sluice 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -13
- data/CHANGELOG +5 -0
- data/lib/sluice/storage/s3/s3.rb +8 -8
- data/lib/sluice/version.rb +1 -1
- metadata +45 -46
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
Y2RlZDExYTlmNTEzODgzOTk0NzM2NTU0YWJhYmRjMzE4ZGQ3NjYwZQ==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c06649f92ec59027bc5d30eb5146ee66b4d2a518
|
4
|
+
data.tar.gz: c28481f3eecd7a60d23e50b5809a7c11d94ae5c0
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
NjU4ZWUzZTY0MDg5ZmM1MjNjODJkNjE2MGIxZWQ1YzkwODliNDRjZmU0Y2Q4
|
11
|
-
ZWUxODc2M2MxNzdiMDEyZGIwOWMxZWQ2YTM3ZjczNzZjOTQyNGU=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
NmYwZjA1MTU5MzQ2NGEwNjI1Y2FiYmUwZmZhNzg4YjAyODJlNTBkMTRhZGQw
|
14
|
-
OTRkZTA1MTZkZDZiMmE2OWQ1OGNhZDUwYTFmYjQyNmQzM2E5YzQ5M2I3MTdi
|
15
|
-
NDJkYzkxOTEwM2Q0Njc4ZDU2MzBmYmYyMWRlYWZiMWZkNWY3NTk=
|
6
|
+
metadata.gz: 99ead2169b18814ab2f993ae559ae03210bff3d899e06a29371d6fdcc2131bc21f682f538e9b68f0de20ee80c4f7133ab7da82c98dc63a70e64839beae11e89b
|
7
|
+
data.tar.gz: 6f640d9e1caff23b2f0a89ecc90453dc2d7bbb5d769af434570e5fd79048c592a853a7adca50bb40ef4fae616fe87a426abf946c490e282e1231f0e78d681c44
|
data/CHANGELOG
CHANGED
data/lib/sluice/storage/s3/s3.rb
CHANGED
@@ -466,7 +466,7 @@ module Sluice
|
|
466
466
|
if files_to_process.size == 0
|
467
467
|
# S3 batches 1000 files per request.
|
468
468
|
# We load up our array with the files to move
|
469
|
-
files_to_process = s3.directories.get(from_loc.bucket, :prefix => from_loc.dir).files.all(marker_opts)
|
469
|
+
files_to_process = s3.directories.get(from_loc.bucket, :prefix => from_loc.dir).files.all(marker_opts).to_a
|
470
470
|
# If we don't have any files after the S3 request, we're complete
|
471
471
|
if files_to_process.size == 0
|
472
472
|
complete = true
|
@@ -593,18 +593,18 @@ module Sluice
|
|
593
593
|
# A helper function to rename a file
|
594
594
|
# TODO: fixup lambda to be Maybe[Proc]
|
595
595
|
Contract String, Maybe[String], Or[Proc, Bool] => Maybe[String]
|
596
|
-
def self.rename_file(filepath, basename,
|
596
|
+
def self.rename_file(filepath, basename, rename_lambda=false)
|
597
597
|
|
598
|
-
if
|
599
|
-
case
|
598
|
+
if rename_lambda.class == Proc
|
599
|
+
case rename_lambda.arity
|
600
600
|
when 2
|
601
|
-
|
601
|
+
rename_lambda.call(basename, filepath)
|
602
602
|
when 1
|
603
|
-
|
603
|
+
rename_lambda.call(basename)
|
604
604
|
when 0
|
605
|
-
|
605
|
+
rename_lambda.call()
|
606
606
|
else
|
607
|
-
raise StorageOperationError "Expect arity of 0, 1 or 2 for
|
607
|
+
raise StorageOperationError "Expect arity of 0, 1 or 2 for rename_lambda, not #{rename_lambda.arity}"
|
608
608
|
end
|
609
609
|
else
|
610
610
|
basename
|
data/lib/sluice/version.rb
CHANGED
metadata
CHANGED
@@ -1,122 +1,121 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sluice
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Dean
|
8
8
|
- Michael Tibben
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-05-
|
12
|
+
date: 2014-05-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: contracts
|
16
|
-
|
16
|
+
version_requirements: !ruby/object:Gem::Requirement
|
17
17
|
requirements:
|
18
18
|
- - ~>
|
19
19
|
- !ruby/object:Gem::Version
|
20
20
|
version: '0.4'
|
21
|
-
|
22
|
-
prerelease: false
|
23
|
-
version_requirements: !ruby/object:Gem::Requirement
|
21
|
+
requirement: !ruby/object:Gem::Requirement
|
24
22
|
requirements:
|
25
23
|
- - ~>
|
26
24
|
- !ruby/object:Gem::Version
|
27
25
|
version: '0.4'
|
26
|
+
prerelease: false
|
27
|
+
type: :runtime
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: fog
|
30
|
-
|
30
|
+
version_requirements: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - ~>
|
33
33
|
- !ruby/object:Gem::Version
|
34
34
|
version: '1.22'
|
35
|
-
|
36
|
-
prerelease: false
|
37
|
-
version_requirements: !ruby/object:Gem::Requirement
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
38
36
|
requirements:
|
39
37
|
- - ~>
|
40
38
|
- !ruby/object:Gem::Version
|
41
39
|
version: '1.22'
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: rspec
|
44
|
-
|
44
|
+
version_requirements: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - ~>
|
47
47
|
- !ruby/object:Gem::Version
|
48
48
|
version: '2.14'
|
49
|
-
- -
|
49
|
+
- - '>='
|
50
50
|
- !ruby/object:Gem::Version
|
51
51
|
version: 2.14.1
|
52
|
-
|
53
|
-
prerelease: false
|
54
|
-
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirement: !ruby/object:Gem::Requirement
|
55
53
|
requirements:
|
56
54
|
- - ~>
|
57
55
|
- !ruby/object:Gem::Version
|
58
56
|
version: '2.14'
|
59
|
-
- -
|
57
|
+
- - '>='
|
60
58
|
- !ruby/object:Gem::Version
|
61
59
|
version: 2.14.1
|
60
|
+
prerelease: false
|
61
|
+
type: :development
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: rspec-nc
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
64
69
|
requirement: !ruby/object:Gem::Requirement
|
65
70
|
requirements:
|
66
|
-
- -
|
71
|
+
- - '>='
|
67
72
|
- !ruby/object:Gem::Version
|
68
73
|
version: '0'
|
69
|
-
type: :development
|
70
74
|
prerelease: false
|
75
|
+
type: :development
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: guard
|
71
78
|
version_requirements: !ruby/object:Gem::Requirement
|
72
79
|
requirements:
|
73
|
-
- -
|
80
|
+
- - '>='
|
74
81
|
- !ruby/object:Gem::Version
|
75
82
|
version: '0'
|
76
|
-
- !ruby/object:Gem::Dependency
|
77
|
-
name: guard
|
78
83
|
requirement: !ruby/object:Gem::Requirement
|
79
84
|
requirements:
|
80
|
-
- -
|
85
|
+
- - '>='
|
81
86
|
- !ruby/object:Gem::Version
|
82
87
|
version: '0'
|
83
|
-
type: :development
|
84
88
|
prerelease: false
|
89
|
+
type: :development
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: guard-rspec
|
85
92
|
version_requirements: !ruby/object:Gem::Requirement
|
86
93
|
requirements:
|
87
|
-
- -
|
94
|
+
- - '>='
|
88
95
|
- !ruby/object:Gem::Version
|
89
96
|
version: '0'
|
90
|
-
- !ruby/object:Gem::Dependency
|
91
|
-
name: guard-rspec
|
92
97
|
requirement: !ruby/object:Gem::Requirement
|
93
98
|
requirements:
|
94
|
-
- -
|
99
|
+
- - '>='
|
95
100
|
- !ruby/object:Gem::Version
|
96
101
|
version: '0'
|
97
|
-
type: :development
|
98
102
|
prerelease: false
|
103
|
+
type: :development
|
104
|
+
- !ruby/object:Gem::Dependency
|
105
|
+
name: coveralls
|
99
106
|
version_requirements: !ruby/object:Gem::Requirement
|
100
107
|
requirements:
|
101
|
-
- -
|
108
|
+
- - '>='
|
102
109
|
- !ruby/object:Gem::Version
|
103
110
|
version: '0'
|
104
|
-
- !ruby/object:Gem::Dependency
|
105
|
-
name: coveralls
|
106
111
|
requirement: !ruby/object:Gem::Requirement
|
107
112
|
requirements:
|
108
|
-
- -
|
113
|
+
- - '>='
|
109
114
|
- !ruby/object:Gem::Version
|
110
115
|
version: '0'
|
111
|
-
type: :development
|
112
116
|
prerelease: false
|
113
|
-
|
114
|
-
|
115
|
-
- - ! '>='
|
116
|
-
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
118
|
-
description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses
|
119
|
-
Fog
|
117
|
+
type: :development
|
118
|
+
description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses Fog
|
120
119
|
email:
|
121
120
|
- support@snowplowanalytics.com
|
122
121
|
executables: []
|
@@ -147,24 +146,24 @@ homepage: http://snowplowanalytics.com
|
|
147
146
|
licenses:
|
148
147
|
- Apache-2.0
|
149
148
|
metadata: {}
|
150
|
-
post_install_message:
|
149
|
+
post_install_message:
|
151
150
|
rdoc_options: []
|
152
151
|
require_paths:
|
153
152
|
- lib
|
154
153
|
required_ruby_version: !ruby/object:Gem::Requirement
|
155
154
|
requirements:
|
156
|
-
- -
|
155
|
+
- - '>='
|
157
156
|
- !ruby/object:Gem::Version
|
158
157
|
version: '0'
|
159
158
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
159
|
requirements:
|
161
|
-
- -
|
160
|
+
- - '>='
|
162
161
|
- !ruby/object:Gem::Version
|
163
162
|
version: '0'
|
164
163
|
requirements: []
|
165
|
-
rubyforge_project:
|
164
|
+
rubyforge_project:
|
166
165
|
rubygems_version: 2.2.2
|
167
|
-
signing_key:
|
166
|
+
signing_key:
|
168
167
|
specification_version: 4
|
169
168
|
summary: Ruby toolkit for cloud-friendly ETL
|
170
169
|
test_files:
|