sluice 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -13
- data/CHANGELOG +5 -0
- data/lib/sluice/storage/s3/s3.rb +8 -8
- data/lib/sluice/version.rb +1 -1
- metadata +45 -46
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
Y2RlZDExYTlmNTEzODgzOTk0NzM2NTU0YWJhYmRjMzE4ZGQ3NjYwZQ==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c06649f92ec59027bc5d30eb5146ee66b4d2a518
|
4
|
+
data.tar.gz: c28481f3eecd7a60d23e50b5809a7c11d94ae5c0
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
NjU4ZWUzZTY0MDg5ZmM1MjNjODJkNjE2MGIxZWQ1YzkwODliNDRjZmU0Y2Q4
|
11
|
-
ZWUxODc2M2MxNzdiMDEyZGIwOWMxZWQ2YTM3ZjczNzZjOTQyNGU=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
NmYwZjA1MTU5MzQ2NGEwNjI1Y2FiYmUwZmZhNzg4YjAyODJlNTBkMTRhZGQw
|
14
|
-
OTRkZTA1MTZkZDZiMmE2OWQ1OGNhZDUwYTFmYjQyNmQzM2E5YzQ5M2I3MTdi
|
15
|
-
NDJkYzkxOTEwM2Q0Njc4ZDU2MzBmYmYyMWRlYWZiMWZkNWY3NTk=
|
6
|
+
metadata.gz: 99ead2169b18814ab2f993ae559ae03210bff3d899e06a29371d6fdcc2131bc21f682f538e9b68f0de20ee80c4f7133ab7da82c98dc63a70e64839beae11e89b
|
7
|
+
data.tar.gz: 6f640d9e1caff23b2f0a89ecc90453dc2d7bbb5d769af434570e5fd79048c592a853a7adca50bb40ef4fae616fe87a426abf946c490e282e1231f0e78d681c44
|
data/CHANGELOG
CHANGED
data/lib/sluice/storage/s3/s3.rb
CHANGED
@@ -466,7 +466,7 @@ module Sluice
|
|
466
466
|
if files_to_process.size == 0
|
467
467
|
# S3 batches 1000 files per request.
|
468
468
|
# We load up our array with the files to move
|
469
|
-
files_to_process = s3.directories.get(from_loc.bucket, :prefix => from_loc.dir).files.all(marker_opts)
|
469
|
+
files_to_process = s3.directories.get(from_loc.bucket, :prefix => from_loc.dir).files.all(marker_opts).to_a
|
470
470
|
# If we don't have any files after the S3 request, we're complete
|
471
471
|
if files_to_process.size == 0
|
472
472
|
complete = true
|
@@ -593,18 +593,18 @@ module Sluice
|
|
593
593
|
# A helper function to rename a file
|
594
594
|
# TODO: fixup lambda to be Maybe[Proc]
|
595
595
|
Contract String, Maybe[String], Or[Proc, Bool] => Maybe[String]
|
596
|
-
def self.rename_file(filepath, basename,
|
596
|
+
def self.rename_file(filepath, basename, rename_lambda=false)
|
597
597
|
|
598
|
-
if
|
599
|
-
case
|
598
|
+
if rename_lambda.class == Proc
|
599
|
+
case rename_lambda.arity
|
600
600
|
when 2
|
601
|
-
|
601
|
+
rename_lambda.call(basename, filepath)
|
602
602
|
when 1
|
603
|
-
|
603
|
+
rename_lambda.call(basename)
|
604
604
|
when 0
|
605
|
-
|
605
|
+
rename_lambda.call()
|
606
606
|
else
|
607
|
-
raise StorageOperationError "Expect arity of 0, 1 or 2 for
|
607
|
+
raise StorageOperationError "Expect arity of 0, 1 or 2 for rename_lambda, not #{rename_lambda.arity}"
|
608
608
|
end
|
609
609
|
else
|
610
610
|
basename
|
data/lib/sluice/version.rb
CHANGED
metadata
CHANGED
@@ -1,122 +1,121 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sluice
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Dean
|
8
8
|
- Michael Tibben
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-05-
|
12
|
+
date: 2014-05-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: contracts
|
16
|
-
|
16
|
+
version_requirements: !ruby/object:Gem::Requirement
|
17
17
|
requirements:
|
18
18
|
- - ~>
|
19
19
|
- !ruby/object:Gem::Version
|
20
20
|
version: '0.4'
|
21
|
-
|
22
|
-
prerelease: false
|
23
|
-
version_requirements: !ruby/object:Gem::Requirement
|
21
|
+
requirement: !ruby/object:Gem::Requirement
|
24
22
|
requirements:
|
25
23
|
- - ~>
|
26
24
|
- !ruby/object:Gem::Version
|
27
25
|
version: '0.4'
|
26
|
+
prerelease: false
|
27
|
+
type: :runtime
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: fog
|
30
|
-
|
30
|
+
version_requirements: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - ~>
|
33
33
|
- !ruby/object:Gem::Version
|
34
34
|
version: '1.22'
|
35
|
-
|
36
|
-
prerelease: false
|
37
|
-
version_requirements: !ruby/object:Gem::Requirement
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
38
36
|
requirements:
|
39
37
|
- - ~>
|
40
38
|
- !ruby/object:Gem::Version
|
41
39
|
version: '1.22'
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: rspec
|
44
|
-
|
44
|
+
version_requirements: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - ~>
|
47
47
|
- !ruby/object:Gem::Version
|
48
48
|
version: '2.14'
|
49
|
-
- -
|
49
|
+
- - '>='
|
50
50
|
- !ruby/object:Gem::Version
|
51
51
|
version: 2.14.1
|
52
|
-
|
53
|
-
prerelease: false
|
54
|
-
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirement: !ruby/object:Gem::Requirement
|
55
53
|
requirements:
|
56
54
|
- - ~>
|
57
55
|
- !ruby/object:Gem::Version
|
58
56
|
version: '2.14'
|
59
|
-
- -
|
57
|
+
- - '>='
|
60
58
|
- !ruby/object:Gem::Version
|
61
59
|
version: 2.14.1
|
60
|
+
prerelease: false
|
61
|
+
type: :development
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: rspec-nc
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
64
69
|
requirement: !ruby/object:Gem::Requirement
|
65
70
|
requirements:
|
66
|
-
- -
|
71
|
+
- - '>='
|
67
72
|
- !ruby/object:Gem::Version
|
68
73
|
version: '0'
|
69
|
-
type: :development
|
70
74
|
prerelease: false
|
75
|
+
type: :development
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: guard
|
71
78
|
version_requirements: !ruby/object:Gem::Requirement
|
72
79
|
requirements:
|
73
|
-
- -
|
80
|
+
- - '>='
|
74
81
|
- !ruby/object:Gem::Version
|
75
82
|
version: '0'
|
76
|
-
- !ruby/object:Gem::Dependency
|
77
|
-
name: guard
|
78
83
|
requirement: !ruby/object:Gem::Requirement
|
79
84
|
requirements:
|
80
|
-
- -
|
85
|
+
- - '>='
|
81
86
|
- !ruby/object:Gem::Version
|
82
87
|
version: '0'
|
83
|
-
type: :development
|
84
88
|
prerelease: false
|
89
|
+
type: :development
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: guard-rspec
|
85
92
|
version_requirements: !ruby/object:Gem::Requirement
|
86
93
|
requirements:
|
87
|
-
- -
|
94
|
+
- - '>='
|
88
95
|
- !ruby/object:Gem::Version
|
89
96
|
version: '0'
|
90
|
-
- !ruby/object:Gem::Dependency
|
91
|
-
name: guard-rspec
|
92
97
|
requirement: !ruby/object:Gem::Requirement
|
93
98
|
requirements:
|
94
|
-
- -
|
99
|
+
- - '>='
|
95
100
|
- !ruby/object:Gem::Version
|
96
101
|
version: '0'
|
97
|
-
type: :development
|
98
102
|
prerelease: false
|
103
|
+
type: :development
|
104
|
+
- !ruby/object:Gem::Dependency
|
105
|
+
name: coveralls
|
99
106
|
version_requirements: !ruby/object:Gem::Requirement
|
100
107
|
requirements:
|
101
|
-
- -
|
108
|
+
- - '>='
|
102
109
|
- !ruby/object:Gem::Version
|
103
110
|
version: '0'
|
104
|
-
- !ruby/object:Gem::Dependency
|
105
|
-
name: coveralls
|
106
111
|
requirement: !ruby/object:Gem::Requirement
|
107
112
|
requirements:
|
108
|
-
- -
|
113
|
+
- - '>='
|
109
114
|
- !ruby/object:Gem::Version
|
110
115
|
version: '0'
|
111
|
-
type: :development
|
112
116
|
prerelease: false
|
113
|
-
|
114
|
-
|
115
|
-
- - ! '>='
|
116
|
-
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
118
|
-
description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses
|
119
|
-
Fog
|
117
|
+
type: :development
|
118
|
+
description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses Fog
|
120
119
|
email:
|
121
120
|
- support@snowplowanalytics.com
|
122
121
|
executables: []
|
@@ -147,24 +146,24 @@ homepage: http://snowplowanalytics.com
|
|
147
146
|
licenses:
|
148
147
|
- Apache-2.0
|
149
148
|
metadata: {}
|
150
|
-
post_install_message:
|
149
|
+
post_install_message:
|
151
150
|
rdoc_options: []
|
152
151
|
require_paths:
|
153
152
|
- lib
|
154
153
|
required_ruby_version: !ruby/object:Gem::Requirement
|
155
154
|
requirements:
|
156
|
-
- -
|
155
|
+
- - '>='
|
157
156
|
- !ruby/object:Gem::Version
|
158
157
|
version: '0'
|
159
158
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
159
|
requirements:
|
161
|
-
- -
|
160
|
+
- - '>='
|
162
161
|
- !ruby/object:Gem::Version
|
163
162
|
version: '0'
|
164
163
|
requirements: []
|
165
|
-
rubyforge_project:
|
164
|
+
rubyforge_project:
|
166
165
|
rubygems_version: 2.2.2
|
167
|
-
signing_key:
|
166
|
+
signing_key:
|
168
167
|
specification_version: 4
|
169
168
|
summary: Ruby toolkit for cloud-friendly ETL
|
170
169
|
test_files:
|