sluice 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/CHANGELOG +5 -0
- data/README.md +1 -1
- data/lib/sluice/storage/s3/s3.rb +11 -12
- data/lib/sluice/version.rb +1 -1
- data/spec/storage/s3/s3_spec.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
Y2U4Y2QxZjc3YTdiYzdiYzkyZTRjYjk5ZDMzNjUwNWQ4ZmZmYWQ0Ng==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YTg2ZGU0M2E2YjdhZGM4YTdiZDU4NzA5Zjc1YWM2MWY5OGY4MWQ2YQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZjNhYzNhYWIyNDgxNjhkOGNhNGY5YjRlMmUxNGNmYWY3Nzc0YTE0YzkwNWU1
|
10
|
+
NzQ5ZDQwOTI2ZjJkNzUyMjE4ZDA4ODQ5NTE3MWIyMzExZDgyNzBmYTY1YTBh
|
11
|
+
NDMzYjhiZjhmNTEzZDg2OTcwNjVkNWJjMmM4MDk1NTA0Njk0MGU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZWVkYWYyYWE0NmE5NzE4MGNiMzU5MTc1ODc5ZTE0MjAxYWUyNjYwYTNlYzY4
|
14
|
+
MjZmMDcyY2YzNzBiNGIwN2M4NDJhMDVjN2Y2OTJhZjA4M2IzZTRjMzM1ZTBj
|
15
|
+
OTBhYjhjMWY1OTk4MzhjODBhYTIyNGU2OWM2YjMwZTE1MmYyNjk=
|
data/CHANGELOG
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
Version 0.4.0 (2016-07-29)
|
2
|
+
--------------------------
|
3
|
+
Represent absence of rename_files lambda with nil, not false (#51)
|
4
|
+
Properly namespace the use of NegativeRegex (#50)
|
5
|
+
|
1
6
|
Version 0.3.4 (2016-07-09)
|
2
7
|
--------------------------
|
3
8
|
Fully qualify RETRIES in Sluice::Storage::S3 (#49)
|
data/README.md
CHANGED
data/lib/sluice/storage/s3/s3.rb
CHANGED
@@ -173,7 +173,7 @@ module Sluice
|
|
173
173
|
# +match_regex+:: a regex string to match the files to move
|
174
174
|
# +alter_filename_lambda+:: lambda to alter the written filename
|
175
175
|
# +flatten+:: strips off any sub-folders below the from_location
|
176
|
-
def self.copy_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=
|
176
|
+
def self.copy_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
|
177
177
|
|
178
178
|
puts " copying inter-account #{describe_from(from_location)} to #{to_location}"
|
179
179
|
processed = []
|
@@ -195,7 +195,7 @@ module Sluice
|
|
195
195
|
# +match_regex+:: a regex string to match the files to copy
|
196
196
|
# +alter_filename_lambda+:: lambda to alter the written filename
|
197
197
|
# +flatten+:: strips off any sub-folders below the from_location
|
198
|
-
def self.copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=
|
198
|
+
def self.copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
|
199
199
|
|
200
200
|
puts " copying #{describe_from(from_files_or_loc)} to #{to_location}"
|
201
201
|
process_files(:copy, s3, from_files_or_loc, [], match_regex, to_location, alter_filename_lambda, flatten)
|
@@ -217,7 +217,7 @@ module Sluice
|
|
217
217
|
# +match_regex+:: a regex string to match the files to copy
|
218
218
|
# +alter_filename_lambda+:: lambda to alter the written filename
|
219
219
|
# +flatten+:: strips off any sub-folders below the from_location
|
220
|
-
def self.copy_files_manifest(s3, manifest, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=
|
220
|
+
def self.copy_files_manifest(s3, manifest, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
|
221
221
|
|
222
222
|
puts " copying with manifest #{describe_from(from_files_or_loc)} to #{to_location}"
|
223
223
|
ignore = manifest.get_entries(s3) # Files to leave untouched
|
@@ -244,7 +244,7 @@ module Sluice
|
|
244
244
|
# +match_regex+:: a regex string to match the files to move
|
245
245
|
# +alter_filename_lambda+:: lambda to alter the written filename
|
246
246
|
# +flatten+:: strips off any sub-folders below the from_location
|
247
|
-
def self.move_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=
|
247
|
+
def self.move_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
|
248
248
|
|
249
249
|
puts " moving inter-account #{describe_from(from_location)} to #{to_location}"
|
250
250
|
processed = []
|
@@ -267,7 +267,7 @@ module Sluice
|
|
267
267
|
# +match_regex+:: a regex string to match the files to move
|
268
268
|
# +alter_filename_lambda+:: lambda to alter the written filename
|
269
269
|
# +flatten+:: strips off any sub-folders below the from_location
|
270
|
-
def self.move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=
|
270
|
+
def self.move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
|
271
271
|
|
272
272
|
puts " moving #{describe_from(from_files_or_loc)} to #{to_location}"
|
273
273
|
process_files(:move, s3, from_files_or_loc, [], match_regex, to_location, alter_filename_lambda, flatten)
|
@@ -359,7 +359,7 @@ module Sluice
|
|
359
359
|
# +to_loc_or_dir+:: S3Location or local directory to process files to
|
360
360
|
# +alter_filename_lambda+:: lambda to alter the written filename
|
361
361
|
# +flatten+:: strips off any sub-folders below the from_loc_or_dir
|
362
|
-
def self.process_files(operation, s3, from_files_or_dir_or_loc, ignore=[], match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=
|
362
|
+
def self.process_files(operation, s3, from_files_or_dir_or_loc, ignore=[], match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=nil, flatten=false)
|
363
363
|
|
364
364
|
# Validate that the file operation makes sense
|
365
365
|
case operation
|
@@ -470,7 +470,7 @@ module Sluice
|
|
470
470
|
filepath = file.key
|
471
471
|
|
472
472
|
# TODO: clean up following https://github.com/snowplow/sluice/issues/25
|
473
|
-
match = if match_regex_or_glob.is_a? NegativeRegex
|
473
|
+
match = if match_regex_or_glob.is_a? Sluice::Storage::NegativeRegex
|
474
474
|
!filepath.match(match_regex_or_glob.regex)
|
475
475
|
else
|
476
476
|
filepath.match(match_regex_or_glob)
|
@@ -574,10 +574,11 @@ module Sluice
|
|
574
574
|
end
|
575
575
|
|
576
576
|
# A helper function to rename a file
|
577
|
-
|
578
|
-
def self.rename_file(filepath, basename, rename_lambda=false)
|
577
|
+
def self.rename_file(filepath, basename, rename_lambda=nil)
|
579
578
|
|
580
|
-
if rename_lambda.
|
579
|
+
if rename_lambda.nil?
|
580
|
+
basename
|
581
|
+
else
|
581
582
|
case rename_lambda.arity
|
582
583
|
when 2
|
583
584
|
rename_lambda.call(basename, filepath)
|
@@ -588,8 +589,6 @@ module Sluice
|
|
588
589
|
else
|
589
590
|
raise StorageOperationError "Expect arity of 0, 1 or 2 for rename_lambda, not #{rename_lambda.arity}"
|
590
591
|
end
|
591
|
-
else
|
592
|
-
basename
|
593
592
|
end
|
594
593
|
end
|
595
594
|
|
data/lib/sluice/version.rb
CHANGED
data/spec/storage/s3/s3_spec.rb
CHANGED
@@ -33,7 +33,7 @@ describe S3 do
|
|
33
33
|
'foobar'
|
34
34
|
}
|
35
35
|
|
36
|
-
S3.rename_file('/dir/subdir/file', 'file', lambda=
|
36
|
+
S3.rename_file('/dir/subdir/file', 'file', lambda=nil).should eql 'file'
|
37
37
|
S3.rename_file('/dir/subdir/file', nil, foobar).should eql 'foobar'
|
38
38
|
S3.rename_file('resources/environments/logs/publish/e-bgp9nsynv7/i-f2b831bd/_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', '_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', concat_subdir).should eql 'i-f2b831bd-_var_log_tomcat7_localhost_access_log.txt-1391958061.gz'
|
39
39
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sluice
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Dean
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-07-
|
12
|
+
date: 2016-07-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: contracts
|