sluice 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- Zjk2NmI3NTI4Zjc5MjA2YWQ4MzEzMTA5MjQ3MWVhOTExNmU1ZWRhMQ==
4
+ Y2U4Y2QxZjc3YTdiYzdiYzkyZTRjYjk5ZDMzNjUwNWQ4ZmZmYWQ0Ng==
5
5
  data.tar.gz: !binary |-
6
- YzcyY2VlYTNmMzBiMTNhOTUxY2VlMTRhMWU3MTNiZjM2ODMzMDNiNQ==
6
+ YTg2ZGU0M2E2YjdhZGM4YTdiZDU4NzA5Zjc1YWM2MWY5OGY4MWQ2YQ==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- ZGE4ZjU1NWY3YTU5ODE2ODFkNDgxZDg2Mjg1ZTYwYWUwMzk4ZmQ1MGYwMTQz
10
- MjcxZGJkOTgyNmU2ZmZlNzk3Mzc4M2NlNTVmOGVmZTBiNTRiZWMzOWJmY2I1
11
- MmYwZDNiNzI3MGUwN2M3NjA2ODBmY2VjM2EzYzg4YjE5MTI5MWM=
9
+ ZjNhYzNhYWIyNDgxNjhkOGNhNGY5YjRlMmUxNGNmYWY3Nzc0YTE0YzkwNWU1
10
+ NzQ5ZDQwOTI2ZjJkNzUyMjE4ZDA4ODQ5NTE3MWIyMzExZDgyNzBmYTY1YTBh
11
+ NDMzYjhiZjhmNTEzZDg2OTcwNjVkNWJjMmM4MDk1NTA0Njk0MGU=
12
12
  data.tar.gz: !binary |-
13
- YjBhYTJmODNmOTI5MDkwNzg1ZjQwYjcxMjY5M2UyOTU3ZWI0NGU5NzZhYThj
14
- YmRhYTcyNzc3MTFkNGVjMTU0ZGUyYThkNzQxOWE5Y2E2MzM5NzRiNGI5NDZl
15
- NDMwOWY0ZmQ4ZjY3NGI1MTgwNWNkMzM4M2RmYjFlODU4YTJjMzM=
13
+ ZWVkYWYyYWE0NmE5NzE4MGNiMzU5MTc1ODc5ZTE0MjAxYWUyNjYwYTNlYzY4
14
+ MjZmMDcyY2YzNzBiNGIwN2M4NDJhMDVjN2Y2OTJhZjA4M2IzZTRjMzM1ZTBj
15
+ OTBhYjhjMWY1OTk4MzhjODBhYTIyNGU2OWM2YjMwZTE1MmYyNjk=
data/CHANGELOG CHANGED
@@ -1,3 +1,8 @@
1
+ Version 0.4.0 (2016-07-29)
2
+ --------------------------
3
+ Represent absence of rename_files lambda with nil, not false (#51)
4
+ Properly namespace the use of NegativeRegex (#50)
5
+
1
6
  Version 0.3.4 (2016-07-09)
2
7
  --------------------------
3
8
  Fully qualify RETRIES in Sluice::Storage::S3 (#49)
data/README.md CHANGED
@@ -26,7 +26,7 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno
26
26
 
27
27
  Or in your Gemfile:
28
28
 
29
- gem 'sluice', '~> 0.3.4'
29
+ gem 'sluice', '~> 0.4.0'
30
30
 
31
31
  ## Usage
32
32
 
@@ -173,7 +173,7 @@ module Sluice
173
173
  # +match_regex+:: a regex string to match the files to move
174
174
  # +alter_filename_lambda+:: lambda to alter the written filename
175
175
  # +flatten+:: strips off any sub-folders below the from_location
176
- def self.copy_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
176
+ def self.copy_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
177
177
 
178
178
  puts " copying inter-account #{describe_from(from_location)} to #{to_location}"
179
179
  processed = []
@@ -195,7 +195,7 @@ module Sluice
195
195
  # +match_regex+:: a regex string to match the files to copy
196
196
  # +alter_filename_lambda+:: lambda to alter the written filename
197
197
  # +flatten+:: strips off any sub-folders below the from_location
198
- def self.copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
198
+ def self.copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
199
199
 
200
200
  puts " copying #{describe_from(from_files_or_loc)} to #{to_location}"
201
201
  process_files(:copy, s3, from_files_or_loc, [], match_regex, to_location, alter_filename_lambda, flatten)
@@ -217,7 +217,7 @@ module Sluice
217
217
  # +match_regex+:: a regex string to match the files to copy
218
218
  # +alter_filename_lambda+:: lambda to alter the written filename
219
219
  # +flatten+:: strips off any sub-folders below the from_location
220
- def self.copy_files_manifest(s3, manifest, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
220
+ def self.copy_files_manifest(s3, manifest, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
221
221
 
222
222
  puts " copying with manifest #{describe_from(from_files_or_loc)} to #{to_location}"
223
223
  ignore = manifest.get_entries(s3) # Files to leave untouched
@@ -244,7 +244,7 @@ module Sluice
244
244
  # +match_regex+:: a regex string to match the files to move
245
245
  # +alter_filename_lambda+:: lambda to alter the written filename
246
246
  # +flatten+:: strips off any sub-folders below the from_location
247
- def self.move_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
247
+ def self.move_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
248
248
 
249
249
  puts " moving inter-account #{describe_from(from_location)} to #{to_location}"
250
250
  processed = []
@@ -267,7 +267,7 @@ module Sluice
267
267
  # +match_regex+:: a regex string to match the files to move
268
268
  # +alter_filename_lambda+:: lambda to alter the written filename
269
269
  # +flatten+:: strips off any sub-folders below the from_location
270
- def self.move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
270
+ def self.move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
271
271
 
272
272
  puts " moving #{describe_from(from_files_or_loc)} to #{to_location}"
273
273
  process_files(:move, s3, from_files_or_loc, [], match_regex, to_location, alter_filename_lambda, flatten)
@@ -359,7 +359,7 @@ module Sluice
359
359
  # +to_loc_or_dir+:: S3Location or local directory to process files to
360
360
  # +alter_filename_lambda+:: lambda to alter the written filename
361
361
  # +flatten+:: strips off any sub-folders below the from_loc_or_dir
362
- def self.process_files(operation, s3, from_files_or_dir_or_loc, ignore=[], match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=false, flatten=false)
362
+ def self.process_files(operation, s3, from_files_or_dir_or_loc, ignore=[], match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=nil, flatten=false)
363
363
 
364
364
  # Validate that the file operation makes sense
365
365
  case operation
@@ -470,7 +470,7 @@ module Sluice
470
470
  filepath = file.key
471
471
 
472
472
  # TODO: clean up following https://github.com/snowplow/sluice/issues/25
473
- match = if match_regex_or_glob.is_a? NegativeRegex
473
+ match = if match_regex_or_glob.is_a? Sluice::Storage::NegativeRegex
474
474
  !filepath.match(match_regex_or_glob.regex)
475
475
  else
476
476
  filepath.match(match_regex_or_glob)
@@ -574,10 +574,11 @@ module Sluice
574
574
  end
575
575
 
576
576
  # A helper function to rename a file
577
- # TODO: fixup lambda to be Maybe[Proc]
578
- def self.rename_file(filepath, basename, rename_lambda=false)
577
+ def self.rename_file(filepath, basename, rename_lambda=nil)
579
578
 
580
- if rename_lambda.class == Proc
579
+ if rename_lambda.nil?
580
+ basename
581
+ else
581
582
  case rename_lambda.arity
582
583
  when 2
583
584
  rename_lambda.call(basename, filepath)
@@ -588,8 +589,6 @@ module Sluice
588
589
  else
589
590
  raise StorageOperationError "Expect arity of 0, 1 or 2 for rename_lambda, not #{rename_lambda.arity}"
590
591
  end
591
- else
592
- basename
593
592
  end
594
593
  end
595
594
 
@@ -15,5 +15,5 @@
15
15
 
16
16
  module Sluice
17
17
  NAME = "sluice"
18
- VERSION = "0.3.4"
18
+ VERSION = "0.4.0"
19
19
  end
@@ -33,7 +33,7 @@ describe S3 do
33
33
  'foobar'
34
34
  }
35
35
 
36
- S3.rename_file('/dir/subdir/file', 'file', lambda=false).should eql 'file'
36
+ S3.rename_file('/dir/subdir/file', 'file', lambda=nil).should eql 'file'
37
37
  S3.rename_file('/dir/subdir/file', nil, foobar).should eql 'foobar'
38
38
  S3.rename_file('resources/environments/logs/publish/e-bgp9nsynv7/i-f2b831bd/_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', '_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', concat_subdir).should eql 'i-f2b831bd-_var_log_tomcat7_localhost_access_log.txt-1391958061.gz'
39
39
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sluice
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alex Dean
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-07-08 00:00:00.000000000 Z
12
+ date: 2016-07-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: contracts