sluice 0.3.4 → 0.4.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Zjk2NmI3NTI4Zjc5MjA2YWQ4MzEzMTA5MjQ3MWVhOTExNmU1ZWRhMQ==
+    Y2U4Y2QxZjc3YTdiYzdiYzkyZTRjYjk5ZDMzNjUwNWQ4ZmZmYWQ0Ng==
   data.tar.gz: !binary |-
-    YzcyY2VlYTNmMzBiMTNhOTUxY2VlMTRhMWU3MTNiZjM2ODMzMDNiNQ==
+    YTg2ZGU0M2E2YjdhZGM4YTdiZDU4NzA5Zjc1YWM2MWY5OGY4MWQ2YQ==
 SHA512:
   metadata.gz: !binary |-
-    ZGE4ZjU1NWY3YTU5ODE2ODFkNDgxZDg2Mjg1ZTYwYWUwMzk4ZmQ1MGYwMTQz
-    MjcxZGJkOTgyNmU2ZmZlNzk3Mzc4M2NlNTVmOGVmZTBiNTRiZWMzOWJmY2I1
-    MmYwZDNiNzI3MGUwN2M3NjA2ODBmY2VjM2EzYzg4YjE5MTI5MWM=
+    ZjNhYzNhYWIyNDgxNjhkOGNhNGY5YjRlMmUxNGNmYWY3Nzc0YTE0YzkwNWU1
+    NzQ5ZDQwOTI2ZjJkNzUyMjE4ZDA4ODQ5NTE3MWIyMzExZDgyNzBmYTY1YTBh
+    NDMzYjhiZjhmNTEzZDg2OTcwNjVkNWJjMmM4MDk1NTA0Njk0MGU=
   data.tar.gz: !binary |-
-    YjBhYTJmODNmOTI5MDkwNzg1ZjQwYjcxMjY5M2UyOTU3ZWI0NGU5NzZhYThj
-    YmRhYTcyNzc3MTFkNGVjMTU0ZGUyYThkNzQxOWE5Y2E2MzM5NzRiNGI5NDZl
-    NDMwOWY0ZmQ4ZjY3NGI1MTgwNWNkMzM4M2RmYjFlODU4YTJjMzM=
+    ZWVkYWYyYWE0NmE5NzE4MGNiMzU5MTc1ODc5ZTE0MjAxYWUyNjYwYTNlYzY4
+    MjZmMDcyY2YzNzBiNGIwN2M4NDJhMDVjN2Y2OTJhZjA4M2IzZTRjMzM1ZTBj
+    OTBhYjhjMWY1OTk4MzhjODBhYTIyNGU2OWM2YjMwZTE1MmYyNjk=

data/CHANGELOG

@@ -1,3 +1,8 @@
+Version 0.4.0 (2016-07-29)
+--------------------------
+Represent absence of rename_files lambda with nil, not false (#51)
+Properly namespace the use of NegativeRegex (#50)
+
 Version 0.3.4 (2016-07-09)
 --------------------------
 Fully qualify RETRIES in Sluice::Storage::S3 (#49)

data/README.md

@@ -26,7 +26,7 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno
 
 Or in your Gemfile:
 
-    gem 'sluice', '~> 0.3.4'
+    gem 'sluice', '~> 0.4.0'
 
 ## Usage
 

data/lib/sluice/storage/s3/s3.rb

@@ -173,7 +173,7 @@ module Sluice
       # +match_regex+:: a regex string to match the files to move
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def self.copy_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def self.copy_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
 
         puts "  copying inter-account #{describe_from(from_location)} to #{to_location}"
         processed = []
@@ -195,7 +195,7 @@ module Sluice
       # +match_regex+:: a regex string to match the files to copy
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def self.copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def self.copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
 
         puts "  copying #{describe_from(from_files_or_loc)} to #{to_location}"
         process_files(:copy, s3, from_files_or_loc, [], match_regex, to_location, alter_filename_lambda, flatten)
@@ -217,7 +217,7 @@ module Sluice
       # +match_regex+:: a regex string to match the files to copy
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def self.copy_files_manifest(s3, manifest, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def self.copy_files_manifest(s3, manifest, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
 
         puts "  copying with manifest #{describe_from(from_files_or_loc)} to #{to_location}"
         ignore = manifest.get_entries(s3) # Files to leave untouched
@@ -244,7 +244,7 @@ module Sluice
       # +match_regex+:: a regex string to match the files to move
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def self.move_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def self.move_files_inter(from_s3, to_s3, from_location, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
 
         puts "  moving inter-account #{describe_from(from_location)} to #{to_location}"
         processed = []
@@ -267,7 +267,7 @@ module Sluice
       # +match_regex+:: a regex string to match the files to move
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def self.move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def self.move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=nil, flatten=false)
 
         puts "  moving #{describe_from(from_files_or_loc)} to #{to_location}"
         process_files(:move, s3, from_files_or_loc, [], match_regex, to_location, alter_filename_lambda, flatten)
@@ -359,7 +359,7 @@ module Sluice
       # +to_loc_or_dir+:: S3Location or local directory to process files to
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_loc_or_dir
-      def self.process_files(operation, s3, from_files_or_dir_or_loc, ignore=[], match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=false, flatten=false)
+      def self.process_files(operation, s3, from_files_or_dir_or_loc, ignore=[], match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=nil, flatten=false)
 
         # Validate that the file operation makes sense
         case operation
@@ -470,7 +470,7 @@ module Sluice
                     filepath = file.key
 
                     # TODO: clean up following https://github.com/snowplow/sluice/issues/25
-                    match = if match_regex_or_glob.is_a? NegativeRegex
+                    match = if match_regex_or_glob.is_a? Sluice::Storage::NegativeRegex
                               !filepath.match(match_regex_or_glob.regex)
                             else
                               filepath.match(match_regex_or_glob)
@@ -574,10 +574,11 @@ module Sluice
       end
 
       # A helper function to rename a file
-      # TODO: fixup lambda to be Maybe[Proc]
-      def self.rename_file(filepath, basename, rename_lambda=false)
+      def self.rename_file(filepath, basename, rename_lambda=nil)
 
-        if rename_lambda.class == Proc
+        if rename_lambda.nil?
+          basename
+        else
           case rename_lambda.arity
           when 2
             rename_lambda.call(basename, filepath)
@@ -588,8 +589,6 @@ module Sluice
           else
             raise StorageOperationError "Expect arity of 0, 1 or 2 for rename_lambda, not #{rename_lambda.arity}"
           end
-        else
-          basename
         end
       end
 

data/lib/sluice/version.rb

@@ -15,5 +15,5 @@
 
 module Sluice
   NAME    = "sluice"
-  VERSION = "0.3.4"
+  VERSION = "0.4.0"
 end

data/spec/storage/s3/s3_spec.rb

@@ -33,7 +33,7 @@ describe S3 do
       'foobar'
     }
 
-    S3.rename_file('/dir/subdir/file', 'file', lambda=false).should eql 'file'
+    S3.rename_file('/dir/subdir/file', 'file', lambda=nil).should eql 'file'
     S3.rename_file('/dir/subdir/file', nil, foobar).should eql 'foobar'
     S3.rename_file('resources/environments/logs/publish/e-bgp9nsynv7/i-f2b831bd/_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', '_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', concat_subdir).should eql 'i-f2b831bd-_var_log_tomcat7_localhost_access_log.txt-1391958061.gz'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sluice
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Alex Dean
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-08 00:00:00.000000000 Z
+date: 2016-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: contracts