sluice 0.0.2 → 0.0.3

data/CHANGELOG

@@ -0,0 +1,14 @@
+Version 0.0.3 (2012-11-07)
+--------------------------
+Added parallel file download
+Some simplification of existing code
+Fixed a couple of bugs in the file move code (around S3 file keys versus local filepaths)
+
+Version 0.0.2 (2012-11-06)
+--------------------------
+Added trail_slash
+Bug fixing in regexify and files_up_to
+
+Version 0.0.1
+-------------
+Initial release of Sluice

data/README.md

@@ -13,7 +13,7 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno
 
 Or in your Gemfile:
 
-    gem 'sluice', '~> 0.0.1'
+    gem 'sluice', '~> 0.0.3'
 
 ## Usage
 
@@ -61,4 +61,4 @@ limitations under the License.
 [emr-etl-runner]: https://github.com/snowplow/snowplow/tree/master/3-etl/emr-etl-runner
 [storage-loader]: https://github.com/snowplow/snowplow/tree/master/4-storage/storage-loader
 
-[license]: http://www.apache.org/licenses/LICENSE-2.0
+[license]: http://www.apache.org/licenses/LICENSE-2.0

data/lib/sluice.rb

@@ -19,5 +19,5 @@ require 'sluice/storage/s3'
 
 module Sluice
   NAME    = "sluice"
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/sluice/storage/s3.rb

@@ -86,11 +86,26 @@ module Sluice
       end
       module_function :is_empty?
 
+      # Download files from an S3 location to
+      # local storage, concurrently
+      #
+      # Parameters:
+      # +s3+:: A Fog::Storage s3 connection
+      # +from_location+:: S3Location to delete files from      
+      # +to_directory+:: Local directory to copy files to
+      # +match_regex+:: a regex string to match the files to delete
+      def download_files(s3, from_location, to_directory, match_regex='.+')
+
+        puts "  downloading files from #{from_location} to #{to_directory}"
+        process_files(:download, s3, from_location, match_regex, to_directory)
+      end
+      module_function :download_files    
+
       # Delete files from S3 locations concurrently
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from+:: S3Location to delete files from
+      # +from_location+:: S3Location to delete files from
       # +match_regex+:: a regex string to match the files to delete
       def delete_files(s3, from_location, match_regex='.+')
 
@@ -103,8 +118,8 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from+:: S3Location to copy files from
-      # +to+:: S3Location to copy files to
+      # +from_location+:: S3Location to copy files from
+      # +to_location+:: S3Location to copy files to
       # +match_regex+:: a regex string to match the files to copy
       # +alter_filename_lambda+:: lambda to alter the written filename
       def copy_files(s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false)
@@ -118,7 +133,7 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from+:: S3Location to move files from
+      # +from_location+:: S3Location to move files from
       # +to+:: S3Location to move files to
       # +match_regex+:: a regex string to match the files to move
       # +alter_filename_lambda+:: lambda to alter the written filename
@@ -129,6 +144,26 @@ module Sluice
       end
       module_function :move_files
 
+      # Download a single file to the exact path specified.
+      # Has no intelligence around filenaming.
+      # Makes sure to create the path as needed.
+      #
+      # Parameters:
+      # +s3+:: A Fog::Storage s3 connection
+      # +from_file:: A Fog::File to download
+      # +to_file:: A local file path
+      def download_file(s3, from_file, to_file)
+
+        FileUtils.mkdir_p(File.dirname(to_file))
+
+        # TODO: deal with bug where Fog hangs indefinitely if network connection dies during download
+
+        local_file = File.open(to_file, "w")
+        local_file.write(from_file.body)
+        local_file.close
+      end
+      module_function :download_file
+
       private
 
       # Concurrent file operations between S3 locations. Supports:
@@ -139,27 +174,27 @@ module Sluice
       # Parameters:
       # +operation+:: Operation to perform. :copy, :delete, :move supported
       # +s3+:: A Fog::Storage s3 connection
-      # +from+:: S3Location to process files from
+      # +from_location+:: S3Location to process files from
       # +match_regex+:: a regex string to match the files to process
-      # +to+:: S3Location to process files to
+      # +to_loc_or_dir+:: S3Location or local directory to process files to
       # +alter_filename_lambda+:: lambda to alter the written filename
-      def process_files(operation, s3, from_location, match_regex='.+', to_location=nil, alter_filename_lambda=false)
+      def process_files(operation, s3, from_location, match_regex='.+', to_loc_or_dir=nil, alter_filename_lambda=false)
 
         # Validate that the file operation makes sense
         case operation
-        when :copy, :move
-          if to_location.nil?
-            raise StorageOperationError "File operation %s requires a to_location to be set" % operation
+        when :copy, :move, :download
+          if to_loc_or_dir.nil?
+            raise StorageOperationError "File operation %s requires the to_loc_or_dir to be set" % operation
           end
         when :delete
-          unless to_location.nil?
-            raise StorageOperationError "File operation %s does not support the to_location argument" % operation
+          unless to_loc_or_dir.nil?
+            raise StorageOperationError "File operation %s does not support the to_loc_or_dir argument" % operation
           end
           if alter_filename_lambda.class == Proc
             raise StorageOperationError "File operation %s does not support the alter_filename_lambda argument" % operation
           end
         else
-          raise StorageOperationError "File operation %s is unsupported. Try :copy, :delete or :move" % operation
+          raise StorageOperationError "File operation %s is unsupported. Try :download, :copy, :delete or :move" % operation
         end
 
         files_to_process = []
@@ -171,7 +206,7 @@ module Sluice
         # If an exception is thrown in a thread that isn't handled, die quickly
         Thread.abort_on_exception = true
 
-        # Create ruby threads to concurrently execute s3 operations
+        # Create Ruby threads to concurrently execute s3 operations
         for i in (0...CONCURRENCY)
 
           # Each thread pops a file off the files_to_process array, and moves it.
@@ -187,10 +222,10 @@ module Sluice
 
                 while !complete && !match do
                   if files_to_process.size == 0
-                    # s3 batches 1000 files per request
-                    # we load up our array with the files to move
+                    # S3 batches 1000 files per request.
+                    # We load up our array with the files to move
                     files_to_process = s3.directories.get(from_location.bucket, :prefix => from_location.dir).files.all(marker_opts)
-                    # if we don't have any files after the s3 request, we're complete
+                    # If we don't have any files after the s3 request, we're complete
                     if files_to_process.size == 0
                       complete = true
                       next
@@ -204,6 +239,7 @@ module Sluice
                   end
 
                   file = files_to_process.pop
+
                   match = if match_regex.is_a? NegativeRegex
                             !file.key.match(match_regex.regex)
                           else
@@ -213,12 +249,13 @@ module Sluice
               end
 
               # If we don't have a match, then we must be complete
-              break unless match # exit the thread
+              break unless match
+
+              # Ignore any EMR-created _$folder$ entries
+              break if file.key.end_with?('_$folder$')
 
               # Match the filename, ignoring directory
               file_match = file.key.match('([^/]+)$')
-
-              # Silently skip any sub-directories in the list
               break unless file_match
 
               if alter_filename_lambda.class == Proc
@@ -227,44 +264,46 @@ module Sluice
                 filename = file_match[1]
               end
 
-              # What are we doing?
+              # What are we doing? Let's determine source and target
+              # Note that target excludes bucket name where relevant
+              source = "#{from_location.bucket}/#{file.key}"
               case operation
+              when :download
+                target = name_file(file.key, filename, from_location.dir_as_path, to_loc_or_dir)
+                puts "    DOWNLOAD #{source} +-> #{target}"
               when :move
-                puts "    MOVE #{from_location.bucket}/#{file.key} -> #{to_location.bucket}/#{to_location.dir_as_path}#{filename}"            
+                target = name_file(file.key, filename, from_location.dir_as_path, to_loc_or_dir.dir_as_path)
+                puts "    MOVE #{source} -> #{to_loc_or_dir.bucket}/#{target}"
               when :copy
-                puts "    COPY #{from_location.bucket}/#{file.key} +-> #{to_location.bucket}/#{to_location.dir_as_path}#{filename}"  
+                target = name_file(file.key, filename, from_location.dir_as_path, to_loc_or_dir.dir_as_path)
+                puts "    COPY #{source} +-> #{to_loc_or_dir.bucket}/#{target}"
               when :delete
-                puts "    DELETE x #{from_location.bucket}/#{file.key}" 
+                # No target
+                puts "    DELETE x #{source}" 
+              end
+
+              # Download is a stand-alone operation vs move/copy/delete
+              if operation == :download
+                retry_x(
+                  download_file(s3, file, target), RETRIES,
+                  "      +/> #{target}",
+                  "Problem downloading #{file.key}. Retrying.")
               end
 
               # A move or copy starts with a copy file
               if [:move, :copy].include? operation
-                i = 0
-                begin
-                  file.copy(to_location.bucket, to_location.dir_as_path + filename)
-                  puts "      +-> #{to_location.bucket}/#{to_location.dir_as_path}#{filename}"
-                rescue
-                  raise unless i < RETRIES
-                  puts "Problem copying #{file.key}. Retrying.", $!, $@
-                  sleep(RETRY_WAIT)  # give us a bit of time before retrying
-                  i += 1
-                  retry
-                end
+                retry_x(
+                  file.copy(to_loc_or_dir.bucket, target), RETRIES,
+                  "      +-> #{to_loc_or_dir.bucket}/#{target}",
+                  "Problem copying #{file.key}. Retrying.")
               end
 
               # A move or delete ends with a delete
               if [:move, :delete].include? operation
-                i = 0
-                begin
-                  file.destroy()
-                  puts "      x #{from_location.bucket}/#{file.key}"
-                rescue
-                  raise unless i < RETRIES
-                  puts "Problem destroying #{file.key}. Retrying.", $!, $@
-                  sleep(RETRY_WAIT) # Give us a bit of time before retrying
-                  i += 1
-                  retry
-                end
+                retry_x(
+                  file.destroy(), RETRIES,
+                  "      x #{source}",
+                  "Problem destroying #{file.key}. Retrying.")
               end
             end
           end
@@ -276,6 +315,70 @@ module Sluice
       end
       module_function :process_files
 
+      # A helper function to attempt to run a
+      # function retries times
+      #
+      # Parameters:
+      # +function+:: Function to run
+      # +retries+:: Number of retries to attempt
+      # +attempt_msg+:: Message to puts on each attempt
+      # +failure_msg+:: Message to puts on each failure
+      def retry_x(function, retries, attempt_msg, failure_msg)
+        i = 0
+        begin
+          function
+          puts attempt_msg
+        rescue
+          raise unless i < retries
+          puts failure_msg
+          sleep(RETRY_WAIT)  # Give us a bit of time before retrying
+          i += 1
+          retry
+        end        
+      end
+      module_function :retry_x
+
+      # A helper function to prepare destination
+      # filenames and paths. This is a bit weird
+      # - it needs to exist because of differences
+      # in the way that Amazon S3, Fog and Unix
+      # treat filepaths versus keys.
+      #
+      # Parameters:
+      # +filepath+:: Path to file (including old filename)
+      # +new_filename+:: Replace the filename in the path with this
+      # +remove_path+:: If this is set, strip this from the front of the path
+      # +add_path+:: If this is set, add this to the front of the path
+      #
+      # TODO: this really needs unit tests
+      def name_file(filepath, new_filename, remove_path=nil, add_path=nil)
+
+        # First, replace the filename in filepath with new one
+        dirname = File.dirname(filepath)
+        new_filepath = (dirname == '.') ? new_filename : dirname + '/' + new_filename
+
+        # Nothing more to do
+        return new_filepath if remove_path.nil?
+
+        # If we have a 'remove_path', it must be found at
+        # the start of the path.
+        # If it's not, you're probably using name_file()
+        # wrong.
+        if !filepath.start_with?(remove_path)
+          raise StorageOperationError, "name_file failed. Filepath '#{filepath}' does not start with '#{remove_path}'"
+        end
+
+        # Okay, let's remove the filepath
+        shortened_filepath = new_filepath[remove_path.length()..-1]
+
+        # Nothing more to do
+        return shortened_filepath if add_path.nil?
+
+        # Add the new filepath on to the start and return
+        add_path + shortened_filepath
+      end
+      module_function :name_file
+
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sluice
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: 
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Alex Dean
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-06 00:00:00 Z
+date: 2012-11-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: fog
@@ -45,6 +45,7 @@ extra_rdoc_files: []
 
 files: 
 - .gitignore
+- CHANGELOG
 - Gemfile
 - LICENSE-2.0.txt
 - README.md