sluice 0.0.7 → 0.0.8

data/CHANGELOG

@@ -1,3 +1,11 @@
+Version 0.0.8 (2013-08-14)
+--------------------------
+Added in ability to process a list of files rather than an S3 location
+
+Version 0.0.7 (2013-XX-XX)
+--------------------------
+Added in upload capability
+
 Version 0.0.6 (2012-12-31)
 --------------------------
 Fixed is_empty? (was actually is_not_empty?)

data/lib/sluice/storage/s3.rb

@@ -9,7 +9,7 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
-# Authors::    Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
 # Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
 # License::   Apache License Version 2.0
 
@@ -67,22 +67,88 @@ module Sluice
       # +access_key_id+:: AWS access key ID
       # +secret_access_key+:: AWS secret access key
       def new_fog_s3_from(region, access_key_id, secret_access_key)
-        Fog::Storage.new({
+        fog = Fog::Storage.new({
           :provider => 'AWS',
           :region => region,
           :aws_access_key_id => access_key_id,
           :aws_secret_access_key => secret_access_key
         })
+        fog.sync_clock
+        fog
       end
       module_function :new_fog_s3_from
 
+      # Return an array of all Fog::Storage::AWS::File's
+      #
+      # Parameters:
+      # +s3+:: A Fog::Storage s3 connection
+      # +location+:: The location to return files from
+      #
+      # Returns array of Fog::Storage::AWS::File's
+      def list_files(s3, location)
+        files_and_dirs = s3.directories.get(location.bucket, prefix: location.dir).files
+
+        files = [] # Can't use a .select because of Ruby deep copy issues (array of non-POROs)
+        files_and_dirs.each { |f|
+          if is_file?(f.key)
+            files << f.dup
+          end
+        }
+        files
+      end
+      module_function :list_files
+
+      # Whether the given path is a directory or not
+      #
+      # Parameters:
+      # +path+:: S3 path in String form
+      #
+      # Returns boolean
+      def is_folder?(path)
+        (path.end_with?('_$folder$') || # EMR-created
+          path.end_with?('/'))
+      end
+      module_function :is_folder?
+
+      # Whether the given path is a file or not
+      #
+      # Parameters:
+      # +path+:: S3 path in String form
+      #
+      # Returns boolean
+      def is_file?(path)
+        !is_folder?(path)
+      end
+      module_function :is_file?
+
+      # Returns the basename for the given path
+      #
+      # Parameters:
+      # +path+:: S3 path in String form
+      #
+      # Returns the basename, or nil if the
+      # path is to a folder
+      def get_basename(path)
+        if is_folder?(path)
+          nil
+        else
+          match = path.match('([^/]+)$')
+          if match
+            match[1]
+          else
+            nil
+          end
+        end
+      end
+      module_function :get_basename
+
       # Determine if a bucket is empty
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
       # +location+:: The location to check
       def is_empty?(s3, location)
-        s3.directories.get(location.bucket, :prefix => location.dir).files().length <= 1
+        list_files(s3, location).length <= 1
       end
       module_function :is_empty?
 
@@ -91,13 +157,13 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from_location+:: S3Location to delete files from      
+      # +from_files_or_loc+:: Array of filepaths or Fog::Storage::AWS::File objects, or S3Location to download files from      
       # +to_directory+:: Local directory to copy files to
       # +match_regex+:: a regex string to match the files to delete
-      def download_files(s3, from_location, to_directory, match_regex='.+')
+      def download_files(s3, from_files_or_loc, to_directory, match_regex='.+')
 
-        puts "  downloading files from #{from_location} to #{to_directory}"
-        process_files(:download, s3, from_location, match_regex, to_directory)
+        puts "  downloading #{describe_from(from_files_or_loc)} to #{to_directory}"
+        process_files(:download, s3, from_files_or_loc, match_regex, to_directory)
       end
       module_function :download_files    
 
@@ -105,12 +171,12 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from_location+:: S3Location to delete files from
+      # +from_files_or_loc+:: Array of filepaths or Fog::Storage::AWS::File objects, or S3Location to delete files from
       # +match_regex+:: a regex string to match the files to delete
-      def delete_files(s3, from_location, match_regex='.+')
+      def delete_files(s3, from_files_or_loc, match_regex='.+')
 
-        puts "  deleting files from #{from_location}"
-        process_files(:delete, s3, from_location, match_regex)
+        puts "  deleting #{describe_from(from_files_or_loc)}"
+        process_files(:delete, s3, from_files_or_loc, match_regex)
       end
       module_function :delete_files
 
@@ -118,15 +184,15 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from_location+:: S3Location to copy files from
+      # +from_files_or_loc+:: Array of filepaths or Fog::Storage::AWS::File objects, or S3Location to copy files from
       # +to_location+:: S3Location to copy files to
       # +match_regex+:: a regex string to match the files to copy
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def copy_files(s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def copy_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
 
-        puts "  copying files from #{from_location} to #{to_location}"
-        process_files(:copy, s3, from_location, match_regex, to_location, alter_filename_lambda, flatten)
+        puts "  copying #{describe_from(from_files_or_loc)} to #{to_location}"
+        process_files(:copy, s3, from_files_or_loc, match_regex, to_location, alter_filename_lambda, flatten)
       end
       module_function :copy_files
 
@@ -134,15 +200,15 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from_location+:: S3Location to move files from
+      # +from_files_or_loc+:: Array of filepaths or Fog::Storage::AWS::File objects, or S3Location to move files from
       # +to_location+:: S3Location to move files to
       # +match_regex+:: a regex string to match the files to move
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_location
-      def move_files(s3, from_location, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
+      def move_files(s3, from_files_or_loc, to_location, match_regex='.+', alter_filename_lambda=false, flatten=false)
 
-        puts "  moving files from #{from_location} to #{to_location}"
-        process_files(:move, s3, from_location, match_regex, to_location, alter_filename_lambda, flatten)
+        puts "  moving #{describe_from(from_files_or_loc)} to #{to_location}"
+        process_files(:move, s3, from_files_or_loc, match_regex, to_location, alter_filename_lambda, flatten)
       end
       module_function :move_files
 
@@ -150,13 +216,13 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from_directory+:: Local directory to upload files from
+      # +from_files_or_dir+:: Local array of files or local directory to upload files from
       # +to_location+:: S3Location to upload files to
       # +match_glob+:: a filesystem glob to match the files to upload
-      def upload_files(s3, from_directory, to_location, match_glob='*')
+      def upload_files(s3, from_files_or_dir, to_location, match_glob='*')
 
-        puts "  uploading files from #{from_directory} to #{to_location}"
-        process_files(:upload, s3, from_directory, match_glob, to_location)
+        puts "  uploading #{describe_from(from_files_or_dir)} to #{to_location}"
+        process_files(:upload, s3, from_files_or_dir, match_glob, to_location)
       end
       module_function :upload_files
 
@@ -188,7 +254,7 @@ module Sluice
       #
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
-      # +from_file:: A Fog::File to download
+      # +from_file:: A Fog::Storage::AWS::File to download
       # +to_file:: A local file path
       def download_file(s3, from_file, to_file)
 
@@ -204,6 +270,22 @@ module Sluice
 
       private
 
+      # Provides string describing from_files_or_dir_or_loc
+      # for logging purposes.
+      #
+      # Parameters:
+      # +from_files_or_dir_or_loc+:: Array of filepaths or Fog::Storage::AWS::File objects, local directory or S3Location to process files from
+      #
+      # Returns a log-friendly string
+      def describe_from(from_files_or_dir_or_loc)
+        if from_files_or_dir_or_loc.is_a?(Array)
+          "#{from_files_or_dir_or_loc.length} file(s)"
+        else
+          "files from #{from_files_or_dir_or_loc}"
+        end
+      end
+      module_function :describe_from
+
       # Concurrent file operations between S3 locations. Supports:
       # - Download
       # - Upload
@@ -214,12 +296,12 @@ module Sluice
       # Parameters:
       # +operation+:: Operation to perform. :download, :upload, :copy, :delete, :move supported
       # +s3+:: A Fog::Storage s3 connection
-      # +from_loc_or_dir+:: S3Location to process files from
+      # +from_files_or_dir_or_loc+:: Array of filepaths or Fog::Storage::AWS::File objects, local directory or S3Location to process files from
       # +match_regex_or_glob+:: a regex or glob string to match the files to process
       # +to_loc_or_dir+:: S3Location or local directory to process files to
       # +alter_filename_lambda+:: lambda to alter the written filename
       # +flatten+:: strips off any sub-folders below the from_loc_or_dir
-      def process_files(operation, s3, from_loc_or_dir, match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=false, flatten=false)
+      def process_files(operation, s3, from_files_or_dir_or_loc, match_regex_or_glob='.+', to_loc_or_dir=nil, alter_filename_lambda=false, flatten=false)
 
         # Validate that the file operation makes sense
         case operation
@@ -238,12 +320,21 @@ module Sluice
           raise StorageOperationError "File operation %s is unsupported. Try :download, :upload, :copy, :delete or :move" % operation
         end
 
-        # If we are uploading, then we can glob the files before we thread
-        if operation == :upload
-          files_to_process = Dir.glob(File.join(from_loc_or_dir, match_regex_or_glob))
+        # If we have an array of files, no additional globbing required
+        if from_files_or_dir_or_loc.is_a?(Array)
+          files_to_process = from_files_or_dir_or_loc # Could be filepaths or Fog::Storage::AWS::File's
+          globbed = true
+        # Otherwise if it's an upload, we can glob now
+        elsif operation == :upload
+          files_to_process = Dir.glob(File.join(from_files_or_dir_or_loc, match_regex_or_glob))
+          globbed = true
+        # Otherwise we'll do threaded globbing later...
         else
           files_to_process = []
+          from_loc = from_files_or_dir_or_loc # Alias
+          globbed = false
         end
+
         threads = []
         mutex = Mutex.new
         complete = false
@@ -261,20 +352,37 @@ module Sluice
             loop do
               file = false
               filepath = false
+              from_bucket = false
+              from_path = false
               match = false
 
               # Critical section:
               # only allow one thread to modify the array at any time
               mutex.synchronize do
 
-                if operation == :upload
-
+                # No need to do further globbing 
+                if globbed
                   if files_to_process.size == 0
                     complete = true
                     next
                   end
 
-                  filepath = files_to_process.pop
+                  file = files_to_process.pop
+                  # Support raw filenames and also Fog::Storage::AWS::File's
+                  if (file.is_a?(Fog::Storage::AWS::File))
+                    from_bucket = file.directory.key # Bucket
+                    from_path = Sluice::Storage.trail_slash(File.dirname(file.key))
+                    filepath = file.key
+                  else
+                    from_bucket = nil # Not used
+                    if from_files_or_dir_or_loc.is_a?(Array)
+                      from_path = Sluice::Storage.trail_slash(File.dirname(file))
+                    else
+                      from_path = from_files_or_dir_or_loc # The root dir
+                    end
+                    filepath = file
+                  end
+
                   match = true # Match is implicit in the glob
                 else
 
@@ -282,8 +390,8 @@ module Sluice
                     if files_to_process.size == 0
                       # S3 batches 1000 files per request.
                       # We load up our array with the files to move
-                      files_to_process = s3.directories.get(from_loc_or_dir.bucket, :prefix => from_loc_or_dir.dir).files.all(marker_opts)
-                      # If we don't have any files after the s3 request, we're complete
+                      files_to_process = s3.directories.get(from_loc.bucket, :prefix => from_loc.dir).files.all(marker_opts)
+                      # If we don't have any files after the S3 request, we're complete
                       if files_to_process.size == 0
                         complete = true
                         next
@@ -297,6 +405,8 @@ module Sluice
                     end
 
                     file = files_to_process.pop
+                    from_bucket = from_loc.bucket
+                    from_path = from_loc.dir_as_path
                     filepath = file.key
 
                     match = if match_regex_or_glob.is_a? NegativeRegex
@@ -308,21 +418,15 @@ module Sluice
                 end
               end
 
-              # If we don't have a match, then we must be complete
               break unless match
+              break if is_folder?(filepath)
 
-              # Ignore any EMR-created _$folder$ entries
-              break if filepath.end_with?('_$folder$')
-
-              # Match the filename, ignoring directories
-              file_match = filepath.match('([^/]+)$')
-              break unless file_match
-
-              # Rename
+              # Name file
+              basename = get_basename(filepath)
               if alter_filename_lambda.class == Proc
-                filename = alter_filename_lambda.call(file_match[1])
+                filename = alter_filename_lambda.call(basename)
               else
-                filename = file_match[1]
+                filename = basename
               end
 
               # What are we doing? Let's determine source and target
@@ -330,22 +434,22 @@ module Sluice
               case operation
               when :upload
                 source = "#{filepath}"
-                target = name_file(filepath, filename, from_loc_or_dir, to_loc_or_dir.dir_as_path, flatten)
+                target = name_file(filepath, filename, from_path, to_loc_or_dir.dir_as_path, flatten)
                 puts "    UPLOAD #{source} +-> #{to_loc_or_dir.bucket}/#{target}"                
               when :download
-                source = "#{from_loc_or_dir.bucket}/#{filepath}"
-                target = name_file(filepath, filename, from_loc_or_dir.dir_as_path, to_loc_or_dir, flatten)
+                source = "#{from_bucket}/#{filepath}"
+                target = name_file(filepath, filename, from_path, to_loc_or_dir, flatten)
                 puts "    DOWNLOAD #{source} +-> #{target}"
               when :move
-                source = "#{from_loc_or_dir.bucket}/#{filepath}"
-                target = name_file(filepath, filename, from_loc_or_dir.dir_as_path, to_loc_or_dir.dir_as_path, flatten)
+                source = "#{from_bucket}/#{filepath}"
+                target = name_file(filepath, filename, from_path, to_loc_or_dir.dir_as_path, flatten)
                 puts "    MOVE #{source} -> #{to_loc_or_dir.bucket}/#{target}"
               when :copy
-                source = "#{from_loc_or_dir.bucket}/#{filepath}"
-                target = name_file(filepath, filename, from_loc_or_dir.dir_as_path, to_loc_or_dir.dir_as_path, flatten)
+                source = "#{from_bucket}/#{filepath}"
+                target = name_file(filepath, filename, from_path, to_loc_or_dir.dir_as_path, flatten)
                 puts "    COPY #{source} +-> #{to_loc_or_dir.bucket}/#{target}"
               when :delete
-                source = "#{from_loc_or_dir.bucket}/#{filepath}"
+                source = "#{from_bucket}/#{filepath}"
                 # No target
                 puts "    DELETE x #{source}" 
               end

data/lib/sluice.rb

@@ -19,5 +19,5 @@ require 'sluice/storage/s3'
 
 module Sluice
   NAME    = "sluice"
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/sluice.gemspec

@@ -34,5 +34,5 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   # Dependencies
-  gem.add_dependency 'fog', '~> 1.6.0'
+  gem.add_dependency 'fog', '~> 1.14.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sluice
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-11 00:00:00.000000000 Z
+date: 2013-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -19,7 +19,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,7 +27,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.14.0
 description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses
   Fog
 email:
@@ -67,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Ruby toolkit for cloud-friendly ETL