sluice 0.1.5 → 0.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NjFiZjc1OTY1NjY1ZWE2YWY2NDExZWZjMTJkYTQyM2IzYWNmZGVlOA==
+  data.tar.gz: !binary |-
+    Y2RlZDExYTlmNTEzODgzOTk0NzM2NTU0YWJhYmRjMzE4ZGQ3NjYwZQ==
+SHA512:
+  metadata.gz: !binary |-
+    OWNlY2YzMjBiZjYwNzgwYmUzODNhNGM2Yzc3NDEyNzI3MmRkYWFmMTkyMDIy
+    NjU4ZWUzZTY0MDg5ZmM1MjNjODJkNjE2MGIxZWQ1YzkwODliNDRjZmU0Y2Q4
+    ZWUxODc2M2MxNzdiMDEyZGIwOWMxZWQ2YTM3ZjczNzZjOTQyNGU=
+  data.tar.gz: !binary |-
+    NmYwZjA1MTU5MzQ2NGEwNjI1Y2FiYmUwZmZhNzg4YjAyODJlNTBkMTRhZGQw
+    OTRkZTA1MTZkZDZiMmE2OWQ1OGNhZDUwYTFmYjQyNmQzM2E5YzQ5M2I3MTdi
+    NDJkYzkxOTEwM2Q0Njc4ZDU2MzBmYmYyMWRlYWZiMWZkNWY3NTk=

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+cache: bundler
+
+rvm:
+  - 1.9.3
+  - jruby
+  - 2.0.0
+  - 2.1.0
+
+script: 'bundle exec rspec spec'

data/CHANGELOG

@@ -1,3 +1,20 @@
+Version 0.2.0 (2014-05-16)
+--------------------------
+Bumped Contracts to 0.4 (#22)
+Bumped Fog to 1.22.0 (#24)
+Added gem button to README (#18)
+Added Coveralls code coverage to project (#17)
+Added Code Climate button to README (#23)
+Added Travis support to project (#16)
+Added initial unit tests (#15)
+Added in FogFile = Fog::Storage::AWS::File-based contracts (#12)
+Added additional contracts (#20)
+Broke up s3.rb into separate files (#21)
+Updated alter_filename_lambda to accept original filepath as 2nd arg (#19)
+Overrode equality operator for S3::Location to support tests (#10)
+Fixed break bugs in core Sluice process flow (#6)
+Made is_empty? work if another folder starts with the same name as this one (#5)
+
 Version 0.1.5 (2013-10-13)
 --------------------------
 Fixed is_empty? returns true if folder contains 1 file (#9)

data/Guardfile

@@ -0,0 +1,11 @@
+guard 'rspec' do
+  # watch /lib/ files
+  watch(%r{^lib/(.+).rb$}) do |m|
+    "spec/#{m[1]}_spec.rb"
+  end
+
+  # watch /spec/ files
+  watch(%r{^spec/(.+).rb$}) do |m|
+    "spec/#{m[1]}.rb"
+  end
+end

data/README.md

@@ -1,4 +1,8 @@
 # Sluice
+[![Gem Version](https://badge.fury.io/rb/sluice.svg)](http://badge.fury.io/rb/sluice)
+[![Build Status](https://travis-ci.org/snowplow/sluice.png)](https://travis-ci.org/snowplow/sluice)
+[![Code Climate](https://codeclimate.com/github/snowplow/sluice.png)](https://codeclimate.com/github/snowplow/sluice)
+[![Coverage Status](https://coveralls.io/repos/snowplow/sluice/badge.png?branch=master)](https://coveralls.io/r/snowplow/sluice?branch=master)
 
 Sluice is a Ruby gem (built with [Bundler] [bundler]) to help you build cloud-friendly ETL (extract, transform, load) processes.
 
@@ -21,7 +25,7 @@ Sluice has been extracted from a pair of Ruby ETL applications built by the [Sno
 
 Or in your Gemfile:
 
-    gem 'sluice', '~> 0.1.0'
+    gem 'sluice', '~> 0.2.0'
 
 ## Usage
 
@@ -32,7 +36,7 @@ Rubydoc and usage examples to come.
 To hack on Sluice locally:
 
     $ gem build sluice.gemspec
-    $ sudo gem install sluice-0.1.0.gem
+    $ sudo gem install sluice-0.2.0.gem
 
 To contribute:
 
@@ -48,7 +52,7 @@ Sluice was developed by [Alex Dean] [alexanderdean] ([Snowplow Analytics] [snowp
 
 ## Copyright and license
 
-Sluice is copyright 2012-2013 Snowplow Analytics Ltd.
+Sluice is copyright 2012-2014 Snowplow Analytics Ltd.
 
 Licensed under the [Apache License, Version 2.0] [license] (the "License");
 you may not use this software except in compliance with the License.

data/lib/sluice.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0,
 # and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -10,14 +10,12 @@
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
 # Author::    Alex Dean (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
 # License::   Apache License Version 2.0
 
 require 'sluice/errors'
 require 'sluice/storage/storage'
-require 'sluice/storage/s3'
-
-module Sluice
-  NAME    = "sluice"
-  VERSION = "0.1.5"
-end
+require 'sluice/storage/s3/contracts'
+require 'sluice/storage/s3/location'
+require 'sluice/storage/s3/manifest'
+require 'sluice/storage/s3/s3'

data/lib/sluice/errors.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0,
 # and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -10,7 +10,7 @@
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
 # Author::    Alex Dean (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
 # License::   Apache License Version 2.0
 
 # All errors

data/lib/sluice/storage/s3/contracts.rb

@@ -0,0 +1,32 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'fog'
+require 'fog/aws/models/storage/file'
+
+require 'contracts'
+include Contracts
+
+module Sluice
+  module Storage
+    module S3
+
+      # Aliases for Contracts
+      FogStorage = Fog::Storage::AWS::Real
+      FogFile = Fog::Storage::AWS::File
+
+    end
+  end
+end

data/lib/sluice/storage/s3/location.rb

@@ -0,0 +1,77 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'contracts'
+include Contracts
+
+module Sluice
+  module Storage
+    module S3
+
+      # Class to describe an S3 location
+      # TODO: if we are going to require trailing line-breaks on
+      # buckets, maybe we should make that clearer?
+      class Location
+        
+        attr_reader :bucket, :dir
+
+        # Location constructor
+        #
+        # Parameters:
+        # +s3location+:: the s3 location config string e.g. "bucket/directory"
+        Contract String => Location
+        def initialize(s3_location)
+          @s3_location = s3_location
+
+          s3_location_match = s3_location.match('^s3n?://([^/]+)/?(.*)/$')
+          raise ArgumentError, 'Bad S3 location %s' % s3_location unless s3_location_match
+
+          @bucket = s3_location_match[1]
+          @dir = s3_location_match[2]
+          self
+        end
+
+        Contract nil => String
+        def dir_as_path
+          if @dir.length > 0
+            return @dir+'/'
+          else
+            return ''
+          end
+        end
+
+        Contract nil => String
+        def to_s
+          @s3_location
+        end
+
+        Contract Any => Bool
+        def ==(o)
+          o.class == self.class && o.state == state
+        end
+        alias_method :eql?, :==
+
+      protected
+
+        Contract nil => [String, String, String]
+        def state
+          [@s3_location, @bucket, @dir]
+        end
+
+      end
+
+    end
+  end
+end

data/lib/sluice/storage/s3/manifest.rb

@@ -0,0 +1,129 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'set'
+
+require 'contracts'
+include Contracts
+
+module Sluice
+  module Storage
+    module S3
+
+      # Legitimate manifest scopes:
+      # 1. :filename - store only the filename
+      #                in the manifest
+      # 2. :relpath  - store the relative path
+      #                to the file in the manifest
+      # 3. :abspath  - store the absolute path
+      #                to the file in the manifest
+      # 4. :bucket   - store bucket PLUS absolute
+      #                path to the file in the manifest
+      #
+      # TODO: add support for 2-4. Currently only 1 supported 
+      class ManifestScope
+
+        @@scopes = Set::[](:filename) # TODO add :relpath, :abspath, :bucket
+
+        def self.valid?(val)
+          val.is_a?(Symbol) &&
+            @@scopes.include?(val)
+        end
+      end
+
+      # Class to read and maintain a manifest.
+      class Manifest
+        attr_reader :s3_location, :scope, :manifest_file
+
+        # Manifest constructor
+        #
+        # Parameters:
+        # +path+:: full path to the manifest file
+        # +scope+:: whether file entries in the
+        #           manifest should be scoped to
+        #           filename, relative path, absolute
+        #           path, or absolute path and bucket
+        Contract Location, ManifestScope => nil
+        def initialize(s3_location, scope)
+          @s3_location = s3_location
+          @scope = scope
+          @manifest_file = "%ssluice-%s-manifest" % [s3_location.dir_as_path, scope.to_s]
+          nil
+        end
+
+        # Get the current file entries in the manifest
+        #
+        # Parameters:
+        # +s3+:: A Fog::Storage s3 connection
+        #
+        # Returns an Array of filenames as Strings
+        Contract FogStorage => ArrayOf[String]
+        def get_entries(s3)
+
+          manifest = self.class.get_manifest(s3, @s3_location, @manifest_file)
+          if manifest.nil?
+            return []
+          end
+
+          manifest.body.split("\n").reject(&:empty?)
+        end
+
+        # Add (i.e. append) the following file entries
+        # to the manifest
+        # Files listed previously in the manifest will
+        # be kept in the new manifest file.
+        #
+        # Parameters:
+        # +s3+:: A Fog::Storage s3 connection
+        # +entries+:: an Array of filenames as Strings
+        #
+        # Returns all entries now in the manifest
+        Contract FogStorage, ArrayOf[String] => ArrayOf[String]
+        def add_entries(s3, entries)
+
+          existing = get_entries(s3)
+          filenames = entries.map { |filepath|
+            File.basename(filepath)
+          } # TODO: update when non-filename-based manifests supported
+          all = (existing + filenames)
+
+          manifest = self.class.get_manifest(s3, @s3_location, @manifest_file)
+          body = all.join("\n")
+          if manifest.nil?
+            bucket = s3.directories.get(s3_location.bucket).files.create(
+              :key  => @manifest_file,
+              :body => body
+            )
+          else
+            manifest.body = body
+            manifest.save
+          end
+
+          all
+        end
+
+        private
+
+        # Helper to get the manifest file
+        Contract FogStorage, Location, String => Maybe[FogFile]
+        def self.get_manifest(s3, s3_location, filename)
+          s3.directories.get(s3_location.bucket, prefix: s3_location.dir).files.get(filename) # TODO: break out into new generic get_file() procedure
+        end
+
+      end
+
+    end
+  end
+end

data/lib/sluice/storage/{s3.rb → s3/s3.rb}

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0,
 # and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -10,10 +10,9 @@
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
 # Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
-# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
 # License::   Apache License Version 2.0
 
-require 'set'
 require 'tmpdir'
 require 'fog'
 require 'thread'
@@ -35,145 +34,6 @@ module Sluice
       RETRY_WAIT = 10  # Seconds
       TIMEOUT_WAIT = 1800 # 30 mins should let even large files upload. +1 https://github.com/snowplow/sluice/issues/7 if this is insufficient or excessive
 
-      # Aliases for Contracts
-      FogStorage = Fog::Storage::AWS::Real
-      # FogFile = Fog::Storage::AWS::File TODO: fix - gives: warning: toplevel constant File referenced by Fog::Storage::AWS::File
-
-      # Class to describe an S3 location
-      # TODO: if we are going to impose trailing line-breaks on
-      # buckets, maybe we should make that clearer?
-      class Location
-        attr_reader :bucket, :dir, :s3location
-
-        # Location constructor
-        #
-        # Parameters:
-        # +s3location+:: the s3 location config string e.g. "bucket/directory"
-        def initialize(s3_location)
-          @s3_location = s3_location
-
-          s3_location_match = s3_location.match('^s3n?://([^/]+)/?(.*)/$')
-          raise ArgumentError, 'Bad S3 location %s' % s3_location unless s3_location_match
-
-          @bucket = s3_location_match[1]
-          @dir = s3_location_match[2]
-        end
-
-        def dir_as_path
-          if @dir.length > 0
-            return @dir+'/'
-          else
-            return ''
-          end
-        end
-
-        def to_s
-          @s3_location
-        end
-      end
-
-      # Legitimate manifest scopes:
-      # 1. :filename - store only the filename
-      #                in the manifest
-      # 2. :relpath  - store the relative path
-      #                to the file in the manifest
-      # 3. :abspath  - store the absolute path
-      #                to the file in the manifest
-      # 4. :bucket   - store bucket PLUS absolute
-      #                path to the file in the manifest
-      #
-      # TODO: add support for 2-4. Currently only 1 supported 
-      class ManifestScope
-
-        @@scopes = Set::[](:filename) # TODO add :relpath, :abspath, :bucket
-
-        def self.valid?(val)
-          val.is_a?(Symbol) &&
-            @@scopes.include?(val)
-        end
-      end
-
-      # Class to read and maintain a manifest.
-      class Manifest
-        attr_reader :s3_location, :scope, :manifest_file
-
-        # Manifest constructor
-        #
-        # Parameters:
-        # +path+:: full path to the manifest file
-        # +scope+:: whether file entries in the
-        #           manifest should be scoped to
-        #           filename, relative path, absolute
-        #           path, or absolute path and bucket
-        Contract Location, ManifestScope => nil
-        def initialize(s3_location, scope)
-          @s3_location = s3_location
-          @scope = scope
-          @manifest_file = "%ssluice-%s-manifest" % [s3_location.dir_as_path, scope.to_s]
-          nil
-        end
-
-        # Get the current file entries in the manifest
-        #
-        # Parameters:
-        # +s3+:: A Fog::Storage s3 connection
-        #
-        # Returns an Array of filenames as Strings
-        Contract FogStorage => ArrayOf[String]
-        def get_entries(s3)
-
-          manifest = self.class.get_manifest(s3, @s3_location, @manifest_file)
-          if manifest.nil?
-            return []
-          end
-
-          manifest.body.split("\n").reject(&:empty?)
-        end
-
-        # Add (i.e. append) the following file entries
-        # to the manifest
-        # Files listed previously in the manifest will
-        # be kept in the new manifest file.
-        #
-        # Parameters:
-        # +s3+:: A Fog::Storage s3 connection
-        # +entries+:: an Array of filenames as Strings
-        #
-        # Returns all entries now in the manifest
-        Contract FogStorage, ArrayOf[String] => ArrayOf[String]
-        def add_entries(s3, entries)
-
-          existing = get_entries(s3)
-          filenames = entries.map { |filepath|
-            File.basename(filepath)
-          } # TODO: update when non-filename-based manifests supported
-          all = (existing + filenames)
-
-          manifest = self.class.get_manifest(s3, @s3_location, @manifest_file)
-          body = all.join("\n")
-          if manifest.nil?
-            bucket = s3.directories.get(s3_location.bucket).files.create(
-              :key  => @manifest_file,
-              :body => body
-            )
-          else
-            manifest.body = body
-            manifest.save
-          end
-
-          all
-        end
-
-        private
-
-        # Helper to get the manifest file
-        # Contract FogStorage, Location, String => Or[FogFile, nil] TODO: fix this. Expected: File, Actual: <Fog::Storage::AWS::File>
-        def self.get_manifest(s3, s3_location, filename)
-          s3.directories.get(s3_location.bucket, prefix: s3_location.dir).files.get(filename) # TODO: break out into new generic get_file() procedure
-        end
-
-      end
-
       # Helper function to instantiate a new Fog::Storage
       # for S3 based on our config options
       #
@@ -201,8 +61,9 @@ module Sluice
       # +location+:: The location to return files from
       #
       # Returns array of Fog::Storage::AWS::File's
+      Contract FogStorage, Location => ArrayOf[FogFile]
       def list_files(s3, location)
-        files_and_dirs = s3.directories.get(location.bucket, prefix: location.dir).files
+        files_and_dirs = s3.directories.get(location.bucket, prefix: location.dir_as_path).files
 
         files = [] # Can't use a .select because of Ruby deep copy issues (array of non-POROs)
         files_and_dirs.each { |f|
@@ -220,6 +81,7 @@ module Sluice
       # +path+:: S3 path in String form
       #
       # Returns boolean
+      Contract String => Bool
       def is_folder?(path)
         (path.end_with?('_$folder$') || # EMR-created
           path.end_with?('/'))
@@ -232,6 +94,7 @@ module Sluice
       # +path+:: S3 path in String form
       #
       # Returns boolean
+      Contract String => Bool
       def is_file?(path)
         !is_folder?(path)
       end
@@ -244,6 +107,7 @@ module Sluice
       #
       # Returns the basename, or nil if the
       # path is to a folder
+      Contract nil => String
       def get_basename(path)
         if is_folder?(path)
           nil
@@ -263,6 +127,7 @@ module Sluice
       # Parameters:
       # +s3+:: A Fog::Storage s3 connection
       # +location+:: The location to check
+      Contract FogStorage, Location => Bool
       def is_empty?(s3, location)
         list_files(s3, location).length == 0
       end
@@ -620,6 +485,7 @@ module Sluice
                     from_path = from_loc.dir_as_path
                     filepath = file.key
 
+                    # TODO: clean up following https://github.com/snowplow/sluice/issues/25
                     match = if match_regex_or_glob.is_a? NegativeRegex
                               !filepath.match(match_regex_or_glob.regex)
                             else
@@ -629,19 +495,19 @@ module Sluice
                   end
                 end
               end
+              # End of mutex.synchronize
 
-              break unless match
-              break if is_folder?(filepath)
+              # Kill this thread's loop (and thus this thread) if we are complete
+              break if complete
+
+              # Skip processing for a folder or file which doesn't match our regexp or glob
+              next if is_folder?(filepath) or not match
 
               # Name file
               basename = get_basename(filepath)
               next if ignore.include?(basename) # Don't process if in our leave list
 
-              if alter_filename_lambda.class == Proc
-                filename = alter_filename_lambda.call(basename)
-              else
-                filename = basename
-              end
+              filename = rename_file(filepath, basename, alter_filename_lambda)
 
               # What are we doing? Let's determine source and target
               # Note that target excludes bucket name where relevant
@@ -724,6 +590,27 @@ module Sluice
       end
       module_function :process_files
 
+      # A helper function to rename a file
+      # TODO: fixup lambda to be Maybe[Proc]
+      Contract String, Maybe[String], Or[Proc, Bool] => Maybe[String]
+      def self.rename_file(filepath, basename, lambda=false)
+
+        if lambda.class == Proc
+          case lambda.arity
+          when 2
+            lambda.call(basename, filepath)
+          when 1
+            lambda.call(basename)
+          when 0
+            lambda.call()
+          else
+            raise StorageOperationError "Expect arity of 0, 1 or 2 for alter_filename_lambda, not #{alter_filename_lambda.arity}"
+          end
+        else
+          basename
+        end
+      end
+
       # A helper function to list all files
       # recursively in a folder
       #

data/lib/sluice/storage/storage.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0,
 # and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -10,7 +10,7 @@
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
 # Author::    Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
-# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
 # License::   Apache License Version 2.0
 
 module Sluice

data/lib/sluice/version.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author::    Alex Dean (mailto:support@snowplowanalytics.com)
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+module Sluice
+  NAME    = "sluice"
+  VERSION = "0.2.0"
+end

data/sluice.gemspec

@@ -1,4 +1,4 @@
-# Copyright (c) 2012 SnowPlow Analytics Ltd. All rights reserved.
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0,
 # and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -10,13 +10,13 @@
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
 # Author::    Alex Dean (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2012 SnowPlow Analytics Ltd
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
 # License::   Apache License Version 2.0
 
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'sluice'
+require 'sluice/version'
 
 Gem::Specification.new do |gem|
   gem.authors       = ["Alex Dean", "Michael Tibben"]
@@ -30,10 +30,17 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = Sluice::NAME
   gem.version       = Sluice::VERSION
+  gem.license       = "Apache-2.0"
   gem.platform      = Gem::Platform::RUBY
   gem.require_paths = ["lib"]
 
   # Dependencies
-  gem.add_dependency 'fog', '~> 1.14.0'
-  gem.add_dependency 'contracts', '~> 0.2.3'
+  gem.add_dependency 'contracts', '~> 0.4'
+  gem.add_dependency 'fog', '~> 1.22'
+
+  gem.add_development_dependency "rspec", "~> 2.14", ">= 2.14.1"
+  gem.add_development_dependency "rspec-nc"
+  gem.add_development_dependency "guard"
+  gem.add_development_dependency "guard-rspec"
+  gem.add_development_dependency "coveralls"
 end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'coveralls'
+Coveralls.wear!
+
+require 'sluice'

data/spec/storage/s3/location_spec.rb

@@ -0,0 +1,47 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'spec_helper'
+
+Location = Sluice::Storage::S3::Location
+
+describe Location do
+
+  it 'should successfully initialize with a valid S3 (s3://) bucket' do
+    loc = Location.new('s3://my-s3-bucket/')
+    loc.bucket.should eql 'my-s3-bucket'
+    loc.dir.should eql ''
+    loc.dir_as_path.should eql ''
+    loc.to_s.should eql 's3://my-s3-bucket/'
+  end
+
+  it 'should successfully initialize with a valid S3 (s3n://) bucket' do
+    loc = Location.new('s3n://my-s3n-bucket/')
+    loc.bucket.should eql 'my-s3n-bucket'
+    loc.dir.should eql ''
+    loc.dir_as_path.should eql ''
+    loc.to_s.should eql 's3n://my-s3n-bucket/'
+  end
+
+  it 'should support object equality tests' do
+    loc1 = Location.new('s3n://my-s3n-bucket/hello/blah/')
+    loc2 = Location.new('s3n://my-s3n-bucket/hello/blah/')
+    loc1.should eql loc2
+    loc2.should eql loc1
+    loc1.should == loc2
+    loc2.should == loc1
+  end
+
+end

data/spec/storage/s3/s3_spec.rb

@@ -0,0 +1,42 @@
+# Copyright (c) 2012-2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Authors::   Alex Dean (mailto:support@snowplowanalytics.com), Michael Tibben
+# Copyright:: Copyright (c) 2012-2014 Snowplow Analytics Ltd
+# License::   Apache License Version 2.0
+
+require 'spec_helper'
+
+S3 = Sluice::Storage::S3
+
+describe S3 do
+
+  it 'should allow filenames to be renamed' do
+
+    concat_subdir = lambda { |basename, filepath|
+      if m = filepath.match('([^/]+)/[^/]+$')
+        return m[1] + '-' + basename
+      else
+        return basename
+      end
+    }
+
+    foobar = lambda {
+      'foobar'
+    }
+
+    S3.rename_file('/dir/subdir/file', 'file', lambda=false).should eql 'file'
+    S3.rename_file('/dir/subdir/file', nil, foobar).should eql 'foobar'
+    S3.rename_file('resources/environments/logs/publish/e-bgp9nsynv7/i-f2b831bd/_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', '_var_log_tomcat7_localhost_access_log.txt-1391958061.gz', concat_subdir).should eql 'i-f2b831bd-_var_log_tomcat7_localhost_access_log.txt-1391958061.gz'
+
+  end
+
+end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sluice
 version: !ruby/object:Gem::Version
-  version: 0.1.5
-  prerelease: 
+  version: 0.2.0
 platform: ruby
 authors:
 - Alex Dean
@@ -10,40 +9,112 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-13 00:00:00.000000000 Z
+date: 2014-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fog
+  name: contracts
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: contracts
+  name: fog
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.3
+        version: '1.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.2.3
+        version: '1.22'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+- !ruby/object:Gem::Dependency
+  name: rspec-nc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A Ruby gem to help you build ETL processes involving Amazon S3. Uses
   Fog
 email:
@@ -52,39 +123,51 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .coveralls.yml
 - .gitignore
+- .travis.yml
 - CHANGELOG
 - Gemfile
+- Guardfile
 - LICENSE-2.0.txt
 - README.md
-- Rakefile
 - lib/sluice.rb
 - lib/sluice/errors.rb
-- lib/sluice/storage/s3.rb
+- lib/sluice/storage/s3/contracts.rb
+- lib/sluice/storage/s3/location.rb
+- lib/sluice/storage/s3/manifest.rb
+- lib/sluice/storage/s3/s3.rb
 - lib/sluice/storage/storage.rb
+- lib/sluice/version.rb
 - sluice.gemspec
+- spec/spec_helper.rb
+- spec/storage/s3/location_spec.rb
+- spec/storage/s3/s3_spec.rb
 homepage: http://snowplowanalytics.com
-licenses: []
+licenses:
+- Apache-2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ruby toolkit for cloud-friendly ETL
-test_files: []
+test_files:
+- spec/spec_helper.rb
+- spec/storage/s3/location_spec.rb
+- spec/storage/s3/s3_spec.rb

data/Rakefile

@@ -1,2 +0,0 @@
-#!/usr/bin/env rake
-require "bundler/gem_tasks"