s3twin 0.0.2 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 639bb3f87f1ec9c0452738d5e82f97a07bac3362
-  data.tar.gz: c3d79d29654cea18f8e2783b1878160b0f296ca4
+  metadata.gz: e0d4ae42cce139cc7fd2957f965f68c10e386661
+  data.tar.gz: c17b586aeef609c1e88458f20c5bdf2855210140
 SHA512:
-  metadata.gz: 301fb46672e726bd6fed665ebdddb4767ab2aba0c1ffcd24347a10564a8c245d0715c31517520389e0cf9cb30753ebfe999185a1281736477fbb017eec17c3d0
-  data.tar.gz: aebfec4092ac05515a00102237cf981dab883d6778d5e5c7ec9d61f035ebb5b7613830edab50d99726dee41838743d8de3853573776c3e4eededd04a5eb1189b
+  metadata.gz: eab5b8a228593e2c15a27ae7ad334903132c613f3cbd2c01526d2ca227859355425f33fcfbdf124fc989f1e3d6091bdadffb085d73951ed17632a4ec5cda865b
+  data.tar.gz: a863873bdd7499318b688d79dcc490535b2b5beb2fdf0b748f3483d1f967ffebd461944a4cad50746bc07b2e69c60233496bcb375d0cd1ca382c56caf03f581c

data/.gitignore

@@ -16,4 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 
-.env
+.env

data/README.md

@@ -1,43 +1,56 @@
 # S3 Twin
 
-Take a mirror of a S3 bucket. It leverages [aws-sdk's](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3/S3Object.html#copy_to-instance_method) `copy_to` function to do the heavy lifting at Amazon, similar to [s3cmd](http://s3tools.org/s3cmd-sync).
-Supports local usage or remote workers, some of which may support scheduling - to keep a bucket's twin up to date.
+Take a mirror of a S3 bucket. It leverages [aws-sdk's](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3/S3Object.html#copy_to-instance_method) `copy_to` function to do the heavy lifting at Amazon, similar to [s3cmd](http://s3tools.org/s3cmd-sync).  
+Supports local usage or remote workers some of which may support scheduling - to keep a bucket's twin up to date.
 
-One way copy only i.e. will not keep two buckets in sync.
-Carries across Public ACLs only, no support for S3 features like server side encryption.
+One way copy only i.e. will not keep two buckets in sync.  
+Carries across Public ACLs only; there is no support for S3 features like server side encryption.
 
-If an object exists in the source bucket and does not exist in the destiantion bucket it's copied.
-If an object exists in both bucket and the [etags](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html) are identical the object is not copied, if the etags do not match the object is copied.
-Does not remove object that have been deleted from the source bucket.
+If an object exists in the source bucket and does not exist in the destiantion bucket then the object is copied.  
+If an object exists in both buckets and the [etags](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html) are identical the object is not copied, if the etags do not match the object is copied.  
+Does not remove objects from the destination bucket that have been deleted from the source bucket.  
 
 Please note; while this works it's the first release so the API may change drastically.  
 
+
 ## Installation
 
 `$ gem install s3twin`
 
+
 ## Usage
 
 `$ s3twin help`
 
-`$ s3twin go`
+`$ s3twin go # run twin now locally`
+
+#### Configuration
+
+`$ s3twin init # initialise a .env file`
 
-Example with payload:
-`$ s3twin go --payload=source_bucket:bar source_access_key:foo source_secret_key:world destination_bucket:bar destination_access_key:foo destination_secret_key:world`
+The configuration payload can be set via an argument `--payload=key:value key2:value2` or `.env` file.  
+If neither of those are present the user will be prompted for input.
+
+_Example with payload:_
+```bash
+$ s3twin go --payload=source_s3_bucket:bar source_aws_access_key_id:foo source_aws_secret_access_key:world destination_s3_bucket:bar destination_aws_access_key_id:foo destination_aws_secret_access_key:world
+```
 
-The payload can be set via an argument `--payload=key:value key2:value2` or `.env` file, if neither of those are present the user will be prompted for input.
 
 ### Remote Workers
 
-Currently only [iron.io] IronWorker tasks (including scheduled) are supported.
+Currently only [iron.io](https://iron.io/) IronWorker tasks (including scheduled) are supported.
 
 #### IronWorker Usage
 
-Create a new project at [https://hud.iron.io](https://hud.iron.io), collect your Iron Token and Project ID.
+Create a new project at [https://hud.iron.io](https://hud.iron.io/), collect your Iron Token and Project ID.
+
+`$ s3twin ironworker help`  
+`$ s3twin ironworker upload # upload worker code`  
+`$ s3twin ironworker go # run worker once now`  
+`$ s3twin ironworker schedule --time # schedule worker for the future`  
 
-`$ s3twin ironworker upload`
-`$ s3twin ironworker go` 
-`$ s3twin ironworker schedule --time `
+Configure `--payload=iron_token:foo iron_project_id:bar`
 
 Optionally create, or append, your `.env` file:
 

data/bin/s3twin

@@ -9,20 +9,34 @@ require 's3twin'
 module S3twin
   class S3twinCLI < Thor
     include Thor::Actions
-    class_option :payload, :type => :hash, :default => { 
-      'source_bucket' => ENV['SOURCE_S3_BUCKET'],
-      'source_access_key' => ENV['SOURCE_AWS_ACCESS_KEY_ID'],
-      'source_secret_key' => ENV['SOURCE_AWS_SECRET_ACCESS_KEY'],
-      'destination_bucket' => ENV['DESTINATION_S3_BUCKET'],
-      'destination_access_key' => ENV['DESTINATION_AWS_ACCESS_KEY_ID'],
-      'destination_secret_key' => ENV['DESTINATION_AWS_SECRET_ACCESS_KEY'] 
+    class_option :payload, :type => :hash, :default => {
+      'source_s3_bucket' => ENV['SOURCE_S3_BUCKET'],
+      'source_aws_access_key_id' => ENV['SOURCE_AWS_ACCESS_KEY_ID'],
+      'source_aws_secret_access_key' => ENV['SOURCE_AWS_SECRET_ACCESS_KEY'],
+      'destination_s3_bucket' => ENV['DESTINATION_S3_BUCKET'],
+      'destination_aws_access_key_id' => ENV['DESTINATION_AWS_ACCESS_KEY_ID'],
+      'destination_aws_secret_access_key' => ENV['DESTINATION_AWS_SECRET_ACCESS_KEY']
     }
 
     desc 'go', 'Make a twin of a S3 Bucket'
     def go
       Twin.go(S3twin::Helpers.prompt_payload(options['payload']))
     end
+
+    desc 'init', 'Initalise a .env file'
+    def init
+      filename = '.env'
+      env = S3twin::Helpers.generate_env(options['payload'])
+      if File.exist?(filename)
+        puts "#{filename} already exists; manually append the below:\n"
+        puts env
+      else
+        File.open(filename, 'w') {|f| f.write(env)}
+        puts "#{filename} created."
+      end
+    end
+
     # worker sub-commands live in ../lib/s3twin/workers    
   end
   S3twinCLI.start
-end
+end

data/lib/s3twin.rb

@@ -1,5 +1,5 @@
 module S3twin
-  VERSION = '0.0.2'
+  VERSION = '0.0.5'
 end
 
 require 's3twin/twin'

data/lib/s3twin/helpers.rb

@@ -1,16 +1,27 @@
 module S3twin::Helpers
   class << self
-
     def prompt_payload(payload)
-      payload['source_bucket'] ||= ask('Source Bucket:')
-      payload['source_access_key'] ||= ask('Source Access Key:')
-      payload['source_secret_key'] ||= ask('Source Secret Key:')
-      payload['destination_bucket'] ||= ask('Destination Bucket:')
-      payload['destination_access_key'] ||= ask('Destination Access Key:')
-      payload['destination_secret_key'] ||= ask('Destination Secret Key:')
+      payload['source_s3_bucket'] ||= ask('Source Bucket:')
+      payload['source_aws_access_key_id'] ||= ask('Source Access Key:')
+      payload['source_aws_secret_access_key'] ||= ask('Source Secret Key:')
+      payload['destination_s3_bucket'] ||= ask('Destination Bucket:')
+      payload['destination_aws_access_key_id'] ||= ask('Destination Access Key:')
+      payload['destination_aws_secret_access_key'] ||= ask('Destination Secret Key:')
       return payload
     end
 
+    def generate_env(payload)
+      env = "IRON_TOKEN='#{payload['iron_token']}'
+IRON_PROJECT_ID='#{payload['iron_project_id']}'
+
+SOURCE_S3_BUCKET='#{payload['source_s3_bucket']}'
+SOURCE_AWS_ACCESS_KEY_ID='#{payload['source_aws_access_key_id']}'
+SOURCE_AWS_SECRET_ACCESS_KEY='#{payload['source_aws_secret_access_key']}'
+
+DESTINATION_S3_BUCKET='#{payload['destination_s3_bucket']}'
+DESTINATION_AWS_ACCESS_KEY_ID='#{payload['destination_aws_access_key_id']}'
+DESTINATION_AWS_SECRET_ACCESS_KEY='#{payload['destination_aws_secret_access_key']}'"
+    end
+
   end
 end
-

data/lib/s3twin/twin.rb

@@ -2,15 +2,15 @@ require 'aws-sdk'
 
 class Twin
   # these exist for ironworker compatibility. It should probaby be seperated out to a wrapper.
-  attr_accessor :source_bucket, :source_access_key, :source_secret_key, :destination_bucket, :destination_access_key, :destination_secret_key
+  attr_accessor :source_s3_bucket, :source_aws_access_key_id, :source_aws_secret_access_key, :destination_s3_bucket, :destination_aws_access_key_id, :destination_aws_secret_access_key
   def run
     params = {
-      'source_bucket' => source_bucket,
-      'source_access_key' => source_access_key,
-      'source_secret_key' => source_secret_key,
-      'destination_bucket' => destination_bucket,
-      'destination_access_key' => destination_access_key,
-      'destination_secret_key' => destination_secret_key
+      'source_s3_bucket' => source_s3_bucket,
+      'source_aws_access_key_id' => source_aws_access_key_id,
+      'source_aws_secret_access_key' => source_aws_secret_access_key,
+      'destination_s3_bucket' => destination_s3_bucket,
+      'destination_aws_access_key_id' => destination_aws_access_key_id,
+      'destination_aws_secret_access_key' => destination_aws_secret_access_key
     }
     Twin.go(params)
   end
@@ -18,36 +18,33 @@ class Twin
   # Real stuff starts here
   class << self
     def go(params)
-      source_bucket = params['source_bucket']
-      source_access_key = params['source_access_key']
-      source_secret_key = params['source_secret_key']
-      destination_bucket = params['destination_bucket']
-      destination_access_key = params['destination_access_key']
-      destination_secret_key = params['destination_secret_key']
-
       puts 'Starting S3Twin run'
       @source = AWS::S3.new(
-        :access_key_id => source_access_key,
-        :secret_access_key => source_secret_key)
-      @source_bucket = @source.buckets[source_bucket]
+        :access_key_id => params['source_aws_access_key_id'],
+        :secret_access_key => params['source_aws_secret_access_key'])
+      @source_bucket = @source.buckets[params['source_s3_bucket']]
 
       @destination = AWS::S3.new(
-        :access_key_id => destination_access_key,
-        :secret_access_key => destination_secret_key)
-      @destination_bucket = @destination.buckets[destination_bucket]
+        :access_key_id => params['destination_aws_access_key_id'],
+        :secret_access_key => params['destination_aws_secret_access_key'])
+      @destination_bucket = @destination.buckets[params['destination_s3_bucket']]
 
       @source_bucket.objects.each do |obj|
         dest_key = @destination_bucket.objects[obj.key] 
-        unless dest_key.exists?
-          puts "Creating: #{obj.key}"
-          obj.copy_to(dest_key, :acl => public_acl(obj))
-        else
-          unless etag_match(obj.key,obj.etag)
-            puts "Updating: #{obj.key}"
+        begin
+          unless dest_key.exists?
+            puts "Creating: #{obj.key}"
             obj.copy_to(dest_key, :acl => public_acl(obj))
           else
-            puts "Skipping: #{obj.key}"
-          end 
+            unless etag_match(obj.key,obj.etag)
+              puts "Updating: #{obj.key}"
+              obj.copy_to(dest_key, :acl => public_acl(obj))
+            else
+              puts "Skipping: #{obj.key}"
+            end
+          end
+        rescue AWS::S3::Errors::AccessDenied
+          puts "Access Denied: #{obj.key}"
         end
       end
       puts 'Completed S3Twin run'
@@ -65,6 +62,8 @@ class Twin
           else 
             return 'private'
           end
+        else
+          return nil
         end
       end
     end

data/s3twin.gemspec

@@ -18,9 +18,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'thor', '~> 0.18'
-  spec.add_dependency 'dotenv', '~> 0.8'
-  spec.add_dependency 'aws-sdk', '~> 1.11'
-  spec.add_dependency 'iron_worker_ng', '~> 0.16'
+  spec.add_dependency 'dotenv', '~> 0.9'
+  spec.add_dependency 'aws-sdk', '~> 1.19'
+  spec.add_dependency 'iron_worker_ng', '~> 1.0'
   
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3twin
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.5
 platform: ruby
 authors:
 - Robert Coleman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-07 00:00:00.000000000 Z
+date: 2013-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -30,42 +30,42 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: iron_worker_ng
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -131,7 +131,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .env.sample
 - .gitignore
 - Gemfile
 - LICENSE

data/.env.sample

@@ -1,10 +0,0 @@
-IRON_TOKEN=''
-IRON_PROJECT_ID=''
-
-SOURCE_S3_BUCKET=''
-SOURCE_AWS_ACCESS_KEY_ID=''
-SOURCE_AWS_SECRET_ACCESS_KEY=''
-
-DESTINATION_S3_BUCKET=''
-DESTINATION_AWS_ACCESS_KEY_ID=''
-DESTINATION_AWS_SECRET_ACCESS_KEY=''