s3twin 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8e79cbdb1aaaedc97087f56a0d1a8c83ea608b33
+  data.tar.gz: 795d2720da9737ca984aaadc034b972560d85d2d
+SHA512:
+  metadata.gz: fdb26f8d21a214c635bed2e2b6de2204e928d58af0df90752a62c33af55b5104f91c0753a0d71d80b9df472b15b5d01eae8a1f545925ab6a5bae84ba871e27eb
+  data.tar.gz: fae2ccd84a0ae1205941550841f4cebf9c63dfb4fec387bf21a39c1facf7ee519cb974ca768a6b2da2790f02f9f2539e7574ee2a69224c57f7f5740ac9a53281

data/.env.sample

@@ -0,0 +1,10 @@
+IRON_TOKEN=''
+IRON_PROJECT_ID=''
+
+SOURCE_S3_BUCKET=''
+SOURCE_AWS_ACCESS_KEY_ID=''
+SOURCE_AWS_SECRET_ACCESS_KEY=''
+
+DESTINATION_S3_BUCKET=''
+DESTINATION_AWS_ACCESS_KEY_ID=''
+DESTINATION_AWS_SECRET_ACCESS_KEY=''

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+.env

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Robert Coleman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# S3 Twin
+
+Take a mirror of a S3 bucket. It leverages [aws-sdk's](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3/S3Object.html#copy_to-instance_method) `copy_to` function to do the heavy lifting at Amazon, similar to [s3cmd](http://s3tools.org/s3cmd-sync).
+Supports local usage or remote workers, some of which may support scheduling - to keep a bucket's twin up to date.
+
+One way copy only i.e. will not keep two buckets in sync.
+Carries across Public ACLs only, no support for S3 features like server side encryption.
+
+If an object exists in the source bucket and does not exist in the destiantion bucket it's copied.
+If an object exists in both bucket and the [etags](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html) are identical the object is not copied, if the etags do not match the object is copied.
+Does not remove object that have been deleted from the source bucket.
+
+Please note; while this works it's the first release so the API may change drastically.  
+
+## Installation
+
+`$ gem install s3twin`
+
+## Usage
+
+`$ s3twin help`
+
+`$ s3twin go`
+
+Example with payload:
+`$ s3twin go --payload=source_bucket:bar source_access_key:foo source_secret_key:world destination_bucket:bar destination_access_key:foo destination_secret_key:world`
+
+The payload can be set via an argument `--payload=key:value key2:value2` or `.env` file, if neither of those are present the user will be prompted for input.
+
+### Remote Workers
+
+Currently only [iron.io] IronWorker tasks (including scheduled) are supported.
+
+#### IronWorker Usage
+
+Create a new project at [https://hud.iron.io](https://hud.iron.io), collect your Iron Token and Project ID.
+
+`$ s3twin ironworker upload`
+`$ s3twin ironworker go` 
+`$ s3twin ironworker schedule --time `
+
+Optionally create, or append, your `.env` file:
+
+```bash
+IRON_TOKEN='foo'
+IRON_PROJECT_ID='bar'
+```
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/s3twin

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require 'thor'
+require 'dotenv'
+Dotenv.load
+
+$:.push File.expand_path('../../lib', __FILE__)
+require 's3twin'
+
+module S3twin
+  class S3twinCLI < Thor
+    include Thor::Actions
+    class_option :payload, :type => :hash, :default => { 
+      'source_bucket' => ENV['SOURCE_S3_BUCKET'],
+      'source_access_key' => ENV['SOURCE_AWS_ACCESS_KEY_ID'],
+      'source_secret_key' => ENV['SOURCE_AWS_SECRET_ACCESS_KEY'],
+      'destination_bucket' => ENV['DESTINATION_S3_BUCKET'],
+      'destination_access_key' => ENV['DESTINATION_AWS_ACCESS_KEY_ID'],
+      'destination_secret_key' => ENV['DESTINATION_AWS_SECRET_ACCESS_KEY'] 
+    }
+
+    desc 'go', 'Make a twin of a S3 Bucket'
+    def go
+      Twin.go(S3twin::Helpers.prompt_payload(options['payload']))
+    end
+    # worker sub-commands live in ../lib/s3twin/workers    
+  end
+  S3twinCLI.start
+end

data/lib/s3twin.rb

@@ -0,0 +1,7 @@
+module S3twin
+  VERSION = '0.0.1'
+end
+
+require 's3twin/twin'
+require 's3twin/helpers'
+require 's3twin/workers'

data/lib/s3twin/helpers.rb

@@ -0,0 +1,16 @@
+module S3twin::Helpers
+  class << self
+
+    def prompt_payload(payload)
+      payload['source_bucket'] ||= ask('Source Bucket:')
+      payload['source_access_key'] ||= ask('Source Access Key:')
+      payload['source_secret_key'] ||= ask('Source Secret Key:')
+      payload['destination_bucket'] ||= ask('Destination Bucket:')
+      payload['destination_access_key'] ||= ask('Destination Access Key:')
+      payload['destination_secret_key'] ||= ask('Destination Secret Key:')
+      return payload
+    end
+
+  end
+end
+

data/lib/s3twin/twin.rb

@@ -0,0 +1,76 @@
+require 'aws-sdk'
+
+class Twin
+  # these exist for ironworker compatibility. It should probaby be seperated out to a wrapper.
+  attr_accessor :source_bucket, :source_access_key, :source_secret_key, :destination_bucket, :destination_access_key, :destination_secret_key
+  def run
+    params = {
+      'source_bucket' => source_bucket,
+      'source_access_key' => source_access_key,
+      'source_secret_key' => source_secret_key,
+      'destination_bucket' => destination_bucket,
+      'destination_access_key' => destination_access_key,
+      'destination_secret_key' => destination_secret_key
+    }
+    Twin.go(params)
+  end
+
+  # Real stuff starts here
+  class << self
+    def go(params)
+      source_bucket = params['source_bucket']
+      source_access_key = params['source_access_key']
+      source_secret_key = params['source_secret_key']
+      destination_bucket = params['destination_bucket']
+      destination_access_key = params['destination_access_key']
+      destination_secret_key = params['destination_secret_key']
+
+      puts 'Starting S3Twin run'
+      @source = AWS::S3.new(
+        :access_key_id => source_access_key,
+        :secret_access_key => source_secret_key)
+      @source_bucket = @source.buckets[source_bucket]
+
+      @destination = AWS::S3.new(
+        :access_key_id => destination_access_key,
+        :secret_access_key => destination_secret_key)
+      @destination_bucket = @destination.buckets[destination_bucket]
+
+      @source_bucket.objects.each do |obj|
+        dest_key = @destination_bucket.objects[obj.key] 
+        unless dest_key.exists?
+          puts "Creating: #{obj.key}"
+          obj.copy_to(dest_key, :acl => public_acl(obj))
+        else
+          unless etag_match(obj.key,obj.etag)
+            puts "Updating: #{obj.key}"
+            obj.copy_to(dest_key, :acl => public_acl(obj))
+          else
+            puts "Skipping: #{obj.key}"
+          end 
+        end
+      end
+      puts 'Completed S3Twin run'
+    end
+
+    private
+    def public_acl(object)
+      object.acl.grants.each do |grant|
+        if grant.grantee.uri == 'http://acs.amazonaws.com/groups/global/AllUsers'
+          case grant.permission.name.to_s
+          when 'read'
+            return 'public_read'
+          when 'read_write'
+            return 'public_read_write'
+          else 
+            return 'private'
+          end
+        end
+      end
+    end
+
+    def etag_match(key,etag)
+     etag == @destination_bucket.objects[key].etag
+    end
+  end
+end

data/lib/s3twin/workers.rb

@@ -0,0 +1,4 @@
+$:.push File.expand_path('../', __FILE__)
+
+require 'thor'
+require 'workers/ironworker'

data/lib/s3twin/workers/ironworker.rb

@@ -0,0 +1,61 @@
+require 'iron_worker_ng'
+
+module S3twin
+  class Ironworker < Thor
+    class_option :name, :type => :string, :default => 'S3Twin'
+    class_option :credentials, :type => :hash, :default => {
+      'token' => ENV['IRON_TOKEN'],
+      'project_id' => ENV['IRON_PROJECT_ID']
+    }
+
+    desc 'upload NAME', 'Upload our worker'
+    def upload
+      client = IronWorkerNG::Client.new(prompt_credentials(options['credentials']))
+      code = IronWorkerNG::Code::Base.new
+      code.runtime = 'ruby'
+      code.name = options['name']
+      code.full_remote_build = true
+      code.merge_exec('lib/s3twin/twin.rb','Twin')
+      code.merge_gem 'aws-sdk'
+      client.codes.create(code)
+    end
+
+    desc 'go WORKER', 'Queue a worker'
+    def go
+      payload = S3twin::Helpers.prompt_payload(options['payload'])
+      client = IronWorkerNG::Client.new(prompt_credentials(options['credentials']))
+      client.tasks.create(options['name'],payload)
+      puts "#{options['name']} queued! Details at https://hud.iron.io/"
+    end
+
+    method_option :time, :type => :hash, :default => {}
+    desc 'schedule WORKER', 'Schedule a worker'
+    def schedule(name='S3Twin')
+      payload = S3twin::Helpers.prompt_payload(options['payload'])
+      time = prompt_time(options['time'])
+      client = IronWorkerNG::Client.new(prompt_credentials(options['credentials']))
+      schedule = client.schedules.create(options['name'],payload,time)
+      puts "#{options['name']} scheduled! (id:#{schedule.id})"
+    end
+
+    private
+    def prompt_time(time)
+      time['run_every'] ||= ask('The amount of time, in seconds, between runs. By default, the task will only run once. Must be greater than 60:').to_i
+      #time['end_at'] ||= ask('The time tasks will stop being queued. Should be a time or datetime:')
+      #time['run_times'] ||= ask('Number of times the task will run:')
+      #time['priority'] ||= ask('The priority queue to run the job in. Higher values means tasks spend less time in the queue once they come off the schedule:', :limited_to => ['0','1','2'])
+      return time
+    end
+    
+    def prompt_credentials(credentials)
+      credentials['token'] ||= ask('Iron.io Token:')
+      credentials['project_id'] ||= ask('Iron.io project ID:')
+      return credentials
+    end
+  end
+  
+  class S3twinCLI < Thor
+    desc 'ironworker SUBCOMMAND ...ARGS', 'IronWorker tasks'
+    subcommand 'ironworker', Ironworker
+  end
+end

data/s3twin.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+require 's3twin'
+
+Gem::Specification.new do |spec|
+  spec.name          = 's3twin'
+  spec.version       = S3twin::VERSION
+  spec.authors       = ['Robert Coleman']
+  spec.email         = ['github@robert.net.nz']
+  spec.description   = %q{Mirror a S3 Bucket via command line, with support for remote workers (and scheduling).}
+  spec.summary       = %q{Mirror a S3 Bucket via command line, with support for remote workers (and scheduling).}
+  spec.homepage      = 'https://github.com/rjocoleman/s3twin'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'thor', '~> 0.18'
+  spec.add_dependency 'dotenv', '~> 0.8'
+  spec.add_dependency 'aws-sdk', '~> 1.11'
+  spec.add_dependency 'iron_worker_ng', '~> 0.16'
+  
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'pry', '~> 0.9'
+  spec.add_development_dependency 'pry-debugger', '~> 0.2'
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: s3twin
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Robert Coleman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: iron_worker_ng
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: pry-debugger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: Mirror a S3 Bucket via command line, with support for remote workers
+  (and scheduling).
+email:
+- github@robert.net.nz
+executables:
+- s3twin
+extensions: []
+extra_rdoc_files: []
+files:
+- .env.sample
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/s3twin
+- lib/s3twin.rb
+- lib/s3twin/helpers.rb
+- lib/s3twin/twin.rb
+- lib/s3twin/workers.rb
+- lib/s3twin/workers/ironworker.rb
+- s3twin.gemspec
+homepage: https://github.com/rjocoleman/s3twin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Mirror a S3 Bucket via command line, with support for remote workers (and
+  scheduling).
+test_files: []