s3sync 0.3.2

data/bin/s3sync

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+#
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+$:.unshift(File.dirname(__FILE__) + '/../lib') unless $:.include?(File.dirname(__FILE__) + '/../lib')
+
+require "s3sync/exceptions"
+require "s3sync/config"
+require "s3sync/cli"
+
+conf = S3Sync::Config.new
+
+# Time to load config and see if we've got everything we need to cook our salad
+begin
+  conf.read
+rescue S3Sync::NoConfigFound => exc
+  # We can't proceed without having those two vars set
+  $stderr.puts "You didn't set up the following environment variables:"
+  $stderr.puts
+  exc.missing_vars.each {|var| $stderr.puts "   * #{var}"}
+  $stderr.puts
+
+  $stderr.puts "I tried to load a config file from the following paths:"
+  $stderr.puts
+  exc.paths_checked.each {|path| $stderr.puts "   * #{path}"}
+  $stderr.puts
+
+  $stderr.puts "You could try to set the `S3SYNC_PATH' environment variable"
+  $stderr.puts "pointing to a file to be loaded as your config file or just"
+  $stderr.puts "export those variables to your environment like this:"
+  $stderr.puts
+  exc.missing_vars.each {|var|
+    $stderr.puts "    $ export #{var}=<value-provided-by-amazon>"
+  }
+  $stderr.puts
+  $stderr.puts "Learn how to do that here: https://github.com/clarete/s3sync"
+  exit
+end
+
+# Step aside, the star of this show is here. Let's try to create the
+# environment to run the requested command. And feed the user back if
+# information needed was not enough
+begin
+  S3Sync::CLI::run conf
+rescue S3Sync::FailureFeedback => exc
+  $stderr.puts exc.message
+  exit 1
+rescue S3Sync::WrongUsage => exc
+  $stderr.puts "Error:\n  #{exc.msg}\n" if exc.msg
+  exit exc.error_code
+end

data/lib/s3sync.rb

@@ -0,0 +1,2 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))

data/lib/s3sync/cli.rb

@@ -0,0 +1,475 @@
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 's3sync/version'
+require 's3sync/exceptions'
+require 's3sync/sync'
+require 'aws/s3'
+require 'cmdparse'
+
+
+module S3Sync
+  module CLI
+
+    AVAILABLE_ACLS = [:public_read, :public_read_write, :private]
+
+    AVAILABLE_METHODS = ['read', 'get', 'put', 'write', 'delete']
+
+    class BaseCmd < CmdParse::Command
+
+      @has_prefix = false
+
+      def has_options?
+        not options.instance_variables.empty?
+      end
+
+      def has_prefix?
+        @has_prefix
+      end
+
+      def usage
+        u = []
+        u << "Usage: #{File.basename commandparser.program_name} #{name} "
+        u << "[options] " if has_options?
+        u << "bucket" if has_args?
+
+        if has_prefix? == 'required'
+          u << ':prefix'
+        elsif has_prefix?
+          u << "[:prefix]"
+        end
+
+        u.join ''
+      end
+
+      protected
+
+      def parse_acl(opt)
+        @acl = nil
+        opt.on("-a", "--acl=ACL", "Options: #{AVAILABLE_ACLS.join ', '}") {|acl|
+          @acl = acl.to_sym
+        }
+      end
+    end
+
+    class ListBuckets < BaseCmd
+      def initialize
+        super 'listbuckets', false, false, false
+
+        @short_desc = "List all available buckets for your user"
+      end
+
+      def run s3, bucket, key, file, args
+        s3.buckets.each do |bkt|
+          puts "#{bkt.name}"
+        end
+      end
+    end
+
+    class CreateBucket < BaseCmd
+      attr_accessor :acl
+
+      def initialize
+        super 'createbucket', false, false
+
+        @short_desc = "Create a new bucket under your user account"
+
+        self.options = CmdParse::OptionParserWrapper.new do |opt|
+          parse_acl(opt)
+        end
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+
+        begin
+          params = {}
+          if @acl
+            raise WrongUsage.new(nil, "Invalid ACL `#{@acl}'. Should be any of #{AVAILABLE_ACLS.join ', '}") if not AVAILABLE_ACLS.include? @acl
+            params.merge!({:acl => @acl})
+          end
+
+          s3.buckets.create bucket, params
+        rescue AWS::S3::Errors::BucketAlreadyExists
+          raise FailureFeedback.new("Bucket `#{bucket}' already exists")
+        end
+      end
+    end
+
+    class DeleteBucket < BaseCmd
+      attr_accessor :force
+
+      def initialize
+        super 'deletebucket', false, false
+
+        @short_desc = "Remove a bucket from your account"
+
+        @force = false
+
+        self.options = CmdParse::OptionParserWrapper.new do |opt|
+          opt.on("-f", "--force", "Clean the bucket then deletes it") {|f|
+            @force = f
+          }
+        end
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+
+        # Getting the bucket
+        bucket_obj = s3.buckets[bucket]
+
+        # Do not kill buckets with content unless explicitly asked
+        if not @force and bucket_obj.objects.count > 0
+          raise FailureFeedback.new("Cowardly refusing to remove non-empty bucket `#{bucket}'. Try with -f.")
+        end
+
+        bucket_obj.delete!
+      end
+    end
+
+    class List < BaseCmd
+      attr_accessor :max_entries
+
+      def initialize
+        super 'list', false, false
+
+        @short_desc = "List items filed under a given bucket"
+
+        @max_entries = 0
+
+        @delimiter = "\t"
+
+        @has_prefix = true
+
+        self.options = CmdParse::OptionParserWrapper.new do |opt|
+          opt.on("-m", "--max-entries=NUM", "Limit the number of entries to output") {|m|
+            @max_entries = m
+          }
+
+          opt.on("-d", "--delimiter=D", "Charactere used to separate columns") {|d|
+            @delimiter = d
+          }
+        end
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+
+        collection = s3.buckets[bucket].objects.with_prefix(key || "")
+
+        if @max_entries > 0
+          collection = collection.page(:per_page => @max_entries)
+        end
+
+        collection.each {|object|
+          o = []
+          o << object.key
+          o << @delimiter
+          o << object.content_length
+          o << @delimiter
+          o << object.last_modified
+          puts o.join
+        }
+      end
+    end
+
+    class Delete < BaseCmd
+      def initialize
+        super 'delete', false, false
+
+        @short_desc = "Delete a key from a bucket"
+
+        @has_prefix = 'required'
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+        raise WrongUsage.new(nil, "You need to inform a key") if not key
+        s3.buckets[bucket].objects[key].delete
+      end
+    end
+
+    class Url < BaseCmd
+      attr_accessor :method
+      attr_accessor :secure
+
+      def initialize
+        super 'url', false, false
+
+        @short_desc = "Generates public urls or authenticated endpoints for the object"
+        @description = "Notice that --method and --public are mutually exclusive"
+        @method = false
+        @public = false
+        @secure = true
+        @expires_in = false
+        @has_prefix = 'required'
+
+        self.options = CmdParse::OptionParserWrapper.new do |opt|
+          opt.on("-m", "--method=METHOD", "Options: #{AVAILABLE_METHODS.join ', '}") {|m|
+            @method = m
+          }
+
+          opt.on("-p", "--public", "Generates a public (not authenticated) URL for the object") {|p|
+            @public = p
+          }
+
+          opt.on("--no-ssl", "Generate an HTTP link, no HTTPS") {
+            @secure = false
+          }
+
+          opt.on("--expires-in=EXPR", "How long the link takes to expire. Format: <# of seconds> | [#d|#h|#m|#s]") { |expr|
+            val = 0
+            expr.scan(/(\d+\w)/) do |track|
+              _, num, what = /(\d+)(\w)/.match(track[0]).to_a
+              num = num.to_i
+
+              case what
+              when "d"; val += num * 86400
+              when "h"; val += num * 3600
+              when "m"; val += num * 60
+              when "s"; val += num
+              end
+            end
+            @expires_in = val > 0 ? val : expr.to_i
+          }
+        end
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+        raise WrongUsage.new(nil, "You need to inform a key") if not key
+        raise WrongUsage.new(nil, "Params --method and --public are mutually exclusive") if (@method and @public)
+        if not AVAILABLE_METHODS.include? method = @method || 'read'
+          raise WrongUsage.new(nil, "Unknown method #{method}")
+        end
+
+        opts = {}
+        opts.merge!({:secure => @secure})
+
+        if @public
+          puts s3.buckets[bucket].objects[key].public_url(opts).to_s
+        else
+          opts.merge!({:expires => @expires_in}) if @expires_in
+          puts s3.buckets[bucket].objects[key].url_for(method.to_sym, opts).to_s
+        end
+      end
+    end
+
+    class Put < BaseCmd
+      def initialize
+        super 'put', false, false
+
+        @short_desc = 'Upload a file to a bucket under a certain prefix'
+        @has_prefix = true
+      end
+
+      def usage
+        "#{super} path/to/local/destination"
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+        raise WrongUsage.new(nil, "You need to inform a file") if not file
+
+        name = S3Sync.safe_join [key, File.basename(file)]
+        s3.buckets[bucket].objects[name].write Pathname.new(file)
+      end
+    end
+
+    class Get < BaseCmd
+      def initialize
+        super 'get', false, false
+        @short_desc = "Retrieve an object and save to the specified file"
+        @has_prefix = 'required'
+      end
+
+      def usage
+        "#{super} path/to/local/destination"
+      end
+
+      def run s3, bucket, key, file, args
+        raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
+        raise WrongUsage.new(nil, "You need to inform a key") if not key
+        raise WrongUsage.new(nil, "You need to inform a file") if not file
+
+        # Saving the content to be downloaded to the current directory if the
+        # destination is a directory
+        path = File.absolute_path file
+        path = S3Sync.safe_join [path, File.basename(key)] if File.directory? path
+        File.open(path, 'wb') do |f|
+          s3.buckets[bucket].objects[key].read do |chunk| f.write(chunk) end
+        end
+      end
+    end
+
+    class Sync < BaseCmd
+      attr_accessor :s3
+      attr_accessor :exclude
+      attr_accessor :keep
+      attr_accessor :dry_run
+      attr_accessor :verbose
+      attr_accessor :acl
+
+      def initialize
+        super 'sync', false, false
+
+        @short_desc = "Synchronize an S3 and a local folder"
+        @s3 = nil
+        @exclude = nil
+        @keep = false
+        @dry_run = false
+        @verbose = false
+
+        self.options = CmdParse::OptionParserWrapper.new do |opt|
+          opt.on("-x EXPR", "--exclude=EXPR", "Skip copying files that matches this pattern. (Ruby REs)") {|v|
+            @exclude = v
+          }
+
+          opt.on("-k", "--keep", "Keep files even if they don't exist in source") {
+            @keep = true
+          }
+
+          parse_acl(opt)
+
+          opt.on("-d", "--dry-run", "Do not download or exclude anything, just show what was planned. Implies `verbose`.") {
+            @dry_run = true
+            @verbose = true
+          }
+
+          opt.on("-v", "--verbose", "Show file names") {
+            @verbose = true
+          }
+        end
+      end
+
+      def usage
+        "Usage: #{File.basename commandparser.program_name} #{name} source destination"
+      end
+
+      def description
+        @description =<<END.strip
+
+Where `source' and `description' might be either local or remote
+addresses. A local address is simply a path in your local file
+system. e.g:
+
+    /tmp/notes.txt
+
+A remote address is a combination of the `bucket` name and
+an optional `prefix`:
+
+    disc.company.com:reports/2013/08/30.html
+
+So, a full example would be something like this
+
+    $ #{File.basename commandparser.program_name} sync Work/reports disc.company.com:reports/2013/08
+
+The above line will update the remote folder `reports/2013/08` with the
+contents of the local folder `Work/reports`.
+END
+      end
+
+      def run s3, bucket, key, file, args
+        @s3 = s3
+        cmd = SyncCommand.new self, *args
+        cmd.run
+      end
+    end
+
+    def run conf
+      cmd = CmdParse::CommandParser.new true
+      cmd.program_name = File.basename $0
+      cmd.program_version = S3Sync::VERSION
+
+      cmd.options = CmdParse::OptionParserWrapper.new do |opt|
+        opt.separator "Global options:"
+      end
+
+      cmd.main_command.short_desc = 'Tool belt for managing your S3 buckets'
+      cmd.main_command.description =<<END.strip
+S3Sync provides a list of commands that will allow you to manage your content
+stored in S3 buckets. To learn about each feature, please use the `help`
+command:
+
+    $ #{File.basename $0} help sync"
+END
+      # Commands used more often
+      cmd.add_command List.new
+      cmd.add_command Delete.new
+      cmd.add_command Url.new
+      cmd.add_command Put.new
+      cmd.add_command Get.new
+      cmd.add_command Sync.new
+
+      # Bucket related options
+      cmd.add_command ListBuckets.new
+      cmd.add_command CreateBucket.new
+      cmd.add_command DeleteBucket.new
+
+      # Built-in commands
+      cmd.add_command CmdParse::HelpCommand.new
+      cmd.add_command CmdParse::VersionCommand.new
+
+      # Defining the `execute` method as a closure, so we can forward the
+      # arguments needed to run the instance of the chosen command.
+      CmdParse::Command.class_eval do
+        define_method :execute, lambda { |args|
+
+          # Connecting to amazon
+          s3 = AWS::S3.new(
+            :access_key_id => conf[:AWS_ACCESS_KEY_ID],
+            :secret_access_key => conf[:AWS_SECRET_ACCESS_KEY],
+          )
+
+          # From the command line
+          key, file = args
+
+          # Parsing the bucket name
+          bucket = nil
+          bucket, key = key.split(':') if key
+
+          # Running our custom method inside of the command class, taking care
+          # of the common errors here, saving duplications in each command;
+          begin
+            run s3, bucket, key, file, args
+          rescue AWS::S3::Errors::AccessDenied
+            raise FailureFeedback.new("Access Denied")
+          rescue AWS::S3::Errors::NoSuchBucket
+            raise FailureFeedback.new("There's no bucket named `#{bucket}'")
+          rescue AWS::S3::Errors::NoSuchKey
+            raise FailureFeedback.new("There's no key named `#{key}' in the bucket `#{bucket}'")
+          rescue AWS::S3::Errors::Base => exc
+            raise FailureFeedback.new("Error: `#{exc.message}'")
+          end
+        }
+      end
+
+      cmd.parse
+    end
+
+    module_function :run
+
+  end
+end

data/lib/s3sync/config.rb

@@ -0,0 +1,98 @@
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# Part of this software was inspired by the original s3sync, so here's their
+# copyright notice:
+
+# This software code is made available "AS IS" without warranties of any
+# kind.  You may copy, display, modify and redistribute the software
+# code either by itself or as incorporated into your code; provided that
+# you do not remove any proprietary notices.  Your use of this software
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code.
+# (c) 2007 alastair brunton
+#
+# modified to search out the yaml in several places, thanks wkharold.
+
+require 'yaml'
+require 's3sync/exceptions'
+
+
+module S3Sync
+
+  class Config < Hash
+
+    REQUIRED_VARS = [:AWS_ACCESS_KEY_ID, :AWS_SECRET_ACCESS_KEY]
+
+    CONFIG_PATHS = ["#{ENV['S3SYNC_PATH']}", "#{ENV['HOME']}/.s3sync.yml", "/etc/s3sync.yml"]
+
+    def read_from_file
+      paths_checked = []
+
+      CONFIG_PATHS.each do |path|
+
+        # Filtering some garbage
+        next if path.nil? or path.strip.empty?
+
+        # Feeding the user feedback in case of failure
+        paths_checked << path
+
+        # Time for the dirty work, let's parse the config file and feed our
+        # internal hash
+        if File.exists? path
+          config = YAML.load_file path
+          config.each_pair do |key, value|
+            self[key.upcase.to_sym] = value
+          end
+          return
+        end
+      end
+
+      return paths_checked
+    end
+
+    def read_from_env
+      REQUIRED_VARS.each do |v|
+        self[v] = ENV[v.to_s] unless ENV[v.to_s].nil?
+      end
+    end
+
+    def read
+      # Reading from file and then trying from env
+      paths_checked = read_from_file
+      read_from_env
+
+      # Checking which variables we have
+      not_found = []
+
+      REQUIRED_VARS.each {|v|
+        not_found << v if self[v].nil?
+      }
+
+      # Cleaning possibly empty env var from CONFIG_PATH
+      paths = (paths_checked || CONFIG_PATHS).select {|e| !e.empty?}
+      raise NoConfigFound.new(not_found, paths) if not_found.count > 0
+    end
+  end
+end

data/lib/s3sync/exceptions.rb

@@ -0,0 +1,55 @@
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module S3Sync
+
+  class SyncException < StandardError
+  end
+
+  class NoConfigFound < SyncException
+
+    attr_accessor :missing_vars
+    attr_accessor :paths_checked
+
+    def initialize missing_vars, paths_checked
+      @missing_vars = missing_vars
+      @paths_checked = paths_checked
+    end
+  end
+
+  class WrongUsage < SyncException
+
+    attr_accessor :error_code
+    attr_accessor :msg
+
+    def initialize(error_code, msg)
+      @error_code = error_code || 1
+      @msg = msg
+    end
+  end
+
+  class FailureFeedback < SyncException
+  end
+
+end

data/lib/s3sync/sync.rb

@@ -0,0 +1,366 @@
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# Part of this software was inspired by the original s3sync, so here's their
+# copyright notice:
+
+# (c) 2007 s3sync.net
+#
+# This software code is made available "AS IS" without warranties of any
+# kind.  You may copy, display, modify and redistribute the software
+# code either by itself or as incorporated into your code; provided that
+# you do not remove any proprietary notices.  Your use of this software
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code.
+
+require 'find'
+require 'fileutils'
+require 's3sync/util'
+
+module S3Sync
+
+  class Location
+    attr_accessor :path
+    attr_accessor :bucket
+
+    def initialize path, bucket=nil
+      raise RuntimeError if path.nil?
+      @path = path
+      @bucket = bucket || nil
+    end
+
+    def to_s
+      out = []
+      out << @bucket unless @bucket.nil?
+      out << @path
+      out.join ':'
+    end
+
+    def local?
+      @bucket.nil?
+    end
+
+    def == other
+      @path == other.path and @bucket == other.bucket
+    end
+
+    alias eql? ==
+  end
+
+  class Node
+    include Comparable
+
+    attr_accessor :base
+    attr_accessor :path
+    attr_accessor :size
+
+    def initialize base, path, size
+      @base = base
+      @path = path
+      @size = size
+    end
+
+    def full
+      S3Sync.safe_join [@base, @path]
+    end
+
+    def == other
+      full == other.full and @size == other.size
+    end
+
+    def <=> other
+      if self.size < other.size
+        -1
+      elsif self.size > other.size
+        1
+      else
+        0
+      end
+    end
+
+    alias eql? ==
+  end
+
+  class LocalDirectory
+    attr_accessor :source
+
+    def initialize source
+      @source = source
+    end
+
+    def list_files
+      nodes = {}
+      Find.find(@source) do |file|
+        st = File.stat file
+
+        # We don't support following symlinks for now, we don't need to follow
+        # folders and I don't think we care about any other thing, right?
+        next unless st.file?
+
+        # Well, we're kinda out of options right now, I'll just yell!
+        if not st.readable?
+          $stderr.puts "WARNING: Skipping unreadable file #{file}"
+          next
+        end
+
+        # We only need the relative path here
+        file_name = file.gsub(/^#{@source}\/?/, '').squeeze('/')
+        node = Node.new(@source.squeeze('/'), file_name, st.size)
+        nodes[node.path] = node
+      end
+
+      return nodes
+    end
+  end
+
+  class SyncCommand
+
+    def self.cmp hash1, hash2
+      same, to_add_to_2 = [], []
+
+      hash1.each do |key, value|
+        value2 = hash2.delete key
+        if value2.nil?
+          to_add_to_2 << value
+        elsif value2.size == value.size
+          same << value
+        else
+          to_add_to_2 << value
+        end
+      end
+
+      to_remove_from_2 = hash2.values
+
+      [same, to_add_to_2, to_remove_from_2]
+    end
+
+    def initialize args, source, destination
+      @args = args
+      @source = source
+      @destination = destination
+    end
+
+    def run
+      # Reading the source and destination using our helper method
+      if (source, destination, bucket = self.class.parse_params [@source, @destination]).nil?
+        raise WrongUsage.new(nil, 'Need a source and a destination')
+      end
+
+      # Getting the trees
+      source_tree, destination_tree = read_trees source, destination
+
+      # Getting the list of resources to be exchanged between the two peers
+      _, to_add, to_remove = self.class.cmp source_tree, destination_tree
+
+      # Removing the items matching the exclude pattern if requested
+      to_add.select! { |e|
+        begin
+          (e.path =~ /#{@args.exclude}/).nil?
+        rescue RegexpError => exc
+          raise WrongUsage.new nil, exc.message
+        end
+      } if @args.exclude
+
+      # Calling the methods that perform the actual IO
+      if source.local?
+        upload_files destination, to_add
+        remove_files destination, to_remove unless @args.keep
+      else
+        download_files destination, source, to_add
+        remove_local_files destination, source, to_remove unless @args.keep
+      end
+    end
+
+    def self.parse_params args
+      # Reading the arbitrary parameters from the command line and getting
+      # modifiable copies to parse
+      source, destination = args; return nil if source.nil? or destination.nil?
+
+      # Sync from one s3 to another is currently not supported
+      if remote_prefix? source and remote_prefix? destination
+        raise WrongUsage.new(nil, 'Both arguments can\'t be on S3')
+      end
+
+      # C'mon, there's rsync out there
+      if !remote_prefix? source and !remote_prefix? destination
+        raise WrongUsage.new(nil, 'One argument must be on S3')
+      end
+
+      source, destination = process_destination source, destination
+      return [Location.new(*source), Location.new(*destination)]
+    end
+
+    def self.remote_prefix?(prefix)
+      # allow for dos-like things e.g. C:\ to be treated as local even with
+      # colon.
+      prefix.include? ':' and not prefix.match '^[A-Za-z]:[\\\\/]'
+    end
+
+    def self.process_file_destination source, destination, file=""
+      if not file.empty?
+        sub = (remote_prefix? source) ? source.split(":")[1] : source
+        file = file.gsub(/^#{sub}/, '')
+      end
+
+      # no slash on end of source means we need to append the last src dir to
+      # dst prefix testing for empty isn't good enough here.. needs to be
+      # "empty apart from potentially having 'bucket:'"
+      if source =~ %r{/$}
+        File.join [destination, file]
+      else
+        if remote_prefix? source
+          _, name = source.split ":"
+          File.join [destination, File.basename(name || ""), file]
+        else
+          source = /^\/?(.*)/.match(source)[1]
+
+          # Corner case: the root of the remote path is empty, we don't want to
+          # add an unnecessary slash here.
+          if destination.end_with? ':'
+            File.join [destination + source, file]
+          else
+            File.join [destination, source, file]
+          end
+        end
+      end
+    end
+
+    def self.process_destination source, destination
+      source, destination = source.dup, destination.dup
+
+      # don't repeat slashes
+      source.squeeze! '/'
+      destination.squeeze! '/'
+
+      # Making sure that local paths won't break our stuff later
+      source.gsub!(/^\.\//, '')
+      destination.gsub!(/^\.\//, '')
+
+      # Parsing the final destination
+      destination = process_file_destination source, destination, ""
+
+      # here's where we find out what direction we're going
+      source_is_s3 = remote_prefix? source
+
+      # canonicalize the S3 stuff
+      remote_prefix = source_is_s3 ? source : destination
+      bucket, remote_prefix = remote_prefix.split ":"
+      remote_prefix ||= ""
+
+      # Just making sure we preserve the direction
+      if source_is_s3
+        [[remote_prefix, bucket], destination]
+      else
+        [source, [remote_prefix, bucket]]
+      end
+    end
+
+    def read_tree_remote location
+      dir = location.path
+      dir += '/' if not dir.empty? and not dir.end_with?('/')
+
+      nodes = {}
+      @args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect do |obj|
+        node = Node.new(location.path, obj.key, obj.content_length)
+        nodes[node.path] = node
+      end
+      return nodes
+    end
+
+    def read_trees source, destination
+      if source.local?
+        source_tree = LocalDirectory.new(source.path).list_files
+        destination_tree = read_tree_remote destination
+      else
+        source_tree = read_tree_remote source
+        destination_tree = LocalDirectory.new(destination.path).list_files
+      end
+
+      [source_tree, destination_tree]
+    end
+
+    def upload_files remote, list
+      list.each do |e|
+        if @args.verbose
+          puts " + #{e.full} => #{remote}#{e.path}"
+        end
+
+        unless @args.dry_run
+          remote_path = "#{remote.path}#{e.path}"
+          @args.s3.buckets[remote.bucket].objects[remote_path].write Pathname.new(e.full), :acl => @args.acl
+        end
+      end
+    end
+
+    def remove_files remote, list
+      if @args.verbose
+        list.each {|e|
+          puts " - #{remote}#{e.path}"
+        }
+      end
+
+      unless @args.dry_run
+        @args.s3.buckets[remote.bucket].objects.delete_if { |obj| list.map(&:path).include? obj.key }
+      end
+    end
+
+    def download_files destination, source, list
+      list.each {|e|
+        path = File.join destination.path, e.path
+
+        if @args.verbose
+          puts " + #{source}#{e.path} => #{path}"
+        end
+
+        unless @args.dry_run
+          obj = @args.s3.buckets[source.bucket].objects[e.path]
+
+          # Making sure this new file will have a safe shelter
+          FileUtils.mkdir_p File.dirname(path)
+
+          # Downloading and saving the files
+          File.open(path, 'wb') do |file|
+            obj.read do |chunk|
+              file.write chunk
+            end
+          end
+        end
+      }
+    end
+
+    def remove_local_files destination, source, list
+      list.each {|e|
+        path = File.join destination.path, e.path
+
+        if @args.verbose
+          puts " * #{e.path} => #{path}"
+        end
+
+        unless @args.dry_run
+          FileUtils.rm_rf path
+        end
+      }
+    end
+  end
+end

data/lib/s3sync/util.rb

@@ -0,0 +1,29 @@
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module S3Sync
+  def S3Sync.safe_join(parts)
+    File.join(*(parts.select {|v| !v.nil? && !v.empty? }))
+  end
+end

data/lib/s3sync/version.rb

@@ -0,0 +1,27 @@
+# s3sync - Tool belt for managing your S3 buckets
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module S3Sync
+  VERSION = "0.3.2"
+end

metadata

@@ -0,0 +1,190 @@
+--- !ruby/object:Gem::Specification
+name: s3sync
+version: !ruby/object:Gem::Version
+  version: 0.3.2
+  prerelease: 
+platform: ruby
+authors:
+- Lincoln de Sousa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cmdparse
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bump
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tool belt for managing your S3 buckets
+email:
+- lincoln@comum.org
+executables:
+- s3sync
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/s3sync
+- lib/s3sync.rb
+- lib/s3sync/cli.rb
+- lib/s3sync/config.rb
+- lib/s3sync/exceptions.rb
+- lib/s3sync/sync.rb
+- lib/s3sync/util.rb
+- lib/s3sync/version.rb
+homepage: https://github.com/clarete/s3sync
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4017142764294384593
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4017142764294384593
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: s3sync is a library that aggregates a good range of features for managing
+  your Amazon S3 buckets. It also provides basic interactive client
+test_files: []