s3ranger 0.1.0

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/s3ranger

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby 
+
+$:.unshift(File.dirname(__FILE__) + '/../lib') unless $:.include?(File.dirname(__FILE__) + '/../lib')
+
+require "s3ranger/exceptions"
+require "s3ranger/config"
+require "s3ranger/cmd"
+
+conf = S3Ranger::Config.new
+
+# Time to load config and see if we've got everything we need to cook our salad
+begin
+  conf.read
+rescue S3Ranger::NoConfigFound => exc
+  # We can't proceed without having those two vars set
+  if not (conf.has_key? "AWS_ACCESS_KEY_ID" and conf.has_key? "AWS_SECRET_ACCESS_KEY")
+    $stderr.puts "You didn't set up your environment variables :("
+    $stderr.puts "I tried the following paths:"
+    exc.paths_checked.each {|path| $stderr.puts " * #{path}/s3config.yml"}
+
+    $stderr.puts "You could try to set the `S3CONF' environment variable."
+    $stderr.puts "Learn how to do that here: https://github.com/clarete/s3ranger"
+    exit
+  end
+end
+
+
+# Step aside, the star of this show is here. Let's try to create the
+# environment to run the requested command. And feed the user back if
+# information needed was not enough
+begin
+  S3Ranger::Cmd.new(conf)
+
+rescue S3Ranger::FailureFeedback => exc
+  $stderr.puts exc.message
+  exit 1
+
+rescue S3Ranger::WrongUsage => exc
+  name = $0.split('/').last
+
+  $stderr.puts <<"ENDUSAGE"
+Usage: #{name} [options] <command> [arg(s)]
+
+Global Options:
+  -h, --help
+  -v, --verbose
+  -n, --dryrun
+  -d, --debug
+      --progress
+
+#{name} listbuckets
+  List all available buckets
+
+#{name} createbucket <bucket>
+  Creates a new bucket
+
+  Options:
+
+    -a <ACL>, --acl=(private|public_read|public_read_write)
+
+#{name} deletebucket [options] <bucket>
+  Removes an existing bucket
+
+  Options:
+    -f, --force  Deletes non-empty buckets (BE CAREFUL)
+
+#{name} list <bucket>[:prefix]
+  List content inside of bucket
+
+  if `prefix' is present, only content under `prefix' will be listed.
+
+#{name} delete <bucket>:<key>
+  Removes `key` from `bucket`
+
+#{name} url [options] <bucket>:<key>
+  Generates a presigned URL for an operation on the object named `key' found on
+  `bucket'.
+
+  Options:
+    --no-ssl
+    --expires-in=(<# of seconds> | [#d|#h|#m|#s])
+
+#{name} get <bucket>:<key> <file>
+  Retrieves the remote `key` object from `bucket` and saves to the local path
+  specified in `file`
+
+#{name} put <bucket>[:<key>] <file>
+  Uploads the file `file` to the `bucket` under the path `key`
+
+#{name} sync <source> <destination>
+
+  One of source or destination must be remote and the other must be local,
+  where local points to a folder in the file system and remote conform to the
+  format `<bucket>[:<key>]`.
+
+  Options:
+    -e <pattern>, --exclude=<pattern>
+    -k, --keep
+    -d, --dry-run
+
+ENDUSAGE
+
+  $stderr.puts "\nCurrent error:\n  #{exc.msg}\n" if exc.msg
+  exit exc.error_code
+end

data/lib/s3ranger/cmd.rb

@@ -0,0 +1,112 @@
+# (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+# (c) 2007 s3sync.net
+#
+# This software code is made available "AS IS" without warranties of any
+# kind.  You may copy, display, modify and redistribute the software
+# code either by itself or as incorporated into your code; provided that
+# you do not remove any proprietary notices.  Your use of this software
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code.
+
+require 'getoptlong'
+require 's3ranger/commands'
+require 's3ranger/util'
+
+module S3Ranger
+
+  class Cmd
+
+    def initialize(conf = conf)
+      # The chain that initializes our command and find the right action
+      options, command, bucket, key, file = read_info_from_args(parse_args())
+
+      # Finding the right command to run
+      (cmd = find_cmd(command)) || (raise WrongUsage.new(nil, "Command `#{command}' does not exist"))
+
+      # Now that we're sure we have things to do, we need to connect to amazon
+      s3 = AWS::S3.new(
+        :access_key_id => conf[:AWS_ACCESS_KEY_ID],
+        :secret_access_key => conf[:AWS_SECRET_ACCESS_KEY],
+      )
+
+      # Calling the actuall command
+      cmd.call({
+        :options => options,
+        :s3 => s3,
+        :bucket => bucket,
+        :key => key,
+        :file => file,
+      })
+    end
+
+    def find_cmd name
+      sym = "_cmd_#{name}".to_sym
+      return nil unless Commands.public_methods.include? sym
+      return Commands.method sym
+    end
+
+    def parse_args
+      options = Hash.new
+
+      args = [
+        ['--help',       '-h', GetoptLong::NO_ARGUMENT],
+        ['--force',      '-f', GetoptLong::NO_ARGUMENT],
+        ['--acl',        '-a', GetoptLong::REQUIRED_ARGUMENT],
+        ['--method',     '-m', GetoptLong::REQUIRED_ARGUMENT],
+        ['--exclude',    '-e', GetoptLong::REQUIRED_ARGUMENT],
+        ['--keep',       '-k', GetoptLong::NO_ARGUMENT],
+        ['--dry-run',    '-d', GetoptLong::NO_ARGUMENT],
+        ['--verbose',    '-v', GetoptLong::NO_ARGUMENT],
+        ['--no-ssl',           GetoptLong::NO_ARGUMENT],
+        ['--expires-in',       GetoptLong::REQUIRED_ARGUMENT],
+      ]
+
+      begin
+        GetoptLong.new(*args).each {|opt, arg| options[opt] = (arg || true)}
+      rescue StandardError => exc
+        raise WrongUsage.new nil, exc.message
+      end
+
+      # Let's just show the help to the user
+      raise WrongUsage.new(0, nil) if options['--help']
+
+      # Returning the options to the next level
+      options
+    end
+
+    def read_info_from_args(options)
+      # Parsing expre date
+      if options['--expires-in'] =~ /d|h|m|s/
+
+        val = 0
+
+        options['--expires-in'].scan(/(\d+\w)/) do |track|
+          _, num, what = /(\d+)(\w)/.match(track[0]).to_a
+          num = num.to_i
+
+          case what
+          when "d"; val += num * 86400
+          when "h"; val += num * 3600
+          when "m"; val += num * 60
+          when "s"; val += num
+          end
+        end
+
+        options['--expires-in'] = val
+      end
+
+      # Reading what to do from the user
+      command = ARGV.shift
+      raise WrongUsage.new(nil, "Need a command (eg.: `list', `listbuckets', etc)") if not command
+
+      key, file = ARGV
+
+      # Parsing the bucket name
+      bucket = nil
+      bucket, key = key.split(':') if key
+
+      # Returning things we need in the next level
+      [options, command, bucket, key, file]
+    end
+  end
+end

data/lib/s3ranger/commands.rb

@@ -0,0 +1,114 @@
+require 's3ranger/exceptions'
+require 's3ranger/sync'
+require 'aws/s3'
+
+
+module Commands
+
+  include S3Ranger
+
+  AVAILABLE_ACLS = [:public_read, :public_read_write, :private]
+
+  AVAILABLE_METHODS = ['read', 'get', 'put', 'write', 'delete']
+
+  def Commands._cmd_listbuckets args
+    args[:s3].buckets.each do |bkt|
+      puts "#{bkt.name}"
+    end
+  end
+
+  def Commands._cmd_createbucket args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+
+    begin
+      params = {}
+      if acl = args[:options]['--acl']
+        raise WrongUsage.new(nil, "Invalid ACL. Should be any of #{EXISTING_ACLS.join ', '}") if not AVAILABLE_ACLS.include? acl
+        params.merge!({:acl => acl.to_sym})
+      end
+
+      args[:s3].buckets.create args[:bucket], params
+    rescue AWS::S3::Errors::BucketAlreadyExists => exc
+      raise FailureFeedback.new("Bucket `#{bucket}' already exists")
+    end
+  end
+
+  def Commands._cmd_deletebucket args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+
+    # Getting the bucket
+    bucket_obj = args[:s3].buckets[args[:bucket]]
+
+    # Do not kill buckets with content unless explicitly asked
+    if not args[:options]['--force'] and bucket_obj.objects.count > 0
+      raise FailureFeedback.new("Cowardly refusing to remove non-empty bucket `#{bucket}'. Try with -f.")
+    end
+
+    begin
+      bucket_obj.delete!
+    rescue AWS::S3::Errors::AccessDenied => exc
+      raise FailureFeedback.new("Access Denied")
+    end
+  end
+
+  def Commands._cmd_list args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+    args[:s3].buckets[args[:bucket]].objects.with_prefix(args[:key] || "").each do |object|
+      puts "#{object.key}\t#{object.content_length}\t#{object.last_modified}"
+    end
+  end
+
+  def Commands._cmd_delete args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+    raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
+    args[:s3].buckets[args[:bucket]].objects[args[:key]].delete
+  end
+
+  def Commands._cmd_url args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+    raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
+
+    method = args[:options]['--method'] || 'read'
+    raise WrongUsage.new(nil, "") unless AVAILABLE_METHODS.include? method
+
+    opts = {}
+    opts.merge!({:secure => args[:options]["--no-ssl"].nil?})
+    opts.merge!({:expires => args[:options]["--expires-in"]}) if args[:options]["--expires-in"]
+    p (args[:s3].buckets[args[:bucket]].objects[args[:key]].url_for method.to_sym, opts).to_s
+  end
+
+  def Commands._cmd_put args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+    raise WrongUsage.new(nil, "You need to inform a file") if not args[:file]
+
+    # key + file name
+    name = S3Ranger.safe_join [args[:key], File.basename(args[:file])]
+    args[:s3].buckets[args[:bucket]].objects[name].write Pathname.new(args[:file])
+  end
+
+  def Commands._cmd_get args
+    raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
+    raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
+    raise WrongUsage.new(nil, "You need to inform a file") if not args[:file]
+
+    # Saving the content to be downloaded to the current directory if the
+    # destination is a directory
+    path = File.absolute_path args[:file]
+    path = S3Ranger.safe_join [path, File.basename(args[:key])] if File.directory? path
+
+    File.open(path, 'wb') do |f|
+      begin
+        args[:s3].buckets[args[:bucket]].objects[args[:key]].read do |chunk| f.write(chunk) end
+      rescue AWS::S3::Errors::NoSuchBucket
+        raise FailureFeedback.new("There's no bucket named `#{args[:bucket]}'")
+      rescue AWS::S3::Errors::NoSuchKey
+        raise FailureFeedback.new("There's no key named `#{args[:key]}' in the bucket `#{args[:bucket]}'")
+      end
+    end
+  end
+
+  def Commands._cmd_sync args
+    cmd = SyncCommand.new args, *ARGV
+    cmd.run
+  end
+end

data/lib/s3ranger/config.rb

@@ -0,0 +1,43 @@
+# This software code is made available "AS IS" without warranties of any
+# kind.  You may copy, display, modify and redistribute the software
+# code either by itself or as incorporated into your code; provided that
+# you do not remove any proprietary notices.  Your use of this software
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code.
+# (c) 2007 alastair brunton
+#
+# modified to search out the yaml in several places, thanks wkharold.
+
+require 'yaml'
+require 's3ranger/exceptions'
+
+
+module S3Ranger
+
+  class Config < Hash
+    def read
+      paths_checked = []
+
+      ["#{ENV['S3CONF']}", "#{ENV['HOME']}/.s3conf", "/etc/s3conf"].each do |path|
+
+        # Filtering some garbage
+        next if path.nil? or path.strip.empty?
+
+        # Feeding the user feedback in case of failure
+        paths_checked << path
+
+        # Time for the dirty work, let's parse the config file and feed our
+        # internal hash
+        if File.exists?("#{path}/s3config.yml")
+          config = YAML.load_file("#{path}/s3config.yml")
+          config.each_pair do |key, value|
+            self[key.upcase.to_sym] = value
+          end
+          return
+        end
+      end
+
+      raise NoConfigFound.new paths_checked
+    end
+  end
+end

data/lib/s3ranger/exceptions.rb

@@ -0,0 +1,38 @@
+# (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+#
+# This software code is made available "AS IS" without warranties of any
+# kind.  You may copy, display, modify and redistribute the software
+# code either by itself or as incorporated into your code; provided that
+# you do not remove any proprietary notices.  Your use of this software
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code.
+
+module S3Ranger
+
+  class SyncException < StandardError
+  end
+
+  class NoConfigFound < SyncException
+
+    attr_accessor :paths_checked
+
+    def initialize(paths_checked)
+      @paths_checked = paths_checked
+    end
+  end
+
+  class WrongUsage < SyncException
+
+    attr_accessor :error_code
+    attr_accessor :msg
+
+    def initialize(error_code, msg)
+      @error_code = error_code || 1
+      @msg = msg
+    end
+  end
+
+  class FailureFeedback < SyncException
+  end
+
+end