s3cp 0.1.1

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.1 / 2011-04-05
+
+* First release
+

data/README.md

@@ -0,0 +1,41 @@
+S3CP: Commands-line tools for Amazon S3 file manipulation
+=============================================================
+
+Just a few simple command-line utilities to list, copy, view S3 files, e.g.,  `s3cp`, `s3ls`, `s3cat`.
+
+### Building ###
+
+    # rake gem
+    # gem install s3cp-0.1.0.gem
+
+### Examples ###
+
+    export AWS_ACCESS_KEY_ID=...
+    export AWS_SECRET_ACCESS_KEY=...
+
+    s3ls s3://mybucket/path/to/some/files
+    s3cat s3://mybucket/path/to/some/file.txt
+    s3cp local_file.bin s3://mybucket/some/path
+    s3mod s3://mybucket/path/to/some/file.txt public-read
+
+Use the `-h` option to learn about command-line options.
+
+All commands support both `s3://bucket/path/to/file` and the legacy `bucket:path/to/file` syntax.
+
+Commands are also TTY-aware;  when run in an interactive shell, their behavior will change.  For example, `s3cat` will launch your favorite `PAGER` or `less` (the default pager) whereas `s3ls` will display N items at a time, where N is the number of display lines on your terminal and pause between pages.
+
+### Dependencies ###
+
+* highline `>=1.5.1`  (console/terminal size guessing)
+* right_aws `=2.1.0`  (underlying Amazon S3 API)
+* right_http_connection `=1.3.0` (required by `right_aws`)
+
+### Target platform ###
+
+* Ruby 1.8.7 / 1.9.2
+
+### License ###
+
+S3CP is is licensed under the terms of the Apache Software License v2.0.
+<http://www.apache.org/licenses/LICENSE-2.0.html>
+

data/bin/s3cat

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 's3cp/s3cat'
+

data/bin/s3cp

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 's3cp/s3cp'
+

data/bin/s3ls

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 's3cp/s3ls'
+

data/bin/s3mod

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby  
+require 's3cp/s3mod'
+
+

data/lib/s3cp/s3cat.rb

@@ -0,0 +1,68 @@
+require 'rubygems'
+require 'right_aws'
+require 'optparse'
+require 'date'
+require 'highline/import'
+require 'tempfile'
+
+require 's3cp/utils'
+
+# Parse arguments
+options = {}
+options[:tty] = $stdout.isatty
+
+op = OptionParser.new do |opts|
+  opts.banner = "s3cat [path]"
+  opts.separator ''
+
+  opts.on("--debug", "Debug mode") do
+    options[:debug] = true
+  end
+
+  opts.on("--tty", "TTY mode") do |tty|
+    options[:tty] = tty
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts op
+    exit
+  end
+end
+op.parse!(ARGV)
+
+unless ARGV.size > 0
+  puts op
+  exit
+end
+
+url = ARGV[0]
+
+if options[:debug]
+  puts "URL: #{url}"
+  puts "Options: \n#{options.inspect}"
+end
+
+@bucket, @prefix = S3CP.bucket_and_key(url)
+fail "Your URL looks funny, doesn't it?" unless @bucket
+
+@s3 = S3CP.connect().interface
+
+if options[:tty]
+  # store contents to file to display with PAGER
+  file = Tempfile.new('s3cat')
+  out = File.new(file.path, File::CREAT|File::RDWR)
+  begin
+    @s3.get(@bucket, @prefix) do |chunk|
+      out.write(chunk)
+    end
+  ensure
+    out.close()
+  end
+  exec "#{ENV['PAGER'] || 'less'} #{file.path}"
+  file.delete()
+else
+  @s3.get(@bucket, @prefix) do |chunk|
+    STDOUT.print(chunk)
+  end
+end
+

data/lib/s3cp/s3cp.rb

@@ -0,0 +1,171 @@
+require 'rubygems'
+require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
+require 'right_aws'
+require 'optparse'
+require 'date'
+require 'highline/import'
+require 'fileutils'
+
+require 's3cp/utils'
+
+# Parse arguments
+options = {}
+options[:verbose] = true if $stdout.isatty
+
+op = OptionParser.new do |opts|
+  opts.banner = "s3cp [path]"
+  opts.separator ''
+
+  opts.on("-r", "Recursive mode") do
+    options[:recursive] = true
+  end
+
+  opts.on("--verbose", "Verbose mode") do
+    options[:verbose] = true
+  end
+
+  opts.on("--debug", "Debug mode") do
+    options[:debug] = true
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts op
+    exit
+  end
+end
+op.parse!(ARGV)
+
+if ARGV.size < 2
+  puts op
+  exit
+end
+
+destination = ARGV.last
+sources = ARGV[0..-2]
+
+def s3?(url)
+  S3CP.bucket_and_key(url)[0]
+end
+
+if options[:debug]
+  puts "URL: #{url}"
+  puts "Options: \n#{options.inspect}"
+end
+
+@bucket = $1
+@prefix = $2
+
+@s3 = S3CP.connect()
+
+def direction(from, to)
+  if s3?(from) && s3?(to)
+    :s3_to_s3
+  elsif !s3?(from) && s3?(to)
+    :local_to_s3
+  elsif s3?(from) && !s3?(to)
+    :s3_to_local
+  else
+    :local_to_local
+  end
+end
+
+def no_slash(path)
+  path = path.match(/\/$/) ? path[0..-2] : path
+  path.match(/^\//) ? path[1..-1] : path
+end
+
+def relative(base, path)
+  no_slash(path[base.length..-1])
+end
+
+def copy(from, to, options)
+  bucket_from, key_from = S3CP.bucket_and_key(from)
+  bucket_to, key_to = S3CP.bucket_and_key(to)
+
+  #puts "bucket_from #{bucket_from.inspect} key_from #{key_from.inspect}"
+  #puts "bucket_to #{bucket_to.inspect} key_from #{key_to.inspect}"
+  #puts "direction #{direction(from, to)}"
+
+  case direction(from, to)
+  when :s3_to_s3
+    if options[:recursive]
+      keys = []
+      @s3.interface.incrementally_list_bucket(bucket_from, :prefix => key_from) do |page|
+        page[:contents].each { |entry| keys << entry[:key] }
+      end
+      keys.each do |key|
+        dest = no_slash(key_to) + '/' + relative(key_from, key)
+        puts "Copy s3://#{bucket_from}/#{key} to s3://#{bucket_to}/#{dest}"
+        @s3.interface.copy(bucket_from, key, bucket_to, dest)
+      end
+    else
+      puts "Copy s3://#{bucket_from}/#{key_from} to s3://#{bucket_to}/#{key_to}"
+      @s3.interface.copy(bucket_from, key_from, bucket_to, key_to)
+    end
+  when :local_to_s3
+    if options[:recursive]
+      files = Dir[from + "/**/*"]
+      files.each do |f|
+        f = File.expand_path(f)
+        key = no_slash(key_to) + '/' + relative(from, f)
+        puts "Copy #{f} to s3://#{bucket_to}/#{key}"
+        @s3.interface.put(bucket_to, key,  File.open(f))
+      end
+    else
+      f = File.expand_path(from)
+      puts "Copy #{f} to s3://#{bucket_to}/#{key_to}"
+      f = File.open(f)
+      begin
+        @s3.interface.put(bucket_to, key_to, f)
+      ensure
+        f.close()
+      end
+    end
+  when :s3_to_local
+    if options[:recursive]
+      keys = []
+      @s3.interface.incrementally_list_bucket(bucket_from, :prefix => key_from) do |page|
+        page[:contents].each { |entry| keys << entry[:key] }
+      end
+      keys.each do |key|
+        dest = File.expand_path(to) + '/' + relative(key_from, key)
+        dest = File.join(dest, File.basename(key)) if File.directory?(dest)
+        puts "Copy s3://#{bucket_from}/#{key} to #{dest}"
+        dir = File.dirname(dest)
+        FileUtils.mkdir_p dir unless File.exist? dir
+        fail "Destination path is not a directory: #{dir}" unless File.directory?(dir)
+        f = File.new(dest, File::CREAT|File::RDWR)
+        begin
+          @s3.interface.get(bucket_from, key) do |chunk|
+            f.write(chunk)
+          end
+        ensure
+          f.close()
+        end
+      end
+    else
+      dest = File.expand_path(to)
+      dest = File.join(dest, File.basename(key_from)) if File.directory?(dest)
+      puts "Copy s3://#{bucket_from}/#{key_from} to #{dest}"
+      f = File.new(dest, File::CREAT|File::RDWR)
+      begin
+        @s3.interface.get(bucket_from, key_from) do |chunk|
+          f.write(chunk)
+        end
+      ensure
+        f.close()
+      end
+    end
+  when :local_to_local
+    if options[:recursive]
+      FileUtils.cp_r from, to
+    else
+      FileUtils.cp from, to
+    end
+  end
+end
+
+sources.each do |source|
+  copy(source, destination, options)
+end
+

data/lib/s3cp/s3ls.rb

@@ -0,0 +1,84 @@
+require 'rubygems'
+require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
+require 'right_aws'
+require 'optparse'
+require 'date'
+require 'highline/import'
+
+require 's3cp/utils'
+
+# Parse arguments
+options = {}
+options[:date_format] = '%x %X'
+options[:rows_per_page] = $terminal.output_rows if $stdout.isatty
+
+op = OptionParser.new do |opts|
+  opts.banner = "s3ls [path]"
+  opts.separator ''
+
+  opts.on("-l", "Long listing format") do
+    options[:long_format] = true
+  end
+
+  opts.on("--date-format FORMAT", "Date format (see http://strfti.me/)") do |custom|
+    options[:custom_params] = custom
+  end
+
+  opts.on("--verbose", "Verbose mode") do
+    options[:verbose] = true
+  end
+
+  opts.on("--rows ROWS", "Rows per page") do |rows|
+    options[:rows_per_page] = rows.to_i
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts op
+    exit
+  end
+end
+op.parse!(ARGV)
+
+unless ARGV.size > 0
+  puts op
+  exit
+end
+
+url = ARGV[0]
+
+if options[:verbose]
+  puts "URL: #{url}"
+  puts "Options: #{options.inspect}"
+end
+
+@bucket, @key = S3CP.bucket_and_key(url)
+fail "Your URL looks funny, doesn't it?" unless @bucket
+
+if options[:verbose]
+  puts "bucket #{@bucket}"
+  puts "key #{@key}"
+end
+
+@s3 = S3CP.connect()
+
+rows = 0
+@s3.interface.incrementally_list_bucket(@bucket, :prefix => @key) do |page|
+  page[:contents].each do |entry|
+    key = entry[:key]
+    last_modified = DateTime.parse(entry[:last_modified])
+    if options[:long_format]
+      puts "#{last_modified.strftime(options[:date_format])} #{key}"
+    else
+      puts key
+    end
+    rows += 1
+    if options[:rows_per_page] && (rows % options[:rows_per_page] == 0)
+      begin
+        print "Continue? (Y/n) "
+        response = STDIN.gets.chomp.downcase
+      end until response == 'n' || response == 'y' || response == ''
+      exit if response == 'n'
+    end
+  end
+end
+

data/lib/s3cp/s3mod.rb

@@ -0,0 +1,35 @@
+require 'rubygems'  
+require 'right_aws'  
+require 'optparse'
+require 's3cp/utils'
+
+op = OptionParser.new do |opts|
+  opts.banner = "s3mod [path] [permission]"
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts op
+    exit
+  end
+end
+
+op.parse!(ARGV)
+
+if ARGV.size < 2
+  puts op
+  exit
+end
+
+def update_permissions(s3, bucket, key, permission)  
+  puts "Setting #{permission} on s3://#{bucket}/#{key}"
+  s3.interface.copy(bucket, key, bucket, key, :replace, {"x-amz-acl" => permission })  
+end  
+
+source  = ARGV[0]
+permission = ARGV.last
+
+LEGAL_MODS = %w{private authenticated-read public-read public-read-write}
+raise "Permissions must be one of the following values: #{LEGAL_MODS}" unless LEGAL_MODS.include?(permission)
+
+@s3 = S3CP.connect()
+bucket,key = S3CP.bucket_and_key(source)
+update_permissions(@s3, bucket, key, permission)  

data/lib/s3cp/utils.rb

@@ -0,0 +1,28 @@
+module S3CP
+  extend self
+
+  # Connect to AWS S3
+  def connect()
+    access_key = ENV["AWS_ACCESS_KEY_ID"]     || raise("Missing environment variable AWS_ACCESS_KEY_ID")
+    secret_key = ENV["AWS_SECRET_ACCESS_KEY"] || raise("Missing environment variable AWS_SECRET_ACCESS_KEY")
+
+    logger = Logger.new('/dev/null')
+    RightAws::S3.new(access_key, secret_key, :logger => logger)
+  end
+
+  # Parse URL and return bucket and key.
+  #
+  # e.g. s3://bucket/path/to/key => ["bucket", "path/to/key"]
+  #      bucket:path/to/key => ["bucket", "path/to/key"]
+  def bucket_and_key(url)
+    if url =~ /s3:\/\/([^\/]+)\/?(.*)/
+      bucket = $1
+      key = $2
+    elsif url =~ /([^:]+):(.*)/
+      bucket = $1
+      key = $2
+    end
+    [bucket, key]
+  end
+end
+

data/lib/s3cp/version.rb

@@ -0,0 +1,4 @@
+
+module S3CP
+  VERSION = "0.1.1"
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification 
+name: s3cp
+version: !ruby/object:Gem::Version 
+  hash: 25
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
+platform: ruby
+authors: 
+- Alex Boisvert
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-09-22 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 6
+        version: "0.6"
+  requirement: *id001
+  type: :runtime
+  name: extensions
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 5
+        - 1
+        version: 1.5.1
+  requirement: *id002
+  type: :runtime
+  name: highline
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+  requirement: *id003
+  type: :runtime
+  name: right_aws
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  requirement: *id004
+  type: :runtime
+  name: right_http_connection
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 2
+        - 5
+        - 0
+        version: 2.5.0
+  requirement: *id005
+  type: :development
+  name: rspec
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 0
+        - 8
+        - 7
+        version: 0.8.7
+  requirement: *id006
+  type: :development
+  name: rake
+  prerelease: false
+description: 
+email: 
+- alex.boisvert@gmail.com
+executables: 
+- s3ls
+- s3cp
+- s3cat
+- s3mod
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- History.txt
+files: 
+- lib/s3cp/s3ls.rb
+- lib/s3cp/s3mod.rb
+- lib/s3cp/s3cp.rb
+- lib/s3cp/version.rb
+- lib/s3cp/utils.rb
+- lib/s3cp/s3cat.rb
+- History.txt
+- README.md
+- bin/s3cp
+- bin/s3cat
+- bin/s3ls
+- bin/s3mod
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Amazon S3 tools to, e.g., list, copy, delete S3 files
+test_files: []
+