cprobert-s3sync 1.3.6

data/lib/s3sync/S3_s3sync_mod.rb

@@ -0,0 +1,143 @@
+#  This software code is made available "AS IS" without warranties of any
+#  kind.  You may copy, display, modify and redistribute the software
+#  code either by itself or as incorporated into your code; provided that
+#  you do not remove any proprietary notices.  Your use of this software
+#  code is at your own risk and you waive any claim against Amazon
+#  Digital Services, Inc. or its affiliates with respect to your use of
+#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
+#  affiliates.
+#  
+# This software code is made available "AS IS" without warranties of any        
+# kind.  You may copy, display, modify and redistribute the software            
+# code either by itself or as incorporated into your code; provided that        
+# you do not remove any proprietary notices.  Your use of this software         
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code. 
+# (c) 2007 s3sync.net
+#
+require 's3sync/S3'
+require 's3sync/HTTPStreaming'
+
+# The purpose of this file is to overlay the S3 library from AWS
+# to add some functionality
+# (without changing the file itself or requiring a specific version)
+# It still isn't perfectly robust, i.e. if radical changes are made
+# to the underlying lib this stuff will need updating.
+
+module S3
+	class AWSAuthConnection
+	
+      def make_http(bucket='', host='', proxy_host=nil, proxy_port=nil, proxy_user=nil, proxy_pass=nil)
+
+         # build the domain based on the calling format
+         server = ''
+         if host != ''
+           server = host           
+         elsif bucket.empty?
+           # for a bucketless request (i.e. list all buckets)
+           # revert to regular domain case since this operation
+           # does not make sense for vanity domains
+           server = @server
+         elsif @calling_format == CallingFormat::SUBDOMAIN
+           server = "#{bucket}.#{@server}" 
+         elsif @calling_format == CallingFormat::VANITY
+           server = bucket 
+         else
+           server = @server
+         end
+         # automatically does the right thing when no proxy
+         http = Net::HTTP::Proxy(proxy_host, proxy_port, proxy_user, proxy_pass).new(server, @port)
+         #http = Net::HTTP.new(server, @port)
+         http.use_ssl = @is_secure
+         http.verify_mode=@verify_mode
+         http.ca_file=@ca_file
+         http.ca_path=@ca_path
+         http.start
+         return http
+      end
+
+      # add support for streaming the response body to an IO stream
+      alias __make_request__ make_request
+      def make_request(method, bucket='', key='', path_args={}, headers={}, data='', metadata={}, streamOut=nil)
+         # build the path based on the calling format
+         path = ''
+         if (not bucket.empty?) and (@calling_format == CallingFormat::REGULAR)
+           path << "/#{bucket}"
+         end
+         # add the slash after the bucket regardless
+         # the key will be appended if it is non-empty
+         path << "/#{key}"
+
+         # build the path_argument string
+         # add the ? in all cases since 
+         # signature and credentials follow path args
+         path << '?'
+         path << S3.path_args_hash_to_string(path_args) 
+
+         req = method_to_request_class(method).new("#{path}")
+
+         set_headers(req, headers)
+         set_headers(req, metadata, METADATA_PREFIX)
+         set_headers(req, {'Connection' => 'keep-alive', 'Keep-Alive' => '300'})
+         
+         set_aws_auth_header(req, @aws_access_key_id, @aws_secret_access_key, bucket, key, path_args)
+         
+         http = $S3syncHttp
+            
+         if req.request_body_permitted?
+           return http.request(req, data, streamOut)
+         else
+           return http.request(req, nil, streamOut)
+         end
+      end
+
+		# a "get" operation that sends the body to an IO stream
+		def get_stream(bucket, key, headers={}, streamOut=nil)
+         return GetResponse.new(make_request('GET', bucket, CGI::escape(key), {}, headers, '', {}, streamOut))
+		end
+		
+		# a "get" operation that sends the body to an IO stream
+		def get_query_stream(bucket, key, path_args={}, headers={}, streamOut=nil)
+         return GetResponse.new(make_request('GET', bucket, CGI::escape(key), path_args, headers, '', {}, streamOut))
+		end
+      
+		def head(bucket, key=nil, headers={})
+         return GetResponse.new(make_request('HEAD', bucket, CGI::escape(key), {}, headers, '', {}))
+		end
+      undef create_bucket
+      def create_bucket(bucket, object)
+         object = S3Object.new(object) if not object.instance_of? S3Object
+         return Response.new(
+            make_request('PUT', bucket, '', {}, {}, object.data, object.metadata)
+         )
+      end
+      # no, because internally the library does not support the header,wait,body paradigm, so this is useless
+      #alias __put__ put
+      #def put(bucket, key, object, headers={})
+      #   headers['Expect'] = "100-continue"
+      #   __put__(bucket, key, object, headers)
+      #end
+
+               
+		# allow ssl validation
+      attr_accessor :verify_mode
+      attr_accessor :ca_path
+      attr_accessor :ca_file      
+
+	end
+   module CallingFormat
+      def CallingFormat.string_to_format(s)
+         case s
+         when 'REGULAR'
+           return CallingFormat::REGULAR
+         when 'SUBDOMAIN'
+           return CallingFormat::SUBDOMAIN
+         when 'VANITY'
+           return CallingFormat::VANITY
+         else
+           raise "Unsupported calling format #{s}"
+         end
+      end
+   end
+   
+end

data/lib/s3sync/S3encoder.rb

@@ -0,0 +1,50 @@
+# This software code is made available "AS IS" without warranties of any        
+# kind.  You may copy, display, modify and redistribute the software            
+# code either by itself or as incorporated into your code; provided that        
+# you do not remove any proprietary notices.  Your use of this software         
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code. 
+# (c) 2007 s3sync.net
+#
+
+# The purpose of this file is to overlay the cgi class
+# to add some functionality
+# (without changing the file itself or requiring a specific version)
+# It still isn't perfectly robust, i.e. if radical changes are made
+# to the underlying lib this stuff will need updating.
+
+require 'cgi'
+require 'iconv' # for UTF-8 conversion
+
+# thanks to http://www.redhillconsulting.com.au/blogs/simon/archives/000326.html
+module S3ExtendCGI
+	def self.included(base)
+		base.extend(ClassMethods)
+		base.class_eval do
+			class << self
+				alias_method :S3Extend_escape_orig, :escape unless method_defined?(:S3Extend_escape_orig)
+				alias_method :escape, :S3Extend_escape
+			end
+		end
+	end
+	module ClassMethods
+		@@exemptSlashesInEscape = false
+		attr_writer :exemptSlashesInEscape
+		@@usePercent20InEscape = false
+		attr_writer :usePercent20InEscape
+		@@nativeCharacterEncoding = "ISO-8859-1"
+		attr_writer :nativeCharacterEncoding
+		@@useUTF8InEscape = false
+		attr_writer :useUTF8InEscape
+		
+		def S3Extend_escape(string)
+			result = string
+			result = Iconv.iconv("UTF-8", @nativeCharacterEncoding, string).join if @useUTF8InEscape 
+			result = S3Extend_escape_orig(result)
+			result.gsub!(/%2f/i, "/") if @exemptSlashesInEscape
+			result.gsub!("+", "%20") if @usePercent20InEscape
+			result
+		end
+	end
+end
+CGI.send(:include, S3ExtendCGI)

data/lib/s3sync/s3config.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+# This software code is made available "AS IS" without warranties of any        
+# kind.  You may copy, display, modify and redistribute the software            
+# code either by itself or as incorporated into your code; provided that        
+# you do not remove any proprietary notices.  Your use of this software         
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code. 
+# (c) 2007 alastair brunton
+#
+# modified to search out the yaml in several places, thanks wkharold.
+require 'yaml'
+
+module S3Config
+  
+  confpath = ["#{ENV['S3CONF']}", "#{ENV['HOME']}/.s3conf", "/etc/s3conf"]
+  
+  confpath.each do |path|
+    if File.exists?(path) and File.directory?(path) and File.exists?("#{path}/s3config.yml")
+      config = YAML.load_file("#{path}/s3config.yml")
+      config.each_pair do |key, value|
+        eval("$#{key.upcase} = '#{value}'")
+      end
+      break
+    end
+  end
+
+   confpath.each do |path|
+    if File.exists?(path) and File.directory?(path) and File.exists?("#{path}/_s3config.yml")
+      config = YAML.load_file("#{path}/_s3config.yml")
+      config.each_pair do |key, value|
+        eval("$#{key.upcase} = '#{value}'")
+      end
+      break
+    end
+  end
+    
+end

data/lib/s3sync/s3try.rb

@@ -0,0 +1,178 @@
+#!/usr/bin/env ruby 
+# This software code is made available "AS IS" without warranties of any        
+# kind.  You may copy, display, modify and redistribute the software            
+# code either by itself or as incorporated into your code; provided that        
+# you do not remove any proprietary notices.  Your use of this software         
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code. 
+# (c) 2007 s3sync.net
+#
+module S3sync
+
+  $AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
+  $AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
+  $AWS_S3_HOST = (ENV["AWS_S3_HOST"] or "s3.amazonaws.com")
+  $HTTP_PROXY_HOST = ENV["HTTP_PROXY_HOST"]
+  $HTTP_PROXY_PORT = ENV["HTTP_PROXY_PORT"]
+  $HTTP_PROXY_USER = ENV["HTTP_PROXY_USER"]
+  $HTTP_PROXY_PASSWORD = ENV["HTTP_PROXY_PASSWORD"]
+  $SSL_CERT_DIR = ENV["SSL_CERT_DIR"]
+  $SSL_CERT_FILE = ENV["SSL_CERT_FILE"]
+  $S3SYNC_RETRIES = (ENV["S3SYNC_RETRIES"] or 100).to_i # number of errors to tolerate 
+  $S3SYNC_WAITONERROR = (ENV["S3SYNC_WAITONERROR"] or 5).to_i # seconds
+  $S3SYNC_NATIVE_CHARSET = (ENV["S3SYNC_NATIVE_CHARSET"] or "ISO-8859-1")
+  $AWS_CALLING_FORMAT = (ENV["AWS_CALLING_FORMAT"] or "REGULAR")
+
+  require 's3sync/S3'
+  require 's3sync/HTTPStreaming'
+  require 's3sync/S3encoder'
+
+  CGI::exemptSlashesInEscape = true
+  CGI::usePercent20InEscape = true
+  CGI::useUTF8InEscape = true
+  CGI::nativeCharacterEncoding = $S3SYNC_NATIVE_CHARSET
+  require 's3sync/S3_s3sync_mod'
+
+
+  $S3syncRetriesLeft = $S3SYNC_RETRIES.to_i
+
+
+  def S3sync.s3trySetup
+
+    # ---------- CONNECT ---------- #
+
+    $S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
+    $S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
+    if $S3syncOptions['--ssl']
+      if $SSL_CERT_DIR
+        $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        $S3syncConnection.ca_path = $SSL_CERT_DIR
+      elsif $SSL_CERT_FILE
+        $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        $S3syncConnection.ca_file = $SSL_CERT_FILE
+      end
+    end
+  end
+
+  def S3sync.s3urlSetup
+    $S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
+    $S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
+    $S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
+  end
+
+  def S3sync.S3tryConnect(bucket, host='')
+    $S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
+  end
+
+  def S3sync.S3try(command, bucket, *args)
+    if(not $S3syncHttp or (bucket != $S3syncLastBucket))
+      $stderr.puts "Creating new connection" if $S3syncOptions['--debug']
+      $S3syncLastBucket = bucket
+      while $S3syncRetriesLeft > 0 do
+        begin
+          S3sync.S3tryConnect(bucket)
+          break
+        rescue Errno::ECONNRESET => e
+          $stderr.puts "Connection reset: #{e}"
+        rescue Errno::ECONNABORTED => e
+          $stderr.puts "Connection aborted: #{e}"
+        rescue Errno::ETIMEDOUT => e
+          $stderr.puts "Connection timed out: #{e}"
+        rescue Timeout::Error => e
+          $stderr.puts "Connection timed out: #{e}"
+        end
+        $S3syncRetriesLeft -= 1
+        $stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds"
+        Kernel.sleep $S3SYNC_WAITONERROR.to_i
+      end
+    end
+
+    result = nil
+    delim = $,
+    $,=' '
+    while $S3syncRetriesLeft > 0 do
+      $stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
+      forceRetry = false
+      now = false
+      hush = false
+      begin
+        result = $S3syncConnection.send(command, bucket, *args)
+      rescue Errno::EPIPE => e
+        forceRetry = true
+        $stderr.puts "Broken pipe: #{e}" 
+      rescue Errno::ECONNRESET => e
+        forceRetry = true
+        $stderr.puts "Connection reset: #{e}" 
+      rescue Errno::ECONNABORTED => e
+        forceRetry = true
+        $stderr.puts "Connection aborted: #{e}" 
+      rescue Errno::ETIMEDOUT => e
+        forceRetry = true
+        $stderr.puts "Connection timed out: #{e}"
+      rescue Timeout::Error => e
+        forceRetry = true
+        $stderr.puts "Connection timed out: #{e}" 
+      rescue EOFError => e
+        # i THINK this is happening like a connection reset
+        forceRetry = true
+        $stderr.puts "EOF error: #{e}"
+      rescue OpenSSL::SSL::SSLError => e
+        forceRetry = true
+        $stderr.puts "SSL Error: #{e}"
+      rescue NoMethodError => e
+        # we get this when using --progress, and the local item is something unreadable
+        $stderr.puts "Null stream error: #{e}"
+        break
+      end
+      if forceRetry
+        # kill and reset connection when we receive a non-50x yet retry-able error
+        S3sync.S3tryConnect(bucket)
+      end
+      begin
+        debug("Response code: #{result.http_response.code}")
+        break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
+        if result.http_response.code.to_i == 301
+          $stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
+        elsif result.http_response.code.to_i == 307
+          # move to the other host
+          host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
+          $stderr.puts("Temporary Redirect to: " + host)
+          debug("Host: " + host)
+          S3sync.S3tryConnect(bucket, host)
+          $S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below
+          forceRetry = true
+          now = true
+          hush = true
+        else
+          $stderr.puts "S3 command failed:\n#{command} #{args}"
+          $stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
+          debug(result.http_response.body)
+        end
+        # only retry 500's, per amazon
+        break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
+      rescue NoMethodError
+        debug("No result available")
+      end
+      $S3syncRetriesLeft -= 1
+      $stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds" unless hush
+      Kernel.sleep $S3SYNC_WAITONERROR.to_i unless now
+    end
+    if $S3syncRetriesLeft <= 0
+      $stderr.puts "Ran out of retries; operations did not complete!"
+    end
+    $, = delim
+    result
+  end
+
+  def S3sync.S3url(command, bucket, *args)
+    S3sync.s3urlSetup() unless $S3syncGenerator
+    result = nil
+    delim = $,
+    $,=' '
+    $stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
+    result = $S3syncGenerator.send(command, bucket, *args)
+    $, = delim
+    result
+  end
+end #module
+

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/s3sync.rb'}"
+puts "Loading s3sync gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)