mgreenly-s3sync 1.2.4

data/lib/s3sync/s3try.rb

@@ -0,0 +1,161 @@
+#!/usr/bin/env ruby 
+# This software code is made available "AS IS" without warranties of any        
+# kind.  You may copy, display, modify and redistribute the software            
+# code either by itself or as incorporated into your code; provided that        
+# you do not remove any proprietary notices.  Your use of this software         
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code. 
+# (c) 2007 s3sync.net
+#
+module S3sync
+
+	$AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]           
+	$AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]   
+	$AWS_S3_HOST = (ENV["AWS_S3_HOST"] or "s3.amazonaws.com")
+   $HTTP_PROXY_HOST = ENV["HTTP_PROXY_HOST"]
+   $HTTP_PROXY_PORT = ENV["HTTP_PROXY_PORT"]
+   $HTTP_PROXY_USER = ENV["HTTP_PROXY_USER"]
+   $HTTP_PROXY_PASSWORD = ENV["HTTP_PROXY_PASSWORD"]
+	$SSL_CERT_DIR = ENV["SSL_CERT_DIR"]
+	$SSL_CERT_FILE = ENV["SSL_CERT_FILE"]
+	$S3SYNC_RETRIES = (ENV["S3SYNC_RETRIES"] or 100).to_i # number of errors to tolerate 
+	$S3SYNC_WAITONERROR = (ENV["S3SYNC_WAITONERROR"] or 30).to_i # seconds
+	$S3SYNC_NATIVE_CHARSET = (ENV["S3SYNC_NATIVE_CHARSET"] or "ISO-8859-1")
+	$AWS_CALLING_FORMAT = (ENV["AWS_CALLING_FORMAT"] or "REGULAR")
+   
+	require 'S3'
+
+	require 'HTTPStreaming'
+	require 'S3encoder'
+	CGI::exemptSlashesInEscape = true
+	CGI::usePercent20InEscape = true
+	CGI::useUTF8InEscape = true
+	CGI::nativeCharacterEncoding = $S3SYNC_NATIVE_CHARSET
+	require 'S3_s3sync_mod'
+
+
+	$S3syncRetriesLeft = $S3SYNC_RETRIES.to_i
+	
+	def S3sync.s3trySetup 	
+		
+		# ---------- CONNECT ---------- #
+
+		$S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
+      $S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
+		if $S3syncOptions['--ssl']
+			if $SSL_CERT_DIR
+				$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+				$S3syncConnection.ca_path = $SSL_CERT_DIR
+			elsif $SSL_CERT_FILE
+				$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+				$S3syncConnection.ca_file = $SSL_CERT_FILE
+			end
+		end
+	end
+	def S3sync.s3urlSetup 	
+		$S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
+      $S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
+      $S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
+	end
+   
+   def S3sync.S3tryConnect(bucket, host='')
+         $S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
+   end
+	
+	def S3sync.S3try(command, bucket, *args)
+      if(not $S3syncHttp or (bucket != $S3syncLastBucket))
+         $stderr.puts "Creating new connection" if $S3syncOptions['--debug']
+         $S3syncLastBucket = bucket
+         S3sync.S3tryConnect(bucket)
+      end
+      
+		result = nil
+		delim = $,
+		$,=' '
+		while $S3syncRetriesLeft > 0 do
+         $stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
+			forceRetry = false
+         now = false
+         hush = false
+			begin
+				result = $S3syncConnection.send(command, bucket, *args)
+			rescue Errno::EPIPE => e
+				forceRetry = true
+				$stderr.puts "Broken pipe: #{e}" 
+			rescue Errno::ECONNRESET => e
+				forceRetry = true
+				$stderr.puts "Connection reset: #{e}" 
+			rescue Errno::ECONNABORTED => e
+				forceRetry = true
+				$stderr.puts "Connection aborted: #{e}" 
+			rescue Errno::ETIMEDOUT => e
+				forceRetry = true
+				$stderr.puts "Connection timed out: #{e}"
+			rescue Timeout::Error => e
+				forceRetry = true
+				$stderr.puts "Connection timed out: #{e}" 
+			rescue EOFError => e
+				# i THINK this is happening like a connection reset
+				forceRetry = true
+				$stderr.puts "EOF error: #{e}"
+			rescue OpenSSL::SSL::SSLError => e
+				forceRetry = true
+				$stderr.puts "SSL Error: #{e}"
+			rescue NoMethodError
+				# we get this when using --progress, and the local item is something unreadable
+				$stderr.puts "Null stream error: #{e}"
+				break
+			end
+         if forceRetry
+				# kill and reset connection when we receive a non-50x yet retry-able error
+				S3sync.S3tryConnect(bucket)
+         end
+			begin
+				debug("Response code: #{result.http_response.code}")
+				break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
+            if result.http_response.code.to_i == 301
+               $stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
+            elsif result.http_response.code.to_i == 307
+               # move to the other host
+               host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
+               $stderr.puts("Temporary Redirect to: " + host)
+               debug("Host: " + host)
+               S3sync.S3tryConnect(bucket, host)
+               $S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below
+               forceRetry = true
+               now = true
+               hush = true
+            else
+               $stderr.puts "S3 command failed:\n#{command} #{args}"
+               $stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
+               debug(result.http_response.body)
+            end
+				# only retry 500's, per amazon
+				break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
+			rescue NoMethodError
+				debug("No result available")
+			end
+			$S3syncRetriesLeft -= 1
+			$stderr.puts "#{$S3syncRetriesLeft} retries left" unless hush
+			Kernel.sleep $S3SYNC_WAITONERROR unless now
+		end
+      if $S3syncRetriesLeft <= 0
+         $stderr.puts "Ran out of retries; operations did not complete!"
+      end
+		$, = delim
+		result
+	end
+   
+	def S3sync.S3url(command, bucket, *args)
+      S3sync.s3urlSetup() unless $S3syncGenerator
+		result = nil
+		delim = $,
+		$,=' '
+      $stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
+      result = $S3syncGenerator.send(command, bucket, *args)
+		$, = delim
+		result
+	end
+	
+end #module
+

data/lib/s3sync/thread_generator.rb

@@ -0,0 +1,383 @@
+#!/usr/bin/env ruby
+#--
+# $Idaemons: /home/cvs/rb/generator.rb,v 1.8 2001/10/03 08:54:32 knu Exp $
+# $RoughId: generator.rb,v 1.10 2003/10/14 19:36:58 knu Exp $
+# $Id: generator.rb,v 1.12 2005/12/31 02:56:46 ocean Exp $
+#++
+#
+# = generator.rb: convert an internal iterator to an external one
+#
+# Copyright (c) 2001,2003 Akinori MUSHA <knu@iDaemons.org>
+#
+# All rights reserved.  You can redistribute and/or modify it under
+# the same terms as Ruby.
+#
+# == Overview
+#
+# This library provides the Generator class, which converts an
+# internal iterator (i.e. an Enumerable object) to an external
+# iterator.  In that form, you can roll many iterators independently.
+#
+# The SyncEnumerator class, which is implemented using Generator,
+# makes it easy to roll many Enumerable objects synchronously.
+#
+# See the respective classes for examples of usage.
+
+
+#
+# Generator converts an internal iterator (i.e. an Enumerable object)
+# to an external iterator.
+#
+# == Example
+#
+#   require 'generator'
+#
+#   # Generator from an Enumerable object
+#   g = Generator.new(['A', 'B', 'C', 'Z'])
+#
+#   while g.next?
+#     puts g.next
+#   end
+#
+#   # Generator from a block
+#   g = Generator.new { |g|
+#     for i in 'A'..'C'
+#       g.yield i
+#     end
+#
+#     g.yield 'Z'
+#   }
+#
+#   # The same result as above
+#   while g.next?
+#     puts g.next
+#   end
+#   
+class Generator
+  include Enumerable
+
+  # Creates a new generator either from an Enumerable object or from a
+  # block.
+  #
+  # In the former, block is ignored even if given.
+  #
+  # In the latter, the given block is called with the generator
+  # itself, and expected to call the +yield+ method for each element.
+  def initialize(enum = nil, &block)
+    if enum
+      @block = proc{|g| enum.each{|value| g.yield value}}
+    else
+      @block = block
+    end
+    @index = 0
+    @queue = []
+    @main_thread = nil
+    @loop_thread.kill if defined?(@loop_thread)
+    @loop_thread = Thread.new do
+      Thread.stop
+      begin
+        @block.call(self)
+      rescue
+        @main_thread.raise $!
+      ensure
+        @main_thread.wakeup
+      end
+    end
+    Thread.pass until @loop_thread.stop?
+    self
+  end
+
+  # Yields an element to the generator.
+  def yield(value)
+    if Thread.current != @loop_thread
+      raise "should be called in Generator.new{|g| ... }"
+    end
+    Thread.critical = true
+    begin
+      @queue << value
+      @main_thread.wakeup
+      Thread.stop
+    ensure
+      Thread.critical = false
+    end
+    self
+  end
+
+  # Returns true if the generator has reached the end.
+  def end?
+    if @queue.empty?
+      if @main_thread
+        raise "should not be called in Generator.new{|g| ... }"
+      end
+      Thread.critical = true
+      begin
+        @main_thread = Thread.current
+        @loop_thread.wakeup
+        Thread.stop
+      rescue ThreadError
+        # ignore
+      ensure
+        @main_thread = nil
+        Thread.critical = false
+      end
+    end
+    @queue.empty?
+  end
+
+  # Returns true if the generator has not reached the end yet.
+  def next?
+    !end?
+  end
+
+  # Returns the current index (position) counting from zero.
+  def index
+    @index
+  end
+
+  # Returns the current index (position) counting from zero.
+  def pos
+    @index
+  end
+
+  # Returns the element at the current position and moves forward.
+  def next
+    raise EOFError.new("no more elements available") if end?
+    @index += 1
+    @queue.shift
+  end
+
+  # Returns the element at the current position.
+  def current
+    raise EOFError.new("no more elements available") if end?
+    @queue.first
+  end
+
+  # Rewinds the generator.
+  def rewind
+    initialize(nil, &@block) if @index.nonzero?
+    self
+  end
+
+  # Rewinds the generator and enumerates the elements.
+  def each
+    rewind
+    until end?
+      yield self.next
+    end
+    self
+  end
+end
+
+#
+# SyncEnumerator creates an Enumerable object from multiple Enumerable
+# objects and enumerates them synchronously.
+#
+# == Example
+#
+#   require 'generator'
+#
+#   s = SyncEnumerator.new([1,2,3], ['a', 'b', 'c'])
+#
+#   # Yields [1, 'a'], [2, 'b'], and [3,'c']
+#   s.each { |row| puts row.join(', ') }
+#
+class SyncEnumerator
+  include Enumerable
+
+  # Creates a new SyncEnumerator which enumerates rows of given
+  # Enumerable objects.
+  def initialize(*enums)
+    @gens = enums.map { |e| Generator.new(e) }
+  end
+
+  # Returns the number of enumerated Enumerable objects, i.e. the size
+  # of each row.
+  def size
+    @gens.size
+  end
+
+  # Returns the number of enumerated Enumerable objects, i.e. the size
+  # of each row.
+  def length
+    @gens.length
+  end
+
+  # Returns true if the given nth Enumerable object has reached the
+  # end.  If no argument is given, returns true if any of the
+  # Enumerable objects has reached the end.
+  def end?(i = nil)
+    if i.nil?
+      @gens.detect { |g| g.end? } ? true : false
+    else
+      @gens[i].end?
+    end
+  end
+
+  # Enumerates rows of the Enumerable objects.
+  def each
+    @gens.each { |g| g.rewind }
+
+    loop do
+      count = 0
+
+      ret = @gens.map { |g|
+	if g.end?
+	  count += 1
+	  nil
+	else
+	  g.next
+	end
+      }
+
+      if count == @gens.size
+	break
+      end
+
+      yield ret
+    end
+
+    self
+  end
+end
+
+if $0 == __FILE__
+  eval DATA.read, nil, $0, __LINE__+4
+end
+
+__END__
+
+require 'test/unit'
+
+class TC_Generator < Test::Unit::TestCase
+  def test_block1
+    g = Generator.new { |g|
+      # no yield's
+    }
+
+    assert_equal(0, g.pos)
+    assert_raises(EOFError) { g.current }
+  end
+
+  def test_block2
+    g = Generator.new { |g|
+      for i in 'A'..'C'
+        g.yield i
+      end
+
+      g.yield 'Z'
+    }
+
+    assert_equal(0, g.pos)
+    assert_equal('A', g.current)
+
+    assert_equal(true, g.next?)
+    assert_equal(0, g.pos)
+    assert_equal('A', g.current)
+    assert_equal(0, g.pos)
+    assert_equal('A', g.next)
+
+    assert_equal(1, g.pos)
+    assert_equal(true, g.next?)
+    assert_equal(1, g.pos)
+    assert_equal('B', g.current)
+    assert_equal(1, g.pos)
+    assert_equal('B', g.next)
+
+    assert_equal(g, g.rewind)
+
+    assert_equal(0, g.pos)
+    assert_equal('A', g.current)
+
+    assert_equal(true, g.next?)
+    assert_equal(0, g.pos)
+    assert_equal('A', g.current)
+    assert_equal(0, g.pos)
+    assert_equal('A', g.next)
+
+    assert_equal(1, g.pos)
+    assert_equal(true, g.next?)
+    assert_equal(1, g.pos)
+    assert_equal('B', g.current)
+    assert_equal(1, g.pos)
+    assert_equal('B', g.next)
+
+    assert_equal(2, g.pos)
+    assert_equal(true, g.next?)
+    assert_equal(2, g.pos)
+    assert_equal('C', g.current)
+    assert_equal(2, g.pos)
+    assert_equal('C', g.next)
+
+    assert_equal(3, g.pos)
+    assert_equal(true, g.next?)
+    assert_equal(3, g.pos)
+    assert_equal('Z', g.current)
+    assert_equal(3, g.pos)
+    assert_equal('Z', g.next)
+
+    assert_equal(4, g.pos)
+    assert_equal(false, g.next?)
+    assert_raises(EOFError) { g.next }
+  end
+
+  def test_each
+    a = [5, 6, 7, 8, 9]
+
+    g = Generator.new(a)
+
+    i = 0
+
+    g.each { |x|
+      assert_equal(a[i], x)
+
+      i += 1
+
+      break if i == 3
+    }
+
+    assert_equal(3, i)
+
+    i = 0
+
+    g.each { |x|
+      assert_equal(a[i], x)
+
+      i += 1
+    }
+
+    assert_equal(5, i)
+  end
+end
+
+class TC_SyncEnumerator < Test::Unit::TestCase
+  def test_each
+    r = ['a'..'f', 1..10, 10..20]
+    ra = r.map { |x| x.to_a }
+
+    a = (0...(ra.map {|x| x.size}.max)).map { |i| ra.map { |x| x[i] } }
+
+    s = SyncEnumerator.new(*r)
+
+    i = 0
+
+    s.each { |x|
+      assert_equal(a[i], x)
+
+      i += 1
+
+      break if i == 3
+    }
+
+    assert_equal(3, i)
+
+    i = 0
+
+    s.each { |x|
+      assert_equal(a[i], x)
+
+      i += 1
+    }
+
+    assert_equal(a.size, i)
+  end
+end