cprobert-s3sync 1.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,143 @@
1
+ # This software code is made available "AS IS" without warranties of any
2
+ # kind. You may copy, display, modify and redistribute the software
3
+ # code either by itself or as incorporated into your code; provided that
4
+ # you do not remove any proprietary notices. Your use of this software
5
+ # code is at your own risk and you waive any claim against Amazon
6
+ # Digital Services, Inc. or its affiliates with respect to your use of
7
+ # this software code. (c) 2006 Amazon Digital Services, Inc. or its
8
+ # affiliates.
9
+ #
10
+ # This software code is made available "AS IS" without warranties of any
11
+ # kind. You may copy, display, modify and redistribute the software
12
+ # code either by itself or as incorporated into your code; provided that
13
+ # you do not remove any proprietary notices. Your use of this software
14
+ # code is at your own risk and you waive any claim against the author
15
+ # with respect to your use of this software code.
16
+ # (c) 2007 s3sync.net
17
+ #
18
+ require 's3sync/S3'
19
+ require 's3sync/HTTPStreaming'
20
+
21
+ # The purpose of this file is to overlay the S3 library from AWS
22
+ # to add some functionality
23
+ # (without changing the file itself or requiring a specific version)
24
+ # It still isn't perfectly robust, i.e. if radical changes are made
25
+ # to the underlying lib this stuff will need updating.
26
+
27
+ module S3
28
+ class AWSAuthConnection
29
+
30
+ def make_http(bucket='', host='', proxy_host=nil, proxy_port=nil, proxy_user=nil, proxy_pass=nil)
31
+
32
+ # build the domain based on the calling format
33
+ server = ''
34
+ if host != ''
35
+ server = host
36
+ elsif bucket.empty?
37
+ # for a bucketless request (i.e. list all buckets)
38
+ # revert to regular domain case since this operation
39
+ # does not make sense for vanity domains
40
+ server = @server
41
+ elsif @calling_format == CallingFormat::SUBDOMAIN
42
+ server = "#{bucket}.#{@server}"
43
+ elsif @calling_format == CallingFormat::VANITY
44
+ server = bucket
45
+ else
46
+ server = @server
47
+ end
48
+ # automatically does the right thing when no proxy
49
+ http = Net::HTTP::Proxy(proxy_host, proxy_port, proxy_user, proxy_pass).new(server, @port)
50
+ #http = Net::HTTP.new(server, @port)
51
+ http.use_ssl = @is_secure
52
+ http.verify_mode=@verify_mode
53
+ http.ca_file=@ca_file
54
+ http.ca_path=@ca_path
55
+ http.start
56
+ return http
57
+ end
58
+
59
+ # add support for streaming the response body to an IO stream
60
+ alias __make_request__ make_request
61
+ def make_request(method, bucket='', key='', path_args={}, headers={}, data='', metadata={}, streamOut=nil)
62
+ # build the path based on the calling format
63
+ path = ''
64
+ if (not bucket.empty?) and (@calling_format == CallingFormat::REGULAR)
65
+ path << "/#{bucket}"
66
+ end
67
+ # add the slash after the bucket regardless
68
+ # the key will be appended if it is non-empty
69
+ path << "/#{key}"
70
+
71
+ # build the path_argument string
72
+ # add the ? in all cases since
73
+ # signature and credentials follow path args
74
+ path << '?'
75
+ path << S3.path_args_hash_to_string(path_args)
76
+
77
+ req = method_to_request_class(method).new("#{path}")
78
+
79
+ set_headers(req, headers)
80
+ set_headers(req, metadata, METADATA_PREFIX)
81
+ set_headers(req, {'Connection' => 'keep-alive', 'Keep-Alive' => '300'})
82
+
83
+ set_aws_auth_header(req, @aws_access_key_id, @aws_secret_access_key, bucket, key, path_args)
84
+
85
+ http = $S3syncHttp
86
+
87
+ if req.request_body_permitted?
88
+ return http.request(req, data, streamOut)
89
+ else
90
+ return http.request(req, nil, streamOut)
91
+ end
92
+ end
93
+
94
+ # a "get" operation that sends the body to an IO stream
95
+ def get_stream(bucket, key, headers={}, streamOut=nil)
96
+ return GetResponse.new(make_request('GET', bucket, CGI::escape(key), {}, headers, '', {}, streamOut))
97
+ end
98
+
99
+ # a "get" operation that sends the body to an IO stream
100
+ def get_query_stream(bucket, key, path_args={}, headers={}, streamOut=nil)
101
+ return GetResponse.new(make_request('GET', bucket, CGI::escape(key), path_args, headers, '', {}, streamOut))
102
+ end
103
+
104
+ def head(bucket, key=nil, headers={})
105
+ return GetResponse.new(make_request('HEAD', bucket, CGI::escape(key), {}, headers, '', {}))
106
+ end
107
+ undef create_bucket
108
+ def create_bucket(bucket, object)
109
+ object = S3Object.new(object) if not object.instance_of? S3Object
110
+ return Response.new(
111
+ make_request('PUT', bucket, '', {}, {}, object.data, object.metadata)
112
+ )
113
+ end
114
+ # no, because internally the library does not support the header,wait,body paradigm, so this is useless
115
+ #alias __put__ put
116
+ #def put(bucket, key, object, headers={})
117
+ # headers['Expect'] = "100-continue"
118
+ # __put__(bucket, key, object, headers)
119
+ #end
120
+
121
+
122
+ # allow ssl validation
123
+ attr_accessor :verify_mode
124
+ attr_accessor :ca_path
125
+ attr_accessor :ca_file
126
+
127
+ end
128
+ module CallingFormat
129
+ def CallingFormat.string_to_format(s)
130
+ case s
131
+ when 'REGULAR'
132
+ return CallingFormat::REGULAR
133
+ when 'SUBDOMAIN'
134
+ return CallingFormat::SUBDOMAIN
135
+ when 'VANITY'
136
+ return CallingFormat::VANITY
137
+ else
138
+ raise "Unsupported calling format #{s}"
139
+ end
140
+ end
141
+ end
142
+
143
+ end
@@ -0,0 +1,50 @@
1
+ # This software code is made available "AS IS" without warranties of any
2
+ # kind. You may copy, display, modify and redistribute the software
3
+ # code either by itself or as incorporated into your code; provided that
4
+ # you do not remove any proprietary notices. Your use of this software
5
+ # code is at your own risk and you waive any claim against the author
6
+ # with respect to your use of this software code.
7
+ # (c) 2007 s3sync.net
8
+ #
9
+
10
+ # The purpose of this file is to overlay the cgi class
11
+ # to add some functionality
12
+ # (without changing the file itself or requiring a specific version)
13
+ # It still isn't perfectly robust, i.e. if radical changes are made
14
+ # to the underlying lib this stuff will need updating.
15
+
16
+ require 'cgi'
17
+ require 'iconv' # for UTF-8 conversion
18
+
19
+ # thanks to http://www.redhillconsulting.com.au/blogs/simon/archives/000326.html
20
+ module S3ExtendCGI
21
+ def self.included(base)
22
+ base.extend(ClassMethods)
23
+ base.class_eval do
24
+ class << self
25
+ alias_method :S3Extend_escape_orig, :escape unless method_defined?(:S3Extend_escape_orig)
26
+ alias_method :escape, :S3Extend_escape
27
+ end
28
+ end
29
+ end
30
+ module ClassMethods
31
+ @@exemptSlashesInEscape = false
32
+ attr_writer :exemptSlashesInEscape
33
+ @@usePercent20InEscape = false
34
+ attr_writer :usePercent20InEscape
35
+ @@nativeCharacterEncoding = "ISO-8859-1"
36
+ attr_writer :nativeCharacterEncoding
37
+ @@useUTF8InEscape = false
38
+ attr_writer :useUTF8InEscape
39
+
40
+ def S3Extend_escape(string)
41
+ result = string
42
+ result = Iconv.iconv("UTF-8", @nativeCharacterEncoding, string).join if @useUTF8InEscape
43
+ result = S3Extend_escape_orig(result)
44
+ result.gsub!(/%2f/i, "/") if @exemptSlashesInEscape
45
+ result.gsub!("+", "%20") if @usePercent20InEscape
46
+ result
47
+ end
48
+ end
49
+ end
50
+ CGI.send(:include, S3ExtendCGI)
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/ruby
2
+ # This software code is made available "AS IS" without warranties of any
3
+ # kind. You may copy, display, modify and redistribute the software
4
+ # code either by itself or as incorporated into your code; provided that
5
+ # you do not remove any proprietary notices. Your use of this software
6
+ # code is at your own risk and you waive any claim against the author
7
+ # with respect to your use of this software code.
8
+ # (c) 2007 alastair brunton
9
+ #
10
+ # modified to search out the yaml in several places, thanks wkharold.
11
+ require 'yaml'
12
+
13
+ module S3Config
14
+
15
+ confpath = ["#{ENV['S3CONF']}", "#{ENV['HOME']}/.s3conf", "/etc/s3conf"]
16
+
17
+ confpath.each do |path|
18
+ if File.exists?(path) and File.directory?(path) and File.exists?("#{path}/s3config.yml")
19
+ config = YAML.load_file("#{path}/s3config.yml")
20
+ config.each_pair do |key, value|
21
+ eval("$#{key.upcase} = '#{value}'")
22
+ end
23
+ break
24
+ end
25
+ end
26
+
27
+ confpath.each do |path|
28
+ if File.exists?(path) and File.directory?(path) and File.exists?("#{path}/_s3config.yml")
29
+ config = YAML.load_file("#{path}/_s3config.yml")
30
+ config.each_pair do |key, value|
31
+ eval("$#{key.upcase} = '#{value}'")
32
+ end
33
+ break
34
+ end
35
+ end
36
+
37
+ end
@@ -0,0 +1,178 @@
1
+ #!/usr/bin/env ruby
2
+ # This software code is made available "AS IS" without warranties of any
3
+ # kind. You may copy, display, modify and redistribute the software
4
+ # code either by itself or as incorporated into your code; provided that
5
+ # you do not remove any proprietary notices. Your use of this software
6
+ # code is at your own risk and you waive any claim against the author
7
+ # with respect to your use of this software code.
8
+ # (c) 2007 s3sync.net
9
+ #
10
+ module S3sync
11
+
12
+ $AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
13
+ $AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
14
+ $AWS_S3_HOST = (ENV["AWS_S3_HOST"] or "s3.amazonaws.com")
15
+ $HTTP_PROXY_HOST = ENV["HTTP_PROXY_HOST"]
16
+ $HTTP_PROXY_PORT = ENV["HTTP_PROXY_PORT"]
17
+ $HTTP_PROXY_USER = ENV["HTTP_PROXY_USER"]
18
+ $HTTP_PROXY_PASSWORD = ENV["HTTP_PROXY_PASSWORD"]
19
+ $SSL_CERT_DIR = ENV["SSL_CERT_DIR"]
20
+ $SSL_CERT_FILE = ENV["SSL_CERT_FILE"]
21
+ $S3SYNC_RETRIES = (ENV["S3SYNC_RETRIES"] or 100).to_i # number of errors to tolerate
22
+ $S3SYNC_WAITONERROR = (ENV["S3SYNC_WAITONERROR"] or 5).to_i # seconds
23
+ $S3SYNC_NATIVE_CHARSET = (ENV["S3SYNC_NATIVE_CHARSET"] or "ISO-8859-1")
24
+ $AWS_CALLING_FORMAT = (ENV["AWS_CALLING_FORMAT"] or "REGULAR")
25
+
26
+ require 's3sync/S3'
27
+ require 's3sync/HTTPStreaming'
28
+ require 's3sync/S3encoder'
29
+
30
+ CGI::exemptSlashesInEscape = true
31
+ CGI::usePercent20InEscape = true
32
+ CGI::useUTF8InEscape = true
33
+ CGI::nativeCharacterEncoding = $S3SYNC_NATIVE_CHARSET
34
+ require 's3sync/S3_s3sync_mod'
35
+
36
+
37
+ $S3syncRetriesLeft = $S3SYNC_RETRIES.to_i
38
+
39
+
40
+ def S3sync.s3trySetup
41
+
42
+ # ---------- CONNECT ---------- #
43
+
44
+ $S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
45
+ $S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
46
+ if $S3syncOptions['--ssl']
47
+ if $SSL_CERT_DIR
48
+ $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
49
+ $S3syncConnection.ca_path = $SSL_CERT_DIR
50
+ elsif $SSL_CERT_FILE
51
+ $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
52
+ $S3syncConnection.ca_file = $SSL_CERT_FILE
53
+ end
54
+ end
55
+ end
56
+
57
+ def S3sync.s3urlSetup
58
+ $S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
59
+ $S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
60
+ $S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
61
+ end
62
+
63
+ def S3sync.S3tryConnect(bucket, host='')
64
+ $S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
65
+ end
66
+
67
+ def S3sync.S3try(command, bucket, *args)
68
+ if(not $S3syncHttp or (bucket != $S3syncLastBucket))
69
+ $stderr.puts "Creating new connection" if $S3syncOptions['--debug']
70
+ $S3syncLastBucket = bucket
71
+ while $S3syncRetriesLeft > 0 do
72
+ begin
73
+ S3sync.S3tryConnect(bucket)
74
+ break
75
+ rescue Errno::ECONNRESET => e
76
+ $stderr.puts "Connection reset: #{e}"
77
+ rescue Errno::ECONNABORTED => e
78
+ $stderr.puts "Connection aborted: #{e}"
79
+ rescue Errno::ETIMEDOUT => e
80
+ $stderr.puts "Connection timed out: #{e}"
81
+ rescue Timeout::Error => e
82
+ $stderr.puts "Connection timed out: #{e}"
83
+ end
84
+ $S3syncRetriesLeft -= 1
85
+ $stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds"
86
+ Kernel.sleep $S3SYNC_WAITONERROR.to_i
87
+ end
88
+ end
89
+
90
+ result = nil
91
+ delim = $,
92
+ $,=' '
93
+ while $S3syncRetriesLeft > 0 do
94
+ $stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
95
+ forceRetry = false
96
+ now = false
97
+ hush = false
98
+ begin
99
+ result = $S3syncConnection.send(command, bucket, *args)
100
+ rescue Errno::EPIPE => e
101
+ forceRetry = true
102
+ $stderr.puts "Broken pipe: #{e}"
103
+ rescue Errno::ECONNRESET => e
104
+ forceRetry = true
105
+ $stderr.puts "Connection reset: #{e}"
106
+ rescue Errno::ECONNABORTED => e
107
+ forceRetry = true
108
+ $stderr.puts "Connection aborted: #{e}"
109
+ rescue Errno::ETIMEDOUT => e
110
+ forceRetry = true
111
+ $stderr.puts "Connection timed out: #{e}"
112
+ rescue Timeout::Error => e
113
+ forceRetry = true
114
+ $stderr.puts "Connection timed out: #{e}"
115
+ rescue EOFError => e
116
+ # i THINK this is happening like a connection reset
117
+ forceRetry = true
118
+ $stderr.puts "EOF error: #{e}"
119
+ rescue OpenSSL::SSL::SSLError => e
120
+ forceRetry = true
121
+ $stderr.puts "SSL Error: #{e}"
122
+ rescue NoMethodError => e
123
+ # we get this when using --progress, and the local item is something unreadable
124
+ $stderr.puts "Null stream error: #{e}"
125
+ break
126
+ end
127
+ if forceRetry
128
+ # kill and reset connection when we receive a non-50x yet retry-able error
129
+ S3sync.S3tryConnect(bucket)
130
+ end
131
+ begin
132
+ debug("Response code: #{result.http_response.code}")
133
+ break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
134
+ if result.http_response.code.to_i == 301
135
+ $stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
136
+ elsif result.http_response.code.to_i == 307
137
+ # move to the other host
138
+ host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
139
+ $stderr.puts("Temporary Redirect to: " + host)
140
+ debug("Host: " + host)
141
+ S3sync.S3tryConnect(bucket, host)
142
+ $S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below
143
+ forceRetry = true
144
+ now = true
145
+ hush = true
146
+ else
147
+ $stderr.puts "S3 command failed:\n#{command} #{args}"
148
+ $stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
149
+ debug(result.http_response.body)
150
+ end
151
+ # only retry 500's, per amazon
152
+ break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
153
+ rescue NoMethodError
154
+ debug("No result available")
155
+ end
156
+ $S3syncRetriesLeft -= 1
157
+ $stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds" unless hush
158
+ Kernel.sleep $S3SYNC_WAITONERROR.to_i unless now
159
+ end
160
+ if $S3syncRetriesLeft <= 0
161
+ $stderr.puts "Ran out of retries; operations did not complete!"
162
+ end
163
+ $, = delim
164
+ result
165
+ end
166
+
167
+ def S3sync.S3url(command, bucket, *args)
168
+ S3sync.s3urlSetup() unless $S3syncGenerator
169
+ result = nil
170
+ delim = $,
171
+ $,=' '
172
+ $stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
173
+ result = $S3syncGenerator.send(command, bucket, *args)
174
+ $, = delim
175
+ result
176
+ end
177
+ end #module
178
+
data/script/console ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+ # File: script/console
3
+ irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
4
+
5
+ libs = " -r irb/completion"
6
+ # Perhaps use a console_lib to store any extra methods I may want available in the cosole
7
+ # libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
8
+ libs << " -r #{File.dirname(__FILE__) + '/../lib/s3sync.rb'}"
9
+ puts "Loading s3sync gem"
10
+ exec "#{irb} #{libs} --simple-prompt"
data/script/destroy ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+ APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
3
+
4
+ begin
5
+ require 'rubigen'
6
+ rescue LoadError
7
+ require 'rubygems'
8
+ require 'rubigen'
9
+ end
10
+ require 'rubigen/scripts/destroy'
11
+
12
+ ARGV.shift if ['--help', '-h'].include?(ARGV[0])
13
+ RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
14
+ RubiGen::Scripts::Destroy.new.run(ARGV)
data/script/generate ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+ APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
3
+
4
+ begin
5
+ require 'rubigen'
6
+ rescue LoadError
7
+ require 'rubygems'
8
+ require 'rubigen'
9
+ end
10
+ require 'rubigen/scripts/generate'
11
+
12
+ ARGV.shift if ['--help', '-h'].include?(ARGV[0])
13
+ RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
14
+ RubiGen::Scripts::Generate.new.run(ARGV)