s3sync 1.2.5 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/s3sync +67 -726
- data/lib/s3sync.rb +2 -0
- data/lib/s3sync/cli.rb +475 -0
- data/lib/s3sync/config.rb +98 -0
- data/lib/s3sync/exceptions.rb +55 -0
- data/lib/s3sync/sync.rb +371 -0
- data/lib/s3sync/util.rb +29 -0
- data/lib/s3sync/version.rb +27 -0
- metadata +177 -54
- data/CHANGELOG +0 -175
- data/README +0 -401
- data/README_s3cmd +0 -172
- data/Rakefile +0 -35
- data/bin/s3cmd +0 -245
- data/lib/HTTPStreaming.rb +0 -103
- data/lib/S3.rb +0 -707
- data/lib/S3_s3sync_mod.rb +0 -143
- data/lib/S3encoder.rb +0 -50
- data/lib/s3config.rb +0 -27
- data/lib/s3try.rb +0 -161
- data/lib/thread_generator.rb +0 -383
- data/lib/version.rb +0 -9
- data/setup.rb +0 -1585
data/lib/S3_s3sync_mod.rb
DELETED
@@ -1,143 +0,0 @@
|
|
1
|
-
# This software code is made available "AS IS" without warranties of any
|
2
|
-
# kind. You may copy, display, modify and redistribute the software
|
3
|
-
# code either by itself or as incorporated into your code; provided that
|
4
|
-
# you do not remove any proprietary notices. Your use of this software
|
5
|
-
# code is at your own risk and you waive any claim against Amazon
|
6
|
-
# Digital Services, Inc. or its affiliates with respect to your use of
|
7
|
-
# this software code. (c) 2006 Amazon Digital Services, Inc. or its
|
8
|
-
# affiliates.
|
9
|
-
#
|
10
|
-
# This software code is made available "AS IS" without warranties of any
|
11
|
-
# kind. You may copy, display, modify and redistribute the software
|
12
|
-
# code either by itself or as incorporated into your code; provided that
|
13
|
-
# you do not remove any proprietary notices. Your use of this software
|
14
|
-
# code is at your own risk and you waive any claim against the author
|
15
|
-
# with respect to your use of this software code.
|
16
|
-
# (c) 2007 s3sync.net
|
17
|
-
#
|
18
|
-
require 'S3'
|
19
|
-
require 'HTTPStreaming'
|
20
|
-
|
21
|
-
# The purpose of this file is to overlay the S3 library from AWS
|
22
|
-
# to add some functionality
|
23
|
-
# (without changing the file itself or requiring a specific version)
|
24
|
-
# It still isn't perfectly robust, i.e. if radical changes are made
|
25
|
-
# to the underlying lib this stuff will need updating.
|
26
|
-
|
27
|
-
module S3
|
28
|
-
class AWSAuthConnection
|
29
|
-
|
30
|
-
def make_http(bucket='', host='', proxy_host=nil, proxy_port=nil, proxy_user=nil, proxy_pass=nil)
|
31
|
-
|
32
|
-
# build the domain based on the calling format
|
33
|
-
server = ''
|
34
|
-
if host != ''
|
35
|
-
server = host
|
36
|
-
elsif bucket.empty?
|
37
|
-
# for a bucketless request (i.e. list all buckets)
|
38
|
-
# revert to regular domain case since this operation
|
39
|
-
# does not make sense for vanity domains
|
40
|
-
server = @server
|
41
|
-
elsif @calling_format == CallingFormat::SUBDOMAIN
|
42
|
-
server = "#{bucket}.#{@server}"
|
43
|
-
elsif @calling_format == CallingFormat::VANITY
|
44
|
-
server = bucket
|
45
|
-
else
|
46
|
-
server = @server
|
47
|
-
end
|
48
|
-
# automatically does the right thing when no proxy
|
49
|
-
http = Net::HTTP::Proxy(proxy_host, proxy_port, proxy_user, proxy_pass).new(server, @port)
|
50
|
-
#http = Net::HTTP.new(server, @port)
|
51
|
-
http.use_ssl = @is_secure
|
52
|
-
http.verify_mode=@verify_mode
|
53
|
-
http.ca_file=@ca_file
|
54
|
-
http.ca_path=@ca_path
|
55
|
-
http.start
|
56
|
-
return http
|
57
|
-
end
|
58
|
-
|
59
|
-
# add support for streaming the response body to an IO stream
|
60
|
-
alias __make_request__ make_request
|
61
|
-
def make_request(method, bucket='', key='', path_args={}, headers={}, data='', metadata={}, streamOut=nil)
|
62
|
-
# build the path based on the calling format
|
63
|
-
path = ''
|
64
|
-
if (not bucket.empty?) and (@calling_format == CallingFormat::REGULAR)
|
65
|
-
path << "/#{bucket}"
|
66
|
-
end
|
67
|
-
# add the slash after the bucket regardless
|
68
|
-
# the key will be appended if it is non-empty
|
69
|
-
path << "/#{key}"
|
70
|
-
|
71
|
-
# build the path_argument string
|
72
|
-
# add the ? in all cases since
|
73
|
-
# signature and credentials follow path args
|
74
|
-
path << '?'
|
75
|
-
path << S3.path_args_hash_to_string(path_args)
|
76
|
-
|
77
|
-
req = method_to_request_class(method).new("#{path}")
|
78
|
-
|
79
|
-
set_headers(req, headers)
|
80
|
-
set_headers(req, metadata, METADATA_PREFIX)
|
81
|
-
set_headers(req, {'Connection' => 'keep-alive', 'Keep-Alive' => '300'})
|
82
|
-
|
83
|
-
set_aws_auth_header(req, @aws_access_key_id, @aws_secret_access_key, bucket, key, path_args)
|
84
|
-
|
85
|
-
http = $S3syncHttp
|
86
|
-
|
87
|
-
if req.request_body_permitted?
|
88
|
-
return http.request(req, data, streamOut)
|
89
|
-
else
|
90
|
-
return http.request(req, nil, streamOut)
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
# a "get" operation that sends the body to an IO stream
|
95
|
-
def get_stream(bucket, key, headers={}, streamOut=nil)
|
96
|
-
return GetResponse.new(make_request('GET', bucket, CGI::escape(key), {}, headers, '', {}, streamOut))
|
97
|
-
end
|
98
|
-
|
99
|
-
# a "get" operation that sends the body to an IO stream
|
100
|
-
def get_query_stream(bucket, key, path_args={}, headers={}, streamOut=nil)
|
101
|
-
return GetResponse.new(make_request('GET', bucket, CGI::escape(key), path_args, headers, '', {}, streamOut))
|
102
|
-
end
|
103
|
-
|
104
|
-
def head(bucket, key=nil, headers={})
|
105
|
-
return GetResponse.new(make_request('HEAD', bucket, CGI::escape(key), {}, headers, '', {}))
|
106
|
-
end
|
107
|
-
undef create_bucket
|
108
|
-
def create_bucket(bucket, object)
|
109
|
-
object = S3Object.new(object) if not object.instance_of? S3Object
|
110
|
-
return Response.new(
|
111
|
-
make_request('PUT', bucket, '', {}, {}, object.data, object.metadata)
|
112
|
-
)
|
113
|
-
end
|
114
|
-
# no, because internally the library does not support the header,wait,body paradigm, so this is useless
|
115
|
-
#alias __put__ put
|
116
|
-
#def put(bucket, key, object, headers={})
|
117
|
-
# headers['Expect'] = "100-continue"
|
118
|
-
# __put__(bucket, key, object, headers)
|
119
|
-
#end
|
120
|
-
|
121
|
-
|
122
|
-
# allow ssl validation
|
123
|
-
attr_accessor :verify_mode
|
124
|
-
attr_accessor :ca_path
|
125
|
-
attr_accessor :ca_file
|
126
|
-
|
127
|
-
end
|
128
|
-
module CallingFormat
|
129
|
-
def CallingFormat.string_to_format(s)
|
130
|
-
case s
|
131
|
-
when 'REGULAR'
|
132
|
-
return CallingFormat::REGULAR
|
133
|
-
when 'SUBDOMAIN'
|
134
|
-
return CallingFormat::SUBDOMAIN
|
135
|
-
when 'VANITY'
|
136
|
-
return CallingFormat::VANITY
|
137
|
-
else
|
138
|
-
raise "Unsupported calling format #{s}"
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
|
143
|
-
end
|
data/lib/S3encoder.rb
DELETED
@@ -1,50 +0,0 @@
|
|
1
|
-
# This software code is made available "AS IS" without warranties of any
|
2
|
-
# kind. You may copy, display, modify and redistribute the software
|
3
|
-
# code either by itself or as incorporated into your code; provided that
|
4
|
-
# you do not remove any proprietary notices. Your use of this software
|
5
|
-
# code is at your own risk and you waive any claim against the author
|
6
|
-
# with respect to your use of this software code.
|
7
|
-
# (c) 2007 s3sync.net
|
8
|
-
#
|
9
|
-
|
10
|
-
# The purpose of this file is to overlay the cgi class
|
11
|
-
# to add some functionality
|
12
|
-
# (without changing the file itself or requiring a specific version)
|
13
|
-
# It still isn't perfectly robust, i.e. if radical changes are made
|
14
|
-
# to the underlying lib this stuff will need updating.
|
15
|
-
|
16
|
-
require 'cgi'
|
17
|
-
require 'iconv' # for UTF-8 conversion
|
18
|
-
|
19
|
-
# thanks to http://www.redhillconsulting.com.au/blogs/simon/archives/000326.html
|
20
|
-
module S3ExtendCGI
|
21
|
-
def self.included(base)
|
22
|
-
base.extend(ClassMethods)
|
23
|
-
base.class_eval do
|
24
|
-
class << self
|
25
|
-
alias_method :S3Extend_escape_orig, :escape unless method_defined?(:S3Extend_escape_orig)
|
26
|
-
alias_method :escape, :S3Extend_escape
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
module ClassMethods
|
31
|
-
@@exemptSlashesInEscape = false
|
32
|
-
attr_writer :exemptSlashesInEscape
|
33
|
-
@@usePercent20InEscape = false
|
34
|
-
attr_writer :usePercent20InEscape
|
35
|
-
@@nativeCharacterEncoding = "ISO-8859-1"
|
36
|
-
attr_writer :nativeCharacterEncoding
|
37
|
-
@@useUTF8InEscape = false
|
38
|
-
attr_writer :useUTF8InEscape
|
39
|
-
|
40
|
-
def S3Extend_escape(string)
|
41
|
-
result = string
|
42
|
-
result = Iconv.iconv("UTF-8", @nativeCharacterEncoding, string).join if @useUTF8InEscape
|
43
|
-
result = S3Extend_escape_orig(result)
|
44
|
-
result.gsub!(/%2f/i, "/") if @exemptSlashesInEscape
|
45
|
-
result.gsub!("+", "%20") if @usePercent20InEscape
|
46
|
-
result
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
CGI.send(:include, S3ExtendCGI)
|
data/lib/s3config.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
#!/usr/bin/ruby
|
2
|
-
# This software code is made available "AS IS" without warranties of any
|
3
|
-
# kind. You may copy, display, modify and redistribute the software
|
4
|
-
# code either by itself or as incorporated into your code; provided that
|
5
|
-
# you do not remove any proprietary notices. Your use of this software
|
6
|
-
# code is at your own risk and you waive any claim against the author
|
7
|
-
# with respect to your use of this software code.
|
8
|
-
# (c) 2007 alastair brunton
|
9
|
-
#
|
10
|
-
# modified to search out the yaml in several places, thanks wkharold.
|
11
|
-
require 'yaml'
|
12
|
-
|
13
|
-
module S3Config
|
14
|
-
|
15
|
-
confpath = ["#{ENV['S3CONF']}", "#{ENV['HOME']}/.s3conf", "/etc/s3conf"]
|
16
|
-
|
17
|
-
confpath.each do |path|
|
18
|
-
if File.exists?(path) and File.directory?(path) and File.exists?("#{path}/s3config.yml")
|
19
|
-
config = YAML.load_file("#{path}/s3config.yml")
|
20
|
-
config.each_pair do |key, value|
|
21
|
-
eval("$#{key.upcase} = '#{value}'")
|
22
|
-
end
|
23
|
-
break
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
data/lib/s3try.rb
DELETED
@@ -1,161 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
# This software code is made available "AS IS" without warranties of any
|
3
|
-
# kind. You may copy, display, modify and redistribute the software
|
4
|
-
# code either by itself or as incorporated into your code; provided that
|
5
|
-
# you do not remove any proprietary notices. Your use of this software
|
6
|
-
# code is at your own risk and you waive any claim against the author
|
7
|
-
# with respect to your use of this software code.
|
8
|
-
# (c) 2007 s3sync.net
|
9
|
-
#
|
10
|
-
module S3sync
|
11
|
-
|
12
|
-
$AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
|
13
|
-
$AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
|
14
|
-
$AWS_S3_HOST = (ENV["AWS_S3_HOST"] or "s3.amazonaws.com")
|
15
|
-
$HTTP_PROXY_HOST = ENV["HTTP_PROXY_HOST"]
|
16
|
-
$HTTP_PROXY_PORT = ENV["HTTP_PROXY_PORT"]
|
17
|
-
$HTTP_PROXY_USER = ENV["HTTP_PROXY_USER"]
|
18
|
-
$HTTP_PROXY_PASSWORD = ENV["HTTP_PROXY_PASSWORD"]
|
19
|
-
$SSL_CERT_DIR = ENV["SSL_CERT_DIR"]
|
20
|
-
$SSL_CERT_FILE = ENV["SSL_CERT_FILE"]
|
21
|
-
$S3SYNC_RETRIES = (ENV["S3SYNC_RETRIES"] or 100).to_i # number of errors to tolerate
|
22
|
-
$S3SYNC_WAITONERROR = (ENV["S3SYNC_WAITONERROR"] or 30).to_i # seconds
|
23
|
-
$S3SYNC_NATIVE_CHARSET = (ENV["S3SYNC_NATIVE_CHARSET"] or "ISO-8859-1")
|
24
|
-
$AWS_CALLING_FORMAT = (ENV["AWS_CALLING_FORMAT"] or "REGULAR")
|
25
|
-
|
26
|
-
require 'S3'
|
27
|
-
|
28
|
-
require 'HTTPStreaming'
|
29
|
-
require 'S3encoder'
|
30
|
-
CGI::exemptSlashesInEscape = true
|
31
|
-
CGI::usePercent20InEscape = true
|
32
|
-
CGI::useUTF8InEscape = true
|
33
|
-
CGI::nativeCharacterEncoding = $S3SYNC_NATIVE_CHARSET
|
34
|
-
require 'S3_s3sync_mod'
|
35
|
-
|
36
|
-
|
37
|
-
$S3syncRetriesLeft = $S3SYNC_RETRIES.to_i
|
38
|
-
|
39
|
-
def S3sync.s3trySetup
|
40
|
-
|
41
|
-
# ---------- CONNECT ---------- #
|
42
|
-
|
43
|
-
$S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
|
44
|
-
$S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
|
45
|
-
if $S3syncOptions['--ssl']
|
46
|
-
if $SSL_CERT_DIR
|
47
|
-
$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
48
|
-
$S3syncConnection.ca_path = $SSL_CERT_DIR
|
49
|
-
elsif $SSL_CERT_FILE
|
50
|
-
$S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
51
|
-
$S3syncConnection.ca_file = $SSL_CERT_FILE
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
def S3sync.s3urlSetup
|
56
|
-
$S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
|
57
|
-
$S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
|
58
|
-
$S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
|
59
|
-
end
|
60
|
-
|
61
|
-
def S3sync.S3tryConnect(bucket, host='')
|
62
|
-
$S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
|
63
|
-
end
|
64
|
-
|
65
|
-
def S3sync.S3try(command, bucket, *args)
|
66
|
-
if(not $S3syncHttp or (bucket != $S3syncLastBucket))
|
67
|
-
$stderr.puts "Creating new connection" if $S3syncOptions['--debug']
|
68
|
-
$S3syncLastBucket = bucket
|
69
|
-
S3sync.S3tryConnect(bucket)
|
70
|
-
end
|
71
|
-
|
72
|
-
result = nil
|
73
|
-
delim = $,
|
74
|
-
$,=' '
|
75
|
-
while $S3syncRetriesLeft > 0 do
|
76
|
-
$stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
|
77
|
-
forceRetry = false
|
78
|
-
now = false
|
79
|
-
hush = false
|
80
|
-
begin
|
81
|
-
result = $S3syncConnection.send(command, bucket, *args)
|
82
|
-
rescue Errno::EPIPE => e
|
83
|
-
forceRetry = true
|
84
|
-
$stderr.puts "Broken pipe: #{e}"
|
85
|
-
rescue Errno::ECONNRESET => e
|
86
|
-
forceRetry = true
|
87
|
-
$stderr.puts "Connection reset: #{e}"
|
88
|
-
rescue Errno::ECONNABORTED => e
|
89
|
-
forceRetry = true
|
90
|
-
$stderr.puts "Connection aborted: #{e}"
|
91
|
-
rescue Errno::ETIMEDOUT => e
|
92
|
-
forceRetry = true
|
93
|
-
$stderr.puts "Connection timed out: #{e}"
|
94
|
-
rescue Timeout::Error => e
|
95
|
-
forceRetry = true
|
96
|
-
$stderr.puts "Connection timed out: #{e}"
|
97
|
-
rescue EOFError => e
|
98
|
-
# i THINK this is happening like a connection reset
|
99
|
-
forceRetry = true
|
100
|
-
$stderr.puts "EOF error: #{e}"
|
101
|
-
rescue OpenSSL::SSL::SSLError => e
|
102
|
-
forceRetry = true
|
103
|
-
$stderr.puts "SSL Error: #{e}"
|
104
|
-
rescue NoMethodError => e
|
105
|
-
# we get this when using --progress, and the local item is something unreadable
|
106
|
-
$stderr.puts "Null stream error: #{e}"
|
107
|
-
break
|
108
|
-
end
|
109
|
-
if forceRetry
|
110
|
-
# kill and reset connection when we receive a non-50x yet retry-able error
|
111
|
-
S3sync.S3tryConnect(bucket)
|
112
|
-
end
|
113
|
-
begin
|
114
|
-
debug("Response code: #{result.http_response.code}")
|
115
|
-
break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
|
116
|
-
if result.http_response.code.to_i == 301
|
117
|
-
$stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
|
118
|
-
elsif result.http_response.code.to_i == 307
|
119
|
-
# move to the other host
|
120
|
-
host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
|
121
|
-
$stderr.puts("Temporary Redirect to: " + host)
|
122
|
-
debug("Host: " + host)
|
123
|
-
S3sync.S3tryConnect(bucket, host)
|
124
|
-
$S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below
|
125
|
-
forceRetry = true
|
126
|
-
now = true
|
127
|
-
hush = true
|
128
|
-
else
|
129
|
-
$stderr.puts "S3 command failed:\n#{command} #{args}"
|
130
|
-
$stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
|
131
|
-
debug(result.http_response.body)
|
132
|
-
end
|
133
|
-
# only retry 500's, per amazon
|
134
|
-
break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
|
135
|
-
rescue NoMethodError
|
136
|
-
debug("No result available")
|
137
|
-
end
|
138
|
-
$S3syncRetriesLeft -= 1
|
139
|
-
$stderr.puts "#{$S3syncRetriesLeft} retries left, sleeping for #{$S3SYNC_WAITONERROR} seconds" unless hush
|
140
|
-
Kernel.sleep $S3SYNC_WAITONERROR.to_i unless now
|
141
|
-
end
|
142
|
-
if $S3syncRetriesLeft <= 0
|
143
|
-
$stderr.puts "Ran out of retries; operations did not complete!"
|
144
|
-
end
|
145
|
-
$, = delim
|
146
|
-
result
|
147
|
-
end
|
148
|
-
|
149
|
-
def S3sync.S3url(command, bucket, *args)
|
150
|
-
S3sync.s3urlSetup() unless $S3syncGenerator
|
151
|
-
result = nil
|
152
|
-
delim = $,
|
153
|
-
$,=' '
|
154
|
-
$stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
|
155
|
-
result = $S3syncGenerator.send(command, bucket, *args)
|
156
|
-
$, = delim
|
157
|
-
result
|
158
|
-
end
|
159
|
-
|
160
|
-
end #module
|
161
|
-
|
data/lib/thread_generator.rb
DELETED
@@ -1,383 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
#--
|
3
|
-
# $Idaemons: /home/cvs/rb/generator.rb,v 1.8 2001/10/03 08:54:32 knu Exp $
|
4
|
-
# $RoughId: generator.rb,v 1.10 2003/10/14 19:36:58 knu Exp $
|
5
|
-
# $Id: generator.rb,v 1.12 2005/12/31 02:56:46 ocean Exp $
|
6
|
-
#++
|
7
|
-
#
|
8
|
-
# = generator.rb: convert an internal iterator to an external one
|
9
|
-
#
|
10
|
-
# Copyright (c) 2001,2003 Akinori MUSHA <knu@iDaemons.org>
|
11
|
-
#
|
12
|
-
# All rights reserved. You can redistribute and/or modify it under
|
13
|
-
# the same terms as Ruby.
|
14
|
-
#
|
15
|
-
# == Overview
|
16
|
-
#
|
17
|
-
# This library provides the Generator class, which converts an
|
18
|
-
# internal iterator (i.e. an Enumerable object) to an external
|
19
|
-
# iterator. In that form, you can roll many iterators independently.
|
20
|
-
#
|
21
|
-
# The SyncEnumerator class, which is implemented using Generator,
|
22
|
-
# makes it easy to roll many Enumerable objects synchronously.
|
23
|
-
#
|
24
|
-
# See the respective classes for examples of usage.
|
25
|
-
|
26
|
-
|
27
|
-
#
|
28
|
-
# Generator converts an internal iterator (i.e. an Enumerable object)
|
29
|
-
# to an external iterator.
|
30
|
-
#
|
31
|
-
# == Example
|
32
|
-
#
|
33
|
-
# require 'generator'
|
34
|
-
#
|
35
|
-
# # Generator from an Enumerable object
|
36
|
-
# g = Generator.new(['A', 'B', 'C', 'Z'])
|
37
|
-
#
|
38
|
-
# while g.next?
|
39
|
-
# puts g.next
|
40
|
-
# end
|
41
|
-
#
|
42
|
-
# # Generator from a block
|
43
|
-
# g = Generator.new { |g|
|
44
|
-
# for i in 'A'..'C'
|
45
|
-
# g.yield i
|
46
|
-
# end
|
47
|
-
#
|
48
|
-
# g.yield 'Z'
|
49
|
-
# }
|
50
|
-
#
|
51
|
-
# # The same result as above
|
52
|
-
# while g.next?
|
53
|
-
# puts g.next
|
54
|
-
# end
|
55
|
-
#
|
56
|
-
class Generator
|
57
|
-
include Enumerable
|
58
|
-
|
59
|
-
# Creates a new generator either from an Enumerable object or from a
|
60
|
-
# block.
|
61
|
-
#
|
62
|
-
# In the former, block is ignored even if given.
|
63
|
-
#
|
64
|
-
# In the latter, the given block is called with the generator
|
65
|
-
# itself, and expected to call the +yield+ method for each element.
|
66
|
-
def initialize(enum = nil, &block)
|
67
|
-
if enum
|
68
|
-
@block = proc{|g| enum.each{|value| g.yield value}}
|
69
|
-
else
|
70
|
-
@block = block
|
71
|
-
end
|
72
|
-
@index = 0
|
73
|
-
@queue = []
|
74
|
-
@main_thread = nil
|
75
|
-
@loop_thread.kill if defined?(@loop_thread)
|
76
|
-
@loop_thread = Thread.new do
|
77
|
-
Thread.stop
|
78
|
-
begin
|
79
|
-
@block.call(self)
|
80
|
-
rescue
|
81
|
-
@main_thread.raise $!
|
82
|
-
ensure
|
83
|
-
@main_thread.wakeup
|
84
|
-
end
|
85
|
-
end
|
86
|
-
Thread.pass until @loop_thread.stop?
|
87
|
-
self
|
88
|
-
end
|
89
|
-
|
90
|
-
# Yields an element to the generator.
|
91
|
-
def yield(value)
|
92
|
-
if Thread.current != @loop_thread
|
93
|
-
raise "should be called in Generator.new{|g| ... }"
|
94
|
-
end
|
95
|
-
Thread.critical = true
|
96
|
-
begin
|
97
|
-
@queue << value
|
98
|
-
@main_thread.wakeup
|
99
|
-
Thread.stop
|
100
|
-
ensure
|
101
|
-
Thread.critical = false
|
102
|
-
end
|
103
|
-
self
|
104
|
-
end
|
105
|
-
|
106
|
-
# Returns true if the generator has reached the end.
|
107
|
-
def end?
|
108
|
-
if @queue.empty?
|
109
|
-
if @main_thread
|
110
|
-
raise "should not be called in Generator.new{|g| ... }"
|
111
|
-
end
|
112
|
-
Thread.critical = true
|
113
|
-
begin
|
114
|
-
@main_thread = Thread.current
|
115
|
-
@loop_thread.wakeup
|
116
|
-
Thread.stop
|
117
|
-
rescue ThreadError
|
118
|
-
# ignore
|
119
|
-
ensure
|
120
|
-
@main_thread = nil
|
121
|
-
Thread.critical = false
|
122
|
-
end
|
123
|
-
end
|
124
|
-
@queue.empty?
|
125
|
-
end
|
126
|
-
|
127
|
-
# Returns true if the generator has not reached the end yet.
|
128
|
-
def next?
|
129
|
-
!end?
|
130
|
-
end
|
131
|
-
|
132
|
-
# Returns the current index (position) counting from zero.
|
133
|
-
def index
|
134
|
-
@index
|
135
|
-
end
|
136
|
-
|
137
|
-
# Returns the current index (position) counting from zero.
|
138
|
-
def pos
|
139
|
-
@index
|
140
|
-
end
|
141
|
-
|
142
|
-
# Returns the element at the current position and moves forward.
|
143
|
-
def next
|
144
|
-
raise EOFError.new("no more elements available") if end?
|
145
|
-
@index += 1
|
146
|
-
@queue.shift
|
147
|
-
end
|
148
|
-
|
149
|
-
# Returns the element at the current position.
|
150
|
-
def current
|
151
|
-
raise EOFError.new("no more elements available") if end?
|
152
|
-
@queue.first
|
153
|
-
end
|
154
|
-
|
155
|
-
# Rewinds the generator.
|
156
|
-
def rewind
|
157
|
-
initialize(nil, &@block) if @index.nonzero?
|
158
|
-
self
|
159
|
-
end
|
160
|
-
|
161
|
-
# Rewinds the generator and enumerates the elements.
|
162
|
-
def each
|
163
|
-
rewind
|
164
|
-
until end?
|
165
|
-
yield self.next
|
166
|
-
end
|
167
|
-
self
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
#
|
172
|
-
# SyncEnumerator creates an Enumerable object from multiple Enumerable
|
173
|
-
# objects and enumerates them synchronously.
|
174
|
-
#
|
175
|
-
# == Example
|
176
|
-
#
|
177
|
-
# require 'generator'
|
178
|
-
#
|
179
|
-
# s = SyncEnumerator.new([1,2,3], ['a', 'b', 'c'])
|
180
|
-
#
|
181
|
-
# # Yields [1, 'a'], [2, 'b'], and [3,'c']
|
182
|
-
# s.each { |row| puts row.join(', ') }
|
183
|
-
#
|
184
|
-
class SyncEnumerator
|
185
|
-
include Enumerable
|
186
|
-
|
187
|
-
# Creates a new SyncEnumerator which enumerates rows of given
|
188
|
-
# Enumerable objects.
|
189
|
-
def initialize(*enums)
|
190
|
-
@gens = enums.map { |e| Generator.new(e) }
|
191
|
-
end
|
192
|
-
|
193
|
-
# Returns the number of enumerated Enumerable objects, i.e. the size
|
194
|
-
# of each row.
|
195
|
-
def size
|
196
|
-
@gens.size
|
197
|
-
end
|
198
|
-
|
199
|
-
# Returns the number of enumerated Enumerable objects, i.e. the size
|
200
|
-
# of each row.
|
201
|
-
def length
|
202
|
-
@gens.length
|
203
|
-
end
|
204
|
-
|
205
|
-
# Returns true if the given nth Enumerable object has reached the
|
206
|
-
# end. If no argument is given, returns true if any of the
|
207
|
-
# Enumerable objects has reached the end.
|
208
|
-
def end?(i = nil)
|
209
|
-
if i.nil?
|
210
|
-
@gens.detect { |g| g.end? } ? true : false
|
211
|
-
else
|
212
|
-
@gens[i].end?
|
213
|
-
end
|
214
|
-
end
|
215
|
-
|
216
|
-
# Enumerates rows of the Enumerable objects.
|
217
|
-
def each
|
218
|
-
@gens.each { |g| g.rewind }
|
219
|
-
|
220
|
-
loop do
|
221
|
-
count = 0
|
222
|
-
|
223
|
-
ret = @gens.map { |g|
|
224
|
-
if g.end?
|
225
|
-
count += 1
|
226
|
-
nil
|
227
|
-
else
|
228
|
-
g.next
|
229
|
-
end
|
230
|
-
}
|
231
|
-
|
232
|
-
if count == @gens.size
|
233
|
-
break
|
234
|
-
end
|
235
|
-
|
236
|
-
yield ret
|
237
|
-
end
|
238
|
-
|
239
|
-
self
|
240
|
-
end
|
241
|
-
end
|
242
|
-
|
243
|
-
if $0 == __FILE__
|
244
|
-
eval DATA.read, nil, $0, __LINE__+4
|
245
|
-
end
|
246
|
-
|
247
|
-
__END__
|
248
|
-
|
249
|
-
require 'test/unit'
|
250
|
-
|
251
|
-
class TC_Generator < Test::Unit::TestCase
|
252
|
-
def test_block1
|
253
|
-
g = Generator.new { |g|
|
254
|
-
# no yield's
|
255
|
-
}
|
256
|
-
|
257
|
-
assert_equal(0, g.pos)
|
258
|
-
assert_raises(EOFError) { g.current }
|
259
|
-
end
|
260
|
-
|
261
|
-
def test_block2
|
262
|
-
g = Generator.new { |g|
|
263
|
-
for i in 'A'..'C'
|
264
|
-
g.yield i
|
265
|
-
end
|
266
|
-
|
267
|
-
g.yield 'Z'
|
268
|
-
}
|
269
|
-
|
270
|
-
assert_equal(0, g.pos)
|
271
|
-
assert_equal('A', g.current)
|
272
|
-
|
273
|
-
assert_equal(true, g.next?)
|
274
|
-
assert_equal(0, g.pos)
|
275
|
-
assert_equal('A', g.current)
|
276
|
-
assert_equal(0, g.pos)
|
277
|
-
assert_equal('A', g.next)
|
278
|
-
|
279
|
-
assert_equal(1, g.pos)
|
280
|
-
assert_equal(true, g.next?)
|
281
|
-
assert_equal(1, g.pos)
|
282
|
-
assert_equal('B', g.current)
|
283
|
-
assert_equal(1, g.pos)
|
284
|
-
assert_equal('B', g.next)
|
285
|
-
|
286
|
-
assert_equal(g, g.rewind)
|
287
|
-
|
288
|
-
assert_equal(0, g.pos)
|
289
|
-
assert_equal('A', g.current)
|
290
|
-
|
291
|
-
assert_equal(true, g.next?)
|
292
|
-
assert_equal(0, g.pos)
|
293
|
-
assert_equal('A', g.current)
|
294
|
-
assert_equal(0, g.pos)
|
295
|
-
assert_equal('A', g.next)
|
296
|
-
|
297
|
-
assert_equal(1, g.pos)
|
298
|
-
assert_equal(true, g.next?)
|
299
|
-
assert_equal(1, g.pos)
|
300
|
-
assert_equal('B', g.current)
|
301
|
-
assert_equal(1, g.pos)
|
302
|
-
assert_equal('B', g.next)
|
303
|
-
|
304
|
-
assert_equal(2, g.pos)
|
305
|
-
assert_equal(true, g.next?)
|
306
|
-
assert_equal(2, g.pos)
|
307
|
-
assert_equal('C', g.current)
|
308
|
-
assert_equal(2, g.pos)
|
309
|
-
assert_equal('C', g.next)
|
310
|
-
|
311
|
-
assert_equal(3, g.pos)
|
312
|
-
assert_equal(true, g.next?)
|
313
|
-
assert_equal(3, g.pos)
|
314
|
-
assert_equal('Z', g.current)
|
315
|
-
assert_equal(3, g.pos)
|
316
|
-
assert_equal('Z', g.next)
|
317
|
-
|
318
|
-
assert_equal(4, g.pos)
|
319
|
-
assert_equal(false, g.next?)
|
320
|
-
assert_raises(EOFError) { g.next }
|
321
|
-
end
|
322
|
-
|
323
|
-
def test_each
|
324
|
-
a = [5, 6, 7, 8, 9]
|
325
|
-
|
326
|
-
g = Generator.new(a)
|
327
|
-
|
328
|
-
i = 0
|
329
|
-
|
330
|
-
g.each { |x|
|
331
|
-
assert_equal(a[i], x)
|
332
|
-
|
333
|
-
i += 1
|
334
|
-
|
335
|
-
break if i == 3
|
336
|
-
}
|
337
|
-
|
338
|
-
assert_equal(3, i)
|
339
|
-
|
340
|
-
i = 0
|
341
|
-
|
342
|
-
g.each { |x|
|
343
|
-
assert_equal(a[i], x)
|
344
|
-
|
345
|
-
i += 1
|
346
|
-
}
|
347
|
-
|
348
|
-
assert_equal(5, i)
|
349
|
-
end
|
350
|
-
end
|
351
|
-
|
352
|
-
class TC_SyncEnumerator < Test::Unit::TestCase
|
353
|
-
def test_each
|
354
|
-
r = ['a'..'f', 1..10, 10..20]
|
355
|
-
ra = r.map { |x| x.to_a }
|
356
|
-
|
357
|
-
a = (0...(ra.map {|x| x.size}.max)).map { |i| ra.map { |x| x[i] } }
|
358
|
-
|
359
|
-
s = SyncEnumerator.new(*r)
|
360
|
-
|
361
|
-
i = 0
|
362
|
-
|
363
|
-
s.each { |x|
|
364
|
-
assert_equal(a[i], x)
|
365
|
-
|
366
|
-
i += 1
|
367
|
-
|
368
|
-
break if i == 3
|
369
|
-
}
|
370
|
-
|
371
|
-
assert_equal(3, i)
|
372
|
-
|
373
|
-
i = 0
|
374
|
-
|
375
|
-
s.each { |x|
|
376
|
-
assert_equal(a[i], x)
|
377
|
-
|
378
|
-
i += 1
|
379
|
-
}
|
380
|
-
|
381
|
-
assert_equal(a.size, i)
|
382
|
-
end
|
383
|
-
end
|