xing-gearman-ruby 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/LICENSE +20 -0
- data/README +9 -0
- data/Rakefile +40 -0
- data/VERSION.yml +4 -0
- data/examples/Net/Gearman/Client.php +290 -0
- data/examples/Net/Gearman/Connection.php +410 -0
- data/examples/Net/Gearman/Exception.php +42 -0
- data/examples/Net/Gearman/Job/Common.php +135 -0
- data/examples/Net/Gearman/Job/Exception.php +45 -0
- data/examples/Net/Gearman/Job/Sleep.php +33 -0
- data/examples/Net/Gearman/Job.php +85 -0
- data/examples/Net/Gearman/Manager.php +320 -0
- data/examples/Net/Gearman/Set.php +215 -0
- data/examples/Net/Gearman/Task.php +300 -0
- data/examples/Net/Gearman/Worker.php +455 -0
- data/examples/client.php +23 -0
- data/examples/client.rb +15 -0
- data/examples/scale_image.rb +32 -0
- data/examples/scale_image_worker.rb +34 -0
- data/examples/server.rb +13 -0
- data/examples/worker.php +15 -0
- data/examples/worker.rb +23 -0
- data/gearman-ruby.gemspec +81 -0
- data/lib/gearman/client.rb +139 -0
- data/lib/gearman/server.rb +94 -0
- data/lib/gearman/task.rb +128 -0
- data/lib/gearman/taskset.rb +241 -0
- data/lib/gearman/testlib.rb +96 -0
- data/lib/gearman/util.rb +212 -0
- data/lib/gearman/worker.rb +341 -0
- data/lib/gearman.rb +76 -0
- data/test/client_test.rb +111 -0
- data/test/mock_client_test.rb +431 -0
- data/test/mock_worker_test.rb +213 -0
- data/test/worker_test.rb +61 -0
- metadata +98 -0
data/lib/gearman/util.rb
ADDED
@@ -0,0 +1,212 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
module Gearman
|
7
|
+
|
8
|
+
class ServerDownException < Exception; end
|
9
|
+
|
10
|
+
# = Util
|
11
|
+
#
|
12
|
+
# == Description
|
13
|
+
# Static helper methods and data used by other classes.
|
14
|
+
class Util
|
15
|
+
# Map from Integer representations of commands used in the network
|
16
|
+
# protocol to more-convenient symbols.
|
17
|
+
COMMANDS = {
|
18
|
+
1 => :can_do, # W->J: FUNC
|
19
|
+
23 => :can_do_timeout, # W->J: FUNC[0]TIMEOUT
|
20
|
+
2 => :cant_do, # W->J: FUNC
|
21
|
+
3 => :reset_abilities, # W->J: --
|
22
|
+
22 => :set_client_id, # W->J: [RANDOM_STRING_NO_WHITESPACE]
|
23
|
+
4 => :pre_sleep, # W->J: --
|
24
|
+
|
25
|
+
6 => :noop, # J->W: --
|
26
|
+
7 => :submit_job, # C->J: FUNC[0]UNIQ[0]ARGS
|
27
|
+
21 => :submit_job_high, # C->J: FUNC[0]UNIQ[0]ARGS
|
28
|
+
18 => :submit_job_bg, # C->J: FUNC[0]UNIQ[0]ARGS
|
29
|
+
|
30
|
+
8 => :job_created, # J->C: HANDLE
|
31
|
+
9 => :grab_job, # W->J: --
|
32
|
+
10 => :no_job, # J->W: --
|
33
|
+
11 => :job_assign, # J->W: HANDLE[0]FUNC[0]ARG
|
34
|
+
|
35
|
+
12 => :work_status, # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
|
36
|
+
13 => :work_complete, # W->J/C: HANDLE[0]RES
|
37
|
+
14 => :work_fail, # W->J/C: HANDLE
|
38
|
+
|
39
|
+
15 => :get_status, # C->J: HANDLE
|
40
|
+
20 => :status_res, # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
|
41
|
+
|
42
|
+
16 => :echo_req, # ?->J: TEXT
|
43
|
+
17 => :echo_res, # J->?: TEXT
|
44
|
+
|
45
|
+
19 => :error, # J->?: ERRCODE[0]ERR_TEXT
|
46
|
+
}
|
47
|
+
|
48
|
+
# Map e.g. 'can_do' => 1
|
49
|
+
NUMS = COMMANDS.invert
|
50
|
+
|
51
|
+
# Default job server port.
|
52
|
+
DEFAULT_PORT = 7003
|
53
|
+
|
54
|
+
@@debug = false
|
55
|
+
|
56
|
+
##
|
57
|
+
# Enable or disable debugging output (off by default).
|
58
|
+
#
|
59
|
+
# @param v print debugging output
|
60
|
+
def Util.debug=(v)
|
61
|
+
@@debug = v
|
62
|
+
end
|
63
|
+
|
64
|
+
##
|
65
|
+
# Construct a request packet.
|
66
|
+
#
|
67
|
+
# @param type_name command type's name (see COMMANDS)
|
68
|
+
# @param arg optional data to pack into the command
|
69
|
+
# @return packet (as a string)
|
70
|
+
def Util.pack_request(type_name, arg='')
|
71
|
+
type_num = NUMS[type_name.to_sym]
|
72
|
+
raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num
|
73
|
+
arg = '' if not arg
|
74
|
+
"\0REQ" + [type_num, arg.size].pack('NN') + arg
|
75
|
+
end
|
76
|
+
|
77
|
+
##
|
78
|
+
# Return a Task based on the passed-in arguments.
|
79
|
+
#
|
80
|
+
# @param args either a single Task object or the arguments accepted by
|
81
|
+
# Task.new
|
82
|
+
# @return Task object
|
83
|
+
def Util.get_task_from_args(*args)
|
84
|
+
if args[0].class == Task
|
85
|
+
return args[0]
|
86
|
+
elsif args.size <= 3
|
87
|
+
return Task.new(*args)
|
88
|
+
else
|
89
|
+
raise InvalidArgsError, 'Incorrect number of args to get_task_from_args'
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
##
|
94
|
+
# Read from a socket, giving up if it doesn't finish quickly enough.
|
95
|
+
# NetworkError is thrown if we don't read all the bytes in time.
|
96
|
+
#
|
97
|
+
# @param sock Socket from which we read
|
98
|
+
# @param len number of bytes to read
|
99
|
+
# @param timeout maximum number of seconds we'll take; nil for no timeout
|
100
|
+
# @return full data that was read
|
101
|
+
def Util.timed_recv(sock, len, timeout=nil)
|
102
|
+
data = ''
|
103
|
+
end_time = Time.now.to_f + timeout if timeout
|
104
|
+
while data.size < len and (not timeout or Time.now.to_f < end_time) do
|
105
|
+
IO::select([sock], nil, nil, timeout ? end_time - Time.now.to_f : nil) \
|
106
|
+
or break
|
107
|
+
data += sock.readpartial(len - data.size)
|
108
|
+
end
|
109
|
+
if data.size < len
|
110
|
+
raise NetworkError, "Read #{data.size} byte(s) instead of #{len}"
|
111
|
+
end
|
112
|
+
data
|
113
|
+
end
|
114
|
+
|
115
|
+
##
|
116
|
+
# Read a response packet from a socket.
|
117
|
+
#
|
118
|
+
# @param sock Socket connected to a job server
|
119
|
+
# @param timeout timeout in seconds, nil for no timeout
|
120
|
+
# @return array consisting of integer packet type and data
|
121
|
+
def Util.read_response(sock, timeout=nil)
|
122
|
+
#debugger
|
123
|
+
end_time = Time.now.to_f + timeout if timeout
|
124
|
+
head = timed_recv(sock, 12, timeout)
|
125
|
+
magic, type, len = head.unpack('a4NN')
|
126
|
+
raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES"
|
127
|
+
buf = len > 0 ?
|
128
|
+
timed_recv(sock, len, timeout ? end_time - Time.now.to_f : nil) : ''
|
129
|
+
type = COMMANDS[type]
|
130
|
+
raise ProtocolError, "Invalid packet type #{type}" unless type
|
131
|
+
[type, buf]
|
132
|
+
end
|
133
|
+
|
134
|
+
##
|
135
|
+
# Send a request packet over a socket.
|
136
|
+
#
|
137
|
+
# @param sock Socket connected to a job server
|
138
|
+
# @param req request packet to send
|
139
|
+
def Util.send_request(sock, req)
|
140
|
+
len = with_safe_socket_op{ sock.write(req) }
|
141
|
+
if len != req.size
|
142
|
+
raise NetworkError, "Wrote #{len} instead of #{req.size}"
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
##
|
147
|
+
# Add default ports to a job server or list of servers.
|
148
|
+
#
|
149
|
+
# @param servers a server hostname or "host:port" or array of servers
|
150
|
+
# @return an array of "host:port" strings
|
151
|
+
def Util.normalize_job_servers(servers)
|
152
|
+
if servers.class == String or servers.class == Symbol
|
153
|
+
servers = [ servers.to_s ]
|
154
|
+
end
|
155
|
+
servers.map {|s| s =~ /:/ ? s : "#{s}:#{DEFAULT_PORT}" }
|
156
|
+
end
|
157
|
+
|
158
|
+
##
|
159
|
+
# Convert job server info and a handle into a string.
|
160
|
+
#
|
161
|
+
# @param hostport "host:port" of job server
|
162
|
+
# @param handle job server-returned handle for a task
|
163
|
+
# @return "host:port//handle"
|
164
|
+
def Util.handle_to_str(hostport, handle)
|
165
|
+
"#{hostport}//#{handle}"
|
166
|
+
end
|
167
|
+
|
168
|
+
##
|
169
|
+
# Reverse Util.handle_to_str.
|
170
|
+
#
|
171
|
+
# @param str "host:port//handle"
|
172
|
+
# @return [hostport, handle]
|
173
|
+
def Util.str_to_handle(str)
|
174
|
+
str =~ %r{^([^:]+:\d+)//(.+)}
|
175
|
+
return [$1, $3]
|
176
|
+
end
|
177
|
+
|
178
|
+
##
|
179
|
+
# Log a message if debugging is enabled.
|
180
|
+
#
|
181
|
+
# @param str message to log
|
182
|
+
def Util.log(str, force=false)
|
183
|
+
puts "#{Time.now.strftime '%Y%m%d %H%M%S'} #{str}" if force or @@debug
|
184
|
+
end
|
185
|
+
|
186
|
+
##
|
187
|
+
# Log a message no matter what.
|
188
|
+
#
|
189
|
+
# @param str message to log
|
190
|
+
def Util.err(str)
|
191
|
+
log(str, true)
|
192
|
+
end
|
193
|
+
|
194
|
+
def self.with_safe_socket_op
|
195
|
+
begin
|
196
|
+
yield
|
197
|
+
rescue Exception => ex
|
198
|
+
raise ServerDownException.new(ex.message)
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def Util.ability_name_with_prefix(prefix,name)
|
203
|
+
"#{prefix}\t#{name}"
|
204
|
+
end
|
205
|
+
|
206
|
+
class << self
|
207
|
+
alias :ability_name_for_perl :ability_name_with_prefix
|
208
|
+
end
|
209
|
+
|
210
|
+
end
|
211
|
+
|
212
|
+
end
|
@@ -0,0 +1,341 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'set'
|
4
|
+
require 'socket'
|
5
|
+
require 'thread'
|
6
|
+
|
7
|
+
module Gearman
|
8
|
+
|
9
|
+
# = Worker
|
10
|
+
#
|
11
|
+
# == Description
|
12
|
+
# A worker that can connect to a Gearman server and perform tasks.
|
13
|
+
#
|
14
|
+
# == Usage
|
15
|
+
# require 'gearman'
|
16
|
+
#
|
17
|
+
# w = Gearman::Worker.new('127.0.0.1')
|
18
|
+
#
|
19
|
+
# # Add a handler for a "sleep" function that takes a single argument, the
|
20
|
+
# # number of seconds to sleep before reporting success.
|
21
|
+
# w.add_ability('sleep') do |data,job|
|
22
|
+
# seconds = data
|
23
|
+
# (1..seconds.to_i).each do |i|
|
24
|
+
# sleep 1
|
25
|
+
# # Report our progress to the job server every second.
|
26
|
+
# job.report_status(i, seconds)
|
27
|
+
# end
|
28
|
+
# # Report success.
|
29
|
+
# true
|
30
|
+
# end
|
31
|
+
# loop { w.work }
|
32
|
+
class Worker
|
33
|
+
# = Ability
|
34
|
+
#
|
35
|
+
# == Description
|
36
|
+
# Information about an ability that we possess.
|
37
|
+
class Ability
|
38
|
+
##
|
39
|
+
# Create a new ability.
|
40
|
+
#
|
41
|
+
# @param block code to run
|
42
|
+
# @param timeout server gives up on us after this many seconds
|
43
|
+
def initialize(block, timeout=nil)
|
44
|
+
@block = block
|
45
|
+
@timeout = timeout
|
46
|
+
end
|
47
|
+
attr_reader :timeout
|
48
|
+
|
49
|
+
##
|
50
|
+
# Run the block of code.
|
51
|
+
#
|
52
|
+
# @param data data passed to us by a client
|
53
|
+
# @param job interface to report job information to the server
|
54
|
+
def run(data, job)
|
55
|
+
@block.call(data, job)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# = Job
|
60
|
+
#
|
61
|
+
# == Description
|
62
|
+
# Interface to allow a worker to report information to a job server.
|
63
|
+
class Job
|
64
|
+
##
|
65
|
+
# Create a new Job.
|
66
|
+
#
|
67
|
+
# @param sock Socket connected to job server
|
68
|
+
# @param handle job server-supplied job handle
|
69
|
+
def initialize(sock, handle)
|
70
|
+
@socket = sock
|
71
|
+
@handle = handle
|
72
|
+
end
|
73
|
+
|
74
|
+
##
|
75
|
+
# Report our status to the job server.
|
76
|
+
def report_status(numerator, denominator)
|
77
|
+
req = Util.pack_request(
|
78
|
+
:work_status, "#{@handle}\0#{numerator}\0#{denominator}")
|
79
|
+
Util.send_request(@socket, req)
|
80
|
+
self
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
##
|
85
|
+
# Create a new worker.
|
86
|
+
#
|
87
|
+
# @param job_servers "host:port"; either a single server or an array
|
88
|
+
# @param opts hash of additional options
|
89
|
+
def initialize(job_servers=nil, opts={})
|
90
|
+
chars = ('a'..'z').to_a
|
91
|
+
@client_id = Array.new(30) { chars[rand(chars.size)] }.join
|
92
|
+
@sockets = {} # "host:port" -> Socket
|
93
|
+
@abilities = {} # "funcname" -> Ability
|
94
|
+
@bad_servers = [] # "host:port"
|
95
|
+
@servers_mutex = Mutex.new
|
96
|
+
%w{client_id reconnect_sec
|
97
|
+
network_timeout_sec}.map {|s| s.to_sym }.each do |k|
|
98
|
+
instance_variable_set "@#{k}", opts[k]
|
99
|
+
opts.delete k
|
100
|
+
end
|
101
|
+
if opts.size > 0
|
102
|
+
raise InvalidArgsError,
|
103
|
+
'Invalid worker args: ' + opts.keys.sort.join(', ')
|
104
|
+
end
|
105
|
+
@reconnect_sec = 30 if not @reconnect_sec
|
106
|
+
@network_timeout_sec = 5 if not @network_timeout_sec
|
107
|
+
self.job_servers = job_servers if job_servers
|
108
|
+
start_reconnect_thread
|
109
|
+
end
|
110
|
+
attr_accessor :client_id, :reconnect_sec, :network_timeout_sec, :bad_servers
|
111
|
+
|
112
|
+
# Start a thread to repeatedly attempt to connect to down job servers.
|
113
|
+
def start_reconnect_thread
|
114
|
+
Thread.new do
|
115
|
+
loop do
|
116
|
+
@servers_mutex.synchronize do
|
117
|
+
# If there are any failed servers, try to reconnect to them.
|
118
|
+
if not @bad_servers.empty?
|
119
|
+
update_job_servers(@sockets.keys + @bad_servers)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
sleep @reconnect_sec
|
123
|
+
end
|
124
|
+
end.run
|
125
|
+
end
|
126
|
+
|
127
|
+
def job_servers
|
128
|
+
servers = nil
|
129
|
+
@servers_mutex.synchronize do
|
130
|
+
servers = @sockets.keys + @bad_servers
|
131
|
+
end
|
132
|
+
servers
|
133
|
+
end
|
134
|
+
|
135
|
+
##
|
136
|
+
# Connect to job servers to be used by this worker.
|
137
|
+
#
|
138
|
+
# @param servers "host:port"; either a single server or an array
|
139
|
+
def job_servers=(servers)
|
140
|
+
@servers_mutex.synchronize do
|
141
|
+
update_job_servers(servers)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
# Internal function to actually connect to servers.
|
146
|
+
# Caller must acquire @servers_mutex before calling us.
|
147
|
+
#
|
148
|
+
# @param servers "host:port"; either a single server or an array
|
149
|
+
def update_job_servers(servers)
|
150
|
+
@bad_servers = []
|
151
|
+
servers = Set.new(Util.normalize_job_servers(servers))
|
152
|
+
# Disconnect from servers that we no longer care about.
|
153
|
+
@sockets.each do |server,sock|
|
154
|
+
if not servers.include? server
|
155
|
+
Util.log "Disconnecting from old server #{server}"
|
156
|
+
sock.close
|
157
|
+
@sockets.delete(server)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
# Connect to new servers.
|
161
|
+
servers.each do |server|
|
162
|
+
if not @sockets[server]
|
163
|
+
begin
|
164
|
+
Util.log "Connecting to server #{server}"
|
165
|
+
@sockets[server] = connect(server)
|
166
|
+
rescue NetworkError, Errno::ECONNRESET
|
167
|
+
@bad_servers << server
|
168
|
+
Util.log "Unable to connect to #{server}"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
private :update_job_servers
|
174
|
+
|
175
|
+
##
|
176
|
+
# Connect to a job server.
|
177
|
+
#
|
178
|
+
# @param hostport "hostname:port"
|
179
|
+
def connect(hostport)
|
180
|
+
begin
|
181
|
+
# FIXME: handle timeouts
|
182
|
+
sock = TCPSocket.new(*hostport.split(':'))
|
183
|
+
rescue Errno::ECONNREFUSED
|
184
|
+
raise NetworkError
|
185
|
+
end
|
186
|
+
# FIXME: catch exceptions; do something smart
|
187
|
+
Util.send_request(sock, Util.pack_request(:set_client_id, @client_id))
|
188
|
+
@abilities.each {|f,a| announce_ability(sock, f, a.timeout) }
|
189
|
+
@sockets[hostport] = sock
|
190
|
+
end
|
191
|
+
private :connect
|
192
|
+
|
193
|
+
##
|
194
|
+
# Announce an ability over a particular socket.
|
195
|
+
#
|
196
|
+
# @param sock Socket connect to a job server
|
197
|
+
# @param func function name (including prefix)
|
198
|
+
# @param timeout the server will give up on us if we don't finish
|
199
|
+
# a task in this many seconds
|
200
|
+
def announce_ability(sock, func, timeout=nil)
|
201
|
+
begin
|
202
|
+
cmd = timeout ? :can_do_timeout : :can_do
|
203
|
+
arg = timeout ? "#{func}\0#{timeout.to_s}" : func
|
204
|
+
Util.send_request(sock, Util.pack_request(cmd, arg))
|
205
|
+
rescue Exception => ex
|
206
|
+
bad_servers << @sockets.keys.detect{|hp| @sockets[hp] == sock}
|
207
|
+
end
|
208
|
+
end
|
209
|
+
private :announce_ability
|
210
|
+
|
211
|
+
##
|
212
|
+
# Add a new ability, announcing it to job servers.
|
213
|
+
#
|
214
|
+
# The passed-in block of code will be executed for jobs of this function
|
215
|
+
# type. It'll receive two arguments, the data supplied by the client and
|
216
|
+
# a Job object. If it returns nil or false, the server will be informed
|
217
|
+
# that the job has failed; otherwise the return value of the block will
|
218
|
+
# be passed back to the client in String form.
|
219
|
+
#
|
220
|
+
# @param func function name (without prefix)
|
221
|
+
# @param timeout the server will give up on us if we don't finish
|
222
|
+
# a task in this many seconds
|
223
|
+
def add_ability(func, timeout=nil, &f)
|
224
|
+
@abilities[func] = Ability.new(f, timeout)
|
225
|
+
@sockets.values.each {|s| announce_ability(s, func, timeout) }
|
226
|
+
end
|
227
|
+
|
228
|
+
##
|
229
|
+
# Let job servers know that we're no longer able to do something.
|
230
|
+
#
|
231
|
+
# @param func function name
|
232
|
+
def remove_ability(func)
|
233
|
+
@abilities.delete(func)
|
234
|
+
req = Util.pack_request(:cant_do, func)
|
235
|
+
@sockets.values.each {|s| Util.send_request(s, req) }
|
236
|
+
end
|
237
|
+
|
238
|
+
##
|
239
|
+
# Handle a job_assign packet.
|
240
|
+
#
|
241
|
+
# @param data data in the packet
|
242
|
+
# @param sock Socket on which the packet arrived
|
243
|
+
# @param hostport "host:port"
|
244
|
+
def handle_job_assign(data, sock, hostport)
|
245
|
+
handle, func, data = data.split("\0", 3)
|
246
|
+
if not func
|
247
|
+
Util.err "Ignoring job_assign with no function from #{hostport}"
|
248
|
+
return false
|
249
|
+
end
|
250
|
+
|
251
|
+
Util.log "Got job_assign with handle #{handle} and #{data.size} byte(s) " +
|
252
|
+
"from #{hostport}"
|
253
|
+
|
254
|
+
ability = @abilities[func]
|
255
|
+
if not ability
|
256
|
+
Util.err "Ignoring job_assign for unsupported func #{func} " +
|
257
|
+
"with handle #{handle} from #{hostport}"
|
258
|
+
Util.send_request(sock, Util.pack_request(:work_fail, handle))
|
259
|
+
return false
|
260
|
+
end
|
261
|
+
|
262
|
+
ret = ability.run(data, Job.new(sock, handle))
|
263
|
+
|
264
|
+
cmd = nil
|
265
|
+
if ret
|
266
|
+
ret = ret.to_s
|
267
|
+
Util.log "Sending work_complete for #{handle} with #{ret.size} byte(s) " +
|
268
|
+
"to #{hostport}"
|
269
|
+
cmd = Util.pack_request(:work_complete, "#{handle}\0#{ret}")
|
270
|
+
else
|
271
|
+
Util.log "Sending work_fail for #{handle} to #{hostport}"
|
272
|
+
cmd = Util.pack_request(:work_fail, handle)
|
273
|
+
end
|
274
|
+
|
275
|
+
Util.send_request(sock, cmd)
|
276
|
+
true
|
277
|
+
end
|
278
|
+
|
279
|
+
##
|
280
|
+
# Do a single job and return.
|
281
|
+
def work
|
282
|
+
req = Util.pack_request(:grab_job)
|
283
|
+
loop do
|
284
|
+
bad_servers = []
|
285
|
+
# We iterate through the servers in sorted order to make testing
|
286
|
+
# easier.
|
287
|
+
servers = nil
|
288
|
+
@servers_mutex.synchronize { servers = @sockets.keys.sort }
|
289
|
+
servers.each do |hostport|
|
290
|
+
Util.log "Sending grab_job to #{hostport}"
|
291
|
+
sock = @sockets[hostport]
|
292
|
+
Util.send_request(sock, req)
|
293
|
+
|
294
|
+
# Now that we've sent grab_job, we need to keep reading packets
|
295
|
+
# until we see a no_job or job_assign response (there may be a noop
|
296
|
+
# waiting for us in response to a previous pre_sleep).
|
297
|
+
loop do
|
298
|
+
begin
|
299
|
+
type, data = Util.read_response(sock, @network_timeout_sec)
|
300
|
+
case type
|
301
|
+
when :no_job
|
302
|
+
Util.log "Got no_job from #{hostport}"
|
303
|
+
break
|
304
|
+
when :job_assign
|
305
|
+
return if handle_job_assign(data, sock, hostport)
|
306
|
+
break
|
307
|
+
else
|
308
|
+
Util.log "Got #{type.to_s} from #{hostport}"
|
309
|
+
end
|
310
|
+
rescue Gearman::ServerDownException, NetworkError, Errno::ECONNRESET
|
311
|
+
Util.log "Server #{hostport} timed out or lost connection; marking bad"
|
312
|
+
bad_servers << hostport
|
313
|
+
break
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
@servers_mutex.synchronize do
|
319
|
+
bad_servers.each do |hostport|
|
320
|
+
@sockets[hostport].close if @sockets[hostport]
|
321
|
+
@bad_servers << hostport if @sockets[hostport]
|
322
|
+
@sockets.delete(hostport)
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
Util.log "Sending pre_sleep and going to sleep for #{@reconnect_sec} sec"
|
327
|
+
@servers_mutex.synchronize do
|
328
|
+
@sockets.values.each do |sock|
|
329
|
+
Util.send_request(sock, Util.pack_request(:pre_sleep))
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
# FIXME: We could optimize things the next time through the 'each' by
|
334
|
+
# sending the first grab_job to one of the servers that had a socket
|
335
|
+
# with data in it. Not bothering with it for now.
|
336
|
+
IO::select(@sockets.values, nil, nil, @reconnect_sec)
|
337
|
+
end
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
341
|
+
end
|
data/lib/gearman.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# = Name
|
4
|
+
# Gearman
|
5
|
+
#
|
6
|
+
# == Description
|
7
|
+
# This file provides a Ruby interface for communicating with the Gearman
|
8
|
+
# distributed job system.
|
9
|
+
#
|
10
|
+
# "Gearman is a system to farm out work to other machines, dispatching
|
11
|
+
# function calls to machines that are better suited to do work, to do work
|
12
|
+
# in parallel, to load balance lots of function calls, or to call functions
|
13
|
+
# between languages." -- http://www.danga.com/gearman/
|
14
|
+
#
|
15
|
+
# == Version
|
16
|
+
# 0.0.1
|
17
|
+
#
|
18
|
+
# == Author
|
19
|
+
# Daniel Erat <dan-ruby@erat.org>
|
20
|
+
#
|
21
|
+
# == License
|
22
|
+
# This program is free software; you can redistribute it and/or modify it
|
23
|
+
# under the terms of either:
|
24
|
+
#
|
25
|
+
# a) the GNU General Public License as published by the Free Software
|
26
|
+
# Foundation; either version 1, or (at your option) any later version,
|
27
|
+
# or
|
28
|
+
#
|
29
|
+
# b) the "Artistic License" which comes with Perl.
|
30
|
+
|
31
|
+
# = Gearman
|
32
|
+
#
|
33
|
+
# == Usage
|
34
|
+
# require 'gearman'
|
35
|
+
#
|
36
|
+
# # Create a new client and tell it about two job servers.
|
37
|
+
# c = Gearman::Client.new
|
38
|
+
# c.job_servers = ['127.0.0.1:7003', '127.0.0.1:7004']
|
39
|
+
#
|
40
|
+
# # Create two tasks, using an "add" function to sum two numbers.
|
41
|
+
# t1 = Gearman::Task.new('add', '5 + 2')
|
42
|
+
# t2 = Gearman::Task.new('add', '1 + 3')
|
43
|
+
#
|
44
|
+
# # Make the tasks print the data they get back from the server.
|
45
|
+
# t1.on_complete {|d| puts "t1 got #{d}" }
|
46
|
+
# t2.on_complete {|d| puts "t2 got #{d}" }
|
47
|
+
#
|
48
|
+
# # Create a taskset, add the two tasks to it, and wait until they finish.
|
49
|
+
# ts = Gearman::TaskSet.new(c)
|
50
|
+
# ts.add_task(t1)
|
51
|
+
# ts.add_task(t2)
|
52
|
+
# ts.wait
|
53
|
+
#
|
54
|
+
# Or, a more simple example:
|
55
|
+
#
|
56
|
+
# c = Gearman::Client.new('127.0.0.1')
|
57
|
+
# puts c.do_task('add', '2 + 2')
|
58
|
+
#
|
59
|
+
module Gearman
|
60
|
+
|
61
|
+
require File.dirname(__FILE__) + '/gearman/client'
|
62
|
+
require File.dirname(__FILE__) + '/gearman/task'
|
63
|
+
require File.dirname(__FILE__) + '/gearman/taskset'
|
64
|
+
require File.dirname(__FILE__) + '/gearman/util'
|
65
|
+
require File.dirname(__FILE__) + '/gearman/worker'
|
66
|
+
|
67
|
+
class InvalidArgsError < Exception
|
68
|
+
end
|
69
|
+
|
70
|
+
class ProtocolError < Exception
|
71
|
+
end
|
72
|
+
|
73
|
+
class NetworkError < Exception
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|