xing-gearman-ruby 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,212 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'socket'
4
+ require 'time'
5
+
6
+ module Gearman
7
+
8
+ class ServerDownException < Exception; end
9
+
10
+ # = Util
11
+ #
12
+ # == Description
13
+ # Static helper methods and data used by other classes.
14
+ class Util
15
+ # Map from Integer representations of commands used in the network
16
+ # protocol to more-convenient symbols.
17
+ COMMANDS = {
18
+ 1 => :can_do, # W->J: FUNC
19
+ 23 => :can_do_timeout, # W->J: FUNC[0]TIMEOUT
20
+ 2 => :cant_do, # W->J: FUNC
21
+ 3 => :reset_abilities, # W->J: --
22
+ 22 => :set_client_id, # W->J: [RANDOM_STRING_NO_WHITESPACE]
23
+ 4 => :pre_sleep, # W->J: --
24
+
25
+ 6 => :noop, # J->W: --
26
+ 7 => :submit_job, # C->J: FUNC[0]UNIQ[0]ARGS
27
+ 21 => :submit_job_high, # C->J: FUNC[0]UNIQ[0]ARGS
28
+ 18 => :submit_job_bg, # C->J: FUNC[0]UNIQ[0]ARGS
29
+
30
+ 8 => :job_created, # J->C: HANDLE
31
+ 9 => :grab_job, # W->J: --
32
+ 10 => :no_job, # J->W: --
33
+ 11 => :job_assign, # J->W: HANDLE[0]FUNC[0]ARG
34
+
35
+ 12 => :work_status, # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
36
+ 13 => :work_complete, # W->J/C: HANDLE[0]RES
37
+ 14 => :work_fail, # W->J/C: HANDLE
38
+
39
+ 15 => :get_status, # C->J: HANDLE
40
+ 20 => :status_res, # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
41
+
42
+ 16 => :echo_req, # ?->J: TEXT
43
+ 17 => :echo_res, # J->?: TEXT
44
+
45
+ 19 => :error, # J->?: ERRCODE[0]ERR_TEXT
46
+ }
47
+
48
+ # Map e.g. 'can_do' => 1
49
+ NUMS = COMMANDS.invert
50
+
51
+ # Default job server port.
52
+ DEFAULT_PORT = 7003
53
+
54
+ @@debug = false
55
+
56
+ ##
57
+ # Enable or disable debugging output (off by default).
58
+ #
59
+ # @param v print debugging output
60
+ def Util.debug=(v)
61
+ @@debug = v
62
+ end
63
+
64
+ ##
65
+ # Construct a request packet.
66
+ #
67
+ # @param type_name command type's name (see COMMANDS)
68
+ # @param arg optional data to pack into the command
69
+ # @return packet (as a string)
70
+ def Util.pack_request(type_name, arg='')
71
+ type_num = NUMS[type_name.to_sym]
72
+ raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num
73
+ arg = '' if not arg
74
+ "\0REQ" + [type_num, arg.size].pack('NN') + arg
75
+ end
76
+
77
+ ##
78
+ # Return a Task based on the passed-in arguments.
79
+ #
80
+ # @param args either a single Task object or the arguments accepted by
81
+ # Task.new
82
+ # @return Task object
83
+ def Util.get_task_from_args(*args)
84
+ if args[0].class == Task
85
+ return args[0]
86
+ elsif args.size <= 3
87
+ return Task.new(*args)
88
+ else
89
+ raise InvalidArgsError, 'Incorrect number of args to get_task_from_args'
90
+ end
91
+ end
92
+
93
+ ##
94
+ # Read from a socket, giving up if it doesn't finish quickly enough.
95
+ # NetworkError is thrown if we don't read all the bytes in time.
96
+ #
97
+ # @param sock Socket from which we read
98
+ # @param len number of bytes to read
99
+ # @param timeout maximum number of seconds we'll take; nil for no timeout
100
+ # @return full data that was read
101
+ def Util.timed_recv(sock, len, timeout=nil)
102
+ data = ''
103
+ end_time = Time.now.to_f + timeout if timeout
104
+ while data.size < len and (not timeout or Time.now.to_f < end_time) do
105
+ IO::select([sock], nil, nil, timeout ? end_time - Time.now.to_f : nil) \
106
+ or break
107
+ data += sock.readpartial(len - data.size)
108
+ end
109
+ if data.size < len
110
+ raise NetworkError, "Read #{data.size} byte(s) instead of #{len}"
111
+ end
112
+ data
113
+ end
114
+
115
+ ##
116
+ # Read a response packet from a socket.
117
+ #
118
+ # @param sock Socket connected to a job server
119
+ # @param timeout timeout in seconds, nil for no timeout
120
+ # @return array consisting of integer packet type and data
121
+ def Util.read_response(sock, timeout=nil)
122
+ #debugger
123
+ end_time = Time.now.to_f + timeout if timeout
124
+ head = timed_recv(sock, 12, timeout)
125
+ magic, type, len = head.unpack('a4NN')
126
+ raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES"
127
+ buf = len > 0 ?
128
+ timed_recv(sock, len, timeout ? end_time - Time.now.to_f : nil) : ''
129
+ type = COMMANDS[type]
130
+ raise ProtocolError, "Invalid packet type #{type}" unless type
131
+ [type, buf]
132
+ end
133
+
134
+ ##
135
+ # Send a request packet over a socket.
136
+ #
137
+ # @param sock Socket connected to a job server
138
+ # @param req request packet to send
139
+ def Util.send_request(sock, req)
140
+ len = with_safe_socket_op{ sock.write(req) }
141
+ if len != req.size
142
+ raise NetworkError, "Wrote #{len} instead of #{req.size}"
143
+ end
144
+ end
145
+
146
+ ##
147
+ # Add default ports to a job server or list of servers.
148
+ #
149
+ # @param servers a server hostname or "host:port" or array of servers
150
+ # @return an array of "host:port" strings
151
+ def Util.normalize_job_servers(servers)
152
+ if servers.class == String or servers.class == Symbol
153
+ servers = [ servers.to_s ]
154
+ end
155
+ servers.map {|s| s =~ /:/ ? s : "#{s}:#{DEFAULT_PORT}" }
156
+ end
157
+
158
+ ##
159
+ # Convert job server info and a handle into a string.
160
+ #
161
+ # @param hostport "host:port" of job server
162
+ # @param handle job server-returned handle for a task
163
+ # @return "host:port//handle"
164
+ def Util.handle_to_str(hostport, handle)
165
+ "#{hostport}//#{handle}"
166
+ end
167
+
168
+ ##
169
+ # Reverse Util.handle_to_str.
170
+ #
171
+ # @param str "host:port//handle"
172
+ # @return [hostport, handle]
173
+ def Util.str_to_handle(str)
174
+ str =~ %r{^([^:]+:\d+)//(.+)}
175
+ return [$1, $3]
176
+ end
177
+
178
+ ##
179
+ # Log a message if debugging is enabled.
180
+ #
181
+ # @param str message to log
182
+ def Util.log(str, force=false)
183
+ puts "#{Time.now.strftime '%Y%m%d %H%M%S'} #{str}" if force or @@debug
184
+ end
185
+
186
+ ##
187
+ # Log a message no matter what.
188
+ #
189
+ # @param str message to log
190
+ def Util.err(str)
191
+ log(str, true)
192
+ end
193
+
194
+ def self.with_safe_socket_op
195
+ begin
196
+ yield
197
+ rescue Exception => ex
198
+ raise ServerDownException.new(ex.message)
199
+ end
200
+ end
201
+
202
+ def Util.ability_name_with_prefix(prefix,name)
203
+ "#{prefix}\t#{name}"
204
+ end
205
+
206
+ class << self
207
+ alias :ability_name_for_perl :ability_name_with_prefix
208
+ end
209
+
210
+ end
211
+
212
+ end
@@ -0,0 +1,341 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'set'
4
+ require 'socket'
5
+ require 'thread'
6
+
7
+ module Gearman
8
+
9
+ # = Worker
10
+ #
11
+ # == Description
12
+ # A worker that can connect to a Gearman server and perform tasks.
13
+ #
14
+ # == Usage
15
+ # require 'gearman'
16
+ #
17
+ # w = Gearman::Worker.new('127.0.0.1')
18
+ #
19
+ # # Add a handler for a "sleep" function that takes a single argument, the
20
+ # # number of seconds to sleep before reporting success.
21
+ # w.add_ability('sleep') do |data,job|
22
+ # seconds = data
23
+ # (1..seconds.to_i).each do |i|
24
+ # sleep 1
25
+ # # Report our progress to the job server every second.
26
+ # job.report_status(i, seconds)
27
+ # end
28
+ # # Report success.
29
+ # true
30
+ # end
31
+ # loop { w.work }
32
+ class Worker
33
+ # = Ability
34
+ #
35
+ # == Description
36
+ # Information about an ability that we possess.
37
+ class Ability
38
+ ##
39
+ # Create a new ability.
40
+ #
41
+ # @param block code to run
42
+ # @param timeout server gives up on us after this many seconds
43
+ def initialize(block, timeout=nil)
44
+ @block = block
45
+ @timeout = timeout
46
+ end
47
+ attr_reader :timeout
48
+
49
+ ##
50
+ # Run the block of code.
51
+ #
52
+ # @param data data passed to us by a client
53
+ # @param job interface to report job information to the server
54
+ def run(data, job)
55
+ @block.call(data, job)
56
+ end
57
+ end
58
+
59
+ # = Job
60
+ #
61
+ # == Description
62
+ # Interface to allow a worker to report information to a job server.
63
+ class Job
64
+ ##
65
+ # Create a new Job.
66
+ #
67
+ # @param sock Socket connected to job server
68
+ # @param handle job server-supplied job handle
69
+ def initialize(sock, handle)
70
+ @socket = sock
71
+ @handle = handle
72
+ end
73
+
74
+ ##
75
+ # Report our status to the job server.
76
+ def report_status(numerator, denominator)
77
+ req = Util.pack_request(
78
+ :work_status, "#{@handle}\0#{numerator}\0#{denominator}")
79
+ Util.send_request(@socket, req)
80
+ self
81
+ end
82
+ end
83
+
84
+ ##
85
+ # Create a new worker.
86
+ #
87
+ # @param job_servers "host:port"; either a single server or an array
88
+ # @param opts hash of additional options
89
+ def initialize(job_servers=nil, opts={})
90
+ chars = ('a'..'z').to_a
91
+ @client_id = Array.new(30) { chars[rand(chars.size)] }.join
92
+ @sockets = {} # "host:port" -> Socket
93
+ @abilities = {} # "funcname" -> Ability
94
+ @bad_servers = [] # "host:port"
95
+ @servers_mutex = Mutex.new
96
+ %w{client_id reconnect_sec
97
+ network_timeout_sec}.map {|s| s.to_sym }.each do |k|
98
+ instance_variable_set "@#{k}", opts[k]
99
+ opts.delete k
100
+ end
101
+ if opts.size > 0
102
+ raise InvalidArgsError,
103
+ 'Invalid worker args: ' + opts.keys.sort.join(', ')
104
+ end
105
+ @reconnect_sec = 30 if not @reconnect_sec
106
+ @network_timeout_sec = 5 if not @network_timeout_sec
107
+ self.job_servers = job_servers if job_servers
108
+ start_reconnect_thread
109
+ end
110
+ attr_accessor :client_id, :reconnect_sec, :network_timeout_sec, :bad_servers
111
+
112
+ # Start a thread to repeatedly attempt to connect to down job servers.
113
+ def start_reconnect_thread
114
+ Thread.new do
115
+ loop do
116
+ @servers_mutex.synchronize do
117
+ # If there are any failed servers, try to reconnect to them.
118
+ if not @bad_servers.empty?
119
+ update_job_servers(@sockets.keys + @bad_servers)
120
+ end
121
+ end
122
+ sleep @reconnect_sec
123
+ end
124
+ end.run
125
+ end
126
+
127
+ def job_servers
128
+ servers = nil
129
+ @servers_mutex.synchronize do
130
+ servers = @sockets.keys + @bad_servers
131
+ end
132
+ servers
133
+ end
134
+
135
+ ##
136
+ # Connect to job servers to be used by this worker.
137
+ #
138
+ # @param servers "host:port"; either a single server or an array
139
+ def job_servers=(servers)
140
+ @servers_mutex.synchronize do
141
+ update_job_servers(servers)
142
+ end
143
+ end
144
+
145
+ # Internal function to actually connect to servers.
146
+ # Caller must acquire @servers_mutex before calling us.
147
+ #
148
+ # @param servers "host:port"; either a single server or an array
149
+ def update_job_servers(servers)
150
+ @bad_servers = []
151
+ servers = Set.new(Util.normalize_job_servers(servers))
152
+ # Disconnect from servers that we no longer care about.
153
+ @sockets.each do |server,sock|
154
+ if not servers.include? server
155
+ Util.log "Disconnecting from old server #{server}"
156
+ sock.close
157
+ @sockets.delete(server)
158
+ end
159
+ end
160
+ # Connect to new servers.
161
+ servers.each do |server|
162
+ if not @sockets[server]
163
+ begin
164
+ Util.log "Connecting to server #{server}"
165
+ @sockets[server] = connect(server)
166
+ rescue NetworkError, Errno::ECONNRESET
167
+ @bad_servers << server
168
+ Util.log "Unable to connect to #{server}"
169
+ end
170
+ end
171
+ end
172
+ end
173
+ private :update_job_servers
174
+
175
+ ##
176
+ # Connect to a job server.
177
+ #
178
+ # @param hostport "hostname:port"
179
+ def connect(hostport)
180
+ begin
181
+ # FIXME: handle timeouts
182
+ sock = TCPSocket.new(*hostport.split(':'))
183
+ rescue Errno::ECONNREFUSED
184
+ raise NetworkError
185
+ end
186
+ # FIXME: catch exceptions; do something smart
187
+ Util.send_request(sock, Util.pack_request(:set_client_id, @client_id))
188
+ @abilities.each {|f,a| announce_ability(sock, f, a.timeout) }
189
+ @sockets[hostport] = sock
190
+ end
191
+ private :connect
192
+
193
+ ##
194
+ # Announce an ability over a particular socket.
195
+ #
196
+ # @param sock Socket connect to a job server
197
+ # @param func function name (including prefix)
198
+ # @param timeout the server will give up on us if we don't finish
199
+ # a task in this many seconds
200
+ def announce_ability(sock, func, timeout=nil)
201
+ begin
202
+ cmd = timeout ? :can_do_timeout : :can_do
203
+ arg = timeout ? "#{func}\0#{timeout.to_s}" : func
204
+ Util.send_request(sock, Util.pack_request(cmd, arg))
205
+ rescue Exception => ex
206
+ bad_servers << @sockets.keys.detect{|hp| @sockets[hp] == sock}
207
+ end
208
+ end
209
+ private :announce_ability
210
+
211
+ ##
212
+ # Add a new ability, announcing it to job servers.
213
+ #
214
+ # The passed-in block of code will be executed for jobs of this function
215
+ # type. It'll receive two arguments, the data supplied by the client and
216
+ # a Job object. If it returns nil or false, the server will be informed
217
+ # that the job has failed; otherwise the return value of the block will
218
+ # be passed back to the client in String form.
219
+ #
220
+ # @param func function name (without prefix)
221
+ # @param timeout the server will give up on us if we don't finish
222
+ # a task in this many seconds
223
+ def add_ability(func, timeout=nil, &f)
224
+ @abilities[func] = Ability.new(f, timeout)
225
+ @sockets.values.each {|s| announce_ability(s, func, timeout) }
226
+ end
227
+
228
+ ##
229
+ # Let job servers know that we're no longer able to do something.
230
+ #
231
+ # @param func function name
232
+ def remove_ability(func)
233
+ @abilities.delete(func)
234
+ req = Util.pack_request(:cant_do, func)
235
+ @sockets.values.each {|s| Util.send_request(s, req) }
236
+ end
237
+
238
+ ##
239
+ # Handle a job_assign packet.
240
+ #
241
+ # @param data data in the packet
242
+ # @param sock Socket on which the packet arrived
243
+ # @param hostport "host:port"
244
+ def handle_job_assign(data, sock, hostport)
245
+ handle, func, data = data.split("\0", 3)
246
+ if not func
247
+ Util.err "Ignoring job_assign with no function from #{hostport}"
248
+ return false
249
+ end
250
+
251
+ Util.log "Got job_assign with handle #{handle} and #{data.size} byte(s) " +
252
+ "from #{hostport}"
253
+
254
+ ability = @abilities[func]
255
+ if not ability
256
+ Util.err "Ignoring job_assign for unsupported func #{func} " +
257
+ "with handle #{handle} from #{hostport}"
258
+ Util.send_request(sock, Util.pack_request(:work_fail, handle))
259
+ return false
260
+ end
261
+
262
+ ret = ability.run(data, Job.new(sock, handle))
263
+
264
+ cmd = nil
265
+ if ret
266
+ ret = ret.to_s
267
+ Util.log "Sending work_complete for #{handle} with #{ret.size} byte(s) " +
268
+ "to #{hostport}"
269
+ cmd = Util.pack_request(:work_complete, "#{handle}\0#{ret}")
270
+ else
271
+ Util.log "Sending work_fail for #{handle} to #{hostport}"
272
+ cmd = Util.pack_request(:work_fail, handle)
273
+ end
274
+
275
+ Util.send_request(sock, cmd)
276
+ true
277
+ end
278
+
279
+ ##
280
+ # Do a single job and return.
281
+ def work
282
+ req = Util.pack_request(:grab_job)
283
+ loop do
284
+ bad_servers = []
285
+ # We iterate through the servers in sorted order to make testing
286
+ # easier.
287
+ servers = nil
288
+ @servers_mutex.synchronize { servers = @sockets.keys.sort }
289
+ servers.each do |hostport|
290
+ Util.log "Sending grab_job to #{hostport}"
291
+ sock = @sockets[hostport]
292
+ Util.send_request(sock, req)
293
+
294
+ # Now that we've sent grab_job, we need to keep reading packets
295
+ # until we see a no_job or job_assign response (there may be a noop
296
+ # waiting for us in response to a previous pre_sleep).
297
+ loop do
298
+ begin
299
+ type, data = Util.read_response(sock, @network_timeout_sec)
300
+ case type
301
+ when :no_job
302
+ Util.log "Got no_job from #{hostport}"
303
+ break
304
+ when :job_assign
305
+ return if handle_job_assign(data, sock, hostport)
306
+ break
307
+ else
308
+ Util.log "Got #{type.to_s} from #{hostport}"
309
+ end
310
+ rescue Gearman::ServerDownException, NetworkError, Errno::ECONNRESET
311
+ Util.log "Server #{hostport} timed out or lost connection; marking bad"
312
+ bad_servers << hostport
313
+ break
314
+ end
315
+ end
316
+ end
317
+
318
+ @servers_mutex.synchronize do
319
+ bad_servers.each do |hostport|
320
+ @sockets[hostport].close if @sockets[hostport]
321
+ @bad_servers << hostport if @sockets[hostport]
322
+ @sockets.delete(hostport)
323
+ end
324
+ end
325
+
326
+ Util.log "Sending pre_sleep and going to sleep for #{@reconnect_sec} sec"
327
+ @servers_mutex.synchronize do
328
+ @sockets.values.each do |sock|
329
+ Util.send_request(sock, Util.pack_request(:pre_sleep))
330
+ end
331
+ end
332
+
333
+ # FIXME: We could optimize things the next time through the 'each' by
334
+ # sending the first grab_job to one of the servers that had a socket
335
+ # with data in it. Not bothering with it for now.
336
+ IO::select(@sockets.values, nil, nil, @reconnect_sec)
337
+ end
338
+ end
339
+ end
340
+
341
+ end
data/lib/gearman.rb ADDED
@@ -0,0 +1,76 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # = Name
4
+ # Gearman
5
+ #
6
+ # == Description
7
+ # This file provides a Ruby interface for communicating with the Gearman
8
+ # distributed job system.
9
+ #
10
+ # "Gearman is a system to farm out work to other machines, dispatching
11
+ # function calls to machines that are better suited to do work, to do work
12
+ # in parallel, to load balance lots of function calls, or to call functions
13
+ # between languages." -- http://www.danga.com/gearman/
14
+ #
15
+ # == Version
16
+ # 0.0.1
17
+ #
18
+ # == Author
19
+ # Daniel Erat <dan-ruby@erat.org>
20
+ #
21
+ # == License
22
+ # This program is free software; you can redistribute it and/or modify it
23
+ # under the terms of either:
24
+ #
25
+ # a) the GNU General Public License as published by the Free Software
26
+ # Foundation; either version 1, or (at your option) any later version,
27
+ # or
28
+ #
29
+ # b) the "Artistic License" which comes with Perl.
30
+
31
+ # = Gearman
32
+ #
33
+ # == Usage
34
+ # require 'gearman'
35
+ #
36
+ # # Create a new client and tell it about two job servers.
37
+ # c = Gearman::Client.new
38
+ # c.job_servers = ['127.0.0.1:7003', '127.0.0.1:7004']
39
+ #
40
+ # # Create two tasks, using an "add" function to sum two numbers.
41
+ # t1 = Gearman::Task.new('add', '5 + 2')
42
+ # t2 = Gearman::Task.new('add', '1 + 3')
43
+ #
44
+ # # Make the tasks print the data they get back from the server.
45
+ # t1.on_complete {|d| puts "t1 got #{d}" }
46
+ # t2.on_complete {|d| puts "t2 got #{d}" }
47
+ #
48
+ # # Create a taskset, add the two tasks to it, and wait until they finish.
49
+ # ts = Gearman::TaskSet.new(c)
50
+ # ts.add_task(t1)
51
+ # ts.add_task(t2)
52
+ # ts.wait
53
+ #
54
+ # Or, a more simple example:
55
+ #
56
+ # c = Gearman::Client.new('127.0.0.1')
57
+ # puts c.do_task('add', '2 + 2')
58
+ #
59
+ module Gearman
60
+
61
+ require File.dirname(__FILE__) + '/gearman/client'
62
+ require File.dirname(__FILE__) + '/gearman/task'
63
+ require File.dirname(__FILE__) + '/gearman/taskset'
64
+ require File.dirname(__FILE__) + '/gearman/util'
65
+ require File.dirname(__FILE__) + '/gearman/worker'
66
+
67
+ class InvalidArgsError < Exception
68
+ end
69
+
70
+ class ProtocolError < Exception
71
+ end
72
+
73
+ class NetworkError < Exception
74
+ end
75
+
76
+ end