gearman-ruby 2.0.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +5 -6
- data/VERSION.yml +2 -2
- data/examples/calculus_client.rb +8 -10
- data/examples/calculus_worker.rb +4 -1
- data/examples/client.php +23 -0
- data/examples/client.rb +6 -10
- data/examples/client_background.rb +7 -7
- data/examples/client_data.rb +4 -4
- data/examples/client_epoch.rb +23 -0
- data/examples/client_exception.rb +4 -2
- data/examples/client_prefix.rb +4 -2
- data/examples/client_reverse.rb +27 -0
- data/examples/scale_image.rb +4 -3
- data/examples/scale_image_worker.rb +1 -1
- data/examples/worker.rb +2 -4
- data/examples/worker_data.rb +2 -2
- data/examples/worker_exception.rb +1 -1
- data/examples/worker_prefix.rb +1 -1
- data/examples/worker_reverse_string.rb +27 -0
- data/examples/worker_reverse_to_file.rb +18 -0
- data/examples/{evented_worker.rb → worker_signals.rb} +18 -8
- data/lib/gearman.rb +68 -21
- data/lib/gearman/client.rb +137 -65
- data/lib/gearman/server.rb +4 -4
- data/lib/gearman/task.rb +140 -20
- data/lib/gearman/taskset.rb +280 -5
- data/lib/gearman/testlib.rb +95 -0
- data/lib/gearman/util.rb +184 -28
- data/lib/gearman/worker.rb +356 -20
- data/test/client_test.rb +145 -0
- data/test/mock_client_test.rb +629 -0
- data/test/mock_worker_test.rb +321 -0
- data/test/util_test.rb +8 -3
- data/test/worker_test.rb +50 -34
- metadata +41 -41
- data/examples/client_echo.rb +0 -16
- data/examples/evented_client.rb +0 -23
- data/examples/worker_echo.rb +0 -20
- data/examples/worker_echo_pprof.rb +0 -5
- data/gearman-ruby.gemspec +0 -111
- data/lib/gearman/evented/client.rb +0 -99
- data/lib/gearman/evented/reactor.rb +0 -86
- data/lib/gearman/evented/worker.rb +0 -118
- data/lib/gearman/job.rb +0 -38
- data/lib/gearman/protocol.rb +0 -110
- data/test/basic_integration_test.rb +0 -121
- data/test/crash_test.rb +0 -69
- data/test/job_test.rb +0 -30
- data/test/protocol_test.rb +0 -132
- data/test/test_helper.rb +0 -31
@@ -0,0 +1,95 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
require 'thread'
|
5
|
+
|
6
|
+
class FakeJobServer
|
7
|
+
def initialize(tester,port=nil)
|
8
|
+
@tester = tester
|
9
|
+
@serv = TCPserver.open(0) if port.nil?
|
10
|
+
@serv = TCPserver.open('localhost',port) unless port.nil?
|
11
|
+
@port = @serv.addr[1]
|
12
|
+
end
|
13
|
+
attr_reader :port
|
14
|
+
|
15
|
+
def server_socket
|
16
|
+
@serv
|
17
|
+
end
|
18
|
+
|
19
|
+
def stop
|
20
|
+
@serv.close
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
@serv = TCPserver.open(@port)
|
25
|
+
end
|
26
|
+
|
27
|
+
def expect_connection
|
28
|
+
sock = @serv.accept
|
29
|
+
return sock
|
30
|
+
end
|
31
|
+
|
32
|
+
def expect_closed(sock)
|
33
|
+
@tester.assert_true(sock.closed?)
|
34
|
+
end
|
35
|
+
|
36
|
+
def expect_request(sock, exp_type, exp_data='', size=12)
|
37
|
+
head = sock.recv(size)
|
38
|
+
magic, type, len = head.unpack('a4NN')
|
39
|
+
@tester.assert("\0REQ" == magic || "\000REQ" == magic)
|
40
|
+
@tester.assert_equal(Gearman::Util::NUMS[exp_type.to_sym], type)
|
41
|
+
data = len > 0 ? sock.recv(len) : ''
|
42
|
+
@tester.assert_equal(exp_data, data)
|
43
|
+
end
|
44
|
+
|
45
|
+
def expect_any_request(sock)
|
46
|
+
head = sock.recv(12)
|
47
|
+
end
|
48
|
+
|
49
|
+
def expect_anything_and_close_socket(sock)
|
50
|
+
head = sock.recv(12)
|
51
|
+
sock.close
|
52
|
+
end
|
53
|
+
|
54
|
+
def send_response(sock, type, data='', bogus_size=nil)
|
55
|
+
type_num = Gearman::Util::NUMS[type.to_sym] || 0
|
56
|
+
response = "\0RES" + [type_num, (bogus_size or data.size)].pack('NN') + data
|
57
|
+
sock.write(response)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
class TestScript
|
62
|
+
def initialize
|
63
|
+
@mutex = Mutex.new
|
64
|
+
@cv = ConditionVariable.new
|
65
|
+
@blocks = []
|
66
|
+
end
|
67
|
+
|
68
|
+
def loop_forever
|
69
|
+
loop do
|
70
|
+
f = nil
|
71
|
+
@mutex.synchronize do
|
72
|
+
@cv.wait(@mutex) if @blocks.empty?
|
73
|
+
f = @blocks[0] if not @blocks.empty?
|
74
|
+
end
|
75
|
+
f.call if f
|
76
|
+
@mutex.synchronize do
|
77
|
+
@blocks.shift
|
78
|
+
@cv.signal if @blocks.empty?
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def exec(&f)
|
84
|
+
@mutex.synchronize do
|
85
|
+
@blocks << f
|
86
|
+
@cv.signal
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def wait
|
91
|
+
@mutex.synchronize do
|
92
|
+
@cv.wait(@mutex) if not @blocks.empty?
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
data/lib/gearman/util.rb
CHANGED
@@ -2,51 +2,207 @@
|
|
2
2
|
|
3
3
|
require 'socket'
|
4
4
|
require 'time'
|
5
|
+
require 'logger'
|
5
6
|
|
6
7
|
module Gearman
|
7
8
|
|
8
9
|
class ServerDownException < Exception; end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
11
|
+
# = Util
|
12
|
+
#
|
13
|
+
# == Description
|
14
|
+
# Static helper methods and data used by other classes.
|
15
|
+
class Util
|
16
|
+
# Map from Integer representations of commands used in the network
|
17
|
+
# protocol to more-convenient symbols.
|
18
|
+
COMMANDS = {
|
19
|
+
1 => :can_do, # W->J: FUNC
|
20
|
+
2 => :cant_do, # W->J: FUNC
|
21
|
+
3 => :reset_abilities, # W->J: --
|
22
|
+
4 => :pre_sleep, # W->J: --
|
23
|
+
#5 => (unused), # - -
|
24
|
+
6 => :noop, # J->W: --
|
25
|
+
7 => :submit_job, # C->J: FUNC[0]UNIQ[0]ARGS
|
26
|
+
8 => :job_created, # J->C: HANDLE
|
27
|
+
9 => :grab_job, # W->J: --
|
28
|
+
10 => :no_job, # J->W: --
|
29
|
+
11 => :job_assign, # J->W: HANDLE[0]FUNC[0]ARG
|
30
|
+
12 => :work_status, # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
|
31
|
+
13 => :work_complete, # W->J/C: HANDLE[0]RES
|
32
|
+
14 => :work_fail, # W->J/C: HANDLE
|
33
|
+
15 => :get_status, # C->J: HANDLE
|
34
|
+
16 => :echo_req, # ?->J: TEXT
|
35
|
+
17 => :echo_res, # J->?: TEXT
|
36
|
+
18 => :submit_job_bg, # C->J: FUNC[0]UNIQ[0]ARGS
|
37
|
+
19 => :error, # J->?: ERRCODE[0]ERR_TEXT
|
38
|
+
20 => :status_res, # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
|
39
|
+
21 => :submit_job_high, # C->J: FUNC[0]UNIQ[0]ARGS
|
40
|
+
22 => :set_client_id, # W->J: [RANDOM_STRING_NO_WHITESPACE]
|
41
|
+
23 => :can_do_timeout, # W->J: FUNC[0]TIMEOUT
|
42
|
+
24 => :all_yours, # REQ Worker
|
43
|
+
25 => :work_exception, # W->J: HANDLE[0]ARG
|
44
|
+
26 => :option_req, # C->J: TEXT
|
45
|
+
27 => :option_res, # J->C: TEXT
|
46
|
+
28 => :work_data, # REQ Worker
|
47
|
+
29 => :work_warning, # W->J/C: HANDLE[0]MSG
|
48
|
+
30 => :grab_job_uniq, # REQ Worker
|
49
|
+
31 => :job_assign_uniq, # RES Worker
|
50
|
+
32 => :submit_job_high_bg, # C->J: FUNC[0]UNIQ[0]ARGS
|
51
|
+
33 => :submit_job_low, # C->J: FUNC[0]UNIQ[0]ARGS
|
52
|
+
34 => :submit_job_low_bg, # C->J: FUNC[0]UNIQ[0]ARGS
|
53
|
+
35 => :submit_job_sched, # REQ Client
|
54
|
+
36 => :submit_job_epoch # C->J: FUNC[0]UNIQ[0]EPOCH[0]ARGS
|
55
|
+
}
|
56
|
+
|
57
|
+
# Map e.g. 'can_do' => 1
|
58
|
+
NUMS = COMMANDS.invert
|
59
|
+
|
60
|
+
# Default job server port.
|
61
|
+
DEFAULT_PORT = 4730
|
62
|
+
|
63
|
+
def Util.logger=(logger)
|
64
|
+
@logger = logger
|
65
|
+
end
|
66
|
+
|
67
|
+
def Util.logger
|
68
|
+
@logger ||=
|
69
|
+
begin
|
70
|
+
l = Logger.new($stdout)
|
71
|
+
l.level = Logger::FATAL
|
72
|
+
l
|
73
|
+
end
|
74
|
+
end
|
15
75
|
|
16
|
-
|
76
|
+
##
|
77
|
+
# Construct a request packet.
|
78
|
+
#
|
79
|
+
# @param type_name command type's name (see COMMANDS)
|
80
|
+
# @param arg optional data to pack into the command
|
81
|
+
# @return packet (as a string)
|
82
|
+
def Util.pack_request(type_name, arg='')
|
83
|
+
type_num = NUMS[type_name.to_sym]
|
84
|
+
raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num
|
85
|
+
arg = '' if not arg
|
86
|
+
"\0REQ" + [type_num, arg.size].pack('NN') + arg
|
87
|
+
end
|
17
88
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
89
|
+
##
|
90
|
+
# Return a Task based on the passed-in arguments.
|
91
|
+
#
|
92
|
+
# @param args either a single Task object or the arguments accepted by
|
93
|
+
# Task.new
|
94
|
+
# @return Task object
|
95
|
+
def Util.get_task_from_args(*args)
|
96
|
+
if (args[0].class == Task || args[0].class.superclass == Task)
|
97
|
+
return args[0]
|
98
|
+
elsif args.size <= 3
|
99
|
+
return Task.new(*args)
|
100
|
+
else
|
101
|
+
raise InvalidArgsError, 'Incorrect number of args to get_task_from_args'
|
24
102
|
end
|
103
|
+
end
|
25
104
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
105
|
+
##
|
106
|
+
# Read from a socket, giving up if it doesn't finish quickly enough.
|
107
|
+
# NetworkError is thrown if we don't read all the bytes in time.
|
108
|
+
#
|
109
|
+
# @param sock Socket from which we read
|
110
|
+
# @param len number of bytes to read
|
111
|
+
# @param timeout maximum number of seconds we'll take; nil for no timeout
|
112
|
+
# @return full data that was read
|
113
|
+
def Util.timed_recv(sock, len, timeout=nil)
|
114
|
+
data = ''
|
115
|
+
end_time = Time.now.to_f + timeout if timeout
|
116
|
+
while data.size < len and (not timeout or Time.now.to_f < end_time) do
|
117
|
+
IO::select([sock], nil, nil, timeout ? end_time - Time.now.to_f : nil) \
|
118
|
+
or break
|
119
|
+
data += sock.readpartial(len - data.size)
|
32
120
|
end
|
121
|
+
if data.size < len
|
122
|
+
raise NetworkError, "Read #{data.size} byte(s) instead of #{len}"
|
123
|
+
end
|
124
|
+
data
|
125
|
+
end
|
126
|
+
|
127
|
+
##
|
128
|
+
# Read a response packet from a socket.
|
129
|
+
#
|
130
|
+
# @param sock Socket connected to a job server
|
131
|
+
# @param timeout timeout in seconds, nil for no timeout
|
132
|
+
# @return array consisting of integer packet type and data
|
133
|
+
def Util.read_response(sock, timeout=nil)
|
134
|
+
#debugger
|
135
|
+
end_time = Time.now.to_f + timeout if timeout
|
136
|
+
head = timed_recv(sock, 12, timeout)
|
137
|
+
magic, type, len = head.unpack('a4NN')
|
138
|
+
raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES"
|
139
|
+
buf = len > 0 ?
|
140
|
+
timed_recv(sock, len, timeout ? end_time - Time.now.to_f : nil) : ''
|
141
|
+
type = COMMANDS[type]
|
142
|
+
raise ProtocolError, "Invalid packet type #{type}" unless type
|
143
|
+
[type, buf]
|
144
|
+
end
|
33
145
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
146
|
+
##
|
147
|
+
# Send a request packet over a socket.
|
148
|
+
#
|
149
|
+
# @param sock Socket connected to a job server
|
150
|
+
# @param req request packet to send
|
151
|
+
def Util.send_request(sock, req)
|
152
|
+
len = with_safe_socket_op{ sock.write(req) }
|
153
|
+
if len != req.size
|
154
|
+
raise NetworkError, "Wrote #{len} instead of #{req.size}"
|
40
155
|
end
|
156
|
+
end
|
41
157
|
|
42
|
-
|
43
|
-
|
158
|
+
##
|
159
|
+
# Add default ports to a job server or list of servers.
|
160
|
+
#
|
161
|
+
# @param servers a server hostname or "host:port" or array of servers
|
162
|
+
# @return an array of "host:port" strings
|
163
|
+
def Util.normalize_job_servers(servers)
|
164
|
+
if servers.class == String or servers.class == Symbol
|
165
|
+
servers = [ servers.to_s ]
|
44
166
|
end
|
167
|
+
servers.map {|s| s =~ /:/ ? s : "#{s}:#{DEFAULT_PORT}" }
|
168
|
+
end
|
169
|
+
|
170
|
+
##
|
171
|
+
# Convert job server info and a handle into a string.
|
172
|
+
#
|
173
|
+
# @param hostport "host:port" of job server
|
174
|
+
# @param handle job server-returned handle for a task
|
175
|
+
# @return "host:port//handle"
|
176
|
+
def Util.handle_to_str(hostport, handle)
|
177
|
+
"#{hostport}//#{handle}"
|
178
|
+
end
|
45
179
|
|
46
|
-
|
47
|
-
|
180
|
+
##
|
181
|
+
# Reverse Util.handle_to_str.
|
182
|
+
#
|
183
|
+
# @param str "host:port//handle"
|
184
|
+
# @return [hostport, handle]
|
185
|
+
def Util.str_to_handle(str)
|
186
|
+
str =~ %r{^([^:]+:\d+)//(.+)}
|
187
|
+
return [$1, $3]
|
188
|
+
end
|
189
|
+
|
190
|
+
def self.with_safe_socket_op
|
191
|
+
begin
|
192
|
+
yield
|
193
|
+
rescue Exception => ex
|
194
|
+
raise ServerDownException.new(ex.message)
|
48
195
|
end
|
196
|
+
end
|
49
197
|
|
198
|
+
def Util.ability_name_with_prefix(prefix,name)
|
199
|
+
"#{prefix}\t#{name}"
|
50
200
|
end
|
51
201
|
|
202
|
+
class << self
|
203
|
+
alias :ability_name_for_perl :ability_name_with_prefix
|
204
|
+
end
|
205
|
+
|
206
|
+
end
|
207
|
+
|
52
208
|
end
|
data/lib/gearman/worker.rb
CHANGED
@@ -1,39 +1,375 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'set'
|
4
|
+
require 'socket'
|
5
|
+
require 'thread'
|
6
|
+
|
1
7
|
module Gearman
|
2
|
-
class Worker
|
3
8
|
|
4
|
-
|
9
|
+
# = Worker
|
10
|
+
#
|
11
|
+
# == Description
|
12
|
+
# A worker that can connect to a Gearman server and perform tasks.
|
13
|
+
#
|
14
|
+
# == Usage
|
15
|
+
# require 'gearman'
|
16
|
+
#
|
17
|
+
# w = Gearman::Worker.new('127.0.0.1')
|
18
|
+
#
|
19
|
+
# # Add a handler for a "sleep" function that takes a single argument, the
|
20
|
+
# # number of seconds to sleep before reporting success.
|
21
|
+
# w.add_ability('sleep') do |data,job|
|
22
|
+
# seconds = data
|
23
|
+
# (1..seconds.to_i).each do |i|
|
24
|
+
# sleep 1
|
25
|
+
# # Report our progress to the job server every second.
|
26
|
+
# job.report_status(i, seconds)
|
27
|
+
# end
|
28
|
+
# # Report success.
|
29
|
+
# true
|
30
|
+
# end
|
31
|
+
# loop { w.work }
|
32
|
+
class Worker
|
33
|
+
# = Ability
|
34
|
+
#
|
35
|
+
# == Description
|
36
|
+
# Information about an ability that we possess.
|
37
|
+
class Ability
|
38
|
+
##
|
39
|
+
# Create a new ability.
|
40
|
+
#
|
41
|
+
# @param block code to run
|
42
|
+
# @param timeout server gives up on us after this many seconds
|
43
|
+
def initialize(block, timeout=nil)
|
44
|
+
@block = block
|
45
|
+
@timeout = timeout
|
46
|
+
end
|
47
|
+
attr_reader :timeout
|
5
48
|
|
6
|
-
|
7
|
-
|
8
|
-
|
49
|
+
##
|
50
|
+
# Run the block of code.
|
51
|
+
#
|
52
|
+
# @param data data passed to us by a client
|
53
|
+
# @param job interface to report job information to the server
|
54
|
+
def run(data, job)
|
55
|
+
@block.call(data, job)
|
56
|
+
end
|
57
|
+
end
|
9
58
|
|
10
|
-
|
59
|
+
# = Job
|
60
|
+
#
|
61
|
+
# == Description
|
62
|
+
# Interface to allow a worker to report information to a job server.
|
63
|
+
class Job
|
64
|
+
##
|
65
|
+
# Create a new Job.
|
66
|
+
#
|
67
|
+
# @param sock Socket connected to job server
|
68
|
+
# @param handle job server-supplied job handle
|
69
|
+
def initialize(sock, handle)
|
70
|
+
@socket = sock
|
71
|
+
@handle = handle
|
72
|
+
end
|
11
73
|
|
12
|
-
|
74
|
+
##
|
75
|
+
# Report our status to the job server.
|
76
|
+
def report_status(numerator, denominator)
|
77
|
+
req = Util.pack_request(
|
78
|
+
:work_status, "#{@handle}\0#{numerator}\0#{denominator}")
|
79
|
+
Util.send_request(@socket, req)
|
80
|
+
self
|
13
81
|
end
|
14
82
|
|
15
|
-
|
16
|
-
|
17
|
-
|
83
|
+
##
|
84
|
+
# Send data before job completes
|
85
|
+
def send_data(data)
|
86
|
+
req = Util.pack_request(:work_data, "#{@handle}\0#{data}")
|
87
|
+
Util.send_request(@socket, req)
|
88
|
+
self
|
18
89
|
end
|
19
90
|
|
20
|
-
|
21
|
-
|
91
|
+
##
|
92
|
+
# Send a warning explicitly
|
93
|
+
def report_warning(warning)
|
94
|
+
req = Util.pack_request(:work_warning, "#{@handle}\0#{warning}")
|
95
|
+
Util.send_request(@socket, req)
|
96
|
+
self
|
22
97
|
end
|
98
|
+
end
|
23
99
|
|
24
|
-
|
25
|
-
|
100
|
+
##
|
101
|
+
# Create a new worker.
|
102
|
+
#
|
103
|
+
# @param job_servers "host:port"; either a single server or an array
|
104
|
+
# @param opts hash of additional options
|
105
|
+
def initialize(job_servers=nil, opts={})
|
106
|
+
chars = ('a'..'z').to_a
|
107
|
+
@client_id = Array.new(30) { chars[rand(chars.size)] }.join
|
108
|
+
@sockets = {} # "host:port" -> Socket
|
109
|
+
@abilities = {} # "funcname" -> Ability
|
110
|
+
@bad_servers = [] # "host:port"
|
111
|
+
@servers_mutex = Mutex.new
|
112
|
+
%w{client_id reconnect_sec
|
113
|
+
network_timeout_sec}.map {|s| s.to_sym }.each do |k|
|
114
|
+
instance_variable_set "@#{k}", opts[k]
|
115
|
+
opts.delete k
|
116
|
+
end
|
117
|
+
if opts.size > 0
|
118
|
+
raise InvalidArgsError,
|
119
|
+
'Invalid worker args: ' + opts.keys.sort.join(', ')
|
26
120
|
end
|
121
|
+
@reconnect_sec = 30 if not @reconnect_sec
|
122
|
+
@network_timeout_sec = 5 if not @network_timeout_sec
|
123
|
+
@worker_enabled = true
|
124
|
+
@status = :preparing
|
125
|
+
self.job_servers = job_servers if job_servers
|
126
|
+
start_reconnect_thread
|
127
|
+
end
|
128
|
+
attr_accessor :client_id, :reconnect_sec, :network_timeout_sec, :bad_servers, :worker_enabled, :status
|
27
129
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
130
|
+
# Start a thread to repeatedly attempt to connect to down job servers.
|
131
|
+
def start_reconnect_thread
|
132
|
+
Thread.new do
|
133
|
+
loop do
|
134
|
+
@servers_mutex.synchronize do
|
135
|
+
# If there are any failed servers, try to reconnect to them.
|
136
|
+
if not @bad_servers.empty?
|
137
|
+
update_job_servers(@sockets.keys + @bad_servers)
|
138
|
+
end
|
34
139
|
end
|
140
|
+
sleep @reconnect_sec
|
35
141
|
end
|
142
|
+
end.run
|
143
|
+
end
|
144
|
+
|
145
|
+
def job_servers
|
146
|
+
servers = nil
|
147
|
+
@servers_mutex.synchronize do
|
148
|
+
servers = @sockets.keys + @bad_servers
|
149
|
+
end
|
150
|
+
servers
|
151
|
+
end
|
152
|
+
|
153
|
+
##
|
154
|
+
# Connect to job servers to be used by this worker.
|
155
|
+
#
|
156
|
+
# @param servers "host:port"; either a single server or an array
|
157
|
+
def job_servers=(servers)
|
158
|
+
@servers_mutex.synchronize do
|
159
|
+
update_job_servers(servers)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# Internal function to actually connect to servers.
|
164
|
+
# Caller must acquire @servers_mutex before calling us.
|
165
|
+
#
|
166
|
+
# @param servers "host:port"; either a single server or an array
|
167
|
+
def update_job_servers(servers)
|
168
|
+
@bad_servers = []
|
169
|
+
servers = Set.new(Util.normalize_job_servers(servers))
|
170
|
+
# Disconnect from servers that we no longer care about.
|
171
|
+
@sockets.each do |server,sock|
|
172
|
+
if not servers.include? server
|
173
|
+
Util.logger.debug "GearmanRuby: Disconnecting from old server #{server}"
|
174
|
+
sock.close
|
175
|
+
@sockets.delete(server)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
# Connect to new servers.
|
179
|
+
servers.each do |server|
|
180
|
+
if not @sockets[server]
|
181
|
+
begin
|
182
|
+
Util.logger.debug "GearmanRuby: Connecting to server #{server}"
|
183
|
+
@sockets[server] = connect(server)
|
184
|
+
rescue NetworkError
|
185
|
+
@bad_servers << server
|
186
|
+
Util.logger.debug "GearmanRuby: Unable to connect to #{server}"
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
private :update_job_servers
|
192
|
+
|
193
|
+
##
|
194
|
+
# Connect to a job server.
|
195
|
+
#
|
196
|
+
# @param hostport "hostname:port"
|
197
|
+
def connect(hostport)
|
198
|
+
begin
|
199
|
+
# FIXME: handle timeouts
|
200
|
+
sock = TCPSocket.new(*hostport.split(':'))
|
201
|
+
rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH
|
202
|
+
raise NetworkError
|
203
|
+
rescue Exception => e
|
204
|
+
Util.logger.debug "GearmanRuby: Unhandled exception while connecting to #{hostport} : #{e} (raising NetworkError exception)"
|
205
|
+
raise NetworkError
|
206
|
+
end
|
207
|
+
# FIXME: catch exceptions; do something smart
|
208
|
+
Util.send_request(sock, Util.pack_request(:set_client_id, @client_id))
|
209
|
+
@abilities.each {|f,a| announce_ability(sock, f, a.timeout) }
|
210
|
+
sock
|
211
|
+
end
|
212
|
+
private :connect
|
213
|
+
|
214
|
+
##
|
215
|
+
# Announce an ability over a particular socket.
|
216
|
+
#
|
217
|
+
# @param sock Socket connect to a job server
|
218
|
+
# @param func function name (including prefix)
|
219
|
+
# @param timeout the server will give up on us if we don't finish
|
220
|
+
# a task in this many seconds
|
221
|
+
def announce_ability(sock, func, timeout=nil)
|
222
|
+
begin
|
223
|
+
cmd = timeout ? :can_do_timeout : :can_do
|
224
|
+
arg = timeout ? "#{func}\0#{timeout.to_s}" : func
|
225
|
+
Util.send_request(sock, Util.pack_request(cmd, arg))
|
226
|
+
rescue Exception => ex
|
227
|
+
bad_servers << @sockets.keys.detect{|hp| @sockets[hp] == sock}
|
228
|
+
end
|
229
|
+
end
|
230
|
+
private :announce_ability
|
231
|
+
|
232
|
+
##
|
233
|
+
# Add a new ability, announcing it to job servers.
|
234
|
+
#
|
235
|
+
# The passed-in block of code will be executed for jobs of this function
|
236
|
+
# type. It'll receive two arguments, the data supplied by the client and
|
237
|
+
# a Job object. If it returns nil or false, the server will be informed
|
238
|
+
# that the job has failed; otherwise the return value of the block will
|
239
|
+
# be passed back to the client in String form.
|
240
|
+
#
|
241
|
+
# @param func function name (without prefix)
|
242
|
+
# @param timeout the server will give up on us if we don't finish
|
243
|
+
# a task in this many seconds
|
244
|
+
def add_ability(func, timeout=nil, &f)
|
245
|
+
@abilities[func] = Ability.new(f, timeout)
|
246
|
+
@sockets.values.each {|s| announce_ability(s, func, timeout) }
|
247
|
+
end
|
248
|
+
|
249
|
+
##
|
250
|
+
# Let job servers know that we're no longer able to do something.
|
251
|
+
#
|
252
|
+
# @param func function name
|
253
|
+
def remove_ability(func)
|
254
|
+
@abilities.delete(func)
|
255
|
+
req = Util.pack_request(:cant_do, func)
|
256
|
+
@sockets.values.each {|s| Util.send_request(s, req) }
|
257
|
+
end
|
258
|
+
|
259
|
+
##
|
260
|
+
# Handle a job_assign packet.
|
261
|
+
#
|
262
|
+
# @param data data in the packet
|
263
|
+
# @param sock Socket on which the packet arrived
|
264
|
+
# @param hostport "host:port"
|
265
|
+
def handle_job_assign(data, sock, hostport)
|
266
|
+
handle, func, data = data.split("\0", 3)
|
267
|
+
if not func
|
268
|
+
Util.logger.error "GearmanRuby: Ignoring job_assign with no function from #{hostport}"
|
269
|
+
return false
|
270
|
+
end
|
271
|
+
|
272
|
+
Util.logger.error "GearmanRuby: Got job_assign with handle #{handle} and #{data.size} byte(s) " +
|
273
|
+
"from #{hostport}"
|
274
|
+
|
275
|
+
ability = @abilities[func]
|
276
|
+
if not ability
|
277
|
+
Util.logger.error "Ignoring job_assign for unsupported func #{func} " +
|
278
|
+
"with handle #{handle} from #{hostport}"
|
279
|
+
Util.send_request(sock, Util.pack_request(:work_fail, handle))
|
280
|
+
return false
|
36
281
|
end
|
37
282
|
|
283
|
+
exception = nil
|
284
|
+
begin
|
285
|
+
ret = ability.run(data, Job.new(sock, handle))
|
286
|
+
rescue Exception => e
|
287
|
+
exception = e
|
288
|
+
end
|
289
|
+
|
290
|
+
|
291
|
+
cmd = if ret && exception.nil?
|
292
|
+
ret = ret.to_s
|
293
|
+
Util.logger.debug "GearmanRuby: Sending work_complete for #{handle} with #{ret.size} byte(s) " +
|
294
|
+
"to #{hostport}"
|
295
|
+
[ Util.pack_request(:work_complete, "#{handle}\0#{ret}") ]
|
296
|
+
elsif exception.nil?
|
297
|
+
Util.logger.debug "GearmanRuby: Sending work_fail for #{handle} to #{hostport}"
|
298
|
+
[ Util.pack_request(:work_fail, handle) ]
|
299
|
+
elsif exception
|
300
|
+
Util.logger.debug "GearmanRuby: Sending work_exception for #{handle} to #{hostport}"
|
301
|
+
[ Util.pack_request(:work_exception, "#{handle}\0#{exception.message}") ]
|
302
|
+
end
|
303
|
+
|
304
|
+
cmd.each {|p| Util.send_request(sock, p) }
|
305
|
+
true
|
38
306
|
end
|
307
|
+
|
308
|
+
##
|
309
|
+
# Do a single job and return.
|
310
|
+
def work
|
311
|
+
req = Util.pack_request(:grab_job)
|
312
|
+
loop do
|
313
|
+
@status = :preparing
|
314
|
+
bad_servers = []
|
315
|
+
# We iterate through the servers in sorted order to make testing
|
316
|
+
# easier.
|
317
|
+
servers = nil
|
318
|
+
@servers_mutex.synchronize { servers = @sockets.keys.sort }
|
319
|
+
servers.each do |hostport|
|
320
|
+
Util.logger.debug "GearmanRuby: Sending grab_job to #{hostport}"
|
321
|
+
sock = @sockets[hostport]
|
322
|
+
Util.send_request(sock, req)
|
323
|
+
|
324
|
+
# Now that we've sent grab_job, we need to keep reading packets
|
325
|
+
# until we see a no_job or job_assign response (there may be a noop
|
326
|
+
# waiting for us in response to a previous pre_sleep).
|
327
|
+
loop do
|
328
|
+
begin
|
329
|
+
type, data = Util.read_response(sock, @network_timeout_sec)
|
330
|
+
case type
|
331
|
+
when :no_job
|
332
|
+
Util.logger.debug "GearmanRuby: Got no_job from #{hostport}"
|
333
|
+
break
|
334
|
+
when :job_assign
|
335
|
+
@status = :working
|
336
|
+
return worker_enabled if handle_job_assign(data, sock, hostport)
|
337
|
+
break
|
338
|
+
else
|
339
|
+
Util.logger.debug "GearmanRuby: Got #{type.to_s} from #{hostport}"
|
340
|
+
end
|
341
|
+
rescue Exception
|
342
|
+
Util.logger.debug "GearmanRuby: Server #{hostport} timed out or lost connection (#{$!.inspect}); marking bad"
|
343
|
+
bad_servers << hostport
|
344
|
+
break
|
345
|
+
end
|
346
|
+
end
|
347
|
+
end
|
348
|
+
|
349
|
+
@servers_mutex.synchronize do
|
350
|
+
bad_servers.each do |hostport|
|
351
|
+
@sockets[hostport].close if @sockets[hostport]
|
352
|
+
@bad_servers << hostport if @sockets[hostport]
|
353
|
+
@sockets.delete(hostport)
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
Util.logger.debug "GearmanRuby: Sending pre_sleep and going to sleep for #{@reconnect_sec} sec"
|
358
|
+
@servers_mutex.synchronize do
|
359
|
+
@sockets.values.each do |sock|
|
360
|
+
Util.send_request(sock, Util.pack_request(:pre_sleep))
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
return false unless worker_enabled
|
365
|
+
@status = :waiting
|
366
|
+
|
367
|
+
# FIXME: We could optimize things the next time through the 'each' by
|
368
|
+
# sending the first grab_job to one of the servers that had a socket
|
369
|
+
# with data in it. Not bothering with it for now.
|
370
|
+
IO::select(@sockets.values, nil, nil, @reconnect_sec)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
end
|
374
|
+
|
39
375
|
end
|