gearman-ruby 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/HOWTO +146 -0
- data/LICENSE +20 -0
- data/README +9 -0
- data/Rakefile +41 -0
- data/TODO +8 -0
- data/VERSION.yml +4 -0
- data/examples/calculus_client.rb +41 -0
- data/examples/calculus_worker.rb +42 -0
- data/examples/client.rb +19 -0
- data/examples/client_background.rb +14 -0
- data/examples/client_data.rb +16 -0
- data/examples/client_echo.rb +16 -0
- data/examples/client_exception.rb +17 -0
- data/examples/client_prefix.rb +15 -0
- data/examples/evented_client.rb +23 -0
- data/examples/evented_worker.rb +26 -0
- data/examples/gearman_environment.sh +25 -0
- data/examples/scale_image.rb +30 -0
- data/examples/scale_image_worker.rb +34 -0
- data/examples/server.rb +15 -0
- data/examples/worker.rb +23 -0
- data/examples/worker_data.rb +16 -0
- data/examples/worker_echo.rb +20 -0
- data/examples/worker_echo_pprof.rb +5 -0
- data/examples/worker_exception.rb +14 -0
- data/examples/worker_prefix.rb +25 -0
- data/gearman-ruby.gemspec +111 -0
- data/lib/gearman.rb +29 -0
- data/lib/gearman/client.rb +80 -0
- data/lib/gearman/evented/client.rb +99 -0
- data/lib/gearman/evented/reactor.rb +86 -0
- data/lib/gearman/evented/worker.rb +118 -0
- data/lib/gearman/job.rb +38 -0
- data/lib/gearman/protocol.rb +110 -0
- data/lib/gearman/server.rb +94 -0
- data/lib/gearman/task.rb +99 -0
- data/lib/gearman/taskset.rb +11 -0
- data/lib/gearman/util.rb +52 -0
- data/lib/gearman/worker.rb +39 -0
- data/test/basic_integration_test.rb +121 -0
- data/test/crash_test.rb +69 -0
- data/test/job_test.rb +30 -0
- data/test/protocol_test.rb +132 -0
- data/test/test_helper.rb +31 -0
- data/test/util_test.rb +12 -0
- data/test/worker_test.rb +45 -0
- metadata +133 -0
@@ -0,0 +1,94 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
require 'gearman'
|
5
|
+
|
6
|
+
module Gearman
|
7
|
+
|
8
|
+
# = Server
|
9
|
+
#
|
10
|
+
# == Description
|
11
|
+
# A client for managing Gearman job servers.
|
12
|
+
class Server
|
13
|
+
##
|
14
|
+
# Create a new client.
|
15
|
+
#
|
16
|
+
# @param job_servers "host:port"; either a single server or an array
|
17
|
+
# @param prefix function name prefix (namespace)
|
18
|
+
def initialize(hostport)
|
19
|
+
@hostport = hostport # "host:port"
|
20
|
+
end
|
21
|
+
attr_reader :hostport
|
22
|
+
|
23
|
+
##
|
24
|
+
# Get a socket for a job server.
|
25
|
+
#
|
26
|
+
# @param hostport job server "host:port"
|
27
|
+
# @return a Socket
|
28
|
+
def socket(num_retries=3)
|
29
|
+
return @socket if @socket
|
30
|
+
num_retries.times do
|
31
|
+
begin
|
32
|
+
sock = TCPSocket.new(*hostport.split(':'))
|
33
|
+
rescue Exception
|
34
|
+
else
|
35
|
+
return @socket = sock
|
36
|
+
end
|
37
|
+
end
|
38
|
+
raise RuntimeError, "Unable to connect to job server #{hostport}"
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# Sends a command to the server.
|
43
|
+
#
|
44
|
+
# @return a response string
|
45
|
+
def send_command(name)
|
46
|
+
response = ''
|
47
|
+
socket.puts(name)
|
48
|
+
while true do
|
49
|
+
if buf = socket.recv_nonblock(65536) rescue nil
|
50
|
+
response << buf
|
51
|
+
return response if response =~ /\n.\n$/
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
##
|
57
|
+
# Returns results of a 'status' command.
|
58
|
+
#
|
59
|
+
# @return a hash of abilities with queued, active and workers keys.
|
60
|
+
def status
|
61
|
+
status = {}
|
62
|
+
if response = send_command('status')
|
63
|
+
response.split("\n").each do |line|
|
64
|
+
if line.match /^([A-Za-z_]+)\t([A-Za-z_]+)\t(\d+)\t(\d+)\t(\d+)$/
|
65
|
+
(status[$1] ||= {})[$2] = { :queue => $3, :active => $4, :workers => $5 }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
status
|
70
|
+
end
|
71
|
+
|
72
|
+
##
|
73
|
+
# Returns results of a 'workers' command.
|
74
|
+
#
|
75
|
+
# @return an array of worker hashes, containing host, status and functions keys.
|
76
|
+
def workers
|
77
|
+
workers = []
|
78
|
+
if response = send_command('workers')
|
79
|
+
response.split("\n").each do |line|
|
80
|
+
if line.match /^(\d+)\s([a-z0-9\:\.]+)\s([^\s]*)\s:\s([a-z_\s\t]+)$/
|
81
|
+
func_parts = $4.split(' ')
|
82
|
+
functions = []
|
83
|
+
while !func_parts.empty?
|
84
|
+
functions << func_parts.shift << '.' << func_parts.shift
|
85
|
+
end
|
86
|
+
workers << { :host => $2, :status => $3, :functions => functions }
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
workers
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
data/lib/gearman/task.rb
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
module Gearman
|
2
|
+
class Task
|
3
|
+
|
4
|
+
attr_reader :name, :payload, :retries_done
|
5
|
+
attr_accessor :retries, :priority, :background, :poll_status_interval
|
6
|
+
|
7
|
+
def initialize(name, payload = nil, opts = {})
|
8
|
+
@name = name.to_s
|
9
|
+
@payload = payload || ''
|
10
|
+
@priority = opts.delete(:priority).to_sym rescue nil
|
11
|
+
@background = opts.delete(:background) ? true : false
|
12
|
+
|
13
|
+
@retries_done = 0
|
14
|
+
@retries = opts.delete(:retries) || 0
|
15
|
+
|
16
|
+
@poll_status_interval = opts.delete(:poll_status_interval)
|
17
|
+
@uniq = opts.has_key?(:uuid) ? opts.delete(:uuid) : `uuidgen`.strip
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# Set a block of code to be executed when this task completes
|
22
|
+
# successfully. The returned data will be passed to the block.
|
23
|
+
def on_complete(&f)
|
24
|
+
@on_complete = f
|
25
|
+
end
|
26
|
+
|
27
|
+
##
|
28
|
+
# Set a block of code to be executed when this task fails.
|
29
|
+
def on_fail(&f)
|
30
|
+
@on_fail = f
|
31
|
+
end
|
32
|
+
|
33
|
+
##
|
34
|
+
# Set a block of code to be executed when this task is retried after
|
35
|
+
# failing. The number of retries that have been attempted (including the
|
36
|
+
# current one) will be passed to the block.
|
37
|
+
def on_retry(&f)
|
38
|
+
@on_retry = f
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# Set a block of code to be executed when a remote exception is sent by a worker.
|
43
|
+
# The block will receive the message of the exception passed from the worker.
|
44
|
+
# The user can return true for retrying or false to mark it as finished
|
45
|
+
#
|
46
|
+
# NOTE: this is actually deprecated, cf. https://bugs.launchpad.net/gearmand/+bug/405732
|
47
|
+
#
|
48
|
+
def on_exception(&f)
|
49
|
+
@on_exception = f
|
50
|
+
end
|
51
|
+
|
52
|
+
##
|
53
|
+
# Set a block of code to be executed when we receive a status update for
|
54
|
+
# this task. The block will receive two arguments, a numerator and
|
55
|
+
# denominator describing the task's status.
|
56
|
+
def on_status(&f)
|
57
|
+
@on_status = f
|
58
|
+
end
|
59
|
+
|
60
|
+
##
|
61
|
+
# Set a block of code to be executed when we receive a warning from a worker.
|
62
|
+
# It is recommended for workers to send work_warning, followed by work_fail if
|
63
|
+
# an exception occurs on their side. Don't expect this behavior from workers NOT
|
64
|
+
# using this very library ATM, though. (cf. https://bugs.launchpad.net/gearmand/+bug/405732)
|
65
|
+
def on_warning(&f)
|
66
|
+
@on_warning = f
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# Set a block of code to be executed when we receive a (partial) data packet for this task.
|
71
|
+
# The data received will be passed as an argument to the block.
|
72
|
+
def on_data(&f)
|
73
|
+
@on_data = f
|
74
|
+
end
|
75
|
+
|
76
|
+
##
|
77
|
+
# Record a failure and check whether we should be retried.
|
78
|
+
#
|
79
|
+
# @return true if we should be resubmitted; false otherwise
|
80
|
+
def should_retry?
|
81
|
+
return false if @retries_done >= @retries
|
82
|
+
@retries_done += 1
|
83
|
+
true
|
84
|
+
end
|
85
|
+
|
86
|
+
def background?
|
87
|
+
background
|
88
|
+
end
|
89
|
+
|
90
|
+
def dispatch(event, *args)
|
91
|
+
callback = instance_variable_get("@#{event}".to_sym)
|
92
|
+
callback.call(*args) if callback
|
93
|
+
end
|
94
|
+
|
95
|
+
def hash
|
96
|
+
@uniq
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/gearman/util.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
module Gearman
|
7
|
+
|
8
|
+
class ServerDownException < Exception; end
|
9
|
+
|
10
|
+
# = Util
|
11
|
+
#
|
12
|
+
# == Description
|
13
|
+
# Static helper methods and data used by other classes.
|
14
|
+
class Util
|
15
|
+
|
16
|
+
@@debug = false
|
17
|
+
|
18
|
+
##
|
19
|
+
# Enable or disable debugging output (off by default).
|
20
|
+
#
|
21
|
+
# @param v print debugging output
|
22
|
+
def Util.debug=(v)
|
23
|
+
@@debug = v
|
24
|
+
end
|
25
|
+
|
26
|
+
##
|
27
|
+
# Log a message if debugging is enabled.
|
28
|
+
#
|
29
|
+
# @param str message to log
|
30
|
+
def Util.log(str, force=false)
|
31
|
+
puts "#{Time.now.strftime '%Y-%m-%d %H:%M:%S'} #{str}" if force or @@debug
|
32
|
+
end
|
33
|
+
|
34
|
+
##
|
35
|
+
# Log a message no matter what.
|
36
|
+
#
|
37
|
+
# @param str message to log
|
38
|
+
def Util.err(str)
|
39
|
+
log(str, true)
|
40
|
+
end
|
41
|
+
|
42
|
+
def Util.ability_name_with_prefix(prefix,name)
|
43
|
+
"#{prefix}\t#{name}"
|
44
|
+
end
|
45
|
+
|
46
|
+
class << self
|
47
|
+
alias :ability_name_for_perl :ability_name_with_prefix
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Gearman
|
2
|
+
class Worker
|
3
|
+
|
4
|
+
attr_reader :abilities
|
5
|
+
|
6
|
+
def initialize(job_servers, opts = {})
|
7
|
+
@reactors = []
|
8
|
+
@abilities = {}
|
9
|
+
|
10
|
+
@job_servers = Array[*job_servers]
|
11
|
+
|
12
|
+
@opts = opts
|
13
|
+
end
|
14
|
+
|
15
|
+
def add_ability(name, timeout = nil, &f)
|
16
|
+
remove_ability(name) if @abilities.has_key?(name)
|
17
|
+
@abilities[name] = { :callback => f, :timeout => timeout }
|
18
|
+
end
|
19
|
+
|
20
|
+
def remove_ability(name)
|
21
|
+
@abilities.delete(name)
|
22
|
+
end
|
23
|
+
|
24
|
+
def has_ability?(name)
|
25
|
+
@abilities.has_key?(name)
|
26
|
+
end
|
27
|
+
|
28
|
+
def work
|
29
|
+
EM.run do
|
30
|
+
@job_servers.each do |hostport|
|
31
|
+
host, port = hostport.split(":")
|
32
|
+
opts = { :abilities => @abilities }.merge(@opts)
|
33
|
+
Gearman::Evented::WorkerReactor.connect(host, port, opts)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
class BasicIntegrationTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
start_gearmand
|
7
|
+
@client = Gearman::Client.new("localhost:4730")
|
8
|
+
@worker = Gearman::Worker.new("localhost:4730")
|
9
|
+
Thread.new { EM.run } unless EM.reactor_running?
|
10
|
+
end
|
11
|
+
|
12
|
+
def teardown
|
13
|
+
stop_gearmand
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_ping_job
|
17
|
+
response = nil
|
18
|
+
|
19
|
+
@worker.add_ability("pingpong") {|data, job| "pong" }
|
20
|
+
@worker.work
|
21
|
+
|
22
|
+
task = Gearman::Task.new("pingpong", "ping")
|
23
|
+
|
24
|
+
task.on_complete {|res| response = res }
|
25
|
+
@client.run task
|
26
|
+
|
27
|
+
assert_equal "pong", response
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_exception_in_worker
|
31
|
+
warning_given = nil
|
32
|
+
failed = false
|
33
|
+
|
34
|
+
task = Gearman::Task.new("crash", "doesntmatter")
|
35
|
+
|
36
|
+
@worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
|
37
|
+
@worker.work
|
38
|
+
|
39
|
+
task.on_warning {|warning| warning_given = warning }
|
40
|
+
task.on_fail { failed = true}
|
41
|
+
@client.run task
|
42
|
+
|
43
|
+
assert_not_nil warning_given
|
44
|
+
assert_equal true, failed
|
45
|
+
assert_equal 0, task.retries_done
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_should_be_able_to_retry_on_worker_exception
|
49
|
+
retry_callback_called = false
|
50
|
+
fail_count = 0
|
51
|
+
|
52
|
+
# Gearman::Util.debug = true
|
53
|
+
task = Gearman::Task.new("crash", "doesntmatter", :retries => 1)
|
54
|
+
|
55
|
+
@worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
|
56
|
+
@worker.work
|
57
|
+
|
58
|
+
task.on_retry {|i| retry_callback_called = true }
|
59
|
+
task.on_fail { fail_count += 1 }
|
60
|
+
@client.run task
|
61
|
+
|
62
|
+
assert_equal true, retry_callback_called
|
63
|
+
assert_equal 1, task.retries_done
|
64
|
+
assert_equal 1, fail_count
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_chunked_response
|
68
|
+
chunks_received = 0
|
69
|
+
|
70
|
+
@worker.add_ability("chunked") do |data, job|
|
71
|
+
5.times {|i| job.send_partial("chunk #{i}") }
|
72
|
+
end
|
73
|
+
@worker.work
|
74
|
+
|
75
|
+
task = Gearman::Task.new("chunked")
|
76
|
+
task.on_data do |data|
|
77
|
+
assert_match /^chunk \d/, data
|
78
|
+
chunks_received += 1
|
79
|
+
end
|
80
|
+
@client.run task
|
81
|
+
|
82
|
+
assert_equal 5, chunks_received
|
83
|
+
end
|
84
|
+
|
85
|
+
def test_background
|
86
|
+
status_received = false
|
87
|
+
|
88
|
+
@worker.add_ability("fireandforget") {|data, job| "this goes to /dev/null" }
|
89
|
+
@worker.work
|
90
|
+
|
91
|
+
task = Gearman::Task.new('fireandforget', 'background', :background => true, :poll_status_interval => 0.1)
|
92
|
+
task.on_complete {|d| flunk "on_complete should never be called for a background job!" }
|
93
|
+
task.on_status {|d| status_received = true }
|
94
|
+
@client.run task
|
95
|
+
|
96
|
+
assert_equal true, status_received
|
97
|
+
end
|
98
|
+
|
99
|
+
def test_non_blocking_run
|
100
|
+
@worker.add_ability("foo") {|data, job| "foo: #{data}" }
|
101
|
+
@worker.work
|
102
|
+
|
103
|
+
task1_complete = false
|
104
|
+
task2_complete = false
|
105
|
+
|
106
|
+
task1 = Gearman::Task.new("foo", 1)
|
107
|
+
task1.on_complete {|d| task1_complete = true}
|
108
|
+
@client.run task1, nil, true
|
109
|
+
|
110
|
+
assert_equal false, task1_complete
|
111
|
+
|
112
|
+
task2 = Gearman::Task.new("foo", 2)
|
113
|
+
task2.on_complete {|d| task2_complete = true}
|
114
|
+
@client.run task2, nil, true
|
115
|
+
|
116
|
+
Thread.new { sleep 0.1 }.join
|
117
|
+
|
118
|
+
assert_equal true, task1_complete
|
119
|
+
assert_equal true, task2_complete
|
120
|
+
end
|
121
|
+
end
|
data/test/crash_test.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
2
|
+
|
3
|
+
# Gearman::Util.debug = true
|
4
|
+
class CrashTest < Test::Unit::TestCase
|
5
|
+
|
6
|
+
def setup
|
7
|
+
teardown_gearmands
|
8
|
+
Thread.new { EM.run } unless EM.reactor_running?
|
9
|
+
end
|
10
|
+
|
11
|
+
def teardown
|
12
|
+
teardown_gearmands
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_worker_should_reconnect_if_gearmand_goes_away
|
16
|
+
start_gearmand
|
17
|
+
worker = Gearman::Worker.new("localhost:4730", :reconnect_sec => 1)
|
18
|
+
worker.add_ability("foo") {|data, job| "noop!" }
|
19
|
+
|
20
|
+
response = nil
|
21
|
+
|
22
|
+
worker.work
|
23
|
+
stop_gearmand
|
24
|
+
start_gearmand
|
25
|
+
|
26
|
+
task = Gearman::Task.new("foo", "ping")
|
27
|
+
task.on_complete {|res| response = res }
|
28
|
+
Gearman::Client.new("localhost:4730").run task
|
29
|
+
|
30
|
+
assert_equal "noop!", response
|
31
|
+
stop_gearmand
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_client_and_worker_should_use_failover_gearmand_if_primary_is_not_available
|
35
|
+
start_gearmand 4731
|
36
|
+
|
37
|
+
worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
|
38
|
+
worker.add_ability("foo") {|data, job| "noop!" }
|
39
|
+
|
40
|
+
response = nil
|
41
|
+
worker.work
|
42
|
+
task = Gearman::Task.new("foo", "ping")
|
43
|
+
task.on_complete {|res| response = res }
|
44
|
+
Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
|
45
|
+
|
46
|
+
assert_equal "noop!", response
|
47
|
+
stop_gearmand 4731
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_client_and_worker_should_switch_to_failover_gearmand_if_primary_goes_down
|
51
|
+
start_gearmand 4730
|
52
|
+
start_gearmand 4731
|
53
|
+
|
54
|
+
worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
|
55
|
+
worker.add_ability("foo") {|data, job| "noop!" }
|
56
|
+
|
57
|
+
response = nil
|
58
|
+
worker.work
|
59
|
+
|
60
|
+
stop_gearmand 4730
|
61
|
+
|
62
|
+
task = Gearman::Task.new("foo", "ping")
|
63
|
+
task.on_complete {|res| response = res }
|
64
|
+
Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
|
65
|
+
|
66
|
+
assert_equal "noop!", response
|
67
|
+
stop_gearmand 4731
|
68
|
+
end
|
69
|
+
end
|