gearman-ruby 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. data/.gitignore +1 -0
  2. data/HOWTO +146 -0
  3. data/LICENSE +20 -0
  4. data/README +9 -0
  5. data/Rakefile +41 -0
  6. data/TODO +8 -0
  7. data/VERSION.yml +4 -0
  8. data/examples/calculus_client.rb +41 -0
  9. data/examples/calculus_worker.rb +42 -0
  10. data/examples/client.rb +19 -0
  11. data/examples/client_background.rb +14 -0
  12. data/examples/client_data.rb +16 -0
  13. data/examples/client_echo.rb +16 -0
  14. data/examples/client_exception.rb +17 -0
  15. data/examples/client_prefix.rb +15 -0
  16. data/examples/evented_client.rb +23 -0
  17. data/examples/evented_worker.rb +26 -0
  18. data/examples/gearman_environment.sh +25 -0
  19. data/examples/scale_image.rb +30 -0
  20. data/examples/scale_image_worker.rb +34 -0
  21. data/examples/server.rb +15 -0
  22. data/examples/worker.rb +23 -0
  23. data/examples/worker_data.rb +16 -0
  24. data/examples/worker_echo.rb +20 -0
  25. data/examples/worker_echo_pprof.rb +5 -0
  26. data/examples/worker_exception.rb +14 -0
  27. data/examples/worker_prefix.rb +25 -0
  28. data/gearman-ruby.gemspec +111 -0
  29. data/lib/gearman.rb +29 -0
  30. data/lib/gearman/client.rb +80 -0
  31. data/lib/gearman/evented/client.rb +99 -0
  32. data/lib/gearman/evented/reactor.rb +86 -0
  33. data/lib/gearman/evented/worker.rb +118 -0
  34. data/lib/gearman/job.rb +38 -0
  35. data/lib/gearman/protocol.rb +110 -0
  36. data/lib/gearman/server.rb +94 -0
  37. data/lib/gearman/task.rb +99 -0
  38. data/lib/gearman/taskset.rb +11 -0
  39. data/lib/gearman/util.rb +52 -0
  40. data/lib/gearman/worker.rb +39 -0
  41. data/test/basic_integration_test.rb +121 -0
  42. data/test/crash_test.rb +69 -0
  43. data/test/job_test.rb +30 -0
  44. data/test/protocol_test.rb +132 -0
  45. data/test/test_helper.rb +31 -0
  46. data/test/util_test.rb +12 -0
  47. data/test/worker_test.rb +45 -0
  48. metadata +133 -0
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'socket'
4
+ require 'gearman'
5
+
6
+ module Gearman
7
+
8
+ # = Server
9
+ #
10
+ # == Description
11
+ # A client for managing Gearman job servers.
12
+ class Server
13
+ ##
14
+ # Create a new client.
15
+ #
16
+ # @param job_servers "host:port"; either a single server or an array
17
+ # @param prefix function name prefix (namespace)
18
+ def initialize(hostport)
19
+ @hostport = hostport # "host:port"
20
+ end
21
+ attr_reader :hostport
22
+
23
+ ##
24
+ # Get a socket for a job server.
25
+ #
26
+ # @param hostport job server "host:port"
27
+ # @return a Socket
28
+ def socket(num_retries=3)
29
+ return @socket if @socket
30
+ num_retries.times do
31
+ begin
32
+ sock = TCPSocket.new(*hostport.split(':'))
33
+ rescue Exception
34
+ else
35
+ return @socket = sock
36
+ end
37
+ end
38
+ raise RuntimeError, "Unable to connect to job server #{hostport}"
39
+ end
40
+
41
+ ##
42
+ # Sends a command to the server.
43
+ #
44
+ # @return a response string
45
+ def send_command(name)
46
+ response = ''
47
+ socket.puts(name)
48
+ while true do
49
+ if buf = socket.recv_nonblock(65536) rescue nil
50
+ response << buf
51
+ return response if response =~ /\n.\n$/
52
+ end
53
+ end
54
+ end
55
+
56
+ ##
57
+ # Returns results of a 'status' command.
58
+ #
59
+ # @return a hash of abilities with queued, active and workers keys.
60
+ def status
61
+ status = {}
62
+ if response = send_command('status')
63
+ response.split("\n").each do |line|
64
+ if line.match /^([A-Za-z_]+)\t([A-Za-z_]+)\t(\d+)\t(\d+)\t(\d+)$/
65
+ (status[$1] ||= {})[$2] = { :queue => $3, :active => $4, :workers => $5 }
66
+ end
67
+ end
68
+ end
69
+ status
70
+ end
71
+
72
+ ##
73
+ # Returns results of a 'workers' command.
74
+ #
75
+ # @return an array of worker hashes, containing host, status and functions keys.
76
+ def workers
77
+ workers = []
78
+ if response = send_command('workers')
79
+ response.split("\n").each do |line|
80
+ if line.match /^(\d+)\s([a-z0-9\:\.]+)\s([^\s]*)\s:\s([a-z_\s\t]+)$/
81
+ func_parts = $4.split(' ')
82
+ functions = []
83
+ while !func_parts.empty?
84
+ functions << func_parts.shift << '.' << func_parts.shift
85
+ end
86
+ workers << { :host => $2, :status => $3, :functions => functions }
87
+ end
88
+ end
89
+ end
90
+ workers
91
+ end
92
+ end
93
+
94
+ end
@@ -0,0 +1,99 @@
1
+ module Gearman
2
+ class Task
3
+
4
+ attr_reader :name, :payload, :retries_done
5
+ attr_accessor :retries, :priority, :background, :poll_status_interval
6
+
7
+ def initialize(name, payload = nil, opts = {})
8
+ @name = name.to_s
9
+ @payload = payload || ''
10
+ @priority = opts.delete(:priority).to_sym rescue nil
11
+ @background = opts.delete(:background) ? true : false
12
+
13
+ @retries_done = 0
14
+ @retries = opts.delete(:retries) || 0
15
+
16
+ @poll_status_interval = opts.delete(:poll_status_interval)
17
+ @uniq = opts.has_key?(:uuid) ? opts.delete(:uuid) : `uuidgen`.strip
18
+ end
19
+
20
+ ##
21
+ # Set a block of code to be executed when this task completes
22
+ # successfully. The returned data will be passed to the block.
23
+ def on_complete(&f)
24
+ @on_complete = f
25
+ end
26
+
27
+ ##
28
+ # Set a block of code to be executed when this task fails.
29
+ def on_fail(&f)
30
+ @on_fail = f
31
+ end
32
+
33
+ ##
34
+ # Set a block of code to be executed when this task is retried after
35
+ # failing. The number of retries that have been attempted (including the
36
+ # current one) will be passed to the block.
37
+ def on_retry(&f)
38
+ @on_retry = f
39
+ end
40
+
41
+ ##
42
+ # Set a block of code to be executed when a remote exception is sent by a worker.
43
+ # The block will receive the message of the exception passed from the worker.
44
+ # The user can return true for retrying or false to mark it as finished
45
+ #
46
+ # NOTE: this is actually deprecated, cf. https://bugs.launchpad.net/gearmand/+bug/405732
47
+ #
48
+ def on_exception(&f)
49
+ @on_exception = f
50
+ end
51
+
52
+ ##
53
+ # Set a block of code to be executed when we receive a status update for
54
+ # this task. The block will receive two arguments, a numerator and
55
+ # denominator describing the task's status.
56
+ def on_status(&f)
57
+ @on_status = f
58
+ end
59
+
60
+ ##
61
+ # Set a block of code to be executed when we receive a warning from a worker.
62
+ # It is recommended for workers to send work_warning, followed by work_fail if
63
+ # an exception occurs on their side. Don't expect this behavior from workers NOT
64
+ # using this very library ATM, though. (cf. https://bugs.launchpad.net/gearmand/+bug/405732)
65
+ def on_warning(&f)
66
+ @on_warning = f
67
+ end
68
+
69
+ ##
70
+ # Set a block of code to be executed when we receive a (partial) data packet for this task.
71
+ # The data received will be passed as an argument to the block.
72
+ def on_data(&f)
73
+ @on_data = f
74
+ end
75
+
76
+ ##
77
+ # Record a failure and check whether we should be retried.
78
+ #
79
+ # @return true if we should be resubmitted; false otherwise
80
+ def should_retry?
81
+ return false if @retries_done >= @retries
82
+ @retries_done += 1
83
+ true
84
+ end
85
+
86
+ def background?
87
+ background
88
+ end
89
+
90
+ def dispatch(event, *args)
91
+ callback = instance_variable_get("@#{event}".to_sym)
92
+ callback.call(*args) if callback
93
+ end
94
+
95
+ def hash
96
+ @uniq
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,11 @@
1
+ module Gearman
2
+ class Taskset < ::Array
3
+
4
+ def self.create(task_or_taskset)
5
+ [*task_or_taskset]
6
+ end
7
+
8
+ alias :add :<<
9
+ alias :add_task :add
10
+ end
11
+ end
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'socket'
4
+ require 'time'
5
+
6
+ module Gearman
7
+
8
+ class ServerDownException < Exception; end
9
+
10
+ # = Util
11
+ #
12
+ # == Description
13
+ # Static helper methods and data used by other classes.
14
+ class Util
15
+
16
+ @@debug = false
17
+
18
+ ##
19
+ # Enable or disable debugging output (off by default).
20
+ #
21
+ # @param v print debugging output
22
+ def Util.debug=(v)
23
+ @@debug = v
24
+ end
25
+
26
+ ##
27
+ # Log a message if debugging is enabled.
28
+ #
29
+ # @param str message to log
30
+ def Util.log(str, force=false)
31
+ puts "#{Time.now.strftime '%Y-%m-%d %H:%M:%S'} #{str}" if force or @@debug
32
+ end
33
+
34
+ ##
35
+ # Log a message no matter what.
36
+ #
37
+ # @param str message to log
38
+ def Util.err(str)
39
+ log(str, true)
40
+ end
41
+
42
+ def Util.ability_name_with_prefix(prefix,name)
43
+ "#{prefix}\t#{name}"
44
+ end
45
+
46
+ class << self
47
+ alias :ability_name_for_perl :ability_name_with_prefix
48
+ end
49
+
50
+ end
51
+
52
+ end
@@ -0,0 +1,39 @@
1
+ module Gearman
2
+ class Worker
3
+
4
+ attr_reader :abilities
5
+
6
+ def initialize(job_servers, opts = {})
7
+ @reactors = []
8
+ @abilities = {}
9
+
10
+ @job_servers = Array[*job_servers]
11
+
12
+ @opts = opts
13
+ end
14
+
15
+ def add_ability(name, timeout = nil, &f)
16
+ remove_ability(name) if @abilities.has_key?(name)
17
+ @abilities[name] = { :callback => f, :timeout => timeout }
18
+ end
19
+
20
+ def remove_ability(name)
21
+ @abilities.delete(name)
22
+ end
23
+
24
+ def has_ability?(name)
25
+ @abilities.has_key?(name)
26
+ end
27
+
28
+ def work
29
+ EM.run do
30
+ @job_servers.each do |hostport|
31
+ host, port = hostport.split(":")
32
+ opts = { :abilities => @abilities }.merge(@opts)
33
+ Gearman::Evented::WorkerReactor.connect(host, port, opts)
34
+ end
35
+ end
36
+ end
37
+
38
+ end
39
+ end
@@ -0,0 +1,121 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ class BasicIntegrationTest < Test::Unit::TestCase
4
+
5
+ def setup
6
+ start_gearmand
7
+ @client = Gearman::Client.new("localhost:4730")
8
+ @worker = Gearman::Worker.new("localhost:4730")
9
+ Thread.new { EM.run } unless EM.reactor_running?
10
+ end
11
+
12
+ def teardown
13
+ stop_gearmand
14
+ end
15
+
16
+ def test_ping_job
17
+ response = nil
18
+
19
+ @worker.add_ability("pingpong") {|data, job| "pong" }
20
+ @worker.work
21
+
22
+ task = Gearman::Task.new("pingpong", "ping")
23
+
24
+ task.on_complete {|res| response = res }
25
+ @client.run task
26
+
27
+ assert_equal "pong", response
28
+ end
29
+
30
+ def test_exception_in_worker
31
+ warning_given = nil
32
+ failed = false
33
+
34
+ task = Gearman::Task.new("crash", "doesntmatter")
35
+
36
+ @worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
37
+ @worker.work
38
+
39
+ task.on_warning {|warning| warning_given = warning }
40
+ task.on_fail { failed = true}
41
+ @client.run task
42
+
43
+ assert_not_nil warning_given
44
+ assert_equal true, failed
45
+ assert_equal 0, task.retries_done
46
+ end
47
+
48
+ def test_should_be_able_to_retry_on_worker_exception
49
+ retry_callback_called = false
50
+ fail_count = 0
51
+
52
+ # Gearman::Util.debug = true
53
+ task = Gearman::Task.new("crash", "doesntmatter", :retries => 1)
54
+
55
+ @worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
56
+ @worker.work
57
+
58
+ task.on_retry {|i| retry_callback_called = true }
59
+ task.on_fail { fail_count += 1 }
60
+ @client.run task
61
+
62
+ assert_equal true, retry_callback_called
63
+ assert_equal 1, task.retries_done
64
+ assert_equal 1, fail_count
65
+ end
66
+
67
+ def test_chunked_response
68
+ chunks_received = 0
69
+
70
+ @worker.add_ability("chunked") do |data, job|
71
+ 5.times {|i| job.send_partial("chunk #{i}") }
72
+ end
73
+ @worker.work
74
+
75
+ task = Gearman::Task.new("chunked")
76
+ task.on_data do |data|
77
+ assert_match /^chunk \d/, data
78
+ chunks_received += 1
79
+ end
80
+ @client.run task
81
+
82
+ assert_equal 5, chunks_received
83
+ end
84
+
85
+ def test_background
86
+ status_received = false
87
+
88
+ @worker.add_ability("fireandforget") {|data, job| "this goes to /dev/null" }
89
+ @worker.work
90
+
91
+ task = Gearman::Task.new('fireandforget', 'background', :background => true, :poll_status_interval => 0.1)
92
+ task.on_complete {|d| flunk "on_complete should never be called for a background job!" }
93
+ task.on_status {|d| status_received = true }
94
+ @client.run task
95
+
96
+ assert_equal true, status_received
97
+ end
98
+
99
+ def test_non_blocking_run
100
+ @worker.add_ability("foo") {|data, job| "foo: #{data}" }
101
+ @worker.work
102
+
103
+ task1_complete = false
104
+ task2_complete = false
105
+
106
+ task1 = Gearman::Task.new("foo", 1)
107
+ task1.on_complete {|d| task1_complete = true}
108
+ @client.run task1, nil, true
109
+
110
+ assert_equal false, task1_complete
111
+
112
+ task2 = Gearman::Task.new("foo", 2)
113
+ task2.on_complete {|d| task2_complete = true}
114
+ @client.run task2, nil, true
115
+
116
+ Thread.new { sleep 0.1 }.join
117
+
118
+ assert_equal true, task1_complete
119
+ assert_equal true, task2_complete
120
+ end
121
+ end
@@ -0,0 +1,69 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ # Gearman::Util.debug = true
4
+ class CrashTest < Test::Unit::TestCase
5
+
6
+ def setup
7
+ teardown_gearmands
8
+ Thread.new { EM.run } unless EM.reactor_running?
9
+ end
10
+
11
+ def teardown
12
+ teardown_gearmands
13
+ end
14
+
15
+ def test_worker_should_reconnect_if_gearmand_goes_away
16
+ start_gearmand
17
+ worker = Gearman::Worker.new("localhost:4730", :reconnect_sec => 1)
18
+ worker.add_ability("foo") {|data, job| "noop!" }
19
+
20
+ response = nil
21
+
22
+ worker.work
23
+ stop_gearmand
24
+ start_gearmand
25
+
26
+ task = Gearman::Task.new("foo", "ping")
27
+ task.on_complete {|res| response = res }
28
+ Gearman::Client.new("localhost:4730").run task
29
+
30
+ assert_equal "noop!", response
31
+ stop_gearmand
32
+ end
33
+
34
+ def test_client_and_worker_should_use_failover_gearmand_if_primary_is_not_available
35
+ start_gearmand 4731
36
+
37
+ worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
38
+ worker.add_ability("foo") {|data, job| "noop!" }
39
+
40
+ response = nil
41
+ worker.work
42
+ task = Gearman::Task.new("foo", "ping")
43
+ task.on_complete {|res| response = res }
44
+ Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
45
+
46
+ assert_equal "noop!", response
47
+ stop_gearmand 4731
48
+ end
49
+
50
+ def test_client_and_worker_should_switch_to_failover_gearmand_if_primary_goes_down
51
+ start_gearmand 4730
52
+ start_gearmand 4731
53
+
54
+ worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
55
+ worker.add_ability("foo") {|data, job| "noop!" }
56
+
57
+ response = nil
58
+ worker.work
59
+
60
+ stop_gearmand 4730
61
+
62
+ task = Gearman::Task.new("foo", "ping")
63
+ task.on_complete {|res| response = res }
64
+ Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
65
+
66
+ assert_equal "noop!", response
67
+ stop_gearmand 4731
68
+ end
69
+ end