gearman-ruby 2.0.0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/Rakefile +5 -6
  2. data/VERSION.yml +2 -2
  3. data/examples/calculus_client.rb +8 -10
  4. data/examples/calculus_worker.rb +4 -1
  5. data/examples/client.php +23 -0
  6. data/examples/client.rb +6 -10
  7. data/examples/client_background.rb +7 -7
  8. data/examples/client_data.rb +4 -4
  9. data/examples/client_epoch.rb +23 -0
  10. data/examples/client_exception.rb +4 -2
  11. data/examples/client_prefix.rb +4 -2
  12. data/examples/client_reverse.rb +27 -0
  13. data/examples/scale_image.rb +4 -3
  14. data/examples/scale_image_worker.rb +1 -1
  15. data/examples/worker.rb +2 -4
  16. data/examples/worker_data.rb +2 -2
  17. data/examples/worker_exception.rb +1 -1
  18. data/examples/worker_prefix.rb +1 -1
  19. data/examples/worker_reverse_string.rb +27 -0
  20. data/examples/worker_reverse_to_file.rb +18 -0
  21. data/examples/{evented_worker.rb → worker_signals.rb} +18 -8
  22. data/lib/gearman.rb +68 -21
  23. data/lib/gearman/client.rb +137 -65
  24. data/lib/gearman/server.rb +4 -4
  25. data/lib/gearman/task.rb +140 -20
  26. data/lib/gearman/taskset.rb +280 -5
  27. data/lib/gearman/testlib.rb +95 -0
  28. data/lib/gearman/util.rb +184 -28
  29. data/lib/gearman/worker.rb +356 -20
  30. data/test/client_test.rb +145 -0
  31. data/test/mock_client_test.rb +629 -0
  32. data/test/mock_worker_test.rb +321 -0
  33. data/test/util_test.rb +8 -3
  34. data/test/worker_test.rb +50 -34
  35. metadata +41 -41
  36. data/examples/client_echo.rb +0 -16
  37. data/examples/evented_client.rb +0 -23
  38. data/examples/worker_echo.rb +0 -20
  39. data/examples/worker_echo_pprof.rb +0 -5
  40. data/gearman-ruby.gemspec +0 -111
  41. data/lib/gearman/evented/client.rb +0 -99
  42. data/lib/gearman/evented/reactor.rb +0 -86
  43. data/lib/gearman/evented/worker.rb +0 -118
  44. data/lib/gearman/job.rb +0 -38
  45. data/lib/gearman/protocol.rb +0 -110
  46. data/test/basic_integration_test.rb +0 -121
  47. data/test/crash_test.rb +0 -69
  48. data/test/job_test.rb +0 -30
  49. data/test/protocol_test.rb +0 -132
  50. data/test/test_helper.rb +0 -31
data/Rakefile CHANGED
@@ -2,17 +2,16 @@ require 'rake'
2
2
  require 'rake/testtask'
3
3
  require 'rake/rdoctask'
4
4
  require 'rcov/rcovtask'
5
-
5
+
6
6
  begin
7
7
  require 'jeweler'
8
8
  Jeweler::Tasks.new do |s|
9
9
  s.name = "gearman-ruby"
10
10
  s.summary = "Library for the Gearman distributed job system"
11
- s.email = "kim.altintop@gmail.com"
12
- s.homepage = "http://github.com/kim/gearman-ruby"
11
+ s.email = "john@unixninjas.org"
12
+ s.homepage = "http://github.com/gearman-ruby/gearman-ruby"
13
13
  s.description = "Library for the Gearman distributed job system"
14
- s.authors = ["Kim Altintop"]
15
- s.add_dependency 'eventmachine', '>= 0.12.8'
14
+ s.authors = ["John Ewart", "Colin Curtin", "Daniel Erat", "Ladislav Martincik", "Pablo Delgado", "Mauro Pompilio", "Antonio Garrote", "Kim Altintop"]
16
15
  end
17
16
  rescue LoadError
18
17
  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
@@ -38,4 +37,4 @@ Rcov::RcovTask.new do |t|
38
37
  t.verbose = true
39
38
  end
40
39
 
41
- task :default => :test
40
+ task :default => :rcov
data/VERSION.yml CHANGED
@@ -1,4 +1,4 @@
1
1
  ---
2
- :patch: 0
3
- :major: 2
2
+ :major: 3
4
3
  :minor: 0
4
+ :patch: 1
@@ -3,9 +3,9 @@ require 'rubygems'
3
3
  require '../lib/gearman'
4
4
  #Gearman::Util.debug = true
5
5
 
6
- # Connect to the local server (at the default port 4730)
6
+ # Connect to the local server (at the default port 7003)
7
7
  client = Gearman::Client.new('localhost')
8
- taskset = Gearman::Taskset.new
8
+ taskset = Gearman::TaskSet.new(client)
9
9
 
10
10
  # Get something to echo
11
11
  puts '[client] Write a basic arithmetic operation:'
@@ -19,23 +19,21 @@ operations.each do |op|
19
19
  # Determining the operation
20
20
  case op
21
21
  when /\+/
22
- type, data = 'addition', op.split('+')
22
+ type, data = 'addition', op.split('+')
23
23
  when /\-/
24
- type, data = 'subtraction', op.split('-')
24
+ type, data = 'subtraction', op.split('-')
25
25
  when /\*/
26
- type, data = 'multiplication', op.split('*')
26
+ type, data = 'multiplication', op.split('*')
27
27
  when /\//
28
- type, data = 'division', op.split('/')
28
+ type, data = 'division', op.split('/')
29
29
  end
30
30
 
31
31
  task = Gearman::Task.new(type, Marshal.dump(data.map {|v| v.to_i}))
32
32
  task.on_complete {|r| puts "[client] #{type} result is: #{r}" }
33
- task.on_warning {|w| puts "[client] warn: #{w}" }
34
- task.on_fail { puts "[client] calculation failed" }
35
33
 
36
34
  # Sending the task to the server
37
35
  puts "[client] Sending values: #{data.inspect}, to the '#{type}' worker"
38
- taskset << task
36
+ taskset.add_task(task)
37
+ taskset.wait(100)
39
38
  end
40
39
 
41
- client.run(taskset)
@@ -5,6 +5,7 @@ require '../lib/gearman'
5
5
  #Gearman::Util.debug = true
6
6
 
7
7
  worker = Gearman::Worker.new('localhost')
8
+ worker.reconnect_sec = 2
8
9
 
9
10
  # Additon ability
10
11
  worker.add_ability('addition') do |data,job|
@@ -39,4 +40,6 @@ worker.add_ability('division') do |data,job|
39
40
  end
40
41
 
41
42
  # Running the workers
42
- worker.work
43
+ loop do
44
+ worker.work
45
+ end
@@ -0,0 +1,23 @@
1
+ <?php
2
+
3
+ require_once 'Net/Gearman/Client.php';
4
+
5
+ $set = new Net_Gearman_Set();
6
+
7
+ function result($func, $handle, $result) {
8
+ var_dump($func);
9
+ var_dump($handle);
10
+ var_dump($result);
11
+ }
12
+
13
+ $task = new Net_Gearman_Task('Sleep', array(
14
+ 'seconds' => 20
15
+ ));
16
+
17
+ $task->attachCallback('result');
18
+ $set->addTask($task);
19
+
20
+ $client = new Net_Gearman_Client(array('localhost:4730', 'localhost:4731'));
21
+ $client->runSet($set);
22
+
23
+ ?>
data/examples/client.rb CHANGED
@@ -4,16 +4,12 @@ require '../lib/gearman'
4
4
  Gearman::Util.debug = true
5
5
 
6
6
  servers = ['localhost:4730', 'localhost:4731']
7
-
7
+
8
8
  client = Gearman::Client.new(servers)
9
- taskset = Gearman::Taskset.new
10
-
11
- task = Gearman::Task.new('sleep', 2)
12
- task.on_complete {|d| puts "TASK 1: #{d}" }
13
- taskset << task
9
+ taskset = Gearman::TaskSet.new(client)
14
10
 
15
- task = Gearman::Task.new('sleep', 2)
16
- task.on_complete {|d| puts "TASK 2: #{d}" }
17
- taskset << task
11
+ task = Gearman::Task.new('sleep', 20)
12
+ task.on_complete {|d| puts d }
18
13
 
19
- client.run(taskset)
14
+ taskset.add_task(task)
15
+ taskset.wait(100)
@@ -1,14 +1,14 @@
1
1
  require 'rubygems'
2
2
  #require 'gearman'
3
3
  require '../lib/gearman'
4
- Gearman::Util.debug = true
5
-
6
- servers = ['localhost:4730', 'localhost:4731']
7
4
 
5
+ servers = ['localhost:4730',]
6
+
8
7
  client = Gearman::Client.new(servers)
8
+ taskset = Gearman::TaskSet.new(client)
9
9
 
10
- task = Gearman::Task.new('sleep', 20, :background => true, :poll_status_interval => 1)
11
- task.on_complete {|d| puts d } #never called
12
- task.on_status {|d| puts "Status: #{d}"}
10
+ task = Gearman::Task.new('sleep', 20, { :background => true })
11
+ task.on_complete {|d| puts d }
13
12
 
14
- client.run(task)
13
+ taskset.add_task(task)
14
+ taskset.wait(100)
@@ -4,13 +4,13 @@ require '../lib/gearman'
4
4
  Gearman::Util.debug = true
5
5
 
6
6
  servers = ['localhost:4730']
7
-
7
+
8
8
  client = Gearman::Client.new(servers)
9
- taskset = Gearman::Taskset.new
9
+ taskset = Gearman::TaskSet.new(client)
10
10
 
11
11
  task = Gearman::Task.new('chunked_transfer')
12
12
  task.on_data {|d| puts d }
13
13
  task.on_complete {|d| puts d }
14
14
 
15
- taskset << task
16
- client.run taskset
15
+ taskset.add_task(task)
16
+ taskset.wait(100)
@@ -0,0 +1,23 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/)
4
+
5
+ require 'rubygems'
6
+ require '../lib/gearman'
7
+
8
+ (1..100).each do
9
+ # Connect to the local server (at the default port 4730)
10
+ client = Gearman::Client.new('localhost')
11
+ taskset = Gearman::TaskSet.new(client)
12
+
13
+ data = rand(36**8).to_s(36)
14
+ # Set scheduled time to some time in the future
15
+ time = Time.now() + rand(10)
16
+ puts "Time as seconds: #{time.to_i}"
17
+ task = Gearman::Task.new("reverse_string", data)
18
+ task.schedule(time)
19
+
20
+ # Sending the task to the server
21
+ puts "[client] Sending task: #{task.inspect}, to the 'reverse_string' worker"
22
+ taskset.add_task(task)
23
+ end
@@ -5,13 +5,15 @@ Gearman::Util.debug = true
5
5
  servers = ['localhost:4730']
6
6
 
7
7
  client = Gearman::Client.new(servers)
8
+ taskset = Gearman::TaskSet.new(client)
8
9
 
9
10
  task = Gearman::Task.new('fail_with_exception', "void")
10
- task.retries = 2
11
+ task.retry_count = 2
11
12
  task.on_complete {|d| puts d }
12
13
  task.on_exception {|ex| puts "This should never be called" }
13
14
  task.on_warning {|warning| puts "WARNING: #{warning}" }
14
15
  task.on_retry { puts "PRE-RETRY HOOK: retry no. #{task.retries_done}" }
15
16
  task.on_fail { puts "TASK FAILED, GIVING UP" }
16
17
 
17
- client.run task
18
+ taskset.add_task(task)
19
+ taskset.wait(100)
@@ -4,12 +4,14 @@ require '../lib/gearman'
4
4
  Gearman::Util.debug = true
5
5
 
6
6
  servers = ['localhost:4730', 'localhost:4731']
7
-
7
+
8
8
  ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep")
9
9
 
10
10
  client = Gearman::Client.new(servers)
11
+ taskset = Gearman::TaskSet.new(client)
11
12
 
12
13
  task = Gearman::Task.new(ability_name_with_prefix, 20)
13
14
  task.on_complete {|d| puts d }
14
15
 
15
- client.run task
16
+ taskset.add_task(task)
17
+ taskset.wait(100)
@@ -0,0 +1,27 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require '../lib/gearman'
4
+
5
+ # Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/)
6
+
7
+ t = nil
8
+ threadcounter = 0
9
+
10
+ client = Gearman::Client.new('localhost')
11
+
12
+
13
+ myid = threadcounter
14
+ threadcounter += 1
15
+ taskset = Gearman::TaskSet.new(client)
16
+
17
+ (1..10000).each do |jid|
18
+ data = rand(36**8).to_s(36)
19
+ result = data.reverse
20
+
21
+ task = Gearman::Task.new("reverse_string", data)
22
+ puts "#{jid} #{data}"
23
+
24
+ time = Time.now() + rand(120) + 10
25
+ task.schedule(time)
26
+ taskset.add_task(task)
27
+ end
@@ -4,8 +4,7 @@ $: << '../lib'
4
4
  require 'gearman'
5
5
  require 'optparse'
6
6
 
7
- Gearman::Util.debug = true
8
- servers = 'localhost:7003'
7
+ servers = 'localhost:4730'
9
8
  format = 'PNG'
10
9
  width, height = 100, 100
11
10
 
@@ -24,7 +23,9 @@ if ARGV.size != 2
24
23
  end
25
24
 
26
25
  client = Gearman::Client.new(servers.split(','), 'example')
26
+ taskset = Gearman::TaskSet.new(client)
27
27
  arg = [width, height, format, File.read(ARGV[0])].join("\0")
28
28
  task = Gearman::Task.new('scale_image', arg)
29
29
  task.on_complete {|d| File.new(ARGV[1],'w').write(d) }
30
- client.run task
30
+ taskset.add_task(task)
31
+ taskset.wait(10)
@@ -31,4 +31,4 @@ worker.add_ability('scale_image') do |data,job|
31
31
  image.to_blob
32
32
  end
33
33
 
34
- worker.work
34
+ loop { worker.work }
data/examples/worker.rb CHANGED
@@ -2,9 +2,7 @@ require 'rubygems'
2
2
  #require 'gearman'
3
3
  require '../lib/gearman'
4
4
 
5
- Gearman::Util.debug = true
6
-
7
- servers = ['localhost:4730', 'localhost:4731']
5
+ servers = ['localhost:4730',]
8
6
  w = Gearman::Worker.new(servers)
9
7
 
10
8
  # Add a handler for a "sleep" function that takes a single argument, the
@@ -20,4 +18,4 @@ w.add_ability('sleep') do |data,job|
20
18
  # Report success.
21
19
  true
22
20
  end
23
- w.work
21
+ loop { w.work }
@@ -9,8 +9,8 @@ worker = Gearman::Worker.new(servers)
9
9
  worker.add_ability('chunked_transfer') do |data, job|
10
10
  5.times do |i|
11
11
  sleep 1
12
- job.send_partial("CHUNK #{i}")
12
+ job.send_data("CHUNK #{i}")
13
13
  end
14
14
  "EOD"
15
15
  end
16
- worker.work
16
+ loop { worker.work }
@@ -11,4 +11,4 @@ w = Gearman::Worker.new(servers)
11
11
  w.add_ability('fail_with_exception') do |data,job|
12
12
  raise Exception.new("Exception in worker (args: #{data.inspect})")
13
13
  end
14
- w.work
14
+ loop { w.work }
@@ -22,4 +22,4 @@ w.add_ability(ability_name_with_prefix) do |data,job|
22
22
  # Report success.
23
23
  true
24
24
  end
25
- w.work
25
+ loop { w.work }
@@ -0,0 +1,27 @@
1
+ require 'rubygems'
2
+ require '../lib/gearman'
3
+
4
+ # String reverse worker
5
+
6
+ servers = ['localhost:4730']
7
+
8
+ t = nil
9
+ jobnum = 0
10
+
11
+ (0..1).each do
12
+ t = Thread.new {
13
+ w = Gearman::Worker.new(servers)
14
+ w.add_ability('reverse_string') do |data,job|
15
+ result = data.reverse
16
+ puts "Job: #{job.inspect} Data: #{data.inspect} Reverse: #{result} "
17
+ puts "Completed job ##{jobnum}"
18
+ jobnum += 1
19
+ result
20
+ end
21
+ loop { w.work }
22
+ }
23
+ end
24
+
25
+ puts "Waiting for threads..."
26
+ t.join
27
+
@@ -0,0 +1,18 @@
1
+ require 'rubygems'
2
+ #require 'gearman'
3
+ require '../lib/gearman'
4
+
5
+ servers = ['localhost:4730']
6
+ w = Gearman::Worker.new(servers)
7
+
8
+ # Add a handler for a "sleep" function that takes a single argument, the
9
+ # number of seconds to sleep before reporting success.
10
+ w.add_ability('reverse_to_file') do |data,job|
11
+ puts "Data: #{data.inspect}"
12
+ word, file = data.split("\0")
13
+ puts "Word: #{word}"
14
+ puts "File: #{file}"
15
+ # Report success.
16
+ true
17
+ end
18
+ loop { w.work }
@@ -1,4 +1,5 @@
1
1
  require 'rubygems'
2
+ #require 'gearman'
2
3
  require '../lib/gearman'
3
4
 
4
5
  Gearman::Util.debug = true
@@ -8,19 +9,28 @@ w = Gearman::Worker.new(servers)
8
9
 
9
10
  # Add a handler for a "sleep" function that takes a single argument, the
10
11
  # number of seconds to sleep before reporting success.
11
- w.add_ability('sleep') do |data, job|
12
+ w.add_ability('sleep') do |data,job|
12
13
  seconds = data
13
- job.report_warning("this is a warning you can safely ignore")
14
-
15
14
  (1..seconds.to_i).each do |i|
16
15
  sleep 1
17
- print i
16
+ Gearman::Util.logger.info i
18
17
  # Report our progress to the job server every second.
19
18
  job.report_status(i, seconds)
20
- job.send_partial(".")
21
19
  end
22
-
23
20
  # Report success.
24
- "SUCCESS"
21
+ true
25
22
  end
26
- w.work
23
+
24
+ # Trap signals while is working
25
+ %w(HUP USR1 ALRM TERM).each do |signal|
26
+ trap(signal) do
27
+ puts "Received signal #{signal} - setting worker_enabled to false. Worker status is [#{w.status}]"
28
+ w.worker_enabled = false
29
+ if w.status == :waiting
30
+ trap(signal, "DEFAULT")
31
+ Process.kill( signal, $$ )
32
+ end
33
+ end
34
+ end
35
+
36
+ loop { w.work or break }
data/lib/gearman.rb CHANGED
@@ -1,29 +1,76 @@
1
- require 'rubygems'
2
- require 'eventmachine'
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # = Name
4
+ # Gearman
5
+ #
6
+ # == Description
7
+ # This file provides a Ruby interface for communicating with the Gearman
8
+ # distributed job system.
9
+ #
10
+ # "Gearman is a system to farm out work to other machines, dispatching
11
+ # function calls to machines that are better suited to do work, to do work
12
+ # in parallel, to load balance lots of function calls, or to call functions
13
+ # between languages." -- http://www.danga.com/gearman/
14
+ #
15
+ # == Version
16
+ # 0.0.1
17
+ #
18
+ # == Author
19
+ # Daniel Erat <dan-ruby@erat.org>
20
+ #
21
+ # == License
22
+ # This program is free software; you can redistribute it and/or modify it
23
+ # under the terms of either:
24
+ #
25
+ # a) the GNU General Public License as published by the Free Software
26
+ # Foundation; either version 1, or (at your option) any later version,
27
+ # or
28
+ #
29
+ # b) the "Artistic License" which comes with Perl.
3
30
 
31
+ # = Gearman
32
+ #
33
+ # == Usage
34
+ # require 'gearman'
35
+ #
36
+ # # Create a new client and tell it about two job servers.
37
+ # c = Gearman::Client.new
38
+ # c.job_servers = ['127.0.0.1:7003', '127.0.0.1:7004']
39
+ #
40
+ # # Create two tasks, using an "add" function to sum two numbers.
41
+ # t1 = Gearman::Task.new('add', '5 + 2')
42
+ # t2 = Gearman::Task.new('add', '1 + 3')
43
+ #
44
+ # # Make the tasks print the data they get back from the server.
45
+ # t1.on_complete {|d| puts "t1 got #{d}" }
46
+ # t2.on_complete {|d| puts "t2 got #{d}" }
47
+ #
48
+ # # Create a taskset, add the two tasks to it, and wait until they finish.
49
+ # ts = Gearman::TaskSet.new(c)
50
+ # ts.add_task(t1)
51
+ # ts.add_task(t2)
52
+ # ts.wait
53
+ #
54
+ # Or, a more simple example:
55
+ #
56
+ # c = Gearman::Client.new('127.0.0.1')
57
+ # puts c.do_task('add', '2 + 2')
58
+ #
4
59
  module Gearman
5
60
 
6
- require File.dirname(__FILE__) + '/gearman/evented/reactor'
7
- require File.dirname(__FILE__) + '/gearman/evented/client'
8
- require File.dirname(__FILE__) + '/gearman/evented/worker'
9
- require File.dirname(__FILE__) + '/gearman/client'
10
- require File.dirname(__FILE__) + '/gearman/task'
11
- require File.dirname(__FILE__) + '/gearman/taskset'
12
- require File.dirname(__FILE__) + '/gearman/util'
13
- require File.dirname(__FILE__) + '/gearman/worker'
14
- require File.dirname(__FILE__) + '/gearman/job'
61
+ require File.dirname(__FILE__) + '/gearman/client'
62
+ require File.dirname(__FILE__) + '/gearman/task'
63
+ require File.dirname(__FILE__) + '/gearman/taskset'
64
+ require File.dirname(__FILE__) + '/gearman/util'
65
+ require File.dirname(__FILE__) + '/gearman/worker'
15
66
 
16
- require File.dirname(__FILE__) + '/gearman/protocol'
17
-
18
-
19
- class InvalidArgsError < Exception
20
- end
67
+ class InvalidArgsError < Exception
68
+ end
21
69
 
22
- class NetworkError < Exception
23
- end
70
+ class ProtocolError < Exception
71
+ end
24
72
 
25
- def log(msg, force = false)
26
- Util.log(msg, force)
27
- end
73
+ class NetworkError < Exception
74
+ end
28
75
 
29
76
  end