gearman-ruby 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/HOWTO +146 -0
- data/LICENSE +20 -0
- data/README +9 -0
- data/Rakefile +41 -0
- data/TODO +8 -0
- data/VERSION.yml +4 -0
- data/examples/calculus_client.rb +41 -0
- data/examples/calculus_worker.rb +42 -0
- data/examples/client.rb +19 -0
- data/examples/client_background.rb +14 -0
- data/examples/client_data.rb +16 -0
- data/examples/client_echo.rb +16 -0
- data/examples/client_exception.rb +17 -0
- data/examples/client_prefix.rb +15 -0
- data/examples/evented_client.rb +23 -0
- data/examples/evented_worker.rb +26 -0
- data/examples/gearman_environment.sh +25 -0
- data/examples/scale_image.rb +30 -0
- data/examples/scale_image_worker.rb +34 -0
- data/examples/server.rb +15 -0
- data/examples/worker.rb +23 -0
- data/examples/worker_data.rb +16 -0
- data/examples/worker_echo.rb +20 -0
- data/examples/worker_echo_pprof.rb +5 -0
- data/examples/worker_exception.rb +14 -0
- data/examples/worker_prefix.rb +25 -0
- data/gearman-ruby.gemspec +111 -0
- data/lib/gearman.rb +29 -0
- data/lib/gearman/client.rb +80 -0
- data/lib/gearman/evented/client.rb +99 -0
- data/lib/gearman/evented/reactor.rb +86 -0
- data/lib/gearman/evented/worker.rb +118 -0
- data/lib/gearman/job.rb +38 -0
- data/lib/gearman/protocol.rb +110 -0
- data/lib/gearman/server.rb +94 -0
- data/lib/gearman/task.rb +99 -0
- data/lib/gearman/taskset.rb +11 -0
- data/lib/gearman/util.rb +52 -0
- data/lib/gearman/worker.rb +39 -0
- data/test/basic_integration_test.rb +121 -0
- data/test/crash_test.rb +69 -0
- data/test/job_test.rb +30 -0
- data/test/protocol_test.rb +132 -0
- data/test/test_helper.rb +31 -0
- data/test/util_test.rb +12 -0
- data/test/worker_test.rb +45 -0
- metadata +133 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require '../lib/gearman'
|
3
|
+
|
4
|
+
Gearman::Util.debug = true
|
5
|
+
|
6
|
+
servers = ['localhost:4730', 'localhost:4731']
|
7
|
+
w = Gearman::Worker.new(servers)
|
8
|
+
|
9
|
+
# Add a handler for a "sleep" function that takes a single argument, the
|
10
|
+
# number of seconds to sleep before reporting success.
|
11
|
+
w.add_ability('sleep') do |data, job|
|
12
|
+
seconds = data
|
13
|
+
job.report_warning("this is a warning you can safely ignore")
|
14
|
+
|
15
|
+
(1..seconds.to_i).each do |i|
|
16
|
+
sleep 1
|
17
|
+
print i
|
18
|
+
# Report our progress to the job server every second.
|
19
|
+
job.report_status(i, seconds)
|
20
|
+
job.send_partial(".")
|
21
|
+
end
|
22
|
+
|
23
|
+
# Report success.
|
24
|
+
"SUCCESS"
|
25
|
+
end
|
26
|
+
w.work
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Start Gearmand
|
4
|
+
echo ' + Starting Gearmand'
|
5
|
+
gearmand --daemon --pidfile=/tmp/gearmand.pid
|
6
|
+
|
7
|
+
# Start the client and the worker(s)
|
8
|
+
echo ' + Starting calculus_worker.rb'
|
9
|
+
ruby calculus_worker.rb &
|
10
|
+
|
11
|
+
sleep 3
|
12
|
+
|
13
|
+
echo ' + Starting calculus_client.rb'
|
14
|
+
ruby calculus_client.rb
|
15
|
+
|
16
|
+
echo ' +++ Example finished +++ '
|
17
|
+
|
18
|
+
# Stop Gearmand
|
19
|
+
echo ' - Stopping Gearmand'
|
20
|
+
kill -9 `cat /tmp/gearmand.pid`
|
21
|
+
|
22
|
+
# Stop the workers
|
23
|
+
echo ' - Stopping calculus_worker.rb'
|
24
|
+
kill -9 `ps ax|grep calculus_worker|grep ruby|awk -F' ' '{print $1}'`
|
25
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
$: << '../lib'
|
4
|
+
require 'gearman'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
Gearman::Util.debug = true
|
8
|
+
servers = 'localhost:7003'
|
9
|
+
format = 'PNG'
|
10
|
+
width, height = 100, 100
|
11
|
+
|
12
|
+
opts = OptionParser.new
|
13
|
+
opts.banner = "Usage: #{$0} [options] <input> <output>"
|
14
|
+
opts.on('-f FORMAT', '--format', 'Scaled image format') { format }
|
15
|
+
opts.on('-h HEIGHT', '--height', 'Scaled image height') { height }
|
16
|
+
opts.on('-s SERVERS', '--servers',
|
17
|
+
'Servers, comma-separated host:port') { servers }
|
18
|
+
opts.on('-w WIDTH', '--width', 'Scaled image width') { width }
|
19
|
+
opts.parse!
|
20
|
+
|
21
|
+
if ARGV.size != 2
|
22
|
+
$stderr.puts opts.banner
|
23
|
+
exit 1
|
24
|
+
end
|
25
|
+
|
26
|
+
client = Gearman::Client.new(servers.split(','), 'example')
|
27
|
+
arg = [width, height, format, File.read(ARGV[0])].join("\0")
|
28
|
+
task = Gearman::Task.new('scale_image', arg)
|
29
|
+
task.on_complete {|d| File.new(ARGV[1],'w').write(d) }
|
30
|
+
client.run task
|
@@ -0,0 +1,34 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
$: << '../lib'
|
4
|
+
require 'gearman'
|
5
|
+
require 'optparse'
|
6
|
+
require 'RMagick'
|
7
|
+
|
8
|
+
Gearman::Util.debug = true
|
9
|
+
servers = 'localhost:7003'
|
10
|
+
|
11
|
+
opts = OptionParser.new
|
12
|
+
opts.banner = "Usage: #{$0} [options]"
|
13
|
+
opts.on('-s SERVERS', '--servers',
|
14
|
+
'Job servers, comma-separated host:port') { servers }
|
15
|
+
opts.parse!
|
16
|
+
|
17
|
+
worker = Gearman::Worker.new(servers.split(','), 'example')
|
18
|
+
|
19
|
+
worker.add_ability('scale_image') do |data,job|
|
20
|
+
width, height, format, data = data.split("\0", 4)
|
21
|
+
width = width.to_f
|
22
|
+
height = height.to_f
|
23
|
+
image = Magick::Image.from_blob(data)[0]
|
24
|
+
orig_ratio = image.columns.to_f / image.rows
|
25
|
+
new_ratio = width / height
|
26
|
+
w = new_ratio < orig_ratio ? width : orig_ratio / new_ratio * width
|
27
|
+
h = new_ratio > orig_ratio ? height : new_ratio / orig_ratio * height
|
28
|
+
puts "Got #{image.inspect}; resizing to #{w}x#{h} #{format}"
|
29
|
+
image.resize!(w, h)
|
30
|
+
image.format = format
|
31
|
+
image.to_blob
|
32
|
+
end
|
33
|
+
|
34
|
+
worker.work
|
data/examples/server.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
# require 'gearman'
|
3
|
+
# require 'gearman/server'
|
4
|
+
require '../lib/gearman'
|
5
|
+
require '../lib/gearman/server'
|
6
|
+
require 'pp'
|
7
|
+
|
8
|
+
Gearman::Util.debug = true
|
9
|
+
w = Gearman::Server.new('localhost:4730')
|
10
|
+
|
11
|
+
loop {
|
12
|
+
pp "Status: ", w.status
|
13
|
+
pp "Workers: ", w.workers
|
14
|
+
sleep 5
|
15
|
+
}
|
data/examples/worker.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
#require 'gearman'
|
3
|
+
require '../lib/gearman'
|
4
|
+
|
5
|
+
Gearman::Util.debug = true
|
6
|
+
|
7
|
+
servers = ['localhost:4730', 'localhost:4731']
|
8
|
+
w = Gearman::Worker.new(servers)
|
9
|
+
|
10
|
+
# Add a handler for a "sleep" function that takes a single argument, the
|
11
|
+
# number of seconds to sleep before reporting success.
|
12
|
+
w.add_ability('sleep') do |data,job|
|
13
|
+
seconds = data
|
14
|
+
(1..seconds.to_i).each do |i|
|
15
|
+
sleep 1
|
16
|
+
print i
|
17
|
+
# Report our progress to the job server every second.
|
18
|
+
job.report_status(i, seconds)
|
19
|
+
end
|
20
|
+
# Report success.
|
21
|
+
true
|
22
|
+
end
|
23
|
+
w.work
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require '../lib/gearman'
|
3
|
+
|
4
|
+
Gearman::Util.debug = true
|
5
|
+
|
6
|
+
servers = ['localhost:4730']
|
7
|
+
worker = Gearman::Worker.new(servers)
|
8
|
+
|
9
|
+
worker.add_ability('chunked_transfer') do |data, job|
|
10
|
+
5.times do |i|
|
11
|
+
sleep 1
|
12
|
+
job.send_partial("CHUNK #{i}")
|
13
|
+
end
|
14
|
+
"EOD"
|
15
|
+
end
|
16
|
+
worker.work
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require '../lib/gearman'
|
3
|
+
# require 'perftools'
|
4
|
+
|
5
|
+
quit = false
|
6
|
+
# PerfTools::CpuProfiler.start("/tmp/worker_echo_profile") do
|
7
|
+
servers = ['localhost:4730']
|
8
|
+
worker = Gearman::Worker.new(servers)
|
9
|
+
|
10
|
+
worker.add_ability('echo') do |data, job|
|
11
|
+
data
|
12
|
+
end
|
13
|
+
worker.add_ability('quit') do |data, job|
|
14
|
+
quit = true
|
15
|
+
end
|
16
|
+
loop {
|
17
|
+
break if quit
|
18
|
+
worker.work
|
19
|
+
}
|
20
|
+
# end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require '../lib/gearman'
|
3
|
+
|
4
|
+
Gearman::Util.debug = true
|
5
|
+
|
6
|
+
servers = ['localhost:4730']
|
7
|
+
w = Gearman::Worker.new(servers)
|
8
|
+
|
9
|
+
# Add a handler for a "sleep" function that takes a single argument, the
|
10
|
+
# number of seconds to sleep before reporting success.
|
11
|
+
w.add_ability('fail_with_exception') do |data,job|
|
12
|
+
raise Exception.new("Exception in worker (args: #{data.inspect})")
|
13
|
+
end
|
14
|
+
w.work
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
#require 'gearman'
|
3
|
+
require '../lib/gearman'
|
4
|
+
|
5
|
+
Gearman::Util.debug = true
|
6
|
+
|
7
|
+
servers = ['localhost:4730', 'localhost:4731']
|
8
|
+
w = Gearman::Worker.new(servers)
|
9
|
+
|
10
|
+
ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep")
|
11
|
+
|
12
|
+
# Add a handler for a "sleep" function that takes a single argument, the
|
13
|
+
# number of seconds to sleep before reporting success.
|
14
|
+
w.add_ability(ability_name_with_prefix) do |data,job|
|
15
|
+
seconds = data
|
16
|
+
(1..seconds.to_i).each do |i|
|
17
|
+
sleep 1
|
18
|
+
print i
|
19
|
+
# Report our progress to the job server every second.
|
20
|
+
job.report_status(i, seconds)
|
21
|
+
end
|
22
|
+
# Report success.
|
23
|
+
true
|
24
|
+
end
|
25
|
+
w.work
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{gearman-ruby}
|
8
|
+
s.version = "2.0.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Kim Altintop"]
|
12
|
+
s.date = %q{2009-12-11}
|
13
|
+
s.description = %q{Library for the Gearman distributed job system}
|
14
|
+
s.email = %q{kim.altintop@gmail.com}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE",
|
17
|
+
"README"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".gitignore",
|
21
|
+
"HOWTO",
|
22
|
+
"LICENSE",
|
23
|
+
"README",
|
24
|
+
"Rakefile",
|
25
|
+
"TODO",
|
26
|
+
"VERSION.yml",
|
27
|
+
"examples/calculus_client.rb",
|
28
|
+
"examples/calculus_worker.rb",
|
29
|
+
"examples/client.rb",
|
30
|
+
"examples/client_background.rb",
|
31
|
+
"examples/client_data.rb",
|
32
|
+
"examples/client_exception.rb",
|
33
|
+
"examples/client_prefix.rb",
|
34
|
+
"examples/evented_client.rb",
|
35
|
+
"examples/evented_worker.rb",
|
36
|
+
"examples/gearman_environment.sh",
|
37
|
+
"examples/scale_image.rb",
|
38
|
+
"examples/scale_image_worker.rb",
|
39
|
+
"examples/server.rb",
|
40
|
+
"examples/worker.rb",
|
41
|
+
"examples/worker_data.rb",
|
42
|
+
"examples/worker_exception.rb",
|
43
|
+
"examples/worker_prefix.rb",
|
44
|
+
"gearman-ruby.gemspec",
|
45
|
+
"lib/gearman.rb",
|
46
|
+
"lib/gearman/client.rb",
|
47
|
+
"lib/gearman/evented/client.rb",
|
48
|
+
"lib/gearman/evented/reactor.rb",
|
49
|
+
"lib/gearman/evented/worker.rb",
|
50
|
+
"lib/gearman/job.rb",
|
51
|
+
"lib/gearman/protocol.rb",
|
52
|
+
"lib/gearman/server.rb",
|
53
|
+
"lib/gearman/task.rb",
|
54
|
+
"lib/gearman/taskset.rb",
|
55
|
+
"lib/gearman/util.rb",
|
56
|
+
"lib/gearman/worker.rb",
|
57
|
+
"test/basic_integration_test.rb",
|
58
|
+
"test/crash_test.rb",
|
59
|
+
"test/job_test.rb",
|
60
|
+
"test/protocol_test.rb",
|
61
|
+
"test/test_helper.rb",
|
62
|
+
"test/util_test.rb",
|
63
|
+
"test/worker_test.rb"
|
64
|
+
]
|
65
|
+
s.homepage = %q{http://github.com/kim/gearman-ruby}
|
66
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
67
|
+
s.require_paths = ["lib"]
|
68
|
+
s.rubygems_version = %q{1.3.5}
|
69
|
+
s.summary = %q{Library for the Gearman distributed job system}
|
70
|
+
s.test_files = [
|
71
|
+
"test/basic_integration_test.rb",
|
72
|
+
"test/crash_test.rb",
|
73
|
+
"test/job_test.rb",
|
74
|
+
"test/protocol_test.rb",
|
75
|
+
"test/test_helper.rb",
|
76
|
+
"test/util_test.rb",
|
77
|
+
"test/worker_test.rb",
|
78
|
+
"examples/calculus_client.rb",
|
79
|
+
"examples/calculus_worker.rb",
|
80
|
+
"examples/client.rb",
|
81
|
+
"examples/client_background.rb",
|
82
|
+
"examples/client_data.rb",
|
83
|
+
"examples/client_echo.rb",
|
84
|
+
"examples/client_exception.rb",
|
85
|
+
"examples/client_prefix.rb",
|
86
|
+
"examples/evented_client.rb",
|
87
|
+
"examples/evented_worker.rb",
|
88
|
+
"examples/scale_image.rb",
|
89
|
+
"examples/scale_image_worker.rb",
|
90
|
+
"examples/server.rb",
|
91
|
+
"examples/worker.rb",
|
92
|
+
"examples/worker_data.rb",
|
93
|
+
"examples/worker_echo.rb",
|
94
|
+
"examples/worker_echo_pprof.rb",
|
95
|
+
"examples/worker_exception.rb",
|
96
|
+
"examples/worker_prefix.rb"
|
97
|
+
]
|
98
|
+
|
99
|
+
if s.respond_to? :specification_version then
|
100
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
101
|
+
s.specification_version = 3
|
102
|
+
|
103
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
104
|
+
s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
|
105
|
+
else
|
106
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
|
107
|
+
end
|
108
|
+
else
|
109
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
|
110
|
+
end
|
111
|
+
end
|
data/lib/gearman.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'eventmachine'
|
3
|
+
|
4
|
+
module Gearman
|
5
|
+
|
6
|
+
require File.dirname(__FILE__) + '/gearman/evented/reactor'
|
7
|
+
require File.dirname(__FILE__) + '/gearman/evented/client'
|
8
|
+
require File.dirname(__FILE__) + '/gearman/evented/worker'
|
9
|
+
require File.dirname(__FILE__) + '/gearman/client'
|
10
|
+
require File.dirname(__FILE__) + '/gearman/task'
|
11
|
+
require File.dirname(__FILE__) + '/gearman/taskset'
|
12
|
+
require File.dirname(__FILE__) + '/gearman/util'
|
13
|
+
require File.dirname(__FILE__) + '/gearman/worker'
|
14
|
+
require File.dirname(__FILE__) + '/gearman/job'
|
15
|
+
|
16
|
+
require File.dirname(__FILE__) + '/gearman/protocol'
|
17
|
+
|
18
|
+
|
19
|
+
class InvalidArgsError < Exception
|
20
|
+
end
|
21
|
+
|
22
|
+
class NetworkError < Exception
|
23
|
+
end
|
24
|
+
|
25
|
+
def log(msg, force = false)
|
26
|
+
Util.log(msg, force)
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module Gearman
|
2
|
+
class Client
|
3
|
+
attr_accessor :uniq, :jobs
|
4
|
+
|
5
|
+
def initialize(job_servers, opts = {})
|
6
|
+
@reactors = []
|
7
|
+
@jobs = {}
|
8
|
+
|
9
|
+
@job_servers = Array[*job_servers]
|
10
|
+
|
11
|
+
@uniq = opts.delete(:uniq)
|
12
|
+
@opts = opts
|
13
|
+
end
|
14
|
+
|
15
|
+
# Run a Task or Taskset
|
16
|
+
def run(taskset, timeout = nil, async = false)
|
17
|
+
timeout ||= 0
|
18
|
+
use_em_stop = EM.reactor_running?
|
19
|
+
EM.run do
|
20
|
+
@taskset ||= Taskset.new
|
21
|
+
@taskset += Taskset.create(taskset)
|
22
|
+
|
23
|
+
job_servers = @job_servers.dup
|
24
|
+
@reactors.each do |reactor|
|
25
|
+
unless reactor.connected?
|
26
|
+
reactor.reconnect true
|
27
|
+
reactor.keep_connected = async
|
28
|
+
reactor.callback { create_job(@taskset.shift, reactor) }
|
29
|
+
job_servers.delete reactor.to_s
|
30
|
+
else
|
31
|
+
create_job(@taskset.shift, reactor)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
job_servers.each do |hostport|
|
35
|
+
host, port = hostport.split(":")
|
36
|
+
reactor = Gearman::Evented::ClientReactor.connect(host, port, @opts)
|
37
|
+
reactor.keep_connected = async
|
38
|
+
reactor.callback { create_job(@taskset.shift, reactor) }
|
39
|
+
@reactors << reactor
|
40
|
+
end
|
41
|
+
|
42
|
+
if timeout > 0
|
43
|
+
if use_em_stop
|
44
|
+
EM.add_timer(timeout) { EM.stop }
|
45
|
+
else
|
46
|
+
sleep timeout
|
47
|
+
end
|
48
|
+
elsif !async
|
49
|
+
Thread.new do
|
50
|
+
loop do
|
51
|
+
sleep 0.1
|
52
|
+
live = 0
|
53
|
+
@reactors.each {|reactor| live += 1 if reactor.connected? }
|
54
|
+
break if live == 0
|
55
|
+
end
|
56
|
+
end.join
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def create_job(task, reactor = nil)
|
64
|
+
return unless task
|
65
|
+
reactor ||= @reactors[rand(@reactors.size)]
|
66
|
+
unless reactor.connected?
|
67
|
+
log "create_job: server #{reactor} not connected"
|
68
|
+
EM.next_tick { create_job(task) }
|
69
|
+
return
|
70
|
+
end
|
71
|
+
|
72
|
+
reactor.submit_job(task) {|handle| create_job(@taskset.shift) }
|
73
|
+
end
|
74
|
+
|
75
|
+
def log(msg)
|
76
|
+
Gearman::Util.log(msg)
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
end
|