gearman-ruby 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. data/.gitignore +1 -0
  2. data/HOWTO +146 -0
  3. data/LICENSE +20 -0
  4. data/README +9 -0
  5. data/Rakefile +41 -0
  6. data/TODO +8 -0
  7. data/VERSION.yml +4 -0
  8. data/examples/calculus_client.rb +41 -0
  9. data/examples/calculus_worker.rb +42 -0
  10. data/examples/client.rb +19 -0
  11. data/examples/client_background.rb +14 -0
  12. data/examples/client_data.rb +16 -0
  13. data/examples/client_echo.rb +16 -0
  14. data/examples/client_exception.rb +17 -0
  15. data/examples/client_prefix.rb +15 -0
  16. data/examples/evented_client.rb +23 -0
  17. data/examples/evented_worker.rb +26 -0
  18. data/examples/gearman_environment.sh +25 -0
  19. data/examples/scale_image.rb +30 -0
  20. data/examples/scale_image_worker.rb +34 -0
  21. data/examples/server.rb +15 -0
  22. data/examples/worker.rb +23 -0
  23. data/examples/worker_data.rb +16 -0
  24. data/examples/worker_echo.rb +20 -0
  25. data/examples/worker_echo_pprof.rb +5 -0
  26. data/examples/worker_exception.rb +14 -0
  27. data/examples/worker_prefix.rb +25 -0
  28. data/gearman-ruby.gemspec +111 -0
  29. data/lib/gearman.rb +29 -0
  30. data/lib/gearman/client.rb +80 -0
  31. data/lib/gearman/evented/client.rb +99 -0
  32. data/lib/gearman/evented/reactor.rb +86 -0
  33. data/lib/gearman/evented/worker.rb +118 -0
  34. data/lib/gearman/job.rb +38 -0
  35. data/lib/gearman/protocol.rb +110 -0
  36. data/lib/gearman/server.rb +94 -0
  37. data/lib/gearman/task.rb +99 -0
  38. data/lib/gearman/taskset.rb +11 -0
  39. data/lib/gearman/util.rb +52 -0
  40. data/lib/gearman/worker.rb +39 -0
  41. data/test/basic_integration_test.rb +121 -0
  42. data/test/crash_test.rb +69 -0
  43. data/test/job_test.rb +30 -0
  44. data/test/protocol_test.rb +132 -0
  45. data/test/test_helper.rb +31 -0
  46. data/test/util_test.rb +12 -0
  47. data/test/worker_test.rb +45 -0
  48. metadata +133 -0
@@ -0,0 +1,26 @@
1
+ require 'rubygems'
2
+ require '../lib/gearman'
3
+
4
+ Gearman::Util.debug = true
5
+
6
+ servers = ['localhost:4730', 'localhost:4731']
7
+ w = Gearman::Worker.new(servers)
8
+
9
+ # Add a handler for a "sleep" function that takes a single argument, the
10
+ # number of seconds to sleep before reporting success.
11
+ w.add_ability('sleep') do |data, job|
12
+ seconds = data
13
+ job.report_warning("this is a warning you can safely ignore")
14
+
15
+ (1..seconds.to_i).each do |i|
16
+ sleep 1
17
+ print i
18
+ # Report our progress to the job server every second.
19
+ job.report_status(i, seconds)
20
+ job.send_partial(".")
21
+ end
22
+
23
+ # Report success.
24
+ "SUCCESS"
25
+ end
26
+ w.work
@@ -0,0 +1,25 @@
1
+ #!/bin/bash
2
+
3
+ # Start Gearmand
4
+ echo ' + Starting Gearmand'
5
+ gearmand --daemon --pidfile=/tmp/gearmand.pid
6
+
7
+ # Start the client and the worker(s)
8
+ echo ' + Starting calculus_worker.rb'
9
+ ruby calculus_worker.rb &
10
+
11
+ sleep 3
12
+
13
+ echo ' + Starting calculus_client.rb'
14
+ ruby calculus_client.rb
15
+
16
+ echo ' +++ Example finished +++ '
17
+
18
+ # Stop Gearmand
19
+ echo ' - Stopping Gearmand'
20
+ kill -9 `cat /tmp/gearmand.pid`
21
+
22
+ # Stop the workers
23
+ echo ' - Stopping calculus_worker.rb'
24
+ kill -9 `ps ax|grep calculus_worker|grep ruby|awk -F' ' '{print $1}'`
25
+
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/ruby
2
+
3
+ $: << '../lib'
4
+ require 'gearman'
5
+ require 'optparse'
6
+
7
+ Gearman::Util.debug = true
8
+ servers = 'localhost:7003'
9
+ format = 'PNG'
10
+ width, height = 100, 100
11
+
12
+ opts = OptionParser.new
13
+ opts.banner = "Usage: #{$0} [options] <input> <output>"
14
+ opts.on('-f FORMAT', '--format', 'Scaled image format') { format }
15
+ opts.on('-h HEIGHT', '--height', 'Scaled image height') { height }
16
+ opts.on('-s SERVERS', '--servers',
17
+ 'Servers, comma-separated host:port') { servers }
18
+ opts.on('-w WIDTH', '--width', 'Scaled image width') { width }
19
+ opts.parse!
20
+
21
+ if ARGV.size != 2
22
+ $stderr.puts opts.banner
23
+ exit 1
24
+ end
25
+
26
+ client = Gearman::Client.new(servers.split(','), 'example')
27
+ arg = [width, height, format, File.read(ARGV[0])].join("\0")
28
+ task = Gearman::Task.new('scale_image', arg)
29
+ task.on_complete {|d| File.new(ARGV[1],'w').write(d) }
30
+ client.run task
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/ruby
2
+
3
+ $: << '../lib'
4
+ require 'gearman'
5
+ require 'optparse'
6
+ require 'RMagick'
7
+
8
+ Gearman::Util.debug = true
9
+ servers = 'localhost:7003'
10
+
11
+ opts = OptionParser.new
12
+ opts.banner = "Usage: #{$0} [options]"
13
+ opts.on('-s SERVERS', '--servers',
14
+ 'Job servers, comma-separated host:port') { servers }
15
+ opts.parse!
16
+
17
+ worker = Gearman::Worker.new(servers.split(','), 'example')
18
+
19
+ worker.add_ability('scale_image') do |data,job|
20
+ width, height, format, data = data.split("\0", 4)
21
+ width = width.to_f
22
+ height = height.to_f
23
+ image = Magick::Image.from_blob(data)[0]
24
+ orig_ratio = image.columns.to_f / image.rows
25
+ new_ratio = width / height
26
+ w = new_ratio < orig_ratio ? width : orig_ratio / new_ratio * width
27
+ h = new_ratio > orig_ratio ? height : new_ratio / orig_ratio * height
28
+ puts "Got #{image.inspect}; resizing to #{w}x#{h} #{format}"
29
+ image.resize!(w, h)
30
+ image.format = format
31
+ image.to_blob
32
+ end
33
+
34
+ worker.work
@@ -0,0 +1,15 @@
1
+ require 'rubygems'
2
+ # require 'gearman'
3
+ # require 'gearman/server'
4
+ require '../lib/gearman'
5
+ require '../lib/gearman/server'
6
+ require 'pp'
7
+
8
+ Gearman::Util.debug = true
9
+ w = Gearman::Server.new('localhost:4730')
10
+
11
+ loop {
12
+ pp "Status: ", w.status
13
+ pp "Workers: ", w.workers
14
+ sleep 5
15
+ }
@@ -0,0 +1,23 @@
1
+ require 'rubygems'
2
+ #require 'gearman'
3
+ require '../lib/gearman'
4
+
5
+ Gearman::Util.debug = true
6
+
7
+ servers = ['localhost:4730', 'localhost:4731']
8
+ w = Gearman::Worker.new(servers)
9
+
10
+ # Add a handler for a "sleep" function that takes a single argument, the
11
+ # number of seconds to sleep before reporting success.
12
+ w.add_ability('sleep') do |data,job|
13
+ seconds = data
14
+ (1..seconds.to_i).each do |i|
15
+ sleep 1
16
+ print i
17
+ # Report our progress to the job server every second.
18
+ job.report_status(i, seconds)
19
+ end
20
+ # Report success.
21
+ true
22
+ end
23
+ w.work
@@ -0,0 +1,16 @@
1
+ require 'rubygems'
2
+ require '../lib/gearman'
3
+
4
+ Gearman::Util.debug = true
5
+
6
+ servers = ['localhost:4730']
7
+ worker = Gearman::Worker.new(servers)
8
+
9
+ worker.add_ability('chunked_transfer') do |data, job|
10
+ 5.times do |i|
11
+ sleep 1
12
+ job.send_partial("CHUNK #{i}")
13
+ end
14
+ "EOD"
15
+ end
16
+ worker.work
@@ -0,0 +1,20 @@
1
+ require 'rubygems'
2
+ require '../lib/gearman'
3
+ # require 'perftools'
4
+
5
+ quit = false
6
+ # PerfTools::CpuProfiler.start("/tmp/worker_echo_profile") do
7
+ servers = ['localhost:4730']
8
+ worker = Gearman::Worker.new(servers)
9
+
10
+ worker.add_ability('echo') do |data, job|
11
+ data
12
+ end
13
+ worker.add_ability('quit') do |data, job|
14
+ quit = true
15
+ end
16
+ loop {
17
+ break if quit
18
+ worker.work
19
+ }
20
+ # end
@@ -0,0 +1,5 @@
1
+ require 'rubygems'
2
+ require 'perftools'
3
+ PerfTools::CpuProfiler.start("/tmp/worker_echo_profile") do
4
+ require File.dirname(__FILE__) + "/worker_echo.rb"
5
+ end
@@ -0,0 +1,14 @@
1
+ require 'rubygems'
2
+ require '../lib/gearman'
3
+
4
+ Gearman::Util.debug = true
5
+
6
+ servers = ['localhost:4730']
7
+ w = Gearman::Worker.new(servers)
8
+
9
+ # Add a handler for a "sleep" function that takes a single argument, the
10
+ # number of seconds to sleep before reporting success.
11
+ w.add_ability('fail_with_exception') do |data,job|
12
+ raise Exception.new("Exception in worker (args: #{data.inspect})")
13
+ end
14
+ w.work
@@ -0,0 +1,25 @@
1
+ require 'rubygems'
2
+ #require 'gearman'
3
+ require '../lib/gearman'
4
+
5
+ Gearman::Util.debug = true
6
+
7
+ servers = ['localhost:4730', 'localhost:4731']
8
+ w = Gearman::Worker.new(servers)
9
+
10
+ ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep")
11
+
12
+ # Add a handler for a "sleep" function that takes a single argument, the
13
+ # number of seconds to sleep before reporting success.
14
+ w.add_ability(ability_name_with_prefix) do |data,job|
15
+ seconds = data
16
+ (1..seconds.to_i).each do |i|
17
+ sleep 1
18
+ print i
19
+ # Report our progress to the job server every second.
20
+ job.report_status(i, seconds)
21
+ end
22
+ # Report success.
23
+ true
24
+ end
25
+ w.work
@@ -0,0 +1,111 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = %q{gearman-ruby}
8
+ s.version = "2.0.0"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Kim Altintop"]
12
+ s.date = %q{2009-12-11}
13
+ s.description = %q{Library for the Gearman distributed job system}
14
+ s.email = %q{kim.altintop@gmail.com}
15
+ s.extra_rdoc_files = [
16
+ "LICENSE",
17
+ "README"
18
+ ]
19
+ s.files = [
20
+ ".gitignore",
21
+ "HOWTO",
22
+ "LICENSE",
23
+ "README",
24
+ "Rakefile",
25
+ "TODO",
26
+ "VERSION.yml",
27
+ "examples/calculus_client.rb",
28
+ "examples/calculus_worker.rb",
29
+ "examples/client.rb",
30
+ "examples/client_background.rb",
31
+ "examples/client_data.rb",
32
+ "examples/client_exception.rb",
33
+ "examples/client_prefix.rb",
34
+ "examples/evented_client.rb",
35
+ "examples/evented_worker.rb",
36
+ "examples/gearman_environment.sh",
37
+ "examples/scale_image.rb",
38
+ "examples/scale_image_worker.rb",
39
+ "examples/server.rb",
40
+ "examples/worker.rb",
41
+ "examples/worker_data.rb",
42
+ "examples/worker_exception.rb",
43
+ "examples/worker_prefix.rb",
44
+ "gearman-ruby.gemspec",
45
+ "lib/gearman.rb",
46
+ "lib/gearman/client.rb",
47
+ "lib/gearman/evented/client.rb",
48
+ "lib/gearman/evented/reactor.rb",
49
+ "lib/gearman/evented/worker.rb",
50
+ "lib/gearman/job.rb",
51
+ "lib/gearman/protocol.rb",
52
+ "lib/gearman/server.rb",
53
+ "lib/gearman/task.rb",
54
+ "lib/gearman/taskset.rb",
55
+ "lib/gearman/util.rb",
56
+ "lib/gearman/worker.rb",
57
+ "test/basic_integration_test.rb",
58
+ "test/crash_test.rb",
59
+ "test/job_test.rb",
60
+ "test/protocol_test.rb",
61
+ "test/test_helper.rb",
62
+ "test/util_test.rb",
63
+ "test/worker_test.rb"
64
+ ]
65
+ s.homepage = %q{http://github.com/kim/gearman-ruby}
66
+ s.rdoc_options = ["--charset=UTF-8"]
67
+ s.require_paths = ["lib"]
68
+ s.rubygems_version = %q{1.3.5}
69
+ s.summary = %q{Library for the Gearman distributed job system}
70
+ s.test_files = [
71
+ "test/basic_integration_test.rb",
72
+ "test/crash_test.rb",
73
+ "test/job_test.rb",
74
+ "test/protocol_test.rb",
75
+ "test/test_helper.rb",
76
+ "test/util_test.rb",
77
+ "test/worker_test.rb",
78
+ "examples/calculus_client.rb",
79
+ "examples/calculus_worker.rb",
80
+ "examples/client.rb",
81
+ "examples/client_background.rb",
82
+ "examples/client_data.rb",
83
+ "examples/client_echo.rb",
84
+ "examples/client_exception.rb",
85
+ "examples/client_prefix.rb",
86
+ "examples/evented_client.rb",
87
+ "examples/evented_worker.rb",
88
+ "examples/scale_image.rb",
89
+ "examples/scale_image_worker.rb",
90
+ "examples/server.rb",
91
+ "examples/worker.rb",
92
+ "examples/worker_data.rb",
93
+ "examples/worker_echo.rb",
94
+ "examples/worker_echo_pprof.rb",
95
+ "examples/worker_exception.rb",
96
+ "examples/worker_prefix.rb"
97
+ ]
98
+
99
+ if s.respond_to? :specification_version then
100
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
101
+ s.specification_version = 3
102
+
103
+ if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
104
+ s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
105
+ else
106
+ s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
107
+ end
108
+ else
109
+ s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
110
+ end
111
+ end
data/lib/gearman.rb ADDED
@@ -0,0 +1,29 @@
1
+ require 'rubygems'
2
+ require 'eventmachine'
3
+
4
+ module Gearman
5
+
6
+ require File.dirname(__FILE__) + '/gearman/evented/reactor'
7
+ require File.dirname(__FILE__) + '/gearman/evented/client'
8
+ require File.dirname(__FILE__) + '/gearman/evented/worker'
9
+ require File.dirname(__FILE__) + '/gearman/client'
10
+ require File.dirname(__FILE__) + '/gearman/task'
11
+ require File.dirname(__FILE__) + '/gearman/taskset'
12
+ require File.dirname(__FILE__) + '/gearman/util'
13
+ require File.dirname(__FILE__) + '/gearman/worker'
14
+ require File.dirname(__FILE__) + '/gearman/job'
15
+
16
+ require File.dirname(__FILE__) + '/gearman/protocol'
17
+
18
+
19
+ class InvalidArgsError < Exception
20
+ end
21
+
22
+ class NetworkError < Exception
23
+ end
24
+
25
+ def log(msg, force = false)
26
+ Util.log(msg, force)
27
+ end
28
+
29
+ end
@@ -0,0 +1,80 @@
1
+ module Gearman
2
+ class Client
3
+ attr_accessor :uniq, :jobs
4
+
5
+ def initialize(job_servers, opts = {})
6
+ @reactors = []
7
+ @jobs = {}
8
+
9
+ @job_servers = Array[*job_servers]
10
+
11
+ @uniq = opts.delete(:uniq)
12
+ @opts = opts
13
+ end
14
+
15
+ # Run a Task or Taskset
16
+ def run(taskset, timeout = nil, async = false)
17
+ timeout ||= 0
18
+ use_em_stop = EM.reactor_running?
19
+ EM.run do
20
+ @taskset ||= Taskset.new
21
+ @taskset += Taskset.create(taskset)
22
+
23
+ job_servers = @job_servers.dup
24
+ @reactors.each do |reactor|
25
+ unless reactor.connected?
26
+ reactor.reconnect true
27
+ reactor.keep_connected = async
28
+ reactor.callback { create_job(@taskset.shift, reactor) }
29
+ job_servers.delete reactor.to_s
30
+ else
31
+ create_job(@taskset.shift, reactor)
32
+ end
33
+ end
34
+ job_servers.each do |hostport|
35
+ host, port = hostport.split(":")
36
+ reactor = Gearman::Evented::ClientReactor.connect(host, port, @opts)
37
+ reactor.keep_connected = async
38
+ reactor.callback { create_job(@taskset.shift, reactor) }
39
+ @reactors << reactor
40
+ end
41
+
42
+ if timeout > 0
43
+ if use_em_stop
44
+ EM.add_timer(timeout) { EM.stop }
45
+ else
46
+ sleep timeout
47
+ end
48
+ elsif !async
49
+ Thread.new do
50
+ loop do
51
+ sleep 0.1
52
+ live = 0
53
+ @reactors.each {|reactor| live += 1 if reactor.connected? }
54
+ break if live == 0
55
+ end
56
+ end.join
57
+ end
58
+ end
59
+ end
60
+
61
+ private
62
+
63
+ def create_job(task, reactor = nil)
64
+ return unless task
65
+ reactor ||= @reactors[rand(@reactors.size)]
66
+ unless reactor.connected?
67
+ log "create_job: server #{reactor} not connected"
68
+ EM.next_tick { create_job(task) }
69
+ return
70
+ end
71
+
72
+ reactor.submit_job(task) {|handle| create_job(@taskset.shift) }
73
+ end
74
+
75
+ def log(msg)
76
+ Gearman::Util.log(msg)
77
+ end
78
+
79
+ end
80
+ end