famoseagle-sweat_shop 0.3.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +12 -7
- data/VERSION.yml +1 -1
- data/config/defaults.yml +2 -1
- data/config/sweatshop.yml +4 -4
- data/lib/message_queue/base.rb +17 -0
- data/lib/message_queue/kestrel.rb +34 -0
- data/lib/message_queue/rabbit-async.rb +83 -0
- data/lib/message_queue/rabbit.rb +42 -0
- data/lib/sweat_shop.rb +53 -18
- data/lib/sweat_shop/worker.rb +18 -15
- data/test/test_functional_worker.rb +5 -4
- data/test/test_sweatshop.rb +1 -3
- metadata +7 -3
- data/lib/kestrel.rb +0 -29
data/README.markdown
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# SweatShop
|
2
2
|
|
3
3
|
SweatShop provides an api to background resource intensive tasks. Much of the api design was copied from Workling, with a few tweaks.
|
4
|
-
Currently, it runs kestrel, but it can support any number of queues.
|
4
|
+
Currently, it runs rabbitmq and kestrel, but it can support any number of queues.
|
5
5
|
|
6
6
|
## Installing
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
8
|
+
gem install sweat_shop
|
9
|
+
freeze in your rails directory
|
10
|
+
cd vendor/gems/sweat_shop
|
11
|
+
rake setup
|
12
12
|
|
13
13
|
## Writing workers
|
14
14
|
|
@@ -36,11 +36,15 @@ queues.
|
|
36
36
|
|
37
37
|
## Running the queue
|
38
38
|
|
39
|
-
SweatShop has been tested with Kestrel, but it will also work with Starling.
|
39
|
+
SweatShop has been tested with Rabbit and Kestrel, but it will also work with Starling. Please use the following resources to install the server:
|
40
40
|
|
41
|
+
Kestrel:
|
41
42
|
http://github.com/robey/kestrel/tree/master
|
42
43
|
|
43
|
-
|
44
|
+
Rabbit:
|
45
|
+
http://github.com/ezmobius/nanite/tree/master
|
46
|
+
|
47
|
+
config/sweatshop.yml specifies the machine address of the queue (default localhost:5672). You can also specify the queue type with the queue param.
|
44
48
|
|
45
49
|
## Running the workers
|
46
50
|
|
@@ -59,6 +63,7 @@ If you would like to run SweatShop as a daemon on a linux machine, use the initd
|
|
59
63
|
|
60
64
|
i_can_daemonize
|
61
65
|
memcache (for kestrel)
|
66
|
+
carrot (for rabbit)
|
62
67
|
|
63
68
|
# LICENSE
|
64
69
|
|
data/VERSION.yml
CHANGED
data/config/defaults.yml
CHANGED
data/config/sweatshop.yml
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
development:
|
2
2
|
servers:
|
3
|
-
- localhost:
|
3
|
+
- localhost:5672
|
4
4
|
enable: true
|
5
5
|
test:
|
6
6
|
servers:
|
7
|
-
- localhost:
|
8
|
-
enable:
|
7
|
+
- localhost:5672
|
8
|
+
enable: false
|
9
9
|
production:
|
10
10
|
servers:
|
11
|
-
- localhost:
|
11
|
+
- localhost:5672
|
12
12
|
enable: true
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module MessageQueue
|
2
|
+
class Base
|
3
|
+
attr_reader :servers
|
4
|
+
def queue_size(queue); end
|
5
|
+
def enqueue(queue, data); end
|
6
|
+
def dequeue(queue); end
|
7
|
+
def confirm(queue); end
|
8
|
+
def subscribe(queue); end
|
9
|
+
def delete(queue); end
|
10
|
+
def client; end
|
11
|
+
def stop; end
|
12
|
+
|
13
|
+
def subscribe?
|
14
|
+
false
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module MessageQueue
|
2
|
+
class Kestrel < Base
|
3
|
+
attr_reader :client, :servers
|
4
|
+
|
5
|
+
def initialize(opts)
|
6
|
+
@servers = opts[:servers]
|
7
|
+
end
|
8
|
+
|
9
|
+
def queue_size(queue)
|
10
|
+
size = 0
|
11
|
+
stats = client.stats
|
12
|
+
servers.each do |server|
|
13
|
+
size += stats[server]["queue_#{queue}_items"].to_i
|
14
|
+
end
|
15
|
+
size
|
16
|
+
end
|
17
|
+
|
18
|
+
def enqueue(queue, data)
|
19
|
+
client.set(queue, data)
|
20
|
+
end
|
21
|
+
|
22
|
+
def dequeue(queue)
|
23
|
+
client.get("#{queue}/open")
|
24
|
+
end
|
25
|
+
|
26
|
+
def confirm(queue)
|
27
|
+
client.get("#{queue}/close")
|
28
|
+
end
|
29
|
+
|
30
|
+
def client
|
31
|
+
@client ||= MemCache.new(servers)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'mq'
|
2
|
+
module MessageQueue
|
3
|
+
class Rabbit < Base
|
4
|
+
attr_accessor :em_thread
|
5
|
+
|
6
|
+
def initialize(opts={})
|
7
|
+
@servers = opts[:servers]
|
8
|
+
@info = {}
|
9
|
+
@host, @port = @servers.first.split(':')
|
10
|
+
@port = @port.to_i
|
11
|
+
end
|
12
|
+
|
13
|
+
def queue_size(queue)
|
14
|
+
num = 0
|
15
|
+
client.queue(queue).status{|messages, consumers| num = messages}
|
16
|
+
num
|
17
|
+
end
|
18
|
+
|
19
|
+
def enqueue(queue, data)
|
20
|
+
client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
|
21
|
+
end
|
22
|
+
|
23
|
+
def dequeue(queue)
|
24
|
+
client.queue(queue).pop do |info, task|
|
25
|
+
@info[queue] = info
|
26
|
+
return Marshal.load(task)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def confirm(queue)
|
31
|
+
if @info[queue]
|
32
|
+
@info[queue].ack
|
33
|
+
@info[queue] = nil
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def client
|
38
|
+
@client ||= begin
|
39
|
+
start_em
|
40
|
+
if servers
|
41
|
+
MQ.new(AMQP.connect(:host => @host, :port => @port))
|
42
|
+
else
|
43
|
+
MQ.new
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def start_em
|
49
|
+
if em_thread.nil? and not EM.reactor_running?
|
50
|
+
self.em_thread = Thread.new{EM.run}
|
51
|
+
['INT', 'TERM'].each do |sig|
|
52
|
+
old = trap(sig) do
|
53
|
+
stop
|
54
|
+
old.call
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def subscribe?
|
61
|
+
true
|
62
|
+
end
|
63
|
+
|
64
|
+
def subscribe(queue, &block)
|
65
|
+
AMQP.start(:host => @host, :port => @port) do
|
66
|
+
mq = MQ.new
|
67
|
+
mq.send(AMQP::Protocol::Basic::Qos.new(:prefetch_size => 0, :prefetch_count => 1, :global => false))
|
68
|
+
mq.queue(queue, :durable => true).subscribe(:ack => true) do |info, task|
|
69
|
+
if task
|
70
|
+
@info[queue] = info
|
71
|
+
task = Marshal.load(task)
|
72
|
+
block.call(task)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def stop
|
79
|
+
em_thread.join(0.15) unless em_thread.nil?
|
80
|
+
AMQP.stop{ EM.stop }
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'carrot'
|
2
|
+
module MessageQueue
|
3
|
+
class Rabbit < Base
|
4
|
+
|
5
|
+
def initialize(opts={})
|
6
|
+
@servers = opts[:servers]
|
7
|
+
@info = {}
|
8
|
+
@host, @port = @servers.first.split(':')
|
9
|
+
@port = @port.to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
def delete(queue)
|
13
|
+
client.queue(queue).delete
|
14
|
+
end
|
15
|
+
|
16
|
+
def queue_size(queue)
|
17
|
+
client.queue(queue).message_count
|
18
|
+
end
|
19
|
+
|
20
|
+
def enqueue(queue, data)
|
21
|
+
client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
|
22
|
+
end
|
23
|
+
|
24
|
+
def dequeue(queue)
|
25
|
+
task = client.queue(queue).pop(:ack => true)
|
26
|
+
return unless task
|
27
|
+
Marshal.load(task)
|
28
|
+
end
|
29
|
+
|
30
|
+
def confirm(queue)
|
31
|
+
client.queue(queue).ack
|
32
|
+
end
|
33
|
+
|
34
|
+
def client
|
35
|
+
@client ||= Carrot.new(:host => @host, :port => @port)
|
36
|
+
end
|
37
|
+
|
38
|
+
def stop
|
39
|
+
client.stop
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/sweat_shop.rb
CHANGED
@@ -3,7 +3,9 @@ require 'digest'
|
|
3
3
|
require 'yaml'
|
4
4
|
|
5
5
|
$:.unshift(File.dirname(__FILE__))
|
6
|
-
require '
|
6
|
+
require 'message_queue/base'
|
7
|
+
require 'message_queue/rabbit'
|
8
|
+
require 'message_queue/kestrel'
|
7
9
|
require 'sweat_shop/worker'
|
8
10
|
|
9
11
|
module SweatShop
|
@@ -29,16 +31,27 @@ module SweatShop
|
|
29
31
|
end
|
30
32
|
|
31
33
|
def do_tasks(workers)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
worker.do_task(task)
|
37
|
-
wait = false
|
34
|
+
if queue.subscribe?
|
35
|
+
EM.run do
|
36
|
+
workers.each do |worker|
|
37
|
+
worker.subscribe
|
38
38
|
end
|
39
39
|
end
|
40
|
-
|
41
|
-
|
40
|
+
else
|
41
|
+
loop do
|
42
|
+
wait = true
|
43
|
+
workers.each do |worker|
|
44
|
+
if task = worker.dequeue
|
45
|
+
worker.do_task(task)
|
46
|
+
wait = false
|
47
|
+
end
|
48
|
+
end
|
49
|
+
if stop?
|
50
|
+
queue.stop
|
51
|
+
exit
|
52
|
+
end
|
53
|
+
sleep 1 if wait
|
54
|
+
end
|
42
55
|
end
|
43
56
|
end
|
44
57
|
|
@@ -54,14 +67,6 @@ module SweatShop
|
|
54
67
|
)
|
55
68
|
end
|
56
69
|
|
57
|
-
def stop
|
58
|
-
@stop = true
|
59
|
-
end
|
60
|
-
|
61
|
-
def stop?
|
62
|
-
@stop
|
63
|
-
end
|
64
|
-
|
65
70
|
def config
|
66
71
|
@config ||= begin
|
67
72
|
defaults = YAML.load_file(File.dirname(__FILE__) + '/../config/defaults.yml')
|
@@ -79,6 +84,15 @@ module SweatShop
|
|
79
84
|
end
|
80
85
|
end
|
81
86
|
|
87
|
+
def stop
|
88
|
+
@stop = true
|
89
|
+
queue.stop if queue.subscribe?
|
90
|
+
end
|
91
|
+
|
92
|
+
def stop?
|
93
|
+
@stop
|
94
|
+
end
|
95
|
+
|
82
96
|
def queue_sizes
|
83
97
|
workers.inject([]) do |all, worker|
|
84
98
|
all << [worker, worker.queue_size]
|
@@ -94,12 +108,33 @@ module SweatShop
|
|
94
108
|
end
|
95
109
|
|
96
110
|
def queue
|
97
|
-
@queue ||=
|
111
|
+
@queue ||= begin
|
112
|
+
queue = config['queue'] || 'rabbit'
|
113
|
+
queue = constantize("MessageQueue::#{queue.capitalize}")
|
114
|
+
queue.new(:servers => config['servers'])
|
115
|
+
end
|
98
116
|
end
|
99
117
|
|
100
118
|
def queue=(queue)
|
101
119
|
@queue = queue
|
102
120
|
end
|
121
|
+
|
122
|
+
def log(msg)
|
123
|
+
return if logger == :silent
|
124
|
+
logger ? logger.debug(msg) : puts(msg)
|
125
|
+
end
|
126
|
+
|
127
|
+
def logger
|
128
|
+
@logger
|
129
|
+
end
|
130
|
+
|
131
|
+
def logger=(logger)
|
132
|
+
@logger = logger
|
133
|
+
end
|
134
|
+
|
135
|
+
def constantize(str)
|
136
|
+
Object.module_eval("#{str}", __FILE__, __LINE__)
|
137
|
+
end
|
103
138
|
end
|
104
139
|
|
105
140
|
if defined?(RAILS_ROOT)
|
data/lib/sweat_shop/worker.rb
CHANGED
@@ -2,8 +2,6 @@ require File.dirname(__FILE__) + '/metaid'
|
|
2
2
|
|
3
3
|
module SweatShop
|
4
4
|
class Worker
|
5
|
-
@@logger = nil
|
6
|
-
|
7
5
|
def self.inherited(subclass)
|
8
6
|
self.workers << subclass
|
9
7
|
end
|
@@ -42,6 +40,10 @@ module SweatShop
|
|
42
40
|
@queue_name ||= self.to_s
|
43
41
|
end
|
44
42
|
|
43
|
+
def self.delete_queue
|
44
|
+
queue.delete(queue_name)
|
45
|
+
end
|
46
|
+
|
45
47
|
def self.queue_size
|
46
48
|
queue.queue_size(queue_name)
|
47
49
|
end
|
@@ -57,6 +59,12 @@ module SweatShop
|
|
57
59
|
def self.confirm
|
58
60
|
queue.confirm(queue_name)
|
59
61
|
end
|
62
|
+
|
63
|
+
def self.subscribe
|
64
|
+
queue.subscribe(queue_name) do |task|
|
65
|
+
do_task(task)
|
66
|
+
end
|
67
|
+
end
|
60
68
|
|
61
69
|
def self.do_tasks
|
62
70
|
while task = dequeue
|
@@ -85,21 +93,20 @@ module SweatShop
|
|
85
93
|
after_task.call(task) if after_task
|
86
94
|
end
|
87
95
|
|
88
|
-
def self.
|
89
|
-
SweatShop.
|
96
|
+
def self.queue
|
97
|
+
SweatShop.queue
|
90
98
|
end
|
91
99
|
|
92
|
-
def self.
|
93
|
-
|
94
|
-
logger ? logger.debug(msg) : puts(msg)
|
100
|
+
def self.workers
|
101
|
+
SweatShop.workers
|
95
102
|
end
|
96
103
|
|
97
|
-
def self.
|
98
|
-
|
104
|
+
def self.config
|
105
|
+
SweatShop.config
|
99
106
|
end
|
100
107
|
|
101
|
-
def self.
|
102
|
-
|
108
|
+
def self.log(msg)
|
109
|
+
SweatShop.log(msg)
|
103
110
|
end
|
104
111
|
|
105
112
|
def self.before_task(&block)
|
@@ -118,10 +125,6 @@ module SweatShop
|
|
118
125
|
end
|
119
126
|
end
|
120
127
|
|
121
|
-
def self.queue
|
122
|
-
SweatShop.queue
|
123
|
-
end
|
124
|
-
|
125
128
|
def self.queue_group(group=nil)
|
126
129
|
group ? meta_def(:_queue_group){ group } : _queue_group
|
127
130
|
end
|
@@ -12,11 +12,10 @@ class WorkerTest < Test::Unit::TestCase
|
|
12
12
|
File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
|
13
13
|
end
|
14
14
|
|
15
|
-
# remove 'x' and start kestrel to run
|
16
15
|
test "daemon" do
|
17
16
|
begin
|
18
17
|
SweatShop.queue = nil
|
19
|
-
SweatShop
|
18
|
+
SweatShop.logger = :silent
|
20
19
|
|
21
20
|
worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
|
22
21
|
sweatd = "#{File.dirname(__FILE__)}/../lib/sweat_shop/sweatd.rb"
|
@@ -27,8 +26,10 @@ class WorkerTest < Test::Unit::TestCase
|
|
27
26
|
|
28
27
|
File.delete('sweatd.log') if File.exist?('sweatd.log')
|
29
28
|
assert_equal 'Hi, Amos', File.read(HelloWorker::TEST_FILE)
|
30
|
-
rescue
|
31
|
-
puts
|
29
|
+
rescue Exception => e
|
30
|
+
puts e.message
|
31
|
+
puts e.backtrace.join("\n")
|
32
|
+
fail "\n\n*** Functional test failed, is the rabbit server running on localhost? ***\n"
|
32
33
|
end
|
33
34
|
end
|
34
35
|
|
data/test/test_sweatshop.rb
CHANGED
@@ -1,10 +1,8 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_mock'
|
2
1
|
require File.dirname(__FILE__) + '/test_helper'
|
3
2
|
require File.dirname(__FILE__) + '/../lib/sweat_shop'
|
4
3
|
|
5
4
|
class SweatShopTest < Test::Unit::TestCase
|
6
5
|
SweatShop.workers = []
|
7
|
-
SweatShop.queue = Kestrel.new(:client => MemCacheMock.new)
|
8
6
|
|
9
7
|
class HelloWorker < SweatShop::Worker
|
10
8
|
def hello(name)
|
@@ -28,7 +26,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
28
26
|
end
|
29
27
|
|
30
28
|
test "uid" do
|
31
|
-
SweatShop
|
29
|
+
SweatShop.logger = :silent
|
32
30
|
uid = HelloWorker.async_hello('Amos')
|
33
31
|
assert_not_nil uid
|
34
32
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: famoseagle-sweat_shop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amos Elliston
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-
|
12
|
+
date: 2009-04-12 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -27,7 +27,11 @@ files:
|
|
27
27
|
- Rakefile
|
28
28
|
- README.markdown
|
29
29
|
- VERSION.yml
|
30
|
-
- lib/
|
30
|
+
- lib/message_queue
|
31
|
+
- lib/message_queue/base.rb
|
32
|
+
- lib/message_queue/kestrel.rb
|
33
|
+
- lib/message_queue/rabbit-async.rb
|
34
|
+
- lib/message_queue/rabbit.rb
|
31
35
|
- lib/sweat_shop
|
32
36
|
- lib/sweat_shop/metaid.rb
|
33
37
|
- lib/sweat_shop/sweatd.rb
|
data/lib/kestrel.rb
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
class Kestrel
|
2
|
-
attr_reader :client, :servers
|
3
|
-
|
4
|
-
def initialize(opts)
|
5
|
-
@servers = opts[:servers]
|
6
|
-
@client = opts[:client] || MemCache.new(@servers)
|
7
|
-
end
|
8
|
-
|
9
|
-
def queue_size(queue)
|
10
|
-
size = 0
|
11
|
-
stats = client.stats
|
12
|
-
servers.each do |server|
|
13
|
-
size += stats[server]["queue_#{queue}_items"].to_i
|
14
|
-
end
|
15
|
-
size
|
16
|
-
end
|
17
|
-
|
18
|
-
def enqueue(queue, data)
|
19
|
-
client.set(queue, data)
|
20
|
-
end
|
21
|
-
|
22
|
-
def dequeue(queue)
|
23
|
-
client.get("#{queue}/open")
|
24
|
-
end
|
25
|
-
|
26
|
-
def confirm(queue)
|
27
|
-
client.get("#{queue}/close")
|
28
|
-
end
|
29
|
-
end
|