sweatshop 1.4.0 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +9 -9
- data/VERSION.yml +1 -1
- data/bin/sweatd +1 -1
- data/lib/message_queue/base.rb +0 -5
- data/lib/{sweat_shop.rb → sweatshop.rb} +14 -22
- data/lib/{sweat_shop → sweatshop}/daemoned.rb +0 -0
- data/lib/{sweat_shop → sweatshop}/metaid.rb +0 -0
- data/lib/{sweat_shop → sweatshop}/sweatd.rb +7 -7
- data/lib/{sweat_shop → sweatshop}/worker.rb +6 -12
- data/script/sweatshop +2 -2
- data/test/hello_worker.rb +2 -2
- data/test/test_functional_worker.rb +10 -10
- data/test/test_sweatshop.rb +14 -14
- metadata +7 -7
data/README.markdown
CHANGED
@@ -1,20 +1,20 @@
|
|
1
|
-
#
|
1
|
+
# Sweatshop
|
2
2
|
|
3
|
-
|
3
|
+
Sweatshop provides an api to background resource intensive tasks. Much of the api design was copied from Workling, with a few tweaks.
|
4
4
|
Currently, it runs rabbitmq and kestrel, but it can support any number of queues.
|
5
5
|
|
6
6
|
## Installing
|
7
7
|
|
8
|
-
gem install
|
9
|
-
freeze in your gems directory (add config.gem '
|
10
|
-
cd vendor/gems/
|
8
|
+
gem install sweatshop
|
9
|
+
freeze in your gems directory (add config.gem 'sweatshop' to your environment)
|
10
|
+
cd vendor/gems/sweatshop
|
11
11
|
rake setup
|
12
12
|
|
13
13
|
## Writing workers
|
14
14
|
|
15
|
-
Put `email_worker.rb` into app/workers and sublcass `
|
15
|
+
Put `email_worker.rb` into app/workers and sublcass `Sweatshop::Worker`:
|
16
16
|
|
17
|
-
class EmailWorker <
|
17
|
+
class EmailWorker < Sweatshop::Worker
|
18
18
|
def send_mail(to)
|
19
19
|
user = User.find_by_id(to)
|
20
20
|
Mailer.deliver_welcome(to)
|
@@ -36,7 +36,7 @@ queues.
|
|
36
36
|
|
37
37
|
## Running the queue
|
38
38
|
|
39
|
-
|
39
|
+
Sweatshop has been tested with Rabbit and Kestrel, but it will also work with Starling. Please use the following resources to install the server:
|
40
40
|
|
41
41
|
Kestrel:
|
42
42
|
http://github.com/robey/kestrel/tree/master
|
@@ -91,7 +91,7 @@ By default, the script will run all workers defined in the app/workers dir. Ever
|
|
91
91
|
script/sweatshop -d
|
92
92
|
script/sweatshop -d stop
|
93
93
|
|
94
|
-
If you would like to run
|
94
|
+
If you would like to run Sweatshop as a daemon on a linux machine, use the initd.sh script provided in the sweatshop/script dir.
|
95
95
|
|
96
96
|
# REQUIREMENTS
|
97
97
|
|
data/VERSION.yml
CHANGED
data/bin/sweatd
CHANGED
data/lib/message_queue/base.rb
CHANGED
@@ -6,9 +6,9 @@ $:.unshift(File.dirname(__FILE__))
|
|
6
6
|
require 'message_queue/base'
|
7
7
|
require 'message_queue/rabbit'
|
8
8
|
require 'message_queue/kestrel'
|
9
|
-
require '
|
9
|
+
require 'sweatshop/worker'
|
10
10
|
|
11
|
-
module
|
11
|
+
module Sweatshop
|
12
12
|
extend self
|
13
13
|
|
14
14
|
def workers
|
@@ -31,30 +31,22 @@ module SweatShop
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def do_tasks(workers)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
34
|
+
loop do
|
35
|
+
wait = true
|
36
|
+
workers.each do |worker|
|
37
|
+
if task = worker.dequeue
|
38
|
+
worker.do_task(task)
|
39
|
+
wait = false
|
38
40
|
end
|
39
41
|
end
|
40
|
-
|
41
|
-
loop do
|
42
|
-
wait = true
|
42
|
+
if stop?
|
43
43
|
workers.each do |worker|
|
44
|
-
|
45
|
-
worker.do_task(task)
|
46
|
-
wait = false
|
47
|
-
end
|
48
|
-
end
|
49
|
-
if stop?
|
50
|
-
workers.each do |worker|
|
51
|
-
worker.stop
|
52
|
-
end
|
53
|
-
queue.stop
|
54
|
-
exit
|
44
|
+
worker.stop
|
55
45
|
end
|
56
|
-
|
46
|
+
queue.stop
|
47
|
+
exit
|
57
48
|
end
|
49
|
+
sleep 1 if wait
|
58
50
|
end
|
59
51
|
end
|
60
52
|
|
@@ -89,7 +81,7 @@ module SweatShop
|
|
89
81
|
|
90
82
|
def stop
|
91
83
|
@stop = true
|
92
|
-
queue.stop
|
84
|
+
queue.stop
|
93
85
|
end
|
94
86
|
|
95
87
|
def stop?
|
File without changes
|
File without changes
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/daemoned'
|
2
2
|
|
3
|
-
module
|
3
|
+
module Sweatshop
|
4
4
|
class Sweatd
|
5
5
|
include Daemoned
|
6
6
|
queues = []
|
@@ -30,12 +30,12 @@ module SweatShop
|
|
30
30
|
|
31
31
|
sig(:term, :int) do
|
32
32
|
puts "Shutting down sweatd..."
|
33
|
-
|
33
|
+
Sweatshop.stop
|
34
34
|
end
|
35
35
|
|
36
36
|
sig(:hup) do
|
37
37
|
puts "Received HUP"
|
38
|
-
|
38
|
+
Sweatshop.stop
|
39
39
|
remove_pid!
|
40
40
|
puts "Restarting sweatd with #{start_cmd}..."
|
41
41
|
`#{start_cmd}`
|
@@ -46,14 +46,14 @@ module SweatShop
|
|
46
46
|
puts "Loading Rails..."
|
47
47
|
require rails_root + '/config/environment'
|
48
48
|
end
|
49
|
-
require File.dirname(__FILE__) + '/../
|
49
|
+
require File.dirname(__FILE__) + '/../sweatshop'
|
50
50
|
end
|
51
51
|
|
52
52
|
daemonize(:kill_timeout => 20) do
|
53
53
|
workers = []
|
54
54
|
|
55
55
|
if groups.any?
|
56
|
-
workers +=
|
56
|
+
workers += Sweatshop.workers_in_group(groups)
|
57
57
|
end
|
58
58
|
|
59
59
|
if queues.any?
|
@@ -64,11 +64,11 @@ module SweatShop
|
|
64
64
|
worker_str = workers.join(',')
|
65
65
|
puts "Starting #{worker_str}..."
|
66
66
|
$0 = "Sweatd: #{worker_str}"
|
67
|
-
|
67
|
+
Sweatshop.do_tasks(workers)
|
68
68
|
else
|
69
69
|
puts "Starting all workers..."
|
70
70
|
$0 = 'Sweatd: all'
|
71
|
-
|
71
|
+
Sweatshop.do_all_tasks
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/metaid'
|
2
2
|
|
3
|
-
module
|
3
|
+
module Sweatshop
|
4
4
|
class Worker
|
5
5
|
def self.inherited(subclass)
|
6
6
|
self.workers << subclass
|
@@ -34,7 +34,7 @@ module SweatShop
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def self.config
|
37
|
-
|
37
|
+
Sweatshop.config
|
38
38
|
end
|
39
39
|
|
40
40
|
def self.queue_name
|
@@ -61,12 +61,6 @@ module SweatShop
|
|
61
61
|
queue.confirm(queue_name)
|
62
62
|
end
|
63
63
|
|
64
|
-
def self.subscribe
|
65
|
-
queue.subscribe(queue_name) do |task|
|
66
|
-
do_task(task)
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
64
|
def self.do_tasks
|
71
65
|
while task = dequeue
|
72
66
|
do_task(task)
|
@@ -92,19 +86,19 @@ module SweatShop
|
|
92
86
|
end
|
93
87
|
|
94
88
|
def self.queue
|
95
|
-
|
89
|
+
Sweatshop.queue(queue_group.to_s)
|
96
90
|
end
|
97
91
|
|
98
92
|
def self.workers
|
99
|
-
|
93
|
+
Sweatshop.workers
|
100
94
|
end
|
101
95
|
|
102
96
|
def self.config
|
103
|
-
|
97
|
+
Sweatshop.config
|
104
98
|
end
|
105
99
|
|
106
100
|
def self.log(msg)
|
107
|
-
|
101
|
+
Sweatshop.log(msg)
|
108
102
|
end
|
109
103
|
|
110
104
|
def self.call_before_task(task)
|
data/script/sweatshop
CHANGED
@@ -7,11 +7,11 @@ if ARGV.include?('-d')
|
|
7
7
|
root = File.expand_path(File.dirname(__FILE__) + '/..')
|
8
8
|
ARGV << "--rails=#{root}"
|
9
9
|
|
10
|
-
system("ruby #{root}/vendor/gems/
|
10
|
+
system("ruby #{root}/vendor/gems/sweatshop/lib/sweatshop/sweatd.rb #{ARGV.join(' ')}")
|
11
11
|
exit $?.exitstatus
|
12
12
|
else
|
13
13
|
puts "Loading Rails..."
|
14
14
|
require File.dirname(__FILE__) + '/../config/environment'
|
15
15
|
puts "Listening for new tasks..."
|
16
|
-
|
16
|
+
Sweatshop.do_all_tasks
|
17
17
|
end
|
data/test/hello_worker.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../lib/
|
2
|
-
class HelloWorker <
|
1
|
+
require File.dirname(__FILE__) + '/../lib/sweatshop'
|
2
|
+
class HelloWorker < Sweatshop::Worker
|
3
3
|
TEST_FILE = File.dirname(__FILE__) + '/test.txt' unless defined?(TEST_FILE)
|
4
4
|
|
5
5
|
def hello(name)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../lib/
|
1
|
+
require File.dirname(__FILE__) + '/../lib/sweatshop'
|
2
2
|
require File.dirname(__FILE__) + '/test_helper'
|
3
3
|
require File.dirname(__FILE__) + '/hello_worker'
|
4
4
|
|
@@ -9,20 +9,20 @@ class WorkerTest < Test::Unit::TestCase
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def teardown
|
12
|
-
|
13
|
-
|
12
|
+
Sweatshop.instance_variable_set("@config", nil)
|
13
|
+
Sweatshop.instance_variable_set("@queues", nil)
|
14
14
|
File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
|
15
15
|
end
|
16
16
|
|
17
17
|
should "daemonize" do
|
18
18
|
begin
|
19
|
-
|
20
|
-
|
19
|
+
Sweatshop.config['enable'] = true
|
20
|
+
Sweatshop.logger = :silent
|
21
21
|
|
22
22
|
HelloWorker.async_hello('Amos')
|
23
23
|
|
24
24
|
worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
|
25
|
-
sweatd = "#{File.dirname(__FILE__)}/../lib/
|
25
|
+
sweatd = "#{File.dirname(__FILE__)}/../lib/sweatshop/sweatd.rb"
|
26
26
|
|
27
27
|
`ruby #{sweatd} --worker-file #{worker} start`
|
28
28
|
`ruby #{sweatd} stop`
|
@@ -39,9 +39,9 @@ class WorkerTest < Test::Unit::TestCase
|
|
39
39
|
|
40
40
|
should "connect to fallback servers if the default one is down" do
|
41
41
|
begin
|
42
|
-
|
43
|
-
|
44
|
-
|
42
|
+
Sweatshop.logger = :silent
|
43
|
+
Sweatshop.config['enable'] = true
|
44
|
+
Sweatshop.config['default']['cluster'] =
|
45
45
|
[
|
46
46
|
'localhost:5671', # invalid
|
47
47
|
'localhost:5672' # valid
|
@@ -53,7 +53,7 @@ class WorkerTest < Test::Unit::TestCase
|
|
53
53
|
HelloWorker.queue.client = nil
|
54
54
|
|
55
55
|
HelloWorker.stop
|
56
|
-
|
56
|
+
Sweatshop.config['default']['cluster'] =
|
57
57
|
[
|
58
58
|
'localhost:5671',# valid
|
59
59
|
'localhost:5672' # invalid
|
data/test/test_sweatshop.rb
CHANGED
@@ -1,23 +1,23 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/test_helper'
|
2
|
-
require File.dirname(__FILE__) + '/../lib/
|
2
|
+
require File.dirname(__FILE__) + '/../lib/sweatshop'
|
3
3
|
|
4
|
-
class
|
5
|
-
|
4
|
+
class SweatshopTest < Test::Unit::TestCase
|
5
|
+
Sweatshop.workers = []
|
6
6
|
|
7
|
-
class HelloWorker <
|
7
|
+
class HelloWorker < Sweatshop::Worker
|
8
8
|
def hello(name)
|
9
9
|
"Hi, #{name}"
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
13
|
-
class GroupedWorker <
|
13
|
+
class GroupedWorker < Sweatshop::Worker
|
14
14
|
queue_group :foo
|
15
15
|
end
|
16
16
|
|
17
17
|
should "group workers" do
|
18
|
-
assert_equal [HelloWorker, GroupedWorker],
|
19
|
-
assert_equal [HelloWorker],
|
20
|
-
assert_equal [GroupedWorker],
|
18
|
+
assert_equal [HelloWorker, GroupedWorker], Sweatshop.workers_in_group(:all)
|
19
|
+
assert_equal [HelloWorker], Sweatshop.workers_in_group(:default)
|
20
|
+
assert_equal [GroupedWorker], Sweatshop.workers_in_group(:foo)
|
21
21
|
end
|
22
22
|
|
23
23
|
should "synch call" do
|
@@ -26,8 +26,8 @@ class SweatShopTest < Test::Unit::TestCase
|
|
26
26
|
end
|
27
27
|
|
28
28
|
should "assign a uid" do
|
29
|
-
|
30
|
-
|
29
|
+
Sweatshop.logger = :silent
|
30
|
+
Sweatshop.config['enable'] = false
|
31
31
|
uid = HelloWorker.async_hello('Amos')
|
32
32
|
assert_not_nil uid
|
33
33
|
end
|
@@ -47,7 +47,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
47
47
|
end
|
48
48
|
|
49
49
|
should "exception handler" do
|
50
|
-
|
50
|
+
Sweatshop.logger = :silent
|
51
51
|
|
52
52
|
exception = nil
|
53
53
|
HelloWorker.on_exception do |e|
|
@@ -60,7 +60,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
60
60
|
|
61
61
|
should "chain before tasks" do
|
62
62
|
MESSAGES = []
|
63
|
-
class BaseWorker <
|
63
|
+
class BaseWorker < Sweatshop::Worker
|
64
64
|
before_task do |task|
|
65
65
|
MESSAGES << 'base'
|
66
66
|
end
|
@@ -72,7 +72,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
72
72
|
end
|
73
73
|
SubWorker.call_before_task('foo')
|
74
74
|
assert_equal ['base', 'sub'], MESSAGES
|
75
|
-
|
76
|
-
|
75
|
+
Sweatshop.workers.delete(BaseWorker)
|
76
|
+
Sweatshop.workers.delete(SubWorker)
|
77
77
|
end
|
78
78
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sweatshop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amos Elliston
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-10-
|
12
|
+
date: 2009-10-20 00:00:00 -07:00
|
13
13
|
default_executable: sweatd
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -43,11 +43,11 @@ files:
|
|
43
43
|
- lib/message_queue/base.rb
|
44
44
|
- lib/message_queue/kestrel.rb
|
45
45
|
- lib/message_queue/rabbit.rb
|
46
|
-
- lib/
|
47
|
-
- lib/
|
48
|
-
- lib/
|
49
|
-
- lib/
|
50
|
-
- lib/
|
46
|
+
- lib/sweatshop.rb
|
47
|
+
- lib/sweatshop/daemoned.rb
|
48
|
+
- lib/sweatshop/metaid.rb
|
49
|
+
- lib/sweatshop/sweatd.rb
|
50
|
+
- lib/sweatshop/worker.rb
|
51
51
|
- script/initd.sh
|
52
52
|
- script/kestrel
|
53
53
|
- script/kestrel.sh
|