sweatshop 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.markdown +9 -9
- data/VERSION.yml +1 -1
- data/bin/sweatd +1 -1
- data/lib/message_queue/base.rb +0 -5
- data/lib/{sweat_shop.rb → sweatshop.rb} +14 -22
- data/lib/{sweat_shop → sweatshop}/daemoned.rb +0 -0
- data/lib/{sweat_shop → sweatshop}/metaid.rb +0 -0
- data/lib/{sweat_shop → sweatshop}/sweatd.rb +7 -7
- data/lib/{sweat_shop → sweatshop}/worker.rb +6 -12
- data/script/sweatshop +2 -2
- data/test/hello_worker.rb +2 -2
- data/test/test_functional_worker.rb +10 -10
- data/test/test_sweatshop.rb +14 -14
- metadata +7 -7
data/README.markdown
CHANGED
@@ -1,20 +1,20 @@
|
|
1
|
-
#
|
1
|
+
# Sweatshop
|
2
2
|
|
3
|
-
|
3
|
+
Sweatshop provides an api to background resource intensive tasks. Much of the api design was copied from Workling, with a few tweaks.
|
4
4
|
Currently, it runs rabbitmq and kestrel, but it can support any number of queues.
|
5
5
|
|
6
6
|
## Installing
|
7
7
|
|
8
|
-
gem install
|
9
|
-
freeze in your gems directory (add config.gem '
|
10
|
-
cd vendor/gems/
|
8
|
+
gem install sweatshop
|
9
|
+
freeze in your gems directory (add config.gem 'sweatshop' to your environment)
|
10
|
+
cd vendor/gems/sweatshop
|
11
11
|
rake setup
|
12
12
|
|
13
13
|
## Writing workers
|
14
14
|
|
15
|
-
Put `email_worker.rb` into app/workers and sublcass `
|
15
|
+
Put `email_worker.rb` into app/workers and sublcass `Sweatshop::Worker`:
|
16
16
|
|
17
|
-
class EmailWorker <
|
17
|
+
class EmailWorker < Sweatshop::Worker
|
18
18
|
def send_mail(to)
|
19
19
|
user = User.find_by_id(to)
|
20
20
|
Mailer.deliver_welcome(to)
|
@@ -36,7 +36,7 @@ queues.
|
|
36
36
|
|
37
37
|
## Running the queue
|
38
38
|
|
39
|
-
|
39
|
+
Sweatshop has been tested with Rabbit and Kestrel, but it will also work with Starling. Please use the following resources to install the server:
|
40
40
|
|
41
41
|
Kestrel:
|
42
42
|
http://github.com/robey/kestrel/tree/master
|
@@ -91,7 +91,7 @@ By default, the script will run all workers defined in the app/workers dir. Ever
|
|
91
91
|
script/sweatshop -d
|
92
92
|
script/sweatshop -d stop
|
93
93
|
|
94
|
-
If you would like to run
|
94
|
+
If you would like to run Sweatshop as a daemon on a linux machine, use the initd.sh script provided in the sweatshop/script dir.
|
95
95
|
|
96
96
|
# REQUIREMENTS
|
97
97
|
|
data/VERSION.yml
CHANGED
data/bin/sweatd
CHANGED
data/lib/message_queue/base.rb
CHANGED
@@ -6,9 +6,9 @@ $:.unshift(File.dirname(__FILE__))
|
|
6
6
|
require 'message_queue/base'
|
7
7
|
require 'message_queue/rabbit'
|
8
8
|
require 'message_queue/kestrel'
|
9
|
-
require '
|
9
|
+
require 'sweatshop/worker'
|
10
10
|
|
11
|
-
module
|
11
|
+
module Sweatshop
|
12
12
|
extend self
|
13
13
|
|
14
14
|
def workers
|
@@ -31,30 +31,22 @@ module SweatShop
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def do_tasks(workers)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
34
|
+
loop do
|
35
|
+
wait = true
|
36
|
+
workers.each do |worker|
|
37
|
+
if task = worker.dequeue
|
38
|
+
worker.do_task(task)
|
39
|
+
wait = false
|
38
40
|
end
|
39
41
|
end
|
40
|
-
|
41
|
-
loop do
|
42
|
-
wait = true
|
42
|
+
if stop?
|
43
43
|
workers.each do |worker|
|
44
|
-
|
45
|
-
worker.do_task(task)
|
46
|
-
wait = false
|
47
|
-
end
|
48
|
-
end
|
49
|
-
if stop?
|
50
|
-
workers.each do |worker|
|
51
|
-
worker.stop
|
52
|
-
end
|
53
|
-
queue.stop
|
54
|
-
exit
|
44
|
+
worker.stop
|
55
45
|
end
|
56
|
-
|
46
|
+
queue.stop
|
47
|
+
exit
|
57
48
|
end
|
49
|
+
sleep 1 if wait
|
58
50
|
end
|
59
51
|
end
|
60
52
|
|
@@ -89,7 +81,7 @@ module SweatShop
|
|
89
81
|
|
90
82
|
def stop
|
91
83
|
@stop = true
|
92
|
-
queue.stop
|
84
|
+
queue.stop
|
93
85
|
end
|
94
86
|
|
95
87
|
def stop?
|
File without changes
|
File without changes
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/daemoned'
|
2
2
|
|
3
|
-
module
|
3
|
+
module Sweatshop
|
4
4
|
class Sweatd
|
5
5
|
include Daemoned
|
6
6
|
queues = []
|
@@ -30,12 +30,12 @@ module SweatShop
|
|
30
30
|
|
31
31
|
sig(:term, :int) do
|
32
32
|
puts "Shutting down sweatd..."
|
33
|
-
|
33
|
+
Sweatshop.stop
|
34
34
|
end
|
35
35
|
|
36
36
|
sig(:hup) do
|
37
37
|
puts "Received HUP"
|
38
|
-
|
38
|
+
Sweatshop.stop
|
39
39
|
remove_pid!
|
40
40
|
puts "Restarting sweatd with #{start_cmd}..."
|
41
41
|
`#{start_cmd}`
|
@@ -46,14 +46,14 @@ module SweatShop
|
|
46
46
|
puts "Loading Rails..."
|
47
47
|
require rails_root + '/config/environment'
|
48
48
|
end
|
49
|
-
require File.dirname(__FILE__) + '/../
|
49
|
+
require File.dirname(__FILE__) + '/../sweatshop'
|
50
50
|
end
|
51
51
|
|
52
52
|
daemonize(:kill_timeout => 20) do
|
53
53
|
workers = []
|
54
54
|
|
55
55
|
if groups.any?
|
56
|
-
workers +=
|
56
|
+
workers += Sweatshop.workers_in_group(groups)
|
57
57
|
end
|
58
58
|
|
59
59
|
if queues.any?
|
@@ -64,11 +64,11 @@ module SweatShop
|
|
64
64
|
worker_str = workers.join(',')
|
65
65
|
puts "Starting #{worker_str}..."
|
66
66
|
$0 = "Sweatd: #{worker_str}"
|
67
|
-
|
67
|
+
Sweatshop.do_tasks(workers)
|
68
68
|
else
|
69
69
|
puts "Starting all workers..."
|
70
70
|
$0 = 'Sweatd: all'
|
71
|
-
|
71
|
+
Sweatshop.do_all_tasks
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/metaid'
|
2
2
|
|
3
|
-
module
|
3
|
+
module Sweatshop
|
4
4
|
class Worker
|
5
5
|
def self.inherited(subclass)
|
6
6
|
self.workers << subclass
|
@@ -34,7 +34,7 @@ module SweatShop
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def self.config
|
37
|
-
|
37
|
+
Sweatshop.config
|
38
38
|
end
|
39
39
|
|
40
40
|
def self.queue_name
|
@@ -61,12 +61,6 @@ module SweatShop
|
|
61
61
|
queue.confirm(queue_name)
|
62
62
|
end
|
63
63
|
|
64
|
-
def self.subscribe
|
65
|
-
queue.subscribe(queue_name) do |task|
|
66
|
-
do_task(task)
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
64
|
def self.do_tasks
|
71
65
|
while task = dequeue
|
72
66
|
do_task(task)
|
@@ -92,19 +86,19 @@ module SweatShop
|
|
92
86
|
end
|
93
87
|
|
94
88
|
def self.queue
|
95
|
-
|
89
|
+
Sweatshop.queue(queue_group.to_s)
|
96
90
|
end
|
97
91
|
|
98
92
|
def self.workers
|
99
|
-
|
93
|
+
Sweatshop.workers
|
100
94
|
end
|
101
95
|
|
102
96
|
def self.config
|
103
|
-
|
97
|
+
Sweatshop.config
|
104
98
|
end
|
105
99
|
|
106
100
|
def self.log(msg)
|
107
|
-
|
101
|
+
Sweatshop.log(msg)
|
108
102
|
end
|
109
103
|
|
110
104
|
def self.call_before_task(task)
|
data/script/sweatshop
CHANGED
@@ -7,11 +7,11 @@ if ARGV.include?('-d')
|
|
7
7
|
root = File.expand_path(File.dirname(__FILE__) + '/..')
|
8
8
|
ARGV << "--rails=#{root}"
|
9
9
|
|
10
|
-
system("ruby #{root}/vendor/gems/
|
10
|
+
system("ruby #{root}/vendor/gems/sweatshop/lib/sweatshop/sweatd.rb #{ARGV.join(' ')}")
|
11
11
|
exit $?.exitstatus
|
12
12
|
else
|
13
13
|
puts "Loading Rails..."
|
14
14
|
require File.dirname(__FILE__) + '/../config/environment'
|
15
15
|
puts "Listening for new tasks..."
|
16
|
-
|
16
|
+
Sweatshop.do_all_tasks
|
17
17
|
end
|
data/test/hello_worker.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../lib/
|
2
|
-
class HelloWorker <
|
1
|
+
require File.dirname(__FILE__) + '/../lib/sweatshop'
|
2
|
+
class HelloWorker < Sweatshop::Worker
|
3
3
|
TEST_FILE = File.dirname(__FILE__) + '/test.txt' unless defined?(TEST_FILE)
|
4
4
|
|
5
5
|
def hello(name)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/../lib/
|
1
|
+
require File.dirname(__FILE__) + '/../lib/sweatshop'
|
2
2
|
require File.dirname(__FILE__) + '/test_helper'
|
3
3
|
require File.dirname(__FILE__) + '/hello_worker'
|
4
4
|
|
@@ -9,20 +9,20 @@ class WorkerTest < Test::Unit::TestCase
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def teardown
|
12
|
-
|
13
|
-
|
12
|
+
Sweatshop.instance_variable_set("@config", nil)
|
13
|
+
Sweatshop.instance_variable_set("@queues", nil)
|
14
14
|
File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
|
15
15
|
end
|
16
16
|
|
17
17
|
should "daemonize" do
|
18
18
|
begin
|
19
|
-
|
20
|
-
|
19
|
+
Sweatshop.config['enable'] = true
|
20
|
+
Sweatshop.logger = :silent
|
21
21
|
|
22
22
|
HelloWorker.async_hello('Amos')
|
23
23
|
|
24
24
|
worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
|
25
|
-
sweatd = "#{File.dirname(__FILE__)}/../lib/
|
25
|
+
sweatd = "#{File.dirname(__FILE__)}/../lib/sweatshop/sweatd.rb"
|
26
26
|
|
27
27
|
`ruby #{sweatd} --worker-file #{worker} start`
|
28
28
|
`ruby #{sweatd} stop`
|
@@ -39,9 +39,9 @@ class WorkerTest < Test::Unit::TestCase
|
|
39
39
|
|
40
40
|
should "connect to fallback servers if the default one is down" do
|
41
41
|
begin
|
42
|
-
|
43
|
-
|
44
|
-
|
42
|
+
Sweatshop.logger = :silent
|
43
|
+
Sweatshop.config['enable'] = true
|
44
|
+
Sweatshop.config['default']['cluster'] =
|
45
45
|
[
|
46
46
|
'localhost:5671', # invalid
|
47
47
|
'localhost:5672' # valid
|
@@ -53,7 +53,7 @@ class WorkerTest < Test::Unit::TestCase
|
|
53
53
|
HelloWorker.queue.client = nil
|
54
54
|
|
55
55
|
HelloWorker.stop
|
56
|
-
|
56
|
+
Sweatshop.config['default']['cluster'] =
|
57
57
|
[
|
58
58
|
'localhost:5671',# valid
|
59
59
|
'localhost:5672' # invalid
|
data/test/test_sweatshop.rb
CHANGED
@@ -1,23 +1,23 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/test_helper'
|
2
|
-
require File.dirname(__FILE__) + '/../lib/
|
2
|
+
require File.dirname(__FILE__) + '/../lib/sweatshop'
|
3
3
|
|
4
|
-
class
|
5
|
-
|
4
|
+
class SweatshopTest < Test::Unit::TestCase
|
5
|
+
Sweatshop.workers = []
|
6
6
|
|
7
|
-
class HelloWorker <
|
7
|
+
class HelloWorker < Sweatshop::Worker
|
8
8
|
def hello(name)
|
9
9
|
"Hi, #{name}"
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
13
|
-
class GroupedWorker <
|
13
|
+
class GroupedWorker < Sweatshop::Worker
|
14
14
|
queue_group :foo
|
15
15
|
end
|
16
16
|
|
17
17
|
should "group workers" do
|
18
|
-
assert_equal [HelloWorker, GroupedWorker],
|
19
|
-
assert_equal [HelloWorker],
|
20
|
-
assert_equal [GroupedWorker],
|
18
|
+
assert_equal [HelloWorker, GroupedWorker], Sweatshop.workers_in_group(:all)
|
19
|
+
assert_equal [HelloWorker], Sweatshop.workers_in_group(:default)
|
20
|
+
assert_equal [GroupedWorker], Sweatshop.workers_in_group(:foo)
|
21
21
|
end
|
22
22
|
|
23
23
|
should "synch call" do
|
@@ -26,8 +26,8 @@ class SweatShopTest < Test::Unit::TestCase
|
|
26
26
|
end
|
27
27
|
|
28
28
|
should "assign a uid" do
|
29
|
-
|
30
|
-
|
29
|
+
Sweatshop.logger = :silent
|
30
|
+
Sweatshop.config['enable'] = false
|
31
31
|
uid = HelloWorker.async_hello('Amos')
|
32
32
|
assert_not_nil uid
|
33
33
|
end
|
@@ -47,7 +47,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
47
47
|
end
|
48
48
|
|
49
49
|
should "exception handler" do
|
50
|
-
|
50
|
+
Sweatshop.logger = :silent
|
51
51
|
|
52
52
|
exception = nil
|
53
53
|
HelloWorker.on_exception do |e|
|
@@ -60,7 +60,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
60
60
|
|
61
61
|
should "chain before tasks" do
|
62
62
|
MESSAGES = []
|
63
|
-
class BaseWorker <
|
63
|
+
class BaseWorker < Sweatshop::Worker
|
64
64
|
before_task do |task|
|
65
65
|
MESSAGES << 'base'
|
66
66
|
end
|
@@ -72,7 +72,7 @@ class SweatShopTest < Test::Unit::TestCase
|
|
72
72
|
end
|
73
73
|
SubWorker.call_before_task('foo')
|
74
74
|
assert_equal ['base', 'sub'], MESSAGES
|
75
|
-
|
76
|
-
|
75
|
+
Sweatshop.workers.delete(BaseWorker)
|
76
|
+
Sweatshop.workers.delete(SubWorker)
|
77
77
|
end
|
78
78
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sweatshop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amos Elliston
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-10-
|
12
|
+
date: 2009-10-20 00:00:00 -07:00
|
13
13
|
default_executable: sweatd
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -43,11 +43,11 @@ files:
|
|
43
43
|
- lib/message_queue/base.rb
|
44
44
|
- lib/message_queue/kestrel.rb
|
45
45
|
- lib/message_queue/rabbit.rb
|
46
|
-
- lib/
|
47
|
-
- lib/
|
48
|
-
- lib/
|
49
|
-
- lib/
|
50
|
-
- lib/
|
46
|
+
- lib/sweatshop.rb
|
47
|
+
- lib/sweatshop/daemoned.rb
|
48
|
+
- lib/sweatshop/metaid.rb
|
49
|
+
- lib/sweatshop/sweatd.rb
|
50
|
+
- lib/sweatshop/worker.rb
|
51
51
|
- script/initd.sh
|
52
52
|
- script/kestrel
|
53
53
|
- script/kestrel.sh
|