famoseagle-sweat_shop 0.3.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
data/README.markdown CHANGED
@@ -1,14 +1,14 @@
1
1
  # SweatShop
2
2
 
3
3
  SweatShop provides an api to background resource intensive tasks. Much of the api design was copied from Workling, with a few tweaks.
4
- Currently, it runs kestrel, but it can support any number of queues.
4
+ Currently, it runs rabbitmq and kestrel, but it can support any number of queues.
5
5
 
6
6
  ## Installing
7
7
 
8
- gem install sweat_shop
9
- freeze in your rails directory
10
- cd vendor/gems/sweat_shop
11
- rake setup
8
+ gem install sweat_shop
9
+ freeze in your rails directory
10
+ cd vendor/gems/sweat_shop
11
+ rake setup
12
12
 
13
13
  ## Writing workers
14
14
 
@@ -36,11 +36,15 @@ queues.
36
36
 
37
37
  ## Running the queue
38
38
 
39
- SweatShop has been tested with Kestrel, but it will also work with Starling. You can install and start kestrel following the instructions here:
39
+ SweatShop has been tested with Rabbit and Kestrel, but it will also work with Starling. Please use the following resources to install the server:
40
40
 
41
+ Kestrel:
41
42
  http://github.com/robey/kestrel/tree/master
42
43
 
43
- config/sweatshop.yml specifies the machine address of the queue (default localhost:22133).
44
+ Rabbit:
45
+ http://github.com/ezmobius/nanite/tree/master
46
+
47
+ config/sweatshop.yml specifies the machine address of the queue (default localhost:5672). You can also specify the queue type with the queue param.
44
48
 
45
49
  ## Running the workers
46
50
 
@@ -59,6 +63,7 @@ If you would like to run SweatShop as a daemon on a linux machine, use the initd
59
63
 
60
64
  i_can_daemonize
61
65
  memcache (for kestrel)
66
+ carrot (for rabbit)
62
67
 
63
68
  # LICENSE
64
69
 
data/VERSION.yml CHANGED
@@ -1,4 +1,4 @@
1
1
  ---
2
2
  :patch: 0
3
3
  :major: 0
4
- :minor: 3
4
+ :minor: 5
data/config/defaults.yml CHANGED
@@ -1,4 +1,5 @@
1
1
  # default options
2
2
  servers:
3
- - localhost:22133
3
+ - localhost:5672
4
4
  enable: true
5
+ queue: rabbit
data/config/sweatshop.yml CHANGED
@@ -1,12 +1,12 @@
1
1
  development:
2
2
  servers:
3
- - localhost:22133
3
+ - localhost:5672
4
4
  enable: true
5
5
  test:
6
6
  servers:
7
- - localhost:22133
8
- enable: true
7
+ - localhost:5672
8
+ enable: false
9
9
  production:
10
10
  servers:
11
- - localhost:22133
11
+ - localhost:5672
12
12
  enable: true
@@ -0,0 +1,17 @@
1
+ module MessageQueue
2
+ class Base
3
+ attr_reader :servers
4
+ def queue_size(queue); end
5
+ def enqueue(queue, data); end
6
+ def dequeue(queue); end
7
+ def confirm(queue); end
8
+ def subscribe(queue); end
9
+ def delete(queue); end
10
+ def client; end
11
+ def stop; end
12
+
13
+ def subscribe?
14
+ false
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,34 @@
1
+ module MessageQueue
2
+ class Kestrel < Base
3
+ attr_reader :client, :servers
4
+
5
+ def initialize(opts)
6
+ @servers = opts[:servers]
7
+ end
8
+
9
+ def queue_size(queue)
10
+ size = 0
11
+ stats = client.stats
12
+ servers.each do |server|
13
+ size += stats[server]["queue_#{queue}_items"].to_i
14
+ end
15
+ size
16
+ end
17
+
18
+ def enqueue(queue, data)
19
+ client.set(queue, data)
20
+ end
21
+
22
+ def dequeue(queue)
23
+ client.get("#{queue}/open")
24
+ end
25
+
26
+ def confirm(queue)
27
+ client.get("#{queue}/close")
28
+ end
29
+
30
+ def client
31
+ @client ||= MemCache.new(servers)
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,83 @@
1
+ require 'mq'
2
+ module MessageQueue
3
+ class Rabbit < Base
4
+ attr_accessor :em_thread
5
+
6
+ def initialize(opts={})
7
+ @servers = opts[:servers]
8
+ @info = {}
9
+ @host, @port = @servers.first.split(':')
10
+ @port = @port.to_i
11
+ end
12
+
13
+ def queue_size(queue)
14
+ num = 0
15
+ client.queue(queue).status{|messages, consumers| num = messages}
16
+ num
17
+ end
18
+
19
+ def enqueue(queue, data)
20
+ client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
21
+ end
22
+
23
+ def dequeue(queue)
24
+ client.queue(queue).pop do |info, task|
25
+ @info[queue] = info
26
+ return Marshal.load(task)
27
+ end
28
+ end
29
+
30
+ def confirm(queue)
31
+ if @info[queue]
32
+ @info[queue].ack
33
+ @info[queue] = nil
34
+ end
35
+ end
36
+
37
+ def client
38
+ @client ||= begin
39
+ start_em
40
+ if servers
41
+ MQ.new(AMQP.connect(:host => @host, :port => @port))
42
+ else
43
+ MQ.new
44
+ end
45
+ end
46
+ end
47
+
48
+ def start_em
49
+ if em_thread.nil? and not EM.reactor_running?
50
+ self.em_thread = Thread.new{EM.run}
51
+ ['INT', 'TERM'].each do |sig|
52
+ old = trap(sig) do
53
+ stop
54
+ old.call
55
+ end
56
+ end
57
+ end
58
+ end
59
+
60
+ def subscribe?
61
+ true
62
+ end
63
+
64
+ def subscribe(queue, &block)
65
+ AMQP.start(:host => @host, :port => @port) do
66
+ mq = MQ.new
67
+ mq.send(AMQP::Protocol::Basic::Qos.new(:prefetch_size => 0, :prefetch_count => 1, :global => false))
68
+ mq.queue(queue, :durable => true).subscribe(:ack => true) do |info, task|
69
+ if task
70
+ @info[queue] = info
71
+ task = Marshal.load(task)
72
+ block.call(task)
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ def stop
79
+ em_thread.join(0.15) unless em_thread.nil?
80
+ AMQP.stop{ EM.stop }
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,42 @@
1
+ require 'carrot'
2
+ module MessageQueue
3
+ class Rabbit < Base
4
+
5
+ def initialize(opts={})
6
+ @servers = opts[:servers]
7
+ @info = {}
8
+ @host, @port = @servers.first.split(':')
9
+ @port = @port.to_i
10
+ end
11
+
12
+ def delete(queue)
13
+ client.queue(queue).delete
14
+ end
15
+
16
+ def queue_size(queue)
17
+ client.queue(queue).message_count
18
+ end
19
+
20
+ def enqueue(queue, data)
21
+ client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
22
+ end
23
+
24
+ def dequeue(queue)
25
+ task = client.queue(queue).pop(:ack => true)
26
+ return unless task
27
+ Marshal.load(task)
28
+ end
29
+
30
+ def confirm(queue)
31
+ client.queue(queue).ack
32
+ end
33
+
34
+ def client
35
+ @client ||= Carrot.new(:host => @host, :port => @port)
36
+ end
37
+
38
+ def stop
39
+ client.stop
40
+ end
41
+ end
42
+ end
data/lib/sweat_shop.rb CHANGED
@@ -3,7 +3,9 @@ require 'digest'
3
3
  require 'yaml'
4
4
 
5
5
  $:.unshift(File.dirname(__FILE__))
6
- require 'kestrel'
6
+ require 'message_queue/base'
7
+ require 'message_queue/rabbit'
8
+ require 'message_queue/kestrel'
7
9
  require 'sweat_shop/worker'
8
10
 
9
11
  module SweatShop
@@ -29,16 +31,27 @@ module SweatShop
29
31
  end
30
32
 
31
33
  def do_tasks(workers)
32
- loop do
33
- wait = true
34
- workers.each do |worker|
35
- if task = worker.dequeue
36
- worker.do_task(task)
37
- wait = false
34
+ if queue.subscribe?
35
+ EM.run do
36
+ workers.each do |worker|
37
+ worker.subscribe
38
38
  end
39
39
  end
40
- exit if stop?
41
- sleep 1 if wait
40
+ else
41
+ loop do
42
+ wait = true
43
+ workers.each do |worker|
44
+ if task = worker.dequeue
45
+ worker.do_task(task)
46
+ wait = false
47
+ end
48
+ end
49
+ if stop?
50
+ queue.stop
51
+ exit
52
+ end
53
+ sleep 1 if wait
54
+ end
42
55
  end
43
56
  end
44
57
 
@@ -54,14 +67,6 @@ module SweatShop
54
67
  )
55
68
  end
56
69
 
57
- def stop
58
- @stop = true
59
- end
60
-
61
- def stop?
62
- @stop
63
- end
64
-
65
70
  def config
66
71
  @config ||= begin
67
72
  defaults = YAML.load_file(File.dirname(__FILE__) + '/../config/defaults.yml')
@@ -79,6 +84,15 @@ module SweatShop
79
84
  end
80
85
  end
81
86
 
87
+ def stop
88
+ @stop = true
89
+ queue.stop if queue.subscribe?
90
+ end
91
+
92
+ def stop?
93
+ @stop
94
+ end
95
+
82
96
  def queue_sizes
83
97
  workers.inject([]) do |all, worker|
84
98
  all << [worker, worker.queue_size]
@@ -94,12 +108,33 @@ module SweatShop
94
108
  end
95
109
 
96
110
  def queue
97
- @queue ||= Kestrel.new(:servers => config['servers'])
111
+ @queue ||= begin
112
+ queue = config['queue'] || 'rabbit'
113
+ queue = constantize("MessageQueue::#{queue.capitalize}")
114
+ queue.new(:servers => config['servers'])
115
+ end
98
116
  end
99
117
 
100
118
  def queue=(queue)
101
119
  @queue = queue
102
120
  end
121
+
122
+ def log(msg)
123
+ return if logger == :silent
124
+ logger ? logger.debug(msg) : puts(msg)
125
+ end
126
+
127
+ def logger
128
+ @logger
129
+ end
130
+
131
+ def logger=(logger)
132
+ @logger = logger
133
+ end
134
+
135
+ def constantize(str)
136
+ Object.module_eval("#{str}", __FILE__, __LINE__)
137
+ end
103
138
  end
104
139
 
105
140
  if defined?(RAILS_ROOT)
@@ -2,8 +2,6 @@ require File.dirname(__FILE__) + '/metaid'
2
2
 
3
3
  module SweatShop
4
4
  class Worker
5
- @@logger = nil
6
-
7
5
  def self.inherited(subclass)
8
6
  self.workers << subclass
9
7
  end
@@ -42,6 +40,10 @@ module SweatShop
42
40
  @queue_name ||= self.to_s
43
41
  end
44
42
 
43
+ def self.delete_queue
44
+ queue.delete(queue_name)
45
+ end
46
+
45
47
  def self.queue_size
46
48
  queue.queue_size(queue_name)
47
49
  end
@@ -57,6 +59,12 @@ module SweatShop
57
59
  def self.confirm
58
60
  queue.confirm(queue_name)
59
61
  end
62
+
63
+ def self.subscribe
64
+ queue.subscribe(queue_name) do |task|
65
+ do_task(task)
66
+ end
67
+ end
60
68
 
61
69
  def self.do_tasks
62
70
  while task = dequeue
@@ -85,21 +93,20 @@ module SweatShop
85
93
  after_task.call(task) if after_task
86
94
  end
87
95
 
88
- def self.workers
89
- SweatShop.workers
96
+ def self.queue
97
+ SweatShop.queue
90
98
  end
91
99
 
92
- def self.log(msg)
93
- return if logger == :silent
94
- logger ? logger.debug(msg) : puts(msg)
100
+ def self.workers
101
+ SweatShop.workers
95
102
  end
96
103
 
97
- def self.logger
98
- @@logger
104
+ def self.config
105
+ SweatShop.config
99
106
  end
100
107
 
101
- def self.logger=(logger)
102
- @@logger = logger
108
+ def self.log(msg)
109
+ SweatShop.log(msg)
103
110
  end
104
111
 
105
112
  def self.before_task(&block)
@@ -118,10 +125,6 @@ module SweatShop
118
125
  end
119
126
  end
120
127
 
121
- def self.queue
122
- SweatShop.queue
123
- end
124
-
125
128
  def self.queue_group(group=nil)
126
129
  group ? meta_def(:_queue_group){ group } : _queue_group
127
130
  end
@@ -12,11 +12,10 @@ class WorkerTest < Test::Unit::TestCase
12
12
  File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
13
13
  end
14
14
 
15
- # remove 'x' and start kestrel to run
16
15
  test "daemon" do
17
16
  begin
18
17
  SweatShop.queue = nil
19
- SweatShop::Worker.logger = :silent
18
+ SweatShop.logger = :silent
20
19
 
21
20
  worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
22
21
  sweatd = "#{File.dirname(__FILE__)}/../lib/sweat_shop/sweatd.rb"
@@ -27,8 +26,10 @@ class WorkerTest < Test::Unit::TestCase
27
26
 
28
27
  File.delete('sweatd.log') if File.exist?('sweatd.log')
29
28
  assert_equal 'Hi, Amos', File.read(HelloWorker::TEST_FILE)
30
- rescue MemCache::MemCacheError => e
31
- puts "\n\n*** Start kestrel on localhost to run all functional tests. ***\n\n"
29
+ rescue Exception => e
30
+ puts e.message
31
+ puts e.backtrace.join("\n")
32
+ fail "\n\n*** Functional test failed, is the rabbit server running on localhost? ***\n"
32
33
  end
33
34
  end
34
35
 
@@ -1,10 +1,8 @@
1
- require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_mock'
2
1
  require File.dirname(__FILE__) + '/test_helper'
3
2
  require File.dirname(__FILE__) + '/../lib/sweat_shop'
4
3
 
5
4
  class SweatShopTest < Test::Unit::TestCase
6
5
  SweatShop.workers = []
7
- SweatShop.queue = Kestrel.new(:client => MemCacheMock.new)
8
6
 
9
7
  class HelloWorker < SweatShop::Worker
10
8
  def hello(name)
@@ -28,7 +26,7 @@ class SweatShopTest < Test::Unit::TestCase
28
26
  end
29
27
 
30
28
  test "uid" do
31
- SweatShop::Worker.logger = :silent
29
+ SweatShop.logger = :silent
32
30
  uid = HelloWorker.async_hello('Amos')
33
31
  assert_not_nil uid
34
32
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: famoseagle-sweat_shop
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amos Elliston
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-03-18 00:00:00 -07:00
12
+ date: 2009-04-12 00:00:00 -07:00
13
13
  default_executable:
14
14
  dependencies: []
15
15
 
@@ -27,7 +27,11 @@ files:
27
27
  - Rakefile
28
28
  - README.markdown
29
29
  - VERSION.yml
30
- - lib/kestrel.rb
30
+ - lib/message_queue
31
+ - lib/message_queue/base.rb
32
+ - lib/message_queue/kestrel.rb
33
+ - lib/message_queue/rabbit-async.rb
34
+ - lib/message_queue/rabbit.rb
31
35
  - lib/sweat_shop
32
36
  - lib/sweat_shop/metaid.rb
33
37
  - lib/sweat_shop/sweatd.rb
data/lib/kestrel.rb DELETED
@@ -1,29 +0,0 @@
1
- class Kestrel
2
- attr_reader :client, :servers
3
-
4
- def initialize(opts)
5
- @servers = opts[:servers]
6
- @client = opts[:client] || MemCache.new(@servers)
7
- end
8
-
9
- def queue_size(queue)
10
- size = 0
11
- stats = client.stats
12
- servers.each do |server|
13
- size += stats[server]["queue_#{queue}_items"].to_i
14
- end
15
- size
16
- end
17
-
18
- def enqueue(queue, data)
19
- client.set(queue, data)
20
- end
21
-
22
- def dequeue(queue)
23
- client.get("#{queue}/open")
24
- end
25
-
26
- def confirm(queue)
27
- client.get("#{queue}/close")
28
- end
29
- end