micro_q 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +1 -1
- data/lib/micro_q/config.rb +4 -2
- data/lib/micro_q/manager/default.rb +4 -2
- data/lib/micro_q/methods/action_mailer.rb +6 -0
- data/lib/micro_q/methods.rb +1 -0
- data/lib/micro_q/queue/default.rb +43 -12
- data/lib/micro_q/queue/redis.rb +97 -0
- data/lib/micro_q/queue.rb +7 -1
- data/lib/micro_q/redis.rb +15 -0
- data/lib/micro_q/util.rb +10 -0
- data/lib/micro_q/version.rb +1 -1
- data/lib/micro_q.rb +5 -0
- data/micro_q.gemspec +5 -2
- data/spec/helpers/queues_examples.rb +37 -0
- data/spec/lib/config_spec.rb +10 -2
- data/spec/lib/manager/default_spec.rb +7 -1
- data/spec/lib/queue/default_spec.rb +97 -33
- data/spec/lib/queue/redis_spec.rb +162 -0
- data/spec/spec_helper.rb +11 -3
- metadata +65 -6
- /data/spec/{wrappers → lib/wrappers}/action_mailer_spec.rb +0 -0
data/.rspec
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
--color
|
2
|
-
--format
|
2
|
+
--format documentation
|
data/lib/micro_q/config.rb
CHANGED
@@ -10,13 +10,15 @@ module MicroQ
|
|
10
10
|
#
|
11
11
|
def initialize
|
12
12
|
@data = {
|
13
|
-
'workers' =>
|
13
|
+
'workers' => 5,
|
14
14
|
'timeout' => 120,
|
15
15
|
'interval' => 5,
|
16
16
|
'middleware' => Middleware::Chain.new,
|
17
17
|
'manager' => Manager::Default,
|
18
18
|
'worker' => Worker::Standard,
|
19
|
-
'queue' => Queue::Default
|
19
|
+
'queue' => Queue::Default,
|
20
|
+
'redis_pool' => { :size => 15, :timeout => 1 },
|
21
|
+
'redis' => { :host => 'localhost', :port => 6379 }
|
20
22
|
}
|
21
23
|
end
|
22
24
|
|
@@ -25,13 +25,15 @@ module MicroQ
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def start
|
28
|
-
|
28
|
+
count = workers.idle_size
|
29
|
+
|
30
|
+
if (messages = queue.dequeue(count)).any?
|
29
31
|
messages.each do |message|
|
30
32
|
workers.perform!(message)
|
31
33
|
end
|
32
34
|
end
|
33
35
|
|
34
|
-
after(
|
36
|
+
after(2) { start }
|
35
37
|
end
|
36
38
|
end
|
37
39
|
end
|
@@ -1,5 +1,11 @@
|
|
1
1
|
module MicroQ
|
2
2
|
module Methods
|
3
|
+
##
|
4
|
+
# Methods that are added to ActionMailer instances
|
5
|
+
#
|
6
|
+
# When mailing asynchronously, the deliver method needs to be
|
7
|
+
# called which means a custom wrapper.
|
8
|
+
#
|
3
9
|
module ActionMailer
|
4
10
|
def async
|
5
11
|
MicroQ::Proxy::ActionMailer.new(
|
data/lib/micro_q/methods.rb
CHANGED
@@ -5,6 +5,9 @@ module MicroQ
|
|
5
5
|
# Handles messages that should be run immediately as well as messages that
|
6
6
|
# should be run at some specified time in the future.
|
7
7
|
#
|
8
|
+
# When shutting down, if the MicroQ.config.queue_file is defined and accessible,
|
9
|
+
# the messages in the queue will be written for persistence.
|
10
|
+
#
|
8
11
|
# Usage:
|
9
12
|
#
|
10
13
|
# item = { 'class' => 'MyWorker', 'args' => [user.id] }
|
@@ -29,6 +32,8 @@ module MicroQ
|
|
29
32
|
def initialize
|
30
33
|
@entries = []
|
31
34
|
@later = []
|
35
|
+
|
36
|
+
load_queues
|
32
37
|
end
|
33
38
|
|
34
39
|
##
|
@@ -39,17 +44,18 @@ module MicroQ
|
|
39
44
|
end
|
40
45
|
|
41
46
|
##
|
42
|
-
#
|
47
|
+
# Asynchronously push a message item to the queue.
|
43
48
|
# Either push it to the immediate portion of the queue or store it for after when
|
44
|
-
# it should be run with the
|
49
|
+
# it should be run with the :when option.
|
45
50
|
#
|
46
51
|
# Options:
|
47
52
|
# when: The time/timestamp after which to run the message.
|
48
53
|
#
|
49
54
|
def sync_push(item, options={})
|
50
|
-
item, options =
|
55
|
+
item, options = MicroQ::Util.stringify(item, options)
|
56
|
+
klass = item['class'] = item['class'].to_s
|
51
57
|
|
52
|
-
MicroQ.middleware.client.call(
|
58
|
+
MicroQ.middleware.client.call(klass, item, options) do
|
53
59
|
if (time = options['when'])
|
54
60
|
@later.push(
|
55
61
|
'when' => time.to_f,
|
@@ -63,11 +69,15 @@ module MicroQ
|
|
63
69
|
|
64
70
|
##
|
65
71
|
# Remove and return all available messages.
|
72
|
+
# Optionally give a limit and return only limit number of messages
|
66
73
|
#
|
67
|
-
def dequeue
|
74
|
+
def dequeue(limit = 30)
|
75
|
+
return [] if limit == 0
|
76
|
+
|
77
|
+
idx = 0
|
68
78
|
[].tap do |items|
|
69
79
|
entries.each do |entry|
|
70
|
-
items << entry
|
80
|
+
items << entry unless (idx += 1) > limit
|
71
81
|
end if entries.any?
|
72
82
|
|
73
83
|
items.each {|i| entries.delete(i) }
|
@@ -76,7 +86,7 @@ module MicroQ
|
|
76
86
|
|
77
87
|
if available.any?
|
78
88
|
available.each do |entry|
|
79
|
-
items << entry['worker']
|
89
|
+
items << entry['worker'] unless (idx += 1) > limit
|
80
90
|
end
|
81
91
|
|
82
92
|
available.each {|a| later.delete(a) }
|
@@ -84,15 +94,36 @@ module MicroQ
|
|
84
94
|
end
|
85
95
|
end
|
86
96
|
|
97
|
+
##
|
98
|
+
# Stop the queue and store items for later
|
99
|
+
#
|
100
|
+
def stop
|
101
|
+
File.open(queue_file, 'w+') do |f|
|
102
|
+
f.write(YAML.dump(entries))
|
103
|
+
end if queue_file?
|
104
|
+
|
105
|
+
terminate
|
106
|
+
end
|
107
|
+
|
87
108
|
private
|
88
109
|
|
89
110
|
##
|
90
|
-
#
|
111
|
+
# Parse the entries back into the queue from the filesystem
|
91
112
|
#
|
92
|
-
def
|
93
|
-
|
94
|
-
|
95
|
-
|
113
|
+
def load_queues
|
114
|
+
if queue_file? && File.exists?(queue_file)
|
115
|
+
@entries = YAML.load(File.new(queue_file).read)
|
116
|
+
|
117
|
+
File.unlink(queue_file)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def queue_file
|
122
|
+
@queue_file ||= MicroQ.config.queue_file
|
123
|
+
end
|
124
|
+
|
125
|
+
def queue_file?
|
126
|
+
queue_file && File.exists?(File.dirname(queue_file))
|
96
127
|
end
|
97
128
|
end
|
98
129
|
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module MicroQ
|
2
|
+
module Queue
|
3
|
+
class Redis
|
4
|
+
include Celluloid
|
5
|
+
|
6
|
+
QUEUES = {
|
7
|
+
:entries => 'micro_q:queue:entries',
|
8
|
+
:later => 'micro_q:queue:later'
|
9
|
+
}.freeze
|
10
|
+
|
11
|
+
##
|
12
|
+
# All of the items in the entries queue
|
13
|
+
#
|
14
|
+
def entries
|
15
|
+
MicroQ.redis do |r|
|
16
|
+
r.lrange(QUEUES[:entries], 0, -1)
|
17
|
+
end.collect(&MicroQ::Util.json_parse)
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# All of the items in the later queue
|
22
|
+
#
|
23
|
+
def later
|
24
|
+
MicroQ.redis do |r|
|
25
|
+
r.zrangebyscore(QUEUES[:later], '-inf', '+inf')
|
26
|
+
end.collect(&MicroQ::Util.json_parse)
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# Asynchronously push a message item to the queue.
|
31
|
+
# Either push it to the immediate portion of the queue (a Redis list)
|
32
|
+
# or store it in the later queue (a Redis sorted-set). The message will
|
33
|
+
# be available for dequeue after the :when time passes.
|
34
|
+
#
|
35
|
+
# Options:
|
36
|
+
# when: The time/timestamp after which to run the message.
|
37
|
+
#
|
38
|
+
def push(item, options = {})
|
39
|
+
async.sync_push(item, options)
|
40
|
+
end
|
41
|
+
|
42
|
+
def sync_push(item, options = {})
|
43
|
+
item, options = MicroQ::Util.stringify(item, options)
|
44
|
+
klass = item['class'] = item['class'].to_s
|
45
|
+
|
46
|
+
MicroQ.middleware.client.call(klass, item, options) do
|
47
|
+
json = JSON.dump(item)
|
48
|
+
|
49
|
+
MicroQ.redis do |r|
|
50
|
+
if (time = options['when'])
|
51
|
+
r.zadd(QUEUES[:later], time.to_f, json)
|
52
|
+
else
|
53
|
+
r.lpush(QUEUES[:entries], json)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
##
|
60
|
+
# Remove and return up to :limit number of items from the
|
61
|
+
# entries queue. Move any available items from the later
|
62
|
+
# queue into the entries for future dequeue.
|
63
|
+
#
|
64
|
+
def dequeue(limit = 30)
|
65
|
+
idx, items = 0, []
|
66
|
+
|
67
|
+
fetch(limit).tap do |(e, later)|
|
68
|
+
e.each {|item| items << item unless (idx += 1) > limit }
|
69
|
+
|
70
|
+
MicroQ.redis do |r|
|
71
|
+
((e - items) + later).each {|l| r.rpush(QUEUES[:entries], l)}
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
items.collect(&MicroQ::Util.json_parse)
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def fetch(limit)
|
81
|
+
return [] unless limit > 0
|
82
|
+
|
83
|
+
time = Time.now.to_f
|
84
|
+
|
85
|
+
MicroQ.redis do |r|
|
86
|
+
[r.multi {
|
87
|
+
limit.times.collect { r.rpop(QUEUES[:entries]) }
|
88
|
+
}.compact,
|
89
|
+
r.multi {
|
90
|
+
r.zrangebyscore(QUEUES[:later], '-inf', time)
|
91
|
+
r.zremrangebyscore(QUEUES[:later], '-inf', time)
|
92
|
+
}.first]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/micro_q/queue.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'micro_q/queue/default'
|
2
|
+
require 'micro_q/queue/redis'
|
2
3
|
|
3
4
|
##
|
4
5
|
# The Queueing interface
|
@@ -14,8 +15,13 @@ require 'micro_q/queue/default'
|
|
14
15
|
# message is the hash the represents an pushed item
|
15
16
|
# options are for items that dont require storing in the message itself
|
16
17
|
# but are important in queueing.
|
18
|
+
#
|
17
19
|
# :method: dequeue
|
18
|
-
# - Remove and return items from the data store
|
20
|
+
# - Remove and return items from the data store (limited to n items)
|
21
|
+
# - :args: (limit = 30) optionally return no more than.
|
22
|
+
#
|
23
|
+
# :method: stop
|
24
|
+
# - Perform any finalizing tasks (e.g. persist the queue)
|
19
25
|
#
|
20
26
|
# You are otherwise able to implement this class in any suitable manner.
|
21
27
|
# If adhering to the other conventions around data structures, keys, etc,
|
data/lib/micro_q/util.rb
CHANGED
@@ -15,6 +15,16 @@ module MicroQ
|
|
15
15
|
rescue
|
16
16
|
end
|
17
17
|
|
18
|
+
def self.json_parse
|
19
|
+
@@json_parse ||= proc {|entry| JSON.parse(entry) }
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.stringify(*args)
|
23
|
+
args.collect do |a|
|
24
|
+
stringify_keys(a)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
18
28
|
##
|
19
29
|
# Copy a hash and convert all keys to strings.
|
20
30
|
# Stringifies to infinite hash depth
|
data/lib/micro_q/version.rb
CHANGED
data/lib/micro_q.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
require 'celluloid'
|
2
|
+
require 'connection_pool'
|
3
|
+
require 'redis'
|
4
|
+
|
2
5
|
require 'micro_q/util'
|
3
6
|
require 'micro_q/config'
|
4
7
|
require 'micro_q/manager'
|
@@ -45,3 +48,5 @@ require 'micro_q/methods'
|
|
45
48
|
require 'micro_q/proxies'
|
46
49
|
require 'micro_q/worker'
|
47
50
|
require 'micro_q/queue'
|
51
|
+
|
52
|
+
require 'micro_q/redis'
|
data/micro_q.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
lib = File.expand_path(
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
4
|
+
require "micro_q/version"
|
5
5
|
|
6
6
|
Gem::Specification.new do |gem|
|
7
7
|
gem.name = "micro_q"
|
@@ -18,6 +18,8 @@ Gem::Specification.new do |gem|
|
|
18
18
|
gem.require_paths = %w(lib)
|
19
19
|
|
20
20
|
gem.add_dependency "celluloid"
|
21
|
+
gem.add_dependency "redis"
|
22
|
+
gem.add_dependency "connection_pool"
|
21
23
|
gem.add_development_dependency "rake"
|
22
24
|
gem.add_development_dependency "rspec"
|
23
25
|
gem.add_development_dependency "timecop"
|
@@ -25,4 +27,5 @@ Gem::Specification.new do |gem|
|
|
25
27
|
gem.add_development_dependency "activerecord", "> 3.2.0"
|
26
28
|
gem.add_development_dependency "actionmailer", "> 3.2.0"
|
27
29
|
gem.add_development_dependency "sqlite3-ruby"
|
30
|
+
gem.add_development_dependency "mock_redis"
|
28
31
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
shared_examples_for 'Queue#sync_push' do
|
2
|
+
it 'should add to the entries' do
|
3
|
+
subject.sync_push(item)
|
4
|
+
|
5
|
+
subject.entries.should include(item)
|
6
|
+
end
|
7
|
+
|
8
|
+
it 'should stringify the class' do
|
9
|
+
subject.sync_push(:class => MyWorker)
|
10
|
+
|
11
|
+
subject.entries.should include('class' => 'MyWorker')
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'should duplicate the item' do
|
15
|
+
subject.sync_push(item)
|
16
|
+
|
17
|
+
before = item.dup
|
18
|
+
subject.entries.should include(before)
|
19
|
+
|
20
|
+
item[:key] = 'new-value'
|
21
|
+
subject.entries.should_not include(item)
|
22
|
+
subject.entries.should include(before)
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'client middleware' do
|
26
|
+
it 'should process the middleware chain' do
|
27
|
+
MicroQ.middleware.client.should_receive(:call) do |w, payload|
|
28
|
+
w.should == 'MyWorker'
|
29
|
+
|
30
|
+
payload['class'].should == 'MyWorker'
|
31
|
+
payload['args'].should == [4]
|
32
|
+
end
|
33
|
+
|
34
|
+
subject.sync_push(item)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/spec/lib/config_spec.rb
CHANGED
@@ -24,8 +24,8 @@ describe MicroQ::Config do
|
|
24
24
|
describe 'defaults' do
|
25
25
|
subject { MicroQ.config }
|
26
26
|
|
27
|
-
it 'should have
|
28
|
-
subject.workers.should ==
|
27
|
+
it 'should have 5 workers' do
|
28
|
+
subject.workers.should == 5
|
29
29
|
end
|
30
30
|
|
31
31
|
it 'should have a 5 second interval' do
|
@@ -44,6 +44,14 @@ describe MicroQ::Config do
|
|
44
44
|
subject.logfile.should == nil
|
45
45
|
end
|
46
46
|
|
47
|
+
it 'should have a redis pool config' do
|
48
|
+
subject.redis_pool.should == { :size => 15, :timeout => 1 }
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should have a redis config' do
|
52
|
+
subject.redis.should == { :host => 'localhost', :port => 6379 }
|
53
|
+
end
|
54
|
+
|
47
55
|
it 'should have the default queue' do
|
48
56
|
subject.manager.should == MicroQ::Manager::Default
|
49
57
|
end
|
@@ -53,10 +53,16 @@ describe MicroQ::Manager::Default do
|
|
53
53
|
@queue = mock(MicroQ::Queue::Default, :dequeue => [@item, @other_item])
|
54
54
|
MicroQ::Queue::Default.stub(:new).and_return(@queue)
|
55
55
|
|
56
|
-
@pool = mock(Celluloid::PoolManager)
|
56
|
+
@pool = mock(Celluloid::PoolManager, :idle_size => 1234, :perform! => nil)
|
57
57
|
MicroQ::Worker::Standard.stub(:pool).and_return(@pool)
|
58
58
|
end
|
59
59
|
|
60
|
+
it 'should dequeue the number of free workers' do
|
61
|
+
@queue.should_receive(:dequeue).with(1234)
|
62
|
+
|
63
|
+
subject.start
|
64
|
+
end
|
65
|
+
|
60
66
|
it 'should perform the items' do
|
61
67
|
@pool.should_receive(:perform!).with(@item) do
|
62
68
|
@pool.should_receive(:perform!).with(@other_item)
|
@@ -1,6 +1,8 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe MicroQ::Queue::Default do
|
4
|
+
let(:item) { { 'class' => 'MyWorker', 'args' => [4] } }
|
5
|
+
|
4
6
|
describe '#entries' do
|
5
7
|
it 'should be empty' do
|
6
8
|
subject.entries.should == []
|
@@ -14,37 +16,7 @@ describe MicroQ::Queue::Default do
|
|
14
16
|
end
|
15
17
|
|
16
18
|
describe '#sync_push' do
|
17
|
-
|
18
|
-
|
19
|
-
it 'should add to the entries' do
|
20
|
-
subject.sync_push(item)
|
21
|
-
|
22
|
-
subject.entries.should include(item)
|
23
|
-
end
|
24
|
-
|
25
|
-
it 'should duplicate the item' do
|
26
|
-
subject.sync_push(item)
|
27
|
-
|
28
|
-
before = item.dup
|
29
|
-
subject.entries.should include(before)
|
30
|
-
|
31
|
-
item[:key] = 'new-value'
|
32
|
-
subject.entries.should_not include(item)
|
33
|
-
subject.entries.should include(before)
|
34
|
-
end
|
35
|
-
|
36
|
-
describe 'client middleware' do
|
37
|
-
it 'should process the middleware chain' do
|
38
|
-
MicroQ.middleware.client.should_receive(:call) do |w, payload|
|
39
|
-
w.should == 'MyWorker'
|
40
|
-
|
41
|
-
payload['class'].should == 'MyWorker'
|
42
|
-
payload['args'].should == [4]
|
43
|
-
end
|
44
|
-
|
45
|
-
subject.sync_push(item)
|
46
|
-
end
|
47
|
-
end
|
19
|
+
it_behaves_like 'Queue#sync_push'
|
48
20
|
|
49
21
|
describe 'when given the "when" key' do
|
50
22
|
let(:worker) { [item, { 'when' => (Time.now + 100).to_i }] }
|
@@ -98,8 +70,6 @@ describe MicroQ::Queue::Default do
|
|
98
70
|
end
|
99
71
|
|
100
72
|
describe '#push' do
|
101
|
-
let(:item) { { 'class' => 'MyWorker', 'args' => [4] } }
|
102
|
-
|
103
73
|
before do
|
104
74
|
@async = mock(Celluloid::ActorProxy)
|
105
75
|
subject.stub(:async).and_return(@async)
|
@@ -193,6 +163,100 @@ describe MicroQ::Queue::Default do
|
|
193
163
|
subject.entries.should == []
|
194
164
|
subject.later.should == [later_item[1].merge('worker' => later_item[0])]
|
195
165
|
end
|
166
|
+
|
167
|
+
describe 'when limited to a certain number' do
|
168
|
+
it 'should return all the available items' do
|
169
|
+
subject.dequeue(2).sort {|x, y| x['args'][0] <=> y['args'][0] }.should == items.first(2)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
describe '#stop' do
|
176
|
+
let(:file_name) { '/some/file/queue.yml' }
|
177
|
+
|
178
|
+
describe 'when there are items in the queue' do
|
179
|
+
let(:other_item) { { 'class' => 'MyWorker', 'args' => ['hello'] } }
|
180
|
+
|
181
|
+
before do
|
182
|
+
MicroQ.configure {|c| c.queue_file = file_name }
|
183
|
+
|
184
|
+
@file = mock(File, :write => nil)
|
185
|
+
File.stub(:open).with(file_name, 'w+').and_yield(@file)
|
186
|
+
File.stub(:exists?).and_return(false)
|
187
|
+
File.stub(:exists?).with(File.dirname(file_name)).and_return(true)
|
188
|
+
|
189
|
+
subject.push(item)
|
190
|
+
subject.push(other_item)
|
191
|
+
end
|
192
|
+
|
193
|
+
it 'should create the target file' do
|
194
|
+
File.should_receive(:open).with(file_name, 'w+')
|
195
|
+
|
196
|
+
subject.stop
|
197
|
+
end
|
198
|
+
|
199
|
+
it 'should write the entries' do
|
200
|
+
@file.should_receive(:write).with(YAML.dump([item, other_item]))
|
201
|
+
|
202
|
+
subject.stop
|
203
|
+
end
|
204
|
+
|
205
|
+
describe 'when the file directory does not exist' do
|
206
|
+
before do
|
207
|
+
File.stub(:exists?).with(File.dirname(file_name)).and_return(false)
|
208
|
+
end
|
209
|
+
|
210
|
+
it 'should not write the file' do
|
211
|
+
File.should_not_receive(:open)
|
212
|
+
|
213
|
+
subject.stop
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
describe '.new' do
|
220
|
+
let!(:file_name) { '/some/other/file/queue.yml' }
|
221
|
+
let(:queue) { -> { subject } }
|
222
|
+
|
223
|
+
before do
|
224
|
+
MicroQ.configure {|c| c.queue_file = file_name }
|
225
|
+
end
|
226
|
+
|
227
|
+
describe 'when there are persisted queue items' do
|
228
|
+
before do
|
229
|
+
File.stub(:exists?).with(File.dirname(file_name)).and_return(true)
|
230
|
+
File.stub(:exists?).with(file_name).and_return(true)
|
231
|
+
File.stub_chain(:new, :read).and_return(YAML.dump([item]))
|
232
|
+
|
233
|
+
File.stub(:unlink)
|
234
|
+
end
|
235
|
+
|
236
|
+
it 'should check the file existence' do
|
237
|
+
File.should_receive(:exists?).with(File.dirname(file_name)).and_return(true)
|
238
|
+
File.should_receive(:exists?).with(file_name).and_return(false)
|
239
|
+
|
240
|
+
queue.call
|
241
|
+
end
|
242
|
+
|
243
|
+
it 'should open the file' do
|
244
|
+
File.should_receive(:new).with(file_name)
|
245
|
+
|
246
|
+
queue.call
|
247
|
+
end
|
248
|
+
|
249
|
+
it 'should remove the file' do
|
250
|
+
File.should_receive(:unlink).with(file_name)
|
251
|
+
|
252
|
+
queue.call
|
253
|
+
end
|
254
|
+
|
255
|
+
it 'should read the file to place the items in the queue' do
|
256
|
+
queue.call
|
257
|
+
|
258
|
+
subject.entries.should == [item]
|
259
|
+
end
|
196
260
|
end
|
197
261
|
end
|
198
262
|
end
|
@@ -0,0 +1,162 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe MicroQ::Queue::Redis do
|
4
|
+
let(:item) { { 'class' => 'MyWorker', 'args' => [4] } }
|
5
|
+
|
6
|
+
describe '#sync_push' do
|
7
|
+
it_behaves_like 'Queue#sync_push'
|
8
|
+
|
9
|
+
describe 'when given the "when" key' do
|
10
|
+
let(:worker) { [item, { 'when' => (Time.now + 100).to_i }] }
|
11
|
+
|
12
|
+
it 'should add to the later' do
|
13
|
+
subject.sync_push(*worker)
|
14
|
+
|
15
|
+
subject.later.should include(item)
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should not be in the entries' do
|
19
|
+
subject.sync_push(*worker)
|
20
|
+
|
21
|
+
subject.entries.should == []
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'should process the middleware chain' do
|
25
|
+
MicroQ.middleware.client.should_receive(:call) do |w, payload, options|
|
26
|
+
w.should == 'MyWorker'
|
27
|
+
|
28
|
+
payload['class'].should == 'MyWorker'
|
29
|
+
payload['args'].should == [4]
|
30
|
+
options['when'].should == (Time.now + 100).to_i
|
31
|
+
end
|
32
|
+
|
33
|
+
subject.sync_push(*worker)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe 'when given the symbol :when key' do
|
38
|
+
let(:worker) { [item, { :when => (Time.now + 100).to_i }] }
|
39
|
+
|
40
|
+
it 'should add to the later' do
|
41
|
+
subject.sync_push(*worker)
|
42
|
+
|
43
|
+
subject.later.should include(item)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should not be in the entries' do
|
47
|
+
subject.sync_push(*worker)
|
48
|
+
|
49
|
+
subject.entries.should == []
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe '#push' do
|
55
|
+
before do
|
56
|
+
@async = mock(Celluloid::ActorProxy)
|
57
|
+
subject.stub(:async).and_return(@async)
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'should asynchronously push the item' do
|
61
|
+
@async.should_receive(:sync_push).with(*item)
|
62
|
+
|
63
|
+
subject.push(*item)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe '#dequeue' do
|
68
|
+
let(:item) { { 'class' => 'MyWorker', 'args' => [] } }
|
69
|
+
|
70
|
+
class MyWorker
|
71
|
+
def perform(*)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe 'when there are entries' do
|
76
|
+
before do
|
77
|
+
subject.sync_push(item)
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'should return the item' do
|
81
|
+
subject.dequeue.should == [item]
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'should remove the item from the list' do
|
85
|
+
subject.dequeue
|
86
|
+
|
87
|
+
subject.entries.should_not include(item)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe 'when there are items to be processed later' do
|
92
|
+
before do
|
93
|
+
subject.sync_push(item, 'when' => (Time.now + 5).to_i)
|
94
|
+
end
|
95
|
+
|
96
|
+
it 'should not return the item' do
|
97
|
+
subject.dequeue.should == []
|
98
|
+
end
|
99
|
+
|
100
|
+
it 'should not remove the item' do
|
101
|
+
subject.dequeue
|
102
|
+
|
103
|
+
subject.later.should == [item]
|
104
|
+
end
|
105
|
+
|
106
|
+
describe 'when the item is in the past' do
|
107
|
+
let(:queue_name) { MicroQ::Queue::Redis::QUEUES[:later] }
|
108
|
+
|
109
|
+
before do
|
110
|
+
MicroQ.redis {|r| r.zadd(queue_name, (Time.now - 2).to_i, item.to_json) } # update its score
|
111
|
+
subject.dequeue # move scheduled items to entries
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'should return the item' do
|
115
|
+
subject.dequeue.should == [item]
|
116
|
+
end
|
117
|
+
|
118
|
+
it 'should remove the item from the list' do
|
119
|
+
subject.dequeue
|
120
|
+
|
121
|
+
subject.later.should == []
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
describe 'when there are many items' do
|
127
|
+
let(:later_item) { [item.dup.tap {|it| it['args'] = %w(special) }, 'when' => (Time.now + 5).to_i] }
|
128
|
+
let(:items) do
|
129
|
+
5.times.collect {|i|
|
130
|
+
item.dup.tap {|it| it['args'] = [i]}
|
131
|
+
}
|
132
|
+
end
|
133
|
+
|
134
|
+
before do
|
135
|
+
items.first(4).each {|item| subject.sync_push(item) }
|
136
|
+
subject.sync_push(items.last, 'when' => (Time.now - 2).to_i)
|
137
|
+
|
138
|
+
subject.sync_push(*later_item)
|
139
|
+
end
|
140
|
+
|
141
|
+
it 'should return all the available items and move the schedule item' do
|
142
|
+
subject.dequeue.sort {|x, y| x['args'][0] <=> y['args'][0] }.should == items.first(4)
|
143
|
+
subject.entries.should include(items.last)
|
144
|
+
|
145
|
+
subject.dequeue.should == items.last(1)
|
146
|
+
end
|
147
|
+
|
148
|
+
it 'should remove the items' do
|
149
|
+
2.times { subject.dequeue }
|
150
|
+
|
151
|
+
subject.entries.should == []
|
152
|
+
subject.later.should == [later_item[0]]
|
153
|
+
end
|
154
|
+
|
155
|
+
describe 'when limited to a certain number' do
|
156
|
+
it 'should return all the available items' do
|
157
|
+
subject.dequeue(2).sort {|x, y| x['args'][0] <=> y['args'][0] }.should == items.first(2)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,12 +1,18 @@
|
|
1
1
|
require 'micro_q'
|
2
2
|
require 'time'
|
3
3
|
require 'timecop'
|
4
|
-
require '
|
4
|
+
require 'mock_redis'
|
5
5
|
|
6
|
-
|
6
|
+
[:methods, :queues].
|
7
|
+
each {|path| require "helpers/#{path}_examples" }
|
7
8
|
|
8
9
|
Celluloid.logger = nil
|
9
10
|
|
11
|
+
# Don't require an actual Redis instance for testing
|
12
|
+
silence_warnings do
|
13
|
+
Redis = MockRedis
|
14
|
+
end
|
15
|
+
|
10
16
|
RSpec.configure do |config|
|
11
17
|
config.treat_symbols_as_metadata_keys_with_true_values = true
|
12
18
|
config.run_all_when_everything_filtered = true
|
@@ -15,6 +21,7 @@ RSpec.configure do |config|
|
|
15
21
|
|
16
22
|
config.before :each do
|
17
23
|
MicroQ.send :clear
|
24
|
+
MicroQ.redis {|r| r.flushdb }
|
18
25
|
end
|
19
26
|
|
20
27
|
config.before :each, :active_record => true do
|
@@ -31,7 +38,7 @@ RSpec.configure do |config|
|
|
31
38
|
);
|
32
39
|
SQL
|
33
40
|
|
34
|
-
# ** Transactional fixtures. **
|
41
|
+
# ** Transactional fixtures. BEGIN **
|
35
42
|
@_db.transaction
|
36
43
|
|
37
44
|
ActiveRecord::Base.establish_connection(
|
@@ -41,6 +48,7 @@ RSpec.configure do |config|
|
|
41
48
|
end
|
42
49
|
|
43
50
|
config.after :each, :active_record => true do
|
51
|
+
# ** Transactional fixtures. END **
|
44
52
|
@_db.rollback
|
45
53
|
end
|
46
54
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: micro_q
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-02-
|
12
|
+
date: 2013-02-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: celluloid
|
@@ -27,6 +27,38 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: redis
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: connection_pool
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
30
62
|
- !ruby/object:Gem::Dependency
|
31
63
|
name: rake
|
32
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -139,6 +171,22 @@ dependencies:
|
|
139
171
|
- - ! '>='
|
140
172
|
- !ruby/object:Gem::Version
|
141
173
|
version: '0'
|
174
|
+
- !ruby/object:Gem::Dependency
|
175
|
+
name: mock_redis
|
176
|
+
requirement: !ruby/object:Gem::Requirement
|
177
|
+
none: false
|
178
|
+
requirements:
|
179
|
+
- - ! '>='
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0'
|
182
|
+
type: :development
|
183
|
+
prerelease: false
|
184
|
+
version_requirements: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
142
190
|
description: ''
|
143
191
|
email: brian.nort@gmail.com
|
144
192
|
executables: []
|
@@ -172,6 +220,8 @@ files:
|
|
172
220
|
- lib/micro_q/proxies/instance.rb
|
173
221
|
- lib/micro_q/queue.rb
|
174
222
|
- lib/micro_q/queue/default.rb
|
223
|
+
- lib/micro_q/queue/redis.rb
|
224
|
+
- lib/micro_q/redis.rb
|
175
225
|
- lib/micro_q/util.rb
|
176
226
|
- lib/micro_q/version.rb
|
177
227
|
- lib/micro_q/worker.rb
|
@@ -180,6 +230,7 @@ files:
|
|
180
230
|
- lib/micro_q/wrappers/action_mailer.rb
|
181
231
|
- micro_q.gemspec
|
182
232
|
- spec/helpers/methods_examples.rb
|
233
|
+
- spec/helpers/queues_examples.rb
|
183
234
|
- spec/lib/config_spec.rb
|
184
235
|
- spec/lib/manager/default_spec.rb
|
185
236
|
- spec/lib/methods/action_mailer_spec.rb
|
@@ -195,10 +246,11 @@ files:
|
|
195
246
|
- spec/lib/proxies/class_spec.rb
|
196
247
|
- spec/lib/proxies/instance_spec.rb
|
197
248
|
- spec/lib/queue/default_spec.rb
|
249
|
+
- spec/lib/queue/redis_spec.rb
|
198
250
|
- spec/lib/util_spec.rb
|
199
251
|
- spec/lib/worker/standard_spec.rb
|
252
|
+
- spec/lib/wrappers/action_mailer_spec.rb
|
200
253
|
- spec/spec_helper.rb
|
201
|
-
- spec/wrappers/action_mailer_spec.rb
|
202
254
|
homepage: http://github.com/bnorton/micro-q
|
203
255
|
licenses: []
|
204
256
|
post_install_message:
|
@@ -211,20 +263,27 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
211
263
|
- - ! '>='
|
212
264
|
- !ruby/object:Gem::Version
|
213
265
|
version: '0'
|
266
|
+
segments:
|
267
|
+
- 0
|
268
|
+
hash: -938614876978684754
|
214
269
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
215
270
|
none: false
|
216
271
|
requirements:
|
217
272
|
- - ! '>='
|
218
273
|
- !ruby/object:Gem::Version
|
219
274
|
version: '0'
|
275
|
+
segments:
|
276
|
+
- 0
|
277
|
+
hash: -938614876978684754
|
220
278
|
requirements: []
|
221
279
|
rubyforge_project:
|
222
|
-
rubygems_version: 1.8.
|
280
|
+
rubygems_version: 1.8.24
|
223
281
|
signing_key:
|
224
282
|
specification_version: 3
|
225
283
|
summary: ''
|
226
284
|
test_files:
|
227
285
|
- spec/helpers/methods_examples.rb
|
286
|
+
- spec/helpers/queues_examples.rb
|
228
287
|
- spec/lib/config_spec.rb
|
229
288
|
- spec/lib/manager/default_spec.rb
|
230
289
|
- spec/lib/methods/action_mailer_spec.rb
|
@@ -240,8 +299,8 @@ test_files:
|
|
240
299
|
- spec/lib/proxies/class_spec.rb
|
241
300
|
- spec/lib/proxies/instance_spec.rb
|
242
301
|
- spec/lib/queue/default_spec.rb
|
302
|
+
- spec/lib/queue/redis_spec.rb
|
243
303
|
- spec/lib/util_spec.rb
|
244
304
|
- spec/lib/worker/standard_spec.rb
|
305
|
+
- spec/lib/wrappers/action_mailer_spec.rb
|
245
306
|
- spec/spec_helper.rb
|
246
|
-
- spec/wrappers/action_mailer_spec.rb
|
247
|
-
has_rdoc:
|
File without changes
|