smq 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +9 -0
- data/VERSION +1 -1
- data/lib/smq.rb +17 -3
- data/lib/smq/queue.rb +19 -7
- data/lib/smq/worker.rb +4 -3
- data/test/test_queue.rb +13 -4
- data/test/test_worker.rb +8 -0
- metadata +21 -15
- data/.gitignore +0 -1
data/README.markdown
CHANGED
@@ -81,6 +81,15 @@ This process, although heavy on `SELECT`s, results in a minimum of table locking
|
|
81
81
|
|
82
82
|
These limitations are deemed acceptable due to the simple nature of this queueing system.
|
83
83
|
|
84
|
+
### Workaround to Locking Limitations
|
85
|
+
|
86
|
+
There is a workaround to the implicit limit of 5 workers per queue, which is to use the "batching" facility. This works by splitting up the messages by the [modulo](http://en.wikipedia.org/wiki/Modulo_operation) of the ID; in effect this means you can then run up to 5 workers per batch:
|
87
|
+
|
88
|
+
SMQ::Worker.new("queue_name", total_batches, this_batch).work do |msg|
|
89
|
+
puts msg.data.inspect
|
90
|
+
msg.ack!
|
91
|
+
end
|
92
|
+
|
84
93
|
## Licensing and Attribution
|
85
94
|
|
86
95
|
SMQ is released under the MIT license as detailed in the LICENSE file that should be distributed with this library; the source code is [freely available](http://github.com/timblair/smq).
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.3.0
|
data/lib/smq.rb
CHANGED
@@ -43,9 +43,23 @@ module SMQ
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def self.flush_all_queues!
|
46
|
-
#
|
47
|
-
#
|
48
|
-
|
46
|
+
# we're totally resetting everything here, including any auto-increment
|
47
|
+
# keys: this should only ever be called for testing.
|
48
|
+
# adapted from: http://www.manu-j.com/blog/post/221/
|
49
|
+
case ActiveRecord::Base.connection.instance_variable_get(:@config)[:adapter]
|
50
|
+
when "mysql"
|
51
|
+
ActiveRecord::Base.connection.tables.each do |table|
|
52
|
+
ActiveRecord::Base.connection.execute("TRUNCATE #{SMQ::Message.table_name}")
|
53
|
+
end
|
54
|
+
when "sqlite", "sqlite3"
|
55
|
+
ActiveRecord::Base.connection.tables.each do |table|
|
56
|
+
ActiveRecord::Base.connection.execute("DELETE FROM #{SMQ::Message.table_name}")
|
57
|
+
ActiveRecord::Base.connection.execute("DELETE FROM sqlite_sequence where name='#{SMQ::Message.table_name}'")
|
58
|
+
end
|
59
|
+
ActiveRecord::Base.connection.execute("VACUUM")
|
60
|
+
else
|
61
|
+
raise "Unsupported connection type: #{ActiveRecord::Base.connection.instance_variable_get(:@config)[:adapter]}."
|
62
|
+
end
|
49
63
|
end
|
50
64
|
|
51
65
|
end
|
data/lib/smq/queue.rb
CHANGED
@@ -25,15 +25,23 @@ module SMQ
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def reserve(worker)
|
28
|
-
find_available.each do |msg|
|
28
|
+
find_available(worker.batches, worker.batch).each do |msg|
|
29
29
|
m = msg.lock!(worker)
|
30
30
|
return m unless m == nil
|
31
31
|
end
|
32
32
|
nil
|
33
33
|
end
|
34
34
|
|
35
|
-
def find_available
|
36
|
-
SMQ::Message.find(
|
35
|
+
def find_available(batches=1, batch=1)
|
36
|
+
SMQ::Message.find(
|
37
|
+
:all,
|
38
|
+
:select => 'id, updated_at',
|
39
|
+
:conditions => [
|
40
|
+
"queue = ? AND (id % ?) = ? AND completed_at IS NULL AND locked_by IS NULL",
|
41
|
+
@name, batches, batch-1
|
42
|
+
],
|
43
|
+
:order => "id ASC", :limit => @batch_size
|
44
|
+
).sort_by { rand() }
|
37
45
|
end
|
38
46
|
|
39
47
|
def length
|
@@ -57,12 +65,16 @@ module SMQ
|
|
57
65
|
private
|
58
66
|
|
59
67
|
def delete_queue_items(where = nil, limit = nil)
|
60
|
-
|
61
|
-
msg = SMQ::Message.find(:first, :select => 'id', :conditions => ["queue = ?", @name], :offset => limit) if !limit.nil?
|
62
|
-
if (msg.nil?)
|
68
|
+
if (limit.nil?)
|
63
69
|
SMQ::Message.delete_all(["queue = ? #{where ? 'AND ' + where : ''}", @name])
|
64
70
|
else
|
65
|
-
|
71
|
+
# delete_all doesn't support a :limit clause, so we have to fake it
|
72
|
+
msg = SMQ::Message.find(
|
73
|
+
:first, :select => 'id',
|
74
|
+
:conditions => ["queue = ? #{where ? 'AND ' + where : ''}", @name],
|
75
|
+
:order => "id ASC", :offset => limit
|
76
|
+
)
|
77
|
+
SMQ::Message.delete_all(["queue = ? AND id < ? #{where ? 'AND ' + where : ''}", @name, msg.id]) unless msg.nil?
|
66
78
|
end
|
67
79
|
end
|
68
80
|
|
data/lib/smq/worker.rb
CHANGED
@@ -4,15 +4,16 @@ module SMQ
|
|
4
4
|
|
5
5
|
class Worker
|
6
6
|
|
7
|
-
attr_accessor :name
|
8
|
-
attr_accessor :queue
|
7
|
+
attr_accessor :name, :queue, :batches, :batch
|
9
8
|
|
10
9
|
@working = false
|
11
10
|
@stopping = false
|
12
11
|
|
13
|
-
def initialize(queue)
|
12
|
+
def initialize(queue, batches=1, batch=1)
|
14
13
|
self.name = "#{Socket.gethostname}:#{Process.pid}" rescue "pid:#{Process.pid}"
|
15
14
|
self.queue = queue.instance_of?(SMQ::Queue) ? queue : SMQ::Queue.new(queue)
|
15
|
+
self.batches = batches
|
16
|
+
self.batch = batch
|
16
17
|
end
|
17
18
|
|
18
19
|
def to_s
|
data/test/test_queue.rb
CHANGED
@@ -89,15 +89,24 @@ class QueueTest < Test::Unit::TestCase
|
|
89
89
|
assert_equal 5, @queue.length
|
90
90
|
end
|
91
91
|
|
92
|
+
def test_should_return_only_correctly_batched_messages
|
93
|
+
populate_queue(@queue, 5)
|
94
|
+
assert_equal 1, @queue.find_available(3,1).length
|
95
|
+
assert_equal 2, @queue.find_available(3,2).length
|
96
|
+
assert_equal 2, @queue.find_available(3,3).length
|
97
|
+
end
|
98
|
+
|
92
99
|
private
|
93
100
|
|
94
101
|
def populate_and_ack_all_but_fail_one(queue = nil)
|
95
102
|
queue ||= @queue
|
96
103
|
populate_queue(queue, 5)
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
104
|
+
SMQ::Message.find(:all, :conditions => ["queue = ?", queue.name]).each do |msg|
|
105
|
+
msg.ack!
|
106
|
+
end
|
107
|
+
failure = SMQ::Message.find(:first)
|
108
|
+
failure.fail
|
109
|
+
failure.save!
|
101
110
|
end
|
102
111
|
|
103
112
|
end
|
data/test/test_worker.rb
CHANGED
@@ -57,4 +57,12 @@ class WorkerTest < Test::Unit::TestCase
|
|
57
57
|
assert !@worker.is_working?, "Working and should have stopped"
|
58
58
|
end
|
59
59
|
|
60
|
+
def test_working_a_populated_queue_in_batches_should_work_all_batch_jobs
|
61
|
+
@worker.batches = 2
|
62
|
+
populate_queue(@worker.queue.name, 5)
|
63
|
+
block_calls = 0
|
64
|
+
@worker.work(true) { block_calls += 1 }
|
65
|
+
assert_equal 2, block_calls
|
66
|
+
end
|
67
|
+
|
60
68
|
end
|
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: smq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
4
|
+
hash: 19
|
5
|
+
prerelease:
|
5
6
|
segments:
|
6
7
|
- 0
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 0.
|
8
|
+
- 3
|
9
|
+
- 0
|
10
|
+
version: 0.3.0
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- Tim Blair
|
@@ -14,16 +15,18 @@ autorequire:
|
|
14
15
|
bindir: bin
|
15
16
|
cert_chain: []
|
16
17
|
|
17
|
-
date:
|
18
|
+
date: 2011-05-16 00:00:00 +01:00
|
18
19
|
default_executable:
|
19
20
|
dependencies:
|
20
21
|
- !ruby/object:Gem::Dependency
|
21
22
|
name: activerecord
|
22
23
|
prerelease: false
|
23
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
24
26
|
requirements:
|
25
27
|
- - ">="
|
26
28
|
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
27
30
|
segments:
|
28
31
|
- 0
|
29
32
|
version: "0"
|
@@ -33,9 +36,11 @@ dependencies:
|
|
33
36
|
name: yajl-ruby
|
34
37
|
prerelease: false
|
35
38
|
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
36
40
|
requirements:
|
37
41
|
- - ">="
|
38
42
|
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
39
44
|
segments:
|
40
45
|
- 0
|
41
46
|
version: "0"
|
@@ -45,9 +50,11 @@ dependencies:
|
|
45
50
|
name: sqlite3-ruby
|
46
51
|
prerelease: false
|
47
52
|
requirement: &id003 !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
48
54
|
requirements:
|
49
55
|
- - ">="
|
50
56
|
- !ruby/object:Gem::Version
|
57
|
+
hash: 3
|
51
58
|
segments:
|
52
59
|
- 0
|
53
60
|
version: "0"
|
@@ -63,7 +70,6 @@ extra_rdoc_files:
|
|
63
70
|
- LICENSE
|
64
71
|
- README.markdown
|
65
72
|
files:
|
66
|
-
- .gitignore
|
67
73
|
- LICENSE
|
68
74
|
- README.markdown
|
69
75
|
- Rakefile
|
@@ -84,34 +90,34 @@ homepage: http://github.com/timblair/smq
|
|
84
90
|
licenses: []
|
85
91
|
|
86
92
|
post_install_message:
|
87
|
-
rdoc_options:
|
88
|
-
|
93
|
+
rdoc_options: []
|
94
|
+
|
89
95
|
require_paths:
|
90
96
|
- lib
|
91
97
|
required_ruby_version: !ruby/object:Gem::Requirement
|
98
|
+
none: false
|
92
99
|
requirements:
|
93
100
|
- - ">="
|
94
101
|
- !ruby/object:Gem::Version
|
102
|
+
hash: 3
|
95
103
|
segments:
|
96
104
|
- 0
|
97
105
|
version: "0"
|
98
106
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
107
|
+
none: false
|
99
108
|
requirements:
|
100
109
|
- - ">="
|
101
110
|
- !ruby/object:Gem::Version
|
111
|
+
hash: 3
|
102
112
|
segments:
|
103
113
|
- 0
|
104
114
|
version: "0"
|
105
115
|
requirements: []
|
106
116
|
|
107
117
|
rubyforge_project:
|
108
|
-
rubygems_version: 1.
|
118
|
+
rubygems_version: 1.5.0
|
109
119
|
signing_key:
|
110
120
|
specification_version: 3
|
111
121
|
summary: Simple Message Queue
|
112
|
-
test_files:
|
113
|
-
|
114
|
-
- test/test_message.rb
|
115
|
-
- test/test_queue.rb
|
116
|
-
- test/test_worker.rb
|
117
|
-
- examples/worker.rb
|
122
|
+
test_files: []
|
123
|
+
|
data/.gitignore
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
pkg
|