mongo-dequeue 0.5.0 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +14 -0
- data/VERSION +1 -1
- data/lib/mongo-dequeue.rb +126 -47
- data/mongo-dequeue.gemspec +3 -1
- data/pkg/mongo-dequeue-0.4.1.gem +0 -0
- data/pkg/mongo-dequeue-0.5.0.gem +0 -0
- data/spec/mongo_dequeue_spec.rb +33 -2
- metadata +4 -2
data/README.rdoc
CHANGED
@@ -45,6 +45,20 @@ queue items. There are no keys or reserved values in the body of an item. Pass i
|
|
45
45
|
body = ["a", "b", "c"]
|
46
46
|
|
47
47
|
queue.push(body,options)
|
48
|
+
|
49
|
+
=== Batch Pushing
|
50
|
+
When you need to push more then a few items, the delay of network requests can start to add up. In these cases, you can
|
51
|
+
add more then a single item with a single call, like so:
|
52
|
+
|
53
|
+
|
54
|
+
queue.batchpush("foo")
|
55
|
+
queue.batchpush("bar")
|
56
|
+
|
57
|
+
To perform the actual push, call the batchprocess method:
|
58
|
+
|
59
|
+
queue.batchprocess()
|
60
|
+
|
61
|
+
|
48
62
|
|
49
63
|
=== Popping Items
|
50
64
|
Pop items with the pop() method. Specifying a timeout (in seconds) will override the default timeout set in
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.1
|
data/lib/mongo-dequeue.rb
CHANGED
@@ -5,13 +5,13 @@ require 'mongo'
|
|
5
5
|
# heavily inspired by https://github.com/skiz/mongo_queue
|
6
6
|
|
7
7
|
class Mongo::Dequeue
|
8
|
-
attr_reader :collection, :config
|
9
|
-
|
8
|
+
attr_reader :collection, :config, :batch
|
9
|
+
|
10
10
|
DEFAULT_CONFIG = {
|
11
11
|
:timeout => 300,
|
12
12
|
:default_priority => 3
|
13
13
|
}.freeze
|
14
|
-
|
14
|
+
|
15
15
|
# Create a new instance of MongoDequeue with the provided mongodb connection and optional configuration.
|
16
16
|
# See +DEFAULT_CONFIG+ for default configuration and possible configuration options.
|
17
17
|
#
|
@@ -23,20 +23,21 @@ class Mongo::Dequeue
|
|
23
23
|
def initialize(collection, opts={})
|
24
24
|
@collection = collection
|
25
25
|
@config = DEFAULT_CONFIG.merge(opts)
|
26
|
+
@batch = []
|
26
27
|
end
|
27
|
-
|
28
|
+
|
28
29
|
# Remove all items from the queue. Use with caution!
|
29
30
|
def flush!
|
30
31
|
collection.drop
|
31
32
|
end
|
32
|
-
|
33
|
+
|
33
34
|
# Insert a new item into the queue.
|
34
35
|
#
|
35
36
|
# Example:
|
36
37
|
# queue.insert(:name => 'Billy', :email => 'billy@example.com', :message => 'Here is the thing you asked for')
|
37
38
|
def push(body, item_opts = {})
|
38
39
|
dup_key = item_opts[:duplicate_key] || Mongo::Dequeue.generate_duplicate_key(body)
|
39
|
-
|
40
|
+
|
40
41
|
selector = {
|
41
42
|
:duplicate_key => dup_key,
|
42
43
|
:complete => false,
|
@@ -55,37 +56,86 @@ class Mongo::Dequeue
|
|
55
56
|
},
|
56
57
|
'$inc' => {:count => 1 }
|
57
58
|
}
|
58
|
-
|
59
|
+
|
59
60
|
id = collection.update(selector, item, :upsert => true)
|
60
61
|
end
|
61
|
-
|
62
|
+
|
63
|
+
# add a new item into the delayed batch
|
64
|
+
def batchpush(body, item_opts = {})
|
65
|
+
@batch << {
|
66
|
+
:body => body,
|
67
|
+
:duplicate_key => item_opts[:duplicate_key] || Mongo::Dequeue.generate_duplicate_key(body),
|
68
|
+
:priority => item_opts[:priority] || @config[:default_priority]
|
69
|
+
}
|
70
|
+
end
|
71
|
+
|
72
|
+
def batchprocess()
|
73
|
+
js = %Q|
|
74
|
+
function(batch) {
|
75
|
+
var nowutc = new Date();
|
76
|
+
var ret = [];
|
77
|
+
for(i in batch){
|
78
|
+
e = batch[i];
|
79
|
+
//ret.push(e);
|
80
|
+
var query = {
|
81
|
+
'duplicate_key': e.duplicate_key,
|
82
|
+
'complete': false,
|
83
|
+
'locked_at': null
|
84
|
+
};
|
85
|
+
var object = {
|
86
|
+
'$set': {
|
87
|
+
'body': e.body,
|
88
|
+
'inserted_at': nowutc,
|
89
|
+
'complete': false,
|
90
|
+
'locked_till': null,
|
91
|
+
'completed_at': null,
|
92
|
+
'priority': e.priority,
|
93
|
+
'duplicate_key': e.duplicate_key,
|
94
|
+
'completecount': 0
|
95
|
+
},
|
96
|
+
'$inc': {'count': 1}
|
97
|
+
};
|
98
|
+
|
99
|
+
db.#{collection.name}.update(query, object, true);
|
100
|
+
}
|
101
|
+
return ret;
|
102
|
+
}
|
103
|
+
|
|
104
|
+
cmd = BSON::OrderedHash.new
|
105
|
+
cmd['$eval'] = js
|
106
|
+
cmd['args'] = [@batch]
|
107
|
+
result = collection.db.command(cmd)
|
108
|
+
@batch.clear
|
109
|
+
#pp result
|
110
|
+
end
|
111
|
+
|
62
112
|
# Lock and return the next queue message if one is available. Returns nil if none are available. Be sure to
|
63
113
|
# review the README.rdoc regarding proper usage of the locking process identifier (locked_by).
|
64
114
|
# Example:
|
65
115
|
# doc = queue.pop()
|
66
|
-
|
116
|
+
|
67
117
|
# {:body=>"foo", :id=>"4e039c372b70275e345206e4"}
|
68
118
|
|
69
119
|
def pop(opts = {})
|
70
120
|
begin
|
71
121
|
timeout = opts[:timeout] || @config[:timeout]
|
72
122
|
cmd = BSON::OrderedHash.new
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
123
|
+
cmd['findandmodify'] = collection.name
|
124
|
+
cmd['update'] = {'$set' => {:locked_till => Time.now.utc+timeout}}
|
125
|
+
cmd['query'] = {:complete => false, '$or'=>[{:locked_till=> nil},{:locked_till=>{'$lt'=>Time.now.utc}}] }
|
126
|
+
cmd['sort'] = {:priority=>-1,:inserted_at=>1}
|
127
|
+
cmd['limit'] = 1
|
128
|
+
cmd['new'] = true
|
129
|
+
result = collection.db.command(cmd)
|
80
130
|
rescue Mongo::OperationFailure => of
|
81
|
-
|
131
|
+
return nil
|
82
132
|
end
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
133
|
+
return {
|
134
|
+
:body => result['value']['body'],
|
135
|
+
:id => result['value']['_id'].to_s
|
136
|
+
}
|
87
137
|
end
|
88
|
-
|
138
|
+
|
89
139
|
# Remove the document from the queue. This should be called when the work is done and the document is no longer needed.
|
90
140
|
# You must provide the process identifier that the document was locked with to complete it.
|
91
141
|
def complete(id)
|
@@ -97,22 +147,22 @@ class Mongo::Dequeue
|
|
97
147
|
cmd['limit'] = 1
|
98
148
|
collection.db.command(cmd)
|
99
149
|
rescue Mongo::OperationFailure => of
|
100
|
-
|
101
|
-
|
150
|
+
#opfailure happens when item has been already completed
|
151
|
+
return nil
|
102
152
|
end
|
103
153
|
end
|
104
|
-
|
154
|
+
|
105
155
|
# Removes completed job history
|
106
156
|
def cleanup()
|
107
157
|
collection.remove({:complete=>true});
|
108
158
|
end
|
109
|
-
|
159
|
+
|
110
160
|
# Provides some information about what is in the queue. We are using an eval to ensure that a
|
111
161
|
# lock is obtained during the execution of this query so that the results are not skewed.
|
112
162
|
# please be aware that it will lock the database during the execution, so avoid using it too
|
113
163
|
# often, even though it it very tiny and should be relatively fast.
|
114
164
|
def stats
|
115
|
-
|
165
|
+
js = "function queue_stat(){
|
116
166
|
return db.eval(
|
117
167
|
function(){
|
118
168
|
var nowutc = new Date();
|
@@ -126,43 +176,72 @@ class Mongo::Dequeue
|
|
126
176
|
'$reduce': function(obj, prev){prev.count += (obj.completecount - 1);},
|
127
177
|
'initial': {count: 0}
|
128
178
|
});
|
129
|
-
|
130
|
-
|
179
|
+
var p = db.#{collection.name}.group({
|
180
|
+
'key': {'priority':1},
|
181
|
+
'cond': {},
|
182
|
+
'$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}},
|
183
|
+
'initial': {complete: 0, waiting:0}
|
184
|
+
});
|
185
|
+
var tasks = db.#{collection.name}.group({
|
186
|
+
'key': {'body.task':1},
|
187
|
+
'cond': {},
|
188
|
+
'$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}},
|
189
|
+
'initial': {complete: 0, waiting:0}
|
190
|
+
});
|
191
|
+
|
192
|
+
return [a, c, t, l, rc[0] ? rc[0].count : 0, p, tasks];
|
131
193
|
}
|
132
194
|
);
|
133
195
|
}"
|
134
|
-
|
196
|
+
|
197
|
+
#possible additions
|
198
|
+
|
199
|
+
#db.job_queue.group({
|
200
|
+
#'key': {'priority':1},
|
201
|
+
#'cond': {},
|
202
|
+
#'$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}},
|
203
|
+
#'initial': {complete: 0, waiting:0}
|
204
|
+
#});
|
205
|
+
|
206
|
+
#db.job_queue.group({
|
207
|
+
#'key': {'body.task':1},
|
208
|
+
#'cond': {},
|
209
|
+
#'$reduce': function(obj, prev){if(obj.complete){prev.complete += 1;}else{prev.waiting += 1;}},
|
210
|
+
#'initial': {complete: 0, waiting:0}
|
211
|
+
#});
|
212
|
+
|
213
|
+
available, complete, total, locked, redundant_completes, priority, tasks = collection.db.eval(js)
|
135
214
|
{ :locked => locked.to_i,
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
215
|
+
:complete => complete.to_i,
|
216
|
+
:available => available.to_i,
|
217
|
+
:total => total.to_i,
|
218
|
+
:redundantcompletes => redundant_completes,
|
219
|
+
:priority => priority,
|
220
|
+
:tasks => tasks
|
140
221
|
}
|
141
222
|
end
|
142
|
-
|
223
|
+
|
143
224
|
def self.generate_duplicate_key(body)
|
144
225
|
return Digest::MD5.hexdigest(body) if body.class == "String"
|
145
226
|
return Digest::MD5.hexdigest(body) if body.class == "Fixnum"
|
146
|
-
#else
|
227
|
+
#else
|
147
228
|
return Digest::MD5.hexdigest(body.to_json) #won't ever match a duplicate. Need a better way to handle hashes and arrays.
|
148
229
|
end
|
149
|
-
|
230
|
+
|
150
231
|
def peek
|
151
232
|
firstfew = collection.find({
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
233
|
+
:complete => false,
|
234
|
+
'$or'=>[{:locked_till=> nil},{:locked_till=>{'$lt'=>Time.now.utc}}]
|
235
|
+
},
|
236
|
+
:sort => [[:priority, :descending],[:inserted_at, :ascending]],
|
237
|
+
:limit => 10)
|
157
238
|
return firstfew
|
158
239
|
end
|
159
|
-
|
160
|
-
|
240
|
+
|
161
241
|
protected
|
162
|
-
|
242
|
+
|
163
243
|
def value_of(result) #:nodoc:
|
164
244
|
result['okay'] == 0 ? nil : result['value']
|
165
245
|
end
|
166
|
-
|
167
|
-
|
246
|
+
|
168
247
|
end
|
data/mongo-dequeue.gemspec
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{mongo-dequeue}
|
8
|
-
s.version = "0.5.
|
8
|
+
s.version = "0.5.1"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["TelegramSam"]
|
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
|
|
30
30
|
"pkg/mongo-dequeue-0.2.1.gem",
|
31
31
|
"pkg/mongo-dequeue-0.3.0.gem",
|
32
32
|
"pkg/mongo-dequeue-0.4.0.gem",
|
33
|
+
"pkg/mongo-dequeue-0.4.1.gem",
|
34
|
+
"pkg/mongo-dequeue-0.5.0.gem",
|
33
35
|
"spec/mongo_dequeue_spec.rb",
|
34
36
|
"spec/spec_helper.rb"
|
35
37
|
]
|
Binary file
|
Binary file
|
data/spec/mongo_dequeue_spec.rb
CHANGED
@@ -37,7 +37,7 @@ describe Mongo::Dequeue do
|
|
37
37
|
end
|
38
38
|
|
39
39
|
describe "Inserting a standard Job" do
|
40
|
-
before(:
|
40
|
+
before(:all) do
|
41
41
|
@item = insert_and_inspect({:message => 'MongoQueueSpec', :foo => 5})
|
42
42
|
end
|
43
43
|
|
@@ -70,6 +70,26 @@ describe Mongo::Dequeue do
|
|
70
70
|
@item['body']['foo'].should be(5)
|
71
71
|
end
|
72
72
|
end
|
73
|
+
|
74
|
+
describe "bulk inserting multiple jobs" do
|
75
|
+
before(:all) do
|
76
|
+
@queue.batchpush({:message => 'MongoQueueSpec1', :foo => 5})
|
77
|
+
@queue.batchpush({:message => 'MongoQueueSpec2', :foo => 5})
|
78
|
+
@queue.batchpush({:message => 'MongoQueueSpec3', :foo => 5})
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should correctly count items in batch" do
|
82
|
+
@queue.batch.length.should be(3)
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should correctly add items on process" do
|
86
|
+
@queue.batchprocess()
|
87
|
+
@queue.send(:collection).count.should == 3
|
88
|
+
@queue.batch.length.should == 0
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
73
93
|
|
74
94
|
describe "Inserting different body types" do
|
75
95
|
before(:each) do
|
@@ -261,7 +281,7 @@ describe Mongo::Dequeue do
|
|
261
281
|
@b = insert_and_inspect("b")
|
262
282
|
@c = insert_and_inspect("c")
|
263
283
|
@d = insert_and_inspect("d")
|
264
|
-
@e = insert_and_inspect("
|
284
|
+
@e = insert_and_inspect({:task => "foo"})
|
265
285
|
|
266
286
|
@ap = @queue.pop(:timeout => 1)
|
267
287
|
@bp = @queue.pop
|
@@ -289,6 +309,17 @@ describe Mongo::Dequeue do
|
|
289
309
|
it "should count redundant completes" do
|
290
310
|
@stats[:redundantcompletes].should == 0
|
291
311
|
end
|
312
|
+
it "should count priorities" do
|
313
|
+
#pp @stats[:priority]
|
314
|
+
@stats[:priority].should == [{"priority"=>3.0, "complete"=>1.0, "waiting"=>4.0}]
|
315
|
+
end
|
316
|
+
it "should count tasks" do
|
317
|
+
#pp @stats[:tasks]
|
318
|
+
@stats[:tasks].should == [
|
319
|
+
{"body.task"=>nil, "complete"=>1.0, "waiting"=>3.0},
|
320
|
+
{"body.task"=>"foo", "complete"=>0.0, "waiting"=>1.0}
|
321
|
+
]
|
322
|
+
end
|
292
323
|
|
293
324
|
|
294
325
|
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: mongo-dequeue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.5.
|
5
|
+
version: 0.5.1
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- TelegramSam
|
@@ -102,6 +102,8 @@ files:
|
|
102
102
|
- pkg/mongo-dequeue-0.2.1.gem
|
103
103
|
- pkg/mongo-dequeue-0.3.0.gem
|
104
104
|
- pkg/mongo-dequeue-0.4.0.gem
|
105
|
+
- pkg/mongo-dequeue-0.4.1.gem
|
106
|
+
- pkg/mongo-dequeue-0.5.0.gem
|
105
107
|
- spec/mongo_dequeue_spec.rb
|
106
108
|
- spec/spec_helper.rb
|
107
109
|
has_rdoc: true
|
@@ -118,7 +120,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
118
120
|
requirements:
|
119
121
|
- - ">="
|
120
122
|
- !ruby/object:Gem::Version
|
121
|
-
hash:
|
123
|
+
hash: 2125946661699072658
|
122
124
|
segments:
|
123
125
|
- 0
|
124
126
|
version: "0"
|