perfectqueue 0.8.15 → 0.8.16
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +10 -0
- data/lib/perfectqueue/backend/rdb_compat.rb +28 -18
- data/lib/perfectqueue/client.rb +1 -2
- data/lib/perfectqueue/engine.rb +6 -0
- data/lib/perfectqueue/multiprocess/fork_processor.rb +1 -1
- data/lib/perfectqueue/multiprocess/thread_processor.rb +19 -4
- data/lib/perfectqueue/signal_queue.rb +18 -10
- data/lib/perfectqueue/version.rb +1 -1
- data/lib/perfectqueue/worker.rb +1 -1
- data/spec/queue_spec.rb +19 -0
- metadata +2 -2
data/ChangeLog
CHANGED
@@ -1,4 +1,14 @@
|
|
1
1
|
|
2
|
+
== 2012-09-03 version 0.8.16
|
3
|
+
|
4
|
+
* Support task_prefetch option (default = 0, meaning no-prefetch)
|
5
|
+
* Wait 0.5 seconds before starting processors to avoid a spike of the
|
6
|
+
number of concurrent connections
|
7
|
+
* Increased default child_heartbeat_limit to 60 seconds
|
8
|
+
* Fixed a potential deadlock problem in shutdown process
|
9
|
+
* Fixed detach time wait
|
10
|
+
|
11
|
+
|
2
12
|
== 2012-08-30 version 0.8.15
|
3
13
|
|
4
14
|
* Improved detach process routines
|
@@ -148,29 +148,39 @@ SQL
|
|
148
148
|
now = (options[:now] || Time.now).to_i
|
149
149
|
next_timeout = now + alive_time
|
150
150
|
|
151
|
+
fetched = []
|
152
|
+
|
151
153
|
connect {
|
152
154
|
while true
|
153
155
|
rows = 0
|
154
|
-
|
155
|
-
@db.
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
156
|
+
begin
|
157
|
+
@db.transaction do
|
158
|
+
@db.fetch(@sql, now, now) {|row|
|
159
|
+
unless row[:created_at]
|
160
|
+
# finished task
|
161
|
+
@db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
|
162
|
+
|
163
|
+
else
|
164
|
+
## optimistic lock is not needed because the row is locked for update
|
165
|
+
#n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
|
166
|
+
n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
|
167
|
+
if n > 0
|
168
|
+
attributes = create_attributes(nil, row)
|
169
|
+
task_token = Token.new(row[:id])
|
170
|
+
task = AcquiredTask.new(@client, row[:id], attributes, task_token)
|
171
|
+
fetched.push task
|
172
|
+
break if fetched.size >= max_acquire
|
173
|
+
end
|
169
174
|
end
|
170
|
-
end
|
171
175
|
|
172
|
-
|
173
|
-
|
176
|
+
rows += 1
|
177
|
+
}
|
178
|
+
end
|
179
|
+
rescue
|
180
|
+
raise if fetched.empty?
|
181
|
+
end
|
182
|
+
unless fetched.empty?
|
183
|
+
return fetched
|
174
184
|
end
|
175
185
|
break nil if rows < MAX_SELECT_ROW
|
176
186
|
end
|
data/lib/perfectqueue/client.rb
CHANGED
@@ -24,7 +24,6 @@ module PerfectQueue
|
|
24
24
|
|
25
25
|
@backend = Backend.new_backend(self, @config)
|
26
26
|
|
27
|
-
@max_acquire = @config[:max_acquire] || 1
|
28
27
|
@retention_time = @config[:retention_time] || 300
|
29
28
|
@alive_time = @config[:alive_time] || 300
|
30
29
|
@retry_wait = @config[:retry_wait] || 300 # TODO retry wait algorithm
|
@@ -65,7 +64,7 @@ module PerfectQueue
|
|
65
64
|
# :alive_time => nil
|
66
65
|
def acquire(options={})
|
67
66
|
alive_time = options[:alive_time] || @alive_time
|
68
|
-
max_acquire = options[:max_acquire] ||
|
67
|
+
max_acquire = options[:max_acquire] || 1
|
69
68
|
|
70
69
|
@backend.acquire(alive_time, max_acquire, options)
|
71
70
|
end
|
data/lib/perfectqueue/engine.rb
CHANGED
@@ -74,6 +74,12 @@ module PerfectQueue
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def run
|
77
|
+
@processors.each {|c|
|
78
|
+
c.keepalive
|
79
|
+
# add wait time before starting processors to avoid
|
80
|
+
# a spike of the number of concurrent connections.
|
81
|
+
sleep rand # upto 1 second, average 0.5 seoncd
|
82
|
+
}
|
77
83
|
until @finish_flag.set?
|
78
84
|
@processors.each {|c| c.keepalive }
|
79
85
|
@finish_flag.wait(@child_keepalive_interval)
|
@@ -33,7 +33,7 @@ module PerfectQueue
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def restart(immediate, config)
|
36
|
-
@child_heartbeat_limit = config[:child_heartbeat_limit] ||
|
36
|
+
@child_heartbeat_limit = config[:child_heartbeat_limit] || 60.0
|
37
37
|
@child_kill_interval = config[:child_kill_interval] || 2.0
|
38
38
|
@child_graceful_kill_limit = config[:child_graceful_kill_limit] || nil
|
39
39
|
@child_fork_frequency_limit = config[:child_fork_frequency_limit] || 5.0
|
@@ -59,6 +59,7 @@ module PerfectQueue
|
|
59
59
|
def restart(immediate, config)
|
60
60
|
@poll_interval = config[:poll_interval] || 1.0
|
61
61
|
@log = config[:logger]
|
62
|
+
@task_prefetch = config[:task_prefetch] || 0
|
62
63
|
@config = config
|
63
64
|
|
64
65
|
@tm.stop_task(immediate)
|
@@ -92,11 +93,25 @@ module PerfectQueue
|
|
92
93
|
def run_loop
|
93
94
|
PerfectQueue.open(@config) {|queue|
|
94
95
|
until @finish_flag.set?
|
95
|
-
|
96
|
-
if
|
97
|
-
process(task)
|
98
|
-
else
|
96
|
+
tasks = queue.poll_multi(:max_acquire=>1+@task_prefetch)
|
97
|
+
if tasks == nil || tasks.empty?
|
99
98
|
@finish_flag.wait(@poll_interval)
|
99
|
+
else
|
100
|
+
begin
|
101
|
+
while task = tasks.shift
|
102
|
+
process(task)
|
103
|
+
end
|
104
|
+
ensure
|
105
|
+
# TODO do not call release! because rdb_compat backend
|
106
|
+
# doesn't have a mechanism to detect preemption.
|
107
|
+
# release! could cause a problem that multiple
|
108
|
+
# workers run one task concurrently.
|
109
|
+
#tasks.each {|task|
|
110
|
+
# # ignoring errors doesn't cause serious problems
|
111
|
+
# # because it's same as failure of this server.
|
112
|
+
# task.release! rescue nil
|
113
|
+
#}
|
114
|
+
end
|
100
115
|
end
|
101
116
|
end
|
102
117
|
}
|
@@ -32,19 +32,26 @@ module PerfectQueue
|
|
32
32
|
@queue = []
|
33
33
|
@mutex = Mutex.new
|
34
34
|
@cond = ConditionVariable.new
|
35
|
+
@finished = false
|
35
36
|
|
36
37
|
block.call(self) if block
|
37
38
|
end
|
38
39
|
|
39
|
-
def trap(sig, &block)
|
40
|
+
def trap(sig, command=nil, &block)
|
40
41
|
sig = sig.to_sym
|
41
42
|
old = @handlers[sig]
|
42
43
|
|
43
|
-
|
44
|
-
|
44
|
+
if block
|
45
|
+
Kernel.trap(sig) do
|
46
|
+
enqueue(sig)
|
47
|
+
end
|
48
|
+
@handlers[sig] = block
|
49
|
+
|
50
|
+
else
|
51
|
+
Kernel.trap(sig, command)
|
52
|
+
@handlers.delete(sig)
|
45
53
|
end
|
46
54
|
|
47
|
-
@handlers[sig] = block
|
48
55
|
old
|
49
56
|
end
|
50
57
|
|
@@ -57,7 +64,8 @@ module PerfectQueue
|
|
57
64
|
end
|
58
65
|
|
59
66
|
def stop
|
60
|
-
|
67
|
+
@finished = true
|
68
|
+
enqueue(nil) # TODO causes recursive lock?
|
61
69
|
end
|
62
70
|
|
63
71
|
def shutdown
|
@@ -66,16 +74,16 @@ module PerfectQueue
|
|
66
74
|
end
|
67
75
|
|
68
76
|
def run
|
69
|
-
|
70
|
-
until finished
|
77
|
+
@owner_thread = Thread.current
|
78
|
+
until @finished
|
71
79
|
h = nil
|
72
80
|
@mutex.synchronize do
|
73
81
|
while @queue.empty?
|
74
|
-
@cond.wait(@mutex)
|
82
|
+
@cond.wait(@mutex, 0.5)
|
75
83
|
end
|
76
84
|
sig = @queue.shift
|
77
85
|
if sig == nil
|
78
|
-
finished = true
|
86
|
+
@finished = true
|
79
87
|
else
|
80
88
|
h = @handlers[sig]
|
81
89
|
end
|
@@ -95,7 +103,7 @@ module PerfectQueue
|
|
95
103
|
|
96
104
|
private
|
97
105
|
def enqueue(sig)
|
98
|
-
if Thread.current == @
|
106
|
+
if Thread.current == @owner_thread
|
99
107
|
@queue << sig
|
100
108
|
if @mutex.locked?
|
101
109
|
@cond.signal
|
data/lib/perfectqueue/version.rb
CHANGED
data/lib/perfectqueue/worker.rb
CHANGED
data/spec/queue_spec.rb
CHANGED
@@ -224,5 +224,24 @@ describe Queue do
|
|
224
224
|
queue['task99'].metadata
|
225
225
|
}.should raise_error NotFoundError
|
226
226
|
end
|
227
|
+
|
228
|
+
it 'prefetch' do
|
229
|
+
now = Time.now.to_i
|
230
|
+
queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
|
231
|
+
queue.submit('task02', 'type2', {"a"=>2}, :now=>now+1)
|
232
|
+
queue.submit('task03', 'type3', {"a"=>3}, :now=>now+2)
|
233
|
+
|
234
|
+
tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
|
235
|
+
tasks.size.should == 2
|
236
|
+
tasks[0].key.should == 'task01'
|
237
|
+
tasks[1].key.should == 'task02'
|
238
|
+
|
239
|
+
tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
|
240
|
+
tasks.size.should == 1
|
241
|
+
tasks[0].key.should == 'task03'
|
242
|
+
|
243
|
+
tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
|
244
|
+
tasks.should == nil
|
245
|
+
end
|
227
246
|
end
|
228
247
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.16
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-10-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sequel
|