perfectqueue 0.8.15 → 0.8.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +10 -0
- data/lib/perfectqueue/backend/rdb_compat.rb +28 -18
- data/lib/perfectqueue/client.rb +1 -2
- data/lib/perfectqueue/engine.rb +6 -0
- data/lib/perfectqueue/multiprocess/fork_processor.rb +1 -1
- data/lib/perfectqueue/multiprocess/thread_processor.rb +19 -4
- data/lib/perfectqueue/signal_queue.rb +18 -10
- data/lib/perfectqueue/version.rb +1 -1
- data/lib/perfectqueue/worker.rb +1 -1
- data/spec/queue_spec.rb +19 -0
- metadata +2 -2
data/ChangeLog
CHANGED
@@ -1,4 +1,14 @@
|
|
1
1
|
|
2
|
+
== 2012-09-03 version 0.8.16
|
3
|
+
|
4
|
+
* Support task_prefetch option (default = 0, meaning no-prefetch)
|
5
|
+
* Wait 0.5 seconds before starting processors to avoid a spike of the
|
6
|
+
number of concurrent connections
|
7
|
+
* Increased default child_heartbeat_limit to 60 seconds
|
8
|
+
* Fixed a potential deadlock problem in shutdown process
|
9
|
+
* Fixed detach time wait
|
10
|
+
|
11
|
+
|
2
12
|
== 2012-08-30 version 0.8.15
|
3
13
|
|
4
14
|
* Improved detach process routines
|
@@ -148,29 +148,39 @@ SQL
|
|
148
148
|
now = (options[:now] || Time.now).to_i
|
149
149
|
next_timeout = now + alive_time
|
150
150
|
|
151
|
+
fetched = []
|
152
|
+
|
151
153
|
connect {
|
152
154
|
while true
|
153
155
|
rows = 0
|
154
|
-
|
155
|
-
@db.
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
156
|
+
begin
|
157
|
+
@db.transaction do
|
158
|
+
@db.fetch(@sql, now, now) {|row|
|
159
|
+
unless row[:created_at]
|
160
|
+
# finished task
|
161
|
+
@db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
|
162
|
+
|
163
|
+
else
|
164
|
+
## optimistic lock is not needed because the row is locked for update
|
165
|
+
#n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
|
166
|
+
n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
|
167
|
+
if n > 0
|
168
|
+
attributes = create_attributes(nil, row)
|
169
|
+
task_token = Token.new(row[:id])
|
170
|
+
task = AcquiredTask.new(@client, row[:id], attributes, task_token)
|
171
|
+
fetched.push task
|
172
|
+
break if fetched.size >= max_acquire
|
173
|
+
end
|
169
174
|
end
|
170
|
-
end
|
171
175
|
|
172
|
-
|
173
|
-
|
176
|
+
rows += 1
|
177
|
+
}
|
178
|
+
end
|
179
|
+
rescue
|
180
|
+
raise if fetched.empty?
|
181
|
+
end
|
182
|
+
unless fetched.empty?
|
183
|
+
return fetched
|
174
184
|
end
|
175
185
|
break nil if rows < MAX_SELECT_ROW
|
176
186
|
end
|
data/lib/perfectqueue/client.rb
CHANGED
@@ -24,7 +24,6 @@ module PerfectQueue
|
|
24
24
|
|
25
25
|
@backend = Backend.new_backend(self, @config)
|
26
26
|
|
27
|
-
@max_acquire = @config[:max_acquire] || 1
|
28
27
|
@retention_time = @config[:retention_time] || 300
|
29
28
|
@alive_time = @config[:alive_time] || 300
|
30
29
|
@retry_wait = @config[:retry_wait] || 300 # TODO retry wait algorithm
|
@@ -65,7 +64,7 @@ module PerfectQueue
|
|
65
64
|
# :alive_time => nil
|
66
65
|
def acquire(options={})
|
67
66
|
alive_time = options[:alive_time] || @alive_time
|
68
|
-
max_acquire = options[:max_acquire] ||
|
67
|
+
max_acquire = options[:max_acquire] || 1
|
69
68
|
|
70
69
|
@backend.acquire(alive_time, max_acquire, options)
|
71
70
|
end
|
data/lib/perfectqueue/engine.rb
CHANGED
@@ -74,6 +74,12 @@ module PerfectQueue
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def run
|
77
|
+
@processors.each {|c|
|
78
|
+
c.keepalive
|
79
|
+
# add wait time before starting processors to avoid
|
80
|
+
# a spike of the number of concurrent connections.
|
81
|
+
sleep rand # upto 1 second, average 0.5 seoncd
|
82
|
+
}
|
77
83
|
until @finish_flag.set?
|
78
84
|
@processors.each {|c| c.keepalive }
|
79
85
|
@finish_flag.wait(@child_keepalive_interval)
|
@@ -33,7 +33,7 @@ module PerfectQueue
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def restart(immediate, config)
|
36
|
-
@child_heartbeat_limit = config[:child_heartbeat_limit] ||
|
36
|
+
@child_heartbeat_limit = config[:child_heartbeat_limit] || 60.0
|
37
37
|
@child_kill_interval = config[:child_kill_interval] || 2.0
|
38
38
|
@child_graceful_kill_limit = config[:child_graceful_kill_limit] || nil
|
39
39
|
@child_fork_frequency_limit = config[:child_fork_frequency_limit] || 5.0
|
@@ -59,6 +59,7 @@ module PerfectQueue
|
|
59
59
|
def restart(immediate, config)
|
60
60
|
@poll_interval = config[:poll_interval] || 1.0
|
61
61
|
@log = config[:logger]
|
62
|
+
@task_prefetch = config[:task_prefetch] || 0
|
62
63
|
@config = config
|
63
64
|
|
64
65
|
@tm.stop_task(immediate)
|
@@ -92,11 +93,25 @@ module PerfectQueue
|
|
92
93
|
def run_loop
|
93
94
|
PerfectQueue.open(@config) {|queue|
|
94
95
|
until @finish_flag.set?
|
95
|
-
|
96
|
-
if
|
97
|
-
process(task)
|
98
|
-
else
|
96
|
+
tasks = queue.poll_multi(:max_acquire=>1+@task_prefetch)
|
97
|
+
if tasks == nil || tasks.empty?
|
99
98
|
@finish_flag.wait(@poll_interval)
|
99
|
+
else
|
100
|
+
begin
|
101
|
+
while task = tasks.shift
|
102
|
+
process(task)
|
103
|
+
end
|
104
|
+
ensure
|
105
|
+
# TODO do not call release! because rdb_compat backend
|
106
|
+
# doesn't have a mechanism to detect preemption.
|
107
|
+
# release! could cause a problem that multiple
|
108
|
+
# workers run one task concurrently.
|
109
|
+
#tasks.each {|task|
|
110
|
+
# # ignoring errors doesn't cause serious problems
|
111
|
+
# # because it's same as failure of this server.
|
112
|
+
# task.release! rescue nil
|
113
|
+
#}
|
114
|
+
end
|
100
115
|
end
|
101
116
|
end
|
102
117
|
}
|
@@ -32,19 +32,26 @@ module PerfectQueue
|
|
32
32
|
@queue = []
|
33
33
|
@mutex = Mutex.new
|
34
34
|
@cond = ConditionVariable.new
|
35
|
+
@finished = false
|
35
36
|
|
36
37
|
block.call(self) if block
|
37
38
|
end
|
38
39
|
|
39
|
-
def trap(sig, &block)
|
40
|
+
def trap(sig, command=nil, &block)
|
40
41
|
sig = sig.to_sym
|
41
42
|
old = @handlers[sig]
|
42
43
|
|
43
|
-
|
44
|
-
|
44
|
+
if block
|
45
|
+
Kernel.trap(sig) do
|
46
|
+
enqueue(sig)
|
47
|
+
end
|
48
|
+
@handlers[sig] = block
|
49
|
+
|
50
|
+
else
|
51
|
+
Kernel.trap(sig, command)
|
52
|
+
@handlers.delete(sig)
|
45
53
|
end
|
46
54
|
|
47
|
-
@handlers[sig] = block
|
48
55
|
old
|
49
56
|
end
|
50
57
|
|
@@ -57,7 +64,8 @@ module PerfectQueue
|
|
57
64
|
end
|
58
65
|
|
59
66
|
def stop
|
60
|
-
|
67
|
+
@finished = true
|
68
|
+
enqueue(nil) # TODO causes recursive lock?
|
61
69
|
end
|
62
70
|
|
63
71
|
def shutdown
|
@@ -66,16 +74,16 @@ module PerfectQueue
|
|
66
74
|
end
|
67
75
|
|
68
76
|
def run
|
69
|
-
|
70
|
-
until finished
|
77
|
+
@owner_thread = Thread.current
|
78
|
+
until @finished
|
71
79
|
h = nil
|
72
80
|
@mutex.synchronize do
|
73
81
|
while @queue.empty?
|
74
|
-
@cond.wait(@mutex)
|
82
|
+
@cond.wait(@mutex, 0.5)
|
75
83
|
end
|
76
84
|
sig = @queue.shift
|
77
85
|
if sig == nil
|
78
|
-
finished = true
|
86
|
+
@finished = true
|
79
87
|
else
|
80
88
|
h = @handlers[sig]
|
81
89
|
end
|
@@ -95,7 +103,7 @@ module PerfectQueue
|
|
95
103
|
|
96
104
|
private
|
97
105
|
def enqueue(sig)
|
98
|
-
if Thread.current == @
|
106
|
+
if Thread.current == @owner_thread
|
99
107
|
@queue << sig
|
100
108
|
if @mutex.locked?
|
101
109
|
@cond.signal
|
data/lib/perfectqueue/version.rb
CHANGED
data/lib/perfectqueue/worker.rb
CHANGED
data/spec/queue_spec.rb
CHANGED
@@ -224,5 +224,24 @@ describe Queue do
|
|
224
224
|
queue['task99'].metadata
|
225
225
|
}.should raise_error NotFoundError
|
226
226
|
end
|
227
|
+
|
228
|
+
it 'prefetch' do
|
229
|
+
now = Time.now.to_i
|
230
|
+
queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
|
231
|
+
queue.submit('task02', 'type2', {"a"=>2}, :now=>now+1)
|
232
|
+
queue.submit('task03', 'type3', {"a"=>3}, :now=>now+2)
|
233
|
+
|
234
|
+
tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
|
235
|
+
tasks.size.should == 2
|
236
|
+
tasks[0].key.should == 'task01'
|
237
|
+
tasks[1].key.should == 'task02'
|
238
|
+
|
239
|
+
tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
|
240
|
+
tasks.size.should == 1
|
241
|
+
tasks[0].key.should == 'task03'
|
242
|
+
|
243
|
+
tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
|
244
|
+
tasks.should == nil
|
245
|
+
end
|
227
246
|
end
|
228
247
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.16
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-10-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sequel
|