perfectqueue 0.8.15 → 0.8.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog CHANGED
@@ -1,4 +1,14 @@
1
1
 
2
+ == 2012-09-03 version 0.8.16
3
+
4
+ * Support task_prefetch option (default = 0, meaning no-prefetch)
5
+ * Wait 0.5 seconds before starting processors to avoid a spike of the
6
+ number of concurrent connections
7
+ * Increased default child_heartbeat_limit to 60 seconds
8
+ * Fixed a potential deadlock problem in shutdown process
9
+ * Fixed detach time wait
10
+
11
+
2
12
  == 2012-08-30 version 0.8.15
3
13
 
4
14
  * Improved detach process routines
@@ -148,29 +148,39 @@ SQL
148
148
  now = (options[:now] || Time.now).to_i
149
149
  next_timeout = now + alive_time
150
150
 
151
+ fetched = []
152
+
151
153
  connect {
152
154
  while true
153
155
  rows = 0
154
- @db.transaction do
155
- @db.fetch(@sql, now, now) {|row|
156
- unless row[:created_at]
157
- # finished task
158
- @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
159
-
160
- else
161
- ## optimistic lock is not needed because the row is locked for update
162
- #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
163
- n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
164
- if n > 0
165
- attributes = create_attributes(nil, row)
166
- task_token = Token.new(row[:id])
167
- task = AcquiredTask.new(@client, row[:id], attributes, task_token)
168
- return [task]
156
+ begin
157
+ @db.transaction do
158
+ @db.fetch(@sql, now, now) {|row|
159
+ unless row[:created_at]
160
+ # finished task
161
+ @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
162
+
163
+ else
164
+ ## optimistic lock is not needed because the row is locked for update
165
+ #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
166
+ n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
167
+ if n > 0
168
+ attributes = create_attributes(nil, row)
169
+ task_token = Token.new(row[:id])
170
+ task = AcquiredTask.new(@client, row[:id], attributes, task_token)
171
+ fetched.push task
172
+ break if fetched.size >= max_acquire
173
+ end
169
174
  end
170
- end
171
175
 
172
- rows += 1
173
- }
176
+ rows += 1
177
+ }
178
+ end
179
+ rescue
180
+ raise if fetched.empty?
181
+ end
182
+ unless fetched.empty?
183
+ return fetched
174
184
  end
175
185
  break nil if rows < MAX_SELECT_ROW
176
186
  end
@@ -24,7 +24,6 @@ module PerfectQueue
24
24
 
25
25
  @backend = Backend.new_backend(self, @config)
26
26
 
27
- @max_acquire = @config[:max_acquire] || 1
28
27
  @retention_time = @config[:retention_time] || 300
29
28
  @alive_time = @config[:alive_time] || 300
30
29
  @retry_wait = @config[:retry_wait] || 300 # TODO retry wait algorithm
@@ -65,7 +64,7 @@ module PerfectQueue
65
64
  # :alive_time => nil
66
65
  def acquire(options={})
67
66
  alive_time = options[:alive_time] || @alive_time
68
- max_acquire = options[:max_acquire] || @max_acquire
67
+ max_acquire = options[:max_acquire] || 1
69
68
 
70
69
  @backend.acquire(alive_time, max_acquire, options)
71
70
  end
@@ -74,6 +74,12 @@ module PerfectQueue
74
74
  end
75
75
 
76
76
  def run
77
+ @processors.each {|c|
78
+ c.keepalive
79
+ # add wait time before starting processors to avoid
80
+ # a spike of the number of concurrent connections.
81
+ sleep rand # upto 1 second, average 0.5 seoncd
82
+ }
77
83
  until @finish_flag.set?
78
84
  @processors.each {|c| c.keepalive }
79
85
  @finish_flag.wait(@child_keepalive_interval)
@@ -33,7 +33,7 @@ module PerfectQueue
33
33
  end
34
34
 
35
35
  def restart(immediate, config)
36
- @child_heartbeat_limit = config[:child_heartbeat_limit] || 10.0
36
+ @child_heartbeat_limit = config[:child_heartbeat_limit] || 60.0
37
37
  @child_kill_interval = config[:child_kill_interval] || 2.0
38
38
  @child_graceful_kill_limit = config[:child_graceful_kill_limit] || nil
39
39
  @child_fork_frequency_limit = config[:child_fork_frequency_limit] || 5.0
@@ -59,6 +59,7 @@ module PerfectQueue
59
59
  def restart(immediate, config)
60
60
  @poll_interval = config[:poll_interval] || 1.0
61
61
  @log = config[:logger]
62
+ @task_prefetch = config[:task_prefetch] || 0
62
63
  @config = config
63
64
 
64
65
  @tm.stop_task(immediate)
@@ -92,11 +93,25 @@ module PerfectQueue
92
93
  def run_loop
93
94
  PerfectQueue.open(@config) {|queue|
94
95
  until @finish_flag.set?
95
- task = queue.poll
96
- if task
97
- process(task)
98
- else
96
+ tasks = queue.poll_multi(:max_acquire=>1+@task_prefetch)
97
+ if tasks == nil || tasks.empty?
99
98
  @finish_flag.wait(@poll_interval)
99
+ else
100
+ begin
101
+ while task = tasks.shift
102
+ process(task)
103
+ end
104
+ ensure
105
+ # TODO do not call release! because rdb_compat backend
106
+ # doesn't have a mechanism to detect preemption.
107
+ # release! could cause a problem that multiple
108
+ # workers run one task concurrently.
109
+ #tasks.each {|task|
110
+ # # ignoring errors doesn't cause serious problems
111
+ # # because it's same as failure of this server.
112
+ # task.release! rescue nil
113
+ #}
114
+ end
100
115
  end
101
116
  end
102
117
  }
@@ -32,19 +32,26 @@ module PerfectQueue
32
32
  @queue = []
33
33
  @mutex = Mutex.new
34
34
  @cond = ConditionVariable.new
35
+ @finished = false
35
36
 
36
37
  block.call(self) if block
37
38
  end
38
39
 
39
- def trap(sig, &block)
40
+ def trap(sig, command=nil, &block)
40
41
  sig = sig.to_sym
41
42
  old = @handlers[sig]
42
43
 
43
- Kernel.trap(sig) do
44
- enqueue(sig)
44
+ if block
45
+ Kernel.trap(sig) do
46
+ enqueue(sig)
47
+ end
48
+ @handlers[sig] = block
49
+
50
+ else
51
+ Kernel.trap(sig, command)
52
+ @handlers.delete(sig)
45
53
  end
46
54
 
47
- @handlers[sig] = block
48
55
  old
49
56
  end
50
57
 
@@ -57,7 +64,8 @@ module PerfectQueue
57
64
  end
58
65
 
59
66
  def stop
60
- enqueue(nil)
67
+ @finished = true
68
+ enqueue(nil) # TODO causes recursive lock?
61
69
  end
62
70
 
63
71
  def shutdown
@@ -66,16 +74,16 @@ module PerfectQueue
66
74
  end
67
75
 
68
76
  def run
69
- finished = false
70
- until finished
77
+ @owner_thread = Thread.current
78
+ until @finished
71
79
  h = nil
72
80
  @mutex.synchronize do
73
81
  while @queue.empty?
74
- @cond.wait(@mutex)
82
+ @cond.wait(@mutex, 0.5)
75
83
  end
76
84
  sig = @queue.shift
77
85
  if sig == nil
78
- finished = true
86
+ @finished = true
79
87
  else
80
88
  h = @handlers[sig]
81
89
  end
@@ -95,7 +103,7 @@ module PerfectQueue
95
103
 
96
104
  private
97
105
  def enqueue(sig)
98
- if Thread.current == @thread
106
+ if Thread.current == @owner_thread
99
107
  @queue << sig
100
108
  if @mutex.locked?
101
109
  @cond.signal
@@ -1,3 +1,3 @@
1
1
  module PerfectQueue
2
- VERSION = "0.8.15"
2
+ VERSION = "0.8.16"
3
3
  end
@@ -59,7 +59,7 @@ module PerfectQueue
59
59
  if @detach
60
60
  wait_time = Time.now + @detach_wait
61
61
  while (w = wait_time - Time.now) > 0
62
- sleep [1, w].max
62
+ sleep [0.5, w].min
63
63
  pid, status = Process.waitpid2(@pid, Process::WNOHANG)
64
64
  break if pid
65
65
  end
data/spec/queue_spec.rb CHANGED
@@ -224,5 +224,24 @@ describe Queue do
224
224
  queue['task99'].metadata
225
225
  }.should raise_error NotFoundError
226
226
  end
227
+
228
+ it 'prefetch' do
229
+ now = Time.now.to_i
230
+ queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
231
+ queue.submit('task02', 'type2', {"a"=>2}, :now=>now+1)
232
+ queue.submit('task03', 'type3', {"a"=>3}, :now=>now+2)
233
+
234
+ tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
235
+ tasks.size.should == 2
236
+ tasks[0].key.should == 'task01'
237
+ tasks[1].key.should == 'task02'
238
+
239
+ tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
240
+ tasks.size.should == 1
241
+ tasks[0].key.should == 'task03'
242
+
243
+ tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
244
+ tasks.should == nil
245
+ end
227
246
  end
228
247
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perfectqueue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.15
4
+ version: 0.8.16
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-08-31 00:00:00.000000000 Z
12
+ date: 2012-10-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sequel