perfectqueue 0.7.32 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/.gitignore +6 -0
  2. data/ChangeLog +0 -62
  3. data/Gemfile +3 -0
  4. data/README.md +239 -0
  5. data/Rakefile +19 -0
  6. data/lib/perfectqueue.rb +68 -4
  7. data/lib/perfectqueue/application.rb +30 -0
  8. data/lib/perfectqueue/application/base.rb +27 -0
  9. data/lib/perfectqueue/application/dispatch.rb +73 -0
  10. data/lib/perfectqueue/application/router.rb +69 -0
  11. data/lib/perfectqueue/backend.rb +44 -47
  12. data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
  13. data/lib/perfectqueue/blocking_flag.rb +84 -0
  14. data/lib/perfectqueue/client.rb +117 -0
  15. data/lib/perfectqueue/command/perfectqueue.rb +108 -323
  16. data/lib/perfectqueue/daemons_logger.rb +80 -0
  17. data/lib/perfectqueue/engine.rb +85 -123
  18. data/lib/perfectqueue/error.rb +53 -0
  19. data/lib/perfectqueue/model.rb +37 -0
  20. data/lib/perfectqueue/multiprocess.rb +31 -0
  21. data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
  22. data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
  23. data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
  24. data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
  25. data/lib/perfectqueue/queue.rb +58 -0
  26. data/lib/perfectqueue/runner.rb +39 -0
  27. data/lib/perfectqueue/signal_queue.rb +112 -0
  28. data/lib/perfectqueue/task.rb +103 -0
  29. data/lib/perfectqueue/task_metadata.rb +98 -0
  30. data/lib/perfectqueue/task_monitor.rb +189 -0
  31. data/lib/perfectqueue/task_status.rb +27 -0
  32. data/lib/perfectqueue/version.rb +1 -3
  33. data/lib/perfectqueue/worker.rb +114 -196
  34. data/perfectqueue.gemspec +24 -0
  35. data/spec/queue_spec.rb +234 -0
  36. data/spec/spec_helper.rb +44 -0
  37. data/spec/worker_spec.rb +81 -0
  38. metadata +93 -40
  39. checksums.yaml +0 -7
  40. data/README.rdoc +0 -224
  41. data/lib/perfectqueue/backend/null.rb +0 -33
  42. data/lib/perfectqueue/backend/rdb.rb +0 -181
  43. data/lib/perfectqueue/backend/simpledb.rb +0 -139
  44. data/test/backend_test.rb +0 -259
  45. data/test/cat.sh +0 -2
  46. data/test/echo.sh +0 -4
  47. data/test/exec_test.rb +0 -61
  48. data/test/fail.sh +0 -2
  49. data/test/huge.sh +0 -2
  50. data/test/stress.rb +0 -99
  51. data/test/success.sh +0 -2
  52. data/test/test_helper.rb +0 -19
@@ -0,0 +1,27 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module TaskStatus
21
+ WAITING = :waiting
22
+ RUNNING = :running
23
+ FINISHED = :finished
24
+ CANCEL_REQUESTED = :cancel_requested
25
+ end
26
+ end
27
+
@@ -1,5 +1,3 @@
1
1
  module PerfectQueue
2
-
3
- VERSION = '0.7.32'
4
-
2
+ VERSION = "0.8.0"
5
3
  end
@@ -1,236 +1,154 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
1
18
 
2
19
  module PerfectQueue
3
20
 
21
+ class Worker
22
+ def self.run(runner, config=nil, &block)
23
+ new(runner, config, &block).run
24
+ end
4
25
 
5
- class MonitorThread
6
- def initialize(engine, conf)
7
- @engine = engine
8
- @log = @engine.log
9
- @backend = engine.backend
10
- @finished = false
11
-
12
- @timeout = conf[:timeout] || 600
13
- @heartbeat_interval = conf[:heartbeat_interval] || @timeout*3/4
14
- @kill_timeout = conf[:kill_timeout] || @timeout*10
15
- @kill_interval = conf[:kill_interval] || 60
16
- @retry_wait = conf[:retry_wait] || nil
17
- @delete_wait = conf[:delete_wait] || 3600
18
-
19
- @token = nil
20
- @heartbeat_time = nil
21
- @kill_time = nil
22
- @kill_proc = nil
23
- @canceled = false
24
- @mutex = Mutex.new
25
- @cond = ConditionVariable.new
26
- end
27
-
28
- def start
29
- @thread = Thread.new(&method(:run))
30
- end
26
+ def initialize(runner, config=nil, &block)
27
+ # initial logger
28
+ STDERR.sync = true
29
+ @log = DaemonsLogger.new(STDERR)
31
30
 
32
- def run
33
- until @finished
34
- @mutex.synchronize {
35
- while true
36
- return if @finished
37
- break if @token
38
- @cond.wait(@mutex)
39
- end
40
- }
41
- process
31
+ @runner = runner
32
+ block = Proc.new { config } if config
33
+ @config_load_proc = block
34
+ @finished = false
42
35
  end
43
- rescue
44
- @engine.stop($!)
45
- end
46
36
 
47
- def process
48
- while true
49
- sleep 1
50
- @mutex.synchronize {
51
- return if @finished
52
- return unless @token
53
- now = Time.now.to_i
54
- try_extend(now)
55
- try_kill(now)
56
- }
37
+ def run
38
+ @sig = install_signal_handlers
39
+ begin
40
+ @engine = Engine.new(@runner, load_config)
41
+ begin
42
+ @engine.run
43
+ ensure
44
+ @engine.shutdown(true)
45
+ end
46
+ ensure
47
+ @sig.shutdown
48
+ end
49
+ return nil
50
+ rescue
51
+ @log.error "#{$!.class}: #{$!}"
52
+ $!.backtrace.each {|x| @log.warn "\t#{x}" }
53
+ return nil
57
54
  end
58
- end
59
55
 
60
- def try_extend(now)
61
- if now >= @heartbeat_time && !@canceled
62
- @log.debug "extending timeout=#{now+@timeout} id=#{@task_id}"
56
+ def stop(immediate)
57
+ @log.info immediate ? "Received immediate stop" : "Received graceful stop"
63
58
  begin
64
- @backend.update(@token, now+@timeout)
65
- rescue CanceledError
66
- @log.info "task id=#{@task_id} is canceled."
67
- @canceled = true
68
- @kill_time = now
59
+ @engine.stop(immediate) if @engine
69
60
  rescue
70
- @log.error "unexpected error id=#{@task_id}: #{$!}"
71
- @canceled = true
72
- @kill_time = now
61
+ @log.error "failed to stop: #{$!}"
62
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
63
+ return false
73
64
  end
74
- @heartbeat_time = now + @heartbeat_interval
65
+ return true
75
66
  end
76
- end
77
67
 
78
- def try_kill(now)
79
- if now >= @kill_time
80
- kill!
81
- @kill_time = now + @kill_interval
68
+ def restart(immediate)
69
+ @log.info immediate ? "Received immediate restart" : "Received graceful restart"
70
+ begin
71
+ @engine.restart(immediate, load_config)
72
+ rescue
73
+ @log.error "failed to restart: #{$!}"
74
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
75
+ return false
76
+ end
77
+ return true
82
78
  end
83
- end
84
79
 
85
- def kill!
86
- if @kill_proc
87
- @log.info "killing id=#{@task_id}..."
80
+ def replace(immediate, command=[$0]+ARGV)
81
+ @log.info immediate ? "Received immediate binary replace" : "Received graceful binary replace"
88
82
  begin
89
- @kill_proc.call
83
+ @engine.replace(immediate, command)
90
84
  rescue
91
- @log.info "kill failed id=#{@task_id}: #{$!.class}: #{$!}"
92
- $!.backtrace.each {|bt|
93
- @log.debug " #{bt}"
94
- }
85
+ @log.error "failed to replace: #{$!}"
86
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
87
+ return false
95
88
  end
89
+ return true
96
90
  end
97
- end
98
91
 
99
- def stop
100
- @mutex.synchronize {
101
- @finished = true
102
- @cond.broadcast
103
- }
104
- end
105
-
106
- def shutdown
107
- @thread.join
108
- end
92
+ def logrotated
93
+ @log.info "reopen a log file"
94
+ @engine.logrotated
95
+ @log.reopen!
96
+ return true
97
+ end
109
98
 
110
- def set(token, task_id)
111
- @mutex.synchronize {
112
- now = Time.now.to_i
113
- @token = token
114
- @task_id = task_id
115
- @heartbeat_time = now + @heartbeat_interval
116
- @kill_time = now + @kill_timeout
117
- @kill_proc = nil
118
- @canceled = false
119
- @cond.broadcast
120
- }
121
- end
99
+ private
100
+ def load_config
101
+ raw_config = @config_load_proc.call
102
+ config = {}
103
+ raw_config.each_pair {|k,v| config[k.to_sym] = v }
122
104
 
123
- def set_kill_proc(kill_proc)
124
- @kill_proc = kill_proc
125
- end
105
+ old_log = @log
106
+ log = DaemonsLogger.new(config[:log] || STDERR)
107
+ old_log.close if old_log
108
+ @log = log
126
109
 
127
- def reset(success)
128
- @mutex.synchronize {
129
- if success
130
- @backend.finish(@token, @delete_wait)
131
- elsif @retry_wait && !@canceled
132
- begin
133
- @backend.update(@token, Time.now.to_i+@retry_wait)
134
- rescue
135
- # ignore CanceledError
136
- end
137
- end
138
- @token = nil
139
- }
140
- end
141
- end
110
+ config[:logger] = log
142
111
 
112
+ return config
113
+ end
143
114
 
144
- class Worker
145
- def initialize(engine, conf)
146
- @engine = engine
147
- @log = @engine.log
115
+ def install_signal_handlers
116
+ SignalQueue.start do |sig|
117
+ sig.trap :TERM do
118
+ stop(false)
119
+ end
120
+ sig.trap :INT do
121
+ stop(false)
122
+ end
148
123
 
149
- @run_class = conf[:run_class]
150
- @monitor = MonitorThread.new(engine, conf)
124
+ sig.trap :QUIT do
125
+ stop(true)
126
+ end
151
127
 
152
- @token = nil
153
- @task = nil
154
- @mutex = Mutex.new
155
- @cond = ConditionVariable.new
156
- end
128
+ sig.trap :USR1 do
129
+ restart(false)
130
+ end
157
131
 
158
- def start
159
- @log.debug "running worker."
160
- @thread = Thread.new(&method(:run))
161
- end
132
+ sig.trap :HUP do
133
+ restart(true)
134
+ end
162
135
 
163
- def run
164
- @monitor.start
165
- begin
166
- while true
167
- @mutex.synchronize {
168
- while true
169
- return if @engine.finished?
170
- break if @token
171
- @cond.wait(@mutex)
172
- end
173
- }
174
- begin
175
- process(@token, @task)
176
- ensure
177
- @token = nil
178
- @engine.release_worker(self)
136
+ sig.trap :CONT do
137
+ replace(false)
179
138
  end
180
- end
181
- ensure
182
- @monitor.stop
183
- end
184
- rescue
185
- @engine.stop($!)
186
- end
187
139
 
188
- def process(token, task)
189
- @log.info "processing task id=#{task.id}"
140
+ sig.trap :WINCH do
141
+ replace(true)
142
+ end
190
143
 
191
- @monitor.set(token, task.id)
192
- success = false
193
- begin
194
- run = @run_class.new(task)
144
+ sig.trap :USR2 do
145
+ logrotated
146
+ end
195
147
 
196
- if run.respond_to?(:kill)
197
- @monitor.set_kill_proc run.method(:kill)
148
+ trap :CHLD, "SIG_IGN"
198
149
  end
199
-
200
- run.run
201
-
202
- @log.info "finished id=#{task.id}"
203
- success = true
204
-
205
- rescue
206
- @log.info "failed id=#{task.id}: #{$!.class}: #{$!}"
207
- $!.backtrace.each {|bt|
208
- @log.debug " #{bt}"
209
- }
210
-
211
- ensure
212
- @monitor.reset(success)
213
150
  end
214
151
  end
215
152
 
216
- def stop
217
- submit(nil, nil)
218
- end
219
-
220
- def shutdown
221
- @monitor.shutdown
222
- @thread.join
223
- end
224
-
225
- def submit(token, task)
226
- @mutex.synchronize {
227
- @token = token
228
- @task = task
229
- @cond.broadcast
230
- }
231
- end
232
- end
233
-
234
-
235
153
  end
236
154
 
@@ -0,0 +1,24 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+ require 'perfectqueue/version'
4
+
5
+ Gem::Specification.new do |gem|
6
+ gem.name = "perfectqueue"
7
+ gem.description = "Highly available distributed cron built on RDBMS"
8
+ gem.homepage = "https://github.com/treasure-data/perfectqueue"
9
+ gem.summary = gem.description
10
+ gem.version = PerfectQueue::VERSION
11
+ gem.authors = ["Sadayuki Furuhashi"]
12
+ gem.email = "frsyuki@gmail.com"
13
+ gem.has_rdoc = false
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_dependency "sequel", "~> 3.26.0"
20
+ gem.add_development_dependency "rake", "~> 0.9.2"
21
+ gem.add_development_dependency "rspec", "~> 2.10.0"
22
+ gem.add_development_dependency "simplecov", "~> 0.5.4"
23
+ gem.add_development_dependency "sqlite3", "~> 1.3.3"
24
+ end
@@ -0,0 +1,234 @@
1
+ require 'spec_helper'
2
+
3
+ describe Queue do
4
+ before do
5
+ @queue = create_test_queue
6
+ end
7
+
8
+ after do
9
+ @queue.client.close
10
+ end
11
+
12
+ it 'is a Queue' do
13
+ @queue.class.should == PerfectQueue::Queue
14
+ end
15
+
16
+ it 'succeess submit' do
17
+ @queue.submit('task01', 'type1', {})
18
+ end
19
+
20
+ it 'fail duplicated submit' do
21
+ now = Time.now.to_i
22
+ @queue.submit('task01', 'type1', {}, :now=>now)
23
+
24
+ lambda {
25
+ @queue.submit('task01', 'type1', {}, :now=>now+1)
26
+ }.should raise_error AlreadyExistsError
27
+
28
+ @queue['task01'].cancel_request!(:now=>now+2)
29
+
30
+ lambda {
31
+ @queue.submit('task01', 'type1', {}, :now=>now+10)
32
+ }.should raise_error AlreadyExistsError
33
+ end
34
+
35
+ it 'list' do
36
+ @queue.submit('task01', 'type1', {"a"=>1})
37
+ @queue.submit('task02', 'type1', {"a"=>2})
38
+ @queue.submit('task03', 'type1', {"a"=>3})
39
+
40
+ a = []
41
+ @queue.each {|t| a << t }
42
+ a.sort_by! {|t| t.key }
43
+
44
+ task01 = a.shift
45
+ task01.finished?.should == false
46
+ task01.type == 'type1'
47
+ task01.key.should == 'task01'
48
+ task01.data["a"].should == 1
49
+
50
+ t2 = a.shift
51
+ t2.finished?.should == false
52
+ t2.type == 'type1'
53
+ t2.key.should == 'task02'
54
+ t2.data["a"].should == 2
55
+
56
+ t3 = a.shift
57
+ t3.finished?.should == false
58
+ t3.type == 'type1'
59
+ t3.key.should == 'task03'
60
+ t3.data["a"].should == 3
61
+
62
+ a.empty?.should == true
63
+ end
64
+
65
+ it 'poll' do
66
+ now = Time.now.to_i
67
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
68
+ @queue.submit('task02', 'type1', {"a"=>2}, :now=>now+1)
69
+ @queue.submit('task03', 'type1', {"a"=>3}, :now=>now+2)
70
+
71
+ task01 = @queue.poll(:now=>now+10)
72
+ task01.key.should == 'task01'
73
+
74
+ t2 = @queue.poll(:now=>now+10)
75
+ t2.key.should == 'task02'
76
+
77
+ t3 = @queue.poll(:now=>now+10)
78
+ t3.key.should == 'task03'
79
+
80
+ t4 = @queue.poll(:now=>now+10)
81
+ t4.should == nil
82
+ end
83
+
84
+ it 'release' do
85
+ now = Time.now.to_i
86
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
87
+
88
+ task01 = @queue.poll(:now=>now+10)
89
+ task01.key.should == 'task01'
90
+
91
+ t2 = @queue.poll(:now=>now+10)
92
+ t2.should == nil
93
+
94
+ task01.release!(:now=>now+10)
95
+
96
+ t3 = @queue.poll(:now=>now+11)
97
+ t3.key.should == 'task01'
98
+ end
99
+
100
+ it 'timeout' do
101
+ now = Time.now.to_i
102
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
103
+
104
+ task01 = @queue.poll(:now=>now+10, :alive_time=>10)
105
+ task01.key.should == 'task01'
106
+
107
+ t2 = @queue.poll(:now=>now+15)
108
+ t2.should == nil
109
+
110
+ t3 = @queue.poll(:now=>now+20)
111
+ t3.key.should == 'task01'
112
+ end
113
+
114
+ it 'heartbeat' do
115
+ now = Time.now.to_i
116
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
117
+
118
+ task01 = @queue.poll(:now=>now+10, :alive_time=>10)
119
+ task01.key.should == 'task01'
120
+
121
+ task01.heartbeat!(:alive_time=>15, :now=>now+10)
122
+
123
+ t2 = @queue.poll(:now=>now+20)
124
+ t2.should == nil
125
+
126
+ t3 = @queue.poll(:now=>now+30)
127
+ t3.key.should == 'task01'
128
+ end
129
+
130
+ it 'retry' do
131
+ now = Time.now.to_i
132
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
133
+
134
+ task01 = @queue.poll(:now=>now+10, :alive_time=>10)
135
+ task01.key.should == 'task01'
136
+
137
+ task01.retry!(:retry_wait=>15, :now=>now+10)
138
+
139
+ t2 = @queue.poll(:now=>now+20)
140
+ t2.should == nil
141
+
142
+ t3 = @queue.poll(:now=>now+30)
143
+ t3.key.should == 'task01'
144
+ end
145
+
146
+ it 'froce_finish' do
147
+ now = Time.now.to_i
148
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
149
+
150
+ task01 = @queue.poll(:now=>now+10)
151
+ task01.key.should == 'task01'
152
+
153
+ @queue['task01'].metadata.running?.should == true
154
+
155
+ @queue['task01'].force_finish!(:now=>now+11)
156
+
157
+ @queue['task01'].metadata.finished?.should == true
158
+ end
159
+
160
+ it 'status' do
161
+ now = Time.now.to_i
162
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
163
+
164
+ # rdb_backend backend can't distinguish running with waiting
165
+ #@queue['task01'].metadata.finished?.should == false
166
+ #@queue['task01'].metadata.running?.should == false
167
+ #@queue['task01'].metadata.waiting?.should == true
168
+ #@queue['task01'].metadata.cancel_requested?.should == false
169
+
170
+ task01 = @queue.poll(:now=>now+10, :alive_time=>10)
171
+ task01.key.should == 'task01'
172
+
173
+ @queue['task01'].metadata.finished?.should == false
174
+ @queue['task01'].metadata.running?.should == true
175
+ @queue['task01'].metadata.waiting?.should == false
176
+ @queue['task01'].metadata.cancel_requested?.should == false
177
+
178
+ task01.cancel_request!
179
+
180
+ # status of cancel_requested running tasks is cancel_requested
181
+ @queue['task01'].metadata.finished?.should == false
182
+ @queue['task01'].metadata.running?.should == false
183
+ @queue['task01'].metadata.waiting?.should == false
184
+ @queue['task01'].metadata.cancel_requested?.should == true
185
+
186
+ task01.finish!
187
+
188
+ @queue['task01'].metadata.finished?.should == true
189
+ @queue['task01'].metadata.running?.should == false
190
+ @queue['task01'].metadata.waiting?.should == false
191
+ @queue['task01'].metadata.cancel_requested?.should == false
192
+ end
193
+
194
+ it 'fail canceling finished task' do
195
+ now = Time.now.to_i
196
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
197
+
198
+ task01 = @queue.poll(:now=>now+10, :alive_time=>10)
199
+ task01.key.should == 'task01'
200
+
201
+ task01.finish!
202
+
203
+ lambda {
204
+ @queue['task01'].cancel_request!
205
+ }.should raise_error AlreadyFinishedError
206
+ end
207
+
208
+ it 'retention_time' do
209
+ now = Time.now.to_i
210
+ @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
211
+
212
+ @queue['task01'].metadata.finished?.should == false
213
+
214
+ task01 = @queue.poll(:now=>now+10, :alive_time=>10)
215
+ task01.key.should == 'task01'
216
+
217
+ task01.finish!(:now=>now+11, :retention_time=>10)
218
+
219
+ @queue.poll(:now=>now+12)
220
+
221
+ @queue['task01'].exists?.should == true
222
+
223
+ @queue.poll(:now=>now+22)
224
+
225
+ @queue['task01'].exists?.should == false
226
+ end
227
+
228
+ it 'get_task_metadata failed with NotFoundError' do
229
+ lambda {
230
+ @queue['task99'].metadata
231
+ }.should raise_error NotFoundError
232
+ end
233
+ end
234
+