perfectqueue 0.7.32 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/.gitignore +6 -0
  2. data/ChangeLog +0 -62
  3. data/Gemfile +3 -0
  4. data/README.md +239 -0
  5. data/Rakefile +19 -0
  6. data/lib/perfectqueue.rb +68 -4
  7. data/lib/perfectqueue/application.rb +30 -0
  8. data/lib/perfectqueue/application/base.rb +27 -0
  9. data/lib/perfectqueue/application/dispatch.rb +73 -0
  10. data/lib/perfectqueue/application/router.rb +69 -0
  11. data/lib/perfectqueue/backend.rb +44 -47
  12. data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
  13. data/lib/perfectqueue/blocking_flag.rb +84 -0
  14. data/lib/perfectqueue/client.rb +117 -0
  15. data/lib/perfectqueue/command/perfectqueue.rb +108 -323
  16. data/lib/perfectqueue/daemons_logger.rb +80 -0
  17. data/lib/perfectqueue/engine.rb +85 -123
  18. data/lib/perfectqueue/error.rb +53 -0
  19. data/lib/perfectqueue/model.rb +37 -0
  20. data/lib/perfectqueue/multiprocess.rb +31 -0
  21. data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
  22. data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
  23. data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
  24. data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
  25. data/lib/perfectqueue/queue.rb +58 -0
  26. data/lib/perfectqueue/runner.rb +39 -0
  27. data/lib/perfectqueue/signal_queue.rb +112 -0
  28. data/lib/perfectqueue/task.rb +103 -0
  29. data/lib/perfectqueue/task_metadata.rb +98 -0
  30. data/lib/perfectqueue/task_monitor.rb +189 -0
  31. data/lib/perfectqueue/task_status.rb +27 -0
  32. data/lib/perfectqueue/version.rb +1 -3
  33. data/lib/perfectqueue/worker.rb +114 -196
  34. data/perfectqueue.gemspec +24 -0
  35. data/spec/queue_spec.rb +234 -0
  36. data/spec/spec_helper.rb +44 -0
  37. data/spec/worker_spec.rb +81 -0
  38. metadata +93 -40
  39. checksums.yaml +0 -7
  40. data/README.rdoc +0 -224
  41. data/lib/perfectqueue/backend/null.rb +0 -33
  42. data/lib/perfectqueue/backend/rdb.rb +0 -181
  43. data/lib/perfectqueue/backend/simpledb.rb +0 -139
  44. data/test/backend_test.rb +0 -259
  45. data/test/cat.sh +0 -2
  46. data/test/echo.sh +0 -4
  47. data/test/exec_test.rb +0 -61
  48. data/test/fail.sh +0 -2
  49. data/test/huge.sh +0 -2
  50. data/test/stress.rb +0 -99
  51. data/test/success.sh +0 -2
  52. data/test/test_helper.rb +0 -19
@@ -0,0 +1,39 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ class Runner
21
+ def initialize(task)
22
+ @task = task
23
+ end
24
+
25
+ attr_accessor :task
26
+
27
+ def queue
28
+ Queue.new(task.client)
29
+ end
30
+
31
+ #def run
32
+ #end
33
+
34
+ def kill(reason)
35
+ # do nothing
36
+ end
37
+ end
38
+ end
39
+
@@ -0,0 +1,112 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+
21
+ class SignalQueue
22
+ def self.start(&block)
23
+ st = new(&block)
24
+ st.start
25
+ return st
26
+ end
27
+
28
+ def initialize(&block)
29
+ require 'thread'
30
+
31
+ @handlers = {}
32
+ @queue = []
33
+ @mutex = Mutex.new
34
+ @cond = ConditionVariable.new
35
+
36
+ block.call(self) if block
37
+ end
38
+
39
+ def trap(sig, &block)
40
+ sig = sig.to_sym
41
+ old = @handlers[sig]
42
+
43
+ Kernel.trap(sig) do
44
+ enqueue(sig)
45
+ end
46
+
47
+ @handlers[sig] = block
48
+ old
49
+ end
50
+
51
+ def start
52
+ @thread = Thread.new(&method(:run))
53
+ end
54
+
55
+ def join
56
+ @thread.join
57
+ end
58
+
59
+ def stop
60
+ enqueue(nil)
61
+ end
62
+
63
+ def shutdown
64
+ stop
65
+ join
66
+ end
67
+
68
+ def run
69
+ finished = false
70
+ until finished
71
+ h = nil
72
+ @mutex.synchronize do
73
+ while @queue.empty?
74
+ @cond.wait(@mutex)
75
+ end
76
+ sig = @queue.shift
77
+ if sig == nil
78
+ finished = true
79
+ else
80
+ h = @handlers[sig]
81
+ end
82
+ end
83
+
84
+ begin
85
+ h.call if h
86
+ rescue
87
+ STDERR.print "#{$!}\n"
88
+ $!.backtrace.each {|bt|
89
+ STDERR.print "\t#{bt}\n"
90
+ STDERR.flush
91
+ }
92
+ end
93
+ end
94
+ end
95
+
96
+ private
97
+ def enqueue(sig)
98
+ if Thread.current == self
99
+ @queue << sig
100
+ if @mutex.locked?
101
+ @cond.signal
102
+ end
103
+ else
104
+ @mutex.synchronize do
105
+ @queue << sig
106
+ @cond.signal
107
+ end
108
+ end
109
+ end
110
+ end
111
+
112
+ end
@@ -0,0 +1,103 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ class Task
21
+ include Model
22
+
23
+ def initialize(client, key)
24
+ super(client)
25
+ @key = key
26
+ end
27
+
28
+ attr_reader :key
29
+
30
+ def cancel_request!(options={})
31
+ @client.cancel_request(@key, options)
32
+ end
33
+
34
+ def force_finish!(options={})
35
+ @client.force_finish(@key, options)
36
+ end
37
+
38
+ def metadata(options={})
39
+ @client.get_task_metadata(@key, options)
40
+ end
41
+
42
+ def exists?(options={})
43
+ metadata(options)
44
+ true
45
+ rescue NotFoundError
46
+ false
47
+ end
48
+
49
+ def preempt(options={})
50
+ @client.preempt(@key, options)
51
+ end
52
+
53
+ def inspect
54
+ "#<#{self.class} @key=#{@key.inspect}>"
55
+ end
56
+ end
57
+
58
+ class TaskWithMetadata < Task
59
+ def initialize(client, key, attributes)
60
+ super(client, key)
61
+ @attributes = attributes
62
+ end
63
+
64
+ def inspect
65
+ "#<#{self.class} @key=#{@key.inspect} @attributes=#{@attributes.inspect}>"
66
+ end
67
+
68
+ include TaskMetadataAccessors
69
+ end
70
+
71
+ class AcquiredTask < TaskWithMetadata
72
+ def initialize(client, key, attributes, task_token)
73
+ super(client, key, attributes)
74
+ @task_token = task_token
75
+ end
76
+
77
+ def heartbeat!(options={})
78
+ @client.heartbeat(@task_token, options)
79
+ end
80
+
81
+ def finish!(options={})
82
+ @client.finish(@task_token, options)
83
+ end
84
+
85
+ def release!(options={})
86
+ @client.release(@task_token, options)
87
+ end
88
+
89
+ def retry!(options={})
90
+ @client.retry(@task_token, options)
91
+ end
92
+
93
+ #def to_json
94
+ # [@key, @task_token, @attributes].to_json
95
+ #end
96
+
97
+ #def self.from_json(data, client)
98
+ # key, task_token, attributes = JSON.load(data)
99
+ # new(client, key, attributes, task_token)
100
+ #end
101
+ end
102
+ end
103
+
@@ -0,0 +1,98 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module TaskMetadataAccessors
21
+ attr_reader :attributes
22
+
23
+ def type
24
+ @attributes[:type]
25
+ end
26
+
27
+ def data
28
+ @attributes[:data]
29
+ end
30
+
31
+ def status
32
+ @attributes[:status]
33
+ end
34
+
35
+ def message
36
+ @attributes[:message]
37
+ end
38
+
39
+ def user
40
+ @attributes[:user]
41
+ end
42
+
43
+ def created_at
44
+ if t = @attributes[:created_at]
45
+ return Time.at(t)
46
+ else
47
+ return nil
48
+ end
49
+ end
50
+
51
+ def timeout
52
+ if t = @attributes[:timeout]
53
+ return Time.at(t)
54
+ else
55
+ return nil
56
+ end
57
+ end
58
+
59
+ def finished?
60
+ status == TaskStatus::FINISHED
61
+ end
62
+
63
+ def running?
64
+ status == TaskStatus::RUNNING
65
+ end
66
+
67
+ def waiting?
68
+ status == TaskStatus::WAITING
69
+ end
70
+
71
+ def running?
72
+ status == TaskStatus::RUNNING
73
+ end
74
+
75
+ def cancel_requested?
76
+ status == TaskStatus::CANCEL_REQUESTED
77
+ end
78
+ end
79
+
80
+ class TaskMetadata
81
+ def initialize(client, key, attributes)
82
+ super(client)
83
+ @key = key
84
+ @attributes = attributes
85
+ end
86
+
87
+ def task
88
+ Task.new(@client, @key)
89
+ end
90
+
91
+ def inspect
92
+ "#<#{self.class} @key=#{@key.inspect} @attributes=#{@attributes.inspect}>"
93
+ end
94
+
95
+ include TaskMetadataAccessors
96
+ end
97
+ end
98
+
@@ -0,0 +1,189 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+
21
+ class TaskMonitor
22
+ def initialize(config, child_heartbeat=nil)
23
+ @config = config
24
+ @log = config[:logger]
25
+ @child_heartbeat = child_heartbeat || Proc.new {}
26
+
27
+ @child_heartbeat_interval = (@config[:child_heartbeat_interval] || 2).to_i
28
+ @task_heartbeat_interval = (@config[:task_heartbeat_interval] || 2).to_i
29
+ @last_child_heartbeat = Time.now.to_i
30
+ @last_task_heartbeat = Time.now.to_i
31
+
32
+ @task = nil
33
+
34
+ @mutex = Monitor.new # support recursive lock
35
+ @cond = @mutex.new_cond
36
+ @finished = false
37
+ end
38
+
39
+ def start
40
+ @thread = Thread.new(&method(:run))
41
+ end
42
+
43
+ def stop
44
+ @finished = true
45
+ @mutex.synchronize {
46
+ @cond.broadcast
47
+ }
48
+ end
49
+
50
+ def join
51
+ @thread.join
52
+ end
53
+
54
+ def set_task(task, runner)
55
+ task.extend(TaskMonitorHook)
56
+ task.task_monitor = self
57
+ task.runner = runner
58
+ @mutex.synchronize {
59
+ @task = task
60
+ @last_task_heartbeat = Time.now.to_i
61
+ @heartbeat_message = nil
62
+ }
63
+ end
64
+
65
+ # callback
66
+ def set_heartbeat_message(task, message)
67
+ @mutex.synchronize {
68
+ if task == @task
69
+ @heartbeat_message = message
70
+ end
71
+ }
72
+ end
73
+
74
+ def stop_task(immediate)
75
+ if immediate
76
+ kill_task ImmediateProcessStopError.new('immediate stop requested')
77
+ else
78
+ kill_task GracefulProcessStopError.new('graceful stop requested')
79
+ end
80
+ end
81
+
82
+ def kill_task(reason)
83
+ @mutex.synchronize {
84
+ if task = @task
85
+ begin
86
+ task.runner.kill(reason) # may recursive lock
87
+ rescue
88
+ @log.error "failed to kill task: #{$!.class}: #{$!}"
89
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
90
+ raise # force exit
91
+ end
92
+ end
93
+ }
94
+ end
95
+
96
+ # callback
97
+ def task_finished(task, &block)
98
+ @mutex.synchronize {
99
+ ret = block.call if block
100
+ if task == @task
101
+ @task = nil
102
+ end
103
+ ret
104
+ }
105
+ end
106
+
107
+ def run
108
+ @mutex.synchronize {
109
+ now = Time.now.to_i
110
+
111
+ until @finished
112
+ next_child_heartbeat = @last_child_heartbeat + @child_heartbeat_interval
113
+
114
+ if @task
115
+ next_task_heartbeat = @last_task_heartbeat + @task_heartbeat_interval
116
+ next_time = [next_child_heartbeat, next_task_heartbeat].min
117
+ else
118
+ next_time = next_child_heartbeat
119
+ end
120
+
121
+ next_wait = [1, next_time - now].max
122
+ @cond.wait(next_wait) if next_wait > 0 # TODO timeout doesn't work?
123
+
124
+ now = Time.now.to_i
125
+ if @task && next_task_heartbeat && now <= next_task_heartbeat
126
+ task_heartbeat
127
+ @last_task_heartbeat = now
128
+ end
129
+
130
+ if now <= next_child_heartbeat
131
+ @child_heartbeat.call # will recursive lock
132
+ @last_child_heartbeat = now
133
+ end
134
+ end
135
+ }
136
+ rescue
137
+ @log.error "Unknown error #{$!.class}: #{$!}. Exiting worker pid=#{Process.pid}"
138
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
139
+ end
140
+
141
+ private
142
+ def task_heartbeat
143
+ @task.heartbeat! :message => @heartbeat_message
144
+ @heartbeat_message = nil
145
+ rescue TaskError
146
+ # finished, cancel_requested, preempted, etc.
147
+ kill_task($!)
148
+ end
149
+ end
150
+
151
+ module TaskMonitorHook
152
+ attr_accessor :task_monitor
153
+ attr_accessor :runner
154
+
155
+ def heartbeat_message=(message)
156
+ @heartbeat_message = message
157
+ @task_monitor.set_heartbeat_message(self, message)
158
+ message
159
+ end
160
+
161
+ attr_reader :heartbeat_message
162
+
163
+ def finish!(*args, &block)
164
+ @task_monitor.task_finished(self) {
165
+ super(*args, &block)
166
+ }
167
+ end
168
+
169
+ def release!(*args, &block)
170
+ @task_monitor.task_finished(self) {
171
+ super(*args, &block)
172
+ }
173
+ end
174
+
175
+ def retry!(*args, &block)
176
+ @task_monitor.task_finished(self) {
177
+ super(*args, &block)
178
+ }
179
+ end
180
+
181
+ def cancel_request!(*args, &block)
182
+ @task_monitor.task_finished(self) {
183
+ super(*args, &block)
184
+ }
185
+ end
186
+ end
187
+
188
+ end
189
+