perfectqueue 0.7.32 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/.gitignore +6 -0
  2. data/ChangeLog +0 -62
  3. data/Gemfile +3 -0
  4. data/README.md +239 -0
  5. data/Rakefile +19 -0
  6. data/lib/perfectqueue.rb +68 -4
  7. data/lib/perfectqueue/application.rb +30 -0
  8. data/lib/perfectqueue/application/base.rb +27 -0
  9. data/lib/perfectqueue/application/dispatch.rb +73 -0
  10. data/lib/perfectqueue/application/router.rb +69 -0
  11. data/lib/perfectqueue/backend.rb +44 -47
  12. data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
  13. data/lib/perfectqueue/blocking_flag.rb +84 -0
  14. data/lib/perfectqueue/client.rb +117 -0
  15. data/lib/perfectqueue/command/perfectqueue.rb +108 -323
  16. data/lib/perfectqueue/daemons_logger.rb +80 -0
  17. data/lib/perfectqueue/engine.rb +85 -123
  18. data/lib/perfectqueue/error.rb +53 -0
  19. data/lib/perfectqueue/model.rb +37 -0
  20. data/lib/perfectqueue/multiprocess.rb +31 -0
  21. data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
  22. data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
  23. data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
  24. data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
  25. data/lib/perfectqueue/queue.rb +58 -0
  26. data/lib/perfectqueue/runner.rb +39 -0
  27. data/lib/perfectqueue/signal_queue.rb +112 -0
  28. data/lib/perfectqueue/task.rb +103 -0
  29. data/lib/perfectqueue/task_metadata.rb +98 -0
  30. data/lib/perfectqueue/task_monitor.rb +189 -0
  31. data/lib/perfectqueue/task_status.rb +27 -0
  32. data/lib/perfectqueue/version.rb +1 -3
  33. data/lib/perfectqueue/worker.rb +114 -196
  34. data/perfectqueue.gemspec +24 -0
  35. data/spec/queue_spec.rb +234 -0
  36. data/spec/spec_helper.rb +44 -0
  37. data/spec/worker_spec.rb +81 -0
  38. metadata +93 -40
  39. checksums.yaml +0 -7
  40. data/README.rdoc +0 -224
  41. data/lib/perfectqueue/backend/null.rb +0 -33
  42. data/lib/perfectqueue/backend/rdb.rb +0 -181
  43. data/lib/perfectqueue/backend/simpledb.rb +0 -139
  44. data/test/backend_test.rb +0 -259
  45. data/test/cat.sh +0 -2
  46. data/test/echo.sh +0 -4
  47. data/test/exec_test.rb +0 -61
  48. data/test/fail.sh +0 -2
  49. data/test/huge.sh +0 -2
  50. data/test/stress.rb +0 -99
  51. data/test/success.sh +0 -2
  52. data/test/test_helper.rb +0 -19
@@ -0,0 +1,109 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module Multiprocess
21
+
22
+ class ChildProcessMonitor
23
+ def initialize(log, pid, rpipe)
24
+ @log = log
25
+ @pid = pid
26
+ @rpipe = rpipe
27
+ @last_heartbeat = Time.now.to_i
28
+
29
+ @kill_start_time = nil
30
+ @last_kill_time = nil
31
+ @kill_immediate = false
32
+
33
+ @rbuf = ''
34
+ end
35
+
36
+ def check_heartbeat(limit)
37
+ @rpipe.read_nonblock(1024, @rbuf)
38
+ @last_heartbeat = Time.now.to_i
39
+ return true
40
+ rescue Errno::EINTR, Errno::EAGAIN
41
+ return Time.now.to_i - @last_heartbeat <= limit
42
+ end
43
+
44
+ def start_killing(immediate)
45
+ if immediate && !@kill_immediate
46
+ @kill_immediate = true # escalation
47
+ elsif @kill_start_time
48
+ return
49
+ end
50
+
51
+ now = Time.now.to_i
52
+ kill_child(now, nil)
53
+ @kill_start_time = now
54
+ end
55
+
56
+ def try_join(kill_interval, graceful_kill_limit)
57
+ return nil unless @kill_start_time
58
+
59
+ begin
60
+ if Process.waitpid(@pid, Process::WNOHANG)
61
+ @log.info "Worker exited pid=#{@pid}"
62
+ return true
63
+ end
64
+ rescue Errno::ECHILD
65
+ # SIGCHLD is trapped in Worker#install_signal_handlers
66
+ @log.info "Worker exited pid=#{@pid}"
67
+ return true
68
+ end
69
+
70
+ # resend signal
71
+ now = Time.now.to_i
72
+ if @last_kill_time + kill_interval <= now
73
+ kill_child(now, graceful_kill_limit)
74
+ end
75
+
76
+ return false
77
+ end
78
+
79
+ def cleanup
80
+ @rpipe.close unless @rpipe.closed?
81
+ end
82
+
83
+ def send_signal(sig)
84
+ begin
85
+ Process.kill(sig, @pid)
86
+ rescue Errno::ESRCH, Errno::EPERM
87
+ # TODO log?
88
+ end
89
+ end
90
+
91
+ private
92
+ def kill_child(now, graceful_kill_limit)
93
+ begin
94
+ if @kill_immediate || (graceful_kill_limit && @kill_start_time + graceful_kill_limit < now)
95
+ @log.debug "sending SIGKILL to pid=#{@pid} for immediate stop"
96
+ Process.kill(:KILL, @pid)
97
+ else
98
+ @log.debug "sending SIGUSR1 to pid=#{@pid} for graceful stop"
99
+ Process.kill(:TERM, @pid)
100
+ end
101
+ rescue Errno::ESRCH, Errno::EPERM
102
+ # TODO log?
103
+ end
104
+ @last_kill_time = now
105
+ end
106
+ end
107
+
108
+ end
109
+ end
@@ -0,0 +1,164 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module Multiprocess
21
+
22
+ class ForkProcessor
23
+ def initialize(runner, config)
24
+ @runner = runner
25
+
26
+ require 'fcntl'
27
+ @stop = false
28
+ @cpm = nil
29
+
30
+ restart(false, config)
31
+ end
32
+
33
+ def restart(immediate, config)
34
+ @child_heartbeat_limit = config[:child_heartbeat_limit] || 10.0
35
+ @child_kill_interval = config[:child_kill_interval] || 2.0
36
+ @child_graceful_kill_limit = config[:child_graceful_kill_limit] || nil
37
+ @log = config[:logger]
38
+ @config = config # for child process
39
+
40
+ if c = @cpm
41
+ c.start_killing(immediate)
42
+ end
43
+ end
44
+
45
+ def stop(immediate)
46
+ @stop = true
47
+ if c = @cpm
48
+ c.start_killing(immediate)
49
+ end
50
+ self
51
+ end
52
+
53
+ def keepalive
54
+ if @stop
55
+ try_join
56
+ return
57
+ end
58
+
59
+ if c = @cpm
60
+ begin
61
+ # receive heartbeat
62
+ keptalive = c.check_heartbeat(@child_heartbeat_limit)
63
+ unless keptalive
64
+ @log.error "Heartbeat broke out. Restarting child process."
65
+ c.start_killing(false)
66
+ end
67
+ rescue EOFError
68
+ @log.error "Heartbeat pipe is closed. Restarting child process."
69
+ c.start_killing(true)
70
+ rescue
71
+ @log.error "Unknown error: #{$!.class}: #{$!}. Restarting child process."
72
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
73
+ c.start_killing(false)
74
+ end
75
+
76
+ try_join
77
+ end
78
+
79
+ unless @cpm
80
+ begin
81
+ @cpm = fork_child
82
+ rescue
83
+ @log.error "Failed to fork child process: #{$!.class}: #{$!}"
84
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
85
+ end
86
+ end
87
+
88
+ nil
89
+ end
90
+
91
+ def join
92
+ while !try_join
93
+ sleep (@child_kill_interval+1) / 2 # TODO
94
+ end
95
+ self
96
+ end
97
+
98
+ def logrotated
99
+ if c = @cpm
100
+ c.send_signal(:CONT)
101
+ end
102
+ end
103
+
104
+ private
105
+ def try_join
106
+ unless @cpm
107
+ return true
108
+ end
109
+ if @cpm.try_join(@child_kill_interval, @child_graceful_kill_limit)
110
+ @cpm.cleanup
111
+ @cpm = nil
112
+ return true
113
+ else
114
+ return false
115
+ end
116
+ end
117
+
118
+ def ensure_fork
119
+ unless @cpm
120
+ @cpm = fork_child
121
+ end
122
+ end
123
+
124
+ INTER_FORK_LOCK = Mutex.new
125
+
126
+ def fork_child
127
+ @runner.before_fork if @runner.respond_to?(:before_fork) # TODO exception handling
128
+
129
+ INTER_FORK_LOCK.lock
130
+ begin
131
+ rpipe, wpipe = IO.pipe
132
+ rpipe.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
133
+ wpipe.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
134
+ ensure
135
+ INTER_FORK_LOCK.unlock
136
+ end
137
+
138
+ pid = fork do
139
+ #STDIN.close
140
+ # pass-through STDOUT
141
+ # pass-through STDERR
142
+ rpipe.close
143
+
144
+ @runner.after_fork if @runner.respond_to?(:after_fork)
145
+
146
+ begin
147
+ ChildProcess.run(@runner, @config, wpipe)
148
+ ensure
149
+ @runner.after_child_end if @runner.respond_to?(:after_child_end) # TODO exception handling
150
+ end
151
+
152
+ exit! 0
153
+ end
154
+
155
+ @log.info "Worker process started. pid=#{pid}"
156
+
157
+ wpipe.close
158
+
159
+ ChildProcessMonitor.new(@log, pid, rpipe)
160
+ end
161
+ end
162
+
163
+ end
164
+ end
@@ -0,0 +1,123 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module Multiprocess
21
+
22
+ class ThreadProcessor
23
+ def initialize(runner, config)
24
+ @runner = runner
25
+
26
+ @running_flag = BlockingFlag.new
27
+ @finish_flag = BlockingFlag.new
28
+
29
+ @tm = TaskMonitor.new(config, method(:child_heartbeat))
30
+
31
+ restart(false, config)
32
+ end
33
+
34
+ def run
35
+ @tm.start
36
+ @running_flag.set_region do
37
+ until @finish_flag.set?
38
+ run_loop
39
+ end
40
+ end
41
+ @tm.join
42
+ ensure
43
+ @thread = nil
44
+ end
45
+
46
+ def join
47
+ while t = @thread
48
+ t.join
49
+ end
50
+ end
51
+
52
+ def keepalive
53
+ unless @thread
54
+ @thread = Thread.new(&method(:run))
55
+ end
56
+ end
57
+
58
+ def restart(immediate, config)
59
+ @poll_interval = config[:poll_interval] || 1.0
60
+ @log = config[:logger]
61
+ @config = config
62
+
63
+ @tm.stop_task(immediate)
64
+
65
+ @finish_flag.set_region do
66
+ @running_flag.wait while @running_flag.set?
67
+ end
68
+ end
69
+
70
+ def stop(immediate)
71
+ @log.info immediate ? "Stopping worker thread immediately" : "Stopping worker thread gracefully"
72
+ @tm.stop_task(immediate)
73
+ @finish_flag.set!
74
+ end
75
+
76
+ def logrotated
77
+ # do nothing
78
+ end
79
+
80
+ private
81
+ def child_heartbeat
82
+ # do nothing
83
+ end
84
+
85
+ def run_loop
86
+ PerfectQueue.open(@config) {|queue|
87
+ until @finish_flag.set?
88
+ task = queue.poll
89
+ if task
90
+ process(task)
91
+ else
92
+ @finish_flag.wait(@poll_interval)
93
+ end
94
+ end
95
+ }
96
+ rescue
97
+ @log.error "Unknown error #{$!.class}: #{$!}. Exiting worker pid=#{Process.pid}"
98
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
99
+ ensure
100
+ @tm.stop
101
+ end
102
+
103
+ def process(task)
104
+ @log.info "acquired task: #{task.inspect}"
105
+ begin
106
+ r = @runner.new(task)
107
+ @tm.set_task(task, r)
108
+ begin
109
+ r.run
110
+ ensure
111
+ @tm.task_finished(task)
112
+ end
113
+ rescue
114
+ @log.error "process failed: #{$!.class}: #{$!}"
115
+ $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
116
+ raise # force exit
117
+ end
118
+ end
119
+ end
120
+
121
+ end
122
+ end
123
+
@@ -0,0 +1,58 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ class Queue
21
+ include Model
22
+
23
+ def initialize(client)
24
+ super(client)
25
+ end
26
+
27
+ def [](key)
28
+ Task.new(@client, key)
29
+ end
30
+
31
+ def each(options={}, &block)
32
+ @client.list(options, &block)
33
+ end
34
+
35
+ include Enumerable
36
+
37
+ def poll(options={})
38
+ options = options.merge({:max_acquire=>1})
39
+ if acquired = poll_multi(options)
40
+ return acquired[0]
41
+ end
42
+ return nil
43
+ end
44
+
45
+ def poll_multi(options={})
46
+ @client.acquire(options)
47
+ end
48
+
49
+ def submit(key, type, data, options={})
50
+ @client.submit(key, type, data, options)
51
+ end
52
+
53
+ def close
54
+ client.close
55
+ end
56
+ end
57
+ end
58
+