perfectqueue 0.7.32 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/.gitignore +6 -0
  2. data/ChangeLog +0 -62
  3. data/Gemfile +3 -0
  4. data/README.md +239 -0
  5. data/Rakefile +19 -0
  6. data/lib/perfectqueue.rb +68 -4
  7. data/lib/perfectqueue/application.rb +30 -0
  8. data/lib/perfectqueue/application/base.rb +27 -0
  9. data/lib/perfectqueue/application/dispatch.rb +73 -0
  10. data/lib/perfectqueue/application/router.rb +69 -0
  11. data/lib/perfectqueue/backend.rb +44 -47
  12. data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
  13. data/lib/perfectqueue/blocking_flag.rb +84 -0
  14. data/lib/perfectqueue/client.rb +117 -0
  15. data/lib/perfectqueue/command/perfectqueue.rb +108 -323
  16. data/lib/perfectqueue/daemons_logger.rb +80 -0
  17. data/lib/perfectqueue/engine.rb +85 -123
  18. data/lib/perfectqueue/error.rb +53 -0
  19. data/lib/perfectqueue/model.rb +37 -0
  20. data/lib/perfectqueue/multiprocess.rb +31 -0
  21. data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
  22. data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
  23. data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
  24. data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
  25. data/lib/perfectqueue/queue.rb +58 -0
  26. data/lib/perfectqueue/runner.rb +39 -0
  27. data/lib/perfectqueue/signal_queue.rb +112 -0
  28. data/lib/perfectqueue/task.rb +103 -0
  29. data/lib/perfectqueue/task_metadata.rb +98 -0
  30. data/lib/perfectqueue/task_monitor.rb +189 -0
  31. data/lib/perfectqueue/task_status.rb +27 -0
  32. data/lib/perfectqueue/version.rb +1 -3
  33. data/lib/perfectqueue/worker.rb +114 -196
  34. data/perfectqueue.gemspec +24 -0
  35. data/spec/queue_spec.rb +234 -0
  36. data/spec/spec_helper.rb +44 -0
  37. data/spec/worker_spec.rb +81 -0
  38. metadata +93 -40
  39. checksums.yaml +0 -7
  40. data/README.rdoc +0 -224
  41. data/lib/perfectqueue/backend/null.rb +0 -33
  42. data/lib/perfectqueue/backend/rdb.rb +0 -181
  43. data/lib/perfectqueue/backend/simpledb.rb +0 -139
  44. data/test/backend_test.rb +0 -259
  45. data/test/cat.sh +0 -2
  46. data/test/echo.sh +0 -4
  47. data/test/exec_test.rb +0 -61
  48. data/test/fail.sh +0 -2
  49. data/test/huge.sh +0 -2
  50. data/test/stress.rb +0 -99
  51. data/test/success.sh +0 -2
  52. data/test/test_helper.rb +0 -19
@@ -0,0 +1,80 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+
21
+ require 'logger'
22
+
23
+ class DaemonsLogger < Logger
24
+ def initialize(dev, shift_age=0, shift_size=1048576)
25
+ @stdout_hook = false
26
+ @stderr_hook = false
27
+ if dev.is_a?(String)
28
+ @path = dev
29
+ @io = File.open(dev, File::WRONLY|File::APPEND)
30
+ else
31
+ @io = dev
32
+ end
33
+ super(@io, shift_size, shift_size)
34
+ end
35
+
36
+ def hook_stdout!
37
+ return nil if @io == STDOUT
38
+ STDOUT.reopen(@io)
39
+ @stdout_hook = true
40
+ self
41
+ end
42
+
43
+ def hook_stderr!
44
+ STDERR.reopen(@io)
45
+ @stderr_hook = true
46
+ self
47
+ end
48
+
49
+ def reopen!
50
+ if @path
51
+ @io.reopen(@path)
52
+ if @stdout_hook
53
+ STDOUT.reopen(@io)
54
+ end
55
+ if @stderr_hook
56
+ STDERR.reopen(@io)
57
+ end
58
+ end
59
+ nil
60
+ end
61
+
62
+ def reopen
63
+ begin
64
+ reopen!
65
+ return true
66
+ rescue
67
+ # TODO log?
68
+ return false
69
+ end
70
+ end
71
+
72
+ def close
73
+ if @path
74
+ @io.close unless @io.closed?
75
+ end
76
+ nil
77
+ end
78
+ end
79
+
80
+ end
@@ -1,152 +1,114 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
1
18
 
2
19
  module PerfectQueue
3
20
 
21
+ class Engine
22
+ def initialize(runner, config)
23
+ @runner = runner
4
24
 
5
- class Engine
6
- def initialize(backend, log, conf)
7
- @backend = backend
8
- @log = log
25
+ @finish_flag = BlockingFlag.new
9
26
 
10
- @timeout = conf[:timeout]
11
- @poll_interval = conf[:poll_interval] || 1
12
- @expire = conf[:expire] || 345600
27
+ processor_type = config[:processor_type] || :process
28
+ case processor_type.to_sym
29
+ when :process
30
+ @processor_class = Multiprocess::ForkProcessor
31
+ when :thread
32
+ @processor_class = Multiprocess::ThreadProcessor
33
+ else
34
+ raise ConfigError, "Unknown processor_type: #{config[:processor_type].inspect}"
35
+ end
13
36
 
14
- num_workers = conf[:workers] || 1
15
- @workers = (1..num_workers).map {
16
- Worker.new(self, conf)
17
- }
18
- @available_workers = @workers.dup
37
+ @processors = []
38
+ restart(false, config)
39
+ end
19
40
 
20
- @finished = false
21
- @error = nil
41
+ def restart(immediate, config)
42
+ return nil if @finish_flag.set?
22
43
 
23
- @mutex = Mutex.new
24
- @cond = ConditionVariable.new
25
- end
44
+ # TODO connection check
26
45
 
27
- attr_reader :backend
28
- attr_reader :log
29
- attr_reader :error
46
+ @log = config[:logger] || Logger.new(STDERR)
30
47
 
31
- def finished?
32
- @finished
33
- end
48
+ num_processors = config[:processors] || 1
34
49
 
35
- def run
36
- @workers.each {|w|
37
- w.start
38
- }
39
-
40
- until finished?
41
- w = acquire_worker
42
- next unless w
43
- begin
44
-
45
- until finished?
46
- now = Time.now.to_i
47
- token, task = @backend.acquire(now+@timeout)
48
-
49
- unless token
50
- sleep @poll_interval
51
- next
52
- end
53
- if task.created_at < now-@expire
54
- @log.warn "canceling expired task id=#{task.id}"
55
- @backend.cancel(token)
56
- next
57
- end
58
-
59
- @log.info "acquired task id=#{task.id}"
60
- w.submit(token, task)
61
- w = nil
62
- break
50
+ # scaling
51
+ extra = num_processors - @processors.length
52
+ if extra > 0
53
+ extra.times do
54
+ @processors << @processor_class.new(@runner, config)
63
55
  end
64
-
65
- ensure
66
- release_worker(w) if w
56
+ elsif extra < 0
57
+ -extra.times do
58
+ c = @processors.shift
59
+ c.stop(immediate)
60
+ c.join
61
+ end
62
+ extra = 0
67
63
  end
68
- end
69
- ensure
70
- @finished = true
71
- end
72
64
 
73
- def stop(err=nil)
74
- @finished = true
75
- @error = error
76
- @workers.each {|w|
77
- w.stop
78
- }
79
-
80
- if err
81
- @log.error "#{err.class}: #{err}"
82
- err.backtrace.each {|x|
83
- @log.error " #{x}"
65
+ @processors[0..(-extra-1)].each {|c|
66
+ c.restart(immediate, config)
84
67
  }
85
- end
86
- end
87
68
 
88
- def shutdown
89
- @finished = true
90
- @workers.each {|w|
91
- w.shutdown
92
- }
93
- end
69
+ @child_keepalive_interval = (config[:child_keepalive_interval] || config[:child_heartbeat_interval] || 2).to_i
94
70
 
95
- def acquire_worker
96
- @mutex.synchronize {
97
- while @available_workers.empty?
98
- return nil if finished?
99
- @cond.wait(@mutex)
100
- end
101
- return @available_workers.pop
102
- }
103
- end
71
+ self
72
+ end
104
73
 
105
- def release_worker(worker)
106
- @mutex.synchronize {
107
- @available_workers.push worker
108
- if @available_workers.size == 1
109
- @cond.broadcast
74
+ def run
75
+ until @finish_flag.set?
76
+ @processors.each {|c| c.keepalive }
77
+ @finish_flag.wait(@child_keepalive_interval)
110
78
  end
111
- }
112
- end
113
- end
79
+ join
80
+ end
114
81
 
82
+ def stop(immediate)
83
+ @processors.each {|c| c.stop(immediate) }
84
+ @finish_flag.set!
85
+ self
86
+ end
115
87
 
116
- class ExecRunner
117
- def initialize(cmd, task)
118
- @cmd = cmd
119
- @task = task
120
- @iobuf = ''
121
- @pid = nil
122
- @kill_signal = :TERM
123
- end
88
+ def join
89
+ @processors.each {|c| c.join }
90
+ self
91
+ end
124
92
 
125
- def run
126
- cmdline = "#{@cmd} #{Shellwords.escape(@task.id)}"
127
- IO.popen(cmdline, "r+") {|io|
128
- @pid = io.pid
129
- io.write(@task.data) rescue nil
130
- io.close_write
131
- begin
132
- while true
133
- io.sysread(1024, @iobuf)
134
- print @iobuf
135
- end
136
- rescue EOFError
93
+ def shutdown(immediate)
94
+ stop(immediate)
95
+ join
96
+ end
97
+
98
+ def replace(immediate, command=[$0]+ARGV)
99
+ return if @replaced_pid
100
+ stop(immediate)
101
+ @replaced_pid = Process.fork do
102
+ exec(*command)
103
+ exit!(127)
137
104
  end
138
- }
139
- if $?.to_i != 0
140
- raise "Command failed"
105
+ self
141
106
  end
142
- end
143
107
 
144
- def kill
145
- Process.kill(@kill_signal, @pid)
146
- @kill_signal = :KILL
108
+ def logrotated
109
+ @processors.each {|c| c.logrotated }
110
+ end
147
111
  end
148
- end
149
-
150
112
 
151
113
  end
152
114
 
@@ -0,0 +1,53 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ class TaskError < StandardError
21
+ end
22
+
23
+ class CancelRequestedError < TaskError
24
+ end
25
+
26
+ class AlreadyFinishedError < TaskError
27
+ end
28
+
29
+ class NotFoundError < TaskError
30
+ end
31
+
32
+ class AlreadyExistsError < TaskError
33
+ end
34
+
35
+ class PreemptedError < TaskError
36
+ end
37
+
38
+ class NotSupportedError < TaskError
39
+ end
40
+
41
+ class ConfigError < RuntimeError
42
+ end
43
+
44
+ class ProcessStopError < RuntimeError
45
+ end
46
+
47
+ class ImmediateProcessStopError < ProcessStopError
48
+ end
49
+
50
+ class GracefulProcessStopError < ProcessStopError
51
+ end
52
+ end
53
+
@@ -0,0 +1,37 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module Model
21
+ def initialize(client)
22
+ @client = client
23
+ end
24
+
25
+ attr_reader :client
26
+
27
+ def config
28
+ @client.config
29
+ end
30
+
31
+ ## TODO
32
+ #def inspect
33
+ # "<#{self.class}>"
34
+ #end
35
+ end
36
+ end
37
+
@@ -0,0 +1,31 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module Multiprocess
21
+ {
22
+ :ChildProcess => 'multiprocess/child_process',
23
+ :ChildProcessMonitor => 'multiprocess/child_process_monitor',
24
+ :ForkProcessor => 'multiprocess/fork_processor',
25
+ :ThreadProcessor => 'multiprocess/thread_processor',
26
+ }.each_pair {|k,v|
27
+ autoload k, File.expand_path(v, File.dirname(__FILE__))
28
+ }
29
+ end
30
+ end
31
+
@@ -0,0 +1,108 @@
1
+ #
2
+ # PerfectQueue
3
+ #
4
+ # Copyright (C) 2012 FURUHASHI Sadayuki
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ module PerfectQueue
20
+ module Multiprocess
21
+
22
+ class ChildProcess < ThreadProcessor
23
+ def self.run(runner, config, wpipe)
24
+ new(runner, config, wpipe).run
25
+ end
26
+
27
+ def initialize(runner, config, wpipe)
28
+ @wpipe = wpipe
29
+ @wpipe.sync = true
30
+ super(runner, config)
31
+ @sig = install_signal_handlers
32
+ end
33
+
34
+ def run
35
+ super
36
+ @sig.shutdown
37
+ end
38
+
39
+ def stop(immediate)
40
+ @log.info "Exiting worker pid=#{Process.pid}"
41
+ super
42
+ end
43
+
44
+ # override
45
+ def join
46
+ # do nothing
47
+ end
48
+
49
+ # override
50
+ def keepalive
51
+ # do nothing
52
+ end
53
+
54
+ def logrotated
55
+ @log.reopen!
56
+ end
57
+
58
+ def child_heartbeat
59
+ @wpipe.write HEARTBEAT_PACKET
60
+ rescue
61
+ @log.error "Parent process unexpectedly died. Exiting worker pid=#{Process.pid}: #{$!}"
62
+ stop(true)
63
+ Process.kill(:KILL, Process.pid)
64
+ exit! 137
65
+ end
66
+
67
+ HEARTBEAT_PACKET = [0].pack('C')
68
+
69
+ private
70
+ def install_signal_handlers
71
+ SignalQueue.start do |sig|
72
+ sig.trap :TERM do
73
+ stop(false)
74
+ end
75
+ sig.trap :INT do
76
+ stop(false)
77
+ end
78
+
79
+ sig.trap :QUIT do
80
+ stop(true)
81
+ end
82
+
83
+ sig.trap :USR1 do
84
+ stop(false)
85
+ end
86
+
87
+ sig.trap :HUP do
88
+ stop(true)
89
+ end
90
+
91
+ sig.trap :CONT do
92
+ stop(false)
93
+ end
94
+
95
+ sig.trap :WINCH do
96
+ stop(true)
97
+ end
98
+
99
+ sig.trap :USR2 do
100
+ logrotated
101
+ end
102
+ end
103
+ end
104
+ end
105
+
106
+ end
107
+ end
108
+