servolux 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +10 -0
- data/Rakefile +3 -0
- data/a.rb +31 -0
- data/b.rb +17 -0
- data/examples/beanstalk.rb +65 -0
- data/examples/echo.rb +52 -0
- data/lib/servolux/piper.rb +58 -56
- data/lib/servolux/prefork.rb +351 -0
- data/lib/servolux.rb +2 -3
- data/spec/piper_spec.rb +4 -2
- metadata +19 -4
data/History.txt
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
== 0.8.0 / 2009-11-12
|
|
2
|
+
|
|
3
|
+
* 2 Major Enhancements
|
|
4
|
+
* Preforking worker pool
|
|
5
|
+
* 1 Minor Enhancement
|
|
6
|
+
* Default return value for Piper#gets (useful for detecting timeouts)
|
|
7
|
+
* 2 Bug Fixes
|
|
8
|
+
* The piper is now using a socket pair for bidirectional communication
|
|
9
|
+
* The piper now defaults to a blocking mode (nil timeout)
|
|
10
|
+
|
|
1
11
|
== 0.7.1 / 2009-11-06
|
|
2
12
|
|
|
3
13
|
* 1 Bug Fix
|
data/Rakefile
CHANGED
|
@@ -9,6 +9,7 @@ ensure_in_path 'lib'
|
|
|
9
9
|
require 'servolux'
|
|
10
10
|
|
|
11
11
|
task :default => 'spec:specdoc'
|
|
12
|
+
task 'gem:release' => ['spec:run', 'rubyforge:release']
|
|
12
13
|
|
|
13
14
|
Bones {
|
|
14
15
|
name 'servolux'
|
|
@@ -20,11 +21,13 @@ Bones {
|
|
|
20
21
|
ignore_file '.gitignore'
|
|
21
22
|
|
|
22
23
|
spec.opts << '--color'
|
|
24
|
+
rubyforge.name 'codeforpeople'
|
|
23
25
|
|
|
24
26
|
use_gmail
|
|
25
27
|
enable_sudo
|
|
26
28
|
|
|
27
29
|
depend_on 'bones-extras', :development => true
|
|
30
|
+
depend_on 'bones-git', :development => true
|
|
28
31
|
depend_on 'logging', :development => true
|
|
29
32
|
depend_on 'rspec', :development => true
|
|
30
33
|
}
|
data/a.rb
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
|
|
2
|
+
$LOAD_PATH.unshift 'lib'
|
|
3
|
+
|
|
4
|
+
require 'servolux'
|
|
5
|
+
|
|
6
|
+
STDOUT.sync = true
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
module DoThis
|
|
10
|
+
def before_executing
|
|
11
|
+
@fd = File.open("#$$.txt", 'w')
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def after_executing
|
|
15
|
+
@fd.close
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def execute
|
|
19
|
+
@fd.puts "Process #$$ @ #{Time.now}"
|
|
20
|
+
sleep 5
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
p = Servolux::Prefork.new :module => DoThis, :timeout => 0.5
|
|
25
|
+
p.start 6
|
|
26
|
+
sleep 10
|
|
27
|
+
p.stop
|
|
28
|
+
|
|
29
|
+
p.errors { |e|
|
|
30
|
+
puts e.inspect
|
|
31
|
+
}
|
data/b.rb
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Preforking Beanstalkd job runner using Servolux.
|
|
2
|
+
#
|
|
3
|
+
# In this example, we prefork 7 processes each of which connect to our
|
|
4
|
+
# Beanstalkd queue and then wait for jobs to process. We are using a module so
|
|
5
|
+
# that we can connect to the beanstalk queue before exectuing and then
|
|
6
|
+
# disconnect from the beanstalk queue after exiting. These methods are called
|
|
7
|
+
# exactly once per child process.
|
|
8
|
+
#
|
|
9
|
+
# A variation on this is to load source code in the before_executing method
|
|
10
|
+
# and initialize an object that will process jobs. This is advantagous because
|
|
11
|
+
# now you can send SIGHUP to a child process and it will restart, loading your
|
|
12
|
+
# Ruby libraries before executing. Now you can do a rolling deploy of new
|
|
13
|
+
# code.
|
|
14
|
+
#
|
|
15
|
+
# def before_executing
|
|
16
|
+
# Kernel.load '/your/source/code.rb'
|
|
17
|
+
# @job_runner = Your::Source::Code::JobRunner.new
|
|
18
|
+
# end
|
|
19
|
+
# --------
|
|
20
|
+
|
|
21
|
+
require 'servolux'
|
|
22
|
+
require 'beanstalk-client'
|
|
23
|
+
|
|
24
|
+
module JobProcessor
|
|
25
|
+
# Open a connection to our beanstalk queue
|
|
26
|
+
def before_executing
|
|
27
|
+
@beanstalk = Beanstalk::Pool.new(['localhost:11300'])
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Close the connection to our beanstalk queue
|
|
31
|
+
def after_executing
|
|
32
|
+
@beanstalk.close
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Reserve a job from the beanstalk queue, and processes jobs as we receive
|
|
36
|
+
# them. We have a timeout set for 2 minutes so that we can send a heartbeat
|
|
37
|
+
# back to the parent process even if the beanstalk queue is empty.
|
|
38
|
+
def execute
|
|
39
|
+
job = @beanstalk.reserve(120)
|
|
40
|
+
if job
|
|
41
|
+
# process job here ...
|
|
42
|
+
job.delete
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Create our preforking worker pool. Each worker will run the code found in
|
|
48
|
+
# the JobProcessor module. We set a timeout of 10 minutes. The child process
|
|
49
|
+
# must send a "heartbeat" message to the parent within this timeout period;
|
|
50
|
+
# otherwise, the parent will halt the child process.
|
|
51
|
+
#
|
|
52
|
+
# Our execute code in the JobProcessor takes this into account. It will wakeup
|
|
53
|
+
# every 2 minutes, if no jobs are reserved from the beanstalk queue, and send
|
|
54
|
+
# the heartbeat message.
|
|
55
|
+
#
|
|
56
|
+
# This also means that if any job processed by a worker takes longer than 10
|
|
57
|
+
# minutes to run, that child worker will be killed.
|
|
58
|
+
pool = Servolux::Prefork.new(:timeout => 600, :module => JobProcessor)
|
|
59
|
+
|
|
60
|
+
# Start up 7 child processes to handle jobs
|
|
61
|
+
pool.start 7
|
|
62
|
+
|
|
63
|
+
# Stop when SIGINT is received.
|
|
64
|
+
trap('INT') { pool.stop }
|
|
65
|
+
Process.waitall
|
data/examples/echo.rb
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Pre-forking echo server using Servolux
|
|
2
|
+
#
|
|
3
|
+
# Run this code using "ruby echo.rb"
|
|
4
|
+
#
|
|
5
|
+
# You can test the server using NetCat from a separate terminal window.
|
|
6
|
+
#
|
|
7
|
+
# echo "hello world" | nc localhost 4242
|
|
8
|
+
#
|
|
9
|
+
# This example was stolen from Ryan Tomayko and modified to demonstrate the
|
|
10
|
+
# Servolux gem. The original can be found here:
|
|
11
|
+
#
|
|
12
|
+
# http://tomayko.com/writings/unicorn-is-unix
|
|
13
|
+
# --------
|
|
14
|
+
|
|
15
|
+
require 'servolux'
|
|
16
|
+
|
|
17
|
+
# Create a socket, bind it to localhost:4242, and start listening.
|
|
18
|
+
# Runs once in the parent; all forked children inherit the socket's
|
|
19
|
+
# file descriptor.
|
|
20
|
+
acceptor = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
|
|
21
|
+
address = Socket.pack_sockaddr_in(4242, '0.0.0.0')
|
|
22
|
+
acceptor.bind(address)
|
|
23
|
+
acceptor.listen(10)
|
|
24
|
+
|
|
25
|
+
# Close the socket when we exit the parent or any child process. This
|
|
26
|
+
# only closes the file descriptor in the calling process, it does not
|
|
27
|
+
# take the socket out of the listening state (until the last fd is
|
|
28
|
+
# closed).
|
|
29
|
+
#
|
|
30
|
+
# The trap is guaranteed to happen, and guaranteed to happen only
|
|
31
|
+
# once, right before the process exits for any reason (unless
|
|
32
|
+
# it's terminated with a SIGKILL).
|
|
33
|
+
trap('EXIT') { acceptor.close }
|
|
34
|
+
|
|
35
|
+
# Create the worker pool passing in the code to execute in each child
|
|
36
|
+
# process.
|
|
37
|
+
pool = Servolux::Prefork.new {
|
|
38
|
+
socket, addr = acceptor.accept
|
|
39
|
+
socket.write "child #$$ echo> "
|
|
40
|
+
socket.flush
|
|
41
|
+
message = socket.gets
|
|
42
|
+
socket.write message
|
|
43
|
+
socket.close
|
|
44
|
+
puts "child #$$ echo'd: '#{message.strip}'"
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# Start up 3 child process to handle echo requests on the socket.
|
|
48
|
+
pool.start 3
|
|
49
|
+
|
|
50
|
+
# Stop the child processes when SIGINT is received.
|
|
51
|
+
trap('INT') { pool.stop }
|
|
52
|
+
Process.waitall
|
data/lib/servolux/piper.rb
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
|
|
2
|
+
require 'socket'
|
|
3
|
+
|
|
2
4
|
# == Synopsis
|
|
3
5
|
# A Piper is used to fork a child proces and then establish a communication
|
|
4
6
|
# pipe between the parent and child. This communication pipe is used to pass
|
|
@@ -47,7 +49,7 @@
|
|
|
47
49
|
class Servolux::Piper
|
|
48
50
|
|
|
49
51
|
# :stopdoc:
|
|
50
|
-
|
|
52
|
+
SIZEOF_INT = [42].pack('I').size
|
|
51
53
|
# :startdoc:
|
|
52
54
|
|
|
53
55
|
# call-seq:
|
|
@@ -90,12 +92,6 @@ class Servolux::Piper
|
|
|
90
92
|
# The timeout in seconds to wait for puts / gets commands.
|
|
91
93
|
attr_accessor :timeout
|
|
92
94
|
|
|
93
|
-
# The read end of the pipe.
|
|
94
|
-
attr_reader :read_io
|
|
95
|
-
|
|
96
|
-
# The write end of the pipe.
|
|
97
|
-
attr_reader :write_io
|
|
98
|
-
|
|
99
95
|
# call-seq:
|
|
100
96
|
# Piper.new( mode = 'r', opts = {} )
|
|
101
97
|
#
|
|
@@ -111,7 +107,10 @@ class Servolux::Piper
|
|
|
111
107
|
# rw read-write read-write
|
|
112
108
|
#
|
|
113
109
|
# The communication timeout can be provided as an option. This is the
|
|
114
|
-
# number of seconds to wait for a +puts+ or +gets+ to succeed.
|
|
110
|
+
# number of seconds to wait for a +puts+ or +gets+ to succeed. This timeout
|
|
111
|
+
# will default to +nil+ such that calls through the pipe will block forever
|
|
112
|
+
# until data is available. You can configure the +puts+ and +gets+ to be
|
|
113
|
+
# non-blocking by setting the timeout to +0+.
|
|
115
114
|
#
|
|
116
115
|
def initialize( *args )
|
|
117
116
|
opts = args.last.is_a?(Hash) ? args.pop : {}
|
|
@@ -121,55 +120,42 @@ class Servolux::Piper
|
|
|
121
120
|
raise ArgumentError, "Unsupported mode #{mode.inspect}"
|
|
122
121
|
end
|
|
123
122
|
|
|
124
|
-
@timeout = opts[:timeout]
|
|
125
|
-
|
|
126
|
-
@read_io, @write_io = IO.pipe('ASCII-8BIT') # encoding for Ruby 1.9
|
|
127
|
-
else
|
|
128
|
-
@read_io, @write_io = IO.pipe
|
|
129
|
-
end
|
|
123
|
+
@timeout = opts.key?(:timeout) ? opts[:timeout] : nil
|
|
124
|
+
socket_pair = Socket.pair(Socket::AF_UNIX, Socket::SOCK_STREAM, 0)
|
|
130
125
|
@child_pid = Kernel.fork
|
|
131
126
|
|
|
132
127
|
if child?
|
|
128
|
+
@socket = socket_pair[1]
|
|
129
|
+
socket_pair[0].close
|
|
130
|
+
|
|
133
131
|
case mode
|
|
134
|
-
when 'r'; close_read
|
|
135
|
-
when 'w'; close_write
|
|
132
|
+
when 'r'; @socket.close_read
|
|
133
|
+
when 'w'; @socket.close_write
|
|
136
134
|
end
|
|
137
135
|
else
|
|
136
|
+
@socket = socket_pair[0]
|
|
137
|
+
socket_pair[1].close
|
|
138
|
+
|
|
138
139
|
case mode
|
|
139
|
-
when 'r'; close_write
|
|
140
|
-
when 'w'; close_read
|
|
140
|
+
when 'r'; @socket.close_write
|
|
141
|
+
when 'w'; @socket.close_read
|
|
141
142
|
end
|
|
142
143
|
end
|
|
143
144
|
end
|
|
144
145
|
|
|
145
|
-
# Close both the
|
|
146
|
-
#
|
|
146
|
+
# Close both the communications socket. This only affects the process from
|
|
147
|
+
# which it was called -- the parent or the child.
|
|
147
148
|
#
|
|
148
149
|
def close
|
|
149
|
-
@
|
|
150
|
-
@write_io.close rescue nil
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
# Close the read end of the communications pipe. This only affects the
|
|
154
|
-
# process from which it was called -- the parent or the child.
|
|
155
|
-
#
|
|
156
|
-
def close_read
|
|
157
|
-
@read_io.close rescue nil
|
|
158
|
-
end
|
|
159
|
-
|
|
160
|
-
# Close the write end of the communications pipe. This only affects the
|
|
161
|
-
# process from which it was called -- the parent or the child.
|
|
162
|
-
#
|
|
163
|
-
def close_write
|
|
164
|
-
@write_io.close rescue nil
|
|
150
|
+
@socket.close rescue nil
|
|
165
151
|
end
|
|
166
152
|
|
|
167
153
|
# Returns +true+ if the communications pipe is readable from the process
|
|
168
154
|
# and there is data waiting to be read.
|
|
169
155
|
#
|
|
170
156
|
def readable?
|
|
171
|
-
return false if @
|
|
172
|
-
r,w,e = Kernel.select([@
|
|
157
|
+
return false if @socket.closed?
|
|
158
|
+
r,w,e = Kernel.select([@socket], nil, nil, @timeout)
|
|
173
159
|
return !(r.nil? or r.empty?)
|
|
174
160
|
end
|
|
175
161
|
|
|
@@ -177,8 +163,8 @@ class Servolux::Piper
|
|
|
177
163
|
# and the write buffer can accept more data.
|
|
178
164
|
#
|
|
179
165
|
def writeable?
|
|
180
|
-
return false if @
|
|
181
|
-
r,w,e = Kernel.select(nil, [@
|
|
166
|
+
return false if @socket.closed?
|
|
167
|
+
r,w,e = Kernel.select(nil, [@socket], nil, @timeout)
|
|
182
168
|
return !(w.nil? or w.empty?)
|
|
183
169
|
end
|
|
184
170
|
|
|
@@ -237,22 +223,27 @@ class Servolux::Piper
|
|
|
237
223
|
@child_pid
|
|
238
224
|
end
|
|
239
225
|
|
|
240
|
-
# Read an object from the communication pipe.
|
|
241
|
-
#
|
|
242
|
-
#
|
|
243
|
-
#
|
|
226
|
+
# Read an object from the communication pipe. If data is available then it
|
|
227
|
+
# is un-marshalled and returned as a Ruby object. If the pipe is closed for
|
|
228
|
+
# reading or if no data is available then the _default_ value is returned.
|
|
229
|
+
# You can pass in the _default_ value; otherwise it will be +nil+.
|
|
244
230
|
#
|
|
245
231
|
# This method will block until the +timeout+ is reached or data can be
|
|
246
232
|
# read from the pipe.
|
|
247
233
|
#
|
|
248
|
-
def gets
|
|
249
|
-
return unless readable?
|
|
234
|
+
def gets( default = nil )
|
|
235
|
+
return default unless readable?
|
|
250
236
|
|
|
251
|
-
data = @
|
|
252
|
-
return if data.nil?
|
|
237
|
+
data = @socket.read SIZEOF_INT
|
|
238
|
+
return default if data.nil?
|
|
239
|
+
|
|
240
|
+
size = data.unpack('I').first
|
|
241
|
+
data = @socket.read size
|
|
242
|
+
return default if data.nil?
|
|
253
243
|
|
|
254
|
-
data.chomp! SEPERATOR
|
|
255
244
|
Marshal.load(data) rescue data
|
|
245
|
+
rescue SystemCallError
|
|
246
|
+
return default
|
|
256
247
|
end
|
|
257
248
|
|
|
258
249
|
# Write an object to the communication pipe. Returns +nil+ if the pipe is
|
|
@@ -267,16 +258,15 @@ class Servolux::Piper
|
|
|
267
258
|
def puts( obj )
|
|
268
259
|
return unless writeable?
|
|
269
260
|
|
|
270
|
-
|
|
271
|
-
@
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
bytes
|
|
261
|
+
data = Marshal.dump(obj)
|
|
262
|
+
@socket.write([data.size].pack('I')) + @socket.write(data)
|
|
263
|
+
rescue SystemCallError
|
|
264
|
+
return nil
|
|
275
265
|
end
|
|
276
266
|
|
|
277
267
|
# Send the given signal to the child process. The signal may be an integer
|
|
278
268
|
# signal number or a POSIX signal name (either with or without a +SIG+
|
|
279
|
-
# prefix).
|
|
269
|
+
# prefix).
|
|
280
270
|
#
|
|
281
271
|
# This method does nothing when called from the child process.
|
|
282
272
|
#
|
|
@@ -286,6 +276,18 @@ class Servolux::Piper
|
|
|
286
276
|
Process.kill(sig, @child_pid)
|
|
287
277
|
end
|
|
288
278
|
|
|
279
|
+
# Returns +true+ if the child process is alive. Returns +nil+ if the child
|
|
280
|
+
# process has not been started.
|
|
281
|
+
#
|
|
282
|
+
# Always returns +nil+ when called from the child process.
|
|
283
|
+
#
|
|
284
|
+
def alive?
|
|
285
|
+
return if @child_pid.nil?
|
|
286
|
+
Process.kill(0, @child_pid)
|
|
287
|
+
true
|
|
288
|
+
rescue Errno::ESRCH, Errno::ENOENT
|
|
289
|
+
false
|
|
290
|
+
end
|
|
291
|
+
|
|
289
292
|
end # class Servolux::Piper
|
|
290
293
|
|
|
291
|
-
# EOF
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
|
|
2
|
+
# == Synopsis
|
|
3
|
+
# The Prefork class provides a pre-forking worker pool for executing tasks in
|
|
4
|
+
# parallel using multiple processes.
|
|
5
|
+
#
|
|
6
|
+
# == Details
|
|
7
|
+
# A pre-forking worker pool is a technique for executing code in parallel in a
|
|
8
|
+
# UNIX environment. Each worker in the pool forks a child process and then
|
|
9
|
+
# executes user supplied code in that child process. The child process can
|
|
10
|
+
# pull jobs from a queue (beanstalkd for example) or listen on a socket for
|
|
11
|
+
# network requests.
|
|
12
|
+
#
|
|
13
|
+
# The code to execute in the child processes is passed as a block to the
|
|
14
|
+
# Prefork initialize method. The child processes executes this code in a loop;
|
|
15
|
+
# that is, your code block should not worry about keeping itself alive. This
|
|
16
|
+
# is handled by the library.
|
|
17
|
+
#
|
|
18
|
+
# If your code raises an exception, it will be captured by the library code
|
|
19
|
+
# and marshalled back to the parent process. This will halt the child process.
|
|
20
|
+
# The Prefork worker pool does not restart dead workers. A method is provided
|
|
21
|
+
# to iterate over workers that have errors, and it is up to the user to handle
|
|
22
|
+
# errors as they please.
|
|
23
|
+
#
|
|
24
|
+
# Instead of passing a block to the initialize method, you can provide a Ruby
|
|
25
|
+
# module that defines an "execute" method. This method will be executed in the
|
|
26
|
+
# child process' run loop. When using a module, you also have the option of
|
|
27
|
+
# defining a "before_executing" method and an "after_executing" method. These
|
|
28
|
+
# methods will be called before the child starts the execute loop and after
|
|
29
|
+
# the execute loop finishes. Each method will be called exactly once. Both
|
|
30
|
+
# methods are optional.
|
|
31
|
+
#
|
|
32
|
+
# Sending a SIGHUP to a child process will cause that child to stop and
|
|
33
|
+
# restart. The child will send a signal to the parent asking to be shutdown.
|
|
34
|
+
# The parent will gracefully halt the child and then start a new child process
|
|
35
|
+
# to replace it.
|
|
36
|
+
#
|
|
37
|
+
# This has the advantage of calling your before/after_executing methods again
|
|
38
|
+
# and reloading any code or resources your worker code will use. The SIGHUP
|
|
39
|
+
# will call Thread#wakeup on the main child process thread; please write your
|
|
40
|
+
# code to respond accordingly to this wakeup call (a thread waiting on a
|
|
41
|
+
# Queue#pop will not return when wakeup is called on the thread).
|
|
42
|
+
#
|
|
43
|
+
# == Examples
|
|
44
|
+
#
|
|
45
|
+
# A pre-forking echo server: http://github.com/TwP/servolux/blob/master/examples/echo.rb
|
|
46
|
+
#
|
|
47
|
+
# Pulling jobs from a beanstalkd work queue: http://github.com/TwP/servolux/blob/master/examples/beanstalk.rb
|
|
48
|
+
#
|
|
49
|
+
# ==== Before / After Executing
|
|
50
|
+
# In this example, we are creating 42 worker processes that will log the
|
|
51
|
+
# process ID and the current time to a file. Each worker will do this every 2
|
|
52
|
+
# seconds. The before/after_executing methods are used to open the file before
|
|
53
|
+
# the run loop starts and to close the file after the run loop completes. The
|
|
54
|
+
# execute method uses the stored file descriptor when logging the message.
|
|
55
|
+
#
|
|
56
|
+
# module RunMe
|
|
57
|
+
# def before_executing
|
|
58
|
+
# @fd = File.open("#{Process.pid}.txt", 'w')
|
|
59
|
+
# end
|
|
60
|
+
#
|
|
61
|
+
# def after_executing
|
|
62
|
+
# @fd.close
|
|
63
|
+
# end
|
|
64
|
+
#
|
|
65
|
+
# def execute
|
|
66
|
+
# @fd.puts "Process #{Process.pid} @ #{Time.now}"
|
|
67
|
+
# sleep 2
|
|
68
|
+
# end
|
|
69
|
+
# end
|
|
70
|
+
#
|
|
71
|
+
# pool = Servolux::Prefork.new(:module => RunMe)
|
|
72
|
+
# pool.start 42
|
|
73
|
+
#
|
|
74
|
+
# ==== Heartbeat
|
|
75
|
+
# When a :timeout is supplied to the constructor, a "heartbeat" is setup
|
|
76
|
+
# between the parent and the child worker. Each loop through the child's
|
|
77
|
+
# execute code must return before :timeout seconds have elapsed. If one
|
|
78
|
+
# iteration through the loop takes longer than :timeout seconds, then the
|
|
79
|
+
# parent process will halt the child worker. An error will be raised in the
|
|
80
|
+
# parent process.
|
|
81
|
+
#
|
|
82
|
+
# pool = Servolux::Prefork.new(:timeout => 2) {
|
|
83
|
+
# puts "Process #{Process.pid} is running."
|
|
84
|
+
# sleep(rand * 5)
|
|
85
|
+
# }
|
|
86
|
+
# pool.start 42
|
|
87
|
+
#
|
|
88
|
+
# Eventually all 42 child processes will be killed by their parents. The
|
|
89
|
+
# random number generator will eventually cause the child to sleep longer than
|
|
90
|
+
# two seconds.
|
|
91
|
+
#
|
|
92
|
+
# What is happening here is that each time the child processes executes the
|
|
93
|
+
# block of code, the Servolux library code will send a "heartbeat" message to
|
|
94
|
+
# the parent. The parent is using a Kernel#select call on the communications
|
|
95
|
+
# pipe to wait for this message. The timeout is passed to the select call, and
|
|
96
|
+
# this will cause it to return +nil+ -- this is the error condition the
|
|
97
|
+
# heartbeat prevents.
|
|
98
|
+
#
|
|
99
|
+
# Use the heartbeat with caution -- allow margins for timing issues and
|
|
100
|
+
# processor load spikes.
|
|
101
|
+
#
|
|
102
|
+
class Servolux::Prefork
|
|
103
|
+
|
|
104
|
+
Timeout = Class.new(::Servolux::Error)
|
|
105
|
+
UnknownSignal = Class.new(::Servolux::Error)
|
|
106
|
+
UnknownResponse = Class.new(::Servolux::Error)
|
|
107
|
+
|
|
108
|
+
# :stopdoc:
|
|
109
|
+
START = "\000START".freeze
|
|
110
|
+
HALT = "\000HALT".freeze
|
|
111
|
+
ERROR = "\000SHIT".freeze
|
|
112
|
+
HEARTBEAT = "\000<3".freeze
|
|
113
|
+
# :startdoc:
|
|
114
|
+
|
|
115
|
+
attr_accessor :timeout # Communication timeout in seconds.
|
|
116
|
+
|
|
117
|
+
# call-seq:
|
|
118
|
+
# Prefork.new { block }
|
|
119
|
+
# Prefork.new( :module => Module )
|
|
120
|
+
#
|
|
121
|
+
# Create a new pre-forking worker pool. You must provide a block of code for
|
|
122
|
+
# the workers to execute in their child processes. This code block can be
|
|
123
|
+
# passed either as a block to this method or as a module via the :module
|
|
124
|
+
# option.
|
|
125
|
+
#
|
|
126
|
+
# If a :timeout is given, then each worker will setup a "heartbeat" between
|
|
127
|
+
# the parent process and the child process. If the child does not respond to
|
|
128
|
+
# the parent within :timeout seconds, then the child process will be halted.
|
|
129
|
+
# If you do not want to use the heartbeat then leave the :timeout unset or
|
|
130
|
+
# manually set it to +nil+.
|
|
131
|
+
#
|
|
132
|
+
# The pre-forking worker pool makes no effort to restart dead workers. It is
|
|
133
|
+
# left to the user to implement this functionality.
|
|
134
|
+
#
|
|
135
|
+
def initialize( opts = {}, &block )
|
|
136
|
+
@timeout = opts[:timeout]
|
|
137
|
+
@module = opts[:module]
|
|
138
|
+
@module = Module.new { define_method :execute, &block } if block
|
|
139
|
+
@workers = []
|
|
140
|
+
|
|
141
|
+
raise ArgumentError, 'No code was given to execute by the workers.' unless @module
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Start up the given _number_ of workers. Each worker will create a child
|
|
145
|
+
# process and run the user supplied code in that child process.
|
|
146
|
+
#
|
|
147
|
+
def start( number )
|
|
148
|
+
@workers.clear
|
|
149
|
+
|
|
150
|
+
number.times {
|
|
151
|
+
@workers << Worker.new(self)
|
|
152
|
+
@workers.last.extend @module
|
|
153
|
+
}
|
|
154
|
+
@workers.each { |worker| worker.start }
|
|
155
|
+
self
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Stop all workers. The current process will wait for each child process to
|
|
159
|
+
# exit before this method will return. The worker instances are not
|
|
160
|
+
# destroyed by this method; this means that the +each_worker+ and the
|
|
161
|
+
# +errors+ methods will still function correctly after stopping the workers.
|
|
162
|
+
#
|
|
163
|
+
def stop
|
|
164
|
+
@workers.each { |worker| worker.stop }
|
|
165
|
+
@workers.each { |worker| worker.wait }
|
|
166
|
+
self
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# call-seq:
|
|
170
|
+
# each_worker { |worker| block }
|
|
171
|
+
#
|
|
172
|
+
# Iterates over all the works and yields each, in turn, to the given
|
|
173
|
+
# _block_.
|
|
174
|
+
#
|
|
175
|
+
def each_worker( &block )
|
|
176
|
+
@workers.each(&block)
|
|
177
|
+
self
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# call-seq:
|
|
181
|
+
# errors { |worker| block }
|
|
182
|
+
#
|
|
183
|
+
# Iterates over all the works and yields the worker to the given _block_
|
|
184
|
+
# only if the worker has an error condition.
|
|
185
|
+
#
|
|
186
|
+
def errors
|
|
187
|
+
@workers.each { |worker| yield worker unless worker.error.nil? }
|
|
188
|
+
self
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# The worker encapsulates the forking of the child process and communication
|
|
192
|
+
# between the parent and the child. Each worker instance is extended with
|
|
193
|
+
# the block or module supplied to the pre-forking pool that created the
|
|
194
|
+
# worker.
|
|
195
|
+
#
|
|
196
|
+
class Worker
|
|
197
|
+
|
|
198
|
+
attr_reader :prefork
|
|
199
|
+
attr_reader :error
|
|
200
|
+
|
|
201
|
+
# Create a new worker that belongs to the _prefork_ pool.
|
|
202
|
+
#
|
|
203
|
+
def initialize( prefork )
|
|
204
|
+
@prefork = prefork
|
|
205
|
+
@thread = nil
|
|
206
|
+
@piper = nil
|
|
207
|
+
@error = nil
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Start this worker. A new process will be forked, and the code supplied
|
|
211
|
+
# by the user to the prefork pool will be executed in the child process.
|
|
212
|
+
#
|
|
213
|
+
def start
|
|
214
|
+
@error = nil
|
|
215
|
+
@piper = ::Servolux::Piper.new('rw', :timeout => @prefork.timeout)
|
|
216
|
+
parent if @piper.parent?
|
|
217
|
+
child if @piper.child?
|
|
218
|
+
self
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Stop this worker. The internal worker thread is stopped and a 'HUP'
|
|
222
|
+
# signal is sent to the child process. This method will return immediately
|
|
223
|
+
# without waiting for the child process to exit. Use the +wait+ method
|
|
224
|
+
# after calling +stop+ if your code needs to know when the child exits.
|
|
225
|
+
#
|
|
226
|
+
def stop
|
|
227
|
+
return if @thread.nil? or @piper.nil? or @piper.child?
|
|
228
|
+
|
|
229
|
+
@thread[:stop] = true
|
|
230
|
+
@thread.wakeup
|
|
231
|
+
Thread.pass until !@thread.status
|
|
232
|
+
kill 'HUP'
|
|
233
|
+
@thread = nil
|
|
234
|
+
self
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Wait for the child process to exit. This method returns immediately when
|
|
238
|
+
# called from the child process or if the child process has not yet been
|
|
239
|
+
# forked.
|
|
240
|
+
#
|
|
241
|
+
def wait
|
|
242
|
+
return if @piper.nil? or @piper.child?
|
|
243
|
+
Process.wait(@piper.pid, Process::WNOHANG|Process::WUNTRACED)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Send this given _signal_ to the child process. The default signal is
|
|
247
|
+
# 'TERM'. This method will return immediately.
|
|
248
|
+
#
|
|
249
|
+
def kill( signal = 'TERM' )
|
|
250
|
+
return if @piper.nil?
|
|
251
|
+
@piper.signal signal
|
|
252
|
+
rescue Errno::ESRCH, Errno::ENOENT
|
|
253
|
+
return nil
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Returns +true+ if the child process is alive. Returns +nil+ if the child
|
|
257
|
+
# process has not been started.
|
|
258
|
+
#
|
|
259
|
+
# Always returns +nil+ when called from the child process.
|
|
260
|
+
#
|
|
261
|
+
def alive?
|
|
262
|
+
return if @piper.nil?
|
|
263
|
+
@piper.alive?
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
private
|
|
268
|
+
|
|
269
|
+
# This code should only be executed in the parent process.
|
|
270
|
+
#
|
|
271
|
+
def parent
|
|
272
|
+
@thread = Thread.new {
|
|
273
|
+
response = nil
|
|
274
|
+
begin
|
|
275
|
+
@piper.puts START
|
|
276
|
+
Thread.current[:stop] = false
|
|
277
|
+
loop {
|
|
278
|
+
break if Thread.current[:stop]
|
|
279
|
+
@piper.puts HEARTBEAT
|
|
280
|
+
response = @piper.gets(ERROR)
|
|
281
|
+
break if Thread.current[:stop]
|
|
282
|
+
|
|
283
|
+
case response
|
|
284
|
+
when HEARTBEAT; next
|
|
285
|
+
when START; break
|
|
286
|
+
when ERROR
|
|
287
|
+
raise Timeout,
|
|
288
|
+
"Child did not respond in a timely fashion. Timeout is set to #{@prefork.timeout} seconds."
|
|
289
|
+
when Exception
|
|
290
|
+
raise response
|
|
291
|
+
else
|
|
292
|
+
raise UnknownResponse,
|
|
293
|
+
"Child returned unknown response: #{response.inspect}"
|
|
294
|
+
end
|
|
295
|
+
}
|
|
296
|
+
rescue Exception => err
|
|
297
|
+
@error = err
|
|
298
|
+
ensure
|
|
299
|
+
@piper.timeout = 0
|
|
300
|
+
@piper.puts HALT rescue nil
|
|
301
|
+
@piper.close
|
|
302
|
+
self.start if START == response
|
|
303
|
+
end
|
|
304
|
+
}
|
|
305
|
+
Thread.pass until @thread[:stop] == false
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# This code should only be executed in the child process. It wraps the
|
|
309
|
+
# user supplied "execute" code in a loop, and takes care of handling
|
|
310
|
+
# signals and communication with the parent.
|
|
311
|
+
#
|
|
312
|
+
def child
|
|
313
|
+
@thread = Thread.current
|
|
314
|
+
|
|
315
|
+
# if we get a HUP signal, then tell the parent process to stop this
|
|
316
|
+
# child process and start a new one to replace it
|
|
317
|
+
Signal.trap('HUP') {
|
|
318
|
+
@piper.puts START rescue nil
|
|
319
|
+
@thread.wakeup
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
before_executing if self.respond_to? :before_executing
|
|
323
|
+
:wait until @piper.gets == START
|
|
324
|
+
|
|
325
|
+
loop {
|
|
326
|
+
signal = @piper.gets(ERROR)
|
|
327
|
+
case signal
|
|
328
|
+
when HEARTBEAT
|
|
329
|
+
execute
|
|
330
|
+
@piper.puts HEARTBEAT
|
|
331
|
+
when HALT
|
|
332
|
+
break
|
|
333
|
+
when ERROR
|
|
334
|
+
raise Timeout,
|
|
335
|
+
"Parent did not respond in a timely fashion. Timeout is set to #{@prefork.timeout} seconds."
|
|
336
|
+
else
|
|
337
|
+
raise UnknownSignal,
|
|
338
|
+
"Child received unknown signal: #{signal.inspect}"
|
|
339
|
+
end
|
|
340
|
+
}
|
|
341
|
+
after_executing if self.respond_to? :after_executing
|
|
342
|
+
rescue Exception => err
|
|
343
|
+
@piper.puts err rescue nil
|
|
344
|
+
ensure
|
|
345
|
+
@piper.close
|
|
346
|
+
exit!
|
|
347
|
+
end
|
|
348
|
+
end # class Worker
|
|
349
|
+
|
|
350
|
+
end # class Servolux::Prefork
|
|
351
|
+
|
data/lib/servolux.rb
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
module Servolux
|
|
3
3
|
|
|
4
4
|
# :stopdoc:
|
|
5
|
-
VERSION = '0.
|
|
5
|
+
VERSION = '0.8.0'
|
|
6
6
|
LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
|
|
7
7
|
PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
|
|
8
8
|
# :startdoc:
|
|
@@ -40,8 +40,7 @@ module Servolux
|
|
|
40
40
|
|
|
41
41
|
end # module Servolux
|
|
42
42
|
|
|
43
|
-
%w[threaded server piper daemon child].each do |lib|
|
|
43
|
+
%w[threaded server piper daemon child prefork].each do |lib|
|
|
44
44
|
require Servolux.libpath('servolux', lib)
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
-
# EOF
|
data/spec/piper_spec.rb
CHANGED
|
@@ -11,7 +11,7 @@ describe Servolux::Piper do
|
|
|
11
11
|
|
|
12
12
|
after :each do
|
|
13
13
|
next if @piper.nil?
|
|
14
|
-
@piper.puts :die
|
|
14
|
+
@piper.puts :die rescue nil
|
|
15
15
|
@piper.close
|
|
16
16
|
@piper = nil
|
|
17
17
|
end
|
|
@@ -40,6 +40,7 @@ describe Servolux::Piper do
|
|
|
40
40
|
end
|
|
41
41
|
@piper.puts obj unless obj.nil?
|
|
42
42
|
}
|
|
43
|
+
exit!
|
|
43
44
|
}
|
|
44
45
|
|
|
45
46
|
@piper.parent {
|
|
@@ -65,7 +66,7 @@ describe Servolux::Piper do
|
|
|
65
66
|
end
|
|
66
67
|
|
|
67
68
|
it 'sends signals from parent to child' do
|
|
68
|
-
@piper = Servolux::Piper.new :timeout => 2
|
|
69
|
+
@piper = Servolux::Piper.new 'rw', :timeout => 2
|
|
69
70
|
|
|
70
71
|
@piper.child {
|
|
71
72
|
Signal.trap('USR2') { @piper.puts "'USR2' was received" rescue nil }
|
|
@@ -77,6 +78,7 @@ describe Servolux::Piper do
|
|
|
77
78
|
Thread.new { sleep 7; exit! }
|
|
78
79
|
@piper.puts :ready
|
|
79
80
|
loop { sleep }
|
|
81
|
+
exit!
|
|
80
82
|
}
|
|
81
83
|
|
|
82
84
|
@piper.parent {
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: servolux
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Tim Pease
|
|
@@ -9,13 +9,23 @@ autorequire:
|
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
11
|
|
|
12
|
-
date: 2009-11-
|
|
12
|
+
date: 2009-11-12 00:00:00 -07:00
|
|
13
13
|
default_executable:
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: bones-extras
|
|
17
17
|
type: :development
|
|
18
18
|
version_requirement:
|
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
20
|
+
requirements:
|
|
21
|
+
- - ">="
|
|
22
|
+
- !ruby/object:Gem::Version
|
|
23
|
+
version: 1.1.0
|
|
24
|
+
version:
|
|
25
|
+
- !ruby/object:Gem::Dependency
|
|
26
|
+
name: bones-git
|
|
27
|
+
type: :development
|
|
28
|
+
version_requirement:
|
|
19
29
|
version_requirements: !ruby/object:Gem::Requirement
|
|
20
30
|
requirements:
|
|
21
31
|
- - ">="
|
|
@@ -50,7 +60,7 @@ dependencies:
|
|
|
50
60
|
requirements:
|
|
51
61
|
- - ">="
|
|
52
62
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: 3.0.
|
|
63
|
+
version: 3.0.1
|
|
54
64
|
version:
|
|
55
65
|
description: |-
|
|
56
66
|
Serv-O-Lux is a collection of Ruby classes that are useful for daemon and
|
|
@@ -69,10 +79,15 @@ files:
|
|
|
69
79
|
- History.txt
|
|
70
80
|
- README.rdoc
|
|
71
81
|
- Rakefile
|
|
82
|
+
- a.rb
|
|
83
|
+
- b.rb
|
|
84
|
+
- examples/beanstalk.rb
|
|
85
|
+
- examples/echo.rb
|
|
72
86
|
- lib/servolux.rb
|
|
73
87
|
- lib/servolux/child.rb
|
|
74
88
|
- lib/servolux/daemon.rb
|
|
75
89
|
- lib/servolux/piper.rb
|
|
90
|
+
- lib/servolux/prefork.rb
|
|
76
91
|
- lib/servolux/server.rb
|
|
77
92
|
- lib/servolux/threaded.rb
|
|
78
93
|
- spec/child_spec.rb
|
|
@@ -105,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
105
120
|
version:
|
|
106
121
|
requirements: []
|
|
107
122
|
|
|
108
|
-
rubyforge_project:
|
|
123
|
+
rubyforge_project: codeforpeople
|
|
109
124
|
rubygems_version: 1.3.5
|
|
110
125
|
signing_key:
|
|
111
126
|
specification_version: 3
|