jobby 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +77 -0
- data/bin/jobby +89 -0
- data/lib/client.rb +38 -0
- data/lib/runner.rb +153 -0
- data/lib/server.rb +192 -0
- data/spec/run_all.rb +6 -0
- data/spec/server_spec.rb +179 -0
- metadata +59 -0
data/README
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
== Jobby
|
2
|
+
|
3
|
+
Jobby is a small utility and library for managing running jobs in concurrent
|
4
|
+
processes. It was initially developed for offloading long running tasks from
|
5
|
+
the webserver in Rails applications, but it has proven to be useful in its own
|
6
|
+
right and has been extracted to work in a general manner.
|
7
|
+
|
8
|
+
|
9
|
+
== Download
|
10
|
+
|
11
|
+
* gem install jobby
|
12
|
+
* http://github.com/Spakman/jobby/tree/master
|
13
|
+
* git clone git://github.com/Spakman/jobby.git
|
14
|
+
|
15
|
+
|
16
|
+
== Contact
|
17
|
+
|
18
|
+
* Mark Somerville <mailto:mark@scottishclimbs.com>
|
19
|
+
* {Rubyforge}[http://jobby.rubyforge.org/]
|
20
|
+
|
21
|
+
|
22
|
+
== Usage
|
23
|
+
|
24
|
+
Jobby is a single 'binary' script that, can be run from the command line and is
|
25
|
+
often used within scripts. You can, of course, also use the Jobby::Server and
|
26
|
+
Jobby::Client classes directly in Ruby programs if you like.
|
27
|
+
|
28
|
+
This bash script attempts to illustrate some simple, if somewhat contrived,
|
29
|
+
usage of processing a directory full of text files (yes, there are better tools
|
30
|
+
for this particular example):
|
31
|
+
|
32
|
+
#!/bin/bash
|
33
|
+
for i in *; do
|
34
|
+
jobby --ruby 'File.rename("#{input}", "#{input}.jobby"' --max-children 4 --input $i
|
35
|
+
done
|
36
|
+
|
37
|
+
The above script runs the specified Ruby code on up to four processes in
|
38
|
+
parallel. In this case, each file in the current directory will be renamed with
|
39
|
+
'.jobby' appended. Standard Ruby string interpolation is used to replace
|
40
|
+
'#{input}' with whatever is specified by --input. You may pass a filepath in
|
41
|
+
place of a string as the --ruby parameter - in this case the specified file is
|
42
|
+
simply loaded (using the Kernel module method 'load') by Ruby.
|
43
|
+
|
44
|
+
The above 'jobby' command is equivalent to:
|
45
|
+
|
46
|
+
jobby --command 'mv #{input} #{input}.jobby' --max-children 4 --input $i
|
47
|
+
|
48
|
+
The difference is that whatever is in the --command parameter is exec'd by the
|
49
|
+
child process rather than interpretted as Ruby code. Again, Ruby string
|
50
|
+
interpolation is used.
|
51
|
+
|
52
|
+
It is important to realise that although the --ruby 'File....' parameter is
|
53
|
+
passed every time jobby is called in this for loop, it is only actually read
|
54
|
+
and used the first time. I'll try to explain why below.
|
55
|
+
|
56
|
+
|
57
|
+
== What happens when you run Jobby?
|
58
|
+
|
59
|
+
Jobby can be thought of as a self-managing daemon and client program rolled
|
60
|
+
into one. The first time you run Jobby, a daemon is started which listens for
|
61
|
+
connections on a socket. The daemon will fork every time it receives a
|
62
|
+
connection (up to --max-children, further requests are queued) and the forked
|
63
|
+
child will run whatever is specified by --ruby or --command.
|
64
|
+
|
65
|
+
The *same* 'jobby' call also runs a client program to connect to the daemon,
|
66
|
+
passing --input as a parameter which will be used by the child process.
|
67
|
+
Subsequent calls to 'jobby' will use the existing daemon.
|
68
|
+
|
69
|
+
|
70
|
+
== Running multiple Jobby daemons
|
71
|
+
|
72
|
+
Since the --ruby and --command parameters are ignored (and the ones used for
|
73
|
+
the first 'jobby' call are used), you may wonder how to specify different types
|
74
|
+
of jobs to be run. The most straightforward way to do this is to specify a
|
75
|
+
different socket for the Jobby daemon, using the --socket option. Then, the
|
76
|
+
'jobby' command will run whatever --ruby or --command is specified for the
|
77
|
+
daemon running on that socket.
|
data/bin/jobby
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# Copyright (C) 2008 Mark Somerville
|
3
|
+
#
|
4
|
+
# This library is free software; you can redistribute it and/or
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
6
|
+
# License as published by the Free Software Foundation; either
|
7
|
+
# version 2.1 of the License, or any later version.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
17
|
+
|
18
|
+
require 'logger'
|
19
|
+
require 'optparse'
|
20
|
+
require 'etc'
|
21
|
+
require "#{File.dirname(__FILE__)}/../lib/server"
|
22
|
+
require "#{File.dirname(__FILE__)}/../lib/client"
|
23
|
+
require "#{File.dirname(__FILE__)}/../lib/runner"
|
24
|
+
|
25
|
+
$0 = "jobby" # set the process name
|
26
|
+
|
27
|
+
options = {}
|
28
|
+
OptionParser.new do |opts|
|
29
|
+
opts.banner = "Usage: jobby [options]"
|
30
|
+
|
31
|
+
opts.on("-v", "--[no-]verbose", "Run verbosely") do |verbose|
|
32
|
+
options[:verbose] = verbose
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("-h", "--help", "Show this message") do
|
36
|
+
puts opts
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.on("--version", "Show version") do
|
41
|
+
puts "0.1.1"
|
42
|
+
exit
|
43
|
+
end
|
44
|
+
|
45
|
+
opts.on("-s", "--socket [SOCKETFILE]", "Connect to this UNIX socket") do |socket|
|
46
|
+
options[:socket] = socket
|
47
|
+
end
|
48
|
+
|
49
|
+
opts.on("-i", "--input [INPUT]", "Pass this string to the child process (can be used instead of STDIN)") do |input|
|
50
|
+
options[:input] = input
|
51
|
+
end
|
52
|
+
|
53
|
+
opts.on("-f", "--flush", "Shutdown the server on the specified socket") do |flush|
|
54
|
+
options[:flush] = flush
|
55
|
+
end
|
56
|
+
|
57
|
+
opts.on("-w", "--wipe", "Shutdown the server and terminate the children on the specified socket immediately") do |wipe|
|
58
|
+
options[:wipe] = wipe
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.separator ""
|
62
|
+
opts.separator "Server options:"
|
63
|
+
|
64
|
+
opts.on("-l", "--log [LOGFILE]", "Log to this file") do |logfile|
|
65
|
+
options[:log] = logfile
|
66
|
+
end
|
67
|
+
|
68
|
+
opts.on("-m", "--max-children [MAXCHILDREN]", "Run MAXCHILDREN forked processes at any one time") do |forked_children|
|
69
|
+
options[:max_child_processes] = forked_children.to_i
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on("-u", "--user [USER]", "Run the processes as this user (probably requires superuser priviledges)") do |user|
|
73
|
+
options[:user] = user
|
74
|
+
end
|
75
|
+
|
76
|
+
opts.on("-g", "--group [GROUP]", "Run the processes as this group (probably requires superuser priviledges)") do |group|
|
77
|
+
options[:group] = group
|
78
|
+
end
|
79
|
+
|
80
|
+
opts.on("-r", "--ruby [RUBY]", "Run this Ruby code in the forked children") do |ruby|
|
81
|
+
options[:ruby] = ruby
|
82
|
+
end
|
83
|
+
|
84
|
+
opts.on("-c", "--command [COMMAND]", "Run this shell code in the forked children") do |command|
|
85
|
+
options[:command] = command
|
86
|
+
end
|
87
|
+
end.parse!
|
88
|
+
|
89
|
+
Jobby::Runner.new(options).run
|
data/lib/client.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# Copyright (C) 2008 Mark Somerville
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2 of the License, or any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require 'socket'
|
18
|
+
|
19
|
+
module Jobby
|
20
|
+
class Client
|
21
|
+
# Creates a new client. Passing a block here is a shortcut for calling send and then close.
|
22
|
+
def initialize(socket_path, &block)
|
23
|
+
@socket = UNIXSocket.open(socket_path)
|
24
|
+
if block_given?
|
25
|
+
yield(self)
|
26
|
+
close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def send(message = "")
|
31
|
+
@socket.send(message, 0)
|
32
|
+
end
|
33
|
+
|
34
|
+
def close
|
35
|
+
@socket.close
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/runner.rb
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
# Copyright (C) 2008 Mark Somerville
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2 of the License, or any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
module Jobby
|
18
|
+
class Runner
|
19
|
+
DEFAULT_OPTIONS = {}
|
20
|
+
DEFAULT_OPTIONS[:socket] = "/tmp/jobby.socket"
|
21
|
+
DEFAULT_OPTIONS[:log] = $stderr
|
22
|
+
DEFAULT_OPTIONS[:max_child_processes] = 1
|
23
|
+
DEFAULT_OPTIONS[:exit_on_error] = true
|
24
|
+
|
25
|
+
def initialize(options = {})
|
26
|
+
@options = DEFAULT_OPTIONS.merge options
|
27
|
+
if @options[:wipe]
|
28
|
+
@options[:input] = "||JOBBY WIPE||"
|
29
|
+
elsif @options[:flush]
|
30
|
+
@options[:input] = "||JOBBY FLUSH||"
|
31
|
+
elsif @options[:input].nil?
|
32
|
+
message "--input not supplied, reading from STDIN (use ctrl-d to end input)"
|
33
|
+
@options[:input] = $stdin.read
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Tries to connect a client to the server. If there isn't a server detected on
|
38
|
+
# the socket this process is forked and a server is started. Then, another
|
39
|
+
# client tries to connect to the server.
|
40
|
+
#
|
41
|
+
# We may consider using fork and exec instead of just fork since COW [1]
|
42
|
+
# semantics are broken using Ruby 1.8.x. There is a patch, for those who can
|
43
|
+
# use it [2].
|
44
|
+
#
|
45
|
+
# [1] - http://blog.beaver.net/2005/03/ruby_gc_and_copyonwrite.html
|
46
|
+
#
|
47
|
+
# [2] - http://izumi.plan99.net/blog/index.php/2008/01/14/making-ruby’s-garbage-collector-copy-on-write-friendly-part-7/
|
48
|
+
#
|
49
|
+
# TODO: this code is pretty ugly.
|
50
|
+
def run
|
51
|
+
change_process_ownership
|
52
|
+
begin
|
53
|
+
if @options[:flush] or @options[:wipe]
|
54
|
+
if @options[:wipe]
|
55
|
+
message "Stopping Jobby server and terminating forked children..."
|
56
|
+
else
|
57
|
+
message "Stopping Jobby server..."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
run_client
|
61
|
+
rescue Errno::EACCES => exception
|
62
|
+
return error(exception.message)
|
63
|
+
rescue Errno::ENOENT, Errno::ECONNREFUSED
|
64
|
+
# Connect failed, fork and start the server process
|
65
|
+
message "There doesn't seem to be a server listening on #{@options[:socket]} - starting one..." if @options[:verbose]
|
66
|
+
exit if @options[:flush] or @options[:wipe]
|
67
|
+
fork do
|
68
|
+
begin
|
69
|
+
Jobby::Server.new(@options[:socket], @options[:max_child_processes], @options[:log]).run(&get_proc_from_options)
|
70
|
+
rescue Exception => exception
|
71
|
+
return error(exception.message)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
sleep 2 # give the server time to start
|
75
|
+
begin
|
76
|
+
run_client
|
77
|
+
rescue Errno::ECONNREFUSED
|
78
|
+
return error("Couldn't connect to the server process")
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
protected
|
84
|
+
|
85
|
+
# Creates a client and gets it to try to send a message to the Server on
|
86
|
+
# @options[:socket]. If this doesn't succeed, either a Errno::EACCES,
|
87
|
+
# Errno::ENOENT or Errno::ECONNREFUSED will probably be raised.
|
88
|
+
def run_client
|
89
|
+
message "Trying to connect to server on #{@options[:socket]}..." if @options[:verbose]
|
90
|
+
Jobby::Client.new(@options[:socket]) { |client| client.send(@options[:input]) }
|
91
|
+
message "Client has run successfully!" if @options[:verbose]
|
92
|
+
end
|
93
|
+
|
94
|
+
# Creates a Proc object that will be run by any children that the Server forks.
|
95
|
+
# The input parameter that is passed to the Proc is the input string that a
|
96
|
+
# Client will send to the server.
|
97
|
+
def get_proc_from_options
|
98
|
+
if @options[:ruby].nil? and @options[:command].nil?
|
99
|
+
return error("No server found on #{@options[:socket]} and you didn't give --ruby or --command to execute")
|
100
|
+
|
101
|
+
elsif not @options[:ruby].nil? and not @options[:command].nil?
|
102
|
+
return error("You can only specify --ruby or --command, not both")
|
103
|
+
|
104
|
+
elsif @options[:ruby] # can be either some Ruby code or a filepath
|
105
|
+
if File.file?(File.expand_path(@options[:ruby]))
|
106
|
+
return lambda { |input, logger|
|
107
|
+
ARGV << input
|
108
|
+
# read and eval this rather than Kernel.load so that the code in the
|
109
|
+
# file can use the local variables in this block
|
110
|
+
instance_eval(File.read(File.expand_path(@options[:ruby])))
|
111
|
+
}
|
112
|
+
else
|
113
|
+
return lambda { |input, logger|
|
114
|
+
instance_eval(@options[:ruby])
|
115
|
+
}
|
116
|
+
end
|
117
|
+
|
118
|
+
elsif @options[:command]
|
119
|
+
return lambda { |input, logger|
|
120
|
+
exec(eval("\"#{@options[:command].gsub('"', '\"')}\""))
|
121
|
+
}
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
# Changes the user and group ownership of the current process if
|
126
|
+
# @options[:user] or @options[:group] are set. This might be a privileged
|
127
|
+
# operation.
|
128
|
+
def change_process_ownership
|
129
|
+
begin
|
130
|
+
if @options[:group]
|
131
|
+
message "Setting group ownership to #{@options[:group]}..." if @options[:verbose]
|
132
|
+
Process::GID.change_privilege Etc.getgrnam(@options[:group]).gid
|
133
|
+
end
|
134
|
+
if @options[:user]
|
135
|
+
message "Setting user ownership to #{@options[:user]}..." if @options[:verbose]
|
136
|
+
Process::UID.change_privilege Etc.getpwnam(@options[:user]).uid
|
137
|
+
end
|
138
|
+
rescue Errno::EPERM
|
139
|
+
return error("You don't have permission to change the process ownership - perhaps you should be root?")
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def error(text)
|
144
|
+
puts " ERROR - #{text}"
|
145
|
+
exit -1 if @options[:exit_on_error]
|
146
|
+
return false
|
147
|
+
end
|
148
|
+
|
149
|
+
def message(text)
|
150
|
+
puts " #{text}"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
data/lib/server.rb
ADDED
@@ -0,0 +1,192 @@
|
|
1
|
+
# Copyright (C) 2008 Mark Somerville
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2 of the License, or any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require 'fileutils'
|
18
|
+
require 'socket'
|
19
|
+
require 'logger'
|
20
|
+
require 'thread'
|
21
|
+
|
22
|
+
module Jobby
|
23
|
+
# This is a generic server class which accepts connections on a UNIX socket. On
|
24
|
+
# receiving a connection, the server process forks and runs the specified block.
|
25
|
+
#
|
26
|
+
# ==Example
|
27
|
+
#
|
28
|
+
# Jobby::Server.new("/tmp/jobby.socket", 3, "/var/log/jobby.log").run do
|
29
|
+
# # This code will be run in the forked children
|
30
|
+
# puts "#{Process.pid}: I'm toilet trained!"
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# ==Stopping the server
|
34
|
+
#
|
35
|
+
# A client process can send one of two special strings to stop the server.
|
36
|
+
#
|
37
|
+
# "||JOBBY FLUSH||" will stop the server forking any more children and shut
|
38
|
+
# it down.
|
39
|
+
#
|
40
|
+
# "||JOBBY WIPE||" will stop the server forking any more children, kill 9
|
41
|
+
# any existing children and shut it down.
|
42
|
+
#
|
43
|
+
# ==Log rotation
|
44
|
+
#
|
45
|
+
# The server can receive USR1 signals as notification that the logfile has been
|
46
|
+
# rotated. This will close and re-open the handle to the log file. Since the
|
47
|
+
# server process forks to produce children, they too can handle USR1 signals on
|
48
|
+
# log rotation.
|
49
|
+
#
|
50
|
+
# To tell all Jobby processes that the log file has been rotated, use something
|
51
|
+
# like:
|
52
|
+
#
|
53
|
+
# % pkill -USR1 -f jobby
|
54
|
+
#
|
55
|
+
class Server
|
56
|
+
def initialize(socket_path, max_forked_processes, log)
|
57
|
+
@socket_path = socket_path
|
58
|
+
@max_forked_processes = max_forked_processes.to_i
|
59
|
+
@log = log
|
60
|
+
@queue = Queue.new
|
61
|
+
end
|
62
|
+
|
63
|
+
# Starts the server and listens for connections. The specified block is run in
|
64
|
+
# the child processes. When a connection is received, the input parameter is
|
65
|
+
# immediately added to the queue.
|
66
|
+
def run(&block)
|
67
|
+
connect_to_socket_and_start_logging
|
68
|
+
unless block_given?
|
69
|
+
@logger.error "No block given, exiting"
|
70
|
+
terminate
|
71
|
+
end
|
72
|
+
start_forking_thread(block)
|
73
|
+
loop do
|
74
|
+
client = @socket.accept
|
75
|
+
input = ""
|
76
|
+
while bytes = client.read(128)
|
77
|
+
input += bytes
|
78
|
+
end
|
79
|
+
if input == "||JOBBY FLUSH||"
|
80
|
+
terminate
|
81
|
+
elsif input == "||JOBBY WIPE||"
|
82
|
+
terminate_children
|
83
|
+
terminate
|
84
|
+
else
|
85
|
+
@queue << input
|
86
|
+
end
|
87
|
+
client.close
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
protected
|
92
|
+
|
93
|
+
# Runs a thread to manage the forked processes. It will block, waiting for a
|
94
|
+
# child to finish if the maximum number of forked processes are already
|
95
|
+
# running. It will then, read from the queue and fork off a new process.
|
96
|
+
#
|
97
|
+
# The input variable that is passed to the block is the message that is
|
98
|
+
# received from Client#send.
|
99
|
+
def start_forking_thread(block)
|
100
|
+
Thread.new do
|
101
|
+
@pids = []
|
102
|
+
loop do
|
103
|
+
if @pids.length >= @max_forked_processes
|
104
|
+
begin
|
105
|
+
reap_child
|
106
|
+
rescue Errno::ECHILD
|
107
|
+
end
|
108
|
+
end
|
109
|
+
# fork and run code that performs the actual work
|
110
|
+
input = @queue.pop
|
111
|
+
@pids << fork do
|
112
|
+
$0 = "jobby: child" # set the process name
|
113
|
+
@logger.info "Child process started (#{Process.pid})"
|
114
|
+
block.call(input, @logger)
|
115
|
+
exit 0
|
116
|
+
end
|
117
|
+
Thread.new do
|
118
|
+
reap_child
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
# Checks if a process is already listening on the socket. If not, removes the
|
125
|
+
# socket file (if it's there) and starts a server. Throws an Errno::EADDRINUSE
|
126
|
+
# exception if an existing server is detected.
|
127
|
+
def connect_to_socket_and_start_logging
|
128
|
+
unless File.exists? @socket_path
|
129
|
+
connect_to_socket
|
130
|
+
else
|
131
|
+
begin
|
132
|
+
# test for a server on the socket
|
133
|
+
test_socket = UNIXSocket.open(@socket_path)
|
134
|
+
test_socket.close # got this far - seems like there is a server already
|
135
|
+
raise Errno::EADDRINUSE.new("it seems like there is already a server listening on #{@socket_path}")
|
136
|
+
rescue Errno::ECONNREFUSED
|
137
|
+
# probably not a server on that socket - start one
|
138
|
+
FileUtils.rm(@socket_path, :force => true)
|
139
|
+
connect_to_socket
|
140
|
+
end
|
141
|
+
end
|
142
|
+
start_logging
|
143
|
+
end
|
144
|
+
|
145
|
+
def connect_to_socket
|
146
|
+
@socket = UNIXServer.open(@socket_path)
|
147
|
+
FileUtils.chmod 0770, @socket_path
|
148
|
+
@socket.listen 10
|
149
|
+
end
|
150
|
+
|
151
|
+
def start_logging
|
152
|
+
@logger = Logger.new @log
|
153
|
+
@logger.info "Server started at #{Time.now}"
|
154
|
+
Signal.trap("USR1") do
|
155
|
+
rotate_log
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def reap_child
|
160
|
+
@pids.delete Process.wait
|
161
|
+
end
|
162
|
+
|
163
|
+
def rotate_log
|
164
|
+
@logger.close
|
165
|
+
@logger = Logger.new @log
|
166
|
+
@logger.info "USR1 received, rotating log file"
|
167
|
+
end
|
168
|
+
|
169
|
+
# Cleans up the server and exits the process with a return code 0.
|
170
|
+
def terminate
|
171
|
+
@queue.clear
|
172
|
+
@logger.info "Flush received - terminating server"
|
173
|
+
@socket.close
|
174
|
+
FileUtils.rm(@socket_path, :force => true)
|
175
|
+
exit! 0
|
176
|
+
end
|
177
|
+
|
178
|
+
# Stops any more children being forked and terminates the existing ones. A kill
|
179
|
+
# 9 signal is used as you will likely be run when termination is needed
|
180
|
+
# immediately, perhaps due to 'runaway' children.
|
181
|
+
def terminate_children
|
182
|
+
@queue.clear
|
183
|
+
@logger.info "Wipe received - terminating forked children"
|
184
|
+
@pids.each do |pid|
|
185
|
+
begin
|
186
|
+
Process.kill 9, pid
|
187
|
+
rescue Errno::ESRCH
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
data/spec/run_all.rb
ADDED
data/spec/server_spec.rb
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
require "#{File.dirname(__FILE__)}/../lib/server"
|
2
|
+
require "#{File.dirname(__FILE__)}/../lib/client"
|
3
|
+
require 'fileutils'
|
4
|
+
$0 = "jobby spec"
|
5
|
+
|
6
|
+
|
7
|
+
# Due to the multi-process nature of these specs, there are a bunch of sleep calls
|
8
|
+
# around. This is, of course, pretty brittle but I can't think of a better way of
|
9
|
+
# handling it. If they start failing 'randomly', I would first start by increasing
|
10
|
+
# the sleep times.
|
11
|
+
|
12
|
+
describe Jobby::Server do
|
13
|
+
|
14
|
+
def run_server(socket, max_child_processes, log_filepath, &block)
|
15
|
+
@server_pid = fork do
|
16
|
+
Jobby::Server.new(socket, max_child_processes, log_filepath).run(&block)
|
17
|
+
end
|
18
|
+
sleep 0.2
|
19
|
+
end
|
20
|
+
|
21
|
+
def terminate_server
|
22
|
+
Process.kill 15, @server_pid
|
23
|
+
if File.exists? @child_filepath
|
24
|
+
FileUtils.rm @child_filepath
|
25
|
+
end
|
26
|
+
FileUtils.rm @log_filepath, :force => true
|
27
|
+
sleep 0.5
|
28
|
+
end
|
29
|
+
|
30
|
+
before :all do
|
31
|
+
@socket = File.expand_path("#{File.dirname(__FILE__)}/jobby_server.sock")
|
32
|
+
@max_child_processes = 2
|
33
|
+
@log_filepath = File.expand_path("#{File.dirname(__FILE__)}/jobby_server.log")
|
34
|
+
@child_filepath = File.expand_path("#{File.dirname(__FILE__)}/jobby_child")
|
35
|
+
end
|
36
|
+
|
37
|
+
before :each do
|
38
|
+
run_server(@socket, @max_child_processes, @log_filepath) {
|
39
|
+
File.open(@child_filepath, "a+") do |file|
|
40
|
+
file << "#{Process.pid}"
|
41
|
+
end
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
after :each do
|
46
|
+
terminate_server
|
47
|
+
end
|
48
|
+
|
49
|
+
after :all do
|
50
|
+
FileUtils.rm @socket, :force => true
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should listen on a UNIX socket" do
|
54
|
+
lambda { UNIXSocket.open(@socket).close }.should_not raise_error
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should allow the children to log from within the called block" do
|
58
|
+
terminate_server
|
59
|
+
sleep 0.5
|
60
|
+
run_server(@socket, @max_child_processes, @log_filepath) { |input, logger|
|
61
|
+
logger.info "I can log!"
|
62
|
+
}
|
63
|
+
sleep 1
|
64
|
+
client_socket = UNIXSocket.open(@socket)
|
65
|
+
client_socket.send("hiya", 0)
|
66
|
+
client_socket.close
|
67
|
+
sleep 1
|
68
|
+
File.read(@log_filepath).should match(/I can log!/)
|
69
|
+
end
|
70
|
+
|
71
|
+
it "should throw an exception if there is already a process listening on the socket" do
|
72
|
+
lambda { Jobby::Server.new(@socket, @max_child_processes, @log_filepath).run { true } }.should raise_error(Errno::EADDRINUSE, "Address already in use - it seems like there is already a server listening on #{@socket}")
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should set the correct permissions on the socket file" do
|
76
|
+
`stat --format=%a,%F #{@socket}`.strip.should eql("770,socket")
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should log when it is started" do
|
80
|
+
File.read(@log_filepath).should match(/Server started at/)
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should be able to accept an IO object instead of a log filepath" do
|
84
|
+
terminate_server
|
85
|
+
sleep 1
|
86
|
+
io_filepath = File.expand_path("#{File.dirname(__FILE__)}/io_log_test.log")
|
87
|
+
FileUtils.rm io_filepath, :force => true
|
88
|
+
io = File.open(io_filepath, "a+")
|
89
|
+
run_server(@socket, @max_child_processes, io) {}
|
90
|
+
terminate_server
|
91
|
+
sleep 0.5
|
92
|
+
File.readlines(io_filepath).length.should eql(1)
|
93
|
+
FileUtils.rm io_filepath
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should flush and reload the log file when it receieves the USR1 signal" do
|
97
|
+
FileUtils.rm @log_filepath
|
98
|
+
Process.kill "USR1", @server_pid
|
99
|
+
sleep 0.2
|
100
|
+
File.read(@log_filepath).should match(/USR1 received, rotating log file/)
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should not run if a block is not given" do
|
104
|
+
terminate_server
|
105
|
+
sleep 0.5
|
106
|
+
run_server(@socket, @max_child_processes, @log_filepath)
|
107
|
+
sleep 0.5
|
108
|
+
lambda { UNIXSocket.open(@socket).close }.should raise_error
|
109
|
+
end
|
110
|
+
|
111
|
+
it "should read all of the provided message" do
|
112
|
+
terminate_server
|
113
|
+
sleep 0.5
|
114
|
+
run_server(@socket, @max_child_processes, @log_filepath) { |input, logger|
|
115
|
+
File.open(@child_filepath, "a+") do |file|
|
116
|
+
file << "#{input}"
|
117
|
+
end
|
118
|
+
}
|
119
|
+
Jobby::Client.new(@socket) { |c| c.send("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890") }
|
120
|
+
sleep 0.5
|
121
|
+
File.read(@child_filepath).should eql("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890")
|
122
|
+
end
|
123
|
+
|
124
|
+
it "should fork off a child and run the specified code when it receives a connection" do
|
125
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
126
|
+
sleep 0.2
|
127
|
+
File.read(@child_filepath).should_not eql(@server_pid.to_s)
|
128
|
+
end
|
129
|
+
|
130
|
+
it "should only fork off a certain number of children - the others should have to wait (in an internal queue)" do
|
131
|
+
terminate_server
|
132
|
+
run_server(@socket, @max_child_processes, @log_filepath) do
|
133
|
+
sleep 2
|
134
|
+
File.open(@child_filepath, "a+") do |file|
|
135
|
+
file << "#{Process.pid}\n"
|
136
|
+
end
|
137
|
+
end
|
138
|
+
(@max_child_processes + 2).times do |i|
|
139
|
+
Thread.new do
|
140
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
141
|
+
end
|
142
|
+
end
|
143
|
+
sleep 2.5
|
144
|
+
File.readlines(@child_filepath).length.should eql(@max_child_processes)
|
145
|
+
sleep 4
|
146
|
+
File.readlines(@child_filepath).length.should eql(@max_child_processes + 2)
|
147
|
+
end
|
148
|
+
|
149
|
+
it "should receive a flush command from the client and terminate while the children continue processing" do
|
150
|
+
terminate_server
|
151
|
+
sleep 1
|
152
|
+
run_server(@socket, 1, @log_filepath) do
|
153
|
+
sleep 2
|
154
|
+
end
|
155
|
+
2.times do |i|
|
156
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
157
|
+
end
|
158
|
+
sleep 1
|
159
|
+
Jobby::Client.new(@socket) { |c| c.send("||JOBBY FLUSH||") }
|
160
|
+
sleep 1.5
|
161
|
+
lambda { Jobby::Client.new(@socket) { |c| c.send("hello?") } }.should raise_error(Errno::ENOENT)
|
162
|
+
`pgrep -f 'jobby spec' | wc -l`.strip.should eql("2")
|
163
|
+
end
|
164
|
+
|
165
|
+
it "should receive a wipe command from the client and terminate, taking the children with it" do
|
166
|
+
terminate_server
|
167
|
+
run_server(@socket, 1, @log_filepath) do
|
168
|
+
sleep 2
|
169
|
+
end
|
170
|
+
2.times do |i|
|
171
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
172
|
+
end
|
173
|
+
sleep 1
|
174
|
+
Jobby::Client.new(@socket) { |c| c.send("||JOBBY WIPE||") }
|
175
|
+
sleep 2.5
|
176
|
+
lambda { Jobby::Client.new(@socket) { |c| c.send("hello?") } }.should raise_error(Errno::ENOENT)
|
177
|
+
`pgrep -f 'jobby spec'`.strip.should eql("#{Process.pid}")
|
178
|
+
end
|
179
|
+
end
|
metadata
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: jobby
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Mark Somerville
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2008-06-17 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description:
|
17
|
+
email: mark@scottishclimbs.com
|
18
|
+
executables:
|
19
|
+
- jobby
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files:
|
23
|
+
- README
|
24
|
+
files:
|
25
|
+
- bin/jobby
|
26
|
+
- lib/runner.rb
|
27
|
+
- lib/server.rb
|
28
|
+
- lib/client.rb
|
29
|
+
- spec/server_spec.rb
|
30
|
+
- spec/run_all.rb
|
31
|
+
- README
|
32
|
+
has_rdoc: true
|
33
|
+
homepage: http://mark.scottishclimbs.com/
|
34
|
+
post_install_message:
|
35
|
+
rdoc_options: []
|
36
|
+
|
37
|
+
require_paths:
|
38
|
+
- lib
|
39
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: "0"
|
44
|
+
version:
|
45
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - ">="
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: "0"
|
50
|
+
version:
|
51
|
+
requirements: []
|
52
|
+
|
53
|
+
rubyforge_project: jobby
|
54
|
+
rubygems_version: 1.0.1
|
55
|
+
signing_key:
|
56
|
+
specification_version: 2
|
57
|
+
summary: Jobby is a small utility and library for managing running jobs in concurrent processes.
|
58
|
+
test_files: []
|
59
|
+
|