jobby 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +77 -0
- data/bin/jobby +89 -0
- data/lib/client.rb +38 -0
- data/lib/runner.rb +153 -0
- data/lib/server.rb +192 -0
- data/spec/run_all.rb +6 -0
- data/spec/server_spec.rb +179 -0
- metadata +59 -0
data/README
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
== Jobby
|
2
|
+
|
3
|
+
Jobby is a small utility and library for managing running jobs in concurrent
|
4
|
+
processes. It was initially developed for offloading long running tasks from
|
5
|
+
the webserver in Rails applications, but it has proven to be useful in its own
|
6
|
+
right and has been extracted to work in a general manner.
|
7
|
+
|
8
|
+
|
9
|
+
== Download
|
10
|
+
|
11
|
+
* gem install jobby
|
12
|
+
* http://github.com/Spakman/jobby/tree/master
|
13
|
+
* git clone git://github.com/Spakman/jobby.git
|
14
|
+
|
15
|
+
|
16
|
+
== Contact
|
17
|
+
|
18
|
+
* Mark Somerville <mailto:mark@scottishclimbs.com>
|
19
|
+
* {Rubyforge}[http://jobby.rubyforge.org/]
|
20
|
+
|
21
|
+
|
22
|
+
== Usage
|
23
|
+
|
24
|
+
Jobby is a single 'binary' script that, can be run from the command line and is
|
25
|
+
often used within scripts. You can, of course, also use the Jobby::Server and
|
26
|
+
Jobby::Client classes directly in Ruby programs if you like.
|
27
|
+
|
28
|
+
This bash script attempts to illustrate some simple, if somewhat contrived,
|
29
|
+
usage of processing a directory full of text files (yes, there are better tools
|
30
|
+
for this particular example):
|
31
|
+
|
32
|
+
#!/bin/bash
|
33
|
+
for i in *; do
|
34
|
+
jobby --ruby 'File.rename("#{input}", "#{input}.jobby"' --max-children 4 --input $i
|
35
|
+
done
|
36
|
+
|
37
|
+
The above script runs the specified Ruby code on up to four processes in
|
38
|
+
parallel. In this case, each file in the current directory will be renamed with
|
39
|
+
'.jobby' appended. Standard Ruby string interpolation is used to replace
|
40
|
+
'#{input}' with whatever is specified by --input. You may pass a filepath in
|
41
|
+
place of a string as the --ruby parameter - in this case the specified file is
|
42
|
+
simply loaded (using the Kernel module method 'load') by Ruby.
|
43
|
+
|
44
|
+
The above 'jobby' command is equivalent to:
|
45
|
+
|
46
|
+
jobby --command 'mv #{input} #{input}.jobby' --max-children 4 --input $i
|
47
|
+
|
48
|
+
The difference is that whatever is in the --command parameter is exec'd by the
|
49
|
+
child process rather than interpretted as Ruby code. Again, Ruby string
|
50
|
+
interpolation is used.
|
51
|
+
|
52
|
+
It is important to realise that although the --ruby 'File....' parameter is
|
53
|
+
passed every time jobby is called in this for loop, it is only actually read
|
54
|
+
and used the first time. I'll try to explain why below.
|
55
|
+
|
56
|
+
|
57
|
+
== What happens when you run Jobby?
|
58
|
+
|
59
|
+
Jobby can be thought of as a self-managing daemon and client program rolled
|
60
|
+
into one. The first time you run Jobby, a daemon is started which listens for
|
61
|
+
connections on a socket. The daemon will fork every time it receives a
|
62
|
+
connection (up to --max-children, further requests are queued) and the forked
|
63
|
+
child will run whatever is specified by --ruby or --command.
|
64
|
+
|
65
|
+
The *same* 'jobby' call also runs a client program to connect to the daemon,
|
66
|
+
passing --input as a parameter which will be used by the child process.
|
67
|
+
Subsequent calls to 'jobby' will use the existing daemon.
|
68
|
+
|
69
|
+
|
70
|
+
== Running multiple Jobby daemons
|
71
|
+
|
72
|
+
Since the --ruby and --command parameters are ignored (and the ones used for
|
73
|
+
the first 'jobby' call are used), you may wonder how to specify different types
|
74
|
+
of jobs to be run. The most straightforward way to do this is to specify a
|
75
|
+
different socket for the Jobby daemon, using the --socket option. Then, the
|
76
|
+
'jobby' command will run whatever --ruby or --command is specified for the
|
77
|
+
daemon running on that socket.
|
data/bin/jobby
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# Copyright (C) 2008 Mark Somerville
|
3
|
+
#
|
4
|
+
# This library is free software; you can redistribute it and/or
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
6
|
+
# License as published by the Free Software Foundation; either
|
7
|
+
# version 2.1 of the License, or any later version.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
17
|
+
|
18
|
+
require 'logger'
|
19
|
+
require 'optparse'
|
20
|
+
require 'etc'
|
21
|
+
require "#{File.dirname(__FILE__)}/../lib/server"
|
22
|
+
require "#{File.dirname(__FILE__)}/../lib/client"
|
23
|
+
require "#{File.dirname(__FILE__)}/../lib/runner"
|
24
|
+
|
25
|
+
$0 = "jobby" # set the process name
|
26
|
+
|
27
|
+
options = {}
|
28
|
+
OptionParser.new do |opts|
|
29
|
+
opts.banner = "Usage: jobby [options]"
|
30
|
+
|
31
|
+
opts.on("-v", "--[no-]verbose", "Run verbosely") do |verbose|
|
32
|
+
options[:verbose] = verbose
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("-h", "--help", "Show this message") do
|
36
|
+
puts opts
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.on("--version", "Show version") do
|
41
|
+
puts "0.1.1"
|
42
|
+
exit
|
43
|
+
end
|
44
|
+
|
45
|
+
opts.on("-s", "--socket [SOCKETFILE]", "Connect to this UNIX socket") do |socket|
|
46
|
+
options[:socket] = socket
|
47
|
+
end
|
48
|
+
|
49
|
+
opts.on("-i", "--input [INPUT]", "Pass this string to the child process (can be used instead of STDIN)") do |input|
|
50
|
+
options[:input] = input
|
51
|
+
end
|
52
|
+
|
53
|
+
opts.on("-f", "--flush", "Shutdown the server on the specified socket") do |flush|
|
54
|
+
options[:flush] = flush
|
55
|
+
end
|
56
|
+
|
57
|
+
opts.on("-w", "--wipe", "Shutdown the server and terminate the children on the specified socket immediately") do |wipe|
|
58
|
+
options[:wipe] = wipe
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.separator ""
|
62
|
+
opts.separator "Server options:"
|
63
|
+
|
64
|
+
opts.on("-l", "--log [LOGFILE]", "Log to this file") do |logfile|
|
65
|
+
options[:log] = logfile
|
66
|
+
end
|
67
|
+
|
68
|
+
opts.on("-m", "--max-children [MAXCHILDREN]", "Run MAXCHILDREN forked processes at any one time") do |forked_children|
|
69
|
+
options[:max_child_processes] = forked_children.to_i
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on("-u", "--user [USER]", "Run the processes as this user (probably requires superuser priviledges)") do |user|
|
73
|
+
options[:user] = user
|
74
|
+
end
|
75
|
+
|
76
|
+
opts.on("-g", "--group [GROUP]", "Run the processes as this group (probably requires superuser priviledges)") do |group|
|
77
|
+
options[:group] = group
|
78
|
+
end
|
79
|
+
|
80
|
+
opts.on("-r", "--ruby [RUBY]", "Run this Ruby code in the forked children") do |ruby|
|
81
|
+
options[:ruby] = ruby
|
82
|
+
end
|
83
|
+
|
84
|
+
opts.on("-c", "--command [COMMAND]", "Run this shell code in the forked children") do |command|
|
85
|
+
options[:command] = command
|
86
|
+
end
|
87
|
+
end.parse!
|
88
|
+
|
89
|
+
Jobby::Runner.new(options).run
|
data/lib/client.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# Copyright (C) 2008 Mark Somerville
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2 of the License, or any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require 'socket'
|
18
|
+
|
19
|
+
module Jobby
|
20
|
+
class Client
|
21
|
+
# Creates a new client. Passing a block here is a shortcut for calling send and then close.
|
22
|
+
def initialize(socket_path, &block)
|
23
|
+
@socket = UNIXSocket.open(socket_path)
|
24
|
+
if block_given?
|
25
|
+
yield(self)
|
26
|
+
close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def send(message = "")
|
31
|
+
@socket.send(message, 0)
|
32
|
+
end
|
33
|
+
|
34
|
+
def close
|
35
|
+
@socket.close
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/runner.rb
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
# Copyright (C) 2008 Mark Somerville
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2 of the License, or any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
module Jobby
|
18
|
+
class Runner
|
19
|
+
DEFAULT_OPTIONS = {}
|
20
|
+
DEFAULT_OPTIONS[:socket] = "/tmp/jobby.socket"
|
21
|
+
DEFAULT_OPTIONS[:log] = $stderr
|
22
|
+
DEFAULT_OPTIONS[:max_child_processes] = 1
|
23
|
+
DEFAULT_OPTIONS[:exit_on_error] = true
|
24
|
+
|
25
|
+
def initialize(options = {})
|
26
|
+
@options = DEFAULT_OPTIONS.merge options
|
27
|
+
if @options[:wipe]
|
28
|
+
@options[:input] = "||JOBBY WIPE||"
|
29
|
+
elsif @options[:flush]
|
30
|
+
@options[:input] = "||JOBBY FLUSH||"
|
31
|
+
elsif @options[:input].nil?
|
32
|
+
message "--input not supplied, reading from STDIN (use ctrl-d to end input)"
|
33
|
+
@options[:input] = $stdin.read
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Tries to connect a client to the server. If there isn't a server detected on
|
38
|
+
# the socket this process is forked and a server is started. Then, another
|
39
|
+
# client tries to connect to the server.
|
40
|
+
#
|
41
|
+
# We may consider using fork and exec instead of just fork since COW [1]
|
42
|
+
# semantics are broken using Ruby 1.8.x. There is a patch, for those who can
|
43
|
+
# use it [2].
|
44
|
+
#
|
45
|
+
# [1] - http://blog.beaver.net/2005/03/ruby_gc_and_copyonwrite.html
|
46
|
+
#
|
47
|
+
# [2] - http://izumi.plan99.net/blog/index.php/2008/01/14/making-ruby’s-garbage-collector-copy-on-write-friendly-part-7/
|
48
|
+
#
|
49
|
+
# TODO: this code is pretty ugly.
|
50
|
+
def run
|
51
|
+
change_process_ownership
|
52
|
+
begin
|
53
|
+
if @options[:flush] or @options[:wipe]
|
54
|
+
if @options[:wipe]
|
55
|
+
message "Stopping Jobby server and terminating forked children..."
|
56
|
+
else
|
57
|
+
message "Stopping Jobby server..."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
run_client
|
61
|
+
rescue Errno::EACCES => exception
|
62
|
+
return error(exception.message)
|
63
|
+
rescue Errno::ENOENT, Errno::ECONNREFUSED
|
64
|
+
# Connect failed, fork and start the server process
|
65
|
+
message "There doesn't seem to be a server listening on #{@options[:socket]} - starting one..." if @options[:verbose]
|
66
|
+
exit if @options[:flush] or @options[:wipe]
|
67
|
+
fork do
|
68
|
+
begin
|
69
|
+
Jobby::Server.new(@options[:socket], @options[:max_child_processes], @options[:log]).run(&get_proc_from_options)
|
70
|
+
rescue Exception => exception
|
71
|
+
return error(exception.message)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
sleep 2 # give the server time to start
|
75
|
+
begin
|
76
|
+
run_client
|
77
|
+
rescue Errno::ECONNREFUSED
|
78
|
+
return error("Couldn't connect to the server process")
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
protected
|
84
|
+
|
85
|
+
# Creates a client and gets it to try to send a message to the Server on
|
86
|
+
# @options[:socket]. If this doesn't succeed, either a Errno::EACCES,
|
87
|
+
# Errno::ENOENT or Errno::ECONNREFUSED will probably be raised.
|
88
|
+
def run_client
|
89
|
+
message "Trying to connect to server on #{@options[:socket]}..." if @options[:verbose]
|
90
|
+
Jobby::Client.new(@options[:socket]) { |client| client.send(@options[:input]) }
|
91
|
+
message "Client has run successfully!" if @options[:verbose]
|
92
|
+
end
|
93
|
+
|
94
|
+
# Creates a Proc object that will be run by any children that the Server forks.
|
95
|
+
# The input parameter that is passed to the Proc is the input string that a
|
96
|
+
# Client will send to the server.
|
97
|
+
def get_proc_from_options
|
98
|
+
if @options[:ruby].nil? and @options[:command].nil?
|
99
|
+
return error("No server found on #{@options[:socket]} and you didn't give --ruby or --command to execute")
|
100
|
+
|
101
|
+
elsif not @options[:ruby].nil? and not @options[:command].nil?
|
102
|
+
return error("You can only specify --ruby or --command, not both")
|
103
|
+
|
104
|
+
elsif @options[:ruby] # can be either some Ruby code or a filepath
|
105
|
+
if File.file?(File.expand_path(@options[:ruby]))
|
106
|
+
return lambda { |input, logger|
|
107
|
+
ARGV << input
|
108
|
+
# read and eval this rather than Kernel.load so that the code in the
|
109
|
+
# file can use the local variables in this block
|
110
|
+
instance_eval(File.read(File.expand_path(@options[:ruby])))
|
111
|
+
}
|
112
|
+
else
|
113
|
+
return lambda { |input, logger|
|
114
|
+
instance_eval(@options[:ruby])
|
115
|
+
}
|
116
|
+
end
|
117
|
+
|
118
|
+
elsif @options[:command]
|
119
|
+
return lambda { |input, logger|
|
120
|
+
exec(eval("\"#{@options[:command].gsub('"', '\"')}\""))
|
121
|
+
}
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
# Changes the user and group ownership of the current process if
|
126
|
+
# @options[:user] or @options[:group] are set. This might be a privileged
|
127
|
+
# operation.
|
128
|
+
def change_process_ownership
|
129
|
+
begin
|
130
|
+
if @options[:group]
|
131
|
+
message "Setting group ownership to #{@options[:group]}..." if @options[:verbose]
|
132
|
+
Process::GID.change_privilege Etc.getgrnam(@options[:group]).gid
|
133
|
+
end
|
134
|
+
if @options[:user]
|
135
|
+
message "Setting user ownership to #{@options[:user]}..." if @options[:verbose]
|
136
|
+
Process::UID.change_privilege Etc.getpwnam(@options[:user]).uid
|
137
|
+
end
|
138
|
+
rescue Errno::EPERM
|
139
|
+
return error("You don't have permission to change the process ownership - perhaps you should be root?")
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def error(text)
|
144
|
+
puts " ERROR - #{text}"
|
145
|
+
exit -1 if @options[:exit_on_error]
|
146
|
+
return false
|
147
|
+
end
|
148
|
+
|
149
|
+
def message(text)
|
150
|
+
puts " #{text}"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
data/lib/server.rb
ADDED
@@ -0,0 +1,192 @@
|
|
1
|
+
# Copyright (C) 2008 Mark Somerville
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2 of the License, or any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require 'fileutils'
|
18
|
+
require 'socket'
|
19
|
+
require 'logger'
|
20
|
+
require 'thread'
|
21
|
+
|
22
|
+
module Jobby
|
23
|
+
# This is a generic server class which accepts connections on a UNIX socket. On
|
24
|
+
# receiving a connection, the server process forks and runs the specified block.
|
25
|
+
#
|
26
|
+
# ==Example
|
27
|
+
#
|
28
|
+
# Jobby::Server.new("/tmp/jobby.socket", 3, "/var/log/jobby.log").run do
|
29
|
+
# # This code will be run in the forked children
|
30
|
+
# puts "#{Process.pid}: I'm toilet trained!"
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# ==Stopping the server
|
34
|
+
#
|
35
|
+
# A client process can send one of two special strings to stop the server.
|
36
|
+
#
|
37
|
+
# "||JOBBY FLUSH||" will stop the server forking any more children and shut
|
38
|
+
# it down.
|
39
|
+
#
|
40
|
+
# "||JOBBY WIPE||" will stop the server forking any more children, kill 9
|
41
|
+
# any existing children and shut it down.
|
42
|
+
#
|
43
|
+
# ==Log rotation
|
44
|
+
#
|
45
|
+
# The server can receive USR1 signals as notification that the logfile has been
|
46
|
+
# rotated. This will close and re-open the handle to the log file. Since the
|
47
|
+
# server process forks to produce children, they too can handle USR1 signals on
|
48
|
+
# log rotation.
|
49
|
+
#
|
50
|
+
# To tell all Jobby processes that the log file has been rotated, use something
|
51
|
+
# like:
|
52
|
+
#
|
53
|
+
# % pkill -USR1 -f jobby
|
54
|
+
#
|
55
|
+
class Server
|
56
|
+
def initialize(socket_path, max_forked_processes, log)
|
57
|
+
@socket_path = socket_path
|
58
|
+
@max_forked_processes = max_forked_processes.to_i
|
59
|
+
@log = log
|
60
|
+
@queue = Queue.new
|
61
|
+
end
|
62
|
+
|
63
|
+
# Starts the server and listens for connections. The specified block is run in
|
64
|
+
# the child processes. When a connection is received, the input parameter is
|
65
|
+
# immediately added to the queue.
|
66
|
+
def run(&block)
|
67
|
+
connect_to_socket_and_start_logging
|
68
|
+
unless block_given?
|
69
|
+
@logger.error "No block given, exiting"
|
70
|
+
terminate
|
71
|
+
end
|
72
|
+
start_forking_thread(block)
|
73
|
+
loop do
|
74
|
+
client = @socket.accept
|
75
|
+
input = ""
|
76
|
+
while bytes = client.read(128)
|
77
|
+
input += bytes
|
78
|
+
end
|
79
|
+
if input == "||JOBBY FLUSH||"
|
80
|
+
terminate
|
81
|
+
elsif input == "||JOBBY WIPE||"
|
82
|
+
terminate_children
|
83
|
+
terminate
|
84
|
+
else
|
85
|
+
@queue << input
|
86
|
+
end
|
87
|
+
client.close
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
protected
|
92
|
+
|
93
|
+
# Runs a thread to manage the forked processes. It will block, waiting for a
|
94
|
+
# child to finish if the maximum number of forked processes are already
|
95
|
+
# running. It will then, read from the queue and fork off a new process.
|
96
|
+
#
|
97
|
+
# The input variable that is passed to the block is the message that is
|
98
|
+
# received from Client#send.
|
99
|
+
def start_forking_thread(block)
|
100
|
+
Thread.new do
|
101
|
+
@pids = []
|
102
|
+
loop do
|
103
|
+
if @pids.length >= @max_forked_processes
|
104
|
+
begin
|
105
|
+
reap_child
|
106
|
+
rescue Errno::ECHILD
|
107
|
+
end
|
108
|
+
end
|
109
|
+
# fork and run code that performs the actual work
|
110
|
+
input = @queue.pop
|
111
|
+
@pids << fork do
|
112
|
+
$0 = "jobby: child" # set the process name
|
113
|
+
@logger.info "Child process started (#{Process.pid})"
|
114
|
+
block.call(input, @logger)
|
115
|
+
exit 0
|
116
|
+
end
|
117
|
+
Thread.new do
|
118
|
+
reap_child
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
# Checks if a process is already listening on the socket. If not, removes the
|
125
|
+
# socket file (if it's there) and starts a server. Throws an Errno::EADDRINUSE
|
126
|
+
# exception if an existing server is detected.
|
127
|
+
def connect_to_socket_and_start_logging
|
128
|
+
unless File.exists? @socket_path
|
129
|
+
connect_to_socket
|
130
|
+
else
|
131
|
+
begin
|
132
|
+
# test for a server on the socket
|
133
|
+
test_socket = UNIXSocket.open(@socket_path)
|
134
|
+
test_socket.close # got this far - seems like there is a server already
|
135
|
+
raise Errno::EADDRINUSE.new("it seems like there is already a server listening on #{@socket_path}")
|
136
|
+
rescue Errno::ECONNREFUSED
|
137
|
+
# probably not a server on that socket - start one
|
138
|
+
FileUtils.rm(@socket_path, :force => true)
|
139
|
+
connect_to_socket
|
140
|
+
end
|
141
|
+
end
|
142
|
+
start_logging
|
143
|
+
end
|
144
|
+
|
145
|
+
def connect_to_socket
|
146
|
+
@socket = UNIXServer.open(@socket_path)
|
147
|
+
FileUtils.chmod 0770, @socket_path
|
148
|
+
@socket.listen 10
|
149
|
+
end
|
150
|
+
|
151
|
+
def start_logging
|
152
|
+
@logger = Logger.new @log
|
153
|
+
@logger.info "Server started at #{Time.now}"
|
154
|
+
Signal.trap("USR1") do
|
155
|
+
rotate_log
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def reap_child
|
160
|
+
@pids.delete Process.wait
|
161
|
+
end
|
162
|
+
|
163
|
+
def rotate_log
|
164
|
+
@logger.close
|
165
|
+
@logger = Logger.new @log
|
166
|
+
@logger.info "USR1 received, rotating log file"
|
167
|
+
end
|
168
|
+
|
169
|
+
# Cleans up the server and exits the process with a return code 0.
|
170
|
+
def terminate
|
171
|
+
@queue.clear
|
172
|
+
@logger.info "Flush received - terminating server"
|
173
|
+
@socket.close
|
174
|
+
FileUtils.rm(@socket_path, :force => true)
|
175
|
+
exit! 0
|
176
|
+
end
|
177
|
+
|
178
|
+
# Stops any more children being forked and terminates the existing ones. A kill
|
179
|
+
# 9 signal is used as you will likely be run when termination is needed
|
180
|
+
# immediately, perhaps due to 'runaway' children.
|
181
|
+
def terminate_children
|
182
|
+
@queue.clear
|
183
|
+
@logger.info "Wipe received - terminating forked children"
|
184
|
+
@pids.each do |pid|
|
185
|
+
begin
|
186
|
+
Process.kill 9, pid
|
187
|
+
rescue Errno::ESRCH
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
data/spec/run_all.rb
ADDED
data/spec/server_spec.rb
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
require "#{File.dirname(__FILE__)}/../lib/server"
|
2
|
+
require "#{File.dirname(__FILE__)}/../lib/client"
|
3
|
+
require 'fileutils'
|
4
|
+
$0 = "jobby spec"
|
5
|
+
|
6
|
+
|
7
|
+
# Due to the multi-process nature of these specs, there are a bunch of sleep calls
|
8
|
+
# around. This is, of course, pretty brittle but I can't think of a better way of
|
9
|
+
# handling it. If they start failing 'randomly', I would first start by increasing
|
10
|
+
# the sleep times.
|
11
|
+
|
12
|
+
describe Jobby::Server do
|
13
|
+
|
14
|
+
def run_server(socket, max_child_processes, log_filepath, &block)
|
15
|
+
@server_pid = fork do
|
16
|
+
Jobby::Server.new(socket, max_child_processes, log_filepath).run(&block)
|
17
|
+
end
|
18
|
+
sleep 0.2
|
19
|
+
end
|
20
|
+
|
21
|
+
def terminate_server
|
22
|
+
Process.kill 15, @server_pid
|
23
|
+
if File.exists? @child_filepath
|
24
|
+
FileUtils.rm @child_filepath
|
25
|
+
end
|
26
|
+
FileUtils.rm @log_filepath, :force => true
|
27
|
+
sleep 0.5
|
28
|
+
end
|
29
|
+
|
30
|
+
before :all do
|
31
|
+
@socket = File.expand_path("#{File.dirname(__FILE__)}/jobby_server.sock")
|
32
|
+
@max_child_processes = 2
|
33
|
+
@log_filepath = File.expand_path("#{File.dirname(__FILE__)}/jobby_server.log")
|
34
|
+
@child_filepath = File.expand_path("#{File.dirname(__FILE__)}/jobby_child")
|
35
|
+
end
|
36
|
+
|
37
|
+
before :each do
|
38
|
+
run_server(@socket, @max_child_processes, @log_filepath) {
|
39
|
+
File.open(@child_filepath, "a+") do |file|
|
40
|
+
file << "#{Process.pid}"
|
41
|
+
end
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
after :each do
|
46
|
+
terminate_server
|
47
|
+
end
|
48
|
+
|
49
|
+
after :all do
|
50
|
+
FileUtils.rm @socket, :force => true
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should listen on a UNIX socket" do
|
54
|
+
lambda { UNIXSocket.open(@socket).close }.should_not raise_error
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should allow the children to log from within the called block" do
|
58
|
+
terminate_server
|
59
|
+
sleep 0.5
|
60
|
+
run_server(@socket, @max_child_processes, @log_filepath) { |input, logger|
|
61
|
+
logger.info "I can log!"
|
62
|
+
}
|
63
|
+
sleep 1
|
64
|
+
client_socket = UNIXSocket.open(@socket)
|
65
|
+
client_socket.send("hiya", 0)
|
66
|
+
client_socket.close
|
67
|
+
sleep 1
|
68
|
+
File.read(@log_filepath).should match(/I can log!/)
|
69
|
+
end
|
70
|
+
|
71
|
+
it "should throw an exception if there is already a process listening on the socket" do
|
72
|
+
lambda { Jobby::Server.new(@socket, @max_child_processes, @log_filepath).run { true } }.should raise_error(Errno::EADDRINUSE, "Address already in use - it seems like there is already a server listening on #{@socket}")
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should set the correct permissions on the socket file" do
|
76
|
+
`stat --format=%a,%F #{@socket}`.strip.should eql("770,socket")
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should log when it is started" do
|
80
|
+
File.read(@log_filepath).should match(/Server started at/)
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should be able to accept an IO object instead of a log filepath" do
|
84
|
+
terminate_server
|
85
|
+
sleep 1
|
86
|
+
io_filepath = File.expand_path("#{File.dirname(__FILE__)}/io_log_test.log")
|
87
|
+
FileUtils.rm io_filepath, :force => true
|
88
|
+
io = File.open(io_filepath, "a+")
|
89
|
+
run_server(@socket, @max_child_processes, io) {}
|
90
|
+
terminate_server
|
91
|
+
sleep 0.5
|
92
|
+
File.readlines(io_filepath).length.should eql(1)
|
93
|
+
FileUtils.rm io_filepath
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should flush and reload the log file when it receieves the USR1 signal" do
|
97
|
+
FileUtils.rm @log_filepath
|
98
|
+
Process.kill "USR1", @server_pid
|
99
|
+
sleep 0.2
|
100
|
+
File.read(@log_filepath).should match(/USR1 received, rotating log file/)
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should not run if a block is not given" do
|
104
|
+
terminate_server
|
105
|
+
sleep 0.5
|
106
|
+
run_server(@socket, @max_child_processes, @log_filepath)
|
107
|
+
sleep 0.5
|
108
|
+
lambda { UNIXSocket.open(@socket).close }.should raise_error
|
109
|
+
end
|
110
|
+
|
111
|
+
it "should read all of the provided message" do
|
112
|
+
terminate_server
|
113
|
+
sleep 0.5
|
114
|
+
run_server(@socket, @max_child_processes, @log_filepath) { |input, logger|
|
115
|
+
File.open(@child_filepath, "a+") do |file|
|
116
|
+
file << "#{input}"
|
117
|
+
end
|
118
|
+
}
|
119
|
+
Jobby::Client.new(@socket) { |c| c.send("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890") }
|
120
|
+
sleep 0.5
|
121
|
+
File.read(@child_filepath).should eql("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890")
|
122
|
+
end
|
123
|
+
|
124
|
+
it "should fork off a child and run the specified code when it receives a connection" do
|
125
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
126
|
+
sleep 0.2
|
127
|
+
File.read(@child_filepath).should_not eql(@server_pid.to_s)
|
128
|
+
end
|
129
|
+
|
130
|
+
it "should only fork off a certain number of children - the others should have to wait (in an internal queue)" do
|
131
|
+
terminate_server
|
132
|
+
run_server(@socket, @max_child_processes, @log_filepath) do
|
133
|
+
sleep 2
|
134
|
+
File.open(@child_filepath, "a+") do |file|
|
135
|
+
file << "#{Process.pid}\n"
|
136
|
+
end
|
137
|
+
end
|
138
|
+
(@max_child_processes + 2).times do |i|
|
139
|
+
Thread.new do
|
140
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
141
|
+
end
|
142
|
+
end
|
143
|
+
sleep 2.5
|
144
|
+
File.readlines(@child_filepath).length.should eql(@max_child_processes)
|
145
|
+
sleep 4
|
146
|
+
File.readlines(@child_filepath).length.should eql(@max_child_processes + 2)
|
147
|
+
end
|
148
|
+
|
149
|
+
it "should receive a flush command from the client and terminate while the children continue processing" do
|
150
|
+
terminate_server
|
151
|
+
sleep 1
|
152
|
+
run_server(@socket, 1, @log_filepath) do
|
153
|
+
sleep 2
|
154
|
+
end
|
155
|
+
2.times do |i|
|
156
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
157
|
+
end
|
158
|
+
sleep 1
|
159
|
+
Jobby::Client.new(@socket) { |c| c.send("||JOBBY FLUSH||") }
|
160
|
+
sleep 1.5
|
161
|
+
lambda { Jobby::Client.new(@socket) { |c| c.send("hello?") } }.should raise_error(Errno::ENOENT)
|
162
|
+
`pgrep -f 'jobby spec' | wc -l`.strip.should eql("2")
|
163
|
+
end
|
164
|
+
|
165
|
+
it "should receive a wipe command from the client and terminate, taking the children with it" do
|
166
|
+
terminate_server
|
167
|
+
run_server(@socket, 1, @log_filepath) do
|
168
|
+
sleep 2
|
169
|
+
end
|
170
|
+
2.times do |i|
|
171
|
+
Jobby::Client.new(@socket) { |c| c.send("hiya") }
|
172
|
+
end
|
173
|
+
sleep 1
|
174
|
+
Jobby::Client.new(@socket) { |c| c.send("||JOBBY WIPE||") }
|
175
|
+
sleep 2.5
|
176
|
+
lambda { Jobby::Client.new(@socket) { |c| c.send("hello?") } }.should raise_error(Errno::ENOENT)
|
177
|
+
`pgrep -f 'jobby spec'`.strip.should eql("#{Process.pid}")
|
178
|
+
end
|
179
|
+
end
|
metadata
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: jobby
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Mark Somerville
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2008-06-17 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description:
|
17
|
+
email: mark@scottishclimbs.com
|
18
|
+
executables:
|
19
|
+
- jobby
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files:
|
23
|
+
- README
|
24
|
+
files:
|
25
|
+
- bin/jobby
|
26
|
+
- lib/runner.rb
|
27
|
+
- lib/server.rb
|
28
|
+
- lib/client.rb
|
29
|
+
- spec/server_spec.rb
|
30
|
+
- spec/run_all.rb
|
31
|
+
- README
|
32
|
+
has_rdoc: true
|
33
|
+
homepage: http://mark.scottishclimbs.com/
|
34
|
+
post_install_message:
|
35
|
+
rdoc_options: []
|
36
|
+
|
37
|
+
require_paths:
|
38
|
+
- lib
|
39
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: "0"
|
44
|
+
version:
|
45
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - ">="
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: "0"
|
50
|
+
version:
|
51
|
+
requirements: []
|
52
|
+
|
53
|
+
rubyforge_project: jobby
|
54
|
+
rubygems_version: 1.0.1
|
55
|
+
signing_key:
|
56
|
+
specification_version: 2
|
57
|
+
summary: Jobby is a small utility and library for managing running jobs in concurrent processes.
|
58
|
+
test_files: []
|
59
|
+
|