spawn_server 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +5 -0
- data/LICENSE +20 -0
- data/README.rdoc +18 -0
- data/Rakefile +58 -0
- data/VERSION +1 -0
- data/lib/spawn/server.rb +210 -0
- data/lib/spawn/task.rb +68 -0
- data/lib/spawn_server.rb +2 -0
- data/test/spawn_server_test.rb +7 -0
- data/test/test_helper.rb +10 -0
- metadata +92 -0
data/.document
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Jeff Moss
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
= spawn_server
|
2
|
+
|
3
|
+
Description goes here.
|
4
|
+
|
5
|
+
== Note on Patches/Pull Requests
|
6
|
+
|
7
|
+
* Fork the project.
|
8
|
+
* Make your feature addition or bug fix.
|
9
|
+
* Add tests for it. This is important so I don't break it in a
|
10
|
+
future version unintentionally.
|
11
|
+
* Commit, do not mess with rakefile, version, or history.
|
12
|
+
(if you want to have your own version, that is fine but
|
13
|
+
bump version in a commit by itself I can ignore when I pull)
|
14
|
+
* Send me a pull request. Bonus points for topic branches.
|
15
|
+
|
16
|
+
== Copyright
|
17
|
+
|
18
|
+
Copyright (c) 2010 Jeff Moss. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "spawn_server"
|
8
|
+
gem.summary = %Q{TODO: one-line summary of your gem}
|
9
|
+
gem.description = %Q{TODO: longer description of your gem}
|
10
|
+
gem.email = "jeff.moss@matchbin.com"
|
11
|
+
gem.homepage = "http://github.com/zardinuk/spawn_server"
|
12
|
+
gem.authors = ["Jeff Moss"]
|
13
|
+
gem.add_development_dependency "thoughtbot-shoulda"
|
14
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
15
|
+
gem.add_dependency "spawn"
|
16
|
+
gem.add_dependency "log4r"
|
17
|
+
end
|
18
|
+
rescue LoadError
|
19
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
20
|
+
end
|
21
|
+
|
22
|
+
require 'rake/testtask'
|
23
|
+
Rake::TestTask.new(:test) do |test|
|
24
|
+
test.libs << 'lib' << 'test'
|
25
|
+
test.pattern = 'test/**/*_test.rb'
|
26
|
+
test.verbose = true
|
27
|
+
end
|
28
|
+
|
29
|
+
begin
|
30
|
+
require 'rcov/rcovtask'
|
31
|
+
Rcov::RcovTask.new do |test|
|
32
|
+
test.libs << 'test'
|
33
|
+
test.pattern = 'test/**/*_test.rb'
|
34
|
+
test.verbose = true
|
35
|
+
end
|
36
|
+
rescue LoadError
|
37
|
+
task :rcov do
|
38
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
task :test => :check_dependencies
|
43
|
+
|
44
|
+
task :default => :test
|
45
|
+
|
46
|
+
require 'rake/rdoctask'
|
47
|
+
Rake::RDocTask.new do |rdoc|
|
48
|
+
if File.exist?('VERSION')
|
49
|
+
version = File.read('VERSION')
|
50
|
+
else
|
51
|
+
version = ""
|
52
|
+
end
|
53
|
+
|
54
|
+
rdoc.rdoc_dir = 'rdoc'
|
55
|
+
rdoc.title = "spawn_server #{version}"
|
56
|
+
rdoc.rdoc_files.include('README*')
|
57
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
58
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.0
|
data/lib/spawn/server.rb
ADDED
@@ -0,0 +1,210 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'spawn/task'
|
3
|
+
|
4
|
+
module Spawn
|
5
|
+
class Server
|
6
|
+
POLLING_INTERVAL = 60
|
7
|
+
|
8
|
+
attr_accessor :pids
|
9
|
+
attr_accessor :tasks
|
10
|
+
attr_accessor :interval
|
11
|
+
attr_accessor :start_times
|
12
|
+
|
13
|
+
def initialize(tasks, interval=POLLING_INTERVAL)
|
14
|
+
puts "Initializing spawn server"
|
15
|
+
|
16
|
+
self.pids = {}
|
17
|
+
self.start_times = {}
|
18
|
+
self.tasks = tasks
|
19
|
+
self.interval = interval
|
20
|
+
|
21
|
+
# trap INT for debugging, in production we want to leave children active
|
22
|
+
Signal.trap('INT') do
|
23
|
+
process_list = self.pids.map{|k,v| "#{k} (#{v})" }.join(', ')
|
24
|
+
print " *** Interrupt received, killing #{process_list} ***\n"
|
25
|
+
Process.kill('INT', *self.pids.values.map{|h| h.keys }.flatten )
|
26
|
+
exit(1)
|
27
|
+
end
|
28
|
+
|
29
|
+
start_tasks
|
30
|
+
end
|
31
|
+
|
32
|
+
def logger
|
33
|
+
if instance_variables.include?(:@logger)
|
34
|
+
@logger
|
35
|
+
else
|
36
|
+
@logger = Logger.new(File.open('log/spawn_server.log', 'w'))
|
37
|
+
@logger.level = Logger::DEBUG
|
38
|
+
@logger
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def start_tasks
|
43
|
+
puts self.tasks.inspect
|
44
|
+
|
45
|
+
# first load in pid files
|
46
|
+
self.tasks.each do |id, params|
|
47
|
+
self.pids[id] = {}
|
48
|
+
|
49
|
+
params[:max_threads].times do |thread_num|
|
50
|
+
thread_pid_file = pid_file(id, thread_num+1)
|
51
|
+
|
52
|
+
puts thread_pid_file
|
53
|
+
|
54
|
+
if File.exists?(thread_pid_file)
|
55
|
+
pid = File.open(thread_pid_file).read.to_i
|
56
|
+
if (Process.kill(0, pid) rescue false)
|
57
|
+
self.pids[id][pid] = File.stat(thread_pid_file).mtime
|
58
|
+
else
|
59
|
+
File.unlink(thread_pid_file) rescue nil
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
if params[:reload]
|
65
|
+
case params[:reload]
|
66
|
+
when :all
|
67
|
+
self.pids[id].each do |pid,mtime|
|
68
|
+
process_stop(id, pid, true)
|
69
|
+
end
|
70
|
+
when :parent
|
71
|
+
self.pids[id].each do |pid,mtime|
|
72
|
+
process_stop(id, pid, false)
|
73
|
+
end
|
74
|
+
else
|
75
|
+
puts "Unrecognized value for :reload parameter in #{id}: #{params[:reload]}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
loop do
|
81
|
+
Thread.new do
|
82
|
+
self.tasks.each do |id, params|
|
83
|
+
puts "Checking if #{id} is running"
|
84
|
+
|
85
|
+
if (!process_running?(id, params[:max_threads]))
|
86
|
+
puts "It's not, starting..."
|
87
|
+
# not running, or not enough running, start it...
|
88
|
+
process_start(id, params) do
|
89
|
+
if params[:task].is_a?(String)
|
90
|
+
Rake::Task[params[:task]].invoke
|
91
|
+
elsif params[:task].is_a?(Proc)
|
92
|
+
puts "Initializing task #{id}"
|
93
|
+
params[:task].call
|
94
|
+
end
|
95
|
+
exit
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# check if the processes are running too long, this needs to be done after "process_running" is called so that the pid list is cleared, it's not optimal this way but good enough for now
|
100
|
+
self.pids[id].each do |pid,mtime|
|
101
|
+
if params[:max_life] && params[:max_life] < time_running(id,pid)
|
102
|
+
puts "Before process stop (#{pid}):\n#{process_list}"
|
103
|
+
process_stop(id,pid)
|
104
|
+
puts "After process stop (#{pid}):\n#{process_list}"
|
105
|
+
|
106
|
+
# BackgroundCheck.deliver_process_max_life(id, time_running(id,pid), params[:max_life]) rescue nil
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
# Run the above code in a thread so that the interval is more consistent
|
113
|
+
sleep(self.interval)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def process_list
|
118
|
+
`ps uxf`
|
119
|
+
end
|
120
|
+
|
121
|
+
def time_running(id, pid)
|
122
|
+
if self.pids[id][pid]
|
123
|
+
Time.now - self.pids[id][pid]
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def pid_file(id, num=1)
|
128
|
+
"tmp/#{id}.#{num}.pid"
|
129
|
+
end
|
130
|
+
|
131
|
+
def process_running?(id, quota=1)
|
132
|
+
running = 0
|
133
|
+
process_list = self.pids[id]
|
134
|
+
if process_list && process_list.is_a?(Hash)
|
135
|
+
process_list.each do |pid,mtime|
|
136
|
+
# do this first to wipe the defunct children, won't block
|
137
|
+
# Process.wait(pid, Process::WNOHANG)
|
138
|
+
|
139
|
+
if (Process.kill(0, pid) rescue false)
|
140
|
+
running += 1
|
141
|
+
else
|
142
|
+
# look for this pid file and remove it
|
143
|
+
self.tasks[id][:max_threads].times do |i|
|
144
|
+
thread_pid_file = pid_file(id,i+1)
|
145
|
+
if File.exists?(thread_pid_file)
|
146
|
+
existing_pid = File.open(thread_pid_file).read
|
147
|
+
if existing_pid.to_i == pid.to_i
|
148
|
+
File.unlink(thread_pid_file)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
self.pids[id].delete(pid)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
return (running >= quota)
|
158
|
+
end
|
159
|
+
|
160
|
+
def process_stop(id, pid, recursive=true)
|
161
|
+
return unless pid
|
162
|
+
|
163
|
+
if recursive
|
164
|
+
# recursive, look through all the stat files for this one's child processes and call recursively
|
165
|
+
Dir.glob('/proc/*/status').each do |stat_file|
|
166
|
+
File.open(stat_file).each do |line|
|
167
|
+
if line =~ /^PPid:\s*#{pid}$/
|
168
|
+
process_stop(id, stat_file.match(/[0-9]+/)[0].to_i)
|
169
|
+
break
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
puts "Stopping process #{pid} (#{id})"
|
176
|
+
Process.kill(9, pid) rescue nil
|
177
|
+
end
|
178
|
+
|
179
|
+
def process_start(id, params={})
|
180
|
+
pid = Spawn::Task.new do
|
181
|
+
CACHE.reset if defined?(CACHE)
|
182
|
+
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
183
|
+
yield
|
184
|
+
end
|
185
|
+
pid = pid.handle.to_i
|
186
|
+
|
187
|
+
puts "Started process #{pid} (#{id})"
|
188
|
+
begin
|
189
|
+
params[:max_threads].times do |thread_num|
|
190
|
+
thread_pid_file = pid_file(id, thread_num+1)
|
191
|
+
if File.exists?(thread_pid_file)
|
192
|
+
# this pid must be running still, cause we must have just checked
|
193
|
+
else
|
194
|
+
File.open(thread_pid_file, 'w'){|f| f << "#{pid}" }
|
195
|
+
break
|
196
|
+
end
|
197
|
+
end
|
198
|
+
rescue Exception => e
|
199
|
+
# email this log message
|
200
|
+
puts e.message
|
201
|
+
end
|
202
|
+
self.pids[id][pid] = Time.now
|
203
|
+
|
204
|
+
if params[:priority]
|
205
|
+
Process.setpriority(Process::PRIO_PROCESS, pid, params[:priority])
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
end
|
210
|
+
end
|
data/lib/spawn/task.rb
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
module Spawn
|
2
|
+
class Task
|
3
|
+
attr_accessor :handle
|
4
|
+
|
5
|
+
# socket to close in child process
|
6
|
+
@@resources = []
|
7
|
+
|
8
|
+
# set the resource to disconnect from in the child process (when forking)
|
9
|
+
def self.resource_to_close(resource)
|
10
|
+
@@resources << resource
|
11
|
+
end
|
12
|
+
|
13
|
+
# close all the resources added by calls to resource_to_close
|
14
|
+
def self.close_resources
|
15
|
+
@@resources.each do |resource|
|
16
|
+
resource.close if resource && resource.respond_to?(:close) && !resource.closed?
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# Spawns a long-running section of code and returns the ID of the spawned process.
|
21
|
+
def initialize(options={})
|
22
|
+
self.handle = fork_it(options) { yield }
|
23
|
+
end
|
24
|
+
|
25
|
+
def wait(sids = [])
|
26
|
+
# wait for all threads and/or forks (if a single sid passed in, convert to array first)
|
27
|
+
Array(sids).each do |sid|
|
28
|
+
if sid.type == :thread
|
29
|
+
sid.handle.join()
|
30
|
+
else
|
31
|
+
begin
|
32
|
+
Process.wait(sid.handle)
|
33
|
+
rescue
|
34
|
+
# if the process is already done, ignore the error
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
protected
|
41
|
+
def fork_it(options)
|
42
|
+
child = fork do
|
43
|
+
begin
|
44
|
+
start = Time.now
|
45
|
+
|
46
|
+
# set the nice priority if needed
|
47
|
+
Process.setpriority(Process::PRIO_PROCESS, 0, options[:nice]) if options[:nice]
|
48
|
+
|
49
|
+
# disconnect from the listening socket, et al
|
50
|
+
self.class.close_resources
|
51
|
+
|
52
|
+
# run the block of code that takes so long
|
53
|
+
yield
|
54
|
+
|
55
|
+
rescue => ex
|
56
|
+
$stderr.puts "spawn> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message} #{ex.backtrace}"
|
57
|
+
ensure
|
58
|
+
exit!(0)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# detach from child process (parent may still wait for detached process if they wish)
|
63
|
+
Process.detach(child)
|
64
|
+
|
65
|
+
return child
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/spawn_server.rb
ADDED
data/test/test_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: spawn_server
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Jeff Moss
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-09-07 00:00:00 -06:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: thoughtbot-shoulda
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
33
|
+
type: :development
|
34
|
+
version_requirements: *id001
|
35
|
+
description: Spawns processes using fork call, monitors them and other neat things.
|
36
|
+
email: jeff.moss@matchbin.com
|
37
|
+
executables: []
|
38
|
+
|
39
|
+
extensions: []
|
40
|
+
|
41
|
+
extra_rdoc_files:
|
42
|
+
- LICENSE
|
43
|
+
- README.rdoc
|
44
|
+
files:
|
45
|
+
- .document
|
46
|
+
- .gitignore
|
47
|
+
- LICENSE
|
48
|
+
- README.rdoc
|
49
|
+
- Rakefile
|
50
|
+
- VERSION
|
51
|
+
- lib/spawn/server.rb
|
52
|
+
- lib/spawn/task.rb
|
53
|
+
- lib/spawn_server.rb
|
54
|
+
- test/spawn_server_test.rb
|
55
|
+
- test/test_helper.rb
|
56
|
+
has_rdoc: true
|
57
|
+
homepage: http://github.com/zardinuk/spawn_server
|
58
|
+
licenses: []
|
59
|
+
|
60
|
+
post_install_message:
|
61
|
+
rdoc_options:
|
62
|
+
- --charset=UTF-8
|
63
|
+
require_paths:
|
64
|
+
- lib
|
65
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
hash: 3
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
hash: 3
|
80
|
+
segments:
|
81
|
+
- 0
|
82
|
+
version: "0"
|
83
|
+
requirements: []
|
84
|
+
|
85
|
+
rubyforge_project:
|
86
|
+
rubygems_version: 1.3.7
|
87
|
+
signing_key:
|
88
|
+
specification_version: 3
|
89
|
+
summary: Forks new processes and monitors them and other neat things
|
90
|
+
test_files:
|
91
|
+
- test/test_helper.rb
|
92
|
+
- test/spawn_server_test.rb
|