spawn_server 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.document ADDED
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
data/.gitignore ADDED
@@ -0,0 +1,5 @@
1
+ *.sw?
2
+ .DS_Store
3
+ coverage
4
+ rdoc
5
+ pkg
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2009 Jeff Moss
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.rdoc ADDED
@@ -0,0 +1,18 @@
1
+ = spawn_server
2
+
3
+ Description goes here.
4
+
5
+ == Note on Patches/Pull Requests
6
+
7
+ * Fork the project.
8
+ * Make your feature addition or bug fix.
9
+ * Add tests for it. This is important so I don't break it in a
10
+ future version unintentionally.
11
+ * Commit, do not mess with rakefile, version, or history.
12
+ (if you want to have your own version, that is fine but
13
+ bump version in a commit by itself I can ignore when I pull)
14
+ * Send me a pull request. Bonus points for topic branches.
15
+
16
+ == Copyright
17
+
18
+ Copyright (c) 2010 Jeff Moss. See LICENSE for details.
data/Rakefile ADDED
@@ -0,0 +1,58 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "spawn_server"
8
+ gem.summary = %Q{TODO: one-line summary of your gem}
9
+ gem.description = %Q{TODO: longer description of your gem}
10
+ gem.email = "jeff.moss@matchbin.com"
11
+ gem.homepage = "http://github.com/zardinuk/spawn_server"
12
+ gem.authors = ["Jeff Moss"]
13
+ gem.add_development_dependency "thoughtbot-shoulda"
14
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
15
+ gem.add_dependency "spawn"
16
+ gem.add_dependency "log4r"
17
+ end
18
+ rescue LoadError
19
+ puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
20
+ end
21
+
22
+ require 'rake/testtask'
23
+ Rake::TestTask.new(:test) do |test|
24
+ test.libs << 'lib' << 'test'
25
+ test.pattern = 'test/**/*_test.rb'
26
+ test.verbose = true
27
+ end
28
+
29
+ begin
30
+ require 'rcov/rcovtask'
31
+ Rcov::RcovTask.new do |test|
32
+ test.libs << 'test'
33
+ test.pattern = 'test/**/*_test.rb'
34
+ test.verbose = true
35
+ end
36
+ rescue LoadError
37
+ task :rcov do
38
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
39
+ end
40
+ end
41
+
42
+ task :test => :check_dependencies
43
+
44
+ task :default => :test
45
+
46
+ require 'rake/rdoctask'
47
+ Rake::RDocTask.new do |rdoc|
48
+ if File.exist?('VERSION')
49
+ version = File.read('VERSION')
50
+ else
51
+ version = ""
52
+ end
53
+
54
+ rdoc.rdoc_dir = 'rdoc'
55
+ rdoc.title = "spawn_server #{version}"
56
+ rdoc.rdoc_files.include('README*')
57
+ rdoc.rdoc_files.include('lib/**/*.rb')
58
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.0
@@ -0,0 +1,210 @@
1
+ require 'logger'
2
+ require 'spawn/task'
3
+
4
+ module Spawn
5
+ class Server
6
+ POLLING_INTERVAL = 60
7
+
8
+ attr_accessor :pids
9
+ attr_accessor :tasks
10
+ attr_accessor :interval
11
+ attr_accessor :start_times
12
+
13
+ def initialize(tasks, interval=POLLING_INTERVAL)
14
+ puts "Initializing spawn server"
15
+
16
+ self.pids = {}
17
+ self.start_times = {}
18
+ self.tasks = tasks
19
+ self.interval = interval
20
+
21
+ # trap INT for debugging, in production we want to leave children active
22
+ Signal.trap('INT') do
23
+ process_list = self.pids.map{|k,v| "#{k} (#{v})" }.join(', ')
24
+ print " *** Interrupt received, killing #{process_list} ***\n"
25
+ Process.kill('INT', *self.pids.values.map{|h| h.keys }.flatten )
26
+ exit(1)
27
+ end
28
+
29
+ start_tasks
30
+ end
31
+
32
+ def logger
33
+ if instance_variables.include?(:@logger)
34
+ @logger
35
+ else
36
+ @logger = Logger.new(File.open('log/spawn_server.log', 'w'))
37
+ @logger.level = Logger::DEBUG
38
+ @logger
39
+ end
40
+ end
41
+
42
+ def start_tasks
43
+ puts self.tasks.inspect
44
+
45
+ # first load in pid files
46
+ self.tasks.each do |id, params|
47
+ self.pids[id] = {}
48
+
49
+ params[:max_threads].times do |thread_num|
50
+ thread_pid_file = pid_file(id, thread_num+1)
51
+
52
+ puts thread_pid_file
53
+
54
+ if File.exists?(thread_pid_file)
55
+ pid = File.open(thread_pid_file).read.to_i
56
+ if (Process.kill(0, pid) rescue false)
57
+ self.pids[id][pid] = File.stat(thread_pid_file).mtime
58
+ else
59
+ File.unlink(thread_pid_file) rescue nil
60
+ end
61
+ end
62
+ end
63
+
64
+ if params[:reload]
65
+ case params[:reload]
66
+ when :all
67
+ self.pids[id].each do |pid,mtime|
68
+ process_stop(id, pid, true)
69
+ end
70
+ when :parent
71
+ self.pids[id].each do |pid,mtime|
72
+ process_stop(id, pid, false)
73
+ end
74
+ else
75
+ puts "Unrecognized value for :reload parameter in #{id}: #{params[:reload]}"
76
+ end
77
+ end
78
+ end
79
+
80
+ loop do
81
+ Thread.new do
82
+ self.tasks.each do |id, params|
83
+ puts "Checking if #{id} is running"
84
+
85
+ if (!process_running?(id, params[:max_threads]))
86
+ puts "It's not, starting..."
87
+ # not running, or not enough running, start it...
88
+ process_start(id, params) do
89
+ if params[:task].is_a?(String)
90
+ Rake::Task[params[:task]].invoke
91
+ elsif params[:task].is_a?(Proc)
92
+ puts "Initializing task #{id}"
93
+ params[:task].call
94
+ end
95
+ exit
96
+ end
97
+ end
98
+
99
+ # check if the processes are running too long, this needs to be done after "process_running" is called so that the pid list is cleared, it's not optimal this way but good enough for now
100
+ self.pids[id].each do |pid,mtime|
101
+ if params[:max_life] && params[:max_life] < time_running(id,pid)
102
+ puts "Before process stop (#{pid}):\n#{process_list}"
103
+ process_stop(id,pid)
104
+ puts "After process stop (#{pid}):\n#{process_list}"
105
+
106
+ # BackgroundCheck.deliver_process_max_life(id, time_running(id,pid), params[:max_life]) rescue nil
107
+ end
108
+ end
109
+ end
110
+ end
111
+
112
+ # Run the above code in a thread so that the interval is more consistent
113
+ sleep(self.interval)
114
+ end
115
+ end
116
+
117
+ def process_list
118
+ `ps uxf`
119
+ end
120
+
121
+ def time_running(id, pid)
122
+ if self.pids[id][pid]
123
+ Time.now - self.pids[id][pid]
124
+ end
125
+ end
126
+
127
+ def pid_file(id, num=1)
128
+ "tmp/#{id}.#{num}.pid"
129
+ end
130
+
131
+ def process_running?(id, quota=1)
132
+ running = 0
133
+ process_list = self.pids[id]
134
+ if process_list && process_list.is_a?(Hash)
135
+ process_list.each do |pid,mtime|
136
+ # do this first to wipe the defunct children, won't block
137
+ # Process.wait(pid, Process::WNOHANG)
138
+
139
+ if (Process.kill(0, pid) rescue false)
140
+ running += 1
141
+ else
142
+ # look for this pid file and remove it
143
+ self.tasks[id][:max_threads].times do |i|
144
+ thread_pid_file = pid_file(id,i+1)
145
+ if File.exists?(thread_pid_file)
146
+ existing_pid = File.open(thread_pid_file).read
147
+ if existing_pid.to_i == pid.to_i
148
+ File.unlink(thread_pid_file)
149
+ end
150
+ end
151
+ end
152
+ self.pids[id].delete(pid)
153
+ end
154
+ end
155
+ end
156
+
157
+ return (running >= quota)
158
+ end
159
+
160
+ def process_stop(id, pid, recursive=true)
161
+ return unless pid
162
+
163
+ if recursive
164
+ # recursive, look through all the stat files for this one's child processes and call recursively
165
+ Dir.glob('/proc/*/status').each do |stat_file|
166
+ File.open(stat_file).each do |line|
167
+ if line =~ /^PPid:\s*#{pid}$/
168
+ process_stop(id, stat_file.match(/[0-9]+/)[0].to_i)
169
+ break
170
+ end
171
+ end
172
+ end
173
+ end
174
+
175
+ puts "Stopping process #{pid} (#{id})"
176
+ Process.kill(9, pid) rescue nil
177
+ end
178
+
179
+ def process_start(id, params={})
180
+ pid = Spawn::Task.new do
181
+ CACHE.reset if defined?(CACHE)
182
+ ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
183
+ yield
184
+ end
185
+ pid = pid.handle.to_i
186
+
187
+ puts "Started process #{pid} (#{id})"
188
+ begin
189
+ params[:max_threads].times do |thread_num|
190
+ thread_pid_file = pid_file(id, thread_num+1)
191
+ if File.exists?(thread_pid_file)
192
+ # this pid must be running still, cause we must have just checked
193
+ else
194
+ File.open(thread_pid_file, 'w'){|f| f << "#{pid}" }
195
+ break
196
+ end
197
+ end
198
+ rescue Exception => e
199
+ # email this log message
200
+ puts e.message
201
+ end
202
+ self.pids[id][pid] = Time.now
203
+
204
+ if params[:priority]
205
+ Process.setpriority(Process::PRIO_PROCESS, pid, params[:priority])
206
+ end
207
+ end
208
+
209
+ end
210
+ end
data/lib/spawn/task.rb ADDED
@@ -0,0 +1,68 @@
1
+ module Spawn
2
+ class Task
3
+ attr_accessor :handle
4
+
5
+ # socket to close in child process
6
+ @@resources = []
7
+
8
+ # set the resource to disconnect from in the child process (when forking)
9
+ def self.resource_to_close(resource)
10
+ @@resources << resource
11
+ end
12
+
13
+ # close all the resources added by calls to resource_to_close
14
+ def self.close_resources
15
+ @@resources.each do |resource|
16
+ resource.close if resource && resource.respond_to?(:close) && !resource.closed?
17
+ end
18
+ end
19
+
20
+ # Spawns a long-running section of code and returns the ID of the spawned process.
21
+ def initialize(options={})
22
+ self.handle = fork_it(options) { yield }
23
+ end
24
+
25
+ def wait(sids = [])
26
+ # wait for all threads and/or forks (if a single sid passed in, convert to array first)
27
+ Array(sids).each do |sid|
28
+ if sid.type == :thread
29
+ sid.handle.join()
30
+ else
31
+ begin
32
+ Process.wait(sid.handle)
33
+ rescue
34
+ # if the process is already done, ignore the error
35
+ end
36
+ end
37
+ end
38
+ end
39
+
40
+ protected
41
+ def fork_it(options)
42
+ child = fork do
43
+ begin
44
+ start = Time.now
45
+
46
+ # set the nice priority if needed
47
+ Process.setpriority(Process::PRIO_PROCESS, 0, options[:nice]) if options[:nice]
48
+
49
+ # disconnect from the listening socket, et al
50
+ self.class.close_resources
51
+
52
+ # run the block of code that takes so long
53
+ yield
54
+
55
+ rescue => ex
56
+ $stderr.puts "spawn> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message} #{ex.backtrace}"
57
+ ensure
58
+ exit!(0)
59
+ end
60
+ end
61
+
62
+ # detach from child process (parent may still wait for detached process if they wish)
63
+ Process.detach(child)
64
+
65
+ return child
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,2 @@
1
+ require 'spawn/task'
2
+ require 'spawn/server'
@@ -0,0 +1,7 @@
1
+ require 'test_helper'
2
+
3
+ class SpawnServerTest < Test::Unit::TestCase
4
+ should "probably rename this file and start testing for real" do
5
+ flunk "hey buddy, you should probably rename this file and start testing for real"
6
+ end
7
+ end
@@ -0,0 +1,10 @@
1
+ require 'rubygems'
2
+ require 'test/unit'
3
+ require 'shoulda'
4
+
5
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
6
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
7
+ require 'spawn_server'
8
+
9
+ class Test::Unit::TestCase
10
+ end
metadata ADDED
@@ -0,0 +1,92 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: spawn_server
3
+ version: !ruby/object:Gem::Version
4
+ hash: 23
5
+ prerelease: false
6
+ segments:
7
+ - 1
8
+ - 0
9
+ - 0
10
+ version: 1.0.0
11
+ platform: ruby
12
+ authors:
13
+ - Jeff Moss
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2010-09-07 00:00:00 -06:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: thoughtbot-shoulda
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ hash: 3
30
+ segments:
31
+ - 0
32
+ version: "0"
33
+ type: :development
34
+ version_requirements: *id001
35
+ description: Spawns processes using fork call, monitors them and other neat things.
36
+ email: jeff.moss@matchbin.com
37
+ executables: []
38
+
39
+ extensions: []
40
+
41
+ extra_rdoc_files:
42
+ - LICENSE
43
+ - README.rdoc
44
+ files:
45
+ - .document
46
+ - .gitignore
47
+ - LICENSE
48
+ - README.rdoc
49
+ - Rakefile
50
+ - VERSION
51
+ - lib/spawn/server.rb
52
+ - lib/spawn/task.rb
53
+ - lib/spawn_server.rb
54
+ - test/spawn_server_test.rb
55
+ - test/test_helper.rb
56
+ has_rdoc: true
57
+ homepage: http://github.com/zardinuk/spawn_server
58
+ licenses: []
59
+
60
+ post_install_message:
61
+ rdoc_options:
62
+ - --charset=UTF-8
63
+ require_paths:
64
+ - lib
65
+ required_ruby_version: !ruby/object:Gem::Requirement
66
+ none: false
67
+ requirements:
68
+ - - ">="
69
+ - !ruby/object:Gem::Version
70
+ hash: 3
71
+ segments:
72
+ - 0
73
+ version: "0"
74
+ required_rubygems_version: !ruby/object:Gem::Requirement
75
+ none: false
76
+ requirements:
77
+ - - ">="
78
+ - !ruby/object:Gem::Version
79
+ hash: 3
80
+ segments:
81
+ - 0
82
+ version: "0"
83
+ requirements: []
84
+
85
+ rubyforge_project:
86
+ rubygems_version: 1.3.7
87
+ signing_key:
88
+ specification_version: 3
89
+ summary: Forks new processes and monitors them and other neat things
90
+ test_files:
91
+ - test/test_helper.rb
92
+ - test/spawn_server_test.rb