spawnling 2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG +55 -0
- data/LICENSE +22 -0
- data/README.md +221 -0
- data/init.rb +1 -0
- data/lib/patches.rb +135 -0
- data/lib/spawn.rb +223 -0
- data/lib/spawn/cucumber.rb +32 -0
- data/spec/spawn/spawn_spec.rb +63 -0
- data/spec/spec_helper.rb +13 -0
- metadata +69 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
data.tar.gz: 22c752d3c7f7e6f38838941833c5b3e05cd376e1
|
4
|
+
metadata.gz: 3ed0947f67a6a6937970268bcb2988dc56184dda
|
5
|
+
SHA512:
|
6
|
+
data.tar.gz: 858a4f1a283da28138a84b801f2943901a86bf49250804bd46477f01b5e667927bab24de0350a5f8887f210af57ae233b3e2cb45647404ff0677bb877eb3ca8c
|
7
|
+
metadata.gz: 949044c486e08b519984b6734c009568143d7d64325838f8df47f6cee75d0e8903a831f3552cf4373d25ca8a749381067fae25c327357609c917e11182f7e3a0
|
data/CHANGELOG
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
v0.1 - 2007/09/13
|
2
|
+
|
3
|
+
initial version
|
4
|
+
|
5
|
+
--------------------------------------------------
|
6
|
+
v0.2 - 2007/09/28
|
7
|
+
|
8
|
+
* return PID of the child process
|
9
|
+
* added ":detach => false" option
|
10
|
+
|
11
|
+
--------------------------------------------------
|
12
|
+
v0.3 - 2007/10/15
|
13
|
+
|
14
|
+
* added ':method => :thread' for threaded spawns
|
15
|
+
* removed ':detach => false' option in favor of more generic implementation
|
16
|
+
* added ability to set configuration of the form 'Spawn::method :thread'
|
17
|
+
* added patch to ActiveRecord::Base to allow for more efficient reconnect in child processes
|
18
|
+
* added monkey patch for http://dev.rubyonrails.org/ticket/7579
|
19
|
+
* added wait() method to wait for spawned code blocks
|
20
|
+
* don't allow threading if allow_concurrency=false
|
21
|
+
|
22
|
+
--------------------------------------------------
|
23
|
+
v0.4 - 2008/1/26
|
24
|
+
|
25
|
+
* default to :thread on windows, still :fork on all other platforms
|
26
|
+
* raise exception when used with :method=>:true and allow_concurrency != true
|
27
|
+
|
28
|
+
--------------------------------------------------
|
29
|
+
v0.5 - 2008/3/1
|
30
|
+
* also default to :thread on JRuby (java)
|
31
|
+
* added new :method => :yield which doesn't fork or thread, this is useful for testing
|
32
|
+
* fixed problem with connections piling up on PostgreSQL
|
33
|
+
|
34
|
+
--------------------------------------------------
|
35
|
+
v0.6 - 2008/04/21
|
36
|
+
* only apply clear_reloadable_connections patch on Rails 1.x (7579 fixed in Rails 2.x)
|
37
|
+
* made it more responsive in more environments by disconnecting from the listener socket in the forked process
|
38
|
+
|
39
|
+
--------------------------------------------------
|
40
|
+
v0.7 - 2008/04/24
|
41
|
+
* more generic mechanism for closing resources after fork
|
42
|
+
* check for existence of Mongrel before patching it
|
43
|
+
|
44
|
+
--------------------------------------------------
|
45
|
+
v0.8 - 2008/05/02
|
46
|
+
* call exit! within the ensure block so that at_exit handlers aren't called on exceptions
|
47
|
+
* set logger from RAILS_DEFAULT_LOGGER if available, else STDERR
|
48
|
+
|
49
|
+
--------------------------------------------------
|
50
|
+
v0.9 - 2008/05/11
|
51
|
+
* added ability to set nice level for child process
|
52
|
+
|
53
|
+
--------------------------------------------------
|
54
|
+
v1.0 - 2010/10/09
|
55
|
+
* merged edged to master, let's call this version 1.0
|
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2007 Tom Anderson (tom@squeat.com)
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person
|
4
|
+
obtaining a copy of this software and associated documentation
|
5
|
+
files (the "Software"), to deal in the Software without
|
6
|
+
restriction, including without limitation the rights to use,
|
7
|
+
copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8
|
+
copies of the Software, and to permit persons to whom the
|
9
|
+
Software is furnished to do so, subject to the following
|
10
|
+
conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be
|
13
|
+
included in all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
17
|
+
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
19
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
20
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
21
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
22
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,221 @@
|
|
1
|
+
#News
|
2
|
+
|
3
|
+
Now runs with ruby 1.9 (and later). Because ruby "stole" the name "spawn", this gem
|
4
|
+
now has been redefined to use "Spawn.new(&block)" instead of "spawn(&block)". Other
|
5
|
+
than that nothing has changed in the basic usage. Read below for detailed usage.
|
6
|
+
|
7
|
+
# Spawn
|
8
|
+
|
9
|
+
This gem provides a 'Spawn' class to easily fork OR thread long-running sections of
|
10
|
+
code so that your application can return results to your users more quickly.
|
11
|
+
It works by creating new database connections in ActiveRecord::Base for the
|
12
|
+
spawned block so that you don't have to worry about database connections working,
|
13
|
+
they just do.
|
14
|
+
|
15
|
+
The gem also patches ActiveRecord::Base to handle some known bugs when using
|
16
|
+
threads if you prefer using the threaded model over forking.
|
17
|
+
|
18
|
+
## Installation
|
19
|
+
|
20
|
+
The name of the gem is "spawn-block" (unfortunately somebody took the name "spawn" before
|
21
|
+
I was able to convert this to a gem).
|
22
|
+
|
23
|
+
### git
|
24
|
+
|
25
|
+
If you want to live on the latest master branch, add this to your Gemfile,
|
26
|
+
|
27
|
+
gem 'spawn-block', :git => 'git://github.com/tra/spawn'
|
28
|
+
|
29
|
+
and use bundler to manage it (bundle install, bundle update).
|
30
|
+
|
31
|
+
### rubygem
|
32
|
+
|
33
|
+
If you'd rather install from the latest gem,
|
34
|
+
|
35
|
+
gem 'spawn-block', '~>2.0'
|
36
|
+
|
37
|
+
### configure
|
38
|
+
|
39
|
+
Make sure that ActiveRecord reconnects to your database automatically when needed,
|
40
|
+
for instance put
|
41
|
+
|
42
|
+
production/development:
|
43
|
+
...
|
44
|
+
reconnect: true
|
45
|
+
|
46
|
+
into your config/database.yml.
|
47
|
+
|
48
|
+
## Usage
|
49
|
+
|
50
|
+
Here's a simple example of how to demonstrate the spawn plugin.
|
51
|
+
In one of your controllers, insert this code (after installing the plugin of course):
|
52
|
+
```ruby
|
53
|
+
Spawn.new do
|
54
|
+
logger.info("I feel sleepy...")
|
55
|
+
sleep 11
|
56
|
+
logger.info("Time to wake up!")
|
57
|
+
end
|
58
|
+
```
|
59
|
+
If everything is working correctly, your controller should finish quickly then you'll see
|
60
|
+
the last log message several seconds later.
|
61
|
+
|
62
|
+
If you need to wait for the spawned processes/threads, then pass the objects returned by
|
63
|
+
spawn to Spawn.wait(), like this:
|
64
|
+
```ruby
|
65
|
+
spawns = []
|
66
|
+
N.times do |i|
|
67
|
+
# spawn N blocks of code
|
68
|
+
spawns << Spawn.new do
|
69
|
+
something(i)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
# wait for all N blocks of code to finish running
|
73
|
+
Spawn.wait(spawns)
|
74
|
+
```
|
75
|
+
## Options
|
76
|
+
|
77
|
+
The options you can pass to spawn are:
|
78
|
+
|
79
|
+
<table>
|
80
|
+
<tr><th>Option</th><th>Values</th></tr>
|
81
|
+
<tr><td>:method</td><td>:fork, :thread, :yield</td></tr>
|
82
|
+
<tr><td>:nice</td><td>integer value 0-19, 19 = really nice</td></tr>
|
83
|
+
<tr><td>:kill</td><td>boolean value indicating whether the parent should kill the spawned process
|
84
|
+
when it exits (only valid when :method => :fork)</td></tr>
|
85
|
+
<tr><td>:argv</td><td>string to override the process name</td></tr>
|
86
|
+
</table>
|
87
|
+
|
88
|
+
Any option to spawn can be set as a default so that you don't have to pass them in
|
89
|
+
to every call of spawn. To configure the spawn default options, add a line to
|
90
|
+
your configuration file(s) like this:
|
91
|
+
```ruby
|
92
|
+
Spawn::default_options {:method => :thread}
|
93
|
+
```
|
94
|
+
If you don't set any default options, the :method will default to :fork. To
|
95
|
+
specify different values for different environments, add the default_options call to
|
96
|
+
he appropriate environment file (development.rb, test.rb). For testing you can set
|
97
|
+
the default :method to :yield so that the code is run inline.
|
98
|
+
```ruby
|
99
|
+
# in environment.rb
|
100
|
+
Spawn.method :method => :fork, :nice => 7
|
101
|
+
# in test.rb, will override the environment.rb setting
|
102
|
+
Spawn.method :method => :yield
|
103
|
+
```
|
104
|
+
This allows you to set your production and development environments to use different
|
105
|
+
methods according to your needs.
|
106
|
+
|
107
|
+
### be nice
|
108
|
+
|
109
|
+
If you want your forked child to run at a lower priority than the parent process, pass in
|
110
|
+
the :nice option like this:
|
111
|
+
```ruby
|
112
|
+
Spawn.new(:nice => 7) do
|
113
|
+
do_something_nicely
|
114
|
+
end
|
115
|
+
```
|
116
|
+
### fork me
|
117
|
+
|
118
|
+
By default, spawn will use the fork to spawn child processes. You can configure it to
|
119
|
+
do threading either by telling the spawn method when you call it or by configuring your
|
120
|
+
environment.
|
121
|
+
For example, this is how you can tell spawn to use threading on the call,
|
122
|
+
```ruby
|
123
|
+
Spawn.new(:method => :thread) do
|
124
|
+
something
|
125
|
+
end
|
126
|
+
```
|
127
|
+
When you use threaded spawning, make sure that your application is thread-safe. Rails
|
128
|
+
can be switched to thread-safe mode with (not sure if this is needed anymore)
|
129
|
+
```ruby
|
130
|
+
# Enable threaded mode
|
131
|
+
config.threadsafe!
|
132
|
+
```
|
133
|
+
in environments/your_environment.rb
|
134
|
+
|
135
|
+
### kill or be killed
|
136
|
+
|
137
|
+
Depending on your application, you may want the children processes to go away when
|
138
|
+
the parent process exits. By default spawn lets the children live after the
|
139
|
+
parent dies. But you can tell it to kill the children by setting the :kill option
|
140
|
+
to true.
|
141
|
+
|
142
|
+
### a process by any other name
|
143
|
+
|
144
|
+
If you'd like to be able to identify which processes are spawned by looking at the
|
145
|
+
output of ps then set the :argv option with a string of your choice.
|
146
|
+
You should then be able to see this string as the process name when
|
147
|
+
listing the running processes (ps).
|
148
|
+
|
149
|
+
For example, if you do something like this,
|
150
|
+
```ruby
|
151
|
+
3.times do |i|
|
152
|
+
Spawn.new(:argv => "spawn -#{i}-") do
|
153
|
+
something(i)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
```
|
157
|
+
then in the shell,
|
158
|
+
```shell
|
159
|
+
$ ps -ef | grep spawn
|
160
|
+
502 2645 2642 0 0:00.01 ttys002 0:00.02 spawn -0-
|
161
|
+
502 2646 2642 0 0:00.02 ttys002 0:00.02 spawn -1-
|
162
|
+
502 2647 2642 0 0:00.02 ttys002 0:00.03 spawn -2-
|
163
|
+
```
|
164
|
+
The length of the process name may be limited by your OS so you might want to experiment
|
165
|
+
to see how long it can be (it may be limited by the length of the original process name).
|
166
|
+
|
167
|
+
## Forking vs. Threading
|
168
|
+
|
169
|
+
There are several tradeoffs for using threading vs. forking. Forking was chosen as the
|
170
|
+
default primarily because it requires no configuration to get it working out of the box.
|
171
|
+
|
172
|
+
Forking advantages:
|
173
|
+
|
174
|
+
- more reliable? - the ActiveRecord code is generally not deemed to be thread-safe.
|
175
|
+
Even though spawn attempts to patch known problems with the threaded implementation,
|
176
|
+
there are no guarantees. Forking is heavier but should be fairly reliable.
|
177
|
+
- keep running - this could also be a disadvantage, but you may find you want to fork
|
178
|
+
off a process that could have a life longer than its parent. For example, maybe you
|
179
|
+
want to restart your server without killing the spawned processes.
|
180
|
+
We don't necessarily condone this (i.e. haven't tried it) but it's technically possible.
|
181
|
+
- easier - forking works out of the box with spawn, threading requires you set
|
182
|
+
allow_concurrency=true (for older versions of Rails).
|
183
|
+
Also, beware of automatic reloading of classes in development
|
184
|
+
mode (config.cache_classes = false).
|
185
|
+
|
186
|
+
Threading advantages:
|
187
|
+
- less filling - threads take less resources... how much less? it depends. Some
|
188
|
+
flavors of Unix are pretty efficient at forking so the threading advantage may not
|
189
|
+
be as big as you think... but then again, maybe it's more than you think. :wink:
|
190
|
+
- debugging - you can set breakpoints in your threads
|
191
|
+
|
192
|
+
## Acknowledgements
|
193
|
+
|
194
|
+
This plugin was initially inspired by Scott Persinger's blog post on how to use fork
|
195
|
+
in rails for background processing (link no longer available).
|
196
|
+
|
197
|
+
Further inspiration for the threading implementation came from [Jonathon Rochkind's
|
198
|
+
blog post](http://bibwild.wordpress.com/2007/08/28/threading-in-rails/) on threading in rails.
|
199
|
+
|
200
|
+
Also thanks to all who have helped debug problems and suggest improvements
|
201
|
+
including:
|
202
|
+
|
203
|
+
- Ahmed Adam, Tristan Schneiter, Scott Haug, Andrew Garfield, Eugene Otto, Dan Sharp,
|
204
|
+
Olivier Ruffin, Adrian Duyzer, Cyrille Labesse
|
205
|
+
|
206
|
+
- Garry Tan, Matt Jankowski (Rails 2.2.x fixes), Mina Naguib (Rails 2.3.6 fix)
|
207
|
+
|
208
|
+
- Tim Kadom, Mauricio Marcon Zaffari, Danial Pearce, Hongli Lai, Scott Wadden
|
209
|
+
(passenger fixes)
|
210
|
+
|
211
|
+
- Will Bryant, James Sanders (memcache fix)
|
212
|
+
|
213
|
+
- David Kelso, Richard Hirner, Luke van der Hoeven (gemification and Rails 3 support)
|
214
|
+
|
215
|
+
- Jay Caines-Gooby, Eric Stewart (Unicorn fix)
|
216
|
+
|
217
|
+
- Dan Sharp for the changes that allow it to work with Ruby 1.9 and later
|
218
|
+
|
219
|
+
- <your name here>
|
220
|
+
|
221
|
+
Copyright (c) 2007-present Tom Anderson (tom@squeat.com), see LICENSE
|
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'spawn'
|
data/lib/patches.rb
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
if defined?(ActiveRecord)
|
2
|
+
# see activerecord/lib/active_record/connection_adaptors/abstract/connection_specification.rb
|
3
|
+
class ActiveRecord::Base
|
4
|
+
# reconnect without disconnecting
|
5
|
+
if ::Spawn::RAILS_3_x || ::Spawn::RAILS_2_2
|
6
|
+
def self.spawn_reconnect(klass=self)
|
7
|
+
# keep ancestors' connection_handlers around to avoid them being garbage collected in the forked child
|
8
|
+
@@ancestor_connection_handlers ||= []
|
9
|
+
@@ancestor_connection_handlers << self.connection_handler
|
10
|
+
self.connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
|
11
|
+
|
12
|
+
establish_connection
|
13
|
+
end
|
14
|
+
else
|
15
|
+
def self.spawn_reconnect(klass=self)
|
16
|
+
spec = @@defined_connections[klass.name]
|
17
|
+
konn = active_connections[klass.name]
|
18
|
+
# remove from internal arrays before calling establish_connection so that
|
19
|
+
# the connection isn't disconnected when it calls AR::Base.remove_connection
|
20
|
+
@@defined_connections.delete_if { |key, value| value == spec }
|
21
|
+
active_connections.delete_if { |key, value| value == konn }
|
22
|
+
establish_connection(spec ? spec.config : nil)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# this patch not needed on Rails 2.x and later
|
27
|
+
if ::Spawn::RAILS_1_x
|
28
|
+
# monkey patch to fix threading problems,
|
29
|
+
# see: http://dev.rubyonrails.org/ticket/7579
|
30
|
+
def self.clear_reloadable_connections!
|
31
|
+
if @@allow_concurrency
|
32
|
+
# Hash keyed by thread_id in @@active_connections. Hash of hashes.
|
33
|
+
@@active_connections.each do |thread_id, conns|
|
34
|
+
conns.each do |name, conn|
|
35
|
+
if conn.requires_reloading?
|
36
|
+
conn.disconnect!
|
37
|
+
@@active_connections[thread_id].delete(name)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
else
|
42
|
+
# Just one level hash, no concurrency.
|
43
|
+
@@active_connections.each do |name, conn|
|
44
|
+
if conn.requires_reloading?
|
45
|
+
conn.disconnect!
|
46
|
+
@@active_connections.delete(name)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# just borrowing from the mongrel & passenger patches:
|
56
|
+
if defined?(Unicorn::HttpServer) && defined?(Unicorn::HttpRequest::REQ)
|
57
|
+
class Unicorn::HttpServer
|
58
|
+
REQ = Unicorn::HttpRequest::REQ
|
59
|
+
alias_method :orig_process_client, :process_client
|
60
|
+
def process_client(client)
|
61
|
+
::Spawn.resources_to_close(client, REQ)
|
62
|
+
orig_process_client(client)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
elsif defined? Unicorn::HttpServer
|
66
|
+
class Unicorn::HttpServer
|
67
|
+
alias_method :orig_process_client, :process_client
|
68
|
+
def process_client(client)
|
69
|
+
::Spawn.resources_to_close(client, @request)
|
70
|
+
orig_process_client(client)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# see mongrel/lib/mongrel.rb
|
76
|
+
# it's possible that this is not defined if you're running outside of mongrel
|
77
|
+
# examples: ./script/runner or ./script/console
|
78
|
+
if defined? Mongrel::HttpServer
|
79
|
+
class Mongrel::HttpServer
|
80
|
+
# redefine Montrel::HttpServer::process_client so that we can intercept
|
81
|
+
# the socket that is being used so Spawn can close it upon forking
|
82
|
+
alias_method :orig_process_client, :process_client
|
83
|
+
def process_client(client)
|
84
|
+
::Spawn.resources_to_close(client, @socket)
|
85
|
+
orig_process_client(client)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
need_passenger_patch = true
|
91
|
+
if defined? PhusionPassenger::VERSION_STRING
|
92
|
+
# The VERSION_STRING variable was defined sometime after 2.1.0.
|
93
|
+
# We don't need passenger patch for 2.2.2 or later.
|
94
|
+
pv = PhusionPassenger::VERSION_STRING.split('.').collect{|s| s.to_i}
|
95
|
+
need_passenger_patch = pv[0] < 2 || (pv[0] == 2 && (pv[1] < 2 || (pv[1] == 2 && pv[2] < 2)))
|
96
|
+
end
|
97
|
+
|
98
|
+
if need_passenger_patch
|
99
|
+
# Patch for work with passenger < 2.1.0
|
100
|
+
if defined? Passenger::Railz::RequestHandler
|
101
|
+
class Passenger::Railz::RequestHandler
|
102
|
+
alias_method :orig_process_request, :process_request
|
103
|
+
def process_request(headers, input, output)
|
104
|
+
::Spawn.resources_to_close(input, output)
|
105
|
+
orig_process_request(headers, input, output)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
# Patch for work with passenger >= 2.1.0
|
111
|
+
if defined? PhusionPassenger::Railz::RequestHandler
|
112
|
+
class PhusionPassenger::Railz::RequestHandler
|
113
|
+
alias_method :orig_process_request, :process_request
|
114
|
+
def process_request(headers, input, output)
|
115
|
+
::Spawn.resources_to_close(input, output)
|
116
|
+
orig_process_request(headers, input, output)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# Patch for passenger with Rails >= 2.3.0 (uses rack)
|
122
|
+
if defined? PhusionPassenger::Rack::RequestHandler
|
123
|
+
class PhusionPassenger::Rack::RequestHandler
|
124
|
+
alias_method :orig_process_request, :process_request
|
125
|
+
def process_request(headers, input, output)
|
126
|
+
::Spawn.resources_to_close(input, output)
|
127
|
+
orig_process_request(headers, input, output)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
if defined?(::ActiveSupport::Cache::MemCacheStore) && ::ActiveSupport::Cache.autoload?(:MemCacheStore).nil?
|
134
|
+
::ActiveSupport::Cache::MemCacheStore.delegate :reset, :to => :@data
|
135
|
+
end
|
data/lib/spawn.rb
ADDED
@@ -0,0 +1,223 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
class Spawn
|
4
|
+
if defined? ::Rails
|
5
|
+
RAILS_1_x = (::Rails::VERSION::MAJOR == 1) unless defined?(RAILS_1_x)
|
6
|
+
RAILS_2_2 = ((::Rails::VERSION::MAJOR == 2 && ::Rails::VERSION::MINOR >= 2)) unless defined?(RAILS_2_2)
|
7
|
+
RAILS_3_x = (::Rails::VERSION::MAJOR > 2) unless defined?(RAILS_3_x)
|
8
|
+
else
|
9
|
+
RAILS_1_x = nil
|
10
|
+
RAILS_2_2 = nil
|
11
|
+
RAILS_3_x = nil
|
12
|
+
end
|
13
|
+
|
14
|
+
@@default_options = {
|
15
|
+
# default to forking (unless windows or jruby)
|
16
|
+
:method => ((RUBY_PLATFORM =~ /(win32|mingw32|java)/) ? :thread : :fork),
|
17
|
+
:nice => nil,
|
18
|
+
:kill => false,
|
19
|
+
:argv => nil
|
20
|
+
}
|
21
|
+
|
22
|
+
# things to close in child process
|
23
|
+
@@resources = []
|
24
|
+
|
25
|
+
# forked children to kill on exit
|
26
|
+
@@punks = []
|
27
|
+
|
28
|
+
# in some environments, logger isn't defined
|
29
|
+
@@logger = defined?(::Rails) ? ::Rails.logger : ::Logger.new(STDERR)
|
30
|
+
|
31
|
+
attr_accessor :type
|
32
|
+
attr_accessor :handle
|
33
|
+
|
34
|
+
# Set the options to use every time spawn is called unless specified
|
35
|
+
# otherwise. For example, in your environment, do something like
|
36
|
+
# this:
|
37
|
+
# Spawn::default_options = {:nice => 5}
|
38
|
+
# to default to using the :nice option with a value of 5 on every call.
|
39
|
+
# Valid options are:
|
40
|
+
# :method => (:thread | :fork | :yield)
|
41
|
+
# :nice => nice value of the forked process
|
42
|
+
# :kill => whether or not the parent process will kill the
|
43
|
+
# spawned child process when the parent exits
|
44
|
+
# :argv => changes name of the spawned process as seen in ps
|
45
|
+
def self.default_options(options = {})
|
46
|
+
@@default_options.merge!(options)
|
47
|
+
@@logger.info "spawn> default options = #{options.inspect}" if @@logger
|
48
|
+
end
|
49
|
+
|
50
|
+
# set the resources to disconnect from in the child process (when forking)
|
51
|
+
def self.resources_to_close(*resources)
|
52
|
+
@@resources = resources
|
53
|
+
end
|
54
|
+
|
55
|
+
# close all the resources added by calls to resource_to_close
|
56
|
+
def self.close_resources
|
57
|
+
@@resources.each do |resource|
|
58
|
+
resource.close if resource && resource.respond_to?(:close) && !resource.closed?
|
59
|
+
end
|
60
|
+
# in case somebody spawns recursively
|
61
|
+
@@resources.clear
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.alive?(pid)
|
65
|
+
begin
|
66
|
+
Process::kill 0, pid
|
67
|
+
# if the process is alive then kill won't throw an exception
|
68
|
+
true
|
69
|
+
rescue Errno::ESRCH
|
70
|
+
false
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.kill_punks
|
75
|
+
@@punks.each do |punk|
|
76
|
+
if alive?(punk)
|
77
|
+
@@logger.info "spawn> parent(#{Process.pid}) killing child(#{punk})" if @@logger
|
78
|
+
begin
|
79
|
+
Process.kill("TERM", punk)
|
80
|
+
rescue
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
@@punks = []
|
85
|
+
end
|
86
|
+
# register to kill marked children when parent exits
|
87
|
+
at_exit { Spawn.kill_punks }
|
88
|
+
|
89
|
+
# Spawns a long-running section of code and returns the ID of the spawned process.
|
90
|
+
# By default the process will be a forked process. To use threading, pass
|
91
|
+
# :method => :thread or override the default behavior in the environment by setting
|
92
|
+
# 'Spawn::method :thread'.
|
93
|
+
def initialize(opts = {})
|
94
|
+
raise "Must give block of code to be spawned" unless block_given?
|
95
|
+
options = @@default_options.merge(symbolize_options(opts))
|
96
|
+
# setting options[:method] will override configured value in default_options[:method]
|
97
|
+
if options[:method] == :yield
|
98
|
+
yield
|
99
|
+
elsif options[:method].respond_to?(:call)
|
100
|
+
options[:method].call(proc { yield })
|
101
|
+
elsif options[:method] == :thread
|
102
|
+
# for versions before 2.2, check for allow_concurrency
|
103
|
+
if RAILS_2_2 || ActiveRecord::Base.respond_to?(:allow_concurrency) ?
|
104
|
+
ActiveRecord::Base.allow_concurrency : Rails.application.config.allow_concurrency
|
105
|
+
@type = :thread
|
106
|
+
@handle = thread_it(options) { yield }
|
107
|
+
else
|
108
|
+
@@logger.error("spawn(:method=>:thread) only allowed when allow_concurrency=true")
|
109
|
+
raise "spawn requires config.active_record.allow_concurrency=true when used with :method=>:thread"
|
110
|
+
end
|
111
|
+
else
|
112
|
+
@type = :fork
|
113
|
+
@handle = fork_it(options) { yield }
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.wait(sids = [])
|
118
|
+
# wait for all threads and/or forks (if a single sid passed in, convert to array first)
|
119
|
+
Array(sids).each do |sid|
|
120
|
+
if sid.type == :thread
|
121
|
+
sid.handle.join()
|
122
|
+
else
|
123
|
+
begin
|
124
|
+
Process.wait(sid.handle)
|
125
|
+
rescue
|
126
|
+
# if the process is already done, ignore the error
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
# clean up connections from expired threads
|
131
|
+
ActiveRecord::Base.verify_active_connections!() if defined?(ActiveRecord)
|
132
|
+
end
|
133
|
+
|
134
|
+
protected
|
135
|
+
|
136
|
+
def fork_it(options)
|
137
|
+
# The problem with rails is that it only has one connection (per class),
|
138
|
+
# so when we fork a new process, we need to reconnect.
|
139
|
+
@@logger.debug "spawn> parent PID = #{Process.pid}" if @@logger
|
140
|
+
child = fork do
|
141
|
+
begin
|
142
|
+
start = Time.now
|
143
|
+
@@logger.debug "spawn> child PID = #{Process.pid}" if @@logger
|
144
|
+
|
145
|
+
# this child has no children of it's own to kill (yet)
|
146
|
+
@@punks = []
|
147
|
+
|
148
|
+
# set the nice priority if needed
|
149
|
+
Process.setpriority(Process::PRIO_PROCESS, 0, options[:nice]) if options[:nice]
|
150
|
+
|
151
|
+
# disconnect from the listening socket, et al
|
152
|
+
Spawn.close_resources
|
153
|
+
if defined?(Rails)
|
154
|
+
# get a new database connection so the parent can keep the original one
|
155
|
+
ActiveRecord::Base.spawn_reconnect
|
156
|
+
# close the memcache connection so the parent can keep the original one
|
157
|
+
Rails.cache.reset if Rails.cache.respond_to?(:reset)
|
158
|
+
end
|
159
|
+
|
160
|
+
# set the process name
|
161
|
+
$0 = options[:argv] if options[:argv]
|
162
|
+
|
163
|
+
# run the block of code that takes so long
|
164
|
+
yield
|
165
|
+
|
166
|
+
rescue => ex
|
167
|
+
@@logger.error "spawn> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message}" if @@logger
|
168
|
+
@@logger.error "spawn> " + ex.backtrace.join("\n") if @@logger
|
169
|
+
ensure
|
170
|
+
begin
|
171
|
+
# to be safe, catch errors on closing the connnections too
|
172
|
+
ActiveRecord::Base.connection_handler.clear_all_connections! if defined?(ActiveRecord)
|
173
|
+
ensure
|
174
|
+
@@logger.info "spawn> child[#{Process.pid}] took #{Time.now - start} sec" if @@logger
|
175
|
+
# ensure log is flushed since we are using exit!
|
176
|
+
@@logger.flush if @@logger && @@logger.respond_to?(:flush)
|
177
|
+
# this child might also have children to kill if it called spawn
|
178
|
+
Spawn.kill_punks
|
179
|
+
# this form of exit doesn't call at_exit handlers
|
180
|
+
exit!(0)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
# detach from child process (parent may still wait for detached process if they wish)
|
186
|
+
Process.detach(child)
|
187
|
+
|
188
|
+
# remove dead children from the target list to avoid memory leaks
|
189
|
+
@@punks.delete_if {|punk| !alive?(punk)}
|
190
|
+
|
191
|
+
# mark this child for death when this process dies
|
192
|
+
if options[:kill]
|
193
|
+
@@punks << child
|
194
|
+
@@logger.debug "spawn> death row = #{@@punks.inspect}" if @@logger
|
195
|
+
end
|
196
|
+
|
197
|
+
# return Spawn::Id.new(:fork, child)
|
198
|
+
return child
|
199
|
+
end
|
200
|
+
|
201
|
+
def thread_it(options)
|
202
|
+
# clean up stale connections from previous threads
|
203
|
+
ActiveRecord::Base.verify_active_connections!() if defined?(ActiveRecord)
|
204
|
+
thr = Thread.new do
|
205
|
+
# run the long-running code block
|
206
|
+
ActiveRecord::Base.connection_pool.with_connection { yield }
|
207
|
+
end
|
208
|
+
thr.priority = -options[:nice] if options[:nice]
|
209
|
+
return thr
|
210
|
+
end
|
211
|
+
|
212
|
+
# In case we don't have rails, can't call opts.symbolize_keys
|
213
|
+
def symbolize_options(hash)
|
214
|
+
hash.inject({}) do |new_hash, (key, value)|
|
215
|
+
new_hash[key.to_sym] = value
|
216
|
+
new_hash
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
220
|
+
Spawnling = Spawn
|
221
|
+
|
222
|
+
# patches depends on Spawn so require it after the class
|
223
|
+
require 'patches'
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module SpawnExtensions
|
2
|
+
# FIXME don't know how to tell Spawn to use #add_spawn_proc without extended
|
3
|
+
# using extended forces to make methods class methods while this is not very clean
|
4
|
+
def self.extended(base)
|
5
|
+
Spawn::method proc{ |block| add_spawn_proc(block) }
|
6
|
+
end
|
7
|
+
|
8
|
+
# Calls the spawn that was created
|
9
|
+
#
|
10
|
+
# Can be used to keep control over forked processes in your tests
|
11
|
+
def call_last_spawn_proc
|
12
|
+
spawns = SpawnExtensions.spawn_procs
|
13
|
+
|
14
|
+
raise "No spawn procs left" if spawns.empty?
|
15
|
+
|
16
|
+
spawns.pop.call
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def self.spawn_procs
|
22
|
+
@@spawn_procs ||= []
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.add_spawn_proc(block)
|
26
|
+
spawn_procs << block
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
# Extend cucumber to take control over spawns
|
32
|
+
World(SpawnExtensions)
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe Spawn do
|
4
|
+
|
5
|
+
describe "yields" do
|
6
|
+
before(:each) do
|
7
|
+
Spawn::method :yield
|
8
|
+
define_spawned
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should be able to yield directly" do
|
12
|
+
Spawned.hello.should == "hello"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "override" do
|
17
|
+
before(:each) do
|
18
|
+
Spawn::method(proc{ "foo" })
|
19
|
+
define_spawned
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should be able to return a proc" do
|
23
|
+
Spawned.hello.should == "foo"
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
describe "delegate to a proc" do
|
29
|
+
before(:each) do
|
30
|
+
Spawn::method( proc{ |block| block })
|
31
|
+
define_spawned
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should be able to return a proc" do
|
35
|
+
Spawned.hello.should be_kind_of(Proc)
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should be able to return a proc" do
|
39
|
+
Spawned.hello.call.should == "hello"
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
after(:each) do
|
45
|
+
Object.send(:remove_const, :Spawned)
|
46
|
+
end
|
47
|
+
|
48
|
+
def define_spawned
|
49
|
+
cls = Class.new do
|
50
|
+
extend Spawn
|
51
|
+
|
52
|
+
def self.hello
|
53
|
+
spawn do
|
54
|
+
"hello"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
Object.const_set :Spawned, cls
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
gem 'rspec'
|
3
|
+
require 'spec'
|
4
|
+
require 'active_record'
|
5
|
+
require 'action_controller'
|
6
|
+
require 'rails/version'
|
7
|
+
|
8
|
+
$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
|
9
|
+
|
10
|
+
require 'spawn'
|
11
|
+
Spec::Runner.configure do |config|
|
12
|
+
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: spawnling
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: "2.1"
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tom Anderson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-08-08 00:00:00 Z
|
13
|
+
dependencies: []
|
14
|
+
|
15
|
+
description: |-
|
16
|
+
This plugin provides a 'Spawn' class to easily fork OR
|
17
|
+
thread long-running sections of code so that your application can return
|
18
|
+
results to your users more quickly. This plugin works by creating new database
|
19
|
+
connections in ActiveRecord::Base for the spawned block.
|
20
|
+
|
21
|
+
The plugin also patches ActiveRecord::Base to handle some known bugs when using
|
22
|
+
threads (see lib/patches.rb).
|
23
|
+
email:
|
24
|
+
- tom@squeat.com
|
25
|
+
executables: []
|
26
|
+
|
27
|
+
extensions: []
|
28
|
+
|
29
|
+
extra_rdoc_files: []
|
30
|
+
|
31
|
+
files:
|
32
|
+
- lib/patches.rb
|
33
|
+
- lib/spawn/cucumber.rb
|
34
|
+
- lib/spawn.rb
|
35
|
+
- spec/spawn/spawn_spec.rb
|
36
|
+
- spec/spec_helper.rb
|
37
|
+
- CHANGELOG
|
38
|
+
- LICENSE
|
39
|
+
- README.md
|
40
|
+
- init.rb
|
41
|
+
homepage: http://github.com/tra/spawn
|
42
|
+
licenses: []
|
43
|
+
|
44
|
+
metadata: {}
|
45
|
+
|
46
|
+
post_install_message:
|
47
|
+
rdoc_options: []
|
48
|
+
|
49
|
+
require_paths:
|
50
|
+
- lib
|
51
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: "0"
|
56
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 1.3.6
|
61
|
+
requirements: []
|
62
|
+
|
63
|
+
rubyforge_project:
|
64
|
+
rubygems_version: 2.0.3
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: Easily fork OR thread long-running sections of code in Ruby
|
68
|
+
test_files: []
|
69
|
+
|