pitchfork 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of pitchfork might be problematic. Click here for more details.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/Rakefile +1 -0
- data/lib/pitchfork/version.rb +3 -0
- data/lib/pitchfork.rb +160 -0
- data/pitchfork.gemspec +24 -0
- metadata +52 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/lib/pitchfork.rb
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
require "pitchfork/version"
|
2
|
+
|
3
|
+
module Pitchfork
|
4
|
+
class PitchforkError < ::StandardError; end
|
5
|
+
class MissingBlock < PitchforkError; end
|
6
|
+
class InvalidHook < PitchforkError; end
|
7
|
+
|
8
|
+
HOOKS = [
|
9
|
+
:start, # In parent, before looping through collection
|
10
|
+
:before_fork, # In parent, inside loop, before calling `fork`
|
11
|
+
:parent_fork, # In parent, after fork
|
12
|
+
:child_fork, # In child, after fork
|
13
|
+
:work_done, # In parent, after child exits
|
14
|
+
:complete # In parent, after looping through the collection
|
15
|
+
]
|
16
|
+
|
17
|
+
def self.work(collection, options = {}, &block)
|
18
|
+
Handler.new(collection, options).work(&block)
|
19
|
+
end
|
20
|
+
|
21
|
+
class Handler
|
22
|
+
attr_accessor :collection
|
23
|
+
|
24
|
+
def initialize(collection, config = {})
|
25
|
+
@collection = collection
|
26
|
+
@config = {:forks => 2, :name => "pitchfork"}.merge(config)
|
27
|
+
@hooks = {}
|
28
|
+
@children = {}
|
29
|
+
@status = :work
|
30
|
+
@master_pid = Process.pid
|
31
|
+
end
|
32
|
+
|
33
|
+
def work
|
34
|
+
puts "Current pid is: #{@master_pid}"
|
35
|
+
register_signals
|
36
|
+
|
37
|
+
procline "Spawning workers ..."
|
38
|
+
run_hook :start
|
39
|
+
|
40
|
+
collection.each do |data|
|
41
|
+
break unless work?
|
42
|
+
|
43
|
+
run_hook :before_fork
|
44
|
+
|
45
|
+
if @child = fork
|
46
|
+
run_hook :parent_fork
|
47
|
+
@children[@child] = true
|
48
|
+
else
|
49
|
+
procline "worker"
|
50
|
+
run_hook :child_fork
|
51
|
+
yield data
|
52
|
+
exit
|
53
|
+
end
|
54
|
+
|
55
|
+
# Stop forking and wait for a child if we've reached the limit
|
56
|
+
if @children.size >= @config[:forks]
|
57
|
+
procline "Waiting for workers to finish ..."
|
58
|
+
pid = Process.wait
|
59
|
+
run_hook :work_done, $?.exitstatus == 0, $?.exitstatus
|
60
|
+
@children.delete(pid)
|
61
|
+
end
|
62
|
+
|
63
|
+
if pause?
|
64
|
+
loop do
|
65
|
+
break unless pause?
|
66
|
+
sleep 3
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
remaining = Process.waitall
|
72
|
+
remaining.each do |pid,status|
|
73
|
+
run_hook :work_done, status.exitstatus == 0, status.exitstatus
|
74
|
+
end
|
75
|
+
|
76
|
+
run_hook :complete
|
77
|
+
collection
|
78
|
+
end
|
79
|
+
|
80
|
+
def on(type, hook)
|
81
|
+
raise InvalidHook.new(<<-ERRMSG) unless HOOKS.include?(type)
|
82
|
+
Pitchfork hook ':#{type}' does not exist.
|
83
|
+
Valid hooks are: #{HOOKS.collect {|c| ":#{c}"}.join(", ")}
|
84
|
+
ERRMSG
|
85
|
+
|
86
|
+
@hooks[type] = hook
|
87
|
+
end
|
88
|
+
|
89
|
+
def work?
|
90
|
+
@status == :work
|
91
|
+
end
|
92
|
+
|
93
|
+
def pause?
|
94
|
+
@status == :pause
|
95
|
+
end
|
96
|
+
|
97
|
+
def quit?
|
98
|
+
@status == :quit
|
99
|
+
end
|
100
|
+
|
101
|
+
def register_signals
|
102
|
+
trap('TERM') { shutdown! }
|
103
|
+
trap('INT') { shutdown! }
|
104
|
+
trap('QUIT') { shutdown }
|
105
|
+
trap('USR2') { pause! }
|
106
|
+
trap('CONT') { restart! }
|
107
|
+
end
|
108
|
+
|
109
|
+
def shutdown
|
110
|
+
procline "Shutting down ..."
|
111
|
+
@status = :quit
|
112
|
+
end
|
113
|
+
|
114
|
+
def shutdown!
|
115
|
+
shutdown
|
116
|
+
|
117
|
+
@children.keys.each do |pid|
|
118
|
+
if system("ps -p #{pid}")
|
119
|
+
Process.kill("KILL", pid) rescue nil
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def pause!
|
125
|
+
procline "Paused processing"
|
126
|
+
@status = :pause
|
127
|
+
end
|
128
|
+
|
129
|
+
def restart!
|
130
|
+
procline "Restarting ..."
|
131
|
+
@status = :work
|
132
|
+
end
|
133
|
+
|
134
|
+
def master?
|
135
|
+
Process.pid == @master_pid
|
136
|
+
end
|
137
|
+
|
138
|
+
def child?
|
139
|
+
!master?
|
140
|
+
end
|
141
|
+
|
142
|
+
def procline(msg)
|
143
|
+
line = "#{@config[:name]}: "
|
144
|
+
line << "[#{@children.size}] " if master?
|
145
|
+
line << msg
|
146
|
+
$0 = line
|
147
|
+
end
|
148
|
+
|
149
|
+
def run_hook(type, *args)
|
150
|
+
if hook = @hooks[type]
|
151
|
+
if type == :work_done
|
152
|
+
hook.call(*args)
|
153
|
+
else
|
154
|
+
hook.call
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
data/pitchfork.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "pitchfork/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "pitchfork"
|
7
|
+
s.version = Pitchfork::VERSION
|
8
|
+
s.authors = ["Peter Bui, 8tracks"]
|
9
|
+
s.email = ["peter@paydrotalks.com"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{Easy way to run parallel tasks with Unix fork}
|
12
|
+
s.description = %q{Easy way to run parallel tasks with Unix fork}
|
13
|
+
|
14
|
+
# s.rubyforge_project = "pitchfork"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
# specify any dependencies here; for example:
|
22
|
+
# s.add_development_dependency "rspec"
|
23
|
+
# s.add_runtime_dependency "rest-client"
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pitchfork
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Peter Bui, 8tracks
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2011-11-11 00:00:00.000000000Z
|
13
|
+
dependencies: []
|
14
|
+
description: Easy way to run parallel tasks with Unix fork
|
15
|
+
email:
|
16
|
+
- peter@paydrotalks.com
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- .gitignore
|
22
|
+
- Gemfile
|
23
|
+
- Rakefile
|
24
|
+
- lib/pitchfork.rb
|
25
|
+
- lib/pitchfork/version.rb
|
26
|
+
- pitchfork.gemspec
|
27
|
+
homepage: ''
|
28
|
+
licenses: []
|
29
|
+
post_install_message:
|
30
|
+
rdoc_options: []
|
31
|
+
require_paths:
|
32
|
+
- lib
|
33
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
41
|
+
requirements:
|
42
|
+
- - ! '>='
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
45
|
+
requirements: []
|
46
|
+
rubyforge_project:
|
47
|
+
rubygems_version: 1.8.10
|
48
|
+
signing_key:
|
49
|
+
specification_version: 3
|
50
|
+
summary: Easy way to run parallel tasks with Unix fork
|
51
|
+
test_files: []
|
52
|
+
has_rdoc:
|