forkoff 0.0.0

Sign up to get free protection for your applications and to get access to all the features.
data/README ADDED
@@ -0,0 +1,146 @@
1
+ NAME
2
+
3
+ forkoff
4
+
5
+ SYNOPSIS
6
+
7
+ brain-dead simple parallel processing for ruby
8
+
9
+ URI
10
+
11
+ http://rubyforge.org/projects/codeforpeople
12
+
13
+ INSTALL
14
+
15
+ gem install forkoff
16
+
17
+ DESCRIPTION
18
+
19
+ forkoff works for any enumerable object, iterating a code block to run in a
20
+ child process and collecting the results. forkoff can limit the number of
21
+ child processes which is, by default, 8.
22
+
23
+ SAMPLES
24
+
25
+ <========< samples/a.rb >========>
26
+
27
+ ~ > cat samples/a.rb
28
+
29
+ #
30
+ # forkoff makes it trivial to do parallel processing with ruby, the following
31
+ # prints out each word in a separate process
32
+ #
33
+
34
+ require 'forkoff'
35
+
36
+ %w( hey you ).forkoff!{|word| puts "#{ word } from #{ Process.pid }"}
37
+
38
+ ~ > ruby samples/a.rb
39
+
40
+ hey from 3239
41
+ you from 3240
42
+
43
+
44
+ <========< samples/b.rb >========>
45
+
46
+ ~ > cat samples/b.rb
47
+
48
+ #
49
+ # for example, this takes only 1 second or so to complete
50
+ #
51
+
52
+ require 'forkoff'
53
+
54
+ a = Time.now.to_f
55
+
56
+ results =
57
+ (0..7).forkoff do |i|
58
+
59
+ sleep 1
60
+
61
+ i ** 2
62
+
63
+ end
64
+
65
+ b = Time.now.to_f
66
+
67
+ elapsed = b - a
68
+
69
+ puts "elapsed: #{ elapsed }"
70
+ puts "results: #{ results.inspect }"
71
+
72
+ ~ > ruby samples/b.rb
73
+
74
+ elapsed: 1.07044386863708
75
+ results: [0, 1, 4, 9, 16, 25, 36, 49]
76
+
77
+
78
+ <========< samples/c.rb >========>
79
+
80
+ ~ > cat samples/c.rb
81
+
82
+ #
83
+ # forkoff does *NOT* spawn processes in batches, waiting for each batch to
84
+ # complete. rather, it keeps a certain number of processes busy until all
85
+ # results have been gathered. in otherwords the following will ensure that 2
86
+ # processes are running at all times, until the list is complete. note that
87
+ # the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
88
+ #
89
+
90
+ require 'forkoff'
91
+
92
+ pid = Process.pid
93
+
94
+ a = Time.now.to_f
95
+
96
+ pstrees =
97
+ %w( a b c d ).forkoff! :processes => 2 do |letter|
98
+ sleep 1
99
+ { letter => ` pstree -l 2 #{ pid } ` }
100
+ end
101
+
102
+
103
+ b = Time.now.to_f
104
+
105
+ puts
106
+ puts "pid: #{ pid }"
107
+ puts "elapsed: #{ b - a }"
108
+ puts
109
+
110
+ require 'yaml'
111
+
112
+ pstrees.each do |pstree|
113
+ y pstree
114
+ end
115
+
116
+ ~ > ruby samples/c.rb
117
+
118
+
119
+ pid: 3254
120
+ elapsed: 2.12998485565186
121
+
122
+ ---
123
+ a: |
124
+ -+- 03254 ahoward ruby -Ilib samples/c.rb
125
+ |-+- 03255 ahoward ruby -Ilib samples/c.rb
126
+ \-+- 03256 ahoward ruby -Ilib samples/c.rb
127
+
128
+ ---
129
+ b: |
130
+ -+- 03254 ahoward ruby -Ilib samples/c.rb
131
+ |-+- 03255 ahoward ruby -Ilib samples/c.rb
132
+ \-+- 03256 ahoward ruby -Ilib samples/c.rb
133
+
134
+ ---
135
+ c: |
136
+ -+- 03254 ahoward ruby -Ilib samples/c.rb
137
+ |-+- 03261 ahoward (ruby)
138
+ \-+- 03262 ahoward ruby -Ilib samples/c.rb
139
+
140
+ ---
141
+ d: |
142
+ -+- 03254 ahoward ruby -Ilib samples/c.rb
143
+ |-+- 03261 ahoward ruby -Ilib samples/c.rb
144
+ \-+- 03262 ahoward ruby -Ilib samples/c.rb
145
+
146
+
@@ -0,0 +1,35 @@
1
+ lib, version = File::basename(File::dirname(File::expand_path(__FILE__))).split %r/-/, 2
2
+
3
+ require 'rubygems'
4
+
5
+ Gem::Specification::new do |spec|
6
+ $VERBOSE = nil
7
+
8
+ shiteless = lambda do |list|
9
+ list.delete_if do |file|
10
+ file =~ %r/\.svn/ or
11
+ file =~ %r/\.tmp/
12
+ end
13
+ end
14
+
15
+ spec.name = lib
16
+ spec.version = version
17
+ spec.platform = Gem::Platform::RUBY
18
+ spec.summary = lib
19
+
20
+ spec.files = shiteless[Dir::glob("**/**")]
21
+ spec.executables = shiteless[Dir::glob("bin/*")].map{|exe| File::basename exe}
22
+
23
+ spec.require_path = "lib"
24
+ spec.autorequire = lib
25
+
26
+ spec.has_rdoc = File::exist? "doc"
27
+ spec.test_suite_file = "test/#{ lib }.rb" if File::directory? "test"
28
+ #spec.add_dependency 'lib', '>= version'
29
+
30
+ spec.extensions << "extconf.rb" if File::exists? "extconf.rb"
31
+
32
+ spec.author = "Ara T. Howard"
33
+ spec.email = "ara.t.howard@gmail.com"
34
+ spec.homepage = "http://codeforpeople.com/lib/ruby/#{ lib }/"
35
+ end
@@ -0,0 +1,32 @@
1
+ require 'pathname'
2
+
3
+ $VERBOSE=nil
4
+
5
+ def indent s, n = 2
6
+ ws = ' ' * n
7
+ s.gsub %r/^/, ws
8
+ end
9
+
10
+ template = IO::read 'README.tmpl'
11
+
12
+ samples = ''
13
+ prompt = '~ > '
14
+
15
+ Dir['sample*/*'].sort.each do |sample|
16
+ samples << "\n" << " <========< #{ sample } >========>" << "\n\n"
17
+
18
+ cmd = "cat #{ sample }"
19
+ samples << indent(prompt + cmd, 2) << "\n\n"
20
+ samples << indent(`#{ cmd }`, 4) << "\n"
21
+
22
+ cmd = "ruby #{ sample }"
23
+ samples << indent(prompt + cmd, 2) << "\n\n"
24
+
25
+ cmd = "ruby -Ilib #{ sample }"
26
+ samples << indent(`#{ cmd } 2>&1`, 4) << "\n"
27
+ end
28
+
29
+ #samples.gsub! %r/^/, ' '
30
+
31
+ readme = template.gsub %r/^\s*@samples\s*$/, samples
32
+ print readme
@@ -0,0 +1,214 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rbconfig'
3
+ require 'find'
4
+ require 'ftools'
5
+ require 'tempfile'
6
+ include Config
7
+
8
+ LIBDIR = "lib"
9
+ LIBDIR_MODE = 0644
10
+
11
+ BINDIR = "bin"
12
+ BINDIR_MODE = 0755
13
+
14
+
15
+ $srcdir = CONFIG["srcdir"]
16
+ $version = CONFIG["MAJOR"]+"."+CONFIG["MINOR"]
17
+ $libdir = File.join(CONFIG["libdir"], "ruby", $version)
18
+ $archdir = File.join($libdir, CONFIG["arch"])
19
+ $site_libdir = $:.find {|x| x =~ /site_ruby$/}
20
+ $bindir = CONFIG["bindir"] || CONFIG['BINDIR']
21
+ $ruby_install_name = CONFIG['ruby_install_name'] || CONFIG['RUBY_INSTALL_NAME'] || 'ruby'
22
+ $ruby_ext = CONFIG['EXEEXT'] || ''
23
+ $ruby = File.join($bindir, ($ruby_install_name + $ruby_ext))
24
+
25
+ if !$site_libdir
26
+ $site_libdir = File.join($libdir, "site_ruby")
27
+ elsif $site_libdir !~ %r/#{Regexp.quote($version)}/
28
+ $site_libdir = File.join($site_libdir, $version)
29
+ end
30
+
31
+ def install_rb(srcdir=nil, destdir=nil, mode=nil, bin=nil)
32
+ #{{{
33
+ path = []
34
+ dir = []
35
+ Find.find(srcdir) do |f|
36
+ next unless FileTest.file?(f)
37
+ next if (f = f[srcdir.length+1..-1]) == nil
38
+ next if (/CVS$/ =~ File.dirname(f))
39
+ next if (/\.svn/ =~ File.dirname(f))
40
+ next if f =~ %r/\.lnk/
41
+ next if f =~ %r/\.svn/
42
+ next if f =~ %r/\.swp/
43
+ next if f =~ %r/\.svn/
44
+ path.push f
45
+ dir |= [File.dirname(f)]
46
+ end
47
+ for f in dir
48
+ next if f == "."
49
+ next if f == "CVS"
50
+ File::makedirs(File.join(destdir, f))
51
+ end
52
+ for f in path
53
+ next if (/\~$/ =~ f)
54
+ next if (/^\./ =~ File.basename(f))
55
+ unless bin
56
+ File::install(File.join(srcdir, f), File.join(destdir, f), mode, true)
57
+ else
58
+ from = File.join(srcdir, f)
59
+ to = File.join(destdir, f)
60
+ shebangify(from) do |sf|
61
+ $deferr.print from, " -> ", File::catname(from, to), "\n"
62
+ $deferr.printf "chmod %04o %s\n", mode, to
63
+ File::install(sf, to, mode, false)
64
+ end
65
+ end
66
+ end
67
+ #}}}
68
+ end
69
+ def shebangify f
70
+ #{{{
71
+ open(f) do |fd|
72
+ buf = fd.read 42
73
+ if buf =~ %r/^\s*#\s*!.*ruby/o
74
+ ftmp = Tempfile::new("#{ $$ }_#{ File::basename(f) }")
75
+ begin
76
+ fd.rewind
77
+ ftmp.puts "#!#{ $ruby }"
78
+ while((buf = fd.read(8192)))
79
+ ftmp.write buf
80
+ end
81
+ ftmp.close
82
+ yield ftmp.path
83
+ ensure
84
+ ftmp.close!
85
+ end
86
+ else
87
+ yield f
88
+ end
89
+ end
90
+ #}}}
91
+ end
92
+ def ARGV.switch
93
+ #{{{
94
+ return nil if self.empty?
95
+ arg = self.shift
96
+ return nil if arg == '--'
97
+ if arg =~ /^-(.)(.*)/
98
+ return arg if $1 == '-'
99
+ raise 'unknown switch "-"' if $2.index('-')
100
+ self.unshift "-#{$2}" if $2.size > 0
101
+ "-#{$1}"
102
+ else
103
+ self.unshift arg
104
+ nil
105
+ end
106
+ #}}}
107
+ end
108
+ def ARGV.req_arg
109
+ #{{{
110
+ self.shift || raise('missing argument')
111
+ #}}}
112
+ end
113
+ def linkify d, linked = []
114
+ #--{{{
115
+ if test ?d, d
116
+ versioned = Dir[ File::join(d, "*-[0-9].[0-9].[0-9].rb") ]
117
+ versioned.each do |v|
118
+ src, dst = v, v.gsub(%r/\-[\d\.]+\.rb$/, '.rb')
119
+ lnk = nil
120
+ begin
121
+ if test ?l, dst
122
+ lnk = "#{ dst }.lnk"
123
+ puts "#{ dst } -> #{ lnk }"
124
+ File::rename dst, lnk
125
+ end
126
+ unless test ?e, dst
127
+ puts "#{ src } -> #{ dst }"
128
+ File::copy src, dst
129
+ linked << dst
130
+ end
131
+ ensure
132
+ if lnk
133
+ at_exit do
134
+ puts "#{ lnk } -> #{ dst }"
135
+ File::rename lnk, dst
136
+ end
137
+ end
138
+ end
139
+ end
140
+ end
141
+ linked
142
+ #--}}}
143
+ end
144
+
145
+
146
+ #
147
+ # main program
148
+ #
149
+
150
+ libdir = $site_libdir
151
+ bindir = $bindir
152
+ no_linkify = false
153
+ linked = nil
154
+ help = false
155
+
156
+ usage = <<-usage
157
+ #{ File::basename $0 }
158
+ -d, --destdir <destdir>
159
+ -l, --libdir <libdir>
160
+ -b, --bindir <bindir>
161
+ -r, --ruby <ruby>
162
+ -n, --no_linkify
163
+ -s, --sudo
164
+ -h, --help
165
+ usage
166
+
167
+ begin
168
+ while switch = ARGV.switch
169
+ case switch
170
+ when '-d', '--destdir'
171
+ libdir = ARGV.req_arg
172
+ when '-l', '--libdir'
173
+ libdir = ARGV.req_arg
174
+ when '-b', '--bindir'
175
+ bindir = ARGV.req_arg
176
+ when '-r', '--ruby'
177
+ $ruby = ARGV.req_arg
178
+ when '-n', '--no_linkify'
179
+ no_linkify = true
180
+ when '-s', '--sudo'
181
+ sudo = 'sudo'
182
+ when '-h', '--help'
183
+ help = true
184
+ else
185
+ raise "unknown switch #{switch.dump}"
186
+ end
187
+ end
188
+ rescue
189
+ STDERR.puts $!.to_s
190
+ STDERR.puts usage
191
+ exit 1
192
+ end
193
+
194
+ if help
195
+ STDOUT.puts usage
196
+ exit
197
+ end
198
+
199
+ system "#{ sudo } #{ $ruby } pre-install.rb" if test(?s, 'pre-install.rb')
200
+
201
+ unless no_linkify
202
+ linked = linkify('lib') + linkify('bin')
203
+ end
204
+
205
+ system "#{ $ruby } extconf.rb && make && #{ sudo } make install" if test(?s, 'extconf.rb')
206
+
207
+ install_rb(LIBDIR, libdir, LIBDIR_MODE)
208
+ install_rb(BINDIR, bindir, BINDIR_MODE, bin=true)
209
+
210
+ if linked
211
+ linked.each{|path| File::rm_f path}
212
+ end
213
+
214
+ system "#{ sudo } #{ $ruby } post-install.rb" if test(?s, 'post-install.rb')
@@ -0,0 +1,92 @@
1
+ require 'thread'
2
+
3
+ module Enumerable
4
+
5
+ def forkoff options = {}, &block
6
+ n = Integer( options['processes'] || options[:processes] || 8 )
7
+ done = Object.new
8
+ qs = Array.new(n){ Queue.new }
9
+ results = Array.new(n){ Queue.new }
10
+
11
+ #
12
+ # consumers
13
+ #
14
+ threads = []
15
+
16
+ n.times do |i|
17
+ thread =
18
+ Thread.new(i) do |i|
19
+ Thread.current.abort_on_exception = true
20
+
21
+ loop do
22
+ value = qs[i].pop
23
+ break if value == done
24
+ args, index = value
25
+
26
+ r, w = IO.pipe
27
+ pid = fork
28
+
29
+ unless pid
30
+ r.close
31
+ result =
32
+ begin
33
+ block.call(*args)
34
+ rescue Object => e
35
+ e
36
+ end
37
+ w.write( Marshal.dump( result ) )
38
+ exit
39
+ end
40
+
41
+ w.close
42
+ result = Marshal.load( r.read )
43
+ results[i].push( [result, index] )
44
+ Process.waitpid pid
45
+ end
46
+
47
+ results[i].push( done )
48
+ end
49
+
50
+ threads << thread
51
+ end
52
+
53
+ #
54
+ # producer
55
+ #
56
+ each_with_index do |args, i|
57
+ qs[ i.modulo(qs.size) ].push( [args, i] )
58
+ end
59
+
60
+ #
61
+ # mark the end of each queue
62
+ #
63
+ qs.each do |q|
64
+ q.push done
65
+ end
66
+
67
+ #
68
+ # wait for all threads to complete
69
+ #
70
+ threads.each do |t|
71
+ t.value
72
+ end
73
+
74
+ #
75
+ # gather results
76
+ #
77
+ list = []
78
+
79
+ results.each do |q|
80
+ loop do
81
+ value = q.pop
82
+ break if value == done
83
+ result, index = value
84
+ list[index] = result
85
+ end
86
+ end
87
+
88
+ list
89
+ end
90
+
91
+ alias_method 'forkoff!', 'forkoff'
92
+ end
@@ -0,0 +1,8 @@
1
+ #
2
+ # forkoff makes it trivial to do parallel processing with ruby, the following
3
+ # prints out each word in a separate process
4
+ #
5
+
6
+ require 'forkoff'
7
+
8
+ %w( hey you ).forkoff!{|word| puts "#{ word } from #{ Process.pid }"}
@@ -0,0 +1,23 @@
1
+ #
2
+ # for example, this takes only 1 second or so to complete
3
+ #
4
+
5
+ require 'forkoff'
6
+
7
+ a = Time.now.to_f
8
+
9
+ results =
10
+ (0..7).forkoff do |i|
11
+
12
+ sleep 1
13
+
14
+ i ** 2
15
+
16
+ end
17
+
18
+ b = Time.now.to_f
19
+
20
+ elapsed = b - a
21
+
22
+ puts "elapsed: #{ elapsed }"
23
+ puts "results: #{ results.inspect }"
@@ -0,0 +1,33 @@
1
+ #
2
+ # forkoff does *NOT* spawn processes in batches, waiting for each batch to
3
+ # complete. rather, it keeps a certain number of processes busy until all
4
+ # results have been gathered. in otherwords the following will ensure that 2
5
+ # processes are running at all times, until the list is complete. note that
6
+ # the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
7
+ #
8
+
9
+ require 'forkoff'
10
+
11
+ pid = Process.pid
12
+
13
+ a = Time.now.to_f
14
+
15
+ pstrees =
16
+ %w( a b c d ).forkoff! :processes => 2 do |letter|
17
+ sleep 1
18
+ { letter => ` pstree -l 2 #{ pid } ` }
19
+ end
20
+
21
+
22
+ b = Time.now.to_f
23
+
24
+ puts
25
+ puts "pid: #{ pid }"
26
+ puts "elapsed: #{ b - a }"
27
+ puts
28
+
29
+ require 'yaml'
30
+
31
+ pstrees.each do |pstree|
32
+ y pstree
33
+ end
metadata ADDED
@@ -0,0 +1,62 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: forkoff
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Ara T. Howard
8
+ autorequire: forkoff
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2008-04-17 00:00:00 -06:00
13
+ default_executable:
14
+ dependencies: []
15
+
16
+ description:
17
+ email: ara.t.howard@gmail.com
18
+ executables: []
19
+
20
+ extensions: []
21
+
22
+ extra_rdoc_files: []
23
+
24
+ files:
25
+ - gemspec.rb
26
+ - gen_readme.rb
27
+ - install.rb
28
+ - lib
29
+ - lib/forkoff.rb
30
+ - README
31
+ - samples
32
+ - samples/a.rb
33
+ - samples/b.rb
34
+ - samples/c.rb
35
+ has_rdoc: false
36
+ homepage: http://codeforpeople.com/lib/ruby/forkoff/
37
+ post_install_message:
38
+ rdoc_options: []
39
+
40
+ require_paths:
41
+ - lib
42
+ required_ruby_version: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: "0"
47
+ version:
48
+ required_rubygems_version: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: "0"
53
+ version:
54
+ requirements: []
55
+
56
+ rubyforge_project:
57
+ rubygems_version: 1.0.1
58
+ signing_key:
59
+ specification_version: 2
60
+ summary: forkoff
61
+ test_files: []
62
+