pwrake 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +30 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +57 -0
- data/Rakefile +1 -0
- data/bin/pwrake +36 -0
- data/lib/pwrake/application.rb +187 -0
- data/lib/pwrake/counter.rb +54 -0
- data/lib/pwrake/file_utils.rb +70 -0
- data/lib/pwrake/gfarm_feature.rb +229 -0
- data/lib/pwrake/graphviz.rb +113 -0
- data/lib/pwrake/locality_aware_queue.rb +254 -0
- data/lib/pwrake/logger.rb +153 -0
- data/lib/pwrake/master.rb +109 -0
- data/lib/pwrake/option.rb +367 -0
- data/lib/pwrake/profiler.rb +88 -0
- data/lib/pwrake/rake_modify.rb +14 -0
- data/lib/pwrake/shell.rb +151 -0
- data/lib/pwrake/task_algorithm.rb +171 -0
- data/lib/pwrake/task_queue.rb +167 -0
- data/lib/pwrake/timer.rb +22 -0
- data/lib/pwrake/version.rb +3 -0
- data/lib/pwrake.rb +18 -0
- data/pwrake.gemspec +19 -0
- data/setup.rb +1585 -0
- data/spec/001/Rakefile +5 -0
- data/spec/002/Rakefile +19 -0
- data/spec/003/Rakefile +9 -0
- data/spec/004/Rakefile +7 -0
- data/spec/005/Rakefile +11 -0
- data/spec/006/Rakefile +3 -0
- data/spec/006/pwrake_conf.yaml +3 -0
- data/spec/007/Rakefile +22 -0
- data/spec/008/Rakefile +3 -0
- data/spec/008/pwrake_conf.yaml +3 -0
- data/spec/009/Rakefile +18 -0
- data/spec/009/pwrake_conf.yaml +2 -0
- data/spec/helper.rb +82 -0
- data/spec/hosts +3 -0
- data/spec/pwrake_spec.rb +83 -0
- metadata +119 -0
data/spec/001/Rakefile
ADDED
data/spec/002/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require "rake/clean"
|
2
|
+
|
3
|
+
B = 4.times.map do |i|
|
4
|
+
file "B#{i}.dat" do |t|
|
5
|
+
sh "echo `hostname`:`pwd`"
|
6
|
+
sh "echo '#{t.name}' > #{t.name}"
|
7
|
+
end.name
|
8
|
+
end
|
9
|
+
|
10
|
+
A = 4.times.map do |i|
|
11
|
+
file "A#{i}.dat" => "B#{i}.dat" do |t|
|
12
|
+
sh "echo `hostname`:`pwd`"
|
13
|
+
sh "echo '#{t.name}' > #{t.name}"
|
14
|
+
end.name
|
15
|
+
end
|
16
|
+
|
17
|
+
task :default => A
|
18
|
+
|
19
|
+
CLEAN.include A,B
|
data/spec/003/Rakefile
ADDED
data/spec/004/Rakefile
ADDED
data/spec/005/Rakefile
ADDED
data/spec/006/Rakefile
ADDED
data/spec/007/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
t = task "A" do |t|
|
3
|
+
puts "task A #{Thread.current.inspect}"
|
4
|
+
c = 10.times.map do |i|
|
5
|
+
task "C#{i}" do |t|
|
6
|
+
print "task #{t.name} #{Thread.current.inspect}\n"
|
7
|
+
sh "echo #{t.name}; sleep 1"
|
8
|
+
end.name
|
9
|
+
end
|
10
|
+
|
11
|
+
(task "B" => c).invoke
|
12
|
+
print "invoke end\n"
|
13
|
+
end
|
14
|
+
|
15
|
+
task :default => ["A"] do |t|
|
16
|
+
print "task #{t.name} #{Thread.current.inspect}\n"
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
p Thread.current
|
21
|
+
|
22
|
+
p RUBY_RELEASE_DATE
|
data/spec/008/Rakefile
ADDED
data/spec/009/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require "rake/clean"
|
2
|
+
|
3
|
+
B = 4.times.map do |i|
|
4
|
+
task "B#{i}.dat" do |t|
|
5
|
+
sh "echo `hostname`:`pwd`"
|
6
|
+
end.name
|
7
|
+
end
|
8
|
+
|
9
|
+
A = 4.times.map do |i|
|
10
|
+
task "A#{i}.dat" => "B#{i}.dat" do |t|
|
11
|
+
sh "echo `hostname`:`pwd`"
|
12
|
+
end.name
|
13
|
+
end
|
14
|
+
|
15
|
+
task :default => A
|
16
|
+
|
17
|
+
CLEAN.include A,B
|
18
|
+
CLEAN.include '*.csv'
|
data/spec/helper.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
class Helper
|
2
|
+
|
3
|
+
@@spec_dir = File.expand_path(File.dirname(__FILE__))+"/"
|
4
|
+
@@pwrake = @@spec_dir+'../bin/pwrake'
|
5
|
+
|
6
|
+
@@show_command = false
|
7
|
+
@@show_result = false
|
8
|
+
|
9
|
+
def self.show=(f)
|
10
|
+
if f
|
11
|
+
@@show_command = true
|
12
|
+
@@show_result = true
|
13
|
+
else
|
14
|
+
@@show_command = false
|
15
|
+
@@show_result = false
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(dir=nil,args=nil)
|
20
|
+
@dir = @@spec_dir+(dir||"")
|
21
|
+
@args = args
|
22
|
+
end
|
23
|
+
|
24
|
+
attr_reader :n_files, :filelist, :result, :status
|
25
|
+
attr_reader :elapsed_time
|
26
|
+
|
27
|
+
def clean
|
28
|
+
Dir.chdir(@dir) do
|
29
|
+
`rake -q clean`
|
30
|
+
end
|
31
|
+
self
|
32
|
+
end
|
33
|
+
|
34
|
+
def run
|
35
|
+
cmd = "sh -c '#{@@pwrake} #{@args} 2>&1'"
|
36
|
+
if @@show_command
|
37
|
+
puts
|
38
|
+
puts "-- dir: #{@dir}"
|
39
|
+
puts "-- cmd: #{cmd}"
|
40
|
+
end
|
41
|
+
Dir.chdir(@dir) do
|
42
|
+
tm = Time.now
|
43
|
+
@result = `#{cmd}`
|
44
|
+
@status = $?
|
45
|
+
@elapsed_time = Time.now - tm
|
46
|
+
system "touch dummy; rm dummy"
|
47
|
+
@filelist = Dir.glob("*")
|
48
|
+
@n_files = @filelist.size
|
49
|
+
end
|
50
|
+
if @@show_result
|
51
|
+
puts @result
|
52
|
+
puts "-- status: #{@status.inspect}\n"
|
53
|
+
end
|
54
|
+
self
|
55
|
+
end
|
56
|
+
|
57
|
+
def success?
|
58
|
+
@status && @status.success?
|
59
|
+
end
|
60
|
+
|
61
|
+
def output_lines
|
62
|
+
@result.split("\n")
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
def self.read_hosts(file,ssh=nil)
|
67
|
+
cores = []
|
68
|
+
open(file) do |f|
|
69
|
+
while l = f.gets
|
70
|
+
l = $1 if /^([^#]*)#/ =~ l
|
71
|
+
host, ncore, group = l.split
|
72
|
+
if host
|
73
|
+
host = `ssh #{host} hostname`.chomp if ssh
|
74
|
+
ncore = (ncore || 1).to_i
|
75
|
+
cores.concat( [host] * ncore )
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
cores
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
data/spec/hosts
ADDED
data/spec/pwrake_spec.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
$spec_dir = File.expand_path(File.dirname(__FILE__))
|
2
|
+
require $spec_dir+"/helper"
|
3
|
+
$hosts = $spec_dir+"/hosts"
|
4
|
+
|
5
|
+
RSpec.configure do |config|
|
6
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
7
|
+
config.filter_run :focus
|
8
|
+
config.run_all_when_everything_filtered = true
|
9
|
+
end
|
10
|
+
|
11
|
+
Helper.show=false
|
12
|
+
|
13
|
+
describe Helper do
|
14
|
+
|
15
|
+
%w[ h V D n N P q g G T t v W X j ].each do |a|
|
16
|
+
context "dir=001 arg=-#{a}" do
|
17
|
+
subject { Helper.new("001","-"+a).run }
|
18
|
+
it { should be_success }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
%w[ N q g G t v X j ].each do |a|
|
23
|
+
context "dir=002 arg=-#{a}" do
|
24
|
+
subject { Helper.new("002","-"+a).clean.run }
|
25
|
+
it { should be_success }
|
26
|
+
its(:n_files) { should eq 9 }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
if File.exist?($hosts)
|
31
|
+
context "dir=002 --hostfile" do
|
32
|
+
subject { Helper.new("002","-F ../hosts").clean.run }
|
33
|
+
it { should be_success }
|
34
|
+
its(:n_files) { should eq 9 }
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
context "dir=003 w task argument" do
|
39
|
+
subject { Helper.new("003","hello[foo,bar]").run }
|
40
|
+
it { should be_success }
|
41
|
+
its(:result) { should eq "foo,bar\nfoo,bar\n" }
|
42
|
+
end
|
43
|
+
|
44
|
+
context "dir=004 -j4 elapsed time" do
|
45
|
+
subject { Helper.new("004","-j4").run }
|
46
|
+
it { should be_success }
|
47
|
+
its(:elapsed_time) { should be_within(1).of(2) } # 1..3 sec
|
48
|
+
end
|
49
|
+
|
50
|
+
if File.exist?($hosts)
|
51
|
+
context "dir=005 --hostfile" do
|
52
|
+
subject { Helper.new("005","-q -F ../hosts").run }
|
53
|
+
it { should be_success }
|
54
|
+
its("output_lines.sort") { should eq Helper.read_hosts($hosts,true).sort }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context "dir=006 PASS_ENV" do
|
59
|
+
subject { Helper.new("006","-q -F ../hosts ENV1=pass_successfully").run }
|
60
|
+
it { should be_success }
|
61
|
+
its(:result) { should eq "pass_successfully\n" }
|
62
|
+
end
|
63
|
+
|
64
|
+
# context "dir=007 invoke-in-task", :focus=>true do
|
65
|
+
context "dir=007 invoke-in-task" do
|
66
|
+
subject { Helper.new("007","-j10").run }
|
67
|
+
it { should be_success }
|
68
|
+
its(:elapsed_time) { should be_within(1).of(2) } # 1..3 sec
|
69
|
+
end
|
70
|
+
|
71
|
+
context "dir=008 --show-conf & PASS_ENV" do
|
72
|
+
subject { Helper.new("008","--show-conf ENV1=hoge").run }
|
73
|
+
it { should be_success }
|
74
|
+
its(:result) { should match(/ ENV1: hoge/) }
|
75
|
+
end
|
76
|
+
|
77
|
+
context "dir=009 PROFILE w GNU_TIME" do
|
78
|
+
subject { Helper.new("009").clean.run }
|
79
|
+
it { should be_success }
|
80
|
+
its(:n_files) { should eq 3 }
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
metadata
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pwrake
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 61
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 9
|
9
|
+
- 3
|
10
|
+
version: 0.9.3
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Masahiro TANAKA
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-12-26 00:00:00 Z
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: Parallel workflow extension for Rake
|
22
|
+
email:
|
23
|
+
- masa16.tanaka@gmail.com
|
24
|
+
executables:
|
25
|
+
- pwrake
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files: []
|
29
|
+
|
30
|
+
files:
|
31
|
+
- .gitignore
|
32
|
+
- Gemfile
|
33
|
+
- LICENSE.txt
|
34
|
+
- README.md
|
35
|
+
- Rakefile
|
36
|
+
- bin/pwrake
|
37
|
+
- lib/pwrake.rb
|
38
|
+
- lib/pwrake/application.rb
|
39
|
+
- lib/pwrake/counter.rb
|
40
|
+
- lib/pwrake/file_utils.rb
|
41
|
+
- lib/pwrake/gfarm_feature.rb
|
42
|
+
- lib/pwrake/graphviz.rb
|
43
|
+
- lib/pwrake/locality_aware_queue.rb
|
44
|
+
- lib/pwrake/logger.rb
|
45
|
+
- lib/pwrake/master.rb
|
46
|
+
- lib/pwrake/option.rb
|
47
|
+
- lib/pwrake/profiler.rb
|
48
|
+
- lib/pwrake/rake_modify.rb
|
49
|
+
- lib/pwrake/shell.rb
|
50
|
+
- lib/pwrake/task_algorithm.rb
|
51
|
+
- lib/pwrake/task_queue.rb
|
52
|
+
- lib/pwrake/timer.rb
|
53
|
+
- lib/pwrake/version.rb
|
54
|
+
- pwrake.gemspec
|
55
|
+
- setup.rb
|
56
|
+
- spec/001/Rakefile
|
57
|
+
- spec/002/Rakefile
|
58
|
+
- spec/003/Rakefile
|
59
|
+
- spec/004/Rakefile
|
60
|
+
- spec/005/Rakefile
|
61
|
+
- spec/006/Rakefile
|
62
|
+
- spec/006/pwrake_conf.yaml
|
63
|
+
- spec/007/Rakefile
|
64
|
+
- spec/008/Rakefile
|
65
|
+
- spec/008/pwrake_conf.yaml
|
66
|
+
- spec/009/Rakefile
|
67
|
+
- spec/009/pwrake_conf.yaml
|
68
|
+
- spec/helper.rb
|
69
|
+
- spec/hosts
|
70
|
+
- spec/pwrake_spec.rb
|
71
|
+
homepage: http://masa16.github.com/pwrake
|
72
|
+
licenses: []
|
73
|
+
|
74
|
+
post_install_message:
|
75
|
+
rdoc_options: []
|
76
|
+
|
77
|
+
require_paths:
|
78
|
+
- lib
|
79
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
80
|
+
none: false
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
hash: 3
|
85
|
+
segments:
|
86
|
+
- 0
|
87
|
+
version: "0"
|
88
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ">="
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
hash: 3
|
94
|
+
segments:
|
95
|
+
- 0
|
96
|
+
version: "0"
|
97
|
+
requirements: []
|
98
|
+
|
99
|
+
rubyforge_project:
|
100
|
+
rubygems_version: 1.8.24
|
101
|
+
signing_key:
|
102
|
+
specification_version: 3
|
103
|
+
summary: Adding Parallel and Distributed feature to Rake
|
104
|
+
test_files:
|
105
|
+
- spec/001/Rakefile
|
106
|
+
- spec/002/Rakefile
|
107
|
+
- spec/003/Rakefile
|
108
|
+
- spec/004/Rakefile
|
109
|
+
- spec/005/Rakefile
|
110
|
+
- spec/006/Rakefile
|
111
|
+
- spec/006/pwrake_conf.yaml
|
112
|
+
- spec/007/Rakefile
|
113
|
+
- spec/008/Rakefile
|
114
|
+
- spec/008/pwrake_conf.yaml
|
115
|
+
- spec/009/Rakefile
|
116
|
+
- spec/009/pwrake_conf.yaml
|
117
|
+
- spec/helper.rb
|
118
|
+
- spec/hosts
|
119
|
+
- spec/pwrake_spec.rb
|