scout-gear 2.0.0 → 5.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +59 -2
- data/VERSION +1 -1
- data/bin/scout +231 -24
- data/lib/scout/cmd.rb +344 -0
- data/lib/scout/concurrent_stream.rb +259 -0
- data/lib/scout/exceptions.rb +15 -8
- data/lib/scout/indiferent_hash/options.rb +8 -26
- data/lib/scout/log/color.rb +2 -2
- data/lib/scout/log/fingerprint.rb +11 -1
- data/lib/scout/log/progress/report.rb +0 -1
- data/lib/scout/log/progress/util.rb +1 -1
- data/lib/scout/log/progress.rb +4 -4
- data/lib/scout/log.rb +10 -2
- data/lib/scout/meta_extension.rb +15 -1
- data/lib/scout/misc/digest.rb +56 -0
- data/lib/scout/misc/filesystem.rb +26 -0
- data/lib/scout/misc/format.rb +1 -2
- data/lib/scout/misc/insist.rb +56 -0
- data/lib/scout/misc.rb +4 -11
- data/lib/scout/open/lock.rb +61 -0
- data/lib/scout/open/remote.rb +120 -0
- data/lib/scout/open/stream.rb +372 -0
- data/lib/scout/open/util.rb +225 -0
- data/lib/scout/open.rb +169 -0
- data/lib/scout/path/find.rb +67 -21
- data/lib/scout/path/tmpfile.rb +8 -0
- data/lib/scout/path/util.rb +14 -1
- data/lib/scout/path.rb +6 -30
- data/lib/scout/persist/open.rb +17 -0
- data/lib/scout/persist/path.rb +15 -0
- data/lib/scout/persist/serialize.rb +140 -0
- data/lib/scout/persist.rb +54 -0
- data/lib/scout/resource/path.rb +15 -0
- data/lib/scout/resource/produce/rake.rb +69 -0
- data/lib/scout/resource/produce.rb +246 -0
- data/lib/scout/resource/scout.rb +3 -0
- data/lib/scout/resource.rb +37 -0
- data/lib/scout/simple_opt/accessor.rb +1 -1
- data/lib/scout/simple_opt/doc.rb +4 -22
- data/lib/scout/simple_opt/parse.rb +4 -3
- data/lib/scout/tmpfile.rb +39 -1
- data/lib/scout/workflow/definition.rb +72 -0
- data/lib/scout/workflow/documentation.rb +77 -0
- data/lib/scout/workflow/step/info.rb +77 -0
- data/lib/scout/workflow/step.rb +96 -0
- data/lib/scout/workflow/task/inputs.rb +112 -0
- data/lib/scout/workflow/task.rb +141 -0
- data/lib/scout/workflow/usage.rb +294 -0
- data/lib/scout/workflow/util.rb +11 -0
- data/lib/scout/workflow.rb +39 -0
- data/lib/scout-gear.rb +4 -0
- data/lib/scout.rb +1 -0
- data/lib/workflow-scout.rb +2 -0
- data/scout-gear.gemspec +66 -5
- data/scout_commands/alias +48 -0
- data/scout_commands/find +83 -0
- data/scout_commands/glob +0 -0
- data/scout_commands/rbbt +23 -0
- data/scout_commands/workflow/task +707 -0
- data/test/scout/indiferent_hash/test_options.rb +11 -1
- data/test/scout/misc/test_digest.rb +30 -0
- data/test/scout/misc/test_filesystem.rb +30 -0
- data/test/scout/misc/test_insist.rb +13 -0
- data/test/scout/open/test_lock.rb +52 -0
- data/test/scout/open/test_remote.rb +25 -0
- data/test/scout/open/test_stream.rb +515 -0
- data/test/scout/open/test_util.rb +73 -0
- data/test/scout/path/test_find.rb +28 -0
- data/test/scout/persist/test_open.rb +37 -0
- data/test/scout/persist/test_path.rb +37 -0
- data/test/scout/persist/test_serialize.rb +114 -0
- data/test/scout/resource/test_path.rb +40 -0
- data/test/scout/resource/test_produce.rb +62 -0
- data/test/scout/test_cmd.rb +85 -0
- data/test/scout/test_concurrent_stream.rb +29 -0
- data/test/scout/test_misc.rb +0 -7
- data/test/scout/test_open.rb +146 -0
- data/test/scout/test_path.rb +3 -1
- data/test/scout/test_persist.rb +83 -0
- data/test/scout/test_resource.rb +26 -0
- data/test/scout/test_workflow.rb +87 -0
- data/test/scout/workflow/step/test_info.rb +28 -0
- data/test/scout/workflow/task/test_inputs.rb +182 -0
- data/test/scout/workflow/test_step.rb +36 -0
- data/test/scout/workflow/test_task.rb +178 -0
- data/test/scout/workflow/test_usage.rb +26 -0
- data/test/scout/workflow/test_util.rb +17 -0
- data/test/test_helper.rb +17 -0
- data/test/test_scout-gear.rb +0 -0
- metadata +64 -3
@@ -0,0 +1,83 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestPersist < Test::Unit::TestCase
|
5
|
+
def test_string
|
6
|
+
TmpFile.with_file do |tmpfile|
|
7
|
+
Path.setup(tmpfile)
|
8
|
+
obj = "TEST"
|
9
|
+
type = :string
|
10
|
+
refute tmpdir.persist.glob("*").any?
|
11
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
|
12
|
+
assert tmpdir.persist.glob("*").any?
|
13
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ raise "Error" }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_float_array
|
18
|
+
TmpFile.with_file do |tmpfile|
|
19
|
+
Path.setup(tmpfile)
|
20
|
+
obj = [1.2,2.2]
|
21
|
+
type = :float_array
|
22
|
+
refute tmpdir.persist.glob("*").any?
|
23
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
|
24
|
+
assert tmpdir.persist.glob("*").any?
|
25
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ raise "Error" }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_string_update
|
30
|
+
TmpFile.with_file do |tmpfile|
|
31
|
+
Path.setup(tmpfile)
|
32
|
+
obj = "TEST"
|
33
|
+
type = :string
|
34
|
+
refute tmpdir.persist.glob("*").any?
|
35
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
|
36
|
+
assert tmpdir.persist.glob("*").any?
|
37
|
+
assert_raises ScoutException do
|
38
|
+
Persist.persist(tmpfile, type, :dir => tmpdir.persist, :update => true){ raise ScoutException }
|
39
|
+
end
|
40
|
+
assert_raises ScoutException do
|
41
|
+
Persist.persist(tmpfile, type, :persist_dir => tmpdir.persist, :persist_update => true){ raise ScoutException }
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_stream
|
47
|
+
TmpFile.with_file do |tmpfile|
|
48
|
+
Path.setup(tmpfile)
|
49
|
+
obj = "TEST\nTEST"
|
50
|
+
stream = StringIO.new obj
|
51
|
+
stream.rewind
|
52
|
+
res = Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ stream }
|
53
|
+
assert IO === res
|
54
|
+
assert_equal obj, res.read
|
55
|
+
assert_equal obj, Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ raise ScoutException }
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_update_time
|
60
|
+
TmpFile.with_file do |dir|
|
61
|
+
Path.setup(dir)
|
62
|
+
obj = "TEST"
|
63
|
+
type = :string
|
64
|
+
|
65
|
+
Open.write(dir.file, "TEST")
|
66
|
+
assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file) }
|
67
|
+
Open.rm(dir.file)
|
68
|
+
assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file) }
|
69
|
+
|
70
|
+
sleep 1
|
71
|
+
Open.write(dir.file2, "TEST2")
|
72
|
+
assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file2) }
|
73
|
+
assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file2){ Open.read(dir.file2) }
|
74
|
+
|
75
|
+
sleep 1
|
76
|
+
Open.write(dir.file3, "TEST3")
|
77
|
+
sleep 1
|
78
|
+
Open.touch tmpdir.persist.glob("*").first
|
79
|
+
assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file3){ Open.read(dir.file3) }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestResourceUnit < Test::Unit::TestCase
|
5
|
+
module TestResource
|
6
|
+
extend Resource
|
7
|
+
|
8
|
+
self.subdir = Path.setup('tmp/test-resource')
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def test_root
|
13
|
+
|
14
|
+
p = TestResource.root.some_file
|
15
|
+
assert p.find(:user).include?(ENV["HOME"])
|
16
|
+
end
|
17
|
+
|
18
|
+
def __test_identify
|
19
|
+
assert_equal 'etc/', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/etc/'))
|
20
|
+
assert_equal 'share/databases/', Rbbt.identify('/usr/local/share/rbbt/databases/')
|
21
|
+
assert_equal 'share/databases/DATABASE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE')
|
22
|
+
assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
|
23
|
+
assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/share/databases/DATABASE/FILE'))
|
24
|
+
assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestWorkflow < Test::Unit::TestCase
|
5
|
+
|
6
|
+
module Pantry
|
7
|
+
extend Resource
|
8
|
+
self.subdir = 'share/pantry'
|
9
|
+
|
10
|
+
Pantry.claim Pantry.eggs, :proc do
|
11
|
+
Log.info "Buying Eggs in the store"
|
12
|
+
"Eggs"
|
13
|
+
end
|
14
|
+
|
15
|
+
Pantry.claim Pantry.flour, :proc do
|
16
|
+
Log.info "Buying Flour in the store"
|
17
|
+
"Flour"
|
18
|
+
end
|
19
|
+
|
20
|
+
Pantry.claim Pantry.blueberries, :proc do
|
21
|
+
Log.info "Buying Bluberries in the store"
|
22
|
+
"Bluberries"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
module Baking
|
27
|
+
def self.whisk(eggs)
|
28
|
+
"Whisking eggs from #{eggs}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.mix(base, mixer)
|
32
|
+
"Mixing base (#{base}) with mixer (#{mixer})"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.bake(batter)
|
36
|
+
"Baking batter (#{batter})"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
module Baking
|
41
|
+
extend Workflow
|
42
|
+
|
43
|
+
dep :prepare_batter
|
44
|
+
task :bake_muffin_tray => :string do
|
45
|
+
Baking.bake(step(:prepare_batter).load)
|
46
|
+
end
|
47
|
+
|
48
|
+
dep :whisk_eggs
|
49
|
+
input :add_bluberries, :boolean
|
50
|
+
task :prepare_batter => :string do |add_bluberries|
|
51
|
+
whisked_eggs = step(:whisk_eggs).load
|
52
|
+
batter = Baking.mix(whisked_eggs, Pantry.flour.produce)
|
53
|
+
|
54
|
+
if add_bluberries
|
55
|
+
batter = Baking.mix(batter, Pantry.blueberries.produce)
|
56
|
+
end
|
57
|
+
|
58
|
+
batter
|
59
|
+
end
|
60
|
+
|
61
|
+
task :whisk_eggs => :string do
|
62
|
+
Baking.whisk(Pantry.eggs.produce)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
setup do
|
67
|
+
Baking.directory = tmpdir.var.jobs.baking.find
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_task
|
71
|
+
wf = Workflow.annonymous_workflow do
|
72
|
+
task :length => :integer do
|
73
|
+
self.length
|
74
|
+
end
|
75
|
+
end
|
76
|
+
bindings = "12345"
|
77
|
+
assert_equal 5, wf.tasks[:length].exec_on(bindings)
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_baking
|
81
|
+
assert_equal "Baking batter (Mixing base (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour)) with mixer (share/pantry/blueberries))",
|
82
|
+
Baking.job(:bake_muffin_tray, :add_bluberries => true).run
|
83
|
+
|
84
|
+
assert_equal "Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))",
|
85
|
+
Baking.job(:bake_muffin_tray).run
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
require 'scout/workflow'
|
4
|
+
|
5
|
+
class TestStepInfo < Test::Unit::TestCase
|
6
|
+
def test_dependency
|
7
|
+
Log.with_severity 0 do
|
8
|
+
tmpfile = tmpdir.test_step
|
9
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
10
|
+
s.length
|
11
|
+
end
|
12
|
+
|
13
|
+
assert_equal 2, step1.exec
|
14
|
+
assert_equal 2, step1.run
|
15
|
+
|
16
|
+
step2 = Step.new tmpfile.step2 do
|
17
|
+
step1 = dependencies.first
|
18
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
19
|
+
end
|
20
|
+
|
21
|
+
step2.dependencies = [step1]
|
22
|
+
|
23
|
+
assert_equal "12 has 2 characters", step2.run
|
24
|
+
assert_equal "12 has 2 characters", step2.run
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
@@ -0,0 +1,182 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/workflow/util'
|
5
|
+
|
6
|
+
class TestTaskInput < Test::Unit::TestCase
|
7
|
+
def example_workflow
|
8
|
+
Workflow.annonymous_workflow do
|
9
|
+
input :string, :string, "", "String"
|
10
|
+
input :integer, :integer, "", 1
|
11
|
+
input :float, :float, "", 1.1
|
12
|
+
input :boolean, :boolean, "", false
|
13
|
+
input :array, :array, "", %w(a b)
|
14
|
+
input :path, :path, "", "dir/subdir/file"
|
15
|
+
input :file, :file, ""
|
16
|
+
input :integer_array, :integer_array, "", [1, 2]
|
17
|
+
input :float_array, :float_array, "", [1.1, 2.2]
|
18
|
+
input :boolean_array, :boolean_array, "", [true, false, true]
|
19
|
+
input :path_array, :path_array, "", %w(dir/subdir/file1 dir/subdir/file2)
|
20
|
+
input :file_array, :file_array
|
21
|
+
task :task => :array do
|
22
|
+
inputs
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def example_task
|
28
|
+
example_workflow.tasks[:task]
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_assign_inputs
|
32
|
+
task = self.example_task
|
33
|
+
|
34
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
|
35
|
+
|
36
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
37
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_boolean
|
42
|
+
task = self.example_task
|
43
|
+
|
44
|
+
assert_equal [:boolean], task.assign_inputs(:string => "String", :integer => 1, :boolean => true).last
|
45
|
+
|
46
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
47
|
+
Open.open(integer_array_file) do |f|
|
48
|
+
inputs, _ = task.assign_inputs(:string => "String", :integer => 2, :integer_array => f)
|
49
|
+
input_pos = task.inputs.index{|p| p.first == :integer_array}
|
50
|
+
assert File === inputs[input_pos]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_keep_stream
|
56
|
+
task = self.example_task
|
57
|
+
|
58
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
|
59
|
+
|
60
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
61
|
+
Open.open(integer_array_file) do |f|
|
62
|
+
inputs, _ = task.assign_inputs(:string => "String", :integer => 2, :integer_array => f)
|
63
|
+
input_pos = task.inputs.index{|p| p.first == :integer_array}
|
64
|
+
assert File === inputs[input_pos]
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
def test_digest_inputs
|
71
|
+
task = self.example_task
|
72
|
+
|
73
|
+
TmpFile.with_file("2\n3") do |integer_array_file|
|
74
|
+
assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
|
75
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def test_digest_stream
|
80
|
+
task = self.example_task
|
81
|
+
|
82
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
|
83
|
+
|
84
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
85
|
+
hash1 = Open.open(integer_array_file) do |f|
|
86
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
|
87
|
+
end
|
88
|
+
hash2 = Open.open(integer_array_file) do |f|
|
89
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
|
90
|
+
end
|
91
|
+
assert_equal hash1, hash2
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
|
97
|
+
def test_digest_file
|
98
|
+
task = self.example_task
|
99
|
+
|
100
|
+
TmpFile.with_file("2\n3") do |integer_array_file|
|
101
|
+
assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
|
102
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_save_and_load
|
107
|
+
task = self.example_task
|
108
|
+
|
109
|
+
TmpFile.with_file("2\n3") do |integer_array_file|
|
110
|
+
inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
|
111
|
+
original_digest = task.digest_inputs(inputs)
|
112
|
+
|
113
|
+
TmpFile.with_file do |save_directory|
|
114
|
+
task.save_inputs(save_directory, inputs)
|
115
|
+
new_inputs = task.load_inputs(save_directory)
|
116
|
+
new_digest = task.digest_inputs(new_inputs)
|
117
|
+
assert_equal original_digest, new_digest
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_save_and_load_file
|
123
|
+
task = self.example_task
|
124
|
+
|
125
|
+
TmpFile.with_file("TEST") do |somefile|
|
126
|
+
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
127
|
+
original_digest = task.digest_inputs(inputs)
|
128
|
+
|
129
|
+
TmpFile.with_file do |save_directory|
|
130
|
+
task.save_inputs(save_directory, inputs)
|
131
|
+
Open.rm somefile
|
132
|
+
new_inputs = task.load_inputs(save_directory)
|
133
|
+
new_digest = task.digest_inputs(new_inputs)
|
134
|
+
assert_equal original_digest, new_digest
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_save_and_load_file_with_copy
|
140
|
+
task = self.example_task
|
141
|
+
|
142
|
+
TmpFile.with_file("TEST") do |somefile|
|
143
|
+
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
144
|
+
original_digest = task.digest_inputs(inputs)
|
145
|
+
|
146
|
+
TmpFile.with_file do |save_directory|
|
147
|
+
task.save_inputs(save_directory, inputs)
|
148
|
+
TmpFile.with_file do |copy_directory|
|
149
|
+
Open.cp save_directory, copy_directory
|
150
|
+
Open.rm_rf save_directory
|
151
|
+
new_inputs = task.load_inputs(copy_directory)
|
152
|
+
new_digest = task.digest_inputs(new_inputs)
|
153
|
+
assert_equal original_digest, new_digest
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
|
160
|
+
def test_save_and_load_file_array
|
161
|
+
task = self.example_task
|
162
|
+
|
163
|
+
TmpFile.with_file do |dir|
|
164
|
+
file1 = File.join(dir, 'subdir1/file')
|
165
|
+
file2 = File.join(dir, 'subdir2/file')
|
166
|
+
|
167
|
+
Open.write(file1, "TEST1")
|
168
|
+
Open.write(file2, "TEST2")
|
169
|
+
inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
|
170
|
+
original_digest = task.digest_inputs(inputs)
|
171
|
+
|
172
|
+
TmpFile.with_file do |save_directory|
|
173
|
+
task.save_inputs(save_directory, inputs)
|
174
|
+
Open.rm(file1)
|
175
|
+
Open.rm(file2)
|
176
|
+
new_inputs = task.load_inputs(save_directory)
|
177
|
+
new_digest = task.digest_inputs(new_inputs)
|
178
|
+
assert_equal original_digest, new_digest
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestWorkflowStep < Test::Unit::TestCase
|
5
|
+
|
6
|
+
def test_step
|
7
|
+
TmpFile.with_file do |tmpfile|
|
8
|
+
step = Step.new tmpfile, ["12"] do |s|
|
9
|
+
s.length
|
10
|
+
end
|
11
|
+
step.type = :integer
|
12
|
+
|
13
|
+
assert_equal 2, step.run
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_dependency
|
18
|
+
Log.with_severity 0 do
|
19
|
+
tmpfile = tmpdir.test_step
|
20
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
21
|
+
s.length
|
22
|
+
end
|
23
|
+
|
24
|
+
step2 = Step.new tmpfile.step2 do
|
25
|
+
step1 = dependencies.first
|
26
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
27
|
+
end
|
28
|
+
|
29
|
+
step2.dependencies = [step1]
|
30
|
+
|
31
|
+
|
32
|
+
assert_equal "12 has 2 characters", step2.run
|
33
|
+
assert_equal "12 has 2 characters", step2.run
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestTask < Test::Unit::TestCase
|
5
|
+
def test_basic_task
|
6
|
+
task = Task.setup do |s=""|
|
7
|
+
(self + s).length
|
8
|
+
end
|
9
|
+
|
10
|
+
assert_equal 4, task.exec_on("1234")
|
11
|
+
assert_equal 6, task.exec_on("1234","56")
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_step
|
15
|
+
task = Task.setup do |s=""|
|
16
|
+
s.length
|
17
|
+
end
|
18
|
+
|
19
|
+
s = task.job('test', ['12'])
|
20
|
+
assert_equal 2, s.run
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_override_inputs
|
24
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
25
|
+
input :input1, :string
|
26
|
+
task :step1 => :string do |i| i end
|
27
|
+
|
28
|
+
dep :step1, :input1 => 1
|
29
|
+
input :input2, :string
|
30
|
+
task :step2 => :string do |i| i end
|
31
|
+
end
|
32
|
+
|
33
|
+
job = wf.job(:step2, :input1 => 2)
|
34
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
35
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_override_inputs_block
|
39
|
+
Log.severity = 0
|
40
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
41
|
+
input :input1, :string
|
42
|
+
task :step1 => :string do |i| i end
|
43
|
+
|
44
|
+
dep :step1 do |id,options|
|
45
|
+
{:inputs => {:input1 => 1}}
|
46
|
+
end
|
47
|
+
input :input2, :string
|
48
|
+
task :step2 => :string do |i| i end
|
49
|
+
end
|
50
|
+
|
51
|
+
job = wf.job(:step2, :input1 => 2)
|
52
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
53
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_task_override_dep
|
57
|
+
Log.severity = 0
|
58
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
59
|
+
input :input1, :integer
|
60
|
+
task :step1 => :integer do |i| i end
|
61
|
+
|
62
|
+
dep :step1
|
63
|
+
input :input2, :integer, "Integer", 3
|
64
|
+
task :step2 => :integer do |i| i * step(:step1).load end
|
65
|
+
end
|
66
|
+
|
67
|
+
assert_equal 6, wf.job(:step2, :input1 => 2, :input2 => 3).run
|
68
|
+
|
69
|
+
step1_job = wf.job(:step1, :input1 => 6)
|
70
|
+
assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
|
71
|
+
|
72
|
+
assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
|
73
|
+
|
74
|
+
assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).run
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_input_dep
|
78
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
79
|
+
input :v1, :integer
|
80
|
+
input :v2, :integer
|
81
|
+
task :sum => :integer do |v1,v2|
|
82
|
+
v1 + v2
|
83
|
+
end
|
84
|
+
|
85
|
+
input :input1, :integer
|
86
|
+
task :step1 => :integer do |i| i end
|
87
|
+
|
88
|
+
input :input2, :integer
|
89
|
+
task :step2 => :integer do |i| i end
|
90
|
+
|
91
|
+
dep :step1, :input1 => 2
|
92
|
+
dep :step2, :input2 => 3
|
93
|
+
dep :sum, :v1 => :step1, :v2 => :step2
|
94
|
+
task :my_sum => :integer do
|
95
|
+
dependencies.last.load
|
96
|
+
end
|
97
|
+
|
98
|
+
dep :my_sum
|
99
|
+
task :double => :integer do
|
100
|
+
step(:my_sum).load * 2
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
job = wf.job(:my_sum)
|
105
|
+
assert_equal 5, job.run
|
106
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
107
|
+
|
108
|
+
job = wf.job(:double)
|
109
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
110
|
+
|
111
|
+
step1 = wf.job(:step1, :input1 => 3)
|
112
|
+
job = wf.job(:double, "TaskInputs#step1" => step1)
|
113
|
+
assert_equal 12, job.run
|
114
|
+
assert_not_equal Task::DEFAULT_NAME, job.name
|
115
|
+
|
116
|
+
step1 = wf.job(:step1, :input1 => 4)
|
117
|
+
job = wf.job(:double, "TaskInputs#step1" => step1)
|
118
|
+
assert_equal 14, job.run
|
119
|
+
assert_not_equal Task::DEFAULT_NAME, job.name
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_input_dep_override
|
123
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
124
|
+
input :v1, :integer
|
125
|
+
input :v2, :integer
|
126
|
+
task :sum => :integer do |v1,v2|
|
127
|
+
v1 + v2
|
128
|
+
end
|
129
|
+
|
130
|
+
input :input1, :integer
|
131
|
+
task :step1 => :integer do |i| i end
|
132
|
+
|
133
|
+
input :input2, :integer
|
134
|
+
task :step2 => :integer do |i| i end
|
135
|
+
|
136
|
+
dep :step1
|
137
|
+
dep :step2
|
138
|
+
task :my_sum => :integer do
|
139
|
+
dependencies.inject(0){|acc,d| acc += d.load }
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
step2 = wf.job(:step2, :input2 => 4)
|
144
|
+
job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> step2)
|
145
|
+
assert_equal 6, job.exec
|
146
|
+
|
147
|
+
job = wf.job(:my_sum, :input1 => 2, :input2 => 3)
|
148
|
+
assert_equal 5, job.run
|
149
|
+
|
150
|
+
TmpFile.with_file(4) do |file|
|
151
|
+
job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> file)
|
152
|
+
assert_equal 6, job.run
|
153
|
+
assert_not_equal Task::DEFAULT_NAME, job.name
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def test_input_rename
|
158
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
159
|
+
input :v1, :integer
|
160
|
+
input :v2, :integer
|
161
|
+
task :sum => :integer do |v1,v2|
|
162
|
+
v1 + v2
|
163
|
+
end
|
164
|
+
|
165
|
+
input :vv1, :integer
|
166
|
+
input :vv2, :integer
|
167
|
+
dep :sum, :v1 => :vv1, :v2 => :vv2
|
168
|
+
task :my_sum => :integer do
|
169
|
+
dependencies.last.load
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
Log.severity = 0
|
174
|
+
job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
|
175
|
+
assert_equal 5, job.run
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/workflow'
|
5
|
+
class TestWorkflowUsage < Test::Unit::TestCase
|
6
|
+
module UsageWorkflow
|
7
|
+
extend Workflow
|
8
|
+
|
9
|
+
desc "Desc"
|
10
|
+
input :array, :array, "Array"
|
11
|
+
task :step1 => :string do
|
12
|
+
end
|
13
|
+
|
14
|
+
dep :step1
|
15
|
+
desc "Desc2"
|
16
|
+
input :float, :float, "Float"
|
17
|
+
task :step2 => :string do
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def __test_usage
|
22
|
+
UsageWorkflow.tasks[:step1].doc
|
23
|
+
UsageWorkflow.doc
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/workflow'
|
5
|
+
|
6
|
+
class TestWorkflowUtil < Test::Unit::TestCase
|
7
|
+
def test_annonymous_workflow
|
8
|
+
wf = Workflow.annonymous_workflow "TEST" do
|
9
|
+
task :length => :integer do
|
10
|
+
self.length
|
11
|
+
end
|
12
|
+
end
|
13
|
+
bindings = "12345"
|
14
|
+
assert_equal 5, wf.tasks[:length].exec_on(bindings)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
data/test/test_helper.rb
CHANGED
@@ -22,6 +22,23 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
22
22
|
#require 'scout/helper/misc/development'
|
23
23
|
require 'scout/tmpfile'
|
24
24
|
require 'scout/log'
|
25
|
+
require 'scout/persist'
|
26
|
+
require 'scout/workflow'
|
25
27
|
|
26
28
|
class Test::Unit::TestCase
|
29
|
+
|
30
|
+
def tmpdir
|
31
|
+
@tmpdir = Path.setup('tmp/test_tmpdir').find
|
32
|
+
end
|
33
|
+
|
34
|
+
setup do
|
35
|
+
Persist.cache_dir = tmpdir.var.cache
|
36
|
+
Open.remote_cache_dir = tmpdir.var.cache
|
37
|
+
Workflow.directory = tmpdir.var.jobs
|
38
|
+
Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }
|
39
|
+
end
|
40
|
+
|
41
|
+
teardown do
|
42
|
+
Open.rm_rf tmpdir
|
43
|
+
end
|
27
44
|
end
|
File without changes
|