scout-gear 2.0.0 → 5.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +59 -2
- data/VERSION +1 -1
- data/bin/scout +231 -24
- data/lib/scout/cmd.rb +344 -0
- data/lib/scout/concurrent_stream.rb +259 -0
- data/lib/scout/exceptions.rb +15 -8
- data/lib/scout/indiferent_hash/options.rb +8 -26
- data/lib/scout/log/color.rb +2 -2
- data/lib/scout/log/fingerprint.rb +11 -1
- data/lib/scout/log/progress/report.rb +0 -1
- data/lib/scout/log/progress/util.rb +1 -1
- data/lib/scout/log/progress.rb +4 -4
- data/lib/scout/log.rb +10 -2
- data/lib/scout/meta_extension.rb +15 -1
- data/lib/scout/misc/digest.rb +56 -0
- data/lib/scout/misc/filesystem.rb +26 -0
- data/lib/scout/misc/format.rb +1 -2
- data/lib/scout/misc/insist.rb +56 -0
- data/lib/scout/misc.rb +4 -11
- data/lib/scout/open/lock.rb +61 -0
- data/lib/scout/open/remote.rb +120 -0
- data/lib/scout/open/stream.rb +372 -0
- data/lib/scout/open/util.rb +225 -0
- data/lib/scout/open.rb +169 -0
- data/lib/scout/path/find.rb +67 -21
- data/lib/scout/path/tmpfile.rb +8 -0
- data/lib/scout/path/util.rb +14 -1
- data/lib/scout/path.rb +6 -30
- data/lib/scout/persist/open.rb +17 -0
- data/lib/scout/persist/path.rb +15 -0
- data/lib/scout/persist/serialize.rb +140 -0
- data/lib/scout/persist.rb +54 -0
- data/lib/scout/resource/path.rb +15 -0
- data/lib/scout/resource/produce/rake.rb +69 -0
- data/lib/scout/resource/produce.rb +246 -0
- data/lib/scout/resource/scout.rb +3 -0
- data/lib/scout/resource.rb +37 -0
- data/lib/scout/simple_opt/accessor.rb +1 -1
- data/lib/scout/simple_opt/doc.rb +4 -22
- data/lib/scout/simple_opt/parse.rb +4 -3
- data/lib/scout/tmpfile.rb +39 -1
- data/lib/scout/workflow/definition.rb +72 -0
- data/lib/scout/workflow/documentation.rb +77 -0
- data/lib/scout/workflow/step/info.rb +77 -0
- data/lib/scout/workflow/step.rb +96 -0
- data/lib/scout/workflow/task/inputs.rb +112 -0
- data/lib/scout/workflow/task.rb +141 -0
- data/lib/scout/workflow/usage.rb +294 -0
- data/lib/scout/workflow/util.rb +11 -0
- data/lib/scout/workflow.rb +39 -0
- data/lib/scout-gear.rb +4 -0
- data/lib/scout.rb +1 -0
- data/lib/workflow-scout.rb +2 -0
- data/scout-gear.gemspec +66 -5
- data/scout_commands/alias +48 -0
- data/scout_commands/find +83 -0
- data/scout_commands/glob +0 -0
- data/scout_commands/rbbt +23 -0
- data/scout_commands/workflow/task +707 -0
- data/test/scout/indiferent_hash/test_options.rb +11 -1
- data/test/scout/misc/test_digest.rb +30 -0
- data/test/scout/misc/test_filesystem.rb +30 -0
- data/test/scout/misc/test_insist.rb +13 -0
- data/test/scout/open/test_lock.rb +52 -0
- data/test/scout/open/test_remote.rb +25 -0
- data/test/scout/open/test_stream.rb +515 -0
- data/test/scout/open/test_util.rb +73 -0
- data/test/scout/path/test_find.rb +28 -0
- data/test/scout/persist/test_open.rb +37 -0
- data/test/scout/persist/test_path.rb +37 -0
- data/test/scout/persist/test_serialize.rb +114 -0
- data/test/scout/resource/test_path.rb +40 -0
- data/test/scout/resource/test_produce.rb +62 -0
- data/test/scout/test_cmd.rb +85 -0
- data/test/scout/test_concurrent_stream.rb +29 -0
- data/test/scout/test_misc.rb +0 -7
- data/test/scout/test_open.rb +146 -0
- data/test/scout/test_path.rb +3 -1
- data/test/scout/test_persist.rb +83 -0
- data/test/scout/test_resource.rb +26 -0
- data/test/scout/test_workflow.rb +87 -0
- data/test/scout/workflow/step/test_info.rb +28 -0
- data/test/scout/workflow/task/test_inputs.rb +182 -0
- data/test/scout/workflow/test_step.rb +36 -0
- data/test/scout/workflow/test_task.rb +178 -0
- data/test/scout/workflow/test_usage.rb +26 -0
- data/test/scout/workflow/test_util.rb +17 -0
- data/test/test_helper.rb +17 -0
- data/test/test_scout-gear.rb +0 -0
- metadata +64 -3
@@ -0,0 +1,83 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestPersist < Test::Unit::TestCase
|
5
|
+
def test_string
|
6
|
+
TmpFile.with_file do |tmpfile|
|
7
|
+
Path.setup(tmpfile)
|
8
|
+
obj = "TEST"
|
9
|
+
type = :string
|
10
|
+
refute tmpdir.persist.glob("*").any?
|
11
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
|
12
|
+
assert tmpdir.persist.glob("*").any?
|
13
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ raise "Error" }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_float_array
|
18
|
+
TmpFile.with_file do |tmpfile|
|
19
|
+
Path.setup(tmpfile)
|
20
|
+
obj = [1.2,2.2]
|
21
|
+
type = :float_array
|
22
|
+
refute tmpdir.persist.glob("*").any?
|
23
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
|
24
|
+
assert tmpdir.persist.glob("*").any?
|
25
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ raise "Error" }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_string_update
|
30
|
+
TmpFile.with_file do |tmpfile|
|
31
|
+
Path.setup(tmpfile)
|
32
|
+
obj = "TEST"
|
33
|
+
type = :string
|
34
|
+
refute tmpdir.persist.glob("*").any?
|
35
|
+
assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
|
36
|
+
assert tmpdir.persist.glob("*").any?
|
37
|
+
assert_raises ScoutException do
|
38
|
+
Persist.persist(tmpfile, type, :dir => tmpdir.persist, :update => true){ raise ScoutException }
|
39
|
+
end
|
40
|
+
assert_raises ScoutException do
|
41
|
+
Persist.persist(tmpfile, type, :persist_dir => tmpdir.persist, :persist_update => true){ raise ScoutException }
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_stream
|
47
|
+
TmpFile.with_file do |tmpfile|
|
48
|
+
Path.setup(tmpfile)
|
49
|
+
obj = "TEST\nTEST"
|
50
|
+
stream = StringIO.new obj
|
51
|
+
stream.rewind
|
52
|
+
res = Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ stream }
|
53
|
+
assert IO === res
|
54
|
+
assert_equal obj, res.read
|
55
|
+
assert_equal obj, Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ raise ScoutException }
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_update_time
|
60
|
+
TmpFile.with_file do |dir|
|
61
|
+
Path.setup(dir)
|
62
|
+
obj = "TEST"
|
63
|
+
type = :string
|
64
|
+
|
65
|
+
Open.write(dir.file, "TEST")
|
66
|
+
assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file) }
|
67
|
+
Open.rm(dir.file)
|
68
|
+
assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file) }
|
69
|
+
|
70
|
+
sleep 1
|
71
|
+
Open.write(dir.file2, "TEST2")
|
72
|
+
assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file2) }
|
73
|
+
assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file2){ Open.read(dir.file2) }
|
74
|
+
|
75
|
+
sleep 1
|
76
|
+
Open.write(dir.file3, "TEST3")
|
77
|
+
sleep 1
|
78
|
+
Open.touch tmpdir.persist.glob("*").first
|
79
|
+
assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file3){ Open.read(dir.file3) }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestResourceUnit < Test::Unit::TestCase
|
5
|
+
module TestResource
|
6
|
+
extend Resource
|
7
|
+
|
8
|
+
self.subdir = Path.setup('tmp/test-resource')
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def test_root
|
13
|
+
|
14
|
+
p = TestResource.root.some_file
|
15
|
+
assert p.find(:user).include?(ENV["HOME"])
|
16
|
+
end
|
17
|
+
|
18
|
+
def __test_identify
|
19
|
+
assert_equal 'etc/', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/etc/'))
|
20
|
+
assert_equal 'share/databases/', Rbbt.identify('/usr/local/share/rbbt/databases/')
|
21
|
+
assert_equal 'share/databases/DATABASE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE')
|
22
|
+
assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
|
23
|
+
assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/share/databases/DATABASE/FILE'))
|
24
|
+
assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestWorkflow < Test::Unit::TestCase
|
5
|
+
|
6
|
+
module Pantry
|
7
|
+
extend Resource
|
8
|
+
self.subdir = 'share/pantry'
|
9
|
+
|
10
|
+
Pantry.claim Pantry.eggs, :proc do
|
11
|
+
Log.info "Buying Eggs in the store"
|
12
|
+
"Eggs"
|
13
|
+
end
|
14
|
+
|
15
|
+
Pantry.claim Pantry.flour, :proc do
|
16
|
+
Log.info "Buying Flour in the store"
|
17
|
+
"Flour"
|
18
|
+
end
|
19
|
+
|
20
|
+
Pantry.claim Pantry.blueberries, :proc do
|
21
|
+
Log.info "Buying Bluberries in the store"
|
22
|
+
"Bluberries"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
module Baking
|
27
|
+
def self.whisk(eggs)
|
28
|
+
"Whisking eggs from #{eggs}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.mix(base, mixer)
|
32
|
+
"Mixing base (#{base}) with mixer (#{mixer})"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.bake(batter)
|
36
|
+
"Baking batter (#{batter})"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
module Baking
|
41
|
+
extend Workflow
|
42
|
+
|
43
|
+
dep :prepare_batter
|
44
|
+
task :bake_muffin_tray => :string do
|
45
|
+
Baking.bake(step(:prepare_batter).load)
|
46
|
+
end
|
47
|
+
|
48
|
+
dep :whisk_eggs
|
49
|
+
input :add_bluberries, :boolean
|
50
|
+
task :prepare_batter => :string do |add_bluberries|
|
51
|
+
whisked_eggs = step(:whisk_eggs).load
|
52
|
+
batter = Baking.mix(whisked_eggs, Pantry.flour.produce)
|
53
|
+
|
54
|
+
if add_bluberries
|
55
|
+
batter = Baking.mix(batter, Pantry.blueberries.produce)
|
56
|
+
end
|
57
|
+
|
58
|
+
batter
|
59
|
+
end
|
60
|
+
|
61
|
+
task :whisk_eggs => :string do
|
62
|
+
Baking.whisk(Pantry.eggs.produce)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
setup do
|
67
|
+
Baking.directory = tmpdir.var.jobs.baking.find
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_task
|
71
|
+
wf = Workflow.annonymous_workflow do
|
72
|
+
task :length => :integer do
|
73
|
+
self.length
|
74
|
+
end
|
75
|
+
end
|
76
|
+
bindings = "12345"
|
77
|
+
assert_equal 5, wf.tasks[:length].exec_on(bindings)
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_baking
|
81
|
+
assert_equal "Baking batter (Mixing base (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour)) with mixer (share/pantry/blueberries))",
|
82
|
+
Baking.job(:bake_muffin_tray, :add_bluberries => true).run
|
83
|
+
|
84
|
+
assert_equal "Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))",
|
85
|
+
Baking.job(:bake_muffin_tray).run
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
require 'scout/workflow'
|
4
|
+
|
5
|
+
class TestStepInfo < Test::Unit::TestCase
|
6
|
+
def test_dependency
|
7
|
+
Log.with_severity 0 do
|
8
|
+
tmpfile = tmpdir.test_step
|
9
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
10
|
+
s.length
|
11
|
+
end
|
12
|
+
|
13
|
+
assert_equal 2, step1.exec
|
14
|
+
assert_equal 2, step1.run
|
15
|
+
|
16
|
+
step2 = Step.new tmpfile.step2 do
|
17
|
+
step1 = dependencies.first
|
18
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
19
|
+
end
|
20
|
+
|
21
|
+
step2.dependencies = [step1]
|
22
|
+
|
23
|
+
assert_equal "12 has 2 characters", step2.run
|
24
|
+
assert_equal "12 has 2 characters", step2.run
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
@@ -0,0 +1,182 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/workflow/util'
|
5
|
+
|
6
|
+
class TestTaskInput < Test::Unit::TestCase
|
7
|
+
def example_workflow
|
8
|
+
Workflow.annonymous_workflow do
|
9
|
+
input :string, :string, "", "String"
|
10
|
+
input :integer, :integer, "", 1
|
11
|
+
input :float, :float, "", 1.1
|
12
|
+
input :boolean, :boolean, "", false
|
13
|
+
input :array, :array, "", %w(a b)
|
14
|
+
input :path, :path, "", "dir/subdir/file"
|
15
|
+
input :file, :file, ""
|
16
|
+
input :integer_array, :integer_array, "", [1, 2]
|
17
|
+
input :float_array, :float_array, "", [1.1, 2.2]
|
18
|
+
input :boolean_array, :boolean_array, "", [true, false, true]
|
19
|
+
input :path_array, :path_array, "", %w(dir/subdir/file1 dir/subdir/file2)
|
20
|
+
input :file_array, :file_array
|
21
|
+
task :task => :array do
|
22
|
+
inputs
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def example_task
|
28
|
+
example_workflow.tasks[:task]
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_assign_inputs
|
32
|
+
task = self.example_task
|
33
|
+
|
34
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
|
35
|
+
|
36
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
37
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_boolean
|
42
|
+
task = self.example_task
|
43
|
+
|
44
|
+
assert_equal [:boolean], task.assign_inputs(:string => "String", :integer => 1, :boolean => true).last
|
45
|
+
|
46
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
47
|
+
Open.open(integer_array_file) do |f|
|
48
|
+
inputs, _ = task.assign_inputs(:string => "String", :integer => 2, :integer_array => f)
|
49
|
+
input_pos = task.inputs.index{|p| p.first == :integer_array}
|
50
|
+
assert File === inputs[input_pos]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_keep_stream
|
56
|
+
task = self.example_task
|
57
|
+
|
58
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
|
59
|
+
|
60
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
61
|
+
Open.open(integer_array_file) do |f|
|
62
|
+
inputs, _ = task.assign_inputs(:string => "String", :integer => 2, :integer_array => f)
|
63
|
+
input_pos = task.inputs.index{|p| p.first == :integer_array}
|
64
|
+
assert File === inputs[input_pos]
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
def test_digest_inputs
|
71
|
+
task = self.example_task
|
72
|
+
|
73
|
+
TmpFile.with_file("2\n3") do |integer_array_file|
|
74
|
+
assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
|
75
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def test_digest_stream
|
80
|
+
task = self.example_task
|
81
|
+
|
82
|
+
assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
|
83
|
+
|
84
|
+
TmpFile.with_file("1\n2") do |integer_array_file|
|
85
|
+
hash1 = Open.open(integer_array_file) do |f|
|
86
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
|
87
|
+
end
|
88
|
+
hash2 = Open.open(integer_array_file) do |f|
|
89
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
|
90
|
+
end
|
91
|
+
assert_equal hash1, hash2
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
|
97
|
+
def test_digest_file
|
98
|
+
task = self.example_task
|
99
|
+
|
100
|
+
TmpFile.with_file("2\n3") do |integer_array_file|
|
101
|
+
assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
|
102
|
+
task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_save_and_load
|
107
|
+
task = self.example_task
|
108
|
+
|
109
|
+
TmpFile.with_file("2\n3") do |integer_array_file|
|
110
|
+
inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
|
111
|
+
original_digest = task.digest_inputs(inputs)
|
112
|
+
|
113
|
+
TmpFile.with_file do |save_directory|
|
114
|
+
task.save_inputs(save_directory, inputs)
|
115
|
+
new_inputs = task.load_inputs(save_directory)
|
116
|
+
new_digest = task.digest_inputs(new_inputs)
|
117
|
+
assert_equal original_digest, new_digest
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_save_and_load_file
|
123
|
+
task = self.example_task
|
124
|
+
|
125
|
+
TmpFile.with_file("TEST") do |somefile|
|
126
|
+
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
127
|
+
original_digest = task.digest_inputs(inputs)
|
128
|
+
|
129
|
+
TmpFile.with_file do |save_directory|
|
130
|
+
task.save_inputs(save_directory, inputs)
|
131
|
+
Open.rm somefile
|
132
|
+
new_inputs = task.load_inputs(save_directory)
|
133
|
+
new_digest = task.digest_inputs(new_inputs)
|
134
|
+
assert_equal original_digest, new_digest
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_save_and_load_file_with_copy
|
140
|
+
task = self.example_task
|
141
|
+
|
142
|
+
TmpFile.with_file("TEST") do |somefile|
|
143
|
+
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
144
|
+
original_digest = task.digest_inputs(inputs)
|
145
|
+
|
146
|
+
TmpFile.with_file do |save_directory|
|
147
|
+
task.save_inputs(save_directory, inputs)
|
148
|
+
TmpFile.with_file do |copy_directory|
|
149
|
+
Open.cp save_directory, copy_directory
|
150
|
+
Open.rm_rf save_directory
|
151
|
+
new_inputs = task.load_inputs(copy_directory)
|
152
|
+
new_digest = task.digest_inputs(new_inputs)
|
153
|
+
assert_equal original_digest, new_digest
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
|
160
|
+
def test_save_and_load_file_array
|
161
|
+
task = self.example_task
|
162
|
+
|
163
|
+
TmpFile.with_file do |dir|
|
164
|
+
file1 = File.join(dir, 'subdir1/file')
|
165
|
+
file2 = File.join(dir, 'subdir2/file')
|
166
|
+
|
167
|
+
Open.write(file1, "TEST1")
|
168
|
+
Open.write(file2, "TEST2")
|
169
|
+
inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
|
170
|
+
original_digest = task.digest_inputs(inputs)
|
171
|
+
|
172
|
+
TmpFile.with_file do |save_directory|
|
173
|
+
task.save_inputs(save_directory, inputs)
|
174
|
+
Open.rm(file1)
|
175
|
+
Open.rm(file2)
|
176
|
+
new_inputs = task.load_inputs(save_directory)
|
177
|
+
new_digest = task.digest_inputs(new_inputs)
|
178
|
+
assert_equal original_digest, new_digest
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestWorkflowStep < Test::Unit::TestCase
|
5
|
+
|
6
|
+
def test_step
|
7
|
+
TmpFile.with_file do |tmpfile|
|
8
|
+
step = Step.new tmpfile, ["12"] do |s|
|
9
|
+
s.length
|
10
|
+
end
|
11
|
+
step.type = :integer
|
12
|
+
|
13
|
+
assert_equal 2, step.run
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_dependency
|
18
|
+
Log.with_severity 0 do
|
19
|
+
tmpfile = tmpdir.test_step
|
20
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
21
|
+
s.length
|
22
|
+
end
|
23
|
+
|
24
|
+
step2 = Step.new tmpfile.step2 do
|
25
|
+
step1 = dependencies.first
|
26
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
27
|
+
end
|
28
|
+
|
29
|
+
step2.dependencies = [step1]
|
30
|
+
|
31
|
+
|
32
|
+
assert_equal "12 has 2 characters", step2.run
|
33
|
+
assert_equal "12 has 2 characters", step2.run
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestTask < Test::Unit::TestCase
|
5
|
+
def test_basic_task
|
6
|
+
task = Task.setup do |s=""|
|
7
|
+
(self + s).length
|
8
|
+
end
|
9
|
+
|
10
|
+
assert_equal 4, task.exec_on("1234")
|
11
|
+
assert_equal 6, task.exec_on("1234","56")
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_step
|
15
|
+
task = Task.setup do |s=""|
|
16
|
+
s.length
|
17
|
+
end
|
18
|
+
|
19
|
+
s = task.job('test', ['12'])
|
20
|
+
assert_equal 2, s.run
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_override_inputs
|
24
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
25
|
+
input :input1, :string
|
26
|
+
task :step1 => :string do |i| i end
|
27
|
+
|
28
|
+
dep :step1, :input1 => 1
|
29
|
+
input :input2, :string
|
30
|
+
task :step2 => :string do |i| i end
|
31
|
+
end
|
32
|
+
|
33
|
+
job = wf.job(:step2, :input1 => 2)
|
34
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
35
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_override_inputs_block
|
39
|
+
Log.severity = 0
|
40
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
41
|
+
input :input1, :string
|
42
|
+
task :step1 => :string do |i| i end
|
43
|
+
|
44
|
+
dep :step1 do |id,options|
|
45
|
+
{:inputs => {:input1 => 1}}
|
46
|
+
end
|
47
|
+
input :input2, :string
|
48
|
+
task :step2 => :string do |i| i end
|
49
|
+
end
|
50
|
+
|
51
|
+
job = wf.job(:step2, :input1 => 2)
|
52
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
53
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_task_override_dep
|
57
|
+
Log.severity = 0
|
58
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
59
|
+
input :input1, :integer
|
60
|
+
task :step1 => :integer do |i| i end
|
61
|
+
|
62
|
+
dep :step1
|
63
|
+
input :input2, :integer, "Integer", 3
|
64
|
+
task :step2 => :integer do |i| i * step(:step1).load end
|
65
|
+
end
|
66
|
+
|
67
|
+
assert_equal 6, wf.job(:step2, :input1 => 2, :input2 => 3).run
|
68
|
+
|
69
|
+
step1_job = wf.job(:step1, :input1 => 6)
|
70
|
+
assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
|
71
|
+
|
72
|
+
assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
|
73
|
+
|
74
|
+
assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).run
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_input_dep
|
78
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
79
|
+
input :v1, :integer
|
80
|
+
input :v2, :integer
|
81
|
+
task :sum => :integer do |v1,v2|
|
82
|
+
v1 + v2
|
83
|
+
end
|
84
|
+
|
85
|
+
input :input1, :integer
|
86
|
+
task :step1 => :integer do |i| i end
|
87
|
+
|
88
|
+
input :input2, :integer
|
89
|
+
task :step2 => :integer do |i| i end
|
90
|
+
|
91
|
+
dep :step1, :input1 => 2
|
92
|
+
dep :step2, :input2 => 3
|
93
|
+
dep :sum, :v1 => :step1, :v2 => :step2
|
94
|
+
task :my_sum => :integer do
|
95
|
+
dependencies.last.load
|
96
|
+
end
|
97
|
+
|
98
|
+
dep :my_sum
|
99
|
+
task :double => :integer do
|
100
|
+
step(:my_sum).load * 2
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
job = wf.job(:my_sum)
|
105
|
+
assert_equal 5, job.run
|
106
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
107
|
+
|
108
|
+
job = wf.job(:double)
|
109
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
110
|
+
|
111
|
+
step1 = wf.job(:step1, :input1 => 3)
|
112
|
+
job = wf.job(:double, "TaskInputs#step1" => step1)
|
113
|
+
assert_equal 12, job.run
|
114
|
+
assert_not_equal Task::DEFAULT_NAME, job.name
|
115
|
+
|
116
|
+
step1 = wf.job(:step1, :input1 => 4)
|
117
|
+
job = wf.job(:double, "TaskInputs#step1" => step1)
|
118
|
+
assert_equal 14, job.run
|
119
|
+
assert_not_equal Task::DEFAULT_NAME, job.name
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_input_dep_override
|
123
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
124
|
+
input :v1, :integer
|
125
|
+
input :v2, :integer
|
126
|
+
task :sum => :integer do |v1,v2|
|
127
|
+
v1 + v2
|
128
|
+
end
|
129
|
+
|
130
|
+
input :input1, :integer
|
131
|
+
task :step1 => :integer do |i| i end
|
132
|
+
|
133
|
+
input :input2, :integer
|
134
|
+
task :step2 => :integer do |i| i end
|
135
|
+
|
136
|
+
dep :step1
|
137
|
+
dep :step2
|
138
|
+
task :my_sum => :integer do
|
139
|
+
dependencies.inject(0){|acc,d| acc += d.load }
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
step2 = wf.job(:step2, :input2 => 4)
|
144
|
+
job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> step2)
|
145
|
+
assert_equal 6, job.exec
|
146
|
+
|
147
|
+
job = wf.job(:my_sum, :input1 => 2, :input2 => 3)
|
148
|
+
assert_equal 5, job.run
|
149
|
+
|
150
|
+
TmpFile.with_file(4) do |file|
|
151
|
+
job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> file)
|
152
|
+
assert_equal 6, job.run
|
153
|
+
assert_not_equal Task::DEFAULT_NAME, job.name
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def test_input_rename
|
158
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
159
|
+
input :v1, :integer
|
160
|
+
input :v2, :integer
|
161
|
+
task :sum => :integer do |v1,v2|
|
162
|
+
v1 + v2
|
163
|
+
end
|
164
|
+
|
165
|
+
input :vv1, :integer
|
166
|
+
input :vv2, :integer
|
167
|
+
dep :sum, :v1 => :vv1, :v2 => :vv2
|
168
|
+
task :my_sum => :integer do
|
169
|
+
dependencies.last.load
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
Log.severity = 0
|
174
|
+
job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
|
175
|
+
assert_equal 5, job.run
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/workflow'
|
5
|
+
class TestWorkflowUsage < Test::Unit::TestCase
|
6
|
+
module UsageWorkflow
|
7
|
+
extend Workflow
|
8
|
+
|
9
|
+
desc "Desc"
|
10
|
+
input :array, :array, "Array"
|
11
|
+
task :step1 => :string do
|
12
|
+
end
|
13
|
+
|
14
|
+
dep :step1
|
15
|
+
desc "Desc2"
|
16
|
+
input :float, :float, "Float"
|
17
|
+
task :step2 => :string do
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def __test_usage
|
22
|
+
UsageWorkflow.tasks[:step1].doc
|
23
|
+
UsageWorkflow.doc
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/workflow'
|
5
|
+
|
6
|
+
class TestWorkflowUtil < Test::Unit::TestCase
|
7
|
+
def test_annonymous_workflow
|
8
|
+
wf = Workflow.annonymous_workflow "TEST" do
|
9
|
+
task :length => :integer do
|
10
|
+
self.length
|
11
|
+
end
|
12
|
+
end
|
13
|
+
bindings = "12345"
|
14
|
+
assert_equal 5, wf.tasks[:length].exec_on(bindings)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
data/test/test_helper.rb
CHANGED
@@ -22,6 +22,23 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
22
22
|
#require 'scout/helper/misc/development'
|
23
23
|
require 'scout/tmpfile'
|
24
24
|
require 'scout/log'
|
25
|
+
require 'scout/persist'
|
26
|
+
require 'scout/workflow'
|
25
27
|
|
26
28
|
class Test::Unit::TestCase
|
29
|
+
|
30
|
+
def tmpdir
|
31
|
+
@tmpdir = Path.setup('tmp/test_tmpdir').find
|
32
|
+
end
|
33
|
+
|
34
|
+
setup do
|
35
|
+
Persist.cache_dir = tmpdir.var.cache
|
36
|
+
Open.remote_cache_dir = tmpdir.var.cache
|
37
|
+
Workflow.directory = tmpdir.var.jobs
|
38
|
+
Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }
|
39
|
+
end
|
40
|
+
|
41
|
+
teardown do
|
42
|
+
Open.rm_rf tmpdir
|
43
|
+
end
|
27
44
|
end
|
File without changes
|