scout-gear 7.3.0 → 8.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
@@ -0,0 +1,112 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
class TestTSVUnzip < Test::Unit::TestCase
|
6
|
+
def test_unzip
|
7
|
+
content1 =<<-EOF
|
8
|
+
#: :sep=" "
|
9
|
+
#ID ValueA ValueB
|
10
|
+
row1 A1|A11 B1|B11
|
11
|
+
row2 A2|A22 B2|B22
|
12
|
+
EOF
|
13
|
+
|
14
|
+
tsv = TSV.open(content1)
|
15
|
+
unzip = tsv.unzip("ValueA", delete: true)
|
16
|
+
assert_equal "B1", unzip["row1:A1"]["ValueB"]
|
17
|
+
assert_equal "B22", unzip["row2:A22"]["ValueB"]
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_unzip_list
|
21
|
+
content1 =<<-EOF
|
22
|
+
#: :sep=" "#:type=:list
|
23
|
+
#ID ValueA ValueB
|
24
|
+
row1 A1 B1
|
25
|
+
row2 A2 B2
|
26
|
+
EOF
|
27
|
+
|
28
|
+
tsv = TSV.open(content1)
|
29
|
+
unzip = tsv.unzip("ValueA", delete: true)
|
30
|
+
assert_equal "B1", unzip["row1:A1"]["ValueB"]
|
31
|
+
assert_equal "B2", unzip["row2:A2"]["ValueB"]
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_unzip_merge
|
35
|
+
content1 =<<-EOF
|
36
|
+
#: :sep=" "
|
37
|
+
#ID ValueA ValueB
|
38
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
39
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
40
|
+
EOF
|
41
|
+
|
42
|
+
tsv = TSV.open(content1)
|
43
|
+
unzip = tsv.unzip("ValueA", delete: true, merge: true)
|
44
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
45
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
46
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_unzip_merge_stream
|
50
|
+
content1 =<<-EOF
|
51
|
+
#: :sep=" "
|
52
|
+
#ID ValueA ValueB
|
53
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
54
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
55
|
+
EOF
|
56
|
+
|
57
|
+
tsv = TSV.open(content1)
|
58
|
+
unzip = tsv.unzip("ValueA", delete: true, merge: true, target: :stream).tsv
|
59
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
60
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
61
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_unzip_merge_stream_parser
|
65
|
+
content1 =<<-EOF
|
66
|
+
#: :sep=" "
|
67
|
+
#ID ValueA ValueB
|
68
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
69
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
70
|
+
EOF
|
71
|
+
|
72
|
+
TmpFile.with_file(content1) do |filename|
|
73
|
+
unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream).tsv
|
74
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
75
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
76
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_unzip_merge_stream_parser_one2many
|
81
|
+
content1 =<<-EOF
|
82
|
+
#: :sep=" "
|
83
|
+
#ID ValueA ValueB
|
84
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
85
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
86
|
+
EOF
|
87
|
+
|
88
|
+
TmpFile.with_file(content1) do |filename|
|
89
|
+
unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream, one2one: false).tsv
|
90
|
+
assert_equal ["B1", "B11", "B11.1"], unzip["row1:A1"]["ValueB"].uniq
|
91
|
+
assert_equal ["B1", "B11", "B11.1"], unzip["row1:A11"]["ValueB"].uniq
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def test_unzip_target
|
97
|
+
content1 =<<-EOF
|
98
|
+
#: :sep=" "
|
99
|
+
#ID ValueA ValueB
|
100
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
101
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
102
|
+
EOF
|
103
|
+
|
104
|
+
tsv = TSV.open(content1)
|
105
|
+
unzip = TSV.setup({}, :key_field => "Key", :fields => %w(ValueA ValueB))
|
106
|
+
tsv.unzip("ValueA", delete: true, merge: true, target: unzip)
|
107
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
108
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
109
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestStepStatus < Test::Unit::TestCase
|
5
|
+
def test_dependency
|
6
|
+
tmpfile = tmpdir.test_step
|
7
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
8
|
+
s.length
|
9
|
+
end
|
10
|
+
|
11
|
+
step2 = Step.new tmpfile.step2 do
|
12
|
+
step1 = dependencies.first
|
13
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
14
|
+
end
|
15
|
+
|
16
|
+
step2.dependencies = [step1]
|
17
|
+
|
18
|
+
sss 0
|
19
|
+
Misc.with_env "SCOUT_UPDATE", "true" do
|
20
|
+
step2.run
|
21
|
+
assert step2.updated?
|
22
|
+
|
23
|
+
sleep 0.1
|
24
|
+
step1.clean
|
25
|
+
step1.run
|
26
|
+
refute step2.updated?
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
@@ -71,8 +71,8 @@ class TestTaskInput < Test::Unit::TestCase
|
|
71
71
|
task = self.example_task
|
72
72
|
|
73
73
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
74
|
-
assert_equal task.
|
75
|
-
task.
|
74
|
+
assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
|
75
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
76
76
|
end
|
77
77
|
end
|
78
78
|
|
@@ -83,10 +83,10 @@ class TestTaskInput < Test::Unit::TestCase
|
|
83
83
|
|
84
84
|
TmpFile.with_file("1\n2") do |integer_array_file|
|
85
85
|
hash1 = Open.open(integer_array_file) do |f|
|
86
|
-
task.
|
86
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
|
87
87
|
end
|
88
88
|
hash2 = Open.open(integer_array_file) do |f|
|
89
|
-
task.
|
89
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
|
90
90
|
end
|
91
91
|
assert_equal hash1, hash2
|
92
92
|
end
|
@@ -98,8 +98,8 @@ class TestTaskInput < Test::Unit::TestCase
|
|
98
98
|
task = self.example_task
|
99
99
|
|
100
100
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
101
|
-
assert_equal task.
|
102
|
-
task.
|
101
|
+
assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
|
102
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
103
103
|
end
|
104
104
|
end
|
105
105
|
|
@@ -108,12 +108,12 @@ class TestTaskInput < Test::Unit::TestCase
|
|
108
108
|
|
109
109
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
110
110
|
inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
|
111
|
-
original_digest = task.
|
111
|
+
original_digest = task.process_inputs(inputs).last
|
112
112
|
|
113
113
|
TmpFile.with_file do |save_directory|
|
114
114
|
task.save_inputs(save_directory, inputs)
|
115
115
|
new_inputs = task.load_inputs(save_directory)
|
116
|
-
new_digest = task.
|
116
|
+
new_digest = task.process_inputs(new_inputs).last
|
117
117
|
assert_equal original_digest, new_digest
|
118
118
|
end
|
119
119
|
end
|
@@ -124,13 +124,13 @@ class TestTaskInput < Test::Unit::TestCase
|
|
124
124
|
|
125
125
|
TmpFile.with_file("TEST") do |somefile|
|
126
126
|
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
127
|
-
original_digest = task.
|
127
|
+
original_digest = task.process_inputs(inputs).last
|
128
128
|
|
129
129
|
TmpFile.with_file do |save_directory|
|
130
130
|
task.save_inputs(save_directory, inputs)
|
131
131
|
Open.rm somefile
|
132
132
|
new_inputs = task.load_inputs(save_directory)
|
133
|
-
new_digest = task.
|
133
|
+
new_digest = task.process_inputs(new_inputs).last
|
134
134
|
assert_equal original_digest, new_digest
|
135
135
|
end
|
136
136
|
end
|
@@ -141,7 +141,7 @@ class TestTaskInput < Test::Unit::TestCase
|
|
141
141
|
|
142
142
|
TmpFile.with_file("TEST") do |somefile|
|
143
143
|
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
144
|
-
original_digest = task.
|
144
|
+
original_digest = task.process_inputs(inputs).last
|
145
145
|
|
146
146
|
TmpFile.with_file do |save_directory|
|
147
147
|
task.save_inputs(save_directory, inputs)
|
@@ -149,7 +149,7 @@ class TestTaskInput < Test::Unit::TestCase
|
|
149
149
|
Open.cp save_directory, copy_directory
|
150
150
|
Open.rm_rf save_directory
|
151
151
|
new_inputs = task.load_inputs(copy_directory)
|
152
|
-
new_digest = task.
|
152
|
+
new_digest = task.process_inputs(new_inputs).last
|
153
153
|
assert_equal original_digest, new_digest
|
154
154
|
end
|
155
155
|
end
|
@@ -167,14 +167,14 @@ class TestTaskInput < Test::Unit::TestCase
|
|
167
167
|
Open.write(file1, "TEST1")
|
168
168
|
Open.write(file2, "TEST2")
|
169
169
|
inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
|
170
|
-
original_digest = task.
|
170
|
+
original_digest = task.process_inputs(inputs).last
|
171
171
|
|
172
172
|
TmpFile.with_file do |save_directory|
|
173
173
|
task.save_inputs(save_directory, inputs)
|
174
174
|
Open.rm(file1)
|
175
175
|
Open.rm(file2)
|
176
176
|
new_inputs = task.load_inputs(save_directory)
|
177
|
-
new_digest = task.
|
177
|
+
new_digest = task.process_inputs(new_inputs).last
|
178
178
|
assert_equal original_digest, new_digest
|
179
179
|
end
|
180
180
|
end
|
@@ -75,7 +75,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
75
75
|
end
|
76
76
|
assert_equal times/2, lines.length
|
77
77
|
|
78
|
-
stream = step2.run
|
78
|
+
stream = step2.run(true)
|
79
79
|
assert step1.streaming?
|
80
80
|
assert step2.streaming?
|
81
81
|
|
@@ -137,7 +137,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
137
137
|
|
138
138
|
step3.recursive_clean
|
139
139
|
|
140
|
-
stream = step3.run
|
140
|
+
stream = step3.run(true)
|
141
141
|
out = []
|
142
142
|
while l = stream.gets
|
143
143
|
out << l
|
@@ -218,7 +218,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
218
218
|
step4.dependencies = [step2, step3]
|
219
219
|
|
220
220
|
lines = []
|
221
|
-
io = step4.run
|
221
|
+
io = step4.run(true)
|
222
222
|
Log::ProgressBar.with_bar severity: 0 do |b|
|
223
223
|
while line = io.gets
|
224
224
|
b.tick
|
@@ -21,38 +21,6 @@ class TestTask < Test::Unit::TestCase
|
|
21
21
|
assert_equal 2, s.run
|
22
22
|
end
|
23
23
|
|
24
|
-
def test_override_inputs
|
25
|
-
wf = Workflow.annonymous_workflow "TaskInputs" do
|
26
|
-
input :input1, :string
|
27
|
-
task :step1 => :string do |i| i end
|
28
|
-
|
29
|
-
dep :step1, :input1 => 1
|
30
|
-
input :input2, :string
|
31
|
-
task :step2 => :string do |i| i end
|
32
|
-
end
|
33
|
-
|
34
|
-
job = wf.job(:step2, :input1 => 2)
|
35
|
-
assert_equal Task::DEFAULT_NAME, job.name
|
36
|
-
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
37
|
-
end
|
38
|
-
|
39
|
-
def test_override_inputs_block
|
40
|
-
wf = Workflow.annonymous_workflow "TaskInputs" do
|
41
|
-
input :input1, :string
|
42
|
-
task :step1 => :string do |i| i end
|
43
|
-
|
44
|
-
dep :step1 do |id,options|
|
45
|
-
{:inputs => {:input1 => 1}}
|
46
|
-
end
|
47
|
-
input :input2, :string
|
48
|
-
task :step2 => :string do |i| i end
|
49
|
-
end
|
50
|
-
|
51
|
-
job = wf.job(:step2, :input1 => 2)
|
52
|
-
assert_equal Task::DEFAULT_NAME, job.name
|
53
|
-
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
54
|
-
end
|
55
|
-
|
56
24
|
def test_task_override_dep
|
57
25
|
wf = Workflow.annonymous_workflow "TaskInputs" do
|
58
26
|
input :input1, :integer
|
@@ -108,11 +76,13 @@ class TestTask < Test::Unit::TestCase
|
|
108
76
|
assert_equal Task::DEFAULT_NAME, job.name
|
109
77
|
|
110
78
|
step1 = wf.job(:step1, :input1 => 3)
|
79
|
+
assert_equal 3, step1.run
|
111
80
|
job = wf.job(:double, "TaskInputs#step1" => step1)
|
112
81
|
assert_equal 12, job.run
|
113
82
|
assert_not_equal Task::DEFAULT_NAME, job.name
|
114
83
|
|
115
84
|
step1 = wf.job(:step1, :input1 => 4)
|
85
|
+
assert_equal 4, step1.run
|
116
86
|
job = wf.job(:double, "TaskInputs#step1" => step1)
|
117
87
|
assert_equal 14, job.run
|
118
88
|
assert_not_equal Task::DEFAULT_NAME, job.name
|
@@ -172,5 +142,171 @@ class TestTask < Test::Unit::TestCase
|
|
172
142
|
job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
|
173
143
|
assert_equal 5, job.run
|
174
144
|
end
|
145
|
+
|
146
|
+
def test_defaults_in_dep_block
|
147
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
148
|
+
input :v1, :integer
|
149
|
+
input :v2, :integer
|
150
|
+
task :sum => :integer do |v1,v2|
|
151
|
+
v1 + v2
|
152
|
+
end
|
153
|
+
|
154
|
+
input :vv1, :integer
|
155
|
+
input :vv2, :integer, nil, 3
|
156
|
+
dep :sum, :v1 => :placeholder, :v2 => :placeholder do |jobname,options,dependencies|
|
157
|
+
raise "Non-numeric value where integer expected" unless Numeric === options[:vv1]
|
158
|
+
{inputs: {v1: options[:vv1], v2: options[:vv2]} }
|
159
|
+
end
|
160
|
+
task :my_sum => :integer do
|
161
|
+
dependencies.last.load
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
job = wf.job(:my_sum, :vv1 => "2")
|
166
|
+
assert_equal 5, job.run
|
167
|
+
end
|
168
|
+
|
169
|
+
def test_dependency_jobname
|
170
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
171
|
+
input :v1, :integer
|
172
|
+
input :v2, :integer
|
173
|
+
task :sum => :integer do |v1,v2|
|
174
|
+
v1 + v2
|
175
|
+
end
|
176
|
+
|
177
|
+
input :vv1, :integer
|
178
|
+
input :vv2, :integer
|
179
|
+
dep :sum, :v1 => :vv1, :v2 => :vv2, :jobname => "OTHER_NAME"
|
180
|
+
task :my_sum => :integer do
|
181
|
+
dependencies.last.load
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
job = wf.job(:my_sum, "TEST_NAME", :vv1 => 2, :vv2 => 3)
|
186
|
+
assert_equal 5, job.run
|
187
|
+
assert_equal "TEST_NAME", job.clean_name
|
188
|
+
assert_equal "OTHER_NAME", job.step(:sum).clean_name
|
189
|
+
end
|
190
|
+
|
191
|
+
def test_no_param_last_job
|
192
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
193
|
+
input :v1, :integer
|
194
|
+
input :v2, :integer
|
195
|
+
task :sum => :integer do |v1,v2|
|
196
|
+
v1 + v2
|
197
|
+
end
|
198
|
+
|
199
|
+
dep :sum
|
200
|
+
task :my_sum => :integer do
|
201
|
+
dependencies.last.load
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
|
206
|
+
assert_equal 5, job.run
|
207
|
+
refute_equal Task::DEFAULT_NAME, job.name
|
208
|
+
end
|
209
|
+
|
210
|
+
def test_no_param_last_job_block
|
211
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
212
|
+
input :v1, :integer
|
213
|
+
input :v2, :integer
|
214
|
+
task :sum => :integer do |v1,v2|
|
215
|
+
v1 + v2
|
216
|
+
end
|
217
|
+
|
218
|
+
dep :sum do |jobname,options|
|
219
|
+
{inputs: options}
|
220
|
+
end
|
221
|
+
task :my_sum => :integer do
|
222
|
+
dependencies.last.load
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
|
227
|
+
assert_equal 5, job.run
|
228
|
+
refute_equal Task::DEFAULT_NAME, job.name
|
229
|
+
end
|
230
|
+
|
231
|
+
|
232
|
+
def test_override_inputs_block
|
233
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
234
|
+
input :input1, :string
|
235
|
+
task :step1 => :string do |i| i end
|
236
|
+
|
237
|
+
dep :step1, :input1 => 1 do |id,options|
|
238
|
+
{:inputs => options}
|
239
|
+
end
|
240
|
+
input :input2, :string
|
241
|
+
task :step2 => :string do |i| step(:step1).load end
|
242
|
+
end
|
243
|
+
|
244
|
+
job = wf.job(:step2, :input1 => 2)
|
245
|
+
assert_equal 1, job.run
|
246
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
247
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
248
|
+
end
|
249
|
+
|
250
|
+
def test_override_inputs_block_array
|
251
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
252
|
+
input :input1, :string
|
253
|
+
task :step1 => :string do |i| i end
|
254
|
+
|
255
|
+
dep :step1, :input1 => 1 do |id,options|
|
256
|
+
[{:inputs => options}]
|
257
|
+
end
|
258
|
+
input :input2, :string
|
259
|
+
task :step2 => :string do |i| step(:step1).load end
|
260
|
+
end
|
261
|
+
|
262
|
+
job = wf.job(:step2, :input1 => 2)
|
263
|
+
assert_equal 1, job.run
|
264
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
265
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
266
|
+
end
|
267
|
+
|
268
|
+
def test_override_inputs
|
269
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
270
|
+
input :input1, :string
|
271
|
+
task :step1 => :string do |i| i end
|
272
|
+
|
273
|
+
dep :step1, :input1 => 1
|
274
|
+
input :input2, :string
|
275
|
+
task :step2 => :string do |i| step(:step1).load end
|
276
|
+
end
|
277
|
+
|
278
|
+
job = wf.job(:step2, :input1 => 2)
|
279
|
+
assert_equal 1, job.run
|
280
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
281
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
282
|
+
end
|
283
|
+
|
284
|
+
def test_non_default_inputs
|
285
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
286
|
+
input :input1, :integer, "", 1
|
287
|
+
input :input2, :integer, "", 0
|
288
|
+
task :step1 => :string do |i1,i2| i1 + i2 end
|
289
|
+
|
290
|
+
dep :step1, :input2 => 1
|
291
|
+
task :step2 => :string do |i| step(:step1).load end
|
292
|
+
|
293
|
+
dep :step2
|
294
|
+
task :step3 => :string do |i| step(:step1).load end
|
295
|
+
end
|
296
|
+
|
297
|
+
job = wf.job(:step3, :input1 => 1)
|
298
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
299
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
300
|
+
assert_equal 2, job.run
|
301
|
+
|
302
|
+
job = wf.job(:step3, :input1 => 2)
|
303
|
+
assert_equal 3, job.run
|
304
|
+
|
305
|
+
|
306
|
+
job = wf.job(:step3)
|
307
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
308
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
309
|
+
end
|
310
|
+
|
175
311
|
end
|
176
312
|
|
@@ -9,27 +9,54 @@ class TestWorkflowUsage < Test::Unit::TestCase
|
|
9
9
|
self.name = "UsageWorkflow"
|
10
10
|
|
11
11
|
self.title = "Workflow to test documentation"
|
12
|
-
self.description = "Use this workflow to
|
12
|
+
self.description = "Use this workflow to evaluate if the documentation is correctly presented."
|
13
13
|
|
14
14
|
desc "Desc"
|
15
15
|
input :array, :array, "Array"
|
16
|
-
task :step1 => :string do
|
16
|
+
task :step1 => :string do |a|
|
17
|
+
a * ", "
|
17
18
|
end
|
18
19
|
|
19
|
-
dep :step1
|
20
20
|
desc "Desc2"
|
21
|
+
dep :step1
|
22
|
+
input :float, :float, "Float"
|
23
|
+
task :step2 => :string do |f|
|
24
|
+
step(:step1).load + " " + f.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "Desc2_fixed"
|
28
|
+
dep :step1, :array => %w(a b)
|
21
29
|
input :float, :float, "Float"
|
22
|
-
task :
|
30
|
+
task :step2_fixed => :string do
|
31
|
+
end
|
32
|
+
|
33
|
+
desc 'Desc3'
|
34
|
+
task_alias :step3, UsageWorkflow, :step2
|
35
|
+
|
36
|
+
desc "Desc3"
|
37
|
+
dep :step3, :array => %w(a b)
|
38
|
+
task :step3_fixed => :string do
|
23
39
|
end
|
24
40
|
end
|
25
41
|
|
26
42
|
|
27
43
|
def test_workflow_usage
|
28
|
-
assert_match "
|
44
|
+
assert_match "evaluate if the documentation", UsageWorkflow.usage
|
29
45
|
end
|
30
46
|
|
31
47
|
def test_task_usage
|
32
|
-
assert_match
|
48
|
+
assert_match /Desc2/, UsageWorkflow.tasks[:step2].usage(UsageWorkflow)
|
49
|
+
assert_match /--array/, UsageWorkflow.tasks[:step2].usage(UsageWorkflow)
|
50
|
+
assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
|
51
|
+
assert_match /Desc3/, UsageWorkflow.tasks[:step3].usage(UsageWorkflow)
|
52
|
+
assert_match /--array/, UsageWorkflow.tasks[:step3].usage(UsageWorkflow)
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_task_input_fixed
|
56
|
+
assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
|
57
|
+
refute_match /--array/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
|
58
|
+
assert_match /Desc3/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
|
59
|
+
refute_match /--array/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
|
33
60
|
end
|
34
61
|
end
|
35
62
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scout-gear
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 8.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-05-
|
11
|
+
date: 2023-05-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: term-ansicolor
|
@@ -114,6 +114,7 @@ files:
|
|
114
114
|
- lib/scout/log/progress.rb
|
115
115
|
- lib/scout/log/progress/report.rb
|
116
116
|
- lib/scout/log/progress/util.rb
|
117
|
+
- lib/scout/log/trap.rb
|
117
118
|
- lib/scout/meta_extension.rb
|
118
119
|
- lib/scout/misc.rb
|
119
120
|
- lib/scout/misc/digest.rb
|
@@ -126,6 +127,7 @@ files:
|
|
126
127
|
- lib/scout/named_array.rb
|
127
128
|
- lib/scout/open.rb
|
128
129
|
- lib/scout/open/lock.rb
|
130
|
+
- lib/scout/open/lock/lockfile.rb
|
129
131
|
- lib/scout/open/remote.rb
|
130
132
|
- lib/scout/open/stream.rb
|
131
133
|
- lib/scout/open/util.rb
|
@@ -153,6 +155,8 @@ files:
|
|
153
155
|
- lib/scout/simple_opt/setup.rb
|
154
156
|
- lib/scout/tmpfile.rb
|
155
157
|
- lib/scout/tsv.rb
|
158
|
+
- lib/scout/tsv/attach.rb
|
159
|
+
- lib/scout/tsv/change_id.rb
|
156
160
|
- lib/scout/tsv/dumper.rb
|
157
161
|
- lib/scout/tsv/index.rb
|
158
162
|
- lib/scout/tsv/open.rb
|
@@ -163,11 +167,14 @@ files:
|
|
163
167
|
- lib/scout/tsv/persist/fix_width_table.rb
|
164
168
|
- lib/scout/tsv/persist/serialize.rb
|
165
169
|
- lib/scout/tsv/persist/tokyocabinet.rb
|
170
|
+
- lib/scout/tsv/transformer.rb
|
166
171
|
- lib/scout/tsv/traverse.rb
|
167
172
|
- lib/scout/tsv/util.rb
|
168
173
|
- lib/scout/tsv/util/filter.rb
|
169
174
|
- lib/scout/tsv/util/process.rb
|
175
|
+
- lib/scout/tsv/util/reorder.rb
|
170
176
|
- lib/scout/tsv/util/select.rb
|
177
|
+
- lib/scout/tsv/util/unzip.rb
|
171
178
|
- lib/scout/work_queue.rb
|
172
179
|
- lib/scout/work_queue/socket.rb
|
173
180
|
- lib/scout/work_queue/worker.rb
|
@@ -182,6 +189,7 @@ files:
|
|
182
189
|
- lib/scout/workflow/step/load.rb
|
183
190
|
- lib/scout/workflow/step/progress.rb
|
184
191
|
- lib/scout/workflow/step/provenance.rb
|
192
|
+
- lib/scout/workflow/step/status.rb
|
185
193
|
- lib/scout/workflow/task.rb
|
186
194
|
- lib/scout/workflow/task/inputs.rb
|
187
195
|
- lib/scout/workflow/usage.rb
|
@@ -244,22 +252,28 @@ files:
|
|
244
252
|
- test/scout/tsv/persist/test_adapter.rb
|
245
253
|
- test/scout/tsv/persist/test_fix_width_table.rb
|
246
254
|
- test/scout/tsv/persist/test_tokyocabinet.rb
|
255
|
+
- test/scout/tsv/test_attach.rb
|
256
|
+
- test/scout/tsv/test_change_id.rb
|
247
257
|
- test/scout/tsv/test_dumper.rb
|
248
258
|
- test/scout/tsv/test_index.rb
|
249
259
|
- test/scout/tsv/test_open.rb
|
250
260
|
- test/scout/tsv/test_parser.rb
|
251
261
|
- test/scout/tsv/test_persist.rb
|
262
|
+
- test/scout/tsv/test_transformer.rb
|
252
263
|
- test/scout/tsv/test_traverse.rb
|
253
264
|
- test/scout/tsv/test_util.rb
|
254
265
|
- test/scout/tsv/util/test_filter.rb
|
255
266
|
- test/scout/tsv/util/test_process.rb
|
267
|
+
- test/scout/tsv/util/test_reorder.rb
|
256
268
|
- test/scout/tsv/util/test_select.rb
|
269
|
+
- test/scout/tsv/util/test_unzip.rb
|
257
270
|
- test/scout/work_queue/test_socket.rb
|
258
271
|
- test/scout/work_queue/test_worker.rb
|
259
272
|
- test/scout/workflow/step/test_dependencies.rb
|
260
273
|
- test/scout/workflow/step/test_info.rb
|
261
274
|
- test/scout/workflow/step/test_load.rb
|
262
275
|
- test/scout/workflow/step/test_provenance.rb
|
276
|
+
- test/scout/workflow/step/test_status.rb
|
263
277
|
- test/scout/workflow/task/test_inputs.rb
|
264
278
|
- test/scout/workflow/test_definition.rb
|
265
279
|
- test/scout/workflow/test_documentation.rb
|
@@ -273,7 +287,7 @@ homepage: http://github.com/mikisvaz/scout-gear
|
|
273
287
|
licenses:
|
274
288
|
- MIT
|
275
289
|
metadata: {}
|
276
|
-
post_install_message:
|
290
|
+
post_install_message:
|
277
291
|
rdoc_options: []
|
278
292
|
require_paths:
|
279
293
|
- lib
|
@@ -288,8 +302,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
288
302
|
- !ruby/object:Gem::Version
|
289
303
|
version: '0'
|
290
304
|
requirements: []
|
291
|
-
rubygems_version: 3.
|
292
|
-
signing_key:
|
305
|
+
rubygems_version: 3.4.11
|
306
|
+
signing_key:
|
293
307
|
specification_version: 4
|
294
308
|
summary: basic gear for scouts
|
295
309
|
test_files: []
|