scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
@@ -0,0 +1,112 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
class TestTSVUnzip < Test::Unit::TestCase
|
6
|
+
def test_unzip
|
7
|
+
content1 =<<-EOF
|
8
|
+
#: :sep=" "
|
9
|
+
#ID ValueA ValueB
|
10
|
+
row1 A1|A11 B1|B11
|
11
|
+
row2 A2|A22 B2|B22
|
12
|
+
EOF
|
13
|
+
|
14
|
+
tsv = TSV.open(content1)
|
15
|
+
unzip = tsv.unzip("ValueA", delete: true)
|
16
|
+
assert_equal "B1", unzip["row1:A1"]["ValueB"]
|
17
|
+
assert_equal "B22", unzip["row2:A22"]["ValueB"]
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_unzip_list
|
21
|
+
content1 =<<-EOF
|
22
|
+
#: :sep=" "#:type=:list
|
23
|
+
#ID ValueA ValueB
|
24
|
+
row1 A1 B1
|
25
|
+
row2 A2 B2
|
26
|
+
EOF
|
27
|
+
|
28
|
+
tsv = TSV.open(content1)
|
29
|
+
unzip = tsv.unzip("ValueA", delete: true)
|
30
|
+
assert_equal "B1", unzip["row1:A1"]["ValueB"]
|
31
|
+
assert_equal "B2", unzip["row2:A2"]["ValueB"]
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_unzip_merge
|
35
|
+
content1 =<<-EOF
|
36
|
+
#: :sep=" "
|
37
|
+
#ID ValueA ValueB
|
38
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
39
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
40
|
+
EOF
|
41
|
+
|
42
|
+
tsv = TSV.open(content1)
|
43
|
+
unzip = tsv.unzip("ValueA", delete: true, merge: true)
|
44
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
45
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
46
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_unzip_merge_stream
|
50
|
+
content1 =<<-EOF
|
51
|
+
#: :sep=" "
|
52
|
+
#ID ValueA ValueB
|
53
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
54
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
55
|
+
EOF
|
56
|
+
|
57
|
+
tsv = TSV.open(content1)
|
58
|
+
unzip = tsv.unzip("ValueA", delete: true, merge: true, target: :stream).tsv
|
59
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
60
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
61
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_unzip_merge_stream_parser
|
65
|
+
content1 =<<-EOF
|
66
|
+
#: :sep=" "
|
67
|
+
#ID ValueA ValueB
|
68
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
69
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
70
|
+
EOF
|
71
|
+
|
72
|
+
TmpFile.with_file(content1) do |filename|
|
73
|
+
unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream).tsv
|
74
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
75
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
76
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_unzip_merge_stream_parser_one2many
|
81
|
+
content1 =<<-EOF
|
82
|
+
#: :sep=" "
|
83
|
+
#ID ValueA ValueB
|
84
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
85
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
86
|
+
EOF
|
87
|
+
|
88
|
+
TmpFile.with_file(content1) do |filename|
|
89
|
+
unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream, one2one: false).tsv
|
90
|
+
assert_equal ["B1", "B11", "B11.1"], unzip["row1:A1"]["ValueB"].uniq
|
91
|
+
assert_equal ["B1", "B11", "B11.1"], unzip["row1:A11"]["ValueB"].uniq
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def test_unzip_target
|
97
|
+
content1 =<<-EOF
|
98
|
+
#: :sep=" "
|
99
|
+
#ID ValueA ValueB
|
100
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
101
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
102
|
+
EOF
|
103
|
+
|
104
|
+
tsv = TSV.open(content1)
|
105
|
+
unzip = TSV.setup({}, :key_field => "Key", :fields => %w(ValueA ValueB))
|
106
|
+
tsv.unzip("ValueA", delete: true, merge: true, target: unzip)
|
107
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
108
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
109
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestStepStatus < Test::Unit::TestCase
|
5
|
+
def test_dependency
|
6
|
+
tmpfile = tmpdir.test_step
|
7
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
8
|
+
s.length
|
9
|
+
end
|
10
|
+
|
11
|
+
step2 = Step.new tmpfile.step2 do
|
12
|
+
step1 = dependencies.first
|
13
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
14
|
+
end
|
15
|
+
|
16
|
+
step2.dependencies = [step1]
|
17
|
+
|
18
|
+
sss 0
|
19
|
+
Misc.with_env "SCOUT_UPDATE", "true" do
|
20
|
+
step2.run
|
21
|
+
assert step2.updated?
|
22
|
+
|
23
|
+
sleep 0.1
|
24
|
+
step1.clean
|
25
|
+
step1.run
|
26
|
+
refute step2.updated?
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
@@ -71,8 +71,8 @@ class TestTaskInput < Test::Unit::TestCase
|
|
71
71
|
task = self.example_task
|
72
72
|
|
73
73
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
74
|
-
assert_equal task.
|
75
|
-
task.
|
74
|
+
assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
|
75
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
76
76
|
end
|
77
77
|
end
|
78
78
|
|
@@ -83,10 +83,10 @@ class TestTaskInput < Test::Unit::TestCase
|
|
83
83
|
|
84
84
|
TmpFile.with_file("1\n2") do |integer_array_file|
|
85
85
|
hash1 = Open.open(integer_array_file) do |f|
|
86
|
-
task.
|
86
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
|
87
87
|
end
|
88
88
|
hash2 = Open.open(integer_array_file) do |f|
|
89
|
-
task.
|
89
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
|
90
90
|
end
|
91
91
|
assert_equal hash1, hash2
|
92
92
|
end
|
@@ -98,8 +98,8 @@ class TestTaskInput < Test::Unit::TestCase
|
|
98
98
|
task = self.example_task
|
99
99
|
|
100
100
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
101
|
-
assert_equal task.
|
102
|
-
task.
|
101
|
+
assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
|
102
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
103
103
|
end
|
104
104
|
end
|
105
105
|
|
@@ -108,12 +108,12 @@ class TestTaskInput < Test::Unit::TestCase
|
|
108
108
|
|
109
109
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
110
110
|
inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
|
111
|
-
original_digest = task.
|
111
|
+
original_digest = task.process_inputs(inputs).last
|
112
112
|
|
113
113
|
TmpFile.with_file do |save_directory|
|
114
114
|
task.save_inputs(save_directory, inputs)
|
115
115
|
new_inputs = task.load_inputs(save_directory)
|
116
|
-
new_digest = task.
|
116
|
+
new_digest = task.process_inputs(new_inputs).last
|
117
117
|
assert_equal original_digest, new_digest
|
118
118
|
end
|
119
119
|
end
|
@@ -124,13 +124,13 @@ class TestTaskInput < Test::Unit::TestCase
|
|
124
124
|
|
125
125
|
TmpFile.with_file("TEST") do |somefile|
|
126
126
|
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
127
|
-
original_digest = task.
|
127
|
+
original_digest = task.process_inputs(inputs).last
|
128
128
|
|
129
129
|
TmpFile.with_file do |save_directory|
|
130
130
|
task.save_inputs(save_directory, inputs)
|
131
131
|
Open.rm somefile
|
132
132
|
new_inputs = task.load_inputs(save_directory)
|
133
|
-
new_digest = task.
|
133
|
+
new_digest = task.process_inputs(new_inputs).last
|
134
134
|
assert_equal original_digest, new_digest
|
135
135
|
end
|
136
136
|
end
|
@@ -141,7 +141,7 @@ class TestTaskInput < Test::Unit::TestCase
|
|
141
141
|
|
142
142
|
TmpFile.with_file("TEST") do |somefile|
|
143
143
|
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
144
|
-
original_digest = task.
|
144
|
+
original_digest = task.process_inputs(inputs).last
|
145
145
|
|
146
146
|
TmpFile.with_file do |save_directory|
|
147
147
|
task.save_inputs(save_directory, inputs)
|
@@ -149,7 +149,7 @@ class TestTaskInput < Test::Unit::TestCase
|
|
149
149
|
Open.cp save_directory, copy_directory
|
150
150
|
Open.rm_rf save_directory
|
151
151
|
new_inputs = task.load_inputs(copy_directory)
|
152
|
-
new_digest = task.
|
152
|
+
new_digest = task.process_inputs(new_inputs).last
|
153
153
|
assert_equal original_digest, new_digest
|
154
154
|
end
|
155
155
|
end
|
@@ -167,14 +167,14 @@ class TestTaskInput < Test::Unit::TestCase
|
|
167
167
|
Open.write(file1, "TEST1")
|
168
168
|
Open.write(file2, "TEST2")
|
169
169
|
inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
|
170
|
-
original_digest = task.
|
170
|
+
original_digest = task.process_inputs(inputs).last
|
171
171
|
|
172
172
|
TmpFile.with_file do |save_directory|
|
173
173
|
task.save_inputs(save_directory, inputs)
|
174
174
|
Open.rm(file1)
|
175
175
|
Open.rm(file2)
|
176
176
|
new_inputs = task.load_inputs(save_directory)
|
177
|
-
new_digest = task.
|
177
|
+
new_digest = task.process_inputs(new_inputs).last
|
178
178
|
assert_equal original_digest, new_digest
|
179
179
|
end
|
180
180
|
end
|
@@ -75,7 +75,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
75
75
|
end
|
76
76
|
assert_equal times/2, lines.length
|
77
77
|
|
78
|
-
stream = step2.run
|
78
|
+
stream = step2.run(true)
|
79
79
|
assert step1.streaming?
|
80
80
|
assert step2.streaming?
|
81
81
|
|
@@ -137,7 +137,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
137
137
|
|
138
138
|
step3.recursive_clean
|
139
139
|
|
140
|
-
stream = step3.run
|
140
|
+
stream = step3.run(true)
|
141
141
|
out = []
|
142
142
|
while l = stream.gets
|
143
143
|
out << l
|
@@ -218,7 +218,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
218
218
|
step4.dependencies = [step2, step3]
|
219
219
|
|
220
220
|
lines = []
|
221
|
-
io = step4.run
|
221
|
+
io = step4.run(true)
|
222
222
|
Log::ProgressBar.with_bar severity: 0 do |b|
|
223
223
|
while line = io.gets
|
224
224
|
b.tick
|
@@ -21,38 +21,6 @@ class TestTask < Test::Unit::TestCase
|
|
21
21
|
assert_equal 2, s.run
|
22
22
|
end
|
23
23
|
|
24
|
-
def test_override_inputs
|
25
|
-
wf = Workflow.annonymous_workflow "TaskInputs" do
|
26
|
-
input :input1, :string
|
27
|
-
task :step1 => :string do |i| i end
|
28
|
-
|
29
|
-
dep :step1, :input1 => 1
|
30
|
-
input :input2, :string
|
31
|
-
task :step2 => :string do |i| i end
|
32
|
-
end
|
33
|
-
|
34
|
-
job = wf.job(:step2, :input1 => 2)
|
35
|
-
assert_equal Task::DEFAULT_NAME, job.name
|
36
|
-
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
37
|
-
end
|
38
|
-
|
39
|
-
def test_override_inputs_block
|
40
|
-
wf = Workflow.annonymous_workflow "TaskInputs" do
|
41
|
-
input :input1, :string
|
42
|
-
task :step1 => :string do |i| i end
|
43
|
-
|
44
|
-
dep :step1 do |id,options|
|
45
|
-
{:inputs => {:input1 => 1}}
|
46
|
-
end
|
47
|
-
input :input2, :string
|
48
|
-
task :step2 => :string do |i| i end
|
49
|
-
end
|
50
|
-
|
51
|
-
job = wf.job(:step2, :input1 => 2)
|
52
|
-
assert_equal Task::DEFAULT_NAME, job.name
|
53
|
-
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
54
|
-
end
|
55
|
-
|
56
24
|
def test_task_override_dep
|
57
25
|
wf = Workflow.annonymous_workflow "TaskInputs" do
|
58
26
|
input :input1, :integer
|
@@ -108,11 +76,13 @@ class TestTask < Test::Unit::TestCase
|
|
108
76
|
assert_equal Task::DEFAULT_NAME, job.name
|
109
77
|
|
110
78
|
step1 = wf.job(:step1, :input1 => 3)
|
79
|
+
assert_equal 3, step1.run
|
111
80
|
job = wf.job(:double, "TaskInputs#step1" => step1)
|
112
81
|
assert_equal 12, job.run
|
113
82
|
assert_not_equal Task::DEFAULT_NAME, job.name
|
114
83
|
|
115
84
|
step1 = wf.job(:step1, :input1 => 4)
|
85
|
+
assert_equal 4, step1.run
|
116
86
|
job = wf.job(:double, "TaskInputs#step1" => step1)
|
117
87
|
assert_equal 14, job.run
|
118
88
|
assert_not_equal Task::DEFAULT_NAME, job.name
|
@@ -172,5 +142,171 @@ class TestTask < Test::Unit::TestCase
|
|
172
142
|
job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
|
173
143
|
assert_equal 5, job.run
|
174
144
|
end
|
145
|
+
|
146
|
+
def test_defaults_in_dep_block
|
147
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
148
|
+
input :v1, :integer
|
149
|
+
input :v2, :integer
|
150
|
+
task :sum => :integer do |v1,v2|
|
151
|
+
v1 + v2
|
152
|
+
end
|
153
|
+
|
154
|
+
input :vv1, :integer
|
155
|
+
input :vv2, :integer, nil, 3
|
156
|
+
dep :sum, :v1 => :placeholder, :v2 => :placeholder do |jobname,options,dependencies|
|
157
|
+
raise "Non-numeric value where integer expected" unless Numeric === options[:vv1]
|
158
|
+
{inputs: {v1: options[:vv1], v2: options[:vv2]} }
|
159
|
+
end
|
160
|
+
task :my_sum => :integer do
|
161
|
+
dependencies.last.load
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
job = wf.job(:my_sum, :vv1 => "2")
|
166
|
+
assert_equal 5, job.run
|
167
|
+
end
|
168
|
+
|
169
|
+
def test_dependency_jobname
|
170
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
171
|
+
input :v1, :integer
|
172
|
+
input :v2, :integer
|
173
|
+
task :sum => :integer do |v1,v2|
|
174
|
+
v1 + v2
|
175
|
+
end
|
176
|
+
|
177
|
+
input :vv1, :integer
|
178
|
+
input :vv2, :integer
|
179
|
+
dep :sum, :v1 => :vv1, :v2 => :vv2, :jobname => "OTHER_NAME"
|
180
|
+
task :my_sum => :integer do
|
181
|
+
dependencies.last.load
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
job = wf.job(:my_sum, "TEST_NAME", :vv1 => 2, :vv2 => 3)
|
186
|
+
assert_equal 5, job.run
|
187
|
+
assert_equal "TEST_NAME", job.clean_name
|
188
|
+
assert_equal "OTHER_NAME", job.step(:sum).clean_name
|
189
|
+
end
|
190
|
+
|
191
|
+
def test_no_param_last_job
|
192
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
193
|
+
input :v1, :integer
|
194
|
+
input :v2, :integer
|
195
|
+
task :sum => :integer do |v1,v2|
|
196
|
+
v1 + v2
|
197
|
+
end
|
198
|
+
|
199
|
+
dep :sum
|
200
|
+
task :my_sum => :integer do
|
201
|
+
dependencies.last.load
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
|
206
|
+
assert_equal 5, job.run
|
207
|
+
refute_equal Task::DEFAULT_NAME, job.name
|
208
|
+
end
|
209
|
+
|
210
|
+
def test_no_param_last_job_block
|
211
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
212
|
+
input :v1, :integer
|
213
|
+
input :v2, :integer
|
214
|
+
task :sum => :integer do |v1,v2|
|
215
|
+
v1 + v2
|
216
|
+
end
|
217
|
+
|
218
|
+
dep :sum do |jobname,options|
|
219
|
+
{inputs: options}
|
220
|
+
end
|
221
|
+
task :my_sum => :integer do
|
222
|
+
dependencies.last.load
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
|
227
|
+
assert_equal 5, job.run
|
228
|
+
refute_equal Task::DEFAULT_NAME, job.name
|
229
|
+
end
|
230
|
+
|
231
|
+
|
232
|
+
def test_override_inputs_block
|
233
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
234
|
+
input :input1, :string
|
235
|
+
task :step1 => :string do |i| i end
|
236
|
+
|
237
|
+
dep :step1, :input1 => 1 do |id,options|
|
238
|
+
{:inputs => options}
|
239
|
+
end
|
240
|
+
input :input2, :string
|
241
|
+
task :step2 => :string do |i| step(:step1).load end
|
242
|
+
end
|
243
|
+
|
244
|
+
job = wf.job(:step2, :input1 => 2)
|
245
|
+
assert_equal 1, job.run
|
246
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
247
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
248
|
+
end
|
249
|
+
|
250
|
+
def test_override_inputs_block_array
|
251
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
252
|
+
input :input1, :string
|
253
|
+
task :step1 => :string do |i| i end
|
254
|
+
|
255
|
+
dep :step1, :input1 => 1 do |id,options|
|
256
|
+
[{:inputs => options}]
|
257
|
+
end
|
258
|
+
input :input2, :string
|
259
|
+
task :step2 => :string do |i| step(:step1).load end
|
260
|
+
end
|
261
|
+
|
262
|
+
job = wf.job(:step2, :input1 => 2)
|
263
|
+
assert_equal 1, job.run
|
264
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
265
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
266
|
+
end
|
267
|
+
|
268
|
+
def test_override_inputs
|
269
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
270
|
+
input :input1, :string
|
271
|
+
task :step1 => :string do |i| i end
|
272
|
+
|
273
|
+
dep :step1, :input1 => 1
|
274
|
+
input :input2, :string
|
275
|
+
task :step2 => :string do |i| step(:step1).load end
|
276
|
+
end
|
277
|
+
|
278
|
+
job = wf.job(:step2, :input1 => 2)
|
279
|
+
assert_equal 1, job.run
|
280
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
281
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
282
|
+
end
|
283
|
+
|
284
|
+
def test_non_default_inputs
|
285
|
+
wf = Workflow.annonymous_workflow "TaskInputs" do
|
286
|
+
input :input1, :integer, "", 1
|
287
|
+
input :input2, :integer, "", 0
|
288
|
+
task :step1 => :string do |i1,i2| i1 + i2 end
|
289
|
+
|
290
|
+
dep :step1, :input2 => 1
|
291
|
+
task :step2 => :string do |i| step(:step1).load end
|
292
|
+
|
293
|
+
dep :step2
|
294
|
+
task :step3 => :string do |i| step(:step1).load end
|
295
|
+
end
|
296
|
+
|
297
|
+
job = wf.job(:step3, :input1 => 1)
|
298
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
299
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
300
|
+
assert_equal 2, job.run
|
301
|
+
|
302
|
+
job = wf.job(:step3, :input1 => 2)
|
303
|
+
assert_equal 3, job.run
|
304
|
+
|
305
|
+
|
306
|
+
job = wf.job(:step3)
|
307
|
+
assert_equal Task::DEFAULT_NAME, job.name
|
308
|
+
assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
|
309
|
+
end
|
310
|
+
|
175
311
|
end
|
176
312
|
|
@@ -9,27 +9,54 @@ class TestWorkflowUsage < Test::Unit::TestCase
|
|
9
9
|
self.name = "UsageWorkflow"
|
10
10
|
|
11
11
|
self.title = "Workflow to test documentation"
|
12
|
-
self.description = "Use this workflow to
|
12
|
+
self.description = "Use this workflow to evaluate if the documentation is correctly presented."
|
13
13
|
|
14
14
|
desc "Desc"
|
15
15
|
input :array, :array, "Array"
|
16
|
-
task :step1 => :string do
|
16
|
+
task :step1 => :string do |a|
|
17
|
+
a * ", "
|
17
18
|
end
|
18
19
|
|
19
|
-
dep :step1
|
20
20
|
desc "Desc2"
|
21
|
+
dep :step1
|
22
|
+
input :float, :float, "Float"
|
23
|
+
task :step2 => :string do |f|
|
24
|
+
step(:step1).load + " " + f.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "Desc2_fixed"
|
28
|
+
dep :step1, :array => %w(a b)
|
21
29
|
input :float, :float, "Float"
|
22
|
-
task :
|
30
|
+
task :step2_fixed => :string do
|
31
|
+
end
|
32
|
+
|
33
|
+
desc 'Desc3'
|
34
|
+
task_alias :step3, UsageWorkflow, :step2
|
35
|
+
|
36
|
+
desc "Desc3"
|
37
|
+
dep :step3, :array => %w(a b)
|
38
|
+
task :step3_fixed => :string do
|
23
39
|
end
|
24
40
|
end
|
25
41
|
|
26
42
|
|
27
43
|
def test_workflow_usage
|
28
|
-
assert_match "
|
44
|
+
assert_match "evaluate if the documentation", UsageWorkflow.usage
|
29
45
|
end
|
30
46
|
|
31
47
|
def test_task_usage
|
32
|
-
assert_match
|
48
|
+
assert_match /Desc2/, UsageWorkflow.tasks[:step2].usage(UsageWorkflow)
|
49
|
+
assert_match /--array/, UsageWorkflow.tasks[:step2].usage(UsageWorkflow)
|
50
|
+
assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
|
51
|
+
assert_match /Desc3/, UsageWorkflow.tasks[:step3].usage(UsageWorkflow)
|
52
|
+
assert_match /--array/, UsageWorkflow.tasks[:step3].usage(UsageWorkflow)
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_task_input_fixed
|
56
|
+
assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
|
57
|
+
refute_match /--array/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
|
58
|
+
assert_match /Desc3/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
|
59
|
+
refute_match /--array/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
|
33
60
|
end
|
34
61
|
end
|
35
62
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scout-gear
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 8.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-05-
|
11
|
+
date: 2023-05-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: term-ansicolor
|
@@ -114,6 +114,7 @@ files:
|
|
114
114
|
- lib/scout/log/progress.rb
|
115
115
|
- lib/scout/log/progress/report.rb
|
116
116
|
- lib/scout/log/progress/util.rb
|
117
|
+
- lib/scout/log/trap.rb
|
117
118
|
- lib/scout/meta_extension.rb
|
118
119
|
- lib/scout/misc.rb
|
119
120
|
- lib/scout/misc/digest.rb
|
@@ -126,6 +127,7 @@ files:
|
|
126
127
|
- lib/scout/named_array.rb
|
127
128
|
- lib/scout/open.rb
|
128
129
|
- lib/scout/open/lock.rb
|
130
|
+
- lib/scout/open/lock/lockfile.rb
|
129
131
|
- lib/scout/open/remote.rb
|
130
132
|
- lib/scout/open/stream.rb
|
131
133
|
- lib/scout/open/util.rb
|
@@ -153,6 +155,8 @@ files:
|
|
153
155
|
- lib/scout/simple_opt/setup.rb
|
154
156
|
- lib/scout/tmpfile.rb
|
155
157
|
- lib/scout/tsv.rb
|
158
|
+
- lib/scout/tsv/attach.rb
|
159
|
+
- lib/scout/tsv/change_id.rb
|
156
160
|
- lib/scout/tsv/dumper.rb
|
157
161
|
- lib/scout/tsv/index.rb
|
158
162
|
- lib/scout/tsv/open.rb
|
@@ -163,11 +167,14 @@ files:
|
|
163
167
|
- lib/scout/tsv/persist/fix_width_table.rb
|
164
168
|
- lib/scout/tsv/persist/serialize.rb
|
165
169
|
- lib/scout/tsv/persist/tokyocabinet.rb
|
170
|
+
- lib/scout/tsv/transformer.rb
|
166
171
|
- lib/scout/tsv/traverse.rb
|
167
172
|
- lib/scout/tsv/util.rb
|
168
173
|
- lib/scout/tsv/util/filter.rb
|
169
174
|
- lib/scout/tsv/util/process.rb
|
175
|
+
- lib/scout/tsv/util/reorder.rb
|
170
176
|
- lib/scout/tsv/util/select.rb
|
177
|
+
- lib/scout/tsv/util/unzip.rb
|
171
178
|
- lib/scout/work_queue.rb
|
172
179
|
- lib/scout/work_queue/socket.rb
|
173
180
|
- lib/scout/work_queue/worker.rb
|
@@ -182,6 +189,7 @@ files:
|
|
182
189
|
- lib/scout/workflow/step/load.rb
|
183
190
|
- lib/scout/workflow/step/progress.rb
|
184
191
|
- lib/scout/workflow/step/provenance.rb
|
192
|
+
- lib/scout/workflow/step/status.rb
|
185
193
|
- lib/scout/workflow/task.rb
|
186
194
|
- lib/scout/workflow/task/inputs.rb
|
187
195
|
- lib/scout/workflow/usage.rb
|
@@ -244,22 +252,28 @@ files:
|
|
244
252
|
- test/scout/tsv/persist/test_adapter.rb
|
245
253
|
- test/scout/tsv/persist/test_fix_width_table.rb
|
246
254
|
- test/scout/tsv/persist/test_tokyocabinet.rb
|
255
|
+
- test/scout/tsv/test_attach.rb
|
256
|
+
- test/scout/tsv/test_change_id.rb
|
247
257
|
- test/scout/tsv/test_dumper.rb
|
248
258
|
- test/scout/tsv/test_index.rb
|
249
259
|
- test/scout/tsv/test_open.rb
|
250
260
|
- test/scout/tsv/test_parser.rb
|
251
261
|
- test/scout/tsv/test_persist.rb
|
262
|
+
- test/scout/tsv/test_transformer.rb
|
252
263
|
- test/scout/tsv/test_traverse.rb
|
253
264
|
- test/scout/tsv/test_util.rb
|
254
265
|
- test/scout/tsv/util/test_filter.rb
|
255
266
|
- test/scout/tsv/util/test_process.rb
|
267
|
+
- test/scout/tsv/util/test_reorder.rb
|
256
268
|
- test/scout/tsv/util/test_select.rb
|
269
|
+
- test/scout/tsv/util/test_unzip.rb
|
257
270
|
- test/scout/work_queue/test_socket.rb
|
258
271
|
- test/scout/work_queue/test_worker.rb
|
259
272
|
- test/scout/workflow/step/test_dependencies.rb
|
260
273
|
- test/scout/workflow/step/test_info.rb
|
261
274
|
- test/scout/workflow/step/test_load.rb
|
262
275
|
- test/scout/workflow/step/test_provenance.rb
|
276
|
+
- test/scout/workflow/step/test_status.rb
|
263
277
|
- test/scout/workflow/task/test_inputs.rb
|
264
278
|
- test/scout/workflow/test_definition.rb
|
265
279
|
- test/scout/workflow/test_documentation.rb
|
@@ -273,7 +287,7 @@ homepage: http://github.com/mikisvaz/scout-gear
|
|
273
287
|
licenses:
|
274
288
|
- MIT
|
275
289
|
metadata: {}
|
276
|
-
post_install_message:
|
290
|
+
post_install_message:
|
277
291
|
rdoc_options: []
|
278
292
|
require_paths:
|
279
293
|
- lib
|
@@ -288,8 +302,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
288
302
|
- !ruby/object:Gem::Version
|
289
303
|
version: '0'
|
290
304
|
requirements: []
|
291
|
-
rubygems_version: 3.
|
292
|
-
signing_key:
|
305
|
+
rubygems_version: 3.4.11
|
306
|
+
signing_key:
|
293
307
|
specification_version: 4
|
294
308
|
summary: basic gear for scouts
|
295
309
|
test_files: []
|