scout-gear 7.3.0 → 8.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +20 -9
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +26 -23
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/log/color.rb +4 -1
  10. data/lib/scout/log/progress/report.rb +1 -1
  11. data/lib/scout/log/progress/util.rb +58 -54
  12. data/lib/scout/log/progress.rb +1 -1
  13. data/lib/scout/log/trap.rb +107 -0
  14. data/lib/scout/log.rb +56 -21
  15. data/lib/scout/meta_extension.rb +13 -6
  16. data/lib/scout/misc/digest.rb +1 -1
  17. data/lib/scout/misc/format.rb +12 -0
  18. data/lib/scout/misc/insist.rb +1 -1
  19. data/lib/scout/misc/monitor.rb +11 -0
  20. data/lib/scout/misc/system.rb +10 -0
  21. data/lib/scout/named_array.rb +65 -3
  22. data/lib/scout/open/lock/lockfile.rb +587 -0
  23. data/lib/scout/open/lock.rb +28 -2
  24. data/lib/scout/open/remote.rb +4 -0
  25. data/lib/scout/open/stream.rb +90 -15
  26. data/lib/scout/open/util.rb +13 -3
  27. data/lib/scout/path/find.rb +9 -1
  28. data/lib/scout/path/util.rb +35 -0
  29. data/lib/scout/persist/serialize.rb +18 -5
  30. data/lib/scout/persist.rb +28 -12
  31. data/lib/scout/resource/path.rb +53 -0
  32. data/lib/scout/resource/produce.rb +0 -8
  33. data/lib/scout/resource/util.rb +2 -1
  34. data/lib/scout/tmpfile.rb +7 -8
  35. data/lib/scout/tsv/attach.rb +177 -0
  36. data/lib/scout/tsv/change_id.rb +40 -0
  37. data/lib/scout/tsv/dumper.rb +72 -46
  38. data/lib/scout/tsv/index.rb +69 -13
  39. data/lib/scout/tsv/open.rb +138 -84
  40. data/lib/scout/tsv/parser.rb +135 -80
  41. data/lib/scout/tsv/path.rb +1 -2
  42. data/lib/scout/tsv/persist/adapter.rb +15 -45
  43. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  44. data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
  45. data/lib/scout/tsv/persist.rb +4 -0
  46. data/lib/scout/tsv/transformer.rb +141 -0
  47. data/lib/scout/tsv/traverse.rb +96 -92
  48. data/lib/scout/tsv/util/filter.rb +9 -0
  49. data/lib/scout/tsv/util/reorder.rb +81 -0
  50. data/lib/scout/tsv/util/select.rb +78 -33
  51. data/lib/scout/tsv/util/unzip.rb +86 -0
  52. data/lib/scout/tsv/util.rb +60 -11
  53. data/lib/scout/tsv.rb +26 -3
  54. data/lib/scout/work_queue/socket.rb +6 -1
  55. data/lib/scout/work_queue/worker.rb +5 -2
  56. data/lib/scout/work_queue.rb +15 -8
  57. data/lib/scout/workflow/definition.rb +21 -2
  58. data/lib/scout/workflow/step/dependencies.rb +24 -4
  59. data/lib/scout/workflow/step/info.rb +36 -5
  60. data/lib/scout/workflow/step/provenance.rb +8 -7
  61. data/lib/scout/workflow/step/status.rb +45 -0
  62. data/lib/scout/workflow/step.rb +100 -34
  63. data/lib/scout/workflow/task/inputs.rb +14 -20
  64. data/lib/scout/workflow/task.rb +81 -46
  65. data/lib/scout/workflow/usage.rb +8 -6
  66. data/scout-gear.gemspec +24 -20
  67. data/scout_commands/workflow/task +34 -7
  68. data/test/scout/open/test_stream.rb +60 -58
  69. data/test/scout/path/test_find.rb +10 -1
  70. data/test/scout/resource/test_produce.rb +15 -0
  71. data/test/scout/test_meta_extension.rb +25 -0
  72. data/test/scout/test_named_array.rb +18 -0
  73. data/test/scout/test_persist.rb +6 -0
  74. data/test/scout/test_tsv.rb +212 -2
  75. data/test/scout/test_work_queue.rb +21 -19
  76. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  77. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  78. data/test/scout/tsv/test_attach.rb +227 -0
  79. data/test/scout/tsv/test_change_id.rb +98 -0
  80. data/test/scout/tsv/test_dumper.rb +1 -1
  81. data/test/scout/tsv/test_index.rb +35 -3
  82. data/test/scout/tsv/test_open.rb +160 -2
  83. data/test/scout/tsv/test_parser.rb +19 -2
  84. data/test/scout/tsv/test_persist.rb +2 -0
  85. data/test/scout/tsv/test_transformer.rb +108 -0
  86. data/test/scout/tsv/test_traverse.rb +88 -3
  87. data/test/scout/tsv/test_util.rb +1 -0
  88. data/test/scout/tsv/util/test_reorder.rb +94 -0
  89. data/test/scout/tsv/util/test_select.rb +25 -11
  90. data/test/scout/tsv/util/test_unzip.rb +112 -0
  91. data/test/scout/work_queue/test_socket.rb +0 -1
  92. data/test/scout/workflow/step/test_status.rb +31 -0
  93. data/test/scout/workflow/task/test_inputs.rb +14 -14
  94. data/test/scout/workflow/test_step.rb +3 -3
  95. data/test/scout/workflow/test_task.rb +168 -32
  96. data/test/scout/workflow/test_usage.rb +33 -6
  97. metadata +20 -6
@@ -0,0 +1,112 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+ class TestTSVUnzip < Test::Unit::TestCase
6
+ def test_unzip
7
+ content1 =<<-EOF
8
+ #: :sep=" "
9
+ #ID ValueA ValueB
10
+ row1 A1|A11 B1|B11
11
+ row2 A2|A22 B2|B22
12
+ EOF
13
+
14
+ tsv = TSV.open(content1)
15
+ unzip = tsv.unzip("ValueA", delete: true)
16
+ assert_equal "B1", unzip["row1:A1"]["ValueB"]
17
+ assert_equal "B22", unzip["row2:A22"]["ValueB"]
18
+ end
19
+
20
+ def test_unzip_list
21
+ content1 =<<-EOF
22
+ #: :sep=" "#:type=:list
23
+ #ID ValueA ValueB
24
+ row1 A1 B1
25
+ row2 A2 B2
26
+ EOF
27
+
28
+ tsv = TSV.open(content1)
29
+ unzip = tsv.unzip("ValueA", delete: true)
30
+ assert_equal "B1", unzip["row1:A1"]["ValueB"]
31
+ assert_equal "B2", unzip["row2:A2"]["ValueB"]
32
+ end
33
+
34
+ def test_unzip_merge
35
+ content1 =<<-EOF
36
+ #: :sep=" "
37
+ #ID ValueA ValueB
38
+ row1 A1|A11|A11 B1|B11|B11.1
39
+ row2 A2|A22|A22 B2|B22|B22.2
40
+ EOF
41
+
42
+ tsv = TSV.open(content1)
43
+ unzip = tsv.unzip("ValueA", delete: true, merge: true)
44
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
45
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
46
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
47
+ end
48
+
49
+ def test_unzip_merge_stream
50
+ content1 =<<-EOF
51
+ #: :sep=" "
52
+ #ID ValueA ValueB
53
+ row1 A1|A11|A11 B1|B11|B11.1
54
+ row2 A2|A22|A22 B2|B22|B22.2
55
+ EOF
56
+
57
+ tsv = TSV.open(content1)
58
+ unzip = tsv.unzip("ValueA", delete: true, merge: true, target: :stream).tsv
59
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
60
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
61
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
62
+ end
63
+
64
+ def test_unzip_merge_stream_parser
65
+ content1 =<<-EOF
66
+ #: :sep=" "
67
+ #ID ValueA ValueB
68
+ row1 A1|A11|A11 B1|B11|B11.1
69
+ row2 A2|A22|A22 B2|B22|B22.2
70
+ EOF
71
+
72
+ TmpFile.with_file(content1) do |filename|
73
+ unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream).tsv
74
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
75
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
76
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
77
+ end
78
+ end
79
+
80
+ def test_unzip_merge_stream_parser_one2many
81
+ content1 =<<-EOF
82
+ #: :sep=" "
83
+ #ID ValueA ValueB
84
+ row1 A1|A11|A11 B1|B11|B11.1
85
+ row2 A2|A22|A22 B2|B22|B22.2
86
+ EOF
87
+
88
+ TmpFile.with_file(content1) do |filename|
89
+ unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream, one2one: false).tsv
90
+ assert_equal ["B1", "B11", "B11.1"], unzip["row1:A1"]["ValueB"].uniq
91
+ assert_equal ["B1", "B11", "B11.1"], unzip["row1:A11"]["ValueB"].uniq
92
+ end
93
+ end
94
+
95
+
96
+ def test_unzip_target
97
+ content1 =<<-EOF
98
+ #: :sep=" "
99
+ #ID ValueA ValueB
100
+ row1 A1|A11|A11 B1|B11|B11.1
101
+ row2 A2|A22|A22 B2|B22|B22.2
102
+ EOF
103
+
104
+ tsv = TSV.open(content1)
105
+ unzip = TSV.setup({}, :key_field => "Key", :fields => %w(ValueA ValueB))
106
+ tsv.unzip("ValueA", delete: true, merge: true, target: unzip)
107
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
108
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
109
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
110
+ end
111
+ end
112
+
@@ -22,7 +22,6 @@ class TestSocket < Test::Unit::TestCase
22
22
  end
23
23
 
24
24
  def __test_speed
25
- sss 0
26
25
  socket = WorkQueue::Socket.new
27
26
 
28
27
  num = 10_000
@@ -0,0 +1,31 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestStepStatus < Test::Unit::TestCase
5
+ def test_dependency
6
+ tmpfile = tmpdir.test_step
7
+ step1 = Step.new tmpfile.step1, ["12"] do |s|
8
+ s.length
9
+ end
10
+
11
+ step2 = Step.new tmpfile.step2 do
12
+ step1 = dependencies.first
13
+ step1.inputs.first + " has " + step1.load.to_s + " characters"
14
+ end
15
+
16
+ step2.dependencies = [step1]
17
+
18
+ sss 0
19
+ Misc.with_env "SCOUT_UPDATE", "true" do
20
+ step2.run
21
+ assert step2.updated?
22
+
23
+ sleep 0.1
24
+ step1.clean
25
+ step1.run
26
+ refute step2.updated?
27
+ end
28
+ end
29
+
30
+ end
31
+
@@ -71,8 +71,8 @@ class TestTaskInput < Test::Unit::TestCase
71
71
  task = self.example_task
72
72
 
73
73
  TmpFile.with_file("2\n3") do |integer_array_file|
74
- assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
75
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
74
+ assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
75
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
76
76
  end
77
77
  end
78
78
 
@@ -83,10 +83,10 @@ class TestTaskInput < Test::Unit::TestCase
83
83
 
84
84
  TmpFile.with_file("1\n2") do |integer_array_file|
85
85
  hash1 = Open.open(integer_array_file) do |f|
86
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
86
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
87
87
  end
88
88
  hash2 = Open.open(integer_array_file) do |f|
89
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
89
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
90
90
  end
91
91
  assert_equal hash1, hash2
92
92
  end
@@ -98,8 +98,8 @@ class TestTaskInput < Test::Unit::TestCase
98
98
  task = self.example_task
99
99
 
100
100
  TmpFile.with_file("2\n3") do |integer_array_file|
101
- assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
102
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
101
+ assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
102
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
103
103
  end
104
104
  end
105
105
 
@@ -108,12 +108,12 @@ class TestTaskInput < Test::Unit::TestCase
108
108
 
109
109
  TmpFile.with_file("2\n3") do |integer_array_file|
110
110
  inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
111
- original_digest = task.digest_inputs(inputs)
111
+ original_digest = task.process_inputs(inputs).last
112
112
 
113
113
  TmpFile.with_file do |save_directory|
114
114
  task.save_inputs(save_directory, inputs)
115
115
  new_inputs = task.load_inputs(save_directory)
116
- new_digest = task.digest_inputs(new_inputs)
116
+ new_digest = task.process_inputs(new_inputs).last
117
117
  assert_equal original_digest, new_digest
118
118
  end
119
119
  end
@@ -124,13 +124,13 @@ class TestTaskInput < Test::Unit::TestCase
124
124
 
125
125
  TmpFile.with_file("TEST") do |somefile|
126
126
  inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
127
- original_digest = task.digest_inputs(inputs)
127
+ original_digest = task.process_inputs(inputs).last
128
128
 
129
129
  TmpFile.with_file do |save_directory|
130
130
  task.save_inputs(save_directory, inputs)
131
131
  Open.rm somefile
132
132
  new_inputs = task.load_inputs(save_directory)
133
- new_digest = task.digest_inputs(new_inputs)
133
+ new_digest = task.process_inputs(new_inputs).last
134
134
  assert_equal original_digest, new_digest
135
135
  end
136
136
  end
@@ -141,7 +141,7 @@ class TestTaskInput < Test::Unit::TestCase
141
141
 
142
142
  TmpFile.with_file("TEST") do |somefile|
143
143
  inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
144
- original_digest = task.digest_inputs(inputs)
144
+ original_digest = task.process_inputs(inputs).last
145
145
 
146
146
  TmpFile.with_file do |save_directory|
147
147
  task.save_inputs(save_directory, inputs)
@@ -149,7 +149,7 @@ class TestTaskInput < Test::Unit::TestCase
149
149
  Open.cp save_directory, copy_directory
150
150
  Open.rm_rf save_directory
151
151
  new_inputs = task.load_inputs(copy_directory)
152
- new_digest = task.digest_inputs(new_inputs)
152
+ new_digest = task.process_inputs(new_inputs).last
153
153
  assert_equal original_digest, new_digest
154
154
  end
155
155
  end
@@ -167,14 +167,14 @@ class TestTaskInput < Test::Unit::TestCase
167
167
  Open.write(file1, "TEST1")
168
168
  Open.write(file2, "TEST2")
169
169
  inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
170
- original_digest = task.digest_inputs(inputs)
170
+ original_digest = task.process_inputs(inputs).last
171
171
 
172
172
  TmpFile.with_file do |save_directory|
173
173
  task.save_inputs(save_directory, inputs)
174
174
  Open.rm(file1)
175
175
  Open.rm(file2)
176
176
  new_inputs = task.load_inputs(save_directory)
177
- new_digest = task.digest_inputs(new_inputs)
177
+ new_digest = task.process_inputs(new_inputs).last
178
178
  assert_equal original_digest, new_digest
179
179
  end
180
180
  end
@@ -75,7 +75,7 @@ class TestWorkflowStep < Test::Unit::TestCase
75
75
  end
76
76
  assert_equal times/2, lines.length
77
77
 
78
- stream = step2.run
78
+ stream = step2.run(true)
79
79
  assert step1.streaming?
80
80
  assert step2.streaming?
81
81
 
@@ -137,7 +137,7 @@ class TestWorkflowStep < Test::Unit::TestCase
137
137
 
138
138
  step3.recursive_clean
139
139
 
140
- stream = step3.run
140
+ stream = step3.run(true)
141
141
  out = []
142
142
  while l = stream.gets
143
143
  out << l
@@ -218,7 +218,7 @@ class TestWorkflowStep < Test::Unit::TestCase
218
218
  step4.dependencies = [step2, step3]
219
219
 
220
220
  lines = []
221
- io = step4.run
221
+ io = step4.run(true)
222
222
  Log::ProgressBar.with_bar severity: 0 do |b|
223
223
  while line = io.gets
224
224
  b.tick
@@ -21,38 +21,6 @@ class TestTask < Test::Unit::TestCase
21
21
  assert_equal 2, s.run
22
22
  end
23
23
 
24
- def test_override_inputs
25
- wf = Workflow.annonymous_workflow "TaskInputs" do
26
- input :input1, :string
27
- task :step1 => :string do |i| i end
28
-
29
- dep :step1, :input1 => 1
30
- input :input2, :string
31
- task :step2 => :string do |i| i end
32
- end
33
-
34
- job = wf.job(:step2, :input1 => 2)
35
- assert_equal Task::DEFAULT_NAME, job.name
36
- assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
37
- end
38
-
39
- def test_override_inputs_block
40
- wf = Workflow.annonymous_workflow "TaskInputs" do
41
- input :input1, :string
42
- task :step1 => :string do |i| i end
43
-
44
- dep :step1 do |id,options|
45
- {:inputs => {:input1 => 1}}
46
- end
47
- input :input2, :string
48
- task :step2 => :string do |i| i end
49
- end
50
-
51
- job = wf.job(:step2, :input1 => 2)
52
- assert_equal Task::DEFAULT_NAME, job.name
53
- assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
54
- end
55
-
56
24
  def test_task_override_dep
57
25
  wf = Workflow.annonymous_workflow "TaskInputs" do
58
26
  input :input1, :integer
@@ -108,11 +76,13 @@ class TestTask < Test::Unit::TestCase
108
76
  assert_equal Task::DEFAULT_NAME, job.name
109
77
 
110
78
  step1 = wf.job(:step1, :input1 => 3)
79
+ assert_equal 3, step1.run
111
80
  job = wf.job(:double, "TaskInputs#step1" => step1)
112
81
  assert_equal 12, job.run
113
82
  assert_not_equal Task::DEFAULT_NAME, job.name
114
83
 
115
84
  step1 = wf.job(:step1, :input1 => 4)
85
+ assert_equal 4, step1.run
116
86
  job = wf.job(:double, "TaskInputs#step1" => step1)
117
87
  assert_equal 14, job.run
118
88
  assert_not_equal Task::DEFAULT_NAME, job.name
@@ -172,5 +142,171 @@ class TestTask < Test::Unit::TestCase
172
142
  job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
173
143
  assert_equal 5, job.run
174
144
  end
145
+
146
+ def test_defaults_in_dep_block
147
+ wf = Workflow.annonymous_workflow "TaskInputs" do
148
+ input :v1, :integer
149
+ input :v2, :integer
150
+ task :sum => :integer do |v1,v2|
151
+ v1 + v2
152
+ end
153
+
154
+ input :vv1, :integer
155
+ input :vv2, :integer, nil, 3
156
+ dep :sum, :v1 => :placeholder, :v2 => :placeholder do |jobname,options,dependencies|
157
+ raise "Non-numeric value where integer expected" unless Numeric === options[:vv1]
158
+ {inputs: {v1: options[:vv1], v2: options[:vv2]} }
159
+ end
160
+ task :my_sum => :integer do
161
+ dependencies.last.load
162
+ end
163
+ end
164
+
165
+ job = wf.job(:my_sum, :vv1 => "2")
166
+ assert_equal 5, job.run
167
+ end
168
+
169
+ def test_dependency_jobname
170
+ wf = Workflow.annonymous_workflow "TaskInputs" do
171
+ input :v1, :integer
172
+ input :v2, :integer
173
+ task :sum => :integer do |v1,v2|
174
+ v1 + v2
175
+ end
176
+
177
+ input :vv1, :integer
178
+ input :vv2, :integer
179
+ dep :sum, :v1 => :vv1, :v2 => :vv2, :jobname => "OTHER_NAME"
180
+ task :my_sum => :integer do
181
+ dependencies.last.load
182
+ end
183
+ end
184
+
185
+ job = wf.job(:my_sum, "TEST_NAME", :vv1 => 2, :vv2 => 3)
186
+ assert_equal 5, job.run
187
+ assert_equal "TEST_NAME", job.clean_name
188
+ assert_equal "OTHER_NAME", job.step(:sum).clean_name
189
+ end
190
+
191
+ def test_no_param_last_job
192
+ wf = Workflow.annonymous_workflow "TaskInputs" do
193
+ input :v1, :integer
194
+ input :v2, :integer
195
+ task :sum => :integer do |v1,v2|
196
+ v1 + v2
197
+ end
198
+
199
+ dep :sum
200
+ task :my_sum => :integer do
201
+ dependencies.last.load
202
+ end
203
+ end
204
+
205
+ job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
206
+ assert_equal 5, job.run
207
+ refute_equal Task::DEFAULT_NAME, job.name
208
+ end
209
+
210
+ def test_no_param_last_job_block
211
+ wf = Workflow.annonymous_workflow "TaskInputs" do
212
+ input :v1, :integer
213
+ input :v2, :integer
214
+ task :sum => :integer do |v1,v2|
215
+ v1 + v2
216
+ end
217
+
218
+ dep :sum do |jobname,options|
219
+ {inputs: options}
220
+ end
221
+ task :my_sum => :integer do
222
+ dependencies.last.load
223
+ end
224
+ end
225
+
226
+ job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
227
+ assert_equal 5, job.run
228
+ refute_equal Task::DEFAULT_NAME, job.name
229
+ end
230
+
231
+
232
+ def test_override_inputs_block
233
+ wf = Workflow.annonymous_workflow "TaskInputs" do
234
+ input :input1, :string
235
+ task :step1 => :string do |i| i end
236
+
237
+ dep :step1, :input1 => 1 do |id,options|
238
+ {:inputs => options}
239
+ end
240
+ input :input2, :string
241
+ task :step2 => :string do |i| step(:step1).load end
242
+ end
243
+
244
+ job = wf.job(:step2, :input1 => 2)
245
+ assert_equal 1, job.run
246
+ assert_equal Task::DEFAULT_NAME, job.name
247
+ assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
248
+ end
249
+
250
+ def test_override_inputs_block_array
251
+ wf = Workflow.annonymous_workflow "TaskInputs" do
252
+ input :input1, :string
253
+ task :step1 => :string do |i| i end
254
+
255
+ dep :step1, :input1 => 1 do |id,options|
256
+ [{:inputs => options}]
257
+ end
258
+ input :input2, :string
259
+ task :step2 => :string do |i| step(:step1).load end
260
+ end
261
+
262
+ job = wf.job(:step2, :input1 => 2)
263
+ assert_equal 1, job.run
264
+ assert_equal Task::DEFAULT_NAME, job.name
265
+ assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
266
+ end
267
+
268
+ def test_override_inputs
269
+ wf = Workflow.annonymous_workflow "TaskInputs" do
270
+ input :input1, :string
271
+ task :step1 => :string do |i| i end
272
+
273
+ dep :step1, :input1 => 1
274
+ input :input2, :string
275
+ task :step2 => :string do |i| step(:step1).load end
276
+ end
277
+
278
+ job = wf.job(:step2, :input1 => 2)
279
+ assert_equal 1, job.run
280
+ assert_equal Task::DEFAULT_NAME, job.name
281
+ assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
282
+ end
283
+
284
+ def test_non_default_inputs
285
+ wf = Workflow.annonymous_workflow "TaskInputs" do
286
+ input :input1, :integer, "", 1
287
+ input :input2, :integer, "", 0
288
+ task :step1 => :string do |i1,i2| i1 + i2 end
289
+
290
+ dep :step1, :input2 => 1
291
+ task :step2 => :string do |i| step(:step1).load end
292
+
293
+ dep :step2
294
+ task :step3 => :string do |i| step(:step1).load end
295
+ end
296
+
297
+ job = wf.job(:step3, :input1 => 1)
298
+ assert_equal Task::DEFAULT_NAME, job.name
299
+ assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
300
+ assert_equal 2, job.run
301
+
302
+ job = wf.job(:step3, :input1 => 2)
303
+ assert_equal 3, job.run
304
+
305
+
306
+ job = wf.job(:step3)
307
+ assert_equal Task::DEFAULT_NAME, job.name
308
+ assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
309
+ end
310
+
175
311
  end
176
312
 
@@ -9,27 +9,54 @@ class TestWorkflowUsage < Test::Unit::TestCase
9
9
  self.name = "UsageWorkflow"
10
10
 
11
11
  self.title = "Workflow to test documentation"
12
- self.description = "Use this workflow to test if the documentation is correctly presented."
12
+ self.description = "Use this workflow to evaluate if the documentation is correctly presented."
13
13
 
14
14
  desc "Desc"
15
15
  input :array, :array, "Array"
16
- task :step1 => :string do
16
+ task :step1 => :string do |a|
17
+ a * ", "
17
18
  end
18
19
 
19
- dep :step1
20
20
  desc "Desc2"
21
+ dep :step1
22
+ input :float, :float, "Float"
23
+ task :step2 => :string do |f|
24
+ step(:step1).load + " " + f.to_s
25
+ end
26
+
27
+ desc "Desc2_fixed"
28
+ dep :step1, :array => %w(a b)
21
29
  input :float, :float, "Float"
22
- task :step2 => :string do
30
+ task :step2_fixed => :string do
31
+ end
32
+
33
+ desc 'Desc3'
34
+ task_alias :step3, UsageWorkflow, :step2
35
+
36
+ desc "Desc3"
37
+ dep :step3, :array => %w(a b)
38
+ task :step3_fixed => :string do
23
39
  end
24
40
  end
25
41
 
26
42
 
27
43
  def test_workflow_usage
28
- assert_match "test if the documentation", UsageWorkflow.usage
44
+ assert_match "evaluate if the documentation", UsageWorkflow.usage
29
45
  end
30
46
 
31
47
  def test_task_usage
32
- assert_match "Desc2", UsageWorkflow.tasks[:step2].usage
48
+ assert_match /Desc2/, UsageWorkflow.tasks[:step2].usage(UsageWorkflow)
49
+ assert_match /--array/, UsageWorkflow.tasks[:step2].usage(UsageWorkflow)
50
+ assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
51
+ assert_match /Desc3/, UsageWorkflow.tasks[:step3].usage(UsageWorkflow)
52
+ assert_match /--array/, UsageWorkflow.tasks[:step3].usage(UsageWorkflow)
53
+ end
54
+
55
+ def test_task_input_fixed
56
+ assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
57
+ refute_match /--array/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
58
+ assert_match /Desc3/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
59
+ refute_match /--array/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
33
60
  end
34
61
  end
35
62
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scout-gear
3
3
  version: !ruby/object:Gem::Version
4
- version: 7.3.0
4
+ version: 8.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-05-12 00:00:00.000000000 Z
11
+ date: 2023-05-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: term-ansicolor
@@ -114,6 +114,7 @@ files:
114
114
  - lib/scout/log/progress.rb
115
115
  - lib/scout/log/progress/report.rb
116
116
  - lib/scout/log/progress/util.rb
117
+ - lib/scout/log/trap.rb
117
118
  - lib/scout/meta_extension.rb
118
119
  - lib/scout/misc.rb
119
120
  - lib/scout/misc/digest.rb
@@ -126,6 +127,7 @@ files:
126
127
  - lib/scout/named_array.rb
127
128
  - lib/scout/open.rb
128
129
  - lib/scout/open/lock.rb
130
+ - lib/scout/open/lock/lockfile.rb
129
131
  - lib/scout/open/remote.rb
130
132
  - lib/scout/open/stream.rb
131
133
  - lib/scout/open/util.rb
@@ -153,6 +155,8 @@ files:
153
155
  - lib/scout/simple_opt/setup.rb
154
156
  - lib/scout/tmpfile.rb
155
157
  - lib/scout/tsv.rb
158
+ - lib/scout/tsv/attach.rb
159
+ - lib/scout/tsv/change_id.rb
156
160
  - lib/scout/tsv/dumper.rb
157
161
  - lib/scout/tsv/index.rb
158
162
  - lib/scout/tsv/open.rb
@@ -163,11 +167,14 @@ files:
163
167
  - lib/scout/tsv/persist/fix_width_table.rb
164
168
  - lib/scout/tsv/persist/serialize.rb
165
169
  - lib/scout/tsv/persist/tokyocabinet.rb
170
+ - lib/scout/tsv/transformer.rb
166
171
  - lib/scout/tsv/traverse.rb
167
172
  - lib/scout/tsv/util.rb
168
173
  - lib/scout/tsv/util/filter.rb
169
174
  - lib/scout/tsv/util/process.rb
175
+ - lib/scout/tsv/util/reorder.rb
170
176
  - lib/scout/tsv/util/select.rb
177
+ - lib/scout/tsv/util/unzip.rb
171
178
  - lib/scout/work_queue.rb
172
179
  - lib/scout/work_queue/socket.rb
173
180
  - lib/scout/work_queue/worker.rb
@@ -182,6 +189,7 @@ files:
182
189
  - lib/scout/workflow/step/load.rb
183
190
  - lib/scout/workflow/step/progress.rb
184
191
  - lib/scout/workflow/step/provenance.rb
192
+ - lib/scout/workflow/step/status.rb
185
193
  - lib/scout/workflow/task.rb
186
194
  - lib/scout/workflow/task/inputs.rb
187
195
  - lib/scout/workflow/usage.rb
@@ -244,22 +252,28 @@ files:
244
252
  - test/scout/tsv/persist/test_adapter.rb
245
253
  - test/scout/tsv/persist/test_fix_width_table.rb
246
254
  - test/scout/tsv/persist/test_tokyocabinet.rb
255
+ - test/scout/tsv/test_attach.rb
256
+ - test/scout/tsv/test_change_id.rb
247
257
  - test/scout/tsv/test_dumper.rb
248
258
  - test/scout/tsv/test_index.rb
249
259
  - test/scout/tsv/test_open.rb
250
260
  - test/scout/tsv/test_parser.rb
251
261
  - test/scout/tsv/test_persist.rb
262
+ - test/scout/tsv/test_transformer.rb
252
263
  - test/scout/tsv/test_traverse.rb
253
264
  - test/scout/tsv/test_util.rb
254
265
  - test/scout/tsv/util/test_filter.rb
255
266
  - test/scout/tsv/util/test_process.rb
267
+ - test/scout/tsv/util/test_reorder.rb
256
268
  - test/scout/tsv/util/test_select.rb
269
+ - test/scout/tsv/util/test_unzip.rb
257
270
  - test/scout/work_queue/test_socket.rb
258
271
  - test/scout/work_queue/test_worker.rb
259
272
  - test/scout/workflow/step/test_dependencies.rb
260
273
  - test/scout/workflow/step/test_info.rb
261
274
  - test/scout/workflow/step/test_load.rb
262
275
  - test/scout/workflow/step/test_provenance.rb
276
+ - test/scout/workflow/step/test_status.rb
263
277
  - test/scout/workflow/task/test_inputs.rb
264
278
  - test/scout/workflow/test_definition.rb
265
279
  - test/scout/workflow/test_documentation.rb
@@ -273,7 +287,7 @@ homepage: http://github.com/mikisvaz/scout-gear
273
287
  licenses:
274
288
  - MIT
275
289
  metadata: {}
276
- post_install_message:
290
+ post_install_message:
277
291
  rdoc_options: []
278
292
  require_paths:
279
293
  - lib
@@ -288,8 +302,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
288
302
  - !ruby/object:Gem::Version
289
303
  version: '0'
290
304
  requirements: []
291
- rubygems_version: 3.2.15
292
- signing_key:
305
+ rubygems_version: 3.4.11
306
+ signing_key:
293
307
  specification_version: 4
294
308
  summary: basic gear for scouts
295
309
  test_files: []