scout-gear 10.4.0 → 10.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -656
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +40 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +83 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +95 -19
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +3 -0
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/entity.rb +5 -0
  40. data/lib/scout/tsv/index.rb +88 -36
  41. data/lib/scout/tsv/open.rb +21 -8
  42. data/lib/scout/tsv/parser.rb +153 -90
  43. data/lib/scout/tsv/path.rb +7 -2
  44. data/lib/scout/tsv/stream.rb +48 -6
  45. data/lib/scout/tsv/transformer.rb +4 -3
  46. data/lib/scout/tsv/traverse.rb +26 -18
  47. data/lib/scout/tsv/util/process.rb +7 -0
  48. data/lib/scout/tsv/util/reorder.rb +25 -15
  49. data/lib/scout/tsv/util/select.rb +9 -1
  50. data/lib/scout/tsv/util/sort.rb +90 -2
  51. data/lib/scout/tsv/util/unzip.rb +56 -0
  52. data/lib/scout/tsv/util.rb +52 -5
  53. data/lib/scout/tsv.rb +45 -27
  54. data/lib/scout/work_queue/socket.rb +8 -0
  55. data/lib/scout/work_queue/worker.rb +22 -5
  56. data/lib/scout/work_queue.rb +38 -24
  57. data/lib/scout/workflow/definition.rb +11 -10
  58. data/lib/scout/workflow/deployment/orchestrator.rb +20 -3
  59. data/lib/scout/workflow/deployment/trace.rb +205 -0
  60. data/lib/scout/workflow/deployment.rb +1 -0
  61. data/lib/scout/workflow/documentation.rb +1 -1
  62. data/lib/scout/workflow/step/archive.rb +42 -0
  63. data/lib/scout/workflow/step/children.rb +51 -0
  64. data/lib/scout/workflow/step/config.rb +1 -1
  65. data/lib/scout/workflow/step/dependencies.rb +24 -7
  66. data/lib/scout/workflow/step/file.rb +19 -0
  67. data/lib/scout/workflow/step/info.rb +37 -9
  68. data/lib/scout/workflow/step/progress.rb +11 -2
  69. data/lib/scout/workflow/step/status.rb +8 -1
  70. data/lib/scout/workflow/step.rb +80 -25
  71. data/lib/scout/workflow/task/dependencies.rb +4 -1
  72. data/lib/scout/workflow/task/inputs.rb +91 -41
  73. data/lib/scout/workflow/task.rb +54 -57
  74. data/lib/scout/workflow/usage.rb +1 -1
  75. data/lib/scout/workflow/util.rb +4 -0
  76. data/lib/scout/workflow.rb +110 -13
  77. data/lib/scout-gear.rb +2 -0
  78. data/lib/scout.rb +0 -1
  79. data/scout-gear.gemspec +80 -23
  80. data/scout_commands/rbbt +2 -0
  81. data/test/data/person/brothers +4 -0
  82. data/test/data/person/identifiers +10 -0
  83. data/test/data/person/marriages +3 -0
  84. data/test/data/person/parents +6 -0
  85. data/test/scout/association/test_fields.rb +105 -0
  86. data/test/scout/association/test_index.rb +70 -0
  87. data/test/scout/association/test_item.rb +21 -0
  88. data/test/scout/entity/test_format.rb +19 -0
  89. data/test/scout/entity/test_identifiers.rb +58 -0
  90. data/test/scout/entity/test_object.rb +0 -0
  91. data/test/scout/entity/test_property.rb +345 -0
  92. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  93. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  94. data/test/scout/persist/engine/test_sharder.rb +31 -0
  95. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  96. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  97. data/test/scout/persist/test_tsv.rb +146 -0
  98. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  99. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  100. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  101. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  102. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  103. data/test/scout/{tsv/persist → persist/tsv/adapter}/test_tkrzw.rb +3 -6
  104. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  105. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  106. data/test/scout/test_association.rb +51 -0
  107. data/test/scout/test_entity.rb +40 -0
  108. data/test/scout/test_tsv.rb +33 -4
  109. data/test/scout/test_work_queue.rb +3 -2
  110. data/test/scout/test_workflow.rb +16 -15
  111. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  112. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  113. data/test/scout/tsv/test_annotation.rb +52 -0
  114. data/test/scout/tsv/test_attach.rb +226 -1
  115. data/test/scout/tsv/test_change_id.rb +25 -0
  116. data/test/scout/tsv/test_csv.rb +50 -0
  117. data/test/scout/tsv/test_dumper.rb +38 -0
  118. data/test/scout/tsv/test_entity.rb +0 -0
  119. data/test/scout/tsv/test_index.rb +82 -0
  120. data/test/scout/tsv/test_open.rb +44 -0
  121. data/test/scout/tsv/test_parser.rb +70 -0
  122. data/test/scout/tsv/test_stream.rb +22 -0
  123. data/test/scout/tsv/test_transformer.rb +27 -3
  124. data/test/scout/tsv/test_traverse.rb +78 -0
  125. data/test/scout/tsv/util/test_process.rb +16 -0
  126. data/test/scout/tsv/util/test_reorder.rb +67 -0
  127. data/test/scout/tsv/util/test_sort.rb +28 -1
  128. data/test/scout/tsv/util/test_unzip.rb +32 -0
  129. data/test/scout/work_queue/test_socket.rb +4 -1
  130. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  131. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  132. data/test/scout/workflow/step/test_archive.rb +28 -0
  133. data/test/scout/workflow/step/test_children.rb +25 -0
  134. data/test/scout/workflow/step/test_info.rb +16 -0
  135. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  136. data/test/scout/workflow/task/test_inputs.rb +45 -1
  137. data/test/scout/workflow/test_definition.rb +52 -0
  138. data/test/scout/workflow/test_step.rb +57 -0
  139. data/test/scout/workflow/test_task.rb +26 -1
  140. data/test/scout/workflow/test_usage.rb +4 -4
  141. data/test/test_helper.rb +23 -1
  142. metadata +71 -14
  143. data/lib/scout/tsv/persist.rb +0 -27
  144. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  145. data/test/scout/tsv/test_persist.rb +0 -45
@@ -197,5 +197,83 @@ row2 A2|A22 B2|B22
197
197
  assert_equal %w(row2), tsv.keys
198
198
  end
199
199
  end
200
+
201
+ def test_traverse_named
202
+ content =<<-EOF
203
+ #Id ValueA ValueB OtherID
204
+ row1 a|aa|aaa b Id1|Id2
205
+ row2 A B Id3
206
+ row3 a C Id4
207
+ EOF
208
+
209
+ TmpFile.with_file(content) do |filename|
210
+ tsv = TSV.open(File.open(filename), :sep => /\s+/)
211
+
212
+ new_key, new_fields = tsv.through :key, ["ValueB"] do |key, values|
213
+ assert_include %w(b B C), values["ValueB"].first
214
+ end
215
+ end
216
+ end
217
+
218
+ def test_traverse_change_key
219
+ content =<<-EOF
220
+ #Id ValueA ValueB OtherID
221
+ row1 a|aa|aaa b Id1|Id2
222
+ row2 A B Id3
223
+ row3 a C Id4
224
+ EOF
225
+
226
+ TmpFile.with_file(content) do |filename|
227
+ tsv = TSV.open(File.open(filename), :sep => /\s+/)
228
+
229
+ new_key, new_fields = tsv.through "ValueA" do |key, values|
230
+ assert_include tsv.keys, values["Id"].first
231
+ end
232
+
233
+ assert_equal "ValueA", new_key
234
+ end
235
+ end
236
+
237
+ def test_traverse_flat
238
+ content =<<-EOF
239
+ #: :type=:flat
240
+ #Row vA
241
+ row1 a b Id1
242
+ row2 A B Id3
243
+ row3 a C Id4
244
+ EOF
245
+
246
+ TmpFile.with_file(content) do |filename|
247
+ tsv = TSV.open(filename, :sep => /\s+/, :type => :flat)
248
+ keys = []
249
+ tsv.through "vA" do |k,v|
250
+ keys << k
251
+ end
252
+ assert_include keys, "B"
253
+
254
+ tsv.through :key do |k,v|
255
+ assert_equal 3, v.length
256
+ end
257
+
258
+ end
259
+ end
260
+
261
+ def test_traverse_flat_same
262
+ content =<<-EOF
263
+ #Id ValueA
264
+ row1 a aa aaa
265
+ row2 A
266
+ row3 a
267
+ EOF
268
+
269
+ TmpFile.with_file(content) do |filename|
270
+ tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
271
+ data = {}
272
+ k, f = tsv.traverse "Id", ["ValueA"] do |k,v|
273
+ data[k] = v
274
+ end
275
+ assert_equal %w(a aa aaa), data["row1"]
276
+ end
277
+ end
200
278
  end
201
279
 
@@ -63,5 +63,21 @@ row2 AA BB Id33
63
63
 
64
64
  assert_equal %w(), tsv["row1"]["ValueC"]
65
65
  end
66
+
67
+ def test_remove_duplicates
68
+ content =<<-EOF
69
+ #Id ValueA ValueB OtherID
70
+ row1 a|A|a|a b|B|b| Id1|Id2|Id1|Id1
71
+ row2 aa|aa|AA|AA b1|b2|B1|B2 Id1|Id1|Id2|Id2
72
+ EOF
73
+
74
+ TmpFile.with_file(content) do |filename|
75
+ tsv = TSV.open(filename, :sep => /\s+/)
76
+ assert_equal %w(a A a), tsv.remove_duplicates["row1"]["ValueA"]
77
+ assert tsv.remove_duplicates["row1"]["ValueB"].include?("")
78
+ end
79
+
80
+ end
81
+
66
82
  end
67
83
 
@@ -58,6 +58,23 @@ row2 A1
58
58
  assert_equal "row2", r["A1"]
59
59
  end
60
60
 
61
+ def test_reorder_single_to_double
62
+ content =<<-'EOF'
63
+ #: :sep=/\s+/#:type=:single
64
+ #Id ValueA
65
+ row1 a1
66
+ row2 a1
67
+ EOF
68
+
69
+ tsv = TmpFile.with_file(content) do |filename|
70
+ TSV.open(filename)
71
+ end
72
+
73
+ r = tsv.reorder "ValueA", type: :flat
74
+
75
+ assert_equal ["row1", "row2"], r["a1"]
76
+ end
77
+
61
78
  def test_transpose
62
79
  content =<<-EOF
63
80
  #: :type=:list
@@ -90,5 +107,55 @@ rowa a|aa b|BB C|CC
90
107
  end
91
108
  end
92
109
 
110
+ def test_slice_empty
111
+ content =<<-EOF
112
+ #ID ValueA ValueB Comment
113
+ row1 a b c
114
+ row2 A B C
115
+ EOF
116
+
117
+ TmpFile.with_file(content) do |filename|
118
+ tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
119
+ tsv = tsv.slice []
120
+ assert tsv.fields.empty?
121
+ TmpFile.with_file do |tmpfile|
122
+ Open.write(tmpfile, tsv.to_s)
123
+ tsv = TSV.open tmpfile
124
+ assert tsv.fields.empty?
125
+ end
126
+ end
127
+ end
128
+
129
+ def test_reorder_flat
130
+ content =<<-EOF
131
+ #Id ValueA
132
+ row1 a aa aaa
133
+ row2 A
134
+ row3 a
135
+ EOF
136
+
137
+ TmpFile.with_file(content) do |filename|
138
+ tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
139
+
140
+ assert_equal ["row1", "row3"].sort, tsv.reorder("ValueA")["a"]
141
+ end
142
+ end
143
+
144
+ def test_reorder_flat_same
145
+ content =<<-EOF
146
+ #Id ValueA
147
+ row1 a aa aaa
148
+ row2 A
149
+ row3 a
150
+ EOF
151
+
152
+ TmpFile.with_file(content) do |filename|
153
+ tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
154
+
155
+ reordered = tsv.reorder("Id", ["ValueA"])
156
+ assert_equal %w(ValueA), reordered.fields
157
+ assert_equal ["a", "aa", "aaa"].sort, reordered["row1"]
158
+ end
159
+ end
93
160
  end
94
161
 
@@ -5,7 +5,7 @@ require 'scout/tsv'
5
5
 
6
6
  class TestClass < Test::Unit::TestCase
7
7
 
8
- def test_sort_by
8
+ def test_sort_by_empty
9
9
  content =<<-EOF
10
10
  #ID ValueA ValueB Comment
11
11
  row1 a B c
@@ -19,5 +19,32 @@ row2 A b C
19
19
  end
20
20
  end
21
21
 
22
+ def test_sort_by
23
+ content =<<-EOF
24
+ #ID ValueA ValueB Comment
25
+ row1 a B c
26
+ row2 A b C
27
+ EOF
28
+
29
+ TmpFile.with_file(content) do |filename|
30
+ tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
31
+ assert_equal %w(row2 row1), tsv.sort_by("ValueA"){|k,v| v }.collect{|k,v| k}
32
+ assert_equal %w(row1 row2), tsv.sort_by("ValueB"){|k,v| v }.collect{|k,v| k}
33
+ end
34
+ end
35
+
36
+ def test_sort
37
+ content =<<-EOF
38
+ #ID ValueA ValueB Comment
39
+ row1 a B c
40
+ row2 A b C
41
+ EOF
42
+
43
+ TmpFile.with_file(content) do |filename|
44
+ tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
45
+ assert_equal %w(row2 row1), tsv.sort("ValueA"){|a,b| a[1] <=> b[1] }.collect{|k,v| k}
46
+ assert_equal %w(row1 row2), tsv.sort("ValueB"){|a,b| a[1] <=> b[1] }.collect{|k,v| k}
47
+ end
48
+ end
22
49
  end
23
50
 
@@ -108,5 +108,37 @@ row2 A2|A22|A22 B2|B22|B22.2
108
108
  assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
109
109
  assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
110
110
  end
111
+
112
+ def test_unzip_replicates
113
+ content =<<-EOF
114
+ #Id ValueA ValueB OtherID
115
+ row1 a|aa|aaa b|bb|bbb Id1|Id2|Id3
116
+ row2 A B Id3
117
+ EOF
118
+
119
+ TmpFile.with_file(content) do |filename|
120
+ tsv = TSV.open(filename, :sep => /\s+/)
121
+
122
+ assert_equal 4, tsv.unzip_replicates.length
123
+ assert_equal %w(aa bb Id2), tsv.unzip_replicates["row1(1)"]
124
+ end
125
+ end
126
+
127
+
128
+ def test_unzip_zip
129
+ content =<<-EOF
130
+ #Id ValueA ValueB OtherID
131
+ row1 a|A|a|a b|B|b| Id1|Id2|Id1|Id1
132
+ row2 aa|aa|AA|AA b1|b2|B1|B2 Id1|Id1|Id2|Id2
133
+ EOF
134
+
135
+ TmpFile.with_file(content) do |filename|
136
+ tsv = TSV.open(filename, :sep => /\s+/)
137
+ assert_equal ["b", "b", ""], tsv.unzip("ValueA", merge: true)["row1:a"]["ValueB"]
138
+ assert_equal ["b", "b", "", "B"].sort, tsv.unzip("ValueA", merge: true).zip(true)["row1"]["ValueB"].sort
139
+ end
140
+
141
+ end
142
+
111
143
  end
112
144
 
@@ -2,6 +2,9 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
2
  require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
3
 
4
4
  class TestSocket < Test::Unit::TestCase
5
+
6
+ class DoneProcessing end
7
+
5
8
  def test_simple
6
9
  socket = WorkQueue::Socket.new
7
10
 
@@ -24,7 +27,7 @@ class TestSocket < Test::Unit::TestCase
24
27
  def __test_speed
25
28
  socket = WorkQueue::Socket.new
26
29
 
27
- num = 10_000
30
+ num = 50_000
28
31
 
29
32
  Thread.new do
30
33
  num.times do |i|
@@ -5,8 +5,9 @@ class TestOrchestrator < Test::Unit::TestCase
5
5
  setup do
6
6
  module TestWF
7
7
  extend Workflow
8
+ self.name = "TestWF"
8
9
 
9
- MULT = 0.01
10
+ MULT = 0.1
10
11
  task :a => :text do
11
12
  sleep(TestWF::MULT * (rand(10) + 2))
12
13
  end
@@ -34,11 +35,11 @@ class TestOrchestrator < Test::Unit::TestCase
34
35
  end
35
36
  end
36
37
 
37
- def TODO_test_orchestrate_resources
38
+ def test_orchestrate_resources
38
39
 
39
40
  jobs =[]
40
41
 
41
- num = 10
42
+ num = 5
42
43
  num.times do |i|
43
44
  jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
44
45
  end
@@ -64,16 +65,14 @@ TestWF:
64
65
  cpus: 15
65
66
  EOF
66
67
 
67
- orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
68
- Log.with_severity 0 do
69
- orchestrator.process(rules, jobs)
70
- end
68
+ orchestrator = Workflow::Orchestrator.new(0.1, "cpus" => 30, "IO" => 10, "size" => 10 )
69
+ orchestrator.process(rules, jobs)
71
70
 
72
71
  data = Workflow.trace jobs, :plot_data => true
73
- eend = data.column("End.second").values.collect{|v| v.to_f}.max
72
+ eend = data.column("End.second").values.collect{|v| v.to_f }.max
74
73
  second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
75
74
  (0..eend.to_i).each do |second|
76
- tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
75
+ tasks = data.select("Start.second"){|s| s <= second }.select("End.second"){|s| s > second}
77
76
  cpus = 0
78
77
  tasks.through :key, ["Workflow", "Task"] do |k, values|
79
78
  workflow, task = values
@@ -82,11 +81,11 @@ TestWF:
82
81
  second_cpus[second] = cpus
83
82
  end
84
83
 
85
- assert Misc.mean(second_cpus.values) > 15
84
+ assert Misc.mean(second_cpus.values) > 15
86
85
  assert Misc.mean(second_cpus.values) < 30
87
86
  end
88
87
 
89
- def TODO_test_orchestrate_erase
88
+ def test_orchestrate_erase
90
89
 
91
90
  jobs =[]
92
91
 
@@ -119,9 +118,7 @@ TestWF:
119
118
  EOF
120
119
 
121
120
  orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
122
- Log.with_severity 3 do
123
- orchestrator.process(rules, jobs)
124
- end
121
+ orchestrator.process(rules, jobs)
125
122
 
126
123
  jobs.each do |job|
127
124
  assert job.step(:c).dependencies.empty?
@@ -131,7 +128,7 @@ TestWF:
131
128
 
132
129
  end
133
130
 
134
- def TODO_test_orchestrate_default
131
+ def test_orchestrate_default
135
132
 
136
133
  jobs =[]
137
134
 
@@ -166,9 +163,7 @@ TestWF:
166
163
  EOF
167
164
 
168
165
  orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
169
- Log.with_severity 3 do
170
- orchestrator.process(rules, jobs)
171
- end
166
+ orchestrator.process(rules, jobs)
172
167
 
173
168
  jobs.each do |job|
174
169
  assert job.step(:c).dependencies.empty?
@@ -178,7 +173,7 @@ TestWF:
178
173
 
179
174
  end
180
175
 
181
- def TODO_test_orchestrate_top_level
176
+ def test_orchestrate_top_level
182
177
 
183
178
  jobs =[]
184
179
 
@@ -211,9 +206,7 @@ TestWF:
211
206
  EOF
212
207
 
213
208
  orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
214
- Log.with_severity 3 do
215
- orchestrator.process(rules, jobs)
216
- end
209
+ orchestrator.process(rules, jobs)
217
210
 
218
211
  jobs.each do |job|
219
212
  next unless job.task_name.to_s == 'd'
@@ -224,7 +217,7 @@ TestWF:
224
217
 
225
218
  end
226
219
 
227
- def TODO_test_orchestrate_top_level_double_dep
220
+ def test_orchestrate_top_level_double_dep
228
221
 
229
222
  jobs =[]
230
223
 
@@ -257,9 +250,7 @@ TestWF:
257
250
  EOF
258
251
 
259
252
  orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
260
- Log.with_severity 3 do
261
- orchestrator.process(rules, jobs)
262
- end
253
+ orchestrator.process(rules, jobs)
263
254
 
264
255
  jobs.each do |job|
265
256
  next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
@@ -0,0 +1,25 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+
6
+ class TestWorkflowTrace < Test::Unit::TestCase
7
+ def test_true
8
+ m = Module.new do
9
+ extend Workflow
10
+ self.name = "TestWF"
11
+
12
+ input :option1
13
+ task :step1 do end
14
+
15
+ dep :step1
16
+ input :option2
17
+ task :step2 do end
18
+ end
19
+
20
+ job = m.job(:step2)
21
+ job.run
22
+ assert_equal 1, Workflow.trace([job])["TestWF#step1"]["Calls"]
23
+ end
24
+ end
25
+
@@ -0,0 +1,28 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestStepArchive < Test::Unit::TestCase
5
+ def test_archive
6
+ m = Module.new do
7
+ extend Workflow
8
+ self.name = "TestWF"
9
+
10
+ input :option1
11
+ task :step1 do end
12
+
13
+ dep :step1
14
+ input :option2
15
+ task :step2 do end
16
+ end
17
+
18
+ job = m.job(:step2, option1: "Option1", option2: "Option2")
19
+ job.run
20
+ job.archive_deps
21
+ assert_include job.archived_info, job.step(:step1).path
22
+ assert_equal :done, job.archived_info[job.step(:step1).path][:status]
23
+
24
+ assert_equal "Option1", job.archived_inputs[:option1]
25
+ assert_equal "Option1", job.inputs.concat(job.archived_inputs)[:option1]
26
+ end
27
+ end
28
+
@@ -0,0 +1,25 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/workflow'
5
+
6
+ class TestStepChildren < Test::Unit::TestCase
7
+ def test_child
8
+ TmpFile.with_file do |tmpfile|
9
+ step = Step.new tmpfile, ["12"] do |s|
10
+ pid = child do
11
+ Open.write(self.file(:somefile), 'TEST')
12
+ end
13
+ Process.waitpid pid
14
+ s.length
15
+ end
16
+ step.type = :integer
17
+
18
+ assert_equal 2, step.run
19
+ assert_equal 1, step.info[:children_pids].length
20
+ assert_include step.files, 'somefile'
21
+ end
22
+ end
23
+
24
+ end
25
+
@@ -42,4 +42,20 @@ class TestStepInfo < Test::Unit::TestCase
42
42
 
43
43
  end
44
44
  end
45
+
46
+ def test_messages
47
+ TmpFile.with_file do |tmpdir|
48
+ Path.setup(tmpdir)
49
+ tmpfile = tmpdir.test_step
50
+ step1 = Step.new tmpfile.step1, ["12"] do |s|
51
+ log :msg, "Message1"
52
+ log :msg, "Message2"
53
+ s.length
54
+ end
55
+
56
+ step1.run
57
+
58
+ assert_equal %w(Message1 Message2), step1.messages
59
+ end
60
+ end
45
61
  end
@@ -244,8 +244,8 @@ class TestTaskDependencies < Test::Unit::TestCase
244
244
  task :step2 => :string do |i| step(:step1).load end
245
245
  end
246
246
 
247
- job = wf.job(:step2, :input1 => 2)
248
- assert_equal 1, job.run
247
+ job = wf.job(:step2, :input1 => "2")
248
+ assert_equal "1", job.run
249
249
  assert_equal Task::DEFAULT_NAME, job.name
250
250
  assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
251
251
  end
@@ -261,9 +261,9 @@ class TestTaskDependencies < Test::Unit::TestCase
261
261
  task :step2 => :string do |i| step(:step1).load end
262
262
  end
263
263
 
264
- job = wf.job(:step2, "SOME_NAME", :input1 => 2)
264
+ job = wf.job(:step2, "SOME_NAME", :input1 => "2")
265
265
  assert_equal "SOME_NAME", job.step(:step1).clean_name
266
- assert_equal 2, job.run
266
+ assert_equal "2", job.run
267
267
  end
268
268
 
269
269
  def test_override_inputs_block_array
@@ -278,8 +278,8 @@ class TestTaskDependencies < Test::Unit::TestCase
278
278
  task :step2 => :string do |i| step(:step1).load end
279
279
  end
280
280
 
281
- job = wf.job(:step2, :input1 => 2)
282
- assert_equal 1, job.run
281
+ job = wf.job(:step2, :input1 => "2")
282
+ assert_equal "1", job.run
283
283
  assert_equal Task::DEFAULT_NAME, job.name
284
284
  assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
285
285
  end
@@ -294,8 +294,8 @@ class TestTaskDependencies < Test::Unit::TestCase
294
294
  task :step2 => :string do |i| step(:step1).load end
295
295
  end
296
296
 
297
- job = wf.job(:step2, :input1 => 2)
298
- assert_equal 1, job.run
297
+ job = wf.job(:step2, :input1 => "2")
298
+ assert_equal "1", job.run
299
299
  assert_equal Task::DEFAULT_NAME, job.name
300
300
  assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
301
301
  end
@@ -313,13 +313,13 @@ class TestTaskDependencies < Test::Unit::TestCase
313
313
  task :step3 => :string do |i| step(:step1).load end
314
314
  end
315
315
 
316
- job = wf.job(:step3, :input1 => 1)
316
+ job = wf.job(:step3, :input1 => "1")
317
317
  assert_equal Task::DEFAULT_NAME, job.name
318
318
  assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
319
- assert_equal 2, job.run
319
+ assert_equal "2", job.run
320
320
 
321
- job = wf.job(:step3, :input1 => 2)
322
- assert_equal 3, job.run
321
+ job = wf.job(:step3, :input1 => "2")
322
+ assert_equal "3", job.run
323
323
 
324
324
 
325
325
  job = wf.job(:step3)
@@ -341,16 +341,16 @@ class TestTaskDependencies < Test::Unit::TestCase
341
341
  dep :step1, :canfail => true
342
342
  task :step2 => :string do |i|
343
343
  if step(:step1).error?
344
- 0
344
+ "0"
345
345
  else
346
346
  step(:step1).load
347
347
  end
348
348
  end
349
349
  end
350
350
 
351
- assert_equal 1, wf.job(:step2, :input1 => 1).run
352
- assert_equal 2, wf.job(:step2, :input1 => 2).run
353
- assert_equal 0, wf.job(:step2, :input1 => -2).run
351
+ assert_equal "1", wf.job(:step2, :input1 => "1").run
352
+ assert_equal "2", wf.job(:step2, :input1 => "2").run
353
+ assert_equal "0", wf.job(:step2, :input1 => "-2").run
354
354
  end
355
355
 
356
356
  end
@@ -1,7 +1,7 @@
1
1
  require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
2
  require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
3
 
4
- require 'scout/workflow/util'
4
+ require 'scout/workflow'
5
5
 
6
6
  class TestTaskInput < Test::Unit::TestCase
7
7
  def example_workflow
@@ -227,4 +227,48 @@ class TestTaskInput < Test::Unit::TestCase
227
227
  end
228
228
  end
229
229
  end
230
+
231
+ def test_recursive_inputs
232
+ w = Module.new do
233
+ extend Workflow
234
+
235
+ self.name = "SaluteWF"
236
+
237
+ input :name, :string
238
+ task :salute => :string do |name|
239
+ "Hi #{name}"
240
+ end
241
+
242
+ task_alias :salute_miguel, self, :salute, :name => "Miguel"
243
+ end
244
+
245
+ assert w.tasks[:salute_miguel].recursive_inputs.empty?
246
+ end
247
+
248
+ def test_recursive_inputs_with_overriden_deps
249
+ w = Module.new do
250
+ extend Workflow
251
+
252
+ self.name = "SaluteWF"
253
+
254
+ input :name, :string
255
+ task :salute => :string do |name|
256
+ "Hi #{name}"
257
+ end
258
+
259
+ task_alias :salute_miguel, self, :salute, :name => "Miguel"
260
+
261
+ dep :salute
262
+ task :repeat_salute => :array do
263
+ [step(:salute).load] * 3
264
+ end
265
+
266
+ dep :salute_miguel
267
+ task_alias :repeat_salute_miguel, self, :repeat_salute, "SaluteWF#salute" => :salute_miguel
268
+ end
269
+
270
+ assert_equal ["Hi Miguel"] * 3, w.job(:repeat_salute_miguel).run
271
+
272
+ assert w.tasks[:repeat_salute_miguel].recursive_inputs.empty?
273
+ end
230
274
  end