scout-gear 6.0.0 → 7.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +465 -432
  3. data/VERSION +1 -1
  4. data/bin/scout +5 -1
  5. data/lib/rbbt-scout.rb +5 -0
  6. data/lib/scout/concurrent_stream.rb +6 -2
  7. data/lib/scout/config.rb +168 -0
  8. data/lib/scout/exceptions.rb +9 -0
  9. data/lib/scout/indiferent_hash/options.rb +1 -0
  10. data/lib/scout/indiferent_hash.rb +4 -2
  11. data/lib/scout/log/color.rb +31 -2
  12. data/lib/scout/log/progress/report.rb +1 -0
  13. data/lib/scout/log/progress/util.rb +3 -1
  14. data/lib/scout/log/progress.rb +7 -3
  15. data/lib/scout/log.rb +8 -3
  16. data/lib/scout/misc/digest.rb +1 -3
  17. data/lib/scout/misc/monitor.rb +3 -0
  18. data/lib/scout/misc/system.rb +15 -0
  19. data/lib/scout/misc.rb +1 -0
  20. data/lib/scout/named_array.rb +68 -0
  21. data/lib/scout/open/stream.rb +58 -26
  22. data/lib/scout/path/find.rb +27 -3
  23. data/lib/scout/path/util.rb +7 -4
  24. data/lib/scout/persist/serialize.rb +7 -14
  25. data/lib/scout/persist.rb +21 -1
  26. data/lib/scout/resource/produce.rb +7 -94
  27. data/lib/scout/resource/software.rb +176 -0
  28. data/lib/scout/tsv/dumper.rb +107 -0
  29. data/lib/scout/tsv/index.rb +49 -0
  30. data/lib/scout/tsv/parser.rb +317 -0
  31. data/lib/scout/tsv/path.rb +13 -0
  32. data/lib/scout/tsv/persist/adapter.rb +348 -0
  33. data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
  34. data/lib/scout/tsv/persist.rb +15 -0
  35. data/lib/scout/tsv/traverse.rb +48 -0
  36. data/lib/scout/tsv/util.rb +24 -0
  37. data/lib/scout/tsv.rb +27 -0
  38. data/lib/scout/work_queue/worker.rb +16 -11
  39. data/lib/scout/work_queue.rb +63 -21
  40. data/lib/scout/workflow/definition.rb +93 -4
  41. data/lib/scout/workflow/step/config.rb +18 -0
  42. data/lib/scout/workflow/step/dependencies.rb +40 -0
  43. data/lib/scout/workflow/step/file.rb +15 -0
  44. data/lib/scout/workflow/step/info.rb +33 -6
  45. data/lib/scout/workflow/step/provenance.rb +148 -0
  46. data/lib/scout/workflow/step.rb +70 -20
  47. data/lib/scout/workflow/task.rb +5 -4
  48. data/lib/scout/workflow/usage.rb +1 -1
  49. data/lib/scout/workflow.rb +11 -3
  50. data/lib/scout-gear.rb +1 -0
  51. data/lib/scout.rb +1 -0
  52. data/scout-gear.gemspec +38 -3
  53. data/scout_commands/find +1 -1
  54. data/scout_commands/workflow/task +16 -10
  55. data/share/software/install_helpers +523 -0
  56. data/test/scout/log/test_progress.rb +0 -2
  57. data/test/scout/misc/test_system.rb +21 -0
  58. data/test/scout/open/test_stream.rb +160 -1
  59. data/test/scout/path/test_find.rb +14 -7
  60. data/test/scout/resource/test_software.rb +24 -0
  61. data/test/scout/test_config.rb +66 -0
  62. data/test/scout/test_meta_extension.rb +10 -0
  63. data/test/scout/test_named_array.rb +19 -0
  64. data/test/scout/test_persist.rb +35 -0
  65. data/test/scout/test_semaphore.rb +1 -1
  66. data/test/scout/test_tmpfile.rb +2 -2
  67. data/test/scout/test_tsv.rb +74 -0
  68. data/test/scout/test_work_queue.rb +63 -8
  69. data/test/scout/tsv/persist/test_adapter.rb +34 -0
  70. data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
  71. data/test/scout/tsv/test_dumper.rb +44 -0
  72. data/test/scout/tsv/test_index.rb +64 -0
  73. data/test/scout/tsv/test_parser.rb +173 -0
  74. data/test/scout/tsv/test_persist.rb +36 -0
  75. data/test/scout/tsv/test_traverse.rb +9 -0
  76. data/test/scout/tsv/test_util.rb +0 -0
  77. data/test/scout/work_queue/test_worker.rb +49 -1
  78. data/test/scout/workflow/step/test_dependencies.rb +25 -0
  79. data/test/scout/workflow/step/test_info.rb +15 -17
  80. data/test/scout/workflow/step/test_load.rb +16 -18
  81. data/test/scout/workflow/step/test_provenance.rb +25 -0
  82. data/test/scout/workflow/test_step.rb +206 -10
  83. data/test/scout/workflow/test_task.rb +0 -3
  84. data/test/test_helper.rb +6 -0
  85. metadata +37 -2
@@ -15,22 +15,218 @@ class TestWorkflowStep < Test::Unit::TestCase
15
15
  end
16
16
 
17
17
  def test_dependency
18
- Log.with_severity 0 do
19
- tmpfile = tmpdir.test_step
20
- step1 = Step.new tmpfile.step1, ["12"] do |s|
21
- s.length
18
+ tmpfile = tmpdir.test_step
19
+ step1 = Step.new tmpfile.step1, ["12"] do |s|
20
+ s.length
21
+ end
22
+
23
+ step2 = Step.new tmpfile.step2 do
24
+ step1 = dependencies.first
25
+ step1.inputs.first + " has " + step1.load.to_s + " characters"
26
+ end
27
+
28
+ step2.dependencies = [step1]
29
+
30
+
31
+ assert_equal "12 has 2 characters", step2.run
32
+ assert_equal "12 has 2 characters", step2.run
33
+ end
34
+
35
+ def test_streaming
36
+ tmpfile = tmpdir.test_step
37
+
38
+ times = 10_000
39
+ sleep = 1 / times
40
+
41
+ step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
42
+ Open.open_pipe do |sin|
43
+ times.times do |i|
44
+ sin.puts "line-#{i}"
45
+ sleep sleep
46
+ end
22
47
  end
48
+ end
49
+ step1.type = :array
23
50
 
24
- step2 = Step.new tmpfile.step2 do
25
- step1 = dependencies.first
26
- step1.inputs.first + " has " + step1.load.to_s + " characters"
51
+ step2 = Step.new tmpfile.step2 do
52
+ step1 = dependencies.first
53
+ stream = step1.get_stream
54
+
55
+ Open.open_pipe do |sin|
56
+ while line = stream.gets
57
+ num = line.split("-").last
58
+ next if num.to_i % 2 == 1
59
+ sin.puts line
60
+ end
27
61
  end
62
+ end
63
+ step2.type = :array
64
+ step2.dependencies = [step1]
65
+
66
+ step1.run
67
+ step1.join
68
+ assert step1.path.read.end_with? "line-#{times-1}\n"
28
69
 
29
- step2.dependencies = [step1]
70
+ step1.clean
30
71
 
72
+ stream = step2.exec
73
+ lines = []
74
+ while line = stream.gets
75
+ lines << line
76
+ end
77
+ assert_equal times/2, lines.length
78
+
79
+ stream = step2.run
80
+ assert step1.streaming?
81
+ assert step2.streaming?
31
82
 
32
- assert_equal "12 has 2 characters", step2.run
33
- assert_equal "12 has 2 characters", step2.run
83
+ lines = []
84
+ while line = stream.gets
85
+ lines << line
34
86
  end
87
+
88
+ assert step1.path.read.end_with? "line-#{times-1}\n"
89
+ assert_equal times/2, lines.length
90
+ assert_equal times/2, step2.path.read.split("\n").length
91
+ end
92
+
93
+ def test_streaming_duplicate
94
+ tmpfile = tmpdir.test_step
95
+
96
+ times = 10_000
97
+ sleep = 0.01 / times
98
+
99
+ step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
100
+ Open.open_pipe do |sin|
101
+ times.times do |i|
102
+ sin.puts "line-#{i}"
103
+ sleep sleep
104
+ end
105
+ end
106
+ end
107
+ step1.type = :array
108
+
109
+ step2 = Step.new tmpfile.step2 do
110
+ step1 = dependencies.first
111
+ stream = step1.get_stream
112
+
113
+ Open.open_pipe do |sin|
114
+ while line = stream.gets
115
+ num = line.split("-").last
116
+ next if num.to_i % 2 == 1
117
+ sin.puts line
118
+ end
119
+ end
120
+ end
121
+ step2.type = :array
122
+ step2.dependencies = [step1]
123
+
124
+ step3 = Step.new tmpfile.step3 do
125
+ step1, step2 = dependencies
126
+ stream = step2.get_stream
127
+
128
+ Open.open_pipe do |sin|
129
+ while line = stream.gets
130
+ num = line.split("-").last
131
+ next if num.to_i % 2 == 1
132
+ sin.puts line
133
+ end
134
+ end
135
+ end
136
+ step3.type = :array
137
+ step3.dependencies = [step1, step2]
138
+
139
+ step3.recursive_clean
140
+
141
+ stream = step3.run
142
+ out = []
143
+ while l = stream.gets
144
+ out << l
145
+ end
146
+ assert_equal times/2, out.length
147
+ end
148
+
149
+ def test_fork_stream
150
+ tmpfile = tmpdir.test_step
151
+
152
+ times = 10_000
153
+ sleep = 0.1 / times
154
+
155
+ step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
156
+ Open.open_pipe do |sin|
157
+ times.times do |i|
158
+ sin.puts "line-#{i}"
159
+ sleep sleep
160
+ end
161
+ end
162
+ end
163
+ step1.type = :array
164
+
165
+ step2 = Step.new tmpfile.step2 do
166
+ step1 = dependencies.first
167
+ stream = step1.get_stream
168
+
169
+ Open.open_pipe do |sin|
170
+ while line = stream.gets
171
+ num = line.split("-").last
172
+ next if num.to_i % 2 == 1
173
+ sin.puts "S2: " + line
174
+ end
175
+ end
176
+ end
177
+ step2.type = :array
178
+ step2.dependencies = [step1]
179
+
180
+ step3 = Step.new tmpfile.step3 do
181
+ step1 = dependencies.first
182
+ stream = step1.get_stream
183
+
184
+ Open.open_pipe do |sin|
185
+ while line = stream.gets
186
+ num = line.split("-").last
187
+ next if num.to_i % 2 == 0
188
+ sin.puts "S3: " + line
189
+ end
190
+ end
191
+ end
192
+ step3.type = :array
193
+ step3.dependencies = [step1]
194
+
195
+
196
+ step4 = Step.new tmpfile.step4 do
197
+ step2, step3 = dependencies
198
+
199
+ mutex = Mutex.new
200
+ Open.open_pipe do |sin|
201
+ t2 = Thread.new do
202
+ stream2 = step2.get_stream
203
+ while line = stream2.gets
204
+ sin.puts line
205
+ end
206
+ end
207
+
208
+ t3 = Thread.new do
209
+ stream3 = step3.get_stream
210
+ while line = stream3.gets
211
+ sin.puts line
212
+ end
213
+ end
214
+ t2.join
215
+ t3.join
216
+ end
217
+ end
218
+ step4.type = :array
219
+ step4.dependencies = [step2, step3]
220
+
221
+ lines = []
222
+ io = step4.run
223
+ Log::ProgressBar.with_bar do |b|
224
+ while line = io.gets
225
+ b.tick
226
+ lines << line.strip
227
+ end
228
+ end
229
+
230
+ assert_equal times, lines.length
35
231
  end
36
232
  end
@@ -37,7 +37,6 @@ class TestTask < Test::Unit::TestCase
37
37
  end
38
38
 
39
39
  def test_override_inputs_block
40
- Log.severity = 0
41
40
  wf = Workflow.annonymous_workflow "TaskInputs" do
42
41
  input :input1, :string
43
42
  task :step1 => :string do |i| i end
@@ -55,7 +54,6 @@ class TestTask < Test::Unit::TestCase
55
54
  end
56
55
 
57
56
  def test_task_override_dep
58
- Log.severity = 0
59
57
  wf = Workflow.annonymous_workflow "TaskInputs" do
60
58
  input :input1, :integer
61
59
  task :step1 => :integer do |i| i end
@@ -171,7 +169,6 @@ class TestTask < Test::Unit::TestCase
171
169
  end
172
170
  end
173
171
 
174
- Log.severity = 0
175
172
  job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
176
173
  assert_equal 5, job.run
177
174
  end
data/test/test_helper.rb CHANGED
@@ -22,16 +22,22 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
22
22
  #require 'scout/helper/misc/development'
23
23
  require 'scout/tmpfile'
24
24
  require 'scout/log'
25
+ require 'scout/open'
25
26
  require 'scout/persist'
26
27
  require 'scout/workflow'
27
28
 
28
29
  class Test::Unit::TestCase
29
30
 
31
+ def assert_equal_path(path1, path2)
32
+ assert_equal File.expand_path(path1), File.expand_path(path2)
33
+ end
34
+
30
35
  def tmpdir
31
36
  @tmpdir = Path.setup('tmp/test_tmpdir').find
32
37
  end
33
38
 
34
39
  setup do
40
+ Log::ProgressBar.default_severity = 0
35
41
  Persist.cache_dir = tmpdir.var.cache
36
42
  Open.remote_cache_dir = tmpdir.var.cache
37
43
  Workflow.directory = tmpdir.var.jobs
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scout-gear
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.0
4
+ version: 7.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-04-29 00:00:00.000000000 Z
11
+ date: 2023-05-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: term-ansicolor
@@ -97,10 +97,12 @@ files:
97
97
  - Rakefile
98
98
  - VERSION
99
99
  - bin/scout
100
+ - lib/rbbt-scout.rb
100
101
  - lib/scout-gear.rb
101
102
  - lib/scout.rb
102
103
  - lib/scout/cmd.rb
103
104
  - lib/scout/concurrent_stream.rb
105
+ - lib/scout/config.rb
104
106
  - lib/scout/exceptions.rb
105
107
  - lib/scout/indiferent_hash.rb
106
108
  - lib/scout/indiferent_hash/case_insensitive.rb
@@ -119,6 +121,8 @@ files:
119
121
  - lib/scout/misc/format.rb
120
122
  - lib/scout/misc/insist.rb
121
123
  - lib/scout/misc/monitor.rb
124
+ - lib/scout/misc/system.rb
125
+ - lib/scout/named_array.rb
122
126
  - lib/scout/open.rb
123
127
  - lib/scout/open/lock.rb
124
128
  - lib/scout/open/remote.rb
@@ -137,6 +141,7 @@ files:
137
141
  - lib/scout/resource/produce.rb
138
142
  - lib/scout/resource/produce/rake.rb
139
143
  - lib/scout/resource/scout.rb
144
+ - lib/scout/resource/software.rb
140
145
  - lib/scout/resource/util.rb
141
146
  - lib/scout/semaphore.rb
142
147
  - lib/scout/simple_opt.rb
@@ -146,6 +151,16 @@ files:
146
151
  - lib/scout/simple_opt/parse.rb
147
152
  - lib/scout/simple_opt/setup.rb
148
153
  - lib/scout/tmpfile.rb
154
+ - lib/scout/tsv.rb
155
+ - lib/scout/tsv/dumper.rb
156
+ - lib/scout/tsv/index.rb
157
+ - lib/scout/tsv/parser.rb
158
+ - lib/scout/tsv/path.rb
159
+ - lib/scout/tsv/persist.rb
160
+ - lib/scout/tsv/persist/adapter.rb
161
+ - lib/scout/tsv/persist/tokyocabinet.rb
162
+ - lib/scout/tsv/traverse.rb
163
+ - lib/scout/tsv/util.rb
149
164
  - lib/scout/work_queue.rb
150
165
  - lib/scout/work_queue/socket.rb
151
166
  - lib/scout/work_queue/worker.rb
@@ -153,8 +168,12 @@ files:
153
168
  - lib/scout/workflow/definition.rb
154
169
  - lib/scout/workflow/documentation.rb
155
170
  - lib/scout/workflow/step.rb
171
+ - lib/scout/workflow/step/config.rb
172
+ - lib/scout/workflow/step/dependencies.rb
173
+ - lib/scout/workflow/step/file.rb
156
174
  - lib/scout/workflow/step/info.rb
157
175
  - lib/scout/workflow/step/load.rb
176
+ - lib/scout/workflow/step/provenance.rb
158
177
  - lib/scout/workflow/task.rb
159
178
  - lib/scout/workflow/task/inputs.rb
160
179
  - lib/scout/workflow/usage.rb
@@ -171,6 +190,7 @@ files:
171
190
  - scout_commands/workflow/task_old
172
191
  - share/color/color_names
173
192
  - share/color/diverging_colors.hex
193
+ - share/software/install_helpers
174
194
  - test/scout/indiferent_hash/test_case_insensitive.rb
175
195
  - test/scout/indiferent_hash/test_options.rb
176
196
  - test/scout/log/test_color.rb
@@ -178,6 +198,7 @@ files:
178
198
  - test/scout/misc/test_digest.rb
179
199
  - test/scout/misc/test_filesystem.rb
180
200
  - test/scout/misc/test_insist.rb
201
+ - test/scout/misc/test_system.rb
181
202
  - test/scout/open/test_lock.rb
182
203
  - test/scout/open/test_remote.rb
183
204
  - test/scout/open/test_stream.rb
@@ -189,6 +210,7 @@ files:
189
210
  - test/scout/persist/test_serialize.rb
190
211
  - test/scout/resource/test_path.rb
191
212
  - test/scout/resource/test_produce.rb
213
+ - test/scout/resource/test_software.rb
192
214
  - test/scout/resource/test_util.rb
193
215
  - test/scout/simple_opt/test_doc.rb
194
216
  - test/scout/simple_opt/test_get.rb
@@ -196,22 +218,35 @@ files:
196
218
  - test/scout/simple_opt/test_setup.rb
197
219
  - test/scout/test_cmd.rb
198
220
  - test/scout/test_concurrent_stream.rb
221
+ - test/scout/test_config.rb
199
222
  - test/scout/test_indiferent_hash.rb
200
223
  - test/scout/test_log.rb
201
224
  - test/scout/test_meta_extension.rb
202
225
  - test/scout/test_misc.rb
226
+ - test/scout/test_named_array.rb
203
227
  - test/scout/test_open.rb
204
228
  - test/scout/test_path.rb
205
229
  - test/scout/test_persist.rb
206
230
  - test/scout/test_resource.rb
207
231
  - test/scout/test_semaphore.rb
208
232
  - test/scout/test_tmpfile.rb
233
+ - test/scout/test_tsv.rb
209
234
  - test/scout/test_work_queue.rb
210
235
  - test/scout/test_workflow.rb
236
+ - test/scout/tsv/persist/test_adapter.rb
237
+ - test/scout/tsv/persist/test_tokyocabinet.rb
238
+ - test/scout/tsv/test_dumper.rb
239
+ - test/scout/tsv/test_index.rb
240
+ - test/scout/tsv/test_parser.rb
241
+ - test/scout/tsv/test_persist.rb
242
+ - test/scout/tsv/test_traverse.rb
243
+ - test/scout/tsv/test_util.rb
211
244
  - test/scout/work_queue/test_socket.rb
212
245
  - test/scout/work_queue/test_worker.rb
246
+ - test/scout/workflow/step/test_dependencies.rb
213
247
  - test/scout/workflow/step/test_info.rb
214
248
  - test/scout/workflow/step/test_load.rb
249
+ - test/scout/workflow/step/test_provenance.rb
215
250
  - test/scout/workflow/task/test_inputs.rb
216
251
  - test/scout/workflow/test_definition.rb
217
252
  - test/scout/workflow/test_documentation.rb