scout-gear 8.0.0 → 9.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (111) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +48 -9
  3. data/Rakefile +6 -1
  4. data/VERSION +1 -1
  5. data/bin/scout +16 -4
  6. data/doc/lib/scout/path.md +35 -0
  7. data/doc/lib/scout/workflow/task.md +13 -0
  8. data/lib/rbbt-scout.rb +2 -1
  9. data/lib/scout/cmd.rb +32 -29
  10. data/lib/scout/concurrent_stream.rb +36 -19
  11. data/lib/scout/exceptions.rb +10 -0
  12. data/lib/scout/indiferent_hash.rb +17 -0
  13. data/lib/scout/log/color.rb +11 -11
  14. data/lib/scout/log/progress/report.rb +8 -5
  15. data/lib/scout/log/progress/util.rb +3 -0
  16. data/lib/scout/log/trap.rb +3 -3
  17. data/lib/scout/log.rb +67 -36
  18. data/lib/scout/meta_extension.rb +34 -0
  19. data/lib/scout/misc/digest.rb +11 -2
  20. data/lib/scout/misc/filesystem.rb +2 -3
  21. data/lib/scout/misc/format.rb +12 -7
  22. data/lib/scout/misc/monitor.rb +11 -0
  23. data/lib/scout/misc/system.rb +48 -0
  24. data/lib/scout/named_array.rb +8 -0
  25. data/lib/scout/offsite/ssh.rb +174 -0
  26. data/lib/scout/offsite/step.rb +100 -0
  27. data/lib/scout/offsite/sync.rb +55 -0
  28. data/lib/scout/offsite.rb +3 -0
  29. data/lib/scout/open/lock.rb +5 -24
  30. data/lib/scout/open/remote.rb +12 -1
  31. data/lib/scout/open/stream.rb +109 -122
  32. data/lib/scout/open/util.rb +9 -0
  33. data/lib/scout/open.rb +12 -11
  34. data/lib/scout/path/find.rb +15 -10
  35. data/lib/scout/path/util.rb +5 -0
  36. data/lib/scout/path.rb +1 -1
  37. data/lib/scout/persist/serialize.rb +4 -4
  38. data/lib/scout/persist.rb +1 -1
  39. data/lib/scout/resource/open.rb +8 -0
  40. data/lib/scout/resource/path.rb +16 -9
  41. data/lib/scout/resource/software.rb +4 -2
  42. data/lib/scout/resource/util.rb +10 -4
  43. data/lib/scout/resource.rb +2 -0
  44. data/lib/scout/tsv/dumper.rb +5 -1
  45. data/lib/scout/tsv/index.rb +28 -86
  46. data/lib/scout/tsv/open.rb +35 -14
  47. data/lib/scout/tsv/parser.rb +22 -5
  48. data/lib/scout/tsv/persist/tokyocabinet.rb +2 -0
  49. data/lib/scout/tsv/stream.rb +204 -0
  50. data/lib/scout/tsv/transformer.rb +11 -0
  51. data/lib/scout/tsv.rb +9 -2
  52. data/lib/scout/work_queue/worker.rb +2 -2
  53. data/lib/scout/work_queue.rb +37 -12
  54. data/lib/scout/workflow/definition.rb +2 -1
  55. data/lib/scout/workflow/deployment/orchestrator.rb +254 -0
  56. data/lib/scout/workflow/deployment.rb +1 -0
  57. data/lib/scout/workflow/step/dependencies.rb +46 -14
  58. data/lib/scout/workflow/step/file.rb +5 -0
  59. data/lib/scout/workflow/step/info.rb +13 -3
  60. data/lib/scout/workflow/step/inputs.rb +5 -0
  61. data/lib/scout/workflow/step/load.rb +1 -1
  62. data/lib/scout/workflow/step/provenance.rb +1 -0
  63. data/lib/scout/workflow/step/status.rb +27 -9
  64. data/lib/scout/workflow/step.rb +82 -30
  65. data/lib/scout/workflow/task/dependencies.rb +116 -0
  66. data/lib/scout/workflow/task/inputs.rb +36 -17
  67. data/lib/scout/workflow/task.rb +12 -109
  68. data/lib/scout/workflow/usage.rb +57 -41
  69. data/lib/scout/workflow.rb +19 -13
  70. data/lib/scout-gear.rb +2 -0
  71. data/lib/scout.rb +6 -0
  72. data/scout-gear.gemspec +38 -7
  73. data/scout_commands/doc +37 -0
  74. data/scout_commands/find +1 -0
  75. data/scout_commands/offsite +30 -0
  76. data/scout_commands/resource/produce +66 -0
  77. data/scout_commands/template +52 -0
  78. data/scout_commands/update +29 -0
  79. data/scout_commands/workflow/info +15 -3
  80. data/scout_commands/workflow/install +105 -0
  81. data/scout_commands/workflow/task +46 -6
  82. data/share/software/install_helpers +2 -2
  83. data/share/templates/command +25 -0
  84. data/share/templates/workflow.rb +14 -0
  85. data/test/scout/offsite/test_ssh.rb +15 -0
  86. data/test/scout/offsite/test_step.rb +32 -0
  87. data/test/scout/offsite/test_sync.rb +36 -0
  88. data/test/scout/offsite/test_task.rb +0 -0
  89. data/test/scout/resource/test_path.rb +6 -0
  90. data/test/scout/test_named_array.rb +6 -0
  91. data/test/scout/test_persist.rb +3 -2
  92. data/test/scout/test_tsv.rb +17 -0
  93. data/test/scout/test_work_queue.rb +64 -42
  94. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  95. data/test/scout/tsv/test_index.rb +14 -0
  96. data/test/scout/tsv/test_parser.rb +35 -0
  97. data/test/scout/tsv/test_stream.rb +200 -0
  98. data/test/scout/tsv/test_transformer.rb +12 -0
  99. data/test/scout/workflow/deployment/test_orchestrator.rb +272 -0
  100. data/test/scout/workflow/step/test_dependencies.rb +68 -0
  101. data/test/scout/workflow/step/test_info.rb +17 -0
  102. data/test/scout/workflow/step/test_status.rb +0 -1
  103. data/test/scout/workflow/task/test_dependencies.rb +357 -0
  104. data/test/scout/workflow/task/test_inputs.rb +52 -0
  105. data/test/scout/workflow/test_definition.rb +18 -0
  106. data/test/scout/workflow/test_documentation.rb +24 -0
  107. data/test/scout/workflow/test_step.rb +109 -0
  108. data/test/scout/workflow/test_task.rb +0 -287
  109. data/test/test_scout.rb +9 -0
  110. metadata +89 -5
  111. data/scout_commands/workflow/task_old +0 -706
@@ -1,706 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'scout'
4
-
5
- def report_options(options)
6
- if options.nil? or options.empty?
7
- puts "No options"
8
- else
9
- options.each do |key, value|
10
- puts [Log.color(:cyan, key), Misc.fingerprint(value)] * ": "
11
- end
12
- end
13
- end
14
-
15
- def usage(workflow = nil, task = nil, exception=nil, abridge = false)
16
- puts SOPT.doc
17
- puts
18
- if workflow.nil?
19
- puts "No workflow specified. Use `scout workflow list` to list available workflows."
20
- exit! -1
21
- end
22
-
23
- if task.nil?
24
- workflow.load_tasks if workflow.respond_to? :load_tasks
25
- workflow.doc nil, abridge
26
- puts
27
- puts "E.g. scout workflow task #{workflow.to_s} #{workflow.tasks.keys.first.to_s} -h"
28
- else
29
- puts Log.color :magenta, workflow.to_s
30
- puts Log.color :magenta, "=" * workflow.to_s.length
31
- if workflow.documentation[:title] and not workflow.documentation[:title].empty?
32
- puts
33
- puts Misc.format_paragraph workflow.documentation[:title]
34
- end
35
- if workflow.documentation[:description] and not workflow.documentation[:description].empty?
36
- puts
37
- puts Misc.format_paragraph workflow.documentation[:description]
38
- end
39
- puts
40
- workflow.doc(task, abridge)
41
- end
42
-
43
- print_error(exception.message, exception.backtrace) if exception
44
-
45
- true
46
- end
47
-
48
- def get_value_stream(value)
49
- if value == "-"
50
- io = Misc.open_pipe do |sin|
51
- while not STDIN.eof?
52
- sin.write STDIN.read(2048)
53
- end
54
- sin.close
55
- end
56
- else
57
- io = Open.open(value)
58
- end
59
- class << io
60
- attr_accessor :filename
61
- end
62
- io.filename = value
63
- io
64
- end
65
-
66
- def fix_options(workflow, task, job_options)
67
- input_types = IndiferentHash.setup workflow.rec_input_types(task.name)
68
- input_options = IndiferentHash.setup workflow.rec_input_options(task.name)
69
-
70
- job_options_cleaned = {}
71
-
72
- job_options.each do |name, value|
73
- type = input_types[name]
74
- type = type.to_sym if type
75
-
76
- if Step === value
77
- job_options_cleaned[name] = value
78
- next
79
- end
80
-
81
- if Path === value && Step === value.resource
82
- job_options_cleaned[name] = value
83
- next
84
- end
85
-
86
- value = case type
87
- when nil
88
- value
89
- when :boolean
90
- TrueClass === value or %w(true TRUE T yes).include? value
91
- when :float
92
- value.to_f
93
- when :path
94
- Path.setup(value)
95
- when :integer
96
- value.to_i
97
- when :text
98
- if input_options[name] and input_options[name][:stream] and String === value
99
- get_value_stream(value)
100
- else
101
- case
102
- when value == '-'
103
- STDIN.read
104
- when (String === value and File.exist?(value) and not File.directory?(value))
105
- Open.read(value)
106
- else
107
- value
108
- end
109
- end
110
- when :array
111
- if input_options[name] && input_options[name][:stream] && String === value && Misc.is_filename?(value) && !! input_options[name][:nofile]
112
- get_value_stream(value)
113
- elsif input_options[name] and input_options[name][:stream] and value == "-"
114
- STDIN
115
- else
116
- if Array === value || IO === value
117
- value
118
- else
119
- array_separator = $array_separator
120
- str = case
121
- when value == '-'
122
- array_separator ||= "\n"
123
- STDIN.read
124
- when (String === value and File.exist?(value) and not (input_options[name] && input_options[name][:nofile]))
125
- array_separator ||= "\n"
126
- Open.read(value)
127
- else
128
- value
129
- end
130
-
131
- if array_separator
132
- str.split(/#{array_separator}/)
133
- else
134
- str.split(/[,|\s]/)
135
- end
136
- end
137
- end
138
- when :tsv
139
- if input_options[name] and input_options[name][:stream] and String === value
140
- TSV::Parser.new(value == '-' ? STDIN : Open.open(value), :filename => value )
141
- else
142
- case value
143
- when TSV
144
- value
145
- when '-'
146
- TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
147
- when (Misc.is_filename?(value) and String)
148
- TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
149
- when IO
150
- TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
151
- else
152
- TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
153
- end
154
- end
155
- when :directory
156
- Path.setup(File.expand_path(value))
157
- else
158
- value
159
- end
160
-
161
- job_options_cleaned[name] = value
162
- end
163
-
164
- job_options_cleaned
165
- end
166
-
167
- options = SOPT.setup <<EOF
168
- Enact workflow tasks
169
-
170
- $ scout workflow task <workflow> [<task>] [<options>]
171
-
172
- Examine workflows and enact tasks from them. If no `task` is specified, a list
173
- of available tasks is shown. If a `task` is given it will be enacted with the
174
- parameters specified in `options`. Use *-h* option to display the description
175
- of a task, including the parameters it accepts; and some examples, if
176
- available. Examples can be enacted using `scout workflow example <workflow>
177
- [<task>] [<example>]`.
178
-
179
- When a task is enacted a job is instantiated. This job is identified by the
180
- `jobname` (which is *#{Task::DEFAULT_NAME}* unless specified otherwise) and the values of the
181
- parameters; these two things determine the filename under which the job result
182
- will be saved. If the taks is enacted using the same `jobname` and parameters
183
- it will result in the same job, pointing to the same result file.
184
-
185
- The first time a job is executed it will save the result. The saved result will
186
- be returned directly if the same task is re-enacted. Once the job is done you
187
- can redo it using the `clean` parameter, this cleans the last step of the task.
188
- The `recursive_clean` cleans all the job dependency steps recursively.
189
-
190
- -h--help Show this help
191
- -ha--abridge Abridge help
192
- -wd--workdir* Change the working directory of the workflow
193
- -wda--workdir_all* Change the working directory of ALL workflow
194
- -as--array_separator* Change the character that separates elements of Arrays, ',', '|', or '\\n' by default
195
- -fs--field_separator* Change the character that separates fields of TSV files '\\t' by default
196
- -jn--jobname* Job name to use. The name '#{Task::DEFAULT_NAME}' is used by default
197
- -pn--printname Print the name of the job and exit without starting it
198
- -pf--printpath Print the path of the job result
199
- -cl--clean Clean the last step of the job so that it gets recomputed
200
- -ct--clean_task* Clean a particular dependency task
201
- -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
202
- -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
203
- --fork Run job asyncronously and monitor progress. It monitors detached processes as well
204
- --orchestrate* Run the job through the orchestrator
205
- --detach Run job asyncronously and detach process
206
- --exec Run job with no persistence
207
- -O--output* Save job result into file
208
- -jf--job_file* Output one of the job produced files
209
- -ljf--list_job_files List all the files produced in that step
210
- --load_inputs* Load inputs from a directory
211
- --info Show the job info
212
- -prov--provenance Report the jobs provenance
213
- -W--workflows* Load a list of workflows
214
- -R--requires* Require a list of files
215
- -pro--produce* Prepare dependencies
216
- -proc--produce_cpus* Number of dependencies prepared in parallel
217
- -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
218
- -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
219
- -PERF--procpath_performance* Measure performance using procpath
220
- -r--relay* Relay job to SSH server
221
- -sr--slurm_relay* Relay job to SSH SLURM server
222
- -rdep--relay_dependencies* Relay dependencies instead of main job
223
- -pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
224
- EOF
225
-
226
- workflow = ARGV.shift
227
- usage and exit! -1 if workflow.nil?
228
-
229
- task = ARGV.shift
230
-
231
- # Set log, fork, clean, recursive_clean and help
232
- help = !!options.delete(:help)
233
- do_fork = !!options.delete(:fork)
234
- detach = !!options.delete(:detach)
235
- do_exec = !!options.delete(:exec)
236
- clean_task = options.delete(:clean_task)
237
- clean = !!options.delete(:clean) || clean_task
238
- override_deps = options.delete(:override_deps)
239
- recursive_clean = !!options.delete(:recursive_clean)
240
- out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT
241
-
242
- $array_separator = options.delete(:array_separator)
243
- $field_separator = options.delete(:field_separator) || "\t"
244
-
245
- # Get workflow
246
-
247
- if Scout.etc.remote_workflows.exists?
248
- remote_workflows = Scout.etc.remote_workflows.yaml
249
- else
250
- remote_workflows = {}
251
- end
252
-
253
- #Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
254
- Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
255
-
256
- workflow = Workflow.require_workflow workflow
257
-
258
- if clean_task
259
- ENV["RBBT_UPDATE"] = 'true'
260
- end
261
-
262
- if options[:update_all_jobs]
263
- ENV["RBBT_UPDATE_ALL_JOBS"] = 'true'
264
- ENV["RBBT_UPDATE"] = 'true'
265
- end
266
-
267
- if options[:workflows]
268
- require 'scout/workflow'
269
- workflows = options[:workflows].split(',')
270
- workflows.each do |workflow|
271
- workflow.strip!
272
- Workflow.require_workflow workflow
273
- end
274
- end
275
-
276
- if options[:requires]
277
- requires = options[:requires].split(',')
278
- requires.each do |req|
279
- req.strip!
280
- require req
281
- end
282
- end
283
-
284
- if options[:remote_workflow_tasks]
285
- Workflow.load_remote_tasks(options[:remote_workflow_tasks])
286
- end
287
-
288
- # Set task
289
- namespace = nil, nil
290
-
291
- case
292
- when task.nil?
293
- usage workflow, nil, nil, options[:abridge] and exit 0
294
- else
295
- task_name = task.to_sym
296
- begin
297
- task = workflow.tasks[task_name]
298
- raise Workflow::TaskNotFoundException.new workflow, task_name if task.nil?
299
- rescue Workflow::TaskNotFoundException
300
- usage workflow, nil, nil, options[:abridge]
301
-
302
- puts
303
- puts Log.color :magenta, "## Error"
304
- puts
305
- puts $!.message
306
- puts
307
-
308
- exit -1
309
- end
310
- end
311
-
312
- usage workflow, task, nil, options[:abridge] and exit 0 if help
313
-
314
- name = options.delete(:jobname)
315
-
316
- # get job args
317
- job_options = workflow.get_SOPT(task)
318
-
319
- if options[:load_inputs]
320
- task_info = workflow.task_info(task_name)
321
- job_options = Workflow.load_inputs(options[:load_inputs], task_info[:inputs], task_info[:input_types]).merge(job_options)
322
- end
323
-
324
- job_options = fix_options(workflow, task, job_options)
325
- saved_job_options = job_options
326
-
327
- workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir))) if options[:workdir]
328
-
329
- if override_deps
330
- override_deps.split($array_separator || ",").each do |part|
331
- t_, value = part.split("=")
332
- job_options.merge!( t_ => value)
333
- end
334
- end
335
-
336
- #- get job
337
-
338
- job = workflow.job(task.name, name, job_options)
339
- $job = job
340
-
341
- # clean job
342
- if clean
343
- job.clean
344
- sleep 1
345
- end
346
-
347
- if clean_task
348
- clean_task.split(",").each do |clean_task|
349
- if clean_task.include? "#"
350
- clean_workflow, clean_task = clean_task.split("#")
351
- end
352
-
353
- job.rec_dependencies.each do |dep|
354
- next unless dep.task_name.to_s == clean_task.to_s
355
- next unless clean_workflow.nil? || clean_workflow == dep.workflow.to_s
356
- dep.clean
357
- dep.set_info :status, :cleaned
358
- end
359
-
360
- job.clean if job.task_name.to_s == clean_task.to_s
361
- end
362
- end
363
-
364
- if recursive_clean
365
- job.recursive_clean
366
- end
367
-
368
- require 'pp'
369
-
370
- # run
371
- begin
372
- if options[:info]
373
- pp job.info
374
- exit 0
375
- end
376
-
377
- if options.delete(:printname)
378
- puts job.name
379
- exit 0
380
- end
381
-
382
- if do_exec or (job.respond_to?(:is_exec) and job.is_exec)
383
- res = job.exec(:stream)
384
-
385
- result_type = job.result_type
386
-
387
- res = JSON.parse(res.read) if (defined?(RemoteStep) and RemoteStep === job) && %w(array float integer boolean).include?(result_type.to_s)
388
-
389
- case
390
- when res.respond_to?(:gets)
391
- begin
392
- Misc.consume_stream(res, false, out)
393
- rescue EOFError, IOError
394
- end
395
- res.join if res.respond_to? :join
396
- when Array === res
397
- out.puts res * "\n"
398
- when TSV === res
399
- out.puts res
400
- when Hash === res
401
- out.puts res.to_yaml
402
- when IO === res
403
- while block = res.read(2048)
404
- out.write block
405
- end
406
- else
407
- out.puts res
408
- end
409
- exit 0
410
- end
411
-
412
- if options.delete(:provenance)
413
- if options.delete(:printpath)
414
- puts job.path
415
- else
416
- puts Step.prov_report(job)
417
- end
418
- exit 0
419
- end
420
-
421
- def match_dependencies(queries, dependencies)
422
- queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
423
-
424
- matched = []
425
- queries.each do |q|
426
- matched += dependencies.select do |dep|
427
- if Array === q
428
- q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
429
- else
430
- q.to_s == dep.task_name.to_s
431
- end
432
- end
433
- end
434
-
435
- matched
436
- end
437
-
438
- def replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay = false, run_type = :run)
439
- jobs_to_relay.each do |job|
440
- ComputeDependency.setup(job, :bootstrap)
441
- next if job.done?
442
- Log.low "Relaying #{Misc.fingerprint job} to #{server}"
443
- jmeta = class << job; self; end
444
-
445
- job.instance_variable_set(:@job, job)
446
- job.instance_variable_set(:@host, server)
447
- job.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
448
-
449
- jmeta.define_method :run do |*args|
450
- if done?
451
- load
452
- else
453
- RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
454
- Step.migrate(@job, 'user', :source => @host)
455
- load
456
- end
457
- end
458
- #job.dependencies = []
459
-
460
- #([job] + job.rec_dependencies).each do |j|
461
- # next if job.done?
462
- # jmeta = class << j; self; end
463
-
464
- # j.instance_variable_set(:@job, job)
465
- # j.instance_variable_set(:@host, server)
466
- # j.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
467
-
468
- # jmeta.define_method :run do |*args|
469
- # if done?
470
- # load
471
- # else
472
- # RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
473
- # Step.migrate(@job, 'user', :source => @host)
474
- # load
475
- # end
476
- # end
477
- #end
478
- end
479
- end
480
-
481
-
482
- if server = options.delete(:relay)
483
- require 'scout/workflow/remote_workflow'
484
- relay_dependencies = options.delete(:relay_dependencies).split(",")
485
- produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
486
-
487
- jobs_to_relay = relay_dependencies ? match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
488
- jobs_to_relay.reject!{|d| d.done? }
489
-
490
- replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :run)
491
- end
492
-
493
- if server = options.delete(:slurm_relay)
494
- require 'scout/workflow/remote_workflow'
495
- relay_dependencies = options.delete(:relay_dependencies).split(",")
496
- produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
497
- jobs_to_relay = relay_dependencies ? match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
498
- jobs_to_relay.reject!{|d| d.done? }
499
-
500
- replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :orchestrate)
501
- end
502
-
503
-
504
- if options[:procpath_performance]
505
- require 'scout/util/procpath'
506
- current_pid = job.info[:pid]
507
- job.fork
508
- job.soft_grace
509
- sleep 2 if job.info[:pid] == current_pid
510
- if job.info[:pid] != current_pid
511
- pid = job.info[:pid]
512
- begin
513
- ProcPath.monitor(pid, options[:procpath_performance])
514
- rescue Errno::ECHILD
515
- Log.warn "Procpath didn't find process #{pid} to monitor. Maybe it finished already"
516
- rescue
517
- Log.warn "Procpath failed: #{$!.message}"
518
- end
519
- end
520
- end
521
-
522
- if tasks = options.delete(:produce)
523
- tasks = tasks.split(",")
524
- produce_cpus = (options[:produce_cpus] || 1)
525
- puts Step.produce_dependencies(job, tasks, produce_cpus)
526
- exit 0
527
- end
528
-
529
-
530
- if do_fork
531
- ENV["RBBT_NO_PROGRESS"] = "true"
532
- if detach
533
- job.fork
534
- Process.detach job.pid if job.pid
535
- puts Log.color(:magenta, "Issued: ") + Log.color(:magenta, job.pid ? job.pid.to_s : 'no pid') + ' -- ' + job.path
536
- exit 0
537
- end
538
-
539
- job.fork
540
- elsif options[:orchestrate]
541
- require 'scout/workflow/util/orchestrator'
542
- rules = case options[:orchestrate]
543
- when 'none', 'open', 'default'
544
- nil
545
- else
546
- YAML.parse(Open.read(options[:orchestrate]))
547
- end
548
- if rules
549
- Workflow::Orchestrator.process rules, job
550
- else
551
- Workflow::Orchestrator.process job
552
- end unless job.done?
553
- else
554
- job.run(:stream)
555
- res = job
556
- end
557
-
558
- if options.delete(:printpath)
559
- job.join if job.running?
560
- raise job.messages.last if (job.error? || job.aborted?) && job.messages
561
- if Open.remote? job.path
562
- puts job.url + Log.color(:blue, "?_format=raw")
563
- else
564
- puts job.path
565
- end
566
- exit 0
567
- end
568
-
569
- if do_fork
570
- puts
571
- space = 1
572
- Log.tty_size ||= 100
573
-
574
- while not job.done?
575
- message = (job.messages and job.messages.any?) ? job.messages.last.strip : "no message"
576
- status = job.status || "no status"
577
- if job.info and job.info.include? :issued
578
- issued = job.info[:issued]
579
- issued = Time.parse(issued) unless Time === issued
580
- time = Time.now - issued
581
- end
582
-
583
- space.times do
584
- Log.clear_line
585
- end
586
-
587
- puts "#{Log.color :blue, job.path}"
588
- str = "Waiting on #{Log.color :blue, job.info[:pid] || job.pid} (#{time ? time.to_i : '?'} sec. ago) " << [Log.color(:cyan, status.to_s),message.strip].compact*" "
589
- puts Misc.format_paragraph str, Log.tty_size
590
-
591
- space = 2 + Log.uncolor(str).length / Log.tty_size
592
- sleep 2
593
- end
594
- raise job.messages.last if job.error?
595
-
596
- if job.info and job.info.include? :issued
597
- issued = job.info[:issued]
598
- issued = Time.parse(issued) unless Time === issued
599
- time = Time.now - issued
600
- end
601
-
602
- space.times do
603
- Log.clear_line
604
- end
605
-
606
- if Open.remote?(job.path)
607
- out.puts job.path + Log.color(:blue, "?_format=raw")
608
- else
609
- out.puts job.path
610
- end
611
-
612
- exit 0
613
- end
614
- rescue ParameterException
615
- SOPT.delete_inputs(workflow.rec_inputs(task.name))
616
- usage(workflow, task, $!)
617
- puts Log.color :magenta, "Options:"
618
- puts
619
- report_options saved_job_options
620
- puts
621
- exit! -1
622
- end
623
-
624
- if options.delete(:list_job_files)
625
- out.puts job.files * "\n"
626
- exit 0
627
- end
628
-
629
- if job_file = options.delete(:job_file)
630
- job.join
631
- file = job.file(job_file)
632
- out.puts Path === file ? file.read : file
633
- exit 0
634
- end
635
-
636
- case res
637
- #when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
638
- when (defined?(RemoteStep) and RemoteStep)
639
- res = job.result
640
- if res.respond_to? :gets
641
- begin
642
- Misc.consume_stream(res, false, out)
643
- rescue EOFError, IOError
644
- end
645
- res.join if res.respond_to? :join
646
- elsif res.nil?
647
- job.join
648
- raise job.get_exception if job.error? || job.aborted?
649
- puts Open.read(job.path, :nocache => true, :nofail => true)
650
- else
651
- if Array === res
652
- out.puts res * "\n"
653
- else
654
- out.puts res.to_s
655
- end
656
- end
657
- when Step
658
- if res.streaming?
659
- io = TSV.stream res
660
- Misc.consume_stream(io, false, out)
661
- io.join if io.respond_to? :join
662
- elsif IO === res.result
663
- begin
664
- io = res.stream
665
- Misc.consume_stream(io, false, out)
666
- io.join if io.respond_to? :join
667
- rescue Aborted, Interrupt
668
- Log.error "Process interrupted. Aborting step"
669
- res.abort
670
- begin
671
- io.abort if io.respond_to? :abort
672
- io.join if io.respond_to? :join
673
- ensure
674
- exit! -1
675
- end
676
- rescue Exception
677
- Log.exception $!
678
- res.abort
679
- begin
680
- io.abort if io.respond_to? :abort
681
- io.join if io.respond_to? :join
682
- ensure
683
- exit! -1
684
- end
685
- end
686
- elsif detach
687
- exit! 0
688
- else
689
- res.join if res.running?
690
- if %w(float integer string boolean).include?(res.result_type.to_s)
691
- out.puts res.load
692
- else
693
- Open.open(res.path, :mode => 'rb') do |io|
694
- Misc.consume_stream(io, false, out)
695
- end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
696
- end if res.done?
697
- end
698
- else
699
- if Array === res
700
- out.puts res * "\n"
701
- else
702
- out.puts res.to_s
703
- end
704
- end
705
-
706
- exit 0