scout-gear 2.0.0 → 5.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +59 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +231 -24
  5. data/lib/scout/cmd.rb +344 -0
  6. data/lib/scout/concurrent_stream.rb +259 -0
  7. data/lib/scout/exceptions.rb +15 -8
  8. data/lib/scout/indiferent_hash/options.rb +8 -26
  9. data/lib/scout/log/color.rb +2 -2
  10. data/lib/scout/log/fingerprint.rb +11 -1
  11. data/lib/scout/log/progress/report.rb +0 -1
  12. data/lib/scout/log/progress/util.rb +1 -1
  13. data/lib/scout/log/progress.rb +4 -4
  14. data/lib/scout/log.rb +10 -2
  15. data/lib/scout/meta_extension.rb +15 -1
  16. data/lib/scout/misc/digest.rb +56 -0
  17. data/lib/scout/misc/filesystem.rb +26 -0
  18. data/lib/scout/misc/format.rb +1 -2
  19. data/lib/scout/misc/insist.rb +56 -0
  20. data/lib/scout/misc.rb +4 -11
  21. data/lib/scout/open/lock.rb +61 -0
  22. data/lib/scout/open/remote.rb +120 -0
  23. data/lib/scout/open/stream.rb +372 -0
  24. data/lib/scout/open/util.rb +225 -0
  25. data/lib/scout/open.rb +169 -0
  26. data/lib/scout/path/find.rb +67 -21
  27. data/lib/scout/path/tmpfile.rb +8 -0
  28. data/lib/scout/path/util.rb +14 -1
  29. data/lib/scout/path.rb +6 -30
  30. data/lib/scout/persist/open.rb +17 -0
  31. data/lib/scout/persist/path.rb +15 -0
  32. data/lib/scout/persist/serialize.rb +140 -0
  33. data/lib/scout/persist.rb +54 -0
  34. data/lib/scout/resource/path.rb +15 -0
  35. data/lib/scout/resource/produce/rake.rb +69 -0
  36. data/lib/scout/resource/produce.rb +246 -0
  37. data/lib/scout/resource/scout.rb +3 -0
  38. data/lib/scout/resource.rb +37 -0
  39. data/lib/scout/simple_opt/accessor.rb +1 -1
  40. data/lib/scout/simple_opt/doc.rb +4 -22
  41. data/lib/scout/simple_opt/parse.rb +4 -3
  42. data/lib/scout/tmpfile.rb +39 -1
  43. data/lib/scout/workflow/definition.rb +72 -0
  44. data/lib/scout/workflow/documentation.rb +77 -0
  45. data/lib/scout/workflow/step/info.rb +77 -0
  46. data/lib/scout/workflow/step.rb +96 -0
  47. data/lib/scout/workflow/task/inputs.rb +112 -0
  48. data/lib/scout/workflow/task.rb +141 -0
  49. data/lib/scout/workflow/usage.rb +294 -0
  50. data/lib/scout/workflow/util.rb +11 -0
  51. data/lib/scout/workflow.rb +39 -0
  52. data/lib/scout-gear.rb +4 -0
  53. data/lib/scout.rb +1 -0
  54. data/lib/workflow-scout.rb +2 -0
  55. data/scout-gear.gemspec +66 -5
  56. data/scout_commands/alias +48 -0
  57. data/scout_commands/find +83 -0
  58. data/scout_commands/glob +0 -0
  59. data/scout_commands/rbbt +23 -0
  60. data/scout_commands/workflow/task +707 -0
  61. data/test/scout/indiferent_hash/test_options.rb +11 -1
  62. data/test/scout/misc/test_digest.rb +30 -0
  63. data/test/scout/misc/test_filesystem.rb +30 -0
  64. data/test/scout/misc/test_insist.rb +13 -0
  65. data/test/scout/open/test_lock.rb +52 -0
  66. data/test/scout/open/test_remote.rb +25 -0
  67. data/test/scout/open/test_stream.rb +515 -0
  68. data/test/scout/open/test_util.rb +73 -0
  69. data/test/scout/path/test_find.rb +28 -0
  70. data/test/scout/persist/test_open.rb +37 -0
  71. data/test/scout/persist/test_path.rb +37 -0
  72. data/test/scout/persist/test_serialize.rb +114 -0
  73. data/test/scout/resource/test_path.rb +40 -0
  74. data/test/scout/resource/test_produce.rb +62 -0
  75. data/test/scout/test_cmd.rb +85 -0
  76. data/test/scout/test_concurrent_stream.rb +29 -0
  77. data/test/scout/test_misc.rb +0 -7
  78. data/test/scout/test_open.rb +146 -0
  79. data/test/scout/test_path.rb +3 -1
  80. data/test/scout/test_persist.rb +83 -0
  81. data/test/scout/test_resource.rb +26 -0
  82. data/test/scout/test_workflow.rb +87 -0
  83. data/test/scout/workflow/step/test_info.rb +28 -0
  84. data/test/scout/workflow/task/test_inputs.rb +182 -0
  85. data/test/scout/workflow/test_step.rb +36 -0
  86. data/test/scout/workflow/test_task.rb +178 -0
  87. data/test/scout/workflow/test_usage.rb +26 -0
  88. data/test/scout/workflow/test_util.rb +17 -0
  89. data/test/test_helper.rb +17 -0
  90. data/test/test_scout-gear.rb +0 -0
  91. metadata +64 -3
@@ -0,0 +1,707 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout-gear'
4
+ require 'workflow-scout'
5
+
6
+ def report_options(options)
7
+ if options.nil? or options.empty?
8
+ puts "No options"
9
+ else
10
+ options.each do |key, value|
11
+ puts [Log.color(:cyan, key), Misc.fingerprint(value)] * ": "
12
+ end
13
+ end
14
+ end
15
+
16
+ def usage(workflow = nil, task = nil, exception=nil, abridge = false)
17
+ puts SOPT.doc
18
+ puts
19
+ if workflow.nil?
20
+ puts "No workflow specified. Use `scout workflow list` to list available workflows."
21
+ exit! -1
22
+ end
23
+
24
+ if task.nil?
25
+ workflow.load_tasks if workflow.respond_to? :load_tasks
26
+ workflow.doc nil, abridge
27
+ puts
28
+ puts "E.g. scout workflow task #{workflow.to_s} #{workflow.tasks.keys.first.to_s} -h"
29
+ else
30
+ puts Log.color :magenta, workflow.to_s
31
+ puts Log.color :magenta, "=" * workflow.to_s.length
32
+ if workflow.documentation[:title] and not workflow.documentation[:title].empty?
33
+ puts
34
+ puts Misc.format_paragraph workflow.documentation[:title]
35
+ end
36
+ if workflow.documentation[:description] and not workflow.documentation[:description].empty?
37
+ puts
38
+ puts Misc.format_paragraph workflow.documentation[:description]
39
+ end
40
+ puts
41
+ workflow.doc(task, abridge)
42
+ end
43
+
44
+ print_error(exception.message, exception.backtrace) if exception
45
+
46
+ true
47
+ end
48
+
49
+ def get_value_stream(value)
50
+ if value == "-"
51
+ io = Misc.open_pipe do |sin|
52
+ while not STDIN.eof?
53
+ sin.write STDIN.read(2048)
54
+ end
55
+ sin.close
56
+ end
57
+ else
58
+ io = Open.open(value)
59
+ end
60
+ class << io
61
+ attr_accessor :filename
62
+ end
63
+ io.filename = value
64
+ io
65
+ end
66
+
67
+ def fix_options(workflow, task, job_options)
68
+ input_types = IndiferentHash.setup workflow.rec_input_types(task.name)
69
+ input_options = IndiferentHash.setup workflow.rec_input_options(task.name)
70
+
71
+ job_options_cleaned = {}
72
+
73
+ job_options.each do |name, value|
74
+ type = input_types[name]
75
+ type = type.to_sym if type
76
+
77
+ if Step === value
78
+ job_options_cleaned[name] = value
79
+ next
80
+ end
81
+
82
+ if Path === value && Step === value.resource
83
+ job_options_cleaned[name] = value
84
+ next
85
+ end
86
+
87
+ value = case type
88
+ when nil
89
+ value
90
+ when :boolean
91
+ TrueClass === value or %w(true TRUE T yes).include? value
92
+ when :float
93
+ value.to_f
94
+ when :path
95
+ Path.setup(value)
96
+ when :integer
97
+ value.to_i
98
+ when :text
99
+ if input_options[name] and input_options[name][:stream] and String === value
100
+ get_value_stream(value)
101
+ else
102
+ case
103
+ when value == '-'
104
+ STDIN.read
105
+ when (String === value and File.exist?(value) and not File.directory?(value))
106
+ Open.read(value)
107
+ else
108
+ value
109
+ end
110
+ end
111
+ when :array
112
+ if input_options[name] && input_options[name][:stream] && String === value && Misc.is_filename?(value) && !! input_options[name][:nofile]
113
+ get_value_stream(value)
114
+ elsif input_options[name] and input_options[name][:stream] and value == "-"
115
+ STDIN
116
+ else
117
+ if Array === value || IO === value
118
+ value
119
+ else
120
+ array_separator = $array_separator
121
+ str = case
122
+ when value == '-'
123
+ array_separator ||= "\n"
124
+ STDIN.read
125
+ when (String === value and File.exist?(value) and not (input_options[name] && input_options[name][:nofile]))
126
+ array_separator ||= "\n"
127
+ Open.read(value)
128
+ else
129
+ value
130
+ end
131
+
132
+ if array_separator
133
+ str.split(/#{array_separator}/)
134
+ else
135
+ str.split(/[,|\s]/)
136
+ end
137
+ end
138
+ end
139
+ when :tsv
140
+ if input_options[name] and input_options[name][:stream] and String === value
141
+ TSV::Parser.new(value == '-' ? STDIN : Open.open(value), :filename => value )
142
+ else
143
+ case value
144
+ when TSV
145
+ value
146
+ when '-'
147
+ TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
148
+ when (Misc.is_filename?(value) and String)
149
+ TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
150
+ when IO
151
+ TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
152
+ else
153
+ TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
154
+ end
155
+ end
156
+ when :directory
157
+ Path.setup(File.expand_path(value))
158
+ else
159
+ value
160
+ end
161
+
162
+ job_options_cleaned[name] = value
163
+ end
164
+
165
+ job_options_cleaned
166
+ end
167
+
168
+ options = SOPT.setup <<EOF
169
+ Enact workflow tasks
170
+
171
+ $ scout workflow task <workflow> [<task>] [<options>]
172
+
173
+ Examine workflows and enact tasks from them. If no `task` is specified, a list
174
+ of available tasks is shown. If a `task` is given it will be enacted with the
175
+ parameters specified in `options`. Use *-h* option to display the description
176
+ of a task, including the parameters it accepts; and some examples, if
177
+ available. Examples can be enacted using `scout workflow example <workflow>
178
+ [<task>] [<example>]`.
179
+
180
+ When a task is enacted a job is instantiated. This job is identified by the
181
+ `jobname` (which is *#{Task::DEFAULT_NAME}* unless specified otherwise) and the values of the
182
+ parameters; these two things determine the filename under which the job result
183
+ will be saved. If the taks is enacted using the same `jobname` and parameters
184
+ it will result in the same job, pointing to the same result file.
185
+
186
+ The first time a job is executed it will save the result. The saved result will
187
+ be returned directly if the same task is re-enacted. Once the job is done you
188
+ can redo it using the `clean` parameter, this cleans the last step of the task.
189
+ The `recursive_clean` cleans all the job dependency steps recursively.
190
+
191
+ -h--help Show this help
192
+ -ha--abridge Abridge help
193
+ -wd--workdir* Change the working directory of the workflow
194
+ -wda--workdir_all* Change the working directory of ALL workflow
195
+ -as--array_separator* Change the character that separates elements of Arrays, ',', '|', or '\\n' by default
196
+ -fs--field_separator* Change the character that separates fields of TSV files '\\t' by default
197
+ -jn--jobname* Job name to use. The name '#{Task::DEFAULT_NAME}' is used by default
198
+ -pn--printname Print the name of the job and exit without starting it
199
+ -pf--printpath Print the path of the job result
200
+ -cl--clean Clean the last step of the job so that it gets recomputed
201
+ -ct--clean_task* Clean a particular dependency task
202
+ -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
203
+ -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
204
+ --fork Run job asyncronously and monitor progress. It monitors detached processes as well
205
+ --orchestrate* Run the job through the orchestrator
206
+ --detach Run job asyncronously and detach process
207
+ --exec Run job with no persistence
208
+ -O--output* Save job result into file
209
+ -jf--job_file* Output one of the job produced files
210
+ -ljf--list_job_files List all the files produced in that step
211
+ --load_inputs* Load inputs from a directory
212
+ --info Show the job info
213
+ -prov--provenance Report the jobs provenance
214
+ -W--workflows* Load a list of workflows
215
+ -R--requires* Require a list of files
216
+ -pro--produce* Prepare dependencies
217
+ -proc--produce_cpus* Number of dependencies prepared in parallel
218
+ -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
219
+ -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
220
+ -PERF--procpath_performance* Measure performance using procpath
221
+ -r--relay* Relay job to SSH server
222
+ -sr--slurm_relay* Relay job to SSH SLURM server
223
+ -rdep--relay_dependencies* Relay dependencies instead of main job
224
+ -pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
225
+ EOF
226
+
227
+ workflow = ARGV.shift
228
+ usage and exit! -1 if workflow.nil?
229
+
230
+ task = ARGV.shift
231
+
232
+ # Set log, fork, clean, recursive_clean and help
233
+ help = !!options.delete(:help)
234
+ do_fork = !!options.delete(:fork)
235
+ detach = !!options.delete(:detach)
236
+ do_exec = !!options.delete(:exec)
237
+ clean_task = options.delete(:clean_task)
238
+ clean = !!options.delete(:clean) || clean_task
239
+ override_deps = options.delete(:override_deps)
240
+ recursive_clean = !!options.delete(:recursive_clean)
241
+ out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT
242
+
243
+ $array_separator = options.delete(:array_separator)
244
+ $field_separator = options.delete(:field_separator) || "\t"
245
+
246
+ # Get workflow
247
+
248
+ if Scout.etc.remote_workflows.exists?
249
+ remote_workflows = Scout.etc.remote_workflows.yaml
250
+ else
251
+ remote_workflows = {}
252
+ end
253
+
254
+ #Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
255
+ Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
256
+
257
+ workflow = Workflow.require_workflow workflow
258
+
259
+ if clean_task
260
+ ENV["RBBT_UPDATE"] = 'true'
261
+ end
262
+
263
+ if options[:update_all_jobs]
264
+ ENV["RBBT_UPDATE_ALL_JOBS"] = 'true'
265
+ ENV["RBBT_UPDATE"] = 'true'
266
+ end
267
+
268
+ if options[:workflows]
269
+ require 'scout/workflow'
270
+ workflows = options[:workflows].split(',')
271
+ workflows.each do |workflow|
272
+ workflow.strip!
273
+ Workflow.require_workflow workflow
274
+ end
275
+ end
276
+
277
+ if options[:requires]
278
+ requires = options[:requires].split(',')
279
+ requires.each do |req|
280
+ req.strip!
281
+ require req
282
+ end
283
+ end
284
+
285
+ if options[:remote_workflow_tasks]
286
+ Workflow.load_remote_tasks(options[:remote_workflow_tasks])
287
+ end
288
+
289
+ # Set task
290
+ namespace = nil, nil
291
+
292
+ case
293
+ when task.nil?
294
+ usage workflow, nil, nil, options[:abridge] and exit 0
295
+ else
296
+ task_name = task.to_sym
297
+ begin
298
+ task = workflow.tasks[task_name]
299
+ raise Workflow::TaskNotFoundException.new workflow, task_name if task.nil?
300
+ rescue Workflow::TaskNotFoundException
301
+ usage workflow, nil, nil, options[:abridge]
302
+
303
+ puts
304
+ puts Log.color :magenta, "## Error"
305
+ puts
306
+ puts $!.message
307
+ puts
308
+
309
+ exit -1
310
+ end
311
+ end
312
+
313
+ usage workflow, task, nil, options[:abridge] and exit 0 if help
314
+
315
+ name = options.delete(:jobname)
316
+
317
+ # get job args
318
+ job_options = workflow.get_SOPT(task)
319
+
320
+ if options[:load_inputs]
321
+ task_info = workflow.task_info(task_name)
322
+ job_options = Workflow.load_inputs(options[:load_inputs], task_info[:inputs], task_info[:input_types]).merge(job_options)
323
+ end
324
+
325
+ job_options = fix_options(workflow, task, job_options)
326
+ saved_job_options = job_options
327
+
328
+ workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir))) if options[:workdir]
329
+
330
+ if override_deps
331
+ override_deps.split($array_separator || ",").each do |part|
332
+ t_, value = part.split("=")
333
+ job_options.merge!( t_ => value)
334
+ end
335
+ end
336
+
337
+ #- get job
338
+
339
+ job = workflow.job(task.name, name, job_options)
340
+ $job = job
341
+
342
+ # clean job
343
+ if clean
344
+ job.clean
345
+ sleep 1
346
+ end
347
+
348
+ if clean_task
349
+ clean_task.split(",").each do |clean_task|
350
+ if clean_task.include? "#"
351
+ clean_workflow, clean_task = clean_task.split("#")
352
+ end
353
+
354
+ job.rec_dependencies.each do |dep|
355
+ next unless dep.task_name.to_s == clean_task.to_s
356
+ next unless clean_workflow.nil? || clean_workflow == dep.workflow.to_s
357
+ dep.clean
358
+ dep.set_info :status, :cleaned
359
+ end
360
+
361
+ job.clean if job.task_name.to_s == clean_task.to_s
362
+ end
363
+ end
364
+
365
+ if recursive_clean
366
+ job.recursive_clean
367
+ end
368
+
369
+ require 'pp'
370
+
371
+ # run
372
+ begin
373
+ if options[:info]
374
+ pp job.info
375
+ exit 0
376
+ end
377
+
378
+ if options.delete(:printname)
379
+ puts job.name
380
+ exit 0
381
+ end
382
+
383
+ if do_exec or (job.respond_to?(:is_exec) and job.is_exec)
384
+ res = job.exec(:stream)
385
+
386
+ result_type = job.result_type
387
+
388
+ res = JSON.parse(res.read) if (defined?(RemoteStep) and RemoteStep === job) && %w(array float integer boolean).include?(result_type.to_s)
389
+
390
+ case
391
+ when res.respond_to?(:gets)
392
+ begin
393
+ Misc.consume_stream(res, false, out)
394
+ rescue EOFError, IOError
395
+ end
396
+ res.join if res.respond_to? :join
397
+ when Array === res
398
+ out.puts res * "\n"
399
+ when TSV === res
400
+ out.puts res
401
+ when Hash === res
402
+ out.puts res.to_yaml
403
+ when IO === res
404
+ while block = res.read(2048)
405
+ out.write block
406
+ end
407
+ else
408
+ out.puts res
409
+ end
410
+ exit 0
411
+ end
412
+
413
+ if options.delete(:provenance)
414
+ if options.delete(:printpath)
415
+ puts job.path
416
+ else
417
+ puts Step.prov_report(job)
418
+ end
419
+ exit 0
420
+ end
421
+
422
+ def match_dependencies(queries, dependencies)
423
+ queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
424
+
425
+ matched = []
426
+ queries.each do |q|
427
+ matched += dependencies.select do |dep|
428
+ if Array === q
429
+ q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
430
+ else
431
+ q.to_s == dep.task_name.to_s
432
+ end
433
+ end
434
+ end
435
+
436
+ matched
437
+ end
438
+
439
+ def replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay = false, run_type = :run)
440
+ jobs_to_relay.each do |job|
441
+ ComputeDependency.setup(job, :bootstrap)
442
+ next if job.done?
443
+ Log.low "Relaying #{Misc.fingerprint job} to #{server}"
444
+ jmeta = class << job; self; end
445
+
446
+ job.instance_variable_set(:@job, job)
447
+ job.instance_variable_set(:@host, server)
448
+ job.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
449
+
450
+ jmeta.define_method :run do |*args|
451
+ if done?
452
+ load
453
+ else
454
+ RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
455
+ Step.migrate(@job, 'user', :source => @host)
456
+ load
457
+ end
458
+ end
459
+ #job.dependencies = []
460
+
461
+ #([job] + job.rec_dependencies).each do |j|
462
+ # next if job.done?
463
+ # jmeta = class << j; self; end
464
+
465
+ # j.instance_variable_set(:@job, job)
466
+ # j.instance_variable_set(:@host, server)
467
+ # j.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
468
+
469
+ # jmeta.define_method :run do |*args|
470
+ # if done?
471
+ # load
472
+ # else
473
+ # RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
474
+ # Step.migrate(@job, 'user', :source => @host)
475
+ # load
476
+ # end
477
+ # end
478
+ #end
479
+ end
480
+ end
481
+
482
+
483
+ if server = options.delete(:relay)
484
+ require 'scout/workflow/remote_workflow'
485
+ relay_dependencies = options.delete(:relay_dependencies).split(",")
486
+ produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
487
+
488
+ jobs_to_relay = relay_dependencies ? match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
489
+ jobs_to_relay.reject!{|d| d.done? }
490
+
491
+ replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :run)
492
+ end
493
+
494
+ if server = options.delete(:slurm_relay)
495
+ require 'scout/workflow/remote_workflow'
496
+ relay_dependencies = options.delete(:relay_dependencies).split(",")
497
+ produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
498
+ jobs_to_relay = relay_dependencies ? match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
499
+ jobs_to_relay.reject!{|d| d.done? }
500
+
501
+ replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :orchestrate)
502
+ end
503
+
504
+
505
+ if options[:procpath_performance]
506
+ require 'scout/util/procpath'
507
+ current_pid = job.info[:pid]
508
+ job.fork
509
+ job.soft_grace
510
+ sleep 2 if job.info[:pid] == current_pid
511
+ if job.info[:pid] != current_pid
512
+ pid = job.info[:pid]
513
+ begin
514
+ ProcPath.monitor(pid, options[:procpath_performance])
515
+ rescue Errno::ECHILD
516
+ Log.warn "Procpath didn't find process #{pid} to monitor. Maybe it finished already"
517
+ rescue
518
+ Log.warn "Procpath failed: #{$!.message}"
519
+ end
520
+ end
521
+ end
522
+
523
+ if tasks = options.delete(:produce)
524
+ tasks = tasks.split(",")
525
+ produce_cpus = (options[:produce_cpus] || 1)
526
+ puts Step.produce_dependencies(job, tasks, produce_cpus)
527
+ exit 0
528
+ end
529
+
530
+
531
+ if do_fork
532
+ ENV["RBBT_NO_PROGRESS"] = "true"
533
+ if detach
534
+ job.fork
535
+ Process.detach job.pid if job.pid
536
+ puts Log.color(:magenta, "Issued: ") + Log.color(:magenta, job.pid ? job.pid.to_s : 'no pid') + ' -- ' + job.path
537
+ exit 0
538
+ end
539
+
540
+ job.fork
541
+ elsif options[:orchestrate]
542
+ require 'scout/workflow/util/orchestrator'
543
+ rules = case options[:orchestrate]
544
+ when 'none', 'open', 'default'
545
+ nil
546
+ else
547
+ YAML.parse(Open.read(options[:orchestrate]))
548
+ end
549
+ if rules
550
+ Workflow::Orchestrator.process rules, job
551
+ else
552
+ Workflow::Orchestrator.process job
553
+ end unless job.done?
554
+ else
555
+ job.run(:stream)
556
+ res = job
557
+ end
558
+
559
+ if options.delete(:printpath)
560
+ job.join if job.running?
561
+ raise job.messages.last if (job.error? || job.aborted?) && job.messages
562
+ if Open.remote? job.path
563
+ puts job.url + Log.color(:blue, "?_format=raw")
564
+ else
565
+ puts job.path
566
+ end
567
+ exit 0
568
+ end
569
+
570
+ if do_fork
571
+ puts
572
+ space = 1
573
+ Log.tty_size ||= 100
574
+
575
+ while not job.done?
576
+ message = (job.messages and job.messages.any?) ? job.messages.last.strip : "no message"
577
+ status = job.status || "no status"
578
+ if job.info and job.info.include? :issued
579
+ issued = job.info[:issued]
580
+ issued = Time.parse(issued) unless Time === issued
581
+ time = Time.now - issued
582
+ end
583
+
584
+ space.times do
585
+ Log.clear_line
586
+ end
587
+
588
+ puts "#{Log.color :blue, job.path}"
589
+ str = "Waiting on #{Log.color :blue, job.info[:pid] || job.pid} (#{time ? time.to_i : '?'} sec. ago) " << [Log.color(:cyan, status.to_s),message.strip].compact*" "
590
+ puts Misc.format_paragraph str, Log.tty_size
591
+
592
+ space = 2 + Log.uncolor(str).length / Log.tty_size
593
+ sleep 2
594
+ end
595
+ raise job.messages.last if job.error?
596
+
597
+ if job.info and job.info.include? :issued
598
+ issued = job.info[:issued]
599
+ issued = Time.parse(issued) unless Time === issued
600
+ time = Time.now - issued
601
+ end
602
+
603
+ space.times do
604
+ Log.clear_line
605
+ end
606
+
607
+ if Open.remote?(job.path)
608
+ out.puts job.path + Log.color(:blue, "?_format=raw")
609
+ else
610
+ out.puts job.path
611
+ end
612
+
613
+ exit 0
614
+ end
615
+ rescue ParameterException
616
+ SOPT.delete_inputs(workflow.rec_inputs(task.name))
617
+ usage(workflow, task, $!)
618
+ puts Log.color :magenta, "Options:"
619
+ puts
620
+ report_options saved_job_options
621
+ puts
622
+ exit! -1
623
+ end
624
+
625
+ if options.delete(:list_job_files)
626
+ out.puts job.files * "\n"
627
+ exit 0
628
+ end
629
+
630
+ if job_file = options.delete(:job_file)
631
+ job.join
632
+ file = job.file(job_file)
633
+ out.puts Path === file ? file.read : file
634
+ exit 0
635
+ end
636
+
637
+ case res
638
+ #when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
639
+ when (defined?(RemoteStep) and RemoteStep)
640
+ res = job.result
641
+ if res.respond_to? :gets
642
+ begin
643
+ Misc.consume_stream(res, false, out)
644
+ rescue EOFError, IOError
645
+ end
646
+ res.join if res.respond_to? :join
647
+ elsif res.nil?
648
+ job.join
649
+ raise job.get_exception if job.error? || job.aborted?
650
+ puts Open.read(job.path, :nocache => true, :nofail => true)
651
+ else
652
+ if Array === res
653
+ out.puts res * "\n"
654
+ else
655
+ out.puts res.to_s
656
+ end
657
+ end
658
+ when Step
659
+ if res.streaming?
660
+ io = TSV.get_stream res
661
+ Misc.consume_stream(io, false, out)
662
+ io.join if io.respond_to? :join
663
+ elsif IO === res.result
664
+ begin
665
+ io = res.get_stream
666
+ Misc.consume_stream(io, false, out)
667
+ io.join if io.respond_to? :join
668
+ rescue Aborted, Interrupt
669
+ Log.error "Process interrupted. Aborting step"
670
+ res.abort
671
+ begin
672
+ io.abort if io.respond_to? :abort
673
+ io.join if io.respond_to? :join
674
+ ensure
675
+ exit! -1
676
+ end
677
+ rescue Exception
678
+ Log.exception $!
679
+ res.abort
680
+ begin
681
+ io.abort if io.respond_to? :abort
682
+ io.join if io.respond_to? :join
683
+ ensure
684
+ exit! -1
685
+ end
686
+ end
687
+ elsif detach
688
+ exit! 0
689
+ else
690
+ res.join if res.running?
691
+ if %w(float integer string boolean).include?(res.result_type.to_s)
692
+ out.puts res.load
693
+ else
694
+ Open.open(res.path, :mode => 'rb') do |io|
695
+ Misc.consume_stream(io, false, out)
696
+ end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
697
+ end if res.done?
698
+ end
699
+ else
700
+ if Array === res
701
+ out.puts res * "\n"
702
+ else
703
+ out.puts res.to_s
704
+ end
705
+ end
706
+
707
+ exit 0