scout-gear 10.11.6 → 10.11.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +16 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +10 -10
  5. data/lib/scout/association/fields.rb +15 -15
  6. data/lib/scout/association/index.rb +6 -6
  7. data/lib/scout/association/item.rb +18 -8
  8. data/lib/scout/association.rb +4 -4
  9. data/lib/scout/entity/identifiers.rb +5 -5
  10. data/lib/scout/entity/property.rb +2 -2
  11. data/lib/scout/entity.rb +1 -1
  12. data/lib/scout/knowledge_base/description.rb +10 -10
  13. data/lib/scout/knowledge_base/entity.rb +6 -6
  14. data/lib/scout/knowledge_base/list.rb +1 -1
  15. data/lib/scout/knowledge_base/query.rb +4 -4
  16. data/lib/scout/knowledge_base/registry.rb +6 -6
  17. data/lib/scout/knowledge_base/traverse.rb +7 -40
  18. data/lib/scout/persist/engine/fix_width_table.rb +6 -6
  19. data/lib/scout/persist/engine/packed_index.rb +2 -2
  20. data/lib/scout/persist/engine/sharder.rb +4 -4
  21. data/lib/scout/persist/engine/tkrzw.rb +1 -1
  22. data/lib/scout/persist/engine/tokyocabinet.rb +2 -2
  23. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +1 -1
  24. data/lib/scout/persist/tsv/adapter/packed_index.rb +1 -1
  25. data/lib/scout/persist/tsv/adapter/tkrzw.rb +1 -1
  26. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +3 -3
  27. data/lib/scout/persist/tsv/serialize.rb +3 -3
  28. data/lib/scout/persist/tsv.rb +1 -1
  29. data/lib/scout/semaphore.rb +78 -3
  30. data/lib/scout/tsv/annotation/repo.rb +4 -4
  31. data/lib/scout/tsv/annotation.rb +2 -2
  32. data/lib/scout/tsv/attach.rb +7 -7
  33. data/lib/scout/tsv/change_id/translate.rb +1 -1
  34. data/lib/scout/tsv/csv.rb +3 -3
  35. data/lib/scout/tsv/dumper.rb +8 -8
  36. data/lib/scout/tsv/index.rb +1 -1
  37. data/lib/scout/tsv/open.rb +3 -3
  38. data/lib/scout/tsv/stream.rb +2 -2
  39. data/lib/scout/tsv/traverse.rb +4 -4
  40. data/lib/scout/tsv/util/filter.rb +9 -9
  41. data/lib/scout/tsv/util/process.rb +2 -2
  42. data/lib/scout/tsv/util/reorder.rb +2 -2
  43. data/lib/scout/tsv/util/select.rb +3 -3
  44. data/lib/scout/tsv/util/unzip.rb +2 -2
  45. data/lib/scout/tsv/util.rb +1 -1
  46. data/lib/scout/tsv.rb +2 -2
  47. data/lib/scout/work_queue/socket.rb +2 -2
  48. data/lib/scout/work_queue/worker.rb +4 -4
  49. data/lib/scout/work_queue.rb +5 -5
  50. data/lib/scout/workflow/definition.rb +18 -16
  51. data/lib/scout/workflow/deployment/local.rb +81 -62
  52. data/lib/scout/workflow/deployment/orchestrator/batches.rb +66 -5
  53. data/lib/scout/workflow/deployment/orchestrator/chains.rb +47 -30
  54. data/lib/scout/workflow/deployment/orchestrator/rules.rb +3 -3
  55. data/lib/scout/workflow/deployment/orchestrator/workload.rb +11 -22
  56. data/lib/scout/workflow/deployment/scheduler/job.rb +34 -36
  57. data/lib/scout/workflow/deployment/scheduler/lfs.rb +1 -1
  58. data/lib/scout/workflow/deployment/scheduler/pbs.rb +4 -4
  59. data/lib/scout/workflow/deployment/scheduler/slurm.rb +2 -2
  60. data/lib/scout/workflow/deployment/scheduler.rb +23 -12
  61. data/lib/scout/workflow/deployment/trace.rb +2 -2
  62. data/lib/scout/workflow/documentation.rb +4 -4
  63. data/lib/scout/workflow/export.rb +1 -1
  64. data/lib/scout/workflow/path.rb +2 -2
  65. data/lib/scout/workflow/step/children.rb +1 -1
  66. data/lib/scout/workflow/step/dependencies.rb +36 -3
  67. data/lib/scout/workflow/step/info.rb +5 -19
  68. data/lib/scout/workflow/step/inputs.rb +1 -1
  69. data/lib/scout/workflow/step/progress.rb +2 -2
  70. data/lib/scout/workflow/step/provenance.rb +4 -4
  71. data/lib/scout/workflow/step/status.rb +23 -9
  72. data/lib/scout/workflow/step.rb +19 -17
  73. data/lib/scout/workflow/task/dependencies.rb +10 -3
  74. data/lib/scout/workflow/task/info.rb +3 -3
  75. data/lib/scout/workflow/task/inputs.rb +8 -8
  76. data/lib/scout/workflow/task.rb +37 -22
  77. data/lib/scout/workflow/usage.rb +13 -13
  78. data/lib/scout/workflow/util.rb +1 -1
  79. data/lib/scout/workflow.rb +6 -6
  80. data/scout-gear.gemspec +3 -3
  81. data/scout_commands/alias +1 -1
  82. data/scout_commands/batch/clean +12 -12
  83. data/scout_commands/batch/list +26 -25
  84. data/scout_commands/batch/tail +9 -5
  85. data/scout_commands/cat +1 -1
  86. data/scout_commands/doc +2 -2
  87. data/scout_commands/entity +4 -4
  88. data/scout_commands/find +1 -1
  89. data/scout_commands/kb/config +1 -1
  90. data/scout_commands/kb/entities +1 -1
  91. data/scout_commands/kb/list +1 -1
  92. data/scout_commands/kb/query +2 -2
  93. data/scout_commands/kb/register +1 -1
  94. data/scout_commands/kb/show +1 -1
  95. data/scout_commands/kb/traverse +1 -1
  96. data/scout_commands/log +6 -6
  97. data/scout_commands/resource/produce +2 -2
  98. data/scout_commands/resource/sync +1 -1
  99. data/scout_commands/system/clean +7 -7
  100. data/scout_commands/system/status +4 -4
  101. data/scout_commands/template +1 -1
  102. data/scout_commands/update +1 -1
  103. data/scout_commands/workflow/info +1 -1
  104. data/scout_commands/workflow/install +1 -1
  105. data/scout_commands/workflow/list +2 -2
  106. data/scout_commands/workflow/process +2 -2
  107. data/scout_commands/workflow/prov +3 -3
  108. data/scout_commands/workflow/task +36 -11
  109. data/scout_commands/workflow/trace +1 -1
  110. data/scout_commands/workflow/write_info +2 -2
  111. data/share/templates/command +1 -1
  112. data/test/scout/association/test_item.rb +5 -0
  113. data/test/scout/entity/test_property.rb +3 -3
  114. data/test/scout/knowledge_base/test_description.rb +1 -1
  115. data/test/scout/knowledge_base/test_traverse.rb +2 -2
  116. data/test/scout/persist/engine/test_packed_index.rb +6 -6
  117. data/test/scout/persist/test_tsv.rb +4 -4
  118. data/test/scout/persist/tsv/adapter/test_packed_index.rb +4 -4
  119. data/test/scout/persist/tsv/adapter/test_sharder.rb +23 -23
  120. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +1 -1
  121. data/test/scout/persist/tsv/test_serialize.rb +1 -1
  122. data/test/scout/test_association.rb +1 -1
  123. data/test/scout/test_tsv.rb +2 -2
  124. data/test/scout/test_workflow.rb +2 -2
  125. data/test/scout/tsv/test_annotation.rb +4 -4
  126. data/test/scout/tsv/test_index.rb +1 -1
  127. data/test/scout/tsv/test_open.rb +2 -2
  128. data/test/scout/tsv/test_parser.rb +2 -2
  129. data/test/scout/tsv/test_stream.rb +1 -1
  130. data/test/scout/tsv/test_transformer.rb +1 -1
  131. data/test/scout/tsv/util/test_filter.rb +1 -1
  132. data/test/scout/tsv/util/test_melt.rb +1 -1
  133. data/test/scout/tsv/util/test_reorder.rb +1 -1
  134. data/test/scout/work_queue/test_socket.rb +3 -3
  135. data/test/scout/work_queue/test_worker.rb +2 -2
  136. data/test/scout/workflow/deployment/orchestrator/test_batches.rb +13 -3
  137. data/test/scout/workflow/deployment/orchestrator/test_chains.rb +15 -13
  138. data/test/scout/workflow/deployment/orchestrator/test_workload.rb +1 -1
  139. data/test/scout/workflow/deployment/test_local.rb +2 -2
  140. data/test/scout/workflow/deployment/test_scheduler.rb +1 -2
  141. data/test/scout/workflow/step/test_children.rb +1 -1
  142. data/test/scout/workflow/step/test_dependencies.rb +36 -1
  143. data/test/scout/workflow/step/test_info.rb +3 -35
  144. data/test/scout/workflow/step/test_load.rb +1 -1
  145. data/test/scout/workflow/step/test_provenance.rb +1 -1
  146. data/test/scout/workflow/step/test_status.rb +33 -1
  147. data/test/scout/workflow/task/test_dependencies.rb +9 -7
  148. data/test/scout/workflow/task/test_inputs.rb +1 -1
  149. data/test/scout/workflow/test_definition.rb +1 -1
  150. data/test/scout/workflow/test_documentation.rb +1 -1
  151. data/test/scout/workflow/test_entity.rb +2 -2
  152. data/test/scout/workflow/test_step.rb +13 -13
  153. data/test/scout/workflow/test_usage.rb +1 -1
  154. data/test/test_helper.rb +1 -1
  155. metadata +2 -2
data/scout_commands/alias CHANGED
@@ -17,7 +17,7 @@ earlier, like --dev or --log. Edit #{Scout.ect.cmd_alias.find} to fine tune.
17
17
  EOF
18
18
  if options[:help]
19
19
  if defined? scout_usage
20
- scout_usage
20
+ scout_usage
21
21
  else
22
22
  puts SOPT.doc
23
23
  end
@@ -25,7 +25,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
25
25
  EOF
26
26
  if options[:help]
27
27
  if defined? scout_usage
28
- scout_usage
28
+ scout_usage
29
29
  else
30
30
  puts SOPT.doc
31
31
  end
@@ -109,7 +109,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
109
109
  end
110
110
 
111
111
  if File.exist?(fstatus = File.join(dir, 'job.status'))
112
- fstatus_txt = Open.read(fstatus)
112
+ fstatus_txt = Open.read(fstatus)
113
113
  begin
114
114
  if job_batch_system == "lsf"
115
115
  nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
@@ -140,28 +140,28 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
140
140
  aborted = error = true if ! done && aborted.nil? && error.nil?
141
141
  #if done || error || aborted || running || queued || jobid || search
142
142
  # select = false
143
- # select = true if done && exit_status && exit_status.to_i == 0
144
- # select = true if error && exit_status && exit_status.to_i != 0
143
+ # select = true if done && exit_status && exit_status.to_i == 0
144
+ # select = true if error && exit_status && exit_status.to_i != 0
145
145
  # select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
146
- # select = select && jobid.split(",").include?(id) if jobid
146
+ # select = select && jobid.split(",").include?(id) if jobid
147
147
  # select = select && cmd.match(/#{search}/) if search
148
- # next unless select
148
+ # next unless select
149
149
  #end
150
150
 
151
- if done || error || aborted || queued || jobid
151
+ if done || error || aborted || queued || jobid
152
152
  select = false
153
153
  select = true if done && exit_status == 0
154
- select = true if error && exit_status && exit_status != 0
154
+ select = true if error && exit_status && exit_status != 0
155
155
  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
156
156
  is_running = exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
157
157
  select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
158
158
  select = true if jobid && jobid.split(",").include?(id)
159
159
  select = select && cmd.match(/#{search}/) if search
160
- next unless select
160
+ next unless select
161
161
  elsif search
162
162
  select = false
163
163
  select = true if search && cmd.match(/#{search}/)
164
- next unless select
164
+ next unless select
165
165
  end
166
166
 
167
167
 
@@ -180,7 +180,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
180
180
 
181
181
  if options[:batch_parameters]
182
182
  puts Log.color(:magenta, "BATCH parameters: ")
183
- case job_batch_system
183
+ case job_batch_system
184
184
  when 'slurm'
185
185
  puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
186
186
  when 'lsf'
@@ -206,6 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
206
206
  Open.rm_rf dir unless dry_run
207
207
  end
208
208
 
209
- puts
209
+ puts
210
210
  puts "Found #{count} jobs"
211
211
 
@@ -12,7 +12,7 @@ List all batch jobs
12
12
 
13
13
  $ #{$0} [<options>] <filename> [<other|->]*
14
14
 
15
- $ rbbt slurm list [options]
15
+ $ rbbt slurm list [options]
16
16
 
17
17
  -h--help Print this help
18
18
  -d--done Done jobs only
@@ -33,7 +33,7 @@ $ rbbt slurm list [options]
33
33
  EOF
34
34
  if options[:help]
35
35
  if defined? scout_usage
36
- scout_usage
36
+ scout_usage
37
37
  else
38
38
  puts SOPT.doc
39
39
  end
@@ -132,7 +132,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
132
132
  end
133
133
 
134
134
  if File.exist?(fstatus = File.join(dir, 'job.status'))
135
- fstatus_txt = Open.read(fstatus)
135
+ fstatus_txt = Open.read(fstatus)
136
136
  begin
137
137
  if job_batch_system == "lsf"
138
138
  nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
@@ -170,47 +170,47 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
170
170
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
171
171
 
172
172
  is_running = exit_status.nil? && ( (running_jobs.include?(id) && (deps.nil? || (running_jobs & deps).empty?)) || different_system )
173
- if done || error || aborted || running || queued || jobid
173
+ if done || error || aborted || running || queued || jobid
174
174
  select = false
175
175
  select = true if done && exit_status == 0
176
- select = true if error && exit_status && exit_status != 0
176
+ select = true if error && exit_status && exit_status != 0
177
177
  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
178
178
  select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
179
179
  select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
180
180
  select = true if jobid && jobid.split(",").include?(id)
181
181
  select = select && step_path.match(/#{search}/) if search
182
- next unless select
182
+ next unless select
183
183
  elsif search
184
184
  select = false
185
185
  select = true if search && cmd.match(/#{search}/)
186
- next unless select
186
+ next unless select
187
187
  end
188
188
 
189
189
 
190
190
  count += 1
191
191
 
192
192
  if options[:compressed] && tail.nil?
193
- status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
193
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
194
194
  if different_system
195
- status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
195
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
196
196
  else
197
197
  #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
198
- status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :
199
- (running_jobs.include?(id) || $norunningjobs ?
200
- (is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) :
198
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :
199
+ (running_jobs.include?(id) || $norunningjobs ?
200
+ (is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) :
201
201
  Log.color(:red, id) )
202
- end
202
+ end
203
203
  prog_rep = []
204
- if options[:progress]
204
+ if options[:progress]
205
205
  step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
206
206
  if step_line
207
207
  step_path = step_line.split(": ").last.strip
208
208
  step = Step.new step_path
209
209
  has_bar = false
210
210
  [step].reverse.each do |j|
211
- next if j.done?
211
+ next if j.done?
212
212
  if j.file(:progress).exists?
213
- bar = Log::ProgressBar.new
213
+ bar = Log::ProgressBar.new
214
214
  bar.load(j.file(:progress).yaml)
215
215
  rep = bar.report_msg.split("·")[1]
216
216
  rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
@@ -238,7 +238,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
238
238
  puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id) )
239
239
  else
240
240
  puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
241
- end
241
+ end
242
242
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
243
243
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
244
244
  puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
@@ -247,7 +247,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
247
247
 
248
248
  if options[:batch_parameters]
249
249
  puts Log.color(:magenta, "BATCH parameters: ")
250
- case job_batch_system
250
+ case job_batch_system
251
251
  when 'slurm'
252
252
  text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
253
253
  when 'lsf'
@@ -295,14 +295,14 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
295
295
  ticks = 1 if ticks == 0
296
296
  time_elapsed = 1 if time_elapsed == 0
297
297
  puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
298
- puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
298
+ puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
299
299
  puts Log.color(:yellow, "Time: ") + Misc.format_seconds((eend - start))
300
300
 
301
301
  end
302
302
 
303
303
  if options[:sacct_peformance]
304
304
  begin
305
- raise "sacct not supported for LSF" unless batch_system == 'slurm'
305
+ raise "sacct not supported for LSF" unless batch_system == 'slurm'
306
306
  tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
307
307
  values = tsv[tsv.keys.first]
308
308
  if values.compact.any?
@@ -324,20 +324,21 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
324
324
  puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
325
325
  else
326
326
  puts Log.color(:magenta, "Log tail: ")
327
- puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | grep -v \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
327
+ pad_tail = 1000 + tail.to_i
328
+ puts CMD.cmd("tail -n #{pad_tail} #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | grep -v \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
328
329
  end
329
330
  end
330
331
 
331
- if options[:progress]
332
+ if options[:progress]
332
333
  step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
333
334
  if step_line
334
335
  step_path = step_line.split(": ").last.strip
335
336
  step = Step.new step_path
336
337
  has_bar = false
337
338
  (step.rec_dependencies.to_a + [step]).reverse.each do |j|
338
- next if j.done?
339
+ next if j.done?
339
340
  if j.file(:progress).exists?
340
- bar = Log::ProgressBar.new
341
+ bar = Log::ProgressBar.new
341
342
  bar.load(j.file(:progress).yaml)
342
343
  puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
343
344
  has_bar = true
@@ -352,6 +353,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
352
353
 
353
354
  end
354
355
 
355
- puts
356
+ puts
356
357
  puts Log.color :clear, "Found #{count} jobs"
357
358
 
@@ -8,14 +8,14 @@ options = SOPT.setup <<EOF
8
8
 
9
9
  Queue a job in Marenostrum
10
10
 
11
- $ rbbt slurm tail <directory|jobid> [options]
11
+ $ rbbt slurm tail <directory|jobid|step> [options]
12
12
 
13
13
  -h--help Print this help
14
14
  EOF
15
15
 
16
16
  if options[:help]
17
17
  if defined? rbbt_usage
18
- rbbt_usage
18
+ rbbt_usage
19
19
  else
20
20
  puts SOPT.doc
21
21
  end
@@ -29,18 +29,22 @@ directory = ARGV.shift
29
29
 
30
30
  raise ParameterException if directory.nil?
31
31
 
32
+ workdir = File.expand_path('~/scout-batch')
33
+ Path.setup(workdir)
34
+
32
35
  if directory =~ /^[0-9]*$/
33
- workdir = File.expand_path('~/scout-batch')
34
- Path.setup(workdir)
35
36
 
36
37
  workdir.glob("**/job.id").each do |file|
37
38
  next unless directory == Open.read(file).strip
38
39
  directory = File.dirname(file)
39
40
  break
40
41
  end
42
+ elsif ! File.exist?(File.join(directory, 'command.batch'))
43
+ out = CMD.cmd("grep -r -l 'STEP_PATH: .*#{directory}' '#{workdir}'").read.split("\n").first
44
+ directory = out.sub("/command.batch", '') if out and not out.empty?
41
45
  end
42
46
 
43
- raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
47
+ raise ParameterException, "Could not identify job #{directory}" unless File.exist?(File.join(directory, 'command.batch'))
44
48
 
45
49
  require 'rbbt/hpc/slurm'
46
50
 
data/scout_commands/cat CHANGED
@@ -17,7 +17,7 @@ $ #{$0} [<options>] (<resource> <path>|<path>)
17
17
  EOF
18
18
  if options[:help]
19
19
  if defined? scout_usage
20
- scout_usage
20
+ scout_usage
21
21
  else
22
22
  puts SOPT.doc
23
23
  end
data/scout_commands/doc CHANGED
@@ -16,7 +16,7 @@ If you don't specify the module it prints those available
16
16
  EOF
17
17
  if options[:help]
18
18
  if defined? scout_usage
19
- scout_usage
19
+ scout_usage
20
20
  else
21
21
  puts SOPT.doc
22
22
  end
@@ -26,7 +26,7 @@ end
26
26
 
27
27
  module_name = ARGV.first
28
28
  if module_name.nil?
29
- puts Scout.doc.lib.scout.glob("**/*.md").collect{|f|
29
+ puts Scout.doc.lib.scout.glob("**/*.md").collect{|f|
30
30
  f.find.unset_extension.sub(Scout.doc.lib.scout.find + "/", '')
31
31
  } * "\n"
32
32
  else
@@ -21,7 +21,7 @@ type, property, entity = ARGV
21
21
 
22
22
  if options[:help] and type.nil?
23
23
  if defined? scout_usage
24
- scout_usage
24
+ scout_usage
25
25
  else
26
26
  puts SOPT.doc
27
27
  end
@@ -63,12 +63,12 @@ end
63
63
  entity_module = Kernel.const_get type
64
64
 
65
65
  entity_module.annotations.uniq.each do |attr|
66
- SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute"
66
+ SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute"
67
67
  end
68
68
 
69
69
  if options[:help] and property.nil?
70
70
  if defined? scout_usage
71
- scout_usage
71
+ scout_usage
72
72
  else
73
73
  puts SOPT.doc
74
74
  end
@@ -99,7 +99,7 @@ if options[:help]
99
99
  end * " "
100
100
 
101
101
  SOPT.synopsys['[<property_parameters>]*'] = property_str
102
-
102
+
103
103
  if defined? scout_usage
104
104
  scout_usage
105
105
  else
data/scout_commands/find CHANGED
@@ -17,7 +17,7 @@ $ #{$0} [<options>] (<resource> <path>|<path>)
17
17
  EOF
18
18
  if options[:help]
19
19
  if defined? scout_usage
20
- scout_usage
20
+ scout_usage
21
21
  else
22
22
  puts SOPT.doc
23
23
  end
@@ -18,7 +18,7 @@ $ #{$0} [<options>] <name> [knowledge_base]
18
18
  EOF
19
19
  if options.delete :help
20
20
  if defined? scout_usage
21
- scout_usage
21
+ scout_usage
22
22
  else
23
23
  puts SOPT.doc
24
24
  end
@@ -16,7 +16,7 @@ $ #{$0} [<options>] <entity> <identifier_files>
16
16
  EOF
17
17
  if options.delete :help
18
18
  if defined? scout_usage
19
- scout_usage
19
+ scout_usage
20
20
  else
21
21
  puts SOPT.doc
22
22
  end
@@ -16,7 +16,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
16
16
  EOF
17
17
  if options[:help]
18
18
  if defined? scout_usage
19
- scout_usage
19
+ scout_usage
20
20
  else
21
21
  puts SOPT.doc
22
22
  end
@@ -21,7 +21,7 @@ $ #{$0} [<options>] <name> <entity>
21
21
  EOF
22
22
  if options[:help]
23
23
  if defined? scout_usage
24
- scout_usage
24
+ scout_usage
25
25
  else
26
26
  puts SOPT.doc
27
27
  end
@@ -70,7 +70,7 @@ if matches.any?
70
70
  else
71
71
  matches.each do |match|
72
72
  puts Log.color :magenta, (reverse ? match.invert : match)
73
- puts index[match].prety_print
73
+ puts index[match].prety_print
74
74
  end
75
75
  end
76
76
  else
@@ -23,7 +23,7 @@ $ #{$0} [<options>] <name> <filename>
23
23
  EOF
24
24
  if options.delete :help
25
25
  if defined? scout_usage
26
- scout_usage
26
+ scout_usage
27
27
  else
28
28
  puts SOPT.doc
29
29
  end
@@ -16,7 +16,7 @@ $ #{$0} [<options>] <name>
16
16
  EOF
17
17
  if options[:help]
18
18
  if defined? scout_usage
19
- scout_usage
19
+ scout_usage
20
20
  else
21
21
  puts SOPT.doc
22
22
  end
@@ -20,7 +20,7 @@ $ #{$0} [<options>] <traversal>
20
20
  EOF
21
21
  if options[:help]
22
22
  if defined? scout_usage
23
- scout_usage
23
+ scout_usage
24
24
  else
25
25
  puts SOPT.doc
26
26
  end
data/scout_commands/log CHANGED
@@ -23,7 +23,7 @@ NONE
23
23
  EOF
24
24
  if options[:help]
25
25
  if defined? scout_usage
26
- scout_usage
26
+ scout_usage
27
27
  else
28
28
  puts SOPT.doc
29
29
  end
@@ -31,11 +31,11 @@ if options[:help]
31
31
  end
32
32
 
33
33
  if ARGV.empty?
34
- if Scout.etc.log_severity.exists?
35
- puts Scout.etc.log_severity.read
36
- else
37
- puts Scout.etc.log_severity.find + ' does not exist'
38
- end
34
+ if Scout.etc.log_severity.exists?
35
+ puts Scout.etc.log_severity.read
36
+ else
37
+ puts Scout.etc.log_severity.find + ' does not exist'
38
+ end
39
39
  else
40
40
  level = ARGV[0]
41
41
  level = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE)[level.to_i] if level =~ /^\d+$/
@@ -8,7 +8,7 @@ options = SOPT.setup <<EOF
8
8
 
9
9
  Produce a resource
10
10
 
11
- $ #{$0} [<options>] <Resource> <file>
11
+ $ #{$0} [<options>] <Resource> <file>
12
12
 
13
13
  -h--help Print this help
14
14
  -W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
@@ -17,7 +17,7 @@ $ #{$0} [<options>] <Resource> <file>
17
17
  EOF
18
18
  if options[:help]
19
19
  if defined? scout_usage
20
- scout_usage
20
+ scout_usage
21
21
  else
22
22
  puts SOPT.doc
23
23
  end
@@ -19,7 +19,7 @@ Path map defaults to :user
19
19
  EOF
20
20
  if options[:help]
21
21
  if defined? scout_usage
22
- scout_usage
22
+ scout_usage
23
23
  else
24
24
  puts SOPT.doc
25
25
  end
@@ -47,7 +47,7 @@ if locks.any?
47
47
  locks.each do |file,info|
48
48
  if force or (info[:pid] && ! Misc.pid_alive?(info[:pid]))
49
49
  puts " Removing #{ file }"
50
- File.unlink file
50
+ File.unlink file
51
51
  end
52
52
  end
53
53
  end
@@ -59,7 +59,7 @@ if persists.any?
59
59
  persists.each do |file,info|
60
60
  if force or (info[:pid] and Misc.pid_alive? info[:pid])
61
61
  puts " Removing #{ file }"
62
- File.unlink file
62
+ File.unlink file
63
63
  end
64
64
  end
65
65
  end
@@ -71,7 +71,7 @@ if sensiblewrites.any?
71
71
  sensiblewrites.each do |file,info|
72
72
  if force or (info[:pid] and Misc.pid_alive? info[:pid])
73
73
  puts " Removing #{ file }"
74
- File.unlink file
74
+ File.unlink file
75
75
  end
76
76
  end
77
77
  end
@@ -134,13 +134,13 @@ TSV.traverse jobs do |file,i|
134
134
  end
135
135
  end
136
136
 
137
- if (force && status !~ /done/) or
137
+ if (force && status !~ /done/) or
138
138
  status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync|waiting)$/ or
139
139
  (status == "noinfo" and not done) or
140
- status == ""
140
+ status == ""
141
141
 
142
- puts " Removing #{ file } - #{status}"
143
- Step.clean(file)
142
+ puts " Removing #{ file } - #{status}"
143
+ Step.clean(file)
144
144
  end
145
145
  end
146
146
 
@@ -177,7 +177,7 @@ workflows.sort.each do |workflow,tasks|
177
177
  if options[:quick] and i[:done]
178
178
  status = 'done'
179
179
  str << " #{ Step.prov_status_msg status }"
180
- if inputs and inputs.any?
180
+ if inputs and inputs.any?
181
181
  str << input_msg(file, inputs)
182
182
  end
183
183
 
@@ -210,15 +210,15 @@ workflows.sort.each do |workflow,tasks|
210
210
  status = status.to_s
211
211
  if status != "done" and pid and not Misc.pid_alive?(pid)
212
212
  if File.exist? file
213
- status << Log.color(:red, " (out of sync)")
213
+ status << Log.color(:red, " (out of sync)")
214
214
  else
215
- status << Log.color(:red, " (dead)")
215
+ status << Log.color(:red, " (dead)")
216
216
  end
217
217
  end
218
218
  str << " #{ Step.prov_status_msg status }"
219
219
  str << " (dirty)" if status == 'done' && Step.new(file).dirty?
220
220
 
221
- if inputs and inputs.any?
221
+ if inputs and inputs.any?
222
222
  str << input_msg(file, inputs)
223
223
  end
224
224
 
@@ -19,7 +19,7 @@ Print a template making the substitutions that follow
19
19
  EOF
20
20
  if options[:help]
21
21
  if defined? scout_usage
22
- scout_usage
22
+ scout_usage
23
23
  else
24
24
  puts SOPT.doc
25
25
  end
@@ -16,7 +16,7 @@ If no gem is specified 'scout-gear' is selected.
16
16
  EOF
17
17
  if options[:help]
18
18
  if defined? scout_usage
19
- scout_usage
19
+ scout_usage
20
20
  else
21
21
  puts SOPT.doc
22
22
  end
@@ -16,7 +16,7 @@ $ #{$0} [<options>] <step_path>
16
16
  EOF
17
17
  if options[:help]
18
18
  if defined? scout_usage
19
- scout_usage
19
+ scout_usage
20
20
  else
21
21
  puts SOPT.doc
22
22
  end
@@ -22,7 +22,7 @@ workflows install automatically as required.
22
22
  EOF
23
23
  if options[:help]
24
24
  if defined? scout_usage
25
- scout_usage
25
+ scout_usage
26
26
  else
27
27
  puts SOPT.doc
28
28
  end
@@ -8,13 +8,13 @@ options = SOPT.setup <<EOF
8
8
 
9
9
  List all workflows
10
10
 
11
- $ #{$0} [<options>]
11
+ $ #{$0} [<options>]
12
12
 
13
13
  -h--help Print this help
14
14
  EOF
15
15
  if options[:help]
16
16
  if defined? scout_usage
17
- scout_usage
17
+ scout_usage
18
18
  else
19
19
  puts SOPT.doc
20
20
  end
@@ -22,7 +22,7 @@ $ #{$0} [<options>] ([<workflow>] [<task>] [<name>] | <filename>)
22
22
  EOF
23
23
  if options[:help]
24
24
  if defined? scout_usage
25
- scout_usage
25
+ scout_usage
26
26
  else
27
27
  puts SOPT.doc
28
28
  end
@@ -69,7 +69,7 @@ begin
69
69
  end
70
70
 
71
71
  jobs = files.collect{|file| Workflow.queue_job(file) }
72
-
72
+
73
73
 
74
74
  begin
75
75
  Workflow.produce(jobs, **options)
@@ -19,7 +19,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
19
19
  EOF
20
20
  if options[:help]
21
21
  if defined? scout_usage
22
- scout_usage
22
+ scout_usage
23
23
  else
24
24
  puts SOPT.doc
25
25
  end
@@ -77,10 +77,10 @@ def adjacency(step)
77
77
  info[:dependencies].each do |task,name,path|
78
78
  dep = get_step path
79
79
  _id, _edges, _node_info = adjacency(dep)
80
- edges << [_id, id]
80
+ edges << [_id, id]
81
81
  edges.concat _edges
82
82
  node_info.merge!(_node_info)
83
- end
83
+ end
84
84
  end
85
85
 
86
86
  [id, edges, node_info]