scout-gear 10.11.6 → 10.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +16 -2
- data/VERSION +1 -1
- data/bin/scout +10 -10
- data/lib/scout/association/fields.rb +15 -15
- data/lib/scout/association/index.rb +6 -6
- data/lib/scout/association/item.rb +18 -8
- data/lib/scout/association.rb +4 -4
- data/lib/scout/entity/identifiers.rb +5 -5
- data/lib/scout/entity/property.rb +2 -2
- data/lib/scout/entity.rb +1 -1
- data/lib/scout/knowledge_base/description.rb +10 -10
- data/lib/scout/knowledge_base/entity.rb +6 -6
- data/lib/scout/knowledge_base/list.rb +1 -1
- data/lib/scout/knowledge_base/query.rb +4 -4
- data/lib/scout/knowledge_base/registry.rb +6 -6
- data/lib/scout/knowledge_base/traverse.rb +7 -40
- data/lib/scout/persist/engine/fix_width_table.rb +6 -6
- data/lib/scout/persist/engine/packed_index.rb +2 -2
- data/lib/scout/persist/engine/sharder.rb +4 -4
- data/lib/scout/persist/engine/tkrzw.rb +1 -1
- data/lib/scout/persist/engine/tokyocabinet.rb +2 -2
- data/lib/scout/persist/tsv/adapter/fix_width_table.rb +1 -1
- data/lib/scout/persist/tsv/adapter/packed_index.rb +1 -1
- data/lib/scout/persist/tsv/adapter/tkrzw.rb +1 -1
- data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +3 -3
- data/lib/scout/persist/tsv/serialize.rb +3 -3
- data/lib/scout/persist/tsv.rb +1 -1
- data/lib/scout/semaphore.rb +78 -3
- data/lib/scout/tsv/annotation/repo.rb +4 -4
- data/lib/scout/tsv/annotation.rb +2 -2
- data/lib/scout/tsv/attach.rb +7 -7
- data/lib/scout/tsv/change_id/translate.rb +1 -1
- data/lib/scout/tsv/csv.rb +3 -3
- data/lib/scout/tsv/dumper.rb +8 -8
- data/lib/scout/tsv/index.rb +1 -1
- data/lib/scout/tsv/open.rb +3 -3
- data/lib/scout/tsv/stream.rb +2 -2
- data/lib/scout/tsv/traverse.rb +4 -4
- data/lib/scout/tsv/util/filter.rb +9 -9
- data/lib/scout/tsv/util/process.rb +2 -2
- data/lib/scout/tsv/util/reorder.rb +2 -2
- data/lib/scout/tsv/util/select.rb +3 -3
- data/lib/scout/tsv/util/unzip.rb +2 -2
- data/lib/scout/tsv/util.rb +1 -1
- data/lib/scout/tsv.rb +2 -2
- data/lib/scout/work_queue/socket.rb +2 -2
- data/lib/scout/work_queue/worker.rb +4 -4
- data/lib/scout/work_queue.rb +5 -5
- data/lib/scout/workflow/definition.rb +18 -16
- data/lib/scout/workflow/deployment/local.rb +82 -62
- data/lib/scout/workflow/deployment/orchestrator/batches.rb +66 -5
- data/lib/scout/workflow/deployment/orchestrator/chains.rb +47 -30
- data/lib/scout/workflow/deployment/orchestrator/rules.rb +3 -3
- data/lib/scout/workflow/deployment/orchestrator/workload.rb +11 -22
- data/lib/scout/workflow/deployment/scheduler/job.rb +34 -36
- data/lib/scout/workflow/deployment/scheduler/lfs.rb +1 -1
- data/lib/scout/workflow/deployment/scheduler/pbs.rb +4 -4
- data/lib/scout/workflow/deployment/scheduler/slurm.rb +2 -2
- data/lib/scout/workflow/deployment/scheduler.rb +23 -12
- data/lib/scout/workflow/deployment/trace.rb +2 -2
- data/lib/scout/workflow/documentation.rb +4 -4
- data/lib/scout/workflow/export.rb +1 -1
- data/lib/scout/workflow/path.rb +2 -2
- data/lib/scout/workflow/step/children.rb +1 -1
- data/lib/scout/workflow/step/dependencies.rb +36 -3
- data/lib/scout/workflow/step/info.rb +5 -19
- data/lib/scout/workflow/step/inputs.rb +1 -1
- data/lib/scout/workflow/step/progress.rb +2 -2
- data/lib/scout/workflow/step/provenance.rb +4 -4
- data/lib/scout/workflow/step/status.rb +23 -9
- data/lib/scout/workflow/step.rb +19 -17
- data/lib/scout/workflow/task/dependencies.rb +10 -3
- data/lib/scout/workflow/task/info.rb +3 -3
- data/lib/scout/workflow/task/inputs.rb +14 -8
- data/lib/scout/workflow/task.rb +37 -22
- data/lib/scout/workflow/usage.rb +13 -13
- data/lib/scout/workflow/util.rb +1 -1
- data/lib/scout/workflow.rb +6 -6
- data/scout-gear.gemspec +3 -3
- data/scout_commands/alias +1 -1
- data/scout_commands/batch/clean +12 -12
- data/scout_commands/batch/list +26 -25
- data/scout_commands/batch/tail +9 -5
- data/scout_commands/cat +1 -1
- data/scout_commands/doc +2 -2
- data/scout_commands/entity +4 -4
- data/scout_commands/find +1 -1
- data/scout_commands/kb/config +1 -1
- data/scout_commands/kb/entities +1 -1
- data/scout_commands/kb/list +1 -1
- data/scout_commands/kb/query +2 -2
- data/scout_commands/kb/register +1 -1
- data/scout_commands/kb/show +1 -1
- data/scout_commands/kb/traverse +1 -1
- data/scout_commands/log +6 -6
- data/scout_commands/resource/produce +2 -2
- data/scout_commands/resource/sync +1 -1
- data/scout_commands/system/clean +7 -7
- data/scout_commands/system/status +4 -4
- data/scout_commands/template +1 -1
- data/scout_commands/update +1 -1
- data/scout_commands/workflow/info +1 -1
- data/scout_commands/workflow/install +1 -1
- data/scout_commands/workflow/list +2 -2
- data/scout_commands/workflow/process +2 -2
- data/scout_commands/workflow/prov +3 -3
- data/scout_commands/workflow/task +36 -11
- data/scout_commands/workflow/trace +1 -1
- data/scout_commands/workflow/write_info +2 -2
- data/share/templates/command +1 -1
- data/test/scout/association/test_item.rb +5 -0
- data/test/scout/entity/test_property.rb +3 -3
- data/test/scout/knowledge_base/test_description.rb +1 -1
- data/test/scout/knowledge_base/test_traverse.rb +2 -2
- data/test/scout/persist/engine/test_packed_index.rb +6 -6
- data/test/scout/persist/test_tsv.rb +4 -4
- data/test/scout/persist/tsv/adapter/test_packed_index.rb +4 -4
- data/test/scout/persist/tsv/adapter/test_sharder.rb +23 -23
- data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +1 -1
- data/test/scout/persist/tsv/test_serialize.rb +1 -1
- data/test/scout/test_association.rb +1 -1
- data/test/scout/test_tsv.rb +2 -2
- data/test/scout/test_workflow.rb +2 -2
- data/test/scout/tsv/test_annotation.rb +4 -4
- data/test/scout/tsv/test_index.rb +1 -1
- data/test/scout/tsv/test_open.rb +2 -2
- data/test/scout/tsv/test_parser.rb +2 -2
- data/test/scout/tsv/test_stream.rb +1 -1
- data/test/scout/tsv/test_transformer.rb +1 -1
- data/test/scout/tsv/util/test_filter.rb +1 -1
- data/test/scout/tsv/util/test_melt.rb +1 -1
- data/test/scout/tsv/util/test_reorder.rb +1 -1
- data/test/scout/work_queue/test_socket.rb +3 -3
- data/test/scout/work_queue/test_worker.rb +2 -2
- data/test/scout/workflow/deployment/orchestrator/test_batches.rb +13 -3
- data/test/scout/workflow/deployment/orchestrator/test_chains.rb +15 -13
- data/test/scout/workflow/deployment/orchestrator/test_workload.rb +1 -1
- data/test/scout/workflow/deployment/test_local.rb +2 -2
- data/test/scout/workflow/deployment/test_scheduler.rb +1 -2
- data/test/scout/workflow/step/test_children.rb +1 -1
- data/test/scout/workflow/step/test_dependencies.rb +36 -1
- data/test/scout/workflow/step/test_info.rb +3 -35
- data/test/scout/workflow/step/test_load.rb +1 -1
- data/test/scout/workflow/step/test_provenance.rb +1 -1
- data/test/scout/workflow/step/test_status.rb +33 -1
- data/test/scout/workflow/task/test_dependencies.rb +9 -7
- data/test/scout/workflow/task/test_inputs.rb +1 -1
- data/test/scout/workflow/test_definition.rb +1 -1
- data/test/scout/workflow/test_documentation.rb +1 -1
- data/test/scout/workflow/test_entity.rb +2 -2
- data/test/scout/workflow/test_step.rb +13 -13
- data/test/scout/workflow/test_usage.rb +1 -1
- data/test/test_helper.rb +1 -1
- metadata +2 -2
data/scout-gear.gemspec
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
-
# stub: scout-gear 10.11.
|
|
5
|
+
# stub: scout-gear 10.11.8 ruby lib
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |s|
|
|
8
8
|
s.name = "scout-gear".freeze
|
|
9
|
-
s.version = "10.11.
|
|
9
|
+
s.version = "10.11.8".freeze
|
|
10
10
|
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
|
12
12
|
s.require_paths = ["lib".freeze]
|
|
@@ -284,7 +284,7 @@ Gem::Specification.new do |s|
|
|
|
284
284
|
]
|
|
285
285
|
s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
|
|
286
286
|
s.licenses = ["MIT".freeze]
|
|
287
|
-
s.rubygems_version = "3.7.
|
|
287
|
+
s.rubygems_version = "3.7.2".freeze
|
|
288
288
|
s.summary = "basic gear for scouts".freeze
|
|
289
289
|
|
|
290
290
|
s.specification_version = 4
|
data/scout_commands/alias
CHANGED
data/scout_commands/batch/clean
CHANGED
|
@@ -25,7 +25,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
|
|
|
25
25
|
EOF
|
|
26
26
|
if options[:help]
|
|
27
27
|
if defined? scout_usage
|
|
28
|
-
scout_usage
|
|
28
|
+
scout_usage
|
|
29
29
|
else
|
|
30
30
|
puts SOPT.doc
|
|
31
31
|
end
|
|
@@ -109,7 +109,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
109
109
|
end
|
|
110
110
|
|
|
111
111
|
if File.exist?(fstatus = File.join(dir, 'job.status'))
|
|
112
|
-
fstatus_txt = Open.read(fstatus)
|
|
112
|
+
fstatus_txt = Open.read(fstatus)
|
|
113
113
|
begin
|
|
114
114
|
if job_batch_system == "lsf"
|
|
115
115
|
nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
|
|
@@ -140,28 +140,28 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
140
140
|
aborted = error = true if ! done && aborted.nil? && error.nil?
|
|
141
141
|
#if done || error || aborted || running || queued || jobid || search
|
|
142
142
|
# select = false
|
|
143
|
-
# select = true if done && exit_status && exit_status.to_i == 0
|
|
144
|
-
# select = true if error && exit_status && exit_status.to_i != 0
|
|
143
|
+
# select = true if done && exit_status && exit_status.to_i == 0
|
|
144
|
+
# select = true if error && exit_status && exit_status.to_i != 0
|
|
145
145
|
# select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
|
|
146
|
-
# select = select && jobid.split(",").include?(id) if jobid
|
|
146
|
+
# select = select && jobid.split(",").include?(id) if jobid
|
|
147
147
|
# select = select && cmd.match(/#{search}/) if search
|
|
148
|
-
# next unless select
|
|
148
|
+
# next unless select
|
|
149
149
|
#end
|
|
150
150
|
|
|
151
|
-
if done || error || aborted || queued || jobid
|
|
151
|
+
if done || error || aborted || queued || jobid
|
|
152
152
|
select = false
|
|
153
153
|
select = true if done && exit_status == 0
|
|
154
|
-
select = true if error && exit_status && exit_status != 0
|
|
154
|
+
select = true if error && exit_status && exit_status != 0
|
|
155
155
|
select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
|
|
156
156
|
is_running = exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
|
|
157
157
|
select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
|
|
158
158
|
select = true if jobid && jobid.split(",").include?(id)
|
|
159
159
|
select = select && cmd.match(/#{search}/) if search
|
|
160
|
-
next unless select
|
|
160
|
+
next unless select
|
|
161
161
|
elsif search
|
|
162
162
|
select = false
|
|
163
163
|
select = true if search && cmd.match(/#{search}/)
|
|
164
|
-
next unless select
|
|
164
|
+
next unless select
|
|
165
165
|
end
|
|
166
166
|
|
|
167
167
|
|
|
@@ -180,7 +180,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
180
180
|
|
|
181
181
|
if options[:batch_parameters]
|
|
182
182
|
puts Log.color(:magenta, "BATCH parameters: ")
|
|
183
|
-
case job_batch_system
|
|
183
|
+
case job_batch_system
|
|
184
184
|
when 'slurm'
|
|
185
185
|
puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
|
|
186
186
|
when 'lsf'
|
|
@@ -206,6 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
206
206
|
Open.rm_rf dir unless dry_run
|
|
207
207
|
end
|
|
208
208
|
|
|
209
|
-
puts
|
|
209
|
+
puts
|
|
210
210
|
puts "Found #{count} jobs"
|
|
211
211
|
|
data/scout_commands/batch/list
CHANGED
|
@@ -12,7 +12,7 @@ List all batch jobs
|
|
|
12
12
|
|
|
13
13
|
$ #{$0} [<options>] <filename> [<other|->]*
|
|
14
14
|
|
|
15
|
-
$ rbbt slurm list [options]
|
|
15
|
+
$ rbbt slurm list [options]
|
|
16
16
|
|
|
17
17
|
-h--help Print this help
|
|
18
18
|
-d--done Done jobs only
|
|
@@ -33,7 +33,7 @@ $ rbbt slurm list [options]
|
|
|
33
33
|
EOF
|
|
34
34
|
if options[:help]
|
|
35
35
|
if defined? scout_usage
|
|
36
|
-
scout_usage
|
|
36
|
+
scout_usage
|
|
37
37
|
else
|
|
38
38
|
puts SOPT.doc
|
|
39
39
|
end
|
|
@@ -132,7 +132,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
132
132
|
end
|
|
133
133
|
|
|
134
134
|
if File.exist?(fstatus = File.join(dir, 'job.status'))
|
|
135
|
-
fstatus_txt = Open.read(fstatus)
|
|
135
|
+
fstatus_txt = Open.read(fstatus)
|
|
136
136
|
begin
|
|
137
137
|
if job_batch_system == "lsf"
|
|
138
138
|
nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
|
|
@@ -170,47 +170,47 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
170
170
|
cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
|
|
171
171
|
|
|
172
172
|
is_running = exit_status.nil? && ( (running_jobs.include?(id) && (deps.nil? || (running_jobs & deps).empty?)) || different_system )
|
|
173
|
-
if done || error || aborted || running || queued || jobid
|
|
173
|
+
if done || error || aborted || running || queued || jobid
|
|
174
174
|
select = false
|
|
175
175
|
select = true if done && exit_status == 0
|
|
176
|
-
select = true if error && exit_status && exit_status != 0
|
|
176
|
+
select = true if error && exit_status && exit_status != 0
|
|
177
177
|
select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
|
|
178
178
|
select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
|
|
179
179
|
select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
|
|
180
180
|
select = true if jobid && jobid.split(",").include?(id)
|
|
181
181
|
select = select && step_path.match(/#{search}/) if search
|
|
182
|
-
next unless select
|
|
182
|
+
next unless select
|
|
183
183
|
elsif search
|
|
184
184
|
select = false
|
|
185
185
|
select = true if search && cmd.match(/#{search}/)
|
|
186
|
-
next unless select
|
|
186
|
+
next unless select
|
|
187
187
|
end
|
|
188
188
|
|
|
189
189
|
|
|
190
190
|
count += 1
|
|
191
191
|
|
|
192
192
|
if options[:compressed] && tail.nil?
|
|
193
|
-
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
|
|
193
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
|
|
194
194
|
if different_system
|
|
195
|
-
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
|
|
195
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
|
|
196
196
|
else
|
|
197
197
|
#status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
|
|
198
|
-
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :
|
|
199
|
-
(running_jobs.include?(id) || $norunningjobs ?
|
|
200
|
-
(is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) :
|
|
198
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :
|
|
199
|
+
(running_jobs.include?(id) || $norunningjobs ?
|
|
200
|
+
(is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) :
|
|
201
201
|
Log.color(:red, id) )
|
|
202
|
-
end
|
|
202
|
+
end
|
|
203
203
|
prog_rep = []
|
|
204
|
-
if options[:progress]
|
|
204
|
+
if options[:progress]
|
|
205
205
|
step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
|
|
206
206
|
if step_line
|
|
207
207
|
step_path = step_line.split(": ").last.strip
|
|
208
208
|
step = Step.new step_path
|
|
209
209
|
has_bar = false
|
|
210
210
|
[step].reverse.each do |j|
|
|
211
|
-
next if j.done?
|
|
211
|
+
next if j.done?
|
|
212
212
|
if j.file(:progress).exists?
|
|
213
|
-
bar = Log::ProgressBar.new
|
|
213
|
+
bar = Log::ProgressBar.new
|
|
214
214
|
bar.load(j.file(:progress).yaml)
|
|
215
215
|
rep = bar.report_msg.split("·")[1]
|
|
216
216
|
rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
|
|
@@ -238,7 +238,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
238
238
|
puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id) )
|
|
239
239
|
else
|
|
240
240
|
puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
|
|
241
|
-
end
|
|
241
|
+
end
|
|
242
242
|
puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
|
|
243
243
|
puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
|
|
244
244
|
puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
|
|
@@ -247,7 +247,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
247
247
|
|
|
248
248
|
if options[:batch_parameters]
|
|
249
249
|
puts Log.color(:magenta, "BATCH parameters: ")
|
|
250
|
-
case job_batch_system
|
|
250
|
+
case job_batch_system
|
|
251
251
|
when 'slurm'
|
|
252
252
|
text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
|
|
253
253
|
when 'lsf'
|
|
@@ -295,14 +295,14 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
295
295
|
ticks = 1 if ticks == 0
|
|
296
296
|
time_elapsed = 1 if time_elapsed == 0
|
|
297
297
|
puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
|
|
298
|
-
puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
|
|
298
|
+
puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
|
|
299
299
|
puts Log.color(:yellow, "Time: ") + Misc.format_seconds((eend - start))
|
|
300
300
|
|
|
301
301
|
end
|
|
302
302
|
|
|
303
303
|
if options[:sacct_peformance]
|
|
304
304
|
begin
|
|
305
|
-
raise "sacct not supported for LSF" unless batch_system == 'slurm'
|
|
305
|
+
raise "sacct not supported for LSF" unless batch_system == 'slurm'
|
|
306
306
|
tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
|
|
307
307
|
values = tsv[tsv.keys.first]
|
|
308
308
|
if values.compact.any?
|
|
@@ -324,20 +324,21 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
324
324
|
puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
|
|
325
325
|
else
|
|
326
326
|
puts Log.color(:magenta, "Log tail: ")
|
|
327
|
-
|
|
327
|
+
pad_tail = 1000 + tail.to_i
|
|
328
|
+
puts CMD.cmd("tail -n #{pad_tail} #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | grep -v \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
|
|
328
329
|
end
|
|
329
330
|
end
|
|
330
331
|
|
|
331
|
-
if options[:progress]
|
|
332
|
+
if options[:progress]
|
|
332
333
|
step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
|
|
333
334
|
if step_line
|
|
334
335
|
step_path = step_line.split(": ").last.strip
|
|
335
336
|
step = Step.new step_path
|
|
336
337
|
has_bar = false
|
|
337
338
|
(step.rec_dependencies.to_a + [step]).reverse.each do |j|
|
|
338
|
-
next if j.done?
|
|
339
|
+
next if j.done?
|
|
339
340
|
if j.file(:progress).exists?
|
|
340
|
-
bar = Log::ProgressBar.new
|
|
341
|
+
bar = Log::ProgressBar.new
|
|
341
342
|
bar.load(j.file(:progress).yaml)
|
|
342
343
|
puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
|
|
343
344
|
has_bar = true
|
|
@@ -352,6 +353,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
|
352
353
|
|
|
353
354
|
end
|
|
354
355
|
|
|
355
|
-
puts
|
|
356
|
+
puts
|
|
356
357
|
puts Log.color :clear, "Found #{count} jobs"
|
|
357
358
|
|
data/scout_commands/batch/tail
CHANGED
|
@@ -8,14 +8,14 @@ options = SOPT.setup <<EOF
|
|
|
8
8
|
|
|
9
9
|
Queue a job in Marenostrum
|
|
10
10
|
|
|
11
|
-
$ rbbt slurm tail <directory|jobid> [options]
|
|
11
|
+
$ rbbt slurm tail <directory|jobid|step> [options]
|
|
12
12
|
|
|
13
13
|
-h--help Print this help
|
|
14
14
|
EOF
|
|
15
15
|
|
|
16
16
|
if options[:help]
|
|
17
17
|
if defined? rbbt_usage
|
|
18
|
-
rbbt_usage
|
|
18
|
+
rbbt_usage
|
|
19
19
|
else
|
|
20
20
|
puts SOPT.doc
|
|
21
21
|
end
|
|
@@ -29,18 +29,22 @@ directory = ARGV.shift
|
|
|
29
29
|
|
|
30
30
|
raise ParameterException if directory.nil?
|
|
31
31
|
|
|
32
|
+
workdir = File.expand_path('~/scout-batch')
|
|
33
|
+
Path.setup(workdir)
|
|
34
|
+
|
|
32
35
|
if directory =~ /^[0-9]*$/
|
|
33
|
-
workdir = File.expand_path('~/scout-batch')
|
|
34
|
-
Path.setup(workdir)
|
|
35
36
|
|
|
36
37
|
workdir.glob("**/job.id").each do |file|
|
|
37
38
|
next unless directory == Open.read(file).strip
|
|
38
39
|
directory = File.dirname(file)
|
|
39
40
|
break
|
|
40
41
|
end
|
|
42
|
+
elsif ! File.exist?(File.join(directory, 'command.batch'))
|
|
43
|
+
out = CMD.cmd("grep -r -l 'STEP_PATH: .*#{directory}' '#{workdir}'").read.split("\n").first
|
|
44
|
+
directory = out.sub("/command.batch", '') if out and not out.empty?
|
|
41
45
|
end
|
|
42
46
|
|
|
43
|
-
raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
|
|
47
|
+
raise ParameterException, "Could not identify job #{directory}" unless File.exist?(File.join(directory, 'command.batch'))
|
|
44
48
|
|
|
45
49
|
require 'rbbt/hpc/slurm'
|
|
46
50
|
|
data/scout_commands/cat
CHANGED
data/scout_commands/doc
CHANGED
|
@@ -16,7 +16,7 @@ If you don't specify the module it prints those available
|
|
|
16
16
|
EOF
|
|
17
17
|
if options[:help]
|
|
18
18
|
if defined? scout_usage
|
|
19
|
-
scout_usage
|
|
19
|
+
scout_usage
|
|
20
20
|
else
|
|
21
21
|
puts SOPT.doc
|
|
22
22
|
end
|
|
@@ -26,7 +26,7 @@ end
|
|
|
26
26
|
|
|
27
27
|
module_name = ARGV.first
|
|
28
28
|
if module_name.nil?
|
|
29
|
-
puts Scout.doc.lib.scout.glob("**/*.md").collect{|f|
|
|
29
|
+
puts Scout.doc.lib.scout.glob("**/*.md").collect{|f|
|
|
30
30
|
f.find.unset_extension.sub(Scout.doc.lib.scout.find + "/", '')
|
|
31
31
|
} * "\n"
|
|
32
32
|
else
|
data/scout_commands/entity
CHANGED
|
@@ -21,7 +21,7 @@ type, property, entity = ARGV
|
|
|
21
21
|
|
|
22
22
|
if options[:help] and type.nil?
|
|
23
23
|
if defined? scout_usage
|
|
24
|
-
scout_usage
|
|
24
|
+
scout_usage
|
|
25
25
|
else
|
|
26
26
|
puts SOPT.doc
|
|
27
27
|
end
|
|
@@ -63,12 +63,12 @@ end
|
|
|
63
63
|
entity_module = Kernel.const_get type
|
|
64
64
|
|
|
65
65
|
entity_module.annotations.uniq.each do |attr|
|
|
66
|
-
SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute"
|
|
66
|
+
SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute"
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
if options[:help] and property.nil?
|
|
70
70
|
if defined? scout_usage
|
|
71
|
-
scout_usage
|
|
71
|
+
scout_usage
|
|
72
72
|
else
|
|
73
73
|
puts SOPT.doc
|
|
74
74
|
end
|
|
@@ -99,7 +99,7 @@ if options[:help]
|
|
|
99
99
|
end * " "
|
|
100
100
|
|
|
101
101
|
SOPT.synopsys['[<property_parameters>]*'] = property_str
|
|
102
|
-
|
|
102
|
+
|
|
103
103
|
if defined? scout_usage
|
|
104
104
|
scout_usage
|
|
105
105
|
else
|
data/scout_commands/find
CHANGED
data/scout_commands/kb/config
CHANGED
data/scout_commands/kb/entities
CHANGED
data/scout_commands/kb/list
CHANGED
data/scout_commands/kb/query
CHANGED
|
@@ -21,7 +21,7 @@ $ #{$0} [<options>] <name> <entity>
|
|
|
21
21
|
EOF
|
|
22
22
|
if options[:help]
|
|
23
23
|
if defined? scout_usage
|
|
24
|
-
scout_usage
|
|
24
|
+
scout_usage
|
|
25
25
|
else
|
|
26
26
|
puts SOPT.doc
|
|
27
27
|
end
|
|
@@ -70,7 +70,7 @@ if matches.any?
|
|
|
70
70
|
else
|
|
71
71
|
matches.each do |match|
|
|
72
72
|
puts Log.color :magenta, (reverse ? match.invert : match)
|
|
73
|
-
puts index[match].prety_print
|
|
73
|
+
puts index[match].prety_print
|
|
74
74
|
end
|
|
75
75
|
end
|
|
76
76
|
else
|
data/scout_commands/kb/register
CHANGED
data/scout_commands/kb/show
CHANGED
data/scout_commands/kb/traverse
CHANGED
data/scout_commands/log
CHANGED
|
@@ -23,7 +23,7 @@ NONE
|
|
|
23
23
|
EOF
|
|
24
24
|
if options[:help]
|
|
25
25
|
if defined? scout_usage
|
|
26
|
-
scout_usage
|
|
26
|
+
scout_usage
|
|
27
27
|
else
|
|
28
28
|
puts SOPT.doc
|
|
29
29
|
end
|
|
@@ -31,11 +31,11 @@ if options[:help]
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
if ARGV.empty?
|
|
34
|
-
if Scout.etc.log_severity.exists?
|
|
35
|
-
puts Scout.etc.log_severity.read
|
|
36
|
-
else
|
|
37
|
-
puts Scout.etc.log_severity.find + ' does not exist'
|
|
38
|
-
end
|
|
34
|
+
if Scout.etc.log_severity.exists?
|
|
35
|
+
puts Scout.etc.log_severity.read
|
|
36
|
+
else
|
|
37
|
+
puts Scout.etc.log_severity.find + ' does not exist'
|
|
38
|
+
end
|
|
39
39
|
else
|
|
40
40
|
level = ARGV[0]
|
|
41
41
|
level = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE)[level.to_i] if level =~ /^\d+$/
|
|
@@ -8,7 +8,7 @@ options = SOPT.setup <<EOF
|
|
|
8
8
|
|
|
9
9
|
Produce a resource
|
|
10
10
|
|
|
11
|
-
$ #{$0} [<options>] <Resource> <file>
|
|
11
|
+
$ #{$0} [<options>] <Resource> <file>
|
|
12
12
|
|
|
13
13
|
-h--help Print this help
|
|
14
14
|
-W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
|
|
@@ -17,7 +17,7 @@ $ #{$0} [<options>] <Resource> <file>
|
|
|
17
17
|
EOF
|
|
18
18
|
if options[:help]
|
|
19
19
|
if defined? scout_usage
|
|
20
|
-
scout_usage
|
|
20
|
+
scout_usage
|
|
21
21
|
else
|
|
22
22
|
puts SOPT.doc
|
|
23
23
|
end
|
data/scout_commands/system/clean
CHANGED
|
@@ -47,7 +47,7 @@ if locks.any?
|
|
|
47
47
|
locks.each do |file,info|
|
|
48
48
|
if force or (info[:pid] && ! Misc.pid_alive?(info[:pid]))
|
|
49
49
|
puts " Removing #{ file }"
|
|
50
|
-
File.unlink file
|
|
50
|
+
File.unlink file
|
|
51
51
|
end
|
|
52
52
|
end
|
|
53
53
|
end
|
|
@@ -59,7 +59,7 @@ if persists.any?
|
|
|
59
59
|
persists.each do |file,info|
|
|
60
60
|
if force or (info[:pid] and Misc.pid_alive? info[:pid])
|
|
61
61
|
puts " Removing #{ file }"
|
|
62
|
-
File.unlink file
|
|
62
|
+
File.unlink file
|
|
63
63
|
end
|
|
64
64
|
end
|
|
65
65
|
end
|
|
@@ -71,7 +71,7 @@ if sensiblewrites.any?
|
|
|
71
71
|
sensiblewrites.each do |file,info|
|
|
72
72
|
if force or (info[:pid] and Misc.pid_alive? info[:pid])
|
|
73
73
|
puts " Removing #{ file }"
|
|
74
|
-
File.unlink file
|
|
74
|
+
File.unlink file
|
|
75
75
|
end
|
|
76
76
|
end
|
|
77
77
|
end
|
|
@@ -134,13 +134,13 @@ TSV.traverse jobs do |file,i|
|
|
|
134
134
|
end
|
|
135
135
|
end
|
|
136
136
|
|
|
137
|
-
if (force && status !~ /done/) or
|
|
137
|
+
if (force && status !~ /done/) or
|
|
138
138
|
status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync|waiting)$/ or
|
|
139
139
|
(status == "noinfo" and not done) or
|
|
140
|
-
status == ""
|
|
140
|
+
status == ""
|
|
141
141
|
|
|
142
|
-
puts " Removing #{ file } - #{status}"
|
|
143
|
-
Step.clean(file)
|
|
142
|
+
puts " Removing #{ file } - #{status}"
|
|
143
|
+
Step.clean(file)
|
|
144
144
|
end
|
|
145
145
|
end
|
|
146
146
|
|
|
@@ -177,7 +177,7 @@ workflows.sort.each do |workflow,tasks|
|
|
|
177
177
|
if options[:quick] and i[:done]
|
|
178
178
|
status = 'done'
|
|
179
179
|
str << " #{ Step.prov_status_msg status }"
|
|
180
|
-
if inputs and inputs.any?
|
|
180
|
+
if inputs and inputs.any?
|
|
181
181
|
str << input_msg(file, inputs)
|
|
182
182
|
end
|
|
183
183
|
|
|
@@ -210,15 +210,15 @@ workflows.sort.each do |workflow,tasks|
|
|
|
210
210
|
status = status.to_s
|
|
211
211
|
if status != "done" and pid and not Misc.pid_alive?(pid)
|
|
212
212
|
if File.exist? file
|
|
213
|
-
status << Log.color(:red, " (out of sync)")
|
|
213
|
+
status << Log.color(:red, " (out of sync)")
|
|
214
214
|
else
|
|
215
|
-
status << Log.color(:red, " (dead)")
|
|
215
|
+
status << Log.color(:red, " (dead)")
|
|
216
216
|
end
|
|
217
217
|
end
|
|
218
218
|
str << " #{ Step.prov_status_msg status }"
|
|
219
219
|
str << " (dirty)" if status == 'done' && Step.new(file).dirty?
|
|
220
220
|
|
|
221
|
-
if inputs and inputs.any?
|
|
221
|
+
if inputs and inputs.any?
|
|
222
222
|
str << input_msg(file, inputs)
|
|
223
223
|
end
|
|
224
224
|
|
data/scout_commands/template
CHANGED
data/scout_commands/update
CHANGED
|
@@ -8,13 +8,13 @@ options = SOPT.setup <<EOF
|
|
|
8
8
|
|
|
9
9
|
List all workflows
|
|
10
10
|
|
|
11
|
-
$ #{$0} [<options>]
|
|
11
|
+
$ #{$0} [<options>]
|
|
12
12
|
|
|
13
13
|
-h--help Print this help
|
|
14
14
|
EOF
|
|
15
15
|
if options[:help]
|
|
16
16
|
if defined? scout_usage
|
|
17
|
-
scout_usage
|
|
17
|
+
scout_usage
|
|
18
18
|
else
|
|
19
19
|
puts SOPT.doc
|
|
20
20
|
end
|
|
@@ -22,7 +22,7 @@ $ #{$0} [<options>] ([<workflow>] [<task>] [<name>] | <filename>)
|
|
|
22
22
|
EOF
|
|
23
23
|
if options[:help]
|
|
24
24
|
if defined? scout_usage
|
|
25
|
-
scout_usage
|
|
25
|
+
scout_usage
|
|
26
26
|
else
|
|
27
27
|
puts SOPT.doc
|
|
28
28
|
end
|
|
@@ -69,7 +69,7 @@ begin
|
|
|
69
69
|
end
|
|
70
70
|
|
|
71
71
|
jobs = files.collect{|file| Workflow.queue_job(file) }
|
|
72
|
-
|
|
72
|
+
|
|
73
73
|
|
|
74
74
|
begin
|
|
75
75
|
Workflow.produce(jobs, **options)
|