scout-gear 8.1.0 → 9.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +22 -0
- data/VERSION +1 -1
- data/bin/scout +1 -0
- data/lib/rbbt-scout.rb +2 -1
- data/lib/scout/cmd.rb +9 -5
- data/lib/scout/indiferent_hash.rb +17 -0
- data/lib/scout/log/progress/report.rb +1 -0
- data/lib/scout/log.rb +14 -11
- data/lib/scout/misc/filesystem.rb +2 -3
- data/lib/scout/offsite/ssh.rb +3 -0
- data/lib/scout/offsite/step.rb +20 -3
- data/lib/scout/open/stream.rb +0 -1
- data/lib/scout/open.rb +8 -8
- data/lib/scout/path.rb +1 -1
- data/lib/scout/persist/serialize.rb +1 -1
- data/lib/scout/resource/open.rb +8 -0
- data/lib/scout/resource/path.rb +12 -9
- data/lib/scout/resource/software.rb +4 -2
- data/lib/scout/resource.rb +2 -0
- data/lib/scout/tsv/dumper.rb +3 -1
- data/lib/scout/tsv/parser.rb +13 -3
- data/lib/scout/work_queue.rb +1 -0
- data/lib/scout/workflow/deployment/orchestrator.rb +17 -8
- data/lib/scout/workflow/step/dependencies.rb +9 -3
- data/lib/scout/workflow/step/info.rb +8 -0
- data/lib/scout/workflow/step/inputs.rb +5 -0
- data/lib/scout/workflow/step/status.rb +22 -2
- data/lib/scout/workflow/step.rb +11 -4
- data/lib/scout/workflow/task/dependencies.rb +2 -0
- data/lib/scout/workflow/task/inputs.rb +14 -9
- data/lib/scout/workflow/task.rb +4 -2
- data/lib/scout/workflow/usage.rb +21 -33
- data/lib/scout/workflow.rb +17 -13
- data/scout-gear.gemspec +9 -3
- data/scout_commands/resource/produce +66 -0
- data/scout_commands/template +52 -0
- data/scout_commands/workflow/install +3 -0
- data/scout_commands/workflow/task +24 -5
- data/share/software/install_helpers +2 -2
- data/share/templates/command +25 -0
- data/share/templates/workflow.rb +14 -0
- data/test/scout/offsite/test_step.rb +0 -1
- data/test/scout/test_persist.rb +2 -2
- data/test/scout/test_work_queue.rb +1 -1
- data/test/scout/tsv/test_parser.rb +21 -0
- data/test/scout/workflow/step/test_info.rb +0 -1
- data/test/scout/workflow/task/test_dependencies.rb +2 -0
- data/test/scout/workflow/task/test_inputs.rb +0 -1
- data/test/scout/workflow/test_definition.rb +1 -1
- metadata +8 -2
@@ -1,6 +1,6 @@
|
|
1
1
|
class Step
|
2
2
|
def abort(exception = nil)
|
3
|
-
if info[:pid] != Process.pid && Misc.
|
3
|
+
if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
|
4
4
|
Process.kill pid
|
5
5
|
else
|
6
6
|
while @result && streaming? && stream = self.stream
|
@@ -15,7 +15,7 @@ class Step
|
|
15
15
|
|
16
16
|
def updated?
|
17
17
|
return false if self.error? && self.recoverable_error?
|
18
|
-
return true
|
18
|
+
return true if self.done? && ! ENV["SCOUT_UPDATE"]
|
19
19
|
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
20
20
|
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
21
21
|
|
@@ -28,6 +28,7 @@ class Step
|
|
28
28
|
@info = nil
|
29
29
|
@info_load_time = nil
|
30
30
|
Open.rm path if Open.exist?(path)
|
31
|
+
Open.rm tmp_path if Open.exist?(tmp_path)
|
31
32
|
Open.rm info_file if Open.exist?(info_file)
|
32
33
|
Open.rm_rf files_dir if Open.exist?(files_dir)
|
33
34
|
end
|
@@ -40,4 +41,23 @@ class Step
|
|
40
41
|
clean
|
41
42
|
end
|
42
43
|
|
44
|
+
def canfail?
|
45
|
+
@compute && @compute.include?(:canfail)
|
46
|
+
end
|
47
|
+
|
48
|
+
def started?
|
49
|
+
return true if done?
|
50
|
+
return false unless Open.exist?(info_file)
|
51
|
+
pid = info[:pid]
|
52
|
+
return false unless pid
|
53
|
+
return Misc.pid_alive?(pid)
|
54
|
+
end
|
55
|
+
|
56
|
+
def waiting?
|
57
|
+
present? and not started?
|
58
|
+
end
|
59
|
+
|
60
|
+
def dirty?
|
61
|
+
done? && ! updated?
|
62
|
+
end
|
43
63
|
end
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -8,16 +8,18 @@ require_relative 'step/dependencies'
|
|
8
8
|
require_relative 'step/provenance'
|
9
9
|
require_relative 'step/config'
|
10
10
|
require_relative 'step/progress'
|
11
|
+
require_relative 'step/inputs'
|
11
12
|
|
12
13
|
class Step
|
13
14
|
|
14
|
-
attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
|
15
|
-
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task)
|
15
|
+
attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
|
16
|
+
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
|
16
17
|
@path = path
|
17
18
|
@inputs = inputs
|
18
19
|
@dependencies = dependencies
|
19
20
|
@id = id
|
20
21
|
@non_default_inputs = non_default_inputs
|
22
|
+
@provided_inputs = provided_inputs
|
21
23
|
@compute = compute
|
22
24
|
@task = task
|
23
25
|
@mutex = Mutex.new
|
@@ -209,7 +211,7 @@ class Step
|
|
209
211
|
if done?
|
210
212
|
Open.open(self.path)
|
211
213
|
else
|
212
|
-
if running?
|
214
|
+
if running? || waiting?
|
213
215
|
join
|
214
216
|
Open.open(self.path)
|
215
217
|
else
|
@@ -251,7 +253,7 @@ class Step
|
|
251
253
|
|
252
254
|
def join
|
253
255
|
consume_all_streams
|
254
|
-
while @result.nil? && (present? && ! terminated?)
|
256
|
+
while @result.nil? && (present? && ! (terminated? || done?))
|
255
257
|
sleep 0.1
|
256
258
|
end
|
257
259
|
raise self.exception if self.exception
|
@@ -278,6 +280,7 @@ class Step
|
|
278
280
|
def step(task_name)
|
279
281
|
dependencies.each do |dep|
|
280
282
|
return dep if dep.task_name == task_name
|
283
|
+
return dep if dep.overriden_task == task_name
|
281
284
|
rec_dep = dep.step(task_name)
|
282
285
|
return rec_dep if rec_dep
|
283
286
|
end
|
@@ -295,4 +298,8 @@ class Step
|
|
295
298
|
def fingerprint
|
296
299
|
digest_str
|
297
300
|
end
|
301
|
+
|
302
|
+
def task_signature
|
303
|
+
[workflow.to_s, task_name] * "#"
|
304
|
+
end
|
298
305
|
end
|
@@ -55,6 +55,8 @@ module Task
|
|
55
55
|
dep = provided_inputs[overriden]
|
56
56
|
dep = Step.new dep unless Step === dep
|
57
57
|
dep.type = workflow.tasks[task].type
|
58
|
+
dep.overriden_task = task
|
59
|
+
dep.overriden_workflow = workflow
|
58
60
|
dependencies << dep
|
59
61
|
non_default_inputs << overriden
|
60
62
|
next
|
@@ -20,7 +20,7 @@ module Task
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def assign_inputs(provided_inputs = {})
|
23
|
+
def assign_inputs(provided_inputs = {}, id = nil)
|
24
24
|
if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
|
25
25
|
case provided_inputs
|
26
26
|
when Array
|
@@ -43,6 +43,8 @@ module Task
|
|
43
43
|
if ! provided.nil? && provided != value
|
44
44
|
non_default_inputs << name.to_sym
|
45
45
|
input_array << provided
|
46
|
+
elsif options && options[:jobname]
|
47
|
+
input_array << id
|
46
48
|
else
|
47
49
|
input_array << value
|
48
50
|
end
|
@@ -53,13 +55,14 @@ module Task
|
|
53
55
|
[input_array, non_default_inputs]
|
54
56
|
end
|
55
57
|
|
56
|
-
def process_inputs(provided_inputs = {})
|
57
|
-
input_array, non_default_inputs = assign_inputs provided_inputs
|
58
|
+
def process_inputs(provided_inputs = {}, id = nil)
|
59
|
+
input_array, non_default_inputs = assign_inputs provided_inputs, id
|
58
60
|
digest_str = Misc.digest_str(input_array)
|
59
61
|
[input_array, non_default_inputs, digest_str]
|
60
62
|
end
|
61
63
|
|
62
64
|
def save_file_input(orig_file, directory)
|
65
|
+
orig_file = orig_file.path if Step === orig_file
|
63
66
|
basename = File.basename(orig_file)
|
64
67
|
digest = Misc.digest(orig_file)
|
65
68
|
if basename.include? '.'
|
@@ -81,7 +84,10 @@ module Task
|
|
81
84
|
value = provided_inputs[name]
|
82
85
|
input_file = File.join(directory, name.to_s)
|
83
86
|
|
84
|
-
if
|
87
|
+
if Path.is_filename?(value)
|
88
|
+
relative_file = save_file_input(value, directory)
|
89
|
+
Open.write(input_file + ".as_file", relative_file)
|
90
|
+
elsif type == :file
|
85
91
|
relative_file = save_file_input(value, directory)
|
86
92
|
Persist.save(relative_file, input_file, :file)
|
87
93
|
elsif type == :file_array
|
@@ -89,13 +95,10 @@ module Task
|
|
89
95
|
save_file_input(orig_file, directory)
|
90
96
|
end
|
91
97
|
Persist.save(new_files, input_file, type)
|
92
|
-
elsif Path.is_filename?(value)
|
93
|
-
relative_file = save_file_input(value, directory)
|
94
|
-
Open.write(input_file + ".as_file", relative_file)
|
95
98
|
elsif Open.is_stream?(value)
|
96
|
-
Persist.save(
|
99
|
+
Persist.save(value, input_file, type)
|
97
100
|
elsif Open.has_stream?(value)
|
98
|
-
Persist.save(
|
101
|
+
Persist.save(value.stream, input_file, type)
|
99
102
|
else
|
100
103
|
Persist.save(value, input_file, type)
|
101
104
|
end
|
@@ -112,6 +115,8 @@ module Task
|
|
112
115
|
value = Open.read(filename).strip
|
113
116
|
value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
|
114
117
|
inputs[name] = value
|
118
|
+
elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
|
119
|
+
inputs[name] = filename
|
115
120
|
else
|
116
121
|
inputs[name] = Persist.load(filename, type)
|
117
122
|
end
|
data/lib/scout/workflow/task.rb
CHANGED
@@ -42,11 +42,12 @@ module Task
|
|
42
42
|
def job(id = DEFAULT_NAME, provided_inputs = nil )
|
43
43
|
provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
|
44
44
|
provided_inputs = {} if provided_inputs.nil?
|
45
|
+
IndiferentHash.setup(provided_inputs)
|
45
46
|
id = DEFAULT_NAME if id.nil?
|
46
47
|
|
47
48
|
provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
|
48
49
|
|
49
|
-
inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
|
50
|
+
inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
|
50
51
|
|
51
52
|
compute = {}
|
52
53
|
dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
|
@@ -90,7 +91,8 @@ module Task
|
|
90
91
|
Log.debug "ID #{self.name} #{id} - Clean"
|
91
92
|
end
|
92
93
|
NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
|
93
|
-
|
94
|
+
step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
|
95
|
+
Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
|
94
96
|
end
|
95
97
|
end
|
96
98
|
end
|
data/lib/scout/workflow/usage.rb
CHANGED
@@ -13,28 +13,30 @@ module Task
|
|
13
13
|
if paragraph
|
14
14
|
str.puts "\n" << Misc.format_paragraph(paragraph)
|
15
15
|
end
|
16
|
-
str.puts
|
17
16
|
else
|
18
17
|
title = self.name.to_s
|
19
18
|
str.puts Log.color :yellow, title
|
20
19
|
str.puts Log.color :yellow, "-" * title.length
|
21
20
|
str.puts "\n" << Misc.format_paragraph(description)
|
22
|
-
str.puts
|
23
21
|
end
|
24
22
|
else
|
25
23
|
title = self.name.to_s
|
26
24
|
str.puts Log.color :yellow, title
|
27
25
|
str.puts Log.color :yellow, "-" * title.length
|
28
|
-
str.puts
|
29
26
|
end
|
30
27
|
|
31
28
|
|
32
29
|
selects = []
|
30
|
+
|
33
31
|
if inputs && inputs.any?
|
32
|
+
str.puts
|
34
33
|
str.puts Log.color(:magenta, "Inputs")
|
35
34
|
str.puts
|
36
35
|
str.puts SOPT.input_array_doc(inputs)
|
37
|
-
|
36
|
+
|
37
|
+
inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
|
38
|
+
selects << [name, options[:select_options]] if options[:select_options]
|
39
|
+
end
|
38
40
|
end
|
39
41
|
|
40
42
|
deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
|
@@ -44,14 +46,21 @@ module Task
|
|
44
46
|
deps.each do |dep_workflow,task_name,options|
|
45
47
|
next if task_name.nil?
|
46
48
|
task = dep_workflow.tasks[task_name]
|
49
|
+
|
47
50
|
next if task.inputs.nil?
|
51
|
+
|
48
52
|
inputs = task.inputs.reject{|name, _| seen.include? name }
|
49
53
|
inputs = task.inputs.reject{|name, _| options.include? name }
|
50
54
|
next unless inputs.any?
|
55
|
+
task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
|
56
|
+
selects << [i, options[:select_options]] if options[:select_options]
|
57
|
+
end
|
58
|
+
|
51
59
|
dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
|
52
60
|
dep_inputs[dep] = inputs
|
53
61
|
end
|
54
62
|
|
63
|
+
str.puts
|
55
64
|
str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
|
56
65
|
dep_inputs.each do |dep,inputs|
|
57
66
|
str.puts
|
@@ -59,54 +68,31 @@ module Task
|
|
59
68
|
str.puts
|
60
69
|
str.puts SOPT.input_array_doc(inputs)
|
61
70
|
end
|
62
|
-
|
63
|
-
#task_inputs = dep_inputs deps, workflow
|
64
|
-
#task_inputs.each do |task,new_inputs|
|
65
|
-
# new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t|
|
66
|
-
# t.to_sym == :select and task.input_options[i][:select_options]
|
67
|
-
# end.each do |i,t|
|
68
|
-
# selects << [i, task.input_options[i][:select_options]]
|
69
|
-
# end
|
70
|
-
|
71
|
-
# next if new_inputs.empty?
|
72
|
-
|
73
|
-
# if task.workflow and task.workflow != workflow
|
74
|
-
# puts " #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
|
75
|
-
# else
|
76
|
-
# puts " #{Log.color :yellow, task.name.to_s}:"
|
77
|
-
# end
|
78
|
-
# puts unless Log.compact
|
79
|
-
# puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
|
80
|
-
# puts unless Log.compact
|
81
|
-
#end
|
82
|
-
#puts
|
83
71
|
end
|
84
72
|
|
85
73
|
case
|
86
74
|
when inputs && inputs.select{|name,type| type == :array }.any?
|
75
|
+
str.puts
|
87
76
|
str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
|
88
77
|
also works in addition to the others. You may use the '--array_separator' option
|
89
78
|
the change this default. Whenever a file is specified it may also accept STDIN using
|
90
79
|
the '-' character."))
|
91
|
-
str.puts
|
92
80
|
|
93
81
|
when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
|
94
|
-
str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
95
82
|
str.puts
|
83
|
+
str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
96
84
|
end
|
97
85
|
|
98
86
|
str.puts
|
99
87
|
str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
|
100
|
-
str.puts
|
101
88
|
|
102
89
|
if selects.any?
|
103
|
-
str.puts Log.color(:magenta, "Input select options")
|
104
90
|
str.puts
|
91
|
+
str.puts Log.color(:magenta, "Input select options")
|
105
92
|
selects.collect{|p| p}.uniq.each do |input,options|
|
93
|
+
str.puts
|
106
94
|
str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
107
|
-
str.puts unless Log.compact
|
108
95
|
end
|
109
|
-
str.puts
|
110
96
|
end
|
111
97
|
str.rewind
|
112
98
|
str.read
|
@@ -249,7 +235,9 @@ module Workflow
|
|
249
235
|
|
250
236
|
str.puts
|
251
237
|
|
252
|
-
if
|
238
|
+
if tasks.nil?
|
239
|
+
str.puts Log.color(:title, "No tasks")
|
240
|
+
elsif task.nil?
|
253
241
|
|
254
242
|
if self.documentation[:description] and not self.documentation[:description].empty?
|
255
243
|
str.puts Misc.format_paragraph self.documentation[:description]
|
@@ -284,7 +272,7 @@ module Workflow
|
|
284
272
|
|
285
273
|
prov_string = prov_string(dep_tree(name))
|
286
274
|
str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
|
287
|
-
end
|
275
|
+
end
|
288
276
|
|
289
277
|
else
|
290
278
|
|
data/lib/scout/workflow.rb
CHANGED
@@ -25,20 +25,24 @@ module Workflow
|
|
25
25
|
base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
|
26
26
|
end
|
27
27
|
|
28
|
-
def self.require_workflow(
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
28
|
+
def self.require_workflow(workflow_name_orig)
|
29
|
+
first = nil
|
30
|
+
workflow_name_orig.split("+").each do |workflow_name|
|
31
|
+
workflow = workflow_name
|
32
|
+
workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
|
33
|
+
workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
|
34
|
+
workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
|
35
|
+
if Open.exists?(workflow)
|
36
|
+
self.main = nil
|
37
|
+
workflow = workflow.find if Path === workflow
|
38
|
+
$LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
|
39
|
+
load workflow
|
40
|
+
else
|
41
|
+
raise "Workflow #{workflow_name} not found"
|
42
|
+
end
|
43
|
+
first ||= self.main || workflows.last
|
40
44
|
end
|
41
|
-
|
45
|
+
first
|
42
46
|
end
|
43
47
|
|
44
48
|
def job(name, *args)
|
data/scout-gear.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear
|
5
|
+
# stub: scout-gear 9.0.0 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "
|
9
|
+
s.version = "9.0.0"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2023-06-
|
14
|
+
s.date = "2023-06-27"
|
15
15
|
s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
@@ -77,6 +77,7 @@ Gem::Specification.new do |s|
|
|
77
77
|
"lib/scout/persist/path.rb",
|
78
78
|
"lib/scout/persist/serialize.rb",
|
79
79
|
"lib/scout/resource.rb",
|
80
|
+
"lib/scout/resource/open.rb",
|
80
81
|
"lib/scout/resource/path.rb",
|
81
82
|
"lib/scout/resource/produce.rb",
|
82
83
|
"lib/scout/resource/produce/rake.rb",
|
@@ -126,6 +127,7 @@ Gem::Specification.new do |s|
|
|
126
127
|
"lib/scout/workflow/step/dependencies.rb",
|
127
128
|
"lib/scout/workflow/step/file.rb",
|
128
129
|
"lib/scout/workflow/step/info.rb",
|
130
|
+
"lib/scout/workflow/step/inputs.rb",
|
129
131
|
"lib/scout/workflow/step/load.rb",
|
130
132
|
"lib/scout/workflow/step/progress.rb",
|
131
133
|
"lib/scout/workflow/step/provenance.rb",
|
@@ -143,6 +145,8 @@ Gem::Specification.new do |s|
|
|
143
145
|
"scout_commands/glob",
|
144
146
|
"scout_commands/offsite",
|
145
147
|
"scout_commands/rbbt",
|
148
|
+
"scout_commands/resource/produce",
|
149
|
+
"scout_commands/template",
|
146
150
|
"scout_commands/update",
|
147
151
|
"scout_commands/workflow/info",
|
148
152
|
"scout_commands/workflow/install",
|
@@ -151,6 +155,8 @@ Gem::Specification.new do |s|
|
|
151
155
|
"share/color/color_names",
|
152
156
|
"share/color/diverging_colors.hex",
|
153
157
|
"share/software/install_helpers",
|
158
|
+
"share/templates/command",
|
159
|
+
"share/templates/workflow.rb",
|
154
160
|
"test/scout/indiferent_hash/test_case_insensitive.rb",
|
155
161
|
"test/scout/indiferent_hash/test_options.rb",
|
156
162
|
"test/scout/log/test_color.rb",
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Produce a resource
|
10
|
+
|
11
|
+
$ #{$0} [<options>] <Resource> <file>
|
12
|
+
|
13
|
+
-h--help Print this help
|
14
|
+
-W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
|
15
|
+
-r--requires* Files to require; 'all' for all in Scout.etc.requires:
|
16
|
+
-f--force Force the production if the file is already present
|
17
|
+
EOF
|
18
|
+
if options[:help]
|
19
|
+
if defined? scout_usage
|
20
|
+
scout_usage
|
21
|
+
else
|
22
|
+
puts SOPT.doc
|
23
|
+
end
|
24
|
+
exit 0
|
25
|
+
end
|
26
|
+
|
27
|
+
case options[:workflows]
|
28
|
+
when nil, false, "false", "none"
|
29
|
+
when "all"
|
30
|
+
Scout.etc.workflows.list.each do |workflow|
|
31
|
+
Workflow.require_workflow file
|
32
|
+
end if Scout.etc.workflows.exists?
|
33
|
+
else
|
34
|
+
options[:workflows].split(/[ ,;|]/).each do |workflow|
|
35
|
+
Workflow.require_workflow workflow
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
case options[:requires]
|
40
|
+
when nil, false, "false", "none"
|
41
|
+
when "all"
|
42
|
+
Scout.etc.requires.list.each do |file|
|
43
|
+
require file
|
44
|
+
end if Scout.etc.requires.exists?
|
45
|
+
else
|
46
|
+
options[:requires].split(/[ ,;|]/).each do |file|
|
47
|
+
require file
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
resource, path = ARGV
|
52
|
+
|
53
|
+
begin
|
54
|
+
resource = Kernel.const_get(resource)
|
55
|
+
rescue
|
56
|
+
begin
|
57
|
+
resource = Workflow.require_workflow resource
|
58
|
+
rescue
|
59
|
+
raise "Resource not found: #{ resource }"
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
force = options[:force]
|
64
|
+
|
65
|
+
puts resource[path].produce(force).find
|
66
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Print a template
|
10
|
+
|
11
|
+
$ #{$0} [<options>] <template> [<VAR>=<value>]*
|
12
|
+
|
13
|
+
Print a template making the substitutions that follow
|
14
|
+
|
15
|
+
-h--help Print this help
|
16
|
+
-s--sub* List a substitutions as a parameter instead
|
17
|
+
-s--var* List a substitutions as a parameter, variable name
|
18
|
+
-s--value* List a substitutions as a parameter, value
|
19
|
+
EOF
|
20
|
+
if options[:help]
|
21
|
+
if defined? scout_usage
|
22
|
+
scout_usage
|
23
|
+
else
|
24
|
+
puts SOPT.doc
|
25
|
+
end
|
26
|
+
exit 0
|
27
|
+
end
|
28
|
+
|
29
|
+
template, *subs = ARGV
|
30
|
+
|
31
|
+
raise MissingParameterException, :template if template.nil?
|
32
|
+
|
33
|
+
txt = Scout.share.templates[template].find_with_extension('rb').read
|
34
|
+
|
35
|
+
if options[:sub]
|
36
|
+
subs ||= []
|
37
|
+
subs << options[:sub]
|
38
|
+
end
|
39
|
+
|
40
|
+
if options[:var]
|
41
|
+
subs ||= []
|
42
|
+
subs << [options[:var], options[:value]] * "="
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
subs.each do |sub|
|
47
|
+
key, value = sub.split("=")
|
48
|
+
|
49
|
+
txt.gsub!(key, value)
|
50
|
+
end
|
51
|
+
|
52
|
+
puts txt
|
@@ -49,6 +49,7 @@ Misc.in_dir(workflow_dir) do
|
|
49
49
|
Log.info "Updating: " + workflow
|
50
50
|
Misc.in_dir(Misc.snake_case(workflow)) do
|
51
51
|
`git pull`
|
52
|
+
`git checkout scout 2> /dev/null`
|
52
53
|
`git submodule init`
|
53
54
|
`git submodule update`
|
54
55
|
end
|
@@ -56,6 +57,7 @@ Misc.in_dir(workflow_dir) do
|
|
56
57
|
Misc.in_dir(workflow) do
|
57
58
|
Log.info "Updating: " + workflow
|
58
59
|
`git pull`
|
60
|
+
`git checkout scout 2> /dev/null`
|
59
61
|
`git submodule init`
|
60
62
|
`git submodule update`
|
61
63
|
end
|
@@ -93,6 +95,7 @@ Misc.in_dir(workflow_dir) do
|
|
93
95
|
end
|
94
96
|
Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
|
95
97
|
Misc.in_dir(Misc.snake_case(workflow)) do
|
98
|
+
`git checkout scout 2> /dev/null`
|
96
99
|
`git submodule init`
|
97
100
|
`git submodule update`
|
98
101
|
end
|
@@ -18,7 +18,7 @@ $ #{$0} [<options>] <workflow> <task>
|
|
18
18
|
-h--help Print this help
|
19
19
|
--nostream Disable job streaming
|
20
20
|
--update Update jobs with newer dependencies
|
21
|
-
-jn--
|
21
|
+
-jn--jobname* Name to use as job identifier
|
22
22
|
-li--load_inputs* Directory with inputs files to load
|
23
23
|
-pf--print_filepath Print the file path
|
24
24
|
-prov--provenance Print the step provenance
|
@@ -26,6 +26,7 @@ $ #{$0} [<options>] <workflow> <task>
|
|
26
26
|
-rcl--recursive_clean Clean all steps
|
27
27
|
-ct--clean_task* Clean a particular task
|
28
28
|
-d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
|
29
|
+
-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
|
29
30
|
EOF
|
30
31
|
|
31
32
|
workflow_name, task_name = ARGV
|
@@ -38,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
|
|
38
39
|
|
39
40
|
options[:help] = true if task.nil?
|
40
41
|
|
41
|
-
help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options,
|
42
|
-
:help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :
|
42
|
+
help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options,
|
43
|
+
:help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
|
43
44
|
:deploy => 'serial'
|
44
45
|
|
45
46
|
if help
|
@@ -59,6 +60,13 @@ else
|
|
59
60
|
job_inputs = task.get_SOPT
|
60
61
|
end
|
61
62
|
|
63
|
+
if override_deps
|
64
|
+
override_deps.split($array_separator || ",").each do |part|
|
65
|
+
t_, value = part.split("=")
|
66
|
+
job_inputs.merge!( t_ => value)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
62
70
|
job = workflow.job(task_name, jobname, job_inputs)
|
63
71
|
|
64
72
|
job.recursive_clean if recursive_clean
|
@@ -79,6 +87,7 @@ if clean_task
|
|
79
87
|
end
|
80
88
|
|
81
89
|
job.clean if job.task_name.to_s == clean_task.to_s
|
90
|
+
job.clean unless job.updated?
|
82
91
|
end
|
83
92
|
end
|
84
93
|
|
@@ -92,9 +101,19 @@ else
|
|
92
101
|
orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
|
93
102
|
orchestrator.process({}, job)
|
94
103
|
when "slurm"
|
95
|
-
|
104
|
+
require 'rbbt-scout'
|
105
|
+
require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
|
106
|
+
HPC::BATCH_MODULE = HPC.batch_system "SLURM"
|
107
|
+
HPC::BATCH_MODULE.orchestrate_job(job, {})
|
108
|
+
job.grace
|
96
109
|
else
|
97
|
-
|
110
|
+
if deploy.end_with?('-slurm')
|
111
|
+
server = deploy.sub('-slurm','')
|
112
|
+
OffsiteStep.setup(job, server: server, slurm: true)
|
113
|
+
else
|
114
|
+
OffsiteStep.setup(job, server: deploy)
|
115
|
+
end
|
116
|
+
|
98
117
|
job.run
|
99
118
|
end unless job.done?
|
100
119
|
|
@@ -32,6 +32,7 @@ else
|
|
32
32
|
local basedir="$(realpath $1)"
|
33
33
|
local path="$(realpath "$2")"
|
34
34
|
local pre=""
|
35
|
+
basedir=${basedir%.}
|
35
36
|
while [[ ! $path = $basedir* ]]; do
|
36
37
|
pre=${pre}../
|
37
38
|
basedir=$(dirname $basedir)
|
@@ -44,9 +45,8 @@ function link(){
|
|
44
45
|
local source="$1"
|
45
46
|
local target="$2"
|
46
47
|
local rel_source=$(relative_path "$(dirname $target)" "$source")
|
47
|
-
|
48
|
+
|
48
49
|
[ -h "$target" ] && rm "$target"
|
49
|
-
echo ln -s "$rel_source" "$target" 1>&2
|
50
50
|
[ -h "$target" ] || ln -s "$rel_source" "$target"
|
51
51
|
}
|
52
52
|
|