scout-gear 8.1.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +22 -0
- data/VERSION +1 -1
- data/bin/scout +1 -0
- data/lib/rbbt-scout.rb +2 -1
- data/lib/scout/cmd.rb +9 -5
- data/lib/scout/indiferent_hash.rb +17 -0
- data/lib/scout/log/progress/report.rb +1 -0
- data/lib/scout/log.rb +14 -11
- data/lib/scout/misc/filesystem.rb +2 -3
- data/lib/scout/offsite/ssh.rb +3 -0
- data/lib/scout/offsite/step.rb +20 -3
- data/lib/scout/open/stream.rb +0 -1
- data/lib/scout/open.rb +8 -8
- data/lib/scout/path.rb +1 -1
- data/lib/scout/persist/serialize.rb +1 -1
- data/lib/scout/resource/open.rb +8 -0
- data/lib/scout/resource/path.rb +12 -9
- data/lib/scout/resource/software.rb +4 -2
- data/lib/scout/resource.rb +2 -0
- data/lib/scout/tsv/dumper.rb +3 -1
- data/lib/scout/tsv/parser.rb +13 -3
- data/lib/scout/work_queue.rb +1 -0
- data/lib/scout/workflow/deployment/orchestrator.rb +17 -8
- data/lib/scout/workflow/step/dependencies.rb +9 -3
- data/lib/scout/workflow/step/info.rb +8 -0
- data/lib/scout/workflow/step/inputs.rb +5 -0
- data/lib/scout/workflow/step/status.rb +22 -2
- data/lib/scout/workflow/step.rb +11 -4
- data/lib/scout/workflow/task/dependencies.rb +2 -0
- data/lib/scout/workflow/task/inputs.rb +14 -9
- data/lib/scout/workflow/task.rb +4 -2
- data/lib/scout/workflow/usage.rb +21 -33
- data/lib/scout/workflow.rb +17 -13
- data/scout-gear.gemspec +9 -3
- data/scout_commands/resource/produce +66 -0
- data/scout_commands/template +52 -0
- data/scout_commands/workflow/install +3 -0
- data/scout_commands/workflow/task +24 -5
- data/share/software/install_helpers +2 -2
- data/share/templates/command +25 -0
- data/share/templates/workflow.rb +14 -0
- data/test/scout/offsite/test_step.rb +0 -1
- data/test/scout/test_persist.rb +2 -2
- data/test/scout/test_work_queue.rb +1 -1
- data/test/scout/tsv/test_parser.rb +21 -0
- data/test/scout/workflow/step/test_info.rb +0 -1
- data/test/scout/workflow/task/test_dependencies.rb +2 -0
- data/test/scout/workflow/task/test_inputs.rb +0 -1
- data/test/scout/workflow/test_definition.rb +1 -1
- metadata +8 -2
@@ -1,6 +1,6 @@
|
|
1
1
|
class Step
|
2
2
|
def abort(exception = nil)
|
3
|
-
if info[:pid] != Process.pid && Misc.
|
3
|
+
if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
|
4
4
|
Process.kill pid
|
5
5
|
else
|
6
6
|
while @result && streaming? && stream = self.stream
|
@@ -15,7 +15,7 @@ class Step
|
|
15
15
|
|
16
16
|
def updated?
|
17
17
|
return false if self.error? && self.recoverable_error?
|
18
|
-
return true
|
18
|
+
return true if self.done? && ! ENV["SCOUT_UPDATE"]
|
19
19
|
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
20
20
|
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
21
21
|
|
@@ -28,6 +28,7 @@ class Step
|
|
28
28
|
@info = nil
|
29
29
|
@info_load_time = nil
|
30
30
|
Open.rm path if Open.exist?(path)
|
31
|
+
Open.rm tmp_path if Open.exist?(tmp_path)
|
31
32
|
Open.rm info_file if Open.exist?(info_file)
|
32
33
|
Open.rm_rf files_dir if Open.exist?(files_dir)
|
33
34
|
end
|
@@ -40,4 +41,23 @@ class Step
|
|
40
41
|
clean
|
41
42
|
end
|
42
43
|
|
44
|
+
def canfail?
|
45
|
+
@compute && @compute.include?(:canfail)
|
46
|
+
end
|
47
|
+
|
48
|
+
def started?
|
49
|
+
return true if done?
|
50
|
+
return false unless Open.exist?(info_file)
|
51
|
+
pid = info[:pid]
|
52
|
+
return false unless pid
|
53
|
+
return Misc.pid_alive?(pid)
|
54
|
+
end
|
55
|
+
|
56
|
+
def waiting?
|
57
|
+
present? and not started?
|
58
|
+
end
|
59
|
+
|
60
|
+
def dirty?
|
61
|
+
done? && ! updated?
|
62
|
+
end
|
43
63
|
end
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -8,16 +8,18 @@ require_relative 'step/dependencies'
|
|
8
8
|
require_relative 'step/provenance'
|
9
9
|
require_relative 'step/config'
|
10
10
|
require_relative 'step/progress'
|
11
|
+
require_relative 'step/inputs'
|
11
12
|
|
12
13
|
class Step
|
13
14
|
|
14
|
-
attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
|
15
|
-
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task)
|
15
|
+
attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
|
16
|
+
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
|
16
17
|
@path = path
|
17
18
|
@inputs = inputs
|
18
19
|
@dependencies = dependencies
|
19
20
|
@id = id
|
20
21
|
@non_default_inputs = non_default_inputs
|
22
|
+
@provided_inputs = provided_inputs
|
21
23
|
@compute = compute
|
22
24
|
@task = task
|
23
25
|
@mutex = Mutex.new
|
@@ -209,7 +211,7 @@ class Step
|
|
209
211
|
if done?
|
210
212
|
Open.open(self.path)
|
211
213
|
else
|
212
|
-
if running?
|
214
|
+
if running? || waiting?
|
213
215
|
join
|
214
216
|
Open.open(self.path)
|
215
217
|
else
|
@@ -251,7 +253,7 @@ class Step
|
|
251
253
|
|
252
254
|
def join
|
253
255
|
consume_all_streams
|
254
|
-
while @result.nil? && (present? && ! terminated?)
|
256
|
+
while @result.nil? && (present? && ! (terminated? || done?))
|
255
257
|
sleep 0.1
|
256
258
|
end
|
257
259
|
raise self.exception if self.exception
|
@@ -278,6 +280,7 @@ class Step
|
|
278
280
|
def step(task_name)
|
279
281
|
dependencies.each do |dep|
|
280
282
|
return dep if dep.task_name == task_name
|
283
|
+
return dep if dep.overriden_task == task_name
|
281
284
|
rec_dep = dep.step(task_name)
|
282
285
|
return rec_dep if rec_dep
|
283
286
|
end
|
@@ -295,4 +298,8 @@ class Step
|
|
295
298
|
def fingerprint
|
296
299
|
digest_str
|
297
300
|
end
|
301
|
+
|
302
|
+
def task_signature
|
303
|
+
[workflow.to_s, task_name] * "#"
|
304
|
+
end
|
298
305
|
end
|
@@ -55,6 +55,8 @@ module Task
|
|
55
55
|
dep = provided_inputs[overriden]
|
56
56
|
dep = Step.new dep unless Step === dep
|
57
57
|
dep.type = workflow.tasks[task].type
|
58
|
+
dep.overriden_task = task
|
59
|
+
dep.overriden_workflow = workflow
|
58
60
|
dependencies << dep
|
59
61
|
non_default_inputs << overriden
|
60
62
|
next
|
@@ -20,7 +20,7 @@ module Task
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def assign_inputs(provided_inputs = {})
|
23
|
+
def assign_inputs(provided_inputs = {}, id = nil)
|
24
24
|
if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
|
25
25
|
case provided_inputs
|
26
26
|
when Array
|
@@ -43,6 +43,8 @@ module Task
|
|
43
43
|
if ! provided.nil? && provided != value
|
44
44
|
non_default_inputs << name.to_sym
|
45
45
|
input_array << provided
|
46
|
+
elsif options && options[:jobname]
|
47
|
+
input_array << id
|
46
48
|
else
|
47
49
|
input_array << value
|
48
50
|
end
|
@@ -53,13 +55,14 @@ module Task
|
|
53
55
|
[input_array, non_default_inputs]
|
54
56
|
end
|
55
57
|
|
56
|
-
def process_inputs(provided_inputs = {})
|
57
|
-
input_array, non_default_inputs = assign_inputs provided_inputs
|
58
|
+
def process_inputs(provided_inputs = {}, id = nil)
|
59
|
+
input_array, non_default_inputs = assign_inputs provided_inputs, id
|
58
60
|
digest_str = Misc.digest_str(input_array)
|
59
61
|
[input_array, non_default_inputs, digest_str]
|
60
62
|
end
|
61
63
|
|
62
64
|
def save_file_input(orig_file, directory)
|
65
|
+
orig_file = orig_file.path if Step === orig_file
|
63
66
|
basename = File.basename(orig_file)
|
64
67
|
digest = Misc.digest(orig_file)
|
65
68
|
if basename.include? '.'
|
@@ -81,7 +84,10 @@ module Task
|
|
81
84
|
value = provided_inputs[name]
|
82
85
|
input_file = File.join(directory, name.to_s)
|
83
86
|
|
84
|
-
if
|
87
|
+
if Path.is_filename?(value)
|
88
|
+
relative_file = save_file_input(value, directory)
|
89
|
+
Open.write(input_file + ".as_file", relative_file)
|
90
|
+
elsif type == :file
|
85
91
|
relative_file = save_file_input(value, directory)
|
86
92
|
Persist.save(relative_file, input_file, :file)
|
87
93
|
elsif type == :file_array
|
@@ -89,13 +95,10 @@ module Task
|
|
89
95
|
save_file_input(orig_file, directory)
|
90
96
|
end
|
91
97
|
Persist.save(new_files, input_file, type)
|
92
|
-
elsif Path.is_filename?(value)
|
93
|
-
relative_file = save_file_input(value, directory)
|
94
|
-
Open.write(input_file + ".as_file", relative_file)
|
95
98
|
elsif Open.is_stream?(value)
|
96
|
-
Persist.save(
|
99
|
+
Persist.save(value, input_file, type)
|
97
100
|
elsif Open.has_stream?(value)
|
98
|
-
Persist.save(
|
101
|
+
Persist.save(value.stream, input_file, type)
|
99
102
|
else
|
100
103
|
Persist.save(value, input_file, type)
|
101
104
|
end
|
@@ -112,6 +115,8 @@ module Task
|
|
112
115
|
value = Open.read(filename).strip
|
113
116
|
value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
|
114
117
|
inputs[name] = value
|
118
|
+
elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
|
119
|
+
inputs[name] = filename
|
115
120
|
else
|
116
121
|
inputs[name] = Persist.load(filename, type)
|
117
122
|
end
|
data/lib/scout/workflow/task.rb
CHANGED
@@ -42,11 +42,12 @@ module Task
|
|
42
42
|
def job(id = DEFAULT_NAME, provided_inputs = nil )
|
43
43
|
provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
|
44
44
|
provided_inputs = {} if provided_inputs.nil?
|
45
|
+
IndiferentHash.setup(provided_inputs)
|
45
46
|
id = DEFAULT_NAME if id.nil?
|
46
47
|
|
47
48
|
provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
|
48
49
|
|
49
|
-
inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
|
50
|
+
inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
|
50
51
|
|
51
52
|
compute = {}
|
52
53
|
dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
|
@@ -90,7 +91,8 @@ module Task
|
|
90
91
|
Log.debug "ID #{self.name} #{id} - Clean"
|
91
92
|
end
|
92
93
|
NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
|
93
|
-
|
94
|
+
step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
|
95
|
+
Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
|
94
96
|
end
|
95
97
|
end
|
96
98
|
end
|
data/lib/scout/workflow/usage.rb
CHANGED
@@ -13,28 +13,30 @@ module Task
|
|
13
13
|
if paragraph
|
14
14
|
str.puts "\n" << Misc.format_paragraph(paragraph)
|
15
15
|
end
|
16
|
-
str.puts
|
17
16
|
else
|
18
17
|
title = self.name.to_s
|
19
18
|
str.puts Log.color :yellow, title
|
20
19
|
str.puts Log.color :yellow, "-" * title.length
|
21
20
|
str.puts "\n" << Misc.format_paragraph(description)
|
22
|
-
str.puts
|
23
21
|
end
|
24
22
|
else
|
25
23
|
title = self.name.to_s
|
26
24
|
str.puts Log.color :yellow, title
|
27
25
|
str.puts Log.color :yellow, "-" * title.length
|
28
|
-
str.puts
|
29
26
|
end
|
30
27
|
|
31
28
|
|
32
29
|
selects = []
|
30
|
+
|
33
31
|
if inputs && inputs.any?
|
32
|
+
str.puts
|
34
33
|
str.puts Log.color(:magenta, "Inputs")
|
35
34
|
str.puts
|
36
35
|
str.puts SOPT.input_array_doc(inputs)
|
37
|
-
|
36
|
+
|
37
|
+
inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
|
38
|
+
selects << [name, options[:select_options]] if options[:select_options]
|
39
|
+
end
|
38
40
|
end
|
39
41
|
|
40
42
|
deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
|
@@ -44,14 +46,21 @@ module Task
|
|
44
46
|
deps.each do |dep_workflow,task_name,options|
|
45
47
|
next if task_name.nil?
|
46
48
|
task = dep_workflow.tasks[task_name]
|
49
|
+
|
47
50
|
next if task.inputs.nil?
|
51
|
+
|
48
52
|
inputs = task.inputs.reject{|name, _| seen.include? name }
|
49
53
|
inputs = task.inputs.reject{|name, _| options.include? name }
|
50
54
|
next unless inputs.any?
|
55
|
+
task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
|
56
|
+
selects << [i, options[:select_options]] if options[:select_options]
|
57
|
+
end
|
58
|
+
|
51
59
|
dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
|
52
60
|
dep_inputs[dep] = inputs
|
53
61
|
end
|
54
62
|
|
63
|
+
str.puts
|
55
64
|
str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
|
56
65
|
dep_inputs.each do |dep,inputs|
|
57
66
|
str.puts
|
@@ -59,54 +68,31 @@ module Task
|
|
59
68
|
str.puts
|
60
69
|
str.puts SOPT.input_array_doc(inputs)
|
61
70
|
end
|
62
|
-
|
63
|
-
#task_inputs = dep_inputs deps, workflow
|
64
|
-
#task_inputs.each do |task,new_inputs|
|
65
|
-
# new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t|
|
66
|
-
# t.to_sym == :select and task.input_options[i][:select_options]
|
67
|
-
# end.each do |i,t|
|
68
|
-
# selects << [i, task.input_options[i][:select_options]]
|
69
|
-
# end
|
70
|
-
|
71
|
-
# next if new_inputs.empty?
|
72
|
-
|
73
|
-
# if task.workflow and task.workflow != workflow
|
74
|
-
# puts " #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
|
75
|
-
# else
|
76
|
-
# puts " #{Log.color :yellow, task.name.to_s}:"
|
77
|
-
# end
|
78
|
-
# puts unless Log.compact
|
79
|
-
# puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
|
80
|
-
# puts unless Log.compact
|
81
|
-
#end
|
82
|
-
#puts
|
83
71
|
end
|
84
72
|
|
85
73
|
case
|
86
74
|
when inputs && inputs.select{|name,type| type == :array }.any?
|
75
|
+
str.puts
|
87
76
|
str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
|
88
77
|
also works in addition to the others. You may use the '--array_separator' option
|
89
78
|
the change this default. Whenever a file is specified it may also accept STDIN using
|
90
79
|
the '-' character."))
|
91
|
-
str.puts
|
92
80
|
|
93
81
|
when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
|
94
|
-
str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
95
82
|
str.puts
|
83
|
+
str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
96
84
|
end
|
97
85
|
|
98
86
|
str.puts
|
99
87
|
str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
|
100
|
-
str.puts
|
101
88
|
|
102
89
|
if selects.any?
|
103
|
-
str.puts Log.color(:magenta, "Input select options")
|
104
90
|
str.puts
|
91
|
+
str.puts Log.color(:magenta, "Input select options")
|
105
92
|
selects.collect{|p| p}.uniq.each do |input,options|
|
93
|
+
str.puts
|
106
94
|
str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
107
|
-
str.puts unless Log.compact
|
108
95
|
end
|
109
|
-
str.puts
|
110
96
|
end
|
111
97
|
str.rewind
|
112
98
|
str.read
|
@@ -249,7 +235,9 @@ module Workflow
|
|
249
235
|
|
250
236
|
str.puts
|
251
237
|
|
252
|
-
if
|
238
|
+
if tasks.nil?
|
239
|
+
str.puts Log.color(:title, "No tasks")
|
240
|
+
elsif task.nil?
|
253
241
|
|
254
242
|
if self.documentation[:description] and not self.documentation[:description].empty?
|
255
243
|
str.puts Misc.format_paragraph self.documentation[:description]
|
@@ -284,7 +272,7 @@ module Workflow
|
|
284
272
|
|
285
273
|
prov_string = prov_string(dep_tree(name))
|
286
274
|
str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
|
287
|
-
end
|
275
|
+
end
|
288
276
|
|
289
277
|
else
|
290
278
|
|
data/lib/scout/workflow.rb
CHANGED
@@ -25,20 +25,24 @@ module Workflow
|
|
25
25
|
base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
|
26
26
|
end
|
27
27
|
|
28
|
-
def self.require_workflow(
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
28
|
+
def self.require_workflow(workflow_name_orig)
|
29
|
+
first = nil
|
30
|
+
workflow_name_orig.split("+").each do |workflow_name|
|
31
|
+
workflow = workflow_name
|
32
|
+
workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
|
33
|
+
workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
|
34
|
+
workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
|
35
|
+
if Open.exists?(workflow)
|
36
|
+
self.main = nil
|
37
|
+
workflow = workflow.find if Path === workflow
|
38
|
+
$LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
|
39
|
+
load workflow
|
40
|
+
else
|
41
|
+
raise "Workflow #{workflow_name} not found"
|
42
|
+
end
|
43
|
+
first ||= self.main || workflows.last
|
40
44
|
end
|
41
|
-
|
45
|
+
first
|
42
46
|
end
|
43
47
|
|
44
48
|
def job(name, *args)
|
data/scout-gear.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear
|
5
|
+
# stub: scout-gear 9.0.0 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "
|
9
|
+
s.version = "9.0.0"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2023-06-
|
14
|
+
s.date = "2023-06-27"
|
15
15
|
s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
@@ -77,6 +77,7 @@ Gem::Specification.new do |s|
|
|
77
77
|
"lib/scout/persist/path.rb",
|
78
78
|
"lib/scout/persist/serialize.rb",
|
79
79
|
"lib/scout/resource.rb",
|
80
|
+
"lib/scout/resource/open.rb",
|
80
81
|
"lib/scout/resource/path.rb",
|
81
82
|
"lib/scout/resource/produce.rb",
|
82
83
|
"lib/scout/resource/produce/rake.rb",
|
@@ -126,6 +127,7 @@ Gem::Specification.new do |s|
|
|
126
127
|
"lib/scout/workflow/step/dependencies.rb",
|
127
128
|
"lib/scout/workflow/step/file.rb",
|
128
129
|
"lib/scout/workflow/step/info.rb",
|
130
|
+
"lib/scout/workflow/step/inputs.rb",
|
129
131
|
"lib/scout/workflow/step/load.rb",
|
130
132
|
"lib/scout/workflow/step/progress.rb",
|
131
133
|
"lib/scout/workflow/step/provenance.rb",
|
@@ -143,6 +145,8 @@ Gem::Specification.new do |s|
|
|
143
145
|
"scout_commands/glob",
|
144
146
|
"scout_commands/offsite",
|
145
147
|
"scout_commands/rbbt",
|
148
|
+
"scout_commands/resource/produce",
|
149
|
+
"scout_commands/template",
|
146
150
|
"scout_commands/update",
|
147
151
|
"scout_commands/workflow/info",
|
148
152
|
"scout_commands/workflow/install",
|
@@ -151,6 +155,8 @@ Gem::Specification.new do |s|
|
|
151
155
|
"share/color/color_names",
|
152
156
|
"share/color/diverging_colors.hex",
|
153
157
|
"share/software/install_helpers",
|
158
|
+
"share/templates/command",
|
159
|
+
"share/templates/workflow.rb",
|
154
160
|
"test/scout/indiferent_hash/test_case_insensitive.rb",
|
155
161
|
"test/scout/indiferent_hash/test_options.rb",
|
156
162
|
"test/scout/log/test_color.rb",
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Produce a resource
|
10
|
+
|
11
|
+
$ #{$0} [<options>] <Resource> <file>
|
12
|
+
|
13
|
+
-h--help Print this help
|
14
|
+
-W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
|
15
|
+
-r--requires* Files to require; 'all' for all in Scout.etc.requires:
|
16
|
+
-f--force Force the production if the file is already present
|
17
|
+
EOF
|
18
|
+
if options[:help]
|
19
|
+
if defined? scout_usage
|
20
|
+
scout_usage
|
21
|
+
else
|
22
|
+
puts SOPT.doc
|
23
|
+
end
|
24
|
+
exit 0
|
25
|
+
end
|
26
|
+
|
27
|
+
case options[:workflows]
|
28
|
+
when nil, false, "false", "none"
|
29
|
+
when "all"
|
30
|
+
Scout.etc.workflows.list.each do |workflow|
|
31
|
+
Workflow.require_workflow file
|
32
|
+
end if Scout.etc.workflows.exists?
|
33
|
+
else
|
34
|
+
options[:workflows].split(/[ ,;|]/).each do |workflow|
|
35
|
+
Workflow.require_workflow workflow
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
case options[:requires]
|
40
|
+
when nil, false, "false", "none"
|
41
|
+
when "all"
|
42
|
+
Scout.etc.requires.list.each do |file|
|
43
|
+
require file
|
44
|
+
end if Scout.etc.requires.exists?
|
45
|
+
else
|
46
|
+
options[:requires].split(/[ ,;|]/).each do |file|
|
47
|
+
require file
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
resource, path = ARGV
|
52
|
+
|
53
|
+
begin
|
54
|
+
resource = Kernel.const_get(resource)
|
55
|
+
rescue
|
56
|
+
begin
|
57
|
+
resource = Workflow.require_workflow resource
|
58
|
+
rescue
|
59
|
+
raise "Resource not found: #{ resource }"
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
force = options[:force]
|
64
|
+
|
65
|
+
puts resource[path].produce(force).find
|
66
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Print a template
|
10
|
+
|
11
|
+
$ #{$0} [<options>] <template> [<VAR>=<value>]*
|
12
|
+
|
13
|
+
Print a template making the substitutions that follow
|
14
|
+
|
15
|
+
-h--help Print this help
|
16
|
+
-s--sub* List a substitutions as a parameter instead
|
17
|
+
-s--var* List a substitutions as a parameter, variable name
|
18
|
+
-s--value* List a substitutions as a parameter, value
|
19
|
+
EOF
|
20
|
+
if options[:help]
|
21
|
+
if defined? scout_usage
|
22
|
+
scout_usage
|
23
|
+
else
|
24
|
+
puts SOPT.doc
|
25
|
+
end
|
26
|
+
exit 0
|
27
|
+
end
|
28
|
+
|
29
|
+
template, *subs = ARGV
|
30
|
+
|
31
|
+
raise MissingParameterException, :template if template.nil?
|
32
|
+
|
33
|
+
txt = Scout.share.templates[template].find_with_extension('rb').read
|
34
|
+
|
35
|
+
if options[:sub]
|
36
|
+
subs ||= []
|
37
|
+
subs << options[:sub]
|
38
|
+
end
|
39
|
+
|
40
|
+
if options[:var]
|
41
|
+
subs ||= []
|
42
|
+
subs << [options[:var], options[:value]] * "="
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
subs.each do |sub|
|
47
|
+
key, value = sub.split("=")
|
48
|
+
|
49
|
+
txt.gsub!(key, value)
|
50
|
+
end
|
51
|
+
|
52
|
+
puts txt
|
@@ -49,6 +49,7 @@ Misc.in_dir(workflow_dir) do
|
|
49
49
|
Log.info "Updating: " + workflow
|
50
50
|
Misc.in_dir(Misc.snake_case(workflow)) do
|
51
51
|
`git pull`
|
52
|
+
`git checkout scout 2> /dev/null`
|
52
53
|
`git submodule init`
|
53
54
|
`git submodule update`
|
54
55
|
end
|
@@ -56,6 +57,7 @@ Misc.in_dir(workflow_dir) do
|
|
56
57
|
Misc.in_dir(workflow) do
|
57
58
|
Log.info "Updating: " + workflow
|
58
59
|
`git pull`
|
60
|
+
`git checkout scout 2> /dev/null`
|
59
61
|
`git submodule init`
|
60
62
|
`git submodule update`
|
61
63
|
end
|
@@ -93,6 +95,7 @@ Misc.in_dir(workflow_dir) do
|
|
93
95
|
end
|
94
96
|
Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
|
95
97
|
Misc.in_dir(Misc.snake_case(workflow)) do
|
98
|
+
`git checkout scout 2> /dev/null`
|
96
99
|
`git submodule init`
|
97
100
|
`git submodule update`
|
98
101
|
end
|
@@ -18,7 +18,7 @@ $ #{$0} [<options>] <workflow> <task>
|
|
18
18
|
-h--help Print this help
|
19
19
|
--nostream Disable job streaming
|
20
20
|
--update Update jobs with newer dependencies
|
21
|
-
-jn--
|
21
|
+
-jn--jobname* Name to use as job identifier
|
22
22
|
-li--load_inputs* Directory with inputs files to load
|
23
23
|
-pf--print_filepath Print the file path
|
24
24
|
-prov--provenance Print the step provenance
|
@@ -26,6 +26,7 @@ $ #{$0} [<options>] <workflow> <task>
|
|
26
26
|
-rcl--recursive_clean Clean all steps
|
27
27
|
-ct--clean_task* Clean a particular task
|
28
28
|
-d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
|
29
|
+
-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
|
29
30
|
EOF
|
30
31
|
|
31
32
|
workflow_name, task_name = ARGV
|
@@ -38,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
|
|
38
39
|
|
39
40
|
options[:help] = true if task.nil?
|
40
41
|
|
41
|
-
help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options,
|
42
|
-
:help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :
|
42
|
+
help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options,
|
43
|
+
:help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
|
43
44
|
:deploy => 'serial'
|
44
45
|
|
45
46
|
if help
|
@@ -59,6 +60,13 @@ else
|
|
59
60
|
job_inputs = task.get_SOPT
|
60
61
|
end
|
61
62
|
|
63
|
+
if override_deps
|
64
|
+
override_deps.split($array_separator || ",").each do |part|
|
65
|
+
t_, value = part.split("=")
|
66
|
+
job_inputs.merge!( t_ => value)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
62
70
|
job = workflow.job(task_name, jobname, job_inputs)
|
63
71
|
|
64
72
|
job.recursive_clean if recursive_clean
|
@@ -79,6 +87,7 @@ if clean_task
|
|
79
87
|
end
|
80
88
|
|
81
89
|
job.clean if job.task_name.to_s == clean_task.to_s
|
90
|
+
job.clean unless job.updated?
|
82
91
|
end
|
83
92
|
end
|
84
93
|
|
@@ -92,9 +101,19 @@ else
|
|
92
101
|
orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
|
93
102
|
orchestrator.process({}, job)
|
94
103
|
when "slurm"
|
95
|
-
|
104
|
+
require 'rbbt-scout'
|
105
|
+
require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
|
106
|
+
HPC::BATCH_MODULE = HPC.batch_system "SLURM"
|
107
|
+
HPC::BATCH_MODULE.orchestrate_job(job, {})
|
108
|
+
job.grace
|
96
109
|
else
|
97
|
-
|
110
|
+
if deploy.end_with?('-slurm')
|
111
|
+
server = deploy.sub('-slurm','')
|
112
|
+
OffsiteStep.setup(job, server: server, slurm: true)
|
113
|
+
else
|
114
|
+
OffsiteStep.setup(job, server: deploy)
|
115
|
+
end
|
116
|
+
|
98
117
|
job.run
|
99
118
|
end unless job.done?
|
100
119
|
|
@@ -32,6 +32,7 @@ else
|
|
32
32
|
local basedir="$(realpath $1)"
|
33
33
|
local path="$(realpath "$2")"
|
34
34
|
local pre=""
|
35
|
+
basedir=${basedir%.}
|
35
36
|
while [[ ! $path = $basedir* ]]; do
|
36
37
|
pre=${pre}../
|
37
38
|
basedir=$(dirname $basedir)
|
@@ -44,9 +45,8 @@ function link(){
|
|
44
45
|
local source="$1"
|
45
46
|
local target="$2"
|
46
47
|
local rel_source=$(relative_path "$(dirname $target)" "$source")
|
47
|
-
|
48
|
+
|
48
49
|
[ -h "$target" ] && rm "$target"
|
49
|
-
echo ln -s "$rel_source" "$target" 1>&2
|
50
50
|
[ -h "$target" ] || ln -s "$rel_source" "$target"
|
51
51
|
}
|
52
52
|
|