scout-gear 8.1.0 → 9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +22 -0
  3. data/VERSION +1 -1
  4. data/bin/scout +1 -0
  5. data/lib/rbbt-scout.rb +2 -1
  6. data/lib/scout/cmd.rb +9 -5
  7. data/lib/scout/indiferent_hash.rb +17 -0
  8. data/lib/scout/log/progress/report.rb +1 -0
  9. data/lib/scout/log.rb +14 -11
  10. data/lib/scout/misc/filesystem.rb +2 -3
  11. data/lib/scout/offsite/ssh.rb +3 -0
  12. data/lib/scout/offsite/step.rb +20 -3
  13. data/lib/scout/open/stream.rb +0 -1
  14. data/lib/scout/open.rb +8 -8
  15. data/lib/scout/path.rb +1 -1
  16. data/lib/scout/persist/serialize.rb +1 -1
  17. data/lib/scout/resource/open.rb +8 -0
  18. data/lib/scout/resource/path.rb +12 -9
  19. data/lib/scout/resource/software.rb +4 -2
  20. data/lib/scout/resource.rb +2 -0
  21. data/lib/scout/tsv/dumper.rb +3 -1
  22. data/lib/scout/tsv/parser.rb +13 -3
  23. data/lib/scout/work_queue.rb +1 -0
  24. data/lib/scout/workflow/deployment/orchestrator.rb +17 -8
  25. data/lib/scout/workflow/step/dependencies.rb +9 -3
  26. data/lib/scout/workflow/step/info.rb +8 -0
  27. data/lib/scout/workflow/step/inputs.rb +5 -0
  28. data/lib/scout/workflow/step/status.rb +22 -2
  29. data/lib/scout/workflow/step.rb +11 -4
  30. data/lib/scout/workflow/task/dependencies.rb +2 -0
  31. data/lib/scout/workflow/task/inputs.rb +14 -9
  32. data/lib/scout/workflow/task.rb +4 -2
  33. data/lib/scout/workflow/usage.rb +21 -33
  34. data/lib/scout/workflow.rb +17 -13
  35. data/scout-gear.gemspec +9 -3
  36. data/scout_commands/resource/produce +66 -0
  37. data/scout_commands/template +52 -0
  38. data/scout_commands/workflow/install +3 -0
  39. data/scout_commands/workflow/task +24 -5
  40. data/share/software/install_helpers +2 -2
  41. data/share/templates/command +25 -0
  42. data/share/templates/workflow.rb +14 -0
  43. data/test/scout/offsite/test_step.rb +0 -1
  44. data/test/scout/test_persist.rb +2 -2
  45. data/test/scout/test_work_queue.rb +1 -1
  46. data/test/scout/tsv/test_parser.rb +21 -0
  47. data/test/scout/workflow/step/test_info.rb +0 -1
  48. data/test/scout/workflow/task/test_dependencies.rb +2 -0
  49. data/test/scout/workflow/task/test_inputs.rb +0 -1
  50. data/test/scout/workflow/test_definition.rb +1 -1
  51. metadata +8 -2
@@ -1,6 +1,6 @@
1
1
  class Step
2
2
  def abort(exception = nil)
3
- if info[:pid] != Process.pid && Misc.alive?(pid)
3
+ if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
4
4
  Process.kill pid
5
5
  else
6
6
  while @result && streaming? && stream = self.stream
@@ -15,7 +15,7 @@ class Step
15
15
 
16
16
  def updated?
17
17
  return false if self.error? && self.recoverable_error?
18
- return true unless ENV["SCOUT_UPDATE"]
18
+ return true if self.done? && ! ENV["SCOUT_UPDATE"]
19
19
  newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
20
20
  newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
21
21
 
@@ -28,6 +28,7 @@ class Step
28
28
  @info = nil
29
29
  @info_load_time = nil
30
30
  Open.rm path if Open.exist?(path)
31
+ Open.rm tmp_path if Open.exist?(tmp_path)
31
32
  Open.rm info_file if Open.exist?(info_file)
32
33
  Open.rm_rf files_dir if Open.exist?(files_dir)
33
34
  end
@@ -40,4 +41,23 @@ class Step
40
41
  clean
41
42
  end
42
43
 
44
+ def canfail?
45
+ @compute && @compute.include?(:canfail)
46
+ end
47
+
48
+ def started?
49
+ return true if done?
50
+ return false unless Open.exist?(info_file)
51
+ pid = info[:pid]
52
+ return false unless pid
53
+ return Misc.pid_alive?(pid)
54
+ end
55
+
56
+ def waiting?
57
+ present? and not started?
58
+ end
59
+
60
+ def dirty?
61
+ done? && ! updated?
62
+ end
43
63
  end
@@ -8,16 +8,18 @@ require_relative 'step/dependencies'
8
8
  require_relative 'step/provenance'
9
9
  require_relative 'step/config'
10
10
  require_relative 'step/progress'
11
+ require_relative 'step/inputs'
11
12
 
12
13
  class Step
13
14
 
14
- attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
15
- def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task)
15
+ attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
16
+ def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
16
17
  @path = path
17
18
  @inputs = inputs
18
19
  @dependencies = dependencies
19
20
  @id = id
20
21
  @non_default_inputs = non_default_inputs
22
+ @provided_inputs = provided_inputs
21
23
  @compute = compute
22
24
  @task = task
23
25
  @mutex = Mutex.new
@@ -209,7 +211,7 @@ class Step
209
211
  if done?
210
212
  Open.open(self.path)
211
213
  else
212
- if running?
214
+ if running? || waiting?
213
215
  join
214
216
  Open.open(self.path)
215
217
  else
@@ -251,7 +253,7 @@ class Step
251
253
 
252
254
  def join
253
255
  consume_all_streams
254
- while @result.nil? && (present? && ! terminated?)
256
+ while @result.nil? && (present? && ! (terminated? || done?))
255
257
  sleep 0.1
256
258
  end
257
259
  raise self.exception if self.exception
@@ -278,6 +280,7 @@ class Step
278
280
  def step(task_name)
279
281
  dependencies.each do |dep|
280
282
  return dep if dep.task_name == task_name
283
+ return dep if dep.overriden_task == task_name
281
284
  rec_dep = dep.step(task_name)
282
285
  return rec_dep if rec_dep
283
286
  end
@@ -295,4 +298,8 @@ class Step
295
298
  def fingerprint
296
299
  digest_str
297
300
  end
301
+
302
+ def task_signature
303
+ [workflow.to_s, task_name] * "#"
304
+ end
298
305
  end
@@ -55,6 +55,8 @@ module Task
55
55
  dep = provided_inputs[overriden]
56
56
  dep = Step.new dep unless Step === dep
57
57
  dep.type = workflow.tasks[task].type
58
+ dep.overriden_task = task
59
+ dep.overriden_workflow = workflow
58
60
  dependencies << dep
59
61
  non_default_inputs << overriden
60
62
  next
@@ -20,7 +20,7 @@ module Task
20
20
  end
21
21
  end
22
22
 
23
- def assign_inputs(provided_inputs = {})
23
+ def assign_inputs(provided_inputs = {}, id = nil)
24
24
  if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
25
25
  case provided_inputs
26
26
  when Array
@@ -43,6 +43,8 @@ module Task
43
43
  if ! provided.nil? && provided != value
44
44
  non_default_inputs << name.to_sym
45
45
  input_array << provided
46
+ elsif options && options[:jobname]
47
+ input_array << id
46
48
  else
47
49
  input_array << value
48
50
  end
@@ -53,13 +55,14 @@ module Task
53
55
  [input_array, non_default_inputs]
54
56
  end
55
57
 
56
- def process_inputs(provided_inputs = {})
57
- input_array, non_default_inputs = assign_inputs provided_inputs
58
+ def process_inputs(provided_inputs = {}, id = nil)
59
+ input_array, non_default_inputs = assign_inputs provided_inputs, id
58
60
  digest_str = Misc.digest_str(input_array)
59
61
  [input_array, non_default_inputs, digest_str]
60
62
  end
61
63
 
62
64
  def save_file_input(orig_file, directory)
65
+ orig_file = orig_file.path if Step === orig_file
63
66
  basename = File.basename(orig_file)
64
67
  digest = Misc.digest(orig_file)
65
68
  if basename.include? '.'
@@ -81,7 +84,10 @@ module Task
81
84
  value = provided_inputs[name]
82
85
  input_file = File.join(directory, name.to_s)
83
86
 
84
- if type == :file
87
+ if Path.is_filename?(value)
88
+ relative_file = save_file_input(value, directory)
89
+ Open.write(input_file + ".as_file", relative_file)
90
+ elsif type == :file
85
91
  relative_file = save_file_input(value, directory)
86
92
  Persist.save(relative_file, input_file, :file)
87
93
  elsif type == :file_array
@@ -89,13 +95,10 @@ module Task
89
95
  save_file_input(orig_file, directory)
90
96
  end
91
97
  Persist.save(new_files, input_file, type)
92
- elsif Path.is_filename?(value)
93
- relative_file = save_file_input(value, directory)
94
- Open.write(input_file + ".as_file", relative_file)
95
98
  elsif Open.is_stream?(value)
96
- Persist.save(input_file, value, type)
99
+ Persist.save(value, input_file, type)
97
100
  elsif Open.has_stream?(value)
98
- Persist.save(input_file, value.stream, type)
101
+ Persist.save(value.stream, input_file, type)
99
102
  else
100
103
  Persist.save(value, input_file, type)
101
104
  end
@@ -112,6 +115,8 @@ module Task
112
115
  value = Open.read(filename).strip
113
116
  value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
114
117
  inputs[name] = value
118
+ elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
119
+ inputs[name] = filename
115
120
  else
116
121
  inputs[name] = Persist.load(filename, type)
117
122
  end
@@ -42,11 +42,12 @@ module Task
42
42
  def job(id = DEFAULT_NAME, provided_inputs = nil )
43
43
  provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
44
44
  provided_inputs = {} if provided_inputs.nil?
45
+ IndiferentHash.setup(provided_inputs)
45
46
  id = DEFAULT_NAME if id.nil?
46
47
 
47
48
  provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
48
49
 
49
- inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
50
+ inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
50
51
 
51
52
  compute = {}
52
53
  dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
@@ -90,7 +91,8 @@ module Task
90
91
  Log.debug "ID #{self.name} #{id} - Clean"
91
92
  end
92
93
  NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
93
- Step.new path.find, inputs, dependencies, id, non_default_inputs, compute, &self
94
+ step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
95
+ Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
94
96
  end
95
97
  end
96
98
  end
@@ -13,28 +13,30 @@ module Task
13
13
  if paragraph
14
14
  str.puts "\n" << Misc.format_paragraph(paragraph)
15
15
  end
16
- str.puts
17
16
  else
18
17
  title = self.name.to_s
19
18
  str.puts Log.color :yellow, title
20
19
  str.puts Log.color :yellow, "-" * title.length
21
20
  str.puts "\n" << Misc.format_paragraph(description)
22
- str.puts
23
21
  end
24
22
  else
25
23
  title = self.name.to_s
26
24
  str.puts Log.color :yellow, title
27
25
  str.puts Log.color :yellow, "-" * title.length
28
- str.puts
29
26
  end
30
27
 
31
28
 
32
29
  selects = []
30
+
33
31
  if inputs && inputs.any?
32
+ str.puts
34
33
  str.puts Log.color(:magenta, "Inputs")
35
34
  str.puts
36
35
  str.puts SOPT.input_array_doc(inputs)
37
- str.puts
36
+
37
+ inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
38
+ selects << [name, options[:select_options]] if options[:select_options]
39
+ end
38
40
  end
39
41
 
40
42
  deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
@@ -44,14 +46,21 @@ module Task
44
46
  deps.each do |dep_workflow,task_name,options|
45
47
  next if task_name.nil?
46
48
  task = dep_workflow.tasks[task_name]
49
+
47
50
  next if task.inputs.nil?
51
+
48
52
  inputs = task.inputs.reject{|name, _| seen.include? name }
49
53
  inputs = task.inputs.reject{|name, _| options.include? name }
50
54
  next unless inputs.any?
55
+ task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
56
+ selects << [i, options[:select_options]] if options[:select_options]
57
+ end
58
+
51
59
  dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
52
60
  dep_inputs[dep] = inputs
53
61
  end
54
62
 
63
+ str.puts
55
64
  str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
56
65
  dep_inputs.each do |dep,inputs|
57
66
  str.puts
@@ -59,54 +68,31 @@ module Task
59
68
  str.puts
60
69
  str.puts SOPT.input_array_doc(inputs)
61
70
  end
62
-
63
- #task_inputs = dep_inputs deps, workflow
64
- #task_inputs.each do |task,new_inputs|
65
- # new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t|
66
- # t.to_sym == :select and task.input_options[i][:select_options]
67
- # end.each do |i,t|
68
- # selects << [i, task.input_options[i][:select_options]]
69
- # end
70
-
71
- # next if new_inputs.empty?
72
-
73
- # if task.workflow and task.workflow != workflow
74
- # puts " #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
75
- # else
76
- # puts " #{Log.color :yellow, task.name.to_s}:"
77
- # end
78
- # puts unless Log.compact
79
- # puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
80
- # puts unless Log.compact
81
- #end
82
- #puts
83
71
  end
84
72
 
85
73
  case
86
74
  when inputs && inputs.select{|name,type| type == :array }.any?
75
+ str.puts
87
76
  str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
88
77
  also works in addition to the others. You may use the '--array_separator' option
89
78
  the change this default. Whenever a file is specified it may also accept STDIN using
90
79
  the '-' character."))
91
- str.puts
92
80
 
93
81
  when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
94
- str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
95
82
  str.puts
83
+ str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
96
84
  end
97
85
 
98
86
  str.puts
99
87
  str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
100
- str.puts
101
88
 
102
89
  if selects.any?
103
- str.puts Log.color(:magenta, "Input select options")
104
90
  str.puts
91
+ str.puts Log.color(:magenta, "Input select options")
105
92
  selects.collect{|p| p}.uniq.each do |input,options|
93
+ str.puts
106
94
  str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
107
- str.puts unless Log.compact
108
95
  end
109
- str.puts
110
96
  end
111
97
  str.rewind
112
98
  str.read
@@ -249,7 +235,9 @@ module Workflow
249
235
 
250
236
  str.puts
251
237
 
252
- if task.nil?
238
+ if tasks.nil?
239
+ str.puts Log.color(:title, "No tasks")
240
+ elsif task.nil?
253
241
 
254
242
  if self.documentation[:description] and not self.documentation[:description].empty?
255
243
  str.puts Misc.format_paragraph self.documentation[:description]
@@ -284,7 +272,7 @@ module Workflow
284
272
 
285
273
  prov_string = prov_string(dep_tree(name))
286
274
  str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
287
- end
275
+ end
288
276
 
289
277
  else
290
278
 
@@ -25,20 +25,24 @@ module Workflow
25
25
  base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
26
26
  end
27
27
 
28
- def self.require_workflow(workflow_name)
29
- workflow = workflow_name
30
- workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
31
- workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
32
- workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
33
- if Open.exists?(workflow)
34
- self.main = nil
35
- workflow = workflow.find if Path === workflow
36
- $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
37
- load workflow
38
- else
39
- raise "Workflow #{workflow_name} not found"
28
+ def self.require_workflow(workflow_name_orig)
29
+ first = nil
30
+ workflow_name_orig.split("+").each do |workflow_name|
31
+ workflow = workflow_name
32
+ workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
33
+ workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
34
+ workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
35
+ if Open.exists?(workflow)
36
+ self.main = nil
37
+ workflow = workflow.find if Path === workflow
38
+ $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
39
+ load workflow
40
+ else
41
+ raise "Workflow #{workflow_name} not found"
42
+ end
43
+ first ||= self.main || workflows.last
40
44
  end
41
- self.main || workflows.last
45
+ first
42
46
  end
43
47
 
44
48
  def job(name, *args)
data/scout-gear.gemspec CHANGED
@@ -2,16 +2,16 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: scout-gear 8.1.0 ruby lib
5
+ # stub: scout-gear 9.0.0 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "scout-gear".freeze
9
- s.version = "8.1.0"
9
+ s.version = "9.0.0"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib".freeze]
13
13
  s.authors = ["Miguel Vazquez".freeze]
14
- s.date = "2023-06-15"
14
+ s.date = "2023-06-27"
15
15
  s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
16
16
  s.email = "mikisvaz@gmail.com".freeze
17
17
  s.executables = ["scout".freeze]
@@ -77,6 +77,7 @@ Gem::Specification.new do |s|
77
77
  "lib/scout/persist/path.rb",
78
78
  "lib/scout/persist/serialize.rb",
79
79
  "lib/scout/resource.rb",
80
+ "lib/scout/resource/open.rb",
80
81
  "lib/scout/resource/path.rb",
81
82
  "lib/scout/resource/produce.rb",
82
83
  "lib/scout/resource/produce/rake.rb",
@@ -126,6 +127,7 @@ Gem::Specification.new do |s|
126
127
  "lib/scout/workflow/step/dependencies.rb",
127
128
  "lib/scout/workflow/step/file.rb",
128
129
  "lib/scout/workflow/step/info.rb",
130
+ "lib/scout/workflow/step/inputs.rb",
129
131
  "lib/scout/workflow/step/load.rb",
130
132
  "lib/scout/workflow/step/progress.rb",
131
133
  "lib/scout/workflow/step/provenance.rb",
@@ -143,6 +145,8 @@ Gem::Specification.new do |s|
143
145
  "scout_commands/glob",
144
146
  "scout_commands/offsite",
145
147
  "scout_commands/rbbt",
148
+ "scout_commands/resource/produce",
149
+ "scout_commands/template",
146
150
  "scout_commands/update",
147
151
  "scout_commands/workflow/info",
148
152
  "scout_commands/workflow/install",
@@ -151,6 +155,8 @@ Gem::Specification.new do |s|
151
155
  "share/color/color_names",
152
156
  "share/color/diverging_colors.hex",
153
157
  "share/software/install_helpers",
158
+ "share/templates/command",
159
+ "share/templates/workflow.rb",
154
160
  "test/scout/indiferent_hash/test_case_insensitive.rb",
155
161
  "test/scout/indiferent_hash/test_options.rb",
156
162
  "test/scout/log/test_color.rb",
@@ -0,0 +1,66 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Produce a resource
10
+
11
+ $ #{$0} [<options>] <Resource> <file>
12
+
13
+ -h--help Print this help
14
+ -W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
15
+ -r--requires* Files to require; 'all' for all in Scout.etc.requires:
16
+ -f--force Force the production if the file is already present
17
+ EOF
18
+ if options[:help]
19
+ if defined? scout_usage
20
+ scout_usage
21
+ else
22
+ puts SOPT.doc
23
+ end
24
+ exit 0
25
+ end
26
+
27
+ case options[:workflows]
28
+ when nil, false, "false", "none"
29
+ when "all"
30
+ Scout.etc.workflows.list.each do |workflow|
31
+ Workflow.require_workflow file
32
+ end if Scout.etc.workflows.exists?
33
+ else
34
+ options[:workflows].split(/[ ,;|]/).each do |workflow|
35
+ Workflow.require_workflow workflow
36
+ end
37
+ end
38
+
39
+ case options[:requires]
40
+ when nil, false, "false", "none"
41
+ when "all"
42
+ Scout.etc.requires.list.each do |file|
43
+ require file
44
+ end if Scout.etc.requires.exists?
45
+ else
46
+ options[:requires].split(/[ ,;|]/).each do |file|
47
+ require file
48
+ end
49
+ end
50
+
51
+ resource, path = ARGV
52
+
53
+ begin
54
+ resource = Kernel.const_get(resource)
55
+ rescue
56
+ begin
57
+ resource = Workflow.require_workflow resource
58
+ rescue
59
+ raise "Resource not found: #{ resource }"
60
+ end
61
+ end
62
+
63
+ force = options[:force]
64
+
65
+ puts resource[path].produce(force).find
66
+
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Print a template
10
+
11
+ $ #{$0} [<options>] <template> [<VAR>=<value>]*
12
+
13
+ Print a template making the substitutions that follow
14
+
15
+ -h--help Print this help
16
+ -s--sub* List a substitutions as a parameter instead
17
+ -s--var* List a substitutions as a parameter, variable name
18
+ -s--value* List a substitutions as a parameter, value
19
+ EOF
20
+ if options[:help]
21
+ if defined? scout_usage
22
+ scout_usage
23
+ else
24
+ puts SOPT.doc
25
+ end
26
+ exit 0
27
+ end
28
+
29
+ template, *subs = ARGV
30
+
31
+ raise MissingParameterException, :template if template.nil?
32
+
33
+ txt = Scout.share.templates[template].find_with_extension('rb').read
34
+
35
+ if options[:sub]
36
+ subs ||= []
37
+ subs << options[:sub]
38
+ end
39
+
40
+ if options[:var]
41
+ subs ||= []
42
+ subs << [options[:var], options[:value]] * "="
43
+ end
44
+
45
+
46
+ subs.each do |sub|
47
+ key, value = sub.split("=")
48
+
49
+ txt.gsub!(key, value)
50
+ end
51
+
52
+ puts txt
@@ -49,6 +49,7 @@ Misc.in_dir(workflow_dir) do
49
49
  Log.info "Updating: " + workflow
50
50
  Misc.in_dir(Misc.snake_case(workflow)) do
51
51
  `git pull`
52
+ `git checkout scout 2> /dev/null`
52
53
  `git submodule init`
53
54
  `git submodule update`
54
55
  end
@@ -56,6 +57,7 @@ Misc.in_dir(workflow_dir) do
56
57
  Misc.in_dir(workflow) do
57
58
  Log.info "Updating: " + workflow
58
59
  `git pull`
60
+ `git checkout scout 2> /dev/null`
59
61
  `git submodule init`
60
62
  `git submodule update`
61
63
  end
@@ -93,6 +95,7 @@ Misc.in_dir(workflow_dir) do
93
95
  end
94
96
  Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
95
97
  Misc.in_dir(Misc.snake_case(workflow)) do
98
+ `git checkout scout 2> /dev/null`
96
99
  `git submodule init`
97
100
  `git submodule update`
98
101
  end
@@ -18,7 +18,7 @@ $ #{$0} [<options>] <workflow> <task>
18
18
  -h--help Print this help
19
19
  --nostream Disable job streaming
20
20
  --update Update jobs with newer dependencies
21
- -jn--job_name* Name to use as job identifier
21
+ -jn--jobname* Name to use as job identifier
22
22
  -li--load_inputs* Directory with inputs files to load
23
23
  -pf--print_filepath Print the file path
24
24
  -prov--provenance Print the step provenance
@@ -26,6 +26,7 @@ $ #{$0} [<options>] <workflow> <task>
26
26
  -rcl--recursive_clean Clean all steps
27
27
  -ct--clean_task* Clean a particular task
28
28
  -d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
29
+ -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
29
30
  EOF
30
31
 
31
32
  workflow_name, task_name = ARGV
@@ -38,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
38
39
 
39
40
  options[:help] = true if task.nil?
40
41
 
41
- help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options,
42
- :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :job_name, :print_filepath, :deploy,
42
+ help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options,
43
+ :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
43
44
  :deploy => 'serial'
44
45
 
45
46
  if help
@@ -59,6 +60,13 @@ else
59
60
  job_inputs = task.get_SOPT
60
61
  end
61
62
 
63
+ if override_deps
64
+ override_deps.split($array_separator || ",").each do |part|
65
+ t_, value = part.split("=")
66
+ job_inputs.merge!( t_ => value)
67
+ end
68
+ end
69
+
62
70
  job = workflow.job(task_name, jobname, job_inputs)
63
71
 
64
72
  job.recursive_clean if recursive_clean
@@ -79,6 +87,7 @@ if clean_task
79
87
  end
80
88
 
81
89
  job.clean if job.task_name.to_s == clean_task.to_s
90
+ job.clean unless job.updated?
82
91
  end
83
92
  end
84
93
 
@@ -92,9 +101,19 @@ else
92
101
  orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
93
102
  orchestrator.process({}, job)
94
103
  when "slurm"
95
- SLURM.orchestrate(job)
104
+ require 'rbbt-scout'
105
+ require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
106
+ HPC::BATCH_MODULE = HPC.batch_system "SLURM"
107
+ HPC::BATCH_MODULE.orchestrate_job(job, {})
108
+ job.grace
96
109
  else
97
- OffsiteStep.setup(job, server: deploy, provided_inputs: job_inputs)
110
+ if deploy.end_with?('-slurm')
111
+ server = deploy.sub('-slurm','')
112
+ OffsiteStep.setup(job, server: server, slurm: true)
113
+ else
114
+ OffsiteStep.setup(job, server: deploy)
115
+ end
116
+
98
117
  job.run
99
118
  end unless job.done?
100
119
 
@@ -32,6 +32,7 @@ else
32
32
  local basedir="$(realpath $1)"
33
33
  local path="$(realpath "$2")"
34
34
  local pre=""
35
+ basedir=${basedir%.}
35
36
  while [[ ! $path = $basedir* ]]; do
36
37
  pre=${pre}../
37
38
  basedir=$(dirname $basedir)
@@ -44,9 +45,8 @@ function link(){
44
45
  local source="$1"
45
46
  local target="$2"
46
47
  local rel_source=$(relative_path "$(dirname $target)" "$source")
47
-
48
+
48
49
  [ -h "$target" ] && rm "$target"
49
- echo ln -s "$rel_source" "$target" 1>&2
50
50
  [ -h "$target" ] || ln -s "$rel_source" "$target"
51
51
  }
52
52