scout-gear 8.1.0 → 9.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +22 -0
  3. data/VERSION +1 -1
  4. data/bin/scout +1 -0
  5. data/lib/rbbt-scout.rb +2 -1
  6. data/lib/scout/cmd.rb +9 -5
  7. data/lib/scout/indiferent_hash.rb +17 -0
  8. data/lib/scout/log/progress/report.rb +1 -0
  9. data/lib/scout/log.rb +14 -11
  10. data/lib/scout/misc/filesystem.rb +2 -3
  11. data/lib/scout/offsite/ssh.rb +3 -0
  12. data/lib/scout/offsite/step.rb +20 -3
  13. data/lib/scout/open/stream.rb +0 -1
  14. data/lib/scout/open.rb +8 -8
  15. data/lib/scout/path.rb +1 -1
  16. data/lib/scout/persist/serialize.rb +1 -1
  17. data/lib/scout/resource/open.rb +8 -0
  18. data/lib/scout/resource/path.rb +12 -9
  19. data/lib/scout/resource/software.rb +4 -2
  20. data/lib/scout/resource.rb +2 -0
  21. data/lib/scout/tsv/dumper.rb +3 -1
  22. data/lib/scout/tsv/parser.rb +13 -3
  23. data/lib/scout/work_queue.rb +1 -0
  24. data/lib/scout/workflow/deployment/orchestrator.rb +17 -8
  25. data/lib/scout/workflow/step/dependencies.rb +9 -3
  26. data/lib/scout/workflow/step/info.rb +8 -0
  27. data/lib/scout/workflow/step/inputs.rb +5 -0
  28. data/lib/scout/workflow/step/status.rb +22 -2
  29. data/lib/scout/workflow/step.rb +11 -4
  30. data/lib/scout/workflow/task/dependencies.rb +2 -0
  31. data/lib/scout/workflow/task/inputs.rb +14 -9
  32. data/lib/scout/workflow/task.rb +4 -2
  33. data/lib/scout/workflow/usage.rb +21 -33
  34. data/lib/scout/workflow.rb +17 -13
  35. data/scout-gear.gemspec +9 -3
  36. data/scout_commands/resource/produce +66 -0
  37. data/scout_commands/template +52 -0
  38. data/scout_commands/workflow/install +3 -0
  39. data/scout_commands/workflow/task +24 -5
  40. data/share/software/install_helpers +2 -2
  41. data/share/templates/command +25 -0
  42. data/share/templates/workflow.rb +14 -0
  43. data/test/scout/offsite/test_step.rb +0 -1
  44. data/test/scout/test_persist.rb +2 -2
  45. data/test/scout/test_work_queue.rb +1 -1
  46. data/test/scout/tsv/test_parser.rb +21 -0
  47. data/test/scout/workflow/step/test_info.rb +0 -1
  48. data/test/scout/workflow/task/test_dependencies.rb +2 -0
  49. data/test/scout/workflow/task/test_inputs.rb +0 -1
  50. data/test/scout/workflow/test_definition.rb +1 -1
  51. metadata +8 -2
@@ -1,6 +1,6 @@
1
1
  class Step
2
2
  def abort(exception = nil)
3
- if info[:pid] != Process.pid && Misc.alive?(pid)
3
+ if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
4
4
  Process.kill pid
5
5
  else
6
6
  while @result && streaming? && stream = self.stream
@@ -15,7 +15,7 @@ class Step
15
15
 
16
16
  def updated?
17
17
  return false if self.error? && self.recoverable_error?
18
- return true unless ENV["SCOUT_UPDATE"]
18
+ return true if self.done? && ! ENV["SCOUT_UPDATE"]
19
19
  newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
20
20
  newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
21
21
 
@@ -28,6 +28,7 @@ class Step
28
28
  @info = nil
29
29
  @info_load_time = nil
30
30
  Open.rm path if Open.exist?(path)
31
+ Open.rm tmp_path if Open.exist?(tmp_path)
31
32
  Open.rm info_file if Open.exist?(info_file)
32
33
  Open.rm_rf files_dir if Open.exist?(files_dir)
33
34
  end
@@ -40,4 +41,23 @@ class Step
40
41
  clean
41
42
  end
42
43
 
44
+ def canfail?
45
+ @compute && @compute.include?(:canfail)
46
+ end
47
+
48
+ def started?
49
+ return true if done?
50
+ return false unless Open.exist?(info_file)
51
+ pid = info[:pid]
52
+ return false unless pid
53
+ return Misc.pid_alive?(pid)
54
+ end
55
+
56
+ def waiting?
57
+ present? and not started?
58
+ end
59
+
60
+ def dirty?
61
+ done? && ! updated?
62
+ end
43
63
  end
@@ -8,16 +8,18 @@ require_relative 'step/dependencies'
8
8
  require_relative 'step/provenance'
9
9
  require_relative 'step/config'
10
10
  require_relative 'step/progress'
11
+ require_relative 'step/inputs'
11
12
 
12
13
  class Step
13
14
 
14
- attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :compute
15
- def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, compute = nil, &task)
15
+ attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
16
+ def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
16
17
  @path = path
17
18
  @inputs = inputs
18
19
  @dependencies = dependencies
19
20
  @id = id
20
21
  @non_default_inputs = non_default_inputs
22
+ @provided_inputs = provided_inputs
21
23
  @compute = compute
22
24
  @task = task
23
25
  @mutex = Mutex.new
@@ -209,7 +211,7 @@ class Step
209
211
  if done?
210
212
  Open.open(self.path)
211
213
  else
212
- if running?
214
+ if running? || waiting?
213
215
  join
214
216
  Open.open(self.path)
215
217
  else
@@ -251,7 +253,7 @@ class Step
251
253
 
252
254
  def join
253
255
  consume_all_streams
254
- while @result.nil? && (present? && ! terminated?)
256
+ while @result.nil? && (present? && ! (terminated? || done?))
255
257
  sleep 0.1
256
258
  end
257
259
  raise self.exception if self.exception
@@ -278,6 +280,7 @@ class Step
278
280
  def step(task_name)
279
281
  dependencies.each do |dep|
280
282
  return dep if dep.task_name == task_name
283
+ return dep if dep.overriden_task == task_name
281
284
  rec_dep = dep.step(task_name)
282
285
  return rec_dep if rec_dep
283
286
  end
@@ -295,4 +298,8 @@ class Step
295
298
  def fingerprint
296
299
  digest_str
297
300
  end
301
+
302
+ def task_signature
303
+ [workflow.to_s, task_name] * "#"
304
+ end
298
305
  end
@@ -55,6 +55,8 @@ module Task
55
55
  dep = provided_inputs[overriden]
56
56
  dep = Step.new dep unless Step === dep
57
57
  dep.type = workflow.tasks[task].type
58
+ dep.overriden_task = task
59
+ dep.overriden_workflow = workflow
58
60
  dependencies << dep
59
61
  non_default_inputs << overriden
60
62
  next
@@ -20,7 +20,7 @@ module Task
20
20
  end
21
21
  end
22
22
 
23
- def assign_inputs(provided_inputs = {})
23
+ def assign_inputs(provided_inputs = {}, id = nil)
24
24
  if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
25
25
  case provided_inputs
26
26
  when Array
@@ -43,6 +43,8 @@ module Task
43
43
  if ! provided.nil? && provided != value
44
44
  non_default_inputs << name.to_sym
45
45
  input_array << provided
46
+ elsif options && options[:jobname]
47
+ input_array << id
46
48
  else
47
49
  input_array << value
48
50
  end
@@ -53,13 +55,14 @@ module Task
53
55
  [input_array, non_default_inputs]
54
56
  end
55
57
 
56
- def process_inputs(provided_inputs = {})
57
- input_array, non_default_inputs = assign_inputs provided_inputs
58
+ def process_inputs(provided_inputs = {}, id = nil)
59
+ input_array, non_default_inputs = assign_inputs provided_inputs, id
58
60
  digest_str = Misc.digest_str(input_array)
59
61
  [input_array, non_default_inputs, digest_str]
60
62
  end
61
63
 
62
64
  def save_file_input(orig_file, directory)
65
+ orig_file = orig_file.path if Step === orig_file
63
66
  basename = File.basename(orig_file)
64
67
  digest = Misc.digest(orig_file)
65
68
  if basename.include? '.'
@@ -81,7 +84,10 @@ module Task
81
84
  value = provided_inputs[name]
82
85
  input_file = File.join(directory, name.to_s)
83
86
 
84
- if type == :file
87
+ if Path.is_filename?(value)
88
+ relative_file = save_file_input(value, directory)
89
+ Open.write(input_file + ".as_file", relative_file)
90
+ elsif type == :file
85
91
  relative_file = save_file_input(value, directory)
86
92
  Persist.save(relative_file, input_file, :file)
87
93
  elsif type == :file_array
@@ -89,13 +95,10 @@ module Task
89
95
  save_file_input(orig_file, directory)
90
96
  end
91
97
  Persist.save(new_files, input_file, type)
92
- elsif Path.is_filename?(value)
93
- relative_file = save_file_input(value, directory)
94
- Open.write(input_file + ".as_file", relative_file)
95
98
  elsif Open.is_stream?(value)
96
- Persist.save(input_file, value, type)
99
+ Persist.save(value, input_file, type)
97
100
  elsif Open.has_stream?(value)
98
- Persist.save(input_file, value.stream, type)
101
+ Persist.save(value.stream, input_file, type)
99
102
  else
100
103
  Persist.save(value, input_file, type)
101
104
  end
@@ -112,6 +115,8 @@ module Task
112
115
  value = Open.read(filename).strip
113
116
  value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
114
117
  inputs[name] = value
118
+ elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
119
+ inputs[name] = filename
115
120
  else
116
121
  inputs[name] = Persist.load(filename, type)
117
122
  end
@@ -42,11 +42,12 @@ module Task
42
42
  def job(id = DEFAULT_NAME, provided_inputs = nil )
43
43
  provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
44
44
  provided_inputs = {} if provided_inputs.nil?
45
+ IndiferentHash.setup(provided_inputs)
45
46
  id = DEFAULT_NAME if id.nil?
46
47
 
47
48
  provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
48
49
 
49
- inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
50
+ inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
50
51
 
51
52
  compute = {}
52
53
  dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
@@ -90,7 +91,8 @@ module Task
90
91
  Log.debug "ID #{self.name} #{id} - Clean"
91
92
  end
92
93
  NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
93
- Step.new path.find, inputs, dependencies, id, non_default_inputs, compute, &self
94
+ step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
95
+ Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
94
96
  end
95
97
  end
96
98
  end
@@ -13,28 +13,30 @@ module Task
13
13
  if paragraph
14
14
  str.puts "\n" << Misc.format_paragraph(paragraph)
15
15
  end
16
- str.puts
17
16
  else
18
17
  title = self.name.to_s
19
18
  str.puts Log.color :yellow, title
20
19
  str.puts Log.color :yellow, "-" * title.length
21
20
  str.puts "\n" << Misc.format_paragraph(description)
22
- str.puts
23
21
  end
24
22
  else
25
23
  title = self.name.to_s
26
24
  str.puts Log.color :yellow, title
27
25
  str.puts Log.color :yellow, "-" * title.length
28
- str.puts
29
26
  end
30
27
 
31
28
 
32
29
  selects = []
30
+
33
31
  if inputs && inputs.any?
32
+ str.puts
34
33
  str.puts Log.color(:magenta, "Inputs")
35
34
  str.puts
36
35
  str.puts SOPT.input_array_doc(inputs)
37
- str.puts
36
+
37
+ inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
38
+ selects << [name, options[:select_options]] if options[:select_options]
39
+ end
38
40
  end
39
41
 
40
42
  deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
@@ -44,14 +46,21 @@ module Task
44
46
  deps.each do |dep_workflow,task_name,options|
45
47
  next if task_name.nil?
46
48
  task = dep_workflow.tasks[task_name]
49
+
47
50
  next if task.inputs.nil?
51
+
48
52
  inputs = task.inputs.reject{|name, _| seen.include? name }
49
53
  inputs = task.inputs.reject{|name, _| options.include? name }
50
54
  next unless inputs.any?
55
+ task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
56
+ selects << [i, options[:select_options]] if options[:select_options]
57
+ end
58
+
51
59
  dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
52
60
  dep_inputs[dep] = inputs
53
61
  end
54
62
 
63
+ str.puts
55
64
  str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
56
65
  dep_inputs.each do |dep,inputs|
57
66
  str.puts
@@ -59,54 +68,31 @@ module Task
59
68
  str.puts
60
69
  str.puts SOPT.input_array_doc(inputs)
61
70
  end
62
-
63
- #task_inputs = dep_inputs deps, workflow
64
- #task_inputs.each do |task,new_inputs|
65
- # new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t|
66
- # t.to_sym == :select and task.input_options[i][:select_options]
67
- # end.each do |i,t|
68
- # selects << [i, task.input_options[i][:select_options]]
69
- # end
70
-
71
- # next if new_inputs.empty?
72
-
73
- # if task.workflow and task.workflow != workflow
74
- # puts " #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
75
- # else
76
- # puts " #{Log.color :yellow, task.name.to_s}:"
77
- # end
78
- # puts unless Log.compact
79
- # puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
80
- # puts unless Log.compact
81
- #end
82
- #puts
83
71
  end
84
72
 
85
73
  case
86
74
  when inputs && inputs.select{|name,type| type == :array }.any?
75
+ str.puts
87
76
  str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
88
77
  also works in addition to the others. You may use the '--array_separator' option
89
78
  the change this default. Whenever a file is specified it may also accept STDIN using
90
79
  the '-' character."))
91
- str.puts
92
80
 
93
81
  when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
94
- str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
95
82
  str.puts
83
+ str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
96
84
  end
97
85
 
98
86
  str.puts
99
87
  str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
100
- str.puts
101
88
 
102
89
  if selects.any?
103
- str.puts Log.color(:magenta, "Input select options")
104
90
  str.puts
91
+ str.puts Log.color(:magenta, "Input select options")
105
92
  selects.collect{|p| p}.uniq.each do |input,options|
93
+ str.puts
106
94
  str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
107
- str.puts unless Log.compact
108
95
  end
109
- str.puts
110
96
  end
111
97
  str.rewind
112
98
  str.read
@@ -249,7 +235,9 @@ module Workflow
249
235
 
250
236
  str.puts
251
237
 
252
- if task.nil?
238
+ if tasks.nil?
239
+ str.puts Log.color(:title, "No tasks")
240
+ elsif task.nil?
253
241
 
254
242
  if self.documentation[:description] and not self.documentation[:description].empty?
255
243
  str.puts Misc.format_paragraph self.documentation[:description]
@@ -284,7 +272,7 @@ module Workflow
284
272
 
285
273
  prov_string = prov_string(dep_tree(name))
286
274
  str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
287
- end
275
+ end
288
276
 
289
277
  else
290
278
 
@@ -25,20 +25,24 @@ module Workflow
25
25
  base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
26
26
  end
27
27
 
28
- def self.require_workflow(workflow_name)
29
- workflow = workflow_name
30
- workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
31
- workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
32
- workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
33
- if Open.exists?(workflow)
34
- self.main = nil
35
- workflow = workflow.find if Path === workflow
36
- $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
37
- load workflow
38
- else
39
- raise "Workflow #{workflow_name} not found"
28
+ def self.require_workflow(workflow_name_orig)
29
+ first = nil
30
+ workflow_name_orig.split("+").each do |workflow_name|
31
+ workflow = workflow_name
32
+ workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
33
+ workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
34
+ workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
35
+ if Open.exists?(workflow)
36
+ self.main = nil
37
+ workflow = workflow.find if Path === workflow
38
+ $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
39
+ load workflow
40
+ else
41
+ raise "Workflow #{workflow_name} not found"
42
+ end
43
+ first ||= self.main || workflows.last
40
44
  end
41
- self.main || workflows.last
45
+ first
42
46
  end
43
47
 
44
48
  def job(name, *args)
data/scout-gear.gemspec CHANGED
@@ -2,16 +2,16 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: scout-gear 8.1.0 ruby lib
5
+ # stub: scout-gear 9.0.0 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "scout-gear".freeze
9
- s.version = "8.1.0"
9
+ s.version = "9.0.0"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib".freeze]
13
13
  s.authors = ["Miguel Vazquez".freeze]
14
- s.date = "2023-06-15"
14
+ s.date = "2023-06-27"
15
15
  s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
16
16
  s.email = "mikisvaz@gmail.com".freeze
17
17
  s.executables = ["scout".freeze]
@@ -77,6 +77,7 @@ Gem::Specification.new do |s|
77
77
  "lib/scout/persist/path.rb",
78
78
  "lib/scout/persist/serialize.rb",
79
79
  "lib/scout/resource.rb",
80
+ "lib/scout/resource/open.rb",
80
81
  "lib/scout/resource/path.rb",
81
82
  "lib/scout/resource/produce.rb",
82
83
  "lib/scout/resource/produce/rake.rb",
@@ -126,6 +127,7 @@ Gem::Specification.new do |s|
126
127
  "lib/scout/workflow/step/dependencies.rb",
127
128
  "lib/scout/workflow/step/file.rb",
128
129
  "lib/scout/workflow/step/info.rb",
130
+ "lib/scout/workflow/step/inputs.rb",
129
131
  "lib/scout/workflow/step/load.rb",
130
132
  "lib/scout/workflow/step/progress.rb",
131
133
  "lib/scout/workflow/step/provenance.rb",
@@ -143,6 +145,8 @@ Gem::Specification.new do |s|
143
145
  "scout_commands/glob",
144
146
  "scout_commands/offsite",
145
147
  "scout_commands/rbbt",
148
+ "scout_commands/resource/produce",
149
+ "scout_commands/template",
146
150
  "scout_commands/update",
147
151
  "scout_commands/workflow/info",
148
152
  "scout_commands/workflow/install",
@@ -151,6 +155,8 @@ Gem::Specification.new do |s|
151
155
  "share/color/color_names",
152
156
  "share/color/diverging_colors.hex",
153
157
  "share/software/install_helpers",
158
+ "share/templates/command",
159
+ "share/templates/workflow.rb",
154
160
  "test/scout/indiferent_hash/test_case_insensitive.rb",
155
161
  "test/scout/indiferent_hash/test_options.rb",
156
162
  "test/scout/log/test_color.rb",
@@ -0,0 +1,66 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Produce a resource
10
+
11
+ $ #{$0} [<options>] <Resource> <file>
12
+
13
+ -h--help Print this help
14
+ -W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
15
+ -r--requires* Files to require; 'all' for all in Scout.etc.requires:
16
+ -f--force Force the production if the file is already present
17
+ EOF
18
+ if options[:help]
19
+ if defined? scout_usage
20
+ scout_usage
21
+ else
22
+ puts SOPT.doc
23
+ end
24
+ exit 0
25
+ end
26
+
27
+ case options[:workflows]
28
+ when nil, false, "false", "none"
29
+ when "all"
30
+ Scout.etc.workflows.list.each do |workflow|
31
+ Workflow.require_workflow file
32
+ end if Scout.etc.workflows.exists?
33
+ else
34
+ options[:workflows].split(/[ ,;|]/).each do |workflow|
35
+ Workflow.require_workflow workflow
36
+ end
37
+ end
38
+
39
+ case options[:requires]
40
+ when nil, false, "false", "none"
41
+ when "all"
42
+ Scout.etc.requires.list.each do |file|
43
+ require file
44
+ end if Scout.etc.requires.exists?
45
+ else
46
+ options[:requires].split(/[ ,;|]/).each do |file|
47
+ require file
48
+ end
49
+ end
50
+
51
+ resource, path = ARGV
52
+
53
+ begin
54
+ resource = Kernel.const_get(resource)
55
+ rescue
56
+ begin
57
+ resource = Workflow.require_workflow resource
58
+ rescue
59
+ raise "Resource not found: #{ resource }"
60
+ end
61
+ end
62
+
63
+ force = options[:force]
64
+
65
+ puts resource[path].produce(force).find
66
+
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'scout'
4
+
5
+ $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
6
+
7
+ options = SOPT.setup <<EOF
8
+
9
+ Print a template
10
+
11
+ $ #{$0} [<options>] <template> [<VAR>=<value>]*
12
+
13
+ Print a template making the substitutions that follow
14
+
15
+ -h--help Print this help
16
+ -s--sub* List a substitutions as a parameter instead
17
+ -s--var* List a substitutions as a parameter, variable name
18
+ -s--value* List a substitutions as a parameter, value
19
+ EOF
20
+ if options[:help]
21
+ if defined? scout_usage
22
+ scout_usage
23
+ else
24
+ puts SOPT.doc
25
+ end
26
+ exit 0
27
+ end
28
+
29
+ template, *subs = ARGV
30
+
31
+ raise MissingParameterException, :template if template.nil?
32
+
33
+ txt = Scout.share.templates[template].find_with_extension('rb').read
34
+
35
+ if options[:sub]
36
+ subs ||= []
37
+ subs << options[:sub]
38
+ end
39
+
40
+ if options[:var]
41
+ subs ||= []
42
+ subs << [options[:var], options[:value]] * "="
43
+ end
44
+
45
+
46
+ subs.each do |sub|
47
+ key, value = sub.split("=")
48
+
49
+ txt.gsub!(key, value)
50
+ end
51
+
52
+ puts txt
@@ -49,6 +49,7 @@ Misc.in_dir(workflow_dir) do
49
49
  Log.info "Updating: " + workflow
50
50
  Misc.in_dir(Misc.snake_case(workflow)) do
51
51
  `git pull`
52
+ `git checkout scout 2> /dev/null`
52
53
  `git submodule init`
53
54
  `git submodule update`
54
55
  end
@@ -56,6 +57,7 @@ Misc.in_dir(workflow_dir) do
56
57
  Misc.in_dir(workflow) do
57
58
  Log.info "Updating: " + workflow
58
59
  `git pull`
60
+ `git checkout scout 2> /dev/null`
59
61
  `git submodule init`
60
62
  `git submodule update`
61
63
  end
@@ -93,6 +95,7 @@ Misc.in_dir(workflow_dir) do
93
95
  end
94
96
  Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
95
97
  Misc.in_dir(Misc.snake_case(workflow)) do
98
+ `git checkout scout 2> /dev/null`
96
99
  `git submodule init`
97
100
  `git submodule update`
98
101
  end
@@ -18,7 +18,7 @@ $ #{$0} [<options>] <workflow> <task>
18
18
  -h--help Print this help
19
19
  --nostream Disable job streaming
20
20
  --update Update jobs with newer dependencies
21
- -jn--job_name* Name to use as job identifier
21
+ -jn--jobname* Name to use as job identifier
22
22
  -li--load_inputs* Directory with inputs files to load
23
23
  -pf--print_filepath Print the file path
24
24
  -prov--provenance Print the step provenance
@@ -26,6 +26,7 @@ $ #{$0} [<options>] <workflow> <task>
26
26
  -rcl--recursive_clean Clean all steps
27
27
  -ct--clean_task* Clean a particular task
28
28
  -d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
29
+ -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
29
30
  EOF
30
31
 
31
32
  workflow_name, task_name = ARGV
@@ -38,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
38
39
 
39
40
  options[:help] = true if task.nil?
40
41
 
41
- help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options,
42
- :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :job_name, :print_filepath, :deploy,
42
+ help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options,
43
+ :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
43
44
  :deploy => 'serial'
44
45
 
45
46
  if help
@@ -59,6 +60,13 @@ else
59
60
  job_inputs = task.get_SOPT
60
61
  end
61
62
 
63
+ if override_deps
64
+ override_deps.split($array_separator || ",").each do |part|
65
+ t_, value = part.split("=")
66
+ job_inputs.merge!( t_ => value)
67
+ end
68
+ end
69
+
62
70
  job = workflow.job(task_name, jobname, job_inputs)
63
71
 
64
72
  job.recursive_clean if recursive_clean
@@ -79,6 +87,7 @@ if clean_task
79
87
  end
80
88
 
81
89
  job.clean if job.task_name.to_s == clean_task.to_s
90
+ job.clean unless job.updated?
82
91
  end
83
92
  end
84
93
 
@@ -92,9 +101,19 @@ else
92
101
  orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
93
102
  orchestrator.process({}, job)
94
103
  when "slurm"
95
- SLURM.orchestrate(job)
104
+ require 'rbbt-scout'
105
+ require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
106
+ HPC::BATCH_MODULE = HPC.batch_system "SLURM"
107
+ HPC::BATCH_MODULE.orchestrate_job(job, {})
108
+ job.grace
96
109
  else
97
- OffsiteStep.setup(job, server: deploy, provided_inputs: job_inputs)
110
+ if deploy.end_with?('-slurm')
111
+ server = deploy.sub('-slurm','')
112
+ OffsiteStep.setup(job, server: server, slurm: true)
113
+ else
114
+ OffsiteStep.setup(job, server: deploy)
115
+ end
116
+
98
117
  job.run
99
118
  end unless job.done?
100
119
 
@@ -32,6 +32,7 @@ else
32
32
  local basedir="$(realpath $1)"
33
33
  local path="$(realpath "$2")"
34
34
  local pre=""
35
+ basedir=${basedir%.}
35
36
  while [[ ! $path = $basedir* ]]; do
36
37
  pre=${pre}../
37
38
  basedir=$(dirname $basedir)
@@ -44,9 +45,8 @@ function link(){
44
45
  local source="$1"
45
46
  local target="$2"
46
47
  local rel_source=$(relative_path "$(dirname $target)" "$source")
47
-
48
+
48
49
  [ -h "$target" ] && rm "$target"
49
- echo ln -s "$rel_source" "$target" 1>&2
50
50
  [ -h "$target" ] || ln -s "$rel_source" "$target"
51
51
  }
52
52