scout-gear 10.11.6 → 10.11.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +16 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +10 -10
  5. data/lib/scout/association/fields.rb +15 -15
  6. data/lib/scout/association/index.rb +6 -6
  7. data/lib/scout/association/item.rb +18 -8
  8. data/lib/scout/association.rb +4 -4
  9. data/lib/scout/entity/identifiers.rb +5 -5
  10. data/lib/scout/entity/property.rb +2 -2
  11. data/lib/scout/entity.rb +1 -1
  12. data/lib/scout/knowledge_base/description.rb +10 -10
  13. data/lib/scout/knowledge_base/entity.rb +6 -6
  14. data/lib/scout/knowledge_base/list.rb +1 -1
  15. data/lib/scout/knowledge_base/query.rb +4 -4
  16. data/lib/scout/knowledge_base/registry.rb +6 -6
  17. data/lib/scout/knowledge_base/traverse.rb +7 -40
  18. data/lib/scout/persist/engine/fix_width_table.rb +6 -6
  19. data/lib/scout/persist/engine/packed_index.rb +2 -2
  20. data/lib/scout/persist/engine/sharder.rb +4 -4
  21. data/lib/scout/persist/engine/tkrzw.rb +1 -1
  22. data/lib/scout/persist/engine/tokyocabinet.rb +2 -2
  23. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +1 -1
  24. data/lib/scout/persist/tsv/adapter/packed_index.rb +1 -1
  25. data/lib/scout/persist/tsv/adapter/tkrzw.rb +1 -1
  26. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +3 -3
  27. data/lib/scout/persist/tsv/serialize.rb +3 -3
  28. data/lib/scout/persist/tsv.rb +1 -1
  29. data/lib/scout/semaphore.rb +78 -3
  30. data/lib/scout/tsv/annotation/repo.rb +4 -4
  31. data/lib/scout/tsv/annotation.rb +2 -2
  32. data/lib/scout/tsv/attach.rb +7 -7
  33. data/lib/scout/tsv/change_id/translate.rb +1 -1
  34. data/lib/scout/tsv/csv.rb +3 -3
  35. data/lib/scout/tsv/dumper.rb +8 -8
  36. data/lib/scout/tsv/index.rb +1 -1
  37. data/lib/scout/tsv/open.rb +3 -3
  38. data/lib/scout/tsv/stream.rb +2 -2
  39. data/lib/scout/tsv/traverse.rb +4 -4
  40. data/lib/scout/tsv/util/filter.rb +9 -9
  41. data/lib/scout/tsv/util/process.rb +2 -2
  42. data/lib/scout/tsv/util/reorder.rb +2 -2
  43. data/lib/scout/tsv/util/select.rb +3 -3
  44. data/lib/scout/tsv/util/unzip.rb +2 -2
  45. data/lib/scout/tsv/util.rb +1 -1
  46. data/lib/scout/tsv.rb +2 -2
  47. data/lib/scout/work_queue/socket.rb +2 -2
  48. data/lib/scout/work_queue/worker.rb +4 -4
  49. data/lib/scout/work_queue.rb +5 -5
  50. data/lib/scout/workflow/definition.rb +18 -16
  51. data/lib/scout/workflow/deployment/local.rb +82 -62
  52. data/lib/scout/workflow/deployment/orchestrator/batches.rb +66 -5
  53. data/lib/scout/workflow/deployment/orchestrator/chains.rb +47 -30
  54. data/lib/scout/workflow/deployment/orchestrator/rules.rb +3 -3
  55. data/lib/scout/workflow/deployment/orchestrator/workload.rb +11 -22
  56. data/lib/scout/workflow/deployment/scheduler/job.rb +34 -36
  57. data/lib/scout/workflow/deployment/scheduler/lfs.rb +1 -1
  58. data/lib/scout/workflow/deployment/scheduler/pbs.rb +4 -4
  59. data/lib/scout/workflow/deployment/scheduler/slurm.rb +2 -2
  60. data/lib/scout/workflow/deployment/scheduler.rb +23 -12
  61. data/lib/scout/workflow/deployment/trace.rb +2 -2
  62. data/lib/scout/workflow/documentation.rb +4 -4
  63. data/lib/scout/workflow/export.rb +1 -1
  64. data/lib/scout/workflow/path.rb +2 -2
  65. data/lib/scout/workflow/step/children.rb +1 -1
  66. data/lib/scout/workflow/step/dependencies.rb +36 -3
  67. data/lib/scout/workflow/step/info.rb +5 -19
  68. data/lib/scout/workflow/step/inputs.rb +1 -1
  69. data/lib/scout/workflow/step/progress.rb +2 -2
  70. data/lib/scout/workflow/step/provenance.rb +4 -4
  71. data/lib/scout/workflow/step/status.rb +23 -9
  72. data/lib/scout/workflow/step.rb +19 -17
  73. data/lib/scout/workflow/task/dependencies.rb +10 -3
  74. data/lib/scout/workflow/task/info.rb +3 -3
  75. data/lib/scout/workflow/task/inputs.rb +14 -8
  76. data/lib/scout/workflow/task.rb +37 -22
  77. data/lib/scout/workflow/usage.rb +13 -13
  78. data/lib/scout/workflow/util.rb +1 -1
  79. data/lib/scout/workflow.rb +6 -6
  80. data/scout-gear.gemspec +3 -3
  81. data/scout_commands/alias +1 -1
  82. data/scout_commands/batch/clean +12 -12
  83. data/scout_commands/batch/list +26 -25
  84. data/scout_commands/batch/tail +9 -5
  85. data/scout_commands/cat +1 -1
  86. data/scout_commands/doc +2 -2
  87. data/scout_commands/entity +4 -4
  88. data/scout_commands/find +1 -1
  89. data/scout_commands/kb/config +1 -1
  90. data/scout_commands/kb/entities +1 -1
  91. data/scout_commands/kb/list +1 -1
  92. data/scout_commands/kb/query +2 -2
  93. data/scout_commands/kb/register +1 -1
  94. data/scout_commands/kb/show +1 -1
  95. data/scout_commands/kb/traverse +1 -1
  96. data/scout_commands/log +6 -6
  97. data/scout_commands/resource/produce +2 -2
  98. data/scout_commands/resource/sync +1 -1
  99. data/scout_commands/system/clean +7 -7
  100. data/scout_commands/system/status +4 -4
  101. data/scout_commands/template +1 -1
  102. data/scout_commands/update +1 -1
  103. data/scout_commands/workflow/info +1 -1
  104. data/scout_commands/workflow/install +1 -1
  105. data/scout_commands/workflow/list +2 -2
  106. data/scout_commands/workflow/process +2 -2
  107. data/scout_commands/workflow/prov +3 -3
  108. data/scout_commands/workflow/task +36 -11
  109. data/scout_commands/workflow/trace +1 -1
  110. data/scout_commands/workflow/write_info +2 -2
  111. data/share/templates/command +1 -1
  112. data/test/scout/association/test_item.rb +5 -0
  113. data/test/scout/entity/test_property.rb +3 -3
  114. data/test/scout/knowledge_base/test_description.rb +1 -1
  115. data/test/scout/knowledge_base/test_traverse.rb +2 -2
  116. data/test/scout/persist/engine/test_packed_index.rb +6 -6
  117. data/test/scout/persist/test_tsv.rb +4 -4
  118. data/test/scout/persist/tsv/adapter/test_packed_index.rb +4 -4
  119. data/test/scout/persist/tsv/adapter/test_sharder.rb +23 -23
  120. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +1 -1
  121. data/test/scout/persist/tsv/test_serialize.rb +1 -1
  122. data/test/scout/test_association.rb +1 -1
  123. data/test/scout/test_tsv.rb +2 -2
  124. data/test/scout/test_workflow.rb +2 -2
  125. data/test/scout/tsv/test_annotation.rb +4 -4
  126. data/test/scout/tsv/test_index.rb +1 -1
  127. data/test/scout/tsv/test_open.rb +2 -2
  128. data/test/scout/tsv/test_parser.rb +2 -2
  129. data/test/scout/tsv/test_stream.rb +1 -1
  130. data/test/scout/tsv/test_transformer.rb +1 -1
  131. data/test/scout/tsv/util/test_filter.rb +1 -1
  132. data/test/scout/tsv/util/test_melt.rb +1 -1
  133. data/test/scout/tsv/util/test_reorder.rb +1 -1
  134. data/test/scout/work_queue/test_socket.rb +3 -3
  135. data/test/scout/work_queue/test_worker.rb +2 -2
  136. data/test/scout/workflow/deployment/orchestrator/test_batches.rb +13 -3
  137. data/test/scout/workflow/deployment/orchestrator/test_chains.rb +15 -13
  138. data/test/scout/workflow/deployment/orchestrator/test_workload.rb +1 -1
  139. data/test/scout/workflow/deployment/test_local.rb +2 -2
  140. data/test/scout/workflow/deployment/test_scheduler.rb +1 -2
  141. data/test/scout/workflow/step/test_children.rb +1 -1
  142. data/test/scout/workflow/step/test_dependencies.rb +36 -1
  143. data/test/scout/workflow/step/test_info.rb +3 -35
  144. data/test/scout/workflow/step/test_load.rb +1 -1
  145. data/test/scout/workflow/step/test_provenance.rb +1 -1
  146. data/test/scout/workflow/step/test_status.rb +33 -1
  147. data/test/scout/workflow/task/test_dependencies.rb +9 -7
  148. data/test/scout/workflow/task/test_inputs.rb +1 -1
  149. data/test/scout/workflow/test_definition.rb +1 -1
  150. data/test/scout/workflow/test_documentation.rb +1 -1
  151. data/test/scout/workflow/test_entity.rb +2 -2
  152. data/test/scout/workflow/test_step.rb +13 -13
  153. data/test/scout/workflow/test_usage.rb +1 -1
  154. data/test/test_helper.rb +1 -1
  155. metadata +2 -2
@@ -30,7 +30,8 @@ class Step
30
30
  end
31
31
 
32
32
  def save_info(info = nil)
33
- Persist.save(@info = info, info_file, SERIALIZER)
33
+ purged = Annotation.purge(@info = info)
34
+ Persist.save(purged, info_file, SERIALIZER)
34
35
  @info_load_time = Time.now
35
36
  end
36
37
 
@@ -82,6 +83,7 @@ class Step
82
83
 
83
84
  if start && eend
84
85
  time = eend - start
86
+ Log.warn "No issue time #{self.path}" if issued.nil?
85
87
  total_time = eend - issued
86
88
  if total_time - time > 1
87
89
  time_str = "#{Misc.format_seconds_short(time)} (#{Misc.format_seconds_short(total_time)})"
@@ -93,7 +95,7 @@ class Step
93
95
  message = Log.color(:time, time_str)
94
96
  end
95
97
  end
96
- report_status value, message
98
+ report_status value, message
97
99
  end
98
100
 
99
101
  if key == :message
@@ -136,7 +138,7 @@ class Step
136
138
  def set_info(key, value)
137
139
  merge_info(key => value)
138
140
  end
139
-
141
+
140
142
  def report_status(status, message = nil)
141
143
  if message.nil?
142
144
  Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
@@ -184,22 +186,6 @@ class Step
184
186
  ! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
185
187
  end
186
188
 
187
- def overriden?
188
- @overriden = overriden_task || overriden_workflow || overriden_deps.any? if @overriden.nil?
189
- @overriden
190
- end
191
-
192
- def overriden_deps
193
- dependencies.select{|d| d.overriden? }
194
- end
195
-
196
- def recursive_overriden_deps
197
- overriden_deps = dependencies.select{|d|
198
- d.overriden?
199
- }
200
- (overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
201
- end
202
-
203
189
  def exception
204
190
  return nil unless info[:exception]
205
191
  begin
@@ -3,7 +3,7 @@ class Step
3
3
  if clean_name != name
4
4
  #hash = name[clean_name.length..-1]
5
5
  #inputs_dir += hash
6
- Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir}"
6
+ Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir} #{Log.fingerprint provided_inputs}"
7
7
  self.task.save_inputs(inputs_dir, provided_inputs)
8
8
  else
9
9
  Log.medium "Saving no input job: #{Log.fingerprint inputs_dir}"
@@ -27,9 +27,9 @@ class Step
27
27
  end
28
28
 
29
29
  def monitor_stream(stream, options = {}, &block)
30
- case options[:bar]
30
+ case options[:bar]
31
31
  when TrueClass
32
- bar = progress_bar
32
+ bar = progress_bar
33
33
  when Hash
34
34
  bar = progress_bar options[:bar]
35
35
  when Numeric
@@ -58,10 +58,10 @@ class Step
58
58
  str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
59
59
  prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
60
60
  else
61
- prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
61
+ prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
62
62
  end
63
63
 
64
- if $inputs and $inputs.any?
64
+ if $inputs and $inputs.any?
65
65
  job_inputs = Step.new(path).recursive_inputs.to_hash
66
66
  IndiferentHash.setup(job_inputs)
67
67
 
@@ -119,7 +119,7 @@ class Step
119
119
  step.dependencies.each do |dep|
120
120
  if dep.input_dependencies.any?
121
121
  dep.input_dependencies.each do |id|
122
- input_name, _dep = dep.recursive_inputs.select{|f,d|
122
+ input_name, _dep = dep.recursive_inputs.select{|f,d|
123
123
  d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
124
124
  }.keys.last
125
125
  if input_name
@@ -135,7 +135,7 @@ class Step
135
135
  indent = prov_indent(step, offset, input_dependencies)
136
136
  str << indent + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
137
137
 
138
- step.dependencies.dup.tap{|l|
138
+ step.dependencies.dup.tap{|l|
139
139
  l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
140
140
  }.each do |dep|
141
141
  path = dep.path
@@ -5,7 +5,7 @@ class Step
5
5
  begin
6
6
  s = Misc.abort_child pid, true
7
7
  Log.medium "Aborted pid #{path} #{s}"
8
- rescue
8
+ rescue
9
9
  Log.debug("Aborted job #{pid} was not killed: #{$!.message}")
10
10
  end
11
11
  else
@@ -17,35 +17,49 @@ class Step
17
17
  end
18
18
 
19
19
  def recoverable_error?
20
- self.error? && ! (ScoutException === self.exception)
20
+ self.error? && ! (ENV['SCOUT_NO_RECOVERABLE_ERROR'].to_s.downcase == 'true') && ! (ScoutException === self.exception)
21
21
  end
22
22
 
23
23
  def newer_dependencies
24
+ rec_dependencies = self.rec_dependencies
25
+ rec_dependencies = rec_dependencies.select{|dep| Open.exists?(dep.info_file) }
26
+ rec_dependencies = rec_dependencies.reject{|dep| dep.error? && ! dep.recoverable_error? }
24
27
  newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
25
28
  newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
26
29
  newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
27
30
  newer
28
31
  end
29
32
 
33
+ def cleaned_dependencies
34
+ return []
35
+ rec_dependencies = self.rec_dependencies
36
+ cleaned = rec_dependencies.select{|dep| dep.info[:status] == :cleaned }
37
+ cleaned += input_dependencies.select{|dep| dep.info[:status] == :cleaned }
38
+ cleaned += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| dep.info[:status] == :cleaned }
39
+ cleaned
40
+ end
41
+
30
42
  def updated?
31
43
  return false if self.error? && self.recoverable_error?
32
- return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
44
+ return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"].to_s.downcase == 'true'
33
45
  newer = newer_dependencies
46
+ cleaned = cleaned_dependencies
34
47
 
35
48
  Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
36
- newer.empty?
49
+ Log.low "Cleaned deps found for #{Log.fingerprint self}: #{Log.fingerprint cleaned}" if cleaned.any?
50
+ newer.empty? && cleaned.empty?
37
51
  end
38
52
 
39
53
  def clean
40
54
  Log.debug "Cleaning job files: #{path}"
41
- @take_stream = nil
55
+ @take_stream = nil
42
56
  @result = nil
43
57
  @info = nil
44
58
  @info_load_time = nil
45
59
  @done = nil
46
- Open.rm path if Open.exist_or_link?(path)
47
- Open.rm tmp_path if Open.exist_or_link?(tmp_path)
48
- Open.rm info_file if Open.exist_or_link?(info_file)
60
+ Open.rm path if Open.exist_or_link?(path) && Path.can_write?(path)
61
+ Open.rm tmp_path if Open.exist_or_link?(tmp_path) && Path.can_write?(tmp_path)
62
+ Open.rm info_file if Open.exist_or_link?(info_file) && Path.can_write?(info_file)
49
63
  Open.rm_rf files_dir if Open.exist_or_link?(files_dir)
50
64
  self
51
65
  end
@@ -59,7 +73,7 @@ class Step
59
73
  dependencies.each do |dep|
60
74
  dep.recursive_clean
61
75
  end
62
- clean
76
+ clean if Open.exists?(self.info_file)
63
77
  end
64
78
 
65
79
  def canfail?
@@ -13,17 +13,18 @@ require_relative 'step/inputs'
13
13
  require_relative 'step/children'
14
14
  require_relative 'step/archive'
15
15
 
16
- class Step
16
+ class Step
17
17
 
18
18
  attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
19
- def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context: nil, &task)
19
+ def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, overriden = nil, exec_context: nil, &task)
20
20
  @path = path
21
21
  @inputs = inputs
22
22
  @dependencies = dependencies
23
23
  @id = id
24
24
  @non_default_inputs = non_default_inputs
25
25
  @provided_inputs = provided_inputs
26
- @compute = compute
26
+ @compute = compute
27
+ @overriden = overriden unless overriden.nil?
27
28
  @task = task
28
29
  @mutex = Mutex.new
29
30
  @tee_copies = 1
@@ -36,12 +37,12 @@ class Step
36
37
 
37
38
  def provided_inputs
38
39
  @provided_inputs ||= begin
39
- if info_file && Open.exists?(info_file)
40
- info[:provided_inputs]
41
- else
42
- {}
43
- end
44
- end
40
+ if info_file && Open.exists?(info_file)
41
+ info[:provided_inputs]
42
+ else
43
+ {}
44
+ end
45
+ end
45
46
  end
46
47
 
47
48
  def non_default_inputs
@@ -97,7 +98,7 @@ class Step
97
98
  return @id if @id
98
99
  return info[:clean_name] if info.include? :clean_name
99
100
  if m = name.match(/(.+?)(?:_[a-z0-9]{32})?(?:\..*)?$/)
100
- return m[1]
101
+ return m[1]
101
102
  end
102
103
  return name.split(".").first
103
104
  end
@@ -189,9 +190,10 @@ class Step
189
190
  end
190
191
  end
191
192
 
192
-
193
193
  return @result || self.load if done?
194
+
194
195
  prepare_dependencies
196
+
195
197
  begin
196
198
 
197
199
 
@@ -200,7 +202,7 @@ class Step
200
202
 
201
203
 
202
204
  reset_info :status => :setup, :issued => Time.now,
203
- :pid => Process.pid, :pid_hostname => Misc.hostname,
205
+ :pid => Process.pid, :pid_hostname => Misc.hostname,
204
206
  :task_name => task_name, :workflow => workflow.to_s,
205
207
  :provided_inputs => IndiferentHash.serializable(provided_inputs),
206
208
  :non_default_inputs => non_default_inputs,
@@ -313,7 +315,7 @@ class Step
313
315
  end
314
316
 
315
317
  def streaming?
316
- @take_stream || IO === @result || StringIO === @result
318
+ @take_stream || IO === @result || StringIO === @result
317
319
  end
318
320
 
319
321
  def stream
@@ -344,16 +346,16 @@ class Step
344
346
  end
345
347
 
346
348
  def consume_all_streams
347
- threads = []
349
+ threads = []
348
350
  while @result && streaming? && stream = self.stream
349
351
  threads << Open.consume_stream(stream, true)
350
352
  end
351
353
 
352
354
  threads.compact!
353
355
 
354
- threads.each do |t|
356
+ threads.each do |t|
355
357
  begin
356
- t.join
358
+ t.join
357
359
  rescue Exception
358
360
  threads.compact.each{|t| t.raise(Aborted); t.join }
359
361
  raise $!
@@ -388,7 +390,7 @@ class Step
388
390
 
389
391
  raise self.exception if self.exception
390
392
 
391
- raise "Error in job #{self.path}" if self.error? or self.aborted?
393
+ raise "Error in job #{self.path}" if self.error? or self.aborted?
392
394
 
393
395
  self
394
396
  end
@@ -2,7 +2,7 @@ module Task
2
2
  def dependencies(id, provided_inputs, non_default_inputs = [], compute = {})
3
3
  return [] if deps.nil?
4
4
  dependencies = []
5
-
5
+
6
6
  provided_inputs ||= {}
7
7
 
8
8
  # Helper function
@@ -66,8 +66,15 @@ module Task
66
66
 
67
67
  if provided_inputs.include?(overriden = [workflow.name, task] * "#")
68
68
  dep = provided_inputs[overriden]
69
- dep = Step.new dep unless Step === dep
70
- dep = dep.dup
69
+ dep = case dep
70
+ when Step
71
+ dep
72
+ when String
73
+ Step.new dep
74
+ when Symbol
75
+ dependencies.select{|dep| dep.task_name == dep }.last
76
+ end
77
+
71
78
  dep.type = workflow.tasks[task].type
72
79
  dep.overriden_task = task
73
80
  dep.overriden_workflow = workflow
@@ -42,8 +42,8 @@ module Workflow
42
42
  def rec_input_use(task_name)
43
43
  input_use = {}
44
44
  task = self.tasks[task_name]
45
- task.inputs.each do |name,_|
46
- input_use[name] ||= {}
45
+ task.inputs.each do |name,_|
46
+ input_use[name] ||= {}
47
47
  input_use[name][self] ||= []
48
48
  input_use[name][self] << task_name
49
49
  end
@@ -69,7 +69,7 @@ module Workflow
69
69
  raise "No '#{name}' task in '#{self.name}' Workflow" if task.nil?
70
70
  id = File.join(self.name, name.to_s)
71
71
  @task_info ||= {}
72
- @task_info[id] ||= begin
72
+ @task_info[id] ||= begin
73
73
  description = task.description
74
74
  returns = task.returns
75
75
 
@@ -1,11 +1,15 @@
1
1
  require 'scout/named_array'
2
2
  module Task
3
3
  def self.format_input(value, type, options = {})
4
+ return nil if value.nil?
4
5
  return value if IO === value || StringIO === value || Step === value
5
6
 
6
- if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options && (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
7
+ return value.to_i if type == :integer && ! value.nil?
8
+ return value.to_f if type == :float && ! value.nil?
9
+
10
+ if String === value && Path.is_filename?(value) && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options && (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
7
11
  if Open.exists?(value) && ! Open.directory?(value)
8
- Persist.load(value, type)
12
+ Persist.load(value, type)
9
13
  else
10
14
  Persist.deserialize(value, type)
11
15
  end
@@ -13,6 +17,8 @@ module Task
13
17
  if m = type.to_s.match(/(.*)_array/)
14
18
  if Array === value
15
19
  value.collect{|v| self.format_input(v, m[1].to_sym, options) }
20
+ else
21
+ value
16
22
  end
17
23
  else
18
24
  value
@@ -96,7 +102,7 @@ module Task
96
102
  basename += "-#{digest}"
97
103
  end
98
104
  new_file = File.join(directory, 'saved_input_files', basename)
99
- relative_file = File.join('.', 'saved_input_files', basename)
105
+ relative_file = File.join('.', 'saved_input_files', basename)
100
106
  Open.link orig_file, new_file
101
107
  relative_file
102
108
  end
@@ -104,7 +110,7 @@ module Task
104
110
  def self.save_input(directory, name, type, value)
105
111
  input_file = File.join(directory, name.to_s)
106
112
 
107
- if Path.is_filename?(value)
113
+ if Path.is_filename?(value)
108
114
  if type == :path
109
115
  Open.write(input_file + ".as_path", value)
110
116
  elsif Path.step_file?(value)
@@ -114,7 +120,7 @@ module Task
114
120
  Open.write(input_file + ".as_file", relative_file)
115
121
  end
116
122
  elsif Step === value
117
- Open.write(input_file + ".as_step", value.short_path)
123
+ Open.write(input_file + ".as_step", value.identify_path)
118
124
  elsif type == :file
119
125
  relative_file = save_file_input(value, directory)
120
126
  Persist.save(relative_file, input_file, :file)
@@ -153,8 +159,8 @@ module Task
153
159
  value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
154
160
  value
155
161
  elsif filename.end_with?('.as_step')
156
- value = Open.read(filename).strip
157
- Step.load value
162
+ value = Path.setup Open.read(filename).strip
163
+ Step.load value.find
158
164
  elsif filename.end_with?('.as_path')
159
165
  value = Open.read(filename).strip
160
166
  Path.setup value
@@ -183,7 +189,7 @@ module Task
183
189
  self.recursive_inputs.each do |p|
184
190
  name, type, desc, value, options = p
185
191
  next if seen.include?(name)
186
- filename = File.join(directory, name.to_s)
192
+ filename = File.join(directory, name.to_s)
187
193
  value = Task.load_input_from_file(filename, type, options)
188
194
  inputs[name] = value unless value.nil?
189
195
  seen << name
@@ -44,10 +44,23 @@ module Task
44
44
 
45
45
  memory_inputs = nil if Array === memory_inputs && memory_inputs.compact.empty?
46
46
  memory_inputs = nil if Hash === memory_inputs && memory_inputs.empty?
47
- Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
47
+ Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {workflow: self.workflow, task: self.name, id: id, provided_inputs: memory_inputs}) do
48
+
49
+ #{{{ Provided inputs and ID
48
50
  provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
49
51
  provided_inputs = {} if provided_inputs.nil?
50
- IndiferentHash.setup(provided_inputs)
52
+ IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
53
+
54
+ provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
55
+
56
+ provided_input_names = case provided_inputs
57
+ when nil
58
+ []
59
+ when Array
60
+ inputs.collect{|name,*| name }[0..provided_inputs.length]
61
+ when Hash
62
+ provided_inputs.keys
63
+ end
51
64
 
52
65
  jobname_input = nil
53
66
  inputs.each do |name,type,desc,default,input_options|
@@ -56,12 +69,14 @@ module Task
56
69
  end
57
70
 
58
71
  id = provided_inputs[jobname_input] if jobname_input && id.nil?
59
- #id = provided_inputs[:id] if provided_inputs.include?(:id)
72
+ id = DEFAULT_NAME if id.nil?
73
+ id = Path.sanitize_filename(id, 150)
60
74
 
75
+ #{{{ Missing inputs
61
76
  missing_inputs = []
62
77
  self.inputs.each do |input,type,desc,val,options|
63
78
  next unless options && options[:required]
64
- missing_inputs << input unless provided_inputs.include?(input)
79
+ missing_inputs << input unless provided_input_names.include?(input.to_s) || provided_input_names.include?(input.to_sym)
65
80
  end if self.inputs
66
81
 
67
82
  if missing_inputs.length == 1
@@ -72,25 +87,32 @@ module Task
72
87
  raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
73
88
  end
74
89
 
75
- provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
76
-
90
+ #{{{ Process inputs
77
91
  job_inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
92
+ non_default_inputs.uniq!
93
+ NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
94
+ step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
78
95
 
96
+ #{{{ Dependencies
79
97
  compute = {}
80
98
  dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
81
99
 
82
- non_default_inputs.uniq!
83
-
84
- non_default_inputs.delete_if{|k| k.to_s.include? "#" } unless dependencies.select{|d| d.overriden? }.any?
100
+ #{{{ Overrides
101
+ override_inputs = provided_input_names.select{|k| (String === k) && k.include?("#") }
102
+ overriden = override_inputs.any? && dependencies.select{|dep| dep.overrider? || dep.overriden? }.any?
85
103
 
86
- id = DEFAULT_NAME if id.nil?
104
+ non_default_inputs.delete_if{|input| override_inputs.include? input }
105
+ non_default_provided_inputs = non_default_inputs & provided_input_names
106
+ non_default_provided_inputs.delete jobname_input if provided_inputs[jobname_input] == id
87
107
 
88
- sanitized_id = Path.sanitize_filename(id, 150)
89
- if non_default_inputs.any? && !(non_default_inputs == [jobname_input] && provided_inputs[jobname_input] == id)
108
+ #{{{ Hash and Path
109
+ if overriden || non_default_provided_inputs.any?
90
110
  hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
91
- name = [sanitized_id, hash] * "_"
111
+ Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies, overriden: override_inputs)}"
112
+ name = [id, hash] * "_"
92
113
  else
93
- name = sanitized_id
114
+ Log.debug "ID #{self.name} #{id} - Clean"
115
+ name = id
94
116
  end
95
117
 
96
118
  extension = self.extension
@@ -110,17 +132,10 @@ module Task
110
132
  end
111
133
 
112
134
  path = directory[name]
113
-
114
135
  path = path.set_extension(extension) if extension
115
136
 
116
- if hash
117
- Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
118
- else
119
- Log.debug "ID #{self.name} #{id} - Clean"
120
- end
121
- NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
122
137
  step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
123
- Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
138
+ Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, overriden, &self
124
139
  end
125
140
  end
126
141
 
@@ -5,19 +5,19 @@ module Task
5
5
  str = StringIO.new
6
6
 
7
7
  if description
8
- title, paragraph = description.split("\n\n")
8
+ title, _, paragraph = description.partition("\n\n")
9
9
  if title.length < Misc::MAX_TTY_LINE_WIDTH
10
10
  title = self.name.to_s + " - " + title
11
11
  str.puts Log.color :yellow, title
12
12
  str.puts Log.color :yellow, "-" * title.length
13
13
  if paragraph
14
- str.puts "\n" << Misc.format_paragraph(paragraph)
14
+ str.puts "\n" << Misc.format_paragraph(paragraph)
15
15
  end
16
16
  else
17
17
  title = self.name.to_s
18
18
  str.puts Log.color :yellow, title
19
19
  str.puts Log.color :yellow, "-" * title.length
20
- str.puts "\n" << Misc.format_paragraph(description)
20
+ str.puts "\n" << Misc.format_paragraph(description)
21
21
  end
22
22
  else
23
23
  title = self.name.to_s
@@ -92,7 +92,7 @@ module Task
92
92
  str.puts
93
93
  str.puts Log.color(:magenta, "Input select options")
94
94
  selects.collect{|p| p}.uniq.each do |input,options|
95
- str.puts
95
+ str.puts
96
96
  str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
97
97
  end
98
98
  end
@@ -164,7 +164,7 @@ module Workflow
164
164
  end
165
165
 
166
166
  def _prov_tasks(tree)
167
- tasks = []
167
+ tasks = []
168
168
  heap = tree.values
169
169
  heap = [tree]
170
170
  while heap.any?
@@ -199,7 +199,7 @@ module Workflow
199
199
  else
200
200
  description += ";" + task_name.to_s
201
201
  end
202
-
202
+
203
203
  seen << [workflow, task_name]
204
204
  end
205
205
  description
@@ -212,9 +212,9 @@ module Workflow
212
212
 
213
213
  offset_str = " " * offset
214
214
 
215
- lines << offset_str
215
+ lines << offset_str
216
216
 
217
- tree.each do |p,dtree|
217
+ tree.each do |p,dtree|
218
218
  next if seen.include?(p)
219
219
  seen.push(p)
220
220
  workflow, task = p
@@ -233,7 +233,7 @@ module Workflow
233
233
  str.puts Log.color :magenta, title
234
234
  str.puts Log.color :magenta, "=" * title.length
235
235
  else
236
- str.puts Log.color :magenta, self.name
236
+ str.puts Log.color :magenta, self.name
237
237
  str.puts Log.color :magenta, "=" * self.name.length
238
238
  end
239
239
 
@@ -244,14 +244,14 @@ module Workflow
244
244
  elsif task.nil?
245
245
 
246
246
  if self.documentation[:description] and not self.documentation[:description].empty?
247
- str.puts Misc.format_paragraph self.documentation[:description]
247
+ str.puts Misc.format_paragraph self.documentation[:description]
248
248
  str.puts
249
249
  end
250
250
 
251
251
  str.puts Log.color :magenta, "## TASKS"
252
252
  if self.documentation[:task_description] and not self.documentation[:task_description].empty?
253
253
  str.puts
254
- str.puts Misc.format_paragraph self.documentation[:task_description]
254
+ str.puts Misc.format_paragraph self.documentation[:task_description]
255
255
  end
256
256
  str.puts
257
257
 
@@ -276,7 +276,7 @@ module Workflow
276
276
 
277
277
  prov_string = prov_string(dep_tree(name))
278
278
  str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
279
- end
279
+ end
280
280
 
281
281
  else
282
282
 
@@ -302,7 +302,7 @@ module Workflow
302
302
  offset, workflow, task_name = m.values_at 1, 2, 3
303
303
  str.puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
304
304
  else
305
- str.puts Log.color :blue, line
305
+ str.puts Log.color :blue, line
306
306
  end
307
307
  end
308
308
  str.puts
@@ -18,7 +18,7 @@ module Workflow
18
18
  end
19
19
 
20
20
  def all_tasks
21
- tasks.keys
21
+ tasks.nil? ? [] : tasks.keys
22
22
  end
23
23
 
24
24
  def self.list
@@ -27,9 +27,9 @@ module Workflow
27
27
  end
28
28
 
29
29
  def workflow_dir
30
- @workflow_dir ||
31
- ENV["SCOUT_WORKFLOW_DIR"] ||
32
- begin
30
+ @workflow_dir ||
31
+ ENV["SCOUT_WORKFLOW_DIR"] ||
32
+ begin
33
33
  workflow_dir_config = Path.setup("etc/workflow_dir")
34
34
  if workflow_dir_config.exists?
35
35
  Path.setup(workflow_dir_config.read.strip)
@@ -40,9 +40,9 @@ module Workflow
40
40
  end
41
41
 
42
42
  def workflow_repo
43
- @workflow_repo ||
44
- ENV["SCOUT_WORKFLOW_REPO"] ||
45
- begin
43
+ @workflow_repo ||
44
+ ENV["SCOUT_WORKFLOW_REPO"] ||
45
+ begin
46
46
  workflow_repo_config = Path.setup("etc/workflow_repo")
47
47
  if workflow_repo_config.exists?
48
48
  workflow_repo_config.read.strip