scout-gear 10.4.0 → 10.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -656
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +41 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +87 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +97 -21
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +3 -0
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/index.rb +88 -36
  40. data/lib/scout/tsv/open.rb +21 -8
  41. data/lib/scout/tsv/parser.rb +153 -90
  42. data/lib/scout/tsv/path.rb +7 -2
  43. data/lib/scout/tsv/stream.rb +48 -6
  44. data/lib/scout/tsv/transformer.rb +5 -3
  45. data/lib/scout/tsv/traverse.rb +28 -19
  46. data/lib/scout/tsv/util/process.rb +7 -0
  47. data/lib/scout/tsv/util/reorder.rb +25 -15
  48. data/lib/scout/tsv/util/select.rb +9 -1
  49. data/lib/scout/tsv/util/sort.rb +90 -2
  50. data/lib/scout/tsv/util/unzip.rb +56 -0
  51. data/lib/scout/tsv/util.rb +52 -5
  52. data/lib/scout/tsv.rb +42 -27
  53. data/lib/scout/work_queue/socket.rb +8 -0
  54. data/lib/scout/work_queue/worker.rb +22 -5
  55. data/lib/scout/work_queue.rb +41 -24
  56. data/lib/scout/workflow/definition.rb +15 -12
  57. data/lib/scout/workflow/deployment/orchestrator.rb +21 -3
  58. data/lib/scout/workflow/deployment/trace.rb +205 -0
  59. data/lib/scout/workflow/deployment.rb +1 -0
  60. data/lib/scout/workflow/documentation.rb +1 -1
  61. data/lib/scout/workflow/step/archive.rb +42 -0
  62. data/lib/scout/workflow/step/children.rb +51 -0
  63. data/lib/scout/workflow/step/config.rb +1 -1
  64. data/lib/scout/workflow/step/dependencies.rb +25 -8
  65. data/lib/scout/workflow/step/file.rb +19 -0
  66. data/lib/scout/workflow/step/info.rb +37 -9
  67. data/lib/scout/workflow/step/progress.rb +11 -2
  68. data/lib/scout/workflow/step/status.rb +9 -1
  69. data/lib/scout/workflow/step.rb +80 -25
  70. data/lib/scout/workflow/task/dependencies.rb +5 -2
  71. data/lib/scout/workflow/task/inputs.rb +91 -41
  72. data/lib/scout/workflow/task.rb +54 -57
  73. data/lib/scout/workflow/usage.rb +1 -1
  74. data/lib/scout/workflow/util.rb +4 -0
  75. data/lib/scout/workflow.rb +110 -13
  76. data/lib/scout-gear.rb +2 -0
  77. data/lib/scout.rb +0 -1
  78. data/scout-gear.gemspec +78 -23
  79. data/scout_commands/rbbt +2 -0
  80. data/test/data/person/brothers +4 -0
  81. data/test/data/person/identifiers +10 -0
  82. data/test/data/person/marriages +3 -0
  83. data/test/data/person/parents +6 -0
  84. data/test/scout/association/test_fields.rb +105 -0
  85. data/test/scout/association/test_index.rb +70 -0
  86. data/test/scout/association/test_item.rb +21 -0
  87. data/test/scout/entity/test_format.rb +19 -0
  88. data/test/scout/entity/test_identifiers.rb +58 -0
  89. data/test/scout/entity/test_object.rb +0 -0
  90. data/test/scout/entity/test_property.rb +345 -0
  91. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  92. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  93. data/test/scout/persist/engine/test_sharder.rb +31 -0
  94. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  95. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  96. data/test/scout/persist/test_tsv.rb +146 -0
  97. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  98. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  99. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  100. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  101. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  102. data/test/scout/{tsv/persist → persist/tsv/adapter}/test_tkrzw.rb +3 -6
  103. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  104. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  105. data/test/scout/test_association.rb +51 -0
  106. data/test/scout/test_entity.rb +40 -0
  107. data/test/scout/test_tsv.rb +33 -4
  108. data/test/scout/test_work_queue.rb +5 -2
  109. data/test/scout/test_workflow.rb +31 -14
  110. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  111. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  112. data/test/scout/tsv/test_annotation.rb +52 -0
  113. data/test/scout/tsv/test_attach.rb +255 -1
  114. data/test/scout/tsv/test_change_id.rb +25 -0
  115. data/test/scout/tsv/test_csv.rb +50 -0
  116. data/test/scout/tsv/test_dumper.rb +38 -0
  117. data/test/scout/tsv/test_index.rb +82 -0
  118. data/test/scout/tsv/test_open.rb +44 -0
  119. data/test/scout/tsv/test_parser.rb +70 -0
  120. data/test/scout/tsv/test_stream.rb +22 -0
  121. data/test/scout/tsv/test_transformer.rb +27 -3
  122. data/test/scout/tsv/test_traverse.rb +78 -0
  123. data/test/scout/tsv/util/test_process.rb +16 -0
  124. data/test/scout/tsv/util/test_reorder.rb +67 -0
  125. data/test/scout/tsv/util/test_sort.rb +28 -1
  126. data/test/scout/tsv/util/test_unzip.rb +32 -0
  127. data/test/scout/work_queue/test_socket.rb +4 -1
  128. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  129. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  130. data/test/scout/workflow/step/test_archive.rb +28 -0
  131. data/test/scout/workflow/step/test_children.rb +25 -0
  132. data/test/scout/workflow/step/test_info.rb +16 -0
  133. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  134. data/test/scout/workflow/task/test_inputs.rb +45 -1
  135. data/test/scout/workflow/test_definition.rb +52 -0
  136. data/test/scout/workflow/test_step.rb +57 -0
  137. data/test/scout/workflow/test_task.rb +26 -1
  138. data/test/scout/workflow/test_usage.rb +4 -4
  139. data/test/test_helper.rb +23 -1
  140. metadata +69 -14
  141. data/lib/scout/tsv/persist.rb +0 -27
  142. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  143. data/test/scout/tsv/test_persist.rb +0 -45
@@ -19,15 +19,17 @@ class Step
19
19
  @info_load_time = Time.now
20
20
  end
21
21
 
22
-
23
-
24
22
  def save_info(info = nil)
25
23
  Persist.save(info, info_file, SERIALIZER)
26
24
  @info_load_time = Time.now
27
25
  end
28
26
 
29
- def clear_info
30
- save_info(@info = {})
27
+ def reset_info(info = {})
28
+ save_info(@info = info)
29
+ end
30
+
31
+ def init_info
32
+ log :waiting unless info_file.nil? || Open.exists?(info_file)
31
33
  end
32
34
 
33
35
  def info
@@ -47,20 +49,36 @@ class Step
47
49
  def merge_info(new_info)
48
50
  info = self.info
49
51
  new_info.each do |key,value|
52
+ value = Annotation.purge(value)
50
53
  if key == :status
51
- message = new_info[:messages]
54
+ message = new_info[:message]
52
55
  if message.nil? && (value == :done || value == :error || value == :aborted)
56
+ issued = info[:issued]
53
57
  start = info[:start]
54
58
  eend = new_info[:end]
55
59
  if start && eend
56
60
  time = eend - start
57
- time_str = Misc.format_seconds_short(time)
61
+ total_time = eend - issued
62
+ if total_time - time > 1
63
+ time_str = "#{Misc.format_seconds_short(time)} (#{Misc.format_seconds_short(total_time)})"
64
+ else
65
+ time_str = Misc.format_seconds_short(time)
66
+ end
67
+ info[:time_elapsed] = time
68
+ info[:total_time_elapsed] = total_time
58
69
  message = Log.color(:time, time_str)
59
70
  end
60
71
  end
61
72
  report_status value, message
62
73
  end
63
74
 
75
+ if key == :message
76
+ messages = info[:messages] || []
77
+ messages << value
78
+ info[:messages] = messages
79
+ next
80
+ end
81
+
64
82
  if Exception === value
65
83
  begin
66
84
  Marshal.dump(value)
@@ -111,12 +129,16 @@ class Step
111
129
  end
112
130
 
113
131
  if message
114
- merge_info :status => status, :messages => message
132
+ merge_info :status => status, :message => message
115
133
  else
116
134
  merge_info :status => status
117
135
  end
118
136
  end
119
137
 
138
+ def messages
139
+ info[:messages]
140
+ end
141
+
120
142
  def status
121
143
  info[:status].tap{|s| s.nil? ? s : s.to_sym }
122
144
  end
@@ -134,7 +156,8 @@ class Step
134
156
  end
135
157
 
136
158
  def overriden?
137
- overriden_task || overriden_workflow || dependencies.select{|d| d.overriden? }.any?
159
+ @overriden = overriden_task || overriden_workflow || overriden_deps.any? if @overriden.nil?
160
+ @overriden
138
161
  end
139
162
 
140
163
  def overriden_deps
@@ -145,10 +168,15 @@ class Step
145
168
  info[:exception]
146
169
  end
147
170
 
148
- def marshal_dump
171
+ # Marshal Step
172
+ def _dump(level)
149
173
  @path
150
174
  end
151
175
 
176
+ def self._load(path)
177
+ Step.new path
178
+ end
179
+
152
180
  def marshal_load(path)
153
181
  Step.new path
154
182
  end
@@ -1,5 +1,5 @@
1
1
  class Step
2
- def progress_bar(msg = "Progress", options = nil)
2
+ def progress_bar(msg = "Progress", options = nil, &block)
3
3
  if Hash === msg and options.nil?
4
4
  options = msg
5
5
  msg = nil
@@ -8,7 +8,16 @@ class Step
8
8
 
9
9
  max = options[:max]
10
10
  Open.mkdir files_dir
11
- Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
11
+ bar = Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
12
+
13
+ if block_given?
14
+ bar.init
15
+ res = yield bar
16
+ bar.remove
17
+ res
18
+ else
19
+ bar
20
+ end
12
21
  end
13
22
  end
14
23
 
@@ -1,11 +1,18 @@
1
1
  class Step
2
2
  def abort(exception = nil)
3
3
  if (pid = info[:pid]) && pid != Process.pid && Misc.pid_alive?(pid)
4
- Process.kill pid
4
+ Log.debug "Kill process #{pid} to abort step #{Log.fingerprint self}"
5
+ begin
6
+ s = Misc.abort_child pid, true
7
+ Log.medium "Aborted pid #{path} #{s}"
8
+ rescue
9
+ Log.debug("Aborted job #{pid} was not killed: #{$!.message}")
10
+ end
5
11
  else
6
12
  while @result && streaming? && stream = self.stream
7
13
  stream.abort(exception)
8
14
  end
15
+ @take_stream.abort(exception) if streaming?
9
16
  end
10
17
  end
11
18
 
@@ -19,6 +26,7 @@ class Step
19
26
  newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
20
27
  newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
21
28
 
29
+ Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
22
30
  newer.empty?
23
31
  end
24
32
 
@@ -1,5 +1,6 @@
1
1
  require 'scout/path'
2
2
  require 'scout/persist'
3
+ require 'scout/semaphore'
3
4
  require_relative 'step/info'
4
5
  require_relative 'step/status'
5
6
  require_relative 'step/load'
@@ -9,11 +10,13 @@ require_relative 'step/provenance'
9
10
  require_relative 'step/config'
10
11
  require_relative 'step/progress'
11
12
  require_relative 'step/inputs'
13
+ require_relative 'step/children'
14
+ require_relative 'step/archive'
12
15
 
13
16
  class Step
14
17
 
15
- attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow
16
- def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, &task)
18
+ attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
19
+ def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context = nil, &task)
17
20
  @path = path
18
21
  @inputs = inputs
19
22
  @dependencies = dependencies
@@ -24,6 +27,7 @@ class Step
24
27
  @task = task
25
28
  @mutex = Mutex.new
26
29
  @tee_copies = 1
30
+ @exec_context = exec_context || self
27
31
  end
28
32
 
29
33
  def synchronize(&block)
@@ -44,10 +48,11 @@ class Step
44
48
 
45
49
  def dependencies
46
50
  @dependencies ||= begin
47
- if Open.exists?(info_file)
51
+ if info_file && Open.exists?(info_file) && info[:dependencies]
48
52
  info[:dependencies].collect do |path|
53
+ path = path.last if Array === path
49
54
  Step.load(path)
50
- end if info[:dependencies]
55
+ end
51
56
  else
52
57
  []
53
58
  end
@@ -56,7 +61,7 @@ class Step
56
61
 
57
62
  attr_accessor :type
58
63
  def type
59
- @type ||= @task.respond_to?(:type) ? @task.type : info[:type]
64
+ @type ||= (@task.respond_to?(:type) && @task.type) ? @task.type : info[:type]
60
65
  end
61
66
 
62
67
  def name
@@ -70,7 +75,9 @@ class Step
70
75
  def clean_name
71
76
  return @id if @id
72
77
  return info[:clean_name] if info.include? :clean_name
73
- return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
78
+ if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
79
+ return m[1]
80
+ end
74
81
  return name.split(".").first
75
82
  end
76
83
 
@@ -82,10 +89,16 @@ class Step
82
89
 
83
90
  def workflow
84
91
  @workflow ||= @task.workflow if Task === @task
85
- @workflow ||= info[:workflow] if Open.exist?(info_file)
92
+ @workflow ||= info[:workflow] if info_file && Open.exist?(info_file)
86
93
  @workflow ||= path.split("/")[-3]
87
94
  end
88
95
 
96
+ def full_task_name
97
+ return nil if task_name.nil?
98
+ return task_name.to_s if workflow.nil?
99
+ [workflow, task_name] * "#"
100
+ end
101
+
89
102
  def exec
90
103
 
91
104
  if inputs
@@ -107,7 +120,7 @@ class Step
107
120
 
108
121
  @result = begin
109
122
  @in_exec = true
110
- self.instance_exec(*inputs, &task)
123
+ @exec_context.instance_exec(*inputs, &task)
111
124
  ensure
112
125
  @in_exec = false
113
126
  end
@@ -127,23 +140,37 @@ class Step
127
140
  def run(stream = false)
128
141
  return @result || self.load if done?
129
142
  prepare_dependencies
130
- run_dependencies
131
143
  begin
132
- @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
133
- clear_info
144
+
145
+ case stream
146
+ when TrueClass, :stream
147
+ no_load = :stream
148
+ when :no_load
149
+ no_load = true
150
+ else
151
+ no_load = false
152
+ end
153
+
154
+ @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
134
155
  input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
135
- merge_info :status => :start, :start => Time.now,
156
+
157
+
158
+ reset_info :status => :setup, :issued => Time.now,
136
159
  :pid => Process.pid, :pid_hostname => Misc.hostname,
137
160
  :task_name => task_name, :workflow => workflow.to_s,
138
- :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
139
- :dependencies => dependencies.collect{|d| d.path }
161
+ :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
162
+ :dependencies => (dependencies || []) .collect{|d| d.path }
140
163
 
164
+ run_dependencies
165
+
166
+ set_info :start, Time.now
167
+ log :start
141
168
  @exec_result = exec
142
169
 
143
170
  if @exec_result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
144
171
  Open.mv self.tmp_path, self.path
145
172
  else
146
- @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream)
173
+ @exec_result = @exec_result.stream if @exec_result.respond_to?(:stream) && ! (TSV === @exec_result)
147
174
  end
148
175
 
149
176
  @exec_result
@@ -155,13 +182,25 @@ class Step
155
182
  @exec_result
156
183
  end
157
184
  end
185
+
186
+ if TrueClass === no_load
187
+ consume_all_streams if streaming?
188
+ @result = nil
189
+ elsif no_load && ! (IO === @result)
190
+ @result = nil
191
+ end
192
+
193
+ @result
158
194
  rescue Exception => e
159
- merge_info :status => :error, :exception => e, :end => Time.now
160
- abort_dependencies
161
- raise e
195
+ merge_info :status => :error, :exception => e, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
196
+ begin
197
+ abort_dependencies
198
+ ensure
199
+ raise e
200
+ end
162
201
  ensure
163
202
  if ! (error? || aborted?)
164
- if streaming?
203
+ if @result && streaming?
165
204
  ConcurrentStream.setup(@result) do
166
205
  merge_info :status => :done, :end => Time.now
167
206
  end
@@ -187,12 +226,22 @@ class Step
187
226
  end
188
227
  end
189
228
 
190
- def fork
191
- Process.fork do
192
- clear_info unless present?
193
- run(false)
229
+ def fork(noload = false, semaphore = nil)
230
+ pid = Process.fork do
231
+ Signal.trap(:TERM) do
232
+ raise Aborted, "Recieved TERM Signal on forked process #{Process.pid}"
233
+ end
234
+ reset_info status: :queue, pid: Process.pid unless present?
235
+ if semaphore
236
+ ScoutSemaphore.synchronize(semaphore) do
237
+ run(noload)
238
+ end
239
+ else
240
+ run(noload)
241
+ end
194
242
  join
195
243
  end
244
+ Process.detach pid
196
245
  grace
197
246
  self
198
247
  end
@@ -272,17 +321,23 @@ class Step
272
321
  while @result.nil? && (present? && ! (terminated? || done?))
273
322
  sleep 0.1
274
323
  end
324
+
325
+ Misc.wait_child info[:pid] if info[:pid]
326
+
275
327
  raise self.exception if self.exception
328
+
276
329
  raise "Error in job #{self.path}" if self.error? or self.aborted?
330
+
277
331
  self
278
332
  end
279
333
 
280
334
  def produce(with_fork: false)
335
+ clean if error? && recoverable_error?
281
336
  if with_fork
282
337
  self.fork
283
338
  self.join
284
339
  else
285
- run
340
+ run(:no_load)
286
341
  end
287
342
  self
288
343
  end
@@ -305,7 +360,7 @@ class Step
305
360
  end
306
361
 
307
362
  def short_path
308
- Scout.identify @path
363
+ Resource.identify @path
309
364
  end
310
365
 
311
366
  def digest_str
@@ -19,7 +19,7 @@ module Task
19
19
  step_inputs.each do |k,v|
20
20
  if Symbol === v
21
21
  input_dep = dependencies.select{|d| d.task_name == v }.first
22
- resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || k
22
+ resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || v
23
23
  else
24
24
  resolved_inputs[k] = v
25
25
  end
@@ -31,6 +31,9 @@ module Task
31
31
  compute_options << :produce if definition_options[:produce]
32
32
  compute_options << :stream if definition_options[:stream]
33
33
  compute[job.path] = compute_options if compute_options.any?
34
+
35
+ job.overriden = false if definition_options[:not_overriden]
36
+
34
37
  [job, step_inputs]
35
38
  end
36
39
 
@@ -98,7 +101,7 @@ module Task
98
101
  else
99
102
  dep = _res
100
103
  dependencies << dep
101
- dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
104
+ dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
102
105
  non_default_inputs.concat(dep_non_default_inputs)
103
106
  end
104
107
  end
@@ -40,7 +40,16 @@ module Task
40
40
  input_names << name
41
41
  provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
42
42
  provided = Task.format_input(provided, type, options || {})
43
- if ! provided.nil? && provided != value
43
+ if provided == value
44
+ same_as_default = true
45
+ elsif String === provided && Symbol === value && provided == value.to_s
46
+ same_as_default = true
47
+ elsif String === value && Symbol === provided && provided.to_s == value
48
+ same_as_default = true
49
+ else
50
+ same_as_default = false
51
+ end
52
+ if ! provided.nil? && ! same_as_default
44
53
  non_default_inputs << name.to_sym
45
54
  input_array << provided
46
55
  elsif options && options[:jobname]
@@ -61,7 +70,7 @@ module Task
61
70
  [input_array, non_default_inputs, digest_str]
62
71
  end
63
72
 
64
- def save_file_input(orig_file, directory)
73
+ def self.save_file_input(orig_file, directory)
65
74
  orig_file = orig_file.path if Step === orig_file
66
75
  basename = File.basename(orig_file)
67
76
  digest = Misc.digest(orig_file)
@@ -76,38 +85,65 @@ module Task
76
85
  relative_file
77
86
  end
78
87
 
88
+ def self.save_input(directory, name, type, value)
89
+ input_file = File.join(directory, name.to_s)
90
+
91
+ if Path.is_filename?(value)
92
+ if type == :path
93
+ Open.write(input_file + ".as_path", value)
94
+ else
95
+ relative_file = save_file_input(value, directory)
96
+ Open.write(input_file + ".as_file", relative_file)
97
+ end
98
+ elsif Step === value
99
+ Open.write(input_file + ".as_step", value.short_path)
100
+ elsif type == :file
101
+ relative_file = save_file_input(value, directory)
102
+ Persist.save(relative_file, input_file, :file)
103
+ elsif type == :file_array
104
+ new_files = value.collect do |orig_file|
105
+ save_file_input(orig_file, directory)
106
+ end
107
+ Persist.save(new_files, input_file, type)
108
+ elsif Open.is_stream?(value)
109
+ Open.sensible_write(input_file, value)
110
+ elsif Open.has_stream?(value)
111
+ Open.sensible_write(input_file, value.stream)
112
+ else
113
+ Persist.save(value, input_file, type)
114
+ end
115
+ end
116
+
79
117
  def save_inputs(directory, provided_inputs = {})
80
- #input_array, non_default_inputs = assign_inputs(provided_inputs)
81
118
  self.recursive_inputs.each_with_index do |p,i|
82
119
  name, type, desc, value, options = p
83
120
  next unless provided_inputs.include?(name)
84
121
  value = provided_inputs[name]
85
- input_file = File.join(directory, name.to_s)
86
-
87
- if Path.is_filename?(value)
88
- if type == :path
89
- Open.write(input_file + ".as_path", value)
90
- else
91
- relative_file = save_file_input(value, directory)
92
- Open.write(input_file + ".as_file", relative_file)
93
- end
94
- elsif Step === value
95
- Open.write(input_file + ".as_step", value.short_path)
96
- elsif type == :file
97
- relative_file = save_file_input(value, directory)
98
- Persist.save(relative_file, input_file, :file)
99
- elsif type == :file_array
100
- new_files = value.collect do |orig_file|
101
- save_file_input(orig_file, directory)
102
- end
103
- Persist.save(new_files, input_file, type)
104
- elsif Open.is_stream?(value)
105
- Persist.save(value, input_file, type)
106
- elsif Open.has_stream?(value)
107
- Persist.save(value.stream, input_file, type)
122
+
123
+ Task.save_input(directory, name, type, value)
124
+ end
125
+ end
126
+
127
+
128
+ def self.load_input_from_file(filename, type, options = nil)
129
+ if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
130
+ if filename.end_with?('.as_file')
131
+ value = Open.read(filename).strip
132
+ value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
133
+ value
134
+ elsif filename.end_with?('.as_step')
135
+ value = Open.read(filename).strip
136
+ Step.load value
137
+ elsif filename.end_with?('.as_path')
138
+ value = Open.read(filename).strip
139
+ Path.setup value
140
+ elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
141
+ filename
108
142
  else
109
- Persist.save(value, input_file, type)
143
+ Persist.load(filename, type)
110
144
  end
145
+ else
146
+ return nil
111
147
  end
112
148
  end
113
149
 
@@ -116,21 +152,35 @@ module Task
116
152
  self.recursive_inputs.each do |p|
117
153
  name, type, desc, value, options = p
118
154
  filename = File.join(directory, name.to_s)
119
- if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
120
- if filename.end_with?('.as_file')
121
- value = Open.read(filename).strip
122
- value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
123
- inputs[name] = value
124
- elsif filename.end_with?('.as_step')
125
- value = Open.read(filename).strip
126
- inputs[name] = Step.load value
127
- elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
128
- inputs[name] = filename
129
- else
130
- inputs[name] = Persist.load(filename, type)
131
- end
132
- end
155
+ value = Task.load_input_from_file(filename, type, options)
156
+ inputs[name] = value unless value.nil?
157
+ end
158
+
159
+ Dir.glob(File.join(directory, "*#*")).each do |file|
160
+ override_dep, _, extension = File.basename(file).partition(".")
161
+
162
+ inputs[override_dep] = Task.load_input_from_file(file, :file)
133
163
  end
164
+
134
165
  inputs
135
166
  end
167
+
168
+ def recursive_inputs(overriden = [])
169
+ return inputs.dup if deps.nil?
170
+ deps.inject(inputs.dup) do |acc,dep|
171
+ workflow, task, options = dep
172
+ next acc if workflow.nil? || task.nil?
173
+ next acc if overriden.include?([workflow.name, task.to_s] * "#")
174
+ overriden.concat options.keys.select{|k| k.to_s.include?("#") } if options
175
+
176
+ workflow.tasks[task].recursive_inputs(overriden).dup.each do |info|
177
+ name, _ = info
178
+ next if options.include?(name.to_sym) || options.include?(name.to_s)
179
+ acc << info
180
+ end
181
+
182
+ acc
183
+ end
184
+ end
185
+
136
186
  end