scout-gear 1.2.0 → 5.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +4 -0
  3. data/.vimproject +663 -4
  4. data/Rakefile +1 -0
  5. data/VERSION +1 -1
  6. data/bin/scout +235 -0
  7. data/lib/scout/cmd.rb +344 -0
  8. data/lib/scout/concurrent_stream.rb +259 -0
  9. data/lib/scout/exceptions.rb +15 -8
  10. data/lib/scout/indiferent_hash/options.rb +8 -26
  11. data/lib/scout/indiferent_hash.rb +0 -30
  12. data/lib/scout/log/color.rb +2 -2
  13. data/lib/scout/log/fingerprint.rb +11 -1
  14. data/lib/scout/log/progress/report.rb +0 -1
  15. data/lib/scout/log/progress/util.rb +1 -1
  16. data/lib/scout/log/progress.rb +4 -4
  17. data/lib/scout/log.rb +10 -2
  18. data/lib/scout/meta_extension.rb +15 -1
  19. data/lib/scout/misc/digest.rb +56 -0
  20. data/lib/scout/misc/filesystem.rb +26 -0
  21. data/lib/scout/misc/format.rb +226 -0
  22. data/lib/scout/misc/insist.rb +56 -0
  23. data/lib/scout/misc.rb +5 -11
  24. data/lib/scout/open/lock.rb +61 -0
  25. data/lib/scout/open/remote.rb +120 -0
  26. data/lib/scout/open/stream.rb +372 -0
  27. data/lib/scout/open/util.rb +225 -0
  28. data/lib/scout/open.rb +169 -0
  29. data/lib/scout/path/find.rb +78 -26
  30. data/lib/scout/path/tmpfile.rb +8 -0
  31. data/lib/scout/path/util.rb +17 -5
  32. data/lib/scout/path.rb +13 -31
  33. data/lib/scout/persist/open.rb +17 -0
  34. data/lib/scout/persist/path.rb +15 -0
  35. data/lib/scout/persist/serialize.rb +140 -0
  36. data/lib/scout/persist.rb +54 -0
  37. data/lib/scout/resource/path.rb +15 -0
  38. data/lib/scout/resource/produce/rake.rb +69 -0
  39. data/lib/scout/resource/produce.rb +246 -0
  40. data/lib/scout/resource/scout.rb +3 -0
  41. data/lib/scout/resource.rb +37 -0
  42. data/lib/scout/simple_opt/accessor.rb +54 -0
  43. data/lib/scout/simple_opt/doc.rb +102 -0
  44. data/lib/scout/simple_opt/get.rb +57 -0
  45. data/lib/scout/simple_opt/parse.rb +67 -0
  46. data/lib/scout/simple_opt/setup.rb +26 -0
  47. data/lib/scout/simple_opt.rb +5 -0
  48. data/lib/scout/tmpfile.rb +39 -1
  49. data/lib/scout/workflow/definition.rb +72 -0
  50. data/lib/scout/workflow/documentation.rb +77 -0
  51. data/lib/scout/workflow/step/info.rb +77 -0
  52. data/lib/scout/workflow/step.rb +96 -0
  53. data/lib/scout/workflow/task/inputs.rb +112 -0
  54. data/lib/scout/workflow/task.rb +141 -0
  55. data/lib/scout/workflow/usage.rb +294 -0
  56. data/lib/scout/workflow/util.rb +11 -0
  57. data/lib/scout/workflow.rb +39 -0
  58. data/lib/scout-gear.rb +5 -0
  59. data/lib/scout.rb +1 -0
  60. data/lib/workflow-scout.rb +2 -0
  61. data/scout-gear.gemspec +78 -5
  62. data/scout_commands/alias +48 -0
  63. data/scout_commands/find +83 -0
  64. data/scout_commands/glob +0 -0
  65. data/scout_commands/rbbt +23 -0
  66. data/scout_commands/workflow/task +707 -0
  67. data/test/scout/indiferent_hash/test_case_insensitive.rb +16 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +37 -1
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/simple_opt/test_get.rb +11 -0
  83. data/test/scout/simple_opt/test_parse.rb +10 -0
  84. data/test/scout/simple_opt/test_setup.rb +77 -0
  85. data/test/scout/test_cmd.rb +85 -0
  86. data/test/scout/test_concurrent_stream.rb +29 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +28 -0
  94. data/test/scout/workflow/task/test_inputs.rb +182 -0
  95. data/test/scout/workflow/test_step.rb +36 -0
  96. data/test/scout/workflow/test_task.rb +178 -0
  97. data/test/scout/workflow/test_usage.rb +26 -0
  98. data/test/scout/workflow/test_util.rb +17 -0
  99. data/test/test_helper.rb +17 -0
  100. data/test/test_scout-gear.rb +0 -0
  101. metadata +76 -3
data/lib/scout/tmpfile.rb CHANGED
@@ -1,7 +1,10 @@
1
- require 'fileutils'
2
1
  require_relative 'misc'
2
+ require_relative 'log'
3
+ require 'fileutils'
3
4
 
4
5
  module TmpFile
6
+ MAX_FILE_LENGTH = 150
7
+
5
8
  def self.user_tmp(subdir = nil)
6
9
  if subdir
7
10
  File.join(ENV["HOME"],"/tmp/scout", subdir)
@@ -89,4 +92,39 @@ module TmpFile
89
92
  end
90
93
  end
91
94
  end
95
+
96
+ def self.tmp_for_file(file, tmp_options = {}, other_options = {})
97
+ tmp_for_file = IndiferentHash.process_options tmp_options, :file
98
+ return tmp_for_file unless tmp_for_file.nil?
99
+
100
+ prefix = IndiferentHash.process_options tmp_options, :prefix
101
+
102
+ if prefix.nil?
103
+ perfile = file.to_s.gsub(/\//, '>')
104
+ else
105
+ perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>')
106
+ end
107
+
108
+ perfile.sub!(/\.b?gz$/,'')
109
+
110
+ if other_options.include? :filters
111
+ other_options[:filters].each do |match,value|
112
+ perfile = perfile + "&F[#{match}=#{Misc.digest(value)}]"
113
+ end
114
+ end
115
+
116
+ persistence_dir = IndiferentHash.process_options(tmp_options, :dir) || TmpFile.tmpdir
117
+ Path.setup(persistence_dir) unless Path === persistence_dir
118
+
119
+ filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
120
+ clean_options = other_options.dup
121
+ clean_options.delete :unnamed
122
+ clean_options.delete "unnamed"
123
+
124
+ filename = filename[0..MAX_FILE_LENGTH] << Misc.digest(filename[MAX_FILE_LENGTH+1..-1]) if filename.length > MAX_FILE_LENGTH + 10
125
+
126
+ filename += ":" << Misc.digest(clean_options) unless clean_options.empty?
127
+
128
+ persistence_dir[filename]
129
+ end
92
130
  end
@@ -0,0 +1,72 @@
1
+ require_relative '../meta_extension'
2
+
3
+ module Workflow
4
+ extend MetaExtension
5
+ extension_attr :name, :tasks
6
+
7
+ class << self
8
+ attr_accessor :directory
9
+
10
+ def directory
11
+ @directory ||= Path.setup('var/jobs')
12
+ end
13
+
14
+ end
15
+
16
+ def name
17
+ @name ||= self.to_s
18
+ end
19
+
20
+ attr_accessor :directory
21
+ def directory
22
+ @directory ||= Workflow.directory[name]
23
+ end
24
+
25
+ def directory=(directory)
26
+ @directory = directory
27
+ @tasks.each{|name,d| d.directory = directory[name] } if @tasks
28
+ end
29
+
30
+ def annotate_next_task(type, obj)
31
+ @annotate_next_task ||= {}
32
+ @annotate_next_task[type] ||= []
33
+ @annotate_next_task[type] << obj
34
+ end
35
+
36
+ def dep(*args, &block)
37
+ case args.length
38
+ when 3
39
+ workflow, task, options = args
40
+ when 2
41
+ if Hash === args.last
42
+ task, options = args
43
+ else
44
+ workflow, task = args
45
+ end
46
+ when 1
47
+ task = args.first
48
+ end
49
+ workflow = self if workflow.nil?
50
+ options = {} if options.nil?
51
+ annotate_next_task :deps, [workflow, task, options, block, args]
52
+ end
53
+
54
+ def input(*args)
55
+ annotate_next_task(:inputs, args)
56
+ end
57
+
58
+ def task(name_and_type, &block)
59
+ name, type = name_and_type.collect.first
60
+ @tasks ||= IndiferentHash.setup({})
61
+ begin
62
+ @annotate_next_task ||= {}
63
+ @tasks[name] = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
64
+ ensure
65
+ @annotate_next_task = {}
66
+ end
67
+ end
68
+
69
+ def desc(description)
70
+ annotate_next_task(:desc, description)
71
+ end
72
+ end
@@ -0,0 +1,77 @@
1
+ module Workflow
2
+
3
+ def self.doc_parse_first_line(str)
4
+ if str.match(/^([^\n]*)\n\n(.*)/sm)
5
+ str.replace $2
6
+ $1
7
+ else
8
+ ""
9
+ end
10
+ end
11
+
12
+ def self.doc_parse_up_to(str, pattern, keep = false)
13
+ pre, _pat, _post = str.partition pattern
14
+ if _pat
15
+ [pre, (keep ? _pat << _post : _post)]
16
+ else
17
+ _post
18
+ end
19
+ end
20
+
21
+ def self.doc_parse_chunks(str, pattern)
22
+ parts = str.split(pattern)
23
+ return {} if parts.length < 2
24
+ tasks = Hash[*parts[1..-1].collect{|v| v.strip}]
25
+ tasks.delete_if{|t,d| d.empty?}
26
+ tasks
27
+ end
28
+
29
+ def self.parse_workflow_doc(doc)
30
+ title = doc_parse_first_line doc
31
+ description, task_info = doc_parse_up_to doc, /^# Tasks/i
32
+ task_description, tasks = doc_parse_up_to task_info, /^##/, true
33
+ tasks = doc_parse_chunks tasks, /## (.*)/
34
+ {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
35
+ end
36
+
37
+ def documentation_markdown
38
+ return "" if @libdir.nil?
39
+ file = @libdir['workflow.md'].find
40
+ file = @libdir['README.md'].find unless file.exists?
41
+ if file.exists?
42
+ file.read
43
+ else
44
+ ""
45
+ end
46
+ end
47
+
48
+ def load_documentation
49
+ return if @documentation
50
+ @documentation ||= Workflow.parse_workflow_doc documentation_markdown
51
+ @documentation[:tasks].each do |task, description|
52
+ if task.include? "#"
53
+ workflow, task = task.split("#")
54
+ workflow = begin
55
+ Kernel.const_get workflow
56
+ rescue
57
+ next
58
+ end
59
+ else
60
+ workflow = self
61
+ end
62
+
63
+ task = task.to_sym
64
+ if workflow.tasks.include? task
65
+ workflow.tasks[task].description = description
66
+ else
67
+ Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task"
68
+ end
69
+ end
70
+ end
71
+
72
+ attr_accessor :documentation
73
+ def documentation
74
+ load_documentation if @documentation.nil?
75
+ @documentation
76
+ end
77
+ end
@@ -0,0 +1,77 @@
1
+ class Step
2
+ def info_file
3
+ @info_file ||= @path + ".info"
4
+ end
5
+
6
+ def load_info
7
+ @info = Persist.load(info_file, :marshal) || {}
8
+ @info_load_time = Time.now
9
+ end
10
+
11
+ def save_info
12
+ Persist.save(@info, info_file, :marshal)
13
+ @info_load_time = Time.now
14
+ end
15
+
16
+ def info
17
+ outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
18
+
19
+ if @info.nil? || outdated
20
+ load_info
21
+ end
22
+
23
+ @info
24
+ end
25
+
26
+ def merge_info(new_info)
27
+ info = self.info
28
+ new_info.each do |key,value|
29
+ report_status new_info[:status], new_info[:message] if key == :status
30
+ if info.include?(key)
31
+ case info[key]
32
+ when Array
33
+ info[key].concat Array === value ? value : [value]
34
+ when Hash
35
+ info[key].merge! value
36
+ else
37
+ info[key] = value
38
+ end
39
+ else
40
+ info[key] = value
41
+ end
42
+ end
43
+ save_info
44
+ end
45
+
46
+ def set_info(key, value)
47
+ merge_info(key => value)
48
+ end
49
+
50
+ def init_info
51
+ @info = {
52
+ :status => :waiting
53
+ }
54
+ end
55
+
56
+ def report_status(status, message = nil)
57
+ if message.nil?
58
+ Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
59
+ else
60
+ Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
61
+ end
62
+ end
63
+
64
+ def log(status, message = nil)
65
+ report_status status, message
66
+ if message
67
+ merge_info :status => status, :messages => [message]
68
+ else
69
+ merge_info :status => status
70
+ end
71
+ end
72
+
73
+ def status
74
+ info[:status]
75
+ end
76
+
77
+ end
@@ -0,0 +1,96 @@
1
+ require_relative '../path'
2
+ require_relative '../persist'
3
+ require_relative 'step/info'
4
+
5
+ class Step
6
+
7
+ attr_accessor :path, :inputs, :dependencies, :task
8
+ def initialize(path, inputs = nil, dependencies = [], &task)
9
+ @path = path
10
+ @inputs = inputs
11
+ @dependencies = dependencies
12
+ @task = task
13
+ end
14
+
15
+ attr_accessor :type
16
+ def type
17
+ @type ||= @task.respond_to?(:type) ? @task.type : nil
18
+ end
19
+
20
+ def name
21
+ @name ||= File.basename(@path)
22
+ end
23
+
24
+ def task_name
25
+ @task_name ||= @task.name if @task.respond_to?(:name)
26
+ end
27
+
28
+ def exec
29
+ self.instance_exec(*inputs, &task)
30
+ end
31
+
32
+ attr_reader :result
33
+ def run
34
+ @result = Persist.persist(name, type, :path => path) do
35
+ begin
36
+ merge_info :status => :start, :start => Time.now,
37
+ :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
38
+ :inputs => inputs,
39
+ :dependencies => dependencies.collect{|d| d.path }
40
+
41
+ dependencies.each{|dep| dep.run }
42
+ @result = exec
43
+ ensure
44
+ if streaming?
45
+ ConcurrentStream.setup(@result) do
46
+ merge_info :status => :done, :end => Time.now
47
+ end
48
+ log :streaming
49
+ else
50
+ merge_info :status => :done, :end => Time.now
51
+ end
52
+ end
53
+ end
54
+ end
55
+
56
+ def done?
57
+ Open.exist?(path)
58
+ end
59
+
60
+ def streaming?
61
+ IO === @result || StringIO === @result
62
+ end
63
+
64
+ def join
65
+ if streaming?
66
+ Open.consume_stream(@result, false)
67
+ @result = nil
68
+ end
69
+ end
70
+
71
+ def produce
72
+ run
73
+ join
74
+ end
75
+
76
+ def load
77
+ return @result unless @result.nil? || streaming?
78
+ join
79
+ done? ? Persist.load(path, type) : exec
80
+ end
81
+
82
+ def clean
83
+ FileUtils.rm path.find if path.exist?
84
+ end
85
+
86
+ def step(task_name)
87
+ dependencies.each do |dep|
88
+ return dep if dep.task_name == task_name
89
+ end
90
+ nil
91
+ end
92
+
93
+ def digest_str
94
+ path
95
+ end
96
+ end
@@ -0,0 +1,112 @@
1
+ module Task
2
+
3
+ def format_input(value, type, options = {})
4
+ return value if IO === value || StringIO === value
5
+
6
+ value = value.load if Step === value
7
+ if String === value && ! [:path, :file].include?(type)
8
+ if Open.exists?(value)
9
+ Persist.load(value, type)
10
+ else
11
+ Persist.deserialize(value, type)
12
+ end
13
+ else
14
+ if m = type.to_s.match(/(.*)_array/)
15
+ if Array === value
16
+ value.collect{|v| format_input(v, m[1].to_sym, options) }
17
+ end
18
+ else
19
+ value
20
+ end
21
+ end
22
+ end
23
+
24
+ def assign_inputs(provided_inputs = {})
25
+ if self.inputs.nil?
26
+ case provided_inputs
27
+ when Array
28
+ return [provided_inputs, provided_inputs]
29
+ else
30
+ return [[], []]
31
+ end
32
+ end
33
+
34
+ input_array = []
35
+ non_default_inputs = []
36
+ self.inputs.each_with_index do |p,i|
37
+ name, type, desc, value, options = p
38
+ provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
39
+ provided = format_input(provided, type, options || {})
40
+ if ! provided.nil? && provided != value
41
+ non_default_inputs << name
42
+ input_array << provided
43
+ else
44
+ input_array << value
45
+ end
46
+ end
47
+
48
+ [input_array, non_default_inputs]
49
+ end
50
+
51
+ def digest_inputs(provided_inputs = {})
52
+ input_array, non_default_inputs = assign_inputs(provided_inputs)
53
+ if Array === provided_inputs
54
+ Misc.digest(input_array)
55
+ else
56
+ Misc.digest(input_array)
57
+ end
58
+ end
59
+
60
+ def process_inputs(provided_inputs = {})
61
+ input_array, non_default_inputs = assign_inputs(provided_inputs)
62
+ digest = Misc.digest(input_array)
63
+ [input_array, non_default_inputs, digest]
64
+ end
65
+
66
+ def save_file_input(orig_file, directory)
67
+ basename = File.basename(orig_file)
68
+ digest = Misc.digest(orig_file)
69
+ if basename.include? '.'
70
+ basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
71
+ else
72
+ basename += "-#{digest}"
73
+ end
74
+ new_file = File.join(directory, 'saved_input_files', basename)
75
+ relative_file = File.join('.', 'saved_input_files', basename)
76
+ Open.link orig_file, new_file
77
+ relative_file
78
+ end
79
+
80
+ def save_inputs(directory, provided_inputs = {})
81
+ input_array, non_default_inputs = assign_inputs(provided_inputs)
82
+ self.inputs.each_with_index do |p,i|
83
+ name, type, desc, value, options = p
84
+ next unless non_default_inputs.include?(name)
85
+ input_file = File.join(directory, name.to_s)
86
+
87
+ if type == :file
88
+ relative_file = save_file_input(input_array[i], directory)
89
+ Persist.save(relative_file, input_file, type)
90
+ elsif type == :file_array
91
+ new_files = input_array[i].collect do |orig_file|
92
+ save_file_input(orig_file, directory)
93
+ end
94
+ Persist.save(new_files, input_file, type)
95
+ else
96
+ Persist.save(input_array[i], input_file, type)
97
+ end
98
+ end
99
+ end
100
+
101
+ def load_inputs(directory)
102
+ self.inputs.collect do |p|
103
+ name, type, desc, value, options = p
104
+ filename = File.join(directory, name.to_s)
105
+ if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
106
+ Persist.load(filename, type)
107
+ else
108
+ value
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,141 @@
1
+ require_relative '../meta_extension'
2
+ require_relative 'step'
3
+ require_relative 'task/inputs'
4
+
5
+ module Task
6
+ extend MetaExtension
7
+ extension_attr :name, :type, :inputs, :deps, :directory, :description
8
+
9
+ DEFAULT_NAME = "Default"
10
+
11
+ class << self
12
+ attr_accessor :default_directory
13
+
14
+ def default_directory
15
+ @default_directory ||= Path.setup('var/jobs/Task')
16
+ end
17
+ end
18
+
19
+ def directory
20
+ @directory ||= Task.default_directory
21
+ end
22
+
23
+ def exec_on(binding = self, *inputs)
24
+ binding.instance_exec(*inputs, &self)
25
+ end
26
+
27
+ def dependencies(id, provided_inputs, non_default_inputs = [])
28
+ return [] if deps.nil?
29
+ dependencies = []
30
+
31
+ provided_inputs ||= {}
32
+
33
+ load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
34
+ task = hash_options[:task] if hash_options.include?(:task)
35
+ workflow = hash_options[:workflow] if hash_options.include?(:workflow)
36
+ id = hash_options[:id] if hash_options.include? :id
37
+
38
+ hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
39
+ inputs = IndiferentHash.add_defaults hash_inputs, inputs
40
+
41
+ resolved_inputs = {}
42
+ inputs.each do |k,v|
43
+ if Symbol === v
44
+ input_dep = dependencies.select{|d| d.task_name == v}.first
45
+ resolved_inputs[k] = input_dep || inputs[v] || k
46
+ else
47
+ resolved_inputs[k] = v
48
+ end
49
+ end
50
+ workflow.job(task, id, resolved_inputs)
51
+ end
52
+
53
+ deps.each do |workflow,task,options,block=nil|
54
+ if provided_inputs.include?(overriden = [workflow.name, task] * "#")
55
+ dep = provided_inputs[overriden]
56
+ dep = Step.new dep unless Step === dep
57
+ dep.type = workflow.tasks[task].type
58
+ dependencies << dep
59
+ non_default_inputs << overriden
60
+ next
61
+ end
62
+
63
+ options ||= {}
64
+ if block
65
+ inputs = IndiferentHash.add_defaults options.dup, provided_inputs
66
+
67
+ res = block.call id, inputs, dependencies
68
+
69
+ case res
70
+ when Step
71
+ dep = res
72
+ dependencies << dep
73
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
74
+ non_default_inputs.concat(dep_non_default_inputs - options.keys)
75
+ when Hash
76
+ new_options = res
77
+ dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
78
+ dependencies << dep
79
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
80
+ dep_non_default_inputs -= options.keys
81
+ if new_options.include?(:inputs)
82
+ dep_non_default_inputs -= new_options[:inputs].keys
83
+ else
84
+ dep_non_default_inputs -= new_options.keys
85
+ end
86
+ non_default_inputs.concat(dep_non_default_inputs)
87
+ when Array
88
+ res.each do |_res|
89
+ if Hash === _res
90
+ new_options = _res
91
+ dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
92
+ dependencies << dep
93
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
94
+ dep_non_default_inputs -= options.keys
95
+ if new_options.include?(:inputs)
96
+ dep_non_default_inputs -= new_options[:inputs].keys
97
+ else
98
+ dep_non_default_inputs -= new_options.keys
99
+ end
100
+ non_default_inputs.concat(dep_non_default_inputs)
101
+ else
102
+ dep = _res
103
+ dependencies << dep
104
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
105
+ non_default_inputs.concat(dep_non_default_inputs - options.keys)
106
+ end
107
+ end
108
+ end
109
+ else
110
+ inputs = IndiferentHash.add_defaults options.dup, provided_inputs
111
+ dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
112
+ dependencies << dep
113
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
114
+ non_default_inputs.concat(dep_non_default_inputs - options.keys)
115
+ end
116
+ end
117
+
118
+ dependencies
119
+ end
120
+
121
+ def job(id = DEFAULT_NAME, provided_inputs = nil )
122
+ provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
123
+ provided_inputs = {} if provided_inputs.nil?
124
+
125
+ inputs, non_default_inputs, input_hash = process_inputs provided_inputs
126
+
127
+ dependencies = dependencies(id, provided_inputs, non_default_inputs)
128
+
129
+ non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
130
+
131
+ if non_default_inputs.any?
132
+ hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
133
+ Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
134
+ id = [id, hash] * "_"
135
+ end
136
+
137
+ path = directory[id]
138
+
139
+ Step.new path, inputs, dependencies, &self
140
+ end
141
+ end