rbbt-util 5.32.12 → 5.32.17

data/lib/rbbt/workflow.rb

@@ -1,4 +1,5 @@
 require 'rbbt/workflow/definition'
+require 'rbbt/workflow/dependencies'
 require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
 require 'rbbt/workflow/accessor'
@@ -228,6 +229,47 @@ module Workflow
                   end
   end
 
+  TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
+  DEBUG_JOB_HASH = ENV["RBBT_DEBUG_JOB_HASH"] == 'true'
+  def step_path(taskname, jobname, inputs, dependencies, extension = nil)
+    raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
+    if inputs.length > 0 or dependencies.any?
+      tagged_jobname = case TAG
+                       when :hash
+                         clean_inputs = Annotated.purge(inputs)
+                         clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
+                         key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
+                         key_str = Misc.obj2str(key_obj)
+                         hash_str = Misc.digest(key_str)
+                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}" if DEBUG_JOB_HASH
+                         jobname + '_' << hash_str
+                       when :inputs
+                         all_inputs = {}
+                         inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
+                           all_inputs[f] = i
+                         end
+                         dependencies.each do |dep|
+                           ri = dep.recursive_inputs
+                           ri.zip(ri.fields).each do |i,f|
+                             all_inputs[f] = i
+                           end
+                         end
+
+                         all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
+                       else
+                         jobname
+                       end
+    else
+      tagged_jobname = jobname
+    end
+
+    if extension and not extension.empty?
+      tagged_jobname = tagged_jobname + ('.' << extension.to_s)
+    end
+
+    workdir[taskname][tagged_jobname].find
+  end
   def import_task(workflow, orig, new)
     orig_task = workflow.tasks[orig]
     new_task = orig_task.dup
@@ -411,12 +453,22 @@ module Workflow
 
     overriden = has_overriden_inputs || overriden_deps.any?
 
+    extension = task.extension
+
+    if extension == :dep_task
+      extension = nil
+      if dependencies.any?
+        dep_basename = File.basename(dependencies.last.path)
+        extension = dep_basename.split(".").last if dep_basename.include?('.')
+      end
+    end
+
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
-      step_path = step_path taskname, jobname, [], [], task.extension
+      step_path = step_path taskname, jobname, [], [], extension
       input_values = task.take_input_values(inputs)
     else
       input_values = task.take_input_values(inputs)
-      step_path = step_path taskname, jobname, input_values, dependencies, task.extension
+      step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
     job = get_job_step step_path, task, input_values, dependencies

data/lib/rbbt/workflow/accessor.rb

@@ -7,7 +7,7 @@ module ComputeDependency
     dep.extend ComputeDependency
     dep.compute = value
   end
-  
+
   def canfail?
     compute == :canfail || (Array === compute && compute.include?(:canfail))
   end
@@ -72,83 +72,6 @@ module Workflow
                        end
   end
 
-  def rec_dependencies(taskname)
-    @rec_dependencies ||= {}
-    @rec_dependencies[taskname] ||= begin
-                                      if task_dependencies.include? taskname
-
-                                        deps = task_dependencies[taskname]
-
-                                        #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
-
-                                        all_deps = []
-                                        deps.each do |dep| 
-                                          if DependencyBlock === dep
-                                            all_deps << dep.dependency if dep.dependency
-                                          else
-                                            all_deps << dep unless Proc === dep
-                                          end
-
-                                          begin
-                                            case dep
-                                            when Array
-                                              wf, t, o = dep
-
-                                              wf.rec_dependencies(t.to_sym).each do |d|
-                                                if Array === d
-                                                  new = d.dup
-                                                else
-                                                  new = [dep.first, d]
-                                                end
-
-                                                if Hash === o and not o.empty? 
-                                                  if Hash === new.last
-                                                    hash = new.last.dup
-                                                    o.each{|k,v| hash[k] ||= v}
-                                                    new[new.length-1] = hash
-                                                  else
-                                                    new.push o.dup
-                                                  end
-                                                end
-
-                                                all_deps << new
-                                              end if wf && t
-
-                                            when String, Symbol
-                                              rec_deps = rec_dependencies(dep.to_sym)
-                                              all_deps.concat rec_deps
-                                            when DependencyBlock
-                                              dep = dep.dependency
-                                              raise TryAgain
-                                            end
-                                          rescue TryAgain
-                                            retry
-                                          end
-                                        end
-                                        all_deps.uniq
-                                      else
-                                        []
-                                      end
-                                    end
-  end
-
-  def task_from_dep(dep)
-    task = case dep
-           when Array
-             dep.first.tasks[dep[1]] 
-           when String
-             tasks[dep.to_sym]
-           when Symbol
-             tasks[dep.to_sym]
-           end
-    raise "Unknown dependency: #{Misc.fingerprint dep}" if task.nil?
-    task
-  end
-
-  #def rec_inputs(taskname)
-  #  [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat(task_from_dep(tn).inputs) }.uniq
-  #end
-
   def rec_inputs(taskname)
     task = task_from_dep(taskname)
     deps = rec_dependencies(taskname)
@@ -294,174 +217,24 @@ module Workflow
     _inputs
   end
 
-  def override_dependencies(inputs)
-    override_dependencies = IndiferentHash.setup({})
-    return override_dependencies if inputs.nil?
-    inputs.each do |key,value|
-      if String === key && m = key.match(/(.*)#(.*)/)
-        workflow, task = m.values_at 1, 2
-        workflow = self.to_s if workflow.empty?
-        override_dependencies[workflow] ||= IndiferentHash.setup({})
-        override_dependencies[workflow][task] = value
-      end
-    end
-    override_dependencies
-  end
-
-  def setup_override_dependency(dep, workflow, task_name)
-    dep = Step === dep ? dep : Workflow.load_step(dep)
-    dep.workflow = workflow
-    dep.info[:name] = dep.name
-    dep.original_task_name ||= dep.task_name if dep.workflow
-    begin
-      workflow = Kernel.const_get workflow if String === workflow
-      dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
-    rescue
-      Log.exception $!
-    end
-    dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym
-    dep
-  end
-
-  def real_dependencies(task, orig_jobname, inputs, dependencies)
-    real_dependencies = []
-    path_deps = {}
-
-    override_dependencies = override_dependencies(inputs)
-
-    dependencies.each do |dependency|
-      _inputs = IndiferentHash.setup(inputs.dup)
-      jobname = orig_jobname
-      jobname = _inputs[:jobname] if _inputs.include? :jobname
-
-      real_dep = case dependency
-                 when Array
-                   workflow, dep_task, options = dependency
-
-                   if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
-                     setup_override_dependency(value, workflow, dep_task)
-                   else
-
-                     compute = options[:compute] if options
-
-                     all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
-
-                     _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
-                     jobname = _inputs.delete :jobname if _inputs.include? :jobname
-
-                     job = workflow._job(dep_task, jobname, _inputs)
-                     ComputeDependency.setup(job, compute) if compute
-                     job
-                   end
-                 when Step
-                   dependency
-                 when Symbol
-                   if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
-                     setup_override_dependency(value, self, dependency)
-                   else
-                     _job(dependency, jobname, _inputs)
-                   end
-                 when Proc
-                   if DependencyBlock === dependency
-                     orig_dep = dependency.dependency 
-                     wf, task_name, options = orig_dep
-
-                     options = {} if options.nil?
-                     compute = options[:compute]
-
-                     options = IndiferentHash.setup(options.dup)
-                     dep = dependency.call jobname, _inputs.merge(options), real_dependencies
-
-                     dep = [dep] unless Array === dep
-
-                     new_=[]
-                     dep.each{|d| 
-                       next if d.nil?
-                       if Hash === d
-                         d[:workflow] ||= wf 
-                         d[:task] ||= task_name
-                         _override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
-                         d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
-                               setup_override_dependency(value, d[:workflow], d[:task])
-                             else
-                               task_info = d[:workflow].task_info(d[:task])
-
-                               _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                               d[:workflow]._job(d[:task], d[:jobname], _inputs) 
-                             end
-                       end
-                       ComputeDependency.setup(d, compute) if compute
-                       new_ << d
-                     }
-                     dep = new_
-                   else
-                     _inputs = IndiferentHash.setup(_inputs.dup)
-                     dep = dependency.call jobname, _inputs, real_dependencies
-                     if Hash === dep
-                       dep[:workflow] ||= wf || self
-                       _override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
-                       if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
-                         setup_override_dependency(value, dep[:workflow], dep[:task])
-                       else
-                         task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
-                         _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
-                       end
-                     end
-                   end
-
-                   dep
-                 else
-                   raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
-                 end
-
-      real_dependencies << real_dep
-    end
-    real_dependencies.flatten.compact
+  def task_from_dep(dep)
+    task = case dep
+           when Array
+             dep.first.tasks[dep[1]] 
+           when String
+             tasks[dep.to_sym]
+           when Symbol
+             tasks[dep.to_sym]
+           end
+    raise "Unknown dependency: #{Misc.fingerprint dep}" if task.nil?
+    task
   end
 
-  TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
-  DEBUG_JOB_HASH = ENV["RBBT_DEBUG_JOB_HASH"] == 'true'
-  def step_path(taskname, jobname, inputs, dependencies, extension = nil)
-    raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
-    if inputs.length > 0 or dependencies.any?
-      tagged_jobname = case TAG
-                       when :hash
-                         clean_inputs = Annotated.purge(inputs)
-                         clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << d.short_path : d }
-                         key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
-                         key_str = Misc.obj2str(key_obj)
-                         hash_str = Misc.digest(key_str)
-                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}" if DEBUG_JOB_HASH
-                         jobname + '_' << hash_str
-                       when :inputs
-                         all_inputs = {}
-                         inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
-                           all_inputs[f] = i
-                         end
-                         dependencies.each do |dep|
-                           ri = dep.recursive_inputs
-                           ri.zip(ri.fields).each do |i,f|
-                             all_inputs[f] = i
-                           end
-                         end
-
-                         all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
-                       else
-                         jobname
-                       end
-    else
-      tagged_jobname = jobname
-    end
+  #def rec_inputs(taskname)
+  #  [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat(task_from_dep(tn).inputs) }.uniq
+  #end
 
-    if extension and not extension.empty?
-      tagged_jobname = tagged_jobname + ('.' << extension.to_s)
-    end
 
-    workdir[taskname][tagged_jobname].find
-  end
 
   def id_for(path)
     if workdir.respond_to? :find

data/lib/rbbt/workflow/definition.rb

@@ -73,10 +73,11 @@ module Workflow
   REMOVE_DEP_TASKS = ENV["RBBT_REMOVE_DEP_TASKS"] == "true"
   def dep_task(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
-    extension workflow.tasks[oname].extension if workflow.tasks.include?(oname) unless @extension
+    extension :dep_task unless @extension
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      Step.wait_for_jobs dependencies
       dep = dependencies.last
       dep.join
       raise dep.get_exception if dep.error?

data/lib/rbbt/workflow/dependencies.rb

@@ -0,0 +1,195 @@
+module Workflow
+  def rec_dependencies(taskname, seen = [])
+    @rec_dependencies ||= {}
+    @rec_dependencies[taskname] ||= [] unless task_dependencies.include?(taskname)
+    @rec_dependencies[taskname] ||= begin
+
+                                      deps = task_dependencies[taskname]
+
+                                      all_deps = []
+                                      deps.each do |dep| 
+                                        next if seen.include?(dep)
+                                        if DependencyBlock === dep
+                                          all_deps << dep.dependency if dep.dependency
+                                        else
+                                          all_deps << dep unless Proc === dep
+                                        end
+
+                                        begin
+                                          case dep
+                                          when Array
+                                            wf, t, o = dep
+
+                                            wf.rec_dependencies(t.to_sym, seen + [dep]).each do |d|
+                                              if Array === d
+                                                new = d.dup
+                                              else
+                                                new = [dep.first, d]
+                                              end
+
+                                              if Hash === o and not o.empty? 
+                                                if Hash === new.last
+                                                  hash = new.last.dup
+                                                  o.each{|k,v| hash[k] ||= v}
+                                                  new[new.length-1] = hash
+                                                else
+                                                  new.push o.dup
+                                                end
+                                              end
+
+                                              all_deps << new
+                                            end if wf && t
+
+                                          when String, Symbol
+                                            rec_deps = rec_dependencies(dep.to_sym, seen + [dep])
+                                            all_deps.concat rec_deps
+                                          when DependencyBlock
+                                            dep = dep.dependency
+                                            raise TryAgain
+                                          end
+                                        rescue TryAgain
+                                          retry
+                                        end
+                                      end
+                                      all_deps.uniq
+                                    end
+  end
+
+  def override_dependencies(inputs)
+    override_dependencies = IndiferentHash.setup({})
+    return override_dependencies if inputs.nil?
+    inputs.each do |key,value|
+      if String === key && m = key.match(/(.*)#(.*)/)
+        workflow, task = m.values_at 1, 2
+        workflow = self.to_s if workflow.empty?
+        override_dependencies[workflow] ||= IndiferentHash.setup({})
+        override_dependencies[workflow][task] = value
+      end
+    end
+    override_dependencies
+  end
+
+  def setup_override_dependency(dep, workflow, task_name)
+    return [] if dep == :skip || dep == 'skip'
+    dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
+    dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name if dep.workflow
+
+
+    begin
+      workflow = Kernel.const_get workflow if String === workflow
+      dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
+    rescue
+      Log.exception $!
+    end
+    dep.task_name = task_name
+    dep.overriden = dep.original_task_name.to_sym
+
+    dep.extend step_module
+
+    dep
+  end
+
+  def real_dependencies(task, orig_jobname, inputs, dependencies)
+    real_dependencies = []
+    path_deps = {}
+
+    override_dependencies = override_dependencies(inputs)
+
+    dependencies.each do |dependency|
+      _inputs = IndiferentHash.setup(inputs.dup)
+      jobname = orig_jobname
+      jobname = _inputs[:jobname] if _inputs.include? :jobname
+
+      real_dep = case dependency
+                 when Array
+                   workflow, dep_task, options = dependency
+
+                   if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
+                     setup_override_dependency(value, workflow, dep_task)
+                   else
+
+                     compute = options[:compute] if options
+
+                     all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
+
+                     _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
+                     jobname = _inputs.delete :jobname if _inputs.include? :jobname
+
+                     job = workflow._job(dep_task, jobname, _inputs)
+                     ComputeDependency.setup(job, compute) if compute
+                     job
+                   end
+                 when Step
+                   dependency
+                 when Symbol
+                   if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
+                     setup_override_dependency(value, self, dependency)
+                   else
+                     _job(dependency, jobname, _inputs)
+                   end
+                 when Proc
+                   if DependencyBlock === dependency
+                     orig_dep = dependency.dependency 
+                     wf, task_name, options = orig_dep
+
+                     if override_dependencies[wf.to_s] && value = override_dependencies[wf.to_s][task_name]
+                       dep = setup_override_dependency(value, wf, task_name)
+                     else
+
+                       options = {} if options.nil?
+                       compute = options[:compute]
+
+                       options = IndiferentHash.setup(options.dup)
+                       dep = dependency.call jobname, _inputs.merge(options), real_dependencies
+
+                       dep = [dep] unless Array === dep
+
+                       new_=[]
+                       dep.each{|d| 
+                         next if d.nil?
+                         if Hash === d
+                           d[:workflow] ||= wf 
+                           d[:task] ||= task_name
+                           _override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
+                           d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
+                                 setup_override_dependency(value, d[:workflow], d[:task])
+                               else
+                                 task_info = d[:workflow].task_info(d[:task])
+
+                                 _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
+                                 d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                               end
+                         end
+                         ComputeDependency.setup(d, compute) if compute
+                         new_ << d
+                       }
+                       dep = new_
+                     end
+                   else
+                     _inputs = IndiferentHash.setup(_inputs.dup)
+                     dep = dependency.call jobname, _inputs, real_dependencies
+                     if Hash === dep
+                       dep[:workflow] ||= wf || self
+                       _override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
+                       if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
+                         setup_override_dependency(value, dep[:workflow], dep[:task])
+                       else
+                         task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
+                         _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
+                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
+                       end
+                     end
+                   end
+
+                   dep
+                 else
+                   raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
+                 end
+
+      real_dependencies << real_dep
+    end
+    real_dependencies.flatten.compact
+  end
+end