rbbt-util 5.32.16 → 5.32.21

data/lib/rbbt/workflow/definition.rb

@@ -77,14 +77,15 @@ module Workflow
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join
       raise dep.get_exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
         self.copy_files_dir
@@ -92,15 +93,17 @@ module Workflow
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
         Open.ln_h dep.path, self.tmp_path
+
         case remove.to_s
         when 'true'
           dep.clean
         when 'recursive'
-          dep.rec_dependencies.each do |d|
+          (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+            next if d.overriden
             d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
           end
           dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
-        end
+        end unless dep.overriden
       else
         if Open.exists?(dep.files_dir)
           Open.rm_rf self.files_dir 

data/lib/rbbt/workflow/dependencies.rb

@@ -0,0 +1,196 @@
+module Workflow
+  def rec_dependencies(taskname, seen = [])
+    @rec_dependencies ||= {}
+    @rec_dependencies[taskname] ||= [] unless task_dependencies.include?(taskname)
+    @rec_dependencies[taskname] ||= begin
+
+                                      deps = task_dependencies[taskname]
+
+                                      all_deps = []
+                                      deps.each do |dep| 
+                                        next if seen.include?(dep)
+                                        if DependencyBlock === dep
+                                          all_deps << dep.dependency if dep.dependency
+                                        else
+                                          all_deps << dep unless Proc === dep
+                                        end
+
+                                        begin
+                                          case dep
+                                          when Array
+                                            wf, t, o = dep
+
+                                            wf.rec_dependencies(t.to_sym, seen + [dep]).each do |d|
+                                              if Array === d
+                                                new = d.dup
+                                              else
+                                                new = [dep.first, d]
+                                              end
+
+                                              if Hash === o and not o.empty? 
+                                                if Hash === new.last
+                                                  hash = new.last.dup
+                                                  o.each{|k,v| hash[k] ||= v}
+                                                  new[new.length-1] = hash
+                                                else
+                                                  new.push o.dup
+                                                end
+                                              end
+
+                                              all_deps << new
+                                            end if wf && t
+
+                                          when String, Symbol
+                                            rec_deps = rec_dependencies(dep.to_sym, seen + [dep])
+                                            all_deps.concat rec_deps
+                                          when DependencyBlock
+                                            dep = dep.dependency
+                                            raise TryAgain
+                                          end
+                                        rescue TryAgain
+                                          retry
+                                        end
+                                      end
+                                      all_deps.uniq
+                                    end
+  end
+
+  def override_dependencies(inputs)
+    override_dependencies = IndiferentHash.setup({})
+    return override_dependencies if inputs.nil?
+    inputs.each do |key,value|
+      if String === key && m = key.match(/(.*)#(.*)/)
+        workflow, task = m.values_at 1, 2
+        workflow = self.to_s if workflow.empty?
+        override_dependencies[workflow] ||= IndiferentHash.setup({})
+        override_dependencies[workflow][task] = value
+      end
+    end
+    override_dependencies
+  end
+
+  def setup_override_dependency(dep, workflow, task_name)
+    return [] if dep == :skip || dep == 'skip'
+    dep = Step === dep ? dep.dup : Workflow.load_step(dep)
+    dep.workflow = workflow
+    dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name if dep.workflow
+
+
+    begin
+      workflow = Kernel.const_get workflow if String === workflow
+      dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
+    rescue
+      Log.exception $!
+    end
+
+    dep.task_name = task_name
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
+
+    dep.extend step_module
+
+    dep
+  end
+
+  def real_dependencies(task, orig_jobname, inputs, dependencies)
+    real_dependencies = []
+    path_deps = {}
+
+    override_dependencies = override_dependencies(inputs)
+
+    dependencies.each do |dependency|
+      _inputs = IndiferentHash.setup(inputs.dup)
+      jobname = orig_jobname
+      jobname = _inputs[:jobname] if _inputs.include? :jobname
+
+      real_dep = case dependency
+                 when Array
+                   workflow, dep_task, options = dependency
+
+                   if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
+                     setup_override_dependency(value, workflow, dep_task)
+                   else
+
+                     compute = options[:compute] if options
+
+                     all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
+
+                     _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
+                     jobname = _inputs.delete :jobname if _inputs.include? :jobname
+
+                     job = workflow._job(dep_task, jobname, _inputs)
+                     ComputeDependency.setup(job, compute) if compute
+                     job
+                   end
+                 when Step
+                   dependency
+                 when Symbol
+                   if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
+                     setup_override_dependency(value, self, dependency)
+                   else
+                     _job(dependency, jobname, _inputs)
+                   end
+                 when Proc
+                   if DependencyBlock === dependency
+                     orig_dep = dependency.dependency 
+                     wf, task_name, options = orig_dep
+
+                     if override_dependencies[wf.to_s] && value = override_dependencies[wf.to_s][task_name]
+                       dep = setup_override_dependency(value, wf, task_name)
+                     else
+
+                       options = {} if options.nil?
+                       compute = options[:compute]
+
+                       options = IndiferentHash.setup(options.dup)
+                       dep = dependency.call jobname, _inputs.merge(options), real_dependencies
+
+                       dep = [dep] unless Array === dep
+
+                       new_=[]
+                       dep.each{|d| 
+                         next if d.nil?
+                         if Hash === d
+                           d[:workflow] ||= wf 
+                           d[:task] ||= task_name
+                           _override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
+                           d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
+                                 setup_override_dependency(value, d[:workflow], d[:task])
+                               else
+                                 task_info = d[:workflow].task_info(d[:task])
+
+                                 _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
+                                 d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                               end
+                         end
+                         ComputeDependency.setup(d, compute) if compute
+                         new_ << d
+                       }
+                       dep = new_
+                     end
+                   else
+                     _inputs = IndiferentHash.setup(_inputs.dup)
+                     dep = dependency.call jobname, _inputs, real_dependencies
+                     if Hash === dep
+                       dep[:workflow] ||= wf || self
+                       _override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
+                       if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
+                         setup_override_dependency(value, dep[:workflow], dep[:task])
+                       else
+                         task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
+                         _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
+                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
+                       end
+                     end
+                   end
+
+                   dep
+                 else
+                   raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
+                 end
+
+      real_dependencies << real_dep
+    end
+    real_dependencies.flatten.compact
+  end
+end

data/lib/rbbt/workflow/step.rb

@@ -4,6 +4,8 @@ require 'rbbt/util/log'
 require 'rbbt/util/semaphore'
 require 'rbbt/workflow/step/accessor'
 require 'rbbt/workflow/step/prepare'
+require 'rbbt/workflow/step/status'
+require 'rbbt/workflow/step/info'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
@@ -35,41 +37,6 @@ class Step
   end
 
 
-  def overriden?
-    return true if @overriden
-    return true if dependencies.select{|dep| dep.overriden? }.any?
-    info[:archived_info].each do |f,i|
-      return true if i[:overriden] || i["overriden"]
-    end if info[:archived_info]
-    return false
-  end
-
-  def overriden
-    @overriden
-    #if @overriden.nil? 
-    #  return false if dependencies.nil?
-    #  dependencies.select{|dep| dep.overriden? }.any?
-    #else
-    #  @overriden
-    #end
-  end
-
-  def overriden_deps
-    ord = []
-    deps = dependencies.dup
-    while dep = deps.shift
-      case dep.overriden
-      when FalseClass
-        next
-      when Symbol
-        ord << dep
-      else
-        deps += dep.dependencies
-      end
-    end
-    ord
-  end
-
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
     path = path.call if Proc === path
@@ -99,35 +66,6 @@ class Step
   end
 
 
-  def load_inputs_from_info
-    if info[:inputs]
-      info_inputs = info[:inputs]
-      if task && task.respond_to?(:inputs) && task.inputs
-        IndiferentHash.setup info_inputs
-        @inputs = NamedArray.setup info_inputs.values_at(*task.inputs.collect{|name| name.to_s}), task.inputs
-      else
-        @inputs = NamedArray.setup info_inputs.values, info_inputs.keys
-      end
-    else
-      nil
-    end
-  end
-
-  def load_dependencies_from_info
-    relocated = nil
-    @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
-      if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
-        Workflow._load_step dep_path
-      else
-        next if FalseClass === relocated
-        new_path = Workflow.relocate(path, dep_path)
-        relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
-        Workflow._load_step new_path
-      end
-    end.compact
-    @relocated = relocated
-  end
-
 
   def inputs
     return @inputs if NamedArray === @inputs
@@ -151,54 +89,6 @@ class Step
     end
   end
 
-  def archive_deps
-    self.set_info :archived_info, archived_info
-    self.set_info :archived_dependencies, info[:dependencies]
-  end
-
-  def archived_info
-    return info[:archived_info] if info[:archived_info]
-
-    archived_info = {}
-    dependencies.each do |dep|
-      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
-        archived_info[dep.path] = dep.overriden
-      else
-        archived_info[dep.path] = dep.info
-      end
-      archived_info.merge!(dep.archived_info)
-    end if dependencies
-
-    archived_info
-  end
-
-  def archived_inputs
-    return {} unless info[:archived_dependencies]
-    archived_info = self.archived_info
-
-    all_inputs = IndiferentHash.setup({})
-    deps = info[:archived_dependencies].collect{|p| p.last}
-    seen = []
-    while path = deps.pop
-      dep_info = archived_info[path]
-      if dep_info
-        dep_info[:inputs].each do |k,v|
-          all_inputs[k] = v unless all_inputs.include?(k)
-        end if dep_info[:inputs]
-        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
-      end
-      seen << path
-    end
-
-    all_inputs
-  end
-
-  def dependencies=(dependencies)
-    @dependencies = dependencies
-    set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
-  end
-
   def recursive_inputs
     if NamedArray === inputs
       i = {}
@@ -213,7 +103,8 @@ class Step
 
       dep.inputs.zip(dep.inputs.fields).each do |v,f|
         if i.include?(f) && i[f] != v
-          Log.debug "Conflict in #{ f }: #{[Misc.fingerprint(i[f]), Misc.fingerprint(v)] * " <-> "}"
+          next
+          #Log.debug "Variable '#{f}' reused with different values: #{[Misc.fingerprint(i[f]), Misc.fingerprint(v)] * " <-> "}"
         else 
           i[f] = v
         end
@@ -453,34 +344,6 @@ class Step
     res
   end
 
-  def self.clean(path)
-    info_file = Step.info_file path
-    pid_file = Step.pid_file path
-    md5_file = Step.md5_file path
-    files_dir = Step.files_dir path
-    tmp_path = Step.tmp_path path
-
-    if ! (Open.writable?(path) && Open.writable?(info_file))
-      Log.warn "Could not clean #{path}: not writable"
-      return 
-    end
-
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
-
-      @result = nil
-      @pid = nil
-
-      Misc.insist do
-        Open.rm info_file if Open.exists?(info_file)
-        Open.rm md5_file if Open.exists?(md5_file)
-        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
-        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
-        Open.rm pid_file if Open.exists?(pid_file)
-        Open.rm tmp_path if Open.exists?(tmp_path)
-      end
-    end
-  end
-
   def update
     if dirty?
       dependencies.collect{|d| d.update } if dependencies
@@ -488,49 +351,6 @@ class Step
     end
   end
 
-  def clean
-    if ! Open.exists?(info_file)
-      Log.high "Refusing to clean step with no .info file: #{path}"
-      return self
-    end
-    status = []
-    status << "dirty" if done? && dirty?
-    status << "not running" if ! done? && ! running? 
-    status.unshift " " if status.any?
-    Log.high "Cleaning step: #{path}#{status * " "}"
-    Log.stack caller if RBBT_DEBUG_CLEAN
-    abort if ! done? && running?
-    Step.clean(path)
-    self
-  end
-
-  #connected = true means that dependency searching ends when a result is done
-  #but dependencies are absent, meanining that the file could have been dropped
-  #in
-  def rec_dependencies(connected = false, seen = [])
-
-    # A step result with no info_file means that it was manually
-    # placed. In that case, do not consider its dependencies
-    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
-
-    return [] if dependencies.nil? or dependencies.empty?
-
-    new_dependencies = []
-    archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
-
-    dependencies.each{|step| 
-      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
-      next if archived_deps.include? step.path
-      next if seen.include? step.path
-      next if self.done? && connected && ! updatable?
-
-      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
-      new_dependencies.concat r
-      new_dependencies << step
-    }
-    new_dependencies.uniq
-  end
-
   def writable?
     Open.writable?(self.path) && Open.writable?(self.info_file)
   end