rbbt-util 5.32.16 → 5.32.17

data/lib/rbbt/workflow/dependencies.rb

@@ -0,0 +1,195 @@
+module Workflow
+  def rec_dependencies(taskname, seen = [])
+    @rec_dependencies ||= {}
+    @rec_dependencies[taskname] ||= [] unless task_dependencies.include?(taskname)
+    @rec_dependencies[taskname] ||= begin
+
+                                      deps = task_dependencies[taskname]
+
+                                      all_deps = []
+                                      deps.each do |dep| 
+                                        next if seen.include?(dep)
+                                        if DependencyBlock === dep
+                                          all_deps << dep.dependency if dep.dependency
+                                        else
+                                          all_deps << dep unless Proc === dep
+                                        end
+
+                                        begin
+                                          case dep
+                                          when Array
+                                            wf, t, o = dep
+
+                                            wf.rec_dependencies(t.to_sym, seen + [dep]).each do |d|
+                                              if Array === d
+                                                new = d.dup
+                                              else
+                                                new = [dep.first, d]
+                                              end
+
+                                              if Hash === o and not o.empty? 
+                                                if Hash === new.last
+                                                  hash = new.last.dup
+                                                  o.each{|k,v| hash[k] ||= v}
+                                                  new[new.length-1] = hash
+                                                else
+                                                  new.push o.dup
+                                                end
+                                              end
+
+                                              all_deps << new
+                                            end if wf && t
+
+                                          when String, Symbol
+                                            rec_deps = rec_dependencies(dep.to_sym, seen + [dep])
+                                            all_deps.concat rec_deps
+                                          when DependencyBlock
+                                            dep = dep.dependency
+                                            raise TryAgain
+                                          end
+                                        rescue TryAgain
+                                          retry
+                                        end
+                                      end
+                                      all_deps.uniq
+                                    end
+  end
+
+  def override_dependencies(inputs)
+    override_dependencies = IndiferentHash.setup({})
+    return override_dependencies if inputs.nil?
+    inputs.each do |key,value|
+      if String === key && m = key.match(/(.*)#(.*)/)
+        workflow, task = m.values_at 1, 2
+        workflow = self.to_s if workflow.empty?
+        override_dependencies[workflow] ||= IndiferentHash.setup({})
+        override_dependencies[workflow][task] = value
+      end
+    end
+    override_dependencies
+  end
+
+  def setup_override_dependency(dep, workflow, task_name)
+    return [] if dep == :skip || dep == 'skip'
+    dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
+    dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name if dep.workflow
+
+
+    begin
+      workflow = Kernel.const_get workflow if String === workflow
+      dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
+    rescue
+      Log.exception $!
+    end
+    dep.task_name = task_name
+    dep.overriden = dep.original_task_name.to_sym
+
+    dep.extend step_module
+
+    dep
+  end
+
+  def real_dependencies(task, orig_jobname, inputs, dependencies)
+    real_dependencies = []
+    path_deps = {}
+
+    override_dependencies = override_dependencies(inputs)
+
+    dependencies.each do |dependency|
+      _inputs = IndiferentHash.setup(inputs.dup)
+      jobname = orig_jobname
+      jobname = _inputs[:jobname] if _inputs.include? :jobname
+
+      real_dep = case dependency
+                 when Array
+                   workflow, dep_task, options = dependency
+
+                   if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
+                     setup_override_dependency(value, workflow, dep_task)
+                   else
+
+                     compute = options[:compute] if options
+
+                     all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
+
+                     _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
+                     jobname = _inputs.delete :jobname if _inputs.include? :jobname
+
+                     job = workflow._job(dep_task, jobname, _inputs)
+                     ComputeDependency.setup(job, compute) if compute
+                     job
+                   end
+                 when Step
+                   dependency
+                 when Symbol
+                   if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
+                     setup_override_dependency(value, self, dependency)
+                   else
+                     _job(dependency, jobname, _inputs)
+                   end
+                 when Proc
+                   if DependencyBlock === dependency
+                     orig_dep = dependency.dependency 
+                     wf, task_name, options = orig_dep
+
+                     if override_dependencies[wf.to_s] && value = override_dependencies[wf.to_s][task_name]
+                       dep = setup_override_dependency(value, wf, task_name)
+                     else
+
+                       options = {} if options.nil?
+                       compute = options[:compute]
+
+                       options = IndiferentHash.setup(options.dup)
+                       dep = dependency.call jobname, _inputs.merge(options), real_dependencies
+
+                       dep = [dep] unless Array === dep
+
+                       new_=[]
+                       dep.each{|d| 
+                         next if d.nil?
+                         if Hash === d
+                           d[:workflow] ||= wf 
+                           d[:task] ||= task_name
+                           _override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
+                           d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
+                                 setup_override_dependency(value, d[:workflow], d[:task])
+                               else
+                                 task_info = d[:workflow].task_info(d[:task])
+
+                                 _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
+                                 d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                               end
+                         end
+                         ComputeDependency.setup(d, compute) if compute
+                         new_ << d
+                       }
+                       dep = new_
+                     end
+                   else
+                     _inputs = IndiferentHash.setup(_inputs.dup)
+                     dep = dependency.call jobname, _inputs, real_dependencies
+                     if Hash === dep
+                       dep[:workflow] ||= wf || self
+                       _override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
+                       if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
+                         setup_override_dependency(value, dep[:workflow], dep[:task])
+                       else
+                         task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
+                         _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
+                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
+                       end
+                     end
+                   end
+
+                   dep
+                 else
+                   raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
+                 end
+
+      real_dependencies << real_dep
+    end
+    real_dependencies.flatten.compact
+  end
+end

data/lib/rbbt/workflow/step.rb

@@ -4,6 +4,8 @@ require 'rbbt/util/log'
 require 'rbbt/util/semaphore'
 require 'rbbt/workflow/step/accessor'
 require 'rbbt/workflow/step/prepare'
+require 'rbbt/workflow/step/status'
+require 'rbbt/workflow/step/info'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
@@ -35,41 +37,6 @@ class Step
   end
 
 
-  def overriden?
-    return true if @overriden
-    return true if dependencies.select{|dep| dep.overriden? }.any?
-    info[:archived_info].each do |f,i|
-      return true if i[:overriden] || i["overriden"]
-    end if info[:archived_info]
-    return false
-  end
-
-  def overriden
-    @overriden
-    #if @overriden.nil? 
-    #  return false if dependencies.nil?
-    #  dependencies.select{|dep| dep.overriden? }.any?
-    #else
-    #  @overriden
-    #end
-  end
-
-  def overriden_deps
-    ord = []
-    deps = dependencies.dup
-    while dep = deps.shift
-      case dep.overriden
-      when FalseClass
-        next
-      when Symbol
-        ord << dep
-      else
-        deps += dep.dependencies
-      end
-    end
-    ord
-  end
-
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
     path = path.call if Proc === path
@@ -99,35 +66,6 @@ class Step
   end
 
 
-  def load_inputs_from_info
-    if info[:inputs]
-      info_inputs = info[:inputs]
-      if task && task.respond_to?(:inputs) && task.inputs
-        IndiferentHash.setup info_inputs
-        @inputs = NamedArray.setup info_inputs.values_at(*task.inputs.collect{|name| name.to_s}), task.inputs
-      else
-        @inputs = NamedArray.setup info_inputs.values, info_inputs.keys
-      end
-    else
-      nil
-    end
-  end
-
-  def load_dependencies_from_info
-    relocated = nil
-    @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
-      if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
-        Workflow._load_step dep_path
-      else
-        next if FalseClass === relocated
-        new_path = Workflow.relocate(path, dep_path)
-        relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
-        Workflow._load_step new_path
-      end
-    end.compact
-    @relocated = relocated
-  end
-
 
   def inputs
     return @inputs if NamedArray === @inputs
@@ -151,54 +89,6 @@ class Step
     end
   end
 
-  def archive_deps
-    self.set_info :archived_info, archived_info
-    self.set_info :archived_dependencies, info[:dependencies]
-  end
-
-  def archived_info
-    return info[:archived_info] if info[:archived_info]
-
-    archived_info = {}
-    dependencies.each do |dep|
-      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
-        archived_info[dep.path] = dep.overriden
-      else
-        archived_info[dep.path] = dep.info
-      end
-      archived_info.merge!(dep.archived_info)
-    end if dependencies
-
-    archived_info
-  end
-
-  def archived_inputs
-    return {} unless info[:archived_dependencies]
-    archived_info = self.archived_info
-
-    all_inputs = IndiferentHash.setup({})
-    deps = info[:archived_dependencies].collect{|p| p.last}
-    seen = []
-    while path = deps.pop
-      dep_info = archived_info[path]
-      if dep_info
-        dep_info[:inputs].each do |k,v|
-          all_inputs[k] = v unless all_inputs.include?(k)
-        end if dep_info[:inputs]
-        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
-      end
-      seen << path
-    end
-
-    all_inputs
-  end
-
-  def dependencies=(dependencies)
-    @dependencies = dependencies
-    set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
-  end
-
   def recursive_inputs
     if NamedArray === inputs
       i = {}
@@ -213,7 +103,8 @@ class Step
 
       dep.inputs.zip(dep.inputs.fields).each do |v,f|
         if i.include?(f) && i[f] != v
-          Log.debug "Conflict in #{ f }: #{[Misc.fingerprint(i[f]), Misc.fingerprint(v)] * " <-> "}"
+          next
+          #Log.debug "Variable '#{f}' reused with different values: #{[Misc.fingerprint(i[f]), Misc.fingerprint(v)] * " <-> "}"
         else 
           i[f] = v
         end
@@ -453,34 +344,6 @@ class Step
     res
   end
 
-  def self.clean(path)
-    info_file = Step.info_file path
-    pid_file = Step.pid_file path
-    md5_file = Step.md5_file path
-    files_dir = Step.files_dir path
-    tmp_path = Step.tmp_path path
-
-    if ! (Open.writable?(path) && Open.writable?(info_file))
-      Log.warn "Could not clean #{path}: not writable"
-      return 
-    end
-
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
-
-      @result = nil
-      @pid = nil
-
-      Misc.insist do
-        Open.rm info_file if Open.exists?(info_file)
-        Open.rm md5_file if Open.exists?(md5_file)
-        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
-        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
-        Open.rm pid_file if Open.exists?(pid_file)
-        Open.rm tmp_path if Open.exists?(tmp_path)
-      end
-    end
-  end
-
   def update
     if dirty?
       dependencies.collect{|d| d.update } if dependencies
@@ -488,22 +351,6 @@ class Step
     end
   end
 
-  def clean
-    if ! Open.exists?(info_file)
-      Log.high "Refusing to clean step with no .info file: #{path}"
-      return self
-    end
-    status = []
-    status << "dirty" if done? && dirty?
-    status << "not running" if ! done? && ! running? 
-    status.unshift " " if status.any?
-    Log.high "Cleaning step: #{path}#{status * " "}"
-    Log.stack caller if RBBT_DEBUG_CLEAN
-    abort if ! done? && running?
-    Step.clean(path)
-    self
-  end
-
   #connected = true means that dependency searching ends when a result is done
   #but dependencies are absent, meanining that the file could have been dropped
   #in
@@ -516,7 +363,11 @@ class Step
     return [] if dependencies.nil? or dependencies.empty?
 
     new_dependencies = []
-    archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+    if self.overriden?
+      archived_deps = []
+    else
+      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+    end
 
     dependencies.each{|step| 
       #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
@@ -528,6 +379,7 @@ class Step
       new_dependencies.concat r
       new_dependencies << step
     }
+
     new_dependencies.uniq
   end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -26,6 +26,7 @@ class Step
       threads = jobs.collect do |j| 
         Thread.new do
           begin
+            j.soft_grace
             j.join unless j.done?
           rescue Exception
             Log.error "Exception waiting for job: #{Log.color :blue, j.path}"
@@ -206,45 +207,6 @@ class Step
     #@status_lock
   end
 
-  def info(check_lock = true)
-    return {:status => :noinfo} if info_file.nil? or not Open.exists? info_file
-    begin
-      Misc.insist do
-        begin
-          return @info_cache if @info_cache and @info_cache_time and Open.ctime(info_file) < @info_cache_time 
-        rescue Exception
-          raise $!
-        end
-
-        begin
-          @info_cache = Misc.insist(3, 1.6, info_file) do
-            Misc.insist(2, 1, info_file) do
-              Misc.insist(3, 0.2, info_file) do
-                raise TryAgain, "Info locked" if check_lock and info_lock.locked?
-                info_lock.lock if check_lock and false
-                begin
-                  Open.open(info_file, :mode => 'rb') do |file|
-                    Step.load_serialized_info(file)
-                  end
-                ensure
-                  info_lock.unlock if check_lock and false
-                end
-              end
-            end
-          end
-          @info_cache_time = Time.now
-          @info_cache
-        end
-      end
-    rescue Exception
-      Log.debug{"Error loading info file: " + info_file}
-      Log.exception $!
-      Open.rm info_file
-      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
-      raise $!
-    end
-  end
-
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
@@ -286,274 +248,6 @@ class Step
     end
   end
 
-  def status
-    begin
-      info[:status]
-    rescue Exception
-      Log.error "Exception reading status: #{$!.message}" 
-      :error
-    end
-  end
-
-  def status=(status)
-    set_info(:status, status)
-  end
-
-  def messages
-    if messages = info[:messages]
-      messages
-    else
-      set_info(:messages, []) if self.respond_to?(:set_info)
-    end
-  end
-
-  def message(message)
-    message = Log.uncolor(message)
-    set_info(:messages, (messages || []) << message)
-  end
-
-  def self.status_color(status)
-    status = status.split(">").last
-    case status
-    when "starting"
-      :yellow
-    when "error", "aborted"
-      :red
-    when "done"
-      :green
-    else
-      :cyan
-    end
-  end
-
-  def self.log_block(status, message, path, &block)
-    start = Time.now
-    status = status.to_s
-    status_color = self.status_color status
-
-    Log.info do 
-      now = Time.now
-      str = Log.color :reset
-      str << "#{ Log.color status_color, status}"
-      str << ": #{ message }" if message and message != :result
-      str << " -- #{Log.color :blue, path.to_s}" if path
-      str << " #{Log.color :yellow, Process.pid}"
-      str
-    end
-    res = yield
-    eend = Time.now
-    Log.info do 
-      now = Time.now
-      str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.2f" % (eend - start)}"
-      str << ": #{ res }" if message == :result
-      str << " -- #{Log.color :blue, path.to_s}" if path
-      str << " #{Log.color :yellow, Process.pid}"
-      str
-    end
-    res
-  end
-
-  def self.log_string(status, message, path)
-    Log.info do 
-
-      status = status.to_s
-      status_color = self.status_color status
-
-      str = Log.color :reset
-      str << "#{ Log.color status_color, status}"
-      str << ": #{ message }" if message
-      str << " -- #{Log.color :blue, path.to_s}" if path
-      str << " #{Log.color :yellow, Process.pid}"
-      str
-    end
-  end
-
-  def self.log_progress(status, options = {}, path = nil, &block)
-    options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
-    max = Misc.process_options options, :max
-    Log::ProgressBar.with_bar(max, options) do |bar|
-      begin
-        res = yield bar
-        raise KeepBar.new res if IO === res
-        res
-      rescue
-        Log.exception $!
-        raise $!
-      end
-    end
-  end
-
-  def log_progress(status, options = {}, &block)
-    Step.log_progress(status, options, file(:progress), &block)
-  end
-
-  def progress_bar(msg = "Progress", options = nil)
-    if Hash === msg and options.nil?
-      options = msg
-      msg = nil
-    end
-    options = {} if options.nil?
-
-    max = options[:max]
-    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
-  end
-
-  def self.log(status, message, path, &block)
-    if block
-      if Hash === message
-        log_progress(status, message, path, &block)
-      else
-        log_block(status, message, path, &block)
-      end
-    else
-      log_string(status, message, path)
-    end
-  end
-
-  def log(status, message = nil, &block)
-    self.status = status
-    if message
-      self.message Log.uncolor(message)
-    end
-    Step.log(status, message, path, &block)
-  end
-
-  def exception(ex, msg = nil)
-    ex_class = ex.class.to_s
-    backtrace = ex.backtrace if ex.respond_to?(:backtrace)
-    message = ex.message if ex.respond_to?(:message)
-    set_info :backtrace, backtrace
-    set_info :exception, {:class => ex_class, :message => message, :backtrace => backtrace}
-    if msg.nil?
-      log :error, "#{ex_class} -- #{message}"
-    else
-      log :error, "#{msg} -- #{message}"
-    end
-    self._abort
-  end
-
-  def get_exception
-    if info[:exception].nil?
-      return Aborted if aborted?
-      return Exception.new(messages.last) if error?
-      Exception.new "" 
-    else
-      ex_class, ex_message, ex_backtrace = info[:exception].values_at :class, :message, :backtrace
-      begin
-        klass = Kernel.const_get(ex_class)
-        ex = klass.new ex_message
-        ex.set_backtrace ex_backtrace unless ex_backtrace.nil? or ex_backtrace.empty?
-        ex
-      rescue
-        Log.exception $!
-        Exception.new ex_message
-      end
-    end
-  end
-
-  def recoverable_error?
-    return true if aborted?
-    return false unless error?
-    begin
-      return true unless info[:exception]
-      klass = Kernel.const_get(info[:exception][:class])
-      ! (klass <= RbbtException)
-    rescue Exception
-      true
-    end
-  end
-
-  def started?
-    Open.exists?(path) or (Open.exists?(pid_file) && Open.exists?(info_file))
-  end
-
-  def waiting?
-    Open.exists?(info_file) and not started?
-  end
-
-  def dirty_files
-    rec_dependencies = self.rec_dependencies(true)
-    return [] if rec_dependencies.empty?
-    canfail_paths = self.canfail_paths
-
-    dirty_files = rec_dependencies.reject{|dep|
-      (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
-        ! Open.exists?(dep.info_file) ||
-        (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
-        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
-    }
-  end
-
-  def dirty?
-    return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
-    return false unless done? || status == :done
-    return false unless ENV["RBBT_UPDATE"] == "true"
-
-    status = self.status
-
-    if done? and not (status == :done or status == :ending or status == :producing) and not status == :noinfo
-      return true 
-    end
-
-    if status == :done and not done?
-      return true 
-    end
-
-    if dirty_files.any?
-      Log.low "Some dirty files found for #{self.path}: #{Misc.fingerprint dirty_files}"
-      true
-    else
-      ! self.updated?
-    end
-  end
-
-  def done?
-    path and Open.exists? path
-  end
-
-  def streaming?
-    (IO === @result) or (not @saved_stream.nil?) or status == :streaming
-  end
-
-  def noinfo?
-    status == :noinfo
-  end
-
-  def running? 
-    return false if ! (started? || status == :ending)
-    return nil unless Open.exist?(self.pid_file)
-    pid = Open.read(self.pid_file).to_i
-
-    return false if done? or error? or aborted? 
-
-    if Misc.pid_exists?(pid) 
-      pid
-    else
-      done? or error? or aborted? 
-    end
-  end
-
-  def stalled?
-    started? && ! (done? || running? || done? || error? || aborted?)
-  end
-
-  def missing?
-    status == :done && ! Open.exists?(path)
-  end
-
-  def error?
-    status == :error
-  end
-
-  def nopid?
-    ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
-  end
-
-  def aborted?
-    status = self.status
-    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
-  end
-
   # {{{ INFO
 
   def files_dir
@@ -650,10 +344,6 @@ class Step
     provenance
   end
 
-  def resumable?
-    task && task.resumable
-  end
-
   def config(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     options ||= {}