rbbt-util 5.32.16 → 5.32.21

data/lib/rbbt/workflow/step/accessor.rb

@@ -26,6 +26,7 @@ class Step
       threads = jobs.collect do |j| 
         Thread.new do
           begin
+            j.soft_grace
             j.join unless j.done?
           rescue Exception
             Log.error "Exception waiting for job: #{Log.color :blue, j.path}"
@@ -206,45 +207,6 @@ class Step
     #@status_lock
   end
 
-  def info(check_lock = true)
-    return {:status => :noinfo} if info_file.nil? or not Open.exists? info_file
-    begin
-      Misc.insist do
-        begin
-          return @info_cache if @info_cache and @info_cache_time and Open.ctime(info_file) < @info_cache_time 
-        rescue Exception
-          raise $!
-        end
-
-        begin
-          @info_cache = Misc.insist(3, 1.6, info_file) do
-            Misc.insist(2, 1, info_file) do
-              Misc.insist(3, 0.2, info_file) do
-                raise TryAgain, "Info locked" if check_lock and info_lock.locked?
-                info_lock.lock if check_lock and false
-                begin
-                  Open.open(info_file, :mode => 'rb') do |file|
-                    Step.load_serialized_info(file)
-                  end
-                ensure
-                  info_lock.unlock if check_lock and false
-                end
-              end
-            end
-          end
-          @info_cache_time = Time.now
-          @info_cache
-        end
-      end
-    rescue Exception
-      Log.debug{"Error loading info file: " + info_file}
-      Log.exception $!
-      Open.rm info_file
-      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
-      raise $!
-    end
-  end
-
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
@@ -286,274 +248,6 @@ class Step
     end
   end
 
-  def status
-    begin
-      info[:status]
-    rescue Exception
-      Log.error "Exception reading status: #{$!.message}" 
-      :error
-    end
-  end
-
-  def status=(status)
-    set_info(:status, status)
-  end
-
-  def messages
-    if messages = info[:messages]
-      messages
-    else
-      set_info(:messages, []) if self.respond_to?(:set_info)
-    end
-  end
-
-  def message(message)
-    message = Log.uncolor(message)
-    set_info(:messages, (messages || []) << message)
-  end
-
-  def self.status_color(status)
-    status = status.split(">").last
-    case status
-    when "starting"
-      :yellow
-    when "error", "aborted"
-      :red
-    when "done"
-      :green
-    else
-      :cyan
-    end
-  end
-
-  def self.log_block(status, message, path, &block)
-    start = Time.now
-    status = status.to_s
-    status_color = self.status_color status
-
-    Log.info do 
-      now = Time.now
-      str = Log.color :reset
-      str << "#{ Log.color status_color, status}"
-      str << ": #{ message }" if message and message != :result
-      str << " -- #{Log.color :blue, path.to_s}" if path
-      str << " #{Log.color :yellow, Process.pid}"
-      str
-    end
-    res = yield
-    eend = Time.now
-    Log.info do 
-      now = Time.now
-      str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.2f" % (eend - start)}"
-      str << ": #{ res }" if message == :result
-      str << " -- #{Log.color :blue, path.to_s}" if path
-      str << " #{Log.color :yellow, Process.pid}"
-      str
-    end
-    res
-  end
-
-  def self.log_string(status, message, path)
-    Log.info do 
-
-      status = status.to_s
-      status_color = self.status_color status
-
-      str = Log.color :reset
-      str << "#{ Log.color status_color, status}"
-      str << ": #{ message }" if message
-      str << " -- #{Log.color :blue, path.to_s}" if path
-      str << " #{Log.color :yellow, Process.pid}"
-      str
-    end
-  end
-
-  def self.log_progress(status, options = {}, path = nil, &block)
-    options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
-    max = Misc.process_options options, :max
-    Log::ProgressBar.with_bar(max, options) do |bar|
-      begin
-        res = yield bar
-        raise KeepBar.new res if IO === res
-        res
-      rescue
-        Log.exception $!
-        raise $!
-      end
-    end
-  end
-
-  def log_progress(status, options = {}, &block)
-    Step.log_progress(status, options, file(:progress), &block)
-  end
-
-  def progress_bar(msg = "Progress", options = nil)
-    if Hash === msg and options.nil?
-      options = msg
-      msg = nil
-    end
-    options = {} if options.nil?
-
-    max = options[:max]
-    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
-  end
-
-  def self.log(status, message, path, &block)
-    if block
-      if Hash === message
-        log_progress(status, message, path, &block)
-      else
-        log_block(status, message, path, &block)
-      end
-    else
-      log_string(status, message, path)
-    end
-  end
-
-  def log(status, message = nil, &block)
-    self.status = status
-    if message
-      self.message Log.uncolor(message)
-    end
-    Step.log(status, message, path, &block)
-  end
-
-  def exception(ex, msg = nil)
-    ex_class = ex.class.to_s
-    backtrace = ex.backtrace if ex.respond_to?(:backtrace)
-    message = ex.message if ex.respond_to?(:message)
-    set_info :backtrace, backtrace
-    set_info :exception, {:class => ex_class, :message => message, :backtrace => backtrace}
-    if msg.nil?
-      log :error, "#{ex_class} -- #{message}"
-    else
-      log :error, "#{msg} -- #{message}"
-    end
-    self._abort
-  end
-
-  def get_exception
-    if info[:exception].nil?
-      return Aborted if aborted?
-      return Exception.new(messages.last) if error?
-      Exception.new "" 
-    else
-      ex_class, ex_message, ex_backtrace = info[:exception].values_at :class, :message, :backtrace
-      begin
-        klass = Kernel.const_get(ex_class)
-        ex = klass.new ex_message
-        ex.set_backtrace ex_backtrace unless ex_backtrace.nil? or ex_backtrace.empty?
-        ex
-      rescue
-        Log.exception $!
-        Exception.new ex_message
-      end
-    end
-  end
-
-  def recoverable_error?
-    return true if aborted?
-    return false unless error?
-    begin
-      return true unless info[:exception]
-      klass = Kernel.const_get(info[:exception][:class])
-      ! (klass <= RbbtException)
-    rescue Exception
-      true
-    end
-  end
-
-  def started?
-    Open.exists?(path) or (Open.exists?(pid_file) && Open.exists?(info_file))
-  end
-
-  def waiting?
-    Open.exists?(info_file) and not started?
-  end
-
-  def dirty_files
-    rec_dependencies = self.rec_dependencies(true)
-    return [] if rec_dependencies.empty?
-    canfail_paths = self.canfail_paths
-
-    dirty_files = rec_dependencies.reject{|dep|
-      (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
-        ! Open.exists?(dep.info_file) ||
-        (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
-        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
-    }
-  end
-
-  def dirty?
-    return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
-    return false unless done? || status == :done
-    return false unless ENV["RBBT_UPDATE"] == "true"
-
-    status = self.status
-
-    if done? and not (status == :done or status == :ending or status == :producing) and not status == :noinfo
-      return true 
-    end
-
-    if status == :done and not done?
-      return true 
-    end
-
-    if dirty_files.any?
-      Log.low "Some dirty files found for #{self.path}: #{Misc.fingerprint dirty_files}"
-      true
-    else
-      ! self.updated?
-    end
-  end
-
-  def done?
-    path and Open.exists? path
-  end
-
-  def streaming?
-    (IO === @result) or (not @saved_stream.nil?) or status == :streaming
-  end
-
-  def noinfo?
-    status == :noinfo
-  end
-
-  def running? 
-    return false if ! (started? || status == :ending)
-    return nil unless Open.exist?(self.pid_file)
-    pid = Open.read(self.pid_file).to_i
-
-    return false if done? or error? or aborted? 
-
-    if Misc.pid_exists?(pid) 
-      pid
-    else
-      done? or error? or aborted? 
-    end
-  end
-
-  def stalled?
-    started? && ! (done? || running? || done? || error? || aborted?)
-  end
-
-  def missing?
-    status == :done && ! Open.exists?(path)
-  end
-
-  def error?
-    status == :error
-  end
-
-  def nopid?
-    ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
-  end
-
-  def aborted?
-    status = self.status
-    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
-  end
-
   # {{{ INFO
 
   def files_dir
@@ -650,10 +344,6 @@ class Step
     provenance
   end
 
-  def resumable?
-    task && task.resumable
-  end
-
   def config(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     options ||= {}

data/lib/rbbt/workflow/step/dependencies.rb

@@ -1,4 +1,3 @@
-
 class Step
 
   STREAM_CACHE = {}
@@ -127,7 +126,7 @@ class Step
   end
 
   def input_dependencies
-    (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
+    @input_dependencies ||= (recursive_inputs.flatten.select{|i| Step === i } + recursive_inputs.flatten.select{|dep| Path === dep && Step === dep.resource }.collect{|dep| dep.resource })
   end
 
   def execute_dependency(dependency, log = true)
@@ -207,7 +206,7 @@ class Step
     if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
-        other_steps = dep_step[step.path]
+        other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
         return unless other_steps.length > 1
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
@@ -369,6 +368,7 @@ class Step
       end
       next unless step.dependencies and step.dependencies.any?
       (step.dependencies + step.input_dependencies).each do |step_dep|
+        next unless step.dependencies.include?(step_dep)
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
@@ -483,4 +483,76 @@ class Step
     kill_children
   end
 
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
+  def overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
+    end
+    ord
+  end
+
+  def dependencies=(dependencies)
+    @dependencies = dependencies
+    set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
+  end
+
+  #connected = true means that dependency searching ends when a result is done
+  #but dependencies are absent, meanining that the file could have been dropped
+  #in
+  def rec_dependencies(connected = false, seen = [])
+
+    # A step result with no info_file means that it was manually
+    # placed. In that case, do not consider its dependencies
+    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
+
+    return [] if dependencies.nil? or dependencies.empty?
+
+    new_dependencies = []
+    if self.overriden?
+      archived_deps = []
+    else
+      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+    end
+
+    dependencies.each{|step| 
+      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
+      next if archived_deps.include? step.path
+      next if seen.include? step.path
+      next if self.done? && connected && ! updatable?
+
+      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
+      new_dependencies.concat r
+      new_dependencies << step
+    }
+
+    new_dependencies.uniq
+  end
+
 end

data/lib/rbbt/workflow/step/info.rb

@@ -0,0 +1,294 @@
+class Step
+  def info(check_lock = true)
+    return {:status => :noinfo} if info_file.nil? || ! Open.exists?(info_file)
+
+    begin
+      Misc.insist do
+
+        begin
+          return @info_cache if @info_cache and @info_cache_time and Open.ctime(info_file) < @info_cache_time 
+        rescue Exception
+          raise $!
+        end
+
+
+        begin
+          @info_cache = Misc.insist(3, 1.6, info_file) do
+            Misc.insist(2, 1, info_file) do
+              Misc.insist(3, 0.2, info_file) do
+                raise TryAgain, "Info locked" if check_lock and info_lock.locked?
+                info_lock.lock if check_lock and false
+                begin
+                  Open.open(info_file, :mode => 'rb') do |file|
+                    Step.load_serialized_info(file)
+                  end
+                ensure
+                  info_lock.unlock if check_lock and false
+                end
+              end
+            end
+          end
+          @info_cache_time = Time.now
+          @info_cache
+        end
+      end
+    rescue Exception
+      Log.debug{"Error loading info file: " + info_file}
+      Log.exception $!
+      Open.rm info_file
+      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
+      raise $!
+    end
+  end
+
+  def load_inputs_from_info
+    if info[:inputs]
+      info_inputs = info[:inputs]
+      if task && task.respond_to?(:inputs) && task.inputs
+        IndiferentHash.setup info_inputs
+        @inputs = NamedArray.setup info_inputs.values_at(*task.inputs.collect{|name| name.to_s}), task.inputs
+      else
+        @inputs = NamedArray.setup info_inputs.values, info_inputs.keys
+      end
+    else
+      nil
+    end
+  end
+
+  def load_dependencies_from_info
+    relocated = nil
+    @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
+      if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
+        Workflow._load_step dep_path
+      else
+        next if FalseClass === relocated
+        new_path = Workflow.relocate(path, dep_path)
+        relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
+        Workflow._load_step new_path
+      end
+    end.compact
+    @relocated = relocated
+  end
+
+
+  def archive_deps
+    self.set_info :archived_info, archived_info
+    self.set_info :archived_dependencies, info[:dependencies]
+  end
+
+  def archived_info
+    return info[:archived_info] if info[:archived_info]
+
+    archived_info = {}
+    dependencies.each do |dep|
+      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
+      archived_info.merge!(dep.archived_info)
+    end if dependencies
+
+    archived_info
+  end
+
+  def archived_inputs
+    return {} unless info[:archived_dependencies]
+    archived_info = self.archived_info
+
+    all_inputs = IndiferentHash.setup({})
+    deps = info[:archived_dependencies].collect{|p| p.last}
+    seen = []
+    while path = deps.pop
+      dep_info = archived_info[path]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
+      seen << path
+    end
+
+    all_inputs
+  end
+
+  def status
+    begin
+      info[:status]
+    rescue Exception
+      Log.error "Exception reading status: #{$!.message}" 
+      :error
+    end
+  end
+
+  def status=(status)
+    set_info(:status, status)
+  end
+
+  def messages
+    if messages = info[:messages]
+      messages
+    else
+      set_info(:messages, []) if self.respond_to?(:set_info)
+    end
+  end
+
+  def message(message)
+    message = Log.uncolor(message)
+    set_info(:messages, (messages || []) << message)
+  end
+
+  def self.status_color(status)
+    status = status.split(">").last
+    case status
+    when "starting"
+      :yellow
+    when "error", "aborted"
+      :red
+    when "done"
+      :green
+    else
+      :cyan
+    end
+  end
+
+  def self.log_block(status, message, path, &block)
+    start = Time.now
+    status = status.to_s
+    status_color = self.status_color status
+
+    Log.info do 
+      now = Time.now
+      str = Log.color :reset
+      str << "#{ Log.color status_color, status}"
+      str << ": #{ message }" if message and message != :result
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
+      str
+    end
+    res = yield
+    eend = Time.now
+    Log.info do 
+      now = Time.now
+      str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.2f" % (eend - start)}"
+      str << ": #{ res }" if message == :result
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
+      str
+    end
+    res
+  end
+
+  def self.log_string(status, message, path)
+    Log.info do 
+
+      status = status.to_s
+      status_color = self.status_color status
+
+      str = Log.color :reset
+      str << "#{ Log.color status_color, status}"
+      str << ": #{ message }" if message
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
+      str
+    end
+  end
+
+  def self.log_progress(status, options = {}, path = nil, &block)
+    options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
+    max = Misc.process_options options, :max
+    Log::ProgressBar.with_bar(max, options) do |bar|
+      begin
+        res = yield bar
+        raise KeepBar.new res if IO === res
+        res
+      rescue
+        Log.exception $!
+        raise $!
+      end
+    end
+  end
+
+  def log_progress(status, options = {}, &block)
+    Step.log_progress(status, options, file(:progress), &block)
+  end
+
+  def progress_bar(msg = "Progress", options = nil)
+    if Hash === msg and options.nil?
+      options = msg
+      msg = nil
+    end
+    options = {} if options.nil?
+
+    max = options[:max]
+    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+  end
+
+  def self.log(status, message, path, &block)
+    if block
+      if Hash === message
+        log_progress(status, message, path, &block)
+      else
+        log_block(status, message, path, &block)
+      end
+    else
+      log_string(status, message, path)
+    end
+  end
+
+  def log(status, message = nil, &block)
+    self.status = status
+    if message
+      self.message Log.uncolor(message)
+    end
+    Step.log(status, message, path, &block)
+  end
+
+  def exception(ex, msg = nil)
+    ex_class = ex.class.to_s
+    backtrace = ex.backtrace if ex.respond_to?(:backtrace)
+    message = ex.message if ex.respond_to?(:message)
+    set_info :backtrace, backtrace
+    set_info :exception, {:class => ex_class, :message => message, :backtrace => backtrace}
+    if msg.nil?
+      log :error, "#{ex_class} -- #{message}"
+    else
+      log :error, "#{msg} -- #{message}"
+    end
+    self._abort
+  end
+
+  def get_exception
+    if info[:exception].nil?
+      return Aborted if aborted?
+      return Exception.new(messages.last) if error?
+      Exception.new "" 
+    else
+      ex_class, ex_message, ex_backtrace = info[:exception].values_at :class, :message, :backtrace
+      begin
+        klass = Kernel.const_get(ex_class)
+        ex = klass.new ex_message
+        ex.set_backtrace ex_backtrace unless ex_backtrace.nil? or ex_backtrace.empty?
+        ex
+      rescue
+        Log.exception $!
+        Exception.new ex_message
+      end
+    end
+  end
+
+  def recoverable_error?
+    return true if aborted?
+    return false unless error?
+    begin
+      return true unless info[:exception]
+      klass = Kernel.const_get(info[:exception][:class])
+      ! (klass <= RbbtException)
+    rescue Exception
+      true
+    end
+  end
+
+end