rbbt-util 5.32.16 → 5.32.17

data/lib/rbbt/workflow/step/dependencies.rb

@@ -1,4 +1,3 @@
-
 class Step
 
   STREAM_CACHE = {}
@@ -127,7 +126,7 @@ class Step
   end
 
   def input_dependencies
-    (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
+    @input_dependencies ||= (recursive_inputs.flatten.select{|i| Step === i } + recursive_inputs.flatten.select{|dep| Path === dep && Step === dep.resource }.collect{|dep| dep.resource })
   end
 
   def execute_dependency(dependency, log = true)
@@ -207,7 +206,7 @@ class Step
     if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
-        other_steps = dep_step[step.path]
+        other_steps = dep_step[step.path].uniq
         return unless other_steps.length > 1
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
@@ -369,6 +368,7 @@ class Step
       end
       next unless step.dependencies and step.dependencies.any?
       (step.dependencies + step.input_dependencies).each do |step_dep|
+        next unless step.dependencies.include?(step_dep)
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
@@ -483,4 +483,44 @@ class Step
     kill_children
   end
 
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
+  def overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
+    end
+    ord
+  end
+
+  def dependencies=(dependencies)
+    @dependencies = dependencies
+    set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
+  end
+
 end

data/lib/rbbt/workflow/step/info.rb

@@ -0,0 +1,294 @@
+class Step
+  def info(check_lock = true)
+    return {:status => :noinfo} if info_file.nil? || ! Open.exists?(info_file)
+
+    begin
+      Misc.insist do
+
+        begin
+          return @info_cache if @info_cache and @info_cache_time and Open.ctime(info_file) < @info_cache_time 
+        rescue Exception
+          raise $!
+        end
+
+
+        begin
+          @info_cache = Misc.insist(3, 1.6, info_file) do
+            Misc.insist(2, 1, info_file) do
+              Misc.insist(3, 0.2, info_file) do
+                raise TryAgain, "Info locked" if check_lock and info_lock.locked?
+                info_lock.lock if check_lock and false
+                begin
+                  Open.open(info_file, :mode => 'rb') do |file|
+                    Step.load_serialized_info(file)
+                  end
+                ensure
+                  info_lock.unlock if check_lock and false
+                end
+              end
+            end
+          end
+          @info_cache_time = Time.now
+          @info_cache
+        end
+      end
+    rescue Exception
+      Log.debug{"Error loading info file: " + info_file}
+      Log.exception $!
+      Open.rm info_file
+      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
+      raise $!
+    end
+  end
+
+  def load_inputs_from_info
+    if info[:inputs]
+      info_inputs = info[:inputs]
+      if task && task.respond_to?(:inputs) && task.inputs
+        IndiferentHash.setup info_inputs
+        @inputs = NamedArray.setup info_inputs.values_at(*task.inputs.collect{|name| name.to_s}), task.inputs
+      else
+        @inputs = NamedArray.setup info_inputs.values, info_inputs.keys
+      end
+    else
+      nil
+    end
+  end
+
+  def load_dependencies_from_info
+    relocated = nil
+    @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
+      if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
+        Workflow._load_step dep_path
+      else
+        next if FalseClass === relocated
+        new_path = Workflow.relocate(path, dep_path)
+        relocated = true if Open.exists?(new_path) || Open.exists?(new_path + '.info')
+        Workflow._load_step new_path
+      end
+    end.compact
+    @relocated = relocated
+  end
+
+
+  def archive_deps
+    self.set_info :archived_info, archived_info
+    self.set_info :archived_dependencies, info[:dependencies]
+  end
+
+  def archived_info
+    return info[:archived_info] if info[:archived_info]
+
+    archived_info = {}
+    dependencies.each do |dep|
+      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
+      archived_info.merge!(dep.archived_info)
+    end if dependencies
+
+    archived_info
+  end
+
+  def archived_inputs
+    return {} unless info[:archived_dependencies]
+    archived_info = self.archived_info
+
+    all_inputs = IndiferentHash.setup({})
+    deps = info[:archived_dependencies].collect{|p| p.last}
+    seen = []
+    while path = deps.pop
+      dep_info = archived_info[path]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
+      seen << path
+    end
+
+    all_inputs
+  end
+
+  def status
+    begin
+      info[:status]
+    rescue Exception
+      Log.error "Exception reading status: #{$!.message}" 
+      :error
+    end
+  end
+
+  def status=(status)
+    set_info(:status, status)
+  end
+
+  def messages
+    if messages = info[:messages]
+      messages
+    else
+      set_info(:messages, []) if self.respond_to?(:set_info)
+    end
+  end
+
+  def message(message)
+    message = Log.uncolor(message)
+    set_info(:messages, (messages || []) << message)
+  end
+
+  def self.status_color(status)
+    status = status.split(">").last
+    case status
+    when "starting"
+      :yellow
+    when "error", "aborted"
+      :red
+    when "done"
+      :green
+    else
+      :cyan
+    end
+  end
+
+  def self.log_block(status, message, path, &block)
+    start = Time.now
+    status = status.to_s
+    status_color = self.status_color status
+
+    Log.info do 
+      now = Time.now
+      str = Log.color :reset
+      str << "#{ Log.color status_color, status}"
+      str << ": #{ message }" if message and message != :result
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
+      str
+    end
+    res = yield
+    eend = Time.now
+    Log.info do 
+      now = Time.now
+      str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.2f" % (eend - start)}"
+      str << ": #{ res }" if message == :result
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
+      str
+    end
+    res
+  end
+
+  def self.log_string(status, message, path)
+    Log.info do 
+
+      status = status.to_s
+      status_color = self.status_color status
+
+      str = Log.color :reset
+      str << "#{ Log.color status_color, status}"
+      str << ": #{ message }" if message
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
+      str
+    end
+  end
+
+  def self.log_progress(status, options = {}, path = nil, &block)
+    options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
+    max = Misc.process_options options, :max
+    Log::ProgressBar.with_bar(max, options) do |bar|
+      begin
+        res = yield bar
+        raise KeepBar.new res if IO === res
+        res
+      rescue
+        Log.exception $!
+        raise $!
+      end
+    end
+  end
+
+  def log_progress(status, options = {}, &block)
+    Step.log_progress(status, options, file(:progress), &block)
+  end
+
+  def progress_bar(msg = "Progress", options = nil)
+    if Hash === msg and options.nil?
+      options = msg
+      msg = nil
+    end
+    options = {} if options.nil?
+
+    max = options[:max]
+    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+  end
+
+  def self.log(status, message, path, &block)
+    if block
+      if Hash === message
+        log_progress(status, message, path, &block)
+      else
+        log_block(status, message, path, &block)
+      end
+    else
+      log_string(status, message, path)
+    end
+  end
+
+  def log(status, message = nil, &block)
+    self.status = status
+    if message
+      self.message Log.uncolor(message)
+    end
+    Step.log(status, message, path, &block)
+  end
+
+  def exception(ex, msg = nil)
+    ex_class = ex.class.to_s
+    backtrace = ex.backtrace if ex.respond_to?(:backtrace)
+    message = ex.message if ex.respond_to?(:message)
+    set_info :backtrace, backtrace
+    set_info :exception, {:class => ex_class, :message => message, :backtrace => backtrace}
+    if msg.nil?
+      log :error, "#{ex_class} -- #{message}"
+    else
+      log :error, "#{msg} -- #{message}"
+    end
+    self._abort
+  end
+
+  def get_exception
+    if info[:exception].nil?
+      return Aborted if aborted?
+      return Exception.new(messages.last) if error?
+      Exception.new "" 
+    else
+      ex_class, ex_message, ex_backtrace = info[:exception].values_at :class, :message, :backtrace
+      begin
+        klass = Kernel.const_get(ex_class)
+        ex = klass.new ex_message
+        ex.set_backtrace ex_backtrace unless ex_backtrace.nil? or ex_backtrace.empty?
+        ex
+      rescue
+        Log.exception $!
+        Exception.new ex_message
+      end
+    end
+  end
+
+  def recoverable_error?
+    return true if aborted?
+    return false unless error?
+    begin
+      return true unless info[:exception]
+      klass = Kernel.const_get(info[:exception][:class])
+      ! (klass <= RbbtException)
+    rescue Exception
+      true
+    end
+  end
+
+end

data/lib/rbbt/workflow/step/status.rb

@@ -0,0 +1,146 @@
+class Step
+  def self.clean(path)
+    info_file = Step.info_file path
+    pid_file = Step.pid_file path
+    md5_file = Step.md5_file path
+    files_dir = Step.files_dir path
+    tmp_path = Step.tmp_path path
+
+    if ! (Open.writable?(path) && Open.writable?(info_file))
+      Log.warn "Could not clean #{path}: not writable"
+      return 
+    end
+
+    if ENV["RBBT_DEBUG_CLEAN"] == 'true'
+      raise "DO NOT CLEAN" 
+    end
+
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
+
+      @result = nil
+      @pid = nil
+
+      Misc.insist do
+        Open.rm info_file if Open.exists?(info_file)
+        Open.rm md5_file if Open.exists?(md5_file)
+        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
+        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
+        Open.rm pid_file if Open.exists?(pid_file)
+        Open.rm tmp_path if Open.exists?(tmp_path)
+      end
+    end
+  end
+
+  def clean
+    if ! Open.exists?(info_file)
+      Log.high "Refusing to clean step with no .info file: #{path}"
+      return self
+    end
+    status = []
+    status << "dirty" if done? && dirty?
+    status << "not running" if ! done? && ! running? 
+    status.unshift " " if status.any?
+    Log.high "Cleaning step: #{path}#{status * " "}"
+    Log.stack caller if RBBT_DEBUG_CLEAN
+    abort if ! done? && running?
+    Step.clean(path)
+    @done = false
+    self
+  end
+
+  def resumable?
+    (task && task.resumable) || status == :waiting || status == :cleaned
+  end
+
+  def started?
+    Open.exists?(path) or (Open.exists?(pid_file) && Open.exists?(info_file))
+  end
+
+  def waiting?
+    Open.exists?(info_file) and not started?
+  end
+
+  def dirty_files
+    rec_dependencies = self.rec_dependencies(true)
+    return [] if rec_dependencies.empty?
+    canfail_paths = self.canfail_paths
+
+    dirty_files = rec_dependencies.reject{|dep|
+      (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
+        ! Open.exists?(dep.info_file) ||
+        (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
+        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
+    }
+  end
+
+  def dirty?
+    return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
+    return false unless done? || status == :done
+    return false unless ENV["RBBT_UPDATE"] == "true"
+
+    status = self.status
+
+    if done? and not (status == :done or status == :ending or status == :producing) and not status == :noinfo
+      return true 
+    end
+
+    if status == :done and not done?
+      return true 
+    end
+
+    if dirty_files.any?
+      Log.low "Some dirty files found for #{self.path}: #{Misc.fingerprint dirty_files}"
+      true
+    else
+      ! self.updated?
+    end
+  end
+
+  def done?
+    @done ||= path and Open.exists? path
+  end
+
+  def streaming?
+    (IO === @result) or (not @saved_stream.nil?) or status == :streaming
+  end
+
+  def noinfo?
+    status == :noinfo
+  end
+
+  def running? 
+    return false if ! (started? || status == :ending)
+    return nil unless Open.exist?(self.pid_file)
+    pid = Open.read(self.pid_file).to_i
+
+    return false if done? or error? or aborted? 
+
+    if Misc.pid_exists?(pid) 
+      pid
+    else
+      done? or error? or aborted? 
+    end
+  end
+
+  def stalled?
+    started? && ! (done? || running? || done? || error? || aborted?)
+  end
+
+  def missing?
+    status == :done && ! Open.exists?(path)
+  end
+
+  def error?
+    status == :error
+  end
+
+  def nopid?
+    ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
+  end
+
+  def aborted?
+    status = self.status
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo && status != :waiting) && nopid?)
+  end
+
+end