rbbt-util 5.32.16 → 5.32.21

data/lib/rbbt/workflow/step/status.rb

@@ -0,0 +1,146 @@
+class Step
+  def self.clean(path)
+    info_file = Step.info_file path
+    pid_file = Step.pid_file path
+    md5_file = Step.md5_file path
+    files_dir = Step.files_dir path
+    tmp_path = Step.tmp_path path
+
+    if ! (Open.writable?(path) && Open.writable?(info_file))
+      Log.warn "Could not clean #{path}: not writable"
+      return 
+    end
+
+    if ENV["RBBT_DEBUG_CLEAN"] == 'true'
+      raise "DO NOT CLEAN" 
+    end
+
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
+
+      @result = nil
+      @pid = nil
+
+      Misc.insist do
+        Open.rm info_file if Open.exists?(info_file)
+        Open.rm md5_file if Open.exists?(md5_file)
+        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
+        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
+        Open.rm pid_file if Open.exists?(pid_file)
+        Open.rm tmp_path if Open.exists?(tmp_path)
+      end
+    end
+  end
+
+  def clean
+    if ! Open.exists?(info_file)
+      Log.high "Refusing to clean step with no .info file: #{path}"
+      return self
+    end
+    status = []
+    status << "dirty" if done? && dirty?
+    status << "not running" if ! done? && ! running? 
+    status.unshift " " if status.any?
+    Log.high "Cleaning step: #{path}#{status * " "}"
+    Log.stack caller if RBBT_DEBUG_CLEAN
+    abort if ! done? && running?
+    Step.clean(path)
+    @done = false
+    self
+  end
+
+  def resumable?
+    (task && task.resumable) || status == :waiting || status == :cleaned
+  end
+
+  def started?
+    Open.exists?(path) or (Open.exists?(pid_file) && Open.exists?(info_file))
+  end
+
+  def waiting?
+    Open.exists?(info_file) and not started?
+  end
+
+  def dirty_files
+    rec_dependencies = self.rec_dependencies(true)
+    return [] if rec_dependencies.empty?
+    canfail_paths = self.canfail_paths
+
+    dirty_files = rec_dependencies.reject{|dep|
+      (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
+        ! Open.exists?(dep.info_file) ||
+        (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
+        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
+    }
+  end
+
+  def dirty?
+    return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
+    return false unless done? || status == :done
+    return false unless ENV["RBBT_UPDATE"] == "true"
+
+    status = self.status
+
+    if done? and not (status == :done or status == :ending or status == :producing) and not status == :noinfo
+      return true 
+    end
+
+    if status == :done and not done?
+      return true 
+    end
+
+    if dirty_files.any?
+      Log.low "Some dirty files found for #{self.path}: #{Misc.fingerprint dirty_files}"
+      true
+    else
+      ! self.updated?
+    end
+  end
+
+  def done?
+    @done ||= path and Open.exists? path
+  end
+
+  def streaming?
+    (IO === @result) or (not @saved_stream.nil?) or status == :streaming
+  end
+
+  def noinfo?
+    status == :noinfo
+  end
+
+  def running? 
+    return false if ! (started? || status == :ending)
+    return nil unless Open.exist?(self.pid_file)
+    pid = Open.read(self.pid_file).to_i
+
+    return false if done? or error? or aborted? 
+
+    if Misc.pid_exists?(pid) 
+      pid
+    else
+      done? or error? or aborted? 
+    end
+  end
+
+  def stalled?
+    started? && ! (done? || running? || done? || error? || aborted?)
+  end
+
+  def missing?
+    status == :done && ! Open.exists?(path)
+  end
+
+  def error?
+    status == :error
+  end
+
+  def nopid?
+    ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
+  end
+
+  def aborted?
+    status = self.status
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo && status != :waiting) && nopid?)
+  end
+
+end

data/lib/rbbt/workflow/usage.rb

@@ -67,11 +67,13 @@ end
 
 module Workflow
 
-  def dep_tree(name)
+  def dep_tree(name, seen = [])
     @dep_tree ||= {}
     @dep_tree[name] ||= begin
                         dep_tree = {}
                         self.task_dependencies[name.to_sym].reverse.each do |dep|
+                          next if seen.include? dep
+                          seen << dep
                           dep = dep.first if Array === dep && dep.length == 1
                           dep = dep.dependency if DependencyBlock === dep
 
@@ -87,7 +89,7 @@ module Workflow
 
                           key = [workflow, task]
 
-                          dep_tree[key] = workflow.dep_tree(task)
+                          dep_tree[key] = workflow.dep_tree(task, seen)
                         end if name && self.task_dependencies[name.to_sym]
                         dep_tree
                       end

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -81,7 +81,7 @@ module Workflow
     def self.candidates(workload, rules)
       if rules.empty?
         candidates = workload.select{|k,v| v.empty? }.
-          collect{|k,v| k}.
+          collect{|k,v| k }.
           reject{|k| k.done? }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.

data/lib/rbbt/workflow/util/provenance.rb

@@ -47,6 +47,8 @@ class Step
     if input.nil? || input.empty?
       input_str = nil
     else
+      input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
+      input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
       input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
     end
 
@@ -91,21 +93,28 @@ class Step
     status = :unsync if status == :done and not Open.exist?(path)
     status = :notfound if status == :noinfo and not Open.exist?(path)
 
-    str = " " * offset
-    str << prov_report_msg(status, name, path, info, input)
+
+    this_step_msg = prov_report_msg(status, name, path, info, input)
 
     input_dependencies = {}
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
-          input_name = dep.inputs.fields.zip(dep.inputs).select{|f,d| d == id || String === d && d.start_with?(id.files_dir) }.first.first
+          input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+            d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
+          }.last.first
           input_dependencies[id] ||= []
           input_dependencies[id] << [dep, input_name]
         end
       end
     end
 
-    step.dependencies.reverse.each do |dep|
+    str = ""
+    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    step.dependencies.dup.tap{|l| 
+      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    }.each do |dep|
       path = dep.path
       new = ! seen.include?(path)
       if new
@@ -126,6 +135,9 @@ class Step
         end
       end
     end if step.dependencies
+
+    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
     str
   end
 end

data/share/rbbt_commands/system/clean

@@ -34,6 +34,7 @@ force = options.delete :force
 dirty = options.delete :dirty
 time = options.delete :older
 
+ENV["RBBT_DEBUG_CLEAN"] = 'false'
 ENV["RBBT_UPDATE"] = 'true' if dirty
 
 time = Misc.timespan time, 'd' if time

data/share/rbbt_commands/workflow/prov

@@ -131,6 +131,6 @@ if options[:plot]
   end
 
 else
-  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats]).strip
+  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats])
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.16
+  version: 5.32.21
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-05 00:00:00.000000000 Z
+date: 2021-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -322,6 +322,7 @@ files:
 - lib/rbbt/workflow/accessor.rb
 - lib/rbbt/workflow/annotate.rb
 - lib/rbbt/workflow/definition.rb
+- lib/rbbt/workflow/dependencies.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
 - lib/rbbt/workflow/integration/ansible.rb
@@ -339,8 +340,10 @@ files:
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/accessor.rb
 - lib/rbbt/workflow/step/dependencies.rb
+- lib/rbbt/workflow/step/info.rb
 - lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
+- lib/rbbt/workflow/step/status.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb