rbbt-util 5.28.11 → 5.29.2

data/lib/rbbt/persist.rb

@@ -31,8 +31,8 @@ module Persist
     path = path.find if Path === path
     file = file.find if Path === file
     if by_link
-      patht = File.lstat(path).mtime
-      filet = File.lstat(file).mtime
+      patht = File.exists?(path) ? File.lstat(path).mtime : nil
+      filet = File.exists?(file) ? File.lstat(file).mtime : nil
     else
       patht = Open.mtime(path)
       filet = Open.mtime(file)

data/lib/rbbt/resource/path.rb

@@ -297,7 +297,7 @@ module Path
 
     raise "No resource defined to produce file: #{ self }" if resource.nil?
 
-    resource.produce self, force
+    resource.produce self, force if Resource === resource
 
     self
   end

data/lib/rbbt/tsv/attach.rb

@@ -243,6 +243,7 @@ module TSV
     Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
 
     if complete
+      Log.warn "Attaching through index and completing empty rows; keys with wrong format may appear (#{other.key_field} insted of #{self.key_field})" if index
       fill = TrueClass === complete ? nil : complete
       field_length = self.fields.length 
       common_fields = (other.fields & self.fields)
@@ -255,11 +256,11 @@ module TSV
       case type
       when :single
         missing.each do |k|
-          self[k] = nil
+          self[k] = fill
         end
       when :list
         missing.each do |k|
-          values = [nil] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -267,8 +268,9 @@ module TSV
           self[k] = values
         end
       when :double
+        fill = [] if fill.nil?
         missing.each do |k|
-          values = [[]] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -276,8 +278,9 @@ module TSV
           self[k] = values
         end
       when :flat
+        fill = [] if fill.nil?
         missing.each do |k|
-          self[k] = []
+          self[k] = fill
         end
       end
     end

data/lib/rbbt/tsv/parallel.rb

@@ -2,6 +2,3 @@ require 'rbbt/util/concurrency'
 
 require 'rbbt/tsv/parallel/through'
 require 'rbbt/tsv/parallel/traverse'
-
-module TSV
-end

data/lib/rbbt/util/misc/indiferent_hash.rb

@@ -61,6 +61,14 @@ module IndiferentHash
       super(key)
     end
   end
+
+  def clean_version
+    clean = {}
+    each do |k,v|
+      clean[k.to_s] = v unless clean.include? k.to_s
+    end
+    clean
+  end
 end
 
 module CaseInsensitiveHash

data/lib/rbbt/util/misc/inspect.rb

@@ -287,7 +287,9 @@ module Misc
           when Symbol 
             obj.to_s
           when (defined?(Path) and Path)
-            if Step === obj.resource
+            if defined?(Step) && Open.exists?(Step.info_file(obj))
+              obj2str(Workflow.load_step(obj))
+            elsif defined?(Step) && Step === obj.resource
               "Step file: " + obj
             else
               if obj.exists?
@@ -322,7 +324,11 @@ module Misc
             remove_long_items(obj)
           when File 
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
@@ -330,7 +336,11 @@ module Misc
             "<IO:" << obj.short_path << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
 
               if obj.respond_to? :obj2str

data/lib/rbbt/workflow.rb

@@ -385,7 +385,7 @@ module Workflow
       next if default == v 
       next if (String === default and Symbol === v and v.to_s == default)
       next if (Symbol === default and String === v and v == default.to_s)
-      real_inputs[k] = v 
+      real_inputs[k.to_sym] = v 
     end
 
     jobname_input_value = inputs[jobname_input] || all_defaults[jobname_input]
@@ -410,6 +410,7 @@ module Workflow
     job.workflow = self
     job.clean_name = jobname
     job.overriden = overriden
+    job.real_inputs = real_inputs.keys
     job
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -16,6 +16,10 @@ end
 
 module Workflow
 
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
   def log(status, message = nil, &block)
     Step.log(status, message, nil, &block)
   end
@@ -301,7 +305,9 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
     dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name if dep.workflow
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
@@ -309,7 +315,7 @@ module Workflow
       Log.exception $!
     end
     dep.task_name = task_name
-    dep.overriden = true
+    dep.overriden = dep.original_task_name.to_sym
     dep
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -77,6 +77,7 @@ module Workflow
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
       dep = dependencies.last.join
+      raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
       forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
       if forget

data/lib/rbbt/workflow/examples.rb

@@ -50,8 +50,8 @@ module Workflow
         case input_types[input]
         when :file
           Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.read$/
-            inputs[input.to_sym]  = Open.read(file)
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             inputs[input.to_sym]  = Open.realpath(file)
           end

data/lib/rbbt/workflow/step.rb

@@ -12,6 +12,9 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name
+
+  RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
   class << self
     attr_accessor :lock_dir
@@ -143,11 +146,13 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      dep_info[:inputs].each do |k,v|
-        all_inputs[k] = v unless all_inputs.include?(k)
-      end if dep_info[:inputs]
-      deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-      deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
       seen << path
     end
 
@@ -454,6 +459,7 @@ class Step
     status << "not running" if ! done? && ! running? 
     status.unshift " " if status.any?
     Log.high "Cleaning step: #{path}#{status * " "}"
+    Log.stack caller if RBBT_DEBUG_CLEAN
     abort if ! done? && running?
     Step.clean(path)
     self

data/lib/rbbt/workflow/step/accessor.rb

@@ -8,6 +8,16 @@ class Step
                        end
                      end
 
+  def self.serialize_info(info)
+    info = info.clean_version if IndiferentHash === info
+    INFO_SERIALIZER.dump(info)
+  end
+
+  def self.load_serialized_info(io)
+    IndiferentHash.setup(INFO_SERIALIZER.load(io))
+  end
+
+
   def self.wait_for_jobs(jobs)
     jobs = [jobs] if Step === jobs
     begin
@@ -59,7 +69,7 @@ class Step
   def self.step_info(path)
     begin
       Open.open(info_file(path), :mode => 'rb') do |f|
-        INFO_SERIALIZER.load(f)
+        self.load_serialized_info(f)
       end
     rescue Exception
       Log.exception $!
@@ -83,18 +93,22 @@ class Step
 
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          Open.ln_s(value, path)
+        else
+          Open.write(path + '.yaml', value.to_yaml)
+        end
       when Array === value
-        Open.write(path, value * "\n")
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
       when IO === value
-        Open.write(path, value)
-      when type == "file"
-        if String === value && File.exists?(value)
-          Open.link(value, path)
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
         else
-          Open.write(path + '.read', value.to_s)
+          Open.write(path, value)
         end
-      when Step === value
-        value = value.produce.load 
       else
         Open.write(path, value.to_s)
       end
@@ -104,18 +118,24 @@ class Step
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = job.task_name
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
-    task_info = workflow.task_info(task_name)
-    input_types = task_info[:input_types]
-    task_inputs = task_info[:inputs]
-    input_defaults = task_info[:input_defaults]
+    if workflow
+      task_info = workflow.task_info(task_name)
+      input_types = task_info[:input_types]
+      task_inputs = task_info[:inputs]
+      input_defaults = task_info[:input_defaults]
+    else
+      task_info = input_types = task_inputs = input_defaults = {}
+    end
 
     inputs = {}
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
-      next if options and ! options.include?(name)
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
       next if value.nil?
       next if input_defaults[name] == value
       inputs[name] = value
@@ -188,7 +208,7 @@ class Step
                 info_lock.lock if check_lock and false
                 begin
                   Open.open(info_file, :mode => 'rb') do |file|
-                    IndiferentHash.setup(INFO_SERIALIZER.load(file)) #|| {}
+                    Step.load_serialized_info(file)
                   end
                 ensure
                   info_lock.unlock if check_lock and false
@@ -204,7 +224,7 @@ class Step
       Log.debug{"Error loading info file: " + info_file}
       Log.exception $!
       Open.rm info_file
-      Misc.sensiblewrite(info_file, INFO_SERIALIZER.dump({:status => :error, :messages => ["Info file lost"]}))
+      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
       raise $!
     end
   end
@@ -212,10 +232,10 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
-      @info_cache = i
-      Misc.sensiblewrite(info_file, INFO_SERIALIZER.dump(i), :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
+      @info_cache = IndiferentHash.setup(i)
       @info_cache_time = Time.now
     end
   end
@@ -227,9 +247,9 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
       i[key] = value 
+      dump = Step.serialize_info(i)
       @info_cache = IndiferentHash.setup(i)
-      dump = INFO_SERIALIZER.dump(i)
-      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now
       value
     end
@@ -242,9 +262,9 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = info(false)
       i.merge! hash
+      dump = Step.serialize_info(i)
       @info_cache = IndiferentHash.setup(i)
-      dump = INFO_SERIALIZER.dump(i)
-      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now
       value
     end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,12 +92,18 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
+        if ! (job.resumable? && (job.updated? && ! job.dirty?))
+          Log.high "About to clean -- status: #{status}, present #{File.exists?(job.path)}, " +
+            %w(done? error? recoverable_error? noinfo? updated? dirty? aborted? running? resumable?).
+            collect{|v| [v, job.send(v)]*": "} * ", " if RBBT_DEBUG_CLEAN
+
+          job.clean
+        end
         job.set_info :status, :cleaned
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started? || ! Workflow.job_path?(job.path)
 
       canfail = ComputeDependency === job && job.canfail?
     end
@@ -121,10 +127,9 @@ class Step
   end
 
   def input_dependencies
-    inputs.flatten.select{|i| Step === i}
+    (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
-
   def execute_dependency(dependency, log = true)
     task_name = self.task_name
     canfail_paths = self.canfail_paths

data/lib/rbbt/workflow/step/run.rb

@@ -112,7 +112,7 @@ class Step
   end
 
   def updatable?
-    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?))
+    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?)) || (ENV["RBBT_UPDATE"] && ! (done? && ! Open.exists?(info_file)))
   end
 
   def dependency_checks
@@ -122,13 +122,12 @@ class Step
       reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
       reject{|dependency| dependency.error? }.
       #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
-      #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
       select{|dependency| dependency.updatable? }.
       collect{|dependency| Workflow.relocate_dependency(self, dependency)}
   end
 
   def input_checks
-    inputs.select{|i| Step === i }.
+    (inputs.select{|i| Step === i } + inputs.select{|i| Path === i && Step === i.resource}.collect{|i| i.resource}).
       select{|dependency| dependency.updatable? }
   end
 
@@ -154,25 +153,28 @@ class Step
     canfail_paths = self.canfail_paths
     this_mtime = Open.mtime(self.path) if Open.exists?(self.path)
 
-    checks.each do |dep| 
-      next unless dep.updatable?
-      dep_done = dep.done?
+    outdated_time = checks.select{|dep| dep.updatable? && dep.done? && Persist.newer?(path, dep.path) }
+    outdated_dep = checks.reject{|dep| dep.done? || (dep.error? && ! dep.recoverable_error? && canfail_paths.include?(dep.path)) }
 
-      begin
-        if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
-          outdated_time << dep
-        end
-      rescue
-      end
+    #checks.each do |dep| 
+    #  next unless dep.updatable?
+    #  dep_done = dep.done?
 
-      # Is this pointless? this would mean some dep got updated after a later
-      # dep but but before this one.
-      #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+    #  begin
+    #    if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
+    #      outdated_time << dep
+    #    end
+    #  rescue
+    #  end
 
-      if (! dep_done && ! canfail_paths.include?(dep.path))
-        outdated_dep << dep
-      end
-    end
+    #  # Is this pointless? this would mean some dep got updated after a later
+    #  # dep but but before this one.
+    #  #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+
+    #  if (! dep_done && ! canfail_paths.include?(dep.path))
+    #    outdated_dep << dep
+    #  end
+    #end
 
     Log.high "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
     Log.high "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
@@ -215,7 +217,7 @@ class Step
         no_load = :stream if no_load
 
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result_type = @task.result_type
+        result_type = @task.result_type if @task
         result_type = info[:result_type] if result_type.nil?
         result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step