rbbt-util 5.32.16 → 5.32.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2898a052d60bd9d6c05e3d277482e2221d07c18067b592fe7b8fefd796e8a6a
-  data.tar.gz: 55bdd8b9a78d2d35a2c4c1574da11113c66b7621128076f29128941a5a0caf77
+  metadata.gz: 2e7aa425415a69c6615edd3fd7f38ce4c0bba0b1a911e7f97e927bf433483401
+  data.tar.gz: d2e8045de7f45d961afd8671b694c4c92f629dbc41d96057c84b9e97fc8cf15c
 SHA512:
-  metadata.gz: 3dd350c68a0c931da771f789aa934e3a94cf52be343101743a14eb2e049b782665db8f941da1f28e7503386af3f17c6ef4e53afa0ef388aecb65c9571ea974a5
-  data.tar.gz: 110a0a0eed2343835e3c4598f9c91c0280e28c64911a4fa18662c423d4f3d1774893b2e0ed848676a60878cae608a9b3d2e50de7c1c724935894971320628f2e
+  metadata.gz: b4f577d5d647b6c9ce1226062e7f5a459e6dee13e151e782d5938494a3e061d4b3fcc41bd14a8276ea646e58b726571445a45db7f2fe03dc34f4ca28d94c2d9a
+  data.tar.gz: d8432ea0d5232ddd9ac89c9e9cf0880a677cf708772ea5d0a6b6e28bd2204831d676e613a218c616c950c509c381a0ab2cf1acf88b98a85e9d4f18c3480ef48c

data/lib/rbbt/resource/path.rb

@@ -159,6 +159,12 @@ module Path
   end
 
   def find(where = nil, caller_lib = nil, paths = nil)
+
+    if located?
+      self.original ||= self
+      return self
+    end
+
     @path ||= {}
     rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil 
     key_elems = [where, caller_lib, rsearch_paths, paths]
@@ -167,12 +173,6 @@ module Path
 
     return @path[key] if @path[key]
 
-    if located?
-      @path[key] = self
-      self.original ||= self
-      return self
-    end
-
     @path[key] ||= begin
                      paths = [paths, rsearch_paths, self.search_paths, SEARCH_PATHS].reverse.compact.inject({}){|acc,h| acc.merge! h; acc }
                      where = paths[:default] if where == :default

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -332,7 +332,7 @@ module TSV
     Log.low{"Traversing #{name} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
     begin
       case obj
-      when (defined? FastContainers and FastContainers::PriorityQueue)
+      when (defined? FastContainers && FastContainers::PriorityQueue === obj)
         traverse_priority_queue(obj, options, &block)
       when TSV
         traverse_tsv(obj, options, &block)

data/lib/rbbt/util/log.rb

@@ -322,8 +322,13 @@ module Log
 
   def self.exception(e)
     stack = caller
-    error([e.class.to_s, e.message].compact * ": " )
-    error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+    if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+      error([e.class.to_s, e.message].compact * ": " )
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+    else
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
+      error([e.class.to_s, e.message].compact * ": " )
+    end
   end
 
   def self.deprecated(m)
@@ -370,11 +375,17 @@ module Log
 
   def self.stack(stack)
     LOG_MUTEX.synchronize do
-
-      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+      if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+        STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack).each do |line|
         STDERR.puts line
       end
+      else
+        STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+        color_stack(stack.reverse).each do |line|
+          STDERR.puts line
+        end
+      end
     end
   end
 

data/lib/rbbt/util/misc/inspect.rb

@@ -320,7 +320,7 @@ module Misc
               end
             end
           when String
-            if Misc.is_filename?(obj) and ! %w(. ..).include?(obj)
+            if Misc.is_filename?(obj, false) and ! %w(. ..).include?(obj) and ! Path === obj
               obj2str Path.setup(obj.dup)
             else
               obj = obj.chomp if String === obj

data/lib/rbbt/util/open.rb

@@ -240,6 +240,7 @@ module Open
   end
 
   def self.find_repo_dir(file)
+    return nil
     self.repository_dirs.each do |dir|
       dir = dir + '/' unless dir.chars[-1] == "/"
 

data/lib/rbbt/workflow.rb

@@ -1,4 +1,5 @@
 require 'rbbt/workflow/definition'
+require 'rbbt/workflow/dependencies'
 require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
 require 'rbbt/workflow/accessor'
@@ -228,6 +229,47 @@ module Workflow
                   end
   end
 
+  TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
+  DEBUG_JOB_HASH = ENV["RBBT_DEBUG_JOB_HASH"] == 'true'
+  def step_path(taskname, jobname, inputs, dependencies, extension = nil)
+    raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
+    if inputs.length > 0 or dependencies.any?
+      tagged_jobname = case TAG
+                       when :hash
+                         clean_inputs = Annotated.purge(inputs)
+                         clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
+                         key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
+                         key_str = Misc.obj2str(key_obj)
+                         hash_str = Misc.digest(key_str)
+                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}" if DEBUG_JOB_HASH
+                         jobname + '_' << hash_str
+                       when :inputs
+                         all_inputs = {}
+                         inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
+                           all_inputs[f] = i
+                         end
+                         dependencies.each do |dep|
+                           ri = dep.recursive_inputs
+                           ri.zip(ri.fields).each do |i,f|
+                             all_inputs[f] = i
+                           end
+                         end
+
+                         all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
+                       else
+                         jobname
+                       end
+    else
+      tagged_jobname = jobname
+    end
+
+    if extension and not extension.empty?
+      tagged_jobname = tagged_jobname + ('.' << extension.to_s)
+    end
+
+    workdir[taskname][tagged_jobname].find
+  end
   def import_task(workflow, orig, new)
     orig_task = workflow.tasks[orig]
     new_task = orig_task.dup
@@ -417,7 +459,7 @@ module Workflow
       extension = nil
       if dependencies.any?
         dep_basename = File.basename(dependencies.last.path)
-        extension = dep_path.split(".").last if dep_basename.include?('.')
+        extension = dep_basename.split(".").last if dep_basename.include?('.')
       end
     end
 

data/lib/rbbt/workflow/accessor.rb

@@ -7,7 +7,7 @@ module ComputeDependency
     dep.extend ComputeDependency
     dep.compute = value
   end
-  
+
   def canfail?
     compute == :canfail || (Array === compute && compute.include?(:canfail))
   end
@@ -72,83 +72,6 @@ module Workflow
                        end
   end
 
-  def rec_dependencies(taskname)
-    @rec_dependencies ||= {}
-    @rec_dependencies[taskname] ||= begin
-                                      if task_dependencies.include? taskname
-
-                                        deps = task_dependencies[taskname]
-
-                                        #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
-
-                                        all_deps = []
-                                        deps.each do |dep| 
-                                          if DependencyBlock === dep
-                                            all_deps << dep.dependency if dep.dependency
-                                          else
-                                            all_deps << dep unless Proc === dep
-                                          end
-
-                                          begin
-                                            case dep
-                                            when Array
-                                              wf, t, o = dep
-
-                                              wf.rec_dependencies(t.to_sym).each do |d|
-                                                if Array === d
-                                                  new = d.dup
-                                                else
-                                                  new = [dep.first, d]
-                                                end
-
-                                                if Hash === o and not o.empty? 
-                                                  if Hash === new.last
-                                                    hash = new.last.dup
-                                                    o.each{|k,v| hash[k] ||= v}
-                                                    new[new.length-1] = hash
-                                                  else
-                                                    new.push o.dup
-                                                  end
-                                                end
-
-                                                all_deps << new
-                                              end if wf && t
-
-                                            when String, Symbol
-                                              rec_deps = rec_dependencies(dep.to_sym)
-                                              all_deps.concat rec_deps
-                                            when DependencyBlock
-                                              dep = dep.dependency
-                                              raise TryAgain
-                                            end
-                                          rescue TryAgain
-                                            retry
-                                          end
-                                        end
-                                        all_deps.uniq
-                                      else
-                                        []
-                                      end
-                                    end
-  end
-
-  def task_from_dep(dep)
-    task = case dep
-           when Array
-             dep.first.tasks[dep[1]] 
-           when String
-             tasks[dep.to_sym]
-           when Symbol
-             tasks[dep.to_sym]
-           end
-    raise "Unknown dependency: #{Misc.fingerprint dep}" if task.nil?
-    task
-  end
-
-  #def rec_inputs(taskname)
-  #  [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat(task_from_dep(tn).inputs) }.uniq
-  #end
-
   def rec_inputs(taskname)
     task = task_from_dep(taskname)
     deps = rec_dependencies(taskname)
@@ -294,174 +217,24 @@ module Workflow
     _inputs
   end
 
-  def override_dependencies(inputs)
-    override_dependencies = IndiferentHash.setup({})
-    return override_dependencies if inputs.nil?
-    inputs.each do |key,value|
-      if String === key && m = key.match(/(.*)#(.*)/)
-        workflow, task = m.values_at 1, 2
-        workflow = self.to_s if workflow.empty?
-        override_dependencies[workflow] ||= IndiferentHash.setup({})
-        override_dependencies[workflow][task] = value
-      end
-    end
-    override_dependencies
-  end
-
-  def setup_override_dependency(dep, workflow, task_name)
-    dep = Step === dep ? dep : Workflow.load_step(dep)
-    dep.workflow = workflow
-    dep.info[:name] = dep.name
-    dep.original_task_name ||= dep.task_name if dep.workflow
-    begin
-      workflow = Kernel.const_get workflow if String === workflow
-      dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
-    rescue
-      Log.exception $!
-    end
-    dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym
-    dep
-  end
-
-  def real_dependencies(task, orig_jobname, inputs, dependencies)
-    real_dependencies = []
-    path_deps = {}
-
-    override_dependencies = override_dependencies(inputs)
-
-    dependencies.each do |dependency|
-      _inputs = IndiferentHash.setup(inputs.dup)
-      jobname = orig_jobname
-      jobname = _inputs[:jobname] if _inputs.include? :jobname
-
-      real_dep = case dependency
-                 when Array
-                   workflow, dep_task, options = dependency
-
-                   if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
-                     setup_override_dependency(value, workflow, dep_task)
-                   else
-
-                     compute = options[:compute] if options
-
-                     all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
-
-                     _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
-                     jobname = _inputs.delete :jobname if _inputs.include? :jobname
-
-                     job = workflow._job(dep_task, jobname, _inputs)
-                     ComputeDependency.setup(job, compute) if compute
-                     job
-                   end
-                 when Step
-                   dependency
-                 when Symbol
-                   if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
-                     setup_override_dependency(value, self, dependency)
-                   else
-                     _job(dependency, jobname, _inputs)
-                   end
-                 when Proc
-                   if DependencyBlock === dependency
-                     orig_dep = dependency.dependency 
-                     wf, task_name, options = orig_dep
-
-                     options = {} if options.nil?
-                     compute = options[:compute]
-
-                     options = IndiferentHash.setup(options.dup)
-                     dep = dependency.call jobname, _inputs.merge(options), real_dependencies
-
-                     dep = [dep] unless Array === dep
-
-                     new_=[]
-                     dep.each{|d| 
-                       next if d.nil?
-                       if Hash === d
-                         d[:workflow] ||= wf 
-                         d[:task] ||= task_name
-                         _override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
-                         d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
-                               setup_override_dependency(value, d[:workflow], d[:task])
-                             else
-                               task_info = d[:workflow].task_info(d[:task])
-
-                               _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                               d[:workflow]._job(d[:task], d[:jobname], _inputs) 
-                             end
-                       end
-                       ComputeDependency.setup(d, compute) if compute
-                       new_ << d
-                     }
-                     dep = new_
-                   else
-                     _inputs = IndiferentHash.setup(_inputs.dup)
-                     dep = dependency.call jobname, _inputs, real_dependencies
-                     if Hash === dep
-                       dep[:workflow] ||= wf || self
-                       _override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
-                       if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
-                         setup_override_dependency(value, dep[:workflow], dep[:task])
-                       else
-                         task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
-                         _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
-                       end
-                     end
-                   end
-
-                   dep
-                 else
-                   raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
-                 end
-
-      real_dependencies << real_dep
-    end
-    real_dependencies.flatten.compact
+  def task_from_dep(dep)
+    task = case dep
+           when Array
+             dep.first.tasks[dep[1]] 
+           when String
+             tasks[dep.to_sym]
+           when Symbol
+             tasks[dep.to_sym]
+           end
+    raise "Unknown dependency: #{Misc.fingerprint dep}" if task.nil?
+    task
   end
 
-  TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
-  DEBUG_JOB_HASH = ENV["RBBT_DEBUG_JOB_HASH"] == 'true'
-  def step_path(taskname, jobname, inputs, dependencies, extension = nil)
-    raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
-    if inputs.length > 0 or dependencies.any?
-      tagged_jobname = case TAG
-                       when :hash
-                         clean_inputs = Annotated.purge(inputs)
-                         clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << d.short_path : d }
-                         key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
-                         key_str = Misc.obj2str(key_obj)
-                         hash_str = Misc.digest(key_str)
-                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}" if DEBUG_JOB_HASH
-                         jobname + '_' << hash_str
-                       when :inputs
-                         all_inputs = {}
-                         inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
-                           all_inputs[f] = i
-                         end
-                         dependencies.each do |dep|
-                           ri = dep.recursive_inputs
-                           ri.zip(ri.fields).each do |i,f|
-                             all_inputs[f] = i
-                           end
-                         end
-
-                         all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
-                       else
-                         jobname
-                       end
-    else
-      tagged_jobname = jobname
-    end
+  #def rec_inputs(taskname)
+  #  [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat(task_from_dep(tn).inputs) }.uniq
+  #end
 
-    if extension and not extension.empty?
-      tagged_jobname = tagged_jobname + ('.' << extension.to_s)
-    end
 
-    workdir[taskname][tagged_jobname].find
-  end
 
   def id_for(path)
     if workdir.respond_to? :find

data/lib/rbbt/workflow/definition.rb

@@ -77,6 +77,7 @@ module Workflow
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      Step.wait_for_jobs dependencies
       dep = dependencies.last
       dep.join
       raise dep.get_exception if dep.error?