rbbt-util 5.23.31 → 5.23.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 109bb2daf8846d12be60271835c62b41d57000ef
-  data.tar.gz: 73fc3ca6821a320f3b8e6ff8e557aa3265183383
+  metadata.gz: b174e2fa2cbfa2a1548380710082c35a98e5d58e
+  data.tar.gz: 74066e9c626e51bd9c625ed741cff14a177a9ea2
 SHA512:
-  metadata.gz: 0dddc6c05a96c78ea6fdb696e5a46c3ce4e29a02867c2a58f7d8751f34da99cfc40051ad3564bcf5d910929c9b61ae9dd3f65a13d14020793cc049d97e39f8b0
-  data.tar.gz: 71c62ea2ef84428217180c01ea7b5b8c4207e3475cfad5f6deefee636c332cb82e05aa02c85de7ec496b8d179feedd634c9bf55fb61bfbf656527f7e7f7515a8
+  metadata.gz: 0f5757b41d2605ff9da15ec39716ab47170675380a9f83e7ed729d50e93f29d74af3f49d6b6dd25a8376922b993216aed0d3db1802339ee793560e21dcde7ee5
+  data.tar.gz: 80e8eab073f7704254b0358381bca50f0d2694b23e9af7e550ed3b75f511702e3a1a36a23f71bb27d2e84b6373379993bdbd8dd00ef40ab34ed95ce5a1fd46d8

data/lib/rbbt/resource/path.rb

@@ -129,7 +129,7 @@ module Path
     @path ||= {}
     rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil 
     key_elems = [where, caller_lib, rsearch_paths, paths]
-    key = Misc.obj2digest(key_elems)
+    key = Misc.obj2digest(key_elems.inspect)
     self.sub!('~/', Etc.getpwuid.dir + '/') if self.include? "~"
 
     return @path[key] if @path[key]

data/lib/rbbt/util/log.rb

@@ -272,6 +272,31 @@ module Log
     end
   end
 
+  def self.count_stack
+    if ! $count_stacks
+      Log.debug "Counting stacks at: " << caller.first
+      return 
+    end
+    $stack_counts ||= {}
+    head = $count_stacks_head
+    stack = caller[1..head+1]
+    stack.reverse.each do |line,i|
+      $stack_counts[line] ||= 0
+      $stack_counts[line] += 1
+    end
+  end
+
+  def self.with_stack_counts(head = 10, total = 100)
+    $count_stacks_head = head
+    $count_stacks = true
+    $stack_counts = {}
+    res = yield
+    $count_stacks = false
+    Log.debug "STACK_COUNTS:\n" + $stack_counts.sort_by{|line,c| c}.reverse.collect{|line,c| [c, line] * " - "}[0..total] * "\n"
+    $stack_counts = {}
+    res
+  end
+
   case ENV['RBBT_LOG'] 
   when 'DEBUG' 
     self.severity = DEBUG

data/lib/rbbt/workflow.rb

@@ -9,6 +9,7 @@ require 'rbbt/workflow/archive'
 module Workflow
 
   STEP_CACHE = {}
+  LOAD_STEP_CACHE = {}
 
   class TaskNotFoundException < Exception 
     def initialize(workflow, task = nil)
@@ -192,6 +193,7 @@ module Workflow
   attr_accessor :task_dependencies, :task_description, :last_task 
   attr_accessor :stream_exports, :asynchronous_exports, :synchronous_exports, :exec_exports
   attr_accessor :step_cache
+  attr_accessor :load_step_cache
   attr_accessor :remote_tasks
 
   #{{{ ATTR DEFAULTS
@@ -230,6 +232,11 @@ module Workflow
     @step_cache ||= Workflow::STEP_CACHE
   end
 
+  def self.load_step_cache
+    @load_step_cache ||= Workflow::LOAD_STEP_CACHE
+  end
+
+
   def helpers
     @helpers ||= {}
   end
@@ -396,7 +403,11 @@ module Workflow
       Misc.insist do
         step.dependencies = step.info[:dependencies].collect do |task, job, path|
           next if job.nil?
-          load_step(path) 
+          if Open.exists?(path)
+            load_step(path) 
+          else
+            Workflow.load_step(path)
+          end
         end
       end
     end
@@ -420,21 +431,6 @@ module Workflow
     step.inputs ||= input_values
     step.dependencies = dependencies if dependencies and (step.dependencies.nil? or step.dependencies.length < dependencies.length)
 
-
-    step
-  end
-
-  def load_id(id)
-    path = if Path === workdir
-             workdir[id].find
-           else
-             File.join(workdir, id)
-           end
-    task = task_for path
-    return remote_tasks[task].load_id(id) if remote_tasks and remote_tasks.include? task
-    step = Step.new path, tasks[task.to_sym]
-    step.load_inputs_from_info
-    set_step_dependencies(step)
     step
   end
 
@@ -467,6 +463,21 @@ module Workflow
     File.join(sr - sl + so)
   end
 
+  def  self.relocate_array(real, list)
+    preal = real.split(/\/+/)
+    prefix = preal[0..-4] * "/" 
+    list.collect do |other|
+      pother = other.split(/\/+/)
+      end_part = pother[-3..-1] * "/"
+      new_path = prefix + "/" << end_part
+      if File.exists? new_path
+        new_path 
+      else
+        Rbbt.var.jobs[end_part].find
+      end
+    end
+  end
+
   def  self.relocate(real, other)
     preal = real.split(/\/+/)
     pother = other.split(/\/+/)
@@ -476,7 +487,8 @@ module Workflow
     Rbbt.var.jobs[end_part].find
   end
 
-  def self.load_step(path)
+
+  def self.__load_step(path)
     step = Step.new path
     relocated = false
     step.dependencies = (step.info[:dependencies] || []).collect do |task,name,dep_path|
@@ -485,14 +497,45 @@ module Workflow
       else
         new_path = relocate(path, dep_path)
         relocated = true if Open.exists?(new_path)
-        Workflow.load_step new_path
+        Workflow._load_step new_path
       end
     end
     step.relocated = relocated
+    step.load_inputs_from_info
 
     step
   end
 
+  def self._load_step(path)
+    Persist.memory("STEP", :path => path, :repo => load_step_cache) do
+      __load_step(path)
+    end
+  end
+
+  def self.load_step(path)
+    begin
+      _load_step(path)
+    ensure
+      load_step_cache.clear
+    end
+  end
+
+  def load_id(id)
+    path = if Path === workdir
+             workdir[id].find
+           else
+             File.join(workdir, id)
+           end
+    task = task_for path
+    return remote_tasks[task].load_id(id) if remote_tasks and remote_tasks.include? task
+    return Workflow.load_step path
+    #step = Step.new path, tasks[task.to_sym]
+    #step.load_inputs_from_info
+    #set_step_dependencies(step)
+    #step
+  end
+
+
   def load_name(task, name)
     return remote_tasks[task].load_step(path) if remote_tasks and remote_tasks.include? task
     task = tasks[task.to_sym] if String === task or Symbol === task

data/lib/rbbt/workflow/accessor.rb

@@ -652,8 +652,9 @@ module Workflow
     name = name.to_sym
     task = tasks[name]
     raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
-    @@task_info ||= {}
-    @@task_info[name] ||= begin description = task.description
+    id = File.join(self.to_s, name.to_s)
+    @task_info ||= {}
+    @task_info[id] ||= begin description = task.description
                            result_description = task.result_description
                            result_type = task.result_type
                            inputs = rec_inputs(name).uniq
@@ -675,9 +676,8 @@ module Workflow
                                       :none
                                     end
 
-
                            dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
-                           { :id => File.join(self.to_s, name.to_s),
+                           info = { :id => id,
                              :description => description,
                              :export => export,
                              :inputs => inputs,

data/lib/rbbt/workflow/step.rb

@@ -98,13 +98,20 @@ class Step
     else
       i = {}
     end
-    dependencies.each do |dep|
-      di = dep.recursive_inputs
-      next unless NamedArray === di
-      di.fields.zip(di).each do |k,v|
-        i[k] = v unless i.include? k
+    rec_dependencies.each do |dep|
+      next unless NamedArray === dep.inputs
+      dep.inputs.zip(dep.inputs.fields).each do |v,f|
+        next if i.include?(f)
+        i[f] = v
       end
     end
+    #dependencies.each do |dep|
+    #  di = dep.recursive_inputs
+    #  next unless NamedArray === di
+    #  di.fields.zip(di).each do |k,v|
+    #    i[k] = v unless i.include? k
+    #  end
+    #end
     v = i.values
     NamedArray.setup v, i.keys 
     v

data/lib/rbbt/workflow/step/dependencies.rb

@@ -302,25 +302,30 @@ class Step
   def canfail_paths
     return Set.new if done? && ! Open.exists?(info_file)
 
-    if !relocated && info[:canfail] 
-      Set.new(info[:canfail])
-    else
-      canfail_paths = Set.new
-      all_deps = dependencies
-      all_deps.each do |dep|
-        next if canfail_paths.include? dep.path
-        canfail_paths += dep.canfail_paths
-        next unless ComputeDependency === dep && dep.canfail?
-        canfail_paths << dep.path
-        canfail_paths += dep.rec_dependencies.collect{|d| d.path }
-      end
-      canfail_paths
-      begin
-        set_info :canfail, canfail_paths.to_a
-      rescue Errno::EROFS
-      end
-      canfail_paths
-    end
+    @canfail_paths ||= begin 
+                         if info[:canfail] 
+                           paths = info[:canfail].uniq
+                           paths = Workflow.relocate_array self.path, paths if relocated
+                           Set.new(paths)
+                         else
+                           canfail_paths = Set.new
+                           all_deps = dependencies
+                           all_deps.each do |dep|
+                             next if canfail_paths.include? dep.path
+                             canfail_paths += dep.canfail_paths
+                             next unless ComputeDependency === dep && dep.canfail?
+                             canfail_paths << dep.path
+                             iii dep.path
+                             canfail_paths += dep.rec_dependencies.collect{|d| d.path }
+                           end
+                           canfail_paths
+                           begin
+                             set_info :canfail, canfail_paths.to_a
+                           rescue Errno::EROFS
+                           end
+                           canfail_paths
+                         end
+                       end
   end
 
   def run_dependencies

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.31
+  version: 5.23.32
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-22 00:00:00.000000000 Z
+date: 2018-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake