rbbt-util 5.20.4 → 5.20.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 940e4cab3bc56e4713e30e94d79ef0e087fe2e3e
-  data.tar.gz: 9faaf4ade7833732f581edee900cf9f81df53b41
+  metadata.gz: 732f48d509a3d1292ceecf5001206984b5681829
+  data.tar.gz: 54bad659a94eefc173f4c6d306f4bd78aca37ea9
 SHA512:
-  metadata.gz: 684c46a31c7050ac87ad83a9877d3051c5bbfac2558da5a9f71f806d01a646fff2126dc110f1f98b796a39b1873c49df6750a1465c53de2d296a631488cfc7c0
-  data.tar.gz: 1a55b1f2ff541b29448a85267ef635deb282e92e9c4f5511254df4b287d74f4ce62ee877eaf018c0a643cf3983798ba617947ac04fd141344071c19e9d9b196f
+  metadata.gz: b8ec9e618e757d8d308e2218c8dab1399bbe2dff8574d48063f07ce3391882b1fa95d78bbf84734b4e29122bda7a1868c650506e2ffe6c02e2e75e89de8254f2
+  data.tar.gz: 06c02c802611a555db90e5bf6402aaec68c3a49b85aa8de3032b30b6e1cb9cea8936b0ba033a52c55e40784b6c2aabfcb487a14908f628519f86c69019458e02

data/lib/rbbt/util/misc/pipes.rb

@@ -79,7 +79,6 @@ module Misc
           Log.medium "Aborted open_pipe: #{$!.message}"
         rescue Exception
           Log.medium "Exception in open_pipe: #{$!.message}"
-          Log.exception $!
           parent.raise $!
           raise $!
         end

data/lib/rbbt/workflow/accessor.rb

@@ -566,9 +566,9 @@ module Workflow
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject(IndiferentHash.setup({})){|acc, tn|
       if Array === tn and tn.first
-        new = tn.first.tasks[tn[1].to_sym].input_types
+        new = tn.first.tasks[tn[1].to_sym].input_defaults
       elsif Symbol === tn
-        new = tasks[tn.to_sym].input_types
+        new = tasks[tn.to_sym].input_defaults
       else
         next acc
       end
@@ -598,9 +598,9 @@ module Workflow
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
       if Array === tn and tn.first
-        new = tn.first.tasks[tn[1].to_sym].input_types
+        new = tn.first.tasks[tn[1].to_sym].input_descriptions
       elsif Symbol === tn
-        new = tasks[tn.to_sym].input_types
+        new = tasks[tn.to_sym].input_descriptions
       else
         next acc
       end
@@ -614,9 +614,9 @@ module Workflow
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
       if Array === tn and tn.first
-        new = tn.first.tasks[tn[1].to_sym].input_types
+        new = tn.first.tasks[tn[1].to_sym].input_options
       elsif Symbol === tn
-        new = tasks[tn.to_sym].input_types
+        new = tasks[tn.to_sym].input_options
       else
         next acc
       end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -1,73 +1,76 @@
 
 class Step
 
-  ##STREAM_CACHE = {}
-  ##STREAM_CACHE_MUTEX = Mutex.new
-  ##def self.purge_stream_cache
-  ##  Log.medium "Purging dup. stream cache"
-  ##  STREAM_CACHE_MUTEX.synchronize do
-  ##    #STREAM_CACHE.collect{|k,s| 
-  ##    #  Thread.new do
-  ##    #    Misc.consume_stream s
-  ##    #  end
-  ##    #}
-  ##    STREAM_CACHE.clear
-  ##  end
-  ##end
-
-  ##def self.dup_stream(stream)
-  ##  case stream
-  ##  when IO, File, Step
-  ##    return stream if stream.respond_to?(:closed?) and stream.closed?
-  ##    return stream if stream.respond_to?(:done?) and stream.done?
-
-  ##    STREAM_CACHE_MUTEX.synchronize do
-  ##      stream_key = Misc.fingerprint(stream)
-  ##      current = STREAM_CACHE[stream_key]
-  ##      case current
-  ##      when nil
-  ##        Log.medium "Not duplicating stream #{stream_key}"
-  ##        STREAM_CACHE[stream_key] = stream
-  ##      when File
-  ##        if Open.exists? current.path 
-  ##          Log.medium "Reopening file #{stream_key}"
-  ##          Open.open(current.path)
-  ##        else
-  ##          new = Misc.dup_stream(current)
-  ##          Log.medium "Duplicating file #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
-  ##          new
-  ##        end
-  ##      when Step
-  ##        job = current
-  ##        current = job.result 
-  ##        new = Misc.dup_stream(current)
-  ##        job.result = current
-  ##        Log.medium "Duplicating step #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
-  ##        new
-  ##      else
-  ##        new = Misc.dup_stream(current)
-  ##        Log.medium "Duplicating stream #{stream_key} #{ Misc.fingerprint(stream) } => #{Misc.fingerprint(new)}"
-  ##        new
-  ##      end
-  ##    end
-  ##  when TSV::Dumper#, TSV::Parser
-  ##    stream = stream.stream
-  ##    return stream if stream.closed?
-
-  ##    STREAM_CACHE_MUTEX.synchronize do
-  ##      if STREAM_CACHE[stream].nil?
-  ##        Log.high "Not duplicating dumper #{ stream.inspect }"
-  ##        STREAM_CACHE[stream] = stream
-  ##      else
-  ##        new = Misc.dup_stream(STREAM_CACHE[stream])
-  ##        Log.high "Duplicating dumper #{ stream.inspect } into #{new.inspect}"
-  ##        new
-  ##      end
-  ##    end
-  ##  else
-  ##    stream
-  ##  end
-  ##end
+  STREAM_CACHE = {}
+  STREAM_CACHE_MUTEX = Mutex.new
+  def self.purge_stream_cache
+    Log.medium "Purging dup. stream cache"
+    STREAM_CACHE_MUTEX.synchronize do
+      #STREAM_CACHE.collect{|k,s| 
+      #  Thread.new do
+      #    Misc.consume_stream s
+      #  end
+      #}
+      STREAM_CACHE.clear
+    end
+  end
+
+  def self.dup_stream(stream)
+    case stream
+    when IO, File, Step
+      return stream if stream.respond_to?(:closed?) and stream.closed?
+      return stream if stream.respond_to?(:done?) and stream.done?
+
+      STREAM_CACHE_MUTEX.synchronize do
+        stream_key = Misc.fingerprint(stream)
+        current = STREAM_CACHE[stream_key]
+        case current
+        when nil, Step
+          Log.medium "Not duplicating stream #{stream_key}"
+          STREAM_CACHE[stream_key] = stream
+        when File
+          if Open.exists? current.path 
+            Log.medium "Reopening file #{stream_key}"
+            Open.open(current.path)
+          else
+            new = Misc.dup_stream(current)
+            Log.medium "Duplicating file #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
+            new
+          end
+        else
+          new = Misc.dup_stream(current)
+          Log.medium "Duplicating stream #{stream_key} #{ Misc.fingerprint(stream) } => #{Misc.fingerprint(new)}"
+          new
+        end
+      end
+    when TSV::Dumper#, TSV::Parser
+      stream = stream.stream
+      return stream if stream.closed?
+
+      STREAM_CACHE_MUTEX.synchronize do
+        if STREAM_CACHE[stream].nil?
+          Log.high "Not duplicating dumper #{ stream.inspect }"
+          STREAM_CACHE[stream] = stream
+        else
+          new = Misc.dup_stream(STREAM_CACHE[stream])
+          Log.high "Duplicating dumper #{ stream.inspect } into #{new.inspect}"
+          new
+        end
+      end
+    else
+      stream
+    end
+  end
+
+  def dup_inputs
+    return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+    Log.low "Dupping inputs for #{path}"
+    dupped_inputs = @inputs.collect do |input|
+      Step.dup_stream input
+    end
+    @inputs.replace dupped_inputs
+    @dupped = true
+  end
 
   def self.prepare_for_execution(job)
     return if (job.done? and not job.dirty?) or 
@@ -76,6 +79,8 @@ class Step
 
     job.clean if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
 
+    job.dup_inputs
+
     raise DependencyError, job if job.error?
   end
 
@@ -152,27 +157,17 @@ class Step
     end
   end
 
-  #def dup_inputs
-  #  return if true or @dupped or ENV["RBBT_NO_STREAM"] == 'true'
-  #  Log.low "Dupping inputs for #{path}"
-  #  dupped_inputs = @inputs.collect do |input|
-  #    Step.dup_stream input
+  #def consolidate_dependencies(path_deps = {})
+  #  return false if @consolidated  or dependencies.nil? or dependencies.empty?
+  #  consolidated_deps = dependencies.collect do |dep|
+  #    dep.consolidate_dependencies(path_deps)
+  #    path = dep.path
+  #    path_deps[path] ||= dep
   #  end
-  #  @inputs.replace dupped_inputs
-  #  @dupped = true
+  #  dependencies.replace consolidated_deps
+  #  @consolidated = true
   #end
 
-  def consolidate_dependencies(path_deps = {})
-    return false if @consolidated  or dependencies.nil? or dependencies.empty?
-    consolidated_deps = dependencies.collect do |dep|
-      dep.consolidate_dependencies(path_deps)
-      path = dep.path
-      path_deps[path] ||= dep
-    end
-    dependencies.replace consolidated_deps
-    @consolidated = true
-  end
-
   #def prepare_dependencies
   #  dep_step = {}
 
@@ -210,9 +205,9 @@ class Step
   #end
 
   def execute_and_dup(step, dep_step, log = true)
-    dup = ! step.result
+    dup = step.result.nil?
     execute_dependency(step, log)
-    if dup and step.streaming? and step.result
+    if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
         other_steps = dep_step[step.path] - [step]
@@ -255,6 +250,10 @@ class Step
   def run_dependencies
     dep_step = {}
 
+    rec_dependencies = self.rec_dependencies
+
+    return if rec_dependencies.empty?
+
     all_deps = rec_dependencies + [self]
 
     dependencies.each do |dep|
@@ -282,6 +281,8 @@ class Step
       end
     end
 
+    self.dup_inputs
+
     required_dep_paths = []
     dep_step.each do |path,list|
       required_dep_paths << path if list.length > 1
@@ -289,6 +290,7 @@ class Step
 
     required_dep_paths.concat dependencies.collect{|dep| dep.path }
 
+
     log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
 
     pre_deps = []
@@ -317,20 +319,24 @@ class Step
       end
     end
 
+    Log.medium "Processing pre dependencies: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     pre_deps.each do |step|
       next if compute_deps.include? step
       execute_and_dup(step, dep_step, false)
     end
 
+    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     compute_pre_deps.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
 
+    Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     last_deps.each do |step|
       next if compute_deps.include? step
       execute_and_dup(step, dep_step)
     end
 
+    Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     compute_last_deps.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end

data/lib/rbbt/workflow/step/run.rb

@@ -20,19 +20,32 @@ class Step
 
   def resolve_input_steps
     step = false
+    pos = 0
     new_inputs = @inputs.collect do |i| 
-      if Step === i
-        step = true
-        if i.done?
-          i.load
-        elsif i.streaming?
-          TSV.get_stream i
+      begin
+        if Step === i
+          step = true
+          if i.done?
+            if (task.input_options[task.inputs[pos]] || {})[:stream]
+              TSV.get_stream i
+            else
+              i.load
+            end
+          elsif i.streaming?
+            TSV.get_stream i
+          else
+            i.join
+            if (task.input_options[task.inputs[pos]] || {})[:stream]
+              TSV.get_stream i
+            else
+              i.load
+            end
+          end
         else
-          i.join
-          i.load
+          i
         end
-      else
-        i
+      ensure
+        pos += 1
       end
     end
     @inputs.replace new_inputs if step
@@ -100,7 +113,7 @@ class Step
             :clean_name => clean_name,
           })
 
-          #dup_inputs
+          dup_inputs
           begin
             run_dependencies
           rescue Exception
@@ -161,7 +174,7 @@ class Step
                 Log.exception $!
               ensure
                 join
-                #Step.purge_stream_cache
+                Step.purge_stream_cache
                 FileUtils.rm pid_file if File.exists?(pid_file)
               end
             end
@@ -179,7 +192,7 @@ class Step
             set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
             set_info :time_elapsed, (time_elapsed = done_time - start_time)
             log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
-            #Step.purge_stream_cache
+            Step.purge_stream_cache
             FileUtils.rm pid_file if File.exists?(pid_file)
           end
 
@@ -190,7 +203,7 @@ class Step
           @result ||= result
           self
         else
-          #Step.purge_stream_cache
+          Step.purge_stream_cache
           @result = prepare_result result, @task.result_description
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.20.4
+  version: 5.20.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-30 00:00:00.000000000 Z
+date: 2016-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake