rbbt-util 5.13.3 → 5.13.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 715d410e93dddd02e14dd6d04f31917470c10b3d
-  data.tar.gz: 473271c8efe67c3465b204078d228618bedbc327
+  metadata.gz: 43ee6ce8a12a65ea0a42e62a3eef0a917c084051
+  data.tar.gz: 91fc982769f590d4f33738c87c459afeaa5dd7be
 SHA512:
-  metadata.gz: 3412998c1d2dce774a3d3a6a41428ab4506a6de2480b73be70890ae97d62be4285d1854881f6af92a551098c93e7b2dc62aaa8867a199e9ad26b3d991a213667
-  data.tar.gz: fb333f016809936f1ab2a928934683a3759a16d5ff3238e088535a7802f62566a732bfc7e7b0e3a7d638ebaa978b6dad6856ef2f3d5795f068213c5c14751e3a
+  metadata.gz: 18a0478c57dd34188715d0eede18d661739f862dcc40a303383254638a93c73ae4a2bf294bcfdf2d5480f895321a8c94eafdcecf0b1a2cbcddd510103ed8311c
+  data.tar.gz: 3a3e30b11a53ea72d7b76065a646a05faec9607c53046f69db78842be35072208e23b235121fe9e909646031b45ddcff4d6f4874e1158cb0b4f30367173868f1

data/bin/rbbt

@@ -13,6 +13,7 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 -cd--command_dir*       #{Log.color :yellow, "Directory from where to load command scripts"}
 --profile               #{Log.color :yellow, "Profile execution"}
 --nocolor               #{Log.color :yellow, "Disable colored output"}
+--nobar                 #{Log.color :yellow, "Disable progress report"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
 EOF
 
@@ -20,6 +21,10 @@ if options[:nocolor]
   Log.nocolor = true 
 end
 
+if options.delete :nobar
+  ENV["RBBT_NO_PROGRESS"] = "true"
+end
+
 locate = options.delete :locate_file
 
 if options[:log]

data/lib/rbbt/persist.rb

@@ -190,7 +190,7 @@ module Persist
     end
   end
 
-  def self.tee_stream_thread(stream, path, type, callback = nil)
+  def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil)
     file, out = Misc.tee_stream(stream)
 
     saver_thread = Thread.new(Thread.current) do |parent|
@@ -209,9 +209,12 @@ module Persist
       end
     end
     ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
+    out.callback = callback
+    out.abort_callback = abort_callback
+    out
   end
 
-  def self.tee_stream_pipe(stream, path, type, callback = nil)
+  def self.tee_stream_pipe(stream, path, type, callback = nil, abort_callback = nil)
     parent_pid = Process.pid
     out = Misc.open_pipe true, false do |sin|
       begin
@@ -280,12 +283,11 @@ module Persist
     if persist_options[:no_load] == :stream 
       case res
       when IO
-        res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
+        res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil, res.respond_to?(:abort_callback)? res.abort_callback : nil)
         ConcurrentStream.setup res do
           begin
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.exception $!
             Log.warn "Lockfile exception: " << $!.message
           end
         end
@@ -293,26 +295,26 @@ module Persist
           begin
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.exception $!
             Log.warn "Lockfile exception: " << $!.message
           end
         end
         raise KeepLocked.new res 
       when TSV::Dumper
-        res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
+        stream = res.stream
+        res = tee_stream(stream, path, type)
         ConcurrentStream.setup res do
           begin
+            stream.callback
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.exception $!
             Log.warn "Lockfile exception: " << $!.message
           end
         end
         res.abort_callback = Proc.new do
           begin
+            stream.abort
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.exception $!
             Log.warn "Lockfile exception: " << $!.message
           end
         end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -1,6 +1,8 @@
 module TSV
   def self.obj_stream(obj)
     case obj
+    when Step
+      obj.result
     when IO, File
       obj
     when TSV::Dumper
@@ -252,6 +254,7 @@ module TSV
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
+      Log.warn "Aborted traversing 2 #{stream_name(obj)}"
     rescue Exception
       Log.warn "Exception traversing #{stream_name(obj)}"
       stream = obj_stream(obj)
@@ -359,7 +362,6 @@ module TSV
     rescue Exception
       stream = obj_stream(store)
       stream.abort if stream.respond_to? :abort
-      Log.exception $!
       raise $!
     end
   end
@@ -423,7 +425,7 @@ module TSV
       begin
         traverse_run(obj, threads, cpus, options, &block)
         into.close if into.respond_to? :close
-      rescue
+      rescue Exception
         stream = obj_stream(obj)
         stream.abort if stream and stream.respond_to? :abort
         stream = obj_stream(into)
@@ -483,9 +485,17 @@ module TSV
           store_into into, e
         rescue Aborted
           Log.warn "Aborted callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
+          stream = nil
+          stream = get_stream obj
+          stream.abort if stream.respond_to? :abort
+          raise $!
         rescue Exception
           Log.warn "Exception callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
           Log.exception $!
+          stream = nil
+          stream = get_stream obj
+          stream.abort if stream.respond_to? :abort
+          raise $!
         ensure
           bar.tick if bar
         end

data/lib/rbbt/tsv/stream.rb

@@ -149,11 +149,12 @@ module TSV
           stream.join if stream.respond_to? :join
         end
       rescue Exception
-        streams.each do |stream|
+        ts = streams.collect do |stream|
           Thread.new do
             stream.abort if stream.respond_to? :abort
           end
         end
+        ts.each do |t| t.join end
         raise $!
       end
     end

data/lib/rbbt/util/log/progress.rb

@@ -25,6 +25,7 @@ module Log
 
     # Used to register a new completed loop iteration.
     def tick(step = nil)
+      return if ENV["RBBT_NO_PROGRESS"] == 'true'
 
       if step.nil?
         @current += 1
@@ -163,6 +164,8 @@ module Log
         Log::LAST.replace "progress"
         length = Log::ProgressBar.cleanup_bars
         length.times{print(io, "\n")}
+      else
+        print(io, "\n") if @last_report == -1
       end
       print(io, up_lines(@depth) << report_msg << down_lines(@depth)) if severity >= Log.severity
       @last_report = Time.now if @last_report == -1
@@ -173,6 +176,8 @@ module Log
         Log::LAST.replace "progress"
         length = Log::ProgressBar.cleanup_bars 
         length.times{print(io, "\n")}
+      else
+        print(io, "\n") if @last_report == -1
       end
       print(io, up_lines(@depth) << throughput_msg << down_lines(@depth)) if severity >= Log.severity
       @last_report = Time.now
@@ -217,10 +222,9 @@ module Log
     end
 
     def self.remove_bar(bar)
-      bar.done unless bar.max
+      bar.done unless bar.max or ENV["RBBT_NO_PROGRESS"] == 'true'
       BAR_MUTEX.synchronize do
         REMOVE << bar
-        Log::LAST.replace "remove"
       end
     end
 

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -98,18 +98,21 @@ module ConcurrentStream
 
   def abort_pids
     @pids.each{|pid| Process.kill :INT, pid } if @pids
+    @pids = []
   end
 
   def abort
+    Log.warn "Aborting stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
     return if @aborted
+    @aborted = true
     begin
-      abort_threads
-      abort_pids
+      @callback = nil
       @abort_callback.call if @abort_callback
-    ensure
       @abort_callback = nil
-      @callback = nil
-      @aborted = true
+      close unless closed?
+
+      abort_threads
+      abort_pids
     end
   end
 

data/lib/rbbt/util/misc/lock.rb

@@ -49,7 +49,6 @@ module Misc
     ensure
       if unlock 
         lockfile.unlock if lockfile.locked?
-        FileUtils.rm lock_path if File.exists? lock_path
       end
     end
 

data/lib/rbbt/util/misc/objects.rb

@@ -7,7 +7,7 @@ module Misc
 
     dup_array = options.delete :dup_array
 
-    if Annotated === field or Entity.respond_to?(:formats) and Entity.formats.include? field
+    if Annotated === field or (Entity.respond_to?(:formats) and Entity.formats.include? field)
       params = options.dup
 
       params[:format] ||= params.delete "format"

data/lib/rbbt/util/misc/pipes.rb

@@ -100,20 +100,26 @@ module Misc
         stream_in2.close unless stream_in2.closed?
         stream.join if stream.respond_to? :join
       rescue Aborted, Interrupt
-        begin
-          stream.abort if stream.respond_to? :abort
-        rescue
-          Log.exception $!
-        end
+        Log.warn "Tee aborting #{Misc.fingerprint stream}"
+        stream.abort if stream.respond_to? :abort
+        stream_out1.abort if stream_out1.respond_to? :abort
+        stream_out2.abort if stream_out2.respond_to? :abort
+        parent.raise $!
       rescue Exception
-        Log.exception $!
         stream.abort if stream.respond_to? :abort
+        parent.raise $!
       end
     end
 
     ConcurrentStream.setup stream_out1, :threads => splitter_thread
     ConcurrentStream.setup stream_out2, :threads => splitter_thread
 
+    stream_out1.callback = stream.callback if stream.respond_to? :callback
+    stream_out1.abort_callback = stream.abort_callback if stream.respond_to? :abort_callback
+
+    stream_out2.callback = stream.callback if stream.respond_to? :callback
+    stream_out2.abort_callback = stream.abort_callback if stream.respond_to? :abort_callback
+
     [stream_out1, stream_out2]
   end
 

data/lib/rbbt/workflow.rb

@@ -188,7 +188,6 @@ module Workflow
   def step_cache
     @step_cache ||= Workflow::STEP_CACHE
   end
-  
 
   def helpers
     @helpers ||= {}
@@ -237,6 +236,7 @@ module Workflow
   def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
     step_path = step_path.call if Proc === step_path
     persist = input_values.nil? ? false : true
+    #persist = false
     key = Path === step_path ? step_path.find : step_path
     step = Persist.memory("Step", :key => key, :repo => step_cache, :persist => persist) do
       step = Step.new step_path, task, input_values, dependencies

data/lib/rbbt/workflow/accessor.rb

@@ -217,7 +217,7 @@ class Step
   end
 
   def aborted?
-    info[:status] == :aborted
+    @aborted || info[:status] == :aborted
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/step.rb

@@ -8,7 +8,7 @@ class Step
   attr_accessor :path, :task, :inputs, :dependencies, :bindings
   attr_accessor :pid
   attr_accessor :exec
-  attr_accessor :result, :mutex
+  attr_accessor :result, :mutex, :seen
 
   class << self
     attr_accessor :lock_dir

data/lib/rbbt/workflow/step/run.rb

@@ -66,13 +66,12 @@ class Step
   def get_stream
     @mutex.synchronize do
       begin
+        return nil if @saved_stream
         if IO === @result 
           @saved_stream = @result 
         else 
           nil
         end
-      ensure
-        @result = nil
       end
     end
   end
@@ -118,29 +117,33 @@ class Step
     end
   end
 
-  def run_dependencies(seen = [])
+  def run_dependencies
+    @seen ||= []
     dependencies.uniq.each do |dependency| 
-
       next if seen.collect{|d| d.path}.include?(dependency.path)
-      seen << dependency
-      seen.concat dependency.rec_dependencies.collect{|d| d } 
+      dependency.seen = seen
+      @seen << dependency
+      @seen.concat dependency.rec_dependencies.collect{|d| d } 
+      @seen.uniq!
     end
 
-    seen.each do |dependency|
+    @seen.each do |dependency|
       next if dependency == self
+      next unless dependencies.include? dependency
       dependency.relay_log self
       dependency.dup_inputs
     end
 
-    seen.each do |dependency| 
+    @seen.each do |dependency| 
       next if dependency == self
+      next unless dependencies.include? dependency
       Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} -- #{Log.color :blue, dependency.path}"
       begin
         if dependency.streaming? 
           next if dependency.running?
           dependency.clean 
         else
-          dependency.clean if not dependency.done? and (dependency.error? or dependency.aborted?)
+          dependency.clean if (dependency.error? or dependency.aborted? or not dependency.done?)
         end
 
         unless dependency.result or dependency.done?
@@ -155,8 +158,8 @@ class Step
       rescue Interrupt
         backtrace = $!.backtrace
         set_info :backtrace, backtrace 
-        log(:error, "Interrupted dependency #{Log.color :yellow, dependency.task.name.to_s}")
         self.abort
+        log(:error, "Interrupted dependency #{Log.color :yellow, dependency.task.name.to_s}")
         raise $!
       rescue Exception
         backtrace = $!.backtrace
@@ -171,8 +174,8 @@ class Step
   def run(no_load = false)
     result = nil
 
-    @mutex.synchronize do
-      begin
+    begin
+      @mutex.synchronize do
         no_load = no_load ? :stream : false
         result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
           if Step === Step.log_relay_step and not self == Step.log_relay_step
@@ -244,8 +247,6 @@ class Step
             result.abort_callback = Proc.new do
               begin
                 log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}" if status == :streaming
-                stop_dependencies
-                abort_stream
               rescue
                 Log.exception $!
               ensure
@@ -267,9 +268,9 @@ class Step
             end
             result.stream.abort_callback = Proc.new do
               begin
-                stop_dependencies
                 log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"  if status == :streaming
               rescue Exception
+                Log.exception $!
               end
             end
           else
@@ -287,9 +288,10 @@ class Step
         else
           @result = prepare_result result, @task.result_description
         end
-      rescue Exception
-        self.abort
       end
+    rescue Exception
+      self.abort
+      raise $!
     end
   end
 
@@ -343,13 +345,7 @@ class Step
 
   def stop_dependencies
     dependencies.each do |dep|
-      Log.warn "Stoping #{Misc.fingerprint dep}"
-      begin
-        dep.abort unless dep.done?
-      rescue Exception
-        Log.exception $!
-      rescue Aborted
-      end
+      dep.abort
     end
     kill_children
   end
@@ -379,11 +375,11 @@ class Step
   def abort_stream
     stream = get_stream if @result
     stream ||= @saved_stream 
+    @saved_stream = nil
     if stream and stream.respond_to? :abort and not stream.aborted?
       begin
         Log.medium "Aborting job stream #{stream.inspect} -- #{Log.color :blue, path}"
         stream.abort 
-        Log.medium "Aborted job stream #{stream.inspect} -- #{Log.color :blue, path}"
         #stream.close unless stream.closed?
       rescue Aborted
         Log.medium "Aborting job stream #{stream.inspect} ABORTED RETRY -- #{Log.color :blue, path}"
@@ -394,19 +390,29 @@ class Step
 
   def abort
     return if @aborted
+    @aborted = true
+    return if error?
+    return if done?
     Log.medium{"#{Log.color :red, "Aborting"} #{Log.color :blue, path}"}
     begin
-      stop_dependencies
-      @aborted = true
       abort_stream
       abort_pid
+      stop_dependencies
     rescue Aborted
       Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
       retry
     rescue Exception
-      Log.exception $!
+      retry
     ensure
-      @aborted = true
+      if Open.exists? path
+        Log.warn "Aborted job had finished. Removing result"
+        begin
+          Open.rm path
+        rescue Exception
+          Log.warn "Exception removing result of aborted job: #{$!.message}"
+        end
+      end
+
       begin
         log(:aborted, "Job aborted")
       rescue Exception

data/share/rbbt_commands/workflow/task

@@ -302,6 +302,7 @@ begin
   end
 
   if do_fork
+    ENV["RBBT_NO_PROGRESS"] = "true"
     if detach
       job.fork
       Process.detach job.pid if job.pid

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.3
+  version: 5.13.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-29 00:00:00.000000000 Z
+date: 2014-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake