rbbt-util 5.13.11 → 5.13.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c06b4610f61069c0be43d2d108ed3ed5eb443b1c
-  data.tar.gz: 17eaf66b9a48b49dca2ceaf7b14d765d13b73b3c
+  metadata.gz: c918ffd6618192626d6874815f0f289813184f28
+  data.tar.gz: 7b49a48b985dfc7657adc68e0eb35b7432967280
 SHA512:
-  metadata.gz: 432cc55382487f8d5af6bba105595a3737259600842a850878ecd05ac56cd2c7cb995489c6ecb6a4deaf23218c730714a010c2e3a12037035a37cccce84345a6
-  data.tar.gz: a5f0d9d82a1dd2c717b716b4e997f19e74c4f8e76b6f32d2b1ff9ff107f25bc04f1f0fa5df13d15f295e9f18c244a3658e473842709f54f5b34cef96935869db
+  metadata.gz: 7334ed69cca371f85944bc6737c4b9e9911fe51e825b93b3ea8aeaf231318c9a3c6dea2d2f3d60dd8772cc5382392c3ac18070b9a53d0132081d3edc2f01055e
+  data.tar.gz: eed1a8230a41553d1c245509d44b04856a90bb0e7bf0452f4f3e1776d1ce3a8491bd51d42c6856950cbb54da1efbb4493d09a860fc7fcc4a377654db1f14201d

data/lib/rbbt/persist.rb

@@ -134,7 +134,7 @@ module Persist
         raise "Unknown persistence: #{ type }"
       end
     rescue
-      Log.warn "Exception loading #{ type } #{ path }: #{$!.message}"
+      Log.medium "Exception loading #{ type } #{ path }: #{$!.message}"
       raise $!
     end
   end
@@ -200,11 +200,11 @@ module Persist
           save_file(path, type, file)
         end
       rescue Aborted
-        Log.warn "Persist stream thread aborted: #{ Log.color :blue, path }"
+        Log.medium "Persist stream thread aborted: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
         raise $!
       rescue Exception
-        Log.warn "Persist stream thread exception: #{ Log.color :blue, path }"
+        Log.medium "Persist stream thread exception: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
         parent.raise $!
         raise $!
@@ -229,10 +229,10 @@ module Persist
             end
             Log.high "Stream pipe saved: #{path}"
           rescue Aborted
-            Log.warn "Persist stream pipe exception: #{ Log.color :blue, path }"
+            Log.medium "Persist stream pipe exception: #{ Log.color :blue, path }"
             stream.abort if stream.respond_to? :abort
           rescue Exception
-            Log.warn "Persist stream pipe exception: #{ Log.color :blue, path }"
+            Log.medium "Persist stream pipe exception: #{ Log.color :blue, path }"
             Log.exception $!
             stream.abort if stream.respond_to? :abort
             stream.join if stream.respond_to? :join
@@ -246,7 +246,7 @@ module Persist
               sin.write block
             end
           rescue Aborted
-            Log.warn "Tee stream thread aborted"
+            Log.medium "Tee stream thread aborted"
             sout.abort if sout.respond_to? :abort
             sin.abort if sin.respond_to? :abort
           rescue Exception
@@ -291,14 +291,14 @@ module Persist
           begin
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.warn "Lockfile exception: " << $!.message
+            Log.medium "Lockfile exception: " << $!.message
           end
         end
         res.abort_callback = Proc.new do
           begin
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.warn "Lockfile exception: " << $!.message
+            Log.medium "Lockfile exception: " << $!.message
           end
         end
         raise KeepLocked.new res 
@@ -310,7 +310,7 @@ module Persist
             stream.callback
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.warn "Lockfile exception: " << $!.message
+            Log.medium "Lockfile exception: " << $!.message
           end
         end
         res.abort_callback = Proc.new do
@@ -318,7 +318,7 @@ module Persist
             stream.abort
             lockfile.unlock if File.exists? lockfile.path and lockfile.locked?
           rescue
-            Log.warn "Lockfile exception: " << $!.message
+            Log.medium "Lockfile exception: " << $!.message
           end
         end
         raise KeepLocked.new res 
@@ -385,7 +385,7 @@ module Persist
       end
 
     rescue
-      Log.warn "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
+      Log.medium "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
       FileUtils.rm path if Open.exists? path 
       raise $!
     end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -235,28 +235,28 @@ module TSV
         raise "Unknown object for traversal: #{Misc.fingerprint obj }"
       end
     rescue IOError
-      Log.warn "IOError traversing #{stream_name(obj)}: #{$!.message}"
+      Log.medium "IOError traversing #{stream_name(obj)}: #{$!.message}"
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
       raise $!
     rescue Errno::EPIPE
-      Log.warn "Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"
+      Log.medium "Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
       raise $!
     rescue Aborted
-      Log.warn "Aborted traversing #{stream_name(obj)}"
+      Log.medium "Aborted traversing #{stream_name(obj)}"
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
-      Log.warn "Aborted traversing 2 #{stream_name(obj)}"
+      Log.medium "Aborted traversing 2 #{stream_name(obj)}"
     rescue Exception
-      Log.warn "Exception traversing #{stream_name(obj)}"
+      Log.medium "Exception traversing #{stream_name(obj)}"
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
@@ -304,7 +304,7 @@ module TSV
           q.process *p
         end
       rescue Aborted, Errno::EPIPE
-        Log.warn "Aborted"
+        Log.medium "Aborted"
       rescue Exception
         Log.exception $!
         raise $!
@@ -312,7 +312,7 @@ module TSV
         q.join
       end
     rescue Interrupt, Aborted
-      Log.warn "Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"
+      Log.medium "Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"
       q.abort
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort
@@ -320,7 +320,7 @@ module TSV
       stream.abort if stream.respond_to? :abort
       raise "Traversal aborted"
     rescue Exception
-      Log.warn "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.message}"
+      Log.medium "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.message}"
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
@@ -356,7 +356,7 @@ module TSV
       end 
       true
     rescue Aborted, Interrupt
-      Log.warn "Aborted storing into #{Misc.fingerprint store}: #{$!.message}"
+      Log.medium "Aborted storing into #{Misc.fingerprint store}: #{$!.message}"
       stream = obj_stream(store)
       stream.abort if stream.respond_to? :abort
     rescue Exception
@@ -484,14 +484,13 @@ module TSV
         begin
           store_into into, e
         rescue Aborted
-          Log.warn "Aborted callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
+          Log.medium "Aborted callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
           stream = nil
           stream = get_stream obj
           stream.abort if stream.respond_to? :abort
           raise $!
         rescue Exception
-          Log.warn "Exception callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
-          Log.exception $!
+          Log.medium "Exception callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
           stream = nil
           stream = get_stream obj
           stream.abort if stream.respond_to? :abort

data/lib/rbbt/util/log/progress/report.rb

@@ -9,6 +9,7 @@ module Log
     end
 
     def print(io, str)
+      return if ENV["RBBT_NO_PROGRESS"] == "true"
       LOG_MUTEX.synchronize do
         STDERR.print str
         Log.logfile.puts str unless Log.logfile.nil?

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -103,7 +103,7 @@ module ConcurrentStream
   end
 
   def abort
-    Log.warn "Aborting stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
+    Log.medium "Aborting stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
     return if @aborted
     @aborted = true
     begin

data/lib/rbbt/util/misc/pipes.rb

@@ -64,8 +64,11 @@ module Misc
         begin
           yield sin
           sin.close if close and not sin.closed?
+        rescue Aborted
+          Log.medium "Aborted open_pipe: #{$!.message}"
         rescue Exception
-          Log.warn "Exception in open_pipe: #{$!.message}"
+          Log.medium "Exception in open_pipe: #{$!.message}"
+          parent.raise $!
           raise $!
         end
       end
@@ -86,14 +89,14 @@ module Misc
           begin 
             stream_in1.write block; 
           rescue IOError
-            Log.warn("Tee stream 1 #{Misc.fingerprint stream} IOError: #{$!.message}");
+            Log.medium("Tee stream 1 #{Misc.fingerprint stream} IOError: #{$!.message}");
             skip1 = true
           end unless skip1 
 
           begin 
             stream_in2.write block
           rescue IOError
-            Log.warn("Tee stream 2 #{Misc.fingerprint stream} IOError: #{$!.message}");
+            Log.medium("Tee stream 2 #{Misc.fingerprint stream} IOError: #{$!.message}");
             skip2 = true
           end unless skip2 
         end
@@ -101,17 +104,17 @@ module Misc
         stream_in2.close unless stream_in2.closed?
         stream.join if stream.respond_to? :join
       rescue Aborted, Interrupt
-        Log.warn "Tee aborting #{Misc.fingerprint stream}"
+        Log.medium "Tee aborting #{Misc.fingerprint stream}"
         stream.abort if stream.respond_to? :abort
         stream_out1.abort if stream_out1.respond_to? :abort
         stream_out2.abort if stream_out2.respond_to? :abort
-        Log.warn "tee_stream_thread aborted: #{$!.message}"
+        Log.medium "tee_stream_thread aborted: #{$!.message}"
       rescue Exception
         stream.abort if stream.respond_to? :abort
         stream_out1.abort if stream_out1.respond_to? :abort
         stream_out2.abort if stream_out2.respond_to? :abort
         stream.join
-        Log.warn "Exception in tee_stream_thread: #{$!.message}"
+        Log.medium "Exception in tee_stream_thread: #{$!.message}"
         raise $!
       end
     end
@@ -153,21 +156,21 @@ module Misc
       return
     end
 
-    Log.medium "Consuming stream #{Misc.fingerprint io}"
     if in_thread
       Thread.new do
         consume_stream(io, false)
       end
     else
+      Log.medium "Consuming stream #{Misc.fingerprint io}"
       begin
         while block = io.read(2048)
         end
         io.join if io.respond_to? :join
       rescue Aborted
-        Log.warn "Consume stream aborted #{Misc.fingerprint io}"
+        Log.medium "Consume stream aborted #{Misc.fingerprint io}"
         io.abort if io.respond_to? :abort
       rescue Exception
-        Log.warn "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
+        Log.medium "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
         io.abort if io.respond_to? :abort
         io.join
         raise $!
@@ -219,11 +222,11 @@ module Misc
 
           Open.mv tmp_path, path
         rescue Aborted
-          Log.warn "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exists? path
         rescue Exception
-          Log.warn "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
+          Log.medium "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exists? path
           raise $!
@@ -400,7 +403,7 @@ module Misc
         save.abort if save.respond_to? :abort
         stream.abort if stream.respond_to? :abort
         stream.join
-        Log.warn "Exception in save_stream: #{$!.message}"
+        Log.medium "Exception in save_stream: #{$!.message}"
         raise $!
       end
     end

data/lib/rbbt/workflow/accessor.rb

@@ -119,25 +119,32 @@ class Step
     set_info(:messages, (messages || []) << message)
   end
 
+  def self.status_color(status)
+    status = status.split(">").last
+    case status
+    when "starting"
+      :yellow
+    when "error", "aborted"
+      :red
+    when "done"
+      :green
+    else
+      :cyan
+    end
+  end
+
   def self.log_block(staus, message, path, &block)
     start = Time.now
     status = status.to_s
-    status_color = case status
-                   when "starting"
-                     :yellow
-                   when "error"
-                     :red
-                   when "done"
-                     :green
-                   else
-                     :cyan
-                   end
+    status_color = self.status_color status
+
     Log.info do 
       now = Time.now
       str = Log.color :reset
       str << "#{ Log.color status_color, status}"
       str << ": #{ message }" if message
       str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
       str
     end
     res = yield
@@ -146,6 +153,7 @@ class Step
       now = Time.now
       str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.1g" % (eend - start)}"
       str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
       str
     end
     res
@@ -155,21 +163,13 @@ class Step
     Log.info do 
 
       status = status.to_s
-      status_color = case status
-                     when "starting"
-                       :yellow
-                     when "error"
-                       :red
-                     when "done"
-                       :green
-                     else
-                       :cyan
-                     end
+      status_color = self.status_color status
 
       str = Log.color :reset
       str << "#{ Log.color status_color, status}"
       str << ": #{ message }" if message
       str << " -- #{Log.color :blue, path.to_s}" if path
+      str << " #{Log.color :yellow, Process.pid}"
       str
     end
   end
@@ -206,6 +206,16 @@ class Step
     Step.log(status, message, path, &block)
   end
 
+  def exception(ex, msg = nil)
+    self._abort
+    set_info :backtrace, ex.backtrace
+    if msg.nil?
+      log :error, "Exception -- #{ex.message}"
+    else
+      log :error, "#{msg} -- #{ex.message}"
+    end
+  end
+
   def started?
     Open.exists? info_file or Open.exists? path
   end

data/lib/rbbt/workflow/step/run.rb

@@ -19,10 +19,13 @@ class Step
             Log.medium "Reopening file #{ Misc.fingerprint(current) }"
             Open.open(current.filename)
           else
-            Log.medium "Duplicating file #{ Misc.fingerprint(current) }"
-            new = current.dup
-            new.rewind
-            new
+            Log.medium "Duplicating file #{ Misc.fingerprint(current) } #{current.inspect}"
+            Misc.dup_stream(current)
+            #STREAM_CACHE[stream] = new
+            #new
+            #new = current.dup
+            #new.rewind
+            #new
           end
 
         else
@@ -149,22 +152,13 @@ class Step
           dependency.run(true) 
         end
       rescue Aborted
-        backtrace = $!.backtrace
-        set_info :backtrace, backtrace 
-        log(:error, "Aborted dependency #{Log.color :yellow, dependency.task.name.to_s}")
-        self.abort
+        Log.error "Aborted dep. #{Log.color :red, dependency.task.name.to_s}"
         raise $!
       rescue Interrupt
-        backtrace = $!.backtrace
-        set_info :backtrace, backtrace 
-        self.abort
-        log(:error, "Interrupted dependency #{Log.color :yellow, dependency.task.name.to_s}")
+        Log.error "Interrupted while in dep. #{Log.color :red, dependency.task.name.to_s}"
         raise $!
       rescue Exception
-        backtrace = $!.backtrace
-        set_info :backtrace, backtrace 
-        log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
-        self.abort
+        Log.error "Exception in dep. #{ Log.color :red, dependency.task.name.to_s }"
         raise $!
       end
     end
@@ -196,8 +190,7 @@ class Step
           dup_inputs
           begin
             run_dependencies
-          rescue
-            log(:error, "Error procesing dependencies")
+          rescue Exception
             stop_dependencies
             raise $!
           end
@@ -206,7 +199,7 @@ class Step
           set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
           set_info :started, (start_time = Time.now)
-          log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+          log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""}"
 
           begin
             result = _exec
@@ -234,12 +227,14 @@ class Step
           case result
           when IO
 
-            log :streaming, "#{Log.color :magenta, "Streaming IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+            log :streaming, "#{Log.color :magenta, "Streaming IO"} #{Log.color :yellow, task.name.to_s || ""}"
             ConcurrentStream.setup result do
               begin
-                set_info :done, (done_time = Time.now)
-                set_info :time_elapsed, (time_elapsed = done_time - start_time)
-                log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
+                if status != :done
+                  set_info :done, (done_time = Time.now)
+                  set_info :time_elapsed, (time_elapsed = done_time - start_time)
+                  log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
+                end
               rescue
                 Log.exception $!
               ensure
@@ -248,7 +243,7 @@ class Step
             end
             result.abort_callback = Proc.new do
               begin
-                log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}" if status == :streaming
+                log :aborted, "#{Log.color :red, "Aborted"} #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
               rescue
                 Log.exception $!
               ensure
@@ -256,12 +251,14 @@ class Step
               end
             end
           when TSV::Dumper
-            log :streaming, "#{Log.color :magenta, "Streaming TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+            log :streaming, "#{Log.color :magenta, "Streaming TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""}"
             ConcurrentStream.setup result.stream do
               begin
-                set_info :done, (done_time = Time.now)
-                set_info :time_elapsed, (time_elapsed = done_time - start_time)
-                log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
+                if status != :done
+                  set_info :done, (done_time = Time.now)
+                  set_info :time_elapsed, (time_elapsed = done_time - start_time)
+                  log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
+                end
               rescue
                 Log.exception $!
               ensure
@@ -270,7 +267,7 @@ class Step
             end
             result.stream.abort_callback = Proc.new do
               begin
-                log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"  if status == :streaming
+                log :aborted, "#{Log.color :red, "Aborted"} #{Log.color :yellow, task.name.to_s || ""}"  if status == :streaming
               rescue Exception
                 Log.exception $!
               end
@@ -278,7 +275,7 @@ class Step
           else
             set_info :done, (done_time = Time.now)
             set_info :time_elapsed, (time_elapsed = done_time - start_time)
-            log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+            log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
           end
 
           result
@@ -292,7 +289,7 @@ class Step
         end
       end
     rescue Exception
-      self.abort
+      exception $!
       raise $!
     end
   end
@@ -390,7 +387,7 @@ class Step
     end
   end
 
-  def abort
+  def _abort
     return if @aborted
     @aborted = true
     return if done?
@@ -406,30 +403,29 @@ class Step
       retry
     ensure
       if Open.exists? path
-        Log.warn "Aborted job had finished. Removing result"
+        Log.warn "Aborted job had finished. Removing result -- #{ path }"
         begin
           Open.rm path
         rescue Exception
           Log.warn "Exception removing result of aborted job: #{$!.message}"
         end
       end
-
-      begin
-        log(:aborted, "Job aborted")
-      rescue Exception
-        Log.exception $!
-      end
     end
     Log.medium{"#{Log.color :red, "Aborted"} #{Log.color :blue, path}"}
   end
 
+  def abort
+    _abort
+    log(:aborted, "Job aborted") unless aborted? or error?
+  end
+
   def join_stream
     stream = get_stream if @result
     if stream
       begin
         Misc.consume_stream stream
       rescue Exception
-        self.abort
+        self._abort
         raise $!
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.11
+  version: 5.13.12
 platform: ruby
 authors:
 - Miguel Vazquez