rbbt-util 5.12.3 → 5.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7e5eb68b9212f47cb33b251ea8d4d0f9c878df1
-  data.tar.gz: 39ecf6d22cde5921db4011ca849d3b5c03ba2f13
+  metadata.gz: 50b93d44cf66b527dc5cb3171ea799ad7618024f
+  data.tar.gz: f8647589d8b4c281074309db7509bd4b26e50aea
 SHA512:
-  metadata.gz: ca4f1549b877b62b6ae97d29b214c6bff957d626819ff14147f0eadaf63b9d394d3bb3e2db31c08f77329ed823d61afba9f44704dc975767a23f5f7fad7be793
-  data.tar.gz: 9e477fc7917df197279869c4bb4a754bf6e60be2d1695ca9f281f5406a1c51d29f311cebd8153430c8349abec29c529cac9e4481283387ef2929630cd5d9c914
+  metadata.gz: b04572a90e96cc1ac50f476df87025f459a0b36258de071179ae7f702da7799e17680a5ceb9d36ecf6e243a2008cd93f45ba7523756568cd99948580cc254356
+  data.tar.gz: ab57d3affae47af95efebbe2d2368418f4b9eae2b7dd8ab980d5d3dfaecd414dbdc0b5ce4615a6770b3d665877c4584ae13c73d7f308f878beb0979e0364d40d

data/lib/rbbt/persist.rb

@@ -111,7 +111,17 @@ module Persist
         res
       when :marshal
         Open.open(path) do |stream|
-          Marshal.load(stream)
+          case stream
+          when StringIO
+            begin
+              Marshal.load(stream) 
+            rescue
+              Log.exception $!
+              raise $!
+            end
+          else
+            Marshal.load(stream)
+          end
         end
       when :yaml
         Open.open(path) do |stream|
@@ -208,7 +218,7 @@ module Persist
   def self.tee_stream_thread(stream, path, type, callback = nil)
     file, out = Misc.tee_stream(stream)
 
-    saver_thread = Thread.new(Thread.current, path, file) do |parent,path,file|
+    saver_thread = Thread.new(Thread.current) do |parent|
       begin
         Thread.current["name"] = "file saver: " + path
         Misc.lock(path) do
@@ -216,11 +226,11 @@ module Persist
         end
       rescue Aborted
         Log.error "Persist stream thread aborted: #{ Log.color :blue, path }"
-        stream.abort if stream.respond_to? :abort
+        file.abort if file.respond_to? :abort
       rescue Exception
         Log.error "Persist stream thread exception: #{ Log.color :blue, path }"
         Log.exception $!
-        stream.abort if stream.respond_to? :abort
+        file.abort if file.respond_to? :abort
         parent.raise $!
       end
     end
@@ -290,8 +300,119 @@ module Persist
     alias tee_stream tee_stream_thread 
   end
 
+  def self.get_result(path, type, persist_options, lockfile, &block)
+    res = yield
+
+    if persist_options[:no_load] == :stream 
+      case res
+      when IO
+        res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
+        ConcurrentStream.setup res do
+          begin
+            lockfile.unlock if lockfile.locked?
+          rescue
+            Log.exception $!
+            Log.warn "Lockfile exception: " << $!.message
+          end
+        end
+        res.abort_callback = Proc.new do
+          begin
+            lockfile.unlock if lockfile.locked?
+          rescue
+            Log.exception $!
+            Log.warn "Lockfile exception: " << $!.message
+          end
+        end
+        raise KeepLocked.new res 
+      when TSV::Dumper
+        res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
+        ConcurrentStream.setup res do
+          begin
+            lockfile.unlock
+          rescue
+            Log.exception $!
+            Log.warn "Lockfile exception: " << $!.message
+          end
+        end
+        res.abort_callback = Proc.new do
+          begin
+            lockfile.unlock
+          rescue
+            Log.exception $!
+            Log.warn "Lockfile exception: " << $!.message
+          end
+        end
+        raise KeepLocked.new res 
+      end
+    end
+
+    case res
+    when IO
+      begin
+        res = case
+              when :array
+                res.read.split "\n"
+              when :tsv
+                TSV.open(res)
+              else
+                res.read
+              end
+        res.join if res.respond_to? :join
+      rescue
+        res.abort if res.respond_to? :abort
+        raise $!
+      end
+    when (defined? TSV and TSV::Dumper)
+      begin
+        io = res.stream
+        res = TSV.open(io)
+        io.join if io.respond_to? :join
+      rescue
+        io.abort if io.respond_to? :abort
+        raise $!
+      end
+    end
+    res
+  end
+
+  def self.persist_file(path, type, persist_options, &block)
+
+    if is_persisted?(path, persist_options)
+      Log.low "Persist up-to-date: #{ path } - #{Misc.fingerprint persist_options}"
+      return path if persist_options[:no_load]
+      return load_file(path, type) 
+    end
+
+    begin
+
+      lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
+      Misc.lock lock_filename do |lockfile|
+
+        if is_persisted?(path, persist_options)
+          Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
+          return path if persist_options[:no_load]
+          return load_file(path, type) 
+        end
+
+        Log.medium "Persist create: #{ path } - #{type} #{Misc.fingerprint persist_options}"
+
+        res = get_result(path, type, persist_options, lockfile, &block)
+
+        Misc.lock(path) do
+          save_file(path, type, res)
+        end
+
+        persist_options[:no_load] ? path : res
+      end
 
-  def self.persist(name, type = nil, persist_options = {})
+    rescue
+      Log.error "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
+      FileUtils.rm path if Open.exists? path 
+      raise $!
+    end
+  end
+
+  def self.persist(name, type = nil, persist_options = {}, &block)
     type ||= :marshal
 
     return (persist_options[:repo] || Persist::MEMORY)[persist_options[:file]] ||= yield if type ==:memory and persist_options[:file] and persist_options[:persist] and persist_options[:persist] != :update
@@ -388,110 +509,7 @@ module Persist
         end
 
       else
-
-        if is_persisted?(path, persist_options)
-          Log.low "Persist up-to-date: #{ path } - #{Misc.fingerprint persist_options}"
-          return path if persist_options[:no_load]
-          return load_file(path, type) 
-        end
-
-        begin
-
-          lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
-          Misc.lock lock_filename do |lockfile|
-
-            if is_persisted?(path, persist_options)
-              Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
-              return path if persist_options[:no_load]
-              return load_file(path, type) 
-            end
-
-            Log.medium "Persist create: #{ path } - #{Misc.fingerprint persist_options}"
-
-            res = yield
-
-            if persist_options[:no_load] == :stream 
-              case res
-              when IO
-                res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
-                ConcurrentStream.setup res do
-                  begin
-                    lockfile.unlock if lockfile.locked?
-                  rescue
-                    Log.exception $!
-                    Log.warn "Lockfile exception: " << $!.message
-                  end
-                end
-                res.abort_callback = Proc.new do
-                  begin
-                    lockfile.unlock if lockfile.locked?
-                  rescue
-                    Log.exception $!
-                    Log.warn "Lockfile exception: " << $!.message
-                  end
-                end
-                raise KeepLocked.new res 
-              when TSV::Dumper
-                res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
-                ConcurrentStream.setup res do
-                  begin
-                    lockfile.unlock
-                  rescue
-                    Log.exception $!
-                    Log.warn "Lockfile exception: " << $!.message
-                  end
-                end
-                res.abort_callback = Proc.new do
-                  begin
-                    lockfile.unlock
-                  rescue
-                    Log.exception $!
-                    Log.warn "Lockfile exception: " << $!.message
-                  end
-                end
-                raise KeepLocked.new res 
-              end
-            end
-
-            case res
-            when IO
-              begin
-                res = case
-                      when :array
-                        res.read.split "\n"
-                      when :tsv
-                        TSV.open(res)
-                      else
-                        res.read
-                      end
-                res.join if res.respond_to? :join
-              rescue
-                res.abort if res.respond_to? :abort
-                raise $!
-              end
-            when (defined? TSV and TSV::Dumper)
-              begin
-                io = res.stream
-                res = TSV.open(io)
-                io.join if io.respond_to? :join
-              rescue
-                io.abort if io.respond_to? :abort
-                raise $!
-              end
-            end
-
-            Misc.lock(path) do
-              save_file(path, type, res)
-            end
-
-            persist_options[:no_load] ? path : res
-          end
-
-        rescue
-          Log.error "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
-          FileUtils.rm path if Open.exists? path 
-          raise $!
-        end
+        persist_file(path, type, persist_options, &block)
       end
 
     end

data/lib/rbbt/tsv/dumper.rb

@@ -59,7 +59,6 @@ module TSV
         @in_stream << k << TSV::Dumper.values_to_s(v, @fields)
       rescue IOError
       rescue Exception
-        Log.exception $!
         raise $!
       end
     end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -13,8 +13,8 @@ module TSV
   def self.stream_name(obj)
     filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
     filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
-    stream_obj = obj_stream(obj)
-    filename_obj.nil? ? stream_obj.inspect : filename_obj + "(#{stream_obj.inspect})"
+    stream_obj = obj_stream(obj) || obj
+    filename_obj.nil? ? Misc.fingerprint(stream_obj) : filename_obj + "(#{Misc.fingerprint(stream_obj)})"
   end
 
   def self.report(msg, obj, into)
@@ -23,71 +23,101 @@ module TSV
     Log.error "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
   end
 
+  #{{{ TRAVERSE OBJECTS
+
   def self.traverse_tsv(tsv, options = {}, &block)
-    callback = Misc.process_options options, :callback
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
     if callback
       tsv.through options[:key_field], options[:fields] do |k,v|
-        callback.call yield(k,v)
+        begin
+          callback.call yield(k,v)
+        ensure
+          bar.tick if bar
+        end
       end
     else
       tsv.through options[:key_field], options[:fields] do |k,v|
-        yield k,v 
+        begin
+          yield k,v 
+        ensure
+          bar.tick if bar
+        end
       end
     end
+    join.call if join
   end
 
   def self.traverse_hash(hash, options = {}, &block)
-    callback = Misc.process_options options, :callback
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
     if callback
       hash.each do |k,v|
-        callback.call yield(k,v)
+        begin
+          callback.call yield(k,v)
+        ensure
+          bar.tick if bar
+        end
       end
     else
       hash.each do |k,v|
-        yield k,v 
+        begin
+          yield k,v 
+        ensure
+          bar.tick if bar
+        end
       end
     end
+    join.call if join
   end
 
   def self.traverse_array(array, options = {}, &block)
-    callback = Misc.process_options options, :callback
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
     if callback
       array.each do |e|
-        res = yield(e)
-        callback.call res
+        begin
+          callback.call yield(e)
+        ensure
+          bar.tick if bar
+        end
       end
     else
       array.each do |e|
-        yield e
+        begin
+          yield e
+        ensure
+          bar.tick if bar
+        end
       end
     end
+    join.call if join
   end
 
   def self.traverse_io_array(io, options = {}, &block)
-    callback = Misc.process_options options, :callback
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
     if callback
       while line = io.gets
-        res = yield line.strip
-        callback.call res
+        begin
+          callback.call yield line.strip
+        ensure
+          bar.tick if bar
+        end
       end
     else
       while line = io.gets
         yield line.strip
       end
     end
+    join.call if join
   end
 
   def self.traverse_io(io, options = {}, &block)
-    filename = io.filename if io.respond_to? :filename
-    callback = Misc.process_options options, :callback
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
     begin
       if callback
         TSV::Parser.traverse(io, options) do |k,v|
-          res = yield k, v
-          callback.call res
+          callback.call yield k, v
         end
       else
         TSV::Parser.traverse(io, options, &block)
@@ -96,10 +126,10 @@ module TSV
       Log.error "Traverse IO error"
       raise $!
     end
+    join.call if join
   end
 
   def self.traverse_obj(obj, options = {}, &block)
-    filename = obj.filename if obj.respond_to? :filename
     if options[:type] == :keys
       options[:fields] = []
       options[:type] = :single
@@ -114,8 +144,7 @@ module TSV
       callback = Misc.process_options options, :callback
       if callback
         obj.traverse(options) do |k,v|
-          res = yield k, v
-          callback.call res
+          callback.call yield k, v
         end
       else
         obj.traverse(options, &block)
@@ -195,9 +224,8 @@ module TSV
 
   def self.traverse_cpus(num, obj, options, &block)
     begin
-      filename = obj.respond_to?(:filename)? obj.filename : "none"
-      callback, cleanup = Misc.process_options options, :callback, :cleanup
-      q = RbbtProcessQueue.new num, cleanup
+      callback, cleanup, join = Misc.process_options options, :callback, :cleanup, :join
+      q = RbbtProcessQueue.new num, cleanup, join
 
       q.callback &callback
       q.init &block
@@ -209,8 +237,16 @@ module TSV
       end
 
       thread.join
+    rescue Interrupt, Aborted
+      Log.error "Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}"
+      stream = obj_stream(obj)
+      stream.abort if stream.respond_to? :abort
+      stream = obj_stream(options[:into])
+      stream.abort if stream.respond_to? :abort
+      q.abort
+      raise $!
     rescue Exception
-      Log.error "Exception traversing in cpus: #{$!.message}"
+      Log.error "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}"
       Log.exception $!
 
       stream = obj_stream(obj)
@@ -295,11 +331,13 @@ module TSV
         close_streams.concat(get_streams_to_close(obj))
         options[:close_streams] = close_streams
 
-        options[:cleanup] = Proc.new do
-          close_streams.uniq.each do |s|
-            s.close unless s.closed?
-          end
-        end if close_streams and close_streams.any?
+        if close_streams and close_streams.any?
+          options[:cleanup] = Proc.new do
+            close_streams.uniq.each do |s|
+              s.close unless s.closed?
+            end
+          end 
+        end
 
         traverse_cpus cpus, obj, options, &block
       end
@@ -323,13 +361,8 @@ module TSV
   end
 
   def self.traverse(obj, options = {}, &block)
-    threads = Misc.process_options options, :threads
-    cpus = Misc.process_options options, :cpus
     into = options[:into]
 
-    threads = nil if threads and threads.to_i <= 1
-    cpus = nil if cpus and cpus.to_i <= 1
-
     if into == :stream
       sout = Misc.open_pipe false, false do |sin|                                                                                                                                           
         begin
@@ -342,13 +375,45 @@ module TSV
       return sout
     end
 
+    threads = Misc.process_options options, :threads
+    cpus = Misc.process_options options, :cpus
+    threads = nil if threads and threads.to_i <= 1
+    cpus = nil if cpus and cpus.to_i <= 1
+
+    bar = Misc.process_options options, :bar
+    bar ||= Misc.process_options options, :progress
+    options[:bar] = case bar
+                    when String
+                      Log::ProgressBar.new_bar(nil, {:desc => bar}) 
+                    when TrueClass
+                      Log::ProgressBar.new_bar(nil, nil) 
+                    when Fixnum
+                      Log::ProgressBar.new_bar(bar) 
+                    when Hash
+                      max = Misc.process_options bar, :max
+                      Log::ProgressBar.new_bar(max, bar) 
+                    else
+                      bar
+                    end
+
     if into
+      bar = Misc.process_options options, :bar
+          
+      options[:join] = Proc.new do
+        Log::ProgressBar.remove_bar(bar)
+      end if bar
+
       options[:callback] = Proc.new do |e|
         begin
           store_into into, e
+        rescue Aborted
+          Log.error "Traversal info #{stream_name into} aborted"
+          raise $!
         rescue Exception
           Log.exception $!
           raise $!
+        ensure
+          bar.tick if bar
         end
       end