rbbt-util 5.10.2 → 5.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cba82109ac8a97d8964ffd2b7eaa3e47444e6bea
-  data.tar.gz: ba62c2032f5225afb66da4d21cf2b92419993e5a
+  metadata.gz: ee7693bddd82af53509c8ed2e5414f1a86ac51ce
+  data.tar.gz: 5122a7d8c8404d7a045e571310cfb34c3e808bbe
 SHA512:
-  metadata.gz: 70cbfbc5301a70c101102221baca363898d075e55b12bb8fd06beb3a559bad40d541b95991a627a7f896635ac580aeb9f5e639bf4c49bdbc6e6328b0b75c1068
-  data.tar.gz: 87d924aa5560c5698adb671b9607c928ecee243aff6a5ec6a002b5b9c600117b138ac2ae289fab9a7889a1e350b0b3105b5e843704cd7e32b0a2f9110e279349
+  metadata.gz: e75a716cf8fd069b161d3f167217b0a329eac88d1458e6133fdbc0b3e51cfcd647e6c53cf2e4cbe9a28ab5499e2822138797688eb75239fffe458975195b6ad3
+  data.tar.gz: ae968a716ff89c35ba2a07300ac56cba42452692a75e2b811051d360fd64ce51c4f95aba36a9d974234450cdfdab846ddd31a367383a41ac80b1b49f177ac303

data/lib/rbbt/persist.rb

@@ -155,11 +155,7 @@ module Persist
           Misc.sensiblewrite(path, content * "\n" + "\n")
         end
       when IO
-        Misc.sensiblewrite(path) do |file|
-          while block = content.read(2048)
-            file.write block
-          end
-        end
+        Misc.sensiblewrite(path, content)
       else
         Misc.sensiblewrite(path, content.to_s)
       end
@@ -181,14 +177,20 @@ module Persist
 
     saver_pid = Process.fork do
       out.close
-      Misc.purge_pipes(stream)
+      stream.close
+      Misc.purge_pipes
       begin
         Misc.lock(path) do
           save_file(path, type, file)
         end
+      rescue Aborted
+        stream.abort if stream.respond_to? :abort
+        raise $!
       rescue Exception
         Log.exception $!
+        Kernel.exit! -1
       end
+      Kernel.exit! 0
     end
     file.close
     ConcurrentStream.setup(out, :pids => [saver_pid], :filename => path)
@@ -203,7 +205,11 @@ module Persist
         Misc.lock(path) do
           save_file(path, type, file)
         end
+      rescue Aborted
+        Log.error "Tee stream thread aborted"
+        stream.abort if stream.respond_to? :abort
       rescue Exception
+        stream.abort if stream.respond_to? :abort
         Log.exception $!
         parent.raise $!
       end
@@ -318,36 +324,70 @@ module Persist
         end
 
         begin
+
           lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
-          Misc.lock lock_filename  do
+          Misc.lock lock_filename  do |lockfile|
             if is_persisted?(path, persist_options)
               Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
               return path if persist_options[:no_load]
               return load_file(path, type) 
             end
 
-            Log.medium "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
+            Log.medium "Persist create: #{ path } - #{Misc.fingerprint persist_options}"
+
             res = yield
 
+            if persist_options[:no_load] == :stream
+              case res
+              when IO
+                res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
+                ConcurrentStream.setup res do
+                  lockfile.unlock
+                end
+                raise KeepLocked.new res 
+              when TSV::Dumper
+                res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
+                ConcurrentStream.setup res do
+                  lockfile.unlock
+                end
+                raise KeepLocked.new res 
+              end
+            end
+
             case res
-            when nil
-              res = load_file(path) unless persist_options[:no_load]
-            when IO, StringIO
-              res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
-              return res if persist_options[:no_load] == :stream
+            when IO
+              begin
+                res = case
+                      when :array
+                        res.read.split "\n"
+                      when :tsv
+                        TSV.open(res)
+                      else
+                        res.read
+                      end
+              rescue
+                res.abort if res.respond_to? :abort
+              ensure
+                res.join if res.respond_to? :join
+              end
             when TSV::Dumper
-              res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
-              return res if persist_options[:no_load] == :stream
-            else
-              Misc.lock(path) do
-                save_file(path, type, res)
+              begin
+                io = res.stream
+                res = TSV.open(io)
+              rescue
+                io.abort if io.respond_to? :abort
+              ensure
+                io.join if io.respond_to? :join
               end
             end
 
-            return path if persist_options[:no_load]
+            Misc.lock(path) do
+              save_file(path, type, res)
+            end
 
-            res
+            persist_options[:no_load] ? path : res
           end
+
         rescue
           Log.high "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
           FileUtils.rm path if Open.exists? path 

data/lib/rbbt/tsv/dumper.rb

@@ -7,8 +7,8 @@ module TSV
         begin
           yield dumper
           dumper.close
+        rescue Aborted
         rescue Exception
-          Log.exception $!
           parent.raise $!
         end
       end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -10,6 +10,19 @@ module TSV
     end
   end
 
+  def self.stream_name(obj)
+    filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
+    filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
+    stream_obj = obj_stream(obj)
+    filename_obj.nil? ? stream_obj.inspect : filename_obj + "(#{stream_obj.inspect})"
+  end
+
+  def self.report(msg, obj, into)
+    into = into[:into] if Hash === into and into.include? :into
+
+    Log.error "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
+  end
+
   def self.traverse_tsv(tsv, options = {}, &block)
     callback = Misc.process_options options, :callback
 
@@ -103,15 +116,21 @@ module TSV
         obj.traverse(options, &block)
       end
     when IO, File, StringIO
-      if options[:type] == :array
-        traverse_io_array(obj, options, &block)
-      else
-        traverse_io(obj, options, &block)
+      begin
+        if options[:type] == :array
+          traverse_io_array(obj, options, &block)
+        else
+          traverse_io(obj, options, &block)
+        end
+      rescue Exception
+        raise Aborted
+      ensure
+        begin
+          obj.close if obj.respond_to? :close and not obj.closed?
+        ensure
+          obj.join if obj.respond_to? :join
+        end
       end
-
-      io = obj
-      obj.join if io.respond_to? :join
-      io.close if io.respond_to? :close and not io.closed?
     when Path
       obj.open do |stream|
         traverse_obj(stream, options, &block)
@@ -120,12 +139,13 @@ module TSV
       traverse_obj(obj.stream, options, &block)
     when (defined? Step and Step)
 
-      case obj.result
-      when IO, TSV::Dumper, TSV
-        traverse_obj(obj.result, options, &block)
+      stream = obj.get_stream
+
+      if stream
+        traverse_obj(stream, options, &block)
       else
         obj.join
-        traverse_obj(obj.path.open, options, &block)
+        traverse_obj(obj.path, options, &block)
       end
     when Array
       traverse_array(obj, options, &block)
@@ -163,45 +183,55 @@ module TSV
   end
 
   def self.traverse_cpus(num, obj, options, &block)
-    filename = obj.respond_to?(:filename)? obj.filename : "none"
-    callback, cleanup = Misc.process_options options, :callback, :cleanup
-    q = RbbtProcessQueue.new num, cleanup
+    begin
+      filename = obj.respond_to?(:filename)? obj.filename : "none"
+      callback, cleanup = Misc.process_options options, :callback, :cleanup
+      q = RbbtProcessQueue.new num, cleanup
 
-    q.callback &callback
-    q.init &block
+      q.callback &callback
+      q.init &block
 
-    traverse_obj(obj, options) do |*p|
-      q.process *p
-    end
+      traverse_obj(obj, options) do |*p|
+        q.process *p
+      end
 
-    into = options[:into]
+      into = options[:into]
+    rescue Exception
+      q.abort
+      raise $!
+    ensure
+      q.join
+    end
 
-    q.join
   end
 
   def self.store_into(store, value)
-    case store
-    when Hash
-      return if value.nil?
-      if Hash === value
-        if TSV === store and store.type == :double
-          store.merge_zip value
+    begin
+      case store
+      when Hash
+        return if value.nil?
+        if Hash === value
+          if TSV === store and store.type == :double
+            store.merge_zip value
+          else
+            store.merge! value
+          end
         else
-          store.merge! value
+          k,v = value
+          store[k] = v
         end
+      when TSV::Dumper
+        return if value.nil?
+        store.add *value
+      when IO
+        return if value.nil?
+        store.puts value.strip
       else
-        k,v = value
-        store[k] = v
-      end
-    when TSV::Dumper
-      return if value.nil?
-      store.add *value
-    when IO
-      return if value.nil?
-      store.puts value.strip
-    else
-      store << value
-    end 
+        store << value
+      end 
+    rescue
+      raise "Error storing into #{store.inspect}: #{$!.message}"
+    end
   end
 
   def self.get_streams_to_close(obj)
@@ -268,30 +298,6 @@ module TSV
     ConcurrentStream.setup(obj_stream(into), :threads => thread)
   end
 
-  def self.stream_name(obj)
-    filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
-    filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
-    stream_obj = obj_stream(obj)
-    filename_obj.nil? ? stream_obj.inspect : filename_obj + "(#{stream_obj.inspect})"
-  end
-
-  def self.report(msg, obj, into)
-    into = into[:into] if Hash === into and into.include? :into
-
-    #filename_into = into.respond_to?(:filename) ? into.filename : nil
-    #filename_into ||= into.respond_to?(:path) ? into.path : nil
-    #stream_into = obj_stream(into)
-    #str_into = filename_into.nil? ? stream_into.inspect : filename_into + "(#{stream_into.inspect})"
-
-    #filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
-    #filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
-    #stream_obj = obj_stream(obj)
-    #str_obj = filename_obj.nil? ? stream_obj.inspect : filename_obj + "(#{stream_obj.inspect})"
-
-    #Log.error "#{ msg } #{filename_obj} - #{filename_into}"
-    Log.error "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
-  end
-
   def self.traverse(obj, options = {}, &block)
     threads = Misc.process_options options, :threads
     cpus = Misc.process_options options, :cpus
@@ -300,12 +306,24 @@ module TSV
     threads = nil if threads and threads.to_i <= 1
     cpus = nil if cpus and cpus.to_i <= 1
 
+    if into == :stream
+      sout = Misc.open_pipe false, false do |sin|                                                                                                                                           
+        begin
+          traverse(obj, options.merge(:into => sin), &block)                                                                                                                                  
+        rescue Exception
+          sin.abort if sin.respond_to? :abort
+        end
+      end                                                                                                                                                                                   
+      return sout
+    end
+
     if into
       options[:callback] = Proc.new do |e|
         begin
           store_into into, e
         rescue Exception
           Log.exception $!
+          raise $!
         end
       end
 

data/lib/rbbt/tsv/parser.rb

@@ -16,9 +16,9 @@ module TSV
 
       # Get line
 
-      Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
+      #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
       line = stream.gets
-      raise "Empty content" if line.nil?
+      raise "Empty content: #{ stream.inspect }" if line.nil?
       line = Misc.fixutf8 line
       line.chomp!
 
@@ -41,7 +41,7 @@ module TSV
         @key_field = @fields.shift
         @key_field = @key_field[(0 + header_hash.length)..-1] # Remove initial hash character
 
-        Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
+        #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
         line = @header_hash != "" ?  Misc.fixutf8(stream.gets) : nil
       end
 
@@ -123,14 +123,20 @@ module TSV
     end
 
     def get_values_flat(parts)
+      begin
+        orig = parts
       if key_position and key_position != 0 and field_positions.nil?
         value = parts.shift
         keys = parts.dup
         return [keys, [value]]
       end
 
-      return parts.shift.split(@sep2, -1).first, parts.collect{|value| value.split(@sep2, -1)}.flatten if 
+        return parts.shift.split(@sep2, -1).first, parts.collect{|value| value.split(@sep2, -1)}.flatten if 
         field_positions.nil? and (key_position.nil? or key_position == 0)
+      rescue
+        eee [:rescue, orig]
+        raise $!
+      end
 
       keys = parts[key_position].split(@sep2, -1)
 
@@ -486,6 +492,7 @@ module TSV
       # parser 
       line_num = 1
       begin
+
         while not line.nil? 
           begin
             progress_monitor.tick(stream.pos) if progress_monitor 
@@ -494,13 +501,14 @@ module TSV
 
             line = Misc.fixutf8(line)
             line = self.process line
+            raise SKIP_LINE if line.empty?
             parts = self.chop_line line
             key, values = self.get_values parts
             values = self.cast_values values if self.cast?
             
             yield key, values
 
-            Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
+            #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
 
             line = stream.gets
 
@@ -515,11 +523,18 @@ module TSV
             end
           rescue END_PARSING
             break
-          rescue IOError
-            Log.exception $!
-            break
+          #rescue IOError
+          #  Log.exception $!
+          #  break
+          rescue Exception
+            stream.abort if stream.respond_to? :abort
+            raise $!
           end
         end
+
+      ensure
+        stream.close
+        stream.join if stream.respond_to? :join
       end
 
       self

data/lib/rbbt/util/concurrency/processes.rb

@@ -27,10 +27,11 @@ class RbbtProcessQueue
             raise p.first if Array === p and Exception === p.first
             @callback.call p
           end
+        rescue Aborted
+          Log.error "Callback thread aborted"
         rescue ClosedStream
         rescue Exception
           Log.exception $!
-          sleep 1
           parent.raise $!
         ensure
           @callback_queue.sread.close unless @callback_queue.sread.closed?
@@ -53,14 +54,13 @@ class RbbtProcessQueue
           @processes[0].join 
           @processes.shift
         end
+      rescue Aborted
+        @processes.each{|p| p.abort }
+        Log.error "Process monitor aborted"
       rescue Exception
-        @processes.each do |p|
-          begin
-            Process.kill :INT, p
-          rescue
-          end
-        end
-        Log.exception $!
+        Log.error "Process monitor exception: #{$!.message}"
+        @processes.each{|p| p.abort }
+        @callback_thread.raise Aborted.new if @callback_thread
         parent.raise $!
       end
     end
@@ -87,8 +87,21 @@ class RbbtProcessQueue
   end
 
   def clean
-    @processes.each{|p| p.abort }
-    @callback_thread.raise Aborted if @callback_thread and @callback_thread.alive?
+    if @process_monitor.alive?
+     @process_monitor.raise Aborted.new
+     aborted = true
+    end
+
+    if @callback_thread and @callback_thread.alive?
+     @callback_thread.raise Aborted.new
+     aborted = true
+    end
+    raise Aborted.new if aborted
+  end
+
+  def abort
+    @process_monitor.raise Aborted.new if @process_monitor.alive?
+    @callback_thread.raise Aborted.new if @callback_thread and @callback_thread.alive?
   end
 
   def process(*e)