rbbt-util 5.10.1 → 5.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5de94d0ec4441889212dc37780f880b38809d71
-  data.tar.gz: 44acad91d91a0a8f7c351d32cd2dba45ca9fc058
+  metadata.gz: cba82109ac8a97d8964ffd2b7eaa3e47444e6bea
+  data.tar.gz: ba62c2032f5225afb66da4d21cf2b92419993e5a
 SHA512:
-  metadata.gz: 1fc109e80a9808fe201f7aa683fd2c0d9a8c0f35e6a1b2f3f9a0ee5f1f62801f192497c51ce09e69e5a721cb301d564227e543c44f2263aaf1faa3da14215c8c
-  data.tar.gz: 78d53fa4a1bc35af227a4d15553204a31405e89459e2b51a60cad762eba536b0e73f0719e51058ca8df03f328ab6117986f6109a2e01c531bb12c3f0b8453576
+  metadata.gz: 70cbfbc5301a70c101102221baca363898d075e55b12bb8fd06beb3a559bad40d541b95991a627a7f896635ac580aeb9f5e639bf4c49bdbc6e6328b0b75c1068
+  data.tar.gz: 87d924aa5560c5698adb671b9607c928ecee243aff6a5ec6a002b5b9c600117b138ac2ae289fab9a7889a1e350b0b3105b5e843704cd7e32b0a2f9110e279349

data/lib/rbbt/persist.rb

@@ -176,72 +176,58 @@ module Persist
     end
   end
 
-  def self.tee_stream(stream, path, type, callback = nil)
-    file_out, file_in = IO.pipe
-    stream_out, stream_in = IO.pipe
+  def self.tee_stream_fork(stream, path, type, callback = nil)
+    file, out = Misc.tee_stream(stream)
 
-    saver_thread = Thread.new(Thread.current) do |parent|
+    saver_pid = Process.fork do
+      out.close
+      Misc.purge_pipes(stream)
       begin
         Misc.lock(path) do
-          save_file(path, type, file_out)
+          save_file(path, type, file)
         end
       rescue Exception
         Log.exception $!
-        parent.raise $!
       end
     end
+    file.close
+    ConcurrentStream.setup(out, :pids => [saver_pid], :filename => path)
+  end
+
+  def self.tee_stream_thread(stream, path, type, callback = nil)
+    file, out = Misc.tee_stream(stream)
 
-    splitter_thread = Thread.new(Thread.current) do |parent|
+    saver_thread = Thread.new(Thread.current, path, file) do |parent,path,file|
       begin
-        while block = stream.read(2048)
-          begin stream_in.write block; rescue Exception; Log.exception $! end
-        begin file_in.write block; rescue Exception;  Log.exception $! end
+        Thread.current["name"] = "file saver: " + path
+        Misc.lock(path) do
+          save_file(path, type, file)
         end
-        file_in.close
-        stream_in.close
-        callback.call if callback
       rescue Exception
         Log.exception $!
         parent.raise $!
       end
     end
+    ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
+  end
 
-    class << stream_out
-      attr_accessor :threads
-
-      def join
-        @threads.each{|t| t.join }
-        @threads = []
-      end
-
-      def close
-        join
-        super
-      end
-
-      def read(*args)
-        res = super(*args)
-        join if eof?
-        res
-      end
-    end
-
-    stream_out.threads = [splitter_thread, saver_thread]
-
-    stream_out
+  class << self
+    alias tee_stream tee_stream_thread 
   end
 
   def self.persist(name, type = nil, persist_options = {})
     type ||= :marshal
-    persist_options = Misc.add_defaults persist_options, :persist => true
 
-    if persist_options[:persist]
+    return (persist_options[:repo] || Persist::MEMORY)[persist_options[:file]] ||= yield if type ==:memory and persist_options[:file]
+
+    if FalseClass != persist_options[:persist]
       other_options = Misc.process_options persist_options, :other
       path = persistence_path(name, persist_options, other_options || {})
 
       case 
       when type.to_sym === :memory
-        Persist::MEMORY[path] ||= yield
+        repo = persist_options[:repo] || Persist::MEMORY
+        repo[path] ||= yield
 
       when (type.to_sym == :annotations and persist_options.include? :annotation_repo)
 
@@ -346,10 +332,12 @@ module Persist
             case res
             when nil
               res = load_file(path) unless persist_options[:no_load]
-            when IO
+            when IO, StringIO
               res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
+              return res if persist_options[:no_load] == :stream
             when TSV::Dumper
               res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
+              return res if persist_options[:no_load] == :stream
             else
               Misc.lock(path) do
                 save_file(path, type, res)
@@ -373,11 +361,17 @@ module Persist
   end
 
   def self.memory(name, options = {}, &block)
-    file = name
-    file << "_" << Misc.hash2md5(options) if options.any?
-    options = Misc.add_defaults options, :persist => true, :file => file
-
-    persist name, :memory, options, &block
+    case options
+    when nil
+      persist name, :memory, :file => name, &block
+    when String
+      persist name, :memory, :file => name + "_" << options, &block
+    else
+      file = name
+      repo = options.delete :repo if options and options.any?
+      file << "_" << (options[:key] ? options[:key] : Misc.hash2md5(options)) if options and options.any?
+      persist name, :memory, options.merge(:repo => repo, :persist => true, :file => file), &block
+    end
   end
 end
 

data/lib/rbbt/resource/path.rb

@@ -196,6 +196,12 @@ module Path
     TSV.open(self.produce, *args)
   end
 
+  def tsv_options(options = {})
+    self.open do |stream|
+      TSV::Parser.new(stream, options).options
+    end
+  end
+
   def traverse(options = {}, &block)
     TSV::Parser.traverse(self.open, options, &block)
   end

data/lib/rbbt/tsv/dumper.rb

@@ -3,9 +3,14 @@ module TSV
     attr_accessor :in_stream, :stream, :options, :filename
     def self.stream(options = {}, filename = nil, &block)
       dumper = TSV::Dumper.new options, filename
-      Thread.new do
-        yield dumper
-        dumper.close
+      Thread.new(Thread.current) do |parent|
+        begin
+          yield dumper
+          dumper.close
+        rescue Exception
+          Log.exception $!
+          parent.raise $!
+        end
       end
       dumper.stream
     end
@@ -19,7 +24,7 @@ module TSV
         @filename = filename
       end
       @filename ||= Misc.fingerprint options
-      @stream, @in_stream = IO.pipe
+      @stream, @in_stream = Misc.pipe
     end
 
     def self.values_to_s(values, fields = nil)
@@ -42,15 +47,33 @@ module TSV
       key_field, fields = Misc.process_options options, :key_field, :fields
 
       str = TSV.header_lines(key_field, fields, options)
+
+      Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
       @in_stream.puts str
     end
 
     def add(k,v)
-      @in_stream << k << TSV::Dumper.values_to_s(v, @options[:fields])
+      @fields ||= @options[:fields]
+      begin
+        Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
+        @in_stream << k << TSV::Dumper.values_to_s(v, @fields)
+      rescue IOError
+      rescue Exception
+        Log.exception $!
+      end
+    end
+
+    def close_out
+      Log.debug "Close out #{@stream.inspect}"
+      @stream.close
+    end
+
+    def close_in
+      @in_stream.close unless @in_stream.closed?
     end
 
     def close
-      @in_stream.close
+      close_in
     end
   end
 end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -1,4 +1,14 @@
 module TSV
+  def self.obj_stream(obj)
+    case obj
+    when IO, File
+      obj
+    when TSV::Dumper
+      obj.stream
+    when TSV::Parser
+      obj.stream
+    end
+  end
 
   def self.traverse_tsv(tsv, options = {}, &block)
     callback = Misc.process_options options, :callback
@@ -43,7 +53,35 @@ module TSV
     end
   end
 
+  def self.traverse_io_array(io, options = {}, &block)
+    callback = Misc.process_options options, :callback
+    if callback
+      while not io.eof?
+        res = yield io.gets.strip
+        callback.call res
+      end
+    else
+      while line = io.gets
+        yield line.strip
+      end
+    end
+  end
+
+  def self.traverse_io(io, options = {}, &block)
+    filename = io.filename if io.respond_to? :filename
+    callback = Misc.process_options options, :callback
+    if callback
+      TSV::Parser.traverse(io, options) do |k,v|
+        res = yield k, v
+        callback.call res
+      end
+    else
+      TSV::Parser.traverse(io, options, &block)
+    end
+  end
+
   def self.traverse_obj(obj, options = {}, &block)
+    filename = obj.filename if obj.respond_to? :filename
     if options[:type] == :keys
       options[:fields] = []
       options[:type] = :single
@@ -64,38 +102,27 @@ module TSV
       else
         obj.traverse(options, &block)
       end
-    when (options[:type] == :array and IO)
-      callback = Misc.process_options options, :callback
-      if callback
-        while not obj.eof?
-          res = yield obj.gets.strip
-          callback.call res
-        end
+    when IO, File, StringIO
+      if options[:type] == :array
+        traverse_io_array(obj, options, &block)
       else
-        while not obj.eof?
-          yield obj.gets.strip
-        end
-      end
-    when IO, File
-      callback = Misc.process_options options, :callback
-      if callback
-        TSV::Parser.traverse(obj, options) do |k,v|
-          res = yield k, v
-          callback.call res
-        end
-      else
-        TSV::Parser.traverse(obj, options, &block)
+        traverse_io(obj, options, &block)
       end
+
+      io = obj
+      obj.join if io.respond_to? :join
+      io.close if io.respond_to? :close and not io.closed?
     when Path
       obj.open do |stream|
         traverse_obj(stream, options, &block)
       end
+    when TSV::Dumper
+      traverse_obj(obj.stream, options, &block)
     when (defined? Step and Step)
+
       case obj.result
-      when IO
+      when IO, TSV::Dumper, TSV
         traverse_obj(obj.result, options, &block)
-      when TSV::Dumper
-        traverse_obj(obj.stream, options, &block)
       else
         obj.join
         traverse_obj(obj.path.open, options, &block)
@@ -117,7 +144,7 @@ module TSV
     if callback
       block = Proc.new do |k,v,mutex|
         v, mutex = nil, v if mutex.nil?
-        res = yield k, v
+        res = yield k, v, mutex
         mutex.synchronize do
           callback.call res
         end
@@ -136,9 +163,9 @@ module TSV
   end
 
   def self.traverse_cpus(num, obj, options, &block)
-    callback = Misc.process_options options, :callback
-
-    q = RbbtProcessQueue.new num
+    filename = obj.respond_to?(:filename)? obj.filename : "none"
+    callback, cleanup = Misc.process_options options, :callback, :cleanup
+    q = RbbtProcessQueue.new num, cleanup
 
     q.callback &callback
     q.init &block
@@ -147,34 +174,59 @@ module TSV
       q.process *p
     end
 
+    into = options[:into]
+
     q.join
-    q.clean
-    nil
   end
 
-  def self.store_into(obj, value)
-    case obj
+  def self.store_into(store, value)
+    case store
     when Hash
       return if value.nil?
       if Hash === value
-        if TSV === obj and obj.type == :double
-          obj.merge_zip value
+        if TSV === store and store.type == :double
+          store.merge_zip value
         else
-          obj.merge! value
+          store.merge! value
         end
       else
         k,v = value
-        obj[k] = v
+        store[k] = v
       end
     when TSV::Dumper
       return if value.nil?
-      obj.add *value
-    when IO, StringIO
+      store.add *value
+    when IO
       return if value.nil?
-      obj.puts value
+      store.puts value.strip
     else
-      obj << value
+      store << value
+    end 
+  end
+
+  def self.get_streams_to_close(obj)
+    close_streams = []
+    case obj
+    when IO, File
+      close_streams << obj
+    when TSV::Parser
+    when TSV::Dumper
+      close_streams << obj.result.in_stream
+    when (defined? Step and Step)
+      case obj.result
+      when IO
+        close_streams << obj.result
+      when TSV::Dumper
+        close_streams << obj.result.in_stream
+      end
+      obj.inputs.each do |input|
+        close_streams = get_streams_to_close(input) + close_streams
+      end
+      obj.dependencies.each do |dependency|
+        close_streams = get_streams_to_close(dependency) + close_streams
+      end
     end 
+    close_streams
   end
 
   def self.traverse_run(obj, threads, cpus, options = {}, &block)
@@ -184,11 +236,62 @@ module TSV
       if threads
         traverse_threads threads, obj, options, &block 
       else
+        close_streams = Misc.process_options(options, :close_streams) || []
+        close_streams = [close_streams] unless Array === close_streams
+
+        close_streams.concat(get_streams_to_close(obj))
+        options[:close_streams] = close_streams
+
+        options[:cleanup] = Proc.new do
+          close_streams.uniq.each do |s|
+            s.close unless s.closed?
+          end
+        end if close_streams and close_streams.any?
+
         traverse_cpus cpus, obj, options, &block
       end
     end
   end
 
+  def self.traverse_stream(obj, threads, cpus, options, &block)
+    into = options[:into]
+    thread = Thread.new(Thread.current, obj) do |parent,obj|
+      begin
+        traverse_run(obj, threads, cpus, options, &block)
+        into.close if into.respond_to? :close
+      rescue Exception
+        Log.exception $!
+        parent.raise $!
+      end
+    end
+    thread.wakeup
+    ConcurrentStream.setup(obj_stream(into), :threads => thread)
+  end
+
+  def self.stream_name(obj)
+    filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
+    filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
+    stream_obj = obj_stream(obj)
+    filename_obj.nil? ? stream_obj.inspect : filename_obj + "(#{stream_obj.inspect})"
+  end
+
+  def self.report(msg, obj, into)
+    into = into[:into] if Hash === into and into.include? :into
+
+    #filename_into = into.respond_to?(:filename) ? into.filename : nil
+    #filename_into ||= into.respond_to?(:path) ? into.path : nil
+    #stream_into = obj_stream(into)
+    #str_into = filename_into.nil? ? stream_into.inspect : filename_into + "(#{stream_into.inspect})"
+
+    #filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
+    #filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
+    #stream_obj = obj_stream(obj)
+    #str_obj = filename_obj.nil? ? stream_obj.inspect : filename_obj + "(#{stream_obj.inspect})"
+
+    #Log.error "#{ msg } #{filename_obj} - #{filename_into}"
+    Log.error "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
+  end
+
   def self.traverse(obj, options = {}, &block)
     threads = Misc.process_options options, :threads
     cpus = Misc.process_options options, :cpus
@@ -198,23 +301,21 @@ module TSV
     cpus = nil if cpus and cpus.to_i <= 1
 
     if into
-      callback = Proc.new do |e|
-        store_into into, e
+      options[:callback] = Proc.new do |e|
+        begin
+          store_into into, e
+        rescue Exception
+          Log.exception $!
+        end
       end
-      options[:callback] = callback
 
       case into
       when TSV::Dumper, IO, StringIO
-        Thread.new(Thread.current) do |parent|
-          begin
-            traverse_run(obj, threads, cpus, options, &block)
-            into.close 
-          rescue Exception
-            parent.raise $!
-          end
-        end
+        traverse_stream(obj, threads, cpus, options, &block)
       else
         traverse_run(obj, threads, cpus, options, &block)
+        into.join if into.respond_to? :join
+        into.close if into.respond_to? :close
       end
 
       into