rbbt-util 5.7.0 → 5.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d4025fea0b4bfd41f869ae4282cd268ebe5f876
-  data.tar.gz: 194ead2946a9a3c215a9136bc2b0c615bf73488a
+  metadata.gz: d2929287d81291dd772ab9ee6f415b52bafad0db
+  data.tar.gz: 61ad4ae0dade13e2cbc0e02c39107243d92a52e0
 SHA512:
-  metadata.gz: 0df26701e5d7e3842a9ddd939c385b7c16884f81eee2fde670d6c7b1c0fe7cc371d05acaa65967eda8716fe7301c5e83e79a8b189e07341597b23052d648c060
-  data.tar.gz: afcd1c9b2c88cb5858278ae50401a5140d729aa2007629f99adff8d3239f3d28d2df4c4791362a450d1892d273bef352941ab14578a6031fc7e43a9f8404a3fd
+  metadata.gz: 14285e88be1d35d8c23fb77ad038d2818ae1df04245bd2fd48bbab4f72b419eff1042764b8173b33c1d96955dee0df7ecfd27a84efb34171abb10bae05169b79
+  data.tar.gz: d96daf9acd2d79ab8b9425d849fd9442b7fd4ee1d614d364d135b9a601d0b7de1278eba414cb464f92f325cb84ab131311dff1937c1bdc2a99d631e7ea5f146b

data/lib/rbbt/annotations.rb

@@ -159,9 +159,12 @@ module Annotated
         object.clean_annotations : 
         object.inject([]){|acc,e| acc << Annotated.purge(e); acc}
     when Hash
+      new = {}
       object.each do |key, value|
-        object[key] = Annotated.purge value
+        Annotated.purge key
+        new[key] = Annotated.purge value
       end
+      new
     else
       object
     end

data/lib/rbbt/annotations/util.rb

@@ -209,6 +209,17 @@ module Annotated
     end
   end
 
+  def marshal_dump(depth)
+   clean_annotations
+  end
+end
 
+class String
+  def marshal_load(str)
+    self.replace str
   end
+end
+
+
+
 

data/lib/rbbt/persist.rb

@@ -276,10 +276,16 @@ module Persist
             Log.medium "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
             res = yield
 
-            Misc.lock(path) do
-              save_file(path, type, res)
+            if res.nil?
+              res = load_file(path) unless persist_options[:no_load]
+            else
+              Misc.lock(path) do
+                save_file(path, type, res)
+              end
             end
 
+            return path if persist_options[:no_load]
+
             res
           end
         rescue

data/lib/rbbt/resource/path.rb

@@ -21,6 +21,7 @@ module Path
   end
 
   def join(name)
+    raise "Invalid path: #{ self }" if self.nil?
     if self.empty?
       self.annotate name.to_s.dup
     else

data/lib/rbbt/tsv/accessor.rb

@@ -7,8 +7,12 @@ module TSV
 
   attr_accessor :unnamed, :serializer_module, :entity_options, :entity_templates
 
+  def info
+    {:key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed}.delete_if{|k,v| v.nil? }
+  end
+
   def annotate(tsv)
-    TSV.setup(tsv, :key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed)
+    TSV.setup(tsv, info)
   end
 
   def entity_options
@@ -149,14 +153,16 @@ module TSV
   def serializer=(serializer)
     @serializer = serializer
     self.send(:[]=, KEY_PREFIX + 'serializer', (serializer.nil? ? SERIALIZED_NIL : TSV_SERIALIZER.dump(serializer)), :entry_key)
-    @serializar_module = serializer.nil? ? nil : SERIALIZER_ALIAS[serializer.to_sym]
+    @serializar_module = serializer.nil? ? TSV::CleanSerializer : SERIALIZER_ALIAS[serializer.to_sym]
   end
 
 
   def serializer_module
-    @serializar_module ||= begin
+    @serializer_module ||= begin
                              serializer = self.serializer
-                             serializer.nil? ? TSV::CleanSerializer : SERIALIZER_ALIAS[serializer.to_sym]
+                             mod = serializer.nil? ? TSV::CleanSerializer : SERIALIZER_ALIAS[serializer.to_sym]
+                             raise "No serializer_module for: #{ serializer.inspect }" if mod.nil?
+                             mod
                            end
   end
 
@@ -170,8 +176,10 @@ module TSV
   def [](key, clean = false)
     value = super(key)
     return value if clean or value.nil?
+    @serializer_module ||= self.serializer_module
+
+    value = @serializer_module.load(value) if @serializer_module and not TSV::CleanSerializer == @serializer_module
 
-    value = serializer_module.load(value) if serializer_module and not TSV::CleanSerializer === serializer_module
     return value if @unnamed or fields.nil?
 
     case type
@@ -186,11 +194,8 @@ module TSV
   end
 
   def []=(key, value, clean = false)
-    if clean or serializer_module.nil? or  TSV::CleanSerializer === serializer_module or value.nil? 
-       return super(key, value)
-    else
-      return super(key, serializer_module.dump(value))
-    end
+    return super(key, value) if clean or value.nil? or TSV::CleanSerializer == self.serializer_module 
+    super(key, @serializer_module.dump(value))
   end
 
   def zip_new(key, values)
@@ -231,20 +236,19 @@ module TSV
   def each
     fields = self.fields
 
-    serializer = self.serializer
     serializer_module = self.serializer_module
     super do |key, value|
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
-      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer === serializer_module
+      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer == serializer_module
 
       # Annotated with Entity and NamedArray
       if not @unnamed
         if not fields.nil? 
           case type
           when :double, :list
-            setup_array value, fields, key, entity_options, entity_templates if Array === value
+            setup_array value, fields, key, entity_options, entity_templates if Array == value
           when :flat, :single
             prepare_entity(value, fields.first, entity_options)
           end
@@ -258,13 +262,12 @@ module TSV
   end
 
   def collect
-    serializer = self.serializer
     serializer_module = self.serializer_module
     super do |key, value|
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
-      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer === serializer_module
+      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer == serializer_module
 
       # Annotated with Entity and NamedArray
       if not @unnamed

data/lib/rbbt/tsv/parallel.rb

@@ -1,47 +1,104 @@
+require 'rbbt/util/concurrency'
+
 module TSV
 
-  def pthrough(num_threads = 100, new_key_field = nil, new_fields = nil, uniq = false, zipped = false)
-    q = Queue.new
-    mutex = Mutex.new
-
-    threads = []
-
-    done = false
-    num_threads.times do |i|
-      threads << Thread.new(Thread.current) do |current|
-        begin
-          loop do
-            p = q.pop
-            p << mutex
-            yield *p
-            next if q.length == 0 and done
-          end
-        rescue Exception
-          current.raise $!
-        end
-      end
-    end
+  def pthrough(num_threads = 10, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
+    q = RbbtThreadQueue.new num_threads
 
-    max = 10_000_000
-    res = through(new_key_field, new_fields, uniq, zipped) do |*p|
-      if q.length >= max
-        Thread.pass
-        q << p
+    q.init(true, &block)
+
+    begin
+      res = through(new_key_field, new_fields, uniq, zipped) do |*p|
+        q.process p
       end
-      q << p
+      q.join
+    ensure
+      q.clean
     end
 
-    done == true
+  end
+
+  def ppthrough_callback(&block)
+    @ppthrough_callback = block
+  end
 
-    Thread.pass while q.length > 0
+  def ppthrough(num_procs = 7, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
 
+    q = RbbtProcessQueue.new num_procs
 
-    threads.each{|t| t.kill }
+    q.callback &@ppthrough_callback
+    @ppthrough_callback = nil
+
+    q.init do |k,v|
+      block.call k,v
+    end
+
+    begin
+      res = through(new_key_field, new_fields, uniq, zipped) do |*p|
+        q.process q
+      end
+      q.join
+    ensure
+      q.clean
+    end
 
     res
   end
 
-  def _pthrough(num_threads = 1, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
-    through(new_key_field, new_fields, uniq, zipped, &block) 
+  def ppthrough(num_procs = 7, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
+
+    q = RbbtProcessQueue.new num_procs
+
+    q.callback &@ppthrough_callback
+    @ppthrough_callback = nil
+
+    _pat_size = 20
+    _pat = "A" << _pat_size.to_s
+
+    num_fields = fields.length
+    pattern = case type
+              when :single, :flat
+                _pat * 2
+              when :list, :double
+                _pat * (num_fields + 1)
+              end
+
+    q.init do |str|
+      _parts = str.unpack(pattern)
+
+      case type
+      when :single
+        k, v = _parts
+      when :list
+        k, *v = _parts
+      when :flat
+        k, v = _parts
+        v = v.split "|"
+      when :double
+        k, *v = _parts
+        v = v.collect{|l| l.split "|" }
+      end
+
+      block.call k,v
+    end
+
+    begin
+      res = through(new_key_field, new_fields, uniq, zipped) do |k,v|
+        case type
+        when :flat
+          v = v * "|" 
+        when :double
+          v = v.collect{|l| l * "|" } if type == :double
+        end
+
+        str = [k,v].flatten.pack(pattern)
+        q.process str
+      end
+      q.join
+    ensure
+      q.clean
+    end
+
+    res
   end
 end

data/lib/rbbt/tsv/util.rb

@@ -230,4 +230,15 @@ module TSV
     new
   end
 
+  def marshal_dump
+    [info, to_hash]
+  end
+end
+
+class Hash
+  def marshal_load(array)
+    info, to_hash = array
+    self.merge! to_hash
+    TSV.setup(self)
+  end
 end

data/lib/rbbt/util/R.rb

@@ -53,7 +53,6 @@ source('#{UTIL}');
     when nil
       "NULL"
     when TSV
-      #"as.matrix(data.frame(c(#{object.transpose("Field").collect{|k,v| "#{k}=" << R.ruby2R(v)}.flatten * ", "}), row.names=#{R.ruby2R object.keys}))"
       "matrix(#{R.ruby2R object.values},dimnames=list(#{R.ruby2R object.keys}, #{R.ruby2R object.fields}))"
     when Symbol
       "#{ object }"

data/lib/rbbt/util/concurrency.rb

@@ -0,0 +1,2 @@
+require 'rbbt/util/concurrency/threads'
+require 'rbbt/util/concurrency/processes'

data/lib/rbbt/util/concurrency/processes.rb

@@ -0,0 +1,96 @@
+require 'rbbt/util/concurrency/processes/worker'
+require 'rbbt/util/concurrency/processes/socket'
+
+
+class RbbtProcessQueue
+  #{{{ RbbtProcessQueue
+
+  attr_accessor :num_processes, :processes, :queue, :process_monitor
+  def initialize(num_processes)
+    @num_processes = num_processes
+    @processes = []
+    @queue = RbbtProcessSocket.new
+  end
+
+  attr_accessor :callback, :callback_queue, :callback_thread
+  def callback(&block)
+    if block_given?
+      @callback = block
+
+      @callback_queue = RbbtProcessSocket.new
+
+      @callback_thread = Thread.new(Thread.current) do |parent|
+        begin
+          loop do
+            p = @callback_queue.pop
+            raise p if Exception === p
+            @callback.call p
+          end
+        rescue ClosedStream
+        rescue Exception
+          Log.debug $!
+          parent.raise $!
+          Thread.exit
+        end
+      end
+    else
+      @callback, @callback_queue, @callback_thread = nil, nil, nil
+    end
+  end
+
+  def init(&block)
+    num_processes.times do |i|
+      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, &block)
+    end
+    @queue.sread.close
+    @callback_queue.swrite.close if @callback_queue
+
+    @process_monitor = Thread.new(Thread.current) do |parent|
+      begin
+        while @processes.any? do
+          pid = Process.wait -1, Process::WNOHANG
+          if pid
+            @processes.delete_if{|p| p.pid == pid}
+            raise "Process #{pid} failed" unless $?.success?
+          else
+            sleep 1
+          end
+        end
+      rescue
+        parent.raise $!
+      ensure
+        Thread.exit
+      end
+    end
+  end
+
+  def close_callback
+    @callback_thread.join if @callback_thread and @callback_thread.alive?
+  end
+
+  def join
+    @queue.push ClosedStream.new
+    @queue.swrite.close
+    begin
+      @process_monitor.join
+    ensure
+      close_callback if @callback
+    end
+  end
+
+  def clean
+    @processes.each{|p| p.abort }.clear
+    @callback_thread.raise Aborted if @callback_thread and @callback_thread.alive?
+  end
+
+  def process(e)
+    @queue.push e
+  end
+
+  def self.each(list, num = 3, &block)
+    q = RbbtProcessQueue.new num
+    q.init(&block)
+    list.each do |elem| q.process elem end
+    q.join
+  end
+end

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -0,0 +1,87 @@
+require 'rbbt/util/semaphore'
+
+class RbbtProcessQueue
+  class RbbtProcessSocket
+
+    Serializer = Marshal
+
+    attr_accessor :sread, :swrite, :write_sem, :read_sem
+    def initialize
+      @sread, @swrite = IO.pipe
+
+      key = rand(100000).to_s;
+      @write_sem = key + '.in'
+      @read_sem = key + '.out'
+      RbbtSemaphore.create_semaphore(@write_sem,1)
+      RbbtSemaphore.create_semaphore(@read_sem,1)
+    end
+
+    def clean
+      @sread.close unless @sread.closed?
+      @swrite.close unless @swrite.closed?
+      RbbtSemaphore.delete_semaphore(@write_sem)
+      RbbtSemaphore.delete_semaphore(@read_sem)
+    end
+
+
+    def dump(obj, stream)
+      case obj
+      when String
+        payload = obj
+        size_head = [payload.bytesize,"S"].pack 'La'
+        str = size_head << payload
+      else
+        payload = Serializer.dump(obj)
+        size_head = [payload.bytesize,"M"].pack 'La'
+        str = size_head << payload
+      end
+
+      write_length = str.length
+      IO.select(nil, [stream])
+      wrote = stream.write(str) 
+      while wrote < write_length
+        wrote += stream.write(str[wrote..-1]) 
+      end
+    end
+
+    def load(stream)
+      size_head = Misc.read_stream stream, 5
+
+      size, type = size_head.unpack('La')
+
+      begin
+        payload = Misc.read_stream stream, size
+        case type
+        when "M"
+          Serializer.load(payload)
+        when "S"
+          payload
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+
+    #{{{ ACCESSOR
+    
+    def push(obj)
+      begin
+        RbbtSemaphore.synchronize(@write_sem) do
+          self.dump(obj, @swrite)
+        end
+      rescue
+        return ClosedStream.new
+      end
+    end
+
+    def pop
+      begin
+        RbbtSemaphore.synchronize(@read_sem) do
+          self.load(@sread)
+        end
+      rescue IOError, ClosedStream
+        return ClosedStream.new
+      end
+    end
+  end
+end