rbbt-util 5.14.18 → 5.14.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 243728b311e7948c452e6de9e299e5938a3ebd24
-  data.tar.gz: 8eb9ee8a10b1653c79465eab2c68f340a39ecc9f
+  metadata.gz: f5bc1786fca01c18645eeb743092710a25ebef1e
+  data.tar.gz: 7b4698982619547c6d37a6b0357f1937d33a7baa
 SHA512:
-  metadata.gz: 46d41fe4394efa9c7ad329e8fa42d0fcd3ea6d2d72cd30a7d2b3f6d7fd8df61565f621ad339abc74d7662a5220df92d2de03beb46f0b5ded72d5a967e532f752
-  data.tar.gz: 20566c536d6b4f4946feedea1e5ad956caf883d8772db394fc8217bbc9eae1bacecb9d5b74fdfde0d45cd94784b82f8ceed01ac3ed286edfa8e35dd3782a12c7
+  metadata.gz: fdf6ad7098498a63382450710dec248108f2956f8919ee72f63ee27fb1c9863eaedd449662ea709fce610c3735d6dc2eac37a3c58ad8c9451935ee32b8076ca1
+  data.tar.gz: 7d59b991703716459c7fec294b7acdffe8a55f33cd5ba989fc634d2e01a9be973ae092863f1aaeb661a16c662935c9ec858ecee98124c8aa4aa6995be25be743

data/etc/app.d/entities.rb

@@ -40,4 +40,3 @@ $annotation_repo = Rbbt.var.sinatra.annotation_repo.find
   end
 end if Rbbt.etc.persist_properties.exists?
 
-

data/lib/rbbt/knowledge_base.rb

@@ -22,12 +22,12 @@ class KnowledgeBase
 
   attr_accessor :namespace, :dir, :indices, :registry, :format, :databases, :entity_options
   def initialize(dir, namespace = nil)
-    @dir = Path.setup(dir).find
+    @dir = Path.setup(dir.dup).find
 
     @namespace = namespace
     @format = IndiferentHash.setup({})
 
-    @registry = IndiferentHash.setup({})
+    @registry ||= IndiferentHash.setup({})
     @entity_options = IndiferentHash.setup({})
 
     @indices = IndiferentHash.setup({})
@@ -59,7 +59,7 @@ class KnowledgeBase
     end
   end
 
-  def syndicate(kb, name)
+  def syndicate(name, kb)
     kb.all_databases.each do |database|
       db_name = [database, name] * "@"
       file, kb_options = kb.registry[database]
@@ -72,7 +72,7 @@ class KnowledgeBase
   end
 
   def all_databases
-    @registry.keys
+    @registry.keys 
   end
 
   def description(name)
@@ -117,43 +117,103 @@ class KnowledgeBase
     {:namespace => namespace, :format => @format}
   end
  
+  #def get_database(name, options = {})
+  #  @databases[Misc.fingerprint([name, options])] ||= \
+  #    begin 
+  #      Persist.memory("Database:" <<[name, self.dir] * "@") do
+  #        options = Misc.add_defaults options, :persist_dir => dir.databases
+  #        persist_options = Misc.pull_keys options, :persist
+
+  #        file, registered_options = registry[name]
+  #        options = open_options.merge(registered_options || {}).merge(options)
+  #        raise "Repo #{ name } not found and not registered" if file.nil?
+
+  #        Log.low "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
+  #        Association.open(file, options, persist_options).
+  #          tap{|tsv| tsv.namespace = self.namespace}
+  #      end
+  #    end
+  #end
+
+
+  #def get_index(name, options = {})
+  #  @indices[Misc.fingerprint([name, options])] ||= \
+  #    begin 
+  #      Persist.memory("Index:" <<[name, self.dir] * "@") do
+  #        options = Misc.add_defaults options, :persist_dir => dir.indices
+  #        persist_options = Misc.pull_keys options, :persist
+
+  #        file, registered_options = registry[name]
+  #        options = open_options.merge(registered_options || {}).merge(options)
+  #        raise "Repo #{ name } not found and not registered" if file.nil?
+
+  #        Log.low "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
+  #        Association.index(file, options, persist_options).
+  #          tap{|tsv| tsv.namespace = self.namespace}
+  #      end
+  #    end
+  #end
+
   def get_database(name, options = {})
-    @databases[Misc.fingerprint([name, options])] ||= \
+    key = name.to_s + "_" + Misc.digest(Misc.fingerprint([name,options,format,namespace]))
+    @databases[key] ||= 
       begin 
-        Persist.memory("Database:" <<[name, self.dir] * "@") do
-          options = Misc.add_defaults options, :persist_dir => dir.databases
+        Persist.memory("Database:" << [key, dir] * "@") do
+          persist_file = dir.databases[key]
+          file, registered_options = registry[name]
+
+          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format
+          options = Misc.add_defaults options, registered_options
+
           persist_options = Misc.pull_keys options, :persist
 
-          file, registered_options = registry[name]
-          options = open_options.merge(registered_options || {}).merge(options)
-          raise "Repo #{ name } not found and not registered" if file.nil?
+          database = if persist_file.exists?
+                    Log.low "Re-opening database #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
+                    Association.open(file, options, persist_options)
+                  else
+                    raise "Repo #{ name } not found and not registered" if file.nil?
+                    Log.low "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
+                    Association.open(file, options, persist_options)
+                  end
+
+          database.namespace = self.namespace
 
-          Log.low "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
-          Association.open(file, options, persist_options).
-            tap{|tsv| tsv.namespace = self.namespace}
+          database
         end
       end
   end
 
-
   def get_index(name, options = {})
-    @indices[Misc.fingerprint([name, options])] ||= \
+    key = name.to_s + "_" + Misc.digest(Misc.fingerprint([name,options]))
+    @indices[key] ||= 
       begin 
-        Persist.memory("Index:" <<[name, self.dir] * "@") do
-          options = Misc.add_defaults options, :persist_dir => dir.indices
+        Persist.memory("Index:" << [key, dir] * "@") do
+          persist_file = dir.indices[key]
+          file, registered_options = registry[name]
+
+          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format
+          options = Misc.add_defaults options, registered_options
+
           persist_options = Misc.pull_keys options, :persist
 
-          file, registered_options = registry[name]
-          options = open_options.merge(registered_options || {}).merge(options)
-          raise "Repo #{ name } not found and not registered" if file.nil?
+          index = if persist_file.exists?
+                    Log.low "Re-opening index #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
+                    Association.index(nil, options, persist_options)
+                  else
+                    options = Misc.add_defaults options, registered_options if registered_options
+                    raise "Repo #{ name } not found and not registered" if file.nil?
+                    Log.low "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
+                    Association.index(file, options, persist_options)
+                  end
+
+          index.namespace = self.namespace
 
-          Log.low "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
-          Association.index(file, options, persist_options).
-            tap{|tsv| tsv.namespace = self.namespace}
+          index
         end
       end
   end
 
+
   #{{{ Add manual database
   
   def add_index(name, source_type, target_type, *fields)
@@ -292,6 +352,8 @@ class KnowledgeBase
 
   def subset(name, entities)
     entities = case entities
+               when :all
+                 {:target => :all, :source => :all}
                when AnnotatedArray
                  format = entities.format if entities.respond_to? :format 
                  format ||= entities.base_entity.to_s
@@ -319,7 +381,7 @@ class KnowledgeBase
   end
 
   def translate(entities, type)
-    if format = @format[type] and format != entities.format
+    if format = @format[type] and (entities.respond_to? :format and format != entities.format)
       entities.to format
     else
       entities

data/lib/rbbt/tsv/attach/util.rb

@@ -34,12 +34,12 @@ module TSV
 
             new_values.flatten if type == :flat
 
-            self[key] = current.concat new_values
+            self[key] = current + new_values
           else
             if type == :double
-              self[key] = current.concat [[]] * fields.length
+              self[key] = current + [[]] * fields.length
             else
-              self[key] = current.concat [nil] * fields.length
+              self[key] = current + [nil] * fields.length
             end
           end
         end

data/lib/rbbt/tsv/manipulate.rb

@@ -528,6 +528,16 @@ module TSV
     new
   end
 
+  def column_values(field, options = {})
+    all = []
+    through :key, field do |k,values|
+      values = Array === values ? values.flatten : [values]
+      all.concat value
+    end
+    prepare_entity(all, field, options = {})
+  end
+
+
   def process_key(&block)
     new = annotate({})
     through do |key, values|

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -334,9 +334,9 @@ module TSV
 
   def self.traverse_cpus(num, obj, options, &block)
     begin
-      callback, cleanup, join = Misc.process_options options, :callback, :cleanup, :join
+      callback, cleanup, join, respawn = Misc.process_options options, :callback, :cleanup, :join, :respawn
 
-      q = RbbtProcessQueue.new num, cleanup, join
+      q = RbbtProcessQueue.new num, cleanup, join, respawn
       q.callback &callback
       q.init &block
 

data/lib/rbbt/util/concurrency/processes.rb

@@ -3,12 +3,13 @@ require 'rbbt/util/concurrency/processes/socket'
 
 class RbbtProcessQueue
 
-  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup, :join
-  def initialize(num_processes, cleanup = nil, join = nil)
+  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup, :join, :reswpan
+  def initialize(num_processes, cleanup = nil, join = nil, reswpan = nil)
     @num_processes = num_processes
     @processes = []
     @cleanup = cleanup
     @join = join
+    @respawn = reswpan
     @queue = RbbtProcessSocket.new
   end
 
@@ -48,7 +49,7 @@ class RbbtProcessQueue
 
   def init(&block)
     num_processes.times do |i|
-      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, &block)
+      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, &block)
     end
     @queue.close_read
 

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -2,46 +2,126 @@ require 'rbbt/util/concurrency/processes/socket'
 class RbbtProcessQueue
   class RbbtProcessQueueWorker
     attr_reader :pid, :queue, :callback_queue, :cleanup, :block
-    def initialize(queue, callback_queue = nil, cleanup = nil, &block)
-      @queue, @callback_queue, @cleanup, @block = queue, callback_queue, cleanup, block
 
-      @pid = Process.fork do
-        begin
-          Misc.pre_fork
+    class Respawn < Exception; end
+
+    def run
+      begin
+        Signal.trap(:INT){ 
+          Kernel.exit! -1
+        }
+
+        @stop = false
+        Signal.trap(:USR1){ 
+          @stop = true
+        }
+
+        loop do
+          p = @queue.pop
+          next if p.nil?
+          raise p if Exception === p
+          raise p.first if Exception === p.first
+          res = @block.call *p
+          @callback_queue.push res if @callback_queue
+          raise Respawn if @stop
+        end
+        Kernel.exit! 0
+      rescue Respawn
+        Kernel.exit! 28
+      rescue ClosedStream
+      rescue Aborted, Interrupt
+        Log.warn "Worker #{Process.pid} aborted"
+        Kernel.exit! 0
+      rescue Exception
+        Log.exception $!
+        @callback_queue.push($!) if @callback_queue
+        Kernel.exit! -1
+      ensure
+        @callback_queue.close_write if @callback_queue 
+      end
+    end
 
-          @cleanup.call if @cleanup
-          @queue.close_write 
+    def run_with_respawn(multiplier = nil)
+      multiplier = case multiplier
+                   when String
+                     multiplier.to_s
+                   when Fixnum
+                     multiplier.to_i
+                   else
+                     3
+                   end
 
-          if @callback_queue
-            Misc.purge_pipes(@callback_queue.swrite) 
-            @callback_queue.close_read 
-          else
-            Misc.purge_pipes
+      status = nil
+      begin
+        @current = Process.fork do
+          run
+        end
+        Log.warn "Worker #{Process.pid} started with #{@current}"
+
+        @monitor_thread = Thread.new do
+          begin
+            while true
+              current = Misc.memory_use(@current) 
+              initial = Misc.memory_use(Process.pid)
+              if current > multiplier * initial
+                Process.kill "USR1", @current
+              end
+              sleep 1
+            end
+          rescue
+            Log.exception $!
           end
+        end
 
-          Signal.trap(:INT){ 
-            Kernel.exit! -1
-          }
-
-          loop do
-            p = @queue.pop
-            next if p.nil?
-            raise p if Exception === p
-            raise p.first if Exception === p.first
-            res = @block.call *p
-            @callback_queue.push res if @callback_queue
+        while true
+          pid, status = Process.waitpid2 @current
+          code = status.to_i >> 8 
+          break unless code == 28
+          @current = Process.fork do
+            run
           end
-          Kernel.exit! 0
-        rescue ClosedStream
-        rescue Aborted, Interrupt
-          Log.warn "Worker #{Process.pid} aborted"
-          Kernel.exit! 0
-        rescue Exception
-          Log.exception $!
-          @callback_queue.push($!) if @callback_queue
-          Kernel.exit! -1
-        ensure
-          @callback_queue.close_write if @callback_queue 
+          Log.warn "Worker #{Process.pid} respawning to #{@current}"
+        end
+      rescue Aborted, Interrupt
+        Log.warn "Worker #{Process.pid} aborted"
+        Kernel.exit! 0
+        Process.kill "INT", @current 
+      rescue Exception
+        Log.exception $!
+        raise $!
+      ensure
+        @monitor_thread.kill
+        Process.kill "INT", @current if Misc.pid_exists? @current
+        @callback_queue.close_write if @callback_queue 
+      end
+
+      if status
+        Kernel.exit! status.to_i >> 8
+      else
+        Kernel.exit! -1
+      end
+    end
+
+    def initialize(queue, callback_queue = nil, cleanup = nil, respawn = false, &block)
+      @queue, @callback_queue, @cleanup, @block = queue, callback_queue, cleanup, block
+
+      @pid = Process.fork do
+        Misc.pre_fork
+
+        @cleanup.call if @cleanup
+        @queue.close_write 
+
+        if @callback_queue
+          Misc.purge_pipes(@callback_queue.swrite) 
+          @callback_queue.close_read 
+        else
+          Misc.purge_pipes
+        end
+
+        if respawn
+          run_with_respawn respawn
+        else
+          run
         end
       end
     end

data/lib/rbbt/util/misc/development.rb

@@ -298,4 +298,8 @@ module Misc
       nil
     end
   end
+
+  def self.memory_use(pid=nil)
+    `ps -o rss -p #{pid || $$}`.strip.split.last.to_i
+  end
 end

data/share/rbbt_commands/association/subset

@@ -19,6 +19,7 @@ Indicate the index file (TokyoCabinet BDB with source~target keys) as the <filen
 -t--target* Target entities
 -tsv--tsv* Output tsv
 EOF
+
 rbbt_usage and exit 0 if options[:help] 
 filename = ARGV.shift
 raise ParameterException, "No filename specified" if filename.nil?

data/test/rbbt/util/concurrency/test_processes.rb

@@ -69,9 +69,36 @@ class TestConcurrencyProcess < Test::Unit::TestCase
       sleep 2
       q.clean
 
+
       q.join
     end
   end
+
+
+  def test_process_respawn
+    q = RbbtProcessQueue.new 2, nil, nil, true
+
+    res = []
+
+    q.callback do |v|
+      res << v
+    end
+
+    q.init do |i|
+      $str ||="-"
+      $str = $str *  (2 + rand(5).to_i)
+      sleep 0.1
+      "."
+    end
+
+    times = 50
+
+    times.times do |i|
+      q.process i
+    end
+
+    q.join
+  end
 end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.18
+  version: 5.14.20
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-26 00:00:00.000000000 Z
+date: 2014-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake