rbbt-util 5.34.11 → 5.34.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32b17775f7faf2220a471559cf7faf7db5f2515001a19e1fe5d70f3b588c06dd
-  data.tar.gz: '09b9518c2dadcdff12799cb9ece8b91a1f096c3953e695854fdf4b937868a345'
+  metadata.gz: 1d36e35e1495c3c319e8aa0485851b870e985bc34ea282f21c8e744adc302f50
+  data.tar.gz: e4df8738231ce9e1f8ebf70090595413fef4555ad9bedde4b52b36ebc219e39e
 SHA512:
-  metadata.gz: 56976a81e8aca514a836fd260d8e57c074252f9ef1e03bfb54496c1107bd6ecf27fa33f9e7d9f4c8ca39f5e8a4a2cc28c560e15cb26ec81855202aea8a53d41c
-  data.tar.gz: b195a055a913b56dd3cb70dc1da1bede56462a7eb55bce0ff437a53abdf556cee39dad8f8e49a6246705116990a666adc24e354345d5759d1f0d3690df2908dd
+  metadata.gz: 3ebc6c5ea53567b06a58854ab0b58c83777d939bb67a80cc28dbe7ebaa992269f59c20c2dba28d799402bfded6cbd9771ad1036e31ebbb365e3995500cd37f44
+  data.tar.gz: 2d3870b76a07c096ee1df74593b6bd641c0f1f8aab68c09827bae57cf64ee97e237b22ee6d16edb81ddcc83b7d8b6f4cb6298da8d203c63f8cdebc408b44fb84

data/lib/rbbt/annotations/annotated_array.rb

@@ -61,8 +61,8 @@ module AnnotatedArray
 
         value = annotate(value)
 
-        value.container       = self
-        value.container_index = pos
+        value.container       = self if value.respond_to?(:container)
+        value.container_index = pos if value.respond_to?(:container_index)
 
         pos += 1
 

data/lib/rbbt/annotations/util.rb

@@ -217,46 +217,66 @@ module Annotated
     end
   end
 
-  def marshal_dump
-    Annotated.purge(self).to_sym.to_s
-  end
-
   def self.to_hash(e)
     hash = {}
-    if Array === e && AnntatedArray === e
+    if Array === e && AnnotatedArray === e
       hash[:literal] = Annotated.purge(e)
       hash[:info] = e.info
     elsif Array === e
       hash = e.collect do |_e|
         _hash = {}
-        _hash[:literal] = _e
-        _hash[:info] = _e.info
+        _hash[:literal] = _e.dup
+        _hash[:info] = _e.info if _e.respond_to?(:info)
         _hash
       end
     else
-      hash[:literal] = e
+      hash[:literal] = e.dup
       hash[:info] = e.info
     end
     hash
   end
 
   def self.load_hash(hash)
-    literal = hash[:literal]
-    info = hash[:info]
-    info[:annotation_types].each do |type|
-      type = Kernel.const_get(type) if String === type
-      type.setup(literal, info) 
+    if Array === hash
+      hash.collect{|h| load_hash(h) }
+    else
+      literal = hash[:literal]
+      info = hash[:info]
+      info[:annotation_types].each do |type|
+        type = Kernel.const_get(type) if String === type
+        type.setup(literal, info) 
+      end
+      literal
     end
-    literal
+  end
+
+  def marshal_dump
+    Annotated.to_hash(self)
   end
 end
 
 class String
-  def marshal_load(str)
-    self.replace str
+  def marshal_load(hash)
+    if Hash === hash
+      e = Annotated.load_hash(hash)
+      self.replace e
+      e.annotate(self)
+      self
+    else
+      self.replace hash
+    end
   end
 end
 
-
-
-
+class Array
+  def marshal_load(hash)
+    if Hash === hash
+      e = Annotated.load_hash(hash)
+      self.replace e
+      e.annotate(self)
+      self
+    else
+      self.replace hash
+    end
+  end
+end

data/lib/rbbt/entity.rb

@@ -5,7 +5,7 @@ module Entity
   
   UNPERSISTED_PREFIX = "entity_unpersisted_property_"
 
-  class MultipleEntity < Exception; end
+  class MultipleEntity < TryAgain; end
 
   class DontPersist < Exception
     attr_accessor :payload
@@ -120,10 +120,6 @@ module Entity
         coder.scalar = clean_annotations
       end
 
-      def marshal_dump
-        clean_annotations
-      end
-
       def consolidate
         self.inject(nil){|acc,e| 
           if acc.nil?
@@ -203,9 +199,11 @@ module Entity
           end
 
           define_method name do |*args|
+
             if self.instance_variable_get("@multiple_result_" + name.to_s)
               return self.instance_variable_get("@multiple_result_" + name.to_s)
             end
+
             obj = if Array === self
                     self
                   elsif self.respond_to?(:container) && Array === self.container
@@ -217,7 +215,8 @@ module Entity
             missing = []
             obj.each do |e|
               begin 
-                e.send(multi_name)
+                res = e.send(multi_name)
+                e.instance_variable_set("@multiple_result_" + name.to_s, res)
               rescue MultipleEntity
                 missing << e
               end
@@ -227,20 +226,26 @@ module Entity
             case res
             when Array
               missing.zip(res).each do |o,res|
-                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
+                raise "Multiple function '#{name}' result 'nil' for element '#{o}'" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
+                o.send(multi_name)
               end
             when Hash
               res.each do |o,res|
-                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
+                raise "Multiple function '#{name}' result 'nil' for element '#{o}'" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
+                o.send(multi_name)
               end
             end
 
             if Array === self
-              self.collect{|o| o.send(multi_name)}
+              res = self.collect{|o| o.send(multi_name)}
+              self.instance_variable_set("@multiple_result_" + name.to_s, res)
+              res
             else
-              self.send(multi_name)
+              res = self.send(multi_name)
+              self.instance_variable_set("@multiple_result_" + name.to_s, res)
+              res
             end
           end
 

data/lib/rbbt/persist/tsv/adapter.rb

@@ -193,8 +193,12 @@ module Persist
     end
 
     def range(*args)
-      self.read_lock do
-        super(*args)
+      begin
+        self.read_lock do
+          super(*args)
+        end
+      rescue
+        []
       end
     end
 
@@ -266,5 +270,6 @@ module Persist
         end
       end
     end
+
   end
 end

data/lib/rbbt/persist/tsv/tokyocabinet/marshal.rb

@@ -0,0 +1,22 @@
+module Persist
+  module TCAdapter
+    def marshal_dump
+      [persistence_path, tokyocabinet_class]
+    end
+  end
+end
+
+class TokyoCabinet::BDB
+  def marshal_load(values)
+    persistence_path, tokyocabinet_class = values
+    Persist::TCAdapter.open persistence_path, false, tokyocabinet_class
+  end
+end
+
+class TokyoCabinet::HDB
+  def marshal_load(values)
+    persistence_path, tokyocabinet_class = values
+    Persist::TCAdapter.open persistence_path, false, tokyocabinet_class
+  end
+end
+

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -101,3 +101,5 @@ module Persist
     database
   end
 end
+
+require 'rbbt/persist/tsv/tokyocabinet/marshal'

data/lib/rbbt/persist.rb

@@ -359,6 +359,12 @@ module Persist
       Log.exception $!
       sleep 1 + rand(2)
       retry
+    rescue TryAgain
+      begin
+        Open.rm path 
+      rescue
+      end if Open.exists? path 
+      raise $!
     rescue Exception
       Log.medium "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
 

data/lib/rbbt/tsv/manipulate.rb

@@ -369,13 +369,14 @@ module TSV
   def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
-    new.key_field = key_field
-    new.fields    = fields.dup unless fields.nil?
-    new.type      = type
-    new.filename  = filename
-    new.namespace = namespace
-    new.entity_options = entity_options
-    new.entity_templates = entity_templates
+    self.annotate(new)
+    #new.key_field = key_field
+    #new.fields    = fields.dup unless fields.nil?
+    #new.type      = type
+    #new.filename  = filename
+    #new.namespace = namespace
+    #new.entity_options = entity_options
+    #new.entity_templates = entity_templates
     
     case
     when (method.nil? and block_given?)

data/lib/rbbt/tsv/marshal.rb

@@ -0,0 +1,17 @@
+module TSV
+  def marshal_dump
+    if defined?(Persist::TCAdapter) && Persist::TCAdapter === self
+      super
+    else
+      [info, to_hash]
+    end
+  end
+end
+
+class Hash
+  def marshal_load(array)
+    info, to_hash = array
+    self.merge! to_hash
+    TSV.setup(self)
+  end
+end

data/lib/rbbt/tsv/util.rb

@@ -388,9 +388,6 @@ module TSV
     new
   end
 
-  def marshal_dump
-    [info, to_hash]
-  end
 
   def to_onehot(boolean = false)
     all_values = values.flatten.uniq.collect{|v| v.to_s}.sort
@@ -405,10 +402,3 @@ module TSV
   end
 end
 
-class Hash
-  def marshal_load(array)
-    info, to_hash = array
-    self.merge! to_hash
-    TSV.setup(self)
-  end
-end

data/lib/rbbt/tsv.rb

@@ -18,6 +18,7 @@ require 'rbbt/tsv/parallel'
 require 'rbbt/tsv/stream'
 require 'rbbt/tsv/melt'
 require 'rbbt/tsv/matrix'
+require 'rbbt/tsv/marshal'
 
 module TSV
   class << self

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -29,6 +29,10 @@ class RbbtProcessQueue
 
     def dump(obj, stream)
       case obj
+      when Annotated
+        payload = @serializer.dump(obj)
+        size_head = [payload.bytesize,"S"].pack 'La'
+        str = size_head << payload
       when String
         payload = obj
         size_head = [payload.bytesize,"C"].pack 'La'
@@ -90,7 +94,7 @@ class RbbtProcessQueue
     def push(obj)
       RbbtSemaphore.synchronize(@write_sem) do
         multiple = MultipleResult === obj
-        obj = Annotated.purge(obj)
+        #obj = Annotated.purge(obj)
         obj.extend MultipleResult if multiple
         self.dump(obj, @swrite)
       end

data/lib/rbbt/util/misc/development.rb

@@ -413,4 +413,15 @@ def self.add_libdir(dir=nil)
     CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
   end
 
+  def self.timeout_insist(time, msg = nil, &block)
+    Misc.insist do
+      begin
+        Timeout.timeout(time, TryAgain, msg, &block)
+      rescue TryAgain
+        Log.low "Timeout detected after #{time} seconds"
+        raise $!
+      end
+    end
+  end
+
 end

data/lib/rbbt/util/misc/objects.rb

@@ -27,10 +27,11 @@ module Misc
       params.merge!(:format => _format) unless _format.nil? or (params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?)))
 
       mod = Entity === field ? field : Entity.formats[field]
-      entity = mod.setup(
-        ((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ),
-        params
-      ) 
+
+      entity = entity.dup
+      entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) 
+
+      entity = mod.setup(entity, params)
     end
 
     entity

data/test/rbbt/annotations/test_util.rb

@@ -9,9 +9,19 @@ end
 class TestAnnotation < Test::Unit::TestCase
   def test_marshal
     a = "STRING"
-    a.extend Annotated
+    a = TestEntityString.setup(a)
 
-    assert !(Annotated === Marshal.load(Marshal.dump(a)))
+    #assert !(Annotated === Marshal.load(Marshal.dump(a)))
+    assert_equal a, Marshal.load(Marshal.dump(a))
+  end
+
+  def test_marshal_array
+    a = ["STRING"]
+    TestEntityString.setup(a)
+
+    #assert !(Annotated === Marshal.load(Marshal.dump(a)))
+    assert Annotated === a
+    assert Annotated === a.first
     assert_equal a, Marshal.load(Marshal.dump(a))
   end
 
@@ -20,5 +30,14 @@ class TestAnnotation < Test::Unit::TestCase
     assert_equal "TEST", Annotated.to_hash(e)[:literal]
     assert_equal 10, Annotated.to_hash(e)[:info][:code]
   end
+
+  def test_traverse_cpus
+    strings = (1..10).to_a.collect{|n| n.to_s }
+    res = TSV.traverse strings, :cpus => 2, :into => [] do |s|
+      TestEntityString.setup(s, :code => s.to_i)
+    end
+
+    assert TestEntityString === res.first
+  end
 end
 

data/test/rbbt/tsv/test_marshal.rb

@@ -0,0 +1,24 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+require 'rbbt/tsv'
+require 'rbbt/tsv/marshal'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+class TestTSVUtil < Test::Unit::TestCase
+
+  def test_marshal
+   content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+
+      assert TSV === Marshal.load(Marshal.dump(tsv))
+      assert_equal tsv.to_hash, Marshal.load(Marshal.dump(tsv)).to_hash
+      assert_equal({1 => 1}, Marshal.load(Marshal.dump({1 => 1})))
+    end
+  end
+end

data/test/rbbt/tsv/test_util.rb

@@ -21,22 +21,6 @@ row2    A    B    Id3
     end
   end
 
-  def test_marshal
-   content =<<-EOF
-#Id    ValueA    ValueB    OtherID
-row1    a|aa|aaa    b    Id1|Id2
-row2    A    B    Id3
-    EOF
-
-    TmpFile.with_file(content) do |filename|
-      tsv = TSV.open(filename, :sep => /\s+/)
-
-      assert TSV === Marshal.load(Marshal.dump(tsv))
-      assert_equal tsv.to_hash, Marshal.load(Marshal.dump(tsv)).to_hash
-      assert_equal({1 => 1}, Marshal.load(Marshal.dump({1 => 1})))
-    end
-  end
-
   def test_replicates
    content =<<-EOF
 #Id    ValueA    ValueB    OtherID

data/test/rbbt/util/misc/test_development.rb

@@ -7,5 +7,16 @@ class TestMiscDevelopment < Test::Unit::TestCase
       puts "hola"
     end
   end
+
+  def __test_timeout
+    Misc.timeout_insist(2) do
+      puts "Start"
+      3.times do
+        puts "ping"
+        sleep rand(4)
+        puts "pong"
+      end
+    end
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.11
+  version: 5.34.12
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -255,6 +255,7 @@ files:
 - lib/rbbt/persist/tsv/packed_index.rb
 - lib/rbbt/persist/tsv/sharder.rb
 - lib/rbbt/persist/tsv/tokyocabinet.rb
+- lib/rbbt/persist/tsv/tokyocabinet/marshal.rb
 - lib/rbbt/resource.rb
 - lib/rbbt/resource/path.rb
 - lib/rbbt/resource/rake.rb
@@ -272,6 +273,7 @@ files:
 - lib/rbbt/tsv/filter.rb
 - lib/rbbt/tsv/index.rb
 - lib/rbbt/tsv/manipulate.rb
+- lib/rbbt/tsv/marshal.rb
 - lib/rbbt/tsv/matrix.rb
 - lib/rbbt/tsv/melt.rb
 - lib/rbbt/tsv/parallel.rb
@@ -536,6 +538,7 @@ files:
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_marshal.rb
 - test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_parallel.rb
 - test/rbbt/tsv/test_parser.rb
@@ -667,6 +670,7 @@ test_files:
 - test/rbbt/tsv/test_change_id.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_marshal.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_csv.rb
 - test/rbbt/tsv/test_accessor.rb