rbbt-util 5.13.36 → 5.13.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 48414e48036dde6708ce828f47fa10e1169cf26c
-  data.tar.gz: 818bb5d1cf90da015d716050c9283821435eecbf
+  metadata.gz: 26df367cce98f5b14b985bd311d4b6594dcb62a9
+  data.tar.gz: 1931a495ac7eb8eb17dcfa0f154be8c9d61e8b7f
 SHA512:
-  metadata.gz: a35a5de8c4870b10e86353030787fb261b7086eac2553e79b86609352d88d69d09f65246cbd07e43f65d1a9bf863c0f00e070c9f380b4f2ae1d541b6f89f61a9
-  data.tar.gz: e649af70a02eb33d3a3255766bf495197baa48d0274fa3a6b8e723a43daa4f8dc497e7aa781291ad83d75b6a50fa4f2913a1530bb9924147ecd670c4ce672fc5
+  metadata.gz: e71fee09c427fdf3faf326b75897dc6d070ec605bc94b3785d8cacf88f2bd3fa1fdd8d017e5e06d5e48cff48960cb8e7debb337b3a4eeb3361d2fd3735b2b1f3
+  data.tar.gz: 14a1b01824edd6bfa2a6ead19828454ea57b1c3846eb1f332a0b15faee9bf5019fdda715732ef57b1326ce3870d8c2f8d8c6c9f2faf6e70218ceea76f373436d

data/lib/rbbt/fix_width_table.rb

@@ -1,6 +1,6 @@
 class FixWidthTable
 
-  attr_accessor :filename, :file, :value_size, :record_size, :range, :size
+  attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask
   def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
     @filename = filename
 
@@ -8,13 +8,14 @@ class FixWidthTable
       Log.debug "FixWidthTable create: #{ filename }"
       @value_size  = value_size
       @range       = range
-      @record_size = @value_size + (@range ? 12 : 4)
+      @record_size = @value_size + (@range ? 16 : 8)
 
       if %w(memory stringio).include? filename.to_s.downcase
         @filename = :memory
         @file     = StringIO.new
       else
         FileUtils.rm @filename if File.exists? @filename
+        FileUtils.mkdir_p File.dirname(@filename) unless File.exists? @filename
         @file = File.open(@filename, 'wb')
       end
 
@@ -24,17 +25,26 @@ class FixWidthTable
     else
       Log.debug "FixWidthTable up-to-date: #{ filename }"
       if in_memory
-        @file        = StringIO.new(Open.read(@filename, :mode => 'rb'), 'r')
+        @file        = StringIO.new(Open.read(@filename, :mode => 'r:ASCII-8BIT'), 'r')
       else
-        @file        = File.open(@filename, 'r')
+        @file        = File.open(@filename, 'r:ASCII-8BIT')
       end
       @value_size  = @file.read(4).unpack("L").first
       @range       = @file.read(1).unpack("C").first == 1
       @record_size = @value_size + (@range ? 12 : 4)
       @size        = (File.size(@filename) - 5) / (@record_size)
     end
+
+    @mask = "a#{value_size}"
+  end
+
+  def persistence_path
+    @filename
   end
 
+  def persistence_path=(value)
+    @filename=value
+  end
 
   CONNECTIONS = {} unless defined? CONNECTIONS
   def self.get(filename, value_size = nil, range = nil, update = false)
@@ -50,28 +60,18 @@ class FixWidthTable
   def format(pos, value)
     padding = value_size - value.length
     if range
-      (pos  + [value + ("\0" * padding)]).pack("llla#{value_size}")
+      (pos  + [padding, value + ("\0" * padding)]).pack("llll#{mask}")
     else
-      [pos, value + ("\0" * padding)].pack("la#{value_size}")
-    end
-  end
-
-  def unformat(format)
-    if range
-      pos_start, pos_end, pos_overlap, value = format.unpack("llla#{value_size}")
-      [[pos_start, pos_end, pos_overlap], value.strip]
-    else
-      pos, value = format.unpack("la#{value_size}")
-      [pos, value.strip]
+      [pos, padding, value + ("\0" * padding)].pack("ll#{mask}")
     end
   end
 
   def add(pos, value)
     format = format(pos, value)
     @file.write format
+
     @size += 1
   end
-  alias << add
 
   def last_pos
     pos(size - 1)
@@ -98,13 +98,15 @@ class FixWidthTable
   def value(index)
     return nil if index < 0 or index >= size
     @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
-    @file.read(value_size).unpack("a#{value_size}").first.strip
+    padding = @file.read(4).unpack("l").first+1
+    txt = @file.read(value_size)
+    txt.unpack(mask).first[0..-padding]
   end
 
-  def read
+  def read(force = false)
     return if @filename == :memory
     @file.close unless @file.closed?
-    @file = File.open(@filename, 'r')
+    @file = File.open(filename, 'r:ASCII-8BIT')
   end
 
   def close
@@ -125,17 +127,20 @@ class FixWidthTable
     end
   end
 
+  def add_range_point(pos, value)
+    @latest ||= []
+    while @latest.any? and @latest[0] < pos[0]
+      @latest.shift
+    end
+    overlap = @latest.length
+    add pos + [overlap], value
+    @latest << pos[1]
+  end
+
   def add_range(data)
-    latest = []
+    @latest = []
     data.sort_by{|value, pos| pos[0] }.each do |value, pos|
-      while latest.any? and latest[0] < pos[0]
-        latest.shift
-      end
-
-      overlap = latest.length
-
-      add pos + [overlap], value
-      latest << pos[1]
+      add_range_point(pos, value)
     end
   end
 
@@ -169,12 +174,14 @@ class FixWidthTable
   end
 
   def get_range(pos)
-    if Range === pos
+    case pos
+    when Range
       r_start = pos.begin
       r_end   = pos.end
+    when Array
+      r_start, r_end = pos
     else
-      r_start = pos.to_i
-      r_end   = pos.to_i
+      r_start, r_end = pos, pos
     end
 
     idx = closest(r_start)
@@ -243,6 +250,7 @@ class FixWidthTable
     end
   end
 
+
   def values_at(*list)
     list.collect{|pos|
       self[pos]

data/lib/rbbt/monitor.rb

@@ -3,7 +3,8 @@ require 'rbbt'
 module Rbbt
 
   LOCK_DIRS = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all +  Rbbt.var.jobs.find_all +
-    Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all + Rbbt.tmp.sensiblewrite_lock_dir.find_all
+    Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all + Rbbt.tmp.sensiblewrite_lock_dir.find_all +
+    Rbbt.tmp.produce_locks.find_all
 
   SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
 

data/lib/rbbt/packed_index.rb

@@ -0,0 +1,79 @@
+class PackedIndex
+  attr_accessor :file, :mask, :mask_length, :offset, :item_size, :stream, :nil_string
+
+  ELEMS = {
+    "I" => ["q", 8],
+    "i" => ["l", 4],
+    "f" => ["f", 4],
+    "F" => ["D", 8],
+  }
+
+  def self.process_mask(mask)
+    str = ""
+    size = 0
+    mask.each do |e|
+      if ELEMS.include? e
+        str << ELEMS[e][0]
+        size += ELEMS[e][1]
+      elsif e =~ /^(\d+)s$/
+        num = $1.to_i
+        str << "a" << num.to_s
+        size += num
+      else
+        e, num = e.split(":")
+        str << e
+        size = (num.nil? ? size + 1 : size + num.to_i)
+      end
+    end
+    [str, size]
+  end
+
+  def initialize(file, write = false, pattern = nil)
+    @file = file
+    if write
+      @stream = Open.open(file, :mode => 'wb')
+      @mask, @item_size = PackedIndex.process_mask pattern
+      header = [@mask.length, @item_size].pack("ll")
+      @stream.write(header)
+      @stream.write(mask)
+      @offset = @mask.length + 8
+    else
+      @stream = Open.open(file, :mode => 'rb')
+      header = @stream.read(8)
+      mask_length, @item_size = header.unpack("ll")
+      @mask = @stream.read(mask_length)
+      @offset = @mask.length + 8
+    end
+    @nil_string = "[NIL]" + "-" * (@item_size - 5)
+  end
+
+  def read
+    close
+    @stream = Open.open(file, :mode => 'rb')
+  end
+
+  def <<(payload)
+    if payload.nil?
+      @stream.write nil_string
+    else
+      @stream.write payload.pack(mask)
+    end
+  end
+
+  def [](position)
+    @stream.seek(position * item_size + offset)
+    encoded = @stream.read(item_size)
+    return nil if encoded == nil_string
+    encoded.unpack mask
+  end
+
+  def values_at(*positions)
+    positions.collect{|p|
+      self[p]
+    }
+  end
+
+  def close
+    stream.close unless stream.closed?
+  end
+end

data/lib/rbbt/persist.rb

@@ -204,7 +204,7 @@ module Persist
     ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
     out.callback = callback
     out.abort_callback = abort_callback
-    out.lockfile = stream.lockfile
+    out.lockfile = stream.lockfile if stream.respond_to? :lockfile
     out
   end
 

data/lib/rbbt/persist/tsv.rb

@@ -1,5 +1,7 @@
 require 'rbbt/persist/tsv/adapter'
 
+require 'rbbt/persist/tsv/fix_width_table'
+
 begin
   require 'rbbt/persist/tsv/tokyocabinet'
 rescue Exception
@@ -46,7 +48,7 @@ module Persist
     end || source.object_id.to_s
   end
 
-  def self.open_database(path, write, serializer = nil, type = "HDB")
+  def self.open_database(path, write, serializer = nil, type = "HDB", options = {})
     case type
     when "LevelDB"
       Persist.open_leveldb(path, write, serializer)
@@ -56,6 +58,13 @@ module Persist
       Persist.open_lmdb(path, write, serializer)
     when 'kch', 'kct'
       Persist.open_kyotocabinet(path, write, serializer, type)
+    when 'fwt'
+      value_size, range, update, in_memory, pos_function = Misc.process_options options.dup, :value_size, :range, :update, :in_memory, :pos_function
+      if pos_function
+        Persist.open_fwt(path, value_size, range, serializer, update, in_memory, &pos_function)
+      else
+        Persist.open_fwt(path, value_size, range, serializer, update, in_memory)
+      end
     else
       Persist.open_tokyocabinet(path, write, serializer, type)
     end
@@ -86,9 +95,9 @@ module Persist
     if is_persisted?(path) and not persist_options[:update]
       Log.debug "TSV persistence up-to-date: #{ path }"
       if persist_options[:shard_function]
-        return open_sharder(path, false, nil, persist_options[:engine], &persist_options[:shard_function]) 
+        return open_sharder(path, false, nil, persist_options[:engine], persist_options, &persist_options[:shard_function]) 
       else
-        return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+        return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
       end
     end
 
@@ -98,9 +107,9 @@ module Persist
           Log.debug "TSV persistence (suddenly) up-to-date: #{ path }"
 
           if persist_options[:shard_function]
-            return open_sharder(path, false, nil, persist_options[:engine], &persist_options[:shard_function]) 
+            return open_sharder(path, false, nil, persist_options[:engine], persist_options, &persist_options[:shard_function]) 
           else
-            return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+            return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
           end
         end
 
@@ -111,9 +120,9 @@ module Persist
         tmp_path = path + '.persist'
 
         data = if persist_options[:shard_function]
-                 open_sharder(tmp_path, true, persist_options[:serializer], persist_options[:engine], &persist_options[:shard_function]) 
+                 open_sharder(tmp_path, true, persist_options[:serializer], persist_options[:engine], persist_options, &persist_options[:shard_function]) 
                else
-                 open_database(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB) 
+                 open_database(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
                end
 
         if TSV === data and data.serializer.nil?
@@ -127,6 +136,7 @@ module Persist
         FileUtils.mv data.persistence_path, path if File.exists? data.persistence_path and not File.exists? path
         tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
         tsv.persistence_path = path
+
         tsv.fix_io if tsv.respond_to? :fix_io
 
         data

data/lib/rbbt/persist/tsv/fix_width_table.rb

@@ -0,0 +1,110 @@
+require 'rbbt/fix_width_table'
+
+module Persist
+
+  module FWTAdapter
+    include Persist::TSVAdapter
+
+    attr_accessor :pos_function
+
+    def self.open(path, value_size, range = false, update = false, in_memory = false, &pos_function)
+      db = CONNECTIONS[path] ||= FixWidthTable.new(path, value_size, range, update, in_memory)
+      db.extend Persist::FWTAdapter
+      db.persistence_path = path
+      db.pos_function = pos_function
+      db
+    end
+
+    def persistence_path=(value)
+      @persistence_path = value
+      @filename = value
+    end
+
+    def metadata_file
+      @metadata_file ||= self.persistence_path + '.metadata'
+    end
+
+    def metadata
+      return {} unless File.exists? metadata_file
+      Open.open(metadata_file, :mode => "rb") do |f|
+        Marshal.load(f)
+      end
+    end
+
+    def set_metadata(k,v)
+      metadata = self.metadata
+      metadata[k] = v
+      Misc.sensiblewrite(metadata_file, Marshal.dump(metadata))
+    end
+
+    def [](key)
+      if TSV::ENTRY_KEYS.include? key
+        metadata[key]
+      else
+        key = pos_function.call(key) if pos_function
+        res = super(key)
+        res.extend MultipleResult
+        res
+      end
+    end
+
+    def []=(key, value)
+      if TSV::ENTRY_KEYS.include? key
+        set_metadata(key, value)
+      else
+        if range
+          add_range_point key, value
+        else
+          add key, value
+        end
+      end
+    end
+     
+    def add(key, value)
+      key = pos_function.call(key) if pos_function and not (range and Array === key)
+      super(key, value)
+    end
+
+    def add_range_point(key, value)
+      key = pos_function.call(key) if pos_function
+      super(key, value)
+    end
+
+    def <<(key, value)
+      self.send(:[]=, *i)
+    end
+
+    def include?(i)
+      return true if Fixnum === i and i < pos(@size)
+      return true if metadata.include? i
+      false
+    end
+
+    def size
+      @size #+ metadata.keys.length
+    end
+
+    def each
+      @size.times do |i|
+        yield i, value(i)
+      end
+    end
+
+    def keys
+      []
+    end
+  end
+
+  def self.open_fwt(path, value_size, range = false, serializer = nil, update = false, in_memory = false, &pos_function)
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::FWTAdapter.open(path, value_size, range, update, in_memory, &pos_function)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+    end
+
+    database
+  end
+end

data/lib/rbbt/persist/tsv/sharder.rb

@@ -1,21 +1,21 @@
 module Persist
   module SharderAdapter
-    def self.open(path, write, type=nil, &block)
+    def self.open(path, write, type=nil, options = {}, &block)
 
-      database = CONNECTIONS[path] ||= Sharder.new(path, write, type, &block)
+      database = CONNECTIONS[path] ||= Sharder.new(path, write, type, options, &block)
 
       database.extend Persist::SharderAdapter unless Persist::SharderAdapter === database
 
       database
     end
-
   end
 
   class Sharder
-    attr_accessor :persistence_path, :shard_function, :databases, :closed, :writable, :mutex, :db_type
+    attr_accessor :persistence_path, :shard_function, :databases, :closed, :writable, :mutex, :db_type, :options
 
-    def initialize(persistence_path, write = false, db_type=nil, &block)
+    def initialize(persistence_path, write = false, db_type=nil, options = {}, &block)
       @shard_function = block
+      @options = options
       @persistence_path = Path.setup(persistence_path)
       @mutex = Mutex.new
       @writable = write
@@ -32,6 +32,7 @@ module Persist
 
     def persistence_path=(path)
       @persistence_path = path
+      databases.values.each{|db| db.persistence_path = File.join(path, File.basename(db.persistence_path))}
     end
 
     def databases
@@ -39,21 +40,32 @@ module Persist
                        hash = {}
                        @persistence_path.glob('shard-*').each do |f|
                          shard = File.basename(f).match(/shard-(.*)/)[1]
-                         hash[shard] = Persist.open_database(f, false, :clean, db_type)
+                         if shard == 'metadata'
+                           hash[shard] = Persist.open_database(f, false, :clean, "HDB", @options)
+                         else
+                           hash[shard] = Persist.open_database(f, false, :clean, db_type, @options)
+                         end
                        end
                        hash
                      end
     end
 
     def database(key)
-      shard = key =~ /__tsv_/ ? "0" : shard_function.call(key)
+      shard = key =~ /__tsv_/ ? "metadata" : shard_function.call(key)
       if databases.include? shard
         databases[shard]
       else
-        database ||= begin
+        if shard == 'metadata'
+          database ||= begin
+                         path = File.join(persistence_path, 'shard-' << shard.to_s)
+                         (writable or File.exists?(path)) ? Persist.open_database(path, writable, :clean, "HDB", @options) : nil
+                     end
+        else
+          database ||= begin
                        path = File.join(persistence_path, 'shard-' << shard.to_s)
-                       (writable or File.exists?(path)) ? Persist.open_database(path, writable, :clean, db_type) : nil
+                       (writable or File.exists?(path)) ? Persist.open_database(path, writable, :clean, db_type, @options) : nil
                      end
+        end
         if database
           databases[shard] = database 
         else
@@ -84,6 +96,7 @@ module Persist
     end
 
     def read(force = false)
+      raise "SIOT"
       return if not write? and not closed and not force
       self.close
       databases.each{|d| d.read }
@@ -204,23 +217,30 @@ module Persist
       databases.values.each{|database| database.write }
     end
 
-    def read
-      databases.values.each{|database| database.read }
+    def read(force = false)
+      databases.values.each{|database| database.read(force) }
     end
 
     def close
       databases.values.each{|database| database.close }
     end
+
+    def size
+      databases.inject(0){|acc,i| 
+        shard, db = i; 
+        acc += db.size 
+      }
+    end
   end
 
-  def self.open_sharder(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB, &shard_function)
+  def self.open_sharder(path, write, serializer = nil, type = TokyoCabinet::HDB, options, &shard_function)
     write = true unless File.exists? path
 
     FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
 
-    database = Persist::SharderAdapter.open(path, write, tokyocabinet_class, &shard_function)
+    database = Persist::SharderAdapter.open(path, write, type, options, &shard_function)
 
-    unless serializer == :clean
+    unless serializer == :clean #or type.to_s == 'fwt'
       TSV.setup database
       database.serializer = serializer if serializer
     end

data/lib/rbbt/resource.rb

@@ -6,6 +6,15 @@ require 'set'
 
  
 module Resource
+
+  class << self
+    attr_accessor :lock_dir
+    
+    def lock_dir
+      @lock_dir ||= Rbbt.tmp.produce_locks.find
+    end
+  end
+
   def self.remote_servers
     @remote_servers = Rbbt.etc.file_servers.exists? ? Rbbt.etc.file_servers.yaml : {}
   end
@@ -65,7 +74,8 @@ module Resource
     begin
       @server_missing_resource_cache ||= Set.new
       raise "Resource Not Found" if @server_missing_resource_cache.include? url
-      Misc.lock final_path do
+      lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
+      Misc.lock lock_filename do
         Net::HTTP.get_response URI(url) do |response|
           case response
           when Net::HTTPSuccess, Net::HTTPOK
@@ -115,15 +125,16 @@ module Resource
     final_path = path.respond_to?(:find) ? (force ? path.find(:user) : path.find) : path
     if not File.exists? final_path or force
       Log.medium "Producing: #{ final_path }"
-      Misc.lock final_path + '.produce' do
+      lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
+      Misc.lock lock_filename do
         if not File.exists? final_path or force
           (remote_server and get_from_server(path, final_path)) or
           begin
             case type
             when :string
-              Open.write(final_path, content)
+              Misc.sensiblewrite(final_path, content)
             when :url
-              Open.write(final_path, Open.open(content))
+              Misc.sensiblewrite(final_path, Open.open(content))
             when :proc
               data = case content.arity
                      when 0
@@ -131,7 +142,7 @@ module Resource
                      when 1
                        content.call final_path
                      end
-              Open.write(final_path, data) unless data.nil?
+              Misc.sensiblewrite(final_path, data) unless data.nil?
             when :rake
               run_rake(path, content, rake_dir)
             when :install

data/lib/rbbt/resource/rake.rb

@@ -38,20 +38,22 @@ module Rake
     raise TaskNotFound if Rake::Task[task].nil?
 
     t = nil
-    #pid = Process.fork{
-    begin
-      Misc.in_dir(dir) do
-        Rake::Task[task].invoke
-
-        Rake::Task.clear
-        Rake::FileTask.clear_files
+    pid = Process.fork{
+      Misc.pre_fork
+      begin
+        Misc.in_dir(dir) do
+          Rake::Task[task].invoke
+
+          Rake::Task.clear
+          Rake::FileTask.clear_files
+        end
+      rescue
+        Log.error "Error in rake: #{$!.message}"
+        raise $!
       end
-    rescue
-      Log.error "Error in rake: #{$!.message}"
-      raise $!
-    end
-    #}
-    #Process.wait(pid)
+    }
+    Process.waitpid(pid)
+    raise "Rake failed" unless $?.success?
 
   end
 end

data/lib/rbbt/tsv/accessor.rb

@@ -94,6 +94,7 @@ module TSV
     begin
       super
     rescue Exception
+      Log.exception $!
       @writable = false
       self
     end
@@ -197,12 +198,7 @@ module TSV
 
   #{{{ GETTERS AND SETTERS
 
-
-  def [](key, clean = false)
-    value = super(key)
-    return value if clean or value.nil?
-    @serializer_module ||= self.serializer_module
-
+  def prepare_value(key, value)
     value = @serializer_module.load(value) if @serializer_module and not TSV::CleanSerializer == @serializer_module
 
     return value if @unnamed or fields.nil?
@@ -218,6 +214,20 @@ module TSV
     value
   end
 
+  def [](key, clean = false)
+    value = super(key)
+    return value if clean or value.nil?
+    @serializer_module ||= self.serializer_module
+
+    if MultipleResult === value
+      res = value.collect{|v| prepare_value key, v }
+      res.extend MultipleResult
+      res
+    else
+      prepare_value key, value
+    end
+  end
+
   def []=(key, value, clean = false)
     return super(key, value) if clean or value.nil? or TSV::CleanSerializer == self.serializer_module 
     super(key, @serializer_module.dump(value))
@@ -634,6 +644,7 @@ Example:
       if merge
         self.through do |key,values|
           field_values = values.delete_at field_pos
+          next if field_values.nil?
           zipped = values.zip_fields
           field_values.zip(zipped).each do |field_value,rest|
             k = [key,field_value]*":"
@@ -648,6 +659,7 @@ Example:
       else
         self.through do |key,values|
           field_values = values.delete_at field_pos
+          next if field_values.nil?
           zipped = Misc.zip_fields(values)
           field_values.zip(zipped).each do |field_value,rest|
             k = [key,field_value]*":"

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -1,7 +1,5 @@
 module TSV
 
-  module MultipleResult; end
-
   def self.obj_stream(obj)
     case obj
     when nil
@@ -510,13 +508,15 @@ module TSV
 
     bar = Misc.process_options options, :bar
     bar ||= Misc.process_options options, :progress
-    max = guess_max(obj)
     options[:bar] = case bar
                     when String
+                      max = guess_max(obj)
                       Log::ProgressBar.new_bar(max, {:desc => bar}) 
                     when TrueClass
+                      max = guess_max(obj)
                       Log::ProgressBar.new_bar(max, nil) 
                     when Fixnum
+                      max = guess_max(obj)
                       Log::ProgressBar.new_bar(bar) 
                     when Hash
                       max = Misc.process_options(bar, :max) || max

data/lib/rbbt/tsv/parser.rb

@@ -554,6 +554,10 @@ module TSV
       self
     end
 
+    def identify_field(field)
+      TSV.identify_field(key_field, fields, field)
+    end
+
     def self.traverse(stream, options = {}, &block)
       parser = Parser.new(stream, options)
       parser.traverse(options, &block)

data/lib/rbbt/tsv/serializers.rb

@@ -1,4 +1,5 @@
 module TSV
+
   class CleanSerializer
     def self.dump(o); o end
     def self.load(o); o end
@@ -26,7 +27,7 @@ module TSV
 
   class StringSerializer
     def self.dump(str); str.to_s; end
-    def self.load(str); str; end
+    def self.load(str); str.dup; end
   end
 
   class StringArraySerializer
@@ -61,7 +62,6 @@ module TSV
     end
   end
 
-
   class TSVSerializer
     def self.dump(tsv)
       tsv.to_s

data/lib/rbbt/tsv/stream.rb

@@ -69,7 +69,6 @@ module TSV
     out = Misc.open_pipe do |sin|
       num_streams = streams.length
 
-
       streams = streams.collect do |stream|
         sorted = Misc.sort_stream(stream)
         stream.annotate sorted if stream.respond_to? :annotate
@@ -150,6 +149,7 @@ module TSV
 
           sin.puts [min, str*sep] * sep
         end
+
         streams.each do |stream|
           stream.join if stream.respond_to? :join
         end

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -3,12 +3,13 @@ require 'rbbt/util/semaphore'
 class RbbtProcessQueue
   class RbbtProcessSocket
 
-    Serializer = Marshal
-
     attr_accessor :sread, :swrite, :write_sem, :read_sem
-    def initialize
+    def initialize(serializer = nil)
       @sread, @swrite = Misc.pipe
 
+      @serializer = serializer || Marshal
+                   
+
       key = "/" << rand(100000000).to_s;
       @write_sem = key + '.in'
       @read_sem = key + '.out'
@@ -30,16 +31,15 @@ class RbbtProcessQueue
       case obj
       when String
         payload = obj
-        size_head = [payload.bytesize,"S"].pack 'La'
+        size_head = [payload.bytesize,"C"].pack 'La'
         str = size_head << payload
       else
-        payload = Serializer.dump(obj)
-        size_head = [payload.bytesize,"M"].pack 'La'
+        payload = @serializer.dump(obj)
+        size_head = [payload.bytesize,"S"].pack 'La'
         str = size_head << payload
       end
 
       write_length = str.length
-      #IO.select(nil, [stream])
       wrote = stream.write(str) 
       while wrote < write_length
         wrote += stream.write(str[wrote..-1]) 
@@ -54,9 +54,9 @@ class RbbtProcessQueue
       begin
         payload = Misc.read_stream stream, size
         case type
-        when "M"
-          Serializer.load(payload)
         when "S"
+          @serializer.load(payload)
+        when "C"
           payload
         end
       rescue TryAgain

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -7,15 +7,6 @@ class RbbtProcessQueue
 
       @pid = Process.fork do
         begin
-
-          #Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
-          #ObjectSpace.each_object(Mutex) do |m| 
-          #  begin 
-          #    m.unlock 
-          #  rescue ThreadError
-          #  end if m.locked? 
-          #end
-          
           Misc.pre_fork
 
           @cleanup.call if @cleanup

data/lib/rbbt/util/misc.rb

@@ -20,6 +20,8 @@ require 'rbbt/util/misc/system'
 require 'rbbt/util/misc/objects'
 require 'rbbt/util/misc/manipulation'
 
+module MultipleResult; end
+
 module Misc
 end
 

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -133,7 +133,6 @@ module ConcurrentStream
 
       abort_threads
       abort_pids
-      iii [:abort, lockfile]
       lockfile.unlock if lockfile and lockfile.locked?
     end
     Log.medium "Aborted stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"

data/test/rbbt/persist/tsv/test_sharder.rb

@@ -35,5 +35,45 @@ class TestSharder < Test::Unit::TestCase
 
     end
   end
+
+  def test_shard_fwt
+    TmpFile.with_file do |dir|
+      shard_function = Proc.new do |key|
+        key[0..(key.index(":")-1)]
+      end
+
+      pos_function = Proc.new do |key|
+        key.split(":").last.to_i
+      end
+
+      size = 10
+      sharder = Persist.persist_tsv(nil, "ShardTest", {}, :update => true, :range => false, :value_size => 64, :engine => 'fwt', :file => dir, :shard_function => shard_function, :pos_function => pos_function, :persist => true, :serializer => :float) do |db|
+        size.times do |v| 
+          v = v + 1
+          chr = "chr" << (v % 5).to_s
+          key = chr + ":" << v.to_s
+          db << [key, v*2]
+        end
+      end
+      sharder.read
+
+      assert_equal dir, sharder.persistence_path
+      assert_equal size, sharder.size
+
+      assert_equal [4.0], sharder["chr2:2"]
+
+      count = 0
+      sharder.through do |k,v|
+        count += 1
+      end
+      assert_equal count, size
+
+      sharder = Persist.open_sharder(dir, false, :float, 'fwt', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
+
+      assert_equal [4.0], sharder["chr2:2"]
+
+      assert_equal size, sharder.size 
+    end
+  end
 end
 

data/test/rbbt/test_packed_index.rb

@@ -0,0 +1,25 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'test_helper.rb')
+require 'rbbt/packed_index'
+
+class TestPackedIndex < Test::Unit::TestCase
+  def test_index
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      100.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
+      pi = PackedIndex.new tmpfile, false
+      100.times do |i|
+        assert_equal i, pi[i][0] 
+        assert_equal i+2, pi[i][1] 
+      end
+      assert_equal nil, pi[100]
+      assert_equal nil, pi[101]
+    end
+  end
+end
+

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -6,7 +6,7 @@ class StopException < StandardError; end
 
 class TestTSVParallelThrough < Test::Unit::TestCase
 
-  def _test_traverse_tsv
+  def test_traverse_tsv
     require 'rbbt/sources/organism'
 
     head = 100
@@ -25,7 +25,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def _test_traverse_tsv_cpus
+  def test_traverse_tsv_cpus
     require 'rbbt/sources/organism'
 
     head = 100
@@ -47,7 +47,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert res.values.compact.flatten.uniq.length > 0
   end
 
-  def _test_traverse_stream
+  def test_traverse_stream
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -61,7 +61,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def _test_traverse_stream_cpus
+  def test_traverse_stream_cpus
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -75,7 +75,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def _test_traverse_stream_keys
+  def test_traverse_stream_keys
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -99,7 +99,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal res.sort, Organism.identifiers("Hsa").tsv(:head => head).keys.sort
   end
   
-  def _test_traverse_array
+  def test_traverse_array
     require 'rbbt/sources/organism'
 
     array = []
@@ -121,7 +121,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array, res
   end
 
-  def _test_traverse_array_threads
+  def test_traverse_array_threads
     require 'rbbt/sources/organism'
 
     array = []
@@ -142,7 +142,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array.sort, res.sort
   end
 
-  def _test_traverse_array_cpus
+  def test_traverse_array_cpus
     require 'rbbt/sources/organism'
 
     array = []
@@ -157,7 +157,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array.sort, res.sort
   end
 
-  def _test_traverse_benchmark
+  def test_traverse_benchmark
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -179,7 +179,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     end
   end
 
-  def _test_traverse_into_dumper
+  def test_traverse_into_dumper
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -197,7 +197,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_into_dumper_threads
+  def test_traverse_into_dumper_threads
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -217,7 +217,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_into_dumper_cpus
+  def test_traverse_into_dumper_cpus
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -238,7 +238,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
 
   #{{{ TRAVERSE DUMPER
 
-  def _test_traverse_dumper
+  def test_traverse_dumper
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -259,7 +259,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_dumper_threads
+  def test_traverse_dumper_threads
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -283,7 +283,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_dumper_cpus
+  def test_traverse_dumper_cpus
     require 'rbbt/sources/organism'
 
     head = 10_000
@@ -305,7 +305,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_dumper_exception
+  def test_traverse_dumper_exception
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -346,7 +346,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     end
   end
 
-  def _test_traverse_into_stream
+  def test_traverse_into_stream
     size = 100
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :into => :stream do |e|
@@ -355,7 +355,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
-  def _test_traverse_progress
+  def test_traverse_progress
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
@@ -389,12 +389,12 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
-  def _test_store_multiple
+  def test_store_multiple
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
       sleep 0.01
-      [e,e+".alt"].extend TSV::MultipleResult
+      [e,e+".alt"].extend MultipleResult
     end
     assert_equal size*2, stream.read.split("\n").length
   end

data/test/rbbt/util/concurrency/processes/test_socket.rb

@@ -11,9 +11,8 @@ class TestConcurrency < Test::Unit::TestCase
     obj3 = "some string"
     obj4 = TSV.setup({1 => 1})
 
-    socket = RbbtProcessQueue::RbbtProcessSocket.new 
+    socket = RbbtProcessQueue::RbbtProcessSocket.new  Marshal
     10.times do
-
       socket.push(obj1)
       socket.push(obj2)
       socket.push(obj3)
@@ -24,7 +23,6 @@ class TestConcurrency < Test::Unit::TestCase
       assert_equal obj3, socket.pop 
       assert_equal obj4, socket.pop 
 
-
     end
 
     socket.swrite.close
@@ -36,4 +34,37 @@ class TestConcurrency < Test::Unit::TestCase
   end
 end
 
+if false and __FILE__ == $0
+  socket = RbbtProcessQueue::RbbtProcessSocket.new 
+
+  obj = "Some string" * 1000
+  Misc.benchmark(1000) do
+    socket.push(obj)
+    socket.pop
+  end
+
+  obj = ["Some string"] * 1000
+  Misc.benchmark(1000) do
+    socket.push(obj)
+    socket.pop
+  end
+  socket.clean
+
+
+  socket = RbbtProcessQueue::RbbtProcessSocket.new Marshal
+
+  obj = "Some string" * 1000
+  Misc.benchmark(1000) do
+    socket.push(obj)
+    socket.pop
+  end
+  socket.clean
 
+  socket = RbbtProcessQueue::RbbtProcessSocket.new TSV::StringArraySerializer
+  obj = ["Some string"] * 1000
+  Misc.benchmark(1000) do
+    socket.push(obj)
+    socket.pop
+  end
+  socket.clean
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.36
+  version: 5.13.37
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-09 00:00:00.000000000 Z
+date: 2014-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -131,10 +131,12 @@ files:
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/knowledge_base.rb
 - lib/rbbt/monitor.rb
+- lib/rbbt/packed_index.rb
 - lib/rbbt/persist.rb
 - lib/rbbt/persist/tsv.rb
 - lib/rbbt/persist/tsv/adapter.rb
 - lib/rbbt/persist/tsv/cdb.rb
+- lib/rbbt/persist/tsv/fix_width_table.rb
 - lib/rbbt/persist/tsv/kyotocabinet.rb
 - lib/rbbt/persist/tsv/leveldb.rb
 - lib/rbbt/persist/tsv/lmdb.rb
@@ -294,6 +296,7 @@ files:
 - test/rbbt/test_fix_width_table.rb
 - test/rbbt/test_knowledge_base.rb
 - test/rbbt/test_monitor.rb
+- test/rbbt/test_packed_index.rb
 - test/rbbt/test_persist.rb
 - test/rbbt/test_resource.rb
 - test/rbbt/test_tsv.rb
@@ -390,6 +393,7 @@ test_files:
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/R/test_eval.rb
+- test/rbbt/test_packed_index.rb
 - test/rbbt/test_association.rb
 - test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb