rbbt-util 5.13.37 → 5.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26df367cce98f5b14b985bd311d4b6594dcb62a9
-  data.tar.gz: 1931a495ac7eb8eb17dcfa0f154be8c9d61e8b7f
+  metadata.gz: 2c1f84fdf126a66b0e8cf03cd96237dd4625c5d7
+  data.tar.gz: 6e9b9ed9bc339fedaa99f1a96d8f537692ebfbb6
 SHA512:
-  metadata.gz: e71fee09c427fdf3faf326b75897dc6d070ec605bc94b3785d8cacf88f2bd3fa1fdd8d017e5e06d5e48cff48960cb8e7debb337b3a4eeb3361d2fd3735b2b1f3
-  data.tar.gz: 14a1b01824edd6bfa2a6ead19828454ea57b1c3846eb1f332a0b15faee9bf5019fdda715732ef57b1326ce3870d8c2f8d8c6c9f2faf6e70218ceea76f373436d
+  metadata.gz: 7879d8241fc10774522d28484ae1cf37bd935067038b1f9e370a2ff4af51adb9f7f183b43c7852694ce4e8b44282ab937b3744b7d8d35072a86e3ccbfb51c48e
+  data.tar.gz: 12ecd4f1f116f4125bb9f4159ab4556910b6e31d0fc3a351c94667758507296914445091575fc8946c975cc305da957949fa79c25913c990e477aed56506e0a0

data/bin/rbbt

@@ -41,9 +41,10 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --profile               #{Log.color :yellow, "Profile execution"}
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
+--nostream              #{Log.color :yellow, "Disable persistance/job streaming"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
 --dump_mem*             #{Log.color :yellow, "Dump strings in memory each second into file"}
---no_lock_id           #{Log.color :yellow, "Do not track lockfiles with ids (for high-througput and high-concurrency"}
+--no_lock_id            #{Log.color :yellow, "Do not track lockfiles with ids (for high-througput and high-concurrency"}
 EOF
 
 
@@ -70,6 +71,10 @@ if mem_dump = options.delete(:dump_mem)
   Rbbt.dump_memory(mem_dump, Symbol)
 end
 
+if options.delete :stream
+  ENV["RBBT_NO_STREAM"] = "true"
+end
+
 if options.delete :nobar
   ENV["RBBT_NO_PROGRESS"] = "true"
 end

data/lib/rbbt/fix_width_table.rb

@@ -1,6 +1,6 @@
 class FixWidthTable
 
-  attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask
+  attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
   def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
     @filename = filename
 
@@ -9,6 +9,7 @@ class FixWidthTable
       @value_size  = value_size
       @range       = range
       @record_size = @value_size + (@range ? 16 : 8)
+      @write = true
 
       if %w(memory stringio).include? filename.to_s.downcase
         @filename = :memory
@@ -21,6 +22,7 @@ class FixWidthTable
 
       @file.write [value_size].pack("L")
       @file.write [@range ? 1 : 0 ].pack("C")
+
       @size = 0
     else
       Log.debug "FixWidthTable up-to-date: #{ filename }"
@@ -31,11 +33,17 @@ class FixWidthTable
       end
       @value_size  = @file.read(4).unpack("L").first
       @range       = @file.read(1).unpack("C").first == 1
-      @record_size = @value_size + (@range ? 12 : 4)
+      @record_size = @value_size + (@range ? 16 : 8)
+      @write = false
+
       @size        = (File.size(@filename) - 5) / (@record_size)
     end
 
-    @mask = "a#{value_size}"
+    @mask = "a#{@value_size}"
+  end
+
+  def write?
+    @write
   end
 
   def persistence_path
@@ -46,15 +54,14 @@ class FixWidthTable
     @filename=value
   end
 
-  CONNECTIONS = {} unless defined? CONNECTIONS
   def self.get(filename, value_size = nil, range = nil, update = false)
     return self.new(filename, value_size, range, update) if filename == :memory
     case
-    when (!File.exists?(filename) or update or not CONNECTIONS.include?(filename))
-      CONNECTIONS[filename] = self.new(filename, value_size, range, update)
+    when (!File.exists?(filename) or update or not Persist::CONNECTIONS.include?(filename))
+      Persist::CONNECTIONS[filename] = self.new(filename, value_size, range, update)
     end
 
-    CONNECTIONS[filename] 
+    Persist::CONNECTIONS[filename] 
   end
 
   def format(pos, value)
@@ -100,16 +107,19 @@ class FixWidthTable
     @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
     padding = @file.read(4).unpack("l").first+1
     txt = @file.read(value_size)
-    txt.unpack(mask).first[0..-padding]
+    str = txt.unpack(mask).first
+    padding > 1 ? str[0..-padding] : str
   end
 
   def read(force = false)
     return if @filename == :memory
+    @write = false
     @file.close unless @file.closed?
     @file = File.open(filename, 'r:ASCII-8BIT')
   end
 
   def close
+    @write = false
     @file.close
   end
 
@@ -191,7 +201,9 @@ class FixWidthTable
 
     idx = 0 if idx < 0
 
-    idx -= overlap(idx) unless overlap(idx).nil?
+    overlap = overlap(idx)
+
+    idx -= overlap unless overlap.nil?
 
     values = []
     l_start = pos(idx)

data/lib/rbbt/monitor.rb

@@ -4,7 +4,7 @@ module Rbbt
 
   LOCK_DIRS = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all +  Rbbt.var.jobs.find_all +
     Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all + Rbbt.tmp.sensiblewrite_lock_dir.find_all +
-    Rbbt.tmp.produce_locks.find_all
+    Rbbt.tmp.produce_locks.find_all + Rbbt.tmp.step_info_locks.find_all
 
   SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
 

data/lib/rbbt/packed_index.rb

@@ -2,10 +2,10 @@ class PackedIndex
   attr_accessor :file, :mask, :mask_length, :offset, :item_size, :stream, :nil_string
 
   ELEMS = {
-    "I" => ["q", 8],
     "i" => ["l", 4],
+    "I" => ["q", 8],
     "f" => ["f", 4],
-    "F" => ["D", 8],
+    "F" => ["d", 8],
   }
 
   def self.process_mask(mask)
@@ -28,6 +28,12 @@ class PackedIndex
     [str, size]
   end
 
+  def size
+    @size ||= begin
+                (File.size(file) - offset) / item_size
+              end
+  end
+
   def initialize(file, write = false, pattern = nil)
     @file = file
     if write
@@ -44,10 +50,18 @@ class PackedIndex
       @mask = @stream.read(mask_length)
       @offset = @mask.length + 8
     end
-    @nil_string = "[NIL]" + "-" * (@item_size - 5)
+    @nil_string = "NIL" << ("-" * (@item_size - 3))
+  end
+
+  def persistence_path
+    @file
+  end
+
+  def persistence_path=(value)
+    @file=value
   end
 
-  def read
+  def read(force = false)
     close
     @stream = Open.open(file, :mode => 'rb')
   end
@@ -63,7 +77,7 @@ class PackedIndex
   def [](position)
     @stream.seek(position * item_size + offset)
     encoded = @stream.read(item_size)
-    return nil if encoded == nil_string
+    return nil if encoded.nil? or encoded == nil_string 
     encoded.unpack mask
   end
 

data/lib/rbbt/persist/tsv.rb

@@ -1,6 +1,7 @@
 require 'rbbt/persist/tsv/adapter'
 
 require 'rbbt/persist/tsv/fix_width_table'
+require 'rbbt/persist/tsv/packed_index'
 
 begin
   require 'rbbt/persist/tsv/tokyocabinet'
@@ -65,6 +66,13 @@ module Persist
       else
         Persist.open_fwt(path, value_size, range, serializer, update, in_memory)
       end
+    when 'pki'
+      pattern, pos_function = Misc.process_options options.dup, :pattern, :pos_function
+      if pos_function
+        Persist.open_pki(path, write, pattern, &pos_function)
+      else
+        Persist.open_pki(path, write, pattern)
+      end
     else
       Persist.open_tokyocabinet(path, write, serializer, type)
     end
@@ -113,7 +121,7 @@ module Persist
           end
         end
 
-        FileUtils.rm path if File.exists? path
+        FileUtils.rm_rf path if File.exists? path
 
         Log.medium "TSV persistence creating: #{ path }"
 

data/lib/rbbt/persist/tsv/fix_width_table.rb

@@ -81,7 +81,7 @@ module Persist
     end
 
     def size
-      @size #+ metadata.keys.length
+      @size
     end
 
     def each

data/lib/rbbt/persist/tsv/packed_index.rb

@@ -0,0 +1,101 @@
+require 'rbbt/packed_index'
+
+module Persist
+
+  module PKIAdapter
+    include Persist::TSVAdapter
+
+    attr_accessor :pos_function
+
+    def self.open(path, write, pattern, &pos_function)
+      db = CONNECTIONS[path] ||= PackedIndex.new(path, write, pattern)
+      db.extend Persist::PKIAdapter
+      db.persistence_path = path
+      db.pos_function = pos_function
+      db
+    end
+
+    def persistence_path=(value)
+      @persistence_path = value
+      @file = value
+    end
+
+    def metadata_file
+      @metadata_file ||= self.persistence_path + '.metadata'
+    end
+
+    def metadata
+      return {} unless File.exists? metadata_file
+      Open.open(metadata_file, :mode => "rb") do |f|
+        Marshal.load(f)
+      end
+    end
+
+    def set_metadata(k,v)
+      metadata = self.metadata
+      metadata[k] = v
+      Misc.sensiblewrite(metadata_file, Marshal.dump(metadata))
+    end
+
+    def [](key, clean = false)
+      if TSV::ENTRY_KEYS.include? key
+        metadata[key]
+      else
+        key = pos_function.call(key) if pos_function and not clean
+        res = super(key)
+        res.extend MultipleResult unless res.nil?
+        res
+      end
+    end
+
+    def value(pos)
+      self.send(:[], pos, true)
+    end
+
+    def []=(key, value)
+      if TSV::ENTRY_KEYS.include? key
+        set_metadata(key, value)
+      else
+        add key, value
+      end
+    end
+     
+    def add(key, value)
+      key = pos_function.call(key) if pos_function 
+      self.send(:<<, value)
+    end
+
+    def add_range_point(key, value)
+      key = pos_function.call(key) if pos_function
+      super(key, value)
+    end
+
+    def include?(i)
+      return true if Fixnum === i and i < size
+      return true if metadata.include? i
+      false
+    end
+
+    def each
+      size.times do |i|
+        yield i, value(i)
+      end
+    end
+
+    def keys
+      []
+    end
+  end
+
+  def self.open_pki(path, write, pattern, &pos_function)
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::PKIAdapter.open(path, write, pattern, &pos_function)
+
+    #TSV.setup database
+
+    #database.serializer = :clean
+
+    database
+  end
+end

data/lib/rbbt/persist/tsv/sharder.rb

@@ -205,7 +205,9 @@ module Persist
     end
 
     def [](key, clean=false)
-      v = database(key).send(:[], key)
+      database = database(key)
+      return nil if database.nil?
+      v = database.send(:[], key)
     end
 
     def <<(p)
@@ -240,9 +242,15 @@ module Persist
 
     database = Persist::SharderAdapter.open(path, write, type, options, &shard_function)
 
-    unless serializer == :clean #or type.to_s == 'fwt'
+    if type.to_s == 'pki'
       TSV.setup database
-      database.serializer = serializer if serializer
+      database.type = :list
+      database.serializer = :clean 
+    else
+      if serializer != :clean 
+        TSV.setup database
+        database.serializer = serializer if serializer
+      end
     end
 
     database

data/lib/rbbt/resource/path.rb

@@ -154,7 +154,7 @@ module Path
     begin
       self.produce
       File.exists? self.find
-    rescue
+    rescue Exception
       false
     end
   end

data/lib/rbbt/resource/rake.rb

@@ -49,6 +49,7 @@ module Rake
         end
       rescue
         Log.error "Error in rake: #{$!.message}"
+        Log.exception $!
         raise $!
       end
     }

data/lib/rbbt/tsv/accessor.rb

@@ -524,19 +524,8 @@ module TSV
     end
   end
 
-  def to_s(keys = nil, no_options = false)
-    if FalseClass === keys or TrueClass === keys
-      no_options = keys
-      keys = nil
-    end
-
-    if keys == :sort
-      with_unnamed do
-        keys = self.keys.sort
-      end
-    end
-
-    io = TSV::Dumper.stream self do |dumper|
+  def dumper_stream(keys = nil, no_options = false)
+    TSV::Dumper.stream self do |dumper|
       dumper.init unless no_options
       begin
         if keys
@@ -554,8 +543,24 @@ module TSV
         Log.exception $!
         raise $!
       end
+      dumper.close
+    end
+  end
+
+  def to_s(keys = nil, no_options = false)
+    if FalseClass === keys or TrueClass === keys
+      no_options = keys
+      keys = nil
+    end
+
+    if keys == :sort
+      with_unnamed do
+        keys = self.keys.sort
+      end
     end
 
+    io = dumper_stream(keys, no_options)
+
     str = ''
     while block = io.read(2048)
       str << block

data/lib/rbbt/tsv/dumper.rb

@@ -4,12 +4,8 @@ module TSV
     def self.stream(options = {}, filename = nil, &block)
       dumper = TSV::Dumper.new options, filename
       Thread.new(Thread.current) do |parent|
-        begin
-          yield dumper
-          dumper.close
-        rescue Exception
-          raise $!
-        end
+        yield dumper
+        dumper.close
       end
       dumper.stream
     end