xrbp 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d22207d3716ce82876aa18980ca4a905cefb569b322d332f81a60879ecff900
-  data.tar.gz: 66f05eab919c17ef84205a7b16afd0a577bdd7d8b19d3c0f2b8a8a90204d7cd8
+  metadata.gz: 4a8b3814b38bdb314bd9c69b5d9f3363b4f0a090f6d41a5849516cc1638d5abe
+  data.tar.gz: fc1ba94351b215389efaacb5013bfa9498a269cf12d7fde3e6958a5fb3f6273a
 SHA512:
-  metadata.gz: 4a1b3a303dce21fff486b306fe95ad5ef930fdc7eaa9331ea2d1c45f28c606d09b6731e96ed326e022873bde6255d67eacc4bddea95df40ee6761cd1314cb096
-  data.tar.gz: 9899f54fe7dad90e19f1eec884c55ea8d5d73b7c109dde1b395d3b0908f9b987b9c6fe6ae7604471e4f0ac98d5dbd0c561e528bb0b9e4fccc39f46bfed6d4199
+  metadata.gz: b8e9e87bc83b3bd361bef7d1deffd87c7637285e15fca3bb5b25f87b34091ebb792137178b06c71d8ac1566a75aadf5ed74fe85d1fc414c5070edecfddce78d4
+  data.tar.gz: 8e8cbea5804ff5f852584be3a699caee69169ad3563bb2a0696e8a3ea9492faa89ae937138bb64c7b65804adc72f49f24538659b85a018b2444b038c4b0b0558

data/examples/nodestore1.rb

@@ -1,14 +1,19 @@
 $: << File.expand_path('../../lib', __FILE__)
 require 'xrbp'
-require 'xrbp/nodestore/backends/rocksdb'
 
-db = XRBP::NodeStore::Backends::RocksDB.new "/var/lib/rippled/rocksdb/rippledb.0899"
+# for rocksdb:
+#require 'xrbp/nodestore/backends/rocksdb'
+#db = XRBP::NodeStore::Backends::RocksDB.new "/home/mmorsi/rippledb/rocksdb/rippledb.beb0"
 
-#ledger = "B506ADD630CB707044B4BFFCD943C1395966692A13DD618E5BD0978A006B43BD"
-#ledger = [ledger].pack("H*")
-#puts db.ledger(ledger)
+# for nudb:
+require 'xrbp/nodestore/backends/nudb'
+db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
+
+ledger = "1B927880277CCF943CB4F6BF57A9BCE48F9583A3DD8EEE9A068D63FB6CD99714"
+ledger = [ledger].pack("H*")
+puts db.ledger(ledger)
 
 #account = "0001bf7468341666f1f47a95e0f4d88e68b5fc7d20d77437cb22954fbbfe6127"
-account = "02c46b3a4130d0a329c47f0da61b829aa5d1ae53c5817e475bcd794e5107be44"
-account = [account].pack("H*")
-puts db.account(account)
+#account = "02c46b3a4130d0a329c47f0da61b829aa5d1ae53c5817e475bcd794e5107be44"
+#account = [account].pack("H*")
+#puts db.account(account)

data/examples/nodestore2.rb

@@ -1,8 +1,13 @@
 $: << File.expand_path('../../lib', __FILE__)
 require 'xrbp'
-require 'xrbp/nodestore/backends/rocksdb'
 
-db = XRBP::NodeStore::Backends::RocksDB.new "/var/lib/rippled/rocksdb/rippledb.0899"
+# for rocksdb:
+#require 'xrbp/nodestore/backends/rocksdb'
+#db = XRBP::NodeStore::Backends::RocksDB.new "/home/mmorsi/rippledb/rocksdb/rippledb.beb0"
+
+# for nudb:
+require 'xrbp/nodestore/backends/nudb'
+db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
 
 db.on :unknown do |hash, node|
   puts "Unknown #{hash}: #{node}"

data/examples/sandbox.rb

@@ -0,0 +1,13 @@
+$: << File.expand_path('../../lib', __FILE__)
+require 'xrbp'
+require 'xrbp/nodestore/backends/nudb'
+
+db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
+store = db.instance_variable_get(:@store)
+
+key = [173, 159, 139, 114, 152, 207, 218, 221, 216, 149, 123, 11, 7, 224, 137, 139, 4, 190, 176, 89, 215, 228, 145, 13, 7, 45, 165, 249, 162, 83, 43, 241].pack("C*")
+val = store.fetch(key).first
+puts val.unpack("C*").join " "
+decompressed = db.send(:decompress, val)
+puts decompressed.unpack("C*").join " "
+type, obj = db.send(:infer_type, decompressed)

data/lib/xrbp/nodestore/backends/decompressor.rb

@@ -0,0 +1,197 @@
+# requires lz4-ruby gem
+require 'lz4-ruby'
+
+module XRBP
+  module NodeStore
+    module Backends
+      # Ported from:
+      #   https://github.com/ripple/rippled/blob/develop/src/ripple/nodestore/impl/codec.h
+      module Decompressor
+        protected
+          def decompress(data)
+            type, remaining = read_varint(data)
+            case(type)
+            when 0
+                                              remaining # uncompressed
+            when 1
+                               decompress_lz4 remaining
+            when 2
+               decompress_compressed_v1_inner remaining
+            when 3
+                     decompress_full_v1_inner remaining
+            when 5
+               decompress_compressed_v2_inner remaining
+            when 6
+                     decompress_full_v2_inner remaining
+
+            else
+              raise "nodeobject codec: bad type=#{type}"
+            end
+          end
+
+        private
+          # https://github.com/ripple/rippled/blob/develop/src/ripple/nodestore/impl/varint.h
+          def read_varint(data)
+            bytes = data.bytes
+
+            t = 0
+            n = 0
+            while(bytes[n] & 0x80) != 0
+              n += 1
+              if n >= bytes.size
+                return t, data
+              end
+            end
+
+            if (n += 1) >= bytes.size
+              return t, data
+            end
+
+            if n == 1 && bytes[0] == 0
+              return t, data[1..-1]
+            end
+
+            used = n
+            while(n -= 1) >= 0
+              d   = bytes[n]
+              t0  = t
+               t *= 127
+               t += d & 0x7F
+               if t <= t0 # overflow
+                 return t, data
+               end
+            end
+
+            return t, data[used..-1]
+          end
+
+          ###
+
+          def decompress_lz4(data)
+            size, remaining = read_varint(data)
+            o = LZ4::Raw::decompress(remaining, size)[0]
+            o
+          end
+
+          ###
+
+          def decompress_compressed_v1_inner(data)
+            raise if data.size < 34
+
+            out = [0, 0, 0, 0,
+                   0, 0, 0, 0,
+                   Format::NODE_OBJ_TYPES[:unknown]] +
+                  [Format::HASH_PREFIXES.invert[:inner_node]].pack("H*")
+                                                             .unpack("C*")
+
+            bytes = data.bytes
+             mask = bytes[0..1].pack("C*").unpack("S").first
+            bytes = bytes[2..-1]
+
+            raise "nodeobject codec v1: empty inner node" if mask == 0
+
+            bit = 0x8000
+              i = 16
+            while i > 0
+              i -= 1
+
+              if (mask & bit) != 0
+                raise "nodeobject codec v1: short inner node" if bytes.size < 32
+
+                out  += bytes[0..31]
+                bytes = bytes[32..-1]
+
+              else
+                out += Array.new(32) { 0 }
+              end
+
+              bit  = bit >> 1
+            end
+
+            out.pack("C*")
+          end
+
+          def decompress_compressed_v2_inner(data)
+            raise if data.size < 67
+
+            out = [0, 0, 0, 0,
+                   0, 0, 0, 0,
+                   Format::NODE_OBJ_TYPES[:unknown]] +
+                  [Format::HASH_PREFIXES.invert[:inner_node_v2]].pack("H*")
+                                                                .unpack("C*")
+
+            bytes = data.bytes
+             mask = bytes[0..1].pack("C*").unpack("S").first
+            bytes = bytes[2..-1]
+
+            raise "nodeobject codec v2: empty inner node" if mask == 0
+
+            depth = bytes[0]
+            bytes = bytes[1..-1]
+
+            bit = 0x8000
+              i = 16
+            while i > 0
+              i -= 1
+
+              if (mask & bit) != 0
+                raise "nodeobject codec v1: short inner node" if bytes.size < 32
+
+                out  += bytes[0..31]
+                bytes = bytes[32..-1]
+
+              else
+                out += Array.new(32) { 0 }
+              end
+
+              bit  = bit >> 1
+            end
+
+            out << depth
+
+             copy = (depth + 1)/2
+            raise "nodeobject codec v2: short inner node" if bytes.size < copy
+             out += bytes[0...copy]
+            bytes = bytes[copy..-1]
+            raise "nodeobject codec v2: long inner node" if bytes.size > 0
+
+             out.pack("C*")
+          end
+
+          ###
+
+          def decompress_full_v1_inner(data)
+            raise if data.size != 512 # 16 32-bit hashes
+
+            out = [0, 0, 0, 0,
+                   0, 0, 0, 0,
+                   Format::NODE_OBJ_TYPES[:unknown]] +
+                  [Format::HASH_PREFIXES.invert[:inner_node]].pack("H*").unpack("C*")
+            (out + data[0...512].bytes).pack("C*")
+          end
+
+
+          def decompress_full_v2_inner(data)
+            bytes = data.bytes
+            depth = bytes[0]
+            bytes = bytes[1..-1]
+             copy = (depth + 1)/2
+
+            raise if bytes.size != 512 + copy # 16 32-bit hashes + copy
+
+            out = [0, 0, 0, 0,
+                   0, 0, 0, 0,
+                   Format::NODE_OBJ_TYPES[:unknown]] +
+                  [Format::HASH_PREFIXES.invert[:inner_node_v2]].pack("H*").unpack("C*")
+
+             out += bytes[0..511]
+            bytes = bytes[512..-1]
+             out << depth
+             out += bytes[0...copy]
+
+            out.pack("C*")
+          end
+      end # module Decompressor
+    end # module Backends
+  end # module NodeStore
+end # module XRBP

data/lib/xrbp/nodestore/backends/nudb.rb

@@ -1 +1,78 @@
-# TODO
+require 'ostruct'
+
+# requires rrudb gem
+require "rudb"
+
+require_relative './decompressor'
+
+module XRBP
+  module NodeStore
+    module Backends
+      class NuDB < DB
+        include Decompressor
+
+        attr_reader :path
+
+        KEY_SIZE = 32
+
+        def initialize(path)
+          @path = path
+          create!
+          open
+        end
+
+        def [](key)
+          decompress(@store.fetch(key)[0])
+        end
+
+        def each
+          dat = File.join(path, "nudb.dat")
+
+          RuDB::each(dat) do |key, val|
+            val = decompress(val)
+            type, obj = infer_type(val)
+
+            if type
+              emit type, key, obj
+            else
+              emit :unknown, key, val
+            end
+
+            # 'mock' iterator
+            iterator = OpenStruct.new(:key   => key,
+                                      :value => val)
+            yield iterator
+          end
+        end
+
+        private
+
+        def create!
+          dat = File.join(path, "nudb.dat")
+          key = File.join(path, "nudb.key")
+          log = File.join(path, "nudb.log")
+
+          RuDB::create :dat_path    => dat,
+                       :key_path    => key,
+                       :log_path    => log,
+                       :app_num     =>   1,
+                       :salt        => RuDB::make_salt,
+                       :key_size    => KEY_SIZE,
+                       :block_size  => RuDB::block_size(key),
+                       :load_factor => 0.5
+        end
+
+
+
+        def open
+          dat = File.join(path, "nudb.dat")
+          key = File.join(path, "nudb.key")
+          log = File.join(path, "nudb.log")
+
+          @store = RuDB::Store.new
+          @store.open(dat, key, log)
+        end
+      end # class NuDB
+    end # module Backends
+  end # module NodeStore
+end # module XRBP

data/lib/xrbp/nodestore/backends/rocksdb.rb

@@ -1,3 +1,4 @@
+# requires rocksdb-ruby gem
 require "rocksdb"
 
 module XRBP

data/lib/xrbp/nodestore/db.rb

@@ -7,6 +7,8 @@ module XRBP
       include Enumerable
       include EventEmitter
 
+      # TODO return nil if db lookup not found
+
       def ledger(hash)
         parse_ledger(self[hash])
       end
@@ -258,6 +260,7 @@ module XRBP
         sha256 = OpenSSL::Digest::SHA256.new
         digest = sha256.digest(sha256.digest(acct))[0..3]
         acct  += digest
+        acct.force_encoding(Encoding::BINARY) # required for Base58 gem
         return Base58.binary_to_base58(acct, :ripple), data[vl..-1]
       end
 

data/lib/xrbp/nodestore/format.rb

@@ -3,6 +3,13 @@ require "bistro"
 module XRBP
   module NodeStore
     module Format
+      NODE_OBJ_TYPES = {
+        :unknown          => 0,
+        :ledger           => 1,
+        :account_node     => 2,
+        :transaction_node => 3
+      }
+
       NODE_TYPES = {
         1 => :ledger,
         2 => :tx,
@@ -15,6 +22,7 @@ module XRBP
         "534E4400" => :tx_node,
         "4D4C4E00" => :leaf_node,
         "4D494E00" => :inner_node,
+        "494E5200" => :inner_node_v2,
         "4C575200" => :ledger_master,
         "53545800" => :tx_sign,
         "56414C00" => :validation,

data/lib/xrbp/version.rb

@@ -1,3 +1,3 @@
 module XRBP
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
 end # module XRBP

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xrbp
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Dev Null Productions
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-09 00:00:00.000000000 Z
+date: 2019-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -196,6 +196,7 @@ files:
 - examples/paginate.rb
 - examples/prioritized.rb
 - examples/round_robin.rb
+- examples/sandbox.rb
 - examples/username.rb
 - examples/validator.rb
 - examples/websocket.rb
@@ -228,6 +229,7 @@ files:
 - lib/xrbp/model/parsers/validator.rb
 - lib/xrbp/model/validator.rb
 - lib/xrbp/nodestore.rb
+- lib/xrbp/nodestore/backends/decompressor.rb
 - lib/xrbp/nodestore/backends/nudb.rb
 - lib/xrbp/nodestore/backends/rocksdb.rb
 - lib/xrbp/nodestore/db.rb