xrbp 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '096cdf11861212228f99c1f93fca43c0ca503107bbd4602356e55991397a29a9'
-  data.tar.gz: b6178472d0dd06f47fb38632d5458ea7ffa86720a657e03e3e9c1cc640d2d4e6
+  metadata.gz: 80c7817a86dc903b6eb5725bb37ecd562fe6a73b1aef5ee81e2b27ff1aeb50a6
+  data.tar.gz: 7003cb99a4488aea91f41d4b8c6bf2fc3bdc577b980d251c858639508f6ae1f1
 SHA512:
-  metadata.gz: 67870bc15932d4b2877ac4fffd4bf07221c8ef4834f0a41f72bffa621f246239f5194922c1d9fbb4933d83054eb88c8bffef93f0b95a9940a1389cb74fab69fd
-  data.tar.gz: 1afd32b1bdf8b9b5be99337da2fc4fd91f29d6dbd8c5deb90a3bf14cbeb9d4a479177c26e4040edf3310b18b11d9d79e1d1532af2855eeefad5b50710211d44b
+  metadata.gz: 9792f31b6e62658be6b94f32b3dae6a8fd99a395b5b513d7d6f1cf1e766a121bc074a043b8a1f884c516bda807e06243806f83f7ad2e397655d7ab69b09a5e51
+  data.tar.gz: 4e360f3e4464758f0b49158239d898b8b901d094ea5be52faadfaa29dccb4f12120c392bcce1d2c0a03a1058b8345bcce6016abce59ec9dd3524bb75142ec3a6

data/.yardopts

@@ -1 +1,2 @@
 --no-private
+--exclude ripple.proto.rb

data/examples/nodestore1.rb

@@ -2,18 +2,18 @@ $: << File.expand_path('../../lib', __FILE__)
 require 'xrbp'
 
 # for rocksdb:
-#require 'xrbp/nodestore/backends/rocksdb'
-#db = XRBP::NodeStore::Backends::RocksDB.new "/home/mmorsi/rippledb/rocksdb/rippledb.beb0"
+require 'xrbp/nodestore/backends/rocksdb'
+db = XRBP::NodeStore::Backends::RocksDB.new "/var/lib/rippled/rocksdb/rippledb.0899"
 
 # for nudb:
-require 'xrbp/nodestore/backends/nudb'
-db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
+#require 'xrbp/nodestore/backends/nudb'
+#db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
 
-ledger = "1B927880277CCF943CB4F6BF57A9BCE48F9583A3DD8EEE9A068D63FB6CD99714"
-ledger = [ledger].pack("H*")
-puts db.ledger(ledger)
+#ledger = "B506ADD630CB707044B4BFFCD943C1395966692A13DD618E5BD0978A006B43BD"
+#ledger = [ledger].pack("H*")
+#puts db.ledger(ledger)
 
 #account = "0001bf7468341666f1f47a95e0f4d88e68b5fc7d20d77437cb22954fbbfe6127"
-#account = "02c46b3a4130d0a329c47f0da61b829aa5d1ae53c5817e475bcd794e5107be44"
-#account = [account].pack("H*")
-#puts db.account(account)
+account = "02c46b3a4130d0a329c47f0da61b829aa5d1ae53c5817e475bcd794e5107be44"
+account = [account].pack("H*")
+puts db.account(account)

data/examples/nodestore2.rb

@@ -2,12 +2,12 @@ $: << File.expand_path('../../lib', __FILE__)
 require 'xrbp'
 
 # for rocksdb:
-#require 'xrbp/nodestore/backends/rocksdb'
-#db = XRBP::NodeStore::Backends::RocksDB.new "/home/mmorsi/rippledb/rocksdb/rippledb.beb0"
+require 'xrbp/nodestore/backends/rocksdb'
+db = XRBP::NodeStore::Backends::RocksDB.new "/var/lib/rippled/rocksdb/rippledb.0899"
 
 # for nudb:
-require 'xrbp/nodestore/backends/nudb'
-db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
+#require 'xrbp/nodestore/backends/nudb'
+#db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
 
 db.on :unknown do |hash, node|
   puts "Unknown #{hash}: #{node}"

data/examples/p2p.rb

@@ -1,5 +1,3 @@
-gem 'openssl', '2.1.3'
-
 $: << File.expand_path('../../lib', __FILE__)
 require 'xrbp'
 
@@ -9,6 +7,7 @@ puts overlay.handshake.response
 
 overlay.read_frames do |frame|
   puts "Message: #{frame.type_name} (#{frame.size} bytes)"
+  # frame.message => protobuf message
 end
 
 overlay.close

data/lib/xrbp/crypto.rb

@@ -1,3 +1,12 @@
+module XRBP
+  # The Crypto module defines methods to create XRPL compliant
+  # keys and subsequent entities (accounts, nodes, validators).
+  module Crypto
+  end # module Crypto
+end # module XRBP
+
+require 'xrbp/crypto/seed'
 require 'xrbp/crypto/key'
 require 'xrbp/crypto/account'
 require 'xrbp/crypto/node'
+require 'xrbp/crypto/validator'

data/lib/xrbp/crypto/account.rb

@@ -2,16 +2,18 @@ require 'base58'
 
 module XRBP
   module Crypto
+    # Generate and new XRPL account.
+    #
+    # @param key [Symbol, Hash] key type to generate or key itself (optional)
+    # @return [Hash] account details containing id and pub/priv key pair
     def self.account(key=nil)
       pub = nil
       if key == :secp256k1 || key.nil?
         key = Key::secp256k1
-        key[:type] = :secp256k1
         pub = key[:public]
 
       elsif key == :ed25519
         key = Key::ed25519
-        key[:type] = :ed25519
         pub = "\xED" + key[:public]
 
       elsif key.is_a?(Hash)
@@ -29,5 +31,29 @@ module XRBP
 
       { :account => Base58.binary_to_base58(account_id  + chksum, :ripple) }.merge(key)
     end
+
+    # Extract Account ID from Address.
+    #
+    # @param account [String] Base58 encoded account address
+    # @return [String, nil] unique account id or nil if input
+    #   if not an account
+    def self.parse_account(account)
+      bin = Base58.base58_to_binary(account, :ripple)
+      typ = bin[0]
+      chk = bin[-4..-1]
+      bin = bin[1...-4]
+
+      return nil unless typ.unpack("C*").first == Key::TOKEN_TYPES[:account_id]
+
+      sha256 = OpenSSL::Digest::SHA256.new
+      return nil unless sha256.digest(sha256.digest(typ + bin))[0..3] == chk
+
+      return bin
+    end
+
+    # Return boolean indicating if the specified account is valid
+    def self.account?(account)
+      parse_account(account) != nil
+    end
   end # module Crypto
 end # module XRBP

data/lib/xrbp/crypto/key.rb

@@ -16,12 +16,8 @@ module XRBP
 
       ###
 
-      def self.priv
-        seed = SecureRandom.random_bytes(32)
-        OpenSSL::Digest::SHA256.new.digest(seed)
-      end
-
-      def self.secp256k1
+      # @return [Hash] new secp256k1 key pair (both public and private components)
+      def self.secp256k1(seed=nil)
           # XXX: the bitcoin secp256k1 implementation (which rippled pulls in / vendors)
           #      has alot of nuances which require special configuration in openssl. For
           #      the time being, mitigate this by pulling in & using the ruby
@@ -34,36 +30,62 @@ module XRBP
 
           spk = Secp256k1::PrivateKey.new
 
-          # XXX: I'd like to generate the private key first & set,
-          #      but for some reason this doesn't work. When the
-          #      keys are loaded for signing/verification later
-          #      the public key is not able to verify the signature
-          #      generated from the private key set in this way.
-          # TODO: Investigate
-           #pk = priv
-          #spk.set_raw_privkey [pk].pack("H*")
+          sd, pk = nil
+          if seed
+            sd,pk = seed,seed
+
+          else
+            sd = Crypto.seed[:seed]
+            pk = Crypto.parse_seed(sd)
+          end
+
+          # FIXME: rippled & ripple-keypairs (& by extension ripple-lib) repeatedly
+          #        hash seed until certain it is less than the order of the
+          #        curve, we should do this as well (for security)
+          #
+          #        https://github.com/ripple/rippled/blob/develop/src/ripple/crypto/impl/GenerateDeterministicKey.cpp
+          #        https://github.com/ripple/ripple-keypairs/blob/master/src/secp256k1.js
+          #
+          #        Also look into if setting raw key here has same effect as
+          #        secp256k1.mul (as invoked in keypairs)
+          sha512 = OpenSSL::Digest::SHA512.new
+          pk = sha512.digest(pk)[0..31]
+
+          spk.set_raw_privkey pk
 
           {  :public => spk.pubkey.serialize.unpack("H*").first,
-            :private => spk.send(:serialize) }
+            :private => spk.send(:serialize),
+               :seed => sd,
+               :type => :secp256k1 }
       end
 
+      # @return [Hash] new ed25519 key pair (both public and private components)
       def self.ed25519
         # XXX openssl 1.1.1 needed for EdDSA support:
         #     https://www.openssl.org/blog/blog/2018/09/11/release111/
         #     Until then use this:
         require "ed25519"
 
-        # FIXME: this works for now (eg generates valid keys),
-        #        but we should do this in the same way rippled does:
-        #        Generate private key, then generate corresponding
-        #        Ed25519 public key
-        key = Ed25519::SigningKey.generate
-        {  :public => key.to_bytes.unpack("H*").first.upcase,
-          :private => key.verify_key.to_bytes.unpack("H*").first.upcase }
+        sd = Crypto.seed[:seed]
+        pk = Crypto.parse_seed(sd)
+
+        sha512 = OpenSSL::Digest::SHA512.new
+        pk = sha512.digest(pk)[0..31]
+
+        key = Ed25519::SigningKey.new(pk)
+        {  :public => key.verify_key.to_bytes.unpack("H*").first.upcase,
+          :private => key.to_bytes.unpack("H*").first.upcase,
+             :seed => sd,
+             :type => :ed25519 }
       end
 
       ###
 
+      # Sign the digest using the specified key, returning the result
+      #
+      # @param key [Hash] key to sign digest with
+      # @param data [String] data to sign (must be exactly 32 bytes long!)
+      # @return [String] signed digest
       def self.sign_digest(key, data)
         raise "unknown key" unless key.is_a?(Hash) && key[:type] && key[:private]
         raise "invalid data" unless data.length == 32
@@ -79,15 +101,57 @@ module XRBP
           sig_raw = pk.ecdsa_sign data, raw: true
           return pk.ecdsa_serialize sig_raw
 
-        #elsif key[:type] == :ed25519
-          # TODO
+        elsif key[:type] == :ed25519
+          # XXX: see note about this library above
+          require "ed25519"
+
+          sd = key[:seed]
+          pk = Crypto.parse_seed(sd)
+
+          sha512 = OpenSSL::Digest::SHA512.new
+          pk = sha512.digest(pk)[0..31]
+
+          pk = Ed25519::SigningKey.new(pk)
+          return pk.sign(data)
         end
 
         raise "unknown key type"
       end
 
+      # Returns bool indicating if data is the result of
+      # signing expected value with given key.
+      #
+      # @param key [Hash] key to use to verify digest
+      # @param data [String] signed data
+      # @param expected [String] original unsigned data
+      # @return [Bool] indicating if signed digest matches
+      #   original data
       def self.verify(key, data, expected)
+        if key[:type] == :secp256k1
+          # XXX: see note about this library above
+          require 'secp256k1'
+
+          pb = Secp256k1::PublicKey.new :pubkey => [key[:public]].pack("H*"),
+                                           :raw => true
+          pv = Secp256k1::PrivateKey.new
+
+          return pb.ecdsa_verify expected,
+                 pv.ecdsa_deserialize(data), raw: true
+
+        elsif key[:type] == :ed25519
+          # XXX: see note about this library above
+          require "ed25519"
+
+          pk = Ed25519::VerifyKey.new([key[:public]].pack("H*"))
+          begin
+            return pk.verify(data, expected)
+          rescue Ed25519::VerifyError
+            return false
+          end
+        end
+
+        raise "unknown key type"
       end
-    end
+    end # module Key
   end # module Crypto
 end # module XRBP

data/lib/xrbp/crypto/node.rb

@@ -2,14 +2,18 @@ require 'base58'
 
 module XRBP
   module Crypto
+    # Generate a new XRPL node.
+    #
+    # @param key [Symbol, Hash] key type to generate or key itself (optional)
+    # @return [Hash] node details containing id and pub/priv key pair
     def self.node(key=nil)
       pub = nil
       if key == :secp256k1 || key.nil?
         key = Key::secp256k1
-        key[:type] = :secp256k1
         pub = key[:public]
 
       elsif key.is_a?(Hash)
+        key = Key::secp256k1(key[:seed]) if key[:seed]
         pub = key[:public]
 
       else
@@ -23,5 +27,29 @@ module XRBP
 
       { :node => Base58.binary_to_base58(node_id + chksum, :ripple) }.merge(key)
     end
+
+    # Extract Node ID from Address.
+    #
+    # @param node [String] Base58 encoded node address
+    # @return [String, nil] unique node id or nil if input
+    #   if not an node
+    def self.parse_node(node)
+      bin = Base58.base58_to_binary(node, :ripple)
+      typ = bin[0]
+      chk = bin[-4..-1]
+      bin = bin[1...-4]
+
+      return nil unless typ.unpack("C*").first == Key::TOKEN_TYPES[:node_public]
+
+      sha256 = OpenSSL::Digest::SHA256.new
+      return nil unless sha256.digest(sha256.digest(typ + bin))[0..3] == chk
+
+      return bin
+    end
+
+    # Return boolean indicating if the specified node is valid
+    def self.node?(node)
+      parse_node(node) != nil
+    end
   end # module Crypto
 end # module XRBP

data/lib/xrbp/crypto/seed.rb

@@ -0,0 +1,58 @@
+require 'base58'
+require 'securerandom'
+
+module XRBP
+  module Crypto
+    # Generate a new XRPL seed.
+    #
+    # @param key [Symbol] key type to generate (optional)
+    # @return [Hash] seed details containing encoded seed and key type
+    def self.seed(key=nil)
+      prefix = nil
+      if key == :secp256k1 || key.nil?
+        prefix = [Key::TOKEN_TYPES[:family_seed]]
+        key = { :type => :secp256k1 }
+
+      elsif key == :ed25519
+        prefix = [0x01, 0xE1, 0x4B]
+        key = { :type => :ed25519 }
+
+      else
+        raise ArgumentError, key
+      end
+
+      sha256 = OpenSSL::Digest::SHA256.new
+      base = SecureRandom.random_bytes(16)
+      pref = (prefix + base.bytes).pack("C*")
+      chk = sha256.digest(sha256.digest(pref))[0..3]
+      { :seed => Base58.binary_to_base58(pref + chk, :ripple) }.merge(key)
+    end
+
+    # Extract Seed ID from Encoding.
+    #
+    # @param seed [String] Base58 encoded seed
+    # @return [String, nil] extracted seed or nil
+    #   if not valid
+    def self.parse_seed(seed)
+      bin = Base58.base58_to_binary(seed, :ripple)
+      typ = bin[0]
+      chk = bin[-4..-1]
+      bin = bin[1...-4]
+
+      # TODO also permit ED25519 prefix (?)
+      return nil unless typ.unpack("C*").first == Key::TOKEN_TYPES[:family_seed]
+
+      sha256 = OpenSSL::Digest::SHA256.new
+      return nil unless sha256.digest(sha256.digest(typ + bin))[0..3] == chk
+
+      return nil unless bin.size == 16
+
+      return bin
+    end
+
+    # Return boolean indicating if the specified seed is valid
+    def self.seed?(seed)
+      parse_seed(seed) != nil
+    end
+  end # module Crypto
+end # module XRBP

data/lib/xrbp/crypto/validator.rb

@@ -0,0 +1,20 @@
+module XRBP
+  module Crypto
+    # Generate a new XRPL validator. Takes same params as {Crypto#node}
+    def self.validator(key=nil)
+      return node(key)
+    end
+
+    # Extract Validator ID from Address.
+    # Takes same params as {Crypto#parse_node}
+    def self.parse_validator(validator)
+      return parse_node(validator)
+    end
+
+    # Return bool indicating if Validator is valid.
+    # Takes same params as {Crypto#node?}
+    def self.validator?(validator)
+      return node?(validator)
+    end
+  end # module Crypto
+end # module XRBP

data/lib/xrbp/nodestore/backends/decompressor.rb

@@ -4,10 +4,15 @@ require 'lz4-ruby'
 module XRBP
   module NodeStore
     module Backends
+      # XRP Ledger Nodestore Decompressor. Currently only the NuDB backend
+      # employs compression but decompression is required to read those entries.
+      #
       # Ported from:
-      #   https://github.com/ripple/rippled/blob/develop/src/ripple/nodestore/impl/codec.h
+      # {https://github.com/ripple/rippled/blob/develop/src/ripple/nodestore/impl/codec.h codec.h}
       module Decompressor
         protected
+          # Principle decompression interface, extracts compression type (if any)
+          # from data and decompresses accordingly.
           def decompress(data)
             type, remaining = read_varint(data)
             case(type)
@@ -30,7 +35,15 @@ module XRBP
           end
 
         private
-          # https://github.com/ripple/rippled/blob/develop/src/ripple/nodestore/impl/varint.h
+          # This is a custom type/size prefix preprended onto
+          # nodestore values.
+          #
+          # See: {https://github.com/ripple/rippled/blob/develop/src/ripple/nodestore/impl/varint.h varint.h}
+          #
+          # @param data [String] binary nodestore data
+          # @return value prefix and remaining data
+          #
+          # @private
           def read_varint(data)
             bytes = data.bytes
 
@@ -67,14 +80,22 @@ module XRBP
 
           ###
 
+          # Decompresses a LZ4-compressed value
+          #
+          # @private
           def decompress_lz4(data)
             size, remaining = read_varint(data)
-            o = LZ4::Raw::decompress(remaining, size)[0]
-            o
+            LZ4::Raw::decompress(remaining, size)[0]
           end
 
           ###
 
+          # Decompresses a Compressed Inner Node (Version 1).
+          # Inner nodes may contain up to 16 32-byte hashes,
+          # though whens stored on disk-the compression algorithm
+          # employs a hash-mask to skip storage of unpopulated hashes.
+          #
+          # @private
           def decompress_compressed_v1_inner(data)
             raise if data.size < 34
 
@@ -84,23 +105,27 @@ module XRBP
                   [Format::HASH_PREFIXES.invert[:inner_node]].pack("H*")
                                                              .unpack("C*")
 
+            # extract mask indicating which hashes are set
             bytes = data.bytes
              mask = bytes[0..1].pack("C*").unpack("S").first
             bytes = bytes[2..-1]
 
             raise "nodeobject codec v1: empty inner node" if mask == 0
 
+            # iterate over hashes...
             bit = 0x8000
               i = 16
             while i > 0
               i -= 1
 
+              # ...if set in mask, extract from data...
               if (mask & bit) != 0
                 raise "nodeobject codec v1: short inner node" if bytes.size < 32
 
                 out  += bytes[0..31]
                 bytes = bytes[32..-1]
 
+              # ...otherwise just set to 0
               else
                 out += Array.new(32) { 0 }
               end
@@ -111,6 +136,11 @@ module XRBP
             out.pack("C*")
           end
 
+          # Decompresses a Compressed Inner Node (Version 2).
+          # Similar to Version 1 but contains additional
+          # variable length data appended onto the result bytes.
+          #
+          # @private
           def decompress_compressed_v2_inner(data)
             raise if data.size < 67
 
@@ -120,6 +150,8 @@ module XRBP
                   [Format::HASH_PREFIXES.invert[:inner_node_v2]].pack("H*")
                                                                 .unpack("C*")
 
+            # Same mask / hash extraction as V1
+
             bytes = data.bytes
              mask = bytes[0..1].pack("C*").unpack("S").first
             bytes = bytes[2..-1]
@@ -147,6 +179,10 @@ module XRBP
               bit  = bit >> 1
             end
 
+            ###
+
+            # Append extra variable-length data onto result
+
             out << depth
 
              copy = (depth + 1)/2
@@ -160,6 +196,9 @@ module XRBP
 
           ###
 
+          # Decompresses a Full (Uncompressed) Inner Node (Version 1).
+          #
+          # @private
           def decompress_full_v1_inner(data)
             raise if data.size != 512 # 16 32-bit hashes
 
@@ -170,7 +209,9 @@ module XRBP
             (out + data[0...512].bytes).pack("C*")
           end
 
-
+          # Decompresses a Full (Uncompressed) Inner Node (Version 2).
+          #
+          # @private
           def decompress_full_v2_inner(data)
             bytes = data.bytes
             depth = bytes[0]

data/lib/xrbp/nodestore/backends/nudb.rb

@@ -8,6 +8,14 @@ require_relative './decompressor'
 module XRBP
   module NodeStore
     module Backends
+      # NuDB nodestore backend, faciliates accessing XRP Ledger data
+      # in a NuDB database. This module accommodates for compression
+      # used in rippled's NuDB nodestore backend implementation
+      #
+      # @example retrieve data from NuDB backend
+      #   require 'nodestore/backends/nudb'
+      #   nudb = NodeStore::Backends::NuDB.new '/var/lib/rippled/db/nudb'
+      #   puts nudb.ledger('B506ADD630CB707044B4BFFCD943C1395966692A13DD618E5BD0978A006B43BD')
       class NuDB < DB
         include Decompressor
 
@@ -21,16 +29,38 @@ module XRBP
           open
         end
 
+        # Retrieve database value for the specified key
+        #
+        # @param key [String] binary key to lookup
+        # @return [String] binary value
         def [](key)
           decompress(@store.fetch(key)[0])
         end
 
+        # Iterate over each database key/value pair,
+        # invoking callback. During iteration will
+        # emit signals specific to the DB types being
+        # parsed
+        #
+        # @example iterating over NuDB entries
+        #   nudb.each do |iterator|
+        #     puts "Key/Value: #{iterator.key}/#{iterator.value}"
+        #   end
+        #
+        # @example handling ledgers via event callback
+        #   nudb.on(:ledger) do |hash, ledger|
+        #     puts "Ledger #{hash}"
+        #     pp ledger
+        #   end
+        #
+        #   # Any Enumerable method that invokes #each will
+        #   # have intended effect
+        #   nudb.to_a
         def each
           dat = File.join(path, "nudb.dat")
 
           RuDB::each(dat) do |key, val|
             val = decompress(val)
-puts key.bytes.join(", ")
             type, obj = infer_type(val)
 
             if type
@@ -48,6 +78,9 @@ puts key.bytes.join(", ")
 
         private
 
+        # Create database if it does not exist
+        #
+        # @private
         def create!
           dat = File.join(path, "nudb.dat")
           key = File.join(path, "nudb.key")
@@ -63,8 +96,9 @@ puts key.bytes.join(", ")
                        :load_factor => 0.5
         end
 
-
-
+        # Open existing database
+        #
+        # @private
         def open
           dat = File.join(path, "nudb.dat")
           key = File.join(path, "nudb.key")

data/lib/xrbp/nodestore/backends/rocksdb.rb

@@ -4,6 +4,13 @@ require "rocksdb"
 module XRBP
   module NodeStore
     module Backends
+      # RocksDB nodestore backend, faciliates accessing XRP Ledger data
+      # in a RocksDB database.
+      #
+      # @example retrieve data from RocksDB backend
+      #   require 'nodestore/backends/rocksdb'
+      #   rocksdb = NodeStore::Backends::RocksDB.new '/var/lib/rippled/db/rocksdb'
+      #   puts rocksdb.ledger('B506ADD630CB707044B4BFFCD943C1395966692A13DD618E5BD0978A006B43BD')
       class RocksDB < DB
         # cap max open files for performance
         MAX_OPEN_FILES = 200
@@ -14,10 +21,33 @@ module XRBP
              :max_open_files => MAX_OPEN_FILES}
         end
 
+        # Retrieve database value for the specified key
+        #
+        # @param key [String] binary key to lookup
+        # @return [String] binary value
         def [](key)
           @db[key]
         end
 
+        # Iterate over each database key/value pair,
+        # invoking callback. During iteration will
+        # emit signals specific to the DB types being
+        # parsed
+        #
+        # @example iterating over RocksDB entries
+        #   rocksdb.each do |iterator|
+        #     puts "Key/Value: #{iterator.key}/#{iterator.value}"
+        #   end
+        #
+        # @example handling account via event callback
+        #   rocksdb.on(:account) do |hash, account|
+        #     puts "Account #{hash}"
+        #     pp account
+        #   end
+        #
+        #   # Any Enumerable method that invokes #each will
+        #   # have intended effect
+        #   rocksdb.to_a
         def each
           iterator = @db.new_iterator
           iterator.seek_to_first

data/lib/xrbp/nodestore/db.rb

@@ -2,25 +2,46 @@ require 'base58'
 require 'openssl'
 
 module XRBP
+  # The NodeStore is the Key/Value DB which rippled persistent stores
+  # ledger data. Implemented via a backend configured at run time,
+  # the NodeStore is used to store the tree-like structures that
+  # consistute the XRP ledger.
+  #
+  # The Keys and Values stored in the NodeStore are custom binary
+  # encodings of tree-node IDs and data. See this module
+  # and the others in this directory for specifics on how keys & values
+  # are stored and extracted.
   module NodeStore
+
+    # Base NodeStore DB module, the client will use this class through
+    # specific DB-type subclass.
+    #
+    # Subclasses should define the <b>[  ]</b> (index) method taking key to
+    # lookup, returning corresponding NodeStore value and *each* method,
+    # iterating over nodestore values (see existing subclasses for
+    # implementation details)
     class DB
       include Enumerable
       include EventEmitter
 
       # TODO return nil if db lookup not found
 
+      # Return the NodeStore Ledger for the given lookup hash
       def ledger(hash)
         parse_ledger(self[hash])
       end
 
+      # Return the NodeStore Account for the given lookup hash
       def account(hash)
         parse_ledger_entry(self[hash])
       end
 
+      # Return the NodeStore Transaction for the given lookup hash
       def tx(hash)
         parse_tx(self[hash])
       end
 
+      # Return the NodeStore InnerNode for the given lookup hash
       def inner_node(hash)
         parse_inner_node(self[hash])
       end
@@ -29,6 +50,9 @@ module XRBP
 
       private
 
+      # Parsers binary ledger representation into structured ledger.
+      #
+      # @private
       def parse_ledger(ledger)
         obj = Format::LEDGER.decode(ledger)
                obj['close_time'] = XRBP::from_xrp_time(obj['close_time']).utc
@@ -39,6 +63,11 @@ module XRBP
         obj
       end
 
+      # Certain data types are prefixed with an 'encoding' header
+      # consisting of a field and/or type. Field, type, and remaining
+      # bytes are returned
+      #
+      # @private
       def parse_encoding(encoding)
              enc = encoding.unpack("C").first
             type = enc >> 4
@@ -59,6 +88,11 @@ module XRBP
         [[type, field], encoding]
       end
 
+      # Parses binary ledger entry into hash. Data returned
+      # in hash includes ledger entry type prefix, index,
+      # and array of parsed fields.
+      #
+      # @private
       def parse_ledger_entry(ledger_entry)
         # validate parsability
                 obj = Format::TYPE_INFER.decode(ledger_entry)
@@ -97,6 +131,9 @@ module XRBP
 
       ###
 
+      # Parse and return series of fields from binary data.
+      #
+      # @private
       def parse_fields(fields)
         parsed = {}
         until fields == "" || fields == "\0" || fields.nil?
@@ -112,6 +149,10 @@ module XRBP
         return parsed
       end
 
+      # Parse single field of specified encoding from data.
+      # Dispatches to corresponding parsing method when appropriate.
+      #
+      # @private
       def parse_field(data, encoding)
         length = encoding.first
 
@@ -135,7 +176,6 @@ module XRBP
         when :vl
           vl, offset = parse_vl(data)
           return data[offset..vl+offset-1], data[vl+offset..-1]
-
         when :account
           return parse_account(data)
         when :array
@@ -147,12 +187,15 @@ module XRBP
         when :vector256
           vl, offset = parse_vl(data)
           return data[offset..vl+offset-1], data[vl+offset..-1]
-
         end
 
         raise
       end
 
+      # Parse variable length header from data buffer. Returns length
+      # extracted from header and the number of bytes in header.
+      #
+      # @private
       def parse_vl(data)
          data = data.bytes
         first = data.first.to_i
@@ -172,6 +215,9 @@ module XRBP
         raise
       end
 
+      # Parse 'Amount' data type from binary data.
+      #
+      # @private
       def parse_amount(data)
         amount = data[0..7].unpack("Q>").first
            xrp = amount < 0x8000000000000000
@@ -195,6 +241,9 @@ module XRBP
                :issuer => issuer }, data
       end
 
+      # Parse 'Account' data type from binary data.
+      #
+      # @private
       def parse_account(data, vl=nil)
         unless vl
           vl,offset = parse_vl(data)
@@ -209,6 +258,9 @@ module XRBP
         return Base58.binary_to_base58(acct, :ripple), data[vl..-1]
       end
 
+      # Parse array of fields from binary data.
+      #
+      # @private
       def parse_array(data, encoding)
         e = Format::ENCODINGS[encoding]
         return nil, data if e == :end_of_array
@@ -229,6 +281,9 @@ module XRBP
         return array, data
       end
 
+      # Parse Object consisting of multiple fields from binary data.
+      #
+      # @private
       def parse_object(data, encoding)
         e = Format::ENCODINGS[encoding]
         case e
@@ -248,6 +303,9 @@ module XRBP
         raise "unknown object type"
       end
 
+      # Parse PathSet from binary data.
+      #
+      # @private
       def parse_pathset(data)
         pathset = [[]]
         until data == "" || data.nil?
@@ -287,6 +345,9 @@ module XRBP
 
       ###
 
+      # Parse Transaction from binary data
+      #
+      # @private
       def parse_tx(tx)
                 obj = Format::TYPE_INFER.decode(tx)
           node_type = Format::NODE_TYPES[obj["node_type"]]
@@ -312,6 +373,9 @@ module XRBP
           :index => index.pack("C*").unpack("H*").first.upcase }
       end
 
+      # Parse InnerNode from binary data.
+      #
+      # @private
       def parse_inner_node(node)
         # verify parsability
                 obj = Format::TYPE_INFER.decode(node)
@@ -323,6 +387,9 @@ module XRBP
 
       protected
 
+      # Return type and extracted structure from binary data.
+      #
+      # @private
       def infer_type(value)
                 obj = Format::TYPE_INFER.decode(value)
           node_type = Format::NODE_TYPES[obj["node_type"]]

data/lib/xrbp/nodestore/format.rb

@@ -17,16 +17,20 @@ module XRBP
         4 => :tx_node
       }
 
-      HASH_PREFIXES = {
-        "54584E00" => :tx_id,
-        "534E4400" => :tx_node,
-        "4D4C4E00" => :leaf_node,
-        "4D494E00" => :inner_node,
-        "494E5200" => :inner_node_v2,
-        "4C575200" => :ledger_master,
-        "53545800" => :tx_sign,
-        "56414C00" => :validation,
-        "50525000" => :proposal
+      HASH_PREFIXES = {               # ASCII value:
+        "54584E00" => :tx_id,         # TXN
+        "534E4400" => :tx_node,       # SND
+        "4D4C4E00" => :leaf_node,     # MLN
+        "4D494E00" => :inner_node,    # MIN
+        "494E5200" => :inner_node_v2, # INR
+        "4C575200" => :ledger_master, # LWR
+        "53545800" => :tx_sign,       # STX
+        "56414C00" => :validation,    # VAL
+        "50525000" => :proposal       # PRP
+
+        # TODO:       :tx_multi_sign  # SMT
+        #             :manifest       # MAN
+        #             :paychan_claim  # CLM
       }
 
       ###

data/lib/xrbp/overlay.rb

@@ -1,3 +1,4 @@
 require_relative './overlay/connection'
 require_relative './overlay/handshake'
 require_relative './overlay/frame'
+require_relative './overlay/messages'

data/lib/xrbp/overlay/connection.rb

@@ -1,5 +1,23 @@
 module XRBP
+  # The Overlay is the Peer-to-Peer (P2P) network established
+  # by rippled node instances to each other. It is what is used
+  # to relay transactions and network state as the consensus
+  # process is executed.
+  #
+  # This module facilitates communication with the Overlay P2P
+  # network from Ruby.
   module Overlay
+
+    # Primary Overlay Connection Interface, use Connection
+    # to send and receive Peer-To-Peer data over the Overlay.
+    #
+    # @example establishing a connection, reading frames
+    #   overlay = XRBP::Overlay::Connection.new "127.0.0.1", 51235
+    #   overlay.connect
+    #
+    #   overlay.read_frames do |frame|
+    #     puts "Message: #{frame.type_name} (#{frame.size} bytes)"
+    #   end
     class Connection
       attr_reader :host, :port
       attr_accessor :node
@@ -10,14 +28,17 @@ module XRBP
         @node = Crypto.node
       end
 
+      # @private
       def socket
         @socket ||= TCPSocket.open(host, port)
       end
 
+      # Indicates if the connection is closed
       def closed?
         socket.closed?
       end
 
+      # @private
       def ssl_socket
         @ssl_socket ||= begin
           ssl_context = OpenSSL::SSL::SSLContext.new
@@ -31,29 +52,38 @@ module XRBP
         end
       end
 
+      # @private
       def handshake
         @handshake ||= Handshake.new self
       end
 
       ###
 
+      # Initiate new connection to peer
       def connect
         ssl_socket.connect
         handshake.execute!
       end
 
-      def close
+      # Close the connection to peer
+      def close!
         ssl_socket.close
       end
 
+      alias :close :close!
+
+      # Send raw data  via this connection
       def write(data)
         ssl_socket.puts(data)
       end
 
+      # Read raw data from connection
       def read
         ssl_socket.gets
       end
 
+      # Read frames from connection until closed, invoking
+      # passed block with each.
       def read_frames
         frame = nil
         remaining = nil
@@ -70,17 +100,15 @@ module XRBP
 
             _, remaining = frame << out
             if frame.complete?
-              # TODO extra specific protobuf data structure
-              #      from data, set on frame
               yield frame
               frame = nil
             end
 
-            # XXX: doesn't feel right to just
-            #      discard remaining, look into this
+            # static assertion: should have no more data
+            raise unless remaining.nil? || remaining.empty?
           end
         end
       end
     end # class Connection
-  end # module WebClient
+  end # module Overlay
 end # module XRBP

data/lib/xrbp/overlay/frame.rb

@@ -3,6 +3,10 @@ require_relative './ripple.proto'
 
 module XRBP
   module Overlay
+    # Overlay Message Frame, prefixes Protobuf based message in
+    # with header describing size and type.
+    #
+    # @private
     class Frame
       TYPE_INFER = Bistro.new([
         'L>', 'size',
@@ -37,14 +41,18 @@ module XRBP
         @type_name ||= self.class.type_name(type)
       end
 
+      def message
+        @message ||= MESSAGES[type_name].decode(data)
+      end
+
       def <<(data)
-        remaining = size - (@data.size + header_size)
+        remaining = size - @data.size
         @data += data[0..remaining-1]
         return @data, data[remaining..-1]
       end
 
       def complete?
-        (@data.size + header_size) == size
+        @data.size == size
       end
     end # class Frame
   end # module WebClient

data/lib/xrbp/overlay/handshake.rb

@@ -1,17 +1,24 @@
-# XXX this module requires the openssl-ruby gem with the following patches:
-#     https://github.com/ruby/openssl/pull/250
-#
-#     Otherwise the ssl_socket#finished and #peer_finished methods will
-#     not be available
-#
-#     Currently the only way to apply this is to checkout openssl-ruby, apply
-#     the patches, and then rebuild/reinstall the gem locally!
-
 require 'base64'
 require 'openssl'
 
 module XRBP
   module Overlay
+
+    # Overlay Connection Handshake, the first message sent from connection
+    # initiator to remote endpoint establishing session. Once connection
+    # is successfully established subsequent messages will be encapsulated
+    # Frames.
+    #
+    # XXX this module requires the openssl-ruby gem with the following patches:
+    #     https://github.com/ruby/openssl/pull/250
+    #
+    #     Otherwise the ssl_socket#finished and #peer_finished methods will
+    #     not be available
+    #
+    #     Currently the only way to apply this is to checkout openssl-ruby, apply
+    #     the patches, and then rebuild/reinstall the gem locally!
+    #
+    # @private
     class Handshake
       attr_reader :connection, :response
 

data/lib/xrbp/overlay/messages.rb

@@ -0,0 +1,27 @@
+module XRBP
+  module Overlay
+    # Map of Protocol Message Type to Class
+    #
+    # See {https://github.com/ripple/rippled/blob/develop/src/ripple/overlay/impl/ProtocolMessage.h ProtocolMessage.h}
+    MESSAGES = {
+      :MTHELLO                => Protocol::TMHello,
+      :MTMANIFESTS            => Protocol::TMManifests,
+      :MTPING                 => Protocol::TMPing,
+      :MTCLUSTER              => Protocol::TMCluster,
+      :MTGET_SHARD_INFO       => Protocol::TMGetShardInfo,
+      :MTSHARD_INFO           => Protocol::TMShardInfo,
+      :MTGET_PEER_SHARD_INFO  => Protocol::TMGetPeerShardInfo,
+      :MTGET_PEERS            => Protocol::TMGetPeers,
+      :MTPEERS                => Protocol::TMPeers,
+      :MTENDPOINTS            => Protocol::TMEndpoints,
+      :MTTRANSACTION          => Protocol::TMTransaction,
+      :MTGET_LEDGER           => Protocol::TMGetLedger,
+      :MTLEDGER_DATA          => Protocol::TMLedgerData,
+      :MTPROPOSE_LEDGER       => Protocol::TMProposeSet,
+      :MTSTATUS_CHANGE        => Protocol::TMStatusChange,
+      :MTHAVE_SET             => Protocol::TMHaveTransactionSet,
+      :MTVALIDATION           => Protocol::TMValidation,
+      :MTGET_OBJECTS          => Protocol::TMGetObjectByHash
+    }
+  end # module Overlay
+end # module XRBP

data/lib/xrbp/overlay/ripple.proto.rb

@@ -74,6 +74,21 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
     optional :lastLink, :bool, 4
     repeated :peerchain, :uint32, 5
   end
+  add_message "protocol.TMLink" do
+    optional :nodePubKey, :bytes, 1
+  end
+  add_message "protocol.TMGetPeerShardInfo" do
+    optional :hops, :uint32, 1
+    optional :lastLink, :bool, 2
+    repeated :peerChain, :message, 3, "protocol.TMLink"
+  end
+  add_message "protocol.TMPeerShardInfo" do
+    optional :shardIndexes, :string, 1
+    optional :nodePubKey, :bytes, 2
+    optional :endpoint, :string, 3
+    optional :lastLink, :bool, 4
+    repeated :peerChain, :message, 5, "protocol.TMLink"
+  end
   add_message "protocol.TMTransaction" do
     optional :rawTransaction, :bytes, 1
     optional :status, :enum, 2, "protocol.TransactionStatus"
@@ -210,6 +225,8 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
     value :MTGET_OBJECTS, 42
     value :MTGET_SHARD_INFO, 50
     value :MTSHARD_INFO, 51
+    value :MTGET_PEER_SHARD_INFO, 52
+    value :MTPEER_SHARD_INFO, 53
   end
   add_enum "protocol.TransactionStatus" do
     value :TSZERO, 0
@@ -275,6 +292,9 @@ module Protocol
   TMCluster = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMCluster").msgclass
   TMGetShardInfo = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMGetShardInfo").msgclass
   TMShardInfo = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMShardInfo").msgclass
+  TMLink = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMLink").msgclass
+  TMGetPeerShardInfo = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMGetPeerShardInfo").msgclass
+  TMPeerShardInfo = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMPeerShardInfo").msgclass
   TMTransaction = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMTransaction").msgclass
   TMStatusChange = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMStatusChange").msgclass
   TMProposeSet = Google::Protobuf::DescriptorPool.generated_pool.lookup("protocol.TMProposeSet").msgclass

data/lib/xrbp/version.rb

@@ -1,3 +1,3 @@
 module XRBP
-  VERSION = '0.1.8'
+  VERSION = '0.1.9'
 end # module XRBP

data/spec/xrbp/crypto/account_spec.rb

@@ -0,0 +1,6 @@
+describe XRBP::Crypto do
+  it "generates valid account" do
+    acct = described_class.account
+    expect(described_class.account?(acct[:account])).to be(true)
+  end
+end

data/spec/xrbp/crypto/key_spec.rb

@@ -0,0 +1,11 @@
+describe XRBP::Crypto::Key do
+  it "generates valid sec256k1 key"
+  it "generates valid ed25519 key"
+
+  it "generates signs and verifies digest" do
+    key = described_class.secp256k1
+    dat = "ABCDEFGHIJLMNOPQRSTUVWYZ12345678"
+    signed = XRBP::Crypto::Key.sign_digest(key, dat)
+    expect(XRBP::Crypto::Key.verify(key, signed, dat))
+  end
+end

data/spec/xrbp/crypto/node_spec.rb

@@ -0,0 +1,6 @@
+describe XRBP::Crypto do
+  it "generates valid node" do
+    node = described_class.node
+    expect(described_class.node?(node[:node])).to be(true)
+  end
+end

data/spec/xrbp/crypto/seed_spec.rb

@@ -0,0 +1,6 @@
+describe XRBP::Crypto do
+  it "generates valid seed" do
+    seed = described_class.seed
+    expect(described_class.seed?(seed[:seed])).to be(true)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xrbp
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Dev Null Productions
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-17 00:00:00.000000000 Z
+date: 2019-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -196,7 +196,6 @@ files:
 - examples/paginate.rb
 - examples/prioritized.rb
 - examples/round_robin.rb
-- examples/sandbox.rb
 - examples/username.rb
 - examples/validator.rb
 - examples/websocket.rb
@@ -207,6 +206,8 @@ files:
 - lib/xrbp/crypto/account.rb
 - lib/xrbp/crypto/key.rb
 - lib/xrbp/crypto/node.rb
+- lib/xrbp/crypto/seed.rb
+- lib/xrbp/crypto/validator.rb
 - lib/xrbp/dsl.rb
 - lib/xrbp/dsl/accounts.rb
 - lib/xrbp/dsl/ledgers.rb
@@ -238,6 +239,7 @@ files:
 - lib/xrbp/overlay/connection.rb
 - lib/xrbp/overlay/frame.rb
 - lib/xrbp/overlay/handshake.rb
+- lib/xrbp/overlay/messages.rb
 - lib/xrbp/overlay/ripple.proto.rb
 - lib/xrbp/plugins.rb
 - lib/xrbp/plugins/base.rb
@@ -286,6 +288,10 @@ files:
 - spec/helpers/force_serializable.rb
 - spec/helpers/test_handshake.rb
 - spec/spec_helper.rb
+- spec/xrbp/crypto/account_spec.rb
+- spec/xrbp/crypto/key_spec.rb
+- spec/xrbp/crypto/node_spec.rb
+- spec/xrbp/crypto/seed_spec.rb
 - spec/xrbp/websocket/client_spec.rb
 - spec/xrbp/websocket/command_spec.rb
 - spec/xrbp/websocket/connection_spec.rb

data/examples/sandbox.rb

@@ -1,14 +0,0 @@
-$: << File.expand_path('../../lib', __FILE__)
-require 'xrbp'
-require 'xrbp/nodestore/backends/nudb'
-
-db = XRBP::NodeStore::Backends::NuDB.new "/var/lib/rippled/nudb/"
-store = db.instance_variable_get(:@store)
-
-key = [47, 126, 65, 231, 47, 167, 3, 91, 161, 174, 133, 152, 2, 230, 171, 139, 209, 40, 100, 217, 100, 131, 160, 49, 186, 119, 31, 84, 75, 116, 125, 81].pack("C*")
-val = store.fetch(key).first
-#puts val.unpack("C*").join " "
-decompressed = db.send(:decompress, val)
-puts decompressed.unpack("C*").join " "
-type, obj = db.send(:infer_type, decompressed)
-puts obj