xrbp 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30fb389dec1e019a299501a035cb5131332fec5b3cc192f24be9ad280aaf60e4
-  data.tar.gz: 8bec54f51b34ebbee7a2f433191d6dbe72e07fb3d2c60c1d3caa0b818c4da1f8
+  metadata.gz: e0ce0d6d6704d00d7fe16278ad96bfd68514f73e7b76486894ae479dee90eef8
+  data.tar.gz: 149a45a103104f1e5cef4da27d6870ba26f98dcb96a4f7e6e0e1bc18038d7367
 SHA512:
-  metadata.gz: f62bb3b3c347d42c29f227afd7398257078b6a30996a82f06d0952a337dcea495b33707a8430432c9945214d0609efde2c73a37472a23dfdeba9ab4ee13aebe3
-  data.tar.gz: d156f474e8d54a4583f3b41516caa9fb6e55d1c1d7edc6d04838f7066e93a560b48485880bdd2b1cf15b7aeb05c93087542dbf1ad80e5f8620a24dee16592e5f
+  metadata.gz: ed3630f368f3706c6c87e7031d25a6c3136489d6de8e3c83120c8a638a9e198d376615ed0c2fe7f5dab1f897f6aff6502a0296b0d3e03ea4ec27813d9d62b027
+  data.tar.gz: f43f8de4dc245cd2b3fe5996fd8a33f5b009ba60ea08858d0e1af9b008a4bfe71855172e734699d32c1b1dc97d853e94285efb4f11a94feac4f2c5b0087754eb

data/examples/crawl_nodes.rb

@@ -25,7 +25,7 @@ connection.on :peers do |node, peers|
 end
 
 connection.on :peer do |node, peer|
-  puts " #{peer.url}"
+  print " #{peer.url.ljust(40)} - Ledgers: #{peer.ledgers}\n"
 end
 
 XRBP::Model::Node.crawl("wss://s1.ripple.com:51235",

data/examples/nodestore1.rb

@@ -0,0 +1,14 @@
+$: << File.expand_path('../../lib', __FILE__)
+require 'xrbp'
+require 'xrbp/nodestore/backends/rocksdb'
+
+db = XRBP::NodeStore::Backends::RocksDB.new "/var/lib/rippled/rocksdb/rippledb.0899"
+
+#ledger = "B506ADD630CB707044B4BFFCD943C1395966692A13DD618E5BD0978A006B43BD"
+#ledger = [ledger].pack("H*")
+#puts db.ledger(ledger)
+
+#account = "0001bf7468341666f1f47a95e0f4d88e68b5fc7d20d77437cb22954fbbfe6127"
+account = "02c46b3a4130d0a329c47f0da61b829aa5d1ae53c5817e475bcd794e5107be44"
+account = [account].pack("H*")
+puts db.account(account)

data/examples/nodestore2.rb

@@ -0,0 +1,42 @@
+$: << File.expand_path('../../lib', __FILE__)
+require 'xrbp'
+require 'xrbp/nodestore/backends/rocksdb'
+
+db = XRBP::NodeStore::Backends::RocksDB.new "/var/lib/rippled/rocksdb/rippledb.0899"
+
+db.on :unknown do |hash, node|
+  puts "Unknown #{hash}: #{node}"
+end
+
+db.on :inner_node do |hash, node|
+  #puts "Inner Node #{hash}"
+end
+
+db.on :ledger do |hash, ledger|
+  #puts "Ledger #{ledger['index']}"
+end
+
+db.on :tx do |hash, tx|
+  puts "Tx #{tx}"
+end
+
+db.on :account do |hash, account|
+  #puts "Account #{account}"
+end
+
+###
+
+tallys = {}
+
+# object iterator invokes event emitters
+db.each do |node|
+          obj = XRBP::NodeStore::Format::TYPE_INFER.decode(node.value)
+    node_type = XRBP::NodeStore::Format::NODE_TYPES[obj["node_type"]]
+  hash_prefix = XRBP::NodeStore::Format::HASH_PREFIXES[obj["hash_prefix"].upcase]
+
+  type = node_type.to_s + "/" + hash_prefix.to_s
+  tallys[type] ||= 0
+  tallys[type]  += 1
+end
+
+puts tallys

data/examples/p2p.rb

@@ -0,0 +1,14 @@
+gem 'openssl', '2.1.3'
+
+$: << File.expand_path('../../lib', __FILE__)
+require 'xrbp'
+
+overlay = XRBP::Overlay::Connection.new "127.0.0.1", 51235
+overlay.connect
+puts overlay.handshake.response
+
+overlay.read_frames do |frame|
+  puts "Message: #{frame.type_name} (#{frame.size} bytes)"
+end
+
+overlay.close

data/lib/xrbp.rb

@@ -6,5 +6,8 @@ require 'xrbp/terminatable'
 require 'xrbp/plugins'
 require 'xrbp/websocket'
 require 'xrbp/webclient'
+require 'xrbp/crypto'
+require 'xrbp/nodestore'
+require 'xrbp/overlay'
 require 'xrbp/model'
 require 'xrbp/dsl'

data/lib/xrbp/common.rb

@@ -7,4 +7,10 @@ module XRBP
   # Created on 2013-01-01
   # https://data.ripple.com/v2/ledgers/32570
   GENESIS_TIME = DateTime.new(2013, 1, 1, 0, 0, 0)
+
+  # Convert XRP Ledger time to local time
+  def self.from_xrp_time(xrp_time)
+    return nil if xrp_time.nil?
+    Time.at(xrp_time + 946684800)
+  end
 end # module XRBP

data/lib/xrbp/core_ext.rb

@@ -22,3 +22,25 @@ class Queue
     end
   end
 end
+
+# @private
+class String
+  # return bignum corresponding to string
+  def to_bn
+    bytes.inject(0) { |bn, b| (bn << 8) | b }
+  end
+end
+
+# @private
+class Integer
+  # return bytes
+  def bytes
+    i = dup
+    b = []
+    until i == 0
+      b << (i & 0xFF)
+      i  = i >> 8
+    end
+    b
+  end
+end

data/lib/xrbp/crypto.rb

@@ -0,0 +1,3 @@
+require 'xrbp/crypto/key'
+require 'xrbp/crypto/account'
+require 'xrbp/crypto/node'

data/lib/xrbp/crypto/account.rb

@@ -0,0 +1,33 @@
+require 'base58'
+
+module XRBP
+  module Crypto
+    def self.account(key=nil)
+      pub = nil
+      if key == :secp256k1 || key.nil?
+        key = Key::secp256k1
+        key[:type] = :secp256k1
+        pub = key[:public]
+
+      elsif key == :ed25519
+        key = Key::ed25519
+        key[:type] = :ed25519
+        pub = "\xED" + key[:public]
+
+      elsif key.is_a?(Hash)
+        pub = key[:public]
+
+      else
+        pub = key
+        key = {:public => pub}
+      end
+
+          sha256 = OpenSSL::Digest::SHA256.new
+       ripemd160 = OpenSSL::Digest::RIPEMD160.new
+      account_id = [Key::TOKEN_TYPES[:account_id]].pack("C") + ripemd160.digest(sha256.digest([pub].pack("H*")))
+          chksum = sha256.digest(sha256.digest(account_id))[0..3]
+
+      { :account => Base58.binary_to_base58(account_id  + chksum, :ripple) }.merge(key)
+    end
+  end # module Crypto
+end # module XRBP

data/lib/xrbp/crypto/key.rb

@@ -0,0 +1,93 @@
+require 'securerandom'
+
+module XRBP
+  module Crypto
+    module Key
+      TOKEN_TYPES = {
+        :none             =>  1, # unused
+        :node_public      => 28,
+        :node_private     => 32,
+        :account_id       =>  0,
+        :account_public   => 35,
+        :account_secret   => 34,
+        :family_generator => 41,
+        :family_seed      => 33
+      }
+
+      ###
+
+      def self.priv
+        seed = SecureRandom.random_bytes(32)
+        OpenSSL::Digest::SHA256.new.digest(seed)
+      end
+
+      def self.secp256k1
+          # XXX: the bitcoin secp256k1 implementation (which rippled pulls in / vendors)
+          #      has alot of nuances which require special configuration in openssl. For
+          #      the time being, mitigate this by pulling in & using the ruby
+          #      btc-secp256k1 bindings:
+          #        https://github.com/cryptape/ruby-bitcoin-secp256k1
+          #
+          #      Perhaps at some point, we can look into implementing this logic in pure-ruby:
+          #        https://medium.com/coinmonks/introduction-to-blockchains-bedrock-the-elliptic-curve-secp256k1-e4bd3bc17d
+          require 'secp256k1'
+
+          spk = Secp256k1::PrivateKey.new
+
+          # XXX: I'd like to generate the private key first & set,
+          #      but for some reason this doesn't work. When the
+          #      keys are loaded for signing/verification later
+          #      the public key is not able to verify the signature
+          #      generated from the private key set in this way.
+          # TODO: Investigate
+           #pk = priv
+          #spk.set_raw_privkey [pk].pack("H*")
+
+          {  :public => spk.pubkey.serialize.unpack("H*").first,
+            :private => spk.send(:serialize) }
+      end
+
+      def self.ed25519
+        # XXX openssl 1.1.1 needed for EdDSA support:
+        #     https://www.openssl.org/blog/blog/2018/09/11/release111/
+        #     Until then use this:
+        require "ed25519"
+
+        # FIXME: this works for now (eg generates valid keys),
+        #        but we should do this in the same way rippled does:
+        #        Generate private key, then generate corresponding
+        #        Ed25519 public key
+        key = Ed25519::SigningKey.generate
+        {  :public => key.to_bytes.unpack("H*").first.upcase,
+          :private => key.verify_key.to_bytes.unpack("H*").first.upcase }
+      end
+
+      ###
+
+      def self.sign_digest(key, data)
+        raise "unknown key" unless key.is_a?(Hash) && key[:type] && key[:private]
+        raise "invalid data" unless data.length == 32
+
+
+        if key[:type] == :secp256k1
+          # XXX: see note about this library above
+          require 'secp256k1'
+
+          pk = Secp256k1::PrivateKey.new
+          pk.set_raw_privkey [key[:private]].pack("H*")
+          #pk.pubkey.deserialize [key[:public]].pack("H*")
+          sig_raw = pk.ecdsa_sign data, raw: true
+          return pk.ecdsa_serialize sig_raw
+
+        #elsif key[:type] == :ed25519
+          # TODO
+        end
+
+        raise "unknown key type"
+      end
+
+      def self.verify(key, data, expected)
+      end
+    end
+  end # module Crypto
+end # module XRBP

data/lib/xrbp/crypto/node.rb

@@ -0,0 +1,27 @@
+require 'base58'
+
+module XRBP
+  module Crypto
+    def self.node(key=nil)
+      pub = nil
+      if key == :secp256k1 || key.nil?
+        key = Key::secp256k1
+        key[:type] = :secp256k1
+        pub = key[:public]
+
+      elsif key.is_a?(Hash)
+        pub = key[:public]
+
+      else
+        pub = key
+        key = {:public => pub}
+      end
+
+       sha256 = OpenSSL::Digest::SHA256.new
+      node_id = [Key::TOKEN_TYPES[:node_public]].pack("C") + [pub].pack("H*")
+       chksum = sha256.digest(sha256.digest(node_id))[0..3]
+
+      { :node => Base58.binary_to_base58(node_id + chksum, :ripple) }.merge(key)
+    end
+  end # module Crypto
+end # module XRBP

data/lib/xrbp/model/node.rb

@@ -11,7 +11,7 @@ module XRBP
       DEFAULT_CRAWL_PORT = 51235
 
       attr_accessor :ip, :port
-      attr_accessor :addr, :version, :uptime, :type
+      attr_accessor :addr, :version, :uptime, :type, :ledgers
 
       # Return unique node id
       def id
@@ -71,6 +71,7 @@ module XRBP
         n.version = p["version"].split("-").last
         n.uptime  = p["uptime"]
         n.type    = p["type"]
+        n.ledgers = p["complete_ledgers"]
 
         n
       end

data/lib/xrbp/nodestore.rb

@@ -0,0 +1,7 @@
+require_relative './nodestore/format'
+
+require_relative './nodestore/db'
+
+# optionally include:
+#require_relative './nodestore/backends/rocksdb'
+#require_relative './nodestore/backends/nudb'

data/lib/xrbp/nodestore/backends/nudb.rb

@@ -0,0 +1 @@
+# TODO

data/lib/xrbp/nodestore/backends/rocksdb.rb

@@ -0,0 +1,43 @@
+require "rocksdb"
+
+module XRBP
+  module NodeStore
+    module Backends
+      class RocksDB < DB
+        # cap max open files for performance
+        MAX_OPEN_FILES = 200
+
+        def initialize(path)
+          @db = ::RocksDB::DB.new path,
+                  {:readonly => true,
+             :max_open_files => MAX_OPEN_FILES}
+        end
+
+        def [](key)
+          @db[key]
+        end
+
+        def each
+          iterator = @db.new_iterator
+          iterator.seek_to_first
+
+          while(iterator.valid)
+            type, obj = infer_type(iterator.value)
+
+            if type
+              emit type, iterator.key, obj
+            else
+              emit :unknown, iterator.key,
+                             iterator.value
+            end
+
+            yield iterator
+            iterator.next
+          end
+
+          iterator.close
+        end
+      end # class RocksDB
+    end # module Backends
+  end # module NodeStore
+end # module XRBP

data/lib/xrbp/nodestore/db.rb

@@ -0,0 +1,324 @@
+require 'base58'
+require 'openssl'
+
+module XRBP
+  module NodeStore
+    class DB
+      include Enumerable
+      include EventEmitter
+
+      def ledger(hash)
+        parse_ledger(self[hash])
+      end
+
+      def account(hash)
+        parse_ledger_entry(self[hash])
+      end
+
+      def tx(hash)
+        parse_tx(self[hash])
+      end
+
+      def inner_node(hash)
+        parse_inner_node(self[hash])
+      end
+
+      ###
+
+      private
+
+      def parse_ledger(ledger)
+        obj = Format::LEDGER.decode(ledger)
+               obj['close_time'] = XRBP::from_xrp_time(obj['close_time']).utc
+        obj['parent_close_time'] = XRBP::from_xrp_time(obj['parent_close_time']).utc
+        obj['parent_hash'].upcase!
+        obj['tx_hash'].upcase!
+        obj['account_hash'].upcase!
+        obj
+      end
+
+      def parse_encoding(encoding)
+            enc = encoding.unpack("C").first
+           type = enc >> 4
+          field = enc  & 0xF
+        encoding = encoding[1..-1]
+
+        if type == 0
+          type = encoding.unpack("C").first
+          encoding = encoding[1..-1]
+        end
+
+        if field == 0
+          field = encoding.unpack("C").first
+          encoding = encoding[1..-1]
+        end
+
+        type = Format::SERIALIZED_TYPES[type]
+        [[type, field], encoding]
+      end
+
+      def parse_ledger_entry(ledger_entry)
+        # validate parsability
+                obj = Format::TYPE_INFER.decode(ledger_entry)
+          node_type = Format::NODE_TYPES[obj["node_type"]]
+        hash_prefix = Format::HASH_PREFIXES[obj["hash_prefix"].upcase]
+        raise unless   node_type == :account_node &&
+                     hash_prefix == :leaf_node
+
+        # discard node type, and hash prefix
+        ledger_entry = ledger_entry[13..-1]
+
+        # verify encoding
+        encoding, ledger_entry = parse_encoding(ledger_entry)
+        raise "Invalid Ledger Entry" unless Format::ENCODINGS[encoding] == :ledger_entry_type
+        ledger_entry = ledger_entry.bytes
+
+        # first byte after encoding is ledger entry type prefix
+        prefix = ledger_entry[0..1].pack("C*")
+
+        # last 32 bytes is entry index
+        index  = ledger_entry[-32..-1].pack("C*")
+                                      .unpack("H*")
+                                      .first
+                                      .upcase
+
+        # remaining bytes are serialized object
+        fields = parse_fields(ledger_entry[2...-32].pack("C*"))
+
+        # TODO instantiate class corresponding to prefix &
+        #      populate attributes w/ fields
+
+        { :prefix => prefix,
+          :index  => index,
+          :fields => fields }
+      end
+
+      ###
+
+      def parse_fields(fields)
+        parsed = {}
+        until fields == "" || fields.nil?
+          encoding, fields = parse_encoding(fields)
+          return parsed if encoding.first.nil?
+
+          e = Format::ENCODINGS[encoding]
+          value, fields = parse_field(fields, encoding)
+          break unless value
+          parsed[e] = value
+        end
+
+        return parsed
+      end
+
+      def parse_field(data, encoding)
+        length = encoding.first
+
+        case length
+        when :uint8
+          return data.unpack("C").first, data[1..-1]
+        when :uint16
+          return data.unpack("S").first, data[2..-1]
+        when :uint32
+          return data.unpack("L").first, data[4..-1]
+        when :uint64
+          return data.unpack("Q").first, data[8..-1]
+        when :hash128
+          return data.unpack("H32").first, data[16..-1]
+        when :hash160
+          return data.unpack("H40").first, data[20..-1]
+        when :hash256
+          return data.unpack("H64").first, data[32..-1]
+
+        when :amount
+          amount = data[0..7].unpack("Q>").first
+             xrp = amount < 0x8000000000000000
+          return  (amount & 0x3FFFFFFFFFFFFFFF), data[8..-1] if xrp
+
+          sign = (amount & 0x4000000000000000) >> 62 # 0 = neg / 1 = pos
+           exp = (amount & 0x3FC0000000000000) >> 54
+          mant = (amount & 0x003FFFFFFFFFFFFF)
+
+          data = data[8..-1]
+          currency = Format::CURRENCY_CODE.decode(data)
+
+          data = data[Format::CURRENCY_CODE.size..-1]
+          issuer, data = parse_account(data, 20)
+
+          # TODO calculate value
+          return { :sign => sign,
+                    :exp => exp,
+               :mantissa => mant,
+               :currency => currency,
+                 :issuer => issuer }, data
+
+        when :vl
+          vl, offset = parse_vl(data)
+          return data[offset..vl+offset-1], data[vl+offset..-1]
+
+        when :account
+          return parse_account(data)
+
+        when :array
+          e = Format::ENCODINGS[encoding]
+          return nil, data if e == :end_of_array
+
+          array = []
+          until data == "" || data.nil?
+            aencoding, data = parse_encoding(data)
+            break if aencoding.first.nil?
+
+            e = Format::ENCODINGS[aencoding]
+            break if e == :end_of_array
+
+            value, data = parse_field(data, aencoding)
+            break unless value
+            array << value
+          end
+
+          return array, data
+
+        when :object
+          e = Format::ENCODINGS[encoding]
+          case e
+          when :end_of_object
+            return nil, data
+
+          when :signer,   :signer_entry,
+               :majority, :memo,
+               :modified_node, :created_node, :deleted_node,
+               :previous_fields, :final_fields, :new_fields
+            # TODO instantiate corresponding classes
+            return parse_fields(data)
+
+          #else:
+          end
+
+        when :pathset
+          pathset = []
+          until data == "" || data.nil?
+            segment = data.unpack("C").first
+            data = data[1..-1]
+            return pathset, data if segment == 0x00 # end of path
+
+            if segment == 0xFF # path boundry
+              pathset << []
+            else
+              if segment & 0x01 # path account
+                issuer, data = parse_account(data, 20)
+              end
+
+              if segment & 0x02 # path currency
+                currency = Format::CURRENCY_CODE.decode(data)
+                data = data[Format::CURRENCY_CODE.size..-1]
+              end
+
+              if segment & 0x03 # path issuer
+                issuer, data = parse_account(data, 20)
+              end
+            end
+          end
+
+          return pathset, data
+
+        when :vector256
+          vl, offset = parse_vl(data)
+          return data[offset..vl+offset-1], data[vl+offset..-1]
+
+        end
+
+        raise
+      end
+
+      def parse_vl(data)
+         data = data.bytes
+        first = data.first.to_i
+        return first, 1 if first <= 192
+
+        data = data[1..-1]
+        second = data.first.to_i
+        if first <= 240
+          return (193+(first-193)*256+second), 2
+
+        elsif first <= 254
+          data = data[1..-1]
+          third = data.first.to_i
+          return (12481 + (first-241)*65536 + second*256 + third), 3
+        end
+
+        raise
+      end
+
+      def parse_account(data, vl=nil)
+        unless vl
+          vl,offset = parse_vl(data)
+          data = data[offset..-1]
+        end
+
+          acct = "\0" + data[0..vl-1]
+        sha256 = OpenSSL::Digest::SHA256.new
+        digest = sha256.digest(sha256.digest(acct))[0..3]
+        acct  += digest
+        return Base58.binary_to_base58(acct, :ripple), data[vl..-1]
+      end
+
+      ###
+
+      def parse_tx(tx)
+                obj = Format::TYPE_INFER.decode(tx)
+          node_type = Format::NODE_TYPES[obj["node_type"]]
+        hash_prefix = Format::HASH_PREFIXES[obj["hash_prefix"].upcase]
+        raise unless   node_type == :tx_node &&
+                     hash_prefix == :tx_node
+
+        # discard node type, and hash prefix
+        tx = tx[13..-1]
+
+        # get node length
+        vl, offset = parse_vl(tx)
+        node, _tx = tx.bytes[offset..vl+offset-1], tx.bytes[vl+offset..-1]
+        node = parse_fields(node.pack("C*"))
+
+        # get meta length
+        vl, offset = parse_vl(_tx.pack("C*"))
+        meta, index = _tx[offset..vl+offset-1], _tx[vl+offset..-1]
+        meta = parse_fields(meta.pack("C*"))
+
+        {  :node =>  node,
+           :meta =>  meta,
+          :index => index.pack("C*").unpack("H*").first.upcase }
+      end
+
+      def parse_inner_node(node)
+        # verify parsability
+                obj = Format::TYPE_INFER.decode(node)
+        hash_prefix = Format::HASH_PREFIXES[obj["hash_prefix"].upcase]
+        raise unless hash_prefix == :inner_node
+
+        Format::INNER_NODE.decode(node)
+      end
+
+      protected
+
+      def infer_type(value)
+                obj = Format::TYPE_INFER.decode(value)
+          node_type = Format::NODE_TYPES[obj["node_type"]]
+        hash_prefix = Format::HASH_PREFIXES[obj["hash_prefix"].upcase]
+
+        if hash_prefix == :inner_node
+          return :inner_node, parse_inner_node(value)
+
+        elsif node_type == :account_node
+          return :account, parse_ledger_entry(value)
+
+        elsif node_type == :tx_node
+          return :tx, parse_tx(value)
+
+        elsif node_type == :ledger
+          return :ledger, parse_ledger(value)
+        end
+
+        return nil
+      end
+    end # class DB
+  end # module NodeStore
+end # module XRBP