reth 0.1.0

data/lib/reth/jsonrpc/server.rb

@@ -0,0 +1,24 @@
+module Reth
+module JSONRPC
+
+  class Server
+    include Concurrent::Async
+
+    def initialize(app, host, port)
+      super()
+
+      @app  = app
+      @host = host
+      @port = port
+    end
+
+    def start
+      Rack::Handler::WEBrick.run App.new(@app), Host: @host, Port: @port
+    rescue
+      puts $!
+      puts $!.backtrace[0,10].join("\n")
+    end
+  end
+
+end
+end

data/lib/reth/jsonrpc/service.rb

@@ -0,0 +1,37 @@
+module Reth
+module JSONRPC
+
+  class Service < ::DEVp2p::Service
+
+    class <<self
+      def register_with_app(app)
+        config = default_config[:jsonrpc]
+        app.register_service self, app, config[:host], config[:port]
+      end
+    end
+
+    name 'jsonrpc'
+    default_config(
+      jsonrpc: {
+        host: '127.0.0.1',
+        port: 8333
+      }
+    )
+
+    def initialize(app, host, port)
+      super(app)
+
+      @app = app
+      @host = host
+      @port = port
+    end
+
+    def start
+      @server = Server.new @app, @host, @port
+      @server.async.start
+    end
+
+  end
+
+end
+end

data/lib/reth/keystore.rb

@@ -0,0 +1,150 @@
+# -*- encoding : ascii-8bit -*-
+
+require 'openssl'
+
+module Reth
+
+  class Keystore
+
+    class PBKDF2
+      attr :name, :params
+
+      def initialize(params=nil)
+        @name = 'pbkdf2'
+        @params = params || mkparams
+      end
+
+      def eval(pw)
+        OpenSSL::PKCS5.pbkdf2_hmac(
+          pw,
+          Utils.decode_hex(params[:salt]),
+          params[:c],
+          params[:dklen],
+          'SHA256'
+        )
+      end
+
+      def mkparams
+        { prf: 'hmac-sha256',
+          dklen: 32,
+          c: 262144,
+          salt: Utils.encode_hex(SecureRandom.random_bytes(16)) }
+      end
+    end
+
+    class AES128CTR
+      attr :name, :params
+
+      def initialize(params=nil)
+        @name = 'aes-128-ctr'
+        @params = params || mkparams
+      end
+
+      def encrypt(text, key)
+        cipher = OpenSSL::Cipher.new name
+        cipher.encrypt
+        cipher.key = key
+        cipher.iv = Utils.decode_hex(params[:iv])
+        cipher.update(text) + cipher.final
+      end
+
+      def decrypt(text, key)
+        cipher = OpenSSL::Cipher.new name
+        cipher.decrypt
+        cipher.key = key
+        cipher.iv = Utils.decode_hex(params[:iv])
+        cipher.update(text) + cipher.final
+      end
+
+      def mkparams
+        {iv: Utils.encode_hex(SecureRandom.random_bytes(16))}
+      end
+    end
+
+    KDF = {
+      'pbkdf2' => PBKDF2
+    }.freeze
+
+    CIPHER = {
+      'aes-128-ctr' => AES128CTR
+    }.freeze
+
+    class <<self
+
+      def make_json(priv, pw, kdf=PBKDF2.new, cipher=AES128CTR.new)
+        derivedkey = kdf.eval pw
+
+        enckey = derivedkey[0,16]
+        c = cipher.encrypt priv, enckey
+
+        mac = Utils.keccak256 "#{derivedkey[16,16]}#{c}"
+        uuid = SecureRandom.uuid
+
+        {
+          crypto: {
+            cipher: cipher.name,
+            ciphertext: Utils.encode_hex(c),
+            cipherparams: cipher.params,
+            kdf: kdf.name,
+            kdfparams: kdf.params,
+            mac: Utils.encode_hex(mac),
+            version: 1
+          },
+          id: uuid,
+          version: 3
+        }
+      end
+
+      def decode_json(jsondata, pw)
+        jsondata = Hashie::Mash.new jsondata
+
+        cryptdata = jsondata.crypto || jsondata.Crypto
+        raise ArgumentError, "JSON data must contain 'crypto' object" unless cryptdata
+
+        kdfparams = cryptdata.kdfparams
+        kdf = KDF[cryptdata.kdf].new kdfparams
+
+        cipherparams = cryptdata.cipherparams
+        cipher = CIPHER[cryptdata.cipher].new cipherparams
+
+        derivedkey = kdf.eval pw
+        raise ValueError, "Derived key must be at least 32 bytes long" unless derivedkey.size >= 32
+
+        enckey = derivedkey[0,16]
+        ct = Utils.decode_hex cryptdata.ciphertext
+        o = cipher.decrypt ct, enckey
+
+        mac1 = Utils.keccak256 "#{derivedkey[16,16]}#{ct}"
+        mac2 = Utils.decode_hex cryptdata.mac
+        raise ValueError, "MAC mismatch. Password incorrect?" unless mac1 == mac2
+
+        o
+      end
+
+      ##
+      # Check if json has the structure of a keystore file version 3.
+      #
+      # Note that this test is not complete, e.g. it doesn't check key
+      # derivation or cipher parameters.
+      #
+      # @param json [Hash] data load from json file
+      # @return [Bool] `true` if the data appears to be valid, otherwise
+      #   `false`
+      #
+      def validate(json)
+        return false unless json.has_key?('crypto') || json.has_key?('Crypto')
+        return false unless json['version'] == 3
+
+        crypto = json['crypto'] || json['Crypto']
+        return false unless crypto.has_key?('cipher')
+        return false unless crypto.has_key?('ciphertext')
+        return false unless crypto.has_key?('kdf')
+        return false unless crypto.has_key?('mac')
+
+        true
+      end
+    end
+
+  end
+
+end

data/lib/reth/leveldb_service.rb

@@ -0,0 +1,79 @@
+# -*- encoding : ascii-8bit -*-
+
+module Reth
+
+  class LevelDBService < ::DEVp2p::Service
+    name 'leveldb'
+
+    attr :db # implement DB::BaseDB interface
+
+    def initialize(app)
+      super(app)
+      @db = DB::LevelDB.new File.join(app.config[:data_dir], 'leveldb')
+    end
+
+    def start
+      # do nothing
+    end
+
+    def stop
+      # do nothing
+    end
+
+    def get(k)
+      @db.get(k)
+    rescue KeyError
+      nil
+    end
+
+    def put(k, v)
+      @db.put(k, v)
+    end
+
+    def commit
+      @db.commit
+    end
+
+    def delete(k)
+      @db.delete(k)
+    end
+
+    def include?(k)
+      @db.include?(k)
+    end
+    alias has_key? include?
+
+    def inc_refcount(k, v)
+      put(k, v)
+    end
+
+    def dec_refcount(k)
+      # do nothing
+    end
+
+    def revert_refcount_changes(epoch)
+      # do nothing
+    end
+
+    def commit_refcount_changes(epoch)
+      # do nothing
+    end
+
+    def cleanup(epoch)
+      # do nothing
+    end
+
+    def put_temporarily(k, v)
+      inc_refcount(k, v)
+      dec_refcount(k)
+    end
+
+    private
+
+    def logger
+      @logger ||= Logger.new 'db'
+    end
+
+  end
+
+end

data/lib/reth/profile.rb

@@ -0,0 +1,66 @@
+# -*- encoding : ascii-8bit -*-
+
+module Reth
+
+  class Profile
+
+    GENESIS_DIR = File.expand_path('../genesisdata', __FILE__)
+
+    ALL = {
+      livenet: {
+        eth: {
+          network_id: 1,
+          genesis: File.join(GENESIS_DIR, 'genesis_frontier.json'),
+          genesis_hash: 'd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3',
+        },
+        discovery: {
+          bootstrap_nodes: [
+            'enode://487611428e6c99a11a9795a6abe7b529e81315ca6aad66e2a2fc76e3adf263faba0d35466c2f8f68d561dbefa8878d4df5f1f2ddb1fbeab7f42ffb8cd328bd4a@5.1.83.226:30303', # C++
+            'enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303', # GO
+            'enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303', # GO2
+            'enode://2676755dd8477ad3beea32b4e5a144fa10444b70dfa3e05effb0fdfa75683ebd4f75709e1f8126cb5317c5a35cae823d503744e790a3a038ae5dd60f51ee9101@144.76.62.101:30303', # Python
+          ]
+        },
+      },
+      testnet: {
+        eth: {
+          network_id: 2,
+          genesis: File.join(GENESIS_DIR, 'genesis_morden.json'),
+          genesis_hash: '0cd786a2425d16f152c658316c423e6ce1181e15c3295826d7c9904cba9ce303',
+          block: {
+            account_initial_nonce: 2 ** 20,
+            homestead_fork_blknum: 494000,
+          },
+        },
+        discovery: {
+          bootstrap_nodes: [
+            'enode://e4533109cc9bd7604e4ff6c095f7a1d807e15b38e9bfeb05d3b7c423ba86af0a9e89abbf40bd9dde4250fef114cd09270fa4e224cbeef8b7bf05a51e8260d6b8@94.242.229.4:40404' # Go
+          ]
+        },
+      }
+    }
+
+    class <<self
+      def all
+        @all ||= Hashie::Mash.new ALL
+      end
+
+      def public(name)
+        all[name]
+      end
+
+      def private(network_id)
+        Hashie::Mash.new({
+          eth: {
+            network_id: network_id
+          },
+          discovery: {
+            bootstrap_nodes: []
+          }
+        })
+      end
+    end
+
+  end
+
+end

data/lib/reth/sync_task.rb

@@ -0,0 +1,273 @@
+# -*- encoding : ascii-8bit -*-
+
+module Reth
+
+  ##
+  # Synchronizes the chain starting from a given blockhash. Blockchain hash
+  # is fetched from a single peer (which led to the unknown blockhash).
+  # Blocks are fetched from the best peers.
+  #
+  class SyncTask
+    MAX_BLOCKS_PER_REQUEST = 32
+    INITIAL_BLOCKHASHES_PER_REQUEST = 16
+    MAX_BLOCKHASHES_PER_REQUEST = 512
+
+    BLOCKS_REQUEST_TIMEOUT = 32
+    BLOCKHASHES_REQUEST_TIMEOUT = 32
+
+    attr :start_block_number, :end_block_number
+
+    def initialize(synchronizer, proto, blockhash, chain_difficulty=0, originator_only=false)
+      @synchronizer = synchronizer
+      @chain = synchronizer.chain
+      @chainservice = synchronizer.chainservice
+
+      @originating_proto = proto
+      @originator_only = originator_only
+
+      @blockhash = blockhash
+      @chain_difficulty = chain_difficulty
+
+      @requests = {} # proto => [cond, result]
+      @start_block_number = @chain.head.number
+      @end_block_number = @start_block_number + 1 # minimum synctask
+
+      @run = Thread.new { run }
+    end
+
+    def run
+      logger.info 'spawning new synctask'
+
+      fetch_hashchain
+    rescue
+      logger.error $!
+      logger.error $!.backtrace[0,20].join("\n")
+      task_exit false
+    end
+
+    def task_exit(success=false)
+      if success
+        logger.debug 'successfully synced'
+      else
+        logger.warn 'syncing failed'
+      end
+
+      @synchronizer.synctask_exited(success)
+    end
+
+    def protocols
+      return [@originating_proto] if @originator_only
+      @synchronizer.protocols
+    end
+
+    def fetch_hashchain
+      logger.debug 'fetching hashchain'
+
+      blockhashes_chain = [@blockhash] # youngest to oldest
+      blockhash = @blockhash = blockhashes_chain.last
+      raise AssertError if @chain.include?(blockhash)
+
+      # get block hashes until we found a known one
+      max_blockhashes_per_request = INITIAL_BLOCKHASHES_PER_REQUEST
+      chain_head_number = @chain.head.number
+      while !@chain.include?(blockhash)
+        blockhashes_batch = []
+
+        # proto with highest difficulty should be the proto we got the
+        # newblock from
+        protos = self.protocols
+        if protos.nil? || protos.empty?
+          logger.warn 'no protocols available'
+          return task_exit(false)
+        end
+
+        protos.each do |proto|
+          logger.debug "syncing with", proto: proto
+          next if proto.stopped?
+
+          raise AssertError if @requests.has_key?(proto)
+          deferred = Concurrent::IVar.new
+          @requests[proto] = deferred
+
+          proto.async.send_getblockhashes blockhash, max_blockhashes_per_request
+          begin
+            blockhashes_batch = deferred.value(BLOCKHASHES_REQUEST_TIMEOUT)
+          rescue Defer::TimedOut
+            logger.warn 'syncing hashchain timed out'
+            next
+          ensure
+            @requests.delete proto
+          end
+
+          if blockhashes_batch.empty?
+            logger.warn 'empty getblockhashes result'
+            next
+          end
+
+          unless blockhashes_batch.all? {|bh| bh.instance_of?(String) }
+            logger.warn "get wrong data type", expected: 'String', received: blockhashes_batch.map(&:class).uniq
+            next
+          end
+
+          break
+        end
+
+        if blockhashes_batch.empty?
+          logger.warn 'syncing failed with all peers', num_protos: protos.size
+          return task_exit(false)
+        end
+
+        if @chain.include?(blockhashes_batch.last)
+          blockhashes_batch.each do |bh| # youngest to oldest
+            blockhash = bh
+
+            if @chain.include?(blockhash)
+              logger.debug "found known blockhash", blockhash: Utils.encode_hex(blockhash), is_genesis: (blockhash == @chain.genesis.full_hash)
+              break
+            else
+              blockhashes_chain.push(blockhash)
+            end
+          end
+        else # no overlap
+          blockhashes_chain.concat blockhashes_batch
+          blockhash = blockhashes_batch.last
+        end
+
+        logger.debug "downloaded #{blockhashes_chain.size} block hashes, ending with #{Utils.encode_hex(blockhashes_chain.last)}"
+        @end_block_number = chain_head_number + blockhashes_chain.size
+        max_blockhashes_per_request = MAX_BLOCKHASHES_PER_REQUEST
+      end
+
+      @start_block_number = @chain.get(blockhash).number
+      @end_block_number = @start_block_number + blockhashes_chain.size
+
+      logger.debug 'computed missing numbers', start_number: @start_block_number, end_number: @end_block_number
+
+      fetch_blocks blockhashes_chain
+    end
+
+    def fetch_blocks(blockhashes_chain)
+      raise ArgumentError, 'no blockhashes' if blockhashes_chain.empty?
+      logger.debug 'fetching blocks', num: blockhashes_chain.size
+
+      blockhashes_chain.reverse! # oldest to youngest
+      num_blocks = blockhashes_chain.size
+      num_fetched = 0
+
+      while !blockhashes_chain.empty?
+        blockhashes_batch = blockhashes_chain[0, MAX_BLOCKS_PER_REQUEST]
+        t_blocks = []
+
+        protos = self.protocols
+        if protos.empty?
+          logger.warn 'no protocols available'
+          return task_exit(false)
+        end
+
+        proto = nil
+        reply_proto = nil
+        protos.each do |_proto|
+          proto = _proto
+
+          next if proto.stopped?
+          raise AssertError if @requests.has_key?(proto)
+
+          logger.debug 'requesting blocks', num: blockhashes_batch.size
+          deferred = Concurrent::IVar.new
+          @requests[proto] = deferred
+
+          proto.async.send_getblocks *blockhashes_batch
+          begin
+            t_blocks = deferred.value(BLOCKS_REQUEST_TIMEOUT)
+          rescue Defer::TimedOut
+            logger.warn 'getblocks timed out, trying next proto'
+            next
+          ensure
+            @requests.delete proto
+          end
+
+          if t_blocks.empty?
+            logger.warn 'empty getblocks reply, trying next proto'
+            next
+          elsif !t_blocks.all? {|b| b.instance_of?(TransientBlock) }
+            logger.warn 'received unexpected data', data: t_blocks
+            t_blocks = []
+            next
+          end
+
+          unless t_blocks.map {|b| b.header.full_hash } == blockhashes_batch[0, t_blocks.size]
+            logger.warn 'received wrong blocks, should ban peer'
+            t_blocks = []
+            next
+          end
+
+          reply_proto = proto
+          break
+        end
+
+        # add received t_blocks
+        num_fetched += t_blocks.size
+        logger.debug "received blocks", num: t_blocks.size, num_fetched: num_fetched, total: num_blocks, missing: (num_blocks - num_fetched)
+
+        if t_blocks.empty?
+          logger.warn 'failed to fetch blocks', missing: blockhashes_chain.size
+          return task_exit(false)
+        end
+
+        t = Time.now
+        logger.debug 'adding blocks', qsize: @chainservice.block_queue.size
+        t_blocks.each do |blk|
+          b = blockhashes_chain.shift
+          raise AssertError unless blk.header.full_hash == b
+          raise AssertError if blockhashes_chain.include?(blk.header.full_hash)
+
+          @chainservice.add_block blk, reply_proto # this blocks if the queue is full
+        end
+        logger.debug 'adding blocks done', took: (Time.now - t)
+      end
+
+      # done
+      last_block = t_blocks.last
+      raise AssertError, 'still missing blocks' unless blockhashes_chain.empty?
+      raise AssertError, 'still missing blocks' unless last_block.header.full_hash == @blockhash
+      logger.debug 'syncing finished'
+
+      # at this time blocks are not in the chain yet, but in the add_block queue
+      if @chain_difficulty >= @chain.head.chain_difficulty
+        @chainservice.broadcast_newblock last_block, @chain_difficulty, proto
+      end
+
+      task_exit(true)
+    rescue
+      logger.error $!
+      logger.error $!.backtrace[0,10].join("\n")
+      task_exit(false)
+    end
+
+    def receive_blocks(proto, t_blocks)
+      logger.debug 'blocks received', proto: proto, num: t_blocks.size
+      unless @requests.has_key?(proto)
+        logger.debug 'unexpected blocks'
+        return
+      end
+      @requests[proto].set t_blocks
+    end
+
+    def receive_blockhashes(proto, blockhashes)
+      logger.debug 'blockhashes received', proto: proto, num: blockhashes.size
+      unless @requests.has_key?(proto)
+        logger.debug 'unexpected blockhashes'
+        return
+      end
+      @requests[proto].set blockhashes
+    end
+
+    private
+
+    def logger
+      @logger ||= Logger.new('eth.sync.task')
+    end
+
+  end
+
+end