ciri-p2p 0.1.0

data/lib/ciri/p2p/discovery/service.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+
+# Copyright (c) 2018 by Jiang Jinyang <jjyruby@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# TODO Items:
+# [x] implement k-buckets algorithm
+# [x] implement peerstore(may use sqlite)
+# [ ] implement a simple scoring system
+# [ ] testing
+require 'async'
+require 'async/io/udp_socket'
+require 'async/io/endpoint/each'
+require 'ciri/utils/logger'
+require 'ciri/core_ext'
+require 'ciri/p2p/node'
+require 'ciri/p2p/address'
+require 'ciri/p2p/peer_store'
+require 'ciri/p2p/kad'
+require_relative 'protocol'
+
+using Ciri::CoreExt
+
+module Ciri
+  module P2P
+    module Discovery
+
+      # Implement the DiscV4 protocol
+      # https://github.com/ethereum/devp2p/blob/master/discv4.md
+      # notice difference between PeerStore and Kad,
+      # we use PeerStore to store all peers we known(upon 8192),
+      # and use Kad to store our neighbours for discovery query.
+      class Service
+        include Utils::Logger
+        # use message classes defined in Discovery
+        include Protocol
+
+        attr_reader :peer_store, :local_node_id, :host, :udp_port, :tcp_port
+
+        # we should consider search from peer_store instead connect to bootnodes everytime 
+        def initialize(peer_store:, host:, udp_port:, tcp_port:, private_key:, discovery_interval_secs: 15)
+          @discovery_interval_secs = discovery_interval_secs
+          @cache = Set.new
+          @host = host
+          @udp_port = udp_port
+          @tcp_port = tcp_port
+          @peer_store = peer_store
+          @private_key = private_key
+          @local_node_id = NodeID.new(private_key)
+          @kad_table = Kad::RoutingTable.new(local_node: Kad::Node.new(@local_node_id.to_bytes))
+          setup_kad_table
+        end
+
+        def run(task: Async::Task.current)
+          # start listening
+          task.async do
+            start_listen
+          end
+          # search peers every x seconds
+          task.reactor.every(@discovery_interval_secs) do
+            task.async do
+              perform_discovery
+            end
+          end
+        end
+
+        private
+        def start_listen(task: Async::Task.current)
+          endpoint = Async::IO::Endpoint.udp(@host, @udp_port)
+          endpoint.bind do |socket|
+            @local_address = socket.local_address
+
+            # update port if port is zero
+            if @udp_port.zero?
+              @udp_port = @local_address.ip_port
+            end
+
+            debug "start discovery server on udp_port: #{@udp_port} tcp_port: #{@tcp_port}\nlocal_node_id: #{@local_node_id}"
+
+            loop do
+              # read discovery message
+              packet, address = socket.recvfrom(Discovery::Protocol::Message::MAX_LEN)
+              handle_request(packet, address)
+            end
+          end
+        end
+
+        MESSAGE_EXPIRATION_IN = 10 * 60 # set 10 minutes later to expiration message
+
+        def handle_request(raw_packet, address, now: Time.now.to_i)
+          msg = Message.decode_message(raw_packet)
+          msg.validate
+          if msg.packet.expiration < now
+            trace("ignore expired message, sender: #{msg.sender}, expired_at: #{msg.packet.expiration}")
+            return
+          end
+          raw_node_id = msg.sender.to_bytes
+          case msg.packet_type
+          when Ping::CODE
+            @kad_table.update(raw_node_id)
+            from = msg.packet.from
+            from_ip = IPAddr.new(from.sender_ip, Socket::AF_INET)
+            from_udp_port = from.sender_udp_port
+            from_tcp_port = from.sender_tcp_port
+            from_address = Address.new(
+              ip: from_ip,
+              udp_port: from_udp_port, 
+              tcp_port: from_tcp_port)
+            debug("receive ping msg from #{from_address.inspect}")
+            # respond pong
+            pong = Pong.new(to: To.from_host_port(from_ip, from_udp_port), 
+                            ping_hash: msg.message_hash, 
+                            expiration: Time.now.to_i + MESSAGE_EXPIRATION_IN)
+            pong_msg = Message.pack(pong, private_key: @private_key).encode_message
+            send_msg(pong_msg, from_ip.to_s, from_udp_port)
+            @peer_store.add_node(Node.new(raw_node_id: raw_node_id, addresses: [from_address]))
+          when Pong::CODE
+            # check pong
+            if @peer_store.has_ping?(raw_node_id, msg.packet.ping_hash)
+              # update peer last seen
+              @peer_store.update_last_seen(msg.sender.to_bytes)
+            else
+              @peer_store.ban_peer(msg.sender.to_bytes)
+            end
+          when FindNode::CODE
+            unless @peer_store.has_seen?(raw_node_id)
+              # consider add to denylist
+              return
+            end
+            nodes = find_neighbours(msg.packet.target, 20).map do |raw_node_id, addr|
+              Neighbors::Node.new(ip: addr.ip.to_i, udp_port: addr.udp_port, tcp_port: addr.tcp_port, node_id: raw_node_id)
+            end
+            neighbors = Neighbors.new(nodes: nodes, expiration: Time.now.to_i + MESSAGE_EXPIRATION_IN)
+            send_msg_to_node(Message.pack(neighbors, private_key: @private_key).encode_message, raw_node_id)
+            @peer_store.update_last_seen(raw_node_id)
+          when Neighbors::CODE
+            unless @peer_store.has_seen?(raw_node_id)
+              # consider add to denylist
+              return
+            end
+            debug("receive neighours #{msg.packet.nodes.size} from #{raw_node_id.to_hex}")
+            msg.packet.nodes.each do |node|
+              raw_id = node.node_id
+              next if raw_id == raw_local_node_id
+              debug("receive neighour #{node} from #{raw_node_id.to_hex}")
+              ip = IPAddr.new(node.ip, Socket::AF_INET)
+              address = Address.new(ip: ip, udp_port: node.udp_port, tcp_port: node.tcp_port)
+              @peer_store.add_node(Node.new(raw_node_id: raw_id, addresses: [address]))
+              # add new discovered node_id
+              @kad_table.update(raw_id)
+            end
+            @kad_table.update(raw_node_id)
+            @peer_store.update_last_seen(raw_node_id)
+          else
+            @peer_store.ban_peer(msg.sender.to_bytes)
+            raise UnknownMessageCodeError.new("can't handle unknown code in discovery protocol, code: #{msg.packet_type}")
+          end
+        rescue StandardError => e
+          @peer_store.ban_peer(msg.sender.to_bytes)
+          error("discovery error: #{e} from address: #{address}\nbacktrace:#{e.backtrace.join("\n")}")
+        end
+
+        def send_ping_to_address(target_node_id, address)
+          send_ping(target_node_id, address[3], address[1])
+        end
+
+        # send discover ping to peer
+        def send_ping(target_node_id, host, port)
+          ping = Ping.new(to: To.from_host_port(host, port), 
+                          from: From.new(
+                            sender_ip: IPAddr.new(@host).to_i,
+                            sender_udp_port: @udp_port,
+                            sender_tcp_port: @tcp_port),
+                            expiration: Time.now.to_i + MESSAGE_EXPIRATION_IN)
+          ping_msg = Message.pack(ping, private_key: @private_key)
+          send_msg(ping_msg.encode_message, host, port)
+          @peer_store.update_ping(target_node_id, ping_msg.message_hash)
+        end
+
+        def send_msg_to_node(msg, raw_node_id)
+          address = @peer_store.get_node_addresses(raw_node_id)&.first
+          raise ArgumentsError.new("can't found peer address of #{raw_node_id.to_hex} from peer_store") unless address
+          send_msg(msg, address.ip.to_s, address.udp_port)
+        end
+
+        def send_msg(msg, host, port)
+          socket = Async::IO::UDPSocket.new(UDPSocket.new)
+          socket.send(msg, 0, host, port)
+        end
+
+        def raw_local_node_id
+            @raw_local_node_id ||= @local_node_id.to_bytes
+        end
+
+        # find nerly neighbours
+        def find_neighbours(raw_node_id, count)
+          @kad_table.find_neighbours(raw_node_id, k: count).map do |node|
+            [node.raw_node_id, @peer_store.get_node_addresses(node.raw_node_id)&.first]
+          end.delete_if do |_, addr|
+            addr.nil?
+          end
+        end
+
+        def setup_kad_table
+          if @kad_table.size.zero?
+            @peer_store.find_bootnodes(20).each do |node|
+              next if raw_local_node_id == node.raw_node_id
+              debug("setup kad_table with #{node}")
+              @kad_table.update(node.raw_node_id)
+            end
+          end
+        end
+
+        def perform_discovery(count_of_query_nodes=15, task: Async::Task.current)
+          query_node = NodeID.new(Key.random)
+          query_target = query_node.to_bytes
+          # randomly search
+          @kad_table.get_random_nodes(15).each do |node|
+            address = @peer_store.get_node_addresses(node.raw_node_id)&.first
+            next unless address
+            # start query node in async task
+            task.async do
+              debug("perform discovery #{address}")
+              send_ping(node.raw_node_id, address.ip.to_s, address.udp_port)
+              query = FindNode.new(target: query_target, expiration: Time.now.to_i + MESSAGE_EXPIRATION_IN)
+              query_msg = Message.pack(query, private_key: @private_key).encode_message
+              send_msg(query_msg, address.ip.to_s, address.udp_port)
+            end
+          end
+        end
+      end
+
+    end
+  end
+end
+

data/lib/ciri/p2p/errors.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+
+# Copyright (c) 2018 by Jiang Jinyang <jjyruby@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+module Ciri
+  module P2P
+
+    class Error < StandardError; end
+    class UselessPeerError < Error; end
+    class DisconnectError < Error; end
+    class UnknownMessageCodeError < Error; end
+    class InvalidMessageError < Error; end
+
+  end
+end
+

data/lib/ciri/p2p/kad.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+
+# Copyright (c) 2018 by Jiang Jinyang <jjyruby@gmail.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+require 'ciri/utils/logger'
+require 'ciri/utils'
+require 'ciri/p2p/node'
+require 'forwardable'
+
+module Ciri
+  module P2P
+
+
+    # Kademlia algorithm
+    # modified from https://github.com/ethereum/py-evm/blob/master/p2p/kademlia.py
+    module Kad
+      K_BITS = 8
+      K_BUCKET_SIZE = 16
+      K_REQUEST_TIMEOUT = 0.9
+      K_IDLE_BUCKET_REFRESH_INTERVAL = 3600
+      K_PUBKEY_SIZE = 512
+      K_ID_SIZE = 256
+      K_MAX_NODE_ID = 2 ** K_ID_SIZE - 1
+
+      class Node
+        attr_reader :id, :raw_node_id
+
+        def initialize(raw_node_id)
+          @raw_node_id = raw_node_id
+          @id = Utils.big_endian_decode(Utils.keccak(raw_node_id))
+        end
+
+        def distance_to(id)
+          @id ^ id
+        end
+
+        def ==(other)
+          self.class == other.class && self.id == other.id
+        end
+
+        def <=>(other)
+          @id <=> other.id
+        end
+      end
+
+      class KBucket
+
+        attr_reader :k_size, :nodes, :start_id, :end_id, :last_updated, :replacement_cache
+
+        def initialize(start_id:, end_id:, k_size: K_BUCKET_SIZE)
+          @start_id = start_id
+          @end_id = end_id
+          @k_size = k_size
+          @nodes = []
+          @replacement_cache = []
+          @last_updated = Time.now.to_i
+        end
+
+        # use to compute node distance with kbucket
+        def midpoint
+          @start_id + (@end_id - @start_id) / 2
+        end
+
+        def distance_to(id)
+          midpoint ^ id
+        end
+
+        # find neighbour nodes
+        def nodes_by_distance_to(id)
+          @nodes.sort_by do |node|
+            node.distance_to(id)
+          end
+        end
+
+        # split to two kbucket by midpoint
+        def split
+          split_point = midpoint
+          lower = KBucket.new(start_id: @start_id, end_id: split_point)
+          upper = KBucket.new(start_id: split_point + 1, end_id: @end_id)
+          @nodes.each do |node|
+            if node.id <= split_point
+              lower.add(node)
+            else
+              upper.add(node)
+            end
+          end
+          @replacement_cache.each do |node|
+            if node.id <= split_point
+              lower.replacement_cache << node
+            else
+              upper.replacement_cache << node
+            end
+          end
+          [lower, upper]
+        end
+
+        def delete(node)
+          @nodes.delete(node)
+        end
+
+        def cover?(node)
+          @start_id <= node.id && node.id <= @end_id
+        end
+
+        def full?
+          @nodes.size == k_size
+        end
+
+        # Try add node into bucket
+        # if node is exists, it is moved to the tail
+        # if the node is node exists and bucket not full, it is added at tail
+        # if the bucket is full, node will added to replacement_cache, and return the head of the list, which should be evicted if it failed to respond to a ping.
+        def add(node)
+          @last_updated = Time.now.to_i
+          if @nodes.include?(node)
+            @nodes.delete(node)
+            @nodes << node
+          elsif @nodes.size < k_size
+            @nodes << node
+          else
+            @replacement_cache << node
+            return head
+          end
+          nil
+        end
+
+        def head
+          @nodes[0]
+        end
+
+        def include?(node)
+          @nodes.include?(node)
+        end
+
+        def size
+          @nodes.size
+        end
+      end
+
+      class RoutingTable
+        attr_reader :buckets, :local_node
+
+        def initialize(local_node:)
+          @local_node = local_node
+          @buckets = [KBucket.new(start_id: 0, end_id: K_MAX_NODE_ID)]
+        end
+
+        def get_random_nodes(count)
+          count = size if count > size
+          nodes = []
+          while nodes.size < count
+            bucket = @buckets.sample
+            next if bucket.nodes.empty?
+            node = bucket.nodes.sample
+            unless nodes.include?(node)
+              nodes << node
+            end
+          end
+          nodes
+        end
+
+        def idle_buckets
+          bucket_idled_at = Time.now.to_i - K_IDLE_BUCKET_REFRESH_INTERVAL
+          @buckets.select do |bucket| 
+            bucket.last_updated < bucket_idled_at
+          end
+        end
+
+        def not_full_buckets
+          @buckets.select do |bucket|
+            !bucket.full?
+          end
+        end
+
+        def delete_node(node)
+          find_bucket_for_node(node).delete(node)
+        end
+
+        def update(raw_node_id)
+          add_node(Node.new(raw_node_id))
+        end
+
+        def add_node(node)
+          raise ArgumentError.new("can't add local_node") if @local_node == node
+          bucket = find_bucket_for_node(node)
+          eviction_candidate = bucket.add(node)
+          # bucket is full, otherwise will return nil
+          if eviction_candidate
+            depth = compute_shared_prefix_bits(bucket.nodes)
+            if bucket.cover?(@local_node) || (depth % K_BITS != 0 && depth != K_ID_SIZE)
+              split_bucket(@buckets.index(bucket))
+              return add_node(node)
+            end
+            return eviction_candidate
+          end
+          nil
+        end
+
+        def buckets_by_distance_to(id)
+          @buckets.sort_by do |bucket|
+            bucket.distance_to(id)
+          end
+        end
+
+        def include?(node)
+          find_bucket_for_node(node).include?(node)
+        end
+
+        def size
+          @buckets.map(&:size).sum
+        end
+
+        def each_node(&blk)
+          @buckets.each do |bucket|
+            bucket.nodes do |node|
+              blk.call(node)
+            end
+          end
+        end
+
+        def find_neighbours(id, k: K_BUCKET_SIZE)
+          # convert id to integer
+          unless id.is_a?(Integer)
+            id = Node.new(id).id
+          end
+          nodes = []
+          buckets_by_distance_to(id).each do |bucket|
+            bucket.nodes_by_distance_to(id).each do |node|
+              if node.id != id
+                nodes << node
+                # find 2 * k nodes to avoid edge cases
+                break if nodes.size == k * 2
+              end
+            end
+          end
+          sort_by_distance(nodes, id)[0...k]
+        end
+
+        # do binary search to find node
+        def find_bucket_for_node(node)
+          @buckets.bsearch do |bucket|
+            bucket.end_id >= node.id
+          end
+        end
+
+        private
+
+        def split_bucket(index)
+          bucket = @buckets[index]
+          a, b = bucket.split
+          @buckets[index] = a
+          @buckets.insert(index + 1, b)
+        end
+
+        def compute_shared_prefix_bits(nodes)
+          return K_ID_SIZE if nodes.size < 2
+          bits = nodes.map{|node| to_binary(node.id) }
+          (1..K_ID_SIZE).each do |i|
+            # check common prefix shared by nodes
+            if bits.map{|b| b[0..i]}.uniq.size != 1
+              return i - 1
+            end
+          end
+        end
+
+        def sort_by_distance(nodes, target_id)
+          nodes.sort_by do |node|
+            node.distance_to(target_id)
+          end
+        end
+
+        def to_binary(x)
+          x.to_s(2).b.rjust(K_ID_SIZE, "\x00".b)
+        end
+
+      end
+
+    end
+  end
+end
+