arf 0.1.0

data/lib/arf/proto/map.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    EMPTY_MAP_MASK = 0x01 << 4
+
+    def self.encode_map(v)
+      t = TYPE_MAP
+      if v.empty?
+        t |= EMPTY_MAP_MASK
+        return [t].pack("C*")
+      end
+
+      keys = []
+      values = []
+      v.each_pair do |key, value|
+        keys << encode(key)
+        values << encode(value)
+      end
+
+      encoded_len = encode_uint64(v.length)
+      keys = keys.join
+      values = values.join
+
+      IO::Buffer.new
+        .write(t)
+        .write_raw(encode_uint64(keys.length + values.length + encoded_len.length))
+        .write_raw(encoded_len)
+        .write_raw(keys)
+        .write_raw(values)
+        .string
+    end
+
+    def self.decode_map(header, io)
+      return {} if header.anybits?(EMPTY_MAP_MASK)
+
+      decode_uint64(io) # Discard full length
+
+      pairs_len = decode_uint64(io)
+      keys = []
+      values = []
+
+      pairs_len.times { keys << decode(io) }
+      pairs_len.times { values << decode(io) }
+
+      keys.zip(values).to_h
+    end
+  end
+end

data/lib/arf/proto/registry.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    module Registry
+      def self.register!(cls)
+        @structs ||= {}
+        id = cls.arf_struct_id
+        fields = Proto.fields_from_struct(cls)
+        @structs[id] = {
+          id:,
+          fields:,
+          type: cls
+        }
+      end
+
+      def self.reset!
+        @structs = {}
+      end
+
+      def self.find(id)
+        @structs ||= {}
+        @structs[id]
+      end
+    end
+  end
+end

data/lib/arf/proto/scalar.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    NUMERIC_SIGNED_MASK = 0x01 << 4
+    NUMERIC_ZERO_MASK = 0x01 << 5
+    NUMERIC_NEGATIVE_MASK = 0x01 << 6
+
+    def self.encode_scalar(v, signed: false)
+      type = TYPE_SCALAR
+      type |= NUMERIC_SIGNED_MASK if signed
+      type |= NUMERIC_ZERO_MASK if v.zero?
+      if v.negative?
+        type |= NUMERIC_NEGATIVE_MASK
+        v *= -1
+      end
+
+      [
+        [type].pack("C*"),
+        v.zero? ? nil : encode_uint64(v)
+      ].compact.join
+    end
+
+    def self.decode_scalar(header, io)
+      return 0 if header.anybits?(NUMERIC_ZERO_MASK)
+
+      v = decode_uint64(io)
+      v *= -1 if header.anybits?(NUMERIC_NEGATIVE_MASK)
+      v
+    end
+
+    def self.encode_uint64(v)
+      bytes = []
+      while v >= 0x80
+        bytes << ((v & 0xFF) | 0x80)
+        v >>= 7
+      end
+      bytes << v
+      bytes.pack("C*")
+    end
+
+    def self.decode_uint64(io)
+      x = 0
+      s = 0
+      b = 0
+      loop do
+        b = io.read(1).getbyte(0)
+        return (x | (b << s)) if b < 0x80
+
+        x |= ((b & 0x7f) << s)
+        s += 7
+      end
+    end
+  end
+end

data/lib/arf/proto/string.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    STRING_EMPTY_MASK = 0x01 << 4
+
+    def self.encode_string(s)
+      t = TYPE_STRING
+      if s.nil? || s.empty?
+        t |= STRING_EMPTY_MASK
+        return [t].pack("C*")
+      end
+
+      s = s.to_s.encode("UTF-8")
+
+      [
+        [t].pack("C*"),
+        encode_uint64(s.bytesize),
+        s
+      ].join
+    end
+
+    def self.decode_string(header, io)
+      return "" if header.anybits?(STRING_EMPTY_MASK)
+
+      size = decode_uint64(io)
+      data = StringIO.new
+      until size.zero?
+        read = io.readpartial(size)
+        data.write(read)
+        size -= read.length
+      end
+      data.string.encode("UTF-8")
+    end
+  end
+end

data/lib/arf/proto/struct.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    def self.fields_from_struct(v)
+      base = v.is_a?(Class) ? v : v.class
+      fields = []
+      base.fields.each do |f|
+        if f[:id].is_a? Integer
+          fields << if f[:type].is_a?(Symbol) ||
+                       f[:type].is_a?(Arf::Types::ArrayType) ||
+                       f[:type].is_a?(Arf::Types::MapType)
+                      f
+                    else
+                      {
+                        id: f[:id],
+                        name: f[:name],
+                        type: base.find_type(f[:type])
+                      }
+                    end
+        else
+          raise UnsupportedNestedUnionError, "Nested unions are not supported" if base.union?
+
+          union_type = base.find_type(f[:type])
+          fields << {
+            id: union_type.fields.first[:id],
+            type: union_type,
+            name: f[:name]
+          }
+        end
+      end
+
+      fields.sort_by! { _1[:id] }
+      fields
+    end
+
+    def self.encode_struct(v)
+      return encode_union(v) if v.union?
+
+      struct_id = v.arf_struct_id
+      fields = fields_from_struct(v)
+      data = []
+      fields.each do |f|
+        data << encode_uint64(f[:id])
+        data << encode_as(v.instance_variable_get("@#{f[:name]}"), f[:type])
+      end
+
+      payload = data.join
+
+      [
+        [TYPE_STRUCT].pack("C*"),
+        encode_string(struct_id),
+        encode_uint64(payload.length),
+        payload
+      ].join
+    end
+
+    def self.decode_struct(_header, io)
+      id_type, id_header = read_type(io)
+      if id_type != TYPE_STRING
+        # :nocov:
+        raise DecodeFailedError, "cannot decode struct: expected String, found #{TYPE_NAME[id_type]}"
+        # :nocov:
+      end
+
+      struct_id = decode_string(id_header, io)
+      bytes_len = decode_uint64(io)
+      reader = IO::LimitReader.new(io, bytes_len)
+      fields = {}
+      loop do
+        id = decode_uint64(reader)
+        fields[id] = decode(reader)
+      rescue EOFError
+        break
+      end
+
+      meta_str = Registry.find(struct_id)
+      raise UnknownMeessageError, "Unknown message ID #{struct_id}" unless meta_str
+
+      inst = meta_str[:type].new
+      inst.decode_fields(fields)
+    end
+  end
+end

data/lib/arf/proto/types.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    TYPE_VOID    = 0b0000
+    TYPE_SCALAR  = 0b0001
+    TYPE_BOOLEAN = 0b0010
+    TYPE_FLOAT   = 0b0011
+    TYPE_STRING  = 0b0100
+    TYPE_BYTES   = 0b0101
+    TYPE_ARRAY   = 0b0110
+    TYPE_MAP     = 0b0111
+    TYPE_STRUCT  = 0b1000
+    TYPE_UNION   = 0b1001
+
+    ALL_PRIMITIVES = [
+      TYPE_VOID,
+      TYPE_SCALAR,
+      TYPE_BOOLEAN,
+      TYPE_FLOAT,
+      TYPE_STRING,
+      TYPE_BYTES,
+      TYPE_ARRAY,
+      TYPE_MAP,
+      TYPE_STRUCT,
+      TYPE_UNION
+    ].freeze
+
+    TYPE_NAME = {
+      TYPE_VOID => "Void",
+      TYPE_SCALAR => "Scalar",
+      TYPE_BOOLEAN => "Boolean",
+      TYPE_FLOAT => "Float",
+      TYPE_STRING => "String",
+      TYPE_BYTES => "Bytes",
+      TYPE_ARRAY => "Array",
+      TYPE_MAP => "Map",
+      TYPE_STRUCT => "Struct",
+      TYPE_UNION => "Union"
+    }.freeze
+
+    SIMPLE_PRIMITIVES = {
+      void: TYPE_VOID,
+      uint8: TYPE_SCALAR,
+      uint16: TYPE_SCALAR,
+      uint32: TYPE_SCALAR,
+      uint64: TYPE_SCALAR,
+      int8: TYPE_SCALAR,
+      int16: TYPE_SCALAR,
+      int32: TYPE_SCALAR,
+      int64: TYPE_SCALAR,
+      bool: TYPE_BOOLEAN,
+      float32: TYPE_FLOAT,
+      float64: TYPE_FLOAT,
+      string: TYPE_STRING,
+      bytes: TYPE_BYTES
+    }.freeze
+
+    def self.read_type(io)
+      b = io.read(1).getbyte(0)
+      decoded = b & 0xF
+      raise UnknownTypeError, format("Unknown type 0x%02x", b) unless ALL_PRIMITIVES.include? decoded
+
+      [decoded, b]
+    end
+  end
+end

data/lib/arf/proto/union.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Arf
+  module Proto
+    def self.encode_union(v)
+      selected = v.__arf_union_set_id
+      f = fields_from_struct(v).find { _1[:id] == selected }
+      payload = encode_as(v.instance_variable_get("@#{f[:name]}"), f[:type])
+
+      [
+        [TYPE_UNION].pack("C*"),
+        encode_uint64(selected),
+        payload
+      ].join
+    end
+
+    def self.decode_union(_header, io)
+      {
+        union: true,
+        id: decode_uint64(io),
+        value: decode(io)
+      }
+    end
+  end
+end

data/lib/arf/proto.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "proto/types"
+require_relative "proto/registry"
+
+require_relative "proto/scalar"
+require_relative "proto/boolean"
+require_relative "proto/bytes"
+require_relative "proto/float"
+require_relative "proto/string"
+require_relative "proto/array"
+require_relative "proto/map"
+require_relative "proto/struct"
+require_relative "proto/union"
+
+require_relative "proto/encoder"
+require_relative "proto/decoder"

data/lib/arf/reactor.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+module Arf
+  class Reactor
+    BEAT_INTERVAL = 3
+
+    def self.instance = (@instance ||= new)
+    def self.connect(host, port, handler) = instance.connect(host, port, handler)
+    def self.post(task = nil, &) = instance.post(task, &)
+    def self.client_writes_pending(client) = instance.client_writes_pending(client)
+    def self.detach_client(client) = instance.detach_client(client)
+    def self.detach(io) = instance.detach(io)
+
+    def self.attach_server(server, handler_class, *handler_args)
+      instance.attach_server(server, handler_class, *handler_args)
+    end
+
+    def initialize
+      @nio = @executor = @thread = nil
+      @stopping = false
+      @todo = Queue.new
+      @spawn_lock = Monitor.new
+      @map = {}
+      @socket_source = {}
+      @log = Arf.configuration.logger.with_fields(subsystem: "Reactor")
+    end
+
+    def timer(interval, &block)
+      @log.debug("Attached timer", interval:, callable: block)
+      Concurrent::TimerTask.new(execution_interval: interval, &block).tap(&:execute)
+    end
+
+    def connect(host, port, handler)
+      @log.debug("Establishing connection", host:, port:, handler:)
+      handler = handler.new
+      @todo << lambda {
+        addr_info = Socket.getaddrinfo(host, port, nil, Socket::SOCK_STREAM).first
+        family = addr_info[4]
+        io = Socket.new(family, Socket::SOCK_STREAM, 0)
+        begin
+          io.connect_nonblock Socket.sockaddr_in(port, host)
+        rescue Errno::EINPROGRESS
+          @log.debug("Connection in progress", id: handler.object_id)
+          @map[io] = @nio.register(io, :w)
+          @map[io].value = [host, port, handler]
+          @socket_source[io] = :client
+          next
+        end
+        @log.debug("Connection succeeded", id: handler.object_id)
+        @map[io] = @nio.register(io, :r)
+        @map[io].value = handler
+        @socket_source[io] = :client
+        @log.debug("Async post_init dispatch", id: handler.object_id)
+        post { handler.registered! }
+        post { handler.post_init } if handler.respond_to?(:post_init)
+      }
+      wakeup
+      handler
+    end
+
+    def attach_server(server, handler_class, *handler_args)
+      @log.debug("Attach server", server:, handler_class:)
+      @todo << lambda {
+        @map[server] = @nio.register(server, :r)
+        @map[server].value = [handler_class, handler_args]
+      }
+      wakeup
+    end
+
+    def detach(io)
+      @log.debug("Detach IO", io:)
+      @todo << lambda {
+        @nio.deregister(io)
+        @map.delete(io)
+        @socket_source.delete(io)
+        io.close
+      }
+      wakeup
+    end
+
+    def detach_client(client)
+      @log.debug("Detach client", client: client.class.name, id: client.object_id)
+      @todo << lambda {
+        io = io_for_client(client)
+        next unless io
+
+        @nio.deregister(io)
+        @map.delete(io)
+        @socket_source.delete(io)
+        io.close
+      }
+      wakeup
+    end
+
+    def post(task = nil, &block)
+      task ||= block
+      spawn
+      source = caller[0]
+      @executor << lambda do
+        task.call
+      rescue Exception => e
+        @log.error("Async post execution failed", e, source:)
+      end
+    end
+
+    def io_for_client(client)
+      @map.each do |k, v|
+        return k if v.value == client
+      end
+      nil
+    end
+
+    def client_writes_pending(client)
+      @log.debug("Writes pending", client: client.class.name, id: client.object_id)
+      @todo << lambda {
+        if (monitor = @map[io_for_client(client)])
+          monitor.interests = :rw
+        else
+          @log.warn("No monitor for client", id: client.object_id)
+        end
+      }
+      wakeup
+      @log.debug("Writes pending registered")
+    end
+
+    private
+
+    def spawn
+      return if @thread&.alive?
+
+      @spawn_lock.synchronize do
+        return if @thread&.alive?
+
+        @nio ||= NIO::Selector.new
+
+        @executor ||= Concurrent::ThreadPoolExecutor.new(
+          min_threads: 1,
+          max_threads: 10,
+          max_queue: 0
+        )
+
+        @thread = Thread.new { run }
+        @thread.name = "Arf::Wire::Reactor loop"
+
+        setup_ping_timer
+        true
+      end
+    end
+
+    def setup_ping_timer
+      @setup_ping_timer ||= timer(BEAT_INTERVAL) do
+        post do
+          @map.each_value do |v|
+            if v.is_a? Wire::BaseConnection
+              @log.debug("Dispatch ping", client: v)
+              v.ping
+            end
+          end
+        end
+      end
+    end
+
+    def handle_server_monitor(monitor)
+      io = monitor.io
+      handler, handler_args = monitor.value
+      return unless monitor.readable?
+
+      client = io.accept_nonblock
+      inst = handler.new(*handler_args)
+      # Basically #attach, but without the round-trip.
+      @map[client] = @nio.register(client, :r)
+      @map[client].value = inst
+      @socket_source[client] = :server
+      post { inst.registered! }
+      post { inst.post_init } if inst.respond_to? :post_init
+    end
+
+    def handle_socket_monitor(monitor)
+      io = monitor.io
+      client = monitor.value
+
+      if client.is_a? Array
+        return unless monitor.writable?
+
+        # This is a socket waiting for connection
+        host, port, handler = client
+        begin
+          io.connect_nonblock Socket.sockaddr_in(port, host)
+        rescue Errno::EISCONN
+          monitor.value = handler
+          monitor.interests = :r
+          @log.debug("Async post_init dispatch", id: handler.object_id)
+          post { handler.registered! }
+          post { handler.post_init } if handler.respond_to?(:post_init)
+        end
+        return
+      end
+
+      begin
+        if monitor.writable?
+          if client.flush_write_buffer(io)
+            monitor.interests = :r
+            detach(io) if client.arf_close_after_writing?
+          end
+          return unless monitor.readable?
+        end
+
+        incoming = io.read_nonblock(4096, exception: false)
+        case incoming
+        when :wait_readable
+          nil
+        when nil
+          post do
+            client.close
+          rescue Exception => e
+            @log.error("Failed closing client", e, id: client.object_id)
+            @nio.deregister(io)
+            @map.delete(io)
+            @socket_source.delete(io)
+          end
+        else
+          post do
+            client.recv(incoming)
+          rescue Exception => e
+            @log.error("Failed calling client.recv", e, id: client.object_id)
+            begin
+              client.close
+            rescue Exception
+              # noop
+            ensure
+              @nio.deregister(io)
+              @map.delete(io)
+              @socket_source.delete(io)
+            end
+          end
+        end
+      rescue Errno::EPIPE, Errno::ECONNRESET => e
+        @log.error("Failed reading from client", e, id: client.object_id)
+        @nio.deregister(io)
+        @map.delete(io)
+        @socket_source.delete(io)
+      end
+    end
+
+    def wakeup
+      spawn || @nio.wakeup
+    end
+
+    def run
+      loop do
+        if @stopping
+          @nio.close
+          break
+        end
+
+        @todo.pop(true).call until @todo.empty?
+
+        next unless (monitors = @nio.select)
+
+        monitors.each do |monitor|
+          case monitor.io
+          when TCPServer then handle_server_monitor(monitor)
+          when TCPSocket, Socket then handle_socket_monitor(monitor)
+          else raise "Unexpected monitor IO on reactor: #{monitor.io.inspect}"
+          end
+        end
+      end
+    end
+  end
+end