volucer-worker 0.1.0

data/lib/volucer/message.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+# The request and response objects a Ruby worker's handler sees.
+#
+# These are the SDK's own types rather than Rack's, because the SDK must work with every
+# Ruby web stack and none of them owns the wire. `Volucer::Rack.env` turns a Request into a
+# Rack environment; a framework adapter is that environment, not these objects.
+
+require 'uri'
+
+module Volucer
+  # One request, as it arrived on the channel.
+  #
+  # `headers` is a list of pairs rather than a Hash, because the same name may arrive twice
+  # and the order is the client's. `#header` is the ordered lookup and `#header_map` is the
+  # collapsed form a Rack environment needs.
+  class Request
+    attr_reader :method, :uri, :protocol, :headers, :remote_addr, :script_filename, :body
+    attr_reader :https
+
+    def initialize(
+      method: 'GET', uri: '/', protocol: 'HTTP/1.1', headers: [], remote_addr: '',
+      script_filename: '', https: false, body: +''
+    )
+      @method = method
+      @uri = uri
+      @protocol = protocol
+      @headers = headers
+      @remote_addr = remote_addr
+      @script_filename = script_filename
+      @https = https
+      @body = body
+    end
+
+    def https?
+      @https
+    end
+
+    # The first value of a header, case-insensitively, or nil.
+    def header(name)
+      lowered = name.downcase
+      @headers.each { |key, value| return value if key.downcase == lowered }
+      nil
+    end
+
+    # Every header as a name => value map, repeated names joined with a comma.
+    #
+    # Repeated names are joined rather than overwritten, because that is what a repeated
+    # HTTP field means. `Set-Cookie` is the exception and it is a *response* header, so the
+    # response side keeps the pair list instead.
+    def header_map
+      @headers.each_with_object({}) do |(key, value), collapsed|
+        collapsed[key] = collapsed.key?(key) ? "#{collapsed[key]}, #{value}" : value
+      end
+    end
+
+    # The path, without the query string.
+    def path
+      @uri.split('?', 2).first
+    end
+
+    # Everything after the first `?`, or an empty string.
+    def query_string
+      _, _, query = @uri.partition('?')
+      query
+    end
+
+    # The query string as a Hash, later values winning.
+    def query
+      URI.decode_www_form(query_string).to_h
+    rescue ArgumentError
+      {}
+    end
+
+    def content_type
+      header('content-type')
+    end
+
+    # Whether this request is a gRPC call.
+    #
+    # The media type decides, on both sides of the channel — see `docs/worker-protocol.md`.
+    # A worker that serves both HTTP routes and gRPC services is the ordinary shape, so this
+    # is the question an adapter asks rather than something the loop decides for it.
+    def grpc?
+      media_type = (content_type || '').strip.downcase
+      return false unless media_type.start_with?('application/grpc')
+
+      rest = media_type['application/grpc'.length..] || ''
+      rest.empty? || [';', '+', '-'].include?(rest[0])
+    end
+  end
+
+  # A response the SDK will send.
+  #
+  # Header order is preserved as given, because the server relays it verbatim and two
+  # `Set-Cookie` headers are two cookies.
+  class Response
+    attr_accessor :status, :headers, :body
+
+    def initialize(status = 200, headers = [], body = +'')
+      @status = status
+      @headers = headers
+      @body = body.b
+    end
+
+    def self.text(body, status = 200)
+      new(status, [['Content-Type', 'text/plain; charset=utf-8']], body)
+    end
+
+    def self.html(body, status = 200)
+      new(status, [['Content-Type', 'text/html; charset=utf-8']], body)
+    end
+
+    def self.json(value, status = 200)
+      new(status, [['Content-Type', 'application/json']], JSON.generate(value))
+    end
+
+    def with_header(name, value)
+      @headers << [name, value]
+      self
+    end
+
+    def with_status(status)
+      @status = status
+      self
+    end
+
+    def with_body(body)
+      @body = body.b
+      self
+    end
+  end
+
+  # One unit of queued work, as a consumer sees it.
+  #
+  # A job is a separate exchange from a request rather than a flag on one, because the two
+  # differ in what an application may assume: a request handler is expected to reset what
+  # the last one left behind, and a consumer is expected to keep its connection pool and
+  # warm caches across jobs.
+  class Job
+    attr_reader :pipeline, :id, :task, :headers, :attempts, :body
+
+    def initialize(pipeline: '', id: '', task: '', headers: [], attempts: 0, body: +'')
+      @pipeline = pipeline
+      @id = id
+      @task = task
+      @headers = headers
+      @attempts = attempts
+      @body = body
+    end
+  end
+end

data/lib/volucer/protocol.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+# The volucer worker protocol, as a Ruby client implements it.
+#
+# The specification is `docs/worker-protocol.md` in `sailantis/volucer`; this module is
+# the other end of it, beside the PHP SDK in `sailantis/volucer-php-sdk` and the Python one
+# in `sailantis/volucer-python-sdk`. All three implement the same document rather than each
+# other, and the wire-format tests assert the same frozen bytes.
+#
+# Everything here is stdlib-only, deliberately: a worker SDK is loaded before an
+# application's own gems are known to work, and a protocol codec that needs a gem is one
+# that can fail for a reason unrelated to the protocol.
+
+require 'json'
+
+module Volucer
+  # The only protocol version that exists so far.
+  PROTOCOL_VERSION = 1
+
+  # The size of a frame header on the wire.
+  HEADER_LEN = 16
+
+  # Refuse absurd payloads before allocating, matching MAX_PAYLOAD_LEN in Rust.
+  MAX_PAYLOAD_LEN = 64 * 1024 * 1024
+
+  # The largest body slice sent in one frame, matching MAX_BODY_CHUNK in Rust.
+  MAX_BODY_CHUNK = 64 * 1024
+
+  # The request id reserved for unsolicited frames.
+  UNSOLICITED = 0
+
+  # The message type table. The numbering *is* the protocol: never reorder, never reuse.
+  module MessageType
+    HELLO = 1
+    HELLO_ACK = 2
+    REQUEST = 3
+    RESPONSE = 4
+    BODY = 5
+    EVENT = 6
+    LOG = 7
+    METRIC = 8
+    CONTROL = 9
+    CONTROL_ACK = 10
+    END_FRAME = 11
+    PING = 12
+    PONG = 13
+    ERROR = 14
+    CALL = 15
+    RETURN = 16
+    JOB = 17
+    ACK = 18
+
+    NAMES = {
+      HELLO => 'HELLO', HELLO_ACK => 'HELLO_ACK', REQUEST => 'REQUEST',
+      RESPONSE => 'RESPONSE', BODY => 'BODY', EVENT => 'EVENT', LOG => 'LOG',
+      METRIC => 'METRIC', CONTROL => 'CONTROL', CONTROL_ACK => 'CONTROL_ACK',
+      END_FRAME => 'END', PING => 'PING', PONG => 'PONG', ERROR => 'ERROR',
+      CALL => 'CALL', RETURN => 'RETURN', JOB => 'JOB', ACK => 'ACK'
+    }.freeze
+
+    # The wire name of a type, for diagnostics.
+    def self.name(type)
+      NAMES.fetch(type, "unknown (#{type})")
+    end
+  end
+
+  # Bits in the `flags` field.
+  module Flags
+    COMPRESSED = 0x0001
+    CHUNKED = 0x0002
+    LAST = 0x0004
+    ERROR = 0x0008
+    NO_CHECKSUM = 0x0010
+  end
+
+  # A framing or message failure.
+  #
+  # The names are the protocol's own: a *framing* failure means the reader can no longer
+  # trust its position in the stream and the worker must be discarded, while a *message*
+  # failure (a wrong request id, a remote error) leaves the stream aligned.
+  class ProtocolError < StandardError
+    attr_reader :fatal
+
+    # Whether the failed operation may nevertheless have taken effect.
+    #
+    # The third answer `RETURN` has room for, and the one a retry has to know about: a call
+    # that was **written** and never answered -- a broken connection, an expired deadline --
+    # may have been served, so retrying it may duplicate a side effect. A refusal the server
+    # actually sent is `false`: nothing happened, and retrying is safe.
+    #
+    # Always `false` on a worker's channel, where the protocol guarantees an answer; it is
+    # {Rpc} that can reach the undecided case, because it reads a socket.
+    attr_reader :unknown
+
+    def initialize(message, fatal: true, unknown: false)
+      super(message)
+      @fatal = fatal
+      @unknown = unknown
+    end
+
+    # Whether the failure may have taken effect anyway, named for the question a caller asks.
+    #
+    # `may_have_taken_effect?` rather than a predicate over `#unknown`, because the sentence a caller
+    # actually needs to answer is "is it safe to retry this?", and `unknown?` reads as "the error is
+    # of an unknown kind" to anyone who has not read this file.
+    def may_have_taken_effect?
+      @unknown
+    end
+
+    # The field's own name, kept because `RETURN` spells it `unknown` on the wire.
+    def unknown?
+      @unknown
+    end
+  end
+
+  # CRC-32C (Castagnoli).
+  #
+  # A table rather than `Zlib.crc32`, which is CRC-32B: a different polynomial, so using it
+  # would make every frame fail its checksum against a correct peer. The check value is the
+  # standard one — `crc32c("123456789") == 0xE3069283` — and the test suite pins it.
+  module Crc32c
+    POLYNOMIAL = 0x82F63B78
+
+    TABLE = Array.new(256) do |index|
+      value = index
+      8.times { value = (value & 1).zero? ? value >> 1 : (value >> 1) ^ POLYNOMIAL }
+      value
+    end.freeze
+
+    def self.digest(data)
+      crc = 0xFFFFFFFF
+      data.each_byte do |byte|
+        crc = TABLE[(crc ^ byte) & 0xFF] ^ (crc >> 8)
+      end
+      crc ^ 0xFFFFFFFF
+    end
+  end
+
+  # A decoded frame header: the 16 bytes that precede a payload.
+  #
+  # A direct transliteration of `Header::decode` in
+  # `crates/volucer-worker/src/header.rs`. Layout, little-endian:
+  #
+  #   offset size field
+  #   0      4    payload_len
+  #   4      1    protocol_version
+  #   5      1    message_type
+  #   6      2    flags
+  #   8      4    request_id
+  #   12     4    crc32c(payload)
+  Header = Struct.new(:payload_len, :message_type, :flags, :request_id, :checksum) do
+    # Parse a header, which must be exactly 16 bytes.
+    #
+    # Exactly, not "at least". A stream that ended mid-header is an error rather than a
+    # clean close: treating it as one would hide a truncated frame.
+    def self.decode(raw)
+      length = raw.bytesize
+      unless length == HEADER_LEN
+        raise ProtocolError, "a frame header is exactly #{HEADER_LEN} bytes, got #{length}"
+      end
+
+      payload_len, version, type, flags, request_id, checksum =
+        raw.unpack('VCCvVV')
+
+      unless version == PROTOCOL_VERSION
+        raise ProtocolError,
+              "unsupported protocol version #{version} (this SDK speaks #{PROTOCOL_VERSION})"
+      end
+
+      unless type.between?(MessageType::HELLO, MessageType::ACK)
+        raise ProtocolError, "unknown message type #{type} (this SDK defines 1..18)"
+      end
+
+      # Refused rather than misread: with no codec negotiated, a "compressed" payload would
+      # be handed to JSON as if it were plain text.
+      unless (flags & Flags::COMPRESSED).zero?
+        raise ProtocolError, 'refusing a compressed frame: no codec is negotiated'
+      end
+
+      new(payload_len, type, flags, request_id, checksum)
+    end
+
+    def last?
+      (flags & Flags::LAST) != 0
+    end
+
+    # The writer's *claim*, not a permission. Whether it may be believed is the reader's
+    # decision, and `Worker#read_payload` refuses one the handshake never agreed to.
+    def declares_no_checksum?
+      (flags & Flags::NO_CHECKSUM) != 0
+    end
+
+    def type_name
+      MessageType.name(message_type)
+    end
+  end
+
+  # A frame ready to send: a message type, a request id and a payload.
+  Frame = Struct.new(:message_type, :request_id, :payload, :flags) do
+    def initialize(message_type, request_id, payload = '', flags = 0)
+      super(message_type, request_id, payload.b, flags)
+    end
+
+    # Serialize to header + payload.
+    #
+    # With the checksum off the field is written as zero and NO_CHECKSUM is raised, which
+    # is what makes the zero mean "not a checksum" rather than "the checksum of an empty
+    # payload".
+    def encode(checksum: true)
+      effective_flags = checksum ? flags : (flags | Flags::NO_CHECKSUM)
+      value = checksum ? Crc32c.digest(payload) : 0
+      [payload.bytesize, PROTOCOL_VERSION, message_type, effective_flags,
+       request_id, value].pack('VCCvVV') + payload
+    end
+  end
+
+  # JSON-encode a payload the way the protocol expects it.
+  #
+  # `JSON.generate` carries no spaces, which is what the golden fixtures in all three SDKs
+  # are written against.
+  def self.json_bytes(value)
+    JSON.generate(value, ascii_only: false).b
+  end
+
+  # Decode a JSON payload, turning a bad payload into a protocol error.
+  def self.decode_json(payload, what)
+    return {} if payload.nil? || payload.empty?
+
+    decoded = begin
+      JSON.parse(payload)
+    rescue JSON::ParserError => e
+      raise ProtocolError, "#{what} was not valid JSON: #{e.message}"
+    end
+
+    unless decoded.is_a?(Hash)
+      raise ProtocolError, "#{what} must be a JSON object"
+    end
+
+    decoded
+  end
+
+  # The extensions, message types and flags are loaded by `volucer/worker`; requiring this
+  # file alone gives a caller the codec without a process.
+end

data/lib/volucer/rack.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+# A Rack adapter: turn a `Volucer::Request` into a Rack environment.
+#
+# This is the half of a Ruby worker that is genuinely new work rather than a transliteration
+# of the PHP SDK. PHP has a SAPI that builds `$_SERVER` from a CGI environment, so the PHP
+# SDK already receives one; Rack is *also* CGI-derived but has its own opinions, and two of
+# them are where a naive adapter goes wrong:
+#
+# * `rack.input` has to respond to `gets`, `each`, `read` and `rewind`. A Rack app, and
+#   `Rack::Lint`, will use all four.
+# * `rack.errors` has to be a stream. Pointing it at the protocol stream would corrupt the
+#   framing the first time an application logged a warning.
+#
+# `SCRIPT_NAME` is empty and `PATH_INFO` is the whole path, which is the shape a middleware
+# stack mounted at the root expects. A framework that needs to route on a prefix does it from
+# `PATH_INFO`, exactly as it does under any other Rack server.
+
+require 'stringio'
+
+module Volucer
+  module Rack
+    # The default server name, when a `Host` header carries none.
+    DEFAULT_SERVER_NAME = 'localhost'
+
+    # Split a `Host` header into a name and a port, with the scheme's default filled in.
+    #
+    # A bracketed IPv6 literal keeps its brackets and its colons; everything else splits on
+    # the last colon, which is the only unambiguous rule a host header has.
+    def self.split_host(host, https)
+      default_port = https ? '443' : '80'
+      return [DEFAULT_SERVER_NAME, default_port] if host.nil? || host.strip.empty?
+
+      host = host.strip
+      if host.start_with?('[')
+        closing = host.index(']')
+        unless closing.nil?
+          name = host[0..closing]
+          rest = host[(closing + 1)..] || ''
+          return [name, rest.start_with?(':') && rest.length > 1 ? rest[1..] : default_port]
+        end
+        return [host, default_port]
+      end
+
+      if host.include?(':')
+        name, _, port = host.rpartition(':')
+        return [name, port] unless name.empty? || port.empty?
+      end
+
+      [host, default_port]
+    end
+
+    # Build the Rack environment for one request.
+    #
+    # `server_port` overrides the port derived from the `Host` header, for a deployment whose
+    # listener is not the port a client sees. The header is the better source when it is
+    # there, so this is an override rather than the default.
+    def self.env(request, server_port = nil)
+      name, port = split_host(request.header('host'), request.https?)
+      port = server_port unless server_port.nil? || server_port.to_s.empty?
+
+      path = decode_path(request.path)
+
+      env = {
+        'REQUEST_METHOD' => request.method,
+        'SCRIPT_NAME' => '',
+        'PATH_INFO' => path,
+        'QUERY_STRING' => request.query_string,
+        'SERVER_NAME' => name,
+        'SERVER_PORT' => port.to_s,
+        'SERVER_PROTOCOL' => request.protocol,
+        'REMOTE_ADDR' => request.remote_addr,
+        'rack.version' => [1, 3],
+        'rack.url_scheme' => request.https? ? 'https' : 'http',
+        'rack.input' => StringIO.new(request.body),
+        'rack.errors' => ErrorsStream.new,
+        'rack.multithread' => false,
+        'rack.multiprocess' => true,
+        'rack.run_once' => false,
+        'rack.hijack?' => false
+      }
+
+      # Routing metadata, not a file to execute — the protocol's own documentation says so,
+      # and a Rack app that opened this path would be a path-traversal bug waiting to happen.
+      # Prefixing it keeps nothing from mistaking it for a CGI variable.
+      env['volucer.script_filename'] = request.script_filename unless request.script_filename.empty?
+
+      request.headers.each do |key, value|
+        upper = key.upcase.tr('-', '_')
+        case upper
+        when 'CONTENT_TYPE', 'CONTENT_LENGTH'
+          env[upper] = value
+        else
+          env["HTTP_#{upper}"] = value
+        end
+      end
+
+      # A Rack server is expected to supply this for a body; the request may have arrived
+      # without one, and a middleware that read `rack.input` to the end would otherwise not
+      # know how much to expect.
+      if !env.key?('CONTENT_LENGTH') && !request.body.empty?
+        env['CONTENT_LENGTH'] = request.body.bytesize.to_s
+      end
+
+      env
+    end
+
+    # Percent-decode a URL path, leaving it as the bytes a Rack app expects.
+    def self.decode_path(raw)
+      decoded = raw.gsub(/%([0-9A-Fa-f]{2})/) { [Regexp.last_match(1)].pack('H2') }
+      decoded.force_encoding(Encoding::UTF_8)
+      # A path that is not valid UTF-8 is still a path a request can carry, so it stays as
+      # bytes rather than raising. Rack apps that care will inspect `PATH_INFO` themselves.
+      decoded.valid_encoding? ? decoded : decoded.dup.force_encoding(Encoding::BINARY)
+    end
+
+    # `rack.errors`, writing somewhere that is not the protocol channel.
+    #
+    # `$stderr` is free (the supervisor drains it), so this is a thin pass-through. An
+    # application's warning is worth seeing, and losing it to a full pipe would be the worst
+    # of both.
+    class ErrorsStream
+      def write(text)
+        $stderr.write(text)
+        text.to_s.length
+      end
+
+      def <<(text)
+        write(text)
+        self
+      end
+
+      def puts(*lines)
+        lines = [''] if lines.empty?
+        lines.each { |line| write("#{line}\n") }
+        nil
+      end
+
+      def flush
+        $stderr.flush
+        self
+      end
+
+      def sync
+        true
+      end
+
+      def sync=(value); end
+
+      def close; end
+    end
+
+    # Wrap a Rack application so it can be handed to `Worker#run`.
+    #
+    # The usual shape of a Ruby worker, which is why this exists as a function rather than as
+    # documentation:
+    #
+    #   require 'volucer'
+    #   require 'my_app'
+    #
+    #   Volucer::Worker.connect.run(&Volucer::Rack.handler(MyApp.new))
+    #
+    # The result is a callable taking `(request, worker)` and returning a `Response`, which
+    # is what `run` wants.
+    def self.handler(app)
+      lambda do |request, _worker|
+        status, headers, body = app.call(env(request))
+
+        collected = +''
+        begin
+          body.each { |chunk| collected << chunk.to_s }
+        ensure
+          body.close if body.respond_to?(:close)
+        end
+
+        Response.new(status.to_i, normalise_headers(headers), collected)
+      end
+    end
+
+    def self.normalise_headers(headers)
+      return [] if headers.nil?
+
+      collected = []
+      headers.each do |name, value|
+        if value.is_a?(Array)
+          # Rack 3 allows an Array value for a header that may repeat; two cookies are two
+          # headers, and joining them would silently break a session.
+          value.each { |one| collected << [name.to_s, one.to_s] }
+        else
+          collected << [name.to_s, value.to_s]
+        end
+      end
+      collected
+    end
+  end
+end