tether 0.1.0

data/lib/tether/nuid.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Tether
+  # Generator for NATS unique identifiers: a 12 character random prefix followed
+  # by a 10 character base62 sequence, incremented by a random step so that
+  # identifiers are neither guessable nor collision prone across processes.
+  #
+  # Instances are not thread safe. Use {NUID.next} for the process wide,
+  # mutex guarded generator.
+  class NUID
+    DIGITS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+    BASE = 62
+    PREFIX_LENGTH = 12
+    SEQUENCE_LENGTH = 10
+    TOTAL_LENGTH = PREFIX_LENGTH + SEQUENCE_LENGTH
+    MAX_SEQUENCE = BASE**SEQUENCE_LENGTH
+    MIN_INCREMENT = 33
+    MAX_INCREMENT = 333
+    PADDING = ("0" * SEQUENCE_LENGTH).freeze
+    private_constant :PADDING
+
+    @mutex = Mutex.new
+    @generator = nil
+
+    class << self
+      # Generates an identifier from the process wide generator.
+      #
+      # @return [String] a 22 character identifier
+      def next
+        @mutex.synchronize { (@generator ||= new).next }
+      end
+    end
+
+    # Seeds a generator with its own random prefix, so two instances in the
+    # same process never collide.
+    def initialize
+      reseed!
+    end
+
+    # Generates the next identifier.
+    #
+    # @return [String] a 22 character identifier
+    def next
+      @sequence += @increment
+      reseed! if @sequence >= MAX_SEQUENCE
+
+      buffer = @prefix + PADDING
+      value = @sequence
+      (TOTAL_LENGTH - 1).downto(PREFIX_LENGTH) do |index|
+        buffer.setbyte(index, DIGITS.getbyte(value % BASE))
+        value /= BASE
+      end
+      buffer
+    end
+
+    private
+
+    def reseed!
+      @prefix = SecureRandom.random_bytes(PREFIX_LENGTH).each_byte.with_object(+"") do |byte, out|
+        out << DIGITS.getbyte(byte % BASE)
+      end
+      @sequence = SecureRandom.random_number(MAX_SEQUENCE)
+      @increment = MIN_INCREMENT + SecureRandom.random_number(MAX_INCREMENT - MIN_INCREMENT)
+    end
+  end
+end

data/lib/tether/protocol/encoder.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Tether
+  module Protocol
+    # Builds complete protocol frames as single binary strings.
+    #
+    # Every method returns one whole frame. The writer takes its lock around the
+    # write of that string, never around the pieces of it: frames from different
+    # threads must not interleave on the wire.
+    module Encoder
+      # The server seeds every connection with `verbose:true, pedantic:true,
+      # echo:true` and merges CONNECT into that struct, so an absent key keeps
+      # the server's default rather than clearing it. Omitting `verbose` alone
+      # makes the server acknowledge every PUB and SUB with `+OK`; omitting
+      # `headers` makes it rewrite HMSG as MSG with the header bytes cut off,
+      # which silently strips all JetStream metadata.
+      DEFAULT_CONNECT = {
+        verbose: false,
+        pedantic: false,
+        tls_required: false,
+        lang: "ruby",
+        version: Tether::VERSION,
+        protocol: 1,
+        echo: true,
+        headers: true,
+        no_responders: true
+      }.freeze
+
+      class << self
+        # Encodes a `PUB` frame.
+        #
+        # @param subject [String]
+        # @param payload [String, nil]
+        # @param reply_to [String, nil]
+        # @return [String] binary encoded frame
+        def pub(subject, payload = nil, reply_to = nil)
+          body = binary(payload)
+          frame = buffer(subject.bytesize + body.bytesize + 64)
+          write_head(frame, "PUB ", subject, reply_to)
+          frame << " " << body.bytesize.to_s << CRLF
+          frame << body << CRLF
+        end
+
+        # Encodes an `HPUB` frame.
+        #
+        # @param subject [String]
+        # @param payload [String, nil]
+        # @param headers [Headers, String] headers, or a pre-encoded header block
+        # @param reply_to [String, nil]
+        # @return [String] binary encoded frame
+        def hpub(subject, payload, headers, reply_to = nil)
+          block = headers.is_a?(Headers) ? headers.to_wire : binary(headers)
+          body = binary(payload)
+          frame = buffer(subject.bytesize + block.bytesize + body.bytesize + 64)
+          write_head(frame, "HPUB ", subject, reply_to)
+          frame << " " << block.bytesize.to_s
+          frame << " " << (block.bytesize + body.bytesize).to_s << CRLF
+          frame << block << body << CRLF
+        end
+
+        # Encodes a `SUB` frame.
+        #
+        # @param subject [String]
+        # @param sid [Integer, String]
+        # @param queue [String, nil] queue group
+        # @return [String] binary encoded frame
+        def sub(subject, sid, queue: nil)
+          frame = buffer(subject.bytesize + 32)
+          frame << "SUB " << binary(subject)
+          frame << " " << binary(queue) if queue
+          frame << " " << sid.to_s << CRLF
+        end
+
+        # Encodes an `UNSUB` frame.
+        #
+        # @param sid [Integer, String]
+        # @param max_msgs [Integer, nil] unsubscribe after this many messages
+        # @return [String] binary encoded frame
+        def unsub(sid, max_msgs = nil)
+          frame = buffer(32)
+          frame << "UNSUB " << sid.to_s
+          frame << " " << max_msgs.to_s if max_msgs
+          frame << CRLF
+        end
+
+        # Encodes a `CONNECT` frame over {DEFAULT_CONNECT}.
+        #
+        # @param options [Hash] overrides for the defaults
+        # @return [String] binary encoded frame
+        # @raise [ArgumentError] if no_responders is requested without headers
+        def connect(options = {})
+          payload = DEFAULT_CONNECT.merge(options)
+          if payload[:no_responders] && !payload[:headers]
+            raise ArgumentError, "no_responders requires headers; the server closes the connection otherwise"
+          end
+
+          json = JSON.generate(payload)
+          frame = buffer(json.bytesize + 16)
+          frame << "CONNECT " << json << CRLF
+        end
+
+        private
+
+        def write_head(frame, verb, subject, reply_to)
+          frame << verb << binary(subject)
+          frame << " " << binary(reply_to) if reply_to
+          frame
+        end
+
+        def buffer(capacity)
+          String.new(capacity: capacity, encoding: BINARY)
+        end
+
+        def binary(value)
+          return EMPTY_PAYLOAD if value.nil?
+          return value if value.encoding == BINARY || value.ascii_only?
+
+          value.b
+        end
+      end
+    end
+  end
+end

data/lib/tether/protocol/parser.rb

@@ -0,0 +1,336 @@
+# frozen_string_literal: true
+
+module Tether
+  module Protocol
+    # Incremental parser for the server side of the NATS protocol.
+    #
+    # Bytes are appended to one buffer and consumed by advancing an index.
+    # Control lines are read in place: subjects, reply subjects, header blocks
+    # and payloads are the only substrings allocated, and integers are scanned
+    # straight out of the buffer. The buffer is reset in place once drained and
+    # compacted only when the consumed prefix passes {COMPACT_THRESHOLD}.
+    #
+    # The parser never raises into its caller. The reader thread is the only
+    # demultiplexer for the whole connection, so its death would stall every
+    # subscription at once; malformed input is reported through
+    # `sink.on_protocol_error` and the parser marks itself {#broken?}.
+    #
+    # The sink must respond to:
+    #
+    # - `on_msg(subject, sid, reply_to, header_bytes, payload)` where `sid` is
+    #   the opaque token this client chose at SUB time, echoed back verbatim,
+    #   and `header_bytes` is nil for `MSG` or the raw header block for `HMSG`
+    # - `on_ping`, `on_pong`, `on_ok`
+    # - `on_info(json)` with the still unparsed JSON text
+    # - `on_error(description)` for an `-ERR` line
+    # - `on_protocol_error(message)`
+    class Parser
+      COMPACT_THRESHOLD = 64 * 1024
+
+      # A leftover at or below this is copied out rather than left in place, so
+      # the buffer does not grow across reads and make every later append
+      # target a large string.
+      SHORT_REMAINDER = 1024
+      SNIPPET_LIMIT = 64
+
+      AWAITING_CONTROL_LINE = :control
+      AWAITING_PAYLOAD = :payload
+
+      MSG_LINE = /\GMSG[ \t]+(\S+)[ \t]+(\S+)[ \t]+(?:(\S+)[ \t]+)?(\d{1,9})[ \t]*\r\n/i
+      HMSG_LINE = /\GHMSG[ \t]+(\S+)[ \t]+(\S+)[ \t]+(?:(\S+)[ \t]+)?(\d{1,9})[ \t]+(\d{1,9})[ \t]*\r\n/i
+
+      CR = 13
+      LF = 10
+      UPPER_M = 77
+      LOWER_M = 109
+      UPPER_H = 72
+      LOWER_H = 104
+      UPPER_P = 80
+      LOWER_P = 112
+      UPPER_I = 73
+      LOWER_I = 105
+      PLUS = 43
+      MINUS = 45
+      UPPER_A = 65
+      UPPER_Z = 90
+      CASE_SHIFT = 32
+      private_constant :MSG_LINE, :HMSG_LINE, :CR, :LF, :UPPER_A, :UPPER_Z, :CASE_SHIFT,
+                       :UPPER_M, :LOWER_M, :UPPER_H, :LOWER_H, :UPPER_P, :LOWER_P,
+                       :UPPER_I, :LOWER_I, :PLUS, :MINUS
+
+      # @return [Symbol] :control or :payload
+      def state = @awaiting_payload ? AWAITING_PAYLOAD : AWAITING_CONTROL_LINE
+
+      # @return [Integer, nil] largest frame accepted, from the server's INFO
+      attr_accessor :max_payload
+
+      # @param sink [Object] receives the parsed frames
+      # @param max_control_line [Integer] sanity bound on a single control line
+      # @param max_payload [Integer, nil] reject a frame larger than the server
+      #   advertised in INFO, rather than buffering toward it
+      def initialize(sink, max_control_line: DEFAULT_MAX_CONTROL_LINE, max_payload: nil)
+        @sink = sink
+        @max_control_line = max_control_line
+        @max_payload = max_payload
+        @buffer = String.new(capacity: COMPACT_THRESHOLD, encoding: BINARY)
+        reset!
+      end
+
+      # Appends bytes and consumes every complete frame in the buffer.
+      #
+      # @param data [String] bytes as read from the socket
+      # @return [self]
+      def <<(data)
+        return self if @broken || data.nil? || data.empty?
+
+        @buffer << (data.encoding == BINARY ? data : data.b)
+        run
+        self
+      end
+      alias parse <<
+
+      # Discards buffered bytes and returns to the initial state.
+      #
+      # @return [void]
+      def reset!
+        @buffer.clear
+        @position = 0
+        @awaiting_payload = false
+        @broken = false
+        clear_frame
+      end
+
+      # A broken parser consumes nothing further; the client is expected to
+      # tear the connection down.
+      #
+      # @return [Boolean] whether a protocol violation stopped this parser
+      def broken?
+        @broken
+      end
+
+      # @return [Integer] bytes buffered but not yet consumed, which is the tail
+      #   of a frame still arriving
+      def pending_bytes
+        @buffer.bytesize - @position
+      end
+
+      private
+
+      def run
+        buffer = @buffer
+        sink = @sink
+        pos = @position
+        size = buffer.bytesize
+
+        loop do
+          if @awaiting_payload
+            needed = @needed
+            break if size - pos < needed + CRLF_SIZE
+
+            unless buffer.getbyte(pos + needed) == CR && buffer.getbyte(pos + needed + 1) == LF
+              @position = pos
+              return fail_protocol!("payload not terminated by CRLF on #{@subject}")
+            end
+
+            if (header_size = @header_size)
+              sink.on_msg(@subject, @sid, @reply_to, buffer.byteslice(pos, header_size),
+                          buffer.byteslice(pos + header_size, needed - header_size))
+            else
+              sink.on_msg(@subject, @sid, @reply_to, nil, buffer.byteslice(pos, needed))
+            end
+            pos += needed + CRLF_SIZE
+            @awaiting_payload = false
+            next
+          end
+
+          line_end = buffer.index(CRLF, pos)
+          unless line_end
+            if size - pos > @max_control_line
+              @position = pos
+              return fail_protocol!("control line exceeds #{@max_control_line} bytes")
+            end
+            break
+          end
+
+          byte = buffer.getbyte(pos)
+          if [UPPER_M, LOWER_M].include?(byte)
+            unless buffer.index(MSG_LINE, pos)
+              @position = pos
+              return malformed(pos, line_end)
+            end
+
+            subject = ::Regexp.last_match(1)
+            sid = ::Regexp.last_match(2)
+            reply_to = ::Regexp.last_match(3)
+            header_size = nil
+            needed = ::Regexp.last_match(4).to_i
+          elsif [UPPER_H, LOWER_H].include?(byte)
+            unless buffer.index(HMSG_LINE, pos)
+              @position = pos
+              return malformed(pos, line_end)
+            end
+
+            subject = ::Regexp.last_match(1)
+            sid = ::Regexp.last_match(2)
+            reply_to = ::Regexp.last_match(3)
+            header_size = ::Regexp.last_match(4).to_i
+            needed = ::Regexp.last_match(5).to_i
+          else
+            @position = pos
+            return unless dispatch(pos, line_end)
+
+            pos = line_end + CRLF_SIZE
+            next
+          end
+
+          if (header_size && header_size > needed) || (@max_payload && needed > @max_payload)
+            @position = pos
+            return reject_sizes(subject, header_size, needed)
+          end
+
+          pos = line_end + CRLF_SIZE
+
+          # The payload is usually already buffered, in which case the frame is
+          # emitted straight from locals and never touches the object at all.
+          if size - pos >= needed + CRLF_SIZE
+            unless buffer.getbyte(pos + needed) == CR && buffer.getbyte(pos + needed + 1) == LF
+              @position = pos
+              return fail_protocol!("payload not terminated by CRLF on #{subject}")
+            end
+
+            if header_size
+              sink.on_msg(subject, sid, reply_to, buffer.byteslice(pos, header_size),
+                          buffer.byteslice(pos + header_size, needed - header_size))
+            else
+              sink.on_msg(subject, sid, reply_to, nil, buffer.byteslice(pos, needed))
+            end
+            pos += needed + CRLF_SIZE
+            next
+          end
+
+          @subject = subject
+          @sid = sid
+          @reply_to = reply_to
+          @header_size = header_size
+          @needed = needed
+          @awaiting_payload = true
+        end
+
+        @position = pos
+        compact
+      end
+
+      def reject_sizes(subject, header_size, needed)
+        return fail_protocol!("header size exceeds total size on #{subject}") if header_size && header_size > needed
+
+        fail_protocol!("frame of #{needed} bytes exceeds max_payload #{@max_payload} on #{subject}")
+      end
+
+      def dispatch(start, line_end)
+        case @buffer.getbyte(start)
+        when UPPER_P, LOWER_P then dispatch_ping_pong(start, line_end)
+        when PLUS then dispatch_ok(start, line_end)
+        when MINUS then dispatch_err(start, line_end)
+        when UPPER_I, LOWER_I then dispatch_info(start, line_end)
+        else unknown_operation(start, line_end)
+        end
+      end
+
+      def dispatch_ping_pong(start, line_end)
+        if verb?(start, line_end, "PING")
+          @sink.on_ping
+          true
+        elsif verb?(start, line_end, "PONG")
+          @sink.on_pong
+          true
+        else
+          unknown_operation(start, line_end)
+        end
+      end
+
+      def dispatch_ok(start, line_end)
+        return unknown_operation(start, line_end) unless verb?(start, line_end, "+OK")
+
+        @sink.on_ok
+        true
+      end
+
+      def dispatch_err(start, line_end)
+        return unknown_operation(start, line_end) unless verb?(start, line_end, "-ERR")
+
+        description = trimmed(start + 4, line_end)
+        @sink.on_error(description.delete_prefix("'").delete_suffix("'"))
+        true
+      end
+
+      def dispatch_info(start, line_end)
+        return unknown_operation(start, line_end) unless verb?(start, line_end, "INFO")
+
+        json = trimmed(start + 4, line_end)
+        return fail_protocol!("malformed INFO: #{snippet(start, line_end)}") if json.empty?
+
+        @sink.on_info(json)
+        true
+      end
+
+      def verb?(start, limit, verb)
+        return false if limit - start < verb.bytesize
+
+        verb.bytesize.times do |offset|
+          expected = verb.getbyte(offset)
+          actual = @buffer.getbyte(start + offset)
+          next if actual == expected
+          return false unless expected.between?(UPPER_A, UPPER_Z) && actual == expected + CASE_SHIFT
+        end
+        true
+      end
+
+      def trimmed(from, to)
+        @buffer.byteslice(from, to - from).strip
+      end
+
+      def snippet(start, line_end)
+        @buffer.byteslice(start, [line_end - start, SNIPPET_LIMIT].min).inspect
+      end
+
+      def clear_frame
+        @subject = nil
+        @sid = nil
+        @reply_to = nil
+        @needed = nil
+        @header_size = nil
+      end
+
+      # Draining the buffer in place is the common case. Otherwise the leftover
+      # is copied out whenever it is short, which keeps the append target small
+      # instead of letting it grow toward COMPACT_THRESHOLD across many reads;
+      # copying a few hundred bytes costs far less than appending into a buffer
+      # that has grown to tens of kilobytes. A long remainder, meaning a large
+      # frame still arriving, is left alone until the threshold.
+      def compact
+        remaining = @buffer.bytesize - @position
+        if remaining.zero?
+          @buffer.clear
+          @position = 0
+        elsif @position >= COMPACT_THRESHOLD || (@position.positive? && remaining <= SHORT_REMAINDER)
+          @buffer = @buffer.byteslice(@position..) || String.new(encoding: BINARY)
+          @position = 0
+        end
+      end
+
+      def malformed(start, line_end)
+        fail_protocol!("malformed control line: #{snippet(start, line_end)}")
+      end
+
+      def unknown_operation(start, line_end)
+        fail_protocol!("unknown protocol operation: #{snippet(start, line_end)}")
+      end
+
+      def fail_protocol!(message)
+        @broken = true
+        @sink.on_protocol_error(message)
+        false
+      end
+    end
+  end
+end

data/lib/tether/protocol.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Tether
+  # The NATS client protocol: a line based text protocol over TCP where every
+  # control line ends in CRLF and payloads are length prefixed.
+  module Protocol
+    CRLF = "\r\n"
+    CRLF_SIZE = 2
+    EMPTY_PAYLOAD = ""
+
+    PING = "PING#{CRLF}".freeze
+    PONG = "PONG#{CRLF}".freeze
+
+    DEFAULT_MAX_CONTROL_LINE = 1024 * 1024
+
+    BINARY = Encoding::BINARY
+  end
+end

data/lib/tether/subscription.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Tether
+  # One subscription, with its own bounded queue and worker thread.
+  #
+  # The reader thread only ever enqueues here, never runs the callback: a slow
+  # handler must not delay delivery to other subscriptions, and must not delay
+  # the PONG that keeps the connection from being dropped as stale.
+  class Subscription
+    DEFAULT_MAX_PENDING = 65_536
+
+    # @return [String] the token sent to the server and echoed back on every message
+    attr_reader :sid
+
+    # @return [String]
+    attr_reader :subject
+
+    # @return [String, nil]
+    attr_reader :queue_group
+
+    # @return [Integer] messages delivered to this subscription
+    attr_reader :delivered
+
+    # @return [Integer] messages dropped because the queue was full
+    attr_reader :dropped
+
+    # @return [Client, nil] used by {Msg#respond}
+    attr_reader :responder
+
+    # Starts a worker thread when a callback is given. Without one, messages
+    # queue up until {#next_message} takes them on the caller's thread.
+    #
+    # @param sid [String]
+    # @param subject [String]
+    # @param queue_group [String, nil]
+    # @param max_pending [Integer] queue depth before this is a slow consumer
+    # @param on_slow_consumer [Proc, nil] called with self when a message is dropped
+    # @param responder [Client, nil] lets {Msg#respond} publish a reply
+    # @param callback [Proc, nil] when given, messages are delivered to it on a worker thread
+    def initialize(sid:, subject:, queue_group: nil, max_pending: DEFAULT_MAX_PENDING,
+                   on_slow_consumer: nil, responder: nil, &callback)
+      @responder = responder
+      @sid = sid
+      @subject = subject
+      @queue_group = queue_group
+      @queue = Thread::SizedQueue.new(max_pending)
+      @on_slow_consumer = on_slow_consumer
+      @callback = callback
+      @delivered = 0
+      @dropped = 0
+      @lock = Mutex.new
+      @worker = start_worker if callback
+    end
+
+    # Enqueues a message. Called on the reader thread, so it never blocks.
+    #
+    # @param msg [Msg]
+    # @return [Boolean] false when the message was dropped
+    def deliver(msg)
+      @queue.push(msg, true)
+      @lock.synchronize { @delivered += 1 }
+      true
+    rescue ThreadError
+      @lock.synchronize { @dropped += 1 }
+      @on_slow_consumer&.call(self)
+      false
+    rescue ClosedQueueError
+      false
+    end
+
+    # Takes the next message, for a subscription with no callback.
+    #
+    # @param deadline [Deadline]
+    # @return [Msg]
+    # @raise [TimeoutError] if the deadline passes first
+    def next_message(deadline = Deadline.infinite)
+      msg = @queue.pop(timeout: deadline.remaining)
+      raise TimeoutError, "no message on #{@subject} within the deadline" if msg.nil? && !@queue.closed?
+      raise ConnectionClosedError, "subscription on #{@subject} was closed" if msg.nil?
+
+      msg
+    end
+
+    # Depth of the queue between the reader thread and this subscription's
+    # worker. Growth here means the handler is not keeping up.
+    #
+    # @return [Integer] messages waiting to be handled
+    def pending
+      @queue.size
+    end
+
+    # @return [Boolean] whether delivery has stopped; a closed subscription never resumes
+    def closed?
+      @queue.closed?
+    end
+
+    # Stops delivery and lets the worker finish what it already took.
+    #
+    # @return [void]
+    def close
+      @queue.close
+      @worker&.join(1)
+      nil
+    end
+
+    private
+
+    def start_worker
+      Thread.new do
+        Thread.current.name = "tether:sub:#{@subject}"
+        while (msg = @queue.pop)
+          invoke(msg)
+        end
+      end
+    end
+
+    def invoke(msg)
+      @callback.call(msg)
+    rescue StandardError => e
+      @on_slow_consumer.nil? ? warn("tether: subscription callback raised: #{e.class}: #{e.message}") : nil
+    end
+  end
+end