pgoutput-client 0.0.0 → 0.1.0

data/lib/pgoutput/client/connection.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    # Thin wrapper around `PG::Connection` for logical replication operations.
+    #
+    # `Connection` hides the small amount of PostgreSQL driver plumbing needed by
+    # the rest of the transport layer. It opens the connection in replication
+    # mode, renders replication commands through {Commands}, and translates
+    # `PG::Error` exceptions into {ConnectionError}.
+    #
+    # @api private
+    class Connection
+      # Configuration associated with this connection.
+      #
+      # @return [Configuration]
+      attr_reader :configuration
+
+      # Open a PostgreSQL connection in database replication mode.
+      #
+      # @param configuration [Configuration] replication configuration
+      # @return [Connection] wrapper around an open `PG::Connection`
+      # @raise [ConnectionError] if the `pg` gem cannot connect
+      def self.open(configuration)
+        require "pg"
+        connection = PG.connect(configuration.database_url, replication: "database")
+        new(configuration:, pg_connection: connection)
+      rescue PG::Error => e
+        raise ConnectionError, e.message
+      end
+
+      # Build a connection wrapper.
+      #
+      # This constructor is public primarily for tests and alternative connection
+      # factories. Normal callers should use {.open}.
+      #
+      # @param configuration [Configuration] replication configuration
+      # @param pg_connection [PG::Connection] connected PostgreSQL driver object
+      # @return [void]
+      def initialize(configuration:, pg_connection:)
+        @configuration = configuration
+        @pg_connection = pg_connection
+      end
+
+      # Execute PostgreSQL's `IDENTIFY_SYSTEM` replication command.
+      #
+      # @return [PG::Result] server identity result
+      # @raise [ConnectionError] if PostgreSQL rejects the command
+      def identify_system
+        exec("IDENTIFY_SYSTEM")
+      end
+
+      # Create the configured logical replication slot.
+      #
+      # @return [PG::Result] command result
+      # @raise [ConnectionError] if PostgreSQL rejects the command
+      def create_replication_slot
+        exec(Commands.create_replication_slot(configuration))
+      end
+
+      # Drop the configured logical replication slot.
+      #
+      # @return [PG::Result] command result
+      # @raise [ConnectionError] if PostgreSQL rejects the command
+      def drop_replication_slot
+        exec(Commands.drop_replication_slot(configuration))
+      end
+
+      # Start streaming logical replication from the configured slot and LSN.
+      #
+      # @return [PG::Result] command result
+      # @raise [ConnectionError] if PostgreSQL rejects the command
+      def start_replication
+        exec(Commands.start_replication(configuration))
+      end
+
+      # Receive one CopyData payload from the server.
+      #
+      # The call is non-blocking because the underlying `pg` call receives
+      # `false` for its blocking argument. `nil` means no complete CopyData
+      # payload is currently available.
+      #
+      # @return [String, nil] raw CopyData payload or `nil`
+      # @raise [ConnectionError] if receiving fails
+      def get_copy_data # rubocop:disable Naming/AccessorMethodName
+        @pg_connection.get_copy_data(false)
+      rescue PG::Error => e
+        raise ConnectionError, e.message
+      end
+
+      # Send one CopyData payload to the server.
+      #
+      # Used for standby status feedback messages.
+      #
+      # @param payload [String] raw CopyData payload
+      # @return [void]
+      # @raise [ConnectionError] if sending fails
+      def put_copy_data(payload)
+        @pg_connection.put_copy_data(payload)
+      rescue PG::Error => e
+        raise ConnectionError, e.message
+      end
+
+      # Close the PostgreSQL connection if it is still open.
+      #
+      # @return [void]
+      def close
+        @pg_connection.close unless @pg_connection.finished?
+      end
+
+      private
+
+      def exec(sql)
+        @pg_connection.exec(sql)
+      rescue PG::Error => e
+        raise ConnectionError, e.message
+      end
+    end
+  end
+end

data/lib/pgoutput/client/errors.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    # Base error class for all pgoutput-client failures.
+    #
+    # Rescue this class when callers want to handle any error raised by the
+    # transport layer without also rescuing unrelated Ruby or PostgreSQL driver
+    # exceptions.
+    #
+    # @api public
+    class Error < StandardError; end
+
+    # Raised when stream configuration is invalid.
+    #
+    # Examples include an empty publication list, invalid replication slot name,
+    # invalid publication name, non-positive protocol version, or non-positive
+    # feedback interval.
+    #
+    # @api public
+    class ConfigurationError < Error; end
+
+    # Raised when a replication protocol envelope cannot be parsed.
+    #
+    # This error represents malformed or unexpected CopyData payloads at the
+    # transport-envelope level. It does not describe pgoutput plugin payload
+    # parsing errors; those belong to the parser layer.
+    #
+    # @api public
+    class ProtocolError < Error; end
+
+    # Raised when a PostgreSQL replication connection operation fails.
+    #
+    # `Connection` converts `PG::Error` instances into this error so public
+    # callers do not need to depend on the PostgreSQL driver's exception classes
+    # for transport-level handling.
+    #
+    # @api public
+    class ConnectionError < Error; end
+  end
+end

data/lib/pgoutput/client/feedback.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    FeedbackData = Data.define(:received_lsn, :flushed_lsn, :applied_lsn, :client_clock, :reply_requested)
+
+    # Standby status feedback message builder.
+    #
+    # Logical replication clients periodically send standby status updates to
+    # tell PostgreSQL which WAL location has been received, flushed, and applied.
+    # `Feedback` models that update and can serialize itself into the CopyData
+    # payload expected by the replication protocol.
+    #
+    # @attr_reader received_lsn [Integer] latest WAL location received by the client
+    # @attr_reader flushed_lsn [Integer] latest WAL location flushed by the client
+    # @attr_reader applied_lsn [Integer] latest WAL location applied by the client
+    # @attr_reader client_clock [Integer] PostgreSQL timestamp in microseconds since 2000-01-01 UTC
+    # @attr_reader reply_requested [Boolean] whether this feedback is responding to an immediate-reply request
+    class Feedback < FeedbackData
+      # Build feedback using the current wall-clock time.
+      #
+      # By default, flushed and applied LSNs follow the received LSN. Callers can
+      # pass lower values if they need to distinguish receipt from durable flush
+      # or application progress.
+      #
+      # @param received_lsn [Integer] latest WAL location received by the client
+      # @param flushed_lsn [Integer] latest WAL location flushed by the client
+      # @param applied_lsn [Integer] latest WAL location applied by the client
+      # @param reply_requested [Boolean] whether this is an immediate reply
+      # @return [Feedback] immutable feedback value
+      def self.now(received_lsn:, flushed_lsn: received_lsn, applied_lsn: flushed_lsn, reply_requested: false)
+        new(received_lsn, flushed_lsn, applied_lsn, current_pg_time, reply_requested)
+      end
+
+      # Build a protocol CopyData payload for standby status update.
+      #
+      # The payload begins with the standby status update tag `r`, followed by
+      # three unsigned 64-bit LSN fields, the PostgreSQL timestamp, and a
+      # one-byte reply-requested flag.
+      #
+      # @return [String] frozen binary CopyData payload
+      def to_copy_data
+        (
+          "r".b +
+          [received_lsn, flushed_lsn, applied_lsn, client_clock].pack("Q>Q>Q>Q>") +
+          [reply_requested ? 1 : 0].pack("C")
+        ).freeze
+      end
+
+      # Current PostgreSQL protocol timestamp.
+      #
+      # PostgreSQL timestamps in replication messages are expressed as
+      # microseconds since 2000-01-01 00:00:00 UTC, not Unix epoch
+      # microseconds.
+      #
+      # @return [Integer] microseconds since 2000-01-01 UTC
+      def self.current_pg_time
+        ((Time.now.utc - Time.utc(2000, 1, 1)) * 1_000_000).to_i
+      end
+    end
+  end
+end

data/lib/pgoutput/client/keepalive.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    KeepaliveData = Data.define(:wal_end, :server_clock, :reply_requested)
+
+    # Immutable primary keepalive replication message.
+    #
+    # PostgreSQL sends keepalive CopyData payloads while a replication stream is
+    # active. The payload layout is:
+    #
+    # ```text
+    # Byte 0      : message tag `k`
+    # Bytes 1-8   : current server WAL end, unsigned 64-bit big-endian
+    # Bytes 9-16  : server clock, PostgreSQL timestamp format
+    # Byte 17     : reply-requested flag, 1 for immediate feedback
+    # ```
+    #
+    # The stream layer uses this message to advance its known WAL position and to
+    # decide whether to send standby status feedback immediately.
+    #
+    # @attr_reader wal_end [Integer] latest server WAL position
+    # @attr_reader server_clock [Integer] PostgreSQL server timestamp in
+    #   microseconds since 2000-01-01 UTC
+    # @attr_reader reply_requested [Boolean] whether PostgreSQL requested
+    #   immediate feedback
+    class Keepalive < KeepaliveData
+      # Parse a keepalive CopyData payload.
+      #
+      # @param bytes [String] raw CopyData payload beginning with `k`
+      # @return [Keepalive] immutable parsed keepalive message
+      # @raise [ProtocolError] if the payload is empty, has the wrong message
+      #   tag, or is too short to contain the required fields
+      def self.parse(bytes)
+        binary = bytes.b
+        raise ProtocolError, "empty CopyData payload" if binary.empty?
+        raise ProtocolError, "expected keepalive message" unless binary.getbyte(0) == "k".ord
+        raise ProtocolError, "truncated keepalive message" if binary.bytesize < 18
+
+        wal_end = unpack_u64(binary, 1)
+        server_clock = unpack_u64(binary, 9)
+        reply_requested = binary.getbyte(17) == 1
+
+        Ractor.make_shareable(new(wal_end, server_clock, reply_requested))
+      end
+
+      # Latest server WAL position formatted as a PostgreSQL LSN string.
+      #
+      # @return [String]
+      def wal_end_lsn = LSN.format(wal_end)
+
+      # @param binary [String]
+      # @param offset [Integer]
+      # @return [Integer]
+      def self.unpack_u64(binary, offset)
+        value = binary.byteslice(offset, 8)&.unpack1("Q>")
+        raise ProtocolError, "failed to unpack uint64" unless value.is_a?(Integer)
+
+        value
+      end
+      private_class_method :unpack_u64
+    end
+  end
+end

data/lib/pgoutput/client/lsn.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    # PostgreSQL Log Sequence Number conversion helpers.
+    #
+    # PostgreSQL represents LSNs as two hexadecimal 32-bit halves separated by a
+    # slash, such as `0/16B6C50`. The replication protocol transmits the same WAL
+    # position as an unsigned 64-bit integer. This module converts between those
+    # two representations.
+    #
+    # @example Parse a textual LSN
+    #   Pgoutput::Client::LSN.parse("0/10")
+    #   # => 16
+    #
+    # @example Format an integer WAL position
+    #   Pgoutput::Client::LSN.format(16)
+    #   # => "0/10"
+    #
+    # @api public
+    module LSN
+      module_function
+
+      # Parse a PostgreSQL LSN string into an integer WAL position.
+      #
+      # @param value [#to_s] LSN string in `HEX/HEX` form
+      # @return [Integer] unsigned 64-bit WAL position
+      # @raise [ArgumentError] if the value is not a valid LSN string
+      def parse(value)
+        high, low = String(value).split("/", 2)
+        raise ArgumentError, "invalid LSN: #{value.inspect}" if high.nil? || low.nil?
+
+        (Integer(high, 16) << 32) + Integer(low, 16)
+      rescue ArgumentError
+        raise ArgumentError, "invalid LSN: #{value.inspect}"
+      end
+
+      # Format an integer WAL position as a PostgreSQL LSN string.
+      #
+      # @param value [#to_int, #to_s] non-negative integer WAL position
+      # @return [String] LSN string in uppercase hexadecimal `HEX/HEX` form
+      # @raise [ArgumentError] if the value is negative or cannot be coerced to
+      #   an integer
+      def format(value)
+        integer = Integer(value)
+        raise ArgumentError, "LSN must be non-negative" if integer.negative?
+
+        high = integer >> 32
+        low = integer & 0xFFFF_FFFF
+        "#{high.to_s(16).upcase}/#{low.to_s(16).upcase}"
+      end
+    end
+  end
+end

data/lib/pgoutput/client/stream.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    # Logical replication stream loop.
+    #
+    # `Stream` consumes PostgreSQL CopyData payloads from a {Connection}. It
+    # understands the two replication envelope message types used by PostgreSQL's
+    # streaming replication protocol:
+    #
+    # * `w` — XLogData, containing logical decoding plugin payload bytes.
+    # * `k` — primary keepalive, optionally requesting immediate feedback.
+    #
+    # The stream yields only XLogData plugin payloads. Keepalive messages are
+    # handled internally by updating the latest known WAL position and sending
+    # standby feedback when requested.
+    #
+    # @api private
+    class Stream
+      # Build a stream loop.
+      #
+      # @param connection [Connection] replication connection
+      # @param configuration [Configuration] stream configuration
+      # @return [void]
+      def initialize(connection:, configuration:)
+        @connection = connection
+        @configuration = configuration
+        @latest_lsn = LSN.parse(configuration.start_lsn_string)
+        @last_feedback_at = monotonic_time
+        @running = false
+      end
+
+      # Start the stream loop.
+      #
+      # The method blocks while the stream is running. For every XLogData
+      # envelope, it yields the raw pgoutput payload and the parsed envelope
+      # metadata. When no CopyData payload is currently available, the loop
+      # continues and checks again.
+      #
+      # @yield [payload, metadata] called for each XLogData payload
+      # @yieldparam payload [String] frozen raw pgoutput payload bytes
+      # @yieldparam metadata [XLogData] parsed WAL envelope metadata
+      # @return [void]
+      # @raise [ArgumentError] if no block is provided
+      # @raise [ProtocolError] if an unknown or malformed replication message is
+      #   received
+      # @raise [ConnectionError] if standby feedback cannot be sent
+      def start(&block)
+        raise ArgumentError, "block required" unless block_given?
+
+        @running = true
+        while @running
+          copy_data = @connection.get_copy_data
+          next unless copy_data
+
+          process_copy_data(copy_data, &block)
+          send_periodic_feedback
+        end
+      end
+
+      # Stop the stream loop after the current iteration.
+      #
+      # @return [void]
+      def stop
+        @running = false
+      end
+
+      private
+
+      def process_copy_data(copy_data)
+        case copy_data.getbyte(0)
+        when "w".ord
+          xlog = XLogData.parse(copy_data)
+          @latest_lsn = xlog.wal_end
+          yield xlog.payload, xlog
+        when "k".ord
+          keepalive = Keepalive.parse(copy_data)
+          @latest_lsn = [@latest_lsn, keepalive.wal_end].max
+          send_feedback(reply_requested: true) if keepalive.reply_requested
+        else
+          raise ProtocolError, "unknown CopyData replication message: #{copy_data.getbyte(0).inspect}"
+        end
+      end
+
+      def send_periodic_feedback
+        return if monotonic_time - @last_feedback_at < @configuration.feedback_interval
+
+        send_feedback(reply_requested: false)
+      end
+
+      def send_feedback(reply_requested:)
+        feedback = Feedback.now(received_lsn: @latest_lsn, reply_requested:)
+        @connection.put_copy_data(feedback.to_copy_data)
+        @last_feedback_at = monotonic_time
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/pgoutput/client/version.rb

@@ -2,6 +2,9 @@
 
 module Pgoutput
   module Client
-    VERSION = "0.0.0"
+    # Current pgoutput-client gem version.
+    #
+    # @return [String]
+    VERSION = "0.1.0"
   end
 end

data/lib/pgoutput/client/xlog_data.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    ReplicationXLogData = Data.define(:wal_start, :wal_end, :server_clock, :payload)
+
+    # Immutable XLogData replication envelope.
+    #
+    # PostgreSQL wraps logical decoding plugin output in an XLogData CopyData
+    # payload while streaming replication is active. The payload layout is:
+    #
+    # ```text
+    # Byte 0      : message tag `w`
+    # Bytes 1-8   : WAL start position, unsigned 64-bit big-endian
+    # Bytes 9-16  : WAL end position, unsigned 64-bit big-endian
+    # Bytes 17-24 : server clock, PostgreSQL timestamp format
+    # Bytes 25..  : logical decoding plugin payload
+    # ```
+    #
+    # Instances are created through {.parse} and made shareable so they can cross
+    # Ractor boundaries with their frozen payload bytes.
+    #
+    # @attr_reader wal_start [Integer] WAL position where this message begins
+    # @attr_reader wal_end [Integer] WAL position after this message
+    # @attr_reader server_clock [Integer] PostgreSQL server timestamp in
+    #   microseconds since 2000-01-01 UTC
+    # @attr_reader payload [String] frozen raw logical decoding plugin payload
+    class XLogData < ReplicationXLogData
+      # Parse an XLogData CopyData payload.
+      #
+      # @param bytes [String] raw CopyData payload beginning with `w`
+      # @return [XLogData] immutable parsed envelope
+      # @raise [ProtocolError] if the payload is empty, has the wrong message
+      #   tag, or is too short to contain the required fields
+      def self.parse(bytes)
+        binary = bytes.b
+        raise ProtocolError, "empty CopyData payload" if binary.empty?
+        raise ProtocolError, "expected XLogData message" unless binary.getbyte(0) == "w".ord
+        raise ProtocolError, "truncated XLogData message" if binary.bytesize < 25
+
+        wal_start = unpack_u64(binary, 1)
+        wal_end = unpack_u64(binary, 9)
+        server_clock = unpack_u64(binary, 17)
+        payload = binary.byteslice(25..)&.freeze || "".b.freeze
+
+        Ractor.make_shareable(new(wal_start, wal_end, server_clock, payload))
+      end
+
+      # Starting WAL position formatted as a PostgreSQL LSN string.
+      #
+      # @return [String]
+      def wal_start_lsn = LSN.format(wal_start)
+
+      # Ending WAL position formatted as a PostgreSQL LSN string.
+      #
+      # @return [String]
+      def wal_end_lsn = LSN.format(wal_end)
+
+      # @param binary [String]
+      # @param offset [Integer]
+      # @return [Integer]
+      def self.unpack_u64(binary, offset)
+        value = binary.byteslice(offset, 8)&.unpack1("Q>")
+        raise ProtocolError, "failed to unpack uint64" unless value.is_a?(Integer)
+
+        value
+      end
+      private_class_method :unpack_u64
+    end
+  end
+end

data/lib/pgoutput/client.rb

@@ -1,10 +1,125 @@
 # frozen_string_literal: true
 
 require_relative "client/version"
+require_relative "client/errors"
+require_relative "client/configuration"
+require_relative "client/lsn"
+require_relative "client/xlog_data"
+require_relative "client/keepalive"
+require_relative "client/feedback"
+require_relative "client/commands"
+require_relative "client/connection"
+require_relative "client/stream"
 
 module Pgoutput
+  # Namespace for PostgreSQL logical replication transport support.
+  #
+  # `Pgoutput::Client` is the replication transport layer of the CDC Ecosystem.
+  # It is responsible for connecting to PostgreSQL in replication mode, creating
+  # or consuming replication slots, issuing `START_REPLICATION`, reading CopyData
+  # messages, and sending standby feedback.
+  #
+  # This namespace intentionally does not parse pgoutput plugin payloads into
+  # table-level changes. Raw plugin bytes are yielded to downstream protocol and
+  # type layers such as `pgoutput-parser` and `pgoutput-decoder`.
+  #
+  # @api public
   module Client
-    class Error < StandardError; end
-    # Your code goes here...
+    # High-level logical replication client facade.
+    #
+    # `Runner` is the simplest public entry point for consumers that want to
+    # stream raw pgoutput payloads without manually managing a replication
+    # connection. It owns the connection lifecycle for one logical replication
+    # stream:
+    #
+    # 1. Build an immutable {Configuration} from keyword arguments.
+    # 2. Open a PostgreSQL replication connection.
+    # 3. Optionally create the configured replication slot.
+    # 4. Start logical replication.
+    # 5. Yield raw pgoutput payload bytes and {XLogData} metadata.
+    # 6. Close the connection when streaming exits.
+    #
+    # @example Stream raw pgoutput messages
+    #   runner = Pgoutput::Client::Runner.new(
+    #     database_url: "postgres://localhost/app",
+    #     slot_name: "cdc_slot",
+    #     publication_names: ["app_publication"]
+    #   )
+    #
+    #   runner.start do |payload, metadata|
+    #     puts "received #{payload.bytesize} bytes at #{metadata.wal_end_lsn}"
+    #   end
+    #
+    # @see Configuration
+    # @see Connection
+    # @see Stream
+    # @api public
+    class Runner
+      # Configuration used by this runner.
+      #
+      # @return [Configuration]
+      attr_reader :configuration
+
+      # Build a runner from configuration keyword arguments.
+      #
+      # The accepted keywords are the same as {Configuration#initialize}. The
+      # resulting configuration object is immutable and reused for the lifetime
+      # of this runner.
+      #
+      # @param options [Hash{Symbol=>Object}] configuration options forwarded to
+      #   {Configuration#initialize}
+      # @return [void]
+      # @raise [ConfigurationError] if the supplied configuration is invalid
+      def initialize(**options)
+        @configuration = Configuration.new(
+          **options # : untyped
+        )
+        @stopped = false
+      end
+
+      # Start streaming raw pgoutput payloads.
+      #
+      # This method blocks until the stream stops or the underlying connection
+      # raises an error. The yielded payload is the raw logical decoding plugin
+      # payload contained inside PostgreSQL's XLogData envelope; callers normally
+      # pass this payload to a pgoutput parser.
+      #
+      # @yield [payload, metadata] called once for each XLogData payload
+      # @yieldparam payload [String] frozen raw pgoutput payload bytes
+      # @yieldparam metadata [XLogData] WAL envelope metadata for the payload
+      # @return [void]
+      # @raise [ArgumentError] if no block is provided
+      # @raise [ConnectionError] if a PostgreSQL connection or command fails
+      # @raise [ProtocolError] if an invalid replication message is received
+      def start(&block)
+        raise ArgumentError, "block required" unless block
+
+        connection = Connection.open(configuration)
+        setup_connection(connection)
+        Stream.new(connection:, configuration:).start { |payload, metadata| block.call(payload, metadata) }
+      ensure
+        connection&.close
+      end
+
+      # Request graceful stop.
+      #
+      # This method records the caller's intent to stop. The current
+      # implementation does not yet interrupt an active {Stream}; it exists as
+      # part of the public lifecycle API and may be wired into cooperative stream
+      # shutdown in a future release.
+      #
+      # @return [void]
+      def stop
+        @stopped = true
+        nil
+      end
+
+      private
+
+      def setup_connection(connection)
+        connection.create_replication_slot if configuration.auto_create_slot
+        connection.start_replication
+      end
+    end
   end
 end

data/lib/pgoutput_client.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# Main convenience require for pgoutput-client.
+#
+# Requiring this file loads the public `Pgoutput::Client` namespace and its
+# transport-layer classes.
+#
+# @example
+#   require "pgoutput_client"
+#
+# @see Pgoutput::Client
+require_relative "pgoutput/client"

data/sig/pg.rbs

@@ -0,0 +1,12 @@
+module PG
+  def self.connect: (String conninfo, ?replication: String) -> Connection
+
+  class Error < StandardError
+  end
+
+  class Connection
+    def exec: (String sql) -> untyped
+    def close: () -> void
+    def finished?: () -> bool
+  end
+end