pgoutput-client 0.0.0 → 0.2.0

data/lib/pgoutput/client/runner.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    # High-level logical replication client facade.
+    #
+    # `Runner` is the simplest public entry point for consumers that want to
+    # stream raw pgoutput payloads without manually managing a replication
+    # connection. It owns the connection lifecycle for one logical replication
+    # stream:
+    #
+    # 1. Build an immutable {Configuration} from keyword arguments.
+    # 2. Open a PostgreSQL replication connection.
+    # 3. Optionally create the configured replication slot.
+    # 4. Start logical replication.
+    # 5. Yield raw pgoutput payload bytes and {XLogData} metadata.
+    # 6. Close the connection when streaming exits.
+    #
+    # If a live stream loses its connection, the runner retries a small number
+    # of times with a backoff and resumes from the latest confirmed WAL
+    # position. Replay, checkpointing, and deduplication are not owned here;
+    # those concerns belong to the downstream CDC runtime and sink layer.
+    #
+    # @example Stream raw pgoutput messages
+    #   runner = Pgoutput::Client::Runner.new(
+    #     database_url: "postgres://localhost/app",
+    #     slot_name: "cdc_slot",
+    #     publication_names: ["app_publication"]
+    #   )
+    #
+    #   runner.start do |payload, metadata|
+    #     puts "received #{payload.bytesize} bytes at #{metadata.wal_end_lsn}"
+    #   end
+    #
+    # @see Configuration
+    # @see Connection
+    # @see Stream
+    # @api public
+    class Runner
+      DEFAULT_RECONNECT_ATTEMPTS = 3
+      DEFAULT_RECONNECT_BACKOFF = 0.5
+
+      # Configuration used by this runner.
+      #
+      # @return [Configuration]
+      attr_reader :configuration
+
+      # Last transport error seen by the runner.
+      #
+      # @return [Exception, nil]
+      attr_reader :last_error
+
+      # Build a runner from configuration keyword arguments.
+      #
+      # The accepted keywords are the same as {Configuration#initialize}. The
+      # resulting configuration object is immutable and reused for the lifetime
+      # of this runner.
+      #
+      # @param options [Hash{Symbol=>Object}] configuration options forwarded to
+      #   {Configuration#initialize}
+      # @return [void]
+      # @raise [ConfigurationError] if the supplied configuration is invalid
+      def initialize(**options)
+        @configuration = Configuration.new(
+          **options # : untyped
+        )
+        @stopped = true
+        @running = false
+        @stream = nil
+        @resume_lsn = configuration.start_lsn
+        @acked_lsn = configuration.start_lsn
+        @slot_created = false
+        @last_error = nil
+        @reconnect_attempts = 0
+      end
+
+      # Start streaming raw pgoutput payloads.
+      #
+      # This method blocks until the stream stops or the underlying connection
+      # raises an error. The yielded payload is the raw logical decoding plugin
+      # payload contained inside PostgreSQL's XLogData envelope; callers normally
+      # pass this payload to a pgoutput parser.
+      #
+      # @yield [payload, metadata] called once for each XLogData payload
+      # @yieldparam payload [String] frozen raw pgoutput payload bytes
+      # @yieldparam metadata [XLogData] WAL envelope metadata for the payload
+      # @return [void]
+      # @raise [ArgumentError] if no block is provided
+      # @raise [ConnectionError] if a PostgreSQL connection or command fails
+      # @raise [ProtocolError] if an invalid replication message is received
+      def start(&block)
+        raise ArgumentError, "block required" unless block
+
+        @stopped = false
+        @running = true
+        @last_error = nil
+        @reconnect_attempts = 0
+
+        loop do
+          current_configuration = configuration_for_resume
+          case run_stream_cycle(current_configuration, &block)
+          when :done
+            break
+          when :retry
+            @reconnect_attempts += 1
+            raise @last_error if @reconnect_attempts > DEFAULT_RECONNECT_ATTEMPTS
+
+            sleep(reconnect_backoff_for(@reconnect_attempts))
+          end
+        end
+      ensure
+        @running = false
+        @stopped = true
+      end
+
+      # Request graceful stop.
+      #
+      # This method records the caller's intent to stop and asks the active
+      # {Stream}, if any, to stop after its current iteration.
+      #
+      # @return [void]
+      def stop
+        @stopped = true
+        @stream&.stop
+        nil
+      end
+
+      # Stop the active stream and start again with the same block.
+      #
+      # The runner is synchronous, so this helper is primarily useful for
+      # supervisors that call it instead of manually calling {#stop} followed by
+      # {#start}.
+      #
+      # @yield [payload, metadata] called once for each XLogData payload
+      # @return [void]
+      def restart(&block)
+        stop
+        start(&block)
+      end
+
+      # Whether the runner is currently inside its streaming loop.
+      #
+      # @return [Boolean]
+      def running?
+        @running
+      end
+
+      # Whether the runner has stopped.
+      #
+      # @return [Boolean]
+      def stopped?
+        !running?
+      end
+
+      # Whether an active replication stream exists.
+      #
+      # @return [Boolean]
+      def connected?
+        !@stream.nil?
+      end
+
+      # Mark a WAL position as durably handled by downstream code.
+      #
+      # This does not checkpoint or persist anything. It only updates transport
+      # feedback state so future standby status updates can distinguish received
+      # WAL from downstream-acknowledged WAL.
+      #
+      # @param lsn [String, Integer] WAL position acknowledged by downstream code
+      # @return [Integer] normalized acknowledged WAL position
+      def ack(lsn)
+        parsed = normalize_lsn_value(lsn)
+        @acked_lsn = [@acked_lsn ? normalize_lsn_value(@acked_lsn) : 0, parsed].max
+        @stream&.ack(@acked_lsn)
+        @acked_lsn
+      end
+
+      # Return an immutable transport status snapshot.
+      #
+      # @return [RunnerState]
+      def monitor
+        RunnerState.new(
+          running: running?,
+          stop_requested: @stopped,
+          connected: connected?,
+          last_received_lsn: current_lsn_string(@stream&.latest_lsn || @resume_lsn),
+          last_feedback_lsn: current_lsn_string(@stream&.acked_lsn || @acked_lsn),
+          last_keepalive_at: @stream&.last_keepalive_at,
+          last_error: @last_error&.message,
+          reconnect_attempts: @reconnect_attempts
+        )
+      end
+
+      private
+
+      def setup_connection(connection)
+        if configuration.auto_create_slot && !@slot_created
+          connection.create_replication_slot
+          @slot_created = true
+        end
+
+        connection.start_replication
+      end
+
+      def run_stream_cycle(configuration, &block)
+        connection = Connection.open(configuration)
+        setup_connection(connection)
+        @stream = Stream.new(connection:, configuration:, acked_lsn: @acked_lsn)
+        @stream.start(&block)
+        :done
+      rescue ConnectionError => e
+        @last_error = e
+        raise if @stopped || @stream.nil?
+
+        @resume_lsn = @stream.latest_lsn || @resume_lsn
+        @acked_lsn = @stream.acked_lsn || @acked_lsn
+        :retry
+      ensure
+        @stream = nil
+        connection&.close
+      end
+
+      def configuration_for_resume
+        return configuration if @resume_lsn.nil?
+
+        Configuration.new(
+          database_url: configuration.database_url,
+          slot_name: configuration.slot_name,
+          publication_names: configuration.publication_names,
+          start_lsn: @resume_lsn,
+          proto_version: configuration.proto_version,
+          binary: configuration.binary,
+          messages: configuration.messages,
+          auto_create_slot: configuration.auto_create_slot,
+          temporary_slot: configuration.temporary_slot,
+          feedback_interval: configuration.feedback_interval
+        )
+      end
+
+      def normalize_lsn_value(value)
+        value.is_a?(String) ? LSN.parse(value) : LSN.parse(LSN.format(value))
+      end
+
+      def current_lsn_string(value)
+        return nil if value.nil?
+
+        value.is_a?(String) ? value : LSN.format(value)
+      end
+
+      def reconnect_backoff_for(attempts)
+        DEFAULT_RECONNECT_BACKOFF * attempts
+      end
+
+      def sleep(duration)
+        Kernel.sleep(duration)
+      end
+    end
+  end
+end

data/lib/pgoutput/client/state.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    RunnerStateData = Data.define(
+      :running,
+      :stop_requested,
+      :connected,
+      :last_received_lsn,
+      :last_feedback_lsn,
+      :last_keepalive_at,
+      :last_error,
+      :reconnect_attempts
+    )
+
+    # Immutable operational snapshot for a {Runner}.
+    #
+    # `RunnerState` is intentionally small and transport-focused. It exposes
+    # connection, feedback, keepalive, and retry state without claiming anything
+    # about downstream processing, sink delivery, or business-level consumption.
+    #
+    # @attr_reader running [Boolean] whether the runner is inside its streaming loop
+    # @attr_reader stop_requested [Boolean] whether graceful stop was requested
+    # @attr_reader connected [Boolean] whether an active replication stream exists
+    # @attr_reader last_received_lsn [String, nil] latest WAL position received from PostgreSQL
+    # @attr_reader last_feedback_lsn [String, nil] latest downstream-acknowledged WAL position
+    #   used for flushed/applied feedback
+    # @attr_reader last_keepalive_at [Time, nil] last time a primary keepalive was observed
+    # @attr_reader last_error [String, nil] last transport error message
+    # @attr_reader reconnect_attempts [Integer] reconnect attempts used by the current/last run
+    # @api public
+    class RunnerState < RunnerStateData
+      # Whether the runner currently has no active stream.
+      #
+      # @return [Boolean]
+      def stopped?
+        !running
+      end
+    end
+  end
+end

data/lib/pgoutput/client/stream.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    # Logical replication stream loop.
+    #
+    # `Stream` consumes PostgreSQL CopyData payloads from a {Connection}. It
+    # understands the two replication envelope message types used by PostgreSQL's
+    # streaming replication protocol:
+    #
+    # * `w` — XLogData, containing logical decoding plugin payload bytes.
+    # * `k` — primary keepalive, optionally requesting immediate feedback.
+    #
+    # The stream yields only XLogData plugin payloads. Keepalive messages are
+    # handled internally by updating the latest known WAL position and sending
+    # standby feedback when requested.
+    #
+    # Feedback separates receipt from downstream acknowledgment. Received LSN
+    # follows the latest WAL position seen from PostgreSQL. Flushed/applied LSN
+    # follows {#ack}, allowing downstream consumers to decide when a WAL position
+    # is safe to report as durable.
+    #
+    # @api private
+    class Stream
+      # Latest WAL position observed from XLogData or keepalive messages.
+      #
+      # @return [Integer]
+      attr_reader :latest_lsn
+
+      # Latest downstream-acknowledged WAL position used as flushed/applied LSN
+      # in standby feedback.
+      #
+      # @return [Integer]
+      attr_reader :acked_lsn
+
+      # Last time a primary keepalive was observed.
+      #
+      # @return [Time, nil]
+      attr_reader :last_keepalive_at
+
+      # Build a stream loop.
+      #
+      # @param connection [Connection] replication connection
+      # @param configuration [Configuration] stream configuration
+      # @param acked_lsn [String, Integer, nil] initial downstream-acknowledged
+      #   WAL position
+      # @return [void]
+      def initialize(connection:, configuration:, acked_lsn: nil)
+        @connection = connection
+        @configuration = configuration
+        @latest_lsn = LSN.parse(configuration.start_lsn_string)
+        @acked_lsn = acked_lsn ? normalize_lsn_value(acked_lsn) : @latest_lsn
+        @last_feedback_at = monotonic_time
+        @last_keepalive_at = nil
+        @running = false
+        @stop_requested = false
+      end
+
+      # Start the stream loop.
+      #
+      # The method blocks while the stream is running. For every XLogData
+      # envelope, it yields the raw pgoutput payload and the parsed envelope
+      # metadata. When no CopyData payload is currently available, the loop
+      # pauses briefly before checking again.
+      #
+      # @yield [payload, metadata] called for each XLogData payload
+      # @yieldparam payload [String] frozen raw pgoutput payload bytes
+      # @yieldparam metadata [XLogData] parsed WAL envelope metadata
+      # @return [void]
+      # @raise [ArgumentError] if no block is provided
+      # @raise [ProtocolError] if an unknown or malformed replication message is
+      #   received
+      # @raise [ConnectionError] if standby feedback cannot be sent
+      def start(&block)
+        raise ArgumentError, "block required" unless block_given?
+
+        @running = true
+        while @running
+          copy_data = @connection.get_copy_data
+          if copy_data.nil?
+            sleep 0.01
+            next
+          end
+
+          process_copy_data(copy_data, &block)
+          send_periodic_feedback
+        end
+      ensure
+        send_feedback(reply_requested: false) if @stop_requested
+        @running = false
+      end
+
+      # Stop the stream loop after the current iteration.
+      #
+      # @return [void]
+      def stop
+        @stop_requested = true
+        @running = false
+        nil
+      end
+
+      # Whether the stream loop is active.
+      #
+      # @return [Boolean]
+      def running?
+        @running
+      end
+
+      # Mark a WAL position as durably handled by downstream code.
+      #
+      # The stream never decides sink durability on its own. Downstream code may
+      # call this after checkpointing, enqueueing, or otherwise making progress
+      # durable. Feedback sent after this call reports the acknowledged LSN as
+      # both flushed and applied.
+      #
+      # @param lsn [String, Integer] WAL position acknowledged by downstream code
+      # @return [Integer] normalized acknowledged WAL position
+      def ack(lsn)
+        parsed = normalize_lsn_value(lsn)
+        @acked_lsn = [@acked_lsn, parsed].max
+      end
+
+      private
+
+      def process_copy_data(copy_data)
+        case copy_data.getbyte(0)
+        when "w".ord
+          xlog = XLogData.parse(copy_data)
+          @latest_lsn = xlog.wal_end
+          yield xlog.payload, xlog
+        when "k".ord
+          keepalive = Keepalive.parse(copy_data)
+          @latest_lsn = [@latest_lsn, keepalive.wal_end].max
+          @last_keepalive_at = Time.now.utc
+          send_feedback(reply_requested: true) if keepalive.reply_requested
+        else
+          raise ProtocolError, "unknown CopyData replication message: #{copy_data.getbyte(0).inspect}"
+        end
+      end
+
+      def send_periodic_feedback
+        return if monotonic_time - @last_feedback_at < @configuration.feedback_interval
+
+        send_feedback(reply_requested: false)
+      end
+
+      def send_feedback(reply_requested:)
+        feedback = Feedback.now(received_lsn: @latest_lsn, flushed_lsn: @acked_lsn, applied_lsn: @acked_lsn,
+                                reply_requested:)
+        @connection.put_copy_data(feedback.to_copy_data)
+        @last_feedback_at = monotonic_time
+      end
+
+      def normalize_lsn_value(value)
+        value.is_a?(String) ? LSN.parse(value) : LSN.parse(LSN.format(value))
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/pgoutput/client/version.rb

@@ -2,6 +2,9 @@
 
 module Pgoutput
   module Client
-    VERSION = "0.0.0"
+    # Current pgoutput-client gem version.
+    #
+    # @return [String]
+    VERSION = "0.2.0"
   end
 end

data/lib/pgoutput/client/xlog_data.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Pgoutput
+  module Client
+    ReplicationXLogData = Data.define(:wal_start, :wal_end, :server_clock, :payload)
+
+    # Immutable XLogData replication envelope.
+    #
+    # PostgreSQL wraps logical decoding plugin output in an XLogData CopyData
+    # payload while streaming replication is active. The payload layout is:
+    #
+    # ```text
+    # Byte 0      : message tag `w`
+    # Bytes 1-8   : WAL start position, unsigned 64-bit big-endian
+    # Bytes 9-16  : WAL end position, unsigned 64-bit big-endian
+    # Bytes 17-24 : server clock, PostgreSQL timestamp format
+    # Bytes 25..  : logical decoding plugin payload
+    # ```
+    #
+    # Instances are created through {.parse} and made shareable so they can cross
+    # Ractor boundaries with their frozen payload bytes.
+    #
+    # @attr_reader wal_start [Integer] WAL position where this message begins
+    # @attr_reader wal_end [Integer] WAL position after this message
+    # @attr_reader server_clock [Integer] PostgreSQL server timestamp in
+    #   microseconds since 2000-01-01 UTC
+    # @attr_reader payload [String] frozen raw logical decoding plugin payload
+    class XLogData < ReplicationXLogData
+      # Parse an XLogData CopyData payload.
+      #
+      # @param bytes [String] raw CopyData payload beginning with `w`
+      # @return [XLogData] immutable parsed envelope
+      # @raise [ProtocolError] if the payload is empty, has the wrong message
+      #   tag, or is too short to contain the required fields
+      def self.parse(bytes)
+        binary = bytes.b
+        raise ProtocolError, "empty CopyData payload" if binary.empty?
+        raise ProtocolError, "expected XLogData message" unless binary.getbyte(0) == "w".ord
+        raise ProtocolError, "truncated XLogData message" if binary.bytesize < 25
+
+        wal_start = unpack_u64(binary, 1)
+        wal_end = unpack_u64(binary, 9)
+        server_clock = unpack_u64(binary, 17)
+        payload = binary.byteslice(25..)&.freeze || "".b.freeze
+
+        Ractor.make_shareable(new(wal_start, wal_end, server_clock, payload))
+      end
+
+      # Starting WAL position formatted as a PostgreSQL LSN string.
+      #
+      # @return [String]
+      def wal_start_lsn = LSN.format(wal_start)
+
+      # Ending WAL position formatted as a PostgreSQL LSN string.
+      #
+      # @return [String]
+      def wal_end_lsn = LSN.format(wal_end)
+
+      # @param binary [String]
+      # @param offset [Integer]
+      # @return [Integer]
+      def self.unpack_u64(binary, offset)
+        value = binary.byteslice(offset, 8)&.unpack1("Q>")
+        raise ProtocolError, "failed to unpack uint64" unless value.is_a?(Integer)
+
+        value
+      end
+      private_class_method :unpack_u64
+    end
+  end
+end

data/lib/pgoutput/client.rb

@@ -1,10 +1,31 @@
 # frozen_string_literal: true
 
 require_relative "client/version"
+require_relative "client/errors"
+require_relative "client/configuration"
+require_relative "client/lsn"
+require_relative "client/xlog_data"
+require_relative "client/keepalive"
+require_relative "client/feedback"
+require_relative "client/state"
+require_relative "client/commands"
+require_relative "client/connection"
+require_relative "client/stream"
+require_relative "client/runner"
 
 module Pgoutput
+  # Namespace for PostgreSQL logical replication transport support.
+  #
+  # `Pgoutput::Client` is the replication transport layer of the CDC Ecosystem.
+  # It is responsible for connecting to PostgreSQL in replication mode, creating
+  # or consuming replication slots, issuing `START_REPLICATION`, reading CopyData
+  # messages, and sending standby feedback.
+  #
+  # This namespace intentionally does not parse pgoutput plugin payloads into
+  # table-level changes. Raw plugin bytes are yielded to downstream protocol and
+  # type layers such as `pgoutput-parser` and `pgoutput-decoder`.
+  #
+  # @api public
   module Client
-    class Error < StandardError; end
-    # Your code goes here...
   end
 end

data/lib/pgoutput_client.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# Main convenience require for pgoutput-client.
+#
+# Requiring this file loads the public `Pgoutput::Client` namespace and its
+# transport-layer classes.
+#
+# @example
+#   require "pgoutput_client"
+#
+# @see Pgoutput::Client
+require_relative "pgoutput/client"

data/sig/pg.rbs

@@ -0,0 +1,12 @@
+module PG
+  def self.connect: (String conninfo, ?replication: String) -> Connection
+
+  class Error < StandardError
+  end
+
+  class Connection
+    def exec: (String sql) -> untyped
+    def close: () -> void
+    def finished?: () -> bool
+  end
+end

data/sig/pgoutput/client/commands.rbs

@@ -0,0 +1,42 @@
+module Pgoutput
+  module Client
+    # SQL command builders for PostgreSQL replication-mode commands.
+    #
+    # PostgreSQL replication commands are issued on a connection opened with the
+    # replication parameter enabled. The methods in this module render the small
+    # command subset needed by `pgoutput-client` and rely on {Configuration} to
+    # validate identifier-like values before interpolation.
+    #
+    # @api private
+    module Commands
+      # Render a `CREATE_REPLICATION_SLOT` command.
+      #
+      # Temporary slots are requested only when
+      # {Configuration#temporary_slot} is true.
+      #
+      # @example Permanent slot
+      #   Commands.create_replication_slot(config)
+      #   # => "CREATE_REPLICATION_SLOT cdc_slot LOGICAL pgoutput"
+      #
+      # @param configuration [Configuration] replication configuration
+      # @return [String] SQL command suitable for `PG::Connection#exec`
+      def self?.create_replication_slot: (untyped configuration) -> ::String
+
+      # Render a `DROP_REPLICATION_SLOT` command.
+      #
+      # @param configuration [Configuration] replication configuration
+      # @return [String] SQL command suitable for `PG::Connection#exec`
+      def self?.drop_replication_slot: (untyped configuration) -> ::String
+
+      # Render a `START_REPLICATION SLOT ... LOGICAL ...` command.
+      #
+      # The command includes the pgoutput options required by PostgreSQL:
+      # `proto_version` and `publication_names`. Optional pgoutput switches such
+      # as `binary` and `messages` are emitted only when enabled.
+      #
+      # @param configuration [Configuration] replication configuration
+      # @return [String] SQL command suitable for `PG::Connection#exec`
+      def self?.start_replication: (untyped configuration) -> ::String
+    end
+  end
+end