pcrd 0.1.0

data/lib/pcrd/readiness/manifest.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Pcrd
+  module Readiness
+    # Builds the target-readiness manifest: for each migrated table, which
+    # secondary objects exist on the source, whether the target already has
+    # them, and runnable DDL to create the missing ones before cutover.
+    #
+    # The load DDL (Schema::DDL) creates only table + primary key; indexes,
+    # constraints, grants, etc. are deferred so the bulk load is fast. This
+    # turns that deferral from tribal knowledge into an explicit checklist.
+    #
+    # Rename/drop aware: an object referencing a dropped column is reported as
+    # not-recreatable; one referencing a renamed column is flagged for manual
+    # regeneration (its source DDL is shown commented out) rather than emitting
+    # silently-wrong SQL.
+    #
+    # Sequences/identity are reported as informational — they are restored
+    # automatically by `pcrd cutover` (Cutover::Sequences), so the manifest does
+    # not emit competing DDL for them.
+    class Manifest
+      # status: :missing (DDL provided) | :present | :needs_review | :info
+      Entry  = Data.define(:category, :name, :status, :detail, :ddl)
+      Table  = Data.define(:table_name, :entries)
+      Result = Data.define(:tables)
+
+      KIND_LABEL = { "f" => "foreign key", "u" => "unique constraint", "c" => "check constraint" }.freeze
+
+      def initialize(source_pool:, target_pool:, config:)
+        @source = source_pool
+        @target = target_pool
+        @config = config
+      end
+
+      def build
+        src = Schema::ObjectReader.new(@source)
+        tgt = Schema::ObjectReader.new(@target)
+
+        tables = (@config.migrate&.tables || []).map do |table_config|
+          Table.new(table_name: table_config.name, entries: entries_for(table_config, src, tgt))
+        end
+
+        Result.new(tables: tables)
+      end
+
+      private
+
+      def entries_for(table_config, src, tgt)
+        name           = table_config.name
+        drops, renames = change_maps(table_config)
+        present_idx    = tgt.indexes(name).map(&:name).to_set
+        present_con    = tgt.constraints(name).map(&:name).to_set
+
+        index_entries(name, src.indexes(name), present_idx, drops, renames) +
+          constraint_entries(name, src.constraints(name), present_con, drops, renames) +
+          sequence_entries(src.identity_columns(name)) +
+          owner_entries(name, src, tgt) +
+          grant_entries(name, src, tgt) +
+          comment_entries(name, src, tgt, drops, renames)
+      end
+
+      def index_entries(table, indexes, present, drops, renames)
+        indexes.map do |ix|
+          review = review_reason(ix.columns, drops, renames)
+          if present.include?(ix.name)
+            entry("index", ix.name, :present, "already on target", nil)
+          elsif review
+            entry("index", ix.name, :needs_review, review, "-- #{review}\n-- #{ix.definition};")
+          else
+            entry("index", ix.name, :missing, ix.unique ? "unique index" : "index",
+                  "#{concurrently(ix.definition)};")
+          end
+        end
+      end
+
+      def constraint_entries(table, constraints, present, drops, renames)
+        constraints.map do |c|
+          label  = KIND_LABEL[c.kind]
+          review = review_reason(c.columns, drops, renames)
+          if present.include?(c.name)
+            entry("constraint", c.name, :present, "already on target", nil)
+          elsif review
+            entry("constraint", c.name, :needs_review, review,
+                  "-- #{review}\n-- ALTER TABLE #{Sql.quote_table(table)} " \
+                  "ADD CONSTRAINT #{Sql.quote_ident(c.name)} #{c.definition};")
+          else
+            fk_note = c.kind == "f" ? "  -- run after all referenced tables are loaded" : ""
+            ddl = "ALTER TABLE #{Sql.quote_table(table)} " \
+                  "ADD CONSTRAINT #{Sql.quote_ident(c.name)} #{c.definition};#{fk_note}"
+            entry("constraint", c.name, :missing, label, ddl)
+          end
+        end
+      end
+
+      def sequence_entries(identity_columns)
+        identity_columns.map do |col|
+          entry("sequence", col.column, :info,
+                "#{col.kind} column — restored automatically by `pcrd cutover`", nil)
+        end
+      end
+
+      def owner_entries(table, src, tgt)
+        source_owner = src.owner(table)
+        return [] unless source_owner
+
+        if source_owner == tgt.owner(table)
+          [entry("owner", source_owner, :present, "owner already #{source_owner}", nil)]
+        else
+          [entry("owner", source_owner, :missing, "set owner to #{source_owner}",
+                 "ALTER TABLE #{Sql.quote_table(table)} OWNER TO #{Sql.quote_ident(source_owner)};")]
+        end
+      end
+
+      def grant_entries(table, src, tgt)
+        target = tgt.grants(table).to_h { |g| [g.grantee, g.privileges] }
+
+        src.grants(table).map do |g|
+          have = target[g.grantee] || []
+          if (g.privileges - have).empty?
+            entry("grant", g.grantee, :present, g.privileges.join(", "), nil)
+          else
+            grantee_sql = g.grantee == "PUBLIC" ? "PUBLIC" : Sql.quote_ident(g.grantee)
+            entry("grant", g.grantee, :missing, g.privileges.join(", "),
+                  "GRANT #{g.privileges.join(', ')} ON #{Sql.quote_table(table)} TO #{grantee_sql};")
+          end
+        end
+      end
+
+      # Comments are rename-safe: only the column identifier changes, so a
+      # renamed column's comment is re-emitted against its target name; dropped
+      # columns are skipped.
+      def comment_entries(table, src, tgt, drops, renames)
+        entries = []
+
+        source_table_comment = src.table_comment(table)
+        if source_table_comment
+          status = source_table_comment == tgt.table_comment(table) ? :present : :missing
+          ddl    = status == :missing ? "COMMENT ON TABLE #{Sql.quote_table(table)} IS #{quote_literal(source_table_comment)};" : nil
+          entries << entry("comment", "(table)", status, truncate(source_table_comment), ddl)
+        end
+
+        target_comments = tgt.column_comments(table)
+        src.column_comments(table).each do |col, comment|
+          next if drops.include?(col)
+
+          target_col = renames[col] || col
+          if target_comments[target_col] == comment
+            entries << entry("comment", target_col, :present, truncate(comment), nil)
+          else
+            entries << entry("comment", target_col, :missing, truncate(comment),
+                             "COMMENT ON COLUMN #{Sql.quote_table(table)}.#{Sql.quote_ident(target_col)} " \
+                             "IS #{quote_literal(comment)};")
+          end
+        end
+
+        entries
+      end
+
+      def quote_literal(str)
+        "'#{str.gsub("'", "''")}'"
+      end
+
+      def truncate(str)
+        str.length > 40 ? "#{str[0, 37]}..." : str
+      end
+
+      # Injects CONCURRENTLY so the index build does not lock out writes.
+      def concurrently(definition)
+        definition.sub(/\ACREATE (UNIQUE )?INDEX /, 'CREATE \1INDEX CONCURRENTLY ')
+      end
+
+      def review_reason(columns, drops, renames)
+        dropped = columns & drops
+        return "references dropped column(s): #{dropped.join(', ')} — not recreated" if dropped.any?
+
+        renamed = columns & renames.keys
+        if renamed.any?
+          pairs = renamed.map { |c| "#{c}->#{renames[c]}" }.join(", ")
+          return "references renamed column(s): #{pairs} — regenerate manually"
+        end
+
+        nil
+      end
+
+      def change_maps(table_config)
+        cols    = table_config.columns || {}
+        drops   = cols.select { |_, spec| spec.drop }.keys.map(&:to_s)
+        renames = cols.each_with_object({}) do |(src, spec), h|
+          h[src.to_s] = spec.rename if spec.rename
+        end
+        [drops, renames]
+      end
+
+      def entry(category, name, status, detail, ddl)
+        Entry.new(category: category, name: name, status: status, detail: detail, ddl: ddl)
+      end
+    end
+  end
+end

data/lib/pcrd/replication/consumer.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Replication
+    # Streams WAL messages from a pgoutput logical replication slot and
+    # buffers complete transactions onto a Thread::Queue for the apply engine.
+    #
+    # Protocol (inside each raw message from the server):
+    #   0x77 ('w') — XLogData: 1 type + 8 wal_start + 8 wal_end + 8 ts + payload
+    #   0x6B ('k') — Primary keepalive: 1 type + 8 wal_end + 8 ts + 1 reply_flag
+    #
+    # The consumer must respond to keepalives with a StandbyStatusUpdate ('r')
+    # within wal_sender_timeout (default 60s) or the server drops the connection.
+    #
+    # Thread model:
+    #   start       — launches a background thread that drives the stream loop
+    #   stop        — signals the thread to exit cleanly after the current poll
+    #   queue       — Thread::Queue; pop from the apply engine side
+    #   advance_lsn — call from apply engine after each applied transaction
+    class Consumer
+      # A complete buffered transaction ready for the apply engine.
+      Transaction = Data.define(:begin_msg, :events, :commit_lsn)
+
+      XLOG_DATA          = 0x77
+      KEEPALIVE          = 0x6B
+      PG_EPOCH_OFFSET_US = 946_684_800 * 1_000_000
+      KEEPALIVE_INTERVAL = 10   # seconds between proactive keepalives to server
+      WAIT_TIMEOUT       = 1    # max seconds per wait_readable; limits stop latency
+
+      # Backpressure cap. The queue holds at most this many buffered
+      # transactions; once full, the stream loop stops reading new WAL (the
+      # server's flow control kicks in) until the apply side drains it. This
+      # bounds memory during a long backfill instead of letting the queue grow
+      # without limit. Tuned via :max_queue.
+      DEFAULT_MAX_QUEUE  = 10_000
+      FULL_QUEUE_BACKOFF = 0.05 # seconds to wait between retries when full
+
+      def initialize(repl_conn:, parser:, slot_name:, pub_name:, start_lsn: "0/0",
+                     max_queue: DEFAULT_MAX_QUEUE)
+        @repl      = repl_conn
+        @parser    = parser
+        @slot_name = slot_name
+        @pub_name  = pub_name
+        @start_lsn = start_lsn
+        @queue     = SizedQueue.new(max_queue)
+        @stop      = false
+        @mutex     = Mutex.new
+        @conf_lsn  = 0   # last applied LSN (int64); advanced by apply engine
+        @last_received_lsn = nil # commit LSN of the most recently buffered txn
+        @thread    = nil
+      end
+
+      attr_reader :queue, :parser, :last_error
+
+      # Commit LSN of the most recent transaction buffered onto the queue, for
+      # observability ("how far has streaming read?"). nil until the first txn.
+      def last_received_lsn
+        @mutex.synchronize { @last_received_lsn }
+      end
+
+      # Number of buffered transactions waiting to be applied (backpressure gauge).
+      def queue_depth
+        @queue.size
+      end
+
+      # Opens the replication connection and starts the background thread.
+      def start
+        @repl.open
+        @repl.start_replication(slot_name: @slot_name, pub_name: @pub_name, start_lsn: @start_lsn)
+        @thread = Thread.new { stream_loop }
+        self
+      end
+
+      # Signals the consumer to stop after the current poll cycle.
+      def stop
+        @mutex.synchronize { @stop = true }
+        @thread&.join(5)
+        @repl.close
+      end
+
+      def stopped?
+        @mutex.synchronize { @stop }
+      end
+
+      # True if the streaming thread exited because of an error. The apply
+      # side polls this when the queue drains empty so a dead consumer
+      # surfaces as a failure instead of looking like "caught up and idle".
+      def failed?
+        @mutex.synchronize { !@last_error.nil? }
+      end
+
+      # Called by the apply engine after a transaction has been applied.
+      # Updates the LSN we report back to the server (WAL reclaim point).
+      def advance_lsn(lsn_string)
+        int = lsn_to_int(lsn_string)
+        @mutex.synchronize { @conf_lsn = [@conf_lsn, int].max }
+      end
+
+      private
+
+      def stream_loop
+        @current_begin  = nil
+        @current_events = []
+        last_keepalive  = monotonic
+
+        loop do
+          # Short wait so stop! is noticed within WAIT_TIMEOUT seconds.
+          break if stopped?
+
+          if @repl.wait_readable(WAIT_TIMEOUT)
+            # Drain all messages available right now in one pass.
+            loop do
+              raw = @repl.get_copy_data
+              break if raw.nil? || raw == false
+              dispatch(raw)
+            end
+          end
+
+          # Send proactive keepalive if nothing has been received recently.
+          if monotonic - last_keepalive >= KEEPALIVE_INTERVAL
+            send_status
+            last_keepalive = monotonic
+          end
+        end
+      rescue => e
+        # Record the failure and let the thread exit. We deliberately do NOT
+        # enqueue anything: a malformed transaction here would be applied as a
+        # no-op and its sentinel "LSN" checkpointed, hiding the failure. The
+        # apply loop detects the dead consumer via #failed? once the queue
+        # drains and raises Replication::Error.
+        @mutex.synchronize { @last_error = e }
+      end
+
+      def dispatch(raw)
+        tag = raw.getbyte(0)
+
+        case tag
+        when XLOG_DATA
+          # Header: 1 tag + 8 wal_start + 8 wal_end + 8 ts = 25 bytes
+          payload = raw.b[25..]
+          handle_pgoutput(@parser.parse(payload))
+
+        when KEEPALIVE
+          # Bytes 1-8: wal_end; byte 17: reply_requested
+          reply_needed = raw.getbyte(17) == 1
+          send_status if reply_needed
+        end
+      end
+
+      def handle_pgoutput(msg)
+        case msg
+        when Pgoutput::Messages::Begin
+          @current_begin  = msg
+          @current_events = []
+
+        when Pgoutput::Messages::Insert,
+             Pgoutput::Messages::Update,
+             Pgoutput::Messages::Delete
+          @current_events << msg
+
+        when Pgoutput::Messages::Commit
+          unless @current_events.empty?
+            enqueue(Transaction.new(
+              begin_msg:  @current_begin,
+              events:     @current_events.dup,
+              commit_lsn: msg.lsn
+            ))
+          end
+          @current_begin  = nil
+          @current_events = []
+
+        when Pgoutput::Messages::Truncate
+          reject_truncate(msg)
+
+        # Relation and Type are cached by the parser; no action needed here.
+        end
+      end
+
+      # pcrd does not replicate TRUNCATE: silently ignoring it would leave the
+      # target diverged from the source with no signal. Halt loudly so the
+      # operator can truncate the target deliberately and resume. The
+      # publication only covers migrated tables, so any TRUNCATE is relevant.
+      def reject_truncate(msg)
+        names = msg.relation_ids.map do |id|
+          (r = @parser.relation(id)) ? "#{r.namespace}.#{r.name}" : "oid:#{id}"
+        end
+        raise Error,
+              "TRUNCATE received for #{names.join(", ")}. pcrd does not replicate " \
+              "TRUNCATE; the target would silently diverge. Migration halted — " \
+              "truncate the target manually if intended, then resume."
+      end
+
+      # Pushes a transaction onto the bounded queue. If the queue is full the
+      # apply side is behind, so we wait — but keep answering the server with
+      # keepalives so it does not drop us past wal_sender_timeout while we
+      # apply backpressure. Returns early if stop is requested.
+      def enqueue(txn)
+        loop do
+          return if stopped?
+
+          begin
+            @queue.push(txn, true) # non-blocking; raises ThreadError when full
+            @mutex.synchronize { @last_received_lsn = txn.commit_lsn }
+            return
+          rescue ThreadError
+            send_status
+            sleep FULL_QUEUE_BACKOFF
+          end
+        end
+      end
+
+      # Sends a StandbyStatusUpdate to keep the connection alive and
+      # tell the server which LSN we have confirmed applying.
+      #   'r' tag + write_lsn (8) + flush_lsn (8) + apply_lsn (8) + ts (8) + reply (1)
+      def send_status
+        lsn = @mutex.synchronize { @conf_lsn }
+        now = ((Time.now.to_f * 1_000_000).to_i - PG_EPOCH_OFFSET_US)
+        msg = "r".b + [lsn, lsn, lsn, now, 0].pack("Q>Q>Q>q>C")
+        @repl.put_copy_data(msg)
+      rescue Connection::Error
+        nil  # connection may be closing; ignore
+      end
+
+      def lsn_to_int(str)
+        return 0 unless str&.include?("/")
+        high, low = str.split("/").map { _1.to_i(16) }
+        (high << 32) | low
+      end
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/pcrd/replication/error.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Replication
+    # Raised when the WAL streaming consumer stops unexpectedly (e.g. the
+    # replication connection drops or pgoutput parsing fails). Carries the
+    # original error as #cause when available.
+    class Error < Pcrd::Error; end
+  end
+end

data/lib/pcrd/replication/pgoutput/messages.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Replication
+    module Pgoutput
+      # Immutable structs for each pgoutput message type.
+      # Produced by Parser#parse; consumed by the WAL consumer (Phase 9).
+      #
+      # Tuple data (new_tuple / old_tuple) is a Hash<column_name, value> where:
+      #   value = String  — column value in text format
+      #   value = nil     — SQL NULL
+      #   value = :toast  — unchanged TOASTed value (not re-sent by server)
+      module Messages
+        # One column description inside a Relation message.
+        RelationColumn = Data.define(
+          :flags,         # Integer: bit 0 set = part of replica identity key
+          :name,          # String: column name
+          :type_id,       # Integer: OID of the column data type
+          :type_modifier  # Integer: atttypmod (-1 means no modifier)
+        )
+
+        # B — transaction begin
+        # lsn: String "X/Y" — final LSN of the transaction
+        # commit_time: Time (UTC)
+        # xid: Integer — transaction ID
+        Begin = Data.define(:lsn, :commit_time, :xid)
+
+        # C — transaction commit
+        # lsn: String "X/Y" — commit LSN
+        # end_lsn: String "X/Y" — LSN after the end of the transaction record
+        # commit_time: Time (UTC)
+        Commit = Data.define(:flags, :lsn, :end_lsn, :commit_time)
+
+        # R — relation (table schema snapshot)
+        # id: Integer — relation OID
+        # namespace: String — schema name (empty for pg_catalog)
+        # name: String — table name
+        # replica_identity: String — one of 'd', 'n', 'f', 'i'
+        # columns: Array<RelationColumn>
+        Relation = Data.define(:id, :namespace, :name, :replica_identity, :columns)
+
+        # T — data type
+        Type = Data.define(:id, :namespace, :name)
+
+        # I — INSERT
+        Insert = Data.define(:relation_id, :new_tuple)
+
+        # U — UPDATE
+        # old_tuple is nil unless REPLICA IDENTITY is FULL or INDEX
+        Update = Data.define(:relation_id, :old_tuple, :new_tuple)
+
+        # D — DELETE
+        # old_tuple contains either the key columns or all columns depending on REPLICA IDENTITY
+        Delete = Data.define(:relation_id, :old_tuple)
+
+        # O — origin (the replication origin this transaction came from)
+        Origin = Data.define(:lsn, :name)
+
+        # A — TRUNCATE (PG 11+)
+        # option_bits: Integer — 1 = CASCADE, 2 = RESTART IDENTITY
+        Truncate = Data.define(:option_bits, :relation_ids)
+
+        # M — logical decoding message (PG 14+)
+        LogicalMessage = Data.define(:flags, :lsn, :prefix, :content)
+      end
+    end
+  end
+end