pcrd 0.1.0

data/lib/pcrd/config/loader.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Pcrd
+  module Config
+    MIGRATE_DEFAULTS = {
+      batch_size: 10_000,
+      lag_threshold_bytes: 1_048_576,        # 1 MB
+      checkpoint_db: "./pcrd_checkpoint.sqlite3"
+    }.freeze
+
+    VERIFY_DEFAULTS  = { sample_size: 1_000 }.freeze
+    CUTOVER_DEFAULTS = { sequence_buffer: 1_000, lag_drain_timeout: 300 }.freeze
+
+    class Loader
+      # Returns a Config::Root. Raises Config::LoadError on any problem.
+      def self.load(path)
+        new(path).load
+      end
+
+      def initialize(path)
+        @path = path
+      end
+
+      def load
+        raw  = read_file
+        data = parse_yaml(raw)
+        validate!(data)
+        build(data)
+      end
+
+      private
+
+      def read_file
+        File.read(@path)
+      rescue Errno::ENOENT
+        raise LoadError, "Config file not found: #{@path}"
+      rescue Errno::EACCES
+        raise LoadError, "Cannot read config file (permission denied): #{@path}"
+      end
+
+      def parse_yaml(raw)
+        YAML.safe_load(raw, symbolize_names: true)
+      rescue Psych::SyntaxError => e
+        raise LoadError, "Config file has invalid YAML: #{e.message}"
+      end
+
+      def validate!(data)
+        result = Schema::DEFINITION.call(data)
+        return if result.success?
+
+        messages = result.errors.messages.map do |msg|
+          "  #{msg.path.join(".")}: #{msg.text}"
+        end
+        raise LoadError, "Config file is invalid:\n#{messages.join("\n")}"
+      end
+
+      def build(data)
+        Root.new(
+          source:   build_connection(data[:source], env_prefix: "SOURCE"),
+          target:   data[:target] ? build_connection(data[:target], env_prefix: "TARGET") : nil,
+          migrate:  data[:migrate] ? build_migrate(data[:migrate]) : nil,
+          analyze:  data[:analyze] ? build_analyze(data[:analyze]) : nil,
+          verify:   data[:verify]  ? build_verify(data[:verify])   : nil,
+          cutover:  data[:cutover] ? build_cutover(data[:cutover]) : nil,
+          path:     @path
+        )
+      end
+
+      def build_connection(raw, env_prefix:)
+        password = raw[:password] ||
+                   ENV["PCRD_#{env_prefix}_PASSWORD"] ||
+                   nil  # falls back to .pgpass / PGPASSWORD at connection time
+        Connection.new(
+          host:     raw[:host],
+          port:     raw.fetch(:port, 5432),
+          database: raw[:database],
+          user:     raw[:user],
+          password: password
+        )
+      end
+
+      def build_migrate(raw)
+        slot_base = derive_slot_base(raw[:tables])
+        MigrateConfig.new(
+          replication_slot:    raw.fetch(:replication_slot, "pcrd_#{slot_base}"),
+          publication:         raw.fetch(:publication,        "pcrd_pub_#{slot_base}"),
+          checkpoint_db:       raw.fetch(:checkpoint_db,      MIGRATE_DEFAULTS[:checkpoint_db]),
+          batch_size:          raw.fetch(:batch_size,          MIGRATE_DEFAULTS[:batch_size]),
+          lag_threshold_bytes: raw.fetch(:lag_threshold_bytes, MIGRATE_DEFAULTS[:lag_threshold_bytes]),
+          max_rows_per_second: raw[:max_rows_per_second],
+          tables:              (raw[:tables] || []).map { build_table(_1) }
+        )
+      end
+
+      def build_table(raw)
+        Table.new(
+          name:                  raw[:name],
+          optimize_column_order: raw.fetch(:optimize_column_order, false),
+          columns:               build_column_specs(raw[:columns] || {}),
+          add_columns:           (raw[:add_columns] || []).map { build_add_column(_1) }
+        )
+      end
+
+      def build_column_specs(raw_columns)
+        raw_columns.transform_keys(&:to_s).transform_values do |spec|
+          spec ||= {}
+          validate_column_spec!(spec)
+          ColumnSpec.new(
+            type:   spec[:type]&.to_s,
+            rename: spec[:rename]&.to_s,
+            drop:   spec.fetch(:drop, false)
+          )
+        end
+      end
+
+      def validate_column_spec!(spec)
+        return unless spec[:drop] && (spec[:type] || spec[:rename])
+
+        raise LoadError,
+              "A column spec cannot combine `drop: true` with `type` or `rename`"
+      end
+
+      def build_add_column(raw)
+        AddColumn.new(
+          name:    raw[:name],
+          type:    raw[:type],
+          default: raw[:default]
+        )
+      end
+
+      def build_analyze(raw)
+        AnalyzeConfig.new(tables: raw[:tables]&.map(&:to_s))
+      end
+
+      def build_verify(raw)
+        VerifyConfig.new(
+          sample_size: raw.fetch(:sample_size, VERIFY_DEFAULTS[:sample_size])
+        )
+      end
+
+      def build_cutover(raw)
+        CutoverConfig.new(
+          sequence_buffer:    raw.fetch(:sequence_buffer,    CUTOVER_DEFAULTS[:sequence_buffer]),
+          lag_drain_timeout:  raw.fetch(:lag_drain_timeout,  CUTOVER_DEFAULTS[:lag_drain_timeout])
+        )
+      end
+
+      # Derives a short stable name for the replication slot / publication
+      # from the first table name when not explicitly configured.
+      def derive_slot_base(tables)
+        first = tables&.first&.dig(:name)
+        first ? first.gsub(/\W/, "_").downcase : "migration"
+      end
+    end
+  end
+end

data/lib/pcrd/config/migrate_config.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    MigrateConfig = Data.define(
+      :replication_slot,
+      :publication,
+      :checkpoint_db,
+      :batch_size,
+      :lag_threshold_bytes,
+      :tables,
+      :max_rows_per_second
+    ) do
+      # max_rows_per_second is optional (nil = unthrottled). Defaulting it here
+      # keeps existing callers and configs without the key working.
+      def initialize(max_rows_per_second: nil, **rest)
+        super(max_rows_per_second: max_rows_per_second, **rest)
+      end
+    end
+  end
+end

data/lib/pcrd/config/root.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    # Top-level config object returned by Config::Loader.load.
+    # target, migrate, analyze, verify, cutover are all optional (may be nil).
+    Root = Data.define(:source, :target, :migrate, :analyze, :verify, :cutover, :path)
+  end
+end

data/lib/pcrd/config/schema.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "dry-schema"
+
+module Pcrd
+  module Config
+    module Schema
+    # Validates a YAML config hash (already natively typed by YAML.safe_load).
+    # Uses Dry::Schema.define (no coercion) since YAML gives us real Ruby types.
+    DEFINITION = Dry::Schema.define do
+      required(:source).hash do
+        required(:host).filled(:string)
+        optional(:port).value(:integer, gt?: 0, lt?: 65_536)
+        required(:database).filled(:string)
+        required(:user).filled(:string)
+        optional(:password).maybe(:string)
+      end
+
+      optional(:target).hash do
+        required(:host).filled(:string)
+        optional(:port).value(:integer, gt?: 0, lt?: 65_536)
+        required(:database).filled(:string)
+        required(:user).filled(:string)
+        optional(:password).maybe(:string)
+      end
+
+      optional(:migrate).hash do
+        optional(:replication_slot).filled(:string)
+        optional(:publication).filled(:string)
+        optional(:checkpoint_db).filled(:string)
+        optional(:batch_size).value(:integer, gt?: 0)
+        optional(:lag_threshold_bytes).value(:integer, gt?: 0)
+        optional(:tables).array(:hash) do
+          required(:name).filled(:string)
+          optional(:optimize_column_order).value(:bool)
+          # columns: dynamic keys (column names) — validated structurally in Loader
+          optional(:columns).value(:hash)
+          optional(:add_columns).array(:hash) do
+            required(:name).filled(:string)
+            required(:type).filled(:string)
+            optional(:default).maybe(:string)
+          end
+        end
+      end
+
+      optional(:analyze).hash do
+        # nil means "use tables from migrate section"
+        optional(:tables).array(:string)
+      end
+
+      optional(:verify).hash do
+        optional(:sample_size).value(:integer, gt?: 0)
+      end
+
+      optional(:cutover).hash do
+        optional(:sequence_buffer).value(:integer, gteq?: 0)
+        optional(:lag_drain_timeout).value(:integer, gt?: 0)
+      end
+    end
+    end  # module Schema
+  end
+end

data/lib/pcrd/config/table.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    # columns:     Hash<source_column_name, ColumnSpec>
+    # add_columns: Array<AddColumn>
+    Table = Data.define(:name, :optimize_column_order, :columns, :add_columns)
+  end
+end

data/lib/pcrd/config/verify_config.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    VerifyConfig = Data.define(:sample_size)
+  end
+end

data/lib/pcrd/config.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    DEFAULT_CONFIG_FILE = "pcrd.config.yml"
+  end
+end

data/lib/pcrd/connection/client.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Pcrd
+  module Connection
+    class Client
+      # Conservative per-session defaults applied to every connection.
+      #
+      #   application_name                     — identifies pcrd in pg_stat_activity
+      #   lock_timeout=5s                      — fail fast instead of blocking
+      #                                          production behind a lock (DDL etc.)
+      #   idle_in_transaction_session_timeout  — release locks if a transaction is
+      #     =60s                                 left open idle (e.g. a stalled tool)
+      #   statement_timeout=0                  — DISABLED on purpose: backfill COPY
+      #                                          and large batches run for a long
+      #                                          time and must not be killed
+      #
+      # Override per pool via `settings:`; values are GUC strings (units allowed).
+      DEFAULT_SESSION_SETTINGS = {
+        "application_name"                    => "pcrd",
+        "lock_timeout"                        => "5s",
+        "idle_in_transaction_session_timeout" => "60s",
+        "statement_timeout"                   => "0"
+      }.freeze
+
+      attr_reader :session_settings
+
+      def initialize(config, settings: {})
+        @config           = config
+        @session_settings = DEFAULT_SESSION_SETTINGS.merge(settings)
+        @conn             = nil
+      end
+
+      # For parameterized queries (SELECT, INSERT with $1 placeholders).
+      def exec(sql, params = [])
+        connection.exec_params(sql, params)
+      rescue PG::Error => e
+        reset_connection!
+        raise Error, e.message
+      end
+
+      # For DDL and multi-statement SQL where no parameter substitution is needed.
+      def exec_sql(sql)
+        connection.exec(sql)
+      rescue PG::Error => e
+        reset_connection!
+        raise Error, e.message
+      end
+
+      # For COPY ... FROM STDIN. Yields the raw PG::Connection so the caller
+      # can call conn.put_copy_data(line) inside the block.
+      def copy_data(sql)
+        connection.copy_data(sql) { yield connection }
+      rescue PG::Error => e
+        # A failed COPY can leave the connection mid-COPY or in an aborted
+        # transaction; reset it like exec/exec_sql so the next call is usable.
+        reset_connection!
+        raise Error, e.message
+      end
+
+      def quote_ident(name)
+        connection.quote_ident(name)
+      end
+
+      def escape_literal(val)
+        connection.escape_literal(val.to_s)
+      end
+
+      def transaction
+        exec("BEGIN")
+        result = yield
+        exec("COMMIT")
+        result
+      rescue StandardError
+        exec("ROLLBACK") rescue nil
+        raise
+      end
+
+      def close
+        @conn&.close
+        @conn = nil
+      end
+
+      def connected?
+        @conn && !@conn.finished?
+      end
+
+      # libpq options string that applies the session settings at connect time
+      # (-c key=value), so they are in force for the very first statement.
+      # application_name is excluded here — it is passed as the dedicated
+      # connect parameter because a -c value is overridden by libpq's
+      # fallback_application_name.
+      def session_options
+        @session_settings
+          .reject { |key, _| key == "application_name" }
+          .map    { |key, value| "-c #{key}=#{value}" }
+          .join(" ")
+      end
+
+      private
+
+      def connection
+        @conn = connect unless connected?
+        @conn
+      end
+
+      def connect
+        PG.connect(
+          host:    @config.host,
+          port:    @config.port,
+          dbname:  @config.database,
+          user:    @config.user,
+          password: @config.password,
+          application_name: @session_settings["application_name"],
+          options: session_options
+        )
+      rescue PG::ConnectionBad => e
+        raise Error, "Cannot connect to #{@config.host}:#{@config.port}/#{@config.database}: #{e.message}"
+      end
+
+      def reset_connection!
+        # If the connection is in an aborted transaction, attempt a rollback
+        # so subsequent commands on the same connection can proceed.
+        @conn&.exec("ROLLBACK") rescue nil
+      end
+    end
+  end
+end

data/lib/pcrd/connection/error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Connection
+    class Error < Pcrd::Error; end
+  end
+end

data/lib/pcrd/connection/replication.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Pcrd
+  module Connection
+    # Manages a PostgreSQL logical replication connection.
+    #
+    # Opened with replication: 'database' so the server accepts streaming
+    # replication protocol commands. Use open → start_replication, then
+    # poll with get_copy_data / respond with put_copy_data.
+    class Replication
+      # START_REPLICATION is replication-protocol SQL, not ordinary SQL: the
+      # slot name and LSN are interpolated as bare tokens, so they must be
+      # validated rather than quoted. Slot names follow PostgreSQL's own rule
+      # (lowercase letters, digits, underscore; max 63). LSN is the standard
+      # hex/hex form. Both are config/checkpoint-derived, so validate them.
+      SLOT_NAME_RE = /\A[a-z0-9_]{1,63}\z/
+      LSN_RE       = %r{\A[0-9A-Fa-f]{1,8}/[0-9A-Fa-f]{1,8}\z}
+
+      def initialize(config)
+        @config = config
+        @conn   = nil
+      end
+
+      def open
+        @conn = PG.connect(
+          host:             @config.host,
+          port:             @config.port,
+          dbname:           @config.database,
+          user:             @config.user,
+          password:         @config.password,
+          application_name: "pcrd-replication",
+          replication:      "database"
+        )
+        self
+      rescue PG::ConnectionBad, PG::Error => e
+        raise Error, "Replication connection failed to " \
+                     "#{@config.host}:#{@config.port}/#{@config.database}: #{e.message}"
+      end
+
+      # Sends START_REPLICATION and enters COPY streaming mode.
+      # Uses send_query + get_result (not exec) so the CopyBoth response is
+      # handled correctly and the connection is left in streaming copy mode.
+      def start_replication(slot_name:, pub_name:, start_lsn: "0/0")
+        validate_slot_name!(slot_name)
+        validate_lsn!(start_lsn)
+        pub_id = pub_name.gsub("'", "''")
+
+        @conn.send_query(
+          "START_REPLICATION SLOT #{slot_name} LOGICAL #{start_lsn} " \
+          "(proto_version '1', publication_names '#{pub_id}')"
+        )
+        @conn.get_result   # reads CopyBothResponse; puts connection in copy mode
+        self
+      rescue PG::Error => e
+        raise Error, "START_REPLICATION failed: #{e.message}"
+      end
+
+      # Waits up to `timeout` seconds for data on the replication socket.
+      # Returns true if data is available, false if the timeout expired.
+      def wait_readable(timeout)
+        @conn.socket_io.wait_readable(timeout)
+      end
+
+      # Returns a String (message bytes), nil (no data yet), or false (stream ended).
+      # Call after wait_readable returns true, or after consume_input.
+      def get_copy_data
+        @conn.consume_input
+        @conn.get_copy_data(true)
+      rescue PG::Error => e
+        raise Error, e.message
+      end
+
+      # Sends a client message (keepalive response) to the server.
+      def put_copy_data(data)
+        @conn.put_copy_data(data)
+      rescue PG::Error => e
+        raise Error, e.message
+      end
+
+      def close
+        @conn&.finish
+        @conn = nil
+      end
+
+      def connected?
+        @conn && !@conn.finished?
+      end
+
+      private
+
+      def validate_slot_name!(slot_name)
+        return if slot_name.to_s.match?(SLOT_NAME_RE)
+
+        raise Error,
+              "Invalid replication slot name #{slot_name.inspect}: must be 1-63 " \
+              "characters of lowercase letters, digits, or underscores."
+      end
+
+      def validate_lsn!(lsn)
+        return if lsn.to_s.match?(LSN_RE)
+
+        raise Error, "Invalid start LSN #{lsn.inspect}: expected hex/hex form like \"0/0\"."
+      end
+    end
+  end
+end

data/lib/pcrd/cutover/orchestrator.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Cutover
+    # Orchestrates the cutover sequence.
+    #
+    # Preconditions (operator's responsibility):
+    #   - Application is in maintenance mode (writes to source have stopped)
+    #   - pcrd migrate is running in streaming mode (or was cleanly stopped)
+    #
+    # Steps:
+    #   1. Verify the migration is at a cuttable phase (backfill complete)
+    #   2. Drain remaining replication lag to zero (with timeout)
+    #   3. Advance sequences on target
+    #   4. Verify row counts match
+    #   5. Print cutover report and "READY" signal
+    class Orchestrator
+      Result = Data.define(
+        :passed,
+        :row_counts,        # Hash<table_name, {source:, target:}>
+        :sequence_results,  # Array<Sequences::SequenceResult>
+        :lag_at_cutover,    # Integer bytes
+        :warnings           # Array<String>
+      )
+
+      def initialize(source_pool:, target_pool:, config:)
+        @source  = source_pool
+        @target  = target_pool
+        @config  = config
+      end
+
+      # Runs the full cutover sequence.
+      # on_progress: optional Proc called with a status string during drain
+      def run(on_progress: nil)
+        warnings   = []
+        table_names = (@config.migrate&.tables || []).map(&:name)
+
+        # 1. Drain remaining lag
+        on_progress&.call("Draining replication lag...")
+        lag = drain_lag(table_names, on_progress: on_progress)
+
+        # 2. Advance sequences
+        on_progress&.call("Advancing target sequences...")
+        seq_results = Sequences.new(
+          source_pool:   @source,
+          target_pool:   @target,
+          safety_buffer: @config.cutover&.sequence_buffer || 1_000
+        ).advance(table_names)
+
+        # 3. Row count verification
+        on_progress&.call("Verifying row counts...")
+        row_counts = verify_counts(table_names, warnings)
+
+        passed = row_counts.all? { |_, v| v[:source] == v[:target] }
+
+        Result.new(
+          passed:           passed,
+          row_counts:       row_counts,
+          sequence_results: seq_results,
+          lag_at_cutover:   lag,
+          warnings:         warnings
+        )
+      end
+
+      private
+
+      def drain_lag(table_names, on_progress:)
+        slot_name = @config.migrate&.replication_slot
+        return 0 unless slot_name
+
+        timeout    = @config.cutover&.lag_drain_timeout || 300
+        deadline   = Time.now + timeout
+        lag_monitor = Monitor::Lag.new(source_pool: @source, slot_name: slot_name)
+
+        loop do
+          lag = lag_monitor.lag_bytes
+          on_progress&.call("  Lag: #{lag ? "#{lag} bytes" : "unknown"}")
+
+          return lag || 0 if lag&.zero?
+          return lag || 0 if !lag  # slot may have been dropped
+
+          if Time.now > deadline
+            on_progress&.call("  Warning: lag did not reach zero within #{timeout}s (#{lag} bytes remaining)")
+            return lag
+          end
+
+          sleep 1
+        end
+      end
+
+      def verify_counts(table_names, warnings)
+        table_names.each_with_object({}) do |name, counts|
+          src_count = @source.exec("SELECT COUNT(*) FROM #{@source.quote_ident(name)}")[0]["count"].to_i
+          tgt_count = @target.exec("SELECT COUNT(*) FROM #{@target.quote_ident(name)}")[0]["count"].to_i
+
+          counts[name] = { source: src_count, target: tgt_count }
+
+          if src_count != tgt_count
+            warnings << "#{name}: row count mismatch (source=#{src_count}, target=#{tgt_count})"
+          end
+        rescue Connection::Error => e
+          warnings << "#{name}: could not verify row count: #{e.message}"
+          counts[name] = { source: nil, target: nil }
+        end
+      end
+    end
+  end
+end