pcrd 0.1.0

data/lib/pcrd/checkpoint/store.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "json"
+
+module Pcrd
+  module Checkpoint
+    # SQLite-backed store for migration progress.
+    #
+    # Tracks two things:
+    #   1. Metadata (phase, LSN watermark, start time) — key/value rows
+    #   2. Completed batches — one row per successfully copied batch,
+    #      with start/end key, row count, duration, and timestamp.
+    #
+    # The per-batch log is what makes resumption safe and auditable:
+    # on resume, `last_completed_key` returns the highest end_key and
+    # the backfill skips straight past it. It also powers throughput
+    # stats and ETA estimates.
+    class Store
+      # A PostgreSQL LSN is two hex segments joined by a slash, e.g. "16/B374D848".
+      LSN_FORMAT = /\A[0-9A-Fa-f]+\/[0-9A-Fa-f]+\z/
+
+      SCHEMA_SQL = <<~SQL.freeze
+        CREATE TABLE IF NOT EXISTS metadata (
+          key   TEXT PRIMARY KEY,
+          value TEXT NOT NULL
+        );
+
+        CREATE TABLE IF NOT EXISTS batches (
+          id           INTEGER PRIMARY KEY AUTOINCREMENT,
+          table_name   TEXT    NOT NULL,
+          start_key    TEXT    NOT NULL,
+          end_key      TEXT    NOT NULL,
+          row_count    INTEGER NOT NULL,
+          duration_ms  INTEGER NOT NULL,
+          completed_at TEXT    NOT NULL
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_batches_table
+          ON batches (table_name, id DESC);
+      SQL
+
+      def initialize(path)
+        @path = path
+        @db   = SQLite3::Database.new(path)
+        @db.results_as_hash = true
+        @db.busy_timeout = 5_000
+        @db.execute_batch(SCHEMA_SQL)
+        # Backfill (recording batches) and the apply worker (recording LSN) hit
+        # this store from two threads at once. SQLite3::Database is a single
+        # connection, so serialize every @db access through one mutex. The lock
+        # is taken only at the lowest level (get_meta/set_meta and the batch
+        # methods) so the public wrappers never re-enter it.
+        @mutex = Mutex.new
+      end
+
+      def close
+        @mutex.synchronize { @db.close unless @db.closed? }
+      end
+
+      # ── phase & LSN metadata ─────────────────────────────────────────────
+
+      def phase
+        val = get_meta("phase")
+        val ? val.to_sym : :new
+      end
+
+      def set_phase(phase)
+        set_meta("phase", phase.to_s)
+      end
+
+      def lsn
+        get_meta("current_lsn")
+      end
+
+      def set_lsn(lsn)
+        unless lsn.is_a?(String) && lsn.match?(LSN_FORMAT)
+          raise ArgumentError, "invalid LSN: #{lsn.inspect}"
+        end
+
+        set_meta("current_lsn", lsn)
+      end
+
+      def backfill_start_lsn
+        get_meta("backfill_start_lsn")
+      end
+
+      def set_backfill_start_lsn(lsn)
+        set_meta("backfill_start_lsn", lsn)
+      end
+
+      # Replication objects this migration created, recorded so a resume can be
+      # cross-checked against the config and cleanup knows what to remove.
+      def replication_slot
+        get_meta("replication_slot")
+      end
+
+      def set_replication_slot(name)
+        set_meta("replication_slot", name)
+      end
+
+      def publication
+        get_meta("publication")
+      end
+
+      def set_publication(name)
+        set_meta("publication", name)
+      end
+
+      def started_at
+        get_meta("started_at")
+      end
+
+      def set_started_at(ts)
+        set_meta("started_at", ts)
+      end
+
+      # ── batch tracking ───────────────────────────────────────────────────
+
+      # Record a successfully completed batch.
+      # Keys are JSON-encoded to support multi-column primary keys.
+      def record_batch(table:, start_key:, end_key:, row_count:, duration_ms:)
+        @mutex.synchronize do
+          @db.execute(
+            "INSERT INTO batches (table_name, start_key, end_key, row_count, duration_ms, completed_at) " \
+            "VALUES (?, ?, ?, ?, ?, ?)",
+            [table.to_s,
+             JSON.generate(start_key),
+             JSON.generate(end_key),
+             row_count.to_i,
+             duration_ms.to_i,
+             Time.now.iso8601]
+          )
+        end
+      end
+
+      # Returns the end_key of the last completed batch for a table, decoded from JSON.
+      # Returns nil if no batches have been recorded for this table (fresh start).
+      def last_completed_key(table:)
+        row = @mutex.synchronize do
+          @db.get_first_row(
+            "SELECT end_key FROM batches WHERE table_name = ? ORDER BY id DESC LIMIT 1",
+            [table.to_s]
+          )
+        end
+        row ? JSON.parse(row["end_key"]) : nil
+      end
+
+      # Returns aggregate stats for a table's completed batches.
+      def batch_stats(table:)
+        row = @mutex.synchronize do
+          @db.get_first_row(
+            "SELECT COUNT(*) AS cnt, SUM(row_count) AS total_rows, " \
+            "AVG(CAST(row_count AS REAL) / NULLIF(duration_ms, 0) * 1000) AS avg_rps " \
+            "FROM batches WHERE table_name = ?",
+            [table.to_s]
+          )
+        end
+        {
+          batch_count:    row["cnt"].to_i,
+          total_rows:     row["total_rows"].to_i,
+          avg_rows_per_sec: row["avg_rps"]&.round(1) || 0.0
+        }
+      end
+
+      # All completed batches for a table, newest first.
+      def batches(table:, limit: 100)
+        rows = @mutex.synchronize do
+          @db.execute(
+            "SELECT * FROM batches WHERE table_name = ? ORDER BY id DESC LIMIT ?",
+            [table.to_s, limit]
+          )
+        end
+        rows.map do |row|
+          {
+            id:           row["id"].to_i,
+            table_name:   row["table_name"],
+            start_key:    JSON.parse(row["start_key"]),
+            end_key:      JSON.parse(row["end_key"]),
+            row_count:    row["row_count"].to_i,
+            duration_ms:  row["duration_ms"].to_i,
+            completed_at: row["completed_at"]
+          }
+        end
+      end
+
+      def total_rows_copied(table:)
+        row = @mutex.synchronize do
+          @db.get_first_row(
+            "SELECT COALESCE(SUM(row_count), 0) AS total FROM batches WHERE table_name = ?",
+            [table.to_s]
+          )
+        end
+        row["total"].to_i
+      end
+
+      private
+
+      def get_meta(key)
+        row = @mutex.synchronize do
+          @db.get_first_row("SELECT value FROM metadata WHERE key = ?", [key])
+        end
+        row ? row["value"] : nil
+      end
+
+      def set_meta(key, value)
+        @mutex.synchronize do
+          @db.execute(
+            "INSERT INTO metadata (key, value) VALUES (?, ?) " \
+            "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+            [key, value.to_s]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/pcrd/cli.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require "thor"
+require "pastel"
+
+module Pcrd
+  class CLI < Thor
+    PASTEL = Pastel.new
+
+    def self.exit_on_failure?
+      true
+    end
+
+    class_option :config, type: :string, aliases: "-c",
+                          desc: "Path to migration YAML config (required for all commands)"
+
+    map %w[--version -v] => :version
+    desc "--version, -v", "Show pcrd version"
+    def version
+      say "pcrd #{Pcrd::VERSION}"
+    end
+
+    desc "analyze", "Analyze column padding for source tables"
+    long_desc <<~DESC
+      Reads the source table schema and reports the current column layout alongside
+      the optimal column ordering for minimal padding waste. Estimates bytes saved
+      per row and total storage reclaimed at current row count.
+
+      With --compare-target, also connects to the target cluster and shows a
+      side-by-side diff: type changes, renames, added/dropped columns, and the
+      padding delta between source and target schemas.
+
+      This command is read-only and requires no migration to be in progress.
+    DESC
+    method_option :table, type: :string, aliases: "-t",
+                          desc: "Analyze a specific table only (default: all tables in config)"
+    method_option :"compare-target", type: :boolean, default: false,
+                  desc: "Compare source and target schemas side-by-side"
+    def analyze
+      config = load_config!
+      Commands::Analyze.new(config, options).run
+    rescue Commands::Analyze::Error => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    rescue Connection::Error => e
+      raise Thor::Error, "Connection failed: #{e.message}"
+    end
+
+    desc "migrate", "Start or resume the migration"
+    long_desc <<~DESC
+      Runs preflight checks, creates the replication slot and publication on source,
+      creates the target table with the new schema, then runs backfill and streaming
+      concurrently until the operator triggers cutover.
+
+      The process is resumable: if interrupted, re-run with --resume to pick up
+      from the last completed backfill batch.
+
+      Use --preflight-only to validate the config and source data without starting
+      the migration. Use --backfill-only to copy existing rows without starting
+      the WAL streaming consumer.
+    DESC
+    method_option :resume, type: :boolean, default: false,
+                  desc: "Resume an interrupted migration from the last checkpoint"
+    method_option :"preflight-only", type: :boolean, default: false,
+                  desc: "Run preflight checks and print DDL only; do not start migration"
+    method_option :"backfill-only", type: :boolean, default: false,
+                  desc: "Copy existing rows only; do not start WAL streaming consumer"
+    method_option :"dry-run", type: :boolean, default: false,
+                  desc: "Print preflight report and target DDL without touching either cluster"
+    method_option :yes, type: :boolean, default: false, aliases: "-y",
+                  desc: "Skip the confirmation prompt before starting migration"
+    method_option :"force-overwrite", type: :boolean, default: false,
+                  desc: "Drop and recreate target tables if they already exist"
+    def migrate
+      config         = load_config!
+      preflight_only = options[:"preflight-only"] || options[:"dry-run"]
+
+      unless preflight_only
+        raise Thor::Error,
+              "ERROR: migrate requires a 'target' section in your config.\n\n" \
+              "Use --preflight-only to validate without a target connection." if config.target.nil?
+
+        raise Thor::Error,
+              "ERROR: migrate requires a 'migrate' section in your config." if config.migrate.nil?
+      end
+
+      result = Preflight.new(config, options).run
+      Output::PreflightPrinter.new.print(result)
+
+      if preflight_only
+        exit(result.passed ? 0 : 1)
+        return
+      end
+
+      unless result.passed
+        raise Thor::Error, "Preflight failed. Fix the issue(s) above before running migrate."
+      end
+
+      unless options[:yes]
+        answer = ask("Proceed with migration? [y/N]")
+        return unless answer.strip.downcase == "y"
+      end
+
+      orchestrator = Migration::Orchestrator.new(config: config, options: options)
+      trap("INT")  { orchestrator.request_stop }
+      trap("TERM") { orchestrator.request_stop }
+      orchestrator.run
+    rescue Replication::Error => e
+      raise Thor::Error, "ERROR: #{e.message}\n\nReplication stopped. Resume with --resume once the cause is resolved."
+    rescue Connection::Error => e
+      raise Thor::Error, "Connection failed: #{e.message}"
+    rescue Pcrd::Error => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    end
+
+    desc "status", "Show current migration phase and replication lag"
+    long_desc <<~DESC
+      Reads the checkpoint database and queries pg_replication_slots to show:
+      current phase, backfill progress, replication lag in bytes and estimated
+      seconds, and whether the migration is ready for cutover.
+    DESC
+    def status
+      config = load_config!
+      Commands::Status.new(config, options).run
+    rescue Config::LoadError => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    end
+
+    desc "readiness", "Report target objects (indexes, constraints) to add before cutover"
+    long_desc <<~DESC
+      The load schema created by `pcrd migrate` is intentionally minimal — just
+      the table and its primary key — so the bulk copy is fast. This command
+      compares the source and target and reports the secondary objects that must
+      be added to the target before cutover: non-PK indexes and foreign-key,
+      unique, and check constraints. It prints runnable DDL (CREATE INDEX
+      CONCURRENTLY / ALTER TABLE ADD CONSTRAINT) for the missing ones.
+
+      Objects that reference dropped or renamed columns are flagged for manual
+      review rather than auto-generated. Sequences and identity columns are
+      listed for reference but are restored automatically by `pcrd cutover`.
+
+      Read-only; safe to run any time after backfill.
+    DESC
+    def readiness
+      config = load_config!
+      result = Commands::Readiness.new(config, options).run
+      Output::ReadinessPrinter.new.print(result)
+    rescue Connection::Error => e
+      raise Thor::Error, "Connection failed: #{e.message}"
+    rescue Pcrd::Error => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    end
+
+    desc "cutover", "Trigger the cutover sequence"
+    long_desc <<~DESC
+      Drains remaining replication lag to zero, advances sequences on the target
+      cluster, runs row-count verification, and prints a cutover report.
+
+      The application must be in maintenance mode before running this command.
+      Use --maintenance-confirmed to skip the interactive confirmation prompt.
+
+      After this command completes, update DATABASE_URL to point at the target
+      cluster and restart the application.
+    DESC
+    method_option :"maintenance-confirmed", type: :boolean, default: false,
+                  desc: "Skip interactive confirmation that the app is in maintenance mode"
+    def cutover
+      config = load_config!
+
+      unless options[:"maintenance-confirmed"]
+        say "\nThe application must be in maintenance mode before continuing."
+        say "Maintenance mode options:"
+        say "  pgBouncer:  PAUSE <database>"
+        say "  Kubernetes: kubectl scale --replicas=0 deployment/app"
+        say "  Rails:      enable maintenance middleware"
+        say ""
+        answer = ask("Is the application in maintenance mode? [y/N]")
+        return unless answer.strip.downcase == "y"
+      end
+
+      source_pool = Connection::Client.new(config.source)
+      target_pool = Connection::Client.new(config.target)
+      printer     = Output::CutoverPrinter.new
+
+      say "\nRunning cutover sequence..."
+      orchestrator = Cutover::Orchestrator.new(
+        source_pool: source_pool,
+        target_pool: target_pool,
+        config:      config
+      )
+
+      result = orchestrator.run(on_progress: ->(msg) { say "  #{msg}" })
+
+      printer.print(result)
+
+      source_pool.close
+      target_pool.close
+
+      exit(result.passed ? 0 : 1)
+    rescue Connection::Error => e
+      raise Thor::Error, "Connection failed: #{e.message}"
+    end
+
+    desc "verify", "Compare row counts and spot-check rows across clusters"
+    long_desc <<~DESC
+      Compares row counts on source and target for each migrated table, then
+      spot-checks a random sample of rows field-by-field. Reports any mismatches.
+
+      Safe to run at any point after backfill completes.
+    DESC
+    method_option :"sample-size", type: :numeric, default: 1_000,
+                  desc: "Number of rows to spot-check per table"
+    method_option :"post-cutover", type: :boolean, default: false,
+                  desc: "Post-cutover mode: compare against the now-live target cluster"
+    def verify
+      config  = load_config!
+      result  = Commands::Verify.new(config, options).run
+      Output::CutoverPrinter.new.print_verify(result)
+      exit(result.passed ? 0 : 1)
+    rescue Connection::Error => e
+      raise Thor::Error, "Connection failed: #{e.message}"
+    rescue Pcrd::Error => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    end
+
+    desc "demo SUBCOMMAND", "Set up and seed a demo database for testing and demonstration"
+    subcommand "demo", Commands::Demo
+
+    desc "cleanup", "Drop replication slot, publication, and checkpoint"
+    long_desc <<~DESC
+      Drops the replication slot and publication on the source cluster and deletes
+      the local checkpoint database. Run this after the application has been
+      successfully cut over to the target cluster and you no longer need to roll back.
+
+      With --drop-source, also drops the source tables. This is irreversible and
+      requires typing the table name to confirm.
+    DESC
+    method_option :"drop-source", type: :boolean, default: false,
+                  desc: "Also drop source tables after cleanup (irreversible; requires confirmation)"
+    def cleanup
+      config = load_config!
+      Commands::Cleanup.new(config, options).run
+    rescue Connection::Error => e
+      raise Thor::Error, "Connection failed: #{e.message}"
+    rescue Pcrd::Error => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    end
+
+    private
+
+    # Resolves the config file path and returns a loaded Config::Root.
+    # Falls back to pcrd.config.yml in the current directory if --config is omitted.
+    # Raises Thor::Error with a clear message if the file cannot be loaded.
+    def load_config!
+      path = options[:config] || default_config_path
+      Config::Loader.load(path)
+    rescue Config::LoadError => e
+      raise Thor::Error, "ERROR: #{e.message}"
+    end
+
+    def default_config_path
+      default = Config::DEFAULT_CONFIG_FILE
+      return default if File.exist?(default)
+
+      raise Thor::Error,
+            "ERROR: No config file found.\n\n" \
+            "Create pcrd.config.yml in the current directory, or pass --config path/to/config.yml\n\n" \
+            "Run `pcrd help migrate` for configuration documentation."
+    end
+
+    def require_config!
+      load_config!
+    end
+  end
+end

data/lib/pcrd/commands/analyze.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Commands
+    class Analyze
+      class Error < Pcrd::Error; end
+
+      def initialize(config, options = {})
+        @config  = config
+        @options = Options.normalize(options)
+      end
+
+      def run
+        validate_config!
+
+        source_pool = Connection::Client.new(@config.source)
+        reader      = Schema::Reader.new(source_pool)
+        packer      = Schema::Packer.new
+        printer     = Output::AnalyzePrinter.new
+
+        if compare_target?
+          run_compare(reader, packer, printer)
+        else
+          run_source_only(reader, packer, printer)
+        end
+
+        source_pool.close
+      end
+
+      private
+
+      def compare_target?
+        @options[:"compare-target"]
+      end
+
+      def run_source_only(reader, packer, printer)
+        tables_to_analyze.each do |table_name|
+          columns   = reader.read(table_name)
+          row_count = reader.estimated_row_count(table_name)
+          report    = packer.report(columns)
+
+          printer.print_table_report(
+            table_name: table_name,
+            row_count:  row_count,
+            report:     report
+          )
+        end
+      end
+
+      def run_compare(reader, packer, printer)
+        validate_compare_config!
+
+        target_pool   = Connection::Client.new(@config.target)
+        target_reader = Schema::Reader.new(target_pool)
+        differ        = Schema::Differ.new
+
+        tables_to_analyze.each do |table_name|
+          source_cols  = reader.read(table_name)
+          row_count    = reader.estimated_row_count(table_name)
+          table_config = find_table_config(table_name)
+
+          target_cols, target_is_live = resolve_target_columns(
+            table_name, table_config, target_reader, source_cols
+          )
+
+          entries = differ.diff(
+            source_columns: source_cols,
+            table_config:   table_config,
+            target_columns: target_cols
+          )
+
+          printer.print_diff_report(
+            table_name:     table_name,
+            row_count:      row_count,
+            diff_entries:   entries,
+            packer:         packer,
+            target_is_live: target_is_live
+          )
+        end
+
+        target_pool.close
+      end
+
+      # Returns [target_columns_or_nil, is_live_boolean].
+      # Prefers a live target DB if the table exists; falls back to synthesis.
+      def resolve_target_columns(table_name, table_config, target_reader, source_cols)
+        if target_reader.table_exists?(table_name)
+          [target_reader.read(table_name), true]
+        else
+          # Synthesize: differ will build target columns from source + spec.
+          [nil, false]
+        end
+      rescue Connection::Error
+        # Target DB unreachable — fall back to synthesis.
+        [nil, false]
+      end
+
+      def tables_to_analyze
+        if @options[:table]
+          [@options[:table]]
+        elsif @config.analyze&.tables&.any?
+          @config.analyze.tables
+        elsif @config.migrate&.tables&.any?
+          @config.migrate.tables.map(&:name)
+        else
+          raise Error, "Nothing to analyze. Add an 'analyze' or 'migrate' section to your " \
+                       "config, or pass --table TABLE_NAME."
+        end
+      end
+
+      def find_table_config(table_name)
+        @config.migrate&.tables&.find { |t| t.name == table_name }
+      end
+
+      def validate_config!
+        raise Error, "source connection is required for analyze" if @config.source.nil?
+      end
+
+      def validate_compare_config!
+        raise Error,
+              "--compare-target requires a 'target' section in your config" if @config.target.nil?
+      end
+    end
+  end
+end