pcrd 0.1.0

data/lib/pcrd/commands/cleanup.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "pastel"
+
+module Pcrd
+  module Commands
+    # Drops the replication publication and slot on source, and deletes the
+    # local checkpoint database.
+    #
+    # Run this after the application has been successfully migrated to the target
+    # cluster and you're confident you won't need to roll back. The source tables
+    # themselves are NOT touched unless --drop-source is passed.
+    #
+    # Timeline recommendation:
+    #   - Verify the app is healthy on the target cluster
+    #   - Wait a few days (or a week) as a rollback window
+    #   - Then run `pcrd cleanup`
+    #   - Optionally run `pcrd cleanup --drop-source` weeks later
+    class Cleanup
+      PASTEL = Pastel.new
+
+      def initialize(config, options = {})
+        @config  = config
+        @options = Options.normalize(options)
+      end
+
+      def run(output: $stdout)
+        output.puts
+        output.puts PASTEL.bold("Cleanup")
+        output.puts PASTEL.dim("─" * 60)
+        output.puts
+
+        drop_slot_and_pub(output)
+        drop_checkpoint(output)
+        drop_source_tables(output) if @options[:"drop-source"]
+
+        output.puts
+        output.puts "  #{PASTEL.green("✓")}  Cleanup complete."
+        output.puts
+      end
+
+      private
+
+      def drop_slot_and_pub(output)
+        return unless @config.source && @config.migrate
+
+        slot = @config.migrate.replication_slot
+        pub  = @config.migrate.publication
+
+        pool = Connection::Client.new(@config.source)
+
+        # Drop replication slot
+        result = pool.exec(
+          "SELECT pg_drop_replication_slot($1) " \
+          "WHERE EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = $1)",
+          [slot]
+        )
+        if result.ntuples > 0
+          output.puts "  #{PASTEL.green("✓")}  Dropped replication slot: #{slot}"
+        else
+          output.puts "  #{PASTEL.dim("·")}  Replication slot not found (already dropped): #{slot}"
+        end
+
+        # Drop publication
+        pool.exec_sql("DROP PUBLICATION IF EXISTS #{pool.quote_ident(pub)}")
+        output.puts "  #{PASTEL.green("✓")}  Dropped publication: #{pub}"
+
+        pool.close
+      rescue Connection::Error => e
+        output.puts "  #{PASTEL.yellow("⚠")}  Could not connect to source to drop slot/publication: #{e.message}"
+        output.puts "     Drop manually: SELECT pg_drop_replication_slot('#{slot}');"
+        output.puts "                    DROP PUBLICATION IF EXISTS #{pub};"
+      end
+
+      def drop_checkpoint(output)
+        path = @config.migrate&.checkpoint_db || "./pcrd_checkpoint.sqlite3"
+
+        if File.exist?(path)
+          File.delete(path)
+          output.puts "  #{PASTEL.green("✓")}  Deleted checkpoint: #{path}"
+        else
+          output.puts "  #{PASTEL.dim("·")}  Checkpoint not found (already deleted): #{path}"
+        end
+      end
+
+      def drop_source_tables(output)
+        return unless @config.source && @config.migrate
+
+        table_names = @config.migrate.tables.map(&:name)
+
+        output.puts
+        output.puts "  #{PASTEL.yellow("⚠")}  Dropping source tables: #{table_names.join(', ')}"
+        output.puts "  #{PASTEL.yellow("This is irreversible.")} Type the first table name to confirm:"
+
+        input = $stdin.gets&.strip
+        unless input == table_names.first
+          output.puts "  #{PASTEL.red("Aborted.")} No tables were dropped."
+          return
+        end
+
+        pool = Connection::Client.new(@config.source)
+        table_names.each do |name|
+          pool.exec_sql("DROP TABLE IF EXISTS public.#{pool.quote_ident(name)} CASCADE")
+          output.puts "  #{PASTEL.green("✓")}  Dropped source table: #{name}"
+        end
+        pool.close
+      rescue Connection::Error => e
+        output.puts "  #{PASTEL.red("✗")}  Failed to drop source tables: #{e.message}"
+      end
+    end
+  end
+end

data/lib/pcrd/commands/demo.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module Pcrd
+  module Commands
+    class Demo < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      class_option :config, type: :string, aliases: "-c",
+                            desc: "Path to migration YAML config (default: pcrd.config.yml)"
+
+      desc "setup", "Create demo schema on the source database"
+      long_desc <<~DESC
+        Creates three tables on the source database: users, agents, and listings.
+
+        The listings table is intentionally ordered with booleans and smallints
+        interleaved among 8-byte columns to demonstrate the padding analysis
+        feature of `pcrd analyze`.
+
+        Any existing demo tables are dropped and recreated.
+
+        If no pcrd.config.yml exists in the current directory, a sample config
+        is written automatically — edit the host/port values to match your setup.
+      DESC
+      def setup
+        config = load_config!
+        pool   = Pcrd::Connection::Client.new(config.source)
+
+        say "Connecting to #{config.source.host}:#{config.source.port}/#{config.source.database}..."
+
+        say "Dropping existing demo tables (if any)..."
+        pool.exec_sql(Pcrd::Demo::Schema::DROP_SQL)
+
+        say "Creating users table..."
+        pool.exec_sql(Pcrd::Demo::Schema::USERS_DDL)
+
+        say "Creating agents table..."
+        pool.exec_sql(Pcrd::Demo::Schema::AGENTS_DDL)
+
+        say "Creating listings table (with intentionally poor column ordering)..."
+        pool.exec_sql(Pcrd::Demo::Schema::LISTINGS_DDL)
+        pool.exec_sql(Pcrd::Demo::Schema::LISTINGS_FK_DDL)
+
+        pool.close
+
+        write_sample_config unless config_file_exists?
+
+        say ""
+        say "Done. Run `pcrd demo seed` to populate with sample data.", :green
+        say "Then run `pcrd analyze` to see the column padding analysis.", :green
+      rescue Pcrd::Connection::Error => e
+        raise Thor::Error, "Connection failed: #{e.message}"
+      rescue Pcrd::Config::LoadError => e
+        raise Thor::Error, e.message
+      end
+
+      desc "seed", "Generate sample data in the demo schema"
+      long_desc <<~DESC
+        Populates the demo tables with realistic fake data.
+
+        Generates users and agents proportional to the listing count, then
+        generates the requested number of listings referencing those agents.
+
+        The data is seeded with a fixed random seed for reproducibility — running
+        seed twice with the same --rows value produces the same rows (useful for
+        testing). Pass --seed to override.
+      DESC
+      method_option :rows, type: :numeric, default: 50_000,
+                    desc: "Number of listing rows to generate (users and agents scale proportionally)"
+      method_option :seed, type: :numeric, default: 42,
+                    desc: "Random seed for reproducible data generation"
+      def seed
+        config    = load_config!
+        pool      = Pcrd::Connection::Client.new(config.source)
+        generator = Pcrd::Demo::Generator.new(pool, seed: options[:seed])
+
+        say "Seeding demo database at #{config.source.host}/#{config.source.database}..."
+        say ""
+
+        counts = generator.generate(listing_count: options[:rows])
+
+        pool.close
+
+        say ""
+        say "Seeding complete:", :green
+        say "  users:    #{format_count(counts[:users])}"
+        say "  agents:   #{format_count(counts[:agents])}"
+        say "  listings: #{format_count(counts[:listings])}"
+        say ""
+        say "Run `pcrd analyze` to see the column padding report."
+      rescue Pcrd::Connection::Error => e
+        raise Thor::Error, "Connection failed: #{e.message}"
+      rescue Pcrd::Config::LoadError => e
+        raise Thor::Error, e.message
+      end
+
+      desc "reset", "Drop all demo tables (non-destructive: data only, not config)"
+      def reset
+        config = load_config!
+        pool   = Pcrd::Connection::Client.new(config.source)
+
+        say "Dropping demo tables on #{config.source.host}/#{config.source.database}..."
+        pool.exec_sql(Pcrd::Demo::Schema::DROP_SQL)
+        pool.close
+
+        say "Done.", :green
+      rescue Pcrd::Connection::Error => e
+        raise Thor::Error, "Connection failed: #{e.message}"
+      rescue Pcrd::Config::LoadError => e
+        raise Thor::Error, e.message
+      end
+
+      private
+
+      def load_config!
+        path = options[:config] || default_config_path
+        Pcrd::Config::Loader.load(path)
+      end
+
+      def default_config_path
+        default = Pcrd::Config::DEFAULT_CONFIG_FILE
+        return default if File.exist?(default)
+
+        # Demo setup can run without a config — we'll write one if absent.
+        # Fall back to a temporary in-memory config using defaults.
+        write_sample_config
+        default
+      end
+
+      def config_file_exists?
+        File.exist?(Pcrd::Config::DEFAULT_CONFIG_FILE)
+      end
+
+      def write_sample_config
+        path = Pcrd::Config::DEFAULT_CONFIG_FILE
+        if File.exist?(path)
+          say "  (#{path} already exists — not overwriting)"
+        else
+          File.write(path, Pcrd::Demo::Schema::SAMPLE_CONFIG)
+          say "  Wrote sample config to #{path} — edit host/port values to match your setup.", :cyan
+        end
+      end
+
+      def format_count(n)
+        n.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+      end
+    end
+  end
+end

data/lib/pcrd/commands/readiness.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Commands
+    # Builds the target-readiness manifest by comparing source and target.
+    # Read-only; does not modify either cluster.
+    class Readiness
+      def initialize(config, options = {})
+        @config  = config
+        @options = Options.normalize(options)
+      end
+
+      def run
+        raise ConfigError, "target connection required for readiness" if @config.target.nil?
+        raise ConfigError, "no tables configured" if (@config.migrate&.tables || []).empty?
+
+        source = Connection::Client.new(@config.source)
+        target = Connection::Client.new(@config.target)
+
+        result = Pcrd::Readiness::Manifest.new(
+          source_pool: source, target_pool: target, config: @config
+        ).build
+
+        source.close
+        target.close
+        result
+      end
+    end
+  end
+end

data/lib/pcrd/commands/status.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "pastel"
+
+module Pcrd
+  module Commands
+    # Displays the current migration state from the checkpoint database and
+    # (optionally) queries the live replication slot for current lag.
+    #
+    # Reads entirely from local state (checkpoint SQLite) so it works without
+    # an active connection to source or target. If source is reachable, also
+    # shows live replication lag and estimated time to cutover readiness.
+    class Status
+      PASTEL = Pastel.new
+
+      PHASE_LABELS = {
+        new:       "not started",
+        backfill:  "backfill in progress",
+        streaming: "streaming (catchup phase)",
+        cutover:   "cutover complete"
+      }.freeze
+
+      def initialize(config, options = {})
+        @config  = config
+        @options = Options.normalize(options)
+      end
+
+      def run
+        checkpoint_path = @config.migrate&.checkpoint_db || "./pcrd_checkpoint.sqlite3"
+
+        unless File.exist?(checkpoint_path)
+          puts
+          puts "  #{PASTEL.yellow("No checkpoint found at #{checkpoint_path}")}"
+          puts "  Run `pcrd migrate` to start the migration."
+          puts
+          return
+        end
+
+        store = Checkpoint::Store.new(checkpoint_path)
+        print_status(store)
+        store.close
+      end
+
+      private
+
+      def print_status(store)
+        phase   = store.phase
+        started = store.started_at
+        lsn     = store.lsn
+
+        puts
+        puts PASTEL.bold("Migration status")
+        puts PASTEL.dim("─" * 60)
+        puts
+
+        puts "  Phase:    #{PASTEL.bold(phase_label(phase))}"
+        puts "  Started:  #{started || PASTEL.dim("unknown")}"
+        puts "  LSN:      #{lsn || PASTEL.dim("none")}" if lsn
+        puts
+
+        tables = @config.migrate&.tables || []
+        if tables.any?
+          puts "  #{PASTEL.bold("Tables:")}"
+          tables.each { |t| print_table_status(store, t.name) }
+          puts
+        end
+
+        print_live_lag(store)
+      end
+
+      def print_table_status(store, table_name)
+        stats = store.batch_stats(table: table_name)
+        last_key = store.last_completed_key(table: table_name)
+
+        total_rows = stats[:total_rows]
+        batches    = stats[:batch_count]
+        rps        = stats[:avg_rows_per_sec]
+
+        if batches.zero?
+          puts "    #{PASTEL.dim("○")}  #{table_name}  #{PASTEL.dim("not started")}"
+        else
+          rps_label = rps > 0 ? "  #{PASTEL.dim("avg #{format_count(rps.to_i)} rows/sec")}" : ""
+          puts "    #{PASTEL.green("✓")}  #{table_name}  " \
+               "#{format_count(total_rows)} rows copied  " \
+               "(#{batches} batch#{batches == 1 ? '' : 'es'})#{rps_label}"
+          puts "       last key: #{last_key.inspect}" if last_key && @options[:verbose]
+        end
+      end
+
+      def print_live_lag(store)
+        return unless @config.source && @config.migrate&.replication_slot
+
+        source_pool = Connection::Client.new(@config.source)
+        lag_monitor = Monitor::Lag.new(
+          source_pool: source_pool,
+          slot_name:   @config.migrate.replication_slot
+        )
+
+        lag = lag_monitor.lag_bytes
+        threshold = @config.migrate&.lag_threshold_bytes || 1_048_576
+
+        if lag.nil?
+          puts "  #{PASTEL.dim("Replication slot not found or not active")}"
+        elsif lag == 0
+          puts "  #{PASTEL.green("Replication lag: 0 bytes")}  #{PASTEL.green("✓ Ready for cutover")}"
+        elsif lag <= threshold
+          puts "  Replication lag: #{PASTEL.green(lag_monitor.summary)}  #{PASTEL.green("✓ Ready for cutover")}"
+        else
+          puts "  Replication lag: #{lag_monitor.summary}"
+        end
+
+        lsn = lag_monitor.confirmed_lsn
+        puts "  Confirmed LSN:   #{lsn}" if lsn
+
+        source_pool.close
+      rescue Connection::Error
+        puts "  #{PASTEL.dim("(source not reachable — showing checkpoint data only)")}"
+      end
+
+      def phase_label(phase)
+        PHASE_LABELS[phase] || phase.to_s
+      end
+
+      def format_count(n)
+        n.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+      end
+    end
+  end
+end

data/lib/pcrd/commands/verify.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Commands
+    # Compares row counts and spot-checks random rows between source and target.
+    #
+    # Safe to run at any time after backfill completes. Does not modify either cluster.
+    class Verify
+      MismatchError = Class.new(StandardError)
+
+      Result = Data.define(:passed, :tables)
+      TableResult = Data.define(:table_name, :source_count, :target_count,
+                                :sample_size, :mismatches)
+
+      def initialize(config, options = {})
+        @config  = config
+        @options = Options.normalize(options)
+      end
+
+      def run
+        validate_config!
+
+        source_pool = Connection::Client.new(@config.source)
+        target_pool = Connection::Client.new(@config.target)
+        sample_size = @options[:"sample-size"] || @config.verify&.sample_size || 1_000
+
+        table_results = (@config.migrate&.tables || []).map do |table_config|
+          verify_table(source_pool, target_pool, table_config, sample_size)
+        end
+
+        source_pool.close
+        target_pool.close
+
+        Result.new(
+          passed: table_results.all? { |r| r.mismatches.empty? && r.source_count == r.target_count },
+          tables: table_results
+        )
+      end
+
+      private
+
+      def verify_table(source_pool, target_pool, table_config, sample_size)
+        table_name = table_config.name
+        src_count = source_pool.exec("SELECT COUNT(*) FROM #{Sql.quote_table(table_name)}")[0]["count"].to_i
+        tgt_count = target_pool.exec("SELECT COUNT(*) FROM #{Sql.quote_table(table_name)}")[0]["count"].to_i
+
+        mismatches = []
+
+        if src_count == tgt_count && src_count > 0
+          mismatches = spot_check(source_pool, target_pool, table_config, sample_size)
+        end
+
+        TableResult.new(
+          table_name:   table_name,
+          source_count: src_count,
+          target_count: tgt_count,
+          sample_size:  [sample_size, src_count].min,
+          mismatches:   mismatches
+        )
+      rescue Connection::Error => e
+        TableResult.new(
+          table_name:   table_name,
+          source_count: nil,
+          target_count: nil,
+          sample_size:  0,
+          mismatches:   ["Connection error: #{e.message}"]
+        )
+      end
+
+      # Samples source rows, transforms each into its expected target shape, and
+      # compares the values field-by-field against the matching target row.
+      # This is what catches a transform that silently corrupts data — a row
+      # count match alone does not.
+      def spot_check(source_pool, target_pool, table_config, sample_size)
+        table_name  = table_config.name
+        reader      = Schema::Reader.new(source_pool)
+        source_cols = reader.read(table_name)
+        pk_cols     = reader.primary_key_columns(table_name)
+        return [] if pk_cols.empty?
+
+        transformer = Transform::RowTransformer.new(table_config, source_cols)
+        pk_target   = map_pk_to_target(pk_cols, table_config)
+
+        sample_rows = sample_source_rows(source_pool, table_name, sample_size)
+        return [] if sample_rows.empty?
+
+        target_table = Sql.quote_table(table_name)
+        conditions   = pk_target.each_with_index
+                                .map { |col, i| "#{Sql.quote_ident(col)} = $#{i + 1}" }
+                                .join(" AND ")
+
+        mismatches = []
+        sample_rows.each do |src_row|
+          expected  = transformer.transform(src_row) # { target_col => value }
+          pk_values = pk_cols.map { |col| src_row[col] }
+          pk_desc   = pk_cols.zip(pk_values).map { |c, v| "#{c}=#{v}" }.join(",")
+
+          tgt_row = target_pool.exec(
+            "SELECT * FROM #{target_table} WHERE #{conditions}", pk_values
+          ).first
+
+          if tgt_row.nil?
+            mismatches << "pk=#{pk_desc}: row missing on target"
+            next
+          end
+
+          expected.each do |col, exp_val|
+            act_val = tgt_row[col]
+            next if values_equal?(exp_val, act_val)
+
+            mismatches << "pk=#{pk_desc} col=#{col}: " \
+                          "source=#{redact(exp_val)} target=#{redact(act_val)}"
+          end
+        end
+
+        mismatches
+      end
+
+      # Samples up to sample_size rows cheaply. ORDER BY random() sorts the whole
+      # table; instead use TABLESAMPLE SYSTEM (page-level random) for large
+      # tables and a plain LIMIT for small ones. Oversample then cap so an
+      # unlucky page selection still tends to fill the sample.
+      def sample_source_rows(pool, table_name, sample_size)
+        quoted = Sql.quote_table(table_name)
+        est    = Schema::Reader.new(pool).estimated_row_count(table_name)
+
+        if est <= sample_size
+          return pool.exec("SELECT * FROM #{quoted} LIMIT $1", [sample_size]).to_a
+        end
+
+        pct  = [[sample_size * 100.0 / est * 3.0, 0.01].max, 100.0].min
+        rows = pool.exec(
+          "SELECT * FROM #{quoted} TABLESAMPLE SYSTEM (#{pct.round(6)}) LIMIT $1",
+          [sample_size]
+        ).to_a
+
+        # TABLESAMPLE can under-fill on small/unlucky page layouts; fall back.
+        rows.empty? ? pool.exec("SELECT * FROM #{quoted} LIMIT $1", [sample_size]).to_a : rows
+      end
+
+      def map_pk_to_target(pk_source_cols, table_config)
+        pk_source_cols.map do |src|
+          spec = table_config.columns&.[](src) || table_config.columns&.[](src.to_sym)
+          spec&.rename || src
+        end
+      end
+
+      # Values come back from libpq as strings (or nil) on both sides, so a
+      # textual comparison correctly treats e.g. int4 99 and int8 99 as equal
+      # while still catching genuinely different values.
+      def values_equal?(expected, actual)
+        return true if expected.nil? && actual.nil?
+        return false if expected.nil? || actual.nil?
+
+        expected.to_s == actual.to_s
+      end
+
+      def redact(val)
+        return "NULL" if val.nil?
+
+        str = val.to_s
+        str.length > 60 ? "#{str[0, 57]}..." : str
+      end
+
+      def validate_config!
+        raise ConfigError, "source connection required" if @config.source.nil?
+        raise ConfigError, "target connection required for verify" if @config.target.nil?
+        raise ConfigError, "no tables configured" if (@config.migrate&.tables || []).empty?
+      end
+    end
+  end
+end

data/lib/pcrd/config/add_column.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    AddColumn = Data.define(:name, :type, :default)
+  end
+end

data/lib/pcrd/config/analyze_config.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    # tables: Array<String> of table names, or nil to use migrate.tables
+    AnalyzeConfig = Data.define(:tables)
+  end
+end

data/lib/pcrd/config/column_spec.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    # Spec for an existing column. All fields are optional:
+    # nil type means keep the current type; nil rename means keep the name;
+    # drop: false means keep the column.
+    ColumnSpec = Data.define(:type, :rename, :drop)
+  end
+end

data/lib/pcrd/config/connection.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    Connection = Data.define(:host, :port, :database, :user, :password)
+  end
+end

data/lib/pcrd/config/cutover_config.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    CutoverConfig = Data.define(:sequence_buffer, :lag_drain_timeout)
+  end
+end

data/lib/pcrd/config/load_error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Pcrd
+  module Config
+    class LoadError < Pcrd::Error; end
+  end
+end