schema_reaper 1.0.0

data/lib/schema_reaper/analyzers/duplicate_index.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # An index whose column list is a leading prefix of another index on the
+    # same table is redundant (the wider index serves both).
+    class DuplicateIndex < Base
+      Registry.register(self)
+
+      def call
+        schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
+              .flat_map { |t| dupes_in(t) }
+      end
+
+      private
+
+      def dupes_in(table)
+        non_pk = table.indexes.reject(&:primary)
+        non_pk.filter_map do |ix|
+          covering = non_pk.find { |o| o != ix && !o.unique && o.covers?(ix) && o.columns != ix.columns }
+          next unless covering
+
+          finding(
+            type: :duplicate_index,
+            table: table.name,
+            index: ix.name,
+            column: ix.columns.join(","),
+            severity: :low,
+            confidence: 0.8,
+            bytes_per_row: 0,
+            evidence: [
+              "#{ix.name} (#{ix.columns.join(", ")}) is a prefix of " \
+              "#{covering.name} (#{covering.columns.join(", ")})"
+            ],
+            suggested_fix: "remove_index :#{table.name}, name: :#{ix.name}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/missing_fk_index.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # A foreign-key column with no index: every parent delete/update scans the
+    # child table. This is a repo-health nag, not dead weight.
+    class MissingFkIndex < Base
+      Registry.register(self)
+
+      def call
+        schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
+              .flat_map { |t| missing_in(t) }
+      end
+
+      private
+
+      def missing_in(table)
+        fk_columns(table).filter_map do |col|
+          next if indexed?(table, col)
+
+          finding(
+            type: :missing_fk_index,
+            table: table.name,
+            column: col,
+            severity: :medium,
+            confidence: 0.9,
+            bytes_per_row: 0,
+            evidence: ["#{col} is a foreign key with no covering index"],
+            suggested_fix: "add_index :#{table.name}, :#{col}"
+          )
+        end
+      end
+
+      def fk_columns(table)
+        (table.foreign_keys + table.column_names.grep(/_id\z/)).uniq
+      end
+
+      def indexed?(table, col)
+        table.indexes.any? { |ix| ix.columns.first == col }
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/registry.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # Analyzers register themselves here so the runner can iterate them.
+    module Registry
+      @classes = []
+
+      class << self
+        attr_reader :classes
+
+        def register(klass)
+          @classes << klass unless @classes.include?(klass)
+        end
+
+        def all = @classes
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/single_value_column.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # A column holding exactly one distinct value across a non-trivial number of
+    # rows carries no information — usually a flag that was never toggled or a
+    # backfill that set everyone the same.
+    class SingleValueColumn < Base
+      Registry.register(self)
+
+      MIN_ROWS = 500
+
+      def call
+        schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
+              .flat_map { |t| flat_in(t) }
+      end
+
+      private
+
+      def flat_in(table)
+        return [] unless table.row_count.to_i >= MIN_ROWS
+
+        table.columns.filter_map do |col|
+          next unless col.single_value?
+          next if col.name == table.primary_key
+          next if config.always_keep_columns.include?(col.name)
+          next if gem_reserved?(table.name, col.name)
+
+          finding(
+            type: :single_value_column,
+            table: table.name,
+            column: col.name,
+            severity: :low,
+            confidence: 0.6,
+            bytes_per_row: col.bytes,
+            evidence: [
+              "pg_stats.n_distinct = 1 across ~#{table.row_count} row(s)",
+              "column has the same value in every row"
+            ],
+            suggested_fix: "confirm the value is not a meaningful default before removing"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/unused_index.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # Indexes with zero scans in pg_stat_user_indexes. Requires stats to be
+    # meaningful (a freshly reset stat table would false-positive), so findings
+    # stay medium confidence and note the caveat.
+    class UnusedIndex < Base
+      Registry.register(self)
+
+      def call
+        schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
+              .flat_map { |t| unused_in(t) }
+      end
+
+      private
+
+      def unused_in(table)
+        table.indexes.filter_map do |ix|
+          next if ix.primary || ix.unique # keep constraint-backing indexes
+          next if ix.scans.nil? # no stats available
+          next unless ix.scans.zero?
+
+          finding(
+            type: :unused_index,
+            table: table.name,
+            index: ix.name,
+            column: ix.columns.join(","),
+            severity: :medium,
+            confidence: 0.55,
+            bytes_per_row: 0,
+            evidence: [
+              "pg_stat_user_indexes.idx_scan = 0 for #{ix.name} (#{ix.columns.join(", ")})",
+              "confirm stats have not been reset recently before dropping"
+            ],
+            suggested_fix: "remove_index :#{table.name}, name: :#{ix.name}  " \
+                           "(use algorithm: :concurrently in production)"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/schema_reaper/baseline.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module SchemaReaper
+  # Persists known finding ids so CI only fails on *new* dead weight.
+  class Baseline
+    def initialize(path)
+      @path = path
+    end
+
+    def ids
+      return [] unless File.exist?(@path)
+
+      JSON.parse(File.read(@path)).fetch("ids", [])
+    rescue JSON::ParserError
+      []
+    end
+
+    def write(findings)
+      FileUtils.mkdir_p(File.dirname(@path))
+      File.write(@path, JSON.pretty_generate(
+                          "generated_at" => Time.now.utc.iso8601,
+                          "ids" => findings.map(&:id).sort
+                        ))
+    end
+
+    # findings not present in the stored baseline
+    def new_among(findings)
+      known = ids
+      findings.reject { |f| known.include?(f.id) }
+    end
+  end
+end

data/lib/schema_reaper/cli.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "thor"
+require_relative "../schema_reaper"
+
+module SchemaReaper
+  # Command-line entry point. See `exe/schema_reaper`.
+  class CLI < Thor
+    def self.exit_on_failure? = true
+
+    class_option :config, type: :string, default: ".schema_reaper.yml",
+                          desc: "path to config file"
+
+    desc "scan", "Scan schema + code and report dead weight"
+    option :format, type: :string, default: "table",
+                    enum: %w[table json markdown sarif]
+    option :ci, type: :boolean, default: false,
+                desc: "exit non-zero on findings not in the baseline"
+    option :record, type: :boolean, default: false,
+                    desc: "append this run to the history log"
+    option :min_confidence, type: :numeric, default: 0.0
+    def scan
+      findings = run.select { |f| f.confidence >= options[:min_confidence] }
+      SchemaReaper.reporter(options[:format]).new(findings).render
+
+      History.new(config.history_log).record(findings) if options[:record]
+      enforce_baseline(findings) if options[:ci]
+    end
+
+    desc "baseline", "Write current findings to the baseline file"
+    def baseline
+      findings = run
+      Baseline.new(config.baseline_path).write(findings)
+      say "wrote #{findings.size} finding(s) to #{config.baseline_path}"
+    end
+
+    desc "trend", "Append a snapshot and print progress over time"
+    def trend
+      History.new(config.history_log).record(run)
+      require "pp"
+      pp History.new(config.history_log).trend
+    end
+
+    desc "generate-migration TABLE COLUMN", "Emit a staged removal migration pair"
+    def generate_migration(table, column)
+      MigrationGenerator.new(table: table, column: column).call
+                        .each { |p| say "created #{p}" }
+    end
+
+    desc "version", "Print version"
+    def version = say(SchemaReaper::VERSION)
+
+    private
+
+    def config = @config ||= Config.load(options[:config])
+
+    def run = Runner.new(config: config).run
+
+    def enforce_baseline(findings)
+      new_ones = Baseline.new(config.baseline_path).new_among(findings)
+      return if new_ones.empty?
+
+      warn "schema_reaper: #{new_ones.size} new finding(s) since baseline"
+      new_ones.each { |f| warn "  - #{f.id}" }
+      exit 1
+    end
+  end
+end

data/lib/schema_reaper/config.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module SchemaReaper
+  # Loaded from .schema_reaper.yml at the project root. Every key has a default
+  # so a missing file still yields a usable config.
+  class Config
+    DEFAULTS = {
+      "database_url" => nil,          # falls back to ENV["DATABASE_URL"]
+      "scan_paths" => %w[app lib config],
+      "view_globs" => %w[
+        app/**/*.erb app/**/*.haml app/**/*.slim app/**/*.jbuilder
+      ],
+      "ignore" => {
+        "tables" => %w[schema_migrations ar_internal_metadata],
+        "columns" => []               # strings, or "/regex/" for a pattern
+      },
+      "always_keep_columns" => %w[id created_at updated_at type],
+      "gem_awareness" => true,        # auto-whitelist columns owned by known gems
+      "min_age_days" => 14,
+      "runtime_log" => ".schema_reaper/runtime.jsonl",
+      "history_log" => ".schema_reaper/history.jsonl",
+      "baseline" => ".schema_reaper/baseline.json",
+      "require" => [] # extra files to load (custom analyzers)
+    }.freeze
+
+    def self.load(path = ".schema_reaper.yml")
+      raw = File.exist?(path) ? (YAML.safe_load_file(path) || {}) : {}
+      new(deep_merge(DEFAULTS, raw))
+    end
+
+    def self.deep_merge(base, override)
+      base.merge(override) do |_key, a, b|
+        a.is_a?(Hash) && b.is_a?(Hash) ? deep_merge(a, b) : b
+      end
+    end
+
+    def initialize(data)
+      @data = data
+    end
+
+    def database_url
+      @data["database_url"] || ENV.fetch("DATABASE_URL", nil)
+    end
+
+    def scan_paths      = @data["scan_paths"]
+    def view_globs      = @data["view_globs"]
+    def ignore_tables   = @data.dig("ignore", "tables").to_a
+    def min_age_days    = @data["min_age_days"]
+    def baseline_path   = @data["baseline"]
+    def runtime_log     = @data["runtime_log"]
+    def history_log     = @data["history_log"]
+    def gem_awareness?  = @data["gem_awareness"] != false
+    def require_paths   = @data["require"].to_a
+
+    def always_keep_columns
+      @data["always_keep_columns"].to_a
+    end
+
+    # Returns true when a column name should be ignored outright.
+    def ignored_column?(name)
+      matchers.any? { |m| m.is_a?(Regexp) ? m.match?(name) : m == name }
+    end
+
+    private
+
+    def matchers
+      @matchers ||= @data.dig("ignore", "columns").to_a.map do |entry|
+        if entry.is_a?(String) && entry.start_with?("/") && entry.end_with?("/")
+          Regexp.new(entry[1..-2])
+        else
+          entry
+        end
+      end
+    end
+  end
+end

data/lib/schema_reaper/finding.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # A single issue reported by an analyzer.
+  Finding = Struct.new(
+    :type,              # Symbol, e.g. :dead_column
+    :table,             # String
+    :column,            # String or nil
+    :index,             # String or nil (index name, for index findings)
+    :severity,          # :low | :medium | :high
+    :confidence,        # 0.0..1.0
+    :bytes_per_row,     # Integer estimate, 0 if unknown
+    :reclaimable_bytes, # Integer, bytes_per_row * row_count
+    :evidence,          # Array<String> human-readable reasons
+    :suggested_fix,     # String
+    keyword_init: true
+  ) do
+    def id
+      [type, table, column, index].compact.join("/")
+    end
+
+    def reclaimable_bytes = self[:reclaimable_bytes] || 0
+
+    def to_h
+      super.merge(id: id, reclaimable_bytes: reclaimable_bytes)
+    end
+  end
+end

data/lib/schema_reaper/gem_awareness.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # Columns and tables that popular gems own and reference indirectly (through
+  # metaprogramming the static scanner cannot see). Detected gems get their
+  # columns whitelisted automatically.
+  module GemAwareness
+    # gem name => { table_glob => [column names] }.  "*" table_glob matches any
+    # table; an exact name matches only that table.
+    MAP = {
+      "devise" => { "*" => %w[
+        encrypted_password reset_password_token reset_password_sent_at
+        remember_created_at sign_in_count current_sign_in_at last_sign_in_at
+        current_sign_in_ip last_sign_in_ip confirmation_token confirmed_at
+        confirmation_sent_at unconfirmed_email failed_attempts unlock_token
+        locked_at provider uid
+      ] },
+      "paper_trail" => { "versions" => %w[item_type item_id event whodunnit object object_changes] },
+      "audited" => { "audits" => %w[auditable_type auditable_id user_type user_id action audited_changes version] },
+      "friendly_id" => { "friendly_id_slugs" => %w[slug sluggable_id sluggable_type scope] },
+      "acts_as_paranoid" => { "*" => %w[deleted_at] },
+      "paranoia" => { "*" => %w[deleted_at] },
+      "counter_culture" => { "*" => %w[] }, # dynamic *_count columns handled by regex in config
+      "activestorage" => {
+        "active_storage_blobs" => %w[key filename content_type metadata service_name byte_size checksum],
+        "active_storage_attachments" => %w[name record_type record_id blob_id],
+        "active_storage_variant_records" => %w[blob_id variation_digest]
+      },
+      "actiontext" => { "action_text_rich_texts" => %w[body record_type record_id name] },
+      "pg_search" => { "pg_search_documents" => %w[content searchable_type searchable_id] },
+      "ahoy_matey" => {
+        "ahoy_visits" => %w[visit_token visitor_token],
+        "ahoy_events" => %w[visit_id name properties]
+      }
+    }.freeze
+
+    # @param installed [Enumerable<String>] gem names present in the bundle
+    # @param tables [Enumerable<#name,#column_names>] or Enumerable<String>
+    # @return [Hash{String => Set<String>}]
+    #
+    # For an exact table_glob the columns are reserved on that table. For "*"
+    # (columns a gem can add to any model table) they are reserved only on
+    # tables that already contain the first ("anchor") column of the list, so an
+    # unrelated `provider`/`uid` column elsewhere is still reportable.
+    def self.reserved_columns(installed:, tables:)
+      installed = installed.to_set
+      index = tables.to_h { |t| t.respond_to?(:name) ? [t.name, t.column_names] : [t, nil] }
+      result = Hash.new { |h, k| h[k] = Set.new }
+
+      MAP.each do |gem_name, table_map|
+        next unless installed.include?(gem_name)
+
+        table_map.each do |table_glob, columns|
+          next if columns.empty?
+
+          if table_glob == "*"
+            anchor = columns.first
+            index.each do |name, cols|
+              result[name].merge(columns) if cols&.include?(anchor)
+            end
+          elsif index.key?(table_glob)
+            result[table_glob].merge(columns)
+          end
+        end
+      end
+      result
+    end
+
+    # Best-effort list of gems in the current bundle.
+    def self.installed_gems
+      if defined?(Bundler)
+        Bundler.load.specs.map(&:name)
+      else
+        Gem::Specification.map(&:name)
+      end
+    rescue StandardError
+      []
+    end
+  end
+end

data/lib/schema_reaper/history.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "time"
+
+module SchemaReaper
+  # Append-only log of scan snapshots, so cleanup progress is visible over time.
+  class History
+    Snapshot = Struct.new(:at, :count, :reclaimable_bytes, :ids, keyword_init: true)
+
+    def initialize(path = ".schema_reaper/history.jsonl")
+      @path = path
+    end
+
+    def record(findings)
+      FileUtils.mkdir_p(File.dirname(@path))
+      row = {
+        "at" => Time.now.utc.iso8601,
+        "count" => findings.size,
+        "reclaimable_bytes" => findings.sum(&:reclaimable_bytes),
+        "ids" => findings.map(&:id).sort
+      }
+      File.open(@path, "a") { |f| f.puts JSON.generate(row) }
+      row
+    end
+
+    def snapshots
+      return [] unless File.exist?(@path)
+
+      File.foreach(@path).filter_map do |line|
+        r = JSON.parse(line)
+        Snapshot.new(at: r["at"], count: r["count"],
+                     reclaimable_bytes: r["reclaimable_bytes"], ids: r["ids"])
+      rescue JSON::ParserError
+        nil
+      end
+    end
+
+    # @return [Hash] delta between the first and last snapshot plus current-vs-previous
+    def trend
+      snaps = snapshots
+      return { snapshots: 0 } if snaps.empty?
+
+      first = snaps.first
+      last = snaps.last
+      prev = snaps[-2] || first
+      {
+        snapshots: snaps.size,
+        first_at: first.at,
+        last_at: last.at,
+        count_change_total: last.count - first.count,
+        count_change_last: last.count - prev.count,
+        newly_introduced: (last.ids - prev.ids),
+        resolved_since_prev: (prev.ids - last.ids),
+        bytes_change_total: last.reclaimable_bytes - first.reclaimable_bytes
+      }
+    end
+  end
+end

data/lib/schema_reaper/introspect/postgres.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Introspect
+    # Reads live schema + planner statistics from PostgreSQL using the `pg` gem
+    # directly, so the host app does not need to boot Rails.
+    class Postgres
+      AVG_TYPE_BYTES = {
+        "boolean" => 1, "smallint" => 2, "integer" => 4, "bigint" => 8,
+        "real" => 4, "double precision" => 8, "numeric" => 8,
+        "date" => 4, "timestamp without time zone" => 8,
+        "timestamp with time zone" => 8, "uuid" => 16
+      }.freeze
+
+      def initialize(url)
+        raise Error, "no database_url configured" if url.nil? || url.empty?
+
+        require "pg"
+        @conn = PG.connect(url)
+      end
+
+      def call
+        DatabaseSchema.new(tables: table_names.map { |n| build_table(n) })
+      end
+
+      private
+
+      def table_names
+        exec(<<~SQL).map { |r| r["tablename"] }
+          SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename
+        SQL
+      end
+
+      def build_table(name)
+        Table.new(
+          name: name,
+          columns: columns_for(name),
+          indexes: indexes_for(name),
+          primary_key: primary_key_for(name),
+          foreign_keys: foreign_keys_for(name),
+          row_count: row_count_for(name)
+        )
+      end
+
+      def columns_for(table)
+        stats = column_stats_for(table)
+        exec(<<~SQL, [table]).map do |r|
+          SELECT column_name, data_type, is_nullable, column_default
+          FROM information_schema.columns
+          WHERE table_schema = 'public' AND table_name = $1
+          ORDER BY ordinal_position
+        SQL
+          s = stats[r["column_name"]] || {}
+          Column.new(
+            name: r["column_name"],
+            sql_type: r["data_type"],
+            null: r["is_nullable"] == "YES",
+            default: r["column_default"],
+            bytes: AVG_TYPE_BYTES.fetch(r["data_type"], 16),
+            null_fraction: s[:null_frac],
+            distinct_values: s[:n_distinct]
+          )
+        end
+      end
+
+      # pg_stats.n_distinct: >= 0 is an absolute count, < 0 is a ratio of rows.
+      def column_stats_for(table)
+        exec(<<~SQL, [table]).each_with_object({}) do |r, h|
+          SELECT attname, null_frac, n_distinct
+          FROM pg_stats WHERE schemaname = 'public' AND tablename = $1
+        SQL
+          nd = r["n_distinct"].to_f
+          h[r["attname"]] = {
+            null_frac: r["null_frac"].to_f,
+            n_distinct: nd >= 0 ? nd.round : nil
+          }
+        end
+      end
+
+      def indexes_for(table)
+        exec(<<~SQL, [table]).map do |r|
+          SELECT i.relname AS name, ix.indisunique AS "unique", ix.indisprimary AS "primary",
+                 s.idx_scan AS scans,
+                 array_to_string(array_agg(a.attname ORDER BY a.attnum), ',') AS cols
+          FROM pg_class t
+          JOIN pg_index ix ON t.oid = ix.indrelid
+          JOIN pg_class i ON i.oid = ix.indexrelid
+          JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
+          LEFT JOIN pg_stat_user_indexes s ON s.indexrelid = i.oid
+          WHERE t.relname = $1
+          GROUP BY i.relname, ix.indisunique, ix.indisprimary, s.idx_scan
+        SQL
+          Index.new(
+            name: r["name"], columns: r["cols"].split(","),
+            unique: r["unique"] == "t", primary: r["primary"] == "t",
+            scans: r["scans"]&.to_i
+          )
+        end
+      end
+
+      def primary_key_for(table)
+        exec(<<~SQL, [table]).map { |r| r["attname"] }.first
+          SELECT a.attname
+          FROM pg_index i
+          JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
+          WHERE i.indrelid = $1::regclass AND i.indisprimary
+        SQL
+      rescue PG::Error
+        nil
+      end
+
+      def foreign_keys_for(table)
+        exec(<<~SQL, [table]).map { |r| r["column_name"] }
+          SELECT kcu.column_name
+          FROM information_schema.table_constraints tc
+          JOIN information_schema.key_column_usage kcu
+            ON tc.constraint_name = kcu.constraint_name
+          WHERE tc.constraint_type = 'FOREIGN KEY'
+            AND tc.table_schema = 'public' AND tc.table_name = $1
+        SQL
+      end
+
+      def row_count_for(table)
+        exec(<<~SQL, [table]).first&.fetch("reltuples")&.to_f&.round
+          SELECT reltuples FROM pg_class WHERE relname = $1
+        SQL
+      end
+
+      def exec(sql, params = nil)
+        (params ? @conn.exec_params(sql, params) : @conn.exec(sql)).to_a
+      end
+    end
+  end
+end