schema_reaper 2.0.1 → 2.1.0

data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # Two FOREIGN KEY constraints on the same columns pointing at the same
+    # key of the same table, with the same ON UPDATE / ON DELETE, MATCH and
+    # deferrability -- only then is either one redundant. (Same columns but a
+    # different ON DELETE is not a duplicate: dropping one changes what a
+    # parent delete does.) Usually a hand-written add_foreign_key layered on a
+    # `t.references ..., foreign_key: true` that already made one. Each copy is
+    # checked on every insert/update of the child and every delete of the
+    # parent, so the extra one is pure overhead.
+    class DuplicateForeignKey < Base
+      Registry.register(self)
+
+      def call
+        schema.tables.reject { |t| config.ignore_tables.include?(t.name) }
+              .flat_map { |t| duplicates_in(t) }
+      end
+
+      private
+
+      def duplicates_in(table)
+        table.foreign_key_constraints
+             .group_by { |fk| [fk.columns, fk.references, fk.behaviour] }
+             .values.select { |group| group.size > 1 }
+             .flat_map { |group| redundant(table, group) }
+      end
+
+      # Keep a validated one (a NOT VALID copy doesn't check existing rows),
+      # then the alphabetically first name -- Rails' fk_rails_<hash> sorts
+      # ahead of hand-named ones. Report the rest.
+      def redundant(table, group)
+        keep, *extra = group.sort_by { |fk| [fk.validated == false ? 1 : 0, fk.name] }
+        extra.map do |fk|
+          finding(
+            type: :duplicate_foreign_key,
+            table: table.name,
+            column: fk.columns.join(","),
+            index: fk.name,
+            severity: :low,
+            confidence: 0.9,
+            bytes_per_row: 0,
+            evidence: [
+              "#{fk.name} and #{keep.name} both constrain (#{fk.columns.join(", ")}) -> #{fk.references}"
+            ],
+            suggested_fix: "remove_foreign_key #{sym(table.name)}, name: :#{fk.name}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/duplicate_index.rb

@@ -40,7 +40,7 @@ module SchemaReaper
           confidence: 0.8,
           bytes_per_row: 0,
           evidence: [evidence_for(index, covering)],
-          suggested_fix: "remove_index :#{table.name}, name: :#{index.name}"
+          suggested_fix: remove_index_fix(table, index)
         )
       end
 

data/lib/schema_reaper/analyzers/missing_fk_index.rb

@@ -32,6 +32,8 @@ module SchemaReaper
 
       def missing_in(table)
         fk_columns(table).filter_map do |col|
+          next if ignored_column?(col)
+
           type_col = polymorphic_type_for(table, col)
 
           # A bare index on the id alone doesn't cover a polymorphic pair --
@@ -56,7 +58,7 @@ module SchemaReaper
       end
 
       def finding_for(table, col, type_col, declared:)
-        referent = declared ? nil : referent_table(col)
+        referent = declared ? nil : referent_table(col, table.name)
         confidence = confidence_for(declared, referent)
 
         finding(
@@ -97,9 +99,9 @@ module SchemaReaper
       end
 
       def fix_for(table, col, type_col)
-        return "add_index :#{table.name}, :#{col}" unless type_col
+        return "add_index #{sym(table.name)}, :#{col}" unless type_col
 
-        "add_index :#{table.name}, %i[#{type_col} #{col}]"
+        "add_index #{sym(table.name)}, %i[#{type_col} #{col}]"
       end
 
       def fk_columns(table)
@@ -109,7 +111,9 @@ module SchemaReaper
       # A table the column plausibly points at: account_id -> accounts,
       # company_id -> companies, address_id -> addresses. Absence is the signal
       # that a *_id name is not really a foreign key.
-      def referent_table(col)
+      # A table outside public looks in its own schema first:
+      # billing.invoices.account_id -> billing.accounts, then accounts.
+      def referent_table(col, from_table)
         return nil unless col.end_with?("_id")
 
         base = col[0..-4]
@@ -117,6 +121,8 @@ module SchemaReaper
 
         candidates = [base, "#{base}s", "#{base}es"]
         candidates << "#{base[0..-2]}ies" if base.end_with?("y")
+        ns = from_table.include?(".") ? from_table.split(".").first : nil
+        candidates = candidates.map { |c| "#{ns}.#{c}" } + candidates if ns
         candidates.find { |name| table_names.include?(name) }
       end
 

data/lib/schema_reaper/analyzers/sequence_exhaustion.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Analyzers
+    # A serial / IDENTITY sequence that has used up a large share of the values
+    # its column can hold. When it runs out, every INSERT into the table fails.
+    # The classic case is a pre-Rails-5.1 `integer` primary key, capped at
+    # 2,147,483,647 even when its sequence is bigint.
+    #
+    # Not dead weight -- a health check, like missing_fk_index -- but it is
+    # read straight from the schema scan the other analyzers already need.
+    class SequenceExhaustion < Base
+      Registry.register(self)
+
+      WARN_AT = 0.5
+      HIGH_AT = 0.75
+
+      def call
+        schema.sequences.filter_map do |seq|
+          next if config.ignore_tables.include?(seq.table)
+          next if seq.used_fraction < WARN_AT
+
+          finding_for(seq)
+        end
+      end
+
+      private
+
+      def finding_for(seq)
+        pct = (seq.used_fraction * 100).round(1)
+        finding(
+          type: :sequence_exhaustion,
+          table: seq.table,
+          column: seq.column,
+          severity: seq.used_fraction >= HIGH_AT ? :high : :medium,
+          confidence: 0.95, # read from the database, not inferred
+          bytes_per_row: 0,
+          evidence: [
+            "#{seq.name} is at #{seq.last_value} of #{seq.max_value} (#{pct}%) " \
+            "for #{seq.column_type} column #{seq.column}",
+            "INSERTs into #{seq.table} fail once it is exhausted"
+          ],
+          suggested_fix: fix_for(seq)
+        )
+      end
+
+      def fix_for(seq)
+        return "the sequence itself is near its MAXVALUE -- raise it with ALTER SEQUENCE" if seq.column_type == "bigint"
+
+        "change_column #{sym(seq.table)}, :#{seq.column}, :bigint  " \
+          "(rewrites the table under an exclusive lock -- plan for it on a large table)"
+      end
+    end
+  end
+end

data/lib/schema_reaper/analyzers/single_value_column.rb

@@ -24,6 +24,7 @@ module SchemaReaper
           next unless col.single_value?
           next if table.primary_key?(col.name)
           next if config.always_keep_columns.include?(col.name)
+          next if ignored_column?(col.name)
           next if gem_reserved?(table.name, col.name)
 
           finding(
@@ -31,16 +32,27 @@ module SchemaReaper
             table: table.name,
             column: col.name,
             severity: :low,
-            confidence: 0.6,
+            confidence: sampled_stats?(table) ? 0.5 : 0.6,
             bytes_per_row: col.bytes,
-            evidence: [
-              "pg_stats.n_distinct = 1 across ~#{table.row_count} row(s)",
-              "column has the same value in every row"
-            ],
+            evidence: evidence_for(table, col),
             suggested_fix: "confirm the value is not a meaningful default before removing"
           )
         end
       end
+
+      # n_distinct counts non-null values only, so "one value" and "the same
+      # value in every row" are different claims when some rows are NULL.
+      def evidence_for(table, col)
+        nulls = col.null_fraction.to_f
+        ev = ["pg_stats.n_distinct = 1 across ~#{table.row_count} row(s)"]
+        ev << if nulls.positive?
+                "one distinct non-null value; #{(nulls * 100).round(1)}% of rows are NULL"
+              else
+                "column has the same value in every row"
+              end
+        ev << "estimated from a #{STATS_SAMPLE_ROWS}-row ANALYZE sample" if sampled_stats?(table)
+        ev
+      end
     end
   end
 end

data/lib/schema_reaper/analyzers/unused_index.rb

@@ -34,15 +34,20 @@ module SchemaReaper
             severity: :medium,
             confidence: 0.55,
             bytes_per_row: 0,
-            evidence: [
-              "pg_stat_user_indexes.idx_scan = 0 for #{ix.name} (#{ix.columns.join(", ")})",
-              "confirm stats have not been reset recently before dropping"
-            ],
-            suggested_fix: "remove_index :#{table.name}, name: :#{ix.name}  " \
-                           "(use algorithm: :concurrently in production)"
+            evidence: evidence_for(ix),
+            suggested_fix: remove_index_fix(table, ix)
           )
         end
       end
+
+      def evidence_for(index)
+        [
+          "pg_stat_user_indexes.idx_scan = 0 for #{index.name} (#{index.columns.join(", ")})",
+          "confirm stats have not been reset recently before dropping",
+          "idx_scan is per server: if read replicas serve queries, check " \
+          "pg_stat_user_indexes on each replica too -- an index used only there looks unused here"
+        ]
+      end
     end
   end
 end

data/lib/schema_reaper/baseline.rb

@@ -2,34 +2,119 @@
 
 require "json"
 require "fileutils"
+require "date"
 
 module SchemaReaper
   # Persists known finding ids so CI only fails on *new* dead weight.
+  #
+  # Each accepted finding can carry a reason and an expiry date. Once an entry
+  # expires, `scan --ci` treats the finding as new again, so "we'll deal with
+  # it later" comes back around instead of living in the file forever.
+  #
+  # Reads the 2.0 format ({"ids": [...]}) as entries with no reason or expiry,
+  # and keeps writing "ids" alongside "entries" so a 2.0.x gem sharing the
+  # file (mixed versions across a team or CI) still sees the accepted set.
+  #
+  # scope: analyzer types (e.g. %w[dead_column]) a write or prune is limited
+  # to -- what `--only` / `--except` selected. Entries of other analyzers
+  # were not looked at in this run, so they are left exactly as they are.
   class Baseline
+    Entry = Struct.new(:id, :reason, :expires_on, keyword_init: true) do
+      def expired?(today = Date.today)
+        !expires_on.nil? && expires_on < today
+      end
+
+      def type
+        id.split("/").first
+      end
+
+      def to_h
+        { "id" => id, "reason" => reason, "expires_on" => expires_on&.iso8601 }.compact
+      end
+    end
+
     def initialize(path)
       @path = path
     end
 
+    def entries
+      @entries ||= read_entries
+    end
+
+    # Ids still accepted today (expired entries excluded).
     def ids
-      return [] unless File.exist?(@path)
+      entries.reject(&:expired?).map(&:id)
+    end
 
-      JSON.parse(File.read(@path)).fetch("ids", [])
-    rescue JSON::ParserError
-      []
+    def expired
+      entries.select(&:expired?)
+    end
+
+    # Replaces the (in-scope part of the) baseline with exactly these
+    # findings. A reason / expiry passed in applies to new entries and to
+    # expired ones being re-accepted -- that is how an expired entry gets
+    # renewed; unexpired entries keep what they had.
+    def write(findings, reason: nil, expires_on: nil, scope: nil)
+      known = entries.to_h { |e| [e.id, e] }
+      kept = entries.reject { |e| in_scope?(e, scope) }
+      written = findings.map(&:id).uniq.map do |id|
+        entry = known[id]
+        next Entry.new(id: id, reason: reason, expires_on: expires_on) unless entry
+        next renew(entry, reason, expires_on) if entry.expired?
+
+        entry
+      end
+      @entries = (kept + written).uniq(&:id).sort_by(&:id)
+      save
     end
 
-    def write(findings)
+    # Drops entries for findings that no longer occur, without accepting any
+    # new ones -- so a fixed finding can't hide a later regression with the
+    # same id. @return [Array<String>] dropped ids
+    def prune(findings, scope: nil)
+      current = findings.map(&:id)
+      dropped = entries.select { |e| in_scope?(e, scope) && !current.include?(e.id) }.map(&:id)
+      @entries = entries.reject { |e| dropped.include?(e.id) }
+      save
+      dropped
+    end
+
+    # findings not present in the stored baseline (or whose entry expired)
+    def new_among(findings)
+      known = ids
+      findings.reject { |f| known.include?(f.id) }
+    end
+
+    private
+
+    def in_scope?(entry, scope)
+      scope.nil? || scope.include?(entry.type)
+    end
+
+    def renew(entry, reason, expires_on)
+      Entry.new(id: entry.id, reason: reason || entry.reason, expires_on: expires_on)
+    end
+
+    def save
       FileUtils.mkdir_p(File.dirname(@path))
       File.write(@path, JSON.pretty_generate(
                           "generated_at" => Time.now.utc.iso8601,
-                          "ids" => findings.map(&:id).sort
+                          "ids" => ids.sort, # read by 2.0.x; unexpired only
+                          "entries" => entries.map(&:to_h)
                         ))
     end
 
-    # findings not present in the stored baseline
-    def new_among(findings)
-      known = ids
-      findings.reject { |f| known.include?(f.id) }
+    def read_entries
+      return [] unless File.exist?(@path)
+
+      data = JSON.parse(File.read(@path))
+      return data.fetch("ids", []).map { |id| Entry.new(id: id) } unless data.key?("entries")
+
+      data["entries"].map do |e|
+        Entry.new(id: e["id"], reason: e["reason"], expires_on: e["expires_on"] && Date.parse(e["expires_on"]))
+      end
+    rescue JSON::ParserError, ArgumentError
+      []
     end
   end
 end

data/lib/schema_reaper/cli/baseline_support.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "date"
+
+module SchemaReaper
+  class CLI < Thor
+    # Output helpers for `baseline` and the `scan --ci` gate.
+    module BaselineSupport
+      private
+
+      def report_written(findings, scope)
+        console.ok("recorded #{findings.size} finding#{"s" unless findings.size == 1} " \
+                   "as the accepted baseline")
+        console.info("file: #{config.baseline_path}")
+        console.info("only #{scope.join(", ")} entries were replaced; other analyzers' are unchanged") if scope
+        console.info("`scan --ci` now fails only on findings that appear after this point.")
+      end
+
+      def report_pruned(dropped)
+        return console.ok("nothing to prune -- every baseline entry still occurs") if dropped.empty?
+
+        console.ok("dropped #{dropped.size} resolved entr#{dropped.size == 1 ? "y" : "ies"}")
+        dropped.each { |id| console.info("  #{id}") }
+      end
+
+      def parse_date(value)
+        return nil unless value
+
+        Date.iso8601(value)
+      rescue ArgumentError
+        console.problem("--expires must be a date like 2027-01-31, got #{value.inspect}")
+        exit 1
+      end
+
+      def warn_expired(expired, findings)
+        current = findings.map(&:id)
+        live = expired.select { |e| current.include?(e.id) }
+        return if live.empty?
+
+        console.notice("#{live.size} baseline entr#{live.size == 1 ? "y has" : "ies have"} expired and count as new:")
+        live.each { |e| console.info("  #{e.id} (expired #{e.expires_on}#{" -- #{e.reason}" if e.reason})") }
+      end
+    end
+  end
+end

data/lib/schema_reaper/cli.rb

@@ -2,16 +2,23 @@
 
 require "thor"
 require_relative "../schema_reaper"
+require_relative "cli/baseline_support"
 
 module SchemaReaper
   # Command-line entry point. See `exe/schema_reaper`.
   class CLI < Thor
+    include BaselineSupport
+
     def self.exit_on_failure?
       true
     end
 
     class_option :config, type: :string, default: ".schema_reaper.yml",
                           desc: "path to config file"
+    class_option :color, type: :boolean, default: nil,
+                         desc: "force colour on/off (default: auto)"
+    class_option :only, type: :array, desc: "run only these analyzers (e.g. dead_column unused_index)"
+    class_option :except, type: :array, desc: "skip these analyzers"
 
     desc "scan", "Scan schema + code and report dead weight"
     option :format, type: :string, default: "table",
@@ -20,11 +27,10 @@ module SchemaReaper
                 desc: "exit non-zero on findings not in the baseline"
     option :record, type: :boolean, default: false,
                     desc: "append this run to the history log"
-    option :min_confidence, type: :numeric, default: 0.0
-    option :color, type: :boolean, default: nil,
-                   desc: "force colour on/off for the table report (default: auto)"
+    option :min_confidence, type: :numeric,
+                            desc: "hide findings below this confidence (default: min_confidence in config, 0.0)"
     def scan
-      findings = run.select { |f| f.confidence >= options[:min_confidence] }
+      findings = run(min_confidence: options[:min_confidence])
       render_report(findings)
 
       History.new(config.history_log).record(findings) if options[:record]
@@ -32,14 +38,20 @@ module SchemaReaper
     end
 
     desc "baseline", "Write current findings to the baseline file"
+    option :reason, type: :string, desc: "why these findings are accepted (set on new and expired entries)"
+    option :expires, type: :string, desc: "YYYY-MM-DD after which `scan --ci` fails on them again " \
+                                          "(renews expired entries)"
+    option :prune, type: :boolean, default: false,
+                   desc: "only drop entries for findings that no longer occur; accept nothing new"
     def baseline
       findings = run
-      Baseline.new(config.baseline_path).write(findings)
+      store = Baseline.new(config.baseline_path)
       console.title("baseline")
-      console.ok("recorded #{findings.size} finding#{"s" unless findings.size == 1} " \
-                 "as the accepted baseline")
-      console.info("file: #{config.baseline_path}")
-      console.info("`scan --ci` now fails only on findings that appear after this point.")
+      scope = runner.selected_types
+      return report_pruned(store.prune(findings, scope: scope)) if options[:prune]
+
+      store.write(findings, reason: options[:reason], expires_on: parse_date(options[:expires]), scope: scope)
+      report_written(findings, scope)
     end
 
     desc "trend", "Append a snapshot and print progress over time"
@@ -49,8 +61,17 @@ module SchemaReaper
       Reporters::Trend.new(history.trend, color: options[:color]).render
     end
 
-    desc "generate-migration TABLE COLUMN", "Emit a staged removal migration pair"
-    def generate_migration(table, column)
+    desc "generate-migration TABLE [COLUMN]",
+         "Emit a staged removal migration pair for a column, or a removal migration for an index (--index) or table"
+    option :index, type: :string, desc: "name of an index on TABLE to remove instead of a column"
+    def generate_migration(table, column = nil)
+      if options[:index] && column
+        console.problem("pass either COLUMN or --index, not both")
+        exit 1
+      end
+      return generate_index_migration(table, options[:index]) if options[:index]
+      return generate_table_migration(table) unless column
+
       paths = MigrationGenerator.new(table: table, column: column).call
       console.title("generate-migration", "#{table}.#{column}")
       console.ok("created two migrations:")
@@ -62,6 +83,16 @@ module SchemaReaper
                    ])
     end
 
+    desc "explain ID", "Show everything known about one finding and the command to act on it"
+    def explain(id)
+      exit 1 unless Explain.new(run, config: config, console: console).call(id)
+    end
+
+    desc "doctor", "Check the database, statistics, runtime log, baseline and alert setup"
+    def doctor
+      exit 1 unless Doctor.new(config: config, console: console).call
+    end
+
     desc "version", "Print version"
     def version
       say(SchemaReaper::VERSION)
@@ -69,12 +100,34 @@ module SchemaReaper
 
     private
 
+    def generate_index_migration(table, index)
+      path = MigrationGenerator.new(table: table, index: index).call.first
+      console.title("generate-migration", "#{table} · #{index}")
+      console.ok("created #{path}")
+      console.info("runs with algorithm: :concurrently, so it does not lock writes to #{table}")
+    end
+
+    def generate_table_migration(table)
+      path = MigrationGenerator.new(table: table).call.first
+      console.title("generate-migration", table)
+      console.ok("created #{path}")
+      console.list("before running it:", [
+                     "confirm no external consumer (reports, ETL, other services) reads #{table}",
+                     "take a backup -- its `down` cannot bring the data back"
+                   ])
+    end
+
     def config
       @config ||= Config.load(options[:config])
     end
 
-    def run
-      Runner.new(config: config).run
+    def runner(min_confidence: nil)
+      @runner ||= Runner.new(config: config, only: options[:only], except: options[:except],
+                             min_confidence: min_confidence)
+    end
+
+    def run(min_confidence: nil)
+      runner(min_confidence: min_confidence).run
     rescue SchemaReaper::Error => e
       console.problem(e.message)
       exit 1
@@ -93,7 +146,9 @@ module SchemaReaper
     end
 
     def enforce_baseline(findings)
-      new_ones = Baseline.new(config.baseline_path).new_among(findings)
+      store = Baseline.new(config.baseline_path)
+      warn_expired(store.expired, findings)
+      new_ones = store.new_among(findings)
       return if new_ones.empty?
 
       console.problem(

data/lib/schema_reaper/config.rb

@@ -9,6 +9,7 @@ module SchemaReaper
   class Config
     DEFAULTS = {
       "database_url" => nil,          # falls back to ENV["DATABASE_URL"]
+      "schemas" => %w[public],        # PostgreSQL schemas to introspect
       "scan_paths" => %w[app lib config],
       "view_globs" => %w[
         app/**/*.erb app/**/*.haml app/**/*.slim app/**/*.jbuilder
@@ -20,9 +21,16 @@ module SchemaReaper
       "always_keep_columns" => %w[id created_at updated_at type],
       "gem_awareness" => true,        # auto-whitelist columns owned by known gems
       "min_age_days" => 14,
+      "min_confidence" => 0.0,        # drop findings below this from every command
+      "runtime_log_max_bytes" => 50 * 1024 * 1024, # rotate the usage log past this size
       "runtime_log" => ".schema_reaper/runtime.jsonl",
       "history_log" => ".schema_reaper/history.jsonl",
       "baseline" => ".schema_reaper/baseline.json",
+      "scan_cache" => ".schema_reaper/scan_cache.json", # per-file scan cache; null disables
+      # global: a column counts as used if its name appears anywhere.
+      # model:  a name in an app/models file only counts for that model's
+      #         table and the tables it mentions -- stricter, finds more.
+      "column_scope" => "global",
       "database_yml" => "config/database.yml", # Rails fallback for the connection
       "require" => [] # extra files to load (custom analyzers)
     }.freeze
@@ -74,6 +82,18 @@ module SchemaReaper
       @data.dig("ignore", "tables").to_a
     end
 
+    def schemas
+      Array(@data["schemas"]).map(&:to_s)
+    end
+
+    def min_confidence
+      @data["min_confidence"].to_f
+    end
+
+    def runtime_log_max_bytes
+      @data["runtime_log_max_bytes"]
+    end
+
     def min_age_days
       @data["min_age_days"]
     end
@@ -90,6 +110,14 @@ module SchemaReaper
       @data["history_log"]
     end
 
+    def scan_cache
+      @data["scan_cache"]
+    end
+
+    def column_scope
+      @data["column_scope"].to_s == "model" ? :model : :global
+    end
+
     def gem_awareness?
       @data["gem_awareness"] != false
     end