schema_reaper 2.0.1 → 2.1.0

data/lib/schema_reaper/reporters/sarif.rb

@@ -8,9 +8,10 @@ module SchemaReaper
     class Sarif
       LEVEL = { high: "error", medium: "warning", low: "note" }.freeze
 
-      def initialize(findings, io: $stdout)
+      def initialize(findings, io: $stdout, locator: SchemaLocator.new)
         @findings = findings
         @io = io
+        @locator = locator
       end
 
       def render
@@ -52,15 +53,23 @@ module SchemaReaper
               "confidence" => f.confidence,
               "reclaimableBytes" => f.reclaimable_bytes
             },
-            "locations" => [{
-              "physicalLocation" => {
-                "artifactLocation" => { "uri" => "db/schema.rb" }
-              }
-            }]
+            "locations" => locations_for(f)
           }
         end
       end
 
+      # GitHub code scanning requires a location on every result, so with no
+      # schema file in the repo fall back to the conventional path, unlined.
+      def locations_for(finding)
+        loc = @locator.locate(finding)
+        return [{ "physicalLocation" => { "artifactLocation" => { "uri" => "db/schema.rb" } } }] unless loc
+
+        [{ "physicalLocation" => {
+          "artifactLocation" => { "uri" => loc.path },
+          "region" => { "startLine" => loc.line }
+        } }]
+      end
+
       def camelize(str)
         str.split("_").map(&:capitalize).join
       end

data/lib/schema_reaper/reporters/slack.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  module Reporters
+    # Slack Block Kit payload for the scheduled report. Slack renders neither
+    # Markdown tables nor messages past ~40k characters, so this is a summary
+    # with the most important findings, not the whole report.
+    class Slack
+      MAX_LISTED = 10
+      SEVERITY_ICON = { high: ":red_circle:", medium: ":large_orange_circle:", low: ":white_circle:" }.freeze
+
+      # changes - {new: [Finding], resolved: [String id]} or nil on a first report
+      def initialize(findings, changes: nil)
+        @findings = findings
+        @changes = changes
+      end
+
+      def payload
+        { "text" => summary, "blocks" => blocks }
+      end
+
+      # Also the notification text, and what a client without Block Kit shows.
+      def summary
+        line = "schema_reaper: #{@findings.size} finding#{"s" unless @findings.size == 1}"
+        return line unless @changes
+
+        "#{line} (#{@changes[:new].size} new, #{@changes[:resolved].size} resolved since the last report)"
+      end
+
+      private
+
+      def blocks
+        out = [{ "type" => "section", "text" => mrkdwn("*#{summary}*") }]
+        listed = @changes ? @changes[:new] : @findings
+        out << list_block(@changes ? "New" : "Top findings", listed) if listed.any?
+        out << resolved_block if @changes && @changes[:resolved].any?
+        out << { "type" => "context",
+                 "elements" => [mrkdwn("Full report: `bundle exec schema_reaper scan`")] }
+        out
+      end
+
+      def list_block(title, findings)
+        lines = findings.first(MAX_LISTED).map do |f|
+          "#{SEVERITY_ICON.fetch(f.severity, "")} `#{f.id}` -- #{f.suggested_fix}"
+        end
+        lines << "_...and #{findings.size - MAX_LISTED} more_" if findings.size > MAX_LISTED
+        { "type" => "section", "text" => mrkdwn(truncate("*#{title}*\n#{lines.join("\n")}")) }
+      end
+
+      def resolved_block
+        ids = @changes[:resolved]
+        text = "*Resolved*\n#{ids.first(MAX_LISTED).map { |id| "~#{id}~" }.join("\n")}"
+        text += "\n_...and #{ids.size - MAX_LISTED} more_" if ids.size > MAX_LISTED
+        { "type" => "section", "text" => mrkdwn(truncate(text)) }
+      end
+
+      # A section's text is capped at 3000 characters by Slack.
+      def truncate(text)
+        text.length > 3000 ? "#{text[0, 2990]}..." : text
+      end
+
+      def mrkdwn(text)
+        { "type" => "mrkdwn", "text" => text }
+      end
+    end
+  end
+end

data/lib/schema_reaper/runner.rb

@@ -7,32 +7,84 @@ module SchemaReaper
   class Runner
     SEVERITY_RANK = { high: 0, medium: 1, low: 2 }.freeze
 
-    def initialize(config: Config.load, root: Dir.pwd, introspector: nil, runtime: nil)
+    # only / except - analyzer type names (strings or symbols) to restrict the
+    #                 run to, or to leave out. Unknown names raise Error.
+    # min_confidence  - drop findings below this; defaults to the config key.
+    def initialize(config: Config.load, root: Dir.pwd, introspector: nil, runtime: nil,
+                   only: nil, except: nil, min_confidence: nil)
       @config = config
       @root = root
       @introspector = introspector
       @runtime = runtime
+      @only = names(only)
+      @except = names(except)
+      @min_confidence = min_confidence || config.min_confidence
       load_plugins
     end
 
     def run
+      selected = analyzers
       db = schema
-      warn_missing_query_history(db)
+      warn_missing_query_history(db) if selected.include?(Analyzers::UnusedIndex)
+      scan = Static::Scanner.new(@config, root: @root).scan
       ctx = Analyzers::Context.new(
         schema: db,
-        used_tokens: Static::Scanner.new(@config, root: @root).call,
+        used_tokens: scan.tokens,
+        scan: scan,
         runtime: runtime_report,
         gem_columns: gem_columns(db),
         gem_owned_tables: gem_owned_tables,
         config: @config
       )
 
-      findings = Analyzers::Registry.all.flat_map { |klass| klass.new(ctx).call }
-      dedupe(findings).sort_by { |f| [-f.confidence, f.id] }
+      findings = selected.flat_map { |klass| klass.new(ctx).call }
+      dedupe(kept_in_code(findings, scan.keeps)).select { |f| f.confidence >= @min_confidence }
+                                                .sort_by do |f|
+        [
+          -f.confidence, f.id
+        ]
+      end
+    end
+
+    # Analyzer types this run covers, as strings, or nil when every analyzer
+    # runs. Callers that persist results (the baseline) use it to leave the
+    # other analyzers' records alone.
+    def selected_types
+      return nil if @only.nil? && @except.nil?
+
+      analyzers.map { |k| k.type.to_s }
     end
 
     private
 
+    # `# schema_reaper:keep TARGET` in code silences a finding. TARGET is a
+    # column (any table), table.column, a table (every finding on it) or an
+    # index name.
+    def kept_in_code(findings, keeps)
+      return findings if keeps.empty?
+
+      findings.reject do |f|
+        targets = [f.table, f.column, f.index, f.column && "#{f.table}.#{f.column}"].compact.map(&:downcase)
+        targets.any? { |t| keeps.key?(t) }
+      end
+    end
+
+    def names(list)
+      list && Array(list).flat_map { |n| n.to_s.split(",") }.map { |n| n.strip.to_sym }.reject(&:empty?)
+    end
+
+    # Plugins load in #initialize, so the registry is complete by the time
+    # the names are checked -- a custom analyzer can be selected like a core one.
+    def analyzers
+      all = Analyzers::Registry.all
+      known = all.map(&:type)
+      unknown = ((@only || []) + (@except || [])) - known
+      raise Error, "unknown analyzer(s): #{unknown.join(", ")} (known: #{known.join(", ")})" if unknown.any?
+
+      all.select { |k| @only.nil? || @only.include?(k.type) }
+         .reject { |k| @except&.include?(k.type) }
+    end
+
     # Silently reporting nothing would look like a clean bill of health, so say
     # why the unused-index analyzer sat this run out.
     def warn_missing_query_history(db)
@@ -72,7 +124,7 @@ module SchemaReaper
     def schema
       return @introspector.call if @introspector
 
-      Introspect::Postgres.new(@config.database_url).call
+      Introspect::Postgres.new(@config.database_url, schemas: @config.schemas).call
     end
 
     def runtime_report

data/lib/schema_reaper/runtime.rb

@@ -15,19 +15,32 @@ module SchemaReaper
         new(accessed: Set.new, observed_days: 0)
       end
 
+      # Reads the rotated generation (path.1) too, so rotation never shortens
+      # the observed window. Several processes append to one log, so lines are
+      # not in time order -- the span is min..max, not first..last line.
       def self.load(path)
-        return empty unless path && File.exist?(path)
+        return empty unless path
+
+        files = [Store.rotated_path(path), path].select { |f| File.exist?(f) }
+        return empty if files.empty?
 
         seen = Set.new
         first = last = nil
-        File.foreach(path) do |line|
-          row = JSON.parse(line)
-          seen << row["key"]
-          ts = row["at"]
-          first ||= ts
-          last = ts
-        rescue JSON::ParserError
-          next
+        files.each do |file|
+          File.foreach(file) do |line|
+            row = JSON.parse(line)
+            seen << row["key"]
+            # Store writes UTC ISO8601 ("2026-01-05T00:00:00Z"), which sorts
+            # correctly as a string -- far cheaper than parsing every line of
+            # a log that can hold millions. Only the extremes are parsed.
+            ts = row["at"]
+            next unless ts.is_a?(String)
+
+            first = ts if first.nil? || ts < first
+            last = ts if last.nil? || ts > last
+          rescue JSON::ParserError
+            next
+          end
         end
         new(accessed: seen, observed_days: day_span(first, last))
       end
@@ -36,6 +49,14 @@ module SchemaReaper
         return 0 unless first && last
 
         ((Time.parse(last) - Time.parse(first)) / 86_400).ceil
+      rescue ArgumentError
+        0
+      end
+
+      # Raw-SQL access recorded by the tracker's SQL subscriber, keyed
+      # "table.*": the table was touched, but by no column we can name.
+      def table_accessed?(table)
+        accessed.any? { |k| k.start_with?("#{table}.") }
       end
 
       def empty?
@@ -52,10 +73,19 @@ module SchemaReaper
     end
 
     # Buffered writer for the usage log. Thread-safe append of JSON lines.
+    #
+    # Past max_bytes the log is renamed to `<path>.1` (replacing any older
+    # generation) and a fresh one started, so disk use stays under roughly
+    # 2 x max_bytes. nil disables rotation.
     class Store
-      def initialize(path:, flush_every: 200)
+      def self.rotated_path(path)
+        "#{path}.1"
+      end
+
+      def initialize(path:, flush_every: 200, max_bytes: Config::DEFAULTS["runtime_log_max_bytes"])
         @path = path
         @flush_every = flush_every
+        @max_bytes = max_bytes
         @buffer = []
         @mutex = Mutex.new
         FileUtils.mkdir_p(File.dirname(path))
@@ -79,6 +109,23 @@ module SchemaReaper
 
         File.open(@path, "a") { |f| f.write(@buffer.join) }
         @buffer.clear
+        rotate_if_needed
+      end
+
+      # Several processes (Puma workers) share one log, so the size check and
+      # the rename happen under an exclusive flock and the size is checked
+      # again once it's held: otherwise two workers that both saw an
+      # oversized log would rotate twice, and the second rename would put a
+      # nearly empty log over the full generation in <path>.1.
+      def rotate_if_needed
+        return unless @max_bytes && File.size?(@path).to_i > @max_bytes
+
+        File.open("#{@path}.lock", File::RDWR | File::CREAT, 0o644) do |lock|
+          lock.flock(File::LOCK_EX)
+          File.rename(@path, self.class.rotated_path(@path)) if File.size?(@path).to_i > @max_bytes
+        end
+      rescue SystemCallError
+        nil
       end
     end
 
@@ -88,16 +135,52 @@ module SchemaReaper
       class << self
         attr_accessor :store, :sample_rate
 
-        def install!(store:, sample_rate: 0.05)
+        # track_sql: also sample the tables named in every executed SQL
+        # statement, so a table read only through find_by_sql, a raw
+        # connection.execute or a view-backed report still counts as used.
+        def install!(store:, sample_rate: 0.05, track_sql: true)
           return if @installed
 
           self.store = store
           self.sample_rate = sample_rate
           require "active_record"
           ActiveRecord::Base.prepend(Hook)
+          subscribe_sql if track_sql
           at_exit { store.flush }
           @installed = true
         end
+
+        private
+
+        def subscribe_sql
+          ActiveSupport::Notifications.subscribe("sql.active_record") do |*, payload|
+            Tracker.note_sql(payload[:sql], payload[:name])
+          end
+        end
+      end
+
+      # The relation after FROM / JOIN / UPDATE / INTO, optionally
+      # schema-qualified and double-quoted: "billing"."invoices" -> billing.invoices.
+      #
+      # INSERT INTO targets are left out on purpose: a table that is only ever
+      # written (an audit log nobody reads) is exactly what dead_table should
+      # be able to find. INSERT ... SELECT still records its FROM tables.
+      SQL_TABLE_RE = /\b(?:FROM|JOIN|UPDATE)\s+((?:"?\w+"?\.)?"?\w+"?)/i.freeze
+      SKIP_SQL_NAMES = %w[SCHEMA TRANSACTION].freeze
+
+      def self.note_sql(sql, name = nil)
+        return unless store && sql
+        return if SKIP_SQL_NAMES.include?(name)
+        return unless rand < sample_rate
+
+        # "public"."users" is the same table the app (and the schema) calls users
+        sql.scan(SQL_TABLE_RE).flatten.map { |t| t.delete('"').delete_prefix("public.") }.uniq.each do |table|
+          next if table.start_with?("pg_", "information_schema")
+
+          store.record(table, "*")
+        end
+      rescue StandardError
+        nil
       end
 
       module Hook

data/lib/schema_reaper/schema.rb

@@ -19,7 +19,9 @@ module SchemaReaper
     end
   end
 
-  Index = Struct.new(:name, :columns, :unique, :primary, :scans, :partial, keyword_init: true) do
+  # partitioned: an index on a partitioned parent (relkind 'I'). PostgreSQL
+  # cannot DROP one CONCURRENTLY, so removal advice differs.
+  Index = Struct.new(:name, :columns, :unique, :primary, :scans, :partial, :partitioned, keyword_init: true) do
     def covers?(other)
       columns.first(other.columns.length) == other.columns
     end
@@ -33,10 +35,33 @@ module SchemaReaper
     end
   end
 
+  # A declared FOREIGN KEY constraint. columns is in key order; references
+  # is the referenced table's display name.
+  # behaviour is an opaque signature of referenced key, ON UPDATE/DELETE,
+  # MATCH and deferrability: equal only when the constraints act alike.
+  ForeignKey = Struct.new(:name, :columns, :references, :behaviour, :validated, keyword_init: true)
+
+  # A column-owned sequence (serial / IDENTITY). max_value is the effective
+  # ceiling: the lower of the sequence's own limit and the column type's.
+  Sequence = Struct.new(:name, :table, :column, :column_type, :last_value, :max_value,
+                        keyword_init: true) do
+    def used_fraction
+      return 0.0 unless max_value&.positive?
+
+      last_value.to_f / max_value
+    end
+  end
+
   Table = Struct.new(
-    :name, :columns, :indexes, :primary_key, :foreign_keys, :row_count,
+    :name, :columns, :indexes, :primary_key, :foreign_keys, :foreign_key_constraints, :row_count,
     keyword_init: true
   ) do
+    # Introspectors (and hand-built test schemas) that predate constraint
+    # details leave this nil.
+    def foreign_key_constraints
+      self[:foreign_key_constraints] || []
+    end
+
     def column_names
       columns.map(&:name)
     end
@@ -61,7 +86,11 @@ module SchemaReaper
     end
   end
 
-  DatabaseSchema = Struct.new(:tables, :index_scan_total, keyword_init: true) do
+  DatabaseSchema = Struct.new(:tables, :index_scan_total, :sequences, keyword_init: true) do
+    def sequences
+      self[:sequences] || []
+    end
+
     def table(name)
       tables.find { |t| t.name == name }
     end

data/lib/schema_reaper/schema_locator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # Finds the line in db/schema.rb (or db/structure.sql) that defines a
+  # finding's table, column or index, so SARIF annotations land on it instead
+  # of line 1 of a file that may not even exist.
+  class SchemaLocator
+    Location = Struct.new(:path, :line, keyword_init: true)
+
+    CANDIDATES = %w[db/schema.rb db/structure.sql].freeze
+
+    def initialize(root: Dir.pwd)
+      @root = root
+      @path = CANDIDATES.find { |p| File.exist?(File.join(root, p)) }
+      @lines = @path ? File.readlines(File.join(root, @path)) : []
+    end
+
+    # @return [Location, nil] nil when there is no schema file at all
+    def locate(finding)
+      return nil unless @path
+
+      Location.new(path: @path, line: line_for(finding) || 1)
+    end
+
+    private
+
+    def line_for(finding)
+      start = table_line(finding.table)
+      return nil unless start
+
+      block = block_after(start)
+      found = (finding.index && (find_in(block, quoted(finding.index)) || find_in(all_lines, quoted(finding.index)))) ||
+              (finding.column && find_in(block, column_pattern(finding.column.split(",").first)))
+      found || (start + 1)
+    end
+
+    # schema.rb:     create_table "users", ...      (schema-qualified: "billing.users")
+    # structure.sql: CREATE TABLE public.users (
+    def table_line(table)
+      bare = table.split(".").last
+      ns = table.include?(".") ? table.split(".").first : "public"
+      rb = /create_table\s+"#{Regexp.escape(table)}"/
+      sql = /CREATE TABLE (?:"?#{Regexp.escape(ns)}"?\.)?"?#{Regexp.escape(bare)}"?\s*\(/
+      @lines.index { |l| l.match?(rb) || l.match?(sql) }
+    end
+
+    # Lines from the table's definition to the next table, as [text, index]
+    # pairs. Indexes in schema.rb sit inside the block; in structure.sql they
+    # come later as CREATE INDEX name ON ..., hence the whole-file fallback
+    # for an index (index names are unique per schema, so it can't mismatch).
+    def block_after(start)
+      rest = all_lines.drop(start + 1)
+      stop = rest.index { |l, _| l.match?(/\A\s*create_table\s|\ACREATE TABLE /) }
+      stop ? rest.first(stop) : rest
+    end
+
+    def all_lines
+      @all_lines ||= @lines.each_with_index.to_a
+    end
+
+    def find_in(pairs, pattern)
+      hit = pairs.find { |l, _| l.match?(pattern) }
+      hit && (hit[1] + 1)
+    end
+
+    def quoted(name)
+      /["\s]#{Regexp.escape(name)}["\s]/
+    end
+
+    # schema.rb: t.string "email"    structure.sql:     email character varying
+    def column_pattern(column)
+      /\A\s*(?:t\.\w+\s+"#{Regexp.escape(column)}"|"?#{Regexp.escape(column)}"?\s+\w)/
+    end
+  end
+end

data/lib/schema_reaper/static/cache.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "digest"
+
+module SchemaReaper
+  module Static
+    # Per-file scan results keyed by mtime + size, so a re-scan only parses
+    # the files that changed. Purely a speed-up: any problem reading or
+    # writing it falls back to a full scan, never a wrong one.
+    class Cache
+      # Bumped whenever what a file contributes changes, so a cache written by
+      # an older scanner is discarded instead of trusted.
+      FORMAT = 3
+
+      # path - where to keep it, or nil to disable
+      # key  - anything else that changes a file's result (gem version, ...)
+      def initialize(path, key:)
+        @path = path
+        # A digest, not #hash: Ruby seeds #hash per process, so a key built
+        # with it never matches the one the previous run saved.
+        @key = Digest::SHA256.hexdigest(JSON.generate([FORMAT, key]))
+        @files = load
+      end
+
+      def lookup(file, kind)
+        stamp = stamp_for(file, kind)
+        hit = @files[file]
+        return hit["entry"].merge("stamp" => stamp) if stamp && hit && hit["stamp"] == stamp
+
+        yield.merge("stamp" => stamp)
+      end
+
+      def save(entries)
+        return unless @path
+
+        files = entries.to_h do |file, entry|
+          [file, { "stamp" => entry["stamp"], "entry" => entry.reject { |k, _| k == "stamp" } }]
+        end
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.write(@path, JSON.generate("key" => @key, "files" => files))
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def load
+        return {} unless @path && File.exist?(@path)
+
+        data = JSON.parse(File.read(@path))
+        data["key"] == @key ? data["files"] : {}
+      rescue StandardError
+        {}
+      end
+
+      def stamp_for(file, kind)
+        st = File.stat(file)
+        "#{kind}:#{st.mtime.to_f}:#{st.size}"
+      rescue SystemCallError
+        nil
+      end
+    end
+  end
+end