schema_reaper 2.0.1 → 2.1.0

data/lib/schema_reaper/doctor.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "time"
+
+module SchemaReaper
+  # `schema_reaper doctor`: checks everything a scan's accuracy depends on and
+  # says what to fix, so a quiet report can be trusted to mean "nothing dead"
+  # rather than "nothing measured".
+  class Doctor
+    Check = Struct.new(:status, :label, :detail, keyword_init: true) # status: :ok | :warn | :fail
+
+    STATS_FRESH_DAYS = 7
+    ICON = { ok: "✓", warn: "!", fail: "✗" }.freeze
+
+    def initialize(config:, root: Dir.pwd, introspector: nil, console: Reporters::Console.new)
+      @config = config
+      @root = root
+      @introspector = introspector
+      @console = console
+      @checks = []
+    end
+
+    # @return [Boolean] true unless a check failed outright
+    def call
+      check_scan_paths
+      check_database
+      check_runtime_log
+      check_baseline
+      check_schema_file
+      check_alerts
+      render
+      @checks.none? { |c| c.status == :fail }
+    end
+
+    private
+
+    def ok(label, detail = nil)
+      @checks << Check.new(status: :ok, label: label, detail: detail)
+    end
+
+    def warn_check(label, detail)
+      @checks << Check.new(status: :warn, label: label, detail: detail)
+    end
+
+    def fail_check(label, detail)
+      @checks << Check.new(status: :fail, label: label, detail: detail)
+    end
+
+    def check_scan_paths
+      missing = @config.scan_paths.reject { |p| Dir.exist?(File.join(@root, p)) }
+      return ok("scan paths", @config.scan_paths.join(", ")) if missing.empty?
+
+      warn_check("scan paths", "not found: #{missing.join(", ")} -- code there can't mark anything as used")
+    end
+
+    def check_database
+      intro = @introspector || Introspect::Postgres.new(@config.database_url, schemas: @config.schemas)
+      diag = intro.diagnostics
+      ok("database", "connected, PostgreSQL #{diag[:server_version]}")
+      check_stats_reset(diag[:stats_reset])
+      check_analyze(diag[:never_analyzed])
+      check_query_history(intro.call)
+      if diag[:in_recovery]
+        warn_check("replica", "this is a read replica -- index usage here doesn't cover the primary")
+      end
+    rescue Error => e
+      fail_check("database", e.message)
+    end
+
+    def check_stats_reset(reset_at)
+      return ok("statistics", "never reset") if reset_at.nil?
+
+      days = ((Time.now - Time.parse(reset_at.to_s)) / 86_400).floor
+      return ok("statistics", "collected for #{days} day(s)") if days >= STATS_FRESH_DAYS
+
+      warn_check("statistics", "reset #{days} day(s) ago -- unused_index needs a longer window to be trusted")
+    end
+
+    def check_analyze(tables)
+      return ok("ANALYZE", "every table has planner statistics") if tables.empty?
+
+      sample = tables.first(5).join(", ") + (tables.size > 5 ? ", ..." : "")
+      warn_check("ANALYZE", "#{tables.size} table(s) never analysed (#{sample}) -- " \
+                            "row counts and null/distinct checks are blind there; run ANALYZE")
+    end
+
+    def check_query_history(schema)
+      return ok("query history", "#{schema.index_scan_total} index scans recorded") if schema.query_history?
+
+      warn_check("query history", "only #{schema.index_scan_total} index scan(s) -- unused_index will sit out; " \
+                                  "scan a database that serves real traffic")
+    end
+
+    def check_runtime_log
+      report = Runtime::Report.load(@config.runtime_log)
+      need = @config.min_age_days || Config::DEFAULTS["min_age_days"]
+      if report.empty?
+        warn_check("runtime log", "none at #{@config.runtime_log} -- dead_column confidence stays capped at " \
+                                  "#{Analyzers::DeadColumn::STATIC_ONLY_CAP}; see README > Runtime signal")
+      elsif report.observed_days < need
+        warn_check("runtime log", "#{report.observed_days} of #{need} day(s) observed -- not trusted yet")
+      else
+        ok("runtime log", "#{report.observed_days} day(s), #{report.accessed.size} column(s) seen")
+      end
+    end
+
+    def check_baseline
+      base = Baseline.new(@config.baseline_path)
+      return ok("baseline", "none (scan --ci would fail on every finding)") if base.entries.empty?
+
+      expired = base.expired
+      return ok("baseline", "#{base.entries.size} accepted finding(s)") if expired.empty?
+
+      warn_check("baseline", "#{expired.size} entr#{expired.size == 1 ? "y has" : "ies have"} expired: " \
+                             "#{expired.map(&:id).first(3).join(", ")}")
+    end
+
+    def check_schema_file
+      path = SchemaLocator::CANDIDATES.find { |p| File.exist?(File.join(@root, p)) }
+      return ok("schema file", path) if path
+
+      warn_check("schema file", "no db/schema.rb or db/structure.sql -- SARIF results can't point at a line")
+    end
+
+    # Only meaningful inside a Rails app with the install generator's
+    # initializer loaded.
+    def check_alerts
+      return unless defined?(AlertConfig) && defined?(Rails)
+
+      cfg = AlertConfig.instance
+      unless cfg.configured?
+        return warn_check("alerts", "no emails or webhook_url configured -- scheduled scans report nowhere")
+      end
+
+      channels = [("email (#{cfg.emails.size})" if cfg.emails?), ("webhook" if cfg.webhook?)].compact
+      ok("alerts", "#{channels.join(" + ")}, alert_on: #{cfg.alert_on}")
+    end
+
+    def render
+      @console.title("doctor")
+      @checks.each do |c|
+        line = format("%<icon>s %<label>-14s %<detail>s", icon: ICON[c.status], label: c.label, detail: c.detail)
+        @console.info(line)
+      end
+      @console.blank
+      counts = @checks.group_by(&:status).transform_values(&:size)
+      @console.info("#{counts.fetch(:ok, 0)} ok, #{counts.fetch(:warn, 0)} warning(s), " \
+                    "#{counts.fetch(:fail, 0)} failed")
+    end
+  end
+end

data/lib/schema_reaper/explain.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # `schema_reaper explain ID`: everything known about one finding, and the
+  # exact command to act on it.
+  class Explain
+    def initialize(findings, config:, root: Dir.pwd, console: Reporters::Console.new)
+      @findings = findings
+      @config = config
+      @root = root
+      @console = console
+    end
+
+    # @return [Boolean] false when no finding matches
+    def call(query)
+      matches = match(query)
+      if matches.size == 1
+        render(matches.first)
+      elsif matches.empty?
+        report_not_found(query)
+      else
+        report_ambiguous(query, matches)
+      end
+      matches.size == 1
+    end
+
+    private
+
+    # An exact id, else every finding whose id contains the query
+    # ("users/legacy_token" finds dead_column/users/legacy_token).
+    def match(query)
+      exact = @findings.select { |f| f.id == query }
+      exact.any? ? exact : @findings.select { |f| f.id.include?(query) }
+    end
+
+    def report_not_found(query)
+      @console.problem("no current finding matches #{query.inspect}")
+      table_finding = @findings.find { |f| f.type == :dead_table && query.split("/").include?(f.table) }
+      if table_finding
+        @console.info("its table is reported whole as #{table_finding.id} -- per-column and per-index " \
+                      "findings on a dead table are folded into that one")
+        return
+      end
+
+      @console.info("list them with `schema_reaper scan --format json` -- ids look like dead_column/users/legacy_token")
+    end
+
+    def report_ambiguous(query, matches)
+      @console.problem("#{matches.size} findings match #{query.inspect} -- be more specific:",
+                       items: matches.map(&:id))
+    end
+
+    def render(finding)
+      @console.title("explain", finding.id)
+      @console.row("type", finding.type)
+      @console.row("severity", finding.severity)
+      @console.row("confidence", format("%.2f", finding.confidence))
+      @console.row("reclaimable", reclaim(finding))
+      @console.row("defined at", location(finding)) if location(finding)
+      @console.row("baseline", baseline_status(finding))
+      @console.blank
+      @console.list("evidence", finding.evidence, colour: :reset)
+      @console.blank
+      @console.list("fix", [finding.suggested_fix], colour: :reset)
+      next_step = next_command(finding)
+      @console.list("next", [next_step], colour: :reset) if next_step
+    end
+
+    def reclaim(finding)
+      return "unknown (row count unavailable -- run ANALYZE)" unless finding.reclaim_known?
+
+      Reporters::Bytes.human(finding.reclaimable_bytes)
+    end
+
+    def location(finding)
+      @location ||= {}
+      @location[finding.id] ||= begin
+        loc = SchemaLocator.new(root: @root).locate(finding)
+        loc && "#{loc.path}:#{loc.line}"
+      end
+    end
+
+    def baseline_status(finding)
+      entry = Baseline.new(@config.baseline_path).entries.find { |e| e.id == finding.id }
+      return "not accepted (fails `scan --ci`)" unless entry
+      return "accepted, but expired #{entry.expires_on}" if entry.expired?
+
+      ["accepted", entry.reason, entry.expires_on && "until #{entry.expires_on}"].compact.join(" -- ")
+    end
+
+    def next_command(finding)
+      case finding.type
+      when :dead_column, :always_null_column, :single_value_column
+        "bundle exec schema_reaper generate-migration #{finding.table} #{finding.column}"
+      when :unused_index, :duplicate_index
+        "bundle exec schema_reaper generate-migration #{finding.table} --index #{finding.index}"
+      when :dead_table
+        "bundle exec schema_reaper generate-migration #{finding.table}"
+      end
+    end
+  end
+end

data/lib/schema_reaper/gem_awareness.rb

@@ -9,6 +9,18 @@ module SchemaReaper
   module GemAwareness
     # gem name => { table_glob => [column names] }.  "*" table_glob matches any
     # table; an exact name matches only that table.
+    def self.owned(tables)
+      tables.to_h { |t| [t, []] }
+    end
+    private_class_method :owned
+
+    SOLID_QUEUE_TABLES = %w[
+      solid_queue_jobs solid_queue_ready_executions solid_queue_scheduled_executions
+      solid_queue_claimed_executions solid_queue_blocked_executions solid_queue_failed_executions
+      solid_queue_recurring_executions solid_queue_recurring_tasks solid_queue_pauses
+      solid_queue_processes solid_queue_semaphores
+    ].freeze
+
     MAP = {
       "devise" => { "*" => %w[
         encrypted_password reset_password_token reset_password_sent_at
@@ -42,7 +54,25 @@ module SchemaReaper
           resource_owner_type resource_owner_id access_token refresh_token
           expires_in revoked_at previous_refresh_token
         ]
-      }
+      },
+      # Gems below own their tables outright and touch every column from
+      # their own code, so the tables are listed with no columns: ownership
+      # alone keeps every analyzer off them (see Base#gem_reserved?).
+      "solid_queue" => owned(SOLID_QUEUE_TABLES),
+      "solid_cache" => owned(%w[solid_cache_entries]),
+      "solid_cable" => owned(%w[solid_cable_messages]),
+      "good_job" => owned(%w[good_jobs good_job_executions good_job_processes good_job_settings
+                             good_job_batches]),
+      "delayed_job_active_record" => owned(%w[delayed_jobs]),
+      "flipper-active_record" => owned(%w[flipper_features flipper_gates]),
+      "acts-as-taggable-on" => owned(%w[tags taggings]),
+      "doorkeeper" => owned(%w[oauth_applications oauth_access_grants oauth_access_tokens]),
+      "noticed" => owned(%w[noticed_events noticed_notifications]),
+      "activerecord-session_store" => owned(%w[sessions]),
+      # Columns these gems add to app tables; the first is the anchor.
+      "discard" => { "*" => %w[discarded_at] },
+      "acts_as_list" => { "*" => %w[position] },
+      "ancestry" => { "*" => %w[ancestry ancestry_depth children_count] }
     }.freeze
 
     # Table names an installed gem creates and owns outright -- as opposed to

data/lib/schema_reaper/inflect.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SchemaReaper
+  # Just enough of ActiveSupport's inflector to map model class names to
+  # table names without depending on it. A wrong guess only weakens a
+  # heuristic -- it never makes a finding more confident.
+  module Inflect
+    module_function
+
+    def underscore(str)
+      str.gsub(/([A-Z\d]+)([A-Z][a-z])/, '\\1_\\2').gsub(/([a-z\d])([A-Z])/, '\\1_\\2').downcase
+    end
+
+    def pluralize(word)
+      case word
+      when /[^aeiou]y\z/ then word.sub(/y\z/, "ies")
+      when /(s|x|z|ch|sh)\z/ then "#{word}es"
+      else "#{word}s"
+      end
+    end
+
+    def singularize(word)
+      case word
+      when /ies\z/ then word.sub(/ies\z/, "y")
+      when /(ss|sh|ch|x|z)es\z/ then word.sub(/es\z/, "")
+      when /s\z/ then word.sub(/s\z/, "")
+      else word
+      end
+    end
+
+    # User -> users, LineItem -> line_items, Address -> addresses
+    def tableize(class_name)
+      pluralize(underscore(class_name))
+    end
+  end
+end