schema_reaper 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/.schema_reaper.yml.example +17 -0
  4. data/CHANGELOG.md +144 -0
  5. data/PRO.md +11 -6
  6. data/README.md +421 -71
  7. data/RELEASE_CHECKLIST.md +13 -2
  8. data/lib/generators/schema_reaper/install/install_generator.rb +11 -11
  9. data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +8 -0
  10. data/lib/schema_reaper/alert_config.rb +21 -1
  11. data/lib/schema_reaper/alert_state.rb +35 -0
  12. data/lib/schema_reaper/analyzers/always_null_column.rb +12 -5
  13. data/lib/schema_reaper/analyzers/base.rb +66 -2
  14. data/lib/schema_reaper/analyzers/dead_column.rb +12 -6
  15. data/lib/schema_reaper/analyzers/dead_table.rb +10 -6
  16. data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb +53 -0
  17. data/lib/schema_reaper/analyzers/duplicate_index.rb +1 -1
  18. data/lib/schema_reaper/analyzers/missing_fk_index.rb +10 -4
  19. data/lib/schema_reaper/analyzers/sequence_exhaustion.rb +55 -0
  20. data/lib/schema_reaper/analyzers/single_value_column.rb +17 -5
  21. data/lib/schema_reaper/analyzers/unused_index.rb +11 -6
  22. data/lib/schema_reaper/baseline.rb +95 -10
  23. data/lib/schema_reaper/cli/baseline_support.rb +45 -0
  24. data/lib/schema_reaper/cli.rb +69 -14
  25. data/lib/schema_reaper/config.rb +28 -0
  26. data/lib/schema_reaper/doctor.rb +151 -0
  27. data/lib/schema_reaper/explain.rb +102 -0
  28. data/lib/schema_reaper/gem_awareness.rb +31 -1
  29. data/lib/schema_reaper/inflect.rb +36 -0
  30. data/lib/schema_reaper/introspect/postgres.rb +258 -55
  31. data/lib/schema_reaper/migration_generator.rb +90 -4
  32. data/lib/schema_reaper/notifier.rb +69 -8
  33. data/lib/schema_reaper/railtie.rb +1 -1
  34. data/lib/schema_reaper/reporters/sarif.rb +15 -6
  35. data/lib/schema_reaper/reporters/slack.rb +67 -0
  36. data/lib/schema_reaper/runner.rb +58 -6
  37. data/lib/schema_reaper/runtime.rb +94 -11
  38. data/lib/schema_reaper/scan_job.rb +24 -2
  39. data/lib/schema_reaper/schema.rb +32 -3
  40. data/lib/schema_reaper/schema_locator.rb +75 -0
  41. data/lib/schema_reaper/static/cache.rb +66 -0
  42. data/lib/schema_reaper/static/scanner.rb +174 -6
  43. data/lib/schema_reaper/tasks/schema_reaper.rake +2 -2
  44. data/lib/schema_reaper/version.rb +1 -1
  45. data/lib/schema_reaper.rb +18 -1
  46. data/schema_reaper.gemspec +1 -2
  47. metadata +13 -4
@@ -2,34 +2,119 @@
2
2
 
3
3
  require "json"
4
4
  require "fileutils"
5
+ require "date"
5
6
 
6
7
  module SchemaReaper
7
8
  # Persists known finding ids so CI only fails on *new* dead weight.
9
+ #
10
+ # Each accepted finding can carry a reason and an expiry date. Once an entry
11
+ # expires, `scan --ci` treats the finding as new again, so "we'll deal with
12
+ # it later" comes back around instead of living in the file forever.
13
+ #
14
+ # Reads the 2.0 format ({"ids": [...]}) as entries with no reason or expiry,
15
+ # and keeps writing "ids" alongside "entries" so a 2.0.x gem sharing the
16
+ # file (mixed versions across a team or CI) still sees the accepted set.
17
+ #
18
+ # scope: analyzer types (e.g. %w[dead_column]) a write or prune is limited
19
+ # to -- what `--only` / `--except` selected. Entries of other analyzers
20
+ # were not looked at in this run, so they are left exactly as they are.
8
21
  class Baseline
22
+ Entry = Struct.new(:id, :reason, :expires_on, keyword_init: true) do
23
+ def expired?(today = Date.today)
24
+ !expires_on.nil? && expires_on < today
25
+ end
26
+
27
+ def type
28
+ id.split("/").first
29
+ end
30
+
31
+ def to_h
32
+ { "id" => id, "reason" => reason, "expires_on" => expires_on&.iso8601 }.compact
33
+ end
34
+ end
35
+
9
36
  def initialize(path)
10
37
  @path = path
11
38
  end
12
39
 
40
+ def entries
41
+ @entries ||= read_entries
42
+ end
43
+
44
+ # Ids still accepted today (expired entries excluded).
13
45
  def ids
14
- return [] unless File.exist?(@path)
46
+ entries.reject(&:expired?).map(&:id)
47
+ end
15
48
 
16
- JSON.parse(File.read(@path)).fetch("ids", [])
17
- rescue JSON::ParserError
18
- []
49
+ def expired
50
+ entries.select(&:expired?)
51
+ end
52
+
53
+ # Replaces the (in-scope part of the) baseline with exactly these
54
+ # findings. A reason / expiry passed in applies to new entries and to
55
+ # expired ones being re-accepted -- that is how an expired entry gets
56
+ # renewed; unexpired entries keep what they had.
57
+ def write(findings, reason: nil, expires_on: nil, scope: nil)
58
+ known = entries.to_h { |e| [e.id, e] }
59
+ kept = entries.reject { |e| in_scope?(e, scope) }
60
+ written = findings.map(&:id).uniq.map do |id|
61
+ entry = known[id]
62
+ next Entry.new(id: id, reason: reason, expires_on: expires_on) unless entry
63
+ next renew(entry, reason, expires_on) if entry.expired?
64
+
65
+ entry
66
+ end
67
+ @entries = (kept + written).uniq(&:id).sort_by(&:id)
68
+ save
19
69
  end
20
70
 
21
- def write(findings)
71
+ # Drops entries for findings that no longer occur, without accepting any
72
+ # new ones -- so a fixed finding can't hide a later regression with the
73
+ # same id. @return [Array<String>] dropped ids
74
+ def prune(findings, scope: nil)
75
+ current = findings.map(&:id)
76
+ dropped = entries.select { |e| in_scope?(e, scope) && !current.include?(e.id) }.map(&:id)
77
+ @entries = entries.reject { |e| dropped.include?(e.id) }
78
+ save
79
+ dropped
80
+ end
81
+
82
+ # findings not present in the stored baseline (or whose entry expired)
83
+ def new_among(findings)
84
+ known = ids
85
+ findings.reject { |f| known.include?(f.id) }
86
+ end
87
+
88
+ private
89
+
90
+ def in_scope?(entry, scope)
91
+ scope.nil? || scope.include?(entry.type)
92
+ end
93
+
94
+ def renew(entry, reason, expires_on)
95
+ Entry.new(id: entry.id, reason: reason || entry.reason, expires_on: expires_on)
96
+ end
97
+
98
+ def save
22
99
  FileUtils.mkdir_p(File.dirname(@path))
23
100
  File.write(@path, JSON.pretty_generate(
24
101
  "generated_at" => Time.now.utc.iso8601,
25
- "ids" => findings.map(&:id).sort
102
+ "ids" => ids.sort, # read by 2.0.x; unexpired only
103
+ "entries" => entries.map(&:to_h)
26
104
  ))
27
105
  end
28
106
 
29
- # findings not present in the stored baseline
30
- def new_among(findings)
31
- known = ids
32
- findings.reject { |f| known.include?(f.id) }
107
+ def read_entries
108
+ return [] unless File.exist?(@path)
109
+
110
+ data = JSON.parse(File.read(@path))
111
+ return data.fetch("ids", []).map { |id| Entry.new(id: id) } unless data.key?("entries")
112
+
113
+ data["entries"].map do |e|
114
+ Entry.new(id: e["id"], reason: e["reason"], expires_on: e["expires_on"] && Date.parse(e["expires_on"]))
115
+ end
116
+ rescue JSON::ParserError, ArgumentError
117
+ []
33
118
  end
34
119
  end
35
120
  end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+
5
+ module SchemaReaper
6
+ class CLI < Thor
7
+ # Output helpers for `baseline` and the `scan --ci` gate.
8
+ module BaselineSupport
9
+ private
10
+
11
+ def report_written(findings, scope)
12
+ console.ok("recorded #{findings.size} finding#{"s" unless findings.size == 1} " \
13
+ "as the accepted baseline")
14
+ console.info("file: #{config.baseline_path}")
15
+ console.info("only #{scope.join(", ")} entries were replaced; other analyzers' are unchanged") if scope
16
+ console.info("`scan --ci` now fails only on findings that appear after this point.")
17
+ end
18
+
19
+ def report_pruned(dropped)
20
+ return console.ok("nothing to prune -- every baseline entry still occurs") if dropped.empty?
21
+
22
+ console.ok("dropped #{dropped.size} resolved entr#{dropped.size == 1 ? "y" : "ies"}")
23
+ dropped.each { |id| console.info(" #{id}") }
24
+ end
25
+
26
+ def parse_date(value)
27
+ return nil unless value
28
+
29
+ Date.iso8601(value)
30
+ rescue ArgumentError
31
+ console.problem("--expires must be a date like 2027-01-31, got #{value.inspect}")
32
+ exit 1
33
+ end
34
+
35
+ def warn_expired(expired, findings)
36
+ current = findings.map(&:id)
37
+ live = expired.select { |e| current.include?(e.id) }
38
+ return if live.empty?
39
+
40
+ console.notice("#{live.size} baseline entr#{live.size == 1 ? "y has" : "ies have"} expired and count as new:")
41
+ live.each { |e| console.info(" #{e.id} (expired #{e.expires_on}#{" -- #{e.reason}" if e.reason})") }
42
+ end
43
+ end
44
+ end
45
+ end
@@ -2,16 +2,23 @@
2
2
 
3
3
  require "thor"
4
4
  require_relative "../schema_reaper"
5
+ require_relative "cli/baseline_support"
5
6
 
6
7
  module SchemaReaper
7
8
  # Command-line entry point. See `exe/schema_reaper`.
8
9
  class CLI < Thor
10
+ include BaselineSupport
11
+
9
12
  def self.exit_on_failure?
10
13
  true
11
14
  end
12
15
 
13
16
  class_option :config, type: :string, default: ".schema_reaper.yml",
14
17
  desc: "path to config file"
18
+ class_option :color, type: :boolean, default: nil,
19
+ desc: "force colour on/off (default: auto)"
20
+ class_option :only, type: :array, desc: "run only these analyzers (e.g. dead_column unused_index)"
21
+ class_option :except, type: :array, desc: "skip these analyzers"
15
22
 
16
23
  desc "scan", "Scan schema + code and report dead weight"
17
24
  option :format, type: :string, default: "table",
@@ -20,11 +27,10 @@ module SchemaReaper
20
27
  desc: "exit non-zero on findings not in the baseline"
21
28
  option :record, type: :boolean, default: false,
22
29
  desc: "append this run to the history log"
23
- option :min_confidence, type: :numeric, default: 0.0
24
- option :color, type: :boolean, default: nil,
25
- desc: "force colour on/off for the table report (default: auto)"
30
+ option :min_confidence, type: :numeric,
31
+ desc: "hide findings below this confidence (default: min_confidence in config, 0.0)"
26
32
  def scan
27
- findings = run.select { |f| f.confidence >= options[:min_confidence] }
33
+ findings = run(min_confidence: options[:min_confidence])
28
34
  render_report(findings)
29
35
 
30
36
  History.new(config.history_log).record(findings) if options[:record]
@@ -32,14 +38,20 @@ module SchemaReaper
32
38
  end
33
39
 
34
40
  desc "baseline", "Write current findings to the baseline file"
41
+ option :reason, type: :string, desc: "why these findings are accepted (set on new and expired entries)"
42
+ option :expires, type: :string, desc: "YYYY-MM-DD after which `scan --ci` fails on them again " \
43
+ "(renews expired entries)"
44
+ option :prune, type: :boolean, default: false,
45
+ desc: "only drop entries for findings that no longer occur; accept nothing new"
35
46
  def baseline
36
47
  findings = run
37
- Baseline.new(config.baseline_path).write(findings)
48
+ store = Baseline.new(config.baseline_path)
38
49
  console.title("baseline")
39
- console.ok("recorded #{findings.size} finding#{"s" unless findings.size == 1} " \
40
- "as the accepted baseline")
41
- console.info("file: #{config.baseline_path}")
42
- console.info("`scan --ci` now fails only on findings that appear after this point.")
50
+ scope = runner.selected_types
51
+ return report_pruned(store.prune(findings, scope: scope)) if options[:prune]
52
+
53
+ store.write(findings, reason: options[:reason], expires_on: parse_date(options[:expires]), scope: scope)
54
+ report_written(findings, scope)
43
55
  end
44
56
 
45
57
  desc "trend", "Append a snapshot and print progress over time"
@@ -49,8 +61,17 @@ module SchemaReaper
49
61
  Reporters::Trend.new(history.trend, color: options[:color]).render
50
62
  end
51
63
 
52
- desc "generate-migration TABLE COLUMN", "Emit a staged removal migration pair"
53
- def generate_migration(table, column)
64
+ desc "generate-migration TABLE [COLUMN]",
65
+ "Emit a staged removal migration pair for a column, or a removal migration for an index (--index) or table"
66
+ option :index, type: :string, desc: "name of an index on TABLE to remove instead of a column"
67
+ def generate_migration(table, column = nil)
68
+ if options[:index] && column
69
+ console.problem("pass either COLUMN or --index, not both")
70
+ exit 1
71
+ end
72
+ return generate_index_migration(table, options[:index]) if options[:index]
73
+ return generate_table_migration(table) unless column
74
+
54
75
  paths = MigrationGenerator.new(table: table, column: column).call
55
76
  console.title("generate-migration", "#{table}.#{column}")
56
77
  console.ok("created two migrations:")
@@ -62,6 +83,16 @@ module SchemaReaper
62
83
  ])
63
84
  end
64
85
 
86
+ desc "explain ID", "Show everything known about one finding and the command to act on it"
87
+ def explain(id)
88
+ exit 1 unless Explain.new(run, config: config, console: console).call(id)
89
+ end
90
+
91
+ desc "doctor", "Check the database, statistics, runtime log, baseline and alert setup"
92
+ def doctor
93
+ exit 1 unless Doctor.new(config: config, console: console).call
94
+ end
95
+
65
96
  desc "version", "Print version"
66
97
  def version
67
98
  say(SchemaReaper::VERSION)
@@ -69,12 +100,34 @@ module SchemaReaper
69
100
 
70
101
  private
71
102
 
103
+ def generate_index_migration(table, index)
104
+ path = MigrationGenerator.new(table: table, index: index).call.first
105
+ console.title("generate-migration", "#{table} · #{index}")
106
+ console.ok("created #{path}")
107
+ console.info("runs with algorithm: :concurrently, so it does not lock writes to #{table}")
108
+ end
109
+
110
+ def generate_table_migration(table)
111
+ path = MigrationGenerator.new(table: table).call.first
112
+ console.title("generate-migration", table)
113
+ console.ok("created #{path}")
114
+ console.list("before running it:", [
115
+ "confirm no external consumer (reports, ETL, other services) reads #{table}",
116
+ "take a backup -- its `down` cannot bring the data back"
117
+ ])
118
+ end
119
+
72
120
  def config
73
121
  @config ||= Config.load(options[:config])
74
122
  end
75
123
 
76
- def run
77
- Runner.new(config: config).run
124
+ def runner(min_confidence: nil)
125
+ @runner ||= Runner.new(config: config, only: options[:only], except: options[:except],
126
+ min_confidence: min_confidence)
127
+ end
128
+
129
+ def run(min_confidence: nil)
130
+ runner(min_confidence: min_confidence).run
78
131
  rescue SchemaReaper::Error => e
79
132
  console.problem(e.message)
80
133
  exit 1
@@ -93,7 +146,9 @@ module SchemaReaper
93
146
  end
94
147
 
95
148
  def enforce_baseline(findings)
96
- new_ones = Baseline.new(config.baseline_path).new_among(findings)
149
+ store = Baseline.new(config.baseline_path)
150
+ warn_expired(store.expired, findings)
151
+ new_ones = store.new_among(findings)
97
152
  return if new_ones.empty?
98
153
 
99
154
  console.problem(
@@ -9,6 +9,7 @@ module SchemaReaper
9
9
  class Config
10
10
  DEFAULTS = {
11
11
  "database_url" => nil, # falls back to ENV["DATABASE_URL"]
12
+ "schemas" => %w[public], # PostgreSQL schemas to introspect
12
13
  "scan_paths" => %w[app lib config],
13
14
  "view_globs" => %w[
14
15
  app/**/*.erb app/**/*.haml app/**/*.slim app/**/*.jbuilder
@@ -20,9 +21,16 @@ module SchemaReaper
20
21
  "always_keep_columns" => %w[id created_at updated_at type],
21
22
  "gem_awareness" => true, # auto-whitelist columns owned by known gems
22
23
  "min_age_days" => 14,
24
+ "min_confidence" => 0.0, # drop findings below this from every command
25
+ "runtime_log_max_bytes" => 50 * 1024 * 1024, # rotate the usage log past this size
23
26
  "runtime_log" => ".schema_reaper/runtime.jsonl",
24
27
  "history_log" => ".schema_reaper/history.jsonl",
25
28
  "baseline" => ".schema_reaper/baseline.json",
29
+ "scan_cache" => ".schema_reaper/scan_cache.json", # per-file scan cache; null disables
30
+ # global: a column counts as used if its name appears anywhere.
31
+ # model: a name in an app/models file only counts for that model's
32
+ # table and the tables it mentions -- stricter, finds more.
33
+ "column_scope" => "global",
26
34
  "database_yml" => "config/database.yml", # Rails fallback for the connection
27
35
  "require" => [] # extra files to load (custom analyzers)
28
36
  }.freeze
@@ -74,6 +82,18 @@ module SchemaReaper
74
82
  @data.dig("ignore", "tables").to_a
75
83
  end
76
84
 
85
+ def schemas
86
+ Array(@data["schemas"]).map(&:to_s)
87
+ end
88
+
89
+ def min_confidence
90
+ @data["min_confidence"].to_f
91
+ end
92
+
93
+ def runtime_log_max_bytes
94
+ @data["runtime_log_max_bytes"]
95
+ end
96
+
77
97
  def min_age_days
78
98
  @data["min_age_days"]
79
99
  end
@@ -90,6 +110,14 @@ module SchemaReaper
90
110
  @data["history_log"]
91
111
  end
92
112
 
113
+ def scan_cache
114
+ @data["scan_cache"]
115
+ end
116
+
117
+ def column_scope
118
+ @data["column_scope"].to_s == "model" ? :model : :global
119
+ end
120
+
93
121
  def gem_awareness?
94
122
  @data["gem_awareness"] != false
95
123
  end
@@ -0,0 +1,151 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module SchemaReaper
6
+ # `schema_reaper doctor`: checks everything a scan's accuracy depends on and
7
+ # says what to fix, so a quiet report can be trusted to mean "nothing dead"
8
+ # rather than "nothing measured".
9
+ class Doctor
10
+ Check = Struct.new(:status, :label, :detail, keyword_init: true) # status: :ok | :warn | :fail
11
+
12
+ STATS_FRESH_DAYS = 7
13
+ ICON = { ok: "✓", warn: "!", fail: "✗" }.freeze
14
+
15
+ def initialize(config:, root: Dir.pwd, introspector: nil, console: Reporters::Console.new)
16
+ @config = config
17
+ @root = root
18
+ @introspector = introspector
19
+ @console = console
20
+ @checks = []
21
+ end
22
+
23
+ # @return [Boolean] true unless a check failed outright
24
+ def call
25
+ check_scan_paths
26
+ check_database
27
+ check_runtime_log
28
+ check_baseline
29
+ check_schema_file
30
+ check_alerts
31
+ render
32
+ @checks.none? { |c| c.status == :fail }
33
+ end
34
+
35
+ private
36
+
37
+ def ok(label, detail = nil)
38
+ @checks << Check.new(status: :ok, label: label, detail: detail)
39
+ end
40
+
41
+ def warn_check(label, detail)
42
+ @checks << Check.new(status: :warn, label: label, detail: detail)
43
+ end
44
+
45
+ def fail_check(label, detail)
46
+ @checks << Check.new(status: :fail, label: label, detail: detail)
47
+ end
48
+
49
+ def check_scan_paths
50
+ missing = @config.scan_paths.reject { |p| Dir.exist?(File.join(@root, p)) }
51
+ return ok("scan paths", @config.scan_paths.join(", ")) if missing.empty?
52
+
53
+ warn_check("scan paths", "not found: #{missing.join(", ")} -- code there can't mark anything as used")
54
+ end
55
+
56
+ def check_database
57
+ intro = @introspector || Introspect::Postgres.new(@config.database_url, schemas: @config.schemas)
58
+ diag = intro.diagnostics
59
+ ok("database", "connected, PostgreSQL #{diag[:server_version]}")
60
+ check_stats_reset(diag[:stats_reset])
61
+ check_analyze(diag[:never_analyzed])
62
+ check_query_history(intro.call)
63
+ if diag[:in_recovery]
64
+ warn_check("replica", "this is a read replica -- index usage here doesn't cover the primary")
65
+ end
66
+ rescue Error => e
67
+ fail_check("database", e.message)
68
+ end
69
+
70
+ def check_stats_reset(reset_at)
71
+ return ok("statistics", "never reset") if reset_at.nil?
72
+
73
+ days = ((Time.now - Time.parse(reset_at.to_s)) / 86_400).floor
74
+ return ok("statistics", "collected for #{days} day(s)") if days >= STATS_FRESH_DAYS
75
+
76
+ warn_check("statistics", "reset #{days} day(s) ago -- unused_index needs a longer window to be trusted")
77
+ end
78
+
79
+ def check_analyze(tables)
80
+ return ok("ANALYZE", "every table has planner statistics") if tables.empty?
81
+
82
+ sample = tables.first(5).join(", ") + (tables.size > 5 ? ", ..." : "")
83
+ warn_check("ANALYZE", "#{tables.size} table(s) never analysed (#{sample}) -- " \
84
+ "row counts and null/distinct checks are blind there; run ANALYZE")
85
+ end
86
+
87
+ def check_query_history(schema)
88
+ return ok("query history", "#{schema.index_scan_total} index scans recorded") if schema.query_history?
89
+
90
+ warn_check("query history", "only #{schema.index_scan_total} index scan(s) -- unused_index will sit out; " \
91
+ "scan a database that serves real traffic")
92
+ end
93
+
94
+ def check_runtime_log
95
+ report = Runtime::Report.load(@config.runtime_log)
96
+ need = @config.min_age_days || Config::DEFAULTS["min_age_days"]
97
+ if report.empty?
98
+ warn_check("runtime log", "none at #{@config.runtime_log} -- dead_column confidence stays capped at " \
99
+ "#{Analyzers::DeadColumn::STATIC_ONLY_CAP}; see README > Runtime signal")
100
+ elsif report.observed_days < need
101
+ warn_check("runtime log", "#{report.observed_days} of #{need} day(s) observed -- not trusted yet")
102
+ else
103
+ ok("runtime log", "#{report.observed_days} day(s), #{report.accessed.size} column(s) seen")
104
+ end
105
+ end
106
+
107
+ def check_baseline
108
+ base = Baseline.new(@config.baseline_path)
109
+ return ok("baseline", "none (scan --ci would fail on every finding)") if base.entries.empty?
110
+
111
+ expired = base.expired
112
+ return ok("baseline", "#{base.entries.size} accepted finding(s)") if expired.empty?
113
+
114
+ warn_check("baseline", "#{expired.size} entr#{expired.size == 1 ? "y has" : "ies have"} expired: " \
115
+ "#{expired.map(&:id).first(3).join(", ")}")
116
+ end
117
+
118
+ def check_schema_file
119
+ path = SchemaLocator::CANDIDATES.find { |p| File.exist?(File.join(@root, p)) }
120
+ return ok("schema file", path) if path
121
+
122
+ warn_check("schema file", "no db/schema.rb or db/structure.sql -- SARIF results can't point at a line")
123
+ end
124
+
125
+ # Only meaningful inside a Rails app with the install generator's
126
+ # initializer loaded.
127
+ def check_alerts
128
+ return unless defined?(AlertConfig) && defined?(Rails)
129
+
130
+ cfg = AlertConfig.instance
131
+ unless cfg.configured?
132
+ return warn_check("alerts", "no emails or webhook_url configured -- scheduled scans report nowhere")
133
+ end
134
+
135
+ channels = [("email (#{cfg.emails.size})" if cfg.emails?), ("webhook" if cfg.webhook?)].compact
136
+ ok("alerts", "#{channels.join(" + ")}, alert_on: #{cfg.alert_on}")
137
+ end
138
+
139
+ def render
140
+ @console.title("doctor")
141
+ @checks.each do |c|
142
+ line = format("%<icon>s %<label>-14s %<detail>s", icon: ICON[c.status], label: c.label, detail: c.detail)
143
+ @console.info(line)
144
+ end
145
+ @console.blank
146
+ counts = @checks.group_by(&:status).transform_values(&:size)
147
+ @console.info("#{counts.fetch(:ok, 0)} ok, #{counts.fetch(:warn, 0)} warning(s), " \
148
+ "#{counts.fetch(:fail, 0)} failed")
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SchemaReaper
4
+ # `schema_reaper explain ID`: everything known about one finding, and the
5
+ # exact command to act on it.
6
+ class Explain
7
+ def initialize(findings, config:, root: Dir.pwd, console: Reporters::Console.new)
8
+ @findings = findings
9
+ @config = config
10
+ @root = root
11
+ @console = console
12
+ end
13
+
14
+ # @return [Boolean] false when no finding matches
15
+ def call(query)
16
+ matches = match(query)
17
+ if matches.size == 1
18
+ render(matches.first)
19
+ elsif matches.empty?
20
+ report_not_found(query)
21
+ else
22
+ report_ambiguous(query, matches)
23
+ end
24
+ matches.size == 1
25
+ end
26
+
27
+ private
28
+
29
+ # An exact id, else every finding whose id contains the query
30
+ # ("users/legacy_token" finds dead_column/users/legacy_token).
31
+ def match(query)
32
+ exact = @findings.select { |f| f.id == query }
33
+ exact.any? ? exact : @findings.select { |f| f.id.include?(query) }
34
+ end
35
+
36
+ def report_not_found(query)
37
+ @console.problem("no current finding matches #{query.inspect}")
38
+ table_finding = @findings.find { |f| f.type == :dead_table && query.split("/").include?(f.table) }
39
+ if table_finding
40
+ @console.info("its table is reported whole as #{table_finding.id} -- per-column and per-index " \
41
+ "findings on a dead table are folded into that one")
42
+ return
43
+ end
44
+
45
+ @console.info("list them with `schema_reaper scan --format json` -- ids look like dead_column/users/legacy_token")
46
+ end
47
+
48
+ def report_ambiguous(query, matches)
49
+ @console.problem("#{matches.size} findings match #{query.inspect} -- be more specific:",
50
+ items: matches.map(&:id))
51
+ end
52
+
53
+ def render(finding)
54
+ @console.title("explain", finding.id)
55
+ @console.row("type", finding.type)
56
+ @console.row("severity", finding.severity)
57
+ @console.row("confidence", format("%.2f", finding.confidence))
58
+ @console.row("reclaimable", reclaim(finding))
59
+ @console.row("defined at", location(finding)) if location(finding)
60
+ @console.row("baseline", baseline_status(finding))
61
+ @console.blank
62
+ @console.list("evidence", finding.evidence, colour: :reset)
63
+ @console.blank
64
+ @console.list("fix", [finding.suggested_fix], colour: :reset)
65
+ next_step = next_command(finding)
66
+ @console.list("next", [next_step], colour: :reset) if next_step
67
+ end
68
+
69
+ def reclaim(finding)
70
+ return "unknown (row count unavailable -- run ANALYZE)" unless finding.reclaim_known?
71
+
72
+ Reporters::Bytes.human(finding.reclaimable_bytes)
73
+ end
74
+
75
+ def location(finding)
76
+ @location ||= {}
77
+ @location[finding.id] ||= begin
78
+ loc = SchemaLocator.new(root: @root).locate(finding)
79
+ loc && "#{loc.path}:#{loc.line}"
80
+ end
81
+ end
82
+
83
+ def baseline_status(finding)
84
+ entry = Baseline.new(@config.baseline_path).entries.find { |e| e.id == finding.id }
85
+ return "not accepted (fails `scan --ci`)" unless entry
86
+ return "accepted, but expired #{entry.expires_on}" if entry.expired?
87
+
88
+ ["accepted", entry.reason, entry.expires_on && "until #{entry.expires_on}"].compact.join(" -- ")
89
+ end
90
+
91
+ def next_command(finding)
92
+ case finding.type
93
+ when :dead_column, :always_null_column, :single_value_column
94
+ "bundle exec schema_reaper generate-migration #{finding.table} #{finding.column}"
95
+ when :unused_index, :duplicate_index
96
+ "bundle exec schema_reaper generate-migration #{finding.table} --index #{finding.index}"
97
+ when :dead_table
98
+ "bundle exec schema_reaper generate-migration #{finding.table}"
99
+ end
100
+ end
101
+ end
102
+ end
@@ -9,6 +9,18 @@ module SchemaReaper
9
9
  module GemAwareness
10
10
  # gem name => { table_glob => [column names] }. "*" table_glob matches any
11
11
  # table; an exact name matches only that table.
12
+ def self.owned(tables)
13
+ tables.to_h { |t| [t, []] }
14
+ end
15
+ private_class_method :owned
16
+
17
+ SOLID_QUEUE_TABLES = %w[
18
+ solid_queue_jobs solid_queue_ready_executions solid_queue_scheduled_executions
19
+ solid_queue_claimed_executions solid_queue_blocked_executions solid_queue_failed_executions
20
+ solid_queue_recurring_executions solid_queue_recurring_tasks solid_queue_pauses
21
+ solid_queue_processes solid_queue_semaphores
22
+ ].freeze
23
+
12
24
  MAP = {
13
25
  "devise" => { "*" => %w[
14
26
  encrypted_password reset_password_token reset_password_sent_at
@@ -42,7 +54,25 @@ module SchemaReaper
42
54
  resource_owner_type resource_owner_id access_token refresh_token
43
55
  expires_in revoked_at previous_refresh_token
44
56
  ]
45
- }
57
+ },
58
+ # Gems below own their tables outright and touch every column from
59
+ # their own code, so the tables are listed with no columns: ownership
60
+ # alone keeps every analyzer off them (see Base#gem_reserved?).
61
+ "solid_queue" => owned(SOLID_QUEUE_TABLES),
62
+ "solid_cache" => owned(%w[solid_cache_entries]),
63
+ "solid_cable" => owned(%w[solid_cable_messages]),
64
+ "good_job" => owned(%w[good_jobs good_job_executions good_job_processes good_job_settings
65
+ good_job_batches]),
66
+ "delayed_job_active_record" => owned(%w[delayed_jobs]),
67
+ "flipper-active_record" => owned(%w[flipper_features flipper_gates]),
68
+ "acts-as-taggable-on" => owned(%w[tags taggings]),
69
+ "doorkeeper" => owned(%w[oauth_applications oauth_access_grants oauth_access_tokens]),
70
+ "noticed" => owned(%w[noticed_events noticed_notifications]),
71
+ "activerecord-session_store" => owned(%w[sessions]),
72
+ # Columns these gems add to app tables; the first is the anchor.
73
+ "discard" => { "*" => %w[discarded_at] },
74
+ "acts_as_list" => { "*" => %w[position] },
75
+ "ancestry" => { "*" => %w[ancestry ancestry_depth children_count] }
46
76
  }.freeze
47
77
 
48
78
  # Table names an installed gem creates and owns outright -- as opposed to