schema_reaper 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/.schema_reaper.yml.example +17 -0
- data/CHANGELOG.md +144 -0
- data/PRO.md +11 -6
- data/README.md +421 -71
- data/RELEASE_CHECKLIST.md +13 -2
- data/lib/generators/schema_reaper/install/install_generator.rb +11 -11
- data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +8 -0
- data/lib/schema_reaper/alert_config.rb +21 -1
- data/lib/schema_reaper/alert_state.rb +35 -0
- data/lib/schema_reaper/analyzers/always_null_column.rb +12 -5
- data/lib/schema_reaper/analyzers/base.rb +66 -2
- data/lib/schema_reaper/analyzers/dead_column.rb +12 -6
- data/lib/schema_reaper/analyzers/dead_table.rb +10 -6
- data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb +53 -0
- data/lib/schema_reaper/analyzers/duplicate_index.rb +1 -1
- data/lib/schema_reaper/analyzers/missing_fk_index.rb +10 -4
- data/lib/schema_reaper/analyzers/sequence_exhaustion.rb +55 -0
- data/lib/schema_reaper/analyzers/single_value_column.rb +17 -5
- data/lib/schema_reaper/analyzers/unused_index.rb +11 -6
- data/lib/schema_reaper/baseline.rb +95 -10
- data/lib/schema_reaper/cli/baseline_support.rb +45 -0
- data/lib/schema_reaper/cli.rb +69 -14
- data/lib/schema_reaper/config.rb +28 -0
- data/lib/schema_reaper/doctor.rb +151 -0
- data/lib/schema_reaper/explain.rb +102 -0
- data/lib/schema_reaper/gem_awareness.rb +31 -1
- data/lib/schema_reaper/inflect.rb +36 -0
- data/lib/schema_reaper/introspect/postgres.rb +258 -55
- data/lib/schema_reaper/migration_generator.rb +90 -4
- data/lib/schema_reaper/notifier.rb +69 -8
- data/lib/schema_reaper/railtie.rb +1 -1
- data/lib/schema_reaper/reporters/sarif.rb +15 -6
- data/lib/schema_reaper/reporters/slack.rb +67 -0
- data/lib/schema_reaper/runner.rb +58 -6
- data/lib/schema_reaper/runtime.rb +94 -11
- data/lib/schema_reaper/scan_job.rb +24 -2
- data/lib/schema_reaper/schema.rb +32 -3
- data/lib/schema_reaper/schema_locator.rb +75 -0
- data/lib/schema_reaper/static/cache.rb +66 -0
- data/lib/schema_reaper/static/scanner.rb +174 -6
- data/lib/schema_reaper/tasks/schema_reaper.rake +2 -2
- data/lib/schema_reaper/version.rb +1 -1
- data/lib/schema_reaper.rb +18 -1
- data/schema_reaper.gemspec +1 -2
- metadata +13 -4
|
@@ -2,34 +2,119 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "fileutils"
|
|
5
|
+
require "date"
|
|
5
6
|
|
|
6
7
|
module SchemaReaper
|
|
7
8
|
# Persists known finding ids so CI only fails on *new* dead weight.
|
|
9
|
+
#
|
|
10
|
+
# Each accepted finding can carry a reason and an expiry date. Once an entry
|
|
11
|
+
# expires, `scan --ci` treats the finding as new again, so "we'll deal with
|
|
12
|
+
# it later" comes back around instead of living in the file forever.
|
|
13
|
+
#
|
|
14
|
+
# Reads the 2.0 format ({"ids": [...]}) as entries with no reason or expiry,
|
|
15
|
+
# and keeps writing "ids" alongside "entries" so a 2.0.x gem sharing the
|
|
16
|
+
# file (mixed versions across a team or CI) still sees the accepted set.
|
|
17
|
+
#
|
|
18
|
+
# scope: analyzer types (e.g. %w[dead_column]) a write or prune is limited
|
|
19
|
+
# to -- what `--only` / `--except` selected. Entries of other analyzers
|
|
20
|
+
# were not looked at in this run, so they are left exactly as they are.
|
|
8
21
|
class Baseline
|
|
22
|
+
Entry = Struct.new(:id, :reason, :expires_on, keyword_init: true) do
|
|
23
|
+
def expired?(today = Date.today)
|
|
24
|
+
!expires_on.nil? && expires_on < today
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def type
|
|
28
|
+
id.split("/").first
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def to_h
|
|
32
|
+
{ "id" => id, "reason" => reason, "expires_on" => expires_on&.iso8601 }.compact
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
9
36
|
def initialize(path)
|
|
10
37
|
@path = path
|
|
11
38
|
end
|
|
12
39
|
|
|
40
|
+
def entries
|
|
41
|
+
@entries ||= read_entries
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Ids still accepted today (expired entries excluded).
|
|
13
45
|
def ids
|
|
14
|
-
|
|
46
|
+
entries.reject(&:expired?).map(&:id)
|
|
47
|
+
end
|
|
15
48
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
49
|
+
def expired
|
|
50
|
+
entries.select(&:expired?)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Replaces the (in-scope part of the) baseline with exactly these
|
|
54
|
+
# findings. A reason / expiry passed in applies to new entries and to
|
|
55
|
+
# expired ones being re-accepted -- that is how an expired entry gets
|
|
56
|
+
# renewed; unexpired entries keep what they had.
|
|
57
|
+
def write(findings, reason: nil, expires_on: nil, scope: nil)
|
|
58
|
+
known = entries.to_h { |e| [e.id, e] }
|
|
59
|
+
kept = entries.reject { |e| in_scope?(e, scope) }
|
|
60
|
+
written = findings.map(&:id).uniq.map do |id|
|
|
61
|
+
entry = known[id]
|
|
62
|
+
next Entry.new(id: id, reason: reason, expires_on: expires_on) unless entry
|
|
63
|
+
next renew(entry, reason, expires_on) if entry.expired?
|
|
64
|
+
|
|
65
|
+
entry
|
|
66
|
+
end
|
|
67
|
+
@entries = (kept + written).uniq(&:id).sort_by(&:id)
|
|
68
|
+
save
|
|
19
69
|
end
|
|
20
70
|
|
|
21
|
-
|
|
71
|
+
# Drops entries for findings that no longer occur, without accepting any
|
|
72
|
+
# new ones -- so a fixed finding can't hide a later regression with the
|
|
73
|
+
# same id. @return [Array<String>] dropped ids
|
|
74
|
+
def prune(findings, scope: nil)
|
|
75
|
+
current = findings.map(&:id)
|
|
76
|
+
dropped = entries.select { |e| in_scope?(e, scope) && !current.include?(e.id) }.map(&:id)
|
|
77
|
+
@entries = entries.reject { |e| dropped.include?(e.id) }
|
|
78
|
+
save
|
|
79
|
+
dropped
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# findings not present in the stored baseline (or whose entry expired)
|
|
83
|
+
def new_among(findings)
|
|
84
|
+
known = ids
|
|
85
|
+
findings.reject { |f| known.include?(f.id) }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
def in_scope?(entry, scope)
|
|
91
|
+
scope.nil? || scope.include?(entry.type)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def renew(entry, reason, expires_on)
|
|
95
|
+
Entry.new(id: entry.id, reason: reason || entry.reason, expires_on: expires_on)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def save
|
|
22
99
|
FileUtils.mkdir_p(File.dirname(@path))
|
|
23
100
|
File.write(@path, JSON.pretty_generate(
|
|
24
101
|
"generated_at" => Time.now.utc.iso8601,
|
|
25
|
-
"ids" =>
|
|
102
|
+
"ids" => ids.sort, # read by 2.0.x; unexpired only
|
|
103
|
+
"entries" => entries.map(&:to_h)
|
|
26
104
|
))
|
|
27
105
|
end
|
|
28
106
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
107
|
+
def read_entries
|
|
108
|
+
return [] unless File.exist?(@path)
|
|
109
|
+
|
|
110
|
+
data = JSON.parse(File.read(@path))
|
|
111
|
+
return data.fetch("ids", []).map { |id| Entry.new(id: id) } unless data.key?("entries")
|
|
112
|
+
|
|
113
|
+
data["entries"].map do |e|
|
|
114
|
+
Entry.new(id: e["id"], reason: e["reason"], expires_on: e["expires_on"] && Date.parse(e["expires_on"]))
|
|
115
|
+
end
|
|
116
|
+
rescue JSON::ParserError, ArgumentError
|
|
117
|
+
[]
|
|
33
118
|
end
|
|
34
119
|
end
|
|
35
120
|
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
|
|
5
|
+
module SchemaReaper
|
|
6
|
+
class CLI < Thor
|
|
7
|
+
# Output helpers for `baseline` and the `scan --ci` gate.
|
|
8
|
+
module BaselineSupport
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def report_written(findings, scope)
|
|
12
|
+
console.ok("recorded #{findings.size} finding#{"s" unless findings.size == 1} " \
|
|
13
|
+
"as the accepted baseline")
|
|
14
|
+
console.info("file: #{config.baseline_path}")
|
|
15
|
+
console.info("only #{scope.join(", ")} entries were replaced; other analyzers' are unchanged") if scope
|
|
16
|
+
console.info("`scan --ci` now fails only on findings that appear after this point.")
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def report_pruned(dropped)
|
|
20
|
+
return console.ok("nothing to prune -- every baseline entry still occurs") if dropped.empty?
|
|
21
|
+
|
|
22
|
+
console.ok("dropped #{dropped.size} resolved entr#{dropped.size == 1 ? "y" : "ies"}")
|
|
23
|
+
dropped.each { |id| console.info(" #{id}") }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def parse_date(value)
|
|
27
|
+
return nil unless value
|
|
28
|
+
|
|
29
|
+
Date.iso8601(value)
|
|
30
|
+
rescue ArgumentError
|
|
31
|
+
console.problem("--expires must be a date like 2027-01-31, got #{value.inspect}")
|
|
32
|
+
exit 1
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def warn_expired(expired, findings)
|
|
36
|
+
current = findings.map(&:id)
|
|
37
|
+
live = expired.select { |e| current.include?(e.id) }
|
|
38
|
+
return if live.empty?
|
|
39
|
+
|
|
40
|
+
console.notice("#{live.size} baseline entr#{live.size == 1 ? "y has" : "ies have"} expired and count as new:")
|
|
41
|
+
live.each { |e| console.info(" #{e.id} (expired #{e.expires_on}#{" -- #{e.reason}" if e.reason})") }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
data/lib/schema_reaper/cli.rb
CHANGED
|
@@ -2,16 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
require "thor"
|
|
4
4
|
require_relative "../schema_reaper"
|
|
5
|
+
require_relative "cli/baseline_support"
|
|
5
6
|
|
|
6
7
|
module SchemaReaper
|
|
7
8
|
# Command-line entry point. See `exe/schema_reaper`.
|
|
8
9
|
class CLI < Thor
|
|
10
|
+
include BaselineSupport
|
|
11
|
+
|
|
9
12
|
def self.exit_on_failure?
|
|
10
13
|
true
|
|
11
14
|
end
|
|
12
15
|
|
|
13
16
|
class_option :config, type: :string, default: ".schema_reaper.yml",
|
|
14
17
|
desc: "path to config file"
|
|
18
|
+
class_option :color, type: :boolean, default: nil,
|
|
19
|
+
desc: "force colour on/off (default: auto)"
|
|
20
|
+
class_option :only, type: :array, desc: "run only these analyzers (e.g. dead_column unused_index)"
|
|
21
|
+
class_option :except, type: :array, desc: "skip these analyzers"
|
|
15
22
|
|
|
16
23
|
desc "scan", "Scan schema + code and report dead weight"
|
|
17
24
|
option :format, type: :string, default: "table",
|
|
@@ -20,11 +27,10 @@ module SchemaReaper
|
|
|
20
27
|
desc: "exit non-zero on findings not in the baseline"
|
|
21
28
|
option :record, type: :boolean, default: false,
|
|
22
29
|
desc: "append this run to the history log"
|
|
23
|
-
option :min_confidence, type: :numeric,
|
|
24
|
-
|
|
25
|
-
desc: "force colour on/off for the table report (default: auto)"
|
|
30
|
+
option :min_confidence, type: :numeric,
|
|
31
|
+
desc: "hide findings below this confidence (default: min_confidence in config, 0.0)"
|
|
26
32
|
def scan
|
|
27
|
-
findings = run
|
|
33
|
+
findings = run(min_confidence: options[:min_confidence])
|
|
28
34
|
render_report(findings)
|
|
29
35
|
|
|
30
36
|
History.new(config.history_log).record(findings) if options[:record]
|
|
@@ -32,14 +38,20 @@ module SchemaReaper
|
|
|
32
38
|
end
|
|
33
39
|
|
|
34
40
|
desc "baseline", "Write current findings to the baseline file"
|
|
41
|
+
option :reason, type: :string, desc: "why these findings are accepted (set on new and expired entries)"
|
|
42
|
+
option :expires, type: :string, desc: "YYYY-MM-DD after which `scan --ci` fails on them again " \
|
|
43
|
+
"(renews expired entries)"
|
|
44
|
+
option :prune, type: :boolean, default: false,
|
|
45
|
+
desc: "only drop entries for findings that no longer occur; accept nothing new"
|
|
35
46
|
def baseline
|
|
36
47
|
findings = run
|
|
37
|
-
Baseline.new(config.baseline_path)
|
|
48
|
+
store = Baseline.new(config.baseline_path)
|
|
38
49
|
console.title("baseline")
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
50
|
+
scope = runner.selected_types
|
|
51
|
+
return report_pruned(store.prune(findings, scope: scope)) if options[:prune]
|
|
52
|
+
|
|
53
|
+
store.write(findings, reason: options[:reason], expires_on: parse_date(options[:expires]), scope: scope)
|
|
54
|
+
report_written(findings, scope)
|
|
43
55
|
end
|
|
44
56
|
|
|
45
57
|
desc "trend", "Append a snapshot and print progress over time"
|
|
@@ -49,8 +61,17 @@ module SchemaReaper
|
|
|
49
61
|
Reporters::Trend.new(history.trend, color: options[:color]).render
|
|
50
62
|
end
|
|
51
63
|
|
|
52
|
-
desc "generate-migration TABLE COLUMN",
|
|
53
|
-
|
|
64
|
+
desc "generate-migration TABLE [COLUMN]",
|
|
65
|
+
"Emit a staged removal migration pair for a column, or a removal migration for an index (--index) or table"
|
|
66
|
+
option :index, type: :string, desc: "name of an index on TABLE to remove instead of a column"
|
|
67
|
+
def generate_migration(table, column = nil)
|
|
68
|
+
if options[:index] && column
|
|
69
|
+
console.problem("pass either COLUMN or --index, not both")
|
|
70
|
+
exit 1
|
|
71
|
+
end
|
|
72
|
+
return generate_index_migration(table, options[:index]) if options[:index]
|
|
73
|
+
return generate_table_migration(table) unless column
|
|
74
|
+
|
|
54
75
|
paths = MigrationGenerator.new(table: table, column: column).call
|
|
55
76
|
console.title("generate-migration", "#{table}.#{column}")
|
|
56
77
|
console.ok("created two migrations:")
|
|
@@ -62,6 +83,16 @@ module SchemaReaper
|
|
|
62
83
|
])
|
|
63
84
|
end
|
|
64
85
|
|
|
86
|
+
desc "explain ID", "Show everything known about one finding and the command to act on it"
|
|
87
|
+
def explain(id)
|
|
88
|
+
exit 1 unless Explain.new(run, config: config, console: console).call(id)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
desc "doctor", "Check the database, statistics, runtime log, baseline and alert setup"
|
|
92
|
+
def doctor
|
|
93
|
+
exit 1 unless Doctor.new(config: config, console: console).call
|
|
94
|
+
end
|
|
95
|
+
|
|
65
96
|
desc "version", "Print version"
|
|
66
97
|
def version
|
|
67
98
|
say(SchemaReaper::VERSION)
|
|
@@ -69,12 +100,34 @@ module SchemaReaper
|
|
|
69
100
|
|
|
70
101
|
private
|
|
71
102
|
|
|
103
|
+
def generate_index_migration(table, index)
|
|
104
|
+
path = MigrationGenerator.new(table: table, index: index).call.first
|
|
105
|
+
console.title("generate-migration", "#{table} · #{index}")
|
|
106
|
+
console.ok("created #{path}")
|
|
107
|
+
console.info("runs with algorithm: :concurrently, so it does not lock writes to #{table}")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def generate_table_migration(table)
|
|
111
|
+
path = MigrationGenerator.new(table: table).call.first
|
|
112
|
+
console.title("generate-migration", table)
|
|
113
|
+
console.ok("created #{path}")
|
|
114
|
+
console.list("before running it:", [
|
|
115
|
+
"confirm no external consumer (reports, ETL, other services) reads #{table}",
|
|
116
|
+
"take a backup -- its `down` cannot bring the data back"
|
|
117
|
+
])
|
|
118
|
+
end
|
|
119
|
+
|
|
72
120
|
def config
|
|
73
121
|
@config ||= Config.load(options[:config])
|
|
74
122
|
end
|
|
75
123
|
|
|
76
|
-
def
|
|
77
|
-
Runner.new(config: config
|
|
124
|
+
def runner(min_confidence: nil)
|
|
125
|
+
@runner ||= Runner.new(config: config, only: options[:only], except: options[:except],
|
|
126
|
+
min_confidence: min_confidence)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def run(min_confidence: nil)
|
|
130
|
+
runner(min_confidence: min_confidence).run
|
|
78
131
|
rescue SchemaReaper::Error => e
|
|
79
132
|
console.problem(e.message)
|
|
80
133
|
exit 1
|
|
@@ -93,7 +146,9 @@ module SchemaReaper
|
|
|
93
146
|
end
|
|
94
147
|
|
|
95
148
|
def enforce_baseline(findings)
|
|
96
|
-
|
|
149
|
+
store = Baseline.new(config.baseline_path)
|
|
150
|
+
warn_expired(store.expired, findings)
|
|
151
|
+
new_ones = store.new_among(findings)
|
|
97
152
|
return if new_ones.empty?
|
|
98
153
|
|
|
99
154
|
console.problem(
|
data/lib/schema_reaper/config.rb
CHANGED
|
@@ -9,6 +9,7 @@ module SchemaReaper
|
|
|
9
9
|
class Config
|
|
10
10
|
DEFAULTS = {
|
|
11
11
|
"database_url" => nil, # falls back to ENV["DATABASE_URL"]
|
|
12
|
+
"schemas" => %w[public], # PostgreSQL schemas to introspect
|
|
12
13
|
"scan_paths" => %w[app lib config],
|
|
13
14
|
"view_globs" => %w[
|
|
14
15
|
app/**/*.erb app/**/*.haml app/**/*.slim app/**/*.jbuilder
|
|
@@ -20,9 +21,16 @@ module SchemaReaper
|
|
|
20
21
|
"always_keep_columns" => %w[id created_at updated_at type],
|
|
21
22
|
"gem_awareness" => true, # auto-whitelist columns owned by known gems
|
|
22
23
|
"min_age_days" => 14,
|
|
24
|
+
"min_confidence" => 0.0, # drop findings below this from every command
|
|
25
|
+
"runtime_log_max_bytes" => 50 * 1024 * 1024, # rotate the usage log past this size
|
|
23
26
|
"runtime_log" => ".schema_reaper/runtime.jsonl",
|
|
24
27
|
"history_log" => ".schema_reaper/history.jsonl",
|
|
25
28
|
"baseline" => ".schema_reaper/baseline.json",
|
|
29
|
+
"scan_cache" => ".schema_reaper/scan_cache.json", # per-file scan cache; null disables
|
|
30
|
+
# global: a column counts as used if its name appears anywhere.
|
|
31
|
+
# model: a name in an app/models file only counts for that model's
|
|
32
|
+
# table and the tables it mentions -- stricter, finds more.
|
|
33
|
+
"column_scope" => "global",
|
|
26
34
|
"database_yml" => "config/database.yml", # Rails fallback for the connection
|
|
27
35
|
"require" => [] # extra files to load (custom analyzers)
|
|
28
36
|
}.freeze
|
|
@@ -74,6 +82,18 @@ module SchemaReaper
|
|
|
74
82
|
@data.dig("ignore", "tables").to_a
|
|
75
83
|
end
|
|
76
84
|
|
|
85
|
+
def schemas
|
|
86
|
+
Array(@data["schemas"]).map(&:to_s)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def min_confidence
|
|
90
|
+
@data["min_confidence"].to_f
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def runtime_log_max_bytes
|
|
94
|
+
@data["runtime_log_max_bytes"]
|
|
95
|
+
end
|
|
96
|
+
|
|
77
97
|
def min_age_days
|
|
78
98
|
@data["min_age_days"]
|
|
79
99
|
end
|
|
@@ -90,6 +110,14 @@ module SchemaReaper
|
|
|
90
110
|
@data["history_log"]
|
|
91
111
|
end
|
|
92
112
|
|
|
113
|
+
def scan_cache
|
|
114
|
+
@data["scan_cache"]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def column_scope
|
|
118
|
+
@data["column_scope"].to_s == "model" ? :model : :global
|
|
119
|
+
end
|
|
120
|
+
|
|
93
121
|
def gem_awareness?
|
|
94
122
|
@data["gem_awareness"] != false
|
|
95
123
|
end
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module SchemaReaper
|
|
6
|
+
# `schema_reaper doctor`: checks everything a scan's accuracy depends on and
|
|
7
|
+
# says what to fix, so a quiet report can be trusted to mean "nothing dead"
|
|
8
|
+
# rather than "nothing measured".
|
|
9
|
+
class Doctor
|
|
10
|
+
Check = Struct.new(:status, :label, :detail, keyword_init: true) # status: :ok | :warn | :fail
|
|
11
|
+
|
|
12
|
+
STATS_FRESH_DAYS = 7
|
|
13
|
+
ICON = { ok: "✓", warn: "!", fail: "✗" }.freeze
|
|
14
|
+
|
|
15
|
+
def initialize(config:, root: Dir.pwd, introspector: nil, console: Reporters::Console.new)
|
|
16
|
+
@config = config
|
|
17
|
+
@root = root
|
|
18
|
+
@introspector = introspector
|
|
19
|
+
@console = console
|
|
20
|
+
@checks = []
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @return [Boolean] true unless a check failed outright
|
|
24
|
+
def call
|
|
25
|
+
check_scan_paths
|
|
26
|
+
check_database
|
|
27
|
+
check_runtime_log
|
|
28
|
+
check_baseline
|
|
29
|
+
check_schema_file
|
|
30
|
+
check_alerts
|
|
31
|
+
render
|
|
32
|
+
@checks.none? { |c| c.status == :fail }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def ok(label, detail = nil)
|
|
38
|
+
@checks << Check.new(status: :ok, label: label, detail: detail)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def warn_check(label, detail)
|
|
42
|
+
@checks << Check.new(status: :warn, label: label, detail: detail)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def fail_check(label, detail)
|
|
46
|
+
@checks << Check.new(status: :fail, label: label, detail: detail)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def check_scan_paths
|
|
50
|
+
missing = @config.scan_paths.reject { |p| Dir.exist?(File.join(@root, p)) }
|
|
51
|
+
return ok("scan paths", @config.scan_paths.join(", ")) if missing.empty?
|
|
52
|
+
|
|
53
|
+
warn_check("scan paths", "not found: #{missing.join(", ")} -- code there can't mark anything as used")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def check_database
|
|
57
|
+
intro = @introspector || Introspect::Postgres.new(@config.database_url, schemas: @config.schemas)
|
|
58
|
+
diag = intro.diagnostics
|
|
59
|
+
ok("database", "connected, PostgreSQL #{diag[:server_version]}")
|
|
60
|
+
check_stats_reset(diag[:stats_reset])
|
|
61
|
+
check_analyze(diag[:never_analyzed])
|
|
62
|
+
check_query_history(intro.call)
|
|
63
|
+
if diag[:in_recovery]
|
|
64
|
+
warn_check("replica", "this is a read replica -- index usage here doesn't cover the primary")
|
|
65
|
+
end
|
|
66
|
+
rescue Error => e
|
|
67
|
+
fail_check("database", e.message)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def check_stats_reset(reset_at)
|
|
71
|
+
return ok("statistics", "never reset") if reset_at.nil?
|
|
72
|
+
|
|
73
|
+
days = ((Time.now - Time.parse(reset_at.to_s)) / 86_400).floor
|
|
74
|
+
return ok("statistics", "collected for #{days} day(s)") if days >= STATS_FRESH_DAYS
|
|
75
|
+
|
|
76
|
+
warn_check("statistics", "reset #{days} day(s) ago -- unused_index needs a longer window to be trusted")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def check_analyze(tables)
|
|
80
|
+
return ok("ANALYZE", "every table has planner statistics") if tables.empty?
|
|
81
|
+
|
|
82
|
+
sample = tables.first(5).join(", ") + (tables.size > 5 ? ", ..." : "")
|
|
83
|
+
warn_check("ANALYZE", "#{tables.size} table(s) never analysed (#{sample}) -- " \
|
|
84
|
+
"row counts and null/distinct checks are blind there; run ANALYZE")
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def check_query_history(schema)
|
|
88
|
+
return ok("query history", "#{schema.index_scan_total} index scans recorded") if schema.query_history?
|
|
89
|
+
|
|
90
|
+
warn_check("query history", "only #{schema.index_scan_total} index scan(s) -- unused_index will sit out; " \
|
|
91
|
+
"scan a database that serves real traffic")
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def check_runtime_log
|
|
95
|
+
report = Runtime::Report.load(@config.runtime_log)
|
|
96
|
+
need = @config.min_age_days || Config::DEFAULTS["min_age_days"]
|
|
97
|
+
if report.empty?
|
|
98
|
+
warn_check("runtime log", "none at #{@config.runtime_log} -- dead_column confidence stays capped at " \
|
|
99
|
+
"#{Analyzers::DeadColumn::STATIC_ONLY_CAP}; see README > Runtime signal")
|
|
100
|
+
elsif report.observed_days < need
|
|
101
|
+
warn_check("runtime log", "#{report.observed_days} of #{need} day(s) observed -- not trusted yet")
|
|
102
|
+
else
|
|
103
|
+
ok("runtime log", "#{report.observed_days} day(s), #{report.accessed.size} column(s) seen")
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def check_baseline
|
|
108
|
+
base = Baseline.new(@config.baseline_path)
|
|
109
|
+
return ok("baseline", "none (scan --ci would fail on every finding)") if base.entries.empty?
|
|
110
|
+
|
|
111
|
+
expired = base.expired
|
|
112
|
+
return ok("baseline", "#{base.entries.size} accepted finding(s)") if expired.empty?
|
|
113
|
+
|
|
114
|
+
warn_check("baseline", "#{expired.size} entr#{expired.size == 1 ? "y has" : "ies have"} expired: " \
|
|
115
|
+
"#{expired.map(&:id).first(3).join(", ")}")
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def check_schema_file
|
|
119
|
+
path = SchemaLocator::CANDIDATES.find { |p| File.exist?(File.join(@root, p)) }
|
|
120
|
+
return ok("schema file", path) if path
|
|
121
|
+
|
|
122
|
+
warn_check("schema file", "no db/schema.rb or db/structure.sql -- SARIF results can't point at a line")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Only meaningful inside a Rails app with the install generator's
|
|
126
|
+
# initializer loaded.
|
|
127
|
+
def check_alerts
|
|
128
|
+
return unless defined?(AlertConfig) && defined?(Rails)
|
|
129
|
+
|
|
130
|
+
cfg = AlertConfig.instance
|
|
131
|
+
unless cfg.configured?
|
|
132
|
+
return warn_check("alerts", "no emails or webhook_url configured -- scheduled scans report nowhere")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
channels = [("email (#{cfg.emails.size})" if cfg.emails?), ("webhook" if cfg.webhook?)].compact
|
|
136
|
+
ok("alerts", "#{channels.join(" + ")}, alert_on: #{cfg.alert_on}")
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def render
|
|
140
|
+
@console.title("doctor")
|
|
141
|
+
@checks.each do |c|
|
|
142
|
+
line = format("%<icon>s %<label>-14s %<detail>s", icon: ICON[c.status], label: c.label, detail: c.detail)
|
|
143
|
+
@console.info(line)
|
|
144
|
+
end
|
|
145
|
+
@console.blank
|
|
146
|
+
counts = @checks.group_by(&:status).transform_values(&:size)
|
|
147
|
+
@console.info("#{counts.fetch(:ok, 0)} ok, #{counts.fetch(:warn, 0)} warning(s), " \
|
|
148
|
+
"#{counts.fetch(:fail, 0)} failed")
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaReaper
|
|
4
|
+
# `schema_reaper explain ID`: everything known about one finding, and the
|
|
5
|
+
# exact command to act on it.
|
|
6
|
+
class Explain
|
|
7
|
+
def initialize(findings, config:, root: Dir.pwd, console: Reporters::Console.new)
|
|
8
|
+
@findings = findings
|
|
9
|
+
@config = config
|
|
10
|
+
@root = root
|
|
11
|
+
@console = console
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# @return [Boolean] false when no finding matches
|
|
15
|
+
def call(query)
|
|
16
|
+
matches = match(query)
|
|
17
|
+
if matches.size == 1
|
|
18
|
+
render(matches.first)
|
|
19
|
+
elsif matches.empty?
|
|
20
|
+
report_not_found(query)
|
|
21
|
+
else
|
|
22
|
+
report_ambiguous(query, matches)
|
|
23
|
+
end
|
|
24
|
+
matches.size == 1
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# An exact id, else every finding whose id contains the query
|
|
30
|
+
# ("users/legacy_token" finds dead_column/users/legacy_token).
|
|
31
|
+
def match(query)
|
|
32
|
+
exact = @findings.select { |f| f.id == query }
|
|
33
|
+
exact.any? ? exact : @findings.select { |f| f.id.include?(query) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def report_not_found(query)
|
|
37
|
+
@console.problem("no current finding matches #{query.inspect}")
|
|
38
|
+
table_finding = @findings.find { |f| f.type == :dead_table && query.split("/").include?(f.table) }
|
|
39
|
+
if table_finding
|
|
40
|
+
@console.info("its table is reported whole as #{table_finding.id} -- per-column and per-index " \
|
|
41
|
+
"findings on a dead table are folded into that one")
|
|
42
|
+
return
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
@console.info("list them with `schema_reaper scan --format json` -- ids look like dead_column/users/legacy_token")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def report_ambiguous(query, matches)
|
|
49
|
+
@console.problem("#{matches.size} findings match #{query.inspect} -- be more specific:",
|
|
50
|
+
items: matches.map(&:id))
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def render(finding)
|
|
54
|
+
@console.title("explain", finding.id)
|
|
55
|
+
@console.row("type", finding.type)
|
|
56
|
+
@console.row("severity", finding.severity)
|
|
57
|
+
@console.row("confidence", format("%.2f", finding.confidence))
|
|
58
|
+
@console.row("reclaimable", reclaim(finding))
|
|
59
|
+
@console.row("defined at", location(finding)) if location(finding)
|
|
60
|
+
@console.row("baseline", baseline_status(finding))
|
|
61
|
+
@console.blank
|
|
62
|
+
@console.list("evidence", finding.evidence, colour: :reset)
|
|
63
|
+
@console.blank
|
|
64
|
+
@console.list("fix", [finding.suggested_fix], colour: :reset)
|
|
65
|
+
next_step = next_command(finding)
|
|
66
|
+
@console.list("next", [next_step], colour: :reset) if next_step
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def reclaim(finding)
|
|
70
|
+
return "unknown (row count unavailable -- run ANALYZE)" unless finding.reclaim_known?
|
|
71
|
+
|
|
72
|
+
Reporters::Bytes.human(finding.reclaimable_bytes)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def location(finding)
|
|
76
|
+
@location ||= {}
|
|
77
|
+
@location[finding.id] ||= begin
|
|
78
|
+
loc = SchemaLocator.new(root: @root).locate(finding)
|
|
79
|
+
loc && "#{loc.path}:#{loc.line}"
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def baseline_status(finding)
|
|
84
|
+
entry = Baseline.new(@config.baseline_path).entries.find { |e| e.id == finding.id }
|
|
85
|
+
return "not accepted (fails `scan --ci`)" unless entry
|
|
86
|
+
return "accepted, but expired #{entry.expires_on}" if entry.expired?
|
|
87
|
+
|
|
88
|
+
["accepted", entry.reason, entry.expires_on && "until #{entry.expires_on}"].compact.join(" -- ")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def next_command(finding)
|
|
92
|
+
case finding.type
|
|
93
|
+
when :dead_column, :always_null_column, :single_value_column
|
|
94
|
+
"bundle exec schema_reaper generate-migration #{finding.table} #{finding.column}"
|
|
95
|
+
when :unused_index, :duplicate_index
|
|
96
|
+
"bundle exec schema_reaper generate-migration #{finding.table} --index #{finding.index}"
|
|
97
|
+
when :dead_table
|
|
98
|
+
"bundle exec schema_reaper generate-migration #{finding.table}"
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -9,6 +9,18 @@ module SchemaReaper
|
|
|
9
9
|
module GemAwareness
|
|
10
10
|
# gem name => { table_glob => [column names] }. "*" table_glob matches any
|
|
11
11
|
# table; an exact name matches only that table.
|
|
12
|
+
def self.owned(tables)
|
|
13
|
+
tables.to_h { |t| [t, []] }
|
|
14
|
+
end
|
|
15
|
+
private_class_method :owned
|
|
16
|
+
|
|
17
|
+
SOLID_QUEUE_TABLES = %w[
|
|
18
|
+
solid_queue_jobs solid_queue_ready_executions solid_queue_scheduled_executions
|
|
19
|
+
solid_queue_claimed_executions solid_queue_blocked_executions solid_queue_failed_executions
|
|
20
|
+
solid_queue_recurring_executions solid_queue_recurring_tasks solid_queue_pauses
|
|
21
|
+
solid_queue_processes solid_queue_semaphores
|
|
22
|
+
].freeze
|
|
23
|
+
|
|
12
24
|
MAP = {
|
|
13
25
|
"devise" => { "*" => %w[
|
|
14
26
|
encrypted_password reset_password_token reset_password_sent_at
|
|
@@ -42,7 +54,25 @@ module SchemaReaper
|
|
|
42
54
|
resource_owner_type resource_owner_id access_token refresh_token
|
|
43
55
|
expires_in revoked_at previous_refresh_token
|
|
44
56
|
]
|
|
45
|
-
}
|
|
57
|
+
},
|
|
58
|
+
# Gems below own their tables outright and touch every column from
|
|
59
|
+
# their own code, so the tables are listed with no columns: ownership
|
|
60
|
+
# alone keeps every analyzer off them (see Base#gem_reserved?).
|
|
61
|
+
"solid_queue" => owned(SOLID_QUEUE_TABLES),
|
|
62
|
+
"solid_cache" => owned(%w[solid_cache_entries]),
|
|
63
|
+
"solid_cable" => owned(%w[solid_cable_messages]),
|
|
64
|
+
"good_job" => owned(%w[good_jobs good_job_executions good_job_processes good_job_settings
|
|
65
|
+
good_job_batches]),
|
|
66
|
+
"delayed_job_active_record" => owned(%w[delayed_jobs]),
|
|
67
|
+
"flipper-active_record" => owned(%w[flipper_features flipper_gates]),
|
|
68
|
+
"acts-as-taggable-on" => owned(%w[tags taggings]),
|
|
69
|
+
"doorkeeper" => owned(%w[oauth_applications oauth_access_grants oauth_access_tokens]),
|
|
70
|
+
"noticed" => owned(%w[noticed_events noticed_notifications]),
|
|
71
|
+
"activerecord-session_store" => owned(%w[sessions]),
|
|
72
|
+
# Columns these gems add to app tables; the first is the anchor.
|
|
73
|
+
"discard" => { "*" => %w[discarded_at] },
|
|
74
|
+
"acts_as_list" => { "*" => %w[position] },
|
|
75
|
+
"ancestry" => { "*" => %w[ancestry ancestry_depth children_count] }
|
|
46
76
|
}.freeze
|
|
47
77
|
|
|
48
78
|
# Table names an installed gem creates and owns outright -- as opposed to
|