schema_reaper 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/.schema_reaper.yml.example +17 -0
- data/CHANGELOG.md +102 -0
- data/README.md +64 -11
- data/RELEASE_CHECKLIST.md +13 -2
- data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +8 -0
- data/lib/schema_reaper/alert_config.rb +21 -1
- data/lib/schema_reaper/alert_state.rb +35 -0
- data/lib/schema_reaper/analyzers/always_null_column.rb +12 -5
- data/lib/schema_reaper/analyzers/base.rb +66 -2
- data/lib/schema_reaper/analyzers/dead_column.rb +2 -2
- data/lib/schema_reaper/analyzers/dead_table.rb +10 -6
- data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb +53 -0
- data/lib/schema_reaper/analyzers/duplicate_index.rb +1 -1
- data/lib/schema_reaper/analyzers/missing_fk_index.rb +10 -4
- data/lib/schema_reaper/analyzers/sequence_exhaustion.rb +55 -0
- data/lib/schema_reaper/analyzers/single_value_column.rb +17 -5
- data/lib/schema_reaper/analyzers/unused_index.rb +11 -6
- data/lib/schema_reaper/baseline.rb +95 -10
- data/lib/schema_reaper/cli/baseline_support.rb +45 -0
- data/lib/schema_reaper/cli.rb +69 -14
- data/lib/schema_reaper/config.rb +28 -0
- data/lib/schema_reaper/doctor.rb +151 -0
- data/lib/schema_reaper/explain.rb +102 -0
- data/lib/schema_reaper/gem_awareness.rb +31 -1
- data/lib/schema_reaper/inflect.rb +36 -0
- data/lib/schema_reaper/introspect/postgres.rb +236 -51
- data/lib/schema_reaper/migration_generator.rb +90 -4
- data/lib/schema_reaper/notifier.rb +58 -6
- data/lib/schema_reaper/railtie.rb +1 -1
- data/lib/schema_reaper/reporters/sarif.rb +15 -6
- data/lib/schema_reaper/reporters/slack.rb +67 -0
- data/lib/schema_reaper/runner.rb +58 -6
- data/lib/schema_reaper/runtime.rb +94 -11
- data/lib/schema_reaper/schema.rb +32 -3
- data/lib/schema_reaper/schema_locator.rb +75 -0
- data/lib/schema_reaper/static/cache.rb +66 -0
- data/lib/schema_reaper/static/scanner.rb +174 -6
- data/lib/schema_reaper/version.rb +1 -1
- data/lib/schema_reaper.rb +7 -0
- data/schema_reaper.gemspec +0 -1
- metadata +13 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 65c5a4cc1857d1fbf940b5907cbb7d8e48de74cebb7b69eb6dd1b8e2a0c26bfa
|
|
4
|
+
data.tar.gz: 679e01d41a797c37dfd94858464127b1ce8789bda1d92cb905f011a2369fe1c8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ef8476dfea36578ef5c97c43fc97c7334371ce38aa14f0d736028d57e42d370b43363922511676af417d76f5fd9b9b35536e98168253efc9c6a663230c56d8c6
|
|
7
|
+
data.tar.gz: cf4f22deed425352ac3d0553deefb34495e4c234170f402167866fa445b4b4bf707fc3d078a3a612fc99477ed39b9c152f96f4cdfb45555ecfa7a0eaaab9cd82
|
data/.rubocop.yml
CHANGED
|
@@ -31,6 +31,7 @@ Metrics/BlockLength:
|
|
|
31
31
|
- "*.gemspec"
|
|
32
32
|
|
|
33
33
|
Metrics/ParameterLists:
|
|
34
|
+
CountKeywordArgs: false # keyword args name themselves at every call site
|
|
34
35
|
Exclude:
|
|
35
36
|
- "spec/**/*"
|
|
36
37
|
|
|
@@ -45,6 +46,8 @@ Metrics/MethodLength:
|
|
|
45
46
|
|
|
46
47
|
Metrics/ClassLength:
|
|
47
48
|
Max: 135
|
|
49
|
+
Exclude:
|
|
50
|
+
- "lib/schema_reaper/introspect/postgres.rb" # mostly SQL heredocs, one per catalog query
|
|
48
51
|
|
|
49
52
|
Metrics/CyclomaticComplexity:
|
|
50
53
|
Max: 12
|
data/.schema_reaper.yml.example
CHANGED
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
---
|
|
3
3
|
# database_url: postgres://localhost/myapp_development # else ENV["DATABASE_URL"]
|
|
4
4
|
|
|
5
|
+
# PostgreSQL schemas to scan. Tables outside public are reported as schema.table.
|
|
6
|
+
schemas:
|
|
7
|
+
- public
|
|
8
|
+
|
|
9
|
+
# Hide findings below this confidence on every command (the CLI flag overrides).
|
|
10
|
+
min_confidence: 0.0
|
|
11
|
+
|
|
5
12
|
scan_paths:
|
|
6
13
|
- app
|
|
7
14
|
- lib
|
|
@@ -33,11 +40,21 @@ gem_awareness: true
|
|
|
33
40
|
|
|
34
41
|
# Runtime usage log written by the tracker (see README > Runtime signal).
|
|
35
42
|
runtime_log: .schema_reaper/runtime.jsonl
|
|
43
|
+
# Rotate it to runtime.jsonl.1 past this many bytes; both are read. null: never.
|
|
44
|
+
runtime_log_max_bytes: 52428800
|
|
36
45
|
|
|
37
46
|
# Append-only snapshot log for `schema_reaper trend`.
|
|
38
47
|
history_log: .schema_reaper/history.jsonl
|
|
39
48
|
|
|
40
49
|
baseline: .schema_reaper/baseline.json
|
|
41
50
|
|
|
51
|
+
# Per-file cache of scan results; repeat scans re-parse only changed files.
|
|
52
|
+
scan_cache: .schema_reaper/scan_cache.json
|
|
53
|
+
|
|
54
|
+
# global: a column name anywhere in code counts as use on every table.
|
|
55
|
+
# model: a name only in an app/models file counts for that model's table and
|
|
56
|
+
# the tables it mentions. Stricter; review its findings before acting.
|
|
57
|
+
column_scope: global
|
|
58
|
+
|
|
42
59
|
# Extra Ruby files to load before analysis -- register custom analyzers here.
|
|
43
60
|
require: []
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,107 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [2.1.0] - 2026-10-14
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- **Introspection no longer mixes up same-named objects across schemas.**
|
|
7
|
+
Row counts were read from `pg_class` by name alone, foreign keys were joined
|
|
8
|
+
on constraint name alone (constraint names are only unique per table), and
|
|
9
|
+
the primary-key lookup resolved the table through `search_path`. All three
|
|
10
|
+
are now qualified by schema. Foreign keys are read from `pg_constraint`, so
|
|
11
|
+
composite keys keep their columns together.
|
|
12
|
+
- **`ignore.columns` now applies to every column analyzer.** It was only
|
|
13
|
+
honoured by `dead_column`; `always_null_column`, `single_value_column` and
|
|
14
|
+
`missing_fk_index` ignored it.
|
|
15
|
+
- **Partitioned tables are reported once, as the parent.** Each partition used
|
|
16
|
+
to appear as its own table -- and, unreferenced by name, got a `dead_table`
|
|
17
|
+
finding. The parent's row count is now summed from its partitions, and its
|
|
18
|
+
indexes' usage from the partitions' indexes, so `unused_index` no longer
|
|
19
|
+
flags every index on a partitioned table (PostgreSQL 12+).
|
|
20
|
+
- **Stats-based findings say when they are estimates.** On tables larger than
|
|
21
|
+
ANALYZE's 30,000-row sample, `always_null_column` (0.85 -> 0.7) and
|
|
22
|
+
`single_value_column` (0.6 -> 0.5) lower their confidence and say so, and
|
|
23
|
+
`single_value_column` no longer claims "the same value in every row" for a
|
|
24
|
+
column that also holds NULLs.
|
|
25
|
+
- **`missing_fk_index`** finds the referenced table in a non-public table's own
|
|
26
|
+
schema.
|
|
27
|
+
- **Runtime `observed_days` no longer depends on line order.** Several
|
|
28
|
+
processes append to one log, so the span is now min..max timestamp rather
|
|
29
|
+
than first..last line.
|
|
30
|
+
|
|
31
|
+
### Added
|
|
32
|
+
- `schemas:` config key to scan PostgreSQL schemas besides `public`. Tables
|
|
33
|
+
outside `public` are reported as `schema.table`; `public` tables keep their
|
|
34
|
+
bare names, so existing baselines and ignore lists still match.
|
|
35
|
+
- `sequence_exhaustion` analyzer (CYCLE and descending sequences excluded): a serial/IDENTITY sequence past 50% (medium)
|
|
36
|
+
or 75% (high) of its column type's range -- e.g. an `integer` key fed by a
|
|
37
|
+
bigint sequence.
|
|
38
|
+
- `duplicate_foreign_key` analyzer: two FK constraints on the same columns
|
|
39
|
+
referencing the same key, with the same ON UPDATE / ON DELETE, MATCH and
|
|
40
|
+
deferrability (anything else changes behaviour when dropped). A validated
|
|
41
|
+
copy is kept over a NOT VALID one.
|
|
42
|
+
- `--only` / `--except` to choose analyzers (custom ones included), and a
|
|
43
|
+
`min_confidence` config key used by every command.
|
|
44
|
+
- `generate-migration TABLE --index NAME` (concurrent `remove_index` with
|
|
45
|
+
`disable_ddl_transaction!`; the migration checks at run time whether the
|
|
46
|
+
index is partitioned, which PostgreSQL can't drop concurrently, and drops it
|
|
47
|
+
plainly then) and `generate-migration TABLE` (`drop_table`). The
|
|
48
|
+
`unused_index` / `duplicate_index` fixes say the same for partitioned
|
|
49
|
+
indexes.
|
|
50
|
+
- The runtime tracker also samples tables read by executed SQL (`track_sql:`,
|
|
51
|
+
on by default; FROM / JOIN / UPDATE, not INSERT targets, so a write-only
|
|
52
|
+
table can still be found), so a table read only through raw SQL isn't
|
|
53
|
+
reported dead.
|
|
54
|
+
- The runtime log rotates to `<path>.1` past `runtime_log_max_bytes` (default
|
|
55
|
+
50 MB), under a file lock so concurrent workers can't rotate twice; both
|
|
56
|
+
generations are read.
|
|
57
|
+
- `schema_reaper doctor`: checks the connection, statistics age, tables never
|
|
58
|
+
`ANALYZE`d, query history, replica status, runtime log length, expired
|
|
59
|
+
baseline entries, the schema file and alert config.
|
|
60
|
+
- `schema_reaper explain ID` (full or partial id): evidence, confidence,
|
|
61
|
+
reclaim, schema line, baseline status and the `generate-migration` command.
|
|
62
|
+
- `# schema_reaper:keep TARGET -- reason` comments in code silence findings.
|
|
63
|
+
- `column_scope: model` (opt-in): a column name seen only in a model file
|
|
64
|
+
counts for that model's table and the tables it mentions, not every table.
|
|
65
|
+
Models are identified by their full class path (`Admin::User` is not
|
|
66
|
+
`User`); STI subclasses map to their parent's table (resolved the way Ruby
|
|
67
|
+
resolves the constant), namespaced models to both possible defaults, and
|
|
68
|
+
abstract bases, POROs and concerns stay shared.
|
|
69
|
+
- Baseline entries carry an optional reason and expiry (`baseline --reason
|
|
70
|
+
--expires YYYY-MM-DD`); `scan --ci` fails on expired ones again, and passing
|
|
71
|
+
`--reason` / `--expires` renews them. `baseline --prune` drops resolved
|
|
72
|
+
entries without accepting new findings. With `--only` / `--except`,
|
|
73
|
+
`baseline` replaces or prunes only those analyzers' entries. The 2.0
|
|
74
|
+
`{"ids": [...]}` file is still read, and the file keeps an `ids` key
|
|
75
|
+
(unexpired entries) so a 2.0.x gem sharing it still works -- though 2.0.x
|
|
76
|
+
ignores reasons and expiry, and rewrites the file in the old format if it
|
|
77
|
+
runs `baseline`.
|
|
78
|
+
- Scheduled reports say what changed since the last one (via `Rails.cache`);
|
|
79
|
+
`config.alert_on = :changes` sends only then. A report only counts as sent
|
|
80
|
+
once a channel accepted it -- a non-2xx webhook response is a failure -- so
|
|
81
|
+
a failed delivery is retried next run instead of suppressed.
|
|
82
|
+
- Slack webhooks get a Block Kit summary instead of a Markdown table Slack
|
|
83
|
+
can't render; other receivers keep the full `{"text": ...}` report
|
|
84
|
+
(`config.webhook_format`).
|
|
85
|
+
- Gem awareness covers solid_queue, solid_cache, solid_cable, good_job,
|
|
86
|
+
delayed_job_active_record, flipper-active_record, acts-as-taggable-on,
|
|
87
|
+
doorkeeper, noticed, activerecord-session_store, discard, acts_as_list and
|
|
88
|
+
ancestry. Every analyzer now leaves a gem-owned table's columns alone.
|
|
89
|
+
- Per-file scan cache (`scan_cache:`), so repeat scans only re-parse changed
|
|
90
|
+
files. Keyed by a SHA-256 digest, so it survives across processes.
|
|
91
|
+
- SARIF results point at the defining line in `db/schema.rb` or
|
|
92
|
+
`db/structure.sql` instead of the top of `db/schema.rb`.
|
|
93
|
+
- `--color` / `--no-color` works on every command.
|
|
94
|
+
|
|
95
|
+
### Changed
|
|
96
|
+
- Column byte estimates use `pg_stats.avg_width` scaled by the non-NULL share
|
|
97
|
+
when the table has been analysed, instead of a fixed per-type guess (16
|
|
98
|
+
bytes for text/jsonb) that also counted NULLs as full width. An all-NULL
|
|
99
|
+
column is estimated at ~0 bytes.
|
|
100
|
+
- `unused_index` evidence notes that `idx_scan` is per server, so indexes used
|
|
101
|
+
only on read replicas look unused on the primary.
|
|
102
|
+
- CI tests Ruby 3.4 and 4.0 too, and runs the live introspection specs against
|
|
103
|
+
a PostgreSQL 16 service.
|
|
104
|
+
|
|
3
105
|
## [2.0.1] - 2026-09-23
|
|
4
106
|
|
|
5
107
|
### Fixed
|
data/README.md
CHANGED
|
@@ -75,7 +75,7 @@ gem "schema_reaper"
|
|
|
75
75
|
| requirement | notes |
|
|
76
76
|
|---|---|
|
|
77
77
|
| Ruby >= 2.7 | CI-tested on 2.7 – 3.3 |
|
|
78
|
-
| PostgreSQL + the `pg` gem | uses your app's own `pg` (every Rails + PostgreSQL app already bundles it); add `gem "pg"` for standalone CLI use. Tables in the `public` schema are scanned. MySQL is on the [roadmap](#roadmap). |
|
|
78
|
+
| PostgreSQL + the `pg` gem | uses your app's own `pg` (every Rails + PostgreSQL app already bundles it); add `gem "pg"` for standalone CLI use. Partitioned tables are reported once, as the parent. Tables in the `public` schema are scanned by default; list others under `schemas:` in the [config](#configuration). MySQL is on the [roadmap](#roadmap). |
|
|
79
79
|
| Rails | optional for the CLI; the rake tasks and production automation need it, and the runtime tracker needs ActiveRecord |
|
|
80
80
|
|
|
81
81
|
> [!IMPORTANT]
|
|
@@ -110,9 +110,16 @@ bundle exec schema_reaper scan --format sarif # GitHub code scanning
|
|
|
110
110
|
bundle exec schema_reaper scan --format json
|
|
111
111
|
bundle exec schema_reaper scan --ci # exit 1 on findings not in the baseline
|
|
112
112
|
bundle exec schema_reaper scan --min-confidence 0.8
|
|
113
|
+
bundle exec schema_reaper scan --only dead_column unused_index # or --except ...
|
|
113
114
|
bundle exec schema_reaper baseline # accept current findings
|
|
115
|
+
bundle exec schema_reaper baseline --reason "billing ETL" --expires 2027-03-31
|
|
116
|
+
bundle exec schema_reaper baseline --prune # drop entries that no longer occur
|
|
117
|
+
bundle exec schema_reaper explain users/legacy_api_token # one finding in full + next step
|
|
118
|
+
bundle exec schema_reaper doctor # is the setup good enough to trust a scan?
|
|
114
119
|
bundle exec schema_reaper trend # snapshot + progress delta
|
|
115
|
-
bundle exec schema_reaper generate-migration users legacy_api_token
|
|
120
|
+
bundle exec schema_reaper generate-migration users legacy_api_token # column: staged pair
|
|
121
|
+
bundle exec schema_reaper generate-migration users --index index_users_on_legacy_token
|
|
122
|
+
bundle exec schema_reaper generate-migration old_audit_logs # whole table
|
|
116
123
|
```
|
|
117
124
|
|
|
118
125
|
<details>
|
|
@@ -124,9 +131,10 @@ bundle exec schema_reaper generate-migration users legacy_api_token
|
|
|
124
131
|
|---|---|---|
|
|
125
132
|
| `--format` | `table` | `table`, `json`, `markdown` or `sarif` |
|
|
126
133
|
| `--ci` | off | exit 1 when a finding isn't in `.schema_reaper/baseline.json` |
|
|
127
|
-
| `--min-confidence N` | `0.0` | hide findings below this confidence (0.0 – 1.0) |
|
|
134
|
+
| `--min-confidence N` | `min_confidence` in config, else `0.0` | hide findings below this confidence (0.0 – 1.0) |
|
|
135
|
+
| `--only A B` / `--except A B` | all | run only, or skip, these analyzers (also on `baseline` and `trend`) |
|
|
128
136
|
| `--record` | off | also append this run to the history log used by `trend` |
|
|
129
|
-
| `--color` / `--no-color` | auto | force colour on or off
|
|
137
|
+
| `--color` / `--no-color` | auto | force colour on or off (works on every command) |
|
|
130
138
|
| `--config PATH` | `.schema_reaper.yml` | config file to load (works on every command) |
|
|
131
139
|
|
|
132
140
|
Colour is automatic on a terminal, and off when output is piped or `NO_COLOR` is set.
|
|
@@ -194,8 +202,14 @@ end
|
|
|
194
202
|
Both channels fire when both are set — not a fallback chain, but two audiences:
|
|
195
203
|
a dev-facing chat channel, and an inbox record for people who don't watch chat.
|
|
196
204
|
|
|
197
|
-
|
|
198
|
-
|
|
205
|
+
Every report says what changed since the previous one (remembered in
|
|
206
|
+
`Rails.cache`). Set `config.alert_on = :changes` to send only when findings
|
|
207
|
+
appeared or were resolved; the default, `:always`, sends every run.
|
|
208
|
+
|
|
209
|
+
- **Webhook** — for a `hooks.slack.com` URL, a Slack Block Kit summary (counts,
|
|
210
|
+
the top or newly added findings, what was resolved); for any other URL, a
|
|
211
|
+
`{"text": "..."}` JSON POST carrying the full Markdown report. Force either
|
|
212
|
+
with `config.webhook_format = :slack` or `:text`. 10-second timeout.
|
|
199
213
|
- **Email** — sent by `SchemaReaper::Mailer`, which inherits your app's
|
|
200
214
|
`ApplicationMailer` (its `default from:`, delivery settings and so on), or
|
|
201
215
|
`ActionMailer::Base` if you don't have one. Delivered with `deliver_later`.
|
|
@@ -295,17 +309,23 @@ for watching the real, live database.
|
|
|
295
309
|
|---|---|---|
|
|
296
310
|
| `dead_column` | column no code path references | static scan (+ runtime) |
|
|
297
311
|
| `dead_table` | table with no model or query reference | static scan + row count |
|
|
298
|
-
| `unused_index` | non-unique index with `idx_scan = 0` (needs query history) | `pg_stat_user_indexes` |
|
|
312
|
+
| `unused_index` | non-unique index with `idx_scan = 0` (needs query history; stats are per server, so check replicas too) | `pg_stat_user_indexes` |
|
|
299
313
|
| `duplicate_index` | index that exactly duplicates another, or is a prefix of a wider one | schema shape |
|
|
300
314
|
| `missing_fk_index` | `*_id` / foreign-key column with no index (polymorphic pairs need a `(type, id)` index) | schema shape |
|
|
301
315
|
| `always_null_column` | `null_frac = 1.0` — no data at all | `pg_stats` |
|
|
302
316
|
| `single_value_column` | one distinct value on a table of 500+ rows | `pg_stats` |
|
|
317
|
+
| `sequence_exhaustion` | serial / IDENTITY sequence past 50% of its column type's range (high at 75%) | `pg_sequences` |
|
|
318
|
+
| `duplicate_foreign_key` | two FK constraints on the same columns to the same table | `pg_constraint` |
|
|
303
319
|
|
|
304
320
|
Columns and tables owned by common gems are whitelisted automatically when the
|
|
305
321
|
gem is in your bundle: **devise**, **devise-api**, **paper_trail**,
|
|
306
322
|
**audited**, **friendly_id**, **paranoia** / **acts_as_paranoid**,
|
|
307
|
-
**activestorage**, **actiontext**, **pg_search**, **ahoy_matey
|
|
308
|
-
**activeadmin
|
|
323
|
+
**activestorage**, **actiontext**, **pg_search**, **ahoy_matey**,
|
|
324
|
+
**activeadmin**, **solid_queue**, **solid_cache**, **solid_cable**,
|
|
325
|
+
**good_job**, **delayed_job_active_record**, **flipper-active_record**,
|
|
326
|
+
**acts-as-taggable-on**, **doorkeeper**, **noticed**,
|
|
327
|
+
**activerecord-session_store**, **discard**, **acts_as_list** and
|
|
328
|
+
**ancestry**. A table a gem owns outright is left alone by every analyzer.
|
|
309
329
|
|
|
310
330
|
## Runtime signal (optional, raises confidence)
|
|
311
331
|
|
|
@@ -322,7 +342,11 @@ SchemaReaper::Runtime::Tracker.install!(
|
|
|
322
342
|
```
|
|
323
343
|
|
|
324
344
|
or, in Rails, boot with `SCHEMA_REAPER_TRACK=1` (and optionally
|
|
325
|
-
`SCHEMA_REAPER_SAMPLE=0.05` for the sample rate).
|
|
345
|
+
`SCHEMA_REAPER_SAMPLE=0.05` for the sample rate). The tracker also samples
|
|
346
|
+
the tables named in executed SQL (pass `track_sql: false` to opt out), so a
|
|
347
|
+
table read only through raw SQL isn't reported as dead. The log rotates to
|
|
348
|
+
`runtime.jsonl.1` past `runtime_log_max_bytes` (50 MB), and both generations
|
|
349
|
+
are read. Let it run in staging or
|
|
326
350
|
production for a couple of weeks. A column unseen in **both** code and
|
|
327
351
|
≥ 14 observed days of runtime data (`min_age_days` in
|
|
328
352
|
[`.schema_reaper.yml`](#configuration)) reaches **0.9** confidence (0.8 if it's
|
|
@@ -406,6 +430,7 @@ override any of these defaults.
|
|
|
406
430
|
```yaml
|
|
407
431
|
database_url: # falls back to DATABASE_URL, then config/database.yml
|
|
408
432
|
database_yml: config/database.yml
|
|
433
|
+
schemas: [public] # PostgreSQL schemas to scan; non-public tables appear as schema.table
|
|
409
434
|
scan_paths: [app, lib, config]
|
|
410
435
|
view_globs: ["app/**/*.erb", "app/**/*.haml", "app/**/*.slim", "app/**/*.jbuilder"]
|
|
411
436
|
ignore:
|
|
@@ -414,14 +439,42 @@ ignore:
|
|
|
414
439
|
always_keep_columns: [id, created_at, updated_at, type]
|
|
415
440
|
gem_awareness: true # auto-whitelist columns owned by known gems
|
|
416
441
|
min_age_days: 14 # days of runtime data before dead_column trusts it
|
|
442
|
+
min_confidence: 0.0 # hide findings below this on every command
|
|
443
|
+
runtime_log_max_bytes: 52428800 # rotate the runtime log past this size (null: never)
|
|
417
444
|
runtime_log: .schema_reaper/runtime.jsonl
|
|
418
445
|
history_log: .schema_reaper/history.jsonl
|
|
419
446
|
baseline: .schema_reaper/baseline.json
|
|
420
447
|
require: [] # extra files to load, e.g. custom analyzers
|
|
448
|
+
scan_cache: .schema_reaper/scan_cache.json # per-file scan cache (null: off)
|
|
449
|
+
column_scope: global # or model -- see "Stricter column matching"
|
|
421
450
|
```
|
|
422
451
|
|
|
423
452
|
</details>
|
|
424
453
|
|
|
454
|
+
### Keeping a finding from code
|
|
455
|
+
|
|
456
|
+
A `schema_reaper:keep` comment anywhere in scanned Ruby or view files silences
|
|
457
|
+
matching findings, with the reason kept next to the code it's about:
|
|
458
|
+
|
|
459
|
+
```ruby
|
|
460
|
+
class User < ApplicationRecord
|
|
461
|
+
# schema_reaper:keep legacy_api_token, users.import_ref -- read by the billing ETL
|
|
462
|
+
end
|
|
463
|
+
```
|
|
464
|
+
|
|
465
|
+
A target is a column (on any table), `table.column`, a table (every finding on
|
|
466
|
+
it), or an index name.
|
|
467
|
+
|
|
468
|
+
### Stricter column matching
|
|
469
|
+
|
|
470
|
+
By default a column counts as used if its name appears anywhere in scanned
|
|
471
|
+
code, so `status` in `Order` also hides a dead `status` on `users`. With
|
|
472
|
+
`column_scope: model`, a name seen only in an `app/models` file counts for that
|
|
473
|
+
model's table and for the tables the model mentions (associations, class
|
|
474
|
+
names); names in controllers, views, jobs and so on still count everywhere.
|
|
475
|
+
It finds more dead columns, at some risk of flagging one used through
|
|
476
|
+
metaprogramming -- review before acting.
|
|
477
|
+
|
|
425
478
|
### Custom analyzers
|
|
426
479
|
|
|
427
480
|
```ruby
|
|
@@ -447,7 +500,7 @@ require:
|
|
|
447
500
|
> ✅ Self-hosted scheduled scans + alerts shipped in **v2.0** — see
|
|
448
501
|
> [Production automation](#production-automation).
|
|
449
502
|
|
|
450
|
-
- Runtime verdict fusion for index
|
|
503
|
+
- Runtime verdict fusion for index findings
|
|
451
504
|
- Orphan-row and `schema.rb`↔DB drift analyzers
|
|
452
505
|
- Disk/$ reclaim from real `pg_total_relation_size`
|
|
453
506
|
- Mountable dashboard engine, trend charts
|
data/RELEASE_CHECKLIST.md
CHANGED
|
@@ -152,8 +152,19 @@ a throwaway PostgreSQL with a deliberately dirty schema.
|
|
|
152
152
|
|
|
153
153
|
## 6. Publish the gem
|
|
154
154
|
|
|
155
|
-
|
|
156
|
-
|
|
155
|
+
Pushing the `vX.Y.Z` tag (§5) runs `.github/workflows/release.yml`: it checks
|
|
156
|
+
the tag matches `VERSION`, is on `main`, has a CHANGELOG section and no AI
|
|
157
|
+
trailers in history; runs the full suite against PostgreSQL; builds the gem
|
|
158
|
+
and diffs the packaged `lib/` against `git ls-files`; then (after approval, if
|
|
159
|
+
the `release` environment requires reviewers) pushes to RubyGems via Trusted
|
|
160
|
+
Publishing and creates the GitHub release from the CHANGELOG section.
|
|
161
|
+
|
|
162
|
+
One-time setup: rubygems.org -> schema_reaper -> Trusted publishers -> add
|
|
163
|
+
repository `aksshatt/schema_reaper`, workflow `release.yml`, environment
|
|
164
|
+
`release`.
|
|
165
|
+
|
|
166
|
+
Manual fallback -- RubyGems requires account MFA; `gem push` prompts for an
|
|
167
|
+
OTP that only the maintainer can enter.
|
|
157
168
|
|
|
158
169
|
```sh
|
|
159
170
|
gem build schema_reaper.gemspec
|
|
@@ -15,4 +15,12 @@
|
|
|
15
15
|
SchemaReaper::AlertConfig.configure do |config|
|
|
16
16
|
# config.emails = %w[dev1@example.com dev2@example.com]
|
|
17
17
|
# config.webhook_url = "https://hooks.slack.com/services/T000/B000/XXXX"
|
|
18
|
+
|
|
19
|
+
# :changes sends a report only when findings appeared or were resolved
|
|
20
|
+
# since the last one (needs Rails.cache); :always sends every run.
|
|
21
|
+
# config.alert_on = :changes
|
|
22
|
+
|
|
23
|
+
# :auto uses Slack Block Kit for hooks.slack.com URLs and a plain
|
|
24
|
+
# {"text": <markdown>} body for anything else. Force with :slack or :text.
|
|
25
|
+
# config.webhook_format = :auto
|
|
18
26
|
end
|
|
@@ -30,11 +30,31 @@ module SchemaReaper
|
|
|
30
30
|
end
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
# alert_on: :always -- send every scheduled report (the 2.0 behaviour)
|
|
34
|
+
# :changes -- send only when findings appeared or were
|
|
35
|
+
# resolved since the last report
|
|
36
|
+
# webhook_format: :auto -- Slack Block Kit for hooks.slack.com, plain
|
|
37
|
+
# {"text": <markdown>} for anything else
|
|
38
|
+
# :slack / :text to force one
|
|
39
|
+
attr_accessor :emails, :webhook_url, :alert_on, :webhook_format
|
|
34
40
|
|
|
35
41
|
def initialize
|
|
36
42
|
@emails = []
|
|
37
43
|
@webhook_url = nil
|
|
44
|
+
@alert_on = :always
|
|
45
|
+
@webhook_format = :auto
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def changes_only?
|
|
49
|
+
alert_on.to_s == "changes"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def slack_blocks?
|
|
53
|
+
case webhook_format.to_s
|
|
54
|
+
when "slack" then true
|
|
55
|
+
when "text" then false
|
|
56
|
+
else webhook_url.to_s.include?("hooks.slack.com")
|
|
57
|
+
end
|
|
38
58
|
end
|
|
39
59
|
|
|
40
60
|
def emails?
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaReaper
|
|
4
|
+
# Remembers which finding ids the last delivered report contained, so the
|
|
5
|
+
# next one can say what changed -- and, with alert_on: :changes, stay quiet
|
|
6
|
+
# when nothing did. Kept in Rails.cache (the manual trigger's cooldown
|
|
7
|
+
# already relies on it) because production filesystems are often
|
|
8
|
+
# ephemeral. Without a cache every report is treated as the first.
|
|
9
|
+
class AlertState
|
|
10
|
+
KEY = "schema_reaper:last_alert_ids"
|
|
11
|
+
|
|
12
|
+
def initialize(cache: default_cache)
|
|
13
|
+
@cache = cache
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# @return [Array<String>, nil] nil when no earlier report is known
|
|
17
|
+
def previous_ids
|
|
18
|
+
@cache&.read(KEY)
|
|
19
|
+
rescue StandardError
|
|
20
|
+
nil
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def remember(ids)
|
|
24
|
+
@cache&.write(KEY, ids.sort)
|
|
25
|
+
rescue StandardError
|
|
26
|
+
nil
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def default_cache
|
|
32
|
+
defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : nil
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -20,6 +20,7 @@ module SchemaReaper
|
|
|
20
20
|
table.columns.filter_map do |col|
|
|
21
21
|
next unless col.always_null?
|
|
22
22
|
next if config.always_keep_columns.include?(col.name)
|
|
23
|
+
next if ignored_column?(col.name)
|
|
23
24
|
next if gem_reserved?(table.name, col.name)
|
|
24
25
|
|
|
25
26
|
finding(
|
|
@@ -27,17 +28,23 @@ module SchemaReaper
|
|
|
27
28
|
table: table.name,
|
|
28
29
|
column: col.name,
|
|
29
30
|
severity: :high,
|
|
30
|
-
confidence: 0.85,
|
|
31
|
+
confidence: sampled_stats?(table) ? 0.7 : 0.85,
|
|
31
32
|
bytes_per_row: col.bytes,
|
|
32
|
-
evidence:
|
|
33
|
-
"pg_stats.null_frac = 1.0 across ~#{table.row_count} row(s)",
|
|
34
|
-
"column carries no data"
|
|
35
|
-
],
|
|
33
|
+
evidence: evidence_for(table),
|
|
36
34
|
suggested_fix: "verify with `SELECT count(#{col.name}) FROM #{table.name}` " \
|
|
37
35
|
"then stage a removal"
|
|
38
36
|
)
|
|
39
37
|
end
|
|
40
38
|
end
|
|
39
|
+
|
|
40
|
+
def evidence_for(table)
|
|
41
|
+
ev = ["pg_stats.null_frac = 1.0 across ~#{table.row_count} row(s)"]
|
|
42
|
+
ev << if sampled_stats?(table)
|
|
43
|
+
"estimated from a #{STATS_SAMPLE_ROWS}-row ANALYZE sample -- a few non-null rows could be missed"
|
|
44
|
+
else
|
|
45
|
+
"column carries no data"
|
|
46
|
+
end
|
|
47
|
+
end
|
|
41
48
|
end
|
|
42
49
|
end
|
|
43
50
|
end
|
|
@@ -11,8 +11,14 @@ module SchemaReaper
|
|
|
11
11
|
# gem_columns - Hash{table_name => Set<column_name>} reserved by gems
|
|
12
12
|
# gem_owned_tables - Set<table_name> created and owned outright by a gem
|
|
13
13
|
# config - Config
|
|
14
|
-
|
|
14
|
+
# scan - Static::Result (optional; per-model detail
|
|
15
|
+
# for column_scope: model)
|
|
16
|
+
Context = Struct.new(:schema, :used_tokens, :runtime, :gem_columns, :gem_owned_tables, :config, :scan,
|
|
15
17
|
keyword_init: true) do
|
|
18
|
+
def scan
|
|
19
|
+
self[:scan] || Static::Result.from_tokens(used_tokens || [])
|
|
20
|
+
end
|
|
21
|
+
|
|
16
22
|
def runtime
|
|
17
23
|
self[:runtime] || Runtime::Report.empty
|
|
18
24
|
end
|
|
@@ -28,6 +34,11 @@ module SchemaReaper
|
|
|
28
34
|
|
|
29
35
|
# Shared plumbing for analyzers: schema access and token lookup helpers.
|
|
30
36
|
class Base
|
|
37
|
+
# ANALYZE reads a sample of 300 x default_statistics_target rows (30,000
|
|
38
|
+
# at the default of 100). Above that, pg_stats' null_frac and n_distinct
|
|
39
|
+
# are estimates: a value held by a handful of rows can be missed.
|
|
40
|
+
STATS_SAMPLE_ROWS = 30_000
|
|
41
|
+
|
|
31
42
|
def self.type
|
|
32
43
|
name.split("::").last.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym
|
|
33
44
|
end
|
|
@@ -61,8 +72,61 @@ module SchemaReaper
|
|
|
61
72
|
ctx.used_tokens.include?(token.to_s.downcase)
|
|
62
73
|
end
|
|
63
74
|
|
|
75
|
+
# Whether code references `column` on `table`. With column_scope: model,
|
|
76
|
+
# a name seen only in model files counts just for the tables those
|
|
77
|
+
# models are, or mention (an association, a class name), so Order's
|
|
78
|
+
# `status` no longer hides a dead `status` on users.
|
|
79
|
+
def column_used?(table, column)
|
|
80
|
+
return used?(column) unless config.column_scope == :model
|
|
81
|
+
|
|
82
|
+
name = column.to_s.downcase
|
|
83
|
+
scan = ctx.scan
|
|
84
|
+
return true if scan.shared.include?(name)
|
|
85
|
+
|
|
86
|
+
forms = table_forms(table)
|
|
87
|
+
scan.models.any? do |model|
|
|
88
|
+
model.tokens.include?(name) && (model.tables.include?(table) || forms.any? { |f| model.tokens.include?(f) })
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def table_forms(table)
|
|
93
|
+
bare = table.to_s.split(".").last
|
|
94
|
+
singular = Inflect.singularize(bare)
|
|
95
|
+
[bare, singular, singular.delete("_")].uniq
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Every column of a table a gem owns outright is the gem's business.
|
|
64
99
|
def gem_reserved?(table, column)
|
|
65
|
-
ctx.gem_columns.fetch(table, []).include?(column)
|
|
100
|
+
gem_owned_table?(table) || ctx.gem_columns.fetch(table, []).include?(column)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def sampled_stats?(table)
|
|
104
|
+
table.row_count.to_i > STATS_SAMPLE_ROWS
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# The `ignore.columns` list (names or /regex/) applies to every
|
|
108
|
+
# column-level analyzer, not only dead_column.
|
|
109
|
+
def ignored_column?(name)
|
|
110
|
+
config.ignored_column?(name)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# The removal command for an index. DROP INDEX CONCURRENTLY is refused
|
|
114
|
+
# on a partitioned index ("cannot drop partitioned index ...
|
|
115
|
+
# concurrently"), and its per-partition indexes can't be dropped on
|
|
116
|
+
# their own while attached, so the only path is a plain drop, which
|
|
117
|
+
# takes an ACCESS EXCLUSIVE lock on the parent and every partition.
|
|
118
|
+
def remove_index_fix(table, index)
|
|
119
|
+
base = "remove_index #{sym(table.name)}, name: :#{index.name}"
|
|
120
|
+
return "#{base} (use algorithm: :concurrently in production)" unless index.partitioned
|
|
121
|
+
|
|
122
|
+
"#{base} (partitioned index: CONCURRENTLY is not possible -- this locks the table " \
|
|
123
|
+
"and all partitions while it runs, so schedule it for a quiet window)"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# A table name as a Ruby symbol literal for suggested fixes: :users, or
|
|
127
|
+
# :"billing.invoices" for a table outside the public schema.
|
|
128
|
+
def sym(name)
|
|
129
|
+
name.to_sym.inspect
|
|
66
130
|
end
|
|
67
131
|
|
|
68
132
|
def gem_owned_table?(table)
|
|
@@ -21,7 +21,7 @@ module SchemaReaper
|
|
|
21
21
|
def dead_in(table)
|
|
22
22
|
table.columns.filter_map do |col|
|
|
23
23
|
next if keep?(table, col)
|
|
24
|
-
next if
|
|
24
|
+
next if column_used?(table.name, col.name)
|
|
25
25
|
next if runtime.read?(table.name, col.name)
|
|
26
26
|
|
|
27
27
|
finding(
|
|
@@ -33,7 +33,7 @@ module SchemaReaper
|
|
|
33
33
|
bytes_per_row: col.bytes,
|
|
34
34
|
evidence: evidence_for(table, col),
|
|
35
35
|
suggested_fix: "Stage removal: `self.ignored_columns += %w[#{col.name}]` on the " \
|
|
36
|
-
"model, deploy, then `remove_column
|
|
36
|
+
"model, deploy, then `remove_column #{sym(table.name)}, :#{col.name}`."
|
|
37
37
|
)
|
|
38
38
|
end
|
|
39
39
|
end
|
|
@@ -24,13 +24,13 @@ module SchemaReaper
|
|
|
24
24
|
|
|
25
25
|
def ignored?(table)
|
|
26
26
|
config.ignore_tables.include?(table.name) ||
|
|
27
|
-
table.name.start_with?("active_storage_", "action_text_", "action_mailbox_") ||
|
|
27
|
+
table.name.split(".").last.start_with?("active_storage_", "action_text_", "action_mailbox_") ||
|
|
28
28
|
gem_owned_table?(table.name)
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def dead(table)
|
|
32
32
|
return if referenced?(table)
|
|
33
|
-
return if runtime.
|
|
33
|
+
return if runtime.table_accessed?(table.name)
|
|
34
34
|
|
|
35
35
|
finding(
|
|
36
36
|
type: :dead_table,
|
|
@@ -40,17 +40,21 @@ module SchemaReaper
|
|
|
40
40
|
confidence: confidence_for(table),
|
|
41
41
|
bytes_per_row: table.columns.sum(&:bytes),
|
|
42
42
|
evidence: evidence_for(table),
|
|
43
|
-
suggested_fix: "confirm no external consumer, then `drop_table
|
|
43
|
+
suggested_fix: "confirm no external consumer, then `drop_table #{sym(table.name)}`"
|
|
44
44
|
)
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
# Match the table name and its singular/camelized model form.
|
|
48
|
+
# A table outside public ("billing.invoices") is still referenced by its
|
|
49
|
+
# bare name in code -- `self.table_name = "billing.invoices"` tokenises
|
|
50
|
+
# to billing + invoices, and the model is Invoice.
|
|
48
51
|
def referenced?(table)
|
|
49
|
-
|
|
50
|
-
|
|
52
|
+
bare = table.name.split(".").last
|
|
53
|
+
singular = singularize(bare)
|
|
54
|
+
forms = [bare, singular, camelize(bare), camelize(singular)]
|
|
51
55
|
return true if forms.any? { |form| used?(form) }
|
|
52
56
|
|
|
53
|
-
embedded_in_identifier?(
|
|
57
|
+
embedded_in_identifier?(bare)
|
|
54
58
|
end
|
|
55
59
|
|
|
56
60
|
def singularize(name)
|