schema_reaper 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/.schema_reaper.yml.example +17 -0
  4. data/CHANGELOG.md +102 -0
  5. data/README.md +64 -11
  6. data/RELEASE_CHECKLIST.md +13 -2
  7. data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +8 -0
  8. data/lib/schema_reaper/alert_config.rb +21 -1
  9. data/lib/schema_reaper/alert_state.rb +35 -0
  10. data/lib/schema_reaper/analyzers/always_null_column.rb +12 -5
  11. data/lib/schema_reaper/analyzers/base.rb +66 -2
  12. data/lib/schema_reaper/analyzers/dead_column.rb +2 -2
  13. data/lib/schema_reaper/analyzers/dead_table.rb +10 -6
  14. data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb +53 -0
  15. data/lib/schema_reaper/analyzers/duplicate_index.rb +1 -1
  16. data/lib/schema_reaper/analyzers/missing_fk_index.rb +10 -4
  17. data/lib/schema_reaper/analyzers/sequence_exhaustion.rb +55 -0
  18. data/lib/schema_reaper/analyzers/single_value_column.rb +17 -5
  19. data/lib/schema_reaper/analyzers/unused_index.rb +11 -6
  20. data/lib/schema_reaper/baseline.rb +95 -10
  21. data/lib/schema_reaper/cli/baseline_support.rb +45 -0
  22. data/lib/schema_reaper/cli.rb +69 -14
  23. data/lib/schema_reaper/config.rb +28 -0
  24. data/lib/schema_reaper/doctor.rb +151 -0
  25. data/lib/schema_reaper/explain.rb +102 -0
  26. data/lib/schema_reaper/gem_awareness.rb +31 -1
  27. data/lib/schema_reaper/inflect.rb +36 -0
  28. data/lib/schema_reaper/introspect/postgres.rb +236 -51
  29. data/lib/schema_reaper/migration_generator.rb +90 -4
  30. data/lib/schema_reaper/notifier.rb +58 -6
  31. data/lib/schema_reaper/railtie.rb +1 -1
  32. data/lib/schema_reaper/reporters/sarif.rb +15 -6
  33. data/lib/schema_reaper/reporters/slack.rb +67 -0
  34. data/lib/schema_reaper/runner.rb +58 -6
  35. data/lib/schema_reaper/runtime.rb +94 -11
  36. data/lib/schema_reaper/schema.rb +32 -3
  37. data/lib/schema_reaper/schema_locator.rb +75 -0
  38. data/lib/schema_reaper/static/cache.rb +66 -0
  39. data/lib/schema_reaper/static/scanner.rb +174 -6
  40. data/lib/schema_reaper/version.rb +1 -1
  41. data/lib/schema_reaper.rb +7 -0
  42. data/schema_reaper.gemspec +0 -1
  43. metadata +13 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 89431a69d7cd997b743e0a7996ce1bae64244946dbea4d0683f8beb477ad1498
4
- data.tar.gz: d43309fe35f2f6b4c669e21413c300997fdabf03f3d77f3295a6742de5f3cea2
3
+ metadata.gz: 65c5a4cc1857d1fbf940b5907cbb7d8e48de74cebb7b69eb6dd1b8e2a0c26bfa
4
+ data.tar.gz: 679e01d41a797c37dfd94858464127b1ce8789bda1d92cb905f011a2369fe1c8
5
5
  SHA512:
6
- metadata.gz: 6d2c527fc15a64ce62680a96a989c1ba0fbda445fd216afe590dd95e3344b1a8c2ef271a102eed01626b1cac1b0e9400874c8cc177f71d1d27efa32fda79e6fb
7
- data.tar.gz: 777a1561b89af7d5795a326d9c30d6764190b14f6a3dbb1af5f22d2fb525f3959d73e2f3beaa25262a9508f94958a1c1cad42d908f879855f1643f0206bc1208
6
+ metadata.gz: ef8476dfea36578ef5c97c43fc97c7334371ce38aa14f0d736028d57e42d370b43363922511676af417d76f5fd9b9b35536e98168253efc9c6a663230c56d8c6
7
+ data.tar.gz: cf4f22deed425352ac3d0553deefb34495e4c234170f402167866fa445b4b4bf707fc3d078a3a612fc99477ed39b9c152f96f4cdfb45555ecfa7a0eaaab9cd82
data/.rubocop.yml CHANGED
@@ -31,6 +31,7 @@ Metrics/BlockLength:
31
31
  - "*.gemspec"
32
32
 
33
33
  Metrics/ParameterLists:
34
+ CountKeywordArgs: false # keyword args name themselves at every call site
34
35
  Exclude:
35
36
  - "spec/**/*"
36
37
 
@@ -45,6 +46,8 @@ Metrics/MethodLength:
45
46
 
46
47
  Metrics/ClassLength:
47
48
  Max: 135
49
+ Exclude:
50
+ - "lib/schema_reaper/introspect/postgres.rb" # mostly SQL heredocs, one per catalog query
48
51
 
49
52
  Metrics/CyclomaticComplexity:
50
53
  Max: 12
@@ -2,6 +2,13 @@
2
2
  ---
3
3
  # database_url: postgres://localhost/myapp_development # else ENV["DATABASE_URL"]
4
4
 
5
+ # PostgreSQL schemas to scan. Tables outside public are reported as schema.table.
6
+ schemas:
7
+ - public
8
+
9
+ # Hide findings below this confidence on every command (the CLI flag overrides).
10
+ min_confidence: 0.0
11
+
5
12
  scan_paths:
6
13
  - app
7
14
  - lib
@@ -33,11 +40,21 @@ gem_awareness: true
33
40
 
34
41
  # Runtime usage log written by the tracker (see README > Runtime signal).
35
42
  runtime_log: .schema_reaper/runtime.jsonl
43
+ # Rotate it to runtime.jsonl.1 past this many bytes; both are read. null: never.
44
+ runtime_log_max_bytes: 52428800
36
45
 
37
46
  # Append-only snapshot log for `schema_reaper trend`.
38
47
  history_log: .schema_reaper/history.jsonl
39
48
 
40
49
  baseline: .schema_reaper/baseline.json
41
50
 
51
+ # Per-file cache of scan results; repeat scans re-parse only changed files.
52
+ scan_cache: .schema_reaper/scan_cache.json
53
+
54
+ # global: a column name anywhere in code counts as use on every table.
55
+ # model: a name only in an app/models file counts for that model's table and
56
+ # the tables it mentions. Stricter; review its findings before acting.
57
+ column_scope: global
58
+
42
59
  # Extra Ruby files to load before analysis -- register custom analyzers here.
43
60
  require: []
data/CHANGELOG.md CHANGED
@@ -1,5 +1,107 @@
1
1
  # Changelog
2
2
 
3
+ ## [2.1.0] - 2026-10-14
4
+
5
+ ### Fixed
6
+ - **Introspection no longer mixes up same-named objects across schemas.**
7
+ Row counts were read from `pg_class` by name alone, foreign keys were joined
8
+ on constraint name alone (constraint names are only unique per table), and
9
+ the primary-key lookup resolved the table through `search_path`. All three
10
+ are now qualified by schema. Foreign keys are read from `pg_constraint`, so
11
+ composite keys keep their columns together.
12
+ - **`ignore.columns` now applies to every column analyzer.** It was only
13
+ honoured by `dead_column`; `always_null_column`, `single_value_column` and
14
+ `missing_fk_index` ignored it.
15
+ - **Partitioned tables are reported once, as the parent.** Each partition used
16
+ to appear as its own table -- and, unreferenced by name, got a `dead_table`
17
+ finding. The parent's row count is now summed from its partitions, and its
18
+ indexes' usage from the partitions' indexes, so `unused_index` no longer
19
+ flags every index on a partitioned table (PostgreSQL 12+).
20
+ - **Stats-based findings say when they are estimates.** On tables larger than
21
+ ANALYZE's 30,000-row sample, `always_null_column` (0.85 -> 0.7) and
22
+ `single_value_column` (0.6 -> 0.5) lower their confidence and say so, and
23
+ `single_value_column` no longer claims "the same value in every row" for a
24
+ column that also holds NULLs.
25
+ - **`missing_fk_index`** finds the referenced table in a non-public table's own
26
+ schema.
27
+ - **Runtime `observed_days` no longer depends on line order.** Several
28
+ processes append to one log, so the span is now min..max timestamp rather
29
+ than first..last line.
30
+
31
+ ### Added
32
+ - `schemas:` config key to scan PostgreSQL schemas besides `public`. Tables
33
+ outside `public` are reported as `schema.table`; `public` tables keep their
34
+ bare names, so existing baselines and ignore lists still match.
35
+ - `sequence_exhaustion` analyzer (CYCLE and descending sequences excluded): a serial/IDENTITY sequence past 50% (medium)
36
+ or 75% (high) of its column type's range -- e.g. an `integer` key fed by a
37
+ bigint sequence.
38
+ - `duplicate_foreign_key` analyzer: two FK constraints on the same columns
39
+ referencing the same key, with the same ON UPDATE / ON DELETE, MATCH and
40
+ deferrability (anything else changes behaviour when dropped). A validated
41
+ copy is kept over a NOT VALID one.
42
+ - `--only` / `--except` to choose analyzers (custom ones included), and a
43
+ `min_confidence` config key used by every command.
44
+ - `generate-migration TABLE --index NAME` (concurrent `remove_index` with
45
+ `disable_ddl_transaction!`; the migration checks at run time whether the
46
+ index is partitioned, which PostgreSQL can't drop concurrently, and drops it
47
+ plainly then) and `generate-migration TABLE` (`drop_table`). The
48
+ `unused_index` / `duplicate_index` fixes say the same for partitioned
49
+ indexes.
50
+ - The runtime tracker also samples tables read by executed SQL (`track_sql:`,
51
+ on by default; FROM / JOIN / UPDATE, not INSERT targets, so a write-only
52
+ table can still be found), so a table read only through raw SQL isn't
53
+ reported dead.
54
+ - The runtime log rotates to `<path>.1` past `runtime_log_max_bytes` (default
55
+ 50 MB), under a file lock so concurrent workers can't rotate twice; both
56
+ generations are read.
57
+ - `schema_reaper doctor`: checks the connection, statistics age, tables never
58
+ `ANALYZE`d, query history, replica status, runtime log length, expired
59
+ baseline entries, the schema file and alert config.
60
+ - `schema_reaper explain ID` (full or partial id): evidence, confidence,
61
+ reclaim, schema line, baseline status and the `generate-migration` command.
62
+ - `# schema_reaper:keep TARGET -- reason` comments in code silence findings.
63
+ - `column_scope: model` (opt-in): a column name seen only in a model file
64
+ counts for that model's table and the tables it mentions, not every table.
65
+ Models are identified by their full class path (`Admin::User` is not
66
+ `User`); STI subclasses map to their parent's table (resolved the way Ruby
67
+ resolves the constant), namespaced models to both possible defaults, and
68
+ abstract bases, POROs and concerns stay shared.
69
+ - Baseline entries carry an optional reason and expiry (`baseline --reason
70
+ --expires YYYY-MM-DD`); `scan --ci` fails on expired ones again, and passing
71
+ `--reason` / `--expires` renews them. `baseline --prune` drops resolved
72
+ entries without accepting new findings. With `--only` / `--except`,
73
+ `baseline` replaces or prunes only those analyzers' entries. The 2.0
74
+ `{"ids": [...]}` file is still read, and the file keeps an `ids` key
75
+ (unexpired entries) so a 2.0.x gem sharing it still works -- though 2.0.x
76
+ ignores reasons and expiry, and rewrites the file in the old format if it
77
+ runs `baseline`.
78
+ - Scheduled reports say what changed since the last one (via `Rails.cache`);
79
+ `config.alert_on = :changes` sends only then. A report only counts as sent
80
+ once a channel accepted it -- a non-2xx webhook response is a failure -- so
81
+ a failed delivery is retried next run instead of suppressed.
82
+ - Slack webhooks get a Block Kit summary instead of a Markdown table Slack
83
+ can't render; other receivers keep the full `{"text": ...}` report
84
+ (`config.webhook_format`).
85
+ - Gem awareness covers solid_queue, solid_cache, solid_cable, good_job,
86
+ delayed_job_active_record, flipper-active_record, acts-as-taggable-on,
87
+ doorkeeper, noticed, activerecord-session_store, discard, acts_as_list and
88
+ ancestry. Every analyzer now leaves a gem-owned table's columns alone.
89
+ - Per-file scan cache (`scan_cache:`), so repeat scans only re-parse changed
90
+ files. Keyed by a SHA-256 digest, so it survives across processes.
91
+ - SARIF results point at the defining line in `db/schema.rb` or
92
+ `db/structure.sql` instead of the top of `db/schema.rb`.
93
+ - `--color` / `--no-color` works on every command.
94
+
95
+ ### Changed
96
+ - Column byte estimates use `pg_stats.avg_width` scaled by the non-NULL share
97
+ when the table has been analysed, instead of a fixed per-type guess (16
98
+ bytes for text/jsonb) that also counted NULLs as full width. An all-NULL
99
+ column is estimated at ~0 bytes.
100
+ - `unused_index` evidence notes that `idx_scan` is per server, so indexes used
101
+ only on read replicas look unused on the primary.
102
+ - CI tests Ruby 3.4 and 4.0 too, and runs the live introspection specs against
103
+ a PostgreSQL 16 service.
104
+
3
105
  ## [2.0.1] - 2026-09-23
4
106
 
5
107
  ### Fixed
data/README.md CHANGED
@@ -75,7 +75,7 @@ gem "schema_reaper"
75
75
  | requirement | notes |
76
76
  |---|---|
77
77
  | Ruby >= 2.7 | CI-tested on 2.7 – 3.3 |
78
- | PostgreSQL + the `pg` gem | uses your app's own `pg` (every Rails + PostgreSQL app already bundles it); add `gem "pg"` for standalone CLI use. Tables in the `public` schema are scanned. MySQL is on the [roadmap](#roadmap). |
78
+ | PostgreSQL + the `pg` gem | uses your app's own `pg` (every Rails + PostgreSQL app already bundles it); add `gem "pg"` for standalone CLI use. Partitioned tables are reported once, as the parent. Tables in the `public` schema are scanned by default; list others under `schemas:` in the [config](#configuration). MySQL is on the [roadmap](#roadmap). |
79
79
  | Rails | optional for the CLI; the rake tasks and production automation need it, and the runtime tracker needs ActiveRecord |
80
80
 
81
81
  > [!IMPORTANT]
@@ -110,9 +110,16 @@ bundle exec schema_reaper scan --format sarif # GitHub code scanning
110
110
  bundle exec schema_reaper scan --format json
111
111
  bundle exec schema_reaper scan --ci # exit 1 on findings not in the baseline
112
112
  bundle exec schema_reaper scan --min-confidence 0.8
113
+ bundle exec schema_reaper scan --only dead_column unused_index # or --except ...
113
114
  bundle exec schema_reaper baseline # accept current findings
115
+ bundle exec schema_reaper baseline --reason "billing ETL" --expires 2027-03-31
116
+ bundle exec schema_reaper baseline --prune # drop entries that no longer occur
117
+ bundle exec schema_reaper explain users/legacy_api_token # one finding in full + next step
118
+ bundle exec schema_reaper doctor # is the setup good enough to trust a scan?
114
119
  bundle exec schema_reaper trend # snapshot + progress delta
115
- bundle exec schema_reaper generate-migration users legacy_api_token
120
+ bundle exec schema_reaper generate-migration users legacy_api_token # column: staged pair
121
+ bundle exec schema_reaper generate-migration users --index index_users_on_legacy_token
122
+ bundle exec schema_reaper generate-migration old_audit_logs # whole table
116
123
  ```
117
124
 
118
125
  <details>
@@ -124,9 +131,10 @@ bundle exec schema_reaper generate-migration users legacy_api_token
124
131
  |---|---|---|
125
132
  | `--format` | `table` | `table`, `json`, `markdown` or `sarif` |
126
133
  | `--ci` | off | exit 1 when a finding isn't in `.schema_reaper/baseline.json` |
127
- | `--min-confidence N` | `0.0` | hide findings below this confidence (0.0 – 1.0) |
134
+ | `--min-confidence N` | `min_confidence` in config, else `0.0` | hide findings below this confidence (0.0 – 1.0) |
135
+ | `--only A B` / `--except A B` | all | run only, or skip, these analyzers (also on `baseline` and `trend`) |
128
136
  | `--record` | off | also append this run to the history log used by `trend` |
129
- | `--color` / `--no-color` | auto | force colour on or off for the table report |
137
+ | `--color` / `--no-color` | auto | force colour on or off (works on every command) |
130
138
  | `--config PATH` | `.schema_reaper.yml` | config file to load (works on every command) |
131
139
 
132
140
  Colour is automatic on a terminal, and off when output is piped or `NO_COLOR` is set.
@@ -194,8 +202,14 @@ end
194
202
  Both channels fire when both are set — not a fallback chain, but two audiences:
195
203
  a dev-facing chat channel, and an inbox record for people who don't watch chat.
196
204
 
197
- - **Webhook** — a Slack-compatible `{"text": "..."}` JSON POST carrying the
198
- Markdown report, with a 10-second timeout.
205
+ Every report says what changed since the previous one (remembered in
206
+ `Rails.cache`). Set `config.alert_on = :changes` to send only when findings
207
+ appeared or were resolved; the default, `:always`, sends every run.
208
+
209
+ - **Webhook** — for a `hooks.slack.com` URL, a Slack Block Kit summary (counts,
210
+ the top or newly added findings, what was resolved); for any other URL, a
211
+ `{"text": "..."}` JSON POST carrying the full Markdown report. Force either
212
+ with `config.webhook_format = :slack` or `:text`. 10-second timeout.
199
213
  - **Email** — sent by `SchemaReaper::Mailer`, which inherits your app's
200
214
  `ApplicationMailer` (its `default from:`, delivery settings and so on), or
201
215
  `ActionMailer::Base` if you don't have one. Delivered with `deliver_later`.
@@ -295,17 +309,23 @@ for watching the real, live database.
295
309
  |---|---|---|
296
310
  | `dead_column` | column no code path references | static scan (+ runtime) |
297
311
  | `dead_table` | table with no model or query reference | static scan + row count |
298
- | `unused_index` | non-unique index with `idx_scan = 0` (needs query history) | `pg_stat_user_indexes` |
312
+ | `unused_index` | non-unique index with `idx_scan = 0` (needs query history; stats are per server, so check replicas too) | `pg_stat_user_indexes` |
299
313
  | `duplicate_index` | index that exactly duplicates another, or is a prefix of a wider one | schema shape |
300
314
  | `missing_fk_index` | `*_id` / foreign-key column with no index (polymorphic pairs need a `(type, id)` index) | schema shape |
301
315
  | `always_null_column` | `null_frac = 1.0` — no data at all | `pg_stats` |
302
316
  | `single_value_column` | one distinct value on a table of 500+ rows | `pg_stats` |
317
+ | `sequence_exhaustion` | serial / IDENTITY sequence past 50% of its column type's range (high at 75%) | `pg_sequences` |
318
+ | `duplicate_foreign_key` | two FK constraints on the same columns to the same table | `pg_constraint` |
303
319
 
304
320
  Columns and tables owned by common gems are whitelisted automatically when the
305
321
  gem is in your bundle: **devise**, **devise-api**, **paper_trail**,
306
322
  **audited**, **friendly_id**, **paranoia** / **acts_as_paranoid**,
307
- **activestorage**, **actiontext**, **pg_search**, **ahoy_matey** and
308
- **activeadmin**.
323
+ **activestorage**, **actiontext**, **pg_search**, **ahoy_matey**,
324
+ **activeadmin**, **solid_queue**, **solid_cache**, **solid_cable**,
325
+ **good_job**, **delayed_job_active_record**, **flipper-active_record**,
326
+ **acts-as-taggable-on**, **doorkeeper**, **noticed**,
327
+ **activerecord-session_store**, **discard**, **acts_as_list** and
328
+ **ancestry**. A table a gem owns outright is left alone by every analyzer.
309
329
 
310
330
  ## Runtime signal (optional, raises confidence)
311
331
 
@@ -322,7 +342,11 @@ SchemaReaper::Runtime::Tracker.install!(
322
342
  ```
323
343
 
324
344
  or, in Rails, boot with `SCHEMA_REAPER_TRACK=1` (and optionally
325
- `SCHEMA_REAPER_SAMPLE=0.05` for the sample rate). Let it run in staging or
345
+ `SCHEMA_REAPER_SAMPLE=0.05` for the sample rate). The tracker also samples
346
+ the tables named in executed SQL (pass `track_sql: false` to opt out), so a
347
+ table read only through raw SQL isn't reported as dead. The log rotates to
348
+ `runtime.jsonl.1` past `runtime_log_max_bytes` (50 MB), and both generations
349
+ are read. Let it run in staging or
326
350
  production for a couple of weeks. A column unseen in **both** code and
327
351
  ≥ 14 observed days of runtime data (`min_age_days` in
328
352
  [`.schema_reaper.yml`](#configuration)) reaches **0.9** confidence (0.8 if it's
@@ -406,6 +430,7 @@ override any of these defaults.
406
430
  ```yaml
407
431
  database_url: # falls back to DATABASE_URL, then config/database.yml
408
432
  database_yml: config/database.yml
433
+ schemas: [public] # PostgreSQL schemas to scan; non-public tables appear as schema.table
409
434
  scan_paths: [app, lib, config]
410
435
  view_globs: ["app/**/*.erb", "app/**/*.haml", "app/**/*.slim", "app/**/*.jbuilder"]
411
436
  ignore:
@@ -414,14 +439,42 @@ ignore:
414
439
  always_keep_columns: [id, created_at, updated_at, type]
415
440
  gem_awareness: true # auto-whitelist columns owned by known gems
416
441
  min_age_days: 14 # days of runtime data before dead_column trusts it
442
+ min_confidence: 0.0 # hide findings below this on every command
443
+ runtime_log_max_bytes: 52428800 # rotate the runtime log past this size (null: never)
417
444
  runtime_log: .schema_reaper/runtime.jsonl
418
445
  history_log: .schema_reaper/history.jsonl
419
446
  baseline: .schema_reaper/baseline.json
420
447
  require: [] # extra files to load, e.g. custom analyzers
448
+ scan_cache: .schema_reaper/scan_cache.json # per-file scan cache (null: off)
449
+ column_scope: global # or model -- see "Stricter column matching"
421
450
  ```
422
451
 
423
452
  </details>
424
453
 
454
+ ### Keeping a finding from code
455
+
456
+ A `schema_reaper:keep` comment anywhere in scanned Ruby or view files silences
457
+ matching findings, with the reason kept next to the code it's about:
458
+
459
+ ```ruby
460
+ class User < ApplicationRecord
461
+ # schema_reaper:keep legacy_api_token, users.import_ref -- read by the billing ETL
462
+ end
463
+ ```
464
+
465
+ A target is a column (on any table), `table.column`, a table (every finding on
466
+ it), or an index name.
467
+
468
+ ### Stricter column matching
469
+
470
+ By default a column counts as used if its name appears anywhere in scanned
471
+ code, so `status` in `Order` also hides a dead `status` on `users`. With
472
+ `column_scope: model`, a name seen only in an `app/models` file counts for that
473
+ model's table and for the tables the model mentions (associations, class
474
+ names); names in controllers, views, jobs and so on still count everywhere.
475
+ It finds more dead columns, at some risk of flagging one used through
476
+ metaprogramming -- review before acting.
477
+
425
478
  ### Custom analyzers
426
479
 
427
480
  ```ruby
@@ -447,7 +500,7 @@ require:
447
500
  > ✅ Self-hosted scheduled scans + alerts shipped in **v2.0** — see
448
501
  > [Production automation](#production-automation).
449
502
 
450
- - Runtime verdict fusion for index and table findings
503
+ - Runtime verdict fusion for index findings
451
504
  - Orphan-row and `schema.rb`↔DB drift analyzers
452
505
  - Disk/$ reclaim from real `pg_total_relation_size`
453
506
  - Mountable dashboard engine, trend charts
data/RELEASE_CHECKLIST.md CHANGED
@@ -152,8 +152,19 @@ a throwaway PostgreSQL with a deliberately dirty schema.
152
152
 
153
153
  ## 6. Publish the gem
154
154
 
155
- RubyGems requires account MFA; `gem push` prompts for an OTP that only the
156
- maintainer can enter.
155
+ Pushing the `vX.Y.Z` tag (§5) runs `.github/workflows/release.yml`: it checks
156
+ the tag matches `VERSION`, is on `main`, has a CHANGELOG section and no AI
157
+ trailers in history; runs the full suite against PostgreSQL; builds the gem
158
+ and diffs the packaged `lib/` against `git ls-files`; then (after approval, if
159
+ the `release` environment requires reviewers) pushes to RubyGems via Trusted
160
+ Publishing and creates the GitHub release from the CHANGELOG section.
161
+
162
+ One-time setup: rubygems.org -> schema_reaper -> Trusted publishers -> add
163
+ repository `aksshatt/schema_reaper`, workflow `release.yml`, environment
164
+ `release`.
165
+
166
+ Manual fallback -- RubyGems requires account MFA; `gem push` prompts for an
167
+ OTP that only the maintainer can enter.
157
168
 
158
169
  ```sh
159
170
  gem build schema_reaper.gemspec
@@ -15,4 +15,12 @@
15
15
  SchemaReaper::AlertConfig.configure do |config|
16
16
  # config.emails = %w[dev1@example.com dev2@example.com]
17
17
  # config.webhook_url = "https://hooks.slack.com/services/T000/B000/XXXX"
18
+
19
+ # :changes sends a report only when findings appeared or were resolved
20
+ # since the last one (needs Rails.cache); :always sends every run.
21
+ # config.alert_on = :changes
22
+
23
+ # :auto uses Slack Block Kit for hooks.slack.com URLs and a plain
24
+ # {"text": <markdown>} body for anything else. Force with :slack or :text.
25
+ # config.webhook_format = :auto
18
26
  end
@@ -30,11 +30,31 @@ module SchemaReaper
30
30
  end
31
31
  end
32
32
 
33
- attr_accessor :emails, :webhook_url
33
+ # alert_on: :always -- send every scheduled report (the 2.0 behaviour)
34
+ # :changes -- send only when findings appeared or were
35
+ # resolved since the last report
36
+ # webhook_format: :auto -- Slack Block Kit for hooks.slack.com, plain
37
+ # {"text": <markdown>} for anything else
38
+ # :slack / :text to force one
39
+ attr_accessor :emails, :webhook_url, :alert_on, :webhook_format
34
40
 
35
41
  def initialize
36
42
  @emails = []
37
43
  @webhook_url = nil
44
+ @alert_on = :always
45
+ @webhook_format = :auto
46
+ end
47
+
48
+ def changes_only?
49
+ alert_on.to_s == "changes"
50
+ end
51
+
52
+ def slack_blocks?
53
+ case webhook_format.to_s
54
+ when "slack" then true
55
+ when "text" then false
56
+ else webhook_url.to_s.include?("hooks.slack.com")
57
+ end
38
58
  end
39
59
 
40
60
  def emails?
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SchemaReaper
4
+ # Remembers which finding ids the last delivered report contained, so the
5
+ # next one can say what changed -- and, with alert_on: :changes, stay quiet
6
+ # when nothing did. Kept in Rails.cache (the manual trigger's cooldown
7
+ # already relies on it) because production filesystems are often
8
+ # ephemeral. Without a cache every report is treated as the first.
9
+ class AlertState
10
+ KEY = "schema_reaper:last_alert_ids"
11
+
12
+ def initialize(cache: default_cache)
13
+ @cache = cache
14
+ end
15
+
16
+ # @return [Array<String>, nil] nil when no earlier report is known
17
+ def previous_ids
18
+ @cache&.read(KEY)
19
+ rescue StandardError
20
+ nil
21
+ end
22
+
23
+ def remember(ids)
24
+ @cache&.write(KEY, ids.sort)
25
+ rescue StandardError
26
+ nil
27
+ end
28
+
29
+ private
30
+
31
+ def default_cache
32
+ defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : nil
33
+ end
34
+ end
35
+ end
@@ -20,6 +20,7 @@ module SchemaReaper
20
20
  table.columns.filter_map do |col|
21
21
  next unless col.always_null?
22
22
  next if config.always_keep_columns.include?(col.name)
23
+ next if ignored_column?(col.name)
23
24
  next if gem_reserved?(table.name, col.name)
24
25
 
25
26
  finding(
@@ -27,17 +28,23 @@ module SchemaReaper
27
28
  table: table.name,
28
29
  column: col.name,
29
30
  severity: :high,
30
- confidence: 0.85,
31
+ confidence: sampled_stats?(table) ? 0.7 : 0.85,
31
32
  bytes_per_row: col.bytes,
32
- evidence: [
33
- "pg_stats.null_frac = 1.0 across ~#{table.row_count} row(s)",
34
- "column carries no data"
35
- ],
33
+ evidence: evidence_for(table),
36
34
  suggested_fix: "verify with `SELECT count(#{col.name}) FROM #{table.name}` " \
37
35
  "then stage a removal"
38
36
  )
39
37
  end
40
38
  end
39
+
40
+ def evidence_for(table)
41
+ ev = ["pg_stats.null_frac = 1.0 across ~#{table.row_count} row(s)"]
42
+ ev << if sampled_stats?(table)
43
+ "estimated from a #{STATS_SAMPLE_ROWS}-row ANALYZE sample -- a few non-null rows could be missed"
44
+ else
45
+ "column carries no data"
46
+ end
47
+ end
41
48
  end
42
49
  end
43
50
  end
@@ -11,8 +11,14 @@ module SchemaReaper
11
11
  # gem_columns - Hash{table_name => Set<column_name>} reserved by gems
12
12
  # gem_owned_tables - Set<table_name> created and owned outright by a gem
13
13
  # config - Config
14
- Context = Struct.new(:schema, :used_tokens, :runtime, :gem_columns, :gem_owned_tables, :config,
14
+ # scan - Static::Result (optional; per-model detail
15
+ # for column_scope: model)
16
+ Context = Struct.new(:schema, :used_tokens, :runtime, :gem_columns, :gem_owned_tables, :config, :scan,
15
17
  keyword_init: true) do
18
+ def scan
19
+ self[:scan] || Static::Result.from_tokens(used_tokens || [])
20
+ end
21
+
16
22
  def runtime
17
23
  self[:runtime] || Runtime::Report.empty
18
24
  end
@@ -28,6 +34,11 @@ module SchemaReaper
28
34
 
29
35
  # Shared plumbing for analyzers: schema access and token lookup helpers.
30
36
  class Base
37
+ # ANALYZE reads a sample of 300 x default_statistics_target rows (30,000
38
+ # at the default of 100). Above that, pg_stats' null_frac and n_distinct
39
+ # are estimates: a value held by a handful of rows can be missed.
40
+ STATS_SAMPLE_ROWS = 30_000
41
+
31
42
  def self.type
32
43
  name.split("::").last.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym
33
44
  end
@@ -61,8 +72,61 @@ module SchemaReaper
61
72
  ctx.used_tokens.include?(token.to_s.downcase)
62
73
  end
63
74
 
75
+ # Whether code references `column` on `table`. With column_scope: model,
76
+ # a name seen only in model files counts just for the tables those
77
+ # models are, or mention (an association, a class name), so Order's
78
+ # `status` no longer hides a dead `status` on users.
79
+ def column_used?(table, column)
80
+ return used?(column) unless config.column_scope == :model
81
+
82
+ name = column.to_s.downcase
83
+ scan = ctx.scan
84
+ return true if scan.shared.include?(name)
85
+
86
+ forms = table_forms(table)
87
+ scan.models.any? do |model|
88
+ model.tokens.include?(name) && (model.tables.include?(table) || forms.any? { |f| model.tokens.include?(f) })
89
+ end
90
+ end
91
+
92
+ def table_forms(table)
93
+ bare = table.to_s.split(".").last
94
+ singular = Inflect.singularize(bare)
95
+ [bare, singular, singular.delete("_")].uniq
96
+ end
97
+
98
+ # Every column of a table a gem owns outright is the gem's business.
64
99
  def gem_reserved?(table, column)
65
- ctx.gem_columns.fetch(table, []).include?(column)
100
+ gem_owned_table?(table) || ctx.gem_columns.fetch(table, []).include?(column)
101
+ end
102
+
103
+ def sampled_stats?(table)
104
+ table.row_count.to_i > STATS_SAMPLE_ROWS
105
+ end
106
+
107
+ # The `ignore.columns` list (names or /regex/) applies to every
108
+ # column-level analyzer, not only dead_column.
109
+ def ignored_column?(name)
110
+ config.ignored_column?(name)
111
+ end
112
+
113
+ # The removal command for an index. DROP INDEX CONCURRENTLY is refused
114
+ # on a partitioned index ("cannot drop partitioned index ...
115
+ # concurrently"), and its per-partition indexes can't be dropped on
116
+ # their own while attached, so the only path is a plain drop, which
117
+ # takes an ACCESS EXCLUSIVE lock on the parent and every partition.
118
+ def remove_index_fix(table, index)
119
+ base = "remove_index #{sym(table.name)}, name: :#{index.name}"
120
+ return "#{base} (use algorithm: :concurrently in production)" unless index.partitioned
121
+
122
+ "#{base} (partitioned index: CONCURRENTLY is not possible -- this locks the table " \
123
+ "and all partitions while it runs, so schedule it for a quiet window)"
124
+ end
125
+
126
+ # A table name as a Ruby symbol literal for suggested fixes: :users, or
127
+ # :"billing.invoices" for a table outside the public schema.
128
+ def sym(name)
129
+ name.to_sym.inspect
66
130
  end
67
131
 
68
132
  def gem_owned_table?(table)
@@ -21,7 +21,7 @@ module SchemaReaper
21
21
  def dead_in(table)
22
22
  table.columns.filter_map do |col|
23
23
  next if keep?(table, col)
24
- next if used?(col.name)
24
+ next if column_used?(table.name, col.name)
25
25
  next if runtime.read?(table.name, col.name)
26
26
 
27
27
  finding(
@@ -33,7 +33,7 @@ module SchemaReaper
33
33
  bytes_per_row: col.bytes,
34
34
  evidence: evidence_for(table, col),
35
35
  suggested_fix: "Stage removal: `self.ignored_columns += %w[#{col.name}]` on the " \
36
- "model, deploy, then `remove_column :#{table.name}, :#{col.name}`."
36
+ "model, deploy, then `remove_column #{sym(table.name)}, :#{col.name}`."
37
37
  )
38
38
  end
39
39
  end
@@ -24,13 +24,13 @@ module SchemaReaper
24
24
 
25
25
  def ignored?(table)
26
26
  config.ignore_tables.include?(table.name) ||
27
- table.name.start_with?("active_storage_", "action_text_", "action_mailbox_") ||
27
+ table.name.split(".").last.start_with?("active_storage_", "action_text_", "action_mailbox_") ||
28
28
  gem_owned_table?(table.name)
29
29
  end
30
30
 
31
31
  def dead(table)
32
32
  return if referenced?(table)
33
- return if runtime.accessed.any? { |k| k.start_with?("#{table.name}.") }
33
+ return if runtime.table_accessed?(table.name)
34
34
 
35
35
  finding(
36
36
  type: :dead_table,
@@ -40,17 +40,21 @@ module SchemaReaper
40
40
  confidence: confidence_for(table),
41
41
  bytes_per_row: table.columns.sum(&:bytes),
42
42
  evidence: evidence_for(table),
43
- suggested_fix: "confirm no external consumer, then `drop_table :#{table.name}`"
43
+ suggested_fix: "confirm no external consumer, then `drop_table #{sym(table.name)}`"
44
44
  )
45
45
  end
46
46
 
47
47
  # Match the table name and its singular/camelized model form.
48
+ # A table outside public ("billing.invoices") is still referenced by its
49
+ # bare name in code -- `self.table_name = "billing.invoices"` tokenises
50
+ # to billing + invoices, and the model is Invoice.
48
51
  def referenced?(table)
49
- singular = singularize(table.name)
50
- forms = [table.name, singular, camelize(table.name), camelize(singular)]
52
+ bare = table.name.split(".").last
53
+ singular = singularize(bare)
54
+ forms = [bare, singular, camelize(bare), camelize(singular)]
51
55
  return true if forms.any? { |form| used?(form) }
52
56
 
53
- embedded_in_identifier?(table.name)
57
+ embedded_in_identifier?(bare)
54
58
  end
55
59
 
56
60
  def singularize(name)