schema_reaper 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/.schema_reaper.yml.example +17 -0
  4. data/CHANGELOG.md +144 -0
  5. data/PRO.md +11 -6
  6. data/README.md +421 -71
  7. data/RELEASE_CHECKLIST.md +13 -2
  8. data/lib/generators/schema_reaper/install/install_generator.rb +11 -11
  9. data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +8 -0
  10. data/lib/schema_reaper/alert_config.rb +21 -1
  11. data/lib/schema_reaper/alert_state.rb +35 -0
  12. data/lib/schema_reaper/analyzers/always_null_column.rb +12 -5
  13. data/lib/schema_reaper/analyzers/base.rb +66 -2
  14. data/lib/schema_reaper/analyzers/dead_column.rb +12 -6
  15. data/lib/schema_reaper/analyzers/dead_table.rb +10 -6
  16. data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb +53 -0
  17. data/lib/schema_reaper/analyzers/duplicate_index.rb +1 -1
  18. data/lib/schema_reaper/analyzers/missing_fk_index.rb +10 -4
  19. data/lib/schema_reaper/analyzers/sequence_exhaustion.rb +55 -0
  20. data/lib/schema_reaper/analyzers/single_value_column.rb +17 -5
  21. data/lib/schema_reaper/analyzers/unused_index.rb +11 -6
  22. data/lib/schema_reaper/baseline.rb +95 -10
  23. data/lib/schema_reaper/cli/baseline_support.rb +45 -0
  24. data/lib/schema_reaper/cli.rb +69 -14
  25. data/lib/schema_reaper/config.rb +28 -0
  26. data/lib/schema_reaper/doctor.rb +151 -0
  27. data/lib/schema_reaper/explain.rb +102 -0
  28. data/lib/schema_reaper/gem_awareness.rb +31 -1
  29. data/lib/schema_reaper/inflect.rb +36 -0
  30. data/lib/schema_reaper/introspect/postgres.rb +258 -55
  31. data/lib/schema_reaper/migration_generator.rb +90 -4
  32. data/lib/schema_reaper/notifier.rb +69 -8
  33. data/lib/schema_reaper/railtie.rb +1 -1
  34. data/lib/schema_reaper/reporters/sarif.rb +15 -6
  35. data/lib/schema_reaper/reporters/slack.rb +67 -0
  36. data/lib/schema_reaper/runner.rb +58 -6
  37. data/lib/schema_reaper/runtime.rb +94 -11
  38. data/lib/schema_reaper/scan_job.rb +24 -2
  39. data/lib/schema_reaper/schema.rb +32 -3
  40. data/lib/schema_reaper/schema_locator.rb +75 -0
  41. data/lib/schema_reaper/static/cache.rb +66 -0
  42. data/lib/schema_reaper/static/scanner.rb +174 -6
  43. data/lib/schema_reaper/tasks/schema_reaper.rake +2 -2
  44. data/lib/schema_reaper/version.rb +1 -1
  45. data/lib/schema_reaper.rb +18 -1
  46. data/schema_reaper.gemspec +1 -2
  47. metadata +13 -4
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SchemaReaper
4
+ # Just enough of ActiveSupport's inflector to map model class names to
5
+ # table names without depending on it. A wrong guess only weakens a
6
+ # heuristic -- it never makes a finding more confident.
7
+ module Inflect
8
+ module_function
9
+
10
+ def underscore(str)
11
+ str.gsub(/([A-Z\d]+)([A-Z][a-z])/, '\\1_\\2').gsub(/([a-z\d])([A-Z])/, '\\1_\\2').downcase
12
+ end
13
+
14
+ def pluralize(word)
15
+ case word
16
+ when /[^aeiou]y\z/ then word.sub(/y\z/, "ies")
17
+ when /(s|x|z|ch|sh)\z/ then "#{word}es"
18
+ else "#{word}s"
19
+ end
20
+ end
21
+
22
+ def singularize(word)
23
+ case word
24
+ when /ies\z/ then word.sub(/ies\z/, "y")
25
+ when /(ss|sh|ch|x|z)es\z/ then word.sub(/es\z/, "")
26
+ when /s\z/ then word.sub(/s\z/, "")
27
+ else word
28
+ end
29
+ end
30
+
31
+ # User -> users, LineItem -> line_items, Address -> addresses
32
+ def tableize(class_name)
33
+ pluralize(underscore(class_name))
34
+ end
35
+ end
36
+ end
@@ -1,7 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SchemaReaper
4
+ # Reads a live database's schema and statistics. PostgreSQL only for now.
4
5
  module Introspect
6
+ # `pg` is deliberately not a runtime dependency -- the host app already
7
+ # bundles its own (every Rails + PostgreSQL app does), and pinning a second
8
+ # version here would fight it. Without this rescue a missing gem surfaces
9
+ # as a baffling NameError instead.
10
+ def self.require_pg!
11
+ require "pg"
12
+ rescue LoadError
13
+ raise Error, "schema_reaper needs the `pg` gem to talk to PostgreSQL -- add `gem \"pg\"` to your Gemfile"
14
+ end
15
+
5
16
  # Reads live schema + planner statistics from PostgreSQL using the `pg` gem
6
17
  # directly, so the host app does not need to boot Rails.
7
18
  class Postgres
@@ -21,24 +32,68 @@ module SchemaReaper
21
32
  # expression itself contains: COALESCE(a, b), ROUND(x, 2), etc.
22
33
  COLUMN_SEPARATOR = 31.chr
23
34
 
24
- def initialize(url)
25
- require "pg" # load first so the PG::Error rescue below can resolve
35
+ # Ceiling of each integer column type, for the sequence-exhaustion check:
36
+ # a bigint sequence feeding an integer column overflows at the column's
37
+ # limit, not the sequence's.
38
+ TYPE_MAX = {
39
+ "smallint" => 32_767, "integer" => 2_147_483_647, "bigint" => 9_223_372_036_854_775_807
40
+ }.freeze
41
+
42
+ # Tables in `public` keep their bare name, so every existing config,
43
+ # baseline and finding id stays valid; tables in any other scanned schema
44
+ # are reported as `schema.table`.
45
+ def self.display_name(namespace, table)
46
+ namespace == "public" ? table : "#{namespace}.#{table}"
47
+ end
48
+
49
+ def initialize(url, schemas: Config::DEFAULTS["schemas"])
50
+ Introspect.require_pg!
26
51
  raise Error, NO_URL if url.nil? || url.empty?
27
52
 
28
- @conn = PG.connect(url)
29
- rescue PG::Error => e
30
- raise Error, "could not connect to the database: #{e.message.strip}"
53
+ @schemas = Array(schemas)
54
+ @conn = connect(url)
31
55
  end
32
56
 
33
57
  def call
34
58
  DatabaseSchema.new(
35
- tables: table_names.map { |n| build_table(n) },
36
- index_scan_total: index_scan_total
59
+ tables: table_refs.map { |ns, name| build_table(ns, name) },
60
+ index_scan_total: index_scan_total,
61
+ sequences: sequences
37
62
  )
38
63
  end
39
64
 
65
+ # Health facts for `schema_reaper doctor`, separate from #call so a scan
66
+ # never pays for them.
67
+ def diagnostics
68
+ {
69
+ server_version: exec("SHOW server_version").first&.fetch("server_version"),
70
+ stats_reset: exec(<<~SQL).first&.fetch("stats_reset"),
71
+ SELECT stats_reset FROM pg_stat_database WHERE datname = current_database()
72
+ SQL
73
+ never_analyzed: never_analyzed,
74
+ in_recovery: exec("SELECT pg_is_in_recovery() AS r").first&.fetch("r") == "t"
75
+ }
76
+ end
77
+
40
78
  private
41
79
 
80
+ def never_analyzed
81
+ exec(<<~SQL, [pg_array(@schemas)]).map { |r| self.class.display_name(r["schemaname"], r["relname"]) }
82
+ SELECT schemaname, relname FROM pg_stat_user_tables
83
+ WHERE schemaname = ANY($1::text[]) AND last_analyze IS NULL AND last_autoanalyze IS NULL
84
+ ORDER BY schemaname, relname
85
+ SQL
86
+ end
87
+
88
+ # Kept out of #initialize: a `rescue PG::Error` there is evaluated for
89
+ # *any* exception, including require_pg!'s, and would itself raise NameError
90
+ # when pg is the thing that failed to load.
91
+ def connect(url)
92
+ PG.connect(url)
93
+ rescue PG::Error => e
94
+ raise Error, "could not connect to the database: #{e.message.strip}"
95
+ end
96
+
42
97
  # Cluster-wide cumulative index scans. Lets the unused-index analyzer tell
43
98
  # "this index is never used" apart from "this database has no query
44
99
  # history", which look identical at the level of a single idx_scan = 0.
@@ -47,29 +102,39 @@ module SchemaReaper
47
102
  .first&.fetch("total")&.to_i
48
103
  end
49
104
 
50
- def table_names
51
- exec(<<~SQL).map { |r| r["tablename"] }
52
- SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename
105
+ # Partitions are left out: the app (and every analyzer) deals with the
106
+ # partitioned parent, so reporting `events_2026_01` as an unreferenced
107
+ # table would be wrong. The parent's row count and index usage are
108
+ # rolled up from its partitions instead.
109
+ def table_refs
110
+ exec(<<~SQL, [pg_array(@schemas)]).map { |r| [r["schemaname"], r["tablename"]] }
111
+ SELECT n.nspname AS schemaname, c.relname AS tablename
112
+ FROM pg_class c
113
+ JOIN pg_namespace n ON n.oid = c.relnamespace
114
+ WHERE c.relkind IN ('r', 'p') AND NOT c.relispartition AND n.nspname = ANY($1::text[])
115
+ ORDER BY n.nspname = 'public' DESC, n.nspname, c.relname
53
116
  SQL
54
117
  end
55
118
 
56
- def build_table(name)
119
+ def build_table(namespace, name)
120
+ fks = foreign_key_constraints_for(namespace, name)
57
121
  Table.new(
58
- name: name,
59
- columns: columns_for(name),
60
- indexes: indexes_for(name),
61
- primary_key: primary_key_for(name),
62
- foreign_keys: foreign_keys_for(name),
63
- row_count: row_count_for(name)
122
+ name: self.class.display_name(namespace, name),
123
+ columns: columns_for(namespace, name),
124
+ indexes: indexes_for(namespace, name),
125
+ primary_key: primary_key_for(namespace, name),
126
+ foreign_keys: fks.flat_map(&:columns).uniq,
127
+ foreign_key_constraints: fks,
128
+ row_count: row_count_for(namespace, name)
64
129
  )
65
130
  end
66
131
 
67
- def columns_for(table)
68
- stats = column_stats_for(table)
69
- exec(<<~SQL, [table]).map do |r|
132
+ def columns_for(namespace, table)
133
+ stats = column_stats_for(namespace, table)
134
+ exec(<<~SQL, [namespace, table]).map do |r|
70
135
  SELECT column_name, data_type, is_nullable, column_default
71
136
  FROM information_schema.columns
72
- WHERE table_schema = 'public' AND table_name = $1
137
+ WHERE table_schema = $1 AND table_name = $2
73
138
  ORDER BY ordinal_position
74
139
  SQL
75
140
  s = stats[r["column_name"]] || {}
@@ -78,37 +143,60 @@ module SchemaReaper
78
143
  sql_type: r["data_type"],
79
144
  null: r["is_nullable"] == "YES",
80
145
  default: r["column_default"],
81
- bytes: AVG_TYPE_BYTES.fetch(r["data_type"], 16),
146
+ bytes: measured_bytes(s) || AVG_TYPE_BYTES.fetch(r["data_type"], 16),
82
147
  null_fraction: s[:null_frac],
83
148
  distinct_values: s[:n_distinct]
84
149
  )
85
150
  end
86
151
  end
87
152
 
153
+ # avg_width covers non-null values only, and a NULL takes no space in
154
+ # the row (just its bit in the null bitmap), so scale by the share of
155
+ # rows that actually hold a value.
156
+ def measured_bytes(stats)
157
+ return 0 if stats[:null_frac].to_f >= 1.0 # all NULL: only the null-bitmap bit, avg_width 0
158
+ return nil unless stats[:avg_width]
159
+
160
+ (stats[:avg_width] * (1.0 - stats[:null_frac].to_f)).round
161
+ end
162
+
88
163
  # pg_stats.n_distinct: >= 0 is an absolute count, < 0 is a ratio of rows.
89
- def column_stats_for(table)
90
- exec(<<~SQL, [table]).each_with_object({}) do |r, h|
91
- SELECT attname, null_frac, n_distinct
92
- FROM pg_stats WHERE schemaname = 'public' AND tablename = $1
164
+ # avg_width is the measured average stored width of the non-null values,
165
+ # far closer to reality than a per-type guess for text/jsonb/arrays.
166
+ def column_stats_for(namespace, table)
167
+ exec(<<~SQL, [namespace, table]).each_with_object({}) do |r, h|
168
+ SELECT attname, null_frac, n_distinct, avg_width
169
+ FROM pg_stats WHERE schemaname = $1 AND tablename = $2
170
+ -- later rows win below: a plain row (inherited = false) beats an inherited one, and a
171
+ -- partitioned parent, which only has inherited rows, still gets those
172
+ ORDER BY inherited DESC
93
173
  SQL
94
174
  nd = r["n_distinct"].to_f
175
+ width = r["avg_width"]&.to_i
95
176
  h[r["attname"]] = {
96
177
  null_frac: r["null_frac"].to_f,
97
- n_distinct: nd >= 0 ? nd.round : nil
178
+ n_distinct: nd >= 0 ? nd.round : nil,
179
+ avg_width: width&.positive? ? width : nil
98
180
  }
99
181
  end
100
182
  end
101
183
 
184
+ # A partitioned parent's index (relkind 'I') is never scanned itself --
185
+ # the planner scans each partition's index -- so its usage is the sum
186
+ # over the leaf partitions, or every such index would look unused.
187
+ #
102
188
  # Columns come back in index-key order via pg_get_indexdef(indexrelid,
103
189
  # column_no, pretty), keyed by column position rather than table attnum
104
190
  # so it renders a plain column and an expression uniformly and never
105
191
  # silently drops one. indnkeyatts excludes INCLUDE columns, which are
106
192
  # payload only. partial (indpred IS NOT NULL) lets analyzers refuse to
107
193
  # treat a conditional index as if it covered every row.
108
- def indexes_for(table)
109
- exec(<<~SQL, [table]).map do |r|
110
- SELECT i.relname AS name, ix.indisunique AS "unique", ix.indisprimary AS "primary",
111
- (ix.indpred IS NOT NULL) AS partial, s.idx_scan AS scans,
194
+ def indexes_for(namespace, table)
195
+ exec(<<~SQL, [namespace, table]).map do |r|
196
+ SELECT i.relname AS name, (i.relkind = 'I') AS partitioned,
197
+ ix.indisunique AS "unique", ix.indisprimary AS "primary",
198
+ (ix.indpred IS NOT NULL) AS partial,
199
+ #{partitioned_index_scans_sql} AS scans,
112
200
  array_to_string(
113
201
  array_agg(pg_get_indexdef(ix.indexrelid, k.ord::int, true) ORDER BY k.ord),
114
202
  '#{COLUMN_SEPARATOR}'
@@ -118,55 +206,170 @@ module SchemaReaper
118
206
  JOIN pg_class i ON i.oid = ix.indexrelid
119
207
  JOIN LATERAL generate_series(1, ix.indnkeyatts) AS k(ord) ON TRUE
120
208
  LEFT JOIN pg_stat_user_indexes s ON s.indexrelid = i.oid
121
- WHERE t.relname = $1 AND t.relnamespace = 'public'::regnamespace
122
- GROUP BY i.relname, ix.indisunique, ix.indisprimary, ix.indpred, s.idx_scan
209
+ WHERE t.relnamespace = $1::regnamespace AND t.relname = $2
210
+ GROUP BY i.oid, i.relname, i.relkind, ix.indisunique, ix.indisprimary, ix.indpred, s.idx_scan
123
211
  SQL
124
- Index.new(
125
- name: r["name"], columns: r["cols"].split(COLUMN_SEPARATOR),
126
- unique: r["unique"] == "t", primary: r["primary"] == "t", partial: r["partial"] == "t",
127
- scans: r["scans"]&.to_i
128
- )
212
+ build_index(r)
129
213
  end
130
214
  end
131
215
 
216
+ def build_index(row)
217
+ Index.new(
218
+ name: row["name"], columns: row["cols"].split(COLUMN_SEPARATOR),
219
+ unique: row["unique"] == "t", primary: row["primary"] == "t", partial: row["partial"] == "t",
220
+ scans: row["scans"]&.to_i, partitioned: row["partitioned"] == "t"
221
+ )
222
+ end
223
+
132
224
  # Every primary-key column, in key order. A composite key needs the whole
133
225
  # list: returning one arbitrary column leaves the rest looking like
134
- # ordinary columns to the analyzers.
135
- def primary_key_for(table)
136
- exec(<<~SQL, [table]).map { |r| r["attname"] }
226
+ # ordinary columns to the analyzers. The relation is resolved by schema
227
+ # and name rather than a ::regclass cast, which would follow search_path.
228
+ def primary_key_for(namespace, table)
229
+ exec(<<~SQL, [namespace, table]).map { |r| r["attname"] }
137
230
  SELECT a.attname
138
231
  FROM pg_index i
232
+ JOIN pg_class t ON t.oid = i.indrelid
139
233
  JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) ON TRUE
140
234
  JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum
141
- WHERE i.indrelid = $1::regclass AND i.indisprimary
235
+ WHERE t.relnamespace = $1::regnamespace AND t.relname = $2 AND i.indisprimary
142
236
  ORDER BY k.ord
143
237
  SQL
144
238
  rescue Error, PG::Error
145
239
  nil
146
240
  end
147
241
 
148
- def foreign_keys_for(table)
149
- exec(<<~SQL, [table]).map { |r| r["column_name"] }
150
- SELECT kcu.column_name
151
- FROM information_schema.table_constraints tc
152
- JOIN information_schema.key_column_usage kcu
153
- ON tc.constraint_name = kcu.constraint_name
154
- WHERE tc.constraint_type = 'FOREIGN KEY'
155
- AND tc.table_schema = 'public' AND tc.table_name = $1
242
+ # Read from pg_constraint rather than information_schema: constraint
243
+ # names are only unique per table, so joining key_column_usage on the
244
+ # name alone mixes up two tables' same-named constraints. conkey keeps
245
+ # a composite key's columns together and in order.
246
+ def foreign_key_constraints_for(namespace, table)
247
+ exec(<<~SQL, [namespace, table]).map { |r| build_foreign_key(r) }
248
+ SELECT c.conname AS name,
249
+ array_to_string(array_agg(a.attname ORDER BY k.ord), ',') AS cols,
250
+ rn.nspname AS ref_schema, rt.relname AS ref_table,
251
+ c.confkey::text AS ref_key, c.confupdtype, c.confdeltype, c.confmatchtype,
252
+ c.condeferrable, c.condeferred, c.convalidated
253
+ FROM pg_constraint c
254
+ JOIN pg_class t ON t.oid = c.conrelid
255
+ JOIN pg_class rt ON rt.oid = c.confrelid
256
+ JOIN pg_namespace rn ON rn.oid = rt.relnamespace
257
+ JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS k(attnum, ord) ON TRUE
258
+ JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum
259
+ WHERE c.contype = 'f' AND t.relnamespace = $1::regnamespace AND t.relname = $2
260
+ #{"AND c.conparentid = 0" if version_num >= 110_000}
261
+ GROUP BY c.conname, rn.nspname, rt.relname, c.confkey, c.confupdtype, c.confdeltype,
262
+ c.confmatchtype, c.condeferrable, c.condeferred, c.convalidated
263
+ ORDER BY c.conname
156
264
  SQL
157
265
  end
158
266
 
267
+ def build_foreign_key(row)
268
+ ForeignKey.new(
269
+ name: row["name"], columns: row["cols"].split(","),
270
+ references: self.class.display_name(row["ref_schema"], row["ref_table"]),
271
+ # Everything that changes what the constraint does. Two FKs are
272
+ # only interchangeable when all of it matches.
273
+ behaviour: [row["ref_key"], row["confupdtype"], row["confdeltype"], row["confmatchtype"],
274
+ row["condeferrable"], row["condeferred"]].join(":"),
275
+ validated: row["convalidated"] == "t"
276
+ )
277
+ end
278
+
159
279
  # pg_class.reltuples is an estimate maintained by ANALYZE/VACUUM. On
160
280
  # PostgreSQL 14+ it is -1 for a relation that has never been analysed,
161
281
  # which is "unknown", not "minus one row". Map anything negative to nil so
162
282
  # analyzers treat the count as unavailable.
163
- def row_count_for(table)
164
- raw = exec(<<~SQL, [table]).first&.fetch("reltuples")&.to_f
165
- SELECT reltuples FROM pg_class WHERE relname = $1
283
+ # A partitioned parent holds no rows itself: sum its leaf partitions,
284
+ # and treat the count as unknown if any leaf is unanalysed.
285
+ def row_count_for(namespace, table)
286
+ return legacy_row_count_for(namespace, table) unless pg12?
287
+
288
+ rows = exec(<<~SQL, [namespace, table])
289
+ SELECT leaf.reltuples
290
+ FROM pg_class t
291
+ JOIN LATERAL (
292
+ SELECT c.reltuples FROM pg_partition_tree(t.oid) pt JOIN pg_class c ON c.oid = pt.relid
293
+ WHERE t.relkind = 'p' AND pt.isleaf
294
+ UNION ALL
295
+ SELECT t.reltuples WHERE t.relkind = 'r'
296
+ ) leaf ON TRUE
297
+ WHERE t.relnamespace = $1::regnamespace AND t.relname = $2
298
+ SQL
299
+ counts = rows.map { |r| r["reltuples"].to_f }
300
+ return nil if counts.empty? || counts.any?(&:negative?)
301
+
302
+ counts.sum.round
303
+ end
304
+
305
+ # A CYCLE sequence wraps instead of running out, and a descending one
306
+ # counts towards min_value, so neither is reported.
307
+ #
308
+ # Sequences owned by a column (serial, bigserial or IDENTITY), with the
309
+ # ceiling that actually applies: the lower of the sequence's max_value
310
+ # and the owning column type's. pg_sequences.last_value is NULL until
311
+ # the sequence has been used, and unreadable without USAGE/SELECT on it.
312
+ def sequences
313
+ exec(<<~SQL, [pg_array(@schemas)]).map { |r| build_sequence(r) }
314
+ SELECT s.schemaname, s.sequencename, s.last_value, s.max_value, s.min_value,
315
+ tn.nspname AS table_schema, t.relname AS table_name, a.attname AS column_name,
316
+ format_type(a.atttypid, NULL) AS column_type
317
+ FROM pg_sequences s
318
+ JOIN pg_class sc ON sc.relname = s.sequencename AND sc.relnamespace = s.schemaname::regnamespace
319
+ JOIN pg_depend d ON d.objid = sc.oid AND d.classid = 'pg_class'::regclass
320
+ AND d.refclassid = 'pg_class'::regclass AND d.deptype IN ('a', 'i')
321
+ JOIN pg_class t ON t.oid = d.refobjid
322
+ JOIN pg_namespace tn ON tn.oid = t.relnamespace
323
+ JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid
324
+ WHERE s.schemaname = ANY($1::text[]) AND s.last_value IS NOT NULL
325
+ AND NOT s.cycle AND s.increment_by > 0
326
+ SQL
327
+ rescue Error
328
+ [] # PG < 10 has no pg_sequences, or no privilege: skip, never fail the scan
329
+ end
330
+
331
+ def build_sequence(row)
332
+ ceiling = [row["max_value"].to_i, TYPE_MAX.fetch(row["column_type"], row["max_value"].to_i)].min
333
+ Sequence.new(
334
+ name: self.class.display_name(row["schemaname"], row["sequencename"]),
335
+ table: self.class.display_name(row["table_schema"], row["table_name"]),
336
+ column: row["column_name"], column_type: row["column_type"],
337
+ last_value: row["last_value"].to_i, max_value: ceiling
338
+ )
339
+ end
340
+
341
+ # PostgreSQL 10/11 lack pg_partition_tree (and 10 lacks conparentid).
342
+ # Both are still worth supporting; on them partitioned parents get the
343
+ # pre-partition-aware treatment.
344
+ def version_num
345
+ @version_num ||= exec("SHOW server_version_num").first["server_version_num"].to_i
346
+ end
347
+
348
+ def pg12?
349
+ version_num >= 120_000
350
+ end
351
+
352
+ def partitioned_index_scans_sql
353
+ return "s.idx_scan" unless pg12?
354
+
355
+ <<~SQL.strip
356
+ CASE WHEN i.relkind = 'I' THEN (
357
+ SELECT sum(cs.idx_scan) FROM pg_partition_tree(i.oid) pt
358
+ JOIN pg_stat_user_indexes cs ON cs.indexrelid = pt.relid
359
+ WHERE pt.isleaf
360
+ ) ELSE s.idx_scan END
361
+ SQL
362
+ end
363
+
364
+ def legacy_row_count_for(namespace, table)
365
+ raw = exec(<<~SQL, [namespace, table]).first&.fetch("reltuples")&.to_f
366
+ SELECT reltuples FROM pg_class WHERE relnamespace = $1::regnamespace AND relname = $2
166
367
  SQL
167
- return nil if raw.nil? || raw.negative?
368
+ raw.nil? || raw.negative? ? nil : raw.round
369
+ end
168
370
 
169
- raw.round
371
+ def pg_array(values)
372
+ PG::TextEncoder::Array.new.encode(values)
170
373
  end
171
374
 
172
375
  def exec(sql, params = nil)
@@ -3,7 +3,9 @@
3
3
  require "fileutils"
4
4
 
5
5
  module SchemaReaper
6
- # Emits a two-step, reversible migration pair for a dead column.
6
+ # Emits removal migrations: a two-step pair for a dead column (ignore, then
7
+ # drop), or a single migration for an index (index:) or a whole table (no
8
+ # column: and no index:).
7
9
  class MigrationGenerator
8
10
  # Enough of Rails' inflector for table names in this comment text -- a
9
11
  # wrong guess doesn't break anything functional, just reads oddly.
@@ -13,19 +15,99 @@ module SchemaReaper
13
15
  [/s\z/, ""] # employees -> employee
14
16
  ].freeze
15
17
 
16
- def initialize(table:, column:, dir: "db/migrate")
18
+ def initialize(table:, column: nil, index: nil, dir: "db/migrate")
19
+ raise ArgumentError, "pass column: or index:, not both" if column && index
20
+
17
21
  @table = table
18
22
  @column = column
23
+ @index = index
19
24
  @dir = dir
20
25
  end
21
26
 
22
27
  def call
23
28
  FileUtils.mkdir_p(@dir)
29
+ return [drop_index] if @index
30
+ return [drop_table] unless @column
31
+
24
32
  [stage_one, stage_two]
25
33
  end
26
34
 
27
35
  private
28
36
 
37
+ # CREATE/DROP INDEX CONCURRENTLY cannot run inside a transaction, hence
38
+ # disable_ddl_transaction!. A partitioned index can't be dropped
39
+ # concurrently at all, and the generator never sees the database, so the
40
+ # migration checks pg_class itself when it runs. The down raises rather
41
+ # than guessing at the index's definition; recreate it from db/schema.rb.
42
+ def drop_index
43
+ down_msg = "recreate #{@index} on :#{@table} from db/schema.rb if you need it back"
44
+ write "remove_#{@index}", <<~RUBY
45
+ # frozen_string_literal: true
46
+
47
+ # Removes #{@index} -- concurrently, so writes to #{@table} are not
48
+ # blocked, unless it is a partitioned index, which PostgreSQL can only
49
+ # drop with a lock on the table and every partition.
50
+ class Remove#{camelize(@index)} < ActiveRecord::Migration[#{migration_version}]
51
+ disable_ddl_transaction!
52
+
53
+ def up
54
+ if partitioned_index?
55
+ say "#{@index} is a partitioned index: dropping it without CONCURRENTLY (locks #{@table})"
56
+ remove_index #{@table.to_sym.inspect}, name: #{@index.to_sym.inspect}, if_exists: true
57
+ else
58
+ remove_index #{@table.to_sym.inspect}, name: #{@index.to_sym.inspect}, algorithm: :concurrently, if_exists: true
59
+ end
60
+ end
61
+
62
+ def down
63
+ raise ActiveRecord::IrreversibleMigration, #{down_msg.inspect}
64
+ end
65
+
66
+ #{partitioned_check}
67
+ end
68
+ RUBY
69
+ end
70
+
71
+ # The generated migration's private helper, indented to sit inside the
72
+ # class body (the first line lands where the template places it).
73
+ def partitioned_check
74
+ [
75
+ "private",
76
+ "",
77
+ "def partitioned_index?",
78
+ " select_value(<<~SQL) == \"I\"",
79
+ " SELECT relkind FROM pg_class WHERE oid = to_regclass(\#{connection.quote(#{qualified_index.inspect})})",
80
+ " SQL",
81
+ "end"
82
+ ].map.with_index { |line, i| i.zero? || line.empty? ? line : " #{line}" }.join("\n")
83
+ end
84
+
85
+ # An index lives in its table's schema; qualify it so to_regclass doesn't
86
+ # depend on search_path.
87
+ def qualified_index
88
+ @table.include?(".") ? "#{@table.split(".").first}.#{@index}" : @index
89
+ end
90
+
91
+ def drop_table
92
+ down_msg = "#{@table} and its rows are gone; restore from a backup if you need them back"
93
+ write "drop_#{@table}", <<~RUBY
94
+ # frozen_string_literal: true
95
+
96
+ # Drops #{@table} and every row in it. Run only after confirming no
97
+ # external consumer (reports, ETL, other services) reads it, and with
98
+ # a backup taken. Remove the #{model} model, if any, in the same deploy.
99
+ class Drop#{camelize(@table)}Table < ActiveRecord::Migration[#{migration_version}]
100
+ def up
101
+ drop_table #{@table.to_sym.inspect}
102
+ end
103
+
104
+ def down
105
+ raise ActiveRecord::IrreversibleMigration, #{down_msg.inspect}
106
+ end
107
+ end
108
+ RUBY
109
+ end
110
+
29
111
  def stage_one
30
112
  note = "Add `self.ignored_columns += %w[#{@column}]` to the #{model} model, " \
31
113
  "then deploy STEP 2 after a soak period."
@@ -67,7 +149,7 @@ module SchemaReaper
67
149
  def write(slug, body)
68
150
  @seq = (@seq || -1) + 1
69
151
  ts = (Time.now + @seq).strftime("%Y%m%d%H%M%S")
70
- path = File.join(@dir, "#{ts}_#{slug}.rb")
152
+ path = File.join(@dir, "#{ts}_#{slug.tr(".", "_")}.rb")
71
153
  File.write(path, body)
72
154
  path
73
155
  end
@@ -81,7 +163,11 @@ module SchemaReaper
81
163
  end
82
164
 
83
165
  def camel
84
- "#{@table}_#{@column}".split("_").map(&:capitalize).join
166
+ camelize("#{@table}_#{@column}")
167
+ end
168
+
169
+ def camelize(str)
170
+ str.split(/[_.]/).map(&:capitalize).join
85
171
  end
86
172
 
87
173
  def model