schema_reaper 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/.schema_reaper.yml.example +17 -0
- data/CHANGELOG.md +144 -0
- data/PRO.md +11 -6
- data/README.md +421 -71
- data/RELEASE_CHECKLIST.md +13 -2
- data/lib/generators/schema_reaper/install/install_generator.rb +11 -11
- data/lib/generators/schema_reaper/install/templates/initializer.rb.tt +8 -0
- data/lib/schema_reaper/alert_config.rb +21 -1
- data/lib/schema_reaper/alert_state.rb +35 -0
- data/lib/schema_reaper/analyzers/always_null_column.rb +12 -5
- data/lib/schema_reaper/analyzers/base.rb +66 -2
- data/lib/schema_reaper/analyzers/dead_column.rb +12 -6
- data/lib/schema_reaper/analyzers/dead_table.rb +10 -6
- data/lib/schema_reaper/analyzers/duplicate_foreign_key.rb +53 -0
- data/lib/schema_reaper/analyzers/duplicate_index.rb +1 -1
- data/lib/schema_reaper/analyzers/missing_fk_index.rb +10 -4
- data/lib/schema_reaper/analyzers/sequence_exhaustion.rb +55 -0
- data/lib/schema_reaper/analyzers/single_value_column.rb +17 -5
- data/lib/schema_reaper/analyzers/unused_index.rb +11 -6
- data/lib/schema_reaper/baseline.rb +95 -10
- data/lib/schema_reaper/cli/baseline_support.rb +45 -0
- data/lib/schema_reaper/cli.rb +69 -14
- data/lib/schema_reaper/config.rb +28 -0
- data/lib/schema_reaper/doctor.rb +151 -0
- data/lib/schema_reaper/explain.rb +102 -0
- data/lib/schema_reaper/gem_awareness.rb +31 -1
- data/lib/schema_reaper/inflect.rb +36 -0
- data/lib/schema_reaper/introspect/postgres.rb +258 -55
- data/lib/schema_reaper/migration_generator.rb +90 -4
- data/lib/schema_reaper/notifier.rb +69 -8
- data/lib/schema_reaper/railtie.rb +1 -1
- data/lib/schema_reaper/reporters/sarif.rb +15 -6
- data/lib/schema_reaper/reporters/slack.rb +67 -0
- data/lib/schema_reaper/runner.rb +58 -6
- data/lib/schema_reaper/runtime.rb +94 -11
- data/lib/schema_reaper/scan_job.rb +24 -2
- data/lib/schema_reaper/schema.rb +32 -3
- data/lib/schema_reaper/schema_locator.rb +75 -0
- data/lib/schema_reaper/static/cache.rb +66 -0
- data/lib/schema_reaper/static/scanner.rb +174 -6
- data/lib/schema_reaper/tasks/schema_reaper.rake +2 -2
- data/lib/schema_reaper/version.rb +1 -1
- data/lib/schema_reaper.rb +18 -1
- data/schema_reaper.gemspec +1 -2
- metadata +13 -4
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SchemaReaper
|
|
4
|
+
# Just enough of ActiveSupport's inflector to map model class names to
|
|
5
|
+
# table names without depending on it. A wrong guess only weakens a
|
|
6
|
+
# heuristic -- it never makes a finding more confident.
|
|
7
|
+
module Inflect
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def underscore(str)
|
|
11
|
+
str.gsub(/([A-Z\d]+)([A-Z][a-z])/, '\\1_\\2').gsub(/([a-z\d])([A-Z])/, '\\1_\\2').downcase
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def pluralize(word)
|
|
15
|
+
case word
|
|
16
|
+
when /[^aeiou]y\z/ then word.sub(/y\z/, "ies")
|
|
17
|
+
when /(s|x|z|ch|sh)\z/ then "#{word}es"
|
|
18
|
+
else "#{word}s"
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def singularize(word)
|
|
23
|
+
case word
|
|
24
|
+
when /ies\z/ then word.sub(/ies\z/, "y")
|
|
25
|
+
when /(ss|sh|ch|x|z)es\z/ then word.sub(/es\z/, "")
|
|
26
|
+
when /s\z/ then word.sub(/s\z/, "")
|
|
27
|
+
else word
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# User -> users, LineItem -> line_items, Address -> addresses
|
|
32
|
+
def tableize(class_name)
|
|
33
|
+
pluralize(underscore(class_name))
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module SchemaReaper
|
|
4
|
+
# Reads a live database's schema and statistics. PostgreSQL only for now.
|
|
4
5
|
module Introspect
|
|
6
|
+
# `pg` is deliberately not a runtime dependency -- the host app already
|
|
7
|
+
# bundles its own (every Rails + PostgreSQL app does), and pinning a second
|
|
8
|
+
# version here would fight it. Without this rescue a missing gem surfaces
|
|
9
|
+
# as a baffling NameError instead.
|
|
10
|
+
def self.require_pg!
|
|
11
|
+
require "pg"
|
|
12
|
+
rescue LoadError
|
|
13
|
+
raise Error, "schema_reaper needs the `pg` gem to talk to PostgreSQL -- add `gem \"pg\"` to your Gemfile"
|
|
14
|
+
end
|
|
15
|
+
|
|
5
16
|
# Reads live schema + planner statistics from PostgreSQL using the `pg` gem
|
|
6
17
|
# directly, so the host app does not need to boot Rails.
|
|
7
18
|
class Postgres
|
|
@@ -21,24 +32,68 @@ module SchemaReaper
|
|
|
21
32
|
# expression itself contains: COALESCE(a, b), ROUND(x, 2), etc.
|
|
22
33
|
COLUMN_SEPARATOR = 31.chr
|
|
23
34
|
|
|
24
|
-
|
|
25
|
-
|
|
35
|
+
# Ceiling of each integer column type, for the sequence-exhaustion check:
|
|
36
|
+
# a bigint sequence feeding an integer column overflows at the column's
|
|
37
|
+
# limit, not the sequence's.
|
|
38
|
+
TYPE_MAX = {
|
|
39
|
+
"smallint" => 32_767, "integer" => 2_147_483_647, "bigint" => 9_223_372_036_854_775_807
|
|
40
|
+
}.freeze
|
|
41
|
+
|
|
42
|
+
# Tables in `public` keep their bare name, so every existing config,
|
|
43
|
+
# baseline and finding id stays valid; tables in any other scanned schema
|
|
44
|
+
# are reported as `schema.table`.
|
|
45
|
+
def self.display_name(namespace, table)
|
|
46
|
+
namespace == "public" ? table : "#{namespace}.#{table}"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def initialize(url, schemas: Config::DEFAULTS["schemas"])
|
|
50
|
+
Introspect.require_pg!
|
|
26
51
|
raise Error, NO_URL if url.nil? || url.empty?
|
|
27
52
|
|
|
28
|
-
@
|
|
29
|
-
|
|
30
|
-
raise Error, "could not connect to the database: #{e.message.strip}"
|
|
53
|
+
@schemas = Array(schemas)
|
|
54
|
+
@conn = connect(url)
|
|
31
55
|
end
|
|
32
56
|
|
|
33
57
|
def call
|
|
34
58
|
DatabaseSchema.new(
|
|
35
|
-
tables:
|
|
36
|
-
index_scan_total: index_scan_total
|
|
59
|
+
tables: table_refs.map { |ns, name| build_table(ns, name) },
|
|
60
|
+
index_scan_total: index_scan_total,
|
|
61
|
+
sequences: sequences
|
|
37
62
|
)
|
|
38
63
|
end
|
|
39
64
|
|
|
65
|
+
# Health facts for `schema_reaper doctor`, separate from #call so a scan
|
|
66
|
+
# never pays for them.
|
|
67
|
+
def diagnostics
|
|
68
|
+
{
|
|
69
|
+
server_version: exec("SHOW server_version").first&.fetch("server_version"),
|
|
70
|
+
stats_reset: exec(<<~SQL).first&.fetch("stats_reset"),
|
|
71
|
+
SELECT stats_reset FROM pg_stat_database WHERE datname = current_database()
|
|
72
|
+
SQL
|
|
73
|
+
never_analyzed: never_analyzed,
|
|
74
|
+
in_recovery: exec("SELECT pg_is_in_recovery() AS r").first&.fetch("r") == "t"
|
|
75
|
+
}
|
|
76
|
+
end
|
|
77
|
+
|
|
40
78
|
private
|
|
41
79
|
|
|
80
|
+
def never_analyzed
|
|
81
|
+
exec(<<~SQL, [pg_array(@schemas)]).map { |r| self.class.display_name(r["schemaname"], r["relname"]) }
|
|
82
|
+
SELECT schemaname, relname FROM pg_stat_user_tables
|
|
83
|
+
WHERE schemaname = ANY($1::text[]) AND last_analyze IS NULL AND last_autoanalyze IS NULL
|
|
84
|
+
ORDER BY schemaname, relname
|
|
85
|
+
SQL
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Kept out of #initialize: a `rescue PG::Error` there is evaluated for
|
|
89
|
+
# *any* exception, including require_pg!'s, and would itself raise NameError
|
|
90
|
+
# when pg is the thing that failed to load.
|
|
91
|
+
def connect(url)
|
|
92
|
+
PG.connect(url)
|
|
93
|
+
rescue PG::Error => e
|
|
94
|
+
raise Error, "could not connect to the database: #{e.message.strip}"
|
|
95
|
+
end
|
|
96
|
+
|
|
42
97
|
# Cluster-wide cumulative index scans. Lets the unused-index analyzer tell
|
|
43
98
|
# "this index is never used" apart from "this database has no query
|
|
44
99
|
# history", which look identical at the level of a single idx_scan = 0.
|
|
@@ -47,29 +102,39 @@ module SchemaReaper
|
|
|
47
102
|
.first&.fetch("total")&.to_i
|
|
48
103
|
end
|
|
49
104
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
105
|
+
# Partitions are left out: the app (and every analyzer) deals with the
|
|
106
|
+
# partitioned parent, so reporting `events_2026_01` as an unreferenced
|
|
107
|
+
# table would be wrong. The parent's row count and index usage are
|
|
108
|
+
# rolled up from its partitions instead.
|
|
109
|
+
def table_refs
|
|
110
|
+
exec(<<~SQL, [pg_array(@schemas)]).map { |r| [r["schemaname"], r["tablename"]] }
|
|
111
|
+
SELECT n.nspname AS schemaname, c.relname AS tablename
|
|
112
|
+
FROM pg_class c
|
|
113
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
114
|
+
WHERE c.relkind IN ('r', 'p') AND NOT c.relispartition AND n.nspname = ANY($1::text[])
|
|
115
|
+
ORDER BY n.nspname = 'public' DESC, n.nspname, c.relname
|
|
53
116
|
SQL
|
|
54
117
|
end
|
|
55
118
|
|
|
56
|
-
def build_table(name)
|
|
119
|
+
def build_table(namespace, name)
|
|
120
|
+
fks = foreign_key_constraints_for(namespace, name)
|
|
57
121
|
Table.new(
|
|
58
|
-
name: name,
|
|
59
|
-
columns: columns_for(name),
|
|
60
|
-
indexes: indexes_for(name),
|
|
61
|
-
primary_key: primary_key_for(name),
|
|
62
|
-
foreign_keys:
|
|
63
|
-
|
|
122
|
+
name: self.class.display_name(namespace, name),
|
|
123
|
+
columns: columns_for(namespace, name),
|
|
124
|
+
indexes: indexes_for(namespace, name),
|
|
125
|
+
primary_key: primary_key_for(namespace, name),
|
|
126
|
+
foreign_keys: fks.flat_map(&:columns).uniq,
|
|
127
|
+
foreign_key_constraints: fks,
|
|
128
|
+
row_count: row_count_for(namespace, name)
|
|
64
129
|
)
|
|
65
130
|
end
|
|
66
131
|
|
|
67
|
-
def columns_for(table)
|
|
68
|
-
stats = column_stats_for(table)
|
|
69
|
-
exec(<<~SQL, [table]).map do |r|
|
|
132
|
+
def columns_for(namespace, table)
|
|
133
|
+
stats = column_stats_for(namespace, table)
|
|
134
|
+
exec(<<~SQL, [namespace, table]).map do |r|
|
|
70
135
|
SELECT column_name, data_type, is_nullable, column_default
|
|
71
136
|
FROM information_schema.columns
|
|
72
|
-
WHERE table_schema =
|
|
137
|
+
WHERE table_schema = $1 AND table_name = $2
|
|
73
138
|
ORDER BY ordinal_position
|
|
74
139
|
SQL
|
|
75
140
|
s = stats[r["column_name"]] || {}
|
|
@@ -78,37 +143,60 @@ module SchemaReaper
|
|
|
78
143
|
sql_type: r["data_type"],
|
|
79
144
|
null: r["is_nullable"] == "YES",
|
|
80
145
|
default: r["column_default"],
|
|
81
|
-
bytes: AVG_TYPE_BYTES.fetch(r["data_type"], 16),
|
|
146
|
+
bytes: measured_bytes(s) || AVG_TYPE_BYTES.fetch(r["data_type"], 16),
|
|
82
147
|
null_fraction: s[:null_frac],
|
|
83
148
|
distinct_values: s[:n_distinct]
|
|
84
149
|
)
|
|
85
150
|
end
|
|
86
151
|
end
|
|
87
152
|
|
|
153
|
+
# avg_width covers non-null values only, and a NULL takes no space in
|
|
154
|
+
# the row (just its bit in the null bitmap), so scale by the share of
|
|
155
|
+
# rows that actually hold a value.
|
|
156
|
+
def measured_bytes(stats)
|
|
157
|
+
return 0 if stats[:null_frac].to_f >= 1.0 # all NULL: only the null-bitmap bit, avg_width 0
|
|
158
|
+
return nil unless stats[:avg_width]
|
|
159
|
+
|
|
160
|
+
(stats[:avg_width] * (1.0 - stats[:null_frac].to_f)).round
|
|
161
|
+
end
|
|
162
|
+
|
|
88
163
|
# pg_stats.n_distinct: >= 0 is an absolute count, < 0 is a ratio of rows.
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
164
|
+
# avg_width is the measured average stored width of the non-null values,
|
|
165
|
+
# far closer to reality than a per-type guess for text/jsonb/arrays.
|
|
166
|
+
def column_stats_for(namespace, table)
|
|
167
|
+
exec(<<~SQL, [namespace, table]).each_with_object({}) do |r, h|
|
|
168
|
+
SELECT attname, null_frac, n_distinct, avg_width
|
|
169
|
+
FROM pg_stats WHERE schemaname = $1 AND tablename = $2
|
|
170
|
+
-- later rows win below: a plain row (inherited = false) beats an inherited one, and a
|
|
171
|
+
-- partitioned parent, which only has inherited rows, still gets those
|
|
172
|
+
ORDER BY inherited DESC
|
|
93
173
|
SQL
|
|
94
174
|
nd = r["n_distinct"].to_f
|
|
175
|
+
width = r["avg_width"]&.to_i
|
|
95
176
|
h[r["attname"]] = {
|
|
96
177
|
null_frac: r["null_frac"].to_f,
|
|
97
|
-
n_distinct: nd >= 0 ? nd.round : nil
|
|
178
|
+
n_distinct: nd >= 0 ? nd.round : nil,
|
|
179
|
+
avg_width: width&.positive? ? width : nil
|
|
98
180
|
}
|
|
99
181
|
end
|
|
100
182
|
end
|
|
101
183
|
|
|
184
|
+
# A partitioned parent's index (relkind 'I') is never scanned itself --
|
|
185
|
+
# the planner scans each partition's index -- so its usage is the sum
|
|
186
|
+
# over the leaf partitions, or every such index would look unused.
|
|
187
|
+
#
|
|
102
188
|
# Columns come back in index-key order via pg_get_indexdef(indexrelid,
|
|
103
189
|
# column_no, pretty), keyed by column position rather than table attnum
|
|
104
190
|
# so it renders a plain column and an expression uniformly and never
|
|
105
191
|
# silently drops one. indnkeyatts excludes INCLUDE columns, which are
|
|
106
192
|
# payload only. partial (indpred IS NOT NULL) lets analyzers refuse to
|
|
107
193
|
# treat a conditional index as if it covered every row.
|
|
108
|
-
def indexes_for(table)
|
|
109
|
-
exec(<<~SQL, [table]).map do |r|
|
|
110
|
-
SELECT i.relname AS name,
|
|
111
|
-
|
|
194
|
+
def indexes_for(namespace, table)
|
|
195
|
+
exec(<<~SQL, [namespace, table]).map do |r|
|
|
196
|
+
SELECT i.relname AS name, (i.relkind = 'I') AS partitioned,
|
|
197
|
+
ix.indisunique AS "unique", ix.indisprimary AS "primary",
|
|
198
|
+
(ix.indpred IS NOT NULL) AS partial,
|
|
199
|
+
#{partitioned_index_scans_sql} AS scans,
|
|
112
200
|
array_to_string(
|
|
113
201
|
array_agg(pg_get_indexdef(ix.indexrelid, k.ord::int, true) ORDER BY k.ord),
|
|
114
202
|
'#{COLUMN_SEPARATOR}'
|
|
@@ -118,55 +206,170 @@ module SchemaReaper
|
|
|
118
206
|
JOIN pg_class i ON i.oid = ix.indexrelid
|
|
119
207
|
JOIN LATERAL generate_series(1, ix.indnkeyatts) AS k(ord) ON TRUE
|
|
120
208
|
LEFT JOIN pg_stat_user_indexes s ON s.indexrelid = i.oid
|
|
121
|
-
WHERE t.
|
|
122
|
-
GROUP BY i.relname, ix.indisunique, ix.indisprimary, ix.indpred, s.idx_scan
|
|
209
|
+
WHERE t.relnamespace = $1::regnamespace AND t.relname = $2
|
|
210
|
+
GROUP BY i.oid, i.relname, i.relkind, ix.indisunique, ix.indisprimary, ix.indpred, s.idx_scan
|
|
123
211
|
SQL
|
|
124
|
-
|
|
125
|
-
name: r["name"], columns: r["cols"].split(COLUMN_SEPARATOR),
|
|
126
|
-
unique: r["unique"] == "t", primary: r["primary"] == "t", partial: r["partial"] == "t",
|
|
127
|
-
scans: r["scans"]&.to_i
|
|
128
|
-
)
|
|
212
|
+
build_index(r)
|
|
129
213
|
end
|
|
130
214
|
end
|
|
131
215
|
|
|
216
|
+
def build_index(row)
|
|
217
|
+
Index.new(
|
|
218
|
+
name: row["name"], columns: row["cols"].split(COLUMN_SEPARATOR),
|
|
219
|
+
unique: row["unique"] == "t", primary: row["primary"] == "t", partial: row["partial"] == "t",
|
|
220
|
+
scans: row["scans"]&.to_i, partitioned: row["partitioned"] == "t"
|
|
221
|
+
)
|
|
222
|
+
end
|
|
223
|
+
|
|
132
224
|
# Every primary-key column, in key order. A composite key needs the whole
|
|
133
225
|
# list: returning one arbitrary column leaves the rest looking like
|
|
134
|
-
# ordinary columns to the analyzers.
|
|
135
|
-
|
|
136
|
-
|
|
226
|
+
# ordinary columns to the analyzers. The relation is resolved by schema
|
|
227
|
+
# and name rather than a ::regclass cast, which would follow search_path.
|
|
228
|
+
def primary_key_for(namespace, table)
|
|
229
|
+
exec(<<~SQL, [namespace, table]).map { |r| r["attname"] }
|
|
137
230
|
SELECT a.attname
|
|
138
231
|
FROM pg_index i
|
|
232
|
+
JOIN pg_class t ON t.oid = i.indrelid
|
|
139
233
|
JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) ON TRUE
|
|
140
234
|
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum
|
|
141
|
-
WHERE
|
|
235
|
+
WHERE t.relnamespace = $1::regnamespace AND t.relname = $2 AND i.indisprimary
|
|
142
236
|
ORDER BY k.ord
|
|
143
237
|
SQL
|
|
144
238
|
rescue Error, PG::Error
|
|
145
239
|
nil
|
|
146
240
|
end
|
|
147
241
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
242
|
+
# Read from pg_constraint rather than information_schema: constraint
|
|
243
|
+
# names are only unique per table, so joining key_column_usage on the
|
|
244
|
+
# name alone mixes up two tables' same-named constraints. conkey keeps
|
|
245
|
+
# a composite key's columns together and in order.
|
|
246
|
+
def foreign_key_constraints_for(namespace, table)
|
|
247
|
+
exec(<<~SQL, [namespace, table]).map { |r| build_foreign_key(r) }
|
|
248
|
+
SELECT c.conname AS name,
|
|
249
|
+
array_to_string(array_agg(a.attname ORDER BY k.ord), ',') AS cols,
|
|
250
|
+
rn.nspname AS ref_schema, rt.relname AS ref_table,
|
|
251
|
+
c.confkey::text AS ref_key, c.confupdtype, c.confdeltype, c.confmatchtype,
|
|
252
|
+
c.condeferrable, c.condeferred, c.convalidated
|
|
253
|
+
FROM pg_constraint c
|
|
254
|
+
JOIN pg_class t ON t.oid = c.conrelid
|
|
255
|
+
JOIN pg_class rt ON rt.oid = c.confrelid
|
|
256
|
+
JOIN pg_namespace rn ON rn.oid = rt.relnamespace
|
|
257
|
+
JOIN LATERAL unnest(c.conkey) WITH ORDINALITY AS k(attnum, ord) ON TRUE
|
|
258
|
+
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum
|
|
259
|
+
WHERE c.contype = 'f' AND t.relnamespace = $1::regnamespace AND t.relname = $2
|
|
260
|
+
#{"AND c.conparentid = 0" if version_num >= 110_000}
|
|
261
|
+
GROUP BY c.conname, rn.nspname, rt.relname, c.confkey, c.confupdtype, c.confdeltype,
|
|
262
|
+
c.confmatchtype, c.condeferrable, c.condeferred, c.convalidated
|
|
263
|
+
ORDER BY c.conname
|
|
156
264
|
SQL
|
|
157
265
|
end
|
|
158
266
|
|
|
267
|
+
def build_foreign_key(row)
|
|
268
|
+
ForeignKey.new(
|
|
269
|
+
name: row["name"], columns: row["cols"].split(","),
|
|
270
|
+
references: self.class.display_name(row["ref_schema"], row["ref_table"]),
|
|
271
|
+
# Everything that changes what the constraint does. Two FKs are
|
|
272
|
+
# only interchangeable when all of it matches.
|
|
273
|
+
behaviour: [row["ref_key"], row["confupdtype"], row["confdeltype"], row["confmatchtype"],
|
|
274
|
+
row["condeferrable"], row["condeferred"]].join(":"),
|
|
275
|
+
validated: row["convalidated"] == "t"
|
|
276
|
+
)
|
|
277
|
+
end
|
|
278
|
+
|
|
159
279
|
# pg_class.reltuples is an estimate maintained by ANALYZE/VACUUM. On
|
|
160
280
|
# PostgreSQL 14+ it is -1 for a relation that has never been analysed,
|
|
161
281
|
# which is "unknown", not "minus one row". Map anything negative to nil so
|
|
162
282
|
# analyzers treat the count as unavailable.
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
283
|
+
# A partitioned parent holds no rows itself: sum its leaf partitions,
|
|
284
|
+
# and treat the count as unknown if any leaf is unanalysed.
|
|
285
|
+
def row_count_for(namespace, table)
|
|
286
|
+
return legacy_row_count_for(namespace, table) unless pg12?
|
|
287
|
+
|
|
288
|
+
rows = exec(<<~SQL, [namespace, table])
|
|
289
|
+
SELECT leaf.reltuples
|
|
290
|
+
FROM pg_class t
|
|
291
|
+
JOIN LATERAL (
|
|
292
|
+
SELECT c.reltuples FROM pg_partition_tree(t.oid) pt JOIN pg_class c ON c.oid = pt.relid
|
|
293
|
+
WHERE t.relkind = 'p' AND pt.isleaf
|
|
294
|
+
UNION ALL
|
|
295
|
+
SELECT t.reltuples WHERE t.relkind = 'r'
|
|
296
|
+
) leaf ON TRUE
|
|
297
|
+
WHERE t.relnamespace = $1::regnamespace AND t.relname = $2
|
|
298
|
+
SQL
|
|
299
|
+
counts = rows.map { |r| r["reltuples"].to_f }
|
|
300
|
+
return nil if counts.empty? || counts.any?(&:negative?)
|
|
301
|
+
|
|
302
|
+
counts.sum.round
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# A CYCLE sequence wraps instead of running out, and a descending one
|
|
306
|
+
# counts towards min_value, so neither is reported.
|
|
307
|
+
#
|
|
308
|
+
# Sequences owned by a column (serial, bigserial or IDENTITY), with the
|
|
309
|
+
# ceiling that actually applies: the lower of the sequence's max_value
|
|
310
|
+
# and the owning column type's. pg_sequences.last_value is NULL until
|
|
311
|
+
# the sequence has been used, and unreadable without USAGE/SELECT on it.
|
|
312
|
+
def sequences
|
|
313
|
+
exec(<<~SQL, [pg_array(@schemas)]).map { |r| build_sequence(r) }
|
|
314
|
+
SELECT s.schemaname, s.sequencename, s.last_value, s.max_value, s.min_value,
|
|
315
|
+
tn.nspname AS table_schema, t.relname AS table_name, a.attname AS column_name,
|
|
316
|
+
format_type(a.atttypid, NULL) AS column_type
|
|
317
|
+
FROM pg_sequences s
|
|
318
|
+
JOIN pg_class sc ON sc.relname = s.sequencename AND sc.relnamespace = s.schemaname::regnamespace
|
|
319
|
+
JOIN pg_depend d ON d.objid = sc.oid AND d.classid = 'pg_class'::regclass
|
|
320
|
+
AND d.refclassid = 'pg_class'::regclass AND d.deptype IN ('a', 'i')
|
|
321
|
+
JOIN pg_class t ON t.oid = d.refobjid
|
|
322
|
+
JOIN pg_namespace tn ON tn.oid = t.relnamespace
|
|
323
|
+
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid
|
|
324
|
+
WHERE s.schemaname = ANY($1::text[]) AND s.last_value IS NOT NULL
|
|
325
|
+
AND NOT s.cycle AND s.increment_by > 0
|
|
326
|
+
SQL
|
|
327
|
+
rescue Error
|
|
328
|
+
[] # PG < 10 has no pg_sequences, or no privilege: skip, never fail the scan
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def build_sequence(row)
|
|
332
|
+
ceiling = [row["max_value"].to_i, TYPE_MAX.fetch(row["column_type"], row["max_value"].to_i)].min
|
|
333
|
+
Sequence.new(
|
|
334
|
+
name: self.class.display_name(row["schemaname"], row["sequencename"]),
|
|
335
|
+
table: self.class.display_name(row["table_schema"], row["table_name"]),
|
|
336
|
+
column: row["column_name"], column_type: row["column_type"],
|
|
337
|
+
last_value: row["last_value"].to_i, max_value: ceiling
|
|
338
|
+
)
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# PostgreSQL 10/11 lack pg_partition_tree (and 10 lacks conparentid).
|
|
342
|
+
# Both are still worth supporting; on them partitioned parents get the
|
|
343
|
+
# pre-partition-aware treatment.
|
|
344
|
+
def version_num
|
|
345
|
+
@version_num ||= exec("SHOW server_version_num").first["server_version_num"].to_i
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
def pg12?
|
|
349
|
+
version_num >= 120_000
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
def partitioned_index_scans_sql
|
|
353
|
+
return "s.idx_scan" unless pg12?
|
|
354
|
+
|
|
355
|
+
<<~SQL.strip
|
|
356
|
+
CASE WHEN i.relkind = 'I' THEN (
|
|
357
|
+
SELECT sum(cs.idx_scan) FROM pg_partition_tree(i.oid) pt
|
|
358
|
+
JOIN pg_stat_user_indexes cs ON cs.indexrelid = pt.relid
|
|
359
|
+
WHERE pt.isleaf
|
|
360
|
+
) ELSE s.idx_scan END
|
|
361
|
+
SQL
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
def legacy_row_count_for(namespace, table)
|
|
365
|
+
raw = exec(<<~SQL, [namespace, table]).first&.fetch("reltuples")&.to_f
|
|
366
|
+
SELECT reltuples FROM pg_class WHERE relnamespace = $1::regnamespace AND relname = $2
|
|
166
367
|
SQL
|
|
167
|
-
|
|
368
|
+
raw.nil? || raw.negative? ? nil : raw.round
|
|
369
|
+
end
|
|
168
370
|
|
|
169
|
-
|
|
371
|
+
def pg_array(values)
|
|
372
|
+
PG::TextEncoder::Array.new.encode(values)
|
|
170
373
|
end
|
|
171
374
|
|
|
172
375
|
def exec(sql, params = nil)
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
require "fileutils"
|
|
4
4
|
|
|
5
5
|
module SchemaReaper
|
|
6
|
-
# Emits a two-step
|
|
6
|
+
# Emits removal migrations: a two-step pair for a dead column (ignore, then
|
|
7
|
+
# drop), or a single migration for an index (index:) or a whole table (no
|
|
8
|
+
# column: and no index:).
|
|
7
9
|
class MigrationGenerator
|
|
8
10
|
# Enough of Rails' inflector for table names in this comment text -- a
|
|
9
11
|
# wrong guess doesn't break anything functional, just reads oddly.
|
|
@@ -13,19 +15,99 @@ module SchemaReaper
|
|
|
13
15
|
[/s\z/, ""] # employees -> employee
|
|
14
16
|
].freeze
|
|
15
17
|
|
|
16
|
-
def initialize(table:, column
|
|
18
|
+
def initialize(table:, column: nil, index: nil, dir: "db/migrate")
|
|
19
|
+
raise ArgumentError, "pass column: or index:, not both" if column && index
|
|
20
|
+
|
|
17
21
|
@table = table
|
|
18
22
|
@column = column
|
|
23
|
+
@index = index
|
|
19
24
|
@dir = dir
|
|
20
25
|
end
|
|
21
26
|
|
|
22
27
|
def call
|
|
23
28
|
FileUtils.mkdir_p(@dir)
|
|
29
|
+
return [drop_index] if @index
|
|
30
|
+
return [drop_table] unless @column
|
|
31
|
+
|
|
24
32
|
[stage_one, stage_two]
|
|
25
33
|
end
|
|
26
34
|
|
|
27
35
|
private
|
|
28
36
|
|
|
37
|
+
# CREATE/DROP INDEX CONCURRENTLY cannot run inside a transaction, hence
|
|
38
|
+
# disable_ddl_transaction!. A partitioned index can't be dropped
|
|
39
|
+
# concurrently at all, and the generator never sees the database, so the
|
|
40
|
+
# migration checks pg_class itself when it runs. The down raises rather
|
|
41
|
+
# than guessing at the index's definition; recreate it from db/schema.rb.
|
|
42
|
+
def drop_index
|
|
43
|
+
down_msg = "recreate #{@index} on :#{@table} from db/schema.rb if you need it back"
|
|
44
|
+
write "remove_#{@index}", <<~RUBY
|
|
45
|
+
# frozen_string_literal: true
|
|
46
|
+
|
|
47
|
+
# Removes #{@index} -- concurrently, so writes to #{@table} are not
|
|
48
|
+
# blocked, unless it is a partitioned index, which PostgreSQL can only
|
|
49
|
+
# drop with a lock on the table and every partition.
|
|
50
|
+
class Remove#{camelize(@index)} < ActiveRecord::Migration[#{migration_version}]
|
|
51
|
+
disable_ddl_transaction!
|
|
52
|
+
|
|
53
|
+
def up
|
|
54
|
+
if partitioned_index?
|
|
55
|
+
say "#{@index} is a partitioned index: dropping it without CONCURRENTLY (locks #{@table})"
|
|
56
|
+
remove_index #{@table.to_sym.inspect}, name: #{@index.to_sym.inspect}, if_exists: true
|
|
57
|
+
else
|
|
58
|
+
remove_index #{@table.to_sym.inspect}, name: #{@index.to_sym.inspect}, algorithm: :concurrently, if_exists: true
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def down
|
|
63
|
+
raise ActiveRecord::IrreversibleMigration, #{down_msg.inspect}
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
#{partitioned_check}
|
|
67
|
+
end
|
|
68
|
+
RUBY
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The generated migration's private helper, indented to sit inside the
|
|
72
|
+
# class body (the first line lands where the template places it).
|
|
73
|
+
def partitioned_check
|
|
74
|
+
[
|
|
75
|
+
"private",
|
|
76
|
+
"",
|
|
77
|
+
"def partitioned_index?",
|
|
78
|
+
" select_value(<<~SQL) == \"I\"",
|
|
79
|
+
" SELECT relkind FROM pg_class WHERE oid = to_regclass(\#{connection.quote(#{qualified_index.inspect})})",
|
|
80
|
+
" SQL",
|
|
81
|
+
"end"
|
|
82
|
+
].map.with_index { |line, i| i.zero? || line.empty? ? line : " #{line}" }.join("\n")
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# An index lives in its table's schema; qualify it so to_regclass doesn't
|
|
86
|
+
# depend on search_path.
|
|
87
|
+
def qualified_index
|
|
88
|
+
@table.include?(".") ? "#{@table.split(".").first}.#{@index}" : @index
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def drop_table
|
|
92
|
+
down_msg = "#{@table} and its rows are gone; restore from a backup if you need them back"
|
|
93
|
+
write "drop_#{@table}", <<~RUBY
|
|
94
|
+
# frozen_string_literal: true
|
|
95
|
+
|
|
96
|
+
# Drops #{@table} and every row in it. Run only after confirming no
|
|
97
|
+
# external consumer (reports, ETL, other services) reads it, and with
|
|
98
|
+
# a backup taken. Remove the #{model} model, if any, in the same deploy.
|
|
99
|
+
class Drop#{camelize(@table)}Table < ActiveRecord::Migration[#{migration_version}]
|
|
100
|
+
def up
|
|
101
|
+
drop_table #{@table.to_sym.inspect}
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def down
|
|
105
|
+
raise ActiveRecord::IrreversibleMigration, #{down_msg.inspect}
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
RUBY
|
|
109
|
+
end
|
|
110
|
+
|
|
29
111
|
def stage_one
|
|
30
112
|
note = "Add `self.ignored_columns += %w[#{@column}]` to the #{model} model, " \
|
|
31
113
|
"then deploy STEP 2 after a soak period."
|
|
@@ -67,7 +149,7 @@ module SchemaReaper
|
|
|
67
149
|
def write(slug, body)
|
|
68
150
|
@seq = (@seq || -1) + 1
|
|
69
151
|
ts = (Time.now + @seq).strftime("%Y%m%d%H%M%S")
|
|
70
|
-
path = File.join(@dir, "#{ts}_#{slug}.rb")
|
|
152
|
+
path = File.join(@dir, "#{ts}_#{slug.tr(".", "_")}.rb")
|
|
71
153
|
File.write(path, body)
|
|
72
154
|
path
|
|
73
155
|
end
|
|
@@ -81,7 +163,11 @@ module SchemaReaper
|
|
|
81
163
|
end
|
|
82
164
|
|
|
83
165
|
def camel
|
|
84
|
-
"#{@table}_#{@column}"
|
|
166
|
+
camelize("#{@table}_#{@column}")
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def camelize(str)
|
|
170
|
+
str.split(/[_.]/).map(&:capitalize).join
|
|
85
171
|
end
|
|
86
172
|
|
|
87
173
|
def model
|