exwiw 0.9.20 → 0.9.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +32 -0
- data/README.md +215 -3
- data/docs/mongodb.md +13 -0
- data/lib/exwiw/adapter/mongodb_adapter.rb +7 -4
- data/lib/exwiw/adapter/mysql_adapter.rb +4 -7
- data/lib/exwiw/adapter/postgresql_adapter.rb +4 -7
- data/lib/exwiw/adapter/sqlite_adapter.rb +4 -7
- data/lib/exwiw/adapter.rb +27 -0
- data/lib/exwiw/cli.rb +190 -3
- data/lib/exwiw/db_introspector/mysql_introspector.rb +160 -0
- data/lib/exwiw/db_introspector/postgresql_introspector.rb +215 -0
- data/lib/exwiw/db_introspector.rb +166 -0
- data/lib/exwiw/db_schema_generator.rb +298 -0
- data/lib/exwiw/default_mask.rb +104 -0
- data/lib/exwiw/mask_value.rb +30 -0
- data/lib/exwiw/mongodb_collection_config.rb +22 -10
- data/lib/exwiw/mongodb_field.rb +5 -1
- data/lib/exwiw/mongoid_schema_generator.rb +368 -36
- data/lib/exwiw/query_ast.rb +1 -1
- data/lib/exwiw/schema_check.rb +167 -0
- data/lib/exwiw/schema_generator.rb +64 -5
- data/lib/exwiw/table_column.rb +7 -1
- data/lib/exwiw/table_config.rb +2 -2
- data/lib/exwiw/version.rb +1 -1
- data/lib/exwiw.rb +7 -0
- data/lib/tasks/exwiw.rake +61 -0
- metadata +8 -1
data/lib/exwiw/cli.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'fileutils'
|
|
3
4
|
require 'logger'
|
|
4
5
|
require 'optparse'
|
|
5
6
|
require 'pathname'
|
|
@@ -11,7 +12,18 @@ require 'exwiw'
|
|
|
11
12
|
|
|
12
13
|
module Exwiw
|
|
13
14
|
class CLI
|
|
14
|
-
KNOWN_SUBCOMMANDS = %w[export explain].freeze
|
|
15
|
+
KNOWN_SUBCOMMANDS = %w[export explain schema].freeze
|
|
16
|
+
|
|
17
|
+
# The verbs `exwiw schema` takes, mirroring the `exwiw:schema:*` rake tasks
|
|
18
|
+
# a Rails application would run instead.
|
|
19
|
+
SCHEMA_VERBS = %w[generate check tidy].freeze
|
|
20
|
+
|
|
21
|
+
# `schema check` exit codes. 1 means "the config needs work" — that is the
|
|
22
|
+
# signal CI acts on — so a failure to *perform* the check (an unreachable
|
|
23
|
+
# database, a malformed config) must not use it, or an infrastructure
|
|
24
|
+
# problem reads as a schema problem and gets "fixed" by regenerating.
|
|
25
|
+
SCHEMA_CHECK_DIRTY_EXIT = 1
|
|
26
|
+
SCHEMA_CHECK_ERROR_EXIT = 2
|
|
15
27
|
|
|
16
28
|
# Config file loaded automatically when --config is omitted, if one exists in
|
|
17
29
|
# the current directory. Kept at the project root (rather than under exwiw/)
|
|
@@ -52,7 +64,8 @@ module Exwiw
|
|
|
52
64
|
REJECTED_CONNECTION_KEYS = %w[host port user database uri password].freeze
|
|
53
65
|
|
|
54
66
|
# Keys that only make sense for `export`. They are skipped when merging config
|
|
55
|
-
# for `explain` so a shared config file does not trip validate_explain_only
|
|
67
|
+
# for `explain` so a shared config file does not trip validate_explain_only!,
|
|
68
|
+
# and for `schema`, which performs no export at all.
|
|
56
69
|
EXPORT_ONLY_CONFIG_KEYS = %w[output_dir output_format insert_only after_insert_hook parallel_workers].freeze
|
|
57
70
|
|
|
58
71
|
def self.start(argv)
|
|
@@ -69,6 +82,16 @@ module Exwiw
|
|
|
69
82
|
"export"
|
|
70
83
|
end
|
|
71
84
|
|
|
85
|
+
# `schema` is the one subcommand taking a verb of its own
|
|
86
|
+
# (`exwiw schema generate`), so the next positional argument belongs to
|
|
87
|
+
# it. An unknown or missing verb is not consumed here — validation
|
|
88
|
+
# reports it with the list of valid ones rather than OptionParser
|
|
89
|
+
# failing on a stray argument.
|
|
90
|
+
@schema_verb =
|
|
91
|
+
if @subcommand == "schema" && !@argv.empty? && !@argv.first.start_with?("-")
|
|
92
|
+
@argv.shift
|
|
93
|
+
end
|
|
94
|
+
|
|
72
95
|
@help = @argv.empty?
|
|
73
96
|
|
|
74
97
|
@database_host = nil
|
|
@@ -78,6 +101,7 @@ module Exwiw
|
|
|
78
101
|
@connection_uri = nil
|
|
79
102
|
@output_dir = nil
|
|
80
103
|
@schema_dir = nil
|
|
104
|
+
@from_db = false
|
|
81
105
|
@config_file_path = nil
|
|
82
106
|
@database_adapter = nil
|
|
83
107
|
@database_name = nil
|
|
@@ -151,9 +175,94 @@ module Exwiw
|
|
|
151
175
|
io: $stdout,
|
|
152
176
|
explain_verbosity: @explain_verbosity,
|
|
153
177
|
).run
|
|
178
|
+
when "schema"
|
|
179
|
+
run_schema(connection_config)
|
|
154
180
|
end
|
|
155
181
|
end
|
|
156
182
|
|
|
183
|
+
# `exwiw schema generate|check|tidy --from-db`: the same three operations
|
|
184
|
+
# the `exwiw:schema:*` rake tasks perform for a Rails application, driven
|
|
185
|
+
# from a database connection instead of from the application's models.
|
|
186
|
+
private def run_schema(connection_config)
|
|
187
|
+
introspector = DbIntrospector.build(connection_config)
|
|
188
|
+
|
|
189
|
+
case @schema_verb
|
|
190
|
+
when "generate" then run_schema_generate(introspector)
|
|
191
|
+
when "tidy" then run_schema_tidy(introspector)
|
|
192
|
+
when "check" then run_schema_check(introspector)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
private def run_schema_generate(introspector)
|
|
197
|
+
DbSchemaGenerator.new(
|
|
198
|
+
introspector: introspector,
|
|
199
|
+
output_dir: @schema_dir,
|
|
200
|
+
safe_new_columns: schema_safe_new_columns?,
|
|
201
|
+
).generate!
|
|
202
|
+
|
|
203
|
+
puts "exwiw: wrote the schema config for #{@database_name} to #{@schema_dir}."
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
private def run_schema_tidy(introspector)
|
|
207
|
+
result = DbSchemaGenerator.new(introspector: introspector, output_dir: @schema_dir).tidy!
|
|
208
|
+
|
|
209
|
+
if result.empty?
|
|
210
|
+
puts "exwiw: schema config is already tidy; nothing to remove."
|
|
211
|
+
return
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
result.removed_tables.each do |name|
|
|
215
|
+
puts "exwiw: removed config for table '#{name}' (no longer exists in the database)."
|
|
216
|
+
end
|
|
217
|
+
result.removed_columns.each do |table_name, columns|
|
|
218
|
+
puts "exwiw: removed column(s) #{columns.join(', ')} from '#{table_name}' (no longer in the table)."
|
|
219
|
+
end
|
|
220
|
+
result.removed_belongs_tos.each do |table_name, targets|
|
|
221
|
+
puts "exwiw: removed belongs_to(s) to #{targets.join(', ')} from '#{table_name}' " \
|
|
222
|
+
"(the target table no longer exists)."
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
private def run_schema_check(introspector)
|
|
227
|
+
# Regenerate exactly as `generate` + `tidy` would, into the copy
|
|
228
|
+
# SchemaCheck hands over, so the report is the diff against a config that
|
|
229
|
+
# has been through both steps rather than only the first.
|
|
230
|
+
regenerator = lambda do |tmp_dir|
|
|
231
|
+
# Explicit: safe mode is not optional here, whatever the library default is.
|
|
232
|
+
DbSchemaGenerator.new(introspector: introspector, output_dir: tmp_dir, safe_new_columns: true).generate!
|
|
233
|
+
DbSchemaGenerator.new(introspector: introspector, output_dir: tmp_dir).tidy!
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
report =
|
|
237
|
+
begin
|
|
238
|
+
SchemaCheck.new(schema_dir: @schema_dir, regenerator: regenerator).run
|
|
239
|
+
rescue StandardError => e
|
|
240
|
+
# See SCHEMA_CHECK_ERROR_EXIT: not being able to run the check is a
|
|
241
|
+
# different answer from "the config is out of date".
|
|
242
|
+
$stderr.puts "exwiw: could not run the schema check (#{e.class}: #{e.message})"
|
|
243
|
+
exit SCHEMA_CHECK_ERROR_EXIT
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
json = JSON.pretty_generate(report)
|
|
247
|
+
puts json
|
|
248
|
+
# A file too, so a caller need not assume stdout carries only the JSON.
|
|
249
|
+
File.write(ENV["EXWIW_SCHEMA_CHECK_OUTPUT"], json + "\n") if ENV["EXWIW_SCHEMA_CHECK_OUTPUT"]
|
|
250
|
+
|
|
251
|
+
return if SchemaCheck.clean?(report)
|
|
252
|
+
|
|
253
|
+
$stderr.puts "exwiw: the schema config is out of date or has undecided masking; " \
|
|
254
|
+
"run `exwiw schema generate --from-db` (then `exwiw schema tidy --from-db`) " \
|
|
255
|
+
"and resolve every `needs_mask_decision` column."
|
|
256
|
+
exit SCHEMA_CHECK_DIRTY_EXIT
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# Safe mode is on unless EXWIW_NEW_COLUMNS=plain, matching the rake task: a
|
|
260
|
+
# first-time bootstrap wants it off, since there every column is new and
|
|
261
|
+
# flagging the whole config at once is noise.
|
|
262
|
+
private def schema_safe_new_columns?
|
|
263
|
+
ENV["EXWIW_NEW_COLUMNS"] != "plain"
|
|
264
|
+
end
|
|
265
|
+
|
|
157
266
|
private def validate_options!
|
|
158
267
|
# Fill in any options not given on the CLI from the config file. Done first
|
|
159
268
|
# so a config-provided `adapter` is in place before normalization below.
|
|
@@ -188,6 +297,16 @@ module Exwiw
|
|
|
188
297
|
resolve_explain_verbosity!
|
|
189
298
|
end
|
|
190
299
|
|
|
300
|
+
# `schema` shares only the connection options with the other subcommands:
|
|
301
|
+
# it neither extracts rows nor reads a dump target, so every check below
|
|
302
|
+
# is about an export it will not perform. Its own requirements — a schema
|
|
303
|
+
# source, an introspectable adapter, a schema dir the verb can use — are
|
|
304
|
+
# validated instead.
|
|
305
|
+
if @subcommand == "schema"
|
|
306
|
+
validate_schema_options!
|
|
307
|
+
return
|
|
308
|
+
end
|
|
309
|
+
|
|
191
310
|
if @database_adapter != "sqlite"
|
|
192
311
|
# When a connection URI is supplied (mongodb only), host/port/database
|
|
193
312
|
# are read from the URI, so none of them are required on the CLI.
|
|
@@ -315,7 +434,9 @@ module Exwiw
|
|
|
315
434
|
|
|
316
435
|
# For `explain`, drop export-only keys so a config shared with `export`
|
|
317
436
|
# does not make validate_explain_only! reject the run.
|
|
318
|
-
|
|
437
|
+
if @subcommand == "explain" || @subcommand == "schema"
|
|
438
|
+
config = config.reject { |k, _| EXPORT_ONLY_CONFIG_KEYS.include?(k) }
|
|
439
|
+
end
|
|
319
440
|
|
|
320
441
|
@database_adapter ||= config["adapter"]
|
|
321
442
|
@schema_dir ||= expand_dir(config["schema_dir"], base)
|
|
@@ -494,6 +615,66 @@ module Exwiw
|
|
|
494
615
|
exit 1
|
|
495
616
|
end
|
|
496
617
|
|
|
618
|
+
# Everything `exwiw schema <verb>` needs, and nothing the export path
|
|
619
|
+
# requires. Runs instead of the export/explain validations, not after them.
|
|
620
|
+
private def validate_schema_options!
|
|
621
|
+
unless SCHEMA_VERBS.include?(@schema_verb)
|
|
622
|
+
$stderr.puts "Usage: exwiw schema #{SCHEMA_VERBS.join('|')} --from-db [options] " \
|
|
623
|
+
"(got #{@schema_verb.inspect})"
|
|
624
|
+
exit 1
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
# The schema source is spelled out rather than assumed. Reading a live
|
|
628
|
+
# database is one way to describe an application's schema and not the
|
|
629
|
+
# only conceivable one, so the flag keeps the command unambiguous today
|
|
630
|
+
# and leaves the reader expecting alternatives tomorrow.
|
|
631
|
+
unless @from_db
|
|
632
|
+
$stderr.puts "`exwiw schema #{@schema_verb}` requires --from-db to say where the schema is read from " \
|
|
633
|
+
"(a live database connection). A Rails application can use the exwiw:schema:#{@schema_verb} " \
|
|
634
|
+
"rake task instead, which reads its models."
|
|
635
|
+
exit 1
|
|
636
|
+
end
|
|
637
|
+
|
|
638
|
+
unless DbIntrospector::SUPPORTED_ADAPTERS.include?(@database_adapter)
|
|
639
|
+
$stderr.puts "--from-db supports the #{DbIntrospector::SUPPORTED_ADAPTERS.join(' and ')} adapters only " \
|
|
640
|
+
"(got '#{@database_adapter}'). sqlite and mongodb schemas are not read this way."
|
|
641
|
+
exit 1
|
|
642
|
+
end
|
|
643
|
+
|
|
644
|
+
{
|
|
645
|
+
"Target database host" => @database_host,
|
|
646
|
+
"Target database port" => @database_port,
|
|
647
|
+
"Target database name" => @database_name,
|
|
648
|
+
"Database user" => @database_user,
|
|
649
|
+
}.each do |name, value|
|
|
650
|
+
if value.nil?
|
|
651
|
+
$stderr.puts "#{name} is required"
|
|
652
|
+
exit 1
|
|
653
|
+
end
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
# Deliberately no DATABASE_PASSWORD requirement, unlike export: schema
|
|
657
|
+
# generation is commonly run in CI against a throwaway database started
|
|
658
|
+
# with trust/empty authentication, and refusing an empty password would
|
|
659
|
+
# make the check unrunnable exactly where it is most wanted. The password
|
|
660
|
+
# is still used when set.
|
|
661
|
+
|
|
662
|
+
if @schema_dir.nil?
|
|
663
|
+
$stderr.puts "Schema dir is required (pass --schema-dir or set schema_dir in the config file)"
|
|
664
|
+
exit 1
|
|
665
|
+
end
|
|
666
|
+
|
|
667
|
+
# `generate` is also the bootstrap command, so it creates the directory;
|
|
668
|
+
# `check` and `tidy` only ever read an existing config, and a missing
|
|
669
|
+
# directory there means the wrong path far more often than an empty one.
|
|
670
|
+
if @schema_verb == "generate"
|
|
671
|
+
FileUtils.mkdir_p(@schema_dir)
|
|
672
|
+
elsif !Dir.exist?(@schema_dir)
|
|
673
|
+
$stderr.puts "Schema dir does not exist: #{@schema_dir}"
|
|
674
|
+
exit 1
|
|
675
|
+
end
|
|
676
|
+
end
|
|
677
|
+
|
|
497
678
|
private def validate_explain_only!
|
|
498
679
|
rejected = []
|
|
499
680
|
rejected << "--output-dir" unless @output_dir.nil?
|
|
@@ -603,6 +784,11 @@ module Exwiw
|
|
|
603
784
|
For mongodb, set verbosity via EXWIW_MONGODB_EXPLAIN_VERBOSITY
|
|
604
785
|
or `explain_verbosity:` in config (queryPlanner (default,
|
|
605
786
|
no query is executed) | executionStats | allPlansExecution).
|
|
787
|
+
schema Maintain the schema config (generate | check | tidy) by
|
|
788
|
+
reading a live database, for applications that cannot be
|
|
789
|
+
loaded to generate it from their models. Requires --from-db;
|
|
790
|
+
mysql and postgresql only. `check` prints a JSON report and
|
|
791
|
+
exits 1 when the config needs work.
|
|
606
792
|
BANNER
|
|
607
793
|
opts.version = Exwiw::VERSION
|
|
608
794
|
|
|
@@ -617,6 +803,7 @@ module Exwiw
|
|
|
617
803
|
v = v.end_with?("/") ? v[0..-2] : v
|
|
618
804
|
@schema_dir = File.expand_path(v)
|
|
619
805
|
end
|
|
806
|
+
opts.on("--from-db", "Read the schema from the database the connection options point at (schema subcommand only; mysql/postgresql). Required by `exwiw schema`.") { @from_db = true }
|
|
620
807
|
opts.on("-c", "--config=CONFIG_FILE_PATH", "Path to the exwiw config YAML. Defaults to ./#{DEFAULT_CONFIG_PATHS.first} (or .#{File.extname(DEFAULT_CONFIG_PATHS.last)}) when present. CLI options take precedence; paths inside the file are resolved relative to the file.") do |v|
|
|
621
808
|
@config_file_path = File.expand_path(v)
|
|
622
809
|
end
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Exwiw
|
|
4
|
+
module DbIntrospector
|
|
5
|
+
# Reads MySQL's `information_schema`, scoped to `DATABASE()` — the database
|
|
6
|
+
# the connection was opened against, which is the one the dump would run in.
|
|
7
|
+
#
|
|
8
|
+
# Connects through MysqlClient, the same wrapper the dump path uses, so the
|
|
9
|
+
# driver choice (mysql2 or trilogy) and the "install the gem" error message
|
|
10
|
+
# are shared rather than reimplemented here.
|
|
11
|
+
class MysqlIntrospector < Base
|
|
12
|
+
# MySQL data_type (the type without its length/precision) -> the
|
|
13
|
+
# ActiveRecord-ish symbol DefaultMask understands. Anything absent maps to
|
|
14
|
+
# nil, which leaves the column unmasked: binary/blob columns must not
|
|
15
|
+
# receive a text mask, and enum/set/geometry/bit have no constant that is
|
|
16
|
+
# valid for every table's declaration.
|
|
17
|
+
TYPE_MAP = {
|
|
18
|
+
"char" => :string,
|
|
19
|
+
"varchar" => :string,
|
|
20
|
+
"tinytext" => :text,
|
|
21
|
+
"text" => :text,
|
|
22
|
+
"mediumtext" => :text,
|
|
23
|
+
"longtext" => :text,
|
|
24
|
+
"tinyint" => :integer,
|
|
25
|
+
"smallint" => :integer,
|
|
26
|
+
"mediumint" => :integer,
|
|
27
|
+
"int" => :integer,
|
|
28
|
+
"integer" => :integer,
|
|
29
|
+
"bigint" => :integer,
|
|
30
|
+
"decimal" => :decimal,
|
|
31
|
+
"numeric" => :decimal,
|
|
32
|
+
"float" => :float,
|
|
33
|
+
"double" => :float,
|
|
34
|
+
"date" => :date,
|
|
35
|
+
"datetime" => :datetime,
|
|
36
|
+
"timestamp" => :datetime,
|
|
37
|
+
"time" => :time,
|
|
38
|
+
"json" => :json,
|
|
39
|
+
}.freeze
|
|
40
|
+
|
|
41
|
+
# MySQL has no boolean type: `BOOLEAN` is an alias for `TINYINT(1)`, and
|
|
42
|
+
# the display width is the only trace of the distinction left in the
|
|
43
|
+
# catalog. Mapping it to :boolean (as every MySQL ORM does) is what makes
|
|
44
|
+
# a flag column mask to false / to its own default rather than to 0.
|
|
45
|
+
BOOLEAN_COLUMN_TYPE = "tinyint(1)"
|
|
46
|
+
|
|
47
|
+
# A default MySQL evaluates per row rather than storing as a literal.
|
|
48
|
+
# `extra` carries DEFAULT_GENERATED for an expression default, but only on
|
|
49
|
+
# servers new enough to support them, so the text is screened as well: a
|
|
50
|
+
# function call, or a bare keyword such as CURRENT_TIMESTAMP, is not a
|
|
51
|
+
# value we can mask with. A literal string that happens to contain
|
|
52
|
+
# parentheses is rejected too — losing a usable default costs nothing more
|
|
53
|
+
# than falling back to the per-type constant, while accepting an
|
|
54
|
+
# expression would write a mask the database re-evaluates.
|
|
55
|
+
EXPRESSION_DEFAULT = /[()]|\Acurrent_(?:timestamp|date|time)\z|\Alocaltime(?:stamp)?\z/i
|
|
56
|
+
|
|
57
|
+
def table_names
|
|
58
|
+
rows(<<~SQL).map { |row| row[0] }.sort
|
|
59
|
+
SELECT table_name
|
|
60
|
+
FROM information_schema.tables
|
|
61
|
+
WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'
|
|
62
|
+
SQL
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def primary_key(table_name)
|
|
66
|
+
names = rows(<<~SQL).map { |row| row[0] }
|
|
67
|
+
SELECT column_name
|
|
68
|
+
FROM information_schema.key_column_usage
|
|
69
|
+
WHERE table_schema = DATABASE()
|
|
70
|
+
AND table_name = #{quote(table_name)}
|
|
71
|
+
AND constraint_name = 'PRIMARY'
|
|
72
|
+
ORDER BY ordinal_position
|
|
73
|
+
SQL
|
|
74
|
+
|
|
75
|
+
case names.size
|
|
76
|
+
when 0 then nil
|
|
77
|
+
when 1 then names.first
|
|
78
|
+
else names
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def columns(table_name)
|
|
83
|
+
sql = <<~SQL
|
|
84
|
+
SELECT column_name, data_type, column_type, character_maximum_length, column_default, extra
|
|
85
|
+
FROM information_schema.columns
|
|
86
|
+
WHERE table_schema = DATABASE() AND table_name = #{quote(table_name)}
|
|
87
|
+
ORDER BY ordinal_position
|
|
88
|
+
SQL
|
|
89
|
+
|
|
90
|
+
rows(sql).map do |name, data_type, column_type, character_maximum_length, column_default, extra|
|
|
91
|
+
type = column_type == BOOLEAN_COLUMN_TYPE ? :boolean : TYPE_MAP[data_type]
|
|
92
|
+
Column.new(
|
|
93
|
+
name: name,
|
|
94
|
+
type: type,
|
|
95
|
+
limit: character_maximum_length&.to_i,
|
|
96
|
+
# MySQL has no array column type; a multi-valued column is JSON,
|
|
97
|
+
# which is masked as JSON rather than as an array.
|
|
98
|
+
array: false,
|
|
99
|
+
default: coerce_default(type, literal_default(column_default, extra)),
|
|
100
|
+
)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def unique_column_names(table_name)
|
|
105
|
+
rows(<<~SQL).map { |row| row[0] }.to_set
|
|
106
|
+
SELECT DISTINCT column_name
|
|
107
|
+
FROM information_schema.statistics
|
|
108
|
+
WHERE table_schema = DATABASE()
|
|
109
|
+
AND table_name = #{quote(table_name)}
|
|
110
|
+
AND non_unique = 0
|
|
111
|
+
SQL
|
|
112
|
+
rescue StandardError => e
|
|
113
|
+
warn_once(
|
|
114
|
+
:unique_column_names,
|
|
115
|
+
"exwiw: could not read the indexes of '#{table_name}' (#{e.class}); " \
|
|
116
|
+
"treating every column as unique-indexed so no constant mask is emitted.",
|
|
117
|
+
)
|
|
118
|
+
nil
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def foreign_keys(table_name)
|
|
122
|
+
# `referenced_table_name IS NOT NULL` is what distinguishes a foreign
|
|
123
|
+
# key's rows from the primary/unique key rows sharing this view.
|
|
124
|
+
build_foreign_keys(table_name, rows(<<~SQL))
|
|
125
|
+
SELECT constraint_name, column_name, referenced_table_name
|
|
126
|
+
FROM information_schema.key_column_usage
|
|
127
|
+
WHERE table_schema = DATABASE()
|
|
128
|
+
AND table_name = #{quote(table_name)}
|
|
129
|
+
AND referenced_table_name IS NOT NULL
|
|
130
|
+
ORDER BY constraint_name, ordinal_position
|
|
131
|
+
SQL
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# The catalog's `column_default` as a plain literal, or nil when it is
|
|
135
|
+
# absent or is an expression (see EXPRESSION_DEFAULT).
|
|
136
|
+
private def literal_default(column_default, extra)
|
|
137
|
+
return nil if column_default.nil?
|
|
138
|
+
return nil if extra.to_s.upcase.include?("DEFAULT_GENERATED")
|
|
139
|
+
return nil if column_default.match?(EXPRESSION_DEFAULT)
|
|
140
|
+
|
|
141
|
+
column_default
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
private def rows(sql)
|
|
145
|
+
connection.query(sql).rows
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Table names reaching this class come from `table_names` (the catalog
|
|
149
|
+
# itself), so they cannot carry an injection; quoting is defensive, for
|
|
150
|
+
# the day a caller passes a name from elsewhere.
|
|
151
|
+
private def quote(value)
|
|
152
|
+
"'#{value.to_s.gsub("\\", "\\\\\\\\").gsub("'", "''")}'"
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
private def connection
|
|
156
|
+
@connection ||= Adapter::MysqlClient.new(@connection_config)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Exwiw
|
|
4
|
+
module DbIntrospector
|
|
5
|
+
# Reads PostgreSQL's catalog, scoped to `current_schema()` — the first
|
|
6
|
+
# schema on the connection's search_path, which is the one an unqualified
|
|
7
|
+
# table name in the dump resolves to.
|
|
8
|
+
#
|
|
9
|
+
# `information_schema` is used where it is unambiguous (tables, columns,
|
|
10
|
+
# primary keys); unique indexes and foreign keys go through `pg_catalog`
|
|
11
|
+
# instead. information_schema only lists what a *constraint* declares, so a
|
|
12
|
+
# bare `CREATE UNIQUE INDEX` would be invisible there, and its
|
|
13
|
+
# `constraint_column_usage` join multiplies the rows of a composite foreign
|
|
14
|
+
# key into a cross product that cannot be grouped back.
|
|
15
|
+
class PostgresqlIntrospector < Base
|
|
16
|
+
# PostgreSQL data_type (information_schema's spelling) -> the
|
|
17
|
+
# ActiveRecord-ish symbol DefaultMask understands. An unmapped type stays
|
|
18
|
+
# nil so no mask is emitted: 'USER-DEFINED' covers every enum and
|
|
19
|
+
# extension type, where the set of valid values is per-column, and
|
|
20
|
+
# bytea/uuid/inet/interval have no constant that is safe to write back.
|
|
21
|
+
TYPE_MAP = {
|
|
22
|
+
"character" => :string,
|
|
23
|
+
"character varying" => :string,
|
|
24
|
+
"text" => :text,
|
|
25
|
+
"smallint" => :integer,
|
|
26
|
+
"integer" => :integer,
|
|
27
|
+
"bigint" => :integer,
|
|
28
|
+
"numeric" => :decimal,
|
|
29
|
+
"decimal" => :decimal,
|
|
30
|
+
"real" => :float,
|
|
31
|
+
"double precision" => :float,
|
|
32
|
+
"boolean" => :boolean,
|
|
33
|
+
"date" => :date,
|
|
34
|
+
"timestamp without time zone" => :datetime,
|
|
35
|
+
"timestamp with time zone" => :datetime,
|
|
36
|
+
"time without time zone" => :time,
|
|
37
|
+
"time with time zone" => :time,
|
|
38
|
+
"json" => :json,
|
|
39
|
+
"jsonb" => :jsonb,
|
|
40
|
+
}.freeze
|
|
41
|
+
|
|
42
|
+
# An ARRAY column reports data_type 'ARRAY' and carries the element type
|
|
43
|
+
# in udt_name, prefixed with an underscore (`_int4`). The element type is
|
|
44
|
+
# mapped so the column is still described accurately, even though
|
|
45
|
+
# DefaultMask emits no mask for an array either way.
|
|
46
|
+
ARRAY_DATA_TYPE = "ARRAY"
|
|
47
|
+
ELEMENT_TYPE_MAP = {
|
|
48
|
+
"bpchar" => :string,
|
|
49
|
+
"varchar" => :string,
|
|
50
|
+
"text" => :text,
|
|
51
|
+
"int2" => :integer,
|
|
52
|
+
"int4" => :integer,
|
|
53
|
+
"int8" => :integer,
|
|
54
|
+
"numeric" => :decimal,
|
|
55
|
+
"float4" => :float,
|
|
56
|
+
"float8" => :float,
|
|
57
|
+
"bool" => :boolean,
|
|
58
|
+
"date" => :date,
|
|
59
|
+
"timestamp" => :datetime,
|
|
60
|
+
"timestamptz" => :datetime,
|
|
61
|
+
"time" => :time,
|
|
62
|
+
"timetz" => :time,
|
|
63
|
+
"json" => :json,
|
|
64
|
+
"jsonb" => :jsonb,
|
|
65
|
+
}.freeze
|
|
66
|
+
|
|
67
|
+
# PostgreSQL renders a stored default back as the SQL text that produced
|
|
68
|
+
# it, so a plain literal arrives with its cast attached
|
|
69
|
+
# (`'member'::user_role`, `0`, `true`) and a computed one as the call that
|
|
70
|
+
# computes it (`now()`, `nextval('...')`). Only the literal forms are
|
|
71
|
+
# recognized, and the cast is stripped: matching what a mask may be built
|
|
72
|
+
# from, rather than trying to exclude every expression, keeps an
|
|
73
|
+
# unfamiliar expression on the safe side of the line.
|
|
74
|
+
QUOTED_LITERAL = /\A'((?:[^']|'')*)'(?:::[^']+)?\z/
|
|
75
|
+
NUMERIC_LITERAL = /\A-?\d+(?:\.\d+)?\z/
|
|
76
|
+
BOOLEAN_LITERAL = /\A(?:true|false)\z/i
|
|
77
|
+
|
|
78
|
+
def table_names
|
|
79
|
+
rows(<<~SQL).map { |row| row[0] }.sort
|
|
80
|
+
SELECT table_name
|
|
81
|
+
FROM information_schema.tables
|
|
82
|
+
WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'
|
|
83
|
+
SQL
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def primary_key(table_name)
|
|
87
|
+
names = rows(<<~SQL, [table_name]).map { |row| row[0] }
|
|
88
|
+
SELECT kcu.column_name
|
|
89
|
+
FROM information_schema.table_constraints tc
|
|
90
|
+
JOIN information_schema.key_column_usage kcu
|
|
91
|
+
ON kcu.constraint_name = tc.constraint_name
|
|
92
|
+
AND kcu.constraint_schema = tc.constraint_schema
|
|
93
|
+
AND kcu.table_name = tc.table_name
|
|
94
|
+
WHERE tc.constraint_type = 'PRIMARY KEY'
|
|
95
|
+
AND tc.table_schema = current_schema()
|
|
96
|
+
AND tc.table_name = $1
|
|
97
|
+
ORDER BY kcu.ordinal_position
|
|
98
|
+
SQL
|
|
99
|
+
|
|
100
|
+
case names.size
|
|
101
|
+
when 0 then nil
|
|
102
|
+
when 1 then names.first
|
|
103
|
+
else names
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def columns(table_name)
|
|
108
|
+
sql = <<~SQL
|
|
109
|
+
SELECT column_name, data_type, udt_name, character_maximum_length, column_default
|
|
110
|
+
FROM information_schema.columns
|
|
111
|
+
WHERE table_schema = current_schema() AND table_name = $1
|
|
112
|
+
ORDER BY ordinal_position
|
|
113
|
+
SQL
|
|
114
|
+
|
|
115
|
+
rows(sql, [table_name]).map do |name, data_type, udt_name, character_maximum_length, column_default|
|
|
116
|
+
array = data_type == ARRAY_DATA_TYPE
|
|
117
|
+
type = array ? ELEMENT_TYPE_MAP[udt_name.to_s.delete_prefix("_")] : TYPE_MAP[data_type]
|
|
118
|
+
Column.new(
|
|
119
|
+
name: name,
|
|
120
|
+
type: type,
|
|
121
|
+
limit: character_maximum_length&.to_i,
|
|
122
|
+
array: array,
|
|
123
|
+
default: coerce_default(type, literal_default(column_default)),
|
|
124
|
+
)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def unique_column_names(table_name)
|
|
129
|
+
# `attnum = ANY(indkey)` keeps an expression index out of the result on
|
|
130
|
+
# its own: its entries are recorded as attnum 0, which no real column
|
|
131
|
+
# has, so the index simply contributes nothing.
|
|
132
|
+
rows(<<~SQL, [table_name]).map { |row| row[0] }.to_set
|
|
133
|
+
SELECT att.attname
|
|
134
|
+
FROM pg_index i
|
|
135
|
+
JOIN pg_class rel ON rel.oid = i.indrelid
|
|
136
|
+
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
|
137
|
+
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ANY(i.indkey)
|
|
138
|
+
WHERE i.indisunique
|
|
139
|
+
AND nsp.nspname = current_schema()
|
|
140
|
+
AND rel.relname = $1
|
|
141
|
+
SQL
|
|
142
|
+
rescue StandardError => e
|
|
143
|
+
warn_once(
|
|
144
|
+
:unique_column_names,
|
|
145
|
+
"exwiw: could not read the indexes of '#{table_name}' (#{e.class}); " \
|
|
146
|
+
"treating every column as unique-indexed so no constant mask is emitted.",
|
|
147
|
+
)
|
|
148
|
+
nil
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def foreign_keys(table_name)
|
|
152
|
+
# `conkey` lists the constrained columns in key order; unnesting it WITH
|
|
153
|
+
# ORDINALITY yields the one-row-per-key-column shape build_foreign_keys
|
|
154
|
+
# groups, so a composite constraint stays recognizable as one.
|
|
155
|
+
build_foreign_keys(table_name, rows(<<~SQL, [table_name]))
|
|
156
|
+
SELECT con.conname, att.attname, ref.relname
|
|
157
|
+
FROM pg_constraint con
|
|
158
|
+
JOIN pg_class rel ON rel.oid = con.conrelid
|
|
159
|
+
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
|
160
|
+
JOIN pg_class ref ON ref.oid = con.confrelid
|
|
161
|
+
JOIN unnest(con.conkey) WITH ORDINALITY AS u(attnum, ord) ON TRUE
|
|
162
|
+
JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = u.attnum
|
|
163
|
+
WHERE con.contype = 'f'
|
|
164
|
+
AND nsp.nspname = current_schema()
|
|
165
|
+
AND rel.relname = $1
|
|
166
|
+
ORDER BY con.conname, u.ord
|
|
167
|
+
SQL
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# The catalog's `column_default` as a plain literal, or nil when it is
|
|
171
|
+
# absent or is an expression (see QUOTED_LITERAL and friends).
|
|
172
|
+
private def literal_default(column_default)
|
|
173
|
+
return nil if column_default.nil?
|
|
174
|
+
|
|
175
|
+
if (match = QUOTED_LITERAL.match(column_default))
|
|
176
|
+
# Inside a SQL string literal a quote is doubled; undouble it so the
|
|
177
|
+
# mask is the value the column actually defaults to.
|
|
178
|
+
return match[1].gsub("''", "'")
|
|
179
|
+
end
|
|
180
|
+
return column_default if column_default.match?(NUMERIC_LITERAL)
|
|
181
|
+
return column_default.downcase if column_default.match?(BOOLEAN_LITERAL)
|
|
182
|
+
|
|
183
|
+
nil
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
private def rows(sql, params = nil)
|
|
187
|
+
params.nil? ? connection.exec(sql).values : connection.exec_params(sql, params).values
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
private def connection
|
|
191
|
+
@connection ||= begin
|
|
192
|
+
require_driver!
|
|
193
|
+
PG.connect(
|
|
194
|
+
host: @connection_config.host,
|
|
195
|
+
port: @connection_config.port,
|
|
196
|
+
user: @connection_config.user,
|
|
197
|
+
password: @connection_config.password,
|
|
198
|
+
dbname: @connection_config.database_name,
|
|
199
|
+
)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# Soft-require the driver, like MysqlClient does, so a host that only ever
|
|
204
|
+
# runs the MySQL path is not forced to build the pg gem — and so the
|
|
205
|
+
# failure names the gem to install instead of surfacing a bare LoadError.
|
|
206
|
+
private def require_driver!
|
|
207
|
+
require "pg"
|
|
208
|
+
rescue LoadError
|
|
209
|
+
raise LoadError,
|
|
210
|
+
"exwiw needs the 'pg' gem to connect to PostgreSQL. " \
|
|
211
|
+
"Add `gem \"pg\"` to your Gemfile."
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|