mutineer 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +680 -0
- data/README.md +210 -499
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_coverage.rb +53 -0
- data/lib/mutineer/cli.rb +293 -42
- data/lib/mutineer/concern_blocks.rb +243 -0
- data/lib/mutineer/config.rb +154 -8
- data/lib/mutineer/coverage_map.rb +265 -27
- data/lib/mutineer/daemon_backend.rb +244 -75
- data/lib/mutineer/daemon_client.rb +73 -10
- data/lib/mutineer/daemon_server.rb +226 -42
- data/lib/mutineer/external_backend.rb +159 -15
- data/lib/mutineer/init.rb +124 -0
- data/lib/mutineer/isolation.rb +78 -24
- data/lib/mutineer/job_plan.rb +234 -44
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +707 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutator_registry.rb +6 -2
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +36 -7
- data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +84 -0
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +302 -14
- data/lib/mutineer/rails_run_env.rb +73 -0
- data/lib/mutineer/rails_worker_db.rb +1596 -57
- data/lib/mutineer/redaction.rb +199 -0
- data/lib/mutineer/reporter.rb +173 -31
- data/lib/mutineer/result.rb +52 -15
- data/lib/mutineer/runner.rb +148 -28
- data/lib/mutineer/statement_lines.rb +24 -11
- data/lib/mutineer/subject.rb +30 -4
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners/rspec.rb +15 -3
- data/lib/mutineer/test_runners.rb +5 -2
- data/lib/mutineer/triage.rb +111 -0
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/lib/mutineer.rb +4 -0
- data/skills/mutineer/SKILL.md +84 -0
- metadata +13 -1
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
|
+
require "uri"
|
|
5
|
+
require_relative "redaction"
|
|
4
6
|
|
|
5
7
|
module Mutineer
|
|
6
8
|
# Per-worker database isolation for the daemon path.
|
|
@@ -15,19 +17,87 @@ module Mutineer
|
|
|
15
17
|
# Isolation model: each parallel worker gets its OWN database so concurrent
|
|
16
18
|
# forks cannot clobber each other's transactional fixtures. {after_fork} runs
|
|
17
19
|
# inside a freshly-forked child and points that child's connection at the
|
|
18
|
-
# worker's database BEFORE any test loads
|
|
19
|
-
# repopulate that isolated database per test. On a slot's first use the worker
|
|
20
|
-
# database starts as a copy of the base test database, so it holds what the
|
|
21
|
-
# booted parent wrote, as the in-process backend sees it.
|
|
20
|
+
# worker's database BEFORE any test loads.
|
|
22
21
|
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
22
|
+
# SQLite copies the base file on a slot's first use ({after_fork} with
|
|
23
|
+
# `seed: true`). Postgres and MySQL copy once, before any worker daemon starts
|
|
24
|
+
# ({provision}). A later fork only connects. Both servers roll back a killed
|
|
25
|
+
# fork's open transaction, so the SQLite re-seed after `error` or `timeout` is
|
|
26
|
+
# skipped. Rows a test commits outside a transaction stay in the slot.
|
|
26
27
|
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
28
|
+
# MySQL has no template database. Each slot is created empty, the app schema
|
|
29
|
+
# is loaded with ActiveRecord::Tasks::DatabaseTasks, and rows are copied
|
|
30
|
+
# table by table. `mysql2` and `trilogy` are accepted. Any other adapter
|
|
31
|
+
# raises NotImplementedError rather than routing at a database that does not exist.
|
|
30
32
|
module RailsWorkerDb
|
|
33
|
+
# Postgres cuts an unquoted identifier at this many bytes.
|
|
34
|
+
POSTGRES_NAME_LIMIT = 63
|
|
35
|
+
# MySQL rejects a database name longer than this many bytes. The shortening
|
|
36
|
+
# rule matches Postgres. A byte limit stays inside the 64-character limit.
|
|
37
|
+
MYSQL_NAME_LIMIT = 64
|
|
38
|
+
# How many worker daemons can hold a lock at once. MySQL GET_LOCK is
|
|
39
|
+
# exclusive, so each daemon takes one name from this pool. A separate guard
|
|
40
|
+
# name covers the handoff and is not one of these slots.
|
|
41
|
+
MYSQL_LOCK_SLOTS = 256
|
|
42
|
+
# Actions information_schema may report for a foreign key. Anything else is
|
|
43
|
+
# rejected so a bad row cannot change the ALTER TABLE statement.
|
|
44
|
+
MYSQL_REFERENTIAL_ACTIONS = ["CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"].freeze
|
|
45
|
+
# TLS and timeout keys from the app's MySQL config that reach the
|
|
46
|
+
# maintenance client unchanged. ActiveRecord passes them to the client the
|
|
47
|
+
# same way. The first group is mysql2's, the second trilogy's. Each client
|
|
48
|
+
# ignores the other's keys.
|
|
49
|
+
MYSQL_CLIENT_KEYS = %i[
|
|
50
|
+
connect_timeout read_timeout write_timeout
|
|
51
|
+
ssl_mode sslkey sslcert sslca sslcapath sslcipher sslverify get_server_public_key
|
|
52
|
+
ssl ssl_ca ssl_capath ssl_cert ssl_key ssl_cipher ssl_crl ssl_crlpath
|
|
53
|
+
tls_ciphersuites tls_min_version tls_max_version
|
|
54
|
+
].freeze
|
|
55
|
+
# Infix that keeps a worker database off Rails' own `<base>-<n>` names.
|
|
56
|
+
WORKER_NAME_MARK = "-mutineer-"
|
|
57
|
+
# Database we connect to while the test database is a template. Advisory
|
|
58
|
+
# locks are per database, so every run must take them here, not on a slot.
|
|
59
|
+
MAINTENANCE_DATABASE = "postgres"
|
|
60
|
+
# Server catalogs a worker copy must never drop, even if a name bug
|
|
61
|
+
# produced one of them.
|
|
62
|
+
RESERVED_DATABASES = %w[postgres template0 template1].freeze
|
|
63
|
+
# Rails keys copied onto the maintenance `PG::Connection`. `database` is
|
|
64
|
+
# not here: that session always uses {MAINTENANCE_DATABASE}. `username` is
|
|
65
|
+
# Rails' spelling of libpq `user`. Anything else (adapter, variables,
|
|
66
|
+
# schema_search_path) is a Rails setting and would make libpq reject the
|
|
67
|
+
# connection.
|
|
68
|
+
LIBPQ_CONNECTION_KEYS = {
|
|
69
|
+
host: :host,
|
|
70
|
+
hostaddr: :hostaddr,
|
|
71
|
+
port: :port,
|
|
72
|
+
user: :user,
|
|
73
|
+
username: :user,
|
|
74
|
+
password: :password,
|
|
75
|
+
service: :service,
|
|
76
|
+
sslmode: :sslmode,
|
|
77
|
+
sslcert: :sslcert,
|
|
78
|
+
sslkey: :sslkey,
|
|
79
|
+
sslrootcert: :sslrootcert,
|
|
80
|
+
sslcrl: :sslcrl,
|
|
81
|
+
sslcompression: :sslcompression,
|
|
82
|
+
ssl_min_protocol_version: :ssl_min_protocol_version,
|
|
83
|
+
ssl_max_protocol_version: :ssl_max_protocol_version,
|
|
84
|
+
connect_timeout: :connect_timeout,
|
|
85
|
+
client_encoding: :client_encoding,
|
|
86
|
+
options: :options,
|
|
87
|
+
application_name: :application_name,
|
|
88
|
+
fallback_application_name: :fallback_application_name,
|
|
89
|
+
keepalives: :keepalives,
|
|
90
|
+
keepalives_idle: :keepalives_idle,
|
|
91
|
+
keepalives_interval: :keepalives_interval,
|
|
92
|
+
keepalives_count: :keepalives_count,
|
|
93
|
+
channel_binding: :channel_binding,
|
|
94
|
+
gssencmode: :gssencmode,
|
|
95
|
+
krbsrvname: :krbsrvname,
|
|
96
|
+
gsslib: :gsslib,
|
|
97
|
+
requiressl: :requiressl,
|
|
98
|
+
target_session_attrs: :target_session_attrs
|
|
99
|
+
}.freeze
|
|
100
|
+
|
|
31
101
|
# True when the app has ActiveRecord loaded. The only condition under which
|
|
32
102
|
# any other method here may touch AR. Never triggers an autoload/require of
|
|
33
103
|
# AR itself.
|
|
@@ -37,10 +107,9 @@ module Mutineer
|
|
|
37
107
|
defined?(ActiveRecord::Base) ? true : false
|
|
38
108
|
end
|
|
39
109
|
|
|
40
|
-
# Derive a per-worker
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
# `storage/test-1.sqlite3`.
|
|
110
|
+
# Derive a per-worker SQLite path by inserting `-<worker>` before the
|
|
111
|
+
# extension. Pure string transform (no AR). `storage/test.sqlite3`, worker
|
|
112
|
+
# 1 -> `storage/test-1.sqlite3`.
|
|
44
113
|
#
|
|
45
114
|
# @param database [String] the base database path.
|
|
46
115
|
# @param worker [Integer] the worker slot index (0..N-1).
|
|
@@ -50,73 +119,294 @@ module Mutineer
|
|
|
50
119
|
"#{database.delete_suffix(ext)}-#{worker}#{ext}"
|
|
51
120
|
end
|
|
52
121
|
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
122
|
+
# Worker database name: `<base>-mutineer-<slot>`. When that is longer than
|
|
123
|
+
# `limit` bytes, the base is shortened on a character boundary and a hash
|
|
124
|
+
# of the full base is added so two slots, and two long bases that share a
|
|
125
|
+
# prefix, stay distinct.
|
|
126
|
+
#
|
|
127
|
+
# @param base [String] the test database name.
|
|
128
|
+
# @param slot [Integer] the worker slot index.
|
|
129
|
+
# @param limit [Integer] maximum name length in bytes.
|
|
130
|
+
# @return [String] a name of at most `limit` bytes, or the hash suffix
|
|
131
|
+
# alone when that suffix itself exceeds the limit.
|
|
132
|
+
def self.worker_database_name(base, slot, limit)
|
|
133
|
+
suffix = "#{WORKER_NAME_MARK}#{slot}"
|
|
134
|
+
full = "#{base}#{suffix}"
|
|
135
|
+
return full if full.bytesize <= limit
|
|
136
|
+
|
|
137
|
+
hash = Digest::SHA256.hexdigest(base)[0, 8]
|
|
138
|
+
tail = "-#{hash}#{suffix}"
|
|
139
|
+
room = limit - tail.bytesize
|
|
140
|
+
"#{utf8_prefix(base, room)}#{tail}"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Postgres worker database name. See {worker_database_name}.
|
|
144
|
+
#
|
|
145
|
+
# @param base [String] the test database name.
|
|
146
|
+
# @param slot [Integer] the worker slot index.
|
|
147
|
+
# @return [String] a name of at most {POSTGRES_NAME_LIMIT} bytes.
|
|
148
|
+
def self.postgres_worker_database(base, slot)
|
|
149
|
+
worker_database_name(base, slot, POSTGRES_NAME_LIMIT)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# MySQL worker database name. See {worker_database_name}.
|
|
153
|
+
#
|
|
154
|
+
# @param base [String] the test database name.
|
|
155
|
+
# @param slot [Integer] the worker slot index.
|
|
156
|
+
# @return [String] a name of at most {MYSQL_NAME_LIMIT} bytes.
|
|
157
|
+
def self.mysql_worker_database(base, slot)
|
|
158
|
+
worker_database_name(base, slot, MYSQL_NAME_LIMIT)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Leading characters of `text` that fit in `max_bytes`. Stops before a
|
|
162
|
+
# character that would cross the limit, so a multibyte name stays valid
|
|
163
|
+
# text. PostgreSQL and MySQL count identifier length in bytes.
|
|
164
|
+
#
|
|
165
|
+
# @param text [String]
|
|
166
|
+
# @param max_bytes [Integer]
|
|
167
|
+
# @return [String]
|
|
168
|
+
def self.utf8_prefix(text, max_bytes)
|
|
169
|
+
return "" if max_bytes <= 0
|
|
170
|
+
|
|
171
|
+
source = text.to_s
|
|
172
|
+
return source if source.bytesize <= max_bytes
|
|
173
|
+
|
|
174
|
+
kept = +""
|
|
175
|
+
source.each_char do |char|
|
|
176
|
+
break if kept.bytesize + char.bytesize > max_bytes
|
|
177
|
+
|
|
178
|
+
kept << char
|
|
179
|
+
end
|
|
180
|
+
kept
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# True when `name` is one of this run's exact worker database names.
|
|
184
|
+
# Equality only: a SQL `LIKE` would treat `_` in the base as a wildcard,
|
|
185
|
+
# so `my_app_test-mutineer-1` must not match base `myXappXtest`.
|
|
186
|
+
# `limit` defaults to the Postgres limit so existing callers stay valid.
|
|
187
|
+
#
|
|
188
|
+
# @param base [String] the test database name.
|
|
189
|
+
# @param name [String] a database name we might drop.
|
|
190
|
+
# @param slots [Integer] how many worker slots this run owns.
|
|
191
|
+
# @param limit [Integer] name limit in bytes. {POSTGRES_NAME_LIMIT} or {MYSQL_NAME_LIMIT}.
|
|
192
|
+
# @return [Boolean]
|
|
193
|
+
def self.owned_database?(base, name, slots:, limit: POSTGRES_NAME_LIMIT)
|
|
194
|
+
(0...slots).any? { |slot| worker_database_name(base, slot, limit) == name }
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# True when `name` is a database this tool may drop. It must be the exact
|
|
198
|
+
# name {postgres_worker_database} would build for `base` and the slot
|
|
199
|
+
# written at the end of `name`. The source database, a server catalog, and
|
|
200
|
+
# any other name are refused.
|
|
201
|
+
#
|
|
202
|
+
# @param base [String] the test database name.
|
|
203
|
+
# @param name [String] a database name we might drop.
|
|
204
|
+
# @return [Boolean]
|
|
205
|
+
def self.worker_database_name?(base, name)
|
|
206
|
+
return false if base.nil? || name.nil?
|
|
207
|
+
|
|
208
|
+
text = name.to_s
|
|
209
|
+
source = base.to_s
|
|
210
|
+
return false if text.empty? || text == source
|
|
211
|
+
return false if RESERVED_DATABASES.include?(text)
|
|
212
|
+
|
|
213
|
+
slot = text[/#{Regexp.escape(WORKER_NAME_MARK)}(\d+)\z/, 1]
|
|
214
|
+
return false unless slot
|
|
215
|
+
|
|
216
|
+
postgres_worker_database(source, Integer(slot, 10)) == text
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Build the AR connection config for one worker from the app's current
|
|
220
|
+
# test config. SQLite, Postgres, and MySQL (mysql2 or trilogy) get a
|
|
221
|
+
# per-worker database name. Anything else raises.
|
|
57
222
|
#
|
|
58
223
|
# @param worker [Integer] the worker slot index.
|
|
59
224
|
# @return [Hash] a symbol-keyed AR configuration hash for the worker database.
|
|
60
|
-
# @raise [NotImplementedError] when the
|
|
225
|
+
# @raise [NotImplementedError] when the adapter has no worker database yet, or the database is in-memory.
|
|
61
226
|
def self.worker_db_config(worker)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
# Config shaping (per_worker_config) is already adapter-general: it derives
|
|
65
|
-
# correct SQLite and Postgres worker-DB names. What is gated is runtime
|
|
66
|
-
# provisioning: SQLite files are created on connect, but Postgres needs an
|
|
67
|
-
# explicit `CREATE DATABASE` per worker. Until that lands, refuse non-SQLite
|
|
68
|
-
# loudly rather than route to a database that does not exist.
|
|
69
|
-
unless adapter.start_with?("sqlite")
|
|
70
|
-
raise NotImplementedError,
|
|
71
|
-
"worker-DB isolation currently provisions SQLite only (got adapter #{adapter.inspect}); " \
|
|
72
|
-
"Postgres per-worker provisioning is not yet supported. Use a SQLite test DB with --daemon, or drop --daemon to run serially."
|
|
73
|
-
end
|
|
227
|
+
per_worker_config(routing_config_hash, worker)
|
|
228
|
+
end
|
|
74
229
|
|
|
75
|
-
|
|
230
|
+
# Config used to name a worker database. MySQL schema load points
|
|
231
|
+
# ActiveRecord at the last slot. Coverage forks must keep the original test
|
|
232
|
+
# database, or the next name is nested and was never created.
|
|
233
|
+
#
|
|
234
|
+
# @return [Hash]
|
|
235
|
+
def self.routing_config_hash
|
|
236
|
+
@mysql_base_config || current_config_hash
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Remember the test database config before schema load switches the connection.
|
|
240
|
+
#
|
|
241
|
+
# @param config_hash [Hash]
|
|
242
|
+
# @return [void]
|
|
243
|
+
def self.remember_mysql_base_config(config_hash)
|
|
244
|
+
@mysql_base_config = config_hash.transform_keys(&:to_sym)
|
|
245
|
+
nil
|
|
76
246
|
end
|
|
77
247
|
|
|
78
|
-
#
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
248
|
+
# Drop the remembered MySQL test config. The next route reads ActiveRecord again.
|
|
249
|
+
#
|
|
250
|
+
# @return [void]
|
|
251
|
+
def self.forget_mysql_base_config
|
|
252
|
+
@mysql_base_config = nil
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Pure config shaping (no AR): swap in the worker database name.
|
|
256
|
+
# SQLite keeps {worker_database_path}. Postgres uses {postgres_worker_database}.
|
|
257
|
+
# MySQL (`mysql2` or `trilogy`) uses {mysql_worker_database}.
|
|
84
258
|
#
|
|
85
259
|
# @param config_hash [Hash] a connection config hash (symbol or string keys).
|
|
86
260
|
# @param worker [Integer] the worker slot index.
|
|
87
261
|
# @return [Hash] the per-worker config (symbol keys), database swapped.
|
|
88
|
-
# @raise [NotImplementedError] for an in-memory
|
|
262
|
+
# @raise [NotImplementedError] for an in-memory database, an empty name, or an adapter other than SQLite, Postgres, or MySQL.
|
|
89
263
|
def self.per_worker_config(config_hash, worker)
|
|
90
264
|
hash = config_hash.transform_keys(&:to_sym)
|
|
265
|
+
adapter = hash[:adapter].to_s
|
|
91
266
|
database = hash[:database].to_s
|
|
267
|
+
if database.empty? && postgres_adapter?(adapter)
|
|
268
|
+
database = database_name_from_url(hash[:url])
|
|
269
|
+
end
|
|
92
270
|
if database.empty? || database == ":memory:"
|
|
93
271
|
raise NotImplementedError,
|
|
94
|
-
"worker-DB isolation needs a file
|
|
272
|
+
"worker-DB isolation needs a file or named database (got #{database.inspect})."
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
name =
|
|
276
|
+
if sqlite_adapter?(adapter)
|
|
277
|
+
worker_database_path(database, worker)
|
|
278
|
+
elsif postgres_adapter?(adapter)
|
|
279
|
+
postgres_worker_database(database, worker)
|
|
280
|
+
elsif mysql_adapter?(adapter)
|
|
281
|
+
mysql_worker_database(database, worker)
|
|
282
|
+
else
|
|
283
|
+
raise NotImplementedError,
|
|
284
|
+
"worker-DB isolation does not support adapter #{adapter.inspect} yet."
|
|
285
|
+
end
|
|
286
|
+
hash.merge(database: name)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Create each worker database before any worker daemon starts.
|
|
290
|
+
# SQLite is a no-op: the file is copied on first use. Postgres drops and
|
|
291
|
+
# recreates each exact slot name from the test database. MySQL creates each
|
|
292
|
+
# slot, loads the schema, and copies rows. The caller must be the only
|
|
293
|
+
# daemon connected to that test database.
|
|
294
|
+
#
|
|
295
|
+
# @param slots [Integer] how many worker slots to create (0..slots-1).
|
|
296
|
+
# @return [void]
|
|
297
|
+
# @raise [NotImplementedError] for an adapter other than SQLite, Postgres, or MySQL.
|
|
298
|
+
# @raise [RuntimeError] when the copy cannot be made, or another run holds the lock.
|
|
299
|
+
def self.provision(slots)
|
|
300
|
+
return unless available?
|
|
301
|
+
|
|
302
|
+
count = Integer(slots)
|
|
303
|
+
return if count < 1
|
|
304
|
+
return if sqlite_adapter?(current_adapter)
|
|
305
|
+
|
|
306
|
+
if postgres_adapter?(current_adapter)
|
|
307
|
+
provision_postgres(count)
|
|
308
|
+
return
|
|
309
|
+
end
|
|
310
|
+
if mysql_adapter?(current_adapter)
|
|
311
|
+
provision_mysql(count)
|
|
312
|
+
return
|
|
95
313
|
end
|
|
96
314
|
|
|
97
|
-
|
|
315
|
+
raise NotImplementedError,
|
|
316
|
+
"worker-DB isolation does not support adapter #{current_adapter.inspect} yet."
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# Take the shared run lock and drop this process's connection to the test
|
|
320
|
+
# database. Worker daemons call this at boot. They do not create databases.
|
|
321
|
+
# SQLite has no lock here. The file copy still happens in {after_fork}.
|
|
322
|
+
#
|
|
323
|
+
# @return [void]
|
|
324
|
+
# @raise [RuntimeError] when another run holds the exclusive lock.
|
|
325
|
+
def self.prepare_worker!
|
|
326
|
+
return unless available?
|
|
327
|
+
return if sqlite_adapter?(current_adapter)
|
|
328
|
+
|
|
329
|
+
if mysql_adapter?(current_adapter)
|
|
330
|
+
prepare_mysql_worker!
|
|
331
|
+
return
|
|
332
|
+
end
|
|
333
|
+
return unless postgres_adapter?(current_adapter)
|
|
334
|
+
|
|
335
|
+
hash = current_config_hash
|
|
336
|
+
base = source_database_name!(hash)
|
|
337
|
+
conn = open_maintenance_connection(hash, base)
|
|
338
|
+
unless try_lock(conn, base, shared: true)
|
|
339
|
+
conn.close
|
|
340
|
+
raise "another Mutineer run is using #{base}"
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
@lock_connection = conn
|
|
344
|
+
disconnect_app!
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Point this process at one worker database. Postgres and MySQL only connect.
|
|
348
|
+
# The slot was copied at {provision}.
|
|
349
|
+
#
|
|
350
|
+
# @param slot [Integer] the worker slot index.
|
|
351
|
+
# @return [void]
|
|
352
|
+
def self.connect(slot)
|
|
353
|
+
ActiveRecord::Base.establish_connection(worker_db_config(slot))
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
# The database config names the test environment declares, and the one
|
|
357
|
+
# each worker copies (the config `ActiveRecord::Base` uses). The tool
|
|
358
|
+
# counts the names that `shared_databases` does not list: more than one
|
|
359
|
+
# means the run must use a single worker.
|
|
360
|
+
#
|
|
361
|
+
# A config with `database_tasks: false` counts: workers would share it like
|
|
362
|
+
# any other. A replica does not, as before: it reads another config's database.
|
|
363
|
+
# No rescue: a failure must reach the tool, which then keeps one worker.
|
|
364
|
+
#
|
|
365
|
+
# @return [Hash{String => Object}] "names" (Array<String>) and "primary" (String).
|
|
366
|
+
def self.database_names
|
|
367
|
+
names = test_configs.reject(&:replica?).map(&:name)
|
|
368
|
+
{ "names" => names, "primary" => primary_database_name }
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# The config name each worker copies: the one `ActiveRecord::Base` uses.
|
|
372
|
+
#
|
|
373
|
+
# @return [String]
|
|
374
|
+
def self.primary_database_name
|
|
375
|
+
ActiveRecord::Base.connection_db_config.name
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# The test environment's database configs, hidden ones and replicas included.
|
|
379
|
+
#
|
|
380
|
+
# @return [Array<ActiveRecord::DatabaseConfigurations::DatabaseConfig>]
|
|
381
|
+
def self.test_configs
|
|
382
|
+
env = ENV["RAILS_ENV"].to_s
|
|
383
|
+
env = "test" if env.empty?
|
|
384
|
+
ActiveRecord::Base.configurations.configs_for(env_name: env, include_hidden: true)
|
|
98
385
|
end
|
|
99
386
|
|
|
100
387
|
# Child-side (after fork): route this process's ActiveRecord at the worker's
|
|
101
388
|
# own database and confirm it is reachable, so a routing failure reads as
|
|
102
|
-
# `error`
|
|
389
|
+
# `error` rather than a false verdict.
|
|
103
390
|
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
106
|
-
#
|
|
107
|
-
#
|
|
108
|
-
# only when the copy differs from `schema.rb` ({schema_current?}: schema
|
|
109
|
-
# version or stored `schema_sha1`, as in a stale or empty base database):
|
|
110
|
-
# `schema.rb` runs with `force: true`, which drops the copied rows of the
|
|
111
|
-
# tables it defines.
|
|
391
|
+
# SQLite with `seed: true` copies the base file first, then loads `schema.rb`
|
|
392
|
+
# only when the copy differs from that file. Postgres and MySQL ignore `seed`
|
|
393
|
+
# and the schema path: the slot was copied at {provision}, and loading
|
|
394
|
+
# `schema.rb` again would drop those rows.
|
|
112
395
|
#
|
|
113
396
|
# @param worker [Integer] the worker slot index.
|
|
114
|
-
# @param schema_path [String, nil]
|
|
115
|
-
# @param seed [Boolean] copy the base
|
|
397
|
+
# @param schema_path [String, nil] the app's Ruby schema file ({ruby_schema_path}), or nil to skip.
|
|
398
|
+
# @param seed [Boolean] copy the base SQLite file into the worker file first.
|
|
116
399
|
# @return [void]
|
|
117
400
|
def self.after_fork(worker, schema_path = nil, seed: false)
|
|
118
401
|
return unless available?
|
|
119
402
|
|
|
403
|
+
forget_lock_connection_after_fork!
|
|
404
|
+
if postgres_adapter?(current_adapter) || mysql_adapter?(current_adapter)
|
|
405
|
+
connect(worker)
|
|
406
|
+
verify_connection!
|
|
407
|
+
return
|
|
408
|
+
end
|
|
409
|
+
|
|
120
410
|
config = worker_db_config(worker)
|
|
121
411
|
base = ActiveRecord::Base.connection.select_value("SELECT file FROM pragma_database_list WHERE name = 'main'") if seed
|
|
122
412
|
ActiveRecord::Base.establish_connection(config)
|
|
@@ -159,10 +449,10 @@ module Mutineer
|
|
|
159
449
|
# file, which catches an edited schema with the same version. No stored
|
|
160
450
|
# checksum counts as out of date, as in Rails' own `schema_up_to_date?`.
|
|
161
451
|
#
|
|
162
|
-
# @param schema_path [String] absolute path to
|
|
452
|
+
# @param schema_path [String] absolute path to the app's Ruby schema file.
|
|
163
453
|
# @return [Boolean]
|
|
164
454
|
def self.schema_current?(schema_path)
|
|
165
|
-
text = File.read(schema_path)
|
|
455
|
+
text = File.read(schema_path, encoding: "UTF-8")
|
|
166
456
|
version = schema_file_version(text)
|
|
167
457
|
conn = ActiveRecord::Base.connection
|
|
168
458
|
base = ActiveRecord::Base
|
|
@@ -185,10 +475,91 @@ module Mutineer
|
|
|
185
475
|
text[/define\(version:\s*([\d_]+)/, 1]&.delete("_")&.to_i
|
|
186
476
|
end
|
|
187
477
|
|
|
478
|
+
# Record each pool that runs SQL in this fork from now on. Run in the fork
|
|
479
|
+
# before its tests, so {touched_shared_databases} also sees a pool that a
|
|
480
|
+
# test disconnects or removes before it ends.
|
|
481
|
+
#
|
|
482
|
+
# @return [void]
|
|
483
|
+
def self.watch_sql
|
|
484
|
+
@sql_pools = {} # used as a set: one entry per pool, not per query
|
|
485
|
+
ActiveSupport::Notifications.subscribe("sql.active_record") do |*, payload|
|
|
486
|
+
connection = payload[:connection]
|
|
487
|
+
@sql_pools[connection.pool] = true if connection.respond_to?(:pool)
|
|
488
|
+
end
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
# Names from `shared` whose database this fork used. Run in a fork after
|
|
492
|
+
# its tests: a name here means a test used a database that parallel
|
|
493
|
+
# workers share, so the verdict cannot be trusted. A pool counts when it
|
|
494
|
+
# is still connected, or when it ran SQL since {watch_sql}. It names a
|
|
495
|
+
# shared database when its config has a shared name, when its class also
|
|
496
|
+
# holds a shared writer pool (a replica in the same `connects_to`), or when
|
|
497
|
+
# its config names the same database as a shared config. Transactional
|
|
498
|
+
# tests connect every registered writing pool, so a pool registered for a
|
|
499
|
+
# shared database shows up here after any test.
|
|
500
|
+
#
|
|
501
|
+
# Only names in {database_names} count: the tool warned about any other
|
|
502
|
+
# name, and no worker shares a database that does not exist.
|
|
503
|
+
#
|
|
504
|
+
# No rescue: a failed check must raise (scored `error`), never read as clean.
|
|
505
|
+
# shortcut: a connection made outside Active Record (a raw driver) is not
|
|
506
|
+
# seen; upgrade when such an app needs `shared_databases`.
|
|
507
|
+
#
|
|
508
|
+
# @param shared [Array<String>] database config names in `shared_databases`.
|
|
509
|
+
# @return [Array<String>]
|
|
510
|
+
def self.touched_shared_databases(shared)
|
|
511
|
+
return [] if shared.empty? || !available?
|
|
512
|
+
|
|
513
|
+
shared &= database_names["names"]
|
|
514
|
+
return [] if shared.empty?
|
|
515
|
+
# Rails 7.0 reads `:all` below as a role name and lists no pool, so the
|
|
516
|
+
# check would always pass. Fail every mutant instead of trusting it.
|
|
517
|
+
raise "shared_databases needs Rails 7.1 or later" if ActiveRecord.version < Gem::Version.new("7.1")
|
|
518
|
+
|
|
519
|
+
pools = ActiveRecord::Base.connection_handler.connection_pool_list(:all)
|
|
520
|
+
owners = pools.filter_map { |p| [pool_owner(p), p.db_config.name] if shared.include?(p.db_config.name) }.to_h
|
|
521
|
+
by_database = test_configs.filter_map { |c| [c.database, c.name] if shared.include?(c.name) && c.database }.to_h
|
|
522
|
+
used = pools.select(&:connected?) | (@sql_pools&.keys || [])
|
|
523
|
+
used.filter_map do |pool|
|
|
524
|
+
config = pool.db_config
|
|
525
|
+
shared.include?(config.name) ? config.name : owners[pool_owner(pool)] || by_database[config.database]
|
|
526
|
+
end.uniq
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# The class a pool belongs to, by name. `connects_to` registers a class's
|
|
530
|
+
# writer and replica pools under it. Rails 8 keeps it in a descriptor.
|
|
531
|
+
#
|
|
532
|
+
# @param pool [ActiveRecord::ConnectionAdapters::ConnectionPool]
|
|
533
|
+
# @return [String]
|
|
534
|
+
def self.pool_owner(pool)
|
|
535
|
+
config = pool.pool_config
|
|
536
|
+
config.respond_to?(:connection_descriptor) ? config.connection_descriptor.name : config.connection_name
|
|
537
|
+
end
|
|
538
|
+
|
|
539
|
+
# The Ruby schema file a SQLite worker loads over an out-of-date copy, as
|
|
540
|
+
# the booted app's config names it. Rails resolves the name: `db/schema.rb`
|
|
541
|
+
# by default, the `schema_dump:` file of the database config (a bare name
|
|
542
|
+
# such as `schema_sqlite.rb` lives in `db/`), or the `SCHEMA` variable.
|
|
543
|
+
# nil when the file does not exist, when the config sets `schema_dump: false`,
|
|
544
|
+
# or when the app uses `structure.sql` (a worker cannot `load` SQL). With nil
|
|
545
|
+
# the worker keeps its copy of the test database as it is.
|
|
546
|
+
#
|
|
547
|
+
# @return [String, nil] absolute path of an existing file, or nil.
|
|
548
|
+
def self.ruby_schema_path
|
|
549
|
+
db_config = ActiveRecord::Base.connection_db_config
|
|
550
|
+
format = schema_format_for(db_config)
|
|
551
|
+
return unless format == :ruby
|
|
552
|
+
|
|
553
|
+
path = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)
|
|
554
|
+
# Expanded so `load` never searches $LOAD_PATH for a relative SCHEMA value.
|
|
555
|
+
path &&= File.expand_path(path)
|
|
556
|
+
path if path && File.exist?(path)
|
|
557
|
+
end
|
|
558
|
+
|
|
188
559
|
# Load a Rails `schema.rb` into the current connection with output silenced
|
|
189
560
|
# (fork child stdout is already File::NULL; this is belt-and-braces).
|
|
190
561
|
#
|
|
191
|
-
# @param schema_path [String] absolute path to
|
|
562
|
+
# @param schema_path [String] absolute path to the app's Ruby schema file.
|
|
192
563
|
# @return [void]
|
|
193
564
|
def self.load_schema(schema_path)
|
|
194
565
|
ActiveRecord::Migration.verbose = false if defined?(ActiveRecord::Migration)
|
|
@@ -201,12 +572,1180 @@ module Mutineer
|
|
|
201
572
|
end
|
|
202
573
|
|
|
203
574
|
# Force a round-trip to the freshly-routed connection so a broken route fails
|
|
204
|
-
# HERE (
|
|
205
|
-
# (
|
|
575
|
+
# HERE (then `error`) instead of later masquerading as a test failure
|
|
576
|
+
# (then a false `killed`).
|
|
206
577
|
#
|
|
207
578
|
# @return [void]
|
|
208
579
|
def self.verify_connection!
|
|
209
580
|
ActiveRecord::Base.connection.execute("SELECT 1")
|
|
210
581
|
end
|
|
582
|
+
|
|
583
|
+
# Symbol-keyed connection hash for the database the app booted. Does not
|
|
584
|
+
# open a new connection.
|
|
585
|
+
#
|
|
586
|
+
# @return [Hash]
|
|
587
|
+
def self.current_config_hash
|
|
588
|
+
ActiveRecord::Base.connection_db_config.configuration_hash.transform_keys(&:to_sym)
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
# Adapter name from {current_config_hash}.
|
|
592
|
+
#
|
|
593
|
+
# @return [String]
|
|
594
|
+
def self.current_adapter
|
|
595
|
+
current_config_hash[:adapter].to_s
|
|
596
|
+
end
|
|
597
|
+
|
|
598
|
+
# @param adapter [String]
|
|
599
|
+
# @return [Boolean]
|
|
600
|
+
def self.sqlite_adapter?(adapter)
|
|
601
|
+
adapter.start_with?("sqlite")
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
# @param adapter [String]
|
|
605
|
+
# @return [Boolean]
|
|
606
|
+
def self.postgres_adapter?(adapter)
|
|
607
|
+
adapter == "postgres" || adapter.start_with?("postgres")
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
# True for the mysql2 and trilogy adapters. The old `mysql` gem is not included.
|
|
611
|
+
#
|
|
612
|
+
# @param adapter [String]
|
|
613
|
+
# @return [Boolean]
|
|
614
|
+
def self.mysql_adapter?(adapter)
|
|
615
|
+
adapter == "mysql2" || adapter == "trilogy"
|
|
616
|
+
end
|
|
617
|
+
|
|
618
|
+
# Classify a MySQL `SELECT VERSION()` text against the support matrix.
|
|
619
|
+
# MariaDB is matched by name first: it reports 10.x or 11.x, and some
|
|
620
|
+
# builds prefix `5.5.5-`, so a numeric check alone would misread it.
|
|
621
|
+
#
|
|
622
|
+
# @param version [String, nil]
|
|
623
|
+
# @return [Symbol] `:supported` (MySQL 8), `:not_supported` (before 8),
|
|
624
|
+
# `:not_tested` (MariaDB, MySQL 9 or later), or `:unreadable`.
|
|
625
|
+
def self.mysql_version_support(version)
|
|
626
|
+
text = version.to_s
|
|
627
|
+
return :not_tested if text.match?(/mariadb/i)
|
|
628
|
+
|
|
629
|
+
major = text[/\A(\d+)\.\d+/, 1]
|
|
630
|
+
return :unreadable unless major
|
|
631
|
+
|
|
632
|
+
major = major.to_i
|
|
633
|
+
return :not_supported if major < 8
|
|
634
|
+
return :not_tested if major > 8
|
|
635
|
+
|
|
636
|
+
:supported
|
|
637
|
+
end
|
|
638
|
+
|
|
639
|
+
# Drop and recreate slots `0...count` from the booted test database.
|
|
640
|
+
# Holds the exclusive advisory lock only for the copy, then a shared lock
|
|
641
|
+
# for the rest of this process so a second run cannot drop those databases.
|
|
642
|
+
#
|
|
643
|
+
# @param count [Integer]
|
|
644
|
+
# @return [void]
|
|
645
|
+
def self.provision_postgres(count)
|
|
646
|
+
hash = current_config_hash
|
|
647
|
+
base = source_database_name!(hash)
|
|
648
|
+
conn = open_maintenance_connection(hash, base)
|
|
649
|
+
unless try_lock(conn, base, shared: false)
|
|
650
|
+
conn.close
|
|
651
|
+
raise "another Mutineer run is using #{base}"
|
|
652
|
+
end
|
|
653
|
+
|
|
654
|
+
begin
|
|
655
|
+
disconnect_app!
|
|
656
|
+
count.times do |slot|
|
|
657
|
+
name = postgres_worker_database(base, slot)
|
|
658
|
+
recreate_from_template(conn, name, base)
|
|
659
|
+
end
|
|
660
|
+
rescue StandardError => e
|
|
661
|
+
unlock(conn, base)
|
|
662
|
+
conn.close
|
|
663
|
+
raise provision_wrapped?(e) ? e : provision_failure(base, base, e.message)
|
|
664
|
+
end
|
|
665
|
+
|
|
666
|
+
unlock(conn, base)
|
|
667
|
+
unless try_lock(conn, base, shared: true)
|
|
668
|
+
conn.close
|
|
669
|
+
raise "another Mutineer run is using #{base}"
|
|
670
|
+
end
|
|
671
|
+
@lock_connection = conn
|
|
672
|
+
warn "[mutineer] provisioned #{count} postgres worker database(s) from #{base}"
|
|
673
|
+
end
|
|
674
|
+
|
|
675
|
+
# Open a session on {MAINTENANCE_DATABASE} with the app's Postgres credentials.
|
|
676
|
+
# Uses the `pg` client the app already loaded. The gem does not depend on it.
|
|
677
|
+
# Endpoint options from the Rails config, including `service` and `url`, are
|
|
678
|
+
# kept. Only the database name changes, to {MAINTENANCE_DATABASE}.
|
|
679
|
+
#
|
|
680
|
+
# @param config_hash [Hash]
|
|
681
|
+
# @param base [String] test database name, named in the error if this fails.
|
|
682
|
+
# @return [PG::Connection]
|
|
683
|
+
def self.open_maintenance_connection(config_hash, base)
|
|
684
|
+
unless defined?(PG::Connection)
|
|
685
|
+
raise "could not provision #{base}: the app's pg client is not loaded"
|
|
686
|
+
end
|
|
687
|
+
|
|
688
|
+
PG::Connection.open(maintenance_connection_options(config_hash))
|
|
689
|
+
rescue StandardError => e
|
|
690
|
+
raise e if e.message.start_with?("could not provision")
|
|
691
|
+
|
|
692
|
+
raise "could not provision #{base}: #{e.message}"
|
|
693
|
+
end
|
|
694
|
+
|
|
695
|
+
# libpq options for the maintenance session. A Rails `url:` fills anything
|
|
696
|
+
# the hash does not set itself. Explicit keys win. `dbname` is always
|
|
697
|
+
# {MAINTENANCE_DATABASE}, never the test database and never a name taken
|
|
698
|
+
# from the URL path.
|
|
699
|
+
#
|
|
700
|
+
# @param config_hash [Hash] symbol or string keys, as Rails stores them.
|
|
701
|
+
# @return [Hash] symbol keys for `PG::Connection.open`.
|
|
702
|
+
# @raise [RuntimeError] when `url:` is present and is not a Postgres URL.
|
|
703
|
+
def self.maintenance_connection_options(config_hash)
|
|
704
|
+
hash = config_hash.transform_keys(&:to_sym)
|
|
705
|
+
opts = options_from_database_url(hash[:url])
|
|
706
|
+
opts.delete(:database)
|
|
707
|
+
LIBPQ_CONNECTION_KEYS.each do |config_key, pg_key|
|
|
708
|
+
next unless hash.key?(config_key)
|
|
709
|
+
|
|
710
|
+
value = hash[config_key]
|
|
711
|
+
next if blank_option?(value)
|
|
712
|
+
|
|
713
|
+
opts[pg_key] = value
|
|
714
|
+
end
|
|
715
|
+
opts[:dbname] = MAINTENANCE_DATABASE
|
|
716
|
+
opts
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
# The test database this run copies from. `database:` wins. A Postgres URL
|
|
720
|
+
# supplies the name when the hash has none. A `service:` entry does not
|
|
721
|
+
# carry the name, so that shape fails here instead of guessing.
|
|
722
|
+
#
|
|
723
|
+
# @param config_hash [Hash]
|
|
724
|
+
# @return [String]
|
|
725
|
+
# @raise [RuntimeError] when no database name is available.
|
|
726
|
+
def self.source_database_name!(config_hash)
|
|
727
|
+
hash = config_hash.transform_keys(&:to_sym)
|
|
728
|
+
name = hash[:database].to_s
|
|
729
|
+
name = database_name_from_url(hash[:url]) if name.empty?
|
|
730
|
+
return name unless name.empty?
|
|
731
|
+
|
|
732
|
+
raise "could not provision: the test database name is missing. " \
|
|
733
|
+
"Set database: or use a URL that includes the database name."
|
|
734
|
+
end
|
|
735
|
+
|
|
736
|
+
# Database name from a postgres URL path, or "" when `url` is blank.
|
|
737
|
+
#
|
|
738
|
+
# @param url [String, nil]
|
|
739
|
+
# @return [String]
|
|
740
|
+
# @raise [RuntimeError] when `url` is present and is not a Postgres URL.
|
|
741
|
+
def self.database_name_from_url(url)
|
|
742
|
+
options_from_database_url(url)[:database].to_s
|
|
743
|
+
end
|
|
744
|
+
|
|
745
|
+
# Host, credentials, database name, and query options from a Rails `url:`.
|
|
746
|
+
# Blank when `url` is blank. The database is returned under `:database`
|
|
747
|
+
# so callers can separate it from the maintenance `dbname`.
|
|
748
|
+
#
|
|
749
|
+
# @param url [String, nil]
|
|
750
|
+
# @return [Hash]
|
|
751
|
+
# @raise [RuntimeError] when `url` is present and is not a Postgres URL.
|
|
752
|
+
def self.options_from_database_url(url)
|
|
753
|
+
return {} if blank_option?(url)
|
|
754
|
+
|
|
755
|
+
uri = URI.parse(url.to_s)
|
|
756
|
+
scheme = uri.scheme.to_s.downcase
|
|
757
|
+
unless scheme == "postgres" || scheme == "postgresql"
|
|
758
|
+
raise "could not provision: the database URL must start with postgres:// or postgresql://"
|
|
759
|
+
end
|
|
760
|
+
|
|
761
|
+
opts = {}
|
|
762
|
+
host = uri.host
|
|
763
|
+
opts[:host] = host if host && !host.empty?
|
|
764
|
+
opts[:port] = uri.port if uri.port
|
|
765
|
+
user = decode_uri_component(uri.user)
|
|
766
|
+
opts[:user] = user if user && !user.empty?
|
|
767
|
+
password = decode_uri_component(uri.password)
|
|
768
|
+
opts[:password] = password if password && !password.empty?
|
|
769
|
+
database = uri.path.to_s.sub(%r{\A/}, "")
|
|
770
|
+
database = decode_uri_component(database) unless database.empty?
|
|
771
|
+
opts[:database] = database if database && !database.empty?
|
|
772
|
+
URI.decode_www_form(uri.query.to_s).each do |key, value|
|
|
773
|
+
pg_key = LIBPQ_CONNECTION_KEYS[key.to_sym]
|
|
774
|
+
next unless pg_key
|
|
775
|
+
next if blank_option?(value)
|
|
776
|
+
|
|
777
|
+
opts[pg_key] = value
|
|
778
|
+
end
|
|
779
|
+
opts
|
|
780
|
+
rescue URI::InvalidURIError, URI::InvalidComponentError, ArgumentError
|
|
781
|
+
# The parser's message repeats the URL, password included. `cause: nil`
|
|
782
|
+
# keeps Ruby from printing that original error under this one.
|
|
783
|
+
raise "could not provision: the database URL is not valid. " \
|
|
784
|
+
"The URL is not shown because it can contain a password. " \
|
|
785
|
+
"Check each part of DATABASE_URL or url: in config/database.yml. " \
|
|
786
|
+
"Percent-encode the user name, password and database name. Encode each " \
|
|
787
|
+
"character that is not an ASCII letter, a digit, -, ., _ or ~. " \
|
|
788
|
+
"For example, write % as %25, @ as %40 and a space as %20.", cause: nil
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
# Percent-decode one URL component. Nil when `text` is blank.
|
|
792
|
+
#
|
|
793
|
+
# @param text [String, nil]
|
|
794
|
+
# @return [String, nil]
|
|
795
|
+
def self.decode_uri_component(text)
|
|
796
|
+
return if blank_option?(text)
|
|
797
|
+
|
|
798
|
+
# `unescape` keeps a literal `+`. The form decoder would turn it into a space.
|
|
799
|
+
# The daemon runs on the app's Ruby, which may be older than 3.4.
|
|
800
|
+
URI::DEFAULT_PARSER.unescape(text.to_s)
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
# True for nil and empty strings. Numbers and false are present values.
|
|
804
|
+
#
|
|
805
|
+
# @param value [Object]
|
|
806
|
+
# @return [Boolean]
|
|
807
|
+
def self.blank_option?(value)
|
|
808
|
+
value.nil? || (value.respond_to?(:empty?) && value.empty?)
|
|
809
|
+
end
|
|
810
|
+
|
|
811
|
+
# @param conn [PG::Connection]
|
|
812
|
+
# @param base [String]
|
|
813
|
+
# @param shared [Boolean] shared lock when true, exclusive when false.
|
|
814
|
+
# @return [Boolean]
|
|
815
|
+
def self.try_lock(conn, base, shared:)
|
|
816
|
+
k1, k2 = lock_keys(base)
|
|
817
|
+
sql = shared ? "SELECT pg_try_advisory_lock_shared($1, $2)" : "SELECT pg_try_advisory_lock($1, $2)"
|
|
818
|
+
conn.exec_params(sql, [k1, k2]).getvalue(0, 0) == "t"
|
|
819
|
+
end
|
|
820
|
+
|
|
821
|
+
# Release one advisory lock this session holds for `base`.
|
|
822
|
+
#
|
|
823
|
+
# @param conn [PG::Connection]
|
|
824
|
+
# @param base [String]
|
|
825
|
+
# @return [void]
|
|
826
|
+
def self.unlock(conn, base)
|
|
827
|
+
k1, k2 = lock_keys(base)
|
|
828
|
+
conn.exec_params("SELECT pg_advisory_unlock($1, $2)", [k1, k2])
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
# Two signed 32-bit keys for `pg_try_advisory_lock(int, int)`, from the base name.
|
|
832
|
+
#
|
|
833
|
+
# @param base [String]
|
|
834
|
+
# @return [Array(Integer, Integer)]
|
|
835
|
+
def self.lock_keys(base)
|
|
836
|
+
Digest::SHA256.digest("mutineer-run:#{base}").unpack("l>l>")
|
|
837
|
+
end
|
|
838
|
+
|
|
839
|
+
# Replace `name` with a template copy of `base`. Terminates other sessions
|
|
840
|
+
# on the slot only. A session on `base` is left alone and fails the copy.
|
|
841
|
+
#
|
|
842
|
+
# @param conn [PG::Connection]
|
|
843
|
+
# @param name [String] exact worker database name.
|
|
844
|
+
# @param base [String] test database used as the template.
|
|
845
|
+
# @return [void]
|
|
846
|
+
def self.recreate_from_template(conn, name, base)
|
|
847
|
+
unless worker_database_name?(base, name)
|
|
848
|
+
raise "refusing to drop #{name}"
|
|
849
|
+
end
|
|
850
|
+
|
|
851
|
+
conn.exec("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = #{conn.escape_literal(name)} AND pid <> pg_backend_pid()")
|
|
852
|
+
conn.exec("DROP DATABASE IF EXISTS #{quote_ident(name)}")
|
|
853
|
+
conn.exec("CREATE DATABASE #{quote_ident(name)} TEMPLATE #{quote_ident(base)}")
|
|
854
|
+
rescue StandardError => e
|
|
855
|
+
raise e if e.message.start_with?("refusing to drop")
|
|
856
|
+
|
|
857
|
+
raise provision_failure(base, name, e.message)
|
|
858
|
+
end
|
|
859
|
+
|
|
860
|
+
# Close every connection this process holds through ActiveRecord, so the
|
|
861
|
+
# test database can be used as a Postgres template and a MySQL slot can be
|
|
862
|
+
# dropped.
|
|
863
|
+
#
|
|
864
|
+
# @return [void]
|
|
865
|
+
def self.disconnect_app!
|
|
866
|
+
handler = ActiveRecord::Base.connection_handler
|
|
867
|
+
if handler.respond_to?(:clear_all_connections!)
|
|
868
|
+
handler.clear_all_connections!
|
|
869
|
+
else
|
|
870
|
+
ActiveRecord::Base.connection_pool.disconnect!
|
|
871
|
+
end
|
|
872
|
+
end
|
|
873
|
+
|
|
874
|
+
# The child inherited the parent's lock session. Drop the Ruby wrapper
|
|
875
|
+
# without sending a disconnect, so the parent's locks stay held.
|
|
876
|
+
# Clearing the held-name list stops the child from sending RELEASE_LOCK
|
|
877
|
+
# on that shared session.
|
|
878
|
+
#
|
|
879
|
+
# @return [void]
|
|
880
|
+
def self.forget_lock_connection_after_fork!
|
|
881
|
+
@mysql_held_locks = []
|
|
882
|
+
conn = @lock_connection
|
|
883
|
+
@lock_connection = nil
|
|
884
|
+
return unless conn
|
|
885
|
+
|
|
886
|
+
detach_inherited_connection(conn)
|
|
887
|
+
ObjectSpace.undefine_finalizer(conn)
|
|
888
|
+
end
|
|
889
|
+
|
|
890
|
+
# Make a fork child's copy of the lock connection safe to free. The
|
|
891
|
+
# client's free function runs at the child's next GC. Left as it is, that
|
|
892
|
+
# sends QUIT (mysql2) or Terminate (pg) on the socket the parent shares,
|
|
893
|
+
# so the server ends the parent's session and every lock it holds.
|
|
894
|
+
# These are the steps ActiveRecord's adapters take in `discard!`.
|
|
895
|
+
#
|
|
896
|
+
# @param conn [Mysql2::Client, Trilogy, PG::Connection]
|
|
897
|
+
# @return [void]
|
|
898
|
+
def self.detach_inherited_connection(conn)
|
|
899
|
+
if conn.respond_to?(:automatic_close=)
|
|
900
|
+
conn.automatic_close = false # mysql2: GC drops only this process's fd
|
|
901
|
+
elsif conn.respond_to?(:discard!)
|
|
902
|
+
conn.discard! # trilogy: closes only this process's fd
|
|
903
|
+
elsif conn.respond_to?(:socket_io)
|
|
904
|
+
conn.socket_io.reopen(IO::NULL) # pg: Terminate goes to /dev/null
|
|
905
|
+
end
|
|
906
|
+
end
|
|
907
|
+
|
|
908
|
+
# Quote a database name for a Postgres identifier. `CREATE DATABASE` cannot
|
|
909
|
+
# take a bound parameter for the name.
|
|
910
|
+
#
|
|
911
|
+
# @param name [String]
|
|
912
|
+
# @return [String]
|
|
913
|
+
def self.quote_ident(name)
|
|
914
|
+
raise ArgumentError, "database name is empty" if name.nil? || name.empty?
|
|
915
|
+
|
|
916
|
+
%("#{name.gsub('"', '""')}")
|
|
917
|
+
end
|
|
918
|
+
|
|
919
|
+
# Failure text for a copy that did not happen. Names the database and the
|
|
920
|
+
# server's cause. A busy template also says to close other sessions.
|
|
921
|
+
#
|
|
922
|
+
# @param base [String]
|
|
923
|
+
# @param name [String]
|
|
924
|
+
# @param cause [String]
|
|
925
|
+
# @return [String]
|
|
926
|
+
def self.provision_failure(base, name, cause)
|
|
927
|
+
message = "could not provision #{name} from #{base}: #{cause}"
|
|
928
|
+
message += ". Close other sessions on #{base}." if cause.match?(/other users|being accessed/i)
|
|
929
|
+
message
|
|
930
|
+
end
|
|
931
|
+
|
|
932
|
+
# Create MySQL slots `0...count`, load the app schema into each, and copy
|
|
933
|
+
# rows from the test database. Holds the guard lock and every pool name
|
|
934
|
+
# while copying, then keeps the guard lock. Pool names are released without
|
|
935
|
+
# a gap so another run cannot drop the new databases. Worker daemons then
|
|
936
|
+
# take one pool name each.
|
|
937
|
+
#
|
|
938
|
+
# @param count [Integer]
|
|
939
|
+
# @return [void]
|
|
940
|
+
# @raise [RuntimeError] when `count` exceeds {MYSQL_LOCK_SLOTS}, the server is
|
|
941
|
+
# MySQL older than 8.0, or another run holds the lock.
|
|
942
|
+
def self.provision_mysql(count)
|
|
943
|
+
if count > MYSQL_LOCK_SLOTS
|
|
944
|
+
raise "MySQL worker databases support at most #{MYSQL_LOCK_SLOTS} workers"
|
|
945
|
+
end
|
|
946
|
+
|
|
947
|
+
hash = routing_config_hash
|
|
948
|
+
remember_mysql_base_config(hash)
|
|
949
|
+
base = hash[:database].to_s
|
|
950
|
+
conn = open_mysql_connection(hash, base)
|
|
951
|
+
if (refusal = check_mysql_server(conn, password: hash[:password]))
|
|
952
|
+
conn.close
|
|
953
|
+
raise "could not provision #{base}: #{refusal}"
|
|
954
|
+
end
|
|
955
|
+
unless try_mysql_lock(conn, base, shared: false)
|
|
956
|
+
conn.close
|
|
957
|
+
raise "another Mutineer run is using #{base}"
|
|
958
|
+
end
|
|
959
|
+
|
|
960
|
+
begin
|
|
961
|
+
disconnect_app!
|
|
962
|
+
count.times do |slot|
|
|
963
|
+
name = mysql_worker_database(base, slot)
|
|
964
|
+
begin
|
|
965
|
+
recreate_mysql_database(conn, name, base)
|
|
966
|
+
load_mysql_schema(hash, name, base)
|
|
967
|
+
copy_mysql_rows(conn, base, name)
|
|
968
|
+
rescue StandardError => e
|
|
969
|
+
raise e if provision_wrapped?(e)
|
|
970
|
+
|
|
971
|
+
raise provision_failure(base, name, e.message)
|
|
972
|
+
end
|
|
973
|
+
end
|
|
974
|
+
raise "another Mutineer run is using #{base}" unless downgrade_mysql_lock_to_shared(conn, base)
|
|
975
|
+
rescue StandardError => e
|
|
976
|
+
release_held_mysql_locks(conn)
|
|
977
|
+
conn.close
|
|
978
|
+
raise e if provision_wrapped?(e)
|
|
979
|
+
|
|
980
|
+
raise provision_failure(base, base, e.message)
|
|
981
|
+
end
|
|
982
|
+
|
|
983
|
+
@lock_connection = conn
|
|
984
|
+
warn "[mutineer] provisioned #{count} mysql worker database(s) from #{base}"
|
|
985
|
+
end
|
|
986
|
+
|
|
987
|
+
# Read the server version and print a warning when the matrix does not
|
|
988
|
+
# test it. Does not close `conn` or raise: the caller does both. A failed
|
|
989
|
+
# version query only warns, with its cause shown. URL passwords, the
|
|
990
|
+
# DATABASE_URL password and the configured `password` are hidden in the
|
|
991
|
+
# cause and in the version text.
|
|
992
|
+
#
|
|
993
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
994
|
+
# @param password [String, nil] the configured password, hidden when it appears.
|
|
995
|
+
# @return [String, nil] the refusal text for MySQL older than 8.0, else nil.
|
|
996
|
+
def self.check_mysql_server(conn, password: nil)
|
|
997
|
+
secret = password.to_s
|
|
998
|
+
hide = lambda do |text|
|
|
999
|
+
out = Redaction.text(text)
|
|
1000
|
+
next out if secret.empty?
|
|
1001
|
+
# A short password would match inside ordinary words, so hide it only as a whole word.
|
|
1002
|
+
pattern = secret.length >= 4 ? secret : /(?<![[:alnum:]])#{Regexp.escape(secret)}(?![[:alnum:]])/
|
|
1003
|
+
out.gsub(pattern, Redaction::REDACTED)
|
|
1004
|
+
end
|
|
1005
|
+
failure = nil
|
|
1006
|
+
raw = begin
|
|
1007
|
+
mysql_rows(conn, "SELECT VERSION() AS version").first&.fetch("version", nil)
|
|
1008
|
+
rescue StandardError => e
|
|
1009
|
+
failure = hide.call("#{e.class}: #{e.message.lines.first.to_s.strip}")
|
|
1010
|
+
nil
|
|
1011
|
+
end
|
|
1012
|
+
support = mysql_version_support(raw)
|
|
1013
|
+
version = raw.nil? ? nil : hide.call(raw)
|
|
1014
|
+
|
|
1015
|
+
case support
|
|
1016
|
+
when :not_supported
|
|
1017
|
+
"MySQL #{version} is not supported. --daemon needs MySQL 8.0 or later. " \
|
|
1018
|
+
"Pass --no-daemon (or daemon: false) to stay serial."
|
|
1019
|
+
when :not_tested
|
|
1020
|
+
warn "[mutineer] server version #{version} is not tested with --daemon. " \
|
|
1021
|
+
"The run continues. If a result looks wrong, pass --no-daemon."
|
|
1022
|
+
nil
|
|
1023
|
+
when :unreadable
|
|
1024
|
+
shown = if failure
|
|
1025
|
+
"an error: #{failure}"
|
|
1026
|
+
elsif version.to_s.empty?
|
|
1027
|
+
"nothing"
|
|
1028
|
+
else
|
|
1029
|
+
"\"#{version}\""
|
|
1030
|
+
end
|
|
1031
|
+
warn "[mutineer] could not read the MySQL server version (got #{shown}). " \
|
|
1032
|
+
"The run continues. --daemon is tested on MySQL 8."
|
|
1033
|
+
nil
|
|
1034
|
+
end
|
|
1035
|
+
end
|
|
1036
|
+
|
|
1037
|
+
# Take one shared GET_LOCK name and drop this process's app connection.
|
|
1038
|
+
#
|
|
1039
|
+
# @return [void]
|
|
1040
|
+
def self.prepare_mysql_worker!
|
|
1041
|
+
hash = current_config_hash
|
|
1042
|
+
base = hash[:database].to_s
|
|
1043
|
+
conn = open_mysql_connection(hash, base)
|
|
1044
|
+
unless try_mysql_lock(conn, base, shared: true)
|
|
1045
|
+
conn.close
|
|
1046
|
+
raise "another Mutineer run is using #{base}"
|
|
1047
|
+
end
|
|
1048
|
+
|
|
1049
|
+
@lock_connection = conn
|
|
1050
|
+
disconnect_app!
|
|
1051
|
+
end
|
|
1052
|
+
|
|
1053
|
+
# Open a session with the app's MySQL client. It starts with no default
|
|
1054
|
+
# database, so it is not tied to the test database while it drops and
|
|
1055
|
+
# creates slots. {restore_mysql_triggers} later selects a slot. Every
|
|
1056
|
+
# other statement names its database. The gem does not depend on the client.
|
|
1057
|
+
#
|
|
1058
|
+
# @param config_hash [Hash]
|
|
1059
|
+
# @param base [String] test database name, named in the error if this fails.
|
|
1060
|
+
# @return [Mysql2::Client, Trilogy]
|
|
1061
|
+
def self.open_mysql_connection(config_hash, base)
|
|
1062
|
+
adapter = config_hash[:adapter].to_s
|
|
1063
|
+
opts = mysql_client_options(config_hash)
|
|
1064
|
+
if adapter == "trilogy"
|
|
1065
|
+
raise "could not provision #{base}: the app's trilogy client is not loaded" unless defined?(Trilogy)
|
|
1066
|
+
|
|
1067
|
+
# Rails' adapter turns a `ssl_mode` name into the constant trilogy wants.
|
|
1068
|
+
if defined?(ActiveRecord::ConnectionAdapters::TrilogyAdapter)
|
|
1069
|
+
ActiveRecord::ConnectionAdapters::TrilogyAdapter.new_client(opts)
|
|
1070
|
+
else
|
|
1071
|
+
Trilogy.new(opts)
|
|
1072
|
+
end
|
|
1073
|
+
else
|
|
1074
|
+
raise "could not provision #{base}: the app's mysql2 client is not loaded" unless defined?(Mysql2::Client)
|
|
1075
|
+
|
|
1076
|
+
Mysql2::Client.new(opts)
|
|
1077
|
+
end
|
|
1078
|
+
rescue StandardError => e
|
|
1079
|
+
raise e if e.message.start_with?("could not provision ")
|
|
1080
|
+
|
|
1081
|
+
raise "could not provision #{base}: #{e.message}"
|
|
1082
|
+
end
|
|
1083
|
+
|
|
1084
|
+
# Connection options shared by mysql2 and trilogy. The database key is left
|
|
1085
|
+
# out so the session is not tied to the test database. A socket wins over
|
|
1086
|
+
# host: trilogy ignores `socket` when `host` is set, and would then use its
|
|
1087
|
+
# compiled-in default instead of the app's socket. TLS and timeout keys in
|
|
1088
|
+
# {MYSQL_CLIENT_KEYS} are kept.
|
|
1089
|
+
#
|
|
1090
|
+
# @param config_hash [Hash]
|
|
1091
|
+
# @return [Hash]
|
|
1092
|
+
def self.mysql_client_options(config_hash)
|
|
1093
|
+
opts = {}
|
|
1094
|
+
socket = config_hash[:socket].to_s
|
|
1095
|
+
if socket.empty?
|
|
1096
|
+
host = config_hash[:host].to_s
|
|
1097
|
+
opts[:host] = host unless host.empty?
|
|
1098
|
+
port = config_hash[:port]
|
|
1099
|
+
opts[:port] = Integer(port) if port && !port.to_s.empty?
|
|
1100
|
+
else
|
|
1101
|
+
opts[:socket] = socket
|
|
1102
|
+
end
|
|
1103
|
+
user = config_hash[:username] || config_hash[:user]
|
|
1104
|
+
opts[:username] = user.to_s if user && !user.to_s.empty?
|
|
1105
|
+
password = config_hash[:password]
|
|
1106
|
+
opts[:password] = password.to_s unless password.nil?
|
|
1107
|
+
MYSQL_CLIENT_KEYS.each { |key| opts[key] = config_hash[key] unless config_hash[key].nil? }
|
|
1108
|
+
# `:encoding` is an ActiveRecord key. mysql2 and trilogy do not share it.
|
|
1109
|
+
# Row copy runs in the server, so the client charset does not change it.
|
|
1110
|
+
opts
|
|
1111
|
+
end
|
|
1112
|
+
|
|
1113
|
+
# Take one pool name, or the guard plus every pool name when `shared` is
|
|
1114
|
+
# false. A miss releases any names this call already took.
|
|
1115
|
+
#
|
|
1116
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1117
|
+
# @param base [String]
|
|
1118
|
+
# @param shared [Boolean]
|
|
1119
|
+
# @return [Boolean]
|
|
1120
|
+
def self.try_mysql_lock(conn, base, shared:)
|
|
1121
|
+
return try_mysql_shared_lock(conn, base) if shared
|
|
1122
|
+
|
|
1123
|
+
try_mysql_exclusive_lock(conn, base)
|
|
1124
|
+
end
|
|
1125
|
+
|
|
1126
|
+
# Take the guard, then every pool name. The guard stays held across the
|
|
1127
|
+
# later downgrade so another run cannot start a copy in that gap.
|
|
1128
|
+
#
|
|
1129
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1130
|
+
# @param base [String]
|
|
1131
|
+
# @return [Boolean]
|
|
1132
|
+
def self.try_mysql_exclusive_lock(conn, base)
|
|
1133
|
+
guard = mysql_guard_lock_name(base)
|
|
1134
|
+
return false unless mysql_get_lock(conn, guard)
|
|
1135
|
+
|
|
1136
|
+
held = [guard]
|
|
1137
|
+
mysql_lock_names(base).each do |name|
|
|
1138
|
+
if mysql_get_lock(conn, name)
|
|
1139
|
+
held << name
|
|
1140
|
+
else
|
|
1141
|
+
held.each { |got| mysql_release_lock(conn, got) }
|
|
1142
|
+
@mysql_held_locks = []
|
|
1143
|
+
return false
|
|
1144
|
+
end
|
|
1145
|
+
end
|
|
1146
|
+
@mysql_held_locks = held
|
|
1147
|
+
true
|
|
1148
|
+
end
|
|
1149
|
+
|
|
1150
|
+
# Take the first free pool name. Does not take the guard, so worker daemons
|
|
1151
|
+
# can connect while the provisioning process still holds that guard.
|
|
1152
|
+
#
|
|
1153
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1154
|
+
# @param base [String]
|
|
1155
|
+
# @return [Boolean]
|
|
1156
|
+
def self.try_mysql_shared_lock(conn, base)
|
|
1157
|
+
mysql_lock_names(base).each do |name|
|
|
1158
|
+
next unless mysql_get_lock(conn, name)
|
|
1159
|
+
|
|
1160
|
+
@mysql_held_locks = [name]
|
|
1161
|
+
return true
|
|
1162
|
+
end
|
|
1163
|
+
@mysql_held_locks = []
|
|
1164
|
+
false
|
|
1165
|
+
end
|
|
1166
|
+
|
|
1167
|
+
# Release every pool name and keep the guard. The guard is never released
|
|
1168
|
+
# first, so this process always holds a lock the next copy needs.
|
|
1169
|
+
#
|
|
1170
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1171
|
+
# @param base [String]
|
|
1172
|
+
# @return [Boolean] false when this session does not hold the guard.
|
|
1173
|
+
def self.downgrade_mysql_lock_to_shared(conn, base)
|
|
1174
|
+
guard = mysql_guard_lock_name(base)
|
|
1175
|
+
held = Array(@mysql_held_locks)
|
|
1176
|
+
return false unless held.include?(guard)
|
|
1177
|
+
|
|
1178
|
+
(held - [guard]).each { |name| mysql_release_lock(conn, name) }
|
|
1179
|
+
@mysql_held_locks = [guard]
|
|
1180
|
+
true
|
|
1181
|
+
end
|
|
1182
|
+
|
|
1183
|
+
# Release the GET_LOCK names this session took for the current run.
|
|
1184
|
+
#
|
|
1185
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1186
|
+
# @return [void]
|
|
1187
|
+
def self.release_held_mysql_locks(conn)
|
|
1188
|
+
Array(@mysql_held_locks).each { |name| mysql_release_lock(conn, name) }
|
|
1189
|
+
@mysql_held_locks = []
|
|
1190
|
+
nil
|
|
1191
|
+
end
|
|
1192
|
+
|
|
1193
|
+
# One GET_LOCK name per worker daemon. The digest keeps each name inside 64
|
|
1194
|
+
# characters for any base database name.
|
|
1195
|
+
#
|
|
1196
|
+
# @param base [String]
|
|
1197
|
+
# @return [Array<String>]
|
|
1198
|
+
def self.mysql_lock_names(base)
|
|
1199
|
+
digest = mysql_lock_digest(base)
|
|
1200
|
+
(0...MYSQL_LOCK_SLOTS).map { |slot| "m:#{digest}:#{slot}" }
|
|
1201
|
+
end
|
|
1202
|
+
|
|
1203
|
+
# Lock held for the whole copy and until worker daemons hold pool names.
|
|
1204
|
+
# Not part of {mysql_lock_names}, so it does not reduce the worker count.
|
|
1205
|
+
#
|
|
1206
|
+
# @param base [String]
|
|
1207
|
+
# @return [String]
|
|
1208
|
+
def self.mysql_guard_lock_name(base)
|
|
1209
|
+
"m:#{mysql_lock_digest(base)}:g"
|
|
1210
|
+
end
|
|
1211
|
+
|
|
1212
|
+
# Shared prefix for the guard and the pool. 16 hex characters keep the full
|
|
1213
|
+
# lock name inside MySQL's 64-character limit.
|
|
1214
|
+
#
|
|
1215
|
+
# @param base [String]
|
|
1216
|
+
# @return [String]
|
|
1217
|
+
def self.mysql_lock_digest(base)
|
|
1218
|
+
Digest::SHA256.hexdigest("mutineer-run:#{base}")[0, 16]
|
|
1219
|
+
end
|
|
1220
|
+
|
|
1221
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1222
|
+
# @param name [String]
|
|
1223
|
+
# @return [Boolean]
|
|
1224
|
+
def self.mysql_get_lock(conn, name)
|
|
1225
|
+
rows = mysql_rows(conn, "SELECT GET_LOCK(#{mysql_quote_string(conn, name)}, 0) AS got")
|
|
1226
|
+
rows.dig(0, "got").to_i == 1
|
|
1227
|
+
end
|
|
1228
|
+
|
|
1229
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1230
|
+
# @param name [String]
|
|
1231
|
+
# @return [void]
|
|
1232
|
+
def self.mysql_release_lock(conn, name)
|
|
1233
|
+
mysql_exec(conn, "SELECT RELEASE_LOCK(#{mysql_quote_string(conn, name)})")
|
|
1234
|
+
end
|
|
1235
|
+
|
|
1236
|
+
# Drop and create `name`. Refuses any name that is not one of `base`'s
|
|
1237
|
+
# worker database names, so a name bug can never drop the test database.
|
|
1238
|
+
#
|
|
1239
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1240
|
+
# @param name [String]
|
|
1241
|
+
# @param base [String] the test database name.
|
|
1242
|
+
# @return [void]
|
|
1243
|
+
def self.recreate_mysql_database(conn, name, base)
|
|
1244
|
+
raise "refusing to drop #{name}" unless owned_database?(base, name, slots: MYSQL_LOCK_SLOTS, limit: MYSQL_NAME_LIMIT)
|
|
1245
|
+
|
|
1246
|
+
quoted = quote_mysql_ident(name)
|
|
1247
|
+
charset, collation = mysql_database_defaults(conn, base)
|
|
1248
|
+
mysql_exec(conn, "DROP DATABASE IF EXISTS #{quoted}")
|
|
1249
|
+
mysql_exec(conn, "CREATE DATABASE #{quoted} CHARACTER SET #{charset} COLLATE #{collation}")
|
|
1250
|
+
end
|
|
1251
|
+
|
|
1252
|
+
# The test database's default character set and collation. A table the
|
|
1253
|
+
# schema creates without its own collation then compares text the same way
|
|
1254
|
+
# in each worker database as in the test database.
|
|
1255
|
+
#
|
|
1256
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1257
|
+
# @param base [String] the test database name.
|
|
1258
|
+
# @return [Array(String, String)] the character set and the collation.
|
|
1259
|
+
# @raise [RuntimeError] when the test database is missing or reports an unexpected name.
|
|
1260
|
+
def self.mysql_database_defaults(conn, base)
|
|
1261
|
+
row = mysql_rows(conn, "SELECT DEFAULT_CHARACTER_SET_NAME AS charset, DEFAULT_COLLATION_NAME AS collation " \
|
|
1262
|
+
"FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = #{mysql_quote_string(conn, base)}").first
|
|
1263
|
+
values = row && [row["charset"].to_s, row["collation"].to_s]
|
|
1264
|
+
raise "cannot read the character set of #{base}" unless values&.all? { |value| value.match?(/\A\w+\z/) }
|
|
1265
|
+
|
|
1266
|
+
values
|
|
1267
|
+
end
|
|
1268
|
+
|
|
1269
|
+
# Load `schema.rb` or `structure.sql` into the empty slot with the app's
|
|
1270
|
+
# DatabaseTasks. A missing file leaves the slot empty so the row copy can
|
|
1271
|
+
# still clone tables that exist only on the test database.
|
|
1272
|
+
#
|
|
1273
|
+
# @param base_hash [Hash] symbol-keyed config for the test database.
|
|
1274
|
+
# @param name [String] worker database name.
|
|
1275
|
+
# @param base [String] test database name, used in the failure text.
|
|
1276
|
+
# @return [void]
|
|
1277
|
+
def self.load_mysql_schema(base_hash, name, base)
|
|
1278
|
+
return unless defined?(ActiveRecord::Tasks::DatabaseTasks)
|
|
1279
|
+
|
|
1280
|
+
switched = false
|
|
1281
|
+
failure = nil
|
|
1282
|
+
begin
|
|
1283
|
+
slot_hash = base_hash.merge(database: name)
|
|
1284
|
+
ActiveRecord::Base.establish_connection(slot_hash)
|
|
1285
|
+
switched = true
|
|
1286
|
+
db_config = mysql_db_config(slot_hash)
|
|
1287
|
+
format = schema_format_for(db_config)
|
|
1288
|
+
file = mysql_schema_file(db_config, format)
|
|
1289
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, format, file) if file
|
|
1290
|
+
rescue StandardError => e
|
|
1291
|
+
failure = e
|
|
1292
|
+
ensure
|
|
1293
|
+
if switched
|
|
1294
|
+
begin
|
|
1295
|
+
ActiveRecord::Base.establish_connection(base_hash)
|
|
1296
|
+
rescue StandardError => e
|
|
1297
|
+
failure ||= e
|
|
1298
|
+
end
|
|
1299
|
+
end
|
|
1300
|
+
end
|
|
1301
|
+
return unless failure
|
|
1302
|
+
raise failure if provision_wrapped?(failure)
|
|
1303
|
+
|
|
1304
|
+
raise provision_failure(base, name, failure.message)
|
|
1305
|
+
end
|
|
1306
|
+
|
|
1307
|
+
# A database config whose name is the slot, so DatabaseTasks loads into that
|
|
1308
|
+
# database and not the test database.
|
|
1309
|
+
#
|
|
1310
|
+
# @param slot_hash [Hash]
|
|
1311
|
+
# @return [ActiveRecord::DatabaseConfigurations::HashConfig]
|
|
1312
|
+
def self.mysql_db_config(slot_hash)
|
|
1313
|
+
env = ENV["RAILS_ENV"].to_s
|
|
1314
|
+
env = "test" if env.empty?
|
|
1315
|
+
ActiveRecord::DatabaseConfigurations::HashConfig.new(env, "primary", slot_hash)
|
|
1316
|
+
end
|
|
1317
|
+
|
|
1318
|
+
# Schema format of one database config. Rails 8.1 lets each config set its
|
|
1319
|
+
# own `schema_format:`; older Rails has only the app-wide setting. A string
|
|
1320
|
+
# setting ("ruby") counts the same as the symbol.
|
|
1321
|
+
#
|
|
1322
|
+
# @param db_config [ActiveRecord::DatabaseConfigurations::DatabaseConfig]
|
|
1323
|
+
# @return [Symbol]
|
|
1324
|
+
def self.schema_format_for(db_config)
|
|
1325
|
+
format = db_config.respond_to?(:schema_format) ? db_config.schema_format : current_schema_format
|
|
1326
|
+
format.respond_to?(:to_sym) ? format.to_sym : format
|
|
1327
|
+
end
|
|
1328
|
+
|
|
1329
|
+
# Schema format the booted app is using.
|
|
1330
|
+
#
|
|
1331
|
+
# @return [Symbol]
|
|
1332
|
+
def self.current_schema_format
|
|
1333
|
+
if ActiveRecord.respond_to?(:schema_format)
|
|
1334
|
+
ActiveRecord.schema_format
|
|
1335
|
+
else
|
|
1336
|
+
ActiveRecord::Base.schema_format
|
|
1337
|
+
end
|
|
1338
|
+
end
|
|
1339
|
+
|
|
1340
|
+
# Absolute path of the schema file for `format`, or nil when it is absent.
|
|
1341
|
+
# DatabaseTasks aborts the process when the file is missing, so the caller
|
|
1342
|
+
# skips the load instead.
|
|
1343
|
+
#
|
|
1344
|
+
# @param db_config [ActiveRecord::DatabaseConfigurations::HashConfig]
|
|
1345
|
+
# @param format [Symbol]
|
|
1346
|
+
# @return [String, nil]
|
|
1347
|
+
def self.mysql_schema_file(db_config, format)
|
|
1348
|
+
path = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)
|
|
1349
|
+
path if path && File.exist?(path)
|
|
1350
|
+
end
|
|
1351
|
+
|
|
1352
|
+
# Copy every base table from `base` into `dest`. Foreign key checks are off.
|
|
1353
|
+
# Generated columns are left out of the INSERT list. A table the schema load
|
|
1354
|
+
# did not create is cloned with CREATE TABLE ... LIKE, then its foreign keys
|
|
1355
|
+
# are added. LIKE does not copy them. Each table's AUTO_INCREMENT is set
|
|
1356
|
+
# from the base after the rows, so a deleted high id is not reused.
|
|
1357
|
+
# Triggers the schema load created are dropped for the copy and created
|
|
1358
|
+
# again after it, so the copied rows match the base.
|
|
1359
|
+
#
|
|
1360
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1361
|
+
# @param base [String]
|
|
1362
|
+
# @param dest [String]
|
|
1363
|
+
# @return [void]
|
|
1364
|
+
def self.copy_mysql_rows(conn, base, dest)
|
|
1365
|
+
mysql_exec(conn, "SET FOREIGN_KEY_CHECKS = 0")
|
|
1366
|
+
# MySQL 8 caches information_schema.TABLES values, AUTO_INCREMENT among
|
|
1367
|
+
# them, for a day by default. The copy reads them live.
|
|
1368
|
+
begin
|
|
1369
|
+
mysql_exec(conn, "SET SESSION information_schema_stats_expiry = 0")
|
|
1370
|
+
rescue StandardError
|
|
1371
|
+
nil # MariaDB has no such cache or variable.
|
|
1372
|
+
end
|
|
1373
|
+
triggers = drop_mysql_triggers(conn, dest)
|
|
1374
|
+
tables = mysql_rows(conn, <<~SQL)
|
|
1375
|
+
SELECT TABLE_NAME AS table_name
|
|
1376
|
+
FROM information_schema.TABLES
|
|
1377
|
+
WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
|
|
1378
|
+
AND TABLE_TYPE = 'BASE TABLE'
|
|
1379
|
+
SQL
|
|
1380
|
+
names = []
|
|
1381
|
+
cloned = []
|
|
1382
|
+
tables.each do |row|
|
|
1383
|
+
table = row["table_name"].to_s
|
|
1384
|
+
next if table.empty?
|
|
1385
|
+
|
|
1386
|
+
names << table
|
|
1387
|
+
cloned << table if ensure_mysql_table(conn, base, dest, table)
|
|
1388
|
+
columns = copyable_mysql_columns(conn, base, table)
|
|
1389
|
+
next if columns.empty?
|
|
1390
|
+
|
|
1391
|
+
dest_table = "#{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)}"
|
|
1392
|
+
base_table = "#{quote_mysql_ident(base)}.#{quote_mysql_ident(table)}"
|
|
1393
|
+
listed = columns.map { |column| quote_mysql_ident(column) }.join(", ")
|
|
1394
|
+
mysql_exec(conn, "DELETE FROM #{dest_table}")
|
|
1395
|
+
mysql_exec(conn, "INSERT INTO #{dest_table} (#{listed}) SELECT #{listed} FROM #{base_table}")
|
|
1396
|
+
end
|
|
1397
|
+
cloned.each { |table| copy_mysql_foreign_keys(conn, base, dest, table) }
|
|
1398
|
+
names.each { |table| copy_mysql_auto_increment(conn, base, dest, table) }
|
|
1399
|
+
restore_mysql_triggers(conn, dest, triggers)
|
|
1400
|
+
ensure
|
|
1401
|
+
begin
|
|
1402
|
+
mysql_exec(conn, "SET FOREIGN_KEY_CHECKS = 1")
|
|
1403
|
+
rescue StandardError
|
|
1404
|
+
nil
|
|
1405
|
+
end
|
|
1406
|
+
end
|
|
1407
|
+
|
|
1408
|
+
# Drop every trigger on `dest` and return what {restore_mysql_triggers}
|
|
1409
|
+
# needs to create each one again. Triggers on one table event come back in
|
|
1410
|
+
# ACTION_ORDER. MySQL 8's CREATE text has no FOLLOWS or PRECEDES, so
|
|
1411
|
+
# creating them in that order keeps their order.
|
|
1412
|
+
#
|
|
1413
|
+
# The CREATE text is read as utf8mb4 with an empty sql_mode. mysql2 0.5.6
|
|
1414
|
+
# and older open utf8mb3, which turns a 4-byte character into "?". Under
|
|
1415
|
+
# ANSI_QUOTES, MySQL quotes the names with double quotes, which the
|
|
1416
|
+
# trigger's own sql_mode may not read. Backticks read in every sql_mode.
|
|
1417
|
+
# The session stays on utf8mb4, so {restore_mysql_triggers} sends the
|
|
1418
|
+
# text back unchanged.
|
|
1419
|
+
#
|
|
1420
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1421
|
+
# @param dest [String]
|
|
1422
|
+
# @return [Array<Hash>] `:name`, `:sql`, `:sql_mode` and `:collation` for each trigger.
|
|
1423
|
+
# @raise [RuntimeError] when SHOW CREATE TRIGGER returns no row for a trigger.
|
|
1424
|
+
def self.drop_mysql_triggers(conn, dest)
|
|
1425
|
+
rows = mysql_rows(conn, <<~SQL)
|
|
1426
|
+
SELECT TRIGGER_NAME AS trigger_name
|
|
1427
|
+
FROM information_schema.TRIGGERS
|
|
1428
|
+
WHERE TRIGGER_SCHEMA = #{mysql_quote_string(conn, dest)}
|
|
1429
|
+
ORDER BY EVENT_OBJECT_TABLE, ACTION_TIMING, EVENT_MANIPULATION, ACTION_ORDER
|
|
1430
|
+
SQL
|
|
1431
|
+
return [] if rows.empty?
|
|
1432
|
+
|
|
1433
|
+
sql_mode = mysql_rows(conn, "SELECT @@SESSION.sql_mode AS sql_mode").first["sql_mode"]
|
|
1434
|
+
mysql_exec(conn, "SET NAMES utf8mb4")
|
|
1435
|
+
mysql_exec(conn, "SET SESSION sql_mode = ''")
|
|
1436
|
+
# Limit: the text comes back as utf8mb4. In a trigger saved with another
|
|
1437
|
+
# client charset, a _binary or _latin1 literal that holds non-ASCII bytes
|
|
1438
|
+
# changes bytes on replay.
|
|
1439
|
+
triggers = rows.map do |row|
|
|
1440
|
+
name = row["trigger_name"].to_s
|
|
1441
|
+
shown = mysql_rows(conn, "SHOW CREATE TRIGGER #{quote_mysql_ident(dest)}.#{quote_mysql_ident(name)}").first
|
|
1442
|
+
raise "cannot read trigger #{name} on #{dest}" unless shown
|
|
1443
|
+
|
|
1444
|
+
{ name: name, sql: shown["sql original statement"].to_s, sql_mode: shown["sql_mode"].to_s,
|
|
1445
|
+
collation: shown["collation_connection"].to_s }
|
|
1446
|
+
end
|
|
1447
|
+
# Every definition is saved before the first drop.
|
|
1448
|
+
triggers.each { |trigger| mysql_exec(conn, "DROP TRIGGER #{quote_mysql_ident(dest)}.#{quote_mysql_ident(trigger[:name])}") }
|
|
1449
|
+
mysql_exec(conn, "SET SESSION sql_mode = #{mysql_quote_string(conn, sql_mode)}")
|
|
1450
|
+
triggers
|
|
1451
|
+
end
|
|
1452
|
+
|
|
1453
|
+
# Create the triggers {drop_mysql_triggers} dropped. Each one is created
|
|
1454
|
+
# with its own sql_mode and connection collation, which decide how MySQL
|
|
1455
|
+
# reads its body. The CREATE text names no database, so this session
|
|
1456
|
+
# switches to `dest`. The sql_mode and collation go back to their values at
|
|
1457
|
+
# the start of this call. The character set stays utf8mb4. A failed create
|
|
1458
|
+
# names the trigger, for example when the DEFINER account needs SET_USER_ID.
|
|
1459
|
+
#
|
|
1460
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1461
|
+
# @param dest [String]
|
|
1462
|
+
# @param triggers [Array<Hash>]
|
|
1463
|
+
# @return [void]
|
|
1464
|
+
# @raise [RuntimeError] when MySQL refuses to create a trigger.
|
|
1465
|
+
def self.restore_mysql_triggers(conn, dest, triggers)
|
|
1466
|
+
return if triggers.empty?
|
|
1467
|
+
|
|
1468
|
+
saved = mysql_rows(conn, "SELECT @@SESSION.sql_mode AS sql_mode, @@SESSION.collation_connection AS collation").first
|
|
1469
|
+
mysql_exec(conn, "USE #{quote_mysql_ident(dest)}")
|
|
1470
|
+
triggers.each do |trigger|
|
|
1471
|
+
mysql_exec(conn, mysql_session_modes_sql(conn, trigger[:sql_mode], trigger[:collation]))
|
|
1472
|
+
mysql_exec(conn, trigger[:sql])
|
|
1473
|
+
rescue StandardError => e
|
|
1474
|
+
raise "cannot recreate trigger #{trigger[:name]} on #{dest}: #{e.message}"
|
|
1475
|
+
end
|
|
1476
|
+
mysql_exec(conn, mysql_session_modes_sql(conn, saved["sql_mode"], saved["collation"]))
|
|
1477
|
+
end
|
|
1478
|
+
|
|
1479
|
+
# SET statement for the session's sql_mode and connection collation.
|
|
1480
|
+
#
|
|
1481
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1482
|
+
# @param sql_mode [String]
|
|
1483
|
+
# @param collation [String]
|
|
1484
|
+
# @return [String]
|
|
1485
|
+
def self.mysql_session_modes_sql(conn, sql_mode, collation)
|
|
1486
|
+
"SET SESSION sql_mode = #{mysql_quote_string(conn, sql_mode)}, " \
|
|
1487
|
+
"collation_connection = #{mysql_quote_string(conn, collation)}"
|
|
1488
|
+
end
|
|
1489
|
+
|
|
1490
|
+
# Create `table` on `dest` when the schema load did not. LIKE copies columns,
|
|
1491
|
+
# including generated columns, and does not copy foreign keys. The caller
|
|
1492
|
+
# adds those keys after every missing table exists.
|
|
1493
|
+
#
|
|
1494
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1495
|
+
# @param base [String]
|
|
1496
|
+
# @param dest [String]
|
|
1497
|
+
# @param table [String]
|
|
1498
|
+
# @return [Boolean] true when this call created the table.
|
|
1499
|
+
def self.ensure_mysql_table(conn, base, dest, table)
|
|
1500
|
+
rows = mysql_rows(conn, <<~SQL)
|
|
1501
|
+
SELECT TABLE_NAME AS table_name
|
|
1502
|
+
FROM information_schema.TABLES
|
|
1503
|
+
WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, dest)}
|
|
1504
|
+
AND TABLE_NAME = #{mysql_quote_string(conn, table)}
|
|
1505
|
+
AND TABLE_TYPE = 'BASE TABLE'
|
|
1506
|
+
SQL
|
|
1507
|
+
return false unless rows.empty?
|
|
1508
|
+
|
|
1509
|
+
mysql_exec(conn, "CREATE TABLE #{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)} LIKE #{quote_mysql_ident(base)}.#{quote_mysql_ident(table)}")
|
|
1510
|
+
true
|
|
1511
|
+
end
|
|
1512
|
+
|
|
1513
|
+
# Add foreign keys from `base.table` onto `dest.table`. Keys that already
|
|
1514
|
+
# exist on `dest` are left alone. A referenced table in `base` is retargeted
|
|
1515
|
+
# at `dest` so the worker does not point back at the test database.
|
|
1516
|
+
#
|
|
1517
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1518
|
+
# @param base [String]
|
|
1519
|
+
# @param dest [String]
|
|
1520
|
+
# @param table [String]
|
|
1521
|
+
# @return [void]
|
|
1522
|
+
def self.copy_mysql_foreign_keys(conn, base, dest, table)
|
|
1523
|
+
existing = mysql_rows(conn, <<~SQL)
|
|
1524
|
+
SELECT CONSTRAINT_NAME AS constraint_name
|
|
1525
|
+
FROM information_schema.TABLE_CONSTRAINTS
|
|
1526
|
+
WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, dest)}
|
|
1527
|
+
AND TABLE_NAME = #{mysql_quote_string(conn, table)}
|
|
1528
|
+
AND CONSTRAINT_TYPE = 'FOREIGN KEY'
|
|
1529
|
+
SQL
|
|
1530
|
+
have = existing.map { |row| row["constraint_name"].to_s }
|
|
1531
|
+
rows = mysql_rows(conn, <<~SQL)
|
|
1532
|
+
SELECT k.CONSTRAINT_NAME AS constraint_name,
|
|
1533
|
+
k.COLUMN_NAME AS column_name,
|
|
1534
|
+
k.ORDINAL_POSITION AS ordinal_position,
|
|
1535
|
+
k.REFERENCED_TABLE_SCHEMA AS referenced_table_schema,
|
|
1536
|
+
k.REFERENCED_TABLE_NAME AS referenced_table_name,
|
|
1537
|
+
k.REFERENCED_COLUMN_NAME AS referenced_column_name,
|
|
1538
|
+
r.UPDATE_RULE AS update_rule,
|
|
1539
|
+
r.DELETE_RULE AS delete_rule,
|
|
1540
|
+
r.MATCH_OPTION AS match_option
|
|
1541
|
+
FROM information_schema.KEY_COLUMN_USAGE k
|
|
1542
|
+
INNER JOIN information_schema.REFERENTIAL_CONSTRAINTS r
|
|
1543
|
+
ON r.CONSTRAINT_SCHEMA = k.CONSTRAINT_SCHEMA
|
|
1544
|
+
AND r.CONSTRAINT_NAME = k.CONSTRAINT_NAME
|
|
1545
|
+
AND r.TABLE_NAME = k.TABLE_NAME
|
|
1546
|
+
WHERE k.TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
|
|
1547
|
+
AND k.TABLE_NAME = #{mysql_quote_string(conn, table)}
|
|
1548
|
+
AND k.REFERENCED_TABLE_NAME IS NOT NULL
|
|
1549
|
+
ORDER BY k.CONSTRAINT_NAME, k.ORDINAL_POSITION
|
|
1550
|
+
SQL
|
|
1551
|
+
rows = rows.reject { |row| have.include?(row["constraint_name"].to_s) }
|
|
1552
|
+
sql = mysql_add_foreign_key_sql(dest, table, rows, base: base)
|
|
1553
|
+
mysql_exec(conn, sql) if sql
|
|
1554
|
+
end
|
|
1555
|
+
|
|
1556
|
+
# One ALTER TABLE that adds every foreign key in `rows`.
|
|
1557
|
+
#
|
|
1558
|
+
# @param dest [String]
|
|
1559
|
+
# @param table [String]
|
|
1560
|
+
# @param rows [Array<Hash>]
|
|
1561
|
+
# @param base [String] test database name. References to it are rewritten to `dest`.
|
|
1562
|
+
# @return [String, nil]
|
|
1563
|
+
def self.mysql_add_foreign_key_sql(dest, table, rows, base:)
|
|
1564
|
+
clauses = mysql_foreign_key_clauses(rows, base: base, dest: dest)
|
|
1565
|
+
return nil if clauses.empty?
|
|
1566
|
+
|
|
1567
|
+
"ALTER TABLE #{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)} #{clauses.join(', ')}"
|
|
1568
|
+
end
|
|
1569
|
+
|
|
1570
|
+
# ADD CONSTRAINT fragments for `rows`, grouped by constraint name.
|
|
1571
|
+
#
|
|
1572
|
+
# @param rows [Array<Hash>]
|
|
1573
|
+
# @param base [String]
|
|
1574
|
+
# @param dest [String]
|
|
1575
|
+
# @return [Array<String>]
|
|
1576
|
+
def self.mysql_foreign_key_clauses(rows, base:, dest:)
|
|
1577
|
+
grouped = {}
|
|
1578
|
+
Array(rows).each do |row|
|
|
1579
|
+
name = row["constraint_name"].to_s
|
|
1580
|
+
next if name.empty?
|
|
1581
|
+
|
|
1582
|
+
(grouped[name] ||= []) << row
|
|
1583
|
+
end
|
|
1584
|
+
grouped.filter_map do |name, cols|
|
|
1585
|
+
ordered = cols.sort_by { |row| row["ordinal_position"].to_i }
|
|
1586
|
+
local_cols = ordered.map { |row| row["column_name"].to_s }
|
|
1587
|
+
ref_cols = ordered.map { |row| row["referenced_column_name"].to_s }
|
|
1588
|
+
ref_table = ordered.first["referenced_table_name"].to_s
|
|
1589
|
+
next if ref_table.empty? || local_cols.any?(&:empty?) || ref_cols.any?(&:empty?)
|
|
1590
|
+
|
|
1591
|
+
ref_schema = ordered.first["referenced_table_schema"].to_s
|
|
1592
|
+
ref_schema = dest if ref_schema.empty? || ref_schema == base
|
|
1593
|
+
delete_rule = mysql_referential_action(ordered.first["delete_rule"])
|
|
1594
|
+
update_rule = mysql_referential_action(ordered.first["update_rule"])
|
|
1595
|
+
match = mysql_match_clause(ordered.first["match_option"])
|
|
1596
|
+
local_sql = local_cols.map { |column| quote_mysql_ident(column) }.join(", ")
|
|
1597
|
+
ref_sql = ref_cols.map { |column| quote_mysql_ident(column) }.join(", ")
|
|
1598
|
+
target = "#{quote_mysql_ident(ref_schema)}.#{quote_mysql_ident(ref_table)}"
|
|
1599
|
+
"ADD CONSTRAINT #{quote_mysql_ident(name)} FOREIGN KEY (#{local_sql}) REFERENCES #{target} (#{ref_sql})#{match} ON DELETE #{delete_rule} ON UPDATE #{update_rule}"
|
|
1600
|
+
end
|
|
1601
|
+
end
|
|
1602
|
+
|
|
1603
|
+
# A foreign-key action safe to interpolate. Unknown text raises.
|
|
1604
|
+
#
|
|
1605
|
+
# @param value [String, nil]
|
|
1606
|
+
# @return [String]
|
|
1607
|
+
# @raise [ArgumentError] when `value` is not one of {MYSQL_REFERENTIAL_ACTIONS}.
|
|
1608
|
+
def self.mysql_referential_action(value)
|
|
1609
|
+
action = value.to_s.strip.upcase.tr("_", " ")
|
|
1610
|
+
return action if MYSQL_REFERENTIAL_ACTIONS.include?(action)
|
|
1611
|
+
|
|
1612
|
+
raise ArgumentError, "unsupported foreign key action #{value.inspect}"
|
|
1613
|
+
end
|
|
1614
|
+
|
|
1615
|
+
# MATCH fragment, or an empty string for the default NONE.
|
|
1616
|
+
#
|
|
1617
|
+
# @param value [String, nil]
|
|
1618
|
+
# @return [String]
|
|
1619
|
+
# @raise [ArgumentError] when `value` is not NONE, FULL, or PARTIAL.
|
|
1620
|
+
def self.mysql_match_clause(value)
|
|
1621
|
+
match = value.to_s.strip.upcase
|
|
1622
|
+
return "" if match.empty? || match == "NONE"
|
|
1623
|
+
return " MATCH #{match}" if %w[FULL PARTIAL].include?(match)
|
|
1624
|
+
|
|
1625
|
+
raise ArgumentError, "unsupported foreign key match #{value.inspect}"
|
|
1626
|
+
end
|
|
1627
|
+
|
|
1628
|
+
# Set `dest.table`'s AUTO_INCREMENT from the base table when the base has one.
|
|
1629
|
+
#
|
|
1630
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1631
|
+
# @param base [String]
|
|
1632
|
+
# @param dest [String]
|
|
1633
|
+
# @param table [String]
|
|
1634
|
+
# @return [void]
|
|
1635
|
+
def self.copy_mysql_auto_increment(conn, base, dest, table)
|
|
1636
|
+
rows = mysql_rows(conn, <<~SQL)
|
|
1637
|
+
SELECT AUTO_INCREMENT AS auto_increment
|
|
1638
|
+
FROM information_schema.TABLES
|
|
1639
|
+
WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
|
|
1640
|
+
AND TABLE_NAME = #{mysql_quote_string(conn, table)}
|
|
1641
|
+
SQL
|
|
1642
|
+
sql = mysql_auto_increment_sql(dest, table, rows.dig(0, "auto_increment"))
|
|
1643
|
+
mysql_exec(conn, sql) if sql
|
|
1644
|
+
end
|
|
1645
|
+
|
|
1646
|
+
# ALTER TABLE statement that sets AUTO_INCREMENT, or nil when `value` is blank.
|
|
1647
|
+
#
|
|
1648
|
+
# @param dest [String]
|
|
1649
|
+
# @param table [String]
|
|
1650
|
+
# @param value [Object]
|
|
1651
|
+
# @return [String, nil]
|
|
1652
|
+
def self.mysql_auto_increment_sql(dest, table, value)
|
|
1653
|
+
return nil if value.nil? || value.to_s.empty?
|
|
1654
|
+
|
|
1655
|
+
number = Integer(value)
|
|
1656
|
+
return nil if number < 1
|
|
1657
|
+
|
|
1658
|
+
"ALTER TABLE #{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)} AUTO_INCREMENT = #{number}"
|
|
1659
|
+
rescue ArgumentError, TypeError
|
|
1660
|
+
nil
|
|
1661
|
+
end
|
|
1662
|
+
|
|
1663
|
+
# Column names that an INSERT may set. Stored and virtual generated columns
|
|
1664
|
+
# are omitted. Expression defaults stay, because they are real columns.
|
|
1665
|
+
#
|
|
1666
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1667
|
+
# @param base [String]
|
|
1668
|
+
# @param table [String]
|
|
1669
|
+
# @return [Array<String>]
|
|
1670
|
+
def self.copyable_mysql_columns(conn, base, table)
|
|
1671
|
+
rows = mysql_rows(conn, <<~SQL)
|
|
1672
|
+
SELECT COLUMN_NAME AS column_name, EXTRA AS extra
|
|
1673
|
+
FROM information_schema.COLUMNS
|
|
1674
|
+
WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
|
|
1675
|
+
AND TABLE_NAME = #{mysql_quote_string(conn, table)}
|
|
1676
|
+
ORDER BY ORDINAL_POSITION
|
|
1677
|
+
SQL
|
|
1678
|
+
rows.filter_map do |row|
|
|
1679
|
+
name = row["column_name"].to_s
|
|
1680
|
+
next if name.empty?
|
|
1681
|
+
next if row["extra"].to_s.match?(/VIRTUAL GENERATED|STORED GENERATED/i)
|
|
1682
|
+
|
|
1683
|
+
name
|
|
1684
|
+
end
|
|
1685
|
+
end
|
|
1686
|
+
|
|
1687
|
+
# Run a statement that does not need its result rows.
|
|
1688
|
+
#
|
|
1689
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1690
|
+
# @param sql [String]
|
|
1691
|
+
# @return [void]
|
|
1692
|
+
def self.mysql_exec(conn, sql)
|
|
1693
|
+
conn.query(sql)
|
|
1694
|
+
nil
|
|
1695
|
+
end
|
|
1696
|
+
|
|
1697
|
+
# Result rows as string-keyed hashes. mysql2 yields hashes from `each`.
|
|
1698
|
+
# trilogy yields them from `each_hash`. Keys are downcased so aliases match.
|
|
1699
|
+
#
|
|
1700
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1701
|
+
# @param sql [String]
|
|
1702
|
+
# @return [Array<Hash>]
|
|
1703
|
+
def self.mysql_rows(conn, sql)
|
|
1704
|
+
result = conn.query(sql)
|
|
1705
|
+
return [] if result.nil?
|
|
1706
|
+
|
|
1707
|
+
rows = []
|
|
1708
|
+
if result.respond_to?(:each_hash)
|
|
1709
|
+
result.each_hash { |row| rows << stringify_mysql_row(row) }
|
|
1710
|
+
else
|
|
1711
|
+
result.each(as: :hash) { |row| rows << stringify_mysql_row(row) }
|
|
1712
|
+
end
|
|
1713
|
+
rows
|
|
1714
|
+
end
|
|
1715
|
+
|
|
1716
|
+
# @param row [Hash]
|
|
1717
|
+
# @return [Hash]
|
|
1718
|
+
def self.stringify_mysql_row(row)
|
|
1719
|
+
row.to_h.transform_keys { |key| key.to_s.downcase }
|
|
1720
|
+
end
|
|
1721
|
+
|
|
1722
|
+
# Quote a string for a SQL literal with the client the app loaded.
|
|
1723
|
+
#
|
|
1724
|
+
# @param conn [Mysql2::Client, Trilogy]
|
|
1725
|
+
# @param value [String]
|
|
1726
|
+
# @return [String]
|
|
1727
|
+
def self.mysql_quote_string(conn, value)
|
|
1728
|
+
"'#{conn.escape(value.to_s)}'"
|
|
1729
|
+
end
|
|
1730
|
+
|
|
1731
|
+
# Quote a MySQL identifier. Backticks inside the name are doubled.
|
|
1732
|
+
#
|
|
1733
|
+
# @param name [String]
|
|
1734
|
+
# @return [String]
|
|
1735
|
+
def self.quote_mysql_ident(name)
|
|
1736
|
+
raise ArgumentError, "database name is empty" if name.nil? || name.empty?
|
|
1737
|
+
raise ArgumentError, "database name contains a null" if name.include?("\0")
|
|
1738
|
+
|
|
1739
|
+
"`#{name.to_s.gsub('`', '``')}`"
|
|
1740
|
+
end
|
|
1741
|
+
|
|
1742
|
+
# True when `error` is already a provision failure this module raised.
|
|
1743
|
+
#
|
|
1744
|
+
# @param error [StandardError]
|
|
1745
|
+
# @return [Boolean]
|
|
1746
|
+
def self.provision_wrapped?(error)
|
|
1747
|
+
message = error.message
|
|
1748
|
+
message.start_with?("could not provision", "another Mutineer run", "refusing to drop")
|
|
1749
|
+
end
|
|
211
1750
|
end
|
|
212
1751
|
end
|