mutineer 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +680 -0
  3. data/README.md +210 -499
  4. data/lib/mutineer/baseline.rb +1 -1
  5. data/lib/mutineer/changed_lines.rb +2 -1
  6. data/lib/mutineer/child_coverage.rb +53 -0
  7. data/lib/mutineer/cli.rb +293 -42
  8. data/lib/mutineer/concern_blocks.rb +243 -0
  9. data/lib/mutineer/config.rb +154 -8
  10. data/lib/mutineer/coverage_map.rb +265 -27
  11. data/lib/mutineer/daemon_backend.rb +244 -75
  12. data/lib/mutineer/daemon_client.rb +73 -10
  13. data/lib/mutineer/daemon_server.rb +226 -42
  14. data/lib/mutineer/external_backend.rb +159 -15
  15. data/lib/mutineer/init.rb +124 -0
  16. data/lib/mutineer/isolation.rb +78 -24
  17. data/lib/mutineer/job_plan.rb +234 -44
  18. data/lib/mutineer/kill_channel.rb +24 -14
  19. data/lib/mutineer/migrate.rb +707 -0
  20. data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
  21. data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
  22. data/lib/mutineer/minitest_integration.rb +44 -14
  23. data/lib/mutineer/mutator_registry.rb +6 -2
  24. data/lib/mutineer/mutators/array_literal.rb +4 -2
  25. data/lib/mutineer/mutators/base.rb +36 -7
  26. data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
  27. data/lib/mutineer/mutators/chain_link.rb +6 -6
  28. data/lib/mutineer/mutators/condition_forcing.rb +6 -3
  29. data/lib/mutineer/mutators/operand_removal.rb +4 -2
  30. data/lib/mutineer/mutators/range_literal.rb +105 -4
  31. data/lib/mutineer/mutators/return_nil.rb +1 -0
  32. data/lib/mutineer/orphan_guard.rb +84 -0
  33. data/lib/mutineer/pairing.rb +62 -1
  34. data/lib/mutineer/project.rb +302 -14
  35. data/lib/mutineer/rails_run_env.rb +73 -0
  36. data/lib/mutineer/rails_worker_db.rb +1596 -57
  37. data/lib/mutineer/redaction.rb +199 -0
  38. data/lib/mutineer/reporter.rb +173 -31
  39. data/lib/mutineer/result.rb +52 -15
  40. data/lib/mutineer/runner.rb +148 -28
  41. data/lib/mutineer/statement_lines.rb +24 -11
  42. data/lib/mutineer/subject.rb +30 -4
  43. data/lib/mutineer/test_runners/minitest.rb +2 -1
  44. data/lib/mutineer/test_runners/rspec.rb +15 -3
  45. data/lib/mutineer/test_runners.rb +5 -2
  46. data/lib/mutineer/triage.rb +111 -0
  47. data/lib/mutineer/version.rb +1 -1
  48. data/lib/mutineer/worker_pool.rb +2 -1
  49. data/lib/mutineer.rb +4 -0
  50. data/skills/mutineer/SKILL.md +84 -0
  51. metadata +13 -1
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "digest"
4
+ require "uri"
5
+ require_relative "redaction"
4
6
 
5
7
  module Mutineer
6
8
  # Per-worker database isolation for the daemon path.
@@ -15,19 +17,87 @@ module Mutineer
15
17
  # Isolation model: each parallel worker gets its OWN database so concurrent
16
18
  # forks cannot clobber each other's transactional fixtures. {after_fork} runs
17
19
  # inside a freshly-forked child and points that child's connection at the
18
- # worker's database BEFORE any test loads; transactional fixtures then
19
- # repopulate that isolated database per test. On a slot's first use the worker
20
- # database starts as a copy of the base test database, so it holds what the
21
- # booted parent wrote, as the in-process backend sees it.
20
+ # worker's database BEFORE any test loads.
22
21
  #
23
- # Scope: SQLite adapter only (per-worker file, hermetic). Postgres per-worker
24
- # DBs (`CREATE DATABASE <db>-<worker>`) are not implemented yet; a non-SQLite
25
- # config raises a clear NotImplementedError rather than silently mis-routing.
22
+ # SQLite copies the base file on a slot's first use ({after_fork} with
23
+ # `seed: true`). Postgres and MySQL copy once, before any worker daemon starts
24
+ # ({provision}). A later fork only connects. Both servers roll back a killed
25
+ # fork's open transaction, so the SQLite re-seed after `error` or `timeout` is
26
+ # skipped. Rows a test commits outside a transaction stay in the slot.
26
27
  #
27
- # Routing failures surface as `error` via {verify_connection!}. Tagging an
28
- # in-test DB failure as `error` (not `killed`) is only observable under
29
- # concurrent load and is not yet implemented.
28
+ # MySQL has no template database. Each slot is created empty, the app schema
29
+ # is loaded with ActiveRecord::Tasks::DatabaseTasks, and rows are copied
30
+ # table by table. `mysql2` and `trilogy` are accepted. Any other adapter
31
+ # raises NotImplementedError rather than routing at a database that does not exist.
30
32
  module RailsWorkerDb
33
+ # Postgres cuts an unquoted identifier at this many bytes.
34
+ POSTGRES_NAME_LIMIT = 63
35
+ # MySQL rejects a database name longer than this many bytes. The shortening
36
+ # rule matches Postgres. A byte limit stays inside the 64-character limit.
37
+ MYSQL_NAME_LIMIT = 64
38
+ # How many worker daemons can hold a lock at once. MySQL GET_LOCK is
39
+ # exclusive, so each daemon takes one name from this pool. A separate guard
40
+ # name covers the handoff and is not one of these slots.
41
+ MYSQL_LOCK_SLOTS = 256
42
+ # Actions information_schema may report for a foreign key. Anything else is
43
+ # rejected so a bad row cannot change the ALTER TABLE statement.
44
+ MYSQL_REFERENTIAL_ACTIONS = ["CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"].freeze
45
+ # TLS and timeout keys from the app's MySQL config that reach the
46
+ # maintenance client unchanged. ActiveRecord passes them to the client the
47
+ # same way. The first group is mysql2's, the second trilogy's. Each client
48
+ # ignores the other's keys.
49
+ MYSQL_CLIENT_KEYS = %i[
50
+ connect_timeout read_timeout write_timeout
51
+ ssl_mode sslkey sslcert sslca sslcapath sslcipher sslverify get_server_public_key
52
+ ssl ssl_ca ssl_capath ssl_cert ssl_key ssl_cipher ssl_crl ssl_crlpath
53
+ tls_ciphersuites tls_min_version tls_max_version
54
+ ].freeze
55
+ # Infix that keeps a worker database off Rails' own `<base>-<n>` names.
56
+ WORKER_NAME_MARK = "-mutineer-"
57
+ # Database we connect to while the test database is a template. Advisory
58
+ # locks are per database, so every run must take them here, not on a slot.
59
+ MAINTENANCE_DATABASE = "postgres"
60
+ # Server catalogs a worker copy must never drop, even if a name bug
61
+ # produced one of them.
62
+ RESERVED_DATABASES = %w[postgres template0 template1].freeze
63
+ # Rails keys copied onto the maintenance `PG::Connection`. `database` is
64
+ # not here: that session always uses {MAINTENANCE_DATABASE}. `username` is
65
+ # Rails' spelling of libpq `user`. Anything else (adapter, variables,
66
+ # schema_search_path) is a Rails setting and would make libpq reject the
67
+ # connection.
68
+ LIBPQ_CONNECTION_KEYS = {
69
+ host: :host,
70
+ hostaddr: :hostaddr,
71
+ port: :port,
72
+ user: :user,
73
+ username: :user,
74
+ password: :password,
75
+ service: :service,
76
+ sslmode: :sslmode,
77
+ sslcert: :sslcert,
78
+ sslkey: :sslkey,
79
+ sslrootcert: :sslrootcert,
80
+ sslcrl: :sslcrl,
81
+ sslcompression: :sslcompression,
82
+ ssl_min_protocol_version: :ssl_min_protocol_version,
83
+ ssl_max_protocol_version: :ssl_max_protocol_version,
84
+ connect_timeout: :connect_timeout,
85
+ client_encoding: :client_encoding,
86
+ options: :options,
87
+ application_name: :application_name,
88
+ fallback_application_name: :fallback_application_name,
89
+ keepalives: :keepalives,
90
+ keepalives_idle: :keepalives_idle,
91
+ keepalives_interval: :keepalives_interval,
92
+ keepalives_count: :keepalives_count,
93
+ channel_binding: :channel_binding,
94
+ gssencmode: :gssencmode,
95
+ krbsrvname: :krbsrvname,
96
+ gsslib: :gsslib,
97
+ requiressl: :requiressl,
98
+ target_session_attrs: :target_session_attrs
99
+ }.freeze
100
+
31
101
  # True when the app has ActiveRecord loaded. The only condition under which
32
102
  # any other method here may touch AR. Never triggers an autoload/require of
33
103
  # AR itself.
@@ -37,10 +107,9 @@ module Mutineer
37
107
  defined?(ActiveRecord::Base) ? true : false
38
108
  end
39
109
 
40
- # Derive a per-worker database path from a base path by inserting `-<worker>`
41
- # before the extension. Pure string transform (no AR) so it is unit-testable
42
- # in the zero-dep suite. `storage/test.sqlite3`, worker 1 ->
43
- # `storage/test-1.sqlite3`.
110
+ # Derive a per-worker SQLite path by inserting `-<worker>` before the
111
+ # extension. Pure string transform (no AR). `storage/test.sqlite3`, worker
112
+ # 1 -> `storage/test-1.sqlite3`.
44
113
  #
45
114
  # @param database [String] the base database path.
46
115
  # @param worker [Integer] the worker slot index (0..N-1).
@@ -50,73 +119,294 @@ module Mutineer
50
119
  "#{database.delete_suffix(ext)}-#{worker}#{ext}"
51
120
  end
52
121
 
53
- # Build the AR connection config for one worker by copying the app's current
54
- # (default test) config and swapping in the per-worker database path. SQLite
55
- # only this pass: a non-SQLite adapter raises so the SQLite-first scope fails
56
- # loud instead of mis-routing.
122
+ # Worker database name: `<base>-mutineer-<slot>`. When that is longer than
123
+ # `limit` bytes, the base is shortened on a character boundary and a hash
124
+ # of the full base is added so two slots, and two long bases that share a
125
+ # prefix, stay distinct.
126
+ #
127
+ # @param base [String] the test database name.
128
+ # @param slot [Integer] the worker slot index.
129
+ # @param limit [Integer] maximum name length in bytes.
130
+ # @return [String] a name of at most `limit` bytes, or the hash suffix
131
+ # alone when that suffix itself exceeds the limit.
132
+ def self.worker_database_name(base, slot, limit)
133
+ suffix = "#{WORKER_NAME_MARK}#{slot}"
134
+ full = "#{base}#{suffix}"
135
+ return full if full.bytesize <= limit
136
+
137
+ hash = Digest::SHA256.hexdigest(base)[0, 8]
138
+ tail = "-#{hash}#{suffix}"
139
+ room = limit - tail.bytesize
140
+ "#{utf8_prefix(base, room)}#{tail}"
141
+ end
142
+
143
+ # Postgres worker database name. See {worker_database_name}.
144
+ #
145
+ # @param base [String] the test database name.
146
+ # @param slot [Integer] the worker slot index.
147
+ # @return [String] a name of at most {POSTGRES_NAME_LIMIT} bytes.
148
+ def self.postgres_worker_database(base, slot)
149
+ worker_database_name(base, slot, POSTGRES_NAME_LIMIT)
150
+ end
151
+
152
+ # MySQL worker database name. See {worker_database_name}.
153
+ #
154
+ # @param base [String] the test database name.
155
+ # @param slot [Integer] the worker slot index.
156
+ # @return [String] a name of at most {MYSQL_NAME_LIMIT} bytes.
157
+ def self.mysql_worker_database(base, slot)
158
+ worker_database_name(base, slot, MYSQL_NAME_LIMIT)
159
+ end
160
+
161
+ # Leading characters of `text` that fit in `max_bytes`. Stops before a
162
+ # character that would cross the limit, so a multibyte name stays valid
163
+ # text. PostgreSQL and MySQL count identifier length in bytes.
164
+ #
165
+ # @param text [String]
166
+ # @param max_bytes [Integer]
167
+ # @return [String]
168
+ def self.utf8_prefix(text, max_bytes)
169
+ return "" if max_bytes <= 0
170
+
171
+ source = text.to_s
172
+ return source if source.bytesize <= max_bytes
173
+
174
+ kept = +""
175
+ source.each_char do |char|
176
+ break if kept.bytesize + char.bytesize > max_bytes
177
+
178
+ kept << char
179
+ end
180
+ kept
181
+ end
182
+
183
+ # True when `name` is one of this run's exact worker database names.
184
+ # Equality only: a SQL `LIKE` would treat `_` in the base as a wildcard,
185
+ # so `my_app_test-mutineer-1` must not match base `myXappXtest`.
186
+ # `limit` defaults to the Postgres limit so existing callers stay valid.
187
+ #
188
+ # @param base [String] the test database name.
189
+ # @param name [String] a database name we might drop.
190
+ # @param slots [Integer] how many worker slots this run owns.
191
+ # @param limit [Integer] name limit in bytes. {POSTGRES_NAME_LIMIT} or {MYSQL_NAME_LIMIT}.
192
+ # @return [Boolean]
193
+ def self.owned_database?(base, name, slots:, limit: POSTGRES_NAME_LIMIT)
194
+ (0...slots).any? { |slot| worker_database_name(base, slot, limit) == name }
195
+ end
196
+
197
+ # True when `name` is a database this tool may drop. It must be the exact
198
+ # name {postgres_worker_database} would build for `base` and the slot
199
+ # written at the end of `name`. The source database, a server catalog, and
200
+ # any other name are refused.
201
+ #
202
+ # @param base [String] the test database name.
203
+ # @param name [String] a database name we might drop.
204
+ # @return [Boolean]
205
+ def self.worker_database_name?(base, name)
206
+ return false if base.nil? || name.nil?
207
+
208
+ text = name.to_s
209
+ source = base.to_s
210
+ return false if text.empty? || text == source
211
+ return false if RESERVED_DATABASES.include?(text)
212
+
213
+ slot = text[/#{Regexp.escape(WORKER_NAME_MARK)}(\d+)\z/, 1]
214
+ return false unless slot
215
+
216
+ postgres_worker_database(source, Integer(slot, 10)) == text
217
+ end
218
+
219
+ # Build the AR connection config for one worker from the app's current
220
+ # test config. SQLite, Postgres, and MySQL (mysql2 or trilogy) get a
221
+ # per-worker database name. Anything else raises.
57
222
  #
58
223
  # @param worker [Integer] the worker slot index.
59
224
  # @return [Hash] a symbol-keyed AR configuration hash for the worker database.
60
- # @raise [NotImplementedError] when the app's database is non-SQLite or in-memory.
225
+ # @raise [NotImplementedError] when the adapter has no worker database yet, or the database is in-memory.
61
226
  def self.worker_db_config(worker)
62
- hash = ActiveRecord::Base.connection_db_config.configuration_hash
63
- adapter = hash[:adapter].to_s
64
- # Config shaping (per_worker_config) is already adapter-general: it derives
65
- # correct SQLite and Postgres worker-DB names. What is gated is runtime
66
- # provisioning: SQLite files are created on connect, but Postgres needs an
67
- # explicit `CREATE DATABASE` per worker. Until that lands, refuse non-SQLite
68
- # loudly rather than route to a database that does not exist.
69
- unless adapter.start_with?("sqlite")
70
- raise NotImplementedError,
71
- "worker-DB isolation currently provisions SQLite only (got adapter #{adapter.inspect}); " \
72
- "Postgres per-worker provisioning is not yet supported. Use a SQLite test DB with --daemon, or drop --daemon to run serially."
73
- end
227
+ per_worker_config(routing_config_hash, worker)
228
+ end
74
229
 
75
- per_worker_config(hash, worker)
230
+ # Config used to name a worker database. MySQL schema load points
231
+ # ActiveRecord at the last slot. Coverage forks must keep the original test
232
+ # database, or the next name is nested and was never created.
233
+ #
234
+ # @return [Hash]
235
+ def self.routing_config_hash
236
+ @mysql_base_config || current_config_hash
237
+ end
238
+
239
+ # Remember the test database config before schema load switches the connection.
240
+ #
241
+ # @param config_hash [Hash]
242
+ # @return [void]
243
+ def self.remember_mysql_base_config(config_hash)
244
+ @mysql_base_config = config_hash.transform_keys(&:to_sym)
245
+ nil
76
246
  end
77
247
 
78
- # Pure config-shaping (no AR): given a connection config hash, return the
79
- # per-worker variant with its database swapped to the worker's own name.
80
- # Adapter-general: SQLite (`storage/test.sqlite3` -> `storage/test-<w>.sqlite3`)
81
- # and Postgres (`myapp_test` -> `myapp_test-<w>`, Rails `parallelize` naming)
82
- # both fall out of {worker_database_path}. Extracted and unit-tested so the
83
- # Postgres shape is proven ready without a live database.
248
+ # Drop the remembered MySQL test config. The next route reads ActiveRecord again.
249
+ #
250
+ # @return [void]
251
+ def self.forget_mysql_base_config
252
+ @mysql_base_config = nil
253
+ end
254
+
255
+ # Pure config shaping (no AR): swap in the worker database name.
256
+ # SQLite keeps {worker_database_path}. Postgres uses {postgres_worker_database}.
257
+ # MySQL (`mysql2` or `trilogy`) uses {mysql_worker_database}.
84
258
  #
85
259
  # @param config_hash [Hash] a connection config hash (symbol or string keys).
86
260
  # @param worker [Integer] the worker slot index.
87
261
  # @return [Hash] the per-worker config (symbol keys), database swapped.
88
- # @raise [NotImplementedError] for an in-memory or empty database (no per-worker split).
262
+ # @raise [NotImplementedError] for an in-memory database, an empty name, or an adapter other than SQLite, Postgres, or MySQL.
89
263
  def self.per_worker_config(config_hash, worker)
90
264
  hash = config_hash.transform_keys(&:to_sym)
265
+ adapter = hash[:adapter].to_s
91
266
  database = hash[:database].to_s
267
+ if database.empty? && postgres_adapter?(adapter)
268
+ database = database_name_from_url(hash[:url])
269
+ end
92
270
  if database.empty? || database == ":memory:"
93
271
  raise NotImplementedError,
94
- "worker-DB isolation needs a file/name-backed database (got #{database.inspect})."
272
+ "worker-DB isolation needs a file or named database (got #{database.inspect})."
273
+ end
274
+
275
+ name =
276
+ if sqlite_adapter?(adapter)
277
+ worker_database_path(database, worker)
278
+ elsif postgres_adapter?(adapter)
279
+ postgres_worker_database(database, worker)
280
+ elsif mysql_adapter?(adapter)
281
+ mysql_worker_database(database, worker)
282
+ else
283
+ raise NotImplementedError,
284
+ "worker-DB isolation does not support adapter #{adapter.inspect} yet."
285
+ end
286
+ hash.merge(database: name)
287
+ end
288
+
289
+ # Create each worker database before any worker daemon starts.
290
+ # SQLite is a no-op: the file is copied on first use. Postgres drops and
291
+ # recreates each exact slot name from the test database. MySQL creates each
292
+ # slot, loads the schema, and copies rows. The caller must be the only
293
+ # daemon connected to that test database.
294
+ #
295
+ # @param slots [Integer] how many worker slots to create (0..slots-1).
296
+ # @return [void]
297
+ # @raise [NotImplementedError] for an adapter other than SQLite, Postgres, or MySQL.
298
+ # @raise [RuntimeError] when the copy cannot be made, or another run holds the lock.
299
+ def self.provision(slots)
300
+ return unless available?
301
+
302
+ count = Integer(slots)
303
+ return if count < 1
304
+ return if sqlite_adapter?(current_adapter)
305
+
306
+ if postgres_adapter?(current_adapter)
307
+ provision_postgres(count)
308
+ return
309
+ end
310
+ if mysql_adapter?(current_adapter)
311
+ provision_mysql(count)
312
+ return
95
313
  end
96
314
 
97
- hash.merge(database: worker_database_path(database, worker))
315
+ raise NotImplementedError,
316
+ "worker-DB isolation does not support adapter #{current_adapter.inspect} yet."
317
+ end
318
+
319
+ # Take the shared run lock and drop this process's connection to the test
320
+ # database. Worker daemons call this at boot. They do not create databases.
321
+ # SQLite has no lock here. The file copy still happens in {after_fork}.
322
+ #
323
+ # @return [void]
324
+ # @raise [RuntimeError] when another run holds the exclusive lock.
325
+ def self.prepare_worker!
326
+ return unless available?
327
+ return if sqlite_adapter?(current_adapter)
328
+
329
+ if mysql_adapter?(current_adapter)
330
+ prepare_mysql_worker!
331
+ return
332
+ end
333
+ return unless postgres_adapter?(current_adapter)
334
+
335
+ hash = current_config_hash
336
+ base = source_database_name!(hash)
337
+ conn = open_maintenance_connection(hash, base)
338
+ unless try_lock(conn, base, shared: true)
339
+ conn.close
340
+ raise "another Mutineer run is using #{base}"
341
+ end
342
+
343
+ @lock_connection = conn
344
+ disconnect_app!
345
+ end
346
+
347
+ # Point this process at one worker database. Postgres and MySQL only connect.
348
+ # The slot was copied at {provision}.
349
+ #
350
+ # @param slot [Integer] the worker slot index.
351
+ # @return [void]
352
+ def self.connect(slot)
353
+ ActiveRecord::Base.establish_connection(worker_db_config(slot))
354
+ end
355
+
356
+ # The database config names the test environment declares, and the one
357
+ # each worker copies (the config `ActiveRecord::Base` uses). The tool
358
+ # counts the names that `shared_databases` does not list: more than one
359
+ # means the run must use a single worker.
360
+ #
361
+ # A config with `database_tasks: false` counts: workers would share it like
362
+ # any other. A replica does not, as before: it reads another config's database.
363
+ # No rescue: a failure must reach the tool, which then keeps one worker.
364
+ #
365
+ # @return [Hash{String => Object}] "names" (Array<String>) and "primary" (String).
366
+ def self.database_names
367
+ names = test_configs.reject(&:replica?).map(&:name)
368
+ { "names" => names, "primary" => primary_database_name }
369
+ end
370
+
371
+ # The config name each worker copies: the one `ActiveRecord::Base` uses.
372
+ #
373
+ # @return [String]
374
+ def self.primary_database_name
375
+ ActiveRecord::Base.connection_db_config.name
376
+ end
377
+
378
+ # The test environment's database configs, hidden ones and replicas included.
379
+ #
380
+ # @return [Array<ActiveRecord::DatabaseConfigurations::DatabaseConfig>]
381
+ def self.test_configs
382
+ env = ENV["RAILS_ENV"].to_s
383
+ env = "test" if env.empty?
384
+ ActiveRecord::Base.configurations.configs_for(env_name: env, include_hidden: true)
98
385
  end
99
386
 
100
387
  # Child-side (after fork): route this process's ActiveRecord at the worker's
101
388
  # own database and confirm it is reachable, so a routing failure reads as
102
- # `error` (via the daemon's child rescue) rather than a false verdict.
389
+ # `error` rather than a false verdict.
103
390
  #
104
- # With `seed: true` (the slot's first use) the worker database first becomes
105
- # a copy of the base test database, schema and rows, so rows the daemon
106
- # parent wrote while it booted (initializers, `--require` files) are there,
107
- # as they are for the in-process backend (#222). The schema is then loaded
108
- # only when the copy differs from `schema.rb` ({schema_current?}: schema
109
- # version or stored `schema_sha1`, as in a stale or empty base database):
110
- # `schema.rb` runs with `force: true`, which drops the copied rows of the
111
- # tables it defines.
391
+ # SQLite with `seed: true` copies the base file first, then loads `schema.rb`
392
+ # only when the copy differs from that file. Postgres and MySQL ignore `seed`
393
+ # and the schema path: the slot was copied at {provision}, and loading
394
+ # `schema.rb` again would drop those rows.
112
395
  #
113
396
  # @param worker [Integer] the worker slot index.
114
- # @param schema_path [String, nil] absolute path to `db/schema.rb`, or nil to skip.
115
- # @param seed [Boolean] copy the base database into the worker database first.
397
+ # @param schema_path [String, nil] the app's Ruby schema file ({ruby_schema_path}), or nil to skip.
398
+ # @param seed [Boolean] copy the base SQLite file into the worker file first.
116
399
  # @return [void]
117
400
  def self.after_fork(worker, schema_path = nil, seed: false)
118
401
  return unless available?
119
402
 
403
+ forget_lock_connection_after_fork!
404
+ if postgres_adapter?(current_adapter) || mysql_adapter?(current_adapter)
405
+ connect(worker)
406
+ verify_connection!
407
+ return
408
+ end
409
+
120
410
  config = worker_db_config(worker)
121
411
  base = ActiveRecord::Base.connection.select_value("SELECT file FROM pragma_database_list WHERE name = 'main'") if seed
122
412
  ActiveRecord::Base.establish_connection(config)
@@ -159,10 +449,10 @@ module Mutineer
159
449
  # file, which catches an edited schema with the same version. No stored
160
450
  # checksum counts as out of date, as in Rails' own `schema_up_to_date?`.
161
451
  #
162
- # @param schema_path [String] absolute path to `db/schema.rb`.
452
+ # @param schema_path [String] absolute path to the app's Ruby schema file.
163
453
  # @return [Boolean]
164
454
  def self.schema_current?(schema_path)
165
- text = File.read(schema_path)
455
+ text = File.read(schema_path, encoding: "UTF-8")
166
456
  version = schema_file_version(text)
167
457
  conn = ActiveRecord::Base.connection
168
458
  base = ActiveRecord::Base
@@ -185,10 +475,91 @@ module Mutineer
185
475
  text[/define\(version:\s*([\d_]+)/, 1]&.delete("_")&.to_i
186
476
  end
187
477
 
478
+ # Record each pool that runs SQL in this fork from now on. Run in the fork
479
+ # before its tests, so {touched_shared_databases} also sees a pool that a
480
+ # test disconnects or removes before it ends.
481
+ #
482
+ # @return [void]
483
+ def self.watch_sql
484
+ @sql_pools = {} # used as a set: one entry per pool, not per query
485
+ ActiveSupport::Notifications.subscribe("sql.active_record") do |*, payload|
486
+ connection = payload[:connection]
487
+ @sql_pools[connection.pool] = true if connection.respond_to?(:pool)
488
+ end
489
+ end
490
+
491
+ # Names from `shared` whose database this fork used. Run in a fork after
492
+ # its tests: a name here means a test used a database that parallel
493
+ # workers share, so the verdict cannot be trusted. A pool counts when it
494
+ # is still connected, or when it ran SQL since {watch_sql}. It names a
495
+ # shared database when its config has a shared name, when its class also
496
+ # holds a shared writer pool (a replica in the same `connects_to`), or when
497
+ # its config names the same database as a shared config. Transactional
498
+ # tests connect every registered writing pool, so a pool registered for a
499
+ # shared database shows up here after any test.
500
+ #
501
+ # Only names in {database_names} count: the tool warned about any other
502
+ # name, and no worker shares a database that does not exist.
503
+ #
504
+ # No rescue: a failed check must raise (scored `error`), never read as clean.
505
+ # shortcut: a connection made outside Active Record (a raw driver) is not
506
+ # seen; upgrade when such an app needs `shared_databases`.
507
+ #
508
+ # @param shared [Array<String>] database config names in `shared_databases`.
509
+ # @return [Array<String>]
510
+ def self.touched_shared_databases(shared)
511
+ return [] if shared.empty? || !available?
512
+
513
+ shared &= database_names["names"]
514
+ return [] if shared.empty?
515
+ # Rails 7.0 reads `:all` below as a role name and lists no pool, so the
516
+ # check would always pass. Fail every mutant instead of trusting it.
517
+ raise "shared_databases needs Rails 7.1 or later" if ActiveRecord.version < Gem::Version.new("7.1")
518
+
519
+ pools = ActiveRecord::Base.connection_handler.connection_pool_list(:all)
520
+ owners = pools.filter_map { |p| [pool_owner(p), p.db_config.name] if shared.include?(p.db_config.name) }.to_h
521
+ by_database = test_configs.filter_map { |c| [c.database, c.name] if shared.include?(c.name) && c.database }.to_h
522
+ used = pools.select(&:connected?) | (@sql_pools&.keys || [])
523
+ used.filter_map do |pool|
524
+ config = pool.db_config
525
+ shared.include?(config.name) ? config.name : owners[pool_owner(pool)] || by_database[config.database]
526
+ end.uniq
527
+ end
528
+
529
+ # The class a pool belongs to, by name. `connects_to` registers a class's
530
+ # writer and replica pools under it. Rails 8 keeps it in a descriptor.
531
+ #
532
+ # @param pool [ActiveRecord::ConnectionAdapters::ConnectionPool]
533
+ # @return [String]
534
+ def self.pool_owner(pool)
535
+ config = pool.pool_config
536
+ config.respond_to?(:connection_descriptor) ? config.connection_descriptor.name : config.connection_name
537
+ end
538
+
539
+ # The Ruby schema file a SQLite worker loads over an out-of-date copy, as
540
+ # the booted app's config names it. Rails resolves the name: `db/schema.rb`
541
+ # by default, the `schema_dump:` file of the database config (a bare name
542
+ # such as `schema_sqlite.rb` lives in `db/`), or the `SCHEMA` variable.
543
+ # nil when the file does not exist, when the config sets `schema_dump: false`,
544
+ # or when the app uses `structure.sql` (a worker cannot `load` SQL). With nil
545
+ # the worker keeps its copy of the test database as it is.
546
+ #
547
+ # @return [String, nil] absolute path of an existing file, or nil.
548
+ def self.ruby_schema_path
549
+ db_config = ActiveRecord::Base.connection_db_config
550
+ format = schema_format_for(db_config)
551
+ return unless format == :ruby
552
+
553
+ path = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)
554
+ # Expanded so `load` never searches $LOAD_PATH for a relative SCHEMA value.
555
+ path &&= File.expand_path(path)
556
+ path if path && File.exist?(path)
557
+ end
558
+
188
559
  # Load a Rails `schema.rb` into the current connection with output silenced
189
560
  # (fork child stdout is already File::NULL; this is belt-and-braces).
190
561
  #
191
- # @param schema_path [String] absolute path to `db/schema.rb`.
562
+ # @param schema_path [String] absolute path to the app's Ruby schema file.
192
563
  # @return [void]
193
564
  def self.load_schema(schema_path)
194
565
  ActiveRecord::Migration.verbose = false if defined?(ActiveRecord::Migration)
@@ -201,12 +572,1180 @@ module Mutineer
201
572
  end
202
573
 
203
574
  # Force a round-trip to the freshly-routed connection so a broken route fails
204
- # HERE (→ `error`) instead of later masquerading as a test failure
205
- # (→ false `killed`).
575
+ # HERE (then `error`) instead of later masquerading as a test failure
576
+ # (then a false `killed`).
206
577
  #
207
578
  # @return [void]
208
579
  def self.verify_connection!
209
580
  ActiveRecord::Base.connection.execute("SELECT 1")
210
581
  end
582
+
583
+ # Symbol-keyed connection hash for the database the app booted. Does not
584
+ # open a new connection.
585
+ #
586
+ # @return [Hash]
587
+ def self.current_config_hash
588
+ ActiveRecord::Base.connection_db_config.configuration_hash.transform_keys(&:to_sym)
589
+ end
590
+
591
+ # Adapter name from {current_config_hash}.
592
+ #
593
+ # @return [String]
594
+ def self.current_adapter
595
+ current_config_hash[:adapter].to_s
596
+ end
597
+
598
+ # @param adapter [String]
599
+ # @return [Boolean]
600
+ def self.sqlite_adapter?(adapter)
601
+ adapter.start_with?("sqlite")
602
+ end
603
+
604
+ # @param adapter [String]
605
+ # @return [Boolean]
606
+ def self.postgres_adapter?(adapter)
607
+ adapter == "postgres" || adapter.start_with?("postgres")
608
+ end
609
+
610
+ # True for the mysql2 and trilogy adapters. The old `mysql` gem is not included.
611
+ #
612
+ # @param adapter [String]
613
+ # @return [Boolean]
614
+ def self.mysql_adapter?(adapter)
615
+ adapter == "mysql2" || adapter == "trilogy"
616
+ end
617
+
618
+ # Classify a MySQL `SELECT VERSION()` text against the support matrix.
619
+ # MariaDB is matched by name first: it reports 10.x or 11.x, and some
620
+ # builds prefix `5.5.5-`, so a numeric check alone would misread it.
621
+ #
622
+ # @param version [String, nil]
623
+ # @return [Symbol] `:supported` (MySQL 8), `:not_supported` (before 8),
624
+ # `:not_tested` (MariaDB, MySQL 9 or later), or `:unreadable`.
625
+ def self.mysql_version_support(version)
626
+ text = version.to_s
627
+ return :not_tested if text.match?(/mariadb/i)
628
+
629
+ major = text[/\A(\d+)\.\d+/, 1]
630
+ return :unreadable unless major
631
+
632
+ major = major.to_i
633
+ return :not_supported if major < 8
634
+ return :not_tested if major > 8
635
+
636
+ :supported
637
+ end
638
+
639
+ # Drop and recreate slots `0...count` from the booted test database.
640
+ # Holds the exclusive advisory lock only for the copy, then a shared lock
641
+ # for the rest of this process so a second run cannot drop those databases.
642
+ #
643
+ # @param count [Integer]
644
+ # @return [void]
645
+ def self.provision_postgres(count)
646
+ hash = current_config_hash
647
+ base = source_database_name!(hash)
648
+ conn = open_maintenance_connection(hash, base)
649
+ unless try_lock(conn, base, shared: false)
650
+ conn.close
651
+ raise "another Mutineer run is using #{base}"
652
+ end
653
+
654
+ begin
655
+ disconnect_app!
656
+ count.times do |slot|
657
+ name = postgres_worker_database(base, slot)
658
+ recreate_from_template(conn, name, base)
659
+ end
660
+ rescue StandardError => e
661
+ unlock(conn, base)
662
+ conn.close
663
+ raise provision_wrapped?(e) ? e : provision_failure(base, base, e.message)
664
+ end
665
+
666
+ unlock(conn, base)
667
+ unless try_lock(conn, base, shared: true)
668
+ conn.close
669
+ raise "another Mutineer run is using #{base}"
670
+ end
671
+ @lock_connection = conn
672
+ warn "[mutineer] provisioned #{count} postgres worker database(s) from #{base}"
673
+ end
674
+
675
+ # Open a session on {MAINTENANCE_DATABASE} with the app's Postgres credentials.
676
+ # Uses the `pg` client the app already loaded. The gem does not depend on it.
677
+ # Endpoint options from the Rails config, including `service` and `url`, are
678
+ # kept. Only the database name changes, to {MAINTENANCE_DATABASE}.
679
+ #
680
+ # @param config_hash [Hash]
681
+ # @param base [String] test database name, named in the error if this fails.
682
+ # @return [PG::Connection]
683
+ def self.open_maintenance_connection(config_hash, base)
684
+ unless defined?(PG::Connection)
685
+ raise "could not provision #{base}: the app's pg client is not loaded"
686
+ end
687
+
688
+ PG::Connection.open(maintenance_connection_options(config_hash))
689
+ rescue StandardError => e
690
+ raise e if e.message.start_with?("could not provision")
691
+
692
+ raise "could not provision #{base}: #{e.message}"
693
+ end
694
+
695
+ # libpq options for the maintenance session. A Rails `url:` fills anything
696
+ # the hash does not set itself. Explicit keys win. `dbname` is always
697
+ # {MAINTENANCE_DATABASE}, never the test database and never a name taken
698
+ # from the URL path.
699
+ #
700
+ # @param config_hash [Hash] symbol or string keys, as Rails stores them.
701
+ # @return [Hash] symbol keys for `PG::Connection.open`.
702
+ # @raise [RuntimeError] when `url:` is present and is not a Postgres URL.
703
+ def self.maintenance_connection_options(config_hash)
704
+ hash = config_hash.transform_keys(&:to_sym)
705
+ opts = options_from_database_url(hash[:url])
706
+ opts.delete(:database)
707
+ LIBPQ_CONNECTION_KEYS.each do |config_key, pg_key|
708
+ next unless hash.key?(config_key)
709
+
710
+ value = hash[config_key]
711
+ next if blank_option?(value)
712
+
713
+ opts[pg_key] = value
714
+ end
715
+ opts[:dbname] = MAINTENANCE_DATABASE
716
+ opts
717
+ end
718
+
719
+ # The test database this run copies from. `database:` wins. A Postgres URL
720
+ # supplies the name when the hash has none. A `service:` entry does not
721
+ # carry the name, so that shape fails here instead of guessing.
722
+ #
723
+ # @param config_hash [Hash]
724
+ # @return [String]
725
+ # @raise [RuntimeError] when no database name is available.
726
+ def self.source_database_name!(config_hash)
727
+ hash = config_hash.transform_keys(&:to_sym)
728
+ name = hash[:database].to_s
729
+ name = database_name_from_url(hash[:url]) if name.empty?
730
+ return name unless name.empty?
731
+
732
+ raise "could not provision: the test database name is missing. " \
733
+ "Set database: or use a URL that includes the database name."
734
+ end
735
+
736
+ # Database name from a postgres URL path, or "" when `url` is blank.
737
+ #
738
+ # @param url [String, nil]
739
+ # @return [String]
740
+ # @raise [RuntimeError] when `url` is present and is not a Postgres URL.
741
+ def self.database_name_from_url(url)
742
+ options_from_database_url(url)[:database].to_s
743
+ end
744
+
745
+ # Host, credentials, database name, and query options from a Rails `url:`.
746
+ # Blank when `url` is blank. The database is returned under `:database`
747
+ # so callers can separate it from the maintenance `dbname`.
748
+ #
749
+ # @param url [String, nil]
750
+ # @return [Hash]
751
+ # @raise [RuntimeError] when `url` is present and is not a Postgres URL.
752
+ def self.options_from_database_url(url)
753
+ return {} if blank_option?(url)
754
+
755
+ uri = URI.parse(url.to_s)
756
+ scheme = uri.scheme.to_s.downcase
757
+ unless scheme == "postgres" || scheme == "postgresql"
758
+ raise "could not provision: the database URL must start with postgres:// or postgresql://"
759
+ end
760
+
761
+ opts = {}
762
+ host = uri.host
763
+ opts[:host] = host if host && !host.empty?
764
+ opts[:port] = uri.port if uri.port
765
+ user = decode_uri_component(uri.user)
766
+ opts[:user] = user if user && !user.empty?
767
+ password = decode_uri_component(uri.password)
768
+ opts[:password] = password if password && !password.empty?
769
+ database = uri.path.to_s.sub(%r{\A/}, "")
770
+ database = decode_uri_component(database) unless database.empty?
771
+ opts[:database] = database if database && !database.empty?
772
+ URI.decode_www_form(uri.query.to_s).each do |key, value|
773
+ pg_key = LIBPQ_CONNECTION_KEYS[key.to_sym]
774
+ next unless pg_key
775
+ next if blank_option?(value)
776
+
777
+ opts[pg_key] = value
778
+ end
779
+ opts
780
+ rescue URI::InvalidURIError, URI::InvalidComponentError, ArgumentError
781
+ # The parser's message repeats the URL, password included. `cause: nil`
782
+ # keeps Ruby from printing that original error under this one.
783
+ raise "could not provision: the database URL is not valid. " \
784
+ "The URL is not shown because it can contain a password. " \
785
+ "Check each part of DATABASE_URL or url: in config/database.yml. " \
786
+ "Percent-encode the user name, password and database name. Encode each " \
787
+ "character that is not an ASCII letter, a digit, -, ., _ or ~. " \
788
+ "For example, write % as %25, @ as %40 and a space as %20.", cause: nil
789
+ end
790
+
791
+ # Percent-decode one URL component. Nil when `text` is blank.
792
+ #
793
+ # @param text [String, nil]
794
+ # @return [String, nil]
795
+ def self.decode_uri_component(text)
796
+ return if blank_option?(text)
797
+
798
+ # `unescape` keeps a literal `+`. The form decoder would turn it into a space.
799
+ # The daemon runs on the app's Ruby, which may be older than 3.4.
800
+ URI::DEFAULT_PARSER.unescape(text.to_s)
801
+ end
802
+
803
+ # True for nil and empty strings. Numbers and false are present values.
804
+ #
805
+ # @param value [Object]
806
+ # @return [Boolean]
807
+ def self.blank_option?(value)
808
+ value.nil? || (value.respond_to?(:empty?) && value.empty?)
809
+ end
810
+
811
+ # @param conn [PG::Connection]
812
+ # @param base [String]
813
+ # @param shared [Boolean] shared lock when true, exclusive when false.
814
+ # @return [Boolean]
815
+ def self.try_lock(conn, base, shared:)
816
+ k1, k2 = lock_keys(base)
817
+ sql = shared ? "SELECT pg_try_advisory_lock_shared($1, $2)" : "SELECT pg_try_advisory_lock($1, $2)"
818
+ conn.exec_params(sql, [k1, k2]).getvalue(0, 0) == "t"
819
+ end
820
+
821
+ # Release one advisory lock this session holds for `base`.
822
+ #
823
+ # @param conn [PG::Connection]
824
+ # @param base [String]
825
+ # @return [void]
826
+ def self.unlock(conn, base)
827
+ k1, k2 = lock_keys(base)
828
+ conn.exec_params("SELECT pg_advisory_unlock($1, $2)", [k1, k2])
829
+ end
830
+
831
+ # Two signed 32-bit keys for `pg_try_advisory_lock(int, int)`, from the base name.
832
+ #
833
+ # @param base [String]
834
+ # @return [Array(Integer, Integer)]
835
+ def self.lock_keys(base)
836
+ Digest::SHA256.digest("mutineer-run:#{base}").unpack("l>l>")
837
+ end
838
+
839
+ # Replace `name` with a template copy of `base`. Terminates other sessions
840
+ # on the slot only. A session on `base` is left alone and fails the copy.
841
+ #
842
+ # @param conn [PG::Connection]
843
+ # @param name [String] exact worker database name.
844
+ # @param base [String] test database used as the template.
845
+ # @return [void]
846
+ def self.recreate_from_template(conn, name, base)
847
+ unless worker_database_name?(base, name)
848
+ raise "refusing to drop #{name}"
849
+ end
850
+
851
+ conn.exec("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = #{conn.escape_literal(name)} AND pid <> pg_backend_pid()")
852
+ conn.exec("DROP DATABASE IF EXISTS #{quote_ident(name)}")
853
+ conn.exec("CREATE DATABASE #{quote_ident(name)} TEMPLATE #{quote_ident(base)}")
854
+ rescue StandardError => e
855
+ raise e if e.message.start_with?("refusing to drop")
856
+
857
+ raise provision_failure(base, name, e.message)
858
+ end
859
+
860
+ # Close every connection this process holds through ActiveRecord, so the
861
+ # test database can be used as a Postgres template and a MySQL slot can be
862
+ # dropped.
863
+ #
864
+ # @return [void]
865
+ def self.disconnect_app!
866
+ handler = ActiveRecord::Base.connection_handler
867
+ if handler.respond_to?(:clear_all_connections!)
868
+ handler.clear_all_connections!
869
+ else
870
+ ActiveRecord::Base.connection_pool.disconnect!
871
+ end
872
+ end
873
+
874
+ # The child inherited the parent's lock session. Drop the Ruby wrapper
875
+ # without sending a disconnect, so the parent's locks stay held.
876
+ # Clearing the held-name list stops the child from sending RELEASE_LOCK
877
+ # on that shared session.
878
+ #
879
+ # @return [void]
880
+ def self.forget_lock_connection_after_fork!
881
+ @mysql_held_locks = []
882
+ conn = @lock_connection
883
+ @lock_connection = nil
884
+ return unless conn
885
+
886
+ detach_inherited_connection(conn)
887
+ ObjectSpace.undefine_finalizer(conn)
888
+ end
889
+
890
+ # Make a fork child's copy of the lock connection safe to free. The
891
+ # client's free function runs at the child's next GC. Left as it is, that
892
+ # sends QUIT (mysql2) or Terminate (pg) on the socket the parent shares,
893
+ # so the server ends the parent's session and every lock it holds.
894
+ # These are the steps ActiveRecord's adapters take in `discard!`.
895
+ #
896
+ # @param conn [Mysql2::Client, Trilogy, PG::Connection]
897
+ # @return [void]
898
+ def self.detach_inherited_connection(conn)
899
+ if conn.respond_to?(:automatic_close=)
900
+ conn.automatic_close = false # mysql2: GC drops only this process's fd
901
+ elsif conn.respond_to?(:discard!)
902
+ conn.discard! # trilogy: closes only this process's fd
903
+ elsif conn.respond_to?(:socket_io)
904
+ conn.socket_io.reopen(IO::NULL) # pg: Terminate goes to /dev/null
905
+ end
906
+ end
907
+
908
+ # Quote a database name for a Postgres identifier. `CREATE DATABASE` cannot
909
+ # take a bound parameter for the name.
910
+ #
911
+ # @param name [String]
912
+ # @return [String]
913
+ def self.quote_ident(name)
914
+ raise ArgumentError, "database name is empty" if name.nil? || name.empty?
915
+
916
+ %("#{name.gsub('"', '""')}")
917
+ end
918
+
919
+ # Failure text for a copy that did not happen. Names the database and the
920
+ # server's cause. A busy template also says to close other sessions.
921
+ #
922
+ # @param base [String]
923
+ # @param name [String]
924
+ # @param cause [String]
925
+ # @return [String]
926
+ def self.provision_failure(base, name, cause)
927
+ message = "could not provision #{name} from #{base}: #{cause}"
928
+ message += ". Close other sessions on #{base}." if cause.match?(/other users|being accessed/i)
929
+ message
930
+ end
931
+
932
+ # Create MySQL slots `0...count`, load the app schema into each, and copy
933
+ # rows from the test database. Holds the guard lock and every pool name
934
+ # while copying, then keeps the guard lock. Pool names are released without
935
+ # a gap so another run cannot drop the new databases. Worker daemons then
936
+ # take one pool name each.
937
+ #
938
+ # @param count [Integer]
939
+ # @return [void]
940
+ # @raise [RuntimeError] when `count` exceeds {MYSQL_LOCK_SLOTS}, the server is
941
+ # MySQL older than 8.0, or another run holds the lock.
942
+ def self.provision_mysql(count)
943
+ if count > MYSQL_LOCK_SLOTS
944
+ raise "MySQL worker databases support at most #{MYSQL_LOCK_SLOTS} workers"
945
+ end
946
+
947
+ hash = routing_config_hash
948
+ remember_mysql_base_config(hash)
949
+ base = hash[:database].to_s
950
+ conn = open_mysql_connection(hash, base)
951
+ if (refusal = check_mysql_server(conn, password: hash[:password]))
952
+ conn.close
953
+ raise "could not provision #{base}: #{refusal}"
954
+ end
955
+ unless try_mysql_lock(conn, base, shared: false)
956
+ conn.close
957
+ raise "another Mutineer run is using #{base}"
958
+ end
959
+
960
+ begin
961
+ disconnect_app!
962
+ count.times do |slot|
963
+ name = mysql_worker_database(base, slot)
964
+ begin
965
+ recreate_mysql_database(conn, name, base)
966
+ load_mysql_schema(hash, name, base)
967
+ copy_mysql_rows(conn, base, name)
968
+ rescue StandardError => e
969
+ raise e if provision_wrapped?(e)
970
+
971
+ raise provision_failure(base, name, e.message)
972
+ end
973
+ end
974
+ raise "another Mutineer run is using #{base}" unless downgrade_mysql_lock_to_shared(conn, base)
975
+ rescue StandardError => e
976
+ release_held_mysql_locks(conn)
977
+ conn.close
978
+ raise e if provision_wrapped?(e)
979
+
980
+ raise provision_failure(base, base, e.message)
981
+ end
982
+
983
+ @lock_connection = conn
984
+ warn "[mutineer] provisioned #{count} mysql worker database(s) from #{base}"
985
+ end
986
+
987
+ # Read the server version and print a warning when the matrix does not
988
+ # test it. Does not close `conn` or raise: the caller does both. A failed
989
+ # version query only warns, with its cause shown. URL passwords, the
990
+ # DATABASE_URL password and the configured `password` are hidden in the
991
+ # cause and in the version text.
992
+ #
993
+ # @param conn [Mysql2::Client, Trilogy]
994
+ # @param password [String, nil] the configured password, hidden when it appears.
995
+ # @return [String, nil] the refusal text for MySQL older than 8.0, else nil.
996
+ def self.check_mysql_server(conn, password: nil)
997
+ secret = password.to_s
998
+ hide = lambda do |text|
999
+ out = Redaction.text(text)
1000
+ next out if secret.empty?
1001
+ # A short password would match inside ordinary words, so hide it only as a whole word.
1002
+ pattern = secret.length >= 4 ? secret : /(?<![[:alnum:]])#{Regexp.escape(secret)}(?![[:alnum:]])/
1003
+ out.gsub(pattern, Redaction::REDACTED)
1004
+ end
1005
+ failure = nil
1006
+ raw = begin
1007
+ mysql_rows(conn, "SELECT VERSION() AS version").first&.fetch("version", nil)
1008
+ rescue StandardError => e
1009
+ failure = hide.call("#{e.class}: #{e.message.lines.first.to_s.strip}")
1010
+ nil
1011
+ end
1012
+ support = mysql_version_support(raw)
1013
+ version = raw.nil? ? nil : hide.call(raw)
1014
+
1015
+ case support
1016
+ when :not_supported
1017
+ "MySQL #{version} is not supported. --daemon needs MySQL 8.0 or later. " \
1018
+ "Pass --no-daemon (or daemon: false) to stay serial."
1019
+ when :not_tested
1020
+ warn "[mutineer] server version #{version} is not tested with --daemon. " \
1021
+ "The run continues. If a result looks wrong, pass --no-daemon."
1022
+ nil
1023
+ when :unreadable
1024
+ shown = if failure
1025
+ "an error: #{failure}"
1026
+ elsif version.to_s.empty?
1027
+ "nothing"
1028
+ else
1029
+ "\"#{version}\""
1030
+ end
1031
+ warn "[mutineer] could not read the MySQL server version (got #{shown}). " \
1032
+ "The run continues. --daemon is tested on MySQL 8."
1033
+ nil
1034
+ end
1035
+ end
1036
+
1037
+ # Take one shared GET_LOCK name and drop this process's app connection.
1038
+ #
1039
+ # @return [void]
1040
+ def self.prepare_mysql_worker!
1041
+ hash = current_config_hash
1042
+ base = hash[:database].to_s
1043
+ conn = open_mysql_connection(hash, base)
1044
+ unless try_mysql_lock(conn, base, shared: true)
1045
+ conn.close
1046
+ raise "another Mutineer run is using #{base}"
1047
+ end
1048
+
1049
+ @lock_connection = conn
1050
+ disconnect_app!
1051
+ end
1052
+
1053
+ # Open a session with the app's MySQL client. It starts with no default
1054
+ # database, so it is not tied to the test database while it drops and
1055
+ # creates slots. {restore_mysql_triggers} later selects a slot. Every
1056
+ # other statement names its database. The gem does not depend on the client.
1057
+ #
1058
+ # @param config_hash [Hash]
1059
+ # @param base [String] test database name, named in the error if this fails.
1060
+ # @return [Mysql2::Client, Trilogy]
1061
+ def self.open_mysql_connection(config_hash, base)
1062
+ adapter = config_hash[:adapter].to_s
1063
+ opts = mysql_client_options(config_hash)
1064
+ if adapter == "trilogy"
1065
+ raise "could not provision #{base}: the app's trilogy client is not loaded" unless defined?(Trilogy)
1066
+
1067
+ # Rails' adapter turns a `ssl_mode` name into the constant trilogy wants.
1068
+ if defined?(ActiveRecord::ConnectionAdapters::TrilogyAdapter)
1069
+ ActiveRecord::ConnectionAdapters::TrilogyAdapter.new_client(opts)
1070
+ else
1071
+ Trilogy.new(opts)
1072
+ end
1073
+ else
1074
+ raise "could not provision #{base}: the app's mysql2 client is not loaded" unless defined?(Mysql2::Client)
1075
+
1076
+ Mysql2::Client.new(opts)
1077
+ end
1078
+ rescue StandardError => e
1079
+ raise e if e.message.start_with?("could not provision ")
1080
+
1081
+ raise "could not provision #{base}: #{e.message}"
1082
+ end
1083
+
1084
+ # Connection options shared by mysql2 and trilogy. The database key is left
1085
+ # out so the session is not tied to the test database. A socket wins over
1086
+ # host: trilogy ignores `socket` when `host` is set, and would then use its
1087
+ # compiled-in default instead of the app's socket. TLS and timeout keys in
1088
+ # {MYSQL_CLIENT_KEYS} are kept.
1089
+ #
1090
+ # @param config_hash [Hash]
1091
+ # @return [Hash]
1092
+ def self.mysql_client_options(config_hash)
1093
+ opts = {}
1094
+ socket = config_hash[:socket].to_s
1095
+ if socket.empty?
1096
+ host = config_hash[:host].to_s
1097
+ opts[:host] = host unless host.empty?
1098
+ port = config_hash[:port]
1099
+ opts[:port] = Integer(port) if port && !port.to_s.empty?
1100
+ else
1101
+ opts[:socket] = socket
1102
+ end
1103
+ user = config_hash[:username] || config_hash[:user]
1104
+ opts[:username] = user.to_s if user && !user.to_s.empty?
1105
+ password = config_hash[:password]
1106
+ opts[:password] = password.to_s unless password.nil?
1107
+ MYSQL_CLIENT_KEYS.each { |key| opts[key] = config_hash[key] unless config_hash[key].nil? }
1108
+ # `:encoding` is an ActiveRecord key. mysql2 and trilogy do not share it.
1109
+ # Row copy runs in the server, so the client charset does not change it.
1110
+ opts
1111
+ end
1112
+
1113
+ # Take one pool name, or the guard plus every pool name when `shared` is
1114
+ # false. A miss releases any names this call already took.
1115
+ #
1116
+ # @param conn [Mysql2::Client, Trilogy]
1117
+ # @param base [String]
1118
+ # @param shared [Boolean]
1119
+ # @return [Boolean]
1120
+ def self.try_mysql_lock(conn, base, shared:)
1121
+ return try_mysql_shared_lock(conn, base) if shared
1122
+
1123
+ try_mysql_exclusive_lock(conn, base)
1124
+ end
1125
+
1126
+ # Take the guard, then every pool name. The guard stays held across the
1127
+ # later downgrade so another run cannot start a copy in that gap.
1128
+ #
1129
+ # @param conn [Mysql2::Client, Trilogy]
1130
+ # @param base [String]
1131
+ # @return [Boolean]
1132
+ def self.try_mysql_exclusive_lock(conn, base)
1133
+ guard = mysql_guard_lock_name(base)
1134
+ return false unless mysql_get_lock(conn, guard)
1135
+
1136
+ held = [guard]
1137
+ mysql_lock_names(base).each do |name|
1138
+ if mysql_get_lock(conn, name)
1139
+ held << name
1140
+ else
1141
+ held.each { |got| mysql_release_lock(conn, got) }
1142
+ @mysql_held_locks = []
1143
+ return false
1144
+ end
1145
+ end
1146
+ @mysql_held_locks = held
1147
+ true
1148
+ end
1149
+
1150
+ # Take the first free pool name. Does not take the guard, so worker daemons
1151
+ # can connect while the provisioning process still holds that guard.
1152
+ #
1153
+ # @param conn [Mysql2::Client, Trilogy]
1154
+ # @param base [String]
1155
+ # @return [Boolean]
1156
+ def self.try_mysql_shared_lock(conn, base)
1157
+ mysql_lock_names(base).each do |name|
1158
+ next unless mysql_get_lock(conn, name)
1159
+
1160
+ @mysql_held_locks = [name]
1161
+ return true
1162
+ end
1163
+ @mysql_held_locks = []
1164
+ false
1165
+ end
1166
+
1167
+ # Release every pool name and keep the guard. The guard is never released
1168
+ # first, so this process always holds a lock the next copy needs.
1169
+ #
1170
+ # @param conn [Mysql2::Client, Trilogy]
1171
+ # @param base [String]
1172
+ # @return [Boolean] false when this session does not hold the guard.
1173
+ def self.downgrade_mysql_lock_to_shared(conn, base)
1174
+ guard = mysql_guard_lock_name(base)
1175
+ held = Array(@mysql_held_locks)
1176
+ return false unless held.include?(guard)
1177
+
1178
+ (held - [guard]).each { |name| mysql_release_lock(conn, name) }
1179
+ @mysql_held_locks = [guard]
1180
+ true
1181
+ end
1182
+
1183
+ # Release the GET_LOCK names this session took for the current run.
1184
+ #
1185
+ # @param conn [Mysql2::Client, Trilogy]
1186
+ # @return [void]
1187
+ def self.release_held_mysql_locks(conn)
1188
+ Array(@mysql_held_locks).each { |name| mysql_release_lock(conn, name) }
1189
+ @mysql_held_locks = []
1190
+ nil
1191
+ end
1192
+
1193
+ # One GET_LOCK name per worker daemon. The digest keeps each name inside 64
1194
+ # characters for any base database name.
1195
+ #
1196
+ # @param base [String]
1197
+ # @return [Array<String>]
1198
+ def self.mysql_lock_names(base)
1199
+ digest = mysql_lock_digest(base)
1200
+ (0...MYSQL_LOCK_SLOTS).map { |slot| "m:#{digest}:#{slot}" }
1201
+ end
1202
+
1203
+ # Lock held for the whole copy and until worker daemons hold pool names.
1204
+ # Not part of {mysql_lock_names}, so it does not reduce the worker count.
1205
+ #
1206
+ # @param base [String]
1207
+ # @return [String]
1208
+ def self.mysql_guard_lock_name(base)
1209
+ "m:#{mysql_lock_digest(base)}:g"
1210
+ end
1211
+
1212
+ # Shared prefix for the guard and the pool. 16 hex characters keep the full
1213
+ # lock name inside MySQL's 64-character limit.
1214
+ #
1215
+ # @param base [String]
1216
+ # @return [String]
1217
+ def self.mysql_lock_digest(base)
1218
+ Digest::SHA256.hexdigest("mutineer-run:#{base}")[0, 16]
1219
+ end
1220
+
1221
+ # @param conn [Mysql2::Client, Trilogy]
1222
+ # @param name [String]
1223
+ # @return [Boolean]
1224
+ def self.mysql_get_lock(conn, name)
1225
+ rows = mysql_rows(conn, "SELECT GET_LOCK(#{mysql_quote_string(conn, name)}, 0) AS got")
1226
+ rows.dig(0, "got").to_i == 1
1227
+ end
1228
+
1229
+ # @param conn [Mysql2::Client, Trilogy]
1230
+ # @param name [String]
1231
+ # @return [void]
1232
+ def self.mysql_release_lock(conn, name)
1233
+ mysql_exec(conn, "SELECT RELEASE_LOCK(#{mysql_quote_string(conn, name)})")
1234
+ end
1235
+
1236
+ # Drop and create `name`. Refuses any name that is not one of `base`'s
1237
+ # worker database names, so a name bug can never drop the test database.
1238
+ #
1239
+ # @param conn [Mysql2::Client, Trilogy]
1240
+ # @param name [String]
1241
+ # @param base [String] the test database name.
1242
+ # @return [void]
1243
+ def self.recreate_mysql_database(conn, name, base)
1244
+ raise "refusing to drop #{name}" unless owned_database?(base, name, slots: MYSQL_LOCK_SLOTS, limit: MYSQL_NAME_LIMIT)
1245
+
1246
+ quoted = quote_mysql_ident(name)
1247
+ charset, collation = mysql_database_defaults(conn, base)
1248
+ mysql_exec(conn, "DROP DATABASE IF EXISTS #{quoted}")
1249
+ mysql_exec(conn, "CREATE DATABASE #{quoted} CHARACTER SET #{charset} COLLATE #{collation}")
1250
+ end
1251
+
1252
+ # The test database's default character set and collation. A table the
1253
+ # schema creates without its own collation then compares text the same way
1254
+ # in each worker database as in the test database.
1255
+ #
1256
+ # @param conn [Mysql2::Client, Trilogy]
1257
+ # @param base [String] the test database name.
1258
+ # @return [Array(String, String)] the character set and the collation.
1259
+ # @raise [RuntimeError] when the test database is missing or reports an unexpected name.
1260
+ def self.mysql_database_defaults(conn, base)
1261
+ row = mysql_rows(conn, "SELECT DEFAULT_CHARACTER_SET_NAME AS charset, DEFAULT_COLLATION_NAME AS collation " \
1262
+ "FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = #{mysql_quote_string(conn, base)}").first
1263
+ values = row && [row["charset"].to_s, row["collation"].to_s]
1264
+ raise "cannot read the character set of #{base}" unless values&.all? { |value| value.match?(/\A\w+\z/) }
1265
+
1266
+ values
1267
+ end
1268
+
1269
+ # Load `schema.rb` or `structure.sql` into the empty slot with the app's
1270
+ # DatabaseTasks. A missing file leaves the slot empty so the row copy can
1271
+ # still clone tables that exist only on the test database.
1272
+ #
1273
+ # @param base_hash [Hash] symbol-keyed config for the test database.
1274
+ # @param name [String] worker database name.
1275
+ # @param base [String] test database name, used in the failure text.
1276
+ # @return [void]
1277
+ def self.load_mysql_schema(base_hash, name, base)
1278
+ return unless defined?(ActiveRecord::Tasks::DatabaseTasks)
1279
+
1280
+ switched = false
1281
+ failure = nil
1282
+ begin
1283
+ slot_hash = base_hash.merge(database: name)
1284
+ ActiveRecord::Base.establish_connection(slot_hash)
1285
+ switched = true
1286
+ db_config = mysql_db_config(slot_hash)
1287
+ format = schema_format_for(db_config)
1288
+ file = mysql_schema_file(db_config, format)
1289
+ ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, format, file) if file
1290
+ rescue StandardError => e
1291
+ failure = e
1292
+ ensure
1293
+ if switched
1294
+ begin
1295
+ ActiveRecord::Base.establish_connection(base_hash)
1296
+ rescue StandardError => e
1297
+ failure ||= e
1298
+ end
1299
+ end
1300
+ end
1301
+ return unless failure
1302
+ raise failure if provision_wrapped?(failure)
1303
+
1304
+ raise provision_failure(base, name, failure.message)
1305
+ end
1306
+
1307
+ # A database config whose name is the slot, so DatabaseTasks loads into that
1308
+ # database and not the test database.
1309
+ #
1310
+ # @param slot_hash [Hash]
1311
+ # @return [ActiveRecord::DatabaseConfigurations::HashConfig]
1312
+ def self.mysql_db_config(slot_hash)
1313
+ env = ENV["RAILS_ENV"].to_s
1314
+ env = "test" if env.empty?
1315
+ ActiveRecord::DatabaseConfigurations::HashConfig.new(env, "primary", slot_hash)
1316
+ end
1317
+
1318
+ # Schema format of one database config. Rails 8.1 lets each config set its
1319
+ # own `schema_format:`; older Rails has only the app-wide setting. A string
1320
+ # setting ("ruby") counts the same as the symbol.
1321
+ #
1322
+ # @param db_config [ActiveRecord::DatabaseConfigurations::DatabaseConfig]
1323
+ # @return [Symbol]
1324
+ def self.schema_format_for(db_config)
1325
+ format = db_config.respond_to?(:schema_format) ? db_config.schema_format : current_schema_format
1326
+ format.respond_to?(:to_sym) ? format.to_sym : format
1327
+ end
1328
+
1329
+ # Schema format the booted app is using.
1330
+ #
1331
+ # @return [Symbol]
1332
+ def self.current_schema_format
1333
+ if ActiveRecord.respond_to?(:schema_format)
1334
+ ActiveRecord.schema_format
1335
+ else
1336
+ ActiveRecord::Base.schema_format
1337
+ end
1338
+ end
1339
+
1340
+ # Absolute path of the schema file for `format`, or nil when it is absent.
1341
+ # DatabaseTasks aborts the process when the file is missing, so the caller
1342
+ # skips the load instead.
1343
+ #
1344
+ # @param db_config [ActiveRecord::DatabaseConfigurations::HashConfig]
1345
+ # @param format [Symbol]
1346
+ # @return [String, nil]
1347
+ def self.mysql_schema_file(db_config, format)
1348
+ path = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config, format)
1349
+ path if path && File.exist?(path)
1350
+ end
1351
+
1352
+ # Copy every base table from `base` into `dest`. Foreign key checks are off.
1353
+ # Generated columns are left out of the INSERT list. A table the schema load
1354
+ # did not create is cloned with CREATE TABLE ... LIKE, then its foreign keys
1355
+ # are added. LIKE does not copy them. Each table's AUTO_INCREMENT is set
1356
+ # from the base after the rows, so a deleted high id is not reused.
1357
+ # Triggers the schema load created are dropped for the copy and created
1358
+ # again after it, so the copied rows match the base.
1359
+ #
1360
+ # @param conn [Mysql2::Client, Trilogy]
1361
+ # @param base [String]
1362
+ # @param dest [String]
1363
+ # @return [void]
1364
+ def self.copy_mysql_rows(conn, base, dest)
1365
+ mysql_exec(conn, "SET FOREIGN_KEY_CHECKS = 0")
1366
+ # MySQL 8 caches information_schema.TABLES values, AUTO_INCREMENT among
1367
+ # them, for a day by default. The copy reads them live.
1368
+ begin
1369
+ mysql_exec(conn, "SET SESSION information_schema_stats_expiry = 0")
1370
+ rescue StandardError
1371
+ nil # MariaDB has no such cache or variable.
1372
+ end
1373
+ triggers = drop_mysql_triggers(conn, dest)
1374
+ tables = mysql_rows(conn, <<~SQL)
1375
+ SELECT TABLE_NAME AS table_name
1376
+ FROM information_schema.TABLES
1377
+ WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
1378
+ AND TABLE_TYPE = 'BASE TABLE'
1379
+ SQL
1380
+ names = []
1381
+ cloned = []
1382
+ tables.each do |row|
1383
+ table = row["table_name"].to_s
1384
+ next if table.empty?
1385
+
1386
+ names << table
1387
+ cloned << table if ensure_mysql_table(conn, base, dest, table)
1388
+ columns = copyable_mysql_columns(conn, base, table)
1389
+ next if columns.empty?
1390
+
1391
+ dest_table = "#{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)}"
1392
+ base_table = "#{quote_mysql_ident(base)}.#{quote_mysql_ident(table)}"
1393
+ listed = columns.map { |column| quote_mysql_ident(column) }.join(", ")
1394
+ mysql_exec(conn, "DELETE FROM #{dest_table}")
1395
+ mysql_exec(conn, "INSERT INTO #{dest_table} (#{listed}) SELECT #{listed} FROM #{base_table}")
1396
+ end
1397
+ cloned.each { |table| copy_mysql_foreign_keys(conn, base, dest, table) }
1398
+ names.each { |table| copy_mysql_auto_increment(conn, base, dest, table) }
1399
+ restore_mysql_triggers(conn, dest, triggers)
1400
+ ensure
1401
+ begin
1402
+ mysql_exec(conn, "SET FOREIGN_KEY_CHECKS = 1")
1403
+ rescue StandardError
1404
+ nil
1405
+ end
1406
+ end
1407
+
1408
+ # Drop every trigger on `dest` and return what {restore_mysql_triggers}
1409
+ # needs to create each one again. Triggers on one table event come back in
1410
+ # ACTION_ORDER. MySQL 8's CREATE text has no FOLLOWS or PRECEDES, so
1411
+ # creating them in that order keeps their order.
1412
+ #
1413
+ # The CREATE text is read as utf8mb4 with an empty sql_mode. mysql2 0.5.6
1414
+ # and older open utf8mb3, which turns a 4-byte character into "?". Under
1415
+ # ANSI_QUOTES, MySQL quotes the names with double quotes, which the
1416
+ # trigger's own sql_mode may not read. Backticks read in every sql_mode.
1417
+ # The session stays on utf8mb4, so {restore_mysql_triggers} sends the
1418
+ # text back unchanged.
1419
+ #
1420
+ # @param conn [Mysql2::Client, Trilogy]
1421
+ # @param dest [String]
1422
+ # @return [Array<Hash>] `:name`, `:sql`, `:sql_mode` and `:collation` for each trigger.
1423
+ # @raise [RuntimeError] when SHOW CREATE TRIGGER returns no row for a trigger.
1424
+ def self.drop_mysql_triggers(conn, dest)
1425
+ rows = mysql_rows(conn, <<~SQL)
1426
+ SELECT TRIGGER_NAME AS trigger_name
1427
+ FROM information_schema.TRIGGERS
1428
+ WHERE TRIGGER_SCHEMA = #{mysql_quote_string(conn, dest)}
1429
+ ORDER BY EVENT_OBJECT_TABLE, ACTION_TIMING, EVENT_MANIPULATION, ACTION_ORDER
1430
+ SQL
1431
+ return [] if rows.empty?
1432
+
1433
+ sql_mode = mysql_rows(conn, "SELECT @@SESSION.sql_mode AS sql_mode").first["sql_mode"]
1434
+ mysql_exec(conn, "SET NAMES utf8mb4")
1435
+ mysql_exec(conn, "SET SESSION sql_mode = ''")
1436
+ # Limit: the text comes back as utf8mb4. In a trigger saved with another
1437
+ # client charset, a _binary or _latin1 literal that holds non-ASCII bytes
1438
+ # changes bytes on replay.
1439
+ triggers = rows.map do |row|
1440
+ name = row["trigger_name"].to_s
1441
+ shown = mysql_rows(conn, "SHOW CREATE TRIGGER #{quote_mysql_ident(dest)}.#{quote_mysql_ident(name)}").first
1442
+ raise "cannot read trigger #{name} on #{dest}" unless shown
1443
+
1444
+ { name: name, sql: shown["sql original statement"].to_s, sql_mode: shown["sql_mode"].to_s,
1445
+ collation: shown["collation_connection"].to_s }
1446
+ end
1447
+ # Every definition is saved before the first drop.
1448
+ triggers.each { |trigger| mysql_exec(conn, "DROP TRIGGER #{quote_mysql_ident(dest)}.#{quote_mysql_ident(trigger[:name])}") }
1449
+ mysql_exec(conn, "SET SESSION sql_mode = #{mysql_quote_string(conn, sql_mode)}")
1450
+ triggers
1451
+ end
1452
+
1453
+ # Create the triggers {drop_mysql_triggers} dropped. Each one is created
1454
+ # with its own sql_mode and connection collation, which decide how MySQL
1455
+ # reads its body. The CREATE text names no database, so this session
1456
+ # switches to `dest`. The sql_mode and collation go back to their values at
1457
+ # the start of this call. The character set stays utf8mb4. A failed create
1458
+ # names the trigger, for example when the DEFINER account needs SET_USER_ID.
1459
+ #
1460
+ # @param conn [Mysql2::Client, Trilogy]
1461
+ # @param dest [String]
1462
+ # @param triggers [Array<Hash>]
1463
+ # @return [void]
1464
+ # @raise [RuntimeError] when MySQL refuses to create a trigger.
1465
+ def self.restore_mysql_triggers(conn, dest, triggers)
1466
+ return if triggers.empty?
1467
+
1468
+ saved = mysql_rows(conn, "SELECT @@SESSION.sql_mode AS sql_mode, @@SESSION.collation_connection AS collation").first
1469
+ mysql_exec(conn, "USE #{quote_mysql_ident(dest)}")
1470
+ triggers.each do |trigger|
1471
+ mysql_exec(conn, mysql_session_modes_sql(conn, trigger[:sql_mode], trigger[:collation]))
1472
+ mysql_exec(conn, trigger[:sql])
1473
+ rescue StandardError => e
1474
+ raise "cannot recreate trigger #{trigger[:name]} on #{dest}: #{e.message}"
1475
+ end
1476
+ mysql_exec(conn, mysql_session_modes_sql(conn, saved["sql_mode"], saved["collation"]))
1477
+ end
1478
+
1479
+ # SET statement for the session's sql_mode and connection collation.
1480
+ #
1481
+ # @param conn [Mysql2::Client, Trilogy]
1482
+ # @param sql_mode [String]
1483
+ # @param collation [String]
1484
+ # @return [String]
1485
+ def self.mysql_session_modes_sql(conn, sql_mode, collation)
1486
+ "SET SESSION sql_mode = #{mysql_quote_string(conn, sql_mode)}, " \
1487
+ "collation_connection = #{mysql_quote_string(conn, collation)}"
1488
+ end
1489
+
1490
+ # Create `table` on `dest` when the schema load did not. LIKE copies columns,
1491
+ # including generated columns, and does not copy foreign keys. The caller
1492
+ # adds those keys after every missing table exists.
1493
+ #
1494
+ # @param conn [Mysql2::Client, Trilogy]
1495
+ # @param base [String]
1496
+ # @param dest [String]
1497
+ # @param table [String]
1498
+ # @return [Boolean] true when this call created the table.
1499
+ def self.ensure_mysql_table(conn, base, dest, table)
1500
+ rows = mysql_rows(conn, <<~SQL)
1501
+ SELECT TABLE_NAME AS table_name
1502
+ FROM information_schema.TABLES
1503
+ WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, dest)}
1504
+ AND TABLE_NAME = #{mysql_quote_string(conn, table)}
1505
+ AND TABLE_TYPE = 'BASE TABLE'
1506
+ SQL
1507
+ return false unless rows.empty?
1508
+
1509
+ mysql_exec(conn, "CREATE TABLE #{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)} LIKE #{quote_mysql_ident(base)}.#{quote_mysql_ident(table)}")
1510
+ true
1511
+ end
1512
+
1513
+ # Add foreign keys from `base.table` onto `dest.table`. Keys that already
1514
+ # exist on `dest` are left alone. A referenced table in `base` is retargeted
1515
+ # at `dest` so the worker does not point back at the test database.
1516
+ #
1517
+ # @param conn [Mysql2::Client, Trilogy]
1518
+ # @param base [String]
1519
+ # @param dest [String]
1520
+ # @param table [String]
1521
+ # @return [void]
1522
+ def self.copy_mysql_foreign_keys(conn, base, dest, table)
1523
+ existing = mysql_rows(conn, <<~SQL)
1524
+ SELECT CONSTRAINT_NAME AS constraint_name
1525
+ FROM information_schema.TABLE_CONSTRAINTS
1526
+ WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, dest)}
1527
+ AND TABLE_NAME = #{mysql_quote_string(conn, table)}
1528
+ AND CONSTRAINT_TYPE = 'FOREIGN KEY'
1529
+ SQL
1530
+ have = existing.map { |row| row["constraint_name"].to_s }
1531
+ rows = mysql_rows(conn, <<~SQL)
1532
+ SELECT k.CONSTRAINT_NAME AS constraint_name,
1533
+ k.COLUMN_NAME AS column_name,
1534
+ k.ORDINAL_POSITION AS ordinal_position,
1535
+ k.REFERENCED_TABLE_SCHEMA AS referenced_table_schema,
1536
+ k.REFERENCED_TABLE_NAME AS referenced_table_name,
1537
+ k.REFERENCED_COLUMN_NAME AS referenced_column_name,
1538
+ r.UPDATE_RULE AS update_rule,
1539
+ r.DELETE_RULE AS delete_rule,
1540
+ r.MATCH_OPTION AS match_option
1541
+ FROM information_schema.KEY_COLUMN_USAGE k
1542
+ INNER JOIN information_schema.REFERENTIAL_CONSTRAINTS r
1543
+ ON r.CONSTRAINT_SCHEMA = k.CONSTRAINT_SCHEMA
1544
+ AND r.CONSTRAINT_NAME = k.CONSTRAINT_NAME
1545
+ AND r.TABLE_NAME = k.TABLE_NAME
1546
+ WHERE k.TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
1547
+ AND k.TABLE_NAME = #{mysql_quote_string(conn, table)}
1548
+ AND k.REFERENCED_TABLE_NAME IS NOT NULL
1549
+ ORDER BY k.CONSTRAINT_NAME, k.ORDINAL_POSITION
1550
+ SQL
1551
+ rows = rows.reject { |row| have.include?(row["constraint_name"].to_s) }
1552
+ sql = mysql_add_foreign_key_sql(dest, table, rows, base: base)
1553
+ mysql_exec(conn, sql) if sql
1554
+ end
1555
+
1556
+ # One ALTER TABLE that adds every foreign key in `rows`.
1557
+ #
1558
+ # @param dest [String]
1559
+ # @param table [String]
1560
+ # @param rows [Array<Hash>]
1561
+ # @param base [String] test database name. References to it are rewritten to `dest`.
1562
+ # @return [String, nil]
1563
+ def self.mysql_add_foreign_key_sql(dest, table, rows, base:)
1564
+ clauses = mysql_foreign_key_clauses(rows, base: base, dest: dest)
1565
+ return nil if clauses.empty?
1566
+
1567
+ "ALTER TABLE #{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)} #{clauses.join(', ')}"
1568
+ end
1569
+
1570
+ # ADD CONSTRAINT fragments for `rows`, grouped by constraint name.
1571
+ #
1572
+ # @param rows [Array<Hash>]
1573
+ # @param base [String]
1574
+ # @param dest [String]
1575
+ # @return [Array<String>]
1576
+ def self.mysql_foreign_key_clauses(rows, base:, dest:)
1577
+ grouped = {}
1578
+ Array(rows).each do |row|
1579
+ name = row["constraint_name"].to_s
1580
+ next if name.empty?
1581
+
1582
+ (grouped[name] ||= []) << row
1583
+ end
1584
+ grouped.filter_map do |name, cols|
1585
+ ordered = cols.sort_by { |row| row["ordinal_position"].to_i }
1586
+ local_cols = ordered.map { |row| row["column_name"].to_s }
1587
+ ref_cols = ordered.map { |row| row["referenced_column_name"].to_s }
1588
+ ref_table = ordered.first["referenced_table_name"].to_s
1589
+ next if ref_table.empty? || local_cols.any?(&:empty?) || ref_cols.any?(&:empty?)
1590
+
1591
+ ref_schema = ordered.first["referenced_table_schema"].to_s
1592
+ ref_schema = dest if ref_schema.empty? || ref_schema == base
1593
+ delete_rule = mysql_referential_action(ordered.first["delete_rule"])
1594
+ update_rule = mysql_referential_action(ordered.first["update_rule"])
1595
+ match = mysql_match_clause(ordered.first["match_option"])
1596
+ local_sql = local_cols.map { |column| quote_mysql_ident(column) }.join(", ")
1597
+ ref_sql = ref_cols.map { |column| quote_mysql_ident(column) }.join(", ")
1598
+ target = "#{quote_mysql_ident(ref_schema)}.#{quote_mysql_ident(ref_table)}"
1599
+ "ADD CONSTRAINT #{quote_mysql_ident(name)} FOREIGN KEY (#{local_sql}) REFERENCES #{target} (#{ref_sql})#{match} ON DELETE #{delete_rule} ON UPDATE #{update_rule}"
1600
+ end
1601
+ end
1602
+
1603
+ # A foreign-key action safe to interpolate. Unknown text raises.
1604
+ #
1605
+ # @param value [String, nil]
1606
+ # @return [String]
1607
+ # @raise [ArgumentError] when `value` is not one of {MYSQL_REFERENTIAL_ACTIONS}.
1608
+ def self.mysql_referential_action(value)
1609
+ action = value.to_s.strip.upcase.tr("_", " ")
1610
+ return action if MYSQL_REFERENTIAL_ACTIONS.include?(action)
1611
+
1612
+ raise ArgumentError, "unsupported foreign key action #{value.inspect}"
1613
+ end
1614
+
1615
+ # MATCH fragment, or an empty string for the default NONE.
1616
+ #
1617
+ # @param value [String, nil]
1618
+ # @return [String]
1619
+ # @raise [ArgumentError] when `value` is not NONE, FULL, or PARTIAL.
1620
+ def self.mysql_match_clause(value)
1621
+ match = value.to_s.strip.upcase
1622
+ return "" if match.empty? || match == "NONE"
1623
+ return " MATCH #{match}" if %w[FULL PARTIAL].include?(match)
1624
+
1625
+ raise ArgumentError, "unsupported foreign key match #{value.inspect}"
1626
+ end
1627
+
1628
+ # Set `dest.table`'s AUTO_INCREMENT from the base table when the base has one.
1629
+ #
1630
+ # @param conn [Mysql2::Client, Trilogy]
1631
+ # @param base [String]
1632
+ # @param dest [String]
1633
+ # @param table [String]
1634
+ # @return [void]
1635
+ def self.copy_mysql_auto_increment(conn, base, dest, table)
1636
+ rows = mysql_rows(conn, <<~SQL)
1637
+ SELECT AUTO_INCREMENT AS auto_increment
1638
+ FROM information_schema.TABLES
1639
+ WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
1640
+ AND TABLE_NAME = #{mysql_quote_string(conn, table)}
1641
+ SQL
1642
+ sql = mysql_auto_increment_sql(dest, table, rows.dig(0, "auto_increment"))
1643
+ mysql_exec(conn, sql) if sql
1644
+ end
1645
+
1646
+ # ALTER TABLE statement that sets AUTO_INCREMENT, or nil when `value` is blank.
1647
+ #
1648
+ # @param dest [String]
1649
+ # @param table [String]
1650
+ # @param value [Object]
1651
+ # @return [String, nil]
1652
+ def self.mysql_auto_increment_sql(dest, table, value)
1653
+ return nil if value.nil? || value.to_s.empty?
1654
+
1655
+ number = Integer(value)
1656
+ return nil if number < 1
1657
+
1658
+ "ALTER TABLE #{quote_mysql_ident(dest)}.#{quote_mysql_ident(table)} AUTO_INCREMENT = #{number}"
1659
+ rescue ArgumentError, TypeError
1660
+ nil
1661
+ end
1662
+
1663
+ # Column names that an INSERT may set. Stored and virtual generated columns
1664
+ # are omitted. Expression defaults stay, because they are real columns.
1665
+ #
1666
+ # @param conn [Mysql2::Client, Trilogy]
1667
+ # @param base [String]
1668
+ # @param table [String]
1669
+ # @return [Array<String>]
1670
+ def self.copyable_mysql_columns(conn, base, table)
1671
+ rows = mysql_rows(conn, <<~SQL)
1672
+ SELECT COLUMN_NAME AS column_name, EXTRA AS extra
1673
+ FROM information_schema.COLUMNS
1674
+ WHERE TABLE_SCHEMA = #{mysql_quote_string(conn, base)}
1675
+ AND TABLE_NAME = #{mysql_quote_string(conn, table)}
1676
+ ORDER BY ORDINAL_POSITION
1677
+ SQL
1678
+ rows.filter_map do |row|
1679
+ name = row["column_name"].to_s
1680
+ next if name.empty?
1681
+ next if row["extra"].to_s.match?(/VIRTUAL GENERATED|STORED GENERATED/i)
1682
+
1683
+ name
1684
+ end
1685
+ end
1686
+
1687
+ # Run a statement that does not need its result rows.
1688
+ #
1689
+ # @param conn [Mysql2::Client, Trilogy]
1690
+ # @param sql [String]
1691
+ # @return [void]
1692
+ def self.mysql_exec(conn, sql)
1693
+ conn.query(sql)
1694
+ nil
1695
+ end
1696
+
1697
+ # Result rows as string-keyed hashes. mysql2 yields hashes from `each`.
1698
+ # trilogy yields them from `each_hash`. Keys are downcased so aliases match.
1699
+ #
1700
+ # @param conn [Mysql2::Client, Trilogy]
1701
+ # @param sql [String]
1702
+ # @return [Array<Hash>]
1703
+ def self.mysql_rows(conn, sql)
1704
+ result = conn.query(sql)
1705
+ return [] if result.nil?
1706
+
1707
+ rows = []
1708
+ if result.respond_to?(:each_hash)
1709
+ result.each_hash { |row| rows << stringify_mysql_row(row) }
1710
+ else
1711
+ result.each(as: :hash) { |row| rows << stringify_mysql_row(row) }
1712
+ end
1713
+ rows
1714
+ end
1715
+
1716
+ # @param row [Hash]
1717
+ # @return [Hash]
1718
+ def self.stringify_mysql_row(row)
1719
+ row.to_h.transform_keys { |key| key.to_s.downcase }
1720
+ end
1721
+
1722
+ # Quote a string for a SQL literal with the client the app loaded.
1723
+ #
1724
+ # @param conn [Mysql2::Client, Trilogy]
1725
+ # @param value [String]
1726
+ # @return [String]
1727
+ def self.mysql_quote_string(conn, value)
1728
+ "'#{conn.escape(value.to_s)}'"
1729
+ end
1730
+
1731
+ # Quote a MySQL identifier. Backticks inside the name are doubled.
1732
+ #
1733
+ # @param name [String]
1734
+ # @return [String]
1735
+ def self.quote_mysql_ident(name)
1736
+ raise ArgumentError, "database name is empty" if name.nil? || name.empty?
1737
+ raise ArgumentError, "database name contains a null" if name.include?("\0")
1738
+
1739
+ "`#{name.to_s.gsub('`', '``')}`"
1740
+ end
1741
+
1742
+ # True when `error` is already a provision failure this module raised.
1743
+ #
1744
+ # @param error [StandardError]
1745
+ # @return [Boolean]
1746
+ def self.provision_wrapped?(error)
1747
+ message = error.message
1748
+ message.start_with?("could not provision", "another Mutineer run", "refusing to drop")
1749
+ end
211
1750
  end
212
1751
  end