ros-apartment 4.0.0.alpha8 → 4.0.0.alpha9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +30 -0
- data/lib/apartment/adapters/abstract_adapter.rb +14 -6
- data/lib/apartment/adapters/postgresql_database_adapter.rb +2 -0
- data/lib/apartment/adapters/postgresql_schema_adapter.rb +2 -0
- data/lib/apartment/adapters/postgresql_transaction_state.rb +34 -0
- data/lib/apartment/config.rb +15 -1
- data/lib/apartment/migrator.rb +5 -1
- data/lib/apartment/patches/connection_handling.rb +20 -1
- data/lib/apartment/pool_manager.rb +11 -0
- data/lib/apartment/pool_reaper.rb +5 -0
- data/lib/apartment/transaction_taint.rb +123 -0
- data/lib/apartment/version.rb +1 -1
- data/lib/apartment.rb +94 -10
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5b4b1a4532066ffd3ebfdfa8c43b0f0cdd4e304f31f9905af8a91550e71c44a6
|
|
4
|
+
data.tar.gz: 75868c2063131bc20022d35142d87aff2c15f7c8b65b7fa3d8cbee43be4f11d1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1f3c98442bf8ea826918440071a3a702615494d8761ef20f990135a10697650e05b72926f28a6cf1053b2aa969b67aee1b8c43249efe73ff58e8802c916f92a3
|
|
7
|
+
data.tar.gz: f23fd1d357e9b5b598516960a0d480f22512b0c8b4481827bb137cc074589a73d4684e4879dc1b2355a12a108fe1b25eaf54eff283aa6c89557c823d91a3e13a
|
data/README.md
CHANGED
|
@@ -338,6 +338,36 @@ Platform notes: parallel migrations use threads. On macOS, libpq has known fork-
|
|
|
338
338
|
|
|
339
339
|
## Known Limitations
|
|
340
340
|
|
|
341
|
+
### Connection poolers in transaction mode (PgBouncer, RDS Proxy)
|
|
342
|
+
|
|
343
|
+
> [!WARNING]
|
|
344
|
+
> With the PostgreSQL `:schema` strategy, **PgBouncer in `transaction` pooling mode can
|
|
345
|
+
> silently serve one tenant another tenant's data.** No error is raised — queries return the
|
|
346
|
+
> wrong tenant's rows. It is safe only on **PostgreSQL 18+** with
|
|
347
|
+
> `track_extra_parameters = IntervalStyle,search_path`; on PostgreSQL 17 and older,
|
|
348
|
+
> transaction mode cannot be made safe and you must use `session` mode.
|
|
349
|
+
|
|
350
|
+
Tenant isolation under `:schema` rests on `search_path`, which is session-scoped state.
|
|
351
|
+
Transaction-mode pooling hands a different backend to each transaction, so a `search_path`
|
|
352
|
+
set by one client is not guaranteed to be the one in effect for the next query. This affects
|
|
353
|
+
v3 and v4 alike; v4 reduces the number of `SET search_path` statements to one per connection
|
|
354
|
+
but does not eliminate them, and one is enough.
|
|
355
|
+
|
|
356
|
+
**Which pooler to use:** PgBouncer on **PostgreSQL 18+** with
|
|
357
|
+
`track_extra_parameters = IntervalStyle,search_path` is the only setup that safely multiplexes
|
|
358
|
+
a schema-per-tenant app — and it needs no changes to Apartment or your application. **RDS Proxy
|
|
359
|
+
is safe but reduces no connections**: a Rails app pins on it *with or without Apartment* (three
|
|
360
|
+
unconditional `SET`s in `configure_connection`, plus the extended query protocol even at
|
|
361
|
+
`prepared_statements: false` — [rails/rails#40207](https://github.com/rails/rails/issues/40207)),
|
|
362
|
+
and AWS offers no session-pinning filters for PostgreSQL. Keep RDS Proxy if you use it for
|
|
363
|
+
failover or IAM; don't adopt it to cut connections.
|
|
364
|
+
|
|
365
|
+
Database-per-tenant (`:database_name`) is not affected.
|
|
366
|
+
|
|
367
|
+
**Read [Connection Poolers](docs/connection-poolers.md) before putting a pooler in front of a
|
|
368
|
+
schema-per-tenant app**, and verify your configuration rather than assuming it: the failure
|
|
369
|
+
is silent, so a working app proves nothing.
|
|
370
|
+
|
|
341
371
|
### `connects_to` with Separate Databases
|
|
342
372
|
|
|
343
373
|
If a model (or its abstract base class) uses `connects_to` to point at a completely different database (not just different roles on the same DB), Apartment's `connection_pool` patch will attempt to create a tenant pool for it.
|
|
@@ -68,15 +68,13 @@ module Apartment
|
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
# Drop a tenant.
|
|
71
|
-
def drop(tenant)
|
|
71
|
+
def drop(tenant)
|
|
72
72
|
drop_tenant(tenant)
|
|
73
73
|
removed_pools = Apartment.pool_manager&.remove_tenant(tenant) || []
|
|
74
74
|
removed_pools.each do |pool_key, pool|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
warn "[Apartment] Pool disconnect failed for '#{pool_key}': #{e.class}: #{e.message}"
|
|
79
|
-
end
|
|
75
|
+
# remove_tenant already took these out of the manager, so deregister_shard's
|
|
76
|
+
# own removal returns nil and cannot disconnect them — we close them here.
|
|
77
|
+
Apartment.disconnect_removed_pool(pool, pool_key)
|
|
80
78
|
begin
|
|
81
79
|
deregister_shard_from_ar_handler(pool_key)
|
|
82
80
|
rescue StandardError => e
|
|
@@ -118,6 +116,16 @@ module Apartment
|
|
|
118
116
|
false
|
|
119
117
|
end
|
|
120
118
|
|
|
119
|
+
# Whether +conn+ sits in an aborted-transaction state that every subsequent
|
|
120
|
+
# statement will fail against until the transaction ends. PostgreSQL is the
|
|
121
|
+
# only supported engine with such a state (PQTRANS_INERROR); MySQL fails the
|
|
122
|
+
# statement and leaves the transaction usable, and its raw connection has no
|
|
123
|
+
# transaction_status at all. Base is conservative: never reclassify.
|
|
124
|
+
# See docs/designs/transaction-taint-detection.md (Evidence E).
|
|
125
|
+
def aborted_transaction?(_conn)
|
|
126
|
+
false
|
|
127
|
+
end
|
|
128
|
+
|
|
121
129
|
# Request-path fail-safe contract. The elevator wraps the
|
|
122
130
|
# tenant switch; on one of these error classes it asks
|
|
123
131
|
# #tenant_container_gone? whether the tenant's storage actually vanished (a
|
|
@@ -10,6 +10,8 @@ module Apartment
|
|
|
10
10
|
# to the environmentified tenant name. Lifecycle operations (create/drop)
|
|
11
11
|
# execute DDL against the default connection.
|
|
12
12
|
class PostgresqlDatabaseAdapter < AbstractAdapter
|
|
13
|
+
include PostgresqlTransactionState
|
|
14
|
+
|
|
13
15
|
def resolve_connection_config(tenant, base_config: nil)
|
|
14
16
|
config = base_config || send(:base_config)
|
|
15
17
|
config.merge('database' => environmentify(tenant))
|
|
@@ -12,6 +12,8 @@ module Apartment
|
|
|
12
12
|
# Apartment.config.postgres_config. Lifecycle operations (create/drop)
|
|
13
13
|
# execute DDL against the default connection.
|
|
14
14
|
class PostgresqlSchemaAdapter < AbstractAdapter
|
|
15
|
+
include PostgresqlTransactionState
|
|
16
|
+
|
|
15
17
|
def shared_pinned_connection?
|
|
16
18
|
!Apartment.config.force_separate_pinned_pool
|
|
17
19
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Apartment
|
|
4
|
+
module Adapters
|
|
5
|
+
# PostgreSQL's aborted-transaction state, shared by both PG adapters
|
|
6
|
+
# (schema-per-tenant and database-per-tenant). Their implementations are
|
|
7
|
+
# identical, so this is the DRY seam rather than two copies.
|
|
8
|
+
#
|
|
9
|
+
# A failed statement inside a transaction moves the connection to
|
|
10
|
+
# PQTRANS_INERROR, where every subsequent statement raises
|
|
11
|
+
# PG::InFailedSqlTransaction until the transaction ends. Apartment heals this
|
|
12
|
+
# at pool checkin; see docs/designs/transaction-taint-detection.md.
|
|
13
|
+
module PostgresqlTransactionState
|
|
14
|
+
def aborted_transaction?(conn)
|
|
15
|
+
# NOT conn.raw_connection. That is the escape hatch for code about to *use*
|
|
16
|
+
# the driver, so it materializes lazy transactions, marks the connection
|
|
17
|
+
# dirty, and CONNECTS one that was never connected — at every checkin, on
|
|
18
|
+
# every pooled connection, which defeats Rails' lazy connect and burns a
|
|
19
|
+
# backend per tenant pool. We only read a status flag, so we take the handle
|
|
20
|
+
# directly. `connected?` keeps us off the never-connected path.
|
|
21
|
+
return false unless conn.connected?
|
|
22
|
+
|
|
23
|
+
raw = conn.instance_variable_get(:@raw_connection)
|
|
24
|
+
return false unless raw.respond_to?(:transaction_status)
|
|
25
|
+
|
|
26
|
+
raw.transaction_status == ::PG::PQTRANS_INERROR
|
|
27
|
+
rescue StandardError
|
|
28
|
+
# A connection too broken to report its own status is not ours to classify;
|
|
29
|
+
# AR's own verify!/reconnect path owns that case.
|
|
30
|
+
false
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
data/lib/apartment/config.rb
CHANGED
|
@@ -27,7 +27,8 @@ module Apartment
|
|
|
27
27
|
:active_record_log, :sql_query_tags,
|
|
28
28
|
:shard_key_prefix,
|
|
29
29
|
:migration_role, :app_role, :schema_cache_per_tenant, :check_pending_migrations,
|
|
30
|
-
:force_separate_pinned_pool, :test_fixture_cleanup, :reap_in_test
|
|
30
|
+
:force_separate_pinned_pool, :test_fixture_cleanup, :reap_in_test,
|
|
31
|
+
:heal_tainted_connections
|
|
31
32
|
|
|
32
33
|
def initialize # rubocop:disable Metrics/AbcSize
|
|
33
34
|
@tenant_strategy = nil
|
|
@@ -63,6 +64,13 @@ module Apartment
|
|
|
63
64
|
@check_pending_migrations = true
|
|
64
65
|
@force_separate_pinned_pool = false
|
|
65
66
|
@reap_in_test = false
|
|
67
|
+
# Reset a tenant connection left in an aborted transaction when it is checked
|
|
68
|
+
# back into its pool. On by default: without it the poisoned connection is
|
|
69
|
+
# served to the next caller, and under pool-per-tenant that connection is the
|
|
70
|
+
# tenant's ONLY connection, so the tenant is dead on that worker until the
|
|
71
|
+
# process restarts. ActiveRecord's active? cannot detect the state.
|
|
72
|
+
# PostgreSQL-only in effect. See docs/designs/transaction-taint-detection.md.
|
|
73
|
+
@heal_tainted_connections = true
|
|
66
74
|
@test_fixture_cleanup = true
|
|
67
75
|
end
|
|
68
76
|
|
|
@@ -258,6 +266,12 @@ module Apartment
|
|
|
258
266
|
"reap_in_test must be true or false, got: #{@reap_in_test.inspect}")
|
|
259
267
|
end
|
|
260
268
|
|
|
269
|
+
unless [true, false].include?(@heal_tainted_connections)
|
|
270
|
+
raise(ConfigurationError,
|
|
271
|
+
'heal_tainted_connections must be true or false, ' \
|
|
272
|
+
"got: #{@heal_tainted_connections.inspect}")
|
|
273
|
+
end
|
|
274
|
+
|
|
261
275
|
unless @tenant_validator.nil? || @tenant_validator == false || @tenant_validator.respond_to?(:call)
|
|
262
276
|
raise(ConfigurationError,
|
|
263
277
|
'tenant_validator must be nil, false, or a callable, ' \
|
data/lib/apartment/migrator.rb
CHANGED
|
@@ -222,7 +222,11 @@ module Apartment
|
|
|
222
222
|
role = Apartment.config.migration_role
|
|
223
223
|
return unless role && Apartment.pool_manager
|
|
224
224
|
|
|
225
|
-
Apartment.pool_manager.evict_by_role(role).each do |pool_key,
|
|
225
|
+
Apartment.pool_manager.evict_by_role(role).each do |pool_key, pool|
|
|
226
|
+
# evict_by_role already removed it from the manager, so deregister_shard has
|
|
227
|
+
# nothing left to disconnect; close it here or a pool AR no longer registers
|
|
228
|
+
# leaks its backend. See docs/designs/out-of-band-tenant-ddl.md.
|
|
229
|
+
Apartment.disconnect_removed_pool(pool, pool_key)
|
|
226
230
|
Apartment.deregister_shard(pool_key)
|
|
227
231
|
end
|
|
228
232
|
rescue StandardError => e
|
|
@@ -86,15 +86,34 @@ module Apartment
|
|
|
86
86
|
# to the reaper and to max_total accounting (a connection leak that also
|
|
87
87
|
# undercounts the cap). Deregister it before re-raising so AR and the
|
|
88
88
|
# manager stay consistent. The next request re-establishes cleanly.
|
|
89
|
+
#
|
|
90
|
+
# deregister_ar_shard, NOT deregister_shard: we are running inside
|
|
91
|
+
# PoolManager's create block, and the full form removes from PoolManager's
|
|
92
|
+
# Concurrent::Map — whose MRI backend guards compute_if_absent and delete
|
|
93
|
+
# with the same non-reentrant mutex, so that would raise ThreadError
|
|
94
|
+
# ("deadlock; recursive locking") and skip this deregistration entirely,
|
|
95
|
+
# orphaning the very pool this rescue exists to reclaim. There is nothing
|
|
96
|
+
# to remove from the manager here regardless: the pool is not stored until
|
|
97
|
+
# this block returns.
|
|
98
|
+
#
|
|
99
|
+
# Reached with +send+ because the AR-only half is private: it is a
|
|
100
|
+
# half-operation, and leaving one publicly reachable is the exact footgun
|
|
101
|
+
# this seam exists to close. This is the one place it is correct.
|
|
89
102
|
begin
|
|
90
103
|
raise(Apartment::PendingMigrationError, tenant) if check_pending_migrations?(pool)
|
|
91
104
|
|
|
92
105
|
load_tenant_schema_cache(tenant, pool) if cfg.schema_cache_per_tenant
|
|
93
106
|
rescue StandardError
|
|
94
|
-
Apartment.
|
|
107
|
+
Apartment.send(:deregister_ar_shard, pool_key)
|
|
95
108
|
raise
|
|
96
109
|
end
|
|
97
110
|
|
|
111
|
+
# After the post-establish checks (a pool that fails them is discarded, so
|
|
112
|
+
# extending it would be wasted), and before the pool is handed out (so the
|
|
113
|
+
# very first checkin is already covered).
|
|
114
|
+
# See docs/designs/transaction-taint-detection.md.
|
|
115
|
+
Apartment::TransactionTaint.install(pool, tenant: tenant, pool_key: pool_key)
|
|
116
|
+
|
|
98
117
|
pool
|
|
99
118
|
end
|
|
100
119
|
rescue Apartment::ApartmentError
|
|
@@ -40,12 +40,22 @@ module Apartment
|
|
|
40
40
|
|
|
41
41
|
# Delete pool first, then timestamp. This ordering prevents a concurrent
|
|
42
42
|
# #get from orphaning a timestamp (get checks @pools, skips touch if absent).
|
|
43
|
+
#
|
|
44
|
+
# @api private
|
|
45
|
+
# Forgetting a pool here is only HALF of discarding it — the pool stays
|
|
46
|
+
# registered in AR's ConnectionHandler, leaking the registration and a live
|
|
47
|
+
# backend if the tenant is never re-accessed. Callers outside the gem want
|
|
48
|
+
# Apartment.deregister_shard (one pool) or Apartment.reset_tenant_pools! (all).
|
|
49
|
+
# These mutators also bypass PoolReaper's in-use guard: they will drop a pool
|
|
50
|
+
# with a checked-out connection or an open transaction.
|
|
51
|
+
# See docs/designs/out-of-band-tenant-ddl.md.
|
|
43
52
|
def remove(tenant_key)
|
|
44
53
|
pool = @pools.delete(tenant_key)
|
|
45
54
|
@timestamps.delete(tenant_key)
|
|
46
55
|
pool
|
|
47
56
|
end
|
|
48
57
|
|
|
58
|
+
# @api private — see #remove.
|
|
49
59
|
def remove_tenant(tenant)
|
|
50
60
|
prefix = "#{tenant}:"
|
|
51
61
|
removed = []
|
|
@@ -58,6 +68,7 @@ module Apartment
|
|
|
58
68
|
removed
|
|
59
69
|
end
|
|
60
70
|
|
|
71
|
+
# @api private — see #remove.
|
|
61
72
|
def evict_by_role(role)
|
|
62
73
|
suffix = ":#{role}"
|
|
63
74
|
removed = []
|
|
@@ -142,6 +142,11 @@ module Apartment
|
|
|
142
142
|
# on_evict with a nil pool.
|
|
143
143
|
return nil unless (pool = @pool_manager.remove(tenant))
|
|
144
144
|
|
|
145
|
+
# We removed the pool ourselves, so deregister_shard's own removal returns nil
|
|
146
|
+
# and it has nothing left to disconnect. AR disconnects only a pool it still
|
|
147
|
+
# finds registered, so a pool that has drifted out of AR's handler would leak
|
|
148
|
+
# its backend unless we close it here. See docs/designs/out-of-band-tenant-ddl.md.
|
|
149
|
+
Apartment.disconnect_removed_pool(pool, tenant)
|
|
145
150
|
deregister_from_ar_handler(tenant)
|
|
146
151
|
Instrumentation.instrument(:evict, tenant: tenant, reason: reason)
|
|
147
152
|
@on_evict&.call(tenant, pool)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Apartment
|
|
4
|
+
# Heals a tenant connection left in an aborted transaction (PostgreSQL's
|
|
5
|
+
# PQTRANS_INERROR) at the moment it is checked back into its pool.
|
|
6
|
+
#
|
|
7
|
+
# WHY CHECKIN. ActiveRecord's +active?+ probes with an empty query, which does
|
|
8
|
+
# NOT error in an aborted transaction, so +verify!+ pronounces a poisoned
|
|
9
|
+
# connection healthy and +checkin+ does not reset it. The pool then serves it to
|
|
10
|
+
# the next caller. Under pool-per-tenant that connection is the ONLY connection
|
|
11
|
+
# for its tenant, so the tenant is dead on that worker until the process
|
|
12
|
+
# restarts, while every other tenant looks fine and nothing in the logs explains
|
|
13
|
+
# it. That amplification is what makes this ours to fix: the defect is generic
|
|
14
|
+
# Rails/PG, the consequence is specific to pool-per-tenant.
|
|
15
|
+
#
|
|
16
|
+
# Checkin is also the one seam that serves both populations correctly. A
|
|
17
|
+
# FIXTURE-PINNED connection must KEEP its transaction (teardown_fixtures owns the
|
|
18
|
+
# rollback) and never reaches ConnectionPool#checkin's body; a production
|
|
19
|
+
# connection must be reset before reuse. We still test +pinned+ explicitly,
|
|
20
|
+
# because this override runs BEFORE the early return in AR's own +checkin+.
|
|
21
|
+
#
|
|
22
|
+
# NEVER issue a raw ROLLBACK here. It destroys the ENCLOSING transaction while
|
|
23
|
+
# ActiveRecord still believes its stack is intact, raises nothing, and lets
|
|
24
|
+
# subsequent writes autocommit — turning a loud failure into silent, permanent
|
|
25
|
+
# database pollution. +reset!+ is ActiveRecord's own primitive: it rolls back,
|
|
26
|
+
# DISCARDs session state, and resets AR's transaction bookkeeping together. It
|
|
27
|
+
# also re-applies the pool's schema_search_path, so a healed tenant connection
|
|
28
|
+
# still points at its own schema (verified — the alternative would be a
|
|
29
|
+
# cross-tenant leak, which is strictly worse than the bug being fixed).
|
|
30
|
+
#
|
|
31
|
+
# Design: docs/designs/transaction-taint-detection.md
|
|
32
|
+
module TransactionTaint
|
|
33
|
+
# Extended onto Apartment-owned tenant pools only. The primary pool and every
|
|
34
|
+
# app-owned pool are left alone: the same Rails defect reaches them, but they
|
|
35
|
+
# are not ours, and the blast radius there is one connection of N rather than a
|
|
36
|
+
# dead tenant. That gap is reported upstream, not patched here.
|
|
37
|
+
module PoolHeal
|
|
38
|
+
attr_accessor :apartment_tenant, :apartment_pool_key, :apartment_taint_warned
|
|
39
|
+
|
|
40
|
+
def checkin(conn)
|
|
41
|
+
TransactionTaint.heal(conn, self)
|
|
42
|
+
super
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
class << self
|
|
47
|
+
# Extend +pool+ with the heal. Called once, at tenant-pool creation.
|
|
48
|
+
def install(pool, tenant:, pool_key:)
|
|
49
|
+
return pool unless Apartment.config&.heal_tainted_connections
|
|
50
|
+
|
|
51
|
+
pool.extend(PoolHeal)
|
|
52
|
+
pool.apartment_tenant = tenant.to_s
|
|
53
|
+
pool.apartment_pool_key = pool_key
|
|
54
|
+
pool
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Best-effort, and it must never raise: an exception here would abort
|
|
58
|
+
# ConnectionPool#checkin and leak the connection out of the pool permanently,
|
|
59
|
+
# which is worse than the taint we came to fix.
|
|
60
|
+
def heal(conn, pool)
|
|
61
|
+
return if conn.pinned
|
|
62
|
+
return unless Apartment.adapter&.aborted_transaction?(conn)
|
|
63
|
+
|
|
64
|
+
open_transactions = conn.open_transactions
|
|
65
|
+
conn.reset!
|
|
66
|
+
warn_once(pool)
|
|
67
|
+
instrument(pool, open_transactions: open_transactions, healed: true)
|
|
68
|
+
rescue StandardError => e
|
|
69
|
+
discard(conn, pool, e)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
# reset! failed, and +super+ is about to put this connection back in the pool's
|
|
75
|
+
# available list. Checking a STILL-POISONED connection back in would rebuild the
|
|
76
|
+
# exact outage we came to fix — the next lease gets it and the tenant stays dead.
|
|
77
|
+
# So drop the handle instead: disconnect! sends no SQL and cannot fail to nil out
|
|
78
|
+
# @raw_connection, so the pool keeps a dead shell that Rails transparently
|
|
79
|
+
# reconnects, fresh, on the next checkout. Checking in a disconnected connection
|
|
80
|
+
# is normal Rails (it is what flush/reap leave behind).
|
|
81
|
+
def discard(conn, pool, error)
|
|
82
|
+
conn.disconnect!
|
|
83
|
+
warn('[Apartment] could not reset the tainted connection for tenant ' \
|
|
84
|
+
"'#{pool.apartment_tenant}' (#{error.class}: #{error.message}); the connection " \
|
|
85
|
+
'was dropped and will be reopened on next use.')
|
|
86
|
+
instrument(pool, open_transactions: nil, healed: false, error: error.class.name)
|
|
87
|
+
rescue StandardError
|
|
88
|
+
nil # Nothing further is safe to attempt inside checkin.
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def instrument(pool, open_transactions:, healed:, error: nil)
|
|
92
|
+
Instrumentation.instrument(
|
|
93
|
+
'transaction_taint',
|
|
94
|
+
tenant: pool.apartment_tenant,
|
|
95
|
+
pool_key: pool.apartment_pool_key,
|
|
96
|
+
open_transactions: open_transactions,
|
|
97
|
+
healed: healed,
|
|
98
|
+
error: error
|
|
99
|
+
)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Once per pool per process. Healing means each taint is a distinct incident
|
|
103
|
+
# rather than a cascade, but an app that poisons on every request would still
|
|
104
|
+
# warn on every request — flooding the log during the exact incident where the
|
|
105
|
+
# signal matters. The notification fires every time; it is the countable
|
|
106
|
+
# channel, and the one to alert on.
|
|
107
|
+
def warn_once(pool)
|
|
108
|
+
return if pool.apartment_taint_warned
|
|
109
|
+
|
|
110
|
+
pool.apartment_taint_warned = true
|
|
111
|
+
warn(
|
|
112
|
+
"[Apartment] the connection for tenant '#{pool.apartment_tenant}' was checked in " \
|
|
113
|
+
'while in an aborted transaction (PostgreSQL PQTRANS_INERROR) and has been reset. ' \
|
|
114
|
+
'Something ran a statement that failed inside a transaction Rails did not unwind. ' \
|
|
115
|
+
'Contain it by wrapping the failing call in ' \
|
|
116
|
+
'ActiveRecord::Base.transaction(requires_new: true); subscribe to ' \
|
|
117
|
+
'transaction_taint.apartment to find the call site. Do NOT add a raw ROLLBACK loop — ' \
|
|
118
|
+
'see docs/testing.md.'
|
|
119
|
+
)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
data/lib/apartment/version.rb
CHANGED
data/lib/apartment.rb
CHANGED
|
@@ -213,23 +213,81 @@ module Apartment # rubocop:disable Metrics/ModuleLength
|
|
|
213
213
|
ActiveRecord::QueryLogs.tags = ActiveRecord::QueryLogs.tags + [:tenant]
|
|
214
214
|
end
|
|
215
215
|
|
|
216
|
-
#
|
|
216
|
+
# Discard one tenant pool, whole: deregister its shard from AR's ConnectionHandler
|
|
217
|
+
# and forget it in PoolManager, disconnecting it either way.
|
|
217
218
|
# Safe to call when AR is not loaded or config is not set (no-op).
|
|
218
219
|
# Used by PoolReaper eviction, AbstractAdapter#drop, and teardown.
|
|
220
|
+
#
|
|
221
|
+
# Both registries are updated because either one alone is wrong: deregistering
|
|
222
|
+
# from AR while the manager still holds the pool wedges the tenant permanently
|
|
223
|
+
# (the manager keeps handing back a pool AR has forgotten), and forgetting it in
|
|
224
|
+
# the manager alone leaks the AR registration and a live backend when the tenant
|
|
225
|
+
# is never re-accessed. Every internal caller already removes from the manager
|
|
226
|
+
# first, so the removal here is a no-op for them.
|
|
227
|
+
#
|
|
228
|
+
# The pool is disconnected here rather than left to AR, which disconnects only a
|
|
229
|
+
# pool it actually finds registered (ConnectionHandler#disconnect_pool_from_pool_manager
|
|
230
|
+
# guards `pool_config.disconnect!` behind `if pool_config`). A manager-held pool
|
|
231
|
+
# with no matching AR registration is reachable — the integration suite swaps the
|
|
232
|
+
# ConnectionHandler per example — and since we have just removed it from the
|
|
233
|
+
# manager, no later `PoolManager#clear` will disconnect it either. Mirrors
|
|
234
|
+
# AbstractAdapter#drop, which already removes, disconnects, then deregisters.
|
|
235
|
+
# (Callers that remove the pool from the manager THEMSELVES must disconnect it
|
|
236
|
+
# themselves too — the removal here returns nil for them, so there is nothing left
|
|
237
|
+
# for us to disconnect. PoolReaper#evict_tenant and Migrator#evict_migration_pools
|
|
238
|
+
# do exactly that.)
|
|
239
|
+
# See docs/designs/out-of-band-tenant-ddl.md.
|
|
240
|
+
#
|
|
241
|
+
# NOT SAFE inside a PoolManager create block — use +deregister_ar_shard+ there.
|
|
242
|
+
# PoolManager's @pools is a Concurrent::Map whose MRI backend guards
|
|
243
|
+
# +compute_if_absent+ and +delete+ with the SAME non-reentrant mutex, so removing
|
|
244
|
+
# a pool from inside the create block raises ThreadError ("deadlock; recursive
|
|
245
|
+
# locking"). The manager removal below is exactly that call.
|
|
246
|
+
#
|
|
247
|
+
# ORDER: AR first, manager removal in an +ensure+. The manager removal is an
|
|
248
|
+
# in-memory delete that cannot meaningfully fail, so it is the step that may run
|
|
249
|
+
# unconditionally; AR's removal does IO and is the one that can raise. Running the
|
|
250
|
+
# fallible step first, with the infallible one guaranteed after, means the call
|
|
251
|
+
# always ends with BOTH registries clear.
|
|
252
|
+
#
|
|
253
|
+
# Manager-first would be a race: between the manager delete and AR's removal, a
|
|
254
|
+
# concurrent tenant switch misses the manager, calls establish_connection — which
|
|
255
|
+
# RETURNS THE STILL-REGISTERED OLD POOL (ConnectionHandler#establish_connection
|
|
256
|
+
# reuses a pool whose db_config is equal) — and stores that doomed pool back in
|
|
257
|
+
# the manager. AR then unregisters and disconnects it, leaving the manager holding
|
|
258
|
+
# a dead pool: the permanent wedge this method exists to prevent, reintroduced.
|
|
259
|
+
# In this order the same interleaving costs at most one failed request, and the
|
|
260
|
+
# ensure clears the manager so the next switch rebuilds cleanly.
|
|
261
|
+
#
|
|
262
|
+
# Deliberately un-rescued at this level: both steps rescue their own failures, and
|
|
263
|
+
# swallowing everything here is what once hid a ThreadError, silently orphaning the
|
|
264
|
+
# pool the caller asked us to discard. Misuse should be loud.
|
|
219
265
|
def deregister_shard(pool_key)
|
|
220
266
|
return unless @config && defined?(ActiveRecord::Base)
|
|
221
267
|
|
|
222
|
-
|
|
223
|
-
|
|
268
|
+
begin
|
|
269
|
+
deregister_ar_shard(pool_key)
|
|
270
|
+
ensure
|
|
271
|
+
disconnect_removed_pool(@pool_manager&.remove(pool_key), pool_key)
|
|
272
|
+
end
|
|
273
|
+
end
|
|
224
274
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
275
|
+
# @api private
|
|
276
|
+
# Disconnect a pool that has been removed from PoolManager. Idempotent with AR's
|
|
277
|
+
# own disconnect on the happy path (ConnectionPool#disconnect! empties @connections,
|
|
278
|
+
# so a second call has nothing left to close); load-bearing only when AR has no
|
|
279
|
+
# matching registration to disconnect, in which case nothing else will close it.
|
|
280
|
+
#
|
|
281
|
+
# Public because the callers that remove a pool from the manager THEMSELVES — and
|
|
282
|
+
# therefore get nil back from deregister_shard's own removal — must disconnect what
|
|
283
|
+
# they removed: PoolReaper#evict_tenant, Migrator#evict_migration_pools,
|
|
284
|
+
# AbstractAdapter#drop. Rescued so one broken pool cannot abort the caller.
|
|
285
|
+
def disconnect_removed_pool(pool, pool_key)
|
|
286
|
+
return unless pool.respond_to?(:disconnect!)
|
|
287
|
+
|
|
288
|
+
pool.disconnect!
|
|
231
289
|
rescue StandardError => e
|
|
232
|
-
warn "[Apartment] Failed to
|
|
290
|
+
warn "[Apartment] Failed to disconnect pool for #{pool_key}: #{e.class}: #{e.message}"
|
|
233
291
|
end
|
|
234
292
|
|
|
235
293
|
# Deregister all tenant pools from AR's ConnectionHandler and clear the
|
|
@@ -255,6 +313,32 @@ module Apartment # rubocop:disable Metrics/ModuleLength
|
|
|
255
313
|
|
|
256
314
|
private
|
|
257
315
|
|
|
316
|
+
# The ActiveRecord half of a discard: deregister the shard (which disconnects the
|
|
317
|
+
# pool AR holds), leaving PoolManager untouched. The ONLY form that is safe inside
|
|
318
|
+
# a PoolManager create block — see the re-entrancy note on +deregister_shard+ —
|
|
319
|
+
# and correct there anyway: compute_if_absent does not store the pool until the
|
|
320
|
+
# block returns, so there is no manager entry to remove.
|
|
321
|
+
#
|
|
322
|
+
# PRIVATE ON PURPOSE. This is a half-operation, and a reachable half-operation is
|
|
323
|
+
# the bug this whole seam exists to eliminate: called on its own, it leaves AR
|
|
324
|
+
# without the pool while PoolManager keeps handing it out, wedging the tenant for
|
|
325
|
+
# the life of the process. The one legitimate caller reaches it with +send+.
|
|
326
|
+
def deregister_ar_shard(pool_key)
|
|
327
|
+
return unless @config && defined?(ActiveRecord::Base)
|
|
328
|
+
|
|
329
|
+
_, separator, role_str = pool_key.to_s.rpartition(':')
|
|
330
|
+
role = separator.empty? || role_str.empty? ? ActiveRecord.writing_role : role_str.to_sym
|
|
331
|
+
|
|
332
|
+
shard_key = :"#{@config.shard_key_prefix}_#{pool_key}"
|
|
333
|
+
ActiveRecord::Base.connection_handler.remove_connection_pool(
|
|
334
|
+
'ActiveRecord::Base',
|
|
335
|
+
role: role,
|
|
336
|
+
shard: shard_key
|
|
337
|
+
)
|
|
338
|
+
rescue StandardError => e
|
|
339
|
+
warn "[Apartment] Failed to deregister AR pool for #{pool_key}: #{e.class}: #{e.message}"
|
|
340
|
+
end
|
|
341
|
+
|
|
258
342
|
# Double-checked locking: the common path (already built) skips the mutex;
|
|
259
343
|
# concurrent first callers serialize so exactly one validator is built.
|
|
260
344
|
# TenantValidator.new subscribes to ActiveSupport::Notifications, so a
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ros-apartment
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.0.0.
|
|
4
|
+
version: 4.0.0.alpha9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ryan Brunner
|
|
@@ -168,6 +168,7 @@ files:
|
|
|
168
168
|
- lib/apartment/adapters/mysql2_adapter.rb
|
|
169
169
|
- lib/apartment/adapters/postgresql_database_adapter.rb
|
|
170
170
|
- lib/apartment/adapters/postgresql_schema_adapter.rb
|
|
171
|
+
- lib/apartment/adapters/postgresql_transaction_state.rb
|
|
171
172
|
- lib/apartment/adapters/sqlite3_adapter.rb
|
|
172
173
|
- lib/apartment/adapters/trilogy_adapter.rb
|
|
173
174
|
- lib/apartment/cli.rb
|
|
@@ -207,6 +208,7 @@ files:
|
|
|
207
208
|
- lib/apartment/tenant_name_validator.rb
|
|
208
209
|
- lib/apartment/tenant_validator.rb
|
|
209
210
|
- lib/apartment/test_fixtures.rb
|
|
211
|
+
- lib/apartment/transaction_taint.rb
|
|
210
212
|
- lib/apartment/version.rb
|
|
211
213
|
- lib/generators/apartment/install/USAGE
|
|
212
214
|
- lib/generators/apartment/install/install_generator.rb
|