legion-transport 1.4.30 → 1.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/CHANGELOG.md +27 -0
- data/Gemfile +1 -0
- data/README.md +124 -7
- data/lib/legion/transport/connection/recovery.rb +241 -0
- data/lib/legion/transport/connection/vault.rb +25 -20
- data/lib/legion/transport/connection.rb +337 -249
- data/lib/legion/transport/errors.rb +2 -0
- data/lib/legion/transport/exchange.rb +61 -28
- data/lib/legion/transport/helper.rb +44 -6
- data/lib/legion/transport/helpers/pool.rb +117 -48
- data/lib/legion/transport/in_process.rb +119 -46
- data/lib/legion/transport/local.rb +222 -18
- data/lib/legion/transport/message.rb +85 -133
- data/lib/legion/transport/messages/direct.rb +2 -2
- data/lib/legion/transport/messages/dynamic.rb +6 -2
- data/lib/legion/transport/messages/task.rb +3 -3
- data/lib/legion/transport/publisher.rb +114 -0
- data/lib/legion/transport/queue.rb +65 -56
- data/lib/legion/transport/routes.rb +109 -17
- data/lib/legion/transport/settings.rb +2 -2
- data/lib/legion/transport/spool/legacy.rb +143 -0
- data/lib/legion/transport/spool/recovery_store.rb +180 -0
- data/lib/legion/transport/spool.rb +554 -105
- data/lib/legion/transport/tenant_migration.rb +408 -0
- data/lib/legion/transport/tenant_provisioner.rb +36 -20
- data/lib/legion/transport/tenant_topology.rb +24 -12
- data/lib/legion/transport/version.rb +1 -1
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7dc701734922367645d3e0c8fd75c358e66dc04d12fb5ea81f9c30ea5cea6f8f
|
|
4
|
+
data.tar.gz: 92e223c6df729c7a64e1fcc80d0aa7187898f97e506e23787d762aeeb609c6df
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7c68ba535132feb33b8339c3c7502441bcb9993925df8196ace33c1223363ac19dea1bd5e3e0e3cf2e3166c2f272fa3977d08c3e0a2b97ab091bd4d3dcb3145d
|
|
7
|
+
data.tar.gz: f4a6ddb3c97f7efbd70497cb704b90218644d3a87b04d37082c3b4a51e28e3a6c32da0508f7ce3b96db4fe62d82d8ad4b4d48ab6110fd1d70a2abf3bee16f98d
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,33 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [1.4.31] - 2026-10-29
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
- Bound pool checkout with a monotonic deadline that includes connection creation; reserve capacity without holding the pool mutex during network operations, wake waiters on shutdown, and reject late connections.
|
|
9
|
+
- Bound connection shutdown across pooled, publisher, log, and build sessions with one five-second deadline; disable recovery, close sockets, and stop old consumer workers before replacing sessions.
|
|
10
|
+
- Keep forced reconnect exclusive to its owning caller and restore existing queue, channel, and consumer references, bindings, and QoS before notifying reconnect callbacks.
|
|
11
|
+
- Reuse and track thread-local publisher channels in pool mode; preserve publisher caches across repeated forced reconnects without accumulating log channels.
|
|
12
|
+
- Start each Bunny session inside its host failover attempt and clean up failed handshakes before advancing to the next host. Fixes #41.
|
|
13
|
+
- Bound recovery snapshot lock acquisition and invalidate the complete reconnect generation on shutdown; cancelled owners cannot reopen transport or disturb a newer session. Stop legacy Bunny readers before intentional socket closure to avoid asynchronous owner-thread errors.
|
|
14
|
+
- Use the supported zero-argument Bunny confirm API with bounded waits, retain Vault PEM owners for the session/recovery lifetime, and keep DLX cleanup failures from masking declaration failures. Fixes #42.
|
|
15
|
+
- Support the common lite-mode channel factory contract, ordered pending-only delivery outside global locks, exchange-scoped direct/topic/fanout routing, and atomic consumer cancellation with session-owned cleanup. Fixes #43.
|
|
16
|
+
- Apply tenant naming at real declaration/binding/DLX boundaries, scope exchange caches by tenant and declaration context, align provisioning with actual resources, and enforce encoded-byte quotas in message/helper publishing and replay. Fixes #44.
|
|
17
|
+
- Correlate mandatory returns on exchanges, retire ambiguous confirm-timeout channels, persist only successful spool writes, and replay claimed wire records asynchronously after connection recovery with at-least-once confirmation semantics.
|
|
18
|
+
- Preserve existing queues, bindings, and DLQ contents on declaration mismatches; surface required DLX failures, report actual HTTP publish outcomes, and default to persistent messages with prefetch 2 while retaining explicit false/zero overrides. Fixes #45.
|
|
19
|
+
- Fail closed when requested encryption is not ready, bound encryption attempts, retain cipher/IV metadata in spool replay, and preserve explicit plaintext and bootstrap exceptions. Fixes #46.
|
|
20
|
+
- Automatically upgrade historical raw JSONL spool records to durable wire envelopes before replay; preserve routing, tenant, encryption intent and false/zero metadata, replace stale IVs, and reuse stored ciphertext across interrupted replay. Retry key readiness and late-loaded exchange classes without a manual trigger. Retain original data on failed or interrupted conversion and hold file ownership across atomic replacement.
|
|
21
|
+
- Keep spool claim cleanup safe when shutdown interrupts draining or conversion before the progress offset or original handle is initialized.
|
|
22
|
+
- Complete tenant queue and DLQ data migration with explicit/registered manifests, nonmutating plans, exclusive source locks, tenant-conflict checks, byte-preserving routed confirmation before source acknowledgement, bounded resumable progress, and an authenticated-host HTTP endpoint.
|
|
23
|
+
- Quarantine malformed and missing/unroutable-destination spool records without blocking healthy work. Add durable inspect/repair/retry APIs, stable recovery IDs, per-record cross-process ownership, atomic ready-state persistence, and delivered tombstones that prevent stale journal markers from resurrecting completed repairs.
|
|
24
|
+
- Recover historical Dynamic destinations from saved function metadata, scope literal legacy exchanges by the saved tenant, and retry unavailable encryption/extension metadata independently of healthy records. Preserve ciphertext and IV during destination repair.
|
|
25
|
+
- Use Bunny's version-aware acknowledgement/rejection APIs during tenant migration, wait for asynchronous requeue visibility, and preserve structured failures when tenant normalization raises without repeating coercion during logging.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
- Direct `Local.subscribe` uses `exchange_name:` for named exchanges; its default is the default exchange, not an unscoped routing-key subscription.
|
|
29
|
+
- New spool records preserve encoded payload bytes and the full envelope. Historical unversioned records upgrade automatically during replay; capacity exhaustion rejects new writes instead of evicting pending data.
|
|
30
|
+
- Standardize changed keyword-based transport methods with anonymous `**` when extra context is unused, preserve intentional keyword forwarding, and use kwargs for internal helpers without migrating existing application-facing positional APIs.
|
|
31
|
+
|
|
5
32
|
## [1.4.30] - 2026-07-31
|
|
6
33
|
|
|
7
34
|
### Fixed
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Legion::Transport is the Ruby gem responsible for connecting LegionIO to its FIFO queue system (RabbitMQ over AMQP 0.9.1). It provides thread-safe connection management, exchange/queue abstractions, message publishing with optional encryption, and consumer wrappers.
|
|
4
4
|
|
|
5
|
-
**Version**: 1.4.
|
|
5
|
+
**Version**: 1.4.31
|
|
6
6
|
|
|
7
7
|
## Features
|
|
8
8
|
|
|
@@ -36,6 +36,8 @@ gem 'legion-transport'
|
|
|
36
36
|
|
|
37
37
|
## Usage
|
|
38
38
|
|
|
39
|
+
Keyword-based recovery, spooling, and tenant-migration APIs accept unrelated keyword context without rejecting the call; named option splats still consume or forward their options. Existing positional constructor and message-publish interfaces remain supported.
|
|
40
|
+
|
|
39
41
|
### Basic Connection
|
|
40
42
|
|
|
41
43
|
```ruby
|
|
@@ -91,6 +93,60 @@ result[:accepted] # => true/false
|
|
|
91
93
|
|
|
92
94
|
Options merge with the message's defaults — you can override `routing_key:` or `persistent:` per-publish without constructing a new message.
|
|
93
95
|
|
|
96
|
+
Mandatory publishing waits for publisher confirmation before reporting routed acceptance. Confirm waits use Bunny's zero-argument API and retire a timed-out channel so late acknowledgments cannot contaminate the next send. A timeout is an unknown delivery outcome, not proof that the broker rejected the message.
|
|
97
|
+
|
|
98
|
+
### Spooling and automatic replay
|
|
99
|
+
|
|
100
|
+
Connection failures can persist the already encoded wire payload, complete AMQP envelope, effective broker destination, and tenant identity. `:spooled` means the disk write succeeded; a full spool or failed write returns `:failed`.
|
|
101
|
+
|
|
102
|
+
- Startup, native recovery, and forced recovery schedule one asynchronous replay worker. Network operations run outside spool-write and connection locks.
|
|
103
|
+
- Replay uses the original ciphertext and IV, checks the stored tenant's encoded-byte quota, and removes records only after routed broker confirmation.
|
|
104
|
+
- Delivery is **at least once**: interruption after broker acceptance but before confirmation is recorded can cause redelivery. Consumers must tolerate duplicates.
|
|
105
|
+
- Concurrent appends are not part of a drainer's claimed file. Shutdown retains the unconfirmed tail for the next startup.
|
|
106
|
+
- Configure `Legion::Settings[:transport][:spool][:directory]` or pass `directory:` to `Spool.setup`; the default is `~/.legionio/spool`. Capacity limits reject new writes rather than deleting pending messages.
|
|
107
|
+
- Manual `Spool.drain` callbacks acknowledge a record only by returning literal `true`.
|
|
108
|
+
|
|
109
|
+
Older unversioned JSONL files upgrade automatically during replay; no migration command or opt-in is required. Their raw payloads are encoded once, with saved routing, tenant and envelope metadata preserved. Saved encryption flags and evidence of an earlier encrypted attempt take precedence over the current encryption default; explicit plaintext and cluster-secret bootstrap requests remain supported. A stale saved IV is replaced with the IV for the newly encoded bytes, not reused.
|
|
110
|
+
|
|
111
|
+
The claimed file is streamed to a locked replacement, flushed and fsynced, then atomically renamed before any upgraded record is published. Interrupted conversion retains the original data; interruption after conversion replays the stored wire bytes without re-encrypting. Missing key readiness or extension classes still loading at startup are retried automatically. Concurrent appends and existing versioned records remain intact.
|
|
112
|
+
|
|
113
|
+
Incomplete destinations, malformed records, and broker-rejected destinations are moved into durable per-record recovery storage under `spool/recovery/`; they do not block healthy records. Missing crypto or extension metadata stays `waiting` and retries automatically. Saved Dynamic function metadata is used when available; a destination that was never recorded is supplied through the repair API, never guessed.
|
|
114
|
+
|
|
115
|
+
#### Inspect, repair, and replay
|
|
116
|
+
|
|
117
|
+
```ruby
|
|
118
|
+
spool = Legion::Transport::Spool
|
|
119
|
+
spool.status # pending, quarantined, waiting, delivered_recoveries
|
|
120
|
+
entry = spool.recovery_entries(status: 'quarantined', limit: 100, offset: 0).first
|
|
121
|
+
original = spool.recovery_record(id: entry.fetch(:id))
|
|
122
|
+
# original[:record] is the parsed original; original[:raw_line] is exact original bytes in Base64.
|
|
123
|
+
spool.repair_record(id: entry.fetch(:id), changes: { exchange: 'task', routing_key: 'worker.run' })
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Repair atomically persists a `ready` wire record and schedules confirmed replay. Destination-only repairs preserve existing ciphertext and IV. Supported changes are `exchange`, `routing_key`, `tenant_id`, `exchange_type`, `payload`, `headers`, and `encrypt`. Changing a wire record's encryption requires an explicit raw `payload`; changing its tenant requires an explicit `exchange`. A completely malformed record can be repaired by supplying its raw payload, exchange, and routing key. Validation/encryption errors leave the stored record unchanged.
|
|
127
|
+
|
|
128
|
+
After broker acceptance the entry becomes `delivered`, retains an idempotence tombstone, and releases its stored payload. Delivered entries cannot be repaired/re-published. `spool.retry_recovery(id: ...)` retries the retained destination after external topology/metadata repair; retries preserve earlier repairs and are idempotent after delivery. Per-record cross-process locks serialize repair and replay without blocking unrelated inspection or spool writes.
|
|
129
|
+
|
|
130
|
+
The host Legion API exposes the same operations under its existing authentication:
|
|
131
|
+
|
|
132
|
+
| Method | Path | Result |
|
|
133
|
+
|--------|------|--------|
|
|
134
|
+
| GET | `/api/transport/spool` | Pending and recovery-state counts |
|
|
135
|
+
| GET | `/api/transport/spool/recovery?status=quarantined&limit=100&offset=0` | Metadata-only page; no message payloads |
|
|
136
|
+
| GET | `/api/transport/spool/recovery/:id` | Explicit original-record/raw-byte inspection |
|
|
137
|
+
| POST | `/api/transport/spool/recovery/:id/repair` | JSON repair fields; `202` means durable and queued, not yet delivered |
|
|
138
|
+
| POST | `/api/transport/spool/recovery/:id/retry` | `202` ready/waiting, `200` already delivered, `422` still quarantined |
|
|
139
|
+
|
|
140
|
+
Unknown IDs return `404`; invalid repairs return `422`; unavailable encryption/storage returns `503`. `pending` includes waiting/ready recovery work but excludes quarantined records and delivered tombstones. Ordinary wire replay and repaired replay remain at least once across a crash between broker acceptance and durable local completion.
|
|
141
|
+
|
|
142
|
+
### Encryption policy
|
|
143
|
+
|
|
144
|
+
`transport.messages.encrypt: true` or `encrypt: true` requires a ready cluster secret. An unready key raises `Legion::Transport::EncryptionNotReady` before publishing or spooling plaintext. Encryption itself is bounded by `crypt.cluster_secret_timeout` (five seconds when unset). Explicit `encrypt: false` and the cluster-secret bootstrap message remain deliberately unencrypted.
|
|
145
|
+
|
|
146
|
+
### HTTP publish outcomes
|
|
147
|
+
|
|
148
|
+
`POST /api/transport/publish` requests mandatory routing and publisher confirmation. Responses distinguish accepted (`201`), spooled/deferred (`202`), unroutable (`422`), confirm timeout (`504`), and failed/nacked (`503`) results. `published: true` is returned only for accepted publication; unexpected exceptions return an error response.
|
|
149
|
+
|
|
94
150
|
### Creating a Queue
|
|
95
151
|
|
|
96
152
|
```ruby
|
|
@@ -125,7 +181,67 @@ exchange.publish(payload, routing_key: 'task.my_runner.my_function')
|
|
|
125
181
|
}
|
|
126
182
|
```
|
|
127
183
|
|
|
128
|
-
Supported server keys: `host:` (string), `hosts:` (array), `server:` (string), `servers:` (array). All are merged and deduped. Port 5672 is injected where omitted.
|
|
184
|
+
Supported server keys: `host:` (string), `hosts:` (array), `server:` (string), `servers:` (array). All are merged and deduped. Port 5672 is injected where omitted. Connection setup attempts each configured host, including the AMQP handshake, and closes failed attempts before moving to the next host.
|
|
185
|
+
|
|
186
|
+
### Pooling and recovery
|
|
187
|
+
|
|
188
|
+
- Set `transport.connection_pool_size` above `1` to enable lazy session pooling. Publisher channels are reused per thread and tracked for cleanup.
|
|
189
|
+
- Pool checkout uses one monotonic deadline for waiting and connection creation. Shutdown wakes blocked callers with `Legion::Transport::PoolClosed`; a late factory result is closed instead of returning to the pool.
|
|
190
|
+
- `Connection.shutdown` shares a five-second teardown deadline across owned sessions, closes their sockets, and stops consumer workers. Channel or session creation racing with shutdown cannot install stale resources.
|
|
191
|
+
- `Connection.force_reconnect` permits one active reconnect. It restores existing consumer/channel/queue objects, declarations, bindings, and QoS before running `on_force_reconnect` callbacks. Cancelled consumers remain cancelled, and repeated reconnects reuse publisher channels.
|
|
192
|
+
- Restoration failures return `false`, close the failed replacement, and retain the subscription snapshot for a later reconnect attempt. Restoration refuses to rebind channels while old consumer workers remain alive.
|
|
193
|
+
- Recovery snapshots have their own five-second deadline. If an old Bunny channel/consumer lock is stuck, reconnect returns `false` without detaching the old session; release the lock and retry.
|
|
194
|
+
- Shutdown invalidates the entire in-flight reconnect generation. A cancelled snapshot, replacement attempt, or cleanup cannot reopen transport, tear down a newer session, or clear a newer reconnect owner's guard.
|
|
195
|
+
- Intentional teardown stops socket readers before closing their sockets, avoiding Bunny 2.x asynchronous network errors in the session-owning thread. An open lite-mode connection needs no network recovery and retains its consumers during `force_reconnect`.
|
|
196
|
+
|
|
197
|
+
### Tenant topology and quotas
|
|
198
|
+
|
|
199
|
+
Enable `transport.tenant_topology.enabled` to scope ordinary queue/exchange declarations, bindings, and DLXs to the active `Legion::TenantContext.current` identity. The default format is `t.<tenant_id>.<name>`; disabled topology and the `default` tenant retain unprefixed names. Configured shared exchanges remain shared. Exchange caches include tenant, effective name, declaration options, and explicit channel context.
|
|
200
|
+
|
|
201
|
+
`TenantProvisioner.provision(tenant_id)` creates the same `task`, `results`, and `events` resources and corresponding dead-letter topology used by producers and consumers. Message and helper publishing enforce per-tenant message/encoded-byte limits; replay preserves the stored tenant instead of using its worker's ambient context.
|
|
202
|
+
|
|
203
|
+
Queue, exchange, and DLX mismatches fail without deleting existing topology or messages. Required DLX provisioning errors prevent source-queue construction.
|
|
204
|
+
|
|
205
|
+
#### Migrating existing unprefixed queues and data
|
|
206
|
+
|
|
207
|
+
`TenantMigration.migrate` plans and transfers existing queue **and dead-letter** data into tenant-scoped topology. It preserves payload bytes, cipher/IV, AMQP properties and routing keys, and acknowledges each source message only after mandatory routed confirmation at its target. A dedicated per-source transfer exchange prevents delivery to unrelated queues sharing the destination exchange. The original exchange is retained in `x-legion-original-exchange`.
|
|
208
|
+
|
|
209
|
+
Pause legacy producers and consumers for the selected queues, enable tenant topology, and specify the owning tenant. Active source consumers and contradictory tenant headers/plaintext metadata are refused; ambient thread context never establishes migration ownership.
|
|
210
|
+
|
|
211
|
+
```ruby
|
|
212
|
+
manifest = [{
|
|
213
|
+
name: 'orders.jobs',
|
|
214
|
+
options: { durable: true, arguments: { 'x-queue-type' => 'quorum',
|
|
215
|
+
'x-dead-letter-exchange' => 'orders.dlx' } },
|
|
216
|
+
bindings: [{ exchange: 'orders', exchange_type: 'topic', routing_key: 'orders.#' }]
|
|
217
|
+
}]
|
|
218
|
+
|
|
219
|
+
plan = Legion::Transport::TenantMigration.migrate(
|
|
220
|
+
tenant_id: 'alpha', queues: manifest, dry_run: true
|
|
221
|
+
)
|
|
222
|
+
result = Legion::Transport::TenantMigration.migrate(
|
|
223
|
+
tenant_id: 'alpha', queues: manifest, max_messages: 10_000, timeout: 30
|
|
224
|
+
)
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
Dry runs inspect source counts/consumers and resolve names without consuming or creating topology. Execution declares all targets and bindings before transfer, migrates the associated `orders.dlx.queue`, and reports per-queue source/target names, moved/remaining counts, status, and errors. Repeat a `pending` result to resume until `completed`; verify all source counts are zero before resuming tenant-scoped producers/consumers. Sources and their bindings are never deleted. Producers must remain paused during cutover so they cannot repopulate legacy queues after verification.
|
|
228
|
+
|
|
229
|
+
Omit `queues:` when the process has already instantiated and bound the tenant's real queue wrappers; declarations and bindings are registered automatically and survive wrapper recreation. After a process restart, or for passive-only declarations, supply the manifest or instantiate/bind those wrappers again. Specify non-default exchange types/options and actual queue arguments; unavailable source metadata is not invented. Shared DLQs and connection-exclusive targets are refused rather than assigned to an arbitrary tenant.
|
|
230
|
+
|
|
231
|
+
`POST /api/transport/tenants/:tenant_id/migrate` accepts the same JSON fields (`queues`, `dry_run`, `max_messages`, `timeout`). Responses are `200` planned/completed, `202` pending, `409` ownership/consumer/topology conflicts, `422` invalid input, or `503` failure. The API inherits the host's authentication.
|
|
232
|
+
|
|
233
|
+
Broker locks exclude concurrent migrations of the same source. Interrupted/unconfirmed transfers are requeued; a repeat invocation resumes the remaining source messages. Delivery is at least once if the target accepted before source acknowledgement was interrupted. Count/time bounds, encoded-byte quotas, and bounded owned-session teardown apply. Lite mode has no persisted broker topology and explicitly rejects this operation.
|
|
234
|
+
|
|
235
|
+
### Lite delivery
|
|
236
|
+
|
|
237
|
+
Lite mode supports exchange-scoped direct, fanout, and topic bindings (`*` matches one word; `#` matches zero or more). Only undelivered messages remain pending; callbacks run outside the global storage mutex, and cancellation unregisters callbacks before completing. Session shutdown retires its own queues and consumers without resetting unrelated sessions.
|
|
238
|
+
|
|
239
|
+
Direct `Local.subscribe` callers must specify `exchange_name:` for named exchanges; omitting it subscribes only to the default exchange:
|
|
240
|
+
|
|
241
|
+
```ruby
|
|
242
|
+
Legion::Transport::Local.subscribe('worker.run', exchange_name: 'events') { |payload| puts payload }
|
|
243
|
+
Legion::Transport::Local.publish('events', 'worker.run', 'work')
|
|
244
|
+
```
|
|
129
245
|
|
|
130
246
|
## Configuration
|
|
131
247
|
|
|
@@ -149,7 +265,7 @@ Configuration is managed through `legion-settings` with environment variable ove
|
|
|
149
265
|
{
|
|
150
266
|
"type": "rabbitmq",
|
|
151
267
|
"connected": false,
|
|
152
|
-
"logger_level": "
|
|
268
|
+
"logger_level": "warn",
|
|
153
269
|
"messages": {
|
|
154
270
|
"encrypt": false,
|
|
155
271
|
"ttl": null,
|
|
@@ -171,8 +287,7 @@ Configuration is managed through `legion-settings` with environment variable ove
|
|
|
171
287
|
"block": false,
|
|
172
288
|
"auto_delete": false,
|
|
173
289
|
"arguments": {
|
|
174
|
-
"x-
|
|
175
|
-
"x-overflow": "reject-publish"
|
|
290
|
+
"x-queue-type": "quorum"
|
|
176
291
|
}
|
|
177
292
|
},
|
|
178
293
|
"connection": {
|
|
@@ -189,7 +304,7 @@ Configuration is managed through `legion-settings` with environment variable ove
|
|
|
189
304
|
"port": 5672,
|
|
190
305
|
"vhost": "/",
|
|
191
306
|
"recovery_attempts": 10,
|
|
192
|
-
"logger_level": "
|
|
307
|
+
"logger_level": "warn",
|
|
193
308
|
"connected": false
|
|
194
309
|
},
|
|
195
310
|
"channel": {
|
|
@@ -228,10 +343,12 @@ TLS can be configured through transport settings:
|
|
|
228
343
|
tls_key: '/path/to/key.pem',
|
|
229
344
|
ca_certs: '/path/to/ca.pem',
|
|
230
345
|
verify_peer: true,
|
|
231
|
-
|
|
346
|
+
vault_pki: false
|
|
232
347
|
}
|
|
233
348
|
```
|
|
234
349
|
|
|
350
|
+
Vault-issued PEM files remain owned by their Bunny session through native recovery and are explicitly removed when that session is retired or connection creation fails.
|
|
351
|
+
|
|
235
352
|
## Dependencies
|
|
236
353
|
|
|
237
354
|
| Gem | Version | Purpose |
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'timeout'
|
|
4
|
+
|
|
5
|
+
module Legion
|
|
6
|
+
module Transport
|
|
7
|
+
module Connection
|
|
8
|
+
# A forced reconnect replaces the session, unlike Bunny's automatic recovery.
|
|
9
|
+
# Keep the original channels: actors retain queues, consumers and channel references.
|
|
10
|
+
# Capture before teardown; restore only after the old transport and workers stop.
|
|
11
|
+
class Recovery
|
|
12
|
+
def self.capture(sessions:, excluded_channels: [], timeout: Legion::Transport::Connection::SHUTDOWN_TIMEOUT, **)
|
|
13
|
+
Timeout.timeout(timeout) do
|
|
14
|
+
new(sessions: sessions.compact.uniq, excluded_channels: excluded_channels)
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def initialize(sessions:, excluded_channels: [], **)
|
|
19
|
+
@excluded_channels = excluded_channels
|
|
20
|
+
@registered_sessions = {}
|
|
21
|
+
@states = sessions.filter_map { |session| capture_session(session: session) }
|
|
22
|
+
@mutex = Mutex.new
|
|
23
|
+
@restored = false
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def restore(session:, **)
|
|
27
|
+
@mutex.synchronize do
|
|
28
|
+
return true if @restored || @states.empty?
|
|
29
|
+
raise IOError, 'replacement session is not open' unless session&.open?
|
|
30
|
+
raise IOError, 'partially restored session must be torn down before retrying' if @attempted_session.equal?(session)
|
|
31
|
+
|
|
32
|
+
ensure_quiescent_sources!
|
|
33
|
+
@attempted_session = session
|
|
34
|
+
active = @states.to_h { |state| [state, active_consumers(state: state)] }
|
|
35
|
+
@states.each do |state|
|
|
36
|
+
state[:channels].each { |channel| restore_channel(channel: channel, session: session) }
|
|
37
|
+
end
|
|
38
|
+
restore_topology(active: active)
|
|
39
|
+
active.each do |state, consumers|
|
|
40
|
+
consumers.each do |consumer|
|
|
41
|
+
consumer.consumer_tag = state[:consumer_tags].fetch(consumer)
|
|
42
|
+
consumer.channel.basic_consume_with(consumer)
|
|
43
|
+
@registered_sessions[consumer] = session
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
@restored = true
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def capture_session(session:, **)
|
|
53
|
+
# Network recovery closes channels without unregistering them. Explicit
|
|
54
|
+
# Channel#close unregisters them, so retained closed channels still need recovery.
|
|
55
|
+
channels = session.instance_variable_get(:@channel_mutex).synchronize do
|
|
56
|
+
session.instance_variable_get(:@channels).values.reject do |channel|
|
|
57
|
+
channel.number.zero? || @excluded_channels.include?(channel)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
return if channels.empty?
|
|
61
|
+
|
|
62
|
+
registrations = channels.flat_map do |channel|
|
|
63
|
+
channel.instance_variable_get(:@consumer_mutex).synchronize { channel.consumers.to_a }
|
|
64
|
+
end
|
|
65
|
+
consumers = registrations.map(&:last)
|
|
66
|
+
queues = (channels.flat_map { |channel| channel.queues.values } + consumers.map(&:queue).grep(::Bunny::Queue)).uniq
|
|
67
|
+
exchanges = channels.flat_map { |channel| channel.exchanges.values }.uniq
|
|
68
|
+
state = { session: session, channels: channels, consumers: consumers,
|
|
69
|
+
consumer_names: consumers.to_h { |consumer| [consumer, consumer.queue_name] },
|
|
70
|
+
consumer_tags: registrations.to_h { |tag, consumer| [consumer, tag] },
|
|
71
|
+
queues: [], exchanges: [], queue_bindings: [], exchange_bindings: [] }
|
|
72
|
+
if session.respond_to?(:topology_registry)
|
|
73
|
+
capture_registry(state: state, registry: session.topology_registry, queues: queues, exchanges: exchanges)
|
|
74
|
+
else
|
|
75
|
+
capture_objects(state: state, queues: queues, exchanges: exchanges)
|
|
76
|
+
end
|
|
77
|
+
state
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def capture_registry(state:, registry:, queues:, exchanges:, **)
|
|
81
|
+
queues_by_name = queues.group_by(&:name)
|
|
82
|
+
exchanges_by_name = exchanges.to_h { |exchange| [exchange.name, exchange] }
|
|
83
|
+
registry.filtered_queues.each do |record|
|
|
84
|
+
objects = queues_by_name[record.name] || []
|
|
85
|
+
options = if objects.empty?
|
|
86
|
+
{ durable: record.durable, exclusive: record.exclusive, auto_delete: record.auto_delete, arguments: record.arguments }
|
|
87
|
+
else
|
|
88
|
+
objects.first.options.dup
|
|
89
|
+
end
|
|
90
|
+
state[:queues] << { name: record.name, server_named: record.server_named?, options: options, objects: objects,
|
|
91
|
+
channel: recovery_channel(state: state, channel: objects.first&.channel || record.channel) }
|
|
92
|
+
end
|
|
93
|
+
registry.filtered_exchanges.reject(&:predeclared?).each do |record|
|
|
94
|
+
object = exchanges_by_name[record.name]
|
|
95
|
+
options = if object
|
|
96
|
+
object.instance_variable_get(:@options).dup
|
|
97
|
+
else
|
|
98
|
+
{ durable: record.durable, auto_delete: record.auto_delete, arguments: record.arguments }
|
|
99
|
+
end
|
|
100
|
+
state[:exchanges] << { name: record.name, type: record.type, options: options,
|
|
101
|
+
channel: recovery_channel(state: state, channel: object&.channel || record.channel) }
|
|
102
|
+
end
|
|
103
|
+
capture_registry_bindings(state: state, registry: registry)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def capture_registry_bindings(state:, registry:, **)
|
|
107
|
+
registry.filtered_queue_bindings.each do |binding|
|
|
108
|
+
state[:queue_bindings] << { queue: binding.destination, exchange: binding.source,
|
|
109
|
+
options: { routing_key: binding.routing_key, arguments: binding.arguments },
|
|
110
|
+
channel: recovery_channel(state: state, channel: binding.channel) }
|
|
111
|
+
end
|
|
112
|
+
registry.filtered_exchange_bindings.each do |binding|
|
|
113
|
+
state[:exchange_bindings] << { source: binding.source, destination: binding.destination,
|
|
114
|
+
options: { routing_key: binding.routing_key, arguments: binding.arguments },
|
|
115
|
+
channel: recovery_channel(state: state, channel: binding.channel) }
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def capture_objects(state:, queues:, exchanges:, **)
|
|
120
|
+
queues.group_by(&:name).each do |name, objects|
|
|
121
|
+
queue = objects.first
|
|
122
|
+
state[:queues] << { name: name, server_named: queue.server_named?, options: queue.options.dup,
|
|
123
|
+
objects: objects, channel: queue.channel }
|
|
124
|
+
objects.each do |object|
|
|
125
|
+
object.instance_variable_get(:@bindings).each do |binding|
|
|
126
|
+
state[:queue_bindings] << { queue: name, exchange: binding[:exchange], channel: object.channel,
|
|
127
|
+
options: { routing_key: binding[:routing_key], arguments: binding[:arguments] } }
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
exchanges.each do |exchange|
|
|
132
|
+
unless exchange.predefined?
|
|
133
|
+
state[:exchanges] << { name: exchange.name, type: exchange.type, channel: exchange.channel,
|
|
134
|
+
options: exchange.instance_variable_get(:@options).dup }
|
|
135
|
+
end
|
|
136
|
+
exchange.instance_variable_get(:@bindings).each do |binding|
|
|
137
|
+
source = binding[:source]
|
|
138
|
+
state[:exchange_bindings] << { source: source.respond_to?(:name) ? source.name : source,
|
|
139
|
+
destination: exchange.name, channel: exchange.channel, options: binding[:opts].dup }
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
state[:queue_bindings].uniq!
|
|
143
|
+
state[:exchange_bindings].uniq!
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def recovery_channel(state:, channel:, **)
|
|
147
|
+
state[:channels].include?(channel) ? channel : state[:channels].first
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def active_consumers(state:, **)
|
|
151
|
+
# Channel#close retains the consumer map, whereas Basic.CancelOk removes it.
|
|
152
|
+
# Recheck after teardown so a cancellation after capture is not resurrected.
|
|
153
|
+
state[:consumers].select do |consumer|
|
|
154
|
+
tag = state[:consumer_tags].fetch(consumer)
|
|
155
|
+
source = @registered_sessions.fetch(consumer, state[:session])
|
|
156
|
+
registered = !source.respond_to?(:topology_registry) || source.topology_registry.consumers.key?(tag)
|
|
157
|
+
registered && consumer.channel.consumers[tag].equal?(consumer)
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def ensure_quiescent_sources!
|
|
162
|
+
channels = @states.flat_map { |state| state[:channels] }
|
|
163
|
+
channels.map(&:connection).uniq.each do |source|
|
|
164
|
+
reader = source.instance_variable_get(:@reader_loop)&.instance_variable_get(:@thread)
|
|
165
|
+
raise IOError, 'old session transport or reader is still running' if source.transport.open? || reader&.alive?
|
|
166
|
+
end
|
|
167
|
+
# Bunny 3.x delivers plain integer tags. A callback from an old work pool
|
|
168
|
+
# could acknowledge an unrelated new delivery if allowed to survive rebinding.
|
|
169
|
+
return unless channels.any? { |channel| Array(channel.work_pool.threads).any?(&:alive?) }
|
|
170
|
+
|
|
171
|
+
raise IOError, 'old consumer workers are still running'
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def restore_channel(channel:, session:, **)
|
|
175
|
+
number = session.next_channel_id
|
|
176
|
+
raise IOError, 'replacement session channel limit exhausted' if number.negative?
|
|
177
|
+
|
|
178
|
+
pool = channel.work_pool
|
|
179
|
+
channel.instance_variable_set(:@connection, session)
|
|
180
|
+
channel.instance_variable_set(:@logger, session.logger)
|
|
181
|
+
channel.instance_variable_set(:@frame_max, session.frame_max)
|
|
182
|
+
channel.id = number
|
|
183
|
+
channel.instance_variable_set(:@work_pool, ::Bunny::ConsumerWorkPool.new(pool.size, pool.abort_on_exception,
|
|
184
|
+
pool.instance_variable_get(:@shutdown_timeout)))
|
|
185
|
+
channel.send(:reset_continuations)
|
|
186
|
+
# Bunny 2.x uses versioned delivery tags; advance before accepting deliveries.
|
|
187
|
+
channel.increment_recoveries_counter if channel.respond_to?(:increment_recoveries_counter)
|
|
188
|
+
channel.open
|
|
189
|
+
channel.recover_prefetch_setting
|
|
190
|
+
channel.recover_confirm_mode
|
|
191
|
+
channel.recover_tx_mode
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def restore_topology(active:, **)
|
|
195
|
+
# Bunny's aggregate recovery either swallows declaration errors (2.x) or
|
|
196
|
+
# replaces consumers and loses the original queue references (3.x). Use its
|
|
197
|
+
# recorded topology and protocol APIs, but let every failure reach the caller.
|
|
198
|
+
active.each_key do |state|
|
|
199
|
+
state[:exchanges].each do |exchange|
|
|
200
|
+
next if exchange[:options][:no_declare]
|
|
201
|
+
|
|
202
|
+
exchange[:channel].exchange_declare(exchange[:name], exchange[:type], exchange[:options])
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
names = active.to_h do |state, _consumers|
|
|
206
|
+
[state, state[:queues].to_h { |queue| [queue[:name], restore_queue(queue: queue)] }]
|
|
207
|
+
end
|
|
208
|
+
active.each do |state, consumers|
|
|
209
|
+
state[:exchange_bindings].each do |binding|
|
|
210
|
+
binding[:channel].exchange_bind(binding[:source], binding[:destination], binding[:options])
|
|
211
|
+
end
|
|
212
|
+
state[:queue_bindings].each do |binding|
|
|
213
|
+
binding[:channel].queue_bind(names[state].fetch(binding[:queue], binding[:queue]), binding[:exchange], binding[:options])
|
|
214
|
+
end
|
|
215
|
+
consumers.each do |consumer|
|
|
216
|
+
next if consumer.queue.respond_to?(:name)
|
|
217
|
+
|
|
218
|
+
original_name = state[:consumer_names].fetch(consumer)
|
|
219
|
+
consumer.instance_variable_set(:@queue, names[state].fetch(original_name, original_name))
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def restore_queue(queue:, **)
|
|
225
|
+
name = queue[:name]
|
|
226
|
+
unless queue[:options][:no_declare]
|
|
227
|
+
requested_name = queue[:server_named] ? '' : name
|
|
228
|
+
name = queue[:channel].queue_declare(requested_name, queue[:options]).queue
|
|
229
|
+
end
|
|
230
|
+
queue[:objects].each do |object|
|
|
231
|
+
object.channel.deregister_queue(object)
|
|
232
|
+
object.instance_variable_set(:@name, name)
|
|
233
|
+
object.channel.register_queue(object)
|
|
234
|
+
object.channel.record_queue(object) if object.channel.respond_to?(:record_queue)
|
|
235
|
+
end
|
|
236
|
+
name
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
@@ -11,22 +11,25 @@ module Legion
|
|
|
11
11
|
|
|
12
12
|
# Provides Vault PKI-based cert issuance for Bunny mTLS connections.
|
|
13
13
|
# Activated when `transport.tls.vault_pki: true` AND `Legion::Crypt::Mtls.enabled?`.
|
|
14
|
-
#
|
|
15
|
-
# by the OS when the process exits — or call cleanup_pki_tempfiles explicitly).
|
|
14
|
+
# Keep PEM owners alive until their session is retired, including native recovery.
|
|
16
15
|
|
|
17
|
-
def vault_pki_tls_options
|
|
16
|
+
def vault_pki_tls_options(tempfiles: nil, **)
|
|
18
17
|
return {} unless vault_pki_enabled?
|
|
19
18
|
return {} unless defined?(Legion::Crypt::Mtls)
|
|
20
19
|
return {} unless Legion::Crypt::Mtls.enabled?
|
|
21
20
|
|
|
21
|
+
created_files = []
|
|
22
22
|
node_name = pki_node_name
|
|
23
23
|
cert_data = Legion::Crypt::Mtls.issue_cert(common_name: node_name)
|
|
24
24
|
Legion::Transport.logger.info(
|
|
25
25
|
"[mTLS] Issued PKI cert for #{node_name}: serial=#{cert_data[:serial]} expiry=#{cert_data[:expiry]}"
|
|
26
26
|
)
|
|
27
27
|
|
|
28
|
-
build_bunny_tls_opts(cert_data)
|
|
28
|
+
options = build_bunny_tls_opts(cert_data: cert_data, tempfiles: created_files)
|
|
29
|
+
(tempfiles || (@pki_tempfiles ||= [])).concat(created_files)
|
|
30
|
+
options
|
|
29
31
|
rescue StandardError => e
|
|
32
|
+
cleanup_pki_tempfiles(tempfiles: created_files)
|
|
30
33
|
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.vault_pki_tls_options')
|
|
31
34
|
{}
|
|
32
35
|
end
|
|
@@ -43,15 +46,13 @@ module Legion
|
|
|
43
46
|
|
|
44
47
|
private
|
|
45
48
|
|
|
46
|
-
def build_bunny_tls_opts(cert_data)
|
|
47
|
-
cert_file = write_pki_tempfile('legion-cert-',
|
|
48
|
-
key_file
|
|
49
|
-
ca_files
|
|
50
|
-
write_pki_tempfile("legion-ca-#{i}-",
|
|
49
|
+
def build_bunny_tls_opts(cert_data:, tempfiles:, **)
|
|
50
|
+
cert_file = write_pki_tempfile(prefix: 'legion-cert-', content: cert_data[:cert], tempfiles: tempfiles)
|
|
51
|
+
key_file = write_pki_tempfile(prefix: 'legion-key-', content: cert_data[:key], tempfiles: tempfiles)
|
|
52
|
+
ca_files = Array(cert_data[:ca_chain]).map.with_index do |pem, i|
|
|
53
|
+
write_pki_tempfile(prefix: "legion-ca-#{i}-", content: pem, tempfiles: tempfiles)
|
|
51
54
|
end
|
|
52
55
|
|
|
53
|
-
track_pki_tempfiles([cert_file, key_file] + ca_files)
|
|
54
|
-
|
|
55
56
|
{
|
|
56
57
|
tls: true,
|
|
57
58
|
tls_cert: cert_file,
|
|
@@ -61,17 +62,21 @@ module Legion
|
|
|
61
62
|
}
|
|
62
63
|
end
|
|
63
64
|
|
|
64
|
-
def write_pki_tempfile(prefix
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
65
|
+
def write_pki_tempfile(prefix:, content:, tempfiles:, **)
|
|
66
|
+
file = Tempfile.new([prefix, '.pem'])
|
|
67
|
+
tempfiles << file
|
|
68
|
+
file.write(content)
|
|
69
|
+
file.close
|
|
70
|
+
file.path
|
|
70
71
|
end
|
|
71
72
|
|
|
72
|
-
def
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
def cleanup_pki_tempfiles(tempfiles: @pki_tempfiles, **)
|
|
74
|
+
tempfiles&.each do |file|
|
|
75
|
+
file.close!
|
|
76
|
+
rescue StandardError => e
|
|
77
|
+
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.cleanup_pki_tempfiles')
|
|
78
|
+
end
|
|
79
|
+
tempfiles&.clear
|
|
75
80
|
end
|
|
76
81
|
|
|
77
82
|
def pki_node_name
|