legion-transport 1.4.23 → 1.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -5
- data/CLAUDE.md +30 -147
- data/lib/legion/transport/exchange.rb +1 -1
- data/lib/legion/transport/queue.rb +30 -5
- data/lib/legion/transport/queues/agent.rb +1 -1
- data/lib/legion/transport/queues/node.rb +1 -1
- data/lib/legion/transport/settings.rb +1 -1
- data/lib/legion/transport/tenant_provisioner.rb +31 -17
- data/lib/legion/transport/tenant_quota.rb +43 -10
- data/lib/legion/transport/tenant_topology.rb +23 -4
- data/lib/legion/transport/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9dbb99fd575325dfdccd23836ec69ae6fd3b931d3a57c7159cec211b61b7caac
|
|
4
|
+
data.tar.gz: 66c2f33752023d583eaf07ebd8c437863c1355409aff0fa9e5a92d1c7e95455e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 11c1c2451dc9bca677ba8abf01890fe05d4e1692d3cdd7d4c87a9fa953267fd73017b74f7210faddd901da9ac28f28cba4d46e3b2242a6c74a814ab32f170606
|
|
7
|
+
data.tar.gz: 0d95280b4c5d4b147735037962a2cac2cd4f41308a21143ade1f69d1360fc8b9131dbd465a073de8bdb7d5626740d996c330f403d1beac14610390bb337829c5
|
data/CHANGELOG.md
CHANGED
|
@@ -2,12 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
## [1.4.24] - 2026-05-11
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
- `TenantProvisioner#provision` and `#deprovision` now use `Legion::Transport::Connection.channel` (the public API) instead of the non-existent `Legion::Transport.connection.create_channel`. (Closes #15)
|
|
9
|
+
- `TenantProvisioner` now wraps internally-acquired channels in `ensure` blocks so channels are closed even when provision/deprovision raises. (Closes #15)
|
|
10
|
+
- `TenantProvisioner#deprovision` now guards against accidental deletion of global exchanges: skips when topology is disabled, and refuses to delete when `tenant_id` is nil, blank, or `'default'`. (Closes #15)
|
|
11
|
+
- `TenantTopology#shared?` no longer over-matches: previously `start_with?('legion.')` matched `legion.controlled`; now uses exact set membership with explicit dot-separated sub-path check (`name == entry || name.start_with?("#{entry}.")`). (Closes #15)
|
|
12
|
+
- `TenantTopology` now reads `shared_exchanges` and `prefix_format` from `transport.tenant_topology` settings with sensible defaults, instead of hardcoding them. (Closes #15)
|
|
13
|
+
- `TenantQuota` now uses per-tenant mutexes instead of a single global mutex, and sweeps stale counter entries (entries inactive for `STALE_SECONDS = 300`) to prevent unbounded map growth. (Closes #15)
|
|
14
|
+
- DLX exchange declarations now use an isolated channel to prevent cascading failures
|
|
15
|
+
- Added self-healing delete-and-recreate logic for mismatched DLX exchanges (PreconditionFailed)
|
|
16
|
+
- Added `exclusive: true` to Node and Agent queues for RabbitMQ 4.x compatibility (transient_nonexcl_queues deprecation)
|
|
17
|
+
- Fixed `topology_mode?` to check `worker?` instead of `agent?` for exchange/queue declarations
|
|
7
18
|
|
|
8
|
-
### Removed
|
|
9
|
-
- Unnecessary `defined?(Legion::Logging)` guards in routes.rb — legion-logging is a hard gemspec dependency
|
|
10
|
-
- Unnecessary `defined?(Legion::Settings)` and `Legion.const_defined?('Settings')` guards in transport.rb, settings.rb, helper.rb, tenant_quota.rb, and tenant_topology.rb — legion-settings is a hard gemspec dependency
|
|
11
19
|
|
|
12
20
|
## [1.4.23] - 2026-05-05
|
|
13
21
|
|
data/CLAUDE.md
CHANGED
|
@@ -2,109 +2,42 @@ Always run a full `bundle exec rspec` and `bundle exec rubocop -A` and fix all e
|
|
|
2
2
|
|
|
3
3
|
# legion-transport: AMQP Transport Layer for LegionIO
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
- **Parent**: `/Users/miverso2/rubymine/legion/CLAUDE.md`
|
|
7
|
-
|
|
8
|
-
## Purpose
|
|
9
|
-
|
|
10
|
-
Ruby gem that manages the connection between LegionIO and its FIFO queue system (RabbitMQ over AMQP 0.9.1). Provides abstractions for exchanges, queues, messages, and consumers with thread-safe connection management.
|
|
11
|
-
|
|
12
|
-
**GitHub**: https://github.com/LegionIO/legion-transport
|
|
13
|
-
**Version**: 1.4.14
|
|
14
|
-
**License**: Apache-2.0
|
|
5
|
+
RabbitMQ over AMQP 0.9.1. Thread-safe connection management, exchanges, queues, messages, consumers.
|
|
15
6
|
|
|
16
7
|
## Architecture
|
|
17
8
|
|
|
18
9
|
```
|
|
19
10
|
Legion::Transport
|
|
20
|
-
├── Connection # Thread-safe
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
├──
|
|
24
|
-
├──
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
├── Queue # Base queue class (extends Bunny::Queue)
|
|
34
|
-
│ └── Queues/
|
|
35
|
-
│ ├── Node # Node queue
|
|
36
|
-
│ ├── Agent # Per-agent queue (auto-delete, routing key: agent.<agent_id>)
|
|
37
|
-
│ ├── NodeCrypt # Node encryption queue
|
|
38
|
-
│ ├── NodeStatus # Node status queue
|
|
39
|
-
│ ├── TaskLog # Task logging queue
|
|
40
|
-
│ ├── TaskUpdate # Task update queue
|
|
41
|
-
│ └── RegionOutbound # Cross-region outbound queue for mesh routing
|
|
42
|
-
├── Message # Base message class with publish, encode, encrypt
|
|
43
|
-
│ └── Messages/
|
|
44
|
-
│ ├── Task # Task messages with dynamic routing keys
|
|
45
|
-
│ ├── SubTask # Subtask messages (conditions, transforms)
|
|
46
|
-
│ ├── Dynamic # Dynamic function-based messages
|
|
47
|
-
│ ├── CheckSubtask
|
|
48
|
-
│ ├── LexRegister
|
|
49
|
-
│ ├── RequestClusterSecret
|
|
50
|
-
│ ├── TaskLog
|
|
51
|
-
│ └── TaskUpdate
|
|
52
|
-
├── Consumer # AMQP consumer with auto-generated tags
|
|
53
|
-
├── Common # Shared utilities (channel mgmt, options merging, consumer tags)
|
|
54
|
-
├── Helper # Injectable transport mixin for LEX extensions
|
|
55
|
-
├── Local # In-memory pub/sub for local development mode (no RabbitMQ)
|
|
56
|
-
├── Spool # Disk-backed message buffer: persist messages when RabbitMQ unavailable, replay on reconnect
|
|
57
|
-
├── TenantProvisioner # Creates per-tenant exchanges + queues on first publish; idempotent with TTL cache
|
|
58
|
-
├── TenantQuota # Per-tenant rate limiting and message quota enforcement
|
|
59
|
-
├── TenantTopology # Tracks per-tenant exchange/queue topology (discovery, cleanup)
|
|
60
|
-
├── Settings # Default configuration with env var overrides
|
|
61
|
-
└── Version # 1.4.14
|
|
11
|
+
├── Connection # Thread-safe session/channel, SSL, Vault creds
|
|
12
|
+
├── InProcess # Lite mode adapter
|
|
13
|
+
├── Exchange # Topic exchanges: Task, Node, Agent, Crypt, Extensions, Lex, Logging
|
|
14
|
+
├── Queue # Node, Agent, NodeCrypt, NodeStatus, TaskLog, TaskUpdate, RegionOutbound
|
|
15
|
+
├── Message # Publish, encode, encrypt; Task, SubTask, Dynamic, LexRegister, etc.
|
|
16
|
+
├── Consumer # Auto-generated consumer tags
|
|
17
|
+
├── Common # Channel mgmt, options merging
|
|
18
|
+
├── Helper # Injectable transport mixin for LEX extensions
|
|
19
|
+
├── Local # In-memory pub/sub for lite mode
|
|
20
|
+
├── Spool # Disk-backed buffer, replay on reconnect
|
|
21
|
+
├── TenantProvisioner # Per-tenant exchanges/queues on first publish
|
|
22
|
+
├── TenantQuota # Per-tenant rate limiting
|
|
23
|
+
└── TenantTopology # Topology tracking
|
|
62
24
|
```
|
|
63
25
|
|
|
64
26
|
## Key Design Patterns
|
|
65
27
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
`Connection.force_reconnect` performs a socket-first teardown (closes the underlying TCP socket before calling `Bunny::Session#close`) to break stuck connections that don't respond to the normal close protocol. Tracks recovery rate over a 5-window/60s sliding window. Calls registered `on_force_reconnect` callbacks after reconnection. Sets a shutdown flag to prevent reconnection loops during intentional shutdown.
|
|
69
|
-
|
|
70
|
-
```ruby
|
|
71
|
-
Legion::Transport::Connection.force_reconnect
|
|
72
|
-
Legion::Transport::Connection.on_force_reconnect { |info| alert_on_call(info) }
|
|
73
|
-
```
|
|
74
|
-
|
|
75
|
-
### AMQP Client / Lite Mode
|
|
76
|
-
Uses `bunny` gem for AMQP 0.9.1. The entry point sets `Legion::Transport::TYPE` and `Legion::Transport::CONNECTOR` as constants. When `LEGION_MODE=lite` env var is set, `TYPE = 'local'` and `CONNECTOR = InProcess` instead of Bunny. `Connection.lite_mode?` checks `TYPE == 'local'`. `Connection.setup` returns an InProcess session in lite mode, skipping Bunny entirely.
|
|
77
|
-
|
|
78
|
-
### Thread-Safe Connection Management
|
|
79
|
-
- `Concurrent::AtomicReference` wraps the AMQP session (one per process)
|
|
80
|
-
- `Concurrent::ThreadLocalVar` provides per-thread channels
|
|
81
|
-
- Auto-recovery on blocked/unblocked/recovery events
|
|
28
|
+
**Force Reconnect** — `Connection.force_reconnect` performs socket-first teardown (closes TCP socket before `Bunny::Session#close`) to break stuck connections. Tracks recovery rate over a 5-window/60s sliding window. Calls registered `on_force_reconnect` callbacks. Sets shutdown flag to prevent reconnection loops.
|
|
82
29
|
|
|
83
|
-
|
|
84
|
-
`Common#options_builder` deep-merges default options -> class options -> instance options. Used by both Exchange and Queue constructors.
|
|
30
|
+
**Lite Mode** — `LEGION_MODE=lite` sets `TYPE='local'`, `CONNECTOR=InProcess`. `Connection.setup` returns an InProcess session, skipping Bunny entirely.
|
|
85
31
|
|
|
86
|
-
|
|
87
|
-
Both Exchange and Queue classes catch `PreconditionFailed` errors (parameter mismatch with existing RabbitMQ declarations) and attempt to delete + recreate once before raising.
|
|
32
|
+
**Thread-Safe Connections** — `Concurrent::AtomicReference` wraps the AMQP session (one per process). `Concurrent::ThreadLocalVar` provides per-thread channels. Auto-recovery on blocked/unblocked/recovery events.
|
|
88
33
|
|
|
89
|
-
|
|
34
|
+
**Auto-Recreate on Mismatch** — Exchange and Queue classes catch `PreconditionFailed` (parameter mismatch with existing declarations) and delete + recreate once before raising.
|
|
90
35
|
|
|
91
|
-
|
|
92
|
-
|-----|---------|
|
|
93
|
-
| `bunny` (>= 2.23) | CRuby AMQP client |
|
|
94
|
-
| `concurrent-ruby` (>= 1.2) | Thread-safe data structures |
|
|
95
|
-
| `legion-json` | JSON serialization |
|
|
96
|
-
| `legion-settings` | Configuration management |
|
|
97
|
-
|
|
98
|
-
Optional runtime dependencies:
|
|
99
|
-
- `legion-crypt` - Message encryption support
|
|
100
|
-
- `legion-data` - Database models (used by Dynamic/SubTask messages)
|
|
101
|
-
- `legion-logging` - Structured logging
|
|
36
|
+
**Options Merging** — `Common#options_builder` deep-merges default -> class -> instance options for Exchange and Queue constructors.
|
|
102
37
|
|
|
103
38
|
## Configuration
|
|
104
39
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
| Env Var | Default | Description |
|
|
40
|
+
| Setting | Default | Description |
|
|
108
41
|
|---------|---------|-------------|
|
|
109
42
|
| `transport.connection.host` | `127.0.0.1` | RabbitMQ host |
|
|
110
43
|
| `transport.connection.port` | `5672` | RabbitMQ port |
|
|
@@ -116,77 +49,27 @@ Settings are loaded via `Legion::Transport::Settings` with env var overrides:
|
|
|
116
49
|
| `transport.messages.ttl` | `nil` | Message TTL |
|
|
117
50
|
| `transport.messages.persistent` | `true` | Persistent messages |
|
|
118
51
|
|
|
119
|
-
Vault integration:
|
|
120
|
-
|
|
121
|
-
## File Map
|
|
122
|
-
|
|
123
|
-
| Path | Purpose |
|
|
124
|
-
|------|---------|
|
|
125
|
-
| `lib/legion/transport.rb` | Entry point, connector detection (Bunny vs InProcess), logger/settings |
|
|
126
|
-
| `lib/legion/transport/connection.rb` | Session/channel lifecycle (setup, reconnect, shutdown, lite_mode?) |
|
|
127
|
-
| `lib/legion/transport/connection/ssl.rb` | TLS settings module |
|
|
128
|
-
| `lib/legion/transport/connection/vault.rb` | Vault PKI integration (stub) |
|
|
129
|
-
| `lib/legion/transport/in_process.rb` | Lite mode adapter: stub Session, Channel, Exchange, Queue, Consumer |
|
|
130
|
-
| `lib/legion/transport/common.rb` | Shared module (channel access, deep_merge, consumer tags) |
|
|
131
|
-
| `lib/legion/transport/helper.rb` | Injectable transport mixin for LEX extensions |
|
|
132
|
-
| `lib/legion/transport/exchange.rb` | Base Exchange class |
|
|
133
|
-
| `lib/legion/transport/exchanges/agent.rb` | Agent exchange for identity-bound communication |
|
|
134
|
-
| `lib/legion/transport/queue.rb` | Base Queue class |
|
|
135
|
-
| `lib/legion/transport/queues/agent.rb` | Per-agent queue (auto-delete, keyed by agent_id) |
|
|
136
|
-
| `lib/legion/transport/message.rb` | Base Message class with publish/encode/encrypt |
|
|
137
|
-
| `lib/legion/transport/consumer.rb` | AMQP consumer wrapper |
|
|
138
|
-
| `lib/legion/transport/spool.rb` | Disk-backed message buffer (~/.legionio/spool, 10MB/file, 500MB total, 3-day TTL) |
|
|
139
|
-
| `lib/legion/transport/tenant_provisioner.rb` | Per-tenant exchange/queue creation with TTL idempotency cache |
|
|
140
|
-
| `lib/legion/transport/tenant_quota.rb` | Per-tenant rate limiting and message quota enforcement |
|
|
141
|
-
| `lib/legion/transport/tenant_topology.rb` | Per-tenant topology tracking (discovery, cleanup) |
|
|
142
|
-
| `lib/legion/transport/queues/region_outbound.rb` | Cross-region outbound queue for mesh routing |
|
|
143
|
-
| `lib/legion/transport/settings.rb` | Default config, env var loading, host resolution |
|
|
144
|
-
| `lib/legion/transport/version.rb` | Version constant |
|
|
145
|
-
| `spec/` | RSpec test suite |
|
|
52
|
+
Vault integration: credentials via `lease://rabbitmq#username` URI refs; lease path configured in `crypt.vault.leases.rabbitmq.path`.
|
|
146
53
|
|
|
147
54
|
## Node vs Agent Exchange
|
|
148
55
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
|
152
|
-
|
|
153
|
-
| `node` | `node.<fqdn/nodename>` | Infrastructure: swarm coordination, service heartbeats, non-identity traffic |
|
|
154
|
-
| `agent` | `agent.<agent_id>` | Identity-bound: GAIA cognitive frames, preference queries, proactive messages |
|
|
155
|
-
|
|
156
|
-
**Agent queue defaults** differ from standard queues:
|
|
157
|
-
- `durable: false` — agent queues are ephemeral (recreated on connect)
|
|
158
|
-
- `auto_delete: true` — cleaned up when the agent disconnects
|
|
159
|
-
- Dead letter exchange: `agent.dlx`
|
|
160
|
-
- Agent ID defaults to `Legion::Settings['client']['name']` if not provided
|
|
56
|
+
| Exchange | Routing Key | Use Case |
|
|
57
|
+
|----------|-------------|----------|
|
|
58
|
+
| `node` | `node.<fqdn>` | Infrastructure: swarm coordination, heartbeats |
|
|
59
|
+
| `agent` | `agent.<agent_id>` | Identity-bound: GAIA frames, preferences, proactive messages |
|
|
161
60
|
|
|
162
|
-
|
|
61
|
+
Agent queue differences: `durable:false`, `auto_delete:true`, DLX: `agent.dlx`. Agent ID defaults to `Legion::Settings['client']['name']`.
|
|
163
62
|
|
|
164
63
|
## Queue Defaults
|
|
165
64
|
|
|
166
|
-
|
|
167
|
-
- `
|
|
168
|
-
- `manual_ack: true` - explicit acknowledgment required
|
|
169
|
-
- `x-max-priority: 255` - priority queue support
|
|
170
|
-
- `x-overflow: reject-publish` - rejects new messages when full
|
|
65
|
+
- `durable: true`, `manual_ack: true`
|
|
66
|
+
- `x-max-priority: 255`, `x-overflow: reject-publish`
|
|
171
67
|
- Dead letter exchange: `<exchange>.dlx`
|
|
172
68
|
|
|
173
69
|
## Exchange Defaults
|
|
174
70
|
|
|
175
|
-
|
|
176
|
-
- `type: topic` - supports routing key pattern matching
|
|
177
|
-
- `durable: true` - survives broker restart
|
|
178
|
-
- `auto_delete: false` - persists when no queues bound
|
|
71
|
+
- `type: topic`, `durable: true`, `auto_delete: false`
|
|
179
72
|
|
|
180
73
|
## Testing
|
|
181
74
|
|
|
182
|
-
|
|
183
|
-
bundle install
|
|
184
|
-
bundle exec rspec
|
|
185
|
-
bundle exec rubocop
|
|
186
|
-
```
|
|
187
|
-
|
|
188
|
-
Spec count: 448+ examples (force_reconnect, recovery tracking, tenant provisioner specs added in v1.4.x)
|
|
189
|
-
|
|
190
|
-
---
|
|
191
|
-
|
|
192
|
-
**Maintained By**: Matthew Iverson (@Esity)
|
|
75
|
+
448+ specs. Run `bundle exec rspec` and `bundle exec rubocop`.
|
|
@@ -125,7 +125,7 @@ module Legion
|
|
|
125
125
|
def topology_mode?
|
|
126
126
|
return true unless defined?(Legion::Mode)
|
|
127
127
|
|
|
128
|
-
Legion::Mode.infra? || Legion::Mode.
|
|
128
|
+
Legion::Mode.infra? || (Legion::Mode.respond_to?(:worker?) && Legion::Mode.worker?)
|
|
129
129
|
end
|
|
130
130
|
|
|
131
131
|
def safely_close_channel(error_channel)
|
|
@@ -89,12 +89,37 @@ module Legion
|
|
|
89
89
|
dlx_name = merged_options.dig(:arguments, :'x-dead-letter-exchange')
|
|
90
90
|
return if dlx_name.nil? || dlx_name.empty?
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
dlx_ch = nil
|
|
93
|
+
dlx_ch = Legion::Transport::Connection.session.create_channel
|
|
94
|
+
declare_dlx(dlx_name, dlx_ch)
|
|
95
|
+
rescue Legion::Transport::CONNECTOR::PreconditionFailed => e
|
|
96
|
+
handle_exception(e, level: :warn, handled: true, operation: 'transport.queue.ensure_dlx', dlx: dlx_name)
|
|
97
|
+
recreate_dlx(dlx_name)
|
|
96
98
|
rescue StandardError => e
|
|
97
99
|
handle_exception(e, level: :warn, handled: true, operation: 'transport.queue.ensure_dlx', dlx: dlx_name)
|
|
100
|
+
ensure
|
|
101
|
+
dlx_ch&.close if dlx_ch&.open?
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def declare_dlx(dlx_name, dlx_channel)
|
|
105
|
+
dlx_channel.exchange_declare(dlx_name, 'fanout', durable: true, auto_delete: false)
|
|
106
|
+
dlx_channel.queue_declare("#{dlx_name}.queue", durable: true, auto_delete: false,
|
|
107
|
+
arguments: { 'x-queue-type': 'classic' })
|
|
108
|
+
dlx_channel.queue_bind("#{dlx_name}.queue", dlx_name, routing_key: '#')
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def recreate_dlx(dlx_name)
|
|
112
|
+
log.warn "DLX #{dlx_name} exists with wrong parameters, deleting and recreating"
|
|
113
|
+
ch = Legion::Transport::Connection.session.create_channel
|
|
114
|
+
ch.exchange_delete(dlx_name)
|
|
115
|
+
ch.queue_delete("#{dlx_name}.queue")
|
|
116
|
+
ch.close
|
|
117
|
+
ch = Legion::Transport::Connection.session.create_channel
|
|
118
|
+
declare_dlx(dlx_name, ch)
|
|
119
|
+
rescue StandardError => e
|
|
120
|
+
handle_exception(e, level: :warn, handled: true, operation: 'transport.queue.recreate_dlx', dlx: dlx_name)
|
|
121
|
+
ensure
|
|
122
|
+
ch&.close if ch&.open?
|
|
98
123
|
end
|
|
99
124
|
|
|
100
125
|
def queue_name
|
|
@@ -142,7 +167,7 @@ module Legion
|
|
|
142
167
|
def topology_mode?
|
|
143
168
|
return true unless defined?(Legion::Mode)
|
|
144
169
|
|
|
145
|
-
Legion::Mode.infra? || Legion::Mode.
|
|
170
|
+
Legion::Mode.infra? || (Legion::Mode.respond_to?(:worker?) && Legion::Mode.worker?)
|
|
146
171
|
end
|
|
147
172
|
|
|
148
173
|
def safely_close_channel(tmp_channel)
|
|
@@ -18,7 +18,7 @@ module Legion
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
def queue_options
|
|
21
|
-
{ durable: false, auto_delete: true, arguments: { 'x-dead-letter-exchange': 'agent.dlx', 'x-queue-type': 'classic' } }
|
|
21
|
+
{ durable: false, auto_delete: true, exclusive: true, arguments: { 'x-dead-letter-exchange': 'agent.dlx', 'x-queue-type': 'classic' } }
|
|
22
22
|
end
|
|
23
23
|
end
|
|
24
24
|
end
|
|
@@ -9,7 +9,7 @@ module Legion
|
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
def queue_options
|
|
12
|
-
{ durable: false, auto_delete: true, arguments: { 'x-dead-letter-exchange': 'node.dlx' } }
|
|
12
|
+
{ durable: false, auto_delete: true, exclusive: true, arguments: { 'x-dead-letter-exchange': 'node.dlx', 'x-queue-type': 'classic' } }
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
end
|
|
@@ -98,7 +98,7 @@ module Legion
|
|
|
98
98
|
def self.tenant_topology
|
|
99
99
|
{
|
|
100
100
|
enabled: false,
|
|
101
|
-
prefix_format: 't.%<tenant_id>s
|
|
101
|
+
prefix_format: 't.%<tenant_id>s.%<name>s',
|
|
102
102
|
shared_exchanges: %w[legion.control legion.health legion.audit],
|
|
103
103
|
auto_provision: true,
|
|
104
104
|
quotas: {}
|
|
@@ -11,14 +11,17 @@ module Legion
|
|
|
11
11
|
EXCHANGE_TYPES = %w[tasks results events].freeze
|
|
12
12
|
|
|
13
13
|
def self.provision(tenant_id, channel: nil)
|
|
14
|
-
ch = channel || Legion::Transport.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
ch = channel || Legion::Transport::Connection.channel
|
|
15
|
+
begin
|
|
16
|
+
EXCHANGE_TYPES.each do |type|
|
|
17
|
+
name = TenantTopology.exchange_name(type, tenant_id: tenant_id)
|
|
18
|
+
ch.topic(name, durable: true)
|
|
19
|
+
end
|
|
20
|
+
dlx = TenantTopology.exchange_name('dlx', tenant_id: tenant_id)
|
|
21
|
+
ch.fanout(dlx, durable: true)
|
|
22
|
+
ensure
|
|
23
|
+
ch.close if ch.respond_to?(:close) && !channel
|
|
18
24
|
end
|
|
19
|
-
dlx = TenantTopology.exchange_name('dlx', tenant_id: tenant_id)
|
|
20
|
-
ch.fanout(dlx, durable: true)
|
|
21
|
-
ch.close unless channel
|
|
22
25
|
log.info "Provisioned tenant topology for tenant_id=#{tenant_id}"
|
|
23
26
|
rescue StandardError => e
|
|
24
27
|
handle_exception(e, level: :warn, handled: false, operation: 'transport.tenant_provisioner.provision',
|
|
@@ -27,18 +30,29 @@ module Legion
|
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
def self.deprovision(tenant_id, channel: nil)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
33
|
+
return log.debug('Skipping deprovision: topology disabled') unless TenantTopology.enabled?
|
|
34
|
+
|
|
35
|
+
tid = tenant_id.to_s.strip
|
|
36
|
+
if tid.empty? || tid == 'default'
|
|
37
|
+
log.warn "Skipping deprovision: refusing to delete global exchanges for tenant_id=#{tenant_id.inspect}"
|
|
38
|
+
return
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
ch = channel || Legion::Transport::Connection.channel
|
|
42
|
+
begin
|
|
43
|
+
(EXCHANGE_TYPES + ['dlx']).each do |type|
|
|
44
|
+
name = TenantTopology.exchange_name(type, tenant_id: tenant_id)
|
|
45
|
+
begin
|
|
46
|
+
ch.exchange_delete(name)
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
handle_exception(e, level: :warn, handled: true, operation: 'transport.tenant_provisioner.deprovision_exchange',
|
|
49
|
+
tenant_id: tenant_id, exchange_name: name)
|
|
50
|
+
nil
|
|
51
|
+
end
|
|
39
52
|
end
|
|
53
|
+
ensure
|
|
54
|
+
ch.close if ch.respond_to?(:close) && !channel
|
|
40
55
|
end
|
|
41
|
-
ch.close unless channel
|
|
42
56
|
log.info "Deprovisioned tenant topology for tenant_id=#{tenant_id}"
|
|
43
57
|
rescue StandardError => e
|
|
44
58
|
handle_exception(e, level: :warn, handled: false, operation: 'transport.tenant_provisioner.deprovision',
|
|
@@ -12,9 +12,11 @@ module Legion
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
WINDOW_SECONDS = 1
|
|
15
|
+
STALE_SECONDS = 300
|
|
15
16
|
|
|
16
|
-
@counters
|
|
17
|
-
@
|
|
17
|
+
@counters = {}
|
|
18
|
+
@mutexes = {}
|
|
19
|
+
@registry_mutex = Mutex.new
|
|
18
20
|
|
|
19
21
|
class << self
|
|
20
22
|
def check_publish(tenant_id, message_size: 0)
|
|
@@ -25,13 +27,15 @@ module Legion
|
|
|
25
27
|
return true if msg_limit.nil? && size_limit.nil?
|
|
26
28
|
|
|
27
29
|
now = current_window
|
|
28
|
-
|
|
29
|
-
|
|
30
|
+
|
|
31
|
+
tenant_mutex(tenant_id).synchronize do
|
|
32
|
+
@counters[tenant_id] ||= { window: now, count: 0, bytes: 0, updated_at: now }
|
|
30
33
|
entry = @counters[tenant_id]
|
|
31
34
|
if entry[:window] != now
|
|
32
|
-
entry[:window]
|
|
33
|
-
entry[:count]
|
|
34
|
-
entry[:bytes]
|
|
35
|
+
entry[:window] = now
|
|
36
|
+
entry[:count] = 0
|
|
37
|
+
entry[:bytes] = 0
|
|
38
|
+
entry[:updated_at] = now
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
if msg_limit && entry[:count] >= msg_limit
|
|
@@ -44,9 +48,12 @@ module Legion
|
|
|
44
48
|
raise QuotaExceededError, "Tenant #{tenant_id} exceeded byte rate quota (#{size_limit} bytes/s)"
|
|
45
49
|
end
|
|
46
50
|
|
|
47
|
-
entry[:count]
|
|
48
|
-
entry[:bytes]
|
|
51
|
+
entry[:count] += 1
|
|
52
|
+
entry[:bytes] += message_size
|
|
53
|
+
entry[:updated_at] = now
|
|
49
54
|
end
|
|
55
|
+
|
|
56
|
+
sweep_stale!
|
|
50
57
|
true
|
|
51
58
|
end
|
|
52
59
|
|
|
@@ -55,11 +62,37 @@ module Legion
|
|
|
55
62
|
end
|
|
56
63
|
|
|
57
64
|
def reset!
|
|
58
|
-
@
|
|
65
|
+
@registry_mutex.synchronize do
|
|
66
|
+
@counters.clear
|
|
67
|
+
@mutexes.clear
|
|
68
|
+
end
|
|
59
69
|
end
|
|
60
70
|
|
|
61
71
|
private
|
|
62
72
|
|
|
73
|
+
def tenant_mutex(tenant_id)
|
|
74
|
+
@registry_mutex.synchronize do
|
|
75
|
+
@mutexes[tenant_id] ||= Mutex.new
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def sweep_stale!
|
|
80
|
+
stale_cutoff = current_window - (STALE_SECONDS / WINDOW_SECONDS)
|
|
81
|
+
|
|
82
|
+
stale_ids = @registry_mutex.synchronize do
|
|
83
|
+
@counters.each_with_object([]) do |(tid, entry), ids|
|
|
84
|
+
ids << tid if entry[:updated_at] && entry[:updated_at] < stale_cutoff
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
stale_ids.each do |tid|
|
|
89
|
+
@registry_mutex.synchronize do
|
|
90
|
+
@counters.delete(tid)
|
|
91
|
+
@mutexes.delete(tid)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
63
96
|
def current_window
|
|
64
97
|
(::Time.now.to_f / WINDOW_SECONDS).floor
|
|
65
98
|
end
|
|
@@ -7,7 +7,8 @@ module Legion
|
|
|
7
7
|
module TenantTopology
|
|
8
8
|
extend Legion::Logging::Helper
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
DEFAULT_SHARED_EXCHANGES = %w[legion.control legion.health legion.audit].freeze
|
|
11
|
+
DEFAULT_PREFIX_FORMAT = 't.%<tenant_id>s.%<name>s'
|
|
11
12
|
|
|
12
13
|
def self.exchange_name(base_name, tenant_id: nil)
|
|
13
14
|
return base_name unless enabled?
|
|
@@ -15,7 +16,7 @@ module Legion
|
|
|
15
16
|
tid = tenant_id || current_tenant_id
|
|
16
17
|
return base_name if tid.nil? || tid == 'default' || shared?(base_name)
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
format(prefix_format, tenant_id: tid, name: base_name)
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
def self.queue_name(base_name, tenant_id: nil)
|
|
@@ -24,11 +25,13 @@ module Legion
|
|
|
24
25
|
tid = tenant_id || current_tenant_id
|
|
25
26
|
return base_name if tid.nil? || tid == 'default'
|
|
26
27
|
|
|
27
|
-
|
|
28
|
+
format(prefix_format, tenant_id: tid, name: base_name)
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
def self.shared?(name)
|
|
31
|
-
|
|
32
|
+
configured_shared_exchanges.any? do |entry|
|
|
33
|
+
name == entry || name.start_with?("#{entry}.")
|
|
34
|
+
end
|
|
32
35
|
end
|
|
33
36
|
|
|
34
37
|
def self.enabled?
|
|
@@ -45,6 +48,22 @@ module Legion
|
|
|
45
48
|
nil
|
|
46
49
|
end
|
|
47
50
|
|
|
51
|
+
private_class_method def self.prefix_format
|
|
52
|
+
settings = transport_settings
|
|
53
|
+
return DEFAULT_PREFIX_FORMAT unless settings.is_a?(Hash)
|
|
54
|
+
|
|
55
|
+
settings.dig(:tenant_topology, :prefix_format) || DEFAULT_PREFIX_FORMAT
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private_class_method def self.configured_shared_exchanges
|
|
59
|
+
settings = transport_settings
|
|
60
|
+
return DEFAULT_SHARED_EXCHANGES unless settings.is_a?(Hash)
|
|
61
|
+
|
|
62
|
+
Array(settings.dig(:tenant_topology, :shared_exchanges)).tap do |arr|
|
|
63
|
+
return DEFAULT_SHARED_EXCHANGES if arr.empty?
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
48
67
|
private_class_method def self.transport_settings
|
|
49
68
|
Legion::Settings[:transport] || {}
|
|
50
69
|
rescue StandardError => e
|