legion-transport 1.4.31 → 1.4.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +5 -2
- data/.gitignore +1 -0
- data/CHANGELOG.md +21 -0
- data/Gemfile +10 -0
- data/README.md +45 -1
- data/legion-transport.gemspec +2 -1
- data/lib/legion/transport/connection/ssl.rb +3 -21
- data/lib/legion/transport/connection/vault.rb +3 -23
- data/lib/legion/transport/connection.rb +12 -12
- data/lib/legion/transport/errors.rb +9 -0
- data/lib/legion/transport/exchange.rb +0 -2
- data/lib/legion/transport/in_process.rb +21 -9
- data/lib/legion/transport/local.rb +68 -13
- data/lib/legion/transport/message.rb +33 -20
- data/lib/legion/transport/messages/check_subtask.rb +3 -1
- data/lib/legion/transport/messages/dynamic.rb +7 -3
- data/lib/legion/transport/messages/region_re_route.rb +1 -1
- data/lib/legion/transport/messages/subtask.rb +10 -13
- data/lib/legion/transport/messages/task.rb +6 -8
- data/lib/legion/transport/messages/task_update.rb +4 -9
- data/lib/legion/transport/publisher.rb +15 -1
- data/lib/legion/transport/queue.rb +0 -2
- data/lib/legion/transport/queues/region_outbound.rb +1 -1
- data/lib/legion/transport/queues/task_update.rb +5 -1
- data/lib/legion/transport/settings.rb +35 -7
- data/lib/legion/transport/spool.rb +7 -1
- data/lib/legion/transport/task/envelope.rb +45 -0
- data/lib/legion/transport/task/protocol.rb +58 -0
- data/lib/legion/transport/version.rb +1 -1
- data/lib/legion/transport.rb +1 -5
- metadata +19 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 03b783d62b4f350acbc74c3cd55501ad8a86af2123006085e0d78ef0d22101e0
|
|
4
|
+
data.tar.gz: b5d109c65eb57c52050c61cfe3e5de6330c0a9ad5eaebd814e90355f0ec08264
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 95416ee45e311a0ba0c3883556ce6e91e384cc7d7980de85e60cb73a5cbe937381e69e3ceeb080f2cf939f095844826e7858588e6f0fa0616bde6933a7bb532a
|
|
7
|
+
data.tar.gz: e78f9eedf46b3beb2aa69c586ffe41b9c258f6acb1583117ee689dfb3c762d97a7869ba4bf38f98e4373469cf1760caea7c203f0b53ce5dc5e4674e9ac02472a
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -8,15 +8,18 @@ on:
|
|
|
8
8
|
|
|
9
9
|
jobs:
|
|
10
10
|
ci:
|
|
11
|
-
uses: LegionIO/.github/.github/workflows/ci.yml@
|
|
11
|
+
uses: LegionIO/.github/.github/workflows/ci.yml@e0fc7364b73f446d4cb9bdc162a66ed34bf0daa7
|
|
12
12
|
with:
|
|
13
13
|
needs-rabbitmq: true
|
|
14
|
+
companion-gems: ${{ github.event_name == 'pull_request' && '{"legion-settings":{"repository":"LegionIO/legion-settings","ref":"8e73a9ef3c13958e3e1e7fc98532e41283159706"}}' || '{}' }}
|
|
14
15
|
|
|
15
16
|
lint:
|
|
16
17
|
uses: LegionIO/.github/.github/workflows/lint-patterns.yml@main
|
|
17
18
|
|
|
18
19
|
security:
|
|
19
|
-
uses: LegionIO/.github/.github/workflows/security-scan.yml@
|
|
20
|
+
uses: LegionIO/.github/.github/workflows/security-scan.yml@e0fc7364b73f446d4cb9bdc162a66ed34bf0daa7
|
|
21
|
+
with:
|
|
22
|
+
companion-gems: ${{ github.event_name == 'pull_request' && '{"legion-settings":{"repository":"LegionIO/legion-settings","ref":"8e73a9ef3c13958e3e1e7fc98532e41283159706"}}' || '{}' }}
|
|
20
23
|
|
|
21
24
|
version-changelog:
|
|
22
25
|
uses: LegionIO/.github/.github/workflows/version-changelog.yml@main
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [1.4.32] - 2026-10-30
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- Add strict `Message#publish!` and raw `Publisher.publish!` APIs backed by Transport-owned local `transport.task_publication` policy; return explicit accepted results or raise `PublishRejected` exposing the original result. Keep legacy non-task publication behavior unchanged and disable duplicate spooling for task outbox handoffs by default.
|
|
9
|
+
- Preserve canonical task lineage, policy identity, typed engine/schema context, attempts, deadlines and schedule metadata across task subclasses and raw-byte retries.
|
|
10
|
+
- Own the immutable task wire contract in `Legion::Transport::Task::Protocol`, with status/route lookup and terminal recognition helpers independent of runtime settings, Data and Core. Nest task protocol and envelope implementations under `lib/legion/transport/task/`.
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- Preserve fixed task stage routes and canonical statuses despite local configuration drift, reject missing task routes, and route meaningful short templates/structured conditions without string-length bypasses.
|
|
14
|
+
- Implement lite manual acknowledgement, rejection/requeue, redelivery metadata and ownership-safe cancellation/channel recovery. Resume existing surviving consumers after cancellation/channel closure, including self-cancellation, without delivering onto a closing session. Preserve callback failures as recoverable pending work without claiming delivery or looping synchronously on rejects.
|
|
15
|
+
- Declare Base64 as a direct runtime dependency so standalone consumers can load spool support on Ruby 3.4 and newer without incidental dependencies.
|
|
16
|
+
- Read message limits and metadata, connection/TLS policy, Vault configuration and routing directly from live `Legion::Settings[:transport]`; remove cached settings snapshots and optional-dependency fallbacks.
|
|
17
|
+
- Reject missing or invalid payload safety limits with a configuration error before broker publication.
|
|
18
|
+
- Restore persisted date/time and numeric-string AMQP timestamps to Unix seconds before spool replay; preserve ciphertext and IVs, and retain unparseable timestamp records in recovery without blocking healthy messages.
|
|
19
|
+
- Restore the public `Connection.settings` API used by subscription actors to create dedicated channels. Read the canonical transport settings on every call so live configuration replacement remains effective.
|
|
20
|
+
- Preserve encryption IVs through task-envelope metadata handling for both existing `publish` and opt-in `publish!` callers.
|
|
21
|
+
- Honor Dynamic constructor and per-publication routing overrides, and retain the supported `task.scheduled` and `transformer.exception` statuses.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
- Require generic `legion-settings >= 1.4.5`; register Transport defaults at startup, never during policy reads, and document local publication policy versus stable wire identities and lite-mode durability limits.
|
|
25
|
+
|
|
5
26
|
## [1.4.31] - 2026-10-29
|
|
6
27
|
|
|
7
28
|
### Fixed
|
data/Gemfile
CHANGED
|
@@ -3,6 +3,16 @@
|
|
|
3
3
|
source 'https://rubygems.org'
|
|
4
4
|
|
|
5
5
|
gemspec
|
|
6
|
+
|
|
7
|
+
dependency_root = if File.directory?(File.expand_path('.ci-dependencies',
|
|
8
|
+
__dir__))
|
|
9
|
+
File.expand_path('.ci-dependencies', __dir__)
|
|
10
|
+
else
|
|
11
|
+
File.expand_path('..', __dir__)
|
|
12
|
+
end
|
|
13
|
+
settings_path = File.join(dependency_root, 'legion-settings')
|
|
14
|
+
gem 'legion-settings', path: settings_path if File.directory?(settings_path)
|
|
15
|
+
|
|
6
16
|
group :test do
|
|
7
17
|
gem 'rake'
|
|
8
18
|
gem 'rspec'
|
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Legion::Transport is the Ruby gem responsible for connecting LegionIO to its FIFO queue system (RabbitMQ over AMQP 0.9.1). It provides thread-safe connection management, exchange/queue abstractions, message publishing with optional encryption, and consumer wrappers.
|
|
4
4
|
|
|
5
|
-
**Version**: 1.4.
|
|
5
|
+
**Version**: 1.4.32
|
|
6
6
|
|
|
7
7
|
## Features
|
|
8
8
|
|
|
@@ -95,12 +95,54 @@ Options merge with the message's defaults — you can override `routing_key:` or
|
|
|
95
95
|
|
|
96
96
|
Mandatory publishing waits for publisher confirmation before reporting routed acceptance. Confirm waits use Bunny's zero-argument API and retire a timed-out channel so late acknowledgments cannot contaminate the next send. A timeout is an unknown delivery outcome, not proof that the broker rejected the message.
|
|
97
97
|
|
|
98
|
+
#### Task pipeline publication
|
|
99
|
+
|
|
100
|
+
Task producers and durable outbox relays can opt into `publish!` when they need explicit acceptance. The existing `publish(options = nil)` API remains available; existing callers do not automatically acquire strict publication policy:
|
|
101
|
+
|
|
102
|
+
```ruby
|
|
103
|
+
message = Legion::Transport::Messages::Task.new(
|
|
104
|
+
function: 'process_order', routing_key: 'orders.process',
|
|
105
|
+
task_id: 42, parent_id: 7, master_id: 1,
|
|
106
|
+
principal_id: 'order-service', principal_type: 'service',
|
|
107
|
+
args: { order_id: 123 }
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
begin
|
|
111
|
+
result = message.publish!(spool: false)
|
|
112
|
+
# result[:accepted] == true and result[:status] == :accepted
|
|
113
|
+
rescue Legion::Transport::PublishRejected => error
|
|
114
|
+
# error.result has the original :status, :accepted, destination and error details.
|
|
115
|
+
# Keep/release the outbox intent for retry; do not acknowledge it as delivered.
|
|
116
|
+
end
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
`Message#publish!(**options)` and `Publisher.publish!(exchange:, payload:, envelope:, **options)` use live, process-local `Legion::Settings[:transport][:task_publication]` through `Legion::Transport::Settings::Helper#transport_task_publication_settings`. Transport registers its defaults at startup with generic `legion-settings >= 1.4.5`; reads never register defaults or cache a loader. Message limits and metadata, connection/TLS configuration, and routing read the live canonical Transport subtree. Defaults enable publisher confirmation, mandatory routing and structured results, use a 5000ms confirmation timeout, and disable the secondary disk spool. Constructor options and call options can explicitly override policy; disabling confirmation or mandatory routing weakens the corresponding guarantee. The bang APIs always return a structured accepted result or raise: spooled, unroutable, nacked, failed and ambiguous timeout outcomes are **not** accepted.
|
|
120
|
+
|
|
121
|
+
The raw `Publisher.publish!` API is for subscription retries: pass the original bytes and full AMQP envelope, including content encoding, IV, message/correlation IDs, and retry headers. It never re-encodes, re-encrypts or spools the payload. The message API continues using the existing Publisher/Spool implementation. Both message publication APIs preserve task encryption IVs; legacy `publish` retains its existing return convention.
|
|
122
|
+
|
|
123
|
+
Task, SubTask, Dynamic, CheckSubtask and TaskUpdate preserve canonical task metadata in both the JSON body and AMQP headers: lineage, identity/tenant/worker policy, engine/schema/options, execution attempts/tokens, depth/deadline, schedule/occurrence and definition version. Explicit business arguments stay under `args`, separate from orchestration values such as `results`, `function_args`, `conditions` and `transformation`. Dynamic retains its legacy flat-argument fallback; supply `args:` to prevent that fallback from passing orchestration values to the runner. Headers include the fixed `task_protocol_version` (`1`). An empty or missing task routing key is rejected rather than silently sent to the empty route.
|
|
124
|
+
|
|
125
|
+
`require 'legion/transport/task/protocol'` exposes the immutable wire contract `Legion::Transport::Task::Protocol`: `STATUSES`, `TERMINAL_STATUSES`, `LEGACY_TERMINAL_STATUSES`, `ROUTES`, `MESSAGE_TYPES`, `VERSION` and `METADATA_FIELDS`. Include or extend its `Helper` for `task_status(name:)`, `task_route(name:)` and `task_terminal_status?(status:)`. Status/route lookup rejects unknown names; terminal recognition includes legacy terminal spellings. Shared task-envelope behavior lives in `legion/transport/task/envelope` under `Legion::Transport::Task::Envelope`.
|
|
126
|
+
|
|
127
|
+
Stage routes remain `task.subtask.check`, `task.subtask.conditioner`, `task.subtask.transform` and `task.update`. CheckSubtask and TaskUpdate always use these fixed routes; Task, SubTask and Dynamic retain explicit target routing keys, including per-publication overrides. Local settings, including obsolete root `task_execution` overrides, cannot rename wire identities or redefine metadata/version/terminal semantics. Settings is not distributed-consistent storage: publication policy is local, while wire identity is fixed and durable execution authority belongs to the hosting application's database.
|
|
128
|
+
|
|
129
|
+
Nonblank short templates such as `{}` and structured conditions are routed to their stages rather than bypassed by string-length heuristics. TaskUpdate accepts canonical statuses, including the previously supported `task.scheduled` and `transformer.exception`. `Messages::Task` needs no database and supports an explicit exchange/routing key; `Messages::Dynamic` resolves its existing `function_id` through the host's optional Data model.
|
|
130
|
+
|
|
131
|
+
#### Lite-mode delivery limits
|
|
132
|
+
|
|
133
|
+
With `LEGION_MODE=lite`, strict publication accepts only a matching declared queue when mandatory routing is enabled. Acceptance means **queued in process memory**, not durable broker storage or successful execution. Restart/session shutdown loses pending work; there are no RabbitMQ durability, priority, TTL or dead-letter guarantees.
|
|
134
|
+
|
|
135
|
+
Manual consumers must acknowledge successful work. `queue.acknowledge(tag)` removes an outstanding delivery; `queue.reject(tag, requeue: true)` retains the original bytes/properties and marks redelivery. Cancellation and channel closure return unacknowledged work to surviving session-owned queues and resume already-subscribed consumers without waiting for another publication. Session shutdown closes its consumers before handing shared work to surviving sessions. `requeue: false` discards it. A synchronous rejection is left pending until subsequent queue activity or a replacement subscription, avoiding a tight retry loop. Callback failures propagate and retain pending work; a failing initial backlog subscription is cancelled so replacement consumers can recover it. Direct `Local.subscribe` retains its automatic-acknowledgement API.
|
|
136
|
+
|
|
137
|
+
Use the Data task/outbox APIs in the hosting application when crash-durable intent is required. Transport does not depend on Data and does not claim lite publication provides that durability.
|
|
138
|
+
|
|
98
139
|
### Spooling and automatic replay
|
|
99
140
|
|
|
100
141
|
Connection failures can persist the already encoded wire payload, complete AMQP envelope, effective broker destination, and tenant identity. `:spooled` means the disk write succeeded; a full spool or failed write returns `:failed`.
|
|
101
142
|
|
|
102
143
|
- Startup, native recovery, and forced recovery schedule one asynchronous replay worker. Network operations run outside spool-write and connection locks.
|
|
103
144
|
- Replay uses the original ciphertext and IV, checks the stored tenant's encoded-byte quota, and removes records only after routed broker confirmation.
|
|
145
|
+
- Replay restores numeric strings and saved date/time strings to Unix timestamp seconds, preserving timezone offsets. Unparseable timestamp strings are quarantined with their original records instead of blocking later messages.
|
|
104
146
|
- Delivery is **at least once**: interruption after broker acceptance but before confirmation is recorded can cause redelivery. Consumers must tolerate duplicates.
|
|
105
147
|
- Concurrent appends are not part of a drainer's claimed file. Shutdown retains the unconfirmed tail for the next startup.
|
|
106
148
|
- Configure `Legion::Settings[:transport][:spool][:directory]` or pass `directory:` to `Spool.setup`; the default is `~/.legionio/spool`. Capacity limits reject new writes rather than deleting pending messages.
|
|
@@ -259,6 +301,8 @@ Configuration is managed through `legion-settings` with environment variable ove
|
|
|
259
301
|
| TTL | `transport.messages.ttl` | `nil` |
|
|
260
302
|
| Persistent | `transport.messages.persistent` | `true` |
|
|
261
303
|
|
|
304
|
+
`Legion::Transport::Connection.settings` exposes the current `Legion::Settings[:transport]` subtree for subscription actors and other connection consumers. It does not cache settings; replacing the live configuration takes effect on the next call.
|
|
305
|
+
|
|
262
306
|
### Full Default Settings
|
|
263
307
|
|
|
264
308
|
```json
|
data/legion-transport.gemspec
CHANGED
|
@@ -26,10 +26,11 @@ Gem::Specification.new do |spec|
|
|
|
26
26
|
'rubygems_mfa_required' => 'true'
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
spec.add_dependency 'base64'
|
|
29
30
|
spec.add_dependency 'bunny', '>= 2.23'
|
|
30
31
|
spec.add_dependency 'concurrent-ruby', '>= 1.2'
|
|
31
32
|
spec.add_dependency 'legion-json', '>= 1.2.0'
|
|
32
33
|
spec.add_dependency 'legion-logging', '>= 1.5.0'
|
|
33
|
-
spec.add_dependency 'legion-settings', '>= 1.
|
|
34
|
+
spec.add_dependency 'legion-settings', '>= 1.4.5'
|
|
34
35
|
spec.add_dependency 'logger'
|
|
35
36
|
end
|
|
@@ -10,8 +10,8 @@ module Legion
|
|
|
10
10
|
|
|
11
11
|
def tls_options(tls_config: nil, port: nil)
|
|
12
12
|
if defined?(Legion::Crypt::TLS)
|
|
13
|
-
tls_config
|
|
14
|
-
port
|
|
13
|
+
tls_config = Legion::Settings[:transport][:tls] if tls_config.nil?
|
|
14
|
+
port ||= Legion::Settings[:transport][:connection][:port]
|
|
15
15
|
|
|
16
16
|
tls = Legion::Crypt::TLS.resolve(tls_config, port: port)
|
|
17
17
|
return {} unless tls[:enabled]
|
|
@@ -32,7 +32,7 @@ module Legion
|
|
|
32
32
|
private
|
|
33
33
|
|
|
34
34
|
def direct_tls_options
|
|
35
|
-
transport =
|
|
35
|
+
transport = Legion::Settings[:transport]
|
|
36
36
|
return {} unless transport[:tls]
|
|
37
37
|
|
|
38
38
|
Legion::Transport.logger.info '[Transport] TLS enabled for RabbitMQ connection'
|
|
@@ -45,24 +45,6 @@ module Legion
|
|
|
45
45
|
verify_peer: transport[:verify_peer] != false
|
|
46
46
|
}
|
|
47
47
|
end
|
|
48
|
-
|
|
49
|
-
def tls_settings
|
|
50
|
-
return {} unless defined?(Legion::Settings)
|
|
51
|
-
|
|
52
|
-
Legion::Settings[:transport][:tls] || {}
|
|
53
|
-
rescue StandardError => e
|
|
54
|
-
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.ssl.tls_settings')
|
|
55
|
-
{}
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def transport_port
|
|
59
|
-
return nil unless defined?(Legion::Settings)
|
|
60
|
-
|
|
61
|
-
Legion::Settings[:transport][:connection][:port]
|
|
62
|
-
rescue StandardError => e
|
|
63
|
-
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.ssl.transport_port')
|
|
64
|
-
nil
|
|
65
|
-
end
|
|
66
48
|
end
|
|
67
49
|
end
|
|
68
50
|
end
|
|
@@ -19,7 +19,7 @@ module Legion
|
|
|
19
19
|
return {} unless Legion::Crypt::Mtls.enabled?
|
|
20
20
|
|
|
21
21
|
created_files = []
|
|
22
|
-
node_name =
|
|
22
|
+
node_name = Legion::Settings[:client][:name]
|
|
23
23
|
cert_data = Legion::Crypt::Mtls.issue_cert(common_name: node_name)
|
|
24
24
|
Legion::Transport.logger.info(
|
|
25
25
|
"[mTLS] Issued PKI cert for #{node_name}: serial=#{cert_data[:serial]} expiry=#{cert_data[:expiry]}"
|
|
@@ -35,10 +35,10 @@ module Legion
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def vault_pki_enabled?
|
|
38
|
-
tls =
|
|
38
|
+
tls = Legion::Settings[:transport][:tls]
|
|
39
39
|
return false unless tls.is_a?(Hash)
|
|
40
40
|
|
|
41
|
-
tls[:vault_pki]
|
|
41
|
+
tls[:vault_pki] == true
|
|
42
42
|
rescue StandardError => e
|
|
43
43
|
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.vault_pki_enabled')
|
|
44
44
|
false
|
|
@@ -78,26 +78,6 @@ module Legion
|
|
|
78
78
|
end
|
|
79
79
|
tempfiles&.clear
|
|
80
80
|
end
|
|
81
|
-
|
|
82
|
-
def pki_node_name
|
|
83
|
-
return 'legion.internal' unless defined?(Legion::Settings)
|
|
84
|
-
|
|
85
|
-
name = Legion::Settings[:client]&.dig(:name) || Legion::Settings[:client]&.dig('name')
|
|
86
|
-
name || 'legion.internal'
|
|
87
|
-
rescue StandardError => e
|
|
88
|
-
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.pki_node_name')
|
|
89
|
-
'legion.internal'
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
def transport_tls_settings
|
|
93
|
-
return {} unless defined?(Legion::Settings)
|
|
94
|
-
|
|
95
|
-
tls = Legion::Settings[:transport][:tls]
|
|
96
|
-
tls.is_a?(Hash) ? tls : {}
|
|
97
|
-
rescue StandardError => e
|
|
98
|
-
handle_exception(e, level: :warn, handled: true, operation: 'transport.connection.transport_tls_settings')
|
|
99
|
-
{}
|
|
100
|
-
end
|
|
101
81
|
end
|
|
102
82
|
end
|
|
103
83
|
end
|
|
@@ -21,14 +21,14 @@ module Legion
|
|
|
21
21
|
include Legion::Transport::Connection::SSL
|
|
22
22
|
include Legion::Transport::Connection::Vault
|
|
23
23
|
|
|
24
|
-
def lite_mode?
|
|
25
|
-
Legion::Transport::TYPE == 'local'
|
|
26
|
-
end
|
|
27
|
-
|
|
28
24
|
def settings
|
|
29
25
|
Legion::Settings[:transport]
|
|
30
26
|
end
|
|
31
27
|
|
|
28
|
+
def lite_mode?
|
|
29
|
+
Legion::Transport::TYPE == 'local'
|
|
30
|
+
end
|
|
31
|
+
|
|
32
32
|
def new
|
|
33
33
|
clone
|
|
34
34
|
end
|
|
@@ -55,7 +55,7 @@ module Legion
|
|
|
55
55
|
log.info("Using transport connector: #{Legion::Transport::CONNECTOR}")
|
|
56
56
|
return setup_lite(generation: generation) if lite_mode?
|
|
57
57
|
|
|
58
|
-
pool_size =
|
|
58
|
+
pool_size = Legion::Settings[:transport][:connection_pool_size].to_i
|
|
59
59
|
if pool_size > 1
|
|
60
60
|
setup_pool(pool_size: pool_size, connection_name: connection_name, generation: generation)
|
|
61
61
|
elsif session.respond_to?(:open?) && session.open?
|
|
@@ -85,8 +85,8 @@ module Legion
|
|
|
85
85
|
end
|
|
86
86
|
raise IOError, 'transport session unavailable (recovery in progress)' unless current_session&.open?
|
|
87
87
|
|
|
88
|
-
ch = current_session.create_channel(nil,
|
|
89
|
-
ch.prefetch(
|
|
88
|
+
ch = current_session.create_channel(nil, Legion::Settings[:transport][:channel][:default_worker_pool_size], false, 10)
|
|
89
|
+
ch.prefetch(Legion::Settings[:transport][:prefetch])
|
|
90
90
|
reconnect_mutex.synchronize do
|
|
91
91
|
check_generation!(generation: generation)
|
|
92
92
|
channel_thread.value = ch
|
|
@@ -233,8 +233,8 @@ module Legion
|
|
|
233
233
|
cached_channel = channel_thread.value
|
|
234
234
|
return cached_channel if Legion::Transport::Publisher.channel_usable?(channel: cached_channel)
|
|
235
235
|
|
|
236
|
-
ch = build_session.create_channel(nil,
|
|
237
|
-
ch.prefetch(
|
|
236
|
+
ch = build_session.create_channel(nil, Legion::Settings[:transport][:channel][:default_worker_pool_size], false, 10)
|
|
237
|
+
ch.prefetch(Legion::Settings[:transport][:prefetch])
|
|
238
238
|
reconnect_mutex.synchronize do
|
|
239
239
|
check_generation!(generation: generation)
|
|
240
240
|
raise IOError, 'build session closed during channel creation' unless @build_session&.value.equal?(build_session)
|
|
@@ -394,7 +394,7 @@ module Legion
|
|
|
394
394
|
end
|
|
395
395
|
|
|
396
396
|
def apply_qos_and_close(qos_channel:, **)
|
|
397
|
-
qos_channel.basic_qos(
|
|
397
|
+
qos_channel.basic_qos(Legion::Settings[:transport][:prefetch], true)
|
|
398
398
|
ensure
|
|
399
399
|
safe_close_channel(chan: qos_channel)
|
|
400
400
|
end
|
|
@@ -433,7 +433,7 @@ module Legion
|
|
|
433
433
|
create_session_with_failover(connection_name: connection_name)
|
|
434
434
|
end
|
|
435
435
|
primary = pool.checkout
|
|
436
|
-
qos_channel = primary.create_channel(nil,
|
|
436
|
+
qos_channel = primary.create_channel(nil, Legion::Settings[:transport][:channel][:session_worker_pool_size])
|
|
437
437
|
apply_qos_and_close(qos_channel: qos_channel)
|
|
438
438
|
reconnect_mutex.synchronize do
|
|
439
439
|
check_generation!(generation: generation)
|
|
@@ -653,7 +653,7 @@ module Legion
|
|
|
653
653
|
@log_channel = nil
|
|
654
654
|
tear_down_session(sess: session) if session
|
|
655
655
|
current_session = create_session_with_failover(connection_name: connection_name)
|
|
656
|
-
qos_channel = current_session.create_channel(nil,
|
|
656
|
+
qos_channel = current_session.create_channel(nil, Legion::Settings[:transport][:channel][:session_worker_pool_size])
|
|
657
657
|
apply_qos_and_close(qos_channel: qos_channel)
|
|
658
658
|
reconnect_mutex.synchronize do
|
|
659
659
|
check_generation!(generation: generation)
|
|
@@ -9,5 +9,14 @@ module Legion
|
|
|
9
9
|
class ClusterUnavailable < StandardError; end
|
|
10
10
|
class PayloadTooLarge < StandardError; end
|
|
11
11
|
class EncryptionNotReady < StandardError; end
|
|
12
|
+
|
|
13
|
+
class PublishRejected < StandardError
|
|
14
|
+
attr_reader :result
|
|
15
|
+
|
|
16
|
+
def initialize(result:, **)
|
|
17
|
+
@result = result
|
|
18
|
+
super("Publication rejected: #{result[:status]}#{": #{result[:error]}" if result[:error]}")
|
|
19
|
+
end
|
|
20
|
+
end
|
|
12
21
|
end
|
|
13
22
|
end
|
|
@@ -10,7 +10,7 @@ module Legion
|
|
|
10
10
|
class ChannelAlreadyClosed < StandardError; end
|
|
11
11
|
class ChannelLevelException < StandardError; end
|
|
12
12
|
|
|
13
|
-
DeliveryInfo = Struct.new(:delivery_tag, :routing_key, :exchange, keyword_init: true)
|
|
13
|
+
DeliveryInfo = Struct.new(:delivery_tag, :routing_key, :exchange, :redelivered, keyword_init: true)
|
|
14
14
|
|
|
15
15
|
class MessageProperties < Hash
|
|
16
16
|
FIELDS = %i[content_type headers timestamp content_encoding delivery_mode priority correlation_id reply_to
|
|
@@ -133,9 +133,13 @@ module Legion
|
|
|
133
133
|
Legion::Transport::Local.bind(queue: queue, exchange_name: exchange_name, routing_key: options.fetch(:routing_key, ''))
|
|
134
134
|
end
|
|
135
135
|
|
|
136
|
-
def acknowledge(
|
|
136
|
+
def acknowledge(delivery_tag, multiple = false, **options) # rubocop:disable Style/OptionalBooleanParameter
|
|
137
|
+
Legion::Transport::Local.settle(owner: self, delivery_tag: delivery_tag, multiple: options.fetch(:multiple, multiple))
|
|
138
|
+
end
|
|
137
139
|
|
|
138
|
-
def reject(
|
|
140
|
+
def reject(delivery_tag, requeue = false, **options) # rubocop:disable Style/OptionalBooleanParameter
|
|
141
|
+
Legion::Transport::Local.settle(owner: self, delivery_tag: delivery_tag, requeue: options.fetch(:requeue, requeue))
|
|
142
|
+
end
|
|
139
143
|
|
|
140
144
|
def exchange_delete(name, *)
|
|
141
145
|
Legion::Transport::Local.delete_exchange(name: name)
|
|
@@ -199,16 +203,21 @@ module Legion
|
|
|
199
203
|
|
|
200
204
|
tag = consumer_tag || SecureRandom.uuid
|
|
201
205
|
consumer = Consumer.new(@channel, self, tag, !manual_ack, false)
|
|
202
|
-
consumer.register(queue: @local_queue, owner: @channel) do |message|
|
|
203
|
-
delivery_info = DeliveryInfo.new(delivery_tag:
|
|
206
|
+
consumer.register(queue: @local_queue, owner: @channel, manual_ack: manual_ack) do |message|
|
|
207
|
+
delivery_info = DeliveryInfo.new(delivery_tag: message.delivery_tag, routing_key: message.routing_key,
|
|
208
|
+
exchange: message.exchange, redelivered: message.redelivered == true)
|
|
204
209
|
callback.call(delivery_info, MessageProperties.new(**message.properties), message.payload)
|
|
205
210
|
end
|
|
206
211
|
consumer
|
|
207
212
|
end
|
|
208
213
|
|
|
209
|
-
def acknowledge(
|
|
214
|
+
def acknowledge(delivery_tag, multiple = false, **) # rubocop:disable Style/OptionalBooleanParameter
|
|
215
|
+
@channel.acknowledge(delivery_tag, multiple, **)
|
|
216
|
+
end
|
|
210
217
|
|
|
211
|
-
def reject(
|
|
218
|
+
def reject(delivery_tag, requeue = false, **) # rubocop:disable Style/OptionalBooleanParameter
|
|
219
|
+
@channel.reject(delivery_tag, requeue, **)
|
|
220
|
+
end
|
|
212
221
|
|
|
213
222
|
def delete(_options = {}, **)
|
|
214
223
|
Legion::Transport::Local.delete_queue(name: @name)
|
|
@@ -223,9 +232,12 @@ module Legion
|
|
|
223
232
|
@cancelled = false
|
|
224
233
|
end
|
|
225
234
|
|
|
226
|
-
def register(queue:, owner:, **, &callback)
|
|
227
|
-
@subscription = Legion::Transport::Local.add_subscription(queue: queue, owner: owner, &callback)
|
|
235
|
+
def register(queue:, owner:, manual_ack: true, **, &callback)
|
|
236
|
+
@subscription = Legion::Transport::Local.add_subscription(queue: queue, owner: owner, manual_ack: manual_ack, &callback)
|
|
228
237
|
Legion::Transport::Local.deliver_pending(queue: queue)
|
|
238
|
+
rescue StandardError
|
|
239
|
+
cancel
|
|
240
|
+
raise
|
|
229
241
|
end
|
|
230
242
|
|
|
231
243
|
def cancel
|
|
@@ -5,13 +5,14 @@ require 'legion/logging/helper'
|
|
|
5
5
|
module Legion
|
|
6
6
|
module Transport
|
|
7
7
|
module Local
|
|
8
|
-
PendingMessage = Struct.new(:exchange, :routing_key, :payload, :properties, keyword_init: true)
|
|
9
|
-
QueueState = Struct.new(:name, :key, :pending, :subscribers, :owners, :bindings, :draining, :implicit, :exchange_name,
|
|
8
|
+
PendingMessage = Struct.new(:exchange, :routing_key, :payload, :properties, :delivery_tag, :redelivered, :requeued, keyword_init: true)
|
|
9
|
+
QueueState = Struct.new(:name, :key, :pending, :unacked, :subscribers, :owners, :bindings, :draining, :implicit, :exchange_name,
|
|
10
10
|
keyword_init: true)
|
|
11
|
-
Subscription = Struct.new(:queue, :owner, :callback, :running_thread, :completed, keyword_init: true)
|
|
11
|
+
Subscription = Struct.new(:queue, :owner, :callback, :manual_ack, :running_thread, :completed, keyword_init: true)
|
|
12
12
|
|
|
13
13
|
@queues = {}
|
|
14
14
|
@mutex = Mutex.new
|
|
15
|
+
@next_delivery_tag = 0
|
|
15
16
|
|
|
16
17
|
class << self
|
|
17
18
|
include Legion::Logging::Helper
|
|
@@ -74,14 +75,15 @@ module Legion
|
|
|
74
75
|
end
|
|
75
76
|
end
|
|
76
77
|
|
|
77
|
-
def add_subscription(queue:, owner: nil, **, &callback)
|
|
78
|
+
def add_subscription(queue:, owner: nil, manual_ack: false, **, &callback)
|
|
78
79
|
raise ArgumentError, 'subscriber callback required' unless callback
|
|
79
80
|
|
|
80
81
|
@mutex.synchronize do
|
|
81
82
|
ensure_queue!(queue: queue)
|
|
82
83
|
raise IOError, 'local consumer channel is closed' if owner && !owner.open?
|
|
83
84
|
|
|
84
|
-
subscription = Subscription.new(queue: queue, owner: owner, callback: callback,
|
|
85
|
+
subscription = Subscription.new(queue: queue, owner: owner, callback: callback, manual_ack: manual_ack,
|
|
86
|
+
completed: ConditionVariable.new)
|
|
85
87
|
queue.subscribers << subscription
|
|
86
88
|
subscription
|
|
87
89
|
end
|
|
@@ -91,7 +93,9 @@ module Legion
|
|
|
91
93
|
@mutex.synchronize do
|
|
92
94
|
unregister(subscription: subscription)
|
|
93
95
|
await_completion(subscription: subscription)
|
|
96
|
+
requeue_unacked(subscription: subscription)
|
|
94
97
|
end
|
|
98
|
+
deliver_pending(queue: subscription.queue)
|
|
95
99
|
true
|
|
96
100
|
end
|
|
97
101
|
|
|
@@ -100,17 +104,25 @@ module Legion
|
|
|
100
104
|
end
|
|
101
105
|
|
|
102
106
|
def release_owner(owner:, **)
|
|
103
|
-
@mutex.synchronize do
|
|
107
|
+
queues = @mutex.synchronize do
|
|
104
108
|
subscriptions = []
|
|
109
|
+
affected = []
|
|
105
110
|
@queues.delete_if do |_key, queue|
|
|
106
|
-
queue.owners.delete(owner)
|
|
111
|
+
owned = queue.owners.delete(owner)
|
|
107
112
|
removed = !queue.implicit && queue.owners.empty?
|
|
108
|
-
|
|
113
|
+
cancelled = queue.subscribers.select { |subscriber| removed || subscriber.owner.equal?(owner) }
|
|
114
|
+
subscriptions.concat(cancelled)
|
|
115
|
+
affected << queue if !removed && (owned || !cancelled.empty?)
|
|
109
116
|
queue.pending.clear if removed
|
|
110
117
|
removed
|
|
111
118
|
end
|
|
112
119
|
cancel_subscriptions(subscriptions: subscriptions)
|
|
120
|
+
affected
|
|
113
121
|
end
|
|
122
|
+
# Session shutdown drains surviving queues after all its channels close.
|
|
123
|
+
return true if owner.respond_to?(:connection) && owner.connection && !owner.connection.open?
|
|
124
|
+
|
|
125
|
+
queues.each { |queue| deliver_pending(queue: queue) }
|
|
114
126
|
true
|
|
115
127
|
end
|
|
116
128
|
|
|
@@ -136,6 +148,26 @@ module Legion
|
|
|
136
148
|
@mutex.synchronize { @queues.values.sum { |queue| queue.name == queue_name ? queue.pending.size : 0 } }
|
|
137
149
|
end
|
|
138
150
|
|
|
151
|
+
def settle(owner:, delivery_tag:, multiple: false, requeue: false, **)
|
|
152
|
+
queues = @mutex.synchronize do
|
|
153
|
+
matched = @queues.values.reject do |queue|
|
|
154
|
+
entries = queue.unacked.select do |tag, (subscription, _message)|
|
|
155
|
+
subscription.owner.equal?(owner) && (tag == delivery_tag || (multiple && (delivery_tag.zero? || tag <= delivery_tag)))
|
|
156
|
+
end
|
|
157
|
+
entries.each do |tag, (_subscription, message)|
|
|
158
|
+
queue.unacked.delete(tag)
|
|
159
|
+
requeue_message(queue: queue, message: message) if requeue
|
|
160
|
+
end
|
|
161
|
+
entries.empty?
|
|
162
|
+
end
|
|
163
|
+
raise ArgumentError, "Unknown local delivery tag #{delivery_tag}" if matched.empty?
|
|
164
|
+
|
|
165
|
+
matched
|
|
166
|
+
end
|
|
167
|
+
queues.each { |queue| deliver_pending(queue: queue) } if requeue
|
|
168
|
+
true
|
|
169
|
+
end
|
|
170
|
+
|
|
139
171
|
def deliver_pending(queue:, **)
|
|
140
172
|
acquired = @mutex.synchronize do
|
|
141
173
|
next false if queue.draining || queue.pending.empty? || queue.subscribers.empty?
|
|
@@ -155,11 +187,15 @@ module Legion
|
|
|
155
187
|
subscription = queue.subscribers.shift
|
|
156
188
|
queue.subscribers << subscription
|
|
157
189
|
subscription.running_thread = Thread.current
|
|
158
|
-
|
|
190
|
+
message = queue.pending.shift.dup
|
|
191
|
+
message.delivery_tag = (@next_delivery_tag += 1)
|
|
192
|
+
message.requeued = false
|
|
193
|
+
queue.unacked[message.delivery_tag] = [subscription, message] if subscription.manual_ack
|
|
194
|
+
[subscription, subscription.callback, message]
|
|
159
195
|
end
|
|
160
196
|
break unless delivery
|
|
161
197
|
|
|
162
|
-
deliver(queue: queue, subscription: delivery[0], callback: delivery[1], message: delivery[2])
|
|
198
|
+
break unless deliver(queue: queue, subscription: delivery[0], callback: delivery[1], message: delivery[2])
|
|
163
199
|
end
|
|
164
200
|
ensure
|
|
165
201
|
@mutex.synchronize { queue.draining = nil if queue.draining.equal?(Thread.current) } if acquired
|
|
@@ -183,7 +219,7 @@ module Legion
|
|
|
183
219
|
|
|
184
220
|
def new_queue(name:, implicit: false, exchange_name: nil, **)
|
|
185
221
|
key = implicit ? [true, exchange_name, name] : [false, name]
|
|
186
|
-
QueueState.new(name: name, key: key, pending: [], subscribers: [], owners: [], bindings: [],
|
|
222
|
+
QueueState.new(name: name, key: key, pending: [], unacked: {}, subscribers: [], owners: [], bindings: [],
|
|
187
223
|
implicit: implicit, exchange_name: exchange_name)
|
|
188
224
|
end
|
|
189
225
|
|
|
@@ -199,7 +235,6 @@ module Legion
|
|
|
199
235
|
|
|
200
236
|
def unregister(subscription:, **)
|
|
201
237
|
subscription.callback = nil
|
|
202
|
-
subscription.owner = nil
|
|
203
238
|
subscription.queue.subscribers.delete(subscription)
|
|
204
239
|
end
|
|
205
240
|
|
|
@@ -211,14 +246,17 @@ module Legion
|
|
|
211
246
|
subscriptions.each do |subscription|
|
|
212
247
|
unregister(subscription: subscription)
|
|
213
248
|
await_completion(subscription: subscription)
|
|
249
|
+
requeue_unacked(subscription: subscription)
|
|
214
250
|
end
|
|
215
251
|
end
|
|
216
252
|
|
|
217
253
|
def deliver(queue:, subscription:, callback:, message:, **)
|
|
218
254
|
callback.call(message)
|
|
255
|
+
!message.requeued || subscription.callback.nil?
|
|
219
256
|
rescue StandardError
|
|
220
257
|
@mutex.synchronize do
|
|
221
|
-
queue.
|
|
258
|
+
outstanding = queue.unacked.delete(message.delivery_tag)
|
|
259
|
+
requeue_message(queue: queue, message: message) if !subscription.manual_ack || outstanding
|
|
222
260
|
end
|
|
223
261
|
raise
|
|
224
262
|
ensure
|
|
@@ -228,6 +266,23 @@ module Legion
|
|
|
228
266
|
end
|
|
229
267
|
end
|
|
230
268
|
|
|
269
|
+
def requeue_unacked(subscription:, **)
|
|
270
|
+
queue = subscription.queue
|
|
271
|
+
entries = queue.unacked.select { |_tag, (subscriber, _message)| subscriber.equal?(subscription) }
|
|
272
|
+
entries.reverse_each do |tag, (_subscriber, message)|
|
|
273
|
+
queue.unacked.delete(tag)
|
|
274
|
+
requeue_message(queue: queue, message: message)
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def requeue_message(queue:, message:, **)
|
|
279
|
+
return unless @queues[queue.key].equal?(queue)
|
|
280
|
+
|
|
281
|
+
message.redelivered = true
|
|
282
|
+
message.requeued = true
|
|
283
|
+
queue.pending.unshift(message)
|
|
284
|
+
end
|
|
285
|
+
|
|
231
286
|
def routes?(queue:, exchange_name:, routing_key:, exchange_type:, **)
|
|
232
287
|
return queue.exchange_name == exchange_name && queue.name == routing_key if queue.implicit
|
|
233
288
|
return queue.name == routing_key if exchange_name.empty?
|