omq 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +157 -0
- data/README.md +31 -4
- data/lib/omq/drop_queue.rb +54 -0
- data/lib/omq/engine.rb +103 -61
- data/lib/omq/monitor_event.rb +16 -0
- data/lib/omq/options.rb +6 -2
- data/lib/omq/pair.rb +2 -2
- data/lib/omq/pub_sub.rb +13 -12
- data/lib/omq/push_pull.rb +4 -4
- data/lib/omq/queue_interface.rb +73 -0
- data/lib/omq/readable.rb +2 -0
- data/lib/omq/req_rep.rb +4 -4
- data/lib/omq/router_dealer.rb +4 -4
- data/lib/omq/routing/dealer.rb +1 -1
- data/lib/omq/routing/fan_out.rb +26 -5
- data/lib/omq/routing/pair.rb +2 -2
- data/lib/omq/routing/pull.rb +1 -1
- data/lib/omq/routing/push.rb +2 -0
- data/lib/omq/routing/rep.rb +2 -2
- data/lib/omq/routing/req.rb +8 -3
- data/lib/omq/routing/round_robin.rb +4 -12
- data/lib/omq/routing/router.rb +2 -2
- data/lib/omq/routing/sub.rb +1 -2
- data/lib/omq/routing/xpub.rb +1 -1
- data/lib/omq/routing/xsub.rb +2 -2
- data/lib/omq/routing.rb +41 -11
- data/lib/omq/socket.rb +49 -2
- data/lib/omq/transport/inproc.rb +25 -7
- data/lib/omq/transport/ipc.rb +16 -4
- data/lib/omq/transport/tcp.rb +31 -8
- data/lib/omq/version.rb +1 -1
- data/lib/omq/writable.rb +1 -0
- data/lib/omq.rb +6 -16
- metadata +4 -15
- data/lib/omq/channel.rb +0 -14
- data/lib/omq/client_server.rb +0 -37
- data/lib/omq/peer.rb +0 -26
- data/lib/omq/radio_dish.rb +0 -74
- data/lib/omq/routing/channel.rb +0 -83
- data/lib/omq/routing/client.rb +0 -56
- data/lib/omq/routing/dish.rb +0 -78
- data/lib/omq/routing/gather.rb +0 -46
- data/lib/omq/routing/peer.rb +0 -101
- data/lib/omq/routing/radio.rb +0 -140
- data/lib/omq/routing/scatter.rb +0 -82
- data/lib/omq/routing/server.rb +0 -101
- data/lib/omq/scatter_gather.rb +0 -23
- data/lib/omq/single_frame.rb +0 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d5ea3ce51d24a9181cfe7a4c1f2680ed57c5dbba078010a54de5b1c13d27d484
|
|
4
|
+
data.tar.gz: 97f7e7fcb27a250e71af4732ff06298e7311af052a9d9a6b3868d73059373174
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9f60b7d45d1b3783b2bde482a6f826ec3b89bfff6a9532836dbd2a8a22622709e7c298cc7c2abae6e596641973636b8972efa92de7258bd1e4164ff62d782237
|
|
7
|
+
data.tar.gz: 7024b97bb46c9005b73bc7959cc4138bd5498a514af26253441613aa05206d357d1fbd002afff5820c9a3baff0c4351d66bcf1aa738955783272a2c2b459a697
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,163 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- **QoS infrastructure** — `Options#qos` attribute (default 0) and inproc
|
|
8
|
+
command queue support for QoS-enabled connections. The
|
|
9
|
+
[omq-qos](https://github.com/paddor/omq-qos) gem activates delivery
|
|
10
|
+
guarantees via prepends.
|
|
11
|
+
- **REQ send/recv ordering** — REQ sockets now enforce strict
|
|
12
|
+
send/recv/send/recv alternation. Calling `#send` twice without a
|
|
13
|
+
`#receive` in between raises `SocketError`.
|
|
14
|
+
- **DirectPipe command frame support** — `DirectPipe#receive_message`
|
|
15
|
+
accepts a block for command frames, matching the `Protocol::ZMTP::Connection`
|
|
16
|
+
interface. Enables inproc transports to handle ACK/NACK and other
|
|
17
|
+
command-level protocols.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- **`send_pump_idle?` visibility** — moved above `private` in `RoundRobin`
|
|
22
|
+
and `FanOut` so `Engine#drain_send_queues` can call it during socket close.
|
|
23
|
+
|
|
24
|
+
- **`Socket#monitor`** — observe connection lifecycle events via a
|
|
25
|
+
block-based API. Returns an `Async::Task` that yields `MonitorEvent`
|
|
26
|
+
(Data.define) instances for `:listening`, `:accepted`, `:connected`,
|
|
27
|
+
`:connect_delayed`, `:connect_retried`, `:handshake_succeeded`,
|
|
28
|
+
`:handshake_failed`, `:accept_failed`, `:bind_failed`, `:disconnected`,
|
|
29
|
+
`:closed`, and `:monitor_stopped`. Event types align with libzmq's
|
|
30
|
+
`zmq_socket_monitor` where applicable. Pattern-matchable, zero overhead
|
|
31
|
+
when no monitor is attached.
|
|
32
|
+
- **Pluggable transport registry** — `Engine.transports` is a scheme →
|
|
33
|
+
module hash. Built-in transports (`tcp`, `ipc`, `inproc`) are registered
|
|
34
|
+
at load time. External gems register via
|
|
35
|
+
`OMQ::Engine.transports["scheme"] = MyTransport`. Each transport
|
|
36
|
+
implements `.bind(endpoint, engine)` → Listener, `.connect(endpoint,
|
|
37
|
+
engine)`, and optionally `.validate_endpoint!(endpoint)`. Listeners
|
|
38
|
+
implement `#start_accept_loops(parent_task, &on_accepted)`, `#stop`,
|
|
39
|
+
`#endpoint`, and optionally `#port`.
|
|
40
|
+
- **Mutable error lists** — `CONNECTION_LOST` and `CONNECTION_FAILED` are
|
|
41
|
+
no longer frozen at load time. Transport plugins can append error classes
|
|
42
|
+
(e.g. `OpenSSL::SSL::SSLError`) before the first `#bind`/`#connect`,
|
|
43
|
+
which freezes both arrays.
|
|
44
|
+
|
|
45
|
+
- **`on_mute` option** — controls behavior when a socket enters the mute state
|
|
46
|
+
(HWM full). PUB, XPUB, and RADIO default to `on_mute: :drop_newest` — slow
|
|
47
|
+
subscribers are skipped in the fan-out rather than blocking the publisher.
|
|
48
|
+
SUB, XSUB, and DISH accept `on_mute: :drop_newest` or `:drop_oldest` to
|
|
49
|
+
drop messages on the receive side instead of applying backpressure. All other
|
|
50
|
+
socket types default to `:block` (existing behavior).
|
|
51
|
+
- **`DropQueue`** — bounded queue with `:drop_newest` (tail drop) and
|
|
52
|
+
`:drop_oldest` (head drop) strategies. Used by recv queues when `on_mute`
|
|
53
|
+
is a drop strategy.
|
|
54
|
+
- **`Routing.build_queue`** — factory method for building send/recv queues
|
|
55
|
+
based on HWM and mute strategy. Supports HWM of `0` or `nil` for unbounded
|
|
56
|
+
queues.
|
|
57
|
+
|
|
58
|
+
### Changed
|
|
59
|
+
|
|
60
|
+
- **`max_message_size` defaults to 1 MiB** — frames exceeding this limit cause
|
|
61
|
+
the connection to be dropped before the body is read from the wire, preventing
|
|
62
|
+
a malicious peer from causing arbitrary memory allocation. Set `socket.max_message_size = nil`
|
|
63
|
+
to restore the previous unlimited behavior.
|
|
64
|
+
- **Accept loops moved into Listeners** — `TCP::Listener` and
|
|
65
|
+
`IPC::Listener` now own their accept loop logic via
|
|
66
|
+
`#start_accept_loops(parent_task, &on_accepted)`. Engine delegates
|
|
67
|
+
via duck-type check. This enables external transports to define
|
|
68
|
+
custom accept behavior without modifying Engine.
|
|
69
|
+
- `Engine#transport_for` uses registry lookup instead of `case/when`.
|
|
70
|
+
- `Engine#validate_endpoint!` delegates to transport module.
|
|
71
|
+
- `Engine#bind` reads `listener.port` instead of parsing the endpoint
|
|
72
|
+
string.
|
|
73
|
+
|
|
74
|
+
### Removed
|
|
75
|
+
|
|
76
|
+
- **Draft socket types extracted** — `RADIO`, `DISH`, `CLIENT`, `SERVER`,
|
|
77
|
+
`SCATTER`, `GATHER`, `CHANNEL`, and `PEER` are no longer bundled with `omq`.
|
|
78
|
+
Use the [omq-draft](https://github.com/paddor/omq-draft) gem and require
|
|
79
|
+
the relevant entry point (`omq/draft/radiodish`, `omq/draft/clientserver`,
|
|
80
|
+
etc.).
|
|
81
|
+
- **UDP transport extracted** — `udp://` endpoints are provided by
|
|
82
|
+
`omq-draft` (via `require "omq/draft/radiodish"`). No longer registered by
|
|
83
|
+
default.
|
|
84
|
+
- **`Routing.for` plugin registry** — draft socket type removal added
|
|
85
|
+
`Routing.register(socket_type, strategy_class)` for external gems to
|
|
86
|
+
register routing strategies. Unknown types fall through the built-in
|
|
87
|
+
`case` to this registry before raising `ArgumentError`.
|
|
88
|
+
|
|
89
|
+
- **TLS transport** — extracted to the
|
|
90
|
+
[omq-transport-tls](https://github.com/paddor/omq-transport-tls) gem.
|
|
91
|
+
(Experimental) `require "omq/transport/tls"` to restore `tls+tcp://` support.
|
|
92
|
+
- `tls_context` / `tls_context=` removed from `Options` and `Socket`
|
|
93
|
+
(provided by omq-transport-tls).
|
|
94
|
+
- `OpenSSL::SSL::SSLError` removed from `CONNECTION_LOST` (added back
|
|
95
|
+
by omq-transport-tls).
|
|
96
|
+
- TLS benchmark transport removed from `bench_helper.rb` and `plot.rb`.
|
|
97
|
+
|
|
98
|
+
## 0.11.0
|
|
99
|
+
|
|
100
|
+
### Added
|
|
101
|
+
|
|
102
|
+
- **`backend:` kwarg** — all socket types accept `backend: :ffi` to use
|
|
103
|
+
the libzmq FFI backend (via the [omq-ffi](https://github.com/paddor/omq-ffi)
|
|
104
|
+
gem). Default is `:ruby` (pure Ruby ZMTP). Enables interop testing and
|
|
105
|
+
access to libzmq-specific features without changing the socket API.
|
|
106
|
+
- **TLS transport (`tls+tcp://`)** — TLS v1.3 on top of TCP using Ruby's
|
|
107
|
+
stdlib `openssl`. Set `socket.tls_context` to an `OpenSSL::SSL::SSLContext`
|
|
108
|
+
before bind/connect. Per-socket (not per-endpoint), frozen on first use.
|
|
109
|
+
SNI set automatically from the endpoint hostname. Bad TLS handshakes are
|
|
110
|
+
dropped without killing the accept loop. `OpenSSL::SSL::SSLError` added
|
|
111
|
+
to `CONNECTION_LOST` for automatic reconnection on TLS failures.
|
|
112
|
+
Accompanied by a draft RFC (`rfc/zmtp-tls.md`) defining the transport
|
|
113
|
+
mapping for ZMTP 3.1 over TLS.
|
|
114
|
+
- **PUB/RADIO fan-out pre-encoding** — ZMTP frames are encoded once per
|
|
115
|
+
message and written as raw wire bytes to all non-CURVE subscribers.
|
|
116
|
+
Eliminates redundant `Frame.new` + `#to_wire` calls during fan-out.
|
|
117
|
+
CURVE connections (which encrypt at the ZMTP level) still encode
|
|
118
|
+
per-connection. TLS, NULL, and PLAIN all benefit since TLS encrypts
|
|
119
|
+
below ZMTP. Requires protocol-zmtp `Frame.encode_message` and
|
|
120
|
+
`Connection#write_wire`.
|
|
121
|
+
- **CURVE benchmarks** — all per-pattern benchmarks now include CURVE
|
|
122
|
+
(via rbnacl) alongside inproc, ipc, tcp, and tls transports.
|
|
123
|
+
- **Engine `connection_wrapper` hook** — optional proc on Engine that wraps
|
|
124
|
+
new connections (both inproc and tcp/ipc) at creation time. Used by the
|
|
125
|
+
omq-ractor gem for per-connection serialization (Marshal for tcp/ipc,
|
|
126
|
+
`Ractor.make_shareable` for inproc).
|
|
127
|
+
- **Queue-style interface** — readable sockets gain `#dequeue(timeout:)`,
|
|
128
|
+
`#pop`, `#wait`, and `#each`; writable sockets gain `#enqueue` and
|
|
129
|
+
`#push`. Inspired by `Async::Queue`. `#wait` blocks indefinitely
|
|
130
|
+
(ignores `read_timeout`); `#each` returns gracefully on timeout.
|
|
131
|
+
- **Recv pump fairness** — each connection yields to the fiber scheduler
|
|
132
|
+
after 64 messages or 1 MB (whichever comes first). Prevents a fast or
|
|
133
|
+
large-message connection from starving slower peers when the consumer
|
|
134
|
+
keeps up. Byte counting gracefully handles non-string messages (e.g.
|
|
135
|
+
deserialized objects from connection wrappers).
|
|
136
|
+
- **Per-pattern benchmark suite** — `bench/{push_pull,req_rep,router_dealer,dealer_dealer,pub_sub,pair}/omq.rb`
|
|
137
|
+
with shared helpers (`bench_helper.rb`) and UnicodePlot braille line
|
|
138
|
+
charts (`plot.rb`). Each benchmark measures throughput (msg/s) and
|
|
139
|
+
bandwidth (MB/s) across transports (inproc, ipc, tcp, tls, curve),
|
|
140
|
+
message sizes (64 B–64 KB), and peer counts (1, 3). Plots are written to per-directory
|
|
141
|
+
`README.md` files for easy diffing across versions.
|
|
142
|
+
|
|
143
|
+
### Changed
|
|
144
|
+
|
|
145
|
+
- **SUB/XSUB `prefix:` kwarg renamed to `subscribe:`** — aligns with
|
|
146
|
+
ZeroMQ conventions. `subscribe: nil` (no subscription) remains the
|
|
147
|
+
default; pass `subscribe: ''` to subscribe to everything, or
|
|
148
|
+
`subscribe: 'topic.'` for a prefix filter.
|
|
149
|
+
- **Scenario benchmarks moved to `bench/scenarios/`** — broker,
|
|
150
|
+
draft_types, flush_batching, hwm_backpressure, large_messages,
|
|
151
|
+
multiframe, pubsub_fanout, ractors_vs_async, ractors_vs_fork,
|
|
152
|
+
reconnect_storm, and reqrep_throughput moved from `bench/` top level.
|
|
153
|
+
|
|
154
|
+
### Removed
|
|
155
|
+
|
|
156
|
+
- **Old flat benchmarks** — `bench/throughput.rb`, `bench/latency.rb`,
|
|
157
|
+
`bench/pipeline_mbps.rb`, `bench/run_all.sh` replaced by per-pattern
|
|
158
|
+
benchmarks.
|
|
159
|
+
- **`bench/cli/`** — CLI-specific benchmarks (fib pipeline, latency,
|
|
160
|
+
throughput shell scripts) moved to the omq-cli repository.
|
|
161
|
+
|
|
5
162
|
## 0.10.0 — 2026-04-01
|
|
6
163
|
|
|
7
164
|
### Added
|
data/README.md
CHANGED
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
|
|
10
10
|
OMQ builds ZeroMQ socket patterns on top of [protocol-zmtp](https://github.com/paddor/protocol-zmtp) (a pure Ruby [ZMTP 3.1](https://rfc.zeromq.org/spec/23/) codec) using [Async](https://github.com/socketry/async) fibers. It speaks native ZeroMQ on the wire and interoperates with libzmq, pyzmq, CZMQ, and everything else in the ZMQ ecosystem.
|
|
11
11
|
|
|
12
|
-
> **
|
|
12
|
+
> **980k msg/s** inproc | **38k msg/s** ipc | **31k msg/s** tcp
|
|
13
13
|
>
|
|
14
|
-
> **
|
|
14
|
+
> **10 µs** inproc latency | **71 µs** ipc | **82 µs** tcp
|
|
15
15
|
>
|
|
16
16
|
> Ruby 4.0 + YJIT on a Linux VM — see [`bench/`](bench/) for full results
|
|
17
17
|
|
|
@@ -30,7 +30,7 @@ See [GETTING_STARTED.md](GETTING_STARTED.md) for a ~30 min walkthrough of all ma
|
|
|
30
30
|
## Highlights
|
|
31
31
|
|
|
32
32
|
- **Zero dependencies on C** — no extensions, no FFI, no libzmq. `gem install` just works everywhere
|
|
33
|
-
- **Fast** — YJIT-optimized hot paths, batched sends,
|
|
33
|
+
- **Fast** — YJIT-optimized hot paths, batched sends, recv prefetching, direct-pipe inproc bypass. 980k msg/s inproc, 10 µs latency
|
|
34
34
|
- **[`omq` CLI](https://github.com/paddor/omq-cli)** — `gem install omq-cli` for a command-line tool with Ruby eval, Ractor parallelism, and script handlers
|
|
35
35
|
- **Every socket pattern** — req/rep, pub/sub, push/pull, dealer/router, xpub/xsub, pair, and all draft types
|
|
36
36
|
- **Every transport** — tcp, ipc (Unix domain sockets), inproc (in-process queues)
|
|
@@ -125,9 +125,29 @@ pull.close
|
|
|
125
125
|
|
|
126
126
|
The IO thread runs all pumps, reconnection, and heartbeating in the background. When you're inside an `Async` block, OMQ uses the existing reactor instead.
|
|
127
127
|
|
|
128
|
+
### Queue Interface
|
|
129
|
+
|
|
130
|
+
All sockets expose an `Async::Queue`-inspired interface:
|
|
131
|
+
|
|
132
|
+
| Async::Queue | OMQ Socket | Notes |
|
|
133
|
+
|---|---|---|
|
|
134
|
+
| `enqueue(item)` / `push(item)` | `enqueue(msg)` / `push(msg)` | Also: `send(msg)`, `<< msg` |
|
|
135
|
+
| `dequeue(timeout:)` / `pop(timeout:)` | `dequeue(timeout:)` / `pop(timeout:)` | Defaults to socket's `read_timeout` |
|
|
136
|
+
| `wait` | `wait` | Blocks indefinitely (ignores `read_timeout`) |
|
|
137
|
+
| `each` | `each` | Yields messages; returns on close or timeout |
|
|
138
|
+
|
|
139
|
+
```ruby
|
|
140
|
+
pull = OMQ::PULL.bind('inproc://work')
|
|
141
|
+
|
|
142
|
+
# iterate messages like a queue
|
|
143
|
+
pull.each do |msg|
|
|
144
|
+
puts msg.first
|
|
145
|
+
end
|
|
146
|
+
```
|
|
147
|
+
|
|
128
148
|
## Socket Types
|
|
129
149
|
|
|
130
|
-
All sockets are thread-safe. Default HWM is 1000 messages per socket. Classes live under `OMQ::` (alias: `ØMQ`).
|
|
150
|
+
All sockets are thread-safe. Default HWM is 1000 messages per socket. Default `max_message_size` is **1 MiB** — frames larger than this cause the connection to be dropped before the body is read from the wire. Set `socket.max_message_size = nil` to disable the limit or raise it as needed. Classes live under `OMQ::` (alias: `ØMQ`).
|
|
131
151
|
|
|
132
152
|
#### Standard (multipart messages)
|
|
133
153
|
|
|
@@ -142,6 +162,8 @@ All sockets are thread-safe. Default HWM is 1000 messages per socket. Classes li
|
|
|
142
162
|
|
|
143
163
|
#### Draft (single-frame only)
|
|
144
164
|
|
|
165
|
+
These require the `omq-draft` gem.
|
|
166
|
+
|
|
145
167
|
| Pattern | Send | Receive | When HWM full |
|
|
146
168
|
|---------|------|---------|---------------|
|
|
147
169
|
| **CLIENT** / **SERVER** | Round-robin / routing-ID | Fair-queue | Block |
|
|
@@ -163,6 +185,11 @@ echo "hello" | omq req -c tcp://localhost:5555
|
|
|
163
185
|
|
|
164
186
|
See the [omq-cli README](https://github.com/paddor/omq-cli) for full documentation.
|
|
165
187
|
|
|
188
|
+
## Companion Gems
|
|
189
|
+
|
|
190
|
+
- **[omq-ffi](https://github.com/paddor/omq-ffi)** — libzmq FFI backend. Same OMQ socket API, but backed by libzmq instead of the pure Ruby ZMTP stack. Useful for interop testing and when you need libzmq-specific features. Requires libzmq installed.
|
|
191
|
+
- **[omq-ractor](https://github.com/paddor/omq-ractor)** — bridge OMQ sockets into Ruby Ractors for true parallel processing across cores. I/O stays on the main Ractor, worker Ractors do pure computation.
|
|
192
|
+
|
|
166
193
|
## Development
|
|
167
194
|
|
|
168
195
|
```sh
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OMQ
|
|
4
|
+
# A bounded queue that drops messages when full instead of blocking.
|
|
5
|
+
#
|
|
6
|
+
# Two drop strategies:
|
|
7
|
+
# :drop_newest — discard the incoming message (tail drop)
|
|
8
|
+
# :drop_oldest — discard the head, then enqueue the new message
|
|
9
|
+
#
|
|
10
|
+
# Used by SUB/XSUB/DISH recv queues when on_mute is a drop strategy.
|
|
11
|
+
#
|
|
12
|
+
class DropQueue
|
|
13
|
+
# @param limit [Integer] maximum number of items
|
|
14
|
+
# @param strategy [Symbol] :drop_newest or :drop_oldest
|
|
15
|
+
#
|
|
16
|
+
def initialize(limit, strategy: :drop_newest)
|
|
17
|
+
@queue = Thread::SizedQueue.new(limit)
|
|
18
|
+
@strategy = strategy
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Enqueues an item. Drops according to the configured strategy if full.
|
|
22
|
+
#
|
|
23
|
+
# @param item [Object]
|
|
24
|
+
# @return [void]
|
|
25
|
+
#
|
|
26
|
+
def enqueue(item)
|
|
27
|
+
@queue.push(item, true)
|
|
28
|
+
rescue ThreadError
|
|
29
|
+
return if @strategy == :drop_newest
|
|
30
|
+
|
|
31
|
+
# :drop_oldest — discard head, enqueue new
|
|
32
|
+
@queue.pop(true) rescue nil
|
|
33
|
+
retry
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Removes and returns the next item, blocking if empty.
|
|
37
|
+
#
|
|
38
|
+
# @return [Object]
|
|
39
|
+
#
|
|
40
|
+
def dequeue(timeout: nil)
|
|
41
|
+
if timeout
|
|
42
|
+
@queue.pop(timeout: timeout)
|
|
43
|
+
else
|
|
44
|
+
@queue.pop
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @return [Boolean]
|
|
49
|
+
#
|
|
50
|
+
def empty?
|
|
51
|
+
@queue.empty?
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
data/lib/omq/engine.rb
CHANGED
|
@@ -9,6 +9,17 @@ module OMQ
|
|
|
9
9
|
# OMQ::Socket instance. Each socket type creates one Engine.
|
|
10
10
|
#
|
|
11
11
|
class Engine
|
|
12
|
+
# Scheme → transport module registry.
|
|
13
|
+
# Plugins add entries via +Engine.transports["scheme"] = MyTransport+.
|
|
14
|
+
#
|
|
15
|
+
@transports = {}
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
# @return [Hash{String => Module}] registered transports
|
|
19
|
+
attr_reader :transports
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
|
|
12
23
|
# @return [Symbol] socket type (e.g. :REQ, :PAIR)
|
|
13
24
|
#
|
|
14
25
|
attr_reader :socket_type
|
|
@@ -57,12 +68,18 @@ module OMQ
|
|
|
57
68
|
@on_io_thread = false
|
|
58
69
|
@connection_promises = {} # connection => Async::Promise
|
|
59
70
|
@fatal_error = nil
|
|
71
|
+
@monitor_queue = nil
|
|
60
72
|
end
|
|
61
73
|
|
|
62
74
|
|
|
63
75
|
attr_reader :peer_connected, :all_peers_gone, :connections, :parent_task
|
|
64
76
|
|
|
65
|
-
attr_writer :reconnect_enabled
|
|
77
|
+
attr_writer :reconnect_enabled, :monitor_queue
|
|
78
|
+
|
|
79
|
+
# Optional proc that wraps new connections (e.g. for serialization).
|
|
80
|
+
# Called with the raw connection; must return the (possibly wrapped) connection.
|
|
81
|
+
#
|
|
82
|
+
attr_accessor :connection_wrapper
|
|
66
83
|
|
|
67
84
|
|
|
68
85
|
# Spawns an inproc reconnect retry task under @parent_task.
|
|
@@ -87,12 +104,17 @@ module OMQ
|
|
|
87
104
|
# @raise [ArgumentError] on unsupported transport
|
|
88
105
|
#
|
|
89
106
|
def bind(endpoint)
|
|
107
|
+
freeze_error_lists!
|
|
90
108
|
transport = transport_for(endpoint)
|
|
91
109
|
listener = transport.bind(endpoint, self)
|
|
92
110
|
start_accept_loops(listener)
|
|
93
111
|
@listeners << listener
|
|
94
112
|
@last_endpoint = listener.endpoint
|
|
95
|
-
@last_tcp_port =
|
|
113
|
+
@last_tcp_port = listener.respond_to?(:port) ? listener.port : nil
|
|
114
|
+
emit_monitor_event(:listening, endpoint: listener.endpoint)
|
|
115
|
+
rescue => error
|
|
116
|
+
emit_monitor_event(:bind_failed, endpoint: endpoint, detail: { error: error })
|
|
117
|
+
raise
|
|
96
118
|
end
|
|
97
119
|
|
|
98
120
|
|
|
@@ -102,6 +124,7 @@ module OMQ
|
|
|
102
124
|
# @return [void]
|
|
103
125
|
#
|
|
104
126
|
def connect(endpoint)
|
|
127
|
+
freeze_error_lists!
|
|
105
128
|
validate_endpoint!(endpoint)
|
|
106
129
|
@connected_endpoints << endpoint
|
|
107
130
|
if endpoint.start_with?("inproc://")
|
|
@@ -110,6 +133,7 @@ module OMQ
|
|
|
110
133
|
transport.connect(endpoint, self)
|
|
111
134
|
else
|
|
112
135
|
# TCP/IPC connect in background — never blocks the caller
|
|
136
|
+
emit_monitor_event(:connect_delayed, endpoint: endpoint)
|
|
113
137
|
schedule_reconnect(endpoint, delay: 0)
|
|
114
138
|
end
|
|
115
139
|
end
|
|
@@ -163,6 +187,7 @@ module OMQ
|
|
|
163
187
|
# @return [void]
|
|
164
188
|
#
|
|
165
189
|
def handle_accepted(io, endpoint: nil)
|
|
190
|
+
emit_monitor_event(:accepted, endpoint: endpoint)
|
|
166
191
|
spawn_connection(io, as_server: true, endpoint: endpoint)
|
|
167
192
|
end
|
|
168
193
|
|
|
@@ -173,6 +198,7 @@ module OMQ
|
|
|
173
198
|
# @return [void]
|
|
174
199
|
#
|
|
175
200
|
def handle_connected(io, endpoint: nil)
|
|
201
|
+
emit_monitor_event(:connected, endpoint: endpoint)
|
|
176
202
|
spawn_connection(io, as_server: false, endpoint: endpoint)
|
|
177
203
|
end
|
|
178
204
|
|
|
@@ -184,10 +210,12 @@ module OMQ
|
|
|
184
210
|
# @return [void]
|
|
185
211
|
#
|
|
186
212
|
def connection_ready(pipe, endpoint: nil)
|
|
213
|
+
pipe = @connection_wrapper.call(pipe) if @connection_wrapper
|
|
187
214
|
@connections << pipe
|
|
188
215
|
@connection_endpoints[pipe] = endpoint if endpoint
|
|
189
216
|
@routing.connection_added(pipe)
|
|
190
217
|
@peer_connected.resolve(pipe)
|
|
218
|
+
emit_monitor_event(:handshake_succeeded, endpoint: endpoint)
|
|
191
219
|
end
|
|
192
220
|
|
|
193
221
|
|
|
@@ -262,6 +290,13 @@ module OMQ
|
|
|
262
290
|
# @yield [msg] optional per-message transform
|
|
263
291
|
# @return [#stop, nil] pump task handle, or nil for DirectPipe bypass
|
|
264
292
|
#
|
|
293
|
+
# Fairness limits for the recv pump. Yield to the scheduler
|
|
294
|
+
# after reading this many messages or bytes from one connection,
|
|
295
|
+
# whichever comes first. Prevents a fast or large-message
|
|
296
|
+
# connection from starving slower peers.
|
|
297
|
+
RECV_FAIRNESS_MESSAGES = 64
|
|
298
|
+
RECV_FAIRNESS_BYTES = 1 << 20 # 1 MB
|
|
299
|
+
|
|
265
300
|
def start_recv_pump(conn, recv_queue, &transform)
|
|
266
301
|
if conn.is_a?(Transport::Inproc::DirectPipe) && conn.peer
|
|
267
302
|
conn.peer.direct_recv_queue = recv_queue
|
|
@@ -270,11 +305,18 @@ module OMQ
|
|
|
270
305
|
end
|
|
271
306
|
|
|
272
307
|
if transform
|
|
273
|
-
@parent_task.async(transient: true, annotation: "recv pump") do
|
|
308
|
+
@parent_task.async(transient: true, annotation: "recv pump") do |task|
|
|
274
309
|
loop do
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
310
|
+
count = 0
|
|
311
|
+
bytes = 0
|
|
312
|
+
while count < RECV_FAIRNESS_MESSAGES && bytes < RECV_FAIRNESS_BYTES
|
|
313
|
+
msg = conn.receive_message
|
|
314
|
+
msg = transform.call(msg).freeze
|
|
315
|
+
recv_queue.enqueue(msg)
|
|
316
|
+
count += 1
|
|
317
|
+
bytes += msg.is_a?(Array) && msg.first.is_a?(String) ? msg.sum(&:bytesize) : 0
|
|
318
|
+
end
|
|
319
|
+
task.yield
|
|
278
320
|
end
|
|
279
321
|
rescue Async::Stop
|
|
280
322
|
rescue Protocol::ZMTP::Error, *CONNECTION_LOST
|
|
@@ -283,9 +325,17 @@ module OMQ
|
|
|
283
325
|
signal_fatal_error(error)
|
|
284
326
|
end
|
|
285
327
|
else
|
|
286
|
-
@parent_task.async(transient: true, annotation: "recv pump") do
|
|
328
|
+
@parent_task.async(transient: true, annotation: "recv pump") do |task|
|
|
287
329
|
loop do
|
|
288
|
-
|
|
330
|
+
count = 0
|
|
331
|
+
bytes = 0
|
|
332
|
+
while count < RECV_FAIRNESS_MESSAGES && bytes < RECV_FAIRNESS_BYTES
|
|
333
|
+
msg = conn.receive_message
|
|
334
|
+
recv_queue.enqueue(msg)
|
|
335
|
+
count += 1
|
|
336
|
+
bytes += msg.is_a?(Array) && msg.first.is_a?(String) ? msg.sum(&:bytesize) : 0
|
|
337
|
+
end
|
|
338
|
+
task.yield
|
|
289
339
|
end
|
|
290
340
|
rescue Async::Stop
|
|
291
341
|
rescue Protocol::ZMTP::Error, *CONNECTION_LOST
|
|
@@ -307,6 +357,7 @@ module OMQ
|
|
|
307
357
|
@connections.delete(connection)
|
|
308
358
|
@routing.connection_removed(connection)
|
|
309
359
|
connection.close
|
|
360
|
+
emit_monitor_event(:disconnected, endpoint: endpoint)
|
|
310
361
|
|
|
311
362
|
# Signal the connection task to exit.
|
|
312
363
|
done = @connection_promises.delete(connection)
|
|
@@ -365,6 +416,8 @@ module OMQ
|
|
|
365
416
|
@routing.stop rescue nil
|
|
366
417
|
@tasks.each { |t| t.stop rescue nil }
|
|
367
418
|
@tasks.clear
|
|
419
|
+
emit_monitor_event(:closed)
|
|
420
|
+
close_monitor_queue
|
|
368
421
|
end
|
|
369
422
|
|
|
370
423
|
|
|
@@ -485,13 +538,16 @@ module OMQ
|
|
|
485
538
|
)
|
|
486
539
|
conn.handshake!
|
|
487
540
|
start_heartbeat(conn)
|
|
541
|
+
conn = @connection_wrapper.call(conn) if @connection_wrapper
|
|
488
542
|
@connections << conn
|
|
489
543
|
@connection_endpoints[conn] = endpoint if endpoint
|
|
490
544
|
@connection_promises[conn] = done if done
|
|
491
545
|
@routing.connection_added(conn)
|
|
492
546
|
@peer_connected.resolve(conn)
|
|
547
|
+
emit_monitor_event(:handshake_succeeded, endpoint: endpoint)
|
|
493
548
|
conn
|
|
494
|
-
rescue Protocol::ZMTP::Error, *CONNECTION_LOST
|
|
549
|
+
rescue Protocol::ZMTP::Error, *CONNECTION_LOST => error
|
|
550
|
+
emit_monitor_event(:handshake_failed, endpoint: endpoint, detail: { error: error })
|
|
495
551
|
conn&.close
|
|
496
552
|
raise
|
|
497
553
|
end
|
|
@@ -556,6 +612,7 @@ module OMQ
|
|
|
556
612
|
delay = [delay * 2, max_delay].min if max_delay
|
|
557
613
|
# After first attempt with delay: 0, use the configured interval
|
|
558
614
|
delay = ri.is_a?(Range) ? ri.begin : ri if delay == 0
|
|
615
|
+
emit_monitor_event(:connect_retried, endpoint: endpoint, detail: { interval: delay })
|
|
559
616
|
end
|
|
560
617
|
end
|
|
561
618
|
rescue Async::Stop
|
|
@@ -572,70 +629,55 @@ module OMQ
|
|
|
572
629
|
# resolution failures during reconnect are retried with backoff.
|
|
573
630
|
#
|
|
574
631
|
def validate_endpoint!(endpoint)
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
Addrinfo.getaddrinfo(host, nil, nil, :STREAM) if host
|
|
632
|
+
transport = transport_for(endpoint)
|
|
633
|
+
transport.validate_endpoint!(endpoint) if transport.respond_to?(:validate_endpoint!)
|
|
578
634
|
end
|
|
579
635
|
|
|
580
636
|
|
|
637
|
+
|
|
581
638
|
def transport_for(endpoint)
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
639
|
+
scheme = endpoint[/\A([^:]+):\/\//, 1]
|
|
640
|
+
self.class.transports[scheme] or
|
|
641
|
+
raise ArgumentError, "unsupported transport: #{endpoint}"
|
|
642
|
+
end
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
# Delegates accept loop startup to the listener.
|
|
646
|
+
#
|
|
647
|
+
# Stream-based listeners (TCP, IPC, TLS, …) implement
|
|
648
|
+
# +#start_accept_loops+. Inproc listeners do not — connections
|
|
649
|
+
# are established synchronously during +connect+.
|
|
650
|
+
#
|
|
651
|
+
def start_accept_loops(listener)
|
|
652
|
+
return unless listener.respond_to?(:start_accept_loops)
|
|
653
|
+
listener.start_accept_loops(@parent_task) do |io|
|
|
654
|
+
handle_accepted(io, endpoint: listener.endpoint)
|
|
587
655
|
end
|
|
588
656
|
end
|
|
589
657
|
|
|
590
658
|
|
|
591
|
-
def
|
|
592
|
-
return
|
|
593
|
-
|
|
594
|
-
|
|
659
|
+
def freeze_error_lists!
|
|
660
|
+
return if OMQ::CONNECTION_LOST.frozen?
|
|
661
|
+
OMQ::CONNECTION_LOST.freeze
|
|
662
|
+
OMQ::CONNECTION_FAILED.freeze
|
|
595
663
|
end
|
|
596
664
|
|
|
597
665
|
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
def start_accept_loops(listener)
|
|
604
|
-
case listener
|
|
605
|
-
when Transport::TCP::Listener
|
|
606
|
-
tasks = listener.servers.map do |server|
|
|
607
|
-
@parent_task.async(transient: true, annotation: "tcp accept #{listener.endpoint}") do
|
|
608
|
-
loop do
|
|
609
|
-
client = server.accept
|
|
610
|
-
Async::Task.current.defer_stop do
|
|
611
|
-
handle_accepted(IO::Stream::Buffered.wrap(client), endpoint: listener.endpoint)
|
|
612
|
-
end
|
|
613
|
-
end
|
|
614
|
-
rescue Async::Stop
|
|
615
|
-
rescue IOError
|
|
616
|
-
# server closed
|
|
617
|
-
ensure
|
|
618
|
-
server.close rescue nil
|
|
619
|
-
end
|
|
620
|
-
end
|
|
621
|
-
listener.accept_tasks = tasks
|
|
666
|
+
def emit_monitor_event(type, endpoint: nil, detail: nil)
|
|
667
|
+
return unless @monitor_queue
|
|
668
|
+
@monitor_queue.push(MonitorEvent.new(type: type, endpoint: endpoint, detail: detail))
|
|
669
|
+
rescue Async::Stop, ClosedQueueError
|
|
670
|
+
end
|
|
622
671
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
Async::Task.current.defer_stop do
|
|
628
|
-
handle_accepted(IO::Stream::Buffered.wrap(client), endpoint: listener.endpoint)
|
|
629
|
-
end
|
|
630
|
-
end
|
|
631
|
-
rescue Async::Stop
|
|
632
|
-
rescue IOError
|
|
633
|
-
# server closed
|
|
634
|
-
ensure
|
|
635
|
-
listener.server.close rescue nil
|
|
636
|
-
end
|
|
637
|
-
listener.accept_task = task
|
|
638
|
-
end
|
|
672
|
+
|
|
673
|
+
def close_monitor_queue
|
|
674
|
+
return unless @monitor_queue
|
|
675
|
+
@monitor_queue.push(nil)
|
|
639
676
|
end
|
|
640
677
|
end
|
|
678
|
+
|
|
679
|
+
# Register built-in transports.
|
|
680
|
+
Engine.transports["tcp"] = Transport::TCP
|
|
681
|
+
Engine.transports["ipc"] = Transport::IPC
|
|
682
|
+
Engine.transports["inproc"] = Transport::Inproc
|
|
641
683
|
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OMQ
|
|
4
|
+
# Lifecycle event emitted by {Socket#monitor}.
|
|
5
|
+
#
|
|
6
|
+
# @!attribute [r] type
|
|
7
|
+
# @return [Symbol] event type (:listening, :connected, :disconnected, etc.)
|
|
8
|
+
# @!attribute [r] endpoint
|
|
9
|
+
# @return [String, nil] the endpoint involved
|
|
10
|
+
# @!attribute [r] detail
|
|
11
|
+
# @return [Hash, nil] extra context (e.g. { error: }, { interval: }, etc.)
|
|
12
|
+
#
|
|
13
|
+
MonitorEvent = Data.define(:type, :endpoint, :detail) do
|
|
14
|
+
def initialize(type:, endpoint: nil, detail: nil) = super
|
|
15
|
+
end
|
|
16
|
+
end
|
data/lib/omq/options.rb
CHANGED
|
@@ -23,9 +23,11 @@ module OMQ
|
|
|
23
23
|
@heartbeat_interval = nil # seconds, nil = disabled
|
|
24
24
|
@heartbeat_ttl = nil # seconds, nil = use heartbeat_interval
|
|
25
25
|
@heartbeat_timeout = nil # seconds, nil = use heartbeat_interval
|
|
26
|
-
@max_message_size =
|
|
26
|
+
@max_message_size = 1 << 20 # bytes (1 MiB default)
|
|
27
27
|
@conflate = false
|
|
28
|
+
@on_mute = :block # :block, :drop_newest, :drop_oldest
|
|
28
29
|
@mechanism = Protocol::ZMTP::Mechanism::Null.new
|
|
30
|
+
@qos = 0 # 0 = fire-and-forget, 1 = at-least-once (see omq-qos gem)
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
attr_accessor :send_hwm, :recv_hwm,
|
|
@@ -35,7 +37,9 @@ module OMQ
|
|
|
35
37
|
:reconnect_interval,
|
|
36
38
|
:heartbeat_interval, :heartbeat_ttl, :heartbeat_timeout,
|
|
37
39
|
:max_message_size,
|
|
38
|
-
:
|
|
40
|
+
:on_mute,
|
|
41
|
+
:mechanism,
|
|
42
|
+
:qos
|
|
39
43
|
|
|
40
44
|
alias_method :router_mandatory?, :router_mandatory
|
|
41
45
|
alias_method :recv_timeout, :read_timeout
|
data/lib/omq/pair.rb
CHANGED
|
@@ -5,8 +5,8 @@ module OMQ
|
|
|
5
5
|
include Readable
|
|
6
6
|
include Writable
|
|
7
7
|
|
|
8
|
-
def initialize(endpoints = nil, linger: 0)
|
|
9
|
-
_init_engine(:PAIR, linger: linger)
|
|
8
|
+
def initialize(endpoints = nil, linger: 0, backend: nil)
|
|
9
|
+
_init_engine(:PAIR, linger: linger, backend: backend)
|
|
10
10
|
_attach(endpoints, default: :connect)
|
|
11
11
|
end
|
|
12
12
|
end
|