omq 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +157 -0
  3. data/README.md +31 -4
  4. data/lib/omq/drop_queue.rb +54 -0
  5. data/lib/omq/engine.rb +103 -61
  6. data/lib/omq/monitor_event.rb +16 -0
  7. data/lib/omq/options.rb +6 -2
  8. data/lib/omq/pair.rb +2 -2
  9. data/lib/omq/pub_sub.rb +13 -12
  10. data/lib/omq/push_pull.rb +4 -4
  11. data/lib/omq/queue_interface.rb +73 -0
  12. data/lib/omq/readable.rb +2 -0
  13. data/lib/omq/req_rep.rb +4 -4
  14. data/lib/omq/router_dealer.rb +4 -4
  15. data/lib/omq/routing/dealer.rb +1 -1
  16. data/lib/omq/routing/fan_out.rb +26 -5
  17. data/lib/omq/routing/pair.rb +2 -2
  18. data/lib/omq/routing/pull.rb +1 -1
  19. data/lib/omq/routing/push.rb +2 -0
  20. data/lib/omq/routing/rep.rb +2 -2
  21. data/lib/omq/routing/req.rb +8 -3
  22. data/lib/omq/routing/round_robin.rb +4 -12
  23. data/lib/omq/routing/router.rb +2 -2
  24. data/lib/omq/routing/sub.rb +1 -2
  25. data/lib/omq/routing/xpub.rb +1 -1
  26. data/lib/omq/routing/xsub.rb +2 -2
  27. data/lib/omq/routing.rb +41 -11
  28. data/lib/omq/socket.rb +49 -2
  29. data/lib/omq/transport/inproc.rb +25 -7
  30. data/lib/omq/transport/ipc.rb +16 -4
  31. data/lib/omq/transport/tcp.rb +31 -8
  32. data/lib/omq/version.rb +1 -1
  33. data/lib/omq/writable.rb +1 -0
  34. data/lib/omq.rb +6 -16
  35. metadata +4 -15
  36. data/lib/omq/channel.rb +0 -14
  37. data/lib/omq/client_server.rb +0 -37
  38. data/lib/omq/peer.rb +0 -26
  39. data/lib/omq/radio_dish.rb +0 -74
  40. data/lib/omq/routing/channel.rb +0 -83
  41. data/lib/omq/routing/client.rb +0 -56
  42. data/lib/omq/routing/dish.rb +0 -78
  43. data/lib/omq/routing/gather.rb +0 -46
  44. data/lib/omq/routing/peer.rb +0 -101
  45. data/lib/omq/routing/radio.rb +0 -140
  46. data/lib/omq/routing/scatter.rb +0 -82
  47. data/lib/omq/routing/server.rb +0 -101
  48. data/lib/omq/scatter_gather.rb +0 -23
  49. data/lib/omq/single_frame.rb +0 -18
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3a81ea3e22e2133f508016129454fca72393068b259400e3d47aaf92d0948136
4
- data.tar.gz: 71839df55e8ae9edb9db9710b77c5449853cde909c60867958a46e8350f27ead
3
+ metadata.gz: d5ea3ce51d24a9181cfe7a4c1f2680ed57c5dbba078010a54de5b1c13d27d484
4
+ data.tar.gz: 97f7e7fcb27a250e71af4732ff06298e7311af052a9d9a6b3868d73059373174
5
5
  SHA512:
6
- metadata.gz: 9e9d603fb6f44b53626e37d88da5dea1297f932bfbc7b3af0b32e5ab4ebdc6c2151f4b7482bca70d5060d31ffc49ba3c771603b6ed1b4dd17ed51733339de67d
7
- data.tar.gz: b40525791795b8e040a4a275a628c1cdca42eab43fcabfab76407de97f669ff9c1b21d05490509c0e0f5cd0b53505c7f85281343b2c312bdb89fd3eee7d4af75
6
+ metadata.gz: 9f60b7d45d1b3783b2bde482a6f826ec3b89bfff6a9532836dbd2a8a22622709e7c298cc7c2abae6e596641973636b8972efa92de7258bd1e4164ff62d782237
7
+ data.tar.gz: 7024b97bb46c9005b73bc7959cc4138bd5498a514af26253441613aa05206d357d1fbd002afff5820c9a3baff0c4351d66bcf1aa738955783272a2c2b459a697
data/CHANGELOG.md CHANGED
@@ -2,6 +2,163 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ ### Added
6
+
7
+ - **QoS infrastructure** — `Options#qos` attribute (default 0) and inproc
8
+ command queue support for QoS-enabled connections. The
9
+ [omq-qos](https://github.com/paddor/omq-qos) gem activates delivery
10
+ guarantees via prepends.
11
+ - **REQ send/recv ordering** — REQ sockets now enforce strict
12
+ send/recv/send/recv alternation. Calling `#send` twice without a
13
+ `#receive` in between raises `SocketError`.
14
+ - **DirectPipe command frame support** — `DirectPipe#receive_message`
15
+ accepts a block for command frames, matching the `Protocol::ZMTP::Connection`
16
+ interface. Enables inproc transports to handle ACK/NACK and other
17
+ command-level protocols.
18
+
19
+ ### Fixed
20
+
21
+ - **`send_pump_idle?` visibility** — moved above `private` in `RoundRobin`
22
+ and `FanOut` so `Engine#drain_send_queues` can call it during socket close.
23
+
24
+ - **`Socket#monitor`** — observe connection lifecycle events via a
25
+ block-based API. Returns an `Async::Task` that yields `MonitorEvent`
26
+ (Data.define) instances for `:listening`, `:accepted`, `:connected`,
27
+ `:connect_delayed`, `:connect_retried`, `:handshake_succeeded`,
28
+ `:handshake_failed`, `:accept_failed`, `:bind_failed`, `:disconnected`,
29
+ `:closed`, and `:monitor_stopped`. Event types align with libzmq's
30
+ `zmq_socket_monitor` where applicable. Pattern-matchable, zero overhead
31
+ when no monitor is attached.
32
+ - **Pluggable transport registry** — `Engine.transports` is a scheme →
33
+ module hash. Built-in transports (`tcp`, `ipc`, `inproc`) are registered
34
+ at load time. External gems register via
35
+ `OMQ::Engine.transports["scheme"] = MyTransport`. Each transport
36
+ implements `.bind(endpoint, engine)` → Listener, `.connect(endpoint,
37
+ engine)`, and optionally `.validate_endpoint!(endpoint)`. Listeners
38
+ implement `#start_accept_loops(parent_task, &on_accepted)`, `#stop`,
39
+ `#endpoint`, and optionally `#port`.
40
+ - **Mutable error lists** — `CONNECTION_LOST` and `CONNECTION_FAILED` are
41
+ no longer frozen at load time. Transport plugins can append error classes
42
+ (e.g. `OpenSSL::SSL::SSLError`) before the first `#bind`/`#connect`,
43
+ which freezes both arrays.
44
+
45
+ - **`on_mute` option** — controls behavior when a socket enters the mute state
46
+ (HWM full). PUB, XPUB, and RADIO default to `on_mute: :drop_newest` — slow
47
+ subscribers are skipped in the fan-out rather than blocking the publisher.
48
+ SUB, XSUB, and DISH accept `on_mute: :drop_newest` or `:drop_oldest` to
49
+ drop messages on the receive side instead of applying backpressure. All other
50
+ socket types default to `:block` (existing behavior).
51
+ - **`DropQueue`** — bounded queue with `:drop_newest` (tail drop) and
52
+ `:drop_oldest` (head drop) strategies. Used by recv queues when `on_mute`
53
+ is a drop strategy.
54
+ - **`Routing.build_queue`** — factory method for building send/recv queues
55
+ based on HWM and mute strategy. Supports HWM of `0` or `nil` for unbounded
56
+ queues.
57
+
58
+ ### Changed
59
+
60
+ - **`max_message_size` defaults to 1 MiB** — frames exceeding this limit cause
61
+ the connection to be dropped before the body is read from the wire, preventing
62
+ a malicious peer from causing arbitrary memory allocation. Set `socket.max_message_size = nil`
63
+ to restore the previous unlimited behavior.
64
+ - **Accept loops moved into Listeners** — `TCP::Listener` and
65
+ `IPC::Listener` now own their accept loop logic via
66
+ `#start_accept_loops(parent_task, &on_accepted)`. Engine delegates
67
+ via duck-type check. This enables external transports to define
68
+ custom accept behavior without modifying Engine.
69
+ - `Engine#transport_for` uses registry lookup instead of `case/when`.
70
+ - `Engine#validate_endpoint!` delegates to transport module.
71
+ - `Engine#bind` reads `listener.port` instead of parsing the endpoint
72
+ string.
73
+
74
+ ### Removed
75
+
76
+ - **Draft socket types extracted** — `RADIO`, `DISH`, `CLIENT`, `SERVER`,
77
+ `SCATTER`, `GATHER`, `CHANNEL`, and `PEER` are no longer bundled with `omq`.
78
+ Use the [omq-draft](https://github.com/paddor/omq-draft) gem and require
79
+ the relevant entry point (`omq/draft/radiodish`, `omq/draft/clientserver`,
80
+ etc.).
81
+ - **UDP transport extracted** — `udp://` endpoints are provided by
82
+ `omq-draft` (via `require "omq/draft/radiodish"`). No longer registered by
83
+ default.
84
+ - **`Routing.for` plugin registry** — draft socket type removal added
85
+ `Routing.register(socket_type, strategy_class)` for external gems to
86
+ register routing strategies. Unknown types fall through the built-in
87
+ `case` to this registry before raising `ArgumentError`.
88
+
89
+ - **TLS transport** — extracted to the
90
+ [omq-transport-tls](https://github.com/paddor/omq-transport-tls) gem.
91
+ (Experimental) `require "omq/transport/tls"` to restore `tls+tcp://` support.
92
+ - `tls_context` / `tls_context=` removed from `Options` and `Socket`
93
+ (provided by omq-transport-tls).
94
+ - `OpenSSL::SSL::SSLError` removed from `CONNECTION_LOST` (added back
95
+ by omq-transport-tls).
96
+ - TLS benchmark transport removed from `bench_helper.rb` and `plot.rb`.
97
+
98
+ ## 0.11.0
99
+
100
+ ### Added
101
+
102
+ - **`backend:` kwarg** — all socket types accept `backend: :ffi` to use
103
+ the libzmq FFI backend (via the [omq-ffi](https://github.com/paddor/omq-ffi)
104
+ gem). Default is `:ruby` (pure Ruby ZMTP). Enables interop testing and
105
+ access to libzmq-specific features without changing the socket API.
106
+ - **TLS transport (`tls+tcp://`)** — TLS v1.3 on top of TCP using Ruby's
107
+ stdlib `openssl`. Set `socket.tls_context` to an `OpenSSL::SSL::SSLContext`
108
+ before bind/connect. Per-socket (not per-endpoint), frozen on first use.
109
+ SNI set automatically from the endpoint hostname. Bad TLS handshakes are
110
+ dropped without killing the accept loop. `OpenSSL::SSL::SSLError` added
111
+ to `CONNECTION_LOST` for automatic reconnection on TLS failures.
112
+ Accompanied by a draft RFC (`rfc/zmtp-tls.md`) defining the transport
113
+ mapping for ZMTP 3.1 over TLS.
114
+ - **PUB/RADIO fan-out pre-encoding** — ZMTP frames are encoded once per
115
+ message and written as raw wire bytes to all non-CURVE subscribers.
116
+ Eliminates redundant `Frame.new` + `#to_wire` calls during fan-out.
117
+ CURVE connections (which encrypt at the ZMTP level) still encode
118
+ per-connection. TLS, NULL, and PLAIN all benefit since TLS encrypts
119
+ below ZMTP. Requires protocol-zmtp `Frame.encode_message` and
120
+ `Connection#write_wire`.
121
+ - **CURVE benchmarks** — all per-pattern benchmarks now include CURVE
122
+ (via rbnacl) alongside inproc, ipc, tcp, and tls transports.
123
+ - **Engine `connection_wrapper` hook** — optional proc on Engine that wraps
124
+ new connections (both inproc and tcp/ipc) at creation time. Used by the
125
+ omq-ractor gem for per-connection serialization (Marshal for tcp/ipc,
126
+ `Ractor.make_shareable` for inproc).
127
+ - **Queue-style interface** — readable sockets gain `#dequeue(timeout:)`,
128
+ `#pop`, `#wait`, and `#each`; writable sockets gain `#enqueue` and
129
+ `#push`. Inspired by `Async::Queue`. `#wait` blocks indefinitely
130
+ (ignores `read_timeout`); `#each` returns gracefully on timeout.
131
+ - **Recv pump fairness** — each connection yields to the fiber scheduler
132
+ after 64 messages or 1 MB (whichever comes first). Prevents a fast or
133
+ large-message connection from starving slower peers when the consumer
134
+ keeps up. Byte counting gracefully handles non-string messages (e.g.
135
+ deserialized objects from connection wrappers).
136
+ - **Per-pattern benchmark suite** — `bench/{push_pull,req_rep,router_dealer,dealer_dealer,pub_sub,pair}/omq.rb`
137
+ with shared helpers (`bench_helper.rb`) and UnicodePlot braille line
138
+ charts (`plot.rb`). Each benchmark measures throughput (msg/s) and
139
+ bandwidth (MB/s) across transports (inproc, ipc, tcp, tls, curve),
140
+ message sizes (64 B–64 KB), and peer counts (1, 3). Plots are written to per-directory
141
+ `README.md` files for easy diffing across versions.
142
+
143
+ ### Changed
144
+
145
+ - **SUB/XSUB `prefix:` kwarg renamed to `subscribe:`** — aligns with
146
+ ZeroMQ conventions. `subscribe: nil` (no subscription) remains the
147
+ default; pass `subscribe: ''` to subscribe to everything, or
148
+ `subscribe: 'topic.'` for a prefix filter.
149
+ - **Scenario benchmarks moved to `bench/scenarios/`** — broker,
150
+ draft_types, flush_batching, hwm_backpressure, large_messages,
151
+ multiframe, pubsub_fanout, ractors_vs_async, ractors_vs_fork,
152
+ reconnect_storm, and reqrep_throughput moved from `bench/` top level.
153
+
154
+ ### Removed
155
+
156
+ - **Old flat benchmarks** — `bench/throughput.rb`, `bench/latency.rb`,
157
+ `bench/pipeline_mbps.rb`, `bench/run_all.sh` replaced by per-pattern
158
+ benchmarks.
159
+ - **`bench/cli/`** — CLI-specific benchmarks (fib pipeline, latency,
160
+ throughput shell scripts) moved to the omq-cli repository.
161
+
5
162
  ## 0.10.0 — 2026-04-01
6
163
 
7
164
  ### Added
data/README.md CHANGED
@@ -9,9 +9,9 @@
9
9
 
10
10
  OMQ builds ZeroMQ socket patterns on top of [protocol-zmtp](https://github.com/paddor/protocol-zmtp) (a pure Ruby [ZMTP 3.1](https://rfc.zeromq.org/spec/23/) codec) using [Async](https://github.com/socketry/async) fibers. It speaks native ZeroMQ on the wire and interoperates with libzmq, pyzmq, CZMQ, and everything else in the ZMQ ecosystem.
11
11
 
12
- > **234k msg/s** inproc | **49k msg/s** ipc | **36k msg/s** tcp
12
+ > **980k msg/s** inproc | **38k msg/s** ipc | **31k msg/s** tcp
13
13
  >
14
- > **12 µs** inproc latency | **51 µs** ipc | **62 µs** tcp
14
+ > **10 µs** inproc latency | **71 µs** ipc | **82 µs** tcp
15
15
  >
16
16
  > Ruby 4.0 + YJIT on a Linux VM — see [`bench/`](bench/) for full results
17
17
 
@@ -30,7 +30,7 @@ See [GETTING_STARTED.md](GETTING_STARTED.md) for a ~30 min walkthrough of all ma
30
30
  ## Highlights
31
31
 
32
32
  - **Zero dependencies on C** — no extensions, no FFI, no libzmq. `gem install` just works everywhere
33
- - **Fast** — YJIT-optimized hot paths, batched sends, 234k msg/s inproc with 12 µs latency
33
+ - **Fast** — YJIT-optimized hot paths, batched sends, recv prefetching, direct-pipe inproc bypass. 980k msg/s inproc, 10 µs latency
34
34
  - **[`omq` CLI](https://github.com/paddor/omq-cli)** — `gem install omq-cli` for a command-line tool with Ruby eval, Ractor parallelism, and script handlers
35
35
  - **Every socket pattern** — req/rep, pub/sub, push/pull, dealer/router, xpub/xsub, pair, and all draft types
36
36
  - **Every transport** — tcp, ipc (Unix domain sockets), inproc (in-process queues)
@@ -125,9 +125,29 @@ pull.close
125
125
 
126
126
  The IO thread runs all pumps, reconnection, and heartbeating in the background. When you're inside an `Async` block, OMQ uses the existing reactor instead.
127
127
 
128
+ ### Queue Interface
129
+
130
+ All sockets expose an `Async::Queue`-inspired interface:
131
+
132
+ | Async::Queue | OMQ Socket | Notes |
133
+ |---|---|---|
134
+ | `enqueue(item)` / `push(item)` | `enqueue(msg)` / `push(msg)` | Also: `send(msg)`, `<< msg` |
135
+ | `dequeue(timeout:)` / `pop(timeout:)` | `dequeue(timeout:)` / `pop(timeout:)` | Defaults to socket's `read_timeout` |
136
+ | `wait` | `wait` | Blocks indefinitely (ignores `read_timeout`) |
137
+ | `each` | `each` | Yields messages; returns on close or timeout |
138
+
139
+ ```ruby
140
+ pull = OMQ::PULL.bind('inproc://work')
141
+
142
+ # iterate messages like a queue
143
+ pull.each do |msg|
144
+ puts msg.first
145
+ end
146
+ ```
147
+
128
148
  ## Socket Types
129
149
 
130
- All sockets are thread-safe. Default HWM is 1000 messages per socket. Classes live under `OMQ::` (alias: `ØMQ`).
150
+ All sockets are thread-safe. Default HWM is 1000 messages per socket. Default `max_message_size` is **1 MiB** — frames larger than this cause the connection to be dropped before the body is read from the wire. Set `socket.max_message_size = nil` to disable the limit or raise it as needed. Classes live under `OMQ::` (alias: `ØMQ`).
131
151
 
132
152
  #### Standard (multipart messages)
133
153
 
@@ -142,6 +162,8 @@ All sockets are thread-safe. Default HWM is 1000 messages per socket. Classes li
142
162
 
143
163
  #### Draft (single-frame only)
144
164
 
165
+ These require the `omq-draft` gem.
166
+
145
167
  | Pattern | Send | Receive | When HWM full |
146
168
  |---------|------|---------|---------------|
147
169
  | **CLIENT** / **SERVER** | Round-robin / routing-ID | Fair-queue | Block |
@@ -163,6 +185,11 @@ echo "hello" | omq req -c tcp://localhost:5555
163
185
 
164
186
  See the [omq-cli README](https://github.com/paddor/omq-cli) for full documentation.
165
187
 
188
+ ## Companion Gems
189
+
190
+ - **[omq-ffi](https://github.com/paddor/omq-ffi)** — libzmq FFI backend. Same OMQ socket API, but backed by libzmq instead of the pure Ruby ZMTP stack. Useful for interop testing and when you need libzmq-specific features. Requires libzmq installed.
191
+ - **[omq-ractor](https://github.com/paddor/omq-ractor)** — bridge OMQ sockets into Ruby Ractors for true parallel processing across cores. I/O stays on the main Ractor, worker Ractors do pure computation.
192
+
166
193
  ## Development
167
194
 
168
195
  ```sh
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OMQ
4
+ # A bounded queue that drops messages when full instead of blocking.
5
+ #
6
+ # Two drop strategies:
7
+ # :drop_newest — discard the incoming message (tail drop)
8
+ # :drop_oldest — discard the head, then enqueue the new message
9
+ #
10
+ # Used by SUB/XSUB/DISH recv queues when on_mute is a drop strategy.
11
+ #
12
+ class DropQueue
13
+ # @param limit [Integer] maximum number of items
14
+ # @param strategy [Symbol] :drop_newest or :drop_oldest
15
+ #
16
+ def initialize(limit, strategy: :drop_newest)
17
+ @queue = Thread::SizedQueue.new(limit)
18
+ @strategy = strategy
19
+ end
20
+
21
+ # Enqueues an item. Drops according to the configured strategy if full.
22
+ #
23
+ # @param item [Object]
24
+ # @return [void]
25
+ #
26
+ def enqueue(item)
27
+ @queue.push(item, true)
28
+ rescue ThreadError
29
+ return if @strategy == :drop_newest
30
+
31
+ # :drop_oldest — discard head, enqueue new
32
+ @queue.pop(true) rescue nil
33
+ retry
34
+ end
35
+
36
+ # Removes and returns the next item, blocking if empty.
37
+ #
38
+ # @return [Object]
39
+ #
40
+ def dequeue(timeout: nil)
41
+ if timeout
42
+ @queue.pop(timeout: timeout)
43
+ else
44
+ @queue.pop
45
+ end
46
+ end
47
+
48
+ # @return [Boolean]
49
+ #
50
+ def empty?
51
+ @queue.empty?
52
+ end
53
+ end
54
+ end
data/lib/omq/engine.rb CHANGED
@@ -9,6 +9,17 @@ module OMQ
9
9
  # OMQ::Socket instance. Each socket type creates one Engine.
10
10
  #
11
11
  class Engine
12
+ # Scheme → transport module registry.
13
+ # Plugins add entries via +Engine.transports["scheme"] = MyTransport+.
14
+ #
15
+ @transports = {}
16
+
17
+ class << self
18
+ # @return [Hash{String => Module}] registered transports
19
+ attr_reader :transports
20
+ end
21
+
22
+
12
23
  # @return [Symbol] socket type (e.g. :REQ, :PAIR)
13
24
  #
14
25
  attr_reader :socket_type
@@ -57,12 +68,18 @@ module OMQ
57
68
  @on_io_thread = false
58
69
  @connection_promises = {} # connection => Async::Promise
59
70
  @fatal_error = nil
71
+ @monitor_queue = nil
60
72
  end
61
73
 
62
74
 
63
75
  attr_reader :peer_connected, :all_peers_gone, :connections, :parent_task
64
76
 
65
- attr_writer :reconnect_enabled
77
+ attr_writer :reconnect_enabled, :monitor_queue
78
+
79
+ # Optional proc that wraps new connections (e.g. for serialization).
80
+ # Called with the raw connection; must return the (possibly wrapped) connection.
81
+ #
82
+ attr_accessor :connection_wrapper
66
83
 
67
84
 
68
85
  # Spawns an inproc reconnect retry task under @parent_task.
@@ -87,12 +104,17 @@ module OMQ
87
104
  # @raise [ArgumentError] on unsupported transport
88
105
  #
89
106
  def bind(endpoint)
107
+ freeze_error_lists!
90
108
  transport = transport_for(endpoint)
91
109
  listener = transport.bind(endpoint, self)
92
110
  start_accept_loops(listener)
93
111
  @listeners << listener
94
112
  @last_endpoint = listener.endpoint
95
- @last_tcp_port = extract_tcp_port(listener.endpoint)
113
+ @last_tcp_port = listener.respond_to?(:port) ? listener.port : nil
114
+ emit_monitor_event(:listening, endpoint: listener.endpoint)
115
+ rescue => error
116
+ emit_monitor_event(:bind_failed, endpoint: endpoint, detail: { error: error })
117
+ raise
96
118
  end
97
119
 
98
120
 
@@ -102,6 +124,7 @@ module OMQ
102
124
  # @return [void]
103
125
  #
104
126
  def connect(endpoint)
127
+ freeze_error_lists!
105
128
  validate_endpoint!(endpoint)
106
129
  @connected_endpoints << endpoint
107
130
  if endpoint.start_with?("inproc://")
@@ -110,6 +133,7 @@ module OMQ
110
133
  transport.connect(endpoint, self)
111
134
  else
112
135
  # TCP/IPC connect in background — never blocks the caller
136
+ emit_monitor_event(:connect_delayed, endpoint: endpoint)
113
137
  schedule_reconnect(endpoint, delay: 0)
114
138
  end
115
139
  end
@@ -163,6 +187,7 @@ module OMQ
163
187
  # @return [void]
164
188
  #
165
189
  def handle_accepted(io, endpoint: nil)
190
+ emit_monitor_event(:accepted, endpoint: endpoint)
166
191
  spawn_connection(io, as_server: true, endpoint: endpoint)
167
192
  end
168
193
 
@@ -173,6 +198,7 @@ module OMQ
173
198
  # @return [void]
174
199
  #
175
200
  def handle_connected(io, endpoint: nil)
201
+ emit_monitor_event(:connected, endpoint: endpoint)
176
202
  spawn_connection(io, as_server: false, endpoint: endpoint)
177
203
  end
178
204
 
@@ -184,10 +210,12 @@ module OMQ
184
210
  # @return [void]
185
211
  #
186
212
  def connection_ready(pipe, endpoint: nil)
213
+ pipe = @connection_wrapper.call(pipe) if @connection_wrapper
187
214
  @connections << pipe
188
215
  @connection_endpoints[pipe] = endpoint if endpoint
189
216
  @routing.connection_added(pipe)
190
217
  @peer_connected.resolve(pipe)
218
+ emit_monitor_event(:handshake_succeeded, endpoint: endpoint)
191
219
  end
192
220
 
193
221
 
@@ -262,6 +290,13 @@ module OMQ
262
290
  # @yield [msg] optional per-message transform
263
291
  # @return [#stop, nil] pump task handle, or nil for DirectPipe bypass
264
292
  #
293
+ # Fairness limits for the recv pump. Yield to the scheduler
294
+ # after reading this many messages or bytes from one connection,
295
+ # whichever comes first. Prevents a fast or large-message
296
+ # connection from starving slower peers.
297
+ RECV_FAIRNESS_MESSAGES = 64
298
+ RECV_FAIRNESS_BYTES = 1 << 20 # 1 MB
299
+
265
300
  def start_recv_pump(conn, recv_queue, &transform)
266
301
  if conn.is_a?(Transport::Inproc::DirectPipe) && conn.peer
267
302
  conn.peer.direct_recv_queue = recv_queue
@@ -270,11 +305,18 @@ module OMQ
270
305
  end
271
306
 
272
307
  if transform
273
- @parent_task.async(transient: true, annotation: "recv pump") do
308
+ @parent_task.async(transient: true, annotation: "recv pump") do |task|
274
309
  loop do
275
- msg = conn.receive_message
276
- msg = transform.call(msg).freeze
277
- recv_queue.enqueue(msg)
310
+ count = 0
311
+ bytes = 0
312
+ while count < RECV_FAIRNESS_MESSAGES && bytes < RECV_FAIRNESS_BYTES
313
+ msg = conn.receive_message
314
+ msg = transform.call(msg).freeze
315
+ recv_queue.enqueue(msg)
316
+ count += 1
317
+ bytes += msg.is_a?(Array) && msg.first.is_a?(String) ? msg.sum(&:bytesize) : 0
318
+ end
319
+ task.yield
278
320
  end
279
321
  rescue Async::Stop
280
322
  rescue Protocol::ZMTP::Error, *CONNECTION_LOST
@@ -283,9 +325,17 @@ module OMQ
283
325
  signal_fatal_error(error)
284
326
  end
285
327
  else
286
- @parent_task.async(transient: true, annotation: "recv pump") do
328
+ @parent_task.async(transient: true, annotation: "recv pump") do |task|
287
329
  loop do
288
- recv_queue.enqueue(conn.receive_message)
330
+ count = 0
331
+ bytes = 0
332
+ while count < RECV_FAIRNESS_MESSAGES && bytes < RECV_FAIRNESS_BYTES
333
+ msg = conn.receive_message
334
+ recv_queue.enqueue(msg)
335
+ count += 1
336
+ bytes += msg.is_a?(Array) && msg.first.is_a?(String) ? msg.sum(&:bytesize) : 0
337
+ end
338
+ task.yield
289
339
  end
290
340
  rescue Async::Stop
291
341
  rescue Protocol::ZMTP::Error, *CONNECTION_LOST
@@ -307,6 +357,7 @@ module OMQ
307
357
  @connections.delete(connection)
308
358
  @routing.connection_removed(connection)
309
359
  connection.close
360
+ emit_monitor_event(:disconnected, endpoint: endpoint)
310
361
 
311
362
  # Signal the connection task to exit.
312
363
  done = @connection_promises.delete(connection)
@@ -365,6 +416,8 @@ module OMQ
365
416
  @routing.stop rescue nil
366
417
  @tasks.each { |t| t.stop rescue nil }
367
418
  @tasks.clear
419
+ emit_monitor_event(:closed)
420
+ close_monitor_queue
368
421
  end
369
422
 
370
423
 
@@ -485,13 +538,16 @@ module OMQ
485
538
  )
486
539
  conn.handshake!
487
540
  start_heartbeat(conn)
541
+ conn = @connection_wrapper.call(conn) if @connection_wrapper
488
542
  @connections << conn
489
543
  @connection_endpoints[conn] = endpoint if endpoint
490
544
  @connection_promises[conn] = done if done
491
545
  @routing.connection_added(conn)
492
546
  @peer_connected.resolve(conn)
547
+ emit_monitor_event(:handshake_succeeded, endpoint: endpoint)
493
548
  conn
494
- rescue Protocol::ZMTP::Error, *CONNECTION_LOST
549
+ rescue Protocol::ZMTP::Error, *CONNECTION_LOST => error
550
+ emit_monitor_event(:handshake_failed, endpoint: endpoint, detail: { error: error })
495
551
  conn&.close
496
552
  raise
497
553
  end
@@ -556,6 +612,7 @@ module OMQ
556
612
  delay = [delay * 2, max_delay].min if max_delay
557
613
  # After first attempt with delay: 0, use the configured interval
558
614
  delay = ri.is_a?(Range) ? ri.begin : ri if delay == 0
615
+ emit_monitor_event(:connect_retried, endpoint: endpoint, detail: { interval: delay })
559
616
  end
560
617
  end
561
618
  rescue Async::Stop
@@ -572,70 +629,55 @@ module OMQ
572
629
  # resolution failures during reconnect are retried with backoff.
573
630
  #
574
631
  def validate_endpoint!(endpoint)
575
- return unless endpoint.start_with?("tcp://")
576
- host = URI.parse(endpoint.sub("tcp://", "http://")).hostname
577
- Addrinfo.getaddrinfo(host, nil, nil, :STREAM) if host
632
+ transport = transport_for(endpoint)
633
+ transport.validate_endpoint!(endpoint) if transport.respond_to?(:validate_endpoint!)
578
634
  end
579
635
 
580
636
 
637
+
581
638
  def transport_for(endpoint)
582
- case endpoint
583
- when /\Atcp:\/\// then Transport::TCP
584
- when /\Aipc:\/\// then Transport::IPC
585
- when /\Ainproc:\/\// then Transport::Inproc
586
- else raise ArgumentError, "unsupported transport: #{endpoint}"
639
+ scheme = endpoint[/\A([^:]+):\/\//, 1]
640
+ self.class.transports[scheme] or
641
+ raise ArgumentError, "unsupported transport: #{endpoint}"
642
+ end
643
+
644
+
645
+ # Delegates accept loop startup to the listener.
646
+ #
647
+ # Stream-based listeners (TCP, IPC, TLS, …) implement
648
+ # +#start_accept_loops+. Inproc listeners do not — connections
649
+ # are established synchronously during +connect+.
650
+ #
651
+ def start_accept_loops(listener)
652
+ return unless listener.respond_to?(:start_accept_loops)
653
+ listener.start_accept_loops(@parent_task) do |io|
654
+ handle_accepted(io, endpoint: listener.endpoint)
587
655
  end
588
656
  end
589
657
 
590
658
 
591
- def extract_tcp_port(endpoint)
592
- return nil unless endpoint&.start_with?("tcp://")
593
- port = endpoint.split(":").last.to_i
594
- port.positive? ? port : nil
659
+ def freeze_error_lists!
660
+ return if OMQ::CONNECTION_LOST.frozen?
661
+ OMQ::CONNECTION_LOST.freeze
662
+ OMQ::CONNECTION_FAILED.freeze
595
663
  end
596
664
 
597
665
 
598
- # Spawns accept loops for a listener under @parent_task.
599
- #
600
- # TCP listeners have multiple server sockets (IPv4/IPv6);
601
- # IPC listeners have one. Inproc listeners have none.
602
- #
603
- def start_accept_loops(listener)
604
- case listener
605
- when Transport::TCP::Listener
606
- tasks = listener.servers.map do |server|
607
- @parent_task.async(transient: true, annotation: "tcp accept #{listener.endpoint}") do
608
- loop do
609
- client = server.accept
610
- Async::Task.current.defer_stop do
611
- handle_accepted(IO::Stream::Buffered.wrap(client), endpoint: listener.endpoint)
612
- end
613
- end
614
- rescue Async::Stop
615
- rescue IOError
616
- # server closed
617
- ensure
618
- server.close rescue nil
619
- end
620
- end
621
- listener.accept_tasks = tasks
666
+ def emit_monitor_event(type, endpoint: nil, detail: nil)
667
+ return unless @monitor_queue
668
+ @monitor_queue.push(MonitorEvent.new(type: type, endpoint: endpoint, detail: detail))
669
+ rescue Async::Stop, ClosedQueueError
670
+ end
622
671
 
623
- when Transport::IPC::Listener
624
- task = @parent_task.async(transient: true, annotation: "ipc accept #{listener.endpoint}") do
625
- loop do
626
- client = listener.server.accept
627
- Async::Task.current.defer_stop do
628
- handle_accepted(IO::Stream::Buffered.wrap(client), endpoint: listener.endpoint)
629
- end
630
- end
631
- rescue Async::Stop
632
- rescue IOError
633
- # server closed
634
- ensure
635
- listener.server.close rescue nil
636
- end
637
- listener.accept_task = task
638
- end
672
+
673
+ def close_monitor_queue
674
+ return unless @monitor_queue
675
+ @monitor_queue.push(nil)
639
676
  end
640
677
  end
678
+
679
+ # Register built-in transports.
680
+ Engine.transports["tcp"] = Transport::TCP
681
+ Engine.transports["ipc"] = Transport::IPC
682
+ Engine.transports["inproc"] = Transport::Inproc
641
683
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OMQ
4
+ # Lifecycle event emitted by {Socket#monitor}.
5
+ #
6
+ # @!attribute [r] type
7
+ # @return [Symbol] event type (:listening, :connected, :disconnected, etc.)
8
+ # @!attribute [r] endpoint
9
+ # @return [String, nil] the endpoint involved
10
+ # @!attribute [r] detail
11
+ # @return [Hash, nil] extra context (e.g. { error: }, { interval: }, etc.)
12
+ #
13
+ MonitorEvent = Data.define(:type, :endpoint, :detail) do
14
+ def initialize(type:, endpoint: nil, detail: nil) = super
15
+ end
16
+ end
data/lib/omq/options.rb CHANGED
@@ -23,9 +23,11 @@ module OMQ
23
23
  @heartbeat_interval = nil # seconds, nil = disabled
24
24
  @heartbeat_ttl = nil # seconds, nil = use heartbeat_interval
25
25
  @heartbeat_timeout = nil # seconds, nil = use heartbeat_interval
26
- @max_message_size = nil # bytes, nil = unlimited
26
+ @max_message_size = 1 << 20 # bytes (1 MiB default)
27
27
  @conflate = false
28
+ @on_mute = :block # :block, :drop_newest, :drop_oldest
28
29
  @mechanism = Protocol::ZMTP::Mechanism::Null.new
30
+ @qos = 0 # 0 = fire-and-forget, 1 = at-least-once (see omq-qos gem)
29
31
  end
30
32
 
31
33
  attr_accessor :send_hwm, :recv_hwm,
@@ -35,7 +37,9 @@ module OMQ
35
37
  :reconnect_interval,
36
38
  :heartbeat_interval, :heartbeat_ttl, :heartbeat_timeout,
37
39
  :max_message_size,
38
- :mechanism
40
+ :on_mute,
41
+ :mechanism,
42
+ :qos
39
43
 
40
44
  alias_method :router_mandatory?, :router_mandatory
41
45
  alias_method :recv_timeout, :read_timeout
data/lib/omq/pair.rb CHANGED
@@ -5,8 +5,8 @@ module OMQ
5
5
  include Readable
6
6
  include Writable
7
7
 
8
- def initialize(endpoints = nil, linger: 0)
9
- _init_engine(:PAIR, linger: linger)
8
+ def initialize(endpoints = nil, linger: 0, backend: nil)
9
+ _init_engine(:PAIR, linger: linger, backend: backend)
10
10
  _attach(endpoints, default: :connect)
11
11
  end
12
12
  end