omq 0.22.1 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +162 -0
  3. data/README.md +17 -21
  4. data/lib/omq/channel.rb +35 -0
  5. data/lib/omq/client_server.rb +72 -0
  6. data/lib/omq/constants.rb +68 -0
  7. data/lib/omq/engine/connection_lifecycle.rb +22 -8
  8. data/lib/omq/engine/heartbeat.rb +3 -4
  9. data/lib/omq/engine/maintenance.rb +4 -5
  10. data/lib/omq/engine/reconnect.rb +12 -11
  11. data/lib/omq/engine/recv_pump.rb +10 -10
  12. data/lib/omq/engine/socket_lifecycle.rb +26 -9
  13. data/lib/omq/engine.rb +202 -90
  14. data/lib/omq/peer.rb +49 -0
  15. data/lib/omq/pub_sub.rb +2 -2
  16. data/lib/omq/radio_dish.rb +122 -0
  17. data/lib/omq/reactor.rb +14 -5
  18. data/lib/omq/readable.rb +5 -1
  19. data/lib/omq/routing/channel.rb +110 -0
  20. data/lib/omq/routing/client.rb +70 -0
  21. data/lib/omq/routing/conn_send_pump.rb +5 -8
  22. data/lib/omq/routing/dealer.rb +3 -15
  23. data/lib/omq/routing/dish.rb +94 -0
  24. data/lib/omq/routing/fan_out.rb +12 -16
  25. data/lib/omq/routing/gather.rb +60 -0
  26. data/lib/omq/routing/pair.rb +7 -26
  27. data/lib/omq/routing/peer.rb +95 -0
  28. data/lib/omq/routing/pub.rb +2 -13
  29. data/lib/omq/routing/pull.rb +3 -15
  30. data/lib/omq/routing/push.rb +4 -13
  31. data/lib/omq/routing/radio.rb +187 -0
  32. data/lib/omq/routing/rep.rb +5 -19
  33. data/lib/omq/routing/req.rb +6 -18
  34. data/lib/omq/routing/round_robin.rb +15 -19
  35. data/lib/omq/routing/router.rb +5 -19
  36. data/lib/omq/routing/scatter.rb +76 -0
  37. data/lib/omq/routing/server.rb +90 -0
  38. data/lib/omq/routing/sub.rb +3 -15
  39. data/lib/omq/routing/xpub.rb +2 -13
  40. data/lib/omq/routing/xsub.rb +8 -25
  41. data/lib/omq/scatter_gather.rb +56 -0
  42. data/lib/omq/socket.rb +8 -23
  43. data/lib/omq/transport/inproc/{direct_pipe.rb → pipe.rb} +26 -24
  44. data/lib/omq/transport/inproc.rb +22 -14
  45. data/lib/omq/transport/ipc.rb +41 -13
  46. data/lib/omq/transport/tcp.rb +59 -23
  47. data/lib/omq/transport/udp.rb +281 -0
  48. data/lib/omq/version.rb +1 -1
  49. data/lib/omq/writable.rb +11 -42
  50. data/lib/omq.rb +9 -64
  51. metadata +17 -3
  52. data/lib/omq/monitor_event.rb +0 -16
@@ -14,7 +14,7 @@ module OMQ
14
14
  # This reduces inproc from 3 queue hops to 2 (send_queue →
15
15
  # recv_queue), eliminating the internal pipe queue in between.
16
16
  #
17
- class DirectPipe
17
+ class Pipe
18
18
  # @return [String] peer's socket type
19
19
  #
20
20
  attr_reader :peer_socket_type
@@ -39,7 +39,7 @@ module OMQ
39
39
  attr_reader :peer_identity
40
40
 
41
41
 
42
- # @return [DirectPipe, nil] the other end of this pipe pair
42
+ # @return [Pipe, nil] the other end of this pipe pair
43
43
  #
44
44
  attr_accessor :peer
45
45
 
@@ -50,12 +50,6 @@ module OMQ
50
50
  attr_reader :direct_recv_queue
51
51
 
52
52
 
53
- # @return [Proc, nil] optional transform applied before
54
- # enqueuing into {#direct_recv_queue}
55
- #
56
- attr_accessor :direct_recv_transform
57
-
58
-
59
53
  # @param send_queue [Async::Queue, nil] outgoing command queue
60
54
  # (nil for non-PUB/SUB types that don't exchange commands)
61
55
  # @param receive_queue [Async::Queue, nil] incoming command queue
@@ -75,18 +69,27 @@ module OMQ
75
69
  end
76
70
 
77
71
 
78
- # Sets the direct recv queue. Drains any messages that were
79
- # buffered before the queue was available.
72
+ # Wires up the direct recv fast-path. Called once by the recv
73
+ # pump when the receiving side of an inproc pipe pair is set up.
74
+ # After this, peer-side {#send_message} calls enqueue straight
75
+ # into +queue+ instead of hopping through the intermediate pipe
76
+ # queue and a recv pump fiber.
80
77
  #
81
- # @param queue [Async::LimitedQueue, nil]
78
+ # Drains any messages the peer buffered into +@pending_direct+
79
+ # before the queue was available.
80
+ #
81
+ # @param queue [Async::LimitedQueue]
82
+ # @param transform [Proc, nil] optional per-message transform
82
83
  # @return [void]
83
84
  #
84
- def direct_recv_queue=(queue)
85
- @direct_recv_queue = queue
86
- if queue && @pending_direct
87
- @pending_direct.each { |msg| queue.enqueue(msg) }
88
- @pending_direct = nil
89
- end
85
+ def wire_direct_recv(queue, transform)
86
+ @direct_recv_transform = transform
87
+ @direct_recv_queue = queue
88
+
89
+ return unless @pending_direct
90
+
91
+ @pending_direct.each { |msg| queue.enqueue(msg) }
92
+ @pending_direct = nil
90
93
  end
91
94
 
92
95
 
@@ -97,6 +100,7 @@ module OMQ
97
100
  #
98
101
  def send_message(parts)
99
102
  raise IOError, "closed" if @closed
103
+
100
104
  if @direct_recv_queue
101
105
  @direct_recv_queue.enqueue(apply_transform(parts))
102
106
  elsif @send_queue
@@ -112,7 +116,7 @@ module OMQ
112
116
 
113
117
  # Batched form, for parity with Protocol::ZMTP::Connection. The
114
118
  # work-stealing pumps call this when they dequeue more than one
115
- # message at once; DirectPipe just loops — no mutex to amortize.
119
+ # message at once; Pipe just loops — no mutex to amortize.
116
120
  #
117
121
  # @param messages [Array<Array<String>>]
118
122
  # @return [void]
@@ -145,14 +149,13 @@ module OMQ
145
149
  #
146
150
  def receive_message
147
151
  loop do
148
- item = @receive_queue.dequeue
149
-
150
- raise EOFError, "connection closed" if item.nil?
152
+ item = @receive_queue.dequeue or raise EOFError, "connection closed"
151
153
 
152
154
  if item.is_a?(Array) && item.first == :command
153
155
  if block_given?
154
156
  yield Protocol::ZMTP::Codec::Frame.new(item[1].to_body, command: true)
155
157
  end
158
+
156
159
  next
157
160
  end
158
161
 
@@ -179,8 +182,7 @@ module OMQ
179
182
  # @return [Protocol::ZMTP::Codec::Frame]
180
183
  #
181
184
  def read_frame
182
- item = @receive_queue.dequeue
183
- raise EOFError, "connection closed" if item.nil?
185
+ item = @receive_queue.dequeue or raise EOFError, "connection closed"
184
186
 
185
187
  if item.is_a?(Array) && item.first == :command
186
188
  Protocol::ZMTP::Codec::Frame.new(item[1].to_body, command: true)
@@ -206,7 +208,7 @@ module OMQ
206
208
 
207
209
  def apply_transform(parts)
208
210
  if @direct_recv_transform
209
- @direct_recv_transform.call(parts).freeze
211
+ @direct_recv_transform.call(parts)
210
212
  else
211
213
  parts
212
214
  end
@@ -2,7 +2,7 @@
2
2
 
3
3
  require "async"
4
4
  require "async/queue"
5
- require_relative "inproc/direct_pipe"
5
+ require_relative "inproc/pipe"
6
6
 
7
7
  module OMQ
8
8
  module Transport
@@ -15,6 +15,9 @@ module OMQ
15
15
  # prevent shared mutable state without copying.
16
16
  #
17
17
  module Inproc
18
+ Engine.transports["inproc"] = self
19
+
20
+
18
21
  # Socket types that exchange commands (SUBSCRIBE/CANCEL) over inproc.
19
22
  #
20
23
  COMMAND_TYPES = %i[PUB SUB XPUB XSUB RADIO DISH].freeze
@@ -28,14 +31,19 @@ module OMQ
28
31
 
29
32
 
30
33
  class << self
31
- # Binds an engine to an inproc endpoint.
34
+ # @return [Hash{String => Engine}] bound inproc endpoints
35
+ #
36
+ attr_reader :registry
37
+
38
+
39
+ # Creates a bound inproc listener.
32
40
  #
33
41
  # @param endpoint [String] e.g. "inproc://my-endpoint"
34
42
  # @param engine [Engine] the owning engine
35
43
  # @return [Listener]
36
44
  # @raise [ArgumentError] if endpoint is already bound
37
45
  #
38
- def bind(endpoint, engine)
46
+ def listener(endpoint, engine, **)
39
47
  @mutex.synchronize do
40
48
  if @registry.key?(endpoint)
41
49
  raise ArgumentError, "endpoint already bound: #{endpoint}"
@@ -57,7 +65,7 @@ module OMQ
57
65
  # @param engine [Engine] the connecting engine
58
66
  # @return [void]
59
67
  #
60
- def connect(endpoint, engine)
68
+ def connect(endpoint, engine, **)
61
69
  bound_engine = @mutex.synchronize { @registry[endpoint] }
62
70
  bound_engine ||= await_bind(endpoint, engine) or return
63
71
  establish_link(engine, bound_engine, endpoint)
@@ -114,8 +122,8 @@ module OMQ
114
122
  end
115
123
 
116
124
 
117
- # Decides whether a DirectPipe pair needs command queues.
118
- # DirectPipe's fast path skips queues entirely; command queues
125
+ # Decides whether a Pipe pair needs command queues.
126
+ # Pipe's fast path skips queues entirely; command queues
119
127
  # are only needed for socket types that exchange ZMTP commands
120
128
  # (e.g. ROUTER/DEALER identity, PUB/SUB subscriptions) or when
121
129
  # either side enables QoS ≥ 1.
@@ -129,11 +137,11 @@ module OMQ
129
137
  end
130
138
 
131
139
 
132
- # Builds a bidirectional {DirectPipe} pair for client + server.
140
+ # Builds a bidirectional {Pipe} pair for client + server.
133
141
  # When +needs_cmds+ is false the pipes have no command queues
134
142
  # (fast path — all traffic bypasses Async::Queue entirely).
135
143
  #
136
- # @return [Array(DirectPipe, DirectPipe)] client, server
144
+ # @return [Array(Pipe, Pipe)] client, server
137
145
  #
138
146
  def make_pipe_pair(ce, se, ct, st, needs_cmds)
139
147
  if needs_cmds
@@ -141,12 +149,12 @@ module OMQ
141
149
  b_to_a = Async::Queue.new
142
150
  end
143
151
 
144
- client = DirectPipe.new(send_queue: needs_cmds ? a_to_b : nil,
145
- receive_queue: needs_cmds ? b_to_a : nil,
146
- peer_identity: se.options.identity, peer_type: st.to_s)
147
- server = DirectPipe.new(send_queue: needs_cmds ? b_to_a : nil,
148
- receive_queue: needs_cmds ? a_to_b : nil,
149
- peer_identity: ce.options.identity, peer_type: ct.to_s)
152
+ client = Pipe.new(send_queue: needs_cmds ? a_to_b : nil,
153
+ receive_queue: needs_cmds ? b_to_a : nil,
154
+ peer_identity: se.options.identity, peer_type: st.to_s)
155
+ server = Pipe.new(send_queue: needs_cmds ? b_to_a : nil,
156
+ receive_queue: needs_cmds ? a_to_b : nil,
157
+ peer_identity: ce.options.identity, peer_type: ct.to_s)
150
158
 
151
159
  client.peer = server
152
160
  server.peer = client
@@ -11,14 +11,17 @@ module OMQ
11
11
  # (paths starting with @).
12
12
  #
13
13
  module IPC
14
+ Engine.transports["ipc"] = self
15
+
16
+
14
17
  class << self
15
- # Binds an IPC server.
18
+ # Creates a bound IPC listener.
16
19
  #
17
20
  # @param endpoint [String] e.g. "ipc:///tmp/my.sock" or "ipc://@abstract"
18
21
  # @param engine [Engine]
19
22
  # @return [Listener]
20
23
  #
21
- def bind(endpoint, engine)
24
+ def listener(endpoint, engine, **)
22
25
  path = parse_path(endpoint)
23
26
  sock_path = to_socket_path(path)
24
27
 
@@ -31,18 +34,14 @@ module OMQ
31
34
  end
32
35
 
33
36
 
34
- # Connects to an IPC endpoint.
37
+ # Creates an IPC dialer for an endpoint.
35
38
  #
36
39
  # @param endpoint [String]
37
40
  # @param engine [Engine]
38
- # @return [void]
41
+ # @return [Dialer]
39
42
  #
40
- def connect(endpoint, engine)
41
- path = parse_path(endpoint)
42
- sock_path = to_socket_path(path)
43
- sock = UNIXSocket.new(sock_path)
44
- apply_buffer_sizes(sock, engine.options)
45
- engine.handle_connected(IO::Stream::Buffered.wrap(sock), endpoint: endpoint)
43
+ def dialer(endpoint, engine, **)
44
+ Dialer.new(endpoint, engine)
46
45
  end
47
46
 
48
47
 
@@ -63,9 +62,6 @@ module OMQ
63
62
  end
64
63
 
65
64
 
66
- private
67
-
68
-
69
65
  # Extracts path from "ipc://path".
70
66
  #
71
67
  def parse_path(endpoint)
@@ -92,6 +88,38 @@ module OMQ
92
88
  end
93
89
 
94
90
 
91
+ # An IPC dialer — stateful factory for outgoing connections.
92
+ #
93
+ class Dialer
94
+ # @return [String] the endpoint this dialer connects to
95
+ #
96
+ attr_reader :endpoint
97
+
98
+
99
+ # @param endpoint [String]
100
+ # @param engine [Engine]
101
+ #
102
+ def initialize(endpoint, engine)
103
+ @endpoint = endpoint
104
+ @engine = engine
105
+ end
106
+
107
+
108
+ # Establishes a Unix socket connection to the endpoint.
109
+ #
110
+ # @return [void]
111
+ #
112
+ def connect
113
+ path = IPC.parse_path(@endpoint)
114
+ sock_path = IPC.to_socket_path(path)
115
+ sock = UNIXSocket.new(sock_path)
116
+ IPC.apply_buffer_sizes(sock, @engine.options)
117
+ @engine.handle_connected(IO::Stream::Buffered.wrap(sock), endpoint: @endpoint)
118
+ end
119
+
120
+ end
121
+
122
+
95
123
  # A bound IPC listener.
96
124
  #
97
125
  class Listener
@@ -9,14 +9,17 @@ module OMQ
9
9
  # TCP transport using Ruby sockets with Async.
10
10
  #
11
11
  module TCP
12
+ Engine.transports["tcp"] = self
13
+
14
+
12
15
  class << self
13
- # Binds a TCP server.
16
+ # Creates a bound TCP listener.
14
17
  #
15
18
  # @param endpoint [String] e.g. "tcp://127.0.0.1:5555" or "tcp://*:0"
16
19
  # @param engine [Engine]
17
20
  # @return [Listener]
18
21
  #
19
- def bind(endpoint, engine)
22
+ def listener(endpoint, engine, **)
20
23
  host, port = self.parse_endpoint(endpoint)
21
24
  lookup_host = normalize_bind_host(host)
22
25
 
@@ -34,6 +37,17 @@ module OMQ
34
37
  end
35
38
 
36
39
 
40
+ # Creates a TCP dialer for an endpoint.
41
+ #
42
+ # @param endpoint [String] e.g. "tcp://127.0.0.1:5555"
43
+ # @param engine [Engine]
44
+ # @return [Dialer]
45
+ #
46
+ def dialer(endpoint, engine, **)
47
+ Dialer.new(endpoint, engine)
48
+ end
49
+
50
+
37
51
  # Validates that the endpoint's host can be resolved.
38
52
  #
39
53
  # @param endpoint [String]
@@ -46,21 +60,6 @@ module OMQ
46
60
  end
47
61
 
48
62
 
49
- # Connects to a TCP endpoint.
50
- #
51
- # @param endpoint [String] e.g. "tcp://127.0.0.1:5555"
52
- # @param engine [Engine]
53
- # @return [void]
54
- #
55
- def connect(endpoint, engine)
56
- host, port = self.parse_endpoint(endpoint)
57
- host = normalize_connect_host(host)
58
- sock = ::Socket.tcp(host, port, connect_timeout: connect_timeout(engine.options))
59
- apply_buffer_sizes(sock, engine.options)
60
- engine.handle_connected(IO::Stream::Buffered.wrap(sock), endpoint: endpoint)
61
- end
62
-
63
-
64
63
  # Normalizes the bind host:
65
64
  # "*" → nil (dual-stack wildcard via AI_PASSIVE)
66
65
  # "" / nil / "localhost" → loopback_host (::1 on IPv6-capable hosts, else 127.0.0.1)
@@ -142,6 +141,42 @@ module OMQ
142
141
  end
143
142
 
144
143
 
144
+ # A TCP dialer — stateful factory for outgoing connections.
145
+ #
146
+ # Created once per {Engine#connect}, stored in +@dialers[endpoint]+.
147
+ # Reconnect calls {#connect} directly — no transport lookup or opts
148
+ # replay needed.
149
+ #
150
+ class Dialer
151
+ # @return [String] the endpoint this dialer connects to
152
+ #
153
+ attr_reader :endpoint
154
+
155
+
156
+ # @param endpoint [String] e.g. "tcp://127.0.0.1:5555"
157
+ # @param engine [Engine]
158
+ #
159
+ def initialize(endpoint, engine)
160
+ @endpoint = endpoint
161
+ @engine = engine
162
+ end
163
+
164
+
165
+ # Establishes a TCP connection to the endpoint.
166
+ #
167
+ # @return [void]
168
+ #
169
+ def connect
170
+ host, port = TCP.parse_endpoint(@endpoint)
171
+ host = TCP.normalize_connect_host(host)
172
+ sock = ::Socket.tcp(host, port, connect_timeout: TCP.connect_timeout(@engine.options))
173
+ TCP.apply_buffer_sizes(sock, @engine.options)
174
+ @engine.handle_connected(IO::Stream::Buffered.wrap(sock), endpoint: @endpoint)
175
+ end
176
+
177
+ end
178
+
179
+
145
180
  # A bound TCP listener.
146
181
  #
147
182
  class Listener
@@ -168,7 +203,7 @@ module OMQ
168
203
  @servers = servers
169
204
  @port = port
170
205
  @engine = engine
171
- @tasks = []
206
+ @barrier = nil
172
207
  end
173
208
 
174
209
 
@@ -179,10 +214,11 @@ module OMQ
179
214
  # @yieldparam io [IO::Stream::Buffered]
180
215
  #
181
216
  def start_accept_loops(parent_task)
182
- @tasks = @servers.map do |server|
183
- # TODO: use this server's exact host:port (@endpoint might not be unique)
184
- annotation = "tcp accept #{@endpoint}"
185
- parent_task.async(transient: true, annotation:) do
217
+ @barrier = Async::Barrier.new(parent: parent_task)
218
+
219
+ @servers.each do |server|
220
+ annotation = "tcp accept #{server.local_address.inspect_sockaddr}"
221
+ @barrier.async(transient: true, annotation:) do
186
222
  loop do
187
223
  client, _addr = server.accept
188
224
  TCP.apply_buffer_sizes(client, @engine.options)
@@ -207,7 +243,7 @@ module OMQ
207
243
  # @return [void]
208
244
  #
209
245
  def stop
210
- @tasks.each(&:stop)
246
+ @barrier&.stop
211
247
  @servers.each { |s| s.close rescue nil }
212
248
  end
213
249