omq-cli 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -0
- data/lib/omq/cli/base_runner.rb +1 -12
- data/lib/omq/cli/cli_parser.rb +6 -12
- data/lib/omq/cli/pipe.rb +111 -118
- data/lib/omq/cli/pub_sub.rb +1 -3
- data/lib/omq/cli/push_pull.rb +1 -3
- data/lib/omq/cli/radio_dish.rb +1 -3
- data/lib/omq/cli/scatter_gather.rb +1 -3
- data/lib/omq/cli/version.rb +1 -1
- data/lib/omq/cli.rb +0 -3
- metadata +6 -15
- data/lib/omq/cli/parallel_recv_runner.rb +0 -150
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a03141c60a8af670aa4de42bc5b5d286fbf3a975c5a455e9276bc2d4b87a0fb8
|
|
4
|
+
data.tar.gz: 2755cb1054df95a64b5faebe481e93ac2e3f391e878f54b5a30a17514dde399a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f245e7f524d745382074347852f1d34e5e1f46e633fe024b7888c05942a399639e69e876ae92c5ace5172800b0dc6a3bd8cd088d3b52f389ea0fb212e0f8da58
|
|
7
|
+
data.tar.gz: 34992fe5962fd2639b0c3e92597e3d2d6a134f3ac4fecadfa3d64b1d566994cf9f8ef36ce08fc929ab97bfcf7a7b4dd7f9c1de241b8c87e2a207ce17a838c7d2
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.1 — 2026-04-07
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- **Pipe `-P` preresolves TCP hostnames** — DNS resolution happens on the
|
|
8
|
+
main thread before spawning Ractors, avoiding `Ractor::IsolationError`
|
|
9
|
+
on `Resolv::DefaultResolver`. All resolved addresses (IPv4 + IPv6) are
|
|
10
|
+
passed to workers.
|
|
11
|
+
|
|
12
|
+
## 0.7.0 — 2026-04-07
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- **`-P` restricted to pipe only** — parallel Ractor workers are no longer
|
|
17
|
+
available on recv-only socket types (pull, sub, gather, dish). Pipe workers
|
|
18
|
+
now use bare Ractors with their own Async reactors and OMQ sockets, removing
|
|
19
|
+
the `omq-ractor` dependency entirely.
|
|
20
|
+
- **`-P` range capped to 1..16** — default is still `nproc`, clamped to 16.
|
|
21
|
+
`-P 1` is valid (single Ractor worker, no sockets on main thread).
|
|
22
|
+
- **Removed `omq-ractor` dependency** — no longer needed.
|
|
23
|
+
- **Removed `ParallelRecvRunner`** — the Ractor bridge infrastructure for
|
|
24
|
+
non-pipe socket types has been deleted.
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- **Pipe `-P` END blocks execute after timeout** — worker recv loops now
|
|
29
|
+
catch `IO::TimeoutError` so BEGIN/END expressions run to completion.
|
|
30
|
+
|
|
3
31
|
## 0.6.0 — 2026-04-07
|
|
4
32
|
|
|
5
33
|
### Added
|
data/lib/omq/cli/base_runner.rb
CHANGED
|
@@ -51,7 +51,7 @@ module OMQ
|
|
|
51
51
|
|
|
52
52
|
def setup_socket
|
|
53
53
|
@sock = create_socket
|
|
54
|
-
attach_endpoints
|
|
54
|
+
attach_endpoints
|
|
55
55
|
setup_curve
|
|
56
56
|
setup_subscriptions
|
|
57
57
|
compile_expr
|
|
@@ -265,17 +265,6 @@ module OMQ
|
|
|
265
265
|
end
|
|
266
266
|
|
|
267
267
|
|
|
268
|
-
# Parallel recv-eval: delegates to ParallelRecvRunner.
|
|
269
|
-
#
|
|
270
|
-
def run_parallel_recv(task)
|
|
271
|
-
# @sock was created by call() before run_loop; close it now so it doesn't
|
|
272
|
-
# steal messages from the N worker sockets ParallelRecvRunner creates.
|
|
273
|
-
@sock&.close
|
|
274
|
-
@sock = nil
|
|
275
|
-
ParallelRecvRunner.new(@klass, config, @fmt, method(:output)).run(task)
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
|
|
279
268
|
def wait_for_loops(receiver, sender)
|
|
280
269
|
if config.data || config.file || config.send_expr || config.recv_expr || config.target
|
|
281
270
|
sender.wait
|
data/lib/omq/cli/cli_parser.rb
CHANGED
|
@@ -367,9 +367,9 @@ module OMQ
|
|
|
367
367
|
require "omq" unless defined?(OMQ::VERSION)
|
|
368
368
|
opts[:scripts] << (v == "-" ? :stdin : (v.start_with?("./", "../") ? File.expand_path(v) : v))
|
|
369
369
|
}
|
|
370
|
-
o.on("-P", "--parallel [N]", Integer, "Parallel Ractor workers (default: nproc
|
|
370
|
+
o.on("-P", "--parallel [N]", Integer, "Parallel Ractor workers for pipe (default: nproc, max 16)") { |v|
|
|
371
371
|
require "etc"
|
|
372
|
-
opts[:parallel] = v || Etc.nprocessors
|
|
372
|
+
opts[:parallel] = [v || Etc.nprocessors, 16].min
|
|
373
373
|
}
|
|
374
374
|
|
|
375
375
|
o.separator "\nCURVE encryption (requires system libsodium):"
|
|
@@ -486,16 +486,10 @@ module OMQ
|
|
|
486
486
|
abort "--send-eval and --target are mutually exclusive" if opts[:send_expr] && opts[:target]
|
|
487
487
|
|
|
488
488
|
if opts[:parallel]
|
|
489
|
-
abort "-P/--parallel
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
elsif RECV_ONLY.include?(type_name)
|
|
494
|
-
abort "-P/--parallel on #{type_name} requires --recv-eval (-e)" unless opts[:recv_expr]
|
|
495
|
-
abort "-P/--parallel requires all endpoints to use --connect (not --bind)" if opts[:binds].any?
|
|
496
|
-
else
|
|
497
|
-
abort "-P/--parallel is only valid for pipe or recv-only socket types (#{RECV_ONLY.join(', ')})"
|
|
498
|
-
end
|
|
489
|
+
abort "-P/--parallel is only valid for pipe" unless type_name == "pipe"
|
|
490
|
+
abort "-P/--parallel must be 1..16" unless (1..16).include?(opts[:parallel])
|
|
491
|
+
all_pipe_eps = opts[:in_endpoints] + opts[:out_endpoints] + opts[:endpoints]
|
|
492
|
+
abort "-P/--parallel requires all endpoints to use --connect (not --bind)" if all_pipe_eps.any?(&:bind?)
|
|
499
493
|
end
|
|
500
494
|
|
|
501
495
|
(opts[:connects] + opts[:binds]).each do |url|
|
data/lib/omq/cli/pipe.rb
CHANGED
|
@@ -130,145 +130,138 @@ module OMQ
|
|
|
130
130
|
|
|
131
131
|
|
|
132
132
|
def run_parallel(task)
|
|
133
|
+
OMQ.freeze_for_ractors!
|
|
133
134
|
in_eps, out_eps = resolve_endpoints
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
workers
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
pull&.close
|
|
142
|
-
push&.close
|
|
135
|
+
in_eps = preresolve_tcp(in_eps)
|
|
136
|
+
out_eps = preresolve_tcp(out_eps)
|
|
137
|
+
workers = spawn_workers(config, in_eps, out_eps)
|
|
138
|
+
workers.each do |w|
|
|
139
|
+
w.join
|
|
140
|
+
rescue ::Ractor::RemoteError => e
|
|
141
|
+
$stderr.write("omq: Ractor error: #{e.cause&.message || e.message}\n")
|
|
143
142
|
end
|
|
144
143
|
end
|
|
145
144
|
|
|
146
145
|
|
|
147
|
-
def
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
146
|
+
def spawn_workers(config, in_eps, out_eps)
|
|
147
|
+
config.parallel.times.map do
|
|
148
|
+
::Ractor.new(config, in_eps, out_eps) do |cfg, ins, outs|
|
|
149
|
+
Async do
|
|
150
|
+
pull = OMQ::PULL.new(linger: cfg.linger)
|
|
151
|
+
push = OMQ::PUSH.new(linger: cfg.linger)
|
|
152
|
+
pull.recv_timeout = cfg.timeout if cfg.timeout
|
|
153
|
+
push.send_timeout = cfg.timeout if cfg.timeout
|
|
154
|
+
pull.reconnect_interval = cfg.reconnect_ivl if cfg.reconnect_ivl
|
|
155
|
+
push.reconnect_interval = cfg.reconnect_ivl if cfg.reconnect_ivl
|
|
156
|
+
pull.heartbeat_interval = cfg.heartbeat_ivl if cfg.heartbeat_ivl
|
|
157
|
+
push.heartbeat_interval = cfg.heartbeat_ivl if cfg.heartbeat_ivl
|
|
158
|
+
pull.send_hwm = cfg.send_hwm if cfg.send_hwm
|
|
159
|
+
pull.recv_hwm = cfg.recv_hwm if cfg.recv_hwm
|
|
160
|
+
push.send_hwm = cfg.send_hwm if cfg.send_hwm
|
|
161
|
+
push.recv_hwm = cfg.recv_hwm if cfg.recv_hwm
|
|
162
|
+
pull.sndbuf = cfg.sndbuf if cfg.sndbuf
|
|
163
|
+
pull.rcvbuf = cfg.rcvbuf if cfg.rcvbuf
|
|
164
|
+
push.sndbuf = cfg.sndbuf if cfg.sndbuf
|
|
165
|
+
push.rcvbuf = cfg.rcvbuf if cfg.rcvbuf
|
|
166
|
+
|
|
167
|
+
OMQ::CLI::SocketSetup.attach_endpoints(pull, ins, verbose: cfg.verbose >= 1)
|
|
168
|
+
OMQ::CLI::SocketSetup.attach_endpoints(push, outs, verbose: cfg.verbose >= 1)
|
|
169
|
+
|
|
170
|
+
Barrier do |barrier|
|
|
171
|
+
barrier.async { pull.peer_connected.wait }
|
|
172
|
+
barrier.async { push.peer_connected.wait }
|
|
173
|
+
end
|
|
154
174
|
|
|
175
|
+
begin_proc, end_proc, eval_proc =
|
|
176
|
+
OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(cfg.recv_expr)
|
|
177
|
+
|
|
178
|
+
fmt_in = OMQ::CLI::Formatter.new(cfg.format, compress: cfg.compress_in || cfg.compress)
|
|
179
|
+
fmt_out = OMQ::CLI::Formatter.new(cfg.format, compress: cfg.compress_out || cfg.compress)
|
|
180
|
+
|
|
181
|
+
_ctx = Object.new
|
|
182
|
+
_ctx.instance_exec(&begin_proc) if begin_proc
|
|
183
|
+
|
|
184
|
+
n_count = cfg.count
|
|
185
|
+
begin
|
|
186
|
+
if eval_proc
|
|
187
|
+
if n_count && n_count > 0
|
|
188
|
+
n_count.times do
|
|
189
|
+
parts = pull.receive
|
|
190
|
+
break if parts.nil?
|
|
191
|
+
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
192
|
+
_ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
|
|
193
|
+
)
|
|
194
|
+
next if parts.nil?
|
|
195
|
+
push << fmt_out.compress(parts) unless parts.empty?
|
|
196
|
+
end
|
|
197
|
+
else
|
|
198
|
+
loop do
|
|
199
|
+
parts = pull.receive
|
|
200
|
+
break if parts.nil?
|
|
201
|
+
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
202
|
+
_ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
|
|
203
|
+
)
|
|
204
|
+
next if parts.nil?
|
|
205
|
+
push << fmt_out.compress(parts) unless parts.empty?
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
else
|
|
209
|
+
if n_count && n_count > 0
|
|
210
|
+
n_count.times do
|
|
211
|
+
parts = pull.receive
|
|
212
|
+
break if parts.nil?
|
|
213
|
+
push << fmt_out.compress(fmt_in.decompress(parts))
|
|
214
|
+
end
|
|
215
|
+
else
|
|
216
|
+
loop do
|
|
217
|
+
parts = pull.receive
|
|
218
|
+
break if parts.nil?
|
|
219
|
+
push << fmt_out.compress(fmt_in.decompress(parts))
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
rescue IO::TimeoutError, Async::TimeoutError
|
|
224
|
+
# recv timed out — fall through to END block
|
|
225
|
+
end
|
|
155
226
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
227
|
+
if end_proc
|
|
228
|
+
out = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
229
|
+
_ctx.instance_exec(&end_proc)
|
|
230
|
+
)
|
|
231
|
+
push << fmt_out.compress(out) if out && !out.empty?
|
|
232
|
+
end
|
|
233
|
+
ensure
|
|
234
|
+
pull&.close
|
|
235
|
+
push&.close
|
|
162
236
|
end
|
|
163
237
|
end
|
|
164
238
|
end
|
|
165
239
|
end
|
|
166
240
|
|
|
167
241
|
|
|
168
|
-
|
|
169
|
-
return unless config.transient
|
|
170
|
-
task.async do
|
|
171
|
-
pairs[0][0].all_peers_gone.wait
|
|
172
|
-
pairs.each do |pull, _|
|
|
173
|
-
pull.reconnect_enabled = false
|
|
174
|
-
pull.close_read
|
|
175
|
-
end
|
|
176
|
-
end
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
def build_worker_data
|
|
181
|
-
# Pack worker config into a shareable Hash passed via omq.data —
|
|
182
|
-
# Ruby 4.0 forbids Ractor blocks from closing over outer locals.
|
|
183
|
-
::Ractor.make_shareable({
|
|
184
|
-
recv_src: config.recv_expr,
|
|
185
|
-
fmt_format: config.format,
|
|
186
|
-
compr_in: config.compress_in || config.compress,
|
|
187
|
-
compr_out: config.compress_out || config.compress,
|
|
188
|
-
n_count: config.count,
|
|
189
|
-
})
|
|
190
|
-
end
|
|
191
|
-
|
|
242
|
+
# ── Shared helpers ────────────────────────────────────────────────
|
|
192
243
|
|
|
193
|
-
def spawn_workers(pairs, worker_data)
|
|
194
|
-
pairs.map do |pull, push|
|
|
195
|
-
OMQ::Ractor.new(pull, push, serialize: false, data: worker_data) do |omq|
|
|
196
|
-
pull_p, push_p = omq.sockets
|
|
197
|
-
d = omq.data
|
|
198
|
-
|
|
199
|
-
# Re-compile expression inside Ractor (Procs are not shareable)
|
|
200
|
-
begin_proc, end_proc, eval_proc =
|
|
201
|
-
OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(d[:recv_src])
|
|
202
|
-
|
|
203
|
-
fmt_in = OMQ::CLI::Formatter.new(d[:fmt_format], compress: d[:compr_in])
|
|
204
|
-
fmt_out = OMQ::CLI::Formatter.new(d[:fmt_format], compress: d[:compr_out])
|
|
205
|
-
# Use a dedicated context object so @ivar expressions in BEGIN/END/eval
|
|
206
|
-
# work inside Ractors (self in a Ractor is shareable; Object.new is not).
|
|
207
|
-
_ctx = Object.new
|
|
208
|
-
_ctx.instance_exec(&begin_proc) if begin_proc
|
|
209
|
-
|
|
210
|
-
n_count = d[:n_count]
|
|
211
|
-
if eval_proc
|
|
212
|
-
if n_count && n_count > 0
|
|
213
|
-
n_count.times do
|
|
214
|
-
parts = pull_p.receive
|
|
215
|
-
break if parts.nil?
|
|
216
|
-
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
217
|
-
_ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
|
|
218
|
-
)
|
|
219
|
-
next if parts.nil?
|
|
220
|
-
push_p << fmt_out.compress(parts) unless parts.empty?
|
|
221
|
-
end
|
|
222
|
-
else
|
|
223
|
-
loop do
|
|
224
|
-
parts = pull_p.receive
|
|
225
|
-
break if parts.nil?
|
|
226
|
-
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
227
|
-
_ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
|
|
228
|
-
)
|
|
229
|
-
next if parts.nil?
|
|
230
|
-
push_p << fmt_out.compress(parts) unless parts.empty?
|
|
231
|
-
end
|
|
232
|
-
end
|
|
233
|
-
else
|
|
234
|
-
if n_count && n_count > 0
|
|
235
|
-
n_count.times do
|
|
236
|
-
parts = pull_p.receive
|
|
237
|
-
break if parts.nil?
|
|
238
|
-
push_p << fmt_out.compress(fmt_in.decompress(parts))
|
|
239
|
-
end
|
|
240
|
-
else
|
|
241
|
-
loop do
|
|
242
|
-
parts = pull_p.receive
|
|
243
|
-
break if parts.nil?
|
|
244
|
-
push_p << fmt_out.compress(fmt_in.decompress(parts))
|
|
245
|
-
end
|
|
246
|
-
end
|
|
247
|
-
end
|
|
248
244
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
245
|
+
# Resolves TCP hostnames to IP addresses so Ractors don't touch
|
|
246
|
+
# Resolv::DefaultResolver (which is not shareable).
|
|
247
|
+
#
|
|
248
|
+
def preresolve_tcp(endpoints)
|
|
249
|
+
endpoints.flat_map do |ep|
|
|
250
|
+
url = ep.url
|
|
251
|
+
if url.start_with?("tcp://")
|
|
252
|
+
host, port = OMQ::Transport::TCP.parse_endpoint(url)
|
|
253
|
+
Addrinfo.getaddrinfo(host, port, nil, :STREAM).map do |addr|
|
|
254
|
+
ip = addr.ip_address
|
|
255
|
+
ip = "[#{ip}]" if ip.include?(":")
|
|
256
|
+
Endpoint.new("tcp://#{ip}:#{addr.ip_port}", ep.bind?)
|
|
254
257
|
end
|
|
258
|
+
else
|
|
259
|
+
ep
|
|
255
260
|
end
|
|
256
261
|
end
|
|
257
262
|
end
|
|
258
263
|
|
|
259
264
|
|
|
260
|
-
def join_workers(workers)
|
|
261
|
-
workers.each do |w|
|
|
262
|
-
w.value
|
|
263
|
-
rescue Ractor::RemoteError => e
|
|
264
|
-
$stderr.write("omq: Ractor error: #{e.cause&.message || e.message}\n")
|
|
265
|
-
end
|
|
266
|
-
end
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
# ── Shared helpers ────────────────────────────────────────────────
|
|
270
|
-
|
|
271
|
-
|
|
272
265
|
def with_timeout(seconds)
|
|
273
266
|
if seconds
|
|
274
267
|
Async::Task.current.with_timeout(seconds) { yield }
|
data/lib/omq/cli/pub_sub.rb
CHANGED
|
@@ -10,9 +10,7 @@ module OMQ
|
|
|
10
10
|
|
|
11
11
|
# Runner for SUB sockets (subscribe and receive published messages).
|
|
12
12
|
class SubRunner < BaseRunner
|
|
13
|
-
def run_loop(task)
|
|
14
|
-
config.parallel ? run_parallel_recv(task) : run_recv_logic
|
|
15
|
-
end
|
|
13
|
+
def run_loop(task) = run_recv_logic
|
|
16
14
|
end
|
|
17
15
|
end
|
|
18
16
|
end
|
data/lib/omq/cli/push_pull.rb
CHANGED
|
@@ -10,9 +10,7 @@ module OMQ
|
|
|
10
10
|
|
|
11
11
|
# Runner for PULL sockets (receive-only pipeline consumer).
|
|
12
12
|
class PullRunner < BaseRunner
|
|
13
|
-
def run_loop(task)
|
|
14
|
-
config.parallel ? run_parallel_recv(task) : run_recv_logic
|
|
15
|
-
end
|
|
13
|
+
def run_loop(task) = run_recv_logic
|
|
16
14
|
end
|
|
17
15
|
end
|
|
18
16
|
end
|
data/lib/omq/cli/radio_dish.rb
CHANGED
|
@@ -23,9 +23,7 @@ module OMQ
|
|
|
23
23
|
|
|
24
24
|
# Runner for DISH sockets (draft; group-based subscribe).
|
|
25
25
|
class DishRunner < BaseRunner
|
|
26
|
-
def run_loop(task)
|
|
27
|
-
config.parallel ? run_parallel_recv(task) : run_recv_logic
|
|
28
|
-
end
|
|
26
|
+
def run_loop(task) = run_recv_logic
|
|
29
27
|
end
|
|
30
28
|
end
|
|
31
29
|
end
|
data/lib/omq/cli/version.rb
CHANGED
data/lib/omq/cli.rb
CHANGED
|
@@ -9,7 +9,6 @@ require_relative "cli/expression_evaluator"
|
|
|
9
9
|
require_relative "cli/socket_setup"
|
|
10
10
|
require_relative "cli/routing_helper"
|
|
11
11
|
require_relative "cli/transient_monitor"
|
|
12
|
-
require_relative "cli/parallel_recv_runner"
|
|
13
12
|
require_relative "cli/base_runner"
|
|
14
13
|
require_relative "cli/push_pull"
|
|
15
14
|
require_relative "cli/pub_sub"
|
|
@@ -199,8 +198,6 @@ module OMQ
|
|
|
199
198
|
require "console"
|
|
200
199
|
|
|
201
200
|
CliParser.validate_gems!(config)
|
|
202
|
-
require "omq/ractor" if config.parallel
|
|
203
|
-
|
|
204
201
|
trap("INT") { Process.exit!(0) }
|
|
205
202
|
trap("TERM") { Process.exit!(0) }
|
|
206
203
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: omq-cli
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Patrik Wenger
|
|
@@ -16,6 +16,9 @@ dependencies:
|
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
18
|
version: '0.15'
|
|
19
|
+
- - ">="
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: 0.15.2
|
|
19
22
|
type: :runtime
|
|
20
23
|
prerelease: false
|
|
21
24
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -23,20 +26,9 @@ dependencies:
|
|
|
23
26
|
- - "~>"
|
|
24
27
|
- !ruby/object:Gem::Version
|
|
25
28
|
version: '0.15'
|
|
26
|
-
-
|
|
27
|
-
name: omq-ractor
|
|
28
|
-
requirement: !ruby/object:Gem::Requirement
|
|
29
|
-
requirements:
|
|
30
|
-
- - "~>"
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: '0.1'
|
|
33
|
-
type: :runtime
|
|
34
|
-
prerelease: false
|
|
35
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
-
requirements:
|
|
37
|
-
- - "~>"
|
|
29
|
+
- - ">="
|
|
38
30
|
- !ruby/object:Gem::Version
|
|
39
|
-
version:
|
|
31
|
+
version: 0.15.2
|
|
40
32
|
- !ruby/object:Gem::Dependency
|
|
41
33
|
name: omq-rfc-clientserver
|
|
42
34
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -173,7 +165,6 @@ files:
|
|
|
173
165
|
- lib/omq/cli/expression_evaluator.rb
|
|
174
166
|
- lib/omq/cli/formatter.rb
|
|
175
167
|
- lib/omq/cli/pair.rb
|
|
176
|
-
- lib/omq/cli/parallel_recv_runner.rb
|
|
177
168
|
- lib/omq/cli/pipe.rb
|
|
178
169
|
- lib/omq/cli/pub_sub.rb
|
|
179
170
|
- lib/omq/cli/push_pull.rb
|
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module OMQ
|
|
4
|
-
module CLI
|
|
5
|
-
# Manages N OMQ::Ractor workers for parallel recv-eval.
|
|
6
|
-
#
|
|
7
|
-
# Each worker gets its own input socket connecting to the external
|
|
8
|
-
# endpoints; ZMQ distributes work naturally. Results are collected via
|
|
9
|
-
# an inproc PULL back to the main task for output.
|
|
10
|
-
#
|
|
11
|
-
class ParallelRecvRunner
|
|
12
|
-
# @param klass [Class] OMQ socket class for worker input sockets
|
|
13
|
-
# @param config [Config] frozen CLI configuration
|
|
14
|
-
# @param fmt [Formatter] message formatter
|
|
15
|
-
# @param output_fn [Method] callable for writing output
|
|
16
|
-
def initialize(klass, config, fmt, output_fn)
|
|
17
|
-
@klass = klass
|
|
18
|
-
@config = config
|
|
19
|
-
@fmt = fmt
|
|
20
|
-
@output_fn = output_fn
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
# Spawns N Ractor workers, distributes incoming messages, and collects results.
|
|
25
|
-
#
|
|
26
|
-
# @param task [Async::Task] parent async task
|
|
27
|
-
# @return [void]
|
|
28
|
-
def run(task)
|
|
29
|
-
cfg = @config
|
|
30
|
-
n_workers = cfg.parallel
|
|
31
|
-
inproc = "inproc://omq-out-#{object_id}"
|
|
32
|
-
|
|
33
|
-
# Pack worker config into a shareable Hash passed via omq.data —
|
|
34
|
-
# Ruby 4.0 forbids Ractor blocks from closing over outer locals.
|
|
35
|
-
worker_data = ::Ractor.make_shareable({
|
|
36
|
-
recv_src: cfg.recv_expr,
|
|
37
|
-
fmt_sym: cfg.format,
|
|
38
|
-
fmt_compr: cfg.compress,
|
|
39
|
-
})
|
|
40
|
-
|
|
41
|
-
# Create N input sockets in the main Async context
|
|
42
|
-
input_socks = n_workers.times.map do
|
|
43
|
-
sock = SocketSetup.build(@klass, cfg)
|
|
44
|
-
SocketSetup.setup_subscriptions(sock, cfg)
|
|
45
|
-
cfg.connects.each { |url| sock.connect(url) }
|
|
46
|
-
sock
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
with_timeout(cfg.timeout) { input_socks.each { |s| s.peer_connected.wait } }
|
|
50
|
-
|
|
51
|
-
# Inproc collector: one bound PULL to receive all worker output
|
|
52
|
-
collector = OMQ::PULL.new(linger: cfg.linger)
|
|
53
|
-
collector.recv_timeout = cfg.timeout if cfg.timeout
|
|
54
|
-
collector.bind(inproc)
|
|
55
|
-
|
|
56
|
-
# N output sockets connecting to the collector
|
|
57
|
-
output_socks = n_workers.times.map do
|
|
58
|
-
s = OMQ::PUSH.new(linger: cfg.linger)
|
|
59
|
-
s.connect(inproc)
|
|
60
|
-
s
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
workers = n_workers.times.map do |i|
|
|
64
|
-
OMQ::Ractor.new(input_socks[i], output_socks[i], serialize: false, data: worker_data) do |omq|
|
|
65
|
-
pull_p, push_p = omq.sockets
|
|
66
|
-
d = omq.data
|
|
67
|
-
|
|
68
|
-
# Re-compile expression inside Ractor (Procs are not shareable)
|
|
69
|
-
begin_proc, end_proc, eval_proc =
|
|
70
|
-
OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(d[:recv_src])
|
|
71
|
-
|
|
72
|
-
formatter = OMQ::CLI::Formatter.new(d[:fmt_sym], compress: d[:fmt_compr])
|
|
73
|
-
# Use a dedicated context object so @ivar expressions in BEGIN/END/eval
|
|
74
|
-
# work inside Ractors (self in a Ractor is shareable; Object.new is not).
|
|
75
|
-
_ctx = Object.new
|
|
76
|
-
_ctx.instance_exec(&begin_proc) if begin_proc
|
|
77
|
-
|
|
78
|
-
if eval_proc
|
|
79
|
-
loop do
|
|
80
|
-
parts = pull_p.receive
|
|
81
|
-
break if parts.nil?
|
|
82
|
-
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
83
|
-
_ctx.instance_exec(formatter.decompress(parts), &eval_proc)
|
|
84
|
-
)
|
|
85
|
-
next if parts.nil?
|
|
86
|
-
push_p << parts unless parts.empty?
|
|
87
|
-
end
|
|
88
|
-
else
|
|
89
|
-
loop do
|
|
90
|
-
parts = pull_p.receive
|
|
91
|
-
break if parts.nil?
|
|
92
|
-
push_p << formatter.decompress(parts)
|
|
93
|
-
end
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
if end_proc
|
|
97
|
-
out = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
98
|
-
_ctx.instance_exec(&end_proc)
|
|
99
|
-
)
|
|
100
|
-
push_p << out if out && !out.empty?
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
# Collect loop: drain inproc PULL → output
|
|
107
|
-
n_count = cfg.count
|
|
108
|
-
if n_count && n_count > 0
|
|
109
|
-
n_count.times do
|
|
110
|
-
parts = collector.receive
|
|
111
|
-
break if parts.nil?
|
|
112
|
-
@output_fn.call(parts)
|
|
113
|
-
end
|
|
114
|
-
else
|
|
115
|
-
loop do
|
|
116
|
-
parts = collector.receive
|
|
117
|
-
break if parts.nil?
|
|
118
|
-
@output_fn.call(parts)
|
|
119
|
-
end
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
# Inject nil into each worker's input port so it exits its loop
|
|
124
|
-
# without waiting for recv_timeout (workers don't self-terminate
|
|
125
|
-
# when the collect loop exits on count).
|
|
126
|
-
workers.each do |w|
|
|
127
|
-
w.close
|
|
128
|
-
rescue Ractor::RemoteError => e
|
|
129
|
-
$stderr.puts "omq: Ractor error: #{e.cause&.message || e.message}"
|
|
130
|
-
end
|
|
131
|
-
ensure
|
|
132
|
-
input_socks&.each(&:close)
|
|
133
|
-
output_socks&.each(&:close)
|
|
134
|
-
collector&.close
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
private
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def with_timeout(seconds)
|
|
142
|
-
if seconds
|
|
143
|
-
Async::Task.current.with_timeout(seconds) { yield }
|
|
144
|
-
else
|
|
145
|
-
yield
|
|
146
|
-
end
|
|
147
|
-
end
|
|
148
|
-
end
|
|
149
|
-
end
|
|
150
|
-
end
|