omq-cli 0.5.4 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -0
- data/lib/omq/cli/base_runner.rb +1 -12
- data/lib/omq/cli/cli_parser.rb +19 -13
- data/lib/omq/cli/config.rb +2 -0
- data/lib/omq/cli/formatter.rb +2 -0
- data/lib/omq/cli/pipe.rb +96 -121
- data/lib/omq/cli/pub_sub.rb +1 -3
- data/lib/omq/cli/push_pull.rb +1 -3
- data/lib/omq/cli/radio_dish.rb +1 -3
- data/lib/omq/cli/scatter_gather.rb +1 -3
- data/lib/omq/cli/version.rb +1 -1
- data/lib/omq/cli.rb +0 -3
- metadata +6 -15
- data/lib/omq/cli/parallel_recv_runner.rb +0 -150
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8e3472d73b7d467c20072e00da82aa7222c1cc99cc99e4147f833cb010422e11
|
|
4
|
+
data.tar.gz: 0d2f5fff5723cc45af838de506d92d47581dbb24feb302d63b297565393cd182
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 10001c2717aded533e3c902c481d34bf5a6870e55207cec33b04c7043f7f19cfb71c68f99f467554ca619cebba352a6258bc634538d781caeeefb76cc0813b18
|
|
7
|
+
data.tar.gz: 1fd287acb7bd0fb85d7c260a74c7ef32208c4893b819f780e1a9b05e3ea7219619a4b00e41847ae03a2a5cb1356348416e17d4e74f1a5cb4c8fcf42cd5741013
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,40 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.0 — 2026-04-07
|
|
4
|
+
|
|
5
|
+
### Changed
|
|
6
|
+
|
|
7
|
+
- **`-P` restricted to pipe only** — parallel Ractor workers are no longer
|
|
8
|
+
available on recv-only socket types (pull, sub, gather, dish). Pipe workers
|
|
9
|
+
now use bare Ractors with their own Async reactors and OMQ sockets, removing
|
|
10
|
+
the `omq-ractor` dependency entirely.
|
|
11
|
+
- **`-P` range capped to 1..16** — default is still `nproc`, clamped to 16.
|
|
12
|
+
`-P 1` is valid (single Ractor worker, no sockets on main thread).
|
|
13
|
+
- **Removed `omq-ractor` dependency** — no longer needed.
|
|
14
|
+
- **Removed `ParallelRecvRunner`** — the Ractor bridge infrastructure for
|
|
15
|
+
non-pipe socket types has been deleted.
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- **Pipe `-P` END blocks execute after timeout** — worker recv loops now
|
|
20
|
+
catch `IO::TimeoutError` so BEGIN/END expressions run to completion.
|
|
21
|
+
|
|
22
|
+
## 0.6.0 — 2026-04-07
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
|
|
26
|
+
- **Modal `--compress` for pipe `--in`/`--out`** — `--compress` after `--in`
|
|
27
|
+
decompresses input, after `--out` compresses output. Enables mixed pipelines
|
|
28
|
+
like plain input → compressed output:
|
|
29
|
+
`omq pipe --in -c src --out --compress -c dst`.
|
|
30
|
+
Without `--in`/`--out`, `--compress` applies to both directions as before.
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
|
|
34
|
+
- **Abort with clear message on decompression failure** — receiving an
|
|
35
|
+
uncompressed message with `--compress` now prints a hint instead of
|
|
36
|
+
silently killing the Async task with exit code 0.
|
|
37
|
+
|
|
3
38
|
## 0.5.4 — 2026-04-07
|
|
4
39
|
|
|
5
40
|
### Fixed
|
data/lib/omq/cli/base_runner.rb
CHANGED
|
@@ -51,7 +51,7 @@ module OMQ
|
|
|
51
51
|
|
|
52
52
|
def setup_socket
|
|
53
53
|
@sock = create_socket
|
|
54
|
-
attach_endpoints
|
|
54
|
+
attach_endpoints
|
|
55
55
|
setup_curve
|
|
56
56
|
setup_subscriptions
|
|
57
57
|
compile_expr
|
|
@@ -265,17 +265,6 @@ module OMQ
|
|
|
265
265
|
end
|
|
266
266
|
|
|
267
267
|
|
|
268
|
-
# Parallel recv-eval: delegates to ParallelRecvRunner.
|
|
269
|
-
#
|
|
270
|
-
def run_parallel_recv(task)
|
|
271
|
-
# @sock was created by call() before run_loop; close it now so it doesn't
|
|
272
|
-
# steal messages from the N worker sockets ParallelRecvRunner creates.
|
|
273
|
-
@sock&.close
|
|
274
|
-
@sock = nil
|
|
275
|
-
ParallelRecvRunner.new(@klass, config, @fmt, method(:output)).run(task)
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
|
|
279
268
|
def wait_for_loops(receiver, sender)
|
|
280
269
|
if config.data || config.file || config.send_expr || config.recv_expr || config.target
|
|
281
270
|
sender.wait
|
data/lib/omq/cli/cli_parser.rb
CHANGED
|
@@ -219,6 +219,8 @@ module OMQ
|
|
|
219
219
|
rcvbuf: nil,
|
|
220
220
|
conflate: false,
|
|
221
221
|
compress: false,
|
|
222
|
+
compress_in: false,
|
|
223
|
+
compress_out: false,
|
|
222
224
|
send_expr: nil,
|
|
223
225
|
recv_expr: nil,
|
|
224
226
|
parallel: nil,
|
|
@@ -346,7 +348,17 @@ module OMQ
|
|
|
346
348
|
o.on("--conflate", "Keep only last message per subscriber (PUB/RADIO)") { opts[:conflate] = true }
|
|
347
349
|
|
|
348
350
|
o.separator "\nCompression:"
|
|
349
|
-
o.on("-z", "--compress", "Zstandard compression per frame
|
|
351
|
+
o.on("-z", "--compress", "Zstandard compression per frame (modal with --in/--out)") do
|
|
352
|
+
require "zstd-ruby"
|
|
353
|
+
case pipe_side
|
|
354
|
+
when :in
|
|
355
|
+
opts[:compress_in] = true
|
|
356
|
+
when :out
|
|
357
|
+
opts[:compress_out] = true
|
|
358
|
+
else
|
|
359
|
+
opts[:compress] = true
|
|
360
|
+
end
|
|
361
|
+
end
|
|
350
362
|
|
|
351
363
|
o.separator "\nProcessing (-e = incoming, -E = outgoing):"
|
|
352
364
|
o.on("-e", "--recv-eval EXPR", "Eval Ruby for each incoming message ($F = parts)") { |v| opts[:recv_expr] = v }
|
|
@@ -355,9 +367,9 @@ module OMQ
|
|
|
355
367
|
require "omq" unless defined?(OMQ::VERSION)
|
|
356
368
|
opts[:scripts] << (v == "-" ? :stdin : (v.start_with?("./", "../") ? File.expand_path(v) : v))
|
|
357
369
|
}
|
|
358
|
-
o.on("-P", "--parallel [N]", Integer, "Parallel Ractor workers (default: nproc
|
|
370
|
+
o.on("-P", "--parallel [N]", Integer, "Parallel Ractor workers for pipe (default: nproc, max 16)") { |v|
|
|
359
371
|
require "etc"
|
|
360
|
-
opts[:parallel] = v || Etc.nprocessors
|
|
372
|
+
opts[:parallel] = [v || Etc.nprocessors, 16].min
|
|
361
373
|
}
|
|
362
374
|
|
|
363
375
|
o.separator "\nCURVE encryption (requires system libsodium):"
|
|
@@ -474,16 +486,10 @@ module OMQ
|
|
|
474
486
|
abort "--send-eval and --target are mutually exclusive" if opts[:send_expr] && opts[:target]
|
|
475
487
|
|
|
476
488
|
if opts[:parallel]
|
|
477
|
-
abort "-P/--parallel
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
elsif RECV_ONLY.include?(type_name)
|
|
482
|
-
abort "-P/--parallel on #{type_name} requires --recv-eval (-e)" unless opts[:recv_expr]
|
|
483
|
-
abort "-P/--parallel requires all endpoints to use --connect (not --bind)" if opts[:binds].any?
|
|
484
|
-
else
|
|
485
|
-
abort "-P/--parallel is only valid for pipe or recv-only socket types (#{RECV_ONLY.join(', ')})"
|
|
486
|
-
end
|
|
489
|
+
abort "-P/--parallel is only valid for pipe" unless type_name == "pipe"
|
|
490
|
+
abort "-P/--parallel must be 1..16" unless (1..16).include?(opts[:parallel])
|
|
491
|
+
all_pipe_eps = opts[:in_endpoints] + opts[:out_endpoints] + opts[:endpoints]
|
|
492
|
+
abort "-P/--parallel requires all endpoints to use --connect (not --bind)" if all_pipe_eps.any?(&:bind?)
|
|
487
493
|
end
|
|
488
494
|
|
|
489
495
|
(opts[:connects] + opts[:binds]).each do |url|
|
data/lib/omq/cli/config.rb
CHANGED
data/lib/omq/cli/formatter.rb
CHANGED
data/lib/omq/cli/pipe.rb
CHANGED
|
@@ -13,6 +13,8 @@ module OMQ
|
|
|
13
13
|
def initialize(config)
|
|
14
14
|
@config = config
|
|
15
15
|
@fmt = Formatter.new(config.format, compress: config.compress)
|
|
16
|
+
@fmt_in = Formatter.new(config.format, compress: config.compress_in || config.compress)
|
|
17
|
+
@fmt_out = Formatter.new(config.format, compress: config.compress_out || config.compress)
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
|
|
@@ -112,10 +114,10 @@ module OMQ
|
|
|
112
114
|
loop do
|
|
113
115
|
parts = @pull.receive
|
|
114
116
|
break if parts.nil?
|
|
115
|
-
parts = @
|
|
117
|
+
parts = @fmt_in.decompress(parts)
|
|
116
118
|
parts = eval_recv_expr(parts)
|
|
117
119
|
if parts && !parts.empty?
|
|
118
|
-
out = @
|
|
120
|
+
out = @fmt_out.compress(parts)
|
|
119
121
|
@push.send(out)
|
|
120
122
|
end
|
|
121
123
|
i += 1
|
|
@@ -128,140 +130,113 @@ module OMQ
|
|
|
128
130
|
|
|
129
131
|
|
|
130
132
|
def run_parallel(task)
|
|
133
|
+
OMQ.freeze_for_ractors!
|
|
131
134
|
in_eps, out_eps = resolve_endpoints
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
ensure
|
|
138
|
-
pairs&.each do |pull, push|
|
|
139
|
-
pull&.close
|
|
140
|
-
push&.close
|
|
141
|
-
end
|
|
142
|
-
end
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
def build_socket_pairs(n_workers, in_eps, out_eps)
|
|
146
|
-
pull_opts = { linger: config.linger }
|
|
147
|
-
push_opts = { linger: config.linger }
|
|
148
|
-
pull_opts[:recv_timeout] = config.timeout if config.timeout
|
|
149
|
-
push_opts[:send_timeout] = config.timeout if config.timeout
|
|
150
|
-
n_workers.times.map { build_pull_push(pull_opts, push_opts, in_eps, out_eps) }
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
def wait_for_pairs(pairs)
|
|
155
|
-
with_timeout(config.timeout) do
|
|
156
|
-
Barrier do |barrier|
|
|
157
|
-
pairs.each do |pull, push|
|
|
158
|
-
barrier.async(annotation: "wait push peer") { push.peer_connected.wait }
|
|
159
|
-
barrier.async(annotation: "wait pull peer") { pull.peer_connected.wait }
|
|
160
|
-
end
|
|
161
|
-
end
|
|
162
|
-
end
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
def setup_parallel_transient(task, pairs)
|
|
167
|
-
return unless config.transient
|
|
168
|
-
task.async do
|
|
169
|
-
pairs[0][0].all_peers_gone.wait
|
|
170
|
-
pairs.each do |pull, _|
|
|
171
|
-
pull.reconnect_enabled = false
|
|
172
|
-
pull.close_read
|
|
173
|
-
end
|
|
135
|
+
workers = spawn_workers(config, in_eps, out_eps)
|
|
136
|
+
workers.each do |w|
|
|
137
|
+
w.join
|
|
138
|
+
rescue ::Ractor::RemoteError => e
|
|
139
|
+
$stderr.write("omq: Ractor error: #{e.cause&.message || e.message}\n")
|
|
174
140
|
end
|
|
175
141
|
end
|
|
176
142
|
|
|
177
143
|
|
|
178
|
-
def
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
n_count = d[:n_count]
|
|
207
|
-
if eval_proc
|
|
208
|
-
if n_count && n_count > 0
|
|
209
|
-
n_count.times do
|
|
210
|
-
parts = pull_p.receive
|
|
211
|
-
break if parts.nil?
|
|
212
|
-
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
213
|
-
_ctx.instance_exec(formatter.decompress(parts), &eval_proc)
|
|
214
|
-
)
|
|
215
|
-
next if parts.nil?
|
|
216
|
-
push_p << formatter.compress(parts) unless parts.empty?
|
|
217
|
-
end
|
|
218
|
-
else
|
|
219
|
-
loop do
|
|
220
|
-
parts = pull_p.receive
|
|
221
|
-
break if parts.nil?
|
|
222
|
-
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
223
|
-
_ctx.instance_exec(formatter.decompress(parts), &eval_proc)
|
|
224
|
-
)
|
|
225
|
-
next if parts.nil?
|
|
226
|
-
push_p << formatter.compress(parts) unless parts.empty?
|
|
227
|
-
end
|
|
144
|
+
def spawn_workers(config, in_eps, out_eps)
|
|
145
|
+
config.parallel.times.map do
|
|
146
|
+
::Ractor.new(config, in_eps, out_eps) do |cfg, ins, outs|
|
|
147
|
+
Async do
|
|
148
|
+
pull = OMQ::PULL.new(linger: cfg.linger)
|
|
149
|
+
push = OMQ::PUSH.new(linger: cfg.linger)
|
|
150
|
+
pull.recv_timeout = cfg.timeout if cfg.timeout
|
|
151
|
+
push.send_timeout = cfg.timeout if cfg.timeout
|
|
152
|
+
pull.reconnect_interval = cfg.reconnect_ivl if cfg.reconnect_ivl
|
|
153
|
+
push.reconnect_interval = cfg.reconnect_ivl if cfg.reconnect_ivl
|
|
154
|
+
pull.heartbeat_interval = cfg.heartbeat_ivl if cfg.heartbeat_ivl
|
|
155
|
+
push.heartbeat_interval = cfg.heartbeat_ivl if cfg.heartbeat_ivl
|
|
156
|
+
pull.send_hwm = cfg.send_hwm if cfg.send_hwm
|
|
157
|
+
pull.recv_hwm = cfg.recv_hwm if cfg.recv_hwm
|
|
158
|
+
push.send_hwm = cfg.send_hwm if cfg.send_hwm
|
|
159
|
+
push.recv_hwm = cfg.recv_hwm if cfg.recv_hwm
|
|
160
|
+
pull.sndbuf = cfg.sndbuf if cfg.sndbuf
|
|
161
|
+
pull.rcvbuf = cfg.rcvbuf if cfg.rcvbuf
|
|
162
|
+
push.sndbuf = cfg.sndbuf if cfg.sndbuf
|
|
163
|
+
push.rcvbuf = cfg.rcvbuf if cfg.rcvbuf
|
|
164
|
+
|
|
165
|
+
OMQ::CLI::SocketSetup.attach_endpoints(pull, ins, verbose: cfg.verbose >= 1)
|
|
166
|
+
OMQ::CLI::SocketSetup.attach_endpoints(push, outs, verbose: cfg.verbose >= 1)
|
|
167
|
+
|
|
168
|
+
Barrier do |barrier|
|
|
169
|
+
barrier.async { pull.peer_connected.wait }
|
|
170
|
+
barrier.async { push.peer_connected.wait }
|
|
228
171
|
end
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
172
|
+
|
|
173
|
+
begin_proc, end_proc, eval_proc =
|
|
174
|
+
OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(cfg.recv_expr)
|
|
175
|
+
|
|
176
|
+
fmt_in = OMQ::CLI::Formatter.new(cfg.format, compress: cfg.compress_in || cfg.compress)
|
|
177
|
+
fmt_out = OMQ::CLI::Formatter.new(cfg.format, compress: cfg.compress_out || cfg.compress)
|
|
178
|
+
|
|
179
|
+
_ctx = Object.new
|
|
180
|
+
_ctx.instance_exec(&begin_proc) if begin_proc
|
|
181
|
+
|
|
182
|
+
n_count = cfg.count
|
|
183
|
+
begin
|
|
184
|
+
if eval_proc
|
|
185
|
+
if n_count && n_count > 0
|
|
186
|
+
n_count.times do
|
|
187
|
+
parts = pull.receive
|
|
188
|
+
break if parts.nil?
|
|
189
|
+
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
190
|
+
_ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
|
|
191
|
+
)
|
|
192
|
+
next if parts.nil?
|
|
193
|
+
push << fmt_out.compress(parts) unless parts.empty?
|
|
194
|
+
end
|
|
195
|
+
else
|
|
196
|
+
loop do
|
|
197
|
+
parts = pull.receive
|
|
198
|
+
break if parts.nil?
|
|
199
|
+
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
200
|
+
_ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
|
|
201
|
+
)
|
|
202
|
+
next if parts.nil?
|
|
203
|
+
push << fmt_out.compress(parts) unless parts.empty?
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
else
|
|
207
|
+
if n_count && n_count > 0
|
|
208
|
+
n_count.times do
|
|
209
|
+
parts = pull.receive
|
|
210
|
+
break if parts.nil?
|
|
211
|
+
push << fmt_out.compress(fmt_in.decompress(parts))
|
|
212
|
+
end
|
|
213
|
+
else
|
|
214
|
+
loop do
|
|
215
|
+
parts = pull.receive
|
|
216
|
+
break if parts.nil?
|
|
217
|
+
push << fmt_out.compress(fmt_in.decompress(parts))
|
|
218
|
+
end
|
|
219
|
+
end
|
|
241
220
|
end
|
|
221
|
+
rescue IO::TimeoutError, Async::TimeoutError
|
|
222
|
+
# recv timed out — fall through to END block
|
|
242
223
|
end
|
|
243
|
-
end
|
|
244
224
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
225
|
+
if end_proc
|
|
226
|
+
out = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
227
|
+
_ctx.instance_exec(&end_proc)
|
|
228
|
+
)
|
|
229
|
+
push << fmt_out.compress(out) if out && !out.empty?
|
|
230
|
+
end
|
|
231
|
+
ensure
|
|
232
|
+
pull&.close
|
|
233
|
+
push&.close
|
|
250
234
|
end
|
|
251
235
|
end
|
|
252
236
|
end
|
|
253
237
|
end
|
|
254
238
|
|
|
255
239
|
|
|
256
|
-
def join_workers(workers)
|
|
257
|
-
workers.each do |w|
|
|
258
|
-
w.value
|
|
259
|
-
rescue Ractor::RemoteError => e
|
|
260
|
-
$stderr.write("omq: Ractor error: #{e.cause&.message || e.message}\n")
|
|
261
|
-
end
|
|
262
|
-
end
|
|
263
|
-
|
|
264
|
-
|
|
265
240
|
# ── Shared helpers ────────────────────────────────────────────────
|
|
266
241
|
|
|
267
242
|
|
data/lib/omq/cli/pub_sub.rb
CHANGED
|
@@ -10,9 +10,7 @@ module OMQ
|
|
|
10
10
|
|
|
11
11
|
# Runner for SUB sockets (subscribe and receive published messages).
|
|
12
12
|
class SubRunner < BaseRunner
|
|
13
|
-
def run_loop(task)
|
|
14
|
-
config.parallel ? run_parallel_recv(task) : run_recv_logic
|
|
15
|
-
end
|
|
13
|
+
def run_loop(task) = run_recv_logic
|
|
16
14
|
end
|
|
17
15
|
end
|
|
18
16
|
end
|
data/lib/omq/cli/push_pull.rb
CHANGED
|
@@ -10,9 +10,7 @@ module OMQ
|
|
|
10
10
|
|
|
11
11
|
# Runner for PULL sockets (receive-only pipeline consumer).
|
|
12
12
|
class PullRunner < BaseRunner
|
|
13
|
-
def run_loop(task)
|
|
14
|
-
config.parallel ? run_parallel_recv(task) : run_recv_logic
|
|
15
|
-
end
|
|
13
|
+
def run_loop(task) = run_recv_logic
|
|
16
14
|
end
|
|
17
15
|
end
|
|
18
16
|
end
|
data/lib/omq/cli/radio_dish.rb
CHANGED
|
@@ -23,9 +23,7 @@ module OMQ
|
|
|
23
23
|
|
|
24
24
|
# Runner for DISH sockets (draft; group-based subscribe).
|
|
25
25
|
class DishRunner < BaseRunner
|
|
26
|
-
def run_loop(task)
|
|
27
|
-
config.parallel ? run_parallel_recv(task) : run_recv_logic
|
|
28
|
-
end
|
|
26
|
+
def run_loop(task) = run_recv_logic
|
|
29
27
|
end
|
|
30
28
|
end
|
|
31
29
|
end
|
data/lib/omq/cli/version.rb
CHANGED
data/lib/omq/cli.rb
CHANGED
|
@@ -9,7 +9,6 @@ require_relative "cli/expression_evaluator"
|
|
|
9
9
|
require_relative "cli/socket_setup"
|
|
10
10
|
require_relative "cli/routing_helper"
|
|
11
11
|
require_relative "cli/transient_monitor"
|
|
12
|
-
require_relative "cli/parallel_recv_runner"
|
|
13
12
|
require_relative "cli/base_runner"
|
|
14
13
|
require_relative "cli/push_pull"
|
|
15
14
|
require_relative "cli/pub_sub"
|
|
@@ -199,8 +198,6 @@ module OMQ
|
|
|
199
198
|
require "console"
|
|
200
199
|
|
|
201
200
|
CliParser.validate_gems!(config)
|
|
202
|
-
require "omq/ractor" if config.parallel
|
|
203
|
-
|
|
204
201
|
trap("INT") { Process.exit!(0) }
|
|
205
202
|
trap("TERM") { Process.exit!(0) }
|
|
206
203
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: omq-cli
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Patrik Wenger
|
|
@@ -16,6 +16,9 @@ dependencies:
|
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
18
|
version: '0.15'
|
|
19
|
+
- - ">="
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: 0.15.2
|
|
19
22
|
type: :runtime
|
|
20
23
|
prerelease: false
|
|
21
24
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -23,20 +26,9 @@ dependencies:
|
|
|
23
26
|
- - "~>"
|
|
24
27
|
- !ruby/object:Gem::Version
|
|
25
28
|
version: '0.15'
|
|
26
|
-
-
|
|
27
|
-
name: omq-ractor
|
|
28
|
-
requirement: !ruby/object:Gem::Requirement
|
|
29
|
-
requirements:
|
|
30
|
-
- - "~>"
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: '0.1'
|
|
33
|
-
type: :runtime
|
|
34
|
-
prerelease: false
|
|
35
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
-
requirements:
|
|
37
|
-
- - "~>"
|
|
29
|
+
- - ">="
|
|
38
30
|
- !ruby/object:Gem::Version
|
|
39
|
-
version:
|
|
31
|
+
version: 0.15.2
|
|
40
32
|
- !ruby/object:Gem::Dependency
|
|
41
33
|
name: omq-rfc-clientserver
|
|
42
34
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -173,7 +165,6 @@ files:
|
|
|
173
165
|
- lib/omq/cli/expression_evaluator.rb
|
|
174
166
|
- lib/omq/cli/formatter.rb
|
|
175
167
|
- lib/omq/cli/pair.rb
|
|
176
|
-
- lib/omq/cli/parallel_recv_runner.rb
|
|
177
168
|
- lib/omq/cli/pipe.rb
|
|
178
169
|
- lib/omq/cli/pub_sub.rb
|
|
179
170
|
- lib/omq/cli/push_pull.rb
|
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module OMQ
|
|
4
|
-
module CLI
|
|
5
|
-
# Manages N OMQ::Ractor workers for parallel recv-eval.
|
|
6
|
-
#
|
|
7
|
-
# Each worker gets its own input socket connecting to the external
|
|
8
|
-
# endpoints; ZMQ distributes work naturally. Results are collected via
|
|
9
|
-
# an inproc PULL back to the main task for output.
|
|
10
|
-
#
|
|
11
|
-
class ParallelRecvRunner
|
|
12
|
-
# @param klass [Class] OMQ socket class for worker input sockets
|
|
13
|
-
# @param config [Config] frozen CLI configuration
|
|
14
|
-
# @param fmt [Formatter] message formatter
|
|
15
|
-
# @param output_fn [Method] callable for writing output
|
|
16
|
-
def initialize(klass, config, fmt, output_fn)
|
|
17
|
-
@klass = klass
|
|
18
|
-
@config = config
|
|
19
|
-
@fmt = fmt
|
|
20
|
-
@output_fn = output_fn
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
# Spawns N Ractor workers, distributes incoming messages, and collects results.
|
|
25
|
-
#
|
|
26
|
-
# @param task [Async::Task] parent async task
|
|
27
|
-
# @return [void]
|
|
28
|
-
def run(task)
|
|
29
|
-
cfg = @config
|
|
30
|
-
n_workers = cfg.parallel
|
|
31
|
-
inproc = "inproc://omq-out-#{object_id}"
|
|
32
|
-
|
|
33
|
-
# Pack worker config into a shareable Hash passed via omq.data —
|
|
34
|
-
# Ruby 4.0 forbids Ractor blocks from closing over outer locals.
|
|
35
|
-
worker_data = ::Ractor.make_shareable({
|
|
36
|
-
recv_src: cfg.recv_expr,
|
|
37
|
-
fmt_sym: cfg.format,
|
|
38
|
-
fmt_compr: cfg.compress,
|
|
39
|
-
})
|
|
40
|
-
|
|
41
|
-
# Create N input sockets in the main Async context
|
|
42
|
-
input_socks = n_workers.times.map do
|
|
43
|
-
sock = SocketSetup.build(@klass, cfg)
|
|
44
|
-
SocketSetup.setup_subscriptions(sock, cfg)
|
|
45
|
-
cfg.connects.each { |url| sock.connect(url) }
|
|
46
|
-
sock
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
with_timeout(cfg.timeout) { input_socks.each { |s| s.peer_connected.wait } }
|
|
50
|
-
|
|
51
|
-
# Inproc collector: one bound PULL to receive all worker output
|
|
52
|
-
collector = OMQ::PULL.new(linger: cfg.linger)
|
|
53
|
-
collector.recv_timeout = cfg.timeout if cfg.timeout
|
|
54
|
-
collector.bind(inproc)
|
|
55
|
-
|
|
56
|
-
# N output sockets connecting to the collector
|
|
57
|
-
output_socks = n_workers.times.map do
|
|
58
|
-
s = OMQ::PUSH.new(linger: cfg.linger)
|
|
59
|
-
s.connect(inproc)
|
|
60
|
-
s
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
workers = n_workers.times.map do |i|
|
|
64
|
-
OMQ::Ractor.new(input_socks[i], output_socks[i], serialize: false, data: worker_data) do |omq|
|
|
65
|
-
pull_p, push_p = omq.sockets
|
|
66
|
-
d = omq.data
|
|
67
|
-
|
|
68
|
-
# Re-compile expression inside Ractor (Procs are not shareable)
|
|
69
|
-
begin_proc, end_proc, eval_proc =
|
|
70
|
-
OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(d[:recv_src])
|
|
71
|
-
|
|
72
|
-
formatter = OMQ::CLI::Formatter.new(d[:fmt_sym], compress: d[:fmt_compr])
|
|
73
|
-
# Use a dedicated context object so @ivar expressions in BEGIN/END/eval
|
|
74
|
-
# work inside Ractors (self in a Ractor is shareable; Object.new is not).
|
|
75
|
-
_ctx = Object.new
|
|
76
|
-
_ctx.instance_exec(&begin_proc) if begin_proc
|
|
77
|
-
|
|
78
|
-
if eval_proc
|
|
79
|
-
loop do
|
|
80
|
-
parts = pull_p.receive
|
|
81
|
-
break if parts.nil?
|
|
82
|
-
parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
83
|
-
_ctx.instance_exec(formatter.decompress(parts), &eval_proc)
|
|
84
|
-
)
|
|
85
|
-
next if parts.nil?
|
|
86
|
-
push_p << parts unless parts.empty?
|
|
87
|
-
end
|
|
88
|
-
else
|
|
89
|
-
loop do
|
|
90
|
-
parts = pull_p.receive
|
|
91
|
-
break if parts.nil?
|
|
92
|
-
push_p << formatter.decompress(parts)
|
|
93
|
-
end
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
if end_proc
|
|
97
|
-
out = OMQ::CLI::ExpressionEvaluator.normalize_result(
|
|
98
|
-
_ctx.instance_exec(&end_proc)
|
|
99
|
-
)
|
|
100
|
-
push_p << out if out && !out.empty?
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
# Collect loop: drain inproc PULL → output
|
|
107
|
-
n_count = cfg.count
|
|
108
|
-
if n_count && n_count > 0
|
|
109
|
-
n_count.times do
|
|
110
|
-
parts = collector.receive
|
|
111
|
-
break if parts.nil?
|
|
112
|
-
@output_fn.call(parts)
|
|
113
|
-
end
|
|
114
|
-
else
|
|
115
|
-
loop do
|
|
116
|
-
parts = collector.receive
|
|
117
|
-
break if parts.nil?
|
|
118
|
-
@output_fn.call(parts)
|
|
119
|
-
end
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
# Inject nil into each worker's input port so it exits its loop
|
|
124
|
-
# without waiting for recv_timeout (workers don't self-terminate
|
|
125
|
-
# when the collect loop exits on count).
|
|
126
|
-
workers.each do |w|
|
|
127
|
-
w.close
|
|
128
|
-
rescue Ractor::RemoteError => e
|
|
129
|
-
$stderr.puts "omq: Ractor error: #{e.cause&.message || e.message}"
|
|
130
|
-
end
|
|
131
|
-
ensure
|
|
132
|
-
input_socks&.each(&:close)
|
|
133
|
-
output_socks&.each(&:close)
|
|
134
|
-
collector&.close
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
private
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def with_timeout(seconds)
|
|
142
|
-
if seconds
|
|
143
|
-
Async::Task.current.with_timeout(seconds) { yield }
|
|
144
|
-
else
|
|
145
|
-
yield
|
|
146
|
-
end
|
|
147
|
-
end
|
|
148
|
-
end
|
|
149
|
-
end
|
|
150
|
-
end
|