omq-cli 0.7.1 → 0.8.0

data/lib/omq/cli/parallel_worker.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+module OMQ
+  module CLI
+    # Worker that runs inside a Ractor for parallel socket modes (-P).
+    # Each worker owns its own Async reactor and socket instance.
+    #
+    # Supported socket types:
+    #   - pull, gather  (recv-only)
+    #   - rep           (recv-reply with echo/data/eval)
+    #
+    class ParallelWorker
+      def initialize(config, socket_sym, endpoints, output_port, log_port)
+        @config      = config
+        @socket_sym  = socket_sym
+        @endpoints   = endpoints
+        @output_port = output_port
+        @log_port    = log_port
+      end
+
+
+      def call
+        Async do
+          setup_socket
+          log_endpoints
+          start_monitors
+          wait_for_peer
+          compile_expr
+          run_loop
+          run_end_block
+        ensure
+          @sock&.close
+        end
+      end
+
+
+      private
+
+
+      def setup_socket
+        @sock = OMQ.const_get(@socket_sym).new
+        OMQ::CLI::SocketSetup.apply_options(@sock, @config)
+        @sock.identity = @config.identity if @config.identity
+        OMQ::CLI::SocketSetup.attach_endpoints(@sock, @endpoints, verbose: false)
+      end
+
+
+      def log_endpoints
+        return unless @config.verbose >= 1
+        @endpoints.each do |ep|
+          @log_port.send(ep.bind? ? "Bound to #{ep.url}" : "Connecting to #{ep.url}")
+        end
+      end
+
+
+      def start_monitors
+        return unless @config.verbose >= 2
+        trace = @config.verbose >= 3
+        @sock.monitor(verbose: trace) do |event|
+          @log_port.send(format_event(event))
+        end
+      end
+
+
+      def format_event(event)
+        case event.type
+        when :message_sent
+          "omq: >> #{OMQ::CLI::Formatter.preview(event.detail[:parts])}"
+        when :message_received
+          "omq: << #{OMQ::CLI::Formatter.preview(event.detail[:parts])}"
+        else
+          ep = event.endpoint ? " #{event.endpoint}" : ""
+          detail = event.detail ? " #{event.detail}" : ""
+          "omq: #{event.type}#{ep}#{detail}"
+        end
+      end
+
+
+      def wait_for_peer
+        if @config.timeout
+          Fiber.scheduler.with_timeout(@config.timeout) do
+            @sock.peer_connected.wait
+          end
+        else
+          @sock.peer_connected.wait
+        end
+      rescue IO::TimeoutError, Async::TimeoutError
+        # Proceed anyway -- recv will timeout if no messages arrive
+      end
+
+
+      def compile_expr
+        @begin_proc, @end_proc, @eval_proc =
+          OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(@config.recv_expr)
+        @fmt = OMQ::CLI::Formatter.new(@config.format, compress: @config.compress)
+        @ctx = Object.new
+        @ctx.instance_exec(&@begin_proc) if @begin_proc
+      end
+
+
+      def run_loop
+        case @config.type_name
+        when "pull", "gather"
+          run_recv_loop
+        when "rep"
+          run_rep_loop
+        end
+      end
+
+
+      # -- Recv-only loop (PULL, GATHER) -----------------------------------
+
+
+      def run_recv_loop
+        n = @config.count
+        i = 0
+        loop do
+          parts = @sock.receive
+          break if parts.nil?
+          parts = @fmt.decompress(parts)
+          if @eval_proc
+            parts = normalize(
+              @ctx.instance_exec(parts, &@eval_proc)
+            )
+            next if parts.nil?
+          end
+          output(parts) unless parts.empty?
+          i += 1
+          break if n && n > 0 && i >= n
+        end
+      rescue IO::TimeoutError, Async::TimeoutError
+        # recv timed out -- fall through to END block
+      end
+
+
+      # -- REP loop (recv request, process, send reply) --------------------
+
+
+      def run_rep_loop
+        n = @config.count
+        i = 0
+        loop do
+          parts = @sock.receive
+          break if parts.nil?
+          parts = @fmt.decompress(parts)
+          reply = compute_reply(parts)
+          output(reply)
+          @sock.send(@fmt.compress(reply))
+          i += 1
+          break if n && n > 0 && i >= n
+        end
+      rescue IO::TimeoutError, Async::TimeoutError
+        # recv timed out -- fall through to END block
+      end
+
+
+      def compute_reply(parts)
+        if @eval_proc
+          normalize(@ctx.instance_exec(parts, &@eval_proc)) || [""]
+        elsif @config.echo
+          parts
+        elsif @config.data
+          @fmt.decode(@config.data + "\n")
+        else
+          parts
+        end
+      end
+
+
+      # -- Output and helpers ----------------------------------------------
+
+
+      def output(parts)
+        return if @config.quiet || parts.nil?
+        @output_port.send(@fmt.encode(parts))
+      end
+
+
+      def normalize(result)
+        OMQ::CLI::ExpressionEvaluator.normalize_result(result)
+      end
+
+
+      def run_end_block
+        return unless @end_proc
+        out = normalize(@ctx.instance_exec(&@end_proc))
+        output(out) if out && !out.empty?
+      end
+    end
+  end
+end

data/lib/omq/cli/pipe.rb

@@ -5,14 +5,18 @@ module OMQ
     # Runner for the virtual "pipe" socket type (PULL -> eval -> PUSH).
     # Supports sequential and parallel (Ractor-based) processing modes.
     class PipeRunner
+      # Default HWM for pipe sockets when the user hasn't set one.
+      # Much lower than the socket default (1000) to bound memory
+      # with large messages in pipeline stages.
+      PIPE_HWM = 64
+
       # @return [Config] frozen CLI configuration
       attr_reader :config
 
 
       # @param config [Config] frozen CLI configuration
       def initialize(config)
-        @config = config
-        @fmt    = Formatter.new(config.format, compress: config.compress)
+        @config  = config
         @fmt_in  = Formatter.new(config.format, compress: config.compress_in || config.compress)
         @fmt_out = Formatter.new(config.format, compress: config.compress_out || config.compress)
       end
@@ -43,30 +47,28 @@ module OMQ
       end
 
 
-      def attach_endpoints(sock, endpoints)
-        SocketSetup.attach_endpoints(sock, endpoints, verbose: config.verbose >= 1)
-      end
-
-
-      # ── Sequential ───────────────────────────────────────────────────
+      # -- Sequential ---------------------------------------------------
 
 
       def run_sequential(task)
+        set_pipe_process_title
         in_eps, out_eps = resolve_endpoints
-        @pull, @push = build_pull_push(
-          { linger: config.linger, recv_timeout: config.timeout },
-          { linger: config.linger, send_timeout: config.timeout },
-          in_eps, out_eps
-        )
+        @pull, @push = build_pull_push(in_eps, out_eps)
         compile_expr
         @sock = @pull  # for eval instance_exec
         start_event_monitors if config.verbose >= 2
-        with_timeout(config.timeout) do
+        wait_body = proc do
           Barrier do |barrier|
             barrier.async(annotation: "wait push peer") { @push.peer_connected.wait }
             barrier.async(annotation: "wait pull peer") { @pull.peer_connected.wait }
           end
         end
+
+        if config.timeout
+          Fiber.scheduler.with_timeout(config.timeout, &wait_body)
+        else
+          wait_body.call
+        end
         setup_sequential_transient(task)
         @sock.instance_exec(&@recv_begin_proc) if @recv_begin_proc
         sequential_message_loop
@@ -77,23 +79,15 @@ module OMQ
       end
 
 
-      def apply_socket_options(sock)
-        sock.reconnect_interval = config.reconnect_ivl if config.reconnect_ivl
-        sock.heartbeat_interval = config.heartbeat_ivl if config.heartbeat_ivl
-        sock.send_hwm           = config.send_hwm      if config.send_hwm
-        sock.recv_hwm           = config.recv_hwm      if config.recv_hwm
-        sock.sndbuf             = config.sndbuf        if config.sndbuf
-        sock.rcvbuf             = config.rcvbuf        if config.rcvbuf
-      end
-
-
-      def build_pull_push(pull_opts, push_opts, in_eps, out_eps)
-        pull = OMQ::PULL.new(**pull_opts)
-        push = OMQ::PUSH.new(**push_opts)
-        apply_socket_options(pull)
-        apply_socket_options(push)
-        attach_endpoints(pull, in_eps)
-        attach_endpoints(push, out_eps)
+      def build_pull_push(in_eps, out_eps)
+        pull = OMQ::PULL.new
+        push = OMQ::PUSH.new
+        SocketSetup.apply_options(pull, config)
+        SocketSetup.apply_options(push, config)
+        pull.recv_hwm = PIPE_HWM unless config.recv_hwm
+        push.send_hwm = PIPE_HWM unless config.send_hwm
+        SocketSetup.attach_endpoints(pull, in_eps, verbose: config.verbose >= 1)
+        SocketSetup.attach_endpoints(push, out_eps, verbose: config.verbose >= 1)
         [pull, push]
       end
 
@@ -117,8 +111,7 @@ module OMQ
           parts = @fmt_in.decompress(parts)
           parts = eval_recv_expr(parts)
           if parts && !parts.empty?
-            out = @fmt_out.compress(parts)
-            @push.send(out)
+            @push.send(@fmt_out.compress(parts))
           end
           i += 1
           break if n && n > 0 && i >= n
@@ -126,149 +119,47 @@ module OMQ
       end
 
 
-      # ── Parallel ─────────────────────────────────────────────────────
+      # -- Parallel -----------------------------------------------------
 
 
       def run_parallel(task)
+        set_pipe_process_title
         OMQ.freeze_for_ractors!
         in_eps, out_eps = resolve_endpoints
-        in_eps  = preresolve_tcp(in_eps)
-        out_eps = preresolve_tcp(out_eps)
-        workers = spawn_workers(config, in_eps, out_eps)
+        in_eps  = RactorHelpers.preresolve_tcp(in_eps)
+        out_eps = RactorHelpers.preresolve_tcp(out_eps)
+        log_port, log_thread = RactorHelpers.start_log_consumer
+        workers = config.parallel.times.map do
+          ::Ractor.new(config, in_eps, out_eps, log_port) do |cfg, ins, outs, lport|
+            PipeWorker.new(cfg, ins, outs, lport).call
+          end
+        end
         workers.each do |w|
           w.join
         rescue ::Ractor::RemoteError => e
           $stderr.write("omq: Ractor error: #{e.cause&.message || e.message}\n")
         end
+      ensure
+        RactorHelpers.stop_consumer(log_port, log_thread) if log_port
       end
 
 
-      def spawn_workers(config, in_eps, out_eps)
-        config.parallel.times.map do
-          ::Ractor.new(config, in_eps, out_eps) do |cfg, ins, outs|
-            Async do
-              pull = OMQ::PULL.new(linger: cfg.linger)
-              push = OMQ::PUSH.new(linger: cfg.linger)
-              pull.recv_timeout      = cfg.timeout if cfg.timeout
-              push.send_timeout      = cfg.timeout if cfg.timeout
-              pull.reconnect_interval = cfg.reconnect_ivl if cfg.reconnect_ivl
-              push.reconnect_interval = cfg.reconnect_ivl if cfg.reconnect_ivl
-              pull.heartbeat_interval = cfg.heartbeat_ivl if cfg.heartbeat_ivl
-              push.heartbeat_interval = cfg.heartbeat_ivl if cfg.heartbeat_ivl
-              pull.send_hwm          = cfg.send_hwm if cfg.send_hwm
-              pull.recv_hwm          = cfg.recv_hwm if cfg.recv_hwm
-              push.send_hwm          = cfg.send_hwm if cfg.send_hwm
-              push.recv_hwm          = cfg.recv_hwm if cfg.recv_hwm
-              pull.sndbuf            = cfg.sndbuf if cfg.sndbuf
-              pull.rcvbuf            = cfg.rcvbuf if cfg.rcvbuf
-              push.sndbuf            = cfg.sndbuf if cfg.sndbuf
-              push.rcvbuf            = cfg.rcvbuf if cfg.rcvbuf
-
-              OMQ::CLI::SocketSetup.attach_endpoints(pull, ins, verbose: cfg.verbose >= 1)
-              OMQ::CLI::SocketSetup.attach_endpoints(push, outs, verbose: cfg.verbose >= 1)
-
-              Barrier do |barrier|
-                barrier.async { pull.peer_connected.wait }
-                barrier.async { push.peer_connected.wait }
-              end
-
-              begin_proc, end_proc, eval_proc =
-                OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(cfg.recv_expr)
-
-              fmt_in  = OMQ::CLI::Formatter.new(cfg.format, compress: cfg.compress_in || cfg.compress)
-              fmt_out = OMQ::CLI::Formatter.new(cfg.format, compress: cfg.compress_out || cfg.compress)
-
-              _ctx = Object.new
-              _ctx.instance_exec(&begin_proc) if begin_proc
-
-              n_count = cfg.count
-              begin
-                if eval_proc
-                  if n_count && n_count > 0
-                    n_count.times do
-                      parts = pull.receive
-                      break if parts.nil?
-                      parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
-                        _ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
-                      )
-                      next if parts.nil?
-                      push << fmt_out.compress(parts) unless parts.empty?
-                    end
-                  else
-                    loop do
-                      parts = pull.receive
-                      break if parts.nil?
-                      parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
-                        _ctx.instance_exec(fmt_in.decompress(parts), &eval_proc)
-                      )
-                      next if parts.nil?
-                      push << fmt_out.compress(parts) unless parts.empty?
-                    end
-                  end
-                else
-                  if n_count && n_count > 0
-                    n_count.times do
-                      parts = pull.receive
-                      break if parts.nil?
-                      push << fmt_out.compress(fmt_in.decompress(parts))
-                    end
-                  else
-                    loop do
-                      parts = pull.receive
-                      break if parts.nil?
-                      push << fmt_out.compress(fmt_in.decompress(parts))
-                    end
-                  end
-                end
-              rescue IO::TimeoutError, Async::TimeoutError
-                # recv timed out — fall through to END block
-              end
-
-              if end_proc
-                out = OMQ::CLI::ExpressionEvaluator.normalize_result(
-                  _ctx.instance_exec(&end_proc)
-                )
-                push << fmt_out.compress(out) if out && !out.empty?
-              end
-            ensure
-              pull&.close
-              push&.close
-            end
-          end
-        end
-      end
-
-
-      # ── Shared helpers ────────────────────────────────────────────────
+      # -- Process title -------------------------------------------------
 
 
-      # Resolves TCP hostnames to IP addresses so Ractors don't touch
-      # Resolv::DefaultResolver (which is not shareable).
-      #
-      def preresolve_tcp(endpoints)
-        endpoints.flat_map do |ep|
-          url = ep.url
-          if url.start_with?("tcp://")
-            host, port = OMQ::Transport::TCP.parse_endpoint(url)
-            Addrinfo.getaddrinfo(host, port, nil, :STREAM).map do |addr|
-              ip = addr.ip_address
-              ip = "[#{ip}]" if ip.include?(":")
-              Endpoint.new("tcp://#{ip}:#{addr.ip_port}", ep.bind?)
-            end
-          else
-            ep
-          end
-        end
+      def set_pipe_process_title
+        in_eps, out_eps = resolve_endpoints
+        title = ["omq pipe"]
+        title << "-z" if config.compress || config.compress_in || config.compress_out
+        title << "-P#{config.parallel}" if config.parallel
+        title.concat(in_eps.map(&:url))
+        title << "->"
+        title.concat(out_eps.map(&:url))
+        Process.setproctitle(title.join(" "))
       end
 
 
-      def with_timeout(seconds)
-        if seconds
-          Async::Task.current.with_timeout(seconds) { yield }
-        else
-          yield
-        end
-      end
+      # -- Expression eval ----------------------------------------------
 
 
       def compile_expr
@@ -285,9 +176,7 @@ module OMQ
       end
 
 
-      def log(msg)
-        $stderr.write("#{msg}\n") if config.verbose >= 1
-      end
+      # -- Event monitoring ---------------------------------------------
 
 
       def start_event_monitors
@@ -296,9 +185,9 @@ module OMQ
           sock.monitor(verbose: verbose) do |event|
             case event.type
             when :message_sent
-              $stderr.write("omq: >> #{msg_preview(event.detail[:parts])}\n")
+              $stderr.write("omq: >> #{Formatter.preview(event.detail[:parts])}\n")
             when :message_received
-              $stderr.write("omq: << #{msg_preview(event.detail[:parts])}\n")
+              $stderr.write("omq: << #{Formatter.preview(event.detail[:parts])}\n")
             else
               ep = event.endpoint ? " #{event.endpoint}" : ""
               detail = event.detail ? " #{event.detail}" : ""
@@ -307,15 +196,6 @@ module OMQ
           end
         end
       end
-
-
-      def msg_preview(parts)
-        parts.map { |p|
-          bytes = p.b
-          preview = bytes[0, 10].gsub(/[^[:print:]]/, ".")
-          bytes.bytesize > 10 ? "#{preview}... (#{bytes.bytesize}B)" : preview
-        }.join(" | ")
-      end
     end
   end
 end

data/lib/omq/cli/pipe_worker.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module OMQ
+  module CLI
+    # Worker that runs inside a Ractor for pipe -P parallel mode.
+    # Each worker owns its own Async reactor, PULL socket, and PUSH socket.
+    #
+    class PipeWorker
+      def initialize(config, in_eps, out_eps, log_port)
+        @config   = config
+        @in_eps   = in_eps
+        @out_eps  = out_eps
+        @log_port = log_port
+      end
+
+
+      def call
+        Async do
+          setup_sockets
+          log_endpoints if @config.verbose >= 1
+          start_monitors if @config.verbose >= 2
+          wait_for_peers
+          compile_expr
+          run_message_loop
+          run_end_block
+        ensure
+          @pull&.close
+          @push&.close
+        end
+      end
+
+
+      private
+
+
+      def setup_sockets
+        @pull = OMQ::PULL.new
+        @push = OMQ::PUSH.new
+        OMQ::CLI::SocketSetup.apply_options(@pull, @config)
+        OMQ::CLI::SocketSetup.apply_options(@push, @config)
+        @pull.recv_hwm = PipeRunner::PIPE_HWM unless @config.recv_hwm
+        @push.send_hwm = PipeRunner::PIPE_HWM unless @config.send_hwm
+        OMQ::CLI::SocketSetup.attach_endpoints(@pull, @in_eps, verbose: false)
+        OMQ::CLI::SocketSetup.attach_endpoints(@push, @out_eps, verbose: false)
+      end
+
+
+      def log_endpoints
+        @in_eps.each { |ep| @log_port.send(ep.bind? ? "Bound to #{ep.url}" : "Connecting to #{ep.url}") }
+        @out_eps.each { |ep| @log_port.send(ep.bind? ? "Bound to #{ep.url}" : "Connecting to #{ep.url}") }
+      end
+
+
+      def start_monitors
+        trace = @config.verbose >= 3
+        [@pull, @push].each do |sock|
+          sock.monitor(verbose: trace) do |event|
+            @log_port.send(format_event(event))
+          end
+        end
+      end
+
+
+      def format_event(event)
+        case event.type
+        when :message_sent
+          "omq: >> #{OMQ::CLI::Formatter.preview(event.detail[:parts])}"
+        when :message_received
+          "omq: << #{OMQ::CLI::Formatter.preview(event.detail[:parts])}"
+        else
+          ep = event.endpoint ? " #{event.endpoint}" : ""
+          detail = event.detail ? " #{event.detail}" : ""
+          "omq: #{event.type}#{ep}#{detail}"
+        end
+      end
+
+
+      def wait_for_peers
+        Barrier do |barrier|
+          barrier.async { @pull.peer_connected.wait }
+          barrier.async { @push.peer_connected.wait }
+        end
+      end
+
+
+      def compile_expr
+        @begin_proc, @end_proc, @eval_proc =
+          OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(@config.recv_expr)
+        @fmt_in  = OMQ::CLI::Formatter.new(@config.format, compress: @config.compress_in || @config.compress)
+        @fmt_out = OMQ::CLI::Formatter.new(@config.format, compress: @config.compress_out || @config.compress)
+        @ctx = Object.new
+        @ctx.instance_exec(&@begin_proc) if @begin_proc
+      end
+
+
+      def run_message_loop
+        n = @config.count
+        if @eval_proc
+          if n && n > 0
+            n.times do
+              parts = @pull.receive
+              break if parts.nil?
+              parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
+                @ctx.instance_exec(@fmt_in.decompress(parts), &@eval_proc)
+              )
+              next if parts.nil?
+              @push << @fmt_out.compress(parts) unless parts.empty?
+            end
+          else
+            loop do
+              parts = @pull.receive
+              break if parts.nil?
+              parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
+                @ctx.instance_exec(@fmt_in.decompress(parts), &@eval_proc)
+              )
+              next if parts.nil?
+              @push << @fmt_out.compress(parts) unless parts.empty?
+            end
+          end
+        else
+          if n && n > 0
+            n.times do
+              parts = @pull.receive
+              break if parts.nil?
+              @push << @fmt_out.compress(@fmt_in.decompress(parts))
+            end
+          else
+            loop do
+              parts = @pull.receive
+              break if parts.nil?
+              @push << @fmt_out.compress(@fmt_in.decompress(parts))
+            end
+          end
+        end
+      rescue IO::TimeoutError, Async::TimeoutError
+        # recv timed out -- fall through to END block
+      end
+
+
+      def run_end_block
+        return unless @end_proc
+        out = OMQ::CLI::ExpressionEvaluator.normalize_result(
+          @ctx.instance_exec(&@end_proc)
+        )
+        @push << @fmt_out.compress(out) if out && !out.empty?
+      end
+    end
+  end
+end

data/lib/omq/cli/push_pull.rb

@@ -10,6 +10,14 @@ module OMQ
 
     # Runner for PULL sockets (receive-only pipeline consumer).
     class PullRunner < BaseRunner
+      def call(task)
+        config.parallel ? run_parallel_workers(:PULL) : super
+      end
+
+
+      private
+
+
       def run_loop(task) = run_recv_logic
     end
   end