nnq-cli 0.2.0

data/lib/nnq/cli/formatter.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Raised when LZ4 decompression fails.
+    class DecompressError < RuntimeError; end
+
+    # Handles encoding/decoding a single-body message in the configured
+    # format, plus optional LZ4 compression.
+    #
+    # Unlike omq-cli's Formatter, nnq messages are not multipart — one
+    # `String` body per message. The API still accepts/returns a
+    # 1-element array so that `$F`-based eval expressions work the same
+    # way.
+    class Formatter
+      # @param format [Symbol] wire format (:ascii, :quoted, :raw, :jsonl, :msgpack, :marshal)
+      # @param compress [Boolean] whether to apply LZ4 compression
+      def initialize(format, compress: false)
+        @format   = format
+        @compress = compress
+      end
+
+
+      # Encodes a message body into a printable string for output.
+      #
+      # @param msg [Array<String>] single-element array (the body)
+      # @return [String] formatted output line
+      def encode(msg)
+        body = msg.first.to_s
+        case @format
+        when :ascii
+          body.b.gsub(/[^[:print:]\t]/, ".") + "\n"
+        when :quoted
+          body.b.dump[1..-2] + "\n"
+        when :raw
+          body
+        when :jsonl
+          JSON.generate([body]) + "\n"
+        when :msgpack
+          MessagePack.pack([body])
+        when :marshal
+          body.inspect + "\n"
+        end
+      end
+
+
+      # Decodes a formatted input line into a 1-element message array.
+      #
+      # @param line [String] input line (newline-terminated)
+      # @return [Array<String>] 1-element array
+      def decode(line)
+        case @format
+        when :ascii, :marshal
+          [line.chomp]
+        when :quoted
+          ["\"#{line.chomp}\"".undump]
+        when :raw
+          [line]
+        when :jsonl
+          arr = JSON.parse(line.chomp)
+          unless arr.is_a?(Array) && arr.all? { |e| e.is_a?(String) }
+            abort "JSON Lines input must be an array of strings"
+          end
+          arr.first(1)
+        end
+      end
+
+
+      # Decodes one Marshal object from the given IO stream.
+      #
+      # @param io [IO] input stream
+      # @return [Object, nil] deserialized object, or nil on EOF
+      def decode_marshal(io)
+        Marshal.load(io)
+      rescue EOFError, TypeError
+        nil
+      end
+
+
+      # Decodes one MessagePack object from the given IO stream.
+      #
+      # @param io [IO] input stream
+      # @return [Object, nil] deserialized object, or nil on EOF
+      def decode_msgpack(io)
+        @msgpack_unpacker ||= MessagePack::Unpacker.new(io)
+        @msgpack_unpacker.read
+      rescue EOFError
+        nil
+      end
+
+
+      # Compresses the body with LZ4 if compression is enabled.
+      #
+      # @param msg [Array<String>] single-element array
+      # @return [Array<String>] optionally compressed
+      def compress(msg)
+        @compress ? msg.map { |p| RLZ4.compress(p) if p } : msg
+      end
+
+
+      # Decompresses the body with LZ4 if compression is enabled.
+      # nil/empty bodies pass through.
+      #
+      # @param msg [Array<String>] possibly compressed single-element array
+      # @return [Array<String>] decompressed
+      def decompress(msg)
+        @compress ? msg.map { |p| p && !p.empty? ? RLZ4.decompress(p) : p } : msg
+      rescue RLZ4::DecompressError
+        raise DecompressError, "decompression failed (did the sender use --compress?)"
+      end
+
+
+      # Formats a message body for human-readable preview (logging).
+      #
+      # @param msg [Array<String>] single-element array
+      # @return [String] truncated preview
+      def self.preview(msg)
+        body = msg.first.to_s
+        "(#{body.bytesize}B) #{preview_body(body)}"
+      end
+
+
+      def self.preview_body(body)
+        bytes = body.b
+        return "''" if bytes.empty?
+
+        sample    = bytes[0, 12]
+        printable = sample.count("\x20-\x7e")
+
+        if printable < sample.bytesize / 2
+          "[#{bytes.bytesize}B]"
+        elsif bytes.bytesize > 12
+          "#{sample.gsub(/[^[:print:]]/, ".")}..."
+        else
+          sample.gsub(/[^[:print:]]/, ".")
+        end
+      end
+    end
+  end
+end

data/lib/nnq/cli/pair.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Runner for PAIR sockets (bidirectional messaging).
+    class PairRunner < BaseRunner
+      private
+
+
+      def run_loop(task)
+        receiver = recv_async(task)
+        sender   = task.async { run_send_logic }
+        wait_for_loops(receiver, sender)
+      end
+
+
+      def recv_async(task)
+        task.async do
+          n = config.count
+          i = 0
+          loop do
+            msg = recv_msg
+            break if msg.nil?
+            msg = eval_recv_expr(msg)
+            output(msg)
+            i += 1
+            break if n && n > 0 && i >= n
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nnq/cli/pipe.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Runner for the virtual "pipe" socket type (PULL -> eval -> PUSH).
+    # Supports sequential and parallel (Ractor-based) processing modes.
+    class PipeRunner
+      # @return [Config] frozen CLI configuration
+      attr_reader :config
+
+
+      # @param config [Config] frozen CLI configuration
+      def initialize(config)
+        @config  = config
+        @fmt_in  = Formatter.new(config.format, compress: config.compress_in || config.compress)
+        @fmt_out = Formatter.new(config.format, compress: config.compress_out || config.compress)
+      end
+
+
+      # Runs the pipe in sequential or parallel mode based on config.
+      #
+      # @param task [Async::Task] the parent async task
+      # @return [void]
+      def call(task)
+        if config.parallel
+          run_parallel(task)
+        else
+          run_sequential(task)
+        end
+      end
+
+
+      private
+
+
+      def resolve_endpoints
+        if config.in_endpoints.any?
+          [config.in_endpoints, config.out_endpoints]
+        else
+          [[config.endpoints[0]], [config.endpoints[1]]]
+        end
+      end
+
+
+      # -- Sequential ---------------------------------------------------
+
+
+      def run_sequential(task)
+        set_pipe_process_title
+        in_eps, out_eps = resolve_endpoints
+        @pull, @push = build_pull_push(in_eps, out_eps)
+        compile_expr
+        @sock = @pull  # for eval instance_exec
+        start_event_monitors if config.verbose >= 2
+        wait_for_peers_with_timeout if config.timeout
+        setup_sequential_transient(task)
+        @sock.instance_exec(&@recv_begin_proc) if @recv_begin_proc
+        sequential_message_loop(fan_out: out_eps.size > 1)
+        @sock.instance_exec(&@recv_end_proc) if @recv_end_proc
+      ensure
+        @pull&.close
+        @push&.close
+      end
+
+
+      def build_pull_push(in_eps, out_eps)
+        pull = SocketSetup.build(NNQ::PULL0, config)
+        push = SocketSetup.build(NNQ::PUSH0, config)
+        SocketSetup.attach_endpoints(pull, in_eps, verbose: config.verbose)
+        SocketSetup.attach_endpoints(push, out_eps, verbose: config.verbose)
+        [pull, push]
+      end
+
+
+      # With --timeout set, fail fast if peers never show up. Without
+      # it, there's no point waiting: PULL#receive blocks naturally
+      # and PUSH buffers up to send_hwm when no peer is present.
+      def wait_for_peers_with_timeout
+        Fiber.scheduler.with_timeout(config.timeout) do
+          Async::Barrier.new.tap do |barrier|
+            barrier.async(annotation: "wait push peer") { @push.peer_connected.wait }
+            barrier.async(annotation: "wait pull peer") { @pull.peer_connected.wait }
+            barrier.wait
+          end
+        end
+      end
+
+
+      def setup_sequential_transient(task)
+        return unless config.transient
+        task.async do
+          @pull.all_peers_gone.wait
+          @pull.reconnect_enabled = false
+          @pull.close_read
+        end
+      end
+
+
+      def sequential_message_loop(fan_out: false)
+        n = config.count
+        i = 0
+
+        loop do
+          body = @pull.receive
+          break if body.nil?
+          msg = @fmt_in.decompress([body])
+          msg = eval_recv_expr(msg)
+
+          if msg && !msg.empty?
+            out = @fmt_out.compress(msg)
+            @push.send(out.first)
+          end
+
+          # Yield after send so send-pump fibers can drain the queue
+          # before the next message is enqueued. Without this, one pump
+          # monopolizes the shared queue when messages arrive in bursts.
+          # Only needed for multi-output pipes; single-output has no
+          # fairness concern.
+          Async::Task.current.yield if fan_out
+
+          i += 1
+          break if n && n > 0 && i >= n
+        end
+      end
+
+
+      # -- Parallel -----------------------------------------------------
+
+
+      def run_parallel(task)
+        set_pipe_process_title
+        NNQ.freeze_for_ractors! if NNQ.respond_to?(:freeze_for_ractors!)
+        in_eps, out_eps = resolve_endpoints
+        in_eps  = RactorHelpers.preresolve_tcp(in_eps)
+        out_eps = RactorHelpers.preresolve_tcp(out_eps)
+        log_port, log_thread = RactorHelpers.start_log_consumer
+        error_port = Ractor::Port.new
+        error_thread = Thread.new(error_port) do |p|
+          msg = p.receive
+          abort "nnq: #{msg}" unless msg.equal?(RactorHelpers::SHUTDOWN)
+        rescue Ractor::ClosedError
+          # port closed, no error
+        end
+        workers = config.parallel.times.map do
+          ::Ractor.new(config, in_eps, out_eps, log_port, error_port) do |cfg, ins, outs, lport, eport|
+            PipeWorker.new(cfg, ins, outs, lport, eport).call
+          end
+        end
+        workers.each do |w|
+          w.join
+        rescue ::Ractor::RemoteError => e
+          $stderr.write("nnq: Ractor error: #{e.cause&.message || e.message}\n")
+        end
+      ensure
+        RactorHelpers.stop_consumer(error_port, error_thread) if error_port
+        RactorHelpers.stop_consumer(log_port, log_thread) if log_port
+      end
+
+
+      # -- Process title -------------------------------------------------
+
+
+      def set_pipe_process_title
+        in_eps, out_eps = resolve_endpoints
+        title = ["nnq pipe"]
+        title << "-z" if config.compress || config.compress_in || config.compress_out
+        title << "-P#{config.parallel}" if config.parallel
+        title.concat(in_eps.map(&:url))
+        title << "->"
+        title.concat(out_eps.map(&:url))
+        Process.setproctitle(title.join(" "))
+      end
+
+
+      # -- Expression eval ----------------------------------------------
+
+
+      def compile_expr
+        @recv_evaluator  = ExpressionEvaluator.new(config.recv_expr, format: config.format)
+        @recv_begin_proc = @recv_evaluator.begin_proc
+        @recv_eval_proc  = @recv_evaluator.eval_proc
+        @recv_end_proc   = @recv_evaluator.end_proc
+      end
+
+
+      def eval_recv_expr(msg)
+        result = @recv_evaluator.call(msg, @sock)
+        result.equal?(ExpressionEvaluator::SENT) ? nil : result
+      end
+
+
+      # -- Event monitoring ---------------------------------------------
+
+
+      def start_event_monitors
+        verbose = config.verbose >= 3
+        v = config.verbose
+        [@pull, @push].each do |sock|
+          sock.monitor(verbose: verbose) do |event|
+            CLI::Term.write_event(event, v)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nnq/cli/pipe_worker.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Worker that runs inside a Ractor for pipe -P parallel mode.
+    # Each worker owns its own Async reactor, PULL socket, and PUSH socket.
+    #
+    class PipeWorker
+      def initialize(config, in_eps, out_eps, log_port, error_port = nil)
+        @config     = config
+        @in_eps     = in_eps
+        @out_eps    = out_eps
+        @log_port   = log_port
+        @error_port = error_port
+      end
+
+
+      def call
+        Async do
+          setup_sockets
+          log_endpoints if @config.verbose >= 1
+          start_monitors if @config.verbose >= 2
+          wait_for_peers
+          compile_expr
+          run_message_loop
+          run_end_block
+        rescue NNQ::CLI::DecompressError => e
+          @error_port&.send(e.message)
+        ensure
+          @pull&.close
+          @push&.close
+        end
+      end
+
+
+      private
+
+
+      def setup_sockets
+        @pull = NNQ::CLI::SocketSetup.build(NNQ::PULL0, @config)
+        @push = NNQ::CLI::SocketSetup.build(NNQ::PUSH0, @config)
+        NNQ::CLI::SocketSetup.attach_endpoints(@pull, @in_eps, verbose: 0)
+        NNQ::CLI::SocketSetup.attach_endpoints(@push, @out_eps, verbose: 0)
+      end
+
+
+      def log_endpoints
+        @in_eps.each { |ep| @log_port.send(ep.bind? ? "Bound to #{ep.url}" : "Connecting to #{ep.url}") }
+        @out_eps.each { |ep| @log_port.send(ep.bind? ? "Bound to #{ep.url}" : "Connecting to #{ep.url}") }
+      end
+
+
+      def start_monitors
+        trace = @config.verbose >= 3
+        [@pull, @push].each do |sock|
+          sock.monitor(verbose: trace) do |event|
+            @log_port.send(format_event(event))
+          end
+        end
+      end
+
+
+      def format_event(event)
+        case event.type
+        when :message_sent
+          "nnq: >> #{NNQ::CLI::Formatter.preview([event.detail[:body]])}"
+        when :message_received
+          "nnq: << #{NNQ::CLI::Formatter.preview([event.detail[:body]])}"
+        else
+          ep     = event.endpoint ? " #{event.endpoint}" : ""
+          detail = event.detail ? " #{event.detail}" : ""
+          "nnq: #{event.type}#{ep}#{detail}"
+        end
+      end
+
+
+      def wait_for_peers
+        Async::Barrier.new.tap do |barrier|
+          barrier.async { @pull.peer_connected.wait }
+          barrier.async { @push.peer_connected.wait }
+          barrier.wait
+        end
+      end
+
+
+      def compile_expr
+        @begin_proc, @end_proc, @eval_proc =
+          NNQ::CLI::ExpressionEvaluator.compile_inside_ractor(@config.recv_expr)
+        @fmt_in  = NNQ::CLI::Formatter.new(@config.format, compress: @config.compress_in || @config.compress)
+        @fmt_out = NNQ::CLI::Formatter.new(@config.format, compress: @config.compress_out || @config.compress)
+        @ctx = Object.new
+        @ctx.instance_exec(&@begin_proc) if @begin_proc
+      end
+
+
+      def run_message_loop
+        n = @config.count
+        if @eval_proc
+          loop do
+            body = @pull.receive
+            break if body.nil?
+            msg = NNQ::CLI::ExpressionEvaluator.normalize_result(
+              @ctx.instance_exec(@fmt_in.decompress([body]), &@eval_proc)
+            )
+            unless msg.nil? || msg.empty?
+              out = @fmt_out.compress(msg)
+              @push.send(out.first)
+            end
+            n -= 1 if n && n > 0
+            break if n == 0
+          end
+        else
+          loop do
+            body = @pull.receive
+            break if body.nil?
+            out = @fmt_out.compress(@fmt_in.decompress([body]))
+            @push.send(out.first)
+            n -= 1 if n && n > 0
+            break if n == 0
+          end
+        end
+      rescue IO::TimeoutError, Async::TimeoutError
+        # recv timed out -- fall through to END block
+      end
+
+
+      def run_end_block
+        return unless @end_proc
+        out = NNQ::CLI::ExpressionEvaluator.normalize_result(
+          @ctx.instance_exec(&@end_proc)
+        )
+        if out && !out.empty?
+          @push.send(@fmt_out.compress(out).first)
+        end
+      end
+    end
+  end
+end

data/lib/nnq/cli/pub_sub.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Runner for PUB sockets (publish messages to subscribers).
+    class PubRunner < BaseRunner
+      def run_loop(task) = run_send_logic
+    end
+
+
+    # Runner for SUB sockets (subscribe and receive published messages).
+    class SubRunner < BaseRunner
+      def run_loop(task) = run_recv_logic
+    end
+  end
+end

data/lib/nnq/cli/push_pull.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Runner for PUSH sockets (send-only pipeline producer).
+    class PushRunner < BaseRunner
+      def run_loop(task) = run_send_logic
+    end
+
+
+    # Runner for PULL sockets (receive-only pipeline consumer).
+    class PullRunner < BaseRunner
+      private
+
+
+      def run_loop(task) = run_recv_logic
+    end
+  end
+end

data/lib/nnq/cli/ractor_helpers.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module NNQ
+  module CLI
+    # Shared Ractor infrastructure for parallel worker modes.
+    module RactorHelpers
+      # Sentinel value sent through ports to signal consumer threads to exit.
+      # Port#close does not unblock a waiting #receive, so we must send an
+      # explicit shutdown marker.
+      SHUTDOWN = :__nnq_shutdown__
+
+
+      # Resolves TCP hostnames to IP addresses so Ractors don't touch
+      # Resolv::DefaultResolver (which is not shareable).
+      #
+      def self.preresolve_tcp(endpoints)
+        endpoints.flat_map do |ep|
+          url = ep.url
+          if url.start_with?("tcp://")
+            host, port = NNQ::Transport::TCP.parse_endpoint(url)
+            Addrinfo.getaddrinfo(host, port, nil, :STREAM).map do |addr|
+              ip = addr.ip_address
+              ip = "[#{ip}]" if ip.include?(":")
+              Endpoint.new("tcp://#{ip}:#{addr.ip_port}", ep.bind?)
+            end
+          else
+            ep
+          end
+        end
+      end
+
+
+      # Starts a Ractor::Port and a consumer thread that drains log
+      # messages to stderr sequentially. Returns [port, thread].
+      # Send SHUTDOWN through the port to stop the consumer.
+      #
+      def self.start_log_consumer
+        port = Ractor::Port.new
+        thread = Thread.new(port) do |p|
+          loop do
+            msg = p.receive
+            break if msg.equal?(SHUTDOWN)
+            $stderr.write("#{msg}\n")
+          rescue Ractor::ClosedError
+            break
+          end
+        end
+        [port, thread]
+      end
+
+
+      # Starts a Ractor::Port and a consumer thread that drains
+      # formatted output to stdout sequentially. Returns [port, thread].
+      # Send SHUTDOWN through the port to stop the consumer.
+      #
+      def self.start_output_consumer
+        port = Ractor::Port.new
+        thread = Thread.new(port) do |p|
+          loop do
+            msg = p.receive
+            break if msg.equal?(SHUTDOWN)
+            $stdout.write(msg)
+          rescue Ractor::ClosedError
+            break
+          end
+        end
+        [port, thread]
+      end
+
+
+      # Sends the shutdown sentinel and joins the consumer thread.
+      #
+      def self.stop_consumer(port, thread)
+        port.send(SHUTDOWN)
+        thread.join
+      rescue Ractor::ClosedError
+        thread.join(1)
+      end
+    end
+  end
+end