omq-cli 0.12.3 → 0.14.0

data/lib/omq/cli/parallel_worker.rb

@@ -29,8 +29,15 @@ module OMQ
           compile_expr
           run_loop
           run_end_block
-        rescue DecompressError => e
-          @error_port.send(e.message)
+        rescue OMQ::SocketDeadError => error
+          # Socket was killed by a protocol violation on the peer side
+          # (see Engine#signal_fatal_error). Surface the underlying
+          # cause via the log stream and exit cleanly -- the Ractor
+          # completes, consumer threads unblock.
+          reason = error.cause&.message || error.message
+          @log_port.send("omq: #{reason}")
+        rescue => error
+          @error_port.send("#{error.class}: #{error.message}")
         ensure
           @sock&.close
         end
@@ -43,6 +50,8 @@ module OMQ
       def setup_socket
         @sock = @config.ffi ? OMQ.const_get(@socket_sym).new(backend: :ffi) : OMQ.const_get(@socket_sym).new
         OMQ::CLI::SocketSetup.apply_options(@sock, @config)
+        OMQ::CLI::SocketSetup.apply_recv_maxsz(@sock, @config)
+        OMQ::CLI::SocketSetup.apply_compression(@sock, @config, @config.type_name)
         @sock.identity = @config.identity if @config.identity
         OMQ::CLI::SocketSetup.attach_endpoints(@sock, @endpoints, verbose: 0)
       end
@@ -51,20 +60,32 @@ module OMQ
       def log_endpoints
         return unless @config.verbose >= 1
         @endpoints.each do |ep|
-          @log_port.send(OMQ::CLI::Term.format_attach(ep.bind? ? :bind : :connect, ep.url, @config.verbose))
+          @log_port.send(OMQ::CLI::Term.format_attach(ep.bind? ? :bind : :connect, ep.url, @config.timestamps))
         end
       end
 
 
       def start_monitors
-        return unless @config.verbose >= 2
-        trace = @config.verbose >= 3
+        trace      = @config.verbose >= 3
+        log_events = @config.verbose >= 2
         @sock.monitor(verbose: trace) do |event|
-          @log_port.send(OMQ::CLI::Term.format_event(event, @config.verbose))
+          @log_port.send(OMQ::CLI::Term.format_event(event, @config.timestamps)) if log_events
+          kill_on_protocol_error(event)
         end
       end
 
 
+      # Mirrors BaseRunner#kill_on_protocol_error: CLI-level policy
+      # that protocol-level disconnects kill the socket so the
+      # recv loop unblocks with SocketDeadError.
+      def kill_on_protocol_error(event)
+        return unless event.type == :disconnected
+        error = event.detail && event.detail[:error]
+        return unless error.is_a?(Protocol::ZMTP::Error)
+        @sock.engine.signal_fatal_error(error)
+      end
+
+
       def wait_for_peer
         if @config.timeout
           Fiber.scheduler.with_timeout(@config.timeout) do
@@ -81,7 +102,7 @@ module OMQ
       def compile_expr
         @begin_proc, @end_proc, @eval_proc =
           OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(@config.recv_expr)
-        @fmt = OMQ::CLI::Formatter.new(@config.format, compress: @config.compress)
+        @fmt = OMQ::CLI::Formatter.new(@config.format)
         @ctx = Object.new
         @ctx.instance_exec(&@begin_proc) if @begin_proc
       end
@@ -106,7 +127,6 @@ module OMQ
         loop do
           parts = @sock.receive
           break if parts.nil?
-          parts = @fmt.decompress(parts)
           if @eval_proc
             parts = normalize(
               @ctx.instance_exec(parts, &@eval_proc)
@@ -131,10 +151,9 @@ module OMQ
         loop do
           parts = @sock.receive
           break if parts.nil?
-          parts = @fmt.decompress(parts)
           reply = compute_reply(parts)
           output(reply)
-          @sock.send(@fmt.compress(reply))
+          @sock.send(reply)
           i += 1
           break if n && n > 0 && i >= n
         end
@@ -166,7 +185,7 @@ module OMQ
 
 
       def normalize(result)
-        OMQ::CLI::ExpressionEvaluator.normalize_result(result)
+        OMQ::CLI::ExpressionEvaluator.normalize_result(result, format: @config.format)
       end
 
 

data/lib/omq/cli/pipe.rb

@@ -11,9 +11,8 @@ module OMQ
 
       # @param config [Config] frozen CLI configuration
       def initialize(config)
-        @config  = config
-        @fmt_in  = Formatter.new(config.format, compress: config.compress_in || config.compress)
-        @fmt_out = Formatter.new(config.format, compress: config.compress_out || config.compress)
+        @config = config
+        @fmt    = Formatter.new(config.format)
       end
 
 
@@ -51,12 +50,16 @@ module OMQ
         @pull, @push = build_pull_push(in_eps, out_eps)
         compile_expr
         @sock = @pull  # for eval instance_exec
-        start_event_monitors if config.verbose >= 2
+        start_event_monitors
         wait_for_peers_with_timeout if config.timeout
         setup_sequential_transient(task)
         @sock.instance_exec(&@recv_begin_proc) if @recv_begin_proc
         sequential_message_loop(fan_out: out_eps.size > 1)
         @sock.instance_exec(&@recv_end_proc) if @recv_end_proc
+      rescue OMQ::SocketDeadError => error
+        reason = error.cause&.message || error.message
+        $stderr.write("omq: #{reason}\n")
+        exit 1
       ensure
         @pull&.close
         @push&.close
@@ -85,8 +88,11 @@ module OMQ
         push = OMQ::PUSH.new(**kwargs)
         SocketSetup.apply_options(pull, config)
         SocketSetup.apply_options(push, config)
-        SocketSetup.attach_endpoints(pull, in_eps, verbose: config.verbose)
-        SocketSetup.attach_endpoints(push, out_eps, verbose: config.verbose)
+        SocketSetup.apply_recv_maxsz(pull, config)
+        SocketSetup.apply_compression(pull, config, "pull")
+        SocketSetup.apply_compression(push, config, "push")
+        SocketSetup.attach_endpoints(pull, in_eps, verbose: config.verbose, timestamps: config.timestamps)
+        SocketSetup.attach_endpoints(push, out_eps, verbose: config.verbose, timestamps: config.timestamps)
         [pull, push]
       end
 
@@ -107,10 +113,9 @@ module OMQ
         loop do
           parts = @pull.receive
           break if parts.nil?
-          parts = @fmt_in.decompress(parts)
           parts = eval_recv_expr(parts)
           if parts && !parts.empty?
-            @push.send(@fmt_out.compress(parts))
+            @push.send(parts)
           end
           # Yield after send so send-pump fibers can drain the queue
           # before the next message is enqueued. Without this, one pump
@@ -163,7 +168,7 @@ module OMQ
       def set_pipe_process_title
         in_eps, out_eps = resolve_endpoints
         title = ["omq pipe"]
-        title << "-z" if config.compress || config.compress_in || config.compress_out
+        title << "-z" if config.compress
         title << "-P#{config.parallel}" if config.parallel
         title.concat(in_eps.map(&:url))
         title << "->"
@@ -193,13 +198,23 @@ module OMQ
 
 
       def start_event_monitors
-        trace = config.verbose >= 3
+        trace      = config.verbose >= 3
+        log_events = config.verbose >= 2
         [@pull, @push].each do |sock|
           sock.monitor(verbose: trace) do |event|
-            Term.write_event(event, config.verbose)
+            Term.write_event(event, config.timestamps) if log_events
+            kill_on_protocol_error(sock, event)
           end
         end
       end
+
+
+      def kill_on_protocol_error(sock, event)
+        return unless event.type == :disconnected
+        error = event.detail && event.detail[:error]
+        return unless error.is_a?(Protocol::ZMTP::Error)
+        sock.engine.signal_fatal_error(error)
+      end
     end
   end
 end

data/lib/omq/cli/pipe_worker.rb

@@ -19,13 +19,14 @@ module OMQ
         Async do
           setup_sockets
           log_endpoints if @config.verbose >= 1
-          start_monitors if @config.verbose >= 2
+          start_monitors
           wait_for_peers_with_timeout if @config.timeout
           compile_expr
           run_message_loop
           run_end_block
-        rescue OMQ::CLI::DecompressError => e
-          @error_port&.send(e.message)
+        rescue OMQ::SocketDeadError => error
+          reason = error.cause&.message || error.message
+          @log_port.send("omq: #{reason}")
         ensure
           @pull&.close
           @push&.close
@@ -42,6 +43,9 @@ module OMQ
         @push = OMQ::PUSH.new(**kwargs)
         OMQ::CLI::SocketSetup.apply_options(@pull, @config)
         OMQ::CLI::SocketSetup.apply_options(@push, @config)
+        OMQ::CLI::SocketSetup.apply_recv_maxsz(@pull, @config)
+        OMQ::CLI::SocketSetup.apply_compression(@pull, @config, "pull")
+        OMQ::CLI::SocketSetup.apply_compression(@push, @config, "push")
         OMQ::CLI::SocketSetup.attach_endpoints(@pull, @in_eps, verbose: 0)
         OMQ::CLI::SocketSetup.attach_endpoints(@push, @out_eps, verbose: 0)
       end
@@ -49,21 +53,31 @@ module OMQ
 
       def log_endpoints
         (@in_eps + @out_eps).each do |ep|
-          @log_port.send(OMQ::CLI::Term.format_attach(ep.bind? ? :bind : :connect, ep.url, @config.verbose))
+          @log_port.send(OMQ::CLI::Term.format_attach(ep.bind? ? :bind : :connect, ep.url, @config.timestamps))
         end
       end
 
 
       def start_monitors
-        trace = @config.verbose >= 3
+        trace      = @config.verbose >= 3
+        log_events = @config.verbose >= 2
         [@pull, @push].each do |sock|
           sock.monitor(verbose: trace) do |event|
-            @log_port.send(OMQ::CLI::Term.format_event(event, @config.verbose))
+            @log_port.send(OMQ::CLI::Term.format_event(event, @config.timestamps)) if log_events
+            kill_on_protocol_error(sock, event)
           end
         end
       end
 
 
+      def kill_on_protocol_error(sock, event)
+        return unless event.type == :disconnected
+        error = event.detail && event.detail[:error]
+        return unless error.is_a?(Protocol::ZMTP::Error)
+        sock.engine.signal_fatal_error(error)
+      end
+
+
       # With --timeout set, fail fast if peers never show up. Without
       # it, there's no point waiting: PULL#receive blocks naturally
       # and PUSH buffers up to send_hwm when no peer is present.
@@ -80,8 +94,7 @@ module OMQ
       def compile_expr
         @begin_proc, @end_proc, @eval_proc =
           OMQ::CLI::ExpressionEvaluator.compile_inside_ractor(@config.recv_expr)
-        @fmt_in  = OMQ::CLI::Formatter.new(@config.format, compress: @config.compress_in || @config.compress)
-        @fmt_out = OMQ::CLI::Formatter.new(@config.format, compress: @config.compress_out || @config.compress)
+        @fmt = OMQ::CLI::Formatter.new(@config.format)
         @ctx = Object.new
         @ctx.instance_exec(&@begin_proc) if @begin_proc
       end
@@ -91,39 +104,15 @@ module OMQ
         n = @config.count
         if @eval_proc
           if n && n > 0
-            n.times do
-              parts = @pull.receive
-              break if parts.nil?
-              parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
-                @ctx.instance_exec(@fmt_in.decompress(parts), &@eval_proc)
-              )
-              next if parts.nil?
-              @push << @fmt_out.compress(parts) unless parts.empty?
-            end
+            n.times { break unless process_one_eval }
           else
-            loop do
-              parts = @pull.receive
-              break if parts.nil?
-              parts = OMQ::CLI::ExpressionEvaluator.normalize_result(
-                @ctx.instance_exec(@fmt_in.decompress(parts), &@eval_proc)
-              )
-              next if parts.nil?
-              @push << @fmt_out.compress(parts) unless parts.empty?
-            end
+            loop { break unless process_one_eval }
           end
         else
           if n && n > 0
-            n.times do
-              parts = @pull.receive
-              break if parts.nil?
-              @push << @fmt_out.compress(@fmt_in.decompress(parts))
-            end
+            n.times { break unless process_one_passthrough }
           else
-            loop do
-              parts = @pull.receive
-              break if parts.nil?
-              @push << @fmt_out.compress(@fmt_in.decompress(parts))
-            end
+            loop { break unless process_one_passthrough }
           end
         end
       rescue IO::TimeoutError, Async::TimeoutError
@@ -131,12 +120,32 @@ module OMQ
       end
 
 
+      def process_one_eval
+        parts_in = @pull.receive
+        return false if parts_in.nil?
+        parts_out = OMQ::CLI::ExpressionEvaluator.normalize_result(
+          @ctx.instance_exec(parts_in, &@eval_proc), format: @config.format
+        )
+        return true if parts_out.nil? || parts_out.empty?
+        @push << parts_out
+        true
+      end
+
+
+      def process_one_passthrough
+        parts_in = @pull.receive
+        return false if parts_in.nil?
+        @push << parts_in
+        true
+      end
+
+
       def run_end_block
         return unless @end_proc
         out = OMQ::CLI::ExpressionEvaluator.normalize_result(
-          @ctx.instance_exec(&@end_proc)
+          @ctx.instance_exec(&@end_proc), format: @config.format
         )
-        @push << @fmt_out.compress(out) if out && !out.empty?
+        @push << out if out && !out.empty?
       end
     end
   end

data/lib/omq/cli/radio_dish.rb

@@ -13,7 +13,6 @@ module OMQ
       def send_msg(parts)
         return if parts.empty?
         parts = [Marshal.dump(parts)] if config.format == :marshal
-        parts = @fmt.compress(parts)
         group = config.group || parts.shift
         @sock.publish(group, parts.first || "")
         transient_ready!

data/lib/omq/cli/router_dealer.rb

@@ -25,7 +25,6 @@ module OMQ
             break if parts.nil?
             identity = parts.shift
             parts.shift if parts.first == ""
-            parts = @fmt.decompress(parts)
             result = eval_recv_expr([display_routing_id(identity), *parts])
             output(result)
             i += 1
@@ -36,7 +35,7 @@ module OMQ
 
 
       def send_to_peer(id, parts)
-        @sock.send([id, "", *@fmt.compress(parts)])
+        @sock.send([id, "", *parts])
       end
     end
   end

data/lib/omq/cli/socket_setup.rb

@@ -22,6 +22,40 @@ module OMQ
       # it entirely with --recv-maxsz 0.
       DEFAULT_RECV_MAXSZ = 1 << 20
 
+      # Socket types that never receive application data. These opt
+      # out of the default passive-compression behavior -- they would
+      # otherwise advertise a profile and wrap every outgoing frame in
+      # a 4-byte uncompressed sentinel for no benefit, since there's
+      # nothing to decompress in return.
+      PURE_SEND_TYPES = %w[push pub scatter radio].freeze
+
+
+      # Installs ZMTP-Zstd compression on +sock+ based on +type_name+
+      # and the explicit flags in +config+. Three outcomes:
+      #
+      # * +config.compress+ is true --> active compression (auto-dict).
+      #   Outgoing frames are compressed; incoming are decoded.
+      # * +config.compress+ is false and +type_name+ can receive -->
+      #   passive compression (RFC Sec. 6.4). The socket advertises the
+      #   profile so an active peer can compress on the wire and we can
+      #   decode it, but we never compress our own outgoing frames.
+      # * +config.compress+ is false and +type_name+ is in
+      #   +PURE_SEND_TYPES+ --> no compression. Pure senders have no
+      #   incoming traffic to decompress, so passive mode is pure
+      #   overhead on outgoing.
+      #
+      # Callers pass +type_name+ explicitly (rather than reading it off
+      # +config+) so the pipe runners can install different modes on
+      # their push/pull ends of a single pipe.
+      def self.apply_compression(sock, config, type_name)
+        if config.compress
+          sock.compression = OMQ::RFC::Zstd::Compression.auto(level: config.compress_level || -3)
+        elsif !PURE_SEND_TYPES.include?(type_name)
+          sock.compression = OMQ::RFC::Zstd::Compression.auto(passive: true)
+        end
+      end
+
+
       # Apply common socket options from +config+ to +sock+.
       #
       def self.apply_options(sock, config)
@@ -44,46 +78,52 @@ module OMQ
         sock = config.ffi ? klass.new(backend: :ffi) : klass.new
         sock.conflate = true if config.conflate && %w[pub radio].include?(config.type_name)
         apply_options(sock, config)
-        # --recv-maxsz: nil → 1 MiB default; 0 → explicitly unlimited; else → as-is.
+        apply_recv_maxsz(sock, config)
+        sock.identity         = config.identity   if config.identity
+        sock.router_mandatory = true if config.type_name == "router"
+        apply_compression(sock, config, config.type_name)
+        sock
+      end
+
+
+      # --recv-maxsz: nil → 1 MiB default; 0 → explicitly unlimited; else → as-is.
+      def self.apply_recv_maxsz(sock, config)
         sock.max_message_size =
           case config.recv_maxsz
           when nil then DEFAULT_RECV_MAXSZ
           when 0   then nil
           else          config.recv_maxsz
           end
-        sock.identity         = config.identity   if config.identity
-        sock.router_mandatory = true if config.type_name == "router"
-        sock
       end
 
 
       # Bind/connect +sock+ using URL strings from +config.binds+ / +config.connects+.
-      # +verbose+ is the integer verbosity level (0 = silent).
+      # +verbose+ gates logging (>= 1), +timestamps+ controls prefix.
       #
-      def self.attach(sock, config, verbose: 0)
+      def self.attach(sock, config, verbose: 0, timestamps: nil)
         config.binds.each do |url|
           sock.bind(url)
-          CLI::Term.write_attach(:bind, sock.last_endpoint, verbose) if verbose >= 1
+          CLI::Term.write_attach(:bind, sock.last_endpoint, timestamps) if verbose >= 1
         end
         config.connects.each do |url|
           sock.connect(url)
-          CLI::Term.write_attach(:connect, url, verbose) if verbose >= 1
+          CLI::Term.write_attach(:connect, url, timestamps) if verbose >= 1
         end
       end
 
 
       # Bind/connect +sock+ from an Array of Endpoint objects.
       # Used by PipeRunner, which works with structured endpoint lists.
-      # +verbose+ is the integer verbosity level (0 = silent).
+      # +verbose+ gates logging (>= 1), +timestamps+ controls prefix.
       #
-      def self.attach_endpoints(sock, endpoints, verbose: 0)
+      def self.attach_endpoints(sock, endpoints, verbose: 0, timestamps: nil)
         endpoints.each do |ep|
           if ep.bind?
             sock.bind(ep.url)
-            CLI::Term.write_attach(:bind, sock.last_endpoint, verbose) if verbose >= 1
+            CLI::Term.write_attach(:bind, sock.last_endpoint, timestamps) if verbose >= 1
           else
             sock.connect(ep.url)
-            CLI::Term.write_attach(:connect, ep.url, verbose) if verbose >= 1
+            CLI::Term.write_attach(:connect, ep.url, timestamps) if verbose >= 1
           end
         end
       end

data/lib/omq/cli/term.rb

@@ -15,15 +15,18 @@ module OMQ
       module_function
 
 
-      # Returns a stderr log line prefix. At verbose >= 4, prepends an
-      # ISO8601 UTC timestamp with µs precision so log traces become
-      # time-correlatable. Otherwise returns the empty string.
+      # Returns a stderr log line prefix with a UTC ISO8601 timestamp
+      # at the requested precision (:s/:ms/:us), or "" when nil.
       #
-      # @param verbose [Integer]
+      # @param timestamps [Symbol, nil] :s, :ms, :us, or nil (disabled)
       # @return [String]
-      def log_prefix(verbose)
-        return "" unless verbose && verbose >= 4
-        "#{Time.now.utc.strftime("%FT%T.%6N")}Z "
+      def log_prefix(timestamps)
+        case timestamps
+        when nil then ""
+        when :s  then "#{Time.now.utc.strftime("%FT%T")}Z "
+        when :ms then "#{Time.now.utc.strftime("%FT%T.%3N")}Z "
+        when :us then "#{Time.now.utc.strftime("%FT%T.%6N")}Z "
+        end
       end
 
 
@@ -31,18 +34,29 @@ module OMQ
       # trailing newline).
       #
       # @param event [OMQ::MonitorEvent]
-      # @param verbose [Integer]
+      # @param timestamps [Symbol, nil]
       # @return [String]
-      def format_event(event, verbose)
-        prefix = log_prefix(verbose)
+      def format_event(event, timestamps)
+        prefix = log_prefix(timestamps)
         case event.type
         when :message_sent
-          "#{prefix}omq: >> #{Formatter.preview(event.detail[:parts])}"
+          "#{prefix}omq: >> #{Formatter.preview(event.detail[:parts], wire_size: event.detail[:wire_size])}"
         when :message_received
-          "#{prefix}omq: << #{Formatter.preview(event.detail[:parts])}"
+          "#{prefix}omq: << #{Formatter.preview(event.detail[:parts], wire_size: event.detail[:wire_size])}"
+        when :zdict_sent
+          "#{prefix}omq: >> ZDICT (#{event.detail[:size]}B)"
+        when :zdict_received
+          "#{prefix}omq: << ZDICT (#{event.detail[:size]}B)"
         else
           ep     = event.endpoint ? " #{event.endpoint}" : ""
-          detail = event.detail ? " #{event.detail}" : ""
+          detail =
+            if event.detail.is_a?(Hash) && event.detail[:reason]
+              " (#{event.detail[:reason]})"
+            elsif event.detail
+              " #{event.detail}"
+            else
+              ""
+            end
           "#{prefix}omq: #{event.type}#{ep}#{detail}"
         end
       end
@@ -52,22 +66,22 @@ module OMQ
       #
       # @param kind [:bind, :connect]
       # @param url [String]
-      # @param verbose [Integer]
+      # @param timestamps [Symbol, nil]
       # @return [String]
-      def format_attach(kind, url, verbose)
+      def format_attach(kind, url, timestamps)
         verb = kind == :bind ? "Bound to" : "Connecting to"
-        "#{log_prefix(verbose)}omq: #{verb} #{url}"
+        "#{log_prefix(timestamps)}omq: #{verb} #{url}"
       end
 
 
       # Writes one formatted event line to +io+ (default $stderr).
       #
       # @param event [OMQ::MonitorEvent]
-      # @param verbose [Integer]
+      # @param timestamps [Symbol, nil]
       # @param io [#write] writable sink, default $stderr
       # @return [void]
-      def write_event(event, verbose, io: $stderr)
-        io.write("#{format_event(event, verbose)}\n")
+      def write_event(event, timestamps, io: $stderr)
+        io.write("#{format_event(event, timestamps)}\n")
       end
 
 
@@ -76,11 +90,11 @@ module OMQ
       #
       # @param kind [:bind, :connect]
       # @param url [String]
-      # @param verbose [Integer]
+      # @param timestamps [Symbol, nil]
       # @param io [#write]
       # @return [void]
-      def write_attach(kind, url, verbose, io: $stderr)
-        io.write("#{format_attach(kind, url, verbose)}\n")
+      def write_attach(kind, url, timestamps, io: $stderr)
+        io.write("#{format_attach(kind, url, timestamps)}\n")
       end
     end
   end

data/lib/omq/cli/version.rb

@@ -2,6 +2,6 @@
 
 module OMQ
   module CLI
-    VERSION = "0.12.3"
+    VERSION = "0.14.0"
   end
 end

data/lib/omq/cli.rb

@@ -197,6 +197,7 @@ module OMQ
       require "omq/rfc/scattergather"
       require "omq/rfc/channel"
       require "omq/rfc/p2p"
+      require "omq/rfc/zstd"
       require "async"
       require "json"
       require "console"
@@ -247,9 +248,6 @@ module OMQ
                    runner_class.new(config)
                  end
         runner.call(task)
-      rescue DecompressError => e
-        $stderr.puts "omq: #{e.message}"
-        exit 1
       rescue IO::TimeoutError, Async::TimeoutError
         $stderr.puts "omq: timeout" unless config.quiet
         exit 2