iriq 0.2.0 → 0.33.0

data/lib/iriq/cli.rb

@@ -1,3 +1,4 @@
+require "fileutils"
 require "json"
 require "optparse"
 require "stringio"
@@ -18,6 +19,8 @@ module Iriq
     LARGE_BATCH_THRESHOLD = 10
 
     USAGE = <<~TXT
+      iriq — find a URL's shape: the route template behind it (e.g. /users/{id}).
+
       Usage: iriq [options] <input>
              iriq [options] < text
              iriq cluster [options] [file]
@@ -26,29 +29,69 @@ module Iriq
       text via stdin.
 
       Sections (combine freely):
-        -n, --normalize       Shape-normalized form
+        -n, --normalize       Shape — variable parts become placeholders
+        -c, --canonical       Clean form — tidy scheme/host, keep the values
         -p, --parse           Parsed fields
+        -e, --explain         Annotated trace — per-segment notes about why
+                              each placeholder / canonical value was chosen
 
       Corpus + stats:
-            --corpus PATH     Load/create a JSON corpus; observe and save atomically.
-                              -n becomes corpus-informed once it has data.
+            --corpus PATH     Use a specific corpus file (overrides the default).
+                              Extension picks the backend: .db/.sqlite/.sqlite3
+                              are SQLite; anything else is JSON.
+        -C, --no-corpus       Disable corpus persistence for this invocation.
+                              Same as IRIQ_NO_CORPUS=1 in the environment.
+            --reset           Delete the corpus database (default path or the
+                              one resolved via --corpus / IRIQ_CORPUS) and exit.
+            --host MODE       Host-keying strategy for clustering:
+                              full (default), registrable (or reg) strips
+                              subdomains, none ignores host entirely.
             --stats           Print rolling aggregates
+            --reinfer         Replay the source-IRI log through the current
+                              classifier + reducers; rebuilds materialized
+                              views from scratch.
+            --propose-recognizers
+                              Scan observed values for shape patterns that
+                              recur enough to suggest a new Recognizer.
+                              Combine with --json for structured output.
+            --cross-host-shapes
+                              List route shapes that recur across
+                              multiple hosts. Combine with --min-hosts.
+            --activate-above F  With --propose-recognizers, promote every
+                              proposal at or above CONFIDENCE F into a
+                              live Recognizer on the corpus, then
+                              reinfer. Confidence integrates coverage
+                              and cross-host corroboration.
+
+      Environment:
+            IRIQ_CORPUS=PATH    Set the corpus path (overrides the default).
+            IRIQ_NO_CORPUS=1    Disable the default corpus (equivalent to -C).
+
+      Thresholds (apply to --propose-recognizers / --cross-host-shapes):
+            --min-observations N  proposal noise floor (default 20)
+            --min-coverage F      proposal coverage floor (default 0.7)
+            --min-hosts N         proposal: minimum hosts (default 1);
+                                  cross-host-shapes: minimum hosts to
+                                  list (default 2)
 
       Other:
         -h, --help            Show this message
         -j, --json            Emit JSON instead of human-readable output
-        -N, --no-hints        Use {integer_id} placeholders instead of {user_id}
+        -J, --ndjson          Newline-delimited JSON (one object per line). Implies --json.
+        -N, --no-hints        Use {integer} placeholders instead of {user_id}
             --no-scheme-less  Skip foo.com/path extraction (explicit-scheme only)
         -V, --version         Print version
 
       Subcommands:
         cluster [file]        Force cluster view (default for ≥10 IRIs anyway)
+        completion <shell>    Print shell completion script (bash | zsh)
 
       Examples:
         iriq foo.com/users/456
         iriq -n https://foo.com/users/123
         iriq ./access.log                     # auto-detect file → extract URLs
         cat README.md | iriq -n               # one normalized URL per line
+        tail -f access.log | iriq -J          # live stream → NDJSON per IRI
         cat README.md | iriq --corpus c.json
     TXT
 
@@ -62,11 +105,22 @@ module Iriq
 
     # Returns an integer exit code.
     def run(argv)
+      # Pre-scan so an error during option parsing can still honor --json.
+      # Re-set authoritatively from opts once parsing succeeds.
+      @json = json_requested?(argv)
       args, opts = parse_options(argv)
+      @json = opts[:json]
 
       return print_usage(stdout, 0) if opts[:help]
       return print_version          if opts[:version]
 
+      # `iriq completion <shell>` short-circuits — no corpus, no IRI input,
+      # just emit the script bundled with the gem.
+      if args.first == "completion"
+        args.shift
+        return cmd_completion(args)
+      end
+
       explicit_cluster = (args.first == "cluster")
       args.shift if explicit_cluster
 
@@ -79,11 +133,25 @@ module Iriq
       batch_mode = explicit_cluster || positional_is_file ||
                    (args.empty? && piped_stdin?)
 
-      return print_usage(stdout, 0) if args.empty? && !batch_mode
+      # --reset short-circuits: delete the resolved corpus file (+ sidecars)
+      # and exit. Resolves through the same precedence chain as the normal
+      # path so `--reset --corpus other.db` and `IRIQ_CORPUS=… --reset` Just Work.
+      if opts[:reset]
+        return cmd_reset(opts)
+      end
+
+      return print_usage(stdout, 0) if args.empty? && !batch_mode && !opts[:reinfer] && !opts[:propose] && !opts[:cross_host_shapes]
 
-      corpus = opts[:corpus] ? load_corpus(opts[:corpus]) : nil
+      corpus_path = resolve_corpus_path(opts)
+      corpus = corpus_path ? load_corpus(corpus_path, host_strategy: opts[:host_strategy], announce_create: true) : nil
 
-      code = if batch_mode
+      code = if opts[:reinfer]
+        cmd_reinfer(corpus, opts)
+      elsif opts[:propose]
+        cmd_propose(corpus, opts)
+      elsif opts[:cross_host_shapes]
+        cmd_cross_host_shapes(corpus, opts)
+      elsif batch_mode
         cmd_batch(args, opts, corpus, explicit_cluster: explicit_cluster)
       elsif opts[:stats]
         cmd_stats(corpus, opts)
@@ -91,14 +159,12 @@ module Iriq
         cmd_summary(args, opts, corpus)
       end
 
-      corpus.save(opts[:corpus]) if corpus && opts[:corpus]
+      corpus.save(corpus_path) if corpus && corpus_path
       code
     rescue Iriq::ParseError => e
-      stderr.puts "iriq: parse error: #{e.message}"
-      2
+      emit_error("parse_error", e.message, 2, human: "iriq: parse error: #{e.message}")
     rescue OptionParser::ParseError => e
-      stderr.puts "iriq: #{e.message}"
-      1
+      emit_error("option_error", e.message, 1)
     end
 
     def parseable_iri?(input)
@@ -113,22 +179,49 @@ module Iriq
     def parse_options(argv)
       opts = {
         json:        false,
+        ndjson:      false,
         help:        false,
         version:     false,
         hints:       true,
         sections:    [],
-        corpus:      nil,
-        stats:       false,
-        scheme_less: true,
+        corpus:        nil,
+        no_corpus:     false,
+        reset:         false,
+        stats:         false,
+        reinfer:       false,
+        propose:       false,
+        propose_min_obs:      nil,
+        propose_min_coverage: nil,
+        # --min-hosts is generic: it applies to both --propose-recognizers
+        # (proposal threshold) and --cross-host-shapes (cross-host
+        # recurrence threshold).
+        min_hosts:            nil,
+        activate_above:       nil,
+        cross_host_shapes:    false,
+        scheme_less:   true,
+        host_strategy: :full,
       }
       parser = OptionParser.new do |o|
         o.on("-p", "--parse")        { opts[:sections] << :parse }
         o.on("-n", "--normalize")    { opts[:sections] << :normalize }
+        o.on("-c", "--canonical")    { opts[:sections] << :canonical }
+        o.on("-e", "--explain")      { opts[:sections] << :explain }
         o.on("-j", "--json")         { opts[:json]    = true }
+        o.on("-J", "--ndjson")       { opts[:json]    = true; opts[:ndjson] = true }
         o.on("--[no-]hints")         { |v| opts[:hints] = v }
         o.on("-N")                   { opts[:hints] = false }
         o.on("--corpus PATH")        { |v| opts[:corpus] = v }
+        o.on("-C", "--no-corpus")    { opts[:no_corpus] = true }
+        o.on("--reset")              { opts[:reset] = true }
+        o.on("--host MODE")          { |v| opts[:host_strategy] = host_strategy_arg(v) }
         o.on("--stats")              { opts[:stats]   = true }
+        o.on("--reinfer")            { opts[:reinfer] = true }
+        o.on("--propose-recognizers") { opts[:propose] = true }
+        o.on("--min-observations N", Integer) { |v| opts[:propose_min_obs]      = v }
+        o.on("--min-coverage F", Float)       { |v| opts[:propose_min_coverage] = v }
+        o.on("--min-hosts N", Integer)        { |v| opts[:min_hosts]           = v }
+        o.on("--activate-above F", Float)     { |v| opts[:activate_above]       = v }
+        o.on("--cross-host-shapes")           { opts[:cross_host_shapes]        = true }
         o.on("--[no-]scheme-less")   { |v| opts[:scheme_less] = v }
         o.on("-h", "--help")         { opts[:help]    = true }
         o.on("-V", "--version")      { opts[:version] = true }
@@ -149,8 +242,89 @@ module Iriq
       end
     end
 
-    def load_corpus(path)
-      Corpus.open(path)
+    def load_corpus(path, host_strategy: :full, announce_create: false)
+      if announce_create && !File.exist?(path)
+        FileUtils.mkdir_p(File.dirname(path))
+        stderr.puts "iriq: created corpus at #{path} (disable with --no-corpus or IRIQ_NO_CORPUS=1)"
+      end
+      Corpus.open(path, host_strategy: host_strategy)
+    end
+
+    # Resolve the corpus file the CLI should use. Precedence:
+    #   1. --corpus PATH       — explicit always wins (you asked for it)
+    #   2. --no-corpus / IRIQ_NO_CORPUS=1 — opt out of the default
+    #   3. IRIQ_CORPUS=PATH    — env override of the default location
+    #   4. default_corpus_path — platform-aware location
+    def resolve_corpus_path(opts)
+      return opts[:corpus] if opts[:corpus]
+      return nil if opts[:no_corpus] || env_corpus_disabled?
+
+      env_path = ENV["IRIQ_CORPUS"].to_s
+      return env_path unless env_path.empty?
+
+      default_corpus_path
+    end
+
+    # Platform-aware default. XDG-honoring on Linux + BSD, Apple-style on
+    # macOS, %LOCALAPPDATA% on Windows. Same logic in Rust so both
+    # runtimes share the same default.db.
+    def default_corpus_path
+      base = if (xdg = ENV["XDG_DATA_HOME"].to_s) && !xdg.empty?
+        File.join(xdg, "iriq")
+      elsif RUBY_PLATFORM =~ /darwin/
+        File.expand_path("~/Library/Application Support/iriq")
+      elsif RUBY_PLATFORM =~ /mingw|mswin|cygwin/
+        File.join(ENV["LOCALAPPDATA"].to_s.empty? ? File.expand_path("~/AppData/Local") : ENV["LOCALAPPDATA"], "iriq")
+      else
+        File.expand_path("~/.local/share/iriq")
+      end
+      File.join(base, "default.db")
+    end
+
+    def env_corpus_disabled?
+      v = ENV["IRIQ_NO_CORPUS"].to_s.downcase
+      !v.empty? && v != "0" && v != "false" && v != "no"
+    end
+
+    def cmd_reset(opts)
+      path = resolve_reset_path(opts)
+      unless path
+        return emit_error("missing_argument", "no corpus path to reset (use --corpus PATH or unset --no-corpus)", 1)
+      end
+      removed = []
+      [path, "#{path}-wal", "#{path}-shm", "#{path}.tmp"].each do |p|
+        if File.exist?(p)
+          File.delete(p)
+          removed << p
+        end
+      end
+      if removed.empty?
+        stderr.puts "iriq: no corpus to reset at #{path}"
+      else
+        stderr.puts "iriq: reset corpus at #{path}"
+      end
+      0
+    end
+
+    # --reset honors --corpus / IRIQ_CORPUS even when --no-corpus is set —
+    # the user is explicitly addressing a stored file, not the runtime state.
+    def resolve_reset_path(opts)
+      return opts[:corpus] if opts[:corpus]
+      env_path = ENV["IRIQ_CORPUS"].to_s
+      return env_path unless env_path.empty?
+      default_corpus_path
+    end
+
+    # Accept `--host=reg` as a short alias for the `registrable` mode.
+    HOST_STRATEGY_ALIASES = {
+      "full" => :full, "registrable" => :registrable, "reg" => :registrable, "none" => :none,
+    }.freeze
+
+    def host_strategy_arg(value)
+      mode = HOST_STRATEGY_ALIASES[value.to_s.downcase]
+      raise OptionParser::InvalidArgument, "--host: expected full|registrable|reg|none, got #{value.inspect}" unless mode
+
+      mode
     end
 
     def print_usage(io, code)
@@ -171,9 +345,13 @@ module Iriq
 
       data = {}
       data[:parse]     = identifier_hash(iri) if sections.include?(:parse)
+      data[:canonical] = iri.canonical         if sections.include?(:canonical)
       if sections.include?(:normalize)
         data[:normalize] = corpus ? corpus.normalize(iri) : Normalizer.normalize_identifier(iri, hints: opts[:hints])
       end
+      if sections.include?(:explain)
+        data[:explain] = Trace.for(iri, hints: opts[:hints])
+      end
 
       if opts[:json]
         payload = sections.size == 1 ? data.values.first : data
@@ -190,12 +368,21 @@ module Iriq
     # corpus is ephemeral unless --corpus was given.
     def cmd_batch(args, opts, corpus, explicit_cluster: false)
       corpus ||= Corpus.new
+
+      # Per-IRI sections (-n/-p/-c/-e) are independent line to line, so we
+      # stream: read input lazily, extract per line, and emit each IRI as it
+      # arrives (flushed for live `tail -f | iriq -n` pipelines). The aggregate
+      # views below — stats, clusters, the deduped URL list — need the whole
+      # input, so they slurp.
+      if opts[:sections].any?
+        emit_per_iri_sections(lazy_iris(args.first, opts), opts, corpus)
+        return 0
+      end
+
       iris = extract_text(read_text(args.first), opts)
       corpus.batch { iris.each { |iri| corpus.observe(iri) } }
 
-      if opts[:sections].any?
-        emit_per_iri_sections(iris, opts)
-      elsif opts[:stats]
+      if opts[:stats]
         emit_stats(corpus, opts)
       elsif explicit_cluster || iris.size >= LARGE_BATCH_THRESHOLD
         # Either the user asked for clusters explicitly, or the input is
@@ -207,36 +394,68 @@ module Iriq
       0
     end
 
-    # Emit the requested sections (parse/normalize/explain) for each
-    # extracted IRI. -n alone is the cleanest case: one line per URL.
-    def emit_per_iri_sections(iris, opts)
+    # Lazily yield IRIs from the input, one input line at a time, so an
+    # unbounded stream flows through without being buffered in full. Matches
+    # whole-text extraction exactly: a candidate never spans a newline
+    # (URL_CHAR_CLASS excludes whitespace) and `extract` does not dedup.
+    def lazy_iris(path, opts)
+      extractor = Extractor.new(scheme_less: opts[:scheme_less])
+      input_lines(path).lazy.flat_map { |line| extractor.extract(line) }
+    end
+
+    def input_lines(path)
+      if path.nil? || path == "-"
+        stdin.each_line
+      else
+        File.foreach(path)
+      end
+    end
+
+    # Emit the requested sections (parse/normalize/explain) for each extracted
+    # IRI, observing each into `corpus` as it passes. `iris` may be a lazy
+    # enumerator; human and NDJSON output stream (flushed per IRI) while a single
+    # JSON array must be materialized. -n alone is the cleanest case: one line
+    # per URL.
+    def emit_per_iri_sections(iris, opts, corpus)
       sections = opts[:sections]
-      payloads = iris.map { |iri| section_payload(iri, sections, opts) }
 
-      if opts[:json]
+      # A wrapping JSON array can't be emitted incrementally — collect it
+      # (force the lazy enumerator to a real Array so emit_json sees an array).
+      if opts[:json] && !opts[:ndjson]
+        payloads = iris.map { |iri| corpus.observe(iri); section_payload(iri, sections, opts) }.to_a
         out = sections.size == 1 ? payloads.map(&:values).flatten(1) : payloads
-        stdout.puts JSON.generate(out)
-      elsif sections == [:normalize]
-        # Most common case — keep it tight: one URL per line, no headers.
-        payloads.each { |p| stdout.puts p[:normalize] }
-      else
-        payloads.each_with_index do |p, i|
+        return emit_json(out, opts)
+      end
+
+      iris.each_with_index do |iri, i|
+        corpus.observe(iri)
+        p = section_payload(iri, sections, opts)
+        if opts[:ndjson]
+          items = sections.size == 1 ? p.values : [p]
+          items.each { |item| stdout.puts JSON.generate(item) }
+        elsif sections == [:normalize] || sections == [:canonical]
+          # Most common case — keep it tight: one URL per line, no headers.
+          stdout.puts p[sections.first]
+        else
           stdout.puts if i > 0
-          stdout.puts "# #{iris[i].canonical}"
+          stdout.puts "# #{iri.canonical}"
           sections.each_with_index do |sec, j|
             stdout.puts if j > 0  # blank line between sections within one IRI
             case sec
             when :parse     then emit_parse_human(p[:parse])
+            when :canonical then stdout.puts p[:canonical]
             when :normalize then stdout.puts p[:normalize]
             end
           end
         end
+        stdout.flush
       end
     end
 
     def section_payload(iri, sections, opts)
       data = {}
       data[:parse]     = identifier_hash(iri)                                       if sections.include?(:parse)
+      data[:canonical] = iri.canonical                                              if sections.include?(:canonical)
       data[:normalize] = Normalizer.normalize_identifier(iri, hints: opts[:hints])  if sections.include?(:normalize)
       data
     end
@@ -260,7 +479,7 @@ module Iriq
       sorted = counts.sort_by { |k, c| [-c, first[k]] }
 
       if opts[:json]
-        stdout.puts JSON.generate(sorted.map { |k, c| { iri: k, count: c } })
+        emit_json(sorted.map { |k, c| { iri: k, count: c } }, opts)
       elsif sorted.all? { |_, c| c == 1 }
         sorted.each { |k, _| stdout.puts k }
       else
@@ -275,9 +494,153 @@ module Iriq
       0
     end
 
+    # --propose-recognizers: scan observed values for prefix patterns
+    # that recur enough to suggest a new Recognizer. Prints one block
+    # per proposal in human mode, or a JSON array under --json. With
+    # --activate-above F, every proposal at or above coverage F is
+    # promoted to a live Recognizer on the corpus's classifier and the
+    # corpus reinfers to apply the new classifier to existing
+    # observations.
+    def cmd_propose(corpus, opts)
+      return missing("--corpus") unless corpus
+
+      kwargs = {}
+      kwargs[:min_observations] = opts[:propose_min_obs]      if opts[:propose_min_obs]
+      kwargs[:min_coverage]     = opts[:propose_min_coverage] if opts[:propose_min_coverage]
+      kwargs[:min_hosts]        = opts[:min_hosts]            if opts[:min_hosts]
+
+      if opts[:activate_above]
+        activated = corpus.activate_proposals_above(opts[:activate_above], **kwargs)
+        if activated.empty?
+          stdout.puts "no proposals at or above coverage #{opts[:activate_above]}"
+        else
+          activated.each do |r|
+            stdout.puts "activated: #{r.type} (#{r.prefix})"
+          end
+        end
+        return 0
+      end
+
+      proposals = corpus.propose_recognizers(**kwargs)
+
+      if opts[:json]
+        stdout.puts JSON.generate(proposals.map(&:to_h))
+        return 0
+      end
+
+      if proposals.empty?
+        stdout.puts "no recognizer proposals (#{corpus.observed_iri_count} observations scanned)"
+        return 0
+      end
+
+      proposals.each_with_index do |p, i|
+        stdout.puts if i > 0
+        stdout.puts "proposal: #{p.suggested_type} (#{p.prefix})"
+        stdout.puts "  strategy:    #{p.strategy}"
+        stdout.puts "  coverage:    #{format('%.2f', p.coverage)}"
+        stdout.puts "  confidence:  #{format('%.2f', p.confidence)}"
+        stdout.puts "  observations: #{p.observation_count}"
+        stdout.puts "  hosts:       #{p.hosts.to_a.sort.join(', ')}"
+        stdout.puts "  positions:   #{p.positions.size}"
+        stdout.puts "  samples:     #{p.sample_values.first(3).join(', ')}"
+      end
+      0
+    end
+
+    # --reinfer: drop the materialized views in the corpus and replay the
+    # source-IRI log through the current classifier + reducers. Prints a
+    # short before/after summary so the user can see what changed.
+    def cmd_reinfer(corpus, _opts)
+      return missing("--corpus") unless corpus
+
+      n      = corpus.observed_iri_count
+      before = corpus.size
+      corpus.reinfer
+      after  = corpus.size
+
+      stdout.puts "reinferred #{n} observation#{n == 1 ? '' : 's'}: " \
+                  "#{before} → #{after} cluster#{after == 1 ? '' : 's'}"
+      0
+    end
+
+    # `completion <shell>` — emit the bundled shell-completion script.
+    # Scripts live in completions/{iriq.bash,_iriq} alongside the
+    # gem; Homebrew installs them automatically, but the user can also do
+    # `source <(iriq completion bash)` in their shell rc. The Rust CLI
+    # inlines the same scripts — keep them byte-identical (parity-tested).
+    COMPLETIONS_DIR = File.expand_path("../../completions", __dir__).freeze
+    COMPLETION_FILES = {
+      "bash" => File.join(COMPLETIONS_DIR, "iriq.bash"),
+      "zsh"  => File.join(COMPLETIONS_DIR, "_iriq"),
+    }.freeze
+
+    def cmd_completion(args)
+      shell = args.first || default_shell
+      path  = COMPLETION_FILES[shell]
+      unless path
+        return emit_error("unknown_shell", "unknown shell #{shell.inspect} (try bash or zsh)", 1)
+      end
+      stdout.write(File.read(path))
+      0
+    end
+
+    def default_shell
+      shell = ENV["SHELL"].to_s
+      shell.empty? ? "bash" : File.basename(shell).sub(/\.exe\z/, "")
+    end
+
+    # --cross-host-shapes: list route shapes that recur across multiple
+    # hosts in the corpus. One block per shape in human mode, JSON array
+    # under --json. Tunable via --min-hosts (default 2).
+    def cmd_cross_host_shapes(corpus, opts)
+      return missing("--corpus") unless corpus
+
+      kwargs = {}
+      kwargs[:min_hosts] = opts[:min_hosts] if opts[:min_hosts]
+      shapes = corpus.cross_host_shapes(**kwargs)
+
+      if opts[:json]
+        stdout.puts JSON.generate(shapes.map(&:to_h))
+        return 0
+      end
+
+      if shapes.empty?
+        stdout.puts "no cross-host shapes (#{corpus.size} cluster#{corpus.size == 1 ? '' : 's'} scanned)"
+        return 0
+      end
+
+      shapes.each do |s|
+        host_list = s.hosts.to_a.sort.join(", ")
+        stdout.puts "#{s.shape}  (#{s.host_count} host#{s.host_count == 1 ? '' : 's'}: #{host_list})  obs=#{s.observation_count}"
+      end
+      0
+    end
+
     def missing(name)
-      stderr.puts "iriq: missing argument <#{name}>"
-      1
+      emit_error("missing_argument", "missing argument <#{name}>", 1)
+    end
+
+    # Detect whether JSON output was requested by scanning raw argv. Used
+    # before option parsing completes (or when it fails) so errors can still
+    # honor --json. Handles bundled short flags like -nj.
+    def json_requested?(argv)
+      argv.any? do |a|
+        a == "--json" || a == "--ndjson" ||
+          (a.start_with?("-") && !a.start_with?("--") && a.match?(/[jJ]/))
+      end
+    end
+
+    # Emit an error to stderr and return its exit code. Under --json/--ndjson
+    # the error is a structured envelope ({"error":{"code","message"}}) so
+    # agents and pipelines get parseable output on the failure path; otherwise
+    # the plain "iriq: <human>" line (human defaults to "iriq: <message>").
+    def emit_error(code, message, exit_code, human: nil)
+      if @json
+        stderr.puts JSON.generate(error: { code: code, message: message })
+      else
+        stderr.puts(human || "iriq: #{message}")
+      end
+      exit_code
     end
 
     def read_input(path)
@@ -314,6 +677,19 @@ module Iriq
       }.reject { |_, v| v.nil? || (v.respond_to?(:empty?) && v.empty?) }
     end
 
+    # Emit a JSON payload to stdout. When --ndjson is set and the payload is
+    # an Array, write one object per line (newline-delimited JSON) instead of
+    # one wrapping array — friendlier for `jq -c`, streaming pipelines, and
+    # log ingest tools. Non-array payloads (single objects) emit the same
+    # under both flags.
+    def emit_json(payload, opts)
+      if opts[:ndjson] && payload.is_a?(Array)
+        payload.each { |item| stdout.puts JSON.generate(item) }
+      else
+        stdout.puts JSON.generate(payload)
+      end
+    end
+
     def emit_sections(data, sections)
       multi = sections.size > 1
       sections.each_with_index do |sec, i|
@@ -321,41 +697,115 @@ module Iriq
         stdout.puts "# #{sec}" if multi
         case sec
         when :parse     then emit_parse_human(data[:parse])
+        when :canonical then stdout.puts data[:canonical]
         when :normalize then stdout.puts data[:normalize]
+        when :explain   then emit_explain_human(data[:explain])
         end
       end
     end
 
+    # Render the trace hash as a vertically-aligned per-segment table.
+    # path rows first, then query rows.
+    def emit_explain_human(trace)
+      stdout.puts trace[:normalized]
+      emit_trace_section("path",  trace[:path])
+      emit_trace_section("query", trace[:query]) if trace[:query]
+    end
+
+    def emit_trace_section(label, rows)
+      return if rows.nil? || rows.empty?
+
+      stdout.puts
+      stdout.puts "#{label}:"
+      name_width  = rows.map { |r| trace_label(r).length }.max
+      type_width  = rows.map { |r| r[:type].to_s.length }.max
+      out_width   = rows.map { |r| r[:output].to_s.length }.max
+      rows.each do |r|
+        stdout.puts "  #{trace_label(r).ljust(name_width)}  #{r[:type].to_s.ljust(type_width)}  #{r[:output].to_s.ljust(out_width)}#{format_notes(r[:notes])}"
+      end
+    end
+
+    def trace_label(row)
+      # Path rows have :value, query rows have :name=:value.
+      row[:name] ? "#{row[:name]}=#{row[:value]}" : row[:value].to_s
+    end
+
+    def format_notes(notes)
+      return "" if notes.nil? || notes.empty?
+      "  (" + notes.join("; ") + ")"
+    end
+
+    # Render the compact identifier_hash. Keys/values are already filtered;
+    # array/hash values get .inspect, everything else .to_s.
     def emit_parse_human(h)
-      stdout.puts "original:      #{h[:original]}"
-      stdout.puts "kind:          #{h[:kind]}"
-      stdout.puts "scheme:        #{h[:scheme]}" if h[:scheme]
-      stdout.puts "host:          #{h[:host]}"   if h[:host]
-      stdout.puts "port:          #{h[:port]}"   if h[:port]
-      stdout.puts "path_segments: #{h[:path_segments].inspect}" if h[:kind] == :url
-      stdout.puts "query_params:  #{h[:query_params].inspect}" if h[:query_params] && !h[:query_params].empty?
-      stdout.puts "fragment:      #{h[:fragment]}" if h[:fragment]
-      stdout.puts "nss:           #{h[:nss]}"      if h[:nss]
-      stdout.puts "canonical:     #{h[:canonical]}"
+      h.each do |key, value|
+        rendered = value.is_a?(Array) || value.is_a?(Hash) ? value.inspect : value.to_s
+        stdout.puts "#{"#{key}:".ljust(15)}#{rendered}"
+      end
     end
 
     def emit_clusters(clusters, opts)
-      sorted = clusters.sort_by { |c| -c.count }
+      # Stable sort: equal-count clusters keep first-seen order. Ruby's
+      # sort_by is unstable, so ties need the explicit index tie-break to
+      # match the Rust CLI's (stable) sort.
+      sorted = clusters.sort_by.with_index { |c, i| [-c.count, i] }
 
       if opts[:json]
-        stdout.puts JSON.generate(sorted.map(&:to_h))
+        emit_json(sorted.map(&:to_h), opts)
       else
         sorted.each_with_index do |c, i|
           stdout.puts if i > 0
           host  = c.host || "(urn)"
           shape = opts[:hints] ? c.shape : raw_shape_for(c)
           stdout.puts "[#{c.count}] #{host}  #{shape}"
-          c.examples.first(3).each { |e| stdout.puts "    #{e.canonical}" }
-          stdout.puts "    + #{c.count - 3} more" if c.count > 3
+          examples = c.examples.first(3)
+          examples.each { |e| stdout.puts "    #{e.canonical}" }
+          remaining = c.count - examples.size
+          stdout.puts "    + #{remaining} more" if remaining.positive?
+          emit_param_summary(c)
+        end
+      end
+    end
+
+    # One line per param: type, confidence, range (numeric), cardinality,
+    # presence. `page  integer  conf 0.87  1..100  avg 50.5  (10 distinct, 100%)`
+    def emit_param_summary(cluster)
+      rows = cluster.param_summary
+      return if rows.empty?
+
+      width = rows.map { |r| r[:name].length }.max
+      rows.each do |r|
+        bits = ["#{r[:type]}"]
+        bits << "conf #{format_conf(r[:confidence])}" if r[:confidence]
+        if r[:min] && r[:max]
+          bits << format_range(r[:min], r[:max])
+          bits << "avg #{format_num(r[:avg])}" if r[:avg]
         end
+        bits << "(#{r[:cardinality]} distinct, #{format_pct(r[:presence])})"
+        stdout.puts "    #{r[:name].to_s.ljust(width)}  #{bits.join('  ')}"
       end
     end
 
+    # Two-decimal confidence, e.g. 0.87. Matches the Rust CLI formatting.
+    def format_conf(conf)
+      format("%.2f", conf)
+    end
+
+    def format_range(lo, hi)
+      "#{format_num(lo)}..#{format_num(hi)}"
+    end
+
+    def format_num(n)
+      return n.to_s if n.is_a?(Integer)
+      whole = n.to_i
+      return whole.to_s if whole == n
+      n.round(2).to_s
+    end
+
+    def format_pct(frac)
+      "#{(frac * 100).round}%"
+    end
+
     def raw_shape_for(cluster)
       example = cluster.examples.first
       return cluster.shape unless example
@@ -388,7 +838,9 @@ module Iriq
     end
 
     def top(hash)
-      hash.sort_by { |_, n| -n }.first(TOP_N_STATS).to_h
+      # Lex tie-break on equal counts — keeps --stats output deterministic
+      # and Ruby ↔ Rust parity stable regardless of insertion order.
+      hash.sort_by { |k, n| [-n, k] }.first(TOP_N_STATS).to_h
     end
   end
 end