iriq 0.33.0 → 0.35.0

data/lib/iriq/cli.rb

@@ -25,8 +25,8 @@ module Iriq
              iriq [options] < text
              iriq cluster [options] [file]
 
-      <input> may be an IRI, a file path (extracted automatically), or piped
-      text via stdin.
+      <input> may be an IRI, a file to extract IRIs from (an existing file wins
+      unless the argument contains ://), or piped text via stdin.
 
       Sections (combine freely):
         -n, --normalize       Shape — variable parts become placeholders
@@ -34,6 +34,7 @@ module Iriq
         -p, --parse           Parsed fields
         -e, --explain         Annotated trace — per-segment notes about why
                               each placeholder / canonical value was chosen
+                              (mechanical rules only, even with a corpus)
 
       Corpus + stats:
             --corpus PATH     Use a specific corpus file (overrides the default).
@@ -41,11 +42,14 @@ module Iriq
                               are SQLite; anything else is JSON.
         -C, --no-corpus       Disable corpus persistence for this invocation.
                               Same as IRIQ_NO_CORPUS=1 in the environment.
-            --reset           Delete the corpus database (default path or the
-                              one resolved via --corpus / IRIQ_CORPUS) and exit.
+            --reset           Delete the corpus (default path or the one
+                              resolved via --corpus / IRIQ_CORPUS), its SQLite
+                              sidecars and JSON temp files, and exit.
             --host MODE       Host-keying strategy for clustering:
                               full (default), registrable (or reg) strips
-                              subdomains, none ignores host entirely.
+                              subdomains, none ignores host entirely. Keys
+                              IRIs as they're observed (-C included) and on
+                              --reinfer; existing clusters keep their keys.
             --stats           Print rolling aggregates
             --reinfer         Replay the source-IRI log through the current
                               classifier + reducers; rebuilds materialized
@@ -78,20 +82,23 @@ module Iriq
         -h, --help            Show this message
         -j, --json            Emit JSON instead of human-readable output
         -J, --ndjson          Newline-delimited JSON (one object per line). Implies --json.
+                              Streams per IRI only with -n/-p/-c/-e; alone, it
+                              prints the URL list or clusters at end of input.
         -N, --no-hints        Use {integer} placeholders instead of {user_id}
             --no-scheme-less  Skip foo.com/path extraction (explicit-scheme only)
         -V, --version         Print version
 
       Subcommands:
-        cluster [file]        Force cluster view (default for ≥10 IRIs anyway)
+        cluster [file]        Observe file (or stdin), then show every cluster
+                              in the corpus (default view for ≥10 IRIs anyway)
         completion <shell>    Print shell completion script (bash | zsh)
 
       Examples:
         iriq foo.com/users/456
         iriq -n https://foo.com/users/123
-        iriq ./access.log                     # auto-detect file → extract URLs
+        iriq access.log                       # extract URLs (request paths have no host)
         cat README.md | iriq -n               # one normalized URL per line
-        tail -f access.log | iriq -J          # live stream → NDJSON per IRI
+        tail -f app.log | iriq -nJ            # live stream → NDJSON per IRI
         cat README.md | iriq --corpus c.json
     TXT
 
@@ -124,11 +131,10 @@ module Iriq
       explicit_cluster = (args.first == "cluster")
       args.shift if explicit_cluster
 
-      # Auto-detect: a positional argument that isn't parseable as an IRI
-      # but IS an existing file gets treated as a file to extract from. This
-      # is what makes `iriq ./access.log` and `iriq /var/log/foo.log` Just
-      # Work without a separate --extract flag.
-      positional_is_file = args.first && File.file?(args.first) && !parseable_iri?(args.first)
+      # A positional that names an existing file is read as a file, so
+      # `iriq access.log` works without ./ (it also parses as a host). An
+      # argument containing "://" is always an IRI.
+      positional_is_file = args.first && !args.first.include?("://") && File.file?(args.first)
 
       batch_mode = explicit_cluster || positional_is_file ||
                    (args.empty? && piped_stdin?)
@@ -140,6 +146,11 @@ module Iriq
         return cmd_reset(opts)
       end
 
+      # Before any corpus is opened, so a typo doesn't create one.
+      if (missing = missing_input_file(args.first, explicit_cluster))
+        raise InputError.new("file_not_found", "no such file: #{missing}")
+      end
+
       return print_usage(stdout, 0) if args.empty? && !batch_mode && !opts[:reinfer] && !opts[:propose] && !opts[:cross_host_shapes]
 
       corpus_path = resolve_corpus_path(opts)
@@ -165,6 +176,20 @@ module Iriq
       emit_error("parse_error", e.message, 2, human: "iriq: parse error: #{e.message}")
     rescue OptionParser::ParseError => e
       emit_error("option_error", e.message, 1)
+    rescue InputError => e
+      emit_error(e.code, e.message, 1)
+    rescue Iriq::CorpusError => e
+      emit_error("corpus_error", e.message, 1)
+    end
+
+    # Input iriq can't read. `code` is the JSON error envelope's code.
+    class InputError < Iriq::Error
+      attr_reader :code
+
+      def initialize(code, message)
+        super(message)
+        @code = code
+      end
     end
 
     def parseable_iri?(input)
@@ -176,6 +201,15 @@ module Iriq
 
     private
 
+    # The argument iriq would have read as a file but can't find: anything
+    # after `cluster`, or a /, ./, ../ path that isn't an IRI.
+    def missing_input_file(arg, explicit_cluster)
+      return nil if arg.nil? || arg == "-" || File.file?(arg)
+      return arg if explicit_cluster
+
+      arg if arg.start_with?("/", "./", "../") && !parseable_iri?(arg)
+    end
+
     def parse_options(argv)
       opts = {
         json:        false,
@@ -243,11 +277,19 @@ module Iriq
     end
 
     def load_corpus(path, host_strategy: :full, announce_create: false)
-      if announce_create && !File.exist?(path)
-        FileUtils.mkdir_p(File.dirname(path))
-        stderr.puts "iriq: created corpus at #{path} (disable with --no-corpus or IRIQ_NO_CORPUS=1)"
+      creating = announce_create && !File.exist?(path)
+      if creating
+        begin
+          FileUtils.mkdir_p(File.dirname(path))
+        rescue SystemCallError => e
+          raise CorpusError, "corpus #{path}: #{Iriq.os_error_message(e)}"
+        end
       end
-      Corpus.open(path, host_strategy: host_strategy)
+      corpus = Corpus.open(path, host_strategy: host_strategy)
+      # Only announce once the corpus actually exists — mkdir_p can succeed
+      # (e.g. the dir is there but read-only) while the open still fails.
+      stderr.puts "iriq: created corpus at #{path} (disable with --no-corpus or IRIQ_NO_CORPUS=1)" if creating
+      corpus
     end
 
     # Resolve the corpus file the CLI should use. Precedence:
@@ -292,7 +334,7 @@ module Iriq
         return emit_error("missing_argument", "no corpus path to reset (use --corpus PATH or unset --no-corpus)", 1)
       end
       removed = []
-      [path, "#{path}-wal", "#{path}-shm", "#{path}.tmp"].each do |p|
+      [path, "#{path}-wal", "#{path}-shm", "#{path}.tmp", *json_temp_files(path)].each do |p|
         if File.exist?(p)
           File.delete(p)
           removed << p
@@ -306,6 +348,16 @@ module Iriq
       0
     end
 
+    # The JSON writer's per-save temp files, PATH.<pid>.<n>.tmp
+    # (Storage.write_atomically); a save killed before its rename leaves one.
+    def json_temp_files(path)
+      dir = File.dirname(path)
+      return [] unless Dir.exist?(dir)
+
+      pattern = /\A#{Regexp.escape(File.basename(path))}\.\d+\.\d+\.tmp\z/
+      Dir.children(dir).grep(pattern).map { |name| File.join(dir, name) }
+    end
+
     # --reset honors --corpus / IRIQ_CORPUS even when --no-corpus is set —
     # the user is explicitly addressing a stored file, not the runtime state.
     def resolve_reset_path(opts)
@@ -322,7 +374,9 @@ module Iriq
 
     def host_strategy_arg(value)
       mode = HOST_STRATEGY_ALIASES[value.to_s.downcase]
-      raise OptionParser::InvalidArgument, "--host: expected full|registrable|reg|none, got #{value.inspect}" unless mode
+      # OptionParser prefixes the switch itself ("--host bogus" / "--host=bogus"),
+      # so the message carries only the accepted modes.
+      raise OptionParser::InvalidArgument.new(value, "(expected full|registrable|reg|none)") unless mode
 
       mode
     end
@@ -346,9 +400,7 @@ module Iriq
       data = {}
       data[:parse]     = identifier_hash(iri) if sections.include?(:parse)
       data[:canonical] = iri.canonical         if sections.include?(:canonical)
-      if sections.include?(:normalize)
-        data[:normalize] = corpus ? corpus.normalize(iri) : Normalizer.normalize_identifier(iri, hints: opts[:hints])
-      end
+      data[:normalize] = normalize_section(iri, opts, corpus) if sections.include?(:normalize)
       if sections.include?(:explain)
         data[:explain] = Trace.for(iri, hints: opts[:hints])
       end
@@ -367,20 +419,20 @@ module Iriq
     # of URLs (one per line) and a file of prose with URLs both work. The
     # corpus is ephemeral unless --corpus was given.
     def cmd_batch(args, opts, corpus, explicit_cluster: false)
-      corpus ||= Corpus.new
-
-      # Per-IRI sections (-n/-p/-c/-e) are independent line to line, so we
-      # stream: read input lazily, extract per line, and emit each IRI as it
-      # arrives (flushed for live `tail -f | iriq -n` pipelines). The aggregate
-      # views below — stats, clusters, the deduped URL list — need the whole
-      # input, so they slurp.
+      # Per-IRI sections (-n/-p/-c/-e) stream: read input lazily, extract per
+      # line, and emit each IRI as it arrives (flushed for live
+      # `tail -f | iriq -n` pipelines). Each IRI renders as single-input would,
+      # so with -C there is no corpus, not a throwaway one. The aggregate views
+      # below — stats, clusters, the deduped URL list — need the whole input,
+      # so they slurp.
       if opts[:sections].any?
         emit_per_iri_sections(lazy_iris(args.first, opts), opts, corpus)
         return 0
       end
 
-      iris = extract_text(read_text(args.first), opts)
-      corpus.batch { iris.each { |iri| corpus.observe(iri) } }
+      corpus ||= Corpus.new(host_strategy: opts[:host_strategy])
+      iris = extract_text(utf8!(read_text(args.first)), opts)
+      corpus.observe_all(iris)
 
       if opts[:stats]
         emit_stats(corpus, opts)
@@ -400,19 +452,42 @@ module Iriq
     # (URL_CHAR_CLASS excludes whitespace) and `extract` does not dedup.
     def lazy_iris(path, opts)
       extractor = Extractor.new(scheme_less: opts[:scheme_less])
-      input_lines(path).lazy.flat_map { |line| extractor.extract(line) }
+      input_lines(path).lazy.flat_map { |line| extractor.extract(utf8!(line)) }
     end
 
+    # Yields input lines as they arrive. Only the reads are guarded, so a
+    # failure in the caller's block isn't mistaken for a read error.
     def input_lines(path)
-      if path.nil? || path == "-"
-        stdin.each_line
-      else
-        File.foreach(path)
+      return enum_for(:input_lines, path) unless block_given?
+
+      io = path.nil? || path == "-" ? stdin : read_guard { File.open(path) }
+      while (line = read_guard { io.gets })
+        yield line
       end
+    ensure
+      io.close if io && !io.equal?(stdin)
+    end
+
+    # Input iriq can't read is the OS error in the Rust CLI's words,
+    # `iriq: Permission denied (os error 13)`, code read_error.
+    def read_guard
+      yield
+    rescue SystemCallError => e
+      raise InputError.new("read_error", Iriq.os_error_message(e))
     end
 
-    # Emit the requested sections (parse/normalize/explain) for each extracted
-    # IRI, observing each into `corpus` as it passes. `iris` may be a lazy
+    # Input is UTF-8 regardless of locale; anything else is an error, not a
+    # backtrace. Message matches the Rust CLI's io::Error text.
+    def utf8!(text)
+      text = text.dup.force_encoding(Encoding::UTF_8)
+      raise InputError.new("invalid_utf8", "stream did not contain valid UTF-8") unless text.valid_encoding?
+
+      text
+    end
+
+    # Emit the requested sections (parse/canonical/normalize/explain) for each
+    # extracted IRI, observing each into `corpus` (when there is one) and then
+    # rendering it from the corpus as it now stands. `iris` may be a lazy
     # enumerator; human and NDJSON output stream (flushed per IRI) while a single
     # JSON array must be materialized. -n alone is the cleanest case: one line
     # per URL.
@@ -422,14 +497,14 @@ module Iriq
       # A wrapping JSON array can't be emitted incrementally — collect it
       # (force the lazy enumerator to a real Array so emit_json sees an array).
       if opts[:json] && !opts[:ndjson]
-        payloads = iris.map { |iri| corpus.observe(iri); section_payload(iri, sections, opts) }.to_a
+        payloads = iris.map { |iri| corpus&.observe(iri); section_payload(iri, sections, opts, corpus) }.to_a
         out = sections.size == 1 ? payloads.map(&:values).flatten(1) : payloads
         return emit_json(out, opts)
       end
 
       iris.each_with_index do |iri, i|
-        corpus.observe(iri)
-        p = section_payload(iri, sections, opts)
+        corpus&.observe(iri)
+        p = section_payload(iri, sections, opts, corpus)
         if opts[:ndjson]
           items = sections.size == 1 ? p.values : [p]
           items.each { |item| stdout.puts JSON.generate(item) }
@@ -445,6 +520,7 @@ module Iriq
             when :parse     then emit_parse_human(p[:parse])
             when :canonical then stdout.puts p[:canonical]
             when :normalize then stdout.puts p[:normalize]
+            when :explain   then emit_explain_human(p[:explain])
             end
           end
         end
@@ -452,14 +528,21 @@ module Iriq
       end
     end
 
-    def section_payload(iri, sections, opts)
+    # Key order is fixed (parse, canonical, normalize, explain) whatever the
+    # flag order, matching cmd_summary's multi-section JSON.
+    def section_payload(iri, sections, opts, corpus)
       data = {}
-      data[:parse]     = identifier_hash(iri)                                       if sections.include?(:parse)
-      data[:canonical] = iri.canonical                                              if sections.include?(:canonical)
-      data[:normalize] = Normalizer.normalize_identifier(iri, hints: opts[:hints])  if sections.include?(:normalize)
+      data[:parse]     = identifier_hash(iri)                  if sections.include?(:parse)
+      data[:canonical] = iri.canonical                         if sections.include?(:canonical)
+      data[:normalize] = normalize_section(iri, opts, corpus)  if sections.include?(:normalize)
+      data[:explain]   = Trace.for(iri, hints: opts[:hints])   if sections.include?(:explain)
       data
     end
 
+    def normalize_section(iri, opts, corpus)
+      corpus ? corpus.normalize(iri, hints: opts[:hints]) : Normalizer.normalize_identifier(iri, hints: opts[:hints])
+    end
+
     def extract_text(text, opts)
       Extractor.new(scheme_less: opts[:scheme_less]).extract(text)
     end
@@ -497,7 +580,7 @@ module Iriq
     # --propose-recognizers: scan observed values for prefix patterns
     # that recur enough to suggest a new Recognizer. Prints one block
     # per proposal in human mode, or a JSON array under --json. With
-    # --activate-above F, every proposal at or above coverage F is
+    # --activate-above F, every proposal at or above confidence F is
     # promoted to a live Recognizer on the corpus's classifier and the
     # corpus reinfers to apply the new classifier to existing
     # observations.
@@ -512,7 +595,7 @@ module Iriq
       if opts[:activate_above]
         activated = corpus.activate_proposals_above(opts[:activate_above], **kwargs)
         if activated.empty?
-          stdout.puts "no proposals at or above coverage #{opts[:activate_above]}"
+          stdout.puts "no proposals at or above confidence #{opts[:activate_above]}"
         else
           activated.each do |r|
             stdout.puts "activated: #{r.type} (#{r.prefix})"
@@ -643,20 +726,8 @@ module Iriq
       exit_code
     end
 
-    def read_input(path)
-      if path.nil? || path == "-"
-        stdin.read.lines
-      else
-        File.readlines(path)
-      end
-    end
-
     def read_text(path)
-      if path.nil? || path == "-"
-        stdin.read
-      else
-        File.read(path)
-      end
+      read_guard { path.nil? || path == "-" ? stdin.read : File.read(path) }
     end
 
     # Compact identifier hash for parse output (both JSON and human). Drops

data/lib/iriq/cluster.rb

@@ -98,17 +98,19 @@ module Iriq
       end
     end
 
-    # Per-position summary:
+    # Per-position summary, values by descending count then value:
     #   [
     #     { position: 0, stable: true,  values: { "users" => 3 } },
     #     { position: 1, stable: false, values: { "1" => 1, "2" => 1, "3" => 1 } },
     #   ]
+    # Not storage order: SQLite reads values back sorted, memory and JSON
+    # in first-seen order.
     def segment_stats
       @segment_counts.each_with_index.map do |counts, i|
         {
           position: i,
           stable:   counts.size == 1,
-          values:   counts.dup,
+          values:   counts.sort_by { |v, n| [-n, v] }.to_h,
         }
       end
     end
@@ -180,7 +182,11 @@ module Iriq
     # agree on what the corpus "thinks" about a param.
     def param_type(name)
       stats = @param_stats[name]
-      return nil unless stats
+      stats && Cluster.param_type_for(name, stats)
+    end
+
+    # param_type for one param's stats, read without the rest of the cluster.
+    def self.param_type_for(name, stats)
       return nil if stats.total.zero?
 
       type = stats.dominant_type
@@ -243,7 +249,7 @@ module Iriq
     YEAR_MIN_DISTINCT       = 2
     YEAR_MAX_DISTINCT       = 150
 
-    def year_position?(type, stats)
+    def self.year_position?(type, stats)
       return false unless type == :integer
       return false if stats.numeric_count.zero?
       return false if stats.cardinality < YEAR_MIN_DISTINCT
@@ -258,7 +264,7 @@ module Iriq
     HTTP_STATUS_MIN_DISTINCT     = 2
     HTTP_STATUS_MAX_DISTINCT     = 30
 
-    def http_status_position?(type, stats)
+    def self.http_status_position?(type, stats)
       return false unless type == :integer
       return false if stats.numeric_count.zero?
       return false if stats.cardinality < HTTP_STATUS_MIN_DISTINCT
@@ -272,7 +278,7 @@ module Iriq
     # as an enum. Built around the *established* members (values seen at least
     # ENUM_MIN_VALUE_COUNT times) so a stray one-off value is a straggler, not
     # a disqualifier. See ENUM_* constants at the top of this class.
-    def enum?(stats)
+    def self.enum?(stats)
       return false if stats.total < ENUM_MIN_OBSERVATIONS
 
       established = established_values(stats)
@@ -283,7 +289,7 @@ module Iriq
     end
 
     # Values seen often enough to count as real members of the set (vs noise).
-    def established_values(stats)
+    def self.established_values(stats)
       stats.value_counts.select { |_, n| n >= ENUM_MIN_VALUE_COUNT }
     end
 
@@ -299,7 +305,7 @@ module Iriq
     # ordered by descending count (lex tie-break). Stragglers are excluded so
     # the advertised set is what the corpus is actually confident about.
     def enum_values(stats)
-      established_values(stats).sort_by { |v, n| [-n, v] }.map(&:first)
+      Cluster.established_values(stats).sort_by { |v, n| [-n, v] }.map(&:first)
     end
 
     # value_distribution returns the fraction of total observations each
@@ -350,7 +356,7 @@ module Iriq
 
     # Most common type in stats.type_counts excluding `skip` — lex tie-break
     # so the choice is deterministic across runtimes.
-    def dominant_excluding(stats, skip)
+    def self.dominant_excluding(stats, skip)
       best = nil
       best_count = -1
       stats.type_counts.each do |t, n|