iriq 0.30.2 → 0.35.0

data/lib/iriq/position_stats.rb

@@ -48,8 +48,10 @@ module Iriq
     def record_numeric(value, type)
       return unless NUMERIC_TYPES.include?(type)
 
-      n = Float(value) rescue nil
-      return unless n
+      # Huge digit strings parse to ±Infinity; they'd poison the range and
+      # can't be serialized to JSON. They still count in type/value counts.
+      n = Float(value, exception: false)
+      return unless n&.finite?
 
       @numeric_count += 1
       @numeric_min = n if @numeric_min.nil? || n < @numeric_min
@@ -79,8 +81,8 @@ module Iriq
     end
 
     # Most common type. On count ties, breaks lexicographically by type
-    # symbol name so the result is deterministic and matches Go's
-    # DominantType (Go's map iteration is randomized).
+    # symbol name so the result is deterministic and matches the Rust
+    # port's dominant_type regardless of map iteration order.
     def dominant_type
       best = nil
       best_count = -1

data/lib/iriq/recognizer.rb

@@ -32,7 +32,7 @@ module Iriq
     # mutually-exclusive on shape, so the ensemble is effectively a
     # short-circuit OR. As more Recognizers carve out of SegmentClassifier
     # they'll join the pool and the scoring becomes load-bearing.
-    def self.ensemble(segment, *recognizers)
+    def self.ensemble(segment, recognizers)
       best = nil
       best_score = -1.0
       recognizers.each do |r|

data/lib/iriq/recognizer_proposal.rb

@@ -115,7 +115,7 @@ module Iriq
             acc[:positions] << position
             acc[:hosts] << position.host
             # Collect every match; we'll sort + cap to a stable top-N at
-            # emission time so Ruby and Go produce identical samples
+            # emission time so Ruby and Rust produce identical samples
             # regardless of underlying Hash / map iteration order.
             acc[:matches] << value
           end
@@ -130,12 +130,12 @@ module Iriq
 
           RecognizerProposal.new(
             prefix:            prefix,
-            suggested_type:    prefix.chomp("_").to_sym,
+            suggested_type:    suggested_type_for(prefix),
             positions:         acc[:positions].to_a,
             hosts:             acc[:hosts],
             coverage:          coverage,
             observation_count: acc[:matching_count],
-            # Sort + cap to 5 so Ruby and Go produce identical samples
+            # Sort + cap to 5 so Ruby and Rust produce identical samples
             # regardless of underlying Hash / map iteration order. The
             # samples are illustrative for humans; alphabetical is fine.
             sample_values:     acc[:matches].sort.first(5),
@@ -144,8 +144,20 @@ module Iriq
         }.sort_by { |p| [-p.confidence, p.prefix] }
       end
 
+      # Placeholder names iriq already means something by. `ip` is the
+      # ipv4/ipv6 display umbrella, not a classifier type.
+      RESERVED_TYPE_NAMES = (SegmentClassifier::TYPES + %i[ip]).freeze
+
       private
 
+      # A proposal never takes a built-in name — activating `literal_` as
+      # :literal would turn every matching value into a fixed literal — so
+      # those get an `_id` suffix (`literal_` → :literal_id).
+      def suggested_type_for(prefix)
+        name = prefix.chomp("_").to_sym
+        RESERVED_TYPE_NAMES.include?(name) ? :"#{name}_id" : name
+      end
+
       def empty_accumulator
         {
           positions:             Set.new,

data/lib/iriq/reducer.rb

@@ -8,8 +8,7 @@ module Iriq
   # handles it, register it in DEFAULTS — no other module changes.
   module Reducer
     # Each entry: { event_class => [lambda(event, storage) -> result] }.
-    # Lambdas may return the result of the underlying storage call so
-    # callers (Corpus#observe) can pick up the cluster they need to return.
+    # Lambdas return the result of the underlying storage call.
     DEFAULTS = {
       Event::HostSeen        => [->(e, s) { s.increment_host(e.host) }],
       Event::PathLengthSeen  => [->(e, s) { s.increment_path_length(e.length) }],

data/lib/iriq/segment_classifier.rb

@@ -14,8 +14,10 @@ module Iriq
     #
     # `:enum` is similarly corpus-only — it surfaces when a position has a
     # bounded set of distinct values observed across enough samples (see
-    # Cluster::ENUM_* thresholds).
-    TYPES = %i[literal integer float number uuid date year timestamp hash slug
+    # Cluster::ENUM_* thresholds). `:string` is the rung below `:enum`: a
+    # param that varies across free-form literal values but isn't a confident
+    # bounded set yet. Neither is ever returned for a single value.
+    TYPES = %i[literal string integer float number uuid date year timestamp hash slug
                ipv4 ipv6 url email boolean version locale currency phone jwt mime
                file color coordinate country base64 http_status enum opaque_id].freeze
 
@@ -125,9 +127,10 @@ module Iriq
       document: %w[pdf doc docx xls xlsx ppt pptx odt ods odp rtf epub],
       data:     %w[csv tsv json xml yaml yml parquet sqlite db ndjson jsonl],
       text:     %w[txt md log markdown rst],
-      web:      %w[html htm css js mjs cjs ts jsx tsx],
+      web:      %w[html htm css js mjs cjs ts jsx tsx map],
+      font:     %w[woff woff2 ttf otf eot],
       audio:    %w[mp3 wav ogg flac aac m4a opus],
-      video:    %w[mp4 mov avi mkv webm flv wmv m4v],
+      video:    %w[mp4 mov avi mkv webm flv wmv m4v m3u8],
       archive:  %w[zip tar gz bz2 7z rar xz tgz],
       code:     %w[rb py go java c cc cpp h hpp sh swift kt rs],
     }.freeze
@@ -230,6 +233,13 @@ module Iriq
       recognizer
     end
 
+    # A copy registers recognizers without touching the original.
+    def initialize_copy(source)
+      super
+      @recognizers = source.recognizers
+      @cache       = {}
+    end
+
     # Snapshot of the live ensemble. Useful for tests and tooling that
     # want to inspect which Recognizers a corpus is consulting.
     def recognizers
@@ -262,7 +272,7 @@ module Iriq
       # Scored ensemble over the live Recognizer list — built-ins +
       # anything Corpus#activate_proposal has registered for this
       # classifier instance.
-      if (v = Recognizer.ensemble(segment, *@recognizers))
+      if (v = Recognizer.ensemble(segment, @recognizers))
         return v[:type]
       end
 
@@ -502,7 +512,8 @@ module Iriq
     # /pricing/USD both render as /pricing/USD.
     def self.canonical_currency(value)
       return nil if value.nil?
-      up = value.upcase
+      # ASCII-only: full Unicode upcase maps ſ→S / ı→I, forging a code.
+      up = value.upcase(:ascii)
       CURRENCY_CODES.include?(up) ? up : nil
     end
 

data/lib/iriq/specificity.rb

@@ -11,9 +11,9 @@ module Iriq
   # integer could plausibly be a year, an HTTP status, or an ID, so
   # `:integer` claims only TYPED.
   #
-  # Calibration corpus tests in spec/iriq/calibration_spec.rb / Go's
-  # calibration_test.go are the source of truth for whether these
-  # values are well-chosen — adjust them and re-run to validate.
+  # Calibration corpus tests in spec/iriq/calibration_spec.rb are the
+  # source of truth for whether these values are well-chosen — adjust
+  # them and re-run to validate.
   module Specificity
     # Unambiguous semantic shapes — the regex effectively can't fire by
     # accident. (UUID, JWT, email with @, URL with ://, color hex.)

data/lib/iriq/storage/json.rb

@@ -19,24 +19,38 @@ module Iriq
         s
       end
 
+      # Refuses (Iriq::CorpusError) anything that isn't a corpus dump, so a
+      # mistyped --corpus path can't overwrite an unrelated JSON file: it must
+      # be an object with at least one corpus key, or `{}`.
       def load!(path)
-        data = File.read(path)
+        data = begin
+          File.read(path)
+        rescue SystemCallError => e
+          raise CorpusError, "corpus #{path}: #{Iriq.os_error_message(e)}"
+        end
         return self if data.empty?
 
-        load_dump!(JSON.parse(data))
+        dump = begin
+          JSON.parse(data)
+        rescue JSON::ParserError
+          raise CorpusError, "corpus #{path}: not valid JSON"
+        end
+        unless dump.is_a?(Hash) && (dump.empty? || dump.keys.intersect?(DUMP_KEYS))
+          raise CorpusError, "corpus #{path}: not an iriq corpus (no corpus keys at the top level)"
+        end
+
+        load_dump!(dump)
         @path = path
         self
       end
 
-      # save writes atomically (tmp + rename). Defaults to the path passed at
-      # open(); pass an explicit path to write elsewhere.
+      # save writes atomically (unique tmp + rename). Defaults to the path
+      # passed at open(); pass an explicit path to write elsewhere.
       def save(path = nil)
         target = path || @path
         raise ArgumentError, "no path provided" unless target
 
-        tmp = "#{target}.tmp"
-        File.write(tmp, JSON.generate(to_dump))
-        File.rename(tmp, target)
+        Storage.write_atomically(target, JSON.generate(to_dump))
       end
     end
   end

data/lib/iriq/storage/memory.rb

@@ -15,12 +15,17 @@ module Iriq
     #
     #   host_counts / path_length_counts / raw_shape_counts / fingerprint_counts
     #   position_stats(position)
+    #   position_evidence(position, value)             # the narrow read normalize uses
+    #   param_stats(cluster_key, name)                 # one param, without the cluster
     #   each_position_stats { |position, stats| ... }
     #   each_observed_iri { |canonical| ... }
-    #   clear_materialized_views                       # for reinfer
+    #   each_observed_iri_since(mark) { |canonical| ... } # → mark of the last
+    #   clear_materialized_views
+    #   begin_rebuild / install_rebuild / discard_rebuild # for reinfer
     #   clusters / cluster_size
     #
     #   transaction { ... }    # backends may batch within
+    #   turn_over?             # a long batch should commit and let others in
     #   flush                  # commit pending writes (no-op for Memory)
     #   close                  # release resources
     class Memory
@@ -55,8 +60,14 @@ module Iriq
         yield self
       end
 
+      # Yields false: nothing else writes an in-memory corpus.
       def batch
-        yield
+        yield false
+      end
+
+      # Nothing else waits on an in-memory corpus.
+      def turn_over?
+        false
       end
 
       def flush;  end
@@ -108,6 +119,13 @@ module Iriq
         @observed_iris.each(&block)
       end
 
+      # The observations logged after `mark` (0 for all), in order; returns
+      # the mark of the last one.
+      def each_observed_iri_since(mark, &block)
+        @observed_iris.drop(mark).each(&block)
+        [mark, @observed_iris.size].max
+      end
+
       def observed_iri_count
         @observed_iris.size
       end
@@ -138,6 +156,14 @@ module Iriq
         @clusters           = {}
       end
 
+      # Nothing else reads an in-memory corpus mid-rebuild: rebuild in place.
+      def begin_rebuild
+        clear_materialized_views
+      end
+
+      def install_rebuild; end
+      def discard_rebuild; end
+
       # --- Reads ------------------------------------------------------------
 
       def host_counts;        @host_counts;        end
@@ -149,6 +175,12 @@ module Iriq
         @position_stats[position]
       end
 
+      # Built over the live stats, so nothing is copied.
+      def position_evidence(position, value)
+        stats = @position_stats[position]
+        stats && PositionEvidence.from_stats(stats, value)
+      end
+
       def each_position_stats(&block)
         @position_stats.each(&block)
       end
@@ -161,22 +193,32 @@ module Iriq
         @clusters.size
       end
 
-      # O(1) lookup by cluster key — used by Corpus#normalize to pull the
-      # cluster's param_stats for the URL being normalized. nil if no cluster
-      # has been observed under this key yet.
+      # O(1) lookup by cluster key. nil if no cluster has been observed under
+      # this key yet.
       def cluster_for(key)
         @clusters[key]
       end
 
+      def param_stats(key, name)
+        cluster = @clusters[key]
+        cluster && cluster.param_stats[name]
+      end
+
       # --- Bulk load (used by JSON backend) --------------------------------
 
+      # Top-level keys of the dump. A JSON file with none of them isn't a corpus.
+      DUMP_KEYS = %w[host_counts path_length_counts raw_shape_counts fingerprint_counts
+                     max_values_per_position position_stats clusterer observed_iris
+                     activated_recognizers].freeze
+
+      # Missing keys load as empty, so `{}` is a valid (empty) corpus.
       def load_dump!(h)
-        @host_counts        = Hash.new(0).merge(h["host_counts"])
-        @path_length_counts = Hash.new(0).merge(h["path_length_counts"].transform_keys(&:to_i))
-        @raw_shape_counts   = Hash.new(0).merge(h["raw_shape_counts"])
-        @fingerprint_counts = Hash.new(0).merge(h["fingerprint_counts"])
+        @host_counts        = Hash.new(0).merge(h.fetch("host_counts", {}))
+        @path_length_counts = Hash.new(0).merge(h.fetch("path_length_counts", {}).transform_keys(&:to_i))
+        @raw_shape_counts   = Hash.new(0).merge(h.fetch("raw_shape_counts", {}))
+        @fingerprint_counts = Hash.new(0).merge(h.fetch("fingerprint_counts", {}))
         @max_values_per_position = h.fetch("max_values_per_position", PositionStats::DEFAULT_MAX_VALUES)
-        @position_stats = h["position_stats"].each_with_object({}) do |entry, acc|
+        @position_stats = h.fetch("position_stats", []).each_with_object({}) do |entry, acc|
           position = Position.from_dump(entry["position"])
           acc[position] = PositionStats.from_dump(entry["stats"])
         end