iriq 0.33.0 → 0.35.0

data/lib/iriq/recognizer_proposal.rb

@@ -130,7 +130,7 @@ module Iriq
 
           RecognizerProposal.new(
             prefix:            prefix,
-            suggested_type:    prefix.chomp("_").to_sym,
+            suggested_type:    suggested_type_for(prefix),
             positions:         acc[:positions].to_a,
             hosts:             acc[:hosts],
             coverage:          coverage,
@@ -144,8 +144,20 @@ module Iriq
         }.sort_by { |p| [-p.confidence, p.prefix] }
       end
 
+      # Placeholder names iriq already means something by. `ip` is the
+      # ipv4/ipv6 display umbrella, not a classifier type.
+      RESERVED_TYPE_NAMES = (SegmentClassifier::TYPES + %i[ip]).freeze
+
       private
 
+      # A proposal never takes a built-in name — activating `literal_` as
+      # :literal would turn every matching value into a fixed literal — so
+      # those get an `_id` suffix (`literal_` → :literal_id).
+      def suggested_type_for(prefix)
+        name = prefix.chomp("_").to_sym
+        RESERVED_TYPE_NAMES.include?(name) ? :"#{name}_id" : name
+      end
+
       def empty_accumulator
         {
           positions:             Set.new,

data/lib/iriq/reducer.rb

@@ -8,8 +8,7 @@ module Iriq
   # handles it, register it in DEFAULTS — no other module changes.
   module Reducer
     # Each entry: { event_class => [lambda(event, storage) -> result] }.
-    # Lambdas may return the result of the underlying storage call so
-    # callers (Corpus#observe) can pick up the cluster they need to return.
+    # Lambdas return the result of the underlying storage call.
     DEFAULTS = {
       Event::HostSeen        => [->(e, s) { s.increment_host(e.host) }],
       Event::PathLengthSeen  => [->(e, s) { s.increment_path_length(e.length) }],

data/lib/iriq/segment_classifier.rb

@@ -233,6 +233,13 @@ module Iriq
       recognizer
     end
 
+    # A copy registers recognizers without touching the original.
+    def initialize_copy(source)
+      super
+      @recognizers = source.recognizers
+      @cache       = {}
+    end
+
     # Snapshot of the live ensemble. Useful for tests and tooling that
     # want to inspect which Recognizers a corpus is consulting.
     def recognizers
@@ -505,7 +512,8 @@ module Iriq
     # /pricing/USD both render as /pricing/USD.
     def self.canonical_currency(value)
       return nil if value.nil?
-      up = value.upcase
+      # ASCII-only: full Unicode upcase maps ſ→S / ı→I, forging a code.
+      up = value.upcase(:ascii)
       CURRENCY_CODES.include?(up) ? up : nil
     end
 

data/lib/iriq/storage/json.rb

@@ -19,24 +19,38 @@ module Iriq
         s
       end
 
+      # Refuses (Iriq::CorpusError) anything that isn't a corpus dump, so a
+      # mistyped --corpus path can't overwrite an unrelated JSON file: it must
+      # be an object with at least one corpus key, or `{}`.
       def load!(path)
-        data = File.read(path)
+        data = begin
+          File.read(path)
+        rescue SystemCallError => e
+          raise CorpusError, "corpus #{path}: #{Iriq.os_error_message(e)}"
+        end
         return self if data.empty?
 
-        load_dump!(JSON.parse(data))
+        dump = begin
+          JSON.parse(data)
+        rescue JSON::ParserError
+          raise CorpusError, "corpus #{path}: not valid JSON"
+        end
+        unless dump.is_a?(Hash) && (dump.empty? || dump.keys.intersect?(DUMP_KEYS))
+          raise CorpusError, "corpus #{path}: not an iriq corpus (no corpus keys at the top level)"
+        end
+
+        load_dump!(dump)
         @path = path
         self
       end
 
-      # save writes atomically (tmp + rename). Defaults to the path passed at
-      # open(); pass an explicit path to write elsewhere.
+      # save writes atomically (unique tmp + rename). Defaults to the path
+      # passed at open(); pass an explicit path to write elsewhere.
       def save(path = nil)
         target = path || @path
         raise ArgumentError, "no path provided" unless target
 
-        tmp = "#{target}.tmp"
-        File.write(tmp, JSON.generate(to_dump))
-        File.rename(tmp, target)
+        Storage.write_atomically(target, JSON.generate(to_dump))
       end
     end
   end

data/lib/iriq/storage/memory.rb

@@ -15,12 +15,17 @@ module Iriq
     #
     #   host_counts / path_length_counts / raw_shape_counts / fingerprint_counts
     #   position_stats(position)
+    #   position_evidence(position, value)             # the narrow read normalize uses
+    #   param_stats(cluster_key, name)                 # one param, without the cluster
     #   each_position_stats { |position, stats| ... }
     #   each_observed_iri { |canonical| ... }
-    #   clear_materialized_views                       # for reinfer
+    #   each_observed_iri_since(mark) { |canonical| ... } # → mark of the last
+    #   clear_materialized_views
+    #   begin_rebuild / install_rebuild / discard_rebuild # for reinfer
     #   clusters / cluster_size
     #
     #   transaction { ... }    # backends may batch within
+    #   turn_over?             # a long batch should commit and let others in
     #   flush                  # commit pending writes (no-op for Memory)
     #   close                  # release resources
     class Memory
@@ -55,8 +60,14 @@ module Iriq
         yield self
       end
 
+      # Yields false: nothing else writes an in-memory corpus.
       def batch
-        yield
+        yield false
+      end
+
+      # Nothing else waits on an in-memory corpus.
+      def turn_over?
+        false
       end
 
       def flush;  end
@@ -108,6 +119,13 @@ module Iriq
         @observed_iris.each(&block)
       end
 
+      # The observations logged after `mark` (0 for all), in order; returns
+      # the mark of the last one.
+      def each_observed_iri_since(mark, &block)
+        @observed_iris.drop(mark).each(&block)
+        [mark, @observed_iris.size].max
+      end
+
       def observed_iri_count
         @observed_iris.size
       end
@@ -138,6 +156,14 @@ module Iriq
         @clusters           = {}
       end
 
+      # Nothing else reads an in-memory corpus mid-rebuild: rebuild in place.
+      def begin_rebuild
+        clear_materialized_views
+      end
+
+      def install_rebuild; end
+      def discard_rebuild; end
+
       # --- Reads ------------------------------------------------------------
 
       def host_counts;        @host_counts;        end
@@ -149,6 +175,12 @@ module Iriq
         @position_stats[position]
       end
 
+      # Built over the live stats, so nothing is copied.
+      def position_evidence(position, value)
+        stats = @position_stats[position]
+        stats && PositionEvidence.from_stats(stats, value)
+      end
+
       def each_position_stats(&block)
         @position_stats.each(&block)
       end
@@ -161,22 +193,32 @@ module Iriq
         @clusters.size
       end
 
-      # O(1) lookup by cluster key — used by Corpus#normalize to pull the
-      # cluster's param_stats for the URL being normalized. nil if no cluster
-      # has been observed under this key yet.
+      # O(1) lookup by cluster key. nil if no cluster has been observed under
+      # this key yet.
       def cluster_for(key)
         @clusters[key]
       end
 
+      def param_stats(key, name)
+        cluster = @clusters[key]
+        cluster && cluster.param_stats[name]
+      end
+
       # --- Bulk load (used by JSON backend) --------------------------------
 
+      # Top-level keys of the dump. A JSON file with none of them isn't a corpus.
+      DUMP_KEYS = %w[host_counts path_length_counts raw_shape_counts fingerprint_counts
+                     max_values_per_position position_stats clusterer observed_iris
+                     activated_recognizers].freeze
+
+      # Missing keys load as empty, so `{}` is a valid (empty) corpus.
       def load_dump!(h)
-        @host_counts        = Hash.new(0).merge(h["host_counts"])
-        @path_length_counts = Hash.new(0).merge(h["path_length_counts"].transform_keys(&:to_i))
-        @raw_shape_counts   = Hash.new(0).merge(h["raw_shape_counts"])
-        @fingerprint_counts = Hash.new(0).merge(h["fingerprint_counts"])
+        @host_counts        = Hash.new(0).merge(h.fetch("host_counts", {}))
+        @path_length_counts = Hash.new(0).merge(h.fetch("path_length_counts", {}).transform_keys(&:to_i))
+        @raw_shape_counts   = Hash.new(0).merge(h.fetch("raw_shape_counts", {}))
+        @fingerprint_counts = Hash.new(0).merge(h.fetch("fingerprint_counts", {}))
         @max_values_per_position = h.fetch("max_values_per_position", PositionStats::DEFAULT_MAX_VALUES)
-        @position_stats = h["position_stats"].each_with_object({}) do |entry, acc|
+        @position_stats = h.fetch("position_stats", []).each_with_object({}) do |entry, acc|
           position = Position.from_dump(entry["position"])
           acc[position] = PositionStats.from_dump(entry["stats"])
         end