iriq 0.30.2 → 0.35.0

data/lib/iriq/cluster.rb

@@ -30,14 +30,35 @@ module Iriq
     NUMBER_CONFIDENCE_THRESHOLD = 0.8
     NUMBER_SUBTYPE_THRESHOLD    = 0.8
 
+    # Param classification is a confidence ladder: constant → string → enum.
+    # A param with a single observed value is a constant (rendered as-is); one
+    # that varies but isn't yet a trustworthy enum is :string (a generic
+    # placeholder); a bounded, well-supported value set is :enum.
+    #
     # `:enum` thresholds. Promote a param to :enum when the corpus has seen
-    # enough samples to trust the bound, the value set is small, each value
-    # appears more than once (rules out singletons), and the tracked values
-    # account for nearly all observations (lets a few stragglers through).
+    # enough samples to trust the bound (ENUM_MIN_OBSERVATIONS), the *established*
+    # values — those seen at least ENUM_MIN_VALUE_COUNT times — are few
+    # (ENUM_MAX_CARDINALITY) and cover nearly all observations
+    # (ENUM_MIN_COVERAGE). Rare one-off values are stragglers, not
+    # disqualifiers: this is what keeps a single brand-new value from knocking
+    # an established enum back down (the observe-before-normalize case).
     ENUM_MIN_OBSERVATIONS = 20
     ENUM_MAX_CARDINALITY  = 10
-    ENUM_MIN_VALUE_COUNT  = 2
-    ENUM_MIN_COVERAGE     = 0.95
+    ENUM_MIN_VALUE_COUNT  = 3
+    ENUM_MIN_COVERAGE     = 0.9
+    # An enum is a bounded *set*: a single repeated value is a constant, not an
+    # enum, so it takes at least two established members to qualify.
+    ENUM_MIN_MEMBERS      = 2
+
+    # `:string` — a param that has taken on 2+ distinct non-typed values but
+    # isn't (yet) a confident enum. The intermediate rung: we know it varies
+    # and looks like free-form text, but haven't earned the bounded-set claim.
+    STRING_MIN_DISTINCT = 2
+
+    # Confidence smoothing constant. confidence = total / (total + K): a
+    # monotone curve that is 0.5 at K observations and asymptotes to 1.0. The
+    # type names our guess; this number says how much evidence backs it.
+    CONFIDENCE_SMOOTHING = 15
 
     def initialize(key:, host:, scheme:, shape:, max_values: PositionStats::DEFAULT_MAX_VALUES)
       @key            = key
@@ -72,21 +93,24 @@ module Iriq
       return unless identifier.query_params
       identifier.query_params.each do |name, value|
         stats = @param_stats[name] ||= PositionStats.new(max_values: @max_values)
-        stats.observe(value.to_s, classifier.classify(value.to_s))
+        value_s = value.to_s
+        stats.observe(value_s, classifier.classify(value_s))
       end
     end
 
-    # Per-position summary:
+    # Per-position summary, values by descending count then value:
     #   [
     #     { position: 0, stable: true,  values: { "users" => 3 } },
     #     { position: 1, stable: false, values: { "1" => 1, "2" => 1, "3" => 1 } },
     #   ]
+    # Not storage order: SQLite reads values back sorted, memory and JSON
+    # in first-seen order.
     def segment_stats
       @segment_counts.each_with_index.map do |counts, i|
         {
           position: i,
           stable:   counts.size == 1,
-          values:   counts.dup,
+          values:   counts.sort_by { |v, n| [-n, v] }.to_h,
         }
       end
     end
@@ -105,7 +129,9 @@ module Iriq
     end
 
     # Per-param summary, ordered by descending presence. Each entry is:
-    #   { name: "page", count: N, type: :integer, cardinality: K, presence: 0.83 }
+    #   { name: "page", count: N, type: :integer, confidence: 0.83,
+    #     cardinality: K, presence: 0.83 }
+    # confidence is how much evidence backs the type (see param_confidence);
     # presence is count / @count — the fraction of observations that had
     # this param.
     def param_summary
@@ -118,6 +144,7 @@ module Iriq
           name:        name,
           count:       stats.total,
           type:        type,
+          confidence:  param_confidence(stats),
           cardinality: stats.cardinality,
           presence:    @count.positive? ? stats.total.to_f / @count : 0.0,
         }
@@ -155,7 +182,11 @@ module Iriq
     # agree on what the corpus "thinks" about a param.
     def param_type(name)
       stats = @param_stats[name]
-      return nil unless stats
+      stats && Cluster.param_type_for(name, stats)
+    end
+
+    # param_type for one param's stats, read without the rest of the cluster.
+    def self.param_type_for(name, stats)
       return nil if stats.total.zero?
 
       type = stats.dominant_type
@@ -205,6 +236,11 @@ module Iriq
         return hint
       end
 
+      # :string rung — a literal-valued param that has taken on more than one
+      # distinct value varies, so it's a placeholder, not a fixed constant.
+      # Below the enum bar (checked above), so we claim only "free-form text".
+      return :string if type == :literal && stats.cardinality >= STRING_MIN_DISTINCT
+
       type
     end
 
@@ -213,7 +249,7 @@ module Iriq
     YEAR_MIN_DISTINCT       = 2
     YEAR_MAX_DISTINCT       = 150
 
-    def year_position?(type, stats)
+    def self.year_position?(type, stats)
       return false unless type == :integer
       return false if stats.numeric_count.zero?
       return false if stats.cardinality < YEAR_MIN_DISTINCT
@@ -228,7 +264,7 @@ module Iriq
     HTTP_STATUS_MIN_DISTINCT     = 2
     HTTP_STATUS_MAX_DISTINCT     = 30
 
-    def http_status_position?(type, stats)
+    def self.http_status_position?(type, stats)
       return false unless type == :integer
       return false if stats.numeric_count.zero?
       return false if stats.cardinality < HTTP_STATUS_MIN_DISTINCT
@@ -239,21 +275,37 @@ module Iriq
     end
 
     # True when stats shows a bounded set of repeated values worth treating
-    # as an enum. See ENUM_* constants at the top of this class.
-    def enum?(stats)
+    # as an enum. Built around the *established* members (values seen at least
+    # ENUM_MIN_VALUE_COUNT times) so a stray one-off value is a straggler, not
+    # a disqualifier. See ENUM_* constants at the top of this class.
+    def self.enum?(stats)
       return false if stats.total < ENUM_MIN_OBSERVATIONS
-      return false if stats.cardinality.zero? || stats.cardinality > ENUM_MAX_CARDINALITY
-      return false if stats.value_counts.any? { |_, n| n < ENUM_MIN_VALUE_COUNT }
 
-      coverage = stats.value_counts.values.sum.to_f / stats.total
+      established = established_values(stats)
+      return false unless established.size.between?(ENUM_MIN_MEMBERS, ENUM_MAX_CARDINALITY)
+
+      coverage = established.values.sum.to_f / stats.total
       coverage >= ENUM_MIN_COVERAGE
     end
 
-    # Distinct values tracked for this param, ordered by descending count
-    # (lex tie-break). Returned alongside :enum-typed rows in param_summary
-    # so verbose/explain consumers can render the value set.
+    # Values seen often enough to count as real members of the set (vs noise).
+    def self.established_values(stats)
+      stats.value_counts.select { |_, n| n >= ENUM_MIN_VALUE_COUNT }
+    end
+
+    # Confidence that the assigned type is right, given how much evidence backs
+    # it. Monotone in observation count; 0.5 at CONFIDENCE_SMOOTHING, → 1.0.
+    def param_confidence(stats)
+      return 0.0 if stats.total.zero?
+
+      (stats.total.to_f / (stats.total + CONFIDENCE_SMOOTHING)).round(2)
+    end
+
+    # The enum's member values — the established ones (seen enough to be real),
+    # ordered by descending count (lex tie-break). Stragglers are excluded so
+    # the advertised set is what the corpus is actually confident about.
     def enum_values(stats)
-      stats.value_counts.sort_by { |v, n| [-n, v] }.map(&:first)
+      Cluster.established_values(stats).sort_by { |v, n| [-n, v] }.map(&:first)
     end
 
     # value_distribution returns the fraction of total observations each
@@ -304,7 +356,7 @@ module Iriq
 
     # Most common type in stats.type_counts excluding `skip` — lex tie-break
     # so the choice is deterministic across runtimes.
-    def dominant_excluding(stats, skip)
+    def self.dominant_excluding(stats, skip)
       best = nil
       best_count = -1
       stats.type_counts.each do |t, n|
@@ -357,8 +409,8 @@ module Iriq
       if iri.urn?
         ns, value = (iri.nss || "").split(":", 2)
         derived = value ? urn_value_shape(ns, value, classifier) : nil
-        key     = "urn:#{ns}:#{derived}"
-        [key, nil, "urn", key]
+        key     = "#{iri.scheme}:#{ns}:#{derived}"
+        [key, nil, iri.scheme, key]
       else
         shape ||= PathShape.new(classifier: classifier).for(iri.path_segments)
         effective_host = host.nil? ? iri.host : host

data/lib/iriq/clusterer.rb

@@ -47,17 +47,6 @@ module Iriq
       end
     end
 
-    def dump
-      { "clusters" => clusters.each_with_object({}) { |c, h| h[c.key] = c.dump } }
-    end
-
-    def self.from_dump(h, classifier: SegmentClassifier::DEFAULT)
-      c = new(classifier: classifier)
-      restored = h["clusters"].transform_values { |cdump| Cluster.from_dump(cdump) }
-      c.instance_variable_get(:@storage).instance_variable_set(:@clusters, restored)
-      c
-    end
-
     private
 
     def coerce(input)