iriq 0.1.0 → 0.30.2

data/lib/iriq/trace.rb

@@ -0,0 +1,294 @@
+module Iriq
+  # Produces an annotated trace explaining how an identifier got
+  # normalized — segment by segment, with notes for each non-obvious
+  # transformation (currency upcase, IP umbrella, hint suppression,
+  # canonical date, param-name lift, etc.).
+  #
+  #   Trace.for("https://shop.com/pricing/usd?currency=eur")
+  #   # => {
+  #   #      input: "...",
+  #   #      normalized: "https://shop.com/pricing/USD?currency=EUR",
+  #   #      scheme: "https", host: "shop.com",
+  #   #      path:  [...per-segment rows...],
+  #   #      query: [...per-param rows...],
+  #   #    }
+  #
+  # Each row is `{ value, type, output, notes }` for path entries and
+  # `{ name, value, type, output, notes }` for query entries. The string
+  # notes are rendered from structured Iriq::Evidence::Record values;
+  # callers that want the structured form can use Trace.evidence_for.
+  module Trace
+    module_function
+
+    HINT_NOTE_TEMPLATE = "semantic type — surfaced as {%s}, not {%s}".freeze
+
+    # Render-ready Trace output. The public format consumers depend on.
+    def for(input, classifier: SegmentClassifier::DEFAULT, hints: true)
+      iri = coerce(input)
+      normalized = Normalizer.normalize_identifier(iri, classifier: classifier, hints: hints)
+
+      out = {
+        input:      iri.canonical,
+        normalized: normalized,
+        scheme:     iri.scheme,
+        host:       iri.host,
+      }
+      out[:port] = iri.port if iri.port
+
+      if iri.urn?
+        out[:path] = urn_rows(iri, classifier, hints)
+      else
+        out[:path]  = path_rows(iri.path_segments, classifier, hints)
+        if iri.query_params && !iri.query_params.empty?
+          out[:query] = query_rows(iri.query_params, classifier)
+        end
+      end
+
+      out
+    end
+
+    # Structured Evidence list for `input`. Each segment + query param
+    # contributes one classification Evidence plus zero or more
+    # transformation Evidence records (canonical date, IP umbrella
+    # collapse, param-name hint, hint suppression).
+    #
+    # Position + Cluster Evidence are not emitted here — they belong to
+    # corpus-informed trace (Corpus#trace), which a follow-up step lands.
+    def evidence_for(input, classifier: SegmentClassifier::DEFAULT, hints: true)
+      iri      = coerce(input)
+      records  = []
+      segments = iri.urn? ? urn_parts(iri) : (iri.path_segments || [])
+      entries  = SegmentHints.derive(segments, classifier)
+
+      entries.each_with_index do |entry, i|
+        records.concat(segment_evidence(entry, segments, i, classifier, hints))
+      end
+
+      if !iri.urn? && iri.query_params && !iri.query_params.empty?
+        iri.query_params.keys.sort.each do |k|
+          records.concat(query_param_evidence(k, iri.query_params[k].to_s, classifier))
+        end
+      end
+
+      records
+    end
+
+    # ── Evidence builders ────────────────────────────────────────────────
+
+    def segment_evidence(entry, segments, idx, classifier, hints)
+      records = []
+
+      records << Evidence.segment(
+        index: idx, value: entry[:value],
+        source:  :recognizer,
+        payload: { type: entry[:type], variable: entry[:variable], hint: entry[:hint] },
+      )
+
+      if entry[:variable]
+        if entry[:type] == :date && (canon = SegmentClassifier.canonical_date(entry[:value]))
+          if canon != entry[:value]
+            records << Evidence.segment(
+              index: idx, value: entry[:value],
+              source:  :policy,
+              payload: { rule: :canonical_date, before: entry[:value], after: canon },
+              notes:   ["canonical date (#{entry[:value]} → #{canon})"],
+            )
+          end
+        elsif entry[:type] == :currency && (canon = SegmentClassifier.canonical_currency(entry[:value]))
+          if canon != entry[:value]
+            records << Evidence.segment(
+              index: idx, value: entry[:value],
+              source:  :policy,
+              payload: { rule: :canonical_currency, before: entry[:value], after: canon },
+              notes:   ["currency upcase (#{entry[:value]} → #{canon})"],
+            )
+          end
+        else
+          extra = placeholder_decoration_evidence(entry, segments, idx, classifier, hints)
+          records.concat(extra)
+        end
+      end
+
+      records
+    end
+
+    def placeholder_decoration_evidence(entry, segments, idx, classifier, hints)
+      out  = []
+      type = entry[:type]
+
+      if type == :ipv4 || type == :ipv6
+        out << Evidence.segment(
+          index: idx, value: entry[:value],
+          source:  :policy,
+          payload: { rule: :ip_umbrella_collapse, from: type, to: :ip },
+          notes:   ["ip umbrella collapse (#{type} → ip)"],
+        )
+      end
+
+      if hints && entry[:hint].nil? && !SegmentHints::HINT_ELIGIBLE_TYPES.include?(type)
+        if (would_be = would_be_hint(segments, idx, type, classifier))
+          display = SegmentClassifier.display_type(type)
+          out << Evidence.segment(
+            index: idx, value: entry[:value],
+            source:  :neighbor,
+            payload: { rule: :hint_suppression, surfaced: display, would_be: would_be, semantic_type: type },
+            notes:   [format(HINT_NOTE_TEMPLATE, display, would_be)],
+          )
+        end
+      end
+
+      out
+    end
+
+    def query_param_evidence(name, value, classifier)
+      records   = []
+      base_type = classifier.classify(value)
+      effective = base_type
+
+      if (hint = SegmentClassifier.param_name_hint(name, base_type))
+        effective = hint
+        records << Evidence.segment(
+          index: name, value: value,
+          source:  :neighbor,
+          payload: { rule: :param_name_hint, name: name, before: base_type, after: hint },
+          notes:   ["param-name hint (`#{name}=`) lifted #{base_type} → #{hint}"],
+        )
+      end
+
+      records << Evidence.segment(
+        index: name, value: value,
+        source:  :recognizer,
+        payload: { type: effective, variable: SegmentClassifier::DEFAULT.variable?(effective) },
+      )
+
+      if effective == :date && (canon = SegmentClassifier.canonical_date(value))
+        if canon != value
+          records << Evidence.segment(
+            index: name, value: value,
+            source:  :policy,
+            payload: { rule: :canonical_date, before: value, after: canon },
+            notes:   ["canonical date (#{value} → #{canon})"],
+          )
+        end
+      elsif effective == :currency && (canon = SegmentClassifier.canonical_currency(value))
+        if canon != value
+          records << Evidence.segment(
+            index: name, value: value,
+            source:  :policy,
+            payload: { rule: :canonical_currency, before: value, after: canon },
+            notes:   ["currency upcase (#{value} → #{canon})"],
+          )
+        end
+      elsif effective == :ipv4 || effective == :ipv6
+        records << Evidence.segment(
+          index: name, value: value,
+          source:  :policy,
+          payload: { rule: :ip_umbrella_collapse, from: effective, to: :ip },
+          notes:   ["ip umbrella collapse (#{effective} → ip)"],
+        )
+      end
+
+      records
+    end
+
+    # ── View rendering (Evidence → Trace.for hash) ───────────────────────
+
+    def path_rows(segments, classifier, hints)
+      return [] if segments.nil? || segments.empty?
+
+      entries = SegmentHints.derive(segments, classifier)
+      entries.each_with_index.map do |entry, i|
+        ev = segment_evidence(entry, segments, i, classifier, hints)
+        render_segment_row(entry, ev, hints)
+      end
+    end
+
+    def urn_rows(iri, classifier, hints)
+      parts = urn_parts(iri)
+      return [] if parts.empty?
+
+      entries = SegmentHints.derive(parts, classifier)
+      entries.each_with_index.map do |entry, i|
+        ev = segment_evidence(entry, parts, i, classifier, hints)
+        render_segment_row(entry, ev, hints)
+      end
+    end
+
+    def query_rows(params, classifier)
+      params.keys.sort.map do |k|
+        v  = params[k].to_s
+        ev = query_param_evidence(k, v, classifier)
+        render_query_row(k, v, ev)
+      end
+    end
+
+    def render_segment_row(entry, evidence, hints)
+      notes = collect_notes(evidence)
+      value = entry[:value]
+      type  = entry[:type]
+
+      return { value: value, type: type, output: value, notes: notes } unless entry[:variable]
+
+      canon_policy = find_payload(evidence, :canonical_date) || find_payload(evidence, :canonical_currency)
+      if canon_policy
+        return { value: value, type: type, output: canon_policy[:after], notes: notes }
+      end
+
+      placeholder = hints && entry[:hint] ? entry[:hint].to_s : SegmentClassifier.display_type(type).to_s
+      { value: value, type: type, output: "{#{placeholder}}", notes: notes }
+    end
+
+    def render_query_row(name, value, evidence)
+      notes        = collect_notes(evidence)
+      cls          = find_evidence(evidence, source: :recognizer)
+      effective    = cls ? cls.payload[:type] : SegmentClassifier::DEFAULT.classify(value)
+      canon_policy = find_payload(evidence, :canonical_date) || find_payload(evidence, :canonical_currency)
+
+      output =
+        if canon_policy
+          canon_policy[:after]
+        elsif SegmentClassifier::DEFAULT.variable?(effective)
+          "{#{SegmentClassifier.display_type(effective)}}"
+        else
+          value
+        end
+
+      { name: name, value: value, type: effective, output: output, notes: notes }
+    end
+
+    # ── Helpers ──────────────────────────────────────────────────────────
+
+    def coerce(input)
+      input.is_a?(Identifier) ? input : Parser.parse(input)
+    end
+
+    def urn_parts(iri)
+      return [] unless iri.nss
+      iri.nss.include?(":") ? iri.nss.split(":", 2) : [iri.nss]
+    end
+
+    def collect_notes(evidence)
+      evidence.flat_map(&:notes)
+    end
+
+    def find_evidence(evidence, source:)
+      evidence.find { |r| r.source == source }
+    end
+
+    def find_payload(evidence, rule)
+      r = evidence.find { |e| e.source == :policy && e.payload[:rule] == rule }
+      r&.payload
+    end
+
+    def would_be_hint(segments, idx, type, classifier)
+      return nil if idx.zero?
+
+      prev = segments[idx - 1]
+      return nil unless classifier.classify(prev) == :literal
+
+      base   = Inflector.singularize(prev)
+      suffix = type == :uuid ? "_uuid" : "_id"
+      "#{base}#{suffix}"
+    end
+  end
+end

data/lib/iriq/version.rb

@@ -1,3 +1,3 @@
 module Iriq
-  VERSION = "0.1.0"
+  VERSION = "0.30.2"
 end

data/lib/iriq.rb

@@ -3,15 +3,33 @@ require "iriq/errors"
 require "iriq/inflector"
 require "iriq/identifier"
 require "iriq/parser"
+require "iriq/specificity"
+require "iriq/recognizer"
+require "iriq/recognizers/uuid"
+require "iriq/recognizers/date"
+require "iriq/recognizers/integer"
 require "iriq/segment_classifier"
 require "iriq/segment_hints"
+require "iriq/shape"
 require "iriq/path_shape"
 require "iriq/normalizer"
 require "iriq/explanation"
+require "iriq/evidence"
+require "iriq/trace"
 require "iriq/cluster"
 require "iriq/clusterer"
 require "iriq/position_stats"
+require "set"
+
 require "iriq/observation"
+require "iriq/position"
+require "iriq/event"
+require "iriq/reducer"
+require "iriq/registrable_domain"
+require "iriq/storage"
+require "iriq/recognizer_proposal"
+require "iriq/synthesized_recognizer"
+require "iriq/cross_host_shape"
 require "iriq/corpus"
 require "iriq/extractor"
 require "iriq/cli"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iriq
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.30.2
 platform: ruby
 authors:
 - Daniel Pepper
@@ -65,18 +65,37 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.22'
-description: Semantic IRI/URI/URL/URN parsing, normalization, classification, and
-  clustering.
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: IRI extraction, normalization, and clustering.
 executables:
 - iriq
 extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
+- CLAUDE.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
+- Makefile
 - README.md
+- completions/_iriq
+- completions/iriq.bash
+- docs/ARCHITECTURE.md
+- docs/ROADMAP.md
 - exe/iriq
 - iriq.gemspec
 - lib/iriq.rb
@@ -84,7 +103,10 @@ files:
 - lib/iriq/cluster.rb
 - lib/iriq/clusterer.rb
 - lib/iriq/corpus.rb
+- lib/iriq/cross_host_shape.rb
 - lib/iriq/errors.rb
+- lib/iriq/event.rb
+- lib/iriq/evidence.rb
 - lib/iriq/explanation.rb
 - lib/iriq/extractor.rb
 - lib/iriq/identifier.rb
@@ -93,12 +115,26 @@ files:
 - lib/iriq/observation.rb
 - lib/iriq/parser.rb
 - lib/iriq/path_shape.rb
+- lib/iriq/position.rb
 - lib/iriq/position_stats.rb
+- lib/iriq/recognizer.rb
+- lib/iriq/recognizer_proposal.rb
+- lib/iriq/recognizers/date.rb
+- lib/iriq/recognizers/integer.rb
+- lib/iriq/recognizers/uuid.rb
+- lib/iriq/reducer.rb
+- lib/iriq/registrable_domain.rb
 - lib/iriq/segment_classifier.rb
 - lib/iriq/segment_hints.rb
+- lib/iriq/shape.rb
+- lib/iriq/specificity.rb
+- lib/iriq/storage.rb
+- lib/iriq/storage/json.rb
+- lib/iriq/storage/memory.rb
+- lib/iriq/storage/sqlite.rb
+- lib/iriq/synthesized_recognizer.rb
+- lib/iriq/trace.rb
 - lib/iriq/version.rb
-- script/benchmark.rb
-- script/memory.rb
 homepage: https://github.com/dpep/iriq
 licenses:
 - MIT
@@ -110,14 +146,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.11
 specification_version: 4
-summary: Semantic IRI normalization and clustering.
+summary: IRI extraction, normalization, and clustering.
 test_files: []

data/script/benchmark.rb

@@ -1,81 +0,0 @@
-#!/usr/bin/env ruby
-# Performance benchmark for the main hot paths in Iriq.
-#
-# Usage:
-#   bundle exec script/benchmark.rb              # default sizes
-#   bundle exec script/benchmark.rb 50000        # custom "large" size
-#
-# Inputs are generated deterministically from IriGenerator so results are
-# comparable across runs.
-
-require "benchmark"
-require "tempfile"
-
-$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
-$LOAD_PATH.unshift File.expand_path("../spec/support", __dir__)
-require "iriq"
-require "iri_generator"
-
-LARGE = Integer(ARGV[0] || 10_000)
-SMALL = [LARGE / 10, 1_000].min
-HUGE  = LARGE * 10
-
-puts "Iriq benchmark — Ruby #{RUBY_VERSION}, Iriq #{Iriq::VERSION}"
-puts "Sizes: small=#{SMALL}, large=#{LARGE}, huge=#{HUGE}"
-puts
-
-small_urls = IriGenerator.urls(count: SMALL, seed: 1)
-large_urls = IriGenerator.urls(count: LARGE, seed: 1)
-huge_urls  = IriGenerator.urls(count: HUGE,  seed: 1)
-
-# ~ LARGE URLs embedded in prose
-text_blob = small_urls.map { |u| "Some prose about #{u} here, also random words." }.join(" ") * (LARGE / SMALL)
-puts "Text blob: #{text_blob.bytesize / 1024} KB (~#{LARGE} URLs embedded)"
-puts
-
-results = {}
-Benchmark.bm(42) do |x|
-  results[:parse]     = x.report("parse #{LARGE} URLs")                  { large_urls.each { |u| Iriq.parse(u) } }
-  results[:normalize] = x.report("normalize #{LARGE} URLs (deterministic)") { large_urls.each { |u| Iriq.normalize(u) } }
-  results[:explain]   = x.report("explain #{LARGE} URLs (deterministic)")   { large_urls.each { |u| Iriq.explain(u) } }
-  results[:extract]   = x.report("extract from ~#{text_blob.bytesize / 1024} KB text")     { Iriq.extract(text_blob) }
-
-  results[:observe_small] = x.report("Corpus.observe #{SMALL} URLs") do
-    c = Iriq::Corpus.new
-    small_urls.each { |u| c.observe(u) }
-  end
-  results[:observe_large] = x.report("Corpus.observe #{LARGE} URLs") do
-    c = Iriq::Corpus.new
-    large_urls.each { |u| c.observe(u) }
-  end
-  results[:observe_huge] = x.report("Corpus.observe #{HUGE} URLs") do
-    c = Iriq::Corpus.new
-    huge_urls.each { |u| c.observe(u) }
-  end
-
-  results[:roundtrip] = x.report("Corpus save+load (#{LARGE} observations)") do
-    c = Iriq::Corpus.new
-    large_urls.each { |u| c.observe(u) }
-    Tempfile.open(["iriq-bench", ".json"]) do |f|
-      c.save(f.path)
-      Iriq::Corpus.load(f.path)
-    end
-  end
-end
-
-puts
-puts "Throughput summary:"
-[
-  [:parse,         LARGE, "URLs/s"],
-  [:normalize,     LARGE, "URLs/s"],
-  [:explain,       LARGE, "URLs/s"],
-  [:observe_small, SMALL, "URLs/s"],
-  [:observe_large, LARGE, "URLs/s"],
-  [:observe_huge,  HUGE,  "URLs/s"],
-].each do |key, n, unit|
-  per_sec = n / results[key].real
-  printf("  %-30s %12s %s\n", key, per_sec.round.to_s, unit)
-end
-
-extract_mb = text_blob.bytesize / (1024.0 * 1024.0)
-printf("  %-30s %12s MB/s\n", :extract, (extract_mb / results[:extract].real).round(2).to_s)

data/script/memory.rb

@@ -1,121 +0,0 @@
-#!/usr/bin/env ruby
-# Memory profile for the main code paths in Iriq.
-#
-# Usage:
-#   bundle exec script/memory.rb              # default sizes
-#   bundle exec script/memory.rb 50000        # custom corpus size
-#
-# Reports retained memory per operation, cache footprints, and memory
-# growth across corpus sizes (to verify linear scaling — no leaks).
-
-require "objspace"
-
-$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
-$LOAD_PATH.unshift File.expand_path("../spec/support", __dir__)
-require "iriq"
-require "iri_generator"
-
-CORPUS_SIZE = Integer(ARGV[0] || 10_000)
-SIZES       = [1_000, 10_000, 100_000].uniq.sort
-SIZES << CORPUS_SIZE unless SIZES.include?(CORPUS_SIZE)
-SIZES.sort!
-
-# Bytes → KB / MB string for display.
-def fmt_bytes(n)
-  if n < 1024
-    "#{n} B"
-  elsif n < 1024 * 1024
-    format("%.1f KB", n / 1024.0)
-  else
-    format("%.2f MB", n / (1024.0 * 1024.0))
-  end
-end
-
-# Run a block in isolation: GC before + after, return delta in bytes.
-def measure_retained(&block)
-  GC.start
-  before = ObjectSpace.memsize_of_all
-  result = block.call
-  GC.start
-  after  = ObjectSpace.memsize_of_all
-  [after - before, result]
-end
-
-# Reset caches so each scenario starts clean.
-def reset_caches
-  Iriq::SegmentClassifier::DEFAULT.instance_variable_get(:@cache).clear
-  Iriq::Inflector.instance_variable_get(:@cache)&.clear
-end
-
-puts "Iriq memory profile — Ruby #{RUBY_VERSION}, Iriq #{Iriq::VERSION}"
-puts
-
-# -- Section 1: memory growth across corpus sizes --
-puts "── corpus retained memory by N (verifies linear growth) ──"
-printf("  %-12s %-14s %-14s %-10s\n", "N obs", "retained", "per obs", "allocs")
-SIZES.each do |n|
-  reset_caches
-  urls = IriGenerator.urls(count: n, seed: 1)
-  alloc_before = GC.stat(:total_allocated_objects)
-  retained, _ = measure_retained do
-    c = Iriq::Corpus.new
-    urls.each { |u| c.observe(u) }
-    c
-  end
-  alloc_total = GC.stat(:total_allocated_objects) - alloc_before
-  printf("  %-12s %-14s %-14s %-10s\n", n, fmt_bytes(retained), fmt_bytes(retained / n), alloc_total)
-end
-puts
-
-# -- Section 2: corpus state breakdown at CORPUS_SIZE --
-puts "── corpus state breakdown at N=#{CORPUS_SIZE} ──"
-reset_caches
-urls = IriGenerator.urls(count: CORPUS_SIZE, seed: 1)
-corpus = Iriq::Corpus.new
-urls.each { |u| corpus.observe(u) }
-puts "  unique hosts:           #{corpus.host_counts.size}"
-puts "  unique fingerprints:    #{corpus.fingerprint_counts.size}"
-puts "  unique raw shapes:      #{corpus.raw_shape_counts.size}"
-puts "  clusters:               #{corpus.size}"
-puts "  position_stats entries: #{corpus.position_stats.size}"
-puts "  total observed values:  #{corpus.position_stats.sum { |_, s| s.value_counts.size }}"
-puts
-
-# -- Section 3: cache footprints --
-puts "── memoization caches ──"
-classifier_cache = Iriq::SegmentClassifier::DEFAULT.instance_variable_get(:@cache)
-inflector_cache  = Iriq::Inflector.instance_variable_get(:@cache) || {}
-puts "  classifier cache: #{classifier_cache.size} entries (cap #{Iriq::SegmentClassifier::CACHE_MAX})"
-puts "  inflector cache:  #{inflector_cache.size} entries (cap #{Iriq::Inflector::CACHE_MAX})"
-puts
-
-# -- Section 4: per-operation memory cost --
-puts "── retained memory per operation (N=#{CORPUS_SIZE}) ──"
-urls = IriGenerator.urls(count: CORPUS_SIZE, seed: 1)
-text_blob = urls.map { |u| "Some prose about #{u} here." }.join(" ")
-
-[
-  ["parse #{CORPUS_SIZE} URLs (discarded after)", ->{ urls.each { |u| Iriq.parse(u) } }],
-  ["normalize #{CORPUS_SIZE} URLs",               ->{ urls.each { |u| Iriq.normalize(u) } }],
-  ["explain #{CORPUS_SIZE} URLs",                 ->{ urls.each { |u| Iriq.explain(u) } }],
-  ["extract from #{fmt_bytes(text_blob.bytesize)} prose", ->{ Iriq.extract(text_blob) }],
-  ["Corpus.observe #{CORPUS_SIZE} URLs",          ->{ c = Iriq::Corpus.new; urls.each { |u| c.observe(u) }; c }],
-].each do |label, op|
-  reset_caches
-  retained, _ = measure_retained(&op)
-  printf("  %-50s %s\n", label, fmt_bytes(retained))
-end
-puts
-
-# -- Section 5: persistence overhead --
-puts "── save/load roundtrip (N=#{CORPUS_SIZE}) ──"
-require "tempfile"
-reset_caches
-corpus = Iriq::Corpus.new
-urls.each { |u| corpus.observe(u) }
-Tempfile.open(["iriq-mem", ".json"]) do |f|
-  corpus.save(f.path)
-  bytes = File.size(f.path)
-  puts "  JSON file on disk:  #{fmt_bytes(bytes)}"
-  puts "  ratio:              #{format("%.2f bytes/obs", bytes.to_f / CORPUS_SIZE)}"
-end