iriq 0.1.0 → 0.30.2

data/lib/iriq/recognizer_proposal.rb

@@ -0,0 +1,167 @@
+require "set"
+
+module Iriq
+  # A suggestion that a new Recognizer should be added to the system.
+  #
+  # Emitted by Corpus#propose_recognizers. NOT automatically activated —
+  # proposals carry enough evidence for a human to judge whether to add
+  # the Recognizer to the built-in set (or, later, to register it
+  # dynamically via a public Recognizer registry).
+  #
+  #   prefix             — the detected shape signature (e.g. "ghp_")
+  #   suggested_type     — Symbol name we'd register the Recognizer under
+  #                        if accepted (e.g. :ghp)
+  #   positions          — every Position where the proposal matched
+  #   hosts              — distinct hosts the proposal was seen at; a high
+  #                        count is strong evidence the pattern isn't
+  #                        host-local
+  #   coverage           — fraction of sampled observations at affected
+  #                        Positions matching the proposal pattern
+  #   observation_count  — total matching observations across positions
+  #   sample_values      — up to 5 example matches, for the human reviewer
+  #   strategy           — the ProposalStrategy that emitted this record
+  class RecognizerProposal
+    attr_reader :prefix, :suggested_type, :positions, :hosts,
+                :coverage, :confidence, :observation_count,
+                :sample_values, :strategy
+
+    def initialize(prefix:, suggested_type:, positions:, hosts:,
+                   coverage:, observation_count:, sample_values:,
+                   strategy:, confidence: nil)
+      @prefix            = prefix
+      @suggested_type    = suggested_type
+      @positions         = positions.freeze
+      @hosts             = hosts.is_a?(Set) ? hosts.dup.freeze : Set.new(hosts).freeze
+      @coverage          = coverage
+      @observation_count = observation_count
+      @sample_values     = sample_values.freeze
+      @strategy          = strategy
+      @confidence        = confidence.nil? ? compute_confidence : confidence
+    end
+
+    def to_h
+      {
+        prefix:            @prefix,
+        suggested_type:    @suggested_type,
+        positions:         @positions.map(&:to_h),
+        hosts:             @hosts.to_a.sort,
+        coverage:          @coverage,
+        confidence:        @confidence,
+        observation_count: @observation_count,
+        sample_values:     @sample_values,
+        strategy:          @strategy,
+      }
+    end
+
+    private
+
+    # Confidence = coverage + linear cross-host boost, capped at 1.0.
+    # Single-host proposals get their raw coverage as confidence (no
+    # boost). Each additional host adds CROSS_HOST_BOOST_PER_HOST to
+    # the score. A proposal supported by ~10 distinct hosts caps out
+    # regardless of raw coverage; below that, both signals compose.
+    def compute_confidence
+      boost = (@hosts.size - 1) * ProposalStrategy::CROSS_HOST_BOOST_PER_HOST
+      score = @coverage + boost
+      score > 1.0 ? 1.0 : score
+    end
+  end
+
+  # Pluggable proposal-detection strategies. Each strategy.propose(storage, **opts)
+  # returns an array of RecognizerProposal. Adding a new detection rule =
+  # add a class with #propose; register it via DEFAULTS.
+  module ProposalStrategy
+    # Default minimum total matching observations across positions before
+    # we'll emit a proposal. Below this the signal is too noisy.
+    DEFAULT_MIN_OBSERVATIONS = 20
+    # Fraction of sampled observations at affected Positions that must
+    # match the proposal pattern.
+    DEFAULT_MIN_COVERAGE     = 0.7
+    # Minimum number of distinct hosts the proposal must appear at. For
+    # single-host corpora this defaults to 1; bumping to 2+ promotes
+    # cross-host patterns over host-local ones.
+    DEFAULT_MIN_HOSTS        = 1
+    # Confidence boost added per additional host beyond the first. A
+    # pattern seen on 10+ hosts caps out the boost (+0.45 ≈ 1.0 when
+    # combined with any reasonable coverage); single-host patterns get
+    # no boost (their coverage IS their confidence).
+    CROSS_HOST_BOOST_PER_HOST = 0.05
+
+    # Detects `<prefix>_<alphanumeric>` patterns at slug/opaque_id
+    # positions — the GitHub PAT (`ghp_…`), Stripe customer ID (`cus_…`),
+    # AWS-style (`sk_test_…` — partial match), Twilio SID-with-letter-
+    # prefix family. Restricting the suffix to alphanumeric (no further
+    # separators) keeps real slugs (`my-cool-post`, `red_team_member`)
+    # from triggering false proposals.
+    class PrefixUnderscoreId
+      PATTERN = /\A([a-z]+)_([A-Za-z0-9]+)\z/.freeze
+      NAME    = :prefix_underscore_id
+
+      def propose(storage,
+                  min_observations: DEFAULT_MIN_OBSERVATIONS,
+                  min_coverage:     DEFAULT_MIN_COVERAGE,
+                  min_hosts:        DEFAULT_MIN_HOSTS)
+        per_prefix = Hash.new { |h, k| h[k] = empty_accumulator }
+
+        storage.each_position_stats do |position, stats|
+          next unless slug_or_opaque?(stats)
+
+          stats.value_counts.each do |value, count|
+            m = PATTERN.match(value) or next
+            prefix = "#{m[1]}_"
+            acc = per_prefix[prefix]
+            acc[:matching_count] += count
+            acc[:position_observations] += stats.total unless acc[:positions].include?(position)
+            acc[:positions] << position
+            acc[:hosts] << position.host
+            # Collect every match; we'll sort + cap to a stable top-N at
+            # emission time so Ruby and Go produce identical samples
+            # regardless of underlying Hash / map iteration order.
+            acc[:matches] << value
+          end
+        end
+
+        per_prefix.filter_map { |prefix, acc|
+          next nil if acc[:matching_count] < min_observations
+          next nil if acc[:hosts].size < min_hosts
+
+          coverage = acc[:matching_count].to_f / acc[:position_observations]
+          next nil if coverage < min_coverage
+
+          RecognizerProposal.new(
+            prefix:            prefix,
+            suggested_type:    prefix.chomp("_").to_sym,
+            positions:         acc[:positions].to_a,
+            hosts:             acc[:hosts],
+            coverage:          coverage,
+            observation_count: acc[:matching_count],
+            # Sort + cap to 5 so Ruby and Go produce identical samples
+            # regardless of underlying Hash / map iteration order. The
+            # samples are illustrative for humans; alphabetical is fine.
+            sample_values:     acc[:matches].sort.first(5),
+            strategy:          NAME,
+          )
+        }.sort_by { |p| [-p.confidence, p.prefix] }
+      end
+
+      private
+
+      def empty_accumulator
+        {
+          positions:             Set.new,
+          hosts:                 Set.new,
+          matching_count:        0,
+          position_observations: 0,
+          matches:               [],
+        }
+      end
+
+      def slug_or_opaque?(stats)
+        dom = stats.type_counts.max_by { |_, c| c }&.first
+        dom == :slug || dom == :opaque_id
+      end
+    end
+
+    DEFAULTS = [PrefixUnderscoreId.new].freeze
+  end
+end

data/lib/iriq/recognizers/date.rb

@@ -0,0 +1,53 @@
+module Iriq
+  module Recognizers
+    # ISO 8601 (YYYY-MM-DD), slash form (YYYY/MM/DD), and US-style
+    # (M/D/YYYY) date shapes. Compact YYYYMMDD lives on the Integer
+    # recognizer — it sees the digits-only input first.
+    #
+    # Conservative: DD/MM/YYYY is intentionally NOT recognized — from a
+    # bare segment we can't tell it apart from MM/DD/YYYY.
+    class Date < Recognizer
+      ISO_PATTERN     = /\A\d{4}-\d{2}-\d{2}\z/.freeze
+      SLASH_PATTERN   = %r{\A\d{4}/\d{2}/\d{2}\z}.freeze
+      US_PATTERN      = %r{\A(\d{1,2})/(\d{1,2})/(\d{4})\z}.freeze
+
+      def try(segment)
+        has_dash  = segment.include?("-")
+        has_slash = segment.include?("/")
+        return nil unless has_dash || has_slash
+        unless ISO_PATTERN.match?(segment) ||
+               SLASH_PATTERN.match?(segment) ||
+               US_PATTERN.match?(segment)
+          return nil
+        end
+
+        { type: :date, confidence: 1.0, specificity: Specificity::STRUCTURED }
+      end
+
+      # Canonicalize a recognized date to ISO 8601 (YYYY-MM-DD). nil for
+      # non-date / implausible-date values. Day-of-month validity (Feb 30,
+      # Apr 31) deliberately not checked — out of scope for a heuristic.
+      def self.canonical(value)
+        return nil if value.nil?
+
+        case value
+        when ISO_PATTERN
+          plausible?(value[0, 4], value[5, 2], value[8, 2]) ? value : nil
+        when SLASH_PATTERN
+          plausible?(value[0, 4], value[5, 2], value[8, 2]) ? value.tr("/", "-") : nil
+        when US_PATTERN
+          m = ::Regexp.last_match
+          mm, dd, yyyy = m[1].rjust(2, "0"), m[2].rjust(2, "0"), m[3]
+          plausible?(yyyy, mm, dd) ? "#{yyyy}-#{mm}-#{dd}" : nil
+        end
+      end
+
+      def self.plausible?(y, m, d)
+        yi = y.to_i; mi = m.to_i; di = d.to_i
+        yi.between?(1900, 2100) && mi.between?(1, 12) && di.between?(1, 31)
+      end
+    end
+
+    DATE = Date.new
+  end
+end

data/lib/iriq/recognizers/integer.rb

@@ -0,0 +1,37 @@
+module Iriq
+  module Recognizers
+    # Base-10 integer. Also returns :timestamp for plausible UNIX seconds /
+    # millis ranges, and :date for plausible YYYYMMDD compact dates — these
+    # share the digit-only lexical shape, and we want the most specific type.
+    class Integer < Recognizer
+      PATTERN              = /\A\d+\z/.freeze
+      COMPACT_DATE_PATTERN = /\A\d{8}\z/.freeze
+      TS_SECONDS_RANGE     = 1_000_000_000..9_999_999_999
+      TS_MILLIS_RANGE      = 1_000_000_000_000..9_999_999_999_999
+
+      def try(segment)
+        first  = segment.getbyte(0)
+        digit0 = first && first >= 0x30 && first <= 0x39
+        return nil unless digit0 && PATTERN.match?(segment)
+
+        n = segment.to_i
+        if TS_MILLIS_RANGE.cover?(n) || TS_SECONDS_RANGE.cover?(n)
+          return { type: :timestamp, confidence: 1.0, specificity: Specificity::BOUNDED }
+        end
+
+        if COMPACT_DATE_PATTERN.match?(segment)
+          y = segment[0, 4].to_i
+          m = segment[4, 2].to_i
+          d = segment[6, 2].to_i
+          if y.between?(1900, 2100) && m.between?(1, 12) && d.between?(1, 31)
+            return { type: :date, confidence: 1.0, specificity: Specificity::STRUCTURED }
+          end
+        end
+
+        { type: :integer, confidence: 1.0, specificity: Specificity::TYPED }
+      end
+    end
+
+    INTEGER = Integer.new
+  end
+end

data/lib/iriq/recognizers/uuid.rb

@@ -0,0 +1,16 @@
+module Iriq
+  module Recognizers
+    # RFC 4122 UUID. Shape-only — does not validate version/variant bits.
+    class Uuid < Recognizer
+      PATTERN = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/.freeze
+
+      def try(segment)
+        return nil unless segment.size == 36 && segment.include?("-") && PATTERN.match?(segment)
+
+        { type: :uuid, confidence: 1.0, specificity: Specificity::SEMANTIC }
+      end
+    end
+
+    UUID = Uuid.new
+  end
+end

data/lib/iriq/reducer.rb

@@ -0,0 +1,37 @@
+module Iriq
+  # Reducers consume the Event stream emitted by Corpus#observe and update
+  # the storage backend's materialized views. Each Reducer is a callable
+  # that takes (event, storage) and applies the appropriate storage
+  # operation; non-applicable events are no-ops via the EVENT_TYPES gate.
+  #
+  # Adding a new metric is: define a new Event subtype, write a Reducer that
+  # handles it, register it in DEFAULTS — no other module changes.
+  module Reducer
+    # Each entry: { event_class => [lambda(event, storage) -> result] }.
+    # Lambdas may return the result of the underlying storage call so
+    # callers (Corpus#observe) can pick up the cluster they need to return.
+    DEFAULTS = {
+      Event::HostSeen        => [->(e, s) { s.increment_host(e.host) }],
+      Event::PathLengthSeen  => [->(e, s) { s.increment_path_length(e.length) }],
+      Event::RawShapeSeen    => [->(e, s) { s.increment_raw_shape(e.shape) }],
+      Event::FingerprintSeen => [->(e, s) { s.increment_fingerprint(e.shape) }],
+      Event::PositionSeen    => [->(e, s) { s.observe_position(e.position, e.value, e.type) }],
+      Event::ClusterAddition => [->(e, s) { s.add_to_cluster(e.key, e.host, e.scheme, e.shape, e.identifier) }],
+    }.freeze
+
+    module_function
+
+    # Apply the event to the storage via all Reducers registered for its
+    # type. Returns the last non-nil reducer result — used by Corpus#observe
+    # to pick up the Cluster created/updated by Event::ClusterAddition.
+    def apply(event, storage, reducers: DEFAULTS)
+      results = reducers.fetch(event.class, [])
+      result  = nil
+      results.each do |r|
+        rv = r.call(event, storage)
+        result = rv unless rv.nil?
+      end
+      result
+    end
+  end
+end

data/lib/iriq/registrable_domain.rb

@@ -0,0 +1,56 @@
+module Iriq
+  # Heuristic registrable-domain extractor. Strips subdomains so that
+  # api.foo.com and app.foo.com both resolve to foo.com.
+  #
+  # Uses an inline allowlist of the ~50 most common multi-label public
+  # suffixes (.co.uk, .com.au, .gov.uk, etc.) — covers the long tail of
+  # real-world traffic without the ~3 MB cost of bundling the full Public
+  # Suffix List. Niche multi-label TLDs (.priv.no, .tas.gov.au, etc.) will
+  # be over-stripped; install the `public_suffix` gem and wire it in if
+  # accuracy on those matters for your workload.
+  module RegistrableDomain
+    # rubocop:disable Layout/LineLength
+    TWO_LABEL_SUFFIXES = %w[
+      co.uk org.uk gov.uk ac.uk net.uk me.uk ltd.uk plc.uk sch.uk
+      co.jp ac.jp or.jp ne.jp go.jp gr.jp ed.jp lg.jp
+      com.au net.au org.au edu.au gov.au asn.au id.au
+      co.nz net.nz org.nz govt.nz ac.nz school.nz
+      com.br net.br org.br gov.br edu.br
+      com.cn net.cn org.cn gov.cn edu.cn ac.cn
+      co.za net.za org.za gov.za ac.za
+      co.kr ne.kr or.kr re.kr go.kr ac.kr
+      co.in net.in org.in gov.in ac.in
+      co.il net.il org.il gov.il ac.il muni.il
+      com.mx net.mx org.mx gob.mx edu.mx
+      com.ar net.ar org.ar gov.ar
+      com.hk net.hk org.hk gov.hk edu.hk
+      com.tw net.tw org.tw gov.tw edu.tw
+      com.sg net.sg org.sg gov.sg edu.sg per.sg
+      com.tr net.tr org.tr gov.tr edu.tr k12.tr
+    ].to_set.freeze
+    # rubocop:enable Layout/LineLength
+
+    IP_V4_RE = /\A\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/.freeze
+
+    module_function
+
+    # Given an authority (hostname, no port), return the registrable
+    # domain. Returns the input unchanged for IP literals, single-label
+    # hosts (`localhost`), and hosts that already match a 2-label apex.
+    def for(host)
+      return host if host.nil? || host.empty?
+      return host if IP_V4_RE.match?(host)
+
+      labels = host.split(".")
+      return host if labels.size <= 2
+
+      tail_two = labels.last(2).join(".")
+      if TWO_LABEL_SUFFIXES.include?(tail_two)
+        # Multi-label public suffix — keep last 3 labels (`foo.co.uk`).
+        labels.last(3).join(".")
+      else
+        labels.last(2).join(".")
+      end
+    end
+  end
+end