insika 0.2.0 → 0.3.0

data/lib/insika/funnel_store.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Insika
+  # the durable aggregates of the OUTCOME FUNNEL — per-day stage
+  # counts (the fold's cells), the fold cursor and the baseline snapshot.
+  # A dumb domain store: it holds no outcome_store and never folds (C4 owns the
+  # transformation), and it is recomputable by construction — the OutcomeStore
+  # stays the source of truth.
+  #
+  # Key shapes (string keys, Store-contract JSON):
+  #   "funnel"           "acme:store-support:2026-08-14"  -> { "greeted" => 41, "paid" => 3 }
+  #   "funnel_cursor"    "acme:store-support"             -> { "at" => ISO8601 | nil, "ids" => [uuid, …] }
+  #   "funnel_baseline"  "acme:store-support"             -> { from/to/stages/primary/…/frozen_at }
+  #
+  # The no-tenant case is the literal "platform" (the outcome KEY's rule), so
+  # funnel and outcome keys share the same tenant segment and the purge prefix
+  # scan is the same string. A record's `tenant` field is `tenant.to_s` — "" for
+  # a single-tenant write — so every key-builder normalizes a blank tenant to
+  # "platform" HERE (one place), or a single-tenant fold lands in an ""-prefixed
+  # cell that purge("platform") never removes.
+  class FunnelStore
+    SCOPE          = "funnel"          # day cells
+    CURSOR_SCOPE   = "funnel_cursor"   # per (tenant, agent)
+    BASELINE_SCOPE = "funnel_baseline" # per (tenant, agent)
+
+    def initialize(store:)
+      @store = store
+    end
+
+    # Cumulative increment (D2): `counts` is the fold's { stage => 1 } hash for
+    # the reached prefix stages[0..index] — the STORE stays free of the
+    # declaration (D1), it only accumulates. Bumped in ONE transaction.
+    # -> { stage => count } the NEW day counts (string keys, declared order).
+    def add(tenant:, agent:, at:, counts:)
+      id = pair_id(tenant, agent)
+      day_key = day_segment(at)
+      @store.transaction do
+        cell = @store.get(SCOPE, "#{id}:#{day_key}") || {}
+        counts.each { |stage, n| cell[stage.to_s] = cell[stage.to_s].to_i + n }
+        @store.set(SCOPE, "#{id}:#{day_key}", cell)
+        cell
+      end
+    end
+
+    # -> { stage => count } | {} — one day's cell ("YYYY-MM-DD").
+    def day(tenant:, agent:, day:)
+      @store.get(SCOPE, "#{pair_id(tenant, agent)}:#{day}") || {}
+    end
+
+    # -> { "YYYY-MM-DD" => { stage => count } } sorted ascending, bounded by
+    # ISO-date strings from:/to: (inclusive). Empty hash when none.
+    def days(tenant:, agent:, from: nil, to: nil)
+      prefix = "#{pair_id(tenant, agent)}:"
+      @store.list(SCOPE).select { |k| k.start_with?(prefix) }.each_with_object({}) do |k, acc|
+        day = k.delete_prefix(prefix)
+        next unless within?(day, from, to)
+
+        acc[day] = @store.get(SCOPE, k)
+      end.sort.to_h
+    end
+
+    # The fold cursor of the pair. -> { "at" => String | nil, "ids" => [String] }
+    def cursor(tenant:, agent:)
+      record = @store.get(CURSOR_SCOPE, pair_id(tenant, agent))
+      { "at" => record && record["at"], "ids" => Array(record && record["ids"]) }
+    end
+
+    def set_cursor(tenant:, agent:, at:, ids:)
+      @store.set(CURSOR_SCOPE, pair_id(tenant, agent), { "at" => at, "ids" => Array(ids) })
+    end
+
+    # -> Hash | nil — the current baseline record (D5), read back verbatim.
+    def baseline(tenant:, agent:)
+      @store.get(BASELINE_SCOPE, pair_id(tenant, agent))
+    end
+
+    # Overwrites (D5 — one current snapshot per pair, no history).
+    def set_baseline(tenant:, agent:, record:)
+      @store.set(BASELINE_SCOPE, pair_id(tenant, agent), record)
+    end
+
+    # Purge one tenant: its day cells, cursors and baselines (DeleteTenantData,
+    # WS8 — the tenant is the FIRST key segment, so a prefix scan; the key IS
+    # the isolation). -> count removed.
+    def purge(tenant:)
+      prefix = "#{tenant_id(tenant)}:"
+      [SCOPE, CURSOR_SCOPE, BASELINE_SCOPE].sum do |scope|
+        keys = @store.list(scope).select { |k| k.start_with?(prefix) }
+        keys.each { |k| @store.delete(scope, k) }
+        keys.size
+      end
+    end
+
+    # Age-based prune of DAY CELLS ONLY (retention, WS8 — outcomes and their
+    # fold die together). The day is the key's last segment ("YYYY-MM-DD",
+    # lexicographic). Cursors/baselines are tiny and live while their agent
+    # does. -> count removed.
+    def delete_older_than(time)
+      cutoff = time.utc.strftime("%Y-%m-%d")
+      removed = 0
+      @store.list(SCOPE).each do |k|
+        day = k.rpartition(":").last
+        next unless day < cutoff
+
+        @store.delete(SCOPE, k)
+        removed += 1
+      end
+      removed
+    end
+
+    # wipes ONE pair's day cells — the recompute repair path (the
+    # fold rebuilds them from the outcome store, so it must start from zero,
+    # never sum on top). Same tenant normalization as every other key builder
+    # (`""`/nil/"platform" all reach the "platform" segment). -> count removed.
+    def delete_days(tenant:, agent:)
+      prefix = "#{pair_id(tenant, agent)}:"
+      keys = @store.list(SCOPE).select { |k| k.start_with?(prefix) }
+      keys.each { |k| @store.delete(SCOPE, k) }
+      keys.size
+    end
+
+    # The pairs that have any day cell — the Studio's derived drill (D7).
+    # -> [{ tenant: String | nil, agent: String }]
+    def pairs
+      @store.list(SCOPE).map { |k| pair_of(k) }.uniq
+    end
+
+    private
+
+    def pair_id(tenant, agent)
+      "#{tenant_id(tenant)}:#{agent}"
+    end
+
+    # The literal "platform" for a blank tenant — the outcome KEY's rule
+    # (outcome_store.rb's key), normalized in ONE place.
+    def tenant_id(tenant)
+      t = tenant.to_s
+      t.empty? ? "platform" : t
+    end
+
+    def day_segment(at)
+      t = at.is_a?(Time) ? at : Time.parse(at.to_s)
+      t.utc.strftime("%Y-%m-%d")
+    end
+
+    # Tolerant like the series reads: a malformed bound (not an ISO date) reads
+    # as unbounded — never a crash, never a silent empty window.
+    DATE_RE = /\A\d{4}-\d{2}-\d{2}\z/
+
+    def within?(day, from, to)
+      lo = DATE_RE.match?(from.to_s) ? from.to_s : nil
+      hi = DATE_RE.match?(to.to_s) ? to.to_s : nil
+      (lo.nil? || day >= lo) && (hi.nil? || day <= hi)
+    end
+
+    def pair_of(key)
+      tenant, agent, = key.split(":", 3)
+      { tenant: tenant == "platform" ? nil : tenant, agent: agent }
+    end
+  end
+end

data/lib/insika/golden_store.rb

@@ -20,7 +20,7 @@ module Insika
   # validated by `Evals::GoldenLoader.build` — the one validator, so a case authored in
   # the Studio and a case read from disk cannot diverge. `evals/golden/**` stays the
   # export format and the seed for a fresh deploy (`insika evals:import`), for the same
-  # reason the knowledge RFC keeps markdown as both: no converter to keep honest.
+  # reason the knowledge base keeps markdown as both: no converter to keep honest.
   #
   # No version history here, unlike the prompt/skill stores: the corpus on disk IS the
   # backup, and re-importing restores it.

data/lib/insika/grounding/matcher.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "../coercion"
+
+module Insika
+  #   — the pack's deterministic claim extractor + membership check.
+  # The pack's DATA (D7): a `sku` regex for the SKU shape. The engine only
+  # APPLIES it — it never guesses what a SKU looks like. No NLP, no LLM, no gem.
+  #
+  # REVIEW-DECISION (rfc-0029 v2): grounding is **SKU-only**. The techspec's
+  # name half (name_keys indexing ledger lines to flag names) cannot flag
+  # anything: a name found in the text that also came from the ledger is grounded
+  # by construction, and detecting a name that matches NO ledger line needs a
+  # "this is a product name" signal that does not exist without NLP. It was
+  # decorative config that lied — cut, not half-fixed. The ledger keeps ids;
+  # `line` stays in the lean envelope for the model's context, it just does not
+  # feed the matcher.
+  class GroundingMatcher
+    # Catastrophic-backtracking cap. A pathological pattern from the pack would
+    # otherwise spin the reactor's thread in C — a rescue cannot save a hang, and
+    # with_timeout cannot preempt a regex. The final answer is short; 1s is a
+    # hard ceiling, not a budget.
+    REGEX_TIMEOUT = 1.0
+
+    def initialize(sku: nil)
+      @sku = sku && Regexp.new(sku, timeout: REGEX_TIMEOUT) # validated at build; a re-raise here is a bug
+    end
+
+    # Is a SKU pattern actually configured? (The doctor's warning and the
+    # harvest's D3 refusal ask the same question — a matcher without a sku
+    # matches nothing and can verify no claim.)
+    def sku? = !@sku.nil?
+
+    # raw (the pack's matcher Hash) -> GroundingMatcher. The `sku` must compile
+    # and be length-capped. Raises ValidationError.
+    def self.build(raw)
+      h = Coercion.deep_stringify(raw || {})
+      sku = Coercion.presence(h["sku"])
+      if sku
+        begin
+          Regexp.new(sku)
+        rescue RegexpError
+          raise Insika::ValidationError, "grounding.matcher.sku does not compile: #{sku.inspect}"
+        end
+        if sku.bytesize > Grounding::SKU_MAX
+          raise Insika::ValidationError, "grounding.matcher.sku exceeds #{Grounding::SKU_MAX} chars"
+        end
+      end
+      new(sku: sku)
+    end
+
+    # -> [String] SKU-pattern matches in the text, deduped. Capture-group safe:
+    # scans the FULL match (Regexp.last_match(0)), so a pack pattern with groups
+    # or alternation can never inject nils or empty captures into the references
+    # (a nil claim would corrupt the flag detail and crash the sentence cut).
+    def references(text)
+      return [] unless @sku
+
+      text.to_enum(:scan, @sku).map { Regexp.last_match(0) }.uniq
+    end
+
+    # -> [String] references NOT in the evidence set. The SET is the ledger ids
+    # — a claim quoting a ledgered id is grounded; everything else flagged/cut.
+    def ungrounded(references, evidence_ids:)
+      known = Set.new(evidence_ids.map(&:to_s))
+      references.reject { |r| known.include?(r) }
+    end
+  end
+end

data/lib/insika/grounding.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "coercion"
+require_relative "grounding/matcher"
+
+module Insika
+  #   — the profile-facing half of grounding: parse the pack's
+  # `grounding` data into the per-turn Grounding object (mode + matcher).
+  #
+  # Data (on the AgentProfile):
+  #
+  #   { "mode" => "flag"|"enforce"|"off",
+  #     "matcher" => { "sku" => "\b[A-Z]{2,4}\d{4,8}\b" } }
+  #
+  # Absent = the whole feature is OFF (parity). The matcher is built ONCE per
+  # turn (profile parse) and cached on the Grounding object. A `sku` that does
+  # not compile, or exceeds 256 chars, is a ValidationError at build — the
+  # engine refuses only uncompileable data; a matcher with no sku matches
+  # nothing (grounding on is harmless but useless; doctor warns).
+  class Grounding
+    MODES = %w[flag enforce off].freeze
+    SKU_MAX = 256
+
+    attr_reader :mode, :matcher
+
+    # raw: the profile's `grounding` Hash | nil | false -> Grounding | nil.
+    # nil/false = off. A non-Hash (e.g. true) reads as an empty config.
+    def self.parse(raw)
+      return nil if raw.nil? || raw == false
+
+      h = Coercion.deep_stringify(raw.is_a?(Hash) ? raw : {})
+      mode = MODES.include?(h["mode"].to_s) ? h["mode"].to_s : "flag" # default :flag
+      new(mode: mode, matcher: GroundingMatcher.build(h["matcher"]))
+    end
+
+    def initialize(mode:, matcher:)
+      @mode = mode
+      @matcher = matcher
+    end
+
+    def enforce? = mode == "enforce"
+    def off? = mode == "off"
+  end
+end

data/lib/insika/harvest/conversion_gate.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "time"
+require "date"
+
+module Insika
+  module Harvest
+    # C8 — the   second gate: the store's target METRIC RATE over
+    # the criterion's window, compared to the frozen baseline's rate. Outcome
+    # is EVIDENCE — the only verdict this object can produce is "the store is
+    # measurably worse than the accepted state" and "there is nothing to
+    # compare against".
+    #
+    # The ruler is a RATE (metric ÷ first stage), never a total: the frozen
+    # baseline covers a ≥ 28-day span while the window is 72 h — comparing the
+    # raw counts would flag a store that tripled its traffic as "worse" (fewer
+    # absolute sales in 3 days than in 28). Both sides fold the same
+    # denominator, so the comparison measures the per-unit health, which is
+    # what the audit can actually read back.
+    #
+    # Refuse-with-a-named-reason, never pass, on missing data (the P18 lesson
+    # applied to the second ruler): no frozen baseline, no criterion, no
+    # funnel store, a fold that has not converged, a frozen span shorter than
+    # the criterion's `min_span`, a baseline whose rate cannot be computed.
+    class ConversionGate
+      Result = Data.define(:passed, :reason, :metric, :window, :current,
+                           :baseline, :threshold, :snapshot_ref) do
+        # The candidate's conversion_gate record — the operator's review card
+        # shows the ruler (both rates), never just a verdict.
+        def to_h
+          { "passed" => passed, "reason" => reason&.to_s, "metric" => metric,
+            "window" => window, "current" => current, "baseline" => baseline,
+            "threshold" => threshold, "snapshot_ref" => snapshot_ref }
+        end
+      end
+
+      # funnel_store: the   FunnelStore (nil = the gate REFUSES — a
+      # store without a ruler cannot promote on outcome, D6).
+      # criterion: the boot-loaded Harvest::Criterion (nil = refuse).
+      def initialize(funnel_store:, criterion:)
+        @funnel_store = funnel_store
+        @criterion = criterion
+      end
+
+      # -> Result. NEVER passes on missing data (D6).
+      def call(tenant:, agent:)
+        return refuse(:no_criterion) if @criterion.nil?
+        return refuse(:no_funnel) if @funnel_store.nil?
+
+        baseline = @funnel_store.baseline(tenant: tenant, agent: agent)
+        return refuse(:no_frozen_baseline) if baseline.nil?
+
+        # The criterion's metric is a stage NAME from the   declaration
+        # (D5). The seed token "primary" resolves to the frozen snapshot's
+        # primary stage — the declaration's own vocabulary, never a gem
+        # constant .
+        metric = resolve_metric(@criterion.rule.metric, baseline)
+        return refuse(:metric_mismatch) if metric != baseline["primary"].to_s
+
+        # min_span is LIVE (D5): the frozen baseline must cover the span the
+        # criterion pre-registered — a number nobody re-froze to a shorter
+        # span is not a number the gate may compare against.
+        return refuse(:baseline_span_short) if span_short?(baseline, @criterion.rule.min_span)
+
+        baseline_rate = rate_of(baseline["stages"] || {}, metric)
+        # A baseline whose conversion was nil when frozen (no first-stage
+        # events) cannot be the accepted state's ruler.
+        return refuse(:no_baseline_rate) if baseline_rate.nil?
+
+        window_hours = @criterion.rule.window.to_s.delete_suffix("h").to_i
+        window_stages = fold_window(tenant: tenant, agent: agent, hours: window_hours)
+        return refuse(:no_fold) if window_stages.empty?
+
+        current_rate = rate_of(window_stages, metric)
+        # The window folded no comparable denominator (no first-stage events
+        # over the window) — there is nothing to compare.
+        return refuse(:no_fold) if current_rate.nil?
+
+        threshold = @criterion.rule.threshold.to_f
+        snapshot_ref = "funnel:#{tenant_id(tenant)}:#{agent}:#{baseline['frozen_at']}"
+
+        if current_rate >= baseline_rate * (1 - threshold)
+          Result.new(passed: true, reason: nil, metric: metric,
+                     window: @criterion.rule.window.to_s, current: current_rate,
+                     baseline: baseline_rate, threshold: threshold,
+                     snapshot_ref: snapshot_ref)
+        else
+          Result.new(passed: false, reason: :conversion_down,
+                     metric: metric, window: @criterion.rule.window.to_s,
+                     current: current_rate, baseline: baseline_rate,
+                     threshold: threshold, snapshot_ref: snapshot_ref)
+        end
+      end
+
+      private
+
+      def refuse(reason)
+        Result.new(passed: false, reason: reason,
+                   metric: @criterion&.rule&.metric.to_s,
+                   window: @criterion&.rule&.window.to_s,
+                   current: nil, baseline: nil,
+                   threshold: @criterion&.rule&.threshold,
+                   snapshot_ref: nil)
+      end
+
+      # "primary" (the seed token) = the frozen snapshot's primary stage;
+      # anything else is a literal stage name.
+      def resolve_metric(token, baseline)
+        token.to_s == "primary" ? baseline["primary"].to_s : token.to_s
+      end
+
+      # metric ÷ FIRST stage — the same denominator both sides fold, so the
+      # totals' different spans cancel out. nil when the denominator is zero
+      # (the ruler cannot be expressed).
+      def rate_of(stages, metric)
+        first = stages.keys.first
+        denom = stages[first].to_f
+        return nil if denom.zero?
+
+        stages[metric].to_f / denom
+      end
+
+      # "28d" -> 28. The frozen span (from/to) must cover it — FreezeFunnelBaseline
+      # enforces the same number at freeze time; this is the gate's own check
+      # against a hand-written baseline or a moved criterion.
+      def span_short?(baseline, min_span)
+        return false if min_span.to_s.empty?
+
+        from = baseline["from"].to_s
+        to = baseline["to"].to_s
+        return true if from.empty? || to.empty?
+
+        days = (Date.iso8601(to) - Date.iso8601(from)).to_i
+        days < min_span.to_s.delete_suffix("d").to_i
+      rescue Date::Error
+        true
+      end
+
+      # The criterion's window folded over the day cells (the
+      # read) — the fold's day cells SUMMED by stage, so the gate can build
+      # the window's rate the same way the baseline built its own.
+      # -> { stage => count }
+      def fold_window(tenant:, agent:, hours:)
+        now = Time.now.utc
+        from = (now - hours * 3600).strftime("%Y-%m-%d")
+        to = now.strftime("%Y-%m-%d")
+        @funnel_store.days(tenant: tenant, agent: agent, from: from, to: to)
+                     .each_with_object(Hash.new(0)) do |(_day, counts), acc|
+          counts.each { |stage, n| acc[stage] += n }
+        end
+      end
+
+      def tenant_id(tenant)
+        t = tenant.to_s
+        t.empty? ? "platform" : t
+      end
+    end
+  end
+end

data/lib/insika/harvest/criterion.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "digest"
+require "yaml"
+
+module Insika
+  module Harvest
+    # C4 — the frozen conversion criterion, parsed from the deployment's
+    # criterion file. The parity shape (Parity::Criterion) copied: one
+    # file, one ```yaml fence, the WHOLE bytes hashed (an edit to the rationale
+    # invalidates the frozen rule too), strict keys, no defaults. A skill may
+    # land only when the store's ruler is not measurably worse than the
+    # accepted state — the FILE is the criterion; a profile carries no
+    # thresholds, and `GateHarvest`/`PromoteHarvest` receive the loaded
+    # criterion (with its sha) at boot.
+    class Criterion
+      Rule = Data.define(:version, :metric, :window, :threshold, :min_span)
+
+      attr_reader :rule, :path, :sha
+
+      def initialize(rule:, path:, sha:)
+        @rule = rule
+        @path = path
+        @sha = sha
+      end
+
+      # Reads the file, extracts the FIRST ```yaml fence, validates STRICTLY.
+      # Raises Insika::ConfigError when the file is absent or has no block;
+      # Insika::ValidationError on an unknown or missing key, or a malformed
+      # value — every defect named.
+      def self.load(path)
+        bytes = read_bytes(path)
+        block = extract_yaml_block(bytes)
+        parsed = YAML.safe_load(block, permitted_classes: [], aliases: false)
+        rule = build_rule(parsed)
+        new(rule: rule, path: path, sha: "sha256:#{Digest::SHA256.hexdigest(bytes)}")
+      end
+
+      def to_h = @rule.to_h
+
+      class << self
+        private
+
+        def read_bytes(path)
+          File.read(path, encoding: "UTF-8")
+        rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR => e
+          raise Insika::ConfigError, "harvest criterion not readable at #{path}: #{e.class}"
+        end
+
+        def extract_yaml_block(bytes)
+          match = bytes.match(/```yaml\n(.*?)\n```/m)
+          raise Insika::ConfigError, "harvest criterion has no ```yaml block (nothing to pre-register)" unless match
+
+          match[1]
+        end
+
+        def build_rule(parsed)
+          raise Insika::ValidationError, "criterion yaml block must be a mapping" unless parsed.is_a?(Hash)
+
+          raw = parsed.transform_keys(&:to_s)
+          keys = raw.keys.sort
+          missing = Rule.members.map(&:to_s) - keys
+          unknown = keys - Rule.members.map(&:to_s)
+          raise Insika::ValidationError, "harvest criterion is missing key(s): #{missing.join(', ')}" if missing.any?
+          raise Insika::ValidationError, "harvest criterion has unknown key(s): #{unknown.join(', ')}" if unknown.any?
+
+          validate!(raw)
+          Rule.new(**Rule.members.to_h { |m| [m, raw[m.to_s]] })
+        end
+
+        # Each defect named — a criterion nobody can read is a criterion
+        # nobody froze (D5).
+        def validate!(raw)
+          version = raw["version"]
+          raise Insika::ValidationError, "harvest criterion version must be an Integer (got #{version.inspect})" unless version.is_a?(Integer)
+
+          metric = raw["metric"].to_s
+          raise Insika::ValidationError, "harvest criterion metric must be a non-blank string" if metric.empty?
+
+          window = raw["window"].to_s
+          unless window.match?(/\A\d+h\z/)
+            raise Insika::ValidationError, "harvest criterion window must match /\\A\\d+h\\z/ (got #{raw['window'].inspect})"
+          end
+
+          min_span = raw["min_span"].to_s
+          unless min_span.match?(/\A\d+d\z/)
+            raise Insika::ValidationError, "harvest criterion min_span must match /\\A\\d+d\\z/ (got #{raw['min_span'].inspect})"
+          end
+
+          threshold = raw["threshold"]
+          unless threshold.is_a?(Float) && threshold >= 0 && threshold <= 1
+            raise Insika::ValidationError, "harvest criterion threshold must be a Float in 0..1 (got #{threshold.inspect})"
+          end
+        end
+      end
+    end
+  end
+end