insika 0.1.0 → 0.3.0

data/lib/insika/evals/judge.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Insika
   module Evals
-    # The LLM-judge (RFC-0008 §3.3, Fase B). Scores a golden's `rubric` against the
+    # The LLM-judge. Scores a golden's `rubric` against the
     # actual assistant reply — the subjective layer on top of the deterministic
     # asserts. Pure over an injected `ask` callable (prompt -> raw model text), so it's
     # unit-testable without an LLM; the real ask (RubyLLM on the utility_model, temp 0)
@@ -13,7 +13,7 @@ module Insika
     # Conservative by construction: an unparseable judge reply scores 0 (fails) rather
     # than silently passing.
     #
-    # A PANEL, not a single voice (RFC-0013 §3.9). `quorum: N` samples ONE model N
+    # A PANEL, not a single voice. `quorum: N` samples ONE model N
     # times, which measures that model's variance and little else — at temperature 0 it
     # mostly returns the same answer, including the same blind spot. Two DIFFERENT
     # models disagreeing about a rubric is the signal worth having, so `asks:` takes one
@@ -95,7 +95,7 @@ module Insika
         end
       end
 
-      # The `policy` is the one thing a rubric cannot carry alone (RFC-0014 §3.3): how
+      # The `policy` is the one thing a rubric cannot carry alone: how
       # much this store wants the agent to ask before acting is a per-store decision,
       # and a judge that is not TOLD it will guess — half the time wrongly. The
       # deterministic half is `Assertions.policy_checks`; this is the other half.
@@ -149,11 +149,11 @@ module Insika
       end
     end
 
-    # Builds the configured judge panel from `settings["evals"]` (RFC-0013 §3.9).
+    # Builds the configured judge panel from `settings["evals"]`.
     #
     # It lives HERE and not in `evals/run.rb` because the CLI is no longer the only
     # caller: the refinement gate scores a candidate with the SAME judges the operator
-    # configured, and §3.7 is explicit that a second copy of the judge would be the
+    # configured, and is explicit that a second copy of the judge would be the
     # worst possible outcome — the gate would then be grading against a rubric nobody
     # tuned. One builder, two callers.
     module JudgePanel
@@ -164,7 +164,7 @@ module Insika
       # -> [Judge, [model names]] | nil when nobody is configured to ask. NIL AND NOT
       # a no-op judge: a rubric'd case with no judge reads as `judge_pending`, which is
       # visible, where a judge that always passes would be silent.
-      # `llm` (RFC-0017 A2): the graph's own RubyLLM context; nil = the global
+      # `llm`: the graph's own RubyLLM context; nil = the global
       # constant. Only the deployment root and the CLI build judges today, and both
       # are one graph per process — the seam keeps the default honest for the day
       # an embedded graph gates a candidate on its own credentials.
@@ -187,7 +187,7 @@ module Insika
       # Sugar for the callers that only want the judge (the gate).
       def judge(settings, **kw) = build(settings, **kw)&.first
 
-      # The SAME configured models, asked a different question (RFC-0014 §3.4). One
+      # The SAME configured models, asked a different question. One
       # builder because "who judges here" is one operator decision: a pairwise panel
       # configured apart from the rubric panel would let a run be graded by judges
       # nobody chose. -> [Pairwise, [model names]] | nil when nobody is configured.

data/lib/insika/evals/pairwise.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Insika
   module Evals
-    # PAIRWISE AGAINST THE INCUMBENT (RFC-0014 §3.4) — the number that answers "can we
+    # PAIRWISE AGAINST THE INCUMBENT — the number that answers "can we
     # replace it". An absolute 0.72 says a reply cleared a bar we invented; it says
     # nothing about whether the system already answering 403,231 chats would have done
     # better with the same customer.
@@ -29,7 +29,7 @@ module Insika
     # 3. **A split panel stays split.** Averaging "better" and "worse" into
     #    "comparable" invents agreement that nobody expressed.
     #
-    # Cost: 2 provider calls per judge per case (§5). This is why it never runs as
+    # Cost: 2 provider calls per judge per case. This is why it never runs as
     # part of the gate and is opt-in on the CLI.
     class Pairwise
       BETTER = "better"
@@ -39,7 +39,7 @@ module Insika
       UNKNOWN = "unknown"
 
       # `vs` names WHO the reference half actually is: `agent` (model against model)
-      # or `human-assisted` (a person typed part of it, P23a's `origin: operator`).
+      # or `human-assisted` (a person typed part of it,'s `origin: operator`).
       # It rides on the verdict rather than beside it so a report cannot print the
       # outcome without the label.
       Verdict = Struct.new(:outcome, :reason, :vs, :judges, :order_dependent, keyword_init: true) do
@@ -60,12 +60,19 @@ module Insika
       def compare(golden:, turns:)
         return nil unless golden.reference?
 
-        ours = Pairwise.transcript(golden.user_turns, turns)
-        theirs = Pairwise.reference_transcript(golden.reference_messages)
-        return nil if ours.strip.empty?
+        compare_texts(ours: Pairwise.transcript(golden.user_turns, turns),
+                      theirs: Pairwise.reference_transcript(golden.reference_messages),
+                      vs: golden.human_assisted? ? "human-assisted" : "agent")
+      end
+
+      # two transcripts, no golden — the shadow seam. The judge is
+      # the SAME object with the SAME prompt, so shadow pairs and golden cases are
+      # graded by one rule. An empty side returns nil (never a verdict against an
+      # empty string — that was a bug in the golden path until this seam landed).
+      def compare_texts(ours:, theirs:, vs: "agent")
+        return nil if ours.to_s.strip.empty? || theirs.to_s.strip.empty?
 
-        panel = @asks.map { |ask| judge_once(ask, ours, theirs) }
-        combine(panel, vs: golden.human_assisted? ? "human-assisted" : "agent")
+        combine(@asks.map { |ask| judge_once(ask, ours, theirs) }, vs: vs)
       end
 
       # The replayed conversation as the judge reads it: the user turns we sent,
@@ -81,8 +88,13 @@ module Insika
       # is the conversation as the customer received it, and telling it "a person wrote
       # this one" is an invitation to grade the author instead. The fact is carried to
       # the READER as `vs: human-assisted` instead, which is where it changes a decision.
+      # A reference with no text at all returns "" — the caller's empty guard then
+      # refuses instead of judging against an empty string .
       def self.reference_transcript(messages)
-        Array(messages).map do |m|
+        msgs = Array(messages)
+        return "" if msgs.all? { |m| m["text"].to_s.strip.empty? }
+
+        msgs.map do |m|
           speaker = m["role"].to_s == "user" ? "customer" : "assistant"
           "#{speaker}: #{m['text'].to_s.strip}"
         end.join("\n")

data/lib/insika/evals/report.rb

@@ -5,7 +5,7 @@ require "json"
 module Insika
   module Evals
     # Renders a run's [CaseResult] as a machine-readable JSON blob (for the baseline
-    # + gating in Fase C) and a human-readable markdown summary.
+    # gating in) and a human-readable markdown summary.
     # Pure over the results — takes a clock value in, never reads it (so callers stay
     # deterministic/testable).
     module Report
@@ -90,7 +90,7 @@ module Insika
         lines = ["# Eval report — #{at}", "",
                  "**#{h['passed']}/#{h['total'] - h['skipped']} passed** · #{h['failed']} failed" \
                  "#{" · #{h['skipped']} skipped" if h['skipped'].positive?}" \
-                 "#{" · #{h['judge_pending']} awaiting judge (Fase B)" if h['judge_pending'].positive?}", ""]
+                 "#{" · #{h['judge_pending']} awaiting judge" if h['judge_pending'].positive?}", ""]
         results.each do |r|
           if r.skipped?
             # WITH the reason, always: "12 skipped" alone is indistinguishable from a

data/lib/insika/evals/runner.rb

@@ -17,7 +17,7 @@ module Insika
       # `tokens` is what the whole case cost, summed over its turns, or nil when no
       # turn reported usage. `cached` is how much of that was served from the prompt
       # cache, carried separately because it is the number that explains a total.
-      # Only the refinement gate reads them (RFC-0013 §3.9 records a run's cost); the
+      # Only the refinement gate reads them (records a run's cost); the
       # report and the exit code are untouched.
       RunCase = Struct.new(:result, :timings, :tokens, :cached, keyword_init: true)
 
@@ -26,13 +26,13 @@ module Insika
       #
       # capabilities: what the DEPLOYMENT has, per agent — anything answering
       # `#for(agent_id)` with { "tools" =>, "capabilities" => } or nil. Used to skip a
-      # case the deployment cannot satisfy (RFC-0014 §3.2), BEFORE spending a turn on
+      # case the deployment cannot satisfy, BEFORE spending a turn on
       # it. nil (or an unknown agent) = no resolution, and then a case with `requires`
       # RUNS and says so in the report: "could not rule it out" is not a reason to
       # stop testing something, and a suite that shrinks in silence is the failure
       # this feature exists to avoid.
       #
-      # pairwise: an Evals::Pairwise (optional, RFC-0014 §3.4). Only cases carrying a
+      # pairwise: an Evals::Pairwise (optional). Only cases carrying a
       # `reference:` are compared, and the verdict never touches pass/fail — it is the
       # answer to "can we replace it", reported beside the suite's own verdict.
       def initialize(transport:, judge: nil, conv_map: {}, capabilities: nil, pairwise: nil)
@@ -53,8 +53,8 @@ module Insika
         skip = skip_reason(golden)
         return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
 
-        # A backend that resolves state from a pre-existing conversation (e.g. achei-b2b
-        # needs a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
+        # A backend that resolves state from a pre-existing conversation (e.g. a
+        # consumer needing a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
         # synthetic "eval-<id>" keeps the adapter's own multi-turn continuation.
         conv = @conv_map[golden.id] || "eval-#{golden.id}"
         turns = []
@@ -76,7 +76,7 @@ module Insika
         # Subjective layer: only when a judge is configured, the case has a rubric, and
         # the turn ran cleanly (nothing to judge on an errored turn).
         result.judge = @judge.score(golden: golden, result: last) if @judge && result.rubric && result.error.nil?
-        # Against the incumbent (RFC-0014 §3.4). Same rule as the judge: nothing to
+        # Against the incumbent. Same rule as the judge: nothing to
         # compare on a turn that errored — half a conversation would lose the
         # comparison for a reason that has nothing to do with the agent.
         result.pairwise = @pairwise.compare(golden: golden, turns: turns) if @pairwise && result.error.nil?

data/lib/insika/evals/transport.rb

@@ -16,7 +16,7 @@ module Insika
     #
     # `usage` is the turn's token counts as the deployment reported them
     # (`response.completed`), or nil when the provider sent none. It is carried, never
-    # asserted on: the consumer is RFC-0013's refinement budget, which has to bound the
+    # asserted on: the consumer is's refinement budget, which has to bound the
     # cost of a gate replay and cannot invent the number. nil is preserved as nil
     # rather than zeroed — "the provider did not say" and "it cost nothing" are
     # different facts, and a budget that confuses them stops being a budget.
@@ -72,7 +72,7 @@ module Insika
       end
     end
 
-    # What the deployment HAS, per agent (RFC-0014 §3.2), read over the same gated
+    # What the deployment HAS, per agent, read over the same gated
     # `/v1` the replay uses. The eval stays a client: it asks the engine instead of
     # keeping its own idea of which tools exist.
     #

data/lib/insika/event_stream.rb

@@ -20,9 +20,11 @@ module Insika
       # local :error event — the turn never waits on transport.
       MAX_QUEUED = 1000
 
-      def initialize(task_id: nil, session_id: nil, on_close: nil)
+      def initialize(task_id: nil, session_id: nil, tenant: nil, types: nil, on_close: nil)
         @task_id = task_id
         @session_id = session_id
+        @tenant = tenant
+        @types = types
         @on_close = on_close
         @queue = Async::Queue.new
       end
@@ -39,9 +41,21 @@ module Insika
 
       # Meta filter: nil = matches any value. Events with no task_id in
       # meta (e.g. :session_created) reach only subscribers with no task filter.
+      #
+      # A TENANT-scoped subscription is FAIL-CLOSED on the meta's tenant (WS1):
+      # an event that does not carry the tenant (control events, ignored turns)
+      # matches NO tenant subscription. The tenant only ever sees its own.
+      #
+      # `types:` (nil = any) keeps a subscriber's queue to the events it answers
+      # — an alert consumer must not sit behind a full-traffic stream's 1000-cap
+      # (WS6), and a filtered queue is the cheapest way to keep it there.
       def matches?(event)
         meta = event.meta || {}
-        (@task_id.nil? || meta[:task_id] == @task_id) &&
+        owned = @tenant.nil? || meta[:tenant] == @tenant
+
+        owned &&
+          (@types.nil? || @types.include?(event.type)) &&
+          (@task_id.nil? || meta[:task_id] == @task_id) &&
           (@session_id.nil? || meta[:session_id] == @session_id)
       end
 
@@ -103,9 +117,13 @@ module Insika
     end
 
     # nil/nil = all events. Returns the Subscription (the caller iterates with
-    # `#each` on its own fiber).
-    def subscribe(task_id: nil, session_id: nil)
-      sub = Subscription.new(task_id: task_id, session_id: session_id,
+    # `#each` on its own fiber). `tenant:` scopes the stream to one tenant's
+    # events (WS1) — fail-closed, see Subscription#matches?. `types:` (nil =
+    # any) filters by event type so a subscriber's queue only ever holds what
+    # its consumer answers (WS6).
+    def subscribe(task_id: nil, session_id: nil, tenant: nil, types: nil)
+      sub = Subscription.new(task_id: task_id, session_id: session_id, tenant: tenant,
+                             types: types,
                              on_close: ->(s) { @subscriptions.delete(s) })
       @subscriptions << sub
       sub

data/lib/insika/evidence.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  #   — the evidence contract (engine half).
+  #
+  # A tool DECLARES `evidence` in its manifest; the engine then does BOTH jobs
+  # from the same declaration: strips the result down to `{items: [{id, line}]}`
+  # for the model (the lean envelope), and appends every `id` to the session
+  # evidence ledger. No second flag, no "lean but not evidence" mode — a
+  # half-configuration cannot exist, which is what keeps "no claim without a
+  # tool ID" a tautology at the envelope instead of a convention.
+  #
+  # Everything here is pure Ruby, no IO: the ToolEnvelope calls it after the real
+  # tool returns. It does NOT write anything — the ledger write is the envelope's,
+  # via the state.
+  module Evidence
+    # The declaration (D1/C2.1). Parses three authorable shapes:
+    #
+    #   { "evidence": "products" }                             // bare kind
+    #   { "evidence": { "kind": "products" } }                 // full form
+    #   { "evidence": { "kind": "products",
+    #                   "items": "results",
+    #                   "attachments": "cards" } }             // non-default paths
+    #
+    # `kind` is the PILOT PACK's value, never a gem constant (the
+    # engine owns nothing about product shape).
+    Spec = Data.define(:kind, :items_path, :attachments_path) do
+      PATH_RE = /\A[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*\z/
+
+      # String | Hash | nil -> Spec | nil. Raises ValidationError on a blank kind
+      # or an empty/ill-formed path. All at ingestion, never at the turn.
+      def self.parse(raw)
+        return nil if raw.nil? || raw == false
+
+        h = raw.is_a?(String) ? { "kind" => raw } : Coercion.deep_stringify(raw)
+        h = h.is_a?(Hash) ? h : {}
+        kind = Coercion.presence(h["kind"])
+        raise Insika::ValidationError, "evidence.kind is required" if kind.nil?
+
+        items = presence_path(h["items"], "items")
+        attachments = presence_path(h["attachments"], "attachments")
+        new(kind: kind, items_path: items, attachments_path: attachments)
+      end
+
+      def to_h
+        { "kind" => kind, "items" => items_path, "attachments" => attachments_path }.compact
+      end
+
+      def self.presence_path(value, default)
+        s = Coercion.presence(value)
+        s = default if s.nil?
+        raise Insika::ValidationError, "evidence.#{default}: not a dotted path" unless PATH_RE.match?(s)
+
+        s
+      end
+      private_class_method :presence_path
+    end
+
+    # The lean result the model sees — the ONLY thing that survives the envelope.
+    MAX_ITEMS = 16
+    # Line truncation keeps the transcript lean by force (E1).
+    LINE_MAX = 200
+    # Attachments are a channel nicety, never the answer; bounded on purpose.
+    MAX_ATTACHMENTS = 16
+    URL_MAX = 500
+
+    # The attachments contract, validated for the outbox (channel side, never the
+    # model): [{ "type" => "card"|"image", "url" => String, "caption" => String|nil }].
+    # Entries without a String url, or beyond MAX_ATTACHMENTS, are DROPPED — never
+    # a turn failure (the card is a channel nicety, not the answer).
+    def self.valid_attachments(list)
+      Array(list).filter_map do |entry|
+        next unless entry.is_a?(Hash)
+
+        url = (entry["url"] || entry[:url]).to_s
+        next if url.empty?
+
+        caption = Coercion.presence(entry["caption"] || entry[:caption])
+        { "type" => (entry["type"] || entry[:type]).to_s,
+          "url" => url[0, URL_MAX],
+          "caption" => caption }
+      end.first(MAX_ATTACHMENTS)
+    end
+
+    # Result shaping, stateless — callable from any tool-call fiber (the parallel
+    # tool_concurrency path). No shared state in this class.
+    class Processor
+      class << self
+        # -> the parsed Hash the evidence paths dig into. For `evidence_envelope`
+        # the raw body lives under `__insika_body` (D3); a code tool returns the
+        # object itself. Raises JSON::ParserError on a non-JSON envelope body —
+        # the envelope turns that into `{error:}`, nothing recorded (fail closed:
+        # no IDs, no claims to make).
+        def raw(spec, result)
+          return result unless result.is_a?(Hash) && result.key?("__insika_body")
+
+          JSON.parse(result["__insika_body"].to_s)
+        end
+
+        # -> [lean, attachments]. Assumes the shape already passed
+        # SchemaGuard.violation_output. Items are capped and lines truncated; the
+        # model must never see a null where the contract says items.
+        def build(spec, raw)
+          items = SchemaGuard.dig(raw, spec.items_path) || []
+          lean_items = items.first(MAX_ITEMS).map do |item|
+            { "id" => (item["id"] || item[:id]).to_s,
+              "line" => Coercion.utf8((item["line"] || item[:line]).to_s)[0, LINE_MAX] }
+          end
+          lean = { "items" => lean_items }
+          attachments = Insika::Evidence.valid_attachments(SchemaGuard.dig(raw, spec.attachments_path))
+          [lean, attachments]
+        end
+      end
+    end
+  end
+
+  # The session evidence ledger (C4). A session-scoped SET with an `ungrounded`
+  # counter: records every product id that entered the context via an
+  # evidence-declared tool this session , plus the ungrounded
+  # counter that feeds the daily metric. It is NOT a policy object — it records
+  # and answers `ids` / `ungrounded` / `lines`; it never decides.
+  class EvidenceLedger
+    # Oldest-evicted cap: a session that outlives it needs a real cap or the row
+    # grows forever.
+    MAX_IDS = 1_000
+
+    def initialize(store: nil, session_id: nil)
+      @store = store
+      @session_id = session_id
+      @ids = []
+      @ungrounded = 0
+    end
+
+    # The in-memory accumulator (the envelope appends, the validator/enforcer
+    # read the union). The PERSISTED list is appended on flush (Executor, stage 8)
+    # — the envelope never blocks on the store.
+    def record(ids)
+      @ids.concat(Array(ids).map(&:to_s).reject(&:empty?))
+      self
+    end
+
+    # -> the effective set for THIS turn: persisted session evidence (loaded at
+    # build) + the turn's new ids, deduped, capped.
+    def ids
+      (session_ids + @ids).uniq.last(MAX_IDS)
+    end
+
+    attr_reader :ungrounded
+
+    def ungrounded_count(claim)
+      @ungrounded += 1
+      claim
+    end
+
+    # -> self, flushed to the session record. Idempotent. A store OR not-found
+    # failure is swallowed (evidence is audit — it must never fail a committed
+    # turn; a session purged mid-turn by forget_customer/session_purge reads as
+    # "nothing to append", never an explosion).
+    def flush!
+      return self unless @store && @session_id
+
+      @store.append_evidence(@session_id, ids: @ids, ungrounded: @ungrounded)
+      @ids = []
+      @ungrounded = 0
+      self
+    rescue Insika::Error
+      self
+    end
+
+    private
+
+    def session_ids
+      return [] unless @store && @session_id
+
+      session = @store.find(@session_id)
+      Array(session&.evidence&.fetch("ids", [])).map(&:to_s)
+    rescue Insika::NotFoundError
+      []
+    end
+  end
+end