insika 0.1.0 → 0.2.0

data/lib/insika/evals/golden.rb

@@ -2,8 +2,8 @@
 
 require "yaml"
 
-# Evals — the quality harness (RFC-0008). It lives in `lib/` so the engine itself can
-# call it (the refinement gate of RFC-0013 needs to score a candidate agent, and a
+# Evals — the quality harness. It lives in `lib/` so the engine itself can
+# call it (the refinement gate of needs to score a candidate agent, and a
 # second copy of the judge would be the worst possible outcome), but it stays a
 # CLIENT: it reaches a running deployment over HTTP through `HttpTransport` and never
 # reads a store directly. `evals/run.rb` is a thin CLI over this module.
@@ -28,17 +28,17 @@ module Insika
       # Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
       def must_not = Array(expect["must_not"]).map(&:to_s)
 
-      # How much the agent should ask before acting (RFC-0014 §3.3). nil = the store
+      # How much the agent should ask before acting. nil = the store
       # has no opinion and only the rubric decides.
       def policy = GoldenLoader.presence(expect["policy"])
 
-      # What the DEPLOYMENT must have for this case to mean anything (RFC-0014 §3.2).
+      # What the DEPLOYMENT must have for this case to mean anything.
       # Empty = runs everywhere.
       def required_tools = Array(requires["tools"]).map(&:to_s)
       def required_capabilities = Array(requires["capabilities"]).map(&:to_s)
       def requirements? = !(required_tools + required_capabilities).empty?
 
-      # THE INCUMBENT'S CONVERSATION for the same opening (RFC-0014 §3.4) — the other
+      # THE INCUMBENT'S CONVERSATION for the same opening — the other
       # half of a pairwise comparison. Data in the case, not a store read: the eval is
       # a client, and a pair that lives in one reviewable file cannot go stale against
       # a database nobody looked at.
@@ -47,14 +47,14 @@ module Insika
       def reference? = !reference_messages.empty?
 
       # Did a PERSON type part of the reference half? After a handoff the operator's
-      # words are stored as `role: assistant` (P23a), and comparing a model to a human
+      # words are stored as `role: assistant`, and comparing a model to a human
       # and calling it a win is a lie in both directions — so the pair is LABELLED and
       # the report never prints the outcome without it.
       def human_assisted?
         reference_messages.any? { |m| MessageOrigin.origin_of(m) == MessageOrigin::OPERATOR }
       end
 
-      # LLM-judge rubric + threshold (consumed in Fase B — deferred here).
+      # LLM-judge rubric + threshold (consumed in — deferred here).
       def rubric = expect["rubric"]
       def min_score = expect["min_score"]
     end
@@ -126,7 +126,7 @@ module Insika
         raise InvalidGolden, "#{where} needs a 'role' of user or assistant" unless %w[user assistant].include?(role)
 
         text = presence(raw["text"]) || (raise InvalidGolden, "#{where} needs a non-empty 'text'")
-        # The SAME closed vocabulary the engine stamps (P23a). A typo'd marker would
+        # The SAME closed vocabulary the engine stamps. A typo'd marker would
         # read as "absent" downstream, which is how a human turn gets scored as the
         # incumbent's model.
         origin = begin

data/lib/insika/evals/judge.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Insika
   module Evals
-    # The LLM-judge (RFC-0008 §3.3, Fase B). Scores a golden's `rubric` against the
+    # The LLM-judge. Scores a golden's `rubric` against the
     # actual assistant reply — the subjective layer on top of the deterministic
     # asserts. Pure over an injected `ask` callable (prompt -> raw model text), so it's
     # unit-testable without an LLM; the real ask (RubyLLM on the utility_model, temp 0)
@@ -13,7 +13,7 @@ module Insika
     # Conservative by construction: an unparseable judge reply scores 0 (fails) rather
     # than silently passing.
     #
-    # A PANEL, not a single voice (RFC-0013 §3.9). `quorum: N` samples ONE model N
+    # A PANEL, not a single voice. `quorum: N` samples ONE model N
     # times, which measures that model's variance and little else — at temperature 0 it
     # mostly returns the same answer, including the same blind spot. Two DIFFERENT
     # models disagreeing about a rubric is the signal worth having, so `asks:` takes one
@@ -95,7 +95,7 @@ module Insika
         end
       end
 
-      # The `policy` is the one thing a rubric cannot carry alone (RFC-0014 §3.3): how
+      # The `policy` is the one thing a rubric cannot carry alone: how
       # much this store wants the agent to ask before acting is a per-store decision,
       # and a judge that is not TOLD it will guess — half the time wrongly. The
       # deterministic half is `Assertions.policy_checks`; this is the other half.
@@ -149,11 +149,11 @@ module Insika
       end
     end
 
-    # Builds the configured judge panel from `settings["evals"]` (RFC-0013 §3.9).
+    # Builds the configured judge panel from `settings["evals"]`.
     #
     # It lives HERE and not in `evals/run.rb` because the CLI is no longer the only
     # caller: the refinement gate scores a candidate with the SAME judges the operator
-    # configured, and §3.7 is explicit that a second copy of the judge would be the
+    # configured, and is explicit that a second copy of the judge would be the
     # worst possible outcome — the gate would then be grading against a rubric nobody
     # tuned. One builder, two callers.
     module JudgePanel
@@ -164,7 +164,7 @@ module Insika
       # -> [Judge, [model names]] | nil when nobody is configured to ask. NIL AND NOT
       # a no-op judge: a rubric'd case with no judge reads as `judge_pending`, which is
       # visible, where a judge that always passes would be silent.
-      # `llm` (RFC-0017 A2): the graph's own RubyLLM context; nil = the global
+      # `llm`: the graph's own RubyLLM context; nil = the global
       # constant. Only the deployment root and the CLI build judges today, and both
       # are one graph per process — the seam keeps the default honest for the day
       # an embedded graph gates a candidate on its own credentials.
@@ -187,7 +187,7 @@ module Insika
       # Sugar for the callers that only want the judge (the gate).
       def judge(settings, **kw) = build(settings, **kw)&.first
 
-      # The SAME configured models, asked a different question (RFC-0014 §3.4). One
+      # The SAME configured models, asked a different question. One
       # builder because "who judges here" is one operator decision: a pairwise panel
       # configured apart from the rubric panel would let a run be graded by judges
       # nobody chose. -> [Pairwise, [model names]] | nil when nobody is configured.

data/lib/insika/evals/pairwise.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Insika
   module Evals
-    # PAIRWISE AGAINST THE INCUMBENT (RFC-0014 §3.4) — the number that answers "can we
+    # PAIRWISE AGAINST THE INCUMBENT — the number that answers "can we
     # replace it". An absolute 0.72 says a reply cleared a bar we invented; it says
     # nothing about whether the system already answering 403,231 chats would have done
     # better with the same customer.
@@ -29,7 +29,7 @@ module Insika
     # 3. **A split panel stays split.** Averaging "better" and "worse" into
     #    "comparable" invents agreement that nobody expressed.
     #
-    # Cost: 2 provider calls per judge per case (§5). This is why it never runs as
+    # Cost: 2 provider calls per judge per case. This is why it never runs as
     # part of the gate and is opt-in on the CLI.
     class Pairwise
       BETTER = "better"
@@ -39,7 +39,7 @@ module Insika
       UNKNOWN = "unknown"
 
       # `vs` names WHO the reference half actually is: `agent` (model against model)
-      # or `human-assisted` (a person typed part of it, P23a's `origin: operator`).
+      # or `human-assisted` (a person typed part of it,'s `origin: operator`).
       # It rides on the verdict rather than beside it so a report cannot print the
       # outcome without the label.
       Verdict = Struct.new(:outcome, :reason, :vs, :judges, :order_dependent, keyword_init: true) do

data/lib/insika/evals/report.rb

@@ -5,7 +5,7 @@ require "json"
 module Insika
   module Evals
     # Renders a run's [CaseResult] as a machine-readable JSON blob (for the baseline
-    # + gating in Fase C) and a human-readable markdown summary.
+    # gating in) and a human-readable markdown summary.
     # Pure over the results — takes a clock value in, never reads it (so callers stay
     # deterministic/testable).
     module Report
@@ -90,7 +90,7 @@ module Insika
         lines = ["# Eval report — #{at}", "",
                  "**#{h['passed']}/#{h['total'] - h['skipped']} passed** · #{h['failed']} failed" \
                  "#{" · #{h['skipped']} skipped" if h['skipped'].positive?}" \
-                 "#{" · #{h['judge_pending']} awaiting judge (Fase B)" if h['judge_pending'].positive?}", ""]
+                 "#{" · #{h['judge_pending']} awaiting judge" if h['judge_pending'].positive?}", ""]
         results.each do |r|
           if r.skipped?
             # WITH the reason, always: "12 skipped" alone is indistinguishable from a

data/lib/insika/evals/runner.rb

@@ -17,7 +17,7 @@ module Insika
       # `tokens` is what the whole case cost, summed over its turns, or nil when no
       # turn reported usage. `cached` is how much of that was served from the prompt
       # cache, carried separately because it is the number that explains a total.
-      # Only the refinement gate reads them (RFC-0013 §3.9 records a run's cost); the
+      # Only the refinement gate reads them (records a run's cost); the
       # report and the exit code are untouched.
       RunCase = Struct.new(:result, :timings, :tokens, :cached, keyword_init: true)
 
@@ -26,13 +26,13 @@ module Insika
       #
       # capabilities: what the DEPLOYMENT has, per agent — anything answering
       # `#for(agent_id)` with { "tools" =>, "capabilities" => } or nil. Used to skip a
-      # case the deployment cannot satisfy (RFC-0014 §3.2), BEFORE spending a turn on
+      # case the deployment cannot satisfy, BEFORE spending a turn on
       # it. nil (or an unknown agent) = no resolution, and then a case with `requires`
       # RUNS and says so in the report: "could not rule it out" is not a reason to
       # stop testing something, and a suite that shrinks in silence is the failure
       # this feature exists to avoid.
       #
-      # pairwise: an Evals::Pairwise (optional, RFC-0014 §3.4). Only cases carrying a
+      # pairwise: an Evals::Pairwise (optional). Only cases carrying a
       # `reference:` are compared, and the verdict never touches pass/fail — it is the
       # answer to "can we replace it", reported beside the suite's own verdict.
       def initialize(transport:, judge: nil, conv_map: {}, capabilities: nil, pairwise: nil)
@@ -53,8 +53,8 @@ module Insika
         skip = skip_reason(golden)
         return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
 
-        # A backend that resolves state from a pre-existing conversation (e.g. achei-b2b
-        # needs a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
+        # A backend that resolves state from a pre-existing conversation (e.g. a
+        # consumer needing a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
         # synthetic "eval-<id>" keeps the adapter's own multi-turn continuation.
         conv = @conv_map[golden.id] || "eval-#{golden.id}"
         turns = []
@@ -76,7 +76,7 @@ module Insika
         # Subjective layer: only when a judge is configured, the case has a rubric, and
         # the turn ran cleanly (nothing to judge on an errored turn).
         result.judge = @judge.score(golden: golden, result: last) if @judge && result.rubric && result.error.nil?
-        # Against the incumbent (RFC-0014 §3.4). Same rule as the judge: nothing to
+        # Against the incumbent. Same rule as the judge: nothing to
         # compare on a turn that errored — half a conversation would lose the
         # comparison for a reason that has nothing to do with the agent.
         result.pairwise = @pairwise.compare(golden: golden, turns: turns) if @pairwise && result.error.nil?

data/lib/insika/evals/transport.rb

@@ -16,7 +16,7 @@ module Insika
     #
     # `usage` is the turn's token counts as the deployment reported them
     # (`response.completed`), or nil when the provider sent none. It is carried, never
-    # asserted on: the consumer is RFC-0013's refinement budget, which has to bound the
+    # asserted on: the consumer is's refinement budget, which has to bound the
     # cost of a gate replay and cannot invent the number. nil is preserved as nil
     # rather than zeroed — "the provider did not say" and "it cost nothing" are
     # different facts, and a budget that confuses them stops being a budget.
@@ -72,7 +72,7 @@ module Insika
       end
     end
 
-    # What the deployment HAS, per agent (RFC-0014 §3.2), read over the same gated
+    # What the deployment HAS, per agent, read over the same gated
     # `/v1` the replay uses. The eval stays a client: it asks the engine instead of
     # keeping its own idea of which tools exist.
     #

data/lib/insika/event_stream.rb

@@ -20,9 +20,11 @@ module Insika
       # local :error event — the turn never waits on transport.
       MAX_QUEUED = 1000
 
-      def initialize(task_id: nil, session_id: nil, on_close: nil)
+      def initialize(task_id: nil, session_id: nil, tenant: nil, types: nil, on_close: nil)
         @task_id = task_id
         @session_id = session_id
+        @tenant = tenant
+        @types = types
         @on_close = on_close
         @queue = Async::Queue.new
       end
@@ -39,9 +41,21 @@ module Insika
 
       # Meta filter: nil = matches any value. Events with no task_id in
       # meta (e.g. :session_created) reach only subscribers with no task filter.
+      #
+      # A TENANT-scoped subscription is FAIL-CLOSED on the meta's tenant (WS1):
+      # an event that does not carry the tenant (control events, ignored turns)
+      # matches NO tenant subscription. The tenant only ever sees its own.
+      #
+      # `types:` (nil = any) keeps a subscriber's queue to the events it answers
+      # — an alert consumer must not sit behind a full-traffic stream's 1000-cap
+      # (WS6), and a filtered queue is the cheapest way to keep it there.
       def matches?(event)
         meta = event.meta || {}
-        (@task_id.nil? || meta[:task_id] == @task_id) &&
+        owned = @tenant.nil? || meta[:tenant] == @tenant
+
+        owned &&
+          (@types.nil? || @types.include?(event.type)) &&
+          (@task_id.nil? || meta[:task_id] == @task_id) &&
           (@session_id.nil? || meta[:session_id] == @session_id)
       end
 
@@ -103,9 +117,13 @@ module Insika
     end
 
     # nil/nil = all events. Returns the Subscription (the caller iterates with
-    # `#each` on its own fiber).
-    def subscribe(task_id: nil, session_id: nil)
-      sub = Subscription.new(task_id: task_id, session_id: session_id,
+    # `#each` on its own fiber). `tenant:` scopes the stream to one tenant's
+    # events (WS1) — fail-closed, see Subscription#matches?. `types:` (nil =
+    # any) filters by event type so a subscriber's queue only ever holds what
+    # its consumer answers (WS6).
+    def subscribe(task_id: nil, session_id: nil, tenant: nil, types: nil)
+      sub = Subscription.new(task_id: task_id, session_id: session_id, tenant: tenant,
+                             types: types,
                              on_close: ->(s) { @subscriptions.delete(s) })
       @subscriptions << sub
       sub