insika 0.0.1 → 0.1.0

data/lib/insika/evals/report.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # Renders a run's [CaseResult] as a machine-readable JSON blob (for the baseline
+    # + gating in Fase C) and a human-readable markdown summary.
+    # Pure over the results — takes a clock value in, never reads it (so callers stay
+    # deterministic/testable).
+    module Report
+      module_function
+
+      # -> Hash ready for JSON. `at` is an ISO-8601 string stamped by the caller.
+      def to_h(results, at:)
+        passed = results.count(&:pass?)
+        skipped = results.count(&:skipped?)
+        {
+          "at" => at,
+          "total" => results.size,
+          "passed" => passed,
+          # A skipped case is neither: counting it as failed is the lie this outcome
+          # exists to stop, and counting it as passed is worse.
+          "failed" => results.size - passed - skipped,
+          "skipped" => skipped,
+          "judge_pending" => results.count(&:judge_pending?),
+          # Absent (not an empty tally) when nothing was compared — a run with no
+          # pairwise is the normal one, and a zeroed block reads like every case tied.
+          "pairwise" => pairwise_summary(results),
+          "cases" => results.map do |r|
+            {
+              "id" => r.id, "agent" => r.agent, "pass" => r.pass?,
+              "skipped" => r.skipped,
+              "judge_pending" => r.judge_pending?, "error" => r.error,
+              "checks" => r.checks.map { |c| { "name" => c.name, "pass" => c.pass, "detail" => c.detail } },
+              "judge" => (r.judge && { "score" => r.judge.score, "pass" => r.judge.pass, "reason" => r.judge.reason }),
+              "pairwise" => (r.pairwise && { "outcome" => r.pairwise.outcome, "vs" => r.pairwise.vs,
+                                             "reason" => r.pairwise.reason, "judges" => r.pairwise.judges,
+                                             "order_dependent" => r.pairwise.order_dependent })
+            }
+          end
+        }.compact
+      end
+
+      # -> counts by outcome, or nil when no case carried a reference. `human_assisted`
+      # is counted separately and always printed: a "better" against a conversation a
+      # PERSON typed is a different claim from one against the incumbent's model, and
+      # the two must never be summed into one number somebody quotes.
+      def pairwise_summary(results)
+        compared = results.filter_map(&:pairwise)
+        return nil if compared.empty?
+
+        counts = compared.group_by(&:outcome).transform_values(&:size)
+        { "compared" => compared.size,
+          "human_assisted" => compared.count(&:human_assisted?),
+          "order_dependent" => compared.count(&:order_dependent),
+          "outcomes" => counts }
+      end
+
+      def to_json(results, at:)
+        JSON.pretty_generate(to_h(results, at: at))
+      end
+
+      PAIRWISE_MARK = { "better" => "🟢", "comparable" => "🟡", "worse" => "🔴",
+                        "split" => "⚖️", "unknown" => "❔" }.freeze
+
+      # Never without `vs:` — see `pairwise_summary`.
+      def pairwise_line(v)
+        flip = v.order_dependent ? " (order-dependent)" : ""
+        "#{PAIRWISE_MARK.fetch(v.outcome, '·')} vs incumbent (#{v.vs}): #{v.outcome}#{flip} — #{v.reason}"
+      end
+
+      def pairwise_block(summary)
+        counts = summary["outcomes"].map { |k, n| "#{n} #{k}" }.join(" · ")
+        lines = ["", "**vs incumbent** (#{summary['compared']} compared): #{counts}"]
+        if summary["human_assisted"].positive?
+          lines << "- #{summary['human_assisted']} against a HUMAN-ASSISTED transcript (a person typed " \
+                   "part of the reference — not a model-vs-model result)"
+        end
+        if summary["order_dependent"].positive?
+          lines << "- #{summary['order_dependent']} flipped when the transcripts were swapped, " \
+                   "and were reported as comparable"
+        end
+        lines
+      end
+
+      # Human summary. One line per case; failing checks nested underneath.
+      def to_markdown(results, at:)
+        h = to_h(results, at: at)
+        lines = ["# Eval report — #{at}", "",
+                 "**#{h['passed']}/#{h['total'] - h['skipped']} passed** · #{h['failed']} failed" \
+                 "#{" · #{h['skipped']} skipped" if h['skipped'].positive?}" \
+                 "#{" · #{h['judge_pending']} awaiting judge (Fase B)" if h['judge_pending'].positive?}", ""]
+        results.each do |r|
+          if r.skipped?
+            # WITH the reason, always: "12 skipped" alone is indistinguishable from a
+            # suite that quietly stopped testing anything.
+            lines << "- ⏭️ `#{r.id}` (#{r.agent}) — skipped: #{r.skipped}"
+            next
+          end
+
+          lines << "- #{r.pass? ? '✅' : '❌'} `#{r.id}` (#{r.agent})#{'  ⏳ judge pending' if r.judge_pending?}"
+          r.failures.each { |c| lines << "    - ❌ #{c.name}: #{c.detail}" }
+          if r.judge
+            v = r.judge
+            lines << "    - #{v.pass ? '✅' : '❌'} judge: #{v.score} — #{v.reason}"
+          end
+          lines << "    - #{pairwise_line(r.pairwise)}" if r.pairwise
+        end
+        lines.concat(pairwise_block(h["pairwise"])) if h["pairwise"]
+        "#{lines.join("\n")}\n"
+      end
+    end
+  end
+end

data/lib/insika/evals/runner.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require_relative "golden"
+require_relative "assertions"
+
+module Insika
+  module Evals
+    # Replays golden cases through a Transport and evaluates each. Pure over the
+    # Transport (injected) — the fake makes the orchestration unit-testable offline;
+    # the HttpTransport makes a real run.
+    #
+    # Multi-turn: a case's turns replay IN ORDER under one conversation id
+    # (`eval-<id>`); earlier turns build context, the assertion runs on the LAST
+    # turn's result. A turn that errors aborts the rest of that conversation (there's
+    # nothing to continue from) and fails the case.
+    class Runner
+      # `tokens` is what the whole case cost, summed over its turns, or nil when no
+      # turn reported usage. `cached` is how much of that was served from the prompt
+      # cache, carried separately because it is the number that explains a total.
+      # Only the refinement gate reads them (RFC-0013 §3.9 records a run's cost); the
+      # report and the exit code are untouched.
+      RunCase = Struct.new(:result, :timings, :tokens, :cached, keyword_init: true)
+
+      # judge: an Evals::Judge (optional). When set, a case with a rubric whose turn
+      # ran cleanly gets a subjective verdict attached on top of the deterministic pass.
+      #
+      # capabilities: what the DEPLOYMENT has, per agent — anything answering
+      # `#for(agent_id)` with { "tools" =>, "capabilities" => } or nil. Used to skip a
+      # case the deployment cannot satisfy (RFC-0014 §3.2), BEFORE spending a turn on
+      # it. nil (or an unknown agent) = no resolution, and then a case with `requires`
+      # RUNS and says so in the report: "could not rule it out" is not a reason to
+      # stop testing something, and a suite that shrinks in silence is the failure
+      # this feature exists to avoid.
+      #
+      # pairwise: an Evals::Pairwise (optional, RFC-0014 §3.4). Only cases carrying a
+      # `reference:` are compared, and the verdict never touches pass/fail — it is the
+      # answer to "can we replace it", reported beside the suite's own verdict.
+      def initialize(transport:, judge: nil, conv_map: {}, capabilities: nil, pairwise: nil)
+        @transport = transport
+        @judge = judge
+        @conv_map = conv_map || {}
+        @capabilities = capabilities
+        @pairwise = pairwise
+      end
+
+      # [Golden] -> [RunCase]. Each RunCase carries the CaseResult (for the report) +
+      # per-turn timings (for `--mode perf`).
+      def run(goldens)
+        goldens.map { |g| run_case(g) }
+      end
+
+      def run_case(golden)
+        skip = skip_reason(golden)
+        return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
+
+        # A backend that resolves state from a pre-existing conversation (e.g. achei-b2b
+        # needs a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
+        # synthetic "eval-<id>" keeps the adapter's own multi-turn continuation.
+        conv = @conv_map[golden.id] || "eval-#{golden.id}"
+        turns = []
+        timings = []
+        spent = []
+        cached = []
+        golden.user_turns.each do |message|
+          outcome = @transport.turn(agent: golden.agent, conv: conv, message: message)
+          timings << { ttfb: outcome.ttfb, total: outcome.total }
+          spent << billed_tokens(outcome.usage)
+          cached << cached_tokens(outcome.usage)
+          turns << outcome.result
+          break if outcome.result.error
+        end
+        last = turns.last
+        # Tool/content assertions read the last turn (unchanged); the policy checks
+        # read every turn — "one question per reply" is a rule about each of them.
+        result = Assertions.evaluate(golden, last, turns: turns)
+        # Subjective layer: only when a judge is configured, the case has a rubric, and
+        # the turn ran cleanly (nothing to judge on an errored turn).
+        result.judge = @judge.score(golden: golden, result: last) if @judge && result.rubric && result.error.nil?
+        # Against the incumbent (RFC-0014 §3.4). Same rule as the judge: nothing to
+        # compare on a turn that errored — half a conversation would lose the
+        # comparison for a reason that has nothing to do with the agent.
+        result.pairwise = @pairwise.compare(golden: golden, turns: turns) if @pairwise && result.error.nil?
+        RunCase.new(result: result, timings: timings, tokens: sum_tokens(spent),
+                    cached: sum_tokens(cached))
+      end
+
+      private
+
+      # nil when NO turn reported usage; otherwise the sum of the ones that did. A
+      # partially-metered case is reported as what was actually measured, low rather
+      # than absent — a budget under-counting is a smaller lie than a budget that
+      # throws the number away because one leg was silent.
+      def sum_tokens(spent)
+        counted = spent.compact
+        counted.empty? ? nil : counted.sum
+      end
+
+      # What the turn actually SENT, cache included. The engine's `total_tokens` is
+      # input + output and DELIBERATELY excludes the cached prefix (`Executor#usage_of`
+      # reports `cached_tokens` alongside it), so reading it as the cost of a turn
+      # under-reads a cached identity by an order of magnitude.
+      #
+      # Measured on the pilot: one real turn reports `total_tokens: 88` with
+      # `cached_tokens: 26624`. A refinement budget built on the first number would
+      # have let a run send ~300× what its ceiling said. Cached tokens are cheaper
+      # than fresh ones; they are not free, and a ceiling has to see them.
+      def billed_tokens(usage)
+        return nil unless usage.is_a?(Hash)
+
+        total = read(usage, "total_tokens")
+        return nil if total.nil?
+
+        total + cached_tokens(usage).to_i
+      end
+
+      def cached_tokens(usage)
+        return nil unless usage.is_a?(Hash)
+
+        values = [read(usage, "cached_tokens"), read(usage, "cache_creation_tokens")].compact
+        values.empty? ? nil : values.sum
+      end
+
+      def read(usage, key)
+        value = usage[key] || usage[key.to_sym]
+        value&.to_i
+      end
+
+      # -> the reason to skip, or nil to run. A case with no `requires` always runs
+      # (and never even asks), which keeps the whole existing corpus untouched.
+      def skip_reason(golden)
+        return nil unless golden.requirements?
+
+        available = @capabilities&.for(golden.agent)
+        return nil if available.nil? # unresolved: run it, and the report says so
+
+        unmet = Assertions.unmet_requirements(golden, available)
+        unmet.empty? ? nil : unmet.join("; ")
+      end
+    end
+  end
+end

data/lib/insika/evals/transport.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+require_relative "assertions"
+
+module Insika
+  module Evals
+    # How the runner turns a (agent, conversation, message) into a TurnResult. The
+    # runner depends only on this seam, so it's unit-testable with a fake — the real
+    # HttpTransport is exercised against a running insika.
+    #
+    # One turn's outcome: the TurnResult the engine asserts on + perf timing (ttfb/
+    # total in ms) so `--mode perf` reports real latency over the real corpus (#6b).
+    #
+    # `usage` is the turn's token counts as the deployment reported them
+    # (`response.completed`), or nil when the provider sent none. It is carried, never
+    # asserted on: the consumer is RFC-0013's refinement budget, which has to bound the
+    # cost of a gate replay and cannot invent the number. nil is preserved as nil
+    # rather than zeroed — "the provider did not say" and "it cost nothing" are
+    # different facts, and a budget that confuses them stops being a budget.
+    TurnOutcome = Struct.new(:result, :ttfb, :total, :usage, keyword_init: true)
+
+    # Pure reduction of the /v1/responses SSE stream. Kept separate from the HTTP so
+    # it's testable offline with canned frames (server/responses.rb is the producer).
+    module Sse
+      module_function
+
+      # Raw SSE text -> [parsed JSON payload]. Skips `event:` lines, blanks, [DONE].
+      def payloads(text)
+        text.to_s.split("\n\n").flat_map do |frame|
+          frame.each_line.filter_map do |line|
+            next unless line.start_with?("data:")
+
+            p = line.sub(/^data:\s*/, "").strip
+            next if p.empty? || p == "[DONE]"
+
+            begin
+              JSON.parse(p)
+            rescue JSON::ParserError
+              nil
+            end
+          end
+        end
+      end
+
+      # [payload] -> { output_text:, tool_calls:, usage:, error: }. Maps the Responses
+      # frames (see server/responses.rb#frame_for): text deltas accumulate; each
+      # function_call item contributes a tool NAME (this stream carries no per-tool
+      # status — that lives in the ToolTraceStore, an in-process enrichment); a
+      # `response.failed` sets the turn error.
+      def reduce(payloads)
+        text = +""
+        tools = []
+        usage = nil
+        error = nil
+        payloads.each do |o|
+          case o["type"]
+          when "response.output_text.delta"
+            text << o["delta"].to_s
+          when "response.output_item.added"
+            item = o["item"] || {}
+            tools << { "name" => item["name"].to_s, "status" => nil } if item["type"] == "function_call"
+          when "response.completed"
+            usage = o.dig("response", "usage")
+          when "response.failed"
+            error = o.dig("response", "error", "message") || "response.failed"
+          end
+        end
+        { output_text: text, tool_calls: tools, usage: usage, error: error }
+      end
+    end
+
+    # What the deployment HAS, per agent (RFC-0014 §3.2), read over the same gated
+    # `/v1` the replay uses. The eval stays a client: it asks the engine instead of
+    # keeping its own idea of which tools exist.
+    #
+    # `#for(id)` -> { "tools" => [names] | nil, "capabilities" => [names] } | nil.
+    # nil means UNRESOLVED — the agent is not there, the route is not wired, the
+    # deployment is unreachable — and the Runner treats that as "run the case
+    # anyway". Silence must not shrink a suite. Answers are memoized: a corpus has
+    # many cases per agent and this is the same answer every time.
+    class HttpCapabilities
+      def initialize(base_url:, token:, timeout: 10)
+        @base = base_url
+        @token = token
+        @timeout = timeout
+        @cache = {}
+      end
+
+      def for(agent_id)
+        @cache.fetch(agent_id) { @cache[agent_id] = fetch(agent_id) }
+      end
+
+      private
+
+      def fetch(agent_id)
+        uri = URI.join("#{@base}/", "v1/agents/#{URI.encode_www_form_component(agent_id)}")
+        req = Net::HTTP::Get.new(uri)
+        req["Authorization"] = "Bearer #{@token}"
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.read_timeout = @timeout
+        http.open_timeout = @timeout
+        res = http.start { http.request(req) }
+        return nil unless res.code.to_i == 200
+
+        body = JSON.parse(res.body)
+        { "tools" => body["tools"], "capabilities" => Array(body["capabilities"]) }
+      rescue StandardError, JSON::ParserError
+        nil
+      end
+    end
+
+    # Drives real turns over POST /v1/responses — the SAME surface as
+    # scripts/loadtest.rb, so `--mode perf` is apples-to-apples with the loadtest and
+    # closes the real-traffic gap (#6b). A conversation is keyed by `user` (the
+    # adapter continues an existing chat on the same id), so multi-turn goldens replay
+    # in order under one `conv` id.
+    class HttpTransport
+      def initialize(base_url:, token:, timeout: 120)
+        @base = base_url
+        @token = token
+        @timeout = timeout
+      end
+
+      def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def turn(agent:, conv:, message:)
+        uri = URI.join("#{@base}/", "v1/responses")
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@token}"
+        req["Content-Type"] = "application/json"
+        req["Accept"] = "text/event-stream"
+        req.body = JSON.generate(model: "openclaw:#{agent}", user: conv, stream: true, input: message)
+
+        t0 = mono
+        ttfb = nil
+        buffer = +""
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.read_timeout = @timeout
+        http.open_timeout = 10
+
+        status = nil
+        http.start do
+          http.request(req) do |res|
+            status = res.code.to_i
+            return failure("HTTP #{status}", t0) unless status.between?(200, 299)
+
+            res.read_body do |chunk|
+              ttfb ||= (mono - t0) * 1000.0
+              buffer << chunk
+            end
+          end
+        end
+
+        reduced = Sse.reduce(Sse.payloads(buffer))
+        TurnOutcome.new(
+          result: TurnResult.new(output_text: reduced[:output_text],
+                                 tool_calls: reduced[:tool_calls], error: reduced[:error]),
+          ttfb: ttfb, total: (mono - t0) * 1000.0, usage: reduced[:usage]
+        )
+      rescue StandardError => e
+        failure(e.class.to_s, t0)
+      end
+
+      private
+
+      def failure(msg, t0)
+        TurnOutcome.new(result: TurnResult.new(output_text: "", tool_calls: [], error: msg),
+                        ttfb: nil, total: (mono - t0) * 1000.0)
+      end
+    end
+  end
+end

data/lib/insika/event.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Insika
+  # Everything the runtime emits is an Event: a closed canonical catalog — new
+  # types require updating the catalog.
+  #
+  # meta: { task_id:, session_id:, seq:, at: } (seq monotonic per task).
+  # Terminal turn events are :task_completed / :task_failed / :task_cancelled.
+  # :error is NOT a terminal twin — it survives only as the subscription-overflow
+  # signal (EventStream) and the post-terminal failure report (fail_task).
+  Event = Data.define(:type, :data, :meta) do
+    def initialize(type:, data:, meta: {})
+      super
+    end
+
+    def to_h = { type:, **data, meta: meta.compact }
+  end
+end

data/lib/insika/event_stream.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/queue"
+
+module Insika
+  # In-process pub/sub. The stream is concurrent with the
+  # turn: a slow observer NEVER delays execution — each subscriber
+  # has its own queue and `emit` only enqueues. No mutex: one reactor, cooperative
+  # fibers — a plain Array is enough.
+  class EventStream
+    # One subscription = one queue. The consumer blocks on `each` (its own fiber),
+    # never the emitter.
+    class Subscription
+      CLOSED = Object.new # internal end-of-stream sentinel
+      private_constant :CLOSED
+
+      # Cap on events queued per subscriber. A slow consumer
+      # piles up in its OWN queue; on overflow, the subscription closes with a
+      # local :error event — the turn never waits on transport.
+      MAX_QUEUED = 1000
+
+      def initialize(task_id: nil, session_id: nil, on_close: nil)
+        @task_id = task_id
+        @session_id = session_id
+        @on_close = on_close
+        @queue = Async::Queue.new
+      end
+
+      # Binds the subscription to a task_id AFTER creation: the
+      # transport subscribes before the dispatch — when the task_id does not yet
+      # exist — and binds it as soon as the handler returns. This keeps the
+      # per-task cap honest (only events for THIS task enter the queue) and makes
+      # the overflow :error carry the correct task_id. Returns self to chain.
+      def bind(task_id:)
+        @task_id = task_id
+        self
+      end
+
+      # Meta filter: nil = matches any value. Events with no task_id in
+      # meta (e.g. :session_created) reach only subscribers with no task filter.
+      def matches?(event)
+        meta = event.meta || {}
+        (@task_id.nil? || meta[:task_id] == @task_id) &&
+          (@session_id.nil? || meta[:session_id] == @session_id)
+      end
+
+      # Enqueues without EVER blocking. The real queue depth is `@queue.size`
+      # (not a separate counter — avoids drift). On reaching the cap, it enqueues
+      # a local :error and closes; pushes after close are ignored (`@closed`).
+      def push(event)
+        return if @closed
+
+        if @queue.size >= MAX_QUEUED
+          @queue.enqueue(Insika::Event.new(
+                           type: :error,
+                           data: { message: "subscription overflow" },
+                           meta: { task_id: @task_id, session_id: @session_id }
+                         ))
+          close
+          return
+        end
+
+        @queue.enqueue(event)
+      end
+
+      # Blocks the CONSUMER's fiber until #close.
+      def each
+        while (event = @queue.dequeue) != CLOSED
+          yield event
+        end
+      end
+
+      # Idempotent: a second CLOSED is harmless (the `each` stops at the first).
+      # `@on_close` fires only once (avoids removing the subscription twice).
+      def close
+        return if @closed
+
+        @closed = true
+        @queue.enqueue(CLOSED)
+        @on_close&.call(self)
+      end
+    end
+
+    def initialize
+      @subscriptions = []
+    end
+
+    # NEVER raises: an observer's exception is isolated — a
+    # broken observer does not bring down the turn. Synchronous and cheap.
+    #
+    # Iterates over a SNAPSHOT (`dup`): a subscription's cap may close it
+    # DURING the push (overflow -> close -> on_close removes from the array);
+    # mutating the array in the middle of a plain Array#each would skip the next
+    # subscriber.
+    def emit(event)
+      @subscriptions.dup.each do |sub|
+        sub.push(event) if sub.matches?(event)
+      rescue StandardError
+        # a broken observer does not bring down the turn; nothing to propagate
+      end
+      nil
+    end
+
+    # nil/nil = all events. Returns the Subscription (the caller iterates with
+    # `#each` on its own fiber).
+    def subscribe(task_id: nil, session_id: nil)
+      sub = Subscription.new(task_id: task_id, session_id: session_id,
+                             on_close: ->(s) { @subscriptions.delete(s) })
+      @subscriptions << sub
+      sub
+    end
+  end
+end