@kal-elsam/kairo-runtime 0.16.0 → 0.17.0

package/src/global/intelligence/execution-router.js

@@ -0,0 +1,444 @@
+// Task -> provider/model routing. Deliberately NOT a black box: every
+// decision is traceable to the exact keywords that matched and the exact
+// real availability/quota data that ruled a candidate in or out. This is a
+// heuristic keyword classifier, not a learned model — do not oversell it as
+// "AI routing" in any UI copy that surfaces its output.
+//
+// Distinct from `intelligence/router.js`, which selects among model
+// *backends* (Ollama/Zen/OpenRouter) for a different purpose — this module
+// selects among Kairo's execution adapters (codex/claude/opencode-go/
+// opencode-zen/cursor) for running an approved task.
+
+const REPETITIVE_KEYWORDS = [
+  "rename", "boilerplate", "mock", "fixture", "typo", "lint", "format",
+  "docstring", "comment", "changelog", "readme", "scaffold", "stub"
+];
+const REASONING_KEYWORDS = [
+  "architecture", "design", "investigate", "root cause", "race condition",
+  "refactor", "security", "performance", "algorithm", "diagnose", "why does",
+  "tradeoff", "evaluate"
+];
+const MULTI_FILE_KEYWORDS = [
+  "integrate", "multi-file", "across", "backend and frontend", "full feature",
+  "migrate", "end to end", "end-to-end", "wire up", "plumb"
+];
+const QUESTION_WORDS = [
+  "qué", "que", "cómo", "como", "cuál", "cual", "cuáles", "por qué", "porque",
+  "quién", "quien", "dónde", "donde", "cuándo", "cuando",
+  "what", "how", "why", "which", "who", "where", "when",
+  "is ", "are ", "does ", "do ", "can ", "could ", "should ", "explain", "explica", "describe"
+];
+const ACTION_VERBS = [
+  "implementa", "implement", "agrega", "add", "crea", "create", "arregla", "fix",
+  "refactoriza", "refactor", "migra", "migrate", "actualiza", "update", "elimina", "remove",
+  "borra", "delete", "cambia", "change", "escribe", "write", "rename", "renombra",
+  "integra", "integrate", "corrige", "wire up", "build", "construye"
+];
+
+/**
+ * A read-only heuristic — never a model call — for whether a prompt reads
+ * like a question/exploration rather than a change request. Deliberately
+ * conservative: an action verb anywhere in the text always wins (so "explain
+ * this, then fix the bug" still routes as a task), and anything not clearly
+ * question-shaped falls back to "task" (today's existing behavior), so this
+ * can only ever remove work Kairo used to do wrong, never add new
+ * uncertainty to what already worked.
+ * @param {string} text
+ */
+export function isLikelyQuestion(text) {
+  const normalized = String(text ?? "").trim().toLowerCase();
+  if (!normalized) return false;
+  if (ACTION_VERBS.some((verb) => normalized.includes(verb))) return false;
+  if (normalized.endsWith("?")) return true;
+  return QUESTION_WORDS.some((word) => normalized.startsWith(word));
+}
+
+const RISK_KEYWORDS = [
+  "auth", "authentication", "payment", "security", "production", "credential",
+  "secret", "delete", "drop table", "migration", "billing", "pii"
+];
+
+/** Matches in the order they actually appear in the text, so a "why" built from them reads naturally. */
+/**
+ * Matches a task against the project's real skill catalog (name +
+ * description, read by skill-catalog.js from each SKILL.md — never
+ * guessed from a bare folder name). Word-overlap only, sorted by overlap
+ * size — surfaced as transparent evidence in the routing "why", not (yet)
+ * a factor that changes which provider is picked; that needs more design
+ * than a simple overlap heuristic should be trusted with.
+ * @param {string} taskText
+ * @param {Array<{name: string, description: string}>} skills
+ */
+export function matchSkills(taskText, skills = []) {
+  const taskWords = new Set(String(taskText ?? "").toLowerCase().split(/\W+/).filter((word) => word.length > 3));
+  const matches = [];
+  for (const skill of skills) {
+    const descriptionWords = String(skill.description ?? "").toLowerCase().split(/\W+/).filter((word) => word.length > 3);
+    const overlap = descriptionWords.filter((word) => taskWords.has(word));
+    if (overlap.length > 0) matches.push({ name: skill.name, overlap: [...new Set(overlap)] });
+  }
+  return matches.sort((a, b) => b.overlap.length - a.overlap.length);
+}
+
+function countMatches(text, keywords) {
+  return keywords
+    .map((keyword) => ({ keyword, index: text.indexOf(keyword) }))
+    .filter((entry) => entry.index !== -1)
+    .sort((a, b) => a.index - b.index)
+    .map((entry) => entry.keyword);
+}
+
+/**
+ * Pure keyword classifier — no model call, no network, deterministic and
+ * fast enough to run on every task submission. Returns which keywords
+ * actually matched (not just a score) so the router's "why" can quote them.
+ * @param {string} taskText
+ */
+export function classifyTask(taskText) {
+  const text = String(taskText ?? "").toLowerCase();
+  const repetitive = countMatches(text, REPETITIVE_KEYWORDS);
+  const reasoning = countMatches(text, REASONING_KEYWORDS);
+  const multiFile = countMatches(text, MULTI_FILE_KEYWORDS);
+  const risk = countMatches(text, RISK_KEYWORDS);
+  return {
+    repetitive, reasoning, multiFile, risk,
+    repetitionScore: repetitive.length,
+    reasoningScore: reasoning.length,
+    multiFileScore: multiFile.length,
+    riskScore: risk.length
+  };
+}
+
+/**
+ * @param {string} adapterId - "codex" | "claude" | "opencode-go" | "opencode-zen" | "cursor"
+ * @param {ReturnType<typeof import("../runtime/execution-adapters/index.js").inspectExecutionAdapters>} adapters
+ */
+function findAdapter(adapterId, adapters) {
+  const baseId = adapterId.startsWith("opencode") ? "opencode" : adapterId;
+  return adapters.find((adapter) => adapter.id === baseId) ?? null;
+}
+
+/** @param {object|null} usageEntry - a codex/claude usage-probe result (primary/secondary windows) */
+function remainingPercent(usageEntry) {
+  return usageEntry?.primary?.remainingPercent ?? null;
+}
+
+// Below this real remaining-quota percentage, a provider is treated as
+// exhausted for automatic routing — conserved for the tests explicitly
+// listed as this increment's scope, not a newly-invented number.
+const MIN_QUOTA_PERCENT = 5;
+
+/**
+ * The single eligibility policy shared by execution routing, ask routing,
+ * and the FIT widget — one candidate is judged the same way everywhere, so
+ * a provider that FIT recommends is guaranteed to actually be launchable.
+ * Real availability/launchability/quota only; never a capability judgment
+ * (that's scoreAvailableModels' job, applied only to survivors of this).
+ * @param {string} adapterId - "codex" | "claude" | "opencode-go" | "opencode-zen" | "cursor"
+ * @param {{adapters: object[], codexUsage?: object|null, claudeUsage?: object|null, opencodeGoUsage?: object|null}} context
+ * @returns {{ok: boolean, reason: string|null}}
+ * @param {{requireLaunchable?: boolean}} [options] - `requireLaunchable: false`
+ *   is for AI TEAM's recommendation surface only (service.js's snapshot()):
+ *   it lets opencode-go be named as a real, accessible option — you do
+ *   have the model via the Go subscription — without claiming Kairo can
+ *   safely auto-execute through it yet (see opencode.js's checkAvailability
+ *   for why: no per-event way to prove a run didn't silently bill Zen).
+ *   It also lets Cursor be named as a real recommendation (e.g. Builder ->
+ *   Cursor Composer 2.5) even though Kairo never auto-executes through it —
+ *   the user continues that work manually inside the Cursor IDE; this is
+ *   permanent for Cursor, not a temporary safety gap like opencode-go's.
+ *   Real task routing (selectExecutionProvider/selectAskProvider) always
+ *   uses the default `true` — it must never pick something guaranteed to
+ *   fail at launch (run-manager.js's own launchable gate would reject it).
+ */
+export function checkCandidate(adapterId, { adapters, codexUsage, claudeUsage, opencodeGoUsage }, { requireLaunchable = true } = {}) {
+  // Zen carries real PAYG/billing risk (see conversation/service.js's
+  // capabilities.openCodeExecution) — never an automatic pick, regardless
+  // of what its real catalog/benchmarks might otherwise say.
+  if (adapterId === "opencode-zen") return { ok: false, reason: "OpenCode Zen is excluded from automatic routing (PAYG risk)" };
+  // Cursor is a deliberate manual-only destination for real task
+  // execution — recommendation and execution are different capabilities
+  // (see model-intelligence.js's role assignments): Kairo can genuinely
+  // recommend a real Cursor model for a role (Builder -> Cursor Composer
+  // 2.5, say), the same requireLaunchable:false exception opencode-go
+  // already gets below, but real task routing (requireLaunchable: true,
+  // the default) always refuses it — the user continues that work
+  // manually inside the Cursor IDE, never an automatic Kairo-launched run.
+  if (adapterId === "cursor" && requireLaunchable) {
+    return { ok: false, reason: "Cursor is manual-only — continue the work in the Cursor IDE, never auto-executed by Kairo" };
+  }
+
+  const adapter = findAdapter(adapterId, adapters);
+  if (!adapter) return { ok: false, reason: `${adapterId}: no adapter found` };
+  if (!adapter.available) return { ok: false, reason: adapter.reason ?? `${adapterId}: not available` };
+  // "launchable" means safe for Kairo to invoke programmatically — not the
+  // bar a manual recommendation needs. Cursor gets the same exemption
+  // opencode-go already has: real availability is enough to recommend it,
+  // even though (unlike opencode-go) it's exempt from launchable for a
+  // structurally different reason — it will NEVER be launchable, by design.
+  const launchableRequired = requireLaunchable || (adapterId !== "opencode-go" && adapterId !== "cursor");
+  if (launchableRequired && !adapter.launchable) return { ok: false, reason: adapter.reason ?? `${adapterId}: not launchable yet` };
+
+  if (adapterId === "codex") {
+    const left = remainingPercent(codexUsage);
+    if (left != null && left < MIN_QUOTA_PERCENT) return { ok: false, reason: `Codex quota nearly exhausted (${left}% left)` };
+  }
+  if (adapterId === "claude") {
+    const left = remainingPercent(claudeUsage);
+    if (left != null && left < MIN_QUOTA_PERCENT) return { ok: false, reason: `Claude quota nearly exhausted (${left}% left)` };
+  }
+  if (adapterId === "opencode-go") {
+    const windows = opencodeGoUsage?.go?.windows ?? opencodeGoUsage?.windows ?? [];
+    // A cap being hit blocks real requests regardless of other windows
+    // having headroom — any rate-limited window is real evidence of that,
+    // so it's the conservative (fail-closed) reading, not a guess.
+    const limited = windows.find((window) => window.status === "rate-limited");
+    if (limited) return { ok: false, reason: `OpenCode Go ${limited.name} window is rate-limited` };
+  }
+  return { ok: true, reason: null };
+}
+
+/** Picks the provider's default model from its real catalog; null (never a guessed id) if none is marked default. */
+function defaultModelFor(adapterId, catalogs) {
+  const catalogKey = adapterId === "opencode-go" ? "opencodeGo" : adapterId === "opencode-zen" ? "opencodeZen" : adapterId;
+  const catalog = catalogs?.[catalogKey];
+  if (!catalog || catalog.status === "unknown") return null;
+  const models = catalog.models ?? [];
+  return models.find((model) => model.isDefault)?.id ?? models[0]?.id ?? null;
+}
+
+// Short with no reasoning/risk signal doesn't need the account's biggest
+// model — "what does this project do?" or "fix the button color" shouldn't
+// burn the same model as "why is there a race condition in the auth flow?".
+const LIGHT_TASK_MAX_LENGTH = 100;
+
+/**
+ * How much model capability a task (question OR real execution work)
+ * actually needs. Reuses classifyTask()'s real keyword signal rather than
+ * a separate heuristic — anything that reads as reasoning-heavy or touches
+ * a risk keyword still deserves a capable model even if it's short ("why
+ * does auth break?"). Governs which MODEL a provider uses, independent of
+ * which PROVIDER gets picked (candidateOrder's job) — a trivial fix still
+ * goes to whichever provider the task shape favors, just with its
+ * cheapest adequate model instead of automatically reaching for the top.
+ * @param {string} taskText
+ * @returns {"light"|"standard"|"heavy"}
+ */
+export function classifyEffort(taskText) {
+  const text = String(taskText ?? "");
+  const profile = classifyTask(text);
+  // Reasoning or risk always means real complexity, regardless of length —
+  // "why does auth break?" is short but not trivial. Multi-file/integration
+  // scope counts too, UNLESS it's dominated by repetition: "rename this DTO
+  // field across 15 files" spans many files but is mechanical, not complex
+  // — the repetition signal is real evidence the multi-file spread doesn't
+  // make it harder, just wider.
+  if (profile.reasoningScore > 0 || profile.riskScore > 0) return "heavy";
+  if (profile.multiFileScore > 0 && profile.multiFileScore > profile.repetitionScore) return "heavy";
+  if (text.trim().length <= LIGHT_TASK_MAX_LENGTH) return "light";
+  return "standard";
+}
+
+// Anthropic's own public model line naming (Haiku < Sonnet < Opus) is a
+// real, documented capability ordering — not a guess — so it's safe to
+// match against real catalog entries by name.
+const CLAUDE_EFFORT_NAME_PATTERNS = { light: "haiku", standard: "sonnet", heavy: "opus" };
+
+/**
+ * Picks a real model id from the provider's actual catalog for the given
+ * effort tier, falling back to the provider's own default when no matching
+ * tier exists in that catalog (never a fabricated id).
+ * @param {string} adapterId
+ * @param {"light"|"standard"|"heavy"} effort
+ * @param {object} catalogs
+ */
+function pickModelForEffort(adapterId, effort, catalogs) {
+  if (adapterId === "claude") {
+    const models = catalogs?.claude?.models ?? [];
+    const pattern = CLAUDE_EFFORT_NAME_PATTERNS[effort];
+    const tiered = pattern ? models.find((model) => model.id.toLowerCase().includes(pattern)) : null;
+    if (tiered) return tiered.id;
+  }
+  // OpenCode Go's real catalog reports a real per-model cost
+  // (costInputPerMTok) — no naming convention to trust here, but real
+  // price is itself a real, ungamed signal: sort by it and pick the
+  // cheapest/median/priciest model for light/standard/heavy. Codex and
+  // Cursor's real catalogs carry neither a naming convention nor a cost
+  // field (verified: id/displayName/isDefault/hidden only), so picking a
+  // tier for them would be inventing data instead of reading it — they
+  // keep using the provider's own default.
+  if (adapterId === "opencode-go") {
+    const withCost = (catalogs?.opencodeGo?.models ?? []).filter((model) => typeof model.costInputPerMTok === "number");
+    if (withCost.length > 0) {
+      const sorted = [...withCost].sort((a, b) => a.costInputPerMTok - b.costInputPerMTok);
+      if (effort === "light") return sorted[0].id;
+      if (effort === "heavy") return sorted[sorted.length - 1].id;
+      return sorted[Math.floor((sorted.length - 1) / 2)].id;
+    }
+  }
+  return defaultModelFor(adapterId, catalogs);
+}
+
+/**
+ * Decides an ordered candidate list (most to least preferred) from the
+ * classification alone — availability/quota filtering happens next, in
+ * `selectExecutionProvider`. Kept separate so the "why this order" reasoning
+ * stays inspectable.
+ */
+function candidateOrder(profile) {
+  if (profile.riskScore > 0 && (profile.reasoningScore > 0 || profile.multiFileScore > 0)) {
+    return { needsApproval: true, order: [] };
+  }
+  if (profile.reasoningScore >= profile.multiFileScore && profile.reasoningScore >= profile.repetitionScore && profile.reasoningScore > 0) {
+    return { needsApproval: false, order: ["codex", "claude"] };
+  }
+  if (profile.multiFileScore > 0) {
+    return { needsApproval: false, order: ["claude", "codex"] };
+  }
+  if (profile.repetitionScore > 0) {
+    return { needsApproval: false, order: ["opencode-go", "claude"] };
+  }
+  // No signal either way — today's existing default behavior, not a guess.
+  return { needsApproval: false, order: ["claude", "codex"] };
+}
+
+function reasonPhrase(adapterId, profile) {
+  if (adapterId === "codex") return profile.reasoning.length ? `reasoning task (${profile.reasoning.join(", ")})` : "default planning provider";
+  if (adapterId === "claude") return profile.multiFile.length ? `multi-file/integration task (${profile.multiFile.join(", ")})` : "default implementation provider";
+  if (adapterId === "opencode-go") return `repetitive/low-risk task (${profile.repetitive.join(", ")})`;
+  return adapterId;
+}
+
+/**
+ * Orders ask candidates by real remaining quota so repeated questions don't
+ * always burn the same account — but only when BOTH sides have a real
+ * measured number; with either one unknown there's no honest comparison to
+ * make, so it keeps today's existing claude-first default instead of
+ * guessing which side "probably" has more room.
+ * @param {number|null} codexRemaining
+ * @param {number|null} claudeRemaining
+ */
+function pickAskOrder(codexRemaining, claudeRemaining) {
+  if (codexRemaining == null || claudeRemaining == null) return { order: ["claude", "codex"], usedQuota: false };
+  return codexRemaining >= claudeRemaining
+    ? { order: ["codex", "claude"], usedQuota: true }
+    : { order: ["claude", "codex"], usedQuota: true };
+}
+
+/**
+ * Routes a read-only question to a provider — deliberately NOT the same
+ * risk/reasoning gating as selectExecutionProvider: a question ABOUT a
+ * risky topic ("how does auth work here?") is itself completely safe,
+ * unlike actually implementing changes to it, so this never returns
+ * WAIT_FOR_APPROVAL. Just: is a real, quota-healthy, ask-capable provider
+ * available, with its real default model.
+ * @param {object} args
+ * @param {object[]} args.adapters
+ * @param {object|null} [args.codexUsage]
+ * @param {object|null} [args.claudeUsage]
+ * @param {object} [args.catalogs]
+ * @param {string} [args.taskText] - the real question text, used only to size
+ *   how much model capability it needs (see classifyEffort) — never to
+ *   change which provider is picked or to gate on risk.
+ */
+export function selectAskProvider({ adapters, codexUsage = null, claudeUsage = null, catalogs = {}, taskText = "" }) {
+  const effort = classifyEffort(taskText);
+  const { order, usedQuota } = pickAskOrder(remainingPercent(codexUsage), remainingPercent(claudeUsage));
+  const attempts = [];
+  for (const adapterId of order) {
+    const check = checkCandidate(adapterId, { adapters, codexUsage, claudeUsage });
+    attempts.push({ adapterId, ...check });
+    if (check.ok) {
+      const quotaNote = usedQuota ? "; more real quota remaining" : "";
+      return {
+        decision: "ROUTED",
+        provider: adapterId,
+        model: pickModelForEffort(adapterId, effort, catalogs),
+        why: `read-only question (${effort} effort${quotaNote})`,
+        rejectedCandidates: attempts.filter((entry) => !entry.ok)
+      };
+    }
+  }
+  return {
+    decision: "NO_PROVIDER_AVAILABLE",
+    provider: null,
+    model: null,
+    why: `no ask-capable provider available: ${attempts.map((entry) => entry.reason).join("; ")}`,
+    rejectedCandidates: attempts
+  };
+}
+
+/**
+ * The actual router: classification -> ordered candidates -> real
+ * availability/quota filtering -> a real model id from a real catalog, or
+ * an explicit WAIT_FOR_APPROVAL decision when risk is too high to route
+ * automatically. Never returns a provider that failed its availability
+ * check, and never returns a model id that isn't in that provider's real
+ * catalog.
+ *
+ * @param {object} args
+ * @param {string} args.task
+ * @param {object[]} args.adapters - inspectExecutionAdapters() result
+ * @param {object|null} [args.codexUsage]
+ * @param {object|null} [args.claudeUsage]
+ * @param {object} [args.catalogs] - { codex, opencodeGo, opencodeZen, cursor, claude } readXModels() results
+ */
+export function selectExecutionProvider({
+  task, adapters, codexUsage = null, claudeUsage = null, opencodeGoUsage = null, catalogs = {}, skills = []
+}) {
+  const profile = classifyTask(task);
+  const { needsApproval, order } = candidateOrder(profile);
+  const matchedSkills = matchSkills(task, skills);
+  const skillNote = matchedSkills.length > 0
+    ? ` · matches skill "${matchedSkills[0].name}" (${matchedSkills[0].overlap.join(", ")})`
+    : "";
+  // Which provider handles the task and how capable a model it needs are
+  // separate questions — a trivial fix still goes wherever the task shape
+  // favors, just with the cheapest adequate model instead of always
+  // reaching for the account's top one (e.g. Claude Opus/Fable for
+  // "fix the button color").
+  const effort = classifyEffort(task);
+
+  if (needsApproval) {
+    return {
+      decision: "WAIT_FOR_APPROVAL",
+      provider: null,
+      model: null,
+      why: `high risk (${profile.risk.join(", ")}) combined with reasoning/multi-file scope — routing automatically would be unsafe; a human should pick the provider.${skillNote}`,
+      fallback: null,
+      matchedSkills,
+      profile
+    };
+  }
+
+  const attempts = [];
+  for (const adapterId of order) {
+    const check = checkCandidate(adapterId, { adapters, codexUsage, claudeUsage, opencodeGoUsage });
+    attempts.push({ adapterId, ...check });
+    if (check.ok) {
+      const model = pickModelForEffort(adapterId, effort, catalogs);
+      const remaining = order.slice(order.indexOf(adapterId) + 1);
+      return {
+        decision: "ROUTED",
+        provider: adapterId,
+        model,
+        why: `${reasonPhrase(adapterId, profile)} (${effort} effort)${skillNote}`,
+        fallback: remaining[0] ?? null,
+        rejectedCandidates: attempts.filter((entry) => !entry.ok),
+        matchedSkills,
+        profile
+      };
+    }
+  }
+
+  return {
+    decision: "NO_PROVIDER_AVAILABLE",
+    provider: null,
+    model: null,
+    why: `every candidate provider was unavailable: ${attempts.map((entry) => entry.reason).join("; ")}`,
+    fallback: null,
+    rejectedCandidates: attempts,
+    matchedSkills,
+    profile
+  };
+}

package/src/global/intelligence/kairo-telemetry-source.js

@@ -0,0 +1,59 @@
+// Kairo's own real runtime telemetry as a Model Intelligence Foundation
+// source. Every real task Kairo executes already gets its duration, token
+// usage, cost, and outcome recorded per run (see applyEventToMetadata in
+// run-events.js) — this just wires that already-collected data into the
+// same CapabilityRegistry as every external source, with zero new cost
+// and zero new benchmark runs. It's the one source with real 100%
+// coverage of exactly what you actually use: every model, every provider,
+// in your real environment, on your real tasks — not a public
+// benchmark's idea of what matters.
+//
+// Only terminal (finished) runs with a known model are ingested: a run
+// still in progress has no real duration or outcome yet, and a run with
+// no recorded model can't be attributed to one. Kairo directly observing
+// its own runs is treated as `verified: true` — there is no more
+// first-party a measurement can get.
+
+import { isTerminalRunState, RUN_STATES } from "../runtime/run-types.js";
+
+export const TELEMETRY_METRICS = Object.freeze({
+  DURATION_MS: "kairo.durationMs",
+  TOTAL_TOKENS: "kairo.totalTokens",
+  COST: "kairo.cost",
+  SUCCESS: "kairo.success"
+});
+
+/**
+ * @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>} registry
+ * @param {Array<object>} runRecords - listRunRecords() output (real run metadata)
+ */
+export function ingestKairoTelemetryEvidence(registry, runRecords) {
+  for (const run of runRecords ?? []) {
+    if (!run?.model || !run?.agentId) continue;
+    if (!isTerminalRunState(run.state)) continue;
+
+    const id = registry.registerIdentity(run.agentId, run.model);
+    const date = run.completedAt ?? run.updatedAt ?? null;
+    // runId identifies WHICH real run each measurement came from — never
+    // averaged into a single number here; that's for whatever consumes
+    // this evidence to decide, same as every other source in this registry.
+    const addTelemetryEvidence = (metric, value) => {
+      if (value == null) return;
+      registry.addEvidence(id, {
+        metric, value, source: "kairo-telemetry",
+        benchmarkVersion: null, modelConfig: run.runId, date, verified: true
+      });
+    };
+
+    const startedAtMs = Date.parse(run.startedAt ?? "");
+    const completedAtMs = Date.parse(run.completedAt ?? "");
+    if (Number.isFinite(startedAtMs) && Number.isFinite(completedAtMs) && completedAtMs >= startedAtMs) {
+      addTelemetryEvidence(TELEMETRY_METRICS.DURATION_MS, completedAtMs - startedAtMs);
+    }
+
+    addTelemetryEvidence(TELEMETRY_METRICS.TOTAL_TOKENS, run.tokenUsage?.total ?? null);
+    addTelemetryEvidence(TELEMETRY_METRICS.COST, run.cost ?? null);
+    addTelemetryEvidence(TELEMETRY_METRICS.SUCCESS, run.state === RUN_STATES.COMPLETED ? 1 : 0);
+  }
+  return registry;
+}

package/src/global/intelligence/kairobench-runner.js

@@ -0,0 +1,85 @@
+// Runs one KairoBench task (kairobench-tasks.js) against one real model,
+// in an isolated scratch directory that's always deleted afterward — never
+// the user's real project. `runTask` is injectable specifically so this
+// can be exercised in tests with zero cost and zero real execution; the
+// default wires to run-manager's real startRun, which is what actually
+// spends quota when this is genuinely invoked.
+
+import { mkdtemp, readFile as fsReadFile, writeFile as fsWriteFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawn } from "node:child_process";
+import { startRun } from "../runtime/run-manager.js";
+import { readRunState } from "../runtime/run-store.js";
+
+function realShellRun(command, args, cwd) {
+  return new Promise((resolve, reject) => {
+    let child;
+    try {
+      child = spawn(command, args, { cwd });
+    } catch (error) {
+      reject(error);
+      return;
+    }
+    let stdout = "";
+    let stderr = "";
+    child.stdout?.on("data", (chunk) => { stdout += chunk; });
+    child.stderr?.on("data", (chunk) => { stderr += chunk; });
+    child.once("error", reject);
+    child.once("close", (exitCode) => resolve({ stdout, stderr, exitCode }));
+  });
+}
+
+async function defaultRunTask({ adapterId, model, cwd, task, homeDir }) {
+  // Real execution: this is what actually spends real quota/money against
+  // a real provider. Never called by a test — every test injects a fake.
+  const { runId } = await startRun({
+    homeDir, agentId: adapterId, model, cwd, task, permissions: ["yolo"], wait: true
+  });
+  const state = await readRunState(homeDir, runId);
+  const startedAtMs = Date.parse(state?.startedAt ?? "");
+  const completedAtMs = Date.parse(state?.completedAt ?? "");
+  return {
+    runId,
+    durationMs: Number.isFinite(startedAtMs) && Number.isFinite(completedAtMs) ? completedAtMs - startedAtMs : null,
+    tokenUsage: state?.tokenUsage ?? null,
+    cost: state?.cost ?? null
+  };
+}
+
+/**
+ * @param {object} task - one KAIROBENCH_TASKS entry ({id, category, prompt, setup?, verify})
+ * @param {{adapterId: string, model: string, homeDir?: string, runTask?: Function}} options
+ * @returns {Promise<{taskId: string, category: string, adapterId: string, model: string, success: boolean, durationMs: number|null, tokenUsage: object|null, cost: number|null, runId: string|null}>}
+ */
+export async function runKairoBenchTask(task, { adapterId, model, homeDir = null, runTask = defaultRunTask } = {}) {
+  const scratchDir = await mkdtemp(join(tmpdir(), `kairobench-${task.id}-`));
+  try {
+    const io = {
+      readFile: (path, encoding) => fsReadFile(join(scratchDir, path), encoding),
+      writeFile: (path, content) => fsWriteFile(join(scratchDir, path), content),
+      run: (command, args) => realShellRun(command, args, scratchDir)
+    };
+
+    if (task.setup) await task.setup(io);
+
+    const runResult = await runTask({ adapterId, model, cwd: scratchDir, task: task.prompt, homeDir });
+
+    let success = false;
+    try {
+      success = await task.verify(io);
+    } catch {
+      success = false; // a verify that throws (e.g. the expected file was never created) is a real failure, not an error to surface
+    }
+
+    return {
+      taskId: task.id, category: task.category, adapterId, model, success,
+      durationMs: runResult?.durationMs ?? null,
+      tokenUsage: runResult?.tokenUsage ?? null,
+      cost: runResult?.cost ?? null,
+      runId: runResult?.runId ?? null
+    };
+  } finally {
+    await rm(scratchDir, { recursive: true, force: true }).catch(() => {});
+  }
+}

package/src/global/intelligence/kairobench-source.js

@@ -0,0 +1,34 @@
+// Feeds real KairoBench results (kairobench-runner.js) into the Model
+// Intelligence Foundation registry, same shape as every other source:
+// per-task success and duration recorded separately, never blended into
+// one composite score. `verified: true` — Kairo directly ran this exact
+// task against this exact model itself; there is no more first-party a
+// measurement gets.
+
+export function ingestKairoBenchEvidence(registry, results) {
+  for (const result of results ?? []) {
+    if (!result?.adapterId || !result?.model || !result?.taskId) continue;
+    const id = registry.registerIdentity(result.adapterId, result.model);
+    const date = new Date().toISOString();
+
+    registry.addEvidence(id, {
+      metric: `kairobench.${result.taskId}.success`, value: result.success ? 1 : 0,
+      source: "kairobench", benchmarkVersion: null, modelConfig: result.runId ?? null, date, verified: true
+    });
+
+    if (result.durationMs != null) {
+      registry.addEvidence(id, {
+        metric: `kairobench.${result.taskId}.durationMs`, value: result.durationMs,
+        source: "kairobench", benchmarkVersion: null, modelConfig: result.runId ?? null, date, verified: true
+      });
+    }
+
+    if (result.cost != null) {
+      registry.addEvidence(id, {
+        metric: `kairobench.${result.taskId}.cost`, value: result.cost,
+        source: "kairobench", benchmarkVersion: null, modelConfig: result.runId ?? null, date, verified: true
+      });
+    }
+  }
+  return registry;
+}

package/src/global/intelligence/kairobench-tasks.js

@@ -0,0 +1,47 @@
+// KairoBench: a small, hand-authored, reproducible task battery — NOT
+// hundreds of tasks fired at once. Every task runs identically against
+// every real model Kairo has access to (same prompt, same working
+// directory shape, same verification), so results are directly
+// comparable in a way no public leaderboard can be, since it's measured
+// against exactly what Kairo can actually invoke, not a vendor's idea of
+// what matters.
+//
+// Deliberately starts tiny per explicit decision: stage KairoBench, never
+// run a large paid batch at once. Grows only after this batch's results
+// are reviewed.
+//
+// `verify` is a scriptable, deterministic pass/fail check against the
+// task's real working directory — never an LLM-graded "quality" score,
+// which would just be another unverified opinion layered on top. Judged
+// quality scoring is explicitly out of scope for this first version.
+
+export const KAIROBENCH_TASKS = [
+  {
+    id: "implementation-01",
+    category: "Implementation",
+    prompt: "Create a file named answer.txt in the current directory containing exactly the text: 42",
+    /** @param {{readFile: (path: string, encoding: string) => Promise<string>}} io */
+    async verify({ readFile }) {
+      try {
+        const content = await readFile("answer.txt", "utf8");
+        return content.trim() === "42";
+      } catch {
+        return false;
+      }
+    }
+  },
+  {
+    id: "debugging-01",
+    category: "Debugging",
+    prompt: "The file broken.js exports add(a, b) but its return statement is missing, so it always returns undefined. Fix it so add(2, 3) returns 5. Do not change the function signature or exports.",
+    /** @param {{writeFile: (path: string, content: string) => Promise<void>}} io */
+    async setup({ writeFile }) {
+      await writeFile("broken.js", "function add(a, b) {\n  a + b;\n}\n\nmodule.exports = { add };\n");
+    },
+    /** @param {{run: (command: string, args: string[]) => Promise<{stdout: string, exitCode: number}>}} io */
+    async verify({ run }) {
+      const result = await run("node", ["-e", "console.log(require('./broken.js').add(2, 3))"]);
+      return result.exitCode === 0 && result.stdout.trim() === "5";
+    }
+  }
+];