@mrace07/kairo 0.1.0

package/dist/application/context-manager.d.ts

@@ -0,0 +1,22 @@
+import type { Message, Task } from "../domain/models.js";
+import type { TaskStore } from "../domain/ports.js";
+export declare class ContextManager {
+    private readonly store;
+    private readonly selector;
+    constructor(store: TaskStore);
+    /** Builds the bounded message list that is sent to the model for the next turn. */
+    prepare(sessionId: string, task: Task): Promise<Message[]>;
+    /** Converts older conversation evidence into a durable summary before it is omitted. */
+    compact(sessionId: string, task: Task): string;
+    /** Drops leading tool-only messages that Gemini cannot interpret as a fresh conversation. */
+    private cleanStart;
+    /** Renders repository facts and ranked files as concise model guidance. */
+    private profileContext;
+    /** Loads applicable agent instructions for this request without retaining their text. */
+    private instructions;
+    private readBounded;
+    /** Combines the request and latest failure evidence for relevance ranking. */
+    private retrievalQuery;
+    /** Renders the latest persisted failure as an actionable repair instruction. */
+    private repairBrief;
+}

package/dist/application/context-manager.js

@@ -0,0 +1,174 @@
+import { open } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { ContextSelector } from "./context-selector.js";
+const AUTO_COMPACT_AFTER = 48;
+const MODEL_MESSAGE_LIMIT = 32;
+const MAX_INSTRUCTION_BYTES = 16 * 1024;
+const MAX_INSTRUCTION_TOTAL_BYTES = 32 * 1024;
+const excerpt = (value, length = 700) => value.length > length ? `${value.slice(0, length)}…` : value;
+export class ContextManager {
+    store;
+    selector = new ContextSelector();
+    constructor(store) {
+        this.store = store;
+    }
+    /** Builds the bounded message list that is sent to the model for the next turn. */
+    async prepare(sessionId, task) {
+        if (this.store.messageCount(sessionId) >= AUTO_COMPACT_AFTER)
+            this.compact(sessionId, task);
+        const checkpoint = this.store.latestCheckpoint(sessionId);
+        const recent = this.cleanStart(this.store.recentMessages(sessionId, MODEL_MESSAGE_LIMIT));
+        const profile = this.store.repositorySnapshot(sessionId);
+        const repositoryContext = profile && {
+            role: "user",
+            content: await this.profileContext(task, profile),
+            createdAt: profile.createdAt,
+        };
+        const repairBrief = this.repairBrief(task);
+        const repairContext = repairBrief && {
+            role: "user",
+            content: repairBrief,
+            createdAt: Date.now(),
+        };
+        const context = [
+            ...(repositoryContext ? [repositoryContext] : []),
+            ...(repairContext ? [repairContext] : []),
+            ...(checkpoint
+                ? [
+                    {
+                        role: "user",
+                        content: `Context checkpoint from earlier work:\n${checkpoint.summary}`,
+                        createdAt: checkpoint.createdAt,
+                    },
+                ]
+                : []),
+            ...recent,
+        ];
+        return context;
+    }
+    /** Converts older conversation evidence into a durable summary before it is omitted. */
+    compact(sessionId, task) {
+        const current = this.store.latestCheckpoint(sessionId);
+        const lastId = this.store.lastMessageId(sessionId);
+        if (current?.throughMessageId === lastId)
+            return current.summary;
+        const recent = this.store
+            .recentMessages(sessionId, 12)
+            .map((message) => `${message.role}${message.toolName ? `:${message.toolName}` : ""}: ${excerpt(message.content, 420)}`)
+            .join("\n");
+        const summary = [
+            `Task: ${task.prompt}`,
+            `State: ${task.status}`,
+            `Changed files: ${task.changedFiles.length ? task.changedFiles.join(", ") : "none"}`,
+            `Verification: ${task.verificationCommand ? `${task.verificationCommand} (${task.verificationPassed ? "passed" : "not passed"})` : "not run"}`,
+            task.error ? `Last error: ${excerpt(task.error)}` : "",
+            "Recent durable evidence:",
+            recent,
+        ]
+            .filter(Boolean)
+            .join("\n");
+        this.store.saveCheckpoint(sessionId, task.id, summary, lastId);
+        this.store.updateTask(task.id, { summary });
+        return summary;
+    }
+    /** Drops leading tool-only messages that Gemini cannot interpret as a fresh conversation. */
+    cleanStart(messages) {
+        const first = messages.findIndex((message) => message.role === "user" || (message.role === "model" && !message.toolCallId));
+        return first < 0 ? messages.slice(-1) : messages.slice(first);
+    }
+    /** Renders repository facts and ranked files as concise model guidance. */
+    async profileContext(task, profile) {
+        const relevantFiles = this.selector.select(this.retrievalQuery(task), profile);
+        const instructions = await this.instructions(profile, relevantFiles);
+        const verification = profile.verificationCandidates.map((candidate) => {
+            const evidence = candidate.evidence?.map((item) => item.path).join(", ");
+            return `${candidate.label} = ${candidate.command}${evidence ? ` [${evidence}]` : ""}`;
+        });
+        return [
+            "Repository snapshot:",
+            `Root: ${profile.root}`,
+            `Ecosystems: ${profile.ecosystems.join(", ") || "none detected"}`,
+            `Repository state: ${profile.fingerprint.kind}${profile.fingerprint.branch ? ` branch=${profile.fingerprint.branch}` : ""}${profile.fingerprint.head ? ` head=${profile.fingerprint.head.slice(0, 12)}` : ""}${profile.truncated ? " (inventory truncated)" : ""}`,
+            `Changed paths: ${profile.changedPaths.join(", ") || "clean or unavailable"}`,
+            `Package: ${profile.packageName ?? "unknown"}`,
+            `Package manager: ${profile.packageManager}`,
+            `Scripts: ${Object.keys(profile.scripts).length ? Object.keys(profile.scripts).sort().join(", ") : "none detected"}`,
+            `Source roots: ${profile.sourceRoots.join(", ") || "none detected"}`,
+            `Test roots: ${profile.testRoots.join(", ") || "none detected"}`,
+            `Instruction files: ${profile.instructionFiles.join(", ") || "none detected"}`,
+            `Manifests: ${profile.manifestFiles.join(", ") || "none detected"}`,
+            `CI files: ${profile.ciFiles.join(", ") || "none detected"}`,
+            `Build files: ${profile.buildFiles.join(", ") || "none detected"}`,
+            `Documentation: ${profile.documentationFiles.join(", ") || "none detected"}`,
+            `Available verification: ${verification.join("; ") || "none detected"}`,
+            `Relevant files for this task: ${relevantFiles.join(", ") || "use search_files to locate files"}`,
+            instructions,
+            "Use the profile as a guide, inspect files before edits, and choose an appropriate verification command after changes. For custom checks use run_command with verification=true; ordinary inspection commands do not verify a task. Every edit invalidates earlier verification.",
+        ]
+            .filter(Boolean)
+            .join("\n");
+    }
+    /** Loads applicable agent instructions for this request without retaining their text. */
+    async instructions(profile, relevantFiles) {
+        let remaining = MAX_INSTRUCTION_TOTAL_BYTES;
+        const sections = [];
+        for (const path of profile.instructionFiles) {
+            const directory = dirname(path).replaceAll("\\", "/");
+            const global = directory === "." ||
+                path === ".github/copilot-instructions.md" ||
+                path.startsWith(".cursor/rules/");
+            if (!global && !relevantFiles.some((file) => file.startsWith(`${directory}/`)))
+                continue;
+            const length = Math.min(MAX_INSTRUCTION_BYTES, remaining);
+            if (length <= 0)
+                break;
+            const text = await this.readBounded(join(profile.root, path), length);
+            if (!text)
+                continue;
+            sections.push(`Instructions from ${path}:\n${text}`);
+            remaining -= Buffer.byteLength(text);
+        }
+        return sections.join("\n");
+    }
+    async readBounded(path, length) {
+        let handle;
+        try {
+            handle = await open(path, "r");
+            const buffer = Buffer.alloc(length);
+            const { bytesRead } = await handle.read(buffer, 0, length, 0);
+            return buffer.subarray(0, bytesRead).toString("utf8");
+        }
+        catch {
+            return "";
+        }
+        finally {
+            await handle?.close();
+        }
+    }
+    /** Combines the request and latest failure evidence for relevance ranking. */
+    retrievalQuery(task) {
+        return [task.prompt, task.error, task.verificationOutput]
+            .filter((value) => Boolean(value))
+            .map((value) => value.slice(0, 8_000))
+            .join("\n");
+    }
+    /** Renders the latest persisted failure as an actionable repair instruction. */
+    repairBrief(task) {
+        const attempts = this.store.repairAttempts(task.id);
+        const latest = attempts.at(-1);
+        if (!latest)
+            return "";
+        const locations = latest.evidence.fileLocations
+            .map((location) => `${location.path}${location.line ? `:${location.line}` : ""}`)
+            .join(", ");
+        return [
+            `Repair attempt ${attempts.length}/2 after \`${latest.command}\` failed.`,
+            `Failure: ${latest.evidence.summary}`,
+            `Locations: ${locations || "none extracted"}`,
+            `Evidence: ${latest.evidence.excerpts.join(" | ") || "inspect the command output"}`,
+            `Changed files: ${task.changedFiles.join(", ") || "none recorded"}`,
+            `Repair budget remaining: ${Math.max(0, 2 - attempts.length)}.`,
+            "Prioritize the affected files, make a materially different repair, then rerun the same focused verification.",
+        ].join("\n");
+    }
+}

package/dist/application/context-selector.d.ts

@@ -0,0 +1,11 @@
+import type { RepositorySnapshot } from "../domain/models.js";
+export declare class ContextSelector {
+    /** Returns the highest-scoring repository files for a task or verification failure. */
+    select(task: string, profile: RepositorySnapshot, limit?: number): string[];
+    /** Produces normal and camel-case-split query terms for code identifiers. */
+    private terms;
+    /** Adds graph proximity to a file's direct lexical and structural relevance. */
+    private score;
+    /** Scores paths, file text, symbols, and source/test roles independently. */
+    private directScore;
+}

package/dist/application/context-selector.js

@@ -0,0 +1,74 @@
+const STOP_WORDS = new Set([
+    "the",
+    "and",
+    "with",
+    "this",
+    "that",
+    "from",
+    "into",
+    "add",
+    "fix",
+    "make",
+    "code",
+    "file",
+]);
+export class ContextSelector {
+    /** Returns the highest-scoring repository files for a task or verification failure. */
+    select(task, profile, limit = 12) {
+        const terms = this.terms(task);
+        const structural = new Map((profile.files ?? []).map((file) => [file.path, file]));
+        const inventory = profile.entries?.length
+            ? profile.entries
+            : profile.indexedFiles.map((path) => ({ path, kind: "other" }));
+        const files = inventory.map((entry) => ({
+            ...(structural.get(entry.path) ?? {
+                path: entry.path,
+                terms: [],
+                symbols: [],
+                imports: [],
+                relatedFiles: [],
+            }),
+            kind: entry.kind,
+        }));
+        const direct = new Set(files.filter((file) => this.directScore(file, terms, profile) > 0).map((file) => file.path));
+        // Relationship proximity only boosts files that are connected to independently relevant files.
+        return files
+            .map((file) => ({ path: file.path, score: this.score(file, terms, profile, direct) }))
+            .filter((item) => item.score > 0)
+            .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path))
+            .slice(0, limit)
+            .map((item) => item.path);
+    }
+    /** Produces normal and camel-case-split query terms for code identifiers. */
+    terms(value) {
+        return [value, value.replace(/([a-z])([A-Z])/g, "$1 $2")]
+            .flatMap((part) => part.toLowerCase().split(/[^a-z0-9_$]+/))
+            .filter((term) => term.length > 2 && !STOP_WORDS.has(term));
+    }
+    /** Adds graph proximity to a file's direct lexical and structural relevance. */
+    score(file, terms, profile, direct) {
+        const relationshipScore = file.relatedFiles.some((path) => direct.has(path)) ? 3 : 0;
+        return this.directScore(file, terms, profile) + relationshipScore;
+    }
+    /** Scores paths, file text, symbols, and source/test roles independently. */
+    directScore(file, terms, profile) {
+        const lower = file.path.toLowerCase();
+        const pathScore = terms.reduce((score, term) => score + (lower.includes(term) ? 4 : 0), 0);
+        const contentScore = terms.reduce((score, term) => score + (file.terms.includes(term) ? 3 : 0), 0);
+        const symbolScore = terms.reduce((score, term) => score + (file.symbols.includes(term) ? 6 : 0), 0);
+        const sourceScore = profile.sourceRoots.some((root) => lower.startsWith(`${root}/`)) ? 1 : 0;
+        const testScore = profile.testRoots.some((root) => lower.startsWith(`${root}/`)) ? 1 : 0;
+        const changedScore = profile.changedPaths?.includes(file.path) ? 8 : 0;
+        const kindTerms = {
+            instruction: ["instruction", "agent", "rule", "convention"],
+            manifest: ["dependency", "package", "manifest", "setup", "install"],
+            ci: ["ci", "pipeline", "workflow", "action"],
+            build: ["build", "docker", "make", "task"],
+            documentation: ["documentation", "docs", "readme", "architecture"],
+            test: ["test", "spec", "verify"],
+            config: ["config", "configuration", "setting"],
+        };
+        const kindScore = terms.some((term) => kindTerms[file.kind]?.includes(term)) ? 3 : 0;
+        return (pathScore + contentScore + symbolScore + sourceScore + testScore + changedScore + kindScore);
+    }
+}

package/dist/application/evaluated-agent.d.ts

@@ -0,0 +1,7 @@
+import { ProviderError } from "../domain/provider-error.js";
+import type { CodingAgent } from "./coding-agent.js";
+/** Captures failure before the caller closes the store, allowing metrics to be read. */
+export declare function runEvaluatedAgent(agent: CodingAgent, sessionId: string, prompt: string, onProgress?: (text: string) => void): Promise<{
+    error?: string;
+    category?: "agent" | ProviderError["category"];
+}>;

package/dist/application/evaluated-agent.js

@@ -0,0 +1,16 @@
+import { ProviderError } from "../domain/provider-error.js";
+/** Captures failure before the caller closes the store, allowing metrics to be read. */
+export async function runEvaluatedAgent(agent, sessionId, prompt, onProgress) {
+    try {
+        await agent.run(sessionId, prompt, (text) => {
+            if (text.startsWith("\n[") && /: (?:retry|stopped retrying)/.test(text))
+                onProgress?.(text);
+        });
+        return {};
+    }
+    catch (error) {
+        return error instanceof ProviderError
+            ? { error: error.message, category: error.category }
+            : { error: "Agent execution failed.", category: "agent" };
+    }
+}

package/dist/application/evaluation-comparison.d.ts

@@ -0,0 +1,34 @@
+import type { EvaluationAttempt, EvaluationRun } from "../domain/models.js";
+import type { EvaluationStore } from "../domain/ports.js";
+export declare const comparisonMetrics: readonly ["modelTurns", "toolExecutions", "repairs", "verificationFailures", "durationMs"];
+type Metric = (typeof comparisonMetrics)[number];
+export interface ReliabilitySummary {
+    infrastructureFailures: number;
+    codingAttempts: number;
+    codingPassRate: number | null;
+    attempts: number;
+    passed: number;
+    passRate: number | null;
+    averages: Record<Metric, number | null>;
+}
+export interface ReliabilityChange {
+    baseline: ReliabilitySummary;
+    current: ReliabilitySummary;
+    delta: ReliabilitySummary | null;
+}
+export interface EvaluationComparison {
+    baselineRun: EvaluationRun;
+    currentRun: EvaluationRun;
+    comparable: boolean;
+    overall: ReliabilityChange;
+    scenarios: Array<ReliabilityChange & {
+        scenarioId: string;
+    }>;
+}
+/** Averages observed attempts; missing data stays unavailable rather than becoming zero. */
+export declare function summarizeAttempts(attempts: EvaluationAttempt[]): ReliabilitySummary;
+/** Compares observed trial distributions, retaining missing scenarios on either side. */
+export declare function compareEvaluations(baselineRun: EvaluationRun, baselineAttempts: EvaluationAttempt[], currentRun: EvaluationRun, currentAttempts: EvaluationAttempt[]): EvaluationComparison;
+/** Loads a saved comparison; no baseline is a normal state for automatic reporting. */
+export declare function compareWithBaseline(store: EvaluationStore, runId: string): EvaluationComparison | undefined;
+export {};

package/dist/application/evaluation-comparison.js

@@ -0,0 +1,91 @@
+export const comparisonMetrics = [
+    "modelTurns",
+    "toolExecutions",
+    "repairs",
+    "verificationFailures",
+    "durationMs",
+];
+/** Averages observed attempts; missing data stays unavailable rather than becoming zero. */
+export function summarizeAttempts(attempts) {
+    const infrastructure = new Set([
+        "quota",
+        "authentication",
+        "network",
+        "service",
+        "request",
+        "setup",
+    ]);
+    const coding = attempts.filter((attempt) => !infrastructure.has(attempt.failureCategory ?? ""));
+    const passed = attempts.filter((attempt) => attempt.passed).length;
+    const averages = Object.fromEntries(comparisonMetrics.map((metric) => {
+        const values = attempts
+            .map((attempt) => (metric === "durationMs" ? attempt.durationMs : attempt.metrics[metric]))
+            .filter((value) => Number.isFinite(value));
+        return [
+            metric,
+            values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null,
+        ];
+    }));
+    return {
+        infrastructureFailures: attempts.length - coding.length,
+        codingAttempts: coding.length,
+        codingPassRate: coding.length
+            ? coding.filter((attempt) => attempt.passed).length / coding.length
+            : null,
+        attempts: attempts.length,
+        passed,
+        passRate: attempts.length ? passed / attempts.length : null,
+        averages,
+    };
+}
+/** Compares observed trial distributions, retaining missing scenarios on either side. */
+export function compareEvaluations(baselineRun, baselineAttempts, currentRun, currentAttempts) {
+    const comparable = baselineRun.provider === currentRun.provider && baselineRun.model === currentRun.model;
+    const difference = (before, after) => before === null || after === null ? null : after - before;
+    const change = (before, after) => {
+        const baseline = summarizeAttempts(before);
+        const current = summarizeAttempts(after);
+        return {
+            baseline,
+            current,
+            delta: comparable
+                ? {
+                    infrastructureFailures: current.infrastructureFailures - baseline.infrastructureFailures,
+                    codingAttempts: current.codingAttempts - baseline.codingAttempts,
+                    codingPassRate: difference(baseline.codingPassRate, current.codingPassRate),
+                    attempts: current.attempts - baseline.attempts,
+                    passed: current.passed - baseline.passed,
+                    passRate: difference(baseline.passRate, current.passRate),
+                    averages: Object.fromEntries(comparisonMetrics.map((metric) => [
+                        metric,
+                        difference(baseline.averages[metric], current.averages[metric]),
+                    ])),
+                }
+                : null,
+        };
+    };
+    const scenarios = [
+        ...new Set([...baselineAttempts, ...currentAttempts].map((attempt) => attempt.scenarioId)),
+    ].sort();
+    return {
+        baselineRun,
+        currentRun,
+        comparable,
+        overall: change(baselineAttempts, currentAttempts),
+        scenarios: scenarios.map((scenarioId) => ({
+            scenarioId,
+            ...change(baselineAttempts.filter((attempt) => attempt.scenarioId === scenarioId), currentAttempts.filter((attempt) => attempt.scenarioId === scenarioId)),
+        })),
+    };
+}
+/** Loads a saved comparison; no baseline is a normal state for automatic reporting. */
+export function compareWithBaseline(store, runId) {
+    const current = store.evaluationRun(runId);
+    if (!current)
+        throw new Error(`Evaluation run not found: ${runId}`);
+    if (current.suite !== "self" || current.completedAt === undefined)
+        throw new Error("Comparison requires a completed self-evaluation run.");
+    const baseline = store.evaluationBaseline();
+    return (baseline &&
+        compareEvaluations(baseline, store.evaluationAttempts(baseline.id), current, store.evaluationAttempts(current.id)));
+}

package/dist/application/evaluation-harness.d.ts

@@ -0,0 +1,19 @@
+import type { EvaluationResult, ModelTurn } from "../domain/models.js";
+import type { JevSafetyAdvisor } from "../domain/ports.js";
+export type EvaluationScenario = {
+    id: string;
+    prompt: string;
+    steps: ModelTurn[];
+    expect(workspace: string): Promise<boolean>;
+};
+export declare const evaluationScenarios: EvaluationScenario[];
+/** Runs all fixture cases in disposable copies and returns only the final aggregate evidence. */
+export declare function runEvaluationSuite(): Promise<EvaluationResult[]>;
+export type JevEvaluationReport = {
+    off: EvaluationResult[];
+    on: EvaluationResult[];
+};
+/** Runs matched deterministic fixtures to isolate Jev decision overhead from model quality. */
+export declare function runJevEvaluationSuite(): Promise<JevEvaluationReport>;
+/** Runs one scenario with isolated storage so benchmark state cannot affect the user workspace. */
+export declare function runScenario(scenario: EvaluationScenario, jev?: JevSafetyAdvisor): Promise<EvaluationResult>;

package/dist/application/evaluation-harness.js

@@ -0,0 +1,217 @@
+import { cp, mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { SqliteSessionStore } from "../infrastructure/persistence/sqlite-session-store.js";
+import { RepositoryProfiler } from "../infrastructure/repository/repository-profiler.js";
+import { WorkspaceTools, definitions } from "../infrastructure/tools/workspace-tools.js";
+import { CodingAgent } from "./coding-agent.js";
+import { taskMetrics } from "./task-metrics.js";
+const fixtures = join(process.cwd(), "evals", "fixtures");
+/** Supplies fixed model decisions so the first benchmark measures agent mechanics reproducibly. */
+class ScenarioProvider {
+    steps;
+    index = 0;
+    constructor(steps) {
+        this.steps = steps;
+    }
+    /** Returns the next deterministic model decision for this benchmark case. */
+    async stream(_messages, onText) {
+        const step = this.steps[this.index++];
+        if (!step)
+            return { text: "Scenario finished.", toolCalls: [] };
+        if (step.text)
+            onText(step.text);
+        return step;
+    }
+}
+/** Eval runs operate only in a disposable fixture copy, so approvals are deterministic and isolated. */
+class FixtureApproval {
+    /** Approves only actions inside the disposable fixture copy created by this harness. */
+    async approve(_call, _description) {
+        return true;
+    }
+}
+export const evaluationScenarios = [
+    {
+        id: "create-file",
+        prompt: "Create result.txt containing done and verify the project tests.",
+        steps: [
+            {
+                text: "",
+                toolCalls: [
+                    { id: "write", name: "write_file", args: { path: "result.txt", content: "done\n" } },
+                ],
+            },
+            {
+                text: "",
+                toolCalls: [
+                    {
+                        id: "test",
+                        name: "run_command",
+                        args: { command: "node --test test.mjs", verification: true },
+                    },
+                ],
+            },
+            { text: "Created and verified.", toolCalls: [] },
+        ],
+        async expect(workspace) {
+            return (await readFile(join(workspace, "result.txt"), "utf8")) === "done\n";
+        },
+    },
+    {
+        id: "repair-failing-test",
+        prompt: "Fix the failing message test and verify it.",
+        steps: [
+            {
+                text: "",
+                toolCalls: [
+                    {
+                        id: "first-edit",
+                        name: "edit_file",
+                        args: { path: "message.txt", oldText: "bad\n", newText: "still-bad\n" },
+                    },
+                ],
+            },
+            {
+                text: "",
+                toolCalls: [
+                    {
+                        id: "first-test",
+                        name: "run_command",
+                        args: { command: "node --test test.mjs", verification: true },
+                    },
+                ],
+            },
+            {
+                text: "",
+                toolCalls: [
+                    {
+                        id: "fix",
+                        name: "edit_file",
+                        args: { path: "message.txt", oldText: "still-bad\n", newText: "good\n" },
+                    },
+                ],
+            },
+            {
+                text: "",
+                toolCalls: [
+                    {
+                        id: "second-test",
+                        name: "run_command",
+                        args: { command: "node --test test.mjs", verification: true },
+                    },
+                ],
+            },
+            { text: "Fixed and verified.", toolCalls: [] },
+        ],
+        async expect(workspace) {
+            return (await readFile(join(workspace, "message.txt"), "utf8")) === "good\n";
+        },
+    },
+];
+/** Runs all fixture cases in disposable copies and returns only the final aggregate evidence. */
+export async function runEvaluationSuite() {
+    const results = [];
+    for (const scenario of evaluationScenarios)
+        results.push(await runScenario(scenario));
+    return results;
+}
+/** Runs matched deterministic fixtures to isolate Jev decision overhead from model quality. */
+export async function runJevEvaluationSuite() {
+    const advisor = {
+        async assess() {
+            return { risk: "low", confidence: 1 };
+        },
+        async route() {
+            return { value: "build", confidence: 1 };
+        },
+        async recover() {
+            return { value: "repair", confidence: 1 };
+        },
+        async modelTier() {
+            return { value: "balanced", confidence: 1 };
+        },
+    };
+    return {
+        off: await runEvaluationSuite(),
+        on: await Promise.all(evaluationScenarios.map((scenario) => runScenario(scenario, advisor))),
+    };
+}
+/** Runs one scenario with isolated storage so benchmark state cannot affect the user workspace. */
+export async function runScenario(scenario, jev) {
+    const root = await mkdtemp(join(tmpdir(), `kairo-eval-${scenario.id}-`));
+    try {
+        await cp(join(fixtures, scenario.id), root, { recursive: true });
+        const store = await SqliteSessionStore.open(":memory:");
+        try {
+            const session = store.create(root);
+            const tools = await WorkspaceTools.create(root);
+            store.saveRepositorySnapshot(session.id, await new RepositoryProfiler().profile(root));
+            const agent = new CodingAgent(new ScenarioProvider(scenario.steps), store, tools, new FixtureApproval(), definitions, undefined, jev);
+            await agent.run(session.id, scenario.prompt, () => { });
+            const task = agent.status(session.id);
+            const expectationPassed = await scenario.expect(root);
+            const metrics = taskMetrics(store.taskEvents(task.id));
+            const verified = task.verificationPassed === true;
+            return {
+                id: scenario.id,
+                passed: task.status === "completed" && verified && expectationPassed,
+                taskStatus: task.status,
+                verified,
+                expectationPassed,
+                metrics: {
+                    modelTurns: metrics.modelTurns,
+                    toolExecutions: metrics.toolExecutions,
+                    toolFailures: metrics.toolFailures,
+                    approvals: metrics.approvals,
+                    repairs: metrics.repairs,
+                    verificationPasses: metrics.verificationPasses,
+                    verificationFailures: metrics.verificationFailures,
+                    verificationSelections: metrics.verificationSelections,
+                    focusedVerifications: metrics.focusedVerifications,
+                    broadVerifications: metrics.broadVerifications,
+                    repairConverged: metrics.repairConverged,
+                    modelMs: metrics.modelMs,
+                    toolMs: metrics.toolMs,
+                    jevDecisions: metrics.jevDecisions,
+                    jevFailures: metrics.jevFailures,
+                    jevMs: metrics.jevMs,
+                    jevRoutes: metrics.jevRoutes,
+                    jevSafetyChecks: metrics.jevSafetyChecks,
+                    jevRecoveryChecks: metrics.jevRecoveryChecks,
+                },
+            };
+        }
+        finally {
+            store.close();
+        }
+    }
+    catch (error) {
+        return {
+            id: scenario.id,
+            passed: false,
+            taskStatus: "failed",
+            verified: false,
+            expectationPassed: false,
+            error: error.message,
+            metrics: {
+                modelTurns: 0,
+                toolExecutions: 0,
+                toolFailures: 0,
+                approvals: 0,
+                repairs: 0,
+                verificationPasses: 0,
+                verificationFailures: 0,
+                verificationSelections: 0,
+                focusedVerifications: 0,
+                broadVerifications: 0,
+                repairConverged: false,
+                modelMs: 0,
+                toolMs: 0,
+            },
+        };
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+}