@mrace07/kairo 0.1.0

package/dist/infrastructure/tools/workspace-tools.js

@@ -0,0 +1,321 @@
+import { readdir, readFile, realpath, writeFile } from "node:fs/promises";
+import { dirname, relative, resolve } from "node:path";
+import { spawn } from "node:child_process";
+const MAX_OUTPUT = 48_000;
+const ignored = new Set([".git", "node_modules", "dist", "build", "coverage", ".next", ".kairo"]);
+const truncate = (value) => value.length > MAX_OUTPUT ? `${value.slice(0, MAX_OUTPUT)}\n[output truncated]` : value;
+export const definitions = [
+    {
+        name: "submit_plan",
+        description: "Submit the final structured implementation plan after inspecting the repository. Only available in planning mode.",
+        mutating: false,
+        parameters: {
+            type: "object",
+            properties: {
+                goal: { type: "string" },
+                assumptions: { type: "array", items: { type: "string" } },
+                files: {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: { path: { type: "string" }, reason: { type: "string" } },
+                        required: ["path", "reason"],
+                    },
+                },
+                steps: { type: "array", items: { type: "string" } },
+                verification: {
+                    type: "object",
+                    properties: { command: { type: "string" }, reason: { type: "string" } },
+                    required: ["reason"],
+                },
+                risks: { type: "array", items: { type: "string" } },
+            },
+            required: ["goal", "assumptions", "files", "steps", "verification", "risks"],
+        },
+    },
+    {
+        name: "list_files",
+        description: "List workspace files under an optional relative directory. Omit path or use an empty path to list the workspace root.",
+        mutating: false,
+        parameters: { type: "object", properties: { path: { type: "string" } } },
+    },
+    {
+        name: "read_file",
+        description: "Read a UTF-8 text file in the workspace.",
+        mutating: false,
+        parameters: {
+            type: "object",
+            properties: { path: { type: "string" } },
+            required: ["path"],
+        },
+    },
+    {
+        name: "read_file_range",
+        description: "Read a bounded inclusive line range from a UTF-8 workspace file.",
+        mutating: false,
+        parameters: {
+            type: "object",
+            properties: {
+                path: { type: "string" },
+                startLine: { type: "number" },
+                endLine: { type: "number" },
+            },
+            required: ["path", "startLine", "endLine"],
+        },
+    },
+    {
+        name: "search_files",
+        description: "Search UTF-8 workspace files for literal text under an optional relative directory. Omit path or use an empty path to search the workspace root.",
+        mutating: false,
+        parameters: {
+            type: "object",
+            properties: { query: { type: "string" }, path: { type: "string" } },
+            required: ["query"],
+        },
+    },
+    {
+        name: "write_file",
+        description: "Create or replace a UTF-8 workspace file.",
+        mutating: true,
+        parameters: {
+            type: "object",
+            properties: { path: { type: "string" }, content: { type: "string" } },
+            required: ["path", "content"],
+        },
+    },
+    {
+        name: "edit_file",
+        description: "Replace one exact text occurrence in a workspace file.",
+        mutating: true,
+        parameters: {
+            type: "object",
+            properties: {
+                path: { type: "string" },
+                oldText: { type: "string" },
+                newText: { type: "string" },
+            },
+            required: ["path", "oldText", "newText"],
+        },
+    },
+    {
+        name: "run_command",
+        description: "Run a shell command inside the workspace.",
+        mutating: true,
+        parameters: {
+            type: "object",
+            properties: {
+                command: { type: "string" },
+                verification: {
+                    type: "boolean",
+                    description: "Set true only for an actual test, typecheck, lint, build, or explicit task assertion. Ordinary inspection commands are not verification.",
+                },
+            },
+            required: ["command"],
+        },
+    },
+];
+export class WorkspaceTools {
+    root;
+    /** Stores the already-resolved workspace root used for every safety check. */
+    constructor(root) {
+        this.root = root;
+    }
+    /** Resolves the requested workspace once before exposing any file tools. */
+    static async create(workspace) {
+        return new WorkspaceTools(await realpath(workspace));
+    }
+    /** Returns whether an absolute path remains inside the resolved workspace root. */
+    inside(path) {
+        const rel = relative(this.root, path);
+        return rel === "" || (!rel.startsWith("..") && !rel.includes("../"));
+    }
+    /** Validates a user path and rejects traversal or symlink escapes before access. */
+    async filePath(input, forWrite = false) {
+        if (typeof input !== "string" || !input.trim())
+            throw new Error("A non-empty path is required.");
+        const candidate = resolve(this.root, input);
+        if (!this.inside(candidate))
+            throw new Error("Path is outside the workspace.");
+        try {
+            const actual = await realpath(candidate);
+            // Resolving first catches a path that looks local but exits through a symlink.
+            if (!this.inside(actual))
+                throw new Error("Symlink escapes the workspace.");
+            return actual;
+        }
+        catch (error) {
+            if (!forWrite || error.code !== "ENOENT")
+                throw error;
+            const parent = await realpath(dirname(candidate));
+            if (!this.inside(parent))
+                throw new Error("Parent directory escapes the workspace.");
+            return candidate;
+        }
+    }
+    /** Resolves an optional directory, treating a blank value as the workspace root. */
+    directoryPath(input) {
+        const path = input === undefined || input === null || (typeof input === "string" && !input.trim())
+            ? "."
+            : input;
+        return this.filePath(path);
+    }
+    /** Produces the exact action text shown in the terminal approval prompt. */
+    description(call) {
+        return call.name === "run_command"
+            ? `Run command in ${this.root}:\n${String(call.args.command ?? "")}`
+            : `${call.name}: ${String(call.args.path ?? "workspace")}`;
+    }
+    /** Dispatches a validated tool request and converts executor errors into tool results. */
+    async execute(call) {
+        try {
+            switch (call.name) {
+                case "list_files":
+                    return {
+                        ok: true,
+                        output: await this.list(await this.directoryPath(call.args.path)),
+                    };
+                case "read_file":
+                    return {
+                        ok: true,
+                        output: truncate(await readFile(await this.filePath(call.args.path), "utf8")),
+                    };
+                case "read_file_range":
+                    return {
+                        ok: true,
+                        output: await this.readRange(call.args),
+                    };
+                case "search_files":
+                    return {
+                        ok: true,
+                        output: await this.search(String(call.args.query ?? ""), await this.directoryPath(call.args.path)),
+                    };
+                case "write_file": {
+                    const path = await this.filePath(call.args.path, true);
+                    if (typeof call.args.content !== "string")
+                        throw new Error("content must be a string");
+                    await writeFile(path, call.args.content, "utf8");
+                    return { ok: true, output: `Wrote ${relative(this.root, path)}` };
+                }
+                case "edit_file":
+                    return await this.edit(call.args);
+                case "run_command":
+                    return await this.command(String(call.args.command ?? ""));
+                default:
+                    throw new Error(`Unknown tool: ${call.name}`);
+            }
+        }
+        catch (error) {
+            return { ok: false, output: `Tool error: ${error.message}` };
+        }
+    }
+    /** Recursively lists a bounded set of non-ignored workspace files. */
+    async list(dir) {
+        const out = [];
+        const visit = async (current) => {
+            for (const entry of await readdir(current, { withFileTypes: true })) {
+                if (ignored.has(entry.name))
+                    continue;
+                const full = resolve(current, entry.name);
+                const display = relative(this.root, full);
+                if (entry.isDirectory())
+                    await visit(full);
+                else if (entry.isFile())
+                    out.push(display);
+                if (out.length >= 500)
+                    return;
+            }
+        };
+        await visit(dir);
+        return out.length ? truncate(out.sort().join("\n")) : "No files found.";
+    }
+    /** Searches readable workspace files for a literal query with bounded matches. */
+    async search(query, dir) {
+        if (!query)
+            throw new Error("query cannot be empty");
+        const files = (await this.list(dir)).split("\n");
+        const matches = [];
+        for (const file of files) {
+            if (file.startsWith("[") || matches.length >= 200)
+                break;
+            try {
+                const text = await readFile(resolve(this.root, file), "utf8");
+                text.split("\n").forEach((line, index) => {
+                    if (line.includes(query) && matches.length < 200)
+                        matches.push(`${file}:${index + 1}: ${line}`);
+                });
+            }
+            catch {
+                /* binary/unreadable files are skipped */
+            }
+        }
+        return matches.length ? truncate(matches.join("\n")) : "No matches found.";
+    }
+    /** Returns an inclusive, line-numbered slice while enforcing a small read limit. */
+    async readRange(args) {
+        const { startLine, endLine } = args;
+        if (!Number.isInteger(startLine) ||
+            !Number.isInteger(endLine) ||
+            startLine < 1 ||
+            endLine < startLine ||
+            endLine - startLine >= 500)
+            throw new Error("Use an inclusive line range from 1 to at most 500 lines.");
+        const lines = (await readFile(await this.filePath(args.path), "utf8")).split("\n");
+        const start = startLine;
+        const selected = lines.slice(start - 1, endLine);
+        return selected.length
+            ? truncate(selected.map((line, index) => `${start + index}: ${line}`).join("\n"))
+            : "No lines found in this range.";
+    }
+    /** Replaces one unambiguous text occurrence in a workspace file. */
+    async edit(args) {
+        const path = await this.filePath(args.path, true);
+        if (typeof args.oldText !== "string" || typeof args.newText !== "string")
+            throw new Error("oldText and newText must be strings");
+        const text = await readFile(path, "utf8");
+        const first = text.indexOf(args.oldText);
+        if (first < 0)
+            throw new Error("oldText was not found");
+        if (text.indexOf(args.oldText, first + args.oldText.length) >= 0)
+            throw new Error("oldText is ambiguous; provide more context");
+        await writeFile(path, text.replace(args.oldText, args.newText), "utf8");
+        return { ok: true, output: `Edited ${relative(this.root, path)}` };
+    }
+    /** Runs a shell command in the workspace and captures bounded output and timing. */
+    command(command) {
+        if (!command.trim())
+            return Promise.resolve({ ok: false, output: "command cannot be empty" });
+        return new Promise((done) => {
+            const startedAt = Date.now();
+            const child = spawn(command, {
+                cwd: this.root,
+                shell: true,
+                stdio: ["ignore", "pipe", "pipe"],
+            });
+            let output = "";
+            const collect = (chunk) => {
+                output = truncate(output + chunk.toString());
+            };
+            child.stdout.on("data", collect);
+            child.stderr.on("data", collect);
+            const timer = setTimeout(() => child.kill("SIGTERM"), 60_000);
+            child.on("close", (code) => {
+                clearTimeout(timer);
+                done({
+                    ok: code === 0,
+                    output: truncate(output || `Command exited with ${code}`),
+                    exitCode: code,
+                    durationMs: Date.now() - startedAt,
+                });
+            });
+            child.on("error", (error) => {
+                clearTimeout(timer);
+                done({
+                    ok: false,
+                    output: error.message,
+                    exitCode: null,
+                    durationMs: Date.now() - startedAt,
+                });
+            });
+        });
+    }
+}

package/dist/interface/cli/evaluation-comparison-report.d.ts

@@ -0,0 +1,6 @@
+import { type EvaluationComparison } from "../../application/evaluation-comparison.js";
+import type { EvaluationAttempt, EvaluationRun } from "../../domain/models.js";
+/** Displays the selected baseline without including any agent content. */
+export declare function formatBaseline(run: EvaluationRun, attempts: EvaluationAttempt[]): string;
+/** Renders informational aggregate and per-scenario reliability changes. */
+export declare function formatComparison(comparison: EvaluationComparison): string;

package/dist/interface/cli/evaluation-comparison-report.js

@@ -0,0 +1,46 @@
+import { comparisonMetrics, summarizeAttempts, } from "../../application/evaluation-comparison.js";
+/** Formats missing observations explicitly. */
+function rate(value) {
+    return value === null ? "N/A" : `${(value * 100).toFixed(1)}%`;
+}
+/** Labels directional changes without implying statistical significance. */
+function delta(value, higherIsBetter, suffix = "") {
+    if (value === null)
+        return "N/A (missing attempts)";
+    const label = Math.abs(value) < 1e-9
+        ? "NO CHANGE"
+        : value > 0 === higherIsBetter
+            ? "IMPROVEMENT"
+            : "REGRESSION";
+    return `${label} ${value > 0 ? "+" : ""}${value.toFixed(2)}${suffix}`;
+}
+/** Shows both observed denominators and, when compatible, their deltas. */
+function changeLines(label, change) {
+    const { baseline: b, current: c, delta: d } = change;
+    return [
+        `Infrastructure failures: ${b.infrastructureFailures} -> ${c.infrastructureFailures}; coding attempts: ${b.codingAttempts} -> ${c.codingAttempts}; coding-outcome pass rate: ${rate(b.codingPassRate)} -> ${rate(c.codingPassRate)}${d ? ` | ${delta(d.codingPassRate === null ? null : d.codingPassRate * 100, true, " pp")}` : ""}`,
+        `${label}: ${b.passed}/${b.attempts} (${rate(b.passRate)}) -> ${c.passed}/${c.attempts} (${rate(c.passRate)})${d ? ` | ${delta(d.passRate === null ? null : d.passRate * 100, true, " pp")}; passed-attempt change ${d.passed > 0 ? "+" : ""}${d.passed}` : ""}`,
+        ...comparisonMetrics.map((metric) => `  avg ${metric}: ${b.averages[metric]?.toFixed(2) ?? "N/A"} -> ${c.averages[metric]?.toFixed(2) ?? "N/A"}${d ? ` | ${delta(d.averages[metric], false)}` : ""}`),
+    ];
+}
+/** Displays the selected baseline without including any agent content. */
+export function formatBaseline(run, attempts) {
+    const summary = summarizeAttempts(attempts);
+    return `Self-evaluation baseline: ${run.id}\nModel: ${run.provider}/${run.model}  Revision: ${run.sourceRevision}  Trials: ${run.trialCount}\nReliability: ${summary.passed}/${summary.attempts} passed (${rate(summary.passRate)})`;
+}
+/** Renders informational aggregate and per-scenario reliability changes. */
+export function formatComparison(comparison) {
+    const { baselineRun: b, currentRun: c } = comparison;
+    return [
+        "Self-evaluation baseline comparison (informational):",
+        `Baseline: ${b.id}  model=${b.provider}/${b.model}  revision=${b.sourceRevision}  trials=${b.trialCount}`,
+        `Current: ${c.id}  model=${c.provider}/${c.model}  revision=${c.sourceRevision}  trials=${c.trialCount}`,
+        ...(!comparison.comparable
+            ? ["Provider/model mismatch: runs are not directly comparable; deltas are omitted."]
+            : []),
+        "Rates and averages use observed attempts; absent attempts are not counted as failures. Lower resource averages do not establish better correctness.",
+        "Coding-outcome rates exclude classified infrastructure failures. Historical misclassifications cannot be reconstructed from saved metadata.",
+        ...changeLines("Overall", comparison.overall),
+        ...comparison.scenarios.flatMap((scenario) => changeLines(scenario.scenarioId, scenario)),
+    ].join("\n");
+}

package/dist/interface/cli/evaluation-report.d.ts

@@ -0,0 +1,14 @@
+import type { EvaluationResult, EvaluationAttempt, EvaluationRun, LiveEvaluationResult, SelfEvaluationResult } from "../../domain/models.js";
+import type { JevEvaluationReport } from "../../application/evaluation-harness.js";
+/** Reports matched deterministic Jev-off/Jev-on trials without changing evaluator semantics. */
+export declare function formatJevEvaluationReport(report: JevEvaluationReport): string;
+/** Renders benchmark evidence in a compact human-readable report. */
+export declare function formatEvaluationReport(results: EvaluationResult[]): string;
+/** Renders deterministic task evidence and DeepEval's semantic completion verdict side by side. */
+export declare function formatLiveEvaluationReport(results: LiveEvaluationResult[]): string;
+/** Renders real Kairo repository tasks, including trial identity for stochastic agent runs. */
+export declare function formatSelfEvaluationReport(results: SelfEvaluationResult[], runId?: string): string;
+/** Renders compact metadata-only reliability history for baseline comparisons. */
+export declare function formatEvaluationHistory(runs: EvaluationRun[]): string;
+/** Renders one saved run without replaying sensitive task or model content. */
+export declare function formatEvaluationRun(run: EvaluationRun, attempts: EvaluationAttempt[]): string;

package/dist/interface/cli/evaluation-report.js

@@ -0,0 +1,122 @@
+/** Reports matched deterministic Jev-off/Jev-on trials without changing evaluator semantics. */
+export function formatJevEvaluationReport(report) {
+    const summarize = (results) => ({
+        completed: results.filter((result) => result.passed).length,
+        repairs: results.reduce((sum, result) => sum + result.metrics.repairs, 0),
+        verificationFailures: results.reduce((sum, result) => sum + result.metrics.verificationFailures, 0),
+        escalations: results.filter((result) => result.taskStatus === "verification_required").length,
+        jevMs: results.reduce((sum, result) => sum + (result.metrics.jevMs ?? 0), 0),
+        routes: results.reduce((sum, result) => sum + (result.metrics.jevRoutes ?? 0), 0),
+        safety: results.reduce((sum, result) => sum + (result.metrics.jevSafetyChecks ?? 0), 0),
+        recovery: results.reduce((sum, result) => sum + (result.metrics.jevRecoveryChecks ?? 0), 0),
+    });
+    const off = summarize(report.off);
+    const on = summarize(report.on);
+    return [
+        "Kairo deterministic Jev evaluation (matched fixtures)",
+        `Jev off: ${off.completed}/${report.off.length} verified completion; repairs=${off.repairs}; verification failures=${off.verificationFailures}; escalations=${off.escalations}`,
+        `Jev on: ${on.completed}/${report.on.length} verified completion; repairs=${on.repairs}; verification failures=${on.verificationFailures}; escalations=${on.escalations}; Jev latency=${Math.round(on.jevMs)} ms; routing=${on.routes}; safety=${on.safety}; recovery=${on.recovery}`,
+        "This deterministic mode measures integration behavior and metadata overhead, not live Jev decision quality.",
+    ].join("\n");
+}
+/** Renders benchmark evidence in a compact human-readable report. */
+export function formatEvaluationReport(results) {
+    const passed = results.filter((result) => result.passed).length;
+    return [
+        `Kairo scripted evaluation: ${passed}/${results.length} passed`,
+        ...results.map((result) => [
+            `${result.passed ? "PASS" : "FAIL"} ${result.id}`,
+            `status=${result.taskStatus}`,
+            `verified=${result.verified}`,
+            `expectation=${result.expectationPassed}`,
+            `turns=${result.metrics.modelTurns}`,
+            `tools=${result.metrics.toolExecutions}`,
+            `repairs=${result.metrics.repairs}`,
+            `checks=${result.metrics.verificationPasses}/${result.metrics.verificationFailures}`,
+            `verification=${result.metrics.focusedVerifications} focused/${result.metrics.broadVerifications} broad`,
+            `repairConverged=${result.metrics.repairConverged}`,
+            result.error ? `error=${result.error}` : "",
+        ]
+            .filter(Boolean)
+            .join("  ")),
+    ].join("\n");
+}
+/** Renders deterministic task evidence and DeepEval's semantic completion verdict side by side. */
+export function formatLiveEvaluationReport(results) {
+    const passed = results.filter((result) => result.passed).length;
+    return [
+        `Kairo live DeepEval evaluation: ${passed}/${results.length} passed`,
+        ...results.map((result) => [
+            `${result.passed ? "PASS" : "FAIL"} ${result.id}`,
+            `status=${result.taskStatus}`,
+            `verified=${result.verified}`,
+            `expectation=${result.expectationPassed}`,
+            `judge=${result.judge.passed ? "passed" : "failed"}`,
+            result.judge.score === undefined ? "" : `score=${result.judge.score.toFixed(2)}`,
+            result.judge.reason ? `reason=${result.judge.reason}` : "",
+            result.judge.error ? `judgeError=${result.judge.error}` : "",
+            result.error ? `error=${result.error}` : "",
+        ]
+            .filter(Boolean)
+            .join("  ")),
+    ].join("\n");
+}
+/** Renders real Kairo repository tasks, including trial identity for stochastic agent runs. */
+export function formatSelfEvaluationReport(results, runId) {
+    const passed = results.filter((result) => result.passed).length;
+    return [
+        `Kairo self evaluation${runId ? ` (${runId})` : ""}: ${passed}/${results.length} passed`,
+        ...results.map((result) => [
+            `${result.passed ? "PASS" : "FAIL"} ${result.id}`,
+            `trial=${result.trial}`,
+            `retries=${result.metrics.providerRetries ?? "N/A"} waitMs=${result.metrics.providerWaitMs ?? "N/A"}`,
+            result.failureCategory ? `failure=${result.failureCategory}` : "",
+            `status=${result.taskStatus}`,
+            `verified=${result.verified}`,
+            `expectation=${result.expectationPassed}`,
+            `turns=${result.metrics.modelTurns}`,
+            `tools=${result.metrics.toolExecutions}`,
+            `repairs=${result.metrics.repairs}`,
+            `verification=${result.metrics.focusedVerifications} focused/${result.metrics.broadVerifications} broad`,
+            `repairConverged=${result.metrics.repairConverged}`,
+            result.error ? `error=${result.error}` : "",
+        ]
+            .filter(Boolean)
+            .join("  ")),
+    ].join("\n");
+}
+/** Renders compact metadata-only reliability history for baseline comparisons. */
+export function formatEvaluationHistory(runs) {
+    if (!runs.length)
+        return "No saved self-evaluation runs.";
+    return [
+        "Kairo self-evaluation history:",
+        ...runs.map((run) => `${run.id}  ${run.passedCount}/${run.attemptCount} passed  trials=${run.trialCount}  model=${run.provider}/${run.model}  revision=${run.sourceRevision}  ${new Date(run.startedAt).toISOString()}`),
+    ].join("\n");
+}
+/** Renders one saved run without replaying sensitive task or model content. */
+export function formatEvaluationRun(run, attempts) {
+    return [
+        `Kairo self evaluation: ${run.id}`,
+        `Reliability: ${run.passedCount}/${run.attemptCount} passed`,
+        `Model: ${run.provider}/${run.model}  Revision: ${run.sourceRevision}  Trials: ${run.trialCount}`,
+        `Started: ${new Date(run.startedAt).toISOString()}${run.completedAt ? `  Completed: ${new Date(run.completedAt).toISOString()}` : ""}`,
+        ...attempts.map((attempt) => [
+            `${attempt.passed ? "PASS" : "FAIL"} ${attempt.scenarioId}`,
+            `trial=${attempt.trial}`,
+            `retries=${attempt.metrics.providerRetries ?? "N/A"} waitMs=${attempt.metrics.providerWaitMs ?? "N/A"}`,
+            `status=${attempt.taskStatus}`,
+            `verified=${attempt.verified}`,
+            `expectation=${attempt.expectationPassed}`,
+            `turns=${attempt.metrics.modelTurns}`,
+            `tools=${attempt.metrics.toolExecutions}`,
+            `repairs=${attempt.metrics.repairs}`,
+            `verification=${attempt.metrics.focusedVerifications} focused/${attempt.metrics.broadVerifications} broad`,
+            `repairConverged=${attempt.metrics.repairConverged}`,
+            `durationMs=${attempt.durationMs}`,
+            attempt.failureCategory ? `failure=${attempt.failureCategory}` : "",
+        ]
+            .filter(Boolean)
+            .join("  ")),
+    ].join("\n");
+}

package/dist/interface/cli/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};