@mrace07/kairo 0.1.0

package/dist/application/failure-analyzer.d.ts

@@ -0,0 +1,5 @@
+import type { FailureEvidence } from "../domain/models.js";
+export declare class FailureAnalyzer {
+    /** Extracts bounded paths and useful error lines from a failed verification command. */
+    analyze(command: string, output: string): FailureEvidence;
+}

package/dist/application/failure-analyzer.js

@@ -0,0 +1,37 @@
+const MAX_EXCERPTS = 8;
+const MAX_OUTPUT = 8_000;
+export class FailureAnalyzer {
+    /** Extracts bounded paths and useful error lines from a failed verification command. */
+    analyze(command, output) {
+        // Keep persisted repair context small even when a test runner emits a large stack trace.
+        const lines = output
+            .slice(0, MAX_OUTPUT)
+            .split("\n")
+            .map((line) => line.trim());
+        const fileLocations = new Map();
+        for (const line of lines) {
+            // Covers the common JavaScript/TypeScript `path:line[:column]` stack-trace form.
+            for (const match of line.matchAll(/([\w@./-]+\.[cm]?[jt]sx?):(\d+)(?::(\d+))?/g)) {
+                const path = match[1];
+                fileLocations.set(`${path}:${match[2]}:${match[3] ?? ""}`, {
+                    path,
+                    line: Number(match[2]),
+                    column: match[3] ? Number(match[3]) : undefined,
+                });
+            }
+            // Test runners often name the failing file without a source location.
+            const testFile = /(?:FAIL|✖|×)\s+([\w@./-]+\.[cm]?[jt]sx?)/.exec(line)?.[1];
+            if (testFile)
+                fileLocations.set(testFile, { path: testFile });
+        }
+        const excerpts = lines
+            .filter((line) => /(?:error|fail|expect|assert|exception|✖|×)/i.test(line))
+            .filter((line, index, all) => Boolean(line) && all.indexOf(line) === index)
+            .slice(0, MAX_EXCERPTS);
+        return {
+            summary: excerpts[0] || `Verification command failed: ${command}`,
+            fileLocations: [...fileLocations.values()].slice(0, MAX_EXCERPTS),
+            excerpts,
+        };
+    }
+}

package/dist/application/interaction-routing.d.ts

@@ -0,0 +1,9 @@
+/** The bounded interaction classes used before Kairo creates a coding task. */
+export type InteractionIntent = "conversation" | "answer" | "repository_task";
+export type AutoInteractionMode = "plan" | "build";
+/** Keeps obvious social messages local so they spend neither Jev nor coding-model capacity. */
+export declare function greetingResponse(input: string): string | undefined;
+/** Builds minimal, credential-free context for Jev's pre-loop intent decision. */
+export declare function interactionIntentState(input: string): string;
+/** AUTO keeps conversational inputs read-only and enables BUILD only for repository work. */
+export declare function autoInteractionMode(intent: InteractionIntent): AutoInteractionMode;

package/dist/application/interaction-routing.js

@@ -0,0 +1,20 @@
+const simpleGreeting = /^(?:hi|hello|hey|yo|good\s+(?:morning|afternoon|evening)|thanks|thank\s+you)[!.\s]*$/i;
+/** Keeps obvious social messages local so they spend neither Jev nor coding-model capacity. */
+export function greetingResponse(input) {
+    if (!simpleGreeting.test(input.trim()))
+        return undefined;
+    if (/thank/i.test(input))
+        return "You're welcome. What would you like to work on?";
+    return "Hello! What would you like to inspect, plan, change, or verify?";
+}
+/** Builds minimal, credential-free context for Jev's pre-loop intent decision. */
+export function interactionIntentState(input) {
+    return [
+        `User message: ${input.replace(/(?:sk|gsk|or|AIza)[-_a-zA-Z0-9]{12,}/g, "[redacted]").slice(0, 1_500)}`,
+        "Classify the interaction before any repository context or tools are exposed.",
+    ].join("\n");
+}
+/** AUTO keeps conversational inputs read-only and enables BUILD only for repository work. */
+export function autoInteractionMode(intent) {
+    return intent === "repository_task" ? "build" : "plan";
+}

package/dist/application/live-evaluation.d.ts

@@ -0,0 +1,8 @@
+import type { LiveEvaluationResult } from "../domain/models.js";
+export type LiveEvaluationOptions = {
+    onProgress?: (text: string) => void;
+    apiKey: string;
+    model: string;
+};
+/** Evaluates live Gemini runs in disposable fixtures; DeepEval scores behavior, not the fixture assertion. */
+export declare function runLiveEvaluationSuite(options: LiveEvaluationOptions): Promise<LiveEvaluationResult[]>;

package/dist/application/live-evaluation.js

@@ -0,0 +1,185 @@
+import { cp, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { EvaluationDataset, Golden } from "deepeval/dataset";
+import { TaskCompletionMetric } from "deepeval/metrics";
+import { GeminiModel } from "deepeval/models";
+import { SpanType, observe, updateCurrentSpan } from "deepeval/tracing";
+import { SqliteSessionStore } from "../infrastructure/persistence/sqlite-session-store.js";
+import { GeminiProvider } from "../infrastructure/providers/gemini-provider.js";
+import { RepositoryProfiler } from "../infrastructure/repository/repository-profiler.js";
+import { WorkspaceTools, definitions } from "../infrastructure/tools/workspace-tools.js";
+import { CodingAgent } from "./coding-agent.js";
+import { evaluationScenarios } from "./evaluation-harness.js";
+import { taskMetrics } from "./task-metrics.js";
+import { runEvaluatedAgent } from "./evaluated-agent.js";
+const fixtures = join(process.cwd(), "evals", "fixtures");
+/** Evaluates live Gemini runs in disposable fixtures; DeepEval scores behavior, not the fixture assertion. */
+export async function runLiveEvaluationSuite(options) {
+    const dataset = new EvaluationDataset({
+        goldens: evaluationScenarios.map((scenario) => new Golden({
+            name: scenario.id,
+            input: scenario.prompt,
+            expectedOutput: expectedOutcome(scenario),
+        })),
+    });
+    const completion = new TaskCompletionMetric({
+        model: new GeminiModel({ apiKey: options.apiKey, model: options.model }),
+        threshold: 0.7,
+        includeReason: true,
+    });
+    const results = [];
+    let scenarioIndex = 0;
+    for await (const golden of dataset.evalsIterator({
+        metrics: [completion],
+        errorConfig: { ignoreErrors: true },
+        displayConfig: { printResults: false, showIndicator: false },
+        identifier: "kairo-live",
+    })) {
+        const scenario = evaluationScenarios[scenarioIndex++];
+        if (!(golden instanceof Golden))
+            throw new Error("Live coding evaluations require single-turn goldens.");
+        const deterministic = await runObservedScenario(scenario, golden.input, options);
+        results.push(deterministic);
+    }
+    return results.map((result, index) => {
+        const metric = dataset.evalResults[index]?.metricsData?.find((item) => item.name === "Task Completion");
+        const judge = {
+            passed: metric?.success === true,
+            score: metric?.score,
+            reason: metric?.reason,
+            error: metric?.error,
+        };
+        return { ...result, passed: result.passed && judge.passed, judge };
+    });
+}
+/** Runs one actual agent task under a DeepEval agent span, keeping raw workspace data out of the trace. */
+async function runObservedScenario(scenario, prompt, options) {
+    const root = await mkdtemp(join(tmpdir(), `kairo-live-eval-${scenario.id}-`));
+    try {
+        await cp(join(fixtures, scenario.id), root, { recursive: true });
+        const store = await SqliteSessionStore.open(":memory:");
+        try {
+            const session = store.create(root);
+            const tools = await WorkspaceTools.create(root);
+            store.saveRepositorySnapshot(session.id, await new RepositoryProfiler().profile(root));
+            const agent = new CodingAgent(new GeminiProvider(options.apiKey, options.model, definitions), store, tools, new FixtureApproval(), definitions, { provider: "gemini", model: options.model });
+            let failure = {};
+            const runAgent = observe({
+                type: SpanType.AGENT,
+                name: "kairo-coding-agent",
+                availableTools: definitions.map((definition) => definition.name),
+                fn: async (input) => {
+                    failure = await runEvaluatedAgent(agent, session.id, input, options.onProgress);
+                    const task = agent.status(session.id);
+                    const output = finalResponse(store.messages(session.id), task.error);
+                    const events = store.taskEvents(task.id);
+                    updateCurrentSpan({
+                        input,
+                        output,
+                        expectedOutput: expectedOutcome(scenario),
+                        toolsCalled: toolCalls(events),
+                        metadata: {
+                            taskStatus: task.status,
+                            changedFileCount: task.changedFiles.length,
+                            verificationPassed: task.verificationPassed === true,
+                            repairAttempts: store.repairAttempts(task.id).length,
+                        },
+                    });
+                    return output;
+                },
+            });
+            await runAgent(prompt);
+            const task = agent.status(session.id);
+            const expectationPassed = failure.error ? false : await scenario.expect(root);
+            const metrics = taskMetrics(store.taskEvents(task.id));
+            const verified = task.verificationPassed === true;
+            return {
+                id: scenario.id,
+                passed: task.status === "completed" && verified && expectationPassed,
+                taskStatus: task.status,
+                verified,
+                expectationPassed,
+                error: failure.error ?? task.error,
+                failureCategory: failure.category,
+                judge: { passed: false },
+                metrics: {
+                    providerRetries: metrics.providerRetries,
+                    providerWaitMs: metrics.providerWaitMs,
+                    modelTurns: metrics.modelTurns,
+                    toolExecutions: metrics.toolExecutions,
+                    toolFailures: metrics.toolFailures,
+                    approvals: metrics.approvals,
+                    repairs: metrics.repairs,
+                    verificationPasses: metrics.verificationPasses,
+                    verificationFailures: metrics.verificationFailures,
+                    verificationSelections: metrics.verificationSelections,
+                    focusedVerifications: metrics.focusedVerifications,
+                    broadVerifications: metrics.broadVerifications,
+                    repairConverged: metrics.repairConverged,
+                    modelMs: metrics.modelMs,
+                    toolMs: metrics.toolMs,
+                },
+            };
+        }
+        finally {
+            store.close();
+        }
+    }
+    catch (error) {
+        return {
+            id: scenario.id,
+            passed: false,
+            taskStatus: "failed",
+            verified: false,
+            expectationPassed: false,
+            error: error.message,
+            judge: { passed: false },
+            metrics: emptyMetrics(),
+        };
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+}
+/** Eval fixture mutations are intentionally pre-approved, unlike an interactive user workspace. */
+class FixtureApproval {
+    async approve(_call, _description) {
+        return true;
+    }
+}
+/** The final natural-language model message is useful judge context without exposing tool output. */
+function finalResponse(messages, error) {
+    return ([...messages].reverse().find((message) => message.role === "model" && !message.toolCallId)
+        ?.content ??
+        error ??
+        "Agent finished without a final response.");
+}
+/** Tool names retain the agent trajectory shape while withholding paths, arguments, and command output. */
+function toolCalls(events) {
+    return events
+        .filter((event) => event.kind === "tool_requested" && event.name)
+        .map((event) => ({ name: event.name }));
+}
+function expectedOutcome(scenario) {
+    return scenario.id === "create-file"
+        ? "Create result.txt containing done and run the fixture test successfully."
+        : "Repair message.txt so the fixture test passes after the initial failure.";
+}
+function emptyMetrics() {
+    return {
+        modelTurns: 0,
+        toolExecutions: 0,
+        toolFailures: 0,
+        approvals: 0,
+        repairs: 0,
+        verificationPasses: 0,
+        verificationFailures: 0,
+        verificationSelections: 0,
+        focusedVerifications: 0,
+        broadVerifications: 0,
+        repairConverged: false,
+        modelMs: 0,
+        toolMs: 0,
+    };
+}

package/dist/application/model-routing.d.ts

@@ -0,0 +1,12 @@
+import type { ModelSelection } from "../domain/models.js";
+import type { JevModelTier } from "../domain/ports.js";
+export type AvailableModel = ModelSelection & {
+    apiKey: string;
+    tier: JevModelTier;
+};
+/** Selects only from locally credentialed models; the manual selection is the deterministic fallback. */
+export declare function selectAutoModel(available: AvailableModel[], manual: ModelSelection, tier: JevModelTier): AvailableModel | undefined;
+/** Orders replacement models after quota exhaustion, preferring a different provider first. */
+export declare function quotaFallbackModels(available: AvailableModel[], selected: ModelSelection, manual: ModelSelection, tier: JevModelTier): AvailableModel[];
+/** Builds bounded, credential-free metadata for the TypeSafe decision request. */
+export declare function modelRoutingState(request: string, candidates: AvailableModel[]): string;

package/dist/application/model-routing.js

@@ -0,0 +1,40 @@
+/** Selects only from locally credentialed models; the manual selection is the deterministic fallback. */
+export function selectAutoModel(available, manual, tier) {
+    const preferred = available.filter((candidate) => candidate.tier === tier);
+    return (preferred.find((candidate) => candidate.provider === manual.provider && candidate.model === manual.model) ??
+        preferred[0] ??
+        available.find((candidate) => candidate.provider === manual.provider && candidate.model === manual.model) ??
+        available[0]);
+}
+/** Orders replacement models after quota exhaustion, preferring a different provider first. */
+export function quotaFallbackModels(available, selected, manual, tier) {
+    const remaining = available.filter((candidate) => candidate.provider !== selected.provider || candidate.model !== selected.model);
+    const rank = (candidate) => {
+        const sameProvider = candidate.provider === selected.provider;
+        const sameTier = candidate.tier === tier;
+        const manualModel = candidate.provider === manual.provider && candidate.model === manual.model;
+        if (!sameProvider && sameTier)
+            return 0;
+        if (!sameProvider && manualModel)
+            return 1;
+        if (!sameProvider)
+            return 2;
+        if (sameTier)
+            return 3;
+        if (manualModel)
+            return 4;
+        return 5;
+    };
+    return remaining.sort((left, right) => rank(left) - rank(right));
+}
+/** Builds bounded, credential-free metadata for the TypeSafe decision request. */
+export function modelRoutingState(request, candidates) {
+    return [
+        `Task request: ${request.replace(/(?:sk|gsk|or|AIza)[-_a-zA-Z0-9]{12,}/g, "[redacted]").slice(0, 1_500)}`,
+        `Available model tiers: ${candidates
+            .map((candidate) => `${candidate.provider}/${candidate.model}=${candidate.tier}`)
+            .join(", ")
+            .slice(0, 1_500)}`,
+        "Choose the coding-model tier only. Credentials, source files, and tool output are not included.",
+    ].join("\n");
+}

package/dist/application/model-system-instruction.d.ts

@@ -0,0 +1,4 @@
+/** Shared behavior contract applied consistently by every model provider. */
+export declare const modelSystemInstruction = "You are Kairo, a careful coding agent. Work only through the provided tools. Inspect relevant files before changing code. After any edit, run an appropriate verification command before declaring success. When a tool fails, inspect its error and try a materially different repair; do not repeat the same call. Keep tool use focused because outputs may be truncated and execution is bounded. Explain the completed work, verification evidence, and remaining limitations concisely.";
+/** Keeps non-repository answers conversational and prevents accidental tool-oriented behavior. */
+export declare const conversationSystemInstruction = "You are Kairo. Answer the user's general question directly and concisely. Do not claim to inspect, test, change, or know anything about a repository. Tools are unavailable for this response; ask the user to explicitly request repository work if they need it.";

package/dist/application/model-system-instruction.js

@@ -0,0 +1,4 @@
+/** Shared behavior contract applied consistently by every model provider. */
+export const modelSystemInstruction = "You are Kairo, a careful coding agent. Work only through the provided tools. Inspect relevant files before changing code. After any edit, run an appropriate verification command before declaring success. When a tool fails, inspect its error and try a materially different repair; do not repeat the same call. Keep tool use focused because outputs may be truncated and execution is bounded. Explain the completed work, verification evidence, and remaining limitations concisely.";
+/** Keeps non-repository answers conversational and prevents accidental tool-oriented behavior. */
+export const conversationSystemInstruction = "You are Kairo. Answer the user's general question directly and concisely. Do not claim to inspect, test, change, or know anything about a repository. Tools are unavailable for this response; ask the user to explicitly request repository work if they need it.";

package/dist/application/self-evaluation.d.ts

@@ -0,0 +1,26 @@
+import type { EvaluationRun, SelfEvaluationResult, ProviderId } from "../domain/models.js";
+import type { EvaluationStore } from "../domain/ports.js";
+export type SelfEvaluationScenario = {
+    id: string;
+    prompt: string;
+    seed(workspace: string): Promise<void>;
+    /** Hidden behavioral check, separate from the agent's own verification command. */
+    verify(workspace: string): Promise<void>;
+};
+export type SelfEvaluationOptions = {
+    apiKey: string;
+    provider: ProviderId;
+    model: string;
+    evaluationStore: EvaluationStore;
+    trials?: number;
+    sourceRoot?: string;
+    onProgress?: (text: string) => void;
+};
+export type SelfEvaluationRun = {
+    run: EvaluationRun;
+    results: SelfEvaluationResult[];
+};
+/** Seven real Kairo safeguards, deliberately removed from isolated source copies. */
+export declare const selfEvaluationScenarios: SelfEvaluationScenario[];
+/** Runs real provider-backed tasks against clean Git-free snapshots and persists metadata-only evidence. */
+export declare function runSelfEvaluationSuite(options: SelfEvaluationOptions): Promise<SelfEvaluationRun>;