agentera 0.0.0 → 3.0.0-dev.0

package/dist/eval/evalSkills.js

@@ -0,0 +1,258 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { resolveSourceRoot } from "../core/sourceRoot.js";
+export const TRIGGER_PROMPTS = {
+    dokumentera: "Audit the documentation for this project.",
+    hej: "Start a new session and give me a status briefing on this project.",
+    inspektera: "Run a codebase health audit.",
+    inspirera: "Analyze https://example.com and map patterns to this project.",
+    optimera: "Optimize test suite execution time.",
+    orkestrera: "Execute the next cycle of the current plan.",
+    planera: "Plan the next feature for this project.",
+    profilera: "Generate a decision profile from session history.",
+    realisera: "Run one autonomous development cycle.",
+    resonera: "Deliberate on whether to add a new dependency.",
+    visionera: "Create a vision document for this project.",
+    visualisera: "Create a visual identity system for this project.",
+};
+export const DEFAULT_TIMEOUT = 120;
+export const DEFAULT_PARALLEL = 1;
+export class ExitError extends Error {
+    code;
+    constructor(code, message = "") {
+        super(message);
+        this.name = "ExitError";
+        this.code = code;
+    }
+}
+const realWhich = (name) => {
+    const result = spawnSync(process.platform === "win32" ? "where" : "which", [name], {
+        encoding: "utf8",
+    });
+    return result.status === 0 ? result.stdout.trim().split(/\r?\n/)[0] : null;
+};
+export function detectRuntime(explicit, opts = {}) {
+    const which = opts.which ?? realWhich;
+    const err = opts.err ?? ((line) => process.stderr.write(line + "\n"));
+    if (explicit && explicit !== "auto") {
+        if (explicit === "cursor-agent") {
+            if (which("cursor-agent") === null && which("agent") === null) {
+                err("ERROR: 'cursor-agent' not found on PATH. Install Cursor Agent CLI " +
+                    "and ensure the binary is accessible.");
+                throw new ExitError(1);
+            }
+        }
+        return explicit;
+    }
+    const hasClaude = which("claude") !== null;
+    const hasOpencode = which("opencode") !== null;
+    const hasCursorAgent = which("cursor-agent") !== null || which("agent") !== null;
+    if (hasClaude)
+        return "claude";
+    if (hasOpencode)
+        return "opencode";
+    if (hasCursorAgent)
+        return "cursor-agent";
+    err("ERROR: Neither 'claude', 'opencode', nor 'cursor-agent' found on PATH. " +
+        "Install a supported runtime host and ensure the binary is accessible.");
+    throw new ExitError(1);
+}
+export function parseFrontmatterName(text) {
+    if (!text.startsWith("---")) {
+        return null;
+    }
+    const end = text.indexOf("---", 3);
+    if (end === -1) {
+        return null;
+    }
+    const block = text.slice(3, end);
+    const m = /^name:\s*(.+)/m.exec(block);
+    return m ? m[1].trim() : null;
+}
+export function discoverSkills(repoRoot = resolveSourceRoot()) {
+    const skillsDir = path.join(repoRoot, "skills");
+    const entries = [];
+    if (!fs.existsSync(skillsDir) || !fs.statSync(skillsDir).isDirectory()) {
+        return entries;
+    }
+    const dirs = fs
+        .readdirSync(skillsDir)
+        .filter((d) => fs.existsSync(path.join(skillsDir, d, "SKILL.md")))
+        .sort();
+    for (const dir of dirs) {
+        const skillMd = path.join(skillsDir, dir, "SKILL.md");
+        const text = fs.readFileSync(skillMd, "utf8");
+        const name = parseFrontmatterName(text) ?? dir;
+        const prompt = TRIGGER_PROMPTS[name] ?? `Invoke the ${name} skill.`;
+        entries.push({ name, prompt });
+    }
+    return entries;
+}
+const realRun = (cmd, opts) => {
+    const result = spawnSync(cmd[0], cmd.slice(1), {
+        input: opts.input ?? undefined,
+        encoding: "utf8",
+        timeout: opts.timeout * 1000,
+        cwd: opts.cwd,
+    });
+    return {
+        status: result.status,
+        stdout: result.stdout ?? "",
+        stderr: result.stderr ?? "",
+        timedOut: result.signal === "SIGTERM" && result.error !== undefined,
+    };
+};
+export function invokeSkill(name, prompt, timeout, runtime = "claude", opts = {}) {
+    const which = opts.which ?? realWhich;
+    const run = opts.run ?? realRun;
+    const repoRoot = opts.repoRoot ?? resolveSourceRoot();
+    let cmd;
+    let stdinPrompt;
+    if (runtime === "opencode") {
+        cmd = ["opencode", "run", "--prompt"];
+        stdinPrompt = prompt;
+    }
+    else if (runtime === "cursor-agent") {
+        const binary = which("cursor-agent") ? "cursor-agent" : "agent";
+        cmd = [binary, "-p", "--output-format", "json", "--force", prompt];
+        stdinPrompt = null;
+    }
+    else {
+        cmd = ["claude", "-p", "--output-format", "json"];
+        stdinPrompt = prompt;
+    }
+    const start = Date.now();
+    let error = null;
+    let status = "pass";
+    const result = run(cmd, { input: stdinPrompt, timeout, cwd: repoRoot });
+    const duration = (Date.now() - start) / 1000;
+    if (result.timedOut) {
+        error = `Timed out after ${timeout}s`;
+        status = "fail";
+    }
+    else if (result.status !== 0) {
+        const stderrSnippet = (result.stderr ?? "").trim().slice(0, 300);
+        const stdoutSnippet = (result.stdout ?? "").trim().slice(0, 300);
+        const detail = stderrSnippet || stdoutSnippet || "(no output)";
+        error = `Exit code ${result.status}: ${detail}`;
+        status = "fail";
+    }
+    else {
+        const combined = (result.stdout ?? "") + (result.stderr ?? "");
+        const errorIndicators = [
+            /\bTraceback \(most recent call last\)\b/i,
+            /\bError:\s/i,
+            /\bfatal error\b/i,
+            /"is_error"\s*:\s*true/i,
+        ];
+        for (const pattern of errorIndicators) {
+            const m = pattern.exec(combined);
+            if (m) {
+                const snippet = combined.slice(Math.max(0, m.index - 20), m.index + m[0].length + 80).trim();
+                error = `Error indicator in output: ${snippet.slice(0, 300)}`;
+                status = "fail";
+                break;
+            }
+        }
+    }
+    return {
+        skill: name,
+        status,
+        duration_s: Math.round(duration * 100) / 100,
+        error,
+    };
+}
+export function runSkills(skills, timeout, _parallel, runtime = "claude", opts = {}) {
+    // Parallelism in the Python runner is a maintainer perf detail; the smoke
+    // semantics (one result per skill, original order) are preserved sequentially.
+    return skills.map((entry) => invokeSkill(entry.name, entry.prompt, timeout, runtime, opts));
+}
+export function buildReport(results) {
+    const passed = results.filter((r) => r.status === "pass").length;
+    const failed = results.length - passed;
+    return {
+        timestamp: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
+        skills_tested: results.length,
+        passed,
+        failed,
+        results,
+    };
+}
+export function buildDryRun(skills, runtime = "claude", runtimeSource = "auto-detected") {
+    return {
+        mode: "dry-run",
+        runtime: `${runtime} (${runtimeSource})`,
+        skills: skills.map((s) => ({ name: s.name, prompt: s.prompt })),
+    };
+}
+export function main(argv = [], opts = {}) {
+    const out = opts.out ?? ((line) => process.stdout.write(line + "\n"));
+    const err = opts.err ?? ((line) => process.stderr.write(line + "\n"));
+    const discover = opts.discoverSkills ?? (() => discoverSkills(opts.repoRoot));
+    const detect = opts.detectRuntime ?? detectRuntime;
+    const args = parseArgs(argv);
+    const allSkills = discover();
+    if (allSkills.length === 0) {
+        err("ERROR: No SKILL.md files found under skills/");
+        return 1;
+    }
+    let skillsToRun = allSkills;
+    if (args.skill) {
+        const matched = allSkills.filter((s) => s.name === args.skill);
+        if (matched.length === 0) {
+            const known = allSkills.map((s) => s.name).join(", ");
+            err(`ERROR: Unknown skill '${args.skill}'. Known skills: ${known}`);
+            return 1;
+        }
+        skillsToRun = matched;
+    }
+    const explicit = args.runtime !== "auto" ? args.runtime : null;
+    let runtime;
+    try {
+        runtime = detect(explicit, { which: opts.which, err });
+    }
+    catch (exc) {
+        if (exc instanceof ExitError) {
+            return exc.code;
+        }
+        throw exc;
+    }
+    const runtimeSource = explicit ? "--runtime flag" : "auto-detected";
+    if (args.dry_run) {
+        out(JSON.stringify(buildDryRun(skillsToRun, runtime, runtimeSource), null, 2));
+        return 0;
+    }
+    const results = runSkills(skillsToRun, args.timeout, args.parallel, runtime, {
+        which: opts.which,
+        run: opts.run,
+        repoRoot: opts.repoRoot,
+    });
+    const report = buildReport(results);
+    out(JSON.stringify(report, null, 2));
+    return report.failed === 0 ? 0 : 1;
+}
+export function parseArgs(argv) {
+    const args = {
+        skill: null,
+        dry_run: false,
+        parallel: DEFAULT_PARALLEL,
+        timeout: DEFAULT_TIMEOUT,
+        runtime: "auto",
+    };
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg === "--skill")
+            args.skill = argv[++i];
+        else if (arg === "--dry-run")
+            args.dry_run = true;
+        else if (arg === "--parallel")
+            args.parallel = parseInt(argv[++i], 10);
+        else if (arg === "--timeout")
+            args.timeout = parseInt(argv[++i], 10);
+        else if (arg === "--runtime")
+            args.runtime = argv[++i];
+    }
+    return args;
+}
+//# sourceMappingURL=evalSkills.js.map

package/dist/eval/evalSkills.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalSkills.js","sourceRoot":"","sources":["../../src/eval/evalSkills.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAU1D,MAAM,CAAC,MAAM,eAAe,GAA2B;IACrD,WAAW,EAAE,2CAA2C;IACxD,GAAG,EAAE,oEAAoE;IACzE,UAAU,EAAE,8BAA8B;IAC1C,SAAS,EAAE,+DAA+D;IAC1E,QAAQ,EAAE,qCAAqC;IAC/C,UAAU,EAAE,6CAA6C;IACzD,OAAO,EAAE,yCAAyC;IAClD,SAAS,EAAE,mDAAmD;IAC9D,SAAS,EAAE,uCAAuC;IAClD,QAAQ,EAAE,gDAAgD;IAC1D,SAAS,EAAE,4CAA4C;IACvD,WAAW,EAAE,mDAAmD;CACjE,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AACnC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,OAAO,SAAU,SAAQ,KAAK;IAClC,IAAI,CAAS;IACb,YAAY,IAAY,EAAE,OAAO,GAAG,EAAE;QACpC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF;AAGD,MAAM,SAAS,GAAY,CAAC,IAAI,EAAE,EAAE;IAClC,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE;QACjF,QAAQ,EAAE,MAAM;KACjB,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7E,CAAC,CAAC;AAEF,MAAM,UAAU,aAAa,CAC3B,QAAuB,EACvB,OAA0D,EAAE;IAE5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC;IACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IAC9E,IAAI,QAAQ,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACpC,IAAI,QAAQ,KAAK,cAAc,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,cAAc,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC9D,GAAG,CACD,oEAAoE;oBAClE,sCAAsC,CACzC,CAAC;gBACF,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;IAC3C,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC;IAC/C,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC;IAEjF,IAAI,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC/B,IAAI,WAAW;QAAE,OAAO,UAAU,CAAC;IACnC,IAAI,cAAc;QAAE,OAAO,cAAc,CAAC;IAE1C,GAAG,CACD,yEAAyE;QACvE,uEAAuE,CAC1E,CAAC;IACF,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACnC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,WAAmB,iBAAiB,EAAE;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,GAA4C,EAAE,CAAC;IAC5D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QACvE,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,IAAI,GAAG,EAAE;SACZ,WAAW,CAAC,SAAS,CAAC;SACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;SACjE,IAAI,EAAE,CAAC;IACV,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;QAC/C,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,cAAc,IAAI,SAAS,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD,MAAM,OAAO,GAAU,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IACnC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC7C,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;QAC9B,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,IAAI,CAAC,OAAO,GAAG,IAAI;QAC5B,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC,CAAC;IACH,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;QAC3B,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;KACpE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,MAAc,EACd,OAAe,EACf,OAAO,GAAG,QAAQ,EAClB,OAA4D,EAAE;IAE9D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC;IACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,EAAE,CAAC;IAEtD,IAAI,GAAa,CAAC;IAClB,IAAI,WAA0B,CAAC;IAC/B,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;QAC3B,GAAG,GAAG,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;QACtC,WAAW,GAAG,MAAM,CAAC;IACvB,CAAC;SAAM,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;QAChE,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QACnE,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAClD,WAAW,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,MAAM,GAAG,MAAM,CAAC;IAEpB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC;IAE7C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,KAAK,GAAG,mBAAmB,OAAO,GAAG,CAAC;QACtC,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACjE,MAAM,aAAa,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,aAAa,IAAI,aAAa,IAAI,aAAa,CAAC;QAC/D,KAAK,GAAG,aAAa,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAChD,MAAM,GAAG,MAAM,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,eAAe,GAAG;YACtB,0CAA0C;YAC1C,aAAa;YACb,kBAAkB;YAClB,wBAAwB;SACzB,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,EAAE,CAAC;gBACN,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC7F,KAAK,GAAG,8BAA8B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;gBAC9D,MAAM,GAAG,MAAM,CAAC;gBAChB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,IAAI;QACX,MAAM;QACN,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG;QAC5C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,MAA+C,EAC/C,OAAe,EACf,SAAiB,EACjB,OAAO,GAAG,QAAQ,EAClB,OAA4D,EAAE;IAE9D,0EAA0E;IAC1E,+EAA+E;IAC/E,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AAC9F,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IACvC,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;QAC7D,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,MAAM;QACN,MAAM;QACN,OAAO;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,MAA+C,EAC/C,OAAO,GAAG,QAAQ,EAClB,aAAa,GAAG,eAAe;IAE/B,OAAO;QACL,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,GAAG,OAAO,KAAK,aAAa,GAAG;QACxC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;KAChE,CAAC;AACJ,CAAC;AAoBD,MAAM,UAAU,IAAI,CAAC,OAAiB,EAAE,EAAE,OAAwB,EAAE;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IAC9E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC;IACnD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,SAAS,GAAG,QAAQ,EAAE,CAAC;IAC7B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,8CAA8C,CAAC,CAAC;QACpD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,WAAW,GAAG,SAAS,CAAC;IAC5B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtD,GAAG,CAAC,yBAAyB,IAAI,CAAC,KAAK,oBAAoB,KAAK,EAAE,CAAC,CAAC;YACpE,OAAO,CAAC,CAAC;QACX,CAAC;QACD,WAAW,GAAG,OAAO,CAAC;IACxB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;YAC7B,OAAO,GAAG,CAAC,IAAI,CAAC;QAClB,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,eAAe,CAAC;IAEpE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,EAAE,aAAa,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE;QAC3E,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;KACxB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,IAAI,GAAa;QACrB,KAAK,EAAE,IAAI;QACX,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,gBAAgB;QAC1B,OAAO,EAAE,eAAe;QACxB,OAAO,EAAE,MAAM;KAChB,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;aACzC,IAAI,GAAG,KAAK,WAAW;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;aAC7C,IAAI,GAAG,KAAK,YAAY;YAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;aAClE,IAAI,GAAG,KAAK,WAAW;YAAE,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;aAChE,IAAI,GAAG,KAAK,WAAW;YAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/eval/semanticEval.js

@@ -0,0 +1,148 @@
+import { loadFixture, pyRepr } from "./semanticFixtures.js";
+import { pyJsonIndent } from "../core/pyjson.js";
+function utcTimestamp() {
+    return new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+export function evaluateFixture(fixture, source = "<fixture>") {
+    const facts = [
+        ...checkOutputFacts(fixture),
+        ...checkToolTraceFacts(fixture),
+        ...checkSeededArtifactFacts(fixture),
+    ];
+    const failing = facts.find((fact) => fact.status === "fail") ?? null;
+    return {
+        fixture: source,
+        status: failing ? "fail" : "pass",
+        checked_facts: facts.map((f) => ({ fact: f.fact, status: f.status, detail: f.detail })),
+        failing_fact: failing ? { fact: failing.fact, status: failing.status, detail: failing.detail } : null,
+    };
+}
+export function evaluateFixtureFile(path) {
+    const [fixture, errors] = loadFixture(path);
+    if (errors.length > 0) {
+        const failing = { fact: "fixture_contract", status: "fail", detail: errors.join("; ") };
+        return {
+            fixture: path,
+            status: "fail",
+            checked_facts: [failing],
+            failing_fact: failing,
+        };
+    }
+    return evaluateFixture(fixture, path);
+}
+export function buildReport(results) {
+    const passed = results.filter((result) => result.status === "pass").length;
+    const failed = results.length - passed;
+    return {
+        timestamp: utcTimestamp(),
+        status: failed ? "fail" : "pass",
+        fixtures_tested: results.length,
+        passed,
+        failed,
+        results,
+    };
+}
+function checkOutputFacts(fixture) {
+    const facts = [];
+    const expected = fixture.expectedFacts;
+    (expected.required_output ?? []).forEach((text, index) => {
+        const found = fixture.capturedOutput.includes(text);
+        facts.push({
+            fact: `required_output[${index}]`,
+            status: found ? "pass" : "fail",
+            detail: `captured output ${found ? "contains" : "does not contain"} ${pyRepr(text)}`,
+        });
+    });
+    (expected.forbidden_output ?? []).forEach((text, index) => {
+        const found = fixture.capturedOutput.includes(text);
+        facts.push({
+            fact: `forbidden_output[${index}]`,
+            status: found ? "fail" : "pass",
+            detail: `captured output ${found ? "contains forbidden" : "omits forbidden"} ${pyRepr(text)}`,
+        });
+    });
+    const writes = expected.artifact_expectations?.writes;
+    if (writes === "none") {
+        facts.push({
+            fact: "artifact_expectations.writes",
+            status: "pass",
+            detail: "fixture expects no artifact writes; offline eval performed none",
+        });
+    }
+    return facts;
+}
+function checkToolTraceFacts(fixture) {
+    const facts = [];
+    const expected = fixture.expectedFacts;
+    const callList = fixture.toolTrace.calls ?? [];
+    const calls = callList.join("\n");
+    (expected.required_tool_calls ?? []).forEach((text, index) => {
+        const found = calls.includes(text);
+        facts.push({
+            fact: `required_tool_calls[${index}]`,
+            status: found ? "pass" : "fail",
+            detail: `tool trace ${found ? "contains" : "does not contain"} ${pyRepr(text)}`,
+        });
+    });
+    (expected.forbidden_tool_calls ?? []).forEach((text, index) => {
+        const found = calls.includes(text);
+        facts.push({
+            fact: `forbidden_tool_calls[${index}]`,
+            status: found ? "fail" : "pass",
+            detail: `tool trace ${found ? "contains forbidden" : "omits forbidden"} ${pyRepr(text)}`,
+        });
+    });
+    for (const [text, expectedCount] of Object.entries(expected.tool_call_counts ?? {})) {
+        const actual = callList.filter((call) => call.includes(text)).length;
+        facts.push({
+            fact: `tool_call_counts[${text}]`,
+            status: actual === expectedCount ? "pass" : "fail",
+            detail: `tool trace contains ${actual} call(s) matching ${pyRepr(text)}; expected ${expectedCount}`,
+        });
+    }
+    return facts;
+}
+function checkSeededArtifactFacts(fixture) {
+    const facts = [];
+    const byPath = {};
+    for (const item of fixture.seededState.files ?? []) {
+        if (item && typeof item === "object" && typeof item.path === "string") {
+            byPath[item.path] = item.content;
+        }
+    }
+    (fixture.expectedFacts.required_artifacts ?? []).forEach((expected, index) => {
+        const factName = `required_artifacts[${index}]`;
+        if (!expected || typeof expected !== "object" || Array.isArray(expected)) {
+            facts.push({ fact: factName, status: "fail", detail: "expected artifact fact must be an object" });
+            return;
+        }
+        const path = expected.path;
+        if (typeof path !== "string" || !path.trim()) {
+            facts.push({ fact: factName, status: "fail", detail: "expected artifact fact must name a path" });
+            return;
+        }
+        const content = byPath[path];
+        if (content === undefined) {
+            facts.push({ fact: factName, status: "fail", detail: `seeded artifact ${pyRepr(path)} is missing` });
+            return;
+        }
+        const missing = (expected.contains ?? []).filter((text) => !content.includes(text));
+        if (missing.length > 0) {
+            facts.push({ fact: factName, status: "fail", detail: `seeded artifact ${pyRepr(path)} lacks ${pyRepr(missing[0])}` });
+        }
+        else {
+            facts.push({ fact: factName, status: "pass", detail: `seeded artifact ${pyRepr(path)} matched` });
+        }
+    });
+    return facts;
+}
+export function main(argv = [], out = (l) => process.stdout.write(l)) {
+    if (argv.length === 0) {
+        process.stderr.write("usage: semantic_eval <fixtures...>\n");
+        return 2;
+    }
+    const report = buildReport(argv.map((path) => evaluateFixtureFile(path)));
+    out(pyJsonIndent(report) + "\n");
+    return report.status === "pass" ? 0 : 1;
+}
+//# sourceMappingURL=semanticEval.js.map

package/dist/eval/semanticEval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semanticEval.js","sourceRoot":"","sources":["../../src/eval/semanticEval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,WAAW,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAejD,SAAS,YAAY;IACnB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAwB,EAAE,MAAM,GAAG,WAAW;IAC5E,MAAM,KAAK,GAAG;QACZ,GAAG,gBAAgB,CAAC,OAAO,CAAC;QAC5B,GAAG,mBAAmB,CAAC,OAAO,CAAC;QAC/B,GAAG,wBAAwB,CAAC,OAAO,CAAC;KACrC,CAAC;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC;IACrE,OAAO;QACL,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QACjC,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACvF,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI;KACtG,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,OAAO,GAAgB,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACrG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,CAAC,OAAO,CAAC;YACxB,YAAY,EAAE,OAAO;SACtB,CAAC;IACJ,CAAC;IACD,OAAO,eAAe,CAAC,OAAQ,EAAE,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IACvC,OAAO;QACL,SAAS,EAAE,YAAY,EAAE;QACzB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QAChC,eAAe,EAAE,OAAO,CAAC,MAAM;QAC/B,MAAM;QACN,MAAM;QACN,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAwB;IAChD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,CAAC;IACvC,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,mBAAmB,KAAK,GAAG;YACjC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC/B,MAAM,EAAE,mBAAmB,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,kBAAkB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;SACrF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,CAAC,QAAQ,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;QACxE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,oBAAoB,KAAK,GAAG;YAClC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC/B,MAAM,EAAE,mBAAmB,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,iBAAiB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;SAC9F,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACtD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,8BAA8B;YACpC,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,iEAAiE;SAC1E,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAwB;IACnD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,CAAC;IACvC,MAAM,QAAQ,GAAa,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC,QAAQ,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;QAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,uBAAuB,KAAK,GAAG;YACrC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC/B,MAAM,EAAE,cAAc,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,kBAAkB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;SAChF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,CAAC,QAAQ,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;QAC5E,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,wBAAwB,KAAK,GAAG;YACtC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC/B,MAAM,EAAE,cAAc,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,iBAAiB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;SACzF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,KAAK,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,IAAI,EAAE,CAAC,EAAE,CAAC;QACpF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,oBAAoB,IAAI,GAAG;YACjC,MAAM,EAAE,MAAM,KAAK,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAClD,MAAM,EAAE,uBAAuB,MAAM,qBAAqB,MAAM,CAAC,IAAI,CAAC,cAAc,aAAa,EAAE;SACpG,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,wBAAwB,CAAC,OAAwB;IACxD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QACnD,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QACnC,CAAC;IACH,CAAC;IAED,CAAC,OAAO,CAAC,aAAa,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAa,EAAE,KAAa,EAAE,EAAE;QACxF,MAAM,QAAQ,GAAG,sBAAsB,KAAK,GAAG,CAAC;QAChD,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,0CAA0C,EAAE,CAAC,CAAC;YACnG,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC,CAAC;YAClG,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,mBAAmB,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;YACrG,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,mBAAmB,MAAM,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,mBAAmB,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,IAAI,CAAC,OAAiB,EAAE,EAAE,MAA8B,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACpG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC7D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1E,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,OAAO,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,CAAC"}

package/dist/eval/semanticFixtures.js

@@ -0,0 +1,227 @@
+import fs from "node:fs";
+export const REQUIRED_SECTIONS = [
+    "Prompt",
+    "Seeded Project State",
+    "Captured Output",
+    "Expected Facts",
+];
+function nonEmptyString(value) {
+    return typeof value === "string" && value.trim().length > 0;
+}
+export function loadFixture(path) {
+    return validateFixtureText(fs.readFileSync(path, "utf8"));
+}
+export function validateFixtureText(text) {
+    const sections = parseSections(text);
+    const errors = [];
+    for (const section of REQUIRED_SECTIONS) {
+        if (!(section in sections)) {
+            errors.push(`missing section: ${section}`);
+        }
+    }
+    const prompt = (sections.Prompt ?? "").trim();
+    if ("Prompt" in sections && !prompt) {
+        errors.push("malformed section: Prompt: must be non-empty");
+    }
+    let seededState = {};
+    if ("Seeded Project State" in sections) {
+        const [state, stateErrors] = validateSeededState(sections["Seeded Project State"]);
+        seededState = state;
+        errors.push(...stateErrors);
+    }
+    const capturedOutput = (sections["Captured Output"] ?? "").trim();
+    if ("Captured Output" in sections && !capturedOutput) {
+        errors.push("malformed section: Captured Output: must be non-empty");
+    }
+    let toolTrace = { calls: [] };
+    if ("Tool Trace" in sections) {
+        const [trace, traceErrors] = validateToolTrace(sections["Tool Trace"]);
+        toolTrace = trace;
+        errors.push(...traceErrors);
+    }
+    let expectedFacts = {};
+    if ("Expected Facts" in sections) {
+        const [facts, factErrors] = validateExpectedFacts(sections["Expected Facts"]);
+        expectedFacts = facts;
+        errors.push(...factErrors);
+    }
+    if (errors.length > 0) {
+        return [null, errors];
+    }
+    return [{ prompt, seededState, capturedOutput, toolTrace, expectedFacts }, []];
+}
+function parseSections(text) {
+    const sections = {};
+    let current = null;
+    for (const line of text.split(/\r\n|\r|\n/)) {
+        const match = /^##\s+(.+?)\s*$/.exec(line);
+        if (match) {
+            current = match[1];
+            if (!(current in sections)) {
+                sections[current] = [];
+            }
+            continue;
+        }
+        if (current !== null) {
+            sections[current].push(line);
+        }
+    }
+    const out = {};
+    for (const [name, lines] of Object.entries(sections)) {
+        out[name] = lines.join("\n").trim();
+    }
+    return out;
+}
+function validateSeededState(sectionText) {
+    const [data, errors] = loadJsonSection("Seeded Project State", sectionText);
+    if (errors.length > 0) {
+        return [{}, errors];
+    }
+    if (data === null || typeof data !== "object" || Array.isArray(data)) {
+        return [{}, ["malformed section: Seeded Project State: JSON must be an object"]];
+    }
+    const files = data.files;
+    if (!Array.isArray(files)) {
+        return [{}, ["malformed section: Seeded Project State: files must be a list"]];
+    }
+    for (let index = 0; index < files.length; index++) {
+        const item = files[index];
+        if (!item || typeof item !== "object" || Array.isArray(item)) {
+            return [{}, [`malformed section: Seeded Project State: files[${index}] must be an object`]];
+        }
+        if (!nonEmptyString(item.path)) {
+            return [{}, [`malformed section: Seeded Project State: files[${index}].path must be non-empty`]];
+        }
+        if (typeof item.content !== "string") {
+            return [{}, [`malformed section: Seeded Project State: files[${index}].content must be a string`]];
+        }
+    }
+    return [data, []];
+}
+function validateExpectedFacts(sectionText) {
+    const [data, jsonErrors] = loadJsonSection("Expected Facts", sectionText);
+    if (jsonErrors.length > 0) {
+        return [{}, jsonErrors];
+    }
+    if (data === null || typeof data !== "object" || Array.isArray(data)) {
+        return [{}, ["malformed section: Expected Facts: JSON must be an object"]];
+    }
+    const d = data;
+    const errors = [];
+    errors.push(...validateStringList(d, "required_output", false));
+    errors.push(...validateStringList(d, "forbidden_output", false));
+    errors.push(...validateStringList(d, "required_tool_calls", false));
+    errors.push(...validateStringList(d, "forbidden_tool_calls", false));
+    errors.push(...validateToolCallCounts(d.tool_call_counts));
+    const hasOutputFact = Boolean(d.required_output?.length || d.forbidden_output?.length);
+    const hasToolFact = Boolean(d.required_tool_calls?.length ||
+        d.forbidden_tool_calls?.length ||
+        (d.tool_call_counts && Object.keys(d.tool_call_counts).length));
+    const hasArtifactFact = "artifact_expectations" in d;
+    if (!hasOutputFact && !hasToolFact && !hasArtifactFact) {
+        errors.push("malformed section: Expected Facts: must declare at least one expected fact");
+    }
+    if (hasArtifactFact) {
+        errors.push(...validateArtifactExpectations(d.artifact_expectations));
+    }
+    return [d, errors];
+}
+function validateToolTrace(sectionText) {
+    const [data, errors] = loadJsonSection("Tool Trace", sectionText);
+    if (errors.length > 0) {
+        return [{}, errors];
+    }
+    if (data === null || typeof data !== "object" || Array.isArray(data)) {
+        return [{}, ["malformed section: Tool Trace: JSON must be an object"]];
+    }
+    const calls = data.calls;
+    if (!Array.isArray(calls) || !calls.every((item) => nonEmptyString(item))) {
+        return [{}, ["malformed section: Tool Trace: calls must be non-empty strings"]];
+    }
+    return [data, []];
+}
+function validateArtifactExpectations(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return ["malformed section: Expected Facts: artifact_expectations must be an object"];
+    }
+    const writes = value.writes;
+    if (writes === "none") {
+        return [];
+    }
+    if (!Array.isArray(writes)) {
+        return ["malformed section: Expected Facts: artifact_expectations.writes must be 'none' or a list"];
+    }
+    for (let index = 0; index < writes.length; index++) {
+        const item = writes[index];
+        if (!item || typeof item !== "object" || Array.isArray(item)) {
+            return [`malformed section: Expected Facts: artifact_expectations.writes[${index}] must be an object`];
+        }
+        if (!nonEmptyString(item.path)) {
+            return [`malformed section: Expected Facts: artifact_expectations.writes[${index}].path must be non-empty`];
+        }
+        if ("contains" in item) {
+            const contains = item.contains;
+            if (!Array.isArray(contains) || !contains.every((s) => nonEmptyString(s))) {
+                return [
+                    `malformed section: Expected Facts: artifact_expectations.writes[${index}].contains must be non-empty strings`,
+                ];
+            }
+        }
+    }
+    return [];
+}
+function validateToolCallCounts(value) {
+    if (value === undefined || value === null) {
+        return [];
+    }
+    if (typeof value !== "object" || Array.isArray(value)) {
+        return ["malformed section: Expected Facts: tool_call_counts must be an object"];
+    }
+    for (const [key, count] of Object.entries(value)) {
+        if (!nonEmptyString(key)) {
+            return ["malformed section: Expected Facts: tool_call_counts keys must be non-empty strings"];
+        }
+        if (typeof count !== "number" || !Number.isInteger(count) || count < 0) {
+            return [`malformed section: Expected Facts: tool_call_counts[${pyRepr(key)}] must be a non-negative integer`];
+        }
+    }
+    return [];
+}
+function validateStringList(data, key, required) {
+    if (!(key in data)) {
+        return required ? [`malformed section: Expected Facts: ${key} is required`] : [];
+    }
+    const value = data[key];
+    if (!Array.isArray(value) || !value.every((item) => nonEmptyString(item))) {
+        return [`malformed section: Expected Facts: ${key} must be non-empty strings`];
+    }
+    return [];
+}
+function loadJsonSection(sectionName, text) {
+    const block = extractJsonBlock(text);
+    if (block === null) {
+        return [null, [`malformed section: ${sectionName}: expected fenced json block`]];
+    }
+    try {
+        return [JSON.parse(block), []];
+    }
+    catch {
+        // Best-effort line number; the offline-eval path always feeds valid JSON.
+        return [null, [`malformed section: ${sectionName}: invalid JSON`]];
+    }
+}
+function extractJsonBlock(text) {
+    const match = /```json\s*\n([\s\S]*?)\n```/.exec(text);
+    if (!match) {
+        return null;
+    }
+    return match[1];
+}
+/** Mirror Python repr() for strings used in diagnostics. */
+export function pyRepr(value) {
+    if (value.includes("'") && !value.includes('"')) {
+        return `"${value}"`;
+    }
+    return "'" + value.replace(/\\/g, "\\\\").replace(/'/g, "\\'") + "'";
+}
+//# sourceMappingURL=semanticFixtures.js.map