@roackb2/heddle 0.0.37 → 1.0.0

package/dist/src/core/eval/git-artifacts.d.ts

@@ -0,0 +1,26 @@
+import type { EvalChangedFile } from './schema.js';
+export type GitArtifacts = {
+    gitStatusPath: string;
+    gitDiffPath: string;
+    gitDiffStatPath: string;
+    changedFilesPath: string;
+    sessionCatalogPath?: string;
+    traceFiles: string[];
+    changedFiles: EvalChangedFile[];
+};
+export type UntrackedEvalFile = {
+    path: string;
+    additions?: number;
+};
+export declare function collectEvalArtifacts(args: {
+    workspaceRoot: string;
+    outputDir: string;
+    stateDir?: string;
+}): Promise<GitArtifacts>;
+export declare function parseChangedFiles(args: {
+    nameStatus: string;
+    numStat: string;
+    untrackedFiles?: UntrackedEvalFile[];
+}): EvalChangedFile[];
+export declare function writeTextArtifact(path: string, value: string): void;
+//# sourceMappingURL=git-artifacts.d.ts.map

package/dist/src/core/eval/git-artifacts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"git-artifacts.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/git-artifacts.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,MAAM,MAAM,YAAY,GAAG;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,EAAE,eAAe,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,YAAY,CAAC,CAwExB;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,iBAAiB,EAAE,CAAC;CACtC,GAAG,eAAe,EAAE,CAuCpB;AAiFD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAG5D"}

package/dist/src/core/eval/git-artifacts.js

@@ -0,0 +1,211 @@
+import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { basename, dirname, join } from 'node:path';
+import { runCommand } from './process.js';
+export async function collectEvalArtifacts(args) {
+    mkdirSync(args.outputDir, { recursive: true });
+    const status = await runCommand({
+        command: 'git',
+        args: ['status', '--porcelain'],
+        cwd: args.workspaceRoot,
+        timeoutMs: 10_000,
+    });
+    const diff = await runCommand({
+        command: 'git',
+        args: ['diff', '--binary'],
+        cwd: args.workspaceRoot,
+        timeoutMs: 10_000,
+    });
+    const diffStat = await runCommand({
+        command: 'git',
+        args: ['diff', '--stat', '--compact-summary'],
+        cwd: args.workspaceRoot,
+        timeoutMs: 10_000,
+    });
+    const numStat = await runCommand({
+        command: 'git',
+        args: ['diff', '--numstat'],
+        cwd: args.workspaceRoot,
+        timeoutMs: 10_000,
+    });
+    const nameStatus = await runCommand({
+        command: 'git',
+        args: ['diff', '--name-status'],
+        cwd: args.workspaceRoot,
+        timeoutMs: 10_000,
+    });
+    const untrackedFiles = await collectUntrackedFiles(args.workspaceRoot);
+    const untrackedDiffs = await collectUntrackedDiffs({
+        workspaceRoot: args.workspaceRoot,
+        files: untrackedFiles,
+    });
+    const untrackedStats = await collectUntrackedStats({
+        workspaceRoot: args.workspaceRoot,
+        files: untrackedFiles,
+    });
+    const gitStatusPath = join(args.outputDir, 'git-status.txt');
+    const gitDiffPath = join(args.outputDir, 'diff.patch');
+    const gitDiffStatPath = join(args.outputDir, 'diff-stat.txt');
+    const changedFilesPath = join(args.outputDir, 'changed-files.json');
+    const changedFiles = parseChangedFiles({
+        nameStatus: nameStatus.stdout,
+        numStat: numStat.stdout,
+        untrackedFiles,
+    });
+    writeFileSync(gitStatusPath, status.stdout, 'utf8');
+    writeFileSync(gitDiffPath, joinArtifactText([diff.stdout, ...untrackedDiffs]), 'utf8');
+    writeFileSync(gitDiffStatPath, joinArtifactText([diffStat.stdout, ...untrackedStats]), 'utf8');
+    writeFileSync(changedFilesPath, `${JSON.stringify(changedFiles, null, 2)}\n`, 'utf8');
+    const stateRoot = join(args.workspaceRoot, args.stateDir ?? '.heddle');
+    const sessionCatalog = join(stateRoot, 'chat-sessions.catalog.json');
+    const sessionCatalogPath = copyOptionalArtifact(sessionCatalog, join(args.outputDir, 'session-catalog.json'));
+    const traceFiles = copyTraceFiles({
+        sourceTraceDir: join(stateRoot, 'traces'),
+        outputTraceDir: join(args.outputDir, 'traces'),
+    });
+    return {
+        gitStatusPath,
+        gitDiffPath,
+        gitDiffStatPath,
+        changedFilesPath,
+        sessionCatalogPath,
+        traceFiles,
+        changedFiles,
+    };
+}
+export function parseChangedFiles(args) {
+    const statsByPath = new Map();
+    for (const line of args.numStat.split('\n')) {
+        if (!line.trim()) {
+            continue;
+        }
+        const [additions, deletions, ...pathParts] = line.split('\t');
+        const path = pathParts.at(-1);
+        if (!path) {
+            continue;
+        }
+        statsByPath.set(path, {
+            additions: parseStatCount(additions),
+            deletions: parseStatCount(deletions),
+        });
+    }
+    const trackedFiles = args.nameStatus.split('\n')
+        .filter((line) => line.trim().length > 0)
+        .map((line) => {
+        const [status, ...pathParts] = line.split('\t');
+        const path = pathParts.at(-1) ?? '';
+        return {
+            path,
+            status,
+            ...statsByPath.get(path),
+        };
+    })
+        .filter((file) => file.path.length > 0);
+    const trackedPaths = new Set(trackedFiles.map((file) => file.path));
+    const untrackedFiles = (args.untrackedFiles ?? [])
+        .filter((file) => file.path.length > 0 && !trackedPaths.has(file.path))
+        .map((file) => ({
+        path: file.path,
+        status: '??',
+        additions: file.additions,
+        deletions: 0,
+    }));
+    return [...trackedFiles, ...untrackedFiles];
+}
+function parseStatCount(value) {
+    if (!value || value === '-') {
+        return undefined;
+    }
+    const parsed = Number.parseInt(value, 10);
+    return Number.isFinite(parsed) ? parsed : undefined;
+}
+async function collectUntrackedFiles(workspaceRoot) {
+    const result = await runCommand({
+        command: 'git',
+        args: ['ls-files', '--others', '--exclude-standard'],
+        cwd: workspaceRoot,
+        timeoutMs: 10_000,
+    });
+    return result.stdout.split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .sort()
+        .map((path) => ({
+        path,
+        additions: countTextLines(join(workspaceRoot, path)),
+    }));
+}
+async function collectUntrackedDiffs(args) {
+    const patches = [];
+    for (const file of args.files) {
+        const result = await runCommand({
+            command: 'git',
+            args: ['diff', '--binary', '--no-index', '--', '/dev/null', file.path],
+            cwd: args.workspaceRoot,
+            timeoutMs: 10_000,
+        });
+        patches.push(result.stdout);
+    }
+    return patches;
+}
+async function collectUntrackedStats(args) {
+    const stats = [];
+    for (const file of args.files) {
+        const result = await runCommand({
+            command: 'git',
+            args: ['diff', '--stat', '--compact-summary', '--no-index', '--', '/dev/null', file.path],
+            cwd: args.workspaceRoot,
+            timeoutMs: 10_000,
+        });
+        stats.push(result.stdout);
+    }
+    return stats;
+}
+function joinArtifactText(parts) {
+    return parts
+        .map((part) => part.trimEnd())
+        .filter((part) => part.length > 0)
+        .join('\n\n');
+}
+function countTextLines(path) {
+    try {
+        const content = readFileSync(path, 'utf8');
+        if (content.length === 0) {
+            return 0;
+        }
+        const lines = content.split(/\r?\n/);
+        return content.endsWith('\n') ? lines.length - 1 : lines.length;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function writeTextArtifact(path, value) {
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, value, 'utf8');
+}
+function listTraceFiles(traceDir) {
+    if (!existsSync(traceDir)) {
+        return [];
+    }
+    return readdirSync(traceDir)
+        .filter((file) => file.endsWith('.json'))
+        .map((file) => join(traceDir, file))
+        .sort();
+}
+function copyOptionalArtifact(sourcePath, targetPath) {
+    if (!existsSync(sourcePath)) {
+        return undefined;
+    }
+    mkdirSync(dirname(targetPath), { recursive: true });
+    copyFileSync(sourcePath, targetPath);
+    return targetPath;
+}
+function copyTraceFiles(args) {
+    return listTraceFiles(args.sourceTraceDir).map((sourcePath) => {
+        const targetPath = join(args.outputTraceDir, basename(sourcePath));
+        mkdirSync(dirname(targetPath), { recursive: true });
+        copyFileSync(sourcePath, targetPath);
+        return targetPath;
+    });
+}
+//# sourceMappingURL=git-artifacts.js.map

package/dist/src/core/eval/git-artifacts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"git-artifacts.js","sourceRoot":"","sources":["../../../../src/core/eval/git-artifacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxG,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAkB1C,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAI1C;IACC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;QAC9B,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC;QAC/B,GAAG,EAAE,IAAI,CAAC,aAAa;QACvB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC;QAC5B,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;QAC1B,GAAG,EAAE,IAAI,CAAC,aAAa;QACvB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC;QAChC,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,mBAAmB,CAAC;QAC7C,GAAG,EAAE,IAAI,CAAC,aAAa;QACvB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;QAC/B,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;QAC3B,GAAG,EAAE,IAAI,CAAC,aAAa;QACvB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC;QAClC,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,MAAM,EAAE,eAAe,CAAC;QAC/B,GAAG,EAAE,IAAI,CAAC,aAAa;QACvB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC;QACjD,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK,EAAE,cAAc;KACtB,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC;QACjD,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK,EAAE,cAAc;KACtB,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACvD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;IACpE,MAAM,YAAY,GAAG,iBAAiB,CAAC;QACrC,UAAU,EAAE,UAAU,CAAC,MAAM;QAC7B,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,cAAc;KACf,CAAC,CAAC;IACH,aAAa,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpD,aAAa,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACvF,aAAa,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC/F,aAAa,CAAC,gBAAgB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEtF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;IACrE,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC,CAAC;IAC9G,MAAM,UAAU,GAAG,cAAc,CAAC;QAChC,cAAc,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;QACzC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;KAC/C,CAAC,CAAC;IAEH,OAAO;QACL,aAAa;QACb,WAAW;QACX,eAAe;QACf,gBAAgB;QAChB,kBAAkB;QAClB,UAAU;QACV,YAAY;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA4D,CAAC;IACxF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACpB,SAAS,EAAE,cAAc,CAAC,SAAS,CAAC;YACpC,SAAS,EAAE,cAAc,CAAC,SAAS,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC;SAC7C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,OAAO;YACL,IAAI;YACJ,MAAM;YACN,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC;SACzB,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;SAC/C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACtE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACd,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI;QACZ,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,SAAS,EAAE,CAAC;KACb,CAAC,CAAC,CAAC;IACN,OAAO,CAAC,GAAG,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;QAC9B,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,oBAAoB,CAAC;QACpD,GAAG,EAAE,aAAa;QAClB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;SAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACd,IAAI;QACJ,SAAS,EAAE,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;KACrD,CAAC,CAAC,CAAC;AACR,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,IAGpC;IACC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC;YACtE,GAAG,EAAE,IAAI,CAAC,aAAa;YACvB,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,IAGpC;IACC,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC;YACzF,GAAG,EAAE,IAAI,CAAC,aAAa;YACvB,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAe;IACvC,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;SAC7B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACrC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,KAAa;IAC3D,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACtC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,WAAW,CAAC,QAAQ,CAAC;SACzB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SACnC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,oBAAoB,CAAC,UAAkB,EAAE,UAAkB;IAClE,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrC,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,cAAc,CAAC,IAAwD;IAC9E,OAAO,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;QACnE,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACrC,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/core/eval/process.d.ts

@@ -0,0 +1,22 @@
+export type CommandRunResult = {
+    command: string[];
+    exitCode: number | null;
+    stdout: string;
+    stderr: string;
+    durationMs: number;
+    timedOut: boolean;
+};
+export declare function runCommand(args: {
+    command: string;
+    args?: string[];
+    cwd: string;
+    env?: NodeJS.ProcessEnv;
+    timeoutMs?: number;
+}): Promise<CommandRunResult>;
+export declare function runShellCommand(args: {
+    command: string;
+    cwd: string;
+    env?: NodeJS.ProcessEnv;
+    timeoutMs?: number;
+}): Promise<CommandRunResult>;
+//# sourceMappingURL=process.d.ts.map

package/dist/src/core/eval/process.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"process.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/process.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,wBAAsB,UAAU,CAAC,IAAI,EAAE;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA0D5B;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAQ5B"}

package/dist/src/core/eval/process.js

@@ -0,0 +1,65 @@
+import { spawn } from 'node:child_process';
+export async function runCommand(args) {
+    const startedAt = Date.now();
+    const argv = [args.command, ...(args.args ?? [])];
+    return await new Promise((resolve) => {
+        let stdout = '';
+        let stderr = '';
+        let settled = false;
+        let timedOut = false;
+        const child = spawn(args.command, args.args ?? [], {
+            cwd: args.cwd,
+            env: args.env,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        const timeout = args.timeoutMs ?
+            setTimeout(() => {
+                timedOut = true;
+                child.kill('SIGTERM');
+                setTimeout(() => {
+                    if (!child.killed) {
+                        child.kill('SIGKILL');
+                    }
+                }, 2_000).unref();
+            }, args.timeoutMs)
+            : undefined;
+        child.stdout?.on('data', (chunk) => {
+            stdout += Buffer.from(chunk).toString('utf8');
+        });
+        child.stderr?.on('data', (chunk) => {
+            stderr += Buffer.from(chunk).toString('utf8');
+        });
+        const finish = (exitCode) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            if (timeout) {
+                clearTimeout(timeout);
+            }
+            resolve({
+                command: argv,
+                exitCode,
+                stdout,
+                stderr,
+                durationMs: Date.now() - startedAt,
+                timedOut,
+            });
+        };
+        child.on('error', (error) => {
+            stderr += `${error instanceof Error ? error.message : String(error)}\n`;
+            finish(null);
+        });
+        child.on('exit', (code) => finish(code));
+    });
+}
+export async function runShellCommand(args) {
+    return await runCommand({
+        command: '/bin/sh',
+        cwd: args.cwd,
+        env: args.env,
+        timeoutMs: args.timeoutMs,
+        args: ['-lc', args.command],
+    });
+}
+//# sourceMappingURL=process.js.map

package/dist/src/core/eval/process.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"process.js","sourceRoot":"","sources":["../../../../src/core/eval/process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAW3C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAMhC;IACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,OAAO,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACnC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE;YACjD,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QAEH,MAAM,OAAO,GACX,IAAI,CAAC,SAAS,CAAC,CAAC;YACd,UAAU,CAAC,GAAG,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,UAAU,CAAC,GAAG,EAAE;oBACd,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;wBAClB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;gBACH,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;YACpB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC;YACpB,CAAC,CAAC,SAAS,CAAC;QAEd,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,CAAC,QAAuB,EAAE,EAAE;YACzC,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;YACT,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,OAAO,EAAE,CAAC;gBACZ,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,CAAC;gBACN,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,QAAQ;aACT,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAKrC;IACC,OAAO,MAAM,UAAU,CAAC;QACtB,OAAO,EAAE,SAAS;QAClB,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC;KAC5B,CAAC,CAAC;AACL,CAAC"}

package/dist/src/core/eval/progress.d.ts

@@ -0,0 +1,28 @@
+export type EvalProgressEvent = {
+    timestamp: string;
+    caseId: string;
+    phase: string;
+    status: 'started' | 'heartbeat' | 'completed' | 'failed' | 'info';
+    message: string;
+    elapsedMs?: number;
+};
+export declare class EvalProgressReporter {
+    readonly progressPath: string;
+    private readonly caseId;
+    private readonly writeStdout;
+    constructor(args: {
+        caseId: string;
+        progressPath: string;
+        writeStdout?: boolean;
+    });
+    info(phase: string, message: string): void;
+    track<T>(args: {
+        phase: string;
+        message: string;
+        heartbeatMessage?: string;
+        heartbeatMs?: number;
+        run: () => Promise<T>;
+    }): Promise<T>;
+    private write;
+}
+//# sourceMappingURL=progress.d.ts.map

package/dist/src/core/eval/progress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/progress.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,qBAAa,oBAAoB;IAC/B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;gBAE1B,IAAI,EAAE;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAOD,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAU7B,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC;KACvB,GAAG,OAAO,CAAC,CAAC,CAAC;IAwDd,OAAO,CAAC,KAAK;CAMd"}

package/dist/src/core/eval/progress.js

@@ -0,0 +1,94 @@
+import { appendFileSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+export class EvalProgressReporter {
+    progressPath;
+    caseId;
+    writeStdout;
+    constructor(args) {
+        this.caseId = args.caseId;
+        this.progressPath = args.progressPath;
+        this.writeStdout = args.writeStdout ?? true;
+        mkdirSync(dirname(this.progressPath), { recursive: true });
+    }
+    info(phase, message) {
+        this.write({
+            timestamp: new Date().toISOString(),
+            caseId: this.caseId,
+            phase,
+            status: 'info',
+            message,
+        });
+    }
+    async track(args) {
+        const startedAt = Date.now();
+        this.write({
+            timestamp: new Date(startedAt).toISOString(),
+            caseId: this.caseId,
+            phase: args.phase,
+            status: 'started',
+            message: args.message,
+            elapsedMs: 0,
+        });
+        const heartbeatMs = args.heartbeatMs ?? 30_000;
+        const interval = heartbeatMs > 0 ?
+            setInterval(() => {
+                const elapsedMs = Date.now() - startedAt;
+                this.write({
+                    timestamp: new Date().toISOString(),
+                    caseId: this.caseId,
+                    phase: args.phase,
+                    status: 'heartbeat',
+                    message: args.heartbeatMessage ?? `still running ${args.message}`,
+                    elapsedMs,
+                });
+            }, heartbeatMs)
+            : undefined;
+        try {
+            const result = await args.run();
+            const elapsedMs = Date.now() - startedAt;
+            this.write({
+                timestamp: new Date().toISOString(),
+                caseId: this.caseId,
+                phase: args.phase,
+                status: 'completed',
+                message: args.message,
+                elapsedMs,
+            });
+            return result;
+        }
+        catch (error) {
+            const elapsedMs = Date.now() - startedAt;
+            this.write({
+                timestamp: new Date().toISOString(),
+                caseId: this.caseId,
+                phase: args.phase,
+                status: 'failed',
+                message: error instanceof Error ? error.message : String(error),
+                elapsedMs,
+            });
+            throw error;
+        }
+        finally {
+            if (interval) {
+                clearInterval(interval);
+            }
+        }
+    }
+    write(event) {
+        appendFileSync(this.progressPath, `${JSON.stringify(event)}\n`, 'utf8');
+        if (this.writeStdout) {
+            process.stdout.write(formatProgressLine(event));
+        }
+    }
+}
+function formatProgressLine(event) {
+    const elapsed = event.elapsedMs === undefined ? '' : ` (${formatElapsed(event.elapsedMs)})`;
+    return `[${event.caseId}] ${event.status}: ${event.message}${elapsed}\n`;
+}
+function formatElapsed(elapsedMs) {
+    if (elapsedMs < 1000) {
+        return `${elapsedMs}ms`;
+    }
+    return `${Math.round(elapsedMs / 1000)}s`;
+}
+//# sourceMappingURL=progress.js.map

package/dist/src/core/eval/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../../src/core/eval/progress.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,MAAM,OAAO,oBAAoB;IACtB,YAAY,CAAS;IACb,MAAM,CAAS;IACf,WAAW,CAAU;IAEtC,YAAY,IAIX;QACC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC;QAC5C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,KAAa,EAAE,OAAe;QACjC,IAAI,CAAC,KAAK,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK;YACL,MAAM,EAAE,MAAM;YACd,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAI,IAMd;QACC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;YAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,CAAC;SACb,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC;YAChC,WAAW,CAAC,GAAG,EAAE;gBACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACzC,IAAI,CAAC,KAAK,CAAC;oBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,WAAW;oBACnB,OAAO,EAAE,IAAI,CAAC,gBAAgB,IAAI,iBAAiB,IAAI,CAAC,OAAO,EAAE;oBACjE,SAAS;iBACV,CAAC,CAAC;YACL,CAAC,EAAE,WAAW,CAAC;YACjB,CAAC,CAAC,SAAS,CAAC;QAEZ,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC;gBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS;aACV,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC;gBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,SAAS;aACV,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAwB;QACpC,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC;IAC5F,OAAO,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,GAAG,OAAO,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB;IACtC,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;QACrB,OAAO,GAAG,SAAS,IAAI,CAAC;IAC1B,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC;AAC5C,CAAC"}

package/dist/src/core/eval/report-writer.d.ts

@@ -0,0 +1,7 @@
+import type { EvalSuiteReport } from './schema.js';
+export declare function writeEvalSuiteReport(report: EvalSuiteReport): {
+    jsonPath: string;
+    markdownPath: string;
+};
+export declare function formatEvalSuiteMarkdown(report: EvalSuiteReport): string;
+//# sourceMappingURL=report-writer.d.ts.map

package/dist/src/core/eval/report-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-writer.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/report-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAiB,eAAe,EAAE,MAAM,aAAa,CAAC;AAElE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAOxG;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAqBvE"}

package/dist/src/core/eval/report-writer.js

@@ -0,0 +1,159 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { relative, join } from 'node:path';
+export function writeEvalSuiteReport(report) {
+    mkdirSync(report.resultsDir, { recursive: true });
+    const jsonPath = join(report.resultsDir, 'report.json');
+    const markdownPath = join(report.resultsDir, 'report.md');
+    writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+    writeFileSync(markdownPath, formatEvalSuiteMarkdown(report), 'utf8');
+    return { jsonPath, markdownPath };
+}
+export function formatEvalSuiteMarkdown(report) {
+    const passed = report.results.filter((result) => result.status === 'passed').length;
+    const lines = [
+        '# Heddle Agent Eval Report',
+        '',
+        `Target: ${report.target}`,
+        `Started: ${report.startedAt}`,
+        `Finished: ${report.finishedAt}`,
+        `Results: ${passed}/${report.results.length} passed`,
+        '',
+        '| Case | Status | Model | Checks | Outcome | Turns | Mutations | Verification After Mutation |',
+        '| --- | --- | --- | ---: | --- | ---: | ---: | ---: |',
+        ...report.results.map(formatSummaryRow),
+        '',
+    ];
+    for (const result of report.results) {
+        lines.push(...formatRunDetail(result, report.resultsDir), '');
+    }
+    return `${lines.join('\n').trimEnd()}\n`;
+}
+function formatSummaryRow(result) {
+    const passedChecks = result.checks.filter((check) => check.passed).length;
+    return [
+        result.caseId,
+        result.status,
+        result.model ?? 'default',
+        `${passedChecks}/${result.checks.length}`,
+        result.metrics.outcome ?? `exit ${result.agent.exitCode ?? 'unknown'}`,
+        String(result.metrics.assistantTurns),
+        String(result.metrics.mutations),
+        String(result.metrics.verificationCommandsAfterMutation),
+    ].map(escapeCell).join(' | ').replace(/^/, '| ').replace(/$/, ' |');
+}
+function formatRunDetail(result, resultsDir) {
+    const traceFiles = result.artifacts.traceFiles.map((path) => formatPath(path, resultsDir));
+    const lines = [
+        `## ${result.caseId}`,
+        '',
+        '| Field | Value |',
+        '| --- | --- |',
+        `| Status | ${escapeCell(result.status)} |`,
+        `| Model | ${escapeCell(result.model ?? 'default')} |`,
+        `| Max steps | ${escapeCell(String(result.maxSteps ?? 'default'))} |`,
+        `| Agent exit | ${escapeCell(`${result.agent.exitCode ?? 'unknown'}${result.agent.timedOut ? ' (timed out)' : ''}`)} |`,
+        `| Fixture | ${escapeCell(formatFixture(result))} |`,
+        `| Workspace | \`${escapeCell(formatPath(result.workspaceRoot, resultsDir))}\` |`,
+        `| Output | \`${escapeCell(formatPath(result.outputDir, resultsDir))}\` |`,
+        `| Diff | \`${escapeCell(formatPath(result.artifacts.gitDiffPath, resultsDir))}\` |`,
+        `| Diff stat | \`${escapeCell(formatPath(result.artifacts.gitDiffStatPath, resultsDir))}\` |`,
+        `| Changed files JSON | \`${escapeCell(formatPath(result.artifacts.changedFilesPath, resultsDir))}\` |`,
+        `| Git status | \`${escapeCell(formatPath(result.artifacts.gitStatusPath, resultsDir))}\` |`,
+        `| Progress | ${result.artifacts.progressPath ? `\`${escapeCell(formatPath(result.artifacts.progressPath, resultsDir))}\`` : 'none'} |`,
+        `| Session catalog | ${result.artifacts.sessionCatalogPath ? `\`${escapeCell(formatPath(result.artifacts.sessionCatalogPath, resultsDir))}\`` : 'none'} |`,
+        `| Trace files | ${traceFiles.length ? traceFiles.map((path) => `\`${escapeCell(path)}\``).join('<br>') : 'none'} |`,
+        '',
+        '### Milestone Review',
+        '',
+    ];
+    lines.push(...formatReviewSection(result), '', '### Changed Files', '');
+    if (result.artifacts.changedFiles.length === 0) {
+        lines.push('- none');
+    }
+    else {
+        lines.push('| File | Status | + | - |', '| --- | --- | ---: | ---: |');
+        for (const file of result.artifacts.changedFiles) {
+            lines.push(`| ${escapeCell(file.path)} | ${escapeCell(file.status)} | ${file.additions ?? ''} | ${file.deletions ?? ''} |`);
+        }
+    }
+    lines.push('', '### Post-Run Checks', '');
+    if (result.checks.length === 0) {
+        lines.push('- none');
+    }
+    else {
+        for (const check of result.checks) {
+            lines.push(`- ${check.passed ? 'PASS' : 'FAIL'} ${check.name}: \`${check.command}\` (${check.exitCode ?? 'unknown'}, ${check.durationMs}ms)`);
+        }
+    }
+    lines.push('', '### Metrics', '', '| Metric | Value |', '| --- | ---: |', `| Assistant turns | ${result.metrics.assistantTurns} |`, `| Tool calls | ${result.metrics.toolCalls} |`, `| Mutations | ${result.metrics.mutations} |`, `| Verification after mutation | ${result.metrics.verificationCommandsAfterMutation} |`, `| Approvals requested | ${result.metrics.approvalsRequested} |`, `| Approvals resolved | ${result.metrics.approvalsResolved} |`, `| Tool errors | ${result.metrics.toolErrors} |`);
+    lines.push('', '### Agent Verification Commands', '');
+    if (result.metrics.verificationCommandDetails.length === 0) {
+        lines.push('- none detected after first mutation');
+    }
+    else {
+        for (const command of result.metrics.verificationCommandDetails) {
+            lines.push(`- \`${escapeCell(command)}\``);
+        }
+    }
+    lines.push('', '### Rubric', '');
+    if (result.review.requiredOutcomes.length === 0 && result.review.humanQuestions.length === 0) {
+        lines.push('- none');
+    }
+    else {
+        for (const outcome of result.review.requiredOutcomes) {
+            lines.push(`- [ ] ${outcome}`);
+        }
+        for (const question of result.review.humanQuestions) {
+            lines.push(`- [ ] ${question}`);
+        }
+    }
+    if (result.metrics.summary) {
+        lines.push('', '### Final Summary', '', result.metrics.summary);
+    }
+    return lines;
+}
+function formatReviewSection(result) {
+    const lines = [];
+    if (result.review.milestone) {
+        lines.push(`Milestone: ${result.review.milestone}`);
+    }
+    if (result.review.intent) {
+        lines.push('', result.review.intent);
+    }
+    lines.push('', '| Review Field | Items |', '| --- | --- |');
+    lines.push(`| Required outcomes | ${formatListCell(result.review.requiredOutcomes)} |`);
+    lines.push(`| Allowed scope | ${formatListCell(result.review.allowedScope)} |`);
+    lines.push(`| Out of scope | ${formatListCell(result.review.outOfScope)} |`);
+    lines.push(`| Human questions | ${formatListCell(result.review.humanQuestions)} |`);
+    return lines;
+}
+function formatListCell(items) {
+    return items.length ? items.map((item) => `- ${escapeCell(item)}`).join('<br>') : 'none';
+}
+function formatFixture(result) {
+    if (result.fixture.type === 'git-worktree') {
+        return [
+            'git-worktree',
+            result.fixture.ref ? `ref ${result.fixture.ref}` : undefined,
+            result.fixture.resolvedRef ? `commit ${shortSha(result.fixture.resolvedRef)}` : undefined,
+            result.fixture.baselineCommit && result.fixture.baselineCommit !== result.fixture.resolvedRef ?
+                `baseline ${shortSha(result.fixture.baselineCommit)}`
+                : undefined,
+        ].filter(Boolean).join(', ');
+    }
+    return result.fixture.baselineCommit ? `inline, baseline ${shortSha(result.fixture.baselineCommit)}` : 'inline';
+}
+function shortSha(value) {
+    return value.slice(0, 12);
+}
+function escapeCell(value) {
+    return value.replaceAll('|', '\\|').replaceAll('\n', ' ');
+}
+function formatPath(path, basePath) {
+    if (path === basePath) {
+        return '.';
+    }
+    const relativePath = relative(basePath, path);
+    return relativePath && !relativePath.startsWith('..') ? relativePath : path;
+}
+//# sourceMappingURL=report-writer.js.map