autodocs-engine 0.5.0

package/dist/benchmark/pr-scorer.js

@@ -0,0 +1,222 @@
+// src/benchmark/pr-scorer.ts — Score AI output against real commit ground truth
+// Phase 0: File placement accuracy as the primary metric.
+// Designed to be extended with naming, imports, exports in later phases.
+import { dirname, basename, extname } from "node:path";
+// ─── Public API ─────────────────────────────────────────────────────────────
+/**
+ * Score AI-generated files against ground truth from a real commit.
+ * Phase 0: primary metric is file placement accuracy.
+ */
+export function scorePROutput(files, task, tokensUsed, latencyMs, error) {
+    if (error || files.length === 0) {
+        return {
+            score: 0,
+            dimensions: {
+                filePlacement: { score: 0, detail: error ?? "No files generated", passed: false },
+                namingConvention: { score: 0, detail: "No files generated", passed: false },
+                barrelUpdate: { score: 0, detail: "No files generated", passed: false },
+            },
+            filesCreated: [],
+            tokensUsed,
+            latencyMs,
+            error,
+        };
+    }
+    // Filter to implementation files (not test, not barrel, not response wrapper)
+    const implFiles = files.filter(f => !f.path.includes(".test.") &&
+        !f.path.includes(".spec.") &&
+        !f.path.includes("__response__") &&
+        !isBarrelFile(f.path));
+    const filePlacement = scoreFilePlacement(implFiles, task);
+    const namingConvention = scoreNamingConvention(implFiles, task);
+    const barrelUpdate = scoreBarrelUpdate(files, task);
+    // Phase 0 headline: file placement is THE metric
+    // Others tracked as secondary signals
+    const score = filePlacement.score;
+    return {
+        score,
+        dimensions: { filePlacement, namingConvention, barrelUpdate },
+        filesCreated: files.map(f => f.path),
+        tokensUsed,
+        latencyMs,
+    };
+}
+// ─── File Placement Scoring ─────────────────────────────────────────────────
+/**
+ * Score whether the AI placed its files in the correct directory.
+ * Uses path distance: exact match = 100%, parent = 60%, shared prefix scaled.
+ */
+export function scoreFilePlacement(implFiles, task) {
+    if (implFiles.length === 0) {
+        return { score: 0, detail: "No implementation files generated", passed: false };
+    }
+    const expectedDir = normalizeDir(task.groundTruth.directory);
+    // Find the best-scoring file among the AI's output
+    let bestScore = 0;
+    let bestPath = "";
+    for (const file of implFiles) {
+        const fileDir = normalizeDir(dirname(file.path));
+        const s = pathSimilarity(fileDir, expectedDir);
+        if (s > bestScore) {
+            bestScore = s;
+            bestPath = file.path;
+        }
+    }
+    const score = Math.round(bestScore * 100);
+    const passed = bestScore >= 0.9; // Exact or near-exact match
+    let detail;
+    if (bestScore >= 1.0) {
+        detail = `Correct: ${dirname(bestPath)} matches ${task.groundTruth.directory}`;
+    }
+    else if (bestScore >= 0.5) {
+        detail = `Close: ${dirname(bestPath)} near ${task.groundTruth.directory} (${score}%)`;
+    }
+    else if (bestScore > 0) {
+        detail = `Wrong: ${dirname(bestPath)}, expected ${task.groundTruth.directory} (${score}%)`;
+    }
+    else {
+        detail = `Wrong: ${bestPath ? dirname(bestPath) : "unknown"}, expected ${task.groundTruth.directory}`;
+    }
+    return { score, detail, passed };
+}
+/**
+ * Path similarity using common prefix and distance.
+ * Exact dir match = 1.0, parent = 0.6, grandparent = 0.4, etc.
+ */
+export function pathSimilarity(a, b) {
+    if (a === b)
+        return 1.0;
+    const aParts = a.split("/").filter(Boolean);
+    const bParts = b.split("/").filter(Boolean);
+    // Common prefix length
+    let lcp = 0;
+    while (lcp < aParts.length && lcp < bParts.length && aParts[lcp] === bParts[lcp]) {
+        lcp++;
+    }
+    const distance = (aParts.length - lcp) + (bParts.length - lcp);
+    if (distance === 0)
+        return 1.0;
+    // Exponential decay with distance
+    const lambda = Math.log(2); // distance 1 ≈ 0.5
+    let score = Math.exp(-lambda * distance);
+    // Package mismatch penalty (monorepos)
+    const aPkg = getPackage(aParts);
+    const bPkg = getPackage(bParts);
+    if (aPkg && bPkg && aPkg !== bPkg) {
+        score *= 0.5;
+    }
+    return Math.max(0, Math.min(1, score));
+}
+function getPackage(parts) {
+    if ((parts[0] === "packages" || parts[0] === "apps") && parts.length >= 2) {
+        return parts.slice(0, 2).join("/");
+    }
+    return null;
+}
+// ─── Naming Convention Scoring ──────────────────────────────────────────────
+/**
+ * Score whether the AI's filename follows the same naming convention
+ * as the ground truth file.
+ */
+export function scoreNamingConvention(implFiles, task) {
+    if (implFiles.length === 0) {
+        return { score: 0, detail: "No files", passed: false };
+    }
+    const gtName = basename(task.groundTruth.filename, extname(task.groundTruth.filename));
+    const gtConvention = detectConvention(gtName);
+    // Check if any AI file matches the naming convention
+    let bestMatch = false;
+    let bestFile = "";
+    for (const file of implFiles) {
+        const aiName = basename(file.path, extname(file.path));
+        const aiConvention = detectConvention(aiName);
+        if (aiConvention === gtConvention) {
+            bestMatch = true;
+            bestFile = file.path;
+            break;
+        }
+    }
+    if (bestMatch) {
+        return {
+            score: 100,
+            detail: `Convention match: ${gtConvention} (${bestFile})`,
+            passed: true,
+        };
+    }
+    // Check for partial match (e.g., both use lowercase but different separators)
+    const aiFile = implFiles[0];
+    const aiName = basename(aiFile.path, extname(aiFile.path));
+    const aiConvention = detectConvention(aiName);
+    return {
+        score: 0,
+        detail: `Convention mismatch: AI used ${aiConvention}, expected ${gtConvention}`,
+        passed: false,
+    };
+}
+function detectConvention(name) {
+    if (/^[a-z][a-z0-9]*(-[a-z0-9]+)+$/.test(name))
+        return "kebab-case";
+    if (/^[a-z][a-z0-9]*$/.test(name))
+        return "kebab-case"; // single word, treat as kebab
+    if (/^[a-z][a-zA-Z0-9]*$/.test(name) && /[A-Z]/.test(name))
+        return "camelCase";
+    if (/^[A-Z][a-zA-Z0-9]*$/.test(name))
+        return "PascalCase";
+    if (/^[a-z][a-z0-9]*(_[a-z0-9]+)+$/.test(name))
+        return "snake_case";
+    return "unknown";
+}
+// ─── Barrel Update Scoring ──────────────────────────────────────────────────
+/**
+ * Score whether the AI updated the barrel/index file when one exists.
+ * Binary: did the AI produce a modified barrel that includes a new export?
+ */
+export function scoreBarrelUpdate(allFiles, task) {
+    // Only score if the directory had a barrel file
+    if (!task.context.barrelFile) {
+        return { score: 100, detail: "No barrel file — N/A", passed: true };
+    }
+    // Find the AI's barrel file
+    const barrelPath = task.context.barrelFile.path;
+    const barrelName = basename(barrelPath);
+    const aiBarrel = allFiles.find(f => f.path === barrelPath ||
+        f.path.endsWith("/" + barrelName) ||
+        basename(f.path) === barrelName);
+    if (!aiBarrel) {
+        return { score: 0, detail: "Barrel file exists but AI did not update it", passed: false };
+    }
+    // Check if the AI added a new export
+    const originalExports = extractReExports(task.context.barrelFile.content);
+    const aiExports = extractReExports(aiBarrel.content);
+    const newExports = aiExports.filter(e => !originalExports.includes(e));
+    if (newExports.length > 0) {
+        return {
+            score: 100,
+            detail: `Barrel updated: added ${newExports.join(", ")}`,
+            passed: true,
+        };
+    }
+    return {
+        score: 0,
+        detail: "Barrel file present but no new exports added",
+        passed: false,
+    };
+}
+function extractReExports(content) {
+    const exports = [];
+    const regex = /export\s+(?:\*|\{[^}]+\})\s+from\s+["']([^"']+)["']/g;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        exports.push(match[1]);
+    }
+    return exports;
+}
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function normalizeDir(dir) {
+    return dir.replace(/^\.\//, "").replace(/\/$/, "");
+}
+function isBarrelFile(path) {
+    const name = basename(path);
+    return name === "index.ts" || name === "index.tsx" || name === "mod.ts";
+}
+//# sourceMappingURL=pr-scorer.js.map

package/dist/benchmark/pr-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pr-scorer.js","sourceRoot":"","sources":["../../src/benchmark/pr-scorer.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,0DAA0D;AAC1D,yEAAyE;AAGzE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6BvD,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,KAAsB,EACtB,IAAe,EACf,UAAkB,EAClB,SAAiB,EACjB,KAAc;IAEd,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,CAAC;YACR,UAAU,EAAE;gBACV,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,IAAI,oBAAoB,EAAE,MAAM,EAAE,KAAK,EAAE;gBACjF,gBAAgB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC3E,YAAY,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,MAAM,EAAE,KAAK,EAAE;aACxE;YACD,YAAY,EAAE,EAAE;YAChB,UAAU;YACV,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CACtB,CAAC;IAEF,MAAM,aAAa,GAAG,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAEpD,iDAAiD;IACjD,sCAAsC;IACtC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC;IAElC,OAAO;QACL,KAAK;QACL,UAAU,EAAE,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE;QAC7D,YAAY,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACpC,UAAU;QACV,SAAS;KACV,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAA0B,EAC1B,IAAe;IAEf,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAClF,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IAE7D,mDAAmD;IACnD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC/C,IAAI,CAAC,GAAG,SAAS,EAAE,CAAC;YAClB,SAAS,GAAG,CAAC,CAAC;YACd,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,SAAS,IAAI,GAAG,CAAC,CAAC,4BAA4B;IAE7D,IAAI,MAAc,CAAC;IACnB,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,GAAG,YAAY,OAAO,CAAC,QAAQ,CAAC,YAAY,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;IACjF,CAAC;SAAM,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QAC5B,MAAM,GAAG,UAAU,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,WAAW,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC;IACxF,CAAC;SAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,GAAG,UAAU,OAAO,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,WAAW,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC;IAC7F,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,UAAU,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,cAAc,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;IACxG,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,CAAS,EAAE,CAAS;IACjD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAExB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,uBAAuB;IACvB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,OAAO,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACjF,GAAG,EAAE,CAAC;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IAC/D,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAE/B,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;IAC/C,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;IAEzC,uCAAuC;IACvC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClC,KAAK,IAAI,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IACjC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,SAA0B,EAC1B,IAAe;IAEf,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAE9C,qDAAqD;IACrD,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,YAAY,KAAK,YAAY,EAAE,CAAC;YAClC,SAAS,GAAG,IAAI,CAAC;YACjB,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;YACrB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,qBAAqB,YAAY,KAAK,QAAQ,GAAG;YACzD,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAE9C,OAAO;QACL,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,gCAAgC,YAAY,cAAc,YAAY,EAAE;QAChF,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC;AAID,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACpE,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC,CAAC,8BAA8B;IACtF,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC;IAC/E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IAC1D,IAAI,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IACpE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,QAAyB,EACzB,IAAe;IAEf,gDAAgD;IAChD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACtE,CAAC;IAED,4BAA4B;IAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;IAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACjC,CAAC,CAAC,IAAI,KAAK,UAAU;QACrB,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,UAAU,CAAC;QACjC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,UAAU,CAChC,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,6CAA6C,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC5F,CAAC;IAED,qCAAqC;IACrC,MAAM,eAAe,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,yBAAyB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACxD,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,8CAA8C;QACtD,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,sDAAsD,CAAC;IACrE,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+EAA+E;AAE/E,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,QAAQ,CAAC;AAC1E,CAAC"}

package/dist/benchmark/pr-task-gen.d.ts

@@ -0,0 +1,16 @@
+import type { MinedTask } from "./pr-miner.js";
+/**
+ * Generate a task prompt from a mined commit.
+ * Strips directory/location hints and implementation-specific details
+ * to keep the task fair across conditions.
+ */
+export declare function generateTaskPrompt(task: MinedTask, packageName: string): string;
+/**
+ * Clean a commit message for use as a task prompt.
+ * Strips:
+ * - Conventional commit prefixes (feat:, fix:, etc.)
+ * - Directory/path references (to src/utils, in packages/auth)
+ * - Issue/PR references (#123)
+ * - Overly specific implementation details
+ */
+export declare function cleanCommitMessage(message: string): string;

package/dist/benchmark/pr-task-gen.js

@@ -0,0 +1,129 @@
+// src/benchmark/pr-task-gen.ts — Generate task prompts from mined commits
+// Derives fair, deliberately vague prompts that test whether AGENTS.md
+// helps with file placement and convention adherence.
+import ts from "typescript";
+import { basename, extname } from "node:path";
+// ─── Public API ─────────────────────────────────────────────────────────────
+/**
+ * Generate a task prompt from a mined commit.
+ * Strips directory/location hints and implementation-specific details
+ * to keep the task fair across conditions.
+ */
+export function generateTaskPrompt(task, packageName) {
+    const message = task.commitMessage;
+    const file = task.groundTruth;
+    // Try to use the commit message if it's descriptive enough
+    const cleaned = cleanCommitMessage(message);
+    if (cleaned.length >= 30) {
+        return buildPrompt(cleaned, packageName);
+    }
+    // Fall back to deriving from the file's exports
+    const derived = deriveFromFile(file.content, file.filename);
+    if (derived) {
+        return buildPrompt(derived, packageName);
+    }
+    // Last resort: generic prompt from filename
+    const name = basename(file.filename, extname(file.filename));
+    const humanName = name.replace(/[-_]/g, " ");
+    return buildPrompt(`Add a new module for ${humanName} functionality`, packageName);
+}
+// ─── Prompt Construction ────────────────────────────────────────────────────
+function buildPrompt(description, packageName) {
+    // Capitalize first letter
+    const desc = description.charAt(0).toUpperCase() + description.slice(1);
+    return (`${desc.endsWith(".") ? desc : desc + "."}\n\n` +
+        `This is for the ${packageName} project. ` +
+        `Follow the project's conventions for file naming, imports, exports, and code style. ` +
+        `Include the implementation file and any necessary updates to barrel/index files.`);
+}
+// ─── Commit Message Cleaning ────────────────────────────────────────────────
+/**
+ * Clean a commit message for use as a task prompt.
+ * Strips:
+ * - Conventional commit prefixes (feat:, fix:, etc.)
+ * - Directory/path references (to src/utils, in packages/auth)
+ * - Issue/PR references (#123)
+ * - Overly specific implementation details
+ */
+export function cleanCommitMessage(message) {
+    let cleaned = message;
+    // Strip conventional commit prefixes
+    cleaned = cleaned.replace(/^(feat|fix|chore|docs|style|refactor|perf|test|build|ci|revert)(\(.+?\))?:\s*/i, "");
+    // Strip issue/PR references
+    cleaned = cleaned.replace(/\s*\(#\d+\)\s*$/, "");
+    cleaned = cleaned.replace(/#\d+/g, "");
+    // Strip directory/path hints (e.g., "to src/utils", "in packages/auth")
+    cleaned = cleaned.replace(/\s+(to|in|at|from|under)\s+[a-z]+\/[a-zA-Z0-9_\-\/]+/g, "");
+    // Strip filename references (e.g., "add auth-service.ts")
+    cleaned = cleaned.replace(/\s+[a-z][a-z0-9\-_]*\.(ts|tsx|js|jsx)\b/gi, "");
+    // Strip leading/trailing whitespace and common noise words
+    cleaned = cleaned.replace(/^\s*(add|create|implement|introduce)\s+/i, "Add ");
+    return cleaned.trim();
+}
+// ─── File Analysis ──────────────────────────────────────────────────────────
+/**
+ * Derive a task description from the file's exports and structure.
+ */
+function deriveFromFile(content, filename) {
+    const sourceFile = ts.createSourceFile(filename, content, ts.ScriptTarget.Latest, true);
+    const exports = extractExports(sourceFile);
+    if (exports.length === 0)
+        return null;
+    const primary = exports[0];
+    const kind = primary.kind;
+    const name = primary.name;
+    if (!name)
+        return null;
+    const humanName = camelToSpaces(name);
+    switch (kind) {
+        case "function":
+            return `Add a new utility function for ${humanName}`;
+        case "class":
+            return `Add a new ${humanName} class`;
+        case "interface":
+        case "type":
+            return `Add type definitions for ${humanName}`;
+        case "const":
+        case "variable":
+            return `Add a ${humanName} module`;
+        default:
+            return `Add a new module for ${humanName}`;
+    }
+}
+function extractExports(sourceFile) {
+    const exports = [];
+    ts.forEachChild(sourceFile, (node) => {
+        const mods = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined;
+        const isExported = mods?.some(m => m.kind === ts.SyntaxKind.ExportKeyword);
+        if (!isExported)
+            return;
+        if (ts.isFunctionDeclaration(node) && node.name) {
+            exports.push({ name: node.name.text, kind: "function" });
+        }
+        else if (ts.isClassDeclaration(node) && node.name) {
+            exports.push({ name: node.name.text, kind: "class" });
+        }
+        else if (ts.isInterfaceDeclaration(node)) {
+            exports.push({ name: node.name.text, kind: "interface" });
+        }
+        else if (ts.isTypeAliasDeclaration(node)) {
+            exports.push({ name: node.name.text, kind: "type" });
+        }
+        else if (ts.isVariableStatement(node)) {
+            for (const decl of node.declarationList.declarations) {
+                if (ts.isIdentifier(decl.name)) {
+                    exports.push({ name: decl.name.text, kind: "const" });
+                }
+            }
+        }
+    });
+    return exports;
+}
+function camelToSpaces(name) {
+    // CamelCase → "camel case", kebab-case → "kebab case"
+    return name
+        .replace(/([a-z])([A-Z])/g, "$1 $2")
+        .replace(/[-_]/g, " ")
+        .toLowerCase();
+}
+//# sourceMappingURL=pr-task-gen.js.map

package/dist/benchmark/pr-task-gen.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pr-task-gen.js","sourceRoot":"","sources":["../../src/benchmark/pr-task-gen.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AACvE,sDAAsD;AAEtD,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG9C,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAe,EACf,WAAmB;IAEnB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC;IAE9B,2DAA2D;IAC3D,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACzB,OAAO,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC3C,CAAC;IAED,gDAAgD;IAChD,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5D,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC3C,CAAC;IAED,4CAA4C;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC7C,OAAO,WAAW,CAChB,wBAAwB,SAAS,gBAAgB,EACjD,WAAW,CACZ,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,SAAS,WAAW,CAAC,WAAmB,EAAE,WAAmB;IAC3D,0BAA0B;IAC1B,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO,CACL,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,MAAM;QAC/C,mBAAmB,WAAW,YAAY;QAC1C,sFAAsF;QACtF,kFAAkF,CACnF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,IAAI,OAAO,GAAG,OAAO,CAAC;IAEtB,qCAAqC;IACrC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,gFAAgF,EAAE,EAAE,CAAC,CAAC;IAEhH,4BAA4B;IAC5B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IACjD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEvC,wEAAwE;IACxE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,uDAAuD,EAAE,EAAE,CAAC,CAAC;IAEvF,0DAA0D;IAC1D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,2CAA2C,EAAE,EAAE,CAAC,CAAC;IAE3E,2DAA2D;IAC3D,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,0CAA0C,EAAE,MAAM,CAAC,CAAC;IAE9E,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,QAAgB;IACvD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CACpC,QAAQ,EACR,OAAO,EACP,EAAE,CAAC,YAAY,CAAC,MAAM,EACtB,IAAI,CACL,CAAC;IAEF,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1B,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAEtC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,UAAU;YACb,OAAO,kCAAkC,SAAS,EAAE,CAAC;QACvD,KAAK,OAAO;YACV,OAAO,aAAa,SAAS,QAAQ,CAAC;QACxC,KAAK,WAAW,CAAC;QACjB,KAAK,MAAM;YACT,OAAO,4BAA4B,SAAS,EAAE,CAAC;QACjD,KAAK,OAAO,CAAC;QACb,KAAK,UAAU;YACb,OAAO,SAAS,SAAS,SAAS,CAAC;QACrC;YACE,OAAO,wBAAwB,SAAS,EAAE,CAAC;IAC/C,CAAC;AACH,CAAC;AAOD,SAAS,cAAc,CAAC,UAAyB;IAC/C,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,MAAM,UAAU,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC3E,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,IAAI,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;aAAM,IAAI,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;gBACrD,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,sDAAsD;IACtD,OAAO,IAAI;SACR,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC;SACnC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,WAAW,EAAE,CAAC;AACnB,CAAC"}

package/dist/benchmark/report.d.ts

@@ -0,0 +1,9 @@
+import type { BenchmarkResults } from "./types.js";
+/**
+ * Generate a human-readable markdown benchmark report.
+ */
+export declare function generateMarkdownReport(results: BenchmarkResults): string;
+/**
+ * Generate JSON benchmark results (for programmatic consumption).
+ */
+export declare function generateJsonReport(results: BenchmarkResults): string;

package/dist/benchmark/report.js

@@ -0,0 +1,131 @@
+// src/benchmark/report.ts — Generate benchmark reports (Markdown + JSON)
+import { effectSizeLabel } from "./statistics.js";
+// ─── Markdown Report ─────────────────────────────────────────────────────────
+/**
+ * Generate a human-readable markdown benchmark report.
+ */
+export function generateMarkdownReport(results) {
+    const lines = [];
+    const { meta, summary, tasks } = results;
+    lines.push(`# Benchmark Report: ${meta.repoPath.split("/").pop()}`);
+    lines.push(`Generated: ${meta.timestamp} | Model: ${meta.model} | Mode: ${meta.mode}`);
+    lines.push("");
+    // Claim scoping
+    lines.push("> **Claim scope:** This benchmark measures whether AGENTS.md improves AI adherence");
+    lines.push("> to contribution patterns (file placement, naming, imports, exports, registration)");
+    lines.push("> beyond what an AI can infer from reading source code alone.");
+    lines.push("");
+    // Summary table
+    lines.push("## Summary");
+    lines.push("");
+    lines.push("| Condition | Mean Score | Pass Rate | Avg Tokens |");
+    lines.push("|-----------|-----------|-----------|------------|");
+    const conditionLabels = {
+        "treatment": "A: AGENTS.md + source",
+        "realistic-control": "B: Source only",
+        "impoverished-control": "C: Dir listing only",
+        "negative-control": "N: Shuffled AGENTS.md",
+    };
+    for (const cond of meta.conditions) {
+        const data = summary.conditions[cond];
+        if (!data)
+            continue;
+        const label = conditionLabels[cond];
+        const score = data.meanScore.toFixed(1) + "%";
+        const pass = `${Math.round(data.passRate * summary.tasksRun)}/${summary.tasksRun}`;
+        const tokens = Math.round(data.meanTokens).toLocaleString();
+        lines.push(`| ${label} | ${score} | ${pass} | ${tokens} |`);
+    }
+    lines.push("");
+    lines.push(`**Headline (A - B):** ${summary.headlineDelta >= 0 ? "+" : ""}${summary.headlineDelta.toFixed(1)}%`);
+    lines.push(`**Upper bound (A - C):** ${summary.upperBoundDelta >= 0 ? "+" : ""}${summary.upperBoundDelta.toFixed(1)}%`);
+    // Per-task-type breakdown
+    const taskTypes = [...new Set(tasks.map(t => t.taskType))];
+    if (taskTypes.length > 1) {
+        lines.push("");
+        lines.push("## Results by Task Type");
+        lines.push("");
+        lines.push("| Type | Tasks | A Mean | B Mean | A-B Delta |");
+        lines.push("|------|-------|--------|--------|-----------|");
+        for (const type of taskTypes) {
+            const typeTasks = tasks.filter(t => t.taskType === type);
+            const aScores = typeTasks.map(t => t.results["treatment"]?.score ?? 0);
+            const bScores = typeTasks.map(t => t.results["realistic-control"]?.score ?? 0);
+            const aMean = aScores.reduce((s, v) => s + v, 0) / aScores.length;
+            const bMean = bScores.reduce((s, v) => s + v, 0) / bScores.length;
+            const delta = aMean - bMean;
+            lines.push(`| ${type} | ${typeTasks.length} | ${aMean.toFixed(1)}% | ${bMean.toFixed(1)}% | ${delta >= 0 ? "+" : ""}${delta.toFixed(1)}% |`);
+        }
+    }
+    // Statistical analysis (full mode only)
+    if (meta.mode === "full" && summary.pValue !== undefined) {
+        lines.push("");
+        lines.push("## Statistical Analysis");
+        lines.push("");
+        lines.push(`- **Paired t-test:** p = ${summary.pValue.toFixed(4)}`);
+        if (summary.effectSize !== undefined) {
+            lines.push(`- **Cohen's d:** ${summary.effectSize.toFixed(2)} (${effectSizeLabel(summary.effectSize)})`);
+        }
+        if (summary.ci95) {
+            lines.push(`- **95% Bootstrap CI:** [${summary.ci95[0].toFixed(1)}%, ${summary.ci95[1].toFixed(1)}%]`);
+        }
+    }
+    else {
+        lines.push("");
+        lines.push("*Quick mode — directional results only, not statistically powered.*");
+    }
+    // Per-task breakdown
+    lines.push("");
+    lines.push("## Per-Task Results");
+    for (const task of tasks) {
+        lines.push("");
+        lines.push(`### ${task.taskId} (Tier ${task.tier})`);
+        lines.push("");
+        lines.push(`> ${task.prompt}`);
+        lines.push("");
+        for (const cond of meta.conditions) {
+            const run = task.results[cond];
+            if (!run)
+                continue;
+            const label = conditionLabels[cond];
+            const status = run.error ? "ERROR" : run.passed ? "PASS" : "FAIL";
+            lines.push(`**${label}:** ${run.score}% (${run.rawScore}/${run.maxPoints} pts) — ${status}`);
+            if (run.error) {
+                lines.push(`  Error: ${run.error}`);
+            }
+            else {
+                for (const check of run.checks) {
+                    const icon = check.passed ? "[x]" : "[ ]";
+                    lines.push(`  - ${icon} ${check.name} (${check.score}/${check.weight}) — ${check.detail}`);
+                }
+            }
+            lines.push("");
+        }
+    }
+    // Token counts (confound disclosure)
+    lines.push("## Token Analysis");
+    lines.push("");
+    lines.push("| Condition | Mean Input Tokens | Notes |");
+    lines.push("|-----------|------------------|-------|");
+    for (const cond of meta.conditions) {
+        const data = summary.conditions[cond];
+        if (!data)
+            continue;
+        const note = cond === "treatment" ? "Includes AGENTS.md context" :
+            cond === "realistic-control" ? "Sibling files only" :
+                cond === "impoverished-control" ? "Minimal context" :
+                    "Shuffled context";
+        lines.push(`| ${conditionLabels[cond]} | ${Math.round(data.meanTokens).toLocaleString()} | ${note} |`);
+    }
+    lines.push("");
+    lines.push("*Note: Token count asymmetry between conditions may contribute to score differences.*");
+    return lines.join("\n");
+}
+// ─── JSON Report ─────────────────────────────────────────────────────────────
+/**
+ * Generate JSON benchmark results (for programmatic consumption).
+ */
+export function generateJsonReport(results) {
+    return JSON.stringify(results, null, 2);
+}
+//# sourceMappingURL=report.js.map

package/dist/benchmark/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/benchmark/report.ts"],"names":[],"mappings":"AAAA,yEAAyE;AAGzE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAEzC,KAAK,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,SAAS,aAAa,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IACjG,KAAK,CAAC,IAAI,CAAC,qFAAqF,CAAC,CAAC;IAClG,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IAClE,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;IAEjE,MAAM,eAAe,GAAuC;QAC1D,WAAW,EAAE,uBAAuB;QACpC,mBAAmB,EAAE,gBAAgB;QACrC,sBAAsB,EAAE,qBAAqB;QAC7C,kBAAkB,EAAE,uBAAuB;KAC5C,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QAC9C,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,KAAK,MAAM,IAAI,MAAM,MAAM,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACjH,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAExH,0BAA0B;IAC1B,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YACvE,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YAClE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;YAClE,MAAM,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,SAAS,CAAC,MAAM,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/I,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,eAAe,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC3G,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;IACpF,CAAC;IAED,qBAAqB;IACrB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,UAAU,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAClE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,SAAS,WAAW,MAAM,EAAE,CAAC,CAAC;YAE7F,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;oBAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC1C,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC7F,CAAC;YACH,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IACxD,KAAK,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,IAAI,GAAG,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC;YACrD,IAAI,KAAK,mBAAmB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;gBACrD,IAAI,KAAK,sBAAsB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBACrD,kBAAkB,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,KAAK,eAAe,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,cAAc,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC;IACzG,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;IAEpG,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAyB;IAC1D,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC1C,CAAC"}

package/dist/benchmark/runner.d.ts

@@ -0,0 +1,6 @@
+import type { ResolvedConfig } from "../types.js";
+import type { BenchmarkOptions, BenchmarkResults } from "./types.js";
+/**
+ * Run the full benchmark pipeline.
+ */
+export declare function orchestrateBenchmark(options: BenchmarkOptions, llmConfig: ResolvedConfig["llm"]): Promise<BenchmarkResults>;