autodocs-engine 0.5.0

package/dist/benchmark/runner.js

@@ -0,0 +1,183 @@
+// src/benchmark/runner.ts — Benchmark orchestrator
+// Coordinates: analyze → generate tasks → AGENTS.md → A/B/C/N → score → report
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { analyze, formatDeterministic } from "../index.js";
+import { ENGINE_VERSION } from "../types.js";
+import { generateTasksFromAnalysis } from "./task-generator.js";
+import { generateCode } from "./code-generator.js";
+import { scoreGeneratedOutput } from "./scorer.js";
+import { shuffleAgentsMd } from "./shuffler.js";
+import { pairedTTest, cohensD, bootstrapCI } from "./statistics.js";
+import { generateMarkdownReport, generateJsonReport } from "./report.js";
+// ─── Public API ──────────────────────────────────────────────────────────────
+/**
+ * Run the full benchmark pipeline.
+ */
+export async function orchestrateBenchmark(options, llmConfig) {
+    const conditions = [
+        "treatment",
+        "realistic-control",
+        "impoverished-control",
+        "negative-control",
+    ];
+    const log = (msg) => {
+        if (options.verbose)
+            process.stderr.write(`[BENCH] ${msg}\n`);
+    };
+    // Phase 1: Analyze the repo
+    log("Phase 1: Analyzing repository...");
+    const analysis = await analyze({
+        packages: [options.repoPath],
+        rootDir: options.rootDir,
+    });
+    // Phase 2: Generate tasks
+    log("Phase 2: Generating benchmark tasks...");
+    const maxTasks = options.mode === "quick"
+        ? Math.min(options.maxTasks ?? 5, 5)
+        : options.maxTasks ?? 20;
+    const tasks = generateTasksFromAnalysis(analysis, options.repoPath, maxTasks);
+    if (tasks.length === 0) {
+        throw new Error("No benchmark tasks could be generated. This repo may lack contribution patterns.\n"
+            + "Try a repo with structured patterns (e.g., directories with 3+ similar files).");
+    }
+    const patternCount = tasks.filter(t => t.taskType === "pattern").length;
+    const commandCount = tasks.filter(t => t.taskType === "command").length;
+    const archCount = tasks.filter(t => t.taskType === "architecture").length;
+    log(`  ${tasks.length} tasks generated: ${patternCount} pattern, ${commandCount} command, ${archCount} architecture (${tasks.filter(t => t.tier === "A").length} Tier A, ${tasks.filter(t => t.tier === "B").length} Tier B, ${tasks.filter(t => t.tier === "C").length} Tier C)`);
+    // Dry run: show tasks and exit
+    if (options.dryRun) {
+        for (const task of tasks) {
+            process.stderr.write(`\n--- Task: ${task.id} [${task.taskType}] (Tier ${task.tier}) ---\n`);
+            process.stderr.write(`Prompt: ${task.prompt}\n`);
+            process.stderr.write(`Directory: ${task.expectedDirectory}\n`);
+            process.stderr.write(`Pattern: ${task.expectedFilePattern}\n`);
+            process.stderr.write(`Siblings: ${task.context.siblingFiles.map(s => s.path).join(", ")}\n`);
+            process.stderr.write(`Registration: ${task.context.registrationFile?.path ?? "none"}\n`);
+            process.stderr.write(`Barrel: ${task.context.barrelFile?.path ?? "none"}\n`);
+            process.stderr.write(`Max points: ${task.maxScoringPoints}\n`);
+        }
+        // Return empty results for dry run
+        return {
+            meta: {
+                engineVersion: ENGINE_VERSION,
+                model: llmConfig.model,
+                repoPath: options.repoPath,
+                timestamp: new Date().toISOString(),
+                mode: options.mode,
+                conditions,
+            },
+            summary: emptySummary(conditions),
+            tasks: [],
+        };
+    }
+    // Phase 3: Generate AGENTS.md
+    log("Phase 3: Generating AGENTS.md...");
+    let agentsMd = null;
+    try {
+        const formatConfig = {
+            output: { format: "agents.md", dir: "." },
+            llm: llmConfig,
+        };
+        agentsMd = await formatDeterministic(analysis, formatConfig);
+    }
+    catch {
+        log("  Warning: AGENTS.md generation failed (LLM unavailable?). Using JSON analysis as fallback.");
+    }
+    // Generate shuffled AGENTS.md for negative control
+    const shuffledAgentsMd = agentsMd ? shuffleAgentsMd(agentsMd) : null;
+    // Phase 4: Run A/B/C/N for each task
+    log(`Phase 4: Running ${tasks.length} tasks × ${conditions.length} conditions...`);
+    const taskResults = [];
+    for (let i = 0; i < tasks.length; i++) {
+        const task = tasks[i];
+        log(`  Task ${i + 1}/${tasks.length}: ${task.id} (Tier ${task.tier})`);
+        const results = {};
+        for (const condition of conditions) {
+            log(`    Condition: ${condition}...`);
+            const codeResult = await generateCode(task, condition, agentsMd, shuffledAgentsMd, llmConfig);
+            const runResult = scoreGeneratedOutput(codeResult.files, task, codeResult.tokensUsed, codeResult.latencyMs, codeResult.error);
+            results[condition] = runResult;
+            log(`    → ${runResult.score}% (${runResult.rawScore}/${runResult.maxPoints}) ${runResult.passed ? "PASS" : "FAIL"}`);
+        }
+        taskResults.push({
+            taskId: task.id,
+            tier: task.tier,
+            taskType: task.taskType,
+            prompt: task.prompt,
+            results,
+        });
+    }
+    // Phase 5: Compute statistics and generate report
+    log("Phase 5: Computing statistics and generating report...");
+    const summary = computeSummary(taskResults, conditions, options.mode);
+    const benchmarkResults = {
+        meta: {
+            engineVersion: ENGINE_VERSION,
+            model: llmConfig.model,
+            repoPath: options.repoPath,
+            timestamp: new Date().toISOString(),
+            mode: options.mode,
+            conditions,
+        },
+        summary,
+        tasks: taskResults,
+    };
+    // Write reports
+    const outputDir = options.outputDir ?? "./benchmark-results";
+    mkdirSync(outputDir, { recursive: true });
+    writeFileSync(join(outputDir, "results.json"), generateJsonReport(benchmarkResults));
+    writeFileSync(join(outputDir, "REPORT.md"), generateMarkdownReport(benchmarkResults));
+    log(`Reports written to ${outputDir}/`);
+    return benchmarkResults;
+}
+// ─── Summary Computation ─────────────────────────────────────────────────────
+function computeSummary(tasks, conditions, mode) {
+    const n = tasks.length;
+    const conditionData = {};
+    for (const cond of conditions) {
+        const scores = tasks.map(t => t.results[cond]?.score ?? 0);
+        const tokens = tasks.map(t => t.results[cond]?.tokensUsed ?? 0);
+        const passCount = tasks.filter(t => t.results[cond]?.passed).length;
+        conditionData[cond] = {
+            meanScore: scores.reduce((s, v) => s + v, 0) / n,
+            passRate: passCount / n,
+            scores,
+            meanTokens: tokens.reduce((s, v) => s + v, 0) / n,
+        };
+    }
+    const aScores = conditionData["treatment"]?.scores ?? [];
+    const bScores = conditionData["realistic-control"]?.scores ?? [];
+    const cScores = conditionData["impoverished-control"]?.scores ?? [];
+    const headlineDelta = (conditionData["treatment"]?.meanScore ?? 0)
+        - (conditionData["realistic-control"]?.meanScore ?? 0);
+    const upperBoundDelta = (conditionData["treatment"]?.meanScore ?? 0)
+        - (conditionData["impoverished-control"]?.meanScore ?? 0);
+    const summary = {
+        tasksRun: n,
+        conditions: conditionData,
+        headlineDelta,
+        upperBoundDelta,
+    };
+    // Statistical analysis for full mode only (n >= 15)
+    if (mode === "full" && n >= 10) {
+        const tTest = pairedTTest(aScores, bScores);
+        summary.pValue = tTest.p;
+        summary.effectSize = cohensD(aScores, bScores);
+        summary.ci95 = bootstrapCI(aScores, bScores);
+    }
+    return summary;
+}
+function emptySummary(conditions) {
+    const conditionData = {};
+    for (const cond of conditions) {
+        conditionData[cond] = { meanScore: 0, passRate: 0, scores: [], meanTokens: 0 };
+    }
+    return {
+        tasksRun: 0,
+        conditions: conditionData,
+        headlineDelta: 0,
+        upperBoundDelta: 0,
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/benchmark/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA,mDAAmD;AACnD,+EAA+E;AAE/E,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAsB,MAAM,iBAAiB,CAAC;AACxF,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAUzE,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAyB,EACzB,SAAgC;IAEhC,MAAM,UAAU,GAAyB;QACvC,WAAW;QACX,mBAAmB;QACnB,sBAAsB;QACtB,kBAAkB;KACnB,CAAC;IAEF,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,OAAO;YAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAChE,CAAC,CAAC;IAEF,4BAA4B;IAC5B,GAAG,CAAC,kCAAkC,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC;QAC7B,QAAQ,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,0BAA0B;IAC1B,GAAG,CAAC,wCAAwC,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO;QACvC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IAE3B,MAAM,KAAK,GAAG,yBAAyB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE9E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,oFAAoF;cAClF,gFAAgF,CACnF,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;IAC1E,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,qBAAqB,YAAY,aAAa,YAAY,aAAa,SAAS,kBAAkB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC;IAEnR,+BAA+B;IAC/B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,IAAI,SAAS,CAAC,CAAC;YAC5F,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;YAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;YAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,IAAI,MAAM,IAAI,CAAC,CAAC;YACzF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,IAAI,MAAM,IAAI,CAAC,CAAC;YAC7E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACjE,CAAC;QACD,mCAAmC;QACnC,OAAO;YACL,IAAI,EAAE;gBACJ,aAAa,EAAE,cAAc;gBAC7B,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,UAAU;aACX;YACD,OAAO,EAAE,YAAY,CAAC,UAAU,CAAC;YACjC,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;IAED,8BAA8B;IAC9B,GAAG,CAAC,kCAAkC,CAAC,CAAC;IACxC,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,YAAY,GAAG;YACnB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAoB,EAAE,GAAG,EAAE,GAAG,EAAE;YAClD,GAAG,EAAE,SAAS;SACf,CAAC;QACF,QAAQ,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,CAAC,6FAA6F,CAAC,CAAC;IACrG,CAAC;IAED,mDAAmD;IACnD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAErE,qCAAqC;IACrC,GAAG,CAAC,oBAAoB,KAAK,CAAC,MAAM,YAAY,UAAU,CAAC,MAAM,gBAAgB,CAAC,CAAC;IACnF,MAAM,WAAW,GAAiB,EAAE,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,UAAU,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QAEvE,MAAM,OAAO,GAA0C,EAA2C,CAAC;QAEnG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,GAAG,CAAC,kBAAkB,SAAS,KAAK,CAAC,CAAC;YACtC,MAAM,UAAU,GAAG,MAAM,YAAY,CACnC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,SAAS,CACvD,CAAC;YAEF,MAAM,SAAS,GAAG,oBAAoB,CACpC,UAAU,CAAC,KAAK,EAChB,IAAI,EACJ,UAAU,CAAC,UAAU,EACrB,UAAU,CAAC,SAAS,EACpB,UAAU,CAAC,KAAK,CACjB,CAAC;YAEF,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;YAC/B,GAAG,CAAC,SAAS,SAAS,CAAC,KAAK,MAAM,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACxH,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,kDAAkD;IAClD,GAAG,CAAC,wDAAwD,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtE,MAAM,gBAAgB,GAAqB;QACzC,IAAI,EAAE;YACJ,aAAa,EAAE,cAAc;YAC7B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,UAAU;SACX;QACD,OAAO;QACP,KAAK,EAAE,WAAW;KACnB,CAAC;IAEF,gBAAgB;IAChB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,qBAAqB,CAAC;IAC7D,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACrF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,sBAAsB,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAEtF,GAAG,CAAC,sBAAsB,SAAS,GAAG,CAAC,CAAC;IAExC,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,gFAAgF;AAEhF,SAAS,cAAc,CACrB,KAAmB,EACnB,UAAgC,EAChC,IAAsB;IAEtB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,aAAa,GAAmC,EAAoC,CAAC;IAE3F,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC;QAEpE,aAAa,CAAC,IAAI,CAAC,GAAG;YACpB,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC;YAChD,QAAQ,EAAE,SAAS,GAAG,CAAC;YACvB,MAAM;YACN,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC;SAClD,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,aAAa,CAAC,WAAW,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;IACzD,MAAM,OAAO,GAAG,aAAa,CAAC,mBAAmB,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;IACjE,MAAM,OAAO,GAAG,aAAa,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;IAEpE,MAAM,aAAa,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC;UAC9D,CAAC,aAAa,CAAC,mBAAmB,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;IACzD,MAAM,eAAe,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC;UAChE,CAAC,aAAa,CAAC,sBAAsB,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;IAE5D,MAAM,OAAO,GAAqB;QAChC,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,aAAa;QACzB,aAAa;QACb,eAAe;KAChB,CAAC;IAEF,oDAAoD;IACpD,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QACzB,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,YAAY,CAAC,UAAgC;IACpD,MAAM,aAAa,GAAG,EAAoC,CAAC;IAC3D,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACjF,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,aAAa;QACzB,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,CAAC;KACnB,CAAC;AACJ,CAAC"}

package/dist/benchmark/scorer.d.ts

@@ -0,0 +1,6 @@
+import type { BenchmarkTask, GeneratedFile, RunResult } from "./types.js";
+/**
+ * Score generated files against a benchmark task's expected patterns.
+ * Returns a RunResult with per-check scoring breakdown.
+ */
+export declare function scoreGeneratedOutput(files: GeneratedFile[], task: BenchmarkTask, tokensUsed: number, latencyMs: number, error?: string): RunResult;