flowseeker 0.1.7 → 0.1.9

package/dist/eval/accuracyV2.js

@@ -0,0 +1,483 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeAccuracyV2 = computeAccuracyV2;
+exports.computeFailureAnalysis = computeFailureAnalysis;
+exports.computeSemanticContribution = computeSemanticContribution;
+exports.aggregateSemanticContribution = aggregateSemanticContribution;
+exports.aggregateAccuracyV2 = aggregateAccuracyV2;
+exports.computeRoleClassifierReport = computeRoleClassifierReport;
+exports.renderRoleClassifierReportMarkdown = renderRoleClassifierReportMarkdown;
+exports.computeRoleConfusionMatrix = computeRoleConfusionMatrix;
+exports.renderAccuracyV2Markdown = renderAccuracyV2Markdown;
+const fileGroups_1 = require("../pipeline/fileGroups");
+const evaluationMetrics_1 = require("../pipeline/evaluationMetrics");
+function computeAccuracyV2(result, goldenTask) {
+    const fileGroups = (0, fileGroups_1.aggregateEvidenceFiles)(result.units);
+    const topFiles = fileGroups.map((g) => g.relativePath);
+    const topK = (k) => topFiles.slice(0, k);
+    const primaryEditPaths = fileGroups
+        .filter((g) => g.role === "edit_candidate")
+        .map((g) => g.relativePath);
+    const allFilePaths = topFiles;
+    // edit_recall@K: at least one expectedEditFile in top K
+    const editRecallAtK = (k) => goldenTask.expectedEditFiles.length === 0
+        ? true
+        : topK(k).some((f) => goldenTask.expectedEditFiles.some((ef) => (0, evaluationMetrics_1.pathMatches)(f, ef)));
+    // primary_edit_precision@3: fraction of top 3 primary edit candidates in expectedEditFiles
+    const top3Edit = primaryEditPaths.slice(0, 3);
+    const primaryEditPrecision3 = top3Edit.length > 0
+        ? top3Edit.filter((f) => goldenTask.expectedEditFiles.some((ef) => (0, evaluationMetrics_1.pathMatches)(f, ef))).length / top3Edit.length
+        : 0;
+    // read_context_recall@10: fraction of expectedReadFiles in top 10
+    const readRecall10 = goldenTask.expectedReadFiles.length > 0
+        ? goldenTask.expectedReadFiles.filter((rf) => topK(10).some((f) => (0, evaluationMetrics_1.pathMatches)(f, rf))).length / goldenTask.expectedReadFiles.length
+        : 1;
+    // test_recall@10: fraction of expectedTestFiles in top 10
+    const testRecall10 = goldenTask.expectedTestFiles.length > 0
+        ? goldenTask.expectedTestFiles.filter((tf) => topK(10).some((f) => (0, evaluationMetrics_1.pathMatches)(f, tf))).length / goldenTask.expectedTestFiles.length
+        : 1;
+    // must_not_edit_violation: any mustNotEditFile appears as primary edit candidate
+    const mustNotEditViolation = goldenTask.mustNotEditFiles.length > 0 &&
+        primaryEditPaths.some((f) => goldenTask.mustNotEditFiles.some((mf) => (0, evaluationMetrics_1.pathMatches)(f, mf)));
+    // noise_promoted_as_edit: mustNotEdit/noise files promoted to edit candidates
+    const noisePromoted = primaryEditPaths.filter((f) => goldenTask.mustNotEditFiles.some((mf) => (0, evaluationMetrics_1.pathMatches)(f, mf)));
+    const noisePromotedAsEditCount = noisePromoted.length;
+    const noisePromotedAsEditTotal = primaryEditPaths.length;
+    // required_slot_coverage / missing_slot_rate
+    const foundSlots = new Set(result.profile.blueprint.requiredSlots);
+    const coveredSlots = goldenTask.requiredSlots.filter((s) => foundSlots.has(s));
+    const requiredSlotCoverage = goldenTask.requiredSlots.length > 0
+        ? coveredSlots.length / goldenTask.requiredSlots.length
+        : 1;
+    const missingSlotRate = goldenTask.requiredSlots.length > 0
+        ? (goldenTask.requiredSlots.length - coveredSlots.length) / goldenTask.requiredSlots.length
+        : 0;
+    // first3_hit_rate: firstThreeFiles contains an expected edit, read, or test file
+    const first3 = result.solvePacket?.firstThreeFiles ?? [];
+    const allExpected = [
+        ...goldenTask.expectedEditFiles,
+        ...goldenTask.expectedReadFiles,
+        ...goldenTask.expectedTestFiles,
+    ];
+    const first3Hit = allExpected.length > 0 &&
+        first3.some((f) => allExpected.some((ef) => (0, evaluationMetrics_1.pathMatches)(f, ef)));
+    return {
+        taskId: goldenTask.id,
+        editRecall1: editRecallAtK(1),
+        editRecall3: editRecallAtK(3),
+        editRecall5: editRecallAtK(5),
+        editRecall10: editRecallAtK(10),
+        primaryEditPrecision3,
+        readContextRecall10: readRecall10,
+        testRecall10,
+        mustNotEditViolation,
+        noisePromotedAsEditCount,
+        noisePromotedAsEditTotal,
+        requiredSlotCoverage,
+        missingSlotRate,
+        first3Hit,
+    };
+}
+// ── Failure analysis ──────────────────────────────────────────────────────────
+function computeFailureAnalysis(result, goldenTask, v2) {
+    const fileGroups = (0, fileGroups_1.aggregateEvidenceFiles)(result.units);
+    const topFiles = fileGroups.map((g) => g.relativePath);
+    const top10Files = topFiles.slice(0, 10);
+    const primaryEditPaths = fileGroups.filter((g) => g.role === "edit_candidate").map((g) => g.relativePath);
+    // Missed expected files (not in top 10)
+    const inTop10 = (f) => top10Files.some((tf) => (0, evaluationMetrics_1.pathMatches)(tf, f));
+    const missedEditFiles = goldenTask.expectedEditFiles.filter((ef) => !inTop10(ef));
+    const missedReadFiles = goldenTask.expectedReadFiles.filter((rf) => !inTop10(rf));
+    const missedTestFiles = goldenTask.expectedTestFiles.filter((tf) => !inTop10(tf));
+    // wrongTop1: the top-1 file if it's not in any expected list
+    const allGoldenFiles = new Set([
+        ...goldenTask.expectedEditFiles,
+        ...goldenTask.expectedReadFiles,
+        ...goldenTask.expectedTestFiles,
+    ]);
+    const top1 = fileGroups[0];
+    const wrongTop1 = top1 && !allGoldenFiles.has(top1.relativePath) && !Array.from(allGoldenFiles).some((gf) => (0, evaluationMetrics_1.pathMatches)(top1.relativePath, gf))
+        ? top1.relativePath
+        : null;
+    // noise promoted as edit
+    const noisePromotedAsEdit = primaryEditPaths.filter((f) => goldenTask.mustNotEditFiles.some((mf) => (0, evaluationMetrics_1.pathMatches)(f, mf)));
+    // mustNotEditViolations
+    const mustNotEditViolations = primaryEditPaths.filter((f) => goldenTask.mustNotEditFiles.some((mf) => (0, evaluationMetrics_1.pathMatches)(f, mf)));
+    // Missing slots
+    const foundSlots = new Set(result.profile.blueprint.requiredSlots);
+    const missingSlots = goldenTask.requiredSlots.filter((s) => !foundSlots.has(s));
+    // Top files list
+    const topFilesList = fileGroups.slice(0, 10).map((g, i) => ({
+        rank: i + 1,
+        path: g.relativePath,
+        role: g.role,
+        score: g.score,
+    }));
+    // first3Files
+    const first3Files = result.solvePacket?.firstThreeFiles ?? [];
+    // Determine likelyFailureReason
+    let likelyFailureReason = "no_failure_detected";
+    if (v2.mustNotEditViolation) {
+        likelyFailureReason = "must_not_promoted_as_edit";
+    }
+    else if (v2.noisePromotedAsEditCount > 0) {
+        likelyFailureReason = "noisy_primary_edit";
+    }
+    else if (missingSlots.length > 0) {
+        likelyFailureReason = "missing_required_slot";
+    }
+    else if (wrongTop1) {
+        likelyFailureReason = "wrong_top1";
+    }
+    else if (missedEditFiles.length > 0) {
+        likelyFailureReason = "missed_expected_edit";
+    }
+    else if (missedReadFiles.length > 0) {
+        likelyFailureReason = "missed_expected_read";
+    }
+    else if (missedTestFiles.length > 0) {
+        likelyFailureReason = "missed_expected_test";
+    }
+    else if (!v2.editRecall1) {
+        likelyFailureReason = "weak_retrieval_evidence";
+    }
+    return {
+        taskId: goldenTask.id,
+        missedEditFiles,
+        missedReadFiles,
+        missedTestFiles,
+        wrongTop1,
+        noisePromotedAsEdit,
+        mustNotEditViolations,
+        missingSlots,
+        likelyFailureReason,
+        topFiles: topFilesList,
+        first3Files,
+    };
+}
+// ── Semantic contribution ─────────────────────────────────────────────────────
+function computeSemanticContribution(result) {
+    const stats = result.stats;
+    const status = stats.semanticRetrievalStatus ?? "disabled";
+    return {
+        semanticStatus: status,
+        semanticAddedCandidates: status === "active" ? (stats.semanticAddedFileCount ?? 0) : 0,
+        semanticSupportedCandidates: status === "active" ? (stats.semanticHitCount ?? 0) : 0,
+        semanticAcceptedAsTopK: 0, // requires deterministic comparison — not available in single-run mode
+        semanticTop3Delta: 0, // requires deterministic comparison
+        semanticErrorType: status === "error_fallback" ? (stats.semanticErrorType ?? "unknown") : null,
+    };
+}
+function aggregateSemanticContribution(perTask, results) {
+    const anyEnabled = results.some((r) => r.stats.semanticRetrievalStatus === "active");
+    const anyError = results.some((r) => r.stats.semanticRetrievalStatus === "error_fallback");
+    const status = anyEnabled ? "active" : anyError ? "error_fallback" : "disabled";
+    return {
+        semanticEnabled: anyEnabled,
+        semanticStatus: status,
+        semanticAddedCandidates: perTask.reduce((s, t) => s + t.semanticAddedCandidates, 0),
+        semanticSupportedCandidates: perTask.reduce((s, t) => s + t.semanticSupportedCandidates, 0),
+        semanticAcceptedAsTopK: 0, // requires deterministic comparison
+        semanticImprovedTop3: 0, // requires deterministic comparison
+        semanticRegressedTop3: 0, // requires deterministic comparison
+        semanticNoOp: anyEnabled ? perTask.filter((t) => t.semanticTop3Delta === 0).length : 0,
+        semanticErrorType: anyError ? (results.find((r) => r.stats.semanticErrorType)?.stats.semanticErrorType ?? null) : null,
+    };
+}
+// ── Aggregate ─────────────────────────────────────────────────────────────────
+function aggregateAccuracyV2(perTask, failureAnalysis, semanticContribution, roleConfusionMatrix, roleClassifierReport, name) {
+    const n = perTask.length || 1;
+    const avg = (vals) => vals.reduce((s, v) => s + v, 0) / vals.length;
+    const frac = (vals) => vals.filter(Boolean).length / n;
+    const noiseTotal = perTask.reduce((s, t) => s + t.noisePromotedAsEditTotal, 0);
+    const noiseCount = perTask.reduce((s, t) => s + t.noisePromotedAsEditCount, 0);
+    const er1 = frac(perTask.map((t) => t.editRecall1));
+    const er3 = frac(perTask.map((t) => t.editRecall3));
+    const er5 = frac(perTask.map((t) => t.editRecall5));
+    const er10 = frac(perTask.map((t) => t.editRecall10));
+    const pep3 = avg(perTask.map((t) => t.primaryEditPrecision3));
+    const rcr10 = avg(perTask.map((t) => t.readContextRecall10));
+    const tr10 = avg(perTask.map((t) => t.testRecall10));
+    const mnvr = frac(perTask.map((t) => t.mustNotEditViolation));
+    const rsc = avg(perTask.map((t) => t.requiredSlotCoverage));
+    const msr = avg(perTask.map((t) => t.missingSlotRate));
+    const f3h = frac(perTask.map((t) => t.first3Hit));
+    return {
+        name,
+        taskCount: perTask.length,
+        perTask,
+        // camelCase
+        editRecall1: er1,
+        editRecall3: er3,
+        editRecall5: er5,
+        editRecall10: er10,
+        primaryEditPrecision3: pep3,
+        readContextRecall10: rcr10,
+        testRecall10: tr10,
+        mustNotEditViolationRate: mnvr,
+        noisePromotedAsEditRate: noiseTotal > 0 ? noiseCount / noiseTotal : 0,
+        requiredSlotCoverage: rsc,
+        missingSlotRate: msr,
+        first3HitRate: f3h,
+        // Public metric keys
+        "edit_recall@1": er1,
+        "edit_recall@3": er3,
+        "edit_recall@5": er5,
+        "edit_recall@10": er10,
+        "primary_edit_precision@3": pep3,
+        "read_context_recall@10": rcr10,
+        "test_recall@10": tr10,
+        "must_not_edit_violation_rate": mnvr,
+        "noise_promoted_as_edit_rate": noiseTotal > 0 ? noiseCount / noiseTotal : 0,
+        "required_slot_coverage": rsc,
+        "missing_slot_rate": msr,
+        "first3_hit_rate": f3h,
+        // V2 extended fields
+        failureAnalysis,
+        semanticContribution,
+        retrievalModes: {
+            deterministic: "active",
+            semantic_fused_when_configured: semanticContribution.semanticEnabled ? "active" : "disabled_no_config",
+            file_only: "report_only_unavailable",
+            chunk_only_placeholder: "report_only_unavailable",
+            oracle_fusion_upper_bound: "report_only_unavailable",
+        },
+        roleConfusionMatrix,
+        roleClassifierReport,
+    };
+}
+function computeSingleRoleMatrix(results, goldenTasks, riskMode) {
+    const cells = {
+        expected_edit: { predicted_edit: 0, predicted_read: 0, predicted_test: 0, predicted_noise: 0, missing: 0 },
+        expected_read: { predicted_edit: 0, predicted_read: 0, predicted_test: 0, predicted_noise: 0, missing: 0 },
+        expected_test: { predicted_edit: 0, predicted_read: 0, predicted_test: 0, predicted_noise: 0, missing: 0 },
+        must_not: { predicted_edit: 0, predicted_read: 0, predicted_test: 0, predicted_noise: 0, missing: 0 },
+    };
+    const failureExamples = [];
+    let allPredictedEdit = 0;
+    let allPredictedRead = 0;
+    let allPredictedTest = 0;
+    for (let i = 0; i < goldenTasks.length; i++) {
+        const gt = goldenTasks[i];
+        const result = results[i];
+        if (!result)
+            continue;
+        const groups = (0, fileGroups_1.aggregateEvidenceFiles)(result.units);
+        const topPaths = new Set(groups.slice(0, 40).map((g) => g.relativePath));
+        const predictedEdits = new Set(groups.filter((g) => g.role === "edit_candidate").map((g) => g.relativePath));
+        const predictedReads = new Set(groups.filter((g) => g.role === "read_context" || g.role === "impact").map((g) => g.relativePath));
+        const predictedTests = new Set(groups.filter((g) => g.role === "test" || g.role === "verify").map((g) => g.relativePath));
+        const predictedNoise = new Set(result.solvePacket?.noiseRisk.map((f) => f.relativePath) ?? []);
+        const doNotEdit = new Set(result.solvePacket?.doNotEditYet ?? []);
+        allPredictedEdit += predictedEdits.size;
+        allPredictedRead += predictedReads.size;
+        allPredictedTest += predictedTests.size;
+        for (const f of topPaths) {
+            for (const rolePairs of [
+                { files: gt.expectedEditFiles, role: "expected_edit" },
+                { files: gt.expectedReadFiles, role: "expected_read" },
+                { files: gt.expectedTestFiles, role: "expected_test" },
+                { files: gt.mustNotEditFiles, role: "must_not" },
+            ]) {
+                if (rolePairs.files.some((ef) => (0, evaluationMetrics_1.pathMatches)(f, ef))) {
+                    let pred = "missing";
+                    if (predictedEdits.has(f))
+                        pred = "predicted_edit";
+                    else if (predictedReads.has(f))
+                        pred = "predicted_read";
+                    else if (predictedTests.has(f))
+                        pred = "predicted_test";
+                    else if (predictedNoise.has(f) || doNotEdit.has(f))
+                        pred = "predicted_noise";
+                    cells[rolePairs.role][pred]++;
+                    if ((rolePairs.role === "expected_edit" && pred !== "predicted_edit") ||
+                        (rolePairs.role === "must_not" && pred === "predicted_edit") ||
+                        (rolePairs.role === "expected_test" && pred !== "predicted_test" && pred !== "predicted_read")) {
+                        failureExamples.push({ taskId: gt.id, filePath: f, expectedRole: rolePairs.role, predictedRole: pred });
+                    }
+                }
+            }
+        }
+        // Count expected files NOT in top results
+        for (const ef of gt.expectedEditFiles) {
+            if (!Array.from(topPaths).some((p) => (0, evaluationMetrics_1.pathMatches)(p, ef))) {
+                cells.expected_edit.missing++;
+                failureExamples.push({ taskId: gt.id, filePath: ef, expectedRole: "expected_edit", predictedRole: "missing" });
+            }
+        }
+        for (const rf of gt.expectedReadFiles) {
+            if (!Array.from(topPaths).some((p) => (0, evaluationMetrics_1.pathMatches)(p, rf)))
+                cells.expected_read.missing++;
+        }
+        for (const tf of gt.expectedTestFiles) {
+            if (!Array.from(topPaths).some((p) => (0, evaluationMetrics_1.pathMatches)(p, tf)))
+                cells.expected_test.missing++;
+        }
+    }
+    const rowTotals = {
+        expected_edit: Object.values(cells.expected_edit).reduce((s, v) => s + v, 0),
+        expected_read: Object.values(cells.expected_read).reduce((s, v) => s + v, 0),
+        expected_test: Object.values(cells.expected_test).reduce((s, v) => s + v, 0),
+        must_not: Object.values(cells.must_not).reduce((s, v) => s + v, 0),
+    };
+    // Fix: predictedRoleAccuracy = correct predictions / all predictions of that role
+    // expectedRowRecall = correct / expected total (same as before, renamed)
+    const pae = allPredictedEdit || 1;
+    const par = allPredictedRead || 1;
+    const pat = allPredictedTest || 1;
+    return {
+        cells,
+        rowTotals,
+        predictedRoleAccuracy: {
+            edit: cells.expected_edit.predicted_edit / pae,
+            read: cells.expected_read.predicted_read / par,
+            test: cells.expected_test.predicted_test / pat,
+        },
+        expectedRowRecall: {
+            edit: rowTotals.expected_edit > 0 ? cells.expected_edit.predicted_edit / (cells.expected_edit.predicted_edit + cells.expected_edit.missing || 1) : 0,
+            read: rowTotals.expected_read > 0 ? cells.expected_read.predicted_read / (cells.expected_read.predicted_read + cells.expected_read.missing || 1) : 0,
+            test: rowTotals.expected_test > 0 ? cells.expected_test.predicted_test / (cells.expected_test.predicted_test + cells.expected_test.missing || 1) : 0,
+        },
+        failureExamples: failureExamples.slice(0, 20),
+        riskMode,
+    };
+}
+function computeRoleClassifierReport(results, goldenTasks) {
+    // Currently same classifier for all modes; report-only
+    const balanced = computeSingleRoleMatrix(results, goldenTasks, "balanced");
+    const strict = computeSingleRoleMatrix(results, goldenTasks, "strict");
+    const broad = computeSingleRoleMatrix(results, goldenTasks, "broad");
+    return {
+        defaultMode: "balanced",
+        modeImplementation: "same_current_classifier_report_only",
+        modeSemantics: {
+            strict: "prefer fewer edit candidates; aggressively protect must_not/noise",
+            balanced: "current/default behavior",
+            broad: "wider context; lower-confidence files must be marked clearly",
+        },
+        modes: { strict, balanced, broad },
+        failureExamples: balanced.failureExamples,
+    };
+}
+function renderRoleClassifierReportMarkdown(report) {
+    const lines = ["### Role Classifier Report", "", `Default mode: ${report.defaultMode}`,
+        `Implementation: ${report.modeImplementation}`, ""];
+    for (const mode of ["strict", "balanced", "broad"]) {
+        const m = report.modes[mode];
+        lines.push(`#### ${mode} mode`, "");
+        lines.push("| Expected \\ Predicted | edit | read | test | noise | missing | Total |");
+        lines.push("|-----------------------|------|------|------|-------|---------|-------|");
+        for (const row of [
+            { label: "expected_edit", key: "expected_edit" },
+            { label: "expected_read", key: "expected_read" },
+            { label: "expected_test", key: "expected_test" },
+            { label: "must_not", key: "must_not" },
+        ]) {
+            const c = m.cells[row.key];
+            lines.push(`| ${row.label} | ${c.predicted_edit} | ${c.predicted_read} | ${c.predicted_test} | ${c.predicted_noise} | ${c.missing} | ${m.rowTotals[row.key]} |`);
+        }
+        lines.push("", `Predicted-role accuracy: edit=${(m.predictedRoleAccuracy.edit * 100).toFixed(1)}% read=${(m.predictedRoleAccuracy.read * 100).toFixed(1)}% test=${(m.predictedRoleAccuracy.test * 100).toFixed(1)}%`, `Expected-row recall: edit=${(m.expectedRowRecall.edit * 100).toFixed(1)}% read=${(m.expectedRowRecall.read * 100).toFixed(1)}% test=${(m.expectedRowRecall.test * 100).toFixed(1)}%`, "");
+    }
+    if (report.failureExamples.length > 0) {
+        lines.push("### Role Failure Examples", "");
+        for (const ex of report.failureExamples.slice(0, 10)) {
+            lines.push(`- **${ex.taskId}**: \`${ex.filePath}\` expected=${ex.expectedRole} predicted=${ex.predictedRole}`);
+        }
+        lines.push("");
+    }
+    return lines;
+}
+// Kept for backward compat
+function computeRoleConfusionMatrix(results, goldenTasks, riskMode = "balanced") {
+    return computeSingleRoleMatrix(results, goldenTasks, riskMode);
+}
+function renderAccuracyV2Markdown(report) {
+    const lines = [
+        "## Accuracy V2",
+        "",
+        `Tasks: ${report.taskCount}`,
+        "",
+        "### Retrieval Metrics",
+        "",
+        `| Metric | Value |`,
+        `|--------|-------|`,
+        `| edit_recall@1 | ${(report.editRecall1 * 100).toFixed(1)}% |`,
+        `| edit_recall@3 | ${(report.editRecall3 * 100).toFixed(1)}% |`,
+        `| edit_recall@5 | ${(report.editRecall5 * 100).toFixed(1)}% |`,
+        `| edit_recall@10 | ${(report.editRecall10 * 100).toFixed(1)}% |`,
+        `| primary_edit_precision@3 | ${(report.primaryEditPrecision3 * 100).toFixed(1)}% |`,
+        `| read_context_recall@10 | ${(report.readContextRecall10 * 100).toFixed(1)}% |`,
+        `| test_recall@10 | ${(report.testRecall10 * 100).toFixed(1)}% |`,
+        "",
+        "### Safety Metrics",
+        "",
+        `| Metric | Value |`,
+        `|--------|-------|`,
+        `| must_not_edit_violation_rate | ${(report.mustNotEditViolationRate * 100).toFixed(1)}% |`,
+        `| noise_promoted_as_edit_rate | ${(report.noisePromotedAsEditRate * 100).toFixed(1)}% |`,
+        "",
+        "### Flow Completeness",
+        "",
+        `| Metric | Value |`,
+        `|--------|-------|`,
+        `| required_slot_coverage | ${(report.requiredSlotCoverage * 100).toFixed(1)}% |`,
+        `| missing_slot_rate | ${(report.missingSlotRate * 100).toFixed(1)}% |`,
+        `| first3_hit_rate | ${(report.first3HitRate * 100).toFixed(1)}% |`,
+        "",
+        "### Per-Task Detail",
+        "",
+    ];
+    for (const t of report.perTask) {
+        lines.push(`- **${t.taskId}**: editR@1=${t.editRecall1} editR@3=${t.editRecall3} editR@5=${t.editRecall5} editR@10=${t.editRecall10} editP@3=${t.primaryEditPrecision3.toFixed(2)} readR@10=${t.readContextRecall10.toFixed(2)} testR@10=${t.testRecall10.toFixed(2)} mustNotViolation=${t.mustNotEditViolation} noiseAsEdit=${t.noisePromotedAsEditCount}/${t.noisePromotedAsEditTotal} slotCov=${t.requiredSlotCoverage.toFixed(2)} missingSlot=${t.missingSlotRate.toFixed(2)} first3Hit=${t.first3Hit}`);
+    }
+    // Failure analysis section
+    lines.push("");
+    lines.push("### Failure Analysis", "");
+    for (const fa of report.failureAnalysis) {
+        lines.push(`#### ${fa.taskId}`);
+        lines.push(`- **likelyFailureReason:** ${fa.likelyFailureReason}`);
+        lines.push(`- **missedEditFiles:** ${fa.missedEditFiles.join(", ") || "none"}`);
+        lines.push(`- **missedReadFiles:** ${fa.missedReadFiles.join(", ") || "none"}`);
+        lines.push(`- **missedTestFiles:** ${fa.missedTestFiles.join(", ") || "none"}`);
+        lines.push(`- **wrongTop1:** ${fa.wrongTop1 ?? "none"}`);
+        lines.push(`- **noisePromotedAsEdit:** ${fa.noisePromotedAsEdit.join(", ") || "none"}`);
+        lines.push(`- **mustNotEditViolations:** ${fa.mustNotEditViolations.join(", ") || "none"}`);
+        lines.push(`- **missingSlots:** ${fa.missingSlots.join(", ") || "none"}`);
+        lines.push(`- **first3Files:** ${fa.first3Files.join(", ") || "none"}`);
+        lines.push(`- **topFiles:** ${fa.topFiles.map((f) => `${f.rank}.${f.path}[${f.role}]`).join(" ")}`);
+        lines.push("");
+    }
+    // Semantic contribution section
+    const sc = report.semanticContribution;
+    lines.push("### Semantic Contribution", "");
+    lines.push(`| Field | Value |`);
+    lines.push(`|-------|-------|`);
+    lines.push(`| semanticEnabled | ${sc.semanticEnabled} |`);
+    lines.push(`| semanticStatus | ${sc.semanticStatus} |`);
+    lines.push(`| semanticAddedCandidates | ${sc.semanticAddedCandidates} |`);
+    lines.push(`| semanticSupportedCandidates | ${sc.semanticSupportedCandidates} |`);
+    lines.push(`| semanticAcceptedAsTopK | ${sc.semanticAcceptedAsTopK} |`);
+    lines.push(`| semanticImprovedTop3 | ${sc.semanticImprovedTop3} |`);
+    lines.push(`| semanticRegressedTop3 | ${sc.semanticRegressedTop3} |`);
+    lines.push(`| semanticNoOp | ${sc.semanticNoOp} |`);
+    lines.push(`| semanticErrorType | ${sc.semanticErrorType ?? "none"} |`);
+    lines.push("");
+    // Retrieval modes section
+    const rm = report.retrievalModes;
+    lines.push("### Retrieval Modes", "");
+    lines.push(`| Mode | Status |`);
+    lines.push(`|------|--------|`);
+    lines.push(`| deterministic | ${rm.deterministic} |`);
+    lines.push(`| semantic_fused_when_configured | ${rm.semantic_fused_when_configured} |`);
+    lines.push(`| file_only | ${rm.file_only} |`);
+    lines.push(`| chunk_only_placeholder | ${rm.chunk_only_placeholder} |`);
+    lines.push(`| oracle_fusion_upper_bound | ${rm.oracle_fusion_upper_bound} |`);
+    lines.push("");
+    lines.push("");
+    if (report.roleClassifierReport) {
+        lines.push(...renderRoleClassifierReportMarkdown(report.roleClassifierReport));
+    }
+    lines.push("");
+    return lines;
+}
+//# sourceMappingURL=accuracyV2.js.map

package/dist/eval/goldenTask.js

@@ -0,0 +1,192 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateGoldenTask = validateGoldenTask;
+exports.validateGoldenManifest = validateGoldenManifest;
+const fssync = __importStar(require("fs"));
+const path = __importStar(require("path"));
+function validateGoldenTask(task, index) {
+    const issues = [];
+    const prefix = `GoldenTask[${index}]`;
+    if (!task || typeof task !== "object") {
+        return [`${prefix}: not an object`];
+    }
+    const t = task;
+    // Required string fields — must be non-empty strings
+    const stringFields = ["id", "repoType", "repoName", "workspace", "task", "notes"];
+    for (const field of stringFields) {
+        const val = t[field];
+        if (typeof val !== "string" || !val.trim()) {
+            issues.push(`${prefix}: ${field} must be a non-empty string`);
+        }
+    }
+    // Array fields — must be arrays of non-empty strings
+    const stringArrayFields = [
+        { key: "expectedEditFiles", arr: t.expectedEditFiles },
+        { key: "expectedReadFiles", arr: t.expectedReadFiles },
+        { key: "expectedTestFiles", arr: t.expectedTestFiles },
+        { key: "mustNotEditFiles", arr: t.mustNotEditFiles },
+        { key: "requiredSlots", arr: t.requiredSlots },
+        { key: "tags", arr: t.tags },
+    ];
+    for (const { key, arr } of stringArrayFields) {
+        if (!Array.isArray(arr)) {
+            issues.push(`${prefix}: ${key} must be an array`);
+            continue;
+        }
+        for (let i = 0; i < arr.length; i++) {
+            if (typeof arr[i] !== "string" || !arr[i].trim()) {
+                issues.push(`${prefix}: ${key}[${i}] must be a non-empty string`);
+            }
+        }
+    }
+    // Path safety checks for file arrays
+    const fileArrays = [
+        { key: "expectedEditFiles", arr: t.expectedEditFiles },
+        { key: "expectedReadFiles", arr: t.expectedReadFiles },
+        { key: "expectedTestFiles", arr: t.expectedTestFiles },
+        { key: "mustNotEditFiles", arr: t.mustNotEditFiles },
+    ];
+    for (const { key, arr } of fileArrays) {
+        if (!Array.isArray(arr))
+            continue;
+        for (let i = 0; i < arr.length; i++) {
+            const p = arr[i];
+            if (!p || !p.trim())
+                continue;
+            if (p.includes("..")) {
+                issues.push(`${prefix}: ${key}[${i}] contains path traversal (".."): ${p}`);
+            }
+            if (/^[A-Za-z]:[\\/]/.test(p) || p.startsWith("/")) {
+                issues.push(`${prefix}: ${key}[${i}] is an absolute path: ${p}`);
+            }
+        }
+    }
+    // Duplicate file paths across edit/read/test/must-not roles
+    if (Array.isArray(t.expectedEditFiles) && Array.isArray(t.expectedReadFiles) &&
+        Array.isArray(t.expectedTestFiles) && Array.isArray(t.mustNotEditFiles)) {
+        const fileSet = new Set();
+        const allFiles = [];
+        for (const f of t.expectedEditFiles) {
+            allFiles.push({ key: "expectedEditFiles", path: f });
+        }
+        for (const f of t.expectedReadFiles) {
+            allFiles.push({ key: "expectedReadFiles", path: f });
+        }
+        for (const f of t.expectedTestFiles) {
+            allFiles.push({ key: "expectedTestFiles", path: f });
+        }
+        for (const f of t.mustNotEditFiles) {
+            allFiles.push({ key: "mustNotEditFiles", path: f });
+        }
+        for (const { key, path: fp } of allFiles) {
+            if (!fp || !fp.trim())
+                continue;
+            const norm = fp.replace(/\\/g, "/").toLowerCase();
+            if (fileSet.has(norm)) {
+                issues.push(`${prefix}: duplicate file path "${fp}" in ${key} (normalized: ${norm})`);
+            }
+            fileSet.add(norm);
+        }
+    }
+    // Empty expectedEditFiles for normal edit/bugfix/feature tasks
+    const editFiles = Array.isArray(t.expectedEditFiles) ? t.expectedEditFiles : [];
+    const taskTags = Array.isArray(t.tags) ? t.tags : [];
+    const isReadOnly = taskTags.includes("read_only");
+    const isAnalysisOnly = taskTags.includes("analysis_only");
+    if (editFiles.length === 0 && !isReadOnly && !isAnalysisOnly) {
+        issues.push(`${prefix}: expectedEditFiles is empty; tag as read_only or analysis_only if no edits expected`);
+    }
+    return issues;
+}
+function validateGoldenManifest(manifest, workspaceRoot) {
+    if (!manifest || typeof manifest !== "object") {
+        return ["GoldenEvalManifest: not an object"];
+    }
+    const m = manifest;
+    const issues = [];
+    if (typeof m.name !== "string" || !m.name.trim())
+        issues.push("GoldenEvalManifest: name must be a non-empty string");
+    if (typeof m.workspace !== "string" || !m.workspace.trim())
+        issues.push("GoldenEvalManifest: workspace must be a non-empty string");
+    if (!Array.isArray(m.tasks) || m.tasks.length === 0) {
+        issues.push("GoldenEvalManifest: tasks must be a non-empty array");
+        return issues;
+    }
+    // Duplicate task ids
+    const taskIds = new Set();
+    for (let i = 0; i < m.tasks.length; i++) {
+        const t = m.tasks[i];
+        if (t && typeof t.id === "string" && t.id.trim()) {
+            if (taskIds.has(t.id)) {
+                issues.push(`GoldenTask[${i}]: duplicate task id "${t.id}"`);
+            }
+            taskIds.add(t.id);
+        }
+    }
+    for (let i = 0; i < m.tasks.length; i++) {
+        issues.push(...validateGoldenTask(m.tasks[i], i));
+    }
+    // Non-existent expected file paths (only when workspaceRoot is provided)
+    if (workspaceRoot && issues.length === 0) {
+        const wsRoot = workspaceRoot;
+        for (let i = 0; i < m.tasks.length; i++) {
+            const t = m.tasks[i];
+            const fileArrays = [
+                { key: "expectedEditFiles", arr: t.expectedEditFiles },
+                { key: "expectedReadFiles", arr: t.expectedReadFiles },
+                { key: "expectedTestFiles", arr: t.expectedTestFiles },
+                { key: "mustNotEditFiles", arr: t.mustNotEditFiles },
+            ];
+            for (const { key, arr } of fileArrays) {
+                if (!Array.isArray(arr))
+                    continue;
+                for (let j = 0; j < arr.length; j++) {
+                    const fp = arr[j];
+                    if (!fp || !fp.trim())
+                        continue;
+                    if (fp.includes("*"))
+                        continue; // glob patterns not checked
+                    const abs = path.join(wsRoot, fp);
+                    if (!fssync.existsSync(abs)) {
+                        issues.push(`GoldenTask[${i}]: ${key}[${j}] path does not exist: ${fp} (workspace: ${wsRoot})`);
+                    }
+                }
+            }
+        }
+    }
+    return issues;
+}
+//# sourceMappingURL=goldenTask.js.map