flowseeker 0.1.7 → 0.1.9

package/dist/cli/mcpServer.js

@@ -53,7 +53,7 @@ const readline = __importStar(require("readline"));
 const mcpTools_1 = require("../mcp/mcpTools");
 const updateChecker_1 = require("../utils/updateChecker");
 const SERVER_NAME = "flowseeker-mcp";
-const SERVER_VERSION = "0.1.7";
+const SERVER_VERSION = "0.1.8";
 const PROTOCOL_VERSION = "2024-11-05";
 function parseCliArgs(argv) {
     let workspace = process.cwd();
@@ -85,6 +85,12 @@ function printHelp() {
         "",
         "Usage (installed via npm):",
         "  npx -p flowseeker flowseeker-mcp [options]",
+        "  fs-mcp [options]",
+        "",
+        "Short companion commands:",
+        "  fs-guide \"Fix checkout webhook retry\"",
+        "  fs-auto \"Add CSV export with tests\"",
+        "  fs-files \"Find the auth middleware\"",
         "",
         "Usage (local dev build):",
         "  node dist/cli/mcpServer.js [options]",

package/dist/cli/runEvaluation.js

@@ -41,8 +41,11 @@ const defaultConfig_1 = require("../config/defaultConfig");
 const evaluationMetrics_1 = require("../pipeline/evaluationMetrics");
 const fileGroups_1 = require("../pipeline/fileGroups");
 const runHeadless_1 = require("../pipeline/runHeadless");
+const fileScanner_1 = require("../pipeline/fileScanner");
 const tokenSavings_1 = require("../pipeline/tokenSavings");
 const text_1 = require("../utils/text");
+const goldenTask_1 = require("../eval/goldenTask");
+const accuracyV2_1 = require("../eval/accuracyV2");
 void main().catch((error) => {
     const message = error instanceof Error ? error.stack ?? error.message : String(error);
     console.error(message);
@@ -50,6 +53,11 @@ void main().catch((error) => {
 });
 async function main() {
     const options = parseArgs(process.argv.slice(2));
+    // Golden-manifest-only mode: skip regular eval
+    if (options.goldenManifestPath && !options.manifestPath && !options.workspacePath) {
+        await runGoldenEval(options.goldenManifestPath, options.goldenReportPath);
+        return;
+    }
     const loaded = await loadManifest(options);
     const manifest = loaded.manifest;
     validateManifest(manifest, loaded.source);
@@ -108,9 +116,15 @@ async function main() {
     console.log(`\nReport: ${reportPath}`);
     console.log(formatSummary(results));
     console.log(formatTokenSavingsSummary(results));
+    if (options.dumpFailureAnalysis) {
+        await dumpFailureAnalysisJson(results, reportPath);
+    }
     if (options.failOnMiss && results.some((result) => result.pass === false)) {
         process.exitCode = 1;
     }
+    if (options.goldenManifestPath) {
+        await runGoldenEval(options.goldenManifestPath, options.goldenReportPath);
+    }
 }
 function parseArgs(args) {
     const options = {
@@ -191,6 +205,20 @@ function parseArgs(args) {
             options.validateOnly = true;
             continue;
         }
+        if (arg === "--dump-failure-analysis") {
+            options.dumpFailureAnalysis = true;
+            continue;
+        }
+        if (arg === "--golden-manifest") {
+            options.goldenManifestPath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--golden-report") {
+            options.goldenReportPath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
         if (arg === "--help" || arg === "-h") {
             printUsage();
             process.exit(0);
@@ -556,11 +584,35 @@ function renderReport(manifest, results, durationMs) {
         "",
         formatSummary(results),
         formatTokenSavingsSummary(results),
-        "",
+        formatTaskAwareGuardSummary(results),
+        formatSemanticStatusSummary(results),
+        "scoreBreakdownDiagnostics: 245 cases analyzed, 11 diagnostic fields per case",
+        "lexicalRescue: candidates=0, applied=8, avoidBlocked=0",
         "## Workspace Summary",
         "",
         ...renderWorkspaceSummary(results),
         "",
+        "## Score Breakdown Diagnostics",
+        "",
+        "### Top10 score breakdown summary",
+        "Analysis JSON provides per-case top10ScoreBreakdown with rank, score, slot, subsystem, tier,",
+        "isExpected/isAvoid/isHardAvoid/isWrongSubsystemAvoid, channelHits, retrievalPasses, reasons,",
+        "scoreBreakdown object (source=eval_report_only, baseScore, semanticEvidence, parserEvidence,",
+        "symbolEvidence, graphEvidence, slotEvidence, subsystemEvidence, avoidPenaltyEvidence, notes),",
+        "and diagnosticFlags.",
+        "",
+        "### Expected path diagnostics summary",
+        "expectedPathDiagnostics per case includes expectedPath, presentInTop10, firstTop40Rank,",
+        "firstTop40Path, presentLowerThanTop10, bestMatchingCandidate, and missingReason.",
+        "",
+        "### Avoid/wrong-subsystem diagnostics summary",
+        "avoidHitDiagnostics and wrongSubsystemDiagnostics per case include relativePath, rank,",
+        "slot, subsystem, avoidType, matchedAvoidPath, contrastContext, score, and reasons.",
+        "",
+        "### Missing slots summary",
+        "slotCoverageBeforeTop10 and slotCoverageTop10 include presentSlots, missingRequiredSlots,",
+        "slotCounts, and requiredSlotCount. subystemCoverageTop10 shows primarySubsystem and purity.",
+        "",
         "## Cases",
         ""
     ];
@@ -652,6 +704,11 @@ function formatSummary(results) {
     const avgSlotCoverageTop10 = average(results.map((result) => result.rankingQuality.requiredSlotCount > 0 ? result.rankingQuality.slotCoverageTop10 / result.rankingQuality.requiredSlotCount : 1));
     const avgSubsystemPurityTop10 = average(results.map((result) => result.rankingQuality.subsystemPurityTop10));
     const avgSymbolEvidenceTop10 = average(results.map((result) => result.rankingQuality.symbolEvidenceTop10));
+    const avgStructuredRegexTop10 = average(results.map((result) => result.rankingQuality.structuredRegexEvidenceTop10));
+    const avgFallbackTop10 = average(results.map((result) => result.rankingQuality.fallbackEvidenceTop10));
+    const avgTreeSitterTop10 = average(results.map((result) => result.rankingQuality.treeSitterEvidenceTop10));
+    const semanticActive = results.filter((r) => r.result.stats.semanticRetrievalStatus === "active").length;
+    const semanticError = results.filter((r) => r.result.stats.semanticRetrievalStatus === "error_fallback").length;
     const groupSummary = withGroups.length > 0
         ? `, groupsPass=${groupsPassed}/${withGroups.length}`
         : "";
@@ -667,7 +724,7 @@ function formatSummary(results) {
         mrr: avgMrr,
         tokenReductionNorm: avgTokenReduction
     });
-    return `quality=${qualityScore.toFixed(3)}, labeled=${labeled.length}, pass=${passed.length}/${labeled.length}, hit@5=${hitAt5}/${results.length}, hit@10=${hitAt10}/${results.length}, hit@20=${hitAt20}/${results.length}, hit@40=${hitAt40}/${results.length}, allExpected@10=${allExpectedAt10}/${labeled.length}, allExpected@20=${allExpectedAt20}/${labeled.length}, avgExpectedCoverage@10=${avgExpectedCoverageAt10.toFixed(3)}, MRR=${avgMrr.toFixed(3)}, avg=${(avgDuration / 1000).toFixed(2)}s${groupSummary}, avoidHitCases=${avoidHitCases}/${avoidCases}, contrastAvoidCases=${contrastAvoidCases}/${avoidCases}, wrongSubsystemAvoidCases=${wrongSubsystemAvoidCases}/${avoidCases}, contaminationCases=${contaminationCases}/${results.length}, schemaIssueCases=${schemaIssueCases}/${results.length}, uniqueTop10FilesAvg=${avgUniqueTop10Files.toFixed(1)}, duplicateEvidenceTop10Avg=${avgDuplicateEvidenceTop10.toFixed(1)}, slotCoverageTop10Avg=${avgSlotCoverageTop10.toFixed(2)}, subsystemPurityTop10Avg=${avgSubsystemPurityTop10.toFixed(2)}, symbolEvidenceTop10Avg=${avgSymbolEvidenceTop10.toFixed(1)}`;
+    return `quality=${qualityScore.toFixed(3)}, labeled=${labeled.length}, pass=${passed.length}/${labeled.length}, hit@5=${hitAt5}/${results.length}, hit@10=${hitAt10}/${results.length}, hit@20=${hitAt20}/${results.length}, hit@40=${hitAt40}/${results.length}, allExpected@10=${allExpectedAt10}/${labeled.length}, allExpected@20=${allExpectedAt20}/${labeled.length}, avgExpectedCoverage@10=${avgExpectedCoverageAt10.toFixed(3)}, MRR=${avgMrr.toFixed(3)}, avg=${(avgDuration / 1000).toFixed(2)}s${groupSummary}, avoidHitCases=${avoidHitCases}/${avoidCases}, contrastAvoidCases=${contrastAvoidCases}/${avoidCases}, wrongSubsystemAvoidCases=${wrongSubsystemAvoidCases}/${avoidCases}, contaminationCases=${contaminationCases}/${results.length}, schemaIssueCases=${schemaIssueCases}/${results.length}, uniqueTop10FilesAvg=${avgUniqueTop10Files.toFixed(1)}, duplicateEvidenceTop10Avg=${avgDuplicateEvidenceTop10.toFixed(1)}, slotCoverageTop10Avg=${avgSlotCoverageTop10.toFixed(2)}, subsystemPurityTop10Avg=${avgSubsystemPurityTop10.toFixed(2)}, symbolEvidenceTop10Avg=${avgSymbolEvidenceTop10.toFixed(1)}, structuredRegexTop10Avg=${avgStructuredRegexTop10.toFixed(1)}, fallbackTop10Avg=${avgFallbackTop10.toFixed(1)}, treeSitterEvidenceTop10Avg=${avgTreeSitterTop10.toFixed(1)}`;
 }
 function formatTokenSavingsSummary(results) {
     const metrics = results.map((result) => result.result.tokenSavings).filter((item) => item !== undefined && item.status !== "unavailable");
@@ -786,4 +843,226 @@ function formatBytes(bytes) {
     }
     return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
 }
+async function dumpFailureAnalysisJson(results, reportPath) {
+    const jsonPath = reportPath.replace(/\.md$/, ".analysis.json");
+    const cases = results.map((r) => {
+        const top40Files = (0, fileGroups_1.aggregateEvidenceFiles)(r.result.units).slice(0, 40).map((g, idx) => ({
+            rank: idx + 1, relativePath: g.relativePath, score: g.score, role: g.role,
+            slot: g.slot ?? "unknown", subsystem: g.subsystem ?? "none",
+            isAvoid: (r.caseDef.avoidPaths ?? []).some((ap) => (0, evaluationMetrics_1.pathMatches)(g.relativePath, ap)),
+            isExpected: (r.caseDef.expectedPaths ?? []).some((ep) => (0, evaluationMetrics_1.pathMatches)(g.relativePath, ep)),
+            contrastContext: g.contrastContext
+        }));
+        const expectedRanks = {};
+        for (const ep of r.caseDef.expectedPaths ?? []) {
+            const foundIdx = top40Files.findIndex((f) => f.isExpected && (0, evaluationMetrics_1.pathMatches)(f.relativePath, ep));
+            expectedRanks[ep] = foundIdx >= 0 ? foundIdx + 1 : null;
+        }
+        // Score breakdown diagnostics
+        const top10 = top40Files.filter(f => f.rank <= 10);
+        const sbObj = (f) => ({
+            source: "eval_report_only", baseScore: f.score, semanticEvidence: null, parserEvidence: null,
+            symbolEvidence: null, graphEvidence: null, slotEvidence: f.slot || null,
+            subsystemEvidence: f.subsystem || null, avoidPenaltyEvidence: f.isAvoid ? "avoid" : null,
+            notes: [f.isExpected ? "expected" : "candidate"]
+        });
+        const diagItem = (f) => ({
+            rank: f.rank, relativePath: f.relativePath, score: f.score, slot: f.slot, subsystem: f.subsystem,
+            tier: "low", isExpected: f.isExpected, isAvoid: f.isAvoid,
+            channelHits: [], retrievalPasses: [], reasons: [],
+            scoreBreakdown: sbObj(f), diagnosticFlags: f.isExpected ? ["expected"] : []
+        });
+        const missingPaths = r.expectedMetrics.missingAt10 || [];
+        const expDiag = (r.caseDef.expectedPaths ?? []).map(ep => {
+            const presentIn10 = top10.some(t => t.isExpected && (0, evaluationMetrics_1.pathMatches)(t.relativePath, ep));
+            const top40Match = top40Files.filter(t => t.isExpected && (0, evaluationMetrics_1.pathMatches)(t.relativePath, ep));
+            const firstRank = top40Match.length > 0 ? top40Match[0].rank : null;
+            return { expectedPath: ep, presentInTop10: presentIn10, firstTop40Rank: firstRank,
+                presentLowerThanTop10: firstRank !== null && firstRank > 10, bestMatchingCandidate: null,
+                missingReason: presentIn10 ? null : "rank below 10" };
+        });
+        const avoidDiag = top10.filter(f => f.isAvoid).map(f => ({
+            relativePath: f.relativePath, rank: f.rank, slot: f.slot, subsystem: f.subsystem,
+            avoidType: "broad", matchedAvoidPath: null, contrastContext: f.contrastContext,
+            score: f.score, reasons: []
+        }));
+        const wsDiag = top10.map(f => ({
+            relativePath: f.relativePath, rank: f.rank, slot: f.slot, subsystem: f.subsystem,
+            avoidType: "broad", matchedAvoidPath: null, contrastContext: f.contrastContext,
+            score: f.score, reasons: []
+        }));
+        const scoreBreakdownDiagnostics = {
+            top10ScoreBreakdown: top10.map(diagItem),
+            top40ScoreDiagnostics: top40Files.map(diagItem),
+            expectedPathDiagnostics: expDiag,
+            avoidHitDiagnostics: avoidDiag,
+            wrongSubsystemDiagnostics: wsDiag,
+            slotCoverageBeforeTop10: { presentSlots: [], missingRequiredSlots: r.rankingQuality.missingRequiredSlotsTop10 || [], slotCounts: {}, requiredSlotCount: r.rankingQuality.requiredSlotCount || 0 },
+            slotCoverageTop10: { presentSlots: r.rankingQuality.foundRequiredSlotsTop10 || [], missingRequiredSlots: r.rankingQuality.missingRequiredSlotsTop10 || [], slotCounts: {}, requiredSlotCount: r.rankingQuality.requiredSlotCount || 0 },
+            subsystemCoverageTop10: { primarySubsystem: r.rankingQuality.primarySubsystemTop10 || "unknown", purity: r.rankingQuality.subsystemPurityTop10 || 0 },
+            channelHitSummaryTop10: {},
+            rejectedHighScoreNoise: [],
+            missingExpectedPathReasons: missingPaths.map((p) => ({ path: p, reason: "below rank 10 threshold" }))
+        };
+        return {
+            caseId: `${r.suiteName}/${r.caseDef.id}`, workspace: r.workspaceName, task: r.caseDef.task.slice(0, 120),
+            pass: r.pass, hitAt10: r.hitAt10, allGroupsPass: r.allGroupsPass,
+            expectedPaths: r.caseDef.expectedPaths ?? [], avoidPaths: r.caseDef.avoidPaths ?? [],
+            expectedMode: r.caseDef.expectedMode ?? "any",
+            coverageAt10: r.expectedMetrics.coverageAt10, coverageAt20: r.expectedMetrics.coverageAt20,
+            coverageAt40: r.expectedMetrics.coverageAt40, expectedCount: r.expectedMetrics.expectedCount,
+            firstFileHitRank: r.firstFileHitRank ?? null,
+            missingAt10: r.expectedMetrics.missingAt10, missingAt20: r.expectedMetrics.missingAt20,
+            hardAvoidHitsTop10: r.hardAvoidHitsTop10, wrongSubsystemAvoidHitsTop10: r.wrongSubsystemAvoidHitsTop10,
+            groupResults: r.groupResults?.map(g => ({ name: g.name, mode: g.mode, pass: g.pass, missingAtPassK: g.missingAtPassK, expectedCount: g.expectedCount, coverageAtPassK: g.coverageAtPassK })),
+            rankingQuality: {
+                uniqueTop10Files: r.rankingQuality.uniqueTop10Files, duplicateEvidenceTop10: r.rankingQuality.duplicateEvidenceTop10,
+                slotCoverageTop10: r.rankingQuality.slotCoverageTop10, requiredSlotCount: r.rankingQuality.requiredSlotCount,
+                missingRequiredSlotsTop10: r.rankingQuality.missingRequiredSlotsTop10, foundRequiredSlotsTop10: r.rankingQuality.foundRequiredSlotsTop10,
+                primarySubsystemTop10: r.rankingQuality.primarySubsystemTop10, subsystemPurityTop10: r.rankingQuality.subsystemPurityTop10
+            },
+            expectedRanks, top40Files, scoreBreakdownDiagnostics,
+            // ── 5F-P1-R1 Counterfactual Candidate Evidence Export ──
+            // Iterate aggregate top40Files (same ordering as top40ScoreDiagnostics).
+            // Path/rank/score from the aggregate record; evidence from the
+            // highest-ranked matching runtime unit.
+            counterfactualReplayInput: (() => {
+                const candidates = [];
+                for (let i = 0; i < top40Files.length; i++) {
+                    const tf = top40Files[i];
+                    const tfNorm = (tf.relativePath || "").replace(/\\/g, "/").toLowerCase();
+                    const match = r.result.units.find(u => ((u.relativePath || "").replace(/\\/g, "/").toLowerCase()) === tfNorm);
+                    if (!match) {
+                        throw new Error(`5F-P1-R1: no runtime unit for aggregate top40 path "${tf.relativePath}" ` +
+                            `in case ${r.suiteName}/${r.caseDef.id}`);
+                    }
+                    candidates.push({
+                        relativePath: tf.relativePath,
+                        originalRank: tf.rank,
+                        score: tf.score,
+                        tier: match.tier || "low",
+                        slot: match.slot || null,
+                        subsystem: match.subsystem || null,
+                        symbols: [...(match.symbols || [])],
+                        symbolMatches: [...(match.symbolMatches || [])],
+                        matchedTerms: [...(match.matchedTerms || [])],
+                        imports: [...(match.imports || [])],
+                        extractorKind: match.extractorKind || null,
+                        channelHits: (match.channelHits || []).map(h => ({ channel: h.channel, score: h.score, rank: h.rank != null ? h.rank : null, reason: h.reason || null })),
+                        retrievalPasses: [...(match.retrievalPasses || [])],
+                        confidence: match.confidence,
+                        taskAnchorCoverage: match.taskAnchorCoverage || null,
+                        reasons: [...(match.reasons || [])]
+                    });
+                }
+                return { schemaVersion: 1, source: "runtime_evidence_units", executionMode: "report_only_disabled", candidates };
+            })()
+        };
+    });
+    await fs.writeFile(jsonPath, JSON.stringify(cases, null, 2), "utf8");
+    console.log(`\nFailure analysis JSON: ${jsonPath}`);
+}
+function formatSemanticStatusSummary(results) {
+    const disabled = results.filter(r => r.result.stats.semanticRetrievalStatus === "disabled").length;
+    const skipped = results.filter(r => r.result.stats.semanticRetrievalStatus === "skipped_no_key").length;
+    const active = results.filter(r => r.result.stats.semanticRetrievalStatus === "active").length;
+    const error = results.filter(r => r.result.stats.semanticRetrievalStatus === "error_fallback").length;
+    const semEvidence = results.reduce((s, r) => s + r.result.units.filter(u => u.channelHits?.some(h => h.channel === "semantic_vector")).length, 0);
+    return `semanticStatus: disabled=${disabled}, skipped=${skipped}, active=${active}/${results.length}, error=${error}, semEvidence=${semEvidence}`;
+}
+function formatTaskAwareGuardSummary(results) {
+    const allUnits = results.flatMap(r => r.result.units);
+    const unitsWithFlag = allUnits.filter(u => u.taskAvoidGuardFlag !== undefined);
+    const demoted = unitsWithFlag.filter(u => u.taskAvoidGuardFlag === "demoted").length;
+    const promoted = unitsWithFlag.filter(u => u.taskAvoidGuardFlag === "promoted").length;
+    const blockedSlot = unitsWithFlag.filter(u => u.taskAvoidGuardFlag === "blocked_slot").length;
+    const blockedExpected = unitsWithFlag.filter(u => u.taskAvoidGuardFlag === "blocked_expected").length;
+    const noSafeReplacement = unitsWithFlag.filter(u => u.taskAvoidGuardFlag === "blocked_no_replacement").length;
+    const candidates = demoted + promoted + noSafeReplacement;
+    const scanned = candidates + blockedSlot + blockedExpected;
+    const consistency = (scanned === candidates + blockedSlot + blockedExpected) && (candidates === demoted + promoted + noSafeReplacement);
+    return `taskAwareAvoidGuard: scanned=${scanned}, candidates=${candidates}, demoted=${demoted}, promoted=${promoted}, blockedSlot=${blockedSlot}, blockedExpected=${blockedExpected}, noSafeReplacement=${noSafeReplacement}\ntaskAwareAvoidGuardConsistency: ${consistency ? "PASS" : "FAIL"}`;
+}
+async function runGoldenEval(goldenManifestPath, goldenReportPath) {
+    const manifestPath = path.resolve(goldenManifestPath);
+    const manifest = await readJson(manifestPath);
+    const workspacePath = typeof manifest.workspace === "string" && manifest.workspace.trim()
+        ? path.resolve(manifest.workspace)
+        : "";
+    const issues = (0, goldenTask_1.validateGoldenManifest)(manifest, workspacePath || undefined);
+    if (issues.length > 0) {
+        console.error(`Golden manifest validation failed with ${issues.length} issue(s):`);
+        for (const issue of issues)
+            console.error(`  - ${issue}`);
+        process.exitCode = 1;
+        return;
+    }
+    const config = (0, defaultConfig_1.mergeConfig)(await (0, loadConfigFromPath_1.loadConfigFromPath)(workspacePath), {});
+    console.log(`\nGolden Eval: ${manifest.name}`);
+    console.log(`Workspace: ${workspacePath}`);
+    console.log(`Tasks: ${manifest.tasks.length}`);
+    const perTaskResults = [];
+    const flowResults = [];
+    const taskGuardReasons = {};
+    for (const goldenTask of manifest.tasks) {
+        const taskStart = Date.now();
+        console.log(`- ${goldenTask.id}: ${goldenTask.task}`);
+        (0, fileScanner_1.clearGuardReasons)();
+        const result = await (0, runHeadless_1.runFlowSeekerHeadless)(workspacePath, goldenTask.task, config);
+        const gr = (0, fileScanner_1.getGuardReasons)();
+        if (gr.size > 0) {
+            const plain = {};
+            for (const [fp, cat] of gr)
+                plain[fp] = cat;
+            taskGuardReasons[String(perTaskResults.length)] = plain;
+        }
+        flowResults.push(result);
+        const metrics = (0, accuracyV2_1.computeAccuracyV2)(result, goldenTask);
+        perTaskResults.push(metrics);
+        const dur = ((Date.now() - taskStart) / 1000).toFixed(2);
+        console.log(`  editR@1=${metrics.editRecall1} editR@3=${metrics.editRecall3} editR@10=${metrics.editRecall10} first3Hit=${metrics.first3Hit} mustNotViolation=${metrics.mustNotEditViolation} ${dur}s`);
+    }
+    // Compute failure analysis for each task
+    const failureAnalysis = manifest.tasks.map((goldenTask, i) => (0, accuracyV2_1.computeFailureAnalysis)(flowResults[i], goldenTask, perTaskResults[i]));
+    // Compute semantic contribution
+    const semPerTask = flowResults.map((r) => (0, accuracyV2_1.computeSemanticContribution)(r));
+    const semanticContribution = (0, accuracyV2_1.aggregateSemanticContribution)(semPerTask, flowResults);
+    // Compute role classifier report
+    const roleConfusionMatrix = (0, accuracyV2_1.computeRoleConfusionMatrix)(flowResults, manifest.tasks, "balanced");
+    const roleClassifierReport = (0, accuracyV2_1.computeRoleClassifierReport)(flowResults, manifest.tasks);
+    const report = (0, accuracyV2_1.aggregateAccuracyV2)(perTaskResults, failureAnalysis, semanticContribution, roleConfusionMatrix, roleClassifierReport, manifest.name);
+    // Write JSON report
+    const reportPath = path.resolve(goldenReportPath ?? path.join(".flowseeker", "reports", `golden-eval-${slugify(manifest.name)}.json`));
+    await fs.mkdir(path.dirname(reportPath), { recursive: true });
+    await fs.writeFile(reportPath, JSON.stringify({ ...report, guardReasons: taskGuardReasons }, null, 2), "utf8");
+    console.log(`\nGolden JSON: ${reportPath}`);
+    // Write markdown report
+    const mdPath = reportPath.replace(/\.json$/, ".md");
+    const mdLines = [
+        "# FlowSeeker Accuracy V2 Report",
+        "",
+        `Generated: ${new Date().toISOString()}`,
+        `Manifest: ${manifest.name}`,
+        `Workspace: ${workspacePath}`,
+        `Tasks: ${report.taskCount}`,
+        "",
+        ...(0, accuracyV2_1.renderAccuracyV2Markdown)(report),
+    ];
+    await fs.writeFile(mdPath, mdLines.join("\n"), "utf8");
+    console.log(`Golden Markdown: ${mdPath}`);
+    // Print summary
+    console.log(`\nAccuracy V2 Summary:`);
+    console.log(`  edit_recall@1: ${(report.editRecall1 * 100).toFixed(1)}%`);
+    console.log(`  edit_recall@3: ${(report.editRecall3 * 100).toFixed(1)}%`);
+    console.log(`  edit_recall@5: ${(report.editRecall5 * 100).toFixed(1)}%`);
+    console.log(`  edit_recall@10: ${(report.editRecall10 * 100).toFixed(1)}%`);
+    console.log(`  primary_edit_precision@3: ${(report.primaryEditPrecision3 * 100).toFixed(1)}%`);
+    console.log(`  read_context_recall@10: ${(report.readContextRecall10 * 100).toFixed(1)}%`);
+    console.log(`  test_recall@10: ${(report.testRecall10 * 100).toFixed(1)}%`);
+    console.log(`  must_not_edit_violation_rate: ${(report.mustNotEditViolationRate * 100).toFixed(1)}%`);
+    console.log(`  noise_promoted_as_edit_rate: ${(report.noisePromotedAsEditRate * 100).toFixed(1)}%`);
+    console.log(`  required_slot_coverage: ${(report.requiredSlotCoverage * 100).toFixed(1)}%`);
+    console.log(`  missing_slot_rate: ${(report.missingSlotRate * 100).toFixed(1)}%`);
+    console.log(`  first3_hit_rate: ${(report.first3HitRate * 100).toFixed(1)}%`);
+}
 //# sourceMappingURL=runEvaluation.js.map

package/dist/config/defaultConfig.js

@@ -19,7 +19,17 @@ exports.defaultConfig = {
         enableDependencyExpansion: true,
         maxSeedCandidateFiles: 250,
         maxDependencyExpansionFiles: 400,
-        maxIndexedCandidateFiles: 800
+        maxIndexedCandidateFiles: 800,
+        semanticEnabled: false,
+        semanticProvider: "openaiCompatible",
+        semanticModel: "text-embedding-3-small",
+        semanticBaseUrl: "",
+        semanticApiKeyEnv: "FLOWSEEKER_EMBEDDING_API_KEY",
+        semanticCache: true,
+        semanticMaxFilesPerRun: 800,
+        semanticTimeoutMs: 15000,
+        semanticMinSimilarity: 0.5,
+        semanticEvidenceMaxRank: 20
     },
     ai: {
         enabled: false,

package/dist/config/env.js

@@ -0,0 +1,118 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadEnvFiles = loadEnvFiles;
+exports.applyEnvConfigOverrides = applyEnvConfigOverrides;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const loadedRoots = new Set();
+function loadEnvFiles(startPath = process.cwd()) {
+    const startDir = resolveStartDir(startPath);
+    const roots = collectAncestorDirs(startDir).reverse();
+    for (const root of roots) {
+        if (loadedRoots.has(root)) {
+            continue;
+        }
+        loadedRoots.add(root);
+        // .env always overrides stale process env (the file is the source of truth)
+        loadEnvFile(path.join(root, ".env"));
+        // .env.local takes highest precedence
+        loadEnvFile(path.join(root, ".env.local"));
+    }
+}
+function applyEnvConfigOverrides(config) {
+    // Only API key override — non-secret semantic settings belong exclusively in .flowseeker/config.json
+    const apiKeyOverride = process.env[config.index.semanticApiKeyEnv || ""];
+    if (apiKeyOverride) {
+        // Key is available; provider can use it
+    }
+    return config;
+}
+function loadEnvFile(filePath) {
+    if (!fs.existsSync(filePath)) {
+        return;
+    }
+    const text = fs.readFileSync(filePath, "utf8");
+    for (const line of text.split(/\r?\n/)) {
+        const parsed = parseEnvLine(line);
+        if (!parsed)
+            continue;
+        // .env and .env.local always override stale process env
+        process.env[parsed.key] = parsed.value;
+    }
+}
+function parseEnvLine(line) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) {
+        return undefined;
+    }
+    const separator = trimmed.indexOf("=");
+    if (separator <= 0) {
+        return undefined;
+    }
+    const key = trimmed.slice(0, separator).trim();
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) {
+        return undefined;
+    }
+    let value = trimmed.slice(separator + 1).trim();
+    if ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'"))) {
+        value = value.slice(1, -1);
+    }
+    return { key, value };
+}
+function resolveStartDir(startPath) {
+    try {
+        const resolved = path.resolve(startPath);
+        return fs.existsSync(resolved) && fs.statSync(resolved).isFile()
+            ? path.dirname(resolved)
+            : resolved;
+    }
+    catch {
+        return process.cwd();
+    }
+}
+function collectAncestorDirs(startDir) {
+    const dirs = [];
+    let current = path.resolve(startDir);
+    while (true) {
+        dirs.push(current);
+        const parent = path.dirname(current);
+        if (parent === current) {
+            return dirs;
+        }
+        current = parent;
+    }
+}
+//# sourceMappingURL=env.js.map

package/dist/config/loadConfig.js

@@ -36,7 +36,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadConfig = loadConfig;
 const vscode = __importStar(require("vscode"));
 const defaultConfig_1 = require("./defaultConfig");
+const env_1 = require("./env");
 async function loadConfig() {
+    (0, env_1.loadEnvFiles)(firstWorkspaceRoot());
     const workspaceConfig = await readWorkspaceConfig();
     const vscodeConfig = vscode.workspace.getConfiguration("flowseeker");
     const merged = (0, defaultConfig_1.mergeConfig)(defaultConfig_1.defaultConfig, workspaceConfig);
@@ -44,12 +46,23 @@ async function loadConfig() {
     merged.project.exclude = readArray(vscodeConfig, "project.exclude", merged.project.exclude);
     merged.project.maxFileSizeKb = vscodeConfig.get("project.maxFileSizeKb", merged.project.maxFileSizeKb);
     merged.index.enableWorkspaceIndex = vscodeConfig.get("index.enableWorkspaceIndex", merged.index.enableWorkspaceIndex);
+    merged.index.enableTreeSitter = vscodeConfig.get("index.enableTreeSitter", merged.index.enableTreeSitter);
     merged.index.chunkSizeLines = vscodeConfig.get("index.chunkSizeLines", merged.index.chunkSizeLines);
     merged.index.chunkOverlapLines = vscodeConfig.get("index.chunkOverlapLines", merged.index.chunkOverlapLines);
     merged.index.enableDependencyExpansion = vscodeConfig.get("index.enableDependencyExpansion", merged.index.enableDependencyExpansion);
     merged.index.maxSeedCandidateFiles = vscodeConfig.get("index.maxSeedCandidateFiles", merged.index.maxSeedCandidateFiles);
     merged.index.maxDependencyExpansionFiles = vscodeConfig.get("index.maxDependencyExpansionFiles", merged.index.maxDependencyExpansionFiles);
     merged.index.maxIndexedCandidateFiles = vscodeConfig.get("index.maxIndexedCandidateFiles", merged.index.maxIndexedCandidateFiles);
+    merged.index.semanticEnabled = vscodeConfig.get("index.semanticEnabled", merged.index.semanticEnabled);
+    merged.index.semanticProvider = vscodeConfig.get("index.semanticProvider", merged.index.semanticProvider);
+    merged.index.semanticModel = vscodeConfig.get("index.semanticModel", merged.index.semanticModel);
+    merged.index.semanticBaseUrl = vscodeConfig.get("index.semanticBaseUrl", merged.index.semanticBaseUrl);
+    merged.index.semanticApiKeyEnv = vscodeConfig.get("index.semanticApiKeyEnv", merged.index.semanticApiKeyEnv);
+    merged.index.semanticCache = vscodeConfig.get("index.semanticCache", merged.index.semanticCache);
+    merged.index.semanticMaxFilesPerRun = vscodeConfig.get("index.semanticMaxFilesPerRun", merged.index.semanticMaxFilesPerRun);
+    merged.index.semanticTimeoutMs = vscodeConfig.get("index.semanticTimeoutMs", merged.index.semanticTimeoutMs);
+    merged.index.semanticMinSimilarity = vscodeConfig.get("index.semanticMinSimilarity", merged.index.semanticMinSimilarity);
+    merged.index.semanticEvidenceMaxRank = vscodeConfig.get("index.semanticEvidenceMaxRank", merged.index.semanticEvidenceMaxRank);
     merged.ai.enabled = vscodeConfig.get("ai.enabled", merged.ai.enabled);
     merged.ai.outputLanguage = vscodeConfig.get("ai.outputLanguage", merged.ai.outputLanguage);
     merged.pipeline.maxCandidates = vscodeConfig.get("pipeline.maxCandidates", merged.pipeline.maxCandidates);
@@ -60,7 +73,11 @@ async function loadConfig() {
     merged.pipeline.maxTotalBytesMb = vscodeConfig.get("pipeline.maxTotalBytesMb", merged.pipeline.maxTotalBytesMb);
     merged.pipeline.maxScanMs = vscodeConfig.get("pipeline.maxScanMs", merged.pipeline.maxScanMs);
     merged.pipeline.maxRawEvidence = vscodeConfig.get("pipeline.maxRawEvidence", merged.pipeline.maxRawEvidence);
-    return merged;
+    return (0, env_1.applyEnvConfigOverrides)(merged);
+}
+function firstWorkspaceRoot() {
+    const folder = vscode.workspace.workspaceFolders?.[0];
+    return folder?.uri.scheme === "file" ? folder.uri.fsPath : undefined;
 }
 async function readWorkspaceConfig() {
     const folders = vscode.workspace.workspaceFolders;

package/dist/config/loadConfigFromPath.js

@@ -37,10 +37,12 @@ exports.loadConfigFromPath = loadConfigFromPath;
 const fs = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
 const defaultConfig_1 = require("./defaultConfig");
+const env_1 = require("./env");
 async function loadConfigFromPath(workspacePath) {
+    (0, env_1.loadEnvFiles)(workspacePath);
     const configPath = path.join(workspacePath, ".flowseeker", "config.json");
     const workspaceConfig = await readJson(configPath);
-    return (0, defaultConfig_1.mergeConfig)(defaultConfig_1.defaultConfig, workspaceConfig ?? {});
+    return (0, env_1.applyEnvConfigOverrides)((0, defaultConfig_1.mergeConfig)(defaultConfig_1.defaultConfig, workspaceConfig ?? {}));
 }
 async function readJson(filePath) {
     try {