flowseeker 0.1.7

package/dist/commands/runEvaluationSuite.js

@@ -0,0 +1,323 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEvaluationSuite = runEvaluationSuite;
+const path = __importStar(require("path"));
+const vscode = __importStar(require("vscode"));
+const evaluationMetrics_1 = require("../pipeline/evaluationMetrics");
+const fileGroups_1 = require("../pipeline/fileGroups");
+const runPipeline_1 = require("../pipeline/runPipeline");
+const tokenSavings_1 = require("../pipeline/tokenSavings");
+const logger_1 = require("../utils/logger");
+async function runEvaluationSuite() {
+    const folders = vscode.workspace.workspaceFolders;
+    if (!folders || folders.length === 0) {
+        (0, logger_1.logInfo)("Run Evaluation Suite stopped because no workspace folder is open.");
+        vscode.window.showWarningMessage("Open a workspace folder before running FlowSeeker evaluation.");
+        return;
+    }
+    try {
+        const suiteFiles = await findSuiteFiles();
+        if (suiteFiles.length === 0) {
+            vscode.window.showWarningMessage("No FlowSeeker eval suites found. Add JSON suites under .flowseeker/evals/.");
+            (0, logger_1.logInfo)("No eval suites found under .flowseeker/evals/.");
+            return;
+        }
+        const suites = await readSuites(suiteFiles);
+        const totalCases = suites.reduce((sum, suite) => sum + suite.cases.length, 0);
+        if (totalCases === 0) {
+            vscode.window.showWarningMessage("FlowSeeker eval suites were found, but none contain cases.");
+            return;
+        }
+        (0, logger_1.logInfo)(`Starting evaluation. Suites: ${suites.length}; cases: ${totalCases}; workspace: ${folders.map((folder) => folder.uri.fsPath).join("; ")}`);
+        const caseResults = await vscode.window.withProgress({
+            location: vscode.ProgressLocation.Notification,
+            title: "FlowSeeker is running eval suites",
+            cancellable: true
+        }, async (progress, token) => {
+            const results = [];
+            let completed = 0;
+            for (const suite of suites) {
+                for (const caseDef of suite.cases) {
+                    if (token.isCancellationRequested) {
+                        (0, logger_1.logInfo)("Evaluation cancelled by user.");
+                        return results;
+                    }
+                    progress.report({
+                        message: `${suite.name}: ${caseDef.id}`,
+                        increment: totalCases > 0 ? 100 / totalCases : undefined
+                    });
+                    const startedAt = Date.now();
+                    const result = await (0, runPipeline_1.runFlowSeeker)(caseDef.task, { token });
+                    const durationMs = Date.now() - startedAt;
+                    const evaluated = evaluateCase(suite.name, caseDef, result, durationMs);
+                    results.push(evaluated);
+                    completed += 1;
+                    (0, logger_1.logInfo)(formatCaseLog(evaluated, completed, totalCases));
+                }
+            }
+            return results;
+        });
+        if (caseResults.length === 0) {
+            vscode.window.showInformationMessage("FlowSeeker evaluation produced no results.");
+            return;
+        }
+        const reportUri = await writeReport(folders[0].uri, suites, caseResults);
+        (0, logger_1.logInfo)(`Evaluation complete. Report: ${reportUri.fsPath}`);
+        vscode.window.showInformationMessage(`FlowSeeker evaluation complete. Report saved: ${path.basename(reportUri.fsPath)}`);
+        await vscode.commands.executeCommand("vscode.open", reportUri);
+    }
+    catch (error) {
+        (0, logger_1.logError)("Run Evaluation Suite failed.", error);
+        const message = error instanceof Error ? error.message : String(error);
+        vscode.window.showErrorMessage(`FlowSeeker evaluation failed: ${message}`);
+    }
+}
+async function findSuiteFiles() {
+    const found = await vscode.workspace.findFiles(".flowseeker/evals/**/*.json", "**/node_modules/**", 100);
+    const direct = await vscode.workspace.findFiles(".flowseeker/eval.json", "**/node_modules/**", 1);
+    const byKey = new Map();
+    for (const uri of direct.concat(found)) {
+        byKey.set(uri.toString(), uri);
+    }
+    return Array.from(byKey.values()).sort((left, right) => left.fsPath.localeCompare(right.fsPath));
+}
+async function readSuites(uris) {
+    const suites = [];
+    for (const uri of uris) {
+        const bytes = await vscode.workspace.fs.readFile(uri);
+        const parsed = JSON.parse(new TextDecoder("utf-8").decode(bytes));
+        if (!parsed.name || !Array.isArray(parsed.cases)) {
+            throw new Error(`Invalid eval suite: ${uri.fsPath}`);
+        }
+        const cases = parsed.cases.map((item, index) => {
+            if (!item?.id || !item?.task) {
+                throw new Error(`Invalid eval case at ${uri.fsPath}#${index + 1}: id and task are required.`);
+            }
+            return item;
+        });
+        suites.push({
+            name: parsed.name,
+            description: parsed.description,
+            cases
+        });
+    }
+    return suites;
+}
+function evaluateCase(suiteName, caseDef, result, durationMs) {
+    const passAtK = caseDef.passAtK ?? 10;
+    const expectedMetrics = (0, evaluationMetrics_1.computeExpectedPathMetrics)(result.units, caseDef.expectedPaths ?? [], passAtK, caseDef.expectedMode ?? "any");
+    const hardAvoidHitsTop10 = findAvoidHits(result.units.slice(0, 10), caseDef.avoidPaths ?? [], "hard");
+    const contrastAvoidHitsTop10 = findAvoidHits(result.units.slice(0, 10), caseDef.avoidPaths ?? [], "contrast");
+    return {
+        suiteName,
+        caseDef,
+        result,
+        durationMs,
+        firstHitRank: expectedMetrics.firstUnitHitRank,
+        firstFileHitRank: expectedMetrics.firstFileHitRank,
+        expectedMetrics,
+        hitAt5: expectedMetrics.coverageAt5 > 0,
+        hitAt10: expectedMetrics.coverageAt10 > 0,
+        hitAt20: expectedMetrics.coverageAt20 > 0,
+        hitAt40: expectedMetrics.coverageAt40 > 0,
+        pass: expectedMetrics.pass,
+        avoidHitsTop10: uniqueStrings([...hardAvoidHitsTop10, ...contrastAvoidHitsTop10]),
+        hardAvoidHitsTop10,
+        contrastAvoidHitsTop10,
+        rankingQuality: (0, evaluationMetrics_1.computeRankingQualityMetrics)(result)
+    };
+}
+function findAvoidHits(units, avoidPaths, mode = "hard") {
+    if (avoidPaths.length === 0) {
+        return [];
+    }
+    return uniqueStrings(units
+        .filter((unit) => avoidPaths.some((avoidPath) => (0, evaluationMetrics_1.pathMatches)(unit.relativePath, avoidPath)))
+        .filter((unit) => mode === "contrast" ? unit.contrastContext : !unit.contrastContext)
+        .map((unit) => unit.relativePath));
+}
+async function writeReport(rootUri, suites, results) {
+    const reportsDir = vscode.Uri.joinPath(rootUri, ".flowseeker", "reports");
+    await vscode.workspace.fs.createDirectory(reportsDir);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const reportUri = vscode.Uri.joinPath(reportsDir, `eval-${timestamp}.md`);
+    const report = renderReport(suites, results);
+    await vscode.workspace.fs.writeFile(reportUri, new TextEncoder().encode(report));
+    return reportUri;
+}
+function renderReport(suites, results) {
+    const labeled = results.filter((result) => result.pass !== "unlabeled");
+    const passed = labeled.filter((result) => result.pass === true);
+    const hitAt5 = count(results, (result) => result.hitAt5);
+    const hitAt10 = count(results, (result) => result.hitAt10);
+    const hitAt20 = count(results, (result) => result.hitAt20);
+    const hitAt40 = count(results, (result) => result.hitAt40);
+    const avgDurationMs = average(results.map((result) => result.durationMs));
+    const avgMrr = average(labeled.map((result) => (result.firstFileHitRank ? 1 / result.firstFileHitRank : 0)));
+    const lines = [
+        "# FlowSeeker Evaluation Report",
+        "",
+        `Generated: ${new Date().toISOString()}`,
+        `Suites: ${suites.length}`,
+        `Cases: ${results.length}`,
+        `Labeled cases: ${labeled.length}`,
+        `Pass@caseK: ${passed.length}/${labeled.length}`,
+        `Hit@5: ${hitAt5}/${results.length}`,
+        `Hit@10: ${hitAt10}/${results.length}`,
+        `Hit@20: ${hitAt20}/${results.length}`,
+        `Hit@40: ${hitAt40}/${results.length}`,
+        `MRR: ${avgMrr.toFixed(3)}`,
+        `Average duration: ${(avgDurationMs / 1000).toFixed(2)}s`,
+        `Token savings: ${formatTokenSavingsSummary(results)}`,
+        "",
+        "## Suites",
+        "",
+        ...suites.map((suite) => `- ${suite.name}${suite.description ? ` - ${suite.description}` : ""}`),
+        "",
+        "## Cases",
+        ""
+    ];
+    for (const result of results) {
+        lines.push(...renderCase(result));
+    }
+    return `${lines.join("\n")}\n`;
+}
+function renderCase(result) {
+    const status = result.pass === "unlabeled" ? "UNLABELED" : result.pass ? "PASS" : "FAIL";
+    const firstHit = result.firstHitRank ? String(result.firstHitRank) : "none";
+    const firstFileHit = result.firstFileHitRank ? String(result.firstFileHitRank) : "none";
+    const stats = result.result.stats;
+    const topFiles = (0, fileGroups_1.aggregateEvidenceFiles)(result.result.units).slice(0, 15);
+    return [
+        `### ${status} - ${result.suiteName}/${result.caseDef.id}`,
+        "",
+        `Task: ${result.caseDef.task}`,
+        ...(result.caseDef.notes ? [`Notes: ${result.caseDef.notes}`] : []),
+        `Intent: ${result.result.profile.intent}`,
+        `Keywords: ${result.result.profile.keywords.join(", ") || "none"}`,
+        `Strategies: ${result.result.profile.strategies.join(", ") || "none"}`,
+        `Expected paths: ${(result.caseDef.expectedPaths ?? []).join(", ") || "none"}`,
+        `Expected mode: ${result.caseDef.expectedMode ?? "any"}`,
+        `Avoid paths: ${(result.caseDef.avoidPaths ?? []).join(", ") || "none"}`,
+        `First expected unit rank: ${firstHit}`,
+        `First expected file rank: ${firstFileHit}`,
+        `Coverage@5/10/20/40: ${result.expectedMetrics.coverageAt5}/${result.expectedMetrics.expectedCount}, ${result.expectedMetrics.coverageAt10}/${result.expectedMetrics.expectedCount}, ${result.expectedMetrics.coverageAt20}/${result.expectedMetrics.expectedCount}, ${result.expectedMetrics.coverageAt40}/${result.expectedMetrics.expectedCount}`,
+        `Missing expected at 10: ${result.expectedMetrics.missingAt10.join(", ") || "none"}`,
+        `Missing expected at 20: ${result.expectedMetrics.missingAt20.join(", ") || "none"}`,
+        `Avoid hits in top 10: ${result.hardAvoidHitsTop10.join(", ") || "none"}`,
+        `Contrast avoid hits in top 10: ${result.contrastAvoidHitsTop10.join(", ") || "none"}`,
+        `Token savings: ${formatCaseTokenSavings(result)}`,
+        `Context coverage: ${formatContextCoverage(result.result)}`,
+        `Ranking quality: uniqueTop10Files=${result.rankingQuality.uniqueTop10Files}, duplicateEvidenceTop10=${result.rankingQuality.duplicateEvidenceTop10}, slotCoverageTop10=${result.rankingQuality.slotCoverageTop10}/${result.rankingQuality.requiredSlotCount}${result.rankingQuality.missingRequiredSlotsTop10.length ? `, missingSlots=${result.rankingQuality.missingRequiredSlotsTop10.join("|")}` : ""}, contrastContextTop10=${result.rankingQuality.contrastContextTop10}, primarySubsystem=${result.rankingQuality.primarySubsystemTop10 ?? "none"}, subsystemPurityTop10=${result.rankingQuality.subsystemPurityTop10.toFixed(2)}, symbolEvidenceTop10=${result.rankingQuality.symbolEvidenceTop10}`,
+        `Duration: ${(result.durationMs / 1000).toFixed(2)}s`,
+        `Scan: discovered=${stats.discoveredFiles}, scanned=${stats.scannedFiles}, skipped=${stats.skippedFiles}, bytes=${formatBytes(stats.scannedBytes)}, rawEvidence=${stats.rawEvidenceCount}, ranked=${stats.rankedEvidenceCount}, capped=${stats.capped}, stoppedEarly=${stats.stoppedEarly}${stats.stopReason ? ` (${stats.stopReason})` : ""}${formatIndexStats(stats)}`,
+        "",
+        "Top files:",
+        "",
+        ...topFiles.map((group, index) => `${index + 1}. ${group.relativePath} [${group.role}/${group.slot}] score=${group.score.toFixed(1)} evidence=${group.unitCount} dup=${group.duplicateUnitCount} contrast=${group.contrastContext} subsystem=${group.subsystem ?? "none"} kinds=${group.kinds.join(",")}`),
+        "",
+        "Top evidence:",
+        "",
+        ...result.result.units.slice(0, 15).map((unit, index) => {
+            const range = unit.range ? `:${unit.range.startLine}-${unit.range.endLine}` : "";
+            const passes = unit.retrievalPasses.length > 0 ? unit.retrievalPasses.join("+") : "none";
+            const meta = [unit.isDuplicate ? "duplicate" : "", unit.contrastContext ? "contrast" : "", unit.frameworkEdgeReason ? "framework" : ""].filter(Boolean).join(",");
+            return `${index + 1}. ${unit.relativePath}${range} [${unit.role}/${unit.kind}/${unit.slot ?? "unknown"}${meta ? `/${meta}` : ""}] score=${unit.score.toFixed(1)} passes=${passes}`;
+        }),
+        ""
+    ];
+}
+function formatCaseLog(result, completed, total) {
+    const status = result.pass === "unlabeled" ? "UNLABELED" : result.pass ? "PASS" : "FAIL";
+    const firstHit = result.firstFileHitRank ? `file rank ${result.firstFileHitRank}` : "no expected file hit";
+    return `Eval ${completed}/${total} ${status} ${result.suiteName}/${result.caseDef.id}: ${firstHit}; ${(0, tokenSavings_1.formatTokenSavingsLine)(result.result.tokenSavings)}; ${formatContextCoverage(result.result)}; duration ${(result.durationMs / 1000).toFixed(2)}s; raw evidence ${result.result.stats.rawEvidenceCount}.`;
+}
+function formatTokenSavingsSummary(results) {
+    const metrics = results.map((result) => result.result.tokenSavings).filter((item) => item !== undefined && item.status !== "unavailable");
+    if (metrics.length === 0) {
+        return "unavailable";
+    }
+    const avgReduction = average(metrics.map((item) => item.reductionPercent));
+    const avgPacketTokens = average(metrics.map((item) => item.solvePacketTokensEstimate));
+    const avgWorkspaceTokens = average(metrics.map((item) => item.workspaceTokensEstimate));
+    const partial = metrics.filter((item) => item.status === "partial").length;
+    return `avg=${(0, tokenSavings_1.formatReductionPercent)(avgReduction)}, packetAvg=${(0, tokenSavings_1.formatTokenCount)(avgPacketTokens)}, workspaceAvg=${(0, tokenSavings_1.formatTokenCount)(avgWorkspaceTokens)}, partial=${partial}/${metrics.length}`;
+}
+function formatCaseTokenSavings(result) {
+    const metrics = result.result.tokenSavings;
+    if (!metrics || metrics.status === "unavailable") {
+        return "unavailable";
+    }
+    const note = metrics.reason ? `; note=${metrics.reason}` : "";
+    const warnings = metrics.proofWarnings.length > 0 ? `; warnings=${metrics.proofWarnings.join(" | ")}` : "";
+    return `Solve Packet ${(0, tokenSavings_1.formatTokenCount)(metrics.solvePacketTokensEstimate)} / workspace ${(0, tokenSavings_1.formatTokenCount)(metrics.workspaceTokensEstimate)} tokens, candidates=${(0, tokenSavings_1.formatTokenCount)(metrics.candidateTokensEstimate)}, reduction ${(0, tokenSavings_1.formatReductionPercent)(metrics.reductionPercent)}, method=${metrics.method}, tokenizer=${metrics.tokenizer}, basis=${metrics.basis}, status=${metrics.status}, proof=${metrics.proofStatus}, files=${metrics.rankedFileCount}/${metrics.workspaceFileCount}${note}${warnings}`;
+}
+function formatContextCoverage(result) {
+    const coverage = result.contextCoverage;
+    if (!coverage) {
+        return "coverage unavailable";
+    }
+    const missing = coverage.missingRequiredSlots.length > 0 ? `, missing=${coverage.missingRequiredSlots.join("|")}` : "";
+    return `coverage ${coverage.foundRequiredSlots.length}/${coverage.requiredSlots.length} required (${coverage.requiredCoveragePercent.toFixed(0)}%)${missing}`;
+}
+function boolText(value) {
+    return value ? "yes" : "no";
+}
+function uniqueStrings(values) {
+    return Array.from(new Set(values));
+}
+function count(items, predicate) {
+    return items.filter(predicate).length;
+}
+function average(values) {
+    if (values.length === 0) {
+        return 0;
+    }
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function formatBytes(bytes) {
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(1)}KB`;
+    }
+    return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+}
+function formatIndexStats(stats) {
+    if (stats.indexedFiles === undefined) {
+        return "";
+    }
+    return `, indexed=${stats.indexedFiles}, candidates=${stats.candidateFiles ?? "unknown"}, indexReused=${stats.indexReusedFiles ?? 0}, indexUpdated=${stats.indexUpdatedFiles ?? 0}`;
+}
+//# sourceMappingURL=runEvaluationSuite.js.map

package/dist/commands/runReplaySuite.js

@@ -0,0 +1,228 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runReplaySuite = runReplaySuite;
+const path = __importStar(require("path"));
+const vscode = __importStar(require("vscode"));
+const feedback_1 = require("../pipeline/feedback");
+const fileGroups_1 = require("../pipeline/fileGroups");
+const runPipeline_1 = require("../pipeline/runPipeline");
+const tokenSavings_1 = require("../pipeline/tokenSavings");
+const logger_1 = require("../utils/logger");
+async function runReplaySuite() {
+    const folders = vscode.workspace.workspaceFolders;
+    if (!folders || folders.length === 0) {
+        (0, logger_1.logInfo)("Run Real-World Replay stopped because no workspace folder is open.");
+        vscode.window.showWarningMessage("Open a workspace folder before running FlowSeeker replay.");
+        return;
+    }
+    const root = folders[0];
+    try {
+        const records = await (0, feedback_1.readFeedbackRecords)(root.uri.fsPath);
+        if (records.length === 0) {
+            vscode.window.showInformationMessage("No FlowSeeker feedback found yet. Run a chat request and mark the packet as Relevant, Missing file, Not relevant, or AI needed more.");
+            return;
+        }
+        (0, logger_1.logInfo)(`Starting real-world replay from feedback. Cases: ${records.length}; workspace: ${root.uri.fsPath}`);
+        const results = await vscode.window.withProgress({
+            location: vscode.ProgressLocation.Notification,
+            title: "FlowSeeker is replaying real-world feedback",
+            cancellable: true
+        }, async (progress, token) => {
+            const replayResults = [];
+            let completed = 0;
+            for (const { record } of records) {
+                if (token.isCancellationRequested) {
+                    (0, logger_1.logInfo)("Real-world replay cancelled by user.");
+                    return replayResults;
+                }
+                progress.report({
+                    message: `${completed + 1}/${records.length}: ${record.run.task.slice(0, 64)}`,
+                    increment: records.length > 0 ? 100 / records.length : undefined
+                });
+                const startedAt = Date.now();
+                const result = await (0, runPipeline_1.runFlowSeeker)(record.run.task, { token });
+                const replayResult = evaluateReplay(record, result, Date.now() - startedAt);
+                replayResults.push(replayResult);
+                completed += 1;
+                (0, logger_1.logInfo)(formatReplayLog(replayResult, completed, records.length));
+            }
+            return replayResults;
+        });
+        if (results.length === 0) {
+            vscode.window.showInformationMessage("FlowSeeker replay produced no results.");
+            return;
+        }
+        const reportUri = await writeReplayReport(root.uri, results);
+        (0, logger_1.logInfo)(`Real-world replay complete. Report: ${reportUri.fsPath}`);
+        vscode.window.showInformationMessage(`FlowSeeker replay complete. Report saved: ${path.basename(reportUri.fsPath)}`);
+        await vscode.commands.executeCommand("vscode.open", reportUri);
+    }
+    catch (error) {
+        (0, logger_1.logError)("Run Real-World Replay failed.", error);
+        const message = error instanceof Error ? error.message : String(error);
+        vscode.window.showErrorMessage(`FlowSeeker replay failed: ${message}`);
+    }
+}
+function evaluateReplay(feedback, result, durationMs) {
+    const contextGapReported = feedback.kind === "missing_file" || feedback.kind === "agent_needed_more_context" || feedback.outcome?.aiNeededMoreContext === true;
+    const positiveFeedback = feedback.kind === "relevant" || feedback.outcome?.planAccepted === true || feedback.outcome?.fixVerified === true;
+    const negativeFeedback = feedback.kind === "not_relevant" || contextGapReported;
+    return {
+        feedback,
+        result,
+        durationMs,
+        contextGapReported,
+        positiveFeedback,
+        negativeFeedback
+    };
+}
+async function writeReplayReport(rootUri, results) {
+    const reportsDir = vscode.Uri.joinPath(rootUri, ".flowseeker", "reports");
+    await vscode.workspace.fs.createDirectory(reportsDir);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const reportUri = vscode.Uri.joinPath(reportsDir, `real-world-replay-${timestamp}.md`);
+    await vscode.workspace.fs.writeFile(reportUri, new TextEncoder().encode(renderReport(results)));
+    return reportUri;
+}
+function renderReport(results) {
+    const lines = [
+        "# FlowSeeker Real-World Replay Report",
+        "",
+        `Generated: ${new Date().toISOString()}`,
+        `Cases: ${results.length}`,
+        "",
+        "## Summary",
+        "",
+        formatSummary(results),
+        formatTokenSavingsSummary(results),
+        "",
+        "## Meaning",
+        "",
+        "- `Context gap reported` comes from local user/agent feedback.",
+        "- `Current top files` are from replaying the same task against the current workspace.",
+        "- `Original top files` are from the saved feedback snapshot, so drift is visible after ranking changes.",
+        "- Source snippets are not stored in feedback records.",
+        "",
+        "## Cases",
+        ""
+    ];
+    for (const result of results) {
+        lines.push(...renderCase(result));
+    }
+    return `${lines.join("\n")}\n`;
+}
+function renderCase(result) {
+    const feedback = result.feedback;
+    const topCurrent = (0, fileGroups_1.aggregateEvidenceFiles)(result.result.units).slice(0, 12);
+    const originalTop = feedback.run.topFiles.slice(0, 8);
+    const coverage = result.result.contextCoverage;
+    const stats = result.result.stats;
+    return [
+        `### ${feedbackKindLabel(feedback.kind)} - ${feedback.run.task}`,
+        "",
+        `Feedback time: ${feedback.createdAt}`,
+        `Feedback note: ${feedback.note ?? "none"}`,
+        `Context gap reported: ${yesNo(result.contextGapReported)}`,
+        `Positive feedback: ${yesNo(result.positiveFeedback)}`,
+        `Current intent: ${result.result.profile.intent}`,
+        `Original intent: ${feedback.run.intent}`,
+        `Current token savings: ${formatCaseTokenSavings(result.result.tokenSavings)}`,
+        feedback.run.tokenSavings ? `Original token savings: ${(0, tokenSavings_1.formatTokenCount)(feedback.run.tokenSavings.solvePacketTokensEstimate)} / ${(0, tokenSavings_1.formatTokenCount)(feedback.run.tokenSavings.workspaceTokensEstimate)} saved=${(0, tokenSavings_1.formatReductionPercent)(feedback.run.tokenSavings.reductionPercent)} tokenizer=${feedback.run.tokenSavings.tokenizer}` : "Original token savings: unavailable",
+        coverage ? `Current context coverage: ${coverage.foundRequiredSlots.length}/${coverage.requiredSlots.length} required (${coverage.requiredCoveragePercent.toFixed(0)}%)${coverage.missingRequiredSlots.length ? ` missing=${coverage.missingRequiredSlots.join("|")}` : ""}` : "Current context coverage: unavailable",
+        `Duration: ${(result.durationMs / 1000).toFixed(2)}s`,
+        `Scan: discovered=${stats.discoveredFiles}, scanned=${stats.scannedFiles}, skipped=${stats.skippedFiles}, rawEvidence=${stats.rawEvidenceCount}, ranked=${stats.rankedEvidenceCount}, capped=${stats.capped}, stoppedEarly=${stats.stoppedEarly}${stats.stopReason ? ` (${stats.stopReason})` : ""}`,
+        "",
+        "Current top files:",
+        "",
+        ...topCurrent.map((group, index) => `${index + 1}. ${group.relativePath} [${group.role}/${group.slot}] score=${group.score.toFixed(1)} evidence=${group.unitCount}`),
+        "",
+        "Original top files:",
+        "",
+        ...originalTop.map((file, index) => `${index + 1}. ${file.relativePath} [${file.role}/${file.slot ?? "unknown"}] score=${file.score.toFixed(1)} evidence=${file.evidence}`),
+        ""
+    ];
+}
+function formatSummary(results) {
+    const positive = results.filter((result) => result.positiveFeedback).length;
+    const negative = results.filter((result) => result.negativeFeedback).length;
+    const gaps = results.filter((result) => result.contextGapReported).length;
+    const avgCoverage = average(results.map((result) => result.result.contextCoverage?.requiredCoveragePercent ?? 0));
+    const avgDuration = average(results.map((result) => result.durationMs));
+    return `cases=${results.length}, positiveFeedback=${positive}/${results.length}, negativeFeedback=${negative}/${results.length}, contextGapReported=${gaps}/${results.length}, requiredCoverageAvg=${avgCoverage.toFixed(1)}%, avg=${(avgDuration / 1000).toFixed(2)}s`;
+}
+function formatTokenSavingsSummary(results) {
+    const metrics = results.map((result) => result.result.tokenSavings).filter((item) => item !== undefined && item.status !== "unavailable");
+    if (metrics.length === 0) {
+        return "tokenSavings=unavailable";
+    }
+    const avgReduction = average(metrics.map((item) => item.reductionPercent));
+    const avgPacketTokens = average(metrics.map((item) => item.solvePacketTokensEstimate));
+    const avgWorkspaceTokens = average(metrics.map((item) => item.workspaceTokensEstimate));
+    const partial = metrics.filter((item) => item.status === "partial").length;
+    return `tokenSavings avg=${(0, tokenSavings_1.formatReductionPercent)(avgReduction)}, packetAvg=${(0, tokenSavings_1.formatTokenCount)(avgPacketTokens)}, workspaceAvg=${(0, tokenSavings_1.formatTokenCount)(avgWorkspaceTokens)}, partial=${partial}/${metrics.length}`;
+}
+function formatCaseTokenSavings(metrics) {
+    if (!metrics || metrics.status === "unavailable") {
+        return "unavailable";
+    }
+    const warnings = metrics.proofWarnings.length > 0 ? `; warnings=${metrics.proofWarnings.join(" | ")}` : "";
+    return `${(0, tokenSavings_1.formatTokenCount)(metrics.solvePacketTokensEstimate)} / ${(0, tokenSavings_1.formatTokenCount)(metrics.workspaceTokensEstimate)} saved=${(0, tokenSavings_1.formatReductionPercent)(metrics.reductionPercent)} tokenizer=${metrics.tokenizer} proof=${metrics.proofStatus}${warnings}`;
+}
+function formatReplayLog(result, completed, total) {
+    return `Replay ${completed}/${total} ${result.feedback.kind}: gap=${yesNo(result.contextGapReported)}; ${formatCaseTokenSavings(result.result.tokenSavings)}; duration ${(result.durationMs / 1000).toFixed(2)}s.`;
+}
+function feedbackKindLabel(kind) {
+    switch (kind) {
+        case "relevant":
+            return "RELEVANT";
+        case "missing_file":
+            return "MISSING CONTEXT";
+        case "not_relevant":
+            return "NOT RELEVANT";
+        case "agent_needed_more_context":
+            return "AI NEEDED MORE CONTEXT";
+    }
+}
+function yesNo(value) {
+    return value ? "yes" : "no";
+}
+function average(values) {
+    if (values.length === 0) {
+        return 0;
+    }
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+//# sourceMappingURL=runReplaySuite.js.map

package/dist/config/defaultConfig.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultConfig = void 0;
+exports.mergeConfig = mergeConfig;
+exports.defaultConfig = {
+    project: {
+        name: "FlowSeeker Workspace",
+        include: ["**/*"],
+        exclude: ["node_modules", "vendor", "dist", "build", "out", ".git", "coverage", ".next", "target", ".vscode-test", ".turbo", ".gradle", ".dart_tool", ".venv", "venv", "__pycache__", ".mypy_cache", ".pytest_cache", ".flowseeker", ".flowseeker/cache", ".flowseeker/reports", ".flowseeker/feedback", ".flowseeker/evals", ".flowseeker/eval*.json", "logs", "tmp", "temp", "drizzle/meta", "storage/framework", "bootstrap/cache", "**/_gen*.py", "**/_gen*.sh", "**/_bulk_write.js", "**/*.snap", "**/*snapshot*.json"],
+        maxFileSizeKb: 800
+    },
+    index: {
+        enableWorkspaceIndex: true,
+        enableTreeSitter: true,
+        enableLsp: false,
+        enableGitHistory: false,
+        chunkSizeLines: 80,
+        chunkOverlapLines: 12,
+        enableDependencyExpansion: true,
+        maxSeedCandidateFiles: 250,
+        maxDependencyExpansionFiles: 400,
+        maxIndexedCandidateFiles: 800
+    },
+    ai: {
+        enabled: false,
+        provider: "openai",
+        model: "gpt-5.2",
+        baseUrl: "",
+        useDirectProvider: false,
+        apiKeyEnv: "FLOWSEEKER_API_KEY",
+        maxInputTokens: 6000,
+        outputLanguage: "auto"
+    },
+    pipeline: {
+        maxCandidates: 40,
+        maxContextTokens: 4000,
+        graphDepth: 2,
+        minConfidence: 0.45,
+        maxFiles: 5000,
+        maxTotalBytesMb: 120,
+        maxScanMs: 30000,
+        maxRawEvidence: 1000
+    },
+    synonyms: {
+        mail: ["email", "smtp", "mailer", "sendmail", "notification"],
+        payment: ["billing", "invoice", "checkout", "transaction", "stripe"],
+        auth: ["login", "session", "jwt", "token", "permission", "role"],
+        queue: ["job", "worker", "dispatch", "schedule", "retry"],
+        verify: ["verified", "verification", "confirm", "confirmation", "activate", "activation", "email_verified", "emailverified"],
+        welcome: ["onboarding", "greeting", "registration", "registered", "signup", "sign_up"],
+        timestamp: ["time", "date", "datetime", "time_zone", "timezone", "tz", "utc", "created_at", "updated_at", "submitted_at", "scheduled_at"],
+        timezone: ["time_zone", "tz", "utc", "locale", "local_time", "localtime", "browser", "client"],
+        browser: ["client", "frontend", "navigator", "locale", "time_zone", "timezone", "tz"],
+        schedule: ["scheduled", "scheduled_at", "delay", "delayed", "queue_at"],
+        review: ["investigate", "trace", "check", "audit", "inspect"],
+        document: ["file", "attachment", "submission", "upload", "document_type", "type"],
+        upload: ["submit", "submission", "file", "attachment", "storage", "disk"],
+        export: ["download", "report", "csv", "excel", "xlsx", "file"],
+        validation: ["validator", "request", "rule", "rules", "schema", "input"],
+        permission: ["authorize", "policy", "role", "guard", "middleware", "acl"]
+    }
+};
+function mergeConfig(base, override) {
+    return {
+        project: { ...base.project, ...override.project },
+        index: { ...base.index, ...override.index },
+        ai: { ...base.ai, ...override.ai },
+        pipeline: { ...base.pipeline, ...override.pipeline },
+        synonyms: { ...base.synonyms, ...override.synonyms }
+    };
+}
+//# sourceMappingURL=defaultConfig.js.map