flowseeker 0.1.8 → 0.1.9

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # FlowSeeker Changelog
 
+## 0.1.9 - Accuracy Gates, Semantic Diagnostics, And Release Hygiene
+
+- Added semantic-only diagnostic/report-only mode for MCP context output so semantic experiments can be inspected without changing deterministic ranking or primary edit candidates.
+- Hardened MCP context smoke coverage across JSON, human, agent, and compact output profiles, including semantic-only warning checks and default-mode purity checks.
+- Added multi-framework golden accuracy harnesses for Laravel, NestJS, Next.js, and Django with label-quality checks, must-not guards, and reproducible edit-recall metrics.
+- Added Phase 20 deterministic accuracy improvements through guarded `framework_convention` and `role_intent` signals, improving NestJS edit recall at 1 from 0% to 67% with must-not violations still at 0%.
+- Added top1 gap, residual triage, signal simulation, import-graph reality, hard-residual strategy, and closeout reports to document what improved and where simple deterministic boosts are exhausted.
+- Closed Phase 20 with documented limitations: three remaining hard top1 misses require future cross-file workflow/domain evidence infrastructure rather than another blind weight tweak.
+- Added closeout/release-readiness artifacts for the 0.1.9 accuracy work, including final matrix metrics and rejected/unsafe target documentation.
+- Refreshed FlowSeeker logo assets across extension, chat/webview, activity bar, and landing-page public assets.
+- Tightened release hygiene so internal PDF/export artifacts are excluded from git, npm, and VSIX packages.
+
 ## 0.1.8 - Local Semantic, One-Command MCP, And Solve Packet UX
 
 - Added full CLI entry points: `flowseeker`, `fs-guide`, `fs-auto`, `fs-files`, `fs-retrieve`, and `fs-mcp`.

package/dist/chat/nativeChatParticipant.js

@@ -170,7 +170,7 @@ function registerNativeChatParticipant(context, resultProvider, gateway, editSto
         }
     };
     const participant = vscode.chat.createChatParticipant(participantId, handler);
-    participant.iconPath = vscode.Uri.joinPath(context.extensionUri, "resources", "flowseeker-logo-vector-v2.svg");
+    participant.iconPath = vscode.Uri.joinPath(context.extensionUri, "resources", "flowseeker-logo.png");
     participant.followupProvider = {
         provideFollowups(result) {
             const metadata = result.metadata;

package/dist/cli/main.js

@@ -471,6 +471,13 @@ async function runDoctorCommand(args) {
             const hasGuide = stdout.includes("flowseeker_guide");
             const hasAuto = stdout.includes("flowseeker_auto");
             const hasFiles = stdout.includes("flowseeker_files");
+            const hasValidatePlan = stdout.includes("flowseeker_validate_plan");
+            const hasContext = stdout.includes("flowseeker_context");
+            const hasInspect = stdout.includes("flowseeker_inspect");
+            const hasExpand = stdout.includes("flowseeker_expand");
+            const hasStatus = stdout.includes("flowseeker_status");
+            const hasIndex = stdout.includes("flowseeker_index");
+            const hasFeedback = stdout.includes("flowseeker_feedback");
             mcpStatus = {
                 checked: true,
                 initializeOk: stdout.includes('"id":0'),
@@ -478,10 +485,19 @@ async function runDoctorCommand(args) {
                 toolsIncludesGuide: hasGuide,
                 toolsIncludesAuto: hasAuto,
                 toolsIncludesFiles: hasFiles,
+                toolsIncludesValidatePlan: hasValidatePlan,
+                toolsIncludesContext: hasContext,
+                toolsIncludesInspect: hasInspect,
+                toolsIncludesExpand: hasExpand,
+                toolsIncludesStatus: hasStatus,
+                toolsIncludesIndex: hasIndex,
+                toolsIncludesFeedback: hasFeedback,
                 promptsIncludeRetrieve: stdout.includes('"retrieve"'),
                 promptsIncludeGuide: stdout.includes('"guide"'),
                 promptsIncludeAuto: stdout.includes('"auto"'),
-                promptsIncludeFiles: stdout.includes('"files"')
+                promptsIncludeFiles: stdout.includes('"files"'),
+                promptsIncludeValidatePlan: stdout.includes('"validate-plan"'),
+                promptsIncludeContext: stdout.includes('"context"'),
             };
             if (!hasRetrieve)
                 nextActions.push("MCP tools/list missing flowseeker_retrieve. Check MCP server startup.");
@@ -545,8 +561,8 @@ async function runDoctorCommand(args) {
             lines.push("  Self-check: " + (mcp.checked ? (mcp.initializeOk ? "pass" : "init failed") : "not run"));
             if (mcp.checked) {
                 const toolIcons = (name) => mcp[name] ? "present" : "MISSING";
-                lines.push("  Tools: retrieve=" + toolIcons("toolsIncludesRetrieve") + " guide=" + toolIcons("toolsIncludesGuide") + " auto=" + toolIcons("toolsIncludesAuto") + " files=" + toolIcons("toolsIncludesFiles"));
-                lines.push("  Prompts: retrieve=" + toolIcons("promptsIncludeRetrieve") + " guide=" + toolIcons("promptsIncludeGuide") + " auto=" + toolIcons("promptsIncludeAuto") + " files=" + toolIcons("promptsIncludeFiles"));
+                lines.push("  Tools: retrieve=" + toolIcons("toolsIncludesRetrieve") + " guide=" + toolIcons("toolsIncludesGuide") + " auto=" + toolIcons("toolsIncludesAuto") + " files=" + toolIcons("toolsIncludesFiles") + " validate_plan=" + toolIcons("toolsIncludesValidatePlan") + " context=" + toolIcons("toolsIncludesContext") + " inspect=" + toolIcons("toolsIncludesInspect") + " expand=" + toolIcons("toolsIncludesExpand") + " status=" + toolIcons("toolsIncludesStatus") + " index=" + toolIcons("toolsIncludesIndex") + " feedback=" + toolIcons("toolsIncludesFeedback"));
+                lines.push("  Prompts: retrieve=" + toolIcons("promptsIncludeRetrieve") + " guide=" + toolIcons("promptsIncludeGuide") + " auto=" + toolIcons("promptsIncludeAuto") + " files=" + toolIcons("promptsIncludeFiles") + " validate-plan=" + toolIcons("promptsIncludeValidatePlan") + " context=" + toolIcons("promptsIncludeContext"));
             }
             lines.push("  Server command: flowseeker");
             lines.push("  Server args: mcp --workspace " + resolved + "");

package/dist/cli/runEvaluation.js

@@ -41,8 +41,11 @@ const defaultConfig_1 = require("../config/defaultConfig");
 const evaluationMetrics_1 = require("../pipeline/evaluationMetrics");
 const fileGroups_1 = require("../pipeline/fileGroups");
 const runHeadless_1 = require("../pipeline/runHeadless");
+const fileScanner_1 = require("../pipeline/fileScanner");
 const tokenSavings_1 = require("../pipeline/tokenSavings");
 const text_1 = require("../utils/text");
+const goldenTask_1 = require("../eval/goldenTask");
+const accuracyV2_1 = require("../eval/accuracyV2");
 void main().catch((error) => {
     const message = error instanceof Error ? error.stack ?? error.message : String(error);
     console.error(message);
@@ -50,6 +53,11 @@ void main().catch((error) => {
 });
 async function main() {
     const options = parseArgs(process.argv.slice(2));
+    // Golden-manifest-only mode: skip regular eval
+    if (options.goldenManifestPath && !options.manifestPath && !options.workspacePath) {
+        await runGoldenEval(options.goldenManifestPath, options.goldenReportPath);
+        return;
+    }
     const loaded = await loadManifest(options);
     const manifest = loaded.manifest;
     validateManifest(manifest, loaded.source);
@@ -114,6 +122,9 @@ async function main() {
     if (options.failOnMiss && results.some((result) => result.pass === false)) {
         process.exitCode = 1;
     }
+    if (options.goldenManifestPath) {
+        await runGoldenEval(options.goldenManifestPath, options.goldenReportPath);
+    }
 }
 function parseArgs(args) {
     const options = {
@@ -198,6 +209,16 @@ function parseArgs(args) {
             options.dumpFailureAnalysis = true;
             continue;
         }
+        if (arg === "--golden-manifest") {
+            options.goldenManifestPath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--golden-report") {
+            options.goldenReportPath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
         if (arg === "--help" || arg === "-h") {
             printUsage();
             process.exit(0);
@@ -962,4 +983,86 @@ function formatTaskAwareGuardSummary(results) {
     const consistency = (scanned === candidates + blockedSlot + blockedExpected) && (candidates === demoted + promoted + noSafeReplacement);
     return `taskAwareAvoidGuard: scanned=${scanned}, candidates=${candidates}, demoted=${demoted}, promoted=${promoted}, blockedSlot=${blockedSlot}, blockedExpected=${blockedExpected}, noSafeReplacement=${noSafeReplacement}\ntaskAwareAvoidGuardConsistency: ${consistency ? "PASS" : "FAIL"}`;
 }
+async function runGoldenEval(goldenManifestPath, goldenReportPath) {
+    const manifestPath = path.resolve(goldenManifestPath);
+    const manifest = await readJson(manifestPath);
+    const workspacePath = typeof manifest.workspace === "string" && manifest.workspace.trim()
+        ? path.resolve(manifest.workspace)
+        : "";
+    const issues = (0, goldenTask_1.validateGoldenManifest)(manifest, workspacePath || undefined);
+    if (issues.length > 0) {
+        console.error(`Golden manifest validation failed with ${issues.length} issue(s):`);
+        for (const issue of issues)
+            console.error(`  - ${issue}`);
+        process.exitCode = 1;
+        return;
+    }
+    const config = (0, defaultConfig_1.mergeConfig)(await (0, loadConfigFromPath_1.loadConfigFromPath)(workspacePath), {});
+    console.log(`\nGolden Eval: ${manifest.name}`);
+    console.log(`Workspace: ${workspacePath}`);
+    console.log(`Tasks: ${manifest.tasks.length}`);
+    const perTaskResults = [];
+    const flowResults = [];
+    const taskGuardReasons = {};
+    for (const goldenTask of manifest.tasks) {
+        const taskStart = Date.now();
+        console.log(`- ${goldenTask.id}: ${goldenTask.task}`);
+        (0, fileScanner_1.clearGuardReasons)();
+        const result = await (0, runHeadless_1.runFlowSeekerHeadless)(workspacePath, goldenTask.task, config);
+        const gr = (0, fileScanner_1.getGuardReasons)();
+        if (gr.size > 0) {
+            const plain = {};
+            for (const [fp, cat] of gr)
+                plain[fp] = cat;
+            taskGuardReasons[String(perTaskResults.length)] = plain;
+        }
+        flowResults.push(result);
+        const metrics = (0, accuracyV2_1.computeAccuracyV2)(result, goldenTask);
+        perTaskResults.push(metrics);
+        const dur = ((Date.now() - taskStart) / 1000).toFixed(2);
+        console.log(`  editR@1=${metrics.editRecall1} editR@3=${metrics.editRecall3} editR@10=${metrics.editRecall10} first3Hit=${metrics.first3Hit} mustNotViolation=${metrics.mustNotEditViolation} ${dur}s`);
+    }
+    // Compute failure analysis for each task
+    const failureAnalysis = manifest.tasks.map((goldenTask, i) => (0, accuracyV2_1.computeFailureAnalysis)(flowResults[i], goldenTask, perTaskResults[i]));
+    // Compute semantic contribution
+    const semPerTask = flowResults.map((r) => (0, accuracyV2_1.computeSemanticContribution)(r));
+    const semanticContribution = (0, accuracyV2_1.aggregateSemanticContribution)(semPerTask, flowResults);
+    // Compute role classifier report
+    const roleConfusionMatrix = (0, accuracyV2_1.computeRoleConfusionMatrix)(flowResults, manifest.tasks, "balanced");
+    const roleClassifierReport = (0, accuracyV2_1.computeRoleClassifierReport)(flowResults, manifest.tasks);
+    const report = (0, accuracyV2_1.aggregateAccuracyV2)(perTaskResults, failureAnalysis, semanticContribution, roleConfusionMatrix, roleClassifierReport, manifest.name);
+    // Write JSON report
+    const reportPath = path.resolve(goldenReportPath ?? path.join(".flowseeker", "reports", `golden-eval-${slugify(manifest.name)}.json`));
+    await fs.mkdir(path.dirname(reportPath), { recursive: true });
+    await fs.writeFile(reportPath, JSON.stringify({ ...report, guardReasons: taskGuardReasons }, null, 2), "utf8");
+    console.log(`\nGolden JSON: ${reportPath}`);
+    // Write markdown report
+    const mdPath = reportPath.replace(/\.json$/, ".md");
+    const mdLines = [
+        "# FlowSeeker Accuracy V2 Report",
+        "",
+        `Generated: ${new Date().toISOString()}`,
+        `Manifest: ${manifest.name}`,
+        `Workspace: ${workspacePath}`,
+        `Tasks: ${report.taskCount}`,
+        "",
+        ...(0, accuracyV2_1.renderAccuracyV2Markdown)(report),
+    ];
+    await fs.writeFile(mdPath, mdLines.join("\n"), "utf8");
+    console.log(`Golden Markdown: ${mdPath}`);
+    // Print summary
+    console.log(`\nAccuracy V2 Summary:`);
+    console.log(`  edit_recall@1: ${(report.editRecall1 * 100).toFixed(1)}%`);
+    console.log(`  edit_recall@3: ${(report.editRecall3 * 100).toFixed(1)}%`);
+    console.log(`  edit_recall@5: ${(report.editRecall5 * 100).toFixed(1)}%`);
+    console.log(`  edit_recall@10: ${(report.editRecall10 * 100).toFixed(1)}%`);
+    console.log(`  primary_edit_precision@3: ${(report.primaryEditPrecision3 * 100).toFixed(1)}%`);
+    console.log(`  read_context_recall@10: ${(report.readContextRecall10 * 100).toFixed(1)}%`);
+    console.log(`  test_recall@10: ${(report.testRecall10 * 100).toFixed(1)}%`);
+    console.log(`  must_not_edit_violation_rate: ${(report.mustNotEditViolationRate * 100).toFixed(1)}%`);
+    console.log(`  noise_promoted_as_edit_rate: ${(report.noisePromotedAsEditRate * 100).toFixed(1)}%`);
+    console.log(`  required_slot_coverage: ${(report.requiredSlotCoverage * 100).toFixed(1)}%`);
+    console.log(`  missing_slot_rate: ${(report.missingSlotRate * 100).toFixed(1)}%`);
+    console.log(`  first3_hit_rate: ${(report.first3HitRate * 100).toFixed(1)}%`);
+}
 //# sourceMappingURL=runEvaluation.js.map