@hassangameryt/freeflow 0.2.0 → 0.3.0

package/{plugins/freeflow/router/dist → router/dist/benchmarks}/benchmarks.js

@@ -5,8 +5,8 @@ import { dirname, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { performance } from "node:perf_hooks";
 import { approximateTokens, averagePercent as average, defaultJsonRunReportPath, escapeMarkdownTableCell as escapeTable, formatPercent, latencySummary, medianPercent as median, normalizeIterations, parseBenchmarkCliArgs, reductionPercent, writeBenchmarkReportPair, } from "./benchmark-harness.js";
-import { freeflowRetrieve } from "./retrieve.js";
-import { createVault, storeCommandOutput } from "./vault.js";
+import { freeflowSearch } from "../tools/search.js";
+import { createVault, storeCommandOutput } from "../vault/vault.js";
 const DEFAULT_ITERATIONS = 3;
 const DEFAULT_CONTEXT_LINES = 2;
 const LEGACY_SKIP_DIRS = new Set([".git", "node_modules", "dist", "build", "out", ".next", ".nuxt", "coverage", "target"]);
@@ -86,7 +86,7 @@ export function renderRouterBenchmarkReport(report) {
         "npm run bench:router",
         "```",
         "",
-        "The CLI writes machine-readable JSON under `plugins/freeflow/evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
+        "The CLI writes machine-readable JSON under `evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
         "",
         "## Summary",
         "",
@@ -424,12 +424,12 @@ async function createVaultedOutputFixture() {
                 query: "ASSERTION_FAILED payments badge",
                 recovery: { status: "failed", detail: "Baseline proxy does not expose structured recovery metadata." },
             }),
-            "improved-freeflow-router": async () => improvedRetrieveObservation(await freeflowRetrieve({
+            "improved-freeflow-router": async () => improvedRetrieveObservation(await freeflowSearch({
                 action: "query",
                 source: { kind: "vault", root: vaultRoot.path, sessionId, outputId: record.outputId, stream: "stderr" },
                 query: "ASSERTION_FAILED payments badge",
                 preserve: "important",
-            }), "improved-freeflow-router: freeflow_retrieve vault query", stderr, (result) => verifyVaultEvidenceRecovery(vaultRoot.path, sessionId, result)),
+            }), "improved-freeflow-router: freeflow_search vault query", stderr, (result) => verifyVaultEvidenceRecovery(vaultRoot.path, sessionId, result)),
         },
         cleanup: vaultRoot.cleanup,
     };
@@ -468,7 +468,7 @@ async function createExpansionFixture() {
                 recovery: { status: "failed", detail: "Baseline proxy does not expose structured expansion recovery." },
             }),
             "improved-freeflow-router": async () => {
-                const queryResult = await freeflowRetrieve({
+                const queryResult = await freeflowSearch({
                     action: "query",
                     source: { kind: "repo", root: repo.path },
                     query,
@@ -476,15 +476,15 @@ async function createExpansionFixture() {
                 });
                 const evidence = queryResult.evidence?.[0];
                 if (!evidence) {
-                    return improvedRetrieveObservation(queryResult, "improved-freeflow-router: freeflow_retrieve query before expand", body.join("\n"), (result) => verifyRepoEvidenceRecovery(repo.path, result));
+                    return improvedRetrieveObservation(queryResult, "improved-freeflow-router: freeflow_search query before expand", body.join("\n"), (result) => verifyRepoEvidenceRecovery(repo.path, result));
                 }
-                return improvedRetrieveObservation(await freeflowRetrieve({
+                return improvedRetrieveObservation(await freeflowSearch({
                     action: "expand",
                     source: { kind: "repo", root: repo.path },
                     evidence,
                     expansion: "lines_30",
                     preserve: "important",
-                }), "improved-freeflow-router: freeflow_retrieve expand lines_30", body.join("\n"), (result) => verifyRepoEvidenceRecovery(repo.path, result));
+                }), "improved-freeflow-router: freeflow_search expand lines_30", body.join("\n"), (result) => verifyRepoEvidenceRecovery(repo.path, result));
             },
         },
         cleanup: repo.cleanup,
@@ -507,12 +507,12 @@ function repoQueryFixture(repo, options) {
                 skipDirs: LEGACY_SKIP_DIRS,
                 recovery: { status: "failed", detail: "Baseline proxy does not expose structured recovery metadata." },
             }),
-            "improved-freeflow-router": async () => improvedRetrieveObservation(await freeflowRetrieve({
+            "improved-freeflow-router": async () => improvedRetrieveObservation(await freeflowSearch({
                 action: "query",
                 source: { kind: "repo", root: repo.path },
                 query: options.query,
                 preserve: "important",
-            }), "improved-freeflow-router: freeflow_retrieve query", await readRepoBytes(repo.path), (result) => verifyRepoEvidenceRecovery(repo.path, result)),
+            }), "improved-freeflow-router: freeflow_search query", await readRepoBytes(repo.path), (result) => verifyRepoEvidenceRecovery(repo.path, result)),
         },
         cleanup: repo.cleanup,
     };
@@ -618,7 +618,7 @@ async function verifyRepoEvidenceRecovery(root, result) {
     if (!range) {
         return { status: "failed", detail: `Unsupported repo evidence range ${evidence.lines}.` };
     }
-    const recovered = await freeflowRetrieve({
+    const recovered = await freeflowSearch({
         action: "retrieve",
         source: { kind: "repo", root, path: evidence.path },
         lineRange: range,
@@ -635,7 +635,7 @@ async function verifyVaultEvidenceRecovery(vaultRoot, sessionId, result) {
     if (!range) {
         return { status: "failed", detail: `Unsupported vault evidence range ${evidence.lines}.` };
     }
-    const recovered = await freeflowRetrieve({
+    const recovered = await freeflowSearch({
         action: "retrieve",
         source: {
             kind: "vault",
@@ -853,7 +853,7 @@ function normalizeRelativePath(path) {
     return path.split(/[/\\]+/).filter(Boolean).join("/");
 }
 function defaultReportPath() {
-    return resolve(process.cwd(), "plugins/freeflow/evals/reports/runtime/output-router-benchmark-1-report.md");
+    return resolve(process.cwd(), "evals/reports/runtime/output-router-benchmark-1-report.md");
 }
 async function runCli() {
     const { iterations, reportPath, jsonReportPath } = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: defaultReportPath() });

package/{plugins/freeflow/router/dist → router/dist/benchmarks}/codex-qa-benchmarks.js

@@ -4,7 +4,7 @@ import { dirname, join, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { performance } from "node:perf_hooks";
 import { approximateTokens, defaultJsonRunReportPath, escapeMarkdownTableCell as escapeTable, formatPercent, latencySummary, normalizeIterations, parseBenchmarkCliArgs, reductionPercent, writeBenchmarkReportPair, } from "./benchmark-harness.js";
-import { freeflowRetrieve } from "./retrieve.js";
+import { freeflowSearch } from "../tools/search.js";
 const SANDBOX_DOC_PATH = "docs/codex-cli-agent-harness/2026-06-12-pass-3-sandboxing-and-permissions.md";
 const GENERATED_GRAPH_PATH = "graphify-out/graph.html";
 const SANDBOX_QUESTION = "Find the Sandbox Permissions section, report file/lines, and explain UseDefault, RequireEscalated, and WithAdditionalPermissions.";
@@ -77,7 +77,7 @@ export function renderCodexQaBenchmarkReport(report) {
         "npm run bench:router:codex-qa",
         "```",
         "",
-        "The CLI writes machine-readable JSON under `plugins/freeflow/evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
+        "The CLI writes machine-readable JSON under `evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
         "",
         "## Summary",
         "",
@@ -162,7 +162,7 @@ async function runModeIterations(fixture, mode, iterations) {
     };
 }
 async function improvedFreeflowObservation(fixture) {
-    const queryResult = await freeflowRetrieve({
+    const queryResult = await freeflowSearch({
         action: "query",
         source: { kind: "repo", root: fixture.root },
         query: fixture.query,
@@ -171,7 +171,7 @@ async function improvedFreeflowObservation(fixture) {
     });
     const seedEvidence = queryResult.evidence?.[0];
     const expandedResult = seedEvidence
-        ? await freeflowRetrieve({
+        ? await freeflowSearch({
             action: "expand",
             source: { kind: "repo", root: fixture.root },
             evidence: seedEvidence,
@@ -182,7 +182,7 @@ async function improvedFreeflowObservation(fixture) {
     const evidence = expandedResult?.evidence?.[0] ?? seedEvidence;
     const evidenceExcerpt = evidence?.excerpt ?? "";
     return {
-        toolPathUsed: "freeflowRetrieve repo query + expand",
+        toolPathUsed: "freeflowSearch repo query + expand",
         proxyCalls: expandedResult ? 2 : 1,
         rawBytes: await repoRawBytes(fixture.root),
         contextBytes: byteLength(JSON.stringify(queryResult)) + byteLength(JSON.stringify(expandedResult ?? {})),
@@ -437,7 +437,7 @@ function skippedExternalTools() {
     ];
 }
 function defaultReportPath() {
-    return resolve(process.cwd(), "plugins/freeflow/evals/reports/runtime/output-router-codex-qa-benchmark-1-report.md");
+    return resolve(process.cwd(), "evals/reports/runtime/output-router-codex-qa-benchmark-1-report.md");
 }
 async function main() {
     const { iterations, reportPath, jsonReportPath } = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: defaultReportPath() });

package/{plugins/freeflow/router/dist → router/dist/benchmarks}/command-benchmarks.d.ts

@@ -1,4 +1,4 @@
-import type { CommandParserMetadata, ExecutionStatus } from "./types.js";
+import type { CommandParserMetadata, ExecutionStatus } from "../config/types.js";
 export interface RunCommandBenchmarksOptions {
     iterations?: number;
     generatedAt?: string;

package/{plugins/freeflow/router/dist → router/dist/benchmarks}/command-benchmarks.js

@@ -5,8 +5,8 @@ import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { performance } from "node:perf_hooks";
 import { approximateTokens, averagePercent as average, defaultJsonRunReportPath, escapeMarkdownTableCell as escapeTable, formatPercent, latencySummary, medianPercent as median, normalizeIterations, parseBenchmarkCliArgs, reductionPercent, writeBenchmarkReportPair, } from "./benchmark-harness.js";
-import { freeflowRun } from "./run.js";
-import { createVault, readOutputText } from "./vault.js";
+import { freeflowRun } from "../tools/run.js";
+import { createVault, readOutputText } from "../vault/vault.js";
 const DEFAULT_ITERATIONS = 3;
 const DEFAULT_SKIPPED_EXTERNAL_TOOLS = [
     {
@@ -69,7 +69,7 @@ export function renderCommandBenchmarkReport(report) {
         "npm run bench:router:commands",
         "```",
         "",
-        "The CLI writes machine-readable JSON under `plugins/freeflow/evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
+        "The CLI writes machine-readable JSON under `evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
         "",
         "## Summary",
         "",
@@ -283,9 +283,9 @@ function commandFixtureDefinitions() {
         "src/other.ts(4,1): warning TS6133: 'unused' is declared but its value is never read.",
     ].join("\n");
     const gitOutput = [
-        " M plugins/freeflow/router/src/run.ts",
-        "?? plugins/freeflow/router/src/command-benchmarks.ts",
-        " plugins/freeflow/router/src/run.ts | 12 +++++++++---",
+        " M router/src/tools/run.ts",
+        "?? router/src/benchmarks/command-benchmarks.ts",
+        " router/src/tools/run.ts | 12 +++++++++---",
         " 1 file changed, 9 insertions(+), 3 deletions(-)",
     ].join("\n");
     const repetitiveLog = [
@@ -361,7 +361,7 @@ function commandFixtureDefinitions() {
             stdout: gitOutput,
             stderr: "",
             combined: gitOutput,
-            requiredFacts: ["M plugins/freeflow/router/src/run.ts", "1 file changed"],
+            requiredFacts: ["M router/src/tools/run.ts", "1 file changed"],
             thresholds: { largeOutputLines: 2, largeOutputBytes: 10_000 },
         },
         {
@@ -402,7 +402,8 @@ function commandFixtureDefinitions() {
             combined: repeatedOutput,
             requiredFacts: ["REPEATED_OUTPUT identical command output stays recoverable"],
             repeatRuns: 2,
-            notes: ["Exact duplicate detection returns a compact note while current and prior raw outputs remain recoverable."],
+            thresholds: { largeOutputLines: 10, largeOutputBytes: 10_000 },
+            notes: ["Exact duplicate detection returns a compact note; current metadata points to the prior exact raw output."],
         },
     ];
 }
@@ -468,7 +469,7 @@ async function improvedCommandObservation(fixture, vaultRoot) {
     if (!result) {
         return skippedObservation("improved-freeflow-run", "freeflowRun did not produce a result.");
     }
-    const recovery = await verifyCommandRecovery(vault.root, sessionId, result.outputId, fixture.combined, fixture.requiredFacts);
+    const recovery = await verifyCommandRecovery(vault.root, sessionId, result.recovery?.outputId ?? result.outputId, fixture.combined, fixture.requiredFacts);
     const notes = [result.routing.reason, ...(fixture.notes ?? [])];
     if (outputIds.length > 1) {
         notes.push(`repeatedRuns=${outputIds.length}`);
@@ -625,7 +626,7 @@ function formatCommandCorrectnessChecks(correctness) {
     ].join(" ");
 }
 function defaultReportPath() {
-    return resolve(process.cwd(), "plugins/freeflow/evals/reports/runtime/output-router-command-benchmark-1-report.md");
+    return resolve(process.cwd(), "evals/reports/runtime/output-router-command-benchmark-1-report.md");
 }
 async function runCli() {
     const { iterations, reportPath, jsonReportPath } = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: defaultReportPath() });

package/router/dist/benchmarks/context-mode-normalized-benchmarks.d.ts

@@ -0,0 +1,74 @@
+export interface RunContextModeNormalizedBenchmarksOptions {
+    iterations?: number;
+    generatedAt?: string;
+}
+export interface ContextModeNormalizedBenchmarkReport {
+    generatedAt: string;
+    iterations: number;
+    summary: ContextModeNormalizedSummary;
+    fixtures: ContextModeFixtureResult[];
+    notes: string[];
+}
+export interface ContextModeNormalizedSummary {
+    fixtures: number;
+    modeResults: number;
+    freeflow: ContextModeModeSummary;
+    contextModeProxy: ContextModeModeSummary;
+    freeflowBetterModelVisibleBytes: number;
+    freeflowBetterToolCalls: number;
+    allFreeflowFactsPreserved: boolean;
+    allFreeflowRecoveryAvailable: boolean;
+    publicClaimsAllowed: boolean;
+}
+export interface ContextModeModeSummary {
+    passed: number;
+    failed: number;
+    answerAccurate: number;
+    exactFactsPreserved: number;
+    recoveryAvailable: number;
+    totalRawBytes: number;
+    totalModelVisibleBytes: number;
+    totalModelVisibleTokensApprox: number;
+    totalToolCalls: number;
+    totalStorageBytes: number;
+    weightedModelVisibleReductionPercent: number;
+}
+export interface ContextModeFixtureResult {
+    id: string;
+    title: string;
+    category: ContextModeFixtureCategory;
+    expectedFacts: string[];
+    results: ContextModeModeResult[];
+}
+export interface ContextModeModeResult {
+    mode: ContextModeMode;
+    toolPathUsed: string;
+    rawBytes: number;
+    modelVisibleBytes: number;
+    modelVisibleTokensApprox: number;
+    detailsPayloadBytes: number;
+    storageBytes: number;
+    toolCalls: number;
+    latencyMs: {
+        p50: number;
+        p95: number;
+    };
+    modelVisibleReductionPercent: number;
+    answerAccuracy: boolean;
+    exactFactsPreserved: boolean;
+    recoveryAvailable: boolean;
+    recoveryDetail: string;
+    notes: string[];
+}
+type ContextModeFixtureCategory = "command" | "docs" | "logs" | "json-csv" | "repo-search" | "batch";
+type ContextModeMode = "context-mode-normalized-proxy" | "freeflow-owned-tools";
+export declare function runContextModeNormalizedBenchmarks(options?: RunContextModeNormalizedBenchmarksOptions): Promise<ContextModeNormalizedBenchmarkReport>;
+export declare function renderContextModeNormalizedBenchmarkReport(report: ContextModeNormalizedBenchmarkReport): string;
+export interface WriteContextModeNormalizedBenchmarkReportsOptions {
+    jsonReportPath?: string | false;
+}
+export declare function writeContextModeNormalizedBenchmarkReports(report: ContextModeNormalizedBenchmarkReport, markdownReportPath: string, options?: WriteContextModeNormalizedBenchmarkReportsOptions): Promise<{
+    markdown: string;
+    json?: string;
+}>;
+export {};