@hassangameryt/freeflow 0.1.0 → 0.2.0

package/plugins/freeflow/router/dist/benchmark-harness.d.ts

@@ -0,0 +1,32 @@
+export interface LatencySummary {
+    p50: number;
+    p95: number;
+}
+export interface BenchmarkCliDefaults {
+    reportPath: string;
+}
+export interface ParsedBenchmarkCliArgs {
+    iterations?: number;
+    reportPath: string;
+    jsonReportPath?: string | false;
+}
+export declare function normalizeIterations(value: unknown, defaultIterations?: number): number;
+export declare function approximateTokens(bytes: number): number;
+export declare function percentile(values: readonly number[], quantile: number): number;
+export declare function latencySummary(values: readonly number[]): LatencySummary;
+export declare function averagePercent(values: readonly number[]): number;
+export declare function medianPercent(values: readonly number[]): number;
+export declare function reductionPercent(raw: number, routed: number): number;
+export declare function formatPercent(value: number): string;
+export declare function escapeMarkdownTableCell(value: string): string;
+export declare function defaultJsonRunReportPath(markdownReportPath: string): string;
+export declare function parseBenchmarkCliArgs(argv: readonly string[], defaults: BenchmarkCliDefaults): ParsedBenchmarkCliArgs;
+export declare function writeBenchmarkReportPair<TReport>(options: {
+    report: TReport;
+    markdownReportPath: string;
+    jsonReportPath?: string | false | undefined;
+    renderMarkdown(report: TReport): string;
+}): Promise<{
+    markdown: string;
+    json?: string;
+}>;

package/plugins/freeflow/router/dist/benchmark-harness.js

@@ -0,0 +1,92 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { basename, dirname, resolve } from "node:path";
+export function normalizeIterations(value, defaultIterations = 3) {
+    if (typeof value === "number" && Number.isFinite(value) && value > 0) {
+        return Math.max(1, Math.floor(value));
+    }
+    if (Number.isFinite(defaultIterations) && defaultIterations > 0) {
+        return Math.max(1, Math.floor(defaultIterations));
+    }
+    return 1;
+}
+export function approximateTokens(bytes) {
+    return Math.ceil(bytes / 4);
+}
+export function percentile(values, quantile) {
+    if (values.length === 0) {
+        return 0;
+    }
+    const sorted = [...values].sort((a, b) => a - b);
+    const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * quantile) - 1));
+    return sorted[index] ?? 0;
+}
+export function latencySummary(values) {
+    return {
+        p50: percentile(values, 0.5),
+        p95: percentile(values, 0.95),
+    };
+}
+export function averagePercent(values) {
+    if (values.length === 0) {
+        return 0;
+    }
+    return roundPercent(values.reduce((sum, value) => sum + value, 0) / values.length);
+}
+export function medianPercent(values) {
+    return roundPercent(percentile(values, 0.5));
+}
+export function reductionPercent(raw, routed) {
+    if (raw <= 0) {
+        return 0;
+    }
+    return roundPercent(((raw - routed) / raw) * 100);
+}
+export function formatPercent(value) {
+    return `${value.toFixed(2)}%`;
+}
+export function escapeMarkdownTableCell(value) {
+    return value.replace(/\|/g, "\\|").replace(/\n/g, " ");
+}
+export function defaultJsonRunReportPath(markdownReportPath) {
+    const markdownName = basename(markdownReportPath);
+    const jsonName = markdownName.endsWith(".md") ? `${markdownName.slice(0, -".md".length)}.json` : `${markdownName}.json`;
+    return resolve(process.cwd(), "plugins/freeflow/evals/runs/output-router", jsonName);
+}
+export function parseBenchmarkCliArgs(argv, defaults) {
+    let iterations;
+    let reportPath = defaults.reportPath;
+    let jsonReportPath;
+    for (const arg of argv) {
+        if (arg.startsWith("--iterations=")) {
+            const value = Number(arg.slice("--iterations=".length));
+            if (Number.isFinite(value) && value > 0) {
+                iterations = value;
+            }
+        }
+        else if (arg.startsWith("--report=")) {
+            reportPath = resolve(process.cwd(), arg.slice("--report=".length));
+        }
+        else if (arg.startsWith("--json-report=")) {
+            const value = arg.slice("--json-report=".length);
+            jsonReportPath = value === "off" ? false : resolve(process.cwd(), value);
+        }
+    }
+    const parsed = iterations === undefined ? { reportPath } : { iterations, reportPath };
+    return jsonReportPath === undefined ? parsed : { ...parsed, jsonReportPath };
+}
+export async function writeBenchmarkReportPair(options) {
+    await mkdir(dirname(options.markdownReportPath), { recursive: true });
+    await writeFile(options.markdownReportPath, options.renderMarkdown(options.report), "utf8");
+    if (options.jsonReportPath === false) {
+        return { markdown: options.markdownReportPath };
+    }
+    if (options.jsonReportPath) {
+        await mkdir(dirname(options.jsonReportPath), { recursive: true });
+        await writeFile(options.jsonReportPath, `${JSON.stringify(options.report, null, 2)}\n`, "utf8");
+        return { markdown: options.markdownReportPath, json: options.jsonReportPath };
+    }
+    return { markdown: options.markdownReportPath };
+}
+function roundPercent(value) {
+    return Math.round(value * 100) / 100;
+}

package/plugins/freeflow/router/dist/benchmarks.d.ts

@@ -0,0 +1,102 @@
+export interface RunRouterBenchmarksOptions {
+    iterations?: number;
+    generatedAt?: string;
+}
+export interface RouterBenchmarkReport {
+    generatedAt: string;
+    iterations: number;
+    summary: RouterBenchmarkSummary;
+    fixtures: RouterBenchmarkFixtureResult[];
+    skippedExternalTools: SkippedExternalTool[];
+}
+export interface RouterBenchmarkSummary {
+    fixtures: number;
+    modeResults: number;
+    improved: ModeSummary;
+    nativeBaseline: ModeSummary;
+    freeflowBaseline: ModeSummary;
+    generatedFalsePositiveCount: number;
+    sandboxFailureFixed: boolean;
+}
+export interface ModeSummary {
+    passed: number;
+    failed: number;
+    skipped: number;
+    pathCorrect: number;
+    spanCorrect: number;
+    excerptComplete: number;
+    generatedFalsePositiveCount: number;
+    totalRawBytes: number;
+    totalRoutedBytes: number;
+    totalRawTokensApprox: number;
+    totalRoutedTokensApprox: number;
+    weightedByteReductionPercent: number;
+    weightedTokenReductionPercent: number;
+    averageByteReductionPercent: number;
+    medianByteReductionPercent: number;
+    averageTokenReductionPercent: number;
+    medianTokenReductionPercent: number;
+}
+export interface RouterBenchmarkFixtureResult {
+    id: string;
+    title: string;
+    kind: BenchmarkKind;
+    expected: BenchmarkExpected;
+    results: RouterBenchmarkModeResult[];
+}
+export interface RouterBenchmarkModeResult {
+    mode: BenchmarkMode;
+    toolPathUsed: string;
+    skipped: boolean;
+    skipReason?: string;
+    rawBytes: number;
+    rawTokensApprox: number;
+    routedBytes: number;
+    routedTokensApprox: number;
+    byteReductionPercent: number;
+    tokenReductionPercent: number;
+    latencyMs: {
+        p50: number;
+        p95: number;
+    };
+    actualPath?: string;
+    actualLines?: string;
+    correctness: CorrectnessResult;
+    recovery: RecoveryResult;
+    notes: string[];
+}
+export interface BenchmarkExpected {
+    path?: string;
+    pathIncludes?: string;
+    lines?: string;
+    requiredExcerpt?: string[];
+}
+export interface CorrectnessResult {
+    passed: boolean;
+    pathCorrect: boolean;
+    spanCorrect: boolean;
+    excerptComplete: boolean;
+    generatedFalsePositive: boolean;
+}
+export interface RecoveryResult {
+    status: "passed" | "failed" | "not-applicable" | "skipped";
+    detail: string;
+}
+export interface SkippedExternalTool {
+    name: string;
+    reason: string;
+}
+type BenchmarkKind = "repo-query" | "repo-expand" | "vault-query";
+type BenchmarkMode = "native-baseline-proxy" | "pre-hardening-freeflow-proxy" | "improved-freeflow-router";
+export declare function runRouterBenchmarks(options?: RunRouterBenchmarksOptions): Promise<RouterBenchmarkReport>;
+export declare function renderRouterBenchmarkReport(report: RouterBenchmarkReport): string;
+export declare function writeRouterBenchmarkReport(report: RouterBenchmarkReport, reportPath: string): Promise<void>;
+export declare function writeRouterBenchmarkJsonReport(report: RouterBenchmarkReport, reportPath: string): Promise<void>;
+export interface WriteRouterBenchmarkReportsOptions {
+    jsonReportPath?: string | false;
+}
+export declare function writeRouterBenchmarkReports(report: RouterBenchmarkReport, markdownReportPath: string, options?: WriteRouterBenchmarkReportsOptions): Promise<{
+    markdown: string;
+    json?: string;
+}>;
+export {};