@kenkaiiii/ggcoder 5.74.3 → 5.75.0

package/dist/core/semantic-search-benchmark.js

@@ -1,211 +0,0 @@
-/**
- * Semantic AST-chunk retrieval vs whole-file reads — real-API measurement of
- * whether Feature #3 is worth building.
- *
- * The claim (cocoindex-code / oh-my-pi): replacing "grep then read whole files"
- * with "retrieve only the relevant AST chunks" cuts ~70% of the tokens an agent
- * spends locating code, with no loss of answer quality. We test that directly on
- * OUR OWN repo against a live model.
- *
- * For a set of natural-language questions about real files in this repo, we build
- * three context strategies and ask the model the same question with each:
- *
- *   BASELINE (whole-file): deliver the FULL text of the top files a lexical
- *   grep would surface — this is what the agent reads today (read + grep).
- *
- *   SEMANTIC (AST chunks): parse every file into top-level declarations
- *   (functions / classes / interfaces / consts), rank chunks with a real BM25
- *   retriever, and deliver only the top-k chunks. No embedding model needed; a
- *   learned embedding retriever would land between BM25 and ORACLE.
- *
- *   ORACLE (upper bound): deliver only the hand-labelled answer chunk(s) — the
- *   best case any retriever could achieve.
- *
- * We measure, per question and strategy: INPUT tokens delivered (the headline
- * cost) and whether the model's answer was correct (deterministic keyword grade).
- * The verdict: does SEMANTIC reach BASELINE-level correctness at a fraction of
- * the input tokens?
- *
- * Usage:
- *   npx tsx src/core/semantic-search-benchmark.ts
- *
- * Env overrides:
- *   GG_SS_PROVIDER / GG_SS_MODEL   (default openai / gpt-6.1-sol)
- *   GG_SS_TOPK                     (chunks/files delivered, default 3)
- */
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { stream } from "@kenkaiiii/gg-ai";
-import { AuthStorage } from "./auth-storage.js";
-import { chunkFile, bm25Rank, rankFiles } from "./code-retrieval.js";
-const HERE = path.dirname(fileURLToPath(import.meta.url));
-const SRC = path.resolve(HERE, ".."); // packages/ggcoder/src
-const QUESTIONS = [
-    {
-        q: "Which method resolves provider credentials and auto-refreshes expired OAuth tokens, and what happens if it is not logged in?",
-        files: ["core/auth-storage.ts", "core/loop-breaker.ts", "tools/edit-diff.ts"],
-        oracle: { file: "core/auth-storage.ts", symbol: "resolveCredentials" },
-        mustInclude: ["resolvecredentials", "refresh"],
-    },
-    {
-        q: "What function performs fuzzy text matching for the edit tool, and how does it tolerate indentation drift?",
-        files: ["tools/edit-diff.ts", "core/auth-storage.ts", "core/checkpoint-store.ts"],
-        oracle: { file: "tools/edit-diff.ts", symbol: "fuzzyFindText" },
-        mustInclude: ["fuzzyfindtext", "indent"],
-    },
-    {
-        q: "What restore modes does the checkpoint / rewind system support?",
-        files: ["core/checkpoint-store.ts", "core/loop-breaker.ts", "tools/edit-diff.ts"],
-        oracle: { file: "core/checkpoint-store.ts", symbol: "RestoreMode" },
-        mustInclude: ["code", "conversation", "both"],
-    },
-];
-function sleep(ms) {
-    return new Promise((r) => setTimeout(r, ms));
-}
-async function ask(provider, model, c, context, question) {
-    const messages = [
-        {
-            role: "system",
-            content: "You answer questions about a codebase using ONLY the provided context. " +
-                "Be specific: name the exact functions/types involved. If the context is insufficient, say so. " +
-                "Answer in 1-3 sentences.",
-        },
-        { role: "user", content: `CONTEXT:\n${context}\n\nQUESTION: ${question}` },
-    ];
-    let lastErr;
-    for (let attempt = 0; attempt < 4; attempt++) {
-        try {
-            let text = "";
-            const result = stream({
-                provider: provider,
-                model,
-                messages,
-                maxTokens: 512,
-                apiKey: c.apiKey,
-                baseUrl: c.baseUrl,
-                accountId: c.accountId,
-            });
-            for await (const event of result) {
-                if (event.type === "text_delta")
-                    text += event.text;
-            }
-            const response = await result.response;
-            const content = response.message.content;
-            const finalText = typeof content === "string"
-                ? content
-                : content
-                    .filter((p) => p.type === "text")
-                    .map((p) => p.text ?? "")
-                    .join("");
-            return {
-                text: finalText || text,
-                inputTokens: response.usage.inputTokens,
-                outputTokens: response.usage.outputTokens,
-            };
-        }
-        catch (err) {
-            lastErr = err;
-            await sleep(2000 * (attempt + 1));
-        }
-    }
-    throw lastErr;
-}
-export function grade(answer, mustInclude) {
-    const a = answer.toLowerCase();
-    return mustInclude.every((t) => a.includes(t));
-}
-async function main() {
-    const provider = process.env.GG_SS_PROVIDER ?? "openai";
-    const model = process.env.GG_SS_MODEL ?? "gpt-6.1-sol";
-    const topK = Math.max(1, parseInt(process.env.GG_SS_TOPK ?? "3", 10));
-    const auth = new AuthStorage();
-    await auth.load();
-    const cr = await auth.resolveCredentials(provider);
-    const creds = { apiKey: cr.accessToken, baseUrl: cr.baseUrl, accountId: cr.accountId };
-    console.log(`\n🔎 Semantic-search benchmark — ${provider}/${model} (top-${topK})\n`);
-    const rows = [];
-    for (const q of QUESTIONS) {
-        // Load the real corpus files; skip the question if any are missing/moved.
-        const files = new Map();
-        let missing = false;
-        for (const rel of q.files) {
-            const abs = path.join(SRC, rel);
-            if (!fs.existsSync(abs)) {
-                console.log(`   ⚠ skipping question — missing ${rel}`);
-                missing = true;
-                break;
-            }
-            files.set(rel, fs.readFileSync(abs, "utf-8"));
-        }
-        if (missing)
-            continue;
-        // BASELINE context: full text of the top-k files a grep would surface.
-        const baseFiles = rankFiles(q.q, files, topK);
-        const baseContext = baseFiles.map((f) => `// FILE: ${f}\n${files.get(f)}`).join("\n\n");
-        // SEMANTIC context: top-k AST chunks across all corpus files.
-        const allChunks = [...files].flatMap(([rel, src]) => chunkFile(rel, src));
-        const semChunks = bm25Rank(q.q, allChunks, topK);
-        const semContext = semChunks.map((c) => `// ${c.file} → ${c.symbol}\n${c.text}`).join("\n\n");
-        // ORACLE context: just the labelled answer chunk.
-        const oracleChunk = allChunks.find((c) => c.file === q.oracle.file && c.symbol === q.oracle.symbol);
-        const oracleContext = oracleChunk
-            ? `// ${oracleChunk.file} → ${oracleChunk.symbol}\n${oracleChunk.text}`
-            : semContext;
-        process.stdout.write(`▶ ${q.q.slice(0, 64)}…\n`);
-        await sleep(1200);
-        const base = await ask(provider, model, creds, baseContext, q.q);
-        await sleep(1200);
-        const sem = await ask(provider, model, creds, semContext, q.q);
-        await sleep(1200);
-        const oracle = await ask(provider, model, creds, oracleContext, q.q);
-        const row = {
-            q: q.q.slice(0, 40),
-            baseInTok: base.inputTokens,
-            baseOk: grade(base.text, q.mustInclude),
-            semInTok: sem.inputTokens,
-            semOk: grade(sem.text, q.mustInclude),
-            oracleInTok: oracle.inputTokens,
-            oracleOk: grade(oracle.text, q.mustInclude),
-        };
-        rows.push(row);
-        process.stdout.write(`   baseline ${row.baseInTok} in tok ${row.baseOk ? "OK" : "FAIL"} | ` +
-            `semantic ${row.semInTok} in tok ${row.semOk ? "OK" : "FAIL"} | ` +
-            `oracle ${row.oracleInTok} in tok ${row.oracleOk ? "OK" : "FAIL"}\n\n`);
-    }
-    if (rows.length === 0) {
-        console.log("No questions ran (corpus files not found).");
-        return;
-    }
-    // ── Report ──
-    console.log("══════════════════════ RESULTS ══════════════════════\n");
-    console.log("Question                                 | base in-tok | sem in-tok | oracle | ok b/s/o");
-    for (const r of rows) {
-        console.log(`${r.q.padEnd(40)} | ${String(r.baseInTok).padStart(11)} | ${String(r.semInTok).padStart(10)} | ` +
-            `${String(r.oracleInTok).padStart(6)} | ${r.baseOk ? "1" : "0"}/${r.semOk ? "1" : "0"}/${r.oracleOk ? "1" : "0"}`);
-    }
-    const sum = (f) => rows.reduce((s, r) => s + f(r), 0);
-    const baseIn = sum((r) => r.baseInTok);
-    const semIn = sum((r) => r.semInTok);
-    const oracleIn = sum((r) => r.oracleInTok);
-    console.log(`\nInput tokens to answer: baseline ${baseIn} | semantic ${semIn} ` +
-        `(${(((baseIn - semIn) / baseIn) * 100).toFixed(0)}% fewer) | oracle ${oracleIn} ` +
-        `(${(((baseIn - oracleIn) / baseIn) * 100).toFixed(0)}% fewer)`);
-    console.log(`Correctness: baseline ${rows.filter((r) => r.baseOk).length}/${rows.length} | ` +
-        `semantic ${rows.filter((r) => r.semOk).length}/${rows.length} | ` +
-        `oracle ${rows.filter((r) => r.oracleOk).length}/${rows.length}`);
-    console.log(`\nVerdict: worth building if SEMANTIC keeps correctness ≈ baseline while cutting input tokens. ` +
-        `cocoindex claims ~70% fewer; ORACLE shows the ceiling a better (embedding) retriever could reach.\n`);
-}
-// Run when executed directly (not when imported by tests).
-const isDirectRun = process.argv[1]?.endsWith("semantic-search-benchmark.ts") ||
-    process.argv[1]?.endsWith("semantic-search-benchmark.js") ||
-    process.argv[1]?.endsWith("semantic-search-benchmark");
-if (isDirectRun) {
-    main().catch((err) => {
-        console.error("Benchmark failed:", err);
-        process.exit(1);
-    });
-}
-//# sourceMappingURL=semantic-search-benchmark.js.map

package/dist/core/semantic-search-benchmark.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"semantic-search-benchmark.js","sourceRoot":"","sources":["../../src/core/semantic-search-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAA8C,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,uBAAuB;AAc7D,MAAM,SAAS,GAAe;IAC5B;QACE,CAAC,EAAE,8HAA8H;QACjI,KAAK,EAAE,CAAC,sBAAsB,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;QAC7E,MAAM,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,EAAE,oBAAoB,EAAE;QACtE,WAAW,EAAE,CAAC,oBAAoB,EAAE,SAAS,CAAC;KAC/C;IACD;QACE,CAAC,EAAE,2GAA2G;QAC9G,KAAK,EAAE,CAAC,oBAAoB,EAAE,sBAAsB,EAAE,0BAA0B,CAAC;QACjF,MAAM,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,eAAe,EAAE;QAC/D,WAAW,EAAE,CAAC,eAAe,EAAE,QAAQ,CAAC;KACzC;IACD;QACE,CAAC,EAAE,iEAAiE;QACpE,KAAK,EAAE,CAAC,0BAA0B,EAAE,sBAAsB,EAAE,oBAAoB,CAAC;QACjF,MAAM,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,MAAM,EAAE,aAAa,EAAE;QACnE,WAAW,EAAE,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC;KAC9C;CACF,CAAC;AAYF,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAQD,KAAK,UAAU,GAAG,CAChB,QAAgB,EAChB,KAAa,EACb,CAAQ,EACR,OAAe,EACf,QAAgB;IAEhB,MAAM,QAAQ,GAAc;QAC1B;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EACL,yEAAyE;gBACzE,gGAAgG;gBAChG,0BAA0B;SAC7B;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,OAAO,iBAAiB,QAAQ,EAAE,EAAE;KAC3E,CAAC;IACF,IAAI,OAAgB,CAAC;IACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,CAAC;gBACpB,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;gBACL,QAAQ;gBACR,SAAS,EAAE,GAAG;gBACd,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,SAAS,EAAE,CAAC,CAAC,SAAS;aACvB,CAAC,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAoC,EAAE,CAAC;gBAC/D,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;oBAAE,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;YACtD,CAAC;YACD,MAAM,QAAQ,GAAuC,MAAM,MAAM,CAAC,QAAQ,CAAC;YAC3E,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC;YACzC,MAAM,SAAS,GACb,OAAO,OAAO,KAAK,QAAQ;gBACzB,CAAC,CAAC,OAAO;gBACT,CAAC,CAAE,OAAkD;qBAChD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;qBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;qBACxB,IAAI,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO;gBACL,IAAI,EAAE,SAAS,IAAI,IAAI;gBACvB,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;gBACvC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC;YACd,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,MAAc,EAAE,WAAqB;IACzD,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAC/B,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAcD,KAAK,UAAU,IAAI;IACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,QAAQ,CAAC;IACxD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,aAAa,CAAC;IACvD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;IAEtE,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;IAC/B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,KAAK,GAAU,EAAE,MAAM,EAAE,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,CAAC;IAE9F,OAAO,CAAC,GAAG,CAAC,oCAAoC,QAAQ,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,CAAC;IAErF,MAAM,IAAI,GAAU,EAAE,CAAC;IAEvB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,0EAA0E;QAC1E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;gBACvD,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM;YACR,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,OAAO;YAAE,SAAS;QAEtB,uEAAuE;QACvE,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExF,8DAA8D;QAC9D,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE9F,kDAAkD;QAClD,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAChE,CAAC;QACF,MAAM,aAAa,GAAG,WAAW;YAC/B,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,MAAM,WAAW,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACvE,CAAC,CAAC,UAAU,CAAC;QAEf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAErE,MAAM,GAAG,GAAQ;YACf,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACnB,SAAS,EAAE,IAAI,CAAC,WAAW;YAC3B,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC;YACvC,QAAQ,EAAE,GAAG,CAAC,WAAW;YACzB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC;YACrC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC;SAC5C,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,eAAe,GAAG,CAAC,SAAS,WAAW,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK;YACpE,YAAY,GAAG,CAAC,QAAQ,WAAW,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK;YACjE,UAAU,GAAG,CAAC,WAAW,WAAW,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CACzE,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAC1D,OAAO;IACT,CAAC;IAED,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CACT,yFAAyF,CAC1F,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACT,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK;YAC/F,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CACpH,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CACT,sCAAsC,MAAM,eAAe,KAAK,GAAG;QACjE,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,QAAQ,GAAG;QAClF,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAClE,CAAC;IACF,OAAO,CAAC,GAAG,CACT,yBAAyB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK;QAC9E,YAAY,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK;QAClE,UAAU,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CACnE,CAAC;IACF,OAAO,CAAC,GAAG,CACT,iGAAiG;QAC/F,qGAAqG,CACxG,CAAC;AACJ,CAAC;AAED,2DAA2D;AAC3D,MAAM,WAAW,GACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,8BAA8B,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,8BAA8B,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,2BAA2B,CAAC,CAAC;AAEzD,IAAI,WAAW,EAAE,CAAC;IAChB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/core/speed-benchmark.d.ts

@@ -1,132 +0,0 @@
-/**
- * Speed benchmark harness — measures the latency impact of speedProfile
- * optimizations (1-h cache TTL + pre-warming) against the current baseline
- * (5-min cache TTL, no pre-warm).
- *
- * Uses a mock streaming provider that simulates realistic provider timing:
- *   - Cold prefill (cache miss): proportional to uncached input tokens
- *   - Warm prefill (cache hit): 10× faster (cache reads cost 0.1×)
- *   - Output streaming: at a configurable token rate
- *   - Cache TTL: configurable (5 min baseline vs 1 h optimized)
- *
- * No real API calls are made — the mock uses real `setTimeout` for prefill and
- * output-token latency (so wall-clock measurements are meaningful) but a
- * virtual clock for inter-turn delays (so multi-minute gaps are instant).
- *
- * Run via vitest:  npx vitest run src/core/speed-benchmark.test.ts
- * Or as a script:  npx tsx src/core/speed-benchmark.ts
- */
-import { StreamResult, type StreamOptions } from "@kenkaiiii/gg-ai";
-export interface MockTimingConfig {
-    /** ms per uncached input token during prefill (cold start). */
-    coldPrefillMsPerToken: number;
-    /** ms per cached input token during prefill (cache hit). 10× faster. */
-    warmPrefillMsPerToken: number;
-    /** ms per output token (determines streaming rate). ~15ms = 66 tok/s. */
-    outputMsPerToken: number;
-    /** Cache TTL in ms. 5_000 = baseline (5 min), 3_600_000 = optimized (1 h). */
-    cacheTtlMs: number;
-    /** Fixed network overhead per request (TCP + TLS + auth). */
-    networkOverheadMs: number;
-    /** Default output tokens per turn if not specified by the workload. */
-    defaultOutputTokens: number;
-}
-export declare const REALISTIC_TIMING: MockTimingConfig;
-/**
- * A mock streaming provider that simulates LLM timing with cache semantics.
- * Registered as provider "benchmark-mock" so the real agent loop code path
- * is exercised end-to-end.
- */
-export declare class MockBenchmarkProvider {
-    private cache;
-    private config;
-    /** Virtual clock — lets the benchmark simulate multi-minute gaps between
-     *  turns without actually sleeping. The cache TTL check uses this, not
-     *  Date.now(). Only prefill/output latency uses real setTimeout. */
-    private virtualNow;
-    readonly stats: {
-        cacheHits: number;
-        cacheMisses: number;
-        cacheWrites: number;
-        cacheEvictions: number;
-        totalPrefillMs: number;
-        totalOutputMs: number;
-        totalNetworkMs: number;
-        turns: number;
-    };
-    constructor(config?: Partial<MockTimingConfig>);
-    /** Update config (e.g., switch TTL for baseline vs optimized run). */
-    setConfig(config: Partial<MockTimingConfig>): void;
-    /** Advance the virtual clock (for cache TTL simulation without real sleeping). */
-    advanceClock(ms: number): void;
-    /** Clear all cache state and stats (between benchmark runs). */
-    reset(): void;
-    /** Number of entries currently in the mock cache. */
-    getCacheSize(): number;
-    /** Force a cache write (simulates pre-warming). */
-    prewarm(cacheKey: string, tokenCount: number): void;
-    stream(options: StreamOptions): StreamResult;
-    private runStream;
-    /** Compute a stable cache key from the system prompt + tool names. */
-    private computeCacheKey;
-}
-export interface WorkloadTurn {
-    /** User message content. */
-    prompt: string;
-    /** Delay before this turn (simulates user think time). Default: 0. */
-    delayMs?: number;
-    /** Override output tokens for this turn. */
-    outputTokens?: number;
-}
-export interface Workload {
-    name: string;
-    /** System prompt (simulates a realistic coding-agent system prompt). */
-    systemPrompt: string;
-    /** Tool definitions (names only for the mock). */
-    toolNames: string[];
-    turns: WorkloadTurn[];
-}
-export interface TurnMetrics {
-    turnNumber: number;
-    promptPreview: string;
-    delayBeforeTurnMs: number;
-    ttftMs: number;
-    cacheHit: boolean;
-    inputTokens: number;
-    outputTokens: number;
-    wallClockMs: number;
-}
-export interface BenchmarkResult {
-    name: string;
-    config: MockTimingConfig;
-    prewarmed: boolean;
-    turns: TurnMetrics[];
-    totalWallClockMs: number;
-    totalTtftMs: number;
-    cacheHits: number;
-    cacheMisses: number;
-    cacheHitRate: number;
-}
-/** Run a workload against the mock provider and collect per-turn metrics. */
-export declare function runBenchmark(workload: Workload, config: MockTimingConfig, options?: {
-    prewarm?: boolean;
-    name?: string;
-}): Promise<BenchmarkResult>;
-export interface ComparisonResult {
-    baseline: BenchmarkResult;
-    optimized: BenchmarkResult;
-    wallClockImprovement: number;
-    ttftImprovement: number;
-    cacheHitRateImprovement: number;
-}
-export declare function compareResults(baseline: BenchmarkResult, optimized: BenchmarkResult): ComparisonResult;
-/** Format a benchmark result as a readable table. */
-export declare function formatResultTable(result: BenchmarkResult): string;
-/** Format a side-by-side comparison. */
-export declare function formatComparison(comparison: ComparisonResult): string;
-/** A realistic multi-turn coding workload with time gaps that expose the
- *  5-min vs 1-h TTL difference. */
-export declare function createDefaultWorkload(): Workload;
-/** Run baseline vs optimized and return the comparison. Uses scaled-down
- *  timing (10× faster than real) so the benchmark completes in seconds. */
-export declare function runFullBenchmark(): Promise<ComparisonResult>;