knodin 0.5.0

package/dist/src/competitive-runner.js

@@ -0,0 +1,453 @@
+import fs from "node:fs";
+import path from "node:path";
+import { COMPETITIVE_MANIFEST, } from "./competitive-manifest.js";
+const RESERVED_CASE_KEYS = new Set([
+    "id",
+    "axis",
+    "parameters",
+    "note",
+    "blocked",
+    "mutation",
+    "knodin",
+    "oracle",
+    "caseContract",
+    "partial",
+]);
+function responseOf(side) {
+    if (!side || side.ok !== true)
+        return null;
+    const response = side.response;
+    return typeof response === "string" && response.length > 0 ? response : null;
+}
+function normalizedWords(value) {
+    return new Set(value
+        .toLowerCase()
+        .split(/[^a-z0-9_.-]+/)
+        .filter((word) => word.length >= 4));
+}
+function runCheckedInOracle(oracleId, test, competitor) {
+    const contenderResponse = responseOf(findCompetitorSide(test, competitor));
+    const knodinResponse = responseOf(test.knodin);
+    if (!contenderResponse || !knodinResponse) {
+        return {
+            id: oracleId,
+            status: "unavailable",
+            reason: "Both successful response payloads are required for the checked-in oracle.",
+        };
+    }
+    const bothContain = (pattern) => pattern.test(contenderResponse) && pattern.test(knodinResponse);
+    let passed = false;
+    switch (oracleId) {
+        case "contains-create-engine":
+            passed = bothContain(/create[\s_-]*engine/i);
+            break;
+        case "contains-index":
+            passed = bothContain(/index|fresh|status/i);
+            break;
+        case "pack-two-files":
+            passed =
+                bothContain(/src\/context\.ts/i) &&
+                    bothContain(/src\/cli-args\.ts/i) &&
+                    !/(?:^|\/)[^/\n]*\.spec\.ts/i.test(contenderResponse) &&
+                    !/(?:^|\/)[^/\n]*\.spec\.ts/i.test(knodinResponse);
+            break;
+        case "packed-read": {
+            const contenderWords = normalizedWords(contenderResponse);
+            const knodinWords = normalizedWords(knodinResponse);
+            const shared = [...contenderWords].filter((word) => knodinWords.has(word));
+            passed = shared.length >= 2;
+            break;
+        }
+        case "grep-create-engine":
+            passed = bothContain(/createEngine/i);
+            break;
+        case "architecture-facet": {
+            const facet = String(test.id ?? "")
+                .replace(/^architecture-/, "")
+                .replace("_", " ");
+            passed = facet.length > 0 && bothContain(new RegExp(facet.replace(" ", "[ _-]?"), "i"));
+            break;
+        }
+    }
+    return {
+        id: oracleId,
+        status: passed ? "passed" : "failed",
+        ...(passed ? {} : { reason: `Checked-in ${oracleId} oracle did not match both responses.` }),
+    };
+}
+export function applyCompetitiveCaseContracts(competitor, raw, fixtureRevision) {
+    const source = raw && typeof raw === "object"
+        ? raw
+        : {};
+    const results = Array.isArray(source.results) ? source.results : [];
+    const mappings = COMPETITIVE_MANIFEST.find((entry) => entry.id === competitor)?.caseMappings ?? [];
+    return {
+        ...source,
+        results: results.map((test) => {
+            const mapping = mappings.find((candidate) => new RegExp(candidate.idPattern).test(String(test.id ?? "")));
+            if (!mapping) {
+                return {
+                    ...test,
+                    caseContract: {
+                        useCase: String(test.axis ?? test.id ?? "unlabeled competitive case"),
+                        fixtureRevision,
+                        mode: "warm",
+                        scope: {
+                            competitorOperation: "unmatched",
+                            knodinOperation: "unmatched",
+                            dimensions: [],
+                        },
+                        budgets: { bytes: null, tokens: null, items: null },
+                        disposition: {
+                            status: "incomparable",
+                            reason: "No checked-in equivalent-operation mapping exists for this case ID; latency is excluded.",
+                        },
+                        oracle: {
+                            status: "unavailable",
+                            reason: "Correctness is not evaluated for an incomparable case.",
+                        },
+                    },
+                };
+            }
+            return {
+                ...test,
+                caseContract: {
+                    useCase: mapping.useCase,
+                    fixtureRevision,
+                    mode: mapping.mode,
+                    scope: mapping.scope,
+                    budgets: mapping.budgets,
+                    disposition: { status: "comparable", reason: mapping.reason },
+                    oracle: runCheckedInOracle(mapping.oracleId, test, competitor),
+                },
+            };
+        }),
+    };
+}
+export function selectCompetitors(manifest, explicitIds, roadmapIds) {
+    const known = new Set(manifest.map((entry) => entry.id));
+    for (const id of explicitIds) {
+        if (!known.has(id))
+            throw new Error(`Unknown competitor: ${id}`);
+    }
+    const requestedRoadmap = new Set(roadmapIds.map((id) => id.toUpperCase()));
+    const knownRoadmap = new Set(manifest.flatMap((entry) => [...entry.roadmap]).map((id) => id.toUpperCase()));
+    for (const id of requestedRoadmap) {
+        if (!knownRoadmap.has(id))
+            throw new Error(`Unknown roadmap item: ${id}`);
+    }
+    const selected = new Set(explicitIds);
+    for (const entry of manifest) {
+        if (entry.roadmap.some((id) => requestedRoadmap.has(id.toUpperCase())))
+            selected.add(entry.id);
+    }
+    if (selected.size === 0 && explicitIds.length === 0 && roadmapIds.length === 0) {
+        for (const entry of manifest)
+            selected.add(entry.id);
+    }
+    return manifest.filter((entry) => selected.has(entry.id)).map((entry) => entry.id);
+}
+export function createRunDirectory(root, timestamp, commit) {
+    const stamp = timestamp.replace(/[-:.]/g, "");
+    const directory = path.join(root, `${stamp}-${commit.slice(0, 12)}`);
+    try {
+        fs.mkdirSync(directory, { recursive: false });
+    }
+    catch (error) {
+        if (error.code === "EEXIST") {
+            throw new Error(`Competitive run directory already exists: ${directory}`);
+        }
+        throw error;
+    }
+    return directory;
+}
+function findCompetitorSide(test, competitor) {
+    const candidates = [
+        competitor,
+        competitor.replaceAll("-", ""),
+        competitor.replaceAll("-", "_"),
+        "competitor",
+        "gitnexus",
+        "graphify",
+        "serena",
+        "grepai",
+        "repomix",
+        "aider",
+        "code2prompt",
+        "claudeContext",
+        "codegraph",
+    ];
+    for (const key of candidates) {
+        const value = test[key];
+        if (value && typeof value === "object" && ("ok" in value || "medianMs" in value)) {
+            return value;
+        }
+    }
+    for (const [key, value] of Object.entries(test)) {
+        if (RESERVED_CASE_KEYS.has(key) || !value || typeof value !== "object")
+            continue;
+        if ("ok" in value || "medianMs" in value)
+            return value;
+    }
+    return undefined;
+}
+export function normalizeRawResult(competitor, raw) {
+    const rawRecord = raw && typeof raw === "object"
+        ? raw
+        : {};
+    const results = Array.isArray(rawRecord.results)
+        ? (rawRecord.results ?? [])
+        : [];
+    const partialRun = rawRecord.partial === true || rawRecord.status === "partial";
+    let paired = 0;
+    let knodinFaster = 0;
+    let competitorFaster = 0;
+    let competitorErrors = 0;
+    let knodinErrors = 0;
+    let blocked = 0;
+    const ratios = [];
+    const precisions = [];
+    const recalls = [];
+    let oracleCases = 0;
+    let oraclePassed = 0;
+    let oracleFailed = 0;
+    let oracleUnavailable = 0;
+    let incomparable = 0;
+    let exactMatches = 0;
+    let exactMismatches = 0;
+    let falsePositives = 0;
+    let hasExactMatch = false;
+    let hasFalsePositives = false;
+    let knodinResponseBytes = 0;
+    let competitorResponseBytes = 0;
+    let hasKnodinResponseBytes = false;
+    let hasCompetitorResponseBytes = false;
+    const estimatedTokens = [];
+    const peakRssMb = [];
+    const knodinPeakRssMb = [];
+    const competitorPeakRssMb = [];
+    const diskBytes = [];
+    const modeRatios = {
+        cold: { p50: [], p95: [] },
+        warm: { p50: [], p95: [] },
+    };
+    const attributedResources = Array.isArray(rawRecord.resources)
+        ? rawRecord.resources
+        : [];
+    for (const resource of attributedResources) {
+        for (const [owner, target] of [
+            ["knodin", knodinPeakRssMb],
+            ["competitor", competitorPeakRssMb],
+        ]) {
+            const measurement = resource[owner];
+            if (measurement &&
+                typeof measurement === "object" &&
+                "rssBytes" in measurement &&
+                typeof measurement.rssBytes === "number" &&
+                measurement.rssBytes > 0) {
+                const megabytes = measurement.rssBytes / 1024 / 1024;
+                target.push(megabytes);
+                peakRssMb.push(megabytes);
+            }
+        }
+    }
+    for (const test of results) {
+        if (test.blocked)
+            blocked++;
+        const contender = findCompetitorSide(test, competitor);
+        const knodin = test.knodin;
+        if (contender && contender.ok === false)
+            competitorErrors++;
+        if (knodin && knodin.ok === false)
+            knodinErrors++;
+        const oracle = test.oracle;
+        const contract = test.caseContract;
+        if (contract?.disposition.status === "incomparable")
+            incomparable++;
+        if (!test.blocked) {
+            if (contract?.disposition.status === "comparable") {
+                oracleCases++;
+                if (contract.oracle.status === "passed")
+                    oraclePassed++;
+                else if (contract.oracle.status === "failed")
+                    oracleFailed++;
+                else
+                    oracleUnavailable++;
+            }
+            else if (!contract) {
+                oracleUnavailable++;
+            }
+        }
+        if (oracle && typeof oracle === "object") {
+            if (contract?.disposition.status !== "comparable")
+                oracleCases++;
+            if (typeof oracle.exactMatch === "boolean") {
+                hasExactMatch = true;
+                if (oracle.exactMatch)
+                    exactMatches++;
+                else
+                    exactMismatches++;
+            }
+            if (typeof oracle.precision === "number")
+                precisions.push(oracle.precision);
+            if (typeof oracle.recall === "number")
+                recalls.push(oracle.recall);
+            if (typeof oracle.falsePositives === "number") {
+                hasFalsePositives = true;
+                falsePositives += oracle.falsePositives;
+            }
+        }
+        if (typeof knodin?.responseBytes === "number") {
+            hasKnodinResponseBytes = true;
+            knodinResponseBytes += knodin.responseBytes;
+        }
+        if (typeof contender?.responseBytes === "number") {
+            hasCompetitorResponseBytes = true;
+            competitorResponseBytes += contender.responseBytes;
+        }
+        for (const value of [knodin?.estimatedTokens, contender?.estimatedTokens])
+            if (typeof value === "number")
+                estimatedTokens.push(value);
+        if (typeof knodin?.peakRssMb === "number") {
+            peakRssMb.push(knodin.peakRssMb);
+            knodinPeakRssMb.push(knodin.peakRssMb);
+        }
+        if (typeof contender?.peakRssMb === "number") {
+            peakRssMb.push(contender.peakRssMb);
+            competitorPeakRssMb.push(contender.peakRssMb);
+        }
+        for (const value of [knodin?.diskBytes, contender?.diskBytes])
+            if (typeof value === "number")
+                diskBytes.push(value);
+        if (!partialRun &&
+            test.partial !== true &&
+            contract?.disposition.status === "comparable" &&
+            contract.oracle.status === "passed" &&
+            contender?.ok === true &&
+            knodin?.ok === true &&
+            typeof contender.medianMs === "number" &&
+            typeof knodin.medianMs === "number" &&
+            knodin.medianMs > 0) {
+            const mode = contract.mode;
+            const contenderP50 = contender.p50Ms ?? contender.medianMs;
+            const knodinP50 = knodin.p50Ms ?? knodin.medianMs;
+            paired++;
+            if (contender.medianMs < knodin.medianMs)
+                competitorFaster++;
+            else if (knodin.medianMs < contender.medianMs)
+                knodinFaster++;
+            ratios.push(contender.medianMs / knodin.medianMs);
+            if (knodinP50 > 0)
+                modeRatios[mode].p50.push(contenderP50 / knodinP50);
+            if (typeof contender.p95Ms === "number" &&
+                typeof knodin.p95Ms === "number" &&
+                knodin.p95Ms > 0) {
+                modeRatios[mode].p95.push(contender.p95Ms / knodin.p95Ms);
+            }
+        }
+    }
+    ratios.sort((a, b) => a - b);
+    const median = (values) => {
+        const sorted = [...values].sort((left, right) => left - right);
+        return sorted.length ? sorted[Math.floor(sorted.length / 2)] : null;
+    };
+    return {
+        competitor,
+        cases: results.length,
+        paired,
+        knodinFaster,
+        competitorFaster,
+        competitorErrors,
+        knodinErrors,
+        blocked,
+        medianRatio: ratios.length ? ratios[Math.floor(ratios.length / 2)] : null,
+        correctness: {
+            oracleCases,
+            oraclePassed,
+            oracleFailed,
+            oracleUnavailable,
+            incomparable,
+            exactMatches: hasExactMatch ? exactMatches : null,
+            exactMismatches: hasExactMatch ? exactMismatches : null,
+            precision: precisions.length
+                ? precisions.reduce((sum, value) => sum + value, 0) / precisions.length
+                : null,
+            recall: recalls.length
+                ? recalls.reduce((sum, value) => sum + value, 0) / recalls.length
+                : null,
+            falsePositives: hasFalsePositives ? falsePositives : null,
+        },
+        performance: {
+            latencySamples: ratios.length,
+            modes: Object.fromEntries(["cold", "warm"]
+                .filter((mode) => modeRatios[mode].p50.length > 0)
+                .map((mode) => [
+                mode,
+                {
+                    paired: modeRatios[mode].p50.length,
+                    p50Ratio: median(modeRatios[mode].p50),
+                    p95Ratio: median(modeRatios[mode].p95),
+                },
+            ])),
+            knodinResponseBytes: hasKnodinResponseBytes ? knodinResponseBytes : null,
+            competitorResponseBytes: hasCompetitorResponseBytes ? competitorResponseBytes : null,
+            estimatedTokens: estimatedTokens.length
+                ? estimatedTokens.reduce((sum, value) => sum + value, 0)
+                : null,
+            knodinPeakRssMb: knodinPeakRssMb.length ? Math.max(...knodinPeakRssMb) : null,
+            competitorPeakRssMb: competitorPeakRssMb.length ? Math.max(...competitorPeakRssMb) : null,
+            peakRssMb: peakRssMb.length ? Math.max(...peakRssMb) : null,
+            diskBytes: diskBytes.length ? Math.max(...diskBytes) : null,
+        },
+    };
+}
+export function compareSummaries(baseline, current, latencyTolerance = 0.1) {
+    if (!Number.isFinite(latencyTolerance) || latencyTolerance < 0) {
+        throw new Error("Latency tolerance must be a non-negative number");
+    }
+    const reasons = [];
+    if (current.paired < baseline.paired)
+        reasons.push(`Paired cases fell by ${baseline.paired - current.paired}`);
+    if (current.knodinFaster < baseline.knodinFaster) {
+        reasons.push(`knodin wins fell by ${baseline.knodinFaster - current.knodinFaster}`);
+    }
+    if (current.knodinErrors > baseline.knodinErrors) {
+        reasons.push(`knodin errors increased by ${current.knodinErrors - baseline.knodinErrors}`);
+    }
+    if (baseline.performance && current.performance) {
+        for (const mode of ["cold", "warm"]) {
+            const before = baseline.performance.modes?.[mode];
+            const after = current.performance.modes?.[mode];
+            if (!before || !after)
+                continue;
+            for (const percentile of ["p50Ratio", "p95Ratio"]) {
+                const beforeValue = before[percentile];
+                const afterValue = after[percentile];
+                if (beforeValue !== null &&
+                    afterValue !== null &&
+                    afterValue < beforeValue * (1 - latencyTolerance)) {
+                    reasons.push(`${mode} ${percentile.replace("Ratio", "")} competitor/knodin ratio fell from ${beforeValue.toFixed(3)} to ${afterValue.toFixed(3)}`);
+                }
+            }
+        }
+    }
+    else if (baseline.medianRatio !== null &&
+        current.medianRatio !== null &&
+        current.medianRatio < baseline.medianRatio * (1 - latencyTolerance)) {
+        reasons.push(`Competitor/knodin median ratio fell from ${baseline.medianRatio.toFixed(3)} to ${current.medianRatio.toFixed(3)}`);
+    }
+    return {
+        regressed: reasons.length > 0,
+        reasons,
+        deltas: {
+            paired: current.paired - baseline.paired,
+            knodinFaster: current.knodinFaster - baseline.knodinFaster,
+            competitorFaster: current.competitorFaster - baseline.competitorFaster,
+            competitorErrors: current.competitorErrors - baseline.competitorErrors,
+            knodinErrors: current.knodinErrors - baseline.knodinErrors,
+            medianRatio: baseline.medianRatio === null || current.medianRatio === null
+                ? null
+                : current.medianRatio - baseline.medianRatio,
+        },
+    };
+}

package/dist/src/competitive-sandbox.js

@@ -0,0 +1,108 @@
+import childProcess from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+function runGit(args, cwd, environment) {
+    const result = childProcess.spawnSync("git", args, {
+        cwd,
+        env: environment,
+        encoding: "utf8",
+    });
+    if (result.status !== 0) {
+        throw new Error(`Unable to create competitive sandbox: git ${args.join(" ")} failed\n${result.stderr.trim()}`);
+    }
+}
+function executableOnPath(name) {
+    for (const directory of (process.env.PATH ?? "").split(path.delimiter)) {
+        if (!directory)
+            continue;
+        const candidate = path.join(directory, name);
+        try {
+            fs.accessSync(candidate, fs.constants.X_OK);
+            return fs.realpathSync(candidate);
+        }
+        catch {
+            // Continue searching PATH.
+        }
+    }
+    return undefined;
+}
+/**
+ * Give a stateful competitor a disposable checkout and user-level state tree.
+ *
+ * The clone borrows immutable Git objects from the source repository, but owns
+ * its refs, index, worktree, and generated graph data. Dependencies are exposed
+ * read-only-by-convention through a node_modules symlink so benchmark adapters
+ * can execute without a second package installation.
+ */
+export function createCompetitiveSandbox(sourceRepository, sourceFixture, competitorId) {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), `knodin-${competitorId}-sandbox-`));
+    const repository = path.join(root, "repository");
+    const fixture = path.join(root, "fixture");
+    const home = path.join(root, "home");
+    const temporary = path.join(root, "tmp");
+    const cache = path.join(root, "cache");
+    const config = path.join(root, "config");
+    const data = path.join(root, "data");
+    const state = path.join(root, "state");
+    const binaries = path.join(root, "bin");
+    for (const directory of [home, temporary, cache, config, data, state, binaries]) {
+        fs.mkdirSync(directory, { recursive: true });
+    }
+    const realBun = executableOnPath("bun");
+    const environment = {
+        ...process.env,
+        HOME: home,
+        TMPDIR: temporary,
+        XDG_CACHE_HOME: cache,
+        XDG_CONFIG_HOME: config,
+        XDG_DATA_HOME: data,
+        XDG_STATE_HOME: state,
+        NPM_CONFIG_CACHE: path.join(cache, "npm"),
+        BUN_INSTALL_CACHE_DIR: path.join(cache, "bun"),
+        PIP_CACHE_DIR: path.join(cache, "pip"),
+        CLAUDE_CONFIG_DIR: path.join(config, "claude"),
+        GIT_CONFIG_GLOBAL: path.join(config, "gitconfig"),
+        GIT_CONFIG_NOSYSTEM: "1",
+        PATH: `${binaries}${path.delimiter}${process.env.PATH ?? ""}`,
+        RECKON_BENCHMARK_SOURCE_REPOSITORY: sourceRepository,
+        ...(realBun ? { RECKON_BENCHMARK_REAL_BUN: realBun } : {}),
+    };
+    try {
+        if (realBun) {
+            const bunShim = path.join(binaries, "bun");
+            fs.writeFileSync(bunShim, `#!/bin/sh
+if [ "$1" = "src/server.ts" ]; then
+  shift
+  exec node --import tsx "$RECKON_BENCHMARK_SOURCE_REPOSITORY/src/server.ts" "$@"
+fi
+exec "$RECKON_BENCHMARK_REAL_BUN" "$@"
+`, { mode: 0o755 });
+        }
+        runGit(["clone", "--quiet", "--shared", "--no-checkout", sourceRepository, repository], root, environment);
+        const revision = childProcess
+            .spawnSync("git", ["rev-parse", "HEAD"], {
+            cwd: sourceRepository,
+            encoding: "utf8",
+        })
+            .stdout.trim();
+        runGit(["checkout", "--quiet", "--detach", revision], repository, environment);
+        const dependencies = path.join(sourceRepository, "node_modules");
+        if (fs.existsSync(dependencies)) {
+            fs.symlinkSync(dependencies, path.join(repository, "node_modules"), "dir");
+        }
+        fs.cpSync(sourceFixture, fixture, { recursive: true });
+        return {
+            root,
+            repository,
+            fixture,
+            output: path.join(root, "raw-result.json"),
+            environment,
+            dispose: () => fs.rmSync(root, { recursive: true, force: true }),
+        };
+    }
+    catch (error) {
+        fs.rmSync(root, { recursive: true, force: true });
+        throw error;
+    }
+}