knodin 0.7.6 → 0.8.3

package/dist/src/engine/ann-hnsw.js

@@ -0,0 +1,261 @@
+/**
+ * Pure-TypeScript, dependency-free HNSW approximate nearest-neighbor index for
+ * the vector half of `search()` (R18).
+ *
+ * WHY THIS EXISTS, AND WHY IT IS OPT-IN. R16 removed R11(a)'s random-hyperplane
+ * LSH path after measuring it slower AND far less accurate (37% top-5 recall)
+ * than the exact cosine scan on a real 18.6k-symbol corpus. R11(a)'s only test
+ * passed because its 30-symbol fixture never exceeded the candidate floor, so it
+ * degenerated to an exact scan and never proved anything about real pruning.
+ *
+ * The durable lesson, now a hard gate: no approximate search path may engage BY
+ * DEFAULT without a measured >=95% top-10 recall vs the exact scan on a real
+ * corpus, with the method stated. This module is therefore OPT-IN ONLY, behind
+ * `KNODIN_ANN=1`; `search()`'s default remains the exact scan at every corpus
+ * size. See scripts/ann-bench.ts for the recall/latency harness.
+ *
+ * MEASURED, GATE NOT CLEARED (so it stays opt-in). scripts/ann-bench.ts on
+ * /path/to/large-repository (18,639 symbols / 18,639 embeddings / schema 14), 20 queries,
+ * limit=10, two separate processes, one untimed warm-up, cold first-timed query
+ * excluded from the warm median, ef=200:
+ *   Exact scan:  warm median 38.11ms, min 34.97ms
+ *   ANN ef=200:  warm median 34.91ms, min 27.17ms
+ *   Recall vs exact: mean top-5 91.0%, mean top-10 88.0% (one query,
+ *     "resolve import path", scored 0%).
+ * 88.0% top-10 is far better than R11(a)'s LSH (43.3% top-10, AND slower) — the
+ * three queries LSH scored 0% on ("session token", "webhook handler", "metrics
+ * collection") here score 80/80/90% top-10 — but it misses the hard 95% floor,
+ * so the exact scan REMAINS the default. Recall is tunable via KNODIN_ANN_EF (a
+ * larger ef trades latency for recall); default-engagement would require a fresh
+ * measurement clearing 95% at the corpus size where the path engages. Given the
+ * exact scan is 38ms at 18.6k and extrapolates sub-second to ~250k symbols,
+ * there is no corpus today where enabling this by default is worth doing.
+ *
+ * REMOVABILITY. The entire algorithm lives in this one file. It holds no stored
+ * state — the index is built in memory from the already-fetched embedding rows
+ * and cached per (repoPath, indexGeneration) by the caller, invalidated exactly
+ * like mapCache/flowsCache. If a future measurement finds it also fails the
+ * gate, deleting this file + the one guarded `if/else` seam + the cache Map in
+ * index.ts restores the exact scan with zero residue and no schema migration —
+ * the same clean-revert shape R11(a) had.
+ *
+ * DISTANCE KERNEL. Similarity is `embeddings.computeSimilarity` (dot product of
+ * L2-normalized vectors; higher = nearer), called via the namespace import so a
+ * degenerate "scan everything" implementation is observable at the integration
+ * layer (the gate test counts these calls and asserts real pruning).
+ */
+import { DEFAULT_PRNG_SEED, mulberry32 } from "../deterministic-random.js";
+import * as embeddings from "./embeddings.js";
+const DEFAULT_M = 16;
+const DEFAULT_EF_CONSTRUCTION = 200;
+const DEFAULT_EF_SEARCH = 200;
+const DEFAULT_SEED = DEFAULT_PRNG_SEED;
+class Hnsw {
+    nodes = [];
+    entryPoint = -1;
+    maxLevel = -1;
+    M;
+    Mmax0;
+    efConstruction;
+    mL;
+    rand;
+    constructor(params) {
+        this.M = params.M ?? DEFAULT_M;
+        this.Mmax0 = this.M * 2;
+        this.efConstruction = params.efConstruction ?? DEFAULT_EF_CONSTRUCTION;
+        this.mL = 1 / Math.log(this.M);
+        this.rand = mulberry32(params.seed ?? DEFAULT_SEED);
+    }
+    get size() {
+        return this.nodes.length;
+    }
+    randomLevel() {
+        return Math.floor(-Math.log(this.rand() || Number.MIN_VALUE) * this.mL);
+    }
+    sim(a, b) {
+        return embeddings.computeSimilarity(a, b);
+    }
+    insert(id, vec) {
+        const level = this.randomLevel();
+        const idx = this.nodes.length;
+        const node = {
+            id,
+            vec,
+            level,
+            neighbors: Array.from({ length: level + 1 }, () => []),
+        };
+        this.nodes.push(node);
+        if (this.entryPoint === -1) {
+            this.entryPoint = idx;
+            this.maxLevel = level;
+            return;
+        }
+        let ep = this.entryPoint;
+        // Descend from the top down to level+1 with a greedy 1-NN walk.
+        for (let lc = this.maxLevel; lc > level; lc--) {
+            ep = this.greedyDescend(vec, ep, lc).node;
+        }
+        // From min(level, maxLevel) down to 0, run the ef beam and connect.
+        for (let lc = Math.min(level, this.maxLevel); lc >= 0; lc--) {
+            const candidates = this.searchLayer(vec, [ep], this.efConstruction, lc).candidates;
+            const Mlevel = lc === 0 ? this.Mmax0 : this.M;
+            const selected = this.selectNeighbors(candidates, Mlevel);
+            node.neighbors[lc] = selected.slice();
+            // Add reciprocal links, pruning the neighbor's list back to its cap.
+            for (const nbr of selected) {
+                const nbrNode = this.nodes[nbr];
+                nbrNode.neighbors[lc].push(idx);
+                const cap = lc === 0 ? this.Mmax0 : this.M;
+                if (nbrNode.neighbors[lc].length > cap) {
+                    const rescored = nbrNode.neighbors[lc].map((n) => ({
+                        idx: n,
+                        score: this.sim(nbrNode.vec, this.nodes[n].vec),
+                    }));
+                    rescored.sort((x, y) => y.score - x.score);
+                    nbrNode.neighbors[lc] = rescored.slice(0, cap).map((r) => r.idx);
+                }
+            }
+            ep = candidates.length > 0 ? candidates[0].idx : ep;
+        }
+        if (level > this.maxLevel) {
+            this.maxLevel = level;
+            this.entryPoint = idx;
+        }
+    }
+    /**
+     * Greedy single-nearest walk at one layer. Returns the local optimum node
+     * index and the exact number of distance computations it performed (so the
+     * caller can report `stats.visited` as a true distance-call count).
+     */
+    greedyDescend(query, entry, level) {
+        let current = entry;
+        let currentScore = this.sim(query, this.nodes[current].vec);
+        let distanceComputations = 1;
+        let improved = true;
+        while (improved) {
+            improved = false;
+            for (const nbr of this.nodes[current].neighbors[level] ?? []) {
+                const s = this.sim(query, this.nodes[nbr].vec);
+                distanceComputations++;
+                if (s > currentScore) {
+                    currentScore = s;
+                    current = nbr;
+                    improved = true;
+                }
+            }
+        }
+        return { node: current, distanceComputations };
+    }
+    /**
+     * ef-wide best-first beam search at one layer from the given entry points.
+     * Returns the ef best candidates (sorted nearest-first) and the count of
+     * distance computations performed (the pruning instrumentation).
+     */
+    searchLayer(query, entries, ef, level) {
+        const visited = new Set();
+        // Max-heap-ish behavior via sorted arrays; corpora at this layer are small.
+        const candidateHeap = []; // frontier, best-first
+        const resultHeap = []; // ef best so far, worst-first
+        let distanceComputations = 0;
+        for (const e of entries) {
+            if (visited.has(e))
+                continue;
+            visited.add(e);
+            const s = this.sim(query, this.nodes[e].vec);
+            distanceComputations++;
+            candidateHeap.push({ idx: e, score: s });
+            resultHeap.push({ idx: e, score: s });
+        }
+        candidateHeap.sort((a, b) => b.score - a.score);
+        resultHeap.sort((a, b) => a.score - b.score);
+        while (candidateHeap.length > 0) {
+            const c = candidateHeap.shift();
+            if (!c)
+                break;
+            const worst = resultHeap[0];
+            if (worst && c.score < worst.score && resultHeap.length >= ef)
+                break;
+            for (const nbr of this.nodes[c.idx].neighbors[level] ?? []) {
+                if (visited.has(nbr))
+                    continue;
+                visited.add(nbr);
+                const s = this.sim(query, this.nodes[nbr].vec);
+                distanceComputations++;
+                const currentWorst = resultHeap[0];
+                if (resultHeap.length < ef || (currentWorst && s > currentWorst.score)) {
+                    insertSorted(candidateHeap, { idx: nbr, score: s }, true);
+                    insertSorted(resultHeap, { idx: nbr, score: s }, false);
+                    if (resultHeap.length > ef)
+                        resultHeap.shift();
+                }
+            }
+        }
+        const candidates = resultHeap.slice().sort((a, b) => b.score - a.score);
+        return { candidates, visited: distanceComputations };
+    }
+    selectNeighbors(candidates, M) {
+        return candidates
+            .slice()
+            .sort((a, b) => b.score - a.score)
+            .slice(0, M)
+            .map((c) => c.idx);
+    }
+    search(queryVec, efSearch, k) {
+        if (this.entryPoint === -1)
+            return { results: [], stats: { visited: 0 } };
+        const ef = Math.max(efSearch, k);
+        let ep = this.entryPoint;
+        let visited = 0;
+        for (let lc = this.maxLevel; lc > 0; lc--) {
+            // Upper-layer greedy-walk distance calls count toward the search budget.
+            const descent = this.greedyDescend(queryVec, ep, lc);
+            ep = descent.node;
+            visited += descent.distanceComputations;
+        }
+        const { candidates, visited: layer0Visited } = this.searchLayer(queryVec, [ep], ef, 0);
+        visited += layer0Visited;
+        const results = candidates
+            .slice(0, k)
+            .map((c) => ({ id: this.nodes[c.idx].id, score: c.score }));
+        return { results, stats: { visited } };
+    }
+}
+/** Inserts into a score-sorted array; `bestFirst` = descending, else ascending. */
+function insertSorted(arr, item, bestFirst) {
+    let lo = 0;
+    let hi = arr.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        const cmp = bestFirst ? arr[mid].score > item.score : arr[mid].score < item.score;
+        if (cmp)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    arr.splice(lo, 0, item);
+}
+/** Builds an in-memory HNSW index over the given embedding rows. */
+export function buildHnswIndex(rows, params = {}) {
+    const index = new Hnsw(params);
+    for (const row of rows)
+        index.insert(row.id, row.vec);
+    return index;
+}
+/**
+ * Whether the ANN path is engaged. OPT-IN ONLY: the exact scan is the default at
+ * every corpus size until the >=95% top-10 recall gate is measured-and-cleared
+ * at a corpus size where the path would actually engage (see module header).
+ */
+export function annEnabled() {
+    return process.env.KNODIN_ANN === "1";
+}
+/** efSearch beam width, overridable via KNODIN_ANN_EF (a benchmark knob). */
+export function annEfSearch() {
+    const raw = process.env.KNODIN_ANN_EF;
+    if (raw) {
+        const n = Number.parseInt(raw, 10);
+        if (Number.isFinite(n) && n > 0)
+            return n;
+    }
+    return DEFAULT_EF_SEARCH;
+}

package/dist/src/engine/embeddings.js

@@ -0,0 +1,193 @@
+/**
+ * Local-only by design (ADR 003, "Future Considerations" section,
+ * docs/adr/003-local-semantic-search.md): a cloud embedding provider
+ * (OpenAI/Google/MiniMax, as some competitors offer) was evaluated and
+ * deliberately deferred, not merely unbuilt. knodin's core positioning is
+ * zero-auth/local-first/no-egress — sending symbol text to a cloud API for
+ * embedding would break that guarantee for every indexed repo, not just tune
+ * a knob. If ever added, it must be opt-in with `local` staying the default,
+ * and reads as a natural pro-tier feature precisely because enabling it is a
+ * deliberate trade against this file's local/no-egress guarantee.
+ */
+import os from "node:os";
+import path from "node:path";
+let embedderPromise = null;
+let embedderOverride = null;
+const embedderProgressListeners = new Set();
+let embedderReady = false;
+const EMBEDDING_DIMENSIONS = 384;
+const DEFAULT_MODEL_REMOTE_HOST = "https://huggingface.co/";
+/**
+ * One machine-level model cache shared by every repository and every installed
+ * knodin version. An explicit override makes managed/offline environments and
+ * the clean-package acceptance gate deterministic.
+ */
+export function getModelCacheDirectory(options = {}) {
+    const environment = options.env ?? process.env;
+    const override = environment.KNODIN_MODEL_CACHE?.trim();
+    if (override)
+        return path.resolve(override);
+    const platform = options.platform ?? process.platform;
+    const home = options.home ?? os.homedir();
+    if (platform === "win32") {
+        return path.join(environment.LOCALAPPDATA || path.join(home, "AppData", "Local"), "knodin", "models");
+    }
+    if (platform === "darwin")
+        return path.join(home, "Library", "Caches", "knodin", "models");
+    return path.join(environment.XDG_CACHE_HOME || path.join(home, ".cache"), "knodin", "models");
+}
+/**
+ * Remote model origin used only when the shared machine cache is cold.
+ * The default remains the public Hugging Face Hub; managed environments may
+ * point at an approved mirror without changing where model files are cached.
+ */
+export function getModelRemoteHost(environment = process.env) {
+    const configured = environment.KNODIN_MODEL_HOST?.trim() || DEFAULT_MODEL_REMOTE_HOST;
+    let parsed;
+    try {
+        parsed = new URL(configured);
+    }
+    catch {
+        throw new Error(`KNODIN_MODEL_HOST must be an absolute HTTP(S) URL: ${configured}`);
+    }
+    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
+        throw new Error(`KNODIN_MODEL_HOST must use HTTP(S): ${configured}`);
+    }
+    parsed.hash = "";
+    parsed.search = "";
+    return parsed.href.endsWith("/") ? parsed.href : `${parsed.href}/`;
+}
+/**
+ * Number of times the real ONNX model pipeline has actually been loaded in this
+ * process. Observability only — it never changes embedding behavior. The test
+ * suite uses it to guarantee that non-model suites keep the deterministic test
+ * embedder installed and never fall through to native model initialization
+ * (which, repeated across a long-lived worker, is what C50 removes).
+ */
+let realEmbedderLoadCount = 0;
+/** Returns how many times the real ONNX pipeline has loaded in this process. */
+export function getRealEmbedderLoadCount() {
+    return realEmbedderLoadCount;
+}
+/**
+ * Overrides the embedding pipeline (e.g. with a fast mock in tests). Pass `null`
+ * to clear the override and fall back to the real lazily-loaded model.
+ */
+export function setEmbedder(pipeline) {
+    embedderOverride = pipeline;
+    embedderPromise = null;
+    embedderReady = pipeline !== null;
+}
+/**
+ * Release the native ONNX model session if one was loaded, freeing its resident
+ * memory. Best-effort and test-facing: the single long-lived test worker
+ * (isolate:false) otherwise keeps the ~model resident for the whole run, bloating
+ * the process so later subprocess-spawning tests fork slowly. The dedicated
+ * embedding spec calls this in teardown once its real-model assertions are done.
+ * A no-op when only a mock embedder (or nothing) has been used.
+ */
+export async function disposeEmbedder() {
+    const pending = embedderPromise;
+    embedderPromise = null;
+    embedderReady = false;
+    if (!pending)
+        return;
+    try {
+        const pipe = (await pending);
+        if (typeof pipe.dispose === "function")
+            await pipe.dispose();
+    }
+    catch {
+        /* best-effort native teardown; never fail a test on dispose */
+    }
+}
+/**
+ * Lazy-loads and caches the local embedding pipeline.
+ * Uses a lightweight, high-performance 384-dimension model (all-MiniLM-L6-v2)
+ * running in-process via ONNX Runtime Web.
+ * Uses dynamic imports to keep CLI and MCP server startup latency near-zero.
+ */
+export function getEmbedder(onProgress) {
+    if (embedderOverride) {
+        onProgress?.({ status: "ready", name: "local test embedder" });
+        return Promise.resolve(embedderOverride);
+    }
+    if (onProgress && embedderReady)
+        onProgress({ status: "ready", name: "Xenova/all-MiniLM-L6-v2" });
+    else if (onProgress)
+        embedderProgressListeners.add(onProgress);
+    if (!embedderPromise) {
+        embedderPromise = (async () => {
+            try {
+                const { env, pipeline } = await import("@huggingface/transformers");
+                env.cacheDir = getModelCacheDirectory();
+                env.remoteHost = getModelRemoteHost();
+                const notify = (event) => {
+                    for (const listener of embedderProgressListeners) {
+                        try {
+                            listener(event);
+                        }
+                        catch {
+                            /* Observability must not alter model initialization. */
+                        }
+                    }
+                };
+                notify({ status: "loading", name: "Xenova/all-MiniLM-L6-v2" });
+                const pipe = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", {
+                    progress_callback: notify,
+                });
+                realEmbedderLoadCount++;
+                embedderReady = true;
+                notify({ status: "ready", name: "Xenova/all-MiniLM-L6-v2" });
+                embedderProgressListeners.clear();
+                return pipe;
+            }
+            catch (error) {
+                embedderPromise = null; // Reset promise so we can retry on next call
+                embedderProgressListeners.clear();
+                console.error("Failed to initialize Hugging Face embedding pipeline:", error);
+                throw error;
+            }
+        })();
+    }
+    return embedderPromise;
+}
+/**
+ * Generates a normalized Float32Array embedding (384 dimensions) for a given text block.
+ */
+export async function generateEmbeddings(texts, onModelProgress) {
+    if (texts.length === 0)
+        return [];
+    const extractor = await getEmbedder(onModelProgress);
+    const result = await extractor([...texts], { pooling: "mean", normalize: true });
+    const data = result.data instanceof Float32Array ? result.data : Float32Array.from(result.data);
+    const dims = result.dims;
+    if (dims?.length !== 2 || dims[0] !== texts.length || dims[1] !== EMBEDDING_DIMENSIONS) {
+        throw new Error(`Unexpected batched embedding shape: expected [${texts.length}, ${EMBEDDING_DIMENSIONS}], got ${JSON.stringify(dims)}`);
+    }
+    const dimension = dims[1];
+    if (data.length !== texts.length * dimension) {
+        throw new Error(`Unexpected batched embedding data length: expected ${texts.length * dimension}, got ${data.length}`);
+    }
+    return texts.map((_, index) => data.slice(index * dimension, (index + 1) * dimension));
+}
+/** Compatibility wrapper used by query-time search. */
+export async function generateEmbedding(text) {
+    return (await generateEmbeddings([text]))[0];
+}
+/**
+ * Computes the cosine similarity between two normalized Float32Arrays.
+ * Since the vectors are pre-normalized, the cosine similarity simplifies
+ * to the mathematical dot product: sum(u_i * v_i).
+ */
+export function computeSimilarity(queryVec, targetVec) {
+    if (queryVec.length !== targetVec.length) {
+        throw new Error(`Embedding dimension mismatch: query has ${queryVec.length} dims, target has ${targetVec.length}`);
+    }
+    let dotProduct = 0;
+    const len = queryVec.length;
+    for (let i = 0; i < len; i++) {
+        dotProduct += queryVec[i] * targetVec[i];
+    }
+    return dotProduct;
+}

package/dist/src/engine/file-walker.js

@@ -0,0 +1,49 @@
+import fs from "node:fs";
+import path from "node:path";
+import { isIndexablePath } from "./prune.js";
+/**
+ * Byte-order path comparison. Deliberately NOT `localeCompare`: index order must
+ * be identical on every machine, and locale collation is not.
+ */
+export function comparePaths(left, right) {
+    if (left < right)
+        return -1;
+    return left > right ? 1 : 0;
+}
+/**
+ * Deterministically enumerate regular files beneath a repository without
+ * following symlinks. Directory pruning happens before descent so dependency,
+ * build, VCS, worktree, and local graph state never incur a recursive scan.
+ */
+export function walkRepoFiles(repoPath, options = {}) {
+    const root = path.resolve(repoPath);
+    const pending = [{ absolute: root, relative: "" }];
+    const files = [];
+    while (pending.length > 0) {
+        const directory = pending.pop();
+        if (!directory)
+            break;
+        let entries;
+        try {
+            entries = fs.readdirSync(directory.absolute, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        entries.sort((left, right) => comparePaths(left.name, right.name));
+        for (const entry of entries) {
+            const relative = directory.relative ? `${directory.relative}/${entry.name}` : entry.name;
+            if (!isIndexablePath(relative))
+                continue;
+            const absolute = path.join(directory.absolute, entry.name);
+            if (entry.isDirectory()) {
+                pending.push({ absolute, relative });
+            }
+            else if (entry.isFile() && (options.accept?.(relative) ?? true)) {
+                files.push(relative);
+            }
+        }
+    }
+    files.sort(comparePaths);
+    return options.limit === undefined ? files : files.slice(0, Math.max(0, options.limit));
+}