knodin 0.5.0

package/dist/src/engine/embeddings.js

@@ -0,0 +1,193 @@
+/**
+ * Local-only by design (ADR 003, "Future Considerations" section,
+ * docs/adr/003-local-semantic-search.md): a cloud embedding provider
+ * (OpenAI/Google/MiniMax, as some competitors offer) was evaluated and
+ * deliberately deferred, not merely unbuilt. knodin's core positioning is
+ * zero-auth/local-first/no-egress — sending symbol text to a cloud API for
+ * embedding would break that guarantee for every indexed repo, not just tune
+ * a knob. If ever added, it must be opt-in with `local` staying the default,
+ * and reads as a natural pro-tier feature precisely because enabling it is a
+ * deliberate trade against this file's local/no-egress guarantee.
+ */
+import os from "node:os";
+import path from "node:path";
+let embedderPromise = null;
+let embedderOverride = null;
+const embedderProgressListeners = new Set();
+let embedderReady = false;
+const EMBEDDING_DIMENSIONS = 384;
+const DEFAULT_MODEL_REMOTE_HOST = "https://huggingface.co/";
+/**
+ * One machine-level model cache shared by every repository and every installed
+ * knodin version. An explicit override makes managed/offline environments and
+ * the clean-package acceptance gate deterministic.
+ */
+export function getModelCacheDirectory(options = {}) {
+    const environment = options.env ?? process.env;
+    const override = environment.RECKON_MODEL_CACHE?.trim();
+    if (override)
+        return path.resolve(override);
+    const platform = options.platform ?? process.platform;
+    const home = options.home ?? os.homedir();
+    if (platform === "win32") {
+        return path.join(environment.LOCALAPPDATA || path.join(home, "AppData", "Local"), "knodin", "models");
+    }
+    if (platform === "darwin")
+        return path.join(home, "Library", "Caches", "knodin", "models");
+    return path.join(environment.XDG_CACHE_HOME || path.join(home, ".cache"), "knodin", "models");
+}
+/**
+ * Remote model origin used only when the shared machine cache is cold.
+ * The default remains the public Hugging Face Hub; managed environments may
+ * point at an approved mirror without changing where model files are cached.
+ */
+export function getModelRemoteHost(environment = process.env) {
+    const configured = environment.RECKON_MODEL_HOST?.trim() || DEFAULT_MODEL_REMOTE_HOST;
+    let parsed;
+    try {
+        parsed = new URL(configured);
+    }
+    catch {
+        throw new Error(`RECKON_MODEL_HOST must be an absolute HTTP(S) URL: ${configured}`);
+    }
+    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
+        throw new Error(`RECKON_MODEL_HOST must use HTTP(S): ${configured}`);
+    }
+    parsed.hash = "";
+    parsed.search = "";
+    return parsed.href.endsWith("/") ? parsed.href : `${parsed.href}/`;
+}
+/**
+ * Number of times the real ONNX model pipeline has actually been loaded in this
+ * process. Observability only — it never changes embedding behavior. The test
+ * suite uses it to guarantee that non-model suites keep the deterministic test
+ * embedder installed and never fall through to native model initialization
+ * (which, repeated across a long-lived worker, is what C50 removes).
+ */
+let realEmbedderLoadCount = 0;
+/** Returns how many times the real ONNX pipeline has loaded in this process. */
+export function getRealEmbedderLoadCount() {
+    return realEmbedderLoadCount;
+}
+/**
+ * Overrides the embedding pipeline (e.g. with a fast mock in tests). Pass `null`
+ * to clear the override and fall back to the real lazily-loaded model.
+ */
+export function setEmbedder(pipeline) {
+    embedderOverride = pipeline;
+    embedderPromise = null;
+    embedderReady = pipeline !== null;
+}
+/**
+ * Release the native ONNX model session if one was loaded, freeing its resident
+ * memory. Best-effort and test-facing: the single long-lived test worker
+ * (isolate:false) otherwise keeps the ~model resident for the whole run, bloating
+ * the process so later subprocess-spawning tests fork slowly. The dedicated
+ * embedding spec calls this in teardown once its real-model assertions are done.
+ * A no-op when only a mock embedder (or nothing) has been used.
+ */
+export async function disposeEmbedder() {
+    const pending = embedderPromise;
+    embedderPromise = null;
+    embedderReady = false;
+    if (!pending)
+        return;
+    try {
+        const pipe = (await pending);
+        if (typeof pipe.dispose === "function")
+            await pipe.dispose();
+    }
+    catch {
+        /* best-effort native teardown; never fail a test on dispose */
+    }
+}
+/**
+ * Lazy-loads and caches the local embedding pipeline.
+ * Uses a lightweight, high-performance 384-dimension model (all-MiniLM-L6-v2)
+ * running in-process via ONNX Runtime Web.
+ * Uses dynamic imports to keep CLI and MCP server startup latency near-zero.
+ */
+export function getEmbedder(onProgress) {
+    if (embedderOverride) {
+        onProgress?.({ status: "ready", name: "local test embedder" });
+        return Promise.resolve(embedderOverride);
+    }
+    if (onProgress && embedderReady)
+        onProgress({ status: "ready", name: "Xenova/all-MiniLM-L6-v2" });
+    else if (onProgress)
+        embedderProgressListeners.add(onProgress);
+    if (!embedderPromise) {
+        embedderPromise = (async () => {
+            try {
+                const { env, pipeline } = await import("@huggingface/transformers");
+                env.cacheDir = getModelCacheDirectory();
+                env.remoteHost = getModelRemoteHost();
+                const notify = (event) => {
+                    for (const listener of embedderProgressListeners) {
+                        try {
+                            listener(event);
+                        }
+                        catch {
+                            /* Observability must not alter model initialization. */
+                        }
+                    }
+                };
+                notify({ status: "loading", name: "Xenova/all-MiniLM-L6-v2" });
+                const pipe = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", {
+                    progress_callback: notify,
+                });
+                realEmbedderLoadCount++;
+                embedderReady = true;
+                notify({ status: "ready", name: "Xenova/all-MiniLM-L6-v2" });
+                embedderProgressListeners.clear();
+                return pipe;
+            }
+            catch (error) {
+                embedderPromise = null; // Reset promise so we can retry on next call
+                embedderProgressListeners.clear();
+                console.error("Failed to initialize Hugging Face embedding pipeline:", error);
+                throw error;
+            }
+        })();
+    }
+    return embedderPromise;
+}
+/**
+ * Generates a normalized Float32Array embedding (384 dimensions) for a given text block.
+ */
+export async function generateEmbeddings(texts, onModelProgress) {
+    if (texts.length === 0)
+        return [];
+    const extractor = await getEmbedder(onModelProgress);
+    const result = await extractor([...texts], { pooling: "mean", normalize: true });
+    const data = result.data instanceof Float32Array ? result.data : Float32Array.from(result.data);
+    const dims = result.dims;
+    if (dims?.length !== 2 || dims[0] !== texts.length || dims[1] !== EMBEDDING_DIMENSIONS) {
+        throw new Error(`Unexpected batched embedding shape: expected [${texts.length}, ${EMBEDDING_DIMENSIONS}], got ${JSON.stringify(dims)}`);
+    }
+    const dimension = dims[1];
+    if (data.length !== texts.length * dimension) {
+        throw new Error(`Unexpected batched embedding data length: expected ${texts.length * dimension}, got ${data.length}`);
+    }
+    return texts.map((_, index) => data.slice(index * dimension, (index + 1) * dimension));
+}
+/** Compatibility wrapper used by query-time search. */
+export async function generateEmbedding(text) {
+    return (await generateEmbeddings([text]))[0];
+}
+/**
+ * Computes the cosine similarity between two normalized Float32Arrays.
+ * Since the vectors are pre-normalized, the cosine similarity simplifies
+ * to the mathematical dot product: sum(u_i * v_i).
+ */
+export function computeSimilarity(queryVec, targetVec) {
+    if (queryVec.length !== targetVec.length) {
+        throw new Error(`Embedding dimension mismatch: query has ${queryVec.length} dims, target has ${targetVec.length}`);
+    }
+    let dotProduct = 0;
+    const len = queryVec.length;
+    for (let i = 0; i < len; i++) {
+        dotProduct += queryVec[i] * targetVec[i];
+    }
+    return dotProduct;
+}

package/dist/src/engine/file-walker.js

@@ -0,0 +1,43 @@
+import fs from "node:fs";
+import path from "node:path";
+import { isIndexablePath } from "./prune.js";
+function comparePaths(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+/**
+ * Deterministically enumerate regular files beneath a repository without
+ * following symlinks. Directory pruning happens before descent so dependency,
+ * build, VCS, worktree, and local graph state never incur a recursive scan.
+ */
+export function walkRepoFiles(repoPath, options = {}) {
+    const root = path.resolve(repoPath);
+    const pending = [{ absolute: root, relative: "" }];
+    const files = [];
+    while (pending.length > 0) {
+        const directory = pending.pop();
+        if (!directory)
+            break;
+        let entries;
+        try {
+            entries = fs.readdirSync(directory.absolute, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        entries.sort((left, right) => comparePaths(left.name, right.name));
+        for (const entry of entries) {
+            const relative = directory.relative ? `${directory.relative}/${entry.name}` : entry.name;
+            if (!isIndexablePath(relative))
+                continue;
+            const absolute = path.join(directory.absolute, entry.name);
+            if (entry.isDirectory()) {
+                pending.push({ absolute, relative });
+            }
+            else if (entry.isFile() && (options.accept?.(relative) ?? true)) {
+                files.push(relative);
+            }
+        }
+    }
+    files.sort(comparePaths);
+    return options.limit === undefined ? files : files.slice(0, Math.max(0, options.limit));
+}