knodin 0.12.2 → 0.13.1

package/dist/src/engine/embedding-reuse.js

@@ -0,0 +1,57 @@
+import { createHash } from "node:crypto";
+import { EMBEDDING_DIMENSIONS } from "./embeddings.js";
+// ~12 MiB of vector payload, plus fixed-size keys/digests and SQLite overhead.
+// A disposable per-graph acceleration table, not another source of graph truth.
+export const EMBEDDING_REUSE_MAX_ENTRIES = 8192;
+// Bump whenever embeddingInputFor/source preparation changes, even if model stays pinned.
+export const EMBEDDING_INPUT_REPRESENTATION = "name-kind-summary-source:1000:v1";
+const VECTOR_BYTES = EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT;
+function digest(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+export function embeddingReuseKey(modelIdentity, input, representation = EMBEDDING_INPUT_REPRESENTATION) {
+    return digest(JSON.stringify([modelIdentity, representation, input]));
+}
+/** Exact-input cache independent of ephemeral symbol IDs. Stores no source text. */
+export class EmbeddingReuseCache {
+    db;
+    maxEntries;
+    constructor(db, maxEntries = EMBEDDING_REUSE_MAX_ENTRIES) {
+        this.db = db;
+        this.maxEntries = maxEntries;
+        if (!Number.isSafeInteger(maxEntries) ||
+            maxEntries < 1 ||
+            maxEntries > EMBEDDING_REUSE_MAX_ENTRIES)
+            throw new Error("Invalid embedding reuse cache bound");
+        db.run(`CREATE TABLE IF NOT EXISTS embedding_reuse_v1 (
+			cacheKey TEXT PRIMARY KEY CHECK(length(cacheKey) = 64),
+			vector BLOB NOT NULL CHECK(length(vector) = ${VECTOR_BYTES}),
+			digest TEXT NOT NULL CHECK(length(digest) = 64)
+		)`);
+        this.prune();
+    }
+    get(key) {
+        const row = this.db
+            .query("SELECT vector, digest FROM embedding_reuse_v1 WHERE cacheKey = ?")
+            .get(key);
+        if (!row)
+            return;
+        if (row.vector.byteLength === VECTOR_BYTES && digest(row.vector) === row.digest) {
+            const vector = new Float32Array(Uint8Array.from(row.vector).buffer);
+            if (vector.every(Number.isFinite))
+                return vector;
+        }
+        this.db.run("DELETE FROM embedding_reuse_v1 WHERE cacheKey = ?", [key]);
+    }
+    /** Call within the embedding batch transaction; reinsertion refreshes LRU order. */
+    put(key, vector) {
+        if (vector.length !== EMBEDDING_DIMENSIONS || !vector.every(Number.isFinite))
+            return;
+        const bytes = new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
+        this.db.run("INSERT OR REPLACE INTO embedding_reuse_v1(cacheKey, vector, digest) VALUES (?, ?, ?)", [key, bytes, digest(bytes)]);
+    }
+    /** Bound all generations together; stale model/input keys compete in the same LRU. */
+    prune() {
+        this.db.run("DELETE FROM embedding_reuse_v1 WHERE rowid IN (SELECT rowid FROM embedding_reuse_v1 ORDER BY rowid DESC LIMIT -1 OFFSET ?)", [this.maxEntries]);
+    }
+}

package/dist/src/engine/embeddings.js

@@ -9,6 +9,7 @@
  * and reads as a natural pro-tier feature precisely because enabling it is a
  * deliberate trade against this file's local/no-egress guarantee.
  */
+import { randomUUID } from "node:crypto";
 import os from "node:os";
 import path from "node:path";
 let embedderPromise = null;
@@ -20,6 +21,27 @@ export const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
 export const EMBEDDING_MODEL_REVISION = "751bff37182d3f1213fa05d7196b954e230abad9";
 export const EMBEDDING_MODEL_DTYPE = "q8";
 const DEFAULT_MODEL_REMOTE_HOST = "https://huggingface.co/";
+const overrideCacheIdentities = new WeakMap();
+/** Pin reuse to the pipeline contract; test overrides never alias the real model. */
+export function getEmbeddingCacheIdentity() {
+    if (embedderOverride) {
+        let identity = overrideCacheIdentities.get(embedderOverride);
+        if (!identity) {
+            identity = `override:${randomUUID()}`;
+            overrideCacheIdentities.set(embedderOverride, identity);
+        }
+        return identity;
+    }
+    return JSON.stringify([
+        EMBEDDING_MODEL_ID,
+        EMBEDDING_MODEL_REVISION,
+        EMBEDDING_MODEL_DTYPE,
+        EMBEDDING_DIMENSIONS,
+        "mean",
+        "normalize:true",
+        getModelRemoteHost(),
+    ]);
+}
 /**
  * One machine-level model cache shared by every repository and every installed
  * knodin version. An explicit override makes managed/offline environments and

package/dist/src/engine/index-coverage.js

@@ -0,0 +1,215 @@
+import crypto from "node:crypto";
+import path from "node:path";
+import { compareBytes } from "../compare.js";
+export const INDEX_COVERAGE_KEY = "indexCoverageIntent";
+export const INDEX_COVERAGE_PENDING_KEY = "indexCoveragePending";
+export const INDEX_COVERAGE_COMPLETE_KEY = "indexCoverageComplete";
+export function canonicalCoveragePath(value) {
+    return (value.length > 0 &&
+        value.length <= 4096 &&
+        !/[\\\0]/.test(value) &&
+        !path.posix.isAbsolute(value) &&
+        !/^[A-Za-z]:/.test(value) &&
+        path.posix.normalize(value) === value &&
+        value !== "." &&
+        !value.split("/").includes(".."));
+}
+// Preserve Array.sort's UTF-16 code-unit ordering; locale ordering changes digests.
+const sort = (values) => [...new Set(values)].sort(compareBytes);
+function normalize(intent) {
+    if (intent.mode === "full")
+        return { ...intent, prefixes: [], files: [] };
+    const prefixes = [];
+    const accepted = new Set();
+    for (const value of sort(intent.prefixes)) {
+        let parent = value;
+        let covered = false;
+        while (parent.includes("/")) {
+            parent = parent.slice(0, parent.lastIndexOf("/"));
+            if (accepted.has(parent)) {
+                covered = true;
+                break;
+            }
+        }
+        if (!covered) {
+            prefixes.push(value);
+            accepted.add(value);
+        }
+    }
+    const prefixIntent = { ...intent, prefixes, files: [] };
+    return {
+        ...intent,
+        prefixes,
+        files: sort(intent.files).filter((file) => !coverageContains(prefixIntent, file)),
+    };
+}
+export function readIndexCoverage(raw, pending, complete) {
+    if (raw === null && pending === null && complete === null)
+        return {
+            intent: { version: 1, mode: "full", revision: 0, prefixes: [], files: [] },
+            origin: "legacy",
+            intentState: "unknown",
+        };
+    try {
+        if (raw === null || Buffer.byteLength(raw) > 16_000_000)
+            throw new Error("missing or oversized coverage intent");
+        const value = JSON.parse(raw);
+        if (!value ||
+            Object.keys(value).some((key) => !["version", "mode", "revision", "prefixes", "files"].includes(key)) ||
+            value.version !== 1 ||
+            !["full", "scoped"].includes(value.mode) ||
+            !Number.isSafeInteger(value.revision) ||
+            value.revision < 1 ||
+            !Array.isArray(value.prefixes) ||
+            !Array.isArray(value.files) ||
+            value.prefixes.length + value.files.length > 100_000 ||
+            ![...value.prefixes, ...value.files].every((item) => typeof item === "string" && canonicalCoveragePath(item)) ||
+            (value.mode === "full" && (value.prefixes.length > 0 || value.files.length > 0)) ||
+            (value.mode === "scoped" && value.prefixes.length + value.files.length === 0))
+            throw new Error("invalid coverage intent");
+        const normalized = normalize(value);
+        if (JSON.stringify(normalized.prefixes) !== JSON.stringify(value.prefixes) ||
+            JSON.stringify(normalized.files) !== JSON.stringify(value.files) ||
+            (pending !== null && pending !== String(value.revision)) ||
+            (complete !== null &&
+                (!/^[1-9]\d*$/.test(complete) ||
+                    !Number.isSafeInteger(Number(complete)) ||
+                    Number(complete) > value.revision)))
+            throw new Error("noncanonical coverage intent");
+        return {
+            intent: normalized,
+            origin: "explicit",
+            intentState: pending !== null ? "pending" : complete === String(value.revision) ? "complete" : "unknown",
+        };
+    }
+    catch {
+        return { intent: null, origin: "invalid", intentState: "unknown" };
+    }
+}
+export function requireIndexCoverage(state) {
+    if (!state.intent)
+        throw new Error("knodin: malformed index coverage intent; refusing automatic indexing or scope expansion. Restore valid coverage metadata or explicitly rebuild a full candidate.");
+    return state.intent;
+}
+const matchers = new WeakMap();
+function matcher(intent) {
+    let known = matchers.get(intent);
+    if (!known) {
+        known = {
+            files: new Set(intent.files),
+            prefixes: new Set(intent.prefixes),
+            parents: new Set(),
+        };
+        for (const selector of [...intent.files, ...intent.prefixes]) {
+            let parent = selector;
+            while (parent.includes("/")) {
+                parent = parent.slice(0, parent.lastIndexOf("/"));
+                known.parents.add(parent);
+            }
+        }
+        matchers.set(intent, known);
+    }
+    return known;
+}
+export function coverageContains(intent, file) {
+    if (!canonicalCoveragePath(file))
+        return false;
+    if (intent.mode === "full")
+        return true;
+    const known = matcher(intent);
+    if (known.files.has(file))
+        return true;
+    let prefix = file;
+    for (;;) {
+        if (known.prefixes.has(prefix))
+            return true;
+        const end = prefix.lastIndexOf("/");
+        if (end < 0)
+            return false;
+        prefix = prefix.slice(0, end);
+    }
+}
+export function coverageMayContain(intent, directory) {
+    return (directory === "" ||
+        intent.mode === "full" ||
+        coverageContains(intent, directory) ||
+        matcher(intent).parents.has(directory));
+}
+export function mergeIndexCoverage(previous, requested, fresh) {
+    const old = requireIndexCoverage(previous);
+    if (requested &&
+        (requested.prefixes.length + requested.files.length > 100_000 ||
+            ![...requested.prefixes, ...requested.files].every(canonicalCoveragePath)))
+        throw new Error("knodin: invalid or oversized requested index coverage selectors");
+    const full = requested === null || (!fresh && old.mode === "full");
+    const next = normalize({
+        version: 1,
+        mode: full ? "full" : "scoped",
+        revision: Math.max(1, old.revision + 1),
+        prefixes: full ? [] : [...(fresh ? [] : old.prefixes), ...(requested?.prefixes ?? [])],
+        files: full ? [] : [...(fresh ? [] : old.files), ...(requested?.files ?? [])],
+    });
+    if (!readIndexCoverage(JSON.stringify(next), String(next.revision), null).intent)
+        throw new Error("knodin: merged index coverage exceeds the supported intent bounds");
+    if (previous.origin === "explicit" &&
+        JSON.stringify({ ...next, revision: old.revision }) === JSON.stringify(old))
+        return old;
+    return next;
+}
+/** Bound argv size independently of the OS ARG_MAX; every pathspec remains literal. */
+export function coverageGitPathspecBatches(intent) {
+    if (intent?.mode !== "scoped")
+        return [[]];
+    const batches = [];
+    let batch = ["--"];
+    let bytes = 3;
+    for (const selector of [...intent.prefixes, ...intent.files]) {
+        const value = `:(literal)${selector}`;
+        const size = Buffer.byteLength(value) + 1;
+        if (batch.length > 1 && (bytes + size > 32_768 || batch.length > 256)) {
+            batches.push(batch);
+            batch = ["--"];
+            bytes = 3;
+        }
+        batch.push(value);
+        bytes += size;
+    }
+    if (batch.length > 1)
+        batches.push(batch);
+    return batches;
+}
+const digests = new WeakMap();
+export function indexCoverageSummary(state) {
+    const intent = state.intent;
+    const selected = [
+        ...(intent?.prefixes ?? []).slice(0, 8).map((value) => ({ kind: "prefix", value })),
+        ...(intent?.files ?? [])
+            .slice(0, Math.max(0, 8 - (intent?.prefixes.length ?? 0)))
+            .map((value) => ({ kind: "file", value })),
+    ];
+    if (intent && !digests.has(intent))
+        digests.set(intent, crypto.createHash("sha256").update(JSON.stringify(intent)).digest("hex"));
+    return {
+        version: 1,
+        mode: intent?.mode ?? "unknown",
+        revision: intent?.revision ?? null,
+        digest: intent ? (digests.get(intent) ?? null) : null,
+        origin: state.origin,
+        prefixCount: intent?.prefixes.length ?? 0,
+        fileCount: intent?.files.length ?? 0,
+        selectorPreview: {
+            prefixes: selected.filter((x) => x.kind === "prefix").map((x) => x.value.slice(0, 160)),
+            files: selected.filter((x) => x.kind === "file").map((x) => x.value.slice(0, 160)),
+            truncated: !!intent &&
+                (intent.prefixes.length + intent.files.length > 8 ||
+                    selected.some((x) => x.value.length > 160)),
+        },
+        intentState: state.intentState,
+        repositoryComplete: intent?.mode === "full" && state.intentState === "complete",
+        negativeScope: !intent || state.intentState !== "complete"
+            ? "unknown"
+            : intent.mode === "full"
+                ? "repository"
+                : "declared-scope",
+    };
+}