knodin 0.10.8 → 0.12.0

package/dist/src/engine/seal.js

@@ -22,16 +22,114 @@ const SEAL_SCHEMA = [
 function countLines(buf) {
     return buf.toString("utf8").split("\n").length;
 }
+/**
+ * A database claims to be an artifact but carries an attestation the reader
+ * cannot trust.
+ *
+ * Distinct from "no attestation at all", which is an ordinary index and a
+ * routine answer. Every field here is consumed as a decision downstream — the
+ * schema gate, the version gate, and the repository path the artifact is
+ * allowed to serve source for — so a half-populated object must stop at the
+ * reader rather than reach them.
+ */
+export class SealAttestationError extends Error {
+    code = "malformed-attestation";
+    constructor(reason) {
+        super(reason);
+        this.name = "SealAttestationError";
+    }
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
+ * Validates the WHOLE shape, not just the fields the immediate caller reads.
+ * A sealed artifact is transported between machines, so it is untrusted input
+ * rather than something this process wrote (KNODIN-21), and an unchecked cast
+ * would let `undefined` stand where a version or a path belongs.
+ */
+function validateAttestation(value) {
+    const bad = (reason) => {
+        throw new SealAttestationError(`seal attestation is malformed: ${reason}`);
+    };
+    if (!isRecord(value))
+        return bad("not an object");
+    const str = (key, on) => typeof on[key] === "string" ? on[key] : bad(`${key} is not a string`);
+    const num = (key, on) => typeof on[key] === "number" && Number.isFinite(on[key])
+        ? on[key]
+        : bad(`${key} is not a finite number`);
+    const nullableStr = (key, on) => on[key] === null || typeof on[key] === "string"
+        ? on[key]
+        : bad(`${key} is not a string or null`);
+    const obj = (key) => isRecord(value[key]) ? value[key] : bad(`${key} is not an object`);
+    if (value.sealVersion !== 1)
+        bad(`unsupported sealVersion ${String(value.sealVersion)}`);
+    const repository = obj("repository");
+    const health = obj("health");
+    const source = obj("source");
+    const embeddings = obj("embeddings");
+    const policy = source.policy;
+    if (policy !== "all-indexed" && policy !== "excluded")
+        bad(`unsupported source policy ${String(policy)}`);
+    const stripped = embeddings.stripped;
+    if (typeof stripped !== "boolean")
+        bad("embeddings.stripped is not a boolean");
+    return {
+        sealVersion: 1,
+        knodinVersion: str("knodinVersion", value),
+        schemaVersion: num("schemaVersion", value),
+        repository: {
+            identity: str("identity", repository),
+            path: str("path", repository),
+            remoteUrl: nullableStr("remoteUrl", repository),
+        },
+        sealedCommit: str("sealedCommit", value),
+        sealedRef: nullableStr("sealedRef", value),
+        sealedAt: str("sealedAt", value),
+        health: {
+            status: str("status", health),
+            pendingPaths: num("pendingPaths", health),
+            freshness: str("freshness", health),
+        },
+        source: {
+            policy: policy,
+            files: num("files", source),
+            uniqueBlobs: num("uniqueBlobs", source),
+            rawBytes: num("rawBytes", source),
+            storedBytes: num("storedBytes", source),
+        },
+        embeddings: { stripped: stripped, removed: num("removed", embeddings) },
+    };
+}
+/**
+ * `null` when the database carries no attestation; THROWS
+ * `SealAttestationError` when it carries one that does not validate. The split
+ * has teeth: "ordinary index" is something a caller falls back from, while a
+ * database misrepresenting itself must not be quietly downgraded to the same
+ * answer.
+ */
 export function readSealAttestation(db) {
+    let raw;
     try {
         const row = db
             .query("SELECT value FROM meta WHERE key = 'sealAttestation'")
             .get();
-        return row ? JSON.parse(row.value) : null;
+        if (!row)
+            return null;
+        raw = row.value;
     }
     catch {
+        // No `meta` table, or an unreadable one: not a sealed artifact.
         return null;
     }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        throw new SealAttestationError(`seal attestation is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return validateAttestation(parsed);
 }
 /**
  * Reads embedded source out of a sealed artifact.
@@ -64,9 +162,30 @@ export function listSealedFiles(db) {
         .all()
         .map((row) => row.filePath);
 }
-/** True when this database carries embedded source, i.e. can answer with no checkout. */
+/**
+ * Content hash of every embedded file, as recorded at seal time.
+ *
+ * The hash is over the RAW bytes, so a caller can compare against a file on
+ * disk without decompressing the blob.
+ */
+export function listSealedFileHashes(db) {
+    return db
+        .query("SELECT filePath, sha256 FROM sealed_file ORDER BY filePath")
+        .all();
+}
+/**
+ * True when this database carries embedded source, i.e. can answer with no
+ * checkout. A malformed attestation answers false: it is not an artifact any
+ * reader here can serve from, and this predicate's callers route on it rather
+ * than handle it.
+ */
 export function isSealedDatabase(db) {
-    return readSealAttestation(db) !== null;
+    try {
+        return readSealAttestation(db) !== null;
+    }
+    catch {
+        return false;
+    }
 }
 /**
  * Produce a sealed artifact, or refuse with a machine-readable code.

package/dist/src/engine/sealed-open.js

@@ -1,9 +1,10 @@
+import crypto from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import { KNODIN_VERSION as RUNTIME_VERSION } from "../version.js";
 import { KNODIN_SCHEMA_VERSION, registerSealedRepository, unregisterSealedRepository, } from "./index.js";
-import { readSealAttestation } from "./seal.js";
+import { listSealedFileHashes, readSealAttestation, SealAttestationError, } from "./seal.js";
 import { Database } from "./sqlite.js";
 export function openSealedArtifact(artifactPath, options = {}) {
     const resolved = path.resolve(artifactPath);
@@ -20,7 +21,18 @@ export function openSealedArtifact(artifactPath, options = {}) {
             reason: error instanceof Error ? error.message : String(error),
         };
     }
-    const attestation = readSealAttestation(db);
+    let attestation;
+    try {
+        attestation = readSealAttestation(db);
+    }
+    catch (error) {
+        db.close();
+        return {
+            ok: false,
+            code: error instanceof SealAttestationError ? "malformed-attestation" : "unreadable",
+            reason: error instanceof Error ? error.message : String(error),
+        };
+    }
     if (!attestation) {
         db.close();
         return {
@@ -84,10 +96,31 @@ export function openSealedArtifact(artifactPath, options = {}) {
     // only the mount would leave those internal reads falling through to a
     // filesystem that does not have the source, which fails as "" rather than
     // loudly.
-    registerSealedRepository(mountDir, db);
     const attestedPath = attestation.repository.path;
-    if (attestedPath && path.resolve(attestedPath) !== path.resolve(mountDir))
-        registerSealedRepository(attestedPath, db);
+    const registerAttested = attestedPath && path.resolve(attestedPath) !== path.resolve(mountDir);
+    // The attested path is chosen by the artifact, and the registry it lands in
+    // takes precedence over the filesystem for every source read in the
+    // process. An artifact naming a path that exists here would therefore serve
+    // its own bytes to live queries against that checkout (KNODIN-21). Sealing
+    // and opening on ONE machine is a real flow, so this refuses on
+    // DISAGREEMENT rather than on existence: where the embedded blobs match the
+    // files on disk the artifact describes that tree and shadowing it changes
+    // nothing, and where they do not, whichever side is wrong, no read of that
+    // path can be trusted.
+    if (registerAttested) {
+        const conflict = firstSourceDisagreement(db, path.resolve(attestedPath));
+        if (conflict) {
+            db.close();
+            return {
+                ok: false,
+                code: "path-conflict",
+                reason: `artifact attests ${path.resolve(attestedPath)}, which exists here with different source (${conflict}); re-seal from this checkout, or open the artifact where that path is absent`,
+            };
+        }
+    }
+    acquireSealedRepository(mountDir, db);
+    if (registerAttested)
+        acquireSealedRepository(attestedPath, db);
     const sealedAt = Date.parse(attestation.sealedAt);
     const ageDays = Number.isNaN(sealedAt)
         ? 0
@@ -100,13 +133,97 @@ export function openSealedArtifact(artifactPath, options = {}) {
         degraded,
         ageDays,
         close() {
-            unregisterSealedRepository(mountDir);
-            if (attestedPath)
-                unregisterSealedRepository(attestedPath);
+            releaseSealedRepository(mountDir, db);
+            if (registerAttested)
+                releaseSealedRepository(attestedPath, db);
             db.close();
         },
     };
 }
+/**
+ * Open handles per registered path, most recent last.
+ *
+ * The engine's registry is a bare path→database map, so two artifacts
+ * attesting the same repository would have the first `close()` unregister the
+ * other's source and leave it reading `""` (KNODIN-21). Held here rather than
+ * widened there because the map is the engine's chokepoint for a single
+ * answer to "where do bytes for this path come from"; ownership of the
+ * handles that claim it belongs to whoever opened them.
+ */
+const sealedRegistrations = new Map();
+function acquireSealedRepository(repoPath, db) {
+    const key = path.resolve(repoPath);
+    const held = sealedRegistrations.get(key) ?? [];
+    held.push(db);
+    sealedRegistrations.set(key, held);
+    registerSealedRepository(key, db);
+}
+function releaseSealedRepository(repoPath, db) {
+    const key = path.resolve(repoPath);
+    const held = sealedRegistrations.get(key);
+    if (!held)
+        return;
+    const last = held.lastIndexOf(db);
+    if (last === -1)
+        return;
+    held.splice(last, 1);
+    // Re-point at a surviving handle rather than leaving the closed one
+    // registered: an unregister here would fall through to a filesystem that
+    // has no source, and a closed database would throw on the next read.
+    const survivor = held.at(-1);
+    if (survivor)
+        registerSealedRepository(key, survivor);
+    else {
+        sealedRegistrations.delete(key);
+        unregisterSealedRepository(key);
+    }
+}
+/**
+ * First embedded file whose bytes differ from the file at the same relative
+ * path under `dir`, or `null` when every one that exists there agrees.
+ *
+ * Files absent from `dir` are not disagreement — a partially or wholly removed
+ * checkout is precisely what an artifact is for. Compared by hash against the
+ * one recorded at seal time, which is over raw bytes, so nothing is
+ * decompressed to answer this.
+ *
+ * `filePath` comes out of the artifact and is therefore attacker-controlled, so
+ * every candidate is resolved and required to stay under `dir` before it is
+ * opened. A `..` segment would otherwise have this check read and hash files
+ * outside the attested checkout — reachable during `openSealedArtifact`, and
+ * pure attack surface, since a repository-relative path never needs to escape.
+ * An entry that does is reported as disagreement rather than skipped: it is not
+ * a path this artifact can honestly claim, and failing closed keeps a traversal
+ * from being a way to hide an entry from this comparison.
+ */
+function firstSourceDisagreement(db, dir) {
+    let stat;
+    try {
+        stat = fs.statSync(dir);
+    }
+    catch {
+        return null;
+    }
+    if (!stat.isDirectory())
+        return null;
+    const root = path.resolve(dir);
+    for (const entry of listSealedFileHashes(db)) {
+        const candidate = path.resolve(root, entry.filePath);
+        if (candidate !== root && !candidate.startsWith(root + path.sep))
+            return entry.filePath;
+        let onDisk;
+        try {
+            onDisk = fs.readFileSync(candidate);
+        }
+        catch {
+            continue;
+        }
+        const hash = crypto.createHash("sha256").update(onDisk).digest("hex");
+        if (hash !== entry.sha256)
+            return entry.filePath;
+    }
+    return null;
+}
 /**
  * Read at call time rather than imported as a constant so that a test can
  * exercise the soft gate without rebuilding the package version.

package/dist/src/engine/text-matches.js

@@ -25,6 +25,24 @@ import { walkRepoFiles } from "./file-walker.js";
 const WORD_CHARACTER = /[\p{L}\p{N}_]/u;
 /** Upper bound on `TextMatch.enclosingText`, so one row cannot flood a preview. */
 const ENCLOSING_TEXT_LIMIT = 120;
+/**
+ * Largest file this search will read, in bytes.
+ *
+ * R1 deliberately drops the source-extension filter, so the walk reaches
+ * lockfiles, NDJSON dumps, fixtures and generated JSON as well as prose. Four
+ * mebibytes clears every hand-written source or document file by a wide margin
+ * while excluding the generated artefacts that would otherwise decide the
+ * process's peak memory. Oversized files are reported in `uncovered`, never
+ * dropped.
+ */
+export const MAX_SEARCHABLE_FILE_BYTES = 4 * 1024 * 1024;
+/**
+ * Largest number of matches this search will accumulate.
+ *
+ * Past this the result is no longer reviewable and the match objects alone
+ * dominate memory; the honest answer is a bounded list plus `truncated: true`.
+ */
+export const MAX_TEXT_MATCHES = 5_000;
 function isWordCharacter(value) {
     return value !== undefined && WORD_CHARACTER.test(value);
 }
@@ -164,86 +182,101 @@ function classifyAgainstTree(parser, content, raw) {
  * over a large repository touches enough files to reproduce it exactly.
  */
 export async function searchFiles(files, term, loadLanguage, newParser, options = {}) {
-    const caseSensitive = options.caseSensitive ?? true;
-    const matches = [];
-    const uncovered = [];
-    let filesSearched = 0;
+    const state = newSearchState();
     for (const { file, content } of files) {
-        const raw = findOccurrences(content, term, caseSensitive);
-        filesSearched++;
-        if (raw.length === 0)
-            continue;
-        let classified = null;
-        let language = null;
+        await searchOneFile(state, file, content, term, loadLanguage, newParser, options);
+        if (state.truncated)
+            break;
+    }
+    return { ...state, searched: true };
+}
+function newSearchState() {
+    return { matches: [], uncovered: [], filesSearched: 0, truncated: false };
+}
+/**
+ * Search and classify one file, appending to `state`.
+ *
+ * Takes a single file's content rather than a list so that a repository-wide
+ * search never has to hold more than one file in memory at once (KNODIN-24).
+ * Sets `state.truncated` when the match cap is reached; the caller is
+ * responsible for stopping there.
+ */
+async function searchOneFile(state, file, content, term, loadLanguage, newParser, options) {
+    const caseSensitive = options.caseSensitive ?? true;
+    const raw = findOccurrences(content, term, caseSensitive);
+    state.filesSearched++;
+    if (raw.length === 0)
+        return;
+    let classified = null;
+    let language = null;
+    try {
+        language = await loadLanguage(file);
+    }
+    catch {
+        // A grammar that will not load costs classification, not the match.
+        language = null;
+    }
+    if (language) {
+        const parser = newParser();
         try {
-            language = await loadLanguage(file);
+            parser.setLanguage(language);
+            classified = classifyAgainstTree(parser, content, raw);
         }
-        catch {
-            // A grammar that will not load costs classification, not the match.
-            language = null;
+        catch (error) {
+            if (isWasmAbort(error))
+                throw error;
+            // Parsing failed on this file only. The matches are still real and
+            // still reported — just without node evidence.
+            classified = null;
+            state.uncovered.push({
+                file,
+                reason: `matched but could not be parsed for classification: ${error instanceof Error ? error.message : String(error)}`,
+            });
         }
-        if (language) {
-            const parser = newParser();
+        finally {
             try {
-                parser.setLanguage(language);
-                classified = classifyAgainstTree(parser, content, raw);
+                parser.delete();
             }
-            catch (error) {
-                if (isWasmAbort(error))
-                    throw error;
-                // Parsing failed on this file only. The matches are still real and
-                // still reported — just without node evidence.
-                classified = null;
-                uncovered.push({
-                    file,
-                    reason: `matched but could not be parsed for classification: ${error instanceof Error ? error.message : String(error)}`,
-                });
-            }
-            finally {
-                try {
-                    parser.delete();
-                }
-                catch {
-                    /* a dead module has nothing to reclaim */
-                }
+            catch {
+                /* a dead module has nothing to reclaim */
             }
         }
-        const starts = lineStarts(content);
-        raw.forEach((match, position) => {
-            const lineIndex = lineIndexFor(starts, match.startIndex);
-            const lineStart = starts[lineIndex];
-            const nextStart = starts[lineIndex + 1] ?? content.length + 1;
-            const node = classified?.[position] ?? null;
-            matches.push({
-                file,
-                line: lineIndex + 1,
-                column: match.startIndex - lineStart + 1,
-                startIndex: match.startIndex,
-                endIndex: match.endIndex,
-                lineText: content.slice(lineStart, Math.max(lineStart, nextStart - 1)).replace(/\r$/, ""),
-                matchedText: match.matchedText,
-                caseMatchesQuery: match.matchedText === term,
-                withinLargerWord: isWordCharacter(content[match.startIndex - 1]) ||
-                    isWordCharacter(content[match.endIndex]),
-                classification: node ? node.classification : classifyByFileType(file),
-                basis: node ? "parse-node" : language ? "none" : "file-type",
-                nodeType: node ? node.nodeType : null,
-                enclosingText: node ? node.enclosingText : null,
-            });
+    }
+    const starts = lineStarts(content);
+    for (const [position, match] of raw.entries()) {
+        if (state.matches.length >= MAX_TEXT_MATCHES) {
+            state.truncated = true;
+            return;
+        }
+        const lineIndex = lineIndexFor(starts, match.startIndex);
+        const lineStart = starts[lineIndex];
+        const nextStart = starts[lineIndex + 1] ?? content.length + 1;
+        const node = classified?.[position] ?? null;
+        state.matches.push({
+            file,
+            line: lineIndex + 1,
+            column: match.startIndex - lineStart + 1,
+            startIndex: match.startIndex,
+            endIndex: match.endIndex,
+            lineText: content.slice(lineStart, Math.max(lineStart, nextStart - 1)).replace(/\r$/, ""),
+            matchedText: match.matchedText,
+            caseMatchesQuery: match.matchedText === term,
+            withinLargerWord: isWordCharacter(content[match.startIndex - 1]) || isWordCharacter(content[match.endIndex]),
+            classification: node ? node.classification : classifyByFileType(file),
+            basis: node ? "parse-node" : language ? "none" : "file-type",
+            nodeType: node ? node.nodeType : null,
+            enclosingText: node ? node.enclosingText : null,
         });
     }
-    return { matches, uncovered, filesSearched, searched: true };
 }
-/**
- * Read a file for searching, or say why it could not be.
- *
- * Binary detection is a NUL-byte sniff over the head of the file rather than an
- * extension list: the extensions that matter here are open-ended, and a
- * misjudged binary read produces garbage matches rather than an honest skip.
- */
-function readSearchable(absolute) {
+export function readSearchable(absolute) {
     let buffer;
     try {
+        const stats = fs.statSync(absolute);
+        if (stats.size > MAX_SEARCHABLE_FILE_BYTES)
+            return {
+                reason: `too large to search: ${stats.size} bytes exceeds the ${MAX_SEARCHABLE_FILE_BYTES}-byte limit`,
+            };
         buffer = fs.readFileSync(absolute);
     }
     catch (error) {
@@ -264,8 +297,15 @@ function readSearchable(absolute) {
  * Every file that could not be searched is returned in `uncovered` (R5). A
  * rename that silently skipped files would leave a half-renamed repository
  * looking finished, which is the worst available outcome for this operation.
+ *
+ * Files are read and searched one at a time. Reading them all up front held the
+ * entire non-pruned working tree in memory at once, which on a tree carrying
+ * generated dumps or vendored data is gigabytes for a single call (KNODIN-24).
+ *
+ * `readFile` is injected only so a test can observe that reads interleave with
+ * searching rather than preceding it.
  */
-export async function searchRepoText(repoPath, term, loadLanguage, newParser, options = {}) {
+export async function searchRepoText(repoPath, term, loadLanguage, newParser, options = {}, readFile = readSearchable) {
     let candidates;
     try {
         // Checked explicitly because `walkRepoFiles` swallows a missing directory
@@ -291,19 +331,21 @@ export async function searchRepoText(repoPath, term, loadLanguage, newParser, op
             ],
             filesSearched: 0,
             searched: false,
+            truncated: false,
         };
     }
-    const readable = [];
-    const uncovered = [];
-    for (const file of candidates) {
-        const outcome = readSearchable(path.join(repoPath, file));
-        if ("reason" in outcome)
-            uncovered.push({ file, reason: outcome.reason });
-        else
-            readable.push({ file, content: outcome.content });
-    }
-    const result = await searchFiles(readable, term, loadLanguage, newParser, options);
+    const state = newSearchState();
     // Files skipped at read time and files that failed classification are both
     // gaps in the same claim, so they are reported through one list.
-    return { ...result, uncovered: [...uncovered, ...result.uncovered] };
+    for (const file of candidates) {
+        const outcome = readFile(path.join(repoPath, file));
+        if ("reason" in outcome) {
+            state.uncovered.push({ file, reason: outcome.reason });
+            continue;
+        }
+        await searchOneFile(state, file, outcome.content, term, loadLanguage, newParser, options);
+        if (state.truncated)
+            break;
+    }
+    return { ...state, searched: true };
 }

package/dist/src/graph-query-health.js

@@ -101,15 +101,21 @@ export function decorateGraphQueryResult(value, graphState = "healthy", freshnes
         };
     }
     const root = value;
-    const noMatch = hasNoMatches(root) && root.ambiguity === undefined;
+    // An unresolved target is not a no-match: nothing was searched for. Calling
+    // it `no-match` is what let a misspelled symbol read as proof that nothing
+    // calls it (KNODIN-28). Ambiguity is exempted for the same reason.
+    const notFound = root.targetResolution === "not-found";
+    const noMatch = hasNoMatches(root) && root.ambiguity === undefined && !notFound;
+    const state = notFound ? "target-not-found" : noMatch ? "no-match" : graphState;
     return {
         ...root,
         ...(freshness ? { freshness } : {}),
+        ...(notFound && root.status === undefined ? { status: "target-not-found" } : {}),
         ...(noMatch && root.status === undefined ? { status: "no-match" } : {}),
         availability: {
-            state: noMatch ? "no-match" : graphState,
+            state,
             graphState,
-            outcome: noMatch ? "no-match" : "matches",
+            outcome: notFound ? "target-not-found" : noMatch ? "no-match" : "matches",
         },
     };
 }