knodin 0.8.7 → 0.10.0

package/dist/src/engine/seal.js

@@ -0,0 +1,270 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import zlib from "node:zlib";
+import { KNODIN_VERSION } from "../version.js";
+import { Database } from "./sqlite.js";
+/** Storage for embedded source. Content-addressed, so duplicate files cost one blob. */
+const SEAL_SCHEMA = [
+    `CREATE TABLE IF NOT EXISTS sealed_source (
+		sha256 TEXT PRIMARY KEY,
+		bytes BLOB NOT NULL,
+		compressed INTEGER NOT NULL,
+		storedBytes INTEGER NOT NULL,
+		rawBytes INTEGER NOT NULL
+	)`,
+    `CREATE TABLE IF NOT EXISTS sealed_file (
+		filePath TEXT PRIMARY KEY,
+		sha256 TEXT NOT NULL REFERENCES sealed_source(sha256)
+	)`,
+];
+/** Line count under the same `split("\n")` convention the read paths use. */
+function countLines(buf) {
+    return buf.toString("utf8").split("\n").length;
+}
+export function readSealAttestation(db) {
+    try {
+        const row = db
+            .query("SELECT value FROM meta WHERE key = 'sealAttestation'")
+            .get();
+        return row ? JSON.parse(row.value) : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Reads embedded source out of a sealed artifact.
+ *
+ * Returns `null` — never `""` — for a path the artifact does not carry. An
+ * empty string is indistinguishable from an empty file, and downstream that
+ * reads as "this symbol has no body" rather than "this was not covered",
+ * which is exactly the silent false negative the consumer needs to be able to
+ * detect in order to fall back.
+ */
+export function readSealedSource(db, filePath) {
+    const row = db
+        .query("SELECT s.bytes bytes, s.compressed compressed FROM sealed_file f JOIN sealed_source s ON s.sha256 = f.sha256 WHERE f.filePath = ?")
+        .get(filePath);
+    if (!row)
+        return null;
+    const raw = Buffer.from(row.bytes);
+    return (row.compressed ? zlib.zstdDecompressSync(raw) : raw).toString("utf8");
+}
+/**
+ * Every path the sealed artifact carries source for.
+ *
+ * A sealed repository has no directory tree to walk, so anything that
+ * discovers files by traversal (module resolution, package manifests) has to
+ * enumerate coverage instead.
+ */
+export function listSealedFiles(db) {
+    return db
+        .query("SELECT filePath FROM sealed_file ORDER BY filePath")
+        .all()
+        .map((row) => row.filePath);
+}
+/** True when this database carries embedded source, i.e. can answer with no checkout. */
+export function isSealedDatabase(db) {
+    return readSealAttestation(db) !== null;
+}
+/**
+ * Produce a sealed artifact, or refuse with a machine-readable code.
+ *
+ * Refusal is the interesting half. A seal of a dirty or diverged index
+ * produces citations that look exactly like correct ones — the failure is
+ * invisible downstream — so it is refused here rather than annotated.
+ */
+export function sealIndex(options) {
+    const { repoPath, sourceDatabasePath, outputPath, health, identity, commit, ref, dirtyPaths } = options;
+    const stripEmbeddings = options.stripEmbeddings !== false;
+    if (!fs.existsSync(sourceDatabasePath))
+        return { ok: false, code: "database-missing", reason: `no index at ${sourceDatabasePath}` };
+    if (health.status !== "healthy")
+        return {
+            ok: false,
+            code: "index-unhealthy",
+            reason: `index status is ${health.status}; seal requires a healthy index`,
+        };
+    if (dirtyPaths.length > 0)
+        return {
+            ok: false,
+            code: "working-tree-dirty",
+            reason: `${dirtyPaths.length} path(s) differ from the index; re-index before sealing`,
+        };
+    if (!commit)
+        return {
+            ok: false,
+            code: "no-commit",
+            reason: "repository has no resolvable commit to attest",
+        };
+    if (health.pendingPaths > 0)
+        return {
+            ok: false,
+            code: "commit-divergent",
+            reason: `${health.pendingPaths} path(s) pending; the graph does not describe ${commit}`,
+        };
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.rmSync(outputPath, { force: true });
+    // Copy rather than seal in place: the live database keeps serving its own
+    // repository, and a half-written artifact can never corrupt it.
+    fs.copyFileSync(sourceDatabasePath, outputPath);
+    const db = new Database(outputPath);
+    try {
+        db.run("PRAGMA journal_mode = WAL;");
+        for (const statement of SEAL_SCHEMA)
+            db.run(statement);
+        db.run("DELETE FROM sealed_file;");
+        db.run("DELETE FROM sealed_source;");
+        let removedEmbeddings = 0;
+        if (stripEmbeddings) {
+            removedEmbeddings =
+                db.query("SELECT COUNT(*) count FROM symbol_embeddings").get()
+                    ?.count ?? 0;
+            db.run("DELETE FROM symbol_embeddings;");
+        }
+        const indexed = db
+            .query("SELECT filePath, mtimeMs, size FROM index_state ORDER BY filePath")
+            .all();
+        const insertBlob = db.prepare("INSERT OR IGNORE INTO sealed_source (sha256, bytes, compressed, storedBytes, rawBytes) VALUES (?, ?, ?, ?, ?)");
+        const insertFile = db.prepare("INSERT OR REPLACE INTO sealed_file (filePath, sha256) VALUES (?, ?)");
+        const maxEndLine = new Map();
+        for (const row of db
+            .query("SELECT filePath, MAX(endLine) lastLine FROM symbols GROUP BY filePath")
+            .all())
+            maxEndLine.set(row.filePath, row.lastLine);
+        const seen = new Set();
+        const drifted = [];
+        let files = 0;
+        let rawBytes = 0;
+        let storedBytes = 0;
+        db.run("BEGIN TRANSACTION;");
+        try {
+            for (const row of indexed) {
+                if (options.excludeSource?.(row.filePath))
+                    continue;
+                const absolute = path.join(repoPath, row.filePath);
+                let buf;
+                let stat;
+                try {
+                    stat = fs.statSync(absolute);
+                    buf = fs.readFileSync(absolute);
+                }
+                catch {
+                    // A file that vanished between indexing and sealing is simply not
+                    // embedded. Queries for it report unavailable rather than empty.
+                    continue;
+                }
+                // The drift hazard, and the reason this is a refusal rather than a
+                // warning. Seal reads the tree; the graph was built earlier. If a file
+                // changed in between, embedded bytes disagree with recorded line
+                // ranges and `explain` returns the WRONG function's body with full
+                // confidence and no omission — a false positive that looks correct,
+                // which is strictly worse than any false negative here.
+                if ((row.size !== null && row.size !== stat.size) ||
+                    (row.mtimeMs !== null && Math.abs(row.mtimeMs - stat.mtimeMs) > 1)) {
+                    drifted.push(row.filePath);
+                    continue;
+                }
+                // Defence in depth behind the mtime/size check above, and the reason
+                // it is worth having: mtime+size cannot see a same-size edit that
+                // lands inside the mtime comparison window. A range that now runs off
+                // the end of the file is proof the graph and these bytes disagree,
+                // independent of any timestamp.
+                const lastLine = maxEndLine.get(row.filePath);
+                if (lastLine !== undefined && lastLine > countLines(buf)) {
+                    drifted.push(row.filePath);
+                    continue;
+                }
+                const hash = crypto.createHash("sha256").update(buf).digest("hex");
+                files++;
+                rawBytes += buf.length;
+                if (!seen.has(hash)) {
+                    seen.add(hash);
+                    // Compressed PER BLOB, not just in transit: SQLite does not
+                    // transparently compress, so a consumer holds whatever is stored
+                    // here. Transport compression would do nothing for that.
+                    //
+                    // Keep the raw bytes when the compressed form is not smaller.
+                    // zstd framing exceeds the payload on very short files, and
+                    // storing those compressed would make the artifact larger than
+                    // the source it embeds.
+                    const packed = zlib.zstdCompressSync(buf);
+                    const useCompressed = packed.length < buf.length;
+                    const stored = useCompressed ? packed : buf;
+                    storedBytes += stored.length;
+                    insertBlob.run(hash, stored, useCompressed ? 1 : 0, stored.length, buf.length);
+                }
+                insertFile.run(row.filePath, hash);
+            }
+            db.run("COMMIT;");
+        }
+        catch (error) {
+            db.run("ROLLBACK;");
+            throw error;
+        }
+        finally {
+            insertBlob.finalize();
+            insertFile.finalize();
+        }
+        if (drifted.length > 0) {
+            db.close();
+            // Discard the partial artifact rather than shipping one whose coverage
+            // silently excludes the very files that moved.
+            fs.rmSync(outputPath, { force: true });
+            const sample = drifted.slice(0, 5).join(", ");
+            return {
+                ok: false,
+                code: "source-drift",
+                reason: `${drifted.length} file(s) changed since indexing (${sample}${drifted.length > 5 ? ", …" : ""}); re-index before sealing`,
+            };
+        }
+        const attestation = {
+            sealVersion: 1,
+            knodinVersion: KNODIN_VERSION,
+            schemaVersion: db.query("PRAGMA user_version").get()?.user_version ?? 0,
+            repository: { identity, path: repoPath, remoteUrl: options.remoteUrl ?? null },
+            sealedCommit: commit,
+            sealedRef: ref,
+            sealedAt: new Date().toISOString(),
+            health: { ...health },
+            source: {
+                policy: options.excludeSource ? "excluded" : "all-indexed",
+                files,
+                uniqueBlobs: seen.size,
+                rawBytes,
+                storedBytes,
+            },
+            embeddings: { stripped: stripEmbeddings, removed: removedEmbeddings },
+        };
+        db.run("INSERT OR REPLACE INTO meta (key, value) VALUES ('sealAttestation', ?)", [
+            JSON.stringify(attestation),
+        ]);
+        // Reclaim the pages freed by stripping embeddings and by any deletion
+        // above. Without this the artifact keeps every freed page and stripping
+        // saves nothing on disk — the whole reason embeddings are dropped. Must
+        // run outside a transaction, which is why it lands here rather than
+        // alongside the deletes.
+        db.run("VACUUM;");
+        const integrity = db.query("PRAGMA integrity_check").get()?.integrity_check ??
+            "unknown";
+        // Fold the WAL back in: a consumer receives ONE file, and an artifact
+        // whose committed state lives in a sidecar it never got is a silent
+        // truncation rather than a loud failure.
+        db.run("PRAGMA wal_checkpoint(TRUNCATE);");
+        db.run("PRAGMA journal_mode = DELETE;");
+        db.close();
+        for (const suffix of ["-wal", "-shm"])
+            fs.rmSync(`${outputPath}${suffix}`, { force: true });
+        const bytes = fs.statSync(outputPath).size;
+        const sha256 = crypto.createHash("sha256").update(fs.readFileSync(outputPath)).digest("hex");
+        return { ok: true, outputPath, bytes, attestation, integrityCheck: integrity, sha256 };
+    }
+    catch (error) {
+        try {
+            db.close();
+        }
+        catch { }
+        throw error;
+    }
+}

package/dist/src/engine/sealed-open.js

@@ -0,0 +1,116 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { KNODIN_VERSION as RUNTIME_VERSION } from "../version.js";
+import { KNODIN_SCHEMA_VERSION, registerSealedRepository, unregisterSealedRepository, } from "./index.js";
+import { readSealAttestation } from "./seal.js";
+import { Database } from "./sqlite.js";
+export function openSealedArtifact(artifactPath, options = {}) {
+    const resolved = path.resolve(artifactPath);
+    if (!fs.existsSync(resolved))
+        return { ok: false, code: "artifact-missing", reason: `no artifact at ${resolved}` };
+    let db;
+    try {
+        db = new Database(resolved, { readonly: true });
+    }
+    catch (error) {
+        return {
+            ok: false,
+            code: "unreadable",
+            reason: error instanceof Error ? error.message : String(error),
+        };
+    }
+    const attestation = readSealAttestation(db);
+    if (!attestation) {
+        db.close();
+        return {
+            ok: false,
+            code: "not-sealed",
+            reason: "database carries no seal attestation; it is an ordinary index, not an artifact",
+        };
+    }
+    // HARD gate. The graph schema is what every query is written against, so a
+    // mismatch is not a degradation — the answers would be wrong rather than
+    // narrow. A consumer falls back to live source on this code.
+    if (attestation.schemaVersion !== KNODIN_SCHEMA_VERSION) {
+        db.close();
+        return {
+            ok: false,
+            code: "schema-mismatch",
+            reason: `artifact schema ${attestation.schemaVersion} != engine schema ${KNODIN_SCHEMA_VERSION}`,
+        };
+    }
+    const degraded = [];
+    // Stripped embeddings must make semantic search report itself UNAVAILABLE.
+    // Left to return zero results it would be indistinguishable from "nothing
+    // matched", which is the silent false negative this whole design exists to
+    // prevent.
+    if (attestation.embeddings.stripped)
+        degraded.push({
+            capability: "semanticSearch",
+            reason: "embeddings were stripped at seal time; semantic search cannot run",
+        });
+    // SOFT gate, and the asymmetry with the schema check is deliberate: a
+    // sealed artifact is never re-parsed, so a knodin version that parses
+    // differently does not invalidate symbols already extracted.
+    if (attestation.knodinVersion !== KNODIN_VERSION_AT_RUNTIME()) {
+        if (options.strictCompat) {
+            db.close();
+            return {
+                ok: false,
+                code: "schema-mismatch",
+                reason: `--strict-compat: artifact knodin ${attestation.knodinVersion} != runtime`,
+            };
+        }
+        degraded.push({
+            capability: "review",
+            reason: `sealed by knodin ${attestation.knodinVersion}; heuristics may differ`,
+        });
+    }
+    // A sealed artifact has no working tree to write to, and no way to reindex.
+    degraded.push({ capability: "rename", reason: "sealed artifacts are read-only" }, { capability: "index", reason: "sealed artifacts cannot be reindexed" });
+    const mountDir = options.mountDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "knodin-sealed-mount-"));
+    // The artifact lives under `candidates/` so the engine's explicit-database
+    // guard accepts it: that guard refuses a path colliding with the active
+    // database, which is a candidate-isolation invariant rather than an
+    // obstacle to route around.
+    const mountedDb = path.join(mountDir, ".knodin", "candidates", "sealed", "db.sqlite");
+    fs.mkdirSync(path.dirname(mountedDb), { recursive: true });
+    if (!fs.existsSync(mountedDb))
+        fs.copyFileSync(resolved, mountedDb);
+    // Registered under BOTH paths. Callers address the artifact by its mount
+    // directory, but the engine resolves a repository from paths recorded in
+    // the database itself — the checkout the index was built from. Registering
+    // only the mount would leave those internal reads falling through to a
+    // filesystem that does not have the source, which fails as "" rather than
+    // loudly.
+    registerSealedRepository(mountDir, db);
+    const attestedPath = attestation.repository.path;
+    if (attestedPath && path.resolve(attestedPath) !== path.resolve(mountDir))
+        registerSealedRepository(attestedPath, db);
+    const sealedAt = Date.parse(attestation.sealedAt);
+    const ageDays = Number.isNaN(sealedAt)
+        ? 0
+        : Math.floor((Date.now() - sealedAt) / (24 * 60 * 60 * 1000));
+    return {
+        ok: true,
+        repoPath: mountDir,
+        databasePath: mountedDb,
+        attestation,
+        degraded,
+        ageDays,
+        close() {
+            unregisterSealedRepository(mountDir);
+            if (attestedPath)
+                unregisterSealedRepository(attestedPath);
+            db.close();
+        },
+    };
+}
+/**
+ * Read at call time rather than imported as a constant so that a test can
+ * exercise the soft gate without rebuilding the package version.
+ */
+function KNODIN_VERSION_AT_RUNTIME() {
+    return process.env.KNODIN_VERSION_OVERRIDE ?? RUNTIME_VERSION;
+}

package/dist/src/engine/sealed-query.js

@@ -0,0 +1,49 @@
+import { createEngine } from "./index.js";
+import { openSealedArtifact, } from "./sealed-open.js";
+export async function runSealedQuery(artifactPath, symbol, options = {}) {
+    const handle = openSealedArtifact(artifactPath, { strictCompat: options.strictCompat });
+    if (!handle.ok)
+        return { ok: false, code: handle.code, reason: handle.reason };
+    try {
+        const summary = {
+            ok: true,
+            repository: {
+                identity: handle.attestation.repository.identity,
+                commit: handle.attestation.sealedCommit,
+                ref: handle.attestation.sealedRef,
+            },
+            sealedAt: handle.attestation.sealedAt,
+            ageDays: handle.ageDays,
+            knodinVersion: handle.attestation.knodinVersion,
+            schemaVersion: handle.attestation.schemaVersion,
+            coverage: {
+                files: handle.attestation.source.files,
+                uniqueBlobs: handle.attestation.source.uniqueBlobs,
+                rawBytes: handle.attestation.source.rawBytes,
+                storedBytes: handle.attestation.source.storedBytes,
+            },
+            degraded: handle.degraded,
+        };
+        if (!symbol)
+            return summary;
+        // `mode: "sealed"` is the whole reason this wrapper exists. Without it the
+        // engine reconciles against the empty mount, prunes every symbol, and
+        // answers "not found" for content the artifact is carrying — a negative
+        // that looks entirely correct. Too important to leave to each caller.
+        const engine = createEngine({
+            watcher: "disabled",
+            mode: "sealed",
+            databasePath: handle.databasePath,
+        });
+        try {
+            summary.explained = await engine.explain(symbol, handle.repoPath);
+            return summary;
+        }
+        finally {
+            await engine.close();
+        }
+    }
+    finally {
+        handle.close();
+    }
+}