knodin 0.7.6 → 0.8.2

package/dist/src/engine/state-paths.js

@@ -0,0 +1,175 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+/**
+ * Resolves where a repository's knodin state (graph database, wiki, hook
+ * scaffolding) lives.
+ *
+ * A normal checkout keeps its state in-tree at `<repo>/.knodin`, unchanged from
+ * the original hardcoded behaviour. A *mirror* — a read-only shallow clone of a
+ * repository the user never checked out themselves — keeps its state outside the
+ * clone entirely, so that a refetch can discard and replace the source tree
+ * without destroying a graph that took minutes to build.
+ *
+ * Mirror layout, rooted at {@link mirrorRoot}:
+ *
+ * ```
+ * <root>/registry.json      the mirror index
+ * <root>/.staging/<id>      in-progress clones, renamed into place on success
+ * <root>/<id>/source        the shallow clone
+ * <root>/<id>/state         db.sqlite, wiki/, and friends
+ * <root>/<id>/README        marker: local edits here are discarded on refresh
+ * ```
+ *
+ * Every consumer must route through {@link resolveStateDir} /
+ * {@link resolveDbPath} rather than rebuilding `path.join(repo, ".knodin")`
+ * inline, or mirrors silently read from the wrong location.
+ */
+/** Directory name holding the shallow clone inside a mirror entry. */
+export const MIRROR_SOURCE_DIR = "source";
+/** Directory name holding graph state inside a mirror entry. */
+export const MIRROR_STATE_DIR = "state";
+/** In-tree state directory name for a normal checkout. */
+export const IN_TREE_STATE_DIR = ".knodin";
+const REGISTRY_VERSION = 1;
+const EMPTY_REGISTRY = { version: REGISTRY_VERSION, mirrors: [] };
+/**
+ * Root for all mirror storage. Defaults to `~/.knodin/mirrors` rather than an
+ * XDG data directory: a mirror costs minutes of indexing to rebuild, so it needs
+ * to sit somewhere the user will actually notice and can audit with
+ * `knodin remote list`. `KNODIN_MIRROR_ROOT` overrides it.
+ */
+export function mirrorRoot() {
+    const override = process.env.KNODIN_MIRROR_ROOT;
+    return override ? path.resolve(override) : path.join(os.homedir(), ".knodin", "mirrors");
+}
+/** Path to the mirror registry document. */
+export function mirrorRegistryPath() {
+    return path.join(mirrorRoot(), "registry.json");
+}
+/** Root for in-progress clones, renamed into place only once complete. */
+export function mirrorStagingRoot() {
+    return path.join(mirrorRoot(), ".staging");
+}
+/** Entry root for one mirror: the parent of both `source/` and `state/`. */
+export function mirrorEntryPath(identity) {
+    return path.join(mirrorRoot(), identity);
+}
+/** Absolute path to a mirror's shallow clone. */
+export function mirrorSourcePath(identity) {
+    return path.join(mirrorEntryPath(identity), MIRROR_SOURCE_DIR);
+}
+/** Absolute path to a mirror's graph state directory. */
+export function mirrorStatePath(identity) {
+    return path.join(mirrorEntryPath(identity), MIRROR_STATE_DIR);
+}
+let cache = null;
+function isMirrorRecord(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const record = value;
+    return (typeof record.identity === "string" &&
+        typeof record.url === "string" &&
+        typeof record.path === "string" &&
+        typeof record.fetchedAt === "string" &&
+        typeof record.sha === "string");
+}
+function indexByPath(registry) {
+    const byPath = new Map();
+    for (const record of registry.mirrors)
+        byPath.set(path.resolve(record.path), record);
+    return byPath;
+}
+/**
+ * Reads the mirror registry, memoized against the file's mtime and size so a
+ * mirror added by a concurrent `knodin remote add` is picked up without paying a
+ * JSON parse on every path resolution.
+ *
+ * The memo key is (mtimeMs, size), which is a heuristic rather than a guarantee:
+ * an out-of-process write landing in the same millisecond AND producing the same
+ * byte length would not invalidate it. Accepted deliberately — that requires a
+ * concurrent edit of identical length within one millisecond, the in-process
+ * writer drops the memo explicitly, and the cost of being wrong is one stale
+ * read, not corruption. Recorded so it reads as a decision, not an oversight.
+ *
+ * A missing or corrupt registry reads as empty. That is deliberate too: it
+ * degrades to "no mirrors configured", which routes every repository to its
+ * in-tree state — the pre-existing behaviour — rather than failing a query
+ * outright.
+ */
+export function readMirrorRegistry() {
+    const file = mirrorRegistryPath();
+    let stat;
+    try {
+        stat = fs.statSync(file);
+    }
+    catch {
+        cache = null;
+        return EMPTY_REGISTRY;
+    }
+    if (cache?.mtimeMs === stat.mtimeMs && cache?.size === stat.size)
+        return cache.registry;
+    try {
+        const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
+        const mirrors = typeof parsed === "object" &&
+            parsed !== null &&
+            Array.isArray(parsed.mirrors)
+            ? parsed.mirrors.filter(isMirrorRecord)
+            : [];
+        const registry = { version: REGISTRY_VERSION, mirrors };
+        cache = { mtimeMs: stat.mtimeMs, size: stat.size, registry, byPath: indexByPath(registry) };
+        return registry;
+    }
+    catch {
+        cache = null;
+        return EMPTY_REGISTRY;
+    }
+}
+/** Persists the registry and drops the memo so the next read reloads it. */
+export function writeMirrorRegistry(registry) {
+    const file = mirrorRegistryPath();
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    const document = { version: REGISTRY_VERSION, mirrors: registry.mirrors };
+    fs.writeFileSync(file, `${JSON.stringify(document, null, 2)}\n`);
+    cache = null;
+}
+/** Clears the in-process registry memo. Test seam. */
+export function __resetMirrorRegistryCache() {
+    cache = null;
+}
+/**
+ * Returns the mirror record whose clone is at `repoPath`, if any. Matching is on
+ * the resolved path, so a caller passing a relative or unnormalized path still
+ * resolves correctly.
+ */
+export function lookupMirror(repoPath) {
+    readMirrorRegistry();
+    return cache?.byPath.get(path.resolve(repoPath));
+}
+/** True when `repoPath` is a registered read-only mirror clone. */
+export function isMirror(repoPath) {
+    return lookupMirror(repoPath) !== undefined;
+}
+/**
+ * Directory holding knodin state for `repoPath`: in-tree `.knodin` for a normal
+ * checkout, an out-of-tree sibling of the clone for a mirror.
+ */
+export function resolveStateDir(repoPath) {
+    const resolved = path.resolve(repoPath);
+    const mirror = lookupMirror(resolved);
+    if (mirror)
+        return mirrorStatePath(mirror.identity);
+    return path.join(resolved, IN_TREE_STATE_DIR);
+}
+/** Graph database path for `repoPath`. */
+export function resolveDbPath(repoPath) {
+    return path.join(resolveStateDir(repoPath), "db.sqlite");
+}
+/**
+ * Whether opening an index for `repoPath` may write to the repository itself —
+ * false for mirrors, whose source tree is replaced wholesale on refresh and must
+ * stay byte-identical to the remote in the meantime.
+ */
+export function mayWriteToRepository(repoPath) {
+    return !isMirror(repoPath);
+}

package/dist/src/engine/symbol-delete.js

@@ -0,0 +1,58 @@
+/**
+ * Single source of truth for "delete symbols, and everything keyed off them" (R17).
+ *
+ * `symbol_embeddings.symbolId` declares
+ * `REFERENCES symbols(id) ON DELETE CASCADE`, but SQLite disables foreign-key
+ * enforcement per connection by default, so for the life of the project that
+ * cascade never fired: `symbols.id` is AUTOINCREMENT, every re-index of a file
+ * deletes its symbols and reinserts them under NEW ids, and the embeddings
+ * pointing at the old ids were stranded forever. On `/path/to/large-repository` that had
+ * reached 37,258 orphans against 18,629 live symbols — 67% of the table, all
+ * of it scanned by semantic search's exact cosine pass.
+ *
+ * `getOrInitDb` now sets `PRAGMA foreign_keys = ON`, which makes the declared
+ * cascade real. These helpers are the second layer: they delete the embeddings
+ * EXPLICITLY, so correctness does not depend on a per-connection pragma that
+ * some future connection path forgets to set. Every `DELETE FROM symbols` site
+ * in the engine goes through here — the R14 lesson (one policy, many thin
+ * adapters) applied to deletion instead of ignore rules.
+ */
+/**
+ * Shared predicate for "this embedding's symbol no longer exists" — the
+ * single source of truth for both the DELETE below and the `stats` orphan
+ * COUNT in `src/engine/index.ts`, which previously hardcoded the identical
+ * SQL as an independent string literal.
+ */
+export const ORPHANED_EMBEDDING_PREDICATE = "symbolId NOT IN (SELECT id FROM symbols)";
+/**
+ * Purge embeddings whose symbol no longer exists.
+ *
+ * Idempotent and cheap — a single correlated DELETE. Used by the 13 → 14
+ * migration to clean indexes polluted before the cascade worked, and safe to
+ * call on an empty or freshly-created table.
+ *
+ * Deliberately does NOT touch live embeddings: re-embedding a real corpus is
+ * ~40 minutes of CPU, so this must stay a purge and never become a wipe.
+ */
+export function purgeOrphanEmbeddings(db) {
+    db.run(`DELETE FROM symbol_embeddings WHERE ${ORPHANED_EMBEDDING_PREDICATE}`);
+}
+/** Delete every symbol declared in one repo-relative file, plus its embeddings. */
+export function deleteSymbolsForFile(db, filePath) {
+    db.run("DELETE FROM symbol_embeddings WHERE symbolId IN (SELECT id FROM symbols WHERE filePath = ?)", [filePath]);
+    db.run("DELETE FROM symbols WHERE filePath = ?", [filePath]);
+}
+/**
+ * Delete every symbol whose file path matches a LIKE pattern, plus its
+ * embeddings. Used by the dbt path, whose symbols live under synthetic
+ * `.knodin/dbt/%` paths rather than one real file.
+ */
+export function deleteSymbolsMatchingPath(db, likePattern) {
+    db.run("DELETE FROM symbol_embeddings WHERE symbolId IN (SELECT id FROM symbols WHERE filePath LIKE ?)", [likePattern]);
+    db.run("DELETE FROM symbols WHERE filePath LIKE ?", [likePattern]);
+}
+/** Delete every symbol in the index, plus every embedding. Full re-index only. */
+export function deleteAllSymbols(db) {
+    db.run("DELETE FROM symbol_embeddings;");
+    db.run("DELETE FROM symbols;");
+}

package/dist/src/execution-profile.js

@@ -0,0 +1,208 @@
+import childProcess from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { compressOutput } from "./output-compression.js";
+function defaultGlobalConfigPath() {
+    const base = process.env.XDG_CONFIG_HOME
+        ? path.resolve(process.env.XDG_CONFIG_HOME)
+        : path.join(os.homedir(), ".config");
+    return path.join(base, "knodin", "execution.json");
+}
+function readDocument(file) {
+    try {
+        return JSON.parse(fs.readFileSync(file, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+function sandboxLiteral(value) {
+    return `"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
+}
+function macosPolicy(repo, _executable) {
+    const protectedRoots = ["/Users", "/Volumes", "/Network", "/private/tmp", "/private/var/folders"];
+    const protectedSubpaths = protectedRoots
+        .map((item) => `(subpath ${sandboxLiteral(item)})`)
+        .join(" ");
+    return [
+        "(version 1)",
+        "(allow default)",
+        "(deny network*)",
+        "(deny process-fork)",
+        `(deny file-read* ${protectedSubpaths})`,
+        "(deny file-write*)",
+        `(allow file-read* file-write* (subpath ${sandboxLiteral(repo)}) (literal "/dev/null"))`,
+    ].join(" ");
+}
+function unavailable(profile, warning) {
+    return {
+        schemaVersion: 1,
+        status: "unavailable",
+        profile,
+        exit: { code: null, signal: null },
+        containment: {
+            level: "containment-unavailable",
+            platform: process.platform,
+            adapter: "unavailable",
+            filesystem: "unavailable",
+            network: "unavailable",
+            processes: "unavailable",
+            warning,
+        },
+        limits: { timeoutMs: 0, maxOutputBytes: 0, maxProcesses: 1 },
+        output: null,
+        audit: { commandTextRecorded: false, profileRecorded: true },
+    };
+}
+function validateProfile(profileName, value) {
+    if (!value || typeof value !== "object")
+        throw new Error(`execution profile $profileNameis invalid`);
+    const profile = value;
+    if (!path.isAbsolute(profile.executable ?? ""))
+        throw new Error(`execution profile $profileName: executable must be absolute`);
+    if (!Array.isArray(profile.args) ||
+        profile.args.some((arg) => typeof arg !== "string" || arg.includes("\0")))
+        throw new Error(`execution profile ${profileName}: args must be a NUL-free string array`);
+    if (profile.network !== "deny")
+        throw new Error(`execution profile ${profileName}: network must be deny`);
+    if (!Number.isInteger(profile.timeoutMs) ||
+        (profile.timeoutMs ?? 0) < 1 ||
+        (profile.timeoutMs ?? 0) > 300_000)
+        throw new Error(`execution profile ${profileName}: timeoutMs must be 1..300000`);
+    if (!Number.isInteger(profile.maxOutputBytes) ||
+        (profile.maxOutputBytes ?? 0) < 256 ||
+        (profile.maxOutputBytes ?? 0) > 16 * 1024 * 1024)
+        throw new Error(`execution profile ${profileName}: maxOutputBytes must be 256..16777216`);
+    if (profile.maxProcesses !== 1)
+        throw new Error(`execution profile ${profileName}: maxProcesses must be 1 for hard fork denial`);
+    if (profile.containment !== undefined && profile.containment !== "fully-contained")
+        throw new Error(`execution profile ${profileName}: only fully-contained is supported`);
+    return profile;
+}
+function appendAudit(repo, record) {
+    const destination = path.join(repo, ".knodin", "execution-audit.jsonl");
+    fs.mkdirSync(path.dirname(destination), { recursive: true });
+    fs.appendFileSync(destination, `${JSON.stringify(record)}\n`, { mode: 0o600 });
+}
+export async function runExecutionProfile(options) {
+    const repo = fs.realpathSync(options.repo);
+    if (!/^[a-z][a-z0-9_-]{0,63}$/.test(options.profile))
+        throw new Error("execution profile identity must match [a-z][a-z0-9_-]{0,63}");
+    const globalDocument = readDocument(options.globalConfigPath ?? defaultGlobalConfigPath());
+    const repositoryDocument = readDocument(path.join(repo, ".knodin", "execution.json"));
+    if (globalDocument?.execution?.enabled !== true)
+        return unavailable(options.profile, "execution is disabled globally");
+    if (repositoryDocument?.execution?.enabled !== true)
+        return unavailable(options.profile, "execution is disabled for this repository");
+    const configured = repositoryDocument.execution.profiles?.[options.profile];
+    if (!configured)
+        throw new Error(`execution profile not found: ${options.profile}`);
+    const profile = validateProfile(options.profile, configured);
+    if (process.platform !== "darwin" || !fs.existsSync("/usr/bin/sandbox-exec"))
+        return unavailable(options.profile, "no certified containment adapter is available on this platform");
+    const executable = fs.realpathSync(profile.executable);
+    const temporaryDirectory = fs.mkdtempSync(path.join(repo, ".knodin", "execution-tmp-"));
+    const environment = {
+        PATH: `${path.dirname(executable)}:/usr/bin:/bin`,
+        LANG: "C.UTF-8",
+        LC_ALL: "C.UTF-8",
+        TMPDIR: temporaryDirectory,
+    };
+    const policy = macosPolicy(repo, executable);
+    const proc = childProcess.spawn("/usr/bin/sandbox-exec", ["-p", policy, executable, ...profile.args], {
+        cwd: repo,
+        env: environment,
+        shell: false,
+        detached: true,
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = Buffer.alloc(0);
+    let stderr = Buffer.alloc(0);
+    let reason = "exit";
+    let stopped = false;
+    const stop = (next) => {
+        if (stopped || reason !== "exit")
+            return;
+        reason = next;
+        if (proc.pid)
+            try {
+                process.kill(-proc.pid, "SIGKILL");
+            }
+            catch { }
+    };
+    const append = (stream, chunk) => {
+        const remaining = Math.max(0, profile.maxOutputBytes - stdout.length - stderr.length);
+        const accepted = chunk.subarray(0, remaining);
+        if (stream === "stdout")
+            stdout = Buffer.concat([stdout, accepted]);
+        else
+            stderr = Buffer.concat([stderr, accepted]);
+        if (accepted.length < chunk.length)
+            stop("output-limit");
+    };
+    proc.stdout.on("data", (chunk) => append("stdout", chunk));
+    proc.stderr.on("data", (chunk) => append("stderr", chunk));
+    const timer = setTimeout(() => stop("timeout"), profile.timeoutMs);
+    const closed = await new Promise((resolve, reject) => {
+        proc.once("error", reject);
+        proc.once("close", (code, signal) => resolve({ code, signal }));
+    });
+    stopped = true;
+    clearTimeout(timer);
+    fs.rmSync(temporaryDirectory, { recursive: true, force: true });
+    const combined = [
+        ...(stdout.length ? [{ stream: "stdout", text: stdout.toString("utf8") }] : []),
+        ...(stderr.length ? [{ stream: "stderr", text: stderr.toString("utf8") }] : []),
+    ];
+    const output = compressOutput(repo, {
+        events: combined,
+        exitCode: closed.code,
+        signal: closed.signal,
+        lineBudget: 200,
+        byteBudget: Math.min(profile.maxOutputBytes, 16_384),
+        retain: true,
+    });
+    const finalReason = reason;
+    // A non-zero exit only reads as "failed" once the bounded-run reasons that
+    // preempt it (deadline, output cap) have been ruled out.
+    let status;
+    if (finalReason === "timeout")
+        status = "timed-out";
+    else if (finalReason === "output-limit")
+        status = "output-limit";
+    else
+        status = closed.code === 0 ? "completed" : "failed";
+    const result = {
+        schemaVersion: 1,
+        status,
+        profile: options.profile,
+        exit: closed,
+        containment: {
+            level: "fully-contained",
+            platform: process.platform,
+            adapter: "macos-sandbox-exec",
+            filesystem: "repository-write-user-data-read-bounded",
+            network: "deny",
+            processes: "fork-denied",
+            warning: "macOS sandbox-exec is deprecated. Writes are repository-bound; system runtime paths remain readable while user homes, mounted volumes, network paths, and shared temporary roots are read-denied outside the repository.",
+        },
+        limits: {
+            timeoutMs: profile.timeoutMs,
+            maxOutputBytes: profile.maxOutputBytes,
+            maxProcesses: 1,
+        },
+        output,
+        audit: { commandTextRecorded: false, profileRecorded: true },
+    };
+    appendAudit(repo, {
+        timestamp: new Date().toISOString(),
+        profile: options.profile,
+        status,
+        exitCode: closed.code,
+        signal: closed.signal,
+        containment: result.containment.level,
+        outputArtifactId: output.artifact.id,
+    });
+    return result;
+}