knodin 0.9.0 → 0.10.0

package/dist/src/worktree-seed.js

@@ -0,0 +1,172 @@
+import childProcess from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { Database } from "./engine/sqlite.js";
+import { resolveDbPath } from "./engine/state-paths.js";
+import { deriveSharedOverlay } from "./shared-index/overlay.js";
+import { commitRelation } from "./shared-index/selection.js";
+import { KNODIN_VERSION } from "./version.js";
+import { inspectWorktrees } from "./worktree-lifecycle.js";
+const OPT_OUT_ENV = "KNODIN_DISABLE_WORKTREE_SEED";
+function readHead(repoPath) {
+    try {
+        return childProcess
+            .execFileSync("git", ["rev-parse", "HEAD"], {
+            cwd: repoPath,
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        })
+            .trim();
+    }
+    catch {
+        return null;
+    }
+}
+/** Opens `dbPath` read-only and confirms schema + build compatibility with this process. */
+function siblingIsCompatible(dbPath, schemaVersion) {
+    let db = null;
+    try {
+        db = new Database(dbPath, { readonly: true });
+        const version = db.query("PRAGMA user_version").get()?.user_version ?? 0;
+        if (version !== schemaVersion)
+            return null;
+        const versionRow = db
+            .query("SELECT value FROM meta WHERE key = ?")
+            .get("knodinVersion");
+        if (!versionRow || versionRow.value !== KNODIN_VERSION)
+            return null;
+        const headRow = db
+            .query("SELECT value FROM meta WHERE key = ?")
+            .get("lastIndexedHead");
+        return headRow?.value || null;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        db?.close();
+    }
+}
+/**
+ * Finds the cheapest-to-reconcile already-indexed sibling worktree of `repoPath`,
+ * or `null` when no eligible sibling exists (single-worktree repo, no healthy
+ * siblings, or none pass the schema/build compatibility gate).
+ */
+export async function selectSeedSibling(repoPath, schemaVersion, status) {
+    const resolved = path.resolve(repoPath);
+    const head = readHead(resolved);
+    if (!head)
+        return null;
+    let worktrees;
+    try {
+        worktrees = (await inspectWorktrees(resolved, status)).worktrees;
+    }
+    catch {
+        return null;
+    }
+    if (worktrees.length < 2)
+        return null;
+    const eligible = worktrees.filter((worktree) => path.resolve(worktree.path) !== resolved &&
+        worktree.graphInitialized &&
+        worktree.freshness !== "removed");
+    const ranked = eligible
+        .map((worktree) => {
+        const dbPath = resolveDbPath(worktree.path);
+        const indexedHead = siblingIsCompatible(dbPath, schemaVersion);
+        if (!indexedHead)
+            return null;
+        const relation = commitRelation(resolved, head, indexedHead);
+        const distance = relation ? relation.distance : Number.MAX_SAFE_INTEGER;
+        return { worktree, dbPath, indexedHead, distance };
+    })
+        .filter((entry) => entry !== null)
+        .sort((left, right) => left.distance - right.distance);
+    const best = ranked[0];
+    if (!best)
+        return null;
+    return {
+        siblingPath: best.worktree.path,
+        siblingDbPath: best.dbPath,
+        siblingIndexedHead: best.indexedHead,
+    };
+}
+/**
+ * Seeds a brand-new worktree's graph database from an already-indexed sibling's
+ * clean baseline (OS-level copy-on-write clone where supported) and reconciles it
+ * to this worktree's actual HEAD/working tree, instead of building from scratch.
+ */
+export async function seedWorktreeIndex(repoPath, engine, schemaVersion) {
+    const resolved = path.resolve(repoPath);
+    const sibling = await selectSeedSibling(resolved, schemaVersion, (p) => engine.status(p));
+    if (!sibling)
+        return { seeded: false, reason: "no eligible sibling worktree" };
+    let candidate;
+    try {
+        candidate = await engine.createCandidate(resolved, {
+            sourceDatabasePath: sibling.siblingDbPath,
+            sourceLabel: sibling.siblingPath,
+        });
+        const overlay = deriveSharedOverlay(resolved, sibling.siblingIndexedHead);
+        const reconciliation = await engine.reconcileCandidate(candidate, resolved, overlay.changedPaths);
+        const audit = await engine.auditCandidate(candidate, { mode: "deep" });
+        if (!audit.healthy) {
+            return { seeded: false, reason: audit.issues.join(",") || "candidate audit failed" };
+        }
+        await engine.promoteCandidate(candidate);
+        const promoted = candidate;
+        candidate = undefined;
+        const health = await engine.status(resolved, { audit: "deep", ignoreActiveOperation: true });
+        return {
+            seeded: true,
+            reflinked: promoted.reflinked ?? false,
+            seededFrom: sibling.siblingPath,
+            reconciledPaths: reconciliation.reconciled,
+            status: health,
+        };
+    }
+    catch (error) {
+        return { seeded: false, reason: error instanceof Error ? error.message : "seed failed" };
+    }
+    finally {
+        if (candidate) {
+            try {
+                await engine.discardCandidate(candidate);
+            }
+            catch {
+                // Best-effort cleanup; a fresh candidate directory is harmless to leave for the next attempt.
+            }
+        }
+    }
+}
+function toIndexResult(status, reconciledPaths) {
+    return {
+        indexed: reconciledPaths,
+        unchanged: [],
+        verification: {
+            status: status.status,
+            issueCount: status.missing.files.length + status.missing.records.length,
+            missing: status.missing,
+            verifiedAt: status.verification.verifiedAt,
+        },
+    };
+}
+/**
+ * Drop-in replacement for `(repo, options) => engine.index(repo, undefined, false, options)`
+ * used by every `knodin init`-family command: for a brand-new worktree, tries to
+ * seed-and-reconcile from a sibling before falling back to a full from-scratch
+ * index. For a worktree that already has a database (repair/update path),
+ * behaves exactly like the wrapped full index call.
+ */
+export function indexOrSeed(engine, schemaVersion) {
+    const fullIndex = (target, options) => engine.index(target, undefined, false, options);
+    return async (target, options) => {
+        if (fs.existsSync(resolveDbPath(target)))
+            return fullIndex(target, options);
+        if (process.env[OPT_OUT_ENV] === "1")
+            return fullIndex(target, options);
+        const seed = await seedWorktreeIndex(target, engine, schemaVersion);
+        if (seed.seeded)
+            return toIndexResult(seed.status, seed.reconciledPaths);
+        return fullIndex(target, options);
+    };
+}

package/docs/SHARED-INDEX-CONTRACT.md

@@ -114,6 +114,15 @@ the signed output size and digest, rejects trailing compressed data, validates
 the SQLite header and internal commit/schema/fingerprint, reconciles final Git
 and filesystem truth, deep-audits, and only then promotes.
 
+A brand-new worktree's distinct database may itself be populated by cloning an
+already-indexed sibling worktree's baseline (an OS-level copy-on-write file
+clone where the filesystem supports it, falling back to a plain copy
+otherwise) instead of a from-scratch build — see `knodin init`'s seed-and-
+reconcile path in `src/worktree-seed.ts`. That clone is reconciled to the new
+worktree's actual state through this same overlay-diff machinery before
+promotion, so the "distinct, writable" guarantee above is unchanged; only how
+the initial bytes are produced differs.
+
 ## Operator surfaces
 
 - `knodin shared status` is network-free. `--probe` is the explicit S3/auth

package/docs/releases/0.10.0.md

@@ -0,0 +1,127 @@
+# knodin 0.10.0
+
+This release makes a full index roughly 2.8x faster, turns parallel parsing on
+by default, adds sealed artifacts so a repository can be queried with no
+checkout at all, and makes unreadable source distinguishable from source that
+does not exist. It also makes `knodin init` fast on a brand-new git worktree by
+seeding its graph from an already-indexed sibling.
+
+## Sealed artifacts: query a repository with no checkout
+
+An ordinary `.knodin/db.sqlite` stores `filePath` plus line ranges and reads the
+bytes off a live working tree, so it cannot answer anything on a machine that
+has no clone. `knodin seal` embeds those bytes and stamps an attestation
+describing exactly which tree they came from.
+
+- `knodin seal --output <path.sqlite>` produces a portable artifact. Sealing is
+  a post-index transform, never index-time work: an ordinary user with a
+  checkout should not pay for source already on disk, so opting in is
+  structural rather than a matter of flag discipline. The live database is
+  copied, never mutated.
+- `knodin sealed <artifact> [symbol]` inspects or queries an artifact, and the
+  same capability is available as a `sealed` operation on the MCP gateway. Both
+  are read-only; creating an artifact stays CLI-only because it writes to disk.
+- The attestation carries schema version, knodin version, canonical
+  `owner/name` repository identity, the exact commit, **and which ref** was
+  sealed — a citation cannot be checked without knowing whether it came from
+  `main` or from whatever production is running.
+- Refusals are machine-readable codes rather than prose, because a caller
+  branches on them: `database-missing`, `index-unhealthy`, `working-tree-dirty`,
+  `commit-divergent`, `no-commit`, `source-drift`.
+- `source-drift` refuses the whole seal when a file changed between indexing
+  and sealing. Embedded bytes that disagree with recorded line ranges make
+  `explain` return the wrong function's body with full confidence and no
+  omission — a false positive that looks correct, which is worse than any
+  absence. Two checks run: `mtimeMs`/`size` against `index_state`, and a bounds
+  check that recorded line ranges fit the embedded bytes, which holds without
+  consulting a clock at all.
+- **Known residual, stated rather than implied:** a same-size edit that also
+  preserves line count and lands inside the mtime comparison window is still
+  invisible. Closing it needs a content hash in `index_state`, which forces a
+  full reindex and is tracked separately.
+- Uncovered content reads as absent, never as an empty string. An empty string
+  is indistinguishable from an empty file and downstream reads as "this symbol
+  has no body" rather than "this was not covered".
+- Size: profiles are excluded by default under the named rule
+  `profiles-excluded` (58% of one Salesforce payload in 187 files, already
+  captured as graph edges); blobs are zstd-compressed **inside** the database,
+  since SQLite does not transparently compress and a consumer holds the
+  decompressed file resident; storage is content-addressed so duplicate files
+  cost one blob; and embeddings are stripped unless `--keep-embeddings` is
+  passed, with semantic search then reporting itself unavailable rather than
+  returning zero results.
+- Measured on this repository: 832 files, 6.05 MB of source compressing to
+  2.05 MB (2.95x), a 17 MB artifact, `integrity_check: ok`.
+- Opening an artifact requires `mode: "sealed"`. Without it the engine
+  reconciles the graph against an empty mount, prunes every symbol, and answers
+  "not found" for content the artifact is carrying.
+
+## A full index is about 2.8x faster
+
+- Scoping an FTS trigger to the columns it actually mirrors, and removing a
+  whole-body string build from symbol identity, took a full index of this
+  repository from 60.5s to 21.3s.
+- Salesforce metadata is now symbolized by kind rather than as one
+  undifferentiated `salesforce-metadata` symbol carrying identical boilerplate,
+  so FTS and embeddings over it mean something for the first time.
+
+## Parallel parsing is on by default
+
+- The parse pool now runs unless `KNODIN_DISABLE_PARSE_WORKERS=1` is set. Pool
+  size is derived from total memory and core count, and an unsuitable host
+  silently keeps the previous sequential behaviour rather than discovering the
+  problem partway through an index.
+- Files that need a main-thread indexer are kept off the pool. `.page`,
+  `.component`, `.clp`, `.ws`, `.xml`, `.sql`, `.pkb`, `.pks` and `.prisma`
+  classify as generic but hand off to indexers that write to the database
+  themselves, and routing them to a worker made a Visualforce controller edge
+  resolve on some runs and not others.
+
+## Unreadable source is distinguishable from missing source
+
+- Source reads report an availability state instead of collapsing every failure
+  into `""`. A file that is gone, a file that could not be read this time, and a
+  symbol that does not exist are now three different answers.
+- `dead_code` no longer reports a symbol as dead because its file could not be
+  read, and a missing file no longer aborts an entire `map` or `explain`.
+
+## Faster `knodin init` on a new worktree
+
+- A brand-new worktree's graph database is seeded from the cheapest-to-
+  reconcile already-indexed sibling worktree of the same repository, cloned
+  via an OS-level copy-on-write file clone where the filesystem supports it
+  (plain-copy fallback otherwise), then reconciled to the new worktree's
+  actual committed/staged/working/untracked state through the existing
+  candidate allocate/reconcile/audit/promote pipeline before atomic
+  promotion.
+- Seeding is opportunistic: no eligible sibling, a schema/build-compatibility
+  mismatch, or any candidate-audit failure falls straight back to a full
+  from-scratch index, with no error surfaced to the caller. Every git
+  lifecycle operation (pull, reset, merge, rebase, checkout, stash, clean)
+  on a seeded worktree is handled by the same incremental reconciliation and
+  hook machinery as any other worktree.
+- Measured on this repository (810 files, 5,086 symbols): `knodin init` on a
+  new worktree with an eligible sibling completed in roughly 5 seconds versus
+  30-40 seconds for a from-scratch build.
+- Fixes a latent bug in the candidate reconciliation path: a candidate
+  populated from a foreign source (a downloaded shared-index artifact, or now
+  another worktree's database) kept `index_state` mtimes from wherever that
+  source was built, so every file outside the explicit changed-path set could
+  trip the deep audit's drift check once source and target were genuinely
+  different checkouts. `reconcileCandidate` now refreshes that bookkeeping
+  (a stat, not a re-index) for known files outside the changed-path set,
+  which also benefits the existing shared-index restore path.
+- Set `KNODIN_DISABLE_WORKTREE_SEED=1` to always force a full from-scratch
+  index.
+
+No existing worktree's database is touched by that change; it only affects the
+code path for a worktree whose `.knodin/db.sqlite` does not yet exist.
+
+## Also in this release
+
+- Five benchmark runners no longer call `.replace("Z", "Z")`, a no-op that
+  CodeQL correctly flagged as `js/identity-replacement`. Output filenames are
+  byte-identical.
+- Fixed an `update-executor` test that failed roughly one run in three: a
+  version probe raced a 10-second timeout, and the probe timeout is now
+  injectable so the suite does not depend on process-spawn latency.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "knodin",
-	"version": "0.9.0",
+	"version": "0.10.0",
 	"knodin": {
 		"compatibility": "breaking"
 	},
@@ -71,6 +71,7 @@
 		"docs/releases/0.8.6.md",
 		"docs/releases/0.8.7.md",
 		"docs/releases/0.9.0.md",
+		"docs/releases/0.10.0.md",
 		"docs/assets/knodin-favicon.svg",
 		"docs/SYSTEMS-AND-RELATIONSHIPS.md",
 		"docs/TELEMETRY.md",