@msareen/knowledge-hub-builder 0.1.7 → 0.2.0

package/scripts/lib/relocate.ts

@@ -0,0 +1,165 @@
+// Mechanical path rewriting for `khb update --path`: after a hub is moved, replace every literal
+// reference to its old location with the new one.
+//
+// Strictly a conversion, not an interpretation — the same substring, in and out, with no
+// judgement about what a path means. That is what keeps it in the CLI: the alternative is
+// an agent reading each file and deciding, which for a byte-identical prefix swap is both
+// slower and less reliable than a regex. See AGENTS.md, "Division of labor".
+import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { ticker } from "./log";
+
+/** Directories never walked: VCS internals, installed packages, regenerable caches. */
+const SKIP_DIRS = new Set([".git", "node_modules", "inbox"]);
+
+/** A file bigger than this is a corpus artefact, not something holding a path reference. */
+const MAX_BYTES = 8 * 1024 * 1024;
+
+export type Hit = { file: string; count: number };
+export type RewriteResult = {
+  scanned: number;
+  hits: Hit[];
+  /** Files that matched but could not be written — read-only, locked, gone. */
+  failed: { file: string; reason: string }[];
+};
+
+function* walk(dir: string, root = dir): Generator<string> {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return; // unreadable directory: nothing to rewrite in what we cannot open
+  }
+  for (const e of entries) {
+    const p = join(dir, e.name);
+    if (e.isDirectory()) {
+      if (!SKIP_DIRS.has(e.name)) yield* walk(p, root);
+    } else if (e.isFile()) {
+      yield p;
+    }
+  }
+}
+
+/** Heuristic, and the standard one: a NUL byte in the first block means not text. */
+function readText(path: string): string | undefined {
+  try {
+    if (statSync(path).size > MAX_BYTES) return undefined;
+    const buf = readFileSync(path);
+    if (buf.subarray(0, 8192).includes(0)) return undefined;
+    return buf.toString("utf8");
+  } catch {
+    return undefined;
+  }
+}
+
+const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * The spellings one path takes in a hub's files, in a fixed order. A Windows path appears
+ * natively (`D:\a\b`), with forward slashes (`D:/a/b`, common in YAML people typed by
+ * hand), and backslash-escaped inside JSON — which is exactly how `raw/` provenance
+ * headers and `log.md` store a source. All are the same location and all must move.
+ * The order is what lets a matched spelling be mapped back to the same spelling of the
+ * new path, so a JSON-escaped source stays JSON-escaped after the rewrite.
+ */
+function spellings(path: string): string[] {
+  const back = path.replace(/\//g, "\\");
+  return [back.replace(/\\/g, "\\\\"), back, path.replace(/\\/g, "/")];
+}
+
+/**
+ * Pair every spelling of every old path with the matching spelling of the new one.
+ *
+ * Several old paths, not one, because the same directory has more than one true name:
+ * a hub registered as `C:\Users\MANASV~1\…` and later canonicalized to
+ * `C:\Users\Manasvi Sareen\…` has both strings sitting in files written at different
+ * times, and a repair that fixed only the canonical one would leave the rest dangling.
+ */
+function pairs(froms: string[], to: string): { find: string; replace: string }[] {
+  const out = new Map<string, string>();
+  const news = spellings(to);
+  for (const from of froms) {
+    spellings(from).forEach((form, i) => {
+      if (form && !out.has(form)) out.set(form, news[i]);
+    });
+  }
+  // The new path also maps to itself. Two cases need it, and both are ordinary moves:
+  // a hub lifted out of its parent (`…/kb/hub` → `…/kb`) and one pushed down into a
+  // subdirectory of where it stood (`…/kb` → `…/kb/hub`). The paths then share a prefix,
+  // and in the second case the old path matches *inside* every reference that is already
+  // correct — prepending the move a second time. Claiming those matches for an identity
+  // rewrite is what makes the overlap safe, and makes any re-run a no-op.
+  for (const form of news) if (form && !out.has(form)) out.set(form, form);
+  // Longest first, so a `\\`-escaped form is consumed whole rather than partly matched by
+  // a shorter spelling of the same path — and so the identity above wins wherever it and
+  // an old path could both match, alternation being ordered.
+  return [...out].map(([find, replace]) => ({ find, replace })).sort((a, b) => b.find.length - a.find.length);
+}
+
+/**
+ * Replace every reference to any of `froms` with `to` in the text files under `root`.
+ * `dryRun` reports what would change without touching anything.
+ */
+export function rewritePaths(
+  root: string,
+  froms: string[],
+  to: string,
+  opts: { dryRun?: boolean; onStart?: (files: number) => void } = {},
+): RewriteResult {
+  const table = pairs(froms, to);
+  const ci = process.platform === "win32";
+  const key = (s: string) => (ci ? s.toLowerCase() : s);
+  const lookup = new Map(table.map((p) => [key(p.find), p.replace]));
+  // Only where the match ends at a path boundary: without the lookahead, moving `…/old`
+  // would also rewrite `…/older`, a sibling whose name merely starts the same way.
+  const re = new RegExp(
+    `(?:${table.map((p) => escapeRe(p.find)).join("|")})(?=[\\\\/"'\\s,;:)\\]}]|$)`,
+    ci ? "gi" : "g",
+  );
+  const hits: Hit[] = [];
+  const failed: { file: string; reason: string }[] = [];
+  let scanned = 0;
+
+  // Enumerate before reading, so the counter can say "of how many". The walk is directory
+  // entries only — cheap next to opening every file, and worth it for a hub whose raw/ has
+  // grown to thousands of documents and would otherwise sit silent.
+  const files = [...walk(root)];
+  opts.onStart?.(files.length);
+  const progress = ticker("checking", files.length);
+
+  for (const file of files) {
+    progress.tick(hits.length ? `${hits.length} file(s) with references` : "");
+    const text = readText(file);
+    if (text === undefined) continue;
+    scanned++;
+    let count = 0;
+    const next = text.replace(re, (m) => {
+      const to = lookup.get(key(m));
+      if (to === undefined) return m; // not one of ours; leave the text exactly as found
+      if (key(to) === key(m)) return m; // already the new path — matched only to shield it
+      count++;
+      return to;
+    });
+    if (!count) continue;
+    hits.push({ file: relative(root, file) || file, count });
+    if (opts.dryRun) continue;
+    try {
+      writeFileSync(file, next);
+    } catch (e) {
+      failed.push({ file: relative(root, file) || file, reason: (e as Error).message });
+    }
+  }
+  progress.done();
+  return { scanned, hits, failed };
+}
+
+/**
+ * True when the two spellings name one directory — the only case `khb update --path`
+ * refuses, since there is then no move to repair. Overlapping-but-different paths used to
+ * be refused alongside it; `pairs()` now shields the new path from being matched inside
+ * itself, which is what made the overlap safe to rewrite.
+ */
+export function sameLocation(from: string, to: string): boolean {
+  const norm = (p: string) => (process.platform === "win32" ? p.toLowerCase() : p).replace(/[\\/]+$/, "");
+  return norm(from) === norm(to);
+}

package/scripts/lib/schema.ts

@@ -0,0 +1,75 @@
+// The current shape of bundles/<b>/sources.yaml, and a comment-preserving diff/apply against
+// it. Not a persisted schema-version: the "schema" is just this file's current field lists,
+// checked fresh every run — same as MANAGED/RETIRED in paths.ts are static lists re-checked on
+// every upgrade rather than tracked historically. Deliberately excludes util.ts, so it can be
+// imported from lib/upgrade.ts, which must resolve before a hub is.
+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseDocument, isMap, isSeq, YAMLMap } from "yaml";
+
+export type FieldSchema = { key: string; default: unknown } | { key: string; deprecated: true };
+
+/**
+ * Optional, backfillable fields per source `type`. Required fields (`path`, `paths`, `urls`)
+ * are not listed here — a missing one is already a hard validation error in
+ * scripts/ingest/index.ts, not something to silently default.
+ */
+export const SOURCES_SCHEMA: Record<string, FieldSchema[]> = {
+  folder: [{ key: "exclude", default: [] }],
+  files: [{ key: "exclude", default: [] }],
+  web: [],
+};
+
+export type SourcesDiff = { path: string; doc: ReturnType<typeof parseDocument>; changes: string[] };
+
+/** Every bundle directory under a hub, without going through util.ts's HUB singleton. */
+export function listBundleDirs(hub: string): string[] {
+  const dir = join(hub, "bundles");
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .map((d) => join(dir, d))
+    .filter((d) => statSync(d).isDirectory());
+}
+
+/** Diff one bundle's sources.yaml against SOURCES_SCHEMA. Null if there's nothing to change. */
+export function diffSourcesYaml(bundleDir: string): SourcesDiff | null {
+  const path = join(bundleDir, "sources.yaml");
+  if (!existsSync(path)) return null;
+  const doc = parseDocument(readFileSync(path, "utf8"));
+  const sources = doc.get("sources");
+  if (!isSeq(sources)) return null;
+
+  const changes: string[] = [];
+  sources.items.forEach((item, i) => {
+    if (!isMap(item)) return;
+    const type = item.get("type");
+    if (typeof type !== "string") return;
+    const fields = SOURCES_SCHEMA[type];
+    if (!fields) return;
+    for (const field of fields) {
+      if ("deprecated" in field) {
+        if (item.has(field.key)) {
+          item.delete(field.key);
+          changes.push(`sources[${i}] (${type}): remove deprecated '${field.key}'`);
+        }
+      } else if (!item.has(field.key)) {
+        (item as YAMLMap).set(field.key, field.default);
+        changes.push(`sources[${i}] (${type}): add '${field.key}: ${JSON.stringify(field.default)}'`);
+      }
+    }
+  });
+
+  return changes.length ? { path, doc, changes } : null;
+}
+
+/** Diff every bundle in a hub. */
+export function diffSourcesYamlAll(hub: string): SourcesDiff[] {
+  return listBundleDirs(hub)
+    .map(diffSourcesYaml)
+    .filter((d): d is SourcesDiff => d !== null);
+}
+
+/** Write a diff's staged changes back to disk. */
+export function applySourcesDiff(diff: SourcesDiff): void {
+  writeFileSync(diff.path, diff.doc.toString());
+}

package/scripts/lib/upgrade.ts

@@ -5,8 +5,10 @@
 // Nothing here may import util.ts: the drift check runs before a hub is resolved, and
 // util.ts resolves one or exits.
 import { cpSync, mkdirSync, writeFileSync, readFileSync, existsSync, statSync, rmSync } from "node:fs";
-import { join, dirname } from "node:path";
+import { join, dirname, resolve } from "node:path";
 import { PKG, MANAGED, RETIRED, MARKER, markerIn, version } from "./paths";
+import { canonical } from "./registry";
+import { diffSourcesYamlAll } from "./schema";
 
 export type UpgradeResult = {
   /** Version recorded in the hub's marker before the upgrade, if it recorded one. */
@@ -56,16 +58,159 @@ function pruneRetired(hub: string): string[] {
   return gone;
 }
 
-/** Write khb.json with the installed version, preserving the hub's creation date. */
-export function stamp(hub: string, created?: string) {
-  writeFileSync(
-    join(hub, MARKER),
-    JSON.stringify(
-      { khb: version(), created: created ?? new Date().toISOString(), upgraded: new Date().toISOString() },
-      null,
-      2,
-    ) + "\n",
-  );
+/** Whatever is in the hub's marker, or an empty object if it has none or it is broken. */
+export function readMarker(hub: string): Record<string, unknown> {
+  try {
+    const found = markerIn(hub);
+    if (!found) return {};
+    const j = JSON.parse(readFileSync(join(hub, found), "utf8"));
+    return j && typeof j === "object" ? j : {};
+  } catch {
+    return {};
+  }
+}
+
+/** `movedFrom` reads as one path or several — normalise both to a list. */
+const asList = (v: unknown): string[] =>
+  typeof v === "string" ? [v] : Array.isArray(v) ? v.filter((s): s is string => typeof s === "string") : [];
+
+/**
+ * Where the hub is, in the two spellings that matter. `path` is canonical, so it is
+ * comparable; `pathAs` is the spelling khb was actually invoked through, kept only when it
+ * differs — `C:\Users\MANASV~1\…` for `C:\Users\Manasvi Sareen\…`, or a symlink standing in
+ * for its target. That second one is not decoration: files written during a run hold the
+ * spelling of *that* run, so a hub only ever reached by its short name has a `raw/` full of
+ * short-name sources, and a repair that knew only the canonical form would miss every one.
+ */
+function location(hub: string): { path: string; pathAs?: string } {
+  const path = canonical(hub);
+  const pathAs = resolve(hub);
+  return pathAs === path ? { path } : { path, pathAs };
+}
+
+/**
+ * Write khb.json with the installed version. khb owns six keys — `khb`, `created`,
+ * `upgraded`, `path`, `pathAs`, `movedFrom` — and everything else in the marker belongs to
+ * whoever put it there: the hub's `name` and `description`, or a note the user added. Merge
+ * rather than replace, or an upgrade would silently eat what it does not recognise.
+ */
+export function stamp(hub: string, created?: string, extra: Record<string, unknown> = {}) {
+  const path = join(hub, MARKER);
+  const existing = readMarker(hub);
+  const merged: Record<string, unknown> = {
+    ...existing,
+    ...extra,
+    khb: version(),
+    created: created ?? (existing.created as string) ?? new Date().toISOString(),
+    upgraded: new Date().toISOString(),
+    ...location(hub),
+  };
+  if (!location(hub).pathAs) delete merged.pathAs;
+  delete merged.bkr; // pre-rename version field; `khb` replaces it
+  writeFileSync(path, JSON.stringify(merged, null, 2) + "\n");
+}
+
+export type Located = {
+  /** Where the marker said the hub was, when that is no longer where it is. */
+  moved?: string;
+  /** Every stale location recorded and not yet repaired, oldest first. */
+  movedFrom: string[];
+};
+
+/**
+ * Keep the hub's own record of where it lives current, and notice when it changed.
+ *
+ * The marker is the one thing that travels *with* the folder, so it is the only place a
+ * move can be detected without being told: the machine registry knows a hub went missing
+ * but not which live hub it became, and after `~/.khb` is deleted or the folder is opened
+ * on another machine it knows nothing at all. A `path` key costs one string and turns
+ * "khb update --path --from <the path you must now remember>" into a command with no
+ * arguments.
+ *
+ * Stale locations accumulate in `movedFrom` rather than replacing each other: a hub moved
+ * twice before anyone repaired it has references to both former homes, and `update --path`
+ * rewrites the whole list in one pass. `khb update --path` clears it — see `clearMoved`.
+ *
+ * Writes only when something actually changed, so the common case leaves khb.json — and
+ * anyone's git status — untouched.
+ */
+export function recordLocation(hub: string): Located {
+  const now = location(hub);
+  const marker = readMarker(hub);
+  const was = typeof marker.path === "string" ? marker.path : undefined;
+  const movedFrom = asList(marker.movedFrom);
+
+  if (was === now.path) {
+    // Same place. Only one thing can still be missing: the alias this run came in through,
+    // when the hub was last located by its canonical name and is now being reached by a
+    // short name or a symlink. Filled in once, never replaced — replacing it would rewrite
+    // khb.json on every alternation between two equally valid spellings.
+    if (now.pathAs && typeof marker.pathAs !== "string") {
+      try {
+        writeFileSync(
+          join(hub, markerIn(hub) ?? MARKER),
+          JSON.stringify({ ...marker, ...now }, null, 2) + "\n",
+        );
+      } catch {
+        /* read-only hub */
+      }
+    }
+    return { movedFrom };
+  }
+  const moved = was;
+  // The alias first, the canonical form last: `update --path` reports the final entry as the
+  // move, and one directory's two names are better reported under the comparable one.
+  for (const p of [marker.pathAs, was])
+    if (typeof p === "string" && p && !movedFrom.includes(p)) movedFrom.push(p);
+
+  const next: Record<string, unknown> = { ...marker, ...now };
+  if (!now.pathAs) delete next.pathAs;
+  if (movedFrom.length) next.movedFrom = movedFrom;
+  try {
+    // Into the marker as it is *named* here, not MARKER: a hub still carrying a pre-rename
+    // filename is about to be renamed by the drift check, and writing the new name first
+    // would leave it holding two markers.
+    writeFileSync(join(hub, markerIn(hub) ?? MARKER), JSON.stringify(next, null, 2) + "\n");
+  } catch {
+    /* read-only hub: the location is a convenience, never a precondition for the command */
+  }
+  return { moved, movedFrom };
+}
+
+/**
+ * Every location this hub has recorded that is no longer where it stands, oldest first and
+ * ending with the most recent one — which is the move to report.
+ *
+ * Reads `path`/`pathAs` as well as the `movedFrom` backlog, because `khb update --path` runs
+ * outside a hub and therefore outside the drift check that calls `recordLocation`: repairing
+ * a move directly, with no khb command run in between, must work exactly as well.
+ */
+export function staleLocations(hub: string): string[] {
+  const marker = readMarker(hub);
+  const key = (s: string) => (process.platform === "win32" ? s.toLowerCase() : s);
+  const mine = new Set([key(canonical(hub)), key(resolve(hub))]);
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const p of [...asList(marker.movedFrom), marker.pathAs, marker.path]) {
+    if (typeof p !== "string" || !p) continue;
+    // canonical() of a folder that no longer exists is just its resolved form, which is
+    // what makes a former home comparable at all.
+    if (mine.has(key(canonical(p))) || mine.has(key(p)) || seen.has(key(p))) continue;
+    seen.add(key(p));
+    out.push(p);
+  }
+  return out;
+}
+
+/** The move is repaired: drop the backlog, keep `path` pointing where the hub now is. */
+export function clearMoved(hub: string): void {
+  const marker = readMarker(hub);
+  const now = location(hub);
+  if (!("movedFrom" in marker) && marker.path === now.path && marker.pathAs === now.pathAs) return;
+  delete marker.movedFrom;
+  Object.assign(marker, now);
+  if (!now.pathAs) delete marker.pathAs;
+  writeFileSync(join(hub, markerIn(hub) ?? MARKER), JSON.stringify(marker, null, 2) + "\n");
 }
 
 /**
@@ -78,16 +223,40 @@ export function upgradeHub(hub: string): UpgradeResult {
   const found = markerIn(hub)!;
   let created: string | undefined;
   let from: string | undefined;
+  // Carried forward by hand rather than left to stamp(): under a legacy name the file is
+  // deleted below, so its contents must be read out before it goes.
+  let carried: Record<string, unknown> = {};
   try {
     const before = JSON.parse(readFileSync(join(hub, found), "utf8"));
     created = before.created;
     from = before.khb ?? before.bkr;
+    carried = before;
   } catch {
     /* unreadable marker: rewritten below with today's date */
   }
   if (found !== MARKER) rmSync(join(hub, found));
   const synced = syncManaged(hub);
   const pruned = pruneRetired(hub);
-  stamp(hub, created);
+  stamp(hub, created, carried);
   return { from, to: version(), synced, pruned, renamed: found === MARKER ? undefined : found };
 }
+
+/**
+ * A one-line nudge, printed after `upgradeHub` runs (explicit `khb upgrade` or an
+ * auto-triggered one): `khb update` has a path repair and/or a sources.yaml schema backfill
+ * pending. Never a prompt — `upgrade` already runs unattended inside unrelated commands, so
+ * offering more than a suggestion here would mean blocking commands that have nothing to do
+ * with either half of `update`.
+ */
+export function updateHint(hub: string): string | undefined {
+  const pathPending = staleLocations(hub).length > 0;
+  const schemaDiffs = diffSourcesYamlAll(hub);
+  if (!pathPending && !schemaDiffs.length) return undefined;
+  const parts: string[] = [];
+  if (pathPending) parts.push("path: repair needed");
+  if (schemaDiffs.length) {
+    const fields = schemaDiffs.reduce((n, d) => n + d.changes.length, 0);
+    parts.push(`schema: ${fields} field(s) across ${schemaDiffs.length} bundle(s)`);
+  }
+  return `khb: 'khb update' has changes available (${parts.join("; ")}) — run 'khb update' to apply, or 'khb update --dry-run' to preview.`;
+}

package/scripts/lib/util.ts

@@ -124,6 +124,13 @@ export function retargetRaw(bundleDir: string, rawRel: string, source: string):
   return true;
 }
 
+/**
+ * One spelling of a path, for comparing two of them. Absolute, and case-folded on Windows,
+ * where `D:\Corpus\Talk.mp4` and `d:/corpus/talk.mp4` are the same file.
+ */
+export const normPath = (p: string) =>
+  process.platform === "win32" ? resolve(p).toLowerCase() : resolve(p);
+
 export const sha256 = (buf: Buffer | string) => createHash("sha256").update(buf).digest("hex");
 
 /** Hash a file in chunks — corpora contain multi-GB binaries we must not slurp. */

package/scripts/lint.ts

@@ -4,6 +4,9 @@ import { detail, section, totalElapsed } from "./lib/log";
 import { readdirSync, statSync } from "node:fs";
 import { dirname, relative } from "node:path";
 import { parse as parseYaml } from "yaml";
+import { rejectUnknownFlags } from "./lib/args";
+
+rejectUnknownFlags(process.argv.slice(2), "khb lint");
 
 /** OKF v0.1 concept frontmatter. Unknown keys are warned, not rejected — OKF is permissive,
  *  but a `titel:` typo silently loses the field, so it is worth one line of noise. */

package/scripts/new-bundle.ts

@@ -3,8 +3,11 @@
 import { existsSync } from "node:fs";
 import { BUNDLES, join } from "./lib/util";
 import { createBundle, VALID_NAME } from "./lib/scaffold";
+import { rejectUnknownFlags } from "./lib/args";
 
-const [name, scope = "TODO scope"] = process.argv.slice(2);
+const argv = process.argv.slice(2);
+rejectUnknownFlags(argv, 'khb new-bundle <name> ["scope"]');
+const [name, scope = "TODO scope"] = argv;
 if (!name || !VALID_NAME.test(name)) {
   console.error("Usage: khb new-bundle <name> [scope]   (lowercase, digits, hyphens)");
   process.exit(1);

package/scripts/visualize.ts

@@ -7,19 +7,24 @@
 // process. `--port N` pins a specific port.
 import { buildGraphData, readConceptFile } from "./lib/graph";
 import { renderGraphPage } from "./lib/graph-page";
+import { takeFlag, takeOpt, rejectUnknownFlags } from "./lib/args";
 
+const USAGE = "khb visualize [--port N] [--no-open]";
 const argv = process.argv.slice(2);
 // port 0 asks the OS for any free port — no fixed default to collide with something else
-// already running on the machine. Both `--port N` and `--port=N` pin it, since the docs
-// have always shown the spaced form.
-const eqArg = argv.find((a) => a.startsWith("--port="));
-const spacedArg = argv[argv.indexOf("--port") + 1];
-const requestedPort = eqArg
-  ? Number(eqArg.slice("--port=".length))
-  : argv.includes("--port") && spacedArg
-    ? Number(spacedArg)
-    : 0;
-const noOpen = argv.includes("--no-open");
+// already running on the machine. `--port N` and `--port=N` both pin it.
+const portArg = takeOpt(argv, "--port");
+const noOpen = takeFlag(argv, "--no-open");
+rejectUnknownFlags(argv, USAGE);
+
+// A port that is not a port used to become NaN and silently serve on a random one, which
+// looks like the flag working right up until nothing is listening where you expected.
+const requestedPort = portArg === undefined ? 0 : Number(portArg);
+if (!Number.isInteger(requestedPort) || requestedPort < 0 || requestedPort > 65535) {
+  console.error(`--port must be a number between 0 and 65535, not: ${portArg}`);
+  console.error(`Usage: ${USAGE}`);
+  process.exit(1);
+}
 
 function summarize(data: ReturnType<typeof buildGraphData>) {
   const concepts = Object.values(data.bundleGraphs).reduce((n, g) => n + g.concepts.length, 0);

package/skills/ingest/SKILL.md

@@ -64,16 +64,26 @@ empty `sources.yaml` has nothing to re-ingest, so there the only answer is a new
 for it. Do not infer sources from nearby files, do not edit `sources.yaml`, and do not run
 `khb ingest` until the user has answered.
 
+For a `folder` or `files` source, ask one more thing before running `khb ingest`: **anything
+to exclude from this source?** (default: no — most sources want everything ingested). If the
+user names folders, files or patterns to skip, write them into that source's `exclude:` list
+yourself — do not run `khb ingest` until this is settled either, for the same reason as the
+question above: a declaration you have not confirmed is not yet a plan.
+
 Sources live in `bundles/<bundle>/sources.yaml`:
 
 ```yaml
 sources:
   - type: folder          # walk a directory tree
     path: /abs/path/to/project-x
+    exclude:               # optional — skip these before ingesting
+      - drafts/             # a plain entry: matches this path or anything under it
+      - "**/*.tmp"          # a glob (has * ? [): matched with Bun.Glob
   - type: files           # a scattered, explicitly named set
     paths:
       - /abs/path/to/one.pdf
       - /abs/path/to/two.xlsx
+    # exclude: also accepted here, matched against each path's basename
   - type: web
     urls:
       - https://example.com/design-doc
@@ -83,6 +93,10 @@ sources:
     space: PROJX
 ```
 
+`exclude` entries can also be fully-qualified absolute paths (e.g. `D:\corpus\project-x\drafts`
+or `/abs/path/to/project-x/drafts`) instead of paths/patterns relative to the source — either
+form matches, plain or glob.
+
 Nothing is copied by declaring a source.
 
 ## 3. Run it
@@ -103,6 +117,7 @@ locally. Read the summary it prints — the counts are the state of the world:
 | `unchanged, skipped` | already acquired at this exact content hash |
 | `extracted` / `reused from the extraction cache` | converted now / converted by an earlier run or another bundle |
 | `read by OCR` / `transcribed` | lossy routes — see quality, below |
+| `read from a caption sidecar` | a recording whose words were read off its `.vtt`/`.srt` instead |
 | `marked quality: low` | verify these against the source when cataloging |
 | `not extracted` | got a ledger row with an empty `raw`; the per-file line says why |
 
@@ -136,22 +151,79 @@ curation, not transcription.
 | `.xlsx` | `fflate` → one markdown table per sheet | high |
 | `.pdf` (scanned, no text layer) | `pdfium` + `tesseract.js`, automatically | **low** |
 | `.png .jpg .webp .tif .gif` | `tesseract.js`, automatically | **low** |
-| `.mp3 .wav .m4a .mp4 .mov .mkv` | local `whisper` / `faster-whisper` | **low** |
+| `.mp3 .wav .m4a .mp4 .mov .mkv` | local `vno` (whisper.cpp), else `whisper` / `faster-whisper` | **low** |
+| `.vtt .srt` | built-in caption reader | high |
 
 Extracted text is cached hub-wide by content hash at `inbox/extracted/<sha256>.md`, so the
 same file appearing in two bundles converts once.
 
+**A recording next to its captions is one source, not two.** `talk.vtt` (or `talk.en.vtt`,
+or `talk.srt`) beside `talk.mp4` is that recording's words, already written down by someone
+who could hear it — so khb reads them instead of guessing at them with whisper. The pair
+gets one `log.md` row, under the recording; the sidecar earns no row and no `raw/` file of
+its own, and the recording's `extract_tool` names the file the text came from. It is both
+free and better than transcription, so it happens even under `--skip-audio`.
+
+Two things follow. The pair's identity is *both* files, so correcting a caption re-ingests
+the recording rather than leaving a stale row marked unchanged. And khb never picks between
+sidecars: `talk.en.vtt` next to `talk.fr.vtt` is a choice about audience, so it transcribes
+instead and leaves both files to be pointed at explicitly. A caption with no recording
+beside it — or one whose recording this source does not visit, because it is excluded or
+simply not listed — is an ordinary source and gets its own row. That is also the lever:
+excluding a sidecar does not unpair it, since `exclude` governs what earns a `raw/` file and
+a paired sidecar never earns one; exclude the *recording* to have its captions ingested
+alone.
+
+The caption reader drops what belongs to the player and keeps what belongs to the
+transcript: cue indices and timecodes go, `<v Name>` becomes a speaker label, the rolling
+repetition auto-generated captions leave behind is collapsed, and anything longer than five
+minutes gets a coarse `## h:mm:ss` heading per interval so a passage can be found in the
+source recording. Quality is `high` — the words are what the file says, not what an
+extractor guessed — but auto-generated captions are still ASR underneath, so treat a
+transcript that reads like a machine wrote it the way you would treat one.
+
 **OCR needs no setup.** `@hyzyla/pdfium`, `sharp` and `tesseract.js` are dependencies of khb
 itself, so a scanned PDF or a photographed page is read on the first run, in any hub, without
 asking the user to install anything.
 
-Transcription is the one route that can be absent: it wants a `whisper` or `faster-whisper`
-executable on `PATH`.
+Transcription is the one route that can be absent. It wants a transcriber on `PATH`, and
+takes the first of these it finds:
 
 ```
-pip install -U openai-whisper                # transcription (faster-whisper also works)
+npm install -g @msareen/voice-notes-organizer   # vno — whisper.cpp, preferred
+pip install -U openai-whisper                   # whisper (faster-whisper also works)
 ```
 
+`vno` is preferred where both are set up: it is whisper.cpp rather than the Python
+whisper, so it is markedly faster on the same audio and uses whatever acceleration the
+machine has, it installs its own ffmpeg and model, and it emits WebVTT — which means a
+transcript with `## h:mm:ss` anchors instead of an undifferentiated wall of text. khb runs
+it as `vno t <file> -o <cache path> --no-open` with stdin closed, so nothing is written
+beside your recordings and vno's setup offers degrade to printed instructions instead of
+prompts.
+
+khb gates on `vno status` before using it, because installed and ready are different things
+— vno needs ffmpeg, whisper.cpp and a model, and reports on all three. **A vno that is not
+set up is an amber gate, never a red one.** The run does not stop and nothing else is
+affected: whisper takes over if you have it, and if you don't, the recordings pend with an
+empty `raw` exactly like any other unavailable extractor while the rest of the corpus is
+ingested normally. What you get is the reason and the fix, on the file's own line and again
+in `log.md`:
+
+```
+[ 3/12] D:\corpus\standup.m4a
+        no captions beside it — transcribing (minutes per file) …
+  vno is installed but not set up: ffmpeg, whisper.cpp — run:  vno setup
+        pending — vno is installed but not set up: ffmpeg, whisper.cpp — run:  vno setup
+```
+
+Run `vno setup` yourself and re-run the ingest; the pending rows fill in. khb will not run
+it for you — installing software nobody asked it to install is not a conversion step.
+
+Either engine is a local binary doing a reproducible conversion, and its output is
+`quality: low` all the same: it is a machine's guess at audio, and the recording is still
+the thing to re-read when a passage looks wrong.
+
 When any extractor is unavailable khb says so once and records the affected files as pending
 rather than failing the run — a `log.md` row with an empty `raw`, waiting for the dependency.
 If khb ever prints a `bun add` hint for the OCR packages, its own install tree is incomplete;