@msareen/knowledge-hub-builder 0.1.7 → 0.1.8

package/scripts/lib/relocate.ts

@@ -0,0 +1,165 @@
+// Mechanical path rewriting for `khb update --path`: after a hub is moved, replace every literal
+// reference to its old location with the new one.
+//
+// Strictly a conversion, not an interpretation — the same substring, in and out, with no
+// judgement about what a path means. That is what keeps it in the CLI: the alternative is
+// an agent reading each file and deciding, which for a byte-identical prefix swap is both
+// slower and less reliable than a regex. See AGENTS.md, "Division of labor".
+import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { ticker } from "./log";
+
+/** Directories never walked: VCS internals, installed packages, regenerable caches. */
+const SKIP_DIRS = new Set([".git", "node_modules", "inbox"]);
+
+/** A file bigger than this is a corpus artefact, not something holding a path reference. */
+const MAX_BYTES = 8 * 1024 * 1024;
+
+export type Hit = { file: string; count: number };
+export type RewriteResult = {
+  scanned: number;
+  hits: Hit[];
+  /** Files that matched but could not be written — read-only, locked, gone. */
+  failed: { file: string; reason: string }[];
+};
+
+function* walk(dir: string, root = dir): Generator<string> {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return; // unreadable directory: nothing to rewrite in what we cannot open
+  }
+  for (const e of entries) {
+    const p = join(dir, e.name);
+    if (e.isDirectory()) {
+      if (!SKIP_DIRS.has(e.name)) yield* walk(p, root);
+    } else if (e.isFile()) {
+      yield p;
+    }
+  }
+}
+
+/** Heuristic, and the standard one: a NUL byte in the first block means not text. */
+function readText(path: string): string | undefined {
+  try {
+    if (statSync(path).size > MAX_BYTES) return undefined;
+    const buf = readFileSync(path);
+    if (buf.subarray(0, 8192).includes(0)) return undefined;
+    return buf.toString("utf8");
+  } catch {
+    return undefined;
+  }
+}
+
+const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * The spellings one path takes in a hub's files, in a fixed order. A Windows path appears
+ * natively (`D:\a\b`), with forward slashes (`D:/a/b`, common in YAML people typed by
+ * hand), and backslash-escaped inside JSON — which is exactly how `raw/` provenance
+ * headers and `log.md` store a source. All are the same location and all must move.
+ * The order is what lets a matched spelling be mapped back to the same spelling of the
+ * new path, so a JSON-escaped source stays JSON-escaped after the rewrite.
+ */
+function spellings(path: string): string[] {
+  const back = path.replace(/\//g, "\\");
+  return [back.replace(/\\/g, "\\\\"), back, path.replace(/\\/g, "/")];
+}
+
+/**
+ * Pair every spelling of every old path with the matching spelling of the new one.
+ *
+ * Several old paths, not one, because the same directory has more than one true name:
+ * a hub registered as `C:\Users\MANASV~1\…` and later canonicalized to
+ * `C:\Users\Manasvi Sareen\…` has both strings sitting in files written at different
+ * times, and a repair that fixed only the canonical one would leave the rest dangling.
+ */
+function pairs(froms: string[], to: string): { find: string; replace: string }[] {
+  const out = new Map<string, string>();
+  const news = spellings(to);
+  for (const from of froms) {
+    spellings(from).forEach((form, i) => {
+      if (form && !out.has(form)) out.set(form, news[i]);
+    });
+  }
+  // The new path also maps to itself. Two cases need it, and both are ordinary moves:
+  // a hub lifted out of its parent (`…/kb/hub` → `…/kb`) and one pushed down into a
+  // subdirectory of where it stood (`…/kb` → `…/kb/hub`). The paths then share a prefix,
+  // and in the second case the old path matches *inside* every reference that is already
+  // correct — prepending the move a second time. Claiming those matches for an identity
+  // rewrite is what makes the overlap safe, and makes any re-run a no-op.
+  for (const form of news) if (form && !out.has(form)) out.set(form, form);
+  // Longest first, so a `\\`-escaped form is consumed whole rather than partly matched by
+  // a shorter spelling of the same path — and so the identity above wins wherever it and
+  // an old path could both match, alternation being ordered.
+  return [...out].map(([find, replace]) => ({ find, replace })).sort((a, b) => b.find.length - a.find.length);
+}
+
+/**
+ * Replace every reference to any of `froms` with `to` in the text files under `root`.
+ * `dryRun` reports what would change without touching anything.
+ */
+export function rewritePaths(
+  root: string,
+  froms: string[],
+  to: string,
+  opts: { dryRun?: boolean; onStart?: (files: number) => void } = {},
+): RewriteResult {
+  const table = pairs(froms, to);
+  const ci = process.platform === "win32";
+  const key = (s: string) => (ci ? s.toLowerCase() : s);
+  const lookup = new Map(table.map((p) => [key(p.find), p.replace]));
+  // Only where the match ends at a path boundary: without the lookahead, moving `…/old`
+  // would also rewrite `…/older`, a sibling whose name merely starts the same way.
+  const re = new RegExp(
+    `(?:${table.map((p) => escapeRe(p.find)).join("|")})(?=[\\\\/"'\\s,;:)\\]}]|$)`,
+    ci ? "gi" : "g",
+  );
+  const hits: Hit[] = [];
+  const failed: { file: string; reason: string }[] = [];
+  let scanned = 0;
+
+  // Enumerate before reading, so the counter can say "of how many". The walk is directory
+  // entries only — cheap next to opening every file, and worth it for a hub whose raw/ has
+  // grown to thousands of documents and would otherwise sit silent.
+  const files = [...walk(root)];
+  opts.onStart?.(files.length);
+  const progress = ticker("checking", files.length);
+
+  for (const file of files) {
+    progress.tick(hits.length ? `${hits.length} file(s) with references` : "");
+    const text = readText(file);
+    if (text === undefined) continue;
+    scanned++;
+    let count = 0;
+    const next = text.replace(re, (m) => {
+      const to = lookup.get(key(m));
+      if (to === undefined) return m; // not one of ours; leave the text exactly as found
+      if (key(to) === key(m)) return m; // already the new path — matched only to shield it
+      count++;
+      return to;
+    });
+    if (!count) continue;
+    hits.push({ file: relative(root, file) || file, count });
+    if (opts.dryRun) continue;
+    try {
+      writeFileSync(file, next);
+    } catch (e) {
+      failed.push({ file: relative(root, file) || file, reason: (e as Error).message });
+    }
+  }
+  progress.done();
+  return { scanned, hits, failed };
+}
+
+/**
+ * True when the two spellings name one directory — the only case `khb update --path`
+ * refuses, since there is then no move to repair. Overlapping-but-different paths used to
+ * be refused alongside it; `pairs()` now shields the new path from being matched inside
+ * itself, which is what made the overlap safe to rewrite.
+ */
+export function sameLocation(from: string, to: string): boolean {
+  const norm = (p: string) => (process.platform === "win32" ? p.toLowerCase() : p).replace(/[\\/]+$/, "");
+  return norm(from) === norm(to);
+}

package/scripts/lib/schema.ts

@@ -0,0 +1,75 @@
+// The current shape of bundles/<b>/sources.yaml, and a comment-preserving diff/apply against
+// it. Not a persisted schema-version: the "schema" is just this file's current field lists,
+// checked fresh every run — same as MANAGED/RETIRED in paths.ts are static lists re-checked on
+// every upgrade rather than tracked historically. Deliberately excludes util.ts, so it can be
+// imported from lib/upgrade.ts, which must resolve before a hub is.
+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseDocument, isMap, isSeq, YAMLMap } from "yaml";
+
+export type FieldSchema = { key: string; default: unknown } | { key: string; deprecated: true };
+
+/**
+ * Optional, backfillable fields per source `type`. Required fields (`path`, `paths`, `urls`)
+ * are not listed here — a missing one is already a hard validation error in
+ * scripts/ingest/index.ts, not something to silently default.
+ */
+export const SOURCES_SCHEMA: Record<string, FieldSchema[]> = {
+  folder: [{ key: "exclude", default: [] }],
+  files: [{ key: "exclude", default: [] }],
+  web: [],
+};
+
+export type SourcesDiff = { path: string; doc: ReturnType<typeof parseDocument>; changes: string[] };
+
+/** Every bundle directory under a hub, without going through util.ts's HUB singleton. */
+export function listBundleDirs(hub: string): string[] {
+  const dir = join(hub, "bundles");
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .map((d) => join(dir, d))
+    .filter((d) => statSync(d).isDirectory());
+}
+
+/** Diff one bundle's sources.yaml against SOURCES_SCHEMA. Null if there's nothing to change. */
+export function diffSourcesYaml(bundleDir: string): SourcesDiff | null {
+  const path = join(bundleDir, "sources.yaml");
+  if (!existsSync(path)) return null;
+  const doc = parseDocument(readFileSync(path, "utf8"));
+  const sources = doc.get("sources");
+  if (!isSeq(sources)) return null;
+
+  const changes: string[] = [];
+  sources.items.forEach((item, i) => {
+    if (!isMap(item)) return;
+    const type = item.get("type");
+    if (typeof type !== "string") return;
+    const fields = SOURCES_SCHEMA[type];
+    if (!fields) return;
+    for (const field of fields) {
+      if ("deprecated" in field) {
+        if (item.has(field.key)) {
+          item.delete(field.key);
+          changes.push(`sources[${i}] (${type}): remove deprecated '${field.key}'`);
+        }
+      } else if (!item.has(field.key)) {
+        (item as YAMLMap).set(field.key, field.default);
+        changes.push(`sources[${i}] (${type}): add '${field.key}: ${JSON.stringify(field.default)}'`);
+      }
+    }
+  });
+
+  return changes.length ? { path, doc, changes } : null;
+}
+
+/** Diff every bundle in a hub. */
+export function diffSourcesYamlAll(hub: string): SourcesDiff[] {
+  return listBundleDirs(hub)
+    .map(diffSourcesYaml)
+    .filter((d): d is SourcesDiff => d !== null);
+}
+
+/** Write a diff's staged changes back to disk. */
+export function applySourcesDiff(diff: SourcesDiff): void {
+  writeFileSync(diff.path, diff.doc.toString());
+}

package/scripts/lib/upgrade.ts

@@ -5,8 +5,10 @@
 // Nothing here may import util.ts: the drift check runs before a hub is resolved, and
 // util.ts resolves one or exits.
 import { cpSync, mkdirSync, writeFileSync, readFileSync, existsSync, statSync, rmSync } from "node:fs";
-import { join, dirname } from "node:path";
+import { join, dirname, resolve } from "node:path";
 import { PKG, MANAGED, RETIRED, MARKER, markerIn, version } from "./paths";
+import { canonical } from "./registry";
+import { diffSourcesYamlAll } from "./schema";
 
 export type UpgradeResult = {
   /** Version recorded in the hub's marker before the upgrade, if it recorded one. */
@@ -56,16 +58,159 @@ function pruneRetired(hub: string): string[] {
   return gone;
 }
 
-/** Write khb.json with the installed version, preserving the hub's creation date. */
-export function stamp(hub: string, created?: string) {
-  writeFileSync(
-    join(hub, MARKER),
-    JSON.stringify(
-      { khb: version(), created: created ?? new Date().toISOString(), upgraded: new Date().toISOString() },
-      null,
-      2,
-    ) + "\n",
-  );
+/** Whatever is in the hub's marker, or an empty object if it has none or it is broken. */
+export function readMarker(hub: string): Record<string, unknown> {
+  try {
+    const found = markerIn(hub);
+    if (!found) return {};
+    const j = JSON.parse(readFileSync(join(hub, found), "utf8"));
+    return j && typeof j === "object" ? j : {};
+  } catch {
+    return {};
+  }
+}
+
+/** `movedFrom` reads as one path or several — normalise both to a list. */
+const asList = (v: unknown): string[] =>
+  typeof v === "string" ? [v] : Array.isArray(v) ? v.filter((s): s is string => typeof s === "string") : [];
+
+/**
+ * Where the hub is, in the two spellings that matter. `path` is canonical, so it is
+ * comparable; `pathAs` is the spelling khb was actually invoked through, kept only when it
+ * differs — `C:\Users\MANASV~1\…` for `C:\Users\Manasvi Sareen\…`, or a symlink standing in
+ * for its target. That second one is not decoration: files written during a run hold the
+ * spelling of *that* run, so a hub only ever reached by its short name has a `raw/` full of
+ * short-name sources, and a repair that knew only the canonical form would miss every one.
+ */
+function location(hub: string): { path: string; pathAs?: string } {
+  const path = canonical(hub);
+  const pathAs = resolve(hub);
+  return pathAs === path ? { path } : { path, pathAs };
+}
+
+/**
+ * Write khb.json with the installed version. khb owns six keys — `khb`, `created`,
+ * `upgraded`, `path`, `pathAs`, `movedFrom` — and everything else in the marker belongs to
+ * whoever put it there: the hub's `name` and `description`, or a note the user added. Merge
+ * rather than replace, or an upgrade would silently eat what it does not recognise.
+ */
+export function stamp(hub: string, created?: string, extra: Record<string, unknown> = {}) {
+  const path = join(hub, MARKER);
+  const existing = readMarker(hub);
+  const merged: Record<string, unknown> = {
+    ...existing,
+    ...extra,
+    khb: version(),
+    created: created ?? (existing.created as string) ?? new Date().toISOString(),
+    upgraded: new Date().toISOString(),
+    ...location(hub),
+  };
+  if (!location(hub).pathAs) delete merged.pathAs;
+  delete merged.bkr; // pre-rename version field; `khb` replaces it
+  writeFileSync(path, JSON.stringify(merged, null, 2) + "\n");
+}
+
+export type Located = {
+  /** Where the marker said the hub was, when that is no longer where it is. */
+  moved?: string;
+  /** Every stale location recorded and not yet repaired, oldest first. */
+  movedFrom: string[];
+};
+
+/**
+ * Keep the hub's own record of where it lives current, and notice when it changed.
+ *
+ * The marker is the one thing that travels *with* the folder, so it is the only place a
+ * move can be detected without being told: the machine registry knows a hub went missing
+ * but not which live hub it became, and after `~/.khb` is deleted or the folder is opened
+ * on another machine it knows nothing at all. A `path` key costs one string and turns
+ * "khb update --path --from <the path you must now remember>" into a command with no
+ * arguments.
+ *
+ * Stale locations accumulate in `movedFrom` rather than replacing each other: a hub moved
+ * twice before anyone repaired it has references to both former homes, and `update --path`
+ * rewrites the whole list in one pass. `khb update --path` clears it — see `clearMoved`.
+ *
+ * Writes only when something actually changed, so the common case leaves khb.json — and
+ * anyone's git status — untouched.
+ */
+export function recordLocation(hub: string): Located {
+  const now = location(hub);
+  const marker = readMarker(hub);
+  const was = typeof marker.path === "string" ? marker.path : undefined;
+  const movedFrom = asList(marker.movedFrom);
+
+  if (was === now.path) {
+    // Same place. Only one thing can still be missing: the alias this run came in through,
+    // when the hub was last located by its canonical name and is now being reached by a
+    // short name or a symlink. Filled in once, never replaced — replacing it would rewrite
+    // khb.json on every alternation between two equally valid spellings.
+    if (now.pathAs && typeof marker.pathAs !== "string") {
+      try {
+        writeFileSync(
+          join(hub, markerIn(hub) ?? MARKER),
+          JSON.stringify({ ...marker, ...now }, null, 2) + "\n",
+        );
+      } catch {
+        /* read-only hub */
+      }
+    }
+    return { movedFrom };
+  }
+  const moved = was;
+  // The alias first, the canonical form last: `update --path` reports the final entry as the
+  // move, and one directory's two names are better reported under the comparable one.
+  for (const p of [marker.pathAs, was])
+    if (typeof p === "string" && p && !movedFrom.includes(p)) movedFrom.push(p);
+
+  const next: Record<string, unknown> = { ...marker, ...now };
+  if (!now.pathAs) delete next.pathAs;
+  if (movedFrom.length) next.movedFrom = movedFrom;
+  try {
+    // Into the marker as it is *named* here, not MARKER: a hub still carrying a pre-rename
+    // filename is about to be renamed by the drift check, and writing the new name first
+    // would leave it holding two markers.
+    writeFileSync(join(hub, markerIn(hub) ?? MARKER), JSON.stringify(next, null, 2) + "\n");
+  } catch {
+    /* read-only hub: the location is a convenience, never a precondition for the command */
+  }
+  return { moved, movedFrom };
+}
+
+/**
+ * Every location this hub has recorded that is no longer where it stands, oldest first and
+ * ending with the most recent one — which is the move to report.
+ *
+ * Reads `path`/`pathAs` as well as the `movedFrom` backlog, because `khb update --path` runs
+ * outside a hub and therefore outside the drift check that calls `recordLocation`: repairing
+ * a move directly, with no khb command run in between, must work exactly as well.
+ */
+export function staleLocations(hub: string): string[] {
+  const marker = readMarker(hub);
+  const key = (s: string) => (process.platform === "win32" ? s.toLowerCase() : s);
+  const mine = new Set([key(canonical(hub)), key(resolve(hub))]);
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const p of [...asList(marker.movedFrom), marker.pathAs, marker.path]) {
+    if (typeof p !== "string" || !p) continue;
+    // canonical() of a folder that no longer exists is just its resolved form, which is
+    // what makes a former home comparable at all.
+    if (mine.has(key(canonical(p))) || mine.has(key(p)) || seen.has(key(p))) continue;
+    seen.add(key(p));
+    out.push(p);
+  }
+  return out;
+}
+
+/** The move is repaired: drop the backlog, keep `path` pointing where the hub now is. */
+export function clearMoved(hub: string): void {
+  const marker = readMarker(hub);
+  const now = location(hub);
+  if (!("movedFrom" in marker) && marker.path === now.path && marker.pathAs === now.pathAs) return;
+  delete marker.movedFrom;
+  Object.assign(marker, now);
+  if (!now.pathAs) delete marker.pathAs;
+  writeFileSync(join(hub, markerIn(hub) ?? MARKER), JSON.stringify(marker, null, 2) + "\n");
 }
 
 /**
@@ -78,16 +223,40 @@ export function upgradeHub(hub: string): UpgradeResult {
   const found = markerIn(hub)!;
   let created: string | undefined;
   let from: string | undefined;
+  // Carried forward by hand rather than left to stamp(): under a legacy name the file is
+  // deleted below, so its contents must be read out before it goes.
+  let carried: Record<string, unknown> = {};
   try {
     const before = JSON.parse(readFileSync(join(hub, found), "utf8"));
     created = before.created;
     from = before.khb ?? before.bkr;
+    carried = before;
   } catch {
     /* unreadable marker: rewritten below with today's date */
   }
   if (found !== MARKER) rmSync(join(hub, found));
   const synced = syncManaged(hub);
   const pruned = pruneRetired(hub);
-  stamp(hub, created);
+  stamp(hub, created, carried);
   return { from, to: version(), synced, pruned, renamed: found === MARKER ? undefined : found };
 }
+
+/**
+ * A one-line nudge, printed after `upgradeHub` runs (explicit `khb upgrade` or an
+ * auto-triggered one): `khb update` has a path repair and/or a sources.yaml schema backfill
+ * pending. Never a prompt — `upgrade` already runs unattended inside unrelated commands, so
+ * offering more than a suggestion here would mean blocking commands that have nothing to do
+ * with either half of `update`.
+ */
+export function updateHint(hub: string): string | undefined {
+  const pathPending = staleLocations(hub).length > 0;
+  const schemaDiffs = diffSourcesYamlAll(hub);
+  if (!pathPending && !schemaDiffs.length) return undefined;
+  const parts: string[] = [];
+  if (pathPending) parts.push("path: repair needed");
+  if (schemaDiffs.length) {
+    const fields = schemaDiffs.reduce((n, d) => n + d.changes.length, 0);
+    parts.push(`schema: ${fields} field(s) across ${schemaDiffs.length} bundle(s)`);
+  }
+  return `khb: 'khb update' has changes available (${parts.join("; ")}) — run 'khb update' to apply, or 'khb update --dry-run' to preview.`;
+}

package/skills/ingest/SKILL.md

@@ -64,16 +64,26 @@ empty `sources.yaml` has nothing to re-ingest, so there the only answer is a new
 for it. Do not infer sources from nearby files, do not edit `sources.yaml`, and do not run
 `khb ingest` until the user has answered.
 
+For a `folder` or `files` source, ask one more thing before running `khb ingest`: **anything
+to exclude from this source?** (default: no — most sources want everything ingested). If the
+user names folders, files or patterns to skip, write them into that source's `exclude:` list
+yourself — do not run `khb ingest` until this is settled either, for the same reason as the
+question above: a declaration you have not confirmed is not yet a plan.
+
 Sources live in `bundles/<bundle>/sources.yaml`:
 
 ```yaml
 sources:
   - type: folder          # walk a directory tree
     path: /abs/path/to/project-x
+    exclude:               # optional — skip these before ingesting
+      - drafts/             # a plain entry: matches this path or anything under it
+      - "**/*.tmp"          # a glob (has * ? [): matched with Bun.Glob
   - type: files           # a scattered, explicitly named set
     paths:
       - /abs/path/to/one.pdf
       - /abs/path/to/two.xlsx
+    # exclude: also accepted here, matched against each path's basename
   - type: web
     urls:
       - https://example.com/design-doc
@@ -83,6 +93,10 @@ sources:
     space: PROJX
 ```
 
+`exclude` entries can also be fully-qualified absolute paths (e.g. `D:\corpus\project-x\drafts`
+or `/abs/path/to/project-x/drafts`) instead of paths/patterns relative to the source — either
+form matches, plain or glob.
+
 Nothing is copied by declaring a source.
 
 ## 3. Run it