@msareen/knowledge-hub-builder 0.1.3

package/scripts/ingest/acquire.ts

@@ -0,0 +1,189 @@
+// One local file → one raw/*.md, fully extracted.
+//
+// This is the whole of ingest's judgement: pick the extractor by file kind, run it, write
+// the text with provenance. Every acquisition path (folder walk, explicit file list) funnels
+// through here so a `.pdf` behaves identically however it was named, and so there is exactly
+// one place that decides what "acquired" means.
+//
+// Nothing here interprets content. Ingest ends the moment bytes have become text — deciding
+// what the text *says* is the catalog pass (skills/catalog/SKILL.md).
+import { readFileSync, existsSync } from "node:fs";
+import { basename } from "node:path";
+import { writeRaw, sha256File, rawNameFor, retargetRaw } from "../lib/util";
+import { record, isFresh, identify, adopt, type Entry } from "../lib/ledger";
+import {
+  extractCached, ocrCached, ocrImageCached, transcribeCached,
+  extractedBody, extractedPath, type Extraction,
+} from "../lib/extract";
+import { kindOf, extOf, mdName } from "./exts";
+import { detectPasswordProtected, PROTECTABLE } from "./protect";
+import { item, note, outcome } from "../lib/log";
+
+export type Options = {
+  force: boolean;   // re-acquire even when the content hash is unchanged
+  ocr: boolean;     // OCR scanned PDFs and images (default on; ~seconds/page)
+  audio: boolean;   // transcribe audio/video (default on; ~minutes/file)
+};
+
+export type Counters = {
+  copied: number;       // text files, taken verbatim
+  extracted: number;    // converted this run
+  fromCache: number;    // converted by an earlier run or another bundle
+  ocrd: number;
+  transcribed: number;
+  lowQuality: number;   // OCR/ASR output — worth re-reading from source during curation
+  skipped: number;      // unchanged since last ingest
+  moved: number;        // same bytes at a new path — row re-pointed, nothing re-extracted
+  // Everything that did not become a raw/ file — protected, unreadable, no extractor, or
+  // skipped by a flag. One bucket, not one per cause: the per-file line above already gave
+  // the reason, and splitting it into `pending` + `failed` only ever double-counted.
+  pending: number;
+};
+
+export const newCounters = (): Counters => ({
+  copied: 0, extracted: 0, fromCache: 0, ocrd: 0, transcribed: 0,
+  lowQuality: 0, skipped: 0, moved: 0, pending: 0,
+});
+
+/** Record the source with an empty `raw` so the owed work survives this process. */
+function pend(entries: Map<string, Entry>, path: string, hash: string, c: Counters, why: string) {
+  record(entries, { source: path, sha256: hash, fetched: new Date().toISOString(), raw: "" });
+  c.pending++;
+  // The path is already on this item's own line; say only why it stopped.
+  outcome(`pending — ${why}`);
+}
+
+export async function acquireFile(
+  at: string,
+  path: string,
+  name: string,
+  rawDir: string,
+  bundleDir: string,
+  entries: Map<string, Entry>,
+  c: Counters,
+  opts: Options,
+): Promise<void> {
+  // Announce the file first: hashing a multi-GB binary and every extractor below can take
+  // real time, and a run that printed only successes left the slow file unnamed.
+  item(at, path);
+  const kind = kindOf(path);
+  const hash = await sha256File(path);
+  if (kind === "skip") {
+    pend(entries, path, hash, c, "no extractor for this format");
+    return;
+  }
+  // Identity before freshness. A file that moved is the row it already has: resolve that
+  // first, or it reads as an unrelated arrival and earns a second raw/ file, a second row
+  // with an empty `curated`, and eventually a second concept for material already written.
+  let adopted: Entry | undefined;
+  const id = identify(entries, bundleDir, path, hash);
+  if (id.kind === "moved") {
+    adopted = adopt(entries, id.from, path);
+    retargetRaw(bundleDir, adopted.raw, path);
+    c.moved++;
+    outcome(`moved from ${id.from.source} — kept ${adopted.raw}${adopted.curated ? " and its catalog entry" : ""}`);
+    if (!opts.force) return;
+  } else if (id.kind === "copy") {
+    note(`same bytes as ${id.twin.source}, which is also still on disk — ingesting as its own source`);
+  } else if (id.kind === "ambiguous") {
+    note(`same bytes as ${id.twins.length} rows whose sources have all gone — not guessing which one moved here`);
+  }
+
+  if (!opts.force && isFresh(entries, bundleDir, path, hash)) {
+    c.skipped++;
+    outcome("unchanged, skipped");
+    return;
+  }
+
+  // A source keeps its raw/ filename for life. Deriving the name from the path again on
+  // every run means a file that moved and *then* changed re-extracts under a new name,
+  // stranding the old raw file with the concept's Citations still pointing at it. Only fall
+  // back to a fresh name when this source has no raw file in this directory yet.
+  const held = adopted ?? entries.get(path);
+  const file =
+    held?.raw?.startsWith(`raw/${basename(rawDir)}/`)
+      ? basename(held.raw)
+      : rawNameFor(rawDir, mdName(name), path, entries.values());
+  const stamp = (raw: string) => record(entries, { source: path, sha256: hash, fetched: new Date().toISOString(), raw });
+
+  if (kind === "text") {
+    const raw = writeRaw(rawDir, file, { source: path, sha256: hash.slice(0, 12), tool: "copy", quality: "high" }, readFileSync(path, "utf8"));
+    stamp(raw);
+    c.copied++;
+    outcome(`copied → ${raw}`);
+    return;
+  }
+
+  // A password-protected document defeats every extractor identically, and finding that out
+  // costs a byte-level peek instead of a failed conversion. Say so rather than reporting a
+  // mystery empty file — the remedy (supply the password, re-export) is the user's.
+  const ext = extOf(path);
+  if (PROTECTABLE.has(ext) && detectPasswordProtected(path, ext, Bun.file(path).size)) {
+    pend(entries, path, hash, c, "password-protected");
+    return;
+  }
+
+  const hit = existsSync(extractedPath(hash));
+  let res: Extraction;
+
+  if (kind === "doc") {
+    note(hit ? `${ext.slice(1)} — reusing cached extraction` : `extracting ${ext.slice(1)} …`);
+    res = await extractCached(path, hash, ext);
+    // Pages but no text layer: the file is fine, the reader was wrong. OCR is the remedy,
+    // and running it here is what keeps ingest a single pass instead of a hunt afterwards.
+    if (res.status === "needs-ocr") {
+      if (!opts.ocr) {
+        pend(entries, path, hash, c, `scanned, ${res.pages}p (--skip-ocr)`);
+        return;
+      }
+      note(`no text layer, ${res.pages}p — scanned, running OCR (seconds per page)`);
+      res = await ocrCached(path, hash);
+      if (res.status === "ok") c.ocrd++;
+    }
+  } else if (kind === "image") {
+    if (!opts.ocr) {
+      pend(entries, path, hash, c, "image (--skip-ocr)");
+      return;
+    }
+    note(hit ? "image — reusing cached OCR" : "image — running OCR …");
+    res = await ocrImageCached(path, hash);
+    if (res.status === "ok" && !hit) c.ocrd++;
+  } else {
+    if (!opts.audio) {
+      pend(entries, path, hash, c, "audio/video (--skip-audio)");
+      return;
+    }
+    note(hit ? "audio/video — reusing cached transcript" : "transcribing with whisper (minutes per file) …");
+    res = await transcribeCached(path, hash);
+    if (res.status === "ok" && !hit) c.transcribed++;
+  }
+
+  if (res.status !== "ok") {
+    pend(entries, path, hash, c, res.status === "unsupported" ? "no extractor for this format" : res.reason);
+    return;
+  }
+
+  // Copy out of the hash-keyed cache rather than moving: raw/ stays derived and the cache
+  // stays reusable by any other bundle that holds the same content.
+  const raw = writeRaw(rawDir, file, { source: path, sha256: hash.slice(0, 12), tool: res.tool, quality: res.quality }, extractedBody(res.path));
+  stamp(raw);
+  if (hit) c.fromCache++;
+  else if (kind === "doc") c.extracted++;
+  if (res.quality === "low") c.lowQuality++;
+  // Name the tool and the quality on the line: `quality: low` is the flag that tells
+  // curation to re-read the original, and burying it in the file made it easy to miss.
+  outcome(`${hit ? "cached" : "extracted"} → ${raw}  [${res.tool}, quality: ${res.quality}]`);
+}
+
+export function report(c: Counters) {
+  const line = (n: number, s: string) => (n ? console.log(`  ${n} ${s}`) : undefined);
+  line(c.skipped, "unchanged, skipped");
+  line(c.moved, "moved/renamed — existing raw file and catalog entry kept");
+  line(c.copied, "text file(s) copied");
+  line(c.extracted, "extracted");
+  line(c.fromCache, "reused from the extraction cache (inbox/extracted/)");
+  line(c.ocrd, "read by OCR");
+  line(c.transcribed, "transcribed");
+  line(c.lowQuality, "marked `quality: low` — verify against the source when curating");
+  line(c.pending, "not extracted (empty `raw` in log.md)");
+}

package/scripts/ingest/exts.ts

@@ -0,0 +1,29 @@
+// Shared file-type classification for every acquisition path.
+//
+// The kind decides which extractor runs, and every kind here has one — ingest is a single
+// flat pass that converts everything it can, locally, with no agent turn in the middle.
+export const TEXT = [".md", ".txt", ".rst", ".adoc", ".html", ".csv", ".json", ".yaml", ".yml"];
+/** Born-digital documents: khb's own pure-JS libraries read these, no system install. */
+export const DOC = [".pdf", ".docx", ".odt", ".xlsx", ".pptx"];
+/** Pixels. Read by tesseract OCR (lossy — `quality: low`), or by an agent vision pass. */
+export const IMAGE = [".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff", ".gif"];
+/** Audio and video: local whisper. Minutes of CPU per file, hence `--skip-audio`. */
+export const AV = [".mp3", ".wav", ".m4a", ".flac", ".ogg", ".mp4", ".mov", ".mkv", ".webm"];
+
+export const extOf = (p: string) => {
+  const i = p.lastIndexOf(".");
+  return i < 0 ? "" : p.slice(i).toLowerCase();
+};
+
+/** raw/ files are always markdown — don't produce "budget.md.md". */
+export const mdName = (n: string) => (n.toLowerCase().endsWith(".md") ? n : n + ".md");
+
+export type Kind = "text" | "doc" | "image" | "av" | "skip";
+export function kindOf(path: string): Kind {
+  const e = extOf(path);
+  if (TEXT.includes(e)) return "text";
+  if (DOC.includes(e)) return "doc";
+  if (IMAGE.includes(e)) return "image";
+  if (AV.includes(e)) return "av";
+  return "skip";
+}

package/scripts/ingest/files.ts

@@ -0,0 +1,29 @@
+// Explicit file list → raw/files/. Same acquisition as a folder source, for the case where
+// the interesting files are scattered and naming them is easier than naming a root.
+import { existsSync } from "node:fs";
+import { basename } from "../lib/util";
+import type { Entry } from "../lib/ledger";
+import { acquireFile, newCounters, report, type Options } from "./acquire";
+import { detail, item, outcome, pos } from "../lib/log";
+import type { Source } from "./index";
+
+export async function ingestFiles(
+  s: Extract<Source, { type: "files" }>,
+  rawDir: string,
+  bundleDir: string,
+  entries: Map<string, Entry>,
+  opts: Options,
+) {
+  detail(`${s.paths.length} file(s) declared`);
+  const c = newCounters();
+  for (const [i, p] of s.paths.entries()) {
+    const at = pos(i + 1, s.paths.length);
+    if (!existsSync(p)) {
+      item(at, p);
+      outcome("missing, skipped");
+      continue;
+    }
+    await acquireFile(at, p, basename(p), rawDir, bundleDir, entries, c, opts);
+  }
+  report(c);
+}

package/scripts/ingest/folder.ts

@@ -0,0 +1,44 @@
+// Local folder → raw/folder/. Walks the tree and hands every file to acquireFile, which
+// owns the extraction decisions. Unchanged files (same content hash, raw/ copy still
+// present) are skipped.
+import { readdirSync, statSync, existsSync } from "node:fs";
+import { join } from "../lib/util";
+import type { Entry } from "../lib/ledger";
+import { acquireFile, newCounters, report, type Options } from "./acquire";
+import { detail, pos } from "../lib/log";
+import type { Source } from "./index";
+
+export async function ingestFolder(
+  s: Extract<Source, { type: "folder" }>,
+  rawDir: string,
+  bundleDir: string,
+  entries: Map<string, Entry>,
+  opts: Options,
+) {
+  if (!existsSync(s.path)) {
+    console.warn(`  missing folder, skipped: ${s.path}`);
+    return;
+  }
+
+  const walk = (d: string): string[] =>
+    readdirSync(d).flatMap((f) => {
+      const p = join(d, f);
+      return statSync(p).isDirectory() ? walk(p) : [p];
+    });
+
+  // Walk the whole tree up front rather than streaming it: a corpus on a network share can
+  // take a while to enumerate, and knowing the denominator is what makes "[ 3/57]" mean
+  // anything to someone deciding whether to wait.
+  detail(`scanning ${s.path} …`);
+  const files = walk(s.path);
+  detail(`${files.length} file(s) found`);
+
+  const c = newCounters();
+  for (const [i, p] of files.entries()) {
+    // Flatten the subtree into the filename so two `notes.md` in sibling folders don't
+    // collide in raw/, and so the origin stays legible without opening the file.
+    const rel = p.slice(s.path.length + 1).replaceAll(/[\\/]/g, "__");
+    await acquireFile(pos(i + 1, files.length), p, rel, rawDir, bundleDir, entries, c, opts);
+  }
+  report(c);
+}

package/scripts/ingest/index.ts

@@ -0,0 +1,125 @@
+// bun scripts/ingest/index.ts <bundle> [--force] [--skip-ocr] [--skip-audio]
+//
+// Ingest is ONE flat phase: get every declared source into bundles/<b>/raw/ as markdown,
+// with provenance, as cheaply and as locally as possible. It converts; it never interprets.
+// Whatever the text means is the catalog pass's problem (skills/catalog/SKILL.md).
+//
+// Everything mechanically convertible is converted here in one go — text, PDF, DOCX, ODT,
+// XLSX, PPTX, images (OCR), audio and video (whisper) — because leaving half the corpus as
+// "pending, agent please run a CLI" made ingest a multi-round negotiation rather than a
+// step. Sources that need an authenticated API (Confluence, ADO, git hosts) stay with the
+// agent via MCP/CLI, which is a plugin boundary, not a phase.
+import { parse } from "yaml";
+import { read, join, HUB } from "../lib/util";
+import { detail, section, totalElapsed } from "../lib/log";
+import { bundleForIngest, DEFAULT_BUNDLE } from "../lib/scaffold";
+import { readLedger, writeLedger } from "../lib/ledger";
+import { takeFlag } from "../lib/args";
+import { ingestFolder } from "./folder";
+import { ingestFiles } from "./files";
+import { ingestWeb } from "./web";
+import type { Options } from "./acquire";
+
+export type Source =
+  | { type: "folder"; path: string }
+  | { type: "files"; paths: string[] }
+  | { type: "web"; urls: string[] };
+
+const argv = process.argv.slice(2);
+const opts: Options = {
+  force: takeFlag(argv, "--force"),
+  // On by default: a corpus half-ingested because the expensive formats were opt-in is a
+  // corpus you have to remember to come back to. The flags exist for the impatient run.
+  ocr: !takeFlag(argv, "--skip-ocr"),
+  audio: !takeFlag(argv, "--skip-audio"),
+};
+const unknownFlag = argv.find((a) => a.startsWith("--"));
+if (unknownFlag) {
+  console.error(`Unknown ingest flag: ${unknownFlag}`);
+  process.exit(1);
+}
+const positional = argv.filter((a) => !a.startsWith("--"));
+if (positional.length > 1) {
+  console.error("Usage: khb ingest [bundle] [--force] [--skip-ocr] [--skip-audio]");
+  process.exit(1);
+}
+// No bundle named → `default`, created on the spot if the hub has none. Bytes always have
+// somewhere to land; sorting them into real bundles is a later, cheaper decision (a concept
+// is one file, and moving it is a `git mv`). Naming a bundle explicitly stays the norm.
+const bundle = positional[0] ?? DEFAULT_BUNDLE;
+
+const dir = bundleForIngest(bundle);
+let cfg: unknown;
+try {
+  cfg = parse(read(join(dir, "sources.yaml")));
+} catch (e) {
+  console.error(`${bundle}: sources.yaml is not valid YAML: ${(e as Error).message}`);
+  process.exit(1);
+}
+const declared =
+  cfg && typeof cfg === "object" && "sources" in cfg
+    ? (cfg as { sources?: unknown }).sources
+    : undefined;
+if (declared !== undefined && !Array.isArray(declared)) {
+  console.error(`${bundle}: sources.yaml 'sources' must be a list`);
+  process.exit(1);
+}
+const sources = (declared ?? []) as any[];
+for (const [i, s] of sources.entries()) {
+  const at = `sources.yaml sources[${i}]`;
+  if (!s || typeof s !== "object" || typeof s.type !== "string") {
+    console.error(`${bundle}: ${at} must be an object with a string 'type'`);
+    process.exit(1);
+  }
+  if (s.type === "folder" && typeof s.path !== "string") {
+    console.error(`${bundle}: ${at}.path must be a string`);
+    process.exit(1);
+  }
+  for (const [type, field] of [["files", "paths"], ["web", "urls"]] as const) {
+    if (s.type === type && (!Array.isArray(s[field]) || s[field].some((v: unknown) => typeof v !== "string"))) {
+      console.error(`${bundle}: ${at}.${field} must be a list of strings`);
+      process.exit(1);
+    }
+  }
+}
+if (!sources.length) {
+  console.log(`${bundle}: no sources configured.`);
+  console.log(`Declare them in bundles/${bundle}/sources.yaml, then re-run: khb ingest ${bundle}`);
+  process.exit(0);
+}
+
+const entries = readLedger(dir);
+
+// State the whole plan before doing any of it: which hub (a --hub/$KHB_HUB run can target a
+// folder you did not expect), which bundle, and which of the expensive extractors are armed.
+console.log(`khb ingest → bundle '${bundle}'`);
+detail(`hub:      ${HUB}`);
+detail(`bundle:   ${dir}`);
+detail(`sources:  ${sources.length} declared in bundles/${bundle}/sources.yaml`);
+detail(`options:  ocr=${opts.ocr ? "on" : "off"}  audio=${opts.audio ? "on" : "off"}  force=${opts.force ? "on" : "off"}`);
+detail(`ledger:   ${entries.size} existing row(s) in log.md`);
+
+for (const [i, s] of sources.entries()) {
+  const label =
+    s.type === "folder" ? s.path
+    : s.type === "files" ? `${s.paths.length} path(s)`
+    : s.type === "web" ? `${s.urls.length} url(s)`
+    : "";
+  section(`[${i + 1}/${sources.length}] ${s.type}${label ? ` — ${label}` : ""} → bundles/${bundle}/raw/${s.type}/`);
+  const rawDir = join(dir, "raw", s.type);
+  if (s.type === "folder") await ingestFolder(s, rawDir, dir, entries, opts);
+  else if (s.type === "files") await ingestFiles(s, rawDir, dir, entries, opts);
+  else if (s.type === "web") await ingestWeb(s, rawDir, dir, entries, opts);
+  else console.warn(`  '${(s as any).type}' has no scripted ingester — the agent pulls it via MCP/CLI (see skills/ingest/SKILL.md)`);
+}
+
+writeLedger(dir, entries, bundle);
+
+const all = [...entries.values()];
+const pending = all.filter((e) => !e.raw).length;
+const uncurated = all.filter((e) => e.raw && !e.curated).length;
+console.log(`\ndone in ${totalElapsed()}`);
+console.log(`ledger: ${all.length} source(s) in ${join(dir, "log.md")}`);
+if (pending) console.log(`  ${pending} with an empty 'raw' — not extracted; see the reasons above`);
+console.log(`  ${uncurated} in raw/ but not yet cataloged (empty 'curated')`);
+console.log(`Next: catalog ${bundle} — turn raw/ into concept docs (skills/catalog/SKILL.md).`);

package/scripts/ingest/protect.ts

@@ -0,0 +1,42 @@
+// Lightweight, dependency-free heuristics for detecting password-protected documents,
+// so ingest can say "password-protected" instead of reporting a mystery empty extraction.
+// Covers only the two document formats
+// (exts.ts) where a cheap byte-level signal exists:
+//   .pdf  — trailer/xref carries an /Encrypt dictionary reference
+//   .docx — MS wraps an encrypted OOXML payload in an OLE2 (CFB) container instead
+//           of the normal PK zip; that's a hard format marker, not a text scan
+// Legacy binary Office formats (.doc/.xls/.ppt) always use the CFB container whether
+// encrypted or not, so there's no cheap signal and they're not checked here.
+import { openSync, readSync, closeSync } from "node:fs";
+
+export const PROTECTABLE = new Set([".pdf", ".docx"]);
+
+const CFB_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
+
+/** Best-effort: false negatives are possible (unusual encoders), false positives are not — both signals are exact format markers. */
+export function detectPasswordProtected(path: string, ext: string, size: number): boolean {
+  const fd = openSync(path, "r");
+  try {
+    if (ext === ".docx") {
+      const buf = Buffer.alloc(8);
+      readSync(fd, buf, 0, 8, 0);
+      return buf.equals(CFB_MAGIC);
+    }
+    if (ext === ".pdf") {
+      const headLen = Math.min(size, 8192);
+      const headBuf = Buffer.alloc(headLen);
+      readSync(fd, headBuf, 0, headLen, 0);
+      if (headBuf.includes("/Encrypt")) return true;
+      if (size > headLen) {
+        const tailLen = Math.min(8192, size);
+        const tailBuf = Buffer.alloc(tailLen);
+        readSync(fd, tailBuf, 0, tailLen, size - tailLen);
+        if (tailBuf.includes("/Encrypt")) return true;
+      }
+      return false;
+    }
+    return false;
+  } finally {
+    closeSync(fd);
+  }
+}

package/scripts/ingest/web.ts

@@ -0,0 +1,54 @@
+// Internet links → raw/web/. Naive HTML→text; swap in a readability lib later.
+// A fetch is unavoidable (the hash only exists once the body is retrieved), but an
+// unchanged body still short-circuits the rewrite and keeps the ledger row stable.
+import { writeRaw, sha256, rawNameFor } from "../lib/util";
+import { record, isFresh, type Entry } from "../lib/ledger";
+import { detail, item, note, outcome, pos } from "../lib/log";
+import type { Options } from "./acquire";
+import type { Source } from "./index";
+
+export async function ingestWeb(
+  s: Extract<Source, { type: "web" }>,
+  rawDir: string,
+  bundleDir: string,
+  entries: Map<string, Entry>,
+  { force }: Options,
+) {
+  detail(`${s.urls.length} url(s) declared`);
+  let skipped = 0;
+  for (const [i, url] of s.urls.entries()) {
+    item(pos(i + 1, s.urls.length), url);
+    try {
+      note("fetching …");
+      const res = await fetch(url);
+      if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
+      const html = await res.text();
+      const text = html
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/&nbsp;|&amp;|&lt;|&gt;|&quot;/g, (m) => ({ "&nbsp;": " ", "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' })[m]!)
+        .replace(/[ \t]+/g, " ")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+      const hash = sha256(text);
+      if (!force && isFresh(entries, bundleDir, url, hash)) {
+        skipped++;
+        outcome("unchanged, skipped");
+        continue;
+      }
+      const name = new URL(url).pathname.split("/").filter(Boolean).pop() || new URL(url).hostname;
+      const file = rawNameFor(rawDir, `${name}.md`, url, entries.values());
+      const raw = writeRaw(rawDir, file, { source: url, sha256: hash.slice(0, 12), tool: "html-strip", quality: "high" }, text);
+      record(entries, { source: url, sha256: hash, fetched: new Date().toISOString(), raw });
+      outcome(`fetched → ${raw}  [html-strip, ${text.length} chars]`);
+    } catch (e) {
+      // A transient refresh failure must not erase a previously acquired, still-usable
+      // copy. Record a pending row only when this source has never succeeded before.
+      if (!entries.has(url))
+        record(entries, { source: url, sha256: "", fetched: new Date().toISOString(), raw: "" });
+      outcome(`failed — ${e}`);
+    }
+  }
+  if (skipped) console.log(`  ${skipped} unchanged, skipped`);
+}

package/scripts/init.ts

@@ -0,0 +1,93 @@
+// khb init [dir] / khb upgrade — create a hub, or refresh a hub's package-owned files.
+//
+// A hub is the user's knowledge: khb.json + outer.index.md + bundles/. The khb package
+// holds no knowledge, so the contract docs an agent needs (AGENTS.md, skills/, …) are
+// copied INTO the hub — an agent opened on the hub folder must be able to read them
+// without knowing where khb is installed. Those copies are package-owned: `upgrade`
+// overwrites them.
+import { cpSync, mkdirSync, writeFileSync, existsSync, statSync, rmSync } from "node:fs";
+import { join, resolve, basename, dirname } from "node:path";
+import { PKG, HUB_TEMPLATE, MANAGED, RETIRED, MARKER, markerIn, version } from "./lib/paths";
+
+const upgrading = process.env.KHB_SUBCOMMAND === "upgrade";
+const [dirArg] = process.argv.slice(2);
+
+/** Copy every package-owned contract file into the hub, replacing what is there. */
+function syncManaged(hub: string): string[] {
+  const done: string[] = [];
+  for (const f of MANAGED) {
+    const src = join(PKG, f);
+    if (!existsSync(src)) continue;
+    const dest = join(hub, f);
+    mkdirSync(dirname(dest), { recursive: true });
+    cpSync(src, dest, { recursive: true, force: true });
+    done.push(statSync(src).isDirectory() ? `${f}/` : f);
+  }
+  return done;
+}
+
+/** Drop package-owned files that later versions stopped shipping. */
+function pruneRetired(hub: string): string[] {
+  const gone: string[] = [];
+  for (const f of RETIRED) {
+    const p = join(hub, f);
+    if (!existsSync(p)) continue;
+    rmSync(p, { recursive: true, force: true });
+    gone.push(f);
+  }
+  return gone;
+}
+
+function stamp(hub: string, created?: string) {
+  writeFileSync(
+    join(hub, MARKER),
+    JSON.stringify(
+      { khb: version(), created: created ?? new Date().toISOString(), upgraded: new Date().toISOString() },
+      null,
+      2,
+    ) + "\n",
+  );
+}
+
+if (upgrading) {
+  const { HUB } = await import("./lib/util"); // resolves the hub, or exits with guidance
+  // The hub may still carry a marker name from an older version; stamp() writes MARKER,
+  // so drop the old file rather than leaving the hub with two.
+  const found = markerIn(HUB)!;
+  const before = JSON.parse(await Bun.file(join(HUB, found)).text());
+  if (found !== MARKER) rmSync(join(HUB, found));
+  const synced = syncManaged(HUB);
+  const pruned = pruneRetired(HUB);
+  stamp(HUB, before.created);
+  console.log(`Upgraded ${HUB}: ${before.khb ?? before.bkr ?? "?"} -> ${version()}`);
+  console.log(`  refreshed: ${synced.join(", ")}`);
+  if (found !== MARKER) console.log(`  renamed: ${found} -> ${MARKER}`);
+  if (pruned.length) console.log(`  removed (no longer part of the contract): ${pruned.join(", ")}`);
+  console.log(`Your bundles/ and outer.index.md were not touched. Next: khb lint`);
+} else {
+  const hub = resolve(dirArg ?? process.cwd());
+
+  if (markerIn(hub)) {
+    console.error(`Already a KHB hub: ${hub}`);
+    console.error(`To refresh its contract docs:   khb upgrade`);
+    process.exit(1);
+  }
+
+  mkdirSync(join(hub, "bundles"), { recursive: true });
+  cpSync(join(HUB_TEMPLATE, "outer.index.md"), join(hub, "outer.index.md"));
+  // Dotfiles: shipped unprefixed so npm doesn't swallow them, renamed on the way in.
+  // Never clobber — `khb init` may be run inside a folder that is already a git repo.
+  for (const f of ["gitignore", "gitattributes"])
+    if (!existsSync(join(hub, `.${f}`))) cpSync(join(HUB_TEMPLATE, f), join(hub, `.${f}`));
+  const synced = syncManaged(hub);
+  stamp(hub);
+
+  console.log(`Hub created: ${hub}`);
+  console.log(`  khb.json, outer.index.md, bundles/, .gitignore, .gitattributes`);
+  console.log(`  contract docs (package-owned, refreshed by 'khb upgrade'): ${synced.join(", ")}`);
+  console.log(`\nNext:`);
+  console.log(`  cd ${basename(hub)}`);
+  console.log(`  git init                              # optional, but recommended`);
+  console.log(`  khb new-bundle <name> "<scope>"       # your first bundle`);
+  console.log(`\nThen open this folder with Claude or Codex — both load AGENTS.md and the workflow skills.`);
+}

package/scripts/lib/args.ts

@@ -0,0 +1,8 @@
+// Tiny argv helpers shared by the scripts that take flags. Each `take*` removes what it
+// consumed from the array, so whatever is left is the positional arguments.
+export function takeFlag(args: string[], name: string): boolean {
+  const i = args.indexOf(name);
+  if (i < 0) return false;
+  args.splice(i, 1);
+  return true;
+}