@msareen/knowledge-hub-builder 0.1.3

package/scripts/lib/ledger.ts

@@ -0,0 +1,136 @@
+// log.md — the durable ingest ledger (OKF-reserved filename, so lint never treats it
+// as a concept doc). Committed, unlike raw/, so it survives raw/ being deleted and
+// re-derived. Columns: source | sha256 | fetched | raw | curated.
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+
+export type Entry = {
+  source: string;   // canonical URI of the origin: absolute path, url, or tool query
+  sha256: string;   // content hash — drives skip-unchanged and cross-bundle dedup
+  fetched: string;  // ISO timestamp of last acquisition
+  raw: string;      // bundle-relative raw/ path, or "" when extraction is still pending
+  curated: string;  // concept doc(s) distilled from it, or "" when not yet curated
+};
+
+const HEADER = `# {{name}} — ingest log
+
+Ingestion ledger, one row per source. \`khb ingest\` maintains \`source\`, \`sha256\`,
+\`fetched\` and \`raw\`; the agent fills \`curated\` while cataloging (skills/catalog/SKILL.md).
+
+Empty \`raw\` = seen but not extracted (protected, unreadable, or skipped by a flag).
+Empty \`curated\` = in raw/ but not yet distilled into a concept doc.
+
+| source | sha256 | fetched | raw | curated |
+|---|---|---|---|---|
+`;
+
+const unwrap = (s: string) => s.trim().replace(/^`|`$/g, "").trim();
+const wrap = (s: string) => (s ? `\`${s.replaceAll("|", "\\|")}\`` : "");
+
+export function ledgerPath(bundleDir: string) {
+  return join(bundleDir, "log.md");
+}
+
+/** Parse log.md into entries keyed by source. Missing file = empty ledger. */
+export function readLedger(bundleDir: string): Map<string, Entry> {
+  const p = ledgerPath(bundleDir);
+  const out = new Map<string, Entry>();
+  if (!existsSync(p)) return out;
+  for (const line of readFileSync(p, "utf8").split("\n")) {
+    if (!line.startsWith("|")) continue;
+    const cells = line.split("|").slice(1, -1).map(unwrap);
+    if (cells.length < 5) continue;
+    if (cells[0] === "source" || /^-+$/.test(cells[0])) continue; // header / separator
+    const [source, sha256, fetched, raw, curated] = cells;
+    out.set(source, { source, sha256, fetched, raw, curated });
+  }
+  return out;
+}
+
+/** Rewrite log.md, preserving any prose above the table. */
+export function writeLedger(bundleDir: string, entries: Map<string, Entry>, bundleName: string) {
+  const p = ledgerPath(bundleDir);
+  const existing = existsSync(p) ? readFileSync(p, "utf8") : HEADER.replaceAll("{{name}}", bundleName);
+  const preamble = existing.split("\n").filter((l) => !l.startsWith("|")).join("\n").trimEnd();
+  const rows = [...entries.values()]
+    .sort((a, b) => a.source.localeCompare(b.source))
+    .map((e) => `| ${wrap(e.source)} | ${wrap(e.sha256.slice(0, 12))} | ${e.fetched} | ${wrap(e.raw)} | ${wrap(e.curated)} |`);
+  const table = ["", "| source | sha256 | fetched | raw | curated |", "|---|---|---|---|---|", ...rows, ""];
+  writeFileSync(p, preamble + "\n" + table.join("\n"));
+}
+
+/**
+ * Upsert, preserving the agent-owned `curated` column. Returns the merged entry.
+ */
+export function record(entries: Map<string, Entry>, e: Omit<Entry, "curated">): Entry {
+  const merged = { ...e, curated: entries.get(e.source)?.curated ?? "" };
+  entries.set(e.source, merged);
+  return merged;
+}
+
+/**
+ * True when this source is already acquired at this exact content hash and its raw/
+ * file is still on disk — the skip condition for incremental re-ingest.
+ */
+export function isFresh(entries: Map<string, Entry>, bundleDir: string, source: string, hash: string) {
+  const e = entries.get(source);
+  return !!e && e.sha256.startsWith(hash.slice(0, 12)) && !!e.raw && existsSync(join(bundleDir, e.raw));
+}
+
+/** A ledger row acquired from a local file, as opposed to a URL or a tool query. */
+const isLocalSource = (s: string) => !/^[a-z][a-z0-9+.-]*:\/\//i.test(s);
+
+export type Identity =
+  | { kind: "moved"; from: Entry }        // same bytes, old path gone — one file, renamed
+  | { kind: "copy"; twin: Entry }         // same bytes, old path still there — two files
+  | { kind: "ambiguous"; twins: Entry[] } // several rows share these bytes; do not guess
+  | { kind: "new" };
+
+/**
+ * Decide what a not-yet-seen source path actually *is*, by content hash.
+ *
+ * Keying the ledger on the path alone means moving a file reads as a deletion plus an
+ * unrelated arrival: a second raw/ file with identical bytes, a second row with an empty
+ * `curated`, and so a second concept for material already cataloged. The bytes are the
+ * identity; the path is just where they happen to live today.
+ *
+ * Only local-file rows are candidates — a URL cannot have been "moved" on this disk — and
+ * a row is only a move if its old path is genuinely gone. Two live paths with the same
+ * bytes are a copy, which is a real (if redundant) second source and the agent's call, not
+ * ours. Several candidates at once is ambiguous, and guessing there would silently rewire
+ * provenance, so we report and leave it alone.
+ */
+export function identify(entries: Map<string, Entry>, bundleDir: string, source: string, hash: string): Identity {
+  if (entries.has(source)) return { kind: "new" }; // known path: freshness, not identity
+  const twins = [...entries.values()].filter(
+    (e) =>
+      e.source !== source &&
+      isLocalSource(e.source) &&
+      e.sha256.startsWith(hash.slice(0, 12)) &&
+      !!e.raw &&
+      existsSync(join(bundleDir, e.raw)),
+  );
+  if (!twins.length) return { kind: "new" };
+  const orphaned = twins.filter((e) => !existsSync(e.source));
+  if (orphaned.length === 1) return { kind: "moved", from: orphaned[0] };
+  if (orphaned.length > 1) return { kind: "ambiguous", twins: orphaned };
+  return { kind: "copy", twin: twins[0] };
+}
+
+/**
+ * Re-point an existing row at the file's new path, keeping its raw/ file and — the whole
+ * point — its `curated` value, so cataloged material is not offered as backlog again.
+ *
+ * The raw/ filename deliberately does *not* change: concept docs cite raw paths in their
+ * Citations sections, and renaming the file underneath them would break those links to
+ * cosmetically match a path that is already recorded inside the file's own provenance
+ * header. The header is what gets corrected.
+ */
+export function adopt(entries: Map<string, Entry>, from: Entry, source: string): Entry {
+  entries.delete(from.source);
+  const moved = { ...from, source };
+  entries.set(source, moved);
+  return moved;
+}
+
+export { HEADER as LEDGER_HEADER };

package/scripts/lib/log.ts

@@ -0,0 +1,53 @@
+// Progress reporting for the long-running commands.
+//
+// Ingest is mostly waiting, and it waits inside libraries that say nothing: a scanned PDF is
+// seconds per page, a video is minutes per file. A silent process is indistinguishable from
+// a hung one, and after the fact "which file produced that?" is unanswerable. So every unit
+// of work announces itself BEFORE the work starts, carrying its position in the run, and
+// closes with what happened and how long it took.
+//
+// Verbose is the only mode on purpose. There is no --quiet: the per-file line is the audit
+// trail for a pass that rewrites a bundle's raw/, and a run whose output you have to re-run
+// to reconstruct is worse than a noisy one. Nothing here changes what khb does.
+
+const RUN_START = Date.now();
+let itemStart = RUN_START;
+
+export const secs = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
+
+/** Wall-clock time since the process started — the closing line of a command. */
+export const totalElapsed = () => secs(Date.now() - RUN_START);
+
+/** "[ 3/57]" — the counter is right-aligned so filenames stay in one column. */
+export function pos(i: number, n: number): string {
+  return `[${String(i).padStart(String(n).length)}/${n}]`;
+}
+
+/** A blank-line-separated heading: a source, a bundle, a phase. */
+export function section(title: string) {
+  console.log(`\n${title}`);
+}
+
+/** Indented context under a heading — settings, counts, where things are going. */
+export function detail(msg: string) {
+  console.log(`  ${msg}`);
+}
+
+/**
+ * The unit of work about to start. Prints before any work happens — that is the whole
+ * point — and starts this item's clock.
+ */
+export function item(prefix: string, label: string) {
+  itemStart = Date.now();
+  console.log(`  ${prefix} ${label}`);
+}
+
+/** A step inside the current item: what khb is about to do, or what it just learned. */
+export function note(msg: string) {
+  console.log(`         ${msg}`);
+}
+
+/** How the current item ended, with its own elapsed time. Exactly one per item. */
+export function outcome(msg: string) {
+  console.log(`         ${msg} (${secs(Date.now() - itemStart)})`);
+}

package/scripts/lib/paths.ts

@@ -0,0 +1,55 @@
+// Package-side paths. Importing this must never require a hub to exist — `khb init`
+// runs before there is one. Hub-side paths live in util.ts.
+import { fileURLToPath } from "node:url";
+import { join } from "node:path";
+import { readFileSync, existsSync } from "node:fs";
+
+/** Root of the installed @msareen/knowledge-hub-builder package (NOT the user's hub). */
+export const PKG = fileURLToPath(new URL("../..", import.meta.url));
+
+export const TEMPLATE = join(PKG, ".bundle_template");
+export const HUB_TEMPLATE = join(PKG, "templates", "hub");
+
+/** Marker file that identifies a hub root; `khb` walks up from cwd looking for it. */
+export const MARKER = "khb.json";
+
+/**
+ * Marker names used by earlier versions. Still recognised when resolving a hub —
+ * otherwise a hub created before the rename becomes invisible to every command,
+ * `khb upgrade` included, and there is no way in from the CLI. `khb upgrade` renames
+ * the file it finds to MARKER, so each hub carries a legacy name at most once.
+ */
+export const LEGACY_MARKERS = ["bkr.json"];
+
+/**
+ * The marker file present in `dir`, if it is a hub at all. Lives here rather than in
+ * util.ts because `khb init` needs it *before* a hub exists, and importing util.ts
+ * resolves a hub or exits.
+ */
+export const markerIn = (dir: string): string | undefined =>
+  [MARKER, ...LEGACY_MARKERS].find((m) => existsSync(join(dir, m)));
+
+/**
+ * Package-owned files copied into every hub by `khb init` and refreshed by
+ * `khb upgrade`. These are the agent contract — the hub needs its own copies so an
+ * agent opened on the hub folder can read them without knowing where khb is installed.
+ * Anything here is overwritten on upgrade, so users must not edit them.
+ */
+export const MANAGED = [
+  "AGENTS.md",
+  "CLAUDE.md",
+  "SPEC.md",
+  "skills",
+  ".agents/skills",
+  ".claude/skills",
+];
+
+/**
+ * Files that used to be MANAGED and no longer are. `khb upgrade` deletes them from the
+ * hub — a stale copy left behind states an older contract than the one now shipping,
+ * and an agent has no way to tell which is current. Only ever list package-owned names.
+ */
+export const RETIRED = ["AGENT.md", "query.md", "ingest.md", "lint.md"];
+
+export const version = (): string =>
+  JSON.parse(readFileSync(join(PKG, "package.json"), "utf8")).version ?? "0.0.0";

package/scripts/lib/scaffold.ts

@@ -0,0 +1,56 @@
+// Bundle creation, shared by `khb new-bundle` and by ingest's default-bundle fallback.
+// One implementation so a bundle born from a bare `khb ingest` is indistinguishable from
+// one the user named: same template, same {{name}} substitution, same outer.index.md row.
+import { cpSync, readFileSync, writeFileSync, readdirSync, statSync, existsSync } from "node:fs";
+import { HUB, BUNDLES, TEMPLATE, join } from "./util";
+
+export const DEFAULT_BUNDLE = "default";
+
+export const VALID_NAME = /^[a-z0-9][a-z0-9-]*$/;
+
+/** Scaffold bundles/<name>/ from the template and register it in outer.index.md. */
+export function createBundle(name: string, scope: string): string {
+  const dest = join(BUNDLES, name);
+  cpSync(TEMPLATE, dest, { recursive: true });
+
+  // fill {{name}} placeholders
+  const walk = (d: string): string[] =>
+    readdirSync(d).flatMap((f) => (statSync(join(d, f)).isDirectory() ? walk(join(d, f)) : [join(d, f)]));
+  for (const f of walk(dest)) {
+    const c = readFileSync(f, "utf8");
+    if (c.includes("{{name}}")) writeFileSync(f, c.replaceAll("{{name}}", name));
+  }
+
+  // register in outer.index.md (append to first table)
+  const outerPath = join(HUB, "outer.index.md");
+  const lines = readFileSync(outerPath, "utf8").split("\n");
+  const row = `| [${name}](bundles/${name}/index.md) | ${scope} | TODO |`;
+  let lastTableRow = -1;
+  for (let i = 0; i < lines.length; i++) if (lines[i].startsWith("|")) lastTableRow = i;
+  lines.splice(lastTableRow + 1, 0, row);
+  writeFileSync(outerPath, lines.join("\n"));
+
+  return dest;
+}
+
+/**
+ * Resolve the bundle to ingest into, creating `default` if that is the target and it does
+ * not exist yet. A hub with no bundles must still have somewhere for bytes to land — the
+ * alternative is refusing the first ingest anyone ever runs. Only `default` is ever
+ * conjured this way: a misspelled explicit name is a mistake, not a request to scaffold.
+ */
+export function bundleForIngest(name: string): string {
+  const dir = join(BUNDLES, name);
+  if (existsSync(dir)) return dir;
+  if (name !== DEFAULT_BUNDLE) {
+    console.error(`No such bundle: ${name}`);
+    console.error(`Create it:   khb new-bundle ${name} "<scope>"`);
+    process.exit(1);
+  }
+  // The scope line lands in outer.index.md, where every agent reads it — so it must not
+  // read as an instruction to reorganize the hub. Splitting `default` into real bundles is
+  // the user's call, exactly like any other bundle decision.
+  createBundle(DEFAULT_BUNDLE, "Unsorted material — where an ingest with no named bundle lands; moves out when you say which bundle owns it");
+  console.log(`Created bundles/${DEFAULT_BUNDLE}/ — the landing bundle for unrouted material.`);
+  return dir;
+}

package/scripts/lib/util.ts

@@ -0,0 +1,154 @@
+import { readdirSync, readFileSync, existsSync, statSync, mkdirSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { join, basename, dirname, resolve } from "node:path";
+import { MARKER, markerIn } from "./paths";
+
+/**
+ * Find the hub root — the folder holding khb.json, outer.index.md and bundles/.
+ * It is the user's knowledge, and lives wherever they put it; the khb package holds
+ * no knowledge of its own. Precedence: $KHB_HUB (set from --hub by cli.ts) > nearest
+ * ancestor of cwd containing the marker.
+ */
+function resolveHub(): string {
+  const explicit = process.env.KHB_HUB;
+  if (explicit) {
+    const dir = resolve(explicit);
+    if (!markerIn(dir)) {
+      console.error(`Not a KHB hub (no ${MARKER}): ${dir}`);
+      process.exit(1);
+    }
+    return dir;
+  }
+  for (let dir = process.cwd(); ; ) {
+    if (markerIn(dir)) return dir;
+    const up = dirname(dir);
+    if (up === dir) break;
+    dir = up;
+  }
+  console.error(`No KHB hub found in ${process.cwd()} or any parent directory.`);
+  console.error(`Create one:   khb init <dir>`);
+  console.error(`Or point at an existing one:   khb --hub <dir> <command>   (or set $KHB_HUB)`);
+  process.exit(1);
+}
+
+export const HUB = resolveHub();
+export const BUNDLES = join(HUB, "bundles");
+export const INBOX = join(HUB, "inbox");
+export { TEMPLATE, markerIn } from "./paths";
+
+export function listBundles(): string[] {
+  if (!existsSync(BUNDLES)) return [];
+  return readdirSync(BUNDLES).filter((d) => statSync(join(BUNDLES, d)).isDirectory());
+}
+
+export function read(path: string): string {
+  return readFileSync(path, "utf8");
+}
+
+/** All markdown link targets in a file: [text](target) */
+export function mdLinks(md: string): { text: string; target: string }[] {
+  const out: { text: string; target: string }[] = [];
+  for (const m of md.matchAll(/\[([^\]]*)\]\(([^)]+)\)/g)) out.push({ text: m[1], target: m[2] });
+  return out;
+}
+
+/** Bundle names mentioned in a refs.md table (first column). */
+export function refTargets(refsMd: string): string[] {
+  const out: string[] = [];
+  for (const line of refsMd.split("\n")) {
+    const m = line.match(/^\|\s*\[?([a-z0-9][a-z0-9-]*)\]?[^|]*\|/);
+    if (m && !["bundle", "---"].includes(m[1])) out.push(m[1]);
+  }
+  return out;
+}
+
+/**
+ * Provenance recorded on every raw/ file. `source` is the whole point of the phase:
+ * extraction is sometimes lossy, so curation must always be able to walk back to the
+ * original bytes. `tool` and `quality` say how much to trust what follows — `low` means
+ * OCR or a transcript guessed at it, and re-reading the source is the remedy.
+ */
+export type RawMeta = {
+  source: string;
+  sha256?: string;
+  tool?: string;
+  quality?: "high" | "low";
+};
+
+const safeRawName = (name: string) => name.replace(/[^\w.-]+/g, "_") || "source.md";
+
+/**
+ * Keep the readable filename unless another source already owns it. The suffix is based on
+ * source identity, not content, so identical bytes acquired from two origins retain honest
+ * provenance instead of overwriting each other.
+ */
+export function rawNameFor(
+  dir: string,
+  name: string,
+  source: string,
+  entries: Iterable<{ source: string; raw: string }>,
+): string {
+  const safe = safeRawName(name);
+  const rel = `raw/${basename(dir)}/${safe}`;
+  const owner = [...entries].find((e) => e.raw === rel);
+  if ((!owner || owner.source === source) && (owner || !existsSync(join(dir, safe)))) return safe;
+  const stem = safe.toLowerCase().endsWith(".md") ? safe.slice(0, -3) : safe;
+  return `${stem}--${sha256(source).slice(0, 12)}.md`;
+}
+
+/**
+ * Write a raw/ file with provenance front matter. Returns its bundle-relative path.
+ * Silent by design — the caller owns the per-file line, and knows the verb ("copied",
+ * "extracted", "transcribed") that a write on its own cannot.
+ */
+export function writeRaw(dir: string, name: string, meta: RawMeta, body: string): string {
+  mkdirSync(dir, { recursive: true });
+  const safe = safeRawName(name);
+  const fm =
+    `---\nsource: ${JSON.stringify(meta.source)}\nfetched: ${new Date().toISOString()}\n` +
+    (meta.sha256 ? `sha256: ${meta.sha256}\n` : "") +
+    (meta.tool ? `extract_tool: ${JSON.stringify(meta.tool)}\n` : "") +
+    (meta.quality ? `quality: ${meta.quality}\n` : "") +
+    `---\n\n`;
+  writeFileSync(join(dir, safe), fm + body);
+  // dir is <bundle>/raw/<type>; report the path as the ledger stores it
+  const type = basename(dir);
+  return `raw/${type}/${safe}`;
+}
+
+/**
+ * Correct the `source:` line of an already-written raw/ file, for when the same bytes turn
+ * up at a new path. Only that one line moves: the body is unchanged by definition (same
+ * hash), and rewriting the file wholesale would churn a diff for no reason.
+ */
+export function retargetRaw(bundleDir: string, rawRel: string, source: string): boolean {
+  const p = join(bundleDir, rawRel);
+  if (!existsSync(p)) return false;
+  const text = readFileSync(p, "utf8");
+  const end = text.indexOf("\n---", 3);
+  if (!text.startsWith("---\n") || end === -1) return false; // no provenance header to fix
+  const head = text.slice(0, end).replace(/^source:.*$/m, `source: ${JSON.stringify(source)}`);
+  writeFileSync(p, head + text.slice(end));
+  return true;
+}
+
+export const sha256 = (buf: Buffer | string) => createHash("sha256").update(buf).digest("hex");
+
+/** Hash a file in chunks — corpora contain multi-GB binaries we must not slurp. */
+export async function sha256File(path: string): Promise<string> {
+  const h = createHash("sha256");
+  const stream = (await import("node:fs")).createReadStream(path, { highWaterMark: 1 << 20 });
+  for await (const chunk of stream) h.update(chunk as Buffer);
+  return h.digest("hex");
+}
+
+export function bundleDir(name: string): string {
+  const dir = join(BUNDLES, name);
+  if (!existsSync(dir)) {
+    console.error(`No such bundle: ${name}`);
+    process.exit(1);
+  }
+  return dir;
+}
+
+export { join, basename, existsSync, mkdirSync };

package/scripts/lint.ts

@@ -0,0 +1,165 @@
+// khb lint — enforce skills/lint/SKILL.md (structural rules + OKF v0.1 conformance) across the hub
+import { HUB, BUNDLES, listBundles, read, mdLinks, refTargets, join, existsSync } from "./lib/util";
+import { detail, section, totalElapsed } from "./lib/log";
+import { readdirSync, statSync } from "node:fs";
+import { dirname, relative } from "node:path";
+import { parse as parseYaml } from "yaml";
+
+/** OKF v0.1 concept frontmatter. Unknown keys are warned, not rejected — OKF is permissive,
+ *  but a `titel:` typo silently loses the field, so it is worth one line of noise. */
+const OKF_FIELDS = new Set(["type", "title", "description", "resource", "tags", "timestamp"]);
+
+/** Accepts a YAML-parsed Date (unquoted) or an ISO-8601 string (quoted). */
+const isTimestamp = (v: unknown) =>
+  v instanceof Date ? !isNaN(v.getTime()) : typeof v === "string" && !isNaN(Date.parse(v));
+
+let errors = 0, warnings = 0;
+const err = (rule: string, msg: string) => { errors++; console.error(`ERROR ${rule}: ${msg}`); };
+const warn = (rule: string, msg: string) => { warnings++; console.warn(`warn  ${rule}: ${msg}`); };
+
+const stripComments = (md: string) => md.replace(/<!--[\s\S]*?-->/g, "");
+const RESERVED = ["index.md", "log.md", "refs.md"]; // refs.md is KHB-reserved
+const bundles = listBundles();
+const outerIndex = read(join(HUB, "outer.index.md"));
+
+/** All files under dir (relative paths), skipping raw/. */
+function walk(dir: string, base = dir): string[] {
+  return readdirSync(dir).flatMap((f) => {
+    const p = join(dir, f);
+    if (statSync(p).isDirectory()) return f === "raw" ? [] : walk(p, base);
+    return [relative(base, p).replaceAll("\\", "/")];
+  });
+}
+
+console.log(`khb lint → ${HUB}`);
+detail(`${bundles.length} bundle(s): ${bundles.join(", ") || "none"}`);
+
+for (const [bi, b] of bundles.entries()) {
+  const dir = join(BUNDLES, b);
+  // Name the bundle before its findings: an unattributed "ERROR L4" in a fifty-bundle hub
+  // sends you grepping, and a clean bundle should still show that it was actually checked.
+  section(`[${bi + 1}/${bundles.length}] ${b}`);
+
+  // L2 name
+  if (!/^[a-z0-9][a-z0-9-]*$/.test(b)) err("L2", `bad bundle name '${b}'`);
+
+  // L1 required files
+  for (const f of ["index.md", "refs.md", "sources.yaml"])
+    if (!existsSync(join(dir, f))) err("L1", `${b}: missing ${f}`);
+
+  // L3 registered in outer index
+  if (!outerIndex.includes(`bundles/${b}/`)) err("L3", `${b}: not listed in outer.index.md`);
+
+  const files = existsSync(dir) ? walk(dir) : [];
+  const mdFiles = files.filter((f) => f.endsWith(".md"));
+  const concepts = mdFiles.filter((f) => !RESERVED.includes(f.split("/").pop()!));
+  const indexes = mdFiles.filter((f) => f.split("/").pop() === "index.md");
+  detail(`${concepts.length} concept doc(s), ${indexes.length} index file(s)`);
+
+  // Collect all index link targets, resolved to bundle-relative paths
+  const indexed = new Set<string>();
+  for (const idx of indexes) {
+    const md = stripComments(read(join(dir, idx)));
+    for (const l of mdLinks(md)) {
+      if (l.target.startsWith("http")) continue;
+      const resolved = l.target.startsWith("/")
+        ? l.target.slice(1)
+        : relative(dir, join(dir, dirname(idx), l.target)).replaceAll("\\", "/");
+      indexed.add(resolved.replace(/\/$/, ""));
+      // L4b index links resolve (warning — OKF tolerates not-yet-written knowledge)
+      if (!existsSync(join(dir, resolved)))
+        warn("L4", `${b}: ${idx} links to missing ${resolved}`);
+    }
+  }
+
+  // L4a every concept is indexed somewhere
+  for (const c of concepts)
+    if (!indexed.has(c)) err("L4", `${b}: ${c} not listed in any index.md`);
+
+  for (const c of concepts) {
+    const body = read(join(dir, c));
+
+    // L9 OKF conformance: frontmatter must parse and carry a usable field set.
+    // Frontmatter is the machine-readable half of a concept — routing, filtering and any
+    // future index generator read it — so a typo'd key is a silent data loss, not a style nit.
+    const fm = body.match(/^---\n([\s\S]*?)\n---/)?.[1];
+    if (fm === undefined) err("L9", `${b}: ${c} has no YAML frontmatter (OKF requires it)`);
+    else {
+      let meta: Record<string, unknown> | undefined;
+      try {
+        meta = (parseYaml(fm) ?? {}) as Record<string, unknown>;
+      } catch (e) {
+        err("L9", `${b}: ${c} frontmatter is not valid YAML — ${(e as Error).message.split("\n")[0]}`);
+      }
+      if (meta) {
+        const str = (k: string) => (typeof meta![k] === "string" ? (meta![k] as string).trim() : "");
+        // type is the one OKF hard requirement; the rest degrade to warnings so an
+        // in-progress hub still lints clean while its authors fill things in.
+        if (!str("type")) err("L9", `${b}: ${c} frontmatter missing required 'type'`);
+        for (const k of ["title", "description"])
+          if (!str(k)) warn("L9", `${b}: ${c} frontmatter missing '${k}'`);
+        if ("tags" in meta && !Array.isArray(meta.tags))
+          err("L9", `${b}: ${c} 'tags' must be a YAML list, not ${typeof meta.tags}`);
+        if (Array.isArray(meta.tags) && meta.tags.some((t) => typeof t !== "string"))
+          err("L9", `${b}: ${c} 'tags' must contain only strings`);
+        if ("timestamp" in meta && !isTimestamp(meta.timestamp))
+          warn("L9", `${b}: ${c} 'timestamp' is not an ISO-8601 datetime`);
+        for (const k of Object.keys(meta))
+          if (!OKF_FIELDS.has(k)) warn("L9", `${b}: ${c} unknown frontmatter key '${k}'`);
+      }
+    }
+
+    // L6 no cross-bundle links from concept docs
+    for (const l of mdLinks(stripComments(body))) {
+      if (/(^|\/)bundles\//.test(l.target) || l.target.startsWith("../../"))
+        err("L6", `${b}: ${c} links into another bundle (${l.target}) — use refs.md`);
+    }
+  }
+
+  // L7 ref targets exist
+  if (existsSync(join(dir, "refs.md"))) {
+    for (const t of refTargets(read(join(dir, "refs.md"))))
+      if (!bundles.includes(t)) err("L7", `${b}: refs.md targets missing bundle '${t}'`);
+  }
+
+  // L8 raw provenance (warning)
+  const rawDir = join(dir, "raw");
+  if (existsSync(rawDir)) {
+    const rawFiles = (readdirSync(rawDir, { recursive: true }) as string[]).filter((f) => f.endsWith(".md"));
+    detail(`${rawFiles.length} raw/ file(s) checked for provenance`);
+    for (const f of readdirSync(rawDir, { recursive: true }) as string[]) {
+      try {
+        if (!f.endsWith(".md")) continue;
+        const head = read(join(rawDir, f));
+        const rfm = head.match(/^---\n([\s\S]*?)\n---/)?.[1];
+        if (rfm === undefined) { warn("L8", `${b}: raw/${f} missing provenance header`); continue; }
+        // `source` is the whole point of the header: it is how a bad extraction gets re-read.
+        if (!/^source:\s*\S/m.test(rfm)) warn("L8", `${b}: raw/${f} provenance missing 'source'`);
+        const q = rfm.match(/^quality:\s*(\S+)/m)?.[1];
+        if (q && q !== "high" && q !== "low")
+          warn("L8", `${b}: raw/${f} quality '${q}' is not high|low`);
+      } catch {}
+    }
+  }
+}
+
+// L3 reverse: outer index entries exist
+for (const l of mdLinks(outerIndex)) {
+  const m = l.target.match(/^bundles\/([a-z0-9-]+)\//);
+  if (m && !bundles.includes(m[1])) err("L3", `outer.index.md lists missing bundle '${m[1]}'`);
+}
+
+// L5 index prose check (rough): paragraph-length prose in index files
+function proseCheck(name: string, md: string) {
+  for (const block of stripComments(md).split(/\n\s*\n/)) {
+    const t = block.trim();
+    if (!t || /^[#|\-*]/.test(t) || t.startsWith("---")) continue;
+    if (t.split(/\s+/).length > 30) warn("L5", `${name}: paragraph-length prose in an index file`);
+  }
+}
+proseCheck("outer.index.md", outerIndex);
+for (const b of bundles)
+  if (existsSync(join(BUNDLES, b, "index.md"))) proseCheck(`${b}/index.md`, read(join(BUNDLES, b, "index.md")));
+
+console.log(`\nlint: ${errors} error(s), ${warnings} warning(s) across ${bundles.length} bundle(s) in ${totalElapsed()}`);
+process.exit(errors ? 1 : 0);

package/scripts/new-bundle.ts

@@ -0,0 +1,17 @@
+// khb new-bundle <name> ["scope line"] — scaffold from .bundle_template + register.
+// Template comes from the package; the bundle lands in the hub.
+import { existsSync } from "node:fs";
+import { BUNDLES, join } from "./lib/util";
+import { createBundle, VALID_NAME } from "./lib/scaffold";
+
+const [name, scope = "TODO scope"] = process.argv.slice(2);
+if (!name || !VALID_NAME.test(name)) {
+  console.error("Usage: khb new-bundle <name> [scope]   (lowercase, digits, hyphens)");
+  process.exit(1);
+}
+if (existsSync(join(BUNDLES, name))) { console.error(`Bundle '${name}' already exists`); process.exit(1); }
+
+createBundle(name, scope);
+
+console.log(`Created bundles/${name}/ and registered it in outer.index.md`);
+console.log("Next: set its scope line in outer.index.md, add sources to sources.yaml, run: khb lint");