@panaversity/ksor 0.0.20 → 0.0.22

package/templates/scaffold/system/site/lib/source.ts

@@ -1,8 +1,18 @@
 import { docs } from "collections/server";
 import { loader } from "fumadocs-core/source";
 import { lucideIconsPlugin } from "fumadocs-core/source/lucide-icons";
+import { statusBadgesPlugin } from "fumadocs-core/source/plugins/status-badges";
 import type { Node, Root } from "fumadocs-core/page-tree";
 
+import {
+  agentFrontmatter,
+  agentIndexSuffix,
+  caveatStatus,
+  readGovernance,
+  resolveSuccessorUrl,
+} from "./governance";
+import { appName, showGovernance } from "./shared";
+import { renderCaveatBadge } from "@/components/sidebar-status";
 import { orderValue } from "./order-rule";
 import { sortNodes } from "./page-order";
 
@@ -10,7 +20,20 @@ import { sortNodes } from "./page-order";
 export const source = loader({
   baseUrl: "/docs",
   source: docs.toFumadocsSource(),
-  plugins: [lucideIconsPlugin()],
+  plugins: [
+    lucideIconsPlugin(),
+    // The shell's own status plugin, which reads `status` from a document's
+    // frontmatter and puts it on the tree node. This used to be a map of
+    // statuses by url and a second walk over the tree that rewrote each row's
+    // `name` — the plugin does the walk, so the record's own key reaches the
+    // sidebar without us restating it.
+    //
+    // `renderBadge` returns null for anything that is not a caveat, which is
+    // the one rule that is OURS: a reader already assumes a document in the
+    // record is current, so `approved` shows nothing and the marker stays rare
+    // enough to be noticed where it matters.
+    statusBadgesPlugin({ renderBadge: (status) => renderCaveatBadge(status) }),
+  ],
 });
 
 export type KnowledgePage = (typeof source)["$inferPage"];
@@ -35,7 +58,13 @@ function orderOf(page: KnowledgePage): number {
  * state and mutating it would survive a hot reload.
  */
 export function getSortedPageTree(): Root {
-  const orders = new Map(source.getPages().map((page) => [page.url, orderOf(page)] as const));
+  const pages = source.getPages();
+  const orders = new Map(pages.map((page) => [page.url, orderOf(page)] as const));
+  // The caveat status already rides the row: `statusBadgesPlugin` above put it
+  // there while the loader built the tree, so the reader sees it before the
+  // click rather than after (research/site-design.md F3). This function is
+  // left with the one thing the shell has no opinion about — the record's
+  // governed `order:`.
   const tree = source.getPageTree();
   return { ...tree, children: sortNodes(tree.children, orders, 0) };
 }
@@ -71,10 +100,197 @@ export function getSortedPages(): KnowledgePage[] {
   return [...ordered, ...remaining.values()];
 }
 
-export async function getLLMText(page: KnowledgePage): Promise<string> {
+/**
+ * One document as the full-corpus file carries it: heading, then the record's
+ * own governance as frontmatter, then the body.
+ *
+ * The frontmatter is the point. Without it this file served a superseded
+ * document as clean prose, so a consumer ingesting the corpus answered from a
+ * withdrawn policy with nothing in the bytes to say so (research/site-design.md
+ * F1). `pages` resolves a successor pointer to the route a consumer can
+ * actually fetch.
+ */
+export async function getLLMText(
+  page: KnowledgePage,
+  pages: readonly KnowledgePage[] = [],
+): Promise<string> {
   const processed = await page.data.getText("processed");
+  const governance = readGovernance(page.data, page.path);
+  const successor =
+    governance.supersededBy === null
+      ? null
+      : resolveSuccessorUrl(governance.supersededBy, page.path, pages);
+  const front = agentFrontmatter(governance, successor === null ? null : basePath + successor);
+
+  // found live 2026-08-21: the processed markdown arrives with its own leading
+  // blank lines, so every block opened with three of them — and adding the
+  // frontmatter above made it four. One blank line between each part, always.
+  return [`# ${page.data.title} (${basePath}${page.url})`, front.trimEnd(), processed.trimStart()]
+    .filter((part) => part !== "")
+    .join("\n\n");
+}
+
+/** One entry in a record listing — everything a reader needs to choose. */
+export interface RecordEntry {
+  readonly url: string;
+  readonly title: string;
+  readonly description: string | null;
+  /** The document's status when it is a caveat, else null. */
+  readonly status: string | null;
+  /**
+   * Who stands behind it. Null when the record declares no owner — and null
+   * for every document when `site.governance` is off, because an owner is a
+   * governance fact and that key turns the pages plain.
+   */
+  readonly owner: string | null;
+  /** How many documents this entry holds below it; 0 for a leaf. */
+  readonly documents: number;
+}
+
+/**
+ * The record's entry for one page — what any listing needs.
+ *
+ * Exported because the front door leads with the document `Open the record`
+ * opens, which is the first page in governed order and may sit BELOW the top
+ * level, where `entriesUnder(null)` would never return it.
+ */
+export function entryFor(page: KnowledgePage): RecordEntry {
+  const data: Record<string, unknown> = page.data as unknown as Record<string, unknown>;
+  const description = typeof data["description"] === "string" ? data["description"].trim() : "";
+  const status = typeof data["status"] === "string" ? data["status"].trim() : "";
+  return {
+    url: page.url,
+    title: page.data.title,
+    description: description === "" ? null : description,
+    status: caveatStatus(status === "" ? null : status),
+    owner: showGovernance ? readGovernance(page.data, page.path).owner : null,
+    documents: 0,
+  };
+}
+
+/**
+ * How many documents a folder holds, its own index page excluded — the index
+ * IS the entry being counted, not something below it.
+ *
+ * By url in a set, not by adding lengths: whether a folder's index also
+ * appears among its children is the loader's business, and counting it twice
+ * would publish a number the record cannot support.
+ */
+function countDocuments(folder: Extract<Node, { type: "folder" }>): number {
+  const urls = new Set<string>();
+  const walk = (nodes: readonly Node[]): void => {
+    for (const node of nodes) {
+      if (node.type === "page") urls.add(node.url);
+      else if (node.type === "folder") {
+        if (node.index) urls.add(node.index.url);
+        walk(node.children);
+      }
+    }
+  };
+  walk(folder.children);
+  if (folder.index) urls.delete(folder.index.url);
+  return urls.size;
+}
+
+/**
+ * The entries directly below a node of the record, in the governed reading
+ * order — or the top level when `url` is null.
+ *
+ * A folder's own index page is not listed under itself: it IS the page doing
+ * the listing. Without this, `/docs/policies` opened with a card pointing back
+ * at `/docs/policies`.
+ */
+export function entriesUnder(url: string | null): RecordEntry[] {
+  const byUrl = new Map(getSortedPages().map((page) => [page.url, page] as const));
+  const nodes = url === null ? getSortedPageTree().children : childrenOfFolder(url);
+  const entries: RecordEntry[] = [];
+  for (const node of nodes) {
+    const target =
+      node.type === "page" ? node.url : node.type === "folder" ? node.index?.url : null;
+    if (target === undefined || target === null || target === url) continue;
+    const page = byUrl.get(target);
+    if (page === undefined) continue;
+    entries.push(
+      node.type === "folder"
+        ? { ...entryFor(page), documents: countDocuments(node) }
+        : entryFor(page),
+    );
+  }
+  return entries;
+}
 
-  return `# ${page.data.title} (${basePath}${page.url})
+/**
+ * The record as `llms.txt` serves it: the instance name, then every document in
+ * the governed reading order, each carrying its governance when the governance
+ * is a caveat.
+ *
+ * Here rather than in the route, because the home page shows these same bytes
+ * to a reader. Two spellings of the record's index would be two indexes, and
+ * the one on the page would be the one nobody checked.
+ */
+export function recordIndexText(): string {
+  const pages = getSortedPages();
+  const lines = pages.map((page) => {
+    const governance = readGovernance(page.data, page.path);
+    const successor =
+      governance.supersededBy === null
+        ? null
+        : resolveSuccessorUrl(governance.supersededBy, page.path, pages);
+    const link = `- [${page.data.title}](${basePath}${page.url})`;
+    const described = page.data.description ? `${link}: ${page.data.description}` : link;
+    // The successor's route is prefixed like every other URL here, so the line
+    // is usable as-is on a sub-path host.
+    return (
+      described + agentIndexSuffix(governance, successor === null ? null : basePath + successor)
+    );
+  });
+  return `# ${appName}\n\n${lines.join("\n")}\n`;
+}
 
-${processed}`;
+/**
+ * The route of a document's markdown twin — `/docs/policies/terms` becomes
+ * `/md/policies/terms.md`, and the record's own index becomes `/md/index.md`.
+ *
+ * One rule, one place: the docs page derives the same address from its route
+ * params for `rel="alternate"`, and a second spelling of it here would be a
+ * broken link the day either changes.
+ */
+export function markdownPath(url: string): string {
+  const slug = url.replace(/^\/docs\/?/, "").replace(/\/$/, "");
+  return `${basePath}/md/${slug === "" ? "index" : slug}.md`;
+}
+
+/** The children of the folder whose index page is at `url`, or []. */
+function childrenOfFolder(url: string): Node[] {
+  const find = (nodes: readonly Node[]): Node[] | null => {
+    for (const node of nodes) {
+      if (node.type !== "folder") continue;
+      if (node.index?.url === url) return [...node.children];
+      const deeper = find(node.children);
+      if (deeper !== null) return deeper;
+    }
+    return null;
+  };
+  return find(getSortedPageTree().children) ?? [];
+}
+
+/**
+ * Every document's caveat status, keyed by route — the small map the search
+ * dialog needs on the client.
+ *
+ * Search was the last surface where a withdrawn document and the one that
+ * replaced it looked identical, and its snippet quotes the withdrawn figure
+ * (research/site-design.md F3). The dialog runs in the browser over a static
+ * index that has no field for status, so the map travels to it as a prop
+ * instead: a few dozen bytes per caveat document, and nothing at all for a
+ * record whose documents are all approved.
+ */
+export function caveatStatusByUrl(): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const page of source.getPages()) {
+    const raw: unknown = (page.data as unknown as Record<string, unknown>)["status"];
+    const status = caveatStatus(typeof raw === "string" && raw.trim() !== "" ? raw.trim() : null);
+    if (status !== null) out[page.url] = status;
+  }
+  return out;
 }

package/templates/scaffold/system/site/lib/stage-knowledge.ts

@@ -1,11 +1,13 @@
 import {
   copyFileSync,
+  existsSync,
   mkdirSync,
   readFileSync,
   readdirSync,
   rmSync,
   statSync,
   watch,
+  writeFileSync,
 } from "node:fs";
 import path from "node:path";
 
@@ -343,47 +345,191 @@ function planStage(recordDir: string, denied: DenylistManifest): StagePlan {
   return { files: [...documents, ...assets], documents: documents.length, total };
 }
 
+/** How often a waiter looks again. */
+const LOCK_POLL_MS = 25;
+/** How long a wait goes unexplained. A build that looks hung must say why. */
+const LOCK_ANNOUNCE_MS = 10_000;
+
+/** Synchronous, because everything on this path is: a bundler cannot await. */
+function sleepSync(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function isAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM is a process that exists and is not ours to signal.
+    return (error as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+/**
+ * Is this lock abandoned — stamped with a process that no longer exists?
+ *
+ * Blank is the one ambiguous read: the holder writes its pid in the same call
+ * that creates the file, so a blank lock is either a holder caught between the
+ * two (microseconds) or one that died there (forever). Looking twice tells
+ * them apart, and only the second look may break a lock.
+ */
+function lockIsAbandoned(lockFile: string): boolean {
+  for (const look of [0, 1]) {
+    let stamp: string;
+    try {
+      stamp = readFileSync(lockFile, "utf8").trim();
+    } catch {
+      // Released while we read it; the next acquire attempt takes it.
+      return false;
+    }
+    const pid = Number(stamp);
+    if (Number.isInteger(pid) && pid > 0) return !isAlive(pid);
+    if (look === 0) sleepSync(LOCK_POLL_MS * 2);
+  }
+  return true;
+}
+
+/**
+ * Hold the stage lock for the duration of `work`: ONE evaluation writes the
+ * stage at a time, and this file says which.
+ *
+ * A build evaluates `source.config.ts` in more than one process — SEVEN of
+ * them staged the record in one measured `next build` of a scaffolded site
+ * (2026-08-23) — and staging was destructive on every evaluation: delete the
+ * whole stage, refill it. Two of those overlapping is not a rare interleaving,
+ * it is what seven of them do — six concurrent evaluations of a 150-document
+ * record failed 42 of 48 runs, in four shapes: `ENOENT` and `EINVAL` out of `copyFileSync` (the reported one,
+ * issue #100), `ENOTEMPTY` out of `rmSync` *with* its retries already in
+ * place, and — 27 of the 48, the majority — no error at all: staging returned
+ * success and handed the build a stage a third of the record short.
+ *
+ * The silent shape is why this is a lock and not another retry. A crash fails
+ * a build; a short stage PUBLISHES one, with documents missing from /docs,
+ * llms.txt and the search index, and nothing anywhere saying so.
+ *
+ * `wx` is the whole primitive: create-if-absent, atomically, on every
+ * filesystem Node supports — and it stamps the holder's pid in the same call,
+ * so a waiter can tell a live holder from a killed one.
+ *
+ * Waiting on a LIVE holder is unbounded on purpose: it is another evaluation
+ * of the same build, staging the same bytes from the same record, and this
+ * build is not finished until it has. Unbounded is not silent, though — a wait
+ * long enough to look like a hang names what it is waiting for.
+ */
+function withStageLock<T>(stageDir: string, work: () => T): T {
+  const lockFile = `${stageDir}.lock`;
+  let waited = 0;
+  let announced = false;
+  for (;;) {
+    try {
+      writeFileSync(lockFile, String(process.pid), { flag: "wx" });
+      break;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      if (lockIsAbandoned(lockFile)) {
+        rmSync(lockFile, { force: true });
+        continue;
+      }
+      sleepSync(LOCK_POLL_MS);
+      waited += LOCK_POLL_MS;
+      if (waited >= LOCK_ANNOUNCE_MS && !announced) {
+        announced = true;
+        console.warn(
+          `[ksor] waiting on ${path.basename(lockFile)} — another evaluation of this build is ` +
+            "staging the record. Delete that file if no build is running.",
+        );
+      }
+    }
+  }
+  try {
+    return work();
+  } finally {
+    rmSync(lockFile, { force: true });
+  }
+}
+
 /**
  * Remove the stage, asking for the retries this exact failure needs.
  *
- * `force: true` suppresses ENOENT; it does NOT retry anything. Node retries
- * EBUSY / EMFILE / ENFILE / ENOTEMPTY / EPERM only when `maxRetries` is set,
- * and it defaults to zero. The build evaluates `source.config.ts` more than
- * once when the bundler wants it in more than one place, so two runs can
- * overlap: one removing the stage while the other is still copying into it.
- * That surfaced as `ENOTEMPTY` out of `rmSync` and failed the whole site build
- * (CI, 2026-08-21) — a race that is safe to lose, because the stage is a
- * deterministic function of the record and the denylist, so redoing it produces
- * the same bytes.
+ * Callers hold the stage lock, so no OTHER evaluation is writing here — but
+ * `force: true` suppresses ENOENT and does NOT retry anything, and Node
+ * retries EBUSY / EMFILE / ENFILE / ENOTEMPTY / EPERM only when `maxRetries`
+ * is set (it defaults to zero). Those are what a Windows indexer or an
+ * antivirus scanner holding a handle looks like — not ksor, and not something
+ * the lock can serialise. Losing that race is safe: the stage is a
+ * deterministic function of the record and the denylist, so redoing it
+ * produces the same bytes.
  */
 function removeStage(stageDir: string): void {
   rmSync(stageDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
 }
 
-/** Fill a clean stage with exactly the set this build may publish. */
-function fillStage(recordDir: string, stageDir: string, denied: DenylistManifest): void {
-  // The old stage goes first, before any refusal can throw: a refused build
-  // that leaves the previous, more permissive stage on disk hands the next
-  // careless build a filtered copy nothing governs (review finding,
-  // 2026-08-19).
-  removeStage(stageDir);
-  const plan = planStage(recordDir, denied);
-  // An empty record is its own problem, reported by the page that renders it;
-  // an empty AUDIENCE is a misconfiguration that would otherwise surface as
-  // "the record has no documents" against a record full of them.
-  if (plan.documents === 0 && plan.total > 0) {
-    refuse(
-      "ksor-audience-empty",
-      `no document in the record is visible to the ${buildAudience} build (${plan.total} document${plan.total === 1 ? "" : "s"}, all above that tier)`,
-      "a site with nothing on it is a deploy that looks successful and serves nobody — and the record is not empty, this audience's slice of it is",
-      "build a wider audience with KSOR_AUDIENCE, lower default_visibility in instance.md, or give at least one document this tier",
-    );
+/**
+ * Does the stage already hold EXACTLY this plan, byte for byte?
+ *
+ * The wipe-and-refill is the destructive half of staging, and it is pure waste
+ * whenever the answer is yes — which is every evaluation after the first in
+ * one build, since the plan is a deterministic function of the record and the
+ * denylist. Skipping it is not an optimisation: while a wipe is running there
+ * is a window in which the stage is not the record, and an evaluation that has
+ * already returned is reading it. The lock stops two writers colliding; this
+ * stops the second writer existing at all.
+ *
+ * Bytes, not names and not timestamps: the alternative is serving a previous
+ * build's copy of a document that has since been edited.
+ */
+function stageHolds(recordDir: string, stageDir: string, plan: StagePlan): boolean {
+  let staged: string[];
+  try {
+    staged = walkFiles(stageDir);
+  } catch {
+    return false;
   }
-  for (const from of plan.files) {
-    const to = path.join(stageDir, path.relative(recordDir, from));
-    mkdirSync(path.dirname(to), { recursive: true });
-    copyFileSync(from, to);
+  if (staged.length !== plan.files.length) return false;
+  const expected = new Map(
+    plan.files.map((from) => [path.join(stageDir, path.relative(recordDir, from)), from]),
+  );
+  for (const file of staged) {
+    const from = expected.get(file);
+    if (from === undefined) return false;
+    if (!readFileSync(from).equals(readFileSync(file))) return false;
   }
+  return true;
+}
+
+/** Fill a clean stage with exactly the set this build may publish. */
+function fillStage(recordDir: string, stageDir: string, denied: DenylistManifest): void {
+  withStageLock(stageDir, () => {
+    let plan: StagePlan;
+    try {
+      plan = planStage(recordDir, denied);
+      // An empty record is its own problem, reported by the page that renders
+      // it; an empty AUDIENCE is a misconfiguration that would otherwise
+      // surface as "the record has no documents" against a record full of them.
+      if (plan.documents === 0 && plan.total > 0) {
+        refuse(
+          "ksor-audience-empty",
+          `no document in the record is visible to the ${buildAudience} build (${plan.total} document${plan.total === 1 ? "" : "s"}, all above that tier)`,
+          "a site with nothing on it is a deploy that looks successful and serves nobody — and the record is not empty, this audience's slice of it is",
+          "build a wider audience with KSOR_AUDIENCE, lower default_visibility in instance.md, or give at least one document this tier",
+        );
+      }
+    } catch (error) {
+      // No refusal may leave the previous, more permissive stage on disk: it
+      // hands the next careless build a filtered copy nothing governs (review
+      // finding, 2026-08-19). The removal used to lead this function, which is
+      // why nothing could ask whether the stage was already correct.
+      removeStage(stageDir);
+      throw error;
+    }
+    if (stageHolds(recordDir, stageDir, plan)) return;
+    removeStage(stageDir);
+    for (const from of plan.files) {
+      const to = path.join(stageDir, path.relative(recordDir, from));
+      mkdirSync(path.dirname(to), { recursive: true });
+      copyFileSync(from, to);
+    }
+  });
 }
 
 /**
@@ -421,13 +567,17 @@ function refuseVisibilityWithoutAudiences(recordDir: string): void {
  * published build is always staged from scratch.
  */
 function refreshStage(recordDir: string, stageDir: string, denied: DenylistManifest): void {
-  const permitted = new Set(planStage(recordDir, denied).files);
-  for (const staged of walkFiles(stageDir)) {
-    const from = path.join(recordDir, path.relative(stageDir, staged));
-    if (!permitted.has(from)) continue;
-    if (readFileSync(from).equals(readFileSync(staged))) continue;
-    copyFileSync(from, staged);
-  }
+  // Under the lock like every other write here: a save landing while another
+  // evaluation is refilling the stage is the same race from the other side.
+  withStageLock(stageDir, () => {
+    const permitted = new Set(planStage(recordDir, denied).files);
+    for (const staged of walkFiles(stageDir)) {
+      const from = path.join(recordDir, path.relative(stageDir, staged));
+      if (!permitted.has(from)) continue;
+      if (readFileSync(from).equals(readFileSync(staged))) continue;
+      copyFileSync(from, staged);
+    }
+  });
 }
 
 let watching = false;
@@ -484,8 +634,11 @@ export function knowledgeSourceDir(): string {
     // Nothing to filter — serve the record itself, the level-0 fast path.
     // A stage left behind by an earlier model would be a filtered copy of the
     // record nothing governs any more — removed before the refusal below can
-    // throw, so a refused build never leaves one behind either.
-    removeStage(stageDir);
+    // throw, so a refused build never leaves one behind either. Under the lock,
+    // because two evaluations removing one tree is the `ENOTEMPTY` shape of the
+    // same race; the existence check keeps a record that never stages from
+    // taking a lock on every build.
+    if (existsSync(stageDir)) withStageLock(stageDir, () => removeStage(stageDir));
     refuseVisibilityWithoutAudiences(recordDir);
     return RECORD_DIR;
   }

package/templates/scaffold/system/site/lib/utils.ts

@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx";
+import { twMerge } from "tailwind-merge";
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs));
+}

package/templates/scaffold/system/site/package.json

@@ -2,18 +2,24 @@
   "name": "site",
   "version": "0.0.0",
   "private": true,
+  "type": "module",
   "scripts": {
     "build": "next build",
     "dev": "next dev"
   },
   "dependencies": {
-    "fumadocs-core": "16.10.3",
-    "fumadocs-mdx": "15.0.13",
-    "fumadocs-ui": "16.10.3",
+    "class-variance-authority": "0.7.1",
+    "clsx": "2.1.1",
+    "fumadocs-core": "16.14.5",
+    "fumadocs-mdx": "15.3.0",
+    "fumadocs-ui": "16.14.5",
     "lucide-react": "1.31.0",
     "next": "16.2.9",
+    "radix-ui": "1.6.7",
     "react": "19.2.8",
     "react-dom": "19.2.8",
+    "tailwind-merge": "3.6.0",
+    "tw-animate-css": "1.4.0",
     "zod": "4.4.3"
   },
   "devDependencies": {

package/templates/scaffold/knowledge/example.md

@@ -1,23 +0,0 @@
----
-title: Your first governed document
-status: draft
-order: 1
----
-
-This file exists so the record is never empty: browse it with `pnpm dev`,
-then replace it with real knowledge.
-
-A governed document is plain markdown with a small frontmatter header. This
-one carries the two required keys — `title` and `status: draft`. As the
-knowledge matures, documents gain `owner` and `provenance` (who stands behind
-this, and which sources it came from), move to `status: approved`, and — when
-replaced — are marked `superseded`, never deleted.
-
-It also carries `order: 1`, which is how the record decides reading order: a
-document that declares `order` sorts ahead of every document that does not, so
-this one stays first in the sidebar, first in `llms.txt`, and the document the
-home page opens.
-
-Ask your coding agent to run the **intake interview** to define what this
-Knowledge System of Record is authoritative for, then start adding documents
-with the **add-sources** skill. `pnpm check` keeps every document honest.