@intentius/chant 0.44.14 → 0.46.0

package/src/cli/handlers/search.ts

@@ -1,6 +1,6 @@
 import { resolve } from "node:path";
 import { build } from "../../build";
-import { buildGraphIr, buildLiveGraphIr, sourceOverlayGraphs, type GraphIR, type IRNode, type IREdge } from "../../graph-ir";
+import { buildGraphIr, buildLiveGraphIr, collectUnobserved, sourceOverlayGraphs, type GraphIR, type IRNode, type IREdge } from "../../graph-ir";
 import { buildDeclaredPerStack } from "../../graph-declared";
 import { enrichEffectiveTopology } from "../../graph-effective";
 import { reconstructEdges, mergeCatalogs, type ReferenceCatalog } from "../../graph-refs";
@@ -9,7 +9,7 @@ import { discover } from "../../discovery/index";
 import { observeResources } from "../../lifecycle/observe";
 import { replaySnapshots, hasSnapshot } from "../../lifecycle/replay";
 import type { LiveObservation } from "../../graph-ir";
-import { loadChantConfig } from "../../config";
+import { loadChantConfig, matchesDeclaredEnvironment } from "../../config";
 import { loadPlugins, resolveProjectLexicons } from "../plugins";
 import { formatError, formatWarning } from "../format";
 import type { CommandContext } from "../registry";
@@ -61,6 +61,13 @@ export async function runSearch(ctx: CommandContext): Promise<number> {
 
   let ir: GraphIR;
   let source: AnswerSource = { kind: "declared" };
+  // Lexicons whose live read threw (#1263). "Nothing observed" and "could not
+  // observe" are different claims; this is what carries the second one.
+  const liveFailures: string[] = [];
+  // Run-level notes from the read (#1265) — "ownership could not be filtered
+  // on this path" — printed with the provenance footer, after the rows, so the
+  // answer is not preceded by what qualifies it.
+  let liveNotes: string[] = [];
   // Kinds that can exist in the account without being declared (#1278). Known
   // without a scan, so it costs nothing to mention.
   let ambientKinds: string[] = [];
@@ -82,7 +89,10 @@ export async function runSearch(ctx: CommandContext): Promise<number> {
       }));
       return 1;
     }
-    if (config.environments && !config.environments.includes(environment)) {
+    // Membership via matchesDeclaredEnvironment (#1221): handles `{ name,
+    // endpoint }` entries (#1166) — a plain `.includes` never matched those —
+    // and glob-pattern entries like `"pr-*"`.
+    if (config.environments && config.environments.length > 0 && !matchesDeclaredEnvironment(config.environments, environment)) {
       console.error(formatError({ message: `Unknown environment "${environment}"` }));
       return 1;
     }
@@ -122,8 +132,15 @@ export async function runSearch(ctx: CommandContext): Promise<number> {
         stacks,
         ambient: args.ambient === true,
       });
-      for (const e of observed.errors) console.error(formatWarning({ message: e }));
+      // A thrown read is not a warning. Printed as one, it sat between the
+      // ownership-filter notices that also print on a working run and carried
+      // no signal (#1263). Name the lexicon and the cause, as an error.
+      for (const e of observed.errors) {
+        liveFailures.push(e);
+        console.error(formatError({ message: `live read failed — ${e}` }));
+      }
       observations = observed.observations;
+      liveNotes = observed.notes ?? [];
       source = { kind: "live" };
     }
     let live = buildLiveGraphIr(observations);
@@ -189,7 +206,10 @@ export async function runSearch(ctx: CommandContext): Promise<number> {
       stacks.length > 0
         ? await buildDeclaredPerStack(stacks, projectPath)
         : buildGraphIr((await discover(resolve(args.src ?? config.sourceDir ?? "."))).entities, projectPath);
-    ir = sourceOverlayGraphs(declared, live);
+    // Carry the NOT-OBSERVED half of the tri-state (#1089) onto the rows, so a
+    // declared entity nobody could read is painted `_unobserved` and a row can
+    // say so instead of printing blank where a physical id would go (#1263).
+    ir = sourceOverlayGraphs(declared, live, { unobserved: collectUnobserved(observations) });
     // Containment goes on AFTER the overlay, not through it.
     //
     // `sourceOverlayGraphs` admits a live edge only when one end is foreign,
@@ -242,6 +262,7 @@ export async function runSearch(ctx: CommandContext): Promise<number> {
     console.log("(no matches)");
     availableAttrs(terms, ir);
     if (args.explain) explain(terms, matches, ir, nodeById, query);
+    for (const n of liveNotes) console.error(formatWarning({ message: n }));
     return 0;
   }
   for (const n of matches) {
@@ -254,12 +275,28 @@ export async function runSearch(ctx: CommandContext): Promise<number> {
     source.kind === "live" && !matches.some((n) => n.physicalId) && args.env
       ? (await hasSnapshot(String(args.env))) ? "yes" : undefined
       : undefined;
-  provenance(matches, source, recorded);
+  provenance(matches, source, recorded, liveFailures);
+  // Qualifies the provenance line, so it sits with it: one line per distinct
+  // note for the whole run, not one per stack, and after the rows (#1265).
+  for (const n of liveNotes) console.error(formatWarning({ message: n }));
   ambientHint(matches, ambientKinds, args.ambient === true, replayAmbient);
   showMiss(matches, show);
   regionSpread(terms, matches, show);
   derivedSurface(terms, matches, ir, backed);
   if (args.explain) explain(terms, matches, ir, nodeById, query);
+  // The caller asked for a live answer and at least one lexicon could not be
+  // read. The rows above are still printed — they are what the source declares,
+  // labelled as such — but the command did not do what it was asked, and a
+  // script or agent needs to see that without parsing the footer (#1263).
+  if (liveFailures.length > 0) {
+    console.error(formatError({
+      message: `live read failed for ${liveFailures.length} lexicon${liveFailures.length === 1 ? "" : "s"} — answer is declared-only`,
+      hint: recorded
+        ? "answer from the recorded snapshot with --at latest, or drop --live for a declared-only query"
+        : "drop --live for a declared-only query, or record a snapshot with chant lifecycle snapshot",
+    }));
+    return 1;
+  }
   return 0;
 }
 
@@ -379,13 +416,24 @@ function ambientHint(
  * already done. That is a fact about the query, printed for every query, and it
  * encodes no expected answer.
  */
-function provenance(matches: IRNode[], source: AnswerSource, recorded?: string): void {
+function provenance(matches: IRNode[], source: AnswerSource, recorded?: string, liveFailures: string[] = []): void {
   if (source.kind === "declared") {
     console.log("— declared only · no observation · physical ids unavailable");
     return;
   }
   const bound = matches.filter((n) => n.physicalId).length;
   const what = source.kind === "live" ? "live read" : "snapshot";
+  if (liveFailures.length > 0) {
+    // Could not observe, as distinct from observed nothing. The rows carry the
+    // per-entity verdict; this names the lexicons that failed in one line.
+    const lexicons = liveFailures.map((e) => e.split(":")[0]).join(", ");
+    const unobserved = matches.filter((n) => (n.attrs as Record<string, unknown> | undefined)?._unobserved).length;
+    const rest = recorded
+      ? "a snapshot of this environment is recorded — answer from it with --at latest"
+      : "answered from the declared graph · physical ids unavailable";
+    console.log(`— live read failed (${lexicons}) · ${unobserved}/${matches.length} rows unobserved · ${rest}`);
+    return;
+  }
   if (bound === 0) {
     // The estate was asked for and nothing came back bound. Naming it is the
     // difference between "these do not exist" and "nobody could see them".
@@ -742,6 +790,10 @@ function formatRow(n: IRNode, show: string[]): string {
   // skip source-mode AttrRef placeholders (objects).
   const physical = (n as { physicalId?: unknown }).physicalId ?? attrs["physicalId"] ?? attrs["InstanceId"] ?? attrs["Id"];
   if (physical != null && typeof physical !== "object") parts.push(String(physical));
+  // A row nobody could read says so where its physical id would go (#1263). A
+  // blank there reads as "declared, not provisioned", which the read never
+  // established; the reason is the tri-state's (#1089).
+  else if (typeof attrs["_unobserved"] === "string") parts.push(`(unobserved: ${attrs["_unobserved"]})`);
   for (const key of show) {
     // Match the name case-insensitively, and report what was actually found.
     // AWS attribute names are PascalCase and chant's derived ones are not, so a
@@ -763,4 +815,4 @@ function formatRow(n: IRNode, show: string[]): string {
 }
 
 /** Internals exposed for unit tests. */
-export const __searchInternals = { parseQuery, matchTerm, formatRow, explain, describeTerm, derivedSurface, availableAttrs, ambientHint, regionSpread, showMiss };
+export const __searchInternals = { parseQuery, matchTerm, formatRow, explain, describeTerm, derivedSurface, availableAttrs, ambientHint, regionSpread, showMiss, provenance };

package/src/cli/main.ts

@@ -23,7 +23,7 @@ import { runCarveAdvise, runCarveUnknown } from "./handlers/carve";
 import { runCarveEmit } from "./handlers/carve-emit";
 import { runCarveBridge } from "./handlers/carve-bridge";
 import { runCarveApply } from "./handlers/carve-apply";
-import { runLifecycleSnapshot, runLifecycleShow, runLifecycleDiff, runLifecycleRollback, runLifecyclePlan, runLifecycleAffected, runLifecycleLog, runLifecycleUnknown } from "./handlers/lifecycle";
+import { runLifecycleSnapshot, runLifecycleShow, runLifecycleDiff, runLifecycleRollback, runLifecyclePlan, runLifecycleAffected, runLifecycleLog, runLifecycleTeardown, runLifecycleUnknown } from "./handlers/lifecycle";
 import { runComponentsStatus, runComponentsReleaseRecord, runComponentsUnknown } from "./handlers/components";
 import { runGraph } from "./handlers/graph";
 import { runExplain } from "./handlers/explain";
@@ -78,6 +78,8 @@ const BOOLEAN_FLAGS = new Set([
   "--fold",
   "--no-fold",
   "--sandbox",
+  "--yes",
+  "--confirm-prod",
 ]);
 
 /**
@@ -228,6 +230,10 @@ export function parseArgs(args: string[]): ParsedArgs {
       result.emit = args[++i];
     } else if (arg === "--dry-run") {
       result.dryRun = true;
+    } else if (arg === "--yes") {
+      result.yes = true;
+    } else if (arg === "--confirm-prod") {
+      result.confirmProd = true;
     } else if (arg === "--strict") {
       result.strict = true;
     } else if (arg === "--validate") {
@@ -484,6 +490,10 @@ Lifecycle (alias: lc):
   lifecycle plan <env>      Typed change set (create/update/delete/adopt) vs live
   lifecycle affected        Stacks a change affects (--base <ref> [--include-dependents])
                             --json: emit the ChangeSet as JSON
+  lifecycle teardown <env>  Plan what deleting the environment would remove —
+                            marker-scoped (this project's stack + env); --yes
+                            executes the plan (production-like names also need
+                            --confirm-prod, or an interactive confirmation)
   lifecycle log [env]       History of lifecycle snapshots
 
 Component release ledger + status:
@@ -542,7 +552,9 @@ Options:
   --fix                 Auto-fix fixable issues (lint command)
   --force               Force overwrite existing files (import command)
   -w, --watch           Watch for changes and rebuild/re-lint (build, lint)
-  -v, --verbose         Show stack traces on errors
+  -v, --verbose         Show stack traces on errors; (build) list every
+                        resolved build parameter and per-file fold decision
+                        instead of the one-line summaries
   -h, --help            Show this help message
   -p, --profile <name>  Temporal worker profile to use (run command)
   --local               Run an Op with the local in-process executor (default)
@@ -572,8 +584,8 @@ Options:
                         composite factory calls (#1022/#1023), falling back
                         to run per-file for anything else outside the fold
                         subset (a cross-file-only reference, a re-export,
-                        \`export default\`, ...). Logs which path each file
-                        took. DEFAULT since #1134 — this flag forces it on
+                        \`export default\`, ...). Logs a fold/run count
+                        (per-file lines under --verbose). DEFAULT since #1134 — this flag forces it on
                         over a chant.config.ts \`build.fold: false\`.
   --no-fold             (build) Opt out of folding for this invocation: every
                         source module is imported and run, the pre-#1134
@@ -778,6 +790,7 @@ const registry: CommandDef[] = [
   { name: "lifecycle rollback", handler: runLifecycleRollback },
   { name: "lifecycle plan", requiresPlugins: true, handler: runLifecyclePlan },
   { name: "lifecycle affected", requiresPlugins: true, handler: runLifecycleAffected },
+  { name: "lifecycle teardown", requiresPlugins: true, handler: runLifecycleTeardown },
   { name: "lifecycle log", handler: runLifecycleLog },
 
   // Component release ledger + status surface (#568, epic #551)

package/src/cli/registry.ts

@@ -85,6 +85,13 @@ export interface ParsedArgs {
   namespace?: string;
   /** `chant lifecycle rollback --dry-run` — compute the rollback delta and print it; open no PR, push nothing, leave no branch. */
   dryRun?: boolean;
+  /** `chant lifecycle teardown <env> --yes` — execute the planned deletion
+   * (#1222). Without it the command plans and stops. */
+  yes?: boolean;
+  /** `chant lifecycle teardown <env> --yes --confirm-prod` — the non-interactive
+   * form of the extra confirmation a production-like environment name demands
+   * (#1222). Meaningless without `--yes`. */
+  confirmProd?: boolean;
   /** `chant import --verbatim` — keep server-defaulted fields in live import */
   verbatim?: boolean;
   /** `chant lifecycle … --src <dir>` — build root override for lifecycle commands */

package/src/cli/security-boundary.test.ts

@@ -99,7 +99,8 @@ describe("CLI end-to-end — resolveProjectLexicons must not execute project sou
   }
 
   test("`chant build --fold` on a project with no `lexicons` config does not execute the hostile file", () => {
-    const { status, stdout, stderr } = runCli(["build", testDir, "--fold"]);
+    // --verbose: the per-file fold decision lines are behind it (#1424).
+    const { status, stdout, stderr } = runCli(["build", testDir, "--fold", "--verbose"]);
 
     expect(stderr).not.toMatch(/EXFIL/);
     expect(stdout).not.toMatch(/EXFIL/);

package/src/codegen/docs-pages.test.ts

@@ -0,0 +1,194 @@
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync, readFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { afterEach, beforeEach, describe, expect, test } from "vitest";
+
+import { readAuthoredPages } from "./docs-pages";
+import { docsPipeline, writeDocsSite, GENERATED_MARKER_TAG } from "./docs";
+import { buildSidebar, quadrantItems } from "./docs-sidebar";
+import type { DocsConfig, DocsResult, SidebarPage } from "./docs-types";
+
+let root: string;
+
+function config(overrides: Partial<DocsConfig> = {}): DocsConfig {
+  return {
+    name: "fixture",
+    displayName: "Fixture",
+    description: "fixture lexicon",
+    distDir: join(root, "dist"),
+    outDir: join(root, "docs"),
+    srcDir: join(root, "src"),
+    examplesDir: join(root, "examples"),
+    basePath: "/chant/lexicons/fixture/",
+    ...overrides,
+  };
+}
+
+function page(name: string, frontmatter: string, body = "Body.\n"): void {
+  mkdirSync(join(root, "docs", "pages"), { recursive: true });
+  writeFileSync(join(root, "docs", "pages", name), `---\n${frontmatter}\n---\n\n${body}`);
+}
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), "chant-docs-pages-"));
+  mkdirSync(join(root, "dist"), { recursive: true });
+  mkdirSync(join(root, "src"), { recursive: true });
+  mkdirSync(join(root, "examples"), { recursive: true });
+  writeFileSync(
+    join(root, "dist", "manifest.json"),
+    JSON.stringify({ name: "fixture", version: "0.0.0", intrinsics: [{ name: "ref", description: "a ref" }] }),
+  );
+  writeFileSync(join(root, "dist", "meta.json"), JSON.stringify({}));
+  writeFileSync(join(root, "examples", "snippet.ts"), "export const x = 1;\n");
+});
+
+afterEach(() => {
+  rmSync(root, { recursive: true, force: true });
+});
+
+describe("readAuthoredPages", () => {
+  test("returns nothing when docs/pages/ does not exist", () => {
+    expect(readAuthoredPages(config())).toEqual([]);
+  });
+
+  test("reads frontmatter, expands file markers, strips sidebar-only keys", () => {
+    page(
+      "getting-started.mdx",
+      'title: "Getting Started"\ndescription: "Start here"\ndiataxis: tutorial\norder: 1\ngroup: "Intro"\nlabel: "Start"\nhidden: false',
+      "Before\n\n{{file:snippet.ts}}\n\nAfter\n",
+    );
+    const [p] = readAuthoredPages(config());
+    expect(p.slug).toBe("getting-started");
+    expect(p.quadrant).toBe("tutorial");
+    expect(p.order).toBe(1);
+    expect(p.group).toBe("Intro");
+    expect(p.label).toBe("Start");
+    expect(p.hidden).toBe(false);
+    expect(p.content).toContain('```typescript title="snippet.ts"\nexport const x = 1;\n```');
+    expect(p.content).toContain("diataxis: tutorial");
+    expect(p.content).not.toMatch(/^(order|group|label|hidden):/m);
+    expect(p.content).toContain('title: "Getting Started"');
+  });
+
+  test("label defaults to title", () => {
+    page("x.mdx", "title: Plain Title\ndiataxis: reference");
+    expect(readAuthoredPages(config())[0].label).toBe("Plain Title");
+  });
+
+  test("missing diataxis throws with the file path", () => {
+    page("untagged.mdx", "title: Untagged");
+    expect(() => readAuthoredPages(config())).toThrow(/untagged\.mdx.*diataxis/);
+  });
+
+  test("unknown diataxis value throws", () => {
+    page("bad.mdx", "title: Bad\ndiataxis: guide");
+    expect(() => readAuthoredPages(config())).toThrow(/"guide" is not one of/);
+  });
+});
+
+describe("docsPipeline with authored pages", () => {
+  test("authored page wins a slug collision with a generated page", () => {
+    page("intrinsics.mdx", "title: My Intrinsics\ndiataxis: reference", "Hand-written.\n");
+    const result = docsPipeline(config());
+    expect(result.pages.get("intrinsics.mdx")).toContain("Hand-written.");
+    expect(result.sidebarPages.filter((p) => p.slug === "intrinsics")).toHaveLength(1);
+  });
+
+  test("generated reference pages land in the reference quadrant after authored ones", () => {
+    page("howto.mdx", "title: Do It\ndiataxis: how-to");
+    const result = docsPipeline(config());
+    const ref = result.sidebarPages.filter((p) => p.quadrant === "reference").map((p) => p.slug);
+    expect(ref).toEqual(["intrinsics", "serialization"]);
+    expect(result.sidebarPages.find((p) => p.slug === "howto")?.quadrant).toBe("how-to");
+  });
+
+  test("hidden pages are written but kept out of the sidebar", () => {
+    page("secret.mdx", "title: Secret\ndiataxis: reference\nhidden: true");
+    const result = docsPipeline(config());
+    expect(result.pages.has("secret.mdx")).toBe(true);
+    expect(result.sidebarPages.some((p) => p.slug === "secret")).toBe(false);
+  });
+
+  test("authored pages carry a provenance marker naming the source file", () => {
+    page("howto.mdx", "title: Do It\ndiataxis: how-to");
+    const result = docsPipeline(config());
+    const out = result.pages.get("howto.mdx") as string;
+    expect(out).toContain(GENERATED_MARKER_TAG);
+    expect(out).toContain("docs/pages/howto.mdx");
+  });
+
+  test("writeDocsSite reaps the written copy once the source is removed", () => {
+    page("howto.mdx", "title: Do It\ndiataxis: how-to");
+    const cfg = config();
+    writeDocsSite(cfg, docsPipeline(cfg));
+    const written = join(root, "docs", "src", "content", "docs", "howto.mdx");
+    expect(existsSync(written)).toBe(true);
+    rmSync(join(root, "docs", "pages", "howto.mdx"));
+    writeDocsSite(cfg, docsPipeline(cfg));
+    expect(existsSync(written)).toBe(false);
+    const astro = readFileSync(join(root, "docs", "astro.config.mjs"), "utf-8");
+    expect(astro).not.toContain("howto");
+    const schema = readFileSync(join(root, "docs", "src", "content.config.ts"), "utf-8");
+    expect(schema).toContain("diataxis");
+  });
+});
+
+describe("buildSidebar", () => {
+  const sb = (pages: SidebarPage[]) =>
+    buildSidebar(config(), { pages: new Map(), sidebarPages: pages, stats: { resources: 0, properties: 0, services: 0, rules: 0, intrinsics: 0 } } satisfies DocsResult);
+
+  test("groups in fixed quadrant order and omits empty quadrants", () => {
+    const items = sb([
+      { slug: "why", label: "Why", quadrant: "explanation" },
+      { slug: "start", label: "Start", quadrant: "tutorial" },
+      { slug: "rules", label: "All Rules", quadrant: "reference", order: 1000 },
+    ]);
+    expect(items.map((i) => i.label)).toEqual(["← chant docs", "Overview", "Tutorials", "Reference", "Explanation"]);
+  });
+
+  test("orders by order then label and nests groups after loose pages", () => {
+    const items = quadrantItems([
+      { slug: "b", label: "Bravo", quadrant: "reference" },
+      { slug: "a", label: "Alpha", quadrant: "reference" },
+      { slug: "z", label: "Zulu", quadrant: "reference", order: 0 },
+      { slug: "eks", label: "EKS", quadrant: "reference", group: "Vendor" },
+      { slug: "aks", label: "AKS", quadrant: "reference", group: "Vendor" },
+    ]);
+    expect(items).toEqual([
+      { label: "Zulu", slug: "z" },
+      { label: "Alpha", slug: "a" },
+      { label: "Bravo", slug: "b" },
+      { label: "Vendor", items: [{ label: "AKS", slug: "aks" }, { label: "EKS", slug: "eks" }] },
+    ]);
+  });
+});
+
+/**
+ * chant #1377 — `dist/manifest.json` is written by `npm run bundle` (prepack
+ * only), so its version is whatever was last bundled on the machine; on one
+ * checkout it put the fountain docs two minor versions behind what was
+ * already committed. `package.json` is the source of truth, so the rendered
+ * version comes from there and the manifest is only a fallback.
+ */
+describe("lexicon version in generated docs comes from package.json (#1377)", () => {
+  test("package.json beside dist/ wins over a stale manifest", () => {
+    writeFileSync(join(root, "package.json"), JSON.stringify({ name: "fixture", version: "0.37.2" }));
+    const index = docsPipeline(config()).pages.get("index.mdx");
+    expect(index).toContain("**Lexicon version:** 0.37.2");
+    expect(index).not.toContain("0.0.0");
+  });
+
+  test("packageJsonPath overrides the default location", () => {
+    mkdirSync(join(root, "elsewhere"), { recursive: true });
+    writeFileSync(join(root, "elsewhere", "package.json"), JSON.stringify({ version: "1.2.3" }));
+    const index = docsPipeline(config({ packageJsonPath: join(root, "elsewhere", "package.json") })).pages.get(
+      "index.mdx",
+    );
+    expect(index).toContain("**Lexicon version:** 1.2.3");
+  });
+
+  test("falls back to the manifest when no package.json is present", () => {
+    const index = docsPipeline(config()).pages.get("index.mdx");
+    expect(index).toContain("**Lexicon version:** 0.0.0");
+  });
+});

package/src/codegen/docs-pages.ts

@@ -0,0 +1,138 @@
+/**
+ * Authored lexicon doc pages (chant #1731 / #1733).
+ *
+ * A lexicon's prose lives as `.mdx` files under `lexicons/<name>/docs/pages/`.
+ * Each carries a `diataxis` frontmatter field naming its Diátaxis quadrant
+ * (https://diataxis.fr). The pipeline expands `{{file:...}}` markers, stamps a
+ * provenance marker and writes the result into `src/content/docs/`, and the
+ * sidebar is grouped from the field. This replaced two older modes — prose
+ * inside `docs.ts` template literals (`extraPages`) and hand-written content
+ * pages wired through `sidebarExtra` — removed in chant #1757.
+ *
+ * Frontmatter recognised here, on top of Starlight's `title` / `description`:
+ *
+ * - `diataxis`: `tutorial | how-to | reference | explanation`. Required.
+ * - `label`: sidebar label; defaults to `title`.
+ * - `group`: a nested subgroup label inside the quadrant (e.g. "Vendor Composites").
+ * - `order`: number; lower sorts first within its group. Unordered pages follow, by label.
+ * - `hidden`: `true` keeps the page out of the sidebar (reachable by URL only).
+ *
+ * `group` / `order` / `hidden` / `label` are stripped from the written copy so
+ * Starlight's schema never sees them; `diataxis` stays, and the generated
+ * `content.config.ts` declares it.
+ */
+
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { join } from "path";
+
+import { expandFileMarkers } from "./docs-file-markers";
+import type { DocsConfig, Quadrant, SidebarPage } from "./docs-types";
+
+export const QUADRANTS: readonly Quadrant[] = ["tutorial", "how-to", "reference", "explanation"];
+
+/** Sidebar group label for each quadrant, in display order. */
+export const QUADRANT_LABELS: Record<Quadrant, string> = {
+  tutorial: "Tutorials",
+  "how-to": "How-to guides",
+  reference: "Reference",
+  explanation: "Explanation",
+};
+
+export interface AuthoredPage extends SidebarPage {
+  /** Source file name under `pagesDir`, e.g. `getting-started.mdx`. */
+  file: string;
+  /** Page body with the rewritten frontmatter, markers expanded. */
+  content: string;
+}
+
+interface Frontmatter {
+  fields: Map<string, string>;
+  /** Raw frontmatter lines, in order, so unknown keys survive the rewrite. */
+  lines: string[];
+  body: string;
+}
+
+function parseFrontmatter(text: string, file: string): Frontmatter {
+  if (!text.startsWith("---\n")) {
+    throw new Error(`${file}: authored doc page must start with a frontmatter block`);
+  }
+  const end = text.indexOf("\n---", 4);
+  if (end === -1) throw new Error(`${file}: unterminated frontmatter`);
+  const lines = text.slice(4, end).split("\n");
+  const fields = new Map<string, string>();
+  for (const line of lines) {
+    const m = line.match(/^([A-Za-z_][\w-]*):\s*(.*)$/);
+    if (m) fields.set(m[1], m[2].trim());
+  }
+  // Skip the closing `---` and the newline after it.
+  const afterClose = text.indexOf("\n", end + 1);
+  const body = afterClose === -1 ? "" : text.slice(afterClose + 1);
+  return { fields, lines, body };
+}
+
+function unquote(v: string): string {
+  if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) {
+    try {
+      return v.startsWith('"') ? (JSON.parse(v) as string) : v.slice(1, -1);
+    } catch {
+      return v.slice(1, -1);
+    }
+  }
+  return v;
+}
+
+/** Keys this module consumes and removes from the written page. */
+const SIDEBAR_KEYS = new Set(["label", "group", "order", "hidden"]);
+
+/**
+ * Read every authored page under `pagesDir`. Returns an empty list when the
+ * directory does not exist, which is every lexicon before it migrates.
+ */
+export function readAuthoredPages(config: DocsConfig): AuthoredPage[] {
+  const pagesDir = config.pagesDir ?? join(config.outDir, "pages");
+  if (!existsSync(pagesDir)) return [];
+
+  const pages: AuthoredPage[] = [];
+  for (const file of readdirSync(pagesDir).sort()) {
+    if (!file.endsWith(".mdx") && !file.endsWith(".md")) continue;
+    const path = join(pagesDir, file);
+    const { fields, lines, body } = parseFrontmatter(readFileSync(path, "utf-8"), path);
+
+    const title = fields.get("title");
+    if (!title) throw new Error(`${path}: frontmatter needs a title`);
+    const quadrant = fields.get("diataxis");
+    if (!quadrant) {
+      throw new Error(
+        `${path}: frontmatter needs \`diataxis: ${QUADRANTS.join(" | ")}\` (https://diataxis.fr)`,
+      );
+    }
+    if (!QUADRANTS.includes(quadrant as Quadrant)) {
+      throw new Error(`${path}: diataxis "${quadrant}" is not one of ${QUADRANTS.join(", ")}`);
+    }
+
+    const orderRaw = fields.get("order");
+    const order = orderRaw === undefined ? undefined : Number(orderRaw);
+    if (order !== undefined && Number.isNaN(order)) {
+      throw new Error(`${path}: order must be a number, got "${orderRaw}"`);
+    }
+
+    const kept = lines.filter((line) => {
+      const key = line.match(/^([A-Za-z_][\w-]*):/)?.[1];
+      return !(key && SIDEBAR_KEYS.has(key));
+    });
+    let content = body;
+    if (config.examplesDir) content = expandFileMarkers(content, config.examplesDir);
+
+    pages.push({
+      file,
+      slug: file.replace(/\.mdx?$/, ""),
+      label: unquote(fields.get("label") ?? title),
+      quadrant: quadrant as Quadrant,
+      group: fields.has("group") ? unquote(fields.get("group") as string) : undefined,
+      order,
+      hidden: fields.get("hidden") === "true",
+      content: ["---", ...kept, "---", "", content].join("\n"),
+    });
+  }
+  return pages;
+}

package/src/codegen/docs-sections.ts

@@ -42,14 +42,6 @@ export function generateOverview(
 
   const suppress = new Set(config.suppressPages ?? []);
 
-  // Extra pages listed first in reference links
-  if (config.extraPages && config.extraPages.length > 0) {
-    for (const page of config.extraPages) {
-      if (page.sidebar === false) continue;
-      lines.push(`- [${page.title}](./${page.slug})`);
-    }
-  }
-
   if (!suppress.has("intrinsics") && manifest.intrinsics && manifest.intrinsics.length > 0) {
     lines.push(
       `- [Intrinsic Functions](./intrinsics) — ${manifest.intrinsics.length} built-in functions`,