@mandujs/core 0.24.0 → 0.25.0

package/src/content/llms-txt.ts

@@ -0,0 +1,196 @@
+/**
+ * llms.txt Generator (Issue #199)
+ *
+ * Emits a single plain-text index of collection entries in the
+ * `llms.txt` convention (https://llmstxt.org). The output is a
+ * deterministic Markdown-ish digest that LLM ingestion pipelines can
+ * crawl without having to understand the project's routing or MDX
+ * compilation.
+ *
+ * # Default format
+ *
+ * ```text
+ * # {site.name}
+ *
+ * ## docs
+ * - [Introduction](/docs/intro): Getting started
+ * - [CLI](/docs/cli): Command reference
+ *
+ * ## blog
+ * - [Hello world](/blog/hello): First post
+ * ```
+ *
+ * # `full: true` variant
+ *
+ * When `full: true` is passed, every entry's body is inlined after
+ * its heading — useful for tools that want an offline snapshot of
+ * all content. This produces `llms-full.txt` in most conventions;
+ * the filename is the caller's responsibility.
+ */
+
+import type { Collection, CollectionEntry } from "./collection";
+
+/**
+ * Collection shape accepted by llms.txt — we use a structural subset
+ * (just the `all()` method) so callers can pass typed collections
+ * (`Collection<{ title: string }>`) without fighting TypeScript's
+ * invariance on generic classes. The generator never writes back into
+ * the collection, so the type-erasure to `CollectionEntry<unknown>`
+ * is safe at runtime — llms.txt reads `data.title`/`data.description`
+ * via `unknown` narrowing.
+ */
+// Covariant read-only view of a Collection. We deliberately re-declare
+// the method signature with `unknown` so Collection<{...}> assigns
+// structurally. `bivarianceHack` lets the assignability flow.
+interface CollectionReader {
+  all(): Promise<Array<{ slug: string; filePath: string; data: unknown; content: string }>>;
+}
+
+/** Entry in the input array — either a Collection or a pre-loaded triple. */
+export type LLMSTxtInput =
+  | { name: string; collection: CollectionReader }
+  | { name: string; entries: CollectionEntry<unknown>[] };
+
+// Re-export the full Collection type so consumers importing the llms
+// types still see the concrete class alongside the reader alias.
+export type { Collection };
+
+/** Options controlling llms.txt rendering. */
+export interface GenerateLLMSTxtOptions {
+  /**
+   * Top-level site title. Rendered as the `#` heading. When omitted,
+   * the heading line is skipped so callers can prepend their own.
+   */
+  siteName?: string;
+  /**
+   * Short description rendered below the site heading. Ignored
+   * when `siteName` is omitted — the heading anchors the block.
+   */
+  description?: string;
+  /**
+   * Base URL prefix applied to every entry href. Default `/`; pass
+   * an absolute origin (e.g. `https://example.com`) to produce an
+   * outward-facing llms.txt that third-party crawlers can consume
+   * without resolving against the host.
+   */
+  basePath?: string;
+  /**
+   * When true, include each entry's body verbatim under its heading.
+   * This produces the `llms-full.txt` variant — significantly larger
+   * output, but lets consumers avoid a second fetch per entry.
+   */
+  full?: boolean;
+  /**
+   * Include entries with `data.draft === true` (default: false).
+   * Production sites should leave this off so unpublished content
+   * doesn't leak to external crawlers.
+   */
+  includeDrafts?: boolean;
+  /**
+   * Override the per-entry summary line. Defaults to
+   * `entry.data.description ?? ""`. Returning an empty string
+   * omits the trailing `: {summary}` tail.
+   */
+  getSummary?: (entry: CollectionEntry<unknown>) => string;
+}
+
+/**
+ * Generate an llms.txt document from one or more collections.
+ *
+ * Accepts both `Collection` instances (loaded internally) and
+ * pre-loaded entry arrays so callers can pipe in filtered/transformed
+ * data without paying for a second scan.
+ */
+export async function generateLLMSTxt(
+  inputs: LLMSTxtInput[],
+  options: GenerateLLMSTxtOptions = {}
+): Promise<string> {
+  const {
+    siteName,
+    description,
+    basePath = "/",
+    full = false,
+    includeDrafts = false,
+    getSummary,
+  } = options;
+
+  const lines: string[] = [];
+  if (siteName) {
+    lines.push(`# ${siteName}`);
+    if (description) {
+      lines.push("");
+      lines.push(`> ${description}`);
+    }
+    lines.push("");
+  }
+
+  for (const input of inputs) {
+    const entries = await loadInput(input);
+    const visible = includeDrafts
+      ? entries
+      : entries.filter((e) => !(e.data as { draft?: unknown })?.draft);
+    if (visible.length === 0) continue;
+
+    // Sort within each collection by slug for deterministic output —
+    // llms.txt is often diffed by agents/tools, and a stable order
+    // keeps those diffs meaningful.
+    const sorted = [...visible].sort((a, b) =>
+      a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0
+    );
+
+    lines.push(`## ${input.name}`);
+    lines.push("");
+    for (const entry of sorted) {
+      const title =
+        typeof (entry.data as { title?: unknown })?.title === "string"
+          ? String((entry.data as { title: string }).title)
+          : entry.slug || "index";
+      const href = joinHref(basePath, input.name, entry.slug);
+      const summary = getSummary
+        ? getSummary(entry)
+        : typeof (entry.data as { description?: unknown })?.description === "string"
+          ? String((entry.data as { description: string }).description)
+          : "";
+      const tail = summary ? `: ${summary}` : "";
+      lines.push(`- [${title}](${href})${tail}`);
+      if (full) {
+        lines.push("");
+        lines.push(entry.content);
+        lines.push("");
+      }
+    }
+    lines.push("");
+  }
+
+  // Trim the trailing blank line — the "ends in newline" convention
+  // is handled by the single final `\n` below so we don't accumulate
+  // extra blank tail lines across collections.
+  while (lines.length > 0 && lines[lines.length - 1] === "") lines.pop();
+  return lines.join("\n") + "\n";
+}
+
+async function loadInput(input: LLMSTxtInput): Promise<CollectionEntry<unknown>[]> {
+  if ("entries" in input) return input.entries;
+  // CollectionReader.all() yields the same shape as
+  // CollectionEntry<unknown>[] — the cast here is a trivial widening.
+  const entries = await input.collection.all();
+  return entries as CollectionEntry<unknown>[];
+}
+
+function joinHref(base: string, collectionName: string, slug: string): string {
+  const parts = [collectionName, slug].filter((x) => x !== "" && x !== "/");
+  const tail = parts.join("/").replace(/\/+/g, "/");
+  if (base.startsWith("http")) {
+    // Preserve the `//` after the protocol — we only collapse slashes
+    // in the path portion, so `https://example.com/docs/foo` survives
+    // intact instead of becoming `https:/example.com/docs/foo`.
+    const trimmedBase = base.endsWith("/") ? base.slice(0, -1) : base;
+    return tail ? `${trimmedBase}/${tail}` : trimmedBase;
+  }
+  if (base === "" || base === "/") {
+    return `/${tail}`.replace(/\/+/g, "/");
+  }
+  const trimmed = base.endsWith("/") ? base.slice(0, -1) : base;
+  const leading = trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+  return `${leading}/${tail}`.replace(/\/+/g, "/");
+}

package/src/content/prebuild.test.ts

@@ -0,0 +1,249 @@
+/**
+ * Tests for `content/prebuild.ts` (Issue #196).
+ *
+ * We exercise:
+ *   - `discoverPrebuildScripts` globbing + sort determinism
+ *   - `shouldAutoPrebuild` activation policy (content/ OR scripts/prebuild-*.ts)
+ *   - `runPrebuildScripts` with an injected spawn hook (no actual subprocess)
+ *     covering the success path, per-script fail-fast, timeout surface, and
+ *     the empty-discovery no-op fast path.
+ *
+ * Spawning a real `bun` subprocess is intentionally avoided — those paths
+ * belong in the CLI-level integration test. At this layer the Bun runtime
+ * is mocked so the test doubles as a correctness guarantee for any Node
+ * environment that later injects its own spawn hook.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  discoverPrebuildScripts,
+  shouldAutoPrebuild,
+  runPrebuildScripts,
+  PrebuildError,
+  type SpawnHook,
+} from "./prebuild";
+
+const PREFIX = path.join(os.tmpdir(), "mandu-prebuild-test-");
+
+function mktmp(prefix = ""): string {
+  return fs.mkdtempSync(PREFIX + prefix);
+}
+
+function writeFile(dir: string, rel: string, content = "// stub\n"): string {
+  const abs = path.join(dir, rel);
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  fs.writeFileSync(abs, content);
+  return abs;
+}
+
+// ---------------------------------------------------------------------------
+// discoverPrebuildScripts
+// ---------------------------------------------------------------------------
+
+describe("discoverPrebuildScripts", () => {
+  let dir = "";
+  beforeEach(() => { dir = mktmp("discover-"); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+
+  it("returns empty when scripts/ does not exist", () => {
+    expect(discoverPrebuildScripts(dir)).toEqual([]);
+  });
+
+  it("returns empty when scripts/ exists but has no prebuild files", () => {
+    writeFile(dir, "scripts/build.ts");
+    writeFile(dir, "scripts/README.md");
+    expect(discoverPrebuildScripts(dir)).toEqual([]);
+  });
+
+  it("finds prebuild-<name>.ts files", () => {
+    writeFile(dir, "scripts/prebuild-docs.ts");
+    writeFile(dir, "scripts/prebuild-seo.ts");
+    const found = discoverPrebuildScripts(dir);
+    expect(found).toHaveLength(2);
+    expect(found[0]).toContain("prebuild-docs.ts");
+    expect(found[1]).toContain("prebuild-seo.ts");
+  });
+
+  it("sorts lexicographically so numeric prefix controls ordering", () => {
+    writeFile(dir, "scripts/prebuild-20-second.ts");
+    writeFile(dir, "scripts/prebuild-10-first.ts");
+    writeFile(dir, "scripts/prebuild-99-third.ts");
+    const found = discoverPrebuildScripts(dir).map((p) => path.basename(p));
+    expect(found).toEqual([
+      "prebuild-10-first.ts",
+      "prebuild-20-second.ts",
+      "prebuild-99-third.ts",
+    ]);
+  });
+
+  it("accepts .tsx, .js, .mjs extensions", () => {
+    writeFile(dir, "scripts/prebuild-a.tsx");
+    writeFile(dir, "scripts/prebuild-b.js");
+    writeFile(dir, "scripts/prebuild-c.mjs");
+    const found = discoverPrebuildScripts(dir).map((p) => path.basename(p));
+    expect(found).toEqual(["prebuild-a.tsx", "prebuild-b.js", "prebuild-c.mjs"]);
+  });
+
+  it("rejects non-prebuild filenames even if in scripts/", () => {
+    writeFile(dir, "scripts/my-prebuild.ts");       // doesn't start with prebuild
+    writeFile(dir, "scripts/prebuild.py");           // wrong extension
+    writeFile(dir, "scripts/prebuild-valid.ts");
+    const found = discoverPrebuildScripts(dir).map((p) => path.basename(p));
+    expect(found).toEqual(["prebuild-valid.ts"]);
+  });
+
+  it("honors custom scriptsDir", () => {
+    writeFile(dir, "custom/prebuild-1.ts");
+    expect(discoverPrebuildScripts(dir, "custom")).toHaveLength(1);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// shouldAutoPrebuild
+// ---------------------------------------------------------------------------
+
+describe("shouldAutoPrebuild", () => {
+  let dir = "";
+  beforeEach(() => { dir = mktmp("policy-"); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+
+  it("false when neither content/ nor prebuild scripts exist", () => {
+    expect(shouldAutoPrebuild(dir)).toBe(false);
+  });
+
+  it("true when content/ directory exists (even if empty)", () => {
+    fs.mkdirSync(path.join(dir, "content"));
+    expect(shouldAutoPrebuild(dir)).toBe(true);
+  });
+
+  it("true when a prebuild script exists but content/ does not", () => {
+    writeFile(dir, "scripts/prebuild-docs.ts");
+    expect(shouldAutoPrebuild(dir)).toBe(true);
+  });
+
+  it("false when content/ is a file, not a directory", () => {
+    fs.writeFileSync(path.join(dir, "content"), "not a dir");
+    expect(shouldAutoPrebuild(dir)).toBe(false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// runPrebuildScripts with mock spawn
+// ---------------------------------------------------------------------------
+
+function makeMockSpawn(
+  policy: (scriptPath: string) => { exitCode: number | null; durationMs: number } | Promise<{ exitCode: number | null; durationMs: number }>,
+): SpawnHook & { calls: Array<{ scriptPath: string; cwd: string; timeoutMs: number }> } {
+  const calls: Array<{ scriptPath: string; cwd: string; timeoutMs: number }> = [];
+  const hook: SpawnHook = async (args) => {
+    calls.push(args);
+    return await policy(args.scriptPath);
+  };
+  return Object.assign(hook, { calls });
+}
+
+describe("runPrebuildScripts", () => {
+  let dir = "";
+  beforeEach(() => { dir = mktmp("run-"); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+
+  it("returns { ran: 0 } when no scripts exist — silent no-op", async () => {
+    const spawn = makeMockSpawn(() => ({ exitCode: 0, durationMs: 5 }));
+    const result = await runPrebuildScripts({ rootDir: dir, spawn });
+    expect(result.ran).toBe(0);
+    expect(spawn.calls).toHaveLength(0);
+  });
+
+  it("runs discovered scripts in order", async () => {
+    writeFile(dir, "scripts/prebuild-b.ts");
+    writeFile(dir, "scripts/prebuild-a.ts");
+    const order: string[] = [];
+    const spawn = makeMockSpawn(async (p) => {
+      order.push(path.basename(p));
+      return { exitCode: 0, durationMs: 1 };
+    });
+    const result = await runPrebuildScripts({ rootDir: dir, spawn });
+    expect(result.ran).toBe(2);
+    expect(order).toEqual(["prebuild-a.ts", "prebuild-b.ts"]);
+  });
+
+  it("throws PrebuildError with exitCode on first non-zero exit", async () => {
+    writeFile(dir, "scripts/prebuild-1.ts");
+    writeFile(dir, "scripts/prebuild-2.ts");
+    const spawn = makeMockSpawn((p) => ({
+      exitCode: path.basename(p) === "prebuild-1.ts" ? 3 : 0,
+      durationMs: 1,
+    }));
+    let caught: unknown;
+    try {
+      await runPrebuildScripts({ rootDir: dir, spawn });
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(PrebuildError);
+    const err = caught as PrebuildError;
+    expect(err.exitCode).toBe(3);
+    expect(err.scriptPath).toContain("prebuild-1.ts");
+    // prebuild-2.ts must NOT have been invoked (fail-fast chain semantics).
+    expect(spawn.calls).toHaveLength(1);
+  });
+
+  it("wraps non-PrebuildError spawn rejections", async () => {
+    writeFile(dir, "scripts/prebuild.ts");
+    const spawn: SpawnHook = async () => {
+      throw new Error("spawn ENOENT");
+    };
+    let caught: unknown;
+    try {
+      await runPrebuildScripts({ rootDir: dir, spawn });
+    } catch (e) {
+      caught = e;
+    }
+    expect(caught).toBeInstanceOf(PrebuildError);
+    expect((caught as PrebuildError).message).toContain("spawn ENOENT");
+  });
+
+  it("invokes onStart / onFinish callbacks for each script", async () => {
+    writeFile(dir, "scripts/prebuild-a.ts");
+    writeFile(dir, "scripts/prebuild-b.ts");
+    const starts: string[] = [];
+    const finishes: Array<{ name: string; code: number | null }> = [];
+    await runPrebuildScripts({
+      rootDir: dir,
+      spawn: async () => ({ exitCode: 0, durationMs: 1 }),
+      onStart: (p, i, total) => {
+        starts.push(`${i + 1}/${total}:${path.basename(p)}`);
+      },
+      onFinish: (r) => {
+        finishes.push({ name: path.basename(r.scriptPath), code: r.exitCode });
+      },
+    });
+    expect(starts).toEqual(["1/2:prebuild-a.ts", "2/2:prebuild-b.ts"]);
+    expect(finishes).toEqual([
+      { name: "prebuild-a.ts", code: 0 },
+      { name: "prebuild-b.ts", code: 0 },
+    ]);
+  });
+
+  it("threads timeoutMs through to the spawn hook", async () => {
+    writeFile(dir, "scripts/prebuild-1.ts");
+    const spawn = makeMockSpawn(() => ({ exitCode: 0, durationMs: 1 }));
+    await runPrebuildScripts({
+      rootDir: dir,
+      spawn,
+      timeoutMs: 5_000,
+    });
+    expect(spawn.calls[0].timeoutMs).toBe(5_000);
+  });
+
+  it("uses default 2-minute timeout when not overridden", async () => {
+    writeFile(dir, "scripts/prebuild-1.ts");
+    const spawn = makeMockSpawn(() => ({ exitCode: 0, durationMs: 1 }));
+    await runPrebuildScripts({ rootDir: dir, spawn });
+    expect(spawn.calls[0].timeoutMs).toBe(2 * 60 * 1000);
+  });
+});