@suzumiyaaoba/mdxr 0.1.0

package/src/load-user-module.ts

@@ -0,0 +1,191 @@
+import { createHash } from "node:crypto";
+import { existsSync, statSync } from "node:fs";
+import { mkdir, readdir, stat, unlink, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+import { build } from "esbuild";
+import type { Plugin } from "esbuild";
+
+import { isRecord } from "./guards.js";
+import { cacheDir } from "./paths.js";
+
+/**
+ * Packages that must resolve to *this* package's dependencies — a second React
+ * copy would break hooks/context, and `valibot` instances mix across schemas.
+ * Shared with hydrate.ts's pinShared plugin (which adds no `@suzumiyaaoba/mdxr`
+ * since the runtimeModule plugin maps that specifier onto the catalog).
+ */
+export const SHARED_PACKAGES = ["react", "react-dom", "valibot"];
+
+// `@suzumiyaaoba/mdxr`(/components) reach this same package via Node's
+// self-reference (the cache file lives inside the package root).
+/* oxlint-disable require-unicode-regexp -- esbuild serializes onResolve filter
+   regexes to Go's RE2, which rejects the `u` flag. */
+const SHARED_EXTERNALS = new RegExp(
+  `^(?:${[...SHARED_PACKAGES, "@suzumiyaaoba/mdxr"].join("|")})(?:/.*)?$`
+);
+/* oxlint-enable require-unicode-regexp */
+
+const sharedExternals: Plugin = {
+  name: "shared-externals",
+  setup(b) {
+    b.onResolve({ filter: SHARED_EXTERNALS }, (a) => ({
+      external: true,
+      path: a.path,
+    }));
+  },
+};
+
+export interface BundledModule {
+  module: Record<string, unknown>;
+  /** Bundled source — also used as a Tailwind scan source. */
+  code: string;
+}
+
+const INDEX_FILES = ["index.tsx", "index.ts", "index.jsx", "index.js"];
+
+/** Files written but not yet imported — a prune must never unlink these. */
+const inFlight = new Set<string>();
+
+/**
+ * `mdxr serve` writes a fresh `doc-<hash>.mjs` on every rebuild, so the cache
+ * would grow unboundedly over a session. Keep only the newest few files per
+ * prefix — imported modules live in the module registry, not on disk, so
+ * deleting an already-imported file is safe. Best-effort: failures are fine.
+ */
+const pruneCache = async (prefix: string): Promise<void> => {
+  try {
+    const KEEP = 8;
+    // Never touch files younger than this: another process may have written
+    // one and not yet called import() (inFlight only guards this process).
+    const STALE_MS = 10_000;
+    const now = Date.now();
+    const ents = await readdir(cacheDir);
+    const mine = ents.filter(
+      (f) =>
+        f.startsWith(`${prefix}-`) &&
+        f.endsWith(".mjs") &&
+        !inFlight.has(path.join(cacheDir, f))
+    );
+    if (mine.length <= KEEP) {
+      return;
+    }
+    const withMtime = await Promise.all(
+      mine.map(async (f) => {
+        const s = await stat(path.join(cacheDir, f));
+        return { f, mtime: s.mtimeMs };
+      })
+    );
+    withMtime.sort((a, b) => b.mtime - a.mtime);
+    const keep = new Set(withMtime.slice(0, KEEP).map(({ f }) => f));
+    await Promise.all(
+      withMtime
+        .filter(({ f, mtime }) => now - mtime > STALE_MS && !keep.has(f))
+        .map(async ({ f, mtime }) => {
+          const p = path.join(cacheDir, f);
+          // TOCTOU: another process may have rewritten the file since our
+          // readdir/stat (a rewrite refreshes mtime, but we judged staleness
+          // on the old one). Re-stat right before unlinking.
+          const s = await stat(p);
+          if (s.mtimeMs !== mtime || now - s.mtimeMs <= STALE_MS) {
+            return;
+          }
+          await unlink(p);
+        })
+    );
+  } catch {
+    // Cache hygiene is best-effort — never fail a render over it.
+  }
+};
+
+/** `await import` of a missing module: Node's ERR_MODULE_NOT_FOUND, possibly
+ *  wrapped by a module runner that only preserves the message. */
+const isModuleNotFound = (e: unknown): boolean =>
+  isRecord(e) &&
+  (e.code === "ERR_MODULE_NOT_FOUND" ||
+    e.code === "ENOENT" ||
+    (typeof e.message === "string" &&
+      e.message.includes("Cannot find module")));
+
+/**
+ * Write bundled ESM `code` into this package's cache dir (content-hashed, so
+ * repeat imports are free) and import it. Living inside the package makes
+ * bare imports (`react`, `valibot`, `@suzumiyaaoba/mdxr`) resolve to *our*
+ * copies — one React instance shared with renderToStaticMarkup and the
+ * hydrate bundle.
+ */
+export const importBundledCode = async (
+  code: string,
+  prefix: string
+): Promise<Record<string, unknown>> => {
+  await mkdir(cacheDir, { recursive: true });
+  const hash = createHash("sha256").update(code).digest("hex").slice(0, 12);
+  const out = path.join(cacheDir, `${prefix}-${hash}.mjs`);
+  // Always rewrite: refreshing the mtime keeps the live file out of prune's
+  // reach (a stale-mtime hit could otherwise be unlinked while still in use).
+  await writeFile(out, code);
+  inFlight.add(out);
+  let raw: unknown;
+  try {
+    try {
+      raw = await import(pathToFileURL(out).href);
+    } catch (error) {
+      if (!isModuleNotFound(error)) {
+        throw error;
+      }
+      // A concurrent process's pruneCache may have unlinked the file between
+      // our writeFile and the module runner's read (it judged staleness on a
+      // pre-rewrite mtime). Rewrite and retry once on a fresh URL — a `?`
+      // suffix also escapes any cached miss in the importing module runner.
+      await writeFile(out, code);
+      raw = await import(`${pathToFileURL(out).href}?retry`);
+    }
+  } finally {
+    inFlight.delete(out);
+  }
+  void pruneCache(prefix);
+  return isRecord(raw) ? raw : {};
+};
+
+/**
+ * Resolve a user module path: directories collapse to their index file.
+ * Used both when importing the module (SSR) and when bundling it for the
+ * hydration script.
+ */
+export const resolveModuleEntry = (entryPath: string): string => {
+  if (!(existsSync(entryPath) && statSync(entryPath).isDirectory())) {
+    return entryPath;
+  }
+  for (const name of INDEX_FILES) {
+    const candidate = path.join(entryPath, name);
+    if (existsSync(candidate)) {
+      return candidate;
+    }
+  }
+  return entryPath;
+};
+
+/**
+ * Bundle a user-authored TS/TSX module and import it.
+ * The result is written inside this package's directory so that externals
+ * resolve against our node_modules (single React instance).
+ */
+export const loadUserModule = async (
+  entryPath: string
+): Promise<BundledModule> => {
+  const result = await build({
+    bundle: true,
+    entryPoints: [entryPath],
+    format: "esm",
+    jsx: "automatic",
+    jsxImportSource: "react",
+    logLevel: "silent",
+    platform: "node",
+    plugins: [sharedExternals],
+    target: "node20",
+    write: false,
+  });
+  const code = result.outputFiles[0].text;
+  return { code, module: await importBundledCode(code, "components") };
+};

package/src/mdx.ts

@@ -0,0 +1,248 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+import { compile } from "@mdx-js/mdx";
+import { createElement, Fragment } from "react";
+import type { ReactElement } from "react";
+import { renderToStaticMarkup, renderToString } from "react-dom/server";
+import rehypeKatex from "rehype-katex";
+import remarkDirective from "remark-directive";
+import remarkFrontmatter from "remark-frontmatter";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import { VFile } from "vfile";
+import { matter } from "vfile-matter";
+
+import type { ComponentMap } from "./define.js";
+import type { DocContextValue } from "./doc-context.js";
+import { DocContext } from "./doc-context.js";
+import { editorUrl } from "./editor.js";
+import { enhanceRenderError, formatError } from "./format-error.js";
+import { isComponent, isRecord } from "./guards.js";
+import { importBundledCode } from "./load-user-module.js";
+import { rehypeShiki } from "./rehype/shiki.js";
+import { remarkMdxrAlerts } from "./remark/alerts.js";
+import { remarkCodeFile } from "./remark/code-file.js";
+import { remarkCodeMeta } from "./remark/code-meta.js";
+import { remarkMdxrDirectives } from "./remark/directives.js";
+import { remarkFilePaths } from "./remark/file-paths.js";
+import { remarkMdxrHeadings } from "./remark/headings.js";
+import { remarkNoJs } from "./remark/no-js.js";
+import { takeUsedIcons } from "./ui/icon.js";
+
+export interface MdxResult {
+  body: string;
+  frontmatter: Record<string, unknown>;
+  /**
+   * Compiled MDX module source (ESM, `react/jsx-runtime` imports). The same
+   * module is rendered here and re-bundled into the document's hydration
+   * script, so client and server evaluate identical code.
+   */
+  code: string;
+  /**
+   * Every `fileLink(rel, line)` call made during SSR, keyed `rel\0line`. The
+   * hydration bundle replays these so components see identical link results
+   * (no `existsSync` in the browser).
+   */
+  fileLinks: Record<string, string>;
+  /**
+   * Iconify names (`prefix:name`) resolved while rendering — the hydration
+   * bundle registers exactly this subset instead of the full icon sets.
+   */
+  usedIcons: string[];
+  /**
+   * Catalog keys the compiled document references — the hydrate import list.
+   * Extracted from the emitted module rather than observed at render: the
+   * module spreads `props.components`, so proxy-based tracking would record
+   * every catalog key.
+   */
+  usedComponents: string[];
+  /**
+   * ISO timestamp captured at render time and put in `DocContext.now` —
+   * serialized into the hydration spec so relative-time components render
+   * identically on the client.
+   */
+  renderedAt: string;
+  /**
+   * The context value the body was rendered under — anything rendered next to
+   * it (the frontmatter PlanHeader) must use the same Provider or hydration
+   * sees different context.
+   */
+  context: DocContextValue;
+  /** True when the body was rendered for hydration (renderToString). */
+  hydrated: boolean;
+  /**
+   * Re-render the document with `header` as the first child of the same
+   * Provider > Fragment > [header|null, doc] tree the hydration client
+   * mounts (hydrate-runtime's mountDocument). useId() encodes tree position,
+   * so the header must be rendered inside that shape — prepending separately
+   * rendered markup would shift every hydrated id/name/htmlFor. Returns
+   * fresh snapshots too: the pass replays every fileLink/icon lookup.
+   */
+  renderWithHeader: (header: ReactElement) => {
+    fileLinks: Record<string, string>;
+    html: string;
+    usedIcons: string[];
+  };
+}
+
+/**
+ * Which catalog entries the document references is visible in the compiled
+ * module itself: JSX identifiers become `_missingMdxReference("Name", …)`
+ * checks, markdown element overrides read `_components.name`, and a `wrapper`
+ * entry is picked straight off `props.components`. (Spreading `components`
+ * into `_components` makes runtime tracking see every key — hence static
+ * extraction here instead.)
+ */
+const extractUsedComponents = (
+  code: string,
+  components: ComponentMap
+): string[] => {
+  const catalogKeys = new Set(Object.keys(components));
+  const used = new Set<string>();
+  for (const m of code.matchAll(/_missingMdxReference\("(?<name>[^"]+)"/gu)) {
+    const name = m.groups?.name;
+    if (name !== undefined && catalogKeys.has(name)) {
+      used.add(name);
+    }
+  }
+  for (const m of code.matchAll(
+    /_components\.(?<dot>\w+)|_components\["(?<bracket>[^"]+)"\]/gu
+  )) {
+    const name = m.groups?.dot ?? m.groups?.bracket;
+    if (name !== undefined && catalogKeys.has(name)) {
+      used.add(name);
+    }
+  }
+  if (catalogKeys.has("wrapper")) {
+    used.add("wrapper");
+  }
+  return [...used];
+};
+
+export const mdxToHtml = async (
+  source: string,
+  components: ComponentMap,
+  filePath = "document.mdx",
+  opts: { editor?: string; hydrate?: boolean } = {}
+): Promise<MdxResult> => {
+  const file = new VFile({ path: filePath, value: source });
+  matter(file);
+  // vfile-matter sets `file.data.matter`, but its types don't declare it.
+  const fmRaw: unknown = isRecord(file.data) ? file.data.matter : undefined;
+  const frontmatter: Record<string, unknown> = isRecord(fmRaw) ? fmRaw : {};
+
+  // Frontmatter `editor:` overrides the mdxr.config.ts default; "none" or an
+  // unresolvable path disables the link. Only existing files get links.
+  const editor =
+    typeof frontmatter.editor === "string" && frontmatter.editor !== ""
+      ? frontmatter.editor
+      : opts.editor;
+  const dir = file.dirname ?? ".";
+  // Calls are recorded so the hydration bundle can replay identical results —
+  // the client has no filesystem, so a missing map entry means "no link".
+  const fileLinks = new Map<string, string>();
+  const fileLink = (rel: string, line?: string): string | undefined => {
+    const abs = path.resolve(dir, rel);
+    const url = existsSync(abs) ? editorUrl(editor, abs, line) : undefined;
+    if (url !== undefined) {
+      // NUL separator — `rel` may legitimately end in digits, and the client
+      // lookup in hydrate.ts uses the same key shape.
+      fileLinks.set(`${rel}\0${line ?? ""}`, url);
+    }
+    return url;
+  };
+
+  // Compile once: the emitted module is imported for SSR *and* inlined into
+  // the hydration bundle, so both sides run byte-identical document code.
+  const compiled = await compile(file, {
+    baseUrl: import.meta.url,
+    format: "mdx",
+    rehypePlugins: [rehypeKatex, rehypeShiki],
+    remarkPlugins: [
+      remarkFrontmatter,
+      remarkGfm,
+      remarkMath,
+      remarkDirective,
+      remarkMdxrDirectives,
+      remarkMdxrAlerts,
+      remarkNoJs,
+      remarkMdxrHeadings,
+      remarkCodeFile,
+      remarkCodeMeta,
+      remarkFilePaths,
+    ],
+  });
+  // Non-fatal plugin diagnostics (unknown directives, …) reach the user here.
+  for (const m of compiled.messages) {
+    process.stderr.write(`mdxr: warning: ${formatError(m)}\n`);
+  }
+  const code = String(compiled);
+  // The compiled module is imported from this package's cache dir so its
+  // `react/jsx-runtime` resolves to our copy — the same instance the
+  // hydration bundle pins via its resolve plugin.
+  const mod = await importBundledCode(code, "doc");
+
+  const used = extractUsedComponents(code, components);
+
+  const docComponent = mod.default;
+  if (!isComponent(docComponent)) {
+    throw new Error("Compiled document has no default export component.");
+  }
+
+  takeUsedIcons();
+  const renderedAt = new Date();
+  const context: DocContextValue = { fileLink, now: renderedAt };
+  // hydrateRoot reconciles against markup produced by renderToString — its
+  // `<!-- -->` text-boundary comments keep adjacent text expressions from
+  // merging in the DOM. renderToStaticMarkup omits them, so only purely
+  // static documents use it.
+  const hydrated = (opts.hydrate ?? true) && used.length > 0;
+  const renderToMarkup = hydrated ? renderToString : renderToStaticMarkup;
+  // The client mounts Provider > Fragment > [header|null, doc]. SSR must emit
+  // that exact shape: useId() seeds encode a component's position in the
+  // tree, so rendering `doc` directly would diverge from hydration even with
+  // no header (the extra Fragment level changes the generated ids).
+  const renderDocument = (header: ReactElement | null): string =>
+    renderToMarkup(
+      createElement(
+        DocContext.Provider,
+        { value: context },
+        createElement(
+          Fragment,
+          null,
+          header,
+          createElement(docComponent, { components })
+        )
+      )
+    );
+  let body: string;
+  try {
+    body = renderDocument(null);
+  } catch (error) {
+    throw enhanceRenderError(error, Object.keys(components));
+  }
+  return {
+    body,
+    code,
+    context,
+    fileLinks: Object.fromEntries(fileLinks),
+    frontmatter,
+    hydrated,
+    renderWithHeader: (header) => {
+      try {
+        const html = renderDocument(header);
+        return {
+          fileLinks: Object.fromEntries(fileLinks),
+          html,
+          usedIcons: takeUsedIcons(),
+        };
+      } catch (error) {
+        throw enhanceRenderError(error, Object.keys(components));
+      }
+    },
+    renderedAt: renderedAt.toISOString(),
+    usedComponents: used,
+    usedIcons: takeUsedIcons(),
+  };
+};

package/src/paths.ts

@@ -0,0 +1,36 @@
+import module from "node:module";
+import path from "node:path";
+import url from "node:url";
+
+const require = module.createRequire(import.meta.url);
+
+const findPkgRoot = (): string => {
+  try {
+    return path.dirname(
+      url.fileURLToPath(require.resolve("@suzumiyaaoba/mdxr/package.json"))
+    );
+  } catch {
+    // Bundled (dist/*.mjs) and source (src/*.ts) layouts both sit one level
+    // below the package root.
+    return url.fileURLToPath(new URL("..", import.meta.url));
+  }
+};
+
+/** Absolute path to this package's root (works bundled, unbundled, and via npx). */
+export const pkgRoot = findPkgRoot();
+
+/**
+ * This package's `src/` directory. The hydration bundle is always built from
+ * the TypeScript sources — they ship with the package, so packed installs
+ * and local dev produce identical bundles. Building from sources (rather
+ * than the packed `dist/` chunks) is also what makes tree-shaking effective:
+ * every used component is imported from its leaf module, and no barrel file
+ * is ever loaded.
+ */
+export const srcDir = path.join(pkgRoot, "src");
+
+/** Directory where bundled user modules are cached. Kept inside this package so
+ *  bare imports (`react`, `valibot`, `@suzumiyaaoba/mdxr`) resolve to *our*
+ *  copy —
+ *  guaranteeing a single React instance shared with renderToStaticMarkup. */
+export const cacheDir = `${pkgRoot}/.mdxr-cache`;