@pterodoc/core 0.2.0

package/src/model/capture.ts

@@ -0,0 +1,92 @@
+/**
+ * Capturing a site model to disk.
+ *
+ * A captured model is how the tests avoid needing a Docusaurus install, how a
+ * user files a reproducible bug, and how a site whose Docusaurus version this
+ * tool cannot load can still be rendered.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { ConfigError, VERSION } from '../util';
+import type { SiteModel } from './types';
+import type { SourceReader } from './reader';
+
+/** A model as it is stored, with enough provenance to read it later. */
+export interface CapturedModel {
+  /** Format version of this file. */
+  capture: 1;
+  /** Version of pterodoc that wrote it. */
+  pterodoc: string;
+  /** When it was written. */
+  capturedAt: string;
+  /** The model itself. */
+  model: SiteModel;
+}
+
+/** Serialise a model. */
+export function serializeModel(model: SiteModel): string {
+  const captured: CapturedModel = {
+    capture: 1,
+    pterodoc: VERSION,
+    capturedAt: new Date().toISOString(),
+    model,
+  };
+  return `${JSON.stringify(captured, null, 2)}\n`;
+}
+
+/** Write a model to a file, creating its directory. */
+export async function writeCapture(file: string, model: SiteModel): Promise<void> {
+  await fs.mkdir(path.dirname(path.resolve(file)), { recursive: true });
+  await fs.writeFile(file, serializeModel(model), 'utf8');
+}
+
+/**
+ * Read a captured model.
+ *
+ * @param file Path of the capture.
+ */
+export async function readCapture(file: string): Promise<SiteModel> {
+  let raw: string;
+  try {
+    raw = await fs.readFile(file, 'utf8');
+  } catch {
+    throw new ConfigError(`Could not read the model at ${file}.`);
+  }
+
+  let parsed: Partial<CapturedModel>;
+  try {
+    parsed = JSON.parse(raw) as Partial<CapturedModel>;
+  } catch {
+    throw new ConfigError(`${file} is not valid JSON.`);
+  }
+
+  if (parsed.capture !== 1 || !parsed.model) {
+    throw new ConfigError(
+      `${file} is not a pterodoc model capture. Produce one with \`pterodoc capture\`.`,
+    );
+  }
+  return parsed.model;
+}
+
+/**
+ * Read from a model captured earlier.
+ *
+ * @param file Path to the capture.
+ */
+export function createCaptureReader(file: string): SourceReader {
+  let cached: SiteModel | undefined;
+  const load = async (): Promise<SiteModel> => {
+    cached ??= await readCapture(file);
+    return cached;
+  };
+  return {
+    kind: 'captured',
+    async locales(): Promise<string[]> {
+      return [(await load()).locale];
+    },
+    async read(): Promise<SiteModel> {
+      return load();
+    },
+  };
+}

package/src/model/index.ts

@@ -0,0 +1,25 @@
+/**
+ * The site model: what a source produces and what the reconciler consumes.
+ *
+ * This layer knows nothing about where a model came from or where it is going.
+ */
+
+export type {
+  Doc,
+  DocNeighbour,
+  DocsInstance,
+  DocsVersion,
+  SidebarCategoryItem,
+  SidebarCategoryLink,
+  SidebarDocItem,
+  SidebarHtmlItem,
+  SidebarItem,
+  SidebarLinkItem,
+  SiteModel,
+} from './types';
+export { buildPageTree } from './tree';
+export type { BuildPageTreeInput, PageKind, PageNode, PageTree } from './tree';
+export { createMemoryReader } from './reader';
+export type { SourceReader } from './reader';
+export { createCaptureReader, readCapture, serializeModel, writeCapture } from './capture';
+export type { CapturedModel } from './capture';

package/src/model/reader.ts

@@ -0,0 +1,37 @@
+/**
+ * Where a site model comes from.
+ *
+ * The rest of the tool takes a reader rather than calling a source directly,
+ * which is what lets it be tested against a captured model, driven by a build,
+ * or pointed at a real site. The interface lives here, with the model it
+ * produces; each source implements it in its own package.
+ */
+
+import type { SiteModel } from './types';
+
+/** Supplies one site model per locale. */
+export interface SourceReader {
+  /** How this reader found the model, for the plan and for messages. */
+  readonly kind: 'docusaurus' | 'captured' | 'memory';
+  /** Locales this reader can produce, in the order they should be published. */
+  locales(): Promise<string[]>;
+  /** The model for one locale. */
+  read(locale?: string): Promise<SiteModel>;
+}
+
+/**
+ * Read from a model already in hand, which is what a build-time plugin does.
+ *
+ * @param model The model to serve.
+ */
+export function createMemoryReader(model: SiteModel): SourceReader {
+  return {
+    kind: 'memory',
+    async locales(): Promise<string[]> {
+      return [model.locale];
+    },
+    async read(): Promise<SiteModel> {
+      return model;
+    },
+  };
+}

package/src/model/tree.ts

@@ -0,0 +1,416 @@
+/**
+ * The page tree.
+ *
+ * Docusaurus hands over resolved sidebars — autogenerated ones already
+ * expanded — and documents that already know their permalink and their
+ * neighbours. This turns that into the tree of pages the target will mirror:
+ * one node per page, ordered as the sidebar orders them.
+ */
+
+import { titleCase, type IssueCollector } from '../util';
+import type { Doc, DocsVersion, SidebarCategoryItem, SidebarItem } from './types';
+
+/** What produced a page. */
+export type PageKind = 'root' | 'doc' | 'dir' | 'generated-index';
+
+/** One page in the tree. */
+export interface PageNode {
+  /** Path relative to the version root; '' is the root itself. */
+  path: string;
+  /** Last segment of `path`, which becomes the target's slug. */
+  slug: string;
+  /** What produced this page. */
+  kind: PageKind;
+  /** Title shown on the page and in navigation. */
+  title: string;
+  /** Description, for a generated index. */
+  description?: string | undefined;
+  /** The document backing this page, when one does. */
+  doc?: Doc | undefined;
+  /** Parent page; absent on the root. */
+  parent?: PageNode | undefined;
+  /** Children, in sidebar order. */
+  children: PageNode[];
+  /** Ten times the position among siblings, matching WordPress's convention. */
+  menuOrder: number;
+  /** Position in the flattened order; -1 on the root. */
+  navIndex: number;
+  /** Children grouped by the category that named them, for generated indexes. */
+  sections: { label?: string | undefined; children: PageNode[] }[];
+  /** Tree path of the previous page, from Docusaurus's own pagination. */
+  previousPath?: string | undefined;
+  /** Tree path of the next page. */
+  nextPath?: string | undefined;
+  /** Version this page belongs to. */
+  versionName: string;
+  /** False for a page no sidebar mentions. */
+  inSidebar: boolean;
+}
+
+/** The tree, plus the indexes callers need to look pages up. */
+export interface PageTree {
+  /** The version root. */
+  root: PageNode;
+  /** Every page except the root, in sidebar order. */
+  chain: PageNode[];
+  /** Pages by tree path. */
+  byPath: Map<string, PageNode>;
+  /** Pages by the permalink Docusaurus gave their document. */
+  byPermalink: Map<string, PageNode>;
+  /** Pages by their source file, relative to the site directory. */
+  bySourcePath: Map<string, PageNode>;
+}
+
+/** Everything needed to build one version's tree. */
+export interface BuildPageTreeInput {
+  /** The version being published. */
+  version: DocsVersion;
+  /** Sidebar ids to publish, or 'all'. */
+  sidebars: string[] | 'all';
+  /** Title for the version root when no document claims it. */
+  rootTitle: string;
+  /** Publish documents Docusaurus marked unlisted. */
+  includeUnlisted?: boolean;
+  /** Publish documents that belong to no sidebar at all. */
+  includeOrphans?: boolean;
+  /** Where skipped entries are reported. */
+  issues: IssueCollector;
+}
+
+/** The longest directory prefix shared by a set of tree paths. */
+function commonDirectory(paths: string[]): string {
+  if (paths.length === 0) return '';
+  const dirOf = (value: string): string[] => {
+    const parts = value.split('/').filter(Boolean);
+    return parts.slice(0, -1);
+  };
+  let common = dirOf(paths[0]!);
+  for (const value of paths.slice(1)) {
+    const parts = dirOf(value);
+    let index = 0;
+    while (index < common.length && index < parts.length && common[index] === parts[index]) index += 1;
+    common = common.slice(0, index);
+    if (common.length === 0) break;
+  }
+  return common.join('/');
+}
+
+/** A category, flattened. */
+interface FlatCategory {
+  label: string;
+  /** Tree path of the category's own page, when it has one. */
+  linkPath?: string | undefined;
+  /** Tree paths of every document below this category, excluding its own page. */
+  deepPaths: string[];
+  /** Tree paths of the documents directly inside it. */
+  directPaths: string[];
+}
+
+/**
+ * Walk resolved sidebars into document order and the categories that can name
+ * a directory.
+ */
+function flattenSidebars(
+  items: SidebarItem[],
+  docsById: Map<string, Doc>,
+  issues: IssueCollector,
+): { order: string[]; categories: FlatCategory[] } {
+  const order: string[] = [];
+  const categories: FlatCategory[] = [];
+
+  const pathOf = (id: string): string | undefined => docsById.get(id)?.treePath;
+
+  const walk = (nodes: SidebarItem[], deep: string[] | null, direct: string[] | null): void => {
+    for (const node of nodes) {
+      if (node.type === 'ref') {
+        // A ref points into a different sidebar. It is a cross-reference, not a
+        // page of this one, so it never becomes a page here.
+        issues.add({
+          code: 'sidebar-ref-item',
+          severity: 'info',
+          message: `The sidebar cross-references "${node.id}", which belongs to another sidebar and was not published here.`,
+        });
+        continue;
+      }
+
+      if (node.type === 'doc') {
+        const treePath = pathOf(node.id);
+        if (treePath === undefined) {
+          issues.add({
+            code: 'sidebar-doc-missing',
+            severity: 'warning',
+            message: `The sidebar lists "${node.id}", but no such document was loaded.`,
+          });
+          continue;
+        }
+        order.push(treePath);
+        deep?.push(treePath);
+        direct?.push(treePath);
+        continue;
+      }
+
+      if (node.type === 'category') {
+        const category = node as SidebarCategoryItem;
+        const deepPaths: string[] = [];
+        const directPaths: string[] = [];
+        let linkPath: string | undefined;
+
+        if (category.link?.type === 'doc') {
+          // The category's own page comes before its members. It is deliberately
+          // kept out of `deepPaths`: it sits one level above them, and counting
+          // it as a member would drag the shared directory up with it.
+          linkPath = pathOf(category.link.id);
+          if (linkPath !== undefined) order.push(linkPath);
+        }
+        walk(category.items ?? [], deepPaths, directPaths);
+        categories.push({ label: category.label, linkPath, deepPaths, directPaths });
+        // The parent does see it, because it really is one of the parent's pages.
+        if (linkPath !== undefined) deep?.push(linkPath);
+        deep?.push(...deepPaths);
+        continue;
+      }
+
+      // `link` and `html` entries are navigation furniture, not pages.
+      issues.add({
+        code: node.type === 'link' ? 'sidebar-link-item' : 'sidebar-html-item',
+        severity: 'info',
+        message:
+          node.type === 'link'
+            ? `The sidebar link "${node.label}" has no page to publish.`
+            : 'A raw HTML sidebar entry has no page to publish.',
+      });
+    }
+  };
+
+  walk(items, null, null);
+  return { order, categories };
+}
+
+/**
+ * Build one version's page tree.
+ */
+export function buildPageTree(input: BuildPageTreeInput): PageTree {
+  const { version, issues } = input;
+  const sidebarNames =
+    input.sidebars === 'all' ? Object.keys(version.sidebars) : input.sidebars;
+
+  const publishable = version.docs.filter((doc) => {
+    if (doc.unlisted && input.includeUnlisted !== true) {
+      issues.addFor(doc, {
+        code: 'doc-unlisted',
+        severity: 'info',
+        message: `"${doc.id}" is unlisted and was not published.`,
+      });
+      return false;
+    }
+    return true;
+  });
+
+  const docsById = new Map(publishable.map((doc) => [doc.id, doc]));
+
+  const order: string[] = [];
+  const categories: FlatCategory[] = [];
+  for (const name of sidebarNames) {
+    const items = version.sidebars[name];
+    if (!items) {
+      issues.add({
+        code: 'sidebar-missing',
+        severity: 'warning',
+        message: `This version has no sidebar called "${name}".`,
+      });
+      continue;
+    }
+    const flattened = flattenSidebars(items, docsById, issues);
+    order.push(...flattened.order);
+    categories.push(...flattened.categories);
+  }
+
+  // Outermost category wins a directory; the rest become section labels.
+  const claimed = new Map<string, string>();
+  const virtual: FlatCategory[] = [];
+  for (const category of categories) {
+    if (category.deepPaths.length === 0 && category.linkPath === undefined) {
+      issues.add({
+        code: 'sidebar-category-empty',
+        severity: 'info',
+        message: `The sidebar category "${category.label}" has no published documents.`,
+      });
+      continue;
+    }
+    const directory = category.linkPath ?? commonDirectory(category.deepPaths);
+    if (directory && !claimed.has(directory)) claimed.set(directory, category.label);
+    else virtual.push(category);
+  }
+
+  const byPath = new Map<string, PageNode>();
+  const root: PageNode = {
+    path: '',
+    slug: '',
+    kind: 'root',
+    title: input.rootTitle,
+    children: [],
+    menuOrder: 0,
+    navIndex: -1,
+    sections: [],
+    versionName: version.name,
+    inSidebar: true,
+  };
+  byPath.set('', root);
+
+  const parentPathOf = (value: string): string => {
+    const parts = value.split('/').filter(Boolean);
+    return parts.slice(0, -1).join('/');
+  };
+
+  const ensureDir = (dirPath: string): PageNode => {
+    if (dirPath === '') return root;
+    const existing = byPath.get(dirPath);
+    if (existing) return existing;
+    const parent = ensureDir(parentPathOf(dirPath));
+    const slug = dirPath.slice(dirPath.lastIndexOf('/') + 1);
+    const node: PageNode = {
+      path: dirPath,
+      slug,
+      kind: 'dir',
+      title: claimed.get(dirPath) ?? titleCase(slug),
+      parent,
+      children: [],
+      menuOrder: 0,
+      navIndex: -1,
+      sections: [],
+      versionName: version.name,
+      inSidebar: false,
+    };
+    byPath.set(dirPath, node);
+    return node;
+  };
+
+  // Only what a published sidebar reaches becomes a page. A document that
+  // belongs to another sidebar — a generated API reference, say — is not
+  // orphaned, it simply is not part of what was asked for.
+  const reachable = new Set(order);
+  const included = publishable.filter((doc) => {
+    if (doc.treePath === '' || reachable.has(doc.treePath)) return true;
+    if (doc.sidebarName === undefined && input.includeOrphans === true) return true;
+    return false;
+  });
+
+  // The version root can be a document too, when one has the version's own slug.
+  const rootDoc = included.find((doc) => doc.treePath === '');
+  if (rootDoc) {
+    root.doc = rootDoc;
+    if (!input.rootTitle) root.title = rootDoc.title;
+  }
+
+  for (const doc of included) {
+    if (doc.treePath === '') continue;
+    const existing = byPath.get(doc.treePath);
+    if (existing) {
+      // A directory page and a document at the same path: the document wins,
+      // which is how an `index.md` or a front-matter slug takes over a folder.
+      existing.doc = doc;
+      existing.description = doc.description;
+      if (!claimed.has(doc.treePath)) existing.title = doc.title;
+      continue;
+    }
+    const parent = ensureDir(parentPathOf(doc.treePath));
+    const slug = doc.treePath.slice(doc.treePath.lastIndexOf('/') + 1);
+    byPath.set(doc.treePath, {
+      path: doc.treePath,
+      slug,
+      kind: 'doc',
+      // A category that names this page wins the title: on WordPress the page
+      // title is also its navigation label, and the sidebar label is what a
+      // reader saw in the navigation on Docusaurus.
+      title: claimed.get(doc.treePath) ?? doc.title,
+      description: doc.description,
+      doc,
+      parent,
+      children: [],
+      menuOrder: 0,
+      navIndex: -1,
+      sections: [],
+      versionName: version.name,
+      inSidebar: false,
+    });
+  }
+
+  // Sidebar order first: placing a node places any ancestor not yet placed, so
+  // a directory lands at the position of its first member.
+  const placed = new Set<string>(['']);
+  const place = (node: PageNode): void => {
+    if (placed.has(node.path)) return;
+    const parent = node.parent ?? root;
+    place(parent);
+    parent.children.push(node);
+    placed.add(node.path);
+  };
+
+  for (const treePath of order) {
+    const node = byPath.get(treePath);
+    if (!node) continue;
+    node.inSidebar = true;
+    if (node !== root) place(node);
+  }
+
+  for (const node of [...byPath.values()].sort((a, b) => a.path.localeCompare(b.path))) {
+    if (placed.has(node.path)) continue;
+    place(node);
+    issues.addFor(node.doc, {
+      code: 'doc-not-in-sidebar',
+      severity: 'info',
+      message: `"${node.path}" is not in a published sidebar; it was placed after the pages that are.`,
+      path: node.path,
+    });
+  }
+
+  // Grouping for generated indexes.
+  const sectionLabelOf = new Map<string, string>();
+  for (const category of virtual) {
+    for (const treePath of category.directPaths) sectionLabelOf.set(treePath, category.label);
+  }
+
+  const chain: PageNode[] = [];
+  const finish = (node: PageNode): void => {
+    node.children.forEach((child, index) => {
+      child.menuOrder = index * 10;
+    });
+    const sections: PageNode['sections'] = [];
+    for (const child of node.children) {
+      const label = sectionLabelOf.get(child.path);
+      const last = sections[sections.length - 1];
+      if (last && last.label === label) last.children.push(child);
+      else sections.push({ label, children: [child] });
+    }
+    node.sections = sections;
+    for (const child of node.children) {
+      child.navIndex = chain.length;
+      chain.push(child);
+      finish(child);
+    }
+  };
+  finish(root);
+
+  // Pagination walks the pages that exist here.
+  //
+  // Docusaurus's own previous/next skip a category that has no document of its
+  // own, because on that site such a category is not a page. Here it is one:
+  // it has a URL a reader can land on, so sending them past it would be wrong.
+  chain.forEach((node, index) => {
+    const previous = chain[index - 1];
+    const next = chain[index + 1];
+    if (previous) node.previousPath = previous.path;
+    if (next) node.nextPath = next.path;
+  });
+
+  const byPermalink = new Map<string, PageNode>();
+  const bySourcePath = new Map<string, PageNode>();
+  for (const node of byPath.values()) {
+    if (!node.doc) continue;
+    byPermalink.set(node.doc.permalink, node);
+    byPermalink.set(node.doc.permalink.replace(/\/$/, ''), node);
+    bySourcePath.set(node.doc.sourceRelativePath, node);
+  }
+
+  return { root, chain, byPath, byPermalink, bySourcePath };
+}