explico 0.12.0 → 0.13.1

package/src/compiler/exports/markdown-export.ts

@@ -0,0 +1,329 @@
+import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
+import { basename, dirname, extname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import type { Manifest, ManifestLesson, ManifestPart } from '../manifest';
+import { courseFileStem } from './export-pages';
+import {
+  consoleLogger,
+  diagramsOf,
+  figuresOf,
+  imageExtension,
+  imageFile,
+  imagesOf,
+  needsCapture,
+  prepareChapters,
+  readBuiltPage,
+  type ExportLogger,
+  type PreparedChapters,
+} from './export-chapters';
+import type { CapturedFigure } from './latex/capture';
+import { checkItems } from './latex/document';
+import { serializeMarkdownNodes, serializeMarkdownPage } from './markdown/serialize';
+import type { StaticPage } from './static-tree';
+import { zip, type ZipEntry } from './zip';
+
+/**
+ * The Markdown export (`lessonExports.markdown`, decision D2): built with the
+ * site, from the same reading trees and figure capture as the LaTeX export.
+ *
+ * - `downloads/<slug>-markdown.zip`: `<name>-markdown/lesson.md` and its
+ *   `assets/`.
+ * - `downloads/part/<dir>-markdown.zip`: `index.md` (the part's page and its
+ *   contents), one file per chapter, and shared `assets/`.
+ * - `downloads/course-markdown.zip`: `index.md` (the course and its
+ *   contents), each part's page, one file per chapter, and `assets/`.
+ *
+ * A figure a browser draws is a PNG at two device pixels per CSS pixel; an
+ * image is the site's own file; a `<Diagram>` is a PNG with its boxes and
+ * arrows listed as text. Links between files of a download are relative.
+ */
+
+interface Picture {
+  readonly path: string;
+  readonly content: Uint8Array;
+}
+
+/** Every picture a page shows, by float anchor or image `src`, under `assetDir`. */
+async function pagePictures(
+  outputRoot: string,
+  base: string,
+  page: StaticPage,
+  captured: ReadonlyMap<string, CapturedFigure> | undefined,
+  assetDir: string,
+): Promise<Map<string, Picture>> {
+  const pictures = new Map<string, Picture>();
+  const copy = async (key: string, src: string, name: string) => {
+    const file = imageFile(outputRoot, base, page.route, src);
+    pictures.set(key, { path: `${assetDir}/${name}${extname(file).toLowerCase()}`, content: await readFile(file) });
+  };
+  for (const figure of figuresOf(page.blocks)) {
+    // An image file stays itself, SVG included: Markdown shows it as it is.
+    if (figure.picture.kind === 'image' && (!needsCapture(figure) || imageExtension(figure.picture.src) === '.svg')) {
+      await copy(figure.anchor, figure.picture.src, figure.anchor);
+      continue;
+    }
+    const png = captured?.get(figure.anchor)?.png;
+    if (png) pictures.set(figure.anchor, { path: `${assetDir}/${figure.anchor}.png`, content: png });
+  }
+  for (const diagram of diagramsOf(page.blocks)) {
+    const png = captured?.get(diagram.anchor)?.png;
+    if (png) pictures.set(diagram.anchor, { path: `${assetDir}/${diagram.anchor}.png`, content: png });
+  }
+  for (const image of imagesOf(page.blocks)) {
+    if (pictures.has(image.src)) continue;
+    const name = basename(new URL(image.src, 'https://x.invalid/').pathname, extname(image.src)) || 'image';
+    await copy(image.src, image.src, name);
+  }
+  return pictures;
+}
+
+function twoDigits(value: number): string {
+  return String(value).padStart(2, '0');
+}
+
+/** A chapter's file in a part's or the course's download: `02-01-projectile-range.md`. */
+function chapterFile(lesson: ManifestLesson): string {
+  return `${twoDigits(lesson.part)}-${twoDigits(lesson.chapter)}-${lesson.slug.split('/').at(-1) ?? lesson.slug}.md`;
+}
+
+/** A part's page in the course's download: `02-00-projectiles.md`. */
+function partFile(part: ManifestPart): string {
+  return `${twoDigits(part.ordinal)}-00-${part.dir}.md`;
+}
+
+/** The pages of a download, by route, with the file each is written to. */
+type DocumentFiles = ReadonlyMap<string, string>;
+
+/** The same map, seen from one file of the download: its own route maps to `''`. */
+function relativeTo(files: DocumentFiles, route: string): Map<string, string> {
+  return new Map([...files].map(([key, file]) => [key, key === route ? '' : file]));
+}
+
+export async function writeMarkdownExports(
+  directory: URL,
+  manifest: Manifest,
+  options: {
+    readonly logger?: ExportLogger;
+    readonly projectRoot?: string;
+    readonly prepared?: PreparedChapters;
+  } = {},
+): Promise<void> {
+  if (!manifest.lessonExports.markdown) return;
+  const logger = options.logger ?? consoleLogger;
+  const outputRoot = fileURLToPath(directory);
+  const { base, chapters, captures } =
+    options.prepared ?? (await prepareChapters(outputRoot, manifest, { logger, projectRoot: options.projectRoot }));
+  const canonicalBaseUrl = manifest.lessonExports.canonicalBaseUrl;
+  const course = manifest.about.material.name;
+  const version = manifest.about.knowledge.version;
+  const source = (route: string) => ({
+    course,
+    version,
+    ...(canonicalBaseUrl ? { url: `${canonicalBaseUrl}${route ? `${route}/` : ''}` } : {}),
+  });
+
+  /** One chapter's Markdown and pictures, seen from `documentFiles`. */
+  const chapterFiles = async (
+    chapter: (typeof chapters)[number],
+    file: string,
+    assetDir: string,
+    documentFiles: DocumentFiles,
+  ): Promise<ZipEntry[]> => {
+    const { lesson, page } = chapter;
+    const pictures = await pagePictures(outputRoot, base, page, captures.get(page.route), assetDir);
+    const markdown = serializeMarkdownPage(
+      {
+        page,
+        documentFiles: relativeTo(documentFiles, page.route),
+        ...(canonicalBaseUrl ? { canonicalBaseUrl } : {}),
+        pictures: new Map([...pictures].map(([key, picture]) => [key, picture.path])),
+        checkItems: checkItems(manifest, lesson),
+        source: source(page.route),
+      },
+      { number: `${lesson.part}.${lesson.chapter}` },
+    );
+    return [
+      { path: file, content: markdown },
+      ...[...pictures.values()].map((picture) => ({ path: picture.path, content: picture.content })),
+    ];
+  };
+
+  const pending: { target: string; content: Uint8Array }[] = [];
+  const pack = (target: string, folder: string, entries: readonly ZipEntry[]) => {
+    const unique = new Map(entries.map((entry) => [entry.path, entry]));
+    pending.push({
+      target: join(outputRoot, 'downloads', target),
+      content: zip([...unique.values()].map((entry) => ({ path: `${folder}/${entry.path}`, content: entry.content }))),
+    });
+  };
+
+  // Each chapter on its own.
+  for (const chapter of chapters) {
+    const name = chapter.lesson.slug.split('/').at(-1) ?? chapter.lesson.slug;
+    pack(
+      `${chapter.lesson.slug}-markdown.zip`,
+      `${name}-markdown`,
+      await chapterFiles(chapter, 'lesson.md', 'assets', new Map([[chapter.page.route, 'lesson.md']])),
+    );
+  }
+
+  const byId = new Map(chapters.map((chapter) => [chapter.lesson.id, chapter]));
+  const pageSlugs = new Set(manifest.pages.map((page) => page.slug));
+  const parts = manifest.parts
+    .map((part) => ({
+      part,
+      chapters: part.lessons.flatMap((id) => (byId.has(id) ? [byId.get(id)!] : [])),
+      overview: pageSlugs.has(`${part.dir}/index`) ? `${part.dir}/index` : undefined,
+    }))
+    .filter((entry) => entry.chapters.length > 0);
+
+  /** A part's page as Markdown, or its label alone when it has none. */
+  const overviewMarkdown = async (
+    entry: (typeof parts)[number],
+    documentFiles: DocumentFiles,
+    assetDir: string,
+  ): Promise<{ markdown: string; pictures: ZipEntry[] }> => {
+    if (!entry.overview) {
+      return { markdown: serializeMarkdownNodes([{ type: 'heading', depth: 1, children: [{ type: 'text', value: entry.part.label }] }]), pictures: [] };
+    }
+    const { page } = await readBuiltPage(outputRoot, entry.overview);
+    const pictures = await pagePictures(outputRoot, base, page, undefined, assetDir);
+    const markdown = serializeMarkdownPage(
+      {
+        page,
+        documentFiles: relativeTo(documentFiles, page.route),
+        ...(canonicalBaseUrl ? { canonicalBaseUrl } : {}),
+        pictures: new Map([...pictures].map(([key, picture]) => [key, picture.path])),
+        checkItems: new Map(),
+        source: source(page.route),
+      },
+      { number: `Part ${entry.part.ordinal}:` },
+    );
+    return { markdown, pictures: [...pictures.values()].map((picture) => ({ path: picture.path, content: picture.content })) };
+  };
+
+  const contents = (entry: (typeof parts)[number], files: DocumentFiles) =>
+    entry.chapters.map((chapter) => ({
+      type: 'listItem' as const,
+      spread: false,
+      children: [
+        {
+          type: 'paragraph' as const,
+          children: [
+            {
+              type: 'link' as const,
+              url: files.get(chapter.page.route) ?? '',
+              children: [{ type: 'text' as const, value: `${chapter.lesson.part}.${chapter.lesson.chapter} ${chapter.page.title}` }],
+            },
+          ],
+        },
+      ],
+    }));
+
+  // Each part: its page as index.md, then its chapters.
+  for (const entry of parts) {
+    const files: DocumentFiles = new Map([
+      ...(entry.overview ? [[entry.part.dir, 'index.md'] as const] : []),
+      ...entry.chapters.map((chapter) => [chapter.page.route, chapterFile(chapter.lesson)] as const),
+    ]);
+    const overview = await overviewMarkdown(entry, files, 'assets/index');
+    const index =
+      `${overview.markdown}\n` +
+      serializeMarkdownNodes([
+        { type: 'heading', depth: 2, children: [{ type: 'text', value: 'Contents' }] },
+        { type: 'list', ordered: false, spread: false, children: contents(entry, files) },
+      ]);
+    const entries: ZipEntry[] = [{ path: 'index.md', content: index }, ...overview.pictures];
+    for (const chapter of entry.chapters) {
+      const file = chapterFile(chapter.lesson);
+      entries.push(...(await chapterFiles(chapter, file, `assets/${file.replace(/\.md$/u, '')}`, files)));
+    }
+    pack(`part/${entry.part.dir}-markdown.zip`, `${entry.part.dir}-markdown`, entries);
+  }
+
+  // The course: index.md, then each part's page and chapters.
+  if (parts.length > 0) {
+    const files: DocumentFiles = new Map([
+      ['', 'index.md'],
+      ...parts.flatMap((entry) => [
+        ...(entry.overview ? [[entry.part.dir, partFile(entry.part)] as const] : []),
+        ...entry.chapters.map((chapter) => [chapter.page.route, chapterFile(chapter.lesson)] as const),
+      ]),
+    ]);
+    const entries: ZipEntry[] = [];
+    for (const entry of parts) {
+      if (entry.overview) {
+        const file = partFile(entry.part);
+        const overview = await overviewMarkdown(entry, files, `assets/${file.replace(/\.md$/u, '')}`);
+        entries.push({ path: file, content: overview.markdown }, ...overview.pictures);
+      }
+      for (const chapter of entry.chapters) {
+        const file = chapterFile(chapter.lesson);
+        entries.push(...(await chapterFiles(chapter, file, `assets/${file.replace(/\.md$/u, '')}`, files)));
+      }
+    }
+    const index = serializeMarkdownNodes([
+      { type: 'heading', depth: 1, children: [{ type: 'text', value: course }] },
+      {
+        type: 'paragraph',
+        children: [
+          { type: 'text', value: `Content version ${version}.` },
+          ...(canonicalBaseUrl
+            ? [
+                { type: 'text' as const, value: ' Interactive version: ' },
+                { type: 'link' as const, url: canonicalBaseUrl, children: [{ type: 'text' as const, value: canonicalBaseUrl }] },
+                { type: 'text' as const, value: '.' },
+              ]
+            : []),
+        ],
+      },
+      ...(manifest.about.material.summary
+        ? [{ type: 'paragraph' as const, children: [{ type: 'text' as const, value: manifest.about.material.summary }] }]
+        : []),
+      { type: 'heading', depth: 2, children: [{ type: 'text', value: 'Contents' }] },
+      {
+        type: 'list',
+        ordered: false,
+        spread: false,
+        children: parts.map((entry) => ({
+          type: 'listItem' as const,
+          spread: false,
+          children: [
+            {
+              type: 'paragraph' as const,
+              children: entry.overview
+                ? [
+                    {
+                      type: 'link' as const,
+                      url: partFile(entry.part),
+                      children: [{ type: 'text' as const, value: `Part ${entry.part.ordinal}: ${entry.part.label}` }],
+                    },
+                  ]
+                : [{ type: 'text' as const, value: `Part ${entry.part.ordinal}: ${entry.part.label}` }],
+            },
+            { type: 'list' as const, ordered: false, spread: false, children: contents(entry, files) },
+          ],
+        })),
+      },
+    ]);
+    entries.push({ path: 'index.md', content: index });
+    pack('course-markdown.zip', `${courseFileStem(manifest)}-markdown`, entries);
+  }
+
+  // Nothing is published until every file is ready.
+  try {
+    for (const file of pending) {
+      await mkdir(dirname(file.target), { recursive: true });
+      await writeFile(`${file.target}.tmp-${process.pid}`, file.content);
+    }
+    for (const file of pending) await rename(`${file.target}.tmp-${process.pid}`, file.target);
+  } finally {
+    await Promise.all(pending.map((file) => rm(`${file.target}.tmp-${process.pid}`, { force: true })));
+  }
+  logger.info(
+    `Markdown export: ${chapters.length} chapter(s), ${parts.length} part(s)` +
+      (parts.length > 0 ? ' and the course' : '') +
+      '.',
+  );
+}

package/src/compiler/exports/standalone-html.ts

@@ -9,19 +9,21 @@ import {
 import { Buffer } from 'node:buffer';
 import { dirname, extname, join, relative, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import {
-  parse,
-  serialize,
-  type DefaultTreeAdapterMap,
-} from 'parse5';
+import { parse, serialize } from 'parse5';
 
 import type { Manifest } from '../manifest';
-
-type HtmlNode = DefaultTreeAdapterMap['node'];
-type HtmlParent = DefaultTreeAdapterMap['parentNode'];
-type HtmlChild = DefaultTreeAdapterMap['childNode'];
-type HtmlElement = DefaultTreeAdapterMap['element'];
-type HtmlText = DefaultTreeAdapterMap['textNode'];
+import { exportPageRoute, standalonePageSlugs } from './export-pages';
+import {
+  attribute,
+  elements,
+  hasClass,
+  inferBasePrefix,
+  type HtmlChild,
+  type HtmlElement,
+  type HtmlNode,
+  type HtmlParent,
+  type HtmlText,
+} from './html-tree';
 
 const HTML_NAMESPACE =
   'http://www.w3.org/1999/xhtml' as HtmlElement['namespaceURI'];
@@ -115,11 +117,12 @@ const STANDALONE_THEME_SCRIPT = `(() => {
   apply('auto');
 })();`;
 
-/** Inputs for producing one standalone lesson from Astro's final HTML. */
+/** Inputs for producing one standalone page from Astro's final HTML. */
 export interface StandaloneLessonInput {
   readonly html: string;
   readonly sourceFile: string;
   readonly outputRoot: string;
+  /** The page's docs slug: a chapter's, or `index`, `motion/index`, `glossary`. */
   readonly lessonSlug: string;
   readonly canonicalBaseUrl?: string;
 }
@@ -143,45 +146,12 @@ interface ExportContext {
   readonly assetData: Map<string, string>;
 }
 
-function isElement(node: HtmlNode): node is HtmlElement {
-  return 'tagName' in node;
-}
-
-function childrenOf(node: HtmlNode): readonly HtmlChild[] {
-  return 'childNodes' in node ? node.childNodes : [];
-}
-
-function walk(node: HtmlNode, visit: (node: HtmlNode) => void): void {
-  visit(node);
-  for (const child of childrenOf(node)) walk(child, visit);
-  if (isElement(node) && node.tagName === 'template') {
-    const template = node as DefaultTreeAdapterMap['template'];
-    for (const child of template.content.childNodes) walk(child, visit);
-  }
-}
-
-function elements(root: HtmlNode, predicate?: (node: HtmlElement) => boolean): HtmlElement[] {
-  const found: HtmlElement[] = [];
-  walk(root, (node) => {
-    if (isElement(node) && (predicate?.(node) ?? true)) found.push(node);
-  });
-  return found;
-}
-
 function firstElement(root: HtmlNode, tagName: string): HtmlElement {
   const found = elements(root, (node) => node.tagName === tagName)[0];
   if (!found) throw new Error(`Standalone HTML source has no <${tagName}> element.`);
   return found;
 }
 
-function attribute(node: HtmlElement, name: string): string | undefined {
-  return node.attrs.find((entry) => entry.name === name)?.value;
-}
-
-function hasClass(node: HtmlElement, name: string): boolean {
-  return (attribute(node, 'class') ?? '').split(/\s+/u).includes(name);
-}
-
 function setAttribute(node: HtmlElement, name: string, value = ''): void {
   const existing = node.attrs.find((entry) => entry.name === name);
   if (existing) existing.value = value;
@@ -489,7 +459,7 @@ async function bundleBrowserGraph(
   });
   const javascript = result.outputFiles.find((file) => file.path.endsWith('.js'));
   if (!javascript) {
-    throw new Error('Browser bundler produced no JavaScript for the standalone lesson.');
+    throw new Error('Browser bundler produced no JavaScript for the standalone page.');
   }
   const unsupported = result.outputFiles.filter(
     (file) => !file.path.endsWith('.js') && !file.path.endsWith('.css'),
@@ -509,18 +479,6 @@ async function bundleBrowserGraph(
   };
 }
 
-function inferBasePrefix(document: HtmlNode): string {
-  for (const node of elements(document)) {
-    for (const name of ['href', 'src', 'component-url', 'renderer-url']) {
-      const value = attribute(node, name);
-      if (!value?.startsWith('/')) continue;
-      const marker = value.indexOf('/astro-assets/');
-      if (marker !== -1) return `${value.slice(0, marker)}/`;
-    }
-  }
-  return '/';
-}
-
 function normalizedPagePath(pathname: string): string {
   return pathname.replace(/\/index\.html$/iu, '/').replace(/\.html$/iu, '').replace(/\/+$/u, '');
 }
@@ -538,14 +496,17 @@ function rewriteCourseLink(
     throw new Error(`Standalone HTML cannot publish local navigation URL ${JSON.stringify(href)}.`);
   }
 
-  const pagePath = `${basePrefix}${lessonSlug}/`.replace(/\/{2,}/gu, '/');
+  const pagePath = `${basePrefix}${exportPageRoute(lessonSlug)}/`.replace(
+    /\/{2,}/gu,
+    '/',
+  );
   const resolved = new URL(trimmed, `https://explico.invalid${pagePath}`);
   if (normalizedPagePath(resolved.pathname) === normalizedPagePath(pagePath)) {
     return `${resolved.search}${resolved.hash}` || '#_top';
   }
   if (!canonicalBaseUrl) {
     throw new Error(
-      `Lesson ${lessonSlug} links outside itself (${JSON.stringify(href)}), so ` +
+      `Page ${lessonSlug} links outside itself (${JSON.stringify(href)}), so ` +
         '`lessonExports.canonicalBaseUrl` is required for standalone HTML.',
     );
   }
@@ -900,14 +861,14 @@ function auditStandaloneHtml(html: string, lessonSlug: string): void {
   }
 }
 
-/** Build one fully inlined lesson document without writing it. */
+/** Build one fully inlined page document without writing it. */
 export async function buildStandaloneLessonHtml(
   input: StandaloneLessonInput,
 ): Promise<string> {
   const outputRoot = resolve(input.outputRoot);
   const sourceFile = resolve(input.sourceFile);
   if (!insideRoot(sourceFile, outputRoot)) {
-    throw new Error('Standalone lesson source must be inside Astro build output.');
+    throw new Error('Standalone page source must be inside Astro build output.');
   }
   const context: ExportContext = {
     outputRoot,
@@ -936,8 +897,11 @@ export async function buildStandaloneLessonHtml(
   return html;
 }
 
-async function lessonSourceFile(outputRoot: string, slug: string): Promise<string> {
-  const candidates = [join(outputRoot, slug, 'index.html'), join(outputRoot, `${slug}.html`)];
+/** Astro's built HTML file of a docs slug. */
+export async function pageSourceFile(outputRoot: string, slug: string): Promise<string> {
+  const route = exportPageRoute(slug);
+  const candidates = [join(outputRoot, route, 'index.html')];
+  if (route) candidates.push(join(outputRoot, `${route}.html`));
   for (const candidate of candidates) if (await isFile(candidate)) return candidate;
   throw new Error(
     `Standalone HTML could not find Astro's built page for ${slug}; tried ` +
@@ -945,7 +909,12 @@ async function lessonSourceFile(outputRoot: string, slug: string): Promise<strin
   );
 }
 
-/** Emit all opt-in lesson HTML artifacts from Astro's completed static output. */
+/**
+ * Emit the opt-in standalone HTML artifact of every chapter
+ * (`standalonePageSlugs`) from Astro's completed static output, at
+ * `downloads/<docs slug>.html`. A page that is not a chapter is in its
+ * part's and the course's files (`combined-html.ts`).
+ */
 export async function writeStandaloneLessonExports(
   directory: URL,
   manifest: Manifest,
@@ -953,9 +922,9 @@ export async function writeStandaloneLessonExports(
   if (!manifest.lessonExports.standaloneHtml) return;
   const outputRoot = fileURLToPath(directory);
   const pending: { target: string; temporary: string; html: string }[] = [];
-  for (const lesson of manifest.lessons) {
-    const sourceFile = await lessonSourceFile(outputRoot, lesson.slug);
-    const target = join(outputRoot, 'downloads', `${lesson.slug}.html`);
+  for (const slug of standalonePageSlugs(manifest)) {
+    const sourceFile = await pageSourceFile(outputRoot, slug);
+    const target = join(outputRoot, 'downloads', `${slug}.html`);
     pending.push({
       target,
       temporary: `${target}.tmp-${process.pid}`,
@@ -963,13 +932,13 @@ export async function writeStandaloneLessonExports(
         html: await readFile(sourceFile, 'utf8'),
         sourceFile,
         outputRoot,
-        lessonSlug: lesson.slug,
+        lessonSlug: slug,
         canonicalBaseUrl: manifest.lessonExports.canonicalBaseUrl,
       }),
     });
   }
 
-  // Nothing is published until every lesson has passed the asset audit.
+  // Nothing is published until every page has passed the asset audit.
   for (const artifact of pending) {
     await mkdir(dirname(artifact.target), { recursive: true });
     await writeFile(artifact.temporary, artifact.html, 'utf8');