@docpensieve/core 0.1.5 → 0.2.0

package/README.md

@@ -1,3 +1,7 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/Juniors017/docpensieve/main/branding/logo.jpg" alt="DocPensieve — Documentation &amp; Magical Memory" width="180">
+</p>
+
 # @docpensieve/core
 
 > Generation engine

package/client/search.js

@@ -0,0 +1,205 @@
+/// <reference lib="dom" />
+/**
+ * Search, in the reader's browser: the one script a DocPensieve site loads,
+ * and only on its search page.
+ *
+ * The build does the heavy work. It writes an index of each version, and a
+ * search page that already lists every page. This script only reads the
+ * query, filters that list against the index and shows an excerpt. Without
+ * it, the page stays the full list: search adds to the site, nothing depends
+ * on it.
+ *
+ * The functions below are exported so that the ranking can be tested outside
+ * a browser; the page wiring only runs where there is a document. The DOM
+ * types are referenced here, the only file that needs them.
+ */
+
+/**
+ * A page of the index, as the build writes it.
+ *
+ * @typedef {object} IndexEntry
+ * @property {string} title
+ * @property {string} url
+ * @property {string} description
+ * @property {string} text Plain text of the page.
+ */
+
+/**
+ * Lowercases and strips accents, so that "ete" finds "été".
+ *
+ * An accented letter written as one character stays one character: the
+ * normalised text keeps the offsets of the original, which the excerpt
+ * relies on.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function normalize(text) {
+  return text
+    .normalize('NFD')
+    .replace(/\p{Diacritic}/gu, '')
+    .toLowerCase();
+}
+
+/**
+ * @param {string} query
+ * @returns {string[]} The distinct words of the query.
+ */
+export function tokenize(query) {
+  return [
+    ...new Set(
+      normalize(query)
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter(Boolean),
+    ),
+  ];
+}
+
+/**
+ * The pages that hold every word of the query, best first.
+ *
+ * A word in the title weighs more than one in the description, which weighs
+ * more than one in the text; repetitions in the text count up to five, so
+ * that a long page does not win by length alone.
+ *
+ * @param {IndexEntry[]} entries
+ * @param {string} query
+ * @returns {IndexEntry[]}
+ */
+export function rank(entries, query) {
+  const words = tokenize(query);
+  if (words.length === 0) return [];
+
+  /** @type {{ entry: IndexEntry, score: number }[]} */
+  const scored = [];
+  for (const entry of entries) {
+    const title = normalize(entry.title);
+    const description = normalize(entry.description);
+    const text = normalize(entry.text);
+
+    let score = 0;
+    let everyWord = true;
+    for (const word of words) {
+      const inTitle = title.includes(word);
+      const inDescription = description.includes(word);
+      const inText = text.split(word).length - 1;
+      if (!inTitle && !inDescription && inText === 0) {
+        everyWord = false;
+        break;
+      }
+      score += (inTitle ? 10 : 0) + (inDescription ? 3 : 0) + Math.min(inText, 5);
+    }
+    if (everyWord) scored.push({ entry, score });
+  }
+
+  return scored.sort((a, b) => b.score - a.score).map((item) => item.entry);
+}
+
+/**
+ * The passage of a text around the first word of the query found in it.
+ *
+ * @param {string} text
+ * @param {string} query
+ * @param {number} [radius] Characters kept on each side.
+ * @returns {{ before: string, match: string, after: string } | null}
+ */
+export function excerpt(text, query, radius = 80) {
+  const normalized = normalize(text);
+  for (const word of tokenize(query)) {
+    const at = normalized.indexOf(word);
+    if (at < 0) continue;
+    const start = Math.max(0, at - radius);
+    const end = Math.min(text.length, at + word.length + radius);
+    return {
+      before: (start > 0 ? '…' : '') + text.slice(start, at),
+      match: text.slice(at, at + word.length),
+      after: text.slice(at + word.length, end) + (end < text.length ? '…' : ''),
+    };
+  }
+  return null;
+}
+
+/** Wires the search page: reads the query, filters the list, shows excerpts. */
+async function init() {
+  const form = /** @type {HTMLFormElement | null} */ (
+    document.querySelector('form[data-search-page]')
+  );
+  const input = /** @type {HTMLInputElement | null | undefined} */ (
+    form?.querySelector('input[name="q"]')
+  );
+  const list = /** @type {HTMLElement | null} */ (document.querySelector('[data-search-results]'));
+  const status = /** @type {HTMLElement | null} */ (document.querySelector('[data-search-status]'));
+  if (!form || !input || !list || !status) return;
+
+  /** @type {Map<string, HTMLElement>} */
+  const items = new Map();
+  const listed = /** @type {NodeListOf<HTMLElement>} */ (list.querySelectorAll('li[data-url]'));
+  for (const item of listed) items.set(item.dataset.url ?? '', item);
+
+  /** @type {IndexEntry[]} */
+  let entries;
+  try {
+    const response = await fetch(form.dataset.index ?? '');
+    entries = await response.json();
+  } catch {
+    status.textContent = 'The search index could not be loaded: every page is listed below.';
+    return;
+  }
+
+  /** @param {HTMLElement} item */
+  const reset = (item) => {
+    item.hidden = false;
+    item.querySelector('.dp-search-excerpt')?.remove();
+  };
+
+  const show = () => {
+    const query = input.value.trim();
+
+    // Without a query, the page is the full list it was without the script.
+    if (!query) {
+      for (const item of items.values()) {
+        reset(item);
+        list.append(item);
+      }
+      status.textContent = `${items.size} pages.`;
+      return;
+    }
+
+    const found = rank(entries, query);
+    for (const item of items.values()) item.hidden = true;
+    for (const entry of found) {
+      const item = items.get(entry.url);
+      if (!item) continue;
+      reset(item);
+      const passage = excerpt(entry.text, query);
+      if (passage) {
+        const paragraph = document.createElement('p');
+        paragraph.className = 'dp-search-excerpt';
+        const mark = document.createElement('mark');
+        mark.textContent = passage.match;
+        paragraph.append(passage.before, mark, passage.after);
+        item.append(paragraph);
+      }
+      // Appending moves the item: the list ends up best first.
+      list.append(item);
+    }
+
+    status.textContent =
+      found.length === 0
+        ? `No page matches “${query}”.`
+        : `${found.length} ${found.length === 1 ? 'page' : 'pages'} for “${query}”.`;
+  };
+
+  input.value = new URLSearchParams(location.search).get('q') ?? '';
+  input.addEventListener('input', show);
+  // Searching again needs no reload, and the address follows the query, so
+  // that a search can be shared.
+  form.addEventListener('submit', (event) => {
+    event.preventDefault();
+    history.replaceState(null, '', `?q=${encodeURIComponent(input.value.trim())}`);
+    show();
+  });
+  show();
+}
+
+if (typeof document !== 'undefined') init();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docpensieve/core",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "description": "DocPensieve engine: loading, MDX compilation, structured data, site generation",
   "type": "module",
   "license": "MIT",
@@ -16,11 +16,12 @@
   },
   "files": [
     "src",
+    "client",
     "templates",
     "types"
   ],
   "dependencies": {
-    "@docpensieve/shared": "0.1.5",
+    "@docpensieve/shared": "0.2.0",
     "@mdx-js/mdx": "^3.1.1",
     "@shikijs/rehype": "^4.4.3",
     "gray-matter": "^4.0.3",
@@ -43,7 +44,7 @@
     "url": "git+https://github.com/Juniors017/docpensieve.git",
     "directory": "packages/core"
   },
-  "homepage": "https://github.com/Juniors017/docpensieve#readme",
+  "homepage": "https://docpensieve.com",
   "bugs": {
     "url": "https://github.com/Juniors017/docpensieve/issues"
   },

package/src/compiler.js

@@ -8,6 +8,8 @@
  * @module @docpensieve/core/compiler
  */
 
+import { closeSync, openSync, readSync } from 'node:fs';
+import path from 'node:path';
 import { createElement } from 'react';
 import { renderToStaticMarkup } from 'react-dom/server';
 import * as runtime from 'react/jsx-runtime';
@@ -17,6 +19,8 @@ import { evaluate } from '@mdx-js/mdx';
 import rehypeShiki from '@shikijs/rehype';
 import remarkGfm from 'remark-gfm';
 
+import { imageSize } from './image-size.js';
+
 /**
  * Default Shiki themes: the dual theme follows dark mode without JS.
  *
@@ -256,6 +260,75 @@ function rehypeSiteLinks({ url, dirUrl, basePath }) {
   };
 }
 
+/**
+ * Prepares the images of a page for the browser: their dimensions, read from
+ * their file, so that the text does not jump when they arrive; and a lazy
+ * loading for all but the first, which is often in view — and which React
+ * then stops preloading, as it does for every image not loaded lazily.
+ *
+ * It runs before the targets are rewritten, while `src` is still the path
+ * the author wrote next to the page.
+ *
+ * @param {{ filepath?: string, sourceDir?: string }} context `sourceDir` is
+ *   the version's folder, from which an absolute `src` starts.
+ * @returns {() => (tree: any) => void}
+ */
+function rehypeImages({ filepath, sourceDir }) {
+  /** @param {string} src @returns {string | null} */
+  const fileOf = (src) => {
+    let clean;
+    try {
+      clean = decodeURI(src.split('?')[0].split('#')[0]);
+    } catch {
+      return null;
+    }
+    if (clean.startsWith('/')) return sourceDir ? path.join(sourceDir, clean) : null;
+    return filepath ? path.resolve(path.dirname(filepath), clean) : null;
+  };
+
+  /** @param {string} file */
+  const sizeOf = (file) => {
+    try {
+      const handle = openSync(file, 'r');
+      try {
+        const bytes = Buffer.alloc(65536);
+        const read = readSync(handle, bytes, 0, bytes.length, 0);
+        return imageSize(bytes.subarray(0, read), path.extname(file));
+      } finally {
+        closeSync(handle);
+      }
+    } catch {
+      // A missing image is the link check's business, not this one's.
+      return null;
+    }
+  };
+
+  return () => (tree) => {
+    let first = true;
+    walk(tree, (node) => {
+      if (node.type !== 'element' || node.tagName !== 'img') return;
+      const properties = (node.properties ??= {});
+
+      const src = properties.src;
+      if (typeof src === 'string' && src !== '' && !EXTERNAL_TARGET.test(src)) {
+        const file = fileOf(src);
+        const size = file ? sizeOf(file) : null;
+        if (size && properties.width === undefined && properties.height === undefined) {
+          properties.width = size.width;
+          properties.height = size.height;
+        }
+      }
+
+      if (first) {
+        first = false;
+        return;
+      }
+      properties.loading ??= 'lazy';
+      properties.decoding ??= 'async';
+    });
+  };
+}
+
 /**
  * Nests a flat list of headings into a tree, by depth.
  *
@@ -329,20 +402,21 @@ export class Compiler {
    * Compiles a source into an HTML fragment and a table of contents.
    *
    * @param {string} source Markdown/MDX content, frontmatter already removed.
-   * @param {{ filepath?: string, url?: string, dirUrl?: string, basePath?: string }} [context]
+   * @param {{ filepath?: string, url?: string, dirUrl?: string, basePath?: string, sourceDir?: string }} [context]
    *   `filepath` locates errors, `dirUrl` is the base of relative targets and
    *   `basePath` prefixes absolute targets (the version root).
    * @returns {Promise<CompileResult>}
    * @throws {CompileError} Invalid syntax, or a component unknown at use.
    */
   async compile(source, context = {}) {
-    const { filepath, url, dirUrl, basePath } = context;
+    const { filepath, url, dirUrl, basePath, sourceDir } = context;
 
     /** @type {{ id: string, text: string, depth: number }[]} */
     const headings = [];
 
     const rehypePlugins = [
       rehypeHeadingIds(headings),
+      rehypeImages({ filepath, sourceDir }),
       rehypeSiteLinks({ url, dirUrl, basePath }),
       rehypeTableScroll(),
       ...(this.highlight ? [[rehypeShiki, this.highlight]] : []),

package/src/config.js

@@ -13,7 +13,6 @@ import {
   CONFIG_FILENAMES,
   ConfigError,
   DEFAULT_OUT_DIR,
-  NotImplementedError,
   THEME_FRAMEWORKS,
 } from '@docpensieve/shared';
 
@@ -26,6 +25,8 @@ import {
  * @property {boolean} [archived] Version kept but no longer maintained.
  * @property {boolean} [prerelease] Version in preparation, not yet the
  *   reference one. Its pages carry a notice and are not indexed.
+ * @property {string} [logo]    Logo of this version, instead of the project's.
+ * @property {string} [favicon] Favicon of this version, instead of the project's.
  */
 
 /**
@@ -35,7 +36,7 @@ import {
  * @property {string} baseUrl     Deployment prefix, slashes included.
  * @property {string} outDir      Output folder, relative to the root.
  * @property {Version[]} versions At least one.
- * @property {{ framework: string, darkMode?: string, tokens?: Record<string, string>, css?: string, source?: string }} theme
+ * @property {{ framework: string, darkMode?: string, toggle?: boolean, tokens?: Record<string, string>, css?: string, source?: string }} theme
  * @property {string} sidebar     `'auto'`, or the path of a description.
  * @property {boolean} globalComponents
  * @property {boolean} scrollToTop Back-to-top button on every page.
@@ -43,6 +44,9 @@ import {
  * @property {string} [logo]        Image beside the project name, in the header.
  * @property {string} [favicon]     Icon of the browser tab: `.ico`, `.png` or `.svg`.
  * @property {string} [socialImage] Preview of a shared page. Needs `siteUrl`.
+ * @property {boolean} [sitemap] `sitemap.xml` of the published versions.
+ * @property {boolean} [feed] RSS feed of the dated pages of the current version.
+ * @property {boolean} [search] Search field, index and search page of each version.
  * @property {string} [rootDir]   Project root, set by `loadConfig`.
  * @property {string} [configFile] Path of the configuration file, set by `loadConfig`.
  * @property {string} [lang]      Document language, `'en'` by default.
@@ -63,7 +67,8 @@ export const DEFAULT_CONFIG = Object.freeze({
   baseUrl: '/',
   outDir: DEFAULT_OUT_DIR,
   versions: [],
-  theme: { framework: 'tailwind', darkMode: 'class' },
+  // The light / dark switch is on unless the project turns it off (ADR-014).
+  theme: { framework: 'tailwind', darkMode: 'class', toggle: true },
   sidebar: 'auto',
   globalComponents: true,
   scrollToTop: true,
@@ -71,8 +76,18 @@ export const DEFAULT_CONFIG = Object.freeze({
   logo: '',
   favicon: '',
   socialImage: '',
+  // On by default, but only written once siteUrl is set: it lists absolute
+  // addresses.
+  sitemap: true,
+  // Off by default: most documentation pages carry no date, and a feed that
+  // is always empty would be announced in every page.
+  feed: false,
+  search: true,
 });
 
+/** Values of `theme.darkMode`. */
+const DARK_MODES = ['class', 'dark', 'light'];
+
 /**
  * Extensions accepted for each project image, and what to do otherwise.
  *
@@ -163,6 +178,19 @@ export function normalizeConfig(userConfig) {
         hint: 'Letters, digits, dot, dash and underscore, starting with a letter or a digit — "v1.0", "next".',
       });
     }
+    // A version may carry its own logo and favicon — a beta told apart at a
+    // glance. Checked like the project's.
+    for (const field of /** @type {const} */ (['logo', 'favicon'])) {
+      const value = version[field];
+      if (value === undefined) continue;
+      const { extensions, hint } = IMAGE_KINDS[field];
+      if (typeof value !== 'string' || !extensions.includes(path.extname(value).toLowerCase())) {
+        throw new ConfigError(
+          `The ${field} of version "${version.slug}" must be a ${extensions.join(', ')} file: "${String(value)}".`,
+          { hint },
+        );
+      }
+    }
     if (seen.has(version.slug)) {
       throw new ConfigError(`The version slug "${version.slug}" is declared twice.`);
     }
@@ -193,13 +221,23 @@ export function normalizeConfig(userConfig) {
     config.versions[0].current = true;
   }
 
-  // The sidebar is derived from the file tree, and nothing else is written
-  // yet. Accepting a path without reading it would suggest it is used.
+  // 'auto' derives the sidebar from the file tree. Anything else names a JSON
+  // description, which the generator reads from each version's folder: each
+  // version has its own pages, so its own menu.
   if (config.sidebar !== 'auto') {
-    throw new NotImplementedError(
-      `A sidebar described by a file ("${config.sidebar}")`,
-      '4.2 Explicit sidebar',
-    );
+    const file = typeof config.sidebar === 'string' ? config.sidebar : '';
+    if (
+      !file.toLowerCase().endsWith('.json') ||
+      path.isAbsolute(file) ||
+      file.split('/').includes('..')
+    ) {
+      throw new ConfigError(
+        `sidebar must be 'auto' or a .json file within each version folder: "${String(config.sidebar)}".`,
+        {
+          hint: "For instance sidebar: 'sidebar.json', read as docs/v1.0/sidebar.json for that version.",
+        },
+      );
+    }
   }
 
   // siteUrl feeds everything that must be absolute: canonical, JSON-LD.
@@ -251,12 +289,46 @@ export function normalizeConfig(userConfig) {
       });
     }
   }
+  // Both list absolute addresses. Asked for explicitly without siteUrl, they
+  // could only be written wrong; left to their default, they wait for it.
+  for (const field of /** @type {const} */ (['sitemap', 'feed'])) {
+    if (typeof config[field] !== 'boolean') {
+      throw new ConfigError(`${field} must be true or false.`, {
+        hint: `For instance ${field}: true.`,
+      });
+    }
+    if (userConfig[field] === true && !config.siteUrl) {
+      throw new ConfigError(`${field} needs siteUrl.`, {
+        hint: 'It lists absolute addresses: set siteUrl, the public address of the site.',
+      });
+    }
+  }
+
+  if (typeof config.search !== 'boolean') {
+    throw new ConfigError('search must be true or false.', { hint: 'For instance search: false.' });
+  }
+
   if (config.socialImage && !config.siteUrl) {
     throw new ConfigError('socialImage needs siteUrl.', {
       hint: 'Social networks only read an absolute address: set siteUrl, the public address of the site.',
     });
   }
 
+  // 'class' follows the reader's system, and a dark or light class on <html>
+  // wins; 'dark' and 'light' set that class at build time, for a site that
+  // keeps one look whatever the system.
+  if (!DARK_MODES.includes(config.theme.darkMode ?? 'class')) {
+    throw new ConfigError(`Unknown darkMode: "${config.theme.darkMode}".`, {
+      hint: `Accepted values: ${DARK_MODES.join(', ')}.`,
+    });
+  }
+
+  if (config.theme.toggle !== undefined && typeof config.theme.toggle !== 'boolean') {
+    throw new ConfigError('theme.toggle must be true or false.', {
+      hint: 'true adds a light / dark button to the header, with a few lines of inline script.',
+    });
+  }
+
   if (!THEME_FRAMEWORKS.includes(config.theme.framework)) {
     throw new ConfigError(`Unknown theme framework: "${config.theme.framework}".`, {
       hint: `Accepted values: ${THEME_FRAMEWORKS.join(', ')}.`,

package/src/discovery.js

@@ -0,0 +1,128 @@
+/**
+ * What a site offers the programs that read it: the sitemap for search
+ * engines, `robots.txt` that points to it, and the RSS feed of dated pages.
+ *
+ * Pure functions: the generator gathers the published pages and writes the
+ * files.
+ *
+ * @module @docpensieve/core/discovery
+ */
+
+import { toISODate } from './structured-data.js';
+
+/**
+ * A page as the site publishes it.
+ *
+ * @typedef {object} PublishedPage
+ * @property {string} url Page URL, deployment prefix included
+ *   (`/docs/versions/v1.0/guide/`).
+ * @property {Record<string, any>} frontmatter Frontmatter of its source.
+ */
+
+/**
+ * Escapes a value for an XML text or attribute.
+ *
+ * @param {unknown} value
+ * @returns {string}
+ */
+function escapeXml(value) {
+  return String(value)
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+    .replaceAll("'", '&apos;');
+}
+
+/**
+ * Builds `sitemap.xml`.
+ *
+ * `lastmod` is the page's `modified` date, or failing that its `date`; a page
+ * that carries neither is listed without one rather than with a made-up date.
+ *
+ * @param {PublishedPage[]} pages Pages of the versions to list.
+ * @param {string} siteUrl Public address of the site: the sitemap only holds
+ *   absolute addresses.
+ * @returns {string}
+ */
+export function buildSitemap(pages, siteUrl) {
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+  ];
+  for (const page of pages) {
+    const modified = toISODate(page.frontmatter.modified) ?? toISODate(page.frontmatter.date);
+    lines.push('  <url>', `    <loc>${escapeXml(new URL(page.url, siteUrl).href)}</loc>`);
+    if (modified) lines.push(`    <lastmod>${modified}</lastmod>`);
+    lines.push('  </url>');
+  }
+  lines.push('</urlset>', '');
+  return lines.join('\n');
+}
+
+/**
+ * Builds `robots.txt`, which lets every crawler in and names the sitemap.
+ *
+ * @param {string} sitemapUrl Absolute address of the sitemap.
+ * @returns {string}
+ */
+export function buildRobots(sitemapUrl) {
+  return ['User-agent: *', 'Allow: /', `Sitemap: ${sitemapUrl}`, ''].join('\n');
+}
+
+/**
+ * Builds the RSS feed of the dated pages, newest first.
+ *
+ * Only a page with a `date` enters it: a documentation page without one is
+ * reference material, not news, and dating it at build time would announce
+ * every page again at every build.
+ *
+ * @param {PublishedPage[]} pages Pages of the current version.
+ * @param {{
+ *   projectName: string, siteUrl: string, homeUrl: string, feedUrl: string, lang?: string,
+ * }} site `homeUrl` and `feedUrl` are absolute.
+ * @returns {string}
+ */
+export function buildFeed(pages, site) {
+  /** @type {{ page: PublishedPage, date: string }[]} */
+  const dated = [];
+  for (const page of pages) {
+    const date = toISODate(page.frontmatter.date);
+    if (date) dated.push({ page, date });
+  }
+  // ISO dates sort as strings; newest first, as a feed reader expects.
+  dated.sort((a, b) => (a.date < b.date ? 1 : a.date > b.date ? -1 : 0));
+
+  /** @param {string} iso */
+  const rfc822 = (iso) => new Date(`${iso}T00:00:00Z`).toUTCString();
+
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">',
+    '  <channel>',
+    `    <title>${escapeXml(site.projectName)}</title>`,
+    `    <link>${escapeXml(site.homeUrl)}</link>`,
+    `    <description>${escapeXml(`Dated pages of ${site.projectName}`)}</description>`,
+    `    <language>${escapeXml(site.lang ?? 'en')}</language>`,
+    `    <atom:link href="${escapeXml(site.feedUrl)}" rel="self" type="application/rss+xml" />`,
+  ];
+  if (dated.length > 0) lines.push(`    <lastBuildDate>${rfc822(dated[0].date)}</lastBuildDate>`);
+
+  for (const { page, date } of dated) {
+    const link = new URL(page.url, site.siteUrl).href;
+    lines.push(
+      '    <item>',
+      `      <title>${escapeXml(page.frontmatter.title ?? site.projectName)}</title>`,
+      `      <link>${escapeXml(link)}</link>`,
+      `      <guid isPermaLink="true">${escapeXml(link)}</guid>`,
+      `      <pubDate>${rfc822(date)}</pubDate>`,
+    );
+    if (page.frontmatter.description) {
+      lines.push(`      <description>${escapeXml(page.frontmatter.description)}</description>`);
+    }
+    lines.push('    </item>');
+  }
+
+  lines.push('  </channel>', '</rss>', '');
+  return lines.join('\n');
+}