@umami/shiso 0.55.0 → 1.0.0

package/scripts/generate-search-index.mjs

@@ -0,0 +1,298 @@
+/**
+ * Generates the project-local .shiso/search-index.generated.ts cache.
+ *
+ * Client-side search needs the plain text of every page, which only exists in
+ * the MDX sources at build time. Each navigable page is parsed to mdast and
+ * split into heading-bounded sections; headings share the slug algorithm with
+ * rehype-slug (src/lib/slug.ts), so result anchors always match rendered ids.
+ *
+ * Pages marked `hidden` in navigation are excluded, matching their exclusion
+ * from the sidebar and sitemap.
+ *
+ * Run via `pnpm search:index`, or automatically by the Vite plugin in
+ * vite.config.ts. The generated module is dynamically imported by the search
+ * dialog, so Vite splits it out of the initial bundle.
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import remarkFrontmatter from 'remark-frontmatter';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+import { headingText } from './lib/mdast.mjs';
+import { createSlugger, slugifyId } from './lib/slug.mjs';
+import { loadDocsConfig } from './load-docs-config.mjs';
+
+const DEFAULT_ROOT = process.cwd();
+
+const parser = unified().use(remarkParse).use(remarkMdx).use(remarkFrontmatter).use(remarkGfm);
+
+/** Mirrors normalizePageReference in src/lib/docs-config.ts. */
+function normalizePageReference(pageRef) {
+  const value = pageRef
+    .trim()
+    .replace(/\\/g, '/')
+    .replace(/^\/+/, '')
+    .replace(/^docs\//, '')
+    .replace(/\.mdx?$/, '')
+    .replace(/\/+$/, '');
+
+  if (!value) {
+    return { fileSlug: 'index', slug: 'index' };
+  }
+
+  return {
+    fileSlug: value,
+    slug: value === 'index' ? 'index' : value.replace(/\/index$/, '') || 'index',
+  };
+}
+
+/**
+ * One entry per navigation scope, mirroring collectScopeSources in
+ * src/lib/docs-config.ts: ordinary navigation, versions, languages, and
+ * versions nested inside languages. Hidden scopes are excluded from search,
+ * matching how hidden pages are excluded.
+ */
+function collectScopes(navigation) {
+  if (Array.isArray(navigation.versions)) {
+    return navigation.versions
+      .filter(version => !version.hidden)
+      .map(version => ({
+        id: slugifyId(version.version?.trim() || '', 'scope'),
+        version: version.version?.trim(),
+        container: version,
+      }));
+  }
+
+  if (Array.isArray(navigation.languages)) {
+    return navigation.languages
+      .filter(language => !language.hidden)
+      .flatMap(language => {
+        const languageLabel = language.language?.trim();
+
+        if (Array.isArray(language.versions)) {
+          return language.versions
+            .filter(version => !version.hidden)
+            .map(version => ({
+              id: slugifyId(`${languageLabel}-${version.version?.trim()}`, 'scope'),
+              language: languageLabel,
+              version: version.version?.trim(),
+              container: version,
+            }));
+        }
+
+        return [
+          {
+            id: slugifyId(languageLabel || '', 'scope'),
+            language: languageLabel,
+            container: language,
+          },
+        ];
+      });
+  }
+
+  return [{ id: 'default', container: navigation }];
+}
+
+/**
+ * Collects `{ fileSlug, slug }` for every non-hidden page in the navigation
+ * tree. A simplified mirror of the config walker: search only needs page refs
+ * and hidden inheritance, not labels or ordering.
+ */
+function collectVisiblePages(container, pages = [], hidden = false) {
+  const items = [
+    ...(container.pages || []),
+    ...(container.groups || []),
+    ...(container.tabs || []),
+    ...(container.dropdowns || []),
+  ];
+
+  for (const item of items) {
+    if (typeof item === 'string') {
+      if (!hidden) {
+        pages.push(normalizePageReference(item));
+      }
+      continue;
+    }
+
+    if (!item || typeof item !== 'object') {
+      continue;
+    }
+
+    const itemHidden = hidden || item.hidden === true;
+
+    if (typeof item.page === 'string') {
+      if (!itemHidden) {
+        pages.push(normalizePageReference(item.page));
+      }
+      continue;
+    }
+
+    if (typeof item.root === 'string' && !itemHidden) {
+      pages.push(normalizePageReference(item.root));
+    }
+
+    collectVisiblePages(item, pages, itemHidden);
+  }
+
+  return pages;
+}
+
+/** Frontmatter is YAML, but search only needs the title line. */
+function frontmatterTitle(tree) {
+  const yaml = tree.children?.find(node => node.type === 'yaml');
+  const match = yaml?.value?.match(/^title:\s*(.+)$/m);
+  return match ? match[1].trim().replace(/^["']|["']$/g, '') : undefined;
+}
+
+/** Node types whose children are separate blocks, so their text needs a space between. */
+const BLOCK_TYPES = new Set([
+  'root',
+  'list',
+  'listItem',
+  'table',
+  'tableRow',
+  'tableCell',
+  'blockquote',
+  'mdxJsxFlowElement',
+]);
+
+/** Like toText in src/lib/mdast.ts, but block children are space-separated. */
+function blockText(node) {
+  if (typeof node.value === 'string') {
+    return node.value;
+  }
+
+  const separator = BLOCK_TYPES.has(node.type) ? ' ' : '';
+  return (node.children || []).map(blockText).join(separator);
+}
+
+/** Splits a document into heading-bounded sections of plain text. */
+function collectSections(tree) {
+  const slugger = createSlugger();
+  const sections = [{ heading: undefined, id: undefined, parts: [] }];
+
+  for (const node of tree.children || []) {
+    if (node.type === 'yaml' || node.type === 'mdxjsEsm') {
+      continue;
+    }
+
+    if (node.type === 'heading') {
+      const heading = headingText(node);
+      sections.push({ heading, id: slugger.slug(heading), parts: [] });
+      continue;
+    }
+
+    const text = blockText(node).replace(/\s+/g, ' ').trim();
+
+    if (text) {
+      sections.at(-1).parts.push(text);
+    }
+  }
+
+  return sections
+    .map(({ heading, id, parts }) => ({ heading, id, text: parts.join(' ') }))
+    .filter(section => section.heading || section.text);
+}
+
+/** @param {{ config?: object, root?: string, output?: string }} [options] */
+export async function generateSearchIndex({
+  config,
+  root = DEFAULT_ROOT,
+  output = path.join(root, '.shiso/search-index.generated.ts'),
+} = {}) {
+  const docsJson = config ?? (await loadDocsConfig({ root })).config;
+  const docsPrefix = (docsJson.$shiso?.docsPrefix ?? '/docs').replace(/\/+$/, '');
+  const contentDir = (docsJson.$shiso?.contentDir ?? 'content/docs').replace(/^\/+|\/+$/g, '');
+
+  const seen = new Set();
+  const records = [];
+
+  for (const scope of collectScopes(docsJson.navigation || {})) {
+    // Single-scope sites omit scope fields so their index stays unchanged.
+    const scopeFields =
+      scope.id === 'default'
+        ? {}
+        : { scopeId: scope.id, language: scope.language, version: scope.version };
+
+    for (const { fileSlug, slug } of collectVisiblePages(scope.container)) {
+      if (seen.has(fileSlug)) {
+        continue;
+      }
+
+      seen.add(fileSlug);
+
+      let source;
+      let filePath;
+
+      for (const extension of ['mdx', 'md']) {
+        filePath = path.join(root, contentDir, `${fileSlug}.${extension}`);
+        source = await fs.readFile(filePath, 'utf8').catch(() => undefined);
+
+        if (source !== undefined) {
+          break;
+        }
+      }
+
+      if (source === undefined) {
+        // Missing files fail config normalization; search just skips them.
+        continue;
+      }
+
+      const tree = parser.parse(source);
+      const url = slug === 'index' ? docsPrefix || '/' : `${docsPrefix}/${slug}`;
+      const page = frontmatterTitle(tree) || fileSlug;
+
+      for (const { heading, id, text } of collectSections(tree)) {
+        records.push({ url, page, heading, id, text, ...scopeFields });
+      }
+    }
+  }
+
+  // Emitted in the repo's formatter style (single quotes, bare keys, trailing
+  // commas) so the generated file passes `biome check` unchanged.
+  const quote = value => {
+    const escaped = value.replace(/\\/g, '\\\\');
+    const singles = (value.match(/'/g) || []).length;
+    const doubles = (value.match(/"/g) || []).length;
+
+    // Like the formatter: whichever quote needs fewer escapes, single on ties.
+    return singles > doubles
+      ? `"${escaped.replace(/"/g, '\\"')}"`
+      : `'${escaped.replace(/'/g, "\\'")}'`;
+  };
+  const body = records
+    .map(record => {
+      const fields = Object.entries(record)
+        .filter(([, value]) => value !== undefined)
+        .map(([key, value]) => `    ${key}: ${quote(value)},`)
+        .join('\n');
+
+      return `  {\n${fields}\n  },`;
+    })
+    .join('\n');
+
+  const contents = `// Generated by scripts/generate-search-index.mjs. Do not edit by hand.
+import type { SearchRecord } from '@/lib/search';
+
+export const SEARCH_INDEX: SearchRecord[] = [
+${body}
+];
+`;
+
+  const previous = await fs.readFile(output, 'utf8').catch(() => '');
+
+  if (previous !== contents) {
+    await fs.mkdir(path.dirname(output), { recursive: true });
+    await fs.writeFile(output, contents);
+  }
+
+  return { pages: seen.size, records: records.length, changed: previous !== contents };
+}
+
+if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
+  const { pages, records } = await generateSearchIndex();
+  console.log(`Search index: ${records} sections across ${pages} pages`);
+}

package/scripts/lib/mdast.mjs

@@ -0,0 +1,23 @@
+/**
+ * Mirrors the mdast helpers in src/lib/mdast.ts for Node scripts, which must
+ * not import raw TypeScript (that would depend on Node's experimental type
+ * stripping). tests/script-mirrors.test.ts asserts the two stay in sync.
+ */
+
+/** Concatenates the text content of a node, including JSX text children. */
+export function toText(node) {
+  if (!node) {
+    return '';
+  }
+
+  if (typeof node.value === 'string') {
+    return node.value;
+  }
+
+  return (node.children || []).map(toText).join('');
+}
+
+/** Text of a heading node, trimmed. */
+export function headingText(node) {
+  return toText(node).trim();
+}

package/scripts/lib/slug.mjs

@@ -0,0 +1,21 @@
+/**
+ * Mirrors src/lib/slug.ts for Node scripts, which must not import raw
+ * TypeScript (that would depend on Node's experimental type stripping).
+ * tests/script-mirrors.test.ts asserts the two stay in sync.
+ */
+import GithubSlugger, { slug as slugifyOnce } from 'github-slugger';
+
+/** Stateful slugger matching rehype-slug: repeated headings get -1, -2, ... */
+export function createSlugger() {
+  return new GithubSlugger();
+}
+
+/** Stateless slugify for one-off ids that do not need de-duplication. */
+export function slugify(value) {
+  return slugifyOnce(value.trim());
+}
+
+/** Slugifies a label into a config-level identifier, with a fallback. */
+export function slugifyId(value, fallback) {
+  return slugify(value) || fallback;
+}

package/scripts/load-docs-config.mjs

@@ -0,0 +1,244 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+/** Error raised while locating, reading, or parsing a Shiso configuration file. */
+export class DocsConfigLoadError extends Error {
+  constructor(message, { cause, code, sourcePath }) {
+    super(message, { cause });
+    this.name = 'DocsConfigLoadError';
+    this.code = code;
+    this.sourcePath = sourcePath;
+  }
+}
+
+function isObject(value) {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isInsideRoot(projectRoot, candidate) {
+  const relative = path.relative(projectRoot, candidate);
+  return (
+    relative === '' ||
+    (!path.isAbsolute(relative) && !relative.startsWith(`..${path.sep}`) && relative !== '..')
+  );
+}
+
+function parseLocation(source, error) {
+  const reported = error.message.match(/line\s+(\d+)(?:\s+column\s+(\d+))?/i);
+
+  if (reported) {
+    return ` at line ${reported[1]}${reported[2] ? `, column ${reported[2]}` : ''}`;
+  }
+
+  const position = error.message.match(/position\s+(\d+)/i)?.[1];
+  const unexpectedToken = error.message.match(/Unexpected token '([^']+)'/i)?.[1];
+  const parsedOffset = position === undefined ? -1 : Number(position);
+  const offset = parsedOffset >= 0 ? parsedOffset : source.indexOf(unexpectedToken || '');
+
+  if (offset < 0 || (!position && !unexpectedToken)) {
+    return '';
+  }
+
+  const before = source.slice(0, offset);
+  const line = before.split('\n').length;
+  const lastNewline = before.lastIndexOf('\n');
+  const column = offset - lastNewline;
+
+  return ` at line ${line}, column ${column}`;
+}
+
+/** Reads a JSON document while preserving its source path in every error. */
+export async function loadJsonDocument(sourcePath, label = 'JSON document') {
+  const resolvedPath = path.resolve(sourcePath);
+  let source;
+
+  try {
+    source = await fs.readFile(resolvedPath, 'utf8');
+  } catch (error) {
+    const reason =
+      error.code === 'ENOENT' ? 'does not exist' : `could not be read: ${error.message}`;
+
+    throw new DocsConfigLoadError(`${label} "${resolvedPath}" ${reason}.`, {
+      cause: error,
+      code: error.code === 'ENOENT' ? 'NOT_FOUND' : 'READ_FAILED',
+      sourcePath: resolvedPath,
+    });
+  }
+
+  try {
+    return JSON.parse(source);
+  } catch (error) {
+    throw new DocsConfigLoadError(
+      `${label} "${resolvedPath}" contains invalid JSON${parseLocation(source, error)}: ${error.message}`,
+      {
+        cause: error,
+        code: 'INVALID_JSON',
+        sourcePath: resolvedPath,
+      },
+    );
+  }
+}
+
+function referenceError(message, { code, sourcePath }) {
+  return new DocsConfigLoadError(message, { code, sourcePath });
+}
+
+/** Resolves every `$ref` in one project configuration tree. */
+async function resolveConfigReferences(entryPath, projectRoot) {
+  const lexicalRoot = path.resolve(projectRoot);
+  const canonicalRoot = await fs.realpath(projectRoot);
+  const sourcePaths = new Set();
+  const cache = new Map();
+
+  async function canonicalDocumentPath(sourcePath, referringPath) {
+    const resolvedPath = path.resolve(sourcePath);
+
+    if (!isInsideRoot(lexicalRoot, resolvedPath) && !isInsideRoot(canonicalRoot, resolvedPath)) {
+      throw referenceError(
+        `Config reference "${resolvedPath}" from "${referringPath}" leaves the project root "${canonicalRoot}".`,
+        { code: 'REF_OUTSIDE_ROOT', sourcePath: resolvedPath },
+      );
+    }
+
+    let canonicalPath;
+
+    try {
+      canonicalPath = await fs.realpath(resolvedPath);
+    } catch (error) {
+      const reason =
+        error.code === 'ENOENT' ? 'does not exist' : `could not be resolved: ${error.message}`;
+
+      throw new DocsConfigLoadError(`Referenced config "${resolvedPath}" ${reason}.`, {
+        cause: error,
+        code: error.code === 'ENOENT' ? 'NOT_FOUND' : 'READ_FAILED',
+        sourcePath: resolvedPath,
+      });
+    }
+
+    if (!isInsideRoot(canonicalRoot, canonicalPath)) {
+      throw referenceError(
+        `Config reference "${resolvedPath}" from "${referringPath}" resolves outside the project root.`,
+        { code: 'REF_OUTSIDE_ROOT', sourcePath: resolvedPath },
+      );
+    }
+
+    return canonicalPath;
+  }
+
+  async function resolveDocument(sourcePath, stack = []) {
+    const referringPath = stack.at(-1) || entryPath;
+    const canonicalPath = await canonicalDocumentPath(sourcePath, referringPath);
+
+    if (stack.includes(canonicalPath)) {
+      const cycle = [...stack.slice(stack.indexOf(canonicalPath)), canonicalPath]
+        .map(item => path.relative(projectRoot, item) || path.basename(item))
+        .join(' -> ');
+
+      throw referenceError(`Circular config reference detected: ${cycle}.`, {
+        code: 'CIRCULAR_REF',
+        sourcePath: canonicalPath,
+      });
+    }
+
+    if (cache.has(canonicalPath)) {
+      return cache.get(canonicalPath);
+    }
+
+    sourcePaths.add(canonicalPath);
+    const document = await loadJsonDocument(canonicalPath, 'Referenced config');
+    const resolved = await resolveValue(document, canonicalPath, [...stack, canonicalPath]);
+    cache.set(canonicalPath, resolved);
+    return resolved;
+  }
+
+  async function resolveReference(reference, sourcePath, stack) {
+    if (typeof reference !== 'string' || !reference.trim()) {
+      throw referenceError(`Config $ref in "${sourcePath}" must be a non-empty string.`, {
+        code: 'INVALID_REF',
+        sourcePath,
+      });
+    }
+
+    if (path.isAbsolute(reference) || /^[a-z][a-z0-9+.-]*:/i.test(reference)) {
+      throw referenceError(
+        `Config $ref "${reference}" in "${sourcePath}" must be a relative JSON file path.`,
+        { code: 'INVALID_REF', sourcePath },
+      );
+    }
+
+    if (path.extname(reference).toLowerCase() !== '.json') {
+      throw referenceError(`Config $ref "${reference}" in "${sourcePath}" must end in .json.`, {
+        code: 'INVALID_REF',
+        sourcePath,
+      });
+    }
+
+    return resolveDocument(path.resolve(path.dirname(sourcePath), reference), stack);
+  }
+
+  async function resolveValue(value, sourcePath, stack) {
+    if (Array.isArray(value)) {
+      return Promise.all(value.map(item => resolveValue(item, sourcePath, stack)));
+    }
+
+    if (!isObject(value)) {
+      return value;
+    }
+
+    if ('$ref' in value) {
+      const referenced = await resolveReference(value.$ref, sourcePath, stack);
+
+      // Object targets receive shallow sibling overrides. For arrays and
+      // primitives, the reference replaces the whole object and siblings are
+      // intentionally ignored.
+      if (!isObject(referenced)) {
+        return referenced;
+      }
+
+      const siblings = Object.fromEntries(Object.entries(value).filter(([key]) => key !== '$ref'));
+      const resolvedSiblings = await resolveValue(siblings, sourcePath, stack);
+      return { ...referenced, ...resolvedSiblings };
+    }
+
+    const entries = await Promise.all(
+      Object.entries(value).map(async ([key, item]) => [
+        key,
+        await resolveValue(item, sourcePath, stack),
+      ]),
+    );
+    return Object.fromEntries(entries);
+  }
+
+  const config = await resolveDocument(entryPath);
+  return { config, projectRoot: canonicalRoot, sourcePaths: [...sourcePaths] };
+}
+
+/**
+ * Loads the project docs configuration.
+ *
+ * Returning its resolved source path now gives the later `$ref` resolver a
+ * stable place from which to resolve relative references without changing
+ * every build-time consumer again.
+ */
+export async function loadDocsConfig({ root = process.cwd(), configFile = 'docs.json' } = {}) {
+  const requestedRoot = path.resolve(root);
+  const requestedSourcePath = path.resolve(requestedRoot, configFile);
+  const { config, projectRoot, sourcePaths } = await resolveConfigReferences(
+    requestedSourcePath,
+    requestedRoot,
+  );
+  const sourcePath = sourcePaths[0] || requestedSourcePath;
+
+  return { config, projectRoot, sourcePath, sourcePaths };
+}
+
+export async function loadDocsSchema({
+  root = process.cwd(),
+  schemaFile = 'docs.schema.json',
+} = {}) {
+  const projectRoot = path.resolve(root);
+  const sourcePath = path.resolve(projectRoot, schemaFile);
+  const schema = await loadJsonDocument(sourcePath, 'Docs schema');
+
+  return { projectRoot, schema, sourcePath };
+}