@brillout/docpress 0.17.5 → 0.17.6

package/code-blocks/getHighlighter.spec.ts

@@ -0,0 +1,30 @@
+import { describe, it, expect } from 'vitest'
+import { getHighlighter } from './getHighlighter.js'
+import type { Highlighter } from 'shiki'
+
+describe('getHighlighter()', () => {
+  it('highlights lazily embedded grammars regardless of the order in which grammars are loaded', async () => {
+    // The markdown grammar lazily embeds the `yaml` grammar for the frontmatter
+    const code = ['---', 'name: "vike"', '---', '', 'See https://vike.dev/llms.txt'].join('\n')
+    const highlight = (highlighter: Highlighter) => highlighter.codeToHtml(code, { lang: 'md', theme: 'github-light' })
+    // Same options as Rehype Pretty Code
+    const options = { themes: ['github-light' as const], langs: ['plaintext' as const] }
+
+    const highlighter1 = await getHighlighter(options)
+    await highlighter1.loadLanguage('md')
+    const html1 = highlight(highlighter1)
+
+    const highlighter2 = await getHighlighter(options)
+    await highlighter2.loadLanguage('yaml')
+    await highlighter2.loadLanguage('md')
+    const html2 = highlight(highlighter2)
+
+    expect(html1).toBe(html2)
+    // The frontmatter is highlighted as YAML: `name` is a YAML key
+    expect(html1).toContain('>name</span>')
+  })
+  it('caches the highlighter per options', () => {
+    const options = { themes: ['github-light' as const], langs: ['plaintext' as const] }
+    expect(getHighlighter(options)).toBe(getHighlighter({ ...options }))
+  })
+})

package/code-blocks/getHighlighter.ts

@@ -0,0 +1,77 @@
+export { getHighlighter }
+export { warmUpHighlighter }
+export { highlighterTheme }
+
+import { getHighlighter as getHighlighterShiki, bundledLanguagesBase } from 'shiki'
+import type { BundledHighlighterOptions, BundledLanguage, BundledTheme, Highlighter, LanguageRegistration } from 'shiki'
+import type { Plugin } from 'vite'
+
+const highlighterTheme = 'github-light'
+// The options Rehype Pretty Code passes to `getHighlighter()`, see `rehypePrettyCode()` in
+// node_modules/rehype-pretty-code/dist/index.js
+const highlighterOptions: BundledHighlighterOptions<BundledLanguage, BundledTheme> = {
+  themes: [highlighterTheme],
+  langs: ['plaintext'],
+}
+
+// TEMPORARY WORKAROUND
+// TO-DO/eventually: remove this file (and its usages in vite.config.ts) once Rehype Pretty Code loads lazily embedded
+// grammars deterministically, e.g. by using Shiki's `guessEmbeddedLanguages()`.
+//
+// Rehype Pretty Code loads grammars on-demand into a single highlighter that is shared by the client-side and
+// server-side builds. Some grammars *lazily* embed other grammars, e.g. the markdown grammar lazily embeds the `yaml`
+// grammar (for frontmatter) and most other grammars (for fenced code blocks): a lazily embedded grammar is used only if
+// it happens to be already loaded.
+//
+// The highlighting of a ```md code block would thus depend on whether a ```yaml code block was processed before it —
+// which differs between the client-side and server-side builds, leading to a hydration mismatch (React error #418).
+//
+// We make the highlighting deterministic by loading all grammars upfront (~0.5s and ~30MB, once per process).
+//
+// See:
+//  - https://github.com/brillout/docpress/pull/195
+//  - https://github.com/shikijs/shiki/pull/791 (Shiki lazily embeds grammars for performance: "users should already
+//    be loading the languages they need")
+//  - https://github.com/shikijs/shiki/issues/979 (same problem: `wikitext` doesn't load `html`)
+//  - https://github.com/shikijs/shiki/pull/1299 (`guessEmbeddedLanguages()` detects frontmatter, but Rehype Pretty Code
+//    doesn't use it — the problem still exists with rehype-pretty-code@0.14.5 and shiki@4.4.3)
+const highlighters = new Map<string, Promise<Highlighter>>()
+function getHighlighter(options: BundledHighlighterOptions<BundledLanguage, BundledTheme>): Promise<Highlighter> {
+  const key = JSON.stringify(options)
+  let highlighter = highlighters.get(key)
+  if (!highlighter) {
+    highlighter = createHighlighter(options)
+    highlighters.set(key, highlighter)
+  }
+  return highlighter
+}
+
+async function createHighlighter(
+  options: BundledHighlighterOptions<BundledLanguage, BundledTheme>,
+): Promise<Highlighter> {
+  const highlighter = await getHighlighterShiki(options)
+  const langsAll = await Promise.all(
+    Object.values(bundledLanguagesBase).map(async (importLang) => (await importLang()).default),
+  )
+  // We load the grammars that lazily embed other grammars (markdown, mdx, vue, ...) last: Shiki re-compiles such a
+  // grammar each time one of its lazily embedded grammars is loaded.
+  const hasLangsEmbeddedLazy = (langs: LanguageRegistration[]) => langs.some((lang) => lang.embeddedLangsLazy?.length)
+  const langsOrdered = [
+    ...langsAll.filter((langs) => !hasLangsEmbeddedLazy(langs)),
+    ...langsAll.filter(hasLangsEmbeddedLazy),
+  ]
+  for (const langs of langsOrdered) await highlighter.loadLanguage(...langs)
+  return highlighter
+}
+
+// Part of the TEMPORARY WORKAROUND above.
+// We load the grammars before the build starts (instead of during the first MDX transform): Rolldown warns when
+// plugins take a significant share of the build time, which the ~0.5s of grammar loading does for small builds.
+function warmUpHighlighter(): Plugin {
+  return {
+    name: '@brillout/docpress:warmUpHighlighter',
+    async configResolved() {
+      await getHighlighter(highlighterOptions)
+    },
+  }
+}

package/code-blocks/remarkChoiceGroup.ts

@@ -83,7 +83,7 @@ const remarkChoiceGroup: Plugin<[], Root> = (): Transformer<Root> => {
     remarkPkgManager.call(this)(tree, file)
 
     visit(tree, 'mdxJsxFlowElement', (node) => {
-      // Descend into non-container nodes so that a `CustomSelectsContainer` nested inside another JSX
+      // Descend into non-container nodes so that a `ChoiceGroupContainer` nested inside another JSX
       // element (e.g. react-tabs `<Tabs>`/`<TabPanel>`, or a `<div>`) still gets visited and its
       // `choiceGroupAll` attribute injected. (Returning 'skip' here would stop the descent.)
       if (node.name !== 'ChoiceGroupContainer') return
@@ -91,6 +91,10 @@ const remarkChoiceGroup: Plugin<[], Root> = (): Transformer<Root> => {
       const choiceGroupAll: ChoiceGroupWithParent[] = []
 
       visit(node, 'mdxJsxFlowElement', (child) => {
+        // A nested container renders its own dropdowns: e.g. a code block inside a choice of a hidden group (a
+        // `:::Choice` toggled by `<Tabs>`), or inside a blockquote. Don't collect its groups here — it gets its own
+        // `choiceGroupAll` attribute when the outer traversal reaches it.
+        if (child !== node && child.name === 'ChoiceGroupContainer') return 'skip'
         if (child.name !== 'ChoiceGroup') return
 
         const choiceGroup = child.data?.customDataChoiceGroup
@@ -127,7 +131,7 @@ const remarkChoiceGroup: Plugin<[], Root> = (): Transformer<Root> => {
 
       node.attributes.push(expressionToAttribute('choiceGroupAll', choiceGroupAll))
 
-      return 'skip'
+      // Don't return 'skip': nested containers need their own `choiceGroupAll` attribute.
     })
   }
 }

package/code-blocks/remarkPkgManager.ts

@@ -34,7 +34,7 @@ function remarkPkgManager() {
           type: node.type,
           lang: node.lang,
           meta: node.meta,
-          value: convert(node.value, pm.toLowerCase() as 'pnpm' | 'bun' | 'yarn'),
+          value: convertCommands(node.value, pm.toLowerCase() as 'pnpm' | 'bun' | 'yarn'),
         })
       }
 
@@ -45,3 +45,17 @@ function remarkPkgManager() {
     })
   }
 }
+
+// If the string contains `npx`, npm-to-yarn only replaces its first `npx` occurrence and leaves the rest of the string
+// untouched (e.g. the `npm install` line of a multi-line command): convert each line separately.
+function convertCommands(value: string, pm: 'pnpm' | 'yarn' | 'bun'): string {
+  return value
+    .split('\n')
+    .map((line) => (hasNpmCommand(line) ? convert(line, pm) : line))
+    .join('\n')
+}
+// Whether the line contains an `npm`/`npx` command. (Not a package name such as `skills-npm`, nor a comment like
+// `# Make sure you install skills-npm`, which npm-to-yarn would mangle.)
+function hasNpmCommand(line: string): boolean {
+  return /(^|\s)np[mx](\s|$)/.test(line)
+}

package/code-blocks/utils/generateChoiceGroupCode.ts

@@ -56,9 +56,11 @@ function generateChoiceGroupCode(choiceNodes: ChoiceNode[], parent: Parent, hide
   const customHidden = choiceNodes.some((node) =>
     node.children.some((node) => node.type === 'containerDirective' && node.children[0]!.type !== 'code'),
   )
-  const hidden = hide || customHidden
+  // A hidden group doesn't render a dropdown: its choices are toggled by `<Tabs>`, or there is only one choice.
+  const hidden = hide || customHidden || choiceNodes.length === 1
 
   const { choiceGroup, mergedChoiceNodes } = resolveChoiceGroupNodes(choiceNodes)
+  const isBuiltIn = Object.keys(CHOICES_BUILT_IN).includes(choiceGroup.name)
   const attributes: MdxJsxAttribute[] = []
   const children: MdxJsxFlowElement[] = []
   let data: MdxJsxFlowElementData = {}
@@ -95,24 +97,32 @@ function generateChoiceGroupCode(choiceNodes: ChoiceNode[], parent: Parent, hide
       ],
       children: choiceChildren,
       data: {
-        ...(!Object.keys(CHOICES_BUILT_IN).includes(choiceGroup.name) && {
-          customDataParentChoiceGroup: {
-            name: choiceGroup.name,
-            choice: choiceNode.choiceValue,
-            default: choiceGroup.default,
-            emptyChoices: choiceGroup.emptyChoices,
-            lvl,
-          },
-        }),
+        // Mark the choice as parent so that the groups nested in it (e.g. the npm/pnpm toggle of a code block, or
+        // the JS/TS toggle) render their dropdown in this group's `<ChoiceGroupContainer>`, next to this group's
+        // dropdown, and only while this choice is selected.
+        // A hidden group renders no dropdown and its choices usually start with prose (it's toggled by `<Tabs>`), so
+        // the top-right corner of its container isn't the top-right corner of a code block: don't mark its choices,
+        // so that nested groups get their own `<ChoiceGroupContainer>` positioned at the code block instead. (That
+        // container is shown/hidden along with the choice, so no parent tracking is needed.)
+        ...(!isBuiltIn &&
+          !hidden && {
+            customDataParentChoiceGroup: {
+              name: choiceGroup.name,
+              choice: choiceNode.choiceValue,
+              default: choiceGroup.default,
+              emptyChoices: choiceGroup.emptyChoices,
+              lvl,
+            },
+          }),
       },
     })
   }
 
   const choiceGroupAttr: ChoiceGroup = {
     ...choiceGroup,
-    hidden: choiceNodes.length === 1 || hidden,
+    hidden,
     lvl,
-    isBuiltIn: Object.keys(CHOICES_BUILT_IN).includes(choiceGroup.name),
+    isBuiltIn,
   }
 
   attributes.push(expressionToAttribute('choiceGroup', choiceGroupAttr))

package/dist/code-blocks/getHighlighter.d.ts

@@ -0,0 +1,8 @@
+export { getHighlighter };
+export { warmUpHighlighter };
+export { highlighterTheme };
+import type { BundledHighlighterOptions, BundledLanguage, BundledTheme, Highlighter } from 'shiki';
+import type { Plugin } from 'vite';
+declare const highlighterTheme = "github-light";
+declare function getHighlighter(options: BundledHighlighterOptions<BundledLanguage, BundledTheme>): Promise<Highlighter>;
+declare function warmUpHighlighter(): Plugin;

package/dist/code-blocks/getHighlighter.js

@@ -0,0 +1,67 @@
+export { getHighlighter };
+export { warmUpHighlighter };
+export { highlighterTheme };
+import { getHighlighter as getHighlighterShiki, bundledLanguagesBase } from 'shiki';
+const highlighterTheme = 'github-light';
+// The options Rehype Pretty Code passes to `getHighlighter()`, see `rehypePrettyCode()` in
+// node_modules/rehype-pretty-code/dist/index.js
+const highlighterOptions = {
+    themes: [highlighterTheme],
+    langs: ['plaintext'],
+};
+// TEMPORARY WORKAROUND
+// TO-DO/eventually: remove this file (and its usages in vite.config.ts) once Rehype Pretty Code loads lazily embedded
+// grammars deterministically, e.g. by using Shiki's `guessEmbeddedLanguages()`.
+//
+// Rehype Pretty Code loads grammars on-demand into a single highlighter that is shared by the client-side and
+// server-side builds. Some grammars *lazily* embed other grammars, e.g. the markdown grammar lazily embeds the `yaml`
+// grammar (for frontmatter) and most other grammars (for fenced code blocks): a lazily embedded grammar is used only if
+// it happens to be already loaded.
+//
+// The highlighting of a ```md code block would thus depend on whether a ```yaml code block was processed before it —
+// which differs between the client-side and server-side builds, leading to a hydration mismatch (React error #418).
+//
+// We make the highlighting deterministic by loading all grammars upfront (~0.5s and ~30MB, once per process).
+//
+// See:
+//  - https://github.com/brillout/docpress/pull/195
+//  - https://github.com/shikijs/shiki/pull/791 (Shiki lazily embeds grammars for performance: "users should already
+//    be loading the languages they need")
+//  - https://github.com/shikijs/shiki/issues/979 (same problem: `wikitext` doesn't load `html`)
+//  - https://github.com/shikijs/shiki/pull/1299 (`guessEmbeddedLanguages()` detects frontmatter, but Rehype Pretty Code
+//    doesn't use it — the problem still exists with rehype-pretty-code@0.14.5 and shiki@4.4.3)
+const highlighters = new Map();
+function getHighlighter(options) {
+    const key = JSON.stringify(options);
+    let highlighter = highlighters.get(key);
+    if (!highlighter) {
+        highlighter = createHighlighter(options);
+        highlighters.set(key, highlighter);
+    }
+    return highlighter;
+}
+async function createHighlighter(options) {
+    const highlighter = await getHighlighterShiki(options);
+    const langsAll = await Promise.all(Object.values(bundledLanguagesBase).map(async (importLang) => (await importLang()).default));
+    // We load the grammars that lazily embed other grammars (markdown, mdx, vue, ...) last: Shiki re-compiles such a
+    // grammar each time one of its lazily embedded grammars is loaded.
+    const hasLangsEmbeddedLazy = (langs) => langs.some((lang) => lang.embeddedLangsLazy?.length);
+    const langsOrdered = [
+        ...langsAll.filter((langs) => !hasLangsEmbeddedLazy(langs)),
+        ...langsAll.filter(hasLangsEmbeddedLazy),
+    ];
+    for (const langs of langsOrdered)
+        await highlighter.loadLanguage(...langs);
+    return highlighter;
+}
+// Part of the TEMPORARY WORKAROUND above.
+// We load the grammars before the build starts (instead of during the first MDX transform): Rolldown warns when
+// plugins take a significant share of the build time, which the ~0.5s of grammar loading does for small builds.
+function warmUpHighlighter() {
+    return {
+        name: '@brillout/docpress:warmUpHighlighter',
+        async configResolved() {
+            await getHighlighter(highlighterOptions);
+        },
+    };
+}

package/dist/code-blocks/remarkChoiceGroup.js

@@ -67,13 +67,18 @@ const remarkChoiceGroup = () => {
         await remarkDetype.call(this)(tree, file);
         remarkPkgManager.call(this)(tree, file);
         visit(tree, 'mdxJsxFlowElement', (node) => {
-            // Descend into non-container nodes so that a `CustomSelectsContainer` nested inside another JSX
+            // Descend into non-container nodes so that a `ChoiceGroupContainer` nested inside another JSX
             // element (e.g. react-tabs `<Tabs>`/`<TabPanel>`, or a `<div>`) still gets visited and its
             // `choiceGroupAll` attribute injected. (Returning 'skip' here would stop the descent.)
             if (node.name !== 'ChoiceGroupContainer')
                 return;
             const choiceGroupAll = [];
             visit(node, 'mdxJsxFlowElement', (child) => {
+                // A nested container renders its own dropdowns: e.g. a code block inside a choice of a hidden group (a
+                // `:::Choice` toggled by `<Tabs>`), or inside a blockquote. Don't collect its groups here — it gets its own
+                // `choiceGroupAll` attribute when the outer traversal reaches it.
+                if (child !== node && child.name === 'ChoiceGroupContainer')
+                    return 'skip';
                 if (child.name !== 'ChoiceGroup')
                     return;
                 const choiceGroup = child.data?.customDataChoiceGroup;
@@ -104,7 +109,7 @@ const remarkChoiceGroup = () => {
                 }
             });
             node.attributes.push(expressionToAttribute('choiceGroupAll', choiceGroupAll));
-            return 'skip';
+            // Don't return 'skip': nested containers need their own `choiceGroupAll` attribute.
         });
     };
 };

package/dist/code-blocks/remarkPkgManager.js

@@ -25,7 +25,7 @@ function remarkPkgManager() {
                     type: node.type,
                     lang: node.lang,
                     meta: node.meta,
-                    value: convert(node.value, pm.toLowerCase()),
+                    value: convertCommands(node.value, pm.toLowerCase()),
                 });
             }
             const choiceNodes = [...nodes].map(([name, node]) => ({ choiceValue: name, children: [node] }));
@@ -34,3 +34,16 @@ function remarkPkgManager() {
         });
     };
 }
+// If the string contains `npx`, npm-to-yarn only replaces its first `npx` occurrence and leaves the rest of the string
+// untouched (e.g. the `npm install` line of a multi-line command): convert each line separately.
+function convertCommands(value, pm) {
+    return value
+        .split('\n')
+        .map((line) => (hasNpmCommand(line) ? convert(line, pm) : line))
+        .join('\n');
+}
+// Whether the line contains an `npm`/`npx` command. (Not a package name such as `skills-npm`, nor a comment like
+// `# Make sure you install skills-npm`, which npm-to-yarn would mangle.)
+function hasNpmCommand(line) {
+    return /(^|\s)np[mx](\s|$)/.test(line);
+}

package/dist/code-blocks/utils/generateChoiceGroupCode.js

@@ -41,8 +41,10 @@ const CHOICES_BUILT_IN = {
 function generateChoiceGroupCode(choiceNodes, parent, hide = false) {
     let lvl = 0;
     const customHidden = choiceNodes.some((node) => node.children.some((node) => node.type === 'containerDirective' && node.children[0].type !== 'code'));
-    const hidden = hide || customHidden;
+    // A hidden group doesn't render a dropdown: its choices are toggled by `<Tabs>`, or there is only one choice.
+    const hidden = hide || customHidden || choiceNodes.length === 1;
     const { choiceGroup, mergedChoiceNodes } = resolveChoiceGroupNodes(choiceNodes);
+    const isBuiltIn = Object.keys(CHOICES_BUILT_IN).includes(choiceGroup.name);
     const attributes = [];
     const children = [];
     let data = {};
@@ -76,7 +78,15 @@ function generateChoiceGroupCode(choiceNodes, parent, hide = false) {
             ],
             children: choiceChildren,
             data: {
-                ...(!Object.keys(CHOICES_BUILT_IN).includes(choiceGroup.name) && {
+                // Mark the choice as parent so that the groups nested in it (e.g. the npm/pnpm toggle of a code block, or
+                // the JS/TS toggle) render their dropdown in this group's `<ChoiceGroupContainer>`, next to this group's
+                // dropdown, and only while this choice is selected.
+                // A hidden group renders no dropdown and its choices usually start with prose (it's toggled by `<Tabs>`), so
+                // the top-right corner of its container isn't the top-right corner of a code block: don't mark its choices,
+                // so that nested groups get their own `<ChoiceGroupContainer>` positioned at the code block instead. (That
+                // container is shown/hidden along with the choice, so no parent tracking is needed.)
+                ...(!isBuiltIn &&
+                    !hidden && {
                     customDataParentChoiceGroup: {
                         name: choiceGroup.name,
                         choice: choiceNode.choiceValue,
@@ -90,9 +100,9 @@ function generateChoiceGroupCode(choiceNodes, parent, hide = false) {
     }
     const choiceGroupAttr = {
         ...choiceGroup,
-        hidden: choiceNodes.length === 1 || hidden,
+        hidden,
         lvl,
-        isBuiltIn: Object.keys(CHOICES_BUILT_IN).includes(choiceGroup.name),
+        isBuiltIn,
     };
     attributes.push(expressionToAttribute('choiceGroup', choiceGroupAttr));
     const choiceGroupNode = {

package/dist/vite.config.js

@@ -14,12 +14,15 @@ import { transformerNotationHighlight } from '@brillout/shiki-transformers';
 import { rehypeMetaToProps } from './code-blocks/rehypeMetaToProps.js';
 import { shikiTransformerAutoLinks } from './code-blocks/shikiTransformerAutoLinks.js';
 import { remarkChoiceGroup } from './code-blocks/remarkChoiceGroup.js';
+import { getHighlighter, warmUpHighlighter, highlighterTheme } from './code-blocks/getHighlighter.js';
 const root = process.cwd();
 const prettyCode = [
     rehypePrettyCode,
     {
-        theme: 'github-light',
+        theme: highlighterTheme,
         keepBackground: false,
+        // TEMPORARY WORKAROUND, see getHighlighter.ts
+        getHighlighter,
         transformers: [
             transformerNotationDiff(),
             transformerNotationHighlight(),
@@ -36,6 +39,8 @@ const config = {
         parsePageSections(),
         mdx({ rehypePlugins, remarkPlugins, providerImportSource: '@brillout/docpress' }),
         react(),
+        // TEMPORARY WORKAROUND, see getHighlighter.ts
+        warmUpHighlighter(),
     ],
     optimizeDeps: {
         include: ['react', 'react-dom', 'react-dom/client'],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brillout/docpress",
-  "version": "0.17.5",
+  "version": "0.17.6",
   "type": "module",
   "dependencies": {
     "@brillout/picocolors": "^1.0.10",

package/vite.config.ts

@@ -16,13 +16,16 @@ import { transformerNotationHighlight } from '@brillout/shiki-transformers'
 import { rehypeMetaToProps } from './code-blocks/rehypeMetaToProps.js'
 import { shikiTransformerAutoLinks } from './code-blocks/shikiTransformerAutoLinks.js'
 import { remarkChoiceGroup } from './code-blocks/remarkChoiceGroup.js'
+import { getHighlighter, warmUpHighlighter, highlighterTheme } from './code-blocks/getHighlighter.js'
 
 const root = process.cwd()
 const prettyCode = [
   rehypePrettyCode,
   {
-    theme: 'github-light',
+    theme: highlighterTheme,
     keepBackground: false,
+    // TEMPORARY WORKAROUND, see getHighlighter.ts
+    getHighlighter,
     transformers: [
       transformerNotationDiff(),
       transformerNotationHighlight(),
@@ -40,6 +43,8 @@ const config: UserConfig = {
     parsePageSections(),
     mdx({ rehypePlugins, remarkPlugins, providerImportSource: '@brillout/docpress' }) as PluginOption,
     react(),
+    // TEMPORARY WORKAROUND, see getHighlighter.ts
+    warmUpHighlighter(),
   ],
   optimizeDeps: {
     include: ['react', 'react-dom', 'react-dom/client'],