@astrojs/mdx 0.4.0 → 0.7.0

package/src/index.ts

@@ -1,17 +1,18 @@
-import { nodeTypes } from '@mdx-js/mdx';
+import { compile as mdxCompile, nodeTypes } from '@mdx-js/mdx';
 import mdxPlugin, { Options as MdxRollupPluginOptions } from '@mdx-js/rollup';
-import type { AstroIntegration } from 'astro';
+import type { AstroConfig, AstroIntegration } from 'astro';
 import { parse as parseESM } from 'es-module-lexer';
 import rehypeRaw from 'rehype-raw';
-import remarkFrontmatter from 'remark-frontmatter';
 import remarkGfm from 'remark-gfm';
 import type { RemarkMdxFrontmatterOptions } from 'remark-mdx-frontmatter';
-import remarkMdxFrontmatter from 'remark-mdx-frontmatter';
 import remarkShikiTwoslash from 'remark-shiki-twoslash';
 import remarkSmartypants from 'remark-smartypants';
+import { VFile } from 'vfile';
 import type { Plugin as VitePlugin } from 'vite';
+import { rehypeApplyFrontmatterExport, remarkInitializeAstroData } from './astro-data-utils.js';
+import rehypeCollectHeadings from './rehype-collect-headings.js';
 import remarkPrism from './remark-prism.js';
-import { getFileInfo, getFrontmatter } from './utils.js';
+import { getFileInfo, parseFrontmatter } from './utils.js';
 
 type WithExtends<T> = T | { extends: T };
 
@@ -26,7 +27,11 @@ type MdxOptions = {
 	frontmatterOptions?: RemarkMdxFrontmatterOptions;
 };
 
-const DEFAULT_REMARK_PLUGINS = [remarkGfm, remarkSmartypants];
+const DEFAULT_REMARK_PLUGINS: MdxRollupPluginOptions['remarkPlugins'] = [
+	remarkGfm,
+	remarkSmartypants,
+];
+const DEFAULT_REHYPE_PLUGINS: MdxRollupPluginOptions['rehypePlugins'] = [];
 
 function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] = []): T[] {
 	if (Array.isArray(config)) return config;
@@ -34,92 +39,107 @@ function handleExtends<T>(config: WithExtends<T[] | undefined>, defaults: T[] =
 	return [...defaults, ...(config?.extends ?? [])];
 }
 
+function getRemarkPlugins(
+	mdxOptions: MdxOptions,
+	config: AstroConfig
+): MdxRollupPluginOptions['remarkPlugins'] {
+	let remarkPlugins = [
+		// Initialize vfile.data.astroExports before all plugins are run
+		remarkInitializeAstroData,
+		...handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS),
+	];
+	if (config.markdown.syntaxHighlight === 'shiki') {
+		// Default export still requires ".default" chaining for some reason
+		// Workarounds tried:
+		// - "import * as remarkShikiTwoslash"
+		// - "import { default as remarkShikiTwoslash }"
+		const shikiTwoslash = (remarkShikiTwoslash as any).default ?? remarkShikiTwoslash;
+		remarkPlugins.push([shikiTwoslash, config.markdown.shikiConfig]);
+	}
+	if (config.markdown.syntaxHighlight === 'prism') {
+		remarkPlugins.push(remarkPrism);
+	}
+	return remarkPlugins;
+}
+
+function getRehypePlugins(
+	mdxOptions: MdxOptions,
+	config: AstroConfig
+): MdxRollupPluginOptions['rehypePlugins'] {
+	let rehypePlugins = handleExtends(mdxOptions.rehypePlugins, DEFAULT_REHYPE_PLUGINS);
+
+	if (config.markdown.syntaxHighlight === 'shiki' || config.markdown.syntaxHighlight === 'prism') {
+		rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
+	}
+	// getHeadings() is guaranteed by TS, so we can't allow user to override
+	rehypePlugins.push(rehypeCollectHeadings);
+
+	return rehypePlugins;
+}
+
 export default function mdx(mdxOptions: MdxOptions = {}): AstroIntegration {
 	return {
 		name: '@astrojs/mdx',
 		hooks: {
 			'astro:config:setup': ({ updateConfig, config, addPageExtension, command }: any) => {
 				addPageExtension('.mdx');
-				let remarkPlugins = handleExtends(mdxOptions.remarkPlugins, DEFAULT_REMARK_PLUGINS);
-				let rehypePlugins = handleExtends(mdxOptions.rehypePlugins);
-
-				if (config.markdown.syntaxHighlight === 'shiki') {
-					remarkPlugins.push([
-						// Default export still requires ".default" chaining for some reason
-						// Workarounds tried:
-						// - "import * as remarkShikiTwoslash"
-						// - "import { default as remarkShikiTwoslash }"
-						(remarkShikiTwoslash as any).default,
-						config.markdown.shikiConfig,
-					]);
-					rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
-				}
-
-				if (config.markdown.syntaxHighlight === 'prism') {
-					remarkPlugins.push(remarkPrism);
-					rehypePlugins.push([rehypeRaw, { passThrough: nodeTypes }]);
-				}
-
-				remarkPlugins.push(remarkFrontmatter);
-				remarkPlugins.push([
-					remarkMdxFrontmatter,
-					{
-						name: 'frontmatter',
-						...mdxOptions.frontmatterOptions,
-					},
-				]);
 
-				const configuredMdxPlugin = mdxPlugin({
-					remarkPlugins,
-					rehypePlugins,
+				const mdxPluginOpts: MdxRollupPluginOptions = {
+					remarkPlugins: getRemarkPlugins(mdxOptions, config),
+					rehypePlugins: getRehypePlugins(mdxOptions, config),
 					jsx: true,
 					jsxImportSource: 'astro',
 					// Note: disable `.md` support
 					format: 'mdx',
 					mdExtensions: [],
-				});
+				};
 
 				updateConfig({
 					vite: {
 						plugins: [
 							{
 								enforce: 'pre',
-								...configuredMdxPlugin,
-								// Override transform to inject layouts before MDX compilation
-								async transform(this, code, id) {
-									if (!id.endsWith('.mdx')) return;
+								...mdxPlugin(mdxPluginOpts),
+								// Override transform to alter code before MDX compilation
+								// ex. inject layouts
+								async transform(code, id) {
+									if (!id.endsWith('mdx')) return;
 
-									const mdxPluginTransform = configuredMdxPlugin.transform?.bind(this);
-									// If user overrides our default YAML parser,
-									// do not attempt to parse the `layout` via gray-matter
-									if (mdxOptions.frontmatterOptions?.parsers) {
-										return mdxPluginTransform?.(code, id);
-									}
-									const frontmatter = getFrontmatter(code, id);
+									let { data: frontmatter, content: pageContent } = parseFrontmatter(code, id);
 									if (frontmatter.layout) {
-										const { layout, ...content } = frontmatter;
-										code += `\nexport default async function({ children }) {\nconst Layout = (await import(${JSON.stringify(
+										const { layout, ...contentProp } = frontmatter;
+										pageContent += `\n\nexport default async function({ children }) {\nconst Layout = (await import(${JSON.stringify(
 											frontmatter.layout
-										)})).default;\nreturn <Layout content={${JSON.stringify(
-											content
-										)}}>{children}</Layout> }`;
+										)})).default;\nconst frontmatter=${JSON.stringify(
+											contentProp
+										)};\nreturn <Layout frontmatter={frontmatter} content={frontmatter} headings={getHeadings()}>{children}</Layout> }`;
 									}
-									return mdxPluginTransform?.(code, id);
+
+									const compiled = await mdxCompile(new VFile({ value: pageContent, path: id }), {
+										...mdxPluginOpts,
+										rehypePlugins: [
+											...(mdxPluginOpts.rehypePlugins ?? []),
+											() =>
+												rehypeApplyFrontmatterExport(
+													frontmatter,
+													mdxOptions.frontmatterOptions?.name
+												),
+										],
+									});
+
+									return {
+										code: String(compiled.value),
+										map: compiled.map,
+									};
 								},
 							},
 							{
-								name: '@astrojs/mdx',
+								name: '@astrojs/mdx-postprocess',
+								// These transforms must happen *after* JSX runtime transformations
 								transform(code, id) {
 									if (!id.endsWith('.mdx')) return;
 									const [, moduleExports] = parseESM(code);
 
-									// This adds support for injected "page-ssr" scripts in MDX files.
-									// TODO: This should only be happening on page entrypoints, not all imported MDX.
-									// TODO: This code is copy-pasted across all Astro/Vite plugins that deal with page
-									// entrypoints (.astro, .md, .mdx). This should be handled in some centralized place,
-									// or otherwise refactored to not require copy-paste handling logic.
-									code += `\nimport "${'astro:scripts/page-ssr.js'}";`;
-
 									const { fileUrl, fileId } = getFileInfo(id, config);
 									if (!moduleExports.includes('url')) {
 										code += `\nexport const url = ${JSON.stringify(fileUrl)};`;

package/src/rehype-collect-headings.ts

@@ -0,0 +1,50 @@
+import Slugger from 'github-slugger';
+import { visit } from 'unist-util-visit';
+import { jsToTreeNode } from './utils.js';
+
+export interface MarkdownHeading {
+	depth: number;
+	slug: string;
+	text: string;
+}
+
+export default function rehypeCollectHeadings() {
+	const slugger = new Slugger();
+	return function (tree: any) {
+		const headings: MarkdownHeading[] = [];
+		visit(tree, (node) => {
+			if (node.type !== 'element') return;
+			const { tagName } = node;
+			if (tagName[0] !== 'h') return;
+			const [_, level] = tagName.match(/h([0-6])/) ?? [];
+			if (!level) return;
+			const depth = Number.parseInt(level);
+
+			let text = '';
+			visit(node, (child, __, parent) => {
+				if (child.type === 'element' || parent == null) {
+					return;
+				}
+				if (child.type === 'raw' && child.value.match(/^\n?<.*>\n?$/)) {
+					return;
+				}
+				if (new Set(['text', 'raw', 'mdxTextExpression']).has(child.type)) {
+					text += child.value;
+				}
+			});
+
+			node.properties = node.properties || {};
+			if (typeof node.properties.id !== 'string') {
+				let slug = slugger.slug(text);
+				if (slug.endsWith('-')) {
+					slug = slug.slice(0, -1);
+				}
+				node.properties.id = slug;
+			}
+			headings.push({ depth, slug: node.properties.id, text });
+		});
+		tree.children.unshift(
+			jsToTreeNode(`export function getHeadings() { return ${JSON.stringify(headings)} }`)
+		);
+	};
+}

package/src/remark-prism.ts

@@ -1,48 +1,6 @@
-// TODO: discuss extracting this file to @astrojs/prism
-import { addAstro } from '@astrojs/prism/internal';
-import Prism from 'prismjs';
-import loadLanguages from 'prismjs/components/index.js';
+import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
 import { visit } from 'unist-util-visit';
 
-const languageMap = new Map([['ts', 'typescript']]);
-
-function runHighlighter(lang: string, code: string) {
-	let classLanguage = `language-${lang}`;
-
-	if (lang == null) {
-		lang = 'plaintext';
-	}
-
-	const ensureLoaded = (language: string) => {
-		if (language && !Prism.languages[language]) {
-			loadLanguages([language]);
-		}
-	};
-
-	if (languageMap.has(lang)) {
-		ensureLoaded(languageMap.get(lang)!);
-	} else if (lang === 'astro') {
-		ensureLoaded('typescript');
-		addAstro(Prism);
-	} else {
-		ensureLoaded('markup-templating'); // Prism expects this to exist for a number of other langs
-		ensureLoaded(lang);
-	}
-
-	if (lang && !Prism.languages[lang]) {
-		// eslint-disable-next-line no-console
-		console.warn(`Unable to load the language: ${lang}`);
-	}
-
-	const grammar = Prism.languages[lang];
-	let html = code;
-	if (grammar) {
-		html = Prism.highlight(code, grammar, lang);
-	}
-
-	return { classLanguage, html };
-}
-
 /**  */
 export default function remarkPrism() {
 	return (tree: any) =>
@@ -50,7 +8,7 @@ export default function remarkPrism() {
 			let { lang, value } = node;
 			node.type = 'html';
 
-			let { html, classLanguage } = runHighlighter(lang, value);
+			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
 			let classes = [classLanguage];
 			node.value = `<pre class="${classes.join(
 				' '

package/src/utils.ts

@@ -1,4 +1,8 @@
+import type { Options as AcornOpts } from 'acorn';
+import { parse } from 'acorn';
 import type { AstroConfig, SSRError } from 'astro';
+import type { MdxjsEsm } from 'mdast-util-mdx';
+
 import matter from 'gray-matter';
 
 function appendForwardSlash(path: string) {
@@ -43,9 +47,9 @@ export function getFileInfo(id: string, config: AstroConfig): FileInfo {
  * Match YAML exception handling from Astro core errors
  * @see 'astro/src/core/errors.ts'
  */
-export function getFrontmatter(code: string, id: string) {
+export function parseFrontmatter(code: string, id: string) {
 	try {
-		return matter(code).data;
+		return matter(code);
 	} catch (e: any) {
 		if (e.name === 'YAMLException') {
 			const err: SSRError = e;
@@ -58,3 +62,24 @@ export function getFrontmatter(code: string, id: string) {
 		}
 	}
 }
+
+export function jsToTreeNode(
+	jsString: string,
+	acornOpts: AcornOpts = {
+		ecmaVersion: 'latest',
+		sourceType: 'module',
+	}
+): MdxjsEsm {
+	return {
+		type: 'mdxjsEsm',
+		value: '',
+		data: {
+			estree: {
+				body: [],
+				...parse(jsString, acornOpts),
+				type: 'Program',
+				sourceType: 'module',
+			},
+		},
+	};
+}

package/test/fixtures/mdx-escape/src/components/Em.astro

@@ -0,0 +1,7 @@
+<em><slot/></em>
+
+<style>
+  em {
+    color: red;
+  }
+</style>

package/test/fixtures/mdx-escape/src/components/P.astro

@@ -0,0 +1 @@
+<p><slot /></p>

package/test/fixtures/mdx-escape/src/components/Title.astro

@@ -0,0 +1 @@
+<h1><slot/></h1>

package/test/fixtures/mdx-escape/src/pages/html-tag.mdx

@@ -0,0 +1,5 @@
+import P from '../components/P.astro';
+import Em from '../components/Em.astro';
+
+<P>Render <Em>Me</Em></P>
+<P><Em>Me</Em></P>

package/test/fixtures/mdx-escape/src/pages/index.mdx

@@ -0,0 +1,13 @@
+import P from '../components/P.astro';
+import Em from '../components/Em.astro';
+import Title from '../components/Title.astro';
+
+export const components = { p: P, em: Em, h1: Title };
+
+# Hello _there_
+
+# _there_
+
+Hello _there_
+
+_there_

package/test/fixtures/mdx-frontmatter/src/layouts/Base.astro

@@ -1,18 +1,28 @@
 ---
-const { content = { title: "Didn't work" } } = Astro.props;
+const {
+	content = { title: "content didn't work" },
+	frontmatter = { title: "frontmatter didn't work" },
+	headings = [],
+} = Astro.props;
 ---
 
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
 	<meta charset="UTF-8">
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=1.0">
-	<title>{content.title}</title>
 </head>
+
 <body>
-	<h1>{content.title}</h1>
+	<p data-content-title>{content.title}</p>
+	<p data-frontmatter-title>{frontmatter.title}</p>
 	<p data-layout-rendered>Layout rendered!</p>
+	<ul data-headings>
+		{headings.map(heading => <li>{heading.slug}</li>)}
+	</ul>
 	<slot />
 </body>
+
 </html>

package/test/fixtures/mdx-frontmatter/src/pages/index.mdx

@@ -5,3 +5,6 @@ illThrowIfIDontExist: "Oh no, that's scary!"
 ---
 
 {frontmatter.illThrowIfIDontExist}
+
+> Note: newline intentionally missing from the end of this file.
+> Useful since that can be the source of bugs in our compile step.

package/test/fixtures/mdx-frontmatter/src/pages/with-headings.mdx

@@ -0,0 +1,7 @@
+---
+layout: '../layouts/Base.astro'
+---
+
+## Section 1
+
+## Section 2

package/test/fixtures/mdx-frontmatter-injection/astro.config.mjs

@@ -0,0 +1,12 @@
+import { defineConfig } from 'astro/config';
+import mdx from '@astrojs/mdx';
+import { rehypeReadingTime, remarkTitle } from './src/markdown-plugins.mjs';
+
+// https://astro.build/config
+export default defineConfig({
+  site: 'https://astro.build/',
+	integrations: [mdx({
+		remarkPlugins: [remarkTitle],
+		rehypePlugins: [rehypeReadingTime],
+	})],
+});

package/test/fixtures/mdx-frontmatter-injection/node_modules/.bin/astro

@@ -0,0 +1,17 @@
+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
+esac
+
+if [ -z "$NODE_PATH" ]; then
+  export NODE_PATH="/home/runner/work/astro/astro/node_modules/.pnpm/node_modules"
+else
+  export NODE_PATH="$NODE_PATH:/home/runner/work/astro/astro/node_modules/.pnpm/node_modules"
+fi
+if [ -x "$basedir/node" ]; then
+  exec "$basedir/node"  "$basedir/../../../../../../../astro/astro.js" "$@"
+else
+  exec node  "$basedir/../../../../../../../astro/astro.js" "$@"
+fi

package/test/fixtures/mdx-frontmatter-injection/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@test/mdx-frontmatter-injection",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "astro": "workspace:*",
+    "@astrojs/mdx": "workspace:*",
+    "mdast-util-to-string": "^3.1.0",
+    "reading-time": "^1.5.0",
+    "unist-util-visit": "^4.1.0"
+  }
+}

package/test/fixtures/mdx-frontmatter-injection/src/markdown-plugins.mjs

@@ -0,0 +1,20 @@
+import getReadingTime from 'reading-time';
+import { toString } from 'mdast-util-to-string';
+import { visit } from 'unist-util-visit';
+
+export function rehypeReadingTime() {
+	return function (tree, { data }) {
+		const readingTime = getReadingTime(toString(tree));
+		data.astro.frontmatter.injectedReadingTime = readingTime;
+	};
+}
+
+export function remarkTitle() {
+	return function (tree, { data }) {
+		visit(tree, ['heading'], (node) => {
+			if (node.depth === 1) {
+				data.astro.frontmatter.title = toString(node.children);
+			}
+		});
+	};
+}

package/test/fixtures/mdx-frontmatter-injection/src/pages/glob.json.js

@@ -0,0 +1,6 @@
+export async function get() {
+	const docs = await import.meta.glob('./*.mdx', { eager: true });
+	return {
+		body: JSON.stringify(Object.values(docs).map(doc => doc.frontmatter)),
+	}
+}

package/test/fixtures/mdx-frontmatter-injection/src/pages/page-1.mdx

@@ -0,0 +1,3 @@
+# Page 1
+
+Look at that!

package/test/fixtures/mdx-frontmatter-injection/src/pages/page-2.mdx

@@ -0,0 +1,19 @@
+# Page 2
+
+## Table of contents
+
+## Section 1
+
+Some text!
+
+### Subsection 1
+
+Some subsection test!
+
+### Subsection 2
+
+Oh cool, more text!
+
+## Section 2
+
+More content

package/test/fixtures/mdx-frontmatter-injection/src/pages/with-overrides.mdx

@@ -0,0 +1,7 @@
+---
+title: 'Overridden title'
+injectedReadingTime: 
+  text: '1000 min read'
+---
+
+# Working!

package/test/fixtures/mdx-get-headings/src/pages/pages.json.js

@@ -0,0 +1,11 @@
+export async function get() {
+	const mdxPages = await import.meta.glob('./*.mdx', { eager: true });
+
+	return {
+		body: JSON.stringify({
+			headingsByPage: Object.fromEntries(
+				Object.entries(mdxPages ?? {}).map(([k, v]) => [k, v?.getHeadings()])
+			),
+		}),
+	}
+}

package/test/fixtures/mdx-get-headings/src/pages/test-with-jsx-expressions.mdx

@@ -0,0 +1,8 @@
+export const h2Title = "Section 1"
+export const h3Title = "Subsection 1"
+
+# Heading test with JSX expressions
+
+## {h2Title}
+
+### {h3Title}

package/test/fixtures/mdx-get-headings/src/pages/test.mdx

@@ -0,0 +1,9 @@
+# Heading test
+
+## Section 1
+
+### Subsection 1
+
+### Subsection 2
+
+## Section 2

package/test/fixtures/mdx-page/astro.config.ts

@@ -0,0 +1,5 @@
+import mdx from '@astrojs/mdx';
+
+export default {
+	integrations: [mdx()]
+}

package/test/fixtures/mdx-page/node_modules/.bin/astro

@@ -0,0 +1,17 @@
+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
+esac
+
+if [ -z "$NODE_PATH" ]; then
+  export NODE_PATH="/home/runner/work/astro/astro/node_modules/.pnpm/node_modules"
+else
+  export NODE_PATH="$NODE_PATH:/home/runner/work/astro/astro/node_modules/.pnpm/node_modules"
+fi
+if [ -x "$basedir/node" ]; then
+  exec "$basedir/node"  "$basedir/../../../../../../../astro/astro.js" "$@"
+else
+  exec node  "$basedir/../../../../../../../astro/astro.js" "$@"
+fi

package/test/fixtures/mdx-page/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@test/mdx-page",
+  "dependencies": {
+    "astro": "workspace:*",
+    "@astrojs/mdx": "workspace:*"
+  }
+}

package/test/fixtures/mdx-plus-react/astro.config.mjs

@@ -0,0 +1,6 @@
+import mdx from '@astrojs/mdx';
+import react from '@astrojs/react';
+
+export default {
+	integrations: [react(), mdx()]
+}

package/test/fixtures/mdx-plus-react/node_modules/.bin/astro

@@ -0,0 +1,17 @@
+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
+esac
+
+if [ -z "$NODE_PATH" ]; then
+  export NODE_PATH="/home/runner/work/astro/astro/node_modules/.pnpm/node_modules"
+else
+  export NODE_PATH="$NODE_PATH:/home/runner/work/astro/astro/node_modules/.pnpm/node_modules"
+fi
+if [ -x "$basedir/node" ]; then
+  exec "$basedir/node"  "$basedir/../../../../../../../astro/astro.js" "$@"
+else
+  exec node  "$basedir/../../../../../../../astro/astro.js" "$@"
+fi

package/test/fixtures/mdx-plus-react/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/mdx-plus-react",
+  "dependencies": {
+    "astro": "workspace:*",
+    "@astrojs/mdx": "workspace:*",
+    "@astrojs/react": "workspace:*"
+  }
+}

package/test/fixtures/mdx-plus-react/src/components/Component.jsx

@@ -0,0 +1,5 @@
+const Component = () => {
+  return <p>Hello world</p>;
+};
+
+export default Component;