@astrojs/markdown-remark 0.0.0-cloudcannon-fix-20230306211609

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@astrojs/markdown-remark",
+  "version": "0.0.0-cloudcannon-fix-20230306211609",
+  "type": "module",
+  "author": "withastro",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/withastro/astro.git",
+    "directory": "packages/markdown/remark"
+  },
+  "bugs": "https://github.com/withastro/astro/issues",
+  "homepage": "https://astro.build",
+  "main": "./dist/index.js",
+  "exports": {
+    ".": "./dist/index.js",
+    "./dist/internal.js": "./dist/internal.js"
+  },
+  "peerDependencies": {
+    "astro": "0.0.0-cloudcannon-fix-20230306211609"
+  },
+  "dependencies": {
+    "@astrojs/prism": "0.0.0-cloudcannon-fix-20230306211609",
+    "github-slugger": "^1.4.0",
+    "import-meta-resolve": "^2.1.0",
+    "rehype-raw": "^6.1.1",
+    "rehype-stringify": "^9.0.3",
+    "remark-gfm": "^3.0.1",
+    "remark-parse": "^10.0.1",
+    "remark-rehype": "^10.1.0",
+    "remark-smartypants": "^2.0.0",
+    "shiki": "^0.11.1",
+    "unified": "^10.1.2",
+    "unist-util-visit": "^4.1.0",
+    "vfile": "^5.3.2"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.1",
+    "@types/estree": "^1.0.0",
+    "@types/github-slugger": "^1.3.0",
+    "@types/hast": "^2.3.4",
+    "@types/mdast": "^3.0.10",
+    "@types/mocha": "^9.1.1",
+    "@types/unist": "^2.0.6",
+    "astro-scripts": "0.0.0-cloudcannon-fix-20230306211609",
+    "chai": "^4.3.6",
+    "mdast-util-mdx-expression": "^1.3.1",
+    "mocha": "^9.2.2"
+  },
+  "scripts": {
+    "prepublish": "pnpm build",
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc -p tsconfig.json",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
+    "postbuild": "astro-scripts copy \"src/**/*.js\"",
+    "dev": "astro-scripts dev \"src/**/*.ts\"",
+    "test": "mocha --exit --timeout 20000"
+  }
+}

package/src/frontmatter-injection.ts

@@ -0,0 +1,41 @@
+import type { Data, VFile } from 'vfile';
+import type { MarkdownAstroData } from './types.js';
+
+function isValidAstroData(obj: unknown): obj is MarkdownAstroData {
+	if (typeof obj === 'object' && obj !== null && obj.hasOwnProperty('frontmatter')) {
+		const { frontmatter } = obj as any;
+		try {
+			// ensure frontmatter is JSON-serializable
+			JSON.stringify(frontmatter);
+		} catch {
+			return false;
+		}
+		return typeof frontmatter === 'object' && frontmatter !== null;
+	}
+	return false;
+}
+
+export class InvalidAstroDataError extends TypeError {}
+
+export function safelyGetAstroData(vfileData: Data): MarkdownAstroData | InvalidAstroDataError {
+	const { astro } = vfileData;
+
+	if (!astro || !isValidAstroData(astro)) {
+		return new InvalidAstroDataError();
+	}
+
+	return astro;
+}
+
+export function toRemarkInitializeAstroData({
+	userFrontmatter,
+}: {
+	userFrontmatter: Record<string, any>;
+}) {
+	return () =>
+		function (tree: any, vfile: VFile) {
+			if (!vfile.data.astro) {
+				vfile.data.astro = { frontmatter: userFrontmatter };
+			}
+		};
+}

package/src/index.ts

@@ -0,0 +1,154 @@
+import type {
+	AstroMarkdownOptions,
+	MarkdownRenderingOptions,
+	MarkdownRenderingResult,
+	MarkdownVFile,
+} from './types';
+
+import { toRemarkInitializeAstroData } from './frontmatter-injection.js';
+import { loadPlugins } from './load-plugins.js';
+import { rehypeHeadingIds } from './rehype-collect-headings.js';
+import toRemarkContentRelImageError from './remark-content-rel-image-error.js';
+import remarkPrism from './remark-prism.js';
+import scopedStyles from './remark-scoped-styles.js';
+import remarkShiki from './remark-shiki.js';
+
+import rehypeRaw from 'rehype-raw';
+import rehypeStringify from 'rehype-stringify';
+import remarkGfm from 'remark-gfm';
+import markdown from 'remark-parse';
+import markdownToHtml from 'remark-rehype';
+import remarkSmartypants from 'remark-smartypants';
+import { unified } from 'unified';
+import { VFile } from 'vfile';
+
+export { rehypeHeadingIds } from './rehype-collect-headings.js';
+export * from './types.js';
+
+export const markdownConfigDefaults: Omit<Required<AstroMarkdownOptions>, 'drafts'> = {
+	syntaxHighlight: 'shiki',
+	shikiConfig: {
+		langs: [],
+		theme: 'github-dark',
+		wrap: false,
+	},
+	remarkPlugins: [],
+	rehypePlugins: [],
+	remarkRehype: {},
+	gfm: true,
+	smartypants: true,
+};
+
+/** Shared utility for rendering markdown */
+export async function renderMarkdown(
+	content: string,
+	opts: MarkdownRenderingOptions
+): Promise<MarkdownRenderingResult> {
+	let {
+		fileURL,
+		syntaxHighlight = markdownConfigDefaults.syntaxHighlight,
+		shikiConfig = markdownConfigDefaults.shikiConfig,
+		remarkPlugins = markdownConfigDefaults.remarkPlugins,
+		rehypePlugins = markdownConfigDefaults.rehypePlugins,
+		remarkRehype = markdownConfigDefaults.remarkRehype,
+		gfm = markdownConfigDefaults.gfm,
+		smartypants = markdownConfigDefaults.smartypants,
+		contentDir,
+		frontmatter: userFrontmatter = {},
+	} = opts;
+	const input = new VFile({ value: content, path: fileURL });
+	const scopedClassName = opts.$?.scopedClassName;
+
+	let parser = unified()
+		.use(markdown)
+		.use(toRemarkInitializeAstroData({ userFrontmatter }))
+		.use([]);
+
+	if (gfm) {
+		parser.use(remarkGfm);
+	}
+
+	if (smartypants) {
+		parser.use(remarkSmartypants);
+	}
+
+	const loadedRemarkPlugins = await Promise.all(loadPlugins(remarkPlugins));
+	const loadedRehypePlugins = await Promise.all(loadPlugins(rehypePlugins));
+
+	loadedRemarkPlugins.forEach(([plugin, pluginOpts]) => {
+		parser.use([[plugin, pluginOpts]]);
+	});
+
+	if (scopedClassName) {
+		parser.use([scopedStyles(scopedClassName)]);
+	}
+
+	if (syntaxHighlight === 'shiki') {
+		parser.use([await remarkShiki(shikiConfig, scopedClassName)]);
+	} else if (syntaxHighlight === 'prism') {
+		parser.use([remarkPrism(scopedClassName)]);
+	}
+
+	// Apply later in case user plugins resolve relative image paths
+	parser.use([toRemarkContentRelImageError({ contentDir })]);
+
+	parser.use([
+		[
+			markdownToHtml as any,
+			{
+				allowDangerousHtml: true,
+				passThrough: [],
+				...remarkRehype,
+			},
+		],
+	]);
+
+	loadedRehypePlugins.forEach(([plugin, pluginOpts]) => {
+		parser.use([[plugin, pluginOpts]]);
+	});
+
+	parser.use([rehypeHeadingIds, rehypeRaw]).use(rehypeStringify, { allowDangerousHtml: true });
+
+	let vfile: MarkdownVFile;
+	try {
+		vfile = await parser.process(input);
+	} catch (err) {
+		// Ensure that the error message contains the input filename
+		// to make it easier for the user to fix the issue
+		err = prefixError(err, `Failed to parse Markdown file "${input.path}"`);
+		// eslint-disable-next-line no-console
+		console.error(err);
+		throw err;
+	}
+
+	const headings = vfile?.data.__astroHeadings || [];
+	return {
+		metadata: { headings, source: content, html: String(vfile.value) },
+		code: String(vfile.value),
+		vfile,
+	};
+}
+
+function prefixError(err: any, prefix: string) {
+	// If the error is an object with a `message` property, attempt to prefix the message
+	if (err && err.message) {
+		try {
+			err.message = `${prefix}:\n${err.message}`;
+			return err;
+		} catch (error) {
+			// Any errors here are ok, there's fallback code below
+		}
+	}
+
+	// If that failed, create a new error with the desired message and attempt to keep the stack
+	const wrappedError = new Error(`${prefix}${err ? `: ${err}` : ''}`);
+	try {
+		wrappedError.stack = err.stack;
+		// @ts-ignore
+		wrappedError.cause = err;
+	} catch (error) {
+		// It's ok if we could not set the stack or cause - the message is the most important part
+	}
+
+	return wrappedError;
+}

package/src/internal.ts

@@ -0,0 +1,5 @@
+export {
+	InvalidAstroDataError,
+	safelyGetAstroData,
+	toRemarkInitializeAstroData,
+} from './frontmatter-injection.js';

package/src/load-plugins.ts

@@ -0,0 +1,42 @@
+import { resolve as importMetaResolve } from 'import-meta-resolve';
+import path from 'path';
+import * as unified from 'unified';
+import { pathToFileURL } from 'url';
+
+const cwdUrlStr = pathToFileURL(path.join(process.cwd(), 'package.json')).toString();
+
+async function importPlugin(p: string | unified.Plugin): Promise<unified.Plugin> {
+	if (typeof p === 'string') {
+		// Try import from this package first
+		try {
+			const importResult = await import(p);
+			return importResult.default;
+		} catch {}
+
+		// Try import from user project
+		const resolved = await importMetaResolve(p, cwdUrlStr);
+		const importResult = await import(resolved);
+		return importResult.default;
+	}
+
+	return p;
+}
+
+export function loadPlugins(
+	items: (string | [string, any] | unified.Plugin<any[], any> | [unified.Plugin<any[], any>, any])[]
+): Promise<[unified.Plugin, any?]>[] {
+	return items.map((p) => {
+		return new Promise((resolve, reject) => {
+			if (Array.isArray(p)) {
+				const [plugin, opts] = p;
+				return importPlugin(plugin)
+					.then((m) => resolve([m, opts]))
+					.catch((e) => reject(e));
+			}
+
+			return importPlugin(p)
+				.then((m) => resolve([m]))
+				.catch((e) => reject(e));
+		});
+	});
+}

package/src/rehype-collect-headings.ts

@@ -0,0 +1,129 @@
+import { type Expression, type Super } from 'estree';
+import Slugger from 'github-slugger';
+import { type MdxTextExpression } from 'mdast-util-mdx-expression';
+import { type Node } from 'unist';
+import { visit } from 'unist-util-visit';
+
+import { InvalidAstroDataError, safelyGetAstroData } from './frontmatter-injection.js';
+import type { MarkdownAstroData, MarkdownHeading, MarkdownVFile, RehypePlugin } from './types.js';
+
+const rawNodeTypes = new Set(['text', 'raw', 'mdxTextExpression']);
+const codeTagNames = new Set(['code', 'pre']);
+
+export function rehypeHeadingIds(): ReturnType<RehypePlugin> {
+	return function (tree, file: MarkdownVFile) {
+		const headings: MarkdownHeading[] = [];
+		const slugger = new Slugger();
+		const isMDX = isMDXFile(file);
+		const astroData = safelyGetAstroData(file.data);
+		visit(tree, (node) => {
+			if (node.type !== 'element') return;
+			const { tagName } = node;
+			if (tagName[0] !== 'h') return;
+			const [_, level] = tagName.match(/h([0-6])/) ?? [];
+			if (!level) return;
+			const depth = Number.parseInt(level);
+
+			let text = '';
+			visit(node, (child, __, parent) => {
+				if (child.type === 'element' || parent == null) {
+					return;
+				}
+				if (child.type === 'raw') {
+					if (child.value.match(/^\n?<.*>\n?$/)) {
+						return;
+					}
+				}
+				if (rawNodeTypes.has(child.type)) {
+					if (isMDX || codeTagNames.has(parent.tagName)) {
+						let value = child.value;
+						if (isMdxTextExpression(child) && !(astroData instanceof InvalidAstroDataError)) {
+							const frontmatterPath = getMdxFrontmatterVariablePath(child);
+							if (Array.isArray(frontmatterPath) && frontmatterPath.length > 0) {
+								const frontmatterValue = getMdxFrontmatterVariableValue(astroData, frontmatterPath);
+								if (typeof frontmatterValue === 'string') {
+									value = frontmatterValue;
+								}
+							}
+						}
+						text += value;
+					} else {
+						text += child.value.replace(/\{/g, '${');
+					}
+				}
+			});
+
+			node.properties = node.properties || {};
+			if (typeof node.properties.id !== 'string') {
+				let slug = slugger.slug(text);
+
+				if (slug.endsWith('-')) slug = slug.slice(0, -1);
+
+				node.properties.id = slug;
+			}
+
+			headings.push({ depth, slug: node.properties.id, text });
+		});
+
+		file.data.__astroHeadings = headings;
+	};
+}
+
+function isMDXFile(file: MarkdownVFile) {
+	return Boolean(file.history[0]?.endsWith('.mdx'));
+}
+
+/**
+ * Check if an ESTree entry is `frontmatter.*.VARIABLE`.
+ * If it is, return the variable path (i.e. `["*", ..., "VARIABLE"]`) minus the `frontmatter` prefix.
+ */
+function getMdxFrontmatterVariablePath(node: MdxTextExpression): string[] | Error {
+	if (!node.data?.estree || node.data.estree.body.length !== 1) return new Error();
+
+	const statement = node.data.estree.body[0];
+
+	// Check for "[ANYTHING].[ANYTHING]".
+	if (statement?.type !== 'ExpressionStatement' || statement.expression.type !== 'MemberExpression')
+		return new Error();
+
+	let expression: Expression | Super = statement.expression;
+	const expressionPath: string[] = [];
+
+	// Traverse the expression, collecting the variable path.
+	while (
+		expression.type === 'MemberExpression' &&
+		expression.property.type === (expression.computed ? 'Literal' : 'Identifier')
+	) {
+		expressionPath.push(
+			expression.property.type === 'Literal'
+				? String(expression.property.value)
+				: expression.property.name
+		);
+
+		expression = expression.object;
+	}
+
+	// Check for "frontmatter.[ANYTHING]".
+	if (expression.type !== 'Identifier' || expression.name !== 'frontmatter') return new Error();
+
+	return expressionPath.reverse();
+}
+
+function getMdxFrontmatterVariableValue(astroData: MarkdownAstroData, path: string[]) {
+	let value: MdxFrontmatterVariableValue = astroData.frontmatter;
+
+	for (const key of path) {
+		if (!value[key]) return undefined;
+
+		value = value[key];
+	}
+
+	return value;
+}
+
+function isMdxTextExpression(node: Node): node is MdxTextExpression {
+	return node.type === 'mdxTextExpression';
+}
+
+type MdxFrontmatterVariableValue =
+	MarkdownAstroData['frontmatter'][keyof MarkdownAstroData['frontmatter']];

package/src/remark-content-rel-image-error.ts

@@ -0,0 +1,53 @@
+import type { Image } from 'mdast';
+import { visit } from 'unist-util-visit';
+import { pathToFileURL } from 'url';
+import type { VFile } from 'vfile';
+
+/**
+ * `src/content/` does not support relative image paths.
+ * This plugin throws an error if any are found
+ */
+export default function toRemarkContentRelImageError({ contentDir }: { contentDir: URL }) {
+	return function remarkContentRelImageError() {
+		return (tree: any, vfile: VFile) => {
+			if (typeof vfile?.path !== 'string') return;
+
+			const isContentFile = pathToFileURL(vfile.path).href.startsWith(contentDir.href);
+			if (!isContentFile) return;
+
+			const relImagePaths = new Set<string>();
+			visit(tree, 'image', function raiseError(node: Image) {
+				if (isRelativePath(node.url)) {
+					relImagePaths.add(node.url);
+				}
+			});
+			if (relImagePaths.size === 0) return;
+
+			const errorMessage =
+				`Relative image paths are not supported in the content/ directory. Place local images in the public/ directory and use absolute paths (see https://docs.astro.build/en/guides/images/#in-markdown-files)\n` +
+				[...relImagePaths].map((path) => JSON.stringify(path)).join(',\n');
+
+			// Throw raw string to use `astro:markdown` default formatting
+			throw errorMessage;
+		};
+	};
+}
+
+// Following utils taken from `packages/astro/src/core/path.ts`:
+
+function isRelativePath(path: string) {
+	return startsWithDotDotSlash(path) || startsWithDotSlash(path);
+}
+
+function startsWithDotDotSlash(path: string) {
+	const c1 = path[0];
+	const c2 = path[1];
+	const c3 = path[2];
+	return c1 === '.' && c2 === '.' && c3 === '/';
+}
+
+function startsWithDotSlash(path: string) {
+	const c1 = path[0];
+	const c2 = path[1];
+	return c1 === '.' && c2 === '/';
+}

package/src/remark-prism.ts

@@ -0,0 +1,32 @@
+import { runHighlighterWithAstro } from '@astrojs/prism/dist/highlighter';
+import { visit } from 'unist-util-visit';
+const noVisit = new Set(['root', 'html', 'text']);
+
+type MaybeString = string | null | undefined;
+
+/**  */
+function transformer(className: MaybeString) {
+	return function (tree: any) {
+		const visitor = (node: any) => {
+			let { lang, value } = node;
+			node.type = 'html';
+
+			let { html, classLanguage } = runHighlighterWithAstro(lang, value);
+			let classes = [classLanguage];
+			if (className) {
+				classes.push(className);
+			}
+			node.value = `<pre class="${classes.join(
+				' '
+			)}"><code is:raw class="${classLanguage}">${html}</code></pre>`;
+			return node;
+		};
+		return visit(tree, 'code', visitor);
+	};
+}
+
+function plugin(className: MaybeString) {
+	return transformer.bind(null, className);
+}
+
+export default plugin;

package/src/remark-scoped-styles.ts

@@ -0,0 +1,18 @@
+import { visit } from 'unist-util-visit';
+const noVisit = new Set(['root', 'html', 'text']);
+
+/**  */
+export default function scopedStyles(className: string) {
+	const visitor = (node: any) => {
+		if (noVisit.has(node.type)) return;
+
+		const { data } = node;
+		let currentClassName = data?.hProperties?.class ?? '';
+		node.data = node.data || {};
+		node.data.hProperties = node.data.hProperties || {};
+		node.data.hProperties.class = `${className} ${currentClassName}`.trim();
+
+		return node;
+	};
+	return () => (tree: any) => visit(tree, visitor);
+}

package/src/remark-shiki.ts

@@ -0,0 +1,106 @@
+import type * as shiki from 'shiki';
+import { getHighlighter } from 'shiki';
+import { visit } from 'unist-util-visit';
+import type { ShikiConfig } from './types.js';
+
+/**
+ * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
+ * cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
+ * We make this async, so that multiple calls to parse markdown still share the same highlighter.
+ */
+const highlighterCacheAsync = new Map<string, Promise<shiki.Highlighter>>();
+
+const remarkShiki = async (
+	{ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig,
+	scopedClassName?: string | null
+) => {
+	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
+	let highlighterAsync = highlighterCacheAsync.get(cacheID);
+	if (!highlighterAsync) {
+		highlighterAsync = getHighlighter({ theme }).then((hl) => {
+			hl.setColorReplacements({
+				'#000001': 'var(--astro-code-color-text)',
+				'#000002': 'var(--astro-code-color-background)',
+				'#000004': 'var(--astro-code-token-constant)',
+				'#000005': 'var(--astro-code-token-string)',
+				'#000006': 'var(--astro-code-token-comment)',
+				'#000007': 'var(--astro-code-token-keyword)',
+				'#000008': 'var(--astro-code-token-parameter)',
+				'#000009': 'var(--astro-code-token-function)',
+				'#000010': 'var(--astro-code-token-string-expression)',
+				'#000011': 'var(--astro-code-token-punctuation)',
+				'#000012': 'var(--astro-code-token-link)',
+			});
+			return hl;
+		});
+		highlighterCacheAsync.set(cacheID, highlighterAsync);
+	}
+	const highlighter = await highlighterAsync;
+
+	// NOTE: There may be a performance issue here for large sites that use `lang`.
+	// Since this will be called on every page load. Unclear how to fix this.
+	for (const lang of langs) {
+		await highlighter.loadLanguage(lang);
+	}
+
+	return () => (tree: any) => {
+		visit(tree, 'code', (node) => {
+			let lang: string;
+
+			if (typeof node.lang === 'string') {
+				const langExists = highlighter.getLoadedLanguages().includes(node.lang);
+				if (langExists) {
+					lang = node.lang;
+				} else {
+					// eslint-disable-next-line no-console
+					console.warn(`The language "${node.lang}" doesn't exist, falling back to plaintext.`);
+					lang = 'plaintext';
+				}
+			} else {
+				lang = 'plaintext';
+			}
+
+			let html = highlighter!.codeToHtml(node.value, { lang });
+
+			// Q: Couldn't these regexes match on a user's inputted code blocks?
+			// A: Nope! All rendered HTML is properly escaped.
+			// Ex. If a user typed `<span class="line"` into a code block,
+			// It would become this before hitting our regexes:
+			// &lt;span class=&quot;line&quot;
+
+			// Replace "shiki" class naming with "astro" and add "is:raw".
+			html = html.replace(
+				/<pre class="(.*?)shiki(.*?)"/,
+				`<pre is:raw class="$1astro-code$2${scopedClassName ? ' ' + scopedClassName : ''}"`
+			);
+			// Add "user-select: none;" for "+"/"-" diff symbols
+			if (node.lang === 'diff') {
+				html = html.replace(
+					/<span class="line"><span style="(.*?)">([\+|\-])/g,
+					'<span class="line"><span style="$1"><span style="user-select: none;">$2</span>'
+				);
+			}
+			// Handle code wrapping
+			// if wrap=null, do nothing.
+			if (wrap === false) {
+				html = html.replace(/style="(.*?)"/, 'style="$1; overflow-x: auto;"');
+			} else if (wrap === true) {
+				html = html.replace(
+					/style="(.*?)"/,
+					'style="$1; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;"'
+				);
+			}
+
+			// Apply scopedClassName to all nested lines
+			if (scopedClassName) {
+				html = html.replace(/\<span class="line"\>/g, `<span class="line ${scopedClassName}"`);
+			}
+
+			node.type = 'html';
+			node.value = html;
+			node.children = [];
+		});
+	};
+};
+
+export default remarkShiki;