astro-xmdx 0.0.2

package/src/vite-plugin/directive-rewriter.ts

@@ -0,0 +1,272 @@
+/**
+ * Directive rewriting utilities for fallback MDX compilation
+ * @module vite-plugin/directive-rewriter
+ */
+
+import type { Registry } from 'xmdx/registry';
+import { starlightLibrary } from 'xmdx/registry';
+import { collectImportedNames, insertAfterImports } from '../utils/imports.js';
+
+/**
+ * Opening directive state for stack tracking.
+ */
+type DirectiveOpening = {
+  name: string;
+  bracketTitle: string | null;
+  rawAttrs: string;
+  prefix: string;  // Leading whitespace and blockquote markers (e.g., "  ", "> ", "  > > ")
+  componentName: string;
+};
+
+/**
+ * Escapes a value for use in an HTML/JSX attribute.
+ */
+function escapeAttributeValue(value: string): string {
+  return value.replace(/&/g, '&').replace(/"/g, '"');
+}
+
+/**
+ * Normalizes directive attributes, stripping outer braces and filtering out reserved attrs.
+ */
+function normalizeDirectiveAttrs(attrs: string, hasBracketTitle: boolean): string {
+  if (!attrs) {
+    return '';
+  }
+
+  // Strip outer braces from remark-directive syntax: {key="value"} → key="value"
+  let normalized = attrs.trim();
+  if (normalized.startsWith('{') && normalized.endsWith('}')) {
+    normalized = normalized.slice(1, -1).trim();
+  }
+
+  const tokens = normalized.split(/\s+/).filter(Boolean);
+  const cleaned: string[] = [];
+  for (const tok of tokens) {
+    const key = tok.split('=')[0]?.trim() ?? '';
+    if (!key) continue;
+    const lower = key.toLowerCase();
+    if (lower === 'type') continue;
+    if (hasBracketTitle && lower === 'title') continue;
+    cleaned.push(tok);
+  }
+  return cleaned.join(' ');
+}
+
+/**
+ * Parses an opening directive line (e.g., ":::note[Title]").
+ */
+function parseOpeningDirective(
+  afterPrefix: string,
+  supported: Set<string>,
+  prefix: string
+): { name: string; bracketTitle: string | null; rawAttrs: string; prefix: string } | null {
+  // Content is already after the prefix; check for directive start
+  if (!afterPrefix.startsWith(':::')) {
+    return null;
+  }
+
+  let rest = afterPrefix.slice(3);
+  let name = '';
+  while (rest.length > 0 && /[A-Za-z]/.test(rest[0] ?? '')) {
+    name += (rest[0] ?? '').toLowerCase();
+    rest = rest.slice(1);
+  }
+
+  if (!name || !supported.has(name)) {
+    return null;
+  }
+
+  let bracketTitle: string | null = null;
+  if (rest.startsWith('[')) {
+    rest = rest.slice(1);
+    let title = '';
+    while (rest.length > 0) {
+      const ch = rest[0] ?? '';
+      rest = rest.slice(1);
+      if (ch === ']') {
+        bracketTitle = title;
+        break;
+      }
+      title += ch;
+    }
+  }
+
+  const rawAttrs = normalizeDirectiveAttrs(rest.trim(), Boolean(bracketTitle));
+  return { name, bracketTitle, rawAttrs, prefix };
+}
+
+/**
+ * Parses a closing directive line (":::").
+ */
+function parseDirectiveCloser(afterPrefix: string, prefix: string): { prefix: string } | null {
+  // Check if the content after prefix is exactly `:::`
+  if (afterPrefix.trim() === ':::') {
+    return { prefix };
+  }
+  return null;
+}
+
+/**
+ * Rewrites directive syntax (:::note, :::tip, etc.) to JSX component syntax.
+ * Used for fallback MDX compilation when xmdx-core can't handle the file.
+ */
+export function rewriteFallbackDirectives(
+  source: string,
+  registry: Registry | null,
+  hasStarlightConfigured: boolean
+): { code: string; usedComponents: Set<string>; changed: boolean } {
+  if (!source) {
+    return { code: source, usedComponents: new Set(), changed: false };
+  }
+
+  // Get directives from registry, fall back to starlightLibrary defaults
+  const registryDirectives = registry?.getSupportedDirectives().map((name) => name.toLowerCase()) ?? [];
+  const supportedSet = new Set(registryDirectives);
+
+  // Add Starlight directives only if registry is empty AND Starlight is configured
+  const useDefaultDirectives = supportedSet.size === 0 && hasStarlightConfigured;
+  if (useDefaultDirectives) {
+    const starlightDirectives = starlightLibrary.directiveMappings ?? [];
+    for (const mapping of starlightDirectives) {
+      supportedSet.add(mapping.directive.toLowerCase());
+    }
+  }
+
+  const lines = source.split(/\r?\n/);
+  const output: string[] = [];
+  const stack: DirectiveOpening[] = [];
+  const usedComponents = new Set<string>();
+  let changed = false;
+  let inFence = false;
+  let fenceChar: string | null = null;
+
+  for (const line of lines) {
+    // Extract prefix (whitespace + blockquote markers) like we do for directives
+    const prefixMatch = line.match(/^(\s*(?:>\s*)*)/);
+    const prefix = prefixMatch?.[1] ?? '';
+    const afterPrefix = line.slice(prefix.length);
+
+    // Check for code fence after stripping prefix (handles blockquoted code fences)
+    const fenceMatch = afterPrefix.match(/^([`~]{3,})/);
+    if (fenceMatch) {
+      const char = fenceMatch[1]?.[0] ?? null;
+      if (!inFence) {
+        inFence = true;
+        fenceChar = char;
+      } else if (char && fenceChar === char) {
+        inFence = false;
+        fenceChar = null;
+      }
+      output.push(line);
+      continue;
+    }
+
+    if (inFence) {
+      output.push(line);
+      continue;
+    }
+
+    const opening = parseOpeningDirective(afterPrefix, supportedSet, prefix);
+    if (opening) {
+      // Try registry first, then fall back to starlightLibrary
+      const mapping = registry?.getDirectiveMapping(opening.name)
+        ?? (useDefaultDirectives
+          ? starlightLibrary.directiveMappings?.find(m => m.directive.toLowerCase() === opening.name)
+          : null);
+      if (!mapping) {
+        output.push(line);
+        continue;
+      }
+
+      const componentName = mapping.component;
+      const props: string[] = ['data-mf-source="directive"'];
+      if (mapping.injectProps) {
+        for (const [propKey, propSource] of Object.entries(mapping.injectProps)) {
+          if (propSource.source === 'directive_name') {
+            props.push(`${propKey}="${escapeAttributeValue(opening.name)}"`);
+          } else if (propSource.source === 'bracket_title' && opening.bracketTitle) {
+            props.push(`${propKey}="${escapeAttributeValue(opening.bracketTitle)}"`);
+          } else if (propSource.source === 'literal' && propSource.value) {
+            props.push(`${propKey}="${escapeAttributeValue(propSource.value)}"`);
+          }
+        }
+      }
+
+      if (opening.bracketTitle) {
+        props.push(`title="${escapeAttributeValue(opening.bracketTitle)}"`);
+      }
+      if (opening.rawAttrs) {
+        props.push(opening.rawAttrs);
+      }
+
+      const propsStr = props.length > 0 ? ` ${props.join(' ')}` : '';
+      output.push(`${opening.prefix}<${componentName}${propsStr}>`);
+      stack.push({ ...opening, componentName });
+      usedComponents.add(componentName);
+      changed = true;
+      continue;
+    }
+
+    const closer = parseDirectiveCloser(afterPrefix, prefix);
+    if (closer && stack.length > 0) {
+      const opened = stack.pop();
+      if (opened) {
+        output.push(`${opened.prefix}</${opened.componentName}>`);
+        changed = true;
+        continue;
+      }
+    }
+
+    output.push(line);
+  }
+
+  while (stack.length > 0) {
+    const opened = stack.pop();
+    if (opened) {
+      output.push(`${opened.prefix}</${opened.componentName}>`);
+    }
+  }
+
+  return { code: output.join('\n'), usedComponents, changed };
+}
+
+/**
+ * Injects component imports for components used in rewritten directives.
+ */
+export function injectFallbackImports(
+  source: string,
+  usedComponents: Set<string>,
+  registry: Registry | null,
+  hasStarlightConfigured: boolean
+): string {
+  if (!source || usedComponents.size === 0) {
+    return source;
+  }
+
+  const imported = collectImportedNames(source);
+  const importLines: string[] = [];
+
+  for (const componentName of usedComponents) {
+    if (imported.has(componentName)) {
+      continue;
+    }
+    const def = registry?.getComponent(componentName);
+    if (def) {
+      if (def.exportType === 'named') {
+        importLines.push(`import { ${componentName} } from '${def.modulePath}';`);
+      } else {
+        importLines.push(`import ${componentName} from '${def.modulePath}/${componentName}.astro';`);
+      }
+    } else if (componentName === 'Aside' && hasStarlightConfigured) {
+      // Fallback for Starlight Aside component when using default directives
+      // Only inject if Starlight is actually configured to avoid module-not-found errors
+      importLines.push(`import { Aside } from '@astrojs/starlight/components';`);
+    }
+  }
+
+  if (importLines.length === 0) {
+    return source;
+  }
+
+  return insertAfterImports(source, importLines.join('\n'));
+}

package/src/vite-plugin/esbuild-pool.ts

@@ -0,0 +1,173 @@
+/**
+ * Worker pool management for parallel esbuild processing.
+ * Spawns multiple worker threads to handle large batches of JSX files.
+ * Uses inline worker code to avoid Node.js TypeScript loading issues.
+ * @module vite-plugin/esbuild-pool
+ */
+
+import { Worker } from 'node:worker_threads';
+import os from 'node:os';
+
+interface WorkerInput {
+  entries: Array<{ entryName: string; id: string; jsx: string }>;
+}
+
+interface WorkerOutput {
+  results: Array<{ id: string; code: string; map?: string }>;
+  errors: Array<{ id: string; error: string }>;
+}
+
+/**
+ * Inline worker code as a string.
+ * This avoids issues with Node.js not supporting TypeScript in node_modules.
+ */
+const WORKER_CODE = `
+const { parentPort } = require('node:worker_threads');
+const { build: esbuildBuild } = require('esbuild');
+const path = require('node:path');
+
+async function processChunk(input) {
+  const entryMap = new Map();
+  for (const entry of input.entries) {
+    entryMap.set(entry.entryName, { id: entry.id, jsx: entry.jsx });
+  }
+
+  const results = [];
+  const errors = [];
+
+  try {
+    const result = await esbuildBuild({
+      write: false,
+      bundle: false,
+      format: 'esm',
+      sourcemap: 'external',
+      loader: { '.jsx': 'jsx' },
+      jsx: 'transform',
+      jsxFactory: '_jsx',
+      jsxFragment: '_Fragment',
+      entryPoints: Array.from(entryMap.keys()),
+      outdir: 'out',
+      plugins: [{
+        name: 'xmdx-virtual-jsx-worker',
+        setup(build) {
+          build.onResolve({ filter: /^entry\\d+\\.jsx$/ }, args => ({
+            path: args.path, namespace: 'xmdx-jsx'
+          }));
+          build.onResolve({ filter: /.*/ }, args => ({
+            path: args.path, external: true
+          }));
+          build.onLoad({ filter: /.*/, namespace: 'xmdx-jsx' }, args => {
+            const entry = entryMap.get(args.path);
+            return entry ? { contents: entry.jsx, loader: 'jsx' } : null;
+          });
+        }
+      }]
+    });
+
+    for (const output of result.outputFiles || []) {
+      const basename = path.basename(output.path);
+      if (basename.endsWith('.map')) continue;
+      const entryName = basename.replace(/\\.js$/, '.jsx');
+      const entry = entryMap.get(entryName);
+      if (entry) {
+        const mapOutput = result.outputFiles.find(o => o.path === output.path + '.map');
+        results.push({ id: entry.id, code: output.text, map: mapOutput?.text });
+      }
+    }
+  } catch (err) {
+    for (const entry of input.entries) {
+      errors.push({ id: entry.id, error: err.message });
+    }
+  }
+
+  return { results, errors };
+}
+
+parentPort.on('message', async (input) => {
+  const output = await processChunk(input);
+  parentPort.postMessage(output);
+});
+`;
+
+/**
+ * Run esbuild in parallel using worker threads.
+ * Splits the input into chunks and processes them concurrently.
+ *
+ * @param jsxInputs - Array of JSX inputs to transform
+ * @returns Map of file IDs to transformed code and source maps
+ */
+export async function runParallelEsbuild(
+  jsxInputs: Array<{ id: string; jsx: string }>
+): Promise<Map<string, { code: string; map?: string }>> {
+  // Use CPU count - 1, capped at 8 workers max
+  const workerCount = Math.max(1, Math.min(os.cpus().length - 1, 8));
+  const chunkSize = Math.ceil(jsxInputs.length / workerCount);
+
+  // Split into chunks
+  const chunks: Array<Array<{ entryName: string; id: string; jsx: string }>> = [];
+  for (let i = 0; i < jsxInputs.length; i += chunkSize) {
+    const chunk = jsxInputs.slice(i, i + chunkSize).map((input, j) => ({
+      entryName: `entry${i + j}.jsx`,
+      id: input.id,
+      jsx: input.jsx,
+    }));
+    chunks.push(chunk);
+  }
+
+  // Run workers in parallel
+  const workerPromises = chunks.map((chunk) => runWorker(chunk));
+  const results = await Promise.all(workerPromises);
+
+  // Merge results
+  const merged = new Map<string, { code: string; map?: string }>();
+  for (const result of results) {
+    for (const item of result.results) {
+      merged.set(item.id, { code: item.code, map: item.map });
+    }
+    // Log any errors (but don't fail the whole batch)
+    for (const error of result.errors) {
+      console.warn(`[xmdx] Worker esbuild error for ${error.id}: ${error.error}`);
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Run a single worker with the given entries using inline code.
+ */
+function runWorker(
+  entries: Array<{ entryName: string; id: string; jsx: string }>
+): Promise<WorkerOutput> {
+  return new Promise<WorkerOutput>((resolve, reject) => {
+    // Use eval mode to execute inline JavaScript code
+    const worker = new Worker(WORKER_CODE, { eval: true });
+
+    // 60 second timeout
+    const timeout = setTimeout(() => {
+      worker.terminate();
+      reject(new Error('Worker timeout after 60s'));
+    }, 60000);
+
+    worker.on('message', (output: WorkerOutput) => {
+      clearTimeout(timeout);
+      worker.terminate();
+      resolve(output);
+    });
+
+    worker.on('error', (err) => {
+      clearTimeout(timeout);
+      worker.terminate();
+      reject(err);
+    });
+
+    worker.on('exit', (code) => {
+      if (code !== 0) {
+        clearTimeout(timeout);
+        reject(new Error(`Worker exited with code ${code}`));
+      }
+    });
+
+    worker.postMessage({ entries } satisfies WorkerInput);
+  });
+}

package/src/vite-plugin/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Vite plugin modules for Xmdx
+ * @module vite-plugin
+ */
+
+// Re-export types
+export type {
+  XmdxBinding,
+  XmdxCompiler,
+  CompileResult,
+  BatchCompileResult,
+  ParseBlocksResult,
+  XmdxPluginOptions,
+  DocumentFragment,
+  Node,
+  Element,
+  TextNode,
+} from './types.js';
+
+export { DEFAULT_EXTENSIONS } from '../utils/paths.js';
+
+// Re-export binding loader
+export { loadXmdxBinding, resetBindingPromise, ENABLE_SHIKI, IS_MDAST } from './binding-loader.js';
+
+// Re-export JSX module utilities
+export { compileFallbackModule } from './jsx-module.js';
+
+// Re-export directive rewriter
+export { rewriteFallbackDirectives, injectFallbackImports } from './directive-rewriter.js';
+
+// Re-export config normalization utilities
+export { normalizeStarlightComponents } from './normalize-config.js';
+export type { NormalizedStarlightComponents } from './normalize-config.js';
+
+// Re-export shiki highlighter
+export { createShikiHighlighter } from './shiki-highlighter.js';
+export type { ShikiHighlighter } from '../transforms/shiki.js';

package/src/vite-plugin/jsx-module.ts

@@ -0,0 +1,106 @@
+/**
+ * JSX module generation utilities
+ * @module vite-plugin/jsx-module
+ */
+
+import type { SourceMapInput } from 'rollup';
+import type { Registry } from 'xmdx/registry';
+import { transformWithEsbuild } from 'vite';
+import { compile as compileMdx } from '@mdx-js/mdx';
+import remarkGfm from 'remark-gfm';
+import remarkDirective from 'remark-directive';
+import { ESBUILD_JSX_CONFIG } from '../constants.js';
+import { stripFrontmatter } from '../utils/frontmatter.js';
+import { loadXmdxBinding } from './binding-loader.js';
+import { rewriteFallbackDirectives, injectFallbackImports } from './directive-rewriter.js';
+
+/**
+ * Compiles a fallback module using @mdx-js/mdx.
+ * Used for files with patterns that xmdx-core can't handle.
+ */
+export async function compileFallbackModule(
+  filename: string,
+  source: string,
+  virtualId: string,
+  registry: Registry | null,
+  hasStarlightConfigured: boolean
+): Promise<{ code: string; map?: SourceMapInput }> {
+  let frontmatter: Record<string, unknown> = {};
+  try {
+    const binding = await loadXmdxBinding();
+    const frontmatterResult = binding.parseFrontmatter(source);
+    frontmatter = frontmatterResult.frontmatter || {};
+  } catch {
+    frontmatter = {};
+  }
+
+  let sourceWithoutFrontmatter = stripFrontmatter(source);
+  const directiveResult = rewriteFallbackDirectives(sourceWithoutFrontmatter, registry, hasStarlightConfigured);
+  if (directiveResult.changed) {
+    sourceWithoutFrontmatter = injectFallbackImports(
+      directiveResult.code,
+      directiveResult.usedComponents,
+      registry,
+      hasStarlightConfigured
+    );
+  }
+  // Use @mdx-js/mdx to compile files that xmdx can't handle
+  // (e.g., files with import/export statements)
+  // Include remark-gfm for GFM features (tables, strikethrough, task lists)
+  // and remark-directive to handle unconverted ::: directives gracefully
+  const compiled = await compileMdx(sourceWithoutFrontmatter, {
+    jsxImportSource: 'astro',
+    remarkPlugins: [remarkGfm, remarkDirective],
+    // Don't use providerImportSource as it requires @mdx-js/react
+    // which may not be installed
+  });
+
+  // The compiled output is a VFile, get the string value
+  const mdxCode = String(compiled);
+
+  // Normalize MDX default export so we can wrap with Astro createComponent
+  const mdxWithoutDefault = mdxCode
+    .replace(/export default function MDXContent/g, 'function MDXContent')
+    .replace(/export default MDXContent\s*;/g, '')
+    .replace(/export\s*\{\s*MDXContent\s+as\s+default\s*\};?/g, '');
+
+  // Wrap in Astro-compatible module format
+  // @mdx-js/mdx outputs ESM with `export default function MDXContent(...)`
+  // We need to add Content, frontmatter and getHeadings exports for Astro compatibility
+  // Note: MDXContent is the default export function from @mdx-js/mdx
+  const wrappedCode = `
+import { createComponent, renderJSX } from 'astro/runtime/server/index.js';
+import { Fragment } from 'astro/jsx-runtime';
+${mdxWithoutDefault}
+
+// Re-export for Astro compatibility
+// Wrap MDXContent so it renders as an Astro component factory
+const XmdxContent = createComponent(
+  (result, props, _slots) =>
+    renderJSX(
+      result,
+      MDXContent({
+        ...(props ?? {}),
+        // Ensure Astro's Fragment is available for <Fragment slot="..."> usage in MDX.
+        components: { ...(props?.components ?? {}), Fragment },
+      })
+    ),
+  ${JSON.stringify(filename)}
+);
+export { MDXContent };
+export const Content = XmdxContent;
+export const file = ${JSON.stringify(filename)};
+export const url = undefined;
+export function getHeadings() { return []; }
+export const frontmatter = ${JSON.stringify(frontmatter)};
+export default XmdxContent;
+`;
+
+  // Transform JSX through esbuild (same as the main compilation path)
+  const esbuildResult = await transformWithEsbuild(wrappedCode, virtualId, ESBUILD_JSX_CONFIG);
+
+  return {
+    code: esbuildResult.code,
+    map: esbuildResult.map as SourceMapInput | undefined,
+  };
+}