astro-xmdx 0.0.2

package/src/utils/imports.ts

@@ -0,0 +1,201 @@
+/**
+ * Import statement manipulation utilities
+ * @module utils/imports
+ */
+
+import { stripCodeFences } from './mdx-detection.js';
+
+/**
+ * Collect all imported names from JavaScript/JSX code.
+ * Handles default imports, namespace imports, and named imports.
+ *
+ * @example
+ * const code = `
+ *   import React from 'react';
+ *   import { useState, useEffect } from 'react';
+ *   import * as utils from './utils';
+ * `;
+ * const names = collectImportedNames(code);
+ * // Set { 'React', 'useState', 'useEffect', 'utils' }
+ */
+export function collectImportedNames(code: string): Set<string> {
+  const imported = new Set<string>();
+  if (!code || typeof code !== 'string') {
+    return imported;
+  }
+  // Strip code fences to avoid false positives from code examples
+  const codeWithoutFences = stripCodeFences(code);
+  const lines = codeWithoutFences.split(/\r?\n/);
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith('import ') || trimmed.startsWith('import(')) {
+      continue;
+    }
+
+    // Default import: import Foo from 'module'
+    const defaultMatch = trimmed.match(
+      /^import\s+([A-Za-z$_][\w$]*)\s*(?:,|\s+from\s)/
+    );
+    if (defaultMatch?.[1]) {
+      imported.add(defaultMatch[1]);
+    }
+
+    // Namespace import: import * as Foo from 'module'
+    const namespaceMatch = trimmed.match(
+      /^import\s+\*\s+as\s+([A-Za-z$_][\w$]*)\s+from/
+    );
+    if (namespaceMatch?.[1]) {
+      imported.add(namespaceMatch[1]);
+    }
+
+    // Named imports: import { Foo, Bar as Baz } from 'module'
+    const namedMatch = trimmed.match(/import\s+{([^}]+)}\s+from/);
+    if (namedMatch?.[1]) {
+      const parts = namedMatch[1].split(',');
+      for (const part of parts) {
+        const item = part.trim();
+        if (!item) continue;
+        const segments = item.split(/\s+as\s+/);
+        const name = segments[1] ?? segments[0];
+        if (name) {
+          imported.add(name.trim());
+        }
+      }
+    }
+  }
+  return imported;
+}
+
+/**
+ * Insert import statement after existing imports in code.
+ * Finds the position after the last import statement and inserts the new import.
+ *
+ * @example
+ * const code = `
+ * import React from 'react';
+ *
+ * export default function App() {}
+ * `;
+ * const result = insertAfterImports(code, "import { Aside } from '@astrojs/starlight/components';");
+ * // Inserts after the React import
+ */
+export function insertAfterImports(code: string, importLine: string): string {
+  if (!code || typeof code !== 'string') {
+    return importLine;
+  }
+  const lines = code.split(/\r?\n/);
+  let idx = 0;
+  while (idx < lines.length) {
+    const trimmed = lines[idx]?.trim() ?? '';
+    if (!trimmed) {
+      idx += 1;
+      continue;
+    }
+    if (trimmed.startsWith('//') || trimmed.startsWith('/*')) {
+      idx += 1;
+      continue;
+    }
+    if (trimmed.startsWith('import ')) {
+      idx += 1;
+      continue;
+    }
+    break;
+  }
+
+  // MDX requires a blank line between imports and markdown content.
+  // Only add blank line if the next line is markdown content, not JavaScript.
+  const nextLine = lines[idx]?.trim() ?? '';
+  const isJavaScript = nextLine.startsWith('export ') ||
+    nextLine.startsWith('const ') ||
+    nextLine.startsWith('let ') ||
+    nextLine.startsWith('var ') ||
+    nextLine.startsWith('function ') ||
+    nextLine.startsWith('class ') ||
+    nextLine.startsWith('import ') ||
+    nextLine.startsWith('//') ||
+    nextLine.startsWith('/*');
+
+  if (nextLine && !isJavaScript) {
+    // There's markdown content after the insertion point; add blank line after the import
+    lines.splice(idx, 0, importLine, '');
+  } else {
+    lines.splice(idx, 0, importLine);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Extract import statements from MDX/JSX code.
+ * Returns the full import statement strings, preserving their original form.
+ *
+ * Import statements end when:
+ * 1. A semicolon is found
+ * 2. The import completes (has 'from' + module path)
+ * 3. The next line starts with a different statement type
+ *
+ * @example
+ * const code = `
+ * import Card from '~/components/Landing/Card.astro'
+ * import { useState } from 'react';
+ *
+ * # Hello World
+ * `;
+ * const imports = extractImportStatements(code);
+ * // ["import Card from '~/components/Landing/Card.astro'", "import { useState } from 'react';"]
+ */
+export function extractImportStatements(code: string): string[] {
+  const imports: string[] = [];
+  if (!code || typeof code !== 'string') {
+    return imports;
+  }
+  // Strip code fences to avoid false positives from code examples
+  const codeWithoutFences = stripCodeFences(code);
+  const lines = codeWithoutFences.split(/\r?\n/);
+
+  let currentImport = '';
+  let inImport = false;
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    if (!inImport) {
+      // Start of new import statement
+      if (trimmed.startsWith('import ') && !trimmed.startsWith('import(')) {
+        // Check if import is complete on this line
+        // Side-effect import: import './file.js' or import "styles.css" (no 'from' keyword)
+        const isSideEffectImport = /^import\s+['"][^'"]+['"]/.test(trimmed);
+        // Complete import has: import ... from '...' or import ... from "..."
+        const hasFromClause = /from\s+['"][^'"]+['"]/.test(trimmed);
+        if (isSideEffectImport || hasFromClause || trimmed.includes(';')) {
+          // Complete single-line import (with or without semicolon)
+          imports.push(trimmed);
+        } else {
+          // Start of multi-line import (e.g., multi-line named imports)
+          inImport = true;
+          currentImport = trimmed;
+        }
+      }
+    } else {
+      // Continue accumulating multi-line import
+      // Strip inline // comments, but only outside of quoted strings
+      const lineWithoutComment = trimmed.replace(/\s*\/\/.*$/, (match, offset) => {
+        const before = trimmed.slice(0, offset);
+        const singleQuotes = (before.match(/'/g) || []).length;
+        const doubleQuotes = (before.match(/"/g) || []).length;
+        // If inside a string (odd number of unescaped quotes), keep the match
+        if (singleQuotes % 2 !== 0 || doubleQuotes % 2 !== 0) return match;
+        return '';
+      });
+      currentImport += ' ' + lineWithoutComment;
+      const hasFromClause = /from\s+['"][^'"]+['"]/.test(currentImport);
+      if (hasFromClause || trimmed.includes(';')) {
+        // End of multi-line import
+        imports.push(currentImport);
+        currentImport = '';
+        inImport = false;
+      }
+    }
+  }
+
+  return imports;
+}

package/src/utils/mdx-detection.test.ts

@@ -0,0 +1,41 @@
+import { describe, it, expect } from 'bun:test';
+import { stripCodeFences } from './mdx-detection.js';
+
+describe('stripCodeFences', () => {
+  it('should strip basic 3-backtick fence', () => {
+    const input = 'before\n```js\nconst x = 1;\n```\nafter';
+    const result = stripCodeFences(input);
+    expect(result).toBe('before\nafter');
+  });
+
+  it('should strip tilde fences', () => {
+    const input = 'before\n~~~\ncode\n~~~\nafter';
+    const result = stripCodeFences(input);
+    expect(result).toBe('before\nafter');
+  });
+
+  it('should handle 4-backtick fence containing 3-backtick content', () => {
+    const input = 'before\n````md\n```\nnested\n```\n````\nafter';
+    const result = stripCodeFences(input);
+    expect(result).toBe('before\nafter');
+  });
+
+  it('should not close fence with shorter marker', () => {
+    const input = 'before\n````\n```\nstill inside\n````\nafter';
+    const result = stripCodeFences(input);
+    expect(result).toBe('before\nafter');
+  });
+
+  it('should not close fence with different marker type', () => {
+    const input = 'before\n```\n~~~\nstill inside\n```\nafter';
+    const result = stripCodeFences(input);
+    expect(result).toBe('before\nafter');
+  });
+
+  it('should not treat closing fence with info string as closer', () => {
+    const input = 'before\n```\n```js\nstill inside\n```\nafter';
+    const result = stripCodeFences(input);
+    // ```js has an info string so it's not a valid closer; the next ``` closes it
+    expect(result).toBe('before\nafter');
+  });
+});

package/src/utils/mdx-detection.ts

@@ -0,0 +1,209 @@
+/**
+ * MDX pattern detection utilities.
+ * @module utils/mdx-detection
+ */
+
+import { stripFrontmatter } from './frontmatter.js';
+import type { MdxImportHandlingOptions } from '../types.js';
+
+/**
+ * Detailed result from pattern detection.
+ */
+export interface MdxPatternDetectionResult {
+  /** Whether problematic patterns were found */
+  hasProblematicPatterns: boolean;
+  /** Human-readable reason for fallback */
+  reason?: string;
+  /** Disallowed import sources found (if any) */
+  disallowedImports?: string[];
+  /** All import sources found */
+  allImports?: string[];
+}
+
+/**
+ * Strip code fences from content to avoid false positives.
+ * Uses efficient single-pass character scanning instead of split() and per-line regex.
+ */
+export function stripCodeFences(content: string): string {
+  let result = '';
+  let pos = 0;
+  let inFence = false;
+  let fenceMarker = '';
+  let fenceLen = 0;
+
+  while (pos < content.length) {
+    // Find end of current line
+    let lineEnd = content.indexOf('\n', pos);
+    if (lineEnd === -1) lineEnd = content.length;
+
+    const line = content.slice(pos, lineEnd);
+
+    // Check for fence at start of line (after trimming leading whitespace)
+    let i = 0;
+    while (i < line.length && (line[i] === ' ' || line[i] === '\t')) i++;
+
+    const char = line[i];
+    if (char === '`' || char === '~') {
+      let len = 0;
+      const marker = char;
+      while (line[i] === marker) {
+        len++;
+        i++;
+      }
+
+      if (len >= 3) {
+        if (!inFence) {
+          // Opening fence
+          inFence = true;
+          fenceMarker = marker;
+          fenceLen = len;
+          pos = lineEnd + 1;
+          continue;
+        } else if (marker === fenceMarker && len >= fenceLen) {
+          // Check if rest of line is only whitespace (valid closer)
+          const rest = line.slice(i).trim();
+          if (rest === '') {
+            inFence = false;
+            pos = lineEnd + 1;
+            continue;
+          }
+        }
+      }
+    }
+
+    if (!inFence) {
+      result += line;
+      if (lineEnd < content.length) result += '\n';
+    }
+
+    pos = lineEnd + 1;
+  }
+
+  return result;
+}
+
+/**
+ * Convert a glob-like pattern to a regex.
+ * Supports * as wildcard.
+ */
+function patternToRegex(pattern: string): RegExp {
+  const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+  const withWildcard = escaped.replace(/\*/g, '.*');
+  return new RegExp(`^${withWildcard}$`);
+}
+
+/**
+ * Check if an import source matches any of the allowed patterns.
+ */
+function isAllowedImport(importSource: string, allowImports: string[]): boolean {
+  return allowImports.some((pattern) => {
+    const regex = patternToRegex(pattern);
+    return regex.test(importSource);
+  });
+}
+
+/**
+ * Extract import sources from content.
+ */
+function extractImportSources(content: string): string[] {
+  const sources: string[] = [];
+  // Match: import ... from 'source' or import 'source'
+  const importRegex = /^import\s+(?:(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)\s+from\s+)?['"]([^'"]+)['"]/gm;
+  let match;
+  while ((match = importRegex.exec(content)) !== null) {
+    if (match[1]) {
+      sources.push(match[1]);
+    }
+  }
+  return sources;
+}
+
+/**
+ * Detect MDX patterns that markdown-rs cannot parse correctly with detailed results.
+ * This includes:
+ * - MDX import statements (import ... from '...')
+ *
+ * Note: Export statements are now handled by Rust (hoisted_exports) and no longer
+ * trigger fallback. This significantly reduces fallback rate.
+ *
+ * @param source - The markdown/MDX source content
+ * @param options - MDX handling options
+ * @returns Detailed detection result with reasons
+ */
+export function detectProblematicMdxPatterns(
+  source: string,
+  options?: MdxImportHandlingOptions
+): MdxPatternDetectionResult {
+  // Skip frontmatter when checking for imports
+  let content = stripFrontmatter(source);
+
+  // Optionally strip code fences (default: true when options provided)
+  const ignoreCodeFences = options?.ignoreCodeFences ?? true;
+  if (ignoreCodeFences) {
+    content = stripCodeFences(content);
+  }
+
+  // Note: Export statements are now handled by Rust and no longer trigger fallback
+  // The Rust compiler extracts exports via collect_root_statements() and includes
+  // them in hoisted_exports, which TypeScript then injects into the JSX module.
+
+  // Check for import statements
+  const importPatterns = [
+    /^import\s+(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)\s+from\s+['"][^'"]+['"]/m,
+    /^import\s+['"][^'"]+['"]/m,
+  ];
+
+  const hasImports = importPatterns.some((pattern) => pattern.test(content));
+  if (!hasImports) {
+    return { hasProblematicPatterns: false };
+  }
+
+  const allImports = extractImportSources(content);
+
+  // If no allowed imports configured, any import is problematic
+  const allowImports = options?.allowImports;
+  if (!allowImports || allowImports.length === 0) {
+    return {
+      hasProblematicPatterns: true,
+      reason: `Contains imports with no allowImports configured: ${allImports.slice(0, 3).join(', ')}${allImports.length > 3 ? ` (+${allImports.length - 3} more)` : ''}`,
+      allImports,
+      disallowedImports: allImports,
+    };
+  }
+
+  // Check if all imports are from allowed sources
+  const disallowedImports = allImports.filter(
+    (src) => !isAllowedImport(src, allowImports)
+  );
+
+  if (disallowedImports.length > 0) {
+    return {
+      hasProblematicPatterns: true,
+      reason: `Contains disallowed imports: ${disallowedImports.slice(0, 3).join(', ')}${disallowedImports.length > 3 ? ` (+${disallowedImports.length - 3} more)` : ''}`,
+      allImports,
+      disallowedImports,
+    };
+  }
+
+  return {
+    hasProblematicPatterns: false,
+    allImports,
+  };
+}
+
+/**
+ * Detect MDX patterns that markdown-rs cannot parse correctly.
+ * This includes:
+ * - MDX import statements (import ... from '...')
+ * - MDX export statements (export const ..., export default ...)
+ * These are JavaScript constructs that the MDAST pipeline cannot handle.
+ *
+ * @param source - The markdown/MDX source content
+ * @param options - MDX handling options
+ */
+export function hasProblematicMdxPatterns(
+  source: string,
+  options?: MdxImportHandlingOptions
+): boolean {
+  return detectProblematicMdxPatterns(source, options).hasProblematicPatterns;
+}

package/src/utils/paths.test.ts

@@ -0,0 +1,206 @@
+import { describe, test, expect } from 'bun:test';
+import path from 'node:path';
+import {
+  stripQuery,
+  normalizePath,
+  deriveAstroUrl,
+  deriveFileOptions,
+  shouldCompile,
+} from './paths.js';
+
+describe('stripQuery', () => {
+  test('returns path unchanged when no query string', () => {
+    expect(stripQuery('/path/to/file.md')).toBe('/path/to/file.md');
+  });
+
+  test('removes query string from path', () => {
+    expect(stripQuery('/path/to/file.md?foo=bar')).toBe('/path/to/file.md');
+  });
+
+  test('removes empty query string', () => {
+    expect(stripQuery('/path/to/file.md?')).toBe('/path/to/file.md');
+  });
+
+  test('handles multiple query parameters', () => {
+    expect(stripQuery('/path/to/file.md?foo=bar&baz=qux')).toBe(
+      '/path/to/file.md'
+    );
+  });
+
+  test('handles query string with special characters', () => {
+    expect(stripQuery('/file.md?import&raw')).toBe('/file.md');
+  });
+});
+
+describe('normalizePath', () => {
+  test('returns Unix-style path unchanged', () => {
+    expect(normalizePath('path/to/file.md')).toBe('path/to/file.md');
+  });
+
+  test('converts Windows-style path to Unix-style', () => {
+    // Simulate Windows path separator
+    const windowsPath = ['path', 'to', 'file.md'].join('\\');
+    const sep = path.sep;
+    // Mock path.sep temporarily
+    Object.defineProperty(path, 'sep', { value: '\\', writable: true });
+    expect(normalizePath(windowsPath)).toBe('path/to/file.md');
+    // Restore
+    Object.defineProperty(path, 'sep', { value: sep, writable: true });
+  });
+
+  test('handles absolute paths', () => {
+    expect(normalizePath('/absolute/path/file.md')).toBe(
+      '/absolute/path/file.md'
+    );
+  });
+
+  test('handles empty path', () => {
+    expect(normalizePath('')).toBe('');
+  });
+});
+
+describe('deriveAstroUrl', () => {
+  test('returns undefined for empty filePath', () => {
+    expect(deriveAstroUrl('')).toBe(undefined);
+  });
+
+  test('returns undefined for files outside pages directory', () => {
+    const filePath = '/project/src/components/Button.astro';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe(undefined);
+  });
+
+  test('returns / for index.md at pages root', () => {
+    const filePath = '/project/src/pages/index.md';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/');
+  });
+
+  test('returns / for empty relative path after pages', () => {
+    const filePath = '/project/src/pages/';
+    const rootDir = '/project';
+    // Normalized path ends at pages, empty relative
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/');
+  });
+
+  test('derives URL for file in pages subdirectory', () => {
+    const filePath = '/project/src/pages/blog/post.md';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/blog/post');
+  });
+
+  test('removes .md extension from URL', () => {
+    const filePath = '/project/src/pages/about.md';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/about');
+  });
+
+  test('removes .mdx extension from URL', () => {
+    const filePath = '/project/src/pages/docs/intro.mdx';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/docs/intro');
+  });
+
+  test('handles nested index files', () => {
+    const filePath = '/project/src/pages/blog/index.md';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/blog');
+  });
+
+  test('handles deeply nested paths', () => {
+    const filePath = '/project/src/pages/docs/guides/advanced/config.md';
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe(
+      '/docs/guides/advanced/config'
+    );
+  });
+
+  test('handles paths on current platform', () => {
+    // Path separators are normalized based on platform
+    const filePath = path.join('/project', 'src', 'pages', 'about.md');
+    const rootDir = '/project';
+    expect(deriveAstroUrl(filePath, rootDir)).toBe('/about');
+  });
+});
+
+describe('deriveFileOptions', () => {
+  test('derives options for absolute path', () => {
+    const id = '/project/src/pages/index.md';
+    const rootDir = '/project';
+    const options = deriveFileOptions(id, rootDir);
+    expect(options.file).toBe('/project/src/pages/index.md');
+    expect(options.url).toBe('/');
+  });
+
+  test('resolves relative path to absolute', () => {
+    const id = 'src/pages/about.md';
+    const rootDir = '/project';
+    const options = deriveFileOptions(id, rootDir);
+    expect(options.file).toBe('/project/src/pages/about.md');
+    expect(options.url).toBe('/about');
+  });
+
+  test('strips query string before processing', () => {
+    const id = '/project/src/pages/blog.md?raw';
+    const rootDir = '/project';
+    const options = deriveFileOptions(id, rootDir);
+    expect(options.file).toBe('/project/src/pages/blog.md');
+    expect(options.url).toBe('/blog');
+  });
+
+  test('omits url when file is outside pages directory', () => {
+    const id = '/project/src/components/Button.astro';
+    const rootDir = '/project';
+    const options = deriveFileOptions(id, rootDir);
+    expect(options.file).toBe('/project/src/components/Button.astro');
+    expect(options.url).toBe(undefined);
+  });
+
+  test('includes url for nested page files', () => {
+    const id = '/project/src/pages/docs/api.mdx';
+    const rootDir = '/project';
+    const options = deriveFileOptions(id, rootDir);
+    expect(options.file).toBe('/project/src/pages/docs/api.mdx');
+    expect(options.url).toBe('/docs/api');
+  });
+
+  test('handles path without rootDir', () => {
+    const id = '/absolute/path/file.md';
+    const options = deriveFileOptions(id);
+    expect(options.file).toBe('/absolute/path/file.md');
+    // url may or may not be present depending on actual file system
+  });
+});
+
+describe('shouldCompile', () => {
+  test('returns true for .md files', () => {
+    expect(shouldCompile('/path/to/file.md')).toBe(true);
+  });
+
+  test('returns true for .mdx files', () => {
+    expect(shouldCompile('/path/to/file.mdx')).toBe(true);
+  });
+
+  test('returns false for .js files', () => {
+    expect(shouldCompile('/path/to/file.js')).toBe(false);
+  });
+
+  test('returns false for .astro files', () => {
+    expect(shouldCompile('/path/to/file.astro')).toBe(false);
+  });
+
+  test('strips query string before checking extension', () => {
+    expect(shouldCompile('/path/to/file.md?raw')).toBe(true);
+    expect(shouldCompile('/path/to/file.mdx?import')).toBe(true);
+    expect(shouldCompile('/path/to/file.js?raw')).toBe(false);
+  });
+
+  test('returns false for paths with no extension', () => {
+    expect(shouldCompile('/path/to/README')).toBe(false);
+  });
+
+  test('handles uppercase extensions', () => {
+    expect(shouldCompile('/path/to/file.MD')).toBe(false); // Case-sensitive
+    expect(shouldCompile('/path/to/file.MDX')).toBe(false);
+  });
+});