@bfra.me/doc-sync 0.1.0

package/src/generators/mdx-generator.ts

@@ -0,0 +1,289 @@
+/**
+ * @bfra.me/doc-sync/generators/mdx-generator - MDX document structure generation for Starlight
+ */
+
+import type {Result} from '@bfra.me/es/result'
+import type {
+  MDXDocument,
+  MDXFrontmatter,
+  PackageAPI,
+  PackageInfo,
+  ReadmeContent,
+  SyncError,
+} from '../types'
+
+import {err, ok} from '@bfra.me/es/result'
+import {SENTINEL_MARKERS} from '../types'
+
+import {extractCodeBlocks, parseJSXTags} from '../utils/safe-patterns'
+import {sanitizeAttribute, sanitizeForMDX, sanitizeJSXTag} from '../utils/sanitization'
+import {generateAPIReference} from './api-reference-generator'
+import {formatCodeExamples} from './code-example-formatter'
+import {mapToStarlightComponents} from './component-mapper'
+import {generateFrontmatter, stringifyFrontmatter} from './frontmatter-generator'
+
+/**
+ * Options for MDX generation
+ */
+export interface MDXGeneratorOptions {
+  /** Include API reference section */
+  readonly includeAPI?: boolean
+  /** Include examples from JSDoc */
+  readonly includeExamples?: boolean
+  /** Custom frontmatter overrides */
+  readonly frontmatterOverrides?: Partial<MDXFrontmatter>
+  /** Preserve manual content sections */
+  readonly preserveManualContent?: boolean
+  /** Existing MDX content for merging */
+  readonly existingContent?: string
+}
+
+/**
+ * Default options for MDX generation
+ */
+const DEFAULT_OPTIONS: Required<
+  Omit<MDXGeneratorOptions, 'frontmatterOverrides' | 'existingContent'>
+> = {
+  includeAPI: true,
+  includeExamples: true,
+  preserveManualContent: true,
+}
+
+/**
+ * Starlight component imports required for generated documentation
+ */
+const STARLIGHT_IMPORTS = `import { Badge, Card, CardGrid, Tabs, TabItem } from '@astrojs/starlight/components';`
+
+export function generateMDXDocument(
+  packageInfo: PackageInfo,
+  readme: ReadmeContent | undefined,
+  api: PackageAPI | undefined,
+  options: MDXGeneratorOptions = {},
+): Result<MDXDocument, SyncError> {
+  const mergedOptions = {...DEFAULT_OPTIONS, ...options}
+
+  try {
+    const frontmatter = generateFrontmatter(packageInfo, readme, options.frontmatterOverrides)
+    const contentSections = buildContentSections(packageInfo, readme, api, mergedOptions)
+    const content = contentSections.join('\n\n')
+    const rendered = renderMDXDocument(frontmatter, content)
+
+    return ok({
+      frontmatter,
+      content,
+      rendered,
+    })
+  } catch (error) {
+    return err({
+      code: 'GENERATION_ERROR',
+      message: `Failed to generate MDX for ${packageInfo.name}: ${error instanceof Error ? error.message : String(error)}`,
+      packageName: packageInfo.name,
+      cause: error,
+    })
+  }
+}
+
+function buildContentSections(
+  packageInfo: PackageInfo,
+  readme: ReadmeContent | undefined,
+  api: PackageAPI | undefined,
+  options: Required<Omit<MDXGeneratorOptions, 'frontmatterOverrides' | 'existingContent'>>,
+): string[] {
+  const sections: string[] = []
+
+  sections.push(STARLIGHT_IMPORTS)
+  sections.push('')
+  sections.push(generatePackageHeader(packageInfo))
+
+  if (readme !== undefined) {
+    const mappedContent = mapToStarlightComponents(readme, packageInfo)
+    sections.push(mappedContent)
+  }
+
+  if (options.includeAPI && api !== undefined && hasAPIContent(api)) {
+    sections.push(SENTINEL_MARKERS.AUTO_START)
+    sections.push('')
+    sections.push('## API Reference')
+    sections.push('')
+    sections.push(generateAPIReference(api))
+
+    if (options.includeExamples) {
+      const examples = formatCodeExamples(api)
+      if (examples.length > 0) {
+        sections.push('')
+        sections.push('## Examples')
+        sections.push('')
+        sections.push(examples)
+      }
+    }
+
+    sections.push('')
+    sections.push(SENTINEL_MARKERS.AUTO_END)
+  }
+
+  return sections
+}
+
+function generatePackageHeader(packageInfo: PackageInfo): string {
+  const lines: string[] = []
+
+  lines.push(`# ${packageInfo.name}`)
+  lines.push('')
+
+  const badges: string[] = []
+
+  if (packageInfo.keywords?.includes('cli')) {
+    badges.push(`<Badge text="CLI Tool" variant="tip" />`)
+  } else if (packageInfo.keywords?.includes('library')) {
+    badges.push(`<Badge text="Library" variant="tip" />`)
+  } else if (packageInfo.keywords?.includes('config')) {
+    badges.push(`<Badge text="Config" variant="tip" />`)
+  }
+
+  badges.push(`<Badge text="${sanitizeAttribute(`v${packageInfo.version}`)}" variant="note" />`)
+
+  if (badges.length > 0) {
+    lines.push(badges.join('\n'))
+    lines.push('')
+  }
+
+  if (packageInfo.description !== undefined) {
+    lines.push(packageInfo.description)
+    lines.push('')
+  }
+
+  return lines.join('\n')
+}
+
+function hasAPIContent(api: PackageAPI): boolean {
+  return api.functions.length > 0 || api.types.length > 0
+}
+
+function renderMDXDocument(frontmatter: MDXFrontmatter, content: string): string {
+  const frontmatterYaml = stringifyFrontmatter(frontmatter)
+  return `---\n${frontmatterYaml}\n---\n\n${content}\n`
+}
+
+/**
+ * Sanitizes content for safe MDX rendering
+ * Prevents XSS by escaping potentially dangerous content
+ */
+export function sanitizeContent(content: string): string {
+  // Use comprehensive sanitization from utils
+  return sanitizeForMDX(content)
+}
+
+/**
+ * Sanitizes content within MDX while preserving JSX components
+ * Only escapes content that appears to be user-provided text
+ * Now includes sanitization of JSX component attributes to prevent XSS
+ * Uses safe, non-backtracking parsing to prevent ReDoS
+ */
+export function sanitizeTextContent(content: string): string {
+  const jsxTags = parseJSXTags(content)
+  const parts: string[] = []
+  let lastIndex = 0
+
+  for (const {tag, index, isClosing} of jsxTags) {
+    if (index > lastIndex) {
+      parts.push(sanitizeContent(content.slice(lastIndex, index)))
+    }
+
+    if (isClosing) {
+      parts.push(tag)
+    } else {
+      parts.push(sanitizeJSXTag(tag))
+    }
+
+    lastIndex = index + tag.length
+  }
+
+  if (lastIndex < content.length) {
+    parts.push(sanitizeContent(content.slice(lastIndex)))
+  }
+
+  return parts.join('')
+}
+
+export function validateMDXSyntax(mdx: string): Result<true, SyncError> {
+  const unclosedTags = checkForUnclosedTags(mdx)
+  if (unclosedTags.length > 0) {
+    return err({
+      code: 'VALIDATION_ERROR',
+      message: `Unclosed MDX tags: ${unclosedTags.join(', ')}`,
+    })
+  }
+
+  const invalidFrontmatter = checkFrontmatter(mdx)
+  if (invalidFrontmatter !== undefined) {
+    return err({
+      code: 'VALIDATION_ERROR',
+      message: invalidFrontmatter,
+    })
+  }
+
+  return ok(true)
+}
+
+function checkForUnclosedTags(mdx: string): string[] {
+  const unclosed: string[] = []
+  const tagStack: string[] = []
+
+  // Remove code blocks from content before checking for JSX tags
+  // This prevents TypeScript generics like Result<T, E> from being
+  // misinterpreted as unclosed JSX tags
+  const codeBlocks = extractCodeBlocks(mdx)
+  let contentWithoutCodeBlocks = mdx
+  for (const block of codeBlocks) {
+    // Replace code block with empty lines to preserve line numbers
+    const lineCount = block.split('\n').length
+    const placeholder = '\n'.repeat(lineCount)
+    contentWithoutCodeBlocks = contentWithoutCodeBlocks.replace(block, placeholder)
+  }
+
+  const jsxTags = parseJSXTags(contentWithoutCodeBlocks)
+
+  for (const {tag, isClosing, isSelfClosing} of jsxTags) {
+    const tagNameMatch = isClosing
+      ? tag.match(/^<\/([A-Z][a-zA-Z0-9]*)>$/)
+      : tag.match(/^<([A-Z][a-zA-Z0-9]*)/)
+    const tagName = tagNameMatch?.[1]
+
+    if (tagName === undefined) continue
+
+    if (isClosing) {
+      const lastOpenTag = tagStack.pop()
+      if (lastOpenTag !== tagName && lastOpenTag !== undefined) {
+        unclosed.push(lastOpenTag)
+      }
+    } else if (!isSelfClosing) {
+      tagStack.push(tagName)
+    }
+  }
+
+  unclosed.push(...tagStack)
+
+  return unclosed
+}
+
+function checkFrontmatter(mdx: string): string | undefined {
+  if (!mdx.startsWith('---')) {
+    return 'MDX document must start with frontmatter'
+  }
+
+  const secondDashIndex = mdx.indexOf('---', 3)
+  if (secondDashIndex === -1) {
+    return 'Frontmatter is not properly closed'
+  }
+
+  const frontmatterContent = mdx.slice(3, secondDashIndex).trim()
+  if (frontmatterContent.length === 0) {
+    return 'Frontmatter cannot be empty'
+  }
+
+  if (!frontmatterContent.includes('title:')) {
+    return 'Frontmatter must include a title'
+  }
+
+  return undefined
+}

package/src/index.ts

@@ -0,0 +1,131 @@
+// Re-export generators
+export {
+  cleanCodeExample,
+  createBadge,
+  createCard,
+  createCardGrid,
+  createDiffSummary,
+  createTabs,
+  detectLanguage,
+  extractAutoSections,
+  extractManualSections,
+  formatCodeBlock,
+  formatCodeExamples,
+  formatFunctionExamples,
+  formatGroupedExamples,
+  formatTypeExamples,
+  formatUsageExample,
+  generateAPICompact,
+  generateAPIReference,
+  generateCategoryReference,
+  generateFrontmatter,
+  generateInstallTabs,
+  generateMDXDocument,
+  groupExamplesByCategory,
+  hasAutoContent,
+  hasManualContent,
+  mapToStarlightComponents,
+  mergeContent,
+  parseFrontmatter,
+  sanitizeContent,
+  sanitizeTextContent,
+  stringifyFrontmatter,
+  stripSentinelMarkers,
+  validateMarkerPairing,
+  validateMDXSyntax,
+  wrapAutoSection,
+  wrapManualSection,
+} from './generators'
+
+export type {
+  CodeExampleOptions,
+  ComponentMapperConfig,
+  ContentSection,
+  MDXGeneratorOptions,
+  MergeOptions,
+  MergeResult,
+  SectionMapper,
+} from './generators'
+
+// Re-export orchestrator
+export {
+  createPackageScanner,
+  createSyncOrchestrator,
+  createValidationPipeline,
+  filterPackagesByPattern,
+  groupPackagesByScope,
+  isValidFilePath,
+  validateContentString,
+  validateDocument,
+} from './orchestrator'
+
+export type {
+  PackageScannerOptions,
+  ScannedPackage,
+  ScanResult,
+  SyncOrchestrator,
+  SyncOrchestratorOptions,
+  ValidationError,
+  ValidationPipelineOptions,
+  ValidationResult,
+  ValidationWarning,
+} from './orchestrator'
+
+export type {
+  CLIOptions,
+  DocConfig,
+  DocConfigSource,
+  ExportedFunction,
+  ExportedType,
+  FileChangeEvent,
+  FunctionParameter,
+  InferSchema,
+  JSDocInfo,
+  JSDocParam,
+  JSDocTag,
+  MDXDocument,
+  MDXFrontmatter,
+  PackageAPI,
+  PackageInfo,
+  ParseError,
+  ParseErrorCode,
+  ParseResult,
+  ReadmeContent,
+  ReadmeSection,
+  ReExport,
+  SyncError,
+  SyncErrorCode,
+  SyncInfo,
+  SyncResult,
+  SyncSummary,
+} from './types'
+
+export {SENTINEL_MARKERS} from './types'
+
+// Re-export watcher
+export {
+  categorizeFile,
+  consolidateEvents,
+  createDocChangeDetector,
+  createDocDebouncer,
+  createDocWatcher,
+  deduplicateEvents,
+  determineRegenerationScope,
+  filterDocumentationChanges,
+  groupChangesByPackage,
+  hasAnyFileChanged,
+} from './watcher'
+
+export type {
+  BatchChangeHandler,
+  DocChangeDetector,
+  DocChangeDetectorOptions,
+  DocChangeHandler,
+  DocDebouncer,
+  DocDebouncerOptions,
+  DocFileWatcher,
+  DocWatcherOptions,
+  FileCategory,
+  PackageChangeAnalysis,
+  RegenerationScope,
+} from './watcher'

package/src/orchestrator/index.ts

@@ -0,0 +1,21 @@
+export {
+  createPackageScanner,
+  filterPackagesByPattern,
+  groupPackagesByScope,
+} from './package-scanner'
+export type {PackageScannerOptions, ScannedPackage, ScanResult} from './package-scanner'
+
+export {createSyncOrchestrator, isValidFilePath} from './sync-orchestrator'
+export type {SyncOrchestrator, SyncOrchestratorOptions} from './sync-orchestrator'
+
+export {
+  createValidationPipeline,
+  validateContentString,
+  validateDocument,
+} from './validation-pipeline'
+export type {
+  ValidationError,
+  ValidationPipelineOptions,
+  ValidationResult,
+  ValidationWarning,
+} from './validation-pipeline'

package/src/orchestrator/package-scanner.ts

@@ -0,0 +1,276 @@
+import type {Result} from '@bfra.me/es/result'
+import type {PackageAPI, PackageInfo, ReadmeContent, SyncError} from '../types'
+
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+import {err, ok} from '@bfra.me/es/result'
+
+import {analyzePublicAPI, parsePackageComplete, parseReadmeFile} from '../parsers'
+
+export interface PackageScannerOptions {
+  readonly rootDir: string
+  readonly includePatterns?: readonly string[]
+  readonly excludePackages?: readonly string[]
+  readonly parseSourceFiles?: boolean
+  readonly parseReadme?: boolean
+}
+
+export interface ScannedPackage {
+  readonly info: PackageInfo
+  readonly readme?: ReadmeContent
+  readonly api?: PackageAPI
+  readonly sourceFiles: readonly string[]
+  readonly needsDocumentation: boolean
+  readonly existingDocPath?: string
+}
+
+export interface ScanResult {
+  readonly packages: readonly ScannedPackage[]
+  readonly packagesNeedingDocs: readonly ScannedPackage[]
+  readonly errors: readonly SyncError[]
+  readonly durationMs: number
+}
+
+const DEFAULT_OPTIONS: Required<Omit<PackageScannerOptions, 'rootDir' | 'excludePackages'>> = {
+  includePatterns: ['packages/*'],
+  parseSourceFiles: true,
+  parseReadme: true,
+}
+
+export function createPackageScanner(options: PackageScannerOptions): {
+  readonly scan: () => Promise<ScanResult>
+  readonly scanPackage: (packagePath: string) => Promise<Result<ScannedPackage, SyncError>>
+} {
+  const {
+    rootDir,
+    includePatterns = DEFAULT_OPTIONS.includePatterns,
+    excludePackages = [],
+    parseSourceFiles = DEFAULT_OPTIONS.parseSourceFiles,
+    parseReadme = DEFAULT_OPTIONS.parseReadme,
+  } = options
+
+  const docsOutputDir = path.join(rootDir, 'docs', 'src', 'content', 'docs', 'packages')
+
+  async function discoverPackages(): Promise<string[]> {
+    const packagePaths: string[] = []
+
+    for (const pattern of includePatterns) {
+      const baseDir = path.join(rootDir, pattern.replace('/*', ''))
+
+      try {
+        const entries = await fs.readdir(baseDir, {withFileTypes: true})
+
+        for (const entry of entries) {
+          if (!entry.isDirectory()) {
+            continue
+          }
+
+          const packagePath = path.join(baseDir, entry.name)
+          const packageJsonPath = path.join(packagePath, 'package.json')
+
+          try {
+            await fs.access(packageJsonPath)
+            packagePaths.push(packagePath)
+          } catch {
+            // No package.json, skip this directory
+          }
+        }
+      } catch {
+        // Pattern directory doesn't exist, skip
+      }
+    }
+
+    return packagePaths
+  }
+
+  async function findSourceFiles(srcDir: string): Promise<string[]> {
+    const sourceFiles: string[] = []
+
+    try {
+      await collectSourceFiles(srcDir, sourceFiles)
+    } catch {
+      // Source directory doesn't exist
+    }
+
+    return sourceFiles
+  }
+
+  async function collectSourceFiles(dir: string, files: string[]): Promise<void> {
+    const entries = await fs.readdir(dir, {withFileTypes: true})
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name)
+
+      if (entry.isDirectory()) {
+        // Skip test directories
+        if (entry.name === '__tests__' || entry.name === '__mocks__') {
+          continue
+        }
+        await collectSourceFiles(fullPath, files)
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase()
+        if (
+          (ext === '.ts' || ext === '.tsx') &&
+          !entry.name.includes('.test.') &&
+          !entry.name.includes('.spec.')
+        ) {
+          files.push(fullPath)
+        }
+      }
+    }
+  }
+
+  async function scanPackage(packagePath: string): Promise<Result<ScannedPackage, SyncError>> {
+    const packageResult = await parsePackageComplete(packagePath)
+
+    if (!packageResult.success) {
+      return err({
+        code: 'PACKAGE_NOT_FOUND',
+        message: `Failed to parse package at ${packagePath}: ${packageResult.error.message}`,
+        filePath: packagePath,
+        cause: packageResult.error,
+      })
+    }
+
+    const info = packageResult.data
+    const sourceFiles = await findSourceFiles(info.srcPath)
+
+    let readme: ReadmeContent | undefined
+    if (parseReadme && info.readmePath !== undefined) {
+      const readmeResult = await parseReadmeFile(info.readmePath)
+      if (readmeResult.success) {
+        readme = readmeResult.data
+      }
+    }
+
+    let api: PackageAPI | undefined
+    if (parseSourceFiles && sourceFiles.length > 0) {
+      const entryFile = findEntryFile(sourceFiles, info.srcPath)
+      if (entryFile !== undefined) {
+        const analysisResult = analyzePublicAPI(entryFile)
+        if (analysisResult.success) {
+          api = analysisResult.data.api
+        }
+      }
+    }
+
+    const docSlug = buildDocSlug(info.name)
+    const existingDocPath = path.join(docsOutputDir, `${docSlug}.mdx`)
+    let hasExistingDoc = false
+
+    try {
+      await fs.access(existingDocPath)
+      hasExistingDoc = true
+    } catch {
+      // Doc doesn't exist yet
+    }
+
+    return ok({
+      info,
+      readme,
+      api,
+      sourceFiles,
+      needsDocumentation: true,
+      existingDocPath: hasExistingDoc ? existingDocPath : undefined,
+    })
+  }
+
+  return {
+    async scan(): Promise<ScanResult> {
+      const startTime = Date.now()
+      const packagePaths = await discoverPackages()
+      const packages: ScannedPackage[] = []
+      const errors: SyncError[] = []
+
+      for (const packagePath of packagePaths) {
+        const result = await scanPackage(packagePath)
+
+        if (result.success) {
+          const scanned = result.data
+
+          // Check if this package should be excluded
+          if (excludePackages.includes(scanned.info.name)) {
+            continue
+          }
+
+          packages.push(scanned)
+        } else {
+          errors.push(result.error)
+        }
+      }
+
+      const packagesNeedingDocs = packages.filter(pkg => pkg.needsDocumentation)
+
+      return {
+        packages,
+        packagesNeedingDocs,
+        errors,
+        durationMs: Date.now() - startTime,
+      }
+    },
+
+    scanPackage,
+  }
+}
+
+/**
+ * Finds the entry file (index.ts) from a list of source files
+ */
+function findEntryFile(sourceFiles: readonly string[], srcDir: string): string | undefined {
+  const indexPath = path.join(srcDir, 'index.ts')
+  return sourceFiles.find(file => file === indexPath) ?? sourceFiles[0]
+}
+
+function buildDocSlug(packageName: string): string {
+  return getUnscopedName(packageName)
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9-]/g, '-')
+}
+
+function getUnscopedName(packageName: string): string {
+  if (packageName.startsWith('@')) {
+    const slashIndex = packageName.indexOf('/')
+    if (slashIndex > 0) {
+      return packageName.slice(slashIndex + 1)
+    }
+  }
+  return packageName
+}
+
+export function filterPackagesByPattern(
+  packages: readonly ScannedPackage[],
+  pattern: string,
+): ScannedPackage[] {
+  const regex = new RegExp(pattern.replaceAll('*', '.*'), 'i')
+  return packages.filter(pkg => regex.test(pkg.info.name))
+}
+
+export function groupPackagesByScope(
+  packages: readonly ScannedPackage[],
+): Map<string, ScannedPackage[]> {
+  const grouped = new Map<string, ScannedPackage[]>()
+
+  for (const pkg of packages) {
+    const scope = getPackageScope(pkg.info.name) ?? '__unscoped__'
+    const existing = grouped.get(scope)
+
+    if (existing === undefined) {
+      grouped.set(scope, [pkg])
+    } else {
+      existing.push(pkg)
+    }
+  }
+
+  return grouped
+}
+
+function getPackageScope(packageName: string): string | undefined {
+  if (packageName.startsWith('@')) {
+    const slashIndex = packageName.indexOf('/')
+    if (slashIndex > 0) {
+      return packageName.slice(0, slashIndex)
+    }
+  }
+  return undefined
+}