@nuasite/cms-marker 0.0.42

package/README.md

@@ -0,0 +1,240 @@
+# @nuasite/cms-marker
+
+An Astro integration that automatically marks HTML elements with unique identifiers and generates a manifest mapping content to source files and line numbers. Perfect for building CMS editors that need to track where content originates in your codebase.
+
+## Features
+
+- **Automatic Marking**: Adds `data-cms-id` attributes to HTML elements during build
+- **Source Location Tracking**: Maps content back to exact line numbers in `.astro` source files
+- **Variable Detection**: Finds content defined as variables in frontmatter
+- **Nested Content Support**: Handles placeholders for nested CMS-editable elements
+- **Dev & Build Modes**: Works in both development and production builds
+- **Manifest Generation**: Creates JSON manifest with all CMS-editable content
+
+## Installation
+
+```bash
+bun add -D @nuasite/cms-marker
+# or: npm install -D @nuasite/cms-marker
+```
+
+## Usage
+
+Add the integration to your `astro.config.mjs`:
+
+```js
+import cmsMarker from '@nuasite/cms-marker'
+import { defineConfig } from 'astro/config'
+
+export default defineConfig({
+	integrations: [
+		cmsMarker({
+			// Optional configuration
+			attributeName: 'data-cms-id',
+			includeTags: null, // null = all tags, or specify array like ['h1', 'p', 'a']
+			excludeTags: ['html', 'head', 'body', 'script', 'style'],
+			includeEmptyText: false,
+		}),
+	],
+})
+```
+
+## How It Works
+
+### 1. HTML Marking
+
+The integration processes your HTML and adds unique IDs:
+
+```html
+<!-- Before -->
+<h1>Welcome to my site</h1>
+<p>This is some content</p>
+
+<!-- After -->
+<h1 data-cms-id="cms-0">Welcome to my site</h1>
+<p data-cms-id="cms-1">This is some content</p>
+```
+
+### 2. Source Location Tracking
+
+It searches your `.astro` source files to find where content originates:
+
+```astro
+<!-- src/components/Hero.astro -->
+---
+const title = "Welcome to my site";
+---
+<h1>{title}</h1>  <!-- Line 4 -->
+```
+
+### 3. Manifest Generation
+
+Creates a JSON manifest mapping IDs to source locations:
+
+```json
+{
+	"cms-0": {
+		"id": "cms-0",
+		"file": "index.html",
+		"tag": "h1",
+		"text": "Welcome to my site",
+		"sourcePath": "src/components/Hero.astro",
+		"sourceLine": 4
+	}
+}
+```
+
+## Configuration Options
+
+### `attributeName`
+
+- **Type**: `string`
+- **Default**: `'data-cms-id'`
+- The HTML attribute name to use for marking elements.
+
+### `includeTags`
+
+- **Type**: `string[] | null`
+- **Default**: `null`
+- If `null`, all tags are included. Otherwise, only specified tags are marked.
+
+### `excludeTags`
+
+- **Type**: `string[]`
+- **Default**: `['html', 'head', 'body', 'script', 'style']`
+- Tags to exclude from marking.
+
+### `includeEmptyText`
+
+- **Type**: `boolean`
+- **Default**: `false`
+- Whether to mark elements with no text content.
+
+## Manifest Entry Structure
+
+Each entry in the manifest contains:
+
+```typescript
+export interface ManifestEntry {
+	id: string // The CMS ID (e.g., "cms-0")
+	file: string // Output HTML file (e.g., "index.html")
+	tag: string // HTML tag name (e.g., "h1")
+	text: string // Text content with placeholders for nested elements
+	sourcePath?: string // Source .astro file path
+	sourceLine?: number // Line number in source file
+	sourceSnippet?: string // Source code snippet
+	sourceType?: 'static' | 'variable' | 'prop' | 'computed' // Type of source
+	variableName?: string // Variable name if source is a variable
+	childCmsIds?: string[] // IDs of nested CMS elements
+}
+```
+
+## Supported Patterns
+
+### ✅ Static Text
+
+```astro
+<h1>Hello World</h1>
+```
+
+### ✅ Simple Variables
+
+```astro
+---
+const title = "My Title";
+---
+<h1>{title}</h1>
+```
+
+### ✅ Variables with Type Annotations
+
+```astro
+---
+const path: string = "src/pages";
+---
+<pre>{path}</pre>
+```
+
+### ✅ Object Properties
+
+```astro
+---
+const content = {
+  title: "Welcome",
+  subtitle: "Get Started"
+};
+---
+<h1>{content.title}</h1>
+<h2>{content.subtitle}</h2>
+```
+
+### ✅ Nested CMS Elements
+
+```astro
+<h1>Start <span>nested</span> end</h1>
+<!-- Manifest text: "Start {{cms:cms-1}} end" -->
+```
+
+### ✅ Escaped Quotes and Special Characters
+
+```astro
+---
+const text = 'What\'s up';
+const message = "Hello & goodbye";
+---
+<p>{text}</p>
+<span>{message}</span>
+```
+
+### ⚠️ Partial Support
+
+Some patterns have limited support and may not always resolve to exact source locations:
+
+- Complex variable expressions
+- Props passed from parent components
+- Template literals with expressions
+- Computed values
+
+In these cases, the manifest will still include the entry but `sourceLine` may be `undefined`.
+
+## Development
+
+```bash
+# Install dependencies
+bun install
+
+# Run tests
+bun test
+
+# Run tests in watch mode
+bun test --watch
+```
+
+## Testing
+
+The package includes comprehensive tests (27 tests, all passing) covering:
+
+- HTML processing and ID assignment
+- Tag inclusion/exclusion rules
+- Manifest generation
+- Source location finding
+- Variable reference detection
+- Escaped quotes and HTML entities
+- Multiple identical tags disambiguation
+- Edge cases and error handling
+
+Run tests with:
+
+```bash
+bun test
+```
+
+See `src/tests/` for all test cases.
+
+## License
+
+MIT
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit issues and pull requests.

package/package.json

@@ -0,0 +1,42 @@
+{
+	"name": "@nuasite/cms-marker",
+	"description": "Mark html tags with a unique identifier and generate a JSON file with the mapping.",
+	"files": [
+		"dist/**",
+		"src/**",
+		"README.md",
+		"package.json"
+	],
+	"homepage": "https://github.com/nuasite/nuasite/blob/main/packages/cms-marker/README.md",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/nuasite/nuasite.git",
+		"directory": "packages/cms-marker"
+	},
+	"license": "Apache-2.0",
+	"version": "0.0.42",
+	"module": "src/index.ts",
+	"types": "src/index.ts",
+	"type": "module",
+	"dependencies": {
+		"@astrojs/compiler": "catalog:astro",
+		"astro": "catalog:astro",
+		"node-html-parser": "catalog:parsers"
+	},
+	"devDependencies": {
+		"@types/bun": "catalog:build"
+	},
+	"peerDependencies": {
+		"typescript": "catalog:build",
+		"vite": "catalog:build"
+	},
+	"scripts": {
+		"test": "bun test"
+	},
+	"keywords": [
+		"devtools",
+		"nuasite",
+		"tooling",
+		"withastro"
+	]
+}

package/src/astro-transform.ts

@@ -0,0 +1,193 @@
+import { parse as parseAstro } from '@astrojs/compiler'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import type { Plugin } from 'vite'
+
+export interface AstroTransformOptions {
+	markComponents?: boolean
+}
+
+/**
+ * Vite plugin that transforms .astro files to inject source location metadata
+ * This runs during Astro's compilation phase and adds data-astro-source-file and
+ * data-astro-source-line attributes to HTML elements in the template.
+ *
+ * NOTE: Component marking is NOT done here because modifying component tags
+ * in the raw .astro source breaks Astro's JSX-like parser. Component marking
+ * is done at the HTML output level instead (in dev-middleware and build-processor).
+ */
+export function createAstroTransformPlugin(options: AstroTransformOptions = {}): Plugin {
+	// Component marking is intentionally disabled at the transform level
+	// const { markComponents = true } = options;
+
+	return {
+		name: 'astro-cms-source-injector',
+		enforce: 'pre', // Run before Astro's own transforms
+
+		async transform(code: string, id: string) {
+			if (!id.endsWith('.astro')) {
+				return null
+			}
+
+			if (id.includes('node_modules')) {
+				return null
+			}
+
+			try {
+				const rawCode = await fs.readFile(id, 'utf-8')
+				const relativePath = path.relative(process.cwd(), id)
+				const result = await parseAstro(rawCode, { position: true })
+
+				if (!result.ast) {
+					return null
+				}
+
+				const transformed = injectSourceAttributes(rawCode, result.ast, relativePath)
+
+				if (transformed !== rawCode) {
+					return {
+						code: transformed,
+						map: null,
+					}
+				}
+
+				return null
+			} catch (error) {
+				console.warn(`[astro-cms-marker] Failed to transform ${id}:`, error)
+				return null
+			}
+		},
+	}
+}
+
+/**
+ * Inject source location attributes into HTML elements
+ * NOTE: Component marking is NOT done here - it breaks Astro's parser
+ */
+function injectSourceAttributes(code: string, ast: any, filePath: string): string {
+	const lines = code.split('\n')
+	const modifications: Array<{ line: number; column: number; insertion: string }> = []
+
+	// Find the template section (after frontmatter)
+	let inFrontmatter = false
+	let frontmatterEnd = -1
+
+	for (let i = 0; i < lines.length; i++) {
+		if (lines[i]?.trim() === '---') {
+			if (!inFrontmatter) {
+				inFrontmatter = true
+			} else {
+				frontmatterEnd = i
+				break
+			}
+		}
+	}
+
+	// If no frontmatter, start from line 0 (all lines are template)
+	if (frontmatterEnd === -1) {
+		frontmatterEnd = -1 // Will make check start.line > 0
+	}
+
+	// Walk the AST and collect modifications
+	const collectElements = (node: any, depth: number = 0) => {
+		if (!node) {
+			return
+		}
+
+		// Only process regular HTML elements, NOT components
+		if (node.type === 'element' && node.position) {
+			const { start } = node.position
+			const tagName = node.name?.toLowerCase()
+
+			// Only process elements in template section (after frontmatter or from start if no frontmatter)
+			const templateStartLine = frontmatterEnd === -1 ? 0 : frontmatterEnd + 1
+			if (start.line > templateStartLine) {
+				// Skip certain elements
+				if (['html', 'head', 'body', 'script', 'style', 'slot', 'fragment'].includes(tagName)) {
+					// Still process children
+					if (node.children && Array.isArray(node.children)) {
+						for (const child of node.children) {
+							collectElements(child, depth + 1)
+						}
+					}
+					return
+				}
+
+				// Find where to insert the attribute (after tag name, before other attributes or >)
+				const lineIndex = start.line - 1
+				if (lineIndex < 0 || lineIndex >= lines.length) {
+					return
+				}
+
+				const line = lines[lineIndex]
+				const tagStartCol = start.column - 1
+
+				// Find the position after the tag name
+				const tagMatch = line?.slice(tagStartCol).match(/^<(\w+)/)
+				if (!tagMatch) {
+					return
+				}
+
+				const insertCol = tagStartCol + tagMatch[0].length
+				const sourceAttr = ` data-astro-source-file="${filePath}" data-astro-source-line="${start.line}"`
+
+				modifications.push({
+					line: lineIndex,
+					column: insertCol,
+					insertion: sourceAttr,
+				})
+			}
+		}
+
+		// Recursively process children
+		if (node.children && Array.isArray(node.children)) {
+			for (const child of node.children) {
+				collectElements(child, depth + 1)
+			}
+		}
+	}
+
+	// Start walking from root children
+	if (ast.children && Array.isArray(ast.children)) {
+		for (const child of ast.children) {
+			collectElements(child, 0)
+		}
+	}
+
+	// Sort modifications by position (reverse order so we can apply them without recalculating positions)
+	modifications.sort((a, b) => {
+		if (a.line !== b.line) {
+			return b.line - a.line
+		}
+		return b.column - a.column
+	})
+
+	// Apply modifications
+	const modifiedLines = [...lines]
+	for (const mod of modifications) {
+		const line = modifiedLines[mod.line]
+
+		// Validate line exists - if not, there's a bug in AST positions or bounds checking
+		if (line === undefined) {
+			console.error(
+				`[astro-cms-marker] Invalid modification at line ${mod.line + 1} in ${filePath}. ` +
+				`This indicates a bug in @astrojs/compiler AST positions or bounds checking. Skipping modification.`
+			)
+			continue
+		}
+
+		// Validate column is within line bounds
+		if (mod.column < 0 || mod.column > line.length) {
+			console.error(
+				`[astro-cms-marker] Invalid column ${mod.column} at line ${mod.line + 1} in ${filePath}. ` +
+				`Line length is ${line.length}. Skipping modification.`
+			)
+			continue
+		}
+
+		// Apply the modification safely
+		modifiedLines[mod.line] = line.slice(0, mod.column) + mod.insertion + line.slice(mod.column)
+	}
+
+	return modifiedLines.join('\n')
+}

package/src/build-processor.ts

@@ -0,0 +1,164 @@
+import type { AstroIntegrationLogger } from 'astro'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { processHtml } from './html-processor'
+import type { ManifestWriter } from './manifest-writer'
+import type { CmsMarkerOptions } from './types'
+
+// Concurrency limit for parallel processing
+const MAX_CONCURRENT = 10
+
+/**
+ * Get the page path from an HTML file path
+ * For example: /about/index.html -> /about
+ *              /index.html -> /
+ *              /blog/post.html -> /blog/post
+ */
+function getPagePath(htmlPath: string, outDir: string): string {
+	const relPath = path.relative(outDir, htmlPath)
+	const parts = relPath.split(path.sep)
+
+	// Handle index.html files
+	if (parts[parts.length - 1] === 'index.html') {
+		parts.pop()
+		return '/' + parts.join('/')
+	}
+
+	// Handle other .html files (remove extension)
+	const last = parts[parts.length - 1]
+	if (last) {
+		parts[parts.length - 1] = last.replace('.html', '')
+	}
+	return '/' + parts.join('/')
+}
+
+/**
+ * Process a single HTML file
+ */
+async function processFile(
+	filePath: string,
+	outDir: string,
+	config: Required<CmsMarkerOptions>,
+	manifestWriter: ManifestWriter,
+	idCounter: { value: number },
+): Promise<number> {
+	const relPath = path.relative(outDir, filePath)
+	const pagePath = getPagePath(filePath, outDir)
+	const html = await fs.readFile(filePath, 'utf-8')
+
+	// Create ID generator - use atomic increment
+	const pageIdStart = idCounter.value
+	const idGenerator = () => `cms-${idCounter.value++}`
+
+	const result = await processHtml(
+		html,
+		relPath,
+		{
+			attributeName: config.attributeName,
+			includeTags: config.includeTags,
+			excludeTags: config.excludeTags,
+			includeEmptyText: config.includeEmptyText,
+			generateManifest: config.generateManifest,
+			markComponents: config.markComponents,
+			componentDirs: config.componentDirs,
+		},
+		idGenerator,
+	)
+
+	// Note: Source locations are now extracted from Astro's compiler attributes
+	// in html-processor.ts, so we don't need the expensive findSourceLocation calls
+
+	// Add to manifest writer (handles per-page manifest writes)
+	manifestWriter.addPage(pagePath, result.entries, result.components)
+
+	// Write transformed HTML back
+	await fs.writeFile(filePath, result.html, 'utf-8')
+
+	return Object.keys(result.entries).length
+}
+
+/**
+ * Process HTML files in parallel with concurrency limit
+ */
+async function processFilesInBatches(
+	files: string[],
+	outDir: string,
+	config: Required<CmsMarkerOptions>,
+	manifestWriter: ManifestWriter,
+	idCounter: { value: number },
+): Promise<number> {
+	let totalEntries = 0
+
+	// Process files in batches of MAX_CONCURRENT
+	for (let i = 0; i < files.length; i += MAX_CONCURRENT) {
+		const batch = files.slice(i, i + MAX_CONCURRENT)
+		const results = await Promise.all(
+			batch.map(file => processFile(file, outDir, config, manifestWriter, idCounter)),
+		)
+		totalEntries += results.reduce((sum, count) => sum + count, 0)
+	}
+
+	return totalEntries
+}
+
+/**
+ * Process build output - processes all HTML files in parallel
+ */
+export async function processBuildOutput(
+	dir: URL,
+	config: Required<CmsMarkerOptions>,
+	manifestWriter: ManifestWriter,
+	idCounter: { value: number },
+	logger?: AstroIntegrationLogger,
+): Promise<void> {
+	const outDir = fileURLToPath(dir)
+	manifestWriter.setOutDir(outDir)
+
+	const htmlFiles = await findHtmlFiles(outDir)
+
+	if (htmlFiles.length === 0) {
+		logger?.info('No HTML files found to process')
+		return
+	}
+
+	const startTime = Date.now()
+
+	// Process all files in parallel batches
+	await processFilesInBatches(htmlFiles, outDir, config, manifestWriter, idCounter)
+
+	// Finalize manifest (writes global manifest and waits for all per-page writes)
+	const stats = await manifestWriter.finalize()
+
+	const duration = Date.now() - startTime
+	const msg = `Processed ${stats.totalPages} pages with ${stats.totalEntries} entries and ${stats.totalComponents} components in ${duration}ms`
+
+	if (logger) {
+		logger.info(msg)
+	} else {
+		console.log(`[astro-cms-marker] ${msg}`)
+	}
+}
+
+/**
+ * Recursively find all HTML files in a directory (parallel version)
+ */
+async function findHtmlFiles(dir: string): Promise<string[]> {
+	const result: string[] = []
+
+	async function scan(currentDir: string): Promise<void> {
+		const entries = await fs.readdir(currentDir, { withFileTypes: true })
+
+		await Promise.all(entries.map(async (entry) => {
+			const fullPath = path.join(currentDir, entry.name)
+			if (entry.isDirectory()) {
+				await scan(fullPath)
+			} else if (entry.isFile() && fullPath.endsWith('.html')) {
+				result.push(fullPath)
+			}
+		}))
+	}
+
+	await scan(dir)
+	return result
+}