@nuasite/llm-enhancements 0.0.57

package/dist/types/types.d.ts

@@ -0,0 +1,34 @@
+export interface PageMarkdownOptions {
+    /** Directory containing content collections (default: 'src/content') */
+    contentDir?: string;
+    /** Whether to include static (non-collection) pages (default: true) */
+    includeStaticPages?: boolean;
+    /** Whether to include frontmatter in output (default: true) */
+    includeFrontmatter?: boolean;
+    /** Enable /.well-known/llm.md endpoint (default: true) */
+    llmEndpoint?: boolean | LlmEndpointOptions;
+}
+export interface LlmEndpointOptions {
+    /** Site name override */
+    siteName?: string;
+    /** Site description override */
+    description?: string;
+    /** Additional content to append */
+    additionalContent?: string;
+}
+export interface MarkdownOutput {
+    /** YAML frontmatter fields */
+    frontmatter: Record<string, unknown>;
+    /** Markdown body content */
+    body: string;
+    /** Path to the original source file (if from collection) */
+    sourcePath?: string;
+}
+export interface ResolvedOptions {
+    contentDir: string;
+    includeStaticPages: boolean;
+    includeFrontmatter: boolean;
+    llmEndpoint: false | LlmEndpointOptions;
+}
+export declare function resolveOptions(options?: PageMarkdownOptions): ResolvedOptions;
+//# sourceMappingURL=types.d.ts.map

package/dist/types/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,mBAAmB;IACnC,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,uEAAuE;IACvE,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B,0DAA0D;IAC1D,WAAW,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAA;CAC1C;AAED,MAAM,WAAW,kBAAkB;IAClC,yBAAyB;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gCAAgC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,mCAAmC;IACnC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,cAAc;IAC9B,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACpC,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,EAAE,OAAO,CAAA;IAC3B,kBAAkB,EAAE,OAAO,CAAA;IAC3B,WAAW,EAAE,KAAK,GAAG,kBAAkB,CAAA;CACvC;AAED,wBAAgB,cAAc,CAAC,OAAO,GAAE,mBAAwB,GAAG,eAAe,CAQjF"}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@nuasite/llm-enhancements",
+  "description": "Expose pages as .md endpoints for Astro",
+  "files": [
+    "dist/**",
+    "src/**",
+    "README.md",
+    "package.json"
+  ],
+  "homepage": "https://github.com/nuasite/nuasite/blob/main/packages/page-markdown/README.md",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nuasite/nuasite.git",
+    "directory": "packages/page-markdown"
+  },
+  "license": "Apache-2.0",
+  "version": "0.0.57",
+  "module": "src/index.ts",
+  "types": "src/index.ts",
+  "type": "module",
+  "dependencies": {
+    "astro": "^5.16.6",
+    "node-html-parser": "^6.1.13"
+  },
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5",
+    "vite": "^6",
+    "@nuasite/cms-marker": "0.0.57"
+  },
+  "peerDependenciesMeta": {
+    "@nuasite/cms-marker": {
+      "optional": true
+    }
+  },
+  "scripts": {
+    "prepack": "bun run ../../scripts/workspace-deps/resolve-deps.ts"
+  },
+  "keywords": [
+    "markdown",
+    "astro",
+    "nuasite",
+    "withastro",
+    "pages"
+  ]
+}

package/src/build-processor.ts

@@ -0,0 +1,173 @@
+import type { AstroConfig, AstroIntegrationLogger } from 'astro'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { getCollectionContent } from './cms-marker'
+import { htmlToMarkdown } from './html-to-markdown'
+import { generateLlmMarkdown, type PageEntry, type SiteMetadata } from './llm-endpoint'
+import { generateLlmsTxt } from './llms-txt-endpoint'
+import { createCollectionOutput, createStaticOutput, generateMarkdown } from './markdown-generator'
+import { getHtmlPath, getLlmOutputPath, getLlmsTxtOutputPath, getMdOutputPath, injectMarkdownLink, normalizePath } from './paths'
+import type { ResolvedOptions } from './types'
+
+interface PageInfo {
+	pathname: string
+}
+
+/**
+ * Get base URL from Astro config, removing trailing slash
+ */
+function getBaseUrl(config: AstroConfig): string {
+	const site = config.site
+	if (!site) return ''
+	return site.endsWith('/') ? site.slice(0, -1) : site
+}
+
+/**
+ * Process build output and generate .md files for all pages
+ */
+export async function processBuildOutput(
+	dir: URL,
+	pages: PageInfo[],
+	options: ResolvedOptions,
+	logger: AstroIntegrationLogger,
+	config: AstroConfig,
+) {
+	const baseUrl = getBaseUrl(config)
+	const distDir = dir.pathname
+	let collectionCount = 0
+	let staticCount = 0
+	const pageEntries: PageEntry[] = []
+	let siteMetadata: SiteMetadata = {}
+
+	for (const page of pages) {
+		const pagePath = normalizePath(page.pathname === '' ? '/' : `/${page.pathname}`)
+
+		try {
+			const mdPath = getMdOutputPath(distDir, pagePath)
+			const htmlPath = getHtmlPath(distDir, pagePath)
+
+			// Try collection page first
+			const content = await getCollectionContent(pagePath, options.contentDir)
+			if (content) {
+				const output = createCollectionOutput(content.frontmatter, content.body, content.file)
+				const markdown = generateMarkdown(output, {
+					url: pagePath,
+					type: 'collection',
+					sourcePath: content.file,
+				}, options.includeFrontmatter)
+
+				await writeMarkdownFile(mdPath, markdown)
+				await injectLinkIntoHtml(htmlPath, pagePath)
+				pageEntries.push({
+					pathname: pagePath,
+					title: extractTitle(content.frontmatter.title),
+					type: 'collection',
+				})
+				collectionCount++
+				continue
+			}
+
+			// Fall back to static page handling
+			if (!options.includeStaticPages) continue
+
+			const htmlExists = await fileExists(htmlPath)
+			if (!htmlExists) continue
+
+			const html = await fs.readFile(htmlPath, 'utf-8')
+			const { metadata, body } = htmlToMarkdown(html)
+			const output = createStaticOutput(metadata, body)
+
+			const markdown = generateMarkdown(output, {
+				url: pagePath,
+				type: 'static',
+			}, options.includeFrontmatter)
+
+			await writeMarkdownFile(mdPath, markdown)
+			await injectLinkIntoHtml(htmlPath, pagePath)
+			pageEntries.push({
+				pathname: pagePath,
+				title: metadata.title,
+				type: 'static',
+			})
+
+			// Extract site metadata from homepage
+			if (pagePath === '/') {
+				siteMetadata = {
+					title: metadata.title,
+					description: metadata.description,
+				}
+			}
+
+			staticCount++
+		} catch (error) {
+			logger.warn(`Failed to process ${pagePath}: ${error}`)
+		}
+	}
+
+	const total = collectionCount + staticCount
+	if (total > 0) {
+		logger.info(`Generated ${total} .md files (${collectionCount} collection, ${staticCount} static)`)
+	}
+
+	// Generate llm.md if enabled
+	if (options.llmEndpoint !== false) {
+		if (!baseUrl) {
+			logger.warn('Skipping /.well-known/llm.md generation: no `site` configured in astro.config')
+		} else {
+			try {
+				const llmContent = generateLlmMarkdown(pageEntries, { ...siteMetadata, baseUrl }, options.llmEndpoint)
+				const llmPath = getLlmOutputPath(distDir)
+				await writeMarkdownFile(llmPath, llmContent)
+				logger.info('Generated /.well-known/llm.md')
+			} catch (error) {
+				logger.warn(`Failed to generate llm.md: ${error}`)
+			}
+		}
+	}
+
+	// Generate llms.txt if enabled
+	if (options.llmsTxt !== false) {
+		if (!baseUrl) {
+			logger.warn('Skipping /llms.txt generation: no `site` configured in astro.config')
+		} else {
+			try {
+				const llmsTxtContent = generateLlmsTxt(pageEntries, { ...siteMetadata, baseUrl }, options.llmsTxt)
+				const llmsTxtPath = getLlmsTxtOutputPath(distDir)
+				await writeMarkdownFile(llmsTxtPath, llmsTxtContent)
+				logger.info('Generated /llms.txt')
+			} catch (error) {
+				logger.warn(`Failed to generate llms.txt: ${error}`)
+			}
+		}
+	}
+}
+
+async function injectLinkIntoHtml(htmlPath: string, pagePath: string): Promise<void> {
+	try {
+		const html = await fs.readFile(htmlPath, 'utf-8')
+		await fs.writeFile(htmlPath, injectMarkdownLink(html, pagePath), 'utf-8')
+	} catch {
+		// File might not exist for some pages
+	}
+}
+
+async function fileExists(filePath: string): Promise<boolean> {
+	try {
+		await fs.access(filePath)
+		return true
+	} catch {
+		return false
+	}
+}
+
+async function writeMarkdownFile(filePath: string, content: string): Promise<void> {
+	await fs.mkdir(path.dirname(filePath), { recursive: true })
+	await fs.writeFile(filePath, content, 'utf-8')
+}
+
+function extractTitle(title: unknown): string | undefined {
+	if (typeof title === 'string') {
+		return title
+	}
+	return undefined
+}

package/src/cms-marker.ts

@@ -0,0 +1,56 @@
+export interface CollectionInfo {
+	name: string
+	slug: string
+	file: string
+}
+
+export interface ParsedContent {
+	frontmatter: Record<string, { value: string; line: number }>
+	body: string
+	bodyStartLine: number
+	file: string
+	collectionName: string
+	collectionSlug: string
+}
+
+type FindCollectionSource = (pagePath: string, contentDir?: string) => Promise<CollectionInfo | undefined>
+type ParseMarkdownContent = (collectionInfo: CollectionInfo) => Promise<ParsedContent | undefined>
+
+let findCollectionSource: FindCollectionSource | undefined
+let parseMarkdownContent: ParseMarkdownContent | undefined
+let initialized = false
+
+async function init() {
+	if (initialized) return
+	initialized = true
+
+	try {
+		const cmsMarker = await import('@nuasite/cms-marker')
+		findCollectionSource = cmsMarker.findCollectionSource
+		parseMarkdownContent = cmsMarker.parseMarkdownContent
+	} catch {
+		// cms-marker not available
+	}
+}
+
+export async function getCollectionContent(
+	pagePath: string,
+	contentDir: string,
+): Promise<ParsedContent | undefined> {
+	await init()
+
+	if (!findCollectionSource || !parseMarkdownContent) {
+		return undefined
+	}
+
+	const collectionInfo = await findCollectionSource(pagePath, contentDir)
+	if (!collectionInfo) {
+		return undefined
+	}
+
+	return parseMarkdownContent(collectionInfo)
+}
+
+export function hasCmsMarker(): boolean {
+	return findCollectionSource !== undefined && parseMarkdownContent !== undefined
+}

package/src/dev-middleware.ts

@@ -0,0 +1,240 @@
+import type { AstroConfig } from 'astro'
+import type { ViteDevServer } from 'vite'
+import { getCollectionContent } from './cms-marker'
+import { htmlToMarkdown } from './html-to-markdown'
+import { generateLlmMarkdown, type PageEntry, type SiteMetadata } from './llm-endpoint'
+import { generateLlmsTxt } from './llms-txt-endpoint'
+import { createCollectionOutput, createStaticOutput, generateMarkdown } from './markdown-generator'
+import { injectMarkdownLink, LLM_ENDPOINT_PATH, LLMS_TXT_PATH, mdUrlToPagePath, normalizePath } from './paths'
+import type { ResolvedOptions } from './types'
+
+const ASSET_PATTERN = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|json)$/
+
+/**
+ * Generate markdown for a given page path
+ */
+async function generateMarkdownForPath(
+	pagePath: string,
+	host: string,
+	options: ResolvedOptions,
+): Promise<string | null> {
+	// Try collection page first
+	const content = await getCollectionContent(pagePath, options.contentDir)
+	if (content) {
+		const output = createCollectionOutput(content.frontmatter, content.body, content.file)
+		return generateMarkdown(output, {
+			url: pagePath,
+			type: 'collection',
+			sourcePath: content.file,
+		}, options.includeFrontmatter)
+	}
+
+	// Fall back to static page handling
+	if (!options.includeStaticPages) {
+		return null
+	}
+
+	const response = await fetch(`http://${host}${pagePath}`, {
+		headers: { Accept: 'text/html' },
+	})
+
+	if (!response.ok) return null
+
+	const contentType = response.headers.get('content-type')
+	if (!contentType?.includes('text/html')) return null
+
+	const html = await response.text()
+	const { metadata, body } = htmlToMarkdown(html)
+	const output = createStaticOutput(metadata, body)
+
+	return generateMarkdown(output, {
+		url: pagePath,
+		type: 'static',
+	}, options.includeFrontmatter)
+}
+
+/**
+ * Discover all pages and their metadata for the LLM endpoint
+ */
+async function discoverPages(host: string, options: ResolvedOptions): Promise<{ pages: PageEntry[]; siteMetadata: SiteMetadata }> {
+	const pages: PageEntry[] = []
+	let siteMetadata: SiteMetadata = {}
+
+	// Fetch the sitemap or root to discover pages
+	// First try to get homepage metadata
+	try {
+		const homeResponse = await fetch(`http://${host}/`, {
+			headers: { Accept: 'text/html' },
+		})
+		if (homeResponse.ok) {
+			const html = await homeResponse.text()
+			const { metadata } = htmlToMarkdown(html)
+			siteMetadata = {
+				title: metadata.title,
+				description: metadata.description,
+			}
+		}
+	} catch {
+		// Ignore errors
+	}
+
+	// Try to get pages from Astro's dev server manifest via __astro_dev_toolbar__
+	// For now, we'll discover pages by checking common routes and the content directory
+	// In dev mode, we just report what we can discover
+
+	// Check if homepage exists
+	try {
+		const content = await getCollectionContent('/', options.contentDir)
+		if (content) {
+			pages.push({ pathname: '/', title: content.frontmatter.title as string | undefined, type: 'collection' })
+		} else if (options.includeStaticPages) {
+			const response = await fetch(`http://${host}/`, { headers: { Accept: 'text/html' } })
+			if (response.ok) {
+				const html = await response.text()
+				const { metadata } = htmlToMarkdown(html)
+				pages.push({ pathname: '/', title: metadata.title, type: 'static' })
+			}
+		}
+	} catch {
+		// Ignore
+	}
+
+	return { pages, siteMetadata }
+}
+
+/**
+ * Get base URL from Astro config, removing trailing slash
+ */
+function getBaseUrl(config: AstroConfig): string {
+	const site = config.site
+	if (!site) return ''
+	return site.endsWith('/') ? site.slice(0, -1) : site
+}
+
+/**
+ * Create dev server middleware to handle markdown requests
+ */
+export function createDevMiddleware(server: ViteDevServer, options: ResolvedOptions, config: AstroConfig) {
+	const baseUrl = getBaseUrl(config)
+
+	// Serve /llms.txt endpoint (only if site is configured)
+	const llmsTxtOptions = options.llmsTxt
+	if (llmsTxtOptions !== false && baseUrl) {
+		server.middlewares.use(async (req, res, next) => {
+			const url = req.url || ''
+
+			if (url !== LLMS_TXT_PATH) {
+				return next()
+			}
+
+			try {
+				const host = req.headers.host || 'localhost:4321'
+				const { pages, siteMetadata } = await discoverPages(host, options)
+				const content = generateLlmsTxt(pages, { ...siteMetadata, baseUrl }, llmsTxtOptions)
+
+				res.setHeader('Content-Type', 'text/plain; charset=utf-8')
+				res.setHeader('Access-Control-Allow-Origin', '*')
+				res.end(content)
+				return
+			} catch (error) {
+				console.error('[page-markdown] Error generating llms.txt:', error)
+			}
+
+			return next()
+		})
+	}
+
+	// Serve /.well-known/llm.md endpoint (only if site is configured)
+	const llmEndpointOptions = options.llmEndpoint
+	if (llmEndpointOptions !== false && baseUrl) {
+		server.middlewares.use(async (req, res, next) => {
+			const url = req.url || ''
+
+			if (url !== LLM_ENDPOINT_PATH) {
+				return next()
+			}
+
+			try {
+				const host = req.headers.host || 'localhost:4321'
+				const { pages, siteMetadata } = await discoverPages(host, options)
+				const markdown = generateLlmMarkdown(pages, { ...siteMetadata, baseUrl }, llmEndpointOptions)
+
+				res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
+				res.setHeader('Access-Control-Allow-Origin', '*')
+				res.end(markdown)
+				return
+			} catch (error) {
+				console.error('[page-markdown] Error generating llm.md:', error)
+			}
+
+			return next()
+		})
+	}
+
+	// Serve .md endpoints
+	server.middlewares.use(async (req, res, next) => {
+		const url = req.url || ''
+
+		if (!url.endsWith('.md')) {
+			return next()
+		}
+
+		const pagePath = mdUrlToPagePath(url)
+
+		try {
+			const host = req.headers.host || 'localhost:4321'
+			const markdown = await generateMarkdownForPath(pagePath, host, options)
+
+			if (markdown) {
+				res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
+				res.setHeader('Access-Control-Allow-Origin', '*')
+				res.end(markdown)
+				return
+			}
+		} catch (error) {
+			console.error('[page-markdown] Error generating markdown:', error)
+		}
+
+		return next()
+	})
+
+	// Inject alternate link into HTML responses
+	server.middlewares.use((req, res, next) => {
+		const url = req.url || ''
+
+		if (url.endsWith('.md') || ASSET_PATTERN.test(url)) {
+			return next()
+		}
+
+		const originalWrite = res.write
+		const originalEnd = res.end
+		const chunks: Buffer[] = []
+
+		res.write = ((chunk: unknown) => {
+			if (chunk) chunks.push(Buffer.from(chunk as Buffer))
+			return true
+		}) as typeof res.write
+
+		res.end = ((chunk?: unknown, ...args: unknown[]) => {
+			if (chunk) chunks.push(Buffer.from(chunk as Buffer))
+
+			const contentType = res.getHeader('content-type')
+			const isHtml = typeof contentType === 'string' && contentType.includes('text/html')
+
+			res.write = originalWrite
+			res.end = originalEnd
+
+			if (isHtml && chunks.length > 0) {
+				const html = Buffer.concat(chunks).toString('utf8')
+				const pagePath = normalizePath(url)
+				return res.end(injectMarkdownLink(html, pagePath), ...(args as []))
+			}
+
+			return chunks.length > 0
+				? res.end(Buffer.concat(chunks), ...(args as []))
+				: res.end(...(args as []))
+		}) as typeof res.end
+
+		next()
+	})
+}