@nuasite/llm-enhancements 0.0.57

package/src/html-to-markdown.ts

@@ -0,0 +1,351 @@
+import { type HTMLElement, type Node, NodeType, parse } from 'node-html-parser'
+
+/** Elements to exclude from markdown conversion */
+const EXCLUDED_TAGS = new Set([
+	'nav',
+	'footer',
+	'header',
+	'script',
+	'style',
+	'noscript',
+	'svg',
+	'iframe',
+	'form',
+	'button',
+	'input',
+	'select',
+	'textarea',
+])
+
+/** Block-level elements that need newlines around them */
+const BLOCK_ELEMENTS = new Set([
+	'p',
+	'div',
+	'section',
+	'article',
+	'main',
+	'aside',
+	'h1',
+	'h2',
+	'h3',
+	'h4',
+	'h5',
+	'h6',
+	'ul',
+	'ol',
+	'li',
+	'blockquote',
+	'pre',
+	'table',
+	'tr',
+	'thead',
+	'tbody',
+	'figure',
+	'figcaption',
+	'hr',
+	'br',
+])
+
+/**
+ * Extract the main content area from HTML
+ */
+function extractMainContent(root: HTMLElement): HTMLElement {
+	// Try to find main content container in order of preference
+	const selectors = ['main', 'article', '[role="main"]', '.content', '#content']
+
+	for (const selector of selectors) {
+		const element = root.querySelector(selector)
+		if (element) {
+			return element
+		}
+	}
+
+	// Fall back to body content
+	const body = root.querySelector('body')
+	return body ?? root
+}
+
+/**
+ * Check if an element should be excluded from markdown output
+ */
+function shouldExclude(element: HTMLElement): boolean {
+	const tagName = element.tagName?.toLowerCase()
+	if (!tagName) return false
+	return EXCLUDED_TAGS.has(tagName)
+}
+
+/**
+ * Convert an HTML element to markdown
+ */
+function elementToMarkdown(element: HTMLElement, depth = 0): string {
+	const tagName = element.tagName?.toLowerCase()
+
+	if (!tagName) {
+		return ''
+	}
+
+	if (shouldExclude(element)) {
+		return ''
+	}
+
+	// Process children for most elements
+	const childrenMd = () => childNodesToMarkdown(element, depth)
+
+	switch (tagName) {
+		// Headings
+		case 'h1':
+			return `\n# ${childrenMd().trim()}\n`
+		case 'h2':
+			return `\n## ${childrenMd().trim()}\n`
+		case 'h3':
+			return `\n### ${childrenMd().trim()}\n`
+		case 'h4':
+			return `\n#### ${childrenMd().trim()}\n`
+		case 'h5':
+			return `\n##### ${childrenMd().trim()}\n`
+		case 'h6':
+			return `\n###### ${childrenMd().trim()}\n`
+
+		// Paragraphs
+		case 'p':
+			return `\n${childrenMd().trim()}\n`
+
+		// Inline formatting
+		case 'strong':
+		case 'b':
+			return `**${childrenMd()}**`
+		case 'em':
+		case 'i':
+			return `*${childrenMd()}*`
+		case 'code':
+			return `\`${childrenMd()}\``
+		case 's':
+		case 'del':
+		case 'strike':
+			return `~~${childrenMd()}~~`
+
+		// Links
+		case 'a': {
+			const href = element.getAttribute('href') ?? ''
+			const text = childrenMd().trim()
+			if (!text) return ''
+			return `[${text}](${href})`
+		}
+
+		// Images
+		case 'img': {
+			const src = element.getAttribute('src') ?? ''
+			const alt = element.getAttribute('alt') ?? ''
+			return `![${alt}](${src})`
+		}
+
+		// Lists
+		case 'ul':
+		case 'ol':
+			return `\n${childrenMd()}`
+		case 'li': {
+			const parent = element.parentNode as HTMLElement | null
+			const isOrdered = parent?.tagName?.toLowerCase() === 'ol'
+			const prefix = isOrdered ? '1. ' : '- '
+			const content = childrenMd().trim()
+			return `${prefix}${content}\n`
+		}
+
+		// Code blocks
+		case 'pre': {
+			// node-html-parser treats pre content as text, so we need to re-parse innerHTML
+			const innerParsed = parse(element.innerHTML)
+			const codeElement = innerParsed.querySelector('code')
+			const code = codeElement ? codeElement.textContent : element.textContent
+			const lang = codeElement?.getAttribute('class')?.match(/language-(\w+)/)?.[1] ?? ''
+			return `\n\`\`\`${lang}\n${code?.trim()}\n\`\`\`\n`
+		}
+
+		// Blockquotes
+		case 'blockquote': {
+			const content = childrenMd().trim()
+			const lines = content.split('\n')
+			return '\n' + lines.map((line) => `> ${line}`).join('\n') + '\n'
+		}
+
+		// Horizontal rule
+		case 'hr':
+			return '\n---\n'
+
+		// Line break
+		case 'br':
+			return '\n'
+
+		// Tables
+		case 'table':
+			return `\n${convertTable(element)}\n`
+
+		// Figures
+		case 'figure':
+			return `\n${childrenMd()}`
+		case 'figcaption':
+			return `\n*${childrenMd().trim()}*\n`
+
+		// Container elements - just process children
+		case 'div':
+		case 'section':
+		case 'article':
+		case 'main':
+		case 'aside':
+		case 'span':
+		case 'thead':
+		case 'tbody':
+		case 'tfoot':
+			return childrenMd()
+
+		default:
+			return childrenMd()
+	}
+}
+
+/**
+ * Convert child nodes to markdown
+ */
+function childNodesToMarkdown(element: HTMLElement, depth: number): string {
+	let result = ''
+
+	for (const child of element.childNodes) {
+		if (child.nodeType === NodeType.TEXT_NODE) {
+			// Normalize whitespace in text nodes
+			const text = child.textContent?.replace(/\s+/g, ' ') ?? ''
+			result += text
+		} else if (child.nodeType === NodeType.ELEMENT_NODE) {
+			result += elementToMarkdown(child as HTMLElement, depth + 1)
+		}
+	}
+
+	return result
+}
+
+/**
+ * Convert HTML table to markdown table
+ */
+function convertTable(table: HTMLElement): string {
+	const rows = table.querySelectorAll('tr')
+	if (rows.length === 0) return ''
+
+	const result: string[] = []
+	let headerDone = false
+
+	for (const row of rows) {
+		const cells = row.querySelectorAll('th, td')
+		const cellContents = cells.map((cell) => {
+			const content = childNodesToMarkdown(cell as HTMLElement, 0).trim()
+			return content.replace(/\|/g, '\\|') // Escape pipe characters
+		})
+
+		result.push(`| ${cellContents.join(' | ')} |`)
+
+		// Add header separator after first row with th elements
+		if (!headerDone && row.querySelector('th')) {
+			result.push(`| ${cellContents.map(() => '---').join(' | ')} |`)
+			headerDone = true
+		}
+	}
+
+	// If no header row with th, add separator after first row
+	if (!headerDone && result.length > 0) {
+		const firstRow = rows[0]
+		if (firstRow) {
+			const cellCount = firstRow.querySelectorAll('td').length
+			if (cellCount > 0) {
+				result.splice(1, 0, `| ${Array(cellCount).fill('---').join(' | ')} |`)
+			}
+		}
+	}
+
+	return result.join('\n')
+}
+
+/**
+ * Extract page title from HTML
+ */
+function extractTitle(root: HTMLElement): string | undefined {
+	// Try meta title first
+	const title = root.querySelector('title')
+	if (title?.textContent) {
+		return title.textContent.trim()
+	}
+
+	// Try og:title
+	const ogTitle = root.querySelector('meta[property="og:title"]')
+	if (ogTitle) {
+		return ogTitle.getAttribute('content') ?? undefined
+	}
+
+	// Try first h1
+	const h1 = root.querySelector('h1')
+	if (h1?.textContent) {
+		return h1.textContent.trim()
+	}
+
+	return undefined
+}
+
+/**
+ * Extract meta description from HTML
+ */
+function extractDescription(root: HTMLElement): string | undefined {
+	const meta = root.querySelector('meta[name="description"]')
+	if (meta) {
+		return meta.getAttribute('content') ?? undefined
+	}
+
+	const ogDesc = root.querySelector('meta[property="og:description"]')
+	if (ogDesc) {
+		return ogDesc.getAttribute('content') ?? undefined
+	}
+
+	return undefined
+}
+
+export interface HtmlToMarkdownResult {
+	/** Extracted metadata for frontmatter */
+	metadata: {
+		title?: string
+		description?: string
+	}
+	/** Converted markdown body */
+	body: string
+}
+
+/**
+ * Convert HTML string to markdown
+ */
+export function htmlToMarkdown(html: string): HtmlToMarkdownResult {
+	const root = parse(html)
+
+	// Extract metadata
+	const title = extractTitle(root)
+	const description = extractDescription(root)
+
+	// Get main content
+	const mainContent = extractMainContent(root)
+
+	// Convert to markdown
+	let body = elementToMarkdown(mainContent)
+
+	// Clean up the output
+	body = body
+		// Remove excessive newlines
+		.replace(/\n{3,}/g, '\n\n')
+		// Trim whitespace from each line
+		.split('\n')
+		.map((line) => line.trimEnd())
+		.join('\n')
+		// Trim start and end
+		.trim()
+
+	return {
+		metadata: {
+			title,
+			description,
+		},
+		body,
+	}
+}

package/src/index.ts

@@ -0,0 +1,29 @@
+import type { AstroConfig, AstroIntegration } from 'astro'
+import { processBuildOutput } from './build-processor'
+import { createDevMiddleware } from './dev-middleware'
+import { type PageMarkdownOptions, resolveOptions } from './types'
+
+export default function pageMarkdown(options: PageMarkdownOptions = {}): AstroIntegration {
+	const resolvedOptions = resolveOptions(options)
+	let config: AstroConfig
+
+	return {
+		name: 'astro-page-markdown',
+		hooks: {
+			'astro:config:done': ({ config: cfg }) => {
+				config = cfg
+			},
+
+			'astro:server:setup': ({ server, logger }) => {
+				createDevMiddleware(server, resolvedOptions, config)
+				logger.info('Markdown endpoints enabled')
+			},
+
+			'astro:build:done': async ({ dir, pages, logger }) => {
+				await processBuildOutput(dir, pages, resolvedOptions, logger, config)
+			},
+		},
+	}
+}
+
+export type { LlmEndpointOptions, LlmsTxtOptions, MarkdownOutput, PageMarkdownOptions } from './types'

package/src/llm-endpoint.ts

@@ -0,0 +1,80 @@
+import { getMarkdownUrl } from './paths'
+import type { LlmEndpointOptions } from './types'
+
+export interface PageEntry {
+	pathname: string
+	title?: string
+	type: 'collection' | 'static'
+}
+
+export interface SiteMetadata {
+	title?: string
+	description?: string
+	baseUrl?: string
+}
+
+/**
+ * Generate the content for /.well-known/llm.md
+ */
+export function generateLlmMarkdown(
+	pages: PageEntry[],
+	siteMetadata: SiteMetadata,
+	options: LlmEndpointOptions,
+): string {
+	const siteName = options.siteName ?? siteMetadata.title ?? 'Site'
+	const description = options.description ?? siteMetadata.description
+	const baseUrl = options.baseUrl ?? siteMetadata.baseUrl ?? ''
+
+	const lines: string[] = []
+
+	// Frontmatter
+	lines.push('---')
+	lines.push(`generatedAt: ${new Date().toISOString()}`)
+	lines.push('---')
+	lines.push('')
+
+	// Title
+	lines.push(`# ${siteName}`)
+	lines.push('')
+
+	// Description
+	if (description) {
+		lines.push(description)
+		lines.push('')
+	}
+
+	// Markdown endpoints section
+	lines.push('## Markdown Endpoints')
+	lines.push('')
+	lines.push('This site exposes page content as markdown at `.md` URLs.')
+	lines.push('')
+
+	// Pages section
+	if (pages.length > 0) {
+		lines.push('### Pages')
+		lines.push('')
+
+		// List all pages sorted by pathname
+		const sortedPages = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname))
+		for (const page of sortedPages) {
+			const mdUrl = getMarkdownUrl(page.pathname)
+			lines.push(`- [${baseUrl}${mdUrl}](${baseUrl}${mdUrl})${page.title ? ` - ${page.title}` : ''}`)
+		}
+		lines.push('')
+	}
+
+	// Usage section
+	lines.push('## Usage')
+	lines.push('')
+	lines.push('Append `.md` to any page URL to get the markdown version:')
+	lines.push(`- \`${baseUrl}/about\` → \`${baseUrl}/about.md\``)
+	lines.push(`- \`${baseUrl}/blog/hello\` → \`${baseUrl}/blog/hello.md\``)
+
+	// Additional content
+	if (options.additionalContent) {
+		lines.push('')
+		lines.push(options.additionalContent)
+	}
+
+	return lines.join('\n')
+}

package/src/llms-txt-endpoint.ts

@@ -0,0 +1,123 @@
+import { getMarkdownUrl } from './paths'
+import type { LlmsTxtOptions } from './types'
+
+export interface PageEntry {
+	pathname: string
+	title?: string
+	type: 'collection' | 'static'
+}
+
+export interface SiteMetadata {
+	title?: string
+	description?: string
+	baseUrl?: string
+}
+
+/**
+ * Generate the content for /llms.txt
+ *
+ * The llms.txt format follows the convention at https://llmstxt.org/
+ * providing a standardized way to communicate site structure to LLMs and crawlers.
+ */
+export function generateLlmsTxt(
+	pages: PageEntry[],
+	siteMetadata: SiteMetadata,
+	options: LlmsTxtOptions,
+): string {
+	const siteName = options.siteName ?? siteMetadata.title ?? 'Site'
+	const description = options.description ?? siteMetadata.description
+	const baseUrl = options.baseUrl ?? siteMetadata.baseUrl ?? ''
+
+	const lines: string[] = []
+
+	// Title
+	lines.push(`# ${siteName}`)
+	lines.push('')
+
+	// Description as blockquote (llms.txt convention)
+	if (description) {
+		lines.push(`> ${description}`)
+		lines.push('')
+	}
+
+	// LLM-specific guidance
+	if (options.allowCrawling !== false) {
+		lines.push('This site provides markdown versions of all pages for LLM consumption.')
+		lines.push('')
+	}
+
+	// LLM Discovery endpoint
+	lines.push('## LLM Discovery')
+	lines.push('')
+	lines.push(`- [LLM Discovery Endpoint](${baseUrl}/.well-known/llm.md): Full site map with all available markdown endpoints`)
+	lines.push('')
+
+	// Markdown endpoints section
+	lines.push('## Markdown Endpoints')
+	lines.push('')
+	lines.push('All pages are available as markdown by appending `.md` to the URL.')
+	lines.push('')
+
+	// Separate collection and static pages
+	const collectionPages = pages.filter((p) => p.type === 'collection')
+	const staticPages = pages.filter((p) => p.type === 'static')
+
+	// Content pages (from collections)
+	if (collectionPages.length > 0) {
+		lines.push('### Content')
+		lines.push('')
+		const sortedCollection = [...collectionPages].sort((a, b) => a.pathname.localeCompare(b.pathname))
+		for (const page of sortedCollection) {
+			const mdUrl = getMarkdownUrl(page.pathname)
+			const title = page.title ?? page.pathname
+			lines.push(`- [${title}](${baseUrl}${mdUrl}): ${page.pathname}`)
+		}
+		lines.push('')
+	}
+
+	// Static pages
+	if (staticPages.length > 0) {
+		lines.push('### Pages')
+		lines.push('')
+		const sortedStatic = [...staticPages].sort((a, b) => a.pathname.localeCompare(b.pathname))
+		for (const page of sortedStatic) {
+			const mdUrl = getMarkdownUrl(page.pathname)
+			const title = page.title ?? page.pathname
+			lines.push(`- [${title}](${baseUrl}${mdUrl}): ${page.pathname}`)
+		}
+		lines.push('')
+	}
+
+	// If both are empty, show a generic message
+	if (collectionPages.length === 0 && staticPages.length === 0) {
+		lines.push('Append `.md` to any page URL to get the markdown version.')
+		lines.push('')
+		lines.push('Examples:')
+		lines.push('- `/about` → `/about.md`')
+		lines.push('- `/blog/post` → `/blog/post.md`')
+		lines.push('')
+	}
+
+	// Crawling permissions
+	lines.push('## Permissions')
+	lines.push('')
+	if (options.allowCrawling !== false) {
+		lines.push('LLMs and crawlers are welcome to access markdown endpoints.')
+	} else {
+		lines.push('Please respect rate limits when crawling.')
+	}
+
+	// Additional instructions
+	if (options.instructions) {
+		lines.push('')
+		lines.push(options.instructions)
+	}
+
+	// Additional content
+	if (options.additionalContent) {
+		lines.push('')
+		lines.push(options.additionalContent)
+	}
+
+	return lines.join('\n')
+}

package/src/markdown-generator.ts

@@ -0,0 +1,138 @@
+import type { MarkdownOutput } from './types'
+
+/**
+ * Serialize a value to YAML format
+ */
+function yamlValue(value: unknown): string {
+	if (value === null || value === undefined) {
+		return 'null'
+	}
+	if (typeof value === 'string') {
+		// Check if string needs quoting
+		if (
+			value.includes('\n')
+			|| value.includes(':')
+			|| value.includes('#')
+			|| value.startsWith(' ')
+			|| value.endsWith(' ')
+			|| value === ''
+		) {
+			// Use double quotes and escape
+			return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')}"`
+		}
+		return value
+	}
+	if (typeof value === 'number' || typeof value === 'boolean') {
+		return String(value)
+	}
+	if (Array.isArray(value)) {
+		if (value.length === 0) return '[]'
+		return '\n' + value.map((v) => `  - ${yamlValue(v)}`).join('\n')
+	}
+	if (typeof value === 'object') {
+		const entries = Object.entries(value)
+		if (entries.length === 0) return '{}'
+		return '\n' + entries.map(([k, v]) => `  ${k}: ${yamlValue(v)}`).join('\n')
+	}
+	return String(value)
+}
+
+/**
+ * Convert frontmatter object to YAML string
+ */
+function frontmatterToYaml(frontmatter: Record<string, unknown>): string {
+	const lines: string[] = []
+
+	for (const [key, value] of Object.entries(frontmatter)) {
+		const yamlVal = yamlValue(value)
+		if (yamlVal.startsWith('\n')) {
+			lines.push(`${key}:${yamlVal}`)
+		} else {
+			lines.push(`${key}: ${yamlVal}`)
+		}
+	}
+
+	return lines.join('\n')
+}
+
+export interface GenerateOptions {
+	/** URL path of the page */
+	url: string
+	/** Type of content (collection or static) */
+	type: 'collection' | 'static'
+	/** Path to source file (for collections) */
+	sourcePath?: string
+}
+
+/**
+ * Generate complete markdown output with frontmatter
+ */
+export function generateMarkdown(
+	output: MarkdownOutput,
+	options: GenerateOptions,
+	includeFrontmatter = true,
+): string {
+	if (!includeFrontmatter) {
+		return output.body
+	}
+
+	// Build frontmatter with metadata
+	const frontmatter: Record<string, unknown> = {
+		...output.frontmatter,
+		url: options.url,
+		type: options.type,
+		generatedAt: new Date().toISOString(),
+	}
+
+	if (options.sourcePath) {
+		frontmatter.source = options.sourcePath
+	}
+
+	const yamlContent = frontmatterToYaml(frontmatter)
+
+	return `---\n${yamlContent}\n---\n\n${output.body}`
+}
+
+/**
+ * Create a simple markdown output for collection content
+ * that already has markdown body
+ */
+export function createCollectionOutput(
+	frontmatter: Record<string, { value: string; line: number }>,
+	body: string,
+	sourcePath: string,
+): MarkdownOutput {
+	// Convert frontmatter to simple key-value
+	const simpleFrontmatter: Record<string, unknown> = {}
+	for (const [key, { value }] of Object.entries(frontmatter)) {
+		simpleFrontmatter[key] = value
+	}
+
+	return {
+		frontmatter: simpleFrontmatter,
+		body,
+		sourcePath,
+	}
+}
+
+/**
+ * Create markdown output from HTML conversion result
+ */
+export function createStaticOutput(
+	metadata: { title?: string; description?: string },
+	body: string,
+): MarkdownOutput {
+	const frontmatter: Record<string, unknown> = {}
+
+	if (metadata.title) {
+		frontmatter.title = metadata.title
+	}
+	if (metadata.description) {
+		frontmatter.description = metadata.description
+	}
+
+	return {
+		frontmatter,
+		body,
+	}
+}