astro-markdown-for-agents 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Trent Rand
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,95 @@
+# astro-markdown-for-agents
+
+Astro integration that generates markdown variants of your pages for AI agents and supports same-URL markdown negotiation in Astro dev.
+
+## What it does
+
+- Converts built HTML pages into markdown files under `dist/_markdown-cache` (configurable).
+- In Astro dev, serves markdown when `Accept: text/markdown` (or `text/plain`, configurable) is preferred over HTML.
+- Adds agent-oriented headers on markdown responses:
+  - `Content-Type: text/markdown; charset=utf-8`
+  - `Vary: Accept`
+  - `x-markdown-tokens`
+  - `content-signal`
+
+## What it does NOT do
+
+This package is deployment-agnostic. It does **not** include hosting-specific runtime shims.
+
+For static hosting environments, serve-time negotiation should be implemented in the consumer app/runtime.
+
+## Installation
+
+```bash
+bun add astro-markdown-for-agents
+```
+
+## Usage
+
+In `astro.config.mjs`:
+
+```js
+import { defineConfig } from 'astro/config';
+import markdownForAgents from 'astro-markdown-for-agents';
+
+export default defineConfig({
+  integrations: [
+    markdownForAgents(),
+  ],
+});
+```
+
+## Quick verification (dev)
+
+Start Astro dev, then compare responses for the same URL:
+
+```bash
+curl -i -H 'Accept: text/html' http://localhost:4321/
+curl -i -H 'Accept: text/markdown' http://localhost:4321/
+```
+
+You should see HTML for `text/html` and markdown for `text/markdown`.
+
+## Options
+
+```ts
+type Matcher = string | RegExp | ((pathname: string) => boolean);
+
+type MarkdownForAgentsOptions = {
+  include?: Matcher[];
+  exclude?: Matcher[];
+  excludePrefixes?: string[];
+  excludeExtensions?: string[];
+  markdownDir?: string;
+  contentSignals?: {
+    aiTrain?: boolean;
+    search?: boolean;
+    aiInput?: boolean;
+  };
+  maxExtractedChars?: number;
+  preferPlainText?: boolean;
+};
+```
+
+### Defaults
+
+- `markdownDir`: `_markdown-cache`
+- `preferPlainText`: `true`
+- `maxExtractedChars`: `Infinity`
+- `excludePrefixes`: `['/api/', '/_astro/']`
+- `excludeExtensions`: common static assets (`.css`, `.js`, images, fonts, etc.)
+- `contentSignals`: `ai-train=yes, search=yes, ai-input=yes`
+
+## Export surfaces
+
+- Root (`astro-markdown-for-agents`): integration entrypoint
+- `astro-markdown-for-agents/core`: shared core helpers/types
+- `astro-markdown-for-agents/runtime`: runtime-safe helpers for consumer-hosted negotiation shims
+
+## Development notes
+
+This package currently ships source files from `src/` via export maps for Astro/Vite-based consumption.
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "astro-markdown-for-agents",
+  "description": "Astro integration that generates markdown variants of your pages for AI agents",
+  "version": "0.1.0",
+  "type": "module",
+  "private": false,
+  "author": "Trent Rand <contact@trentrand.com>",
+  "license": "MIT",
+  "keywords": [
+    "astro",
+    "integration",
+    "markdown",
+    "ai",
+    "content-negotiation",
+    "llm"
+  ],
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "astro": "^5.0.0"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./astro": "./src/astro.ts",
+    "./core": "./src/core/index.ts",
+    "./runtime": "./src/runtime.ts"
+  },
+  "dependencies": {
+    "turndown": "^7.2.0"
+  },
+  "devDependencies": {
+    "@types/turndown": "^5.0.6"
+  }
+}

package/src/astro.ts

@@ -0,0 +1,88 @@
+import type { AstroIntegration } from 'astro';
+import { fileURLToPath } from 'node:url';
+import {
+  applyMarkdownHeaders,
+  extractPathname,
+  generateMarkdownAssets,
+  htmlToMarkdown,
+  isHtmlResponse,
+  isNegotiablePath,
+  resolveOptions,
+  ensureVaryAccept,
+  shouldHandleDevRequest,
+  type MarkdownForAgentsOptions,
+  writeNodeResponse,
+} from './core/index';
+const DEV_BYPASS_HEADER = 'x-markdown-negotiation-bypass';
+
+export default function markdownForAgents(options: MarkdownForAgentsOptions = {}): AstroIntegration {
+  const resolved = resolveOptions(options);
+
+  return {
+    name: 'markdown-for-agents',
+    hooks: {
+      'astro:server:setup': ({ server }) => {
+        server.middlewares.use(async (req, res, next) => {
+          if (!shouldHandleDevRequest(req, resolved.preferPlainText, DEV_BYPASS_HEADER)) {
+            next();
+            return;
+          }
+
+          const host = req.headers.host;
+          const pathname = extractPathname(req.url);
+
+          if (!host || !pathname || !isNegotiablePath(pathname, resolved)) {
+            next();
+            return;
+          }
+
+          try {
+            const method = (req.method ?? 'GET').toUpperCase();
+            const requestUrl = new URL(req.url ?? '/', `http://${host}`);
+            const htmlResponse = await fetch(requestUrl.toString(), {
+              method,
+              headers: {
+                accept: 'text/html',
+                [DEV_BYPASS_HEADER]: '1',
+              },
+            });
+
+            const headers = new Headers(htmlResponse.headers);
+            ensureVaryAccept(headers);
+
+            if (!htmlResponse.ok || !isHtmlResponse(headers.get('content-type'))) {
+              await writeNodeResponse(res, htmlResponse, headers);
+              return;
+            }
+
+            const html = await htmlResponse.text();
+            const markdown = htmlToMarkdown(html, pathname, resolved.maxExtractedChars);
+            applyMarkdownHeaders(headers, markdown, resolved);
+
+            res.statusCode = htmlResponse.status;
+            res.statusMessage = htmlResponse.statusText;
+
+            for (const [key, value] of headers.entries()) {
+              res.setHeader(key, value);
+            }
+
+            if (method === 'HEAD') {
+              res.end();
+              return;
+            }
+
+            res.end(markdown);
+          } catch (error) {
+            next(error as Error);
+          }
+        });
+      },
+
+      'astro:build:done': async ({ dir, logger }) => {
+        const distDir = fileURLToPath(dir);
+        await generateMarkdownAssets({ distDir, options: resolved });
+        logger.info('[markdown-for-agents] Markdown assets generated');
+      },
+    },
+  };
+}

package/src/core/accept.ts

@@ -0,0 +1,56 @@
+type AcceptEntry = {
+  mediaType: string;
+  q: number;
+  index: number;
+};
+
+export function parseAcceptHeader(value: string): AcceptEntry[] {
+  if (!value) {
+    return [];
+  }
+
+  return value
+    .split(',')
+    .map((item, index) => {
+      const [mediaType, ...params] = item
+        .split(';')
+        .map((part) => part.trim().toLowerCase());
+      let q = 1;
+
+      for (const param of params) {
+        if (!param.startsWith('q=')) {
+          continue;
+        }
+        const parsed = Number(param.slice(2));
+        if (!Number.isNaN(parsed)) {
+          q = parsed;
+        }
+      }
+
+      return { mediaType, q, index };
+    })
+    .filter((entry) => entry.mediaType && entry.q > 0)
+    .sort((a, b) => (b.q !== a.q ? b.q - a.q : a.index - b.index));
+}
+
+export function prefersMarkdown(acceptHeader: string, preferPlainText = true): boolean {
+  const accepted = parseAcceptHeader(acceptHeader);
+  if (accepted.length === 0) {
+    return false;
+  }
+
+  const markdownIndex = accepted.findIndex((entry) =>
+    entry.mediaType === 'text/markdown' || (preferPlainText && entry.mediaType === 'text/plain'),
+  );
+
+  if (markdownIndex === -1) {
+    return false;
+  }
+
+  const htmlIndex = accepted.findIndex(
+    (entry) =>
+      entry.mediaType === 'text/html' || entry.mediaType === 'text/*' || entry.mediaType === '*/*',
+  );
+
+  return htmlIndex === -1 || markdownIndex < htmlIndex;
+}

package/src/core/build.ts

@@ -0,0 +1,74 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { htmlToMarkdown } from './convert';
+import { isNegotiablePath, markdownAssetPath } from './routes';
+import type { ResolvedOptions } from './types';
+
+type GenerateMarkdownAssetsInput = {
+  distDir: string;
+  options: ResolvedOptions;
+};
+
+export async function generateMarkdownAssets(input: GenerateMarkdownAssetsInput): Promise<void> {
+  const htmlFiles = await collectHtmlFiles(input.distDir, input.options.markdownDir);
+
+  for (const htmlFilePath of htmlFiles) {
+    const relativeHtmlPath = path.relative(input.distDir, htmlFilePath);
+    const pathname = htmlFileToRoutePath(relativeHtmlPath);
+    if (!isNegotiablePath(pathname, input.options)) {
+      continue;
+    }
+
+    const html = await fs.readFile(htmlFilePath, 'utf-8');
+    const markdown = htmlToMarkdown(html, pathname, input.options.maxExtractedChars);
+    const outPath = path.join(
+      input.distDir,
+      markdownAssetPath(pathname, input.options.markdownDir).slice(1),
+    );
+
+    await fs.mkdir(path.dirname(outPath), { recursive: true });
+    await fs.writeFile(outPath, markdown, 'utf-8');
+  }
+}
+
+async function collectHtmlFiles(distDir: string, markdownDir: string): Promise<string[]> {
+  const entries = await fs.readdir(distDir, { withFileTypes: true });
+  const files: string[] = [];
+
+  for (const entry of entries) {
+    const absolutePath = path.join(distDir, entry.name);
+
+    if (entry.isDirectory()) {
+      if (entry.name === '_worker.js' || entry.name === markdownDir) {
+        continue;
+      }
+
+      files.push(...(await collectHtmlFiles(absolutePath, markdownDir)));
+      continue;
+    }
+
+    if (entry.isFile() && entry.name.endsWith('.html')) {
+      files.push(absolutePath);
+    }
+  }
+
+  return files;
+}
+
+function htmlFileToRoutePath(relativeHtmlPath: string): string {
+  const normalized = relativeHtmlPath.split(path.sep).join('/');
+
+  if (normalized === 'index.html') {
+    return '/';
+  }
+
+  if (normalized.endsWith('/index.html')) {
+    return `/${normalized.slice(0, -'/index.html'.length)}/`;
+  }
+
+  if (normalized.endsWith('.html')) {
+    return `/${normalized.slice(0, -'.html'.length)}`;
+  }
+
+  return '/';
+}

package/src/core/convert.ts

@@ -0,0 +1,60 @@
+import TurndownService from 'turndown';
+
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+});
+
+export function htmlToMarkdown(
+  html: string,
+  routePath: string,
+  maxExtractedChars: number,
+): string {
+  const title = extractMeta(html, /<title[^>]*>([^<]*)<\/title>/i);
+  const description = extractMeta(
+    html,
+    /<meta\s+name=["']description["']\s+content=["']([^"']*)["'][^>]*>/i,
+  );
+
+  const bodyHtml = extractBodyHtml(html);
+  const markdown = turndown.turndown(bodyHtml).trim().slice(0, maxExtractedChars).trim();
+
+  const lines: string[] = [
+    `# ${title || `Page ${routePath}`}`,
+    '',
+    `- Path: ${routePath}`,
+    `- Generated: ${new Date().toISOString()}`,
+  ];
+
+  if (description) {
+    lines.push(`- Description: ${description}`);
+  }
+
+  lines.push('', '## Extracted Content', '', markdown || '_No extractable body content found._');
+
+  return lines.join('\n');
+}
+
+function extractBodyHtml(html: string): string {
+  const stripped = html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ');
+
+  return stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? stripped;
+}
+
+function extractMeta(html: string, pattern: RegExp): string {
+  const match = html.match(pattern);
+  return decodeHtmlEntities(match?.[1]?.trim() ?? '');
+}
+
+function decodeHtmlEntities(value: string): string {
+  return value
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ');
+}

package/src/core/headers.ts

@@ -0,0 +1,43 @@
+import type { ResolvedOptions } from './types';
+
+export function ensureVaryAccept(headers: Headers): void {
+  const vary = headers.get('vary');
+  if (!vary) {
+    headers.set('vary', 'Accept');
+    return;
+  }
+
+  const values = vary
+    .split(',')
+    .map((part) => part.trim().toLowerCase())
+    .filter(Boolean);
+
+  if (!values.includes('accept')) {
+    headers.set(
+      'vary',
+      [...new Set([...vary.split(',').map((part) => part.trim()), 'Accept'])]
+        .filter(Boolean)
+        .join(', '),
+    );
+  }
+}
+
+export function isHtmlResponse(contentType: string | null): boolean {
+  return (contentType ?? '').toLowerCase().includes('text/html');
+}
+
+export function countTokens(markdown: string): number {
+  return markdown.trim().split(/\s+/).filter(Boolean).length;
+}
+
+export function applyMarkdownHeaders(
+  headers: Headers,
+  markdown: string,
+  options: Pick<ResolvedOptions, 'contentSignalHeader'>,
+): void {
+  headers.set('content-type', 'text/markdown; charset=utf-8');
+  headers.set('x-markdown-tokens', String(countTokens(markdown)));
+  headers.set('content-signal', options.contentSignalHeader);
+  headers.delete('content-length');
+  headers.delete('etag');
+}

package/src/core/index.ts

@@ -0,0 +1,8 @@
+export * from './types';
+export * from './options';
+export * from './accept';
+export * from './headers';
+export * from './routes';
+export * from './convert';
+export * from './build';
+export * from './server';

package/src/core/options.ts

@@ -0,0 +1,50 @@
+import type { MarkdownForAgentsOptions, ResolvedOptions } from './types';
+
+const DEFAULT_EXCLUDE_PREFIXES = ['/api/', '/_astro/'];
+const DEFAULT_EXCLUDE_EXTENSIONS = [
+  '.css',
+  '.js',
+  '.mjs',
+  '.map',
+  '.json',
+  '.xml',
+  '.txt',
+  '.ico',
+  '.png',
+  '.jpg',
+  '.jpeg',
+  '.webp',
+  '.svg',
+  '.gif',
+  '.pdf',
+  '.zip',
+  '.woff',
+  '.woff2',
+];
+
+export function resolveOptions(options: MarkdownForAgentsOptions = {}): ResolvedOptions {
+  return {
+    include: options.include ?? [],
+    exclude: options.exclude ?? [],
+    excludePrefixes: options.excludePrefixes ?? DEFAULT_EXCLUDE_PREFIXES,
+    excludeExtensions: options.excludeExtensions ?? DEFAULT_EXCLUDE_EXTENSIONS,
+    markdownDir: options.markdownDir ?? '_markdown-cache',
+    contentSignalHeader: toContentSignalHeader(options.contentSignals),
+    maxExtractedChars: options.maxExtractedChars ?? Infinity,
+    preferPlainText: options.preferPlainText ?? true,
+  };
+}
+
+function toContentSignalHeader(signals: MarkdownForAgentsOptions['contentSignals']): string {
+  const resolved = {
+    aiTrain: signals?.aiTrain ?? true,
+    search: signals?.search ?? true,
+    aiInput: signals?.aiInput ?? true,
+  };
+
+  return [
+    `ai-train=${resolved.aiTrain ? 'yes' : 'no'}`,
+    `search=${resolved.search ? 'yes' : 'no'}`,
+    `ai-input=${resolved.aiInput ? 'yes' : 'no'}`,
+  ].join(', ');
+}

package/src/core/routes.ts

@@ -0,0 +1,101 @@
+import type { Matcher, ResolvedOptions } from './types';
+
+export function extractPathname(requestUrl?: string): string | null {
+  if (!requestUrl) {
+    return null;
+  }
+
+  const queryIndex = requestUrl.indexOf('?');
+  return queryIndex === -1 ? requestUrl : requestUrl.slice(0, queryIndex);
+}
+
+export function isNegotiablePath(pathname: string, options: ResolvedOptions): boolean {
+  if (!pathname) {
+    return false;
+  }
+
+  if (matchesAny(pathname, options.exclude)) {
+    return false;
+  }
+
+  if (options.include.length > 0 && !matchesAny(pathname, options.include)) {
+    return false;
+  }
+
+  if (options.excludePrefixes.some((prefix) => pathname.startsWith(prefix))) {
+    return false;
+  }
+
+  const lastSegment = pathname.split('/').pop() ?? '';
+  const ext = extensionOf(lastSegment);
+  if (ext && options.excludeExtensions.includes(ext)) {
+    return false;
+  }
+
+  return !lastSegment.includes('.') || ext === '';
+}
+
+export function normalizePathname(pathname: string): string {
+  if (!pathname || pathname === '/') {
+    return '/';
+  }
+
+  if (pathname.endsWith('/')) {
+    return pathname;
+  }
+
+  const lastSegment = pathname.split('/').pop() ?? '';
+  if (lastSegment.includes('.')) {
+    return pathname;
+  }
+
+  return `${pathname}/`;
+}
+
+export function markdownAssetPath(pathname: string, markdownDir: string): string {
+  const normalized = normalizePathname(pathname);
+
+  if (normalized === '/') {
+    return `/${markdownDir}/index.md`;
+  }
+
+  if (normalized.endsWith('/')) {
+    return `/${markdownDir}${normalized}index.md`;
+  }
+
+  return `/${markdownDir}${normalized}.md`;
+}
+
+function extensionOf(value: string): string {
+  const dotIndex = value.lastIndexOf('.');
+  if (dotIndex === -1) {
+    return '';
+  }
+  return value.slice(dotIndex).toLowerCase();
+}
+
+function matchesAny(pathname: string, matchers: Matcher[]): boolean {
+  return matchers.some((matcher) => matches(pathname, matcher));
+}
+
+function matches(pathname: string, matcher: Matcher): boolean {
+  if (typeof matcher === 'function') {
+    return matcher(pathname);
+  }
+
+  if (matcher instanceof RegExp) {
+    return matcher.test(pathname);
+  }
+
+  if (!matcher.includes('*')) {
+    return pathname === matcher;
+  }
+
+  const regex = new RegExp(
+    `^${matcher
+      .replace(/[.+?^${}()|[\]\\]/g, '\\$&')
+      .replace(/\*/g, '.*')}$`,
+  );
+
+  return regex.test(pathname);
+}

package/src/core/server.ts

@@ -0,0 +1,35 @@
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { prefersMarkdown } from './accept';
+
+export function shouldHandleDevRequest(
+  req: IncomingMessage,
+  preferPlainText: boolean,
+  bypassHeaderName: string,
+): boolean {
+  const method = (req.method ?? 'GET').toUpperCase();
+  if (method !== 'GET' && method !== 'HEAD') {
+    return false;
+  }
+
+  if (req.headers[bypassHeaderName] === '1') {
+    return false;
+  }
+
+  return prefersMarkdown(String(req.headers.accept ?? ''), preferPlainText);
+}
+
+export async function writeNodeResponse(
+  res: ServerResponse,
+  response: Response,
+  headers: Headers,
+): Promise<void> {
+  res.statusCode = response.status;
+  res.statusMessage = response.statusText;
+
+  for (const [key, value] of headers.entries()) {
+    res.setHeader(key, value);
+  }
+
+  const body = await response.arrayBuffer();
+  res.end(Buffer.from(body));
+}

package/src/core/types.ts

@@ -0,0 +1,29 @@
+export type Matcher = string | RegExp | ((pathname: string) => boolean);
+
+export type ContentSignals = {
+  aiTrain?: boolean;
+  search?: boolean;
+  aiInput?: boolean;
+};
+
+export type MarkdownForAgentsOptions = {
+  include?: Matcher[];
+  exclude?: Matcher[];
+  excludePrefixes?: string[];
+  excludeExtensions?: string[];
+  markdownDir?: string;
+  contentSignals?: ContentSignals;
+  maxExtractedChars?: number;
+  preferPlainText?: boolean;
+};
+
+export type ResolvedOptions = {
+  include: Matcher[];
+  exclude: Matcher[];
+  excludePrefixes: string[];
+  excludeExtensions: string[];
+  markdownDir: string;
+  contentSignalHeader: string;
+  maxExtractedChars: number;
+  preferPlainText: boolean;
+};

package/src/index.ts

@@ -0,0 +1,3 @@
+export type { MarkdownForAgentsOptions } from './core/index';
+export { default as markdownForAgents } from './astro';
+export { default } from './astro';

package/src/runtime.ts

@@ -0,0 +1,3 @@
+export { prefersMarkdown } from './core/accept';
+export { ensureVaryAccept, isHtmlResponse, applyMarkdownHeaders } from './core/headers';
+export { markdownAssetPath } from './core/routes';