@coywolf/emdash 0.0.0-stage → 0.35.13

package/src/discovery/markdown.ts

@@ -0,0 +1,294 @@
+/**
+ * Portable Text → Markdown, for the per-page Markdown sources served at
+ * `<entry-url>/index.html.md` (the llmstxt.org convention, as in Coywolf SEO).
+ *
+ * Pure and dependency-free so it can be unit-tested with `node --test`.
+ * Handles the core block types EmDash's editor writes: text blocks
+ * (paragraphs, headings h1–h6, blockquotes, bullet and numbered lists at any
+ * depth, with strong/em/code/strike-through and link marks), images, code,
+ * horizontal breaks, and tables. Anything else (embeds, buttons, custom
+ * plugin blocks) is skipped rather than guessed at.
+ */
+
+// biome-ignore lint/suspicious/noExplicitAny: Portable Text is untyped JSON.
+type Node = Record<string, any>;
+
+export interface MarkdownOptions {
+	/** Turn a site-relative href or media URL into an absolute one. Default: unchanged. */
+	absolute?: (url: string) => string;
+}
+
+export interface Frontmatter {
+	[key: string]: string | string[] | undefined;
+}
+
+const HEADING = /^h([1-6])$/;
+const DANGEROUS_SCHEME = /^\s*(javascript|vbscript|data):/i;
+
+const isRecord = (value: unknown): value is Node => typeof value === "object" && value !== null && !Array.isArray(value);
+
+/** Escape characters that would otherwise turn plain text into Markdown syntax. */
+export function escapeText(text: string): string {
+	return text.replace(/([\\`*_[\]<>])/g, "\\$1");
+}
+
+/**
+ * Escape a line's leading characters that would start a block: headings,
+ * quotes, list markers, code fences (``` or ~~~), and thematic breaks or
+ * setext underlines (runs of 3+ "-", "*", "_" or "=", optionally spaced).
+ */
+export function escapeLineStart(line: string): string {
+	// Thematic break / setext underline: the whole line is one repeated marker.
+	if (/^\s*([-*_=])(\s*\1){2,}\s*$/.test(line)) return line.replace(/^(\s*)(\S)/, "$1\\$2");
+	// Fenced code: ``` is already escaped by escapeText; ~~~ is not.
+	if (/^\s*~{3,}/.test(line)) return line.replace(/^(\s*)~/, "$1\\~");
+	return line.replace(/^(\s*)([#>+-]|\d+[.)])(?=\s|$)/, (_m, space: string, marker: string) => `${space}${marker.replace(/([#>+\-.)])/, "\\$1")}`);
+}
+
+function safeHref(href: unknown, options: MarkdownOptions): string | null {
+	if (typeof href !== "string" || !href.trim() || DANGEROUS_SCHEME.test(href)) return null;
+	const url = options.absolute ? options.absolute(href.trim()) : href.trim();
+	return url.replace(/[()\s]/g, (c) => encodeURIComponent(c));
+}
+
+/** Inline content of a text block (or table cell): spans with marks and link annotations. */
+export function renderSpans(children: unknown, markDefs: unknown, options: MarkdownOptions = {}): string {
+	if (!Array.isArray(children)) return "";
+	const defs = new Map<string, Node>();
+	if (Array.isArray(markDefs)) for (const def of markDefs) if (isRecord(def) && typeof def._key === "string") defs.set(def._key, def);
+
+	let out = "";
+	for (const child of children) {
+		if (!isRecord(child)) continue;
+		if (child._type !== "span") {
+			// Inline objects (e.g. a hard break) — keep a break, skip the rest.
+			if (child._type === "break") out += "  \n";
+			continue;
+		}
+		const raw = typeof child.text === "string" ? child.text : "";
+		if (!raw) continue;
+		const marks: string[] = Array.isArray(child.marks) ? child.marks.filter((m: unknown): m is string => typeof m === "string") : [];
+
+		const isCode = marks.includes("code");
+		let text = isCode ? codeSpan(raw) : raw.split("\n").map(escapeText).join("  \n");
+		if (!isCode && text.trim()) {
+			// Keep surrounding whitespace outside the emphasis markers so `** bold**` never happens.
+			const lead = text.match(/^\s*/)?.[0] ?? "";
+			const trail = text.match(/\s*$/)?.[0] ?? "";
+			let core = text.slice(lead.length, text.length - trail.length);
+			if (marks.includes("strike-through")) core = `~~${core}~~`;
+			if (marks.includes("em")) core = `_${core}_`;
+			if (marks.includes("strong")) core = `**${core}**`;
+			text = lead + core + trail;
+		}
+		for (const mark of marks) {
+			const def = defs.get(mark);
+			if (def && (def._type === "link" || typeof def.href === "string")) {
+				const href = safeHref(def.href, options);
+				if (href) text = `[${text}](${href})`;
+			}
+		}
+		out += text;
+	}
+	return out;
+}
+
+function codeSpan(text: string): string {
+	const runs = text.match(/`+/g) ?? [];
+	const fence = "`".repeat(Math.max(0, ...runs.map((r) => r.length)) + 1);
+	const pad = text.startsWith("`") || text.endsWith("`") ? " " : "";
+	return `${fence}${pad}${text}${pad}${fence}`;
+}
+
+function renderCode(block: Node): string | null {
+	const code = typeof block.code === "string" ? block.code : "";
+	if (!code) return null;
+	const runs = code.match(/`{3,}/g) ?? [];
+	const fence = "`".repeat(Math.max(3, ...runs.map((r) => r.length + 1)));
+	const language = typeof block.language === "string" ? block.language.replace(/[^\w+#.-]/g, "") : "";
+	const filename = typeof block.filename === "string" && block.filename.trim() ? `${escapeText(block.filename.trim())}\n\n` : "";
+	return `${filename}${fence}${language}\n${code.replace(/\n+$/, "")}\n${fence}`;
+}
+
+/** The same URL resolution EmDash's image renderer uses for local media. */
+function imageUrl(block: Node): string | null {
+	const asset = isRecord(block.asset) ? block.asset : {};
+	const url = typeof asset.url === "string" && asset.url ? asset.url : typeof asset.src === "string" && asset.src ? asset.src : null;
+	if (url) return url;
+	const key = isRecord(asset.meta) && typeof asset.meta.storageKey === "string" ? asset.meta.storageKey : null;
+	return key ? `/_emdash/api/media/file/${key.split("/").map(encodeURIComponent).join("/")}` : null;
+}
+
+function renderImage(block: Node, options: MarkdownOptions): string | null {
+	const src = safeHref(imageUrl(block), options);
+	if (!src) return null;
+	const asset = isRecord(block.asset) ? block.asset : {};
+	const altRaw = typeof block.alt === "string" ? block.alt : typeof asset.alt === "string" ? asset.alt : "";
+	const alt = altRaw.replace(/[\r\n]+/g, " ").replace(/([\\[\]])/g, "\\$1");
+	let out = `![${alt}](${src})`;
+	const link = typeof block.link === "string" ? block.link : isRecord(block.link) ? block.link.href : null;
+	const href = safeHref(link, options);
+	if (href) out = `[${out}](${href})`;
+	const caption = typeof block.caption === "string" ? block.caption.trim() : "";
+	if (caption) out += `\n\n_${escapeText(caption.replace(/\s+/g, " "))}_`;
+	return out;
+}
+
+function cellText(cell: Node, tableDefs: unknown, options: MarkdownOptions): string {
+	const defs = [...(Array.isArray(tableDefs) ? tableDefs : []), ...(Array.isArray(cell.markDefs) ? cell.markDefs : [])];
+	const content = Array.isArray(cell.content) ? cell.content : Array.isArray(cell.children) ? cell.children : [];
+	return renderSpans(content, defs, options).replace(/ {2}\n|\n/g, "<br>").replace(/\|/g, "\\|").trim();
+}
+
+function renderTable(block: Node, options: MarkdownOptions): string | null {
+	const rows: Node[] = Array.isArray(block.rows) ? block.rows.filter(isRecord) : [];
+	const grid: string[][] = [];
+	for (const row of rows) {
+		const cells: Node[] = Array.isArray(row.cells) ? row.cells.filter(isRecord) : [];
+		const line: string[] = [];
+		for (const cell of cells) {
+			line.push(cellText(cell, block.markDefs, options));
+			const span = typeof cell.colspan === "number" && cell.colspan > 1 ? Math.min(cell.colspan, 100) : 1;
+			for (let i = 1; i < span; i++) line.push("");
+		}
+		grid.push(line);
+	}
+	if (!grid.length) return null;
+	const width = Math.max(...grid.map((r) => r.length));
+	if (!width) return null;
+	const pad = (r: string[]) => [...r, ...Array(width - r.length).fill("")];
+	const line = (r: string[]) => `| ${pad(r).join(" | ")} |`;
+	// GFM tables need a header row: use the first row (the editor's header row when it has one).
+	const [head, ...body] = grid;
+	return [line(head), `| ${Array(width).fill("---").join(" | ")} |`, ...body.map(line)].join("\n");
+}
+
+interface ListState {
+	/** Numbering counters per nesting level for numbered lists. */
+	counters: number[];
+}
+
+function renderListItem(block: Node, state: ListState, options: MarkdownOptions): string {
+	const level = typeof block.level === "number" && block.level > 0 ? Math.min(block.level, 10) : 1;
+	state.counters.length = level;
+	const numbered = block.listItem === "number";
+	let marker = "-";
+	if (numbered) {
+		state.counters[level - 1] = (state.counters[level - 1] ?? 0) + 1;
+		marker = `${state.counters[level - 1]}.`;
+	} else {
+		state.counters[level - 1] = 0;
+	}
+	const indent = "   ".repeat(level - 1);
+	const text = renderSpans(block.children, block.markDefs, options);
+	const continuation = `${indent}${" ".repeat(marker.length + 1)}`;
+	return `${indent}${marker} ${text.split("\n").join(`\n${continuation}`)}`;
+}
+
+function renderTextBlock(block: Node, options: MarkdownOptions): string | null {
+	const text = renderSpans(block.children, block.markDefs, options);
+	if (!text.trim()) return null;
+	const style = typeof block.style === "string" ? block.style : "normal";
+	const heading = HEADING.exec(style);
+	if (heading) return `${"#".repeat(Number(heading[1]))} ${text.replace(/ {2}\n/g, " ").trim()}`;
+	if (style === "blockquote") return text.split("\n").map((l) => `> ${l}`).join("\n");
+	return text.split("\n").map(escapeLineStart).join("\n");
+}
+
+/**
+ * Serialize a Portable Text array to Markdown. Blocks are separated by a
+ * blank line; consecutive list items form one list.
+ */
+export function portableTextToMarkdown(value: unknown, options: MarkdownOptions = {}): string {
+	const blocks: unknown[] = typeof value === "string" ? safeParse(value) : Array.isArray(value) ? value : [];
+	const parts: string[] = [];
+	let list: string[] | null = null;
+	const listState: ListState = { counters: [] };
+
+	const closeList = () => {
+		if (list?.length) parts.push(list.join("\n"));
+		list = null;
+		listState.counters = [];
+	};
+
+	for (const block of blocks) {
+		if (!isRecord(block)) continue;
+		if (block._type === "block" && typeof block.listItem === "string") {
+			list ??= [];
+			list.push(renderListItem(block, listState, options));
+			continue;
+		}
+		closeList();
+		let rendered: string | null = null;
+		switch (block._type) {
+			case "block":
+				rendered = renderTextBlock(block, options);
+				break;
+			case "image":
+				rendered = renderImage(block, options);
+				break;
+			case "code":
+				rendered = renderCode(block);
+				break;
+			case "break":
+				rendered = "---";
+				break;
+			case "table":
+				rendered = renderTable(block, options);
+				break;
+			default:
+				rendered = null; // Unknown block types are skipped.
+		}
+		if (rendered) parts.push(rendered);
+	}
+	closeList();
+	return parts.join("\n\n").trim();
+}
+
+function safeParse(text: string): unknown[] {
+	try {
+		const parsed = JSON.parse(text);
+		return Array.isArray(parsed) ? parsed : [];
+	} catch {
+		return [];
+	}
+}
+
+/** YAML frontmatter with double-quoted scalars (matches Coywolf SEO's .md output). */
+export function frontmatter(fields: Frontmatter): string {
+	const yaml = (value: string) => `"${value.replace(/\r\n|\r|\n/g, " ").replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+	const lines = ["---"];
+	for (const [key, value] of Object.entries(fields)) {
+		if (Array.isArray(value)) {
+			if (!value.length) continue;
+			lines.push(`${key}:`);
+			for (const item of value) lines.push(`  - ${yaml(item)}`);
+		} else if (typeof value === "string" && value.trim()) {
+			lines.push(`${key}: ${yaml(value)}`);
+		}
+	}
+	lines.push("---");
+	return `${lines.join("\n")}\n`;
+}
+
+/** Rough token count: ~4 characters per token (the same heuristic Coywolf SEO uses for X-Markdown-Tokens). */
+export function estimateTokens(text: string): number {
+	return Math.max(1, Math.ceil([...text].length / 4));
+}
+
+/** `<page-url>/index.html.md`, the llmstxt.org convention Coywolf SEO uses. */
+export function markdownUrl(pageUrl: string): string {
+	const url = new URL(pageUrl);
+	url.search = "";
+	url.hash = "";
+	if (!url.pathname.endsWith("/")) url.pathname += "/";
+	url.pathname += "index.html.md";
+	return url.href;
+}
+
+/** The page path an `.../index.html.md` request is for, or null when the path isn't one. */
+export function pagePathFromMarkdownPath(pathname: string): string | null {
+	if (!pathname.endsWith("/index.html.md")) return null;
+	const page = pathname.slice(0, -"index.html.md".length);
+	return page || "/";
+}

package/src/discovery/memory-cache.ts

@@ -0,0 +1,97 @@
+/**
+ * A small per-isolate cache for middleware results, bounded by size rather
+ * than entry count: total bytes are capped, large values aren't kept, and
+ * misses (null results, e.g. a .md request for a random path) get their own
+ * small LRU so they can't crowd out real documents.
+ */
+
+export interface MemoryCacheOptions {
+	ttlMs: number;
+	/** Total size of cached values. */
+	maxBytes: number;
+	/** Values larger than this aren't cached. */
+	maxEntryBytes: number;
+	/** How many miss (null) results to remember. */
+	maxMisses: number;
+	now?: () => number;
+}
+
+interface Slot {
+	value: unknown;
+	bytes: number;
+	expires: number;
+}
+
+/** Approximate size: UTF-16 string length × 2 of the JSON form. */
+export function sizeOf(value: unknown): number {
+	try {
+		return (JSON.stringify(value) ?? "").length * 2;
+	} catch {
+		return Number.POSITIVE_INFINITY;
+	}
+}
+
+export class MemoryCache {
+	private readonly hits = new Map<string, Slot>();
+	private readonly misses = new Map<string, number>();
+	private bytes = 0;
+	private readonly options: MemoryCacheOptions;
+	private readonly now: () => number;
+
+	constructor(options: MemoryCacheOptions) {
+		this.options = options;
+		this.now = options.now ?? Date.now;
+	}
+
+	get totalBytes(): number {
+		return this.bytes;
+	}
+
+	get size(): { hits: number; misses: number } {
+		return { hits: this.hits.size, misses: this.misses.size };
+	}
+
+	/** `undefined` = not cached; `null` = a remembered miss. */
+	get(key: string): unknown | null | undefined {
+		const now = this.now();
+		const slot = this.hits.get(key);
+		if (slot) {
+			if (slot.expires > now) {
+				// Refresh LRU position.
+				this.hits.delete(key);
+				this.hits.set(key, slot);
+				return slot.value;
+			}
+			this.remove(key);
+		}
+		const miss = this.misses.get(key);
+		if (miss !== undefined) {
+			if (miss > now) return null;
+			this.misses.delete(key);
+		}
+		return undefined;
+	}
+
+	set(key: string, value: unknown): void {
+		this.remove(key);
+		this.misses.delete(key);
+		const expires = this.now() + this.options.ttlMs;
+		if (value === null || value === undefined) {
+			while (this.misses.size >= this.options.maxMisses) this.misses.delete(this.misses.keys().next().value as string);
+			if (this.options.maxMisses > 0) this.misses.set(key, expires);
+			return;
+		}
+		const bytes = sizeOf(value);
+		if (bytes > this.options.maxEntryBytes || bytes > this.options.maxBytes) return;
+		while (this.bytes + bytes > this.options.maxBytes && this.hits.size) this.remove(this.hits.keys().next().value as string);
+		this.hits.set(key, { value, bytes, expires });
+		this.bytes += bytes;
+	}
+
+	private remove(key: string): void {
+		const slot = this.hits.get(key);
+		if (!slot) return;
+		this.hits.delete(key);
+		this.bytes -= slot.bytes;
+	}
+}

package/src/discovery/middleware.ts

@@ -0,0 +1,186 @@
+/**
+ * Site URLs for the Discovery module, served through the pack middleware:
+ *
+ * - `/<key>.txt`                       IndexNow key file
+ * - `/news-sitemap.xml`                Google News sitemap (also `/coywolf-news-sitemap.xml`, Coywolf SEO's name)
+ * - `/feed/`, `<term archive>/feed/`  RSS feeds
+ * - `/llms.txt`                        llms.txt
+ * - `<entry-url>/index.html.md`        Markdown source of an entry
+ *
+ * The work happens in the module's public plugin routes (so it runs in the
+ * plugin context, with EmDash's URL resolution and the plugin's KV cache);
+ * this file dispatches to them through EmDash's public route handler and
+ * keeps results briefly in isolate memory.
+ *
+ * There is deliberately no `Accept: text/markdown` negotiation on entry
+ * URLs: Cloudflare's cache key ignores Accept, so a single such request
+ * could put Markdown into the cached HTML page.
+ */
+import type { MiddlewareHandler } from "astro";
+
+import { PLUGIN_ID } from "../core/features.js";
+import { noteReadFailure, routeFailed } from "../core/read-failures.js";
+import type { PackMiddleware } from "../core/module.js";
+import { FEED_EDGE_MAX_AGE, parseFeedPath } from "./feed.js";
+import { keyFromPath } from "./indexnow.js";
+import { pagePathFromMarkdownPath } from "./markdown.js";
+import { MemoryCache } from "./memory-cache.js";
+import { FEATURE } from "./module.js";
+
+type Context = Parameters<MiddlewareHandler>[0];
+
+interface PublicRouteResult {
+	success: boolean;
+	data?: unknown;
+	/** HTTP status of the route's answer (500 for an error the route didn't expect; unset when the plugin isn't enabled). */
+	status?: number;
+	error?: { code?: string };
+}
+type PublicRouteHandler = (pluginId: string, method: string, path: string, request: Request) => Promise<PublicRouteResult>;
+
+/** About 2 MB of documents per isolate; bodies over 256 KB and all but 32 misses aren't kept. */
+const memory = new MemoryCache({ ttlMs: 60_000, maxBytes: 2 * 1024 * 1024, maxEntryBytes: 256 * 1024, maxMisses: 32 });
+
+/** Call one of the module's public routes. Null when it isn't available or answers with an error (e.g. 404; a server error isn't kept). */
+async function callRoute<T>(context: Context, route: string, query: Record<string, string> = {}, keep: "all" | "misses" = "all"): Promise<T | null> {
+	const cacheKey = `${route}?${new URLSearchParams(query)}`;
+	const hit = memory.get(cacheKey);
+	if (hit !== undefined && (keep === "all" || hit === null)) return hit as T | null;
+
+	const locals = context.locals as { emdash?: { handlePublicPluginApiRoute?: PublicRouteHandler } };
+	const dispatch = locals.emdash?.handlePublicPluginApiRoute;
+	if (typeof dispatch !== "function") return null;
+	const url = new URL(`/_emdash/api/plugins/${PLUGIN_ID}/${route}`, context.url);
+	for (const [k, v] of Object.entries(query)) url.searchParams.set(k, v);
+	const result = await dispatch(PLUGIN_ID, "GET", route, new Request(url, { method: "GET" }));
+	const value = result.success ? ((result.data ?? null) as T | null) : null;
+	// A server error (a read that failed) isn't kept as the answer: the next request asks again.
+	if (routeFailed(result)) {
+		noteReadFailure("discovery route");
+		return null;
+	}
+
+	if (keep === "all" || value === null) memory.set(cacheKey, value);
+	return value;
+}
+
+const isRead = (context: Context) => context.request.method === "GET" || context.request.method === "HEAD";
+
+function respond(context: Context, body: string, headers: Record<string, string>): Response {
+	return new Response(context.request.method === "HEAD" ? null : body, { status: 200, headers });
+}
+
+// ── IndexNow key file ────────────────────────────────────────────
+
+export const indexNowKeyMiddleware: PackMiddleware = {
+	module: "discovery",
+	feature: FEATURE.indexnow,
+	handle: async (context) => {
+		if (!isRead(context)) return undefined;
+		const requested = keyFromPath(context.url.pathname);
+		if (!requested) return undefined;
+		const result = await callRoute<{ key: string | null }>(context, "discovery/public/key");
+		if (!result?.key || result.key !== requested) return undefined;
+		return respond(context, result.key, {
+			"Content-Type": "text/plain; charset=utf-8",
+			"Cache-Control": "no-cache",
+			"X-Robots-Tag": "noindex",
+		});
+	},
+};
+
+// ── News sitemap ─────────────────────────────────────────────────
+
+const NEWS_PATHS = new Set(["/news-sitemap.xml", "/coywolf-news-sitemap.xml"]);
+
+export const newsSitemapMiddleware: PackMiddleware = {
+	module: "discovery",
+	feature: FEATURE.news,
+	handle: async (context) => {
+		if (!isRead(context) || !NEWS_PATHS.has(context.url.pathname)) return undefined;
+		const result = await callRoute<{ body: string }>(context, "discovery/public/news");
+		if (!result?.body) return undefined;
+		return respond(context, result.body, {
+			"Content-Type": "application/xml; charset=utf-8",
+			"Cache-Control": "public, max-age=300",
+			// A sitemap never sends a robots noindex header (test/sitemap-headers.test.mjs).
+		});
+	},
+};
+
+// ── RSS feeds ────────────────────────────────────────────────────
+
+interface RouteCache {
+	enabled?: boolean;
+	set(options: { maxAge?: number; swr?: number; tags?: string[] } | false): void;
+}
+
+export const feedMiddleware: PackMiddleware = {
+	module: "discovery",
+	feature: FEATURE.feed,
+	// Feeds are kept in the edge cache (Workers Cache), tagged so a publish, edit or term change clears them.
+	ownsCache: true,
+	handle: async (context) => {
+		if (!isRead(context) || !parseFeedPath(context.url.pathname)) return undefined;
+		// No feed at this path (not a term, or the feed's off): the site answers, as it would without the pack.
+		// Only misses are kept in memory: a feed goes to the edge cache, and a copy from memory could put the feed
+		// from before a publish back there for an hour, right after EmDash cleared it.
+		const doc = await callRoute<{ body: string; tags?: string[] }>(context, "discovery/public/feed", { path: context.url.pathname }, "misses");
+		if (!doc?.body) return undefined;
+		const cache = (context as unknown as { cache?: RouteCache }).cache;
+		if (cache?.enabled) {
+			// Replace any lifetime the site's route rules or Plugins → Performance gave the request; swr 0 so a reader
+			// polling once an hour isn't handed the copy its last poll left.
+			cache.set(false);
+			cache.set({ maxAge: FEED_EDGE_MAX_AGE, swr: 0, tags: doc.tags ?? [] });
+		}
+		return respond(context, doc.body, {
+			"Content-Type": "application/rss+xml; charset=utf-8",
+			"Cache-Control": "public, max-age=900",
+			"X-Content-Type-Options": "nosniff",
+		});
+	},
+};
+
+// ── llms.txt and Markdown sources ────────────────────────────────
+
+interface MarkdownDoc {
+	body: string;
+	tokens: number;
+	markdownUrl: string;
+}
+
+export const llmsMiddleware: PackMiddleware = {
+	module: "discovery",
+	feature: FEATURE.llms,
+	handle: async (context) => {
+		if (!isRead(context)) return undefined;
+		const { pathname } = context.url;
+
+		if (pathname === "/llms.txt") {
+			const result = await callRoute<{ body: string }>(context, "discovery/public/llms");
+			if (!result?.body) return undefined;
+			return respond(context, result.body, {
+				"Content-Type": "text/plain; charset=utf-8",
+				"Cache-Control": "public, max-age=3600",
+				"X-Content-Type-Options": "nosniff",
+			});
+		}
+
+		const pagePath = pagePathFromMarkdownPath(pathname);
+		if (pagePath) {
+			const doc = await callRoute<MarkdownDoc>(context, "discovery/public/markdown", { path: pagePath });
+			if (!doc?.body) return undefined;
+			return respond(context, doc.body, {
+				"Content-Type": "text/markdown; charset=utf-8",
+				"Cache-Control": "public, max-age=3600",
+				"X-Markdown-Tokens": String(doc.tokens),
+				"X-Content-Type-Options": "nosniff",
+			});
+		}
+
+		return undefined;
+	},
+};
+
+export const discoveryMiddleware: PackMiddleware[] = [indexNowKeyMiddleware, newsSitemapMiddleware, feedMiddleware, llmsMiddleware];