@coywolf/emdash 0.0.0-stage → 0.35.13

package/src/discovery/feed-build.ts

@@ -0,0 +1,217 @@
+/**
+ * Builds the RSS feeds (inside the plugin context): the site feed at /feed/
+ * and term feeds at <term archive>/feed/ (see ./feed.ts). Entries come from
+ * EmDash's content API; a term feed's entries are found with one D1 query on
+ * EmDash's taxonomy tables (the content API can't filter by term). URLs come
+ * from the pack-wide resolver, so they match the sitemaps.
+ */
+import { type CollectionSchemaInfo, type PluginContentItem, type PluginContext, getSiteSetting } from "emdash";
+
+import { COLLECTION_SLUG, ancestorTrail, expandPattern, hostDefaultLocale, primaryTerms, termParentMaps, urlOverride } from "../core/content-url.js";
+import { siteName } from "../core/site.js";
+import { workerEnv } from "../shared.js";
+import { entryTitle, pickCollections } from "./build.js";
+import { FEED_PATH, FEED_TAG, type FeedItem, type TermFeedRoute, buildRss, feedDate, parseFeedPath, termFeedRoute, termSegments, withDescendants } from "./feed.js";
+import type { DiscoverySettings } from "./settings.js";
+import { createUrlResolver, siteOrigin } from "./urls.js";
+
+export interface FeedDocument {
+	body: string;
+	count: number;
+	/** Edge cache tags: EmDash clears a collection's tag (its slug) on every live-content write, and a taxonomy's on term edits. */
+	tags: string[];
+}
+
+const str = (value: unknown) => (typeof value === "string" ? value : "");
+
+async function db(binding: string): Promise<D1Database | null> {
+	try {
+		return ((await workerEnv())[binding] as D1Database | undefined) ?? null;
+	} catch {
+		return null;
+	}
+}
+
+interface Term {
+	slug: string;
+	label: string;
+	/** Archive path, e.g. /news/seo/ */
+	path: string;
+	taxonomy: string;
+}
+
+/** The collection and term a term feed path asks for, or null. */
+async function findTerm(d1: D1Database, collections: CollectionSchemaInfo[], segments: string[]): Promise<{ collection: CollectionSchemaInfo; route: TermFeedRoute; term: Term; groups: string[] } | null> {
+	for (const collection of collections) {
+		const override = urlOverride(collection.slug);
+		const route = termFeedRoute(override ? expandPattern(override) : null);
+		if (!route) continue;
+		const path = termSegments(segments, route);
+		if (!path) continue;
+		const slug = path[path.length - 1];
+		const parents = (await termParentMaps(d1, [route.taxonomy])).get(route.taxonomy) ?? new Map<string, string>();
+		// {termpath}: the URL must be the term's full trail (as its entries' URLs are); {term}: just the slug.
+		if (route.hierarchical && ancestorTrail(slug, parents).join("/") !== path.join("/")) continue;
+		const slugs = withDescendants(slug, parents);
+		const defaultLocale = await hostDefaultLocale();
+		const { results } = await d1
+			.prepare(`SELECT slug, label, locale, translation_group FROM taxonomies WHERE name = ? AND slug IN (${slugs.map(() => "?").join(",")})`)
+			.bind(route.taxonomy, ...slugs)
+			.all<{ slug: string; label: string | null; locale: string | null; translation_group: string | null }>();
+		const own = results.filter((r) => r.slug === slug);
+		if (!own.length) continue;
+		const label = (own.find((r) => r.locale === defaultLocale) ?? own[0]).label || slug;
+		const groups = [...new Set(results.map((r) => r.translation_group).filter((g): g is string => Boolean(g)))];
+		return { collection, route, groups, term: { slug, label, taxonomy: route.taxonomy, path: `${route.prefix}/${path.join("/")}/` } };
+	}
+	return null;
+}
+
+/** Ids of a term's published, indexable entries (default locale), newest first. */
+async function termEntryIds(d1: D1Database, collection: string, groups: string[], limit: number): Promise<string[]> {
+	if (!groups.length || !COLLECTION_SLUG.test(collection)) return [];
+	const { results } = await d1
+		.prepare(
+			`SELECT content.id AS id FROM "ec_${collection}" AS content
+			LEFT JOIN _emdash_seo AS seo ON seo.collection = ? AND seo.content_id = content.id
+			WHERE content.status = 'published' AND content.deleted_at IS NULL AND content.published_at IS NOT NULL
+				AND coalesce(seo.seo_no_index, 0) = 0
+				AND (content.locale IS NULL OR content.locale = ?)
+				AND content.translation_group IN (SELECT entry_id FROM content_taxonomies WHERE collection = ? AND taxonomy_id IN (${groups.map(() => "?").join(",")}))
+			ORDER BY content.published_at DESC LIMIT ?`,
+		)
+		.bind(collection, await hostDefaultLocale(), collection, ...groups, limit)
+		.all<{ id: string }>();
+	return results.map((r) => r.id);
+}
+
+/** The newest `limit` published, indexable entries (default locale) of a collection: a page of 100 at a time (the content API's most). */
+async function newest(ctx: PluginContext, collection: string, limit: number): Promise<PluginContentItem[]> {
+	if (!ctx.content) return [];
+	const locale = await hostDefaultLocale();
+	const out: PluginContentItem[] = [];
+	let cursor: string | undefined;
+	for (let pages = 0; pages < 10 && out.length < limit; pages++) {
+		const page = await ctx.content.list(collection, { limit: 100, cursor, orderBy: { publishedAt: "desc" }, where: { status: "published" } });
+		for (const item of page.items) {
+			if (item.publishedAt && !item.seo?.noIndex && (!item.locale || item.locale === locale)) out.push(item);
+			if (out.length >= limit) break;
+		}
+		cursor = page.hasMore ? page.cursor : undefined;
+		if (!cursor) break;
+	}
+	return out;
+}
+
+/** Labels of terms by slug (default locale first). */
+async function termLabels(d1: D1Database | null, taxonomy: string, slugs: string[]): Promise<Map<string, string>> {
+	const out = new Map<string, string>();
+	if (!d1 || !slugs.length) return out;
+	try {
+		const defaultLocale = await hostDefaultLocale();
+		const { results } = await d1
+			.prepare(`SELECT slug, label, locale FROM taxonomies WHERE name = ? AND slug IN (${slugs.map(() => "?").join(",")})`)
+			.bind(taxonomy, ...slugs)
+			.all<{ slug: string; label: string | null; locale: string | null }>();
+		for (const r of results) if (r.label && (!out.has(r.slug) || r.locale === defaultLocale)) out.set(r.slug, r.label);
+	} catch {
+		// No taxonomy table: items go out without a category.
+	}
+	return out;
+}
+
+/** The site logo (or favicon) as the channel image, when it's a GIF, JPEG or PNG (all RSS allows). */
+async function channelImage(origin: string): Promise<string | null> {
+	for (const key of ["logo", "favicon"] as const) {
+		const ref = (await getSiteSetting(key).catch(() => undefined)) as { url?: string; contentType?: string } | undefined;
+		if (ref?.url && /^image\/(png|jpeg|gif)$/.test(ref.contentType ?? "")) return new URL(ref.url, `${origin}/`).href;
+	}
+	return null;
+}
+
+/** The feed for a path (/feed/ or <term archive>/feed/), or null when there's none at that path. */
+export async function buildFeed(ctx: PluginContext, settings: DiscoverySettings, pathname: string, database = "DB"): Promise<FeedDocument | null> {
+	const segments = parseFeedPath(pathname);
+	if (!segments) return null;
+	const { collections: chosen, items: limit } = settings.feed;
+	// An empty selection lists nothing (like the news sitemap).
+	const collections = chosen.length ? await pickCollections(ctx, chosen) : [];
+	if (!collections.length) return null;
+	const origin = siteOrigin(ctx);
+	const d1 = await db(database);
+
+	let entries: Array<{ collection: CollectionSchemaInfo; item: PluginContentItem }> = [];
+	let term: Term | null = null;
+	if (segments.length === 0) {
+		for (const collection of collections) for (const item of await newest(ctx, collection.slug, limit)) entries.push({ collection, item });
+	} else {
+		if (!settings.feed.termFeeds || !d1) return null;
+		const found = await findTerm(d1, collections, segments);
+		if (!found) return null;
+		term = found.term;
+		const ids = await termEntryIds(d1, found.collection.slug, found.groups, limit);
+		const items = await Promise.all(ids.map((id) => ctx.content?.get(found.collection.slug, id) ?? null));
+		entries = items.filter((item): item is PluginContentItem => Boolean(item && item.status === "published" && item.publishedAt)).map((item) => ({ collection: found.collection, item }));
+	}
+
+	entries = entries
+		.filter(({ item }) => !item.seo?.noIndex)
+		.sort((a, b) => (feedDate(b.item.publishedAt)?.getTime() ?? 0) - (feedDate(a.item.publishedAt)?.getTime() ?? 0));
+
+	const resolver = createUrlResolver(ctx);
+	const items: Array<FeedItem & { changed: string }> = [];
+	for (const collection of collections) {
+		const own = entries.filter((e) => e.collection.slug === collection.slug).map((e) => e.item);
+		if (!own.length) continue;
+		const urls = await resolver.urls(collection, own);
+		const taxonomy = term?.taxonomy ?? termFeedRoute(urlOverride(collection.slug) ? expandPattern(urlOverride(collection.slug)!) : null)?.taxonomy ?? "category";
+		const primary = await primaryTerms(ctx, collection.slug, own.map((i) => i.id), [taxonomy]).catch(() => new Map<string, Record<string, string>>());
+		const labels = await termLabels(d1, taxonomy, [...new Set([...primary.values()].map((t) => t[taxonomy]).filter(Boolean))]);
+		const bylines = new Map<string, string>();
+		if (ctx.bylines) {
+			for (let i = 0; i < own.length; i += 100) {
+				const credits = await ctx.bylines.getEntriesBylines(collection.slug, own.slice(i, i + 100).map((x) => x.id)).catch(() => []);
+				for (const c of credits) {
+					const name = c.bylines[0]?.byline.displayName;
+					if (name) bylines.set(c.entryId, name);
+				}
+			}
+		}
+		for (const item of own) {
+			const url = urls.get(item.id);
+			if (!url || !item.publishedAt) continue;
+			const slug = primary.get(item.id)?.[taxonomy];
+			items.push({
+				title: entryTitle(collection, item),
+				url,
+				publishedAt: item.publishedAt,
+				creator: bylines.get(item.id) ?? null,
+				category: slug ? (labels.get(slug) ?? null) : null,
+				description: str(item.data.excerpt).trim() || str(item.seo?.description).trim(),
+				changed: item.updatedAt || item.publishedAt,
+			});
+		}
+	}
+	items.sort((a, b) => (feedDate(b.publishedAt)?.getTime() ?? 0) - (feedDate(a.publishedAt)?.getTime() ?? 0));
+	const listed = items.slice(0, limit);
+	const newestChange = listed
+		.map((i) => feedDate(i.changed))
+		.filter((d): d is Date => Boolean(d))
+		.sort((a, b) => b.getTime() - a.getTime())[0];
+
+	const name = (await siteName(ctx)) || new URL(origin).host;
+	const tagline = str(await getSiteSetting("tagline").catch(() => undefined)).trim();
+	const image = await channelImage(origin);
+	const body = buildRss({
+		title: term ? `${term.label} Archives - ${name}` : name,
+		link: `${origin}${term ? term.path : "/"}`,
+		selfUrl: `${origin}${term ? `${term.path}feed/` : FEED_PATH}`,
+		description: tagline,
+		language: ctx.site.locale,
+		lastBuildDate: (newestChange ?? feedDate(listed[0]?.publishedAt))?.toISOString() ?? null,
+		image: image ? { url: image, title: name, link: `${origin}/` } : null,
+		items: listed.map(({ changed: _, ...item }) => item),
+	});
+	const taxonomies = new Set(collections.map((c) => termFeedRoute(urlOverride(c.slug) ? expandPattern(urlOverride(c.slug)!) : null)?.taxonomy ?? "category"));
+	return { body, count: listed.length, tags: [FEED_TAG, ...collections.map((c) => c.slug), ...[...taxonomies].map((t) => `emdash:taxonomy:${t}`)] };
+}

package/src/discovery/feed.ts

@@ -0,0 +1,172 @@
+/**
+ * RSS feed (pure part): feed URLs and the RSS 2.0 document. No imports, so
+ * `node --test` can load it.
+ *
+ * - `/feed/` lists the newest entries of the chosen collections, at
+ *   WordPress's feed URL.
+ * - `<term archive>/feed/` lists one term's entries (and its child terms'),
+ *   WordPress's category feeds. Only for a collection whose URL override is
+ *   `<prefix>/{term:<taxonomy>|…}/{slug}/` or `{termpath:…}` (e.g.
+ *   `/{termpath:category|uncategorized}/{slug}/` → `/news/seo/feed/`), since
+ *   the term's archive is then the entry URL without the slug.
+ */
+
+export const FEED_PATH = "/feed/";
+/** Edge cache tag on every feed, cleared when the feed settings are saved. */
+export const FEED_TAG = "coywolf:feed";
+/** How long the edge keeps a feed when nothing clears it sooner (changes that clear no tag, such as a byline edit). */
+export const FEED_EDGE_MAX_AGE = 3600;
+/** Items per feed: the default, and the most allowed. */
+export const FEED_DEFAULT_ITEMS = 20;
+export const FEED_MAX_ITEMS = 100;
+/** Deepest term path followed in a feed URL. */
+const MAX_SEGMENTS = 8;
+/** A term slug: letters (any script), digits, "-" and "_", not starting with "_" or "-" (so never /_emdash/…). */
+const SEGMENT = /^[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*$/u;
+
+/**
+ * The term path a feed URL asks for: [] for the site feed (`/feed/`), the
+ * segments before `/feed/` otherwise, null when the path isn't a feed URL.
+ * `/feed` without the slash isn't one: the site redirects it to `/feed/`, as
+ * it does for its other pages.
+ */
+export function parseFeedPath(pathname: string): string[] | null {
+	const match = /^((?:\/[^/]+)*)\/feed\/$/.exec(pathname);
+	if (!match) return null;
+	const segments = match[1].split("/").filter(Boolean);
+	if (segments.length > MAX_SEGMENTS) return null;
+	let decoded: string[];
+	try {
+		decoded = segments.map((s) => decodeURIComponent(s));
+	} catch {
+		return null;
+	}
+	return decoded.every((s) => SEGMENT.test(s)) ? decoded : null;
+}
+
+export interface TermFeedRoute {
+	taxonomy: string;
+	/** {termpath:…}: the URL has the term's ancestors in front. */
+	hierarchical: boolean;
+	/** Static path segments before the term ("" or e.g. "/blog"). */
+	prefix: string;
+}
+
+/**
+ * Where a collection's term feeds live, from its URL override (after
+ * expandPattern, so `{category}` is already `{term:category|uncategorized}`).
+ * Null when the pattern isn't "static prefix, term, slug".
+ */
+export function termFeedRoute(pattern: string | null): TermFeedRoute | null {
+	if (!pattern) return null;
+	const match = /^((?:\/[A-Za-z0-9_-]+)*)\/\{(term|termpath):([A-Za-z0-9_-]+)(?:\|[^{}/]*)?\}\/\{slug\}\/?$/.exec(pattern);
+	return match ? { prefix: match[1], hierarchical: match[2] === "termpath", taxonomy: match[3] } : null;
+}
+
+/** The term path in a feed URL's segments, once the route's prefix is removed (null: not under the prefix). */
+export function termSegments(segments: string[], route: TermFeedRoute): string[] | null {
+	const prefix = route.prefix.split("/").filter(Boolean);
+	if (segments.length <= prefix.length || prefix.some((p, i) => p !== segments[i])) return null;
+	const rest = segments.slice(prefix.length);
+	return route.hierarchical || rest.length === 1 ? rest : null;
+}
+
+/** A term and all its descendants, from child → parent slugs. */
+export function withDescendants(slug: string, parents: ReadonlyMap<string, string>): string[] {
+	const children = new Map<string, string[]>();
+	for (const [child, parent] of parents) children.set(parent, [...(children.get(parent) ?? []), child]);
+	const out = [slug];
+	for (let i = 0; i < out.length && out.length < 5000; i++) {
+		for (const child of children.get(out[i]) ?? []) if (!out.includes(child)) out.push(child);
+	}
+	return out;
+}
+
+export interface FeedItem {
+	title: string;
+	url: string;
+	/** ISO 8601, or SQLite's "YYYY-MM-DD HH:MM:SS" in UTC. */
+	publishedAt: string;
+	creator?: string | null;
+	category?: string | null;
+	description?: string | null;
+}
+
+export interface FeedChannel {
+	title: string;
+	/** The page the feed is for (the home page, or the term's archive). */
+	link: string;
+	/** The feed's own URL (atom:link rel="self"). */
+	selfUrl: string;
+	description: string;
+	language: string;
+	/** Newest change among the items (ISO); now when there are none. */
+	lastBuildDate?: string | null;
+	image?: { url: string; title: string; link: string } | null;
+	items: FeedItem[];
+}
+
+const TZ_SUFFIX = /([zZ]|[+-]\d{2}(:?\d{2})?)$/;
+
+/** Parse a stored timestamp, treating offset-less values as UTC (as EmDash does). */
+export function feedDate(value: string | null | undefined): Date | null {
+	if (!value) return null;
+	let normalized = value.trim();
+	if (normalized.includes(" ") && !normalized.includes("T")) normalized = normalized.replace(" ", "T");
+	if (!TZ_SUFFIX.test(normalized)) normalized += "Z";
+	const time = Date.parse(normalized);
+	return Number.isNaN(time) ? null : new Date(time);
+}
+
+export function escapeXml(text: string): string {
+	return text
+		.replace(/&/g, "&amp;")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&apos;")
+		// Characters XML 1.0 forbids outright.
+		// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping them is the point.
+		.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\uFFFE\uFFFF]/g, "");
+}
+
+/** RSS's <language>: the site locale as a language tag ("en-US"), "en" when unset or malformed. */
+export function feedLanguage(locale: string | null | undefined): string {
+	const value = (locale ?? "").trim().replace(/_/g, "-");
+	return /^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/.test(value) ? value : "en";
+}
+
+/** RSS 2.0 with atom:link rel="self" and dc:creator (as WordPress's feeds have). */
+export function buildRss(channel: FeedChannel, now: Date = new Date()): string {
+	const lines = [
+		'<?xml version="1.0" encoding="UTF-8"?>',
+		'<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/">',
+		"\t<channel>",
+		`\t\t<title>${escapeXml(channel.title)}</title>`,
+		`\t\t<atom:link href="${escapeXml(channel.selfUrl)}" rel="self" type="application/rss+xml" />`,
+		`\t\t<link>${escapeXml(channel.link)}</link>`,
+		`\t\t<description>${escapeXml(channel.description)}</description>`,
+		`\t\t<lastBuildDate>${(feedDate(channel.lastBuildDate) ?? now).toUTCString()}</lastBuildDate>`,
+		`\t\t<language>${escapeXml(feedLanguage(channel.language))}</language>`,
+	];
+	if (channel.image) {
+		lines.push(
+			"\t\t<image>",
+			`\t\t\t<url>${escapeXml(channel.image.url)}</url>`,
+			`\t\t\t<title>${escapeXml(channel.image.title)}</title>`,
+			`\t\t\t<link>${escapeXml(channel.image.link)}</link>`,
+			"\t\t</image>",
+		);
+	}
+	for (const item of channel.items) {
+		const published = feedDate(item.publishedAt);
+		if (!published) continue;
+		lines.push("\t\t<item>", `\t\t\t<title>${escapeXml(item.title)}</title>`, `\t\t\t<link>${escapeXml(item.url)}</link>`);
+		if (item.creator) lines.push(`\t\t\t<dc:creator>${escapeXml(item.creator)}</dc:creator>`);
+		lines.push(`\t\t\t<pubDate>${published.toUTCString()}</pubDate>`);
+		if (item.category) lines.push(`\t\t\t<category>${escapeXml(item.category)}</category>`);
+		lines.push(`\t\t\t<guid isPermaLink="true">${escapeXml(item.url)}</guid>`, `\t\t\t<description>${escapeXml(item.description ?? "")}</description>`, "\t\t</item>");
+	}
+	lines.push("\t</channel>", "</rss>", "");
+	return lines.join("\n");
+}

package/src/discovery/indexnow.ts

@@ -0,0 +1,114 @@
+/**
+ * IndexNow (pure part): key handling, the submission payload, and a batcher
+ * that collects changed URLs for a short window, dedupes them, and submits
+ * them in one request.
+ *
+ * Coywolf SEO pinged Bing once per URL; here several changes in quick
+ * succession (a save followed by a publish, a bulk edit) go out as one POST,
+ * which every IndexNow endpoint accepts (up to 10,000 URLs).
+ */
+
+export const INDEXNOW_ENDPOINTS = {
+	"api.indexnow.org": "https://api.indexnow.org/indexnow",
+	"www.bing.com": "https://www.bing.com/indexnow",
+} as const;
+export type IndexNowEndpoint = keyof typeof INDEXNOW_ENDPOINTS;
+
+/** IndexNow keys: 8–128 characters of a–z, A–Z, 0–9 and "-". */
+export const KEY_PATTERN = /^[a-zA-Z0-9-]{8,128}$/;
+export const MAX_URLS_PER_REQUEST = 10_000;
+
+/** A new 32-character lowercase hex key. */
+export function generateKey(): string {
+	const bytes = new Uint8Array(16);
+	crypto.getRandomValues(bytes);
+	return [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+/** The key a `/<key>.txt` request asks for, or null when the path isn't key-shaped. */
+export function keyFromPath(pathname: string): string | null {
+	const match = /^\/([a-zA-Z0-9-]{8,128})\.txt$/.exec(pathname);
+	return match ? match[1] : null;
+}
+
+export interface IndexNowPayload {
+	host: string;
+	key: string;
+	keyLocation: string;
+	urlList: string[];
+}
+
+/**
+ * The POST body for a batch. URLs on other hosts are dropped: IndexNow
+ * rejects a batch whose URLs don't all belong to the key's host.
+ */
+export function buildPayload(siteUrl: string, key: string, urls: string[]): IndexNowPayload | null {
+	const site = new URL(siteUrl);
+	const urlList = [...new Set(urls)].filter((url) => {
+		try {
+			return new URL(url).host === site.host;
+		} catch {
+			return false;
+		}
+	});
+	if (!urlList.length) return null;
+	return {
+		host: site.host,
+		key,
+		keyLocation: `${site.origin}/${key}.txt`,
+		urlList: urlList.slice(0, MAX_URLS_PER_REQUEST),
+	};
+}
+
+export interface BatcherOptions {
+	/** How long to wait for more URLs before submitting. */
+	windowMs: number;
+	/** Keep work alive past the response (EmDash's `after()` / waitUntil). */
+	defer: (task: () => Promise<void>) => void;
+	sleep?: (ms: number) => Promise<void>;
+}
+
+export type BatchSender = (urls: string[]) => Promise<void>;
+
+/**
+ * Collects URLs for `windowMs`, then hands the deduped list to the most
+ * recent sender. One pending batch per isolate.
+ */
+export class IndexNowBatcher {
+	private pending = new Set<string>();
+	private sender: BatchSender | null = null;
+	private scheduled = false;
+	private readonly sleep: (ms: number) => Promise<void>;
+	private readonly options: BatcherOptions;
+
+	constructor(options: BatcherOptions) {
+		this.options = options;
+		this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+	}
+
+	get size(): number {
+		return this.pending.size;
+	}
+
+	add(urls: string | string[], sender: BatchSender): void {
+		for (const url of Array.isArray(urls) ? urls : [urls]) if (url) this.pending.add(url);
+		if (!this.pending.size) return;
+		this.sender = sender;
+		if (this.scheduled) return;
+		this.scheduled = true;
+		this.options.defer(async () => {
+			await this.sleep(this.options.windowMs);
+			await this.flush();
+		});
+	}
+
+	/** Submit what's pending now. */
+	async flush(): Promise<void> {
+		this.scheduled = false;
+		const urls = [...this.pending];
+		const sender = this.sender;
+		this.pending.clear();
+		this.sender = null;
+		if (urls.length && sender) await sender(urls);
+	}
+}

package/src/discovery/llms.ts

@@ -0,0 +1,79 @@
+/**
+ * llms.txt (pure part), per https://llmstxt.org/ and Coywolf SEO's layout:
+ * an H1 with the site name, a blockquote summary, a short detail paragraph,
+ * one H2 "file list" per collection linking each entry (to its Markdown
+ * source when per-page Markdown is on), and an "## Optional" section for the
+ * long tail.
+ */
+
+/** Links per collection in the main lists; the rest move to "## Optional". */
+export const MAIN_LIMIT = 100;
+export const DEFAULT_MAX_ENTRIES = 500;
+
+export interface LlmsEntry {
+	title: string;
+	url: string;
+	note?: string | null;
+}
+
+export interface LlmsSection {
+	label: string;
+	entries: LlmsEntry[];
+}
+
+export interface LlmsInput {
+	/** Site name (falls back to the site URL). */
+	name: string;
+	siteUrl: string;
+	/** Blockquote summary: the site tagline, or a default. */
+	summary?: string | null;
+	/** Detail paragraph under the summary. */
+	intro?: string | null;
+	/** Whether links point at Markdown sources (changes the default intro). */
+	markdownLinks: boolean;
+	sections: LlmsSection[];
+	/** Total links across the file. Default 500. */
+	maxEntries?: number;
+}
+
+export const DEFAULT_SUMMARY = "A curated, agent-readable index of this site's public content.";
+export const DEFAULT_INTRO_MARKDOWN = "Each link below points at the Markdown source of a page, so an agent can read the content directly.";
+export const DEFAULT_INTRO_HTML = "Each link below points at a page on this site.";
+
+/** Plain text for llms.txt: one line, no Markdown link breakers. */
+export function llmsText(text: string): string {
+	return text.replace(/\s+/g, " ").trim().replace(/([[\]])/g, "\\$1");
+}
+
+function linkLine(entry: LlmsEntry): string {
+	const url = entry.url.replace(/[()\s]/g, (c) => encodeURIComponent(c));
+	const note = entry.note ? llmsText(entry.note) : "";
+	return `- [${llmsText(entry.title) || url}](${url})${note ? `: ${note}` : ""}\n`;
+}
+
+export function buildLlmsTxt(input: LlmsInput): string {
+	const max = Math.max(1, input.maxEntries ?? DEFAULT_MAX_ENTRIES);
+	const name = llmsText(input.name) || input.siteUrl;
+	const summary = input.summary?.trim() ? input.summary.replace(/\s+/g, " ").trim() : DEFAULT_SUMMARY;
+	const intro = input.intro?.trim() ? input.intro.trim() : input.markdownLinks ? DEFAULT_INTRO_MARKDOWN : DEFAULT_INTRO_HTML;
+
+	let out = `# ${name}\n\n> ${summary}\n\n${intro}\n`;
+	let used = 0;
+	const optional: LlmsEntry[] = [];
+
+	for (const section of input.sections) {
+		if (!section.entries.length || used >= max) continue;
+		const main = section.entries.slice(0, Math.min(MAIN_LIMIT, max - used));
+		used += main.length;
+		out += `\n## ${llmsText(section.label)}\n\n`;
+		for (const entry of main) out += linkLine(entry);
+		optional.push(...section.entries.slice(main.length));
+	}
+
+	const rest = optional.slice(0, Math.max(0, max - used));
+	if (rest.length) {
+		out += "\n## Optional\n\n";
+		for (const entry of rest) out += linkLine(entry);
+	}
+	return out;
+}