@coywolf/emdash 0.0.0-stage → 0.35.14

package/src/discovery/module.ts

@@ -0,0 +1,353 @@
+/**
+ * Discovery module: routes and hooks for IndexNow, the news sitemap,
+ * llms.txt and per-entry Markdown. Site URLs are answered by the middleware
+ * in ./middleware.ts, which calls the public routes here.
+ */
+import { type PluginContext, PluginRouteError, after, definePluginRoute } from "emdash";
+import { z } from "zod";
+
+import { absoluteUrl, entryUrl } from "../core/content-url.js";
+import { siteInStaging, siteName } from "../core/site.js";
+import { cachedCtxFeatures, ctxFeatures, invalidateFeatures, isOn, readSiteSetting, registerSiteSetting, requireFeature } from "../core/features.js";
+import { parseInput } from "../shared.js";
+import { LLMS_CACHE, NEWS_CACHE, type CachedDocument, type DocKind, buildMarkdown, readDoc, rebuildNow, scheduleRebuild } from "./build.js";
+import { INDEXNOW_ENDPOINTS, IndexNowBatcher, buildPayload, generateKey } from "./indexnow.js";
+import { buildFeed } from "./feed-build.js";
+import { FEED_PATH, FEED_TAG } from "./feed.js";
+import { purgePageCache } from "../pageCache/lib.js";
+import { markdownUrl } from "./markdown.js";
+import {
+	INDEXNOW_KEY_SETTING,
+	SETTINGS_KEY,
+	type DiscoverySettings,
+	ensureIndexNowKey,
+	loadSettings,
+	readIndexNowKey,
+	selected,
+	settingsSchema,
+} from "./settings.js";
+import { siteOrigin } from "./urls.js";
+
+export const FEATURE = {
+	main: "discovery",
+	indexnow: "discovery.indexnow",
+	news: "discovery.newsSitemap",
+	llms: "discovery.llms",
+	feed: "discovery.feed",
+} as const;
+
+export interface DiscoveryOptions {
+	/** D1 binding of the site database (term feeds read EmDash's taxonomy tables). Default "DB". */
+	database?: string;
+}
+
+// ── IndexNow ─────────────────────────────────────────────────────
+
+const LOG_KEY = "state:discovery:indexnowLog";
+const LOG_LIMIT = 25;
+const urlKey = (collection: string, id: string) => `state:discovery:url:${collection}:${id}`;
+
+export interface PingLogEntry {
+	at: string;
+	endpoint: string;
+	count: number;
+	urls: string[];
+	status: number;
+	ok: boolean;
+	error?: string;
+	trigger: "content" | "manual";
+}
+
+/** Submissions wait this long for more changes, then go out as one request. */
+const BATCH_WINDOW_MS = 3000;
+const batcher = new IndexNowBatcher({ windowMs: BATCH_WINDOW_MS, defer: (task) => after(task) });
+
+async function submit(ctx: PluginContext, urls: string[], trigger: PingLogEntry["trigger"]): Promise<PingLogEntry | null> {
+	const key = await ensureIndexNowKey(ctx);
+	const settings = await loadSettings(ctx);
+	const payload = buildPayload(siteOrigin(ctx), key, urls);
+	if (!payload) return null;
+	const endpoint = INDEXNOW_ENDPOINTS[settings.indexnow.endpoint];
+	const entry: PingLogEntry = {
+		at: new Date().toISOString(),
+		endpoint: settings.indexnow.endpoint,
+		count: payload.urlList.length,
+		urls: payload.urlList.slice(0, 10),
+		status: 0,
+		ok: false,
+		trigger,
+	};
+	try {
+		if (!ctx.http) throw new Error("network:request capability missing");
+		const response = await ctx.http.fetch(endpoint, {
+			method: "POST",
+			headers: { "Content-Type": "application/json; charset=utf-8" },
+			body: JSON.stringify(payload),
+		});
+		entry.status = response.status;
+		entry.ok = response.ok;
+		if (!response.ok) entry.error = (await response.text().catch(() => "")).slice(0, 300) || response.statusText;
+	} catch (error) {
+		entry.error = String(error).slice(0, 300);
+	}
+	if (entry.ok) ctx.log.info("IndexNow submitted", { count: entry.count, status: entry.status });
+	else ctx.log.warn("IndexNow submission failed", { count: entry.count, status: entry.status, error: entry.error });
+	const log = (await ctx.kv.get<PingLogEntry[]>(LOG_KEY)) ?? [];
+	await ctx.kv.set(LOG_KEY, [entry, ...log].slice(0, LOG_LIMIT));
+	return entry;
+}
+
+type ChangeKind = "publish" | "update" | "unpublish" | "delete";
+
+async function queueIndexNow(ctx: PluginContext, settings: DiscoverySettings, collection: string, id: string, kind: ChangeKind, permanent = false) {
+	if (!selected(settings.indexnow.collections, collection)) return;
+	const known = await ctx.kv.get<string>(urlKey(collection, id));
+	const urls: string[] = [];
+
+	if (kind === "publish" || kind === "update") {
+		const item = await ctx.content?.get(collection, id);
+		const path = item && !item.seo?.noIndex ? await entryUrl(ctx, collection, item) : null;
+		const url = path ? absoluteUrl(path, siteOrigin(ctx)) : null;
+		if (url) urls.push(url);
+		// A changed slug: tell engines about the old URL too.
+		if (known && known !== url) urls.push(known);
+		if (url && url !== known) await ctx.kv.set(urlKey(collection, id), url);
+		if (!url && known) await ctx.kv.delete(urlKey(collection, id));
+	} else {
+		// Unpublished or deleted: the entry has no public URL any more, so use the URL remembered at publish.
+		if (known) urls.push(known);
+		if (known && (kind === "unpublish" || permanent)) await ctx.kv.delete(urlKey(collection, id));
+	}
+	if (urls.length) batcher.add(urls, (batch) => submit(ctx, batch, "content").then(() => undefined));
+}
+
+async function onContentChange(ctx: PluginContext, collection: string, id: string | undefined, kind: ChangeKind, permanent = false) {
+	const features = await ctxFeatures(ctx);
+	const llms = isOn(features, FEATURE.llms);
+	const news = isOn(features, FEATURE.news);
+	const kinds: DocKind[] = [...(llms ? (["llms"] as const) : []), ...(news ? (["news"] as const) : [])];
+	if (kinds.length) scheduleRebuild(ctx, kinds);
+	if (id && isOn(features, FEATURE.indexnow)) {
+		// A site in EmDash's staging mode is hidden from search engines: don't tell them about its pages.
+		if (await siteInStaging()) {
+			ctx.log.info("IndexNow: skipped, the site is in staging", { collection, id, kind });
+			return;
+		}
+		await queueIndexNow(ctx, await loadSettings(ctx), collection, id, kind, permanent);
+	}
+}
+
+const idOf = (content: Record<string, unknown>) => (typeof content.id === "string" ? content.id : undefined);
+
+// ── Head link to the Markdown source ─────────────────────────────
+
+// Read on every page render: it comes with the feature switches' query (no query of its own).
+registerSiteSetting(SETTINGS_KEY);
+/** The last stored value and its parsed settings, so a cache hit doesn't parse again. */
+let settingsMemo: { raw: unknown; settings: DiscoverySettings } | null = null;
+async function memoSettings(ctx: PluginContext): Promise<DiscoverySettings> {
+	const read = await readSiteSetting(SETTINGS_KEY).catch(() => null);
+	if (!read) return loadSettings(ctx); // D1 unreadable here: through the plugin context.
+	if (settingsMemo && settingsMemo.raw === read.value) return settingsMemo.settings;
+	const parsed = settingsSchema.safeParse(read.value ?? {});
+	settingsMemo = { raw: read.value, settings: parsed.success ? parsed.data : settingsSchema.parse({}) };
+	return settingsMemo.settings;
+}
+
+const escapeAttr = (s: string) => s.replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+
+// ── Module ───────────────────────────────────────────────────────
+
+export function discoveryModule(options: DiscoveryOptions) {
+	const hooks = {
+		"content:afterPublish": async (event: { content: Record<string, unknown>; collection: string }, ctx: PluginContext) =>
+			onContentChange(ctx, event.collection, idOf(event.content), "publish"),
+		"content:afterUnpublish": async (event: { content: Record<string, unknown>; collection: string }, ctx: PluginContext) =>
+			onContentChange(ctx, event.collection, idOf(event.content), "unpublish"),
+		/** Edits that change the live page directly (collections without draft revisions). */
+		"content:afterSave": async (event: { content: Record<string, unknown>; collection: string; isNew: boolean }, ctx: PluginContext) => {
+			if (event.content.status !== "published" || event.content.draftRevisionId) return;
+			await onContentChange(ctx, event.collection, idOf(event.content), event.isNew ? "publish" : "update");
+		},
+		"content:afterDelete": async (event: { id: string; collection: string; permanent: boolean }, ctx: PluginContext) =>
+			onContentChange(ctx, event.collection, event.id, "delete", event.permanent),
+		/**
+		 * Head links (page:metadata links can't carry a type): the RSS feed on every page, so feed
+		 * readers and apps find it, and the entry's Markdown source on content pages.
+		 */
+		"page:fragments": async (event: { page: { kind: string; url: string; canonical: string | null; title: string | null; pageTitle?: string | null; content?: { collection: string }; seo?: { robots?: string | null } } }, ctx: PluginContext) => {
+			const { page } = event;
+			// The same cached read the hook composer made for this render (no query of its own).
+			const features = await cachedCtxFeatures(ctx);
+			const out: Array<{ kind: "html"; placement: "head"; key: string; html: string }> = [];
+			if (isOn(features, FEATURE.feed)) {
+				try {
+					const href = new URL(FEED_PATH, page.url).href;
+					const name = (await siteName(ctx)) || new URL(page.url).host;
+					out.push({ kind: "html", placement: "head", key: "discovery-feed", html: `<link rel="alternate" type="application/rss+xml" href="${escapeAttr(href)}" title="${escapeAttr(`${name} RSS feed`)}">` });
+				} catch {
+					// No usable page URL: no link.
+				}
+			}
+			if (isOn(features, FEATURE.llms) && page.kind === "content" && page.content && !/noindex/i.test(page.seo?.robots ?? "")) {
+				const settings = await memoSettings(ctx);
+				if (settings.llms.markdown && selected(settings.llms.collections, page.content.collection)) {
+					try {
+						const href = markdownUrl(new URL(page.canonical || page.url, page.url).href);
+						const title = `${page.pageTitle || page.title || "This page"} — Markdown source`;
+						out.push({ kind: "html", placement: "head", key: "discovery-markdown", html: `<link rel="alternate" type="text/markdown" href="${escapeAttr(href)}" title="${escapeAttr(title)}">` });
+					} catch {
+						// No usable page URL: no link.
+					}
+				}
+			}
+			return out;
+		},
+	};
+
+	const routes = {
+		// ── Public (called by the site middleware) ──
+		"discovery/public/key": {
+			public: true,
+			handler: async (ctx: PluginContext) => {
+				await requireFeature(ctx, FEATURE.indexnow);
+				return { key: await readIndexNowKey(ctx) };
+			},
+		},
+		"discovery/public/news": {
+			public: true,
+			handler: async (ctx: PluginContext) => {
+				await requireFeature(ctx, FEATURE.news);
+				return { body: (await readDoc(ctx, "news")).body };
+			},
+		},
+		"discovery/public/feed": {
+			public: true,
+			handler: async (ctx: PluginContext & { input: unknown }) => {
+				await requireFeature(ctx, FEATURE.feed);
+				const { path } = parseInput(z.object({ path: z.string().min(1).max(2000).startsWith("/") }), ctx.input);
+				const doc = await buildFeed(ctx, await loadSettings(ctx), path, options.database);
+				if (!doc) throw PluginRouteError.notFound("No feed at this path.");
+				return doc;
+			},
+		},
+		"discovery/public/llms": {
+			public: true,
+			handler: async (ctx: PluginContext) => {
+				await requireFeature(ctx, FEATURE.llms);
+				return { body: (await readDoc(ctx, "llms")).body };
+			},
+		},
+		"discovery/public/markdown": {
+			public: true,
+			handler: async (ctx: PluginContext & { input: unknown }) => {
+				await requireFeature(ctx, FEATURE.llms);
+				const { path } = parseInput(z.object({ path: z.string().min(1).max(2000).startsWith("/") }), ctx.input);
+				const doc = await buildMarkdown(ctx, await loadSettings(ctx), path);
+				if (!doc) throw PluginRouteError.notFound("No Markdown source for this path.");
+				return doc;
+			},
+		},
+
+		// ── Admin ──
+		"discovery/status": {
+			permission: "plugins:manage" as const,
+			handler: async (ctx: PluginContext) => {
+				const features = await ctxFeatures(ctx);
+				const settings = await loadSettings(ctx);
+				const key = isOn(features, FEATURE.indexnow) ? await ensureIndexNowKey(ctx) : await readIndexNowKey(ctx);
+				const collections = ((await ctx.schema?.listCollections()) ?? [])
+					.filter((c) => c.routable)
+					.map((c) => ({ slug: c.slug, label: c.label, hidden: c.hidden, hasSeo: c.hasSeo }));
+				const origin = siteOrigin(ctx);
+				const doc = async (k: string) => {
+					const d = await ctx.kv.get<CachedDocument>(k);
+					return d ? { count: d.count, builtAt: d.builtAt } : null;
+				};
+				return {
+					staging: await siteInStaging(),
+					features: { indexnow: isOn(features, FEATURE.indexnow), news: isOn(features, FEATURE.news), llms: isOn(features, FEATURE.llms), feed: isOn(features, FEATURE.feed) },
+					settings,
+					collections,
+					key,
+					urls: {
+						keyFile: key ? `${origin}/${key}.txt` : null,
+						news: `${origin}/news-sitemap.xml`,
+						llms: `${origin}/llms.txt`,
+						feed: `${origin}${FEED_PATH}`,
+					},
+					log: (await ctx.kv.get<PingLogEntry[]>(LOG_KEY)) ?? [],
+					cache: { llms: await doc(LLMS_CACHE), news: await doc(NEWS_CACHE) },
+				};
+			},
+		},
+
+		"discovery/save": definePluginRoute({
+			permission: "plugins:manage",
+			methods: ["POST"],
+			request: { body: "json" },
+			handler: async (ctx) => {
+				const settings = parseInput(settingsSchema, ctx.input);
+				await ctx.settings.set(SETTINGS_KEY, settings);
+				settingsMemo = null;
+				// Read with the feature switches: drop that cache too.
+				invalidateFeatures();
+				// Rebuild in the background; the old copies keep serving until then.
+				const features = await ctxFeatures(ctx);
+				const kinds: DocKind[] = [...(isOn(features, FEATURE.llms) ? (["llms"] as const) : []), ...(isOn(features, FEATURE.news) ? (["news"] as const) : [])];
+				if (kinds.length) scheduleRebuild(ctx, kinds, 0);
+				// Feeds in the edge cache were built with the old settings.
+				await purgePageCache({ tags: [FEED_TAG] }).catch((error: unknown) => ctx.log.warn("Discovery: clearing cached feeds failed", { error: String(error) }));
+				return { settings };
+			},
+		}),
+
+		"discovery/regenerate-key": definePluginRoute({
+			permission: "plugins:manage",
+			methods: ["POST"],
+			request: { body: "json" },
+			handler: async (ctx) => {
+				const key = generateKey();
+				await ctx.settings.set(INDEXNOW_KEY_SETTING, key);
+				ctx.log.info("IndexNow key regenerated");
+				return { key };
+			},
+		}),
+
+		/** Submit URLs now (for checking the setup). Defaults to the home page. */
+		"discovery/submit": definePluginRoute({
+			permission: "plugins:manage",
+			methods: ["POST"],
+			request: { body: "json" },
+			handler: async (ctx) => {
+				await requireFeature(ctx, FEATURE.indexnow);
+				if (await siteInStaging()) throw PluginRouteError.badRequest("The site is in staging, hidden from search engines, so nothing is sent to IndexNow. Go live under Settings → General → Site Status first.");
+				const { urls } = parseInput(z.object({ urls: z.array(z.string().url().max(2000)).max(100).optional() }), ctx.input ?? {});
+				await ensureIndexNowKey(ctx);
+				const origin = siteOrigin(ctx);
+				const list = urls?.length ? urls : [`${origin}/`];
+				const foreign = list.filter((u) => new URL(u).host !== new URL(origin).host);
+				if (foreign.length) throw PluginRouteError.badRequest(`Only URLs on ${new URL(origin).host} can be submitted.`);
+				const entry = await submit(ctx, list, "manual");
+				if (!entry) throw PluginRouteError.badRequest("Nothing to submit.");
+				return entry;
+			},
+		}),
+
+		/** Build llms.txt and the news sitemap again now (replacing the stored copies). */
+		"discovery/rebuild": definePluginRoute({
+			permission: "plugins:manage",
+			methods: ["POST"],
+			request: { body: "json" },
+			handler: async (ctx) => {
+				const features = await ctxFeatures(ctx);
+				const out: { llms?: number; news?: number } = {};
+				if (isOn(features, FEATURE.llms)) out.llms = (await rebuildNow(ctx, "llms")).count;
+				if (isOn(features, FEATURE.news)) out.news = (await rebuildNow(ctx, "news")).count;
+				return out;
+			},
+		}),
+	};
+
+	return { routes, hooks };
+}

package/src/discovery/news.ts

@@ -0,0 +1,99 @@
+/**
+ * Google News sitemap (pure part): the 48-hour selection and the XML.
+ * Mirrors Coywolf SEO's news sitemap: articles published in the last
+ * 48 hours (the window Google News considers), newest first, at most 1,000
+ * (Google's per-sitemap limit), each with publication name and language,
+ * publication date, and title.
+ */
+
+export const NEWS_WINDOW_MS = 48 * 60 * 60 * 1000;
+export const NEWS_LIMIT = 1000;
+
+export interface NewsArticle {
+	url: string;
+	title: string;
+	/** Publish date (ISO 8601, or SQLite's "YYYY-MM-DD HH:MM:SS" in UTC). */
+	publishedAt: string;
+}
+
+export interface NewsPublication {
+	name: string;
+	language: string;
+}
+
+const TZ_SUFFIX = /([zZ]|[+-]\d{2}(:?\d{2})?)$/;
+
+/** Parse a stored timestamp, treating offset-less values as UTC (as EmDash does). */
+export function parseDate(value: string | null | undefined): Date | null {
+	if (!value) return null;
+	let normalized = value.trim();
+	if (normalized.includes(" ") && !normalized.includes("T")) normalized = normalized.replace(" ", "T");
+	if (!TZ_SUFFIX.test(normalized)) normalized += "Z";
+	const time = Date.parse(normalized);
+	return Number.isNaN(time) ? null : new Date(time);
+}
+
+/** Articles inside the 48-hour window (not in the future), newest first, capped. */
+export function selectNewsArticles(articles: NewsArticle[], now: number = Date.now(), limit = NEWS_LIMIT): NewsArticle[] {
+	return articles
+		.map((article) => ({ article, time: parseDate(article.publishedAt)?.getTime() ?? Number.NaN }))
+		.filter(({ time }) => !Number.isNaN(time) && time > now - NEWS_WINDOW_MS && time <= now)
+		.sort((a, b) => b.time - a.time)
+		.slice(0, limit)
+		.map(({ article }) => article);
+}
+
+export function escapeXml(text: string): string {
+	return text
+		.replace(/&/g, "&amp;")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/"/g, "&quot;")
+		.replace(/'/g, "&apos;")
+		// Characters XML 1.0 forbids outright.
+		// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping them is the point.
+		.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F￾￿]/g, "");
+}
+
+/**
+ * The language code Google News expects: ISO 639-1/2 ("en", "fr"), except
+ * Chinese, which must be "zh-cn" or "zh-tw".
+ */
+export function newsLanguage(locale: string | null | undefined): string {
+	const value = (locale ?? "").trim().toLowerCase().replace(/_/g, "-");
+	if (!value) return "en";
+	if (value.startsWith("zh")) return /-(tw|hk|mo|hant)/.test(value) ? "zh-tw" : "zh-cn";
+	const base = value.split("-")[0];
+	return /^[a-z]{2,3}$/.test(base) ? base : "en";
+}
+
+/** W3C datetime for <news:publication_date>. */
+function w3c(value: string): string {
+	return parseDate(value)?.toISOString().replace(/\.\d{3}Z$/, "Z") ?? value;
+}
+
+export function buildNewsSitemap(articles: NewsArticle[], publication: NewsPublication): string {
+	const name = escapeXml(publication.name);
+	const language = escapeXml(newsLanguage(publication.language));
+	const lines = [
+		'<?xml version="1.0" encoding="UTF-8"?>',
+		'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">',
+	];
+	for (const article of articles) {
+		lines.push(
+			"\t<url>",
+			`\t\t<loc>${escapeXml(article.url)}</loc>`,
+			"\t\t<news:news>",
+			"\t\t\t<news:publication>",
+			`\t\t\t\t<news:name>${name}</news:name>`,
+			`\t\t\t\t<news:language>${language}</news:language>`,
+			"\t\t\t</news:publication>",
+			`\t\t\t<news:publication_date>${escapeXml(w3c(article.publishedAt))}</news:publication_date>`,
+			`\t\t\t<news:title>${escapeXml(article.title)}</news:title>`,
+			"\t\t</news:news>",
+			"\t</url>",
+		);
+	}
+	lines.push("</urlset>", "");
+	return lines.join("\n");
+}

package/src/discovery/pack.ts

@@ -0,0 +1,58 @@
+import { registerFeatures } from "../core/features.js";
+import type { PackModule } from "../core/module.js";
+import { discoveryMiddleware } from "./middleware.js";
+import { type DiscoveryOptions, FEATURE, discoveryModule } from "./module.js";
+
+const FEATURES = [
+	{
+		id: FEATURE.main,
+		label: "Discovery",
+		description: "Help search engines, feed readers and AI agents find your content: IndexNow, a Google News sitemap, an RSS feed, and llms.txt with Markdown versions of entries.",
+		default: false,
+	},
+	{
+		id: FEATURE.indexnow,
+		label: "IndexNow",
+		description: "Tell Bing, Yandex, Seznam, Naver and other IndexNow engines when entries are published, updated, unpublished, or deleted.",
+		default: false,
+	},
+	{
+		id: FEATURE.news,
+		label: "News sitemap",
+		description: "A Google News sitemap at /news-sitemap.xml with entries published in the last 48 hours.",
+		default: false,
+	},
+	{
+		id: FEATURE.feed,
+		label: "RSS feed",
+		description: "An RSS feed of the newest entries at /feed/ (and a feed per category), linked from every page so feed readers and apps find it.",
+		default: false,
+	},
+	{
+		id: FEATURE.llms,
+		label: "llms.txt and Markdown",
+		description: "An llms.txt index at /llms.txt and a Markdown version of each entry at its URL + index.html.md.",
+		default: false,
+	},
+];
+registerFeatures(FEATURES);
+
+export type { DiscoveryOptions } from "./module.js";
+
+export function discoveryPack(options: DiscoveryOptions): PackModule {
+	const { routes, hooks } = discoveryModule(options);
+	return {
+		id: "discovery",
+		label: "Discovery",
+		features: FEATURES,
+		routes,
+		hooks,
+		// page:fragments adds the feed link and the Markdown link, each checked against its own switch.
+		hookFeature: { "page:fragments": FEATURE.main },
+		adminPages: [{ path: "/discovery", label: "Discovery", icon: "compass" }],
+		capabilities: ["content:read", "schema:read", "bylines:read", "network:request", "hooks.page-fragments:register"],
+		allowedHosts: ["api.indexnow.org", "www.bing.com"],
+	};
+}
+
+export { discoveryMiddleware };

package/src/discovery/settings.ts

@@ -0,0 +1,91 @@
+/**
+ * Discovery settings: one plugin setting ("discoverySettings") edited on the
+ * Discovery admin page, plus the IndexNow key ("discoveryIndexnowKey").
+ */
+import { z } from "zod";
+
+import type { SettingsReader } from "../shared.js";
+import { FEED_DEFAULT_ITEMS, FEED_MAX_ITEMS } from "./feed.js";
+import { DEFAULT_MAX_ENTRIES } from "./llms.js";
+import { INDEXNOW_ENDPOINTS, KEY_PATTERN, generateKey } from "./indexnow.js";
+
+export const SETTINGS_KEY = "discoverySettings";
+export const INDEXNOW_KEY_SETTING = "discoveryIndexnowKey";
+
+const endpoints = Object.keys(INDEXNOW_ENDPOINTS) as [keyof typeof INDEXNOW_ENDPOINTS, ...(keyof typeof INDEXNOW_ENDPOINTS)[]];
+
+export const settingsSchema = z.object({
+	indexnow: z
+		.object({
+			endpoint: z.enum(endpoints).default("api.indexnow.org"),
+			/** Collections whose changes are submitted; empty = every routable collection. */
+			collections: z.array(z.string().max(100)).max(200).default([]),
+		})
+		.default({ endpoint: "api.indexnow.org", collections: [] }),
+	news: z
+		.object({
+			/** Collections listed in the news sitemap (Coywolf SEO's default: posts only). */
+			collections: z.array(z.string().max(100)).max(200).default(["posts"]),
+			/** <news:name>; empty = the site title. */
+			publicationName: z.string().max(200).default(""),
+			/** <news:language>; empty = the site locale. */
+			language: z.string().max(20).default(""),
+		})
+		.default({ collections: ["posts"], publicationName: "", language: "" }),
+	feed: z
+		.object({
+			/** Collections in the feed (default: posts). */
+			collections: z.array(z.string().max(100)).max(200).default(["posts"]),
+			/** Newest entries listed in each feed. */
+			items: z.number().int().min(1).max(FEED_MAX_ITEMS).default(FEED_DEFAULT_ITEMS),
+			/** Term feeds at <term archive>/feed/ (WordPress's category feeds). */
+			termFeeds: z.boolean().default(true),
+		})
+		.default({ collections: ["posts"], items: FEED_DEFAULT_ITEMS, termFeeds: true }),
+	llms: z
+		.object({
+			/** Blockquote summary; empty = the site tagline. */
+			summary: z.string().max(500).default(""),
+			/** Detail paragraph under the summary; empty = the default sentence. */
+			intro: z.string().max(4000).default(""),
+			/** Collections listed in llms.txt; empty = every routable collection. */
+			collections: z.array(z.string().max(100)).max(200).default([]),
+			maxEntries: z.number().int().min(1).max(5000).default(DEFAULT_MAX_ENTRIES),
+			/** Serve `<entry-url>/index.html.md` and link llms.txt to it. */
+			markdown: z.boolean().default(true),
+			/** Optional content license noted in the Markdown frontmatter. */
+			license: z.string().max(200).default(""),
+		})
+		.default({ summary: "", intro: "", collections: [], maxEntries: DEFAULT_MAX_ENTRIES, markdown: true, license: "" }),
+});
+
+export type DiscoverySettings = z.infer<typeof settingsSchema>;
+
+export async function loadSettings(ctx: SettingsReader): Promise<DiscoverySettings> {
+	const stored = await ctx.settings.get<unknown>(SETTINGS_KEY);
+	const parsed = settingsSchema.safeParse(stored ?? {});
+	return parsed.success ? parsed.data : settingsSchema.parse({});
+}
+
+interface SettingsWriter extends SettingsReader {
+	settings: SettingsReader["settings"] & { set(key: string, value: unknown): Promise<void> };
+}
+
+/** The IndexNow key, generating and storing one the first time. */
+export async function ensureIndexNowKey(ctx: SettingsWriter): Promise<string> {
+	const existing = await ctx.settings.get<string>(INDEXNOW_KEY_SETTING);
+	if (typeof existing === "string" && KEY_PATTERN.test(existing)) return existing;
+	const key = generateKey();
+	await ctx.settings.set(INDEXNOW_KEY_SETTING, key);
+	return key;
+}
+
+export async function readIndexNowKey(ctx: SettingsReader): Promise<string | null> {
+	const key = await ctx.settings.get<string>(INDEXNOW_KEY_SETTING);
+	return typeof key === "string" && KEY_PATTERN.test(key) ? key : null;
+}
+
+/** Whether a collection is selected, where an empty selection means "all". */
+export function selected(collections: string[], slug: string): boolean {
+	return collections.length === 0 || collections.includes(slug);
+}

package/src/discovery/urls.ts

@@ -0,0 +1,35 @@
+/**
+ * Public URLs of content entries for discovery (llms.txt, the news sitemap,
+ * IndexNow), from the pack-wide resolver in core/content-url.ts: the site's
+ * `urls` overrides when configured, otherwise EmDash's own resolution
+ * (getPublicUrl, verified once per collection and locale against local
+ * url_pattern interpolation so long lists don't cost two queries an entry).
+ */
+import type { CollectionSchemaInfo, PluginContentItem, PluginContext } from "emdash";
+
+import { absoluteUrl, createEntryUrlResolver } from "../core/content-url.js";
+
+export function siteOrigin(ctx: Pick<PluginContext, "site">): string {
+	return ctx.site.url.replace(/\/+$/, "");
+}
+
+export interface UrlResolver {
+	/** Absolute URLs of published entries, keyed by id (null: none). Terms are read in one batch. */
+	urls(collection: CollectionSchemaInfo, items: PluginContentItem[]): Promise<Map<string, string | null>>;
+}
+
+/** One resolver per build (it remembers collection routes and verdicts). */
+export function createUrlResolver(ctx: PluginContext): UrlResolver {
+	const resolver = createEntryUrlResolver(ctx);
+	const origin = siteOrigin(ctx);
+	return {
+		async urls(collection, items) {
+			const out = new Map<string, string | null>();
+			const usable = items.filter((item) => collection.routable && item.status === "published" && item.slug);
+			for (const item of items) out.set(item.id, null);
+			const paths = await resolver.urls(collection.slug, usable);
+			for (const [id, path] of paths) out.set(id, path ? absoluteUrl(path, origin) : null);
+			return out;
+		},
+	};
+}

package/src/env.d.ts

@@ -0,0 +1,10 @@
+declare module "cloudflare:workers" {
+	export const env: Record<string, unknown>;
+	export function waitUntil(promise: Promise<unknown>): void;
+}
+
+/** Optional peer (src/forms.ts only); the site that uses private form uploads has it installed. */
+declare module "@emdash-cms/plugin-forms" {
+	// biome-ignore lint/suspicious/noExplicitAny: EmDash's ResolvedPlugin; wrapFormsPlugin reads it loosely.
+	export function createPlugin(options?: Record<string, unknown>): any;
+}