@coywolf/emdash 0.0.0-stage → 0.35.14

package/src/pageCache/lib.ts

@@ -0,0 +1,274 @@
+/**
+ * Page cache: keeps Cloudflare's Workers Cache (the edge HTML cache in front of
+ * the Worker, turned on by Astro's cacheCloudflare() provider) in step with the
+ * site. EmDash already purges the pages it tagged when content, menus or site
+ * settings change. This covers what EmDash can't know about: a new deploy (theme
+ * or code changes) and Coywolf Pack settings, which change page output (schema,
+ * blocks, reviews, videos…).
+ *
+ * Purging runs inside the Worker with cache.purge() from cloudflare:workers, so
+ * it needs no API token. Outside Workers Cache it does nothing.
+ */
+
+/** Option row holding the Worker version whose pages are in the cache. */
+export const VERSION_OPTION = "plugin:coywolf-pack:pageCache:version";
+
+/**
+ * Pack routes anyone can call (search, discovery files, video player actions).
+ * Everything else under the pack's API is an admin action, so a successful POST
+ * there purges the cache. The Stream webhook isn't listed: it changes video
+ * details that pages show.
+ */
+const PUBLIC_ROUTE = /^\/_emdash\/api\/plugins\/coywolf-pack\/(search\/|discovery\/public\/|videos\/(like|play|caption|embed|sitemap)(\/|$))/;
+const PACK_API = "/_emdash/api/plugins/coywolf-pack/";
+
+/** True when a request is a pack admin write whose success should purge the cache. */
+export function purgesAfter(method: string, pathname: string): boolean {
+	return method === "POST" && pathname.startsWith(PACK_API) && !PUBLIC_ROUTE.test(pathname);
+}
+
+export type PurgeScope = { purgeEverything: true } | { pathPrefixes: string[] } | { tags: string[] };
+
+/** Cache tag on every redirect the pack serves from the edge cache (src/redirects/middleware.ts). */
+export const REDIRECTS_TAG = "coywolf-redirects";
+/** robots.txt in the edge cache (cleared with everything else after the pack's admin saves and settings changes). */
+export const ROBOTS_TAG = "coywolf-robots";
+
+interface EdgeRouteCache {
+	enabled?: boolean;
+	set(options: { maxAge?: number; swr?: number; tags?: string[] } | false): void;
+}
+
+/**
+ * Keep a response the pack serves itself in the edge cache (Workers Cache) for
+ * `maxAge` seconds, replacing any lifetime the site's route rules or Plugins →
+ * Performance gave the request. For middleware marked `ownsCache`. Does
+ * nothing without Astro route caching.
+ */
+export function keepAtEdge(context: unknown, maxAge: number, tags: string[] = []): void {
+	const cache = (context as { cache?: EdgeRouteCache }).cache;
+	if (!cache?.enabled) return;
+	cache.set(false);
+	cache.set({ maxAge, swr: 0, tags });
+}
+
+/** Keep a response out of the edge cache (for an `ownsCache` middleware's other answers). */
+export function keepOffEdge(context: unknown): void {
+	(context as { cache?: EdgeRouteCache }).cache?.set(false);
+}
+
+/**
+ * What a successful pack admin write has to clear. Most settings change page
+ * output, so everything goes. These don't touch cached pages:
+ * - redirect edits clear only the cached redirects (their tag), and so does
+ *   rewinding the database (it may bring other rules back);
+ * - other backup actions, redirect lookups, the link report, and the Performance
+ *   page's own media rule and warming controls clear nothing;
+ * - robots.txt rules change only /robots.txt.
+ * null: nothing to clear.
+ */
+export function purgeScope(pathname: string): PurgeScope | null {
+	const route = pathname.slice(PACK_API.length);
+	if (/^redirects\/(?!(list|test|removed)$)|^backups\/(rewind|undo)$/.test(route)) return { tags: [REDIRECTS_TAG] };
+	if (/^(redirects|backups|links)\//.test(route) || /^cache\/(media|warm|purge)(\/|$)/.test(route)) return null;
+	// Stored image sizes: new uploads and crops are picked up after a short stopgap lifetime; a backfill run clears the cache itself once when it is done.
+	if (/^images\/variants\/(settings|run|bulk|estimate)$/.test(route)) return null;
+	// Videos admin reads (library, details, status, storage, settings, captions list) change nothing.
+	if (/^videos\/(list|status|detail|options|storage|test|settings|captions\/list)$/.test(route)) return null;
+	if (/^robots\//.test(route)) return { pathPrefixes: ["/robots.txt"] };
+	return { purgeEverything: true };
+}
+
+/** Most sources one redirect save clears by path; more than this clears everything instead. */
+const MAX_REDIRECT_PATHS = 30;
+
+/** True for the redirect writes whose request body redirectPagesScope reads (read it before the route consumes it). */
+export function isRedirectSave(pathname: string): boolean {
+	return /^redirects\/(save|import)$/.test(pathname.slice(PACK_API.length));
+}
+
+/**
+ * The pages a redirect save or import also has to clear: the cached page at
+ * each new source. Cached pages are served before the Worker runs, so the
+ * redirect would otherwise wait for them to expire (purgeScope only clears the
+ * cached redirects). An exact source clears by path prefix, which covers its
+ * trailing-slash form and query strings; a pattern can cover any page, so it
+ * clears everything. null: nothing to clear (disabled rules, an unreadable body).
+ */
+export function redirectPagesScope(body: unknown): PurgeScope | null {
+	if (!body || typeof body !== "object") return null;
+	const input = body as { rules?: unknown };
+	const rules = (Array.isArray(input.rules) ? input.rules : [body]) as Array<{ source?: unknown; isRegex?: unknown; enabled?: unknown }>;
+	const paths = new Set<string>();
+	for (const rule of rules) {
+		if (!rule || typeof rule !== "object" || rule.enabled === false || typeof rule.source !== "string") continue;
+		if (rule.isRegex === true) return { purgeEverything: true };
+		const path = rule.source.trim().split("?")[0].replace(/\/+$/, "") || "/";
+		if (!path.startsWith("/")) continue;
+		if (path === "/") return { purgeEverything: true };
+		paths.add(path);
+	}
+	if (!paths.size) return null;
+	return paths.size > MAX_REDIRECT_PATHS ? { purgeEverything: true } : { pathPrefixes: [...paths] };
+}
+
+/** Release an RPC result's stubs (Symbol.dispose, where the runtime has it). Anything else is left alone. */
+export function disposeRpcResult(result: unknown): void {
+	if (typeof Symbol.dispose !== "symbol" || !result || typeof result !== "object") return;
+	try {
+		(result as { [Symbol.dispose]?: () => void })[Symbol.dispose]?.();
+	} catch {
+		// Already released.
+	}
+}
+
+/** Purge cached pages (all of them unless a scope says otherwise). False when there's no Workers Cache or the purge was refused. */
+export async function purgePageCache(scope: PurgeScope = { purgeEverything: true }): Promise<boolean> {
+	try {
+		const workers = (await import("cloudflare:workers")) as unknown as {
+			cache?: { purge(options: PurgeScope): Promise<{ success?: boolean; errors?: Array<{ message: string }> } | undefined> };
+		};
+		if (!workers.cache?.purge) return false;
+		const result = await workers.cache.purge(scope);
+		try {
+			// The purge API rate-limits (Free-tier limits for Workers Cache) and says so in the result.
+			if (result && result.success === false) {
+				console.error("coywolf-pack: page cache purge refused", result.errors);
+				return false;
+			}
+			return true;
+		} finally {
+			// The result is an RPC result: dispose of it, or the runtime warns that it wasn't.
+			disposeRpcResult(result);
+		}
+	} catch (error) {
+		console.error("coywolf-pack: page cache purge failed", error);
+		return false;
+	}
+}
+
+/** The deployed Worker version (CF_VERSION_METADATA binding), if bound. */
+export function versionId(env: Record<string, unknown>): string | undefined {
+	const metadata = env.CF_VERSION_METADATA as { id?: unknown } | undefined;
+	return typeof metadata?.id === "string" && metadata.id ? metadata.id : undefined;
+}
+
+/**
+ * Purge once per deploy: the first isolate of a new version that sees a request
+ * compares its version with the stored one, purges, and records it. Other
+ * isolates of the same version find it recorded. Racing isolates may both purge,
+ * which is harmless.
+ */
+export async function purgeIfNewVersion(db: D1Database, id: string, known?: string | null): Promise<boolean> {
+	// `known`: the row as read with the feature switches (undefined when not read; then it's read here).
+	const value = known !== undefined ? known : ((await db.prepare("SELECT value FROM options WHERE name = ?").bind(VERSION_OPTION).first<{ value: string }>())?.value ?? null);
+	if (value && JSON.parse(value) === id) return false;
+	if (!(await purgePageCache())) return false;
+	await db
+		.prepare("INSERT INTO options (name, value) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET value = excluded.value")
+		.bind(VERSION_OPTION, JSON.stringify(id))
+		.run();
+	return true;
+}
+
+/** Astro's per-request cache controls (route caching), as far as the pack uses them. */
+interface RouteCache {
+	enabled?: boolean;
+	options?: { maxAge?: number };
+	set(options: { maxAge?: number; swr?: number } | false): void;
+}
+
+/**
+ * Apply the Page cache lifetimes to a request whose route the site made
+ * cacheable (a routeRules entry gave it a maxAge). Other routes are left alone,
+ * so this never turns caching on for anything the site didn't.
+ */
+export function applyPageLifetime(cache: RouteCache | undefined, maxAgeDays: number, refreshDays: number): boolean {
+	if (!cache?.enabled || cache.options?.maxAge === undefined) return false;
+	cache.set({ maxAge: Math.round(maxAgeDays * 86400), swr: Math.round(refreshDays * 86400) });
+	return true;
+}
+
+/** Lifetime of a page rendered with a stopgap (a Stream poster while its media-host copy is made). */
+export const STOPGAP_LIFETIME = { maxAge: 300, swr: 60 } as const;
+
+/** What settlePage asks about a response once it's rendered. */
+export interface SettleChecks {
+	/** A read the render relied on failed (src/core/read-failures.ts): the response isn't edge-cached at all. */
+	failed(): boolean;
+	/** The render used temporary media (src/images/pending.ts): the page is cached for minutes, not days. */
+	stopgap?(): boolean;
+	/** For the cache warmer's own visits: the header that tells it the page wasn't cached (see below). */
+	markHeader?: string;
+	/** Names the request in the log line when a page isn't cached after a failed read (e.g. "GET /about/"). */
+	label?: string;
+	/** What failed (for the same log line). */
+	reason?(): string | undefined;
+}
+
+/**
+ * The last word on a GET or HEAD response's edge caching, given once it's
+ * rendered:
+ * - `failed()`: the pack fell back to "off" or "absent" for something it
+ *   couldn't read, so the response isn't the site's real page. Not cached.
+ * - `stopgap()`: the page showed temporary media. Cached for minutes (STOPGAP_LIFETIME).
+ *
+ * Astro streams pages: components deeper in the page render after `next()`
+ * returns, and they can read settings (a failure noticed only then), mark a
+ * stopgap, or call cache.set(), which turns caching back on after a
+ * set(false). The cache headers are applied once the middleware chain has
+ * returned (Astro's handleCache). So a page about to be cached (HTML, a 200,
+ * a lifetime from the route rules) is read in full first, then asked, then its
+ * caching set, last. Any other response isn't read: one rendered after a
+ * failed read is left out of the cache at once (headers aren't applied until
+ * the chain returns, so nothing it renders later can undo that).
+ *
+ * `markHeader` is for the cache warmer's own visits: a stopgap page (or one
+ * rendered after a failed read) isn't cached at all, and the response carries
+ * `markHeader: 1` so the warmer visits it again later (see
+ * src/pageCache/warm.ts). Not caching it keeps the header out of the cache
+ * (visitors never see it) and makes sure the warmer's next visit renders the
+ * page instead of getting this copy back.
+ */
+export async function settlePage(cache: RouteCache | undefined, response: Response, checks: SettleChecks): Promise<Response> {
+	// No route caching (dev, or a site without a cache provider). Astro's real cache stays `enabled` after set(false).
+	if (!cache?.enabled) return response;
+	const maxAge = cache.options?.maxAge;
+	if (maxAge === undefined || maxAge <= 0 || response.status !== 200 || !response.body || !(response.headers.get("content-type") ?? "").startsWith("text/html")) {
+		if (checks.failed()) cache.set(false);
+		return response;
+	}
+	const body = await response.arrayBuffer();
+	const headers = new Headers(response.headers);
+	if (checks.failed()) {
+		console.error(`coywolf-pack: ${checks.label ?? "a page"} not edge-cached: a read failed while it rendered (last failure: ${checks.reason?.() ?? "unknown"})`);
+		cache.set(false);
+		if (checks.markHeader) headers.set(checks.markHeader, "1");
+	} else if (checks.stopgap?.() && cache.options?.maxAge !== undefined) {
+		// (A component may have turned caching off while rendering, which clears maxAge: that's left alone.)
+		if (checks.markHeader) {
+			cache.set(false);
+			headers.set(checks.markHeader, "1");
+		} else cache.set({ maxAge: Math.min(maxAge, STOPGAP_LIFETIME.maxAge), swr: STOPGAP_LIFETIME.swr });
+	}
+	return new Response(body, { status: response.status, statusText: response.statusText, headers });
+}
+
+/**
+ * Notice when the request clears cached pages by tag (EmDash does on content,
+ * menu, taxonomy, widget and site settings writes, and skips it for saves that
+ * don't change live content, such as draft autosaves): wraps this request's
+ * cache.invalidate. Astro gives the middleware and the route the same cache
+ * object. Returns a function telling whether it was called (and succeeded).
+ */
+export function watchInvalidation(cache: { invalidate?: (...args: unknown[]) => Promise<unknown> } | undefined): () => boolean {
+	let invalidated = false;
+	const original = cache?.invalidate;
+	if (!cache || typeof original !== "function") return () => false;
+	cache.invalidate = async (...args: unknown[]) => {
+		const result = await original.apply(cache, args);
+		invalidated = true;
+		return result;
+	};
+	return () => invalidated;
+}

package/src/pageCache/pack.ts

@@ -0,0 +1,240 @@
+import { PluginRouteError, definePluginRoute } from "emdash";
+import { z } from "zod";
+
+import { registerSiteOption, registerSiteSetting, rememberSiteOption, rememberSiteSetting, siteFeatureOn, siteOption, siteSetting } from "../core/features.js";
+import { parseInput } from "../shared.js";
+import { CLOUDFLARE_API_HOST, CloudflareApiError, applyMediaCacheRule, readMediaCacheRule } from "../images/cloudflare.js";
+import { mediaApiAccess, purgeMediaHost } from "../images/module.js";
+import type { PackMiddleware, PackModule } from "../core/module.js";
+import { workerEnv } from "../shared.js";
+import { ROBOTS_TAG, VERSION_OPTION, keepAtEdge, purgeIfNewVersion, purgePageCache, versionId } from "./lib.js";
+import { usageReport } from "./usage.js";
+import { WARM_DAILY_SETTING, WARM_SETTING, WARM_STATE_OPTION, nextDailyAt, pendingRevisits, readWarmState, remainingUrls, startWarm } from "./warm.js";
+
+/**
+ * Performance is always on (its page sits right below Coywolf Pack). On a site
+ * without Workers Cache its purges do nothing and its lifetimes apply to no
+ * route, so there's nothing to switch off.
+ */
+export const ALWAYS = "*";
+
+/** How long the edge keeps a page, and how long after that it may serve it while refreshing. */
+export const LIFETIME_SETTINGS = { maxAgeDays: "pageCacheMaxAgeDays", refreshDays: "pageCacheRefreshDays" } as const;
+export const LIFETIME_DEFAULTS = { maxAgeDays: 7, refreshDays: 1 } as const;
+registerSiteSetting(LIFETIME_SETTINGS.maxAgeDays);
+registerSiteSetting(LIFETIME_SETTINGS.refreshDays);
+registerSiteSetting(WARM_SETTING);
+registerSiteSetting(WARM_DAILY_SETTING);
+// The deployed version whose pages are cached, read with the switches (no query of its own per isolate).
+registerSiteOption(VERSION_OPTION);
+
+async function siteDb(): Promise<D1Database | undefined> {
+	const env: Record<string, unknown> = await workerEnv().catch(() => ({}));
+	return env.DB as D1Database | undefined;
+}
+
+
+const lifetimeInput = z.object({
+	maxAgeDays: z.number().int().min(1, "Keep pages for at least 1 day.").max(365, "Keep pages for at most 365 days."),
+	refreshDays: z.number().int().min(0).max(365, "Refresh for at most 365 days."),
+});
+
+const apiError = (error: unknown): never => {
+	if (error instanceof CloudflareApiError) throw PluginRouteError.badRequest(error.message);
+	throw error;
+};
+
+export function pageCachePack(): PackModule {
+	return {
+		id: "pageCache",
+		label: "Page cache",
+		features: [],
+		routes: {
+			/** The Performance section: page lifetimes and whether the site has Workers Cache. */
+			"cache/settings": {
+				permission: "plugins:manage" as const,
+				handler: async (ctx: Parameters<typeof mediaApiAccess>[0]) => {
+					const [maxAgeDays, refreshDays] = await Promise.all([
+						ctx.settings.get<number>(LIFETIME_SETTINGS.maxAgeDays),
+						ctx.settings.get<number>(LIFETIME_SETTINGS.refreshDays),
+					]);
+					return {
+						maxAgeDays: maxAgeDays ?? LIFETIME_DEFAULTS.maxAgeDays,
+						refreshDays: refreshDays ?? LIFETIME_DEFAULTS.refreshDays,
+						images: await siteFeatureOn("images"),
+					};
+				},
+			},
+
+			/** Cache warming: the switch and where the current (or last) run is. */
+			"cache/warm/status": {
+				permission: "plugins:manage" as const,
+				handler: async (ctx: Parameters<typeof mediaApiAccess>[0]) => {
+					const db = await siteDb();
+					const state = db ? await readWarmState(db) : null;
+					return {
+						enabled: Boolean(await ctx.settings.get<boolean>(WARM_SETTING)),
+						// The daily refresh: on unless turned off.
+						daily: (await ctx.settings.get<boolean>(WARM_DAILY_SETTING)) !== false,
+						// `revisiting`: pages warmed with a stopgap poster, to be warmed again once their posters are copied (the URLs stay out).
+						// `nextDailyAt`: when the daily refresh starts a new run (ms; only once a run has finished).
+						state: state && { ...state, revisit: undefined, remaining: remainingUrls(state), revisiting: pendingRevisits(state), nextDailyAt: nextDailyAt(state) ?? undefined },
+					};
+				},
+			},
+
+			"cache/warm/settings/save": definePluginRoute({
+				permission: "plugins:manage",
+				methods: ["POST"],
+				request: { body: "json" },
+				handler: async (ctx) => {
+					const { enabled, daily } = parseInput(z.object({ enabled: z.boolean().optional(), daily: z.boolean().optional() }), ctx.input);
+					if (enabled !== undefined) {
+						await ctx.settings.set(WARM_SETTING, enabled);
+						rememberSiteSetting(WARM_SETTING, enabled);
+					}
+					if (daily !== undefined) {
+						await ctx.settings.set(WARM_DAILY_SETTING, daily);
+						rememberSiteSetting(WARM_DAILY_SETTING, daily);
+					}
+					return { enabled, daily };
+				},
+			}),
+
+			/** Warm every page now (also restarts a run in progress). */
+			"cache/warm/start": definePluginRoute({
+				permission: "plugins:manage",
+				methods: ["POST"],
+				request: { body: "none" },
+				handler: async (ctx) => {
+					const db = await siteDb();
+					if (!db) throw PluginRouteError.badRequest("Missing the DB binding.");
+					if (!(await ctx.settings.get<boolean>(WARM_SETTING))) throw PluginRouteError.badRequest("Turn on cache warming first.");
+					const state = await startWarm(db, "manual");
+					return { state: { ...state, remaining: 0 } };
+				},
+			}),
+
+			"cache/settings/save": definePluginRoute({
+				permission: "plugins:manage",
+				methods: ["POST"],
+				request: { body: "json" },
+				handler: async (ctx) => {
+					const input = parseInput(lifetimeInput, ctx.input);
+					await ctx.settings.set(LIFETIME_SETTINGS.maxAgeDays, input.maxAgeDays);
+					await ctx.settings.set(LIFETIME_SETTINGS.refreshDays, input.refreshDays);
+					rememberSiteSetting(LIFETIME_SETTINGS.maxAgeDays, input.maxAgeDays);
+					rememberSiteSetting(LIFETIME_SETTINGS.refreshDays, input.refreshDays);
+					// Pages already cached keep their old lifetime; the pack's purge-after-save clears them.
+					return input;
+				},
+			}),
+
+			/** The media host's Cloudflare Cache Rule (read through the Clean Image URLs token). */
+			"cache/media": {
+				permission: "plugins:manage" as const,
+				handler: async (ctx: Parameters<typeof mediaApiAccess>[0]) => {
+					const access = await mediaApiAccess(ctx);
+					if (!("config" in access)) return { host: access.hostname, reason: access.reason, rule: null };
+					try {
+						const { zone, rule } = await readMediaCacheRule(access.config, access.hostname);
+						return { host: access.hostname, zone, rule };
+					} catch (error) {
+						return { host: access.hostname, reason: error instanceof Error ? error.message : String(error), rule: null };
+					}
+				},
+			},
+
+			"cache/media/apply": definePluginRoute({
+				permission: "plugins:manage",
+				methods: ["POST"],
+				request: { body: "none" },
+				handler: async (ctx) => {
+					const access = await mediaApiAccess(ctx);
+					if (!("config" in access)) throw PluginRouteError.badRequest(access.reason);
+					const zone = await applyMediaCacheRule(access.config, access.hostname).catch(apiError);
+					ctx.log.info("Media cache rule applied", { host: access.hostname, zone });
+					return { host: access.hostname, zone, ...(await readMediaCacheRule(access.config, access.hostname).catch(() => ({ rule: null }))) };
+				},
+			}),
+
+			/** Cloudflare usage (Performance page and the dashboard widget): kept for an hour; `refresh` reads again. */
+			"cache/usage": {
+				permission: "plugins:manage" as const,
+				handler: async (ctx: Parameters<typeof mediaApiAccess>[0] & { input?: unknown }) => {
+					const { refresh } = parseInput(z.object({ refresh: z.union([z.literal("1"), z.literal(1), z.literal(true)]).optional() }), ctx.input ?? {});
+					return usageReport(ctx, Boolean(refresh));
+				},
+			},
+
+			/**
+			 * Clear every cached page now, and the media host's images in the
+			 * zone cache when Clean image URLs has a media host.
+			 */
+			"cache/purge": definePluginRoute({
+				permission: "plugins:manage",
+				methods: ["POST"],
+				request: { body: "none" },
+				handler: async (ctx) => {
+					const pages = await purgePageCache();
+					if (pages && (await ctx.settings.get<boolean>(WARM_SETTING))) {
+						const db = await siteDb();
+						if (db) await startWarm(db, "cleared");
+					}
+					const images = (await siteFeatureOn("images")) ? await purgeMediaHost(ctx) : { purged: false, host: null };
+					if (!pages && !images.purged) {
+						throw PluginRouteError.badRequest(images.message ?? "This site isn't using Workers Cache, so there are no pages to clear.");
+					}
+					ctx.log.info("Page cache cleared", { pages, images: images.purged ? images.host : false });
+					return { pages, images };
+				},
+			}),
+		},
+		// Its page (/performance) is listed by index.ts, always, right below Coywolf Pack.
+		adminPages: [],
+		widgets: [{ id: "cloudflare-usage", title: "Cloudflare usage", size: "third" }],
+		// Clearing the media host's images goes through the Cloudflare API.
+		capabilities: ["network:request"],
+		allowedHosts: [CLOUDFLARE_API_HOST],
+	};
+}
+
+let checkedVersion = false;
+
+/** Once per isolate: if this is a new deploy, clear the cache (never answers a request). */
+export const pageCacheMiddleware: PackMiddleware = {
+	module: "pageCache",
+	feature: ALWAYS,
+	handle: (_context, env, waitUntil) => {
+		if (checkedVersion) return undefined;
+		checkedVersion = true;
+		const id = versionId(env);
+		const db = env.DB as D1Database | undefined;
+		if (id && db)
+			waitUntil(
+				purgeIfNewVersion(db, id, siteOption(VERSION_OPTION))
+					.then(async (purged) => {
+						if (purged && (await siteSetting<boolean>(WARM_SETTING))) rememberSiteOption(WARM_STATE_OPTION, JSON.stringify(await startWarm(db, "deploy")));
+					})
+					.catch((error) => console.error("coywolf-pack: page cache version check failed", error)),
+			);
+		return undefined;
+	},
+};
+
+/** robots.txt is only ever read at the root; a crawler asking at /news/robots.txt gets a small 404, kept at the edge for a day, instead of a rendered 404 page. */
+export const strayRobotsMiddleware: PackMiddleware = {
+	module: "pageCache",
+	feature: ALWAYS,
+	ownsCache: true,
+	handle: (context) => {
+		const { pathname } = context.url;
+		const method = context.request.method;
+		if (!pathname.endsWith("/robots.txt") || pathname === "/robots.txt" || (method !== "GET" && method !== "HEAD")) return undefined;
+		keepAtEdge(context, 86400, [ROBOTS_TAG]);
+		return new Response(method === "HEAD" ? null : "Not found. robots.txt is at the site root.\n", {
+			status: 404,
+			headers: { "Content-Type": "text/plain; charset=utf-8", "Cache-Control": "public, max-age=86400", "X-Robots-Tag": "noindex" },
+		});
+	},
+};

package/src/pageCache/usage-core.ts

@@ -0,0 +1,152 @@
+/**
+ * Cloudflare usage (pure part): plan allowances and prices, plan detection,
+ * the estimated bill, and the cache-warming estimate. No imports, so
+ * `node --test` can load this file.
+ *
+ * Prices and allowances are Cloudflare's published Workers and D1 pricing
+ * (Workers Paid, Standard usage model). The account's allowances are shared
+ * by every Worker and database in it, so the bill is worked out from the
+ * account's totals, not this site's.
+ */
+
+export type Plan = "paid" | "free";
+
+export interface D1Usage {
+	rowsRead: number;
+	rowsWritten: number;
+	readQueries: number;
+	writeQueries: number;
+}
+
+export interface WorkersUsage {
+	requests: number;
+	cpuMs: number;
+	errors: number;
+}
+
+export interface UsageTotals {
+	d1: D1Usage;
+	workers: WorkersUsage;
+}
+
+export const EMPTY_TOTALS: UsageTotals = {
+	d1: { rowsRead: 0, rowsWritten: 0, readQueries: 0, writeQueries: 0 },
+	workers: { requests: 0, cpuMs: 0, errors: 0 },
+};
+
+/** Workers Paid: what the $5 a month includes, and the price of each unit over it. */
+export const PAID = {
+	base: 5,
+	monthly: { requests: 10_000_000, cpuMs: 30_000_000, rowsRead: 25_000_000_000, rowsWritten: 50_000_000, storageBytes: 5e9 },
+	/** Per million (storage: per GB-month). */
+	price: { requests: 0.3, cpuMs: 0.02, rowsRead: 0.001, rowsWritten: 1, storageGb: 0.75 },
+} as const;
+
+/** Workers Free: daily limits (requests and D1 rows reset every day at 00:00 UTC); 10 ms CPU per request; 5 GB of D1 storage. */
+export const FREE = {
+	daily: { requests: 100_000, rowsRead: 5_000_000, rowsWritten: 100_000 },
+	cpuMsPerRequest: 10,
+	storageBytes: 5e9,
+} as const;
+
+interface Entitlement {
+	id?: string;
+	feature?: { key?: string };
+	allocation?: { value?: unknown };
+}
+
+/**
+ * The Workers plan from the account's entitlements: D1's per-database size
+ * limit is 10 GB on Workers Paid and 500 MB on Free. Null when the list
+ * doesn't say (the key is missing, or the token can't read it).
+ */
+export function planFromEntitlements(entitlements: unknown): Plan | null {
+	const list = Array.isArray(entitlements) ? (entitlements as Entitlement[]) : [];
+	const perDb = list.find((e) => (e.feature?.key ?? e.id) === "d1.max_storage_per_db_mb");
+	const mb = Number(perDb?.allocation?.value);
+	if (!Number.isFinite(mb) || mb <= 0) return null;
+	return mb > 500 ? "paid" : "free";
+}
+
+/** Without the entitlements: a day over Free's limits means the account is on Paid (Free would have stopped it). Null when it can't tell. */
+export function planFromDailyPeaks(peaks: { requests: number; rowsRead: number; rowsWritten: number }): Plan | null {
+	return peaks.requests > FREE.daily.requests || peaks.rowsRead > FREE.daily.rowsRead || peaks.rowsWritten > FREE.daily.rowsWritten ? "paid" : null;
+}
+
+const over = (used: number, included: number) => Math.max(0, used - included);
+const round2 = (n: number) => Math.round(n * 100) / 100;
+
+export interface Bill {
+	base: number;
+	overage: { requests: number; cpu: number; rowsRead: number; rowsWritten: number; storage: number };
+	total: number;
+}
+
+/**
+ * The estimated bill for 30 days at this rate on Workers Paid: $5 plus anything
+ * over the included amounts. A billing month isn't exactly these 30 days, so
+ * it's an estimate.
+ */
+export function estimateBill(account: UsageTotals, storageBytes: number): Bill {
+	const m = PAID.monthly;
+	const p = PAID.price;
+	const overage = {
+		requests: round2((over(account.workers.requests, m.requests) / 1e6) * p.requests),
+		cpu: round2((over(account.workers.cpuMs, m.cpuMs) / 1e6) * p.cpuMs),
+		rowsRead: round2((over(account.d1.rowsRead, m.rowsRead) / 1e6) * p.rowsRead),
+		rowsWritten: round2((over(account.d1.rowsWritten, m.rowsWritten) / 1e6) * p.rowsWritten),
+		storage: round2((over(storageBytes, m.storageBytes) / 1e9) * p.storageGb),
+	};
+	const total = round2(PAID.base + Object.values(overage).reduce((a, b) => a + b, 0));
+	return { base: PAID.base, overage, total };
+}
+
+export interface WarmingEstimate {
+	/** Cache warmer visits in the window (from sampled Workers Logs). */
+	visits: number;
+	cpuMs: number;
+	/** Share of the site's requests and CPU time in the same window (0–1). */
+	requestShare: number;
+	cpuShare: number;
+	/** The site's D1 rows read in the window times the warmer's share of requests. */
+	rowsRead: number;
+	days: number;
+}
+
+/** The warmer's share of the site's work over the same days. Logs are sampled, so it's an estimate. */
+export function estimateWarming(warm: { visits: number; cpuMs: number }, site: UsageTotals, days: number): WarmingEstimate {
+	const requestShare = site.workers.requests > 0 ? Math.min(1, warm.visits / site.workers.requests) : 0;
+	const cpuShare = site.workers.cpuMs > 0 ? Math.min(1, warm.cpuMs / site.workers.cpuMs) : 0;
+	return { visits: warm.visits, cpuMs: warm.cpuMs, requestShare, cpuShare, rowsRead: Math.round(site.d1.rowsRead * requestShare), days };
+}
+
+/** GraphQL rows (grouped by database or Worker) folded into one site's totals and the account's. */
+export function foldTotals(
+	d1Rows: Array<{ dimensions?: { databaseId?: string }; sum?: Partial<D1Usage> }>,
+	workerRows: Array<{ dimensions?: { scriptName?: string }; sum?: { requests?: number; errors?: number; cpuTimeUs?: number } }>,
+	site: { databaseId: string | null; worker: string | null },
+): { site: UsageTotals; account: UsageTotals } {
+	const blank = (): UsageTotals => ({ d1: { ...EMPTY_TOTALS.d1 }, workers: { ...EMPTY_TOTALS.workers } });
+	const out = { site: blank(), account: blank() };
+	for (const row of d1Rows) {
+		const s = row.sum ?? {};
+		const targets = [out.account, ...(site.databaseId && row.dimensions?.databaseId === site.databaseId ? [out.site] : [])];
+		for (const t of targets) {
+			t.d1.rowsRead += s.rowsRead ?? 0;
+			t.d1.rowsWritten += s.rowsWritten ?? 0;
+			t.d1.readQueries += s.readQueries ?? 0;
+			t.d1.writeQueries += s.writeQueries ?? 0;
+		}
+	}
+	for (const row of workerRows) {
+		const s = row.sum ?? {};
+		const targets = [out.account, ...(site.worker && row.dimensions?.scriptName === site.worker ? [out.site] : [])];
+		for (const t of targets) {
+			t.workers.requests += s.requests ?? 0;
+			t.workers.errors += s.errors ?? 0;
+			t.workers.cpuMs += (s.cpuTimeUs ?? 0) / 1000;
+		}
+	}
+	for (const t of [out.site, out.account]) t.workers.cpuMs = Math.round(t.workers.cpuMs);
+	return out;
+}