@coywolf/emdash 0.0.0-stage → 0.35.14

package/src/middleware.ts

@@ -0,0 +1,287 @@
+/**
+ * Coywolf Pack site middleware: everything the pack serves outside EmDash's
+ * plugin API (redirects, and module files such as sitemaps or downloads).
+ * Add it first in the site's src/middleware.ts:
+ *
+ * ```ts
+ * import { sequence } from "astro:middleware";
+ * import { coywolfPack } from "@coywolf/emdash/middleware";
+ * export const onRequest = sequence(coywolfPack(), yourMiddleware);
+ * ```
+ *
+ * Each module's handler runs only while its feature is on.
+ *
+ * A GET or HEAD answered after one of the pack's reads failed (it then fails
+ * closed: features at their defaults, settings unset, rules absent) is kept out of the edge
+ * cache, so a page rendered without its schema, reviews, videos or redirects
+ * isn't served for days. See src/core/read-failures.ts and settlePage.
+ */
+import type { MiddlewareHandler } from "astro";
+
+import { invalidateFeatures, knownFeatures, registerSiteOption, rememberSiteOption, siteFeatures, siteOption, siteSetting, isOn } from "./core/features.js";
+import { claimSettle, recordSettingsChange } from "./core/generation.js";
+import { invalidateRobotsCache } from "./robots/middleware.js";
+import { lastReadFailure, noteReadFailure, readFailures } from "./core/read-failures.js";
+import { injectAdminEnhancements } from "./core/settings-enhance.js";
+import type { PackMiddleware } from "./core/module.js";
+import { MIDDLEWARE } from "./modules.js";
+import { ALWAYS, LIFETIME_DEFAULTS, LIFETIME_SETTINGS } from "./pageCache/pack.js";
+import { ROBOTS_EDGE_MAX_AGE } from "./robots/pack.js";
+import { type PurgeScope, ROBOTS_TAG, applyPageLifetime, keepAtEdge, isRedirectSave, purgePageCache, purgeScope, purgesAfter, redirectPagesScope, settlePage, watchInvalidation } from "./pageCache/lib.js";
+import { STOPGAP_HEADER, WARMER_AGENT, WARM_DAILY_SETTING, WARM_SETTING, WARM_STATE_OPTION, scheduleRewarm, startWarm, warmMayHaveWork, warmStep } from "./pageCache/warm.js";
+import { prefetchRedirects } from "./redirects/middleware.js";
+import { notePackMiddleware } from "./search/live-serve.js";
+import { withMediaPreconnect } from "./images/lib.js";
+import { pendingMediaRenders, renderedPendingMedia } from "./images/pending.js";
+import { refreshMediaHost } from "./images/settings.js";
+import { variantsAfterMediaWrite } from "./images/pack.js";
+
+// The warming progress row is read with the feature switches, so requests can skip an idle warmer without a query.
+registerSiteOption(WARM_STATE_OPTION);
+
+/**
+ * False when the progress row as last read with the switches says there's
+ * nothing to warm (the run is done or failed, and no rewarm or daily refresh is due), so
+ * the request skips its warming step and that step's read. A run started (or
+ * rewarm scheduled) in another isolate is seen on this isolate's next read of
+ * the switches (FEATURES_TTL_MS at most); one started here is remembered at
+ * once. True when unknown.
+ */
+function warmingMayHaveWork(daily: boolean): boolean {
+	return warmMayHaveWork(siteOption(WARM_STATE_OPTION), Date.now(), daily);
+}
+
+/** The daily refresh is on unless turned off (read with the switches, so no extra query). */
+async function warmDaily(database?: string): Promise<boolean> {
+	return (await siteSetting<boolean>(WARM_DAILY_SETTING, database)) !== false;
+}
+
+/** Start warming, and remember the new run in this isolate's copy of the progress row. */
+async function startWarming(db: D1Database, reason: string): Promise<void> {
+	const state = await startWarm(db, reason);
+	rememberSiteOption(WARM_STATE_OPTION, JSON.stringify(state));
+}
+
+/** Per isolate: when this isolate last started a warming step (one at a time, at most every few seconds). */
+let warmingUntil = 0;
+const WARM_EVERY_MS = 5_000;
+const WARM_BUDGET_MS = 20_000;
+
+export interface CoywolfPackMiddlewareOptions {
+	/** D1 binding of the site database. Default "DB". */
+	database?: string;
+}
+
+const PASS_THROUGH = /^\/(_astro|_image)\//;
+/** EmDash's site settings API (POST saves General, SEO, Social…). */
+const EMDASH_SETTINGS_API = "/_emdash/api/settings";
+/** Largest redirect save body read for its sources (redirects/import takes up to 2 MiB). */
+const REDIRECT_BODY_MAX = 2 * 1024 * 1024 + 64 * 1024;
+/** EmDash admin pages (not its API): the pack adds its admin enhancements (step-by-step guides on the plugin Settings page). */
+const ADMIN_PAGE = /^\/_emdash\/admin(\/|$)/;
+/**
+ * EmDash's admin manifest (the sidebar). Read with fresh switches: another isolate's
+ * cached read could otherwise keep a module that was just switched on (or off) out of
+ * (or in) the sidebar for up to FEATURES_TTL_MS.
+ */
+const ADMIN_MANIFEST = "/_emdash/api/manifest";
+
+export function coywolfPack(options: CoywolfPackMiddlewareOptions = {}, handlers: PackMiddleware[] = MIDDLEWARE): MiddlewareHandler {
+	return async (context, next) => {
+		if (PASS_THROUGH.test(context.url.pathname)) return next();
+		if (ADMIN_PAGE.test(context.url.pathname) && context.request.method === "GET") return injectAdminEnhancements(await next());
+		let workers: { env: Record<string, unknown>; waitUntil?: (p: Promise<unknown>) => void };
+		try {
+			workers = (await import("cloudflare:workers")) as unknown as typeof workers;
+		} catch {
+			return next(); // Not running on Cloudflare.
+		}
+		// Pages rendered in this isolate may now load the live search client as a file (it's served below).
+		notePackMiddleware();
+		// Reads that fail from here until the response is rendered keep it out of the edge cache.
+		const failuresBefore = readFailures();
+		const readFailed = () => readFailures() !== failuresBefore;
+		const routeCache = (context as unknown as { cache?: Parameters<typeof settlePage>[0] }).cache;
+		const env = workers.env;
+		const waitUntil = (p: Promise<unknown>) => (workers.waitUntil ? workers.waitUntil(p) : void p);
+		// This request's redirect lookup goes to D1 in the same batch as the switches (when they aren't cached).
+		// (Skipped when the switches this isolate last read have Redirects off.)
+		const known = knownFeatures();
+		if (!known || isOn(known, "redirects")) prefetchRedirects(context.url, env, { database: options.database });
+		const features = await siteFeatures(options.database, context.url.pathname === ADMIN_MANIFEST);
+		const db = env[options.database ?? "DB"] as D1Database | undefined;
+		/** Clear cached pages, then (for a full purge, when warming is on) start warming them again. */
+		const purge = (scope: PurgeScope) =>
+			purgePageCache(scope).then(async (purged) => {
+				if (purged && "purgeEverything" in scope && db && (await siteSetting<boolean>(WARM_SETTING, options.database))) await startWarming(db, "settings");
+			});
+		// Page cache: a successful pack admin write (settings, redirects, imports…) can change any page.
+		if (purgesAfter(context.request.method, context.url.pathname)) {
+			// A redirect save also clears the pages cached at its sources; read its body before the route does.
+			// Only a body the route would accept (it checks permissions after this): anything else gets the tag purge alone.
+			const length = Number(context.request.headers.get("content-length"));
+			const redirectBody =
+				isRedirectSave(context.url.pathname) && length > 0 && length <= REDIRECT_BODY_MAX
+					? context.request.clone().json().catch(() => null)
+					: null;
+			const response = await next();
+			if (response.status < 400) {
+				const pages = redirectPagesScope(await redirectBody);
+				const scope = pages && "purgeEverything" in pages ? pages : purgeScope(context.url.pathname);
+				if (pages && !("purgeEverything" in pages)) waitUntil(purge(pages).catch((error) => console.error("coywolf-pack: could not clear pages at redirect sources", error)));
+				// This isolate's settings caches go now; other isolates drop theirs when they see the new generation.
+				invalidateFeatures();
+				const recorded = db
+					? recordSettingsChange(db, scope).catch((error) => console.error("coywolf-pack: could not record the settings change", error))
+					: Promise.resolve();
+				if (scope) waitUntil(recorded.then(() => purge(scope)));
+				else waitUntil(recorded);
+			}
+			return response;
+		}
+		// Content writes: EmDash clears the pages tagged with the collection (home page, archives, many
+		// posts). With warming on, a run starts a minute after the last such write (scheduleRewarm).
+		const method = context.request.method;
+		const invalidated =
+			method !== "GET" && method !== "HEAD" && context.url.pathname.startsWith("/_emdash/api/")
+				? watchInvalidation((context as unknown as { cache?: Parameters<typeof watchInvalidation>[0] }).cache)
+				: null;
+		// Only SEO saves can change it (not General, Social…), so other settings saves don't touch the settings generation.
+		// A staging change (EmDash's Site Status: Go live, or back to staging) changes every page's robots tag.
+		const settingsLength = Number(context.request.headers.get("content-length"));
+		const settingsKeys: { seo: boolean; staging: boolean } =
+			method === "POST" && context.url.pathname === EMDASH_SETTINGS_API && settingsLength > 0 && settingsLength <= 64 * 1024
+				? await context.request
+						.clone()
+						.json()
+						.then((body: unknown) => (body && typeof body === "object" ? { seo: "seo" in body, staging: "staging" in body } : { seo: false, staging: false }))
+						.catch(() => ({ seo: false, staging: false }))
+				: { seo: false, staging: false };
+		const seoSave = settingsKeys.seo;
+		const afterWrite = (response: Response): Response => {
+			// EmDash's SEO settings include "Disallow AI training", whose lines the pack's robots.txt carries: clear it,
+			// here now and (settling purge) again once every isolate's robots.txt cache has expired.
+			if (seoSave && response.status < 400 && isOn(features, "robots")) {
+				invalidateRobotsCache();
+				const scope: PurgeScope = { pathPrefixes: ["/robots.txt"] };
+				waitUntil(
+					(db ? recordSettingsChange(db, scope) : Promise.resolve())
+						.then(() => purge(scope))
+						.catch((error) => console.error("coywolf-pack: could not clear robots.txt after EmDash's settings changed", error)),
+				);
+			}
+			// Going live (or back to staging) changes every page's robots tag: EmDash renders noindex, nofollow on every page
+			// while the site is in staging, and only clears pages tagged with its settings. Clear every cached page (which
+			// restarts warming), and again once every isolate has the new settings (settling purge).
+			if (settingsKeys.staging && response.status < 400) {
+				const scope: PurgeScope = { purgeEverything: true };
+				waitUntil(
+					(db ? recordSettingsChange(db, scope) : Promise.resolve())
+						.then(() => purge(scope))
+						.catch((error) => console.error("coywolf-pack: could not clear pages after the site's staging status changed", error)),
+				);
+			}
+			// Stored image sizes follow media deletes, replaced files and WordPress media imports (new uploads: the media:afterUpload hook).
+			if (method !== "GET" && method !== "HEAD" && response.status < 400) {
+				const work = variantsAfterMediaWrite(method, context.url.pathname, isOn(features, "images"), context.url.origin);
+				if (work) waitUntil(work.catch((error) => console.error("coywolf-pack images: stored sizes after a media change failed", error)));
+			}
+			if (invalidated?.() && response.status < 400 && db) {
+				waitUntil(
+					(async () => {
+						if (!(await siteSetting<boolean>(WARM_SETTING, options.database))) return;
+						const state = await scheduleRewarm(db);
+						if (state) rememberSiteOption(WARM_STATE_OPTION, JSON.stringify(state));
+					})().catch((error) => console.error("coywolf-pack: could not schedule cache warming", error)),
+				);
+			}
+			return response;
+		};
+		// Settings saved in another isolate reach every isolate within the switches' lifetime; pages
+		// rendered meanwhile may have used the old ones, so the first request after that purges once more.
+		if (db) {
+			waitUntil(
+				claimSettle(db)
+					.then((scope) => (scope ? purge(scope) : undefined))
+					.catch((error) => console.error("coywolf-pack: settling purge failed", error)),
+			);
+		}
+		// Page cache lifetimes from Plugins → Performance, on routes the site made cacheable.
+		if (context.request.method === "GET") {
+			const [maxAgeDays, refreshDays] = await Promise.all([
+				siteSetting<number>(LIFETIME_SETTINGS.maxAgeDays, options.database),
+				siteSetting<number>(LIFETIME_SETTINGS.refreshDays, options.database),
+			]);
+			applyPageLifetime(
+				(context as unknown as { cache?: Parameters<typeof applyPageLifetime>[0] }).cache,
+				maxAgeDays ?? LIFETIME_DEFAULTS.maxAgeDays,
+				refreshDays ?? LIFETIME_DEFAULTS.refreshDays,
+			);
+		}
+		// Cache warming rides on real page traffic (see src/pageCache/warm.ts): after this
+		// response, claim a small batch of cold pages and visit them in the background.
+		if (
+			context.request.method === "GET" &&
+			!context.url.pathname.startsWith("/_emdash/") &&
+			context.request.headers.get("user-agent") !== WARMER_AGENT &&
+			Date.now() > warmingUntil &&
+			(await siteSetting<boolean>(WARM_SETTING, options.database)) &&
+			warmingMayHaveWork(await warmDaily(options.database))
+		) {
+			const self = env.SELF as { fetch(request: Request): Promise<Response> } | undefined;
+			if (db && self) {
+				warmingUntil = Date.now() + WARM_BUDGET_MS + WARM_EVERY_MS;
+				const origin = (context.site ?? context.url).origin;
+				waitUntil(
+					warmStep(db, self, origin, { budgetMs: WARM_BUDGET_MS, batchSize: 4, daily: await warmDaily(options.database) })
+						.catch((error) => console.error("coywolf-pack: cache warming failed", error))
+						.finally(() => {
+							warmingUntil = Date.now() + WARM_EVERY_MS;
+						}),
+				);
+			}
+		}
+		for (const handler of handlers) {
+			if (handler.feature !== ALWAYS && !isOn(features, handler.feature)) continue;
+			try {
+				const response = await handler.handle(context, env, waitUntil);
+				if (response) {
+					// The pack's own responses (robots.txt, files…) set their own Cache-Control.
+					// Without this, a site route rule (e.g. on a catch-all page route) would add
+					// its edge lifetime to them too. Redirects set their own edge caching.
+					if (!handler.ownsCache || readFailed()) routeCache?.set?.(false);
+					return response;
+				}
+			} catch (error) {
+				console.error(`coywolf-pack: ${handler.module} middleware failed`, error);
+				// The request goes on to the site instead (a page, or EmDash's own robots.txt): not into the edge cache.
+				noteReadFailure(`${handler.module} middleware`);
+			}
+		}
+		// The last word on edge caching, once the response has rendered (settlePage): not cached after a
+		// failed read; and a page that showed a video's Stream poster or an image's /s/ size while its
+		// stored copy is made (src/images/pending.ts) is cached for minutes, not days, so the next render
+		// uses the copy. The cache warmer's render of such a page isn't cached; it's told so
+		// (STOPGAP_HEADER) and visits again.
+		if (method === "GET" || method === "HEAD") {
+			const before = pendingMediaRenders();
+			const response = await next();
+			// EmDash's own robots.txt (Robots.txt Rules off): kept at the edge for a day, as the pack's is.
+			if (context.url.pathname === "/robots.txt" && response.status === 200 && !readFailed()) keepAtEdge(context, ROBOTS_EDGE_MAX_AGE, [ROBOTS_TAG]);
+			// The media host (read with the feature switches: no extra query) for the page's preconnect header.
+			await refreshMediaHost().catch(() => undefined);
+			const settled = await settlePage(routeCache, response, {
+				failed: readFailed,
+				stopgap: method === "GET" && isOn(features, "images") ? () => renderedPendingMedia(context.locals, before) : undefined,
+				markHeader: context.request.headers.get("user-agent") === WARMER_AGENT ? STOPGAP_HEADER : undefined,
+				label: `${method} ${context.url.pathname}`,
+				reason: lastReadFailure,
+			});
+			return context.url.pathname.startsWith("/_emdash/") ? settled : withMediaPreconnect(settled);
+		}
+		return afterWrite(await next());
+	};
+}
+
+/** Earlier name, kept so existing sites keep working. Serves all pack middleware, not just redirects. */
+export const coywolfRedirects = coywolfPack;

package/src/modules.ts

@@ -0,0 +1,53 @@
+/**
+ * The module list. Order matters: hooks and middleware run in this order.
+ * To add a module: import its factory and middleware, then add one entry to
+ * MODULES (and MIDDLEWARE if it serves site URLs).
+ */
+import { aiPack } from "./ai/pack.js";
+import { backupsDownloadMiddleware, backupsPack } from "./backups/pack.js";
+import { codeBlocksPack } from "./codeBlocks/pack.js";
+import { customBlocksPack } from "./customBlocks/pack.js";
+import type { PackMiddleware, PackModule } from "./core/module.js";
+import { headingsPack } from "./headings/pack.js";
+import { imagesMiddleware, imagesPack } from "./images/pack.js";
+import { pageCacheMiddleware, strayRobotsMiddleware, pageCachePack } from "./pageCache/pack.js";
+import { breadcrumbsPack } from "./breadcrumbs/pack.js";
+import { filesMiddleware, filesPack } from "./files/pack.js";
+import { formUploadsPack } from "./formUploads/pack.js";
+import { discoveryMiddleware, discoveryPack } from "./discovery/pack.js";
+import { linksPack } from "./links/pack.js";
+import { redirectsMiddleware, redirectsPack } from "./redirects/pack.js";
+import { schemaPack } from "./schema/pack.js";
+import { searchLiveMiddleware, searchPack, searchRateLimitMiddleware } from "./search/pack.js";
+import { robotsMiddleware, robotsPack } from "./robots/pack.js";
+import { reviewsPack } from "./reviews/pack.js";
+import type { CoywolfOptions } from "./options.js";
+import { videosCaptionsMiddleware, videosPack, videosSitemapMiddleware } from "./videos/pack.js";
+import { wpImportPack } from "./wpImport/pack.js";
+
+type Factory = (options: CoywolfOptions) => PackModule | null;
+
+export const MODULES: Factory[] = [
+	(o) => (o.backups ? backupsPack(o.backups) : null),
+	(o) => (o.redirects === false ? null : redirectsPack(o.redirects ?? {})),
+	// Before Headings & TOC: imported heading ids must be in place when anchors are stamped.
+	(o) => (o.wpImport === false ? null : wpImportPack()),
+	(o) => (o.headings === false ? null : headingsPack(o.headings ?? {})),
+	(o) => (o.breadcrumbs === false ? null : breadcrumbsPack(o.breadcrumbs ?? {})),
+	(o) => (o.codeBlocks === false ? null : codeBlocksPack(o.codeBlocks ?? {})),
+	(o) => (o.schema === false ? null : schemaPack(o.schema ?? {})),
+	(o) => (o.files === false ? null : filesPack(o.files ?? {})),
+	(o) => (o.formUploads === false ? null : formUploadsPack()),
+	(o) => (o.search === false ? null : searchPack(o.search ?? {})),
+	(o) => (o.ai === false ? null : aiPack(o.ai ?? {})),
+	(o) => (o.discovery === false ? null : discoveryPack(o.discovery ?? {})),
+	(o) => (o.links === false ? null : linksPack()),
+	(o) => (o.videos === false ? null : videosPack(o.videos ?? {})),
+	(o) => (o.robots === false ? null : robotsPack(o.robots ?? {})),
+	(o) => (o.reviews === false ? null : reviewsPack(o.reviews ?? {})),
+	(o) => (o.customBlocks === false || o.contentBlocks === false ? null : customBlocksPack()),
+	(o) => (o.images === false ? null : imagesPack(o.images ?? {})),
+	(o) => (o.pageCache === false ? null : pageCachePack()),
+];
+
+export const MIDDLEWARE: PackMiddleware[] = [pageCacheMiddleware, strayRobotsMiddleware, imagesMiddleware, backupsDownloadMiddleware, redirectsMiddleware, robotsMiddleware, filesMiddleware, searchLiveMiddleware, searchRateLimitMiddleware, ...discoveryMiddleware, videosSitemapMiddleware, videosCaptionsMiddleware];

package/src/options.ts

@@ -0,0 +1,66 @@
+import type { AiOptions } from "./ai/store.js";
+import type { BackupsOptions } from "./backups/module.js";
+import type { HeadingsOptions } from "./headings/pack.js";
+import type { BreadcrumbsOptions } from "./breadcrumbs/pack.js";
+import type { CodeBlocksOptions } from "./codeBlocks/pack.js";
+import type { FilesOptions } from "./files/module.js";
+import type { DiscoveryOptions } from "./discovery/module.js";
+import type { LinksOptions } from "./links/module.js";
+import type { RedirectsOptions } from "./redirects/module.js";
+import type { SchemaOptions } from "./schema/module.js";
+import type { SearchOptions } from "./search/module.js";
+import type { VideosOptions } from "./videos/module.js";
+import type { RobotsOptions } from "./robots/module.js";
+import type { ReviewsOptions } from "./reviews/pack.js";
+import type { TrailingSlash } from "./core/content-url.js";
+
+/**
+ * Modules to include in the build. Backups needs its bindings, so it's
+ * included only when configured; every other module is included unless set
+ * to false. Included modules are then turned on or off on the Coywolf Pack page.
+ */
+export interface CoywolfOptions {
+	backups?: BackupsOptions | false;
+	redirects?: RedirectsOptions | false;
+	headings?: HeadingsOptions | false;
+	breadcrumbs?: BreadcrumbsOptions | false;
+	codeBlocks?: CodeBlocksOptions | false;
+	schema?: SchemaOptions | false;
+	files?: FilesOptions | false;
+	/** Private form uploads (no options here; the Forms plugin is wrapped with privateFormUploads()). `false` leaves the module out. */
+	formUploads?: Record<string, never> | false;
+	search?: SearchOptions | false;
+	ai?: AiOptions | false;
+	discovery?: DiscoveryOptions | false;
+	links?: LinksOptions | false;
+	videos?: VideosOptions | false;
+	robots?: RobotsOptions | false;
+	reviews?: ReviewsOptions | false;
+	/** Custom Blocks: Note, Details, Affiliate disclosure, Quote, Testimonial and Podcast links blocks (no options yet). `false` leaves the module out. */
+	customBlocks?: Record<string, never> | false;
+	/** Clean image URLs (/media/<id>-<w>x<h>.<format>): media bucket and Images binding names. */
+	images?: import("./images/pack.js").ImagesOptions | false;
+	/** Page cache upkeep for Cloudflare Workers Cache (no options yet). `false` leaves the module out. */
+	pageCache?: Record<string, never> | false;
+	/** @deprecated Until 0.12.0 the Custom Blocks module was called Content Blocks; `contentBlocks: false` still leaves it out. */
+	contentBlocks?: Record<string, never> | false;
+	/** WordPress import tools (no options yet). `false` leaves the module out. */
+	wpImport?: Record<string, never> | false;
+	/**
+	 * Entry URL patterns for collections the theme routes differently from
+	 * EmDash's url_pattern, e.g. `{ posts: "/{term:category|uncategorized}/{slug}/" }`
+	 * or, with parent categories in the path, `{ posts: "/{termpath:category|uncategorized}/{slug}/" }`.
+	 * Used by every module that builds entry URLs (see docs/content-urls.md).
+	 */
+	urls?: Record<string, string>;
+	/** Trailing-slash policy for entry URLs. Default: EmDash's (Astro's `trailingSlash`). */
+	trailingSlash?: TrailingSlash;
+	/**
+	 * Parent terms for `{termpath:<taxonomy>}` URLs, taxonomy → term slug →
+	 * parent slug, e.g. `{ category: { seo: "news" } }`. Used only for terms
+	 * that have no parent in EmDash.
+	 */
+	termParents?: Record<string, Record<string, string>>;
+	/** Parent pages for `{pagepath}` URLs, slug → parent slug, e.g. `{ "coywolf-seo": "apps" }`. EmDash entries have no parent of their own. */
+	pageParents?: Record<string, string>;
+}