emdash 0.5.0 → 0.7.0

package/src/emdash-runtime.ts

@@ -19,7 +19,9 @@ import type {
 } from "./astro/integration/runtime.js";
 import type { EmDashManifest, ManifestCollection } from "./astro/types.js";
 import { getAuthMode } from "./auth/mode.js";
+import { getTrustedProxyHeaders } from "./auth/trusted-proxy.js";
 import { isSqlite } from "./database/dialect-helpers.js";
+import { kyselyLogOption } from "./database/instrumentation.js";
 import { runMigrations } from "./database/migrations/runner.js";
 import { RevisionRepository } from "./database/repositories/revision.js";
 import type { ContentItem as ContentItemInternal } from "./database/repositories/types.js";
@@ -88,6 +90,7 @@ function isValidMetadataContribution(c: unknown): c is PageMetadataContribution
 	}
 }
 
+import { after } from "./after.js";
 import { loadBundleFromR2 } from "./api/handlers/marketplace.js";
 import { runSystemCleanup } from "./cleanup.js";
 import {
@@ -153,6 +156,7 @@ import { FTSManager } from "./search/fts-manager.js";
 const FIELD_TYPE_TO_KIND: Record<FieldType, string> = {
 	string: "string",
 	slug: "string",
+	url: "url",
 	text: "richText",
 	number: "number",
 	integer: "number",
@@ -286,6 +290,18 @@ export class EmDashRuntime {
 	private enabledPlugins: Set<string>;
 	private pluginStates: Map<string, string>;
 
+	private _cachedManifest: EmDashManifest | null = null;
+	private _manifestPromise: Promise<EmDashManifest> | null = null;
+	private readonly _manifestCacheKey: string;
+
+	/**
+	 * Set to true after FTS indexes have been verified for this worker
+	 * lifetime so we don't re-scan on every admin request. See
+	 * ensureSearchHealthy().
+	 */
+	private _searchHealthChecked = false;
+	private _searchHealthPromise: Promise<void> | null = null;
+
 	/** Current hook pipeline. Use the `hooks` getter for external access. */
 	get hooks(): HookPipeline {
 		return this._hooks;
@@ -344,6 +360,7 @@ export class EmDashRuntime {
 		},
 		runtimeDeps: RuntimeDependencies,
 		pipelineRef: { current: HookPipeline },
+		manifestCacheKey: string,
 	) {
 		this._db = db;
 		this.storage = storage;
@@ -364,6 +381,7 @@ export class EmDashRuntime {
 		this.pipelineFactoryOptions = pipelineFactoryOptions;
 		this.runtimeDeps = runtimeDeps;
 		this.pipelineRef = pipelineRef;
+		this._manifestCacheKey = manifestCacheKey;
 	}
 
 	/**
@@ -413,6 +431,7 @@ export class EmDashRuntime {
 			this.enabledPlugins.delete(pluginId);
 			await this.rebuildHookPipeline();
 		}
+		this.invalidateManifest();
 	}
 
 	/**
@@ -565,36 +584,46 @@ export class EmDashRuntime {
 	/**
 	 * Create and initialize the runtime
 	 */
-	static async create(deps: RuntimeDependencies): Promise<EmDashRuntime> {
-		// Initialize database
-		const db = await EmDashRuntime.getDatabase(deps);
-
-		// Verify and repair FTS indexes (auto-heal crash corruption)
-		// FTS5 is SQLite-only; on other dialects, search is a no-op until
-		// the pluggable SearchProvider work lands.
-		if (isSqlite(db)) {
+	static async create(
+		deps: RuntimeDependencies,
+		timings?: Array<{ name: string; dur: number; desc?: string }>,
+	): Promise<EmDashRuntime> {
+		// Helper: time a phase and push into the shared timings array when
+		// provided. Uses performance.now() — monotonic across async boundaries.
+		// No-op when `timings` wasn't passed (preserves backwards compatibility
+		// with callers that don't care about per-phase breakdown).
+		const phase = async <T>(name: string, desc: string, fn: () => Promise<T>): Promise<T> => {
+			if (!timings) return fn();
+			const t0 = performance.now();
 			try {
-				const ftsManager = new FTSManager(db);
-				const repaired = await ftsManager.verifyAndRepairAll();
-				if (repaired > 0) {
-					console.log(`Repaired ${repaired} corrupted FTS index(es) at startup`);
-				}
-			} catch {
-				// FTS tables may not exist yet (pre-setup). Non-fatal.
+				return await fn();
+			} finally {
+				timings.push({ name, dur: performance.now() - t0, desc });
 			}
-		}
+		};
 
-		// Initialize storage
+		// Initialize database (connects, runs migrations if needed)
+		const db = await phase("rt.db", "DB init + migrations", () => EmDashRuntime.getDatabase(deps));
+
+		// FTS verify/repair is deferred off the cold-start hot path.
+		// See EmDashRuntime.ensureSearchHealthy().
+
+		// Initialize storage (sync)
 		const storage = EmDashRuntime.getStorage(deps);
 
 		// Fetch plugin states from database
 		let pluginStates: Map<string, string> = new Map();
-		try {
-			const states = await db.selectFrom("_plugin_state").select(["plugin_id", "status"]).execute();
-			pluginStates = new Map(states.map((s) => [s.plugin_id, s.status]));
-		} catch {
-			// Plugin state table may not exist yet
-		}
+		await phase("rt.plugins", "Plugin states", async () => {
+			try {
+				const states = await db
+					.selectFrom("_plugin_state")
+					.select(["plugin_id", "status"])
+					.execute();
+				pluginStates = new Map(states.map((s) => [s.plugin_id, s.status]));
+			} catch {
+				// Plugin state table may not exist yet
+			}
+		});
 
 		// Build set of enabled plugins
 		const enabledPlugins = new Set<string>();
@@ -605,21 +634,25 @@ export class EmDashRuntime {
 			}
 		}
 
-		// Load site info for plugin context extensions
+		// Load site info for plugin context extensions (1 batch query instead of 3)
 		let siteInfo: { siteName?: string; siteUrl?: string; locale?: string } | undefined;
-		try {
-			const optionsRepo = new OptionsRepository(db);
-			const siteName = await optionsRepo.get<string>("emdash:site_title");
-			const siteUrl = await optionsRepo.get<string>("emdash:site_url");
-			const locale = await optionsRepo.get<string>("emdash:locale");
-			siteInfo = {
-				siteName: siteName ?? undefined,
-				siteUrl: siteUrl ?? undefined,
-				locale: locale ?? undefined,
-			};
-		} catch {
-			// Options table may not exist yet (pre-setup)
-		}
+		await phase("rt.site", "Site info options", async () => {
+			try {
+				const optionsRepo = new OptionsRepository(db);
+				const siteOpts = await optionsRepo.getMany<string>([
+					"emdash:site_title",
+					"emdash:site_url",
+					"emdash:locale",
+				]);
+				siteInfo = {
+					siteName: siteOpts.get("emdash:site_title") ?? undefined,
+					siteUrl: siteOpts.get("emdash:site_url") ?? undefined,
+					locale: siteOpts.get("emdash:locale") ?? undefined,
+				};
+			} catch {
+				// Options table may not exist yet (pre-setup)
+			}
+		});
 
 		// Build the full list of pipeline-eligible plugins: all configured
 		// plugins (regardless of current enabled status) plus built-in plugins.
@@ -685,11 +718,15 @@ export class EmDashRuntime {
 		const pipeline = createHookPipeline(enabledPluginList, pipelineFactoryOptions);
 
 		// Load sandboxed plugins (build-time)
-		const sandboxedPlugins = await EmDashRuntime.loadSandboxedPlugins(deps, db);
+		const sandboxedPlugins = await phase("rt.sandbox", "Sandboxed plugins", () =>
+			EmDashRuntime.loadSandboxedPlugins(deps, db),
+		);
 
 		// Cold-start: load marketplace-installed plugins from site R2
 		if (deps.config.marketplace && storage) {
-			await EmDashRuntime.loadMarketplacePlugins(db, storage, deps, sandboxedPlugins);
+			await phase("rt.market", "Marketplace plugins", () =>
+				EmDashRuntime.loadMarketplacePlugins(db, storage, deps, sandboxedPlugins),
+			);
 		}
 
 		// Initialize media providers
@@ -707,7 +744,9 @@ export class EmDashRuntime {
 		}
 
 		// Resolve exclusive hooks — auto-select providers and sync with DB
-		await EmDashRuntime.resolveExclusiveHooks(pipeline, db, deps);
+		await phase("rt.hooks", "Exclusive hook resolution", () =>
+			EmDashRuntime.resolveExclusiveHooks(pipeline, db, deps),
+		);
 
 		// ── Email pipeline ───────────────────────────────────────────────
 		// The email pipeline orchestrates beforeSend → deliver → afterSend.
@@ -740,52 +779,84 @@ export class EmDashRuntime {
 		let cronExecutor: CronExecutor | null = null;
 		let cronScheduler: CronScheduler | null = null;
 
-		try {
-			cronExecutor = new CronExecutor(db, invokeCronHook);
-
-			// Recover stale locks from previous crashes
-			const recovered = await cronExecutor.recoverStaleLocks();
-			if (recovered > 0) {
-				console.log(`[cron] Recovered ${recovered} stale task lock(s)`);
-			}
+		await phase("rt.cron", "Cron init (recovery deferred post-response)", async () => {
+			try {
+				cronExecutor = new CronExecutor(db, invokeCronHook);
+
+				// Recover stale locks from previous crashes. Pure bookkeeping
+				// against the _emdash_cron_tasks table — no request needs the
+				// result — so we defer it past the response via after(). On
+				// Cloudflare this goes into waitUntil (extending the worker
+				// lifetime); on Node it's fire-and-forget (the process stays
+				// up anyway). Saves one cold-start write per D1 isolate.
+				const executorForRecovery = cronExecutor;
+				after(async () => {
+					try {
+						const recovered = await executorForRecovery.recoverStaleLocks();
+						if (recovered > 0) {
+							console.log(`[cron] Recovered ${recovered} stale task lock(s)`);
+						}
+					} catch (error) {
+						// Keep the `[cron]` prefix so a failure is easy to trace back
+						// rather than surfacing as a generic deferred-task error.
+						console.error("[cron] Failed to recover stale task locks:", error);
+					}
+				});
 
-			// Detect platform and create appropriate scheduler.
-			// On Cloudflare Workers, setTimeout is available but unreliable for
-			// long durations — use PiggybackScheduler as default.
-			// In Node/Bun, use NodeCronScheduler with real timers.
-			const isWorkersRuntime =
-				typeof globalThis.navigator !== "undefined" &&
-				globalThis.navigator.userAgent === "Cloudflare-Workers";
-
-			if (isWorkersRuntime) {
-				cronScheduler = new PiggybackScheduler(cronExecutor);
-			} else {
-				cronScheduler = new NodeCronScheduler(cronExecutor);
-			}
+				// Detect platform and create appropriate scheduler.
+				// On Cloudflare Workers, setTimeout is available but unreliable for
+				// long durations — use PiggybackScheduler as default.
+				// In Node/Bun, use NodeCronScheduler with real timers.
+				const isWorkersRuntime =
+					typeof globalThis.navigator !== "undefined" &&
+					globalThis.navigator.userAgent === "Cloudflare-Workers";
 
-			// Register system cleanup to run alongside each scheduler tick.
-			// Pass storage so cleanupPendingUploads can delete orphaned files.
-			cronScheduler.setSystemCleanup(async () => {
-				try {
-					await runSystemCleanup(db, storage ?? undefined);
-				} catch (error) {
-					// Non-fatal -- individual cleanup failures are already logged
-					// by runSystemCleanup. This catches unexpected errors.
-					console.error("[cleanup] System cleanup failed:", error);
+				if (isWorkersRuntime) {
+					cronScheduler = new PiggybackScheduler(cronExecutor);
+				} else {
+					cronScheduler = new NodeCronScheduler(cronExecutor);
 				}
-			});
 
-			// Add cron reschedule callback (merges with existing factory options)
-			pipeline.setContextFactory({
-				cronReschedule: () => cronScheduler?.reschedule(),
-			});
+				// Register system cleanup to run alongside each scheduler tick.
+				// Pass storage so cleanupPendingUploads can delete orphaned files.
+				cronScheduler.setSystemCleanup(async () => {
+					try {
+						await runSystemCleanup(db, storage ?? undefined);
+					} catch (error) {
+						// Non-fatal -- individual cleanup failures are already logged
+						// by runSystemCleanup. This catches unexpected errors.
+						console.error("[cleanup] System cleanup failed:", error);
+					}
+				});
 
-			// Start the scheduler
-			await cronScheduler.start();
-		} catch (error) {
-			console.warn("[cron] Failed to initialize cron system:", error);
-			// Non-fatal — CMS works without cron
-		}
+				// Add cron reschedule callback (merges with existing factory options)
+				pipeline.setContextFactory({
+					cronReschedule: () => cronScheduler?.reschedule(),
+				});
+
+				// Start the scheduler
+				await cronScheduler.start();
+			} catch (error) {
+				console.warn("[cron] Failed to initialize cron system:", error);
+				// Non-fatal — CMS works without cron
+			}
+		});
+
+		// SHA of emdash commit + user config that affects the manifest.
+		// COMMIT captures emdash code changes; plugin IDs/versions and i18n
+		// capture user astro.config changes (e.g. upgrading a plugin package).
+		// DB-driven changes (collections, fields, plugin toggle) go through
+		// invalidateManifest(). Sorted for stability across nondeterministic
+		// plugin ordering.
+		const manifestCacheKey = await hashString(
+			[
+				COMMIT,
+				...deps.plugins.map((p) => `${p.id}@${p.version ?? ""}`).toSorted(),
+				...deps.sandboxedPluginEntries.map((e) => `${e.id}@${e.version}`).toSorted(),
+				virtualConfig?.i18n?.defaultLocale ?? "",
+				(virtualConfig?.i18n?.locales ?? []).toSorted().join(","),
+			].join("|"),
+		);
 
 		return new EmDashRuntime(
 			db,
@@ -806,6 +877,7 @@ export class EmDashRuntime {
 			pipelineFactoryOptions,
 			deps,
 			pipelineRef,
+			manifestCacheKey,
 		);
 	}
 
@@ -837,12 +909,14 @@ export class EmDashRuntime {
 	 * Get or create database instance
 	 */
 	private static async getDatabase(deps: RuntimeDependencies): Promise<Kysely<Database>> {
-		// If a per-request DB override is set (e.g. by the playground middleware
-		// which runs before the runtime init), use that directly. This allows
-		// the runtime to initialize against the real DO database instead of
-		// the dummy singleton dialect.
+		// Only use the per-request `ctx.db` when it's an isolated instance
+		// (playground / DO preview). Plain D1 Sessions set `ctx.db` on every
+		// anonymous request — if we captured one of those session-bound
+		// Kyselys into the cached runtime, every request would accidentally
+		// share one request's session. The configured `deps.createDialect`
+		// path gives us a fresh singleton instead.
 		const ctx = getRequestContext();
-		if (ctx?.db) {
+		if (ctx?.dbIsIsolated && ctx.db) {
 			// eslint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- db in context is typed as unknown to avoid circular deps
 			return ctx.db as Kysely<Database>;
 		}
@@ -878,9 +952,20 @@ export class EmDashRuntime {
 
 		dbInitPromise = (async () => {
 			const dialect = deps.createDialect(dbConfig.config);
-			const db = new Kysely<Database>({ dialect });
+			const db = new Kysely<Database>({ dialect, log: kyselyLogOption() });
 
-			await runMigrations(db);
+			const { applied } = await runMigrations(db);
+
+			// If migrations were applied, the schema changed — clear the
+			// DB-persisted manifest cache so getManifest() rebuilds it.
+			if (applied.length > 0) {
+				try {
+					const options = new OptionsRepository(db);
+					await options.delete("emdash:manifest_cache");
+				} catch {
+					// Non-fatal
+				}
+			}
 
 			// Auto-seed schema if no collections exist and setup hasn't run.
 			// This covers first-load on sites that skip the setup wizard.
@@ -1142,9 +1227,82 @@ export class EmDashRuntime {
 	// =========================================================================
 
 	/**
-	 * Build the manifest (rebuilt on each request for freshness)
+	 * Get the manifest, using an in-memory cache with a DB-persisted
+	 * fallback for cold starts. Avoids N+1 schema registry queries
+	 * on every request.
+	 *
+	 * Cache is invalidated by invalidateManifest(), called from schema
+	 * API routes, MCP server, plugin toggle, and taxonomy def changes.
 	 */
 	async getManifest(): Promise<EmDashManifest> {
+		// When the DB is overridden by an isolated instance (playground /
+		// DO-preview sessions), bypass the module-scoped manifest cache —
+		// its schema may diverge from the configured DB. Plain D1 Sessions
+		// routing does NOT set `dbIsIsolated`, so the cache still applies.
+		if (getRequestContext()?.dbIsIsolated) {
+			return this._buildManifest();
+		}
+
+		if (this._cachedManifest) return this._cachedManifest;
+
+		// DB-persisted cache (1 query instead of N+1 rebuild on cold start).
+		// Keyed by SHA of commit + config to bust on deploys. DB-driven
+		// changes (collections, fields, plugins, taxonomies) go through
+		// invalidateManifest().
+		try {
+			const options = new OptionsRepository(this.db);
+			const cached = await options.get<{ key: string; manifest: EmDashManifest }>(
+				"emdash:manifest_cache",
+			);
+			if (cached && cached.key === this._manifestCacheKey && cached.manifest) {
+				this._cachedManifest = cached.manifest;
+				return cached.manifest;
+			}
+		} catch {
+			// Options table may not exist yet
+		}
+
+		// Full rebuild, then persist. Track which promise is current so
+		// an invalidation during the build can't be overwritten.
+		if (!this._manifestPromise) {
+			let manifestPromise: Promise<EmDashManifest>;
+			const isCurrentLoad = () => this._manifestPromise === manifestPromise;
+			manifestPromise = this._loadManifest(isCurrentLoad);
+			this._manifestPromise = manifestPromise;
+		}
+		return this._manifestPromise;
+	}
+
+	private async _loadManifest(isCurrentLoad: () => boolean): Promise<EmDashManifest> {
+		try {
+			const manifest = await this._buildManifest();
+
+			if (isCurrentLoad()) {
+				this._cachedManifest = manifest;
+
+				try {
+					const options = new OptionsRepository(this.db);
+					await options.set("emdash:manifest_cache", {
+						key: this._manifestCacheKey,
+						manifest,
+					});
+				} catch {
+					// Non-fatal — will just rebuild next time
+				}
+			}
+
+			return manifest;
+		} finally {
+			if (isCurrentLoad()) {
+				this._manifestPromise = null;
+			}
+		}
+	}
+
+	/**
+	 * Build the manifest from database (N+1 collection queries).
+	 */
+	private async _buildManifest(): Promise<EmDashManifest> {
 		// Build collections from database.
 		// Use this.db (ALS-aware getter) so playground mode picks up the
 		// per-session DO database instead of the hardcoded singleton.
@@ -1370,11 +1528,72 @@ export class EmDashRuntime {
 
 	/**
 	 * Invalidate cached data derived from the manifest/schema.
-	 * Called when collections are created, updated, or deleted.
+	 * Called when collections, fields, plugins, or taxonomy defs change.
 	 */
 	invalidateManifest(): void {
-		// Invalidate the URL pattern cache used by resolveEmDashPath
+		this._cachedManifest = null;
+		this._manifestPromise = null;
 		invalidateUrlPatternCache();
+		// Delete DB-persisted cache so the next cold start rebuilds.
+		// Fire-and-forget: in-memory is already cleared for this worker,
+		// DB delete is best-effort for the next cold start.
+		try {
+			const options = new OptionsRepository(this.db);
+			options.delete("emdash:manifest_cache").catch((error) => {
+				console.error("Failed to delete persisted manifest cache", error);
+			});
+		} catch (error) {
+			console.error("Failed to initialize manifest cache invalidation", error);
+		}
+	}
+
+	/**
+	 * Verify and repair FTS indexes on demand. Runs at most once per worker
+	 * lifetime.
+	 *
+	 * Originally called from `EmDashRuntime.create()`, but on a busy D1 link
+	 * (e.g. SIN replica ~80-150ms per query) it added ~1.5s to every cold
+	 * start for a modest-sized site — more than every other init phase
+	 * combined. Anonymous public reads never touch the search write path,
+	 * so the cost isn't paid back for the vast majority of requests.
+	 *
+	 * Instead, search endpoints call this lazily: the first request that
+	 * actually needs the index pays the verify cost (usually fast — no
+	 * rebuild needed), everyone else runs cold-free.
+	 *
+	 * Uses the runtime's singleton database (`this._db`) rather than the
+	 * request-scoped DB. Verify reads only, but `rebuildIndex` writes, and
+	 * a GET search request on D1 carries a `first-unconstrained` session
+	 * that's free to route at a read replica — unsafe for writes. The
+	 * singleton always goes through the default binding, which the D1
+	 * adapter will promote to `first-primary` for write statements.
+	 *
+	 * Safe to call concurrently: repeated callers share the same in-flight
+	 * promise. Errors are swallowed internally so callers don't need to
+	 * defend against FTS not existing yet (pre-setup).
+	 */
+	async ensureSearchHealthy(): Promise<void> {
+		if (this._searchHealthChecked) return;
+		if (this._searchHealthPromise) return this._searchHealthPromise;
+		if (!isSqlite(this._db)) {
+			this._searchHealthChecked = true;
+			return;
+		}
+		this._searchHealthPromise = (async () => {
+			try {
+				const ftsManager = new FTSManager(this._db);
+				const repaired = await ftsManager.verifyAndRepairAll();
+				if (repaired > 0) {
+					console.log(`Repaired ${repaired} corrupted FTS index(es)`);
+				}
+			} catch {
+				// FTS tables may not exist yet (pre-setup). Non-fatal.
+			} finally {
+				this._searchHealthChecked = true;
+				this._searchHealthPromise = null;
+			}
+		})();
+		return this._searchHealthPromise;
 	}
 
 	// =========================================================================
@@ -1862,6 +2081,7 @@ export class EmDashRuntime {
 			const routeRegistry = new PluginRouteRegistry({
 				db: this.db,
 				emailPipeline: this.email ?? undefined,
+				trustedProxyHeaders: getTrustedProxyHeaders(this.config),
 			});
 			routeRegistry.register(trustedPlugin);
 
@@ -2103,7 +2323,7 @@ export class EmDashRuntime {
 
 		try {
 			const headers = sanitizeHeadersForSandbox(request.headers);
-			const meta = extractRequestMeta(request);
+			const meta = extractRequestMeta(request, this.config);
 			const result = await plugin.invokeRoute(routeName, body, {
 				url: request.url,
 				method: request.method,

package/src/import/registry.ts

@@ -4,7 +4,7 @@
  * Manages available import sources and provides URL probing.
  */
 
-import { validateExternalUrl } from "./ssrf.js";
+import { resolveAndValidateExternalUrl } from "./ssrf.js";
 import type { ImportSource, ProbeResult, SourceProbeResult } from "./types.js";
 
 // Regex pattern for URL normalization
@@ -63,8 +63,9 @@ export async function probeUrl(url: string): Promise<ProbeResult> {
 	// Remove trailing slash for consistency
 	normalizedUrl = normalizedUrl.replace(TRAILING_SLASHES_PATTERN, "");
 
-	// SSRF: reject internal/private network targets
-	validateExternalUrl(normalizedUrl);
+	// SSRF: reject internal/private network targets. DNS resolution
+	// catches hostnames that resolve to private addresses.
+	await resolveAndValidateExternalUrl(normalizedUrl);
 
 	const results: SourceProbeResult[] = [];
 	const urlSources = getUrlSources();