@coywolf/emdash 0.0.0-stage → 0.35.14

package/src/core/csv.ts

@@ -0,0 +1,119 @@
+/**
+ * CSV for exports and imports (RFC 4180): quoted fields, doubled quotes,
+ * commas and line breaks inside quotes. No imports, so the admin bundle and
+ * `node --test` can both load it.
+ *
+ * Spreadsheet apps run a cell that starts with = + - @ (or a tab or carriage
+ * return) as a formula, so exports meant for spreadsheets pass `guard: true`:
+ * such cells get a leading apostrophe, which spreadsheets hide. Cells that
+ * start with apostrophes followed by one of those characters get one more, so
+ * `unguardCell` can always undo the guard exactly and a guarded export still
+ * imports back to the same values.
+ */
+
+export type CsvCell = string | number | boolean | null | undefined;
+
+const FORMULA = /^'*[=+\-@\t\r]/;
+const GUARDED = /^'+[=+\-@\t\r]/;
+
+/** Neutralize a cell a spreadsheet would treat as a formula. Reversible with unguardCell. */
+export function guardCell(value: string): string {
+	return FORMULA.test(value) ? `'${value}` : value;
+}
+
+/** Undo guardCell. Values that weren't guarded come back unchanged. */
+export function unguardCell(value: string): string {
+	return GUARDED.test(value) ? value.slice(1) : value;
+}
+
+function cellText(value: CsvCell): string {
+	if (value === null || value === undefined) return "";
+	if (typeof value === "boolean") return value ? "1" : "0";
+	return String(value);
+}
+
+/** Quote a field when it holds a comma, quote, tab or line break (or leading/trailing spaces, which some readers trim). */
+function quote(text: string): string {
+	return /[",\t\r\n]|^\s|\s$/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+
+/** Rows to CSV text (CRLF line ends, as RFC 4180 and Excel expect). */
+export function toCsv(rows: CsvCell[][], options: { guard?: boolean } = {}): string {
+	return `${rows
+		.map((row) =>
+			row
+				.map((value) => {
+					const text = cellText(value);
+					return quote(options.guard && typeof value === "string" ? guardCell(text) : text);
+				})
+				.join(","),
+		)
+		.join("\r\n")}\r\n`;
+}
+
+/**
+ * Parse CSV (or tab-separated) text into rows. The delimiter defaults to a
+ * tab when the first line has one (toCsv quotes tabs, so its header never
+ * does), else a comma. A leading
+ * byte-order mark is dropped, and blank lines are skipped.
+ */
+export function parseCsv(text: string, delimiter?: string): string[][] {
+	const source = text.replace(/^/, "");
+	const firstLine = source.split(/\r?\n/, 1)[0] ?? "";
+	const sep = delimiter ?? (firstLine.includes("\t") ? "\t" : ",");
+	const rows: string[][] = [];
+	let row: string[] = [];
+	let field = "";
+	let quoted = false;
+	let i = 0;
+	const endRow = () => {
+		row.push(field);
+		if (row.length > 1 || row[0] !== "") rows.push(row);
+		row = [];
+		field = "";
+	};
+	while (i < source.length) {
+		const c = source[i];
+		if (quoted) {
+			if (c === '"') {
+				if (source[i + 1] === '"') {
+					field += '"';
+					i += 2;
+					continue;
+				}
+				quoted = false;
+				i++;
+				continue;
+			}
+			field += c;
+			i++;
+			continue;
+		}
+		if (c === '"' && field.trim() === "") {
+			// A quote opens a quoted field only at its start (spaces before it are dropped).
+			field = "";
+			quoted = true;
+			i++;
+		} else if (c === sep) {
+			row.push(field);
+			field = "";
+			i++;
+		} else if (c === "\r" || c === "\n") {
+			endRow();
+			i += c === "\r" && source[i + 1] === "\n" ? 2 : 1;
+		} else {
+			field += c;
+			i++;
+		}
+	}
+	if (field !== "" || row.length) endRow();
+	return rows;
+}
+
+/** CSV rows to objects keyed by the (lowercased, trimmed) header row. */
+export function csvRecords(text: string, delimiter?: string): Array<Record<string, string>> {
+	const [header, ...rows] = parseCsv(text, delimiter);
+	if (!header) return [];
+	const keys = header.map((h) => h.trim().toLowerCase());
+	return rows.map((row) => Object.fromEntries(keys.map((key, i) => [key, row[i] ?? ""])));
+}

package/src/core/d1-batch.ts

@@ -0,0 +1,65 @@
+/**
+ * Same-tick batching for the pack's own D1 reads. The pack reads the site
+ * database through the raw binding (env.DB), which EmDash's per-request
+ * session dialect never sees, so its reads were never coalesced: each was its
+ * own round trip. Reads issued through here in the same tick (before a
+ * zero-delay timer fires) go to D1 as one `db.batch()`: one round trip.
+ *
+ * A statement that fails fails the whole batch in D1, so on a batch error each
+ * statement is retried on its own, and only the failing ones reject. Reads of
+ * different requests are never batched together (Workers ties I/O to the
+ * request that started it).
+ */
+import { requestStore } from "./request-memo.js";
+
+interface Pending {
+	statement: D1PreparedStatement;
+	resolve(result: D1Result<unknown>): void;
+	reject(error: unknown): void;
+}
+
+/** Request context (or OUTSIDE) → database → reads waiting for this tick's batch. */
+const queues = new WeakMap<object, Map<D1Database, Pending[]>>();
+const OUTSIDE = {};
+
+function flush(scope: Map<D1Database, Pending[]>, db: D1Database): void {
+	const queue = scope.get(db) ?? [];
+	scope.delete(db);
+	if (!queue.length) return;
+	if (queue.length === 1 || typeof db.batch !== "function") {
+		for (const p of queue) p.statement.all().then(p.resolve, p.reject);
+		return;
+	}
+	db.batch(queue.map((p) => p.statement)).then(
+		(results) => queue.forEach((p, i) => p.resolve(results[i] as D1Result<unknown>)),
+		() => {
+			// One bad statement fails a D1 batch: run them one by one so the others still answer.
+			for (const p of queue) p.statement.all().then(p.resolve, p.reject);
+		},
+	);
+}
+
+/** Run a read with the other reads of this tick, in one D1 batch. Resolves to its rows. */
+export function batchedAll<T = Record<string, unknown>>(db: D1Database, statement: D1PreparedStatement): Promise<T[]> {
+	return new Promise<D1Result<unknown>>((resolve, reject) => {
+		const owner = requestStore() ?? OUTSIDE;
+		let scope = queues.get(owner);
+		if (!scope) {
+			scope = new Map();
+			queues.set(owner, scope);
+		}
+		let queue = scope.get(db);
+		if (!queue) {
+			queue = [];
+			scope.set(db, queue);
+			const pending = scope;
+			setTimeout(() => flush(pending, db), 0);
+		}
+		queue.push({ statement, resolve, reject });
+	}).then((result) => (result.results ?? []) as T[]);
+}
+
+/** Like batchedAll, the first row (null when none). */
+export async function batchedFirst<T = Record<string, unknown>>(db: D1Database, statement: D1PreparedStatement): Promise<T | null> {
+	return (await batchedAll<T>(db, statement))[0] ?? null;
+}

package/src/core/entry-docs.ts

@@ -0,0 +1,175 @@
+/**
+ * Everything the pack stores about one entry, read for a page render in one
+ * D1 round trip: its plugin-storage docs in several collections (Schema
+ * overrides, the Videos embed index, AI entities…, all keyed
+ * "<collection>:<entry id>") in a single query, plus, when asked, the
+ * entry's own row (for blocks the theme didn't hand over), batched with it.
+ *
+ * Before, each module read its doc on its own (three to five queries per
+ * page). Without a D1 binding (tests, other databases) it falls back to the
+ * plugin context's storage and content APIs, one read each, at once.
+ */
+import type { PluginContext, StorageCollection } from "emdash";
+
+import { batchedAll, batchedFirst } from "./d1-batch.js";
+import { PLUGIN_ID } from "./features.js";
+import { noteReadFailure } from "./read-failures.js";
+import { peekRequestMemo, seedRequestMemo } from "./request-memo.js";
+import { workerEnv } from "../shared.js";
+
+export interface EntryDocsRequest {
+	collection: string;
+	id: string;
+	/** Plugin-storage collections whose doc for this entry (id "<collection>:<entry id>") is wanted. */
+	collections: readonly string[];
+	/** Also read the entry's own data (its row in ec_<collection>). */
+	data?: boolean;
+	/** D1 binding of the site database. Default "DB". */
+	database?: string;
+}
+
+export interface EntryDocs {
+	/** Storage collection → this entry's doc (null when there's none). */
+	docs: Map<string, unknown>;
+	/** The entry's fields (JSON fields parsed), when asked for; null when the entry doesn't exist. */
+	data?: Record<string, unknown> | null;
+}
+
+/** ec_<collection> names EmDash accepts (validateIdentifier). */
+const IDENTIFIER = /^[a-z][a-z0-9_]*$/;
+
+type Row = Record<string, unknown>;
+
+/**
+ * An entry row's fields: JSON text columns (Portable Text, objects, lists)
+ * parsed. Only columns that look like JSON are parsed, and `mustContain`
+ * skips parsing large columns that can't hold what the caller looks for.
+ */
+export function rowData(row: Row | null | undefined, mustContain?: string): Record<string, unknown> | null {
+	if (!row) return null;
+	const out: Record<string, unknown> = {};
+	for (const [key, value] of Object.entries(row)) {
+		if (typeof value === "string" && (value.startsWith("[") || value.startsWith("{"))) {
+			if (mustContain && !value.includes(mustContain)) continue;
+			try {
+				out[key] = JSON.parse(value);
+				continue;
+			} catch {
+				// Not JSON after all: keep the text.
+			}
+		}
+		out[key] = value;
+	}
+	return out;
+}
+
+async function readFromD1(db: D1Database, req: EntryDocsRequest, mustContain?: string): Promise<EntryDocs> {
+	const key = `${req.collection}:${req.id}`;
+	const wantData = req.data && IDENTIFIER.test(req.collection);
+	// Both go out with the page's other pack reads of this tick (one D1 batch).
+	const [rows, entry] = await Promise.all([
+		req.collections.length
+			? batchedAll<{ collection: string; data: string | null }>(
+					db,
+					db
+						.prepare(`SELECT collection, data FROM _plugin_storage WHERE plugin_id = ? AND id = ? AND collection IN (${req.collections.map(() => "?").join(",")})`)
+						.bind(PLUGIN_ID, key, ...req.collections),
+				)
+			: [],
+		wantData ? batchedFirst<Row>(db, db.prepare(`SELECT * FROM "ec_${req.collection}" WHERE id = ? AND deleted_at IS NULL LIMIT 1`).bind(req.id)) : null,
+	]);
+	const docs = new Map<string, unknown>(req.collections.map((c) => [c, null]));
+	for (const row of rows) {
+		try {
+			docs.set(row.collection, row.data ? JSON.parse(row.data) : null);
+		} catch {
+			docs.set(row.collection, null);
+		}
+	}
+	const out: EntryDocs = { docs };
+	if (req.data) out.data = wantData ? rowData(entry, mustContain) : null;
+	return out;
+}
+
+async function readFromContext(ctx: PluginContext, req: EntryDocsRequest): Promise<EntryDocs> {
+	const key = `${req.collection}:${req.id}`;
+	const storage = ctx.storage as Record<string, StorageCollection | undefined>;
+	const [values, entry] = await Promise.all([
+		Promise.all(req.collections.map((c) => storage[c]?.get(key) ?? null)),
+		req.data && ctx.content
+			? ctx.content.get(req.collection, req.id).catch(() => {
+					noteReadFailure("entry data");
+					return null;
+				})
+			: null,
+	]);
+	const out: EntryDocs = { docs: new Map(req.collections.map((c, i) => [c, values[i] ?? null])) };
+	if (req.data) out.data = (entry?.data as Record<string, unknown> | undefined) ?? null;
+	return out;
+}
+
+/**
+ * Read an entry's docs (and data). One D1 batch when the site database is
+ * bound; otherwise the plugin context's reads. `mustContain` (optional): only
+ * JSON columns containing it are parsed from a D1 row.
+ */
+async function readUncached(ctx: PluginContext, req: EntryDocsRequest, mustContain?: string): Promise<EntryDocs> {
+	let db: D1Database | undefined;
+	try {
+		db = (await workerEnv())[req.database ?? "DB"] as D1Database | undefined;
+	} catch {
+		db = undefined; // Not on Workers.
+	}
+	if (db && typeof db.prepare === "function") {
+		try {
+			return await readFromD1(db, req, mustContain);
+		} catch (error) {
+			ctx.log?.warn?.("coywolf-pack: entry docs read failed; reading them one by one", { error: String(error) });
+			// Without content access the fallback can't read the entry's data (it reads as null).
+			if (req.data && !ctx.content) noteReadFailure("entry data");
+		}
+	}
+	return readFromContext(ctx, req);
+}
+
+const docKey = (req: EntryDocsRequest, collection: string) => `coywolf-entry-doc:${collection}:${req.collection}:${req.id}`;
+const dataKey = (req: EntryDocsRequest, mustContain?: string) => `coywolf-entry-data:${req.collection}:${req.id}:${mustContain ?? ""}`;
+
+/**
+ * Read an entry's docs (and data), each at most once per request: callers
+ * that ask for the same entry (the page:metadata hook run more than once,
+ * modules' own hooks) share the reads. What isn't read yet in this request
+ * is one D1 batch when the site database is bound; otherwise the plugin
+ * context's reads. `mustContain` (optional): only JSON columns containing it
+ * are parsed from a D1 row.
+ */
+export async function readEntryDocs(ctx: PluginContext, req: EntryDocsRequest, mustContain?: string): Promise<EntryDocs> {
+	const known = new Map<string, Promise<unknown>>();
+	const missing: string[] = [];
+	for (const c of req.collections) {
+		const hit = peekRequestMemo<unknown>(docKey(req, c));
+		if (hit) known.set(c, hit);
+		else missing.push(c);
+	}
+	const dataHit = req.data ? peekRequestMemo<Record<string, unknown> | null>(dataKey(req, mustContain)) : undefined;
+	const wantData = Boolean(req.data && !dataHit);
+	let data: Promise<Record<string, unknown> | null> | undefined = dataHit;
+	if (missing.length || wantData) {
+		const read = readUncached(ctx, { ...req, collections: missing, data: wantData }, mustContain);
+		for (const c of missing) {
+			const one = read.then((d) => d.docs.get(c) ?? null);
+			seedRequestMemo(docKey(req, c), one);
+			known.set(c, one);
+		}
+		if (wantData) {
+			data = read.then((d) => d.data ?? null);
+			seedRequestMemo(dataKey(req, mustContain), data);
+		}
+		await read;
+	}
+	const collections = [...known];
+	const values = await Promise.all(collections.map(([, v]) => v));
+	const out: EntryDocs = { docs: new Map(collections.map(([c], i) => [c, values[i]])) };
+	if (req.data) out.data = (await data) ?? null;
+	return out;
+}

package/src/core/features-module.ts

@@ -0,0 +1,48 @@
+/**
+ * Routes behind the Features admin page: list every feature with its state,
+ * and save the switches.
+ */
+import { definePluginRoute } from "emdash";
+import { z } from "zod";
+
+import { parseInput } from "../shared.js";
+import { ensureTasks } from "./compose.js";
+import { FEATURES_SETTING, type FeatureMap, ctxFeatures, featureCatalog, invalidateFeatures } from "./features.js";
+import type { PackModule, TaskDef } from "./module.js";
+
+export function featuresRoutes(modules: PackModule[], tasks: TaskDef[]) {
+	return {
+		"features/list": {
+			permission: "plugins:manage" as const,
+			// biome-ignore lint/suspicious/noExplicitAny: plugin context.
+			handler: async (ctx: any) => {
+				await ensureTasks(ctx, tasks);
+				const state = await ctxFeatures(ctx);
+				const known = new Set(featureCatalog().map((f) => f.id));
+				return {
+					modules: modules.map((m) => ({
+						id: m.id,
+						label: m.label,
+						features: m.features.filter((f) => known.has(f.id)).map((f) => ({ ...f, enabled: state[f.id] ?? false })),
+					})),
+				};
+			},
+		},
+
+		"features/save": definePluginRoute({
+			permission: "plugins:manage",
+			methods: ["POST"],
+			request: { body: "json" },
+			handler: async (ctx) => {
+				const { features } = parseInput(z.object({ features: z.record(z.string(), z.boolean()) }), ctx.input);
+				const known = new Set(featureCatalog().map((f) => f.id));
+				const stored = ((await ctx.settings.get<FeatureMap>(FEATURES_SETTING)) ?? {}) as FeatureMap;
+				for (const [id, on] of Object.entries(features)) if (known.has(id)) stored[id] = on;
+				await ctx.settings.set(FEATURES_SETTING, stored);
+				invalidateFeatures();
+				ctx.log.info("Features saved", stored);
+				return { features: await ctxFeatures(ctx) };
+			},
+		}),
+	};
+}