emdash 0.40.0 → 0.41.0

package/src/database/reference-backfill.ts

@@ -0,0 +1,141 @@
+import type { Kysely } from "kysely";
+import { sql } from "kysely";
+import { ulid } from "ulidx";
+
+import { chunks, SQL_BATCH_SIZE } from "../utils/chunks.js";
+import { currentTimestampValue, tableExists } from "./dialect-helpers.js";
+import { REFERENCE_INSERT_BATCH_SIZE } from "./repositories/relation.js";
+import { validateIdentifier } from "./validate.js";
+
+/** What to copy, and which relation to copy it into. */
+export interface ReferenceBackfill {
+	/** Collection owning the field, the relation's parent end. */
+	parentCollection: string;
+	/** The relation's child end, where the column's ids resolve. */
+	childCollection: string;
+	fieldSlug: string;
+	relationId: string;
+	/** The relation's limit for this side. `null` is unlimited. */
+	maxChildren: number | null;
+}
+
+/** The ids one column value holds. */
+function parseColumnIds(value: unknown): string[] {
+	if (typeof value !== "string" || value.length === 0) return [];
+	if (!value.startsWith("[")) return [value];
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(value);
+	} catch {
+		return [value];
+	}
+	if (!Array.isArray(parsed)) return [];
+	return parsed.filter((entry): entry is string => typeof entry === "string" && entry.length > 0);
+}
+
+/**
+ * Copy a reference field's column values in as relation edges.
+ *
+ * A reference field created before relations existed holds its selection in a
+ * TEXT column: one entry id, or a JSON array of them for a multiple-reference
+ * field (`serializeValue` stringifies the array). Binding the field to a relation
+ * moves that selection into `_emdash_content_references`, and this is the copy.
+ *
+ * The column is not touched. On a site that predates pickers it was a free-text
+ * box, so it can hold anything an editor typed, and only the ids that resolve to
+ * an entry become edges — clearing it would destroy the rest. Nothing writes to
+ * it once the field is bound and typegen stops declaring the key, but a content
+ * read still reports the frozen value in `data` beside the live `references`.
+ *
+ * Both ends of an edge are translation groups, so the locale siblings of one
+ * entry contribute to the same parent group. They are read in a fixed order and
+ * their ids deduped, and the result is capped at the relation's limit — a
+ * single-reference field whose locale rows point at different entries keeps the
+ * first rather than storing a selection the relation forbids.
+ *
+ * Every insert is `ON CONFLICT DO NOTHING` against the edge table's unique
+ * constraint, so running this twice adds nothing the first run already wrote.
+ *
+ * Migration 077 carries its own copy of this for the upgrade path. A migration
+ * must not share a module with the handler layer — the bundler then puts the two
+ * in chunks that cycle through the migration runner, and the built package throws
+ * on import — and a shipped migration has to keep behaving the way it did anyway.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- runs against a migration's untyped Kysely as well as the app's
+export async function backfillReferenceEdges(
+	db: Kysely<any>,
+	backfill: ReferenceBackfill,
+): Promise<void> {
+	const parentTable = `ec_${backfill.parentCollection}`;
+	const childTable = `ec_${backfill.childCollection}`;
+	validateIdentifier(parentTable, "content table name");
+	validateIdentifier(childTable, "content table name");
+	validateIdentifier(backfill.fieldSlug, "content field name");
+
+	if (!(await tableExists(db, parentTable)) || !(await tableExists(db, childTable))) return;
+
+	const entries = await sql<{ translation_group: string | null; value: unknown }>`
+		SELECT translation_group, ${sql.ref(backfill.fieldSlug)} AS value
+		FROM ${sql.ref(parentTable)}
+		WHERE ${sql.ref(backfill.fieldSlug)} IS NOT NULL
+		ORDER BY locale, id
+	`.execute(db);
+
+	// Parent group -> the child entry ids it selects, in order, deduped.
+	const selections = new Map<string, string[]>();
+	for (const entry of entries.rows) {
+		if (!entry.translation_group) continue;
+		const existing = selections.get(entry.translation_group) ?? [];
+		for (const id of parseColumnIds(entry.value)) {
+			if (!existing.includes(id)) existing.push(id);
+		}
+		selections.set(entry.translation_group, existing);
+	}
+	if (selections.size === 0) return;
+
+	const childIds = [...new Set([...selections.values()].flat())];
+	const childGroups = new Map<string, string>();
+	for (const batch of chunks(childIds, SQL_BATCH_SIZE)) {
+		// oxlint-disable-next-line no-await-in-loop -- one statement per bind-parameter batch
+		const resolved = await sql<{ id: string; translation_group: string | null }>`
+			SELECT id, translation_group
+			FROM ${sql.ref(childTable)}
+			WHERE id IN (${sql.join(batch)})
+		`.execute(db);
+		for (const row of resolved.rows) {
+			if (row.translation_group) childGroups.set(row.id, row.translation_group);
+		}
+	}
+
+	const now = currentTimestampValue(db);
+	const rows: Array<{ parentGroup: string; childGroup: string; sortOrder: number }> = [];
+	for (const parentGroup of [...selections.keys()].toSorted()) {
+		const groups: string[] = [];
+		for (const id of selections.get(parentGroup) ?? []) {
+			const group = childGroups.get(id);
+			// An id whose entry is gone is dropped; its value stays in the column.
+			if (group && !groups.includes(group)) groups.push(group);
+		}
+		const selected = backfill.maxChildren === null ? groups : groups.slice(0, backfill.maxChildren);
+		for (const [sortOrder, childGroup] of selected.entries()) {
+			rows.push({ parentGroup, childGroup, sortOrder });
+		}
+	}
+
+	// This runs inside the transaction that binds the field, so an editor binding
+	// a field on a large legacy collection waits for it: batch rather than paying
+	// a round trip per edge.
+	for (const batch of chunks(rows, REFERENCE_INSERT_BATCH_SIZE)) {
+		const values = batch.map(
+			(row) =>
+				sql`(${ulid()}, ${backfill.relationId}, ${row.parentGroup}, ${row.childGroup}, ${row.sortOrder}, ${now})`,
+		);
+		// oxlint-disable-next-line no-await-in-loop -- one statement per bind-parameter batch
+		await sql`
+			INSERT INTO ${sql.ref("_emdash_content_references")}
+				(id, relation_id, parent_group, child_group, sort_order, created_at)
+			VALUES ${sql.join(values)}
+			ON CONFLICT DO NOTHING
+		`.execute(db);
+	}
+}

package/src/database/repositories/content.ts

@@ -5,7 +5,7 @@ import type { ContentFieldFilterValue, ContentFieldFilters } from "../../content
 import { keepKnownFields, staleStoredKeys } from "../../content/known-fields.js";
 import { normalizeExplicitDatetime } from "../../datetime-normalization.js";
 import { invalidateCollectionCache } from "../../object-cache/index.js";
-import { isIndexableFieldType, type FieldType } from "../../schema/types.js";
+import { isIndexableFieldType, isStoragelessFieldRow, type FieldType } from "../../schema/types.js";
 import { buildFtsPrefixMatch, buildSlugGlobPrefix } from "../../search/match.js";
 import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js";
 import { isMissingTableError } from "../../utils/db-errors.js";
@@ -619,8 +619,15 @@ export class ContentRepository {
 	/**
 	 * Find content by id, including trashed (soft-deleted) items.
 	 * Used by restore endpoint for ownership checks.
+	 *
+	 * `deletedAt` rides along because the row is already in hand: a caller that
+	 * has to know whether the row is trashed before touching anything else would
+	 * otherwise pay a second read for a column this row already carries.
 	 */
-	async findByIdIncludingTrashed(type: string, id: string): Promise<ContentItem | null> {
+	async findByIdIncludingTrashed(
+		type: string,
+		id: string,
+	): Promise<(ContentItem & { deletedAt: string | null }) | null> {
 		const tableName = getTableName(type);
 
 		const result = await sql<Record<string, unknown>>`
@@ -633,7 +640,10 @@ export class ContentRepository {
 			return null;
 		}
 
-		return this.mapRow(type, row);
+		return {
+			...this.mapRow(type, row),
+			deletedAt: typeof row.deleted_at === "string" ? row.deleted_at : null,
+		};
 	}
 
 	/**
@@ -1019,7 +1029,12 @@ export class ContentRepository {
 		if (!existing) throw new EmDashValidationError("Content item not found");
 
 		const normalizedSnapshot = await this.datetimes.normalizeData(type, revisionData);
-		const { _slug, ...snapshotFields } = normalizedSnapshot;
+		const { _slug } = normalizedSnapshot;
+		// Leading-underscore keys are staged metadata (`_slug`, `_references`), not columns.
+		const snapshotFields: Record<string, unknown> = {};
+		for (const [key, value] of Object.entries(normalizedSnapshot)) {
+			if (!key.startsWith("_")) snapshotFields[key] = value;
+		}
 		const fieldData = writableContentData(snapshotFields);
 		const revisionId = createRevisionId();
 		const now = new Date().toISOString();
@@ -2195,13 +2210,26 @@ export class ContentRepository {
 		return result.rows.map((row) => this.mapRow(type, row));
 	}
 
-	/** Whether any row of `translationGroup` exists, trashed rows included. */
-	async hasTranslationsIncludingTrashed(type: string, translationGroup: string): Promise<boolean> {
+	/**
+	 * Whether any row still shares `translationGroup`, trashed rows included.
+	 * Group-keyed satellite data (term assignments, reference edges) is owned by
+	 * the group, not by a single locale row, so a purge may only cascade to it
+	 * once nothing is left to own it — and a trashed sibling is still restorable.
+	 */
+	async hasTranslationsIncludingTrashed(
+		type: string,
+		translationGroup: string,
+		options: { excludeId?: string } = {},
+	): Promise<boolean> {
 		const tableName = getTableName(type);
+		// Asked before a row is deleted, "does anything else hold this group?"
+		// has to leave that row out of the answer.
+		const exclusion = options.excludeId ? sql`AND id != ${options.excludeId}` : sql``;
 
 		const result = await sql<Record<string, unknown>>`
 			SELECT id FROM ${sql.ref(tableName)}
 			WHERE translation_group = ${translationGroup}
+			${exclusion}
 			LIMIT 1
 		`.execute(this.db);
 
@@ -2970,9 +2998,16 @@ export class ContentRepository {
 			.where("collection.slug", "=", type)
 			.where("field.slug", "in", fields)
 			.where("field.indexed", "=", 1)
-			.select(["field.slug", "field.type"])
+			.select(["field.slug", "field.type", "field.validation"])
 			.execute();
-		const metadata = new Map(rows.map((row) => [row.slug, row.type as FieldType]));
+		// `indexed` alone is not enough: a storage-less field has no column to
+		// filter on. A reference field bound to a relation after it was indexed can
+		// still carry the flag, and its column no longer receives writes.
+		const metadata = new Map(
+			rows
+				.filter((row) => !isStoragelessFieldRow(row))
+				.map((row) => [row.slug, row.type as FieldType]),
+		);
 
 		if (metadata.size === 0 && !(await this.collectionExists(type))) return [];
 

package/src/database/repositories/media.ts

@@ -8,6 +8,7 @@ import {
 import { ulid } from "ulidx";
 
 import { normalizeFocalPoint, type FocalPointUpdate } from "../../media/focal-point.js";
+import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js";
 import type { Database, MediaRow } from "../types.js";
 import type { FindManyResult } from "./types.js";
 import { encodeCursor, decodeCursor } from "./types.js";
@@ -393,6 +394,20 @@ export class MediaRepository {
 		return row ? this.rowToItem(row) : null;
 	}
 
+	/**
+	 * Find the pending media row minted for a signed upload URL.
+	 */
+	async findPendingByStorageKey(storageKey: string): Promise<MediaItem | null> {
+		const row = await this.db
+			.selectFrom("media")
+			.selectAll()
+			.where("storage_key", "=", storageKey)
+			.where("status", "=", "pending")
+			.executeTakeFirst();
+
+		return row ? this.rowToItem(row) : null;
+	}
+
 	/**
 	 * Find media by ID
 	 */
@@ -614,6 +629,15 @@ export class MediaRepository {
 		return null;
 	}
 
+	async isStorageKeyReferenced(storageKey: string): Promise<boolean> {
+		const row = await this.db
+			.selectFrom("media")
+			.select("id")
+			.where("storage_key", "=", storageKey)
+			.executeTakeFirst();
+		return row !== undefined;
+	}
+
 	async delete(id: string): Promise<boolean> {
 		return (await this.deleteWithStorageKey(id)) !== null;
 	}
@@ -682,7 +706,22 @@ export class MediaRepository {
 			.returning("storage_key")
 			.execute();
 
-		return rows.map((r) => r.storage_key);
+		const keys = rows.map((r) => r.storage_key);
+		if (keys.length === 0) return keys;
+
+		// A stored object may still back another media row (for example after a
+		// legacy duplicate registration); never hand such a key to storage deletion.
+		const stillReferenced = new Set<string>();
+		for (const batch of chunks(keys, SQL_BATCH_SIZE)) {
+			const refs = await this.db
+				.selectFrom("media")
+				.select("storage_key")
+				.where("storage_key", "in", batch)
+				.execute();
+			for (const ref of refs) stillReferenced.add(ref.storage_key);
+		}
+
+		return keys.filter((key) => !stillReferenced.has(key));
 	}
 
 	/**

package/src/database/repositories/redirect.ts

@@ -17,9 +17,9 @@ import { encodeCursor, decodeCursor, type FindManyResult } from "./types.js";
 // ---------------------------------------------------------------------------
 
 /**
- * Hard cap on rows stored in `_emdash_404_log`. When exceeded, the oldest
- * rows (by `last_seen_at`) are evicted on insert. Prevents an unauthenticated
- * attacker from growing the table without bound by requesting unique URLs.
+ * Hard cap on rows stored in `_emdash_404_log`. Scheduled maintenance evicts
+ * the oldest rows by `last_seen_at` without adding a read-amplifying count to
+ * the anonymous request path.
  */
 export const MAX_404_LOG_ROWS = 10_000;
 
@@ -646,7 +646,7 @@ export class RedirectRepository {
 		// under concurrency: two requests for the same new path can't both
 		// insert — the second one hits the conflict branch and increments
 		// hits instead of failing with a uniqueness error.
-		const result = await this.db
+		await this.db
 			.insertInto("_emdash_404_log")
 			.values({
 				id,
@@ -667,14 +667,7 @@ export class RedirectRepository {
 					ip,
 				}),
 			)
-			.returning("id")
-			.executeTakeFirst();
-
-		// The conflict branch only updates existing rows, so repeat hits
-		// cannot grow the table. Only enforce the row cap when we actually
-		// inserted a new path.
-		if (result?.id !== id) return;
-		await this.enforce404Cap();
+			.execute();
 	}
 
 	/**
@@ -682,36 +675,26 @@ export class RedirectRepository {
 	 * MAX_404_LOG_ROWS. "Oldest" is by `last_seen_at`, so a path that keeps
 	 * getting hit stays in the table even if it was first seen long ago.
 	 *
-	 * Private — called by `log404` only after a new path was inserted.
+	 * Called by scheduled system cleanup, never by the anonymous request path.
 	 */
-	private async enforce404Cap(): Promise<void> {
-		const countRow = await this.db
-			.selectFrom("_emdash_404_log")
-			.select((eb) => eb.fn.countAll<number>().as("c"))
-			.executeTakeFirst();
-		const count = Number(countRow?.c ?? 0);
-		if (count <= MAX_404_LOG_ROWS) return;
-
-		const excess = count - MAX_404_LOG_ROWS;
-
-		// Evict the oldest rows in a single SQL statement. Using a subquery
-		// (rather than materialising the victim IDs in JS and passing them
-		// back as bind parameters) keeps the statement bounded regardless of
-		// how far over cap the table is — important for existing installs
-		// that crossed the threshold before this cap was introduced.
-		await this.db
+	async cleanup404Log(): Promise<number> {
+		// Keep the newest rows in one statement. Deriving the victims inside the
+		// DELETE makes overlapping cleanup runs idempotent: each statement
+		// evaluates the current newest set instead of acting on a stale count.
+		const result = await this.db
 			.deleteFrom("_emdash_404_log")
 			.where(
 				"id",
-				"in",
+				"not in",
 				this.db
 					.selectFrom("_emdash_404_log")
 					.select("id")
-					.orderBy("last_seen_at", "asc")
-					.orderBy("id", "asc")
-					.limit(excess),
+					.orderBy("last_seen_at", "desc")
+					.orderBy("id", "desc")
+					.limit(MAX_404_LOG_ROWS),
 			)
-			.execute();
+			.executeTakeFirst();
+		return Number(result.numDeletedRows);
 	}
 
 	async find404s(opts: {