emdash 0.26.0 → 0.28.0

package/src/media/usage/content-refresh.ts

@@ -0,0 +1,524 @@
+import { sql, type Kysely } from "kysely";
+
+import { MediaUsageRepository } from "../../database/repositories/media-usage.js";
+import type { Database } from "../../database/types.js";
+import { validateIdentifier } from "../../database/validate.js";
+import { isI18nEnabled } from "../../i18n/config.js";
+import { loadContentMediaUsageFields } from "./content-fields.js";
+import {
+	CONTENT_SOURCE_SCHEMA_VERSION,
+	loadContentMediaUsageSnapshots,
+} from "./content-snapshots.js";
+import {
+	buildContentMediaUsageSourceKey,
+	MEDIA_USAGE_CONTENT_SOURCE_VARIANTS,
+} from "./source-key.js";
+
+export const CONTENT_MEDIA_USAGE_ADAPTER_ID = "content-media";
+export const CONTENT_MEDIA_USAGE_COLLECTION_SCOPE = "collection";
+
+const CONTENT_USAGE_LOCKS_KEY = Symbol.for("emdash.mediaUsage.contentLocks");
+const CONTENT_USAGE_COLLECTION_LOCKS_KEY = Symbol.for("emdash.mediaUsage.collectionLocks");
+const CONTENT_USAGE_REFRESH_MAX_ATTEMPTS = 2;
+
+// These maps only de-dupe usage work inside the current isolate/process. Cross-worker
+// correctness comes from expected-generation guards on repository writes.
+
+export type ContentMediaUsageRefreshErrorCode =
+	| "CONTENT_NOT_FOUND"
+	| "DRAFT_REVISION_NOT_FOUND"
+	| "DRAFT_REVISION_MISMATCH"
+	| "DRAFT_REVISION_INVALID"
+	| "CONTENT_USAGE_REFRESH_ERROR"
+	| "CONTENT_USAGE_DELETE_ERROR"
+	| "CONTENT_USAGE_GENERATION_CONFLICT"
+	| "CONTENT_USAGE_STALE";
+
+export interface ContentMediaUsageRefreshResult {
+	success: boolean;
+	refreshedSourceCount: number;
+	deletedSourceCount: number;
+	failedSourceCount: number;
+	errorCode?: ContentMediaUsageRefreshErrorCode;
+}
+
+const ZERO_RESULT: ContentMediaUsageRefreshResult = {
+	success: true,
+	refreshedSourceCount: 0,
+	deletedSourceCount: 0,
+	failedSourceCount: 0,
+};
+
+export async function refreshContentMediaUsage(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	validateIdentifier(collectionSlug, "collection slug");
+	return withContentUsageCollectionLock(collectionSlug, () =>
+		withContentUsageLock(collectionSlug, contentId, () =>
+			refreshContentMediaUsageUnlocked(db, collectionSlug, contentId),
+		),
+	);
+}
+
+async function refreshContentMediaUsageUnlocked(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	try {
+		let conflictResult: ContentMediaUsageRefreshResult | null = null;
+		for (let attempt = 0; attempt < CONTENT_USAGE_REFRESH_MAX_ATTEMPTS; attempt++) {
+			const result = await refreshContentMediaUsageAttempt(db, collectionSlug, contentId);
+			if (result.errorCode !== "CONTENT_USAGE_GENERATION_CONFLICT") return result;
+			conflictResult = result;
+		}
+
+		return markGenerationConflict(db, collectionSlug, {
+			refreshedSourceCount: conflictResult?.refreshedSourceCount ?? 0,
+			deletedSourceCount: conflictResult?.deletedSourceCount ?? 0,
+		});
+	} catch (error) {
+		console.error(`[media-usage] Failed to refresh ${collectionSlug}/${contentId}:`, error);
+		await markContentMediaUsageCollectionStaleSafely(
+			db,
+			collectionSlug,
+			"CONTENT_USAGE_REFRESH_ERROR",
+		);
+		return {
+			success: false,
+			refreshedSourceCount: 0,
+			deletedSourceCount: 0,
+			failedSourceCount: 0,
+			errorCode: "CONTENT_USAGE_REFRESH_ERROR",
+		};
+	}
+}
+
+async function refreshContentMediaUsageAttempt(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	const repo = new MediaUsageRepository(db);
+	const observedGenerations = await loadObservedContentSourceGenerations(
+		db,
+		collectionSlug,
+		contentId,
+	);
+	const snapshotsResult = await loadContentMediaUsageSnapshots(db, collectionSlug, contentId);
+	if (!snapshotsResult.success) {
+		return markSnapshotFailure(db, collectionSlug, snapshotsResult);
+	}
+
+	if (!(await contentCollectionExists(db, collectionSlug))) {
+		const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId);
+		return { ...ZERO_RESULT, deletedSourceCount };
+	}
+
+	let refreshedSourceCount = 0;
+	for (const snapshot of snapshotsResult.snapshots) {
+		const result = await repo.replaceSourceIfCurrent(
+			snapshot.source,
+			snapshot.occurrences,
+			observedGenerations.get(snapshot.source.sourceKey) ?? null,
+		);
+		if (!result.replaced) {
+			return generationConflictResult({
+				refreshedSourceCount,
+				deletedSourceCount: 0,
+			});
+		}
+		refreshedSourceCount++;
+	}
+	if (!(await contentCollectionExists(db, collectionSlug))) {
+		const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId);
+		return { ...ZERO_RESULT, deletedSourceCount };
+	}
+
+	const expectedSourceKeys = new Set(
+		snapshotsResult.snapshots.map((snapshot) => snapshot.source.sourceKey),
+	);
+	const absentSourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) =>
+		buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }),
+	).filter((sourceKey) => !expectedSourceKeys.has(sourceKey));
+	let deletedSourceCount = 0;
+	for (const sourceKey of absentSourceKeys) {
+		const expectedGeneration = observedGenerations.get(sourceKey) ?? null;
+		if (expectedGeneration === null) continue;
+
+		const result = await repo.deleteSourceIfCurrent(sourceKey, expectedGeneration);
+		if (result.deleted) {
+			deletedSourceCount++;
+			continue;
+		}
+		if (result.source) {
+			return generationConflictResult({
+				refreshedSourceCount,
+				deletedSourceCount,
+			});
+		}
+	}
+
+	return {
+		success: true,
+		refreshedSourceCount,
+		deletedSourceCount,
+		failedSourceCount: 0,
+	};
+}
+
+async function loadObservedContentSourceGenerations(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<Map<string, string | null>> {
+	const generations = new Map<string, string | null>();
+	const sourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) =>
+		buildContentMediaUsageSourceKey({
+			collectionSlug,
+			contentId,
+			sourceVariant,
+		}),
+	);
+	for (const sourceKey of sourceKeys) {
+		generations.set(sourceKey, null);
+	}
+
+	const rows = await db
+		.selectFrom("_emdash_media_usage_sources")
+		.select(["source_key", "current_generation"])
+		.where("source_key", "in", sourceKeys)
+		.execute();
+	for (const row of rows) {
+		generations.set(row.source_key, row.current_generation);
+	}
+
+	return generations;
+}
+
+async function markGenerationConflict(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	counts: Pick<ContentMediaUsageRefreshResult, "refreshedSourceCount" | "deletedSourceCount">,
+): Promise<ContentMediaUsageRefreshResult> {
+	await markContentMediaUsageCollectionStaleSafely(
+		db,
+		collectionSlug,
+		"CONTENT_USAGE_GENERATION_CONFLICT",
+	);
+	return {
+		success: false,
+		refreshedSourceCount: counts.refreshedSourceCount,
+		deletedSourceCount: counts.deletedSourceCount,
+		failedSourceCount: 0,
+		errorCode: "CONTENT_USAGE_GENERATION_CONFLICT",
+	};
+}
+
+function generationConflictResult(
+	counts: Pick<ContentMediaUsageRefreshResult, "refreshedSourceCount" | "deletedSourceCount">,
+): ContentMediaUsageRefreshResult {
+	return {
+		success: false,
+		refreshedSourceCount: counts.refreshedSourceCount,
+		deletedSourceCount: counts.deletedSourceCount,
+		failedSourceCount: 0,
+		errorCode: "CONTENT_USAGE_GENERATION_CONFLICT",
+	};
+}
+
+async function contentCollectionExists(
+	db: Kysely<Database>,
+	collectionSlug: string,
+): Promise<boolean> {
+	const row = await db
+		.selectFrom("_emdash_collections")
+		.select("id")
+		.where("slug", "=", collectionSlug)
+		.executeTakeFirst();
+	return row !== undefined;
+}
+
+export async function deleteContentMediaUsage(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	validateIdentifier(collectionSlug, "collection slug");
+	return withContentUsageCollectionLock(collectionSlug, () =>
+		withContentUsageLock(collectionSlug, contentId, () =>
+			deleteContentMediaUsageUnlocked(db, collectionSlug, contentId),
+		),
+	);
+}
+
+async function deleteContentMediaUsageUnlocked(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	try {
+		const deletedSourceCount = await new MediaUsageRepository(db).deleteContentSources(
+			collectionSlug,
+			contentId,
+		);
+		return { ...ZERO_RESULT, deletedSourceCount };
+	} catch (error) {
+		console.error(
+			`[media-usage] Failed to delete usage for ${collectionSlug}/${contentId}:`,
+			error,
+		);
+		await markContentMediaUsageCollectionStaleSafely(
+			db,
+			collectionSlug,
+			"CONTENT_USAGE_DELETE_ERROR",
+		);
+		return {
+			success: false,
+			refreshedSourceCount: 0,
+			deletedSourceCount: 0,
+			failedSourceCount: 0,
+			errorCode: "CONTENT_USAGE_DELETE_ERROR",
+		};
+	}
+}
+
+export async function deleteContentMediaUsageCollection(
+	db: Kysely<Database>,
+	collectionSlug: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	validateIdentifier(collectionSlug, "collection slug");
+	return withContentUsageCollectionLock(collectionSlug, () =>
+		deleteContentMediaUsageCollectionUnlocked(db, collectionSlug),
+	);
+}
+
+async function deleteContentMediaUsageCollectionUnlocked(
+	db: Kysely<Database>,
+	collectionSlug: string,
+): Promise<ContentMediaUsageRefreshResult> {
+	try {
+		const repo = new MediaUsageRepository(db);
+		const deletedSourceCount = await repo.deleteCollectionSources(collectionSlug);
+		await repo.deleteIndexStatus({
+			adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID,
+			scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE,
+			scopeKey: collectionSlug,
+		});
+		return { ...ZERO_RESULT, deletedSourceCount };
+	} catch (error) {
+		console.error(`[media-usage] Failed to delete usage for collection ${collectionSlug}:`, error);
+		try {
+			await new MediaUsageRepository(db).deleteIndexStatus({
+				adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID,
+				scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE,
+				scopeKey: collectionSlug,
+			});
+		} catch (statusError) {
+			console.error(
+				`[media-usage] Failed to clear usage status for deleted collection ${collectionSlug}:`,
+				statusError,
+			);
+		}
+		return {
+			success: false,
+			refreshedSourceCount: 0,
+			deletedSourceCount: 0,
+			failedSourceCount: 0,
+			errorCode: "CONTENT_USAGE_DELETE_ERROR",
+		};
+	}
+}
+
+export async function refreshContentMediaUsageAfterWrite(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	contentId: string,
+): Promise<void> {
+	const result = await refreshContentMediaUsage(db, collectionSlug, contentId);
+	if (!result.success) {
+		console.error(
+			`[media-usage] Usage refresh for ${collectionSlug}/${contentId} finished with ${result.errorCode}`,
+		);
+	}
+}
+
+export async function markContentMediaUsageCollectionStale(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	lastErrorCode: string,
+): Promise<void> {
+	validateIdentifier(collectionSlug, "collection slug");
+	const repo = new MediaUsageRepository(db);
+	const identity = {
+		adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID,
+		scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE,
+		scopeKey: collectionSlug,
+	};
+	const existing = await repo.findIndexStatus(identity);
+	await repo.upsertIndexStatus({
+		...identity,
+		status: "stale",
+		schemaVersion: existing?.schemaVersion ?? CONTENT_SOURCE_SCHEMA_VERSION,
+		startedAt: existing?.startedAt ?? null,
+		completedAt: existing?.completedAt ?? null,
+		cursor: existing?.cursor ?? null,
+		indexedSourceCount: existing?.indexedSourceCount ?? 0,
+		failedSourceCount: existing?.failedSourceCount ?? 0,
+		lastErrorCode,
+	});
+}
+
+export async function findNonTranslatableSiblingContentIds(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	updatedContentId: string,
+	translationGroup: string | null | undefined,
+	updatedData: Record<string, unknown> | undefined,
+): Promise<string[]> {
+	if (!isI18nEnabled() || !updatedData || !translationGroup) return [];
+
+	validateIdentifier(collectionSlug, "collection slug");
+	const collection = await db
+		.selectFrom("_emdash_collections")
+		.select("id")
+		.where("slug", "=", collectionSlug)
+		.executeTakeFirst();
+	if (!collection) return [];
+
+	const fields = await db
+		.selectFrom("_emdash_fields")
+		.select("slug")
+		.where("collection_id", "=", collection.id)
+		.where("translatable", "=", 0)
+		.execute();
+
+	const touchedNonTranslatableSlugs = fields
+		.filter((field) => field.slug in updatedData)
+		.map((field) => field.slug);
+	if (touchedNonTranslatableSlugs.length === 0) return [];
+
+	const usageFields = await loadContentMediaUsageFields(db, collectionSlug);
+	const usageRelevantSlugs = new Set([
+		...usageFields.extractionFields.map((field) => field.slug),
+		...usageFields.displayFieldSlugs,
+	]);
+	if (!touchedNonTranslatableSlugs.some((slug) => usageRelevantSlugs.has(slug))) return [];
+
+	const tableName = `ec_${collectionSlug}`;
+	const rows = await sql<{ id: string }>`
+		SELECT id
+		FROM ${sql.ref(tableName)}
+		WHERE translation_group = ${translationGroup}
+		AND id != ${updatedContentId}
+		ORDER BY id ASC
+	`.execute(db);
+
+	return rows.rows.map((row) => row.id);
+}
+
+async function markSnapshotFailure(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	result: Exclude<Awaited<ReturnType<typeof loadContentMediaUsageSnapshots>>, { success: true }>,
+): Promise<ContentMediaUsageRefreshResult> {
+	const repo = new MediaUsageRepository(db);
+	if (result.source) {
+		await repo.markSourceAttempted({
+			...result.source,
+			sourceCompleteness: "failed",
+			lastErrorCode: result.error,
+		});
+	}
+	await markContentMediaUsageCollectionStale(db, collectionSlug, result.error);
+	return {
+		success: false,
+		refreshedSourceCount: 0,
+		deletedSourceCount: 0,
+		failedSourceCount: result.source ? 1 : 0,
+		errorCode: result.error,
+	};
+}
+
+export async function markContentMediaUsageCollectionStaleSafely(
+	db: Kysely<Database>,
+	collectionSlug: string,
+	lastErrorCode: ContentMediaUsageRefreshErrorCode,
+): Promise<boolean> {
+	try {
+		await markContentMediaUsageCollectionStale(db, collectionSlug, lastErrorCode);
+		return true;
+	} catch (error) {
+		console.error(`[media-usage] Failed to mark ${collectionSlug} stale:`, error);
+		return false;
+	}
+}
+
+async function withContentUsageLock<T>(
+	collectionSlug: string,
+	contentId: string,
+	fn: () => Promise<T>,
+): Promise<T> {
+	const locks = getContentUsageLocks();
+	const lockKey = `${collectionSlug}\0${contentId}`;
+	const previous = locks.get(lockKey) ?? Promise.resolve();
+	let releaseCurrent!: () => void;
+	const current = new Promise<void>((resolve) => {
+		releaseCurrent = resolve;
+	});
+	const next = previous.catch(() => {}).then(() => current);
+	locks.set(lockKey, next);
+
+	try {
+		await previous.catch(() => {});
+		return await fn();
+	} finally {
+		releaseCurrent();
+		if (locks.get(lockKey) === next) locks.delete(lockKey);
+	}
+}
+
+export async function withContentUsageCollectionLock<T>(
+	collectionSlug: string,
+	fn: () => Promise<T>,
+): Promise<T> {
+	// Coarse by design: row refreshes and collection source deletes must not interleave.
+	const locks = getContentUsageCollectionLocks();
+	const previous = locks.get(collectionSlug) ?? Promise.resolve();
+	let releaseCurrent!: () => void;
+	const current = new Promise<void>((resolve) => {
+		releaseCurrent = resolve;
+	});
+	const next = previous.catch(() => {}).then(() => current);
+	locks.set(collectionSlug, next);
+
+	try {
+		await previous.catch(() => {});
+		return await fn();
+	} finally {
+		releaseCurrent();
+		if (locks.get(collectionSlug) === next) locks.delete(collectionSlug);
+	}
+}
+
+function getContentUsageLocks(): Map<string, Promise<void>> {
+	const global = globalThis as typeof globalThis & Record<symbol, unknown>;
+	const existing = global[CONTENT_USAGE_LOCKS_KEY];
+	// eslint-disable-next-line typescript/no-unsafe-type-assertion -- globalThis symbol slot stores only this map
+	if (existing instanceof Map) return existing as Map<string, Promise<void>>;
+	const locks = new Map<string, Promise<void>>();
+	global[CONTENT_USAGE_LOCKS_KEY] = locks;
+	return locks;
+}
+
+function getContentUsageCollectionLocks(): Map<string, Promise<void>> {
+	const global = globalThis as typeof globalThis & Record<symbol, unknown>;
+	const existing = global[CONTENT_USAGE_COLLECTION_LOCKS_KEY];
+	// eslint-disable-next-line typescript/no-unsafe-type-assertion -- globalThis symbol slot stores only this map
+	if (existing instanceof Map) return existing as Map<string, Promise<void>>;
+	const locks = new Map<string, Promise<void>>();
+	global[CONTENT_USAGE_COLLECTION_LOCKS_KEY] = locks;
+	return locks;
+}