emdash 0.28.0 → 0.28.1

package/src/database/repositories/media-usage.ts

@@ -496,7 +496,7 @@ export class MediaUsageRepository {
 				.executeTakeFirst();
 			deleted = Number(result.numDeletedRows ?? 0) > 0;
 			if (!deleted) return;
-			await trx.deleteFrom("_emdash_media_usage").where("source_key", "=", sourceKey).execute();
+			await this.deleteSourceGenerationOccurrences(trx, sourceKey, expectedCurrentGeneration);
 		});
 
 		return {
@@ -518,7 +518,11 @@ export class MediaUsageRepository {
 				.executeTakeFirst();
 			deleted = Number(result.numDeletedRows ?? 0) > 0;
 			if (!deleted) return;
-			await trx.deleteFrom("_emdash_media_usage").where("source_key", "=", sourceKey).execute();
+			await this.deleteSourceGenerationOccurrences(
+				trx,
+				sourceKey,
+				expectedSource.currentGeneration,
+			);
 		});
 
 		return {
@@ -547,7 +551,11 @@ export class MediaUsageRepository {
 				.executeTakeFirst();
 			deleted = Number(result.numDeletedRows ?? 0) > 0;
 			if (!deleted) return;
-			await trx.deleteFrom("_emdash_media_usage").where("source_key", "=", sourceKey).execute();
+			await this.deleteSourceGenerationOccurrences(
+				trx,
+				sourceKey,
+				expectedSource.currentGeneration,
+			);
 		});
 		const contentPresent = deleted ? false : await this.contentRowExists(tableName, contentId);
 
@@ -903,6 +911,20 @@ export class MediaUsageRepository {
 		});
 	}
 
+	private async deleteSourceGenerationOccurrences(
+		db: DatabaseExecutor,
+		sourceKey: string,
+		generation: string,
+	): Promise<void> {
+		// Guarded source deletes remove only the generation that won the source CAS;
+		// unmatched generations become invisible orphans reclaimed by age-gated cleanup.
+		await db
+			.deleteFrom("_emdash_media_usage")
+			.where("source_key", "=", sourceKey)
+			.where("generation", "=", generation)
+			.execute();
+	}
+
 	private async insertOccurrences(
 		db: DatabaseExecutor,
 		sourceKey: string,

package/src/media/usage/content-repair.ts

@@ -20,6 +20,7 @@ import {
 } from "./content-refresh.js";
 import {
 	CONTENT_SOURCE_SCHEMA_VERSION,
+	type ContentMediaUsageSnapshot,
 	loadContentMediaUsageSnapshots,
 } from "./content-snapshots.js";
 import {
@@ -65,11 +66,48 @@ export interface ContentMediaUsageRepairCollectionResult {
 	completedAt: string | null;
 }
 
+export interface ContentMediaUsageRepairAllResult {
+	status: ContentMediaUsageRepairStatus;
+	collections: ContentMediaUsageRepairCollectionResult[];
+	indexedSourceCount: number;
+	failedSourceCount: number;
+	skippedSourceCount: number;
+	deletedSourceCount: number;
+}
+
 export interface ContentMediaUsageCollectionScan {
 	collectionSlug: string;
 	contentIds: string[];
 }
 
+interface ContentMediaUsageCollectionRecord {
+	id: string;
+	slug: string;
+}
+
+interface ContentMediaUsageInitialCollectionResult {
+	collection: ContentMediaUsageCollectionRecord;
+	result: ContentMediaUsageRepairCollectionResult;
+}
+
+export async function repairContentMediaUsageAll(
+	db: Kysely<Database>,
+): Promise<ContentMediaUsageRepairAllResult> {
+	const collections = await loadContentMediaUsageCollectionRecords(db);
+	const results: ContentMediaUsageInitialCollectionResult[] = [];
+
+	for (const collection of collections) {
+		results.push({
+			collection,
+			result: await repairContentMediaUsageCollectionSafely(db, collection.slug),
+		});
+	}
+
+	return aggregateContentMediaUsageRepairAll(
+		await filterExistingContentMediaUsageCollectionResults(db, results),
+	);
+}
+
 export async function scanContentMediaUsageCollection(
 	db: Kysely<Database>,
 	collectionSlug: string,
@@ -105,6 +143,116 @@ export async function repairContentMediaUsageCollection(
 	);
 }
 
+async function loadContentMediaUsageCollectionRecords(
+	db: Kysely<Database>,
+): Promise<ContentMediaUsageCollectionRecord[]> {
+	return db
+		.selectFrom("_emdash_collections")
+		.select(["id", "slug"])
+		.orderBy("slug", "asc")
+		.execute();
+}
+
+async function repairContentMediaUsageCollectionSafely(
+	db: Kysely<Database>,
+	collectionSlug: string,
+): Promise<ContentMediaUsageRepairCollectionResult> {
+	try {
+		return await repairContentMediaUsageCollection(db, { collectionSlug });
+	} catch (error) {
+		console.error(`[media-usage] Failed to repair collection ${collectionSlug}:`, error);
+		const now = new Date().toISOString();
+		return {
+			scope: contentMediaUsageCollectionScope(collectionSlug),
+			status: "failed",
+			indexedSourceCount: 0,
+			failedSourceCount: 0,
+			skippedSourceCount: 0,
+			deletedSourceCount: 0,
+			lastErrorCode: CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_ERROR,
+			startedAt: now,
+			completedAt: now,
+		};
+	}
+}
+
+async function filterExistingContentMediaUsageCollectionResults(
+	db: Kysely<Database>,
+	results: readonly ContentMediaUsageInitialCollectionResult[],
+): Promise<ContentMediaUsageRepairCollectionResult[]> {
+	const currentCollections = await loadContentMediaUsageCollectionRecordsSafely(db);
+	const currentIdsBySlug = new Map(
+		currentCollections.map((collection) => [collection.slug, collection.id]),
+	);
+	const includedResults: ContentMediaUsageRepairCollectionResult[] = [];
+	const excludedResults: ContentMediaUsageRepairCollectionResult[] = [];
+
+	for (const { collection, result } of results) {
+		if (currentIdsBySlug.get(collection.slug) === collection.id) {
+			includedResults.push(result);
+		} else {
+			excludedResults.push(result);
+		}
+	}
+
+	if (excludedResults.length > 0) {
+		const repo = new MediaUsageRepository(db);
+		for (const result of excludedResults) {
+			await repo.deleteIndexStatus(result.scope);
+		}
+	}
+
+	return includedResults;
+}
+
+async function loadContentMediaUsageCollectionRecordsSafely(
+	db: Kysely<Database>,
+): Promise<ContentMediaUsageCollectionRecord[]> {
+	try {
+		return await loadContentMediaUsageCollectionRecords(db);
+	} catch {
+		// Retry once before failing; returning unpruned results can over-report deleted collections.
+	}
+
+	try {
+		return await loadContentMediaUsageCollectionRecords(db);
+	} catch (error) {
+		console.error("[media-usage] Failed to reconcile all-content repair collections:", error);
+		throw error;
+	}
+}
+
+function aggregateContentMediaUsageRepairAll(
+	collections: readonly ContentMediaUsageRepairCollectionResult[],
+): ContentMediaUsageRepairAllResult {
+	return {
+		status: determineRepairAllStatus(collections),
+		collections: [...collections],
+		indexedSourceCount: sumCollectionRepairCount(collections, "indexedSourceCount"),
+		failedSourceCount: sumCollectionRepairCount(collections, "failedSourceCount"),
+		skippedSourceCount: sumCollectionRepairCount(collections, "skippedSourceCount"),
+		deletedSourceCount: sumCollectionRepairCount(collections, "deletedSourceCount"),
+	};
+}
+
+function determineRepairAllStatus(
+	collections: readonly ContentMediaUsageRepairCollectionResult[],
+): ContentMediaUsageRepairStatus {
+	if (collections.length === 0) return "complete";
+	if (collections.every((collection) => collection.status === "complete")) return "complete";
+	if (collections.some((collection) => collection.status === "stale")) return "stale";
+	if (collections.some((collection) => collection.status === "partial")) return "partial";
+	if (collections.every((collection) => collection.status === "failed")) return "failed";
+	return "partial";
+}
+
+function sumCollectionRepairCount(
+	collections: readonly ContentMediaUsageRepairCollectionResult[],
+	key: "indexedSourceCount" | "failedSourceCount" | "skippedSourceCount" | "deletedSourceCount",
+): number {
+	return collections.reduce((sum, collection) => sum + collection[key], 0);
+}
+
 async function repairContentMediaUsageCollectionUnlocked(
 	db: Kysely<Database>,
 	collectionSlug: string,
@@ -147,6 +295,7 @@ async function repairContentMediaUsageCollectionUnlocked(
 					skippedSourceCount: 0,
 					deletedSourceCount: 0,
 					lastErrorCode: CONTENT_MEDIA_USAGE_REPAIR_ERROR.COLLECTION_NOT_FOUND,
+					missingContentIds: new Set(),
 				},
 				status: "failed",
 				startedAt,
@@ -155,16 +304,14 @@ async function repairContentMediaUsageCollectionUnlocked(
 		}
 		const counts = await repairScannedContentSources(db, repo, scan);
 		const finalScan = await scanContentMediaUsageCollection(db, collectionSlug);
-		const scannedContentCount = finalScan?.contentIds.length ?? scan.contentIds.length;
 		if (!finalScan) {
 			counts.failedSourceCount++;
 			counts.lastErrorCode = CONTENT_MEDIA_USAGE_REPAIR_ERROR.COLLECTION_NOT_FOUND;
-		} else if (!sameContentIds(scan.contentIds, finalScan.contentIds)) {
-			counts.skippedSourceCount++;
-			counts.lastErrorCode = CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_CONFLICT;
+		} else if (!sameContentIds(repairedContentIds(scan.contentIds, counts), finalScan.contentIds)) {
+			markRepairConflict(counts);
 		}
 		const completedAt = new Date().toISOString();
-		const status = determineRepairStatus(counts, scannedContentCount);
+		const status = determineRepairStatus(counts);
 		return await finalizeRepairStatus(repo, {
 			...scope,
 			runToken,
@@ -191,6 +338,7 @@ async function repairContentMediaUsageCollectionUnlocked(
 				skippedSourceCount: 0,
 				deletedSourceCount: 0,
 				lastErrorCode,
+				missingContentIds: new Set(),
 			},
 			status: "failed",
 			startedAt,
@@ -205,6 +353,7 @@ interface RepairCounts {
 	skippedSourceCount: number;
 	deletedSourceCount: number;
 	lastErrorCode: ContentMediaUsageRepairErrorCode | null;
+	missingContentIds: Set<string>;
 }
 
 async function repairScannedContentSources(
@@ -218,6 +367,7 @@ async function repairScannedContentSources(
 		skippedSourceCount: 0,
 		deletedSourceCount: 0,
 		lastErrorCode: null,
+		missingContentIds: new Set(),
 	};
 
 	const fieldDiscovery = await loadContentMediaUsageFields(db, scan.collectionSlug);
@@ -256,7 +406,16 @@ async function repairContentSource(
 		fieldDiscovery,
 	);
 	if (!snapshotsResult.success) {
+		if (snapshotsResult.error === CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_NOT_FOUND) {
+			markRepairConflict(counts);
+			counts.missingContentIds.add(contentId);
+			return;
+		}
 		counts.lastErrorCode = snapshotsResult.error;
+		if (snapshotsResult.snapshots) {
+			// Partial snapshot failures keep absent variants in place; the attempted source is marked below.
+			await repairSnapshotSources(repo, snapshotsResult.snapshots, observedSources, counts);
+		}
 		if (snapshotsResult.source) {
 			const result = await repo.markSourceAttemptedIfMatching(
 				{
@@ -269,8 +428,7 @@ async function repairContentSource(
 			if (result.attempted) {
 				counts.failedSourceCount++;
 			} else {
-				counts.skippedSourceCount++;
-				counts.lastErrorCode = CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_CONFLICT;
+				markRepairConflict(counts);
 			}
 			return;
 		}
@@ -279,8 +437,29 @@ async function repairContentSource(
 		return;
 	}
 
+	const expectedSourceKeys = await repairSnapshotSources(
+		repo,
+		snapshotsResult.snapshots,
+		observedSources,
+		counts,
+	);
+
+	for (const sourceKey of sourceKeys) {
+		if (expectedSourceKeys.has(sourceKey)) continue;
+		const observedSource = observedSources.get(sourceKey);
+		if (!observedSource) continue;
+		await deleteObservedSource(repo, sourceKey, observedSource, counts);
+	}
+}
+
+async function repairSnapshotSources(
+	repo: MediaUsageRepository,
+	snapshots: readonly ContentMediaUsageSnapshot[],
+	observedSources: Map<string, MediaUsageSource>,
+	counts: RepairCounts,
+): Promise<Set<string>> {
 	const expectedSourceKeys = new Set<string>();
-	for (const snapshot of snapshotsResult.snapshots) {
+	for (const snapshot of snapshots) {
 		expectedSourceKeys.add(snapshot.source.sourceKey);
 		const result = await repo.replaceSourceIfMatching(
 			snapshot.source,
@@ -290,17 +469,15 @@ async function repairContentSource(
 		if (result.replaced) {
 			counts.indexedSourceCount++;
 		} else {
-			counts.skippedSourceCount++;
-			counts.lastErrorCode = CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_CONFLICT;
+			markRepairConflict(counts);
 		}
 	}
+	return expectedSourceKeys;
+}
 
-	for (const sourceKey of sourceKeys) {
-		if (expectedSourceKeys.has(sourceKey)) continue;
-		const observedSource = observedSources.get(sourceKey);
-		if (!observedSource) continue;
-		await deleteObservedSource(repo, sourceKey, observedSource, counts);
-	}
+function markRepairConflict(counts: RepairCounts): void {
+	counts.skippedSourceCount++;
+	counts.lastErrorCode ??= CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_CONFLICT;
 }
 
 async function reconcileOrphanedContentSources(
@@ -370,8 +547,7 @@ async function deleteObservedSourceIfContentAbsent(
 	}
 	if (result.contentPresent) return;
 	if (result.source) {
-		counts.skippedSourceCount++;
-		counts.lastErrorCode = CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_CONFLICT;
+		markRepairConflict(counts);
 	}
 }
 
@@ -387,8 +563,7 @@ async function deleteObservedSource(
 		return;
 	}
 	if (result.source) {
-		counts.skippedSourceCount++;
-		counts.lastErrorCode = CONTENT_MEDIA_USAGE_REPAIR_ERROR.CONTENT_USAGE_REPAIR_CONFLICT;
+		markRepairConflict(counts);
 	}
 }
 
@@ -439,11 +614,12 @@ async function finalizeRepairStatus(
 
 function determineRepairStatus(
 	counts: RepairCounts,
-	scannedContentCount: number,
 ): Exclude<ContentMediaUsageRepairStatus, "stale"> {
 	if (counts.failedSourceCount === 0 && counts.skippedSourceCount === 0) return "complete";
 	const trustedProgress = counts.indexedSourceCount + counts.deletedSourceCount;
-	if (trustedProgress === 0 && scannedContentCount > 0) return "failed";
+	if (counts.failedSourceCount > 0 && trustedProgress === 0) {
+		return "failed";
+	}
 	return "partial";
 }
 
@@ -477,6 +653,14 @@ function sameContentIds(left: readonly string[], right: readonly string[]): bool
 	return left.every((id) => rightIds.has(id));
 }
 
+function repairedContentIds(
+	contentIds: readonly string[],
+	counts: Pick<RepairCounts, "missingContentIds">,
+): string[] {
+	if (counts.missingContentIds.size === 0) return [...contentIds];
+	return contentIds.filter((contentId) => !counts.missingContentIds.has(contentId));
+}
+
 function contentMediaUsageCollectionScope(collectionSlug: string): ContentMediaUsageRepairScope {
 	return {
 		adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID,

package/src/media/usage/content-snapshots.ts

@@ -46,6 +46,7 @@ export type LoadContentMediaUsageSnapshotsResult =
 				| "DRAFT_REVISION_MISMATCH"
 				| "DRAFT_REVISION_INVALID";
 			source?: MediaUsageSourceInput;
+			snapshots?: ContentMediaUsageSnapshot[];
 	  };
 
 export interface ContentMediaUsageSnapshot {
@@ -116,6 +117,7 @@ export async function loadContentMediaUsageSnapshots(
 				success: false,
 				error: "DRAFT_REVISION_NOT_FOUND",
 				source: attemptedDraftSource,
+				snapshots,
 			};
 		}
 		if (!revisionResult.success) {
@@ -123,6 +125,7 @@ export async function loadContentMediaUsageSnapshots(
 				success: false,
 				error: "DRAFT_REVISION_INVALID",
 				source: attemptedDraftSource,
+				snapshots,
 			};
 		}
 		const revision = revisionResult.revision;
@@ -131,6 +134,7 @@ export async function loadContentMediaUsageSnapshots(
 				success: false,
 				error: "DRAFT_REVISION_MISMATCH",
 				source: attemptedDraftSource,
+				snapshots,
 			};
 		}