emdash 0.27.0 → 0.28.1

package/src/database/types.ts

@@ -76,6 +76,12 @@ export interface MediaUsageSourceTable {
 	revision_id: string | null;
 	current_generation: string;
 	schema_version: Generated<number>;
+	source_updated_at: Generated<string | null>;
+	source_version: Generated<number | null>;
+	source_fingerprint: Generated<string | null>;
+	source_completeness: Generated<string>;
+	last_attempted_at: Generated<string | null>;
+	last_error_code: Generated<string | null>;
 	indexed_at: Generated<string>;
 	created_at: Generated<string>;
 	updated_at: Generated<string>;
@@ -97,6 +103,21 @@ export interface MediaUsageTable {
 	created_at: Generated<string>;
 }
 
+export interface MediaUsageIndexStatusTable {
+	adapter_id: string;
+	scope_type: string;
+	scope_key: string;
+	status: string;
+	schema_version: Generated<number>;
+	started_at: Generated<string | null>;
+	completed_at: Generated<string | null>;
+	cursor: Generated<string | null>;
+	indexed_source_count: Generated<number>;
+	failed_source_count: Generated<number>;
+	last_error_code: Generated<string | null>;
+	updated_at: Generated<string>;
+}
+
 export interface UserTable {
 	id: string;
 	email: string;
@@ -453,6 +474,7 @@ export interface Database {
 	media: MediaTable;
 	_emdash_media_usage_sources: MediaUsageSourceTable;
 	_emdash_media_usage: MediaUsageTable;
+	_emdash_media_usage_index_status: MediaUsageIndexStatusTable;
 	users: UserTable;
 	credentials: CredentialTable;
 	auth_tokens: AuthTokenTable;

package/src/emdash-runtime.ts

@@ -31,6 +31,12 @@ import type {
 import { validateIdentifier } from "./database/validate.js";
 import { normalizeMediaValue } from "./media/normalize.js";
 import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js";
+import {
+	deleteContentMediaUsage,
+	findNonTranslatableSiblingContentIds,
+	markContentMediaUsageCollectionStale,
+	refreshContentMediaUsageAfterWrite,
+} from "./media/usage/content-refresh.js";
 import type { SandboxedPluginInstance, SandboxRunner } from "./plugins/sandbox/types.js";
 import type {
 	ResolvedPlugin,
@@ -2570,6 +2576,9 @@ export class EmDashRuntime {
 			authorId: body.authorId,
 			bylines: body.bylines,
 		});
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
 
 		// Run afterSave hooks (fire-and-forget)
 		if (result.success && result.data) {
@@ -2664,6 +2673,7 @@ export class EmDashRuntime {
 		// Content table columns = published data (never written by saves).
 		// Draft data lives only in the revisions table.
 		let usesDraftRevisions = false;
+		let draftStorageChanged = false;
 		if (processedData) {
 			try {
 				const collectionInfo = await this.schemaRegistry.getCollectionWithFields(collection);
@@ -2693,6 +2703,7 @@ export class EmDashRuntime {
 						if (bodyWithoutRev.skipRevision && existing.draftRevisionId) {
 							// Autosave: update existing draft revision in place
 							await revisionRepo.updateData(existing.draftRevisionId, mergedData);
+							draftStorageChanged = true;
 						} else {
 							// Create new draft revision
 							const revision = await revisionRepo.create({
@@ -2711,6 +2722,7 @@ export class EmDashRuntime {
 									updated_at = ${new Date().toISOString()}
 								WHERE id = ${resolvedId}
 							`.execute(this.db);
+							draftStorageChanged = true;
 
 							// Fire-and-forget: prune old revisions to prevent unbounded growth
 							void revisionRepo.pruneOldRevisions(collection, resolvedId, 50).catch(() => {});
@@ -2738,6 +2750,43 @@ export class EmDashRuntime {
 		// supporting collections, that's the just-saved draft, not the live
 		// columns.
 		const hydrated = await this.hydrateDraftData(result);
+		if (hydrated.success && hydrated.data) {
+			const contentIdsToRefresh = [resolvedId];
+			if (!usesDraftRevisions && processedData) {
+				try {
+					contentIdsToRefresh.push(
+						...(await findNonTranslatableSiblingContentIds(
+							this.db,
+							collection,
+							resolvedId,
+							hydrated.data.item.translationGroup,
+							processedData,
+						)),
+					);
+				} catch (error) {
+					console.error(
+						`[media-usage] Failed to discover synced i18n siblings for ${collection}/${resolvedId}:`,
+						error,
+					);
+					try {
+						await markContentMediaUsageCollectionStale(
+							this.db,
+							collection,
+							"CONTENT_USAGE_REFRESH_ERROR",
+						);
+					} catch (staleError) {
+						console.error(`[media-usage] Failed to mark ${collection} stale:`, staleError);
+					}
+				}
+			}
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, contentIdsToRefresh);
+		} else if (draftStorageChanged) {
+			try {
+				await markContentMediaUsageCollectionStale(this.db, collection, "CONTENT_USAGE_STALE");
+			} catch (error) {
+				console.error(`[media-usage] Failed to mark ${collection} stale:`, error);
+			}
+		}
 
 		// Run afterSave hooks (fire-and-forget)
 		if (hydrated.success && hydrated.data) {
@@ -2776,6 +2825,9 @@ export class EmDashRuntime {
 
 		// Delete the content
 		const result = await handleContentDelete(this.db, collection, id);
+		if (result.success) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.id]);
+		}
 
 		// Run afterDelete hooks (fire-and-forget)
 		if (result.success) {
@@ -2798,6 +2850,9 @@ export class EmDashRuntime {
 
 	async handleContentRestore(collection: string, id: string) {
 		const result = await handleContentRestore(this.db, collection, id);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
 
 		// Run afterRestore hooks (fire-and-forget)
 		if (result.success) {
@@ -2809,6 +2864,9 @@ export class EmDashRuntime {
 
 	async handleContentPermanentDelete(collection: string, id: string) {
 		const result = await handleContentPermanentDelete(this.db, collection, id);
+		if (result.success) {
+			await this.deleteContentUsageAfterSuccessfulPermanentDelete(collection, result.data.id);
+		}
 
 		// Run afterDelete hooks so plugins (e.g. AI Search) can clean up
 		if (result.success) {
@@ -2823,7 +2881,11 @@ export class EmDashRuntime {
 	}
 
 	async handleContentDuplicate(collection: string, id: string, authorId?: string) {
-		return handleContentDuplicate(this.db, collection, id, authorId);
+		const result = await handleContentDuplicate(this.db, collection, id, authorId);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
+		return result;
 	}
 
 	// =========================================================================
@@ -2836,6 +2898,9 @@ export class EmDashRuntime {
 		options: { publishedAt?: string; requireScheduledDue?: boolean } = {},
 	) {
 		const result = await handleContentPublish(this.db, collection, id, options);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
 
 		// Run afterPublish hooks (fire-and-forget)
 		if (result.success && result.data) {
@@ -2847,6 +2912,9 @@ export class EmDashRuntime {
 
 	async handleContentUnpublish(collection: string, id: string) {
 		const result = await handleContentUnpublish(this.db, collection, id);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
 
 		// Run afterUnpublish hooks (fire-and-forget)
 		if (result.success && result.data) {
@@ -2858,6 +2926,9 @@ export class EmDashRuntime {
 
 	async handleContentSchedule(collection: string, id: string, scheduledAt: string) {
 		const result = await handleContentSchedule(this.db, collection, id, scheduledAt);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
 
 		// Run afterSchedule hooks (fire-and-forget)
 		if (result.success && result.data) {
@@ -2869,6 +2940,9 @@ export class EmDashRuntime {
 
 	async handleContentUnschedule(collection: string, id: string) {
 		const result = await handleContentUnschedule(this.db, collection, id);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
 
 		// Run afterUnschedule hooks (fire-and-forget)
 		if (result.success && result.data) {
@@ -2883,7 +2957,11 @@ export class EmDashRuntime {
 	}
 
 	async handleContentDiscardDraft(collection: string, id: string) {
-		return handleContentDiscardDraft(this.db, collection, id);
+		const result = await handleContentDiscardDraft(this.db, collection, id);
+		if (result.success && result.data) {
+			await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]);
+		}
+		return result;
 	}
 
 	async handleContentCompare(collection: string, id: string) {
@@ -3025,6 +3103,9 @@ export class EmDashRuntime {
 		// behavior for collections that opt out of the draft model.
 		if (!usesDraftRevisions) {
 			const result = await handleRevisionRestore(this.db, revisionId, callerUserId);
+			if (result.success) {
+				await this.refreshContentUsageAfterSuccessfulWrite(revision.collection, [revision.entryId]);
+			}
 			return this.hydrateDraftData(result);
 		}
 
@@ -3060,7 +3141,11 @@ export class EmDashRuntime {
 			// columns and the next `content_get` would surface different
 			// values (the bug that motivated this rewrite).
 			const refetched = await handleContentGet(this.db, revision.collection, revision.entryId);
-			return this.hydrateDraftData(refetched);
+			const hydrated = await this.hydrateDraftData(refetched);
+			if (hydrated.success) {
+				await this.refreshContentUsageAfterSuccessfulWrite(revision.collection, [revision.entryId]);
+			}
+			return hydrated;
 		} catch (error) {
 			console.error("[emdash] revision restore failed:", error);
 			return {
@@ -3073,6 +3158,41 @@ export class EmDashRuntime {
 		}
 	}
 
+	private async refreshContentUsageAfterSuccessfulWrite(
+		collection: string,
+		contentIds: readonly string[],
+	): Promise<void> {
+		for (const contentId of new Set(contentIds)) {
+			try {
+				await refreshContentMediaUsageAfterWrite(this.db, collection, contentId);
+			} catch (error) {
+				console.error(
+					`[media-usage] Failed after content write ${collection}/${contentId}:`,
+					error,
+				);
+			}
+		}
+	}
+
+	private async deleteContentUsageAfterSuccessfulPermanentDelete(
+		collection: string,
+		contentId: string,
+	): Promise<void> {
+		try {
+			const result = await deleteContentMediaUsage(this.db, collection, contentId);
+			if (!result.success) {
+				console.error(
+					`[media-usage] Usage delete for ${collection}/${contentId} finished with ${result.errorCode}`,
+				);
+			}
+		} catch (error) {
+			console.error(
+				`[media-usage] Failed after permanent content delete ${collection}/${contentId}:`,
+				error,
+			);
+		}
+	}
+
 	// =========================================================================
 	// Plugin Routes
 	// =========================================================================

package/src/import/comments.ts

@@ -0,0 +1,159 @@
+/**
+ * Comment import functions
+ *
+ * Import comments from the WordPress plugin API into EmDash's native
+ * comments table, preserving authors, dates, threading, and status.
+ */
+
+import type { Kysely } from "kysely";
+import { ulid } from "ulidx";
+
+import type { Database } from "../database/types.js";
+import { invalidateCommentObjectCache } from "../object-cache/index.js";
+
+/**
+ * Plugin API comment format (matches /emdash/v1/comments items)
+ */
+export interface PluginComment {
+	id: number;
+	post_id: number;
+	parent_id: number | null;
+	author_name: string;
+	author_email: string;
+	/** Plain-text body (the plugin strips HTML) */
+	body: string;
+	/** ISO 8601 UTC timestamp */
+	date_gmt: string;
+	status: "approved" | "pending";
+}
+
+/**
+ * Result of comment import operation
+ */
+export interface CommentsImportResult {
+	/** Number of comments created */
+	imported: number;
+	/** Comments skipped (unresolvable post reference or already imported) */
+	skipped: number;
+	/** Errors encountered */
+	errors: Array<{ comment: string; error: string }>;
+}
+
+/**
+ * Import comments from the plugin API.
+ *
+ * Preserves original timestamps and threading. Comments whose post was
+ * not imported (no entry in `contentIdMap`) are skipped. Re-running the
+ * import is idempotent: a comment with the same post, author email, and
+ * timestamp is not created twice.
+ *
+ * @param comments - Comments from the plugin API (all pages, flat)
+ * @param db - Database connection
+ * @param contentIdMap - WP post ID -> EmDash content ID
+ * @param collectionMap - WP post ID -> EmDash collection slug
+ * @param rootIds - Optional pre-seeded WP-comment-ID -> EmDash-root-ID map.
+ *   The chunked import passes the map accumulated from earlier pages so a
+ *   reply in page N can thread onto a parent imported in page N-1; the
+ *   function adds this page's entries to it.
+ */
+export async function importCommentsFromPlugin(
+	comments: PluginComment[],
+	db: Kysely<Database>,
+	contentIdMap: Map<number, string>,
+	collectionMap: Map<number, string>,
+	rootIds?: Map<number, string>,
+): Promise<CommentsImportResult> {
+	const result: CommentsImportResult = {
+		imported: 0,
+		skipped: 0,
+		errors: [],
+	};
+
+	// WP comment ID -> EmDash comment ID, for parent threading
+	const commentIdMap = new Map<number, string>();
+
+	// WP comment ID -> EmDash ID of its top-level ancestor. EmDash threads
+	// are one level deep (assembleThreads nests replies under roots only),
+	// so deeper WP threads are flattened onto their root comment.
+	const rootIdMap = rootIds ?? new Map<number, string>();
+
+	// Parents must exist before children reference them; WP comment IDs
+	// are chronological, so ID order guarantees parents come first.
+	const sorted = comments.toSorted((a, b) => a.id - b.id);
+
+	for (const comment of sorted) {
+		const label = `${comment.author_name || "Anonymous"} (${comment.date_gmt})`;
+		try {
+			const contentId = contentIdMap.get(comment.post_id);
+			const collection = collectionMap.get(comment.post_id);
+			if (!contentId || !collection) {
+				result.skipped++;
+				continue;
+			}
+
+			const parsed = new Date(comment.date_gmt);
+			const createdAt = Number.isNaN(parsed.getTime())
+				? new Date().toISOString()
+				: parsed.toISOString();
+
+			// Idempotency: same post + author + timestamp + body = already
+			// imported (body included so same-second comments from one author
+			// don't collide). ponytail: one SELECT per comment — fine at
+			// import scale; batch with WHERE IN for six-figure comment counts.
+			const existing = await db
+				.selectFrom("_emdash_comments")
+				.select("id")
+				.where("content_id", "=", contentId)
+				.where("author_email", "=", comment.author_email)
+				.where("created_at", "=", createdAt)
+				.where("body", "=", comment.body)
+				.executeTakeFirst();
+			const parentId =
+				comment.parent_id !== null ? (rootIdMap.get(comment.parent_id) ?? null) : null;
+
+			if (existing) {
+				commentIdMap.set(comment.id, existing.id);
+				rootIdMap.set(comment.id, parentId ?? existing.id);
+				result.skipped++;
+				continue;
+			}
+
+			const id = ulid();
+
+			await db
+				.insertInto("_emdash_comments")
+				.values({
+					id,
+					collection,
+					content_id: contentId,
+					parent_id: parentId,
+					author_name: comment.author_name || "Anonymous",
+					author_email: comment.author_email,
+					author_user_id: null,
+					body: comment.body,
+					status: comment.status,
+					ip_hash: null,
+					user_agent: null,
+					moderation_metadata: null,
+					created_at: createdAt,
+					updated_at: createdAt,
+				})
+				.execute();
+
+			commentIdMap.set(comment.id, id);
+			rootIdMap.set(comment.id, parentId ?? id);
+			result.imported++;
+		} catch (error) {
+			result.errors.push({
+				comment: label,
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+	}
+
+	if (result.imported > 0) {
+		invalidateCommentObjectCache();
+	}
+
+	return result;
+}

package/src/import/index.ts

@@ -44,13 +44,20 @@ export {
 // Sections import
 export { importReusableBlocksAsSections, type SectionsImportResult } from "./sections.js";
 
+// Comments import
+export {
+	importCommentsFromPlugin,
+	type PluginComment,
+	type CommentsImportResult,
+} from "./comments.js";
+
 // Site settings import
 export {
 	importSiteSettings,
 	parseSiteSettingsFromPlugin,
 	type SiteSettingsAnalysis,
 	type SettingsImportResult,
-	type WidgetAreaAnalysis,
+	type SettingsMediaIds,
 } from "./settings.js";
 
 // Registry