@remit/drizzle-service 0.0.98 → 0.0.100

package/src/repos/message.ts

@@ -1,4 +1,3 @@
-import { randomUUID } from "node:crypto";
 import type {
 	CreateMessageInput,
 	IMessageRepository,
@@ -6,8 +5,20 @@ import type {
 	MessageItem,
 	PlacementPredicate,
 } from "@remit/data-ports";
-import { and, asc, eq, gt, inArray, isNull, or, type SQL } from "drizzle-orm";
-
+import {
+	and,
+	asc,
+	eq,
+	exists,
+	gt,
+	inArray,
+	isNotNull,
+	isNull,
+	lt,
+	or,
+	type SQL,
+} from "drizzle-orm";
+import { runBatch, type SqliteBatchItem } from "../batch.js";
 import type { Db } from "../db.js";
 import {
 	CreateFailedConflictError,
@@ -20,22 +31,21 @@ import {
 } from "../id.js";
 import { decodeToken, resultList } from "../pagination.js";
 import {
-	bodyPartContentTable,
 	bodyPartParameterTable,
 	bodyPartStorageTable,
 	bodyPartTable,
+	bodyStructureRefetchTable,
 	envelopeAddressTable,
 	envelopeTable,
 	type MessageDataSchema,
 	messageFlagTable,
 	messageReferenceTable,
 	messageTable,
-	outboxTable,
 	rawMessageStorageTable,
 } from "../schema/message-data.js";
-import { runInTransaction } from "../tx.js";
+import { threadMessageTable } from "../schema/thread-message.js";
+import type { SearchIndex } from "../search/thread-message-search.js";
 import {
-	toBodyPartContentItem,
 	toBodyPartItem,
 	toBodyPartParameterItem,
 	toBodyPartStorageItem,
@@ -47,13 +57,6 @@ import {
 
 type DB = Db<MessageDataSchema>;
 
-const REEMBED_BATCH_SIZE = 500;
-
-export interface ReembedRequest {
-	queued: number;
-	alreadyQueued: number;
-}
-
 function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 	return {
 		messageId: row.messageId,
@@ -74,9 +77,6 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 		movedByRemit: row.movedByRemit,
 		createdAt: row.createdAt,
 		updatedAt: row.updatedAt,
-		...(row.messageIdHeader !== null
-			? { messageIdHeader: row.messageIdHeader }
-			: {}),
 		...(row.bodyStorageKey !== null
 			? { bodyStorageKey: row.bodyStorageKey }
 			: {}),
@@ -111,71 +111,56 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
 	};
 }
 
-/**
- * Emitted into the transactional outbox when a message's rows are deleted, so
- * the search-index worker relays a search-index REMOVE and the vectors are
- * dropped.
- */
-export const MESSAGE_REMOVED_EVENT = "message.removed" as const;
-
-export type SubtreeDb = Pick<Db<Record<string, unknown>>, "delete" | "insert">;
+export type SubtreeDb = Pick<Db<Record<string, unknown>>, "delete">;
 
 /**
- * Delete a message and its whole per-message subtree by message id — the nine
- * child tables, the message row, and its prior CDC outbox rows — then append one
- * `message.removed` outbox row per id so search-index cleanup rides the same
- * transactional outbox as body-sync and move. Runs inside a caller-supplied
- * transaction so the row delete and the removal event commit atomically.
+ * The statements that delete a message and its whole per-message subtree by
+ * message id: the child tables, then the message row. The caller runs them as
+ * one batch so the subtree goes atomically; the last statement returns the
+ * message ids it removed. `guard` rides every statement, so
+ * a guard that reads the message row holds until the last statement removes it.
  */
-export async function deleteMessageSubtree(
-	tx: SubtreeDb,
+export function deleteMessageSubtree(
+	db: SubtreeDb,
 	messageIds: string[],
-): Promise<void> {
-	if (messageIds.length === 0) return;
-
-	await tx
-		.delete(messageFlagTable)
-		.where(inArray(messageFlagTable.messageId, messageIds));
-	await tx
-		.delete(messageReferenceTable)
-		.where(inArray(messageReferenceTable.messageId, messageIds));
-	await tx
-		.delete(envelopeAddressTable)
-		.where(inArray(envelopeAddressTable.messageId, messageIds));
-	await tx
-		.delete(bodyPartParameterTable)
-		.where(inArray(bodyPartParameterTable.messageId, messageIds));
-	await tx
-		.delete(bodyPartStorageTable)
-		.where(inArray(bodyPartStorageTable.messageId, messageIds));
-	await tx
-		.delete(bodyPartContentTable)
-		.where(inArray(bodyPartContentTable.messageId, messageIds));
-	await tx
-		.delete(rawMessageStorageTable)
-		.where(inArray(rawMessageStorageTable.messageId, messageIds));
-	await tx
-		.delete(bodyPartTable)
-		.where(inArray(bodyPartTable.messageId, messageIds));
-	await tx
-		.delete(envelopeTable)
-		.where(inArray(envelopeTable.messageId, messageIds));
-	await tx
-		.delete(messageTable)
-		.where(inArray(messageTable.messageId, messageIds));
-
-	await tx
-		.delete(outboxTable)
-		.where(inArray(outboxTable.messageId, messageIds));
-	await tx.insert(outboxTable).values(
-		messageIds.map((messageId) => ({
-			id: randomUUID(),
-			messageId,
-			event: MESSAGE_REMOVED_EVENT,
-			payload: { messageId },
-			createdAt: new Date(),
-		})),
-	);
+	guard?: SQL,
+): SqliteBatchItem[] {
+	if (messageIds.length === 0) return [];
+	return [
+		db
+			.delete(messageFlagTable)
+			.where(and(inArray(messageFlagTable.messageId, messageIds), guard)),
+		db
+			.delete(messageReferenceTable)
+			.where(and(inArray(messageReferenceTable.messageId, messageIds), guard)),
+		db
+			.delete(envelopeAddressTable)
+			.where(and(inArray(envelopeAddressTable.messageId, messageIds), guard)),
+		db
+			.delete(bodyPartParameterTable)
+			.where(and(inArray(bodyPartParameterTable.messageId, messageIds), guard)),
+		db
+			.delete(bodyPartStorageTable)
+			.where(and(inArray(bodyPartStorageTable.messageId, messageIds), guard)),
+		db
+			.delete(rawMessageStorageTable)
+			.where(and(inArray(rawMessageStorageTable.messageId, messageIds), guard)),
+		db
+			.delete(bodyPartTable)
+			.where(and(inArray(bodyPartTable.messageId, messageIds), guard)),
+		db
+			.delete(bodyStructureRefetchTable)
+			.where(
+				and(inArray(bodyStructureRefetchTable.messageId, messageIds), guard),
+			),
+		db
+			.delete(envelopeTable)
+			.where(and(inArray(envelopeTable.messageId, messageIds), guard)),
+		db
+			.delete(messageTable)
+			.where(and(inArray(messageTable.messageId, messageIds), guard))
+			.returning({ messageId: messageTable.messageId }),
+	];
 }
 
 /**
@@ -207,11 +192,14 @@ const placementTerms = (expected: PlacementPredicate): SQL[] => {
 };
 
 export class DrizzleMessageRepository implements IMessageRepository {
-	constructor(private db: DB) {}
+	constructor(
+		private db: DB,
+		private search: SearchIndex,
+	) {}
 
-	async create(input: CreateMessageInput): Promise<MessageItem> {
+	private rowOf(input: CreateMessageInput) {
 		const now = Date.now();
-		const row = {
+		return {
 			messageId: input.messageId,
 			mailboxId: input.mailboxId,
 			uid: input.uid,
@@ -229,7 +217,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 				input.authenticityVerdict ?? ("NotEvaluated" as const),
 			hasListUnsubscribe: input.hasListUnsubscribe ?? false,
 			movedByRemit: input.movedByRemit ?? false,
-			messageIdHeader: input.messageIdHeader ?? null,
 			bodyStorageKey: input.bodyStorageKey ?? null,
 			originalMailboxId: input.originalMailboxId ?? null,
 			originalUid: input.originalUid ?? null,
@@ -243,14 +230,13 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			createdAt: now,
 			updatedAt: now,
 		};
+	}
+
+	async create(input: CreateMessageInput): Promise<MessageItem> {
+		const row = this.rowOf(input);
 
 		// Faithful to ElectroDB message.create: a duplicate messageId raises a
 		// unique-constraint violation, surfaced as the domain conflict error.
-		//
-		// No outbox event. A freshly created message has neither a body nor a
-		// threadMessage yet, so there is nothing to index — the search-index
-		// relay never drained a creation event, and the rows accumulated forever
-		// (reader#1063). Body-sync appends the event once there is content.
 		try {
 			await this.db.insert(messageTable).values(row);
 		} catch (error) {
@@ -270,17 +256,13 @@ export class DrizzleMessageRepository implements IMessageRepository {
 	async upsertWithStatus(
 		input: CreateMessageInput,
 	): Promise<{ item: MessageItem; created: boolean }> {
-		const existing = await this.db
-			.select()
-			.from(messageTable)
-			.where(eq(messageTable.messageId, input.messageId));
-
-		if (existing.length > 0) {
-			return { item: toMessageItem(existing[0]), created: false };
-		}
-
-		const item = await this.create(input);
-		return { item, created: true };
+		const [inserted] = await this.db
+			.insert(messageTable)
+			.values(this.rowOf(input))
+			.onConflictDoNothing({ target: messageTable.messageId })
+			.returning();
+		if (inserted) return { item: toMessageItem(inserted), created: true };
+		return { item: await this.get(input.messageId), created: false };
 	}
 
 	async get(messageId: string): Promise<MessageItem>;
@@ -318,7 +300,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			bodyPartParameters,
 			rawMessageStorages,
 			bodyPartStorages,
-			bodyPartContents,
 		] = await Promise.all([
 			this.db
 				.select()
@@ -356,10 +337,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 				.select()
 				.from(bodyPartStorageTable)
 				.where(eq(bodyPartStorageTable.messageId, messageId)),
-			this.db
-				.select()
-				.from(bodyPartContentTable)
-				.where(eq(bodyPartContentTable.messageId, messageId)),
 		]);
 
 		if (messages.length === 0) {
@@ -383,7 +360,6 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			bodyPartParameter: bodyPartParameters.map(toBodyPartParameterItem),
 			rawMessageStorage: rawMessageStorages.map(toRawMessageStorageItem),
 			bodyPartStorage: bodyPartStorages.map(toBodyPartStorageItem),
-			bodyPartContent: bodyPartContents.map(toBodyPartContentItem),
 		};
 	}
 
@@ -427,42 +403,16 @@ export class DrizzleMessageRepository implements IMessageRepository {
 			...(input.placementDecidedAt !== undefined
 				? { placementDecidedAt: input.placementDecidedAt }
 				: {}),
-			...(input.messageIdHeader !== undefined
-				? { messageIdHeader: input.messageIdHeader }
-				: {}),
 			...(input.spamReport !== undefined
 				? { spamReport: input.spamReport }
 				: {}),
 			updatedAt: now,
 		};
 
-		// A non-empty bodyStorageKey means body-sync just persisted the parsed
-		// body, so the message now has embeddable content and its threadMessage
-		// exists. Append a search-index event in the same transaction as the write
-		// (transactional outbox) — the search-index worker relays it to SQS and
-		// embeds.
-		// The outbox is append-only: the worker's content-hash gate makes a
-		// redundant pass near-free.
-		const bodySynced =
-			typeof input.bodyStorageKey === "string" &&
-			input.bodyStorageKey.length > 0;
-
-		await runInTransaction(this.db, async (tx) => {
-			await tx
-				.update(messageTable)
-				.set(setValues)
-				.where(eq(messageTable.messageId, messageId));
-
-			if (bodySynced) {
-				await tx.insert(outboxTable).values({
-					id: randomUUID(),
-					messageId,
-					event: "message.body_synced",
-					payload: { messageId },
-					createdAt: new Date(),
-				});
-			}
-		});
+		await this.db
+			.update(messageTable)
+			.set(setValues)
+			.where(eq(messageTable.messageId, messageId));
 		return this.get(messageId);
 	}
 
@@ -492,11 +442,39 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		await this.deleteMany([messageId]);
 	}
 
+	async deleteIfPlacement(
+		messageId: string,
+		expected: PlacementPredicate,
+	): Promise<boolean> {
+		const placed = exists(
+			this.db
+				.select({ messageId: messageTable.messageId })
+				.from(messageTable)
+				.where(
+					and(
+						eq(messageTable.messageId, messageId),
+						...placementTerms(expected),
+					),
+				),
+		);
+		const listings = await this.db
+			.select({ threadMessageId: threadMessageTable.threadMessageId })
+			.from(threadMessageTable)
+			.where(eq(threadMessageTable.messageId, messageId));
+		const results = await runBatch(this.db, [
+			this.db
+				.delete(threadMessageTable)
+				.where(and(eq(threadMessageTable.messageId, messageId), placed)),
+			...deleteMessageSubtree(this.db, [messageId], placed),
+		]);
+		await this.search.sync(listings.map((row) => row.threadMessageId));
+		const removed = results.at(-1);
+		return Array.isArray(removed) && removed.length > 0;
+	}
+
 	async deleteMany(messageIds: string[]): Promise<void> {
 		if (messageIds.length === 0) return;
-		await runInTransaction(this.db, (tx) =>
-			deleteMessageSubtree(tx as unknown as SubtreeDb, messageIds),
-		);
+		await runBatch(this.db, deleteMessageSubtree(this.db, messageIds));
 	}
 
 	async listByMailbox(
@@ -554,6 +532,41 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		return rows.map(toMessageItem);
 	}
 
+	async listStrandedPlacements(
+		mailboxIds: readonly string[],
+		untouchedSince: number,
+	): ReturnType<IMessageRepository["listStrandedPlacements"]> {
+		if (mailboxIds.length === 0) return [];
+		const ids = [...mailboxIds];
+		const rows = await this.db
+			.select()
+			.from(messageTable)
+			.where(
+				and(
+					lt(messageTable.updatedAt, untouchedSince),
+					or(
+						and(
+							eq(messageTable.status, "moving"),
+							inArray(messageTable.originalMailboxId, ids),
+							isNotNull(messageTable.originalUid),
+						),
+						and(
+							eq(messageTable.status, "deleting"),
+							inArray(messageTable.mailboxId, ids),
+						),
+						and(
+							eq(messageTable.status, "moving"),
+							eq(messageTable.uid, 0),
+							isNull(messageTable.originalUid),
+							inArray(messageTable.mailboxId, ids),
+						),
+					),
+				),
+			)
+			.orderBy(asc(messageTable.updatedAt), asc(messageTable.messageId));
+		return rows.map(toMessageItem);
+	}
+
 	async transitionPlacement(
 		messageId: string,
 		expected: PlacementPredicate,
@@ -587,17 +600,68 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		return toMessageItem(rows[0]);
 	}
 
+	async repointIfPlacement(
+		messageId: string,
+		expected: PlacementPredicate,
+		to: { mailboxId: string; uid: number; isDeleted: boolean },
+	): Promise<boolean> {
+		const now = Date.now();
+		const landed = exists(
+			this.db
+				.select({ messageId: messageTable.messageId })
+				.from(messageTable)
+				.where(
+					and(
+						eq(messageTable.messageId, messageId),
+						eq(messageTable.mailboxId, to.mailboxId),
+						eq(messageTable.uid, to.uid),
+					),
+				),
+		);
+		const listings = await this.db
+			.select({ threadMessageId: threadMessageTable.threadMessageId })
+			.from(threadMessageTable)
+			.where(eq(threadMessageTable.messageId, messageId));
+		const [moved] = await runBatch(this.db, [
+			this.db
+				.update(messageTable)
+				.set({
+					uid: to.uid,
+					mailboxId: to.mailboxId,
+					originalUid: null,
+					status: "active",
+					syncStatus: "synced",
+					abandonedMutation: "none",
+					updatedAt: now,
+				})
+				.where(
+					and(
+						eq(messageTable.messageId, messageId),
+						...placementTerms(expected),
+					),
+				)
+				.returning({ messageId: messageTable.messageId }),
+			this.db
+				.update(threadMessageTable)
+				.set({
+					mailboxId: to.mailboxId,
+					uid: to.uid,
+					isDeleted: to.isDeleted,
+					updatedAt: now,
+				})
+				.where(and(eq(threadMessageTable.messageId, messageId), landed)),
+		] as const);
+		await this.search.sync(listings.map((row) => row.threadMessageId));
+		return moved.length > 0;
+	}
+
 	async updateUid(
 		messageId: string,
 		newUid: number,
 		newMailboxId: string,
 	): ReturnType<IMessageRepository["updateUid"]> {
 		// updateUid settles a move against confirmed IMAP state (the destination
-		// mailbox and its COPYUID). The message's search vectors still carry the
-		// old mailbox in their metadata and their body is unchanged, so a normal
-		// re-index would skip them on content hash. Enqueue a move re-index event
-		// in the same transaction as the update; the search-index worker drains it
-		// with force, refreshing the stored mailbox metadata.
+		// mailbox and its COPYUID).
 		//
 		// `originalUid` goes with the settle. It exists to say the row's `uid`
 		// was recorded under `originalMailboxId`, and that stops being true here;
@@ -605,82 +669,27 @@ export class DrizzleMessageRepository implements IMessageRepository {
 		// in-flight one whenever the destination's counter happens to hand back
 		// the source's number, which on a young account is most of them (#1217).
 		// `originalMailboxId` stays: Undo restores the message to it.
-		const rows = await runInTransaction(this.db, async (tx) => {
-			const updated = await tx
-				.update(messageTable)
-				.set({
-					uid: newUid,
-					mailboxId: newMailboxId,
-					originalUid: null,
-					status: "active",
-					syncStatus: "synced",
-					// The settle clears the epitaph as well as the pair. `syncStatus`
-					// alone already gates every reader of it, so this is tidiness
-					// rather than correctness — but a row that has just settled has
-					// nothing it gave up on, and leaving a value there invites a
-					// reader that forgets the gate.
-					abandonedMutation: "none",
-					updatedAt: Date.now(),
-				})
-				.where(eq(messageTable.messageId, messageId))
-				.returning();
-			if (updated.length === 0) {
-				throw new NotFoundError(`Message not found: ${messageId}`);
-			}
-
-			await tx.insert(outboxTable).values({
-				id: randomUUID(),
-				messageId,
-				event: "message.moved",
-				payload: { messageId },
-				createdAt: new Date(),
-			});
-			return updated;
-		});
-		return toMessageItem(rows[0]);
-	}
-
-	async requestReembed(messageIds: string[]): Promise<ReembedRequest> {
-		const request: ReembedRequest = { queued: 0, alreadyQueued: 0 };
-		for (
-			let start = 0;
-			start < messageIds.length;
-			start += REEMBED_BATCH_SIZE
-		) {
-			const batch = messageIds.slice(start, start + REEMBED_BATCH_SIZE);
-			await runInTransaction(this.db, async (tx) => {
-				const present = await tx
-					.select({ messageId: messageTable.messageId })
-					.from(messageTable)
-					.where(inArray(messageTable.messageId, batch));
-				const pending = await tx
-					.selectDistinct({ messageId: outboxTable.messageId })
-					.from(outboxTable)
-					.where(
-						and(
-							inArray(outboxTable.messageId, batch),
-							eq(outboxTable.event, "message.moved"),
-							isNull(outboxTable.processedAt),
-						),
-					);
-				const pendingIds = new Set(pending.map(({ messageId }) => messageId));
-				const toQueue = present.filter(
-					({ messageId }) => !pendingIds.has(messageId),
-				);
-				request.alreadyQueued += present.length - toQueue.length;
-				request.queued += toQueue.length;
-				if (toQueue.length === 0) return;
-				await tx.insert(outboxTable).values(
-					toQueue.map(({ messageId }) => ({
-						id: randomUUID(),
-						messageId,
-						event: "message.moved" as const,
-						payload: { messageId },
-						createdAt: new Date(),
-					})),
-				);
-			});
+		const rows = await this.db
+			.update(messageTable)
+			.set({
+				uid: newUid,
+				mailboxId: newMailboxId,
+				originalUid: null,
+				status: "active",
+				syncStatus: "synced",
+				// The settle clears the epitaph as well as the pair. `syncStatus`
+				// alone already gates every reader of it, so this is tidiness
+				// rather than correctness — but a row that has just settled has
+				// nothing it gave up on, and leaving a value there invites a
+				// reader that forgets the gate.
+				abandonedMutation: "none",
+				updatedAt: Date.now(),
+			})
+			.where(eq(messageTable.messageId, messageId))
+			.returning();
+		if (rows.length === 0) {
+			throw new NotFoundError(`Message not found: ${messageId}`);
 		}
-		return request;
+		return toMessageItem(rows[0]);
 	}
 }

package/src/repos/sync-warning.ts

@@ -0,0 +1,76 @@
+import type {
+	ISyncWarningRepository,
+	SurfacedSyncWarnings,
+	SyncWarningItem,
+	SyncWarningKey,
+} from "@remit/data-ports";
+import { deriveSyncWarningId } from "@remit/data-ports/id";
+import { and, count, desc, eq, gte, inArray, sql } from "drizzle-orm";
+import type { Db } from "../db.js";
+import { syncWarningTable } from "../schema/sync-warning.js";
+
+type DB = Db<Record<string, unknown>>;
+
+export class SyncWarningRepo implements ISyncWarningRepository {
+	constructor(private db: DB) {}
+
+	listByMailboxId = async (mailboxId: string): Promise<SyncWarningItem[]> =>
+		this.db
+			.select()
+			.from(syncWarningTable)
+			.where(eq(syncWarningTable.mailboxId, mailboxId));
+
+	listSurfacedByAccountConfigId = async (
+		accountConfigId: string,
+		minAttempts: number,
+		limit: number,
+	): Promise<SurfacedSyncWarnings> => {
+		const surfaced = and(
+			eq(syncWarningTable.accountConfigId, accountConfigId),
+			gte(syncWarningTable.attempts, minAttempts),
+		);
+		const items = await this.db
+			.select()
+			.from(syncWarningTable)
+			.where(surfaced)
+			.orderBy(desc(syncWarningTable.createdAt))
+			.limit(limit);
+		const [{ total }] = await this.db
+			.select({ total: count() })
+			.from(syncWarningTable)
+			.where(surfaced);
+		return { items, total };
+	};
+
+	recordFailures = async (keys: SyncWarningKey[]): Promise<void> => {
+		if (keys.length === 0) return;
+		const now = Date.now();
+		await this.db
+			.insert(syncWarningTable)
+			.values(
+				keys.map((key) => ({
+					syncWarningId: deriveSyncWarningId(
+						key.accountId,
+						key.mailboxId,
+						key.uidValidity,
+						key.uid,
+						key.stage,
+					),
+					...key,
+					attempts: 1,
+					createdAt: now,
+				})),
+			)
+			.onConflictDoUpdate({
+				target: syncWarningTable.syncWarningId,
+				set: { attempts: sql`${syncWarningTable.attempts} + 1` },
+			});
+	};
+
+	deleteMany = async (syncWarningIds: string[]): Promise<void> => {
+		if (syncWarningIds.length === 0) return;
+		await this.db
+			.delete(syncWarningTable)
+			.where(inArray(syncWarningTable.syncWarningId, syncWarningIds));
+	};
+}