@remit/drizzle-service 0.0.99 → 0.0.101

package/src/repos/mappers.ts

@@ -1,5 +1,4 @@
 import type {
-	BodyPartContentItem,
 	BodyPartItem,
 	BodyPartParameterItem,
 	BodyPartStorageItem,
@@ -9,7 +8,6 @@ import type {
 	RawMessageStorageItem,
 } from "@remit/data-ports";
 import type {
-	bodyPartContentTable,
 	bodyPartParameterTable,
 	bodyPartStorageTable,
 	bodyPartTable,
@@ -155,17 +153,3 @@ export function toBodyPartStorageItem(
 		...(row.dedupHash !== null ? { dedupHash: row.dedupHash } : {}),
 	};
 }
-
-export function toBodyPartContentItem(
-	row: typeof bodyPartContentTable.$inferSelect,
-): BodyPartContentItem {
-	return {
-		bodyPartContentId: row.bodyPartContentId,
-		messageId: row.messageId,
-		bodyPartId: row.bodyPartId,
-		content: row.content,
-		contentLength: row.contentLength,
-		createdAt: row.createdAt,
-		updatedAt: row.updatedAt,
-	};
-}

package/src/repos/message.sqlite.test.ts

@@ -1,25 +1,23 @@
 import assert from "node:assert/strict";
 import { after, before, describe, test } from "node:test";
-import { and, eq, isNull } from "drizzle-orm";
 import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
 } from "../id.js";
-import {
-	type MessageDataSchema,
-	messageDataSchema,
-	outboxTable,
-} from "../schema/message-data.js";
+import { messageDataSchema } from "../schema/message-data.js";
+import { threadMessageTable } from "../schema/thread-message.js";
+import { SearchIndex } from "../search/thread-message-search.js";
 import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
 import { DrizzleMessageRepository } from "./message.js";
 
+const SCHEMA = { ...messageDataSchema, threadMessages: threadMessageTable };
+
 // Runs the ported message repo against a real better-sqlite3 database
-// (RFC 036 D1). Exercises the SQLite-specific seams: the SAVEPOINT-bracketed
-// transaction, the sqlite outbox table (text-json payload), the unique-
-// violation → CreateFailedConflictError mapping, and the JSON/boolean columns.
+// (RFC 036 D1). Exercises the SQLite-specific seams: the batched delete, the
+// unique-violation → CreateFailedConflictError mapping, and the JSON/boolean columns.
 
 describe("DrizzleMessageRepository (sqlite)", () => {
-	let db: SqliteTestDb<MessageDataSchema>;
+	let db: SqliteTestDb<typeof SCHEMA>;
 	let close: () => Promise<void>;
 	let repo: DrizzleMessageRepository;
 
@@ -44,29 +42,19 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 	};
 
 	before(async () => {
-		({ db, close } = await createSqliteTestDb(messageDataSchema));
-		repo = new DrizzleMessageRepository(db);
+		({ db, close } = await createSqliteTestDb(SCHEMA));
+		repo = new DrizzleMessageRepository(db, new SearchIndex(db, db));
 	});
 
 	after(async () => {
 		await close();
 	});
 
-	test("create returns a MessageItem and writes no outbox row", async () => {
+	test("create returns a MessageItem", async () => {
 		const item = await repo.create(BASE_INPUT);
 		assert.equal(item.messageId, MESSAGE_ID);
 		assert.equal(item.status, "active");
 		assert.equal(item.hasListUnsubscribe, true);
-
-		const rows = await db
-			.select()
-			.from(outboxTable)
-			.where(eq(outboxTable.messageId, MESSAGE_ID));
-		assert.equal(
-			rows.length,
-			0,
-			"a message with no body yet has nothing to index",
-		);
 	});
 
 	test("boolean and json columns round-trip", async () => {
@@ -122,26 +110,11 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		assert.equal(afterUpdate.placementVerdict?.action, "MoveToInbox");
 	});
 
-	test("duplicate messageId throws CreateFailedConflictError and appends no outbox row", async () => {
-		const before = await db
-			.select()
-			.from(outboxTable)
-			.where(eq(outboxTable.messageId, MESSAGE_ID));
-
+	test("duplicate messageId throws CreateFailedConflictError", async () => {
 		await assert.rejects(
 			() => repo.create(BASE_INPUT),
 			(err: Error) => err.name === "CreateFailedConflictError",
 		);
-
-		const afterRows = await db
-			.select()
-			.from(outboxTable)
-			.where(eq(outboxTable.messageId, MESSAGE_ID));
-		assert.equal(
-			afterRows.length,
-			before.length,
-			"rolled-back create must not append an outbox row",
-		);
 	});
 
 	test("upsertWithStatus reports created=false for an existing row", async () => {
@@ -153,6 +126,25 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 	// The state every inbound sync row is born in, and the one the re-point gate
 	// in mailbox-service has to accept (#1096): the sync path supplies no
 	// syncStatus and nothing later promotes what the repository writes here.
+	test("two concurrent saves of one message both succeed, one of them creating it", async () => {
+		const messageId = "00000000-0000-0000-1111-000000000099";
+		const input = {
+			...BASE_INPUT,
+			messageId,
+			envelopeId: deriveEnvelopeId(messageId),
+			rootBodyPartId: deriveRootBodyPartId(messageId),
+		};
+
+		const saves = await Promise.all([
+			repo.upsertWithStatus(input),
+			repo.upsertWithStatus(input),
+		]);
+
+		assert.deepEqual(saves.map((save) => save.created).sort(), [false, true]);
+		assert.equal(saves[0]?.item.messageId, messageId);
+		assert.equal(saves[1]?.item.messageId, messageId);
+	});
+
 	test("upsertWithStatus writes pending when the caller names no syncStatus", async () => {
 		const inboundId = "00000000-0000-0000-2222-000000000009";
 		const { item, created } = await repo.upsertWithStatus({
@@ -283,74 +275,8 @@ describe("DrizzleMessageRepository (sqlite)", () => {
 		});
 	});
 
-	test("requestReembed queues a forced re-index for each message that still exists", async () => {
-		const movedRows = () =>
-			db
-				.select()
-				.from(outboxTable)
-				.where(
-					and(
-						eq(outboxTable.messageId, MESSAGE_ID),
-						eq(outboxTable.event, "message.moved"),
-						isNull(outboxTable.processedAt),
-					),
-				);
-		const pendingBefore = (await movedRows()).length;
-
-		const queued = await repo.requestReembed([
-			MESSAGE_ID,
-			"00000000-0000-0000-2222-0000000000fe",
-		]);
-
-		assert.deepEqual(queued, { queued: 1, alreadyQueued: 0 });
-		assert.equal((await movedRows()).length, pendingBefore + 1);
-		const orphan = await db
-			.select()
-			.from(outboxTable)
-			.where(eq(outboxTable.messageId, "00000000-0000-0000-2222-0000000000fe"));
-		assert.equal(orphan.length, 0);
-	});
-
-	test("requestReembed skips a message whose forced re-index is still pending", async () => {
-		const pendingBefore = await db
-			.select()
-			.from(outboxTable)
-			.where(
-				and(
-					eq(outboxTable.messageId, MESSAGE_ID),
-					eq(outboxTable.event, "message.moved"),
-					isNull(outboxTable.processedAt),
-				),
-			);
-		assert.ok(pendingBefore.length > 0);
-
-		const again = await repo.requestReembed([MESSAGE_ID]);
-
-		assert.deepEqual(again, { queued: 0, alreadyQueued: 1 });
-		const pendingAfter = await db
-			.select()
-			.from(outboxTable)
-			.where(
-				and(
-					eq(outboxTable.messageId, MESSAGE_ID),
-					eq(outboxTable.event, "message.moved"),
-					isNull(outboxTable.processedAt),
-				),
-			);
-		assert.equal(pendingAfter.length, pendingBefore.length);
-	});
-
-	test("delete removes the message and appends a removal outbox row", async () => {
+	test("delete removes the message", async () => {
 		await repo.delete(MESSAGE_ID);
 		await assert.rejects(() => repo.get(MESSAGE_ID));
-
-		const rows = await db
-			.select()
-			.from(outboxTable)
-			.where(eq(outboxTable.messageId, MESSAGE_ID));
-		assert.ok(
-			rows.some((r) => r.event === "message.removed"),
-			"delete must append a message.removed outbox row",
-		);
 	});
 });

package/src/repos/message.test.ts

@@ -4,6 +4,7 @@ import {
 	envelopeId as deriveEnvelopeId,
 	rootBodyPartId as deriveRootBodyPartId,
 } from "../id.js";
+import { SearchIndex } from "../search/thread-message-search.js";
 import { DrizzleEnvelopeRepository } from "./envelope.js";
 import { DrizzleMessageRepository } from "./message.js";
 import { createTestDb, type TestDb } from "./test-helpers.js";
@@ -36,7 +37,7 @@ describe("DrizzleMessageRepository", () => {
 
 	before(async () => {
 		({ db, stop } = await createTestDb());
-		messageRepo = new DrizzleMessageRepository(db);
+		messageRepo = new DrizzleMessageRepository(db, new SearchIndex(db, db));
 		envelopeRepo = new DrizzleEnvelopeRepository(db);
 	});
 
@@ -56,25 +57,7 @@ describe("DrizzleMessageRepository", () => {
 			assert.ok(typeof item.updatedAt === "number");
 		});
 
-		test("writes no outbox row — there is nothing to index before the body", async () => {
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { eq } = await import("drizzle-orm");
-			const rows = await db
-				.select()
-				.from(outboxTable)
-				.where(eq(outboxTable.messageId, MESSAGE_ID));
-			assert.equal(rows.length, 0);
-		});
-
-		test("duplicate messageId throws CreateFailedConflictError and writes no extra outbox row", async () => {
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { eq } = await import("drizzle-orm");
-
-			const before = await db
-				.select()
-				.from(outboxTable)
-				.where(eq(outboxTable.messageId, MESSAGE_ID));
-
+		test("duplicate messageId throws CreateFailedConflictError", async () => {
 			await assert.rejects(
 				() => messageRepo.create(BASE_MESSAGE_INPUT),
 				(err: Error) => {
@@ -82,16 +65,6 @@ describe("DrizzleMessageRepository", () => {
 					return true;
 				},
 			);
-
-			const after = await db
-				.select()
-				.from(outboxTable)
-				.where(eq(outboxTable.messageId, MESSAGE_ID));
-			assert.equal(
-				after.length,
-				before.length,
-				"rolled-back create must not add an outbox row",
-			);
 		});
 	});
 
@@ -119,23 +92,6 @@ describe("DrizzleMessageRepository", () => {
 			});
 		});
 
-		test("returns all 10 MessageDescription members", async () => {
-			const desc = await messageRepo.describe(MESSAGE_ID);
-			const keys = Object.keys(desc).sort();
-			assert.deepStrictEqual(keys, [
-				"bodyPart",
-				"bodyPartContent",
-				"bodyPartParameter",
-				"bodyPartStorage",
-				"envelope",
-				"envelopeAddress",
-				"message",
-				"messageFlag",
-				"messageReference",
-				"rawMessageStorage",
-			]);
-		});
-
 		test("message array has 1 row with correct fields", async () => {
 			const desc = await messageRepo.describe(MESSAGE_ID);
 			assert.equal(desc.message.length, 1);
@@ -160,7 +116,6 @@ describe("DrizzleMessageRepository", () => {
 			assert.equal(desc.bodyPartParameter.length, 0);
 			assert.equal(desc.rawMessageStorage.length, 0);
 			assert.equal(desc.bodyPartStorage.length, 0);
-			assert.equal(desc.bodyPartContent.length, 0);
 		});
 
 		test("throws NotFoundError for unknown messageId", async () => {
@@ -235,22 +190,7 @@ describe("DrizzleMessageRepository", () => {
 			);
 		});
 
-		test("updateForMove writes no move reindex event (completion-only)", async () => {
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { and, eq } = await import("drizzle-orm");
-			const rows = await db
-				.select()
-				.from(outboxTable)
-				.where(
-					and(
-						eq(outboxTable.messageId, MOVE_MESSAGE_ID),
-						eq(outboxTable.event, "message.moved"),
-					),
-				);
-			assert.equal(rows.length, 0);
-		});
-
-		test("updateUid completes the move and emits a move reindex event", async () => {
+		test("updateUid completes the move", async () => {
 			const completed = await messageRepo.updateUid(
 				MOVE_MESSAGE_ID,
 				99,
@@ -265,21 +205,6 @@ describe("DrizzleMessageRepository", () => {
 			assert.equal(read.uid, 99);
 			assert.equal(read.status, "active");
 			assert.equal(read.syncStatus, "synced");
-
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { and, eq } = await import("drizzle-orm");
-			const rows = await db
-				.select()
-				.from(outboxTable)
-				.where(
-					and(
-						eq(outboxTable.messageId, MOVE_MESSAGE_ID),
-						eq(outboxTable.event, "message.moved"),
-					),
-				);
-			assert.equal(rows.length, 1, "one move reindex event was enqueued");
-			assert.deepStrictEqual(rows[0].payload, { messageId: MOVE_MESSAGE_ID });
-			assert.equal(rows[0].processedAt, null, "event starts undrained");
 		});
 
 		test("updateUid drops the source uid and keeps the folder Undo restores to", async () => {
@@ -304,7 +229,7 @@ describe("DrizzleMessageRepository", () => {
 			);
 		});
 
-		test("updateUid throws NotFoundError and rolls back its reindex event", async () => {
+		test("updateUid throws NotFoundError for an unknown messageId", async () => {
 			const UNKNOWN = "00000000-0000-0000-9999-000000000002";
 			await assert.rejects(
 				() => messageRepo.updateUid(UNKNOWN, 1, DEST_MAILBOX_ID),
@@ -313,83 +238,10 @@ describe("DrizzleMessageRepository", () => {
 					return true;
 				},
 			);
-
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { eq } = await import("drizzle-orm");
-			const rows = await db
-				.select()
-				.from(outboxTable)
-				.where(eq(outboxTable.messageId, UNKNOWN));
-			assert.equal(rows.length, 0, "rolled-back move writes no outbox row");
 		});
 	});
 
-	describe("update — appends a body_synced outbox row per body write", () => {
-		const SYNC_MESSAGE_ID = "00000000-0000-0000-3333-000000000001";
-		const SYNC_MAILBOX_ID = "00000000-0000-0000-3333-000000000002";
-
-		const bodySyncedRows = async () => {
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { and, eq } = await import("drizzle-orm");
-			return db
-				.select()
-				.from(outboxTable)
-				.where(
-					and(
-						eq(outboxTable.messageId, SYNC_MESSAGE_ID),
-						eq(outboxTable.event, "message.body_synced"),
-					),
-				);
-		};
-
-		before(async () => {
-			await messageRepo.create({
-				messageId: SYNC_MESSAGE_ID,
-				mailboxId: SYNC_MAILBOX_ID,
-				uid: 11,
-				sequenceNumber: 1,
-				rfc822Size: 256,
-				internalDate: NOW,
-				envelopeId: deriveEnvelopeId(SYNC_MESSAGE_ID),
-				rootBodyPartId: deriveRootBodyPartId(SYNC_MESSAGE_ID),
-				status: "active" as const,
-				syncStatus: "synced" as const,
-			});
-		});
-
-		test("a body write appends one body_synced event", async () => {
-			assert.equal((await bodySyncedRows()).length, 0, "none before sync");
-			await messageRepo.update(SYNC_MESSAGE_ID, {
-				bodyStorageKey: "s3://bucket/body-a.eml",
-			});
-			const rows = await bodySyncedRows();
-			assert.equal(rows.length, 1, "one row for one body write");
-			assert.deepStrictEqual(rows[0].payload, { messageId: SYNC_MESSAGE_ID });
-			assert.equal(rows[0].processedAt, null, "event starts undrained");
-		});
-
-		test("each body write appends its own row (append-only, no suppression)", async () => {
-			await messageRepo.update(SYNC_MESSAGE_ID, {
-				bodyStorageKey: "s3://bucket/body-b.eml",
-			});
-			await messageRepo.update(SYNC_MESSAGE_ID, {
-				bodyStorageKey: "s3://bucket/body-c.eml",
-			});
-			assert.equal(
-				(await bodySyncedRows()).length,
-				3,
-				"three body writes leave three rows — coalescing is the worker's job",
-			);
-		});
-
-		test("a classification-only update (no bodyStorageKey) appends nothing", async () => {
-			const before = (await bodySyncedRows()).length;
-			await messageRepo.update(SYNC_MESSAGE_ID, { category: "personal" });
-			assert.equal((await bodySyncedRows()).length, before);
-		});
-	});
-
-	describe("deleteMany — subtree delete + message.removed outbox", () => {
+	describe("deleteMany — subtree delete", () => {
 		const DEL_MESSAGE_ID = "00000000-0000-0000-4444-000000000001";
 		const DEL_MAILBOX_ID = "00000000-0000-0000-4444-000000000002";
 		const DEL_ENVELOPE_ID = deriveEnvelopeId(DEL_MESSAGE_ID);
@@ -480,19 +332,6 @@ describe("DrizzleMessageRepository", () => {
 			);
 		});
 
-		test("leaves exactly one undrained message.removed outbox row", async () => {
-			const { outboxTable } = await import("../schema/message-data.js");
-			const { eq } = await import("drizzle-orm");
-			const rows = await db
-				.select()
-				.from(outboxTable)
-				.where(eq(outboxTable.messageId, DEL_MESSAGE_ID));
-			assert.equal(rows.length, 1, "prior CDC rows are cleared");
-			assert.equal(rows[0].event, "message.removed");
-			assert.equal(rows[0].processedAt, null, "removal starts undrained");
-			assert.deepStrictEqual(rows[0].payload, { messageId: DEL_MESSAGE_ID });
-		});
-
 		test("deleteMany([]) is a no-op", async () => {
 			await messageRepo.deleteMany([]);
 		});