@remit/drizzle-service 0.0.13 → 0.0.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/drizzle-service",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "description": "Drizzle ORM service parameterized by dialect (Postgres / SQLite)",
   "type": "module",
   "main": "src/index.ts",

package/src/index.ts

@@ -40,6 +40,7 @@ export {
 } from "./repos/message.js";
 export { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
 export { MessageLabelRepo } from "./repos/message-label.js";
+export { QuarantineRepo } from "./repos/quarantine.js";
 export { DrizzleThreadMessageRepository } from "./repos/thread-message.js";
 export { DrizzleUnitOfWork } from "./repos/unit-of-work.js";
 export * from "./schema/i4-account-config.js";
@@ -54,6 +55,7 @@ export * from "./schema/i4-organize-job-request.js";
 export * from "./schema/i4-outbox-message.js";
 export * from "./schema/message-data.js";
 export { messageDataSchema } from "./schema/message-data.js";
+export * from "./schema/quarantine.js";
 export {
 	createSqliteDatabase,
 	type SqliteClient,

package/src/repos/cascade-delete.sqlite.test.ts

@@ -11,6 +11,7 @@ import {
 	envelopeTable,
 	mailboxTable,
 	messageTable,
+	quarantineTable,
 } from "../schema.js";
 import { createSqliteTestDb } from "../test-db-sqlite.js";
 import {
@@ -22,11 +23,13 @@ import {
 // this harness pushes must live on a real file (not the in-memory default) for
 // the deleter's own connection to see it. `deleteMessageSubtree` touches every
 // message-data child table, so push the full message-data schema plus the
-// account and mailbox containers.
+// account and mailbox containers, plus quarantine — the cascade deletes those
+// rows set-wise by account, so the table has to exist here too.
 const CASCADE_SCHEMA = {
 	...messageDataSchema,
 	account: accountTable,
 	mailbox: mailboxTable,
+	quarantine: quarantineTable,
 };
 
 const log = { info: () => {} };

package/src/repos/cascade-delete.test.ts

@@ -16,6 +16,7 @@ import {
 	messageTable,
 	outboxMessageTable,
 	outboxTable,
+	quarantineTable,
 	threadMessageTable,
 } from "../schema.js";
 import { createTestDb, type TestDb } from "../test-db.js";
@@ -40,6 +41,7 @@ const ROOT_BODY_PART_ID = "00000000-0000-4000-8000-000000000002";
 const MESSAGE_FLAG_ID = "00000000-0000-4000-8000-000000000003";
 const BODY_PART_ID = "00000000-0000-4000-8000-000000000004";
 const OTHER_MAILBOX_ID = "mbx-cascade-2";
+const QUARANTINE_ID = "quarantine-cascade-1";
 
 describe("runDrizzleCascadeDelete", () => {
 	let db: TestDb;
@@ -228,6 +230,25 @@ describe("runDrizzleCascadeDelete", () => {
 				updatedAt: NOW,
 			},
 		]);
+
+		await db.insert(quarantineTable).values({
+			quarantineId: QUARANTINE_ID,
+			accountConfigId: ACCOUNT_CONFIG_ID,
+			accountId: ACCOUNT_ID,
+			mailboxId: MAILBOX_ID,
+			uidValidity: 1_712_000_000,
+			uid: 40217,
+			mailboxPath: "Clients/Acme Holdings",
+			quarantinedAt: NOW,
+			attempts: 3,
+			failureStage: "BodyParse",
+			failureCode: "UnterminatedMultipartBoundary",
+			failureMessage: "multipart boundary was never closed",
+			workerVersion: "worker 1.0.0",
+			structure: [{ depth: 0, contentType: "multipart/mixed" }],
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
 	});
 
 	after(async () => {
@@ -397,6 +418,16 @@ describe("runDrizzleCascadeDelete", () => {
 			0,
 			"both the Seen and Flagged markers (composite key) must be deleted",
 		);
+		assert.equal(
+			(
+				await db
+					.select()
+					.from(quarantineTable)
+					.where(eq(quarantineTable.quarantineId, QUARANTINE_ID))
+			).length,
+			0,
+			"quarantine rows carry the user's folder names and parser output, so account deletion must take them",
+		);
 	});
 
 	test("emits a message.removed outbox row for search-index cleanup", async () => {

package/src/repos/cascade-delete.ts

@@ -9,6 +9,7 @@ import { messageFlagPushTable } from "../schema/i4-message-flag-push.js";
 import { messagePlacementMoveTable } from "../schema/i4-message-placement-move.js";
 import { outboxMessageTable } from "../schema/i4-outbox-message.js";
 import { messageDataSchema } from "../schema/message-data.js";
+import { quarantineTable } from "../schema/quarantine.js";
 import { threadMessageTable } from "../schema/thread-message.js";
 import { createSqliteDatabase } from "../sqlite-client.js";
 import { runInTransaction } from "../tx.js";
@@ -151,6 +152,13 @@ export const runDrizzleCascadeDelete = async (
 
 		const accountIds = (grouped.get("Account") ?? []).map((k) => k.accountId);
 		if (accountIds.length > 0) {
+			// Quarantine rows are deleted set-wise by account rather than from an
+			// enumerated key, the way message subtrees are. They carry the user's
+			// own folder names and parser output quoting the message, so account
+			// deletion has to take them even though nothing enumerates them.
+			await tx
+				.delete(quarantineTable)
+				.where(inArray(quarantineTable.accountId, accountIds));
 			await tx
 				.delete(accountTable)
 				.where(inArray(accountTable.accountId, accountIds));

package/src/repos/quarantine.test.ts

@@ -0,0 +1,112 @@
+import assert from "node:assert";
+import { after, before, describe, test } from "node:test";
+import { quarantineTable } from "../schema/quarantine.js";
+import { createTestDb, randomId, type TestDb } from "../test-db.js";
+import { QuarantineRepo } from "./quarantine.js";
+
+describe("QuarantineRepo", () => {
+	let db: TestDb;
+	let close: () => Promise<void>;
+	let repo: QuarantineRepo;
+
+	const seed = async (
+		overrides: Partial<typeof quarantineTable.$inferInsert> & {
+			accountConfigId: string;
+		},
+	): Promise<string> => {
+		const quarantineId = randomId();
+		const now = Date.now();
+		await db.insert(quarantineTable).values({
+			quarantineId,
+			accountId: randomId(),
+			mailboxId: randomId(),
+			uidValidity: 1_712_000_000,
+			uid: 40217,
+			mailboxPath: "INBOX",
+			quarantinedAt: now,
+			attempts: 3,
+			failureStage: "BodyParse",
+			failureCode: "UnterminatedMultipartBoundary",
+			failureMessage: "multipart boundary was never closed",
+			workerVersion: "worker 1.0.0",
+			structure: [{ depth: 0, contentType: "multipart/mixed" }],
+			createdAt: now,
+			updatedAt: now,
+			...overrides,
+		});
+		return quarantineId;
+	};
+
+	before(async () => {
+		({ db, close } = await createTestDb());
+		repo = new QuarantineRepo(db as never);
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	test("lists one user's entries newest first", async () => {
+		const accountConfigId = randomId();
+		const older = await seed({ accountConfigId, quarantinedAt: 1_000 });
+		const newer = await seed({ accountConfigId, quarantinedAt: 2_000 });
+
+		const entries = await repo.listByAccountConfigId(accountConfigId);
+
+		assert.deepEqual(
+			entries.map((entry) => entry.quarantineId),
+			[newer, older],
+		);
+	});
+
+	test("never returns another user's entries", async () => {
+		const mine = randomId();
+		await seed({ accountConfigId: mine });
+		await seed({ accountConfigId: randomId() });
+
+		const entries = await repo.listByAccountConfigId(mine);
+
+		assert.equal(entries.length, 1);
+		assert.equal(entries[0].accountConfigId, mine);
+	});
+
+	test("returns the MIME tree as the pre-order walk it was written as", async () => {
+		const accountConfigId = randomId();
+		await seed({
+			accountConfigId,
+			structure: [
+				{ depth: 0, contentType: "multipart/mixed" },
+				{ depth: 1, contentType: "text/plain" },
+				{ depth: 1, contentType: "application/pdf" },
+			],
+		});
+
+		const [entry] = await repo.listByAccountConfigId(accountConfigId);
+
+		assert.deepEqual(entry.structure, [
+			{ depth: 0, contentType: "multipart/mixed" },
+			{ depth: 1, contentType: "text/plain" },
+			{ depth: 1, contentType: "application/pdf" },
+		]);
+	});
+
+	test("carries the optional diagnostics as absent, not null", async () => {
+		const accountConfigId = randomId();
+		await seed({ accountConfigId });
+
+		const [entry] = await repo.listByAccountConfigId(accountConfigId);
+
+		// A folder nobody appointed a role to, a whole-body failure and an
+		// undeclared charset are all ordinary. The API contract makes them
+		// optional, so a null row value must not leak through as `null`.
+		assert.equal(entry.mailboxRole, undefined);
+		assert.equal(entry.failurePartPath, undefined);
+		assert.equal(entry.charset, undefined);
+		// The message-shape fields come off one optional BODYSTRUCTURE, so a
+		// message that failed before it was read carries none of them.
+		assert.equal(entry.contentType, undefined);
+		assert.equal(entry.transferEncoding, undefined);
+		assert.equal(entry.sizeBytes, undefined);
+		assert.equal(entry.messageIdHash, undefined);
+	});
+});

package/src/repos/quarantine.ts

@@ -0,0 +1,57 @@
+import type {
+	IQuarantineRepository,
+	QuarantineItem,
+	QuarantineMimeNodeItem,
+} from "@remit/data-ports";
+import { desc, eq } from "drizzle-orm";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { quarantineTable } from "../schema/quarantine.js";
+
+type DB = NodePgDatabase<Record<string, unknown>>;
+
+function rowToItem(row: typeof quarantineTable.$inferSelect): QuarantineItem {
+	return {
+		quarantineId: row.quarantineId,
+		accountConfigId: row.accountConfigId,
+		accountId: row.accountId,
+		mailboxId: row.mailboxId,
+		uidValidity: row.uidValidity,
+		uid: row.uid,
+		mailboxRole: row.mailboxRole ?? undefined,
+		mailboxPath: row.mailboxPath,
+		quarantinedAt: row.quarantinedAt,
+		attempts: row.attempts,
+		failureStage: row.failureStage,
+		failureCode: row.failureCode,
+		failureMessage: row.failureMessage,
+		failurePartPath: row.failurePartPath ?? undefined,
+		workerVersion: row.workerVersion,
+		contentType: row.contentType ?? undefined,
+		transferEncoding: row.transferEncoding ?? undefined,
+		charset: row.charset ?? undefined,
+		sizeBytes: row.sizeBytes ?? undefined,
+		structure: row.structure as QuarantineMimeNodeItem[],
+		messageIdHash: row.messageIdHash ?? undefined,
+		createdAt: row.createdAt,
+		updatedAt: row.updatedAt,
+	};
+}
+
+/**
+ * Read side of the message quarantine (issue #72). The rows are written by the
+ * sync worker; nothing in the API process creates or clears one.
+ */
+export class QuarantineRepo implements IQuarantineRepository {
+	constructor(private db: DB) {}
+
+	listByAccountConfigId = async (
+		accountConfigId: string,
+	): Promise<QuarantineItem[]> => {
+		const rows = await this.db
+			.select()
+			.from(quarantineTable)
+			.where(eq(quarantineTable.accountConfigId, accountConfigId))
+			.orderBy(desc(quarantineTable.quarantinedAt));
+		return rows.map(rowToItem);
+	};
+}

package/src/repos/thread-message.test.ts

@@ -382,6 +382,281 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
 	});
 });
 
+// ─── searchByDate: the unified listing's cross-folder search mode ─────────────
+// The daily brief's unscoped search reaches every folder of every account in
+// one query, so matching must span the caller-supplied mailbox scope rather
+// than a single mailbox.
+
+describe("DrizzleThreadMessageRepository.searchByDate", {
+	skip: !RUN_INTEG,
+}, () => {
+	let repo: DrizzleThreadMessageRepository;
+	const cleanup: Array<() => Promise<void>> = [];
+
+	before(async () => {
+		await setupDb();
+		repo = new DrizzleThreadMessageRepository(PG_URL);
+	});
+
+	after(async () => {
+		for (const fn of cleanup.reverse()) {
+			await fn();
+		}
+		await repo.close();
+	});
+
+	async function seed(
+		accountConfigId: string,
+		mailboxId: string,
+		rows: Array<Partial<CreateThreadMessageInput>>,
+	): Promise<void> {
+		for (const overrides of rows) {
+			const created = await repo.create(
+				makeInput(accountConfigId, mailboxId, overrides),
+			);
+			cleanup.push(() => repo.delete(accountConfigId, created.threadMessageId));
+		}
+	}
+
+	test("matches across every mailbox in the scope, not just the inbox", async () => {
+		const acct = uuid();
+		const inbox = uuid();
+		const archive = uuid();
+		const spam = uuid();
+		const now = Date.now();
+		await seed(acct, inbox, [
+			{ subject: "invoice inbox", sentDate: now, internalDate: now },
+		]);
+		await seed(acct, archive, [
+			{ subject: "invoice archive", sentDate: now - 1, internalDate: now - 1 },
+		]);
+		await seed(acct, spam, [
+			{ subject: "invoice spam", sentDate: now - 2, internalDate: now - 2 },
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "invoice" },
+			{ excludeDeleted: true, mailboxIds: new Set([inbox, archive, spam]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["invoice inbox", "invoice archive", "invoice spam"],
+			"newest first, across all three folders",
+		);
+	});
+
+	test("a mailbox outside the scope contributes nothing", async () => {
+		const acct = uuid();
+		const inbox = uuid();
+		const excluded = uuid();
+		const now = Date.now();
+		await seed(acct, inbox, [
+			{ subject: "receipt kept", sentDate: now, internalDate: now },
+		]);
+		await seed(acct, excluded, [
+			{ subject: "receipt dropped", sentDate: now - 1, internalDate: now - 1 },
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "receipt" },
+			{ excludeDeleted: true, mailboxIds: new Set([inbox]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["receipt kept"],
+		);
+	});
+
+	test("every whitespace-separated term must match", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{
+				subject: "quarterly invoice",
+				fromEmail: "billing@acme.test",
+				sentDate: now,
+				internalDate: now,
+			},
+			{
+				subject: "quarterly report",
+				fromEmail: "reports@acme.test",
+				sentDate: now - 1,
+				internalDate: now - 1,
+			},
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "quarterly invoice" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["quarterly invoice"],
+		);
+	});
+
+	test("a term matches the From address as well as the subject", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{
+				subject: "no keyword here",
+				fromEmail: "penelope@acme.test",
+				sentDate: now,
+				internalDate: now,
+			},
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "penelope" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]) },
+		);
+
+		assert.equal(result.items.length, 1);
+	});
+
+	test("soft-deleted rows stay out", async () => {
+		const acct = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(acct, mbx, [
+			{ subject: "parcel live", sentDate: now, internalDate: now },
+			{
+				subject: "parcel gone",
+				isDeleted: true,
+				sentDate: now - 1,
+				internalDate: now - 1,
+			},
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "parcel" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["parcel live"],
+		);
+	});
+
+	// A short page means the matches ran out, never that a read window did — the
+	// contract the endpoint documents for search mode.
+	test("pages over matches, and a full page yields a resumable cursor", async () => {
+		const acct = uuid();
+		const inbox = uuid();
+		const archive = uuid();
+		const now = Date.now();
+		await seed(acct, inbox, [
+			{ subject: "noise a", sentDate: now, internalDate: now },
+			{ subject: "gamma one", sentDate: now - 1, internalDate: now - 1 },
+			{ subject: "noise b", sentDate: now - 2, internalDate: now - 2 },
+		]);
+		await seed(acct, archive, [
+			{ subject: "gamma two", sentDate: now - 3, internalDate: now - 3 },
+			{ subject: "gamma three", sentDate: now - 4, internalDate: now - 4 },
+		]);
+
+		const scope = {
+			excludeDeleted: true,
+			mailboxIds: new Set([inbox, archive]),
+		};
+
+		const page1 = await repo.searchByDate(
+			acct,
+			{ query: "gamma" },
+			{ ...scope, limit: 2 },
+		);
+		assert.deepEqual(
+			page1.items.map((r) => r.subject),
+			["gamma one", "gamma two"],
+			"a full page of matches, skipping the non-matching newer rows",
+		);
+		assert.ok(page1.continuationToken, "more matches remain — cursor expected");
+
+		const page2 = await repo.searchByDate(
+			acct,
+			{ query: "gamma" },
+			{ ...scope, limit: 2, continuationToken: page1.continuationToken },
+		);
+		assert.deepEqual(
+			page2.items.map((r) => r.subject),
+			["gamma three"],
+			"the last page is short because the matches ran out",
+		);
+		assert.equal(
+			page2.continuationToken,
+			undefined,
+			"a short page ends the pagination",
+		);
+	});
+
+	test("starred narrows the search without changing the scope", async () => {
+		const acct = uuid();
+		const archive = uuid();
+		const now = Date.now();
+		await seed(acct, archive, [
+			{
+				subject: "delta starred",
+				hasStars: true,
+				sentDate: now,
+				internalDate: now,
+			},
+			{
+				subject: "delta plain",
+				hasStars: false,
+				sentDate: now - 1,
+				internalDate: now - 1,
+			},
+		]);
+
+		const result = await repo.searchByDate(
+			acct,
+			{ query: "delta", starred: true },
+			{ excludeDeleted: true, mailboxIds: new Set([archive]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["delta starred"],
+		);
+	});
+
+	test("another config's mail is never returned", async () => {
+		const mine = uuid();
+		const theirs = uuid();
+		const mbx = uuid();
+		const now = Date.now();
+		await seed(mine, mbx, [
+			{ subject: "epsilon mine", sentDate: now, internalDate: now },
+		]);
+		await seed(theirs, mbx, [
+			{ subject: "epsilon theirs", sentDate: now - 1, internalDate: now - 1 },
+		]);
+
+		const result = await repo.searchByDate(
+			mine,
+			{ query: "epsilon" },
+			{ excludeDeleted: true, mailboxIds: new Set([mbx]) },
+		);
+
+		assert.deepEqual(
+			result.items.map((r) => r.subject),
+			["epsilon mine"],
+		);
+	});
+});
+
 // ─── Native text-search semantics ─────────────────────────────────────────────
 // The type-ahead search box lowercases the query before sending it. These tests
 // pin the Postgres-native behaviour: case- and accent-insensitive substring

package/src/repos/thread-message.ts

@@ -478,6 +478,65 @@ export class DrizzleThreadMessageRepository
 		};
 	}
 
+	/**
+	 * Cross-mailbox search for the unified listing's search mode. Same predicate
+	 * builder and keyset cursor as `searchByMailboxWindow`, with the mailbox
+	 * equality swapped for the caller's scope set. Matching runs in SQL over the
+	 * whole scope, so a short page means the matches are exhausted.
+	 */
+	async searchByDate(
+		accountConfigId: string,
+		search: SearchOptions,
+		options?: {
+			order?: "asc" | "desc";
+			limit?: number;
+			continuationToken?: string;
+			mailboxIds?: Set<string>;
+			excludeDeleted?: boolean;
+		},
+	): Promise<ResultList<ThreadMessageItem>> {
+		const order = options?.order ?? "desc";
+		const limit = clampThreadSearchLimit(options?.limit);
+		const cursor = options?.continuationToken
+			? decodeDateCursor(options.continuationToken)
+			: null;
+
+		const mailboxCond = options?.mailboxIds?.size
+			? inArray(threadMessageTable.mailboxId, [...options.mailboxIds])
+			: undefined;
+
+		const rows = await this.db
+			.select()
+			.from(threadMessageTable)
+			.where(
+				and(
+					eq(threadMessageTable.accountConfigId, accountConfigId),
+					mailboxCond,
+					options?.excludeDeleted
+						? eq(threadMessageTable.isDeleted, false)
+						: undefined,
+					...buildSearchConditions(search),
+					sentDateCursorCond(order, cursor),
+				),
+			)
+			.orderBy(
+				order === "desc"
+					? desc(threadMessageTable.sentDate)
+					: asc(threadMessageTable.sentDate),
+				asc(threadMessageTable.threadMessageId),
+			)
+			.limit(limit);
+
+		const lastRow = rows[rows.length - 1];
+		return {
+			items: rows.map(toItem),
+			continuationToken:
+				rows.length === limit && lastRow
+					? encodeDateCursor(lastRow.sentDate, lastRow.threadMessageId)
+					: undefined,
+		};
+	}
+
 	async listByStarred(
 		accountConfigId: string,
 		options?: {

package/src/schema/quarantine.ts

@@ -0,0 +1,3 @@
+import { entities } from "./active-entities.js";
+
+export const quarantineTable = entities.quarantines;

package/src/schema.ts

@@ -25,4 +25,5 @@ export * from "./schema/i4-message-placement-move.js";
 export * from "./schema/i4-organize-job-request.js";
 export * from "./schema/i4-outbox-message.js";
 export * from "./schema/message-data.js";
+export * from "./schema/quarantine.js";
 export { threadMessageTable } from "./schema/thread-message.js";