@remit/mailbox-service 0.0.41 → 0.0.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.41",
+  "version": "0.0.42",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/list-id-backfill.test.ts

@@ -24,6 +24,7 @@ import type { StorageService } from "@remit/storage-service";
 import {
 	backfillListIds,
 	type ListIdBackfillCheckpoint,
+	type ListIdBackfillCheckpointStore,
 	type ListIdBackfillProgress,
 } from "./list-id-backfill.js";
 
@@ -99,6 +100,22 @@ const message = (overrides: Partial<MessageItem>): MessageItem =>
 		...overrides,
 	}) as unknown as MessageItem;
 
+const asAccount = (accountConfigId: string): AccountConfigItem =>
+	({ accountConfigId }) as unknown as AccountConfigItem;
+
+const inMemoryCheckpointStore = (): ListIdBackfillCheckpointStore => {
+	let checkpoint: ListIdBackfillCheckpoint | undefined;
+	return {
+		load: async () => checkpoint,
+		save: async (next) => {
+			checkpoint = next;
+		},
+		clear: async () => {
+			checkpoint = undefined;
+		},
+	};
+};
+
 interface Harness {
 	accountConfigService: Pick<IAccountConfigRepository, "listAll">;
 	threadMessageService: Pick<
@@ -351,6 +368,37 @@ describe("backfillListIds", () => {
 		assert.equal(harness.updates.length, 2);
 	});
 
+	it("scans accounts in account-id order, whatever order listAll returns", async () => {
+		const accountIds = ["acc-1", "acc-2", "acc-3"];
+		const rows = accountIds.map((accountConfigId) =>
+			row({
+				threadMessageId: `tm-${accountConfigId}`,
+				messageId: `m-${accountConfigId}`,
+				accountConfigId,
+			}),
+		);
+		const harness = buildHarness({
+			accounts: ["acc-3", "acc-1", "acc-2"].map(asAccount),
+			rows,
+			messages: rows.map((r) =>
+				message({
+					messageId: r.messageId,
+					bodyStorageKey: `s3://${r.messageId}`,
+				}),
+			),
+		});
+		const progress: ListIdBackfillProgress[] = [];
+
+		await backfillListIds(harness, {
+			onProgress: (p) => progress.push({ ...p }),
+		});
+
+		assert.deepEqual(
+			progress.map((p) => p.accountConfigId),
+			accountIds,
+		);
+	});
+
 	it("checkpoints after each page and clears it on completion", async () => {
 		const rows = Array.from({ length: 3 }, (_, i) =>
 			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
@@ -380,6 +428,7 @@ describe("backfillListIds", () => {
 		});
 
 		assert.equal(saved.length, 2);
+		assert.equal(saved[0].accountConfigId, "acc-1");
 		assert.equal(saved[0].continuationToken, "2");
 		assert.equal(saved[1].continuationToken, undefined);
 		assert.equal(cleared, true);
@@ -400,7 +449,10 @@ describe("backfillListIds", () => {
 		const result = await backfillListIds(harness, {
 			batchSize: 2,
 			checkpointStore: {
-				load: async () => ({ accountIndex: 0, continuationToken: "2" }),
+				load: async () => ({
+					accountConfigId: "acc-1",
+					continuationToken: "2",
+				}),
 				save: async () => {},
 				clear: async () => {},
 			},
@@ -413,4 +465,109 @@ describe("backfillListIds", () => {
 			["tm-2"],
 		);
 	});
+
+	it("resumes the account it was working through after an earlier account is removed", async () => {
+		const accountIds = ["acc-1", "acc-2", "acc-3"];
+		const rows = [
+			row({
+				threadMessageId: "tm-1",
+				messageId: "m-1",
+				accountConfigId: "acc-1",
+			}),
+			row({
+				threadMessageId: "tm-2a",
+				messageId: "m-2a",
+				accountConfigId: "acc-2",
+			}),
+			row({
+				threadMessageId: "tm-2b",
+				messageId: "m-2b",
+				accountConfigId: "acc-2",
+			}),
+			row({
+				threadMessageId: "tm-3",
+				messageId: "m-3",
+				accountConfigId: "acc-3",
+			}),
+		];
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+
+		const store = inMemoryCheckpointStore();
+
+		const interrupted = buildHarness({
+			accounts: accountIds.map(asAccount),
+			rows,
+			messages,
+			pageSize: 1,
+		});
+		const listByAccount = interrupted.threadMessageService.listByAccount;
+		interrupted.threadMessageService.listByAccount = async (
+			accountConfigId,
+			opts,
+		) => {
+			if (accountConfigId === "acc-2" && opts?.continuationToken) {
+				throw new Error("interrupted");
+			}
+			return listByAccount(accountConfigId, opts);
+		};
+
+		await assert.rejects(
+			backfillListIds(interrupted, { batchSize: 1, checkpointStore: store }),
+			/interrupted/,
+		);
+
+		const resumed = buildHarness({
+			accounts: ["acc-2", "acc-3"].map(asAccount),
+			rows,
+			messages,
+			pageSize: 1,
+		});
+
+		const result = await backfillListIds(resumed, {
+			batchSize: 1,
+			checkpointStore: store,
+		});
+
+		assert.deepEqual(
+			resumed.updates.map((u) => u.threadMessageId),
+			["tm-2b", "tm-3"],
+		);
+		assert.equal(result.backfilled, 2);
+	});
+
+	it("restarts the pass when the checkpointed account no longer exists", async () => {
+		const rows = Array.from({ length: 3 }, (_, i) =>
+			row({ threadMessageId: `tm-${i}`, messageId: `m-${i}` }),
+		);
+		const messages = rows.map((r) =>
+			message({
+				messageId: r.messageId,
+				bodyStorageKey: `s3://${r.messageId}`,
+			}),
+		);
+		const harness = buildHarness({ rows, messages, pageSize: 2 });
+
+		const result = await backfillListIds(harness, {
+			batchSize: 2,
+			checkpointStore: {
+				load: async () => ({
+					accountConfigId: "acc-removed",
+					continuationToken: "2",
+				}),
+				save: async () => {},
+				clear: async () => {},
+			},
+		});
+
+		assert.equal(result.scanned, 3);
+		assert.deepEqual(
+			harness.updates.map((u) => u.threadMessageId),
+			["tm-0", "tm-1", "tm-2"],
+		);
+	});
 });

package/src/list-id-backfill.ts

@@ -11,14 +11,14 @@ import { extractListId } from "./filters/list-id.js";
 const DEFAULT_BATCH_SIZE = 200;
 
 /**
- * Where the full-corpus pass left off: the index into the account list (in
- * `listAll()` order) it was working through, and the page cursor within that
- * account. Resuming skips every account before it entirely and re-opens the
- * in-flight one at its saved cursor, rather than re-scanning the whole corpus
- * from the top after an interruption.
+ * Where the full-corpus pass left off: the identity of the account it was
+ * working through, and the page cursor within that account. Resuming looks the
+ * account up by id and re-opens it at its saved cursor, rather than re-scanning
+ * the whole corpus from the top after an interruption. A checkpoint naming no
+ * configured account restarts the pass.
  */
 export interface ListIdBackfillCheckpoint {
-	accountIndex: number;
+	accountConfigId: string;
 	continuationToken?: string;
 }
 
@@ -138,7 +138,8 @@ const deriveAndApplyListId = async (
  * sync path will populate `listId` for it once the body lands.
  *
  * Chunked by `listByAccount`'s existing keyset pagination, one account at a
- * time in `listAll()` order. A failure reading or parsing one message's
+ * time in account-id order, so an interrupted run and its resume walk the same
+ * sequence. A failure reading or parsing one message's
  * stored body is contained to that message — logged, counted, and the pass
  * continues — the same containment `BodySyncService`'s classification
  * backfill uses for the same reason: one unreadable object must not strand
@@ -151,9 +152,29 @@ export const backfillListIds = async (
 	const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
 	const { logger, checkpointStore } = options;
 
-	const accounts = await deps.accountConfigService.listAll();
+	const accounts = [...(await deps.accountConfigService.listAll())].sort(
+		(left, right) => left.accountConfigId.localeCompare(right.accountConfigId),
+	);
 	const startingCheckpoint = await checkpointStore?.load();
-	const startIndex = startingCheckpoint?.accountIndex ?? 0;
+	const checkpointedIndex = startingCheckpoint
+		? accounts.findIndex(
+				(account) =>
+					account.accountConfigId === startingCheckpoint.accountConfigId,
+			)
+		: -1;
+
+	if (startingCheckpoint && checkpointedIndex === -1) {
+		logger?.info(
+			{ accountConfigId: startingCheckpoint.accountConfigId },
+			"ListId backfill checkpoint names no configured account; restarting the pass",
+		);
+	}
+
+	const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
+	const startContinuationToken =
+		checkpointedIndex === -1
+			? undefined
+			: startingCheckpoint?.continuationToken;
 
 	const totals = emptyTotals();
 	const failedThreadMessageIds: string[] = [];
@@ -165,9 +186,7 @@ export const backfillListIds = async (
 	) {
 		const account = accounts[accountIndex];
 		let continuationToken: string | undefined =
-			accountIndex === startIndex
-				? startingCheckpoint?.continuationToken
-				: undefined;
+			accountIndex === startIndex ? startContinuationToken : undefined;
 
 		do {
 			const page = await deps.threadMessageService.listByAccount(
@@ -231,7 +250,7 @@ export const backfillListIds = async (
 
 			continuationToken = page.continuationToken;
 			await checkpointStore?.save({
-				accountIndex,
+				accountConfigId: account.accountConfigId,
 				continuationToken,
 			});