@remit/imap-worker 0.0.87 → 0.0.89

package/src/scheduler/runner.ts

@@ -4,11 +4,16 @@ import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
 import { getClient } from "@remit/backend/client";
 import { createLogger } from "@remit/logger";
 import { installNodeLogger } from "@remit/logger-lambda/node";
+import { backfillSearchText } from "@remit/mailbox-service";
 import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
-import { createQueueProducer } from "@remit/sqs-client/producer";
 
-import { env } from "expect-env";
 import { emitEvent } from "../emit.js";
+import {
+	setWorkerQueues,
+	sqsWorkerQueues,
+	workerQueues,
+	workerQueueUrlsFromEnv,
+} from "../queues.js";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
@@ -40,9 +45,7 @@ installNodeLogger();
 
 const log = createLogger();
 
-const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
-const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
-const accountFanoutQueueUrl = env.SQS_QUEUE_URL_ACCOUNT_FANOUT;
+setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
 
 const tickIntervalMs = getTickIntervalMs();
 const offlineIntervalMs = getOfflineIntervalMs();
@@ -81,16 +84,20 @@ const runLoop = async (): Promise<void> => {
 	await finishHalfDeletedAccounts({
 		account,
 		configImport: client.configImport,
-		sqsClient: createQueueProducer({ queueUrl: accountFanoutQueueUrl }),
-		accountFanoutQueueUrl,
+		queue: client.queues.accountFanout,
 		log,
 	});
-	await runSchedulerLoop({
+	const searchTextBackfill = backfillSearchText(
+		{ messageService: client.message, storageService: client.storage },
+		log,
+	).then((result) => {
+		log.debug({ ...result }, "Search text backfill finished");
+	});
+	const schedulerLoop = runSchedulerLoop({
 		tick: runSchedulerTick,
 		tickDeps: {
 			accountService: account,
-			sqsClient,
-			queueUrl: mailboxesQueueUrl,
+			queue: workerQueues().mailboxes,
 			log,
 			tickIntervalMs,
 			offlineIntervalMs,
@@ -110,6 +117,7 @@ const runLoop = async (): Promise<void> => {
 		onHeartbeatError: onBeatError,
 		tickIntervalMs,
 	});
+	await Promise.all([searchTextBackfill, schedulerLoop]);
 };
 
 runLoop()

package/src/stranded-placement.sqlite.test.ts

@@ -84,7 +84,7 @@ const placementMover = (w: World): PlacementMoveService =>
 		addressService: w.client.address,
 		mailboxSpecialUseService: w.client.mailboxSpecialUse,
 		mailboxService: w.client.mailbox,
-		sqsQueueUrl: process.env.SQS_QUEUE_URL_MESSAGE_MGMT ?? "",
+		queue: w.client.queues.messageManagement,
 	});
 
 const placementOf = async (a: Arranged) => {
@@ -131,7 +131,8 @@ const queueRefuses = (w: World): void => {
 };
 
 const queueTakes = (w: World): void => {
-	w.queue.answer(async () => ({ Failed: [] }));
+	w.queue.answer(async () => ({}));
+	w.queue.refuse([]);
 };
 
 const redriveLater = async (
@@ -255,9 +256,7 @@ describe("a move whose event dies leaves no row moving", () => {
 
 	it("hands back a move to Trash whose batch entry the queue refused", async () => {
 		const a = await arrange();
-		a.w.queue.answer(async () => ({
-			Failed: [{ Id: "0", Code: "InvalidParameterValue", SenderFault: true }],
-		}));
+		a.w.queue.refuse([0]);
 
 		await assert.rejects(
 			a.w.client.messageMove.deleteMessages(

package/src/test-helpers/test-world.ts

@@ -5,17 +5,23 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
 	_resetForTest,
-	getClient,
 	type RemitClient,
 	setClient,
 } from "@remit/backend/client";
+import { composeSqliteClient } from "@remit/backend/compose-sqlite";
+import { buildSharedDeps } from "@remit/backend/shared-deps";
 import type {
 	AccountItem,
 	MailboxItem,
 	MailboxSpecialUseValue,
 	MessageItem,
 	OutboxMessageItem,
+	QueueProducer,
 } from "@remit/data-ports";
+import {
+	type RecordingQueue,
+	recordingQueue,
+} from "@remit/data-ports/recording-queue";
 import {
 	AccountAuthType,
 	ConnectionState,
@@ -28,13 +34,10 @@ import {
 	eml,
 	FakeImapServer,
 } from "@remit/mailbox-service/test-helpers/fake-imap";
-import {
-	captureQueue,
-	type QueueCapture,
-} from "@remit/mailbox-service/test-helpers/test-queue";
 import { serializeEncryptedPayload } from "@remit/secrets-service";
 import type { createConnectionScopeWithCredentials } from "../connection-scope.js";
 import { syncMessages } from "../handlers/sync-messages.js";
+import { setWorkerQueues } from "../queues.js";
 
 export interface LogLine {
 	level: string;
@@ -93,6 +96,7 @@ export interface SeededMessage {
  */
 export const openStore = async (): Promise<{
 	client: RemitClient;
+	bindWorkerQueues: () => void;
 	world: (accountConfigId?: string) => Promise<World>;
 	close: () => void;
 }> => {
@@ -103,12 +107,38 @@ export const openStore = async (): Promise<{
 	process.env.BETTER_AUTH_URL = "https://mail.example.com";
 	process.env.BETTER_AUTH_SECRET = "a-signing-secret-of-at-least-32-characters";
 	_resetForTest();
-	const client = await getClient();
+	let current = recordingQueue();
+	const queue: QueueProducer<unknown> = {
+		send: (message) => current.send(message),
+		sendBatch: (messages) => current.sendBatch(messages),
+	};
+	const client = await composeSqliteClient(
+		buildSharedDeps({
+			sync: queue,
+			accountFanout: queue,
+			smtp: queue,
+			messageManagement: queue,
+			body: queue,
+		}),
+	);
 	setClient(client);
+	const bindWorkerQueues = (): void =>
+		setWorkerQueues({
+			mailboxes: queue,
+			messages: queue,
+			flags: queue,
+			body: queue,
+			mailboxManagement: queue,
+			messageManagement: queue,
+		});
+	bindWorkerQueues();
 	return {
 		client,
-		world: (accountConfigId = `cfg-${randomUUID()}`) =>
-			createWorld(client, accountConfigId),
+		bindWorkerQueues,
+		world: (accountConfigId = `cfg-${randomUUID()}`) => {
+			current = recordingQueue();
+			return createWorld(client, current, accountConfigId);
+		},
 		close: () => {
 			_resetForTest();
 			rmSync(directory, { recursive: true, force: true });
@@ -135,7 +165,7 @@ export interface World {
 	) => Promise<Array<{ mailboxId: string; uid?: number; isDeleted: boolean }>>;
 	queued: () => Array<Record<string, unknown>>;
 	refuseSends: (error: Error | undefined) => void;
-	queue: QueueCapture;
+	queue: RecordingQueue;
 	draft: (
 		attachment?: Buffer,
 	) => Promise<{ outbox: OutboxMessageItem; attachmentId?: string }>;
@@ -145,10 +175,9 @@ export interface World {
 
 const createWorld = async (
 	client: RemitClient,
+	queue: RecordingQueue,
 	accountConfigId: string,
 ): Promise<World> => {
-	const queue = captureQueue();
-
 	const imap = new FakeImapServer();
 	const logs: LogLine[] = [];
 	const password = await client.secrets.encrypt("app-password");

package/src/thread-join.sqlite.test.ts

@@ -6,14 +6,7 @@ import type {
 	MailboxItem,
 } from "@remit/data-ports";
 import { deriveThreadId } from "@remit/data-ports/id";
-import {
-	AddressRepo,
-	DrizzleEnvelopeRepository,
-	DrizzleMessageRepository,
-	DrizzleThreadMessageRepository,
-	SearchIndex,
-	SyncWarningRepo,
-} from "@remit/drizzle-service";
+import { createSqliteRepositories } from "@remit/drizzle-service";
 import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import { MessageSyncService, SyncWarningService } from "@remit/mailbox-service";
 import type { ImapMessage } from "@remit/mailbox-service/types";
@@ -63,18 +56,16 @@ const syncInOrder = async (
 	messages: ImapMessage[],
 ): Promise<{ threadIds: (messageId: string) => string[] }> => {
 	const { db, sqlite } = createShippedSqliteDb();
-	const typed = db as unknown as ConstructorParameters<
-		typeof DrizzleMessageRepository
-	>[0];
+	const repos = createSqliteRepositories({ app: db, mail: db, search: db });
 	const service = new MessageSyncService(
 		{} as never,
 		{} as IMailboxRepository,
 		{} as IMailboxSpecialUseRepository,
-		new DrizzleMessageRepository(typed, new SearchIndex(typed, typed)),
-		new DrizzleEnvelopeRepository(typed),
-		new AddressRepo(db as never),
-		new DrizzleThreadMessageRepository(db, new SearchIndex(db, db)),
-		new SyncWarningService(new SyncWarningRepo(db as never)),
+		repos.message,
+		repos.envelope,
+		repos.address,
+		repos.threadMessage,
+		new SyncWarningService(repos.syncWarning),
 	);
 	const save = (
 		service as unknown as {

package/src/with-oauth-lifecycle-deps.sqlite.test.ts

@@ -12,7 +12,10 @@ import assert from "node:assert/strict";
 import { after, before, describe, it } from "node:test";
 import type { AccountItem } from "@remit/data-ports";
 import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
-import { AccountRepo } from "@remit/drizzle-service";
+import {
+	type AccountRepo,
+	createSqliteRepositories,
+} from "@remit/drizzle-service";
 import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import { noopLogger } from "@remit/logger";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
@@ -67,7 +70,11 @@ const account = async (): Promise<AccountItem> =>
 before(() => {
 	const db = createShippedSqliteDb();
 	close = db.close;
-	repo = new AccountRepo(db.db, db.db);
+	repo = createSqliteRepositories({
+		app: db.db,
+		mail: db.db,
+		search: db.db,
+	}).account;
 });
 
 after(() => {

package/src/processor.test.ts

@@ -1,58 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type { Logger } from "@remit/logger";
-import type { ImapWorkerStopEvent } from "./events.js";
-import { processEvent } from "./processor.js";
-
-interface CapturedLogEntry {
-	args: unknown[];
-}
-
-const createCapturingLogger = (): {
-	log: Logger;
-	infoCalls: CapturedLogEntry[];
-} => {
-	const infoCalls: CapturedLogEntry[] = [];
-	const noop = () => {};
-	const log = {
-		info: (...args: unknown[]) => infoCalls.push({ args }),
-		warn: noop,
-		error: noop,
-		debug: noop,
-		fatal: noop,
-		trace: noop,
-		child: () => log,
-	} as unknown as Logger;
-	return { log, infoCalls };
-};
-
-describe("processEvent — IMAP_WORKER_STOP", () => {
-	it("returns undefined and logs the stop signal — tombstone fence on the account row already halts work", async () => {
-		const event: ImapWorkerStopEvent = {
-			type: "IMAP_WORKER_STOP",
-			accountConfigId: "acfg_alice_replay_safe_test_id",
-			accountId: "acct_alice_replay_safe_test_id",
-		};
-		const { log, infoCalls } = createCapturingLogger();
-
-		const result = await processEvent(event, log);
-
-		assert.equal(result, undefined);
-		const matched = infoCalls.find((c) => {
-			const [meta, msg] = c.args;
-			return (
-				typeof msg === "string" &&
-				msg === "Imap worker stop signal received" &&
-				typeof meta === "object" &&
-				meta !== null &&
-				(meta as { accountConfigId?: string }).accountConfigId ===
-					event.accountConfigId &&
-				(meta as { accountId?: string }).accountId === event.accountId
-			);
-		});
-		assert.ok(
-			matched,
-			"processor must log the cascade-contract stop signal with both ids",
-		);
-	});
-});