@remit/imap-worker 0.0.88 → 0.0.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/package.json +6 -6
  2. package/src/cli.ts +3 -2
  3. package/src/credential-less-sync-fence.test.ts +5 -9
  4. package/src/e2e-processor-shim.ts +2 -2
  5. package/src/emit.test.ts +25 -42
  6. package/src/emit.ts +94 -119
  7. package/src/events.ts +1 -19
  8. package/src/handlers/delete-account-objects.test.ts +27 -18
  9. package/src/handlers/delete-account-objects.ts +5 -12
  10. package/src/handlers/message-copy.sqlite.test.ts +3 -3
  11. package/src/handlers/message-copy.ts +5 -2
  12. package/src/handlers/message-delete.sqlite.test.ts +1 -1
  13. package/src/handlers/message-delete.ts +7 -4
  14. package/src/handlers/message-move.sqlite.test.ts +1 -1
  15. package/src/handlers/message-move.ts +9 -6
  16. package/src/handlers/placement-move-push.ts +9 -6
  17. package/src/handlers/placement-push.sqlite.test.ts +3 -2
  18. package/src/handlers/probe-message-id.sqlite.test.ts +2 -4
  19. package/src/handlers/sync-mailboxes.ts +8 -4
  20. package/src/handlers/sync-message-body.sqlite.test.ts +2 -2
  21. package/src/handlers/sync-message-body.ts +15 -18
  22. package/src/handlers/sync-messages.sqlite.test.ts +14 -2
  23. package/src/handlers/sync-messages.ts +20 -8
  24. package/src/index.ts +3 -1
  25. package/src/mail-sync-selection.sqlite.test.ts +7 -13
  26. package/src/pending-sync.sqlite.test.ts +26 -28
  27. package/src/processor.ts +10 -16
  28. package/src/queues.ts +40 -0
  29. package/src/scheduler/attachment-sweep.sqlite.test.ts +2 -2
  30. package/src/scheduler/half-deleted-accounts.sqlite.test.ts +20 -27
  31. package/src/scheduler/handler.ts +35 -28
  32. package/src/scheduler/run-tick.sqlite.test.ts +6 -14
  33. package/src/scheduler/run-tick.ts +10 -14
  34. package/src/scheduler/runner.ts +15 -12
  35. package/src/stranded-placement.sqlite.test.ts +9 -5
  36. package/src/test-helpers/test-world.ts +40 -13
  37. package/src/test-helpers/worker-queues.ts +11 -0
  38. package/src/processor.test.ts +0 -58
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/imap-worker",
3
- "version": "0.0.88",
3
+ "version": "0.0.90",
4
4
  "type": "module",
5
5
  "main": "dist/index.js",
6
6
  "exports": {
@@ -53,11 +53,11 @@
53
53
  "@remit/domain-enums": "*",
54
54
  "@remit/mail-oauth-service": "*",
55
55
  "@remit/secrets-service": "*",
56
- "@aws-sdk/client-sqs": "*",
57
- "@aws-sdk/client-ssm": "*",
58
- "expect-env": "*",
59
- "p-map": "*",
60
- "@types/aws-lambda": "*"
56
+ "@aws-sdk/client-sqs": "^3.1090.0",
57
+ "@aws-sdk/client-ssm": "^3.1090.0",
58
+ "expect-env": "^1.0.1",
59
+ "p-map": "^2.1.0",
60
+ "@types/aws-lambda": "^8.10.162"
61
61
  },
62
62
  "license": "MIT",
63
63
  "publishConfig": {
package/src/cli.ts CHANGED
@@ -6,7 +6,8 @@
6
6
  // in npm-scripts/docker-bundle.mjs). Do not route it through the logger — usage
7
7
  // text and argument errors are for the person who typed the command.
8
8
  import { parseArgs } from "node:util";
9
- import { emitEvent } from "./emit.js";
9
+ import { createEmitEvent } from "./emit.js";
10
+ import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "./queues.js";
10
11
 
11
12
  const HELP = `
12
13
  remit-imap-worker - Process IMAP sync events
@@ -206,7 +207,7 @@ const event = buildEvent();
206
207
 
207
208
  console.log(`Enqueueing ${event.type} event for account ${event.accountId}...`);
208
209
 
209
- emitEvent(event)
210
+ createEmitEvent(sqsWorkerQueues(workerQueueUrlsFromEnv()))(event)
210
211
  .then(() => {
211
212
  console.log("Event enqueued successfully");
212
213
  process.exit(0);
@@ -18,7 +18,7 @@ import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
18
18
  import { join } from "node:path";
19
19
  import { after, describe, it } from "node:test";
20
20
  import { fileURLToPath } from "node:url";
21
- import { buildSyncMailboxesCommand } from "@remit/backend/trigger-sync";
21
+ import { buildSyncMailboxesMessage } from "@remit/backend/trigger-sync";
22
22
  import type { AccountItem } from "@remit/data-ports";
23
23
  import { ConnectionState } from "@remit/domain-enums";
24
24
  import {
@@ -33,7 +33,6 @@ import {
33
33
  } from "./with-oauth-lifecycle.js";
34
34
 
35
35
  const QUEUE_NAME = "remit-mailboxes.fifo";
36
- const QUEUE_URL = `http://localhost:9324/000000000000/${QUEUE_NAME}`;
37
36
  const ACCOUNT_ID = "acct_imported";
38
37
  const T0 = 1_700_000_000_000;
39
38
 
@@ -69,15 +68,12 @@ const importedAccount = {
69
68
  } as unknown as AccountItem;
70
69
 
71
70
  const enqueueSync = (store: QueueStore, now: number): void => {
72
- const command = buildSyncMailboxesCommand({
73
- queueUrl: QUEUE_URL,
74
- accountId: ACCOUNT_ID,
75
- });
71
+ const message = buildSyncMailboxesMessage({ accountId: ACCOUNT_ID });
76
72
  store.sendMessage({
77
73
  queueName: QUEUE_NAME,
78
- body: command.input.MessageBody ?? "",
79
- groupId: command.input.MessageGroupId,
80
- deduplicationId: command.input.MessageDeduplicationId,
74
+ body: JSON.stringify(message.body),
75
+ groupId: message.groupId,
76
+ deduplicationId: message.deduplicationId,
81
77
  now,
82
78
  });
83
79
  };
@@ -7,7 +7,7 @@ import {
7
7
  } from "@aws-sdk/client-sqs";
8
8
  import { createLogger } from "@remit/logger";
9
9
  import { installNodeLogger } from "@remit/logger-lambda/node";
10
- import { createQueueProducer } from "@remit/sqs-client/producer";
10
+ import { createSqsClient } from "@remit/sqs-client/producer";
11
11
  import type { Context, SQSBatchResponse, SQSEvent } from "aws-lambda";
12
12
  import { env } from "expect-env";
13
13
  import { handler } from "./index.js";
@@ -101,7 +101,7 @@ if (cluster.isPrimary) {
101
101
 
102
102
  const maxMessages = 10; // SQS API limit
103
103
 
104
- const sqs = createQueueProducer({ queueUrl });
104
+ const sqs = createSqsClient({ queueUrl });
105
105
 
106
106
  let isShuttingDown = false;
107
107
 
package/src/emit.test.ts CHANGED
@@ -1,53 +1,42 @@
1
1
  import assert from "node:assert/strict";
2
- import { afterEach, before, beforeEach, describe, it } from "node:test";
3
- import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
2
+ import { afterEach, beforeEach, describe, it } from "node:test";
4
3
  import {
5
4
  _resetForTest,
6
5
  type RemitClient,
7
6
  setClient,
8
7
  } from "@remit/backend/client";
8
+ import {
9
+ type RecordingQueue,
10
+ recordingQueue,
11
+ } from "@remit/data-ports/recording-queue";
9
12
  import { MailboxLockRepo } from "@remit/drizzle-service";
10
13
  import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
11
- import { mockClient } from "aws-sdk-client-mock";
12
- import type * as Emit from "./emit.js";
14
+ import {
15
+ createEmitEvent,
16
+ type EmitEvent,
17
+ SYNC_MESSAGES_PENDING,
18
+ } from "./emit.js";
13
19
  import type {
14
20
  FlagPushEvent,
15
21
  SyncMailboxesEvent,
16
22
  SyncMessageBodyEvent,
17
23
  SyncMessagesEvent,
18
24
  } from "./events.js";
25
+ import { workerQueuesOn } from "./test-helpers/worker-queues.js";
19
26
 
20
27
  type Emitted<T> = Omit<T, "eventId" | "timestamp">;
21
28
 
22
- // The queue urls are read once, at import. Point them at FIFO queues first —
23
- // this suite is about what emitEvent puts on a FIFO queue, which is what both
24
- // the self-host stack and the deployed stack run (deploy/vps/queues.json).
25
- const fifo = (name: string) =>
26
- `https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
27
-
28
- const sqsMock = mockClient(SQSClient);
29
-
30
- let emitEvent: typeof Emit.emitEvent;
31
- let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
32
29
  let locks: MailboxLockRepo;
33
30
  let closeStore: () => void;
31
+ let queue: RecordingQueue;
34
32
 
35
- const sentCommands = (): SendMessageCommand["input"][] =>
36
- sqsMock.commandCalls(SendMessageCommand).map((call) => call.args[0].input);
33
+ const sentCommands = () => queue.sent;
37
34
 
38
- const bodyOf = (input: SendMessageCommand["input"]): Record<string, unknown> =>
39
- JSON.parse(input.MessageBody ?? "{}") as Record<string, unknown>;
40
-
41
- before(async () => {
42
- process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
43
- process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
44
- process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
45
- ({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
46
- });
35
+ const emitEvent: EmitEvent = (event) =>
36
+ createEmitEvent(workerQueuesOn(queue))(event);
47
37
 
48
38
  beforeEach(() => {
49
- sqsMock.reset();
50
- sqsMock.on(SendMessageCommand).resolves({});
39
+ queue = recordingQueue();
51
40
  const store = createShippedSqliteDb();
52
41
  closeStore = store.close;
53
42
  locks = new MailboxLockRepo(store.db);
@@ -71,8 +60,8 @@ describe("emitEvent on a FIFO queue", () => {
71
60
 
72
61
  const [sent] = sentCommands();
73
62
  if (!sent) throw new Error("expected a send");
74
- assert.equal(sent.MessageGroupId, "acc-1");
75
- assert.equal(sent.MessageDeduplicationId, bodyOf(sent).eventId);
63
+ assert.equal(sent.groupId, "acc-1");
64
+ assert.equal(sent.deduplicationId, sent.body.eventId);
76
65
  });
77
66
 
78
67
  // Issue #37: every SYNC_MESSAGES for a mailbox carried one dedup id, so
@@ -94,16 +83,13 @@ describe("emitEvent on a FIFO queue", () => {
94
83
  "acc-1",
95
84
  "mbx-1",
96
85
  SYNC_MESSAGES_PENDING,
97
- String(bodyOf(first).eventId),
86
+ String(first.body.eventId),
98
87
  );
99
88
  await emitEvent(event);
100
89
 
101
90
  const sent = sentCommands();
102
91
  assert.equal(sent.length, 2);
103
- assert.notEqual(
104
- sent[0]?.MessageDeduplicationId,
105
- sent[1]?.MessageDeduplicationId,
106
- );
92
+ assert.notEqual(sent[0]?.deduplicationId, sent[1]?.deduplicationId);
107
93
  });
108
94
 
109
95
  it("lets a second sync of the same account's mailbox list through", async () => {
@@ -117,10 +103,7 @@ describe("emitEvent on a FIFO queue", () => {
117
103
 
118
104
  const sent = sentCommands();
119
105
  assert.equal(sent.length, 2);
120
- assert.notEqual(
121
- sent[0]?.MessageDeduplicationId,
122
- sent[1]?.MessageDeduplicationId,
123
- );
106
+ assert.notEqual(sent[0]?.deduplicationId, sent[1]?.deduplicationId);
124
107
  });
125
108
 
126
109
  // A FIFO queue without content-based deduplication rejects a send carrying
@@ -139,12 +122,12 @@ describe("emitEvent on a FIFO queue", () => {
139
122
 
140
123
  const [sent] = sentCommands();
141
124
  if (!sent) throw new Error("expected a send");
142
- assert.equal(sent.MessageDeduplicationId, bodyOf(sent).eventId);
125
+ assert.equal(sent.deduplicationId, sent.body.eventId);
143
126
  });
144
127
  });
145
128
 
146
129
  describe("emitEvent on a standard queue", () => {
147
- it("carries no FIFO parameters", async () => {
130
+ it("sends a body sync with no group or deduplication id", async () => {
148
131
  const event: Emitted<SyncMessageBodyEvent> = {
149
132
  type: "SYNC_MESSAGE_BODY",
150
133
  accountId: "acc-1",
@@ -156,7 +139,7 @@ describe("emitEvent on a standard queue", () => {
156
139
 
157
140
  const [sent] = sentCommands();
158
141
  if (!sent) throw new Error("expected a send");
159
- assert.equal(sent.MessageGroupId, undefined);
160
- assert.equal(sent.MessageDeduplicationId, undefined);
142
+ assert.equal(sent.groupId, undefined);
143
+ assert.equal(sent.deduplicationId, undefined);
161
144
  });
162
145
  });
package/src/emit.ts CHANGED
@@ -1,140 +1,115 @@
1
1
  import { randomUUID } from "node:crypto";
2
- import { SendMessageCommand } from "@aws-sdk/client-sqs";
3
2
  import { getClient } from "@remit/backend/client";
3
+ import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
4
4
  import { recordImapFailure } from "@remit/logger-lambda/metrics";
5
- import {
6
- createQueueProducer,
7
- isLocalEndpoint,
8
- } from "@remit/sqs-client/producer";
9
- import { env } from "expect-env";
10
5
  import type { ImapEvent } from "./events.js";
6
+ import type { WorkerQueues } from "./queues.js";
11
7
 
12
8
  type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
13
9
 
14
- export const SYNC_MESSAGES_PENDING = "SYNC_MESSAGES_PENDING";
15
-
16
- const SYNC_MESSAGES_PENDING_TTL_SECONDS = 300;
17
-
18
- const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
19
- const messagesQueueUrl = env.SQS_QUEUE_URL_MESSAGES;
20
- // SYNC_MESSAGE_BODY events route to the single standard body queue (#612). It is
21
- // a standard (non-FIFO) queue, so isFifoQueue() below skips the FIFO
22
- // MessageGroupId/dedup parameters automatically.
23
- const bodyQueueUrl = env.SQS_QUEUE_URL_BODY;
24
- const flagsQueueUrl = env.SQS_QUEUE_URL_FLAGS;
25
- const mailboxMgmtQueueUrl = env.SQS_QUEUE_URL_MAILBOX_MGMT;
26
- const messageMgmtQueueUrl = env.SQS_QUEUE_URL_MESSAGE_MGMT;
27
-
28
- const isLocal = isLocalEndpoint(mailboxesQueueUrl);
10
+ export type EmitEvent = (event: EventInput) => Promise<void>;
29
11
 
30
- const sqs = createQueueProducer({ queueUrl: mailboxesQueueUrl });
12
+ type Send = (event: ImapEvent) => Promise<void>;
31
13
 
32
- const queueUrlMap: Record<ImapEvent["type"], string> = {
33
- SYNC_MAILBOXES: mailboxesQueueUrl,
34
- SYNC_MESSAGES: messagesQueueUrl,
35
- SYNC_MESSAGE_BODY: bodyQueueUrl,
36
- MAILBOX_CREATE: mailboxMgmtQueueUrl,
37
- MAILBOX_RENAME: mailboxMgmtQueueUrl,
38
- MAILBOX_DELETE: mailboxMgmtQueueUrl,
39
- MESSAGE_DELETE: messageMgmtQueueUrl,
40
- MESSAGE_MOVE: messageMgmtQueueUrl,
41
- MESSAGE_COPY: messageMgmtQueueUrl,
42
- EMPTY_TRASH: messageMgmtQueueUrl,
43
- APPEND_SENT_MESSAGE: messageMgmtQueueUrl,
44
- // Rides messageMgmtQueue (issue #1271) rather than a dedicated queue — its
45
- // payload carries only our message id (never a UID, unlike the legacy
46
- // MESSAGE_MOVE event); per-event-type payload shape, same queue.
47
- PLACEMENT_MOVE_PUSH: messageMgmtQueueUrl,
48
- // Rides the existing flags queue (issue #1273) — this is imap-worker's OWN
49
- // re-arm hint (the periodic per-mailbox sync tick catching up a marker
50
- // stuck `pending`), distinct from `FlagPushService`'s user-facing hint
51
- // (remit-mailbox-service, sent from the API on its own SQS client onto
52
- // SQS_QUEUE_URL). Both land on a queue this worker already consumes;
53
- // dispatch is by `type`, not by which queue delivered the message.
54
- FLAG_PUSH: flagsQueueUrl,
55
- };
56
-
57
- /**
58
- * Check if queue URL is a FIFO queue (ends with .fifo)
59
- */
60
- const isFifoQueue = (queueUrl: string): boolean => queueUrl.endsWith(".fifo");
61
-
62
- export interface EmitEventOptions {
63
- /**
64
- * Delay delivery of the message by this many seconds (0-900).
65
- * Useful for retry backoff to avoid overwhelming IMAP servers.
66
- */
67
- delaySeconds?: number;
68
- }
14
+ // The deduplication id is the event's own id, so the queue suppresses a
15
+ // re-send of one event (the retry SQS's own idempotency guard is for) and
16
+ // nothing else. A shared id per account or per mailbox instead made the
17
+ // 5-minute window a rate limiter: the second sync of a mailbox within five
18
+ // minutes was discarded before any worker saw it, so mail that arrived after
19
+ // a sync could not be fetched until the window elapsed (issue #37).
20
+ const fifo =
21
+ (queue: FifoQueueProducer<ImapEvent>): Send =>
22
+ (event) =>
23
+ queue.send({
24
+ body: event,
25
+ groupId: event.accountId,
26
+ deduplicationId: event.eventId,
27
+ });
69
28
 
70
- export const emitEvent = async (
71
- event: EventInput,
72
- options?: EmitEventOptions,
73
- ) => {
74
- const fullEvent: ImapEvent = {
75
- ...event,
76
- eventId: randomUUID(),
77
- timestamp: Date.now(),
78
- } as ImapEvent;
29
+ const standard =
30
+ (queue: QueueProducer<ImapEvent>): Send =>
31
+ (event) =>
32
+ queue.send({ body: event });
79
33
 
80
- const queueUrl = queueUrlMap[event.type];
81
- const useFifo = isFifoQueue(queueUrl);
34
+ export const SYNC_MESSAGES_PENDING = "SYNC_MESSAGES_PENDING";
82
35
 
83
- // ElasticMQ FIFO queues don't support per-message DelaySeconds
84
- const useDelay = options?.delaySeconds && !isLocal;
36
+ const SYNC_MESSAGES_PENDING_TTL_SECONDS = 300;
85
37
 
86
- const send = () =>
87
- sqs.send(
88
- new SendMessageCommand({
89
- QueueUrl: queueUrl,
90
- MessageBody: JSON.stringify(fullEvent),
91
- // Delay delivery for retry backoff (skip for local ElasticMQ)
92
- ...(useDelay && { DelaySeconds: options.delaySeconds }),
93
- // FIFO queue parameters — only set if the queue is FIFO. The
94
- // deduplication id is the event's own id, so the queue suppresses a
95
- // re-send of one event (the retry SQS's own idempotency guard is for)
96
- // and nothing else. A shared id per account or per mailbox instead made
97
- // the 5-minute window a rate limiter: the second sync of a mailbox
98
- // within five minutes was discarded before any worker saw it, so mail
99
- // that arrived after a sync could not be fetched until the window
100
- // elapsed (issue #37).
101
- ...(useFifo && {
102
- MessageGroupId: event.accountId,
103
- MessageDeduplicationId: fullEvent.eventId,
104
- }),
105
- }),
106
- );
107
- if (fullEvent.type !== "SYNC_MESSAGES") {
108
- await send();
109
- return;
38
+ const senderFor = (type: ImapEvent["type"], queues: WorkerQueues): Send => {
39
+ switch (type) {
40
+ case "SYNC_MAILBOXES":
41
+ return fifo(queues.mailboxes);
42
+ case "SYNC_MESSAGES":
43
+ return fifo(queues.messages);
44
+ // SYNC_MESSAGE_BODY routes to the single standard body queue (#612).
45
+ case "SYNC_MESSAGE_BODY":
46
+ return standard(queues.body);
47
+ case "MAILBOX_CREATE":
48
+ case "MAILBOX_RENAME":
49
+ case "MAILBOX_DELETE":
50
+ return standard(queues.mailboxManagement);
51
+ case "MESSAGE_DELETE":
52
+ case "MESSAGE_MOVE":
53
+ case "MESSAGE_COPY":
54
+ case "EMPTY_TRASH":
55
+ case "APPEND_SENT_MESSAGE":
56
+ // Rides messageMgmtQueue (issue #1271) rather than a dedicated queue — its
57
+ // payload carries only our message id (never a UID, unlike the legacy
58
+ // MESSAGE_MOVE event); per-event-type payload shape, same queue.
59
+ case "PLACEMENT_MOVE_PUSH":
60
+ return standard(queues.messageManagement);
61
+ // Rides the existing flags queue (issue #1273) — this is imap-worker's OWN
62
+ // re-arm hint (the periodic per-mailbox sync tick catching up a marker
63
+ // stuck `pending`), distinct from `FlagPushService`'s user-facing hint
64
+ // sent from the API onto the sync queue. Both land on a queue this worker
65
+ // already consumes; dispatch is by `type`, not by which queue delivered
66
+ // the message.
67
+ case "FLAG_PUSH":
68
+ return fifo(queues.flags);
110
69
  }
70
+ };
111
71
 
112
- const { mailboxLock } = await getClient();
113
- const { accountId, mailboxId, eventId } = fullEvent;
114
- const previous = await mailboxLock.get(
115
- accountId,
116
- mailboxId,
117
- SYNC_MESSAGES_PENDING,
118
- );
119
- const acquired = await mailboxLock.tryAcquireLock(
120
- mailboxId,
121
- SYNC_MESSAGES_PENDING,
122
- accountId,
123
- eventId,
124
- SYNC_MESSAGES_PENDING_TTL_SECONDS,
125
- );
126
- if (!acquired) return;
127
- if (previous && previous.ttl < Math.floor(Date.now() / 1000)) {
128
- recordImapFailure("SYNC_PENDING_EXPIRED", "other");
129
- }
72
+ export const createEmitEvent =
73
+ (queues: WorkerQueues): EmitEvent =>
74
+ async (event) => {
75
+ const fullEvent: ImapEvent = {
76
+ ...event,
77
+ eventId: randomUUID(),
78
+ timestamp: Date.now(),
79
+ } as ImapEvent;
130
80
 
131
- await send().catch(async (error: unknown) => {
132
- await mailboxLock.releaseLock(
81
+ const send = () => senderFor(fullEvent.type, queues)(fullEvent);
82
+ if (fullEvent.type !== "SYNC_MESSAGES") {
83
+ await send();
84
+ return;
85
+ }
86
+
87
+ const { mailboxLock } = await getClient();
88
+ const { accountId, mailboxId, eventId } = fullEvent;
89
+ const previous = await mailboxLock.get(
133
90
  accountId,
134
91
  mailboxId,
135
92
  SYNC_MESSAGES_PENDING,
93
+ );
94
+ const acquired = await mailboxLock.tryAcquireLock(
95
+ mailboxId,
96
+ SYNC_MESSAGES_PENDING,
97
+ accountId,
136
98
  eventId,
99
+ SYNC_MESSAGES_PENDING_TTL_SECONDS,
137
100
  );
138
- throw error;
139
- });
140
- };
101
+ if (!acquired) return;
102
+ if (previous && previous.ttl < Math.floor(Date.now() / 1000)) {
103
+ recordImapFailure("SYNC_PENDING_EXPIRED", "other");
104
+ }
105
+
106
+ await send().catch(async (error: unknown) => {
107
+ await mailboxLock.releaseLock(
108
+ accountId,
109
+ mailboxId,
110
+ SYNC_MESSAGES_PENDING,
111
+ eventId,
112
+ );
113
+ throw error;
114
+ });
115
+ };
package/src/events.ts CHANGED
@@ -232,26 +232,8 @@ export interface DeleteAccountObjectsEvent {
232
232
  continuationToken?: string;
233
233
  }
234
234
 
235
- /**
236
- * Sent by the account-deletion fanout worker once per accountId under a
237
- * deleted AccountConfig. The actual stop semantics already happen via the
238
- * account-tombstone fence (`isActive=false` + `deletedAt`) flipped by the
239
- * deletion API: any in-flight or future event for the account is dropped
240
- * by `isAccountDeleted`. This event exists as an explicit signal in the
241
- * cascade contract — when the imap-worker grows real per-account drain or
242
- * connection-teardown semantics it hangs off this hook.
243
- */
244
- export interface ImapWorkerStopEvent {
245
- type: "IMAP_WORKER_STOP";
246
- accountConfigId: string;
247
- accountId: string;
248
- }
249
-
250
235
  /** Union of all event types the worker can process (including non-IMAP ones). */
251
- export type WorkerEvent =
252
- | ImapEvent
253
- | DeleteAccountObjectsEvent
254
- | ImapWorkerStopEvent;
236
+ export type WorkerEvent = ImapEvent | DeleteAccountObjectsEvent;
255
237
 
256
238
  export type MessageManagementEvent =
257
239
  | MessageDeleteEvent
@@ -1,18 +1,22 @@
1
1
  import assert from "node:assert/strict";
2
- import { afterEach, describe, it } from "node:test";
2
+ import { afterEach, beforeEach, describe, it } from "node:test";
3
3
  import {
4
4
  DeleteObjectsCommand,
5
5
  ListObjectsV2Command,
6
6
  S3Client,
7
7
  } from "@aws-sdk/client-s3";
8
- import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
8
+ import {
9
+ type RecordingQueue,
10
+ recordingQueue,
11
+ } from "@remit/data-ports/recording-queue";
9
12
  import { noopLogger } from "@remit/logger";
10
13
  import { mockClient } from "aws-sdk-client-mock";
11
14
  import type { DeleteAccountObjectsEvent } from "../events.js";
15
+ import { workerQueuesOn } from "../test-helpers/worker-queues.js";
12
16
  import { handleDeleteAccountObjects } from "./delete-account-objects.js";
13
17
 
14
18
  const s3Mock = mockClient(S3Client);
15
- const sqsMock = mockClient(SQSClient);
19
+ let queue: RecordingQueue;
16
20
 
17
21
  const event: DeleteAccountObjectsEvent = {
18
22
  type: "DELETE_ACCOUNT_OBJECTS",
@@ -20,9 +24,12 @@ const event: DeleteAccountObjectsEvent = {
20
24
  };
21
25
 
22
26
  describe("handleDeleteAccountObjects", () => {
27
+ beforeEach(() => {
28
+ queue = recordingQueue();
29
+ });
30
+
23
31
  afterEach(() => {
24
32
  s3Mock.reset();
25
- sqsMock.reset();
26
33
  });
27
34
 
28
35
  it("lists and deletes every object under the account prefix in one page", async () => {
@@ -35,7 +42,7 @@ describe("handleDeleteAccountObjects", () => {
35
42
  });
36
43
  s3Mock.on(DeleteObjectsCommand).resolves({});
37
44
 
38
- await handleDeleteAccountObjects(event, noopLogger);
45
+ await handleDeleteAccountObjects(event, noopLogger, workerQueuesOn(queue));
39
46
 
40
47
  const listCall = s3Mock.commandCalls(ListObjectsV2Command)[0];
41
48
  assert.equal(listCall.args[0].input.Prefix, "accounts/cfg-1/");
@@ -45,7 +52,7 @@ describe("handleDeleteAccountObjects", () => {
45
52
  { Key: "accounts/cfg-1/a.eml" },
46
53
  { Key: "accounts/cfg-1/b.eml" },
47
54
  ]);
48
- assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
55
+ assert.equal(queue.sent.length, 0);
49
56
  });
50
57
 
51
58
  it("follows the continuation token across truncated pages", async () => {
@@ -62,7 +69,7 @@ describe("handleDeleteAccountObjects", () => {
62
69
  });
63
70
  s3Mock.on(DeleteObjectsCommand).resolves({});
64
71
 
65
- await handleDeleteAccountObjects(event, noopLogger);
72
+ await handleDeleteAccountObjects(event, noopLogger, workerQueuesOn(queue));
66
73
 
67
74
  const listCalls = s3Mock.commandCalls(ListObjectsV2Command);
68
75
  assert.equal(listCalls.length, 2);
@@ -73,18 +80,18 @@ describe("handleDeleteAccountObjects", () => {
73
80
  it("skips the delete call when a page holds no keys", async () => {
74
81
  s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
75
82
 
76
- await handleDeleteAccountObjects(event, noopLogger);
83
+ await handleDeleteAccountObjects(event, noopLogger, workerQueuesOn(queue));
77
84
 
78
85
  assert.equal(s3Mock.commandCalls(DeleteObjectsCommand).length, 0);
79
86
  });
80
87
 
81
88
  it("re-enqueues with the continuation token when time is nearly up", async () => {
82
89
  s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
83
- sqsMock.on(SendMessageCommand).resolves({});
84
90
 
85
91
  await handleDeleteAccountObjects(
86
92
  { ...event, continuationToken: "tok-mid" },
87
93
  noopLogger,
94
+ workerQueuesOn(queue),
88
95
  () => 5_000,
89
96
  );
90
97
 
@@ -93,13 +100,10 @@ describe("handleDeleteAccountObjects", () => {
93
100
  0,
94
101
  "bails before starting a new page",
95
102
  );
96
- const send = sqsMock.commandCalls(SendMessageCommand)[0];
97
- const body = JSON.parse(
98
- send?.args[0].input.MessageBody ?? "{}",
99
- ) as DeleteAccountObjectsEvent;
100
- assert.equal(body.type, "DELETE_ACCOUNT_OBJECTS");
101
- assert.equal(body.accountConfigId, "cfg-1");
102
- assert.equal(body.continuationToken, "tok-mid");
103
+ const body = queue.sent[0]?.body;
104
+ assert.equal(body?.type, "DELETE_ACCOUNT_OBJECTS");
105
+ assert.equal(body?.accountConfigId, "cfg-1");
106
+ assert.equal(body?.continuationToken, "tok-mid");
103
107
  });
104
108
 
105
109
  it("keeps paging while time remains", async () => {
@@ -109,9 +113,14 @@ describe("handleDeleteAccountObjects", () => {
109
113
  });
110
114
  s3Mock.on(DeleteObjectsCommand).resolves({});
111
115
 
112
- await handleDeleteAccountObjects(event, noopLogger, () => 120_000);
116
+ await handleDeleteAccountObjects(
117
+ event,
118
+ noopLogger,
119
+ workerQueuesOn(queue),
120
+ () => 120_000,
121
+ );
113
122
 
114
123
  assert.equal(s3Mock.commandCalls(ListObjectsV2Command).length, 1);
115
- assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
124
+ assert.equal(queue.sent.length, 0);
116
125
  });
117
126
  });