@remit/imap-worker 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -0
- package/build.mjs +17 -0
- package/package.json +50 -0
- package/src/account-check.test.ts +122 -0
- package/src/account-check.ts +88 -0
- package/src/body-sync-gate.test.ts +185 -0
- package/src/body-sync-gate.ts +86 -0
- package/src/cli.ts +211 -0
- package/src/connection-scope.test.ts +266 -0
- package/src/connection-scope.ts +335 -0
- package/src/e2e-processor-shim.ts +248 -0
- package/src/emit.test.ts +44 -0
- package/src/emit.ts +142 -0
- package/src/events.ts +221 -0
- package/src/handlers/append-sent-message.ts +163 -0
- package/src/handlers/delete-account-objects.test.ts +81 -0
- package/src/handlers/delete-account-objects.ts +116 -0
- package/src/handlers/empty-trash.ts +136 -0
- package/src/handlers/flag-push.test.ts +25 -0
- package/src/handlers/flag-push.ts +224 -0
- package/src/handlers/mailbox-management.ts +266 -0
- package/src/handlers/mailbox-sync-order.test.ts +93 -0
- package/src/handlers/mailbox-sync-order.ts +65 -0
- package/src/handlers/message-copy.ts +219 -0
- package/src/handlers/message-delete.test.ts +176 -0
- package/src/handlers/message-delete.ts +283 -0
- package/src/handlers/message-move.test.ts +168 -0
- package/src/handlers/message-move.ts +298 -0
- package/src/handlers/placement-move-push.test.ts +234 -0
- package/src/handlers/placement-move-push.ts +434 -0
- package/src/handlers/sync-mailboxes.ts +241 -0
- package/src/handlers/sync-message-body.test.ts +375 -0
- package/src/handlers/sync-message-body.ts +337 -0
- package/src/handlers/sync-messages-deleted-account.test.ts +141 -0
- package/src/handlers/sync-messages.test.ts +204 -0
- package/src/handlers/sync-messages.ts +412 -0
- package/src/handlers/sync-reserved-host.test.ts +97 -0
- package/src/index.test.ts +22 -0
- package/src/index.ts +70 -0
- package/src/poller.ts +49 -0
- package/src/processor.test.ts +58 -0
- package/src/processor.ts +66 -0
- package/src/scheduler/config.test.ts +40 -0
- package/src/scheduler/config.ts +52 -0
- package/src/scheduler/decide-due.test.ts +44 -0
- package/src/scheduler/decide-due.ts +26 -0
- package/src/scheduler/handler.ts +52 -0
- package/src/scheduler/local-runner.ts +76 -0
- package/src/scheduler/run-tick.test.ts +248 -0
- package/src/scheduler/run-tick.ts +141 -0
- package/src/with-oauth-lifecycle-deps.ts +62 -0
- package/src/with-oauth-lifecycle.test.ts +227 -0
- package/src/with-oauth-lifecycle.ts +125 -0
- package/tsconfig.json +8 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import { afterEach, describe, mock, test } from "node:test";
|
|
3
|
+
import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
4
|
+
import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
|
|
5
|
+
import { getClient } from "@remit/backend/client";
|
|
6
|
+
import type { AccountItem, MailboxItem } from "@remit/data-ports";
|
|
7
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
8
|
+
import { BodySyncService, type SyncedMessage } from "@remit/mailbox-service";
|
|
9
|
+
import { mockClient } from "aws-sdk-client-mock";
|
|
10
|
+
import { resetBodySyncGateCache } from "../body-sync-gate.js";
|
|
11
|
+
import { __warmPoolSizeForTest } from "../connection-scope.js";
|
|
12
|
+
import type { SyncMessageBodyEvent } from "../events.js";
|
|
13
|
+
import {
|
|
14
|
+
BODY_SYNC_MAX_ATTEMPTS,
|
|
15
|
+
buildRetryableFailureError,
|
|
16
|
+
getBodySyncMaxAttempts,
|
|
17
|
+
resolveBatch,
|
|
18
|
+
syncMessageBody,
|
|
19
|
+
} from "./sync-message-body.js";
|
|
20
|
+
import { BODY_BATCH_SIZE, batchSyncedMessages } from "./sync-messages.js";
|
|
21
|
+
|
|
22
|
+
const silentLogger = (() => {
|
|
23
|
+
const noop = () => {};
|
|
24
|
+
const log = {
|
|
25
|
+
info: noop,
|
|
26
|
+
warn: noop,
|
|
27
|
+
error: noop,
|
|
28
|
+
debug: noop,
|
|
29
|
+
fatal: noop,
|
|
30
|
+
trace: noop,
|
|
31
|
+
child: () => log,
|
|
32
|
+
} as unknown as Logger;
|
|
33
|
+
return log;
|
|
34
|
+
})();
|
|
35
|
+
|
|
36
|
+
const baseEvent = {
|
|
37
|
+
type: "SYNC_MESSAGE_BODY" as const,
|
|
38
|
+
accountId: "test-account-123",
|
|
39
|
+
mailboxId: "test-mailbox-456",
|
|
40
|
+
eventId: "event-789",
|
|
41
|
+
timestamp: 1700000000000,
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
describe("resolveBatch — event-shape preference", () => {
|
|
45
|
+
test("prefers the new messages[] shape and exposes the uid map", () => {
|
|
46
|
+
const event: SyncMessageBodyEvent = {
|
|
47
|
+
...baseEvent,
|
|
48
|
+
messageIds: ["msg-1", "msg-2"],
|
|
49
|
+
messages: [
|
|
50
|
+
{ messageId: "msg-1", uid: 101 },
|
|
51
|
+
{ messageId: "msg-2", uid: 102 },
|
|
52
|
+
],
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const { messageIds, uidByMessageId } = resolveBatch(event);
|
|
56
|
+
|
|
57
|
+
assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
|
|
58
|
+
assert.ok(uidByMessageId);
|
|
59
|
+
assert.equal(uidByMessageId.get("msg-1"), 101);
|
|
60
|
+
assert.equal(uidByMessageId.get("msg-2"), 102);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("derives messageIds from messages[] when both disagree", () => {
|
|
64
|
+
// messages[] is authoritative; a stale messageIds list must not leak in.
|
|
65
|
+
const event: SyncMessageBodyEvent = {
|
|
66
|
+
...baseEvent,
|
|
67
|
+
messageIds: ["stale"],
|
|
68
|
+
messages: [{ messageId: "msg-1", uid: 101 }],
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const { messageIds } = resolveBatch(event);
|
|
72
|
+
|
|
73
|
+
assert.deepEqual(messageIds, ["msg-1"]);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("falls back to legacy messageIds[] with no uid map", () => {
|
|
77
|
+
const event: SyncMessageBodyEvent = {
|
|
78
|
+
...baseEvent,
|
|
79
|
+
messageIds: ["msg-1", "msg-2", "msg-3"],
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const { messageIds, uidByMessageId } = resolveBatch(event);
|
|
83
|
+
|
|
84
|
+
assert.deepEqual(messageIds, ["msg-1", "msg-2", "msg-3"]);
|
|
85
|
+
assert.equal(uidByMessageId, undefined);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("force defaults to false when the event omits it (legacy/bulk events)", () => {
|
|
89
|
+
const event: SyncMessageBodyEvent = {
|
|
90
|
+
...baseEvent,
|
|
91
|
+
messageIds: ["msg-1"],
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
assert.equal(resolveBatch(event).force, false);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("force is carried from the event's read-miss re-arm cue", () => {
|
|
98
|
+
const event: SyncMessageBodyEvent = {
|
|
99
|
+
...baseEvent,
|
|
100
|
+
messageIds: ["msg-1"],
|
|
101
|
+
messages: [{ messageId: "msg-1", uid: 101 }],
|
|
102
|
+
force: true,
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
assert.equal(resolveBatch(event).force, true);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("empty messages[] resolves to an empty batch, not the legacy list", () => {
|
|
109
|
+
const event: SyncMessageBodyEvent = {
|
|
110
|
+
...baseEvent,
|
|
111
|
+
messageIds: ["should-be-ignored"],
|
|
112
|
+
messages: [],
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const { messageIds, uidByMessageId } = resolveBatch(event);
|
|
116
|
+
|
|
117
|
+
assert.deepEqual(messageIds, []);
|
|
118
|
+
assert.ok(uidByMessageId);
|
|
119
|
+
assert.equal(uidByMessageId.size, 0);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
describe("buildRetryableFailureError — the DLQ-propagation signal", () => {
|
|
124
|
+
// Genuine processing failures must propagate (issue #1270): syncMessageBody
|
|
125
|
+
// throws this while SQS redelivery budget remains, instead of swallowing the
|
|
126
|
+
// failure into a fresh re-enqueue. index.ts's SQS handler catches it and
|
|
127
|
+
// reports the record as a batch item failure, so SQS redelivers it — and
|
|
128
|
+
// once the queue's own maxReceiveCount is hit, the record dead-letters into
|
|
129
|
+
// the body-dlq (alarmed in infra/stacks/dev/stacks/remit-worker-monitoring-stack.ts).
|
|
130
|
+
|
|
131
|
+
test("names every failed message id and the current attempt", () => {
|
|
132
|
+
const error = buildRetryableFailureError(["msg-1", "msg-2"], 1);
|
|
133
|
+
|
|
134
|
+
assert.match(error.message, /msg-1/);
|
|
135
|
+
assert.match(error.message, /msg-2/);
|
|
136
|
+
assert.match(error.message, /attempt 1\/3/);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("is a real Error instance so it propagates like any other failure", () => {
|
|
140
|
+
const error = buildRetryableFailureError(["msg-1"], 2);
|
|
141
|
+
assert.ok(error instanceof Error);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test("BODY_SYNC_MAX_ATTEMPTS matches the body queue's maxReceiveCount (3)", () => {
|
|
145
|
+
// See MAX_RECEIVE_COUNT in infra/stacks/dev/stacks/remit-queue-stack.ts.
|
|
146
|
+
assert.equal(BODY_SYNC_MAX_ATTEMPTS, 3);
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
describe("getBodySyncMaxAttempts — env-derived, CDK-injected threshold (#1270)", () => {
|
|
151
|
+
test("parses the CDK-injected env var (derived from the queue's MAX_RECEIVE_COUNT)", () => {
|
|
152
|
+
assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "3" }), 3);
|
|
153
|
+
assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "5" }), 5);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("defaults to 3 when unset (local dev, unit tests) — matches the queue's own default", () => {
|
|
157
|
+
assert.equal(getBodySyncMaxAttempts({}), 3);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("defaults to 3 on a non-numeric or non-positive value", () => {
|
|
161
|
+
assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "nope" }), 3);
|
|
162
|
+
assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "0" }), 3);
|
|
163
|
+
assert.equal(getBodySyncMaxAttempts({ BODY_SYNC_MAX_ATTEMPTS: "-1" }), 3);
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
describe("syncMessageBody — pause gate runs before connection reuse", () => {
|
|
168
|
+
afterEach(() => {
|
|
169
|
+
mockClient(SSMClient).reset();
|
|
170
|
+
resetBodySyncGateCache();
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("paused: acks-and-skips before borrowing a warm connection", async () => {
|
|
174
|
+
const accountId = "paused-account-zzz";
|
|
175
|
+
mockClient(SSMClient)
|
|
176
|
+
.on(GetParameterCommand)
|
|
177
|
+
.resolves({ Parameter: { Value: "false" } });
|
|
178
|
+
|
|
179
|
+
const event: SyncMessageBodyEvent = {
|
|
180
|
+
...baseEvent,
|
|
181
|
+
accountId,
|
|
182
|
+
messageIds: ["msg-1"],
|
|
183
|
+
messages: [{ messageId: "msg-1", uid: 101 }],
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
await syncMessageBody(event, silentLogger);
|
|
187
|
+
|
|
188
|
+
// Returning at the gate must not touch the warm pool (no account lookup,
|
|
189
|
+
// no IMAP connection) — proves the gate is first.
|
|
190
|
+
assert.strictEqual(
|
|
191
|
+
__warmPoolSizeForTest(accountId),
|
|
192
|
+
0,
|
|
193
|
+
"paused handler must not create a warm connection",
|
|
194
|
+
);
|
|
195
|
+
});
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
describe("syncMessageBody — DLQ propagation (integrated, #1270)", () => {
|
|
199
|
+
const accountId = "cap-account-zzz";
|
|
200
|
+
const mailboxId = "cap-mailbox-zzz";
|
|
201
|
+
|
|
202
|
+
const cappedAccount = (): AccountItem =>
|
|
203
|
+
({
|
|
204
|
+
accountId,
|
|
205
|
+
accountConfigId: "cap-acfg-zzz",
|
|
206
|
+
connectionState: "authenticated",
|
|
207
|
+
username: "cap@imap.example.com",
|
|
208
|
+
imapHost: "imap.example.com",
|
|
209
|
+
imapPort: 993,
|
|
210
|
+
imapTls: true,
|
|
211
|
+
// deserializeEncryptedPayload just needs base64-decodable strings; the
|
|
212
|
+
// mocked secrets.decrypt below never inspects the actual bytes.
|
|
213
|
+
passwordHash: JSON.stringify({
|
|
214
|
+
encryptedDek: "",
|
|
215
|
+
encryptedData: "",
|
|
216
|
+
iv: "",
|
|
217
|
+
authTag: "",
|
|
218
|
+
}),
|
|
219
|
+
}) as unknown as AccountItem;
|
|
220
|
+
|
|
221
|
+
const cappedMailbox = (): MailboxItem =>
|
|
222
|
+
({ fullPath: "INBOX" }) as unknown as MailboxItem;
|
|
223
|
+
|
|
224
|
+
afterEach(() => {
|
|
225
|
+
mock.restoreAll();
|
|
226
|
+
mockClient(SSMClient).reset();
|
|
227
|
+
mockClient(SQSClient).reset();
|
|
228
|
+
resetBodySyncGateCache();
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
test("below BODY_SYNC_MAX_ATTEMPTS: throws instead of re-enqueueing, so SQS redelivers the record (issue #1270)", async () => {
|
|
232
|
+
// This is the mechanism that lets a genuine processing failure ever reach
|
|
233
|
+
// the body-dlq: the handler used to swallow every failure into a fresh
|
|
234
|
+
// SQS SendMessage and always return successfully, so the queue's own
|
|
235
|
+
// maxReceiveCount/DLQ never engaged. Throwing here is what index.ts's SQS
|
|
236
|
+
// handler turns into a batchItemFailure, which SQS then redelivers.
|
|
237
|
+
mockClient(SSMClient)
|
|
238
|
+
.on(GetParameterCommand)
|
|
239
|
+
.resolves({ Parameter: { Value: "true" } });
|
|
240
|
+
const sqsMock = mockClient(SQSClient);
|
|
241
|
+
|
|
242
|
+
mock.method((await getClient()).account, "get", async () =>
|
|
243
|
+
cappedAccount(),
|
|
244
|
+
);
|
|
245
|
+
mock.method((await getClient()).mailbox, "get", async () =>
|
|
246
|
+
cappedMailbox(),
|
|
247
|
+
);
|
|
248
|
+
mock.method(
|
|
249
|
+
(await getClient()).secrets,
|
|
250
|
+
"decrypt",
|
|
251
|
+
async () => "fake-password",
|
|
252
|
+
);
|
|
253
|
+
mock.method(BodySyncService.prototype, "syncBodies", async () => ({
|
|
254
|
+
syncedCount: 0,
|
|
255
|
+
syncedMessageIds: [],
|
|
256
|
+
skippedCount: 0,
|
|
257
|
+
failedCount: 1,
|
|
258
|
+
failedMessageIds: ["msg-1"],
|
|
259
|
+
}));
|
|
260
|
+
|
|
261
|
+
const event: SyncMessageBodyEvent = {
|
|
262
|
+
...baseEvent,
|
|
263
|
+
accountId,
|
|
264
|
+
mailboxId,
|
|
265
|
+
messageIds: ["msg-1"],
|
|
266
|
+
messages: [{ messageId: "msg-1", uid: 101 }],
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
await assert.rejects(
|
|
270
|
+
() => syncMessageBody(event, silentLogger, 1),
|
|
271
|
+
(err: unknown) =>
|
|
272
|
+
err instanceof Error &&
|
|
273
|
+
/Body sync failed for 1 message/.test(err.message),
|
|
274
|
+
);
|
|
275
|
+
|
|
276
|
+
assert.equal(
|
|
277
|
+
sqsMock.commandCalls(SendMessageCommand).length,
|
|
278
|
+
0,
|
|
279
|
+
"no manual re-enqueue — SQS's own redelivery owns the retry now",
|
|
280
|
+
);
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
test("cursor_invalid: acks-and-skips without ever calling BodySyncService.syncBodies (#1272)", async () => {
|
|
284
|
+
// The cheap pre-check (isCursorRebuildNeeded, run before borrowing any
|
|
285
|
+
// connection — frugal, epic #1281 invariant 6) catches an already-paused
|
|
286
|
+
// mailbox before BodySyncService is ever invoked. The structural backstop
|
|
287
|
+
// for a cursor that trips *during* this call — guardConnectionCursor's
|
|
288
|
+
// openBox override — is covered directly in mailbox-cursor.test.ts.
|
|
289
|
+
mockClient(SSMClient)
|
|
290
|
+
.on(GetParameterCommand)
|
|
291
|
+
.resolves({ Parameter: { Value: "true" } });
|
|
292
|
+
|
|
293
|
+
mock.method((await getClient()).account, "get", async () =>
|
|
294
|
+
cappedAccount(),
|
|
295
|
+
);
|
|
296
|
+
mock.method((await getClient()).mailbox, "get", async () => ({
|
|
297
|
+
...cappedMailbox(),
|
|
298
|
+
mailboxId,
|
|
299
|
+
cursorState: "cursor_invalid",
|
|
300
|
+
}));
|
|
301
|
+
mock.method(
|
|
302
|
+
(await getClient()).secrets,
|
|
303
|
+
"decrypt",
|
|
304
|
+
async () => "fake-password",
|
|
305
|
+
);
|
|
306
|
+
const syncBodies = mock.method(
|
|
307
|
+
BodySyncService.prototype,
|
|
308
|
+
"syncBodies",
|
|
309
|
+
async () => {
|
|
310
|
+
throw new Error(
|
|
311
|
+
"must not be called while the mailbox cursor is paused",
|
|
312
|
+
);
|
|
313
|
+
},
|
|
314
|
+
);
|
|
315
|
+
|
|
316
|
+
const event: SyncMessageBodyEvent = {
|
|
317
|
+
...baseEvent,
|
|
318
|
+
accountId,
|
|
319
|
+
mailboxId,
|
|
320
|
+
messageIds: ["msg-1"],
|
|
321
|
+
messages: [{ messageId: "msg-1", uid: 101 }],
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
// Must resolve, not reject — a paused cursor is a routine, expected pause
|
|
325
|
+
// (epic #1281 invariant 3), not a fault to retry/DLQ.
|
|
326
|
+
await syncMessageBody(event, silentLogger, 1);
|
|
327
|
+
|
|
328
|
+
assert.equal(
|
|
329
|
+
syncBodies.mock.calls.length,
|
|
330
|
+
0,
|
|
331
|
+
"outbound body fetch must not run while the mailbox cursor is paused",
|
|
332
|
+
);
|
|
333
|
+
});
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
describe("batchSyncedMessages — one batch == one ranged fetch", () => {
|
|
337
|
+
const makeSynced = (count: number): SyncedMessage[] =>
|
|
338
|
+
Array.from({ length: count }, (_, i) => ({
|
|
339
|
+
messageId: `msg-${i}`,
|
|
340
|
+
uid: i + 1,
|
|
341
|
+
}));
|
|
342
|
+
|
|
343
|
+
test("batch size is raised to 200", () => {
|
|
344
|
+
assert.equal(BODY_BATCH_SIZE, 200);
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
test("packs up to 200 messages into a single batch", () => {
|
|
348
|
+
const batches = batchSyncedMessages(makeSynced(200));
|
|
349
|
+
|
|
350
|
+
assert.equal(batches.length, 1);
|
|
351
|
+
assert.equal(batches[0].length, 200);
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
test("splits 201 messages into 200 + 1", () => {
|
|
355
|
+
const batches = batchSyncedMessages(makeSynced(201));
|
|
356
|
+
|
|
357
|
+
assert.equal(batches.length, 2);
|
|
358
|
+
assert.equal(batches[0].length, 200);
|
|
359
|
+
assert.equal(batches[1].length, 1);
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
test("keeps messageId+uid pairs intact per batch", () => {
|
|
363
|
+
const batches = batchSyncedMessages(makeSynced(3));
|
|
364
|
+
|
|
365
|
+
assert.deepEqual(batches[0], [
|
|
366
|
+
{ messageId: "msg-0", uid: 1 },
|
|
367
|
+
{ messageId: "msg-1", uid: 2 },
|
|
368
|
+
{ messageId: "msg-2", uid: 3 },
|
|
369
|
+
]);
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
test("empty input yields no batches", () => {
|
|
373
|
+
assert.deepEqual(batchSyncedMessages([]), []);
|
|
374
|
+
});
|
|
375
|
+
});
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
import { getClient } from "@remit/backend/client";
|
|
2
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
3
|
+
import { MetricUnit, metrics } from "@remit/logger-lambda";
|
|
4
|
+
import {
|
|
5
|
+
BodySyncService,
|
|
6
|
+
type FilterConfig,
|
|
7
|
+
guardConnectionCursor,
|
|
8
|
+
isCursorRebuildNeeded,
|
|
9
|
+
MailboxCursorPausedError,
|
|
10
|
+
PlacementMoveService,
|
|
11
|
+
resolveExhaustedBodySyncFailures,
|
|
12
|
+
} from "@remit/mailbox-service";
|
|
13
|
+
import { env } from "expect-env";
|
|
14
|
+
import { isAccountDeleted } from "../account-check.js";
|
|
15
|
+
import { isBodySyncEnabled } from "../body-sync-gate.js";
|
|
16
|
+
import {
|
|
17
|
+
borrowWarmConnection,
|
|
18
|
+
createConnectionScopeWithCredentials,
|
|
19
|
+
} from "../connection-scope.js";
|
|
20
|
+
import type { SyncMessageBodyEvent } from "../events.js";
|
|
21
|
+
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
22
|
+
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
23
|
+
|
|
24
|
+
const bodySyncEnabledParameterName = env.BODY_SYNC_ENABLED_PARAMETER_NAME;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Fallback when `BODY_SYNC_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
28
|
+
* Matches the body queue's own `MAX_RECEIVE_COUNT` default
|
|
29
|
+
* (`infra/stacks/dev/stacks/remit-queue-stack.ts`) so an environment that
|
|
30
|
+
* never injects the var still behaves like production.
|
|
31
|
+
*/
|
|
32
|
+
const DEFAULT_BODY_SYNC_MAX_ATTEMPTS = 3;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Reads the redelivery-budget-exhaustion threshold. CDK derives
|
|
36
|
+
* `BODY_SYNC_MAX_ATTEMPTS` from the body queue's own `MAX_RECEIVE_COUNT`
|
|
37
|
+
* (`remit-worker-stack.ts`) so the two constants can't drift apart — a
|
|
38
|
+
* hand-copied duplicate here previously risked the worker resolving
|
|
39
|
+
* "last attempt" on a different delivery than the queue's redrive policy
|
|
40
|
+
* actually uses (issue #1270). SQS's own `ApproximateReceiveCount` is the
|
|
41
|
+
* source of truth for how many times a record has been delivered; once it
|
|
42
|
+
* reaches this value, the current invocation is the last attempt before the
|
|
43
|
+
* queue's own redrive would DLQ the record, so retry exhaustion is resolved
|
|
44
|
+
* here (see `resolveExhaustedBodySyncFailures`) instead of letting the
|
|
45
|
+
* record dead-letter with no diagnosis.
|
|
46
|
+
*/
|
|
47
|
+
export const getBodySyncMaxAttempts = (
|
|
48
|
+
processEnv: NodeJS.ProcessEnv = process.env,
|
|
49
|
+
): number => {
|
|
50
|
+
const raw = processEnv.BODY_SYNC_MAX_ATTEMPTS;
|
|
51
|
+
if (!raw) return DEFAULT_BODY_SYNC_MAX_ATTEMPTS;
|
|
52
|
+
const parsed = Number.parseInt(raw, 10);
|
|
53
|
+
return Number.isFinite(parsed) && parsed > 0
|
|
54
|
+
? parsed
|
|
55
|
+
: DEFAULT_BODY_SYNC_MAX_ATTEMPTS;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export const BODY_SYNC_MAX_ATTEMPTS = getBodySyncMaxAttempts();
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The ordered message ids to sync, plus a uid lookup when the event carried the
|
|
62
|
+
* preferred `messages` shape. The uid map lets the body-sync service skip the
|
|
63
|
+
* per-message DDB get; legacy events (ids only) leave it undefined and the
|
|
64
|
+
* service resolves uids itself.
|
|
65
|
+
*/
|
|
66
|
+
export interface ResolvedBatch {
|
|
67
|
+
messageIds: string[];
|
|
68
|
+
uidByMessageId?: Map<string, number>;
|
|
69
|
+
force: boolean;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export const resolveBatch = (event: SyncMessageBodyEvent): ResolvedBatch => {
|
|
73
|
+
const force = event.force === true;
|
|
74
|
+
if (event.messages !== undefined) {
|
|
75
|
+
return {
|
|
76
|
+
messageIds: event.messages.map((m) => m.messageId),
|
|
77
|
+
uidByMessageId: new Map(
|
|
78
|
+
event.messages.map((m): [string, number] => [m.messageId, m.uid]),
|
|
79
|
+
),
|
|
80
|
+
force,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
return { messageIds: event.messageIds, force };
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Build the loud, structured error thrown when a batch still has failed
|
|
88
|
+
* messages and SQS redelivery budget left. Throwing (rather than swallowing
|
|
89
|
+
* into a fresh re-enqueue) is what lets a genuine processing failure reach
|
|
90
|
+
* the body-dlq once `BODY_SYNC_MAX_ATTEMPTS` is exhausted — the SQS-level
|
|
91
|
+
* redrive owns retry scheduling now, not this handler.
|
|
92
|
+
*/
|
|
93
|
+
export const buildRetryableFailureError = (
|
|
94
|
+
failedMessageIds: string[],
|
|
95
|
+
receiveCount: number,
|
|
96
|
+
): Error =>
|
|
97
|
+
new Error(
|
|
98
|
+
`Body sync failed for ${failedMessageIds.length} message(s) ` +
|
|
99
|
+
`(attempt ${receiveCount}/${BODY_SYNC_MAX_ATTEMPTS}): ${failedMessageIds.join(", ")}`,
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
export const syncMessageBody = async (
|
|
103
|
+
event: SyncMessageBodyEvent,
|
|
104
|
+
log: Logger,
|
|
105
|
+
receiveCount = 1,
|
|
106
|
+
): Promise<void> => {
|
|
107
|
+
const { accountId, mailboxId } = event;
|
|
108
|
+
|
|
109
|
+
// Prefer the messageId+uid pairs when present (one ranged FETCH, no
|
|
110
|
+
// per-message UID lookup); fall back to the legacy id-only list otherwise.
|
|
111
|
+
const { messageIds, force } = resolveBatch(event);
|
|
112
|
+
|
|
113
|
+
log.info(
|
|
114
|
+
{
|
|
115
|
+
event: event.type,
|
|
116
|
+
accountId,
|
|
117
|
+
mailboxId,
|
|
118
|
+
messageCount: messageIds.length,
|
|
119
|
+
hasUids: event.messages !== undefined,
|
|
120
|
+
force,
|
|
121
|
+
receiveCount,
|
|
122
|
+
},
|
|
123
|
+
"Handling event",
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
// Pause gate stays first: ack-and-skip before touching any account/IMAP state.
|
|
127
|
+
if (!(await isBodySyncEnabled(bodySyncEnabledParameterName, log))) {
|
|
128
|
+
log.info(
|
|
129
|
+
{
|
|
130
|
+
event: event.type,
|
|
131
|
+
accountId,
|
|
132
|
+
mailboxId,
|
|
133
|
+
messageCount: messageIds.length,
|
|
134
|
+
},
|
|
135
|
+
"Body sync paused via SSM toggle, acking and skipping",
|
|
136
|
+
);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const {
|
|
141
|
+
account: accountService,
|
|
142
|
+
mailbox: mailboxService,
|
|
143
|
+
mailboxSpecialUse: mailboxSpecialUseService,
|
|
144
|
+
message: messageService,
|
|
145
|
+
threadMessage: threadMessageService,
|
|
146
|
+
address: addressService,
|
|
147
|
+
envelope: envelopeService,
|
|
148
|
+
placementMove: markerService,
|
|
149
|
+
filter: filterService,
|
|
150
|
+
filterAnchor: filterAnchorService,
|
|
151
|
+
messageLabel: messageLabelService,
|
|
152
|
+
storage,
|
|
153
|
+
secrets,
|
|
154
|
+
} = await getClient();
|
|
155
|
+
|
|
156
|
+
const account = await accountService.get(accountId);
|
|
157
|
+
if (!account) {
|
|
158
|
+
throw new Error(`Account ${accountId} not found`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (isAccountDeleted(account, log)) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// PLACEMENT_MOVE_PUSH rides messageMgmtQueue (issue #1271) — same queue
|
|
166
|
+
// the general MessageMoveService move/delete/copy machinery uses, not a
|
|
167
|
+
// dedicated one.
|
|
168
|
+
const placementMoveQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
169
|
+
const placementMoveService = placementMoveQueueUrl
|
|
170
|
+
? new PlacementMoveService({
|
|
171
|
+
messageService,
|
|
172
|
+
threadMessageService,
|
|
173
|
+
markerService,
|
|
174
|
+
sqsQueueUrl: placementMoveQueueUrl,
|
|
175
|
+
})
|
|
176
|
+
: undefined;
|
|
177
|
+
|
|
178
|
+
await withOAuthLifecycle(
|
|
179
|
+
buildLifecycleDeps(secrets, accountService),
|
|
180
|
+
account,
|
|
181
|
+
log,
|
|
182
|
+
async (credentials) => {
|
|
183
|
+
const mailbox = await mailboxService.get(accountId, mailboxId);
|
|
184
|
+
|
|
185
|
+
// Cheap frugal skip (epic #1281 invariant 6): a mailbox already known
|
|
186
|
+
// paused never even borrows a connection. This is an optimization only
|
|
187
|
+
// — guardConnectionCursor below is the structural guarantee, so a
|
|
188
|
+
// handler that forgot this check still cannot reach a stale UID.
|
|
189
|
+
if (isCursorRebuildNeeded(mailbox.cursorState)) {
|
|
190
|
+
log.info(
|
|
191
|
+
{ accountId, mailboxId, cursorState: mailbox.cursorState },
|
|
192
|
+
"Mailbox cursor not normal; pausing outbound body sync this round",
|
|
193
|
+
);
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// Warm reuse: borrow a live IMAP connection from the module-scoped pool
|
|
198
|
+
// (keyed by accountId) instead of dialing a fresh one per invocation. A
|
|
199
|
+
// warm container skips TCP+TLS+LOGIN+SELECT; a dead pooled connection is
|
|
200
|
+
// liveness-checked and replaced inside borrowWarmConnection.
|
|
201
|
+
const borrowed = borrowWarmConnection(accountId, () =>
|
|
202
|
+
createConnectionScopeWithCredentials(account, credentials),
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
// A confident, actionable placement verdict moves mail directly on
|
|
206
|
+
// body-sync. Safety lives in the verdict itself (only confident,
|
|
207
|
+
// INBOX/Junk-only) and the movedByRemit loop guard.
|
|
208
|
+
const placementConfig = placementMoveService
|
|
209
|
+
? {
|
|
210
|
+
mailboxSpecialUseService,
|
|
211
|
+
placementMoveService,
|
|
212
|
+
}
|
|
213
|
+
: undefined;
|
|
214
|
+
|
|
215
|
+
// A matched filter's actions (label upsert, exclusive move) apply on
|
|
216
|
+
// the same body-sync pass, reusing the placement mover for the move
|
|
217
|
+
// (RFC 034 Decision 3.1). Absent the placement mover there is no move
|
|
218
|
+
// path, so filters stay off — the two share the same enqueue plumbing.
|
|
219
|
+
const filterConfig: FilterConfig | undefined = placementMoveService
|
|
220
|
+
? {
|
|
221
|
+
filterService,
|
|
222
|
+
filterAnchorService,
|
|
223
|
+
messageLabelService,
|
|
224
|
+
placementMoveService,
|
|
225
|
+
}
|
|
226
|
+
: undefined;
|
|
227
|
+
|
|
228
|
+
const bodySyncService = new BodySyncService(
|
|
229
|
+
messageService,
|
|
230
|
+
storage,
|
|
231
|
+
threadMessageService,
|
|
232
|
+
addressService,
|
|
233
|
+
envelopeService,
|
|
234
|
+
log,
|
|
235
|
+
placementConfig,
|
|
236
|
+
filterConfig,
|
|
237
|
+
);
|
|
238
|
+
|
|
239
|
+
// Guard at the openBox choke point (epic #1281 invariants 3 & 5). The
|
|
240
|
+
// served UIDVALIDITY is only knowable by opening the box, which only a
|
|
241
|
+
// genuine fetch needs — BodySyncService.syncBodies skips connecting
|
|
242
|
+
// entirely when every message already has a stored body (nothing would
|
|
243
|
+
// touch a stored UID either way) — so the check fires lazily, the first
|
|
244
|
+
// time `openBox` is actually called, instead of forcing an extra open
|
|
245
|
+
// on every event.
|
|
246
|
+
const getConnectionChecked = async () =>
|
|
247
|
+
guardConnectionCursor(
|
|
248
|
+
await borrowed.getConnection(),
|
|
249
|
+
{ mailboxService },
|
|
250
|
+
accountId,
|
|
251
|
+
mailbox,
|
|
252
|
+
);
|
|
253
|
+
|
|
254
|
+
// Both the sync attempt AND the retry-exhaustion resolution below run
|
|
255
|
+
// against the SAME borrowed connection, so the mailbox stays open
|
|
256
|
+
// across the two — the connection is only released once both are done.
|
|
257
|
+
await (async () => {
|
|
258
|
+
const result = await bodySyncService.syncBodies(
|
|
259
|
+
messageIds,
|
|
260
|
+
accountId,
|
|
261
|
+
account.accountConfigId,
|
|
262
|
+
mailbox.fullPath,
|
|
263
|
+
getConnectionChecked,
|
|
264
|
+
force,
|
|
265
|
+
);
|
|
266
|
+
|
|
267
|
+
if (result.failedMessageIds.length === 0) {
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (receiveCount < BODY_SYNC_MAX_ATTEMPTS) {
|
|
272
|
+
// Redelivery budget remains: let SQS retry the whole record
|
|
273
|
+
// naturally. Messages that already synced skip via the
|
|
274
|
+
// already-stored guard on the next attempt, so only the failed
|
|
275
|
+
// ones actually redo work. This — not a manual re-enqueue — is
|
|
276
|
+
// what lets a genuine processing failure ever reach the body-dlq.
|
|
277
|
+
throw buildRetryableFailureError(
|
|
278
|
+
result.failedMessageIds,
|
|
279
|
+
receiveCount,
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// Redelivery budget exhausted: a persistent per-message failure is
|
|
284
|
+
// never a steady state (epic #1281 invariant 3). Resolve every
|
|
285
|
+
// failed id into exactly one of the two terminal outcomes instead of
|
|
286
|
+
// letting the record dead-letter with no diagnosis.
|
|
287
|
+
const { reconciledMessageIds, brokenMessageIds } =
|
|
288
|
+
await resolveExhaustedBodySyncFailures(
|
|
289
|
+
{
|
|
290
|
+
messageService,
|
|
291
|
+
threadMessageService,
|
|
292
|
+
storageService: storage,
|
|
293
|
+
log,
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
accountId,
|
|
297
|
+
accountConfigId: account.accountConfigId,
|
|
298
|
+
mailboxId,
|
|
299
|
+
mailboxPath: mailbox.fullPath,
|
|
300
|
+
failedMessageIds: result.failedMessageIds,
|
|
301
|
+
getConnection: getConnectionChecked,
|
|
302
|
+
},
|
|
303
|
+
);
|
|
304
|
+
|
|
305
|
+
if (reconciledMessageIds.length > 0) {
|
|
306
|
+
metrics.addMetric(
|
|
307
|
+
"bodySyncStaleRowReconciled",
|
|
308
|
+
MetricUnit.Count,
|
|
309
|
+
reconciledMessageIds.length,
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
if (brokenMessageIds.length > 0) {
|
|
313
|
+
metrics.addMetric(
|
|
314
|
+
"bodySyncMessageBroken",
|
|
315
|
+
MetricUnit.Count,
|
|
316
|
+
brokenMessageIds.length,
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
})()
|
|
320
|
+
.catch((error: unknown) => {
|
|
321
|
+
// Expected pause (epic #1281 invariant 3), not a fault: ack and
|
|
322
|
+
// skip rather than propagating into SQS retry/DLQ. The read stays
|
|
323
|
+
// served from whatever is already stored; the fetch resumes once
|
|
324
|
+
// the mailbox returns to normal.
|
|
325
|
+
if (error instanceof MailboxCursorPausedError) {
|
|
326
|
+
log.info(
|
|
327
|
+
{ accountId, mailboxId, cursorState: error.state },
|
|
328
|
+
"UIDVALIDITY changed; mailbox cursor tripped, pausing outbound body sync",
|
|
329
|
+
);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
throw error;
|
|
333
|
+
})
|
|
334
|
+
.finally(() => borrowed.release());
|
|
335
|
+
},
|
|
336
|
+
);
|
|
337
|
+
};
|