@remit/imap-worker 0.0.88 → 0.0.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/cli.ts +3 -2
- package/src/credential-less-sync-fence.test.ts +5 -9
- package/src/e2e-processor-shim.ts +2 -2
- package/src/emit.test.ts +25 -42
- package/src/emit.ts +94 -119
- package/src/events.ts +1 -19
- package/src/handlers/delete-account-objects.test.ts +27 -18
- package/src/handlers/delete-account-objects.ts +5 -12
- package/src/handlers/message-copy.sqlite.test.ts +3 -3
- package/src/handlers/message-copy.ts +5 -2
- package/src/handlers/message-delete.sqlite.test.ts +1 -1
- package/src/handlers/message-delete.ts +7 -4
- package/src/handlers/message-move.sqlite.test.ts +1 -1
- package/src/handlers/message-move.ts +9 -6
- package/src/handlers/placement-move-push.ts +9 -6
- package/src/handlers/placement-push.sqlite.test.ts +3 -2
- package/src/handlers/probe-message-id.sqlite.test.ts +2 -4
- package/src/handlers/sync-mailboxes.ts +8 -4
- package/src/handlers/sync-message-body.sqlite.test.ts +2 -2
- package/src/handlers/sync-message-body.ts +15 -18
- package/src/handlers/sync-messages.sqlite.test.ts +14 -2
- package/src/handlers/sync-messages.ts +20 -8
- package/src/index.ts +3 -1
- package/src/mail-sync-selection.sqlite.test.ts +7 -13
- package/src/pending-sync.sqlite.test.ts +26 -28
- package/src/processor.ts +10 -16
- package/src/queues.ts +40 -0
- package/src/scheduler/attachment-sweep.sqlite.test.ts +2 -2
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +20 -27
- package/src/scheduler/handler.ts +35 -28
- package/src/scheduler/run-tick.sqlite.test.ts +6 -14
- package/src/scheduler/run-tick.ts +10 -14
- package/src/scheduler/runner.ts +15 -12
- package/src/stranded-placement.sqlite.test.ts +9 -5
- package/src/test-helpers/test-world.ts +40 -13
- package/src/test-helpers/worker-queues.ts +11 -0
- package/src/processor.test.ts +0 -58
|
@@ -3,23 +3,20 @@ import {
|
|
|
3
3
|
ListObjectsV2Command,
|
|
4
4
|
S3Client,
|
|
5
5
|
} from "@aws-sdk/client-s3";
|
|
6
|
-
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
7
6
|
import type { Logger } from "@remit/logger";
|
|
8
|
-
import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
9
7
|
import { env } from "expect-env";
|
|
10
8
|
import type { DeleteAccountObjectsEvent } from "../events.js";
|
|
9
|
+
import type { WorkerQueues } from "../queues.js";
|
|
11
10
|
|
|
12
11
|
const s3 = new S3Client({});
|
|
13
12
|
|
|
14
|
-
const sqsQueueUrl = env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
15
|
-
const sqs = createQueueProducer({ queueUrl: sqsQueueUrl });
|
|
16
|
-
|
|
17
13
|
const BATCH_SIZE = 1_000;
|
|
18
14
|
const MIN_REMAINING_MS = 30_000;
|
|
19
15
|
|
|
20
16
|
export const handleDeleteAccountObjects = async (
|
|
21
17
|
event: DeleteAccountObjectsEvent,
|
|
22
18
|
log: Logger,
|
|
19
|
+
queues: WorkerQueues,
|
|
23
20
|
getRemainingTimeMs?: () => number,
|
|
24
21
|
): Promise<void> => {
|
|
25
22
|
const BUCKET_NAME = env.S3_BUCKET_NAME;
|
|
@@ -41,7 +38,7 @@ export const handleDeleteAccountObjects = async (
|
|
|
41
38
|
{ accountConfigId, totalDeleted, continuationToken: currentToken },
|
|
42
39
|
"Near timeout, re-enqueuing with continuation token",
|
|
43
40
|
);
|
|
44
|
-
await reenqueue(accountConfigId, currentToken);
|
|
41
|
+
await reenqueue(queues, accountConfigId, currentToken);
|
|
45
42
|
return;
|
|
46
43
|
}
|
|
47
44
|
|
|
@@ -85,6 +82,7 @@ export const handleDeleteAccountObjects = async (
|
|
|
85
82
|
};
|
|
86
83
|
|
|
87
84
|
const reenqueue = async (
|
|
85
|
+
queues: WorkerQueues,
|
|
88
86
|
accountConfigId: string,
|
|
89
87
|
continuationToken: string | undefined,
|
|
90
88
|
): Promise<void> => {
|
|
@@ -94,10 +92,5 @@ const reenqueue = async (
|
|
|
94
92
|
continuationToken,
|
|
95
93
|
};
|
|
96
94
|
|
|
97
|
-
await
|
|
98
|
-
new SendMessageCommand({
|
|
99
|
-
QueueUrl: sqsQueueUrl,
|
|
100
|
-
MessageBody: JSON.stringify(event),
|
|
101
|
-
}),
|
|
102
|
-
);
|
|
95
|
+
await queues.messageManagement.send({ body: event });
|
|
103
96
|
};
|
|
@@ -73,7 +73,7 @@ const arrange = async (
|
|
|
73
73
|
};
|
|
74
74
|
|
|
75
75
|
const run = (a: Arranged, receiveCount = 1) =>
|
|
76
|
-
handleMessageCopy(a.event, a.w.log, receiveCount, a.w.scope);
|
|
76
|
+
handleMessageCopy(a.event, a.w.log, a.w.queues, receiveCount, a.w.scope);
|
|
77
77
|
|
|
78
78
|
describe("a message copy", () => {
|
|
79
79
|
it("lands at the destination, leaves the source alone and settles the copy row", async () => {
|
|
@@ -264,7 +264,7 @@ describe("a fresh copy acted on before the server confirms it", () => {
|
|
|
264
264
|
assert.ok(event, "the move enqueued a MESSAGE_MOVE");
|
|
265
265
|
assert.equal(event.uid, copyUid);
|
|
266
266
|
|
|
267
|
-
await handleMessageMove(event, a.w.log, 1, a.w.scope);
|
|
267
|
+
await handleMessageMove(event, a.w.log, a.w.queues, 1, a.w.scope);
|
|
268
268
|
|
|
269
269
|
assert.deepEqual(a.w.imap.uids("Projects", a.messageIdHeader), []);
|
|
270
270
|
assert.equal(a.w.imap.uids("INBOX", a.messageIdHeader).length, 2);
|
|
@@ -288,7 +288,7 @@ describe("a fresh copy acted on before the server confirms it", () => {
|
|
|
288
288
|
const event = a.w.queued().find(isDelete);
|
|
289
289
|
assert.ok(event, "the delete enqueued a MESSAGE_DELETE");
|
|
290
290
|
|
|
291
|
-
await handleMessageDelete(event, a.w.log, 1, a.w.scope);
|
|
291
|
+
await handleMessageDelete(event, a.w.log, a.w.queues, 1, a.w.scope);
|
|
292
292
|
|
|
293
293
|
assert.deepEqual(a.w.imap.uids("Projects", a.messageIdHeader), []);
|
|
294
294
|
assert.deepEqual(a.w.imap.uids("INBOX"), [a.uid]);
|
|
@@ -14,9 +14,10 @@ import {
|
|
|
14
14
|
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
15
15
|
import { isAccountDeleted } from "../account-check.js";
|
|
16
16
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
17
|
-
import {
|
|
17
|
+
import { createEmitEvent } from "../emit.js";
|
|
18
18
|
import type { MessageCopyEvent } from "../events.js";
|
|
19
19
|
import { findMailboxRow } from "../mailbox-row.js";
|
|
20
|
+
import type { WorkerQueues } from "../queues.js";
|
|
20
21
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
21
22
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
22
23
|
import {
|
|
@@ -68,9 +69,11 @@ export type CopyProbe =
|
|
|
68
69
|
export const handleMessageCopy = async (
|
|
69
70
|
event: MessageCopyEvent,
|
|
70
71
|
log: Logger,
|
|
72
|
+
queues: WorkerQueues,
|
|
71
73
|
receiveCount = 1,
|
|
72
74
|
createConnectionScope = createConnectionScopeWithCredentials,
|
|
73
75
|
): Promise<void> => {
|
|
76
|
+
const emit = createEmitEvent(queues);
|
|
74
77
|
const {
|
|
75
78
|
account: accountService,
|
|
76
79
|
message: messageService,
|
|
@@ -383,7 +386,7 @@ export const handleMessageCopy = async (
|
|
|
383
386
|
}
|
|
384
387
|
}
|
|
385
388
|
|
|
386
|
-
await emitMoveResync(
|
|
389
|
+
await emitMoveResync(emit, {
|
|
387
390
|
accountId,
|
|
388
391
|
sourceMailboxId,
|
|
389
392
|
destinationMailboxId,
|
|
@@ -72,7 +72,7 @@ const arrange = async (
|
|
|
72
72
|
};
|
|
73
73
|
|
|
74
74
|
const run = (a: Arranged, receiveCount = 1, event = a.event) =>
|
|
75
|
-
handleMessageDelete(event, a.w.log, receiveCount, a.w.scope);
|
|
75
|
+
handleMessageDelete(event, a.w.log, a.w.queues, receiveCount, a.w.scope);
|
|
76
76
|
|
|
77
77
|
const resyncs = (w: World): unknown[] =>
|
|
78
78
|
w
|
|
@@ -24,9 +24,10 @@ import {
|
|
|
24
24
|
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
25
25
|
import { isAccountDeleted } from "../account-check.js";
|
|
26
26
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
27
|
-
import {
|
|
27
|
+
import { createEmitEvent } from "../emit.js";
|
|
28
28
|
import type { MessageDeleteEvent } from "../events.js";
|
|
29
29
|
import { findMailboxRow } from "../mailbox-row.js";
|
|
30
|
+
import type { WorkerQueues } from "../queues.js";
|
|
30
31
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
31
32
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
32
33
|
import { resolveExhaustedMessageDeleteFailure } from "./message-delete-terminal.js";
|
|
@@ -153,9 +154,11 @@ const confirmTrashMoveUid = async (
|
|
|
153
154
|
export const handleMessageDelete = async (
|
|
154
155
|
event: MessageDeleteEvent,
|
|
155
156
|
log: Logger,
|
|
157
|
+
queues: WorkerQueues,
|
|
156
158
|
receiveCount = 1,
|
|
157
159
|
createConnectionScope = createConnectionScopeWithCredentials,
|
|
158
160
|
): Promise<void> => {
|
|
161
|
+
const emit = createEmitEvent(queues);
|
|
159
162
|
const {
|
|
160
163
|
account: accountService,
|
|
161
164
|
message: messageService,
|
|
@@ -246,14 +249,14 @@ export const handleMessageDelete = async (
|
|
|
246
249
|
*/
|
|
247
250
|
const emitSettleResync = async (): Promise<void> => {
|
|
248
251
|
if (destinationMailboxId) {
|
|
249
|
-
await emitMoveResync(
|
|
252
|
+
await emitMoveResync(emit, {
|
|
250
253
|
accountId,
|
|
251
254
|
sourceMailboxId: mailboxId,
|
|
252
255
|
destinationMailboxId,
|
|
253
256
|
});
|
|
254
257
|
return;
|
|
255
258
|
}
|
|
256
|
-
await emitMailboxResync(
|
|
259
|
+
await emitMailboxResync(emit, { accountId, mailboxId });
|
|
257
260
|
};
|
|
258
261
|
|
|
259
262
|
/**
|
|
@@ -623,7 +626,7 @@ export const handleMessageDelete = async (
|
|
|
623
626
|
"Refused to delete: the destination mailbox no longer exists (deleted)",
|
|
624
627
|
"message_delete_destination_gone",
|
|
625
628
|
);
|
|
626
|
-
await emitMailboxResync(
|
|
629
|
+
await emitMailboxResync(emit, { accountId, mailboxId });
|
|
627
630
|
return;
|
|
628
631
|
}
|
|
629
632
|
|
|
@@ -68,7 +68,7 @@ const arrange = async (
|
|
|
68
68
|
};
|
|
69
69
|
|
|
70
70
|
const run = (a: Arranged, receiveCount = 1) =>
|
|
71
|
-
handleMessageMove(a.event, a.w.log, receiveCount, a.w.scope);
|
|
71
|
+
handleMessageMove(a.event, a.w.log, a.w.queues, receiveCount, a.w.scope);
|
|
72
72
|
|
|
73
73
|
const resyncs = (w: World): unknown[] =>
|
|
74
74
|
w
|
|
@@ -17,9 +17,10 @@ import {
|
|
|
17
17
|
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
18
18
|
import { isAccountDeleted } from "../account-check.js";
|
|
19
19
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
20
|
-
import {
|
|
20
|
+
import { createEmitEvent } from "../emit.js";
|
|
21
21
|
import type { MessageMoveEvent, SyncMessagesEvent } from "../events.js";
|
|
22
22
|
import { findMailboxRow } from "../mailbox-row.js";
|
|
23
|
+
import type { WorkerQueues } from "../queues.js";
|
|
23
24
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
24
25
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
25
26
|
import { resolveExhaustedMessageMoveFailure } from "./message-move-terminal.js";
|
|
@@ -244,9 +245,11 @@ export const buildThreadMessageMoveUpdate = (
|
|
|
244
245
|
export const handleMessageMove = async (
|
|
245
246
|
event: MessageMoveEvent,
|
|
246
247
|
log: Logger,
|
|
248
|
+
queues: WorkerQueues,
|
|
247
249
|
receiveCount = 1,
|
|
248
250
|
createConnectionScope = createConnectionScopeWithCredentials,
|
|
249
251
|
): Promise<void> => {
|
|
252
|
+
const emit = createEmitEvent(queues);
|
|
250
253
|
const {
|
|
251
254
|
account: accountService,
|
|
252
255
|
message: messageService,
|
|
@@ -343,7 +346,7 @@ export const handleMessageMove = async (
|
|
|
343
346
|
},
|
|
344
347
|
"MESSAGE_MOVE not pushed: source mailbox no longer exists (deleted); stale row reconciled",
|
|
345
348
|
);
|
|
346
|
-
await emitMailboxResync(
|
|
349
|
+
await emitMailboxResync(emit, {
|
|
347
350
|
accountId,
|
|
348
351
|
mailboxId: destinationMailboxId,
|
|
349
352
|
});
|
|
@@ -449,7 +452,7 @@ export const handleMessageMove = async (
|
|
|
449
452
|
{ metric: "message_move_destination_gone" },
|
|
450
453
|
"destination mailbox no longer exists (deleted)",
|
|
451
454
|
);
|
|
452
|
-
await emitMailboxResync(
|
|
455
|
+
await emitMailboxResync(emit, {
|
|
453
456
|
accountId,
|
|
454
457
|
mailboxId: sourceMailboxId,
|
|
455
458
|
});
|
|
@@ -543,7 +546,7 @@ export const handleMessageMove = async (
|
|
|
543
546
|
);
|
|
544
547
|
}
|
|
545
548
|
|
|
546
|
-
await emitMoveResync(
|
|
549
|
+
await emitMoveResync(emit, {
|
|
547
550
|
accountId,
|
|
548
551
|
sourceMailboxId,
|
|
549
552
|
destinationMailboxId,
|
|
@@ -624,7 +627,7 @@ export const handleMessageMove = async (
|
|
|
624
627
|
await settleMoved(newUid);
|
|
625
628
|
},
|
|
626
629
|
() =>
|
|
627
|
-
emitMoveResync(
|
|
630
|
+
emitMoveResync(emit, {
|
|
628
631
|
accountId,
|
|
629
632
|
sourceMailboxId,
|
|
630
633
|
destinationMailboxId,
|
|
@@ -739,7 +742,7 @@ export const handleMessageMove = async (
|
|
|
739
742
|
// re-projects it with the server's own UID. BROKEN, the row has just
|
|
740
743
|
// been put back at the source the server confirmed, and the resync
|
|
741
744
|
// is what carries any drift either folder has picked up since.
|
|
742
|
-
await emitMoveResync(
|
|
745
|
+
await emitMoveResync(emit, {
|
|
743
746
|
accountId,
|
|
744
747
|
sourceMailboxId,
|
|
745
748
|
destinationMailboxId,
|
|
@@ -14,9 +14,10 @@ import {
|
|
|
14
14
|
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
15
15
|
import { isAccountDeleted } from "../account-check.js";
|
|
16
16
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
17
|
-
import {
|
|
17
|
+
import { createEmitEvent } from "../emit.js";
|
|
18
18
|
import type { PlacementMovePushEvent } from "../events.js";
|
|
19
19
|
import { findMailboxRow } from "../mailbox-row.js";
|
|
20
|
+
import type { WorkerQueues } from "../queues.js";
|
|
20
21
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
21
22
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
22
23
|
import {
|
|
@@ -152,9 +153,11 @@ export const attemptMove = async (
|
|
|
152
153
|
export const handlePlacementMovePush = async (
|
|
153
154
|
event: PlacementMovePushEvent,
|
|
154
155
|
log: Logger,
|
|
156
|
+
queues: WorkerQueues,
|
|
155
157
|
receiveCount = 1,
|
|
156
158
|
createConnectionScope = createConnectionScopeWithCredentials,
|
|
157
159
|
): Promise<void> => {
|
|
160
|
+
const emit = createEmitEvent(queues);
|
|
158
161
|
const {
|
|
159
162
|
account: accountService,
|
|
160
163
|
mailbox: mailboxService,
|
|
@@ -238,7 +241,7 @@ export const handlePlacementMovePush = async (
|
|
|
238
241
|
"Placement move not pushed: its source folder is gone; the stale row was reconciled",
|
|
239
242
|
);
|
|
240
243
|
if (destination.kind === "present") {
|
|
241
|
-
await emitMailboxResync(
|
|
244
|
+
await emitMailboxResync(emit, {
|
|
242
245
|
accountId,
|
|
243
246
|
mailboxId: marker.destinationMailboxId,
|
|
244
247
|
});
|
|
@@ -261,7 +264,7 @@ export const handlePlacementMovePush = async (
|
|
|
261
264
|
context,
|
|
262
265
|
"Placement move not pushed: its destination folder is gone; the row was handed back to its source",
|
|
263
266
|
);
|
|
264
|
-
await emitMailboxResync(
|
|
267
|
+
await emitMailboxResync(emit, {
|
|
265
268
|
accountId,
|
|
266
269
|
mailboxId: marker.sourceMailboxId,
|
|
267
270
|
});
|
|
@@ -354,7 +357,7 @@ export const handlePlacementMovePush = async (
|
|
|
354
357
|
},
|
|
355
358
|
"Message no longer at its pending-move source (external delete or moved away); marker dropped, stale row reconciled",
|
|
356
359
|
);
|
|
357
|
-
await emitMoveResync(
|
|
360
|
+
await emitMoveResync(emit, {
|
|
358
361
|
accountId,
|
|
359
362
|
sourceMailboxId: marker.sourceMailboxId,
|
|
360
363
|
destinationMailboxId: marker.destinationMailboxId,
|
|
@@ -404,7 +407,7 @@ export const handlePlacementMovePush = async (
|
|
|
404
407
|
"Placement move confirmed on IMAP; marker cleared",
|
|
405
408
|
);
|
|
406
409
|
|
|
407
|
-
await emitMoveResync(
|
|
410
|
+
await emitMoveResync(emit, {
|
|
408
411
|
accountId,
|
|
409
412
|
sourceMailboxId: marker.sourceMailboxId,
|
|
410
413
|
destinationMailboxId: marker.destinationMailboxId,
|
|
@@ -452,7 +455,7 @@ export const handlePlacementMovePush = async (
|
|
|
452
455
|
);
|
|
453
456
|
|
|
454
457
|
if (outcome === "reconciled") {
|
|
455
|
-
await emitMoveResync(
|
|
458
|
+
await emitMoveResync(emit, {
|
|
456
459
|
accountId,
|
|
457
460
|
sourceMailboxId: marker.sourceMailboxId,
|
|
458
461
|
destinationMailboxId: marker.destinationMailboxId,
|
|
@@ -52,7 +52,7 @@ const arrange = async (): Promise<Arranged> => {
|
|
|
52
52
|
addressService: w.client.address,
|
|
53
53
|
mailboxSpecialUseService: w.client.mailboxSpecialUse,
|
|
54
54
|
mailboxService: w.client.mailbox,
|
|
55
|
-
|
|
55
|
+
queue: w.client.queues.messageManagement,
|
|
56
56
|
});
|
|
57
57
|
return { w, inbox, junk, message, placement };
|
|
58
58
|
};
|
|
@@ -92,7 +92,8 @@ const pushMove = (
|
|
|
92
92
|
a: Arranged,
|
|
93
93
|
event: PlacementMovePushEvent,
|
|
94
94
|
receiveCount = 1,
|
|
95
|
-
) =>
|
|
95
|
+
) =>
|
|
96
|
+
handlePlacementMovePush(event, a.w.log, a.w.queues, receiveCount, a.w.scope);
|
|
96
97
|
|
|
97
98
|
const seenOnServer = (a: Arranged, path: string): boolean[] =>
|
|
98
99
|
a.w.imap
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { afterEach, beforeEach, describe, it, mock } from "node:test";
|
|
3
|
-
import { SQSClient } from "@aws-sdk/client-sqs";
|
|
4
3
|
import type { MailboxItem } from "@remit/data-ports";
|
|
4
|
+
import { recordingQueue } from "@remit/data-ports/recording-queue";
|
|
5
5
|
import {
|
|
6
6
|
AccountAuthType,
|
|
7
7
|
ConnectionState,
|
|
@@ -171,15 +171,13 @@ describe("the Message-ID the move, delete and copy probes ask with", () => {
|
|
|
171
171
|
it("reads the source's Message-ID for a copy", async () => {
|
|
172
172
|
const sourceId = await syncInbox("<report@example.com>");
|
|
173
173
|
const archive = await createMailbox("Archive");
|
|
174
|
-
mock.method(SQSClient.prototype, "send", async () => ({}));
|
|
175
|
-
|
|
176
174
|
const copyId = await new MessageMoveService({
|
|
177
175
|
messageService: messages,
|
|
178
176
|
mailboxService: mailboxes,
|
|
179
177
|
mailboxSpecialUseService: repos.mailboxSpecialUse,
|
|
180
178
|
threadMessageService: threadMessages,
|
|
181
179
|
addressService: repos.address,
|
|
182
|
-
|
|
180
|
+
queue: recordingQueue().fifo,
|
|
183
181
|
}).copyMessage(ACCOUNT_CONFIG_ID, sourceId, archive.mailboxId, accountId);
|
|
184
182
|
|
|
185
183
|
assert.notEqual(copyId, sourceId);
|
|
@@ -21,16 +21,17 @@ import {
|
|
|
21
21
|
MailboxSyncService,
|
|
22
22
|
MailConnectionError,
|
|
23
23
|
type MailCredentials,
|
|
24
|
+
standardMailboxManagement,
|
|
24
25
|
} from "@remit/mailbox-service";
|
|
25
|
-
import { env } from "expect-env";
|
|
26
26
|
import pMap from "p-map";
|
|
27
27
|
import {
|
|
28
28
|
isAccountDeleted,
|
|
29
29
|
isMailSyncDisabled,
|
|
30
30
|
isUnsyncableHost,
|
|
31
31
|
} from "../account-check.js";
|
|
32
|
-
import {
|
|
32
|
+
import { createEmitEvent, type EmitEvent } from "../emit.js";
|
|
33
33
|
import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
|
|
34
|
+
import type { WorkerQueues } from "../queues.js";
|
|
34
35
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
35
36
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
36
37
|
import { createImportedFolder } from "./imported-folder.js";
|
|
@@ -95,6 +96,7 @@ export const mailboxNeedsSync = (
|
|
|
95
96
|
export const syncMailboxes = async (
|
|
96
97
|
event: SyncMailboxesEvent,
|
|
97
98
|
log: Logger,
|
|
99
|
+
queues: WorkerQueues,
|
|
98
100
|
): Promise<void> => {
|
|
99
101
|
const client = await getClient();
|
|
100
102
|
const {
|
|
@@ -118,7 +120,7 @@ export const syncMailboxes = async (
|
|
|
118
120
|
mailboxService,
|
|
119
121
|
new MailboxQueueService({
|
|
120
122
|
mailboxService,
|
|
121
|
-
|
|
123
|
+
enqueue: standardMailboxManagement(queues.mailboxManagement),
|
|
122
124
|
logger: log,
|
|
123
125
|
}),
|
|
124
126
|
),
|
|
@@ -166,6 +168,7 @@ export const syncMailboxes = async (
|
|
|
166
168
|
accountService,
|
|
167
169
|
binder,
|
|
168
170
|
log,
|
|
171
|
+
createEmitEvent(queues),
|
|
169
172
|
);
|
|
170
173
|
} catch (err) {
|
|
171
174
|
// Auth failures are handled by the wrapper — rethrow untouched so it
|
|
@@ -199,6 +202,7 @@ const syncMailboxesForAccount = async (
|
|
|
199
202
|
accountService: IAccountRepository,
|
|
200
203
|
binder: ConfigBinderDeps,
|
|
201
204
|
log: Logger,
|
|
205
|
+
emit: EmitEvent,
|
|
202
206
|
): Promise<void> => {
|
|
203
207
|
const { accountId } = account;
|
|
204
208
|
|
|
@@ -330,7 +334,7 @@ const syncMailboxesForAccount = async (
|
|
|
330
334
|
mailboxCountSynced: skipped,
|
|
331
335
|
});
|
|
332
336
|
|
|
333
|
-
await emitSyncMessagesEvents(accountId, mailboxes,
|
|
337
|
+
await emitSyncMessagesEvents(accountId, mailboxes, emit);
|
|
334
338
|
};
|
|
335
339
|
|
|
336
340
|
type SyncMessagesInput = Omit<SyncMessagesEvent, "eventId" | "timestamp">;
|
|
@@ -73,7 +73,7 @@ const arrange = async (): Promise<Arranged> => {
|
|
|
73
73
|
};
|
|
74
74
|
|
|
75
75
|
const run = (a: Arranged, receiveCount = 1) =>
|
|
76
|
-
syncMessageBody(a.event, a.w.log, receiveCount, a.w.scope);
|
|
76
|
+
syncMessageBody(a.event, a.w.log, a.w.queues, receiveCount, a.w.scope);
|
|
77
77
|
|
|
78
78
|
const bodyFetches = (a: Arranged): string[] => a.w.imap.commands("BODY");
|
|
79
79
|
|
|
@@ -160,7 +160,7 @@ const batchOf = (
|
|
|
160
160
|
});
|
|
161
161
|
|
|
162
162
|
const runBatch = (w: World, event: SyncMessageBodyEvent, receiveCount = 1) =>
|
|
163
|
-
syncMessageBody(event, w.log, receiveCount, w.scope);
|
|
163
|
+
syncMessageBody(event, w.log, w.queues, receiveCount, w.scope);
|
|
164
164
|
|
|
165
165
|
const arrangeFiling = async () => {
|
|
166
166
|
const w = await store.world();
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
createConnectionScopeWithCredentials,
|
|
26
26
|
} from "../connection-scope.js";
|
|
27
27
|
import type { SyncMessageBodyEvent } from "../events.js";
|
|
28
|
+
import type { WorkerQueues } from "../queues.js";
|
|
28
29
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
29
30
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
30
31
|
import { workerVersion } from "../worker-version.js";
|
|
@@ -87,6 +88,7 @@ export const buildRetryableFailureError = (
|
|
|
87
88
|
export const syncMessageBody = async (
|
|
88
89
|
event: SyncMessageBodyEvent,
|
|
89
90
|
log: Logger,
|
|
91
|
+
queues: WorkerQueues,
|
|
90
92
|
receiveCount = 1,
|
|
91
93
|
createConnectionScope = createConnectionScopeWithCredentials,
|
|
92
94
|
): Promise<void> => {
|
|
@@ -156,18 +158,15 @@ export const syncMessageBody = async (
|
|
|
156
158
|
// PLACEMENT_MOVE_PUSH rides messageMgmtQueue (issue #1271) — same queue
|
|
157
159
|
// the general MessageMoveService move/delete/copy machinery uses, not a
|
|
158
160
|
// dedicated one.
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
sqsQueueUrl: placementMoveQueueUrl,
|
|
169
|
-
})
|
|
170
|
-
: undefined;
|
|
161
|
+
const placementMoveService = new PlacementMoveService({
|
|
162
|
+
messageService,
|
|
163
|
+
threadMessageService,
|
|
164
|
+
markerService,
|
|
165
|
+
addressService,
|
|
166
|
+
mailboxSpecialUseService,
|
|
167
|
+
mailboxService,
|
|
168
|
+
queue: queues.messageManagement,
|
|
169
|
+
});
|
|
171
170
|
|
|
172
171
|
await withOAuthLifecycle(
|
|
173
172
|
buildLifecycleDeps(secrets, accountService),
|
|
@@ -220,12 +219,10 @@ export const syncMessageBody = async (
|
|
|
220
219
|
// A confident, actionable placement verdict moves mail directly on
|
|
221
220
|
// body-sync. Safety lives in the verdict itself (only confident,
|
|
222
221
|
// INBOX/Junk-only) and the movedByRemit loop guard.
|
|
223
|
-
const placementConfig =
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
}
|
|
228
|
-
: undefined;
|
|
222
|
+
const placementConfig = {
|
|
223
|
+
mailboxSpecialUseService,
|
|
224
|
+
placementMoveService,
|
|
225
|
+
};
|
|
229
226
|
|
|
230
227
|
// A matched filter's actions (label upsert, exclusive move) apply on
|
|
231
228
|
// the same body-sync pass, reusing the placement mover for the move
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { after, before, describe, it } from "node:test";
|
|
3
|
+
import { createEmitEvent } from "../emit.js";
|
|
3
4
|
import type { SyncMessagesEvent } from "../events.js";
|
|
4
5
|
import { openStore, type World } from "../test-helpers/test-world.js";
|
|
5
6
|
import { drainPendingFlagPushes, syncMessages } from "./sync-messages.js";
|
|
@@ -21,7 +22,12 @@ const syncEvent = (w: World, mailboxId: string): SyncMessagesEvent => ({
|
|
|
21
22
|
});
|
|
22
23
|
|
|
23
24
|
const sync = async (w: World, mailboxId: string): Promise<number> => {
|
|
24
|
-
await syncMessages(
|
|
25
|
+
await syncMessages(
|
|
26
|
+
syncEvent(w, mailboxId),
|
|
27
|
+
w.log,
|
|
28
|
+
w.queues,
|
|
29
|
+
w.connectionFactory,
|
|
30
|
+
);
|
|
25
31
|
return w.imap.connections;
|
|
26
32
|
};
|
|
27
33
|
|
|
@@ -106,7 +112,12 @@ describe("a folder deleted while its sync is in flight", () => {
|
|
|
106
112
|
return { getConnection: () => connection, close: factory.close };
|
|
107
113
|
};
|
|
108
114
|
|
|
109
|
-
await syncMessages(
|
|
115
|
+
await syncMessages(
|
|
116
|
+
syncEvent(w, old.mailboxId),
|
|
117
|
+
w.log,
|
|
118
|
+
w.queues,
|
|
119
|
+
connectionFactory,
|
|
120
|
+
);
|
|
110
121
|
|
|
111
122
|
const account = await w.client.account.get(w.account.accountId);
|
|
112
123
|
assert.notEqual(account.syncPhase, "error");
|
|
@@ -142,6 +153,7 @@ describe("drainPendingFlagPushes (issue #1273)", () => {
|
|
|
142
153
|
w.account,
|
|
143
154
|
inbox.mailboxId,
|
|
144
155
|
w.log,
|
|
156
|
+
createEmitEvent(w.queues),
|
|
145
157
|
);
|
|
146
158
|
|
|
147
159
|
assert.deepEqual(
|
|
@@ -35,12 +35,17 @@ import {
|
|
|
35
35
|
isMailSyncDisabled,
|
|
36
36
|
isUnsyncableHost,
|
|
37
37
|
} from "../account-check.js";
|
|
38
|
-
import {
|
|
38
|
+
import {
|
|
39
|
+
createEmitEvent,
|
|
40
|
+
type EmitEvent,
|
|
41
|
+
SYNC_MESSAGES_PENDING,
|
|
42
|
+
} from "../emit.js";
|
|
39
43
|
import type {
|
|
40
44
|
FlagPushEvent,
|
|
41
45
|
SyncMessageBodyEvent,
|
|
42
46
|
SyncMessagesEvent,
|
|
43
47
|
} from "../events.js";
|
|
48
|
+
import type { WorkerQueues } from "../queues.js";
|
|
44
49
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
45
50
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
46
51
|
import { workerVersion } from "../worker-version.js";
|
|
@@ -77,6 +82,7 @@ export const managedConnectionFor = (
|
|
|
77
82
|
export const syncMessages = async (
|
|
78
83
|
event: SyncMessagesEvent,
|
|
79
84
|
log: Logger,
|
|
85
|
+
queues: WorkerQueues,
|
|
80
86
|
createConnectionFactory = managedConnectionFor,
|
|
81
87
|
): Promise<void> => {
|
|
82
88
|
log.info(
|
|
@@ -212,6 +218,7 @@ export const syncMessages = async (
|
|
|
212
218
|
syncWarningRepository,
|
|
213
219
|
flagPushMarkerService,
|
|
214
220
|
messageFlagService,
|
|
221
|
+
emit: createEmitEvent(queues),
|
|
215
222
|
},
|
|
216
223
|
log,
|
|
217
224
|
);
|
|
@@ -290,6 +297,7 @@ interface SyncDeps {
|
|
|
290
297
|
syncWarningRepository: ISyncWarningRepository;
|
|
291
298
|
flagPushMarkerService: IMessageFlagPushRepository;
|
|
292
299
|
messageFlagService: IMessageFlagRepository;
|
|
300
|
+
emit: EmitEvent;
|
|
293
301
|
}
|
|
294
302
|
|
|
295
303
|
// Bounds concurrent SQS sends while re-arming stuck flag-push markers —
|
|
@@ -314,16 +322,13 @@ const FLAG_PUSH_DRAIN_CONCURRENCY = 5;
|
|
|
314
322
|
* loudly — it must never fail the surrounding SYNC_MESSAGES batch, which is
|
|
315
323
|
* unrelated message-header sync work. The marker stays durable regardless;
|
|
316
324
|
* the next periodic tick tries again.
|
|
317
|
-
*
|
|
318
|
-
* `emit` defaults to the real `emitEvent` (imap-worker's shared SQS
|
|
319
|
-
* producer) and is only ever overridden in tests.
|
|
320
325
|
*/
|
|
321
326
|
export const drainPendingFlagPushes = async (
|
|
322
327
|
flagPushMarkerService: IMessageFlagPushRepository,
|
|
323
328
|
account: AccountItem,
|
|
324
329
|
mailboxId: string,
|
|
325
330
|
log: Logger,
|
|
326
|
-
emit:
|
|
331
|
+
emit: EmitEvent,
|
|
327
332
|
): Promise<void> => {
|
|
328
333
|
const markers = await flagPushMarkerService.listByMailboxId(mailboxId);
|
|
329
334
|
const stuck = markers.filter((marker) => marker.state === "pending");
|
|
@@ -383,6 +388,7 @@ const syncMailboxMessages = async (
|
|
|
383
388
|
syncWarningRepository,
|
|
384
389
|
flagPushMarkerService,
|
|
385
390
|
messageFlagService,
|
|
391
|
+
emit,
|
|
386
392
|
} = deps;
|
|
387
393
|
|
|
388
394
|
// Get the mailbox - it must exist (should have been created by mailbox sync)
|
|
@@ -392,7 +398,13 @@ const syncMailboxMessages = async (
|
|
|
392
398
|
// Periodic drain (issue #1273): independent of the IMAP sync below — no
|
|
393
399
|
// connection needed, just an SQS re-arm — so it runs regardless of this
|
|
394
400
|
// round's sync outcome.
|
|
395
|
-
await drainPendingFlagPushes(
|
|
401
|
+
await drainPendingFlagPushes(
|
|
402
|
+
flagPushMarkerService,
|
|
403
|
+
account,
|
|
404
|
+
mailboxId,
|
|
405
|
+
log,
|
|
406
|
+
emit,
|
|
407
|
+
);
|
|
396
408
|
const isInbox = mailbox.fullPath.toUpperCase() === "INBOX";
|
|
397
409
|
|
|
398
410
|
const syncService = new MessageSyncService(
|
|
@@ -466,7 +478,7 @@ const syncMailboxMessages = async (
|
|
|
466
478
|
messageIds: batch.map((m) => m.messageId),
|
|
467
479
|
messages: batch.map((m) => ({ messageId: m.messageId, uid: m.uid })),
|
|
468
480
|
};
|
|
469
|
-
return
|
|
481
|
+
return emit(bodyEvent);
|
|
470
482
|
},
|
|
471
483
|
{ concurrency: EVENT_EMIT_CONCURRENCY },
|
|
472
484
|
);
|
|
@@ -484,7 +496,7 @@ const syncMailboxMessages = async (
|
|
|
484
496
|
accountId: event.accountId,
|
|
485
497
|
mailboxId,
|
|
486
498
|
};
|
|
487
|
-
await
|
|
499
|
+
await emit(nextSyncEvent);
|
|
488
500
|
return;
|
|
489
501
|
}
|
|
490
502
|
|
package/src/index.ts
CHANGED
|
@@ -9,8 +9,10 @@ import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
|
|
|
9
9
|
import type { WorkerEvent } from "./events.js";
|
|
10
10
|
import { imapFailureKind } from "./failure-kind.js";
|
|
11
11
|
import { processEvent } from "./processor.js";
|
|
12
|
+
import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "./queues.js";
|
|
12
13
|
|
|
13
14
|
installNodeLogger();
|
|
15
|
+
const queues = sqsWorkerQueues(workerQueueUrlsFromEnv());
|
|
14
16
|
|
|
15
17
|
const log = createLogger();
|
|
16
18
|
|
|
@@ -45,7 +47,7 @@ export const handler: SQSHandler = withTelemetry(
|
|
|
45
47
|
|
|
46
48
|
const queue = queueNameFromEventSource(record.eventSourceARN);
|
|
47
49
|
const opStart = Date.now();
|
|
48
|
-
const failed = await processEvent(imapEvent, log, receiveCount)
|
|
50
|
+
const failed = await processEvent(imapEvent, log, queues, receiveCount)
|
|
49
51
|
.then(() => {
|
|
50
52
|
recordQueueEvent({
|
|
51
53
|
queue,
|