@remit/imap-worker 0.0.88 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/cli.ts +7 -0
- package/src/credential-less-sync-fence.test.ts +5 -9
- package/src/e2e-processor-shim.ts +2 -2
- package/src/emit.test.ts +24 -59
- package/src/emit.ts +50 -73
- package/src/events.ts +1 -19
- package/src/handlers/delete-account-objects.test.ts +25 -14
- package/src/handlers/delete-account-objects.ts +2 -11
- package/src/handlers/placement-push.sqlite.test.ts +1 -1
- package/src/handlers/probe-message-id.sqlite.test.ts +2 -4
- package/src/handlers/sync-mailboxes.ts +2 -2
- package/src/handlers/sync-message-body.ts +14 -18
- package/src/index.ts +6 -0
- package/src/mail-sync-selection.sqlite.test.ts +14 -13
- package/src/pending-sync.sqlite.test.ts +27 -28
- package/src/processor.ts +0 -8
- package/src/queues.ts +49 -0
- package/src/scheduler/attachment-sweep.sqlite.test.ts +1 -0
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +20 -27
- package/src/scheduler/handler.ts +8 -6
- package/src/scheduler/run-tick.sqlite.test.ts +6 -14
- package/src/scheduler/run-tick.ts +10 -14
- package/src/scheduler/runner.ts +18 -10
- package/src/stranded-placement.sqlite.test.ts +4 -5
- package/src/test-helpers/test-world.ts +40 -11
- package/src/processor.test.ts +0 -58
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remit/imap-worker",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.89",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -53,11 +53,11 @@
|
|
|
53
53
|
"@remit/domain-enums": "*",
|
|
54
54
|
"@remit/mail-oauth-service": "*",
|
|
55
55
|
"@remit/secrets-service": "*",
|
|
56
|
-
"@aws-sdk/client-sqs": "
|
|
57
|
-
"@aws-sdk/client-ssm": "
|
|
58
|
-
"expect-env": "
|
|
59
|
-
"p-map": "
|
|
60
|
-
"@types/aws-lambda": "
|
|
56
|
+
"@aws-sdk/client-sqs": "^3.1090.0",
|
|
57
|
+
"@aws-sdk/client-ssm": "^3.1090.0",
|
|
58
|
+
"expect-env": "^1.0.1",
|
|
59
|
+
"p-map": "^2.1.0",
|
|
60
|
+
"@types/aws-lambda": "^8.10.162"
|
|
61
61
|
},
|
|
62
62
|
"license": "MIT",
|
|
63
63
|
"publishConfig": {
|
package/src/cli.ts
CHANGED
|
@@ -7,6 +7,11 @@
|
|
|
7
7
|
// text and argument errors are for the person who typed the command.
|
|
8
8
|
import { parseArgs } from "node:util";
|
|
9
9
|
import { emitEvent } from "./emit.js";
|
|
10
|
+
import {
|
|
11
|
+
setWorkerQueues,
|
|
12
|
+
sqsWorkerQueues,
|
|
13
|
+
workerQueueUrlsFromEnv,
|
|
14
|
+
} from "./queues.js";
|
|
10
15
|
|
|
11
16
|
const HELP = `
|
|
12
17
|
remit-imap-worker - Process IMAP sync events
|
|
@@ -204,6 +209,8 @@ const buildEvent = () => {
|
|
|
204
209
|
|
|
205
210
|
const event = buildEvent();
|
|
206
211
|
|
|
212
|
+
setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
|
|
213
|
+
|
|
207
214
|
console.log(`Enqueueing ${event.type} event for account ${event.accountId}...`);
|
|
208
215
|
|
|
209
216
|
emitEvent(event)
|
|
@@ -18,7 +18,7 @@ import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
|
18
18
|
import { join } from "node:path";
|
|
19
19
|
import { after, describe, it } from "node:test";
|
|
20
20
|
import { fileURLToPath } from "node:url";
|
|
21
|
-
import {
|
|
21
|
+
import { buildSyncMailboxesMessage } from "@remit/backend/trigger-sync";
|
|
22
22
|
import type { AccountItem } from "@remit/data-ports";
|
|
23
23
|
import { ConnectionState } from "@remit/domain-enums";
|
|
24
24
|
import {
|
|
@@ -33,7 +33,6 @@ import {
|
|
|
33
33
|
} from "./with-oauth-lifecycle.js";
|
|
34
34
|
|
|
35
35
|
const QUEUE_NAME = "remit-mailboxes.fifo";
|
|
36
|
-
const QUEUE_URL = `http://localhost:9324/000000000000/${QUEUE_NAME}`;
|
|
37
36
|
const ACCOUNT_ID = "acct_imported";
|
|
38
37
|
const T0 = 1_700_000_000_000;
|
|
39
38
|
|
|
@@ -69,15 +68,12 @@ const importedAccount = {
|
|
|
69
68
|
} as unknown as AccountItem;
|
|
70
69
|
|
|
71
70
|
const enqueueSync = (store: QueueStore, now: number): void => {
|
|
72
|
-
const
|
|
73
|
-
queueUrl: QUEUE_URL,
|
|
74
|
-
accountId: ACCOUNT_ID,
|
|
75
|
-
});
|
|
71
|
+
const message = buildSyncMailboxesMessage({ accountId: ACCOUNT_ID });
|
|
76
72
|
store.sendMessage({
|
|
77
73
|
queueName: QUEUE_NAME,
|
|
78
|
-
body:
|
|
79
|
-
groupId:
|
|
80
|
-
deduplicationId:
|
|
74
|
+
body: JSON.stringify(message.body),
|
|
75
|
+
groupId: message.groupId,
|
|
76
|
+
deduplicationId: message.deduplicationId,
|
|
81
77
|
now,
|
|
82
78
|
});
|
|
83
79
|
};
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "@aws-sdk/client-sqs";
|
|
8
8
|
import { createLogger } from "@remit/logger";
|
|
9
9
|
import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
10
|
-
import {
|
|
10
|
+
import { createSqsClient } from "@remit/sqs-client/producer";
|
|
11
11
|
import type { Context, SQSBatchResponse, SQSEvent } from "aws-lambda";
|
|
12
12
|
import { env } from "expect-env";
|
|
13
13
|
import { handler } from "./index.js";
|
|
@@ -101,7 +101,7 @@ if (cluster.isPrimary) {
|
|
|
101
101
|
|
|
102
102
|
const maxMessages = 10; // SQS API limit
|
|
103
103
|
|
|
104
|
-
const sqs =
|
|
104
|
+
const sqs = createSqsClient({ queueUrl });
|
|
105
105
|
|
|
106
106
|
let isShuttingDown = false;
|
|
107
107
|
|
package/src/emit.test.ts
CHANGED
|
@@ -1,53 +1,42 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { afterEach,
|
|
3
|
-
import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
2
|
+
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
4
3
|
import {
|
|
5
4
|
_resetForTest,
|
|
6
5
|
type RemitClient,
|
|
7
6
|
setClient,
|
|
8
7
|
} from "@remit/backend/client";
|
|
8
|
+
import {
|
|
9
|
+
type RecordingQueue,
|
|
10
|
+
recordingQueue,
|
|
11
|
+
} from "@remit/data-ports/recording-queue";
|
|
9
12
|
import { MailboxLockRepo } from "@remit/drizzle-service";
|
|
10
13
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
11
|
-
import {
|
|
12
|
-
import type * as Emit from "./emit.js";
|
|
14
|
+
import { emitEvent, SYNC_MESSAGES_PENDING } from "./emit.js";
|
|
13
15
|
import type {
|
|
14
16
|
FlagPushEvent,
|
|
15
17
|
SyncMailboxesEvent,
|
|
16
|
-
SyncMessageBodyEvent,
|
|
17
18
|
SyncMessagesEvent,
|
|
18
19
|
} from "./events.js";
|
|
20
|
+
import { setWorkerQueues } from "./queues.js";
|
|
19
21
|
|
|
20
22
|
type Emitted<T> = Omit<T, "eventId" | "timestamp">;
|
|
21
23
|
|
|
22
|
-
// The queue urls are read once, at import. Point them at FIFO queues first —
|
|
23
|
-
// this suite is about what emitEvent puts on a FIFO queue, which is what both
|
|
24
|
-
// the self-host stack and the deployed stack run (deploy/vps/queues.json).
|
|
25
|
-
const fifo = (name: string) =>
|
|
26
|
-
`https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
|
|
27
|
-
|
|
28
|
-
const sqsMock = mockClient(SQSClient);
|
|
29
|
-
|
|
30
|
-
let emitEvent: typeof Emit.emitEvent;
|
|
31
|
-
let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
|
|
32
24
|
let locks: MailboxLockRepo;
|
|
33
25
|
let closeStore: () => void;
|
|
26
|
+
let queue: RecordingQueue;
|
|
34
27
|
|
|
35
|
-
const sentCommands = ()
|
|
36
|
-
sqsMock.commandCalls(SendMessageCommand).map((call) => call.args[0].input);
|
|
37
|
-
|
|
38
|
-
const bodyOf = (input: SendMessageCommand["input"]): Record<string, unknown> =>
|
|
39
|
-
JSON.parse(input.MessageBody ?? "{}") as Record<string, unknown>;
|
|
40
|
-
|
|
41
|
-
before(async () => {
|
|
42
|
-
process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
|
|
43
|
-
process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
|
|
44
|
-
process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
|
|
45
|
-
({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
|
|
46
|
-
});
|
|
28
|
+
const sentCommands = () => queue.sent;
|
|
47
29
|
|
|
48
30
|
beforeEach(() => {
|
|
49
|
-
|
|
50
|
-
|
|
31
|
+
queue = recordingQueue();
|
|
32
|
+
setWorkerQueues({
|
|
33
|
+
mailboxes: queue,
|
|
34
|
+
messages: queue,
|
|
35
|
+
flags: queue,
|
|
36
|
+
body: queue,
|
|
37
|
+
mailboxManagement: queue,
|
|
38
|
+
messageManagement: queue,
|
|
39
|
+
});
|
|
51
40
|
const store = createShippedSqliteDb();
|
|
52
41
|
closeStore = store.close;
|
|
53
42
|
locks = new MailboxLockRepo(store.db);
|
|
@@ -71,8 +60,8 @@ describe("emitEvent on a FIFO queue", () => {
|
|
|
71
60
|
|
|
72
61
|
const [sent] = sentCommands();
|
|
73
62
|
if (!sent) throw new Error("expected a send");
|
|
74
|
-
assert.equal(sent.
|
|
75
|
-
assert.equal(sent.
|
|
63
|
+
assert.equal(sent.groupId, "acc-1");
|
|
64
|
+
assert.equal(sent.deduplicationId, sent.body.eventId);
|
|
76
65
|
});
|
|
77
66
|
|
|
78
67
|
// Issue #37: every SYNC_MESSAGES for a mailbox carried one dedup id, so
|
|
@@ -94,16 +83,13 @@ describe("emitEvent on a FIFO queue", () => {
|
|
|
94
83
|
"acc-1",
|
|
95
84
|
"mbx-1",
|
|
96
85
|
SYNC_MESSAGES_PENDING,
|
|
97
|
-
String(
|
|
86
|
+
String(first.body.eventId),
|
|
98
87
|
);
|
|
99
88
|
await emitEvent(event);
|
|
100
89
|
|
|
101
90
|
const sent = sentCommands();
|
|
102
91
|
assert.equal(sent.length, 2);
|
|
103
|
-
assert.notEqual(
|
|
104
|
-
sent[0]?.MessageDeduplicationId,
|
|
105
|
-
sent[1]?.MessageDeduplicationId,
|
|
106
|
-
);
|
|
92
|
+
assert.notEqual(sent[0]?.deduplicationId, sent[1]?.deduplicationId);
|
|
107
93
|
});
|
|
108
94
|
|
|
109
95
|
it("lets a second sync of the same account's mailbox list through", async () => {
|
|
@@ -117,10 +103,7 @@ describe("emitEvent on a FIFO queue", () => {
|
|
|
117
103
|
|
|
118
104
|
const sent = sentCommands();
|
|
119
105
|
assert.equal(sent.length, 2);
|
|
120
|
-
assert.notEqual(
|
|
121
|
-
sent[0]?.MessageDeduplicationId,
|
|
122
|
-
sent[1]?.MessageDeduplicationId,
|
|
123
|
-
);
|
|
106
|
+
assert.notEqual(sent[0]?.deduplicationId, sent[1]?.deduplicationId);
|
|
124
107
|
});
|
|
125
108
|
|
|
126
109
|
// A FIFO queue without content-based deduplication rejects a send carrying
|
|
@@ -139,24 +122,6 @@ describe("emitEvent on a FIFO queue", () => {
|
|
|
139
122
|
|
|
140
123
|
const [sent] = sentCommands();
|
|
141
124
|
if (!sent) throw new Error("expected a send");
|
|
142
|
-
assert.equal(sent.
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
|
|
146
|
-
describe("emitEvent on a standard queue", () => {
|
|
147
|
-
it("carries no FIFO parameters", async () => {
|
|
148
|
-
const event: Emitted<SyncMessageBodyEvent> = {
|
|
149
|
-
type: "SYNC_MESSAGE_BODY",
|
|
150
|
-
accountId: "acc-1",
|
|
151
|
-
mailboxId: "mbx-1",
|
|
152
|
-
messageIds: ["msg-1"],
|
|
153
|
-
};
|
|
154
|
-
|
|
155
|
-
await emitEvent(event);
|
|
156
|
-
|
|
157
|
-
const [sent] = sentCommands();
|
|
158
|
-
if (!sent) throw new Error("expected a send");
|
|
159
|
-
assert.equal(sent.MessageGroupId, undefined);
|
|
160
|
-
assert.equal(sent.MessageDeduplicationId, undefined);
|
|
125
|
+
assert.equal(sent.deduplicationId, sent.body.eventId);
|
|
161
126
|
});
|
|
162
127
|
});
|
package/src/emit.ts
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
3
2
|
import { getClient } from "@remit/backend/client";
|
|
3
|
+
import type { FifoQueueProducer } from "@remit/data-ports";
|
|
4
4
|
import { recordImapFailure } from "@remit/logger-lambda/metrics";
|
|
5
|
-
import {
|
|
6
|
-
createQueueProducer,
|
|
7
|
-
isLocalEndpoint,
|
|
8
|
-
} from "@remit/sqs-client/producer";
|
|
9
|
-
import { env } from "expect-env";
|
|
10
5
|
import type { ImapEvent } from "./events.js";
|
|
6
|
+
import { type WorkerQueues, workerQueues } from "./queues.js";
|
|
11
7
|
|
|
12
8
|
type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
|
|
13
9
|
|
|
@@ -15,50 +11,43 @@ export const SYNC_MESSAGES_PENDING = "SYNC_MESSAGES_PENDING";
|
|
|
15
11
|
|
|
16
12
|
const SYNC_MESSAGES_PENDING_TTL_SECONDS = 300;
|
|
17
13
|
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
// dispatch is by `type`, not by which queue delivered the message.
|
|
54
|
-
FLAG_PUSH: flagsQueueUrl,
|
|
14
|
+
const queueFor = (
|
|
15
|
+
type: ImapEvent["type"],
|
|
16
|
+
queues: WorkerQueues,
|
|
17
|
+
): FifoQueueProducer<ImapEvent> => {
|
|
18
|
+
switch (type) {
|
|
19
|
+
case "SYNC_MAILBOXES":
|
|
20
|
+
return queues.mailboxes;
|
|
21
|
+
case "SYNC_MESSAGES":
|
|
22
|
+
return queues.messages;
|
|
23
|
+
// SYNC_MESSAGE_BODY routes to the single standard body queue (#612).
|
|
24
|
+
case "SYNC_MESSAGE_BODY":
|
|
25
|
+
return queues.body;
|
|
26
|
+
case "MAILBOX_CREATE":
|
|
27
|
+
case "MAILBOX_RENAME":
|
|
28
|
+
case "MAILBOX_DELETE":
|
|
29
|
+
return queues.mailboxManagement;
|
|
30
|
+
case "MESSAGE_DELETE":
|
|
31
|
+
case "MESSAGE_MOVE":
|
|
32
|
+
case "MESSAGE_COPY":
|
|
33
|
+
case "EMPTY_TRASH":
|
|
34
|
+
case "APPEND_SENT_MESSAGE":
|
|
35
|
+
// Rides messageMgmtQueue (issue #1271) rather than a dedicated queue — its
|
|
36
|
+
// payload carries only our message id (never a UID, unlike the legacy
|
|
37
|
+
// MESSAGE_MOVE event); per-event-type payload shape, same queue.
|
|
38
|
+
case "PLACEMENT_MOVE_PUSH":
|
|
39
|
+
return queues.messageManagement;
|
|
40
|
+
// Rides the existing flags queue (issue #1273) — this is imap-worker's OWN
|
|
41
|
+
// re-arm hint (the periodic per-mailbox sync tick catching up a marker
|
|
42
|
+
// stuck `pending`), distinct from `FlagPushService`'s user-facing hint
|
|
43
|
+
// sent from the API onto the sync queue. Both land on a queue this worker
|
|
44
|
+
// already consumes; dispatch is by `type`, not by which queue delivered
|
|
45
|
+
// the message.
|
|
46
|
+
case "FLAG_PUSH":
|
|
47
|
+
return queues.flags;
|
|
48
|
+
}
|
|
55
49
|
};
|
|
56
50
|
|
|
57
|
-
/**
|
|
58
|
-
* Check if queue URL is a FIFO queue (ends with .fifo)
|
|
59
|
-
*/
|
|
60
|
-
const isFifoQueue = (queueUrl: string): boolean => queueUrl.endsWith(".fifo");
|
|
61
|
-
|
|
62
51
|
export interface EmitEventOptions {
|
|
63
52
|
/**
|
|
64
53
|
* Delay delivery of the message by this many seconds (0-900).
|
|
@@ -77,33 +66,21 @@ export const emitEvent = async (
|
|
|
77
66
|
timestamp: Date.now(),
|
|
78
67
|
} as ImapEvent;
|
|
79
68
|
|
|
80
|
-
const
|
|
81
|
-
const useFifo = isFifoQueue(queueUrl);
|
|
82
|
-
|
|
83
|
-
// ElasticMQ FIFO queues don't support per-message DelaySeconds
|
|
84
|
-
const useDelay = options?.delaySeconds && !isLocal;
|
|
69
|
+
const queue = queueFor(event.type, workerQueues());
|
|
85
70
|
|
|
71
|
+
// The deduplication id is the event's own id, so the queue suppresses a
|
|
72
|
+
// re-send of one event (the retry SQS's own idempotency guard is for) and
|
|
73
|
+
// nothing else. A shared id per account or per mailbox instead made the
|
|
74
|
+
// 5-minute window a rate limiter: the second sync of a mailbox within five
|
|
75
|
+
// minutes was discarded before any worker saw it, so mail that arrived after
|
|
76
|
+
// a sync could not be fetched until the window elapsed (issue #37).
|
|
86
77
|
const send = () =>
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
// FIFO queue parameters — only set if the queue is FIFO. The
|
|
94
|
-
// deduplication id is the event's own id, so the queue suppresses a
|
|
95
|
-
// re-send of one event (the retry SQS's own idempotency guard is for)
|
|
96
|
-
// and nothing else. A shared id per account or per mailbox instead made
|
|
97
|
-
// the 5-minute window a rate limiter: the second sync of a mailbox
|
|
98
|
-
// within five minutes was discarded before any worker saw it, so mail
|
|
99
|
-
// that arrived after a sync could not be fetched until the window
|
|
100
|
-
// elapsed (issue #37).
|
|
101
|
-
...(useFifo && {
|
|
102
|
-
MessageGroupId: event.accountId,
|
|
103
|
-
MessageDeduplicationId: fullEvent.eventId,
|
|
104
|
-
}),
|
|
105
|
-
}),
|
|
106
|
-
);
|
|
78
|
+
queue.send({
|
|
79
|
+
body: fullEvent,
|
|
80
|
+
groupId: event.accountId,
|
|
81
|
+
deduplicationId: fullEvent.eventId,
|
|
82
|
+
...(options?.delaySeconds && { delaySeconds: options.delaySeconds }),
|
|
83
|
+
});
|
|
107
84
|
if (fullEvent.type !== "SYNC_MESSAGES") {
|
|
108
85
|
await send();
|
|
109
86
|
return;
|
package/src/events.ts
CHANGED
|
@@ -232,26 +232,8 @@ export interface DeleteAccountObjectsEvent {
|
|
|
232
232
|
continuationToken?: string;
|
|
233
233
|
}
|
|
234
234
|
|
|
235
|
-
/**
|
|
236
|
-
* Sent by the account-deletion fanout worker once per accountId under a
|
|
237
|
-
* deleted AccountConfig. The actual stop semantics already happen via the
|
|
238
|
-
* account-tombstone fence (`isActive=false` + `deletedAt`) flipped by the
|
|
239
|
-
* deletion API: any in-flight or future event for the account is dropped
|
|
240
|
-
* by `isAccountDeleted`. This event exists as an explicit signal in the
|
|
241
|
-
* cascade contract — when the imap-worker grows real per-account drain or
|
|
242
|
-
* connection-teardown semantics it hangs off this hook.
|
|
243
|
-
*/
|
|
244
|
-
export interface ImapWorkerStopEvent {
|
|
245
|
-
type: "IMAP_WORKER_STOP";
|
|
246
|
-
accountConfigId: string;
|
|
247
|
-
accountId: string;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
235
|
/** Union of all event types the worker can process (including non-IMAP ones). */
|
|
251
|
-
export type WorkerEvent =
|
|
252
|
-
| ImapEvent
|
|
253
|
-
| DeleteAccountObjectsEvent
|
|
254
|
-
| ImapWorkerStopEvent;
|
|
236
|
+
export type WorkerEvent = ImapEvent | DeleteAccountObjectsEvent;
|
|
255
237
|
|
|
256
238
|
export type MessageManagementEvent =
|
|
257
239
|
| MessageDeleteEvent
|
|
@@ -1,18 +1,22 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { afterEach, describe, it } from "node:test";
|
|
2
|
+
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
3
3
|
import {
|
|
4
4
|
DeleteObjectsCommand,
|
|
5
5
|
ListObjectsV2Command,
|
|
6
6
|
S3Client,
|
|
7
7
|
} from "@aws-sdk/client-s3";
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
type RecordingQueue,
|
|
10
|
+
recordingQueue,
|
|
11
|
+
} from "@remit/data-ports/recording-queue";
|
|
9
12
|
import { noopLogger } from "@remit/logger";
|
|
10
13
|
import { mockClient } from "aws-sdk-client-mock";
|
|
11
14
|
import type { DeleteAccountObjectsEvent } from "../events.js";
|
|
15
|
+
import { setWorkerQueues } from "../queues.js";
|
|
12
16
|
import { handleDeleteAccountObjects } from "./delete-account-objects.js";
|
|
13
17
|
|
|
14
18
|
const s3Mock = mockClient(S3Client);
|
|
15
|
-
|
|
19
|
+
let queue: RecordingQueue;
|
|
16
20
|
|
|
17
21
|
const event: DeleteAccountObjectsEvent = {
|
|
18
22
|
type: "DELETE_ACCOUNT_OBJECTS",
|
|
@@ -20,9 +24,20 @@ const event: DeleteAccountObjectsEvent = {
|
|
|
20
24
|
};
|
|
21
25
|
|
|
22
26
|
describe("handleDeleteAccountObjects", () => {
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
queue = recordingQueue();
|
|
29
|
+
setWorkerQueues({
|
|
30
|
+
mailboxes: queue,
|
|
31
|
+
messages: queue,
|
|
32
|
+
flags: queue,
|
|
33
|
+
body: queue,
|
|
34
|
+
mailboxManagement: queue,
|
|
35
|
+
messageManagement: queue,
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
|
|
23
39
|
afterEach(() => {
|
|
24
40
|
s3Mock.reset();
|
|
25
|
-
sqsMock.reset();
|
|
26
41
|
});
|
|
27
42
|
|
|
28
43
|
it("lists and deletes every object under the account prefix in one page", async () => {
|
|
@@ -45,7 +60,7 @@ describe("handleDeleteAccountObjects", () => {
|
|
|
45
60
|
{ Key: "accounts/cfg-1/a.eml" },
|
|
46
61
|
{ Key: "accounts/cfg-1/b.eml" },
|
|
47
62
|
]);
|
|
48
|
-
assert.equal(
|
|
63
|
+
assert.equal(queue.sent.length, 0);
|
|
49
64
|
});
|
|
50
65
|
|
|
51
66
|
it("follows the continuation token across truncated pages", async () => {
|
|
@@ -80,7 +95,6 @@ describe("handleDeleteAccountObjects", () => {
|
|
|
80
95
|
|
|
81
96
|
it("re-enqueues with the continuation token when time is nearly up", async () => {
|
|
82
97
|
s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
|
|
83
|
-
sqsMock.on(SendMessageCommand).resolves({});
|
|
84
98
|
|
|
85
99
|
await handleDeleteAccountObjects(
|
|
86
100
|
{ ...event, continuationToken: "tok-mid" },
|
|
@@ -93,13 +107,10 @@ describe("handleDeleteAccountObjects", () => {
|
|
|
93
107
|
0,
|
|
94
108
|
"bails before starting a new page",
|
|
95
109
|
);
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
)
|
|
100
|
-
assert.equal(body.type, "DELETE_ACCOUNT_OBJECTS");
|
|
101
|
-
assert.equal(body.accountConfigId, "cfg-1");
|
|
102
|
-
assert.equal(body.continuationToken, "tok-mid");
|
|
110
|
+
const body = queue.sent[0]?.body;
|
|
111
|
+
assert.equal(body?.type, "DELETE_ACCOUNT_OBJECTS");
|
|
112
|
+
assert.equal(body?.accountConfigId, "cfg-1");
|
|
113
|
+
assert.equal(body?.continuationToken, "tok-mid");
|
|
103
114
|
});
|
|
104
115
|
|
|
105
116
|
it("keeps paging while time remains", async () => {
|
|
@@ -112,6 +123,6 @@ describe("handleDeleteAccountObjects", () => {
|
|
|
112
123
|
await handleDeleteAccountObjects(event, noopLogger, () => 120_000);
|
|
113
124
|
|
|
114
125
|
assert.equal(s3Mock.commandCalls(ListObjectsV2Command).length, 1);
|
|
115
|
-
assert.equal(
|
|
126
|
+
assert.equal(queue.sent.length, 0);
|
|
116
127
|
});
|
|
117
128
|
});
|
|
@@ -3,17 +3,13 @@ import {
|
|
|
3
3
|
ListObjectsV2Command,
|
|
4
4
|
S3Client,
|
|
5
5
|
} from "@aws-sdk/client-s3";
|
|
6
|
-
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
7
6
|
import type { Logger } from "@remit/logger";
|
|
8
|
-
import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
9
7
|
import { env } from "expect-env";
|
|
10
8
|
import type { DeleteAccountObjectsEvent } from "../events.js";
|
|
9
|
+
import { workerQueues } from "../queues.js";
|
|
11
10
|
|
|
12
11
|
const s3 = new S3Client({});
|
|
13
12
|
|
|
14
|
-
const sqsQueueUrl = env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
15
|
-
const sqs = createQueueProducer({ queueUrl: sqsQueueUrl });
|
|
16
|
-
|
|
17
13
|
const BATCH_SIZE = 1_000;
|
|
18
14
|
const MIN_REMAINING_MS = 30_000;
|
|
19
15
|
|
|
@@ -94,10 +90,5 @@ const reenqueue = async (
|
|
|
94
90
|
continuationToken,
|
|
95
91
|
};
|
|
96
92
|
|
|
97
|
-
await
|
|
98
|
-
new SendMessageCommand({
|
|
99
|
-
QueueUrl: sqsQueueUrl,
|
|
100
|
-
MessageBody: JSON.stringify(event),
|
|
101
|
-
}),
|
|
102
|
-
);
|
|
93
|
+
await workerQueues().messageManagement.send({ body: event });
|
|
103
94
|
};
|
|
@@ -52,7 +52,7 @@ const arrange = async (): Promise<Arranged> => {
|
|
|
52
52
|
addressService: w.client.address,
|
|
53
53
|
mailboxSpecialUseService: w.client.mailboxSpecialUse,
|
|
54
54
|
mailboxService: w.client.mailbox,
|
|
55
|
-
|
|
55
|
+
queue: w.client.queues.messageManagement,
|
|
56
56
|
});
|
|
57
57
|
return { w, inbox, junk, message, placement };
|
|
58
58
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { afterEach, beforeEach, describe, it, mock } from "node:test";
|
|
3
|
-
import { SQSClient } from "@aws-sdk/client-sqs";
|
|
4
3
|
import type { MailboxItem } from "@remit/data-ports";
|
|
4
|
+
import { recordingQueue } from "@remit/data-ports/recording-queue";
|
|
5
5
|
import {
|
|
6
6
|
AccountAuthType,
|
|
7
7
|
ConnectionState,
|
|
@@ -171,15 +171,13 @@ describe("the Message-ID the move, delete and copy probes ask with", () => {
|
|
|
171
171
|
it("reads the source's Message-ID for a copy", async () => {
|
|
172
172
|
const sourceId = await syncInbox("<report@example.com>");
|
|
173
173
|
const archive = await createMailbox("Archive");
|
|
174
|
-
mock.method(SQSClient.prototype, "send", async () => ({}));
|
|
175
|
-
|
|
176
174
|
const copyId = await new MessageMoveService({
|
|
177
175
|
messageService: messages,
|
|
178
176
|
mailboxService: mailboxes,
|
|
179
177
|
mailboxSpecialUseService: repos.mailboxSpecialUse,
|
|
180
178
|
threadMessageService: threadMessages,
|
|
181
179
|
addressService: repos.address,
|
|
182
|
-
|
|
180
|
+
queue: recordingQueue(),
|
|
183
181
|
}).copyMessage(ACCOUNT_CONFIG_ID, sourceId, archive.mailboxId, accountId);
|
|
184
182
|
|
|
185
183
|
assert.notEqual(copyId, sourceId);
|
|
@@ -22,7 +22,6 @@ import {
|
|
|
22
22
|
MailConnectionError,
|
|
23
23
|
type MailCredentials,
|
|
24
24
|
} from "@remit/mailbox-service";
|
|
25
|
-
import { env } from "expect-env";
|
|
26
25
|
import pMap from "p-map";
|
|
27
26
|
import {
|
|
28
27
|
isAccountDeleted,
|
|
@@ -31,6 +30,7 @@ import {
|
|
|
31
30
|
} from "../account-check.js";
|
|
32
31
|
import { emitEvent } from "../emit.js";
|
|
33
32
|
import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
|
|
33
|
+
import { workerQueues } from "../queues.js";
|
|
34
34
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
35
35
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
36
36
|
import { createImportedFolder } from "./imported-folder.js";
|
|
@@ -118,7 +118,7 @@ export const syncMailboxes = async (
|
|
|
118
118
|
mailboxService,
|
|
119
119
|
new MailboxQueueService({
|
|
120
120
|
mailboxService,
|
|
121
|
-
|
|
121
|
+
queue: workerQueues().mailboxManagement,
|
|
122
122
|
logger: log,
|
|
123
123
|
}),
|
|
124
124
|
),
|
|
@@ -142,6 +142,7 @@ export const syncMessageBody = async (
|
|
|
142
142
|
flagQueue: flagQueueService,
|
|
143
143
|
calendarSuggestion: calendarSuggestionService,
|
|
144
144
|
calendarStore,
|
|
145
|
+
queues,
|
|
145
146
|
} = await getClient();
|
|
146
147
|
|
|
147
148
|
const account = await accountService.get(accountId);
|
|
@@ -156,18 +157,15 @@ export const syncMessageBody = async (
|
|
|
156
157
|
// PLACEMENT_MOVE_PUSH rides messageMgmtQueue (issue #1271) — same queue
|
|
157
158
|
// the general MessageMoveService move/delete/copy machinery uses, not a
|
|
158
159
|
// dedicated one.
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
sqsQueueUrl: placementMoveQueueUrl,
|
|
169
|
-
})
|
|
170
|
-
: undefined;
|
|
160
|
+
const placementMoveService = new PlacementMoveService({
|
|
161
|
+
messageService,
|
|
162
|
+
threadMessageService,
|
|
163
|
+
markerService,
|
|
164
|
+
addressService,
|
|
165
|
+
mailboxSpecialUseService,
|
|
166
|
+
mailboxService,
|
|
167
|
+
queue: queues.messageManagement,
|
|
168
|
+
});
|
|
171
169
|
|
|
172
170
|
await withOAuthLifecycle(
|
|
173
171
|
buildLifecycleDeps(secrets, accountService),
|
|
@@ -220,12 +218,10 @@ export const syncMessageBody = async (
|
|
|
220
218
|
// A confident, actionable placement verdict moves mail directly on
|
|
221
219
|
// body-sync. Safety lives in the verdict itself (only confident,
|
|
222
220
|
// INBOX/Junk-only) and the movedByRemit loop guard.
|
|
223
|
-
const placementConfig =
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
}
|
|
228
|
-
: undefined;
|
|
221
|
+
const placementConfig = {
|
|
222
|
+
mailboxSpecialUseService,
|
|
223
|
+
placementMoveService,
|
|
224
|
+
};
|
|
229
225
|
|
|
230
226
|
// A matched filter's actions (label upsert, exclusive move) apply on
|
|
231
227
|
// the same body-sync pass, reusing the placement mover for the move
|