@remit/imap-worker 0.0.89 → 0.0.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +3 -9
- package/src/emit.test.ts +28 -10
- package/src/emit.ts +70 -72
- package/src/handlers/delete-account-objects.test.ts +11 -13
- package/src/handlers/delete-account-objects.ts +5 -3
- package/src/handlers/message-copy.sqlite.test.ts +3 -3
- package/src/handlers/message-copy.ts +5 -2
- package/src/handlers/message-delete.sqlite.test.ts +1 -1
- package/src/handlers/message-delete.ts +7 -4
- package/src/handlers/message-move.sqlite.test.ts +1 -1
- package/src/handlers/message-move.ts +9 -6
- package/src/handlers/placement-move-push.ts +9 -6
- package/src/handlers/placement-push.sqlite.test.ts +2 -1
- package/src/handlers/probe-message-id.sqlite.test.ts +1 -1
- package/src/handlers/sync-mailboxes.ts +8 -4
- package/src/handlers/sync-message-body.sqlite.test.ts +2 -2
- package/src/handlers/sync-message-body.ts +2 -1
- package/src/handlers/sync-messages.sqlite.test.ts +14 -2
- package/src/handlers/sync-messages.ts +20 -8
- package/src/index.ts +3 -7
- package/src/mail-sync-selection.sqlite.test.ts +4 -11
- package/src/pending-sync.sqlite.test.ts +9 -10
- package/src/processor.ts +10 -8
- package/src/queues.ts +14 -23
- package/src/scheduler/attachment-sweep.sqlite.test.ts +2 -3
- package/src/scheduler/handler.ts +31 -26
- package/src/scheduler/run-tick.sqlite.test.ts +1 -1
- package/src/scheduler/runner.ts +5 -10
- package/src/stranded-placement.sqlite.test.ts +5 -0
- package/src/test-helpers/test-world.ts +30 -32
- package/src/test-helpers/worker-queues.ts +11 -0
|
@@ -177,7 +177,7 @@ describe("the Message-ID the move, delete and copy probes ask with", () => {
|
|
|
177
177
|
mailboxSpecialUseService: repos.mailboxSpecialUse,
|
|
178
178
|
threadMessageService: threadMessages,
|
|
179
179
|
addressService: repos.address,
|
|
180
|
-
queue: recordingQueue(),
|
|
180
|
+
queue: recordingQueue().fifo,
|
|
181
181
|
}).copyMessage(ACCOUNT_CONFIG_ID, sourceId, archive.mailboxId, accountId);
|
|
182
182
|
|
|
183
183
|
assert.notEqual(copyId, sourceId);
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
MailboxSyncService,
|
|
22
22
|
MailConnectionError,
|
|
23
23
|
type MailCredentials,
|
|
24
|
+
standardMailboxManagement,
|
|
24
25
|
} from "@remit/mailbox-service";
|
|
25
26
|
import pMap from "p-map";
|
|
26
27
|
import {
|
|
@@ -28,9 +29,9 @@ import {
|
|
|
28
29
|
isMailSyncDisabled,
|
|
29
30
|
isUnsyncableHost,
|
|
30
31
|
} from "../account-check.js";
|
|
31
|
-
import {
|
|
32
|
+
import { createEmitEvent, type EmitEvent } from "../emit.js";
|
|
32
33
|
import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
|
|
33
|
-
import {
|
|
34
|
+
import type { WorkerQueues } from "../queues.js";
|
|
34
35
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
35
36
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
36
37
|
import { createImportedFolder } from "./imported-folder.js";
|
|
@@ -95,6 +96,7 @@ export const mailboxNeedsSync = (
|
|
|
95
96
|
export const syncMailboxes = async (
|
|
96
97
|
event: SyncMailboxesEvent,
|
|
97
98
|
log: Logger,
|
|
99
|
+
queues: WorkerQueues,
|
|
98
100
|
): Promise<void> => {
|
|
99
101
|
const client = await getClient();
|
|
100
102
|
const {
|
|
@@ -118,7 +120,7 @@ export const syncMailboxes = async (
|
|
|
118
120
|
mailboxService,
|
|
119
121
|
new MailboxQueueService({
|
|
120
122
|
mailboxService,
|
|
121
|
-
|
|
123
|
+
enqueue: standardMailboxManagement(queues.mailboxManagement),
|
|
122
124
|
logger: log,
|
|
123
125
|
}),
|
|
124
126
|
),
|
|
@@ -166,6 +168,7 @@ export const syncMailboxes = async (
|
|
|
166
168
|
accountService,
|
|
167
169
|
binder,
|
|
168
170
|
log,
|
|
171
|
+
createEmitEvent(queues),
|
|
169
172
|
);
|
|
170
173
|
} catch (err) {
|
|
171
174
|
// Auth failures are handled by the wrapper — rethrow untouched so it
|
|
@@ -199,6 +202,7 @@ const syncMailboxesForAccount = async (
|
|
|
199
202
|
accountService: IAccountRepository,
|
|
200
203
|
binder: ConfigBinderDeps,
|
|
201
204
|
log: Logger,
|
|
205
|
+
emit: EmitEvent,
|
|
202
206
|
): Promise<void> => {
|
|
203
207
|
const { accountId } = account;
|
|
204
208
|
|
|
@@ -330,7 +334,7 @@ const syncMailboxesForAccount = async (
|
|
|
330
334
|
mailboxCountSynced: skipped,
|
|
331
335
|
});
|
|
332
336
|
|
|
333
|
-
await emitSyncMessagesEvents(accountId, mailboxes,
|
|
337
|
+
await emitSyncMessagesEvents(accountId, mailboxes, emit);
|
|
334
338
|
};
|
|
335
339
|
|
|
336
340
|
type SyncMessagesInput = Omit<SyncMessagesEvent, "eventId" | "timestamp">;
|
|
@@ -73,7 +73,7 @@ const arrange = async (): Promise<Arranged> => {
|
|
|
73
73
|
};
|
|
74
74
|
|
|
75
75
|
const run = (a: Arranged, receiveCount = 1) =>
|
|
76
|
-
syncMessageBody(a.event, a.w.log, receiveCount, a.w.scope);
|
|
76
|
+
syncMessageBody(a.event, a.w.log, a.w.queues, receiveCount, a.w.scope);
|
|
77
77
|
|
|
78
78
|
const bodyFetches = (a: Arranged): string[] => a.w.imap.commands("BODY");
|
|
79
79
|
|
|
@@ -160,7 +160,7 @@ const batchOf = (
|
|
|
160
160
|
});
|
|
161
161
|
|
|
162
162
|
const runBatch = (w: World, event: SyncMessageBodyEvent, receiveCount = 1) =>
|
|
163
|
-
syncMessageBody(event, w.log, receiveCount, w.scope);
|
|
163
|
+
syncMessageBody(event, w.log, w.queues, receiveCount, w.scope);
|
|
164
164
|
|
|
165
165
|
const arrangeFiling = async () => {
|
|
166
166
|
const w = await store.world();
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
createConnectionScopeWithCredentials,
|
|
26
26
|
} from "../connection-scope.js";
|
|
27
27
|
import type { SyncMessageBodyEvent } from "../events.js";
|
|
28
|
+
import type { WorkerQueues } from "../queues.js";
|
|
28
29
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
29
30
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
30
31
|
import { workerVersion } from "../worker-version.js";
|
|
@@ -87,6 +88,7 @@ export const buildRetryableFailureError = (
|
|
|
87
88
|
export const syncMessageBody = async (
|
|
88
89
|
event: SyncMessageBodyEvent,
|
|
89
90
|
log: Logger,
|
|
91
|
+
queues: WorkerQueues,
|
|
90
92
|
receiveCount = 1,
|
|
91
93
|
createConnectionScope = createConnectionScopeWithCredentials,
|
|
92
94
|
): Promise<void> => {
|
|
@@ -142,7 +144,6 @@ export const syncMessageBody = async (
|
|
|
142
144
|
flagQueue: flagQueueService,
|
|
143
145
|
calendarSuggestion: calendarSuggestionService,
|
|
144
146
|
calendarStore,
|
|
145
|
-
queues,
|
|
146
147
|
} = await getClient();
|
|
147
148
|
|
|
148
149
|
const account = await accountService.get(accountId);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { after, before, describe, it } from "node:test";
|
|
3
|
+
import { createEmitEvent } from "../emit.js";
|
|
3
4
|
import type { SyncMessagesEvent } from "../events.js";
|
|
4
5
|
import { openStore, type World } from "../test-helpers/test-world.js";
|
|
5
6
|
import { drainPendingFlagPushes, syncMessages } from "./sync-messages.js";
|
|
@@ -21,7 +22,12 @@ const syncEvent = (w: World, mailboxId: string): SyncMessagesEvent => ({
|
|
|
21
22
|
});
|
|
22
23
|
|
|
23
24
|
const sync = async (w: World, mailboxId: string): Promise<number> => {
|
|
24
|
-
await syncMessages(
|
|
25
|
+
await syncMessages(
|
|
26
|
+
syncEvent(w, mailboxId),
|
|
27
|
+
w.log,
|
|
28
|
+
w.queues,
|
|
29
|
+
w.connectionFactory,
|
|
30
|
+
);
|
|
25
31
|
return w.imap.connections;
|
|
26
32
|
};
|
|
27
33
|
|
|
@@ -106,7 +112,12 @@ describe("a folder deleted while its sync is in flight", () => {
|
|
|
106
112
|
return { getConnection: () => connection, close: factory.close };
|
|
107
113
|
};
|
|
108
114
|
|
|
109
|
-
await syncMessages(
|
|
115
|
+
await syncMessages(
|
|
116
|
+
syncEvent(w, old.mailboxId),
|
|
117
|
+
w.log,
|
|
118
|
+
w.queues,
|
|
119
|
+
connectionFactory,
|
|
120
|
+
);
|
|
110
121
|
|
|
111
122
|
const account = await w.client.account.get(w.account.accountId);
|
|
112
123
|
assert.notEqual(account.syncPhase, "error");
|
|
@@ -142,6 +153,7 @@ describe("drainPendingFlagPushes (issue #1273)", () => {
|
|
|
142
153
|
w.account,
|
|
143
154
|
inbox.mailboxId,
|
|
144
155
|
w.log,
|
|
156
|
+
createEmitEvent(w.queues),
|
|
145
157
|
);
|
|
146
158
|
|
|
147
159
|
assert.deepEqual(
|
|
@@ -35,12 +35,17 @@ import {
|
|
|
35
35
|
isMailSyncDisabled,
|
|
36
36
|
isUnsyncableHost,
|
|
37
37
|
} from "../account-check.js";
|
|
38
|
-
import {
|
|
38
|
+
import {
|
|
39
|
+
createEmitEvent,
|
|
40
|
+
type EmitEvent,
|
|
41
|
+
SYNC_MESSAGES_PENDING,
|
|
42
|
+
} from "../emit.js";
|
|
39
43
|
import type {
|
|
40
44
|
FlagPushEvent,
|
|
41
45
|
SyncMessageBodyEvent,
|
|
42
46
|
SyncMessagesEvent,
|
|
43
47
|
} from "../events.js";
|
|
48
|
+
import type { WorkerQueues } from "../queues.js";
|
|
44
49
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
45
50
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
46
51
|
import { workerVersion } from "../worker-version.js";
|
|
@@ -77,6 +82,7 @@ export const managedConnectionFor = (
|
|
|
77
82
|
export const syncMessages = async (
|
|
78
83
|
event: SyncMessagesEvent,
|
|
79
84
|
log: Logger,
|
|
85
|
+
queues: WorkerQueues,
|
|
80
86
|
createConnectionFactory = managedConnectionFor,
|
|
81
87
|
): Promise<void> => {
|
|
82
88
|
log.info(
|
|
@@ -212,6 +218,7 @@ export const syncMessages = async (
|
|
|
212
218
|
syncWarningRepository,
|
|
213
219
|
flagPushMarkerService,
|
|
214
220
|
messageFlagService,
|
|
221
|
+
emit: createEmitEvent(queues),
|
|
215
222
|
},
|
|
216
223
|
log,
|
|
217
224
|
);
|
|
@@ -290,6 +297,7 @@ interface SyncDeps {
|
|
|
290
297
|
syncWarningRepository: ISyncWarningRepository;
|
|
291
298
|
flagPushMarkerService: IMessageFlagPushRepository;
|
|
292
299
|
messageFlagService: IMessageFlagRepository;
|
|
300
|
+
emit: EmitEvent;
|
|
293
301
|
}
|
|
294
302
|
|
|
295
303
|
// Bounds concurrent SQS sends while re-arming stuck flag-push markers —
|
|
@@ -314,16 +322,13 @@ const FLAG_PUSH_DRAIN_CONCURRENCY = 5;
|
|
|
314
322
|
* loudly — it must never fail the surrounding SYNC_MESSAGES batch, which is
|
|
315
323
|
* unrelated message-header sync work. The marker stays durable regardless;
|
|
316
324
|
* the next periodic tick tries again.
|
|
317
|
-
*
|
|
318
|
-
* `emit` defaults to the real `emitEvent` (imap-worker's shared SQS
|
|
319
|
-
* producer) and is only ever overridden in tests.
|
|
320
325
|
*/
|
|
321
326
|
export const drainPendingFlagPushes = async (
|
|
322
327
|
flagPushMarkerService: IMessageFlagPushRepository,
|
|
323
328
|
account: AccountItem,
|
|
324
329
|
mailboxId: string,
|
|
325
330
|
log: Logger,
|
|
326
|
-
emit:
|
|
331
|
+
emit: EmitEvent,
|
|
327
332
|
): Promise<void> => {
|
|
328
333
|
const markers = await flagPushMarkerService.listByMailboxId(mailboxId);
|
|
329
334
|
const stuck = markers.filter((marker) => marker.state === "pending");
|
|
@@ -383,6 +388,7 @@ const syncMailboxMessages = async (
|
|
|
383
388
|
syncWarningRepository,
|
|
384
389
|
flagPushMarkerService,
|
|
385
390
|
messageFlagService,
|
|
391
|
+
emit,
|
|
386
392
|
} = deps;
|
|
387
393
|
|
|
388
394
|
// Get the mailbox - it must exist (should have been created by mailbox sync)
|
|
@@ -392,7 +398,13 @@ const syncMailboxMessages = async (
|
|
|
392
398
|
// Periodic drain (issue #1273): independent of the IMAP sync below — no
|
|
393
399
|
// connection needed, just an SQS re-arm — so it runs regardless of this
|
|
394
400
|
// round's sync outcome.
|
|
395
|
-
await drainPendingFlagPushes(
|
|
401
|
+
await drainPendingFlagPushes(
|
|
402
|
+
flagPushMarkerService,
|
|
403
|
+
account,
|
|
404
|
+
mailboxId,
|
|
405
|
+
log,
|
|
406
|
+
emit,
|
|
407
|
+
);
|
|
396
408
|
const isInbox = mailbox.fullPath.toUpperCase() === "INBOX";
|
|
397
409
|
|
|
398
410
|
const syncService = new MessageSyncService(
|
|
@@ -466,7 +478,7 @@ const syncMailboxMessages = async (
|
|
|
466
478
|
messageIds: batch.map((m) => m.messageId),
|
|
467
479
|
messages: batch.map((m) => ({ messageId: m.messageId, uid: m.uid })),
|
|
468
480
|
};
|
|
469
|
-
return
|
|
481
|
+
return emit(bodyEvent);
|
|
470
482
|
},
|
|
471
483
|
{ concurrency: EVENT_EMIT_CONCURRENCY },
|
|
472
484
|
);
|
|
@@ -484,7 +496,7 @@ const syncMailboxMessages = async (
|
|
|
484
496
|
accountId: event.accountId,
|
|
485
497
|
mailboxId,
|
|
486
498
|
};
|
|
487
|
-
await
|
|
499
|
+
await emit(nextSyncEvent);
|
|
488
500
|
return;
|
|
489
501
|
}
|
|
490
502
|
|
package/src/index.ts
CHANGED
|
@@ -9,14 +9,10 @@ import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
|
|
|
9
9
|
import type { WorkerEvent } from "./events.js";
|
|
10
10
|
import { imapFailureKind } from "./failure-kind.js";
|
|
11
11
|
import { processEvent } from "./processor.js";
|
|
12
|
-
import {
|
|
13
|
-
setWorkerQueues,
|
|
14
|
-
sqsWorkerQueues,
|
|
15
|
-
workerQueueUrlsFromEnv,
|
|
16
|
-
} from "./queues.js";
|
|
12
|
+
import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "./queues.js";
|
|
17
13
|
|
|
18
14
|
installNodeLogger();
|
|
19
|
-
|
|
15
|
+
const queues = sqsWorkerQueues(workerQueueUrlsFromEnv());
|
|
20
16
|
|
|
21
17
|
const log = createLogger();
|
|
22
18
|
|
|
@@ -51,7 +47,7 @@ export const handler: SQSHandler = withTelemetry(
|
|
|
51
47
|
|
|
52
48
|
const queue = queueNameFromEventSource(record.eventSourceARN);
|
|
53
49
|
const opStart = Date.now();
|
|
54
|
-
const failed = await processEvent(imapEvent, log, receiveCount)
|
|
50
|
+
const failed = await processEvent(imapEvent, log, queues, receiveCount)
|
|
55
51
|
.then(() => {
|
|
56
52
|
recordQueueEvent({
|
|
57
53
|
queue,
|
|
@@ -25,8 +25,8 @@ import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
|
25
25
|
import type { Logger } from "@remit/logger";
|
|
26
26
|
import { syncMailboxes } from "./handlers/sync-mailboxes.js";
|
|
27
27
|
import { syncMessages } from "./handlers/sync-messages.js";
|
|
28
|
-
import { setWorkerQueues } from "./queues.js";
|
|
29
28
|
import { runSchedulerTick } from "./scheduler/run-tick.js";
|
|
29
|
+
import { workerQueuesOn } from "./test-helpers/worker-queues.js";
|
|
30
30
|
|
|
31
31
|
const NOW = 1_700_000_000_000;
|
|
32
32
|
|
|
@@ -70,15 +70,6 @@ beforeEach(() => {
|
|
|
70
70
|
mailboxLock: locks,
|
|
71
71
|
} as unknown as RemitClient;
|
|
72
72
|
setClient(client);
|
|
73
|
-
const queue = recordingQueue();
|
|
74
|
-
setWorkerQueues({
|
|
75
|
-
mailboxes: queue,
|
|
76
|
-
messages: queue,
|
|
77
|
-
flags: queue,
|
|
78
|
-
body: queue,
|
|
79
|
-
mailboxManagement: queue,
|
|
80
|
-
messageManagement: queue,
|
|
81
|
-
});
|
|
82
73
|
});
|
|
83
74
|
|
|
84
75
|
afterEach(() => {
|
|
@@ -144,7 +135,7 @@ const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
|
|
|
144
135
|
const result = await runSchedulerTick({
|
|
145
136
|
redriveStrandedPlacements: async () => 0,
|
|
146
137
|
accountService: accounts,
|
|
147
|
-
queue,
|
|
138
|
+
queue: queue.fifo,
|
|
148
139
|
log: silentLogger,
|
|
149
140
|
tickIntervalMs: 60_000,
|
|
150
141
|
offlineIntervalMs: 0,
|
|
@@ -179,6 +170,7 @@ describe("turning mail sync off and on again", () => {
|
|
|
179
170
|
accountId: account.accountId,
|
|
180
171
|
},
|
|
181
172
|
silentLogger,
|
|
173
|
+
workerQueuesOn(recordingQueue()),
|
|
182
174
|
);
|
|
183
175
|
|
|
184
176
|
let connections = 0;
|
|
@@ -191,6 +183,7 @@ describe("turning mail sync off and on again", () => {
|
|
|
191
183
|
mailboxId,
|
|
192
184
|
},
|
|
193
185
|
silentLogger,
|
|
186
|
+
workerQueuesOn(recordingQueue()),
|
|
194
187
|
() => {
|
|
195
188
|
connections += 1;
|
|
196
189
|
throw new Error("a sync that is off must not connect");
|
|
@@ -11,11 +11,15 @@ import {
|
|
|
11
11
|
} from "@remit/data-ports/recording-queue";
|
|
12
12
|
import { MailboxLockRepo } from "@remit/drizzle-service";
|
|
13
13
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
createEmitEvent,
|
|
16
|
+
type EmitEvent,
|
|
17
|
+
SYNC_MESSAGES_PENDING,
|
|
18
|
+
} from "./emit.js";
|
|
15
19
|
import type { SyncMessagesEvent } from "./events.js";
|
|
16
20
|
import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
|
|
17
21
|
import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
|
|
18
|
-
import {
|
|
22
|
+
import { workerQueuesOn } from "./test-helpers/worker-queues.js";
|
|
19
23
|
|
|
20
24
|
const ACCOUNT = "account-1366";
|
|
21
25
|
const FOLDERS = [
|
|
@@ -42,19 +46,14 @@ const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
|
|
|
42
46
|
mailboxId: "mbx-inbox",
|
|
43
47
|
});
|
|
44
48
|
|
|
49
|
+
const emitEvent: EmitEvent = (event) =>
|
|
50
|
+
createEmitEvent(workerQueuesOn(queue))(event);
|
|
51
|
+
|
|
45
52
|
const fanOut = (): Promise<void> =>
|
|
46
53
|
emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
|
|
47
54
|
|
|
48
55
|
beforeEach(() => {
|
|
49
56
|
queue = recordingQueue();
|
|
50
|
-
setWorkerQueues({
|
|
51
|
-
mailboxes: queue,
|
|
52
|
-
messages: queue,
|
|
53
|
-
flags: queue,
|
|
54
|
-
body: queue,
|
|
55
|
-
mailboxManagement: queue,
|
|
56
|
-
messageManagement: queue,
|
|
57
|
-
});
|
|
58
57
|
const store = createShippedSqliteDb();
|
|
59
58
|
closeStore = store.close;
|
|
60
59
|
locks = new MailboxLockRepo(store.db);
|
package/src/processor.ts
CHANGED
|
@@ -12,10 +12,12 @@ import { handlePlacementMovePush } from "./handlers/placement-move-push.js";
|
|
|
12
12
|
import { syncMailboxes } from "./handlers/sync-mailboxes.js";
|
|
13
13
|
import { syncMessageBody } from "./handlers/sync-message-body.js";
|
|
14
14
|
import { syncMessages } from "./handlers/sync-messages.js";
|
|
15
|
+
import type { WorkerQueues } from "./queues.js";
|
|
15
16
|
|
|
16
17
|
export const processEvent = async (
|
|
17
18
|
event: WorkerEvent,
|
|
18
19
|
log: Logger,
|
|
20
|
+
queues: WorkerQueues,
|
|
19
21
|
/**
|
|
20
22
|
* SQS's own delivery count for the record carrying this event (1 on first
|
|
21
23
|
* delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH, FLAG_PUSH,
|
|
@@ -29,31 +31,31 @@ export const processEvent = async (
|
|
|
29
31
|
): Promise<void> => {
|
|
30
32
|
switch (event.type) {
|
|
31
33
|
case "SYNC_MAILBOXES":
|
|
32
|
-
return syncMailboxes(event, log);
|
|
34
|
+
return syncMailboxes(event, log, queues);
|
|
33
35
|
case "SYNC_MESSAGES":
|
|
34
|
-
return syncMessages(event, log);
|
|
36
|
+
return syncMessages(event, log, queues);
|
|
35
37
|
case "SYNC_MESSAGE_BODY":
|
|
36
|
-
return syncMessageBody(event, log, receiveCount);
|
|
38
|
+
return syncMessageBody(event, log, queues, receiveCount);
|
|
37
39
|
case "MAILBOX_CREATE":
|
|
38
40
|
case "MAILBOX_RENAME":
|
|
39
41
|
case "MAILBOX_DELETE":
|
|
40
42
|
return processMailboxManagement(event, log);
|
|
41
43
|
case "MESSAGE_DELETE":
|
|
42
|
-
return handleMessageDelete(event, log, receiveCount);
|
|
44
|
+
return handleMessageDelete(event, log, queues, receiveCount);
|
|
43
45
|
case "MESSAGE_MOVE":
|
|
44
|
-
return handleMessageMove(event, log, receiveCount);
|
|
46
|
+
return handleMessageMove(event, log, queues, receiveCount);
|
|
45
47
|
case "PLACEMENT_MOVE_PUSH":
|
|
46
|
-
return handlePlacementMovePush(event, log, receiveCount);
|
|
48
|
+
return handlePlacementMovePush(event, log, queues, receiveCount);
|
|
47
49
|
case "FLAG_PUSH":
|
|
48
50
|
return handleFlagPush(event, log, receiveCount);
|
|
49
51
|
case "MESSAGE_COPY":
|
|
50
|
-
return handleMessageCopy(event, log, receiveCount);
|
|
52
|
+
return handleMessageCopy(event, log, queues, receiveCount);
|
|
51
53
|
case "EMPTY_TRASH":
|
|
52
54
|
return handleEmptyTrash(event, log);
|
|
53
55
|
case "APPEND_SENT_MESSAGE":
|
|
54
56
|
return handleAppendSentMessage(event, log, receiveCount);
|
|
55
57
|
case "DELETE_ACCOUNT_OBJECTS":
|
|
56
|
-
return handleDeleteAccountObjects(event, log);
|
|
58
|
+
return handleDeleteAccountObjects(event, log, queues);
|
|
57
59
|
default:
|
|
58
60
|
throw new Error(`Unknown event type: ${(event as WorkerEvent).type}`);
|
|
59
61
|
}
|
package/src/queues.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
createFifoQueueProducer,
|
|
4
|
+
createStandardQueueProducer,
|
|
5
|
+
} from "@remit/sqs-client/producer";
|
|
3
6
|
import { env } from "expect-env";
|
|
4
7
|
import type { ImapEvent, WorkerEvent } from "./events.js";
|
|
5
8
|
|
|
@@ -24,26 +27,14 @@ export const workerQueueUrlsFromEnv = (): WorkerQueueUrls => ({
|
|
|
24
27
|
});
|
|
25
28
|
|
|
26
29
|
export const sqsWorkerQueues = (urls: WorkerQueueUrls): WorkerQueues => ({
|
|
27
|
-
mailboxes:
|
|
28
|
-
messages:
|
|
29
|
-
flags:
|
|
30
|
-
body:
|
|
31
|
-
mailboxManagement:
|
|
32
|
-
|
|
30
|
+
mailboxes: createFifoQueueProducer({ queueUrl: urls.mailboxes }),
|
|
31
|
+
messages: createFifoQueueProducer({ queueUrl: urls.messages }),
|
|
32
|
+
flags: createFifoQueueProducer({ queueUrl: urls.flags }),
|
|
33
|
+
body: createStandardQueueProducer({ queueUrl: urls.body }),
|
|
34
|
+
mailboxManagement: createStandardQueueProducer({
|
|
35
|
+
queueUrl: urls.mailboxManagement,
|
|
36
|
+
}),
|
|
37
|
+
messageManagement: createStandardQueueProducer({
|
|
38
|
+
queueUrl: urls.messageManagement,
|
|
39
|
+
}),
|
|
33
40
|
});
|
|
34
|
-
|
|
35
|
-
let composed: WorkerQueues | null = null;
|
|
36
|
-
|
|
37
|
-
/** Register the worker's queues from its entry, before the first event. */
|
|
38
|
-
export const setWorkerQueues = (queues: WorkerQueues): void => {
|
|
39
|
-
composed = queues;
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
export const workerQueues = (): WorkerQueues => {
|
|
43
|
-
if (!composed) {
|
|
44
|
-
throw new Error(
|
|
45
|
-
"no worker queues registered — register them with setWorkerQueues() from the entry",
|
|
46
|
-
);
|
|
47
|
-
}
|
|
48
|
-
return composed;
|
|
49
|
-
};
|
|
@@ -57,9 +57,8 @@ describe("the outbox attachment sweep", () => {
|
|
|
57
57
|
const second = await store.world();
|
|
58
58
|
const secondDraft = await arrange(second);
|
|
59
59
|
|
|
60
|
-
const {
|
|
61
|
-
|
|
62
|
-
await handler(
|
|
60
|
+
const { buildScheduledHandler } = await import("./handler.js");
|
|
61
|
+
await buildScheduledHandler(second.queues)(
|
|
63
62
|
{ time: new Date().toISOString() } as never,
|
|
64
63
|
{} as never,
|
|
65
64
|
() => {},
|
package/src/scheduler/handler.ts
CHANGED
|
@@ -2,11 +2,10 @@ import { getClient } from "@remit/backend/client";
|
|
|
2
2
|
import { createLogger } from "@remit/logger";
|
|
3
3
|
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
4
4
|
import type { ScheduledHandler } from "aws-lambda";
|
|
5
|
-
import {
|
|
5
|
+
import { createEmitEvent } from "../emit.js";
|
|
6
6
|
import {
|
|
7
|
-
setWorkerQueues,
|
|
8
7
|
sqsWorkerQueues,
|
|
9
|
-
|
|
8
|
+
type WorkerQueues,
|
|
10
9
|
workerQueueUrlsFromEnv,
|
|
11
10
|
} from "../queues.js";
|
|
12
11
|
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
@@ -19,8 +18,6 @@ installNodeLogger();
|
|
|
19
18
|
|
|
20
19
|
const log = createLogger();
|
|
21
20
|
|
|
22
|
-
setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
|
|
23
|
-
|
|
24
21
|
/**
|
|
25
22
|
* EventBridge-scheduled entry point for the periodic mailbox-sync tick
|
|
26
23
|
* (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
|
|
@@ -38,27 +35,35 @@ setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
|
|
|
38
35
|
* while a genuine retry of the same invocation carries the same `time` and
|
|
39
36
|
* correctly dedupes.
|
|
40
37
|
*/
|
|
41
|
-
export const
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
export const buildScheduledHandler = (queues: WorkerQueues): ScheduledHandler =>
|
|
39
|
+
withTelemetry(async (event) => {
|
|
40
|
+
const client = await getClient();
|
|
41
|
+
const now = Date.parse(event.time);
|
|
44
42
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
{ client, emit: emitEvent, log },
|
|
60
|
-
account,
|
|
61
|
-
tickNow,
|
|
43
|
+
await runSchedulerTick({
|
|
44
|
+
accountService: client.account,
|
|
45
|
+
queue: queues.mailboxes,
|
|
46
|
+
log,
|
|
47
|
+
tickIntervalMs: getTickIntervalMs(),
|
|
48
|
+
offlineIntervalMs: getOfflineIntervalMs(),
|
|
49
|
+
now,
|
|
50
|
+
// The hosted side is the one that needs this: a browser PUTs straight
|
|
51
|
+
// to block storage, so nothing refuses an upload against a draft that is
|
|
52
|
+
// already gone and this sweep is the only thing that collects it.
|
|
53
|
+
sweepAttachments: buildAttachmentSweep(client, log),
|
|
54
|
+
refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(
|
|
55
|
+
client,
|
|
56
|
+
log,
|
|
62
57
|
),
|
|
58
|
+
redriveStrandedPlacements: (account, tickNow) =>
|
|
59
|
+
redriveStrandedPlacements(
|
|
60
|
+
{ client, emit: createEmitEvent(queues), log },
|
|
61
|
+
account,
|
|
62
|
+
tickNow,
|
|
63
|
+
),
|
|
64
|
+
});
|
|
63
65
|
});
|
|
64
|
-
|
|
66
|
+
|
|
67
|
+
export const handler: ScheduledHandler = buildScheduledHandler(
|
|
68
|
+
sqsWorkerQueues(workerQueueUrlsFromEnv()),
|
|
69
|
+
);
|
|
@@ -65,7 +65,7 @@ const tick = async (
|
|
|
65
65
|
const logs: LogLine[] = [];
|
|
66
66
|
const result = await runSchedulerTick({
|
|
67
67
|
accountService: accounts,
|
|
68
|
-
queue,
|
|
68
|
+
queue: queue.fifo,
|
|
69
69
|
log: capturingLogger(logs),
|
|
70
70
|
tickIntervalMs: TICK_INTERVAL_MS,
|
|
71
71
|
offlineIntervalMs: OFFLINE_INTERVAL_MS,
|
package/src/scheduler/runner.ts
CHANGED
|
@@ -7,13 +7,8 @@ import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
|
7
7
|
import { backfillSearchText } from "@remit/mailbox-service";
|
|
8
8
|
import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
|
|
9
9
|
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
setWorkerQueues,
|
|
13
|
-
sqsWorkerQueues,
|
|
14
|
-
workerQueues,
|
|
15
|
-
workerQueueUrlsFromEnv,
|
|
16
|
-
} from "../queues.js";
|
|
10
|
+
import { createEmitEvent } from "../emit.js";
|
|
11
|
+
import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "../queues.js";
|
|
17
12
|
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
18
13
|
import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
|
|
19
14
|
import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
|
|
@@ -45,7 +40,7 @@ installNodeLogger();
|
|
|
45
40
|
|
|
46
41
|
const log = createLogger();
|
|
47
42
|
|
|
48
|
-
|
|
43
|
+
const queues = sqsWorkerQueues(workerQueueUrlsFromEnv());
|
|
49
44
|
|
|
50
45
|
const tickIntervalMs = getTickIntervalMs();
|
|
51
46
|
const offlineIntervalMs = getOfflineIntervalMs();
|
|
@@ -97,7 +92,7 @@ const runLoop = async (): Promise<void> => {
|
|
|
97
92
|
tick: runSchedulerTick,
|
|
98
93
|
tickDeps: {
|
|
99
94
|
accountService: account,
|
|
100
|
-
queue:
|
|
95
|
+
queue: queues.mailboxes,
|
|
101
96
|
log,
|
|
102
97
|
tickIntervalMs,
|
|
103
98
|
offlineIntervalMs,
|
|
@@ -108,7 +103,7 @@ const runLoop = async (): Promise<void> => {
|
|
|
108
103
|
),
|
|
109
104
|
redriveStrandedPlacements: (stranded, now) =>
|
|
110
105
|
redriveStrandedPlacements(
|
|
111
|
-
{ client, emit:
|
|
106
|
+
{ client, emit: createEmitEvent(queues), log },
|
|
112
107
|
stranded,
|
|
113
108
|
now,
|
|
114
109
|
),
|
|
@@ -447,6 +447,7 @@ describe("the reconcile re-drives a move or delete already stranded", () => {
|
|
|
447
447
|
await handleMessageMove(
|
|
448
448
|
stamped<MessageMoveEvent>(move),
|
|
449
449
|
a.w.log,
|
|
450
|
+
a.w.queues,
|
|
450
451
|
1,
|
|
451
452
|
a.w.scope,
|
|
452
453
|
);
|
|
@@ -469,6 +470,7 @@ describe("the reconcile re-drives a move or delete already stranded", () => {
|
|
|
469
470
|
await handlePlacementMovePush(
|
|
470
471
|
stamped<PlacementMovePushEvent>(push),
|
|
471
472
|
a.w.log,
|
|
473
|
+
a.w.queues,
|
|
472
474
|
1,
|
|
473
475
|
a.w.scope,
|
|
474
476
|
);
|
|
@@ -500,6 +502,7 @@ describe("the reconcile re-drives a move or delete already stranded", () => {
|
|
|
500
502
|
await handleMessageMove(
|
|
501
503
|
stamped<MessageMoveEvent>(emitted[0]),
|
|
502
504
|
a.w.log,
|
|
505
|
+
a.w.queues,
|
|
503
506
|
1,
|
|
504
507
|
a.w.scope,
|
|
505
508
|
);
|
|
@@ -520,6 +523,7 @@ describe("the reconcile re-drives a move or delete already stranded", () => {
|
|
|
520
523
|
messageId: a.message.messageId,
|
|
521
524
|
}),
|
|
522
525
|
a.w.log,
|
|
526
|
+
a.w.queues,
|
|
523
527
|
1,
|
|
524
528
|
a.w.scope,
|
|
525
529
|
);
|
|
@@ -645,6 +649,7 @@ describe("a uid-0 copy whose event is gone", () => {
|
|
|
645
649
|
await handleMessageCopy(
|
|
646
650
|
stamped<MessageCopyEvent>(copy),
|
|
647
651
|
a.w.log,
|
|
652
|
+
a.w.queues,
|
|
648
653
|
1,
|
|
649
654
|
a.w.scope,
|
|
650
655
|
);
|