@remit/imap-worker 0.0.86 → 0.0.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +10 -2
- package/src/account-check.ts +1 -1
- package/src/body-sync-gate.test.ts +1 -1
- package/src/body-sync-gate.ts +1 -1
- package/src/e2e-processor-shim.ts +6 -21
- package/src/emit.ts +1 -1
- package/src/failure-kind.ts +1 -1
- package/src/handlers/append-sent-message.sqlite.test.ts +176 -0
- package/src/handlers/append-sent-message.ts +3 -24
- package/src/handlers/delete-account-objects.test.ts +1 -1
- package/src/handlers/delete-account-objects.ts +1 -1
- package/src/handlers/empty-trash.sqlite.test.ts +209 -0
- package/src/handlers/empty-trash.ts +3 -24
- package/src/handlers/flag-push.ts +11 -18
- package/src/handlers/mailbox-management.sqlite.test.ts +265 -0
- package/src/handlers/mailbox-management.ts +12 -48
- package/src/handlers/message-copy.sqlite.test.ts +298 -0
- package/src/handlers/message-copy.ts +31 -49
- package/src/handlers/message-delete-terminal.ts +12 -10
- package/src/handlers/message-delete.sqlite.test.ts +291 -0
- package/src/handlers/message-delete.ts +26 -32
- package/src/handlers/message-move-terminal.ts +12 -10
- package/src/handlers/message-move.sqlite.test.ts +271 -0
- package/src/handlers/message-move.test.ts +76 -863
- package/src/handlers/message-move.ts +49 -39
- package/src/handlers/placement-move-push.test.ts +96 -329
- package/src/handlers/placement-move-push.ts +75 -29
- package/src/handlers/placement-push.sqlite.test.ts +298 -0
- package/src/handlers/probe-message-id.sqlite.test.ts +194 -0
- package/src/handlers/probe-message-id.ts +15 -0
- package/src/handlers/sync-mailboxes.test.ts +0 -1
- package/src/handlers/sync-mailboxes.ts +1 -1
- package/src/handlers/sync-message-body.sqlite.test.ts +314 -0
- package/src/handlers/sync-message-body.test.ts +3 -299
- package/src/handlers/sync-message-body.ts +21 -11
- package/src/handlers/sync-messages.sqlite.test.ts +183 -0
- package/src/handlers/sync-messages.ts +27 -33
- package/src/index.ts +5 -3
- package/src/mail-sync-selection.sqlite.test.ts +21 -21
- package/src/poller.ts +5 -4
- package/src/processor.test.ts +1 -1
- package/src/processor.ts +1 -1
- package/src/scheduler/attachment-sweep.sqlite.test.ts +74 -0
- package/src/scheduler/attachment-sweep.ts +1 -1
- package/src/scheduler/{calendar-subscriptions.test.ts → calendar-subscriptions.sqlite.test.ts} +43 -55
- package/src/scheduler/calendar-subscriptions.ts +6 -6
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +203 -0
- package/src/scheduler/handler.ts +12 -1
- package/src/scheduler/loop.test.ts +2 -0
- package/src/scheduler/run-tick.sqlite.test.ts +199 -0
- package/src/scheduler/run-tick.ts +39 -1
- package/src/scheduler/runner.ts +21 -1
- package/src/scheduler/stranded-placement-redrive.ts +265 -0
- package/src/stranded-placement.sqlite.test.ts +795 -0
- package/src/test-helpers/test-world.ts +398 -0
- package/src/thread-join.sqlite.test.ts +106 -0
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +146 -15
- package/src/with-oauth-lifecycle.ts +1 -1
- package/src/handlers/append-sent-message.test.ts +0 -529
- package/src/handlers/empty-trash.test.ts +0 -728
- package/src/handlers/flag-push-placement-race.test.ts +0 -385
- package/src/handlers/flag-push.test.ts +0 -325
- package/src/handlers/mailbox-management.test.ts +0 -850
- package/src/handlers/message-copy.test.ts +0 -654
- package/src/handlers/message-delete.test.ts +0 -1869
- package/src/handlers/message-move-terminal.test.ts +0 -412
- package/src/handlers/sync-messages.test.ts +0 -569
- package/src/scheduler/attachment-sweep.test.ts +0 -169
- package/src/scheduler/run-tick.test.ts +0 -478
- package/src/with-oauth-lifecycle.test.ts +0 -318
|
@@ -1,28 +1,31 @@
|
|
|
1
1
|
import { getClient } from "@remit/backend/client";
|
|
2
|
-
import {
|
|
3
|
-
import type { Logger } from "@remit/logger
|
|
4
|
-
import { recordImapFailure } from "@remit/logger-lambda";
|
|
2
|
+
import { MessageMutation, MessageSyncStatus } from "@remit/domain-enums";
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
|
+
import { recordImapFailure } from "@remit/logger-lambda/metrics";
|
|
5
5
|
import {
|
|
6
6
|
guardConnectionCursor,
|
|
7
7
|
type IImapConnection,
|
|
8
8
|
isCursorRebuildNeeded,
|
|
9
9
|
isMessageGoneFromOpenMailbox,
|
|
10
10
|
MailboxCursorPausedError,
|
|
11
|
-
reconcileStaleMessage,
|
|
12
11
|
resolveExhaustedPlacementMoveFailure,
|
|
12
|
+
restoreSourcePlacement,
|
|
13
13
|
} from "@remit/mailbox-service";
|
|
14
14
|
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
15
15
|
import { isAccountDeleted } from "../account-check.js";
|
|
16
16
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
17
17
|
import { emitEvent } from "../emit.js";
|
|
18
18
|
import type { PlacementMovePushEvent } from "../events.js";
|
|
19
|
+
import { findMailboxRow } from "../mailbox-row.js";
|
|
19
20
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
20
21
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
21
22
|
import {
|
|
22
23
|
buildThreadMessageMoveUpdate,
|
|
24
|
+
emitMailboxResync,
|
|
23
25
|
emitMoveResync,
|
|
24
26
|
searchMailboxForHighestMessageIdUid,
|
|
25
27
|
} from "./message-move.js";
|
|
28
|
+
import { probeableMessageIdHeader } from "./probe-message-id.js";
|
|
26
29
|
|
|
27
30
|
export const getPlacementMoveMaxAttempts = (
|
|
28
31
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
@@ -150,6 +153,7 @@ export const handlePlacementMovePush = async (
|
|
|
150
153
|
event: PlacementMovePushEvent,
|
|
151
154
|
log: Logger,
|
|
152
155
|
receiveCount = 1,
|
|
156
|
+
createConnectionScope = createConnectionScopeWithCredentials,
|
|
153
157
|
): Promise<void> => {
|
|
154
158
|
const {
|
|
155
159
|
account: accountService,
|
|
@@ -180,6 +184,11 @@ export const handlePlacementMovePush = async (
|
|
|
180
184
|
}
|
|
181
185
|
|
|
182
186
|
const message = await messageService.get(messageId);
|
|
187
|
+
const placementRead = {
|
|
188
|
+
status: message.status,
|
|
189
|
+
mailboxId: message.mailboxId,
|
|
190
|
+
uid: message.uid,
|
|
191
|
+
};
|
|
183
192
|
|
|
184
193
|
// A newer local action (drag-and-drop move, auto-moved Undo) already
|
|
185
194
|
// changed the message's location away from what this marker still
|
|
@@ -205,32 +214,61 @@ export const handlePlacementMovePush = async (
|
|
|
205
214
|
// died mid-flight already left it here).
|
|
206
215
|
await markerService.updateState(messageId, "processing");
|
|
207
216
|
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
217
|
+
// No MOVE has left for this marker, so the row is settled, not just unmarked.
|
|
218
|
+
const [source, destination] = await Promise.all([
|
|
219
|
+
findMailboxRow(mailboxService, accountId, marker.sourceMailboxId),
|
|
220
|
+
findMailboxRow(mailboxService, accountId, marker.destinationMailboxId),
|
|
221
|
+
]);
|
|
222
|
+
if (source.kind === "gone" || destination.kind === "gone") {
|
|
223
|
+
const context = {
|
|
224
|
+
alert: "placement_move_mailbox_unreachable",
|
|
225
|
+
messageId,
|
|
226
|
+
accountId,
|
|
227
|
+
sourceMailboxId: marker.sourceMailboxId,
|
|
228
|
+
destinationMailboxId: marker.destinationMailboxId,
|
|
229
|
+
};
|
|
230
|
+
if (source.kind === "gone") {
|
|
231
|
+
const reconciled = await messageService.deleteIfPlacement(
|
|
232
|
+
messageId,
|
|
233
|
+
placementRead,
|
|
234
|
+
);
|
|
235
|
+
await markerService.delete(messageId);
|
|
236
|
+
log.error(
|
|
237
|
+
{ ...context, reconciled },
|
|
238
|
+
"Placement move not pushed: its source folder is gone; the stale row was reconciled",
|
|
239
|
+
);
|
|
240
|
+
if (destination.kind === "present") {
|
|
241
|
+
await emitMailboxResync(emitEvent, {
|
|
242
|
+
accountId,
|
|
243
|
+
mailboxId: marker.destinationMailboxId,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
await restoreSourcePlacement(
|
|
249
|
+
{ messageService, threadMessageService },
|
|
223
250
|
{
|
|
251
|
+
accountConfigId,
|
|
224
252
|
messageId,
|
|
225
|
-
accountId,
|
|
226
253
|
sourceMailboxId: marker.sourceMailboxId,
|
|
227
|
-
|
|
254
|
+
uid: message.uid,
|
|
255
|
+
syncStatus: MessageSyncStatus.synced,
|
|
256
|
+
abandonedMutation: MessageMutation.none,
|
|
228
257
|
},
|
|
229
|
-
"Skipping PLACEMENT_MOVE_PUSH: a mailbox no longer exists (deleted); marker dropped",
|
|
230
258
|
);
|
|
259
|
+
await markerService.delete(messageId);
|
|
260
|
+
log.error(
|
|
261
|
+
context,
|
|
262
|
+
"Placement move not pushed: its destination folder is gone; the row was handed back to its source",
|
|
263
|
+
);
|
|
264
|
+
await emitMailboxResync(emitEvent, {
|
|
265
|
+
accountId,
|
|
266
|
+
mailboxId: marker.sourceMailboxId,
|
|
267
|
+
});
|
|
231
268
|
return;
|
|
232
269
|
}
|
|
233
|
-
const
|
|
270
|
+
const sourceMailbox = source.mailbox;
|
|
271
|
+
const destinationMailbox = destination.mailbox;
|
|
234
272
|
|
|
235
273
|
// Cheap frugal skip (epic #1281 invariant 6): either mailbox already known
|
|
236
274
|
// paused never even borrows a connection. Optimization only — the
|
|
@@ -256,7 +294,7 @@ export const handlePlacementMovePush = async (
|
|
|
256
294
|
account,
|
|
257
295
|
log,
|
|
258
296
|
async (credentials) => {
|
|
259
|
-
const scope =
|
|
297
|
+
const scope = createConnectionScope(account, credentials);
|
|
260
298
|
|
|
261
299
|
await scope
|
|
262
300
|
.getConnection()
|
|
@@ -283,7 +321,11 @@ export const handlePlacementMovePush = async (
|
|
|
283
321
|
sourceMailbox.fullPath,
|
|
284
322
|
destinationMailbox.fullPath,
|
|
285
323
|
message.uid,
|
|
286
|
-
|
|
324
|
+
await probeableMessageIdHeader(
|
|
325
|
+
threadMessageService,
|
|
326
|
+
accountConfigId,
|
|
327
|
+
messageId,
|
|
328
|
+
),
|
|
287
329
|
);
|
|
288
330
|
|
|
289
331
|
if (outcome.kind === "trycreate") {
|
|
@@ -298,10 +340,9 @@ export const handlePlacementMovePush = async (
|
|
|
298
340
|
if (outcome.kind === "not-found") {
|
|
299
341
|
// External delete supersedes the marker entirely (epic invariant 2).
|
|
300
342
|
await markerService.delete(messageId);
|
|
301
|
-
const
|
|
302
|
-
{ messageService, threadMessageService },
|
|
303
|
-
accountConfigId,
|
|
343
|
+
const reconciled = await messageService.deleteIfPlacement(
|
|
304
344
|
messageId,
|
|
345
|
+
placementRead,
|
|
305
346
|
);
|
|
306
347
|
log.info(
|
|
307
348
|
{
|
|
@@ -309,7 +350,7 @@ export const handlePlacementMovePush = async (
|
|
|
309
350
|
accountId,
|
|
310
351
|
uid: message.uid,
|
|
311
352
|
sourceMailboxPath: sourceMailbox.fullPath,
|
|
312
|
-
|
|
353
|
+
reconciled,
|
|
313
354
|
},
|
|
314
355
|
"Message no longer at its pending-move source (external delete or moved away); marker dropped, stale row reconciled",
|
|
315
356
|
);
|
|
@@ -395,6 +436,11 @@ export const handlePlacementMovePush = async (
|
|
|
395
436
|
const { outcome } = await resolveExhaustedPlacementMoveFailure(
|
|
396
437
|
{ markerService, messageService, threadMessageService, log },
|
|
397
438
|
{
|
|
439
|
+
placement: {
|
|
440
|
+
status: message.status,
|
|
441
|
+
mailboxId: message.mailboxId,
|
|
442
|
+
uid: message.uid,
|
|
443
|
+
},
|
|
398
444
|
accountId,
|
|
399
445
|
accountConfigId,
|
|
400
446
|
messageId,
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two pushes body sync leaves behind: a placement move (Junk, a filter's
|
|
3
|
+
* folder) and a flag (auto-mark-read). They ride different queues, so either
|
|
4
|
+
* can reach the server first, and the flag must never land on the uid of a
|
|
5
|
+
* move that has not settled (#496).
|
|
6
|
+
*/
|
|
7
|
+
import assert from "node:assert/strict";
|
|
8
|
+
import { after, before, describe, it, mock } from "node:test";
|
|
9
|
+
import type { MailboxItem } from "@remit/data-ports";
|
|
10
|
+
import {
|
|
11
|
+
MailboxCursorState,
|
|
12
|
+
MailboxSpecialUse,
|
|
13
|
+
MessageStatus,
|
|
14
|
+
} from "@remit/domain-enums";
|
|
15
|
+
import { PlacementMoveService } from "@remit/mailbox-service";
|
|
16
|
+
import type { FlagPushEvent, PlacementMovePushEvent } from "../events.js";
|
|
17
|
+
import {
|
|
18
|
+
openStore,
|
|
19
|
+
type SeededMessage,
|
|
20
|
+
type World,
|
|
21
|
+
} from "../test-helpers/test-world.js";
|
|
22
|
+
import { FLAG_PUSH_DEFER_MAX_MS, handleFlagPush } from "./flag-push.js";
|
|
23
|
+
import { handlePlacementMovePush } from "./placement-move-push.js";
|
|
24
|
+
|
|
25
|
+
const SEEN = "\\Seen";
|
|
26
|
+
|
|
27
|
+
let store: Awaited<ReturnType<typeof openStore>>;
|
|
28
|
+
|
|
29
|
+
before(async () => {
|
|
30
|
+
store = await openStore();
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
after(() => store.close());
|
|
34
|
+
|
|
35
|
+
interface Arranged {
|
|
36
|
+
w: World;
|
|
37
|
+
inbox: MailboxItem;
|
|
38
|
+
junk: MailboxItem;
|
|
39
|
+
message: SeededMessage;
|
|
40
|
+
placement: PlacementMoveService;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const arrange = async (): Promise<Arranged> => {
|
|
44
|
+
const w = await store.world();
|
|
45
|
+
const inbox = await w.folder("INBOX");
|
|
46
|
+
const junk = await w.folder("Junk", { specialUse: MailboxSpecialUse.Junk });
|
|
47
|
+
const message = await w.message(inbox);
|
|
48
|
+
const placement = new PlacementMoveService({
|
|
49
|
+
messageService: w.client.message,
|
|
50
|
+
threadMessageService: w.client.threadMessage,
|
|
51
|
+
markerService: w.client.placementMove,
|
|
52
|
+
addressService: w.client.address,
|
|
53
|
+
mailboxSpecialUseService: w.client.mailboxSpecialUse,
|
|
54
|
+
mailboxService: w.client.mailbox,
|
|
55
|
+
sqsQueueUrl: process.env.SQS_QUEUE_URL_MESSAGE_MGMT ?? "",
|
|
56
|
+
});
|
|
57
|
+
return { w, inbox, junk, message, placement };
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const queuedOf = <T>(w: World, type: string): T => {
|
|
61
|
+
const event = w
|
|
62
|
+
.queued()
|
|
63
|
+
.filter((body) => body.type === type)
|
|
64
|
+
.at(-1) as T | undefined;
|
|
65
|
+
assert.ok(event, `a ${type} was enqueued`);
|
|
66
|
+
return event;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const markAsRead = async (a: Arranged): Promise<FlagPushEvent> => {
|
|
70
|
+
await a.w.client.flagQueue.markAsRead(
|
|
71
|
+
a.w.account.accountConfigId,
|
|
72
|
+
a.message.messageId,
|
|
73
|
+
a.w.account.accountId,
|
|
74
|
+
);
|
|
75
|
+
return queuedOf<FlagPushEvent>(a.w, "FLAG_PUSH");
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const moveToJunk = async (a: Arranged): Promise<PlacementMovePushEvent> => {
|
|
79
|
+
await a.placement.moveMessage(
|
|
80
|
+
a.w.account.accountConfigId,
|
|
81
|
+
a.message.messageId,
|
|
82
|
+
a.junk.mailboxId,
|
|
83
|
+
a.w.account.accountId,
|
|
84
|
+
);
|
|
85
|
+
return queuedOf<PlacementMovePushEvent>(a.w, "PLACEMENT_MOVE_PUSH");
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const pushFlag = (a: Arranged, event: FlagPushEvent, receiveCount = 1) =>
|
|
89
|
+
handleFlagPush(event, a.w.log, receiveCount, a.w.scope);
|
|
90
|
+
|
|
91
|
+
const pushMove = (
|
|
92
|
+
a: Arranged,
|
|
93
|
+
event: PlacementMovePushEvent,
|
|
94
|
+
receiveCount = 1,
|
|
95
|
+
) => handlePlacementMovePush(event, a.w.log, receiveCount, a.w.scope);
|
|
96
|
+
|
|
97
|
+
const seenOnServer = (a: Arranged, path: string): boolean[] =>
|
|
98
|
+
a.w.imap
|
|
99
|
+
.messages(path)
|
|
100
|
+
.filter((m) => m.messageIdHeader === a.message.messageIdHeader)
|
|
101
|
+
.map((m) => m.flags.includes(SEEN));
|
|
102
|
+
|
|
103
|
+
describe("a flag push", () => {
|
|
104
|
+
it("stores the flag on the server and clears its marker", async () => {
|
|
105
|
+
const a = await arrange();
|
|
106
|
+
const event = await markAsRead(a);
|
|
107
|
+
|
|
108
|
+
await pushFlag(a, event);
|
|
109
|
+
|
|
110
|
+
assert.deepEqual(seenOnServer(a, "INBOX"), [true]);
|
|
111
|
+
assert.equal(
|
|
112
|
+
await a.w.client.flagPush.find(a.message.messageId, SEEN),
|
|
113
|
+
null,
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it("waits, without connecting, while the message's placement move has not settled", async () => {
|
|
118
|
+
const a = await arrange();
|
|
119
|
+
await moveToJunk(a);
|
|
120
|
+
const event = await markAsRead(a);
|
|
121
|
+
|
|
122
|
+
await pushFlag(a, event);
|
|
123
|
+
|
|
124
|
+
assert.equal(a.w.imap.connections, 0);
|
|
125
|
+
assert.equal(
|
|
126
|
+
(await a.w.client.flagPush.find(a.message.messageId, SEEN))?.state,
|
|
127
|
+
"pending",
|
|
128
|
+
);
|
|
129
|
+
assert.deepEqual(seenOnServer(a, "INBOX"), [false]);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it("lands on the moved message once the placement move has settled", async () => {
|
|
133
|
+
const a = await arrange();
|
|
134
|
+
const move = await moveToJunk(a);
|
|
135
|
+
const flag = await markAsRead(a);
|
|
136
|
+
await pushFlag(a, flag);
|
|
137
|
+
|
|
138
|
+
await pushMove(a, move);
|
|
139
|
+
await pushFlag(a, flag);
|
|
140
|
+
|
|
141
|
+
assert.deepEqual(a.w.imap.uids("INBOX"), []);
|
|
142
|
+
assert.deepEqual(seenOnServer(a, "Junk"), [true]);
|
|
143
|
+
assert.equal(
|
|
144
|
+
await a.w.client.flagPush.find(a.message.messageId, SEEN),
|
|
145
|
+
null,
|
|
146
|
+
);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
// A move that never settles would otherwise park the flag forever, one
|
|
150
|
+
// queue round trip per sync tick.
|
|
151
|
+
it("drops the marker without connecting once the move has been stuck past the window", async (t) => {
|
|
152
|
+
const a = await arrange();
|
|
153
|
+
await moveToJunk(a);
|
|
154
|
+
const event = await markAsRead(a);
|
|
155
|
+
mock.timers.enable({
|
|
156
|
+
apis: ["Date"],
|
|
157
|
+
now: Date.now() + FLAG_PUSH_DEFER_MAX_MS + 1,
|
|
158
|
+
});
|
|
159
|
+
t.after(() => mock.timers.reset());
|
|
160
|
+
|
|
161
|
+
await pushFlag(a, event);
|
|
162
|
+
|
|
163
|
+
assert.equal(a.w.imap.connections, 0);
|
|
164
|
+
assert.equal(
|
|
165
|
+
await a.w.client.flagPush.find(a.message.messageId, SEEN),
|
|
166
|
+
null,
|
|
167
|
+
);
|
|
168
|
+
assert.deepEqual(seenOnServer(a, "INBOX"), [false]);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it("drops the marker without connecting when the mailbox was deleted", async () => {
|
|
172
|
+
const a = await arrange();
|
|
173
|
+
const event = await markAsRead(a);
|
|
174
|
+
await a.w.client.mailbox.delete(a.w.account.accountId, a.inbox.mailboxId);
|
|
175
|
+
|
|
176
|
+
await pushFlag(a, event);
|
|
177
|
+
|
|
178
|
+
assert.equal(a.w.imap.connections, 0);
|
|
179
|
+
assert.equal(
|
|
180
|
+
await a.w.client.flagPush.find(a.message.messageId, SEEN),
|
|
181
|
+
null,
|
|
182
|
+
);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("drops the marker without connecting when the message row is gone", async () => {
|
|
186
|
+
const a = await arrange();
|
|
187
|
+
const event = await markAsRead(a);
|
|
188
|
+
await a.w.client.message.delete(a.message.messageId);
|
|
189
|
+
|
|
190
|
+
await pushFlag(a, event);
|
|
191
|
+
|
|
192
|
+
assert.equal(a.w.imap.connections, 0);
|
|
193
|
+
assert.equal(
|
|
194
|
+
await a.w.client.flagPush.find(a.message.messageId, SEEN),
|
|
195
|
+
null,
|
|
196
|
+
);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("keeps the marker and stores nothing while the cursor is rebuilding", async () => {
|
|
200
|
+
const a = await arrange();
|
|
201
|
+
const event = await markAsRead(a);
|
|
202
|
+
await a.w.client.mailbox.update(a.w.account.accountId, a.inbox.mailboxId, {
|
|
203
|
+
cursorState: MailboxCursorState.rebuilding,
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
await pushFlag(a, event);
|
|
207
|
+
|
|
208
|
+
assert.equal(a.w.imap.connections, 0);
|
|
209
|
+
assert.ok(await a.w.client.flagPush.find(a.message.messageId, SEEN));
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it("stores nothing when the mailbox's UIDVALIDITY changed under it", async () => {
|
|
213
|
+
const a = await arrange();
|
|
214
|
+
const event = await markAsRead(a);
|
|
215
|
+
a.w.imap.setUidValidity("INBOX", 2);
|
|
216
|
+
|
|
217
|
+
await pushFlag(a, event);
|
|
218
|
+
|
|
219
|
+
assert.deepEqual(a.w.imap.commands("STORE"), []);
|
|
220
|
+
assert.ok(await a.w.client.flagPush.find(a.message.messageId, SEEN));
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("keeps the marker and redelivers when the connection drops", async () => {
|
|
224
|
+
const a = await arrange();
|
|
225
|
+
const event = await markAsRead(a);
|
|
226
|
+
a.w.imap.failNext.STORE = new Error("Connection reset");
|
|
227
|
+
|
|
228
|
+
await assert.rejects(pushFlag(a, event), /Connection reset/);
|
|
229
|
+
|
|
230
|
+
assert.ok(await a.w.client.flagPush.find(a.message.messageId, SEEN));
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
describe("a placement move push", () => {
|
|
235
|
+
it("moves the message on the server, settles the row and clears the marker", async () => {
|
|
236
|
+
const a = await arrange();
|
|
237
|
+
const event = await moveToJunk(a);
|
|
238
|
+
|
|
239
|
+
await pushMove(a, event);
|
|
240
|
+
|
|
241
|
+
const [junkUid] = a.w.imap.uids("Junk", a.message.messageIdHeader);
|
|
242
|
+
const row = await a.w.row(a.message.messageId);
|
|
243
|
+
assert.equal(row?.mailboxId, a.junk.mailboxId);
|
|
244
|
+
assert.equal(row?.uid, junkUid);
|
|
245
|
+
assert.equal(row?.status, MessageStatus.active);
|
|
246
|
+
assert.equal(
|
|
247
|
+
await a.w.client.placementMove.find(a.message.messageId),
|
|
248
|
+
null,
|
|
249
|
+
);
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
it("drops the marker without connecting when a newer move took the message elsewhere", async () => {
|
|
253
|
+
const a = await arrange();
|
|
254
|
+
const event = await moveToJunk(a);
|
|
255
|
+
const archive = await a.w.folder("Archive");
|
|
256
|
+
await a.w.client.message.transitionPlacement(
|
|
257
|
+
a.message.messageId,
|
|
258
|
+
{ mailboxId: a.junk.mailboxId },
|
|
259
|
+
{ mailboxId: archive.mailboxId },
|
|
260
|
+
);
|
|
261
|
+
|
|
262
|
+
await pushMove(a, event);
|
|
263
|
+
|
|
264
|
+
assert.equal(a.w.imap.connections, 0);
|
|
265
|
+
assert.equal(
|
|
266
|
+
await a.w.client.placementMove.find(a.message.messageId),
|
|
267
|
+
null,
|
|
268
|
+
);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
it("drops the marker without connecting when the destination mailbox was deleted", async () => {
|
|
272
|
+
const a = await arrange();
|
|
273
|
+
const event = await moveToJunk(a);
|
|
274
|
+
await a.w.client.mailbox.delete(a.w.account.accountId, a.junk.mailboxId);
|
|
275
|
+
|
|
276
|
+
await pushMove(a, event);
|
|
277
|
+
|
|
278
|
+
assert.equal(a.w.imap.connections, 0);
|
|
279
|
+
assert.equal(
|
|
280
|
+
await a.w.client.placementMove.find(a.message.messageId),
|
|
281
|
+
null,
|
|
282
|
+
);
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
it("keeps the marker and moves nothing while the source cursor is rebuilding", async () => {
|
|
286
|
+
const a = await arrange();
|
|
287
|
+
const event = await moveToJunk(a);
|
|
288
|
+
await a.w.client.mailbox.update(a.w.account.accountId, a.inbox.mailboxId, {
|
|
289
|
+
cursorState: MailboxCursorState.rebuilding,
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
await pushMove(a, event);
|
|
293
|
+
|
|
294
|
+
assert.equal(a.w.imap.connections, 0);
|
|
295
|
+
assert.ok(await a.w.client.placementMove.find(a.message.messageId));
|
|
296
|
+
assert.deepEqual(a.w.imap.uids("INBOX"), [a.message.uid]);
|
|
297
|
+
});
|
|
298
|
+
});
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { afterEach, beforeEach, describe, it, mock } from "node:test";
|
|
3
|
+
import { SQSClient } from "@aws-sdk/client-sqs";
|
|
4
|
+
import type { MailboxItem } from "@remit/data-ports";
|
|
5
|
+
import {
|
|
6
|
+
AccountAuthType,
|
|
7
|
+
ConnectionState,
|
|
8
|
+
MailboxCursorState,
|
|
9
|
+
} from "@remit/domain-enums";
|
|
10
|
+
import {
|
|
11
|
+
createSqliteRepositories,
|
|
12
|
+
type DrizzleMessageRepository,
|
|
13
|
+
type DrizzleThreadMessageRepository,
|
|
14
|
+
type MailboxRepo,
|
|
15
|
+
type SqliteRepositories,
|
|
16
|
+
} from "@remit/drizzle-service";
|
|
17
|
+
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
18
|
+
import {
|
|
19
|
+
type IImapConnection,
|
|
20
|
+
type ManagedConnectionFactory,
|
|
21
|
+
MessageMoveService,
|
|
22
|
+
MessageSyncService,
|
|
23
|
+
SyncWarningService,
|
|
24
|
+
} from "@remit/mailbox-service";
|
|
25
|
+
import type { ImapMessage } from "@remit/mailbox-service/types";
|
|
26
|
+
import { probeableMessageIdHeader } from "./probe-message-id.js";
|
|
27
|
+
|
|
28
|
+
const ACCOUNT_CONFIG_ID = "cfg-msgid";
|
|
29
|
+
const UID_VALIDITY = 7;
|
|
30
|
+
|
|
31
|
+
const inboxHolding = (messageIdHeader: string): ImapMessage => ({
|
|
32
|
+
uid: 3,
|
|
33
|
+
seq: 1,
|
|
34
|
+
flags: [],
|
|
35
|
+
internalDate: new Date(0),
|
|
36
|
+
size: 100,
|
|
37
|
+
envelope: {
|
|
38
|
+
date: new Date(0).toISOString(),
|
|
39
|
+
messageId: messageIdHeader,
|
|
40
|
+
subject: "Quarterly report",
|
|
41
|
+
from: [{ name: "Sender", mailbox: "sender", host: "example.com" }],
|
|
42
|
+
sender: [],
|
|
43
|
+
replyTo: [],
|
|
44
|
+
to: [],
|
|
45
|
+
cc: [],
|
|
46
|
+
bcc: [],
|
|
47
|
+
inReplyTo: "",
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
const serverWithInbox = (message: ImapMessage): IImapConnection =>
|
|
52
|
+
({
|
|
53
|
+
openBox: async (path: string) => ({
|
|
54
|
+
name: path,
|
|
55
|
+
readOnly: false,
|
|
56
|
+
uidvalidity: UID_VALIDITY,
|
|
57
|
+
uidnext: message.uid + 1,
|
|
58
|
+
flags: [],
|
|
59
|
+
permFlags: [],
|
|
60
|
+
persistentUIDs: true,
|
|
61
|
+
messages: { total: 1, new: 0 },
|
|
62
|
+
newKeywords: false,
|
|
63
|
+
}),
|
|
64
|
+
getMailboxStatus: async () => ({
|
|
65
|
+
messages: 1,
|
|
66
|
+
recent: 0,
|
|
67
|
+
unseen: 0,
|
|
68
|
+
uidNext: message.uid + 1,
|
|
69
|
+
uidValidity: UID_VALIDITY,
|
|
70
|
+
highestModseq: "0",
|
|
71
|
+
deletedCount: 0,
|
|
72
|
+
}),
|
|
73
|
+
supportsCondstore: () => false,
|
|
74
|
+
search: async () => [message.uid],
|
|
75
|
+
fetchMessages: async () => [message],
|
|
76
|
+
}) as unknown as IImapConnection;
|
|
77
|
+
|
|
78
|
+
describe("the Message-ID the move, delete and copy probes ask with", () => {
|
|
79
|
+
let close: () => void;
|
|
80
|
+
let repos: SqliteRepositories;
|
|
81
|
+
let accountId: string;
|
|
82
|
+
let mailboxes: MailboxRepo;
|
|
83
|
+
let messages: DrizzleMessageRepository;
|
|
84
|
+
let threadMessages: DrizzleThreadMessageRepository;
|
|
85
|
+
|
|
86
|
+
const createMailbox = async (fullPath: string): Promise<MailboxItem> => {
|
|
87
|
+
const created = await mailboxes.create({
|
|
88
|
+
accountId,
|
|
89
|
+
namespacePrefix: "",
|
|
90
|
+
hierarchyDelimiter: "/",
|
|
91
|
+
fullPath,
|
|
92
|
+
uidValidity: UID_VALIDITY,
|
|
93
|
+
uidNext: 1,
|
|
94
|
+
highestModseq: "0",
|
|
95
|
+
messageCount: 0,
|
|
96
|
+
unseenCount: 0,
|
|
97
|
+
deletedCount: 0,
|
|
98
|
+
totalSize: 0,
|
|
99
|
+
lastSyncUid: 0,
|
|
100
|
+
highWaterMarkUid: 0,
|
|
101
|
+
lastMessageSyncAt: 0,
|
|
102
|
+
cursorState: MailboxCursorState.normal,
|
|
103
|
+
} as Parameters<MailboxRepo["create"]>[0]);
|
|
104
|
+
if (created.outcome !== "Created") throw new Error(`${fullPath} taken`);
|
|
105
|
+
return created.mailbox;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const syncInbox = async (messageIdHeader: string): Promise<string> => {
|
|
109
|
+
const inbox = await createMailbox("INBOX");
|
|
110
|
+
const connection = serverWithInbox(inboxHolding(messageIdHeader));
|
|
111
|
+
const factory: ManagedConnectionFactory = {
|
|
112
|
+
getConnection: () => connection,
|
|
113
|
+
close: async () => {},
|
|
114
|
+
};
|
|
115
|
+
await new MessageSyncService(
|
|
116
|
+
factory,
|
|
117
|
+
mailboxes,
|
|
118
|
+
repos.mailboxSpecialUse,
|
|
119
|
+
messages,
|
|
120
|
+
repos.envelope,
|
|
121
|
+
repos.address,
|
|
122
|
+
threadMessages,
|
|
123
|
+
new SyncWarningService(repos.syncWarning),
|
|
124
|
+
).syncMessages(inbox.mailboxId, accountId, ACCOUNT_CONFIG_ID);
|
|
125
|
+
const [synced] = await messages.listAllByMailbox(inbox.mailboxId);
|
|
126
|
+
if (!synced) throw new Error("sync stored no message");
|
|
127
|
+
return synced.messageId;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const probeable = (messageId: string): Promise<string | undefined> =>
|
|
131
|
+
probeableMessageIdHeader(threadMessages, ACCOUNT_CONFIG_ID, messageId);
|
|
132
|
+
|
|
133
|
+
beforeEach(async () => {
|
|
134
|
+
const store = createShippedSqliteDb();
|
|
135
|
+
close = store.close;
|
|
136
|
+
repos = createSqliteRepositories({
|
|
137
|
+
app: store.db,
|
|
138
|
+
mail: store.db,
|
|
139
|
+
search: store.db,
|
|
140
|
+
});
|
|
141
|
+
mailboxes = repos.mailbox;
|
|
142
|
+
messages = repos.message;
|
|
143
|
+
threadMessages = repos.threadMessage;
|
|
144
|
+
const account = await repos.account.create({
|
|
145
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
146
|
+
username: "person@example.com",
|
|
147
|
+
email: "person@example.com",
|
|
148
|
+
authType: AccountAuthType.Password,
|
|
149
|
+
passwordHash: '{"ciphertext":"x"}',
|
|
150
|
+
imapHost: "imap.example.com",
|
|
151
|
+
imapPort: 993,
|
|
152
|
+
imapTls: true,
|
|
153
|
+
imapStartTls: false,
|
|
154
|
+
isActive: true,
|
|
155
|
+
connectionState: ConnectionState.Authenticated,
|
|
156
|
+
});
|
|
157
|
+
accountId = account.accountId;
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
afterEach(() => {
|
|
161
|
+
mock.restoreAll();
|
|
162
|
+
close();
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("reads the Message-ID a real sync stored", async () => {
|
|
166
|
+
const messageId = await syncInbox("<report@example.com>");
|
|
167
|
+
|
|
168
|
+
assert.equal(await probeable(messageId), "<report@example.com>");
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it("reads the source's Message-ID for a copy", async () => {
|
|
172
|
+
const sourceId = await syncInbox("<report@example.com>");
|
|
173
|
+
const archive = await createMailbox("Archive");
|
|
174
|
+
mock.method(SQSClient.prototype, "send", async () => ({}));
|
|
175
|
+
|
|
176
|
+
const copyId = await new MessageMoveService({
|
|
177
|
+
messageService: messages,
|
|
178
|
+
mailboxService: mailboxes,
|
|
179
|
+
mailboxSpecialUseService: repos.mailboxSpecialUse,
|
|
180
|
+
threadMessageService: threadMessages,
|
|
181
|
+
addressService: repos.address,
|
|
182
|
+
sqsQueueUrl: "https://queue.test/messages",
|
|
183
|
+
}).copyMessage(ACCOUNT_CONFIG_ID, sourceId, archive.mailboxId, accountId);
|
|
184
|
+
|
|
185
|
+
assert.notEqual(copyId, sourceId);
|
|
186
|
+
assert.equal(await probeable(copyId), "<report@example.com>");
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("answers unprobeable for a bounce whose Message-ID is <>", async () => {
|
|
190
|
+
const messageId = await syncInbox("<>");
|
|
191
|
+
|
|
192
|
+
assert.equal(await probeable(messageId), undefined);
|
|
193
|
+
});
|
|
194
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { IThreadMessageRepository } from "@remit/data-ports";
|
|
2
|
+
import { isValidMessageId } from "@remit/data-ports/id";
|
|
3
|
+
|
|
4
|
+
export const probeableMessageIdHeader = async (
|
|
5
|
+
threadMessageService: Pick<IThreadMessageRepository, "findByMessageId">,
|
|
6
|
+
accountConfigId: string,
|
|
7
|
+
messageId: string,
|
|
8
|
+
): Promise<string | undefined> => {
|
|
9
|
+
const row = await threadMessageService.findByMessageId(
|
|
10
|
+
accountConfigId,
|
|
11
|
+
messageId,
|
|
12
|
+
);
|
|
13
|
+
const header = row?.messageIdHeader;
|
|
14
|
+
return isValidMessageId(header) ? header : undefined;
|
|
15
|
+
};
|