@remit/imap-worker 0.0.64 → 0.0.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/append-sent-message.ts +2 -15
- package/src/handlers/flag-push.ts +2 -16
- package/src/handlers/message-copy.test.ts +19 -0
- package/src/handlers/message-copy.ts +11 -18
- package/src/handlers/message-delete.test.ts +27 -3
- package/src/handlers/message-delete.ts +15 -26
- package/src/handlers/message-move.test.ts +45 -6
- package/src/handlers/message-move.ts +26 -22
- package/src/handlers/placement-move-push.test.ts +22 -0
- package/src/handlers/placement-move-push.ts +12 -19
- package/src/handlers/sync-message-body.ts +2 -29
package/package.json
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
buildMailMessage,
|
|
11
11
|
renderRawMessage,
|
|
12
12
|
} from "@remit/smtp-service/message-builder";
|
|
13
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
13
14
|
import { isAccountDeleted } from "../account-check.js";
|
|
14
15
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
15
16
|
import type { AppendSentMessageEvent } from "../events.js";
|
|
@@ -29,23 +30,9 @@ const ROW_SURVIVED_ITS_DELETE =
|
|
|
29
30
|
const unfiledAppendRefused = (fullPath: string, error: unknown): string =>
|
|
30
31
|
`Sent, but not filed: the mail server refused to store a copy in ${fullPath} (${error instanceof Error ? error.message : String(error)}).`;
|
|
31
32
|
|
|
32
|
-
/**
|
|
33
|
-
* Fallback when `APPEND_SENT_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
34
|
-
* Matches the shared `MAX_RECEIVE_COUNT` every queue's redrive policy uses,
|
|
35
|
-
* same pattern as `FLAG_PUSH_MAX_ATTEMPTS` / `BODY_SYNC_MAX_ATTEMPTS`.
|
|
36
|
-
*/
|
|
37
|
-
const DEFAULT_APPEND_SENT_MAX_ATTEMPTS = 3;
|
|
38
|
-
|
|
39
33
|
export const getAppendSentMaxAttempts = (
|
|
40
34
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
41
|
-
): number =>
|
|
42
|
-
const raw = processEnv.APPEND_SENT_MAX_ATTEMPTS;
|
|
43
|
-
if (!raw) return DEFAULT_APPEND_SENT_MAX_ATTEMPTS;
|
|
44
|
-
const parsed = Number.parseInt(raw, 10);
|
|
45
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
46
|
-
? parsed
|
|
47
|
-
: DEFAULT_APPEND_SENT_MAX_ATTEMPTS;
|
|
48
|
-
};
|
|
35
|
+
): number => attemptBudget("APPEND_SENT_MAX_ATTEMPTS", 3, processEnv);
|
|
49
36
|
|
|
50
37
|
export const APPEND_SENT_MAX_ATTEMPTS = getAppendSentMaxAttempts();
|
|
51
38
|
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
MailboxCursorPausedError,
|
|
9
9
|
resolveExhaustedFlagPushFailure,
|
|
10
10
|
} from "@remit/mailbox-service";
|
|
11
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
11
12
|
import { isAccountDeleted } from "../account-check.js";
|
|
12
13
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
13
14
|
import type { FlagPushEvent } from "../events.js";
|
|
@@ -15,24 +16,9 @@ import { isNotFoundError } from "../is-not-found.js";
|
|
|
15
16
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
16
17
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
17
18
|
|
|
18
|
-
/**
|
|
19
|
-
* Fallback when `FLAG_PUSH_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
20
|
-
* Matches the shared `MAX_RECEIVE_COUNT` every queue's redrive policy uses
|
|
21
|
-
* (`infra/stacks/dev/stacks/remit-queue-stack.ts`), same pattern as
|
|
22
|
-
* `BODY_SYNC_MAX_ATTEMPTS` (#1270) / `PLACEMENT_MOVE_MAX_ATTEMPTS` (#1289).
|
|
23
|
-
*/
|
|
24
|
-
const DEFAULT_FLAG_PUSH_MAX_ATTEMPTS = 3;
|
|
25
|
-
|
|
26
19
|
export const getFlagPushMaxAttempts = (
|
|
27
20
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
28
|
-
): number =>
|
|
29
|
-
const raw = processEnv.FLAG_PUSH_MAX_ATTEMPTS;
|
|
30
|
-
if (!raw) return DEFAULT_FLAG_PUSH_MAX_ATTEMPTS;
|
|
31
|
-
const parsed = Number.parseInt(raw, 10);
|
|
32
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
33
|
-
? parsed
|
|
34
|
-
: DEFAULT_FLAG_PUSH_MAX_ATTEMPTS;
|
|
35
|
-
};
|
|
21
|
+
): number => attemptBudget("FLAG_PUSH_MAX_ATTEMPTS", 3, processEnv);
|
|
36
22
|
|
|
37
23
|
export const FLAG_PUSH_MAX_ATTEMPTS = getFlagPushMaxAttempts();
|
|
38
24
|
|
|
@@ -229,6 +229,25 @@ describe("handleMessageCopy", () => {
|
|
|
229
229
|
assert.equal(called("threadMessage.update").length, 1);
|
|
230
230
|
});
|
|
231
231
|
|
|
232
|
+
// Issue #1122. `deriveCopyMessageId` reuses the row, so copying the same
|
|
233
|
+
// message into the same folder twice issues a second server COPY and leaves
|
|
234
|
+
// the destination holding both. Nothing here can rule the older one out the
|
|
235
|
+
// way the trash probe's source check does — a COPY leaves the source
|
|
236
|
+
// untouched — so the row must settle on the highest match. Settling on 12
|
|
237
|
+
// points it at the earlier copy and orphans the one just made.
|
|
238
|
+
it("settles on the copy this event made, not on an earlier copy of the same message", async () => {
|
|
239
|
+
h.connection.copyMessages = async () => ({ uidMap: new Map() });
|
|
240
|
+
h.destinationHolds = [12, 42];
|
|
241
|
+
|
|
242
|
+
await handleMessageCopy(event, noopLog, 1, deps());
|
|
243
|
+
|
|
244
|
+
assert.deepEqual(called("message.updateUid")[0]?.args, [
|
|
245
|
+
"new-msg",
|
|
246
|
+
42,
|
|
247
|
+
"dst-mbx",
|
|
248
|
+
]);
|
|
249
|
+
});
|
|
250
|
+
|
|
232
251
|
it("reconciles the optimistic row away when the destination does not hold the copy", async () => {
|
|
233
252
|
h.connection.copyMessages = async () => ({ uidMap: new Map() });
|
|
234
253
|
h.destinationHolds = [];
|
|
@@ -9,13 +9,14 @@ import {
|
|
|
9
9
|
MailboxCursorPausedError,
|
|
10
10
|
reconcileStaleMessage,
|
|
11
11
|
} from "@remit/mailbox-service";
|
|
12
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
12
13
|
import { isAccountDeleted } from "../account-check.js";
|
|
13
14
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
14
15
|
import type { MessageCopyEvent } from "../events.js";
|
|
15
16
|
import { isNotFoundError } from "../is-not-found.js";
|
|
16
17
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
17
18
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
18
|
-
import {
|
|
19
|
+
import { searchMailboxForHighestMessageIdUid } from "./message-move.js";
|
|
19
20
|
|
|
20
21
|
export interface MessageCopyDeps {
|
|
21
22
|
getClient: typeof getClient;
|
|
@@ -31,24 +32,9 @@ const defaultDeps: MessageCopyDeps = {
|
|
|
31
32
|
createConnectionScope: createConnectionScopeWithCredentials,
|
|
32
33
|
};
|
|
33
34
|
|
|
34
|
-
/**
|
|
35
|
-
* Fallback when `MESSAGE_COPY_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
36
|
-
* Matches the `maxReceiveCount` the message queue's redrive policy uses
|
|
37
|
-
* (`remit-messages.fifo`, `deploy/vps/queues.json`), same pattern as
|
|
38
|
-
* `MESSAGE_MOVE_MAX_ATTEMPTS`.
|
|
39
|
-
*/
|
|
40
|
-
const DEFAULT_MESSAGE_COPY_MAX_ATTEMPTS = 3;
|
|
41
|
-
|
|
42
35
|
export const getMessageCopyMaxAttempts = (
|
|
43
36
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
44
|
-
): number =>
|
|
45
|
-
const raw = processEnv.MESSAGE_COPY_MAX_ATTEMPTS;
|
|
46
|
-
if (!raw) return DEFAULT_MESSAGE_COPY_MAX_ATTEMPTS;
|
|
47
|
-
const parsed = Number.parseInt(raw, 10);
|
|
48
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
49
|
-
? parsed
|
|
50
|
-
: DEFAULT_MESSAGE_COPY_MAX_ATTEMPTS;
|
|
51
|
-
};
|
|
37
|
+
): number => attemptBudget("MESSAGE_COPY_MAX_ATTEMPTS", 3, processEnv);
|
|
52
38
|
|
|
53
39
|
export const MESSAGE_COPY_MAX_ATTEMPTS = getMessageCopyMaxAttempts();
|
|
54
40
|
|
|
@@ -202,11 +188,18 @@ export const handleMessageCopy = async (
|
|
|
202
188
|
// handle: a `guardConnectionCursor` wrap is bound to the SOURCE mailbox
|
|
203
189
|
// snapshot alone, and opening the destination through it trips that
|
|
204
190
|
// mailbox into a rebuild.
|
|
191
|
+
//
|
|
192
|
+
// This is the call site with no source-side gate to fall back on
|
|
193
|
+
// (#1122): a COPY leaves the source untouched, so nothing here can rule
|
|
194
|
+
// an older destination copy out the way `confirmTrashMoveUid` does. The
|
|
195
|
+
// fresh copy is the highest matching uid, and `deriveCopyMessageId`
|
|
196
|
+
// makes a repeated copy of the same message a second server COPY, so
|
|
197
|
+
// this probe must never bind to the earlier one.
|
|
205
198
|
const probeDestination = async (
|
|
206
199
|
connection: IImapConnection,
|
|
207
200
|
): Promise<CopyProbe> => {
|
|
208
201
|
if (!copyRow.messageIdHeader) return { kind: "unprobeable" };
|
|
209
|
-
const probedUid = await
|
|
202
|
+
const probedUid = await searchMailboxForHighestMessageIdUid(
|
|
210
203
|
connection,
|
|
211
204
|
destinationMailboxPath,
|
|
212
205
|
copyRow.messageIdHeader,
|
|
@@ -516,6 +516,30 @@ describe("handleMessageDelete", () => {
|
|
|
516
516
|
);
|
|
517
517
|
});
|
|
518
518
|
|
|
519
|
+
// Issue #1122. Trash already holds an older copy of this Message-ID (a
|
|
520
|
+
// resend, a sieve `fileinto` + `keep`, an earlier delete of another
|
|
521
|
+
// folder's copy). Settling on uid 12 points the row at that copy and
|
|
522
|
+
// orphans the one this MOVE just delivered: Empty Trash then expunges by
|
|
523
|
+
// the wrong uid and the fresh copy survives, unreachable.
|
|
524
|
+
it("settles on the copy this move delivered, not on an older copy already in Trash", async () => {
|
|
525
|
+
h.connection.moveMessages = async () => ({ uidMap: new Map() });
|
|
526
|
+
sourceNoLongerHoldsTheUid();
|
|
527
|
+
h.destinationSearchUids = [12, 77];
|
|
528
|
+
|
|
529
|
+
await handleMessageDelete(moveEvent, noopLog, 1, deps());
|
|
530
|
+
|
|
531
|
+
assert.deepEqual(called("message.updateUid")[0]?.args, [
|
|
532
|
+
"msg-1",
|
|
533
|
+
77,
|
|
534
|
+
"trash-mbx",
|
|
535
|
+
]);
|
|
536
|
+
assert.deepEqual(called("threadMessage.update")[0]?.args[2], {
|
|
537
|
+
uid: 77,
|
|
538
|
+
mailboxId: "trash-mbx",
|
|
539
|
+
isDeleted: true,
|
|
540
|
+
});
|
|
541
|
+
});
|
|
542
|
+
|
|
519
543
|
it("probes the DESTINATION mailbox, read-only, by Message-ID", async () => {
|
|
520
544
|
h.connection.moveMessages = async () => ({ uidMap: new Map() });
|
|
521
545
|
sourceNoLongerHoldsTheUid();
|
|
@@ -650,11 +674,11 @@ describe("handleMessageDelete", () => {
|
|
|
650
674
|
assert.equal(called("emitEvent").length, 0);
|
|
651
675
|
});
|
|
652
676
|
|
|
653
|
-
//
|
|
654
|
-
// Message-ID can have several server copies in one account while
|
|
677
|
+
// One Message-ID can have several server copies in one account while
|
|
655
678
|
// `deriveMessageId` gives them one local row. A source that still holds
|
|
656
679
|
// the uid proves the MOVE did not happen, so any hit at the destination
|
|
657
|
-
// is a different copy
|
|
680
|
+
// is a different copy — including the highest one #1122 now takes, since
|
|
681
|
+
// with no fresh copy at all the highest match is still an older copy.
|
|
658
682
|
it("never takes a destination hit while the source still holds the uid (duplicate Message-ID)", async () => {
|
|
659
683
|
h.connection.moveMessages = async () => ({ uidMap: new Map() });
|
|
660
684
|
h.destinationSearchUids = [100];
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
isMessageGoneFromOpenMailbox,
|
|
15
15
|
MailboxCursorPausedError,
|
|
16
16
|
} from "@remit/mailbox-service";
|
|
17
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
17
18
|
import { isAccountDeleted } from "../account-check.js";
|
|
18
19
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
19
20
|
import { emitEvent } from "../emit.js";
|
|
@@ -22,30 +23,18 @@ import { isNotFoundError } from "../is-not-found.js";
|
|
|
22
23
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
23
24
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
24
25
|
import { resolveExhaustedMessageDeleteFailure } from "./message-delete-terminal.js";
|
|
25
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
emitMoveResync,
|
|
28
|
+
searchMailboxForHighestMessageIdUid,
|
|
29
|
+
} from "./message-move.js";
|
|
26
30
|
import {
|
|
27
31
|
buildThreadMessageMoveRevert,
|
|
28
32
|
buildThreadMessageTrashUpdate,
|
|
29
33
|
} from "./thread-message-rows.js";
|
|
30
34
|
|
|
31
|
-
/**
|
|
32
|
-
* Fallback when `MESSAGE_DELETE_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
33
|
-
* Matches the `maxReceiveCount` the redrive policy of the queue `emit.ts`
|
|
34
|
-
* routes MESSAGE_DELETE onto uses (`remit-message-mgmt`,
|
|
35
|
-
* `deploy/vps/queues.json`), same pattern as `MESSAGE_MOVE_MAX_ATTEMPTS`.
|
|
36
|
-
*/
|
|
37
|
-
const DEFAULT_MESSAGE_DELETE_MAX_ATTEMPTS = 3;
|
|
38
|
-
|
|
39
35
|
export const getMessageDeleteMaxAttempts = (
|
|
40
36
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
41
|
-
): number =>
|
|
42
|
-
const raw = processEnv.MESSAGE_DELETE_MAX_ATTEMPTS;
|
|
43
|
-
if (!raw) return DEFAULT_MESSAGE_DELETE_MAX_ATTEMPTS;
|
|
44
|
-
const parsed = Number.parseInt(raw, 10);
|
|
45
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
46
|
-
? parsed
|
|
47
|
-
: DEFAULT_MESSAGE_DELETE_MAX_ATTEMPTS;
|
|
48
|
-
};
|
|
37
|
+
): number => attemptBudget("MESSAGE_DELETE_MAX_ATTEMPTS", 3, processEnv);
|
|
49
38
|
|
|
50
39
|
export const MESSAGE_DELETE_MAX_ATTEMPTS = getMessageDeleteMaxAttempts();
|
|
51
40
|
|
|
@@ -84,14 +73,14 @@ export const deleteAllThreadMessagesForMessage = async (
|
|
|
84
73
|
* never be opened through the source's guard.
|
|
85
74
|
*
|
|
86
75
|
* The source is asked first, and a source that still holds the uid ends it: the
|
|
87
|
-
* MOVE did not happen, so nothing at the destination can be this message.
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
* `fileinto` + `keep`, a
|
|
92
|
-
* `deriveMessageId`
|
|
93
|
-
* ungated probe
|
|
94
|
-
* expunges by
|
|
76
|
+
* MOVE did not happen, so nothing at the destination can be this message. The
|
|
77
|
+
* gate survives #1122's switch to the highest matching uid and is not made
|
|
78
|
+
* redundant by it — highest picks the fresh copy out of several only once a
|
|
79
|
+
* fresh copy exists, and where the MOVE never ran every hit at the destination
|
|
80
|
+
* is an older copy of the same Message-ID (a sieve `fileinto` + `keep`, a
|
|
81
|
+
* multi-label store, a resend) that `deriveMessageId` folds into this one local
|
|
82
|
+
* row. An ungated probe hands back that copy's uid, and Empty Trash then
|
|
83
|
+
* expunges by it. The gate also closes the second half of #912: an empty
|
|
95
84
|
* `uidMap` can mean the MOVE matched nothing at all.
|
|
96
85
|
*
|
|
97
86
|
* A row with no `messageIdHeader`, and a row that is already deleted, have
|
|
@@ -123,7 +112,7 @@ const confirmTrashMoveUid = async (
|
|
|
123
112
|
if (!message) return { outcome: "row-gone" };
|
|
124
113
|
if (!message.messageIdHeader) return { outcome: "unprobeable" };
|
|
125
114
|
|
|
126
|
-
const probedUid = await
|
|
115
|
+
const probedUid = await searchMailboxForHighestMessageIdUid(
|
|
127
116
|
destinationConnection,
|
|
128
117
|
destinationMailboxPath,
|
|
129
118
|
message.messageIdHeader,
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
handleMessageMove,
|
|
13
13
|
MESSAGE_MOVE_MAX_ATTEMPTS,
|
|
14
14
|
moveThenResync,
|
|
15
|
-
|
|
15
|
+
searchMailboxForHighestMessageIdUid,
|
|
16
16
|
} from "./message-move.js";
|
|
17
17
|
|
|
18
18
|
const silentLogger = (() => {
|
|
@@ -343,7 +343,7 @@ describe("handleMessageMove — the move's own pending state gates every attempt
|
|
|
343
343
|
});
|
|
344
344
|
});
|
|
345
345
|
|
|
346
|
-
describe("
|
|
346
|
+
describe("searchMailboxForHighestMessageIdUid — the probe that binds a move to a UID (#912)", () => {
|
|
347
347
|
const isMessageIdCriterion = (
|
|
348
348
|
criterion: unknown,
|
|
349
349
|
): criterion is [string, string, string] =>
|
|
@@ -381,7 +381,7 @@ describe("searchMailboxByMessageId — the probe that binds a move to a UID (#91
|
|
|
381
381
|
},
|
|
382
382
|
} as unknown as IImapConnection;
|
|
383
383
|
|
|
384
|
-
await
|
|
384
|
+
await searchMailboxForHighestMessageIdUid(
|
|
385
385
|
destination,
|
|
386
386
|
"Archive",
|
|
387
387
|
'<a"b@example.com>\r\nUID 1',
|
|
@@ -392,14 +392,14 @@ describe("searchMailboxByMessageId — the probe that binds a move to a UID (#91
|
|
|
392
392
|
]);
|
|
393
393
|
});
|
|
394
394
|
|
|
395
|
-
it("answers null, not
|
|
395
|
+
it("answers null, not another message's UID, when the folder holds no such message", async () => {
|
|
396
396
|
const destination = buildDestination([
|
|
397
397
|
{ uid: 11, messageIdHeader: "<stranger-a@example.com>" },
|
|
398
398
|
{ uid: 12, messageIdHeader: "<stranger-b@example.com>" },
|
|
399
399
|
]);
|
|
400
400
|
|
|
401
401
|
assert.strictEqual(
|
|
402
|
-
await
|
|
402
|
+
await searchMailboxForHighestMessageIdUid(
|
|
403
403
|
destination,
|
|
404
404
|
"Archive",
|
|
405
405
|
"<moved@example.com>",
|
|
@@ -415,7 +415,7 @@ describe("searchMailboxByMessageId — the probe that binds a move to a UID (#91
|
|
|
415
415
|
]);
|
|
416
416
|
|
|
417
417
|
assert.strictEqual(
|
|
418
|
-
await
|
|
418
|
+
await searchMailboxForHighestMessageIdUid(
|
|
419
419
|
destination,
|
|
420
420
|
"Archive",
|
|
421
421
|
"<moved@example.com>",
|
|
@@ -423,4 +423,43 @@ describe("searchMailboxByMessageId — the probe that binds a move to a UID (#91
|
|
|
423
423
|
12,
|
|
424
424
|
);
|
|
425
425
|
});
|
|
426
|
+
|
|
427
|
+
// Issue #1122. The folder already held an older copy of this Message-ID — a
|
|
428
|
+
// sieve `fileinto` + `keep`, a resend, an earlier copy of the same message —
|
|
429
|
+
// and the uid this probe hands back settles the row the fresh copy lives in.
|
|
430
|
+
// Answering 12 binds that row to mail this operation never touched and
|
|
431
|
+
// leaves the copy that just arrived unreachable by any later delete.
|
|
432
|
+
it("answers the newest copy's UID when the folder already held an older copy of it", async () => {
|
|
433
|
+
const destination = buildDestination([
|
|
434
|
+
{ uid: 12, messageIdHeader: "<moved@example.com>" },
|
|
435
|
+
{ uid: 40, messageIdHeader: "<moved@example.com>" },
|
|
436
|
+
]);
|
|
437
|
+
|
|
438
|
+
assert.strictEqual(
|
|
439
|
+
await searchMailboxForHighestMessageIdUid(
|
|
440
|
+
destination,
|
|
441
|
+
"Archive",
|
|
442
|
+
"<moved@example.com>",
|
|
443
|
+
),
|
|
444
|
+
40,
|
|
445
|
+
);
|
|
446
|
+
});
|
|
447
|
+
|
|
448
|
+
// RFC 3501 does not order a SEARCH response, so the fresh copy is the
|
|
449
|
+
// highest uid returned rather than the last one returned.
|
|
450
|
+
it("takes the highest match whatever order the server lists them in", async () => {
|
|
451
|
+
const destination = buildDestination([
|
|
452
|
+
{ uid: 40, messageIdHeader: "<moved@example.com>" },
|
|
453
|
+
{ uid: 12, messageIdHeader: "<moved@example.com>" },
|
|
454
|
+
]);
|
|
455
|
+
|
|
456
|
+
assert.strictEqual(
|
|
457
|
+
await searchMailboxForHighestMessageIdUid(
|
|
458
|
+
destination,
|
|
459
|
+
"Archive",
|
|
460
|
+
"<moved@example.com>",
|
|
461
|
+
),
|
|
462
|
+
40,
|
|
463
|
+
);
|
|
464
|
+
});
|
|
426
465
|
});
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
isPlacementUnsettled,
|
|
11
11
|
MailboxCursorPausedError,
|
|
12
12
|
} from "@remit/mailbox-service";
|
|
13
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
13
14
|
import { isAccountDeleted } from "../account-check.js";
|
|
14
15
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
15
16
|
import { emitEvent } from "../emit.js";
|
|
@@ -19,24 +20,9 @@ import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
|
19
20
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
20
21
|
import { resolveExhaustedMessageMoveFailure } from "./message-move-terminal.js";
|
|
21
22
|
|
|
22
|
-
/**
|
|
23
|
-
* Fallback when `MESSAGE_MOVE_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
24
|
-
* Matches the `maxReceiveCount` the message queue's redrive policy uses
|
|
25
|
-
* (`remit-messages.fifo`, `deploy/vps/queues.json`), same pattern as
|
|
26
|
-
* `FLAG_PUSH_MAX_ATTEMPTS` and `PLACEMENT_MOVE_MAX_ATTEMPTS`.
|
|
27
|
-
*/
|
|
28
|
-
const DEFAULT_MESSAGE_MOVE_MAX_ATTEMPTS = 3;
|
|
29
|
-
|
|
30
23
|
export const getMessageMoveMaxAttempts = (
|
|
31
24
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
32
|
-
): number =>
|
|
33
|
-
const raw = processEnv.MESSAGE_MOVE_MAX_ATTEMPTS;
|
|
34
|
-
if (!raw) return DEFAULT_MESSAGE_MOVE_MAX_ATTEMPTS;
|
|
35
|
-
const parsed = Number.parseInt(raw, 10);
|
|
36
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
37
|
-
? parsed
|
|
38
|
-
: DEFAULT_MESSAGE_MOVE_MAX_ATTEMPTS;
|
|
39
|
-
};
|
|
25
|
+
): number => attemptBudget("MESSAGE_MOVE_MAX_ATTEMPTS", 3, processEnv);
|
|
40
26
|
|
|
41
27
|
export const MESSAGE_MOVE_MAX_ATTEMPTS = getMessageMoveMaxAttempts();
|
|
42
28
|
|
|
@@ -67,11 +53,28 @@ export const emitMoveResync = async (
|
|
|
67
53
|
};
|
|
68
54
|
|
|
69
55
|
/**
|
|
70
|
-
* SEARCH a mailbox for a message by its RFC822 Message-ID
|
|
71
|
-
*
|
|
72
|
-
*
|
|
56
|
+
* SEARCH a mailbox for copies of a message identified by its RFC822 Message-ID
|
|
57
|
+
* header and answer the HIGHEST matching uid. Read-only (EXAMINE, not SELECT) —
|
|
58
|
+
* this is a verification probe, never a write. `null` when nothing matched.
|
|
59
|
+
*
|
|
60
|
+
* The highest uid is the freshly delivered copy only when one exists; deciding
|
|
61
|
+
* that it does is the caller's job, not this function's. Every probe in this
|
|
62
|
+
* package wants the highest: uids ascend with arrival inside a UIDVALIDITY, so a
|
|
63
|
+
* copy that has just landed outranks every copy of the same Message-ID the
|
|
64
|
+
* folder already held. Where nothing was delivered, the highest is simply the
|
|
65
|
+
* newest pre-existing copy, and a caller that has not ruled that case out binds
|
|
66
|
+
* itself to unrelated mail.
|
|
67
|
+
* One Message-ID can have several server copies in one account (a sieve
|
|
68
|
+
* `fileinto` + `keep`, a resend, a repeated COPY — `deriveCopyMessageId` reuses
|
|
69
|
+
* the row, so the second copy is a second server COPY), while `deriveMessageId`
|
|
70
|
+
* is folder-independent and gives them one local row. Taking the lowest bound
|
|
71
|
+
* the fresh copy's row to an older copy's uid and orphaned the fresh one, which
|
|
72
|
+
* a later delete or Empty Trash then expunged by the wrong uid (issue #1122).
|
|
73
|
+
*
|
|
74
|
+
* The maximum is computed rather than read off the tail: RFC 3501 leaves the
|
|
75
|
+
* order of a SEARCH response unspecified, so "last returned" is not "highest".
|
|
73
76
|
*/
|
|
74
|
-
export const
|
|
77
|
+
export const searchMailboxForHighestMessageIdUid = async (
|
|
75
78
|
connection: IImapConnection,
|
|
76
79
|
mailboxPath: string,
|
|
77
80
|
messageIdHeader: string,
|
|
@@ -80,7 +83,8 @@ export const searchMailboxByMessageId = async (
|
|
|
80
83
|
const uids = await connection.search([
|
|
81
84
|
["HEADER", "Message-ID", messageIdHeader],
|
|
82
85
|
]);
|
|
83
|
-
|
|
86
|
+
if (uids.length === 0) return null;
|
|
87
|
+
return uids.reduce((highest, uid) => (uid > highest ? uid : highest));
|
|
84
88
|
};
|
|
85
89
|
|
|
86
90
|
/**
|
|
@@ -287,7 +291,7 @@ export const handleMessageMove = async (
|
|
|
287
291
|
const newUid =
|
|
288
292
|
result.uidMap.get(uid) ??
|
|
289
293
|
(message.messageIdHeader
|
|
290
|
-
? await
|
|
294
|
+
? await searchMailboxForHighestMessageIdUid(
|
|
291
295
|
rawConnection,
|
|
292
296
|
destinationMailboxPath,
|
|
293
297
|
message.messageIdHeader,
|
|
@@ -156,6 +156,28 @@ describe("attemptMove — the IMAP push (issue #1271)", () => {
|
|
|
156
156
|
assert.equal(outcome.newUid, 77);
|
|
157
157
|
});
|
|
158
158
|
|
|
159
|
+
// Issue #1122. The destination already held an older copy of this
|
|
160
|
+
// Message-ID; the marker must bind to the copy this MOVE delivered, or
|
|
161
|
+
// the placement row names mail this push never touched.
|
|
162
|
+
it("moved: binds to the newest copy when the destination already held an older one", async () => {
|
|
163
|
+
const connection = buildConnection({
|
|
164
|
+
uidMap: new Map(),
|
|
165
|
+
destinationSearchUids: [15, 77],
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
const outcome = await attemptMove(
|
|
169
|
+
connection,
|
|
170
|
+
connection,
|
|
171
|
+
"Junk",
|
|
172
|
+
"INBOX",
|
|
173
|
+
42,
|
|
174
|
+
MESSAGE_ID_HEADER,
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
assert.equal(outcome.kind, "moved");
|
|
178
|
+
assert.equal(outcome.newUid, 77);
|
|
179
|
+
});
|
|
180
|
+
|
|
159
181
|
it("throws (never deletes) when unconfirmed at the destination but STILL present at the source", async () => {
|
|
160
182
|
const connection = buildConnection({
|
|
161
183
|
uidMap: new Map(),
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
reconcileStaleMessage,
|
|
11
11
|
resolveExhaustedPlacementMoveFailure,
|
|
12
12
|
} from "@remit/mailbox-service";
|
|
13
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
13
14
|
import { isAccountDeleted } from "../account-check.js";
|
|
14
15
|
import { createConnectionScopeWithCredentials } from "../connection-scope.js";
|
|
15
16
|
import { emitEvent } from "../emit.js";
|
|
@@ -20,27 +21,12 @@ import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
|
20
21
|
import {
|
|
21
22
|
buildThreadMessageMoveUpdate,
|
|
22
23
|
emitMoveResync,
|
|
23
|
-
|
|
24
|
+
searchMailboxForHighestMessageIdUid,
|
|
24
25
|
} from "./message-move.js";
|
|
25
26
|
|
|
26
|
-
/**
|
|
27
|
-
* Fallback when `PLACEMENT_MOVE_MAX_ATTEMPTS` is unset (local dev, unit
|
|
28
|
-
* tests). Matches the placement-move queue's own `MAX_RECEIVE_COUNT` default
|
|
29
|
-
* (`infra/stacks/dev/stacks/remit-queue-stack.ts`), same pattern as
|
|
30
|
-
* `BODY_SYNC_MAX_ATTEMPTS` (#1270).
|
|
31
|
-
*/
|
|
32
|
-
const DEFAULT_PLACEMENT_MOVE_MAX_ATTEMPTS = 3;
|
|
33
|
-
|
|
34
27
|
export const getPlacementMoveMaxAttempts = (
|
|
35
28
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
36
|
-
): number =>
|
|
37
|
-
const raw = processEnv.PLACEMENT_MOVE_MAX_ATTEMPTS;
|
|
38
|
-
if (!raw) return DEFAULT_PLACEMENT_MOVE_MAX_ATTEMPTS;
|
|
39
|
-
const parsed = Number.parseInt(raw, 10);
|
|
40
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
41
|
-
? parsed
|
|
42
|
-
: DEFAULT_PLACEMENT_MOVE_MAX_ATTEMPTS;
|
|
43
|
-
};
|
|
29
|
+
): number => attemptBudget("PLACEMENT_MOVE_MAX_ATTEMPTS", 3, processEnv);
|
|
44
30
|
|
|
45
31
|
export const PLACEMENT_MOVE_MAX_ATTEMPTS = getPlacementMoveMaxAttempts();
|
|
46
32
|
|
|
@@ -111,9 +97,16 @@ export const attemptMove = async (
|
|
|
111
97
|
}
|
|
112
98
|
|
|
113
99
|
// Unconfirmed: either no COPYUID entry, or the server explicitly claimed
|
|
114
|
-
// "not found" — never trust either without independent verification.
|
|
100
|
+
// "not found" — never trust either without independent verification. A
|
|
101
|
+
// message this MOVE delivered is the newest copy of that Message-ID at the
|
|
102
|
+
// destination, so the probe binds to the highest matching uid (#1122).
|
|
103
|
+
// Unlike `confirmTrashMoveUid`, this probe is ungated: where the MOVE never
|
|
104
|
+
// ran and an older copy filed there earlier is the only match, the marker
|
|
105
|
+
// binds to that unrelated mail. Pre-existing, and not narrowed by taking the
|
|
106
|
+
// highest — the source is not asked first here, so nothing rules the case
|
|
107
|
+
// out.
|
|
115
108
|
if (messageIdHeader) {
|
|
116
|
-
const destinationUid = await
|
|
109
|
+
const destinationUid = await searchMailboxForHighestMessageIdUid(
|
|
117
110
|
destinationConnection,
|
|
118
111
|
destinationMailboxPath,
|
|
119
112
|
messageIdHeader,
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
QuarantineService,
|
|
12
12
|
resolveExhaustedBodySyncFailures,
|
|
13
13
|
} from "@remit/mailbox-service";
|
|
14
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
14
15
|
import { env } from "expect-env";
|
|
15
16
|
import { isAccountDeleted } from "../account-check.js";
|
|
16
17
|
import { isBodySyncEnabled } from "../body-sync-gate.js";
|
|
@@ -27,37 +28,9 @@ import { workerVersion } from "../worker-version.js";
|
|
|
27
28
|
|
|
28
29
|
const bodySyncEnabledParameterName = env.BODY_SYNC_ENABLED_PARAMETER_NAME;
|
|
29
30
|
|
|
30
|
-
/**
|
|
31
|
-
* Fallback when `BODY_SYNC_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
32
|
-
* Matches the body queue's own `MAX_RECEIVE_COUNT` default
|
|
33
|
-
* (`infra/stacks/dev/stacks/remit-queue-stack.ts`) so an environment that
|
|
34
|
-
* never injects the var still behaves like production.
|
|
35
|
-
*/
|
|
36
|
-
const DEFAULT_BODY_SYNC_MAX_ATTEMPTS = 3;
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Reads the redelivery-budget-exhaustion threshold. CDK derives
|
|
40
|
-
* `BODY_SYNC_MAX_ATTEMPTS` from the body queue's own `MAX_RECEIVE_COUNT`
|
|
41
|
-
* (`remit-worker-stack.ts`) so the two constants can't drift apart — a
|
|
42
|
-
* hand-copied duplicate here previously risked the worker resolving
|
|
43
|
-
* "last attempt" on a different delivery than the queue's redrive policy
|
|
44
|
-
* actually uses (issue #1270). SQS's own `ApproximateReceiveCount` is the
|
|
45
|
-
* source of truth for how many times a record has been delivered; once it
|
|
46
|
-
* reaches this value, the current invocation is the last attempt before the
|
|
47
|
-
* queue's own redrive would DLQ the record, so retry exhaustion is resolved
|
|
48
|
-
* here (see `resolveExhaustedBodySyncFailures`) instead of letting the
|
|
49
|
-
* record dead-letter with no diagnosis.
|
|
50
|
-
*/
|
|
51
31
|
export const getBodySyncMaxAttempts = (
|
|
52
32
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
53
|
-
): number =>
|
|
54
|
-
const raw = processEnv.BODY_SYNC_MAX_ATTEMPTS;
|
|
55
|
-
if (!raw) return DEFAULT_BODY_SYNC_MAX_ATTEMPTS;
|
|
56
|
-
const parsed = Number.parseInt(raw, 10);
|
|
57
|
-
return Number.isFinite(parsed) && parsed > 0
|
|
58
|
-
? parsed
|
|
59
|
-
: DEFAULT_BODY_SYNC_MAX_ATTEMPTS;
|
|
60
|
-
};
|
|
33
|
+
): number => attemptBudget("BODY_SYNC_MAX_ATTEMPTS", 3, processEnv);
|
|
61
34
|
|
|
62
35
|
export const BODY_SYNC_MAX_ATTEMPTS = getBodySyncMaxAttempts();
|
|
63
36
|
|