@remit/imap-worker 0.0.49 → 0.0.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/flag-push.ts +4 -4
- package/src/handlers/message-move-terminal.test.ts +222 -0
- package/src/handlers/message-move-terminal.ts +122 -0
- package/src/handlers/message-move.test.ts +89 -2
- package/src/handlers/message-move.ts +175 -62
- package/src/handlers/placement-move-push.ts +1 -17
- package/src/processor.ts +6 -7
package/package.json
CHANGED
|
@@ -39,10 +39,10 @@ export const FLAG_PUSH_MAX_ATTEMPTS = getFlagPushMaxAttempts();
|
|
|
39
39
|
/**
|
|
40
40
|
* How long a marker may sit deferred behind a move before it is dropped
|
|
41
41
|
* outright. A move that settles takes seconds to low minutes; one stuck past
|
|
42
|
-
* this window has almost certainly already exhausted its own retries
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
42
|
+
* this window has almost certainly already exhausted its own retries and been
|
|
43
|
+
* resolved as broken by its own terminal resolver, so deferring further would
|
|
44
|
+
* cycle one SQS round trip per sync tick forever instead of surfacing the
|
|
45
|
+
* stall.
|
|
46
46
|
*/
|
|
47
47
|
const DEFAULT_FLAG_PUSH_DEFER_MAX_MS = 10 * 60 * 1000;
|
|
48
48
|
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import type {
|
|
4
|
+
IMessageRepository,
|
|
5
|
+
IThreadMessageRepository,
|
|
6
|
+
} from "@remit/data-ports";
|
|
7
|
+
import type { IImapConnection } from "@remit/mailbox-service";
|
|
8
|
+
import {
|
|
9
|
+
type MessageMoveTerminalLogger,
|
|
10
|
+
type ResolveExhaustedMessageMoveDeps,
|
|
11
|
+
resolveExhaustedMessageMoveFailure,
|
|
12
|
+
} from "./message-move-terminal.js";
|
|
13
|
+
|
|
14
|
+
interface LogEntry {
|
|
15
|
+
obj: Record<string, unknown>;
|
|
16
|
+
msg: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const buildLogger = (): {
|
|
20
|
+
log: MessageMoveTerminalLogger;
|
|
21
|
+
infos: LogEntry[];
|
|
22
|
+
errors: LogEntry[];
|
|
23
|
+
} => {
|
|
24
|
+
const infos: LogEntry[] = [];
|
|
25
|
+
const errors: LogEntry[] = [];
|
|
26
|
+
return {
|
|
27
|
+
log: {
|
|
28
|
+
info: (obj, msg) => infos.push({ obj, msg }),
|
|
29
|
+
error: (obj, msg) => errors.push({ obj, msg }),
|
|
30
|
+
},
|
|
31
|
+
infos,
|
|
32
|
+
errors,
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const buildConnection = (
|
|
37
|
+
present: Set<number>,
|
|
38
|
+
fetchDrops: Set<number> = new Set(),
|
|
39
|
+
): IImapConnection =>
|
|
40
|
+
({
|
|
41
|
+
openBox: async () => ({}) as never,
|
|
42
|
+
fetchMessages: async (uids: number[]) =>
|
|
43
|
+
uids
|
|
44
|
+
.filter((uid) => present.has(uid) && !fetchDrops.has(uid))
|
|
45
|
+
.map((uid) => ({ uid }) as unknown as never),
|
|
46
|
+
search: async (criteria: unknown[]) => {
|
|
47
|
+
const [, value] = (criteria as Array<[string, string]>)[0];
|
|
48
|
+
const uid = Number(value);
|
|
49
|
+
return present.has(uid) ? [uid] : [];
|
|
50
|
+
},
|
|
51
|
+
}) as unknown as IImapConnection;
|
|
52
|
+
|
|
53
|
+
interface MessageRow {
|
|
54
|
+
messageId: string;
|
|
55
|
+
mailboxId: string;
|
|
56
|
+
uid: number;
|
|
57
|
+
status: string;
|
|
58
|
+
syncStatus: string;
|
|
59
|
+
originalMailboxId: string;
|
|
60
|
+
originalUid: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The move's pending state IS the Message row, so these fakes hold real rows
|
|
65
|
+
* and the assertions read the rows back — a resolver that reverted the
|
|
66
|
+
* optimistic move (PR #652's defect) would show up here as a changed row, not
|
|
67
|
+
* as an uncalled mock.
|
|
68
|
+
*/
|
|
69
|
+
const buildRepositories = (row: MessageRow) => {
|
|
70
|
+
const messages = new Map<string, MessageRow>([[row.messageId, row]]);
|
|
71
|
+
const threadMessages = new Map<
|
|
72
|
+
string,
|
|
73
|
+
{ accountConfigId: string; threadMessageId: string }
|
|
74
|
+
>([
|
|
75
|
+
[
|
|
76
|
+
`tm-${row.messageId}`,
|
|
77
|
+
{ accountConfigId: "cfg-1", threadMessageId: `tm-${row.messageId}` },
|
|
78
|
+
],
|
|
79
|
+
]);
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
messages,
|
|
83
|
+
threadMessages,
|
|
84
|
+
messageService: {
|
|
85
|
+
delete: async (messageId: string) => {
|
|
86
|
+
messages.delete(messageId);
|
|
87
|
+
},
|
|
88
|
+
update: async (messageId: string, input: Partial<MessageRow>) => {
|
|
89
|
+
const current = messages.get(messageId);
|
|
90
|
+
if (current) messages.set(messageId, { ...current, ...input });
|
|
91
|
+
},
|
|
92
|
+
} as unknown as Pick<IMessageRepository, "delete">,
|
|
93
|
+
threadMessageService: {
|
|
94
|
+
findAllByMessageId: async () => [...threadMessages.values()],
|
|
95
|
+
deleteMany: async (
|
|
96
|
+
keys: Array<{ accountConfigId: string; threadMessageId: string }>,
|
|
97
|
+
) => {
|
|
98
|
+
for (const key of keys) threadMessages.delete(key.threadMessageId);
|
|
99
|
+
},
|
|
100
|
+
} as unknown as Pick<
|
|
101
|
+
IThreadMessageRepository,
|
|
102
|
+
"findAllByMessageId" | "deleteMany"
|
|
103
|
+
>,
|
|
104
|
+
};
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const pendingMoveRow = (): MessageRow => ({
|
|
108
|
+
messageId: "msg-1",
|
|
109
|
+
mailboxId: "mbx-archive",
|
|
110
|
+
uid: 101,
|
|
111
|
+
status: "moving",
|
|
112
|
+
syncStatus: "failed",
|
|
113
|
+
originalMailboxId: "mbx-inbox",
|
|
114
|
+
originalUid: 101,
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
const input = {
|
|
118
|
+
accountId: "acc-1",
|
|
119
|
+
accountConfigId: "cfg-1",
|
|
120
|
+
messageId: "msg-1",
|
|
121
|
+
uid: 101,
|
|
122
|
+
sourceMailboxPath: "INBOX",
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
describe("resolveExhaustedMessageMoveFailure — the two terminal outcomes (issue #655)", () => {
|
|
126
|
+
it("RECONCILED: the message is gone from the move's source — stale row reconciled, no alarm", async () => {
|
|
127
|
+
const repos = buildRepositories(pendingMoveRow());
|
|
128
|
+
const { log, infos, errors } = buildLogger();
|
|
129
|
+
const deps: ResolveExhaustedMessageMoveDeps = {
|
|
130
|
+
messageService: repos.messageService,
|
|
131
|
+
threadMessageService: repos.threadMessageService,
|
|
132
|
+
log,
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
const result = await resolveExhaustedMessageMoveFailure(deps, {
|
|
136
|
+
...input,
|
|
137
|
+
getConnection: async () => buildConnection(new Set()),
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
assert.equal(result.outcome, "reconciled");
|
|
141
|
+
assert.equal(
|
|
142
|
+
repos.messages.get("msg-1"),
|
|
143
|
+
undefined,
|
|
144
|
+
"the stale Message row is deleted so a resync can re-project it",
|
|
145
|
+
);
|
|
146
|
+
assert.equal(repos.threadMessages.size, 0);
|
|
147
|
+
assert.equal(errors.length, 0, "no alarm for the expected/routine outcome");
|
|
148
|
+
assert.ok(
|
|
149
|
+
infos.some((e) => e.obj.metric === "message_move_stale_row_reconciled"),
|
|
150
|
+
);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("BROKEN: the message is still at the source — the row is left EXACTLY as it stands, alarm logged, never re-thrown", async () => {
|
|
154
|
+
const repos = buildRepositories(pendingMoveRow());
|
|
155
|
+
const { log, errors } = buildLogger();
|
|
156
|
+
const deps: ResolveExhaustedMessageMoveDeps = {
|
|
157
|
+
messageService: repos.messageService,
|
|
158
|
+
threadMessageService: repos.threadMessageService,
|
|
159
|
+
log,
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const result = await resolveExhaustedMessageMoveFailure(deps, {
|
|
163
|
+
...input,
|
|
164
|
+
getConnection: async () => buildConnection(new Set([101])),
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
assert.equal(result.outcome, "broken");
|
|
168
|
+
assert.deepEqual(
|
|
169
|
+
repos.messages.get("msg-1"),
|
|
170
|
+
pendingMoveRow(),
|
|
171
|
+
"a move that never reached the server is never reverted locally (PR #652)",
|
|
172
|
+
);
|
|
173
|
+
assert.equal(repos.threadMessages.size, 1);
|
|
174
|
+
assert.ok(errors.some((e) => e.obj.alert === "message_move_failed"));
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
it("BROKEN: an empty FETCH the SEARCH contradicts never counts as gone", async () => {
|
|
178
|
+
const repos = buildRepositories(pendingMoveRow());
|
|
179
|
+
const { log } = buildLogger();
|
|
180
|
+
const deps: ResolveExhaustedMessageMoveDeps = {
|
|
181
|
+
messageService: repos.messageService,
|
|
182
|
+
threadMessageService: repos.threadMessageService,
|
|
183
|
+
log,
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
const result = await resolveExhaustedMessageMoveFailure(deps, {
|
|
187
|
+
...input,
|
|
188
|
+
getConnection: async () =>
|
|
189
|
+
buildConnection(new Set([101]), new Set([101])),
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
assert.equal(result.outcome, "broken");
|
|
193
|
+
assert.deepEqual(repos.messages.get("msg-1"), pendingMoveRow());
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("an unreachable server reaches no verdict at all — the probe propagates and the row is untouched", async () => {
|
|
197
|
+
const repos = buildRepositories(pendingMoveRow());
|
|
198
|
+
const { log } = buildLogger();
|
|
199
|
+
const deps: ResolveExhaustedMessageMoveDeps = {
|
|
200
|
+
messageService: repos.messageService,
|
|
201
|
+
threadMessageService: repos.threadMessageService,
|
|
202
|
+
log,
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
await assert.rejects(
|
|
206
|
+
() =>
|
|
207
|
+
resolveExhaustedMessageMoveFailure(deps, {
|
|
208
|
+
...input,
|
|
209
|
+
getConnection: async () => {
|
|
210
|
+
throw new Error("ECONNRESET");
|
|
211
|
+
},
|
|
212
|
+
}),
|
|
213
|
+
/ECONNRESET/,
|
|
214
|
+
);
|
|
215
|
+
|
|
216
|
+
assert.deepEqual(
|
|
217
|
+
repos.messages.get("msg-1"),
|
|
218
|
+
pendingMoveRow(),
|
|
219
|
+
"absence is only ever concluded from an answer the server gave",
|
|
220
|
+
);
|
|
221
|
+
});
|
|
222
|
+
});
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type IImapConnection,
|
|
3
|
+
isMessageGoneFromOpenMailbox,
|
|
4
|
+
reconcileStaleMessage,
|
|
5
|
+
type StaleMessageReconcileDeps,
|
|
6
|
+
} from "@remit/mailbox-service";
|
|
7
|
+
|
|
8
|
+
export interface MessageMoveTerminalLogger {
|
|
9
|
+
info(obj: Record<string, unknown>, msg: string): void;
|
|
10
|
+
error(obj: Record<string, unknown>, msg: string): void;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface ResolveExhaustedMessageMoveDeps
|
|
14
|
+
extends StaleMessageReconcileDeps {
|
|
15
|
+
log: MessageMoveTerminalLogger;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface ResolveExhaustedMessageMoveInput {
|
|
19
|
+
accountId: string;
|
|
20
|
+
accountConfigId: string;
|
|
21
|
+
messageId: string;
|
|
22
|
+
uid: number;
|
|
23
|
+
sourceMailboxPath: string;
|
|
24
|
+
getConnection: () => Promise<IImapConnection>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type MessageMoveTerminalOutcome = "reconciled" | "broken";
|
|
28
|
+
|
|
29
|
+
export interface ResolveExhaustedMessageMoveResult {
|
|
30
|
+
outcome: MessageMoveTerminalOutcome;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Resolve a MESSAGE_MOVE failure that has exhausted the message queue's
|
|
35
|
+
* redelivery budget into exactly one of two terminal outcomes, mirroring
|
|
36
|
+
* `resolveExhaustedPlacementMoveFailure` and `resolveExhaustedFlagPushFailure`
|
|
37
|
+
* for the same failure taxonomy (issue #655) — no third, softer outcome.
|
|
38
|
+
*
|
|
39
|
+
* The move's pending state is the Message row itself (`status: moving`,
|
|
40
|
+
* `originalMailboxId`/`originalUid`, `mailboxId` already pointing at the
|
|
41
|
+
* destination), not a separate marker, so the outcomes act on that row.
|
|
42
|
+
*
|
|
43
|
+
* 1. RECONCILED (expected) — the message no longer exists at the move's source
|
|
44
|
+
* on IMAP, confirmed by {@link isMessageGoneFromOpenMailbox} rather than by
|
|
45
|
+
* a FETCH coming back empty. Either the MOVE did execute server-side and
|
|
46
|
+
* the connection dropped before the tagged OK was read, or a foreign client
|
|
47
|
+
* moved or expunged the message; from here those are indistinguishable and
|
|
48
|
+
* have the same answer. The stale Message/ThreadMessage rows are deleted
|
|
49
|
+
* via {@link reconcileStaleMessage} and the caller resyncs both folders, so
|
|
50
|
+
* whichever folder actually holds the message re-projects it with the
|
|
51
|
+
* server's own UID. Metric only, no alarm — routine.
|
|
52
|
+
* 2. BROKEN — the message is still at the source, so the move never took
|
|
53
|
+
* effect, but it keeps failing: broken code or a broken account, not a
|
|
54
|
+
* transient blip. Local state is left exactly as it stands. Reverting the
|
|
55
|
+
* optimistic move here is what PR #652 was pulled for: the local row is the
|
|
56
|
+
* only record that this move is still owed, and a revert races a MOVE that
|
|
57
|
+
* may yet have landed. Logged with an `alert`-shaped entry for an operator
|
|
58
|
+
* alarm; never re-thrown (terminal — the caller acks either way, since
|
|
59
|
+
* retrying a permanently-broken move can never succeed).
|
|
60
|
+
*
|
|
61
|
+
* A server that cannot be reached at exhaustion time never reaches either
|
|
62
|
+
* verdict: the probe throws and the record dead-letters with the row untouched.
|
|
63
|
+
* Absence is only ever concluded from an answer the server gave.
|
|
64
|
+
*
|
|
65
|
+
* An operator reading `message_move_failed` should know one case where the
|
|
66
|
+
* message is not actually at the source: a message another client expunged
|
|
67
|
+
* mid-session can answer an empty FETCH while the server still lists its UID
|
|
68
|
+
* in SEARCH, until it is allowed to send the untagged EXPUNGE. That message
|
|
69
|
+
* lands in BROKEN, and BROKEN is terminal — the row stays pending and the
|
|
70
|
+
* alert stands until someone clears it. The reverse mistake discards the row
|
|
71
|
+
* for live mail, so the cost is paid deliberately.
|
|
72
|
+
*/
|
|
73
|
+
export const resolveExhaustedMessageMoveFailure = async (
|
|
74
|
+
deps: ResolveExhaustedMessageMoveDeps,
|
|
75
|
+
input: ResolveExhaustedMessageMoveInput,
|
|
76
|
+
): Promise<ResolveExhaustedMessageMoveResult> => {
|
|
77
|
+
const {
|
|
78
|
+
accountId,
|
|
79
|
+
accountConfigId,
|
|
80
|
+
messageId,
|
|
81
|
+
uid,
|
|
82
|
+
sourceMailboxPath,
|
|
83
|
+
getConnection,
|
|
84
|
+
} = input;
|
|
85
|
+
|
|
86
|
+
const connection = await getConnection();
|
|
87
|
+
await connection.openBox(sourceMailboxPath, true);
|
|
88
|
+
|
|
89
|
+
if (await isMessageGoneFromOpenMailbox(connection, uid)) {
|
|
90
|
+
const { threadMessagesDeleted } = await reconcileStaleMessage(
|
|
91
|
+
deps,
|
|
92
|
+
accountConfigId,
|
|
93
|
+
messageId,
|
|
94
|
+
);
|
|
95
|
+
deps.log.info(
|
|
96
|
+
{
|
|
97
|
+
metric: "message_move_stale_row_reconciled",
|
|
98
|
+
accountId,
|
|
99
|
+
accountConfigId,
|
|
100
|
+
messageId,
|
|
101
|
+
uid,
|
|
102
|
+
sourceMailboxPath,
|
|
103
|
+
threadMessagesDeleted,
|
|
104
|
+
},
|
|
105
|
+
"Message no longer at its move source after retry exhaustion (move landed server-side, or an external delete or move); stale row reconciled",
|
|
106
|
+
);
|
|
107
|
+
return { outcome: "reconciled" };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
deps.log.error(
|
|
111
|
+
{
|
|
112
|
+
alert: "message_move_failed",
|
|
113
|
+
accountId,
|
|
114
|
+
accountConfigId,
|
|
115
|
+
messageId,
|
|
116
|
+
uid,
|
|
117
|
+
sourceMailboxPath,
|
|
118
|
+
},
|
|
119
|
+
"Message move could not be pushed to IMAP after retry exhaustion; message still exists at its source — local state left pending for operator investigation",
|
|
120
|
+
);
|
|
121
|
+
return { outcome: "broken" };
|
|
122
|
+
};
|
|
@@ -7,7 +7,9 @@ import type { MessageMoveEvent } from "../events.js";
|
|
|
7
7
|
import {
|
|
8
8
|
buildThreadMessageMoveUpdate,
|
|
9
9
|
emitMoveResync,
|
|
10
|
+
getMessageMoveMaxAttempts,
|
|
10
11
|
handleMessageMove,
|
|
12
|
+
MESSAGE_MOVE_MAX_ATTEMPTS,
|
|
11
13
|
moveThenResync,
|
|
12
14
|
} from "./message-move.js";
|
|
13
15
|
|
|
@@ -185,7 +187,39 @@ describe("moveThenResync (#1031)", () => {
|
|
|
185
187
|
});
|
|
186
188
|
});
|
|
187
189
|
|
|
188
|
-
describe("
|
|
190
|
+
describe("getMessageMoveMaxAttempts — env-derived threshold (#655)", () => {
|
|
191
|
+
it("parses the injected env var", () => {
|
|
192
|
+
assert.equal(
|
|
193
|
+
getMessageMoveMaxAttempts({ MESSAGE_MOVE_MAX_ATTEMPTS: "3" }),
|
|
194
|
+
3,
|
|
195
|
+
);
|
|
196
|
+
assert.equal(
|
|
197
|
+
getMessageMoveMaxAttempts({ MESSAGE_MOVE_MAX_ATTEMPTS: "5" }),
|
|
198
|
+
5,
|
|
199
|
+
);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("defaults to the message queue's own maxReceiveCount when unset", () => {
|
|
203
|
+
assert.equal(getMessageMoveMaxAttempts({}), 3);
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
it("defaults on a non-numeric or non-positive value", () => {
|
|
207
|
+
assert.equal(
|
|
208
|
+
getMessageMoveMaxAttempts({ MESSAGE_MOVE_MAX_ATTEMPTS: "nope" }),
|
|
209
|
+
3,
|
|
210
|
+
);
|
|
211
|
+
assert.equal(
|
|
212
|
+
getMessageMoveMaxAttempts({ MESSAGE_MOVE_MAX_ATTEMPTS: "0" }),
|
|
213
|
+
3,
|
|
214
|
+
);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it("MESSAGE_MOVE_MAX_ATTEMPTS is a concrete, positive number at module load", () => {
|
|
218
|
+
assert.ok(MESSAGE_MOVE_MAX_ATTEMPTS > 0);
|
|
219
|
+
});
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
describe("handleMessageMove — the move's own pending state gates every attempt", () => {
|
|
189
223
|
const acctId = "mm-acc-zzz";
|
|
190
224
|
|
|
191
225
|
const cappedAccount = (): AccountItem =>
|
|
@@ -219,13 +253,25 @@ describe("handleMessageMove — deleted mailbox is terminal (#287/#289)", () =>
|
|
|
219
253
|
timestamp: 1700000000000,
|
|
220
254
|
} as MessageMoveEvent;
|
|
221
255
|
|
|
256
|
+
const pendingRow = () => ({
|
|
257
|
+
messageId: "mm-msg-zzz",
|
|
258
|
+
mailboxId: "mm-dst-zzz",
|
|
259
|
+
uid: 10,
|
|
260
|
+
status: "moving",
|
|
261
|
+
syncStatus: "pending",
|
|
262
|
+
});
|
|
263
|
+
|
|
222
264
|
// The client is supplied by injection (`setClient`), so these tests register
|
|
223
265
|
// the repositories they then mock rather than reaching a composition.
|
|
224
266
|
before(() => {
|
|
225
267
|
setClient({
|
|
226
268
|
account: { get: async () => undefined },
|
|
227
269
|
mailbox: { get: async () => undefined },
|
|
228
|
-
message: {
|
|
270
|
+
message: {
|
|
271
|
+
get: async () => [],
|
|
272
|
+
update: async () => undefined,
|
|
273
|
+
updateUid: async () => undefined,
|
|
274
|
+
},
|
|
229
275
|
secrets: { decrypt: async () => undefined },
|
|
230
276
|
} as unknown as RemitClient);
|
|
231
277
|
});
|
|
@@ -236,6 +282,7 @@ describe("handleMessageMove — deleted mailbox is terminal (#287/#289)", () =>
|
|
|
236
282
|
const client = await getClient();
|
|
237
283
|
mock.method(client.account, "get", async () => cappedAccount());
|
|
238
284
|
mock.method(client.secrets, "decrypt", async () => "fake-password");
|
|
285
|
+
mock.method(client.message, "get", async () => [pendingRow()]);
|
|
239
286
|
mock.method(client.mailbox, "get", async () => {
|
|
240
287
|
throw Object.assign(new Error("Mailbox not found: mm-src-zzz"), {
|
|
241
288
|
name: "NotFoundError",
|
|
@@ -252,4 +299,44 @@ describe("handleMessageMove — deleted mailbox is terminal (#287/#289)", () =>
|
|
|
252
299
|
"a deleted mailbox never reaches the IMAP move",
|
|
253
300
|
);
|
|
254
301
|
});
|
|
302
|
+
|
|
303
|
+
// Without this gate a redelivery of an already-confirmed move re-runs the
|
|
304
|
+
// MOVE against a UID the source no longer holds, and on exhaustion the
|
|
305
|
+
// terminal resolver reads the source's honest "gone" as grounds to delete a
|
|
306
|
+
// row that is correct and settled.
|
|
307
|
+
it("acks without connecting when the move already settled", async () => {
|
|
308
|
+
const client = await getClient();
|
|
309
|
+
mock.method(client.account, "get", async () => cappedAccount());
|
|
310
|
+
mock.method(client.secrets, "decrypt", async () => "fake-password");
|
|
311
|
+
mock.method(client.message, "get", async () => [
|
|
312
|
+
{ ...pendingRow(), uid: 4711, status: "active", syncStatus: "synced" },
|
|
313
|
+
]);
|
|
314
|
+
const mailboxGet = mock.method(client.mailbox, "get", async () => {
|
|
315
|
+
throw new Error("a settled move must never resolve a mailbox");
|
|
316
|
+
});
|
|
317
|
+
const update = mock.method(client.message, "update", async () => {});
|
|
318
|
+
|
|
319
|
+
await handleMessageMove(event, silentLogger, MESSAGE_MOVE_MAX_ATTEMPTS);
|
|
320
|
+
|
|
321
|
+
assert.equal(mailboxGet.mock.calls.length, 0);
|
|
322
|
+
assert.equal(
|
|
323
|
+
update.mock.calls.length,
|
|
324
|
+
0,
|
|
325
|
+
"a settled row is never written again",
|
|
326
|
+
);
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
it("acks without connecting when the message row is already gone", async () => {
|
|
330
|
+
const client = await getClient();
|
|
331
|
+
mock.method(client.account, "get", async () => cappedAccount());
|
|
332
|
+
mock.method(client.secrets, "decrypt", async () => "fake-password");
|
|
333
|
+
mock.method(client.message, "get", async () => []);
|
|
334
|
+
const mailboxGet = mock.method(client.mailbox, "get", async () => {
|
|
335
|
+
throw new Error("a missing row must never resolve a mailbox");
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
await handleMessageMove(event, silentLogger, MESSAGE_MOVE_MAX_ATTEMPTS);
|
|
339
|
+
|
|
340
|
+
assert.equal(mailboxGet.mock.calls.length, 0);
|
|
341
|
+
});
|
|
255
342
|
});
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { getClient } from "@remit/backend/client";
|
|
2
2
|
import type { ThreadMessageItem } from "@remit/data-ports";
|
|
3
|
-
import {
|
|
3
|
+
import { MessageSyncStatus } from "@remit/domain-enums";
|
|
4
4
|
import type { Logger } from "@remit/logger-lambda";
|
|
5
|
+
import { recordImapFailure } from "@remit/logger-lambda";
|
|
5
6
|
import {
|
|
6
7
|
guardConnectionCursor,
|
|
8
|
+
type IImapConnection,
|
|
7
9
|
isCursorRebuildNeeded,
|
|
10
|
+
isPlacementUnsettled,
|
|
8
11
|
MailboxCursorPausedError,
|
|
9
12
|
} from "@remit/mailbox-service";
|
|
10
13
|
import { isAccountDeleted } from "../account-check.js";
|
|
@@ -14,6 +17,28 @@ import type { MessageMoveEvent, SyncMessagesEvent } from "../events.js";
|
|
|
14
17
|
import { isNotFoundError } from "../is-not-found.js";
|
|
15
18
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
16
19
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
20
|
+
import { resolveExhaustedMessageMoveFailure } from "./message-move-terminal.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Fallback when `MESSAGE_MOVE_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
24
|
+
* Matches the `maxReceiveCount` the message queue's redrive policy uses
|
|
25
|
+
* (`remit-messages.fifo`, `deploy/vps/queues.json`), same pattern as
|
|
26
|
+
* `FLAG_PUSH_MAX_ATTEMPTS` and `PLACEMENT_MOVE_MAX_ATTEMPTS`.
|
|
27
|
+
*/
|
|
28
|
+
const DEFAULT_MESSAGE_MOVE_MAX_ATTEMPTS = 3;
|
|
29
|
+
|
|
30
|
+
export const getMessageMoveMaxAttempts = (
|
|
31
|
+
processEnv: NodeJS.ProcessEnv = process.env,
|
|
32
|
+
): number => {
|
|
33
|
+
const raw = processEnv.MESSAGE_MOVE_MAX_ATTEMPTS;
|
|
34
|
+
if (!raw) return DEFAULT_MESSAGE_MOVE_MAX_ATTEMPTS;
|
|
35
|
+
const parsed = Number.parseInt(raw, 10);
|
|
36
|
+
return Number.isFinite(parsed) && parsed > 0
|
|
37
|
+
? parsed
|
|
38
|
+
: DEFAULT_MESSAGE_MOVE_MAX_ATTEMPTS;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export const MESSAGE_MOVE_MAX_ATTEMPTS = getMessageMoveMaxAttempts();
|
|
17
42
|
|
|
18
43
|
type EmitSyncMessages = (
|
|
19
44
|
event: Omit<SyncMessagesEvent, "eventId" | "timestamp">,
|
|
@@ -41,6 +66,23 @@ export const emitMoveResync = async (
|
|
|
41
66
|
);
|
|
42
67
|
};
|
|
43
68
|
|
|
69
|
+
/**
|
|
70
|
+
* SEARCH a mailbox for a message by its RFC822 Message-ID header. Read-only
|
|
71
|
+
* (EXAMINE, not SELECT) — this is a verification probe, never a write.
|
|
72
|
+
* Returns the first matching UID, or `null` if nothing matched.
|
|
73
|
+
*/
|
|
74
|
+
export const searchMailboxByMessageId = async (
|
|
75
|
+
connection: IImapConnection,
|
|
76
|
+
mailboxPath: string,
|
|
77
|
+
messageIdHeader: string,
|
|
78
|
+
): Promise<number | null> => {
|
|
79
|
+
await connection.openBox(mailboxPath, true);
|
|
80
|
+
const uids = await connection.search([
|
|
81
|
+
`HEADER Message-ID "${messageIdHeader}"`,
|
|
82
|
+
]);
|
|
83
|
+
return uids[0] ?? null;
|
|
84
|
+
};
|
|
85
|
+
|
|
44
86
|
/**
|
|
45
87
|
* Resync the affected folders only once the IMAP move has resolved. A move that
|
|
46
88
|
* fails (or is retried) must not refresh counts off a move that didn't happen,
|
|
@@ -97,10 +139,18 @@ export const buildThreadMessageMoveUpdate = (
|
|
|
97
139
|
/**
|
|
98
140
|
* Handle MESSAGE_MOVE events.
|
|
99
141
|
* Executes IMAP MOVE command and updates local state with new UID.
|
|
142
|
+
*
|
|
143
|
+
* A failing move retries on SQS redelivery until `receiveCount` reaches
|
|
144
|
+
* {@link MESSAGE_MOVE_MAX_ATTEMPTS}, at which point
|
|
145
|
+
* {@link resolveExhaustedMessageMoveFailure} asks IMAP where the message
|
|
146
|
+
* actually is and settles the row into one terminal outcome (issue #655).
|
|
147
|
+
* Before that, `syncStatus: failed` marked every attempt and nothing ever
|
|
148
|
+
* settled the row, so an exhausted move sat `moving`/`failed` forever.
|
|
100
149
|
*/
|
|
101
150
|
export const handleMessageMove = async (
|
|
102
151
|
event: MessageMoveEvent,
|
|
103
152
|
log: Logger,
|
|
153
|
+
receiveCount = 1,
|
|
104
154
|
): Promise<void> => {
|
|
105
155
|
const {
|
|
106
156
|
account: accountService,
|
|
@@ -140,6 +190,32 @@ export const handleMessageMove = async (
|
|
|
140
190
|
return;
|
|
141
191
|
}
|
|
142
192
|
|
|
193
|
+
const [message] = await messageService.get([messageId]);
|
|
194
|
+
|
|
195
|
+
// The message row is already gone — some other reconciliation path deleted
|
|
196
|
+
// it. The move is moot; ack without touching IMAP.
|
|
197
|
+
if (!message) {
|
|
198
|
+
log.warn(
|
|
199
|
+
{ accountId, messageId },
|
|
200
|
+
"Skipping MESSAGE_MOVE: message row no longer exists",
|
|
201
|
+
);
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// This move already settled — `updateUid` cleared `status: moving` once the
|
|
206
|
+
// server confirmed it. A redelivery reaching here would MOVE a UID the
|
|
207
|
+
// source no longer holds, fail, and on exhaustion read the source's honest
|
|
208
|
+
// "gone" as grounds to reconcile away a row that is correct. There is no
|
|
209
|
+
// marker to find missing (unlike FLAG_PUSH and PLACEMENT_MOVE_PUSH), so the
|
|
210
|
+
// row's own pending marker is what stands in for one.
|
|
211
|
+
if (!isPlacementUnsettled(message)) {
|
|
212
|
+
log.info(
|
|
213
|
+
{ accountId, messageId, uid: message.uid, status: message.status },
|
|
214
|
+
"Skipping MESSAGE_MOVE: the move already settled against confirmed IMAP state",
|
|
215
|
+
);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
143
219
|
await withOAuthLifecycle(
|
|
144
220
|
buildLifecycleDeps(secrets, accountService),
|
|
145
221
|
account,
|
|
@@ -200,56 +276,65 @@ export const handleMessageMove = async (
|
|
|
200
276
|
destinationMailboxPath,
|
|
201
277
|
);
|
|
202
278
|
|
|
203
|
-
// Get new UID from COPYUID response
|
|
204
|
-
|
|
279
|
+
// Get new UID from COPYUID response. A server without UIDPLUS
|
|
280
|
+
// answers a perfectly successful MOVE with no COPYUID entry,
|
|
281
|
+
// so an empty map is UNCONFIRMED, never evidence the message
|
|
282
|
+
// is gone: the destination is asked by Message-ID before any
|
|
283
|
+
// verdict, exactly as `attemptMove` does. Marking the row
|
|
284
|
+
// `failed` and returning (the behaviour issue #655 opens on)
|
|
285
|
+
// left it `moving`/`failed` with no DLQ entry and no metric,
|
|
286
|
+
// because a handler that returns never redelivers.
|
|
287
|
+
const newUid =
|
|
288
|
+
result.uidMap.get(uid) ??
|
|
289
|
+
(message.messageIdHeader
|
|
290
|
+
? await searchMailboxByMessageId(
|
|
291
|
+
rawConnection,
|
|
292
|
+
destinationMailboxPath,
|
|
293
|
+
message.messageIdHeader,
|
|
294
|
+
)
|
|
295
|
+
: null);
|
|
205
296
|
|
|
206
|
-
if (newUid) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
messageId,
|
|
210
|
-
newUid,
|
|
211
|
-
destinationMailboxId,
|
|
297
|
+
if (!newUid) {
|
|
298
|
+
throw new Error(
|
|
299
|
+
`Message move unconfirmed (no COPYUID entry, not found at ${destinationMailboxPath}) — retrying`,
|
|
212
300
|
);
|
|
301
|
+
}
|
|
213
302
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
);
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
log.info(
|
|
235
|
-
{
|
|
236
|
-
messageId,
|
|
237
|
-
oldUid: uid,
|
|
238
|
-
newUid,
|
|
239
|
-
destination: destinationMailboxPath,
|
|
240
|
-
},
|
|
241
|
-
"Message moved successfully",
|
|
303
|
+
// Update message with new UID
|
|
304
|
+
await messageService.updateUid(
|
|
305
|
+
messageId,
|
|
306
|
+
newUid,
|
|
307
|
+
destinationMailboxId,
|
|
308
|
+
);
|
|
309
|
+
|
|
310
|
+
// Update ThreadMessage UID and mailboxId
|
|
311
|
+
const threadMessage = await threadMessageService.findByMessageId(
|
|
312
|
+
account.accountConfigId,
|
|
313
|
+
messageId,
|
|
314
|
+
);
|
|
315
|
+
if (threadMessage) {
|
|
316
|
+
const args = buildThreadMessageMoveUpdate(
|
|
317
|
+
threadMessage,
|
|
318
|
+
newUid,
|
|
319
|
+
destinationMailboxId,
|
|
242
320
|
);
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
321
|
+
await threadMessageService.update(
|
|
322
|
+
threadMessage.accountConfigId,
|
|
323
|
+
threadMessage.threadMessageId,
|
|
324
|
+
args.set,
|
|
325
|
+
{ composites: args.composites },
|
|
248
326
|
);
|
|
249
|
-
await messageService.update(messageId, {
|
|
250
|
-
syncStatus: MessageSyncStatus.failed,
|
|
251
|
-
});
|
|
252
327
|
}
|
|
328
|
+
|
|
329
|
+
log.info(
|
|
330
|
+
{
|
|
331
|
+
messageId,
|
|
332
|
+
oldUid: uid,
|
|
333
|
+
newUid,
|
|
334
|
+
destination: destinationMailboxPath,
|
|
335
|
+
},
|
|
336
|
+
"Message moved successfully",
|
|
337
|
+
);
|
|
253
338
|
},
|
|
254
339
|
() =>
|
|
255
340
|
emitMoveResync(emitEvent, {
|
|
@@ -284,31 +369,59 @@ export const handleMessageMove = async (
|
|
|
284
369
|
);
|
|
285
370
|
const connection = await scope.getConnection();
|
|
286
371
|
await connection.createMailbox(destinationMailboxPath);
|
|
287
|
-
// Re-throw to let the event be retried
|
|
372
|
+
// Re-throw to let the event be retried against the folder just
|
|
373
|
+
// created. Kept unconditional past the attempt budget: the
|
|
374
|
+
// destination now exists, so the record is worth redriving from
|
|
375
|
+
// the DLQ, and the terminal resolver has nothing to settle — it
|
|
376
|
+
// would find the message exactly where it started.
|
|
288
377
|
throw error;
|
|
289
378
|
}
|
|
290
379
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
log.info(
|
|
297
|
-
{ messageId, uid },
|
|
298
|
-
"Message not found on IMAP, updating local state as synced",
|
|
299
|
-
);
|
|
380
|
+
if (receiveCount < MESSAGE_MOVE_MAX_ATTEMPTS) {
|
|
381
|
+
// Transient move failure — expected (connections drop). No
|
|
382
|
+
// alarm; queue redelivery retries, and `failed` marks the row
|
|
383
|
+
// as unsettled meanwhile. It is not a terminal signal: only the
|
|
384
|
+
// resolver below settles anything.
|
|
300
385
|
await messageService.update(messageId, {
|
|
301
|
-
|
|
302
|
-
|
|
386
|
+
syncStatus: MessageSyncStatus.failed,
|
|
387
|
+
});
|
|
388
|
+
throw error;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// Redelivery budget exhausted: resolve into exactly one of the two
|
|
392
|
+
// terminal outcomes instead of dead-lettering with no diagnosis,
|
|
393
|
+
// and never by inferring the server's state from our own failures.
|
|
394
|
+
const { outcome } = await resolveExhaustedMessageMoveFailure(
|
|
395
|
+
{ messageService, threadMessageService, log },
|
|
396
|
+
{
|
|
397
|
+
accountId,
|
|
398
|
+
accountConfigId: account.accountConfigId,
|
|
399
|
+
messageId,
|
|
400
|
+
uid,
|
|
401
|
+
sourceMailboxPath,
|
|
402
|
+
getConnection: scope.getConnection,
|
|
403
|
+
},
|
|
404
|
+
);
|
|
405
|
+
|
|
406
|
+
if (outcome === "reconciled") {
|
|
407
|
+
// Whichever folder the message actually sits in re-projects it
|
|
408
|
+
// with the server's own UID.
|
|
409
|
+
await emitMoveResync(emitEvent, {
|
|
410
|
+
accountId,
|
|
411
|
+
sourceMailboxId,
|
|
412
|
+
destinationMailboxId,
|
|
303
413
|
});
|
|
304
414
|
return;
|
|
305
415
|
}
|
|
306
416
|
|
|
307
|
-
//
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
417
|
+
// Terminal and never re-thrown, so the handler-outcome series
|
|
418
|
+
// records this record as a success. Counted here or it is invisible.
|
|
419
|
+
recordImapFailure("MESSAGE_MOVE_EXHAUSTED", "other");
|
|
420
|
+
log.error(
|
|
421
|
+
{ error: errorMessage },
|
|
422
|
+
"Message move retry exhausted; message still exists at its source",
|
|
423
|
+
);
|
|
424
|
+
// Terminal — never re-thrown, so the caller acks either way.
|
|
312
425
|
})
|
|
313
426
|
.finally(() => scope.disconnect());
|
|
314
427
|
},
|
|
@@ -20,6 +20,7 @@ import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
|
20
20
|
import {
|
|
21
21
|
buildThreadMessageMoveUpdate,
|
|
22
22
|
emitMoveResync,
|
|
23
|
+
searchMailboxByMessageId,
|
|
23
24
|
} from "./message-move.js";
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -48,23 +49,6 @@ interface MoveOutcome {
|
|
|
48
49
|
newUid?: number;
|
|
49
50
|
}
|
|
50
51
|
|
|
51
|
-
/**
|
|
52
|
-
* SEARCH a mailbox for a message by its RFC822 Message-ID header. Read-only
|
|
53
|
-
* (EXAMINE, not SELECT) — this is a verification probe, never a write.
|
|
54
|
-
* Returns the first matching UID, or `null` if nothing matched.
|
|
55
|
-
*/
|
|
56
|
-
const searchMailboxByMessageId = async (
|
|
57
|
-
connection: IImapConnection,
|
|
58
|
-
mailboxPath: string,
|
|
59
|
-
messageIdHeader: string,
|
|
60
|
-
): Promise<number | null> => {
|
|
61
|
-
await connection.openBox(mailboxPath, true);
|
|
62
|
-
const uids = await connection.search([
|
|
63
|
-
`HEADER Message-ID "${messageIdHeader}"`,
|
|
64
|
-
]);
|
|
65
|
-
return uids[0] ?? null;
|
|
66
|
-
};
|
|
67
|
-
|
|
68
52
|
/**
|
|
69
53
|
* Attempt the IMAP MOVE.
|
|
70
54
|
*
|
package/src/processor.ts
CHANGED
|
@@ -18,12 +18,11 @@ export const processEvent = async (
|
|
|
18
18
|
log: Logger,
|
|
19
19
|
/**
|
|
20
20
|
* SQS's own delivery count for the record carrying this event (1 on first
|
|
21
|
-
* delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH, FLAG_PUSH
|
|
22
|
-
* APPEND_SENT_MESSAGE — each knows from it when this is
|
|
23
|
-
* before the queue's
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* dead-lettering blindly.
|
|
21
|
+
* delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH, FLAG_PUSH,
|
|
22
|
+
* APPEND_SENT_MESSAGE and MESSAGE_MOVE — each knows from it when this is
|
|
23
|
+
* the last attempt before the queue's own redrive policy would DLQ the
|
|
24
|
+
* record, so it can resolve retry exhaustion into a terminal outcome
|
|
25
|
+
* (issue #1270) instead of dead-lettering blindly.
|
|
27
26
|
*/
|
|
28
27
|
receiveCount = 1,
|
|
29
28
|
): Promise<void> => {
|
|
@@ -41,7 +40,7 @@ export const processEvent = async (
|
|
|
41
40
|
case "MESSAGE_DELETE":
|
|
42
41
|
return handleMessageDelete(event, log);
|
|
43
42
|
case "MESSAGE_MOVE":
|
|
44
|
-
return handleMessageMove(event, log);
|
|
43
|
+
return handleMessageMove(event, log, receiveCount);
|
|
45
44
|
case "PLACEMENT_MOVE_PUSH":
|
|
46
45
|
return handlePlacementMovePush(event, log, receiveCount);
|
|
47
46
|
case "FLAG_PUSH":
|