@remit/imap-worker 0.0.62 → 0.0.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/credential-less-sync-fence.test.ts +156 -0
- package/src/handlers/empty-trash.ts +1 -1
- package/src/handlers/message-delete-terminal.ts +46 -16
- package/src/handlers/message-delete.test.ts +61 -21
- package/src/handlers/message-delete.ts +8 -82
- package/src/handlers/thread-message-rows.ts +77 -0
- package/src/scheduler/run-tick.test.ts +31 -0
- package/src/scheduler/run-tick.ts +5 -0
- package/src/with-oauth-lifecycle.test.ts +32 -2
- package/src/with-oauth-lifecycle.ts +30 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remit/imap-worker",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.64",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
"dev": "node --import tsx src/e2e-processor-shim.ts"
|
|
26
26
|
},
|
|
27
27
|
"devDependencies": {
|
|
28
|
+
"@remit/queue-sidecar": "*",
|
|
28
29
|
"@remit/storage-service": "*",
|
|
29
30
|
"aws-sdk-client-mock": "*",
|
|
30
31
|
"esbuild": "^0.28.1"
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression test for issue #1120.
|
|
3
|
+
*
|
|
4
|
+
* A sync triggered for an account that stores no credential used to fail the
|
|
5
|
+
* batch item, so the poller left the message undeleted for its whole
|
|
6
|
+
* visibility timeout. `remit-mailboxes.fifo` groups by accountId and a FIFO
|
|
7
|
+
* queue never delivers past an in-flight message in a group, so the account
|
|
8
|
+
* was fenced from every later sync — including the one the user asks for right
|
|
9
|
+
* after saving the password.
|
|
10
|
+
*
|
|
11
|
+
* Runs against the real QueueStore and the real queue definition the stack
|
|
12
|
+
* ships, with the poller's own contract modelled around it: ack deletes the
|
|
13
|
+
* message, a batch item failure leaves it in flight.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import assert from "node:assert/strict";
|
|
17
|
+
import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { after, describe, it } from "node:test";
|
|
20
|
+
import { fileURLToPath } from "node:url";
|
|
21
|
+
import { buildSyncMailboxesCommand } from "@remit/backend/trigger-sync";
|
|
22
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
23
|
+
import { ConnectionState } from "@remit/domain-enums";
|
|
24
|
+
import {
|
|
25
|
+
bootstrapQueues,
|
|
26
|
+
loadQueuesConfig,
|
|
27
|
+
QueueStore,
|
|
28
|
+
} from "@remit/queue-sidecar";
|
|
29
|
+
import { DEFAULT_VISIBILITY_TIMEOUT_SECONDS } from "@remit/sqs-client/poller";
|
|
30
|
+
import {
|
|
31
|
+
type OAuthLifecycleDeps,
|
|
32
|
+
withOAuthLifecycle,
|
|
33
|
+
} from "./with-oauth-lifecycle.js";
|
|
34
|
+
|
|
35
|
+
const QUEUE_NAME = "remit-mailboxes.fifo";
|
|
36
|
+
const QUEUE_URL = `http://localhost:9324/000000000000/${QUEUE_NAME}`;
|
|
37
|
+
const ACCOUNT_ID = "acct_imported";
|
|
38
|
+
const T0 = 1_700_000_000_000;
|
|
39
|
+
|
|
40
|
+
const here = fileURLToPath(new URL(".", import.meta.url));
|
|
41
|
+
const repoRoot = join(here, "..", "..", "..");
|
|
42
|
+
const tmpRoot = join(here, "..", ".tmp", "credential-less-sync-fence");
|
|
43
|
+
|
|
44
|
+
const silentLogger = {
|
|
45
|
+
info: () => {},
|
|
46
|
+
warn: () => {},
|
|
47
|
+
error: () => {},
|
|
48
|
+
debug: () => {},
|
|
49
|
+
trace: () => {},
|
|
50
|
+
fatal: () => {},
|
|
51
|
+
child: () => silentLogger,
|
|
52
|
+
} as never;
|
|
53
|
+
|
|
54
|
+
/** The account the import wizard lands: no password until its owner types one. */
|
|
55
|
+
const importedAccount = {
|
|
56
|
+
accountId: ACCOUNT_ID,
|
|
57
|
+
accountConfigId: "acfg_1",
|
|
58
|
+
username: "alice@example.com",
|
|
59
|
+
email: "alice@example.com",
|
|
60
|
+
authType: "password",
|
|
61
|
+
imapHost: "imap.example.com",
|
|
62
|
+
imapPort: 993,
|
|
63
|
+
imapTls: true,
|
|
64
|
+
imapStartTls: false,
|
|
65
|
+
isActive: true,
|
|
66
|
+
connectionState: ConnectionState.CredentialsMissing,
|
|
67
|
+
createdAt: T0,
|
|
68
|
+
updatedAt: T0,
|
|
69
|
+
} as unknown as AccountItem;
|
|
70
|
+
|
|
71
|
+
const enqueueSync = (store: QueueStore, now: number): void => {
|
|
72
|
+
const command = buildSyncMailboxesCommand({
|
|
73
|
+
queueUrl: QUEUE_URL,
|
|
74
|
+
accountId: ACCOUNT_ID,
|
|
75
|
+
});
|
|
76
|
+
store.sendMessage({
|
|
77
|
+
queueName: QUEUE_NAME,
|
|
78
|
+
body: command.input.MessageBody ?? "",
|
|
79
|
+
groupId: command.input.MessageGroupId,
|
|
80
|
+
deduplicationId: command.input.MessageDeduplicationId,
|
|
81
|
+
now,
|
|
82
|
+
});
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
describe("a credential-less sync trigger (issue #1120)", () => {
|
|
86
|
+
after(() => {
|
|
87
|
+
rmSync(tmpRoot, { recursive: true, force: true });
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("is acked, leaving the account's queue group deliverable", async () => {
|
|
91
|
+
mkdirSync(tmpRoot, { recursive: true });
|
|
92
|
+
const store = new QueueStore(
|
|
93
|
+
join(mkdtempSync(join(tmpRoot, "store-")), "queue.db"),
|
|
94
|
+
);
|
|
95
|
+
bootstrapQueues(
|
|
96
|
+
store,
|
|
97
|
+
loadQueuesConfig(join(repoRoot, "deploy/vps/queues.json")),
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const stateUpdates: string[] = [];
|
|
101
|
+
const deps: OAuthLifecycleDeps = {
|
|
102
|
+
secrets: {
|
|
103
|
+
decrypt: async () => "",
|
|
104
|
+
encrypt: async () => ({}) as never,
|
|
105
|
+
},
|
|
106
|
+
tokenService: { getAccessToken: async () => ({}) as never },
|
|
107
|
+
persistRotatedToken: async () => {},
|
|
108
|
+
updateConnectionState: async (_accountId, state) => {
|
|
109
|
+
stateUpdates.push(state);
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// The GET /config poll while the user is still typing the password.
|
|
114
|
+
enqueueSync(store, T0);
|
|
115
|
+
|
|
116
|
+
const [received] = store.receiveMessages({
|
|
117
|
+
queueName: QUEUE_NAME,
|
|
118
|
+
maxMessages: 1,
|
|
119
|
+
visibilityTimeoutSeconds: DEFAULT_VISIBILITY_TIMEOUT_SECONDS,
|
|
120
|
+
now: T0,
|
|
121
|
+
});
|
|
122
|
+
assert.ok(received, "the trigger is on the queue");
|
|
123
|
+
|
|
124
|
+
// The poller's contract: an ack deletes the message, a batch item failure
|
|
125
|
+
// leaves it in flight until the visibility timeout expires.
|
|
126
|
+
let batchItemFailure: unknown;
|
|
127
|
+
await withOAuthLifecycle(deps, importedAccount, silentLogger, async () => {
|
|
128
|
+
throw new Error("IMAP work must not run without a credential");
|
|
129
|
+
}).then(
|
|
130
|
+
() => store.deleteMessage(QUEUE_NAME, received.receiptHandle),
|
|
131
|
+
(err: unknown) => {
|
|
132
|
+
batchItemFailure = err;
|
|
133
|
+
},
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
// The refresh the user presses after saving the password.
|
|
137
|
+
enqueueSync(store, T0 + 2_000);
|
|
138
|
+
const deliverable = store.receiveMessages({
|
|
139
|
+
queueName: QUEUE_NAME,
|
|
140
|
+
maxMessages: 10,
|
|
141
|
+
now: T0 + 3_000,
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
assert.equal(
|
|
145
|
+
deliverable.length,
|
|
146
|
+
1,
|
|
147
|
+
"the account's group must stay deliverable, not fenced for the visibility timeout",
|
|
148
|
+
);
|
|
149
|
+
assert.equal(
|
|
150
|
+
batchItemFailure,
|
|
151
|
+
undefined,
|
|
152
|
+
"the event is acked, not retried",
|
|
153
|
+
);
|
|
154
|
+
assert.deepEqual(stateUpdates, [ConnectionState.CredentialsMissing]);
|
|
155
|
+
});
|
|
156
|
+
});
|
|
@@ -15,7 +15,7 @@ import type { EmptyTrashEvent } from "../events.js";
|
|
|
15
15
|
import { isNotFoundError } from "../is-not-found.js";
|
|
16
16
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
17
17
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
18
|
-
import { buildThreadMessageUndelete } from "./message-
|
|
18
|
+
import { buildThreadMessageUndelete } from "./thread-message-rows.js";
|
|
19
19
|
|
|
20
20
|
export interface EmptyTrashDeps {
|
|
21
21
|
getClient: typeof getClient;
|
|
@@ -1,11 +1,14 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
1
|
+
import type {
|
|
2
|
+
IMessageRepository,
|
|
3
|
+
IThreadMessageRepository,
|
|
4
|
+
} from "@remit/data-ports";
|
|
3
5
|
import {
|
|
4
6
|
type IImapConnection,
|
|
5
7
|
isMessageGoneFromOpenMailbox,
|
|
6
8
|
reconcileStaleMessage,
|
|
7
9
|
type StaleMessageReconcileDeps,
|
|
8
10
|
} from "@remit/mailbox-service";
|
|
11
|
+
import { buildThreadMessageMoveRevert } from "./thread-message-rows.js";
|
|
9
12
|
|
|
10
13
|
export interface MessageDeleteTerminalLogger {
|
|
11
14
|
info(obj: Record<string, unknown>, msg: string): void;
|
|
@@ -14,7 +17,11 @@ export interface MessageDeleteTerminalLogger {
|
|
|
14
17
|
|
|
15
18
|
export interface ResolveExhaustedMessageDeleteDeps
|
|
16
19
|
extends StaleMessageReconcileDeps {
|
|
17
|
-
messageService: Pick<IMessageRepository, "delete" | "
|
|
20
|
+
messageService: Pick<IMessageRepository, "delete" | "updateUid">;
|
|
21
|
+
threadMessageService: Pick<
|
|
22
|
+
IThreadMessageRepository,
|
|
23
|
+
"findAllByMessageId" | "deleteMany" | "update"
|
|
24
|
+
>;
|
|
18
25
|
log: MessageDeleteTerminalLogger;
|
|
19
26
|
}
|
|
20
27
|
|
|
@@ -22,6 +29,7 @@ export interface ResolveExhaustedMessageDeleteInput {
|
|
|
22
29
|
accountId: string;
|
|
23
30
|
accountConfigId: string;
|
|
24
31
|
messageId: string;
|
|
32
|
+
sourceMailboxId: string;
|
|
25
33
|
uid: number;
|
|
26
34
|
sourceMailboxPath: string;
|
|
27
35
|
getConnection: () => Promise<IImapConnection>;
|
|
@@ -49,15 +57,18 @@ export interface ResolveExhaustedMessageDeleteResult {
|
|
|
49
57
|
* server's own UID. Metric only, no alarm — routine.
|
|
50
58
|
* 2. BROKEN — the message is still at the source, so the delete never took
|
|
51
59
|
* effect, but it keeps failing: broken code or a broken account, not a
|
|
52
|
-
* transient blip. The row
|
|
53
|
-
*
|
|
54
|
-
*
|
|
60
|
+
* transient blip. The row is put back where the server has just said the
|
|
61
|
+
* message is: `mailboxId` and `uid` return to the delete's source pair, and
|
|
62
|
+
* the thread rows lose the deletion mark the optimistic `updateForMove`
|
|
63
|
+
* wrote (issue #1098).
|
|
55
64
|
*
|
|
56
|
-
* `status`
|
|
57
|
-
*
|
|
58
|
-
* `
|
|
59
|
-
*
|
|
60
|
-
*
|
|
65
|
+
* Settling `status` to `active` while leaving `mailboxId` on Trash is what
|
|
66
|
+
* this replaces. `bindsForeignUid` refuses a mismatched pair only while
|
|
67
|
+
* `status` is `moving`, so clearing it read as consistent, and the UI then
|
|
68
|
+
* offered "Delete permanently" on a Trash mailbox with the source folder's
|
|
69
|
+
* uid — an expunge naming a different message. Restoring the pair is not a
|
|
70
|
+
* revert on ambiguity (what PR #652 was pulled for): the presence probe
|
|
71
|
+
* above has just answered, so this writes a placement the server confirmed.
|
|
61
72
|
*
|
|
62
73
|
* A server that cannot be reached at exhaustion time never reaches either
|
|
63
74
|
* verdict: the probe throws and the record dead-letters with the row untouched.
|
|
@@ -71,6 +82,7 @@ export const resolveExhaustedMessageDeleteFailure = async (
|
|
|
71
82
|
accountId,
|
|
72
83
|
accountConfigId,
|
|
73
84
|
messageId,
|
|
85
|
+
sourceMailboxId,
|
|
74
86
|
uid,
|
|
75
87
|
sourceMailboxPath,
|
|
76
88
|
getConnection,
|
|
@@ -100,10 +112,6 @@ export const resolveExhaustedMessageDeleteFailure = async (
|
|
|
100
112
|
return { outcome: "reconciled" };
|
|
101
113
|
}
|
|
102
114
|
|
|
103
|
-
await deps.messageService.update(messageId, {
|
|
104
|
-
status: MessageStatus.active,
|
|
105
|
-
syncStatus: MessageSyncStatus.failed,
|
|
106
|
-
});
|
|
107
115
|
deps.log.error(
|
|
108
116
|
{
|
|
109
117
|
alert: "message_delete_failed",
|
|
@@ -111,9 +119,31 @@ export const resolveExhaustedMessageDeleteFailure = async (
|
|
|
111
119
|
accountConfigId,
|
|
112
120
|
messageId,
|
|
113
121
|
uid,
|
|
122
|
+
sourceMailboxId,
|
|
114
123
|
sourceMailboxPath,
|
|
115
124
|
},
|
|
116
|
-
"Delete could not be pushed to IMAP after retry exhaustion; the message is still at its source — row
|
|
125
|
+
"Delete could not be pushed to IMAP after retry exhaustion; the message is still at its source — row moved back to the source folder",
|
|
117
126
|
);
|
|
127
|
+
|
|
128
|
+
await deps.messageService.updateUid(messageId, uid, sourceMailboxId);
|
|
129
|
+
|
|
130
|
+
const threadMessages = await deps.threadMessageService.findAllByMessageId(
|
|
131
|
+
accountConfigId,
|
|
132
|
+
messageId,
|
|
133
|
+
);
|
|
134
|
+
for (const threadMessage of threadMessages) {
|
|
135
|
+
const args = buildThreadMessageMoveRevert(
|
|
136
|
+
threadMessage,
|
|
137
|
+
uid,
|
|
138
|
+
sourceMailboxId,
|
|
139
|
+
);
|
|
140
|
+
await deps.threadMessageService.update(
|
|
141
|
+
threadMessage.accountConfigId,
|
|
142
|
+
threadMessage.threadMessageId,
|
|
143
|
+
args.set,
|
|
144
|
+
{ composites: args.composites },
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
118
148
|
return { outcome: "broken" };
|
|
119
149
|
};
|
|
@@ -5,14 +5,16 @@ import type { Logger } from "@remit/logger-lambda";
|
|
|
5
5
|
import { renderMetrics, resetMetrics } from "@remit/logger-lambda";
|
|
6
6
|
import type { MessageDeleteEvent } from "../events.js";
|
|
7
7
|
import {
|
|
8
|
-
buildThreadMessageTrashUpdate,
|
|
9
|
-
buildThreadMessageUndelete,
|
|
10
8
|
deleteAllThreadMessagesForMessage,
|
|
11
9
|
getMessageDeleteMaxAttempts,
|
|
12
10
|
handleMessageDelete,
|
|
13
11
|
MESSAGE_DELETE_MAX_ATTEMPTS,
|
|
14
12
|
type MessageDeleteDeps,
|
|
15
13
|
} from "./message-delete.js";
|
|
14
|
+
import {
|
|
15
|
+
buildThreadMessageTrashUpdate,
|
|
16
|
+
buildThreadMessageUndelete,
|
|
17
|
+
} from "./thread-message-rows.js";
|
|
16
18
|
|
|
17
19
|
describe("getMessageDeleteMaxAttempts — env-derived threshold (#980)", () => {
|
|
18
20
|
it("parses the injected env var", () => {
|
|
@@ -676,28 +678,65 @@ describe("handleMessageDelete", () => {
|
|
|
676
678
|
);
|
|
677
679
|
});
|
|
678
680
|
|
|
679
|
-
// A source that still holds the uid at the ceiling means the
|
|
680
|
-
// took effect
|
|
681
|
-
//
|
|
682
|
-
//
|
|
683
|
-
|
|
681
|
+
// Issue #1098. A source that still holds the uid at the ceiling means the
|
|
682
|
+
// MOVE never took effect, so the row goes back to the source folder: the
|
|
683
|
+
// optimistic `updateForMove` left `mailboxId` on Trash while `uid` still
|
|
684
|
+
// named the source's message, and settling `status` to `active` on top of
|
|
685
|
+
// that pair switched off every guard that reads `status === "moving"`.
|
|
686
|
+
// The UI then resolved a Trash `mailboxId` to "Delete permanently", an
|
|
687
|
+
// expunge of Trash by a uid belonging to a different message. This is not
|
|
688
|
+
// a revert on ambiguity (#652, #655): the presence probe has just
|
|
689
|
+
// answered, so the pair written here is the placement the server holds.
|
|
690
|
+
it("moves a message still at its source back to the source folder at the ceiling", async () => {
|
|
684
691
|
h.connection.moveMessages = async () => ({ uidMap: new Map() });
|
|
685
692
|
h.destinationSearchUids = [100];
|
|
693
|
+
h.allThreadMessages = [
|
|
694
|
+
{
|
|
695
|
+
...baseThreadMessage,
|
|
696
|
+
accountConfigId: "cfg-1",
|
|
697
|
+
threadMessageId: "tm-1",
|
|
698
|
+
mailboxId: "trash-mbx",
|
|
699
|
+
isDeleted: true,
|
|
700
|
+
},
|
|
701
|
+
];
|
|
686
702
|
|
|
687
703
|
await handleMessageDelete(moveEvent, noopLog, 3, deps());
|
|
688
704
|
|
|
689
705
|
assert.equal(called("message.delete").length, 0);
|
|
690
706
|
assert.equal(called("threadMessage.deleteMany").length, 0);
|
|
691
|
-
assert.equal(called("threadMessage.update").length, 0);
|
|
692
707
|
|
|
693
|
-
//
|
|
694
|
-
// `
|
|
695
|
-
//
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
708
|
+
// `mailboxId` and `uid` name the same message again, and `updateUid`
|
|
709
|
+
// settles `status` out of `moving` in the same write, so no later
|
|
710
|
+
// delete of this message waits on a mutation that has terminated.
|
|
711
|
+
assert.deepEqual(called("message.updateUid").at(-1)?.args, [
|
|
712
|
+
"msg-1",
|
|
713
|
+
10,
|
|
714
|
+
"src-mbx",
|
|
715
|
+
]);
|
|
716
|
+
assert.equal(
|
|
717
|
+
called("message.update").length,
|
|
718
|
+
0,
|
|
719
|
+
"settling `status` without repairing the pair is what left the row addressable as a Trash expunge",
|
|
720
|
+
);
|
|
721
|
+
|
|
722
|
+
// The listing row follows the Message row out of Trash, which is what
|
|
723
|
+
// makes "Delete permanently" unreachable: the client resolves that
|
|
724
|
+
// outcome from a target whose `mailboxId` is the Trash mailbox.
|
|
725
|
+
assert.deepEqual(called("threadMessage.update").at(-1)?.args, [
|
|
726
|
+
"cfg-1",
|
|
727
|
+
"tm-1",
|
|
728
|
+
{ uid: 10, mailboxId: "src-mbx", isDeleted: false },
|
|
729
|
+
{
|
|
730
|
+
composites: {
|
|
731
|
+
sentDate: baseThreadMessage.sentDate,
|
|
732
|
+
mailboxId: "trash-mbx",
|
|
733
|
+
isRead: true,
|
|
734
|
+
isDeleted: true,
|
|
735
|
+
hasStars: true,
|
|
736
|
+
hasAttachment: false,
|
|
737
|
+
},
|
|
738
|
+
},
|
|
739
|
+
]);
|
|
701
740
|
assert.equal(await imapFailures("MESSAGE_DELETE_EXHAUSTED"), 1);
|
|
702
741
|
});
|
|
703
742
|
|
|
@@ -767,15 +806,16 @@ describe("handleMessageDelete", () => {
|
|
|
767
806
|
assert.equal(called("emitEvent").length, 2);
|
|
768
807
|
});
|
|
769
808
|
|
|
770
|
-
it("settles a still-present message at the ceiling
|
|
809
|
+
it("settles a still-present message at the ceiling back onto its source", async () => {
|
|
771
810
|
moveThrows();
|
|
772
811
|
|
|
773
812
|
await handleMessageDelete(moveEvent, noopLog, 3, deps());
|
|
774
813
|
|
|
775
|
-
assert.deepEqual(called("message.
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
814
|
+
assert.deepEqual(called("message.updateUid").at(-1)?.args, [
|
|
815
|
+
"msg-1",
|
|
816
|
+
10,
|
|
817
|
+
"src-mbx",
|
|
818
|
+
]);
|
|
779
819
|
assert.equal(await imapFailures("MESSAGE_DELETE_EXHAUSTED"), 1);
|
|
780
820
|
});
|
|
781
821
|
});
|
|
@@ -2,7 +2,6 @@ import { getClient } from "@remit/backend/client";
|
|
|
2
2
|
import type {
|
|
3
3
|
IMessageRepository,
|
|
4
4
|
IThreadMessageRepository,
|
|
5
|
-
ThreadMessageItem,
|
|
6
5
|
} from "@remit/data-ports";
|
|
7
6
|
import { isCurrentSchemaVersion } from "@remit/data-ports/mutation-events";
|
|
8
7
|
import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
|
|
@@ -24,6 +23,10 @@ import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
|
24
23
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
25
24
|
import { resolveExhaustedMessageDeleteFailure } from "./message-delete-terminal.js";
|
|
26
25
|
import { emitMoveResync, searchMailboxByMessageId } from "./message-move.js";
|
|
26
|
+
import {
|
|
27
|
+
buildThreadMessageMoveRevert,
|
|
28
|
+
buildThreadMessageTrashUpdate,
|
|
29
|
+
} from "./thread-message-rows.js";
|
|
27
30
|
|
|
28
31
|
/**
|
|
29
32
|
* Fallback when `MESSAGE_DELETE_MAX_ATTEMPTS` is unset (local dev, unit tests).
|
|
@@ -73,82 +76,6 @@ export const deleteAllThreadMessagesForMessage = async (
|
|
|
73
76
|
return rows.length;
|
|
74
77
|
};
|
|
75
78
|
|
|
76
|
-
/**
|
|
77
|
-
* Build the `set` and `composites` payload for the ThreadMessage update on a
|
|
78
|
-
* MESSAGE_DELETE move-to-trash.
|
|
79
|
-
*
|
|
80
|
-
* The CURRENT row state goes in `composites`; the NEW values go in `set`.
|
|
81
|
-
* ElectroDB uses `composites` to run the conditional check on the existing row
|
|
82
|
-
* AND to compute the previous sort-key values needed to recompute the new ones.
|
|
83
|
-
* Passing the NEW values in `composites` makes the conditional check fail with
|
|
84
|
-
* ConditionalCheckFailedException, which ElectroDB wraps as NotFoundError, and
|
|
85
|
-
* the caller silently drops the update. Same root cause as PR #186 fixed for
|
|
86
|
-
* `flag-queue.ts`.
|
|
87
|
-
*/
|
|
88
|
-
type ThreadMessageRowState = Pick<
|
|
89
|
-
ThreadMessageItem,
|
|
90
|
-
| "sentDate"
|
|
91
|
-
| "mailboxId"
|
|
92
|
-
| "isRead"
|
|
93
|
-
| "isDeleted"
|
|
94
|
-
| "hasStars"
|
|
95
|
-
| "hasAttachment"
|
|
96
|
-
>;
|
|
97
|
-
|
|
98
|
-
const currentComposites = (threadMessage: ThreadMessageRowState) => ({
|
|
99
|
-
sentDate: threadMessage.sentDate,
|
|
100
|
-
mailboxId: threadMessage.mailboxId,
|
|
101
|
-
isRead: threadMessage.isRead,
|
|
102
|
-
isDeleted: threadMessage.isDeleted,
|
|
103
|
-
hasStars: threadMessage.hasStars,
|
|
104
|
-
hasAttachment: threadMessage.hasAttachment,
|
|
105
|
-
});
|
|
106
|
-
|
|
107
|
-
export const buildThreadMessageTrashUpdate = (
|
|
108
|
-
threadMessage: ThreadMessageRowState,
|
|
109
|
-
newUid: number,
|
|
110
|
-
destinationMailboxId: string,
|
|
111
|
-
) => ({
|
|
112
|
-
set: {
|
|
113
|
-
uid: newUid,
|
|
114
|
-
mailboxId: destinationMailboxId,
|
|
115
|
-
isDeleted: true,
|
|
116
|
-
},
|
|
117
|
-
composites: currentComposites(threadMessage),
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* The inverse payload: put the thread row back where the message still is on
|
|
122
|
-
* the server. The delete was recorded optimistically, so a delete abandoned
|
|
123
|
-
* before any IMAP write has to hand the row back rather than leave it claiming
|
|
124
|
-
* Trash — an invisible `failed` on a row the user cannot see is the shape of
|
|
125
|
-
* the incident this whole change is about.
|
|
126
|
-
*/
|
|
127
|
-
export const buildThreadMessageMoveRevert = (
|
|
128
|
-
threadMessage: ThreadMessageRowState,
|
|
129
|
-
sourceUid: number,
|
|
130
|
-
sourceMailboxId: string,
|
|
131
|
-
) => ({
|
|
132
|
-
set: {
|
|
133
|
-
uid: sourceUid,
|
|
134
|
-
mailboxId: sourceMailboxId,
|
|
135
|
-
isDeleted: false,
|
|
136
|
-
},
|
|
137
|
-
composites: currentComposites(threadMessage),
|
|
138
|
-
});
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Hand a row back after an abandoned expunge. The mail never left Trash, so
|
|
142
|
-
* only the deletion mark reverts — the uid and mailbox on the row are still
|
|
143
|
-
* where the server has it.
|
|
144
|
-
*/
|
|
145
|
-
export const buildThreadMessageUndelete = (
|
|
146
|
-
threadMessage: ThreadMessageRowState,
|
|
147
|
-
) => ({
|
|
148
|
-
set: { isDeleted: false },
|
|
149
|
-
composites: currentComposites(threadMessage),
|
|
150
|
-
});
|
|
151
|
-
|
|
152
79
|
/**
|
|
153
80
|
* Settle a move to Trash the server left unconfirmed, by asking it two
|
|
154
81
|
* read-only questions instead of one. Both handles wrap the SAME connection but
|
|
@@ -349,6 +276,7 @@ export const handleMessageDelete = async (
|
|
|
349
276
|
accountId,
|
|
350
277
|
accountConfigId,
|
|
351
278
|
messageId,
|
|
279
|
+
sourceMailboxId: mailboxId,
|
|
352
280
|
uid,
|
|
353
281
|
sourceMailboxPath: mailboxPath,
|
|
354
282
|
getConnection,
|
|
@@ -362,11 +290,9 @@ export const handleMessageDelete = async (
|
|
|
362
290
|
// Both verdicts end in a row the server has contradicted, so this delete
|
|
363
291
|
// reconciles rather than waits (R2): whichever folder actually holds the
|
|
364
292
|
// message re-projects it with the server's own uid. RECONCILED, the local
|
|
365
|
-
// rows are gone and the resync rebuilds them. BROKEN, the
|
|
366
|
-
//
|
|
367
|
-
//
|
|
368
|
-
// user sees it in Trash for as long as that row stands, which is the
|
|
369
|
-
// silent misdirection the settle exists to end.
|
|
293
|
+
// rows are gone and the resync rebuilds them. BROKEN, the row has just
|
|
294
|
+
// been put back at the source the server confirmed, and the resync is
|
|
295
|
+
// what carries any drift either folder has picked up since.
|
|
370
296
|
if (destinationMailboxId) {
|
|
371
297
|
await emitMoveResync(emitEvent, {
|
|
372
298
|
accountId,
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { ThreadMessageItem } from "@remit/data-ports";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Build the `set` and `composites` payload for the ThreadMessage update on a
|
|
5
|
+
* MESSAGE_DELETE move-to-trash.
|
|
6
|
+
*
|
|
7
|
+
* The CURRENT row state goes in `composites`; the NEW values go in `set`.
|
|
8
|
+
* ElectroDB uses `composites` to run the conditional check on the existing row
|
|
9
|
+
* AND to compute the previous sort-key values needed to recompute the new ones.
|
|
10
|
+
* Passing the NEW values in `composites` makes the conditional check fail with
|
|
11
|
+
* ConditionalCheckFailedException, which ElectroDB wraps as NotFoundError, and
|
|
12
|
+
* the caller silently drops the update. Same root cause as PR #186 fixed for
|
|
13
|
+
* `flag-queue.ts`.
|
|
14
|
+
*/
|
|
15
|
+
export type ThreadMessageRowState = Pick<
|
|
16
|
+
ThreadMessageItem,
|
|
17
|
+
| "sentDate"
|
|
18
|
+
| "mailboxId"
|
|
19
|
+
| "isRead"
|
|
20
|
+
| "isDeleted"
|
|
21
|
+
| "hasStars"
|
|
22
|
+
| "hasAttachment"
|
|
23
|
+
>;
|
|
24
|
+
|
|
25
|
+
const currentComposites = (threadMessage: ThreadMessageRowState) => ({
|
|
26
|
+
sentDate: threadMessage.sentDate,
|
|
27
|
+
mailboxId: threadMessage.mailboxId,
|
|
28
|
+
isRead: threadMessage.isRead,
|
|
29
|
+
isDeleted: threadMessage.isDeleted,
|
|
30
|
+
hasStars: threadMessage.hasStars,
|
|
31
|
+
hasAttachment: threadMessage.hasAttachment,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
export const buildThreadMessageTrashUpdate = (
|
|
35
|
+
threadMessage: ThreadMessageRowState,
|
|
36
|
+
newUid: number,
|
|
37
|
+
destinationMailboxId: string,
|
|
38
|
+
) => ({
|
|
39
|
+
set: {
|
|
40
|
+
uid: newUid,
|
|
41
|
+
mailboxId: destinationMailboxId,
|
|
42
|
+
isDeleted: true,
|
|
43
|
+
},
|
|
44
|
+
composites: currentComposites(threadMessage),
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The inverse payload: put the thread row back where the message still is on
|
|
49
|
+
* the server. The delete was recorded optimistically, so a delete that gives up
|
|
50
|
+
* with the message still at its source has to hand the row back rather than
|
|
51
|
+
* leave it claiming Trash — an invisible `failed` on a row the user cannot see
|
|
52
|
+
* is the shape of the incident this whole change is about.
|
|
53
|
+
*/
|
|
54
|
+
export const buildThreadMessageMoveRevert = (
|
|
55
|
+
threadMessage: ThreadMessageRowState,
|
|
56
|
+
sourceUid: number,
|
|
57
|
+
sourceMailboxId: string,
|
|
58
|
+
) => ({
|
|
59
|
+
set: {
|
|
60
|
+
uid: sourceUid,
|
|
61
|
+
mailboxId: sourceMailboxId,
|
|
62
|
+
isDeleted: false,
|
|
63
|
+
},
|
|
64
|
+
composites: currentComposites(threadMessage),
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Hand a row back after an abandoned expunge. The mail never left Trash, so
|
|
69
|
+
* only the deletion mark reverts — the uid and mailbox on the row are still
|
|
70
|
+
* where the server has it.
|
|
71
|
+
*/
|
|
72
|
+
export const buildThreadMessageUndelete = (
|
|
73
|
+
threadMessage: ThreadMessageRowState,
|
|
74
|
+
) => ({
|
|
75
|
+
set: { isDeleted: false },
|
|
76
|
+
composites: currentComposites(threadMessage),
|
|
77
|
+
});
|
|
@@ -51,6 +51,7 @@ const baseAccount = (overrides: Partial<AccountItem>): AccountItem =>
|
|
|
51
51
|
username: "user",
|
|
52
52
|
email: "user@example.com",
|
|
53
53
|
authType: "password",
|
|
54
|
+
passwordHash: '{"ciphertext":"x"}',
|
|
54
55
|
imapHost: "imap.example.com",
|
|
55
56
|
imapPort: 993,
|
|
56
57
|
imapTls: true,
|
|
@@ -162,6 +163,36 @@ describe("runSchedulerTick", () => {
|
|
|
162
163
|
assert.equal(sent.length, 0);
|
|
163
164
|
});
|
|
164
165
|
|
|
166
|
+
it("skips an account storing no credential (issue #1120)", async () => {
|
|
167
|
+
const due = baseAccount({ accountId: "acct_due" });
|
|
168
|
+
const credentialless = baseAccount({
|
|
169
|
+
accountId: "acct_no_credential",
|
|
170
|
+
passwordHash: undefined,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
const accountService = fakeAccountService([
|
|
174
|
+
{ items: [due, credentialless], cursor: null },
|
|
175
|
+
]);
|
|
176
|
+
const { sqsClient, sent } = fakeSqsClient();
|
|
177
|
+
|
|
178
|
+
const result = await runSchedulerTick({
|
|
179
|
+
accountService,
|
|
180
|
+
sqsClient,
|
|
181
|
+
queueUrl:
|
|
182
|
+
"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
|
|
183
|
+
log: createNoopLogger(),
|
|
184
|
+
tickIntervalMs: TICK_INTERVAL_MS,
|
|
185
|
+
offlineIntervalMs: OFFLINE_INTERVAL_MS,
|
|
186
|
+
now: NOW,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
assert.equal(result.enqueued, 1);
|
|
190
|
+
assert.equal(result.skipped, 1);
|
|
191
|
+
assert.equal(sent.length, 1);
|
|
192
|
+
const body = JSON.parse(sent[0]?.input.MessageBody ?? "{}");
|
|
193
|
+
assert.equal(body.accountId, "acct_due");
|
|
194
|
+
});
|
|
195
|
+
|
|
165
196
|
it("never logs per-account for ineligible accounts — only the aggregate tick summary (review #1250)", async () => {
|
|
166
197
|
const deleted = baseAccount({ accountId: "acct_deleted", deletedAt: NOW });
|
|
167
198
|
const unsyncable = baseAccount({
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
} from "@remit/backend/trigger-sync";
|
|
6
6
|
import type { AccountItem, IAccountRepository } from "@remit/data-ports";
|
|
7
7
|
import type { Logger } from "@remit/logger-lambda";
|
|
8
|
+
import { hasStoredCredential } from "@remit/mailbox-service/account-credentials";
|
|
8
9
|
import pMap from "p-map";
|
|
9
10
|
import {
|
|
10
11
|
isAccountDeleted,
|
|
@@ -79,6 +80,10 @@ const isEligible = (account: AccountItem): boolean => {
|
|
|
79
80
|
if (isAccountDeleted(account, silentLogger)) return false;
|
|
80
81
|
if (isUnsyncableHost(account, silentLogger)) return false;
|
|
81
82
|
if (isAccountReauthRequired(account, silentLogger)) return false;
|
|
83
|
+
// An account storing no credential can only produce a sync that fails at
|
|
84
|
+
// credential resolution (issue #1120); the tick would enqueue one every
|
|
85
|
+
// tick, forever, for an account the user never finished setting up.
|
|
86
|
+
if (!hasStoredCredential(account)) return false;
|
|
82
87
|
return true;
|
|
83
88
|
};
|
|
84
89
|
|
|
@@ -6,7 +6,7 @@ import { RefreshTokenError } from "@remit/mail-oauth-service";
|
|
|
6
6
|
import {
|
|
7
7
|
MailConnectionError,
|
|
8
8
|
type MailCredentials,
|
|
9
|
-
} from "@remit/mailbox-service";
|
|
9
|
+
} from "@remit/mailbox-service/types";
|
|
10
10
|
import {
|
|
11
11
|
type ConnectionStateValue,
|
|
12
12
|
type OAuthLifecycleDeps,
|
|
@@ -75,7 +75,7 @@ const buildDeps = (
|
|
|
75
75
|
options.resolveCredentials ??
|
|
76
76
|
(async () => {
|
|
77
77
|
recorded.resolveCalls += 1;
|
|
78
|
-
return passwordCreds;
|
|
78
|
+
return { status: "resolved", credentials: passwordCreds };
|
|
79
79
|
}),
|
|
80
80
|
};
|
|
81
81
|
// Wrap resolveCredentials to record the call count when a custom one is given.
|
|
@@ -210,6 +210,36 @@ describe("withOAuthLifecycle", () => {
|
|
|
210
210
|
assert.equal(recorded.stateUpdates.length, 0);
|
|
211
211
|
});
|
|
212
212
|
|
|
213
|
+
it("password account with no passwordHash: ACKs and flips to credentials_missing", async () => {
|
|
214
|
+
const { deps, recorded } = buildDeps();
|
|
215
|
+
delete deps.resolveCredentials;
|
|
216
|
+
const account = buildAccount({ authType: AccountAuthType.Password });
|
|
217
|
+
|
|
218
|
+
await withOAuthLifecycle(deps, account, silentLogger, async () => {
|
|
219
|
+
recorded.workCalls += 1;
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
assert.equal(recorded.workCalls, 0, "no IMAP traffic without a credential");
|
|
223
|
+
assert.deepEqual(recorded.stateUpdates, [
|
|
224
|
+
{ accountId: "acc-1", state: ConnectionState.CredentialsMissing },
|
|
225
|
+
]);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it("OAuth account with no refresh token: ACKs and flips to reauth_required", async () => {
|
|
229
|
+
const { deps, recorded } = buildDeps();
|
|
230
|
+
delete deps.resolveCredentials;
|
|
231
|
+
const account = buildAccount({ authType: AccountAuthType.OauthMicrosoft });
|
|
232
|
+
|
|
233
|
+
await withOAuthLifecycle(deps, account, silentLogger, async () => {
|
|
234
|
+
recorded.workCalls += 1;
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
assert.equal(recorded.workCalls, 0, "no IMAP traffic without a credential");
|
|
238
|
+
assert.deepEqual(recorded.stateUpdates, [
|
|
239
|
+
{ accountId: "acc-1", state: ConnectionState.ReauthRequired },
|
|
240
|
+
]);
|
|
241
|
+
});
|
|
242
|
+
|
|
213
243
|
it("on RefreshTokenError transient: rethrows", async () => {
|
|
214
244
|
const { deps, recorded } = buildDeps({
|
|
215
245
|
resolveCredentials: async () => {
|
|
@@ -8,6 +8,11 @@
|
|
|
8
8
|
* - Credential resolution AND the per-handler work both run inside one
|
|
9
9
|
* try/catch so that a revoked OAuth token is caught regardless of where it
|
|
10
10
|
* surfaces (token mint vs. first IMAP command).
|
|
11
|
+
* - An account storing no credential at all is terminal, not transient: the
|
|
12
|
+
* account is flipped to the state that describes it and the event is ACKed.
|
|
13
|
+
* Retrying instead fenced the account's FIFO group for the message's whole
|
|
14
|
+
* visibility timeout, blocking the sync that follows the credential the
|
|
15
|
+
* user was still typing (issue #1120).
|
|
11
16
|
* - On a terminal auth failure (RefreshTokenError reauth-required, or
|
|
12
17
|
* MailConnectionError auth), flip the account to reauth_required and return
|
|
13
18
|
* WITHOUT rethrowing — this ACKs the SQS message so it is not retried.
|
|
@@ -23,15 +28,17 @@ import type { Logger } from "@remit/logger-lambda";
|
|
|
23
28
|
import { RefreshTokenError } from "@remit/mail-oauth-service";
|
|
24
29
|
import {
|
|
25
30
|
type AccountCredentialsDeps,
|
|
31
|
+
type ConnectionStateValue,
|
|
32
|
+
type CredentialResolution,
|
|
33
|
+
resolveConnectionCredentials,
|
|
34
|
+
} from "@remit/mailbox-service/account-credentials";
|
|
35
|
+
import {
|
|
26
36
|
MailConnectionError,
|
|
27
37
|
type MailCredentials,
|
|
28
|
-
|
|
29
|
-
} from "@remit/mailbox-service";
|
|
38
|
+
} from "@remit/mailbox-service/types";
|
|
30
39
|
import { isAccountReauthRequired } from "./account-check.js";
|
|
31
40
|
|
|
32
|
-
|
|
33
|
-
export type ConnectionStateValue =
|
|
34
|
-
(typeof ConnectionState)[keyof typeof ConnectionState];
|
|
41
|
+
export type { ConnectionStateValue };
|
|
35
42
|
|
|
36
43
|
export interface OAuthLifecycleDeps extends AccountCredentialsDeps {
|
|
37
44
|
/**
|
|
@@ -50,7 +57,7 @@ export interface OAuthLifecycleDeps extends AccountCredentialsDeps {
|
|
|
50
57
|
resolveCredentials?: (
|
|
51
58
|
account: AccountItem,
|
|
52
59
|
deps: AccountCredentialsDeps,
|
|
53
|
-
) => Promise<
|
|
60
|
+
) => Promise<CredentialResolution>;
|
|
54
61
|
}
|
|
55
62
|
|
|
56
63
|
/**
|
|
@@ -73,8 +80,23 @@ export const withOAuthLifecycle = async (
|
|
|
73
80
|
const resolve = deps.resolveCredentials ?? resolveConnectionCredentials;
|
|
74
81
|
|
|
75
82
|
try {
|
|
76
|
-
const
|
|
77
|
-
|
|
83
|
+
const resolution = await resolve(account, deps);
|
|
84
|
+
if (resolution.status === "missing") {
|
|
85
|
+
log.warn(
|
|
86
|
+
{
|
|
87
|
+
accountId: account.accountId,
|
|
88
|
+
reason: resolution.reason,
|
|
89
|
+
connectionState: resolution.terminalState,
|
|
90
|
+
},
|
|
91
|
+
"Account stores no credential; acking the event without IMAP traffic",
|
|
92
|
+
);
|
|
93
|
+
await deps.updateConnectionState(
|
|
94
|
+
account.accountId,
|
|
95
|
+
resolution.terminalState,
|
|
96
|
+
);
|
|
97
|
+
return; // ACK — no retry can produce a credential
|
|
98
|
+
}
|
|
99
|
+
await work(resolution.credentials);
|
|
78
100
|
} catch (err) {
|
|
79
101
|
// Terminal OAuth failure: token revoked / consent withdrawn.
|
|
80
102
|
if (err instanceof RefreshTokenError) {
|