@remit/imap-worker 0.0.86 → 0.0.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +10 -2
- package/src/account-check.ts +1 -1
- package/src/body-sync-gate.test.ts +1 -1
- package/src/body-sync-gate.ts +1 -1
- package/src/e2e-processor-shim.ts +6 -21
- package/src/emit.ts +1 -1
- package/src/failure-kind.ts +1 -1
- package/src/handlers/append-sent-message.sqlite.test.ts +176 -0
- package/src/handlers/append-sent-message.ts +3 -24
- package/src/handlers/delete-account-objects.test.ts +1 -1
- package/src/handlers/delete-account-objects.ts +1 -1
- package/src/handlers/empty-trash.sqlite.test.ts +209 -0
- package/src/handlers/empty-trash.ts +3 -24
- package/src/handlers/flag-push.ts +11 -18
- package/src/handlers/mailbox-management.sqlite.test.ts +265 -0
- package/src/handlers/mailbox-management.ts +12 -48
- package/src/handlers/message-copy.sqlite.test.ts +298 -0
- package/src/handlers/message-copy.ts +31 -49
- package/src/handlers/message-delete-terminal.ts +12 -10
- package/src/handlers/message-delete.sqlite.test.ts +291 -0
- package/src/handlers/message-delete.ts +26 -32
- package/src/handlers/message-move-terminal.ts +12 -10
- package/src/handlers/message-move.sqlite.test.ts +271 -0
- package/src/handlers/message-move.test.ts +76 -863
- package/src/handlers/message-move.ts +49 -39
- package/src/handlers/placement-move-push.test.ts +96 -329
- package/src/handlers/placement-move-push.ts +75 -29
- package/src/handlers/placement-push.sqlite.test.ts +298 -0
- package/src/handlers/probe-message-id.sqlite.test.ts +197 -0
- package/src/handlers/probe-message-id.ts +15 -0
- package/src/handlers/sync-mailboxes.test.ts +0 -1
- package/src/handlers/sync-mailboxes.ts +1 -1
- package/src/handlers/sync-message-body.sqlite.test.ts +314 -0
- package/src/handlers/sync-message-body.test.ts +3 -299
- package/src/handlers/sync-message-body.ts +21 -11
- package/src/handlers/sync-messages.sqlite.test.ts +183 -0
- package/src/handlers/sync-messages.ts +27 -33
- package/src/index.ts +5 -3
- package/src/mail-sync-selection.sqlite.test.ts +16 -18
- package/src/poller.ts +5 -4
- package/src/processor.test.ts +1 -1
- package/src/processor.ts +1 -1
- package/src/scheduler/attachment-sweep.sqlite.test.ts +74 -0
- package/src/scheduler/attachment-sweep.ts +1 -1
- package/src/scheduler/{calendar-subscriptions.test.ts → calendar-subscriptions.sqlite.test.ts} +43 -55
- package/src/scheduler/calendar-subscriptions.ts +6 -6
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +203 -0
- package/src/scheduler/handler.ts +12 -1
- package/src/scheduler/loop.test.ts +2 -0
- package/src/scheduler/run-tick.sqlite.test.ts +192 -0
- package/src/scheduler/run-tick.ts +39 -1
- package/src/scheduler/runner.ts +21 -1
- package/src/scheduler/stranded-placement-redrive.ts +265 -0
- package/src/stranded-placement.sqlite.test.ts +795 -0
- package/src/test-helpers/test-world.ts +398 -0
- package/src/thread-join.sqlite.test.ts +115 -0
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +138 -14
- package/src/with-oauth-lifecycle.ts +1 -1
- package/src/handlers/append-sent-message.test.ts +0 -529
- package/src/handlers/empty-trash.test.ts +0 -728
- package/src/handlers/flag-push-placement-race.test.ts +0 -385
- package/src/handlers/flag-push.test.ts +0 -325
- package/src/handlers/mailbox-management.test.ts +0 -850
- package/src/handlers/message-copy.test.ts +0 -654
- package/src/handlers/message-delete.test.ts +0 -1869
- package/src/handlers/message-move-terminal.test.ts +0 -412
- package/src/handlers/sync-messages.test.ts +0 -569
- package/src/scheduler/attachment-sweep.test.ts +0 -169
- package/src/scheduler/run-tick.test.ts +0 -478
- package/src/with-oauth-lifecycle.test.ts +0 -318
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { getClient } from "@remit/backend/client";
|
|
2
2
|
import { isNotFoundError } from "@remit/data-ports/errors";
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
3
|
+
import { SYNC_WARNING_SURFACE_ATTEMPTS } from "@remit/data-ports/sync-warning-state";
|
|
4
|
+
import type { Logger } from "@remit/logger";
|
|
5
|
+
import { recordImapFailure } from "@remit/logger-lambda/metrics";
|
|
5
6
|
import {
|
|
6
7
|
BodySyncService,
|
|
7
8
|
guardConnectionCursor,
|
|
@@ -12,6 +13,7 @@ import {
|
|
|
12
13
|
QuarantineService,
|
|
13
14
|
resolveExhaustedBodySyncFailures,
|
|
14
15
|
resolveFolderPlacement,
|
|
16
|
+
SyncWarningService,
|
|
15
17
|
} from "@remit/mailbox-service";
|
|
16
18
|
import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
|
|
17
19
|
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
@@ -31,7 +33,12 @@ const bodySyncEnabledParameterName = env.BODY_SYNC_ENABLED_PARAMETER_NAME;
|
|
|
31
33
|
|
|
32
34
|
export const getBodySyncMaxAttempts = (
|
|
33
35
|
processEnv: NodeJS.ProcessEnv = process.env,
|
|
34
|
-
): number =>
|
|
36
|
+
): number =>
|
|
37
|
+
attemptBudget(
|
|
38
|
+
"BODY_SYNC_MAX_ATTEMPTS",
|
|
39
|
+
SYNC_WARNING_SURFACE_ATTEMPTS,
|
|
40
|
+
processEnv,
|
|
41
|
+
);
|
|
35
42
|
|
|
36
43
|
export const BODY_SYNC_MAX_ATTEMPTS = getBodySyncMaxAttempts();
|
|
37
44
|
|
|
@@ -81,6 +88,7 @@ export const syncMessageBody = async (
|
|
|
81
88
|
event: SyncMessageBodyEvent,
|
|
82
89
|
log: Logger,
|
|
83
90
|
receiveCount = 1,
|
|
91
|
+
createConnectionScope = createConnectionScopeWithCredentials,
|
|
84
92
|
): Promise<void> => {
|
|
85
93
|
const { accountId, mailboxId } = event;
|
|
86
94
|
|
|
@@ -125,15 +133,15 @@ export const syncMessageBody = async (
|
|
|
125
133
|
envelope: envelopeService,
|
|
126
134
|
placementMove: markerService,
|
|
127
135
|
filter: filterService,
|
|
128
|
-
filterAnchor: filterAnchorService,
|
|
129
136
|
messageLabel: messageLabelService,
|
|
130
137
|
storage,
|
|
131
138
|
secrets,
|
|
132
139
|
mailboxSpecialUse: mailboxSpecialUseRepository,
|
|
133
140
|
quarantine: quarantineRepository,
|
|
141
|
+
syncWarning: syncWarningRepository,
|
|
134
142
|
flagQueue: flagQueueService,
|
|
135
143
|
calendarSuggestion: calendarSuggestionService,
|
|
136
|
-
|
|
144
|
+
calendarStore,
|
|
137
145
|
} = await getClient();
|
|
138
146
|
|
|
139
147
|
const account = await accountService.get(accountId);
|
|
@@ -156,6 +164,7 @@ export const syncMessageBody = async (
|
|
|
156
164
|
markerService,
|
|
157
165
|
addressService,
|
|
158
166
|
mailboxSpecialUseService,
|
|
167
|
+
mailboxService,
|
|
159
168
|
sqsQueueUrl: placementMoveQueueUrl,
|
|
160
169
|
})
|
|
161
170
|
: undefined;
|
|
@@ -205,7 +214,7 @@ export const syncMessageBody = async (
|
|
|
205
214
|
// warm container skips TCP+TLS+LOGIN+SELECT; a dead pooled connection is
|
|
206
215
|
// liveness-checked and replaced inside borrowWarmConnection.
|
|
207
216
|
const borrowed = borrowWarmConnection(accountId, () =>
|
|
208
|
-
|
|
217
|
+
createConnectionScope(account, credentials),
|
|
209
218
|
);
|
|
210
219
|
|
|
211
220
|
// A confident, actionable placement verdict moves mail directly on
|
|
@@ -220,12 +229,11 @@ export const syncMessageBody = async (
|
|
|
220
229
|
|
|
221
230
|
// A matched filter's actions (label upsert, exclusive move) apply on
|
|
222
231
|
// the same body-sync pass, reusing the placement mover for the move
|
|
223
|
-
// (RFC 034 Decision 3.1).
|
|
224
|
-
// (anchor-only) filter match here instead of being silently skipped.
|
|
232
|
+
// (RFC 034 Decision 3.1).
|
|
225
233
|
const filterConfig = buildFilterConfig({
|
|
226
234
|
filterService,
|
|
227
|
-
filterAnchorService,
|
|
228
235
|
messageLabelService,
|
|
236
|
+
mailboxService,
|
|
229
237
|
placementMoveService,
|
|
230
238
|
});
|
|
231
239
|
|
|
@@ -235,6 +243,7 @@ export const syncMessageBody = async (
|
|
|
235
243
|
threadMessageService,
|
|
236
244
|
addressService,
|
|
237
245
|
envelopeService,
|
|
246
|
+
new SyncWarningService(syncWarningRepository),
|
|
238
247
|
log,
|
|
239
248
|
placementConfig,
|
|
240
249
|
filterConfig,
|
|
@@ -252,7 +261,7 @@ export const syncMessageBody = async (
|
|
|
252
261
|
{ flagQueueService },
|
|
253
262
|
// An invitation becomes a card the first time the message is seen,
|
|
254
263
|
// alongside the filters that run on the same pass (issue #1033).
|
|
255
|
-
{ calendarSuggestionService,
|
|
264
|
+
{ calendarSuggestionService, calendarStore, filterService },
|
|
256
265
|
);
|
|
257
266
|
|
|
258
267
|
// Guard at the openBox choke point (epic #1281 invariants 3 & 5). The
|
|
@@ -274,7 +283,9 @@ export const syncMessageBody = async (
|
|
|
274
283
|
// against the SAME borrowed connection, so the mailbox stays open
|
|
275
284
|
// across the two — the connection is only released once both are done.
|
|
276
285
|
const folder = {
|
|
286
|
+
mailboxId: mailbox.mailboxId,
|
|
277
287
|
fullPath: mailbox.fullPath,
|
|
288
|
+
uidValidity: mailbox.uidValidity,
|
|
278
289
|
placement: await resolveFolderPlacement(
|
|
279
290
|
mailboxSpecialUseService,
|
|
280
291
|
accountId,
|
|
@@ -315,7 +326,6 @@ export const syncMessageBody = async (
|
|
|
315
326
|
const { brokenMessageIds } = await resolveExhaustedBodySyncFailures(
|
|
316
327
|
{
|
|
317
328
|
messageService,
|
|
318
|
-
threadMessageService,
|
|
319
329
|
storageService: storage,
|
|
320
330
|
log,
|
|
321
331
|
},
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { after, before, describe, it } from "node:test";
|
|
3
|
+
import type { SyncMessagesEvent } from "../events.js";
|
|
4
|
+
import { openStore, type World } from "../test-helpers/test-world.js";
|
|
5
|
+
import { drainPendingFlagPushes, syncMessages } from "./sync-messages.js";
|
|
6
|
+
|
|
7
|
+
let store: Awaited<ReturnType<typeof openStore>>;
|
|
8
|
+
|
|
9
|
+
before(async () => {
|
|
10
|
+
store = await openStore();
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
after(() => store.close());
|
|
14
|
+
|
|
15
|
+
const syncEvent = (w: World, mailboxId: string): SyncMessagesEvent => ({
|
|
16
|
+
type: "SYNC_MESSAGES",
|
|
17
|
+
accountId: w.account.accountId,
|
|
18
|
+
mailboxId,
|
|
19
|
+
eventId: `evt-${mailboxId}`,
|
|
20
|
+
timestamp: Date.now(),
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
const sync = async (w: World, mailboxId: string): Promise<number> => {
|
|
24
|
+
await syncMessages(syncEvent(w, mailboxId), w.log, w.connectionFactory);
|
|
25
|
+
return w.imap.connections;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
describe("which SYNC_MESSAGES events reach the server", () => {
|
|
29
|
+
it("a live mailbox does", async () => {
|
|
30
|
+
const w = await store.world();
|
|
31
|
+
const inbox = await w.folder("INBOX");
|
|
32
|
+
|
|
33
|
+
assert.equal(await sync(w, inbox.mailboxId), 1);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// Issue #287: a NotFoundError at the head of the account's FIFO group
|
|
37
|
+
// stalled every later event for the account.
|
|
38
|
+
it("a mailbox whose row is gone is acked without connecting", async () => {
|
|
39
|
+
const w = await store.world();
|
|
40
|
+
const old = await w.folder("Old");
|
|
41
|
+
await w.client.mailbox.delete(w.account.accountId, old.mailboxId);
|
|
42
|
+
|
|
43
|
+
assert.equal(await sync(w, old.mailboxId), 0);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("a mailbox being deleted is acked without connecting", async () => {
|
|
47
|
+
const w = await store.world();
|
|
48
|
+
const old = await w.folder("Old");
|
|
49
|
+
await w.client.mailboxQueue.deleteMailbox(
|
|
50
|
+
old.mailboxId,
|
|
51
|
+
w.account.accountId,
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
assert.equal(await sync(w, old.mailboxId), 0);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("a mailbox not yet created on the server is acked without connecting", async () => {
|
|
58
|
+
const w = await store.world();
|
|
59
|
+
const created = await w.client.mailboxQueue.createMailbox(
|
|
60
|
+
{
|
|
61
|
+
accountId: w.account.accountId,
|
|
62
|
+
namespacePrefix: "",
|
|
63
|
+
hierarchyDelimiter: "/",
|
|
64
|
+
fullPath: "Receipts",
|
|
65
|
+
uidValidity: 0,
|
|
66
|
+
uidNext: 0,
|
|
67
|
+
highestModseq: "0",
|
|
68
|
+
messageCount: 0,
|
|
69
|
+
unseenCount: 0,
|
|
70
|
+
deletedCount: 0,
|
|
71
|
+
totalSize: 0,
|
|
72
|
+
lastSyncUid: 0,
|
|
73
|
+
highWaterMarkUid: 0,
|
|
74
|
+
lastMessageSyncAt: 0,
|
|
75
|
+
},
|
|
76
|
+
w.account.accountId,
|
|
77
|
+
);
|
|
78
|
+
if (created.outcome !== "Created") throw new Error("path taken");
|
|
79
|
+
|
|
80
|
+
assert.equal(await sync(w, created.mailbox.mailboxId), 0);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
// Issue #339: a delete asked for while the round is in flight lands inside
|
|
85
|
+
// the guard, and the round then fails deeper on a folder that is gone.
|
|
86
|
+
describe("a folder deleted while its sync is in flight", () => {
|
|
87
|
+
it("resolves the event and records no error phase on the account", async () => {
|
|
88
|
+
const w = await store.world();
|
|
89
|
+
const old = await w.folder("Old");
|
|
90
|
+
const connectionFactory = () => {
|
|
91
|
+
const factory = w.connectionFactory();
|
|
92
|
+
const connection = factory.getConnection();
|
|
93
|
+
const openBox = connection.openBox;
|
|
94
|
+
let first = true;
|
|
95
|
+
connection.openBox = async (path, readOnly) => {
|
|
96
|
+
if (first) {
|
|
97
|
+
first = false;
|
|
98
|
+
await w.client.mailboxQueue.deleteMailbox(
|
|
99
|
+
old.mailboxId,
|
|
100
|
+
w.account.accountId,
|
|
101
|
+
);
|
|
102
|
+
w.imap.removeFolder("Old");
|
|
103
|
+
}
|
|
104
|
+
return openBox(path, readOnly);
|
|
105
|
+
};
|
|
106
|
+
return { getConnection: () => connection, close: factory.close };
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
await syncMessages(syncEvent(w, old.mailboxId), w.log, connectionFactory);
|
|
110
|
+
|
|
111
|
+
const account = await w.client.account.get(w.account.accountId);
|
|
112
|
+
assert.notEqual(account.syncPhase, "error");
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe("drainPendingFlagPushes (issue #1273)", () => {
|
|
117
|
+
const marker = (w: World, mailboxId: string, messageId: string) =>
|
|
118
|
+
w.client.flagPush.put({
|
|
119
|
+
messageId,
|
|
120
|
+
flagName: "\\Seen",
|
|
121
|
+
accountId: w.account.accountId,
|
|
122
|
+
accountConfigId: w.account.accountConfigId,
|
|
123
|
+
mailboxId,
|
|
124
|
+
operation: "add",
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("re-arms the markers stuck before their wake-up and leaves the ones a driver owns", async () => {
|
|
128
|
+
const w = await store.world();
|
|
129
|
+
const inbox = await w.folder("INBOX");
|
|
130
|
+
await marker(w, inbox.mailboxId, "msg-pending");
|
|
131
|
+
await marker(w, inbox.mailboxId, "msg-queued");
|
|
132
|
+
await w.client.flagPush.updateState("msg-queued", "\\Seen", "queued");
|
|
133
|
+
await marker(w, inbox.mailboxId, "msg-processing");
|
|
134
|
+
await w.client.flagPush.updateState(
|
|
135
|
+
"msg-processing",
|
|
136
|
+
"\\Seen",
|
|
137
|
+
"processing",
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
await drainPendingFlagPushes(
|
|
141
|
+
w.client.flagPush,
|
|
142
|
+
w.account,
|
|
143
|
+
inbox.mailboxId,
|
|
144
|
+
w.log,
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
assert.deepEqual(
|
|
148
|
+
w
|
|
149
|
+
.queued()
|
|
150
|
+
.filter((body) => body.type === "FLAG_PUSH")
|
|
151
|
+
.map((body) => body.messageId),
|
|
152
|
+
["msg-pending"],
|
|
153
|
+
);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("keeps re-arming the others when one re-arm fails, and says so loudly", async () => {
|
|
157
|
+
const w = await store.world();
|
|
158
|
+
const inbox = await w.folder("INBOX");
|
|
159
|
+
await marker(w, inbox.mailboxId, "will-fail");
|
|
160
|
+
await marker(w, inbox.mailboxId, "will-succeed");
|
|
161
|
+
const armed: string[] = [];
|
|
162
|
+
|
|
163
|
+
await drainPendingFlagPushes(
|
|
164
|
+
w.client.flagPush,
|
|
165
|
+
w.account,
|
|
166
|
+
inbox.mailboxId,
|
|
167
|
+
w.log,
|
|
168
|
+
async (event) => {
|
|
169
|
+
const { messageId } = event as unknown as { messageId: string };
|
|
170
|
+
if (messageId === "will-fail") throw new Error("queue down");
|
|
171
|
+
armed.push(messageId);
|
|
172
|
+
},
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
assert.deepEqual(armed, ["will-succeed"]);
|
|
176
|
+
assert.equal(
|
|
177
|
+
w.logs.filter(
|
|
178
|
+
(line) => line.fields.alert === "flag_push_drain_rearm_failed",
|
|
179
|
+
).length,
|
|
180
|
+
1,
|
|
181
|
+
);
|
|
182
|
+
});
|
|
183
|
+
});
|
|
@@ -10,21 +10,24 @@ import type {
|
|
|
10
10
|
IMessageFlagRepository,
|
|
11
11
|
IMessageRepository,
|
|
12
12
|
IQuarantineRepository,
|
|
13
|
+
ISyncWarningRepository,
|
|
13
14
|
IThreadMessageRepository,
|
|
14
|
-
IUnitOfWork,
|
|
15
15
|
} from "@remit/data-ports";
|
|
16
16
|
import { isNotFoundError } from "@remit/data-ports/errors";
|
|
17
17
|
import { SyncPhase } from "@remit/domain-enums";
|
|
18
|
-
import {
|
|
18
|
+
import type { Logger } from "@remit/logger";
|
|
19
|
+
import { recordImapFailure } from "@remit/logger-lambda/metrics";
|
|
19
20
|
import { RefreshTokenError } from "@remit/mail-oauth-service";
|
|
20
21
|
import {
|
|
21
22
|
createManagedConnectionFactory,
|
|
22
23
|
isMailboxMutationInFlight,
|
|
23
24
|
MailConnectionError,
|
|
24
25
|
type MailCredentials,
|
|
26
|
+
type ManagedConnectionFactory,
|
|
25
27
|
MessageSyncService,
|
|
26
28
|
QuarantineService,
|
|
27
29
|
type SyncedMessage,
|
|
30
|
+
SyncWarningService,
|
|
28
31
|
} from "@remit/mailbox-service";
|
|
29
32
|
import pMap from "p-map";
|
|
30
33
|
import {
|
|
@@ -59,22 +62,22 @@ export const batchSyncedMessages = (
|
|
|
59
62
|
return batches;
|
|
60
63
|
};
|
|
61
64
|
|
|
62
|
-
export
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
};
|
|
65
|
+
export const managedConnectionFor = (
|
|
66
|
+
account: Pick<AccountItem, "username" | "imapHost" | "imapPort" | "imapTls">,
|
|
67
|
+
credentials: MailCredentials,
|
|
68
|
+
): ManagedConnectionFactory =>
|
|
69
|
+
createManagedConnectionFactory({
|
|
70
|
+
user: account.username,
|
|
71
|
+
credentials,
|
|
72
|
+
host: account.imapHost,
|
|
73
|
+
port: account.imapPort,
|
|
74
|
+
tls: account.imapTls,
|
|
75
|
+
});
|
|
73
76
|
|
|
74
77
|
export const syncMessages = async (
|
|
75
78
|
event: SyncMessagesEvent,
|
|
76
79
|
log: Logger,
|
|
77
|
-
|
|
80
|
+
createConnectionFactory = managedConnectionFor,
|
|
78
81
|
): Promise<void> => {
|
|
79
82
|
log.info(
|
|
80
83
|
{
|
|
@@ -95,11 +98,11 @@ export const syncMessages = async (
|
|
|
95
98
|
mailboxLock: mailboxLockService,
|
|
96
99
|
mailboxSpecialUse: mailboxSpecialUseService,
|
|
97
100
|
quarantine: quarantineService,
|
|
101
|
+
syncWarning: syncWarningRepository,
|
|
98
102
|
flagPush: flagPushMarkerService,
|
|
99
103
|
messageFlag: messageFlagService,
|
|
100
|
-
unitOfWork,
|
|
101
104
|
secrets,
|
|
102
|
-
} = await
|
|
105
|
+
} = await getClient();
|
|
103
106
|
|
|
104
107
|
await mailboxLockService.releaseLock(
|
|
105
108
|
event.accountId,
|
|
@@ -181,8 +184,8 @@ export const syncMessages = async (
|
|
|
181
184
|
// withOAuthLifecycle owns the reauth/ACK contract (skip-if-reauth, resolve
|
|
182
185
|
// credentials, flip on terminal auth failure, rethrow transient). The
|
|
183
186
|
// mailbox lock and the actual sync run inside the wrapper callback.
|
|
184
|
-
await
|
|
185
|
-
|
|
187
|
+
await withOAuthLifecycle(
|
|
188
|
+
buildLifecycleDeps(secrets, accountService),
|
|
186
189
|
account,
|
|
187
190
|
log,
|
|
188
191
|
async (credentials) => {
|
|
@@ -196,7 +199,7 @@ export const syncMessages = async (
|
|
|
196
199
|
await syncMailboxMessages(
|
|
197
200
|
event,
|
|
198
201
|
account,
|
|
199
|
-
credentials,
|
|
202
|
+
createConnectionFactory(account, credentials),
|
|
200
203
|
{
|
|
201
204
|
accountService,
|
|
202
205
|
mailboxService,
|
|
@@ -206,9 +209,9 @@ export const syncMessages = async (
|
|
|
206
209
|
threadMessageService,
|
|
207
210
|
mailboxSpecialUseService,
|
|
208
211
|
quarantineService,
|
|
212
|
+
syncWarningRepository,
|
|
209
213
|
flagPushMarkerService,
|
|
210
214
|
messageFlagService,
|
|
211
|
-
unitOfWork,
|
|
212
215
|
},
|
|
213
216
|
log,
|
|
214
217
|
);
|
|
@@ -284,9 +287,9 @@ interface SyncDeps {
|
|
|
284
287
|
threadMessageService: IThreadMessageRepository;
|
|
285
288
|
mailboxSpecialUseService: IMailboxSpecialUseRepository;
|
|
286
289
|
quarantineService: IQuarantineRepository;
|
|
290
|
+
syncWarningRepository: ISyncWarningRepository;
|
|
287
291
|
flagPushMarkerService: IMessageFlagPushRepository;
|
|
288
292
|
messageFlagService: IMessageFlagRepository;
|
|
289
|
-
unitOfWork?: IUnitOfWork;
|
|
290
293
|
}
|
|
291
294
|
|
|
292
295
|
// Bounds concurrent SQS sends while re-arming stuck flag-push markers —
|
|
@@ -364,7 +367,7 @@ export const drainPendingFlagPushes = async (
|
|
|
364
367
|
const syncMailboxMessages = async (
|
|
365
368
|
event: SyncMessagesEvent,
|
|
366
369
|
account: AccountItem,
|
|
367
|
-
|
|
370
|
+
connectionFactory: ManagedConnectionFactory,
|
|
368
371
|
deps: SyncDeps,
|
|
369
372
|
log: Logger,
|
|
370
373
|
): Promise<void> => {
|
|
@@ -377,20 +380,11 @@ const syncMailboxMessages = async (
|
|
|
377
380
|
threadMessageService,
|
|
378
381
|
mailboxSpecialUseService,
|
|
379
382
|
quarantineService,
|
|
383
|
+
syncWarningRepository,
|
|
380
384
|
flagPushMarkerService,
|
|
381
385
|
messageFlagService,
|
|
382
|
-
unitOfWork,
|
|
383
386
|
} = deps;
|
|
384
387
|
|
|
385
|
-
// Create a managed connection factory that caches and reuses the connection
|
|
386
|
-
const connectionFactory = createManagedConnectionFactory({
|
|
387
|
-
user: account.username,
|
|
388
|
-
credentials,
|
|
389
|
-
host: account.imapHost,
|
|
390
|
-
port: account.imapPort,
|
|
391
|
-
tls: account.imapTls,
|
|
392
|
-
});
|
|
393
|
-
|
|
394
388
|
// Get the mailbox - it must exist (should have been created by mailbox sync)
|
|
395
389
|
const mailbox = await mailboxService.get(account.accountId, event.mailboxId);
|
|
396
390
|
const mailboxId = mailbox.mailboxId;
|
|
@@ -409,8 +403,8 @@ const syncMailboxMessages = async (
|
|
|
409
403
|
envelopeService,
|
|
410
404
|
addressService,
|
|
411
405
|
threadMessageService,
|
|
406
|
+
new SyncWarningService(syncWarningRepository),
|
|
412
407
|
log,
|
|
413
|
-
unitOfWork,
|
|
414
408
|
flagPushMarkerService,
|
|
415
409
|
messageFlagService,
|
|
416
410
|
new QuarantineService(
|
package/src/index.ts
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
|
+
import { createLogger } from "@remit/logger";
|
|
1
2
|
import {
|
|
2
|
-
createLogger,
|
|
3
3
|
queueNameFromEventSource,
|
|
4
4
|
recordImapFailure,
|
|
5
5
|
recordQueueEvent,
|
|
6
|
-
|
|
7
|
-
} from "@remit/logger-lambda";
|
|
6
|
+
} from "@remit/logger-lambda/metrics";
|
|
7
|
+
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
8
8
|
import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
|
|
9
9
|
import type { WorkerEvent } from "./events.js";
|
|
10
10
|
import { imapFailureKind } from "./failure-kind.js";
|
|
11
11
|
import { processEvent } from "./processor.js";
|
|
12
12
|
|
|
13
|
+
installNodeLogger();
|
|
14
|
+
|
|
13
15
|
const log = createLogger();
|
|
14
16
|
|
|
15
17
|
/**
|
|
@@ -19,14 +19,12 @@ import {
|
|
|
19
19
|
DrizzleMessageRepository,
|
|
20
20
|
MailboxLockRepo,
|
|
21
21
|
MailboxRepo,
|
|
22
|
+
SearchIndex,
|
|
22
23
|
} from "@remit/drizzle-service";
|
|
23
24
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
24
|
-
import type { Logger } from "@remit/logger
|
|
25
|
+
import type { Logger } from "@remit/logger";
|
|
25
26
|
import { syncMailboxes } from "./handlers/sync-mailboxes.js";
|
|
26
|
-
import {
|
|
27
|
-
type SyncMessagesDeps,
|
|
28
|
-
syncMessages,
|
|
29
|
-
} from "./handlers/sync-messages.js";
|
|
27
|
+
import { syncMessages } from "./handlers/sync-messages.js";
|
|
30
28
|
import { runSchedulerTick } from "./scheduler/run-tick.js";
|
|
31
29
|
|
|
32
30
|
const NOW = 1_700_000_000_000;
|
|
@@ -55,9 +53,12 @@ let close: () => void;
|
|
|
55
53
|
beforeEach(() => {
|
|
56
54
|
const store = createShippedSqliteDb();
|
|
57
55
|
close = store.close;
|
|
58
|
-
accounts = new AccountRepo(store.db);
|
|
59
|
-
mailboxes = new MailboxRepo(store.db);
|
|
60
|
-
messages = new DrizzleMessageRepository(
|
|
56
|
+
accounts = new AccountRepo(store.db, store.db);
|
|
57
|
+
mailboxes = new MailboxRepo(store.db, new SearchIndex(store.db, store.db));
|
|
58
|
+
messages = new DrizzleMessageRepository(
|
|
59
|
+
store.db as never,
|
|
60
|
+
new SearchIndex(store.db as never, store.db as never),
|
|
61
|
+
);
|
|
61
62
|
locks = new MailboxLockRepo(store.db);
|
|
62
63
|
client = {
|
|
63
64
|
account: accounts,
|
|
@@ -135,6 +136,7 @@ const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
|
|
|
135
136
|
},
|
|
136
137
|
} as unknown as SQSClient;
|
|
137
138
|
const result = await runSchedulerTick({
|
|
139
|
+
redriveStrandedPlacements: async () => 0,
|
|
138
140
|
accountService: accounts,
|
|
139
141
|
sqsClient,
|
|
140
142
|
queueUrl: "https://queue.test/mailboxes",
|
|
@@ -176,14 +178,7 @@ describe("turning mail sync off and on again", () => {
|
|
|
176
178
|
silentLogger,
|
|
177
179
|
);
|
|
178
180
|
|
|
179
|
-
let
|
|
180
|
-
const deps = {
|
|
181
|
-
getClient: async () => client,
|
|
182
|
-
buildLifecycleDeps: () => ({}),
|
|
183
|
-
withOAuthLifecycle: async () => {
|
|
184
|
-
lifecycleCalls += 1;
|
|
185
|
-
},
|
|
186
|
-
} as unknown as SyncMessagesDeps;
|
|
181
|
+
let connections = 0;
|
|
187
182
|
await syncMessages(
|
|
188
183
|
{
|
|
189
184
|
type: "SYNC_MESSAGES",
|
|
@@ -193,9 +188,12 @@ describe("turning mail sync off and on again", () => {
|
|
|
193
188
|
mailboxId,
|
|
194
189
|
},
|
|
195
190
|
silentLogger,
|
|
196
|
-
|
|
191
|
+
() => {
|
|
192
|
+
connections += 1;
|
|
193
|
+
throw new Error("a sync that is off must not connect");
|
|
194
|
+
},
|
|
197
195
|
);
|
|
198
|
-
assert.equal(
|
|
196
|
+
assert.equal(connections, 0);
|
|
199
197
|
|
|
200
198
|
assert.deepEqual(
|
|
201
199
|
await mailboxes.listAllByAccount(account.accountId),
|
package/src/poller.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
import { createLogger
|
|
1
|
+
import { createLogger } from "@remit/logger";
|
|
2
|
+
import { startMetricsServer } from "@remit/logger-lambda/metrics";
|
|
3
|
+
import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
2
4
|
import { runQueuePoller } from "@remit/sqs-client/poller";
|
|
3
5
|
import { env } from "expect-env";
|
|
4
6
|
import { handler } from "./index.js";
|
|
5
7
|
|
|
8
|
+
installNodeLogger();
|
|
9
|
+
|
|
6
10
|
/**
|
|
7
11
|
* Production queue poller — the deployed form of `e2e-processor-shim.ts`.
|
|
8
12
|
* Polls every imap-worker queue and invokes the production Lambda handler.
|
|
9
|
-
* The search-index queue is NOT polled here: it is its own image/deployment
|
|
10
|
-
* (`remit-search-index-worker`), unlike the e2e shim which piggybacks it
|
|
11
|
-
* onto this same process for test convenience.
|
|
12
13
|
*/
|
|
13
14
|
const log = createLogger();
|
|
14
15
|
|
package/src/processor.test.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
|
-
import type { Logger } from "@remit/logger
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
4
|
import type { ImapWorkerStopEvent } from "./events.js";
|
|
5
5
|
import { processEvent } from "./processor.js";
|
|
6
6
|
|
package/src/processor.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Logger } from "@remit/logger
|
|
1
|
+
import type { Logger } from "@remit/logger";
|
|
2
2
|
import type { WorkerEvent } from "./events.js";
|
|
3
3
|
import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
|
|
4
4
|
import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { after, before, describe, it } from "node:test";
|
|
4
|
+
import { openStore, type World } from "../test-helpers/test-world.js";
|
|
5
|
+
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
6
|
+
|
|
7
|
+
let store: Awaited<ReturnType<typeof openStore>>;
|
|
8
|
+
|
|
9
|
+
before(async () => {
|
|
10
|
+
store = await openStore();
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
after(() => store.close());
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A draft holding one uploaded file, and one object under the same draft that
|
|
17
|
+
* no attachment row names — what a browser leaves behind when it uploads and
|
|
18
|
+
* the reservation lapses before it is completed.
|
|
19
|
+
*/
|
|
20
|
+
const arrange = async (w: World) => {
|
|
21
|
+
const { outbox, attachmentId } = await w.draft(Buffer.from("q1,q2\n1,2\n"));
|
|
22
|
+
await w.client.storage.storeOutboxAttachment({
|
|
23
|
+
accountConfigId: w.account.accountConfigId,
|
|
24
|
+
accountId: w.account.accountId,
|
|
25
|
+
outboxMessageId: outbox.outboxMessageId,
|
|
26
|
+
outboxAttachmentId: `orphan-${randomUUID()}`,
|
|
27
|
+
content: Buffer.from("left behind"),
|
|
28
|
+
});
|
|
29
|
+
return { outboxMessageId: outbox.outboxMessageId, attachmentId };
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const storedIds = async (w: World, outboxMessageId: string) =>
|
|
33
|
+
(
|
|
34
|
+
await w.client.storage.listOutboxAttachments(
|
|
35
|
+
w.account.accountConfigId,
|
|
36
|
+
w.account.accountId,
|
|
37
|
+
outboxMessageId,
|
|
38
|
+
)
|
|
39
|
+
).map((item) => item.outboxAttachmentId);
|
|
40
|
+
|
|
41
|
+
describe("the outbox attachment sweep", () => {
|
|
42
|
+
it("removes the objects no attachment row names and keeps the ones that are", async () => {
|
|
43
|
+
const w = await store.world();
|
|
44
|
+
const { outboxMessageId, attachmentId } = await arrange(w);
|
|
45
|
+
|
|
46
|
+
await buildAttachmentSweep(w.client, w.log)(w.account);
|
|
47
|
+
|
|
48
|
+
assert.deepEqual(await storedIds(w, outboxMessageId), [attachmentId]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("runs for every account on the hosted tick, due a sync or not", async () => {
|
|
52
|
+
const first = await store.world();
|
|
53
|
+
const firstDraft = await arrange(first);
|
|
54
|
+
await first.client.account.update(first.account.accountId, {
|
|
55
|
+
lastSyncAt: Date.now(),
|
|
56
|
+
});
|
|
57
|
+
const second = await store.world();
|
|
58
|
+
const secondDraft = await arrange(second);
|
|
59
|
+
|
|
60
|
+
const { handler } = await import("./handler.js");
|
|
61
|
+
await handler(
|
|
62
|
+
{ time: new Date().toISOString() } as never,
|
|
63
|
+
{} as never,
|
|
64
|
+
() => {},
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
assert.deepEqual(await storedIds(first, firstDraft.outboxMessageId), [
|
|
68
|
+
firstDraft.attachmentId,
|
|
69
|
+
]);
|
|
70
|
+
assert.deepEqual(await storedIds(second, secondDraft.outboxMessageId), [
|
|
71
|
+
secondDraft.attachmentId,
|
|
72
|
+
]);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { RemitClient } from "@remit/backend/client";
|
|
2
2
|
import type { AccountItem } from "@remit/data-ports";
|
|
3
|
-
import type { Logger } from "@remit/logger
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
4
|
import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
|
|
5
5
|
/**
|
|
6
6
|
* Collect attachment bytes the database does not know about. Both scheduler
|