@remit/imap-worker 0.0.86 → 0.0.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +3 -1
  2. package/package.json +10 -2
  3. package/src/account-check.ts +1 -1
  4. package/src/body-sync-gate.test.ts +1 -1
  5. package/src/body-sync-gate.ts +1 -1
  6. package/src/e2e-processor-shim.ts +6 -21
  7. package/src/emit.ts +1 -1
  8. package/src/failure-kind.ts +1 -1
  9. package/src/handlers/append-sent-message.sqlite.test.ts +176 -0
  10. package/src/handlers/append-sent-message.ts +3 -24
  11. package/src/handlers/delete-account-objects.test.ts +1 -1
  12. package/src/handlers/delete-account-objects.ts +1 -1
  13. package/src/handlers/empty-trash.sqlite.test.ts +209 -0
  14. package/src/handlers/empty-trash.ts +3 -24
  15. package/src/handlers/flag-push.ts +11 -18
  16. package/src/handlers/mailbox-management.sqlite.test.ts +265 -0
  17. package/src/handlers/mailbox-management.ts +12 -48
  18. package/src/handlers/message-copy.sqlite.test.ts +298 -0
  19. package/src/handlers/message-copy.ts +31 -49
  20. package/src/handlers/message-delete-terminal.ts +12 -10
  21. package/src/handlers/message-delete.sqlite.test.ts +291 -0
  22. package/src/handlers/message-delete.ts +26 -32
  23. package/src/handlers/message-move-terminal.ts +12 -10
  24. package/src/handlers/message-move.sqlite.test.ts +271 -0
  25. package/src/handlers/message-move.test.ts +76 -863
  26. package/src/handlers/message-move.ts +49 -39
  27. package/src/handlers/placement-move-push.test.ts +96 -329
  28. package/src/handlers/placement-move-push.ts +75 -29
  29. package/src/handlers/placement-push.sqlite.test.ts +298 -0
  30. package/src/handlers/probe-message-id.sqlite.test.ts +197 -0
  31. package/src/handlers/probe-message-id.ts +15 -0
  32. package/src/handlers/sync-mailboxes.test.ts +0 -1
  33. package/src/handlers/sync-mailboxes.ts +1 -1
  34. package/src/handlers/sync-message-body.sqlite.test.ts +314 -0
  35. package/src/handlers/sync-message-body.test.ts +3 -299
  36. package/src/handlers/sync-message-body.ts +21 -11
  37. package/src/handlers/sync-messages.sqlite.test.ts +183 -0
  38. package/src/handlers/sync-messages.ts +27 -33
  39. package/src/index.ts +5 -3
  40. package/src/mail-sync-selection.sqlite.test.ts +16 -18
  41. package/src/poller.ts +5 -4
  42. package/src/processor.test.ts +1 -1
  43. package/src/processor.ts +1 -1
  44. package/src/scheduler/attachment-sweep.sqlite.test.ts +74 -0
  45. package/src/scheduler/attachment-sweep.ts +1 -1
  46. package/src/scheduler/{calendar-subscriptions.test.ts → calendar-subscriptions.sqlite.test.ts} +43 -55
  47. package/src/scheduler/calendar-subscriptions.ts +6 -6
  48. package/src/scheduler/half-deleted-accounts.sqlite.test.ts +203 -0
  49. package/src/scheduler/handler.ts +12 -1
  50. package/src/scheduler/loop.test.ts +2 -0
  51. package/src/scheduler/run-tick.sqlite.test.ts +192 -0
  52. package/src/scheduler/run-tick.ts +39 -1
  53. package/src/scheduler/runner.ts +21 -1
  54. package/src/scheduler/stranded-placement-redrive.ts +265 -0
  55. package/src/stranded-placement.sqlite.test.ts +795 -0
  56. package/src/test-helpers/test-world.ts +398 -0
  57. package/src/thread-join.sqlite.test.ts +115 -0
  58. package/src/with-oauth-lifecycle-deps.sqlite.test.ts +138 -14
  59. package/src/with-oauth-lifecycle.ts +1 -1
  60. package/src/handlers/append-sent-message.test.ts +0 -529
  61. package/src/handlers/empty-trash.test.ts +0 -728
  62. package/src/handlers/flag-push-placement-race.test.ts +0 -385
  63. package/src/handlers/flag-push.test.ts +0 -325
  64. package/src/handlers/mailbox-management.test.ts +0 -850
  65. package/src/handlers/message-copy.test.ts +0 -654
  66. package/src/handlers/message-delete.test.ts +0 -1869
  67. package/src/handlers/message-move-terminal.test.ts +0 -412
  68. package/src/handlers/sync-messages.test.ts +0 -569
  69. package/src/scheduler/attachment-sweep.test.ts +0 -169
  70. package/src/scheduler/run-tick.test.ts +0 -478
  71. package/src/with-oauth-lifecycle.test.ts +0 -318
@@ -1,7 +1,8 @@
1
1
  import { getClient } from "@remit/backend/client";
2
2
  import { isNotFoundError } from "@remit/data-ports/errors";
3
- import type { Logger } from "@remit/logger-lambda";
4
- import { recordImapFailure } from "@remit/logger-lambda";
3
+ import { SYNC_WARNING_SURFACE_ATTEMPTS } from "@remit/data-ports/sync-warning-state";
4
+ import type { Logger } from "@remit/logger";
5
+ import { recordImapFailure } from "@remit/logger-lambda/metrics";
5
6
  import {
6
7
  BodySyncService,
7
8
  guardConnectionCursor,
@@ -12,6 +13,7 @@ import {
12
13
  QuarantineService,
13
14
  resolveExhaustedBodySyncFailures,
14
15
  resolveFolderPlacement,
16
+ SyncWarningService,
15
17
  } from "@remit/mailbox-service";
16
18
  import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
17
19
  import { attemptBudget } from "@remit/sqs-client/attempt-budget";
@@ -31,7 +33,12 @@ const bodySyncEnabledParameterName = env.BODY_SYNC_ENABLED_PARAMETER_NAME;
31
33
 
32
34
  export const getBodySyncMaxAttempts = (
33
35
  processEnv: NodeJS.ProcessEnv = process.env,
34
- ): number => attemptBudget("BODY_SYNC_MAX_ATTEMPTS", 3, processEnv);
36
+ ): number =>
37
+ attemptBudget(
38
+ "BODY_SYNC_MAX_ATTEMPTS",
39
+ SYNC_WARNING_SURFACE_ATTEMPTS,
40
+ processEnv,
41
+ );
35
42
 
36
43
  export const BODY_SYNC_MAX_ATTEMPTS = getBodySyncMaxAttempts();
37
44
 
@@ -81,6 +88,7 @@ export const syncMessageBody = async (
81
88
  event: SyncMessageBodyEvent,
82
89
  log: Logger,
83
90
  receiveCount = 1,
91
+ createConnectionScope = createConnectionScopeWithCredentials,
84
92
  ): Promise<void> => {
85
93
  const { accountId, mailboxId } = event;
86
94
 
@@ -125,15 +133,15 @@ export const syncMessageBody = async (
125
133
  envelope: envelopeService,
126
134
  placementMove: markerService,
127
135
  filter: filterService,
128
- filterAnchor: filterAnchorService,
129
136
  messageLabel: messageLabelService,
130
137
  storage,
131
138
  secrets,
132
139
  mailboxSpecialUse: mailboxSpecialUseRepository,
133
140
  quarantine: quarantineRepository,
141
+ syncWarning: syncWarningRepository,
134
142
  flagQueue: flagQueueService,
135
143
  calendarSuggestion: calendarSuggestionService,
136
- calendarUnitOfWork,
144
+ calendarStore,
137
145
  } = await getClient();
138
146
 
139
147
  const account = await accountService.get(accountId);
@@ -156,6 +164,7 @@ export const syncMessageBody = async (
156
164
  markerService,
157
165
  addressService,
158
166
  mailboxSpecialUseService,
167
+ mailboxService,
159
168
  sqsQueueUrl: placementMoveQueueUrl,
160
169
  })
161
170
  : undefined;
@@ -205,7 +214,7 @@ export const syncMessageBody = async (
205
214
  // warm container skips TCP+TLS+LOGIN+SELECT; a dead pooled connection is
206
215
  // liveness-checked and replaced inside borrowWarmConnection.
207
216
  const borrowed = borrowWarmConnection(accountId, () =>
208
- createConnectionScopeWithCredentials(account, credentials),
217
+ createConnectionScope(account, credentials),
209
218
  );
210
219
 
211
220
  // A confident, actionable placement verdict moves mail directly on
@@ -220,12 +229,11 @@ export const syncMessageBody = async (
220
229
 
221
230
  // A matched filter's actions (label upsert, exclusive move) apply on
222
231
  // the same body-sync pass, reusing the placement mover for the move
223
- // (RFC 034 Decision 3.1). The env-provisioned embedder lets a semantic
224
- // (anchor-only) filter match here instead of being silently skipped.
232
+ // (RFC 034 Decision 3.1).
225
233
  const filterConfig = buildFilterConfig({
226
234
  filterService,
227
- filterAnchorService,
228
235
  messageLabelService,
236
+ mailboxService,
229
237
  placementMoveService,
230
238
  });
231
239
 
@@ -235,6 +243,7 @@ export const syncMessageBody = async (
235
243
  threadMessageService,
236
244
  addressService,
237
245
  envelopeService,
246
+ new SyncWarningService(syncWarningRepository),
238
247
  log,
239
248
  placementConfig,
240
249
  filterConfig,
@@ -252,7 +261,7 @@ export const syncMessageBody = async (
252
261
  { flagQueueService },
253
262
  // An invitation becomes a card the first time the message is seen,
254
263
  // alongside the filters that run on the same pass (issue #1033).
255
- { calendarSuggestionService, calendarUnitOfWork, filterService },
264
+ { calendarSuggestionService, calendarStore, filterService },
256
265
  );
257
266
 
258
267
  // Guard at the openBox choke point (epic #1281 invariants 3 & 5). The
@@ -274,7 +283,9 @@ export const syncMessageBody = async (
274
283
  // against the SAME borrowed connection, so the mailbox stays open
275
284
  // across the two — the connection is only released once both are done.
276
285
  const folder = {
286
+ mailboxId: mailbox.mailboxId,
277
287
  fullPath: mailbox.fullPath,
288
+ uidValidity: mailbox.uidValidity,
278
289
  placement: await resolveFolderPlacement(
279
290
  mailboxSpecialUseService,
280
291
  accountId,
@@ -315,7 +326,6 @@ export const syncMessageBody = async (
315
326
  const { brokenMessageIds } = await resolveExhaustedBodySyncFailures(
316
327
  {
317
328
  messageService,
318
- threadMessageService,
319
329
  storageService: storage,
320
330
  log,
321
331
  },
@@ -0,0 +1,183 @@
1
+ import assert from "node:assert/strict";
2
+ import { after, before, describe, it } from "node:test";
3
+ import type { SyncMessagesEvent } from "../events.js";
4
+ import { openStore, type World } from "../test-helpers/test-world.js";
5
+ import { drainPendingFlagPushes, syncMessages } from "./sync-messages.js";
6
+
7
+ let store: Awaited<ReturnType<typeof openStore>>;
8
+
9
+ before(async () => {
10
+ store = await openStore();
11
+ });
12
+
13
+ after(() => store.close());
14
+
15
+ const syncEvent = (w: World, mailboxId: string): SyncMessagesEvent => ({
16
+ type: "SYNC_MESSAGES",
17
+ accountId: w.account.accountId,
18
+ mailboxId,
19
+ eventId: `evt-${mailboxId}`,
20
+ timestamp: Date.now(),
21
+ });
22
+
23
+ const sync = async (w: World, mailboxId: string): Promise<number> => {
24
+ await syncMessages(syncEvent(w, mailboxId), w.log, w.connectionFactory);
25
+ return w.imap.connections;
26
+ };
27
+
28
+ describe("which SYNC_MESSAGES events reach the server", () => {
29
+ it("a live mailbox does", async () => {
30
+ const w = await store.world();
31
+ const inbox = await w.folder("INBOX");
32
+
33
+ assert.equal(await sync(w, inbox.mailboxId), 1);
34
+ });
35
+
36
+ // Issue #287: a NotFoundError at the head of the account's FIFO group
37
+ // stalled every later event for the account.
38
+ it("a mailbox whose row is gone is acked without connecting", async () => {
39
+ const w = await store.world();
40
+ const old = await w.folder("Old");
41
+ await w.client.mailbox.delete(w.account.accountId, old.mailboxId);
42
+
43
+ assert.equal(await sync(w, old.mailboxId), 0);
44
+ });
45
+
46
+ it("a mailbox being deleted is acked without connecting", async () => {
47
+ const w = await store.world();
48
+ const old = await w.folder("Old");
49
+ await w.client.mailboxQueue.deleteMailbox(
50
+ old.mailboxId,
51
+ w.account.accountId,
52
+ );
53
+
54
+ assert.equal(await sync(w, old.mailboxId), 0);
55
+ });
56
+
57
+ it("a mailbox not yet created on the server is acked without connecting", async () => {
58
+ const w = await store.world();
59
+ const created = await w.client.mailboxQueue.createMailbox(
60
+ {
61
+ accountId: w.account.accountId,
62
+ namespacePrefix: "",
63
+ hierarchyDelimiter: "/",
64
+ fullPath: "Receipts",
65
+ uidValidity: 0,
66
+ uidNext: 0,
67
+ highestModseq: "0",
68
+ messageCount: 0,
69
+ unseenCount: 0,
70
+ deletedCount: 0,
71
+ totalSize: 0,
72
+ lastSyncUid: 0,
73
+ highWaterMarkUid: 0,
74
+ lastMessageSyncAt: 0,
75
+ },
76
+ w.account.accountId,
77
+ );
78
+ if (created.outcome !== "Created") throw new Error("path taken");
79
+
80
+ assert.equal(await sync(w, created.mailbox.mailboxId), 0);
81
+ });
82
+ });
83
+
84
+ // Issue #339: a delete asked for while the round is in flight lands inside
85
+ // the guard, and the round then fails deeper on a folder that is gone.
86
+ describe("a folder deleted while its sync is in flight", () => {
87
+ it("resolves the event and records no error phase on the account", async () => {
88
+ const w = await store.world();
89
+ const old = await w.folder("Old");
90
+ const connectionFactory = () => {
91
+ const factory = w.connectionFactory();
92
+ const connection = factory.getConnection();
93
+ const openBox = connection.openBox;
94
+ let first = true;
95
+ connection.openBox = async (path, readOnly) => {
96
+ if (first) {
97
+ first = false;
98
+ await w.client.mailboxQueue.deleteMailbox(
99
+ old.mailboxId,
100
+ w.account.accountId,
101
+ );
102
+ w.imap.removeFolder("Old");
103
+ }
104
+ return openBox(path, readOnly);
105
+ };
106
+ return { getConnection: () => connection, close: factory.close };
107
+ };
108
+
109
+ await syncMessages(syncEvent(w, old.mailboxId), w.log, connectionFactory);
110
+
111
+ const account = await w.client.account.get(w.account.accountId);
112
+ assert.notEqual(account.syncPhase, "error");
113
+ });
114
+ });
115
+
116
+ describe("drainPendingFlagPushes (issue #1273)", () => {
117
+ const marker = (w: World, mailboxId: string, messageId: string) =>
118
+ w.client.flagPush.put({
119
+ messageId,
120
+ flagName: "\\Seen",
121
+ accountId: w.account.accountId,
122
+ accountConfigId: w.account.accountConfigId,
123
+ mailboxId,
124
+ operation: "add",
125
+ });
126
+
127
+ it("re-arms the markers stuck before their wake-up and leaves the ones a driver owns", async () => {
128
+ const w = await store.world();
129
+ const inbox = await w.folder("INBOX");
130
+ await marker(w, inbox.mailboxId, "msg-pending");
131
+ await marker(w, inbox.mailboxId, "msg-queued");
132
+ await w.client.flagPush.updateState("msg-queued", "\\Seen", "queued");
133
+ await marker(w, inbox.mailboxId, "msg-processing");
134
+ await w.client.flagPush.updateState(
135
+ "msg-processing",
136
+ "\\Seen",
137
+ "processing",
138
+ );
139
+
140
+ await drainPendingFlagPushes(
141
+ w.client.flagPush,
142
+ w.account,
143
+ inbox.mailboxId,
144
+ w.log,
145
+ );
146
+
147
+ assert.deepEqual(
148
+ w
149
+ .queued()
150
+ .filter((body) => body.type === "FLAG_PUSH")
151
+ .map((body) => body.messageId),
152
+ ["msg-pending"],
153
+ );
154
+ });
155
+
156
+ it("keeps re-arming the others when one re-arm fails, and says so loudly", async () => {
157
+ const w = await store.world();
158
+ const inbox = await w.folder("INBOX");
159
+ await marker(w, inbox.mailboxId, "will-fail");
160
+ await marker(w, inbox.mailboxId, "will-succeed");
161
+ const armed: string[] = [];
162
+
163
+ await drainPendingFlagPushes(
164
+ w.client.flagPush,
165
+ w.account,
166
+ inbox.mailboxId,
167
+ w.log,
168
+ async (event) => {
169
+ const { messageId } = event as unknown as { messageId: string };
170
+ if (messageId === "will-fail") throw new Error("queue down");
171
+ armed.push(messageId);
172
+ },
173
+ );
174
+
175
+ assert.deepEqual(armed, ["will-succeed"]);
176
+ assert.equal(
177
+ w.logs.filter(
178
+ (line) => line.fields.alert === "flag_push_drain_rearm_failed",
179
+ ).length,
180
+ 1,
181
+ );
182
+ });
183
+ });
@@ -10,21 +10,24 @@ import type {
10
10
  IMessageFlagRepository,
11
11
  IMessageRepository,
12
12
  IQuarantineRepository,
13
+ ISyncWarningRepository,
13
14
  IThreadMessageRepository,
14
- IUnitOfWork,
15
15
  } from "@remit/data-ports";
16
16
  import { isNotFoundError } from "@remit/data-ports/errors";
17
17
  import { SyncPhase } from "@remit/domain-enums";
18
- import { type Logger, recordImapFailure } from "@remit/logger-lambda";
18
+ import type { Logger } from "@remit/logger";
19
+ import { recordImapFailure } from "@remit/logger-lambda/metrics";
19
20
  import { RefreshTokenError } from "@remit/mail-oauth-service";
20
21
  import {
21
22
  createManagedConnectionFactory,
22
23
  isMailboxMutationInFlight,
23
24
  MailConnectionError,
24
25
  type MailCredentials,
26
+ type ManagedConnectionFactory,
25
27
  MessageSyncService,
26
28
  QuarantineService,
27
29
  type SyncedMessage,
30
+ SyncWarningService,
28
31
  } from "@remit/mailbox-service";
29
32
  import pMap from "p-map";
30
33
  import {
@@ -59,22 +62,22 @@ export const batchSyncedMessages = (
59
62
  return batches;
60
63
  };
61
64
 
62
- export interface SyncMessagesDeps {
63
- getClient: typeof getClient;
64
- withOAuthLifecycle: typeof withOAuthLifecycle;
65
- buildLifecycleDeps: typeof buildLifecycleDeps;
66
- }
67
-
68
- const defaultDeps: SyncMessagesDeps = {
69
- getClient,
70
- withOAuthLifecycle,
71
- buildLifecycleDeps,
72
- };
65
+ export const managedConnectionFor = (
66
+ account: Pick<AccountItem, "username" | "imapHost" | "imapPort" | "imapTls">,
67
+ credentials: MailCredentials,
68
+ ): ManagedConnectionFactory =>
69
+ createManagedConnectionFactory({
70
+ user: account.username,
71
+ credentials,
72
+ host: account.imapHost,
73
+ port: account.imapPort,
74
+ tls: account.imapTls,
75
+ });
73
76
 
74
77
  export const syncMessages = async (
75
78
  event: SyncMessagesEvent,
76
79
  log: Logger,
77
- deps: SyncMessagesDeps = defaultDeps,
80
+ createConnectionFactory = managedConnectionFor,
78
81
  ): Promise<void> => {
79
82
  log.info(
80
83
  {
@@ -95,11 +98,11 @@ export const syncMessages = async (
95
98
  mailboxLock: mailboxLockService,
96
99
  mailboxSpecialUse: mailboxSpecialUseService,
97
100
  quarantine: quarantineService,
101
+ syncWarning: syncWarningRepository,
98
102
  flagPush: flagPushMarkerService,
99
103
  messageFlag: messageFlagService,
100
- unitOfWork,
101
104
  secrets,
102
- } = await deps.getClient();
105
+ } = await getClient();
103
106
 
104
107
  await mailboxLockService.releaseLock(
105
108
  event.accountId,
@@ -181,8 +184,8 @@ export const syncMessages = async (
181
184
  // withOAuthLifecycle owns the reauth/ACK contract (skip-if-reauth, resolve
182
185
  // credentials, flip on terminal auth failure, rethrow transient). The
183
186
  // mailbox lock and the actual sync run inside the wrapper callback.
184
- await deps.withOAuthLifecycle(
185
- deps.buildLifecycleDeps(secrets, accountService),
187
+ await withOAuthLifecycle(
188
+ buildLifecycleDeps(secrets, accountService),
186
189
  account,
187
190
  log,
188
191
  async (credentials) => {
@@ -196,7 +199,7 @@ export const syncMessages = async (
196
199
  await syncMailboxMessages(
197
200
  event,
198
201
  account,
199
- credentials,
202
+ createConnectionFactory(account, credentials),
200
203
  {
201
204
  accountService,
202
205
  mailboxService,
@@ -206,9 +209,9 @@ export const syncMessages = async (
206
209
  threadMessageService,
207
210
  mailboxSpecialUseService,
208
211
  quarantineService,
212
+ syncWarningRepository,
209
213
  flagPushMarkerService,
210
214
  messageFlagService,
211
- unitOfWork,
212
215
  },
213
216
  log,
214
217
  );
@@ -284,9 +287,9 @@ interface SyncDeps {
284
287
  threadMessageService: IThreadMessageRepository;
285
288
  mailboxSpecialUseService: IMailboxSpecialUseRepository;
286
289
  quarantineService: IQuarantineRepository;
290
+ syncWarningRepository: ISyncWarningRepository;
287
291
  flagPushMarkerService: IMessageFlagPushRepository;
288
292
  messageFlagService: IMessageFlagRepository;
289
- unitOfWork?: IUnitOfWork;
290
293
  }
291
294
 
292
295
  // Bounds concurrent SQS sends while re-arming stuck flag-push markers —
@@ -364,7 +367,7 @@ export const drainPendingFlagPushes = async (
364
367
  const syncMailboxMessages = async (
365
368
  event: SyncMessagesEvent,
366
369
  account: AccountItem,
367
- credentials: MailCredentials,
370
+ connectionFactory: ManagedConnectionFactory,
368
371
  deps: SyncDeps,
369
372
  log: Logger,
370
373
  ): Promise<void> => {
@@ -377,20 +380,11 @@ const syncMailboxMessages = async (
377
380
  threadMessageService,
378
381
  mailboxSpecialUseService,
379
382
  quarantineService,
383
+ syncWarningRepository,
380
384
  flagPushMarkerService,
381
385
  messageFlagService,
382
- unitOfWork,
383
386
  } = deps;
384
387
 
385
- // Create a managed connection factory that caches and reuses the connection
386
- const connectionFactory = createManagedConnectionFactory({
387
- user: account.username,
388
- credentials,
389
- host: account.imapHost,
390
- port: account.imapPort,
391
- tls: account.imapTls,
392
- });
393
-
394
388
  // Get the mailbox - it must exist (should have been created by mailbox sync)
395
389
  const mailbox = await mailboxService.get(account.accountId, event.mailboxId);
396
390
  const mailboxId = mailbox.mailboxId;
@@ -409,8 +403,8 @@ const syncMailboxMessages = async (
409
403
  envelopeService,
410
404
  addressService,
411
405
  threadMessageService,
406
+ new SyncWarningService(syncWarningRepository),
412
407
  log,
413
- unitOfWork,
414
408
  flagPushMarkerService,
415
409
  messageFlagService,
416
410
  new QuarantineService(
package/src/index.ts CHANGED
@@ -1,15 +1,17 @@
1
+ import { createLogger } from "@remit/logger";
1
2
  import {
2
- createLogger,
3
3
  queueNameFromEventSource,
4
4
  recordImapFailure,
5
5
  recordQueueEvent,
6
- withTelemetry,
7
- } from "@remit/logger-lambda";
6
+ } from "@remit/logger-lambda/metrics";
7
+ import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
8
8
  import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
9
9
  import type { WorkerEvent } from "./events.js";
10
10
  import { imapFailureKind } from "./failure-kind.js";
11
11
  import { processEvent } from "./processor.js";
12
12
 
13
+ installNodeLogger();
14
+
13
15
  const log = createLogger();
14
16
 
15
17
  /**
@@ -19,14 +19,12 @@ import {
19
19
  DrizzleMessageRepository,
20
20
  MailboxLockRepo,
21
21
  MailboxRepo,
22
+ SearchIndex,
22
23
  } from "@remit/drizzle-service";
23
24
  import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
24
- import type { Logger } from "@remit/logger-lambda";
25
+ import type { Logger } from "@remit/logger";
25
26
  import { syncMailboxes } from "./handlers/sync-mailboxes.js";
26
- import {
27
- type SyncMessagesDeps,
28
- syncMessages,
29
- } from "./handlers/sync-messages.js";
27
+ import { syncMessages } from "./handlers/sync-messages.js";
30
28
  import { runSchedulerTick } from "./scheduler/run-tick.js";
31
29
 
32
30
  const NOW = 1_700_000_000_000;
@@ -55,9 +53,12 @@ let close: () => void;
55
53
  beforeEach(() => {
56
54
  const store = createShippedSqliteDb();
57
55
  close = store.close;
58
- accounts = new AccountRepo(store.db);
59
- mailboxes = new MailboxRepo(store.db);
60
- messages = new DrizzleMessageRepository(store.db as never);
56
+ accounts = new AccountRepo(store.db, store.db);
57
+ mailboxes = new MailboxRepo(store.db, new SearchIndex(store.db, store.db));
58
+ messages = new DrizzleMessageRepository(
59
+ store.db as never,
60
+ new SearchIndex(store.db as never, store.db as never),
61
+ );
61
62
  locks = new MailboxLockRepo(store.db);
62
63
  client = {
63
64
  account: accounts,
@@ -135,6 +136,7 @@ const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
135
136
  },
136
137
  } as unknown as SQSClient;
137
138
  const result = await runSchedulerTick({
139
+ redriveStrandedPlacements: async () => 0,
138
140
  accountService: accounts,
139
141
  sqsClient,
140
142
  queueUrl: "https://queue.test/mailboxes",
@@ -176,14 +178,7 @@ describe("turning mail sync off and on again", () => {
176
178
  silentLogger,
177
179
  );
178
180
 
179
- let lifecycleCalls = 0;
180
- const deps = {
181
- getClient: async () => client,
182
- buildLifecycleDeps: () => ({}),
183
- withOAuthLifecycle: async () => {
184
- lifecycleCalls += 1;
185
- },
186
- } as unknown as SyncMessagesDeps;
181
+ let connections = 0;
187
182
  await syncMessages(
188
183
  {
189
184
  type: "SYNC_MESSAGES",
@@ -193,9 +188,12 @@ describe("turning mail sync off and on again", () => {
193
188
  mailboxId,
194
189
  },
195
190
  silentLogger,
196
- deps,
191
+ () => {
192
+ connections += 1;
193
+ throw new Error("a sync that is off must not connect");
194
+ },
197
195
  );
198
- assert.equal(lifecycleCalls, 0);
196
+ assert.equal(connections, 0);
199
197
 
200
198
  assert.deepEqual(
201
199
  await mailboxes.listAllByAccount(account.accountId),
package/src/poller.ts CHANGED
@@ -1,14 +1,15 @@
1
- import { createLogger, startMetricsServer } from "@remit/logger-lambda";
1
+ import { createLogger } from "@remit/logger";
2
+ import { startMetricsServer } from "@remit/logger-lambda/metrics";
3
+ import { installNodeLogger } from "@remit/logger-lambda/node";
2
4
  import { runQueuePoller } from "@remit/sqs-client/poller";
3
5
  import { env } from "expect-env";
4
6
  import { handler } from "./index.js";
5
7
 
8
+ installNodeLogger();
9
+
6
10
  /**
7
11
  * Production queue poller — the deployed form of `e2e-processor-shim.ts`.
8
12
  * Polls every imap-worker queue and invokes the production Lambda handler.
9
- * The search-index queue is NOT polled here: it is its own image/deployment
10
- * (`remit-search-index-worker`), unlike the e2e shim which piggybacks it
11
- * onto this same process for test convenience.
12
13
  */
13
14
  const log = createLogger();
14
15
 
@@ -1,6 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { describe, it } from "node:test";
3
- import type { Logger } from "@remit/logger-lambda";
3
+ import type { Logger } from "@remit/logger";
4
4
  import type { ImapWorkerStopEvent } from "./events.js";
5
5
  import { processEvent } from "./processor.js";
6
6
 
package/src/processor.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { Logger } from "@remit/logger-lambda";
1
+ import type { Logger } from "@remit/logger";
2
2
  import type { WorkerEvent } from "./events.js";
3
3
  import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
4
4
  import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
@@ -0,0 +1,74 @@
1
+ import assert from "node:assert/strict";
2
+ import { randomUUID } from "node:crypto";
3
+ import { after, before, describe, it } from "node:test";
4
+ import { openStore, type World } from "../test-helpers/test-world.js";
5
+ import { buildAttachmentSweep } from "./attachment-sweep.js";
6
+
7
+ let store: Awaited<ReturnType<typeof openStore>>;
8
+
9
+ before(async () => {
10
+ store = await openStore();
11
+ });
12
+
13
+ after(() => store.close());
14
+
15
+ /**
16
+ * A draft holding one uploaded file, and one object under the same draft that
17
+ * no attachment row names — what a browser leaves behind when it uploads and
18
+ * the reservation lapses before it is completed.
19
+ */
20
+ const arrange = async (w: World) => {
21
+ const { outbox, attachmentId } = await w.draft(Buffer.from("q1,q2\n1,2\n"));
22
+ await w.client.storage.storeOutboxAttachment({
23
+ accountConfigId: w.account.accountConfigId,
24
+ accountId: w.account.accountId,
25
+ outboxMessageId: outbox.outboxMessageId,
26
+ outboxAttachmentId: `orphan-${randomUUID()}`,
27
+ content: Buffer.from("left behind"),
28
+ });
29
+ return { outboxMessageId: outbox.outboxMessageId, attachmentId };
30
+ };
31
+
32
+ const storedIds = async (w: World, outboxMessageId: string) =>
33
+ (
34
+ await w.client.storage.listOutboxAttachments(
35
+ w.account.accountConfigId,
36
+ w.account.accountId,
37
+ outboxMessageId,
38
+ )
39
+ ).map((item) => item.outboxAttachmentId);
40
+
41
+ describe("the outbox attachment sweep", () => {
42
+ it("removes the objects no attachment row names and keeps the ones that are", async () => {
43
+ const w = await store.world();
44
+ const { outboxMessageId, attachmentId } = await arrange(w);
45
+
46
+ await buildAttachmentSweep(w.client, w.log)(w.account);
47
+
48
+ assert.deepEqual(await storedIds(w, outboxMessageId), [attachmentId]);
49
+ });
50
+
51
+ it("runs for every account on the hosted tick, due a sync or not", async () => {
52
+ const first = await store.world();
53
+ const firstDraft = await arrange(first);
54
+ await first.client.account.update(first.account.accountId, {
55
+ lastSyncAt: Date.now(),
56
+ });
57
+ const second = await store.world();
58
+ const secondDraft = await arrange(second);
59
+
60
+ const { handler } = await import("./handler.js");
61
+ await handler(
62
+ { time: new Date().toISOString() } as never,
63
+ {} as never,
64
+ () => {},
65
+ );
66
+
67
+ assert.deepEqual(await storedIds(first, firstDraft.outboxMessageId), [
68
+ firstDraft.attachmentId,
69
+ ]);
70
+ assert.deepEqual(await storedIds(second, secondDraft.outboxMessageId), [
71
+ secondDraft.attachmentId,
72
+ ]);
73
+ });
74
+ });
@@ -1,6 +1,6 @@
1
1
  import type { RemitClient } from "@remit/backend/client";
2
2
  import type { AccountItem } from "@remit/data-ports";
3
- import type { Logger } from "@remit/logger-lambda";
3
+ import type { Logger } from "@remit/logger";
4
4
  import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
5
5
  /**
6
6
  * Collect attachment bytes the database does not know about. Both scheduler