@remit/imap-worker 0.0.89 → 0.0.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/dist/account-check.d.ts +27 -0
  2. package/dist/account-check.js +70 -0
  3. package/dist/body-sync-gate.d.ts +12 -0
  4. package/dist/body-sync-gate.js +53 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +193 -0
  7. package/dist/config-env.d.ts +5 -0
  8. package/dist/config-env.js +37 -0
  9. package/dist/connection-pool.d.ts +22 -0
  10. package/dist/connection-pool.js +97 -0
  11. package/dist/connection-scope.d.ts +8 -0
  12. package/dist/connection-scope.js +10 -0
  13. package/dist/context.d.ts +46 -0
  14. package/dist/context.js +15 -0
  15. package/dist/e2e-processor-shim.d.ts +2 -0
  16. package/dist/e2e-processor-shim.js +181 -0
  17. package/dist/emit.d.ts +15 -0
  18. package/dist/emit.js +76 -0
  19. package/dist/events.d.ts +203 -0
  20. package/dist/events.js +1 -0
  21. package/dist/failure-kind.d.ts +10 -0
  22. package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
  23. package/dist/handlers/append-sent-message.d.ts +3 -0
  24. package/dist/handlers/append-sent-message.js +152 -0
  25. package/dist/handlers/delete-account-objects.d.ts +3 -0
  26. package/dist/handlers/delete-account-objects.js +35 -0
  27. package/dist/handlers/empty-trash.d.ts +14 -0
  28. package/dist/handlers/empty-trash.js +217 -0
  29. package/dist/handlers/flag-push.d.ts +38 -0
  30. package/dist/handlers/flag-push.js +195 -0
  31. package/dist/handlers/imported-folder.d.ts +4 -0
  32. package/dist/handlers/imported-folder.js +23 -0
  33. package/dist/handlers/mailbox-management.d.ts +6 -0
  34. package/dist/handlers/mailbox-management.js +294 -0
  35. package/dist/handlers/mailbox-sync-order.d.ts +21 -0
  36. package/dist/handlers/mailbox-sync-order.js +50 -0
  37. package/dist/handlers/message-copy.d.ts +40 -0
  38. package/dist/handlers/message-copy.js +371 -0
  39. package/dist/handlers/message-delete-terminal.d.ts +60 -0
  40. package/dist/handlers/message-delete-terminal.js +77 -0
  41. package/dist/handlers/message-delete.d.ts +57 -0
  42. package/dist/handlers/message-delete.js +520 -0
  43. package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
  44. package/dist/handlers/message-move-terminal.js +94 -0
  45. package/dist/handlers/message-move.d.ts +146 -0
  46. package/dist/handlers/message-move.js +490 -0
  47. package/dist/handlers/placement-move-push.d.ts +62 -0
  48. package/dist/handlers/placement-move-push.js +307 -0
  49. package/dist/handlers/probe-message-id.d.ts +2 -0
  50. package/dist/handlers/probe-message-id.js +6 -0
  51. package/dist/handlers/sync-mailboxes.d.ts +45 -0
  52. package/dist/handlers/sync-mailboxes.js +233 -0
  53. package/dist/handlers/sync-message-body.d.ts +23 -0
  54. package/dist/handlers/sync-message-body.js +182 -0
  55. package/dist/handlers/sync-messages.d.ts +29 -0
  56. package/dist/handlers/sync-messages.js +285 -0
  57. package/dist/index.d.ts +9 -0
  58. package/dist/index.js +67 -0
  59. package/dist/lambda-context.d.ts +8 -0
  60. package/dist/lambda-context.js +33 -0
  61. package/dist/mailbox-row.d.ts +9 -0
  62. package/dist/mailbox-row.js +6 -0
  63. package/dist/poller.d.ts +1 -0
  64. package/dist/poller.js +51 -0
  65. package/dist/process.d.ts +6 -0
  66. package/dist/process.js +3 -0
  67. package/dist/processor.d.ts +3 -0
  68. package/dist/processor.js +44 -0
  69. package/dist/queues.d.ts +10 -0
  70. package/dist/queues.js +1 -0
  71. package/dist/s3-account-objects.d.ts +3 -0
  72. package/dist/s3-account-objects.js +26 -0
  73. package/dist/scheduler/attachment-sweep.d.ts +11 -0
  74. package/dist/scheduler/attachment-sweep.js +23 -0
  75. package/dist/scheduler/calendar-subscriptions.d.ts +26 -0
  76. package/dist/scheduler/calendar-subscriptions.js +35 -0
  77. package/dist/scheduler/config.d.ts +27 -0
  78. package/{src/scheduler/config.ts → dist/scheduler/config.js} +7 -26
  79. package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
  80. package/dist/scheduler/decide-due.js +17 -0
  81. package/dist/scheduler/handler.d.ts +21 -0
  82. package/{src/scheduler/handler.ts → dist/scheduler/handler.js} +24 -36
  83. package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
  84. package/dist/scheduler/loop.js +8 -0
  85. package/dist/scheduler/run-tick.d.ts +62 -0
  86. package/dist/scheduler/run-tick.js +164 -0
  87. package/dist/scheduler/runner.d.ts +2 -0
  88. package/{src/scheduler/runner.ts → dist/scheduler/runner.js} +58 -84
  89. package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
  90. package/dist/scheduler/stranded-placement-redrive.js +172 -0
  91. package/dist/sqs-queues.d.ts +4 -0
  92. package/dist/sqs-queues.js +22 -0
  93. package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
  94. package/dist/with-oauth-lifecycle-deps.js +17 -0
  95. package/dist/with-oauth-lifecycle.d.ts +56 -0
  96. package/dist/with-oauth-lifecycle.js +92 -0
  97. package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
  98. package/dist/worker-version.js +12 -0
  99. package/package.json +75 -69
  100. package/src/account-check.test.ts +0 -149
  101. package/src/account-check.ts +0 -101
  102. package/src/body-sync-gate.test.ts +0 -185
  103. package/src/body-sync-gate.ts +0 -86
  104. package/src/cli.ts +0 -224
  105. package/src/connection-scope.test.ts +0 -266
  106. package/src/connection-scope.ts +0 -314
  107. package/src/credential-less-sync-fence.test.ts +0 -152
  108. package/src/e2e-processor-shim.ts +0 -232
  109. package/src/emit.test.ts +0 -127
  110. package/src/emit.ts +0 -117
  111. package/src/events.ts +0 -252
  112. package/src/failure-kind.test.ts +0 -42
  113. package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
  114. package/src/handlers/append-sent-message.ts +0 -264
  115. package/src/handlers/delete-account-objects.test.ts +0 -128
  116. package/src/handlers/delete-account-objects.ts +0 -94
  117. package/src/handlers/empty-trash.sqlite.test.ts +0 -209
  118. package/src/handlers/empty-trash.ts +0 -352
  119. package/src/handlers/flag-push.ts +0 -285
  120. package/src/handlers/imported-folder.test.ts +0 -63
  121. package/src/handlers/imported-folder.ts +0 -38
  122. package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
  123. package/src/handlers/mailbox-management.ts +0 -486
  124. package/src/handlers/mailbox-sync-order.test.ts +0 -93
  125. package/src/handlers/mailbox-sync-order.ts +0 -65
  126. package/src/handlers/message-copy.sqlite.test.ts +0 -298
  127. package/src/handlers/message-copy.ts +0 -566
  128. package/src/handlers/message-delete-terminal.ts +0 -147
  129. package/src/handlers/message-delete.sqlite.test.ts +0 -291
  130. package/src/handlers/message-delete.ts +0 -885
  131. package/src/handlers/message-move.sqlite.test.ts +0 -271
  132. package/src/handlers/message-move.test.ts +0 -216
  133. package/src/handlers/message-move.ts +0 -752
  134. package/src/handlers/placement-move-push.test.ts +0 -137
  135. package/src/handlers/placement-move-push.ts +0 -475
  136. package/src/handlers/placement-push.sqlite.test.ts +0 -298
  137. package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
  138. package/src/handlers/probe-message-id.ts +0 -15
  139. package/src/handlers/sync-mailboxes.test.ts +0 -183
  140. package/src/handlers/sync-mailboxes.ts +0 -421
  141. package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
  142. package/src/handlers/sync-message-body.test.ts +0 -150
  143. package/src/handlers/sync-message-body.ts +0 -365
  144. package/src/handlers/sync-messages.sqlite.test.ts +0 -183
  145. package/src/handlers/sync-messages.ts +0 -525
  146. package/src/index.test.ts +0 -22
  147. package/src/index.ts +0 -86
  148. package/src/mail-sync-selection.sqlite.test.ts +0 -234
  149. package/src/mailbox-row.ts +0 -20
  150. package/src/pending-sync.sqlite.test.ts +0 -166
  151. package/src/poller.ts +0 -55
  152. package/src/processor.ts +0 -60
  153. package/src/queues.ts +0 -49
  154. package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -75
  155. package/src/scheduler/attachment-sweep.ts +0 -46
  156. package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
  157. package/src/scheduler/calendar-subscriptions.ts +0 -103
  158. package/src/scheduler/config.test.ts +0 -40
  159. package/src/scheduler/decide-due.test.ts +0 -44
  160. package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
  161. package/src/scheduler/loop.test.ts +0 -143
  162. package/src/scheduler/run-tick.sqlite.test.ts +0 -191
  163. package/src/scheduler/run-tick.ts +0 -281
  164. package/src/scheduler/stranded-placement-redrive.ts +0 -265
  165. package/src/stranded-placement.sqlite.test.ts +0 -794
  166. package/src/test-helpers/test-world.ts +0 -427
  167. package/src/thread-join.sqlite.test.ts +0 -106
  168. package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
  169. package/src/with-oauth-lifecycle-deps.ts +0 -71
  170. package/src/with-oauth-lifecycle.ts +0 -158
  171. package/tsconfig.json +0 -8
@@ -1,264 +0,0 @@
1
- import { getClient } from "@remit/backend/client";
2
- import {
3
- APPENDED_UID_NONE,
4
- APPENDED_UID_UNREPORTED,
5
- isSentCopyFiled,
6
- } from "@remit/data-ports";
7
- import { isNotFoundError } from "@remit/data-ports/errors";
8
- import { OutboxMessageStatus } from "@remit/domain-enums";
9
- import type { Logger } from "@remit/logger";
10
- import { OutboxAttachmentUnavailableError } from "@remit/mailbox-service/outbox-attachment-content";
11
- import {
12
- buildMailMessage,
13
- renderRawMessage,
14
- } from "@remit/smtp-service/message-builder";
15
- import { attemptBudget } from "@remit/sqs-client/attempt-budget";
16
- import { isAccountDeleted } from "../account-check.js";
17
- import { createConnectionScopeWithCredentials } from "../connection-scope.js";
18
- import type { AppendSentMessageEvent } from "../events.js";
19
- import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
20
- import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
21
-
22
- const UNFILED_NO_SENT_MAILBOX =
23
- "Sent, but not filed: this account has no folder appointed to the Sent role and none that a Sent folder could be recognised by. Appoint one in the account's folder settings and later messages will be filed there.";
24
-
25
- const UNFILED_SIGNED_OUT =
26
- "Sent, but not filed: this account has to be signed in again before a copy can be stored in Sent.";
27
-
28
- const ROW_SURVIVED_ITS_DELETE =
29
- "Sent message was filed but its outbox row survived its delete and stays hidden until the boot-time repair";
30
-
31
- const unfiledAppendRefused = (fullPath: string, error: unknown): string =>
32
- error instanceof OutboxAttachmentUnavailableError
33
- ? `Sent, but not filed: the copy for ${fullPath} could not be built because ${error.message}.`
34
- : `Sent, but not filed: the mail server refused to store a copy in ${fullPath} (${error instanceof Error ? error.message : String(error)}).`;
35
-
36
- export const getAppendSentMaxAttempts = (
37
- processEnv: NodeJS.ProcessEnv = process.env,
38
- ): number => attemptBudget("APPEND_SENT_MAX_ATTEMPTS", 3, processEnv);
39
-
40
- export const APPEND_SENT_MAX_ATTEMPTS = getAppendSentMaxAttempts();
41
-
42
- export const handleAppendSentMessage = async (
43
- event: AppendSentMessageEvent,
44
- log: Logger,
45
- /**
46
- * SQS's own delivery count for this record. The APPEND is the last step of a
47
- * message the user has already had delivered, so exhausting the redrive
48
- * budget must settle the row rather than dead-letter it: a DLQ'd record
49
- * leaves the row at `sent`, which no view shows.
50
- */
51
- receiveCount = 1,
52
- createConnectionScope = createConnectionScopeWithCredentials,
53
- ): Promise<void> => {
54
- const {
55
- account: accountService,
56
- outboxMessage: outboxMessageService,
57
- outboxAttachment: outboxAttachmentService,
58
- mailboxSpecialUse: mailboxSpecialUseService,
59
- secrets,
60
- } = await getClient();
61
-
62
- const { accountId, outboxMessageId } = event;
63
-
64
- log.info({ event: event.type, accountId, outboxMessageId }, "Handling event");
65
-
66
- const account = await accountService.get(accountId);
67
- if (isAccountDeleted(account, log)) {
68
- return;
69
- }
70
-
71
- // A row this event names can be gone by the time the event is redelivered:
72
- // its own last attempt deleted it, or the boot repair dropped it on the
73
- // strength of a recorded uid. Either way the work is done, and re-throwing
74
- // the same NotFoundError on every redelivery only dead-letters it.
75
- const outbox = await outboxMessageService
76
- .get(account.accountConfigId, outboxMessageId)
77
- .catch((error: unknown) => {
78
- if (isNotFoundError(error)) return null;
79
- throw error;
80
- });
81
- if (!outbox) {
82
- log.warn(
83
- { accountId, outboxMessageId },
84
- "Skipping APPEND_SENT_MESSAGE: the outbox row is already gone",
85
- );
86
- return;
87
- }
88
- if (outbox.status !== OutboxMessageStatus.sent) {
89
- log.info(
90
- { outboxMessageId, status: outbox.status },
91
- "Outbox message not in sent status, skipping APPEND",
92
- );
93
- return;
94
- }
95
-
96
- // The message left over SMTP; only the filing can still fail. Settling the
97
- // row as `unfiled` keeps it in the Outbox list rather than deleting it, so a
98
- // delivered message stays readable somewhere. Every path out of this handler
99
- // that is not a confirmed APPEND ends here.
100
- const settleUnfiled = async (reason: string): Promise<void> => {
101
- log.error(
102
- { accountId, outboxMessageId, reason },
103
- "Sent message could not be filed, settling the outbox row as unfiled",
104
- );
105
- await outboxMessageService.update(
106
- account.accountConfigId,
107
- outboxMessageId,
108
- {
109
- status: OutboxMessageStatus.unfiled,
110
- lastError: reason,
111
- },
112
- );
113
- };
114
-
115
- // The copy is in Sent, so the row is a leftover and deleting it is all that
116
- // is left to do. Both steps are idempotent against a row that is already
117
- // gone, so nothing here needs to know how far the last attempt got.
118
- //
119
- // Files first, row second, the same order outbox-queue.ts discards a draft
120
- // in. Nothing but this row points at those objects: a row that outlives its
121
- // files is hidden and the boot repair drops it, while files that outlive
122
- // their row name a message nothing can reach and are vouched for forever.
123
- const dropOutboxRow = async (): Promise<void> => {
124
- await outboxAttachmentService.discardAll(
125
- account.accountConfigId,
126
- accountId,
127
- outboxMessageId,
128
- );
129
- await outboxMessageService.delete(account.accountConfigId, outboxMessageId);
130
- log.info(
131
- { outboxMessageId },
132
- "Deleted outbox row after successful APPEND to Sent",
133
- );
134
- };
135
-
136
- // Everything this recovers from happened after the copy reached Sent, so
137
- // re-appending is off the table and the delete is the only step left to
138
- // retry. Below the redrive budget that retry is worth having; at it the
139
- // record would dead-letter, so the row is left at `sent` for the repair.
140
- const retryDropWithinBudget = (error: unknown): void => {
141
- if (receiveCount < APPEND_SENT_MAX_ATTEMPTS) throw error;
142
- log.error({ accountId, outboxMessageId, error }, ROW_SURVIVED_ITS_DELETE);
143
- };
144
-
145
- // A redelivery of an event whose APPEND already landed. The copy is in the
146
- // user's Sent folder and the only step still owed is the delete that failed
147
- // last time; starting from the top would file a second copy of a message the
148
- // user sent once (#858). Ahead of the Sent-mailbox lookup on purpose — a
149
- // message that is already filed does not need one, and an account that lost
150
- // its Sent appointment in between must not settle as unfiled.
151
- if (isSentCopyFiled(outbox)) {
152
- log.info(
153
- { outboxMessageId, appendedUid: outbox.appendedUid },
154
- "Sent copy was already filed by an earlier attempt, skipping the APPEND",
155
- );
156
- await dropOutboxRow().catch(retryDropWithinBudget);
157
- return;
158
- }
159
-
160
- const sentMailbox = await mailboxSpecialUseService.findSentMailbox(accountId);
161
- if (!sentMailbox) {
162
- await settleUnfiled(UNFILED_NO_SENT_MAILBOX);
163
- return;
164
- }
165
-
166
- let appendedUid = APPENDED_UID_NONE;
167
-
168
- const failure = await withOAuthLifecycle(
169
- buildLifecycleDeps(secrets, accountService),
170
- account,
171
- log,
172
- async (credentials) => {
173
- // Inside the retried unit: a file that cannot be read is retried below
174
- // the budget and settles `unfiled` at it, rather than dead-lettering a
175
- // delivered message out of every view.
176
- const attachments = await outboxAttachmentService.contentsFor(
177
- account.accountConfigId,
178
- accountId,
179
- outboxMessageId,
180
- );
181
- const scope = createConnectionScope(account, credentials);
182
-
183
- await scope
184
- .getConnection()
185
- .then(async (connection) => {
186
- const rawMessage = await renderRawMessage(
187
- buildMailMessage(outbox, attachments),
188
- );
189
-
190
- const result = await connection.append(
191
- sentMailbox.fullPath,
192
- rawMessage,
193
- ["\\Seen"],
194
- );
195
-
196
- log.info(
197
- {
198
- outboxMessageId,
199
- sentMailbox: sentMailbox.fullPath,
200
- uid: result.uid,
201
- uidValidity: result.uidValidity,
202
- },
203
- "Appended sent message to Sent mailbox",
204
- );
205
-
206
- // A server without UIDPLUS files the copy and names no uid for
207
- // it, which is still a filed copy and has to read as one.
208
- appendedUid = result.uid > 0 ? result.uid : APPENDED_UID_UNREPORTED;
209
- })
210
- .finally(() => scope.disconnect());
211
-
212
- // The idempotency key, and the only reason a redelivery is safe. It is
213
- // written before the delete and never after it: from here on every step
214
- // can fail and be retried without the user gaining a second copy, and
215
- // the one window that still can is this single row update.
216
- await outboxMessageService.update(
217
- account.accountConfigId,
218
- outboxMessageId,
219
- { appendedUid },
220
- );
221
-
222
- // The message now lives in the IMAP Sent folder. Drop the outbox row so
223
- // the user does not see it twice in the UI (Outbox + Sent). Issue #178.
224
- await dropOutboxRow();
225
- },
226
- ).then(
227
- () => null,
228
- (error: unknown) => {
229
- // Below the redrive budget this is an ordinary retry — connections
230
- // drop, servers go away, and the row is still `sent` for the next
231
- // attempt to pick up. At the budget the record would dead-letter, and a
232
- // dead-lettered APPEND is exactly how a delivered message goes missing.
233
- //
234
- // The budget still binds after a landed APPEND, though no longer to
235
- // hold off a duplicate: the recorded uid does that, and the retries
236
- // the budget allows re-drive the delete alone.
237
- if (receiveCount < APPEND_SENT_MAX_ATTEMPTS) throw error;
238
- return error;
239
- },
240
- );
241
-
242
- // The APPEND landed: the copy is in Sent whatever failed after it, so there
243
- // is nothing to settle as unfiled. A row that outlives its delete holds
244
- // `sent`, which every view hides, until the migrator's boot-time stranded-row
245
- // repair drops it — the next container start, not sooner (#824).
246
- if (isSentCopyFiled({ appendedUid })) {
247
- if (failure) {
248
- log.error(
249
- { accountId, outboxMessageId, reason: String(failure) },
250
- ROW_SURVIVED_ITS_DELETE,
251
- );
252
- }
253
- return;
254
- }
255
-
256
- // A terminal auth failure returns here without throwing — withOAuthLifecycle
257
- // flips the account to reauth_required and ACKs the record, which without
258
- // this would leave the row at `sent` and the message nowhere.
259
- await settleUnfiled(
260
- failure
261
- ? unfiledAppendRefused(sentMailbox.fullPath, failure)
262
- : UNFILED_SIGNED_OUT,
263
- );
264
- };
@@ -1,128 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { afterEach, beforeEach, describe, it } from "node:test";
3
- import {
4
- DeleteObjectsCommand,
5
- ListObjectsV2Command,
6
- S3Client,
7
- } from "@aws-sdk/client-s3";
8
- import {
9
- type RecordingQueue,
10
- recordingQueue,
11
- } from "@remit/data-ports/recording-queue";
12
- import { noopLogger } from "@remit/logger";
13
- import { mockClient } from "aws-sdk-client-mock";
14
- import type { DeleteAccountObjectsEvent } from "../events.js";
15
- import { setWorkerQueues } from "../queues.js";
16
- import { handleDeleteAccountObjects } from "./delete-account-objects.js";
17
-
18
- const s3Mock = mockClient(S3Client);
19
- let queue: RecordingQueue;
20
-
21
- const event: DeleteAccountObjectsEvent = {
22
- type: "DELETE_ACCOUNT_OBJECTS",
23
- accountConfigId: "cfg-1",
24
- };
25
-
26
- describe("handleDeleteAccountObjects", () => {
27
- beforeEach(() => {
28
- queue = recordingQueue();
29
- setWorkerQueues({
30
- mailboxes: queue,
31
- messages: queue,
32
- flags: queue,
33
- body: queue,
34
- mailboxManagement: queue,
35
- messageManagement: queue,
36
- });
37
- });
38
-
39
- afterEach(() => {
40
- s3Mock.reset();
41
- });
42
-
43
- it("lists and deletes every object under the account prefix in one page", async () => {
44
- s3Mock.on(ListObjectsV2Command).resolves({
45
- Contents: [
46
- { Key: "accounts/cfg-1/a.eml" },
47
- { Key: "accounts/cfg-1/b.eml" },
48
- ],
49
- IsTruncated: false,
50
- });
51
- s3Mock.on(DeleteObjectsCommand).resolves({});
52
-
53
- await handleDeleteAccountObjects(event, noopLogger);
54
-
55
- const listCall = s3Mock.commandCalls(ListObjectsV2Command)[0];
56
- assert.equal(listCall.args[0].input.Prefix, "accounts/cfg-1/");
57
-
58
- const deleteCall = s3Mock.commandCalls(DeleteObjectsCommand)[0];
59
- assert.deepEqual(deleteCall.args[0].input.Delete?.Objects, [
60
- { Key: "accounts/cfg-1/a.eml" },
61
- { Key: "accounts/cfg-1/b.eml" },
62
- ]);
63
- assert.equal(queue.sent.length, 0);
64
- });
65
-
66
- it("follows the continuation token across truncated pages", async () => {
67
- s3Mock
68
- .on(ListObjectsV2Command)
69
- .resolvesOnce({
70
- Contents: [{ Key: "accounts/cfg-1/p1.eml" }],
71
- IsTruncated: true,
72
- NextContinuationToken: "tok-2",
73
- })
74
- .resolvesOnce({
75
- Contents: [{ Key: "accounts/cfg-1/p2.eml" }],
76
- IsTruncated: false,
77
- });
78
- s3Mock.on(DeleteObjectsCommand).resolves({});
79
-
80
- await handleDeleteAccountObjects(event, noopLogger);
81
-
82
- const listCalls = s3Mock.commandCalls(ListObjectsV2Command);
83
- assert.equal(listCalls.length, 2);
84
- assert.equal(listCalls[1]?.args[0].input.ContinuationToken, "tok-2");
85
- assert.equal(s3Mock.commandCalls(DeleteObjectsCommand).length, 2);
86
- });
87
-
88
- it("skips the delete call when a page holds no keys", async () => {
89
- s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
90
-
91
- await handleDeleteAccountObjects(event, noopLogger);
92
-
93
- assert.equal(s3Mock.commandCalls(DeleteObjectsCommand).length, 0);
94
- });
95
-
96
- it("re-enqueues with the continuation token when time is nearly up", async () => {
97
- s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
98
-
99
- await handleDeleteAccountObjects(
100
- { ...event, continuationToken: "tok-mid" },
101
- noopLogger,
102
- () => 5_000,
103
- );
104
-
105
- assert.equal(
106
- s3Mock.commandCalls(ListObjectsV2Command).length,
107
- 0,
108
- "bails before starting a new page",
109
- );
110
- const body = queue.sent[0]?.body;
111
- assert.equal(body?.type, "DELETE_ACCOUNT_OBJECTS");
112
- assert.equal(body?.accountConfigId, "cfg-1");
113
- assert.equal(body?.continuationToken, "tok-mid");
114
- });
115
-
116
- it("keeps paging while time remains", async () => {
117
- s3Mock.on(ListObjectsV2Command).resolves({
118
- Contents: [{ Key: "accounts/cfg-1/x.eml" }],
119
- IsTruncated: false,
120
- });
121
- s3Mock.on(DeleteObjectsCommand).resolves({});
122
-
123
- await handleDeleteAccountObjects(event, noopLogger, () => 120_000);
124
-
125
- assert.equal(s3Mock.commandCalls(ListObjectsV2Command).length, 1);
126
- assert.equal(queue.sent.length, 0);
127
- });
128
- });
@@ -1,94 +0,0 @@
1
- import {
2
- DeleteObjectsCommand,
3
- ListObjectsV2Command,
4
- S3Client,
5
- } from "@aws-sdk/client-s3";
6
- import type { Logger } from "@remit/logger";
7
- import { env } from "expect-env";
8
- import type { DeleteAccountObjectsEvent } from "../events.js";
9
- import { workerQueues } from "../queues.js";
10
-
11
- const s3 = new S3Client({});
12
-
13
- const BATCH_SIZE = 1_000;
14
- const MIN_REMAINING_MS = 30_000;
15
-
16
- export const handleDeleteAccountObjects = async (
17
- event: DeleteAccountObjectsEvent,
18
- log: Logger,
19
- getRemainingTimeMs?: () => number,
20
- ): Promise<void> => {
21
- const BUCKET_NAME = env.S3_BUCKET_NAME;
22
- const { accountConfigId, continuationToken } = event;
23
- const prefix = `accounts/${accountConfigId}/`;
24
-
25
- log.info(
26
- { accountConfigId, prefix, hasContinuation: !!continuationToken },
27
- "Deleting account objects from S3",
28
- );
29
-
30
- let currentToken = continuationToken;
31
- let totalDeleted = 0;
32
-
33
- while (true) {
34
- // Check remaining time before starting a new page
35
- if (getRemainingTimeMs && getRemainingTimeMs() < MIN_REMAINING_MS) {
36
- log.info(
37
- { accountConfigId, totalDeleted, continuationToken: currentToken },
38
- "Near timeout, re-enqueuing with continuation token",
39
- );
40
- await reenqueue(accountConfigId, currentToken);
41
- return;
42
- }
43
-
44
- const listResult = await s3.send(
45
- new ListObjectsV2Command({
46
- Bucket: BUCKET_NAME,
47
- Prefix: prefix,
48
- MaxKeys: BATCH_SIZE,
49
- ContinuationToken: currentToken,
50
- }),
51
- );
52
-
53
- const keys = (listResult.Contents ?? [])
54
- .map((obj) => obj.Key)
55
- .filter((k): k is string => k !== undefined);
56
-
57
- if (keys.length > 0) {
58
- await s3.send(
59
- new DeleteObjectsCommand({
60
- Bucket: BUCKET_NAME,
61
- Delete: {
62
- Objects: keys.map((Key) => ({ Key })),
63
- Quiet: true,
64
- },
65
- }),
66
- );
67
- totalDeleted += keys.length;
68
- }
69
-
70
- if (!listResult.IsTruncated) {
71
- break;
72
- }
73
-
74
- currentToken = listResult.NextContinuationToken;
75
- }
76
-
77
- log.info(
78
- { accountConfigId, totalDeleted },
79
- "Finished deleting account objects",
80
- );
81
- };
82
-
83
- const reenqueue = async (
84
- accountConfigId: string,
85
- continuationToken: string | undefined,
86
- ): Promise<void> => {
87
- const event: DeleteAccountObjectsEvent = {
88
- type: "DELETE_ACCOUNT_OBJECTS",
89
- accountConfigId,
90
- continuationToken,
91
- };
92
-
93
- await workerQueues().messageManagement.send({ body: event });
94
- };
@@ -1,209 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { after, before, describe, it } from "node:test";
3
- import type { MailboxItem } from "@remit/data-ports";
4
- import {
5
- MailboxCursorState,
6
- MailboxSpecialUse,
7
- MessageStatus,
8
- } from "@remit/domain-enums";
9
- import type { EmptyTrashEvent } from "../events.js";
10
- import {
11
- openStore,
12
- type SeededMessage,
13
- type World,
14
- } from "../test-helpers/test-world.js";
15
- import { handleEmptyTrash } from "./empty-trash.js";
16
-
17
- let store: Awaited<ReturnType<typeof openStore>>;
18
-
19
- before(async () => {
20
- store = await openStore();
21
- });
22
-
23
- after(() => store.close());
24
-
25
- interface Arranged {
26
- w: World;
27
- inbox: MailboxItem;
28
- trash: MailboxItem;
29
- trashed: SeededMessage[];
30
- }
31
-
32
- const arrange = async (count = 2): Promise<Arranged> => {
33
- const w = await store.world();
34
- const inbox = await w.folder("INBOX");
35
- const trash = await w.folder("Trash", {
36
- specialUse: MailboxSpecialUse.Trash,
37
- });
38
- const trashed: SeededMessage[] = [];
39
- for (let i = 0; i < count; i += 1) trashed.push(await w.message(trash));
40
- return { w, inbox, trash, trashed };
41
- };
42
-
43
- /** Empty Trash through the service the API calls, and hand back its event. */
44
- const emptyTrash = async (a: Arranged): Promise<EmptyTrashEvent> => {
45
- await a.w.client.messageMove.emptyTrash(
46
- a.w.account.accountConfigId,
47
- a.w.account.accountId,
48
- );
49
- const event = a.w
50
- .queued()
51
- .find((body) => body.type === "EMPTY_TRASH") as unknown as EmptyTrashEvent;
52
- assert.ok(event, "Empty Trash enqueued an EMPTY_TRASH");
53
- return event;
54
- };
55
-
56
- const run = (a: Arranged, event: EmptyTrashEvent) =>
57
- handleEmptyTrash(event, a.w.log, a.w.scope);
58
-
59
- const visible = async (a: Arranged): Promise<boolean[]> =>
60
- Promise.all(
61
- a.trashed.map(async (message) => {
62
- const row = await a.w.row(message.messageId);
63
- const listing = await a.w.threadRows(message.messageId);
64
- return (
65
- row?.status === MessageStatus.active &&
66
- listing.length > 0 &&
67
- listing.every((tm) => !tm.isDeleted)
68
- );
69
- }),
70
- );
71
-
72
- describe("Empty Trash", () => {
73
- it("expunges every message on the server and removes both local rows", async () => {
74
- const a = await arrange();
75
- const event = await emptyTrash(a);
76
-
77
- await run(a, event);
78
-
79
- assert.deepEqual(a.w.imap.uids("Trash"), []);
80
- for (const message of a.trashed) {
81
- assert.equal(await a.w.row(message.messageId), undefined);
82
- assert.deepEqual(await a.w.threadRows(message.messageId), []);
83
- }
84
- });
85
-
86
- it("hands every row back when another client emptied the Trash first", async () => {
87
- const a = await arrange();
88
- const event = await emptyTrash(a);
89
- a.w.imap.empty("Trash");
90
-
91
- await run(a, event);
92
-
93
- assert.deepEqual(await visible(a), [true, true]);
94
- });
95
-
96
- it("keeps the row of a message the expunge never reached", async () => {
97
- const a = await arrange();
98
- const event = await emptyTrash(a);
99
- const [kept, gone] = a.trashed;
100
- a.w.imap.expunge("Trash", kept?.uid as number);
101
-
102
- await run(a, event);
103
-
104
- assert.equal(await a.w.row(gone?.messageId ?? ""), undefined);
105
- assert.equal(
106
- (await a.w.row(kept?.messageId ?? ""))?.status,
107
- MessageStatus.active,
108
- );
109
- });
110
-
111
- // A move into Trash that has not settled names Trash while still carrying
112
- // INBOX's uid. Matching it against the expunge would delete whatever Trash
113
- // holds at that number.
114
- it("leaves a message whose move into Trash has not settled", async () => {
115
- const a = await arrange(1);
116
- const moving = await a.w.message(a.inbox);
117
- await a.w.client.messageMove.deleteMessage(
118
- a.w.account.accountConfigId,
119
- moving.messageId,
120
- a.w.account.accountId,
121
- );
122
- const event = await emptyTrash(a);
123
-
124
- await run(a, event);
125
-
126
- const row = await a.w.row(moving.messageId);
127
- assert.equal(row?.status, MessageStatus.moving);
128
- assert.deepEqual(a.w.imap.uids("INBOX"), [moving.uid]);
129
- assert.equal(await a.w.row(a.trashed[0]?.messageId ?? ""), undefined);
130
- });
131
-
132
- it("expunges nothing and hands the rows back when the folder at the path is not the one emptied", async () => {
133
- const a = await arrange();
134
- const event = await emptyTrash(a);
135
- a.w.imap.setUidValidity("Trash", 99);
136
-
137
- await run(a, event);
138
-
139
- assert.deepEqual(a.w.imap.commands("EXPUNGE"), []);
140
- assert.deepEqual(await visible(a), [true, true]);
141
- });
142
-
143
- it("expunges nothing when the Trash role no longer rests on confirmed evidence", async () => {
144
- const a = await arrange();
145
- const event = await emptyTrash(a);
146
- await a.w.client.mailboxSpecialUse.deleteByMailboxId(a.trash.mailboxId);
147
-
148
- await run(a, event);
149
-
150
- assert.equal(a.w.imap.connections, 0);
151
- assert.deepEqual(await visible(a), [true, true]);
152
- });
153
-
154
- it("hands the rows back without connecting while the cursor is rebuilding", async () => {
155
- const a = await arrange();
156
- const event = await emptyTrash(a);
157
- await a.w.client.mailbox.update(a.w.account.accountId, a.trash.mailboxId, {
158
- cursorState: MailboxCursorState.rebuilding,
159
- });
160
-
161
- await run(a, event);
162
-
163
- assert.equal(a.w.imap.connections, 0);
164
- assert.deepEqual(await visible(a), [true, true]);
165
- });
166
-
167
- it("refuses an event minted under an unknown contract and hands the rows back", async () => {
168
- const a = await arrange();
169
- const event = await emptyTrash(a);
170
-
171
- await run(a, { ...event, schemaVersion: 99 } as unknown as EmptyTrashEvent);
172
-
173
- assert.equal(a.w.imap.connections, 0);
174
- assert.deepEqual(await visible(a), [true, true]);
175
- });
176
-
177
- it("acks without connecting when the Trash mailbox was deleted", async () => {
178
- const a = await arrange();
179
- const event = await emptyTrash(a);
180
- await a.w.client.mailbox.delete(a.w.account.accountId, a.trash.mailboxId);
181
-
182
- await run(a, event);
183
-
184
- assert.equal(a.w.imap.connections, 0);
185
- });
186
-
187
- it("empties Trash under its current path when the folder was renamed after enqueue", async () => {
188
- const a = await arrange();
189
- const event = await emptyTrash(a);
190
- a.w.imap.renameFolder("Trash", "Deleted Items");
191
- await a.w.client.mailbox.update(a.w.account.accountId, a.trash.mailboxId, {
192
- fullPath: "Deleted Items",
193
- });
194
-
195
- await run(a, event);
196
-
197
- assert.deepEqual(a.w.imap.uids("Deleted Items"), []);
198
- });
199
-
200
- it("rethrows a failed expunge so the event is retried", async () => {
201
- const a = await arrange();
202
- const event = await emptyTrash(a);
203
- a.w.imap.failNext.EXPUNGE = new Error("Connection reset");
204
-
205
- await assert.rejects(run(a, event), /Connection reset/);
206
-
207
- assert.equal(a.w.imap.uids("Trash").length, 2);
208
- });
209
- });