@remit/imap-worker 0.0.90 → 0.0.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/dist/account-check.d.ts +27 -0
  2. package/dist/account-check.js +70 -0
  3. package/dist/body-sync-gate.d.ts +12 -0
  4. package/dist/body-sync-gate.js +53 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +193 -0
  7. package/dist/config-env.d.ts +5 -0
  8. package/dist/config-env.js +37 -0
  9. package/dist/connection-pool.d.ts +22 -0
  10. package/dist/connection-pool.js +97 -0
  11. package/dist/connection-scope.d.ts +8 -0
  12. package/dist/connection-scope.js +10 -0
  13. package/dist/context.d.ts +46 -0
  14. package/dist/context.js +15 -0
  15. package/dist/e2e-processor-shim.d.ts +2 -0
  16. package/dist/e2e-processor-shim.js +181 -0
  17. package/dist/emit.d.ts +15 -0
  18. package/dist/emit.js +76 -0
  19. package/dist/events.d.ts +203 -0
  20. package/dist/events.js +1 -0
  21. package/dist/failure-kind.d.ts +10 -0
  22. package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
  23. package/dist/handlers/append-sent-message.d.ts +3 -0
  24. package/dist/handlers/append-sent-message.js +152 -0
  25. package/dist/handlers/delete-account-objects.d.ts +3 -0
  26. package/dist/handlers/delete-account-objects.js +35 -0
  27. package/dist/handlers/empty-trash.d.ts +14 -0
  28. package/dist/handlers/empty-trash.js +217 -0
  29. package/dist/handlers/flag-push.d.ts +38 -0
  30. package/dist/handlers/flag-push.js +195 -0
  31. package/dist/handlers/imported-folder.d.ts +4 -0
  32. package/dist/handlers/imported-folder.js +23 -0
  33. package/dist/handlers/mailbox-management.d.ts +6 -0
  34. package/dist/handlers/mailbox-management.js +294 -0
  35. package/dist/handlers/mailbox-sync-order.d.ts +21 -0
  36. package/dist/handlers/mailbox-sync-order.js +50 -0
  37. package/dist/handlers/message-copy.d.ts +40 -0
  38. package/dist/handlers/message-copy.js +371 -0
  39. package/dist/handlers/message-delete-terminal.d.ts +60 -0
  40. package/dist/handlers/message-delete-terminal.js +77 -0
  41. package/dist/handlers/message-delete.d.ts +57 -0
  42. package/dist/handlers/message-delete.js +520 -0
  43. package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
  44. package/dist/handlers/message-move-terminal.js +94 -0
  45. package/dist/handlers/message-move.d.ts +146 -0
  46. package/dist/handlers/message-move.js +490 -0
  47. package/dist/handlers/placement-move-push.d.ts +62 -0
  48. package/dist/handlers/placement-move-push.js +307 -0
  49. package/dist/handlers/probe-message-id.d.ts +2 -0
  50. package/dist/handlers/probe-message-id.js +6 -0
  51. package/dist/handlers/sync-mailboxes.d.ts +45 -0
  52. package/dist/handlers/sync-mailboxes.js +233 -0
  53. package/dist/handlers/sync-message-body.d.ts +23 -0
  54. package/dist/handlers/sync-message-body.js +182 -0
  55. package/dist/handlers/sync-messages.d.ts +29 -0
  56. package/dist/handlers/sync-messages.js +285 -0
  57. package/dist/index.d.ts +9 -0
  58. package/dist/index.js +67 -0
  59. package/dist/lambda-context.d.ts +8 -0
  60. package/dist/lambda-context.js +33 -0
  61. package/dist/mailbox-row.d.ts +9 -0
  62. package/dist/mailbox-row.js +6 -0
  63. package/dist/poller.d.ts +1 -0
  64. package/dist/poller.js +51 -0
  65. package/dist/process.d.ts +6 -0
  66. package/dist/process.js +3 -0
  67. package/dist/processor.d.ts +3 -0
  68. package/dist/processor.js +44 -0
  69. package/dist/queues.d.ts +10 -0
  70. package/dist/queues.js +1 -0
  71. package/dist/s3-account-objects.d.ts +3 -0
  72. package/dist/s3-account-objects.js +26 -0
  73. package/dist/scheduler/attachment-sweep.d.ts +11 -0
  74. package/dist/scheduler/attachment-sweep.js +23 -0
  75. package/dist/scheduler/calendar-subscriptions.d.ts +26 -0
  76. package/dist/scheduler/calendar-subscriptions.js +35 -0
  77. package/dist/scheduler/config.d.ts +27 -0
  78. package/{src/scheduler/config.ts → dist/scheduler/config.js} +7 -26
  79. package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
  80. package/dist/scheduler/decide-due.js +17 -0
  81. package/dist/scheduler/handler.d.ts +21 -0
  82. package/{src/scheduler/handler.ts → dist/scheduler/handler.js} +25 -42
  83. package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
  84. package/dist/scheduler/loop.js +8 -0
  85. package/dist/scheduler/run-tick.d.ts +62 -0
  86. package/dist/scheduler/run-tick.js +164 -0
  87. package/dist/scheduler/runner.d.ts +2 -0
  88. package/{src/scheduler/runner.ts → dist/scheduler/runner.js} +57 -78
  89. package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
  90. package/dist/scheduler/stranded-placement-redrive.js +172 -0
  91. package/dist/sqs-queues.d.ts +4 -0
  92. package/dist/sqs-queues.js +22 -0
  93. package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
  94. package/dist/with-oauth-lifecycle-deps.js +17 -0
  95. package/dist/with-oauth-lifecycle.d.ts +56 -0
  96. package/dist/with-oauth-lifecycle.js +92 -0
  97. package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
  98. package/dist/worker-version.js +12 -0
  99. package/package.json +75 -69
  100. package/src/account-check.test.ts +0 -149
  101. package/src/account-check.ts +0 -101
  102. package/src/body-sync-gate.test.ts +0 -185
  103. package/src/body-sync-gate.ts +0 -86
  104. package/src/cli.ts +0 -218
  105. package/src/connection-scope.test.ts +0 -266
  106. package/src/connection-scope.ts +0 -314
  107. package/src/credential-less-sync-fence.test.ts +0 -152
  108. package/src/e2e-processor-shim.ts +0 -232
  109. package/src/emit.test.ts +0 -145
  110. package/src/emit.ts +0 -115
  111. package/src/events.ts +0 -252
  112. package/src/failure-kind.test.ts +0 -42
  113. package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
  114. package/src/handlers/append-sent-message.ts +0 -264
  115. package/src/handlers/delete-account-objects.test.ts +0 -126
  116. package/src/handlers/delete-account-objects.ts +0 -96
  117. package/src/handlers/empty-trash.sqlite.test.ts +0 -209
  118. package/src/handlers/empty-trash.ts +0 -352
  119. package/src/handlers/flag-push.ts +0 -285
  120. package/src/handlers/imported-folder.test.ts +0 -63
  121. package/src/handlers/imported-folder.ts +0 -38
  122. package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
  123. package/src/handlers/mailbox-management.ts +0 -486
  124. package/src/handlers/mailbox-sync-order.test.ts +0 -93
  125. package/src/handlers/mailbox-sync-order.ts +0 -65
  126. package/src/handlers/message-copy.sqlite.test.ts +0 -298
  127. package/src/handlers/message-copy.ts +0 -569
  128. package/src/handlers/message-delete-terminal.ts +0 -147
  129. package/src/handlers/message-delete.sqlite.test.ts +0 -291
  130. package/src/handlers/message-delete.ts +0 -888
  131. package/src/handlers/message-move.sqlite.test.ts +0 -271
  132. package/src/handlers/message-move.test.ts +0 -216
  133. package/src/handlers/message-move.ts +0 -755
  134. package/src/handlers/placement-move-push.test.ts +0 -137
  135. package/src/handlers/placement-move-push.ts +0 -478
  136. package/src/handlers/placement-push.sqlite.test.ts +0 -299
  137. package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
  138. package/src/handlers/probe-message-id.ts +0 -15
  139. package/src/handlers/sync-mailboxes.test.ts +0 -183
  140. package/src/handlers/sync-mailboxes.ts +0 -425
  141. package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
  142. package/src/handlers/sync-message-body.test.ts +0 -150
  143. package/src/handlers/sync-message-body.ts +0 -366
  144. package/src/handlers/sync-messages.sqlite.test.ts +0 -195
  145. package/src/handlers/sync-messages.ts +0 -537
  146. package/src/index.test.ts +0 -22
  147. package/src/index.ts +0 -82
  148. package/src/mail-sync-selection.sqlite.test.ts +0 -227
  149. package/src/mailbox-row.ts +0 -20
  150. package/src/pending-sync.sqlite.test.ts +0 -165
  151. package/src/poller.ts +0 -55
  152. package/src/processor.ts +0 -62
  153. package/src/queues.ts +0 -40
  154. package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -74
  155. package/src/scheduler/attachment-sweep.ts +0 -46
  156. package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
  157. package/src/scheduler/calendar-subscriptions.ts +0 -103
  158. package/src/scheduler/config.test.ts +0 -40
  159. package/src/scheduler/decide-due.test.ts +0 -44
  160. package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
  161. package/src/scheduler/loop.test.ts +0 -143
  162. package/src/scheduler/run-tick.sqlite.test.ts +0 -191
  163. package/src/scheduler/run-tick.ts +0 -281
  164. package/src/scheduler/stranded-placement-redrive.ts +0 -265
  165. package/src/stranded-placement.sqlite.test.ts +0 -799
  166. package/src/test-helpers/test-world.ts +0 -425
  167. package/src/test-helpers/worker-queues.ts +0 -11
  168. package/src/thread-join.sqlite.test.ts +0 -106
  169. package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
  170. package/src/with-oauth-lifecycle-deps.ts +0 -71
  171. package/src/with-oauth-lifecycle.ts +0 -158
  172. package/tsconfig.json +0 -8
@@ -1,227 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { randomUUID } from "node:crypto";
3
- import { afterEach, beforeEach, describe, it } from "node:test";
4
- import {
5
- _resetForTest,
6
- type RemitClient,
7
- setClient,
8
- } from "@remit/backend/client";
9
- import type { AccountItem } from "@remit/data-ports";
10
- import { recordingQueue } from "@remit/data-ports/recording-queue";
11
- import {
12
- AccountAuthType,
13
- AccountService,
14
- ConnectionState,
15
- MailboxCursorState,
16
- } from "@remit/domain-enums";
17
- import {
18
- type AccountRepo,
19
- createSqliteRepositories,
20
- type DrizzleMessageRepository,
21
- MailboxLockRepo,
22
- type MailboxRepo,
23
- } from "@remit/drizzle-service";
24
- import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
25
- import type { Logger } from "@remit/logger";
26
- import { syncMailboxes } from "./handlers/sync-mailboxes.js";
27
- import { syncMessages } from "./handlers/sync-messages.js";
28
- import { runSchedulerTick } from "./scheduler/run-tick.js";
29
- import { workerQueuesOn } from "./test-helpers/worker-queues.js";
30
-
31
- const NOW = 1_700_000_000_000;
32
-
33
- const silentLogger: Logger = (() => {
34
- const noop = () => {};
35
- const stub = {
36
- info: noop,
37
- warn: noop,
38
- error: noop,
39
- debug: noop,
40
- fatal: noop,
41
- trace: noop,
42
- child: () => stub,
43
- };
44
- return stub as unknown as Logger;
45
- })();
46
-
47
- let accounts: AccountRepo;
48
- let mailboxes: MailboxRepo;
49
- let messages: DrizzleMessageRepository;
50
- let locks: MailboxLockRepo;
51
- let client: RemitClient;
52
- let close: () => void;
53
-
54
- beforeEach(() => {
55
- const store = createShippedSqliteDb();
56
- close = store.close;
57
- const repos = createSqliteRepositories({
58
- app: store.db,
59
- mail: store.db,
60
- search: store.db,
61
- });
62
- accounts = repos.account;
63
- mailboxes = repos.mailbox;
64
- messages = repos.message;
65
- locks = new MailboxLockRepo(store.db);
66
- client = {
67
- account: accounts,
68
- mailbox: mailboxes,
69
- message: messages,
70
- mailboxLock: locks,
71
- } as unknown as RemitClient;
72
- setClient(client);
73
- });
74
-
75
- afterEach(() => {
76
- _resetForTest();
77
- close();
78
- });
79
-
80
- const seedSyncedAccount = async (): Promise<{
81
- account: AccountItem;
82
- mailboxId: string;
83
- messageId: string;
84
- }> => {
85
- const account = await accounts.create({
86
- accountConfigId: "cfg-1181",
87
- username: "person@example.com",
88
- email: "person@example.com",
89
- authType: AccountAuthType.Password,
90
- passwordHash: '{"ciphertext":"x"}',
91
- imapHost: "imap.example.com",
92
- imapPort: 993,
93
- imapTls: true,
94
- imapStartTls: false,
95
- isActive: true,
96
- connectionState: ConnectionState.Authenticated,
97
- });
98
- const created = await mailboxes.create({
99
- accountId: account.accountId,
100
- namespacePrefix: "",
101
- hierarchyDelimiter: "/",
102
- fullPath: "INBOX",
103
- uidValidity: 7,
104
- uidNext: 4,
105
- highestModseq: "42",
106
- messageCount: 1,
107
- unseenCount: 0,
108
- deletedCount: 0,
109
- totalSize: 10,
110
- lastSyncUid: 3,
111
- highWaterMarkUid: 3,
112
- lastMessageSyncAt: NOW,
113
- cursorState: MailboxCursorState.normal,
114
- } as Parameters<MailboxRepo["create"]>[0]);
115
- if (created.outcome !== "Created") throw new Error("INBOX path taken");
116
- const mailbox = created.mailbox;
117
- const messageId = randomUUID();
118
- await messages.create({
119
- messageId,
120
- mailboxId: mailbox.mailboxId,
121
- uid: 3,
122
- sequenceNumber: 1,
123
- rfc822Size: 10,
124
- internalDate: NOW,
125
- envelopeId: randomUUID(),
126
- rootBodyPartId: randomUUID(),
127
- status: "active",
128
- syncStatus: "synced",
129
- });
130
- return { account, mailboxId: mailbox.mailboxId, messageId };
131
- };
132
-
133
- const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
134
- const queue = recordingQueue();
135
- const result = await runSchedulerTick({
136
- redriveStrandedPlacements: async () => 0,
137
- accountService: accounts,
138
- queue: queue.fifo,
139
- log: silentLogger,
140
- tickIntervalMs: 60_000,
141
- offlineIntervalMs: 0,
142
- now: NOW,
143
- });
144
- return {
145
- enqueued: result.enqueued,
146
- accountIds: queue.sent.map((message) => String(message.body.accountId)),
147
- };
148
- };
149
-
150
- const setServices = (
151
- accountId: string,
152
- syncedServices: AccountItem["syncedServices"],
153
- ) => accounts.update(accountId, { syncedServices });
154
-
155
- describe("turning mail sync off and on again", () => {
156
- it("stops every mail sync entry point and keeps the stored mail", async () => {
157
- const { account, mailboxId, messageId } = await seedSyncedAccount();
158
- const storedMailboxes = await mailboxes.listAllByAccount(account.accountId);
159
- const storedMessage = await messages.get(messageId);
160
-
161
- await setServices(account.accountId, [AccountService.Calendar]);
162
-
163
- assert.deepEqual(await tick(), { enqueued: 0, accountIds: [] });
164
-
165
- await syncMailboxes(
166
- {
167
- type: "SYNC_MAILBOXES",
168
- eventId: "evt-mailboxes",
169
- timestamp: NOW,
170
- accountId: account.accountId,
171
- },
172
- silentLogger,
173
- workerQueuesOn(recordingQueue()),
174
- );
175
-
176
- let connections = 0;
177
- await syncMessages(
178
- {
179
- type: "SYNC_MESSAGES",
180
- eventId: "evt-messages",
181
- timestamp: NOW,
182
- accountId: account.accountId,
183
- mailboxId,
184
- },
185
- silentLogger,
186
- workerQueuesOn(recordingQueue()),
187
- () => {
188
- connections += 1;
189
- throw new Error("a sync that is off must not connect");
190
- },
191
- );
192
- assert.equal(connections, 0);
193
-
194
- assert.deepEqual(
195
- await mailboxes.listAllByAccount(account.accountId),
196
- storedMailboxes,
197
- );
198
- assert.deepEqual(await messages.get(messageId), storedMessage);
199
- assert.equal(
200
- (await accounts.get(account.accountId)).connectionState,
201
- ConnectionState.Authenticated,
202
- );
203
- });
204
-
205
- it("resumes from the stored markers when mail is turned back on", async () => {
206
- const { account, messageId } = await seedSyncedAccount();
207
- const storedMailboxes = await mailboxes.listAllByAccount(account.accountId);
208
- const storedMessage = await messages.get(messageId);
209
-
210
- await setServices(account.accountId, [AccountService.Calendar]);
211
- await tick();
212
- await setServices(account.accountId, [
213
- AccountService.Mail,
214
- AccountService.Calendar,
215
- ]);
216
-
217
- assert.deepEqual(await tick(), {
218
- enqueued: 1,
219
- accountIds: [account.accountId],
220
- });
221
- assert.deepEqual(
222
- await mailboxes.listAllByAccount(account.accountId),
223
- storedMailboxes,
224
- );
225
- assert.deepEqual(await messages.get(messageId), storedMessage);
226
- });
227
- });
@@ -1,20 +0,0 @@
1
- import type { IMailboxRepository, MailboxItem } from "@remit/data-ports";
2
- import { isNotFoundError } from "@remit/data-ports/errors";
3
-
4
- export type MailboxRow =
5
- | { kind: "present"; mailbox: MailboxItem }
6
- | { kind: "gone"; mailboxId: string };
7
-
8
- export const findMailboxRow = (
9
- mailboxService: Pick<IMailboxRepository, "get">,
10
- accountId: string,
11
- mailboxId: string,
12
- ): Promise<MailboxRow> =>
13
- mailboxService.get(accountId, mailboxId).then(
14
- (mailbox): MailboxRow =>
15
- mailbox ? { kind: "present", mailbox } : { kind: "gone", mailboxId },
16
- (error: unknown): MailboxRow => {
17
- if (isNotFoundError(error)) return { kind: "gone", mailboxId };
18
- throw error;
19
- },
20
- );
@@ -1,165 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { afterEach, beforeEach, describe, it } from "node:test";
3
- import {
4
- _resetForTest,
5
- type RemitClient,
6
- setClient,
7
- } from "@remit/backend/client";
8
- import {
9
- type RecordingQueue,
10
- recordingQueue,
11
- } from "@remit/data-ports/recording-queue";
12
- import { MailboxLockRepo } from "@remit/drizzle-service";
13
- import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
14
- import {
15
- createEmitEvent,
16
- type EmitEvent,
17
- SYNC_MESSAGES_PENDING,
18
- } from "./emit.js";
19
- import type { SyncMessagesEvent } from "./events.js";
20
- import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
21
- import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
22
- import { workerQueuesOn } from "./test-helpers/worker-queues.js";
23
-
24
- const ACCOUNT = "account-1366";
25
- const FOLDERS = [
26
- { mailboxId: "mbx-inbox", fullPath: "INBOX" },
27
- { mailboxId: "mbx-sent", fullPath: "Sent" },
28
- { mailboxId: "mbx-archive", fullPath: "Archive" },
29
- ];
30
-
31
- let queue: RecordingQueue;
32
- let locks: MailboxLockRepo;
33
- let closeStore: () => void;
34
-
35
- type Queued = { mailboxId: string; eventId: string };
36
-
37
- const queued = (): Queued[] =>
38
- queue.sent.map(({ body }) => ({
39
- mailboxId: String(body.mailboxId),
40
- eventId: String(body.eventId),
41
- }));
42
-
43
- const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
44
- type: "SYNC_MESSAGES",
45
- accountId: ACCOUNT,
46
- mailboxId: "mbx-inbox",
47
- });
48
-
49
- const emitEvent: EmitEvent = (event) =>
50
- createEmitEvent(workerQueuesOn(queue))(event);
51
-
52
- const fanOut = (): Promise<void> =>
53
- emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
54
-
55
- beforeEach(() => {
56
- queue = recordingQueue();
57
- const store = createShippedSqliteDb();
58
- closeStore = store.close;
59
- locks = new MailboxLockRepo(store.db);
60
- setClient({ mailboxLock: locks } as unknown as RemitClient);
61
- });
62
-
63
- afterEach(() => {
64
- _resetForTest();
65
- closeStore();
66
- });
67
-
68
- describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
69
- it("queues one event per folder however many rounds arrive while none has started", async () => {
70
- for (let round = 0; round < 25; round += 1) await fanOut();
71
-
72
- assert.deepEqual(
73
- queued()
74
- .map((event) => event.mailboxId)
75
- .sort(),
76
- ["mbx-archive", "mbx-inbox", "mbx-sent"],
77
- );
78
- });
79
-
80
- it("queues a folder again once its pending sync has started", async () => {
81
- await fanOut();
82
- const inbox = queued().find((event) => event.mailboxId === "mbx-inbox");
83
- if (!inbox) throw new Error("INBOX was not queued");
84
-
85
- await locks.releaseLock(
86
- ACCOUNT,
87
- inbox.mailboxId,
88
- SYNC_MESSAGES_PENDING,
89
- inbox.eventId,
90
- );
91
- await fanOut();
92
-
93
- assert.equal(queued().length, FOLDERS.length + 1);
94
- assert.equal(queued().at(-1)?.mailboxId, "mbx-inbox");
95
- });
96
-
97
- it("keeps a folder's pending marker when a stale event starts", async () => {
98
- await fanOut();
99
- await locks.releaseLock(
100
- ACCOUNT,
101
- "mbx-inbox",
102
- SYNC_MESSAGES_PENDING,
103
- "an-event-from-before-the-marker",
104
- );
105
- await fanOut();
106
-
107
- assert.equal(queued().length, FOLDERS.length);
108
- });
109
-
110
- it("takes the same marker for a move's resync and a delete's resync", async () => {
111
- await emitMoveResync(emitEvent, {
112
- accountId: ACCOUNT,
113
- sourceMailboxId: "mbx-inbox",
114
- destinationMailboxId: "mbx-sent",
115
- });
116
- await emitMailboxResync(emitEvent, {
117
- accountId: ACCOUNT,
118
- mailboxId: "mbx-inbox",
119
- });
120
- await fanOut();
121
-
122
- assert.equal(queued().length, FOLDERS.length);
123
- });
124
-
125
- it("takes over a marker whose event was lost once it has expired", async () => {
126
- await locks.tryAcquireLock(
127
- "mbx-inbox",
128
- "SYNC_MESSAGES_PENDING",
129
- ACCOUNT,
130
- "lost-event",
131
- -1,
132
- );
133
-
134
- await emitEvent(inboxSync());
135
-
136
- assert.equal(queued().length, 1);
137
- });
138
-
139
- it("releases the marker when the send fails, so the next round is not swallowed", async () => {
140
- let sends = 0;
141
- queue.answer(async () => {
142
- sends += 1;
143
- if (sends === 1) throw new Error("queue unreachable");
144
- });
145
-
146
- await assert.rejects(emitEvent(inboxSync()), /queue unreachable/);
147
- await emitEvent(inboxSync());
148
-
149
- assert.equal(queued().length, 2);
150
- });
151
-
152
- it("fails the operation when the marker store fails", async () => {
153
- setClient({
154
- mailboxLock: {
155
- get: async () => null,
156
- tryAcquireLock: async () => {
157
- throw new Error("marker store down");
158
- },
159
- },
160
- } as unknown as RemitClient);
161
-
162
- await assert.rejects(emitEvent(inboxSync()), /marker store down/);
163
- assert.equal(queue.sent.length, 0);
164
- });
165
- });
package/src/poller.ts DELETED
@@ -1,55 +0,0 @@
1
- import { createLogger } from "@remit/logger";
2
- import { startMetricsServer } from "@remit/logger-lambda/metrics";
3
- import { installNodeLogger } from "@remit/logger-lambda/node";
4
- import { runQueuePoller } from "@remit/sqs-client/poller";
5
- import { env } from "expect-env";
6
- import { handler } from "./index.js";
7
-
8
- installNodeLogger();
9
-
10
- /**
11
- * Production queue poller — the deployed form of `e2e-processor-shim.ts`.
12
- * Polls every imap-worker queue and invokes the production Lambda handler.
13
- */
14
- const log = createLogger();
15
-
16
- // /metrics and nothing else, on the compose network (standalone-observability
17
- // D2). No health route on it: worker liveness is a heartbeat file, which keeps
18
- // answering when this server does not.
19
- startMetricsServer();
20
-
21
- await runQueuePoller({
22
- log,
23
- targets: [
24
- {
25
- queueUrl: env.SQS_QUEUE_URL_MAILBOXES,
26
- handler,
27
- functionName: "imap-worker-mailboxes",
28
- },
29
- {
30
- queueUrl: env.SQS_QUEUE_URL_MESSAGES,
31
- handler,
32
- functionName: "imap-worker-messages",
33
- },
34
- {
35
- queueUrl: env.SQS_QUEUE_URL_FLAGS,
36
- handler,
37
- functionName: "imap-worker-flags",
38
- },
39
- {
40
- queueUrl: env.SQS_QUEUE_URL_BODY,
41
- handler,
42
- functionName: "imap-worker-body",
43
- },
44
- {
45
- queueUrl: env.SQS_QUEUE_URL_MAILBOX_MGMT,
46
- handler,
47
- functionName: "imap-worker-mailbox-mgmt",
48
- },
49
- {
50
- queueUrl: env.SQS_QUEUE_URL_MESSAGE_MGMT,
51
- handler,
52
- functionName: "imap-worker-message-mgmt",
53
- },
54
- ],
55
- });
package/src/processor.ts DELETED
@@ -1,62 +0,0 @@
1
- import type { Logger } from "@remit/logger";
2
- import type { WorkerEvent } from "./events.js";
3
- import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
4
- import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
5
- import { handleEmptyTrash } from "./handlers/empty-trash.js";
6
- import { handleFlagPush } from "./handlers/flag-push.js";
7
- import { processMailboxManagement } from "./handlers/mailbox-management.js";
8
- import { handleMessageCopy } from "./handlers/message-copy.js";
9
- import { handleMessageDelete } from "./handlers/message-delete.js";
10
- import { handleMessageMove } from "./handlers/message-move.js";
11
- import { handlePlacementMovePush } from "./handlers/placement-move-push.js";
12
- import { syncMailboxes } from "./handlers/sync-mailboxes.js";
13
- import { syncMessageBody } from "./handlers/sync-message-body.js";
14
- import { syncMessages } from "./handlers/sync-messages.js";
15
- import type { WorkerQueues } from "./queues.js";
16
-
17
- export const processEvent = async (
18
- event: WorkerEvent,
19
- log: Logger,
20
- queues: WorkerQueues,
21
- /**
22
- * SQS's own delivery count for the record carrying this event (1 on first
23
- * delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH, FLAG_PUSH,
24
- * APPEND_SENT_MESSAGE, MESSAGE_MOVE, MESSAGE_DELETE and MESSAGE_COPY — each
25
- * knows from it when this is
26
- * the last attempt before the queue's own redrive policy would DLQ the
27
- * record, so it can resolve retry exhaustion into a terminal outcome
28
- * (issue #1270) instead of dead-lettering blindly.
29
- */
30
- receiveCount = 1,
31
- ): Promise<void> => {
32
- switch (event.type) {
33
- case "SYNC_MAILBOXES":
34
- return syncMailboxes(event, log, queues);
35
- case "SYNC_MESSAGES":
36
- return syncMessages(event, log, queues);
37
- case "SYNC_MESSAGE_BODY":
38
- return syncMessageBody(event, log, queues, receiveCount);
39
- case "MAILBOX_CREATE":
40
- case "MAILBOX_RENAME":
41
- case "MAILBOX_DELETE":
42
- return processMailboxManagement(event, log);
43
- case "MESSAGE_DELETE":
44
- return handleMessageDelete(event, log, queues, receiveCount);
45
- case "MESSAGE_MOVE":
46
- return handleMessageMove(event, log, queues, receiveCount);
47
- case "PLACEMENT_MOVE_PUSH":
48
- return handlePlacementMovePush(event, log, queues, receiveCount);
49
- case "FLAG_PUSH":
50
- return handleFlagPush(event, log, receiveCount);
51
- case "MESSAGE_COPY":
52
- return handleMessageCopy(event, log, queues, receiveCount);
53
- case "EMPTY_TRASH":
54
- return handleEmptyTrash(event, log);
55
- case "APPEND_SENT_MESSAGE":
56
- return handleAppendSentMessage(event, log, receiveCount);
57
- case "DELETE_ACCOUNT_OBJECTS":
58
- return handleDeleteAccountObjects(event, log, queues);
59
- default:
60
- throw new Error(`Unknown event type: ${(event as WorkerEvent).type}`);
61
- }
62
- };
package/src/queues.ts DELETED
@@ -1,40 +0,0 @@
1
- import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
2
- import {
3
- createFifoQueueProducer,
4
- createStandardQueueProducer,
5
- } from "@remit/sqs-client/producer";
6
- import { env } from "expect-env";
7
- import type { ImapEvent, WorkerEvent } from "./events.js";
8
-
9
- export interface WorkerQueues {
10
- mailboxes: FifoQueueProducer<ImapEvent>;
11
- messages: FifoQueueProducer<ImapEvent>;
12
- flags: FifoQueueProducer<ImapEvent>;
13
- body: QueueProducer<ImapEvent>;
14
- mailboxManagement: QueueProducer<ImapEvent>;
15
- messageManagement: QueueProducer<WorkerEvent>;
16
- }
17
-
18
- export type WorkerQueueUrls = Record<keyof WorkerQueues, string>;
19
-
20
- export const workerQueueUrlsFromEnv = (): WorkerQueueUrls => ({
21
- mailboxes: env.SQS_QUEUE_URL_MAILBOXES,
22
- messages: env.SQS_QUEUE_URL_MESSAGES,
23
- flags: env.SQS_QUEUE_URL_FLAGS,
24
- body: env.SQS_QUEUE_URL_BODY,
25
- mailboxManagement: env.SQS_QUEUE_URL_MAILBOX_MGMT,
26
- messageManagement: env.SQS_QUEUE_URL_MESSAGE_MGMT,
27
- });
28
-
29
- export const sqsWorkerQueues = (urls: WorkerQueueUrls): WorkerQueues => ({
30
- mailboxes: createFifoQueueProducer({ queueUrl: urls.mailboxes }),
31
- messages: createFifoQueueProducer({ queueUrl: urls.messages }),
32
- flags: createFifoQueueProducer({ queueUrl: urls.flags }),
33
- body: createStandardQueueProducer({ queueUrl: urls.body }),
34
- mailboxManagement: createStandardQueueProducer({
35
- queueUrl: urls.mailboxManagement,
36
- }),
37
- messageManagement: createStandardQueueProducer({
38
- queueUrl: urls.messageManagement,
39
- }),
40
- });
@@ -1,74 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { randomUUID } from "node:crypto";
3
- import { after, before, describe, it } from "node:test";
4
- import { openStore, type World } from "../test-helpers/test-world.js";
5
- import { buildAttachmentSweep } from "./attachment-sweep.js";
6
-
7
- let store: Awaited<ReturnType<typeof openStore>>;
8
-
9
- before(async () => {
10
- store = await openStore();
11
- });
12
-
13
- after(() => store.close());
14
-
15
- /**
16
- * A draft holding one uploaded file, and one object under the same draft that
17
- * no attachment row names — what a browser leaves behind when it uploads and
18
- * the reservation lapses before it is completed.
19
- */
20
- const arrange = async (w: World) => {
21
- const { outbox, attachmentId } = await w.draft(Buffer.from("q1,q2\n1,2\n"));
22
- await w.client.storage.storeOutboxAttachment({
23
- accountConfigId: w.account.accountConfigId,
24
- accountId: w.account.accountId,
25
- outboxMessageId: outbox.outboxMessageId,
26
- outboxAttachmentId: `orphan-${randomUUID()}`,
27
- content: Buffer.from("left behind"),
28
- });
29
- return { outboxMessageId: outbox.outboxMessageId, attachmentId };
30
- };
31
-
32
- const storedIds = async (w: World, outboxMessageId: string) =>
33
- (
34
- await w.client.storage.listOutboxAttachments(
35
- w.account.accountConfigId,
36
- w.account.accountId,
37
- outboxMessageId,
38
- )
39
- ).map((item) => item.outboxAttachmentId);
40
-
41
- describe("the outbox attachment sweep", () => {
42
- it("removes the objects no attachment row names and keeps the ones that are", async () => {
43
- const w = await store.world();
44
- const { outboxMessageId, attachmentId } = await arrange(w);
45
-
46
- await buildAttachmentSweep(w.client, w.log)(w.account);
47
-
48
- assert.deepEqual(await storedIds(w, outboxMessageId), [attachmentId]);
49
- });
50
-
51
- it("runs for every account on the hosted tick, due a sync or not", async () => {
52
- const first = await store.world();
53
- const firstDraft = await arrange(first);
54
- await first.client.account.update(first.account.accountId, {
55
- lastSyncAt: Date.now(),
56
- });
57
- const second = await store.world();
58
- const secondDraft = await arrange(second);
59
-
60
- const { buildScheduledHandler } = await import("./handler.js");
61
- await buildScheduledHandler(second.queues)(
62
- { time: new Date().toISOString() } as never,
63
- {} as never,
64
- () => {},
65
- );
66
-
67
- assert.deepEqual(await storedIds(first, firstDraft.outboxMessageId), [
68
- firstDraft.attachmentId,
69
- ]);
70
- assert.deepEqual(await storedIds(second, secondDraft.outboxMessageId), [
71
- secondDraft.attachmentId,
72
- ]);
73
- });
74
- });
@@ -1,46 +0,0 @@
1
- import type { RemitClient } from "@remit/backend/client";
2
- import type { AccountItem } from "@remit/data-ports";
3
- import type { Logger } from "@remit/logger";
4
- import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
5
- /**
6
- * Collect attachment bytes the database does not know about. Both scheduler
7
- * entry points build it the same way, so the hosted tick and the self-host loop
8
- * collect identically — this is the only defence on the hosted side, where the
9
- * browser PUTs straight to block storage and nothing consults the database on
10
- * the way in.
11
- */
12
- export const buildAttachmentSweep =
13
- (
14
- client: Pick<RemitClient, "storage" | "outboxAttachment">,
15
- log: Logger,
16
- ): ((account: AccountItem) => Promise<void>) =>
17
- async (account) => {
18
- const { deleted, skipped } = await sweepAbandonedOutboxAttachments(
19
- {
20
- storage: client.storage,
21
- // Reaps the draft's lapsed reservations first, so a row that has
22
- // stopped holding room stops vouching for its bytes too and the
23
- // object is collected on this same pass.
24
- listKnownAttachmentIds: (accountConfigId, outboxMessageId) =>
25
- client.outboxAttachment.reapAndListLive(
26
- accountConfigId,
27
- outboxMessageId,
28
- ),
29
- onSkipped: (outboxMessageId, error) => {
30
- log.error(
31
- { outboxMessageId, error },
32
- "Left a draft's attachment objects alone: its rows could not be read",
33
- );
34
- },
35
- },
36
- account.accountConfigId,
37
- account.accountId,
38
- );
39
-
40
- if (deleted > 0 || skipped > 0) {
41
- log.warn(
42
- { accountId: account.accountId, deleted, skipped },
43
- "Outbox attachment sweep",
44
- );
45
- }
46
- };