@remit/imap-worker 0.0.89 → 0.0.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/dist/account-check.d.ts +27 -0
  2. package/dist/account-check.js +70 -0
  3. package/dist/body-sync-gate.d.ts +12 -0
  4. package/dist/body-sync-gate.js +53 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +193 -0
  7. package/dist/config-env.d.ts +5 -0
  8. package/dist/config-env.js +37 -0
  9. package/dist/connection-pool.d.ts +22 -0
  10. package/dist/connection-pool.js +97 -0
  11. package/dist/connection-scope.d.ts +8 -0
  12. package/dist/connection-scope.js +10 -0
  13. package/dist/context.d.ts +46 -0
  14. package/dist/context.js +15 -0
  15. package/dist/e2e-processor-shim.d.ts +2 -0
  16. package/dist/e2e-processor-shim.js +181 -0
  17. package/dist/emit.d.ts +15 -0
  18. package/dist/emit.js +76 -0
  19. package/dist/events.d.ts +203 -0
  20. package/dist/events.js +1 -0
  21. package/dist/failure-kind.d.ts +10 -0
  22. package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
  23. package/dist/handlers/append-sent-message.d.ts +3 -0
  24. package/dist/handlers/append-sent-message.js +152 -0
  25. package/dist/handlers/delete-account-objects.d.ts +3 -0
  26. package/dist/handlers/delete-account-objects.js +35 -0
  27. package/dist/handlers/empty-trash.d.ts +14 -0
  28. package/dist/handlers/empty-trash.js +217 -0
  29. package/dist/handlers/flag-push.d.ts +38 -0
  30. package/dist/handlers/flag-push.js +195 -0
  31. package/dist/handlers/imported-folder.d.ts +4 -0
  32. package/dist/handlers/imported-folder.js +23 -0
  33. package/dist/handlers/mailbox-management.d.ts +6 -0
  34. package/dist/handlers/mailbox-management.js +294 -0
  35. package/dist/handlers/mailbox-sync-order.d.ts +21 -0
  36. package/dist/handlers/mailbox-sync-order.js +50 -0
  37. package/dist/handlers/message-copy.d.ts +40 -0
  38. package/dist/handlers/message-copy.js +371 -0
  39. package/dist/handlers/message-delete-terminal.d.ts +60 -0
  40. package/dist/handlers/message-delete-terminal.js +77 -0
  41. package/dist/handlers/message-delete.d.ts +57 -0
  42. package/dist/handlers/message-delete.js +520 -0
  43. package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
  44. package/dist/handlers/message-move-terminal.js +94 -0
  45. package/dist/handlers/message-move.d.ts +146 -0
  46. package/dist/handlers/message-move.js +490 -0
  47. package/dist/handlers/placement-move-push.d.ts +62 -0
  48. package/dist/handlers/placement-move-push.js +307 -0
  49. package/dist/handlers/probe-message-id.d.ts +2 -0
  50. package/dist/handlers/probe-message-id.js +6 -0
  51. package/dist/handlers/sync-mailboxes.d.ts +45 -0
  52. package/dist/handlers/sync-mailboxes.js +233 -0
  53. package/dist/handlers/sync-message-body.d.ts +23 -0
  54. package/dist/handlers/sync-message-body.js +182 -0
  55. package/dist/handlers/sync-messages.d.ts +29 -0
  56. package/dist/handlers/sync-messages.js +285 -0
  57. package/dist/index.d.ts +9 -0
  58. package/dist/index.js +67 -0
  59. package/dist/lambda-context.d.ts +8 -0
  60. package/dist/lambda-context.js +33 -0
  61. package/dist/mailbox-row.d.ts +9 -0
  62. package/dist/mailbox-row.js +6 -0
  63. package/dist/poller.d.ts +1 -0
  64. package/dist/poller.js +51 -0
  65. package/dist/process.d.ts +6 -0
  66. package/dist/process.js +3 -0
  67. package/dist/processor.d.ts +3 -0
  68. package/dist/processor.js +44 -0
  69. package/dist/queues.d.ts +10 -0
  70. package/dist/queues.js +1 -0
  71. package/dist/s3-account-objects.d.ts +3 -0
  72. package/dist/s3-account-objects.js +26 -0
  73. package/dist/scheduler/attachment-sweep.d.ts +11 -0
  74. package/dist/scheduler/attachment-sweep.js +23 -0
  75. package/dist/scheduler/calendar-subscriptions.d.ts +26 -0
  76. package/dist/scheduler/calendar-subscriptions.js +35 -0
  77. package/dist/scheduler/config.d.ts +27 -0
  78. package/{src/scheduler/config.ts → dist/scheduler/config.js} +7 -26
  79. package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
  80. package/dist/scheduler/decide-due.js +17 -0
  81. package/dist/scheduler/handler.d.ts +21 -0
  82. package/{src/scheduler/handler.ts → dist/scheduler/handler.js} +24 -36
  83. package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
  84. package/dist/scheduler/loop.js +8 -0
  85. package/dist/scheduler/run-tick.d.ts +62 -0
  86. package/dist/scheduler/run-tick.js +164 -0
  87. package/dist/scheduler/runner.d.ts +2 -0
  88. package/{src/scheduler/runner.ts → dist/scheduler/runner.js} +58 -84
  89. package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
  90. package/dist/scheduler/stranded-placement-redrive.js +172 -0
  91. package/dist/sqs-queues.d.ts +4 -0
  92. package/dist/sqs-queues.js +22 -0
  93. package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
  94. package/dist/with-oauth-lifecycle-deps.js +17 -0
  95. package/dist/with-oauth-lifecycle.d.ts +56 -0
  96. package/dist/with-oauth-lifecycle.js +92 -0
  97. package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
  98. package/dist/worker-version.js +12 -0
  99. package/package.json +75 -69
  100. package/src/account-check.test.ts +0 -149
  101. package/src/account-check.ts +0 -101
  102. package/src/body-sync-gate.test.ts +0 -185
  103. package/src/body-sync-gate.ts +0 -86
  104. package/src/cli.ts +0 -224
  105. package/src/connection-scope.test.ts +0 -266
  106. package/src/connection-scope.ts +0 -314
  107. package/src/credential-less-sync-fence.test.ts +0 -152
  108. package/src/e2e-processor-shim.ts +0 -232
  109. package/src/emit.test.ts +0 -127
  110. package/src/emit.ts +0 -117
  111. package/src/events.ts +0 -252
  112. package/src/failure-kind.test.ts +0 -42
  113. package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
  114. package/src/handlers/append-sent-message.ts +0 -264
  115. package/src/handlers/delete-account-objects.test.ts +0 -128
  116. package/src/handlers/delete-account-objects.ts +0 -94
  117. package/src/handlers/empty-trash.sqlite.test.ts +0 -209
  118. package/src/handlers/empty-trash.ts +0 -352
  119. package/src/handlers/flag-push.ts +0 -285
  120. package/src/handlers/imported-folder.test.ts +0 -63
  121. package/src/handlers/imported-folder.ts +0 -38
  122. package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
  123. package/src/handlers/mailbox-management.ts +0 -486
  124. package/src/handlers/mailbox-sync-order.test.ts +0 -93
  125. package/src/handlers/mailbox-sync-order.ts +0 -65
  126. package/src/handlers/message-copy.sqlite.test.ts +0 -298
  127. package/src/handlers/message-copy.ts +0 -566
  128. package/src/handlers/message-delete-terminal.ts +0 -147
  129. package/src/handlers/message-delete.sqlite.test.ts +0 -291
  130. package/src/handlers/message-delete.ts +0 -885
  131. package/src/handlers/message-move.sqlite.test.ts +0 -271
  132. package/src/handlers/message-move.test.ts +0 -216
  133. package/src/handlers/message-move.ts +0 -752
  134. package/src/handlers/placement-move-push.test.ts +0 -137
  135. package/src/handlers/placement-move-push.ts +0 -475
  136. package/src/handlers/placement-push.sqlite.test.ts +0 -298
  137. package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
  138. package/src/handlers/probe-message-id.ts +0 -15
  139. package/src/handlers/sync-mailboxes.test.ts +0 -183
  140. package/src/handlers/sync-mailboxes.ts +0 -421
  141. package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
  142. package/src/handlers/sync-message-body.test.ts +0 -150
  143. package/src/handlers/sync-message-body.ts +0 -365
  144. package/src/handlers/sync-messages.sqlite.test.ts +0 -183
  145. package/src/handlers/sync-messages.ts +0 -525
  146. package/src/index.test.ts +0 -22
  147. package/src/index.ts +0 -86
  148. package/src/mail-sync-selection.sqlite.test.ts +0 -234
  149. package/src/mailbox-row.ts +0 -20
  150. package/src/pending-sync.sqlite.test.ts +0 -166
  151. package/src/poller.ts +0 -55
  152. package/src/processor.ts +0 -60
  153. package/src/queues.ts +0 -49
  154. package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -75
  155. package/src/scheduler/attachment-sweep.ts +0 -46
  156. package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
  157. package/src/scheduler/calendar-subscriptions.ts +0 -103
  158. package/src/scheduler/config.test.ts +0 -40
  159. package/src/scheduler/decide-due.test.ts +0 -44
  160. package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
  161. package/src/scheduler/loop.test.ts +0 -143
  162. package/src/scheduler/run-tick.sqlite.test.ts +0 -191
  163. package/src/scheduler/run-tick.ts +0 -281
  164. package/src/scheduler/stranded-placement-redrive.ts +0 -265
  165. package/src/stranded-placement.sqlite.test.ts +0 -794
  166. package/src/test-helpers/test-world.ts +0 -427
  167. package/src/thread-join.sqlite.test.ts +0 -106
  168. package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
  169. package/src/with-oauth-lifecycle-deps.ts +0 -71
  170. package/src/with-oauth-lifecycle.ts +0 -158
  171. package/tsconfig.json +0 -8
package/dist/poller.js ADDED
@@ -0,0 +1,51 @@
1
+ import { createLogger } from "@remit/logger";
2
+ import { startMetricsServer } from "@remit/logger-lambda/metrics";
3
+ import { installNodeLogger } from "@remit/logger-lambda/node";
4
+ import { runQueuePoller } from "@remit/sqs-client/poller";
5
+ import { env } from "expect-env";
6
+ import { handler } from "./index.js";
7
+ installNodeLogger();
8
+ /**
9
+ * Production queue poller — the deployed form of `e2e-processor-shim.ts`.
10
+ * Polls every imap-worker queue and invokes the production Lambda handler.
11
+ */
12
+ const log = createLogger();
13
+ // /metrics and nothing else, on the compose network (standalone-observability
14
+ // D2). No health route on it: worker liveness is a heartbeat file, which keeps
15
+ // answering when this server does not.
16
+ startMetricsServer();
17
+ await runQueuePoller({
18
+ log,
19
+ targets: [
20
+ {
21
+ queueUrl: env.SQS_QUEUE_URL_MAILBOXES,
22
+ handler,
23
+ functionName: "imap-worker-mailboxes",
24
+ },
25
+ {
26
+ queueUrl: env.SQS_QUEUE_URL_MESSAGES,
27
+ handler,
28
+ functionName: "imap-worker-messages",
29
+ },
30
+ {
31
+ queueUrl: env.SQS_QUEUE_URL_FLAGS,
32
+ handler,
33
+ functionName: "imap-worker-flags",
34
+ },
35
+ {
36
+ queueUrl: env.SQS_QUEUE_URL_BODY,
37
+ handler,
38
+ functionName: "imap-worker-body",
39
+ },
40
+ {
41
+ queueUrl: env.SQS_QUEUE_URL_MAILBOX_MGMT,
42
+ handler,
43
+ functionName: "imap-worker-mailbox-mgmt",
44
+ },
45
+ {
46
+ queueUrl: env.SQS_QUEUE_URL_MESSAGE_MGMT,
47
+ handler,
48
+ functionName: "imap-worker-message-mgmt",
49
+ },
50
+ ],
51
+ });
@@ -0,0 +1,6 @@
1
+ export type { ConnectionScope } from "@remit/mailbox-service";
2
+ export { type BorrowedConnection, type ConnectImap, type ConnectionPool, type ConnectionPoolOptions, createConnectionPool, type PoolAccount, type PoolOverflow, } from "./connection-pool.js";
3
+ export { type AccountObjectPage, type AccountObjectStore, type AttemptBudgets, DEFAULT_MAILBOX_FRESHNESS_MS, defaultImapWorkerConfig, type ImapFailureKind, type ImapWorkerConfig, type ImapWorkerQueueName, type ProcessContext, type TokenService, type WorkerMetrics, } from "./context.js";
4
+ export type { ImapEvent, WorkerEvent } from "./events.js";
5
+ export { processEvent } from "./processor.js";
6
+ export type { WorkerQueues } from "./queues.js";
@@ -0,0 +1,3 @@
1
+ export { createConnectionPool, } from "./connection-pool.js";
2
+ export { DEFAULT_MAILBOX_FRESHNESS_MS, defaultImapWorkerConfig, } from "./context.js";
3
+ export { processEvent } from "./processor.js";
@@ -0,0 +1,3 @@
1
+ import type { ProcessContext } from "./context.js";
2
+ import type { WorkerEvent } from "./events.js";
3
+ export declare const processEvent: (event: WorkerEvent, ctx: ProcessContext, receiveCount: number) => Promise<void>;
@@ -0,0 +1,44 @@
1
+ import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
2
+ import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
3
+ import { handleEmptyTrash } from "./handlers/empty-trash.js";
4
+ import { handleFlagPush } from "./handlers/flag-push.js";
5
+ import { processMailboxManagement } from "./handlers/mailbox-management.js";
6
+ import { handleMessageCopy } from "./handlers/message-copy.js";
7
+ import { handleMessageDelete } from "./handlers/message-delete.js";
8
+ import { handleMessageMove } from "./handlers/message-move.js";
9
+ import { handlePlacementMovePush } from "./handlers/placement-move-push.js";
10
+ import { syncMailboxes } from "./handlers/sync-mailboxes.js";
11
+ import { syncMessageBody } from "./handlers/sync-message-body.js";
12
+ import { syncMessages } from "./handlers/sync-messages.js";
13
+ export const processEvent = async (event, ctx, receiveCount) => {
14
+ switch (event.type) {
15
+ case "SYNC_MAILBOXES":
16
+ return syncMailboxes(event, ctx);
17
+ case "SYNC_MESSAGES":
18
+ return syncMessages(event, ctx);
19
+ case "SYNC_MESSAGE_BODY":
20
+ return syncMessageBody(event, ctx, receiveCount);
21
+ case "MAILBOX_CREATE":
22
+ case "MAILBOX_RENAME":
23
+ case "MAILBOX_DELETE":
24
+ return processMailboxManagement(event, ctx);
25
+ case "MESSAGE_DELETE":
26
+ return handleMessageDelete(event, ctx, receiveCount);
27
+ case "MESSAGE_MOVE":
28
+ return handleMessageMove(event, ctx, receiveCount);
29
+ case "PLACEMENT_MOVE_PUSH":
30
+ return handlePlacementMovePush(event, ctx, receiveCount);
31
+ case "FLAG_PUSH":
32
+ return handleFlagPush(event, ctx, receiveCount);
33
+ case "MESSAGE_COPY":
34
+ return handleMessageCopy(event, ctx, receiveCount);
35
+ case "EMPTY_TRASH":
36
+ return handleEmptyTrash(event, ctx);
37
+ case "APPEND_SENT_MESSAGE":
38
+ return handleAppendSentMessage(event, ctx, receiveCount);
39
+ case "DELETE_ACCOUNT_OBJECTS":
40
+ return handleDeleteAccountObjects(event, ctx);
41
+ default:
42
+ throw new Error(`Unknown event type: ${event.type}`);
43
+ }
44
+ };
@@ -0,0 +1,10 @@
1
+ import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
2
+ import type { ImapEvent, WorkerEvent } from "./events.js";
3
+ export interface WorkerQueues {
4
+ mailboxes: FifoQueueProducer<ImapEvent>;
5
+ messages: FifoQueueProducer<ImapEvent>;
6
+ flags: FifoQueueProducer<ImapEvent>;
7
+ body: QueueProducer<ImapEvent>;
8
+ mailboxManagement: QueueProducer<ImapEvent>;
9
+ messageManagement: QueueProducer<WorkerEvent>;
10
+ }
package/dist/queues.js ADDED
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,3 @@
1
+ import { S3Client } from "@aws-sdk/client-s3";
2
+ import type { AccountObjectStore } from "./context.js";
3
+ export declare const s3AccountObjects: (bucket: () => string, s3?: S3Client) => AccountObjectStore;
@@ -0,0 +1,26 @@
1
+ import { DeleteObjectsCommand, ListObjectsV2Command, S3Client, } from "@aws-sdk/client-s3";
2
+ const PAGE_SIZE = 1_000;
3
+ export const s3AccountObjects = (bucket, s3 = new S3Client({})) => ({
4
+ list: async (prefix, continuationToken) => {
5
+ const result = await s3.send(new ListObjectsV2Command({
6
+ Bucket: bucket(),
7
+ Prefix: prefix,
8
+ MaxKeys: PAGE_SIZE,
9
+ ContinuationToken: continuationToken,
10
+ }));
11
+ return {
12
+ keys: (result.Contents ?? [])
13
+ .map((object) => object.Key)
14
+ .filter((key) => key !== undefined),
15
+ nextContinuationToken: result.IsTruncated
16
+ ? result.NextContinuationToken
17
+ : undefined,
18
+ };
19
+ },
20
+ delete: async (keys) => {
21
+ await s3.send(new DeleteObjectsCommand({
22
+ Bucket: bucket(),
23
+ Delete: { Objects: keys.map((Key) => ({ Key })), Quiet: true },
24
+ }));
25
+ },
26
+ });
@@ -0,0 +1,11 @@
1
+ import type { RemitClient } from "@remit/backend/client";
2
+ import type { AccountItem } from "@remit/data-ports";
3
+ import type { Logger } from "@remit/logger";
4
+ /**
5
+ * Collect attachment bytes the database does not know about. Both scheduler
6
+ * entry points build it the same way, so the hosted tick and the self-host loop
7
+ * collect identically — this is the only defence on the hosted side, where the
8
+ * browser PUTs straight to block storage and nothing consults the database on
9
+ * the way in.
10
+ */
11
+ export declare const buildAttachmentSweep: (client: Pick<RemitClient, "storage" | "outboxAttachment">, log: Logger) => ((account: AccountItem) => Promise<void>);
@@ -0,0 +1,23 @@
1
+ import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
2
+ /**
3
+ * Collect attachment bytes the database does not know about. Both scheduler
4
+ * entry points build it the same way, so the hosted tick and the self-host loop
5
+ * collect identically — this is the only defence on the hosted side, where the
6
+ * browser PUTs straight to block storage and nothing consults the database on
7
+ * the way in.
8
+ */
9
+ export const buildAttachmentSweep = (client, log) => async (account) => {
10
+ const { deleted, skipped } = await sweepAbandonedOutboxAttachments({
11
+ storage: client.storage,
12
+ // Reaps the draft's lapsed reservations first, so a row that has
13
+ // stopped holding room stops vouching for its bytes too and the
14
+ // object is collected on this same pass.
15
+ listKnownAttachmentIds: (accountConfigId, outboxMessageId) => client.outboxAttachment.reapAndListLive(accountConfigId, outboxMessageId),
16
+ onSkipped: (outboxMessageId, error) => {
17
+ log.error({ outboxMessageId, error }, "Left a draft's attachment objects alone: its rows could not be read");
18
+ },
19
+ }, account.accountConfigId, account.accountId);
20
+ if (deleted > 0 || skipped > 0) {
21
+ log.warn({ accountId: account.accountId, deleted, skipped }, "Outbox attachment sweep");
22
+ }
23
+ };
@@ -0,0 +1,26 @@
1
+ import type { RemitClient } from "@remit/backend/client";
2
+ import { type CalendarFeedFetcher } from "@remit/calendar-service";
3
+ import type { ICalendarCollectionRepository, ICalendarStore } from "@remit/data-ports";
4
+ import type { Logger } from "@remit/logger";
5
+ export interface CalendarSubscriptionTally {
6
+ refreshed: number;
7
+ failed: number;
8
+ notDue: number;
9
+ }
10
+ export interface CalendarSubscriptionRefreshDeps {
11
+ calendarCollection: Pick<ICalendarCollectionRepository, "listEnabledSubscriptions">;
12
+ calendarStore: ICalendarStore;
13
+ log: Logger;
14
+ fetcher?: CalendarFeedFetcher;
15
+ }
16
+ /**
17
+ * Re-reads every enabled subscription whose last fetch is at least
18
+ * `intervalMs` old. A disabled subscription is never listed, which is what
19
+ * stops its refresh while leaving its events in place.
20
+ *
21
+ * One feed's failure is recorded on its own collection and counted here; it
22
+ * never stops the others. The log names the calendar and the reason, never the
23
+ * URL, which for a Google secret address is the whole credential.
24
+ */
25
+ export declare const refreshDueCalendarSubscriptions: (deps: CalendarSubscriptionRefreshDeps, now: number, intervalMs: number) => Promise<CalendarSubscriptionTally>;
26
+ export declare const buildCalendarSubscriptionRefresh: (client: Pick<RemitClient, "calendarCollection" | "calendarStore">, log: Logger) => ((now: number, intervalMs: number) => Promise<CalendarSubscriptionTally>);
@@ -0,0 +1,35 @@
1
+ import { isSubscriptionDue, refreshCalendarSubscription, } from "@remit/calendar-service";
2
+ import pMap from "p-map";
3
+ const REFRESH_CONCURRENCY = 4;
4
+ /**
5
+ * Re-reads every enabled subscription whose last fetch is at least
6
+ * `intervalMs` old. A disabled subscription is never listed, which is what
7
+ * stops its refresh while leaving its events in place.
8
+ *
9
+ * One feed's failure is recorded on its own collection and counted here; it
10
+ * never stops the others. The log names the calendar and the reason, never the
11
+ * URL, which for a Google secret address is the whole credential.
12
+ */
13
+ export const refreshDueCalendarSubscriptions = async (deps, now, intervalMs) => {
14
+ const enabled = await deps.calendarCollection.listEnabledSubscriptions();
15
+ const due = enabled.filter((collection) => isSubscriptionDue(collection, now, intervalMs));
16
+ const outcomes = await pMap(due, (collection) => refreshCalendarSubscription(deps.calendarStore, collection, now, deps.fetcher).then((refresh) => {
17
+ if (refresh.ok)
18
+ return true;
19
+ deps.log.warn({ calendarId: collection.calendarId, reason: refresh.reason }, "Calendar subscription could not be refreshed");
20
+ return false;
21
+ }, (error) => {
22
+ deps.log.error({ calendarId: collection.calendarId, error }, "Calendar subscription refresh failed");
23
+ return false;
24
+ }), { concurrency: REFRESH_CONCURRENCY });
25
+ return {
26
+ refreshed: outcomes.filter(Boolean).length,
27
+ failed: outcomes.filter((ok) => !ok).length,
28
+ notDue: enabled.length - due.length,
29
+ };
30
+ };
31
+ export const buildCalendarSubscriptionRefresh = (client, log) => (now, intervalMs) => refreshDueCalendarSubscriptions({
32
+ calendarCollection: client.calendarCollection,
33
+ calendarStore: client.calendarStore,
34
+ log,
35
+ }, now, intervalMs);
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Periodic mailbox-sync scheduler (#1247, restructured #1251): the tick rate
3
+ * and the offline-sync threshold are separate, independently configured
4
+ * knobs — never hardcode a cadence — following the repo's
5
+ * env-var-with-a-safe-default convention (see `METRICS_PORT` in
6
+ * remit-logger-lambda/src/metrics.ts).
7
+ *
8
+ * `tickIntervalSeconds` drives how often the tick itself runs — the
9
+ * EventBridge schedule rate on a managed deployment, the runner's loop delay
10
+ * everywhere else — and must stay well below `offlineIntervalSeconds` so a tick
11
+ * reliably observes every account crossing the threshold. CDK and this runtime
12
+ * default must agree — see infra/lib/config.ts's `mailboxSync` stage config.
13
+ *
14
+ * `offlineIntervalSeconds` is the only due-ness threshold: an account is due
15
+ * once its last successful sync is older than this interval. There is no
16
+ * "online" tier — client-side polling (useStaleAccountSync) covers an
17
+ * account while its mail is actively open in the web client.
18
+ *
19
+ * The defaults below are a floor for a deployment that sets neither. The
20
+ * standalone stack sets both, at a cadence sized for one box and a handful of
21
+ * accounts, and the checker's stall threshold is derived from those values —
22
+ * see deploy/vps/docker-compose.sqlite.yml's `scheduler` service.
23
+ */
24
+ export declare const getTickIntervalMs: (env?: NodeJS.ProcessEnv) => number;
25
+ export declare const getOfflineIntervalMs: (env?: NodeJS.ProcessEnv) => number;
26
+ export declare const SCHEDULER_PAGE_SIZE = 100;
27
+ export declare const SCHEDULER_ENQUEUE_CONCURRENCY = 10;
@@ -21,35 +21,16 @@
21
21
  * accounts, and the checker's stall threshold is derived from those values —
22
22
  * see deploy/vps/docker-compose.sqlite.yml's `scheduler` service.
23
23
  */
24
-
25
24
  const DEFAULT_TICK_INTERVAL_SECONDS = 60 * 60; // 1 hour
26
25
  const DEFAULT_OFFLINE_INTERVAL_SECONDS = 12 * 60 * 60; // 12 hours
27
-
28
- const parsePositiveIntSeconds = (
29
- raw: string | undefined,
30
- fallback: number,
31
- ): number => {
32
- if (!raw) return fallback;
33
- const parsed = Number.parseInt(raw, 10);
34
- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
26
+ const parsePositiveIntSeconds = (raw, fallback) => {
27
+ if (!raw)
28
+ return fallback;
29
+ const parsed = Number.parseInt(raw, 10);
30
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
35
31
  };
36
-
37
- export const getTickIntervalMs = (
38
- env: NodeJS.ProcessEnv = process.env,
39
- ): number =>
40
- parsePositiveIntSeconds(
41
- env.MAILBOX_SYNC_TICK_INTERVAL_SECONDS,
42
- DEFAULT_TICK_INTERVAL_SECONDS,
43
- ) * 1000;
44
-
45
- export const getOfflineIntervalMs = (
46
- env: NodeJS.ProcessEnv = process.env,
47
- ): number =>
48
- parsePositiveIntSeconds(
49
- env.MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS,
50
- DEFAULT_OFFLINE_INTERVAL_SECONDS,
51
- ) * 1000;
52
-
32
+ export const getTickIntervalMs = (env = process.env) => parsePositiveIntSeconds(env.MAILBOX_SYNC_TICK_INTERVAL_SECONDS, DEFAULT_TICK_INTERVAL_SECONDS) * 1000;
33
+ export const getOfflineIntervalMs = (env = process.env) => parsePositiveIntSeconds(env.MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS, DEFAULT_OFFLINE_INTERVAL_SECONDS) * 1000;
53
34
  // Page size + enqueue concurrency are implementation details of the tick
54
35
  // itself, not stage config — unlike the two intervals above, nothing outside
55
36
  // this worker needs to agree with them.
@@ -10,17 +10,8 @@
10
10
  * misses an account by a few seconds simply catches it on the next tick,
11
11
  * which arrives long before the threshold matters again.
12
12
  */
13
-
14
13
  export interface SchedulableAccount {
15
- readonly accountId: string;
16
- readonly lastSyncAt?: number;
14
+ readonly accountId: string;
15
+ readonly lastSyncAt?: number;
17
16
  }
18
-
19
- export const isSyncDue = (
20
- account: SchedulableAccount,
21
- now: number,
22
- offlineIntervalMs: number,
23
- ): boolean => {
24
- if (!account.lastSyncAt) return true;
25
- return now - account.lastSyncAt >= offlineIntervalMs;
26
- };
17
+ export declare const isSyncDue: (account: SchedulableAccount, now: number, offlineIntervalMs: number) => boolean;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Pure scheduling decision for the periodic mailbox-sync tick (#1247,
3
+ * restructured #1251).
4
+ *
5
+ * One tier: an account is due once its last successful sync is older than
6
+ * `offlineIntervalMs`. An account that has never synced is always due. The
7
+ * tick interval is decoupled from this threshold and runs far more often
8
+ * than `offlineIntervalMs` (see config.ts), so — unlike the prior two-tier
9
+ * design — no slack is needed to compensate for sampling lag: a tick that
10
+ * misses an account by a few seconds simply catches it on the next tick,
11
+ * which arrives long before the threshold matters again.
12
+ */
13
+ export const isSyncDue = (account, now, offlineIntervalMs) => {
14
+ if (!account.lastSyncAt)
15
+ return true;
16
+ return now - account.lastSyncAt >= offlineIntervalMs;
17
+ };
@@ -0,0 +1,21 @@
1
+ import type { ScheduledHandler } from "aws-lambda";
2
+ import type { WorkerQueues } from "../queues.js";
3
+ /**
4
+ * EventBridge-scheduled entry point for the periodic mailbox-sync tick
5
+ * (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
6
+ * (rate schedule, wired in infra/stacks/dev/stacks/remit-worker-stack.ts) and
7
+ * delegates the actual decision + enqueue to `runSchedulerTick` — the same
8
+ * function the timer loop calls on a deployment with no EventBridge (see
9
+ * `runner.ts`), so every deployment runs one code path.
10
+ *
11
+ * Uses the EventBridge event's own `time` (the scheduled fire time) as the
12
+ * tick's `now`, rather than `Date.now()` at processing time. This is what
13
+ * keeps `buildScheduledSyncDedupId`'s time bucket aligned to the schedule
14
+ * instead of wall-clock/Lambda-cold-start jitter (review #1250): every
15
+ * distinct scheduled firing gets a `time` exactly `tickIntervalMs` apart
16
+ * from the last, so consecutive ticks always land in different buckets,
17
+ * while a genuine retry of the same invocation carries the same `time` and
18
+ * correctly dedupes.
19
+ */
20
+ export declare const buildScheduledHandler: (queues: WorkerQueues) => ScheduledHandler;
21
+ export declare const handler: ScheduledHandler;
@@ -1,26 +1,15 @@
1
1
  import { getClient } from "@remit/backend/client";
2
2
  import { createLogger } from "@remit/logger";
3
3
  import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
4
- import type { ScheduledHandler } from "aws-lambda";
5
- import { emitEvent } from "../emit.js";
6
- import {
7
- setWorkerQueues,
8
- sqsWorkerQueues,
9
- workerQueues,
10
- workerQueueUrlsFromEnv,
11
- } from "../queues.js";
4
+ import { lambdaEmitEvent } from "../lambda-context.js";
5
+ import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "../sqs-queues.js";
12
6
  import { buildAttachmentSweep } from "./attachment-sweep.js";
13
7
  import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
14
8
  import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
15
9
  import { runSchedulerTick } from "./run-tick.js";
16
10
  import { redriveStrandedPlacements } from "./stranded-placement-redrive.js";
17
-
18
11
  installNodeLogger();
19
-
20
12
  const log = createLogger();
21
-
22
- setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
23
-
24
13
  /**
25
14
  * EventBridge-scheduled entry point for the periodic mailbox-sync tick
26
15
  * (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
@@ -38,27 +27,26 @@ setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
38
27
  * while a genuine retry of the same invocation carries the same `time` and
39
28
  * correctly dedupes.
40
29
  */
41
- export const handler: ScheduledHandler = withTelemetry(async (event) => {
42
- const client = await getClient();
43
- const now = Date.parse(event.time);
44
-
45
- await runSchedulerTick({
46
- accountService: client.account,
47
- queue: workerQueues().mailboxes,
48
- log,
49
- tickIntervalMs: getTickIntervalMs(),
50
- offlineIntervalMs: getOfflineIntervalMs(),
51
- now,
52
- // The hosted side is the one that needs this: a browser PUTs straight
53
- // to block storage, so nothing refuses an upload against a draft that is
54
- // already gone and this sweep is the only thing that collects it.
55
- sweepAttachments: buildAttachmentSweep(client, log),
56
- refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
57
- redriveStrandedPlacements: (account, tickNow) =>
58
- redriveStrandedPlacements(
59
- { client, emit: emitEvent, log },
60
- account,
61
- tickNow,
62
- ),
63
- });
30
+ export const buildScheduledHandler = (queues) => withTelemetry(async (event) => {
31
+ const client = await getClient();
32
+ const now = Date.parse(event.time);
33
+ await runSchedulerTick({
34
+ accountService: client.account,
35
+ queue: queues.mailboxes,
36
+ log,
37
+ tickIntervalMs: getTickIntervalMs(),
38
+ offlineIntervalMs: getOfflineIntervalMs(),
39
+ now,
40
+ // The hosted side is the one that needs this: a browser PUTs straight
41
+ // to block storage, so nothing refuses an upload against a draft that is
42
+ // already gone and this sweep is the only thing that collects it.
43
+ sweepAttachments: buildAttachmentSweep(client, log),
44
+ refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
45
+ redriveStrandedPlacements: (account, tickNow) => redriveStrandedPlacements({
46
+ client,
47
+ emit: lambdaEmitEvent(queues, client.mailboxLock),
48
+ log,
49
+ }, account, tickNow),
50
+ });
64
51
  });
52
+ export const handler = buildScheduledHandler(sqsWorkerQueues(workerQueueUrlsFromEnv()));
@@ -1,7 +1,5 @@
1
- import { setTimeout as delay } from "node:timers/promises";
2
1
  import type { Heartbeat } from "@remit/sqs-client/heartbeat";
3
2
  import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
4
-
5
3
  /**
6
4
  * The scheduled-sync loop, separated from the wiring in runner.ts for the same
7
5
  * reason `runQueuePoller` is separate from the container that starts it: the
@@ -14,25 +12,11 @@ import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
14
12
  * healthy for as long as it kept crashing.
15
13
  */
16
14
  export interface SchedulerLoopOptions {
17
- readonly tick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
18
- readonly tickDeps: RunSchedulerTickDeps;
19
- readonly heartbeat: Heartbeat;
20
- readonly onHeartbeatError: (error: unknown) => void;
21
- readonly tickIntervalMs: number;
22
- readonly wait?: (ms: number) => Promise<void>;
15
+ readonly tick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
16
+ readonly tickDeps: RunSchedulerTickDeps;
17
+ readonly heartbeat: Heartbeat;
18
+ readonly onHeartbeatError: (error: unknown) => void;
19
+ readonly tickIntervalMs: number;
20
+ readonly wait?: (ms: number) => Promise<void>;
23
21
  }
24
-
25
- export const runSchedulerLoop = async ({
26
- tick,
27
- tickDeps,
28
- heartbeat,
29
- onHeartbeatError,
30
- tickIntervalMs,
31
- wait = (ms) => delay(ms),
32
- }: SchedulerLoopOptions): Promise<never> => {
33
- for (;;) {
34
- await tick(tickDeps);
35
- await heartbeat().catch(onHeartbeatError);
36
- await wait(tickIntervalMs);
37
- }
38
- };
22
+ export declare const runSchedulerLoop: ({ tick, tickDeps, heartbeat, onHeartbeatError, tickIntervalMs, wait, }: SchedulerLoopOptions) => Promise<never>;
@@ -0,0 +1,8 @@
1
+ import { setTimeout as delay } from "node:timers/promises";
2
+ export const runSchedulerLoop = async ({ tick, tickDeps, heartbeat, onHeartbeatError, tickIntervalMs, wait = (ms) => delay(ms), }) => {
3
+ for (;;) {
4
+ await tick(tickDeps);
5
+ await heartbeat().catch(onHeartbeatError);
6
+ await wait(tickIntervalMs);
7
+ }
8
+ };
@@ -0,0 +1,62 @@
1
+ import { type SyncMailboxesEvent } from "@remit/backend/trigger-sync";
2
+ import type { AccountItem, FifoQueueProducer, IAccountRepository } from "@remit/data-ports";
3
+ import type { Logger } from "@remit/logger";
4
+ import type { CalendarSubscriptionTally } from "./calendar-subscriptions.js";
5
+ export interface RunSchedulerTickDeps {
6
+ accountService: Pick<IAccountRepository, "listAllAccountsPage">;
7
+ queue: FifoQueueProducer<SyncMailboxesEvent>;
8
+ log: Logger;
9
+ offlineIntervalMs: number;
10
+ /**
11
+ * How often this tick itself runs — the dedup-id bucket width for
12
+ * `buildScheduledSyncDedupId`, so consecutive ticks each get a fresh id.
13
+ */
14
+ tickIntervalMs: number;
15
+ /** Injectable for tests; defaults to `Date.now()`. */
16
+ now?: number;
17
+ /**
18
+ * Collect attachment bytes the database does not know about, for one account.
19
+ *
20
+ * The tick already walks every account on a cadence, which is the only thing
21
+ * this needs. It exists because an upload URL minted for a draft stays valid
22
+ * after that draft is discarded, and where the browser writes straight to
23
+ * block storage nothing is in the path to refuse it — see
24
+ * `sweepAbandonedOutboxAttachments`. Optional only so a test can leave it out;
25
+ * both real entry points pass it.
26
+ */
27
+ sweepAttachments?: (account: AccountItem) => Promise<void>;
28
+ /**
29
+ * Re-read the calendar subscriptions whose last fetch is older than
30
+ * `offlineIntervalMs` — the same knob that says how stale mail may get.
31
+ * Optional only so a test can leave it out; both real entry points pass it.
32
+ */
33
+ refreshCalendarSubscriptions?: (now: number, intervalMs: number) => Promise<CalendarSubscriptionTally>;
34
+ redriveStrandedPlacements: (account: AccountItem, now: number) => Promise<number>;
35
+ }
36
+ export interface SchedulerTickResult {
37
+ scanned: number;
38
+ enqueued: number;
39
+ skipped: number;
40
+ swept: number;
41
+ sweepFailed: number;
42
+ subscriptionsRefreshed: number;
43
+ subscriptionsFailed: number;
44
+ redriven: number;
45
+ redriveFailed: number;
46
+ }
47
+ /**
48
+ * One tick of the periodic mailbox-sync scheduler (#1247, restructured
49
+ * #1251). Pages through every account (never loading the whole account base
50
+ * into memory), decides per account via `isSyncDue` against the single
51
+ * offline threshold, and enqueues SYNC_MAILBOXES for the accounts that are
52
+ * due — in bounded-concurrency batches, never an unbounded `Promise.all`.
53
+ *
54
+ * Every enqueue goes through the same `triggerAccountSync` the manual
55
+ * POST /sync path uses, with a scheduler-specific, time-bucketed dedup id
56
+ * (`buildScheduledSyncDedupId`, bucketed by `tickIntervalMs`) so this tick
57
+ * can never collide with its own previous tick or with a concurrent manual
58
+ * trigger (see trigger-sync.ts). Concurrent-sync safety for the mailbox
59
+ * itself is MailboxLockService's job, inside the worker handler — this tick
60
+ * only decides "is a sync due" and enqueues; it never talks to IMAP.
61
+ */
62
+ export declare const runSchedulerTick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;