@remit/imap-worker 0.0.90 → 0.0.92

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/dist/account-check.d.ts +27 -0
  2. package/dist/account-check.js +70 -0
  3. package/dist/body-sync-gate.d.ts +12 -0
  4. package/dist/body-sync-gate.js +53 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +193 -0
  7. package/dist/config-env.d.ts +5 -0
  8. package/dist/config-env.js +37 -0
  9. package/dist/connection-pool.d.ts +22 -0
  10. package/dist/connection-pool.js +97 -0
  11. package/dist/connection-scope.d.ts +8 -0
  12. package/dist/connection-scope.js +10 -0
  13. package/dist/context.d.ts +46 -0
  14. package/dist/context.js +15 -0
  15. package/dist/e2e-processor-shim.d.ts +2 -0
  16. package/dist/e2e-processor-shim.js +181 -0
  17. package/dist/emit.d.ts +15 -0
  18. package/dist/emit.js +76 -0
  19. package/dist/events.d.ts +203 -0
  20. package/dist/events.js +1 -0
  21. package/dist/failure-kind.d.ts +10 -0
  22. package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
  23. package/dist/handlers/append-sent-message.d.ts +3 -0
  24. package/dist/handlers/append-sent-message.js +152 -0
  25. package/dist/handlers/delete-account-objects.d.ts +3 -0
  26. package/dist/handlers/delete-account-objects.js +35 -0
  27. package/dist/handlers/empty-trash.d.ts +14 -0
  28. package/dist/handlers/empty-trash.js +217 -0
  29. package/dist/handlers/flag-push.d.ts +38 -0
  30. package/dist/handlers/flag-push.js +195 -0
  31. package/dist/handlers/imported-folder.d.ts +4 -0
  32. package/dist/handlers/imported-folder.js +23 -0
  33. package/dist/handlers/mailbox-management.d.ts +6 -0
  34. package/dist/handlers/mailbox-management.js +294 -0
  35. package/dist/handlers/mailbox-sync-order.d.ts +21 -0
  36. package/dist/handlers/mailbox-sync-order.js +50 -0
  37. package/dist/handlers/message-copy.d.ts +40 -0
  38. package/dist/handlers/message-copy.js +371 -0
  39. package/dist/handlers/message-delete-terminal.d.ts +60 -0
  40. package/dist/handlers/message-delete-terminal.js +77 -0
  41. package/dist/handlers/message-delete.d.ts +57 -0
  42. package/dist/handlers/message-delete.js +520 -0
  43. package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
  44. package/dist/handlers/message-move-terminal.js +94 -0
  45. package/dist/handlers/message-move.d.ts +146 -0
  46. package/dist/handlers/message-move.js +490 -0
  47. package/dist/handlers/placement-move-push.d.ts +62 -0
  48. package/dist/handlers/placement-move-push.js +307 -0
  49. package/dist/handlers/probe-message-id.d.ts +2 -0
  50. package/dist/handlers/probe-message-id.js +6 -0
  51. package/dist/handlers/sync-mailboxes.d.ts +45 -0
  52. package/dist/handlers/sync-mailboxes.js +233 -0
  53. package/dist/handlers/sync-message-body.d.ts +23 -0
  54. package/dist/handlers/sync-message-body.js +182 -0
  55. package/dist/handlers/sync-messages.d.ts +29 -0
  56. package/dist/handlers/sync-messages.js +285 -0
  57. package/dist/index.d.ts +9 -0
  58. package/dist/index.js +67 -0
  59. package/dist/lambda-context.d.ts +8 -0
  60. package/dist/lambda-context.js +33 -0
  61. package/dist/mailbox-row.d.ts +9 -0
  62. package/dist/mailbox-row.js +6 -0
  63. package/dist/poller.d.ts +1 -0
  64. package/dist/poller.js +51 -0
  65. package/dist/process.d.ts +6 -0
  66. package/dist/process.js +3 -0
  67. package/dist/processor.d.ts +3 -0
  68. package/dist/processor.js +44 -0
  69. package/dist/queues.d.ts +10 -0
  70. package/dist/queues.js +1 -0
  71. package/dist/s3-account-objects.d.ts +3 -0
  72. package/dist/s3-account-objects.js +26 -0
  73. package/dist/scheduler/attachment-sweep.d.ts +11 -0
  74. package/dist/scheduler/attachment-sweep.js +23 -0
  75. package/dist/scheduler/calendar-subscriptions.d.ts +22 -0
  76. package/dist/scheduler/calendar-subscriptions.js +35 -0
  77. package/dist/scheduler/config.d.ts +29 -0
  78. package/{src/scheduler/config.ts → dist/scheduler/config.js} +9 -28
  79. package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
  80. package/dist/scheduler/decide-due.js +17 -0
  81. package/dist/scheduler/handler.d.ts +21 -0
  82. package/dist/scheduler/handler.js +50 -0
  83. package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
  84. package/dist/scheduler/loop.js +8 -0
  85. package/dist/scheduler/run-tick.d.ts +66 -0
  86. package/dist/scheduler/run-tick.js +164 -0
  87. package/dist/scheduler/runner.d.ts +2 -0
  88. package/dist/scheduler/runner.js +102 -0
  89. package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
  90. package/dist/scheduler/stranded-placement-redrive.js +172 -0
  91. package/dist/scheduler/tick.d.ts +5 -0
  92. package/dist/scheduler/tick.js +5 -0
  93. package/dist/sqs-queues.d.ts +4 -0
  94. package/dist/sqs-queues.js +22 -0
  95. package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
  96. package/dist/with-oauth-lifecycle-deps.js +17 -0
  97. package/dist/with-oauth-lifecycle.d.ts +56 -0
  98. package/dist/with-oauth-lifecycle.js +92 -0
  99. package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
  100. package/dist/worker-version.js +12 -0
  101. package/package.json +79 -69
  102. package/src/account-check.test.ts +0 -149
  103. package/src/account-check.ts +0 -101
  104. package/src/body-sync-gate.test.ts +0 -185
  105. package/src/body-sync-gate.ts +0 -86
  106. package/src/cli.ts +0 -218
  107. package/src/connection-scope.test.ts +0 -266
  108. package/src/connection-scope.ts +0 -314
  109. package/src/credential-less-sync-fence.test.ts +0 -152
  110. package/src/e2e-processor-shim.ts +0 -232
  111. package/src/emit.test.ts +0 -145
  112. package/src/emit.ts +0 -115
  113. package/src/events.ts +0 -252
  114. package/src/failure-kind.test.ts +0 -42
  115. package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
  116. package/src/handlers/append-sent-message.ts +0 -264
  117. package/src/handlers/delete-account-objects.test.ts +0 -126
  118. package/src/handlers/delete-account-objects.ts +0 -96
  119. package/src/handlers/empty-trash.sqlite.test.ts +0 -209
  120. package/src/handlers/empty-trash.ts +0 -352
  121. package/src/handlers/flag-push.ts +0 -285
  122. package/src/handlers/imported-folder.test.ts +0 -63
  123. package/src/handlers/imported-folder.ts +0 -38
  124. package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
  125. package/src/handlers/mailbox-management.ts +0 -486
  126. package/src/handlers/mailbox-sync-order.test.ts +0 -93
  127. package/src/handlers/mailbox-sync-order.ts +0 -65
  128. package/src/handlers/message-copy.sqlite.test.ts +0 -298
  129. package/src/handlers/message-copy.ts +0 -569
  130. package/src/handlers/message-delete-terminal.ts +0 -147
  131. package/src/handlers/message-delete.sqlite.test.ts +0 -291
  132. package/src/handlers/message-delete.ts +0 -888
  133. package/src/handlers/message-move.sqlite.test.ts +0 -271
  134. package/src/handlers/message-move.test.ts +0 -216
  135. package/src/handlers/message-move.ts +0 -755
  136. package/src/handlers/placement-move-push.test.ts +0 -137
  137. package/src/handlers/placement-move-push.ts +0 -478
  138. package/src/handlers/placement-push.sqlite.test.ts +0 -299
  139. package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
  140. package/src/handlers/probe-message-id.ts +0 -15
  141. package/src/handlers/sync-mailboxes.test.ts +0 -183
  142. package/src/handlers/sync-mailboxes.ts +0 -425
  143. package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
  144. package/src/handlers/sync-message-body.test.ts +0 -150
  145. package/src/handlers/sync-message-body.ts +0 -366
  146. package/src/handlers/sync-messages.sqlite.test.ts +0 -195
  147. package/src/handlers/sync-messages.ts +0 -537
  148. package/src/index.test.ts +0 -22
  149. package/src/index.ts +0 -82
  150. package/src/mail-sync-selection.sqlite.test.ts +0 -227
  151. package/src/mailbox-row.ts +0 -20
  152. package/src/pending-sync.sqlite.test.ts +0 -165
  153. package/src/poller.ts +0 -55
  154. package/src/processor.ts +0 -62
  155. package/src/queues.ts +0 -40
  156. package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -74
  157. package/src/scheduler/attachment-sweep.ts +0 -46
  158. package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
  159. package/src/scheduler/calendar-subscriptions.ts +0 -103
  160. package/src/scheduler/config.test.ts +0 -40
  161. package/src/scheduler/decide-due.test.ts +0 -44
  162. package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
  163. package/src/scheduler/handler.ts +0 -69
  164. package/src/scheduler/loop.test.ts +0 -143
  165. package/src/scheduler/run-tick.sqlite.test.ts +0 -191
  166. package/src/scheduler/run-tick.ts +0 -281
  167. package/src/scheduler/runner.ts +0 -125
  168. package/src/scheduler/stranded-placement-redrive.ts +0 -265
  169. package/src/stranded-placement.sqlite.test.ts +0 -799
  170. package/src/test-helpers/test-world.ts +0 -425
  171. package/src/test-helpers/worker-queues.ts +0 -11
  172. package/src/thread-join.sqlite.test.ts +0 -106
  173. package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
  174. package/src/with-oauth-lifecycle-deps.ts +0 -71
  175. package/src/with-oauth-lifecycle.ts +0 -158
  176. package/tsconfig.json +0 -8
@@ -0,0 +1,164 @@
1
+ import { buildScheduledSyncDedupId, triggerAccountSync, } from "@remit/backend/trigger-sync";
2
+ import { hasStoredCredential } from "@remit/mailbox-service/account-credentials";
3
+ import pMap from "p-map";
4
+ import { isAccountDeleted, isAccountReauthRequired, isMailSyncDisabled, isUnsyncableHost, } from "../account-check.js";
5
+ import { SCHEDULER_ENQUEUE_CONCURRENCY, SCHEDULER_PAGE_SIZE, } from "./config.js";
6
+ import { isSyncDue } from "./decide-due.js";
7
+ /**
8
+ * `isAccountDeleted` / `isUnsyncableHost` / `isAccountReauthRequired` each log
9
+ * one line per ineligible account — the right volume for their real call site
10
+ * (once per SYNC_MAILBOXES event). Run across the whole account base every
11
+ * tick, that becomes one log line per deleted/reauth/placeholder account
12
+ * every 5 minutes, forever (review #1250). The tick already reports the
13
+ * aggregate `skipped` count, so eligibility checks here go through a silent
14
+ * logger — the per-event path (sync-mailboxes.ts) is untouched and keeps
15
+ * logging normally.
16
+ */
17
+ const silentLogger = (() => {
18
+ const noop = () => { };
19
+ const stub = {
20
+ info: noop,
21
+ warn: noop,
22
+ error: noop,
23
+ debug: noop,
24
+ fatal: noop,
25
+ trace: noop,
26
+ child: () => stub,
27
+ };
28
+ return stub;
29
+ })();
30
+ const isEligible = (account) => {
31
+ if (isAccountDeleted(account, silentLogger))
32
+ return false;
33
+ if (isMailSyncDisabled(account, silentLogger))
34
+ return false;
35
+ if (isUnsyncableHost(account, silentLogger))
36
+ return false;
37
+ if (isAccountReauthRequired(account, silentLogger))
38
+ return false;
39
+ // An account storing no credential can only produce a sync that fails at
40
+ // credential resolution (issue #1120); the tick would enqueue one every
41
+ // tick, forever, for an account the user never finished setting up.
42
+ if (!hasStoredCredential(account))
43
+ return false;
44
+ return true;
45
+ };
46
+ /**
47
+ * One tick of the periodic mailbox-sync scheduler (#1247, restructured
48
+ * #1251). Pages through every account (never loading the whole account base
49
+ * into memory), decides per account via `isSyncDue` against the single
50
+ * offline threshold, and enqueues SYNC_MAILBOXES for the accounts that are
51
+ * due — in bounded-concurrency batches, never an unbounded `Promise.all`.
52
+ *
53
+ * Every enqueue goes through the same `triggerAccountSync` the manual
54
+ * POST /sync path uses, with a scheduler-specific, time-bucketed dedup id
55
+ * (`buildScheduledSyncDedupId`, bucketed by `tickIntervalMs`) so this tick
56
+ * can never collide with its own previous tick or with a concurrent manual
57
+ * trigger (see trigger-sync.ts). Concurrent-sync safety for the mailbox
58
+ * itself is MailboxLockService's job, inside the worker handler — this tick
59
+ * only decides "is a sync due" and enqueues; it never talks to IMAP.
60
+ */
61
+ export const runSchedulerTick = async (deps) => {
62
+ const { accountService, queue, log, offlineIntervalMs, tickIntervalMs } = deps;
63
+ const now = deps.now ?? Date.now();
64
+ let cursor;
65
+ let scanned = 0;
66
+ let enqueued = 0;
67
+ let skipped = 0;
68
+ let swept = 0;
69
+ let sweepFailed = 0;
70
+ let redriven = 0;
71
+ let redriveFailed = 0;
72
+ do {
73
+ const page = await accountService.listAllAccountsPage({
74
+ limit: SCHEDULER_PAGE_SIZE,
75
+ cursor,
76
+ });
77
+ scanned += page.items.length;
78
+ const due = page.items.filter((account) => isEligible(account) && isSyncDue(account, now, offlineIntervalMs));
79
+ skipped += page.items.length - due.length;
80
+ await pMap(due, (account) => triggerAccountSync({
81
+ queue,
82
+ accountId: account.accountId,
83
+ dedupId: buildScheduledSyncDedupId(account.accountId, now, tickIntervalMs),
84
+ }), { concurrency: SCHEDULER_ENQUEUE_CONCURRENCY });
85
+ enqueued += due.length;
86
+ if (deps.sweepAttachments) {
87
+ // After the enqueue, never before it. Enqueuing mail is the tick's job
88
+ // and collecting bytes is housekeeping; on a hosted Lambda a slow sweep
89
+ // ahead of the enqueue eats the invocation budget, and a timeout is not
90
+ // catchable, so the remaining pages would never be enqueued at all.
91
+ //
92
+ // Every account, not only the ones due a sync: abandoned bytes have
93
+ // nothing to do with how recently mail was fetched. Bounded to the same
94
+ // concurrency as the enqueues.
95
+ //
96
+ // Contained, because this is housekeeping and enqueuing mail is not. An
97
+ // unreadable input or a storage permission error thrown from here would
98
+ // otherwise reach the loop, which logs and exits, and compose would
99
+ // restart the process every five seconds — no account would ever be
100
+ // enqueued again. A failed sweep costs a tick's collection and nothing
101
+ // else; the count is what makes that visible.
102
+ const sweep = deps.sweepAttachments;
103
+ const outcomes = await pMap(page.items, (account) => sweep(account).then(() => true, (error) => {
104
+ log.error({ accountId: account.accountId, error }, "Outbox attachment sweep failed for account");
105
+ return false;
106
+ }), { concurrency: SCHEDULER_ENQUEUE_CONCURRENCY });
107
+ swept += outcomes.filter(Boolean).length;
108
+ sweepFailed += outcomes.filter((ok) => !ok).length;
109
+ }
110
+ // Contained like the sweep, and for the same reason: a failed re-drive
111
+ // costs one account this tick, never the scheduler. Eligible accounts
112
+ // only, since a re-driven event needs a connection to settle anything.
113
+ const redrives = await pMap(page.items.filter(isEligible), (account) => deps.redriveStrandedPlacements(account, now).then((count) => count, (error) => {
114
+ log.error({ accountId: account.accountId, error }, "Stranded placement re-drive failed for account");
115
+ return null;
116
+ }), { concurrency: SCHEDULER_ENQUEUE_CONCURRENCY });
117
+ for (const count of redrives) {
118
+ if (count === null) {
119
+ redriveFailed += 1;
120
+ continue;
121
+ }
122
+ redriven += count;
123
+ }
124
+ cursor = page.cursor ?? undefined;
125
+ } while (cursor);
126
+ let subscriptionsRefreshed = 0;
127
+ let subscriptionsFailed = 0;
128
+ if (deps.refreshCalendarSubscriptions) {
129
+ // After every page is enqueued, for the same reason the sweep runs after
130
+ // the enqueue: a slow feed must never cost an account its mail. Contained
131
+ // like the sweep, so an unreadable store costs a tick's refresh and not
132
+ // the scheduler.
133
+ const tally = await deps
134
+ .refreshCalendarSubscriptions(now, offlineIntervalMs)
135
+ .catch((error) => {
136
+ log.error({ error }, "Calendar subscription refresh failed");
137
+ return { refreshed: 0, failed: 0, notDue: 0 };
138
+ });
139
+ subscriptionsRefreshed = tally.refreshed;
140
+ subscriptionsFailed = tally.failed;
141
+ }
142
+ log.info({
143
+ scanned,
144
+ enqueued,
145
+ skipped,
146
+ swept,
147
+ sweepFailed,
148
+ subscriptionsRefreshed,
149
+ subscriptionsFailed,
150
+ redriven,
151
+ redriveFailed,
152
+ }, "Scheduled-sync tick complete");
153
+ return {
154
+ scanned,
155
+ enqueued,
156
+ skipped,
157
+ swept,
158
+ sweepFailed,
159
+ subscriptionsRefreshed,
160
+ subscriptionsFailed,
161
+ redriven,
162
+ redriveFailed,
163
+ };
164
+ };
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export {};
@@ -0,0 +1,102 @@
1
+ #!/usr/bin/env node
2
+ import { setTimeout as delay } from "node:timers/promises";
3
+ import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
4
+ import { getClient } from "@remit/backend/client";
5
+ import { createLogger } from "@remit/logger";
6
+ import { installNodeLogger } from "@remit/logger-lambda/node";
7
+ import { backfillSearchText } from "@remit/mailbox-service";
8
+ import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
9
+ import { lambdaEmitEvent } from "../lambda-context.js";
10
+ import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "../sqs-queues.js";
11
+ import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
12
+ import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
13
+ import { runSchedulerLoop } from "./loop.js";
14
+ import { buildAttachmentSweep, redriveStrandedPlacements, runSchedulerTick, } from "./tick.js";
15
+ installNodeLogger();
16
+ /**
17
+ * The scheduled-sync runner for every deployment that has no EventBridge: the
18
+ * self-host stack's `scheduler` service and the local dev compose stack. A
19
+ * managed deployment fires `runSchedulerTick` off an EventBridge schedule (see
20
+ * handler.ts); this process ticks the same function on a plain loop at the same
21
+ * `MAILBOX_SYNC_TICK_INTERVAL_SECONDS` cadence, so there is one scheduling
22
+ * implementation and one set of knobs.
23
+ *
24
+ * A tick failure crashes the process loudly rather than being swallowed —
25
+ * compose's `restart: unless-stopped` brings it back for the next tick.
26
+ *
27
+ * Liveness is a heartbeat file, the same mechanism as a worker's poll loop (D1
28
+ * of docs/design/standalone-observability.md); loop.ts carries where in the loop
29
+ * it is written and why. Clearing the previous generation's file here is what
30
+ * extends that answer across a restart. A completed round is still not the same
31
+ * as mail arriving: the accounts that came due are enqueued for workers to
32
+ * fetch, and whether that fetch lands is what
33
+ * `remit_account_sync_age_seconds` measures.
34
+ */
35
+ const log = createLogger();
36
+ const queues = sqsWorkerQueues(workerQueueUrlsFromEnv());
37
+ const tickIntervalMs = getTickIntervalMs(process.env);
38
+ const offlineIntervalMs = getOfflineIntervalMs(process.env);
39
+ // A persistent failure (e.g. Postgres not up yet at container boot) throws
40
+ // before the loop ever reaches its own `delay`, so `restart: unless-stopped`
41
+ // would otherwise respawn the process immediately — a tight, log-flooding
42
+ // crash loop (review #1250). This fixed pause before exiting is not retry
43
+ // logic (there is nothing to retry here; the container restart IS the
44
+ // retry) — it only paces how fast that restart can happen.
45
+ const CRASH_BACKOFF_MS = 5_000;
46
+ log.info({ tickIntervalMs, offlineIntervalMs }, "Scheduled-sync runner started");
47
+ const runLoop = async () => {
48
+ // Neither call on the monitoring file may take the scheduler down with it. An
49
+ // unreadable or full /data/heartbeat is a reason to keep enqueuing mail, not
50
+ // to stop. A clear that fails leaves the previous generation's file, which
51
+ // ages out at the same threshold; a write that fails is the missed beat that
52
+ // is itself the signal. Two messages because they are different problems: one
53
+ // is a volume this container could not read at boot, the other a write it
54
+ // could not make this round.
55
+ const onClearError = (error) => {
56
+ log.error({ error }, "Scheduled-sync heartbeat clear failed");
57
+ };
58
+ const onBeatError = (error) => {
59
+ log.error({ error }, "Scheduled-sync heartbeat write failed");
60
+ };
61
+ await clearHeartbeats().catch(onClearError);
62
+ const client = await getClient();
63
+ const { account } = client;
64
+ await finishHalfDeletedAccounts({
65
+ account,
66
+ configImport: client.configImport,
67
+ queue: client.queues.accountFanout,
68
+ log,
69
+ });
70
+ const searchTextBackfill = backfillSearchText({ messageService: client.message, storageService: client.storage }, log).then((result) => {
71
+ log.debug({ ...result }, "Search text backfill finished");
72
+ });
73
+ const schedulerLoop = runSchedulerLoop({
74
+ tick: runSchedulerTick,
75
+ tickDeps: {
76
+ accountService: account,
77
+ queue: queues.mailboxes,
78
+ log,
79
+ tickIntervalMs,
80
+ offlineIntervalMs,
81
+ sweepAttachments: buildAttachmentSweep(client, log),
82
+ refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
83
+ redriveStrandedPlacements: (stranded, now) => redriveStrandedPlacements({
84
+ client,
85
+ emit: lambdaEmitEvent(queues, client.mailboxLock),
86
+ log,
87
+ }, stranded, now),
88
+ },
89
+ heartbeat: createHeartbeat("tick"),
90
+ onHeartbeatError: onBeatError,
91
+ tickIntervalMs,
92
+ });
93
+ await Promise.all([searchTextBackfill, schedulerLoop]);
94
+ };
95
+ runLoop()
96
+ .catch(async (error) => {
97
+ log.error({ error }, "Scheduled-sync tick failed");
98
+ await delay(CRASH_BACKOFF_MS);
99
+ })
100
+ .finally(() => {
101
+ process.exit(1);
102
+ });
@@ -0,0 +1,16 @@
1
+ import type { RemitClient } from "@remit/backend/client";
2
+ import type { AccountItem } from "@remit/data-ports";
3
+ import type { Logger } from "@remit/logger";
4
+ import type { MessageCopyEventV3, MessageDeleteEventV3, MessageMoveEventV3, PlacementMovePushEvent } from "../events.js";
5
+ /** Well past three attempts on a 60 s visibility timeout. */
6
+ export declare const STRANDED_PLACEMENT_AFTER_MILLIS: number;
7
+ type Unstamped<T> = Omit<T, "eventId" | "timestamp">;
8
+ export type StrandedPlacementEvent = Unstamped<PlacementMovePushEvent> | Unstamped<MessageDeleteEventV3> | Unstamped<MessageMoveEventV3> | Unstamped<MessageCopyEventV3>;
9
+ export interface StrandedPlacementRedriveDeps {
10
+ client: Pick<RemitClient, "mailbox" | "mailboxSpecialUse" | "message" | "placementMove" | "threadMessage">;
11
+ emit: (event: StrandedPlacementEvent) => Promise<void>;
12
+ log: Logger;
13
+ }
14
+ /** The claim restarts the row's clock; a folder awaiting a cursor rebuild waits until it is normal. */
15
+ export declare const redriveStrandedPlacements: (deps: StrandedPlacementRedriveDeps, account: AccountItem, now: number) => Promise<number>;
16
+ export {};
@@ -0,0 +1,172 @@
1
+ import { trashMailboxAt } from "@remit/data-ports/folder-role";
2
+ import { deriveCopyMessageId, deriveMessageId, isValidMessageId, } from "@remit/data-ports/id";
3
+ import { PATHLESS_MUTATION_EVENT_SCHEMA_VERSION } from "@remit/data-ports/mutation-events";
4
+ import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
5
+ import { isCursorRebuildNeeded } from "@remit/mailbox-service";
6
+ /** Well past three attempts on a 60 s visibility timeout. */
7
+ export const STRANDED_PLACEMENT_AFTER_MILLIS = 60 * 60 * 1000;
8
+ const isStrandedCopy = (row) => row.status === MessageStatus.moving &&
9
+ row.uid === 0 &&
10
+ typeof row.originalUid !== "number";
11
+ /**
12
+ * The message a uid-0 copy was made from: its id derives from the Message-ID
13
+ * the copy's listing row carries, and the copy's own id derives from it.
14
+ */
15
+ const copyRedriveFor = async (client, account, copy) => {
16
+ const listing = await client.threadMessage.findByMessageId(account.accountConfigId, copy.messageId);
17
+ if (!listing)
18
+ return { kind: "unresolvable", reason: "listing row missing" };
19
+ const header = listing.messageIdHeader;
20
+ if (!header || !isValidMessageId(header)) {
21
+ return { kind: "unresolvable", reason: "listing row has no Message-ID" };
22
+ }
23
+ const sourceId = deriveMessageId(account.accountId, header);
24
+ if (deriveCopyMessageId(sourceId, copy.mailboxId) !== copy.messageId) {
25
+ return {
26
+ kind: "unresolvable",
27
+ reason: "copy was not made from the message its Message-ID names",
28
+ };
29
+ }
30
+ const [source] = await client.message.get([sourceId]);
31
+ if (!source)
32
+ return { kind: "unresolvable", reason: "source row missing" };
33
+ if (source.status === MessageStatus.deleted) {
34
+ return { kind: "unresolvable", reason: "source row deleted" };
35
+ }
36
+ if (source.status !== MessageStatus.active)
37
+ return { kind: "wait" };
38
+ return {
39
+ kind: "redrive",
40
+ event: {
41
+ type: "MESSAGE_COPY",
42
+ schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
43
+ accountId: account.accountId,
44
+ sourceMessageId: source.messageId,
45
+ newMessageId: copy.messageId,
46
+ sourceMailboxId: source.mailboxId,
47
+ destinationMailboxId: copy.mailboxId,
48
+ uid: source.uid,
49
+ },
50
+ };
51
+ };
52
+ /** A copy with no source to copy from has nothing to bind and nothing to hide. */
53
+ const removeUnresolvableCopy = async (deps, account, row, reason, now) => {
54
+ const removed = await deps.client.message.deleteIfPlacement(row.messageId, {
55
+ status: MessageStatus.moving,
56
+ mailboxId: row.mailboxId,
57
+ uid: 0,
58
+ });
59
+ if (!removed)
60
+ return false;
61
+ deps.log.error({
62
+ alert: "stranded_copy_unresolvable",
63
+ accountId: account.accountId,
64
+ messageId: row.messageId,
65
+ mailboxId: row.mailboxId,
66
+ reason,
67
+ untouchedForMs: now - row.updatedAt,
68
+ }, "A copy stranded at uid 0 has no source to copy from; the copy row was removed");
69
+ return true;
70
+ };
71
+ const eventFor = async (client, account, row, trashMailboxId) => {
72
+ const { accountId, accountConfigId } = account;
73
+ const { messageId } = row;
74
+ if (row.status === MessageStatus.deleting) {
75
+ return {
76
+ type: "MESSAGE_DELETE",
77
+ schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
78
+ accountId,
79
+ messageId,
80
+ mailboxId: row.mailboxId,
81
+ uid: row.uid,
82
+ operation: "permanent_delete",
83
+ };
84
+ }
85
+ if (typeof row.originalMailboxId !== "string" ||
86
+ typeof row.originalUid !== "number") {
87
+ return undefined;
88
+ }
89
+ const marker = await client.placementMove.find(messageId);
90
+ if (marker && marker.destinationMailboxId === row.mailboxId) {
91
+ return {
92
+ type: "PLACEMENT_MOVE_PUSH",
93
+ accountId,
94
+ accountConfigId,
95
+ messageId,
96
+ };
97
+ }
98
+ if (row.mailboxId === trashMailboxId) {
99
+ return {
100
+ type: "MESSAGE_DELETE",
101
+ schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
102
+ accountId,
103
+ messageId,
104
+ mailboxId: row.originalMailboxId,
105
+ uid: row.originalUid,
106
+ operation: "move_to_trash",
107
+ destinationMailboxId: row.mailboxId,
108
+ };
109
+ }
110
+ return {
111
+ type: "MESSAGE_MOVE",
112
+ schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
113
+ accountId,
114
+ messageId,
115
+ sourceMailboxId: row.originalMailboxId,
116
+ destinationMailboxId: row.mailboxId,
117
+ uid: row.originalUid,
118
+ };
119
+ };
120
+ /** The claim restarts the row's clock; a folder awaiting a cursor rebuild waits until it is normal. */
121
+ export const redriveStrandedPlacements = async (deps, account, now) => {
122
+ const { client, emit, log } = deps;
123
+ const { accountId } = account;
124
+ const mailboxes = await client.mailbox.listAllByAccount(accountId);
125
+ const paused = new Set(mailboxes
126
+ .filter((mailbox) => isCursorRebuildNeeded(mailbox.cursorState))
127
+ .map((mailbox) => mailbox.mailboxId));
128
+ const stranded = (await client.message.listStrandedPlacements(mailboxes.map((mailbox) => mailbox.mailboxId), now - STRANDED_PLACEMENT_AFTER_MILLIS)).filter((row) => !paused.has(row.mailboxId) &&
129
+ !(row.originalMailboxId && paused.has(row.originalMailboxId)));
130
+ if (stranded.length === 0)
131
+ return 0;
132
+ const trash = trashMailboxAt(await client.mailboxSpecialUse.resolveTrashRole(accountId), "resolved");
133
+ const trashMailboxId = trash.allowed ? trash.mailbox.mailboxId : undefined;
134
+ let redriven = 0;
135
+ for (const row of stranded) {
136
+ const copy = isStrandedCopy(row)
137
+ ? await copyRedriveFor(client, account, row)
138
+ : undefined;
139
+ if (copy?.kind === "wait")
140
+ continue;
141
+ if (copy?.kind === "unresolvable") {
142
+ if (await removeUnresolvableCopy(deps, account, row, copy.reason, now)) {
143
+ redriven += 1;
144
+ }
145
+ continue;
146
+ }
147
+ const event = copy?.kind === "redrive"
148
+ ? copy.event
149
+ : await eventFor(client, account, row, trashMailboxId);
150
+ if (!event)
151
+ continue;
152
+ // A copy's earlier COPY may have landed with its answer lost, so the
153
+ // re-drive marks it `failed`: the copy handler then asks the destination
154
+ // before it copies again.
155
+ const claimed = await client.message.transitionPlacement(row.messageId, { status: row.status, mailboxId: row.mailboxId, uid: row.uid }, {
156
+ syncStatus: copy ? MessageSyncStatus.failed : MessageSyncStatus.pending,
157
+ });
158
+ if (!claimed)
159
+ continue;
160
+ await emit(event);
161
+ redriven += 1;
162
+ log.warn({
163
+ metric: "stranded_placement_redriven",
164
+ accountId,
165
+ messageId: row.messageId,
166
+ event: event.type,
167
+ syncStatus: row.syncStatus,
168
+ untouchedForMs: now - row.updatedAt,
169
+ }, "A move or delete had no event left to settle it; re-driven");
170
+ }
171
+ return redriven;
172
+ };
@@ -0,0 +1,5 @@
1
+ export { createEmitEvent, type EmitDeps, type EmitEvent } from "../emit.js";
2
+ export { buildAttachmentSweep } from "./attachment-sweep.js";
3
+ export { DEFAULT_OFFLINE_INTERVAL_MS, DEFAULT_TICK_INTERVAL_MS, } from "./config.js";
4
+ export { type CalendarSubscriptionTally, type RunSchedulerTickDeps, runSchedulerTick, type SchedulerTickResult, } from "./run-tick.js";
5
+ export { redriveStrandedPlacements, type StrandedPlacementEvent, type StrandedPlacementRedriveDeps, } from "./stranded-placement-redrive.js";
@@ -0,0 +1,5 @@
1
+ export { createEmitEvent } from "../emit.js";
2
+ export { buildAttachmentSweep } from "./attachment-sweep.js";
3
+ export { DEFAULT_OFFLINE_INTERVAL_MS, DEFAULT_TICK_INTERVAL_MS, } from "./config.js";
4
+ export { runSchedulerTick, } from "./run-tick.js";
5
+ export { redriveStrandedPlacements, } from "./stranded-placement-redrive.js";
@@ -0,0 +1,4 @@
1
+ import type { WorkerQueues } from "./queues.js";
2
+ export type WorkerQueueUrls = Record<keyof WorkerQueues, string>;
3
+ export declare const workerQueueUrlsFromEnv: () => WorkerQueueUrls;
4
+ export declare const sqsWorkerQueues: (urls: WorkerQueueUrls) => WorkerQueues;
@@ -0,0 +1,22 @@
1
+ import { createFifoQueueProducer, createStandardQueueProducer, } from "@remit/sqs-client/producer";
2
+ import { env } from "expect-env";
3
+ export const workerQueueUrlsFromEnv = () => ({
4
+ mailboxes: env.SQS_QUEUE_URL_MAILBOXES,
5
+ messages: env.SQS_QUEUE_URL_MESSAGES,
6
+ flags: env.SQS_QUEUE_URL_FLAGS,
7
+ body: env.SQS_QUEUE_URL_BODY,
8
+ mailboxManagement: env.SQS_QUEUE_URL_MAILBOX_MGMT,
9
+ messageManagement: env.SQS_QUEUE_URL_MESSAGE_MGMT,
10
+ });
11
+ export const sqsWorkerQueues = (urls) => ({
12
+ mailboxes: createFifoQueueProducer({ queueUrl: urls.mailboxes }),
13
+ messages: createFifoQueueProducer({ queueUrl: urls.messages }),
14
+ flags: createFifoQueueProducer({ queueUrl: urls.flags }),
15
+ body: createStandardQueueProducer({ queueUrl: urls.body }),
16
+ mailboxManagement: createStandardQueueProducer({
17
+ queueUrl: urls.mailboxManagement,
18
+ }),
19
+ messageManagement: createStandardQueueProducer({
20
+ queueUrl: urls.messageManagement,
21
+ }),
22
+ });
@@ -0,0 +1,5 @@
1
+ import type { IAccountRepository } from "@remit/data-ports";
2
+ import type { SecretsService } from "@remit/secrets-service";
3
+ import type { TokenService } from "./context.js";
4
+ import type { OAuthLifecycleDeps } from "./with-oauth-lifecycle.js";
5
+ export declare const buildLifecycleDeps: (secrets: SecretsService, accountService: IAccountRepository, tokenService: TokenService) => OAuthLifecycleDeps;
@@ -0,0 +1,17 @@
1
+ export const buildLifecycleDeps = (secrets, accountService, tokenService) => ({
2
+ secrets,
3
+ tokenService,
4
+ persistRotatedToken: async (accountId, encryptedHash, updatedAt) => {
5
+ await accountService.update(accountId, {
6
+ oauthRefreshTokenHash: encryptedHash,
7
+ oauthTokenUpdatedAt: updatedAt,
8
+ });
9
+ },
10
+ // A state flip with no reason of its own removes the stored one. Left in
11
+ // place it would describe a failure the account has moved on from — the
12
+ // mailbox a sync could not find, still on the card beside a Reconnect
13
+ // button, after the token was revoked hours later.
14
+ updateConnectionState: async (accountId, state, lastError) => {
15
+ await accountService.update(accountId, { connectionState: state, lastError }, lastError === undefined ? ["lastError"] : undefined);
16
+ },
17
+ });
@@ -0,0 +1,56 @@
1
+ /**
2
+ * with-oauth-lifecycle.ts
3
+ *
4
+ * Centralizes the OAuth reauth/ACK contract shared by every IMAP handler.
5
+ *
6
+ * The contract (see issue #472):
7
+ * - If the account already requires reauth, skip all IMAP traffic entirely.
8
+ * - Credential resolution AND the per-handler work both run inside one
9
+ * try/catch so that a revoked OAuth token is caught regardless of where it
10
+ * surfaces (token mint vs. first IMAP command).
11
+ * - An account storing no credential at all is terminal, not transient: the
12
+ * account is flipped to the state that describes it and the event is ACKed.
13
+ * Retrying instead fenced the account's FIFO group for the message's whole
14
+ * visibility timeout, blocking the sync that follows the credential the
15
+ * user was still typing (issue #1120).
16
+ * - On a terminal auth failure (RefreshTokenError reauth-required, or
17
+ * MailConnectionError auth), flip the account to reauth_required and return
18
+ * WITHOUT rethrowing — this ACKs the SQS message so it is not retried.
19
+ * An IMAP refusal is stored with it, because not every refusal a mail server
20
+ * hands back is one re-authenticating clears: "SmtpClientAuthentication is
21
+ * disabled" needs a tenant setting changed, and an account card that only
22
+ * ever says "Re-authentication required" sends the user around a loop.
23
+ * - On transient / config / network errors, rethrow so SQS retries with
24
+ * backoff (let-it-crash).
25
+ *
26
+ * Tokens must NEVER appear in logs — only accountId / errorKind / errorCode.
27
+ */
28
+ import type { AccountItem } from "@remit/data-ports";
29
+ import type { Logger } from "@remit/logger";
30
+ import { type AccountCredentialsDeps, type ConnectionStateValue, type CredentialResolution } from "@remit/mailbox-service/account-credentials";
31
+ import { type MailCredentials } from "@remit/mailbox-service/types";
32
+ export type { ConnectionStateValue };
33
+ export interface OAuthLifecycleDeps extends AccountCredentialsDeps {
34
+ /**
35
+ * Persist the account's connectionState, and with it the sentence the
36
+ * account card shows for the failure. `lastError` carries the server's own
37
+ * words when the failure came with any; it is absent when nothing was said,
38
+ * as for a credential that was never stored or a token simply revoked, and
39
+ * the stored reason is then cleared rather than left to describe an older
40
+ * failure the account has moved on from.
41
+ */
42
+ updateConnectionState: (accountId: string, state: ConnectionStateValue, lastError?: string) => Promise<void>;
43
+ /**
44
+ * Resolve credentials for the account. Defaults to
45
+ * resolveConnectionCredentials; overridable for testing. Kept as the ONLY
46
+ * authType branch in the codebase (see account-credentials.ts).
47
+ */
48
+ resolveCredentials?: (account: AccountItem, deps: AccountCredentialsDeps) => Promise<CredentialResolution>;
49
+ }
50
+ /**
51
+ * Run `work` for an account under the shared OAuth reauth/ACK contract.
52
+ *
53
+ * `work` receives the resolved MailCredentials. Both credential resolution and
54
+ * `work` run inside the same try/catch.
55
+ */
56
+ export declare const withOAuthLifecycle: (deps: OAuthLifecycleDeps, account: AccountItem, log: Logger, work: (credentials: MailCredentials) => Promise<void>) => Promise<void>;