@remit/imap-worker 0.0.90 → 0.0.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/account-check.d.ts +27 -0
- package/dist/account-check.js +70 -0
- package/dist/body-sync-gate.d.ts +12 -0
- package/dist/body-sync-gate.js +53 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +193 -0
- package/dist/config-env.d.ts +5 -0
- package/dist/config-env.js +37 -0
- package/dist/connection-pool.d.ts +22 -0
- package/dist/connection-pool.js +97 -0
- package/dist/connection-scope.d.ts +8 -0
- package/dist/connection-scope.js +10 -0
- package/dist/context.d.ts +46 -0
- package/dist/context.js +15 -0
- package/dist/e2e-processor-shim.d.ts +2 -0
- package/dist/e2e-processor-shim.js +181 -0
- package/dist/emit.d.ts +15 -0
- package/dist/emit.js +76 -0
- package/dist/events.d.ts +203 -0
- package/dist/events.js +1 -0
- package/dist/failure-kind.d.ts +10 -0
- package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
- package/dist/handlers/append-sent-message.d.ts +3 -0
- package/dist/handlers/append-sent-message.js +152 -0
- package/dist/handlers/delete-account-objects.d.ts +3 -0
- package/dist/handlers/delete-account-objects.js +35 -0
- package/dist/handlers/empty-trash.d.ts +14 -0
- package/dist/handlers/empty-trash.js +217 -0
- package/dist/handlers/flag-push.d.ts +38 -0
- package/dist/handlers/flag-push.js +195 -0
- package/dist/handlers/imported-folder.d.ts +4 -0
- package/dist/handlers/imported-folder.js +23 -0
- package/dist/handlers/mailbox-management.d.ts +6 -0
- package/dist/handlers/mailbox-management.js +294 -0
- package/dist/handlers/mailbox-sync-order.d.ts +21 -0
- package/dist/handlers/mailbox-sync-order.js +50 -0
- package/dist/handlers/message-copy.d.ts +40 -0
- package/dist/handlers/message-copy.js +371 -0
- package/dist/handlers/message-delete-terminal.d.ts +60 -0
- package/dist/handlers/message-delete-terminal.js +77 -0
- package/dist/handlers/message-delete.d.ts +57 -0
- package/dist/handlers/message-delete.js +520 -0
- package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
- package/dist/handlers/message-move-terminal.js +94 -0
- package/dist/handlers/message-move.d.ts +146 -0
- package/dist/handlers/message-move.js +490 -0
- package/dist/handlers/placement-move-push.d.ts +62 -0
- package/dist/handlers/placement-move-push.js +307 -0
- package/dist/handlers/probe-message-id.d.ts +2 -0
- package/dist/handlers/probe-message-id.js +6 -0
- package/dist/handlers/sync-mailboxes.d.ts +45 -0
- package/dist/handlers/sync-mailboxes.js +233 -0
- package/dist/handlers/sync-message-body.d.ts +23 -0
- package/dist/handlers/sync-message-body.js +182 -0
- package/dist/handlers/sync-messages.d.ts +29 -0
- package/dist/handlers/sync-messages.js +285 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +67 -0
- package/dist/lambda-context.d.ts +8 -0
- package/dist/lambda-context.js +33 -0
- package/dist/mailbox-row.d.ts +9 -0
- package/dist/mailbox-row.js +6 -0
- package/dist/poller.d.ts +1 -0
- package/dist/poller.js +51 -0
- package/dist/process.d.ts +6 -0
- package/dist/process.js +3 -0
- package/dist/processor.d.ts +3 -0
- package/dist/processor.js +44 -0
- package/dist/queues.d.ts +10 -0
- package/dist/queues.js +1 -0
- package/dist/s3-account-objects.d.ts +3 -0
- package/dist/s3-account-objects.js +26 -0
- package/dist/scheduler/attachment-sweep.d.ts +11 -0
- package/dist/scheduler/attachment-sweep.js +23 -0
- package/dist/scheduler/calendar-subscriptions.d.ts +22 -0
- package/dist/scheduler/calendar-subscriptions.js +35 -0
- package/dist/scheduler/config.d.ts +29 -0
- package/{src/scheduler/config.ts → dist/scheduler/config.js} +9 -28
- package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
- package/dist/scheduler/decide-due.js +17 -0
- package/dist/scheduler/handler.d.ts +21 -0
- package/dist/scheduler/handler.js +50 -0
- package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
- package/dist/scheduler/loop.js +8 -0
- package/dist/scheduler/run-tick.d.ts +66 -0
- package/dist/scheduler/run-tick.js +164 -0
- package/dist/scheduler/runner.d.ts +2 -0
- package/dist/scheduler/runner.js +102 -0
- package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
- package/dist/scheduler/stranded-placement-redrive.js +172 -0
- package/dist/scheduler/tick.d.ts +5 -0
- package/dist/scheduler/tick.js +5 -0
- package/dist/sqs-queues.d.ts +4 -0
- package/dist/sqs-queues.js +22 -0
- package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
- package/dist/with-oauth-lifecycle-deps.js +17 -0
- package/dist/with-oauth-lifecycle.d.ts +56 -0
- package/dist/with-oauth-lifecycle.js +92 -0
- package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
- package/dist/worker-version.js +12 -0
- package/package.json +79 -69
- package/src/account-check.test.ts +0 -149
- package/src/account-check.ts +0 -101
- package/src/body-sync-gate.test.ts +0 -185
- package/src/body-sync-gate.ts +0 -86
- package/src/cli.ts +0 -218
- package/src/connection-scope.test.ts +0 -266
- package/src/connection-scope.ts +0 -314
- package/src/credential-less-sync-fence.test.ts +0 -152
- package/src/e2e-processor-shim.ts +0 -232
- package/src/emit.test.ts +0 -145
- package/src/emit.ts +0 -115
- package/src/events.ts +0 -252
- package/src/failure-kind.test.ts +0 -42
- package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
- package/src/handlers/append-sent-message.ts +0 -264
- package/src/handlers/delete-account-objects.test.ts +0 -126
- package/src/handlers/delete-account-objects.ts +0 -96
- package/src/handlers/empty-trash.sqlite.test.ts +0 -209
- package/src/handlers/empty-trash.ts +0 -352
- package/src/handlers/flag-push.ts +0 -285
- package/src/handlers/imported-folder.test.ts +0 -63
- package/src/handlers/imported-folder.ts +0 -38
- package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
- package/src/handlers/mailbox-management.ts +0 -486
- package/src/handlers/mailbox-sync-order.test.ts +0 -93
- package/src/handlers/mailbox-sync-order.ts +0 -65
- package/src/handlers/message-copy.sqlite.test.ts +0 -298
- package/src/handlers/message-copy.ts +0 -569
- package/src/handlers/message-delete-terminal.ts +0 -147
- package/src/handlers/message-delete.sqlite.test.ts +0 -291
- package/src/handlers/message-delete.ts +0 -888
- package/src/handlers/message-move.sqlite.test.ts +0 -271
- package/src/handlers/message-move.test.ts +0 -216
- package/src/handlers/message-move.ts +0 -755
- package/src/handlers/placement-move-push.test.ts +0 -137
- package/src/handlers/placement-move-push.ts +0 -478
- package/src/handlers/placement-push.sqlite.test.ts +0 -299
- package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
- package/src/handlers/probe-message-id.ts +0 -15
- package/src/handlers/sync-mailboxes.test.ts +0 -183
- package/src/handlers/sync-mailboxes.ts +0 -425
- package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
- package/src/handlers/sync-message-body.test.ts +0 -150
- package/src/handlers/sync-message-body.ts +0 -366
- package/src/handlers/sync-messages.sqlite.test.ts +0 -195
- package/src/handlers/sync-messages.ts +0 -537
- package/src/index.test.ts +0 -22
- package/src/index.ts +0 -82
- package/src/mail-sync-selection.sqlite.test.ts +0 -227
- package/src/mailbox-row.ts +0 -20
- package/src/pending-sync.sqlite.test.ts +0 -165
- package/src/poller.ts +0 -55
- package/src/processor.ts +0 -62
- package/src/queues.ts +0 -40
- package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -74
- package/src/scheduler/attachment-sweep.ts +0 -46
- package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
- package/src/scheduler/calendar-subscriptions.ts +0 -103
- package/src/scheduler/config.test.ts +0 -40
- package/src/scheduler/decide-due.test.ts +0 -44
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
- package/src/scheduler/handler.ts +0 -69
- package/src/scheduler/loop.test.ts +0 -143
- package/src/scheduler/run-tick.sqlite.test.ts +0 -191
- package/src/scheduler/run-tick.ts +0 -281
- package/src/scheduler/runner.ts +0 -125
- package/src/scheduler/stranded-placement-redrive.ts +0 -265
- package/src/stranded-placement.sqlite.test.ts +0 -799
- package/src/test-helpers/test-world.ts +0 -425
- package/src/test-helpers/worker-queues.ts +0 -11
- package/src/thread-join.sqlite.test.ts +0 -106
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
- package/src/with-oauth-lifecycle-deps.ts +0 -71
- package/src/with-oauth-lifecycle.ts +0 -158
- package/tsconfig.json +0 -8
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { buildScheduledSyncDedupId, triggerAccountSync, } from "@remit/backend/trigger-sync";
|
|
2
|
+
import { hasStoredCredential } from "@remit/mailbox-service/account-credentials";
|
|
3
|
+
import pMap from "p-map";
|
|
4
|
+
import { isAccountDeleted, isAccountReauthRequired, isMailSyncDisabled, isUnsyncableHost, } from "../account-check.js";
|
|
5
|
+
import { SCHEDULER_ENQUEUE_CONCURRENCY, SCHEDULER_PAGE_SIZE, } from "./config.js";
|
|
6
|
+
import { isSyncDue } from "./decide-due.js";
|
|
7
|
+
/**
|
|
8
|
+
* `isAccountDeleted` / `isUnsyncableHost` / `isAccountReauthRequired` each log
|
|
9
|
+
* one line per ineligible account — the right volume for their real call site
|
|
10
|
+
* (once per SYNC_MAILBOXES event). Run across the whole account base every
|
|
11
|
+
* tick, that becomes one log line per deleted/reauth/placeholder account
|
|
12
|
+
* every 5 minutes, forever (review #1250). The tick already reports the
|
|
13
|
+
* aggregate `skipped` count, so eligibility checks here go through a silent
|
|
14
|
+
* logger — the per-event path (sync-mailboxes.ts) is untouched and keeps
|
|
15
|
+
* logging normally.
|
|
16
|
+
*/
|
|
17
|
+
const silentLogger = (() => {
|
|
18
|
+
const noop = () => { };
|
|
19
|
+
const stub = {
|
|
20
|
+
info: noop,
|
|
21
|
+
warn: noop,
|
|
22
|
+
error: noop,
|
|
23
|
+
debug: noop,
|
|
24
|
+
fatal: noop,
|
|
25
|
+
trace: noop,
|
|
26
|
+
child: () => stub,
|
|
27
|
+
};
|
|
28
|
+
return stub;
|
|
29
|
+
})();
|
|
30
|
+
const isEligible = (account) => {
|
|
31
|
+
if (isAccountDeleted(account, silentLogger))
|
|
32
|
+
return false;
|
|
33
|
+
if (isMailSyncDisabled(account, silentLogger))
|
|
34
|
+
return false;
|
|
35
|
+
if (isUnsyncableHost(account, silentLogger))
|
|
36
|
+
return false;
|
|
37
|
+
if (isAccountReauthRequired(account, silentLogger))
|
|
38
|
+
return false;
|
|
39
|
+
// An account storing no credential can only produce a sync that fails at
|
|
40
|
+
// credential resolution (issue #1120); the tick would enqueue one every
|
|
41
|
+
// tick, forever, for an account the user never finished setting up.
|
|
42
|
+
if (!hasStoredCredential(account))
|
|
43
|
+
return false;
|
|
44
|
+
return true;
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* One tick of the periodic mailbox-sync scheduler (#1247, restructured
|
|
48
|
+
* #1251). Pages through every account (never loading the whole account base
|
|
49
|
+
* into memory), decides per account via `isSyncDue` against the single
|
|
50
|
+
* offline threshold, and enqueues SYNC_MAILBOXES for the accounts that are
|
|
51
|
+
* due — in bounded-concurrency batches, never an unbounded `Promise.all`.
|
|
52
|
+
*
|
|
53
|
+
* Every enqueue goes through the same `triggerAccountSync` the manual
|
|
54
|
+
* POST /sync path uses, with a scheduler-specific, time-bucketed dedup id
|
|
55
|
+
* (`buildScheduledSyncDedupId`, bucketed by `tickIntervalMs`) so this tick
|
|
56
|
+
* can never collide with its own previous tick or with a concurrent manual
|
|
57
|
+
* trigger (see trigger-sync.ts). Concurrent-sync safety for the mailbox
|
|
58
|
+
* itself is MailboxLockService's job, inside the worker handler — this tick
|
|
59
|
+
* only decides "is a sync due" and enqueues; it never talks to IMAP.
|
|
60
|
+
*/
|
|
61
|
+
export const runSchedulerTick = async (deps) => {
|
|
62
|
+
const { accountService, queue, log, offlineIntervalMs, tickIntervalMs } = deps;
|
|
63
|
+
const now = deps.now ?? Date.now();
|
|
64
|
+
let cursor;
|
|
65
|
+
let scanned = 0;
|
|
66
|
+
let enqueued = 0;
|
|
67
|
+
let skipped = 0;
|
|
68
|
+
let swept = 0;
|
|
69
|
+
let sweepFailed = 0;
|
|
70
|
+
let redriven = 0;
|
|
71
|
+
let redriveFailed = 0;
|
|
72
|
+
do {
|
|
73
|
+
const page = await accountService.listAllAccountsPage({
|
|
74
|
+
limit: SCHEDULER_PAGE_SIZE,
|
|
75
|
+
cursor,
|
|
76
|
+
});
|
|
77
|
+
scanned += page.items.length;
|
|
78
|
+
const due = page.items.filter((account) => isEligible(account) && isSyncDue(account, now, offlineIntervalMs));
|
|
79
|
+
skipped += page.items.length - due.length;
|
|
80
|
+
await pMap(due, (account) => triggerAccountSync({
|
|
81
|
+
queue,
|
|
82
|
+
accountId: account.accountId,
|
|
83
|
+
dedupId: buildScheduledSyncDedupId(account.accountId, now, tickIntervalMs),
|
|
84
|
+
}), { concurrency: SCHEDULER_ENQUEUE_CONCURRENCY });
|
|
85
|
+
enqueued += due.length;
|
|
86
|
+
if (deps.sweepAttachments) {
|
|
87
|
+
// After the enqueue, never before it. Enqueuing mail is the tick's job
|
|
88
|
+
// and collecting bytes is housekeeping; on a hosted Lambda a slow sweep
|
|
89
|
+
// ahead of the enqueue eats the invocation budget, and a timeout is not
|
|
90
|
+
// catchable, so the remaining pages would never be enqueued at all.
|
|
91
|
+
//
|
|
92
|
+
// Every account, not only the ones due a sync: abandoned bytes have
|
|
93
|
+
// nothing to do with how recently mail was fetched. Bounded to the same
|
|
94
|
+
// concurrency as the enqueues.
|
|
95
|
+
//
|
|
96
|
+
// Contained, because this is housekeeping and enqueuing mail is not. An
|
|
97
|
+
// unreadable input or a storage permission error thrown from here would
|
|
98
|
+
// otherwise reach the loop, which logs and exits, and compose would
|
|
99
|
+
// restart the process every five seconds — no account would ever be
|
|
100
|
+
// enqueued again. A failed sweep costs a tick's collection and nothing
|
|
101
|
+
// else; the count is what makes that visible.
|
|
102
|
+
const sweep = deps.sweepAttachments;
|
|
103
|
+
const outcomes = await pMap(page.items, (account) => sweep(account).then(() => true, (error) => {
|
|
104
|
+
log.error({ accountId: account.accountId, error }, "Outbox attachment sweep failed for account");
|
|
105
|
+
return false;
|
|
106
|
+
}), { concurrency: SCHEDULER_ENQUEUE_CONCURRENCY });
|
|
107
|
+
swept += outcomes.filter(Boolean).length;
|
|
108
|
+
sweepFailed += outcomes.filter((ok) => !ok).length;
|
|
109
|
+
}
|
|
110
|
+
// Contained like the sweep, and for the same reason: a failed re-drive
|
|
111
|
+
// costs one account this tick, never the scheduler. Eligible accounts
|
|
112
|
+
// only, since a re-driven event needs a connection to settle anything.
|
|
113
|
+
const redrives = await pMap(page.items.filter(isEligible), (account) => deps.redriveStrandedPlacements(account, now).then((count) => count, (error) => {
|
|
114
|
+
log.error({ accountId: account.accountId, error }, "Stranded placement re-drive failed for account");
|
|
115
|
+
return null;
|
|
116
|
+
}), { concurrency: SCHEDULER_ENQUEUE_CONCURRENCY });
|
|
117
|
+
for (const count of redrives) {
|
|
118
|
+
if (count === null) {
|
|
119
|
+
redriveFailed += 1;
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
redriven += count;
|
|
123
|
+
}
|
|
124
|
+
cursor = page.cursor ?? undefined;
|
|
125
|
+
} while (cursor);
|
|
126
|
+
let subscriptionsRefreshed = 0;
|
|
127
|
+
let subscriptionsFailed = 0;
|
|
128
|
+
if (deps.refreshCalendarSubscriptions) {
|
|
129
|
+
// After every page is enqueued, for the same reason the sweep runs after
|
|
130
|
+
// the enqueue: a slow feed must never cost an account its mail. Contained
|
|
131
|
+
// like the sweep, so an unreadable store costs a tick's refresh and not
|
|
132
|
+
// the scheduler.
|
|
133
|
+
const tally = await deps
|
|
134
|
+
.refreshCalendarSubscriptions(now, offlineIntervalMs)
|
|
135
|
+
.catch((error) => {
|
|
136
|
+
log.error({ error }, "Calendar subscription refresh failed");
|
|
137
|
+
return { refreshed: 0, failed: 0, notDue: 0 };
|
|
138
|
+
});
|
|
139
|
+
subscriptionsRefreshed = tally.refreshed;
|
|
140
|
+
subscriptionsFailed = tally.failed;
|
|
141
|
+
}
|
|
142
|
+
log.info({
|
|
143
|
+
scanned,
|
|
144
|
+
enqueued,
|
|
145
|
+
skipped,
|
|
146
|
+
swept,
|
|
147
|
+
sweepFailed,
|
|
148
|
+
subscriptionsRefreshed,
|
|
149
|
+
subscriptionsFailed,
|
|
150
|
+
redriven,
|
|
151
|
+
redriveFailed,
|
|
152
|
+
}, "Scheduled-sync tick complete");
|
|
153
|
+
return {
|
|
154
|
+
scanned,
|
|
155
|
+
enqueued,
|
|
156
|
+
skipped,
|
|
157
|
+
swept,
|
|
158
|
+
sweepFailed,
|
|
159
|
+
subscriptionsRefreshed,
|
|
160
|
+
subscriptionsFailed,
|
|
161
|
+
redriven,
|
|
162
|
+
redriveFailed,
|
|
163
|
+
};
|
|
164
|
+
};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
3
|
+
import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
|
|
4
|
+
import { getClient } from "@remit/backend/client";
|
|
5
|
+
import { createLogger } from "@remit/logger";
|
|
6
|
+
import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
7
|
+
import { backfillSearchText } from "@remit/mailbox-service";
|
|
8
|
+
import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
|
|
9
|
+
import { lambdaEmitEvent } from "../lambda-context.js";
|
|
10
|
+
import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "../sqs-queues.js";
|
|
11
|
+
import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
|
|
12
|
+
import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
|
|
13
|
+
import { runSchedulerLoop } from "./loop.js";
|
|
14
|
+
import { buildAttachmentSweep, redriveStrandedPlacements, runSchedulerTick, } from "./tick.js";
|
|
15
|
+
installNodeLogger();
|
|
16
|
+
/**
|
|
17
|
+
* The scheduled-sync runner for every deployment that has no EventBridge: the
|
|
18
|
+
* self-host stack's `scheduler` service and the local dev compose stack. A
|
|
19
|
+
* managed deployment fires `runSchedulerTick` off an EventBridge schedule (see
|
|
20
|
+
* handler.ts); this process ticks the same function on a plain loop at the same
|
|
21
|
+
* `MAILBOX_SYNC_TICK_INTERVAL_SECONDS` cadence, so there is one scheduling
|
|
22
|
+
* implementation and one set of knobs.
|
|
23
|
+
*
|
|
24
|
+
* A tick failure crashes the process loudly rather than being swallowed —
|
|
25
|
+
* compose's `restart: unless-stopped` brings it back for the next tick.
|
|
26
|
+
*
|
|
27
|
+
* Liveness is a heartbeat file, the same mechanism as a worker's poll loop (D1
|
|
28
|
+
* of docs/design/standalone-observability.md); loop.ts carries where in the loop
|
|
29
|
+
* it is written and why. Clearing the previous generation's file here is what
|
|
30
|
+
* extends that answer across a restart. A completed round is still not the same
|
|
31
|
+
* as mail arriving: the accounts that came due are enqueued for workers to
|
|
32
|
+
* fetch, and whether that fetch lands is what
|
|
33
|
+
* `remit_account_sync_age_seconds` measures.
|
|
34
|
+
*/
|
|
35
|
+
const log = createLogger();
|
|
36
|
+
const queues = sqsWorkerQueues(workerQueueUrlsFromEnv());
|
|
37
|
+
const tickIntervalMs = getTickIntervalMs(process.env);
|
|
38
|
+
const offlineIntervalMs = getOfflineIntervalMs(process.env);
|
|
39
|
+
// A persistent failure (e.g. Postgres not up yet at container boot) throws
|
|
40
|
+
// before the loop ever reaches its own `delay`, so `restart: unless-stopped`
|
|
41
|
+
// would otherwise respawn the process immediately — a tight, log-flooding
|
|
42
|
+
// crash loop (review #1250). This fixed pause before exiting is not retry
|
|
43
|
+
// logic (there is nothing to retry here; the container restart IS the
|
|
44
|
+
// retry) — it only paces how fast that restart can happen.
|
|
45
|
+
const CRASH_BACKOFF_MS = 5_000;
|
|
46
|
+
log.info({ tickIntervalMs, offlineIntervalMs }, "Scheduled-sync runner started");
|
|
47
|
+
const runLoop = async () => {
|
|
48
|
+
// Neither call on the monitoring file may take the scheduler down with it. An
|
|
49
|
+
// unreadable or full /data/heartbeat is a reason to keep enqueuing mail, not
|
|
50
|
+
// to stop. A clear that fails leaves the previous generation's file, which
|
|
51
|
+
// ages out at the same threshold; a write that fails is the missed beat that
|
|
52
|
+
// is itself the signal. Two messages because they are different problems: one
|
|
53
|
+
// is a volume this container could not read at boot, the other a write it
|
|
54
|
+
// could not make this round.
|
|
55
|
+
const onClearError = (error) => {
|
|
56
|
+
log.error({ error }, "Scheduled-sync heartbeat clear failed");
|
|
57
|
+
};
|
|
58
|
+
const onBeatError = (error) => {
|
|
59
|
+
log.error({ error }, "Scheduled-sync heartbeat write failed");
|
|
60
|
+
};
|
|
61
|
+
await clearHeartbeats().catch(onClearError);
|
|
62
|
+
const client = await getClient();
|
|
63
|
+
const { account } = client;
|
|
64
|
+
await finishHalfDeletedAccounts({
|
|
65
|
+
account,
|
|
66
|
+
configImport: client.configImport,
|
|
67
|
+
queue: client.queues.accountFanout,
|
|
68
|
+
log,
|
|
69
|
+
});
|
|
70
|
+
const searchTextBackfill = backfillSearchText({ messageService: client.message, storageService: client.storage }, log).then((result) => {
|
|
71
|
+
log.debug({ ...result }, "Search text backfill finished");
|
|
72
|
+
});
|
|
73
|
+
const schedulerLoop = runSchedulerLoop({
|
|
74
|
+
tick: runSchedulerTick,
|
|
75
|
+
tickDeps: {
|
|
76
|
+
accountService: account,
|
|
77
|
+
queue: queues.mailboxes,
|
|
78
|
+
log,
|
|
79
|
+
tickIntervalMs,
|
|
80
|
+
offlineIntervalMs,
|
|
81
|
+
sweepAttachments: buildAttachmentSweep(client, log),
|
|
82
|
+
refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
|
|
83
|
+
redriveStrandedPlacements: (stranded, now) => redriveStrandedPlacements({
|
|
84
|
+
client,
|
|
85
|
+
emit: lambdaEmitEvent(queues, client.mailboxLock),
|
|
86
|
+
log,
|
|
87
|
+
}, stranded, now),
|
|
88
|
+
},
|
|
89
|
+
heartbeat: createHeartbeat("tick"),
|
|
90
|
+
onHeartbeatError: onBeatError,
|
|
91
|
+
tickIntervalMs,
|
|
92
|
+
});
|
|
93
|
+
await Promise.all([searchTextBackfill, schedulerLoop]);
|
|
94
|
+
};
|
|
95
|
+
runLoop()
|
|
96
|
+
.catch(async (error) => {
|
|
97
|
+
log.error({ error }, "Scheduled-sync tick failed");
|
|
98
|
+
await delay(CRASH_BACKOFF_MS);
|
|
99
|
+
})
|
|
100
|
+
.finally(() => {
|
|
101
|
+
process.exit(1);
|
|
102
|
+
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { RemitClient } from "@remit/backend/client";
|
|
2
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
|
+
import type { MessageCopyEventV3, MessageDeleteEventV3, MessageMoveEventV3, PlacementMovePushEvent } from "../events.js";
|
|
5
|
+
/** Well past three attempts on a 60 s visibility timeout. */
|
|
6
|
+
export declare const STRANDED_PLACEMENT_AFTER_MILLIS: number;
|
|
7
|
+
type Unstamped<T> = Omit<T, "eventId" | "timestamp">;
|
|
8
|
+
export type StrandedPlacementEvent = Unstamped<PlacementMovePushEvent> | Unstamped<MessageDeleteEventV3> | Unstamped<MessageMoveEventV3> | Unstamped<MessageCopyEventV3>;
|
|
9
|
+
export interface StrandedPlacementRedriveDeps {
|
|
10
|
+
client: Pick<RemitClient, "mailbox" | "mailboxSpecialUse" | "message" | "placementMove" | "threadMessage">;
|
|
11
|
+
emit: (event: StrandedPlacementEvent) => Promise<void>;
|
|
12
|
+
log: Logger;
|
|
13
|
+
}
|
|
14
|
+
/** The claim restarts the row's clock; a folder awaiting a cursor rebuild waits until it is normal. */
|
|
15
|
+
export declare const redriveStrandedPlacements: (deps: StrandedPlacementRedriveDeps, account: AccountItem, now: number) => Promise<number>;
|
|
16
|
+
export {};
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { trashMailboxAt } from "@remit/data-ports/folder-role";
|
|
2
|
+
import { deriveCopyMessageId, deriveMessageId, isValidMessageId, } from "@remit/data-ports/id";
|
|
3
|
+
import { PATHLESS_MUTATION_EVENT_SCHEMA_VERSION } from "@remit/data-ports/mutation-events";
|
|
4
|
+
import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
|
|
5
|
+
import { isCursorRebuildNeeded } from "@remit/mailbox-service";
|
|
6
|
+
/** Well past three attempts on a 60 s visibility timeout. */
|
|
7
|
+
export const STRANDED_PLACEMENT_AFTER_MILLIS = 60 * 60 * 1000;
|
|
8
|
+
const isStrandedCopy = (row) => row.status === MessageStatus.moving &&
|
|
9
|
+
row.uid === 0 &&
|
|
10
|
+
typeof row.originalUid !== "number";
|
|
11
|
+
/**
|
|
12
|
+
* The message a uid-0 copy was made from: its id derives from the Message-ID
|
|
13
|
+
* the copy's listing row carries, and the copy's own id derives from it.
|
|
14
|
+
*/
|
|
15
|
+
const copyRedriveFor = async (client, account, copy) => {
|
|
16
|
+
const listing = await client.threadMessage.findByMessageId(account.accountConfigId, copy.messageId);
|
|
17
|
+
if (!listing)
|
|
18
|
+
return { kind: "unresolvable", reason: "listing row missing" };
|
|
19
|
+
const header = listing.messageIdHeader;
|
|
20
|
+
if (!header || !isValidMessageId(header)) {
|
|
21
|
+
return { kind: "unresolvable", reason: "listing row has no Message-ID" };
|
|
22
|
+
}
|
|
23
|
+
const sourceId = deriveMessageId(account.accountId, header);
|
|
24
|
+
if (deriveCopyMessageId(sourceId, copy.mailboxId) !== copy.messageId) {
|
|
25
|
+
return {
|
|
26
|
+
kind: "unresolvable",
|
|
27
|
+
reason: "copy was not made from the message its Message-ID names",
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const [source] = await client.message.get([sourceId]);
|
|
31
|
+
if (!source)
|
|
32
|
+
return { kind: "unresolvable", reason: "source row missing" };
|
|
33
|
+
if (source.status === MessageStatus.deleted) {
|
|
34
|
+
return { kind: "unresolvable", reason: "source row deleted" };
|
|
35
|
+
}
|
|
36
|
+
if (source.status !== MessageStatus.active)
|
|
37
|
+
return { kind: "wait" };
|
|
38
|
+
return {
|
|
39
|
+
kind: "redrive",
|
|
40
|
+
event: {
|
|
41
|
+
type: "MESSAGE_COPY",
|
|
42
|
+
schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
|
|
43
|
+
accountId: account.accountId,
|
|
44
|
+
sourceMessageId: source.messageId,
|
|
45
|
+
newMessageId: copy.messageId,
|
|
46
|
+
sourceMailboxId: source.mailboxId,
|
|
47
|
+
destinationMailboxId: copy.mailboxId,
|
|
48
|
+
uid: source.uid,
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
};
|
|
52
|
+
/** A copy with no source to copy from has nothing to bind and nothing to hide. */
|
|
53
|
+
const removeUnresolvableCopy = async (deps, account, row, reason, now) => {
|
|
54
|
+
const removed = await deps.client.message.deleteIfPlacement(row.messageId, {
|
|
55
|
+
status: MessageStatus.moving,
|
|
56
|
+
mailboxId: row.mailboxId,
|
|
57
|
+
uid: 0,
|
|
58
|
+
});
|
|
59
|
+
if (!removed)
|
|
60
|
+
return false;
|
|
61
|
+
deps.log.error({
|
|
62
|
+
alert: "stranded_copy_unresolvable",
|
|
63
|
+
accountId: account.accountId,
|
|
64
|
+
messageId: row.messageId,
|
|
65
|
+
mailboxId: row.mailboxId,
|
|
66
|
+
reason,
|
|
67
|
+
untouchedForMs: now - row.updatedAt,
|
|
68
|
+
}, "A copy stranded at uid 0 has no source to copy from; the copy row was removed");
|
|
69
|
+
return true;
|
|
70
|
+
};
|
|
71
|
+
const eventFor = async (client, account, row, trashMailboxId) => {
|
|
72
|
+
const { accountId, accountConfigId } = account;
|
|
73
|
+
const { messageId } = row;
|
|
74
|
+
if (row.status === MessageStatus.deleting) {
|
|
75
|
+
return {
|
|
76
|
+
type: "MESSAGE_DELETE",
|
|
77
|
+
schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
|
|
78
|
+
accountId,
|
|
79
|
+
messageId,
|
|
80
|
+
mailboxId: row.mailboxId,
|
|
81
|
+
uid: row.uid,
|
|
82
|
+
operation: "permanent_delete",
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
if (typeof row.originalMailboxId !== "string" ||
|
|
86
|
+
typeof row.originalUid !== "number") {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
const marker = await client.placementMove.find(messageId);
|
|
90
|
+
if (marker && marker.destinationMailboxId === row.mailboxId) {
|
|
91
|
+
return {
|
|
92
|
+
type: "PLACEMENT_MOVE_PUSH",
|
|
93
|
+
accountId,
|
|
94
|
+
accountConfigId,
|
|
95
|
+
messageId,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
if (row.mailboxId === trashMailboxId) {
|
|
99
|
+
return {
|
|
100
|
+
type: "MESSAGE_DELETE",
|
|
101
|
+
schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
|
|
102
|
+
accountId,
|
|
103
|
+
messageId,
|
|
104
|
+
mailboxId: row.originalMailboxId,
|
|
105
|
+
uid: row.originalUid,
|
|
106
|
+
operation: "move_to_trash",
|
|
107
|
+
destinationMailboxId: row.mailboxId,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
type: "MESSAGE_MOVE",
|
|
112
|
+
schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
|
|
113
|
+
accountId,
|
|
114
|
+
messageId,
|
|
115
|
+
sourceMailboxId: row.originalMailboxId,
|
|
116
|
+
destinationMailboxId: row.mailboxId,
|
|
117
|
+
uid: row.originalUid,
|
|
118
|
+
};
|
|
119
|
+
};
|
|
120
|
+
/** The claim restarts the row's clock; a folder awaiting a cursor rebuild waits until it is normal. */
|
|
121
|
+
export const redriveStrandedPlacements = async (deps, account, now) => {
|
|
122
|
+
const { client, emit, log } = deps;
|
|
123
|
+
const { accountId } = account;
|
|
124
|
+
const mailboxes = await client.mailbox.listAllByAccount(accountId);
|
|
125
|
+
const paused = new Set(mailboxes
|
|
126
|
+
.filter((mailbox) => isCursorRebuildNeeded(mailbox.cursorState))
|
|
127
|
+
.map((mailbox) => mailbox.mailboxId));
|
|
128
|
+
const stranded = (await client.message.listStrandedPlacements(mailboxes.map((mailbox) => mailbox.mailboxId), now - STRANDED_PLACEMENT_AFTER_MILLIS)).filter((row) => !paused.has(row.mailboxId) &&
|
|
129
|
+
!(row.originalMailboxId && paused.has(row.originalMailboxId)));
|
|
130
|
+
if (stranded.length === 0)
|
|
131
|
+
return 0;
|
|
132
|
+
const trash = trashMailboxAt(await client.mailboxSpecialUse.resolveTrashRole(accountId), "resolved");
|
|
133
|
+
const trashMailboxId = trash.allowed ? trash.mailbox.mailboxId : undefined;
|
|
134
|
+
let redriven = 0;
|
|
135
|
+
for (const row of stranded) {
|
|
136
|
+
const copy = isStrandedCopy(row)
|
|
137
|
+
? await copyRedriveFor(client, account, row)
|
|
138
|
+
: undefined;
|
|
139
|
+
if (copy?.kind === "wait")
|
|
140
|
+
continue;
|
|
141
|
+
if (copy?.kind === "unresolvable") {
|
|
142
|
+
if (await removeUnresolvableCopy(deps, account, row, copy.reason, now)) {
|
|
143
|
+
redriven += 1;
|
|
144
|
+
}
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const event = copy?.kind === "redrive"
|
|
148
|
+
? copy.event
|
|
149
|
+
: await eventFor(client, account, row, trashMailboxId);
|
|
150
|
+
if (!event)
|
|
151
|
+
continue;
|
|
152
|
+
// A copy's earlier COPY may have landed with its answer lost, so the
|
|
153
|
+
// re-drive marks it `failed`: the copy handler then asks the destination
|
|
154
|
+
// before it copies again.
|
|
155
|
+
const claimed = await client.message.transitionPlacement(row.messageId, { status: row.status, mailboxId: row.mailboxId, uid: row.uid }, {
|
|
156
|
+
syncStatus: copy ? MessageSyncStatus.failed : MessageSyncStatus.pending,
|
|
157
|
+
});
|
|
158
|
+
if (!claimed)
|
|
159
|
+
continue;
|
|
160
|
+
await emit(event);
|
|
161
|
+
redriven += 1;
|
|
162
|
+
log.warn({
|
|
163
|
+
metric: "stranded_placement_redriven",
|
|
164
|
+
accountId,
|
|
165
|
+
messageId: row.messageId,
|
|
166
|
+
event: event.type,
|
|
167
|
+
syncStatus: row.syncStatus,
|
|
168
|
+
untouchedForMs: now - row.updatedAt,
|
|
169
|
+
}, "A move or delete had no event left to settle it; re-driven");
|
|
170
|
+
}
|
|
171
|
+
return redriven;
|
|
172
|
+
};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { createEmitEvent, type EmitDeps, type EmitEvent } from "../emit.js";
|
|
2
|
+
export { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
3
|
+
export { DEFAULT_OFFLINE_INTERVAL_MS, DEFAULT_TICK_INTERVAL_MS, } from "./config.js";
|
|
4
|
+
export { type CalendarSubscriptionTally, type RunSchedulerTickDeps, runSchedulerTick, type SchedulerTickResult, } from "./run-tick.js";
|
|
5
|
+
export { redriveStrandedPlacements, type StrandedPlacementEvent, type StrandedPlacementRedriveDeps, } from "./stranded-placement-redrive.js";
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { createEmitEvent } from "../emit.js";
|
|
2
|
+
export { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
3
|
+
export { DEFAULT_OFFLINE_INTERVAL_MS, DEFAULT_TICK_INTERVAL_MS, } from "./config.js";
|
|
4
|
+
export { runSchedulerTick, } from "./run-tick.js";
|
|
5
|
+
export { redriveStrandedPlacements, } from "./stranded-placement-redrive.js";
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { createFifoQueueProducer, createStandardQueueProducer, } from "@remit/sqs-client/producer";
|
|
2
|
+
import { env } from "expect-env";
|
|
3
|
+
export const workerQueueUrlsFromEnv = () => ({
|
|
4
|
+
mailboxes: env.SQS_QUEUE_URL_MAILBOXES,
|
|
5
|
+
messages: env.SQS_QUEUE_URL_MESSAGES,
|
|
6
|
+
flags: env.SQS_QUEUE_URL_FLAGS,
|
|
7
|
+
body: env.SQS_QUEUE_URL_BODY,
|
|
8
|
+
mailboxManagement: env.SQS_QUEUE_URL_MAILBOX_MGMT,
|
|
9
|
+
messageManagement: env.SQS_QUEUE_URL_MESSAGE_MGMT,
|
|
10
|
+
});
|
|
11
|
+
export const sqsWorkerQueues = (urls) => ({
|
|
12
|
+
mailboxes: createFifoQueueProducer({ queueUrl: urls.mailboxes }),
|
|
13
|
+
messages: createFifoQueueProducer({ queueUrl: urls.messages }),
|
|
14
|
+
flags: createFifoQueueProducer({ queueUrl: urls.flags }),
|
|
15
|
+
body: createStandardQueueProducer({ queueUrl: urls.body }),
|
|
16
|
+
mailboxManagement: createStandardQueueProducer({
|
|
17
|
+
queueUrl: urls.mailboxManagement,
|
|
18
|
+
}),
|
|
19
|
+
messageManagement: createStandardQueueProducer({
|
|
20
|
+
queueUrl: urls.messageManagement,
|
|
21
|
+
}),
|
|
22
|
+
});
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { IAccountRepository } from "@remit/data-ports";
|
|
2
|
+
import type { SecretsService } from "@remit/secrets-service";
|
|
3
|
+
import type { TokenService } from "./context.js";
|
|
4
|
+
import type { OAuthLifecycleDeps } from "./with-oauth-lifecycle.js";
|
|
5
|
+
export declare const buildLifecycleDeps: (secrets: SecretsService, accountService: IAccountRepository, tokenService: TokenService) => OAuthLifecycleDeps;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const buildLifecycleDeps = (secrets, accountService, tokenService) => ({
|
|
2
|
+
secrets,
|
|
3
|
+
tokenService,
|
|
4
|
+
persistRotatedToken: async (accountId, encryptedHash, updatedAt) => {
|
|
5
|
+
await accountService.update(accountId, {
|
|
6
|
+
oauthRefreshTokenHash: encryptedHash,
|
|
7
|
+
oauthTokenUpdatedAt: updatedAt,
|
|
8
|
+
});
|
|
9
|
+
},
|
|
10
|
+
// A state flip with no reason of its own removes the stored one. Left in
|
|
11
|
+
// place it would describe a failure the account has moved on from — the
|
|
12
|
+
// mailbox a sync could not find, still on the card beside a Reconnect
|
|
13
|
+
// button, after the token was revoked hours later.
|
|
14
|
+
updateConnectionState: async (accountId, state, lastError) => {
|
|
15
|
+
await accountService.update(accountId, { connectionState: state, lastError }, lastError === undefined ? ["lastError"] : undefined);
|
|
16
|
+
},
|
|
17
|
+
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* with-oauth-lifecycle.ts
|
|
3
|
+
*
|
|
4
|
+
* Centralizes the OAuth reauth/ACK contract shared by every IMAP handler.
|
|
5
|
+
*
|
|
6
|
+
* The contract (see issue #472):
|
|
7
|
+
* - If the account already requires reauth, skip all IMAP traffic entirely.
|
|
8
|
+
* - Credential resolution AND the per-handler work both run inside one
|
|
9
|
+
* try/catch so that a revoked OAuth token is caught regardless of where it
|
|
10
|
+
* surfaces (token mint vs. first IMAP command).
|
|
11
|
+
* - An account storing no credential at all is terminal, not transient: the
|
|
12
|
+
* account is flipped to the state that describes it and the event is ACKed.
|
|
13
|
+
* Retrying instead fenced the account's FIFO group for the message's whole
|
|
14
|
+
* visibility timeout, blocking the sync that follows the credential the
|
|
15
|
+
* user was still typing (issue #1120).
|
|
16
|
+
* - On a terminal auth failure (RefreshTokenError reauth-required, or
|
|
17
|
+
* MailConnectionError auth), flip the account to reauth_required and return
|
|
18
|
+
* WITHOUT rethrowing — this ACKs the SQS message so it is not retried.
|
|
19
|
+
* An IMAP refusal is stored with it, because not every refusal a mail server
|
|
20
|
+
* hands back is one re-authenticating clears: "SmtpClientAuthentication is
|
|
21
|
+
* disabled" needs a tenant setting changed, and an account card that only
|
|
22
|
+
* ever says "Re-authentication required" sends the user around a loop.
|
|
23
|
+
* - On transient / config / network errors, rethrow so SQS retries with
|
|
24
|
+
* backoff (let-it-crash).
|
|
25
|
+
*
|
|
26
|
+
* Tokens must NEVER appear in logs — only accountId / errorKind / errorCode.
|
|
27
|
+
*/
|
|
28
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
29
|
+
import type { Logger } from "@remit/logger";
|
|
30
|
+
import { type AccountCredentialsDeps, type ConnectionStateValue, type CredentialResolution } from "@remit/mailbox-service/account-credentials";
|
|
31
|
+
import { type MailCredentials } from "@remit/mailbox-service/types";
|
|
32
|
+
export type { ConnectionStateValue };
|
|
33
|
+
export interface OAuthLifecycleDeps extends AccountCredentialsDeps {
|
|
34
|
+
/**
|
|
35
|
+
* Persist the account's connectionState, and with it the sentence the
|
|
36
|
+
* account card shows for the failure. `lastError` carries the server's own
|
|
37
|
+
* words when the failure came with any; it is absent when nothing was said,
|
|
38
|
+
* as for a credential that was never stored or a token simply revoked, and
|
|
39
|
+
* the stored reason is then cleared rather than left to describe an older
|
|
40
|
+
* failure the account has moved on from.
|
|
41
|
+
*/
|
|
42
|
+
updateConnectionState: (accountId: string, state: ConnectionStateValue, lastError?: string) => Promise<void>;
|
|
43
|
+
/**
|
|
44
|
+
* Resolve credentials for the account. Defaults to
|
|
45
|
+
* resolveConnectionCredentials; overridable for testing. Kept as the ONLY
|
|
46
|
+
* authType branch in the codebase (see account-credentials.ts).
|
|
47
|
+
*/
|
|
48
|
+
resolveCredentials?: (account: AccountItem, deps: AccountCredentialsDeps) => Promise<CredentialResolution>;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Run `work` for an account under the shared OAuth reauth/ACK contract.
|
|
52
|
+
*
|
|
53
|
+
* `work` receives the resolved MailCredentials. Both credential resolution and
|
|
54
|
+
* `work` run inside the same try/catch.
|
|
55
|
+
*/
|
|
56
|
+
export declare const withOAuthLifecycle: (deps: OAuthLifecycleDeps, account: AccountItem, log: Logger, work: (credentials: MailCredentials) => Promise<void>) => Promise<void>;
|