@remit/imap-worker 0.0.90 → 0.0.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/account-check.d.ts +27 -0
- package/dist/account-check.js +70 -0
- package/dist/body-sync-gate.d.ts +12 -0
- package/dist/body-sync-gate.js +53 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +193 -0
- package/dist/config-env.d.ts +5 -0
- package/dist/config-env.js +37 -0
- package/dist/connection-pool.d.ts +22 -0
- package/dist/connection-pool.js +97 -0
- package/dist/connection-scope.d.ts +8 -0
- package/dist/connection-scope.js +10 -0
- package/dist/context.d.ts +46 -0
- package/dist/context.js +15 -0
- package/dist/e2e-processor-shim.d.ts +2 -0
- package/dist/e2e-processor-shim.js +181 -0
- package/dist/emit.d.ts +15 -0
- package/dist/emit.js +76 -0
- package/dist/events.d.ts +203 -0
- package/dist/events.js +1 -0
- package/dist/failure-kind.d.ts +10 -0
- package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
- package/dist/handlers/append-sent-message.d.ts +3 -0
- package/dist/handlers/append-sent-message.js +152 -0
- package/dist/handlers/delete-account-objects.d.ts +3 -0
- package/dist/handlers/delete-account-objects.js +35 -0
- package/dist/handlers/empty-trash.d.ts +14 -0
- package/dist/handlers/empty-trash.js +217 -0
- package/dist/handlers/flag-push.d.ts +38 -0
- package/dist/handlers/flag-push.js +195 -0
- package/dist/handlers/imported-folder.d.ts +4 -0
- package/dist/handlers/imported-folder.js +23 -0
- package/dist/handlers/mailbox-management.d.ts +6 -0
- package/dist/handlers/mailbox-management.js +294 -0
- package/dist/handlers/mailbox-sync-order.d.ts +21 -0
- package/dist/handlers/mailbox-sync-order.js +50 -0
- package/dist/handlers/message-copy.d.ts +40 -0
- package/dist/handlers/message-copy.js +371 -0
- package/dist/handlers/message-delete-terminal.d.ts +60 -0
- package/dist/handlers/message-delete-terminal.js +77 -0
- package/dist/handlers/message-delete.d.ts +57 -0
- package/dist/handlers/message-delete.js +520 -0
- package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
- package/dist/handlers/message-move-terminal.js +94 -0
- package/dist/handlers/message-move.d.ts +146 -0
- package/dist/handlers/message-move.js +490 -0
- package/dist/handlers/placement-move-push.d.ts +62 -0
- package/dist/handlers/placement-move-push.js +307 -0
- package/dist/handlers/probe-message-id.d.ts +2 -0
- package/dist/handlers/probe-message-id.js +6 -0
- package/dist/handlers/sync-mailboxes.d.ts +45 -0
- package/dist/handlers/sync-mailboxes.js +233 -0
- package/dist/handlers/sync-message-body.d.ts +23 -0
- package/dist/handlers/sync-message-body.js +182 -0
- package/dist/handlers/sync-messages.d.ts +29 -0
- package/dist/handlers/sync-messages.js +285 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +67 -0
- package/dist/lambda-context.d.ts +8 -0
- package/dist/lambda-context.js +33 -0
- package/dist/mailbox-row.d.ts +9 -0
- package/dist/mailbox-row.js +6 -0
- package/dist/poller.d.ts +1 -0
- package/dist/poller.js +51 -0
- package/dist/process.d.ts +6 -0
- package/dist/process.js +3 -0
- package/dist/processor.d.ts +3 -0
- package/dist/processor.js +44 -0
- package/dist/queues.d.ts +10 -0
- package/dist/queues.js +1 -0
- package/dist/s3-account-objects.d.ts +3 -0
- package/dist/s3-account-objects.js +26 -0
- package/dist/scheduler/attachment-sweep.d.ts +11 -0
- package/dist/scheduler/attachment-sweep.js +23 -0
- package/dist/scheduler/calendar-subscriptions.d.ts +26 -0
- package/dist/scheduler/calendar-subscriptions.js +35 -0
- package/dist/scheduler/config.d.ts +27 -0
- package/{src/scheduler/config.ts → dist/scheduler/config.js} +7 -26
- package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
- package/dist/scheduler/decide-due.js +17 -0
- package/dist/scheduler/handler.d.ts +21 -0
- package/{src/scheduler/handler.ts → dist/scheduler/handler.js} +25 -42
- package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
- package/dist/scheduler/loop.js +8 -0
- package/dist/scheduler/run-tick.d.ts +62 -0
- package/dist/scheduler/run-tick.js +164 -0
- package/dist/scheduler/runner.d.ts +2 -0
- package/{src/scheduler/runner.ts → dist/scheduler/runner.js} +57 -78
- package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
- package/dist/scheduler/stranded-placement-redrive.js +172 -0
- package/dist/sqs-queues.d.ts +4 -0
- package/dist/sqs-queues.js +22 -0
- package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
- package/dist/with-oauth-lifecycle-deps.js +17 -0
- package/dist/with-oauth-lifecycle.d.ts +56 -0
- package/dist/with-oauth-lifecycle.js +92 -0
- package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
- package/dist/worker-version.js +12 -0
- package/package.json +75 -69
- package/src/account-check.test.ts +0 -149
- package/src/account-check.ts +0 -101
- package/src/body-sync-gate.test.ts +0 -185
- package/src/body-sync-gate.ts +0 -86
- package/src/cli.ts +0 -218
- package/src/connection-scope.test.ts +0 -266
- package/src/connection-scope.ts +0 -314
- package/src/credential-less-sync-fence.test.ts +0 -152
- package/src/e2e-processor-shim.ts +0 -232
- package/src/emit.test.ts +0 -145
- package/src/emit.ts +0 -115
- package/src/events.ts +0 -252
- package/src/failure-kind.test.ts +0 -42
- package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
- package/src/handlers/append-sent-message.ts +0 -264
- package/src/handlers/delete-account-objects.test.ts +0 -126
- package/src/handlers/delete-account-objects.ts +0 -96
- package/src/handlers/empty-trash.sqlite.test.ts +0 -209
- package/src/handlers/empty-trash.ts +0 -352
- package/src/handlers/flag-push.ts +0 -285
- package/src/handlers/imported-folder.test.ts +0 -63
- package/src/handlers/imported-folder.ts +0 -38
- package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
- package/src/handlers/mailbox-management.ts +0 -486
- package/src/handlers/mailbox-sync-order.test.ts +0 -93
- package/src/handlers/mailbox-sync-order.ts +0 -65
- package/src/handlers/message-copy.sqlite.test.ts +0 -298
- package/src/handlers/message-copy.ts +0 -569
- package/src/handlers/message-delete-terminal.ts +0 -147
- package/src/handlers/message-delete.sqlite.test.ts +0 -291
- package/src/handlers/message-delete.ts +0 -888
- package/src/handlers/message-move.sqlite.test.ts +0 -271
- package/src/handlers/message-move.test.ts +0 -216
- package/src/handlers/message-move.ts +0 -755
- package/src/handlers/placement-move-push.test.ts +0 -137
- package/src/handlers/placement-move-push.ts +0 -478
- package/src/handlers/placement-push.sqlite.test.ts +0 -299
- package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
- package/src/handlers/probe-message-id.ts +0 -15
- package/src/handlers/sync-mailboxes.test.ts +0 -183
- package/src/handlers/sync-mailboxes.ts +0 -425
- package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
- package/src/handlers/sync-message-body.test.ts +0 -150
- package/src/handlers/sync-message-body.ts +0 -366
- package/src/handlers/sync-messages.sqlite.test.ts +0 -195
- package/src/handlers/sync-messages.ts +0 -537
- package/src/index.test.ts +0 -22
- package/src/index.ts +0 -82
- package/src/mail-sync-selection.sqlite.test.ts +0 -227
- package/src/mailbox-row.ts +0 -20
- package/src/pending-sync.sqlite.test.ts +0 -165
- package/src/poller.ts +0 -55
- package/src/processor.ts +0 -62
- package/src/queues.ts +0 -40
- package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -74
- package/src/scheduler/attachment-sweep.ts +0 -46
- package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
- package/src/scheduler/calendar-subscriptions.ts +0 -103
- package/src/scheduler/config.test.ts +0 -40
- package/src/scheduler/decide-due.test.ts +0 -44
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
- package/src/scheduler/loop.test.ts +0 -143
- package/src/scheduler/run-tick.sqlite.test.ts +0 -191
- package/src/scheduler/run-tick.ts +0 -281
- package/src/scheduler/stranded-placement-redrive.ts +0 -265
- package/src/stranded-placement.sqlite.test.ts +0 -799
- package/src/test-helpers/test-world.ts +0 -425
- package/src/test-helpers/worker-queues.ts +0 -11
- package/src/thread-join.sqlite.test.ts +0 -106
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
- package/src/with-oauth-lifecycle-deps.ts +0 -71
- package/src/with-oauth-lifecycle.ts +0 -158
- package/tsconfig.json +0 -8
package/dist/poller.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createLogger } from "@remit/logger";
|
|
2
|
+
import { startMetricsServer } from "@remit/logger-lambda/metrics";
|
|
3
|
+
import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
4
|
+
import { runQueuePoller } from "@remit/sqs-client/poller";
|
|
5
|
+
import { env } from "expect-env";
|
|
6
|
+
import { handler } from "./index.js";
|
|
7
|
+
installNodeLogger();
|
|
8
|
+
/**
|
|
9
|
+
* Production queue poller — the deployed form of `e2e-processor-shim.ts`.
|
|
10
|
+
* Polls every imap-worker queue and invokes the production Lambda handler.
|
|
11
|
+
*/
|
|
12
|
+
const log = createLogger();
|
|
13
|
+
// /metrics and nothing else, on the compose network (standalone-observability
|
|
14
|
+
// D2). No health route on it: worker liveness is a heartbeat file, which keeps
|
|
15
|
+
// answering when this server does not.
|
|
16
|
+
startMetricsServer();
|
|
17
|
+
await runQueuePoller({
|
|
18
|
+
log,
|
|
19
|
+
targets: [
|
|
20
|
+
{
|
|
21
|
+
queueUrl: env.SQS_QUEUE_URL_MAILBOXES,
|
|
22
|
+
handler,
|
|
23
|
+
functionName: "imap-worker-mailboxes",
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
queueUrl: env.SQS_QUEUE_URL_MESSAGES,
|
|
27
|
+
handler,
|
|
28
|
+
functionName: "imap-worker-messages",
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
queueUrl: env.SQS_QUEUE_URL_FLAGS,
|
|
32
|
+
handler,
|
|
33
|
+
functionName: "imap-worker-flags",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
queueUrl: env.SQS_QUEUE_URL_BODY,
|
|
37
|
+
handler,
|
|
38
|
+
functionName: "imap-worker-body",
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
queueUrl: env.SQS_QUEUE_URL_MAILBOX_MGMT,
|
|
42
|
+
handler,
|
|
43
|
+
functionName: "imap-worker-mailbox-mgmt",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
queueUrl: env.SQS_QUEUE_URL_MESSAGE_MGMT,
|
|
47
|
+
handler,
|
|
48
|
+
functionName: "imap-worker-message-mgmt",
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { ConnectionScope } from "@remit/mailbox-service";
|
|
2
|
+
export { type BorrowedConnection, type ConnectImap, type ConnectionPool, type ConnectionPoolOptions, createConnectionPool, type PoolAccount, type PoolOverflow, } from "./connection-pool.js";
|
|
3
|
+
export { type AccountObjectPage, type AccountObjectStore, type AttemptBudgets, DEFAULT_MAILBOX_FRESHNESS_MS, defaultImapWorkerConfig, type ImapFailureKind, type ImapWorkerConfig, type ImapWorkerQueueName, type ProcessContext, type TokenService, type WorkerMetrics, } from "./context.js";
|
|
4
|
+
export type { ImapEvent, WorkerEvent } from "./events.js";
|
|
5
|
+
export { processEvent } from "./processor.js";
|
|
6
|
+
export type { WorkerQueues } from "./queues.js";
|
package/dist/process.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
|
|
2
|
+
import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
|
|
3
|
+
import { handleEmptyTrash } from "./handlers/empty-trash.js";
|
|
4
|
+
import { handleFlagPush } from "./handlers/flag-push.js";
|
|
5
|
+
import { processMailboxManagement } from "./handlers/mailbox-management.js";
|
|
6
|
+
import { handleMessageCopy } from "./handlers/message-copy.js";
|
|
7
|
+
import { handleMessageDelete } from "./handlers/message-delete.js";
|
|
8
|
+
import { handleMessageMove } from "./handlers/message-move.js";
|
|
9
|
+
import { handlePlacementMovePush } from "./handlers/placement-move-push.js";
|
|
10
|
+
import { syncMailboxes } from "./handlers/sync-mailboxes.js";
|
|
11
|
+
import { syncMessageBody } from "./handlers/sync-message-body.js";
|
|
12
|
+
import { syncMessages } from "./handlers/sync-messages.js";
|
|
13
|
+
export const processEvent = async (event, ctx, receiveCount) => {
|
|
14
|
+
switch (event.type) {
|
|
15
|
+
case "SYNC_MAILBOXES":
|
|
16
|
+
return syncMailboxes(event, ctx);
|
|
17
|
+
case "SYNC_MESSAGES":
|
|
18
|
+
return syncMessages(event, ctx);
|
|
19
|
+
case "SYNC_MESSAGE_BODY":
|
|
20
|
+
return syncMessageBody(event, ctx, receiveCount);
|
|
21
|
+
case "MAILBOX_CREATE":
|
|
22
|
+
case "MAILBOX_RENAME":
|
|
23
|
+
case "MAILBOX_DELETE":
|
|
24
|
+
return processMailboxManagement(event, ctx);
|
|
25
|
+
case "MESSAGE_DELETE":
|
|
26
|
+
return handleMessageDelete(event, ctx, receiveCount);
|
|
27
|
+
case "MESSAGE_MOVE":
|
|
28
|
+
return handleMessageMove(event, ctx, receiveCount);
|
|
29
|
+
case "PLACEMENT_MOVE_PUSH":
|
|
30
|
+
return handlePlacementMovePush(event, ctx, receiveCount);
|
|
31
|
+
case "FLAG_PUSH":
|
|
32
|
+
return handleFlagPush(event, ctx, receiveCount);
|
|
33
|
+
case "MESSAGE_COPY":
|
|
34
|
+
return handleMessageCopy(event, ctx, receiveCount);
|
|
35
|
+
case "EMPTY_TRASH":
|
|
36
|
+
return handleEmptyTrash(event, ctx);
|
|
37
|
+
case "APPEND_SENT_MESSAGE":
|
|
38
|
+
return handleAppendSentMessage(event, ctx, receiveCount);
|
|
39
|
+
case "DELETE_ACCOUNT_OBJECTS":
|
|
40
|
+
return handleDeleteAccountObjects(event, ctx);
|
|
41
|
+
default:
|
|
42
|
+
throw new Error(`Unknown event type: ${event.type}`);
|
|
43
|
+
}
|
|
44
|
+
};
|
package/dist/queues.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
|
|
2
|
+
import type { ImapEvent, WorkerEvent } from "./events.js";
|
|
3
|
+
export interface WorkerQueues {
|
|
4
|
+
mailboxes: FifoQueueProducer<ImapEvent>;
|
|
5
|
+
messages: FifoQueueProducer<ImapEvent>;
|
|
6
|
+
flags: FifoQueueProducer<ImapEvent>;
|
|
7
|
+
body: QueueProducer<ImapEvent>;
|
|
8
|
+
mailboxManagement: QueueProducer<ImapEvent>;
|
|
9
|
+
messageManagement: QueueProducer<WorkerEvent>;
|
|
10
|
+
}
|
package/dist/queues.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { DeleteObjectsCommand, ListObjectsV2Command, S3Client, } from "@aws-sdk/client-s3";
|
|
2
|
+
const PAGE_SIZE = 1_000;
|
|
3
|
+
export const s3AccountObjects = (bucket, s3 = new S3Client({})) => ({
|
|
4
|
+
list: async (prefix, continuationToken) => {
|
|
5
|
+
const result = await s3.send(new ListObjectsV2Command({
|
|
6
|
+
Bucket: bucket(),
|
|
7
|
+
Prefix: prefix,
|
|
8
|
+
MaxKeys: PAGE_SIZE,
|
|
9
|
+
ContinuationToken: continuationToken,
|
|
10
|
+
}));
|
|
11
|
+
return {
|
|
12
|
+
keys: (result.Contents ?? [])
|
|
13
|
+
.map((object) => object.Key)
|
|
14
|
+
.filter((key) => key !== undefined),
|
|
15
|
+
nextContinuationToken: result.IsTruncated
|
|
16
|
+
? result.NextContinuationToken
|
|
17
|
+
: undefined,
|
|
18
|
+
};
|
|
19
|
+
},
|
|
20
|
+
delete: async (keys) => {
|
|
21
|
+
await s3.send(new DeleteObjectsCommand({
|
|
22
|
+
Bucket: bucket(),
|
|
23
|
+
Delete: { Objects: keys.map((Key) => ({ Key })), Quiet: true },
|
|
24
|
+
}));
|
|
25
|
+
},
|
|
26
|
+
});
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RemitClient } from "@remit/backend/client";
|
|
2
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
|
+
/**
|
|
5
|
+
* Collect attachment bytes the database does not know about. Both scheduler
|
|
6
|
+
* entry points build it the same way, so the hosted tick and the self-host loop
|
|
7
|
+
* collect identically — this is the only defence on the hosted side, where the
|
|
8
|
+
* browser PUTs straight to block storage and nothing consults the database on
|
|
9
|
+
* the way in.
|
|
10
|
+
*/
|
|
11
|
+
export declare const buildAttachmentSweep: (client: Pick<RemitClient, "storage" | "outboxAttachment">, log: Logger) => ((account: AccountItem) => Promise<void>);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
|
|
2
|
+
/**
|
|
3
|
+
* Collect attachment bytes the database does not know about. Both scheduler
|
|
4
|
+
* entry points build it the same way, so the hosted tick and the self-host loop
|
|
5
|
+
* collect identically — this is the only defence on the hosted side, where the
|
|
6
|
+
* browser PUTs straight to block storage and nothing consults the database on
|
|
7
|
+
* the way in.
|
|
8
|
+
*/
|
|
9
|
+
export const buildAttachmentSweep = (client, log) => async (account) => {
|
|
10
|
+
const { deleted, skipped } = await sweepAbandonedOutboxAttachments({
|
|
11
|
+
storage: client.storage,
|
|
12
|
+
// Reaps the draft's lapsed reservations first, so a row that has
|
|
13
|
+
// stopped holding room stops vouching for its bytes too and the
|
|
14
|
+
// object is collected on this same pass.
|
|
15
|
+
listKnownAttachmentIds: (accountConfigId, outboxMessageId) => client.outboxAttachment.reapAndListLive(accountConfigId, outboxMessageId),
|
|
16
|
+
onSkipped: (outboxMessageId, error) => {
|
|
17
|
+
log.error({ outboxMessageId, error }, "Left a draft's attachment objects alone: its rows could not be read");
|
|
18
|
+
},
|
|
19
|
+
}, account.accountConfigId, account.accountId);
|
|
20
|
+
if (deleted > 0 || skipped > 0) {
|
|
21
|
+
log.warn({ accountId: account.accountId, deleted, skipped }, "Outbox attachment sweep");
|
|
22
|
+
}
|
|
23
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { RemitClient } from "@remit/backend/client";
|
|
2
|
+
import { type CalendarFeedFetcher } from "@remit/calendar-service";
|
|
3
|
+
import type { ICalendarCollectionRepository, ICalendarStore } from "@remit/data-ports";
|
|
4
|
+
import type { Logger } from "@remit/logger";
|
|
5
|
+
export interface CalendarSubscriptionTally {
|
|
6
|
+
refreshed: number;
|
|
7
|
+
failed: number;
|
|
8
|
+
notDue: number;
|
|
9
|
+
}
|
|
10
|
+
export interface CalendarSubscriptionRefreshDeps {
|
|
11
|
+
calendarCollection: Pick<ICalendarCollectionRepository, "listEnabledSubscriptions">;
|
|
12
|
+
calendarStore: ICalendarStore;
|
|
13
|
+
log: Logger;
|
|
14
|
+
fetcher?: CalendarFeedFetcher;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Re-reads every enabled subscription whose last fetch is at least
|
|
18
|
+
* `intervalMs` old. A disabled subscription is never listed, which is what
|
|
19
|
+
* stops its refresh while leaving its events in place.
|
|
20
|
+
*
|
|
21
|
+
* One feed's failure is recorded on its own collection and counted here; it
|
|
22
|
+
* never stops the others. The log names the calendar and the reason, never the
|
|
23
|
+
* URL, which for a Google secret address is the whole credential.
|
|
24
|
+
*/
|
|
25
|
+
export declare const refreshDueCalendarSubscriptions: (deps: CalendarSubscriptionRefreshDeps, now: number, intervalMs: number) => Promise<CalendarSubscriptionTally>;
|
|
26
|
+
export declare const buildCalendarSubscriptionRefresh: (client: Pick<RemitClient, "calendarCollection" | "calendarStore">, log: Logger) => ((now: number, intervalMs: number) => Promise<CalendarSubscriptionTally>);
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { isSubscriptionDue, refreshCalendarSubscription, } from "@remit/calendar-service";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
const REFRESH_CONCURRENCY = 4;
|
|
4
|
+
/**
|
|
5
|
+
* Re-reads every enabled subscription whose last fetch is at least
|
|
6
|
+
* `intervalMs` old. A disabled subscription is never listed, which is what
|
|
7
|
+
* stops its refresh while leaving its events in place.
|
|
8
|
+
*
|
|
9
|
+
* One feed's failure is recorded on its own collection and counted here; it
|
|
10
|
+
* never stops the others. The log names the calendar and the reason, never the
|
|
11
|
+
* URL, which for a Google secret address is the whole credential.
|
|
12
|
+
*/
|
|
13
|
+
export const refreshDueCalendarSubscriptions = async (deps, now, intervalMs) => {
|
|
14
|
+
const enabled = await deps.calendarCollection.listEnabledSubscriptions();
|
|
15
|
+
const due = enabled.filter((collection) => isSubscriptionDue(collection, now, intervalMs));
|
|
16
|
+
const outcomes = await pMap(due, (collection) => refreshCalendarSubscription(deps.calendarStore, collection, now, deps.fetcher).then((refresh) => {
|
|
17
|
+
if (refresh.ok)
|
|
18
|
+
return true;
|
|
19
|
+
deps.log.warn({ calendarId: collection.calendarId, reason: refresh.reason }, "Calendar subscription could not be refreshed");
|
|
20
|
+
return false;
|
|
21
|
+
}, (error) => {
|
|
22
|
+
deps.log.error({ calendarId: collection.calendarId, error }, "Calendar subscription refresh failed");
|
|
23
|
+
return false;
|
|
24
|
+
}), { concurrency: REFRESH_CONCURRENCY });
|
|
25
|
+
return {
|
|
26
|
+
refreshed: outcomes.filter(Boolean).length,
|
|
27
|
+
failed: outcomes.filter((ok) => !ok).length,
|
|
28
|
+
notDue: enabled.length - due.length,
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
export const buildCalendarSubscriptionRefresh = (client, log) => (now, intervalMs) => refreshDueCalendarSubscriptions({
|
|
32
|
+
calendarCollection: client.calendarCollection,
|
|
33
|
+
calendarStore: client.calendarStore,
|
|
34
|
+
log,
|
|
35
|
+
}, now, intervalMs);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Periodic mailbox-sync scheduler (#1247, restructured #1251): the tick rate
|
|
3
|
+
* and the offline-sync threshold are separate, independently configured
|
|
4
|
+
* knobs — never hardcode a cadence — following the repo's
|
|
5
|
+
* env-var-with-a-safe-default convention (see `METRICS_PORT` in
|
|
6
|
+
* remit-logger-lambda/src/metrics.ts).
|
|
7
|
+
*
|
|
8
|
+
* `tickIntervalSeconds` drives how often the tick itself runs — the
|
|
9
|
+
* EventBridge schedule rate on a managed deployment, the runner's loop delay
|
|
10
|
+
* everywhere else — and must stay well below `offlineIntervalSeconds` so a tick
|
|
11
|
+
* reliably observes every account crossing the threshold. CDK and this runtime
|
|
12
|
+
* default must agree — see infra/lib/config.ts's `mailboxSync` stage config.
|
|
13
|
+
*
|
|
14
|
+
* `offlineIntervalSeconds` is the only due-ness threshold: an account is due
|
|
15
|
+
* once its last successful sync is older than this interval. There is no
|
|
16
|
+
* "online" tier — client-side polling (useStaleAccountSync) covers an
|
|
17
|
+
* account while its mail is actively open in the web client.
|
|
18
|
+
*
|
|
19
|
+
* The defaults below are a floor for a deployment that sets neither. The
|
|
20
|
+
* standalone stack sets both, at a cadence sized for one box and a handful of
|
|
21
|
+
* accounts, and the checker's stall threshold is derived from those values —
|
|
22
|
+
* see deploy/vps/docker-compose.sqlite.yml's `scheduler` service.
|
|
23
|
+
*/
|
|
24
|
+
export declare const getTickIntervalMs: (env?: NodeJS.ProcessEnv) => number;
|
|
25
|
+
export declare const getOfflineIntervalMs: (env?: NodeJS.ProcessEnv) => number;
|
|
26
|
+
export declare const SCHEDULER_PAGE_SIZE = 100;
|
|
27
|
+
export declare const SCHEDULER_ENQUEUE_CONCURRENCY = 10;
|
|
@@ -21,35 +21,16 @@
|
|
|
21
21
|
* accounts, and the checker's stall threshold is derived from those values —
|
|
22
22
|
* see deploy/vps/docker-compose.sqlite.yml's `scheduler` service.
|
|
23
23
|
*/
|
|
24
|
-
|
|
25
24
|
const DEFAULT_TICK_INTERVAL_SECONDS = 60 * 60; // 1 hour
|
|
26
25
|
const DEFAULT_OFFLINE_INTERVAL_SECONDS = 12 * 60 * 60; // 12 hours
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
)
|
|
32
|
-
if (!raw) return fallback;
|
|
33
|
-
const parsed = Number.parseInt(raw, 10);
|
|
34
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
26
|
+
const parsePositiveIntSeconds = (raw, fallback) => {
|
|
27
|
+
if (!raw)
|
|
28
|
+
return fallback;
|
|
29
|
+
const parsed = Number.parseInt(raw, 10);
|
|
30
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
35
31
|
};
|
|
36
|
-
|
|
37
|
-
export const
|
|
38
|
-
env: NodeJS.ProcessEnv = process.env,
|
|
39
|
-
): number =>
|
|
40
|
-
parsePositiveIntSeconds(
|
|
41
|
-
env.MAILBOX_SYNC_TICK_INTERVAL_SECONDS,
|
|
42
|
-
DEFAULT_TICK_INTERVAL_SECONDS,
|
|
43
|
-
) * 1000;
|
|
44
|
-
|
|
45
|
-
export const getOfflineIntervalMs = (
|
|
46
|
-
env: NodeJS.ProcessEnv = process.env,
|
|
47
|
-
): number =>
|
|
48
|
-
parsePositiveIntSeconds(
|
|
49
|
-
env.MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS,
|
|
50
|
-
DEFAULT_OFFLINE_INTERVAL_SECONDS,
|
|
51
|
-
) * 1000;
|
|
52
|
-
|
|
32
|
+
export const getTickIntervalMs = (env = process.env) => parsePositiveIntSeconds(env.MAILBOX_SYNC_TICK_INTERVAL_SECONDS, DEFAULT_TICK_INTERVAL_SECONDS) * 1000;
|
|
33
|
+
export const getOfflineIntervalMs = (env = process.env) => parsePositiveIntSeconds(env.MAILBOX_SYNC_OFFLINE_INTERVAL_SECONDS, DEFAULT_OFFLINE_INTERVAL_SECONDS) * 1000;
|
|
53
34
|
// Page size + enqueue concurrency are implementation details of the tick
|
|
54
35
|
// itself, not stage config — unlike the two intervals above, nothing outside
|
|
55
36
|
// this worker needs to agree with them.
|
|
@@ -10,17 +10,8 @@
|
|
|
10
10
|
* misses an account by a few seconds simply catches it on the next tick,
|
|
11
11
|
* which arrives long before the threshold matters again.
|
|
12
12
|
*/
|
|
13
|
-
|
|
14
13
|
export interface SchedulableAccount {
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
readonly accountId: string;
|
|
15
|
+
readonly lastSyncAt?: number;
|
|
17
16
|
}
|
|
18
|
-
|
|
19
|
-
export const isSyncDue = (
|
|
20
|
-
account: SchedulableAccount,
|
|
21
|
-
now: number,
|
|
22
|
-
offlineIntervalMs: number,
|
|
23
|
-
): boolean => {
|
|
24
|
-
if (!account.lastSyncAt) return true;
|
|
25
|
-
return now - account.lastSyncAt >= offlineIntervalMs;
|
|
26
|
-
};
|
|
17
|
+
export declare const isSyncDue: (account: SchedulableAccount, now: number, offlineIntervalMs: number) => boolean;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure scheduling decision for the periodic mailbox-sync tick (#1247,
|
|
3
|
+
* restructured #1251).
|
|
4
|
+
*
|
|
5
|
+
* One tier: an account is due once its last successful sync is older than
|
|
6
|
+
* `offlineIntervalMs`. An account that has never synced is always due. The
|
|
7
|
+
* tick interval is decoupled from this threshold and runs far more often
|
|
8
|
+
* than `offlineIntervalMs` (see config.ts), so — unlike the prior two-tier
|
|
9
|
+
* design — no slack is needed to compensate for sampling lag: a tick that
|
|
10
|
+
* misses an account by a few seconds simply catches it on the next tick,
|
|
11
|
+
* which arrives long before the threshold matters again.
|
|
12
|
+
*/
|
|
13
|
+
export const isSyncDue = (account, now, offlineIntervalMs) => {
|
|
14
|
+
if (!account.lastSyncAt)
|
|
15
|
+
return true;
|
|
16
|
+
return now - account.lastSyncAt >= offlineIntervalMs;
|
|
17
|
+
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ScheduledHandler } from "aws-lambda";
|
|
2
|
+
import type { WorkerQueues } from "../queues.js";
|
|
3
|
+
/**
|
|
4
|
+
* EventBridge-scheduled entry point for the periodic mailbox-sync tick
|
|
5
|
+
* (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
|
|
6
|
+
* (rate schedule, wired in infra/stacks/dev/stacks/remit-worker-stack.ts) and
|
|
7
|
+
* delegates the actual decision + enqueue to `runSchedulerTick` — the same
|
|
8
|
+
* function the timer loop calls on a deployment with no EventBridge (see
|
|
9
|
+
* `runner.ts`), so every deployment runs one code path.
|
|
10
|
+
*
|
|
11
|
+
* Uses the EventBridge event's own `time` (the scheduled fire time) as the
|
|
12
|
+
* tick's `now`, rather than `Date.now()` at processing time. This is what
|
|
13
|
+
* keeps `buildScheduledSyncDedupId`'s time bucket aligned to the schedule
|
|
14
|
+
* instead of wall-clock/Lambda-cold-start jitter (review #1250): every
|
|
15
|
+
* distinct scheduled firing gets a `time` exactly `tickIntervalMs` apart
|
|
16
|
+
* from the last, so consecutive ticks always land in different buckets,
|
|
17
|
+
* while a genuine retry of the same invocation carries the same `time` and
|
|
18
|
+
* correctly dedupes.
|
|
19
|
+
*/
|
|
20
|
+
export declare const buildScheduledHandler: (queues: WorkerQueues) => ScheduledHandler;
|
|
21
|
+
export declare const handler: ScheduledHandler;
|
|
@@ -1,23 +1,15 @@
|
|
|
1
1
|
import { getClient } from "@remit/backend/client";
|
|
2
2
|
import { createLogger } from "@remit/logger";
|
|
3
3
|
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
4
|
-
import
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
sqsWorkerQueues,
|
|
8
|
-
type WorkerQueues,
|
|
9
|
-
workerQueueUrlsFromEnv,
|
|
10
|
-
} from "../queues.js";
|
|
4
|
+
import { lambdaEmitEvent } from "../lambda-context.js";
|
|
5
|
+
import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "../sqs-queues.js";
|
|
11
6
|
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
12
7
|
import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
|
|
13
8
|
import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
|
|
14
9
|
import { runSchedulerTick } from "./run-tick.js";
|
|
15
10
|
import { redriveStrandedPlacements } from "./stranded-placement-redrive.js";
|
|
16
|
-
|
|
17
11
|
installNodeLogger();
|
|
18
|
-
|
|
19
12
|
const log = createLogger();
|
|
20
|
-
|
|
21
13
|
/**
|
|
22
14
|
* EventBridge-scheduled entry point for the periodic mailbox-sync tick
|
|
23
15
|
* (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
|
|
@@ -35,35 +27,26 @@ const log = createLogger();
|
|
|
35
27
|
* while a genuine retry of the same invocation carries the same `time` and
|
|
36
28
|
* correctly dedupes.
|
|
37
29
|
*/
|
|
38
|
-
export const buildScheduledHandler = (queues
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
account,
|
|
62
|
-
tickNow,
|
|
63
|
-
),
|
|
64
|
-
});
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
export const handler: ScheduledHandler = buildScheduledHandler(
|
|
68
|
-
sqsWorkerQueues(workerQueueUrlsFromEnv()),
|
|
69
|
-
);
|
|
30
|
+
export const buildScheduledHandler = (queues) => withTelemetry(async (event) => {
|
|
31
|
+
const client = await getClient();
|
|
32
|
+
const now = Date.parse(event.time);
|
|
33
|
+
await runSchedulerTick({
|
|
34
|
+
accountService: client.account,
|
|
35
|
+
queue: queues.mailboxes,
|
|
36
|
+
log,
|
|
37
|
+
tickIntervalMs: getTickIntervalMs(),
|
|
38
|
+
offlineIntervalMs: getOfflineIntervalMs(),
|
|
39
|
+
now,
|
|
40
|
+
// The hosted side is the one that needs this: a browser PUTs straight
|
|
41
|
+
// to block storage, so nothing refuses an upload against a draft that is
|
|
42
|
+
// already gone and this sweep is the only thing that collects it.
|
|
43
|
+
sweepAttachments: buildAttachmentSweep(client, log),
|
|
44
|
+
refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
|
|
45
|
+
redriveStrandedPlacements: (account, tickNow) => redriveStrandedPlacements({
|
|
46
|
+
client,
|
|
47
|
+
emit: lambdaEmitEvent(queues, client.mailboxLock),
|
|
48
|
+
log,
|
|
49
|
+
}, account, tickNow),
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
export const handler = buildScheduledHandler(sqsWorkerQueues(workerQueueUrlsFromEnv()));
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import { setTimeout as delay } from "node:timers/promises";
|
|
2
1
|
import type { Heartbeat } from "@remit/sqs-client/heartbeat";
|
|
3
2
|
import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
|
|
4
|
-
|
|
5
3
|
/**
|
|
6
4
|
* The scheduled-sync loop, separated from the wiring in runner.ts for the same
|
|
7
5
|
* reason `runQueuePoller` is separate from the container that starts it: the
|
|
@@ -14,25 +12,11 @@ import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
|
|
|
14
12
|
* healthy for as long as it kept crashing.
|
|
15
13
|
*/
|
|
16
14
|
export interface SchedulerLoopOptions {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
15
|
+
readonly tick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
|
|
16
|
+
readonly tickDeps: RunSchedulerTickDeps;
|
|
17
|
+
readonly heartbeat: Heartbeat;
|
|
18
|
+
readonly onHeartbeatError: (error: unknown) => void;
|
|
19
|
+
readonly tickIntervalMs: number;
|
|
20
|
+
readonly wait?: (ms: number) => Promise<void>;
|
|
23
21
|
}
|
|
24
|
-
|
|
25
|
-
export const runSchedulerLoop = async ({
|
|
26
|
-
tick,
|
|
27
|
-
tickDeps,
|
|
28
|
-
heartbeat,
|
|
29
|
-
onHeartbeatError,
|
|
30
|
-
tickIntervalMs,
|
|
31
|
-
wait = (ms) => delay(ms),
|
|
32
|
-
}: SchedulerLoopOptions): Promise<never> => {
|
|
33
|
-
for (;;) {
|
|
34
|
-
await tick(tickDeps);
|
|
35
|
-
await heartbeat().catch(onHeartbeatError);
|
|
36
|
-
await wait(tickIntervalMs);
|
|
37
|
-
}
|
|
38
|
-
};
|
|
22
|
+
export declare const runSchedulerLoop: ({ tick, tickDeps, heartbeat, onHeartbeatError, tickIntervalMs, wait, }: SchedulerLoopOptions) => Promise<never>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
2
|
+
export const runSchedulerLoop = async ({ tick, tickDeps, heartbeat, onHeartbeatError, tickIntervalMs, wait = (ms) => delay(ms), }) => {
|
|
3
|
+
for (;;) {
|
|
4
|
+
await tick(tickDeps);
|
|
5
|
+
await heartbeat().catch(onHeartbeatError);
|
|
6
|
+
await wait(tickIntervalMs);
|
|
7
|
+
}
|
|
8
|
+
};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { type SyncMailboxesEvent } from "@remit/backend/trigger-sync";
|
|
2
|
+
import type { AccountItem, FifoQueueProducer, IAccountRepository } from "@remit/data-ports";
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
|
+
import type { CalendarSubscriptionTally } from "./calendar-subscriptions.js";
|
|
5
|
+
export interface RunSchedulerTickDeps {
|
|
6
|
+
accountService: Pick<IAccountRepository, "listAllAccountsPage">;
|
|
7
|
+
queue: FifoQueueProducer<SyncMailboxesEvent>;
|
|
8
|
+
log: Logger;
|
|
9
|
+
offlineIntervalMs: number;
|
|
10
|
+
/**
|
|
11
|
+
* How often this tick itself runs — the dedup-id bucket width for
|
|
12
|
+
* `buildScheduledSyncDedupId`, so consecutive ticks each get a fresh id.
|
|
13
|
+
*/
|
|
14
|
+
tickIntervalMs: number;
|
|
15
|
+
/** Injectable for tests; defaults to `Date.now()`. */
|
|
16
|
+
now?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Collect attachment bytes the database does not know about, for one account.
|
|
19
|
+
*
|
|
20
|
+
* The tick already walks every account on a cadence, which is the only thing
|
|
21
|
+
* this needs. It exists because an upload URL minted for a draft stays valid
|
|
22
|
+
* after that draft is discarded, and where the browser writes straight to
|
|
23
|
+
* block storage nothing is in the path to refuse it — see
|
|
24
|
+
* `sweepAbandonedOutboxAttachments`. Optional only so a test can leave it out;
|
|
25
|
+
* both real entry points pass it.
|
|
26
|
+
*/
|
|
27
|
+
sweepAttachments?: (account: AccountItem) => Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Re-read the calendar subscriptions whose last fetch is older than
|
|
30
|
+
* `offlineIntervalMs` — the same knob that says how stale mail may get.
|
|
31
|
+
* Optional only so a test can leave it out; both real entry points pass it.
|
|
32
|
+
*/
|
|
33
|
+
refreshCalendarSubscriptions?: (now: number, intervalMs: number) => Promise<CalendarSubscriptionTally>;
|
|
34
|
+
redriveStrandedPlacements: (account: AccountItem, now: number) => Promise<number>;
|
|
35
|
+
}
|
|
36
|
+
export interface SchedulerTickResult {
|
|
37
|
+
scanned: number;
|
|
38
|
+
enqueued: number;
|
|
39
|
+
skipped: number;
|
|
40
|
+
swept: number;
|
|
41
|
+
sweepFailed: number;
|
|
42
|
+
subscriptionsRefreshed: number;
|
|
43
|
+
subscriptionsFailed: number;
|
|
44
|
+
redriven: number;
|
|
45
|
+
redriveFailed: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* One tick of the periodic mailbox-sync scheduler (#1247, restructured
|
|
49
|
+
* #1251). Pages through every account (never loading the whole account base
|
|
50
|
+
* into memory), decides per account via `isSyncDue` against the single
|
|
51
|
+
* offline threshold, and enqueues SYNC_MAILBOXES for the accounts that are
|
|
52
|
+
* due — in bounded-concurrency batches, never an unbounded `Promise.all`.
|
|
53
|
+
*
|
|
54
|
+
* Every enqueue goes through the same `triggerAccountSync` the manual
|
|
55
|
+
* POST /sync path uses, with a scheduler-specific, time-bucketed dedup id
|
|
56
|
+
* (`buildScheduledSyncDedupId`, bucketed by `tickIntervalMs`) so this tick
|
|
57
|
+
* can never collide with its own previous tick or with a concurrent manual
|
|
58
|
+
* trigger (see trigger-sync.ts). Concurrent-sync safety for the mailbox
|
|
59
|
+
* itself is MailboxLockService's job, inside the worker handler — this tick
|
|
60
|
+
* only decides "is a sync due" and enqueues; it never talks to IMAP.
|
|
61
|
+
*/
|
|
62
|
+
export declare const runSchedulerTick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
|