@remit/imap-worker 0.0.89 → 0.0.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/account-check.d.ts +27 -0
- package/dist/account-check.js +70 -0
- package/dist/body-sync-gate.d.ts +12 -0
- package/dist/body-sync-gate.js +53 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +193 -0
- package/dist/config-env.d.ts +5 -0
- package/dist/config-env.js +37 -0
- package/dist/connection-pool.d.ts +22 -0
- package/dist/connection-pool.js +97 -0
- package/dist/connection-scope.d.ts +8 -0
- package/dist/connection-scope.js +10 -0
- package/dist/context.d.ts +46 -0
- package/dist/context.js +15 -0
- package/dist/e2e-processor-shim.d.ts +2 -0
- package/dist/e2e-processor-shim.js +181 -0
- package/dist/emit.d.ts +15 -0
- package/dist/emit.js +76 -0
- package/dist/events.d.ts +203 -0
- package/dist/events.js +1 -0
- package/dist/failure-kind.d.ts +10 -0
- package/{src/failure-kind.ts → dist/failure-kind.js} +6 -6
- package/dist/handlers/append-sent-message.d.ts +3 -0
- package/dist/handlers/append-sent-message.js +152 -0
- package/dist/handlers/delete-account-objects.d.ts +3 -0
- package/dist/handlers/delete-account-objects.js +35 -0
- package/dist/handlers/empty-trash.d.ts +14 -0
- package/dist/handlers/empty-trash.js +217 -0
- package/dist/handlers/flag-push.d.ts +38 -0
- package/dist/handlers/flag-push.js +195 -0
- package/dist/handlers/imported-folder.d.ts +4 -0
- package/dist/handlers/imported-folder.js +23 -0
- package/dist/handlers/mailbox-management.d.ts +6 -0
- package/dist/handlers/mailbox-management.js +294 -0
- package/dist/handlers/mailbox-sync-order.d.ts +21 -0
- package/dist/handlers/mailbox-sync-order.js +50 -0
- package/dist/handlers/message-copy.d.ts +40 -0
- package/dist/handlers/message-copy.js +371 -0
- package/dist/handlers/message-delete-terminal.d.ts +60 -0
- package/dist/handlers/message-delete-terminal.js +77 -0
- package/dist/handlers/message-delete.d.ts +57 -0
- package/dist/handlers/message-delete.js +520 -0
- package/{src/handlers/message-move-terminal.ts → dist/handlers/message-move-terminal.d.ts} +18 -105
- package/dist/handlers/message-move-terminal.js +94 -0
- package/dist/handlers/message-move.d.ts +146 -0
- package/dist/handlers/message-move.js +490 -0
- package/dist/handlers/placement-move-push.d.ts +62 -0
- package/dist/handlers/placement-move-push.js +307 -0
- package/dist/handlers/probe-message-id.d.ts +2 -0
- package/dist/handlers/probe-message-id.js +6 -0
- package/dist/handlers/sync-mailboxes.d.ts +45 -0
- package/dist/handlers/sync-mailboxes.js +233 -0
- package/dist/handlers/sync-message-body.d.ts +23 -0
- package/dist/handlers/sync-message-body.js +182 -0
- package/dist/handlers/sync-messages.d.ts +29 -0
- package/dist/handlers/sync-messages.js +285 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +67 -0
- package/dist/lambda-context.d.ts +8 -0
- package/dist/lambda-context.js +33 -0
- package/dist/mailbox-row.d.ts +9 -0
- package/dist/mailbox-row.js +6 -0
- package/dist/poller.d.ts +1 -0
- package/dist/poller.js +51 -0
- package/dist/process.d.ts +6 -0
- package/dist/process.js +3 -0
- package/dist/processor.d.ts +3 -0
- package/dist/processor.js +44 -0
- package/dist/queues.d.ts +10 -0
- package/dist/queues.js +1 -0
- package/dist/s3-account-objects.d.ts +3 -0
- package/dist/s3-account-objects.js +26 -0
- package/dist/scheduler/attachment-sweep.d.ts +11 -0
- package/dist/scheduler/attachment-sweep.js +23 -0
- package/dist/scheduler/calendar-subscriptions.d.ts +26 -0
- package/dist/scheduler/calendar-subscriptions.js +35 -0
- package/dist/scheduler/config.d.ts +27 -0
- package/{src/scheduler/config.ts → dist/scheduler/config.js} +7 -26
- package/{src/scheduler/decide-due.ts → dist/scheduler/decide-due.d.ts} +3 -12
- package/dist/scheduler/decide-due.js +17 -0
- package/dist/scheduler/handler.d.ts +21 -0
- package/{src/scheduler/handler.ts → dist/scheduler/handler.js} +24 -36
- package/{src/scheduler/loop.ts → dist/scheduler/loop.d.ts} +7 -23
- package/dist/scheduler/loop.js +8 -0
- package/dist/scheduler/run-tick.d.ts +62 -0
- package/dist/scheduler/run-tick.js +164 -0
- package/dist/scheduler/runner.d.ts +2 -0
- package/{src/scheduler/runner.ts → dist/scheduler/runner.js} +58 -84
- package/dist/scheduler/stranded-placement-redrive.d.ts +16 -0
- package/dist/scheduler/stranded-placement-redrive.js +172 -0
- package/dist/sqs-queues.d.ts +4 -0
- package/dist/sqs-queues.js +22 -0
- package/dist/with-oauth-lifecycle-deps.d.ts +5 -0
- package/dist/with-oauth-lifecycle-deps.js +17 -0
- package/dist/with-oauth-lifecycle.d.ts +56 -0
- package/dist/with-oauth-lifecycle.js +92 -0
- package/{src/worker-version.ts → dist/worker-version.d.ts} +1 -2
- package/dist/worker-version.js +12 -0
- package/package.json +75 -69
- package/src/account-check.test.ts +0 -149
- package/src/account-check.ts +0 -101
- package/src/body-sync-gate.test.ts +0 -185
- package/src/body-sync-gate.ts +0 -86
- package/src/cli.ts +0 -224
- package/src/connection-scope.test.ts +0 -266
- package/src/connection-scope.ts +0 -314
- package/src/credential-less-sync-fence.test.ts +0 -152
- package/src/e2e-processor-shim.ts +0 -232
- package/src/emit.test.ts +0 -127
- package/src/emit.ts +0 -117
- package/src/events.ts +0 -252
- package/src/failure-kind.test.ts +0 -42
- package/src/handlers/append-sent-message.sqlite.test.ts +0 -176
- package/src/handlers/append-sent-message.ts +0 -264
- package/src/handlers/delete-account-objects.test.ts +0 -128
- package/src/handlers/delete-account-objects.ts +0 -94
- package/src/handlers/empty-trash.sqlite.test.ts +0 -209
- package/src/handlers/empty-trash.ts +0 -352
- package/src/handlers/flag-push.ts +0 -285
- package/src/handlers/imported-folder.test.ts +0 -63
- package/src/handlers/imported-folder.ts +0 -38
- package/src/handlers/mailbox-management.sqlite.test.ts +0 -265
- package/src/handlers/mailbox-management.ts +0 -486
- package/src/handlers/mailbox-sync-order.test.ts +0 -93
- package/src/handlers/mailbox-sync-order.ts +0 -65
- package/src/handlers/message-copy.sqlite.test.ts +0 -298
- package/src/handlers/message-copy.ts +0 -566
- package/src/handlers/message-delete-terminal.ts +0 -147
- package/src/handlers/message-delete.sqlite.test.ts +0 -291
- package/src/handlers/message-delete.ts +0 -885
- package/src/handlers/message-move.sqlite.test.ts +0 -271
- package/src/handlers/message-move.test.ts +0 -216
- package/src/handlers/message-move.ts +0 -752
- package/src/handlers/placement-move-push.test.ts +0 -137
- package/src/handlers/placement-move-push.ts +0 -475
- package/src/handlers/placement-push.sqlite.test.ts +0 -298
- package/src/handlers/probe-message-id.sqlite.test.ts +0 -192
- package/src/handlers/probe-message-id.ts +0 -15
- package/src/handlers/sync-mailboxes.test.ts +0 -183
- package/src/handlers/sync-mailboxes.ts +0 -421
- package/src/handlers/sync-message-body.sqlite.test.ts +0 -314
- package/src/handlers/sync-message-body.test.ts +0 -150
- package/src/handlers/sync-message-body.ts +0 -365
- package/src/handlers/sync-messages.sqlite.test.ts +0 -183
- package/src/handlers/sync-messages.ts +0 -525
- package/src/index.test.ts +0 -22
- package/src/index.ts +0 -86
- package/src/mail-sync-selection.sqlite.test.ts +0 -234
- package/src/mailbox-row.ts +0 -20
- package/src/pending-sync.sqlite.test.ts +0 -166
- package/src/poller.ts +0 -55
- package/src/processor.ts +0 -60
- package/src/queues.ts +0 -49
- package/src/scheduler/attachment-sweep.sqlite.test.ts +0 -75
- package/src/scheduler/attachment-sweep.ts +0 -46
- package/src/scheduler/calendar-subscriptions.sqlite.test.ts +0 -147
- package/src/scheduler/calendar-subscriptions.ts +0 -103
- package/src/scheduler/config.test.ts +0 -40
- package/src/scheduler/decide-due.test.ts +0 -44
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +0 -196
- package/src/scheduler/loop.test.ts +0 -143
- package/src/scheduler/run-tick.sqlite.test.ts +0 -191
- package/src/scheduler/run-tick.ts +0 -281
- package/src/scheduler/stranded-placement-redrive.ts +0 -265
- package/src/stranded-placement.sqlite.test.ts +0 -794
- package/src/test-helpers/test-world.ts +0 -427
- package/src/thread-join.sqlite.test.ts +0 -106
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +0 -269
- package/src/with-oauth-lifecycle-deps.ts +0 -71
- package/src/with-oauth-lifecycle.ts +0 -158
- package/tsconfig.json +0 -8
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
2
|
+
import type { Logger } from "@remit/logger";
|
|
3
|
+
/**
|
|
4
|
+
* True when `host` is a reserved placeholder name that can never resolve.
|
|
5
|
+
*
|
|
6
|
+
* Matches the bare `invalid`/`example` and any subdomain of those namespaces
|
|
7
|
+
* (`mail.example`, `foo.invalid`, …). Suffix matching is anchored on the dot so
|
|
8
|
+
* real hosts like `invalid.com` or `imap.gmail.com` are NOT treated as reserved.
|
|
9
|
+
*/
|
|
10
|
+
export declare const isReservedHost: (host: string) => boolean;
|
|
11
|
+
/**
|
|
12
|
+
* Check if an account's IMAP host can never resolve (reserved TLD).
|
|
13
|
+
* Returns true if the account should be skipped cleanly — no connection
|
|
14
|
+
* attempt, no thrown error, so the event is acked rather than dead-lettered.
|
|
15
|
+
*/
|
|
16
|
+
export declare const isUnsyncableHost: (account: AccountItem, log: Logger) => boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Check if an account is deleted (tombstone pattern).
|
|
19
|
+
* Returns true if the account should be skipped.
|
|
20
|
+
*/
|
|
21
|
+
export declare const isAccountDeleted: (account: AccountItem, log: Logger) => boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Check if an account requires re-authentication (e.g. OAuth token revoked).
|
|
24
|
+
* Returns true if the account should be skipped until the user re-auths.
|
|
25
|
+
*/
|
|
26
|
+
export declare const isAccountReauthRequired: (account: AccountItem, log: Logger) => boolean;
|
|
27
|
+
export declare const isMailSyncDisabled: (account: AccountItem, log: Logger) => boolean;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { AccountService } from "@remit/domain-enums";
|
|
2
|
+
/**
|
|
3
|
+
* RFC 2606 reserved placeholder namespaces that are guaranteed never to resolve
|
|
4
|
+
* and are not used by any local/dev/e2e environment: `.invalid` (the actual
|
|
5
|
+
* smoke-test placeholder) and `.example`. An account pointed at one of these can
|
|
6
|
+
* never connect, so syncing it would retry and dead-letter forever — we skip it.
|
|
7
|
+
*
|
|
8
|
+
* `localhost`/`.localhost` and `.test` are deliberately NOT skipped: the e2e and
|
|
9
|
+
* mailfuzz suites run their IMAP server on `localhost`, and `.test` is commonly
|
|
10
|
+
* used for local testing.
|
|
11
|
+
*/
|
|
12
|
+
const RESERVED_NAMES = ["invalid", "example"];
|
|
13
|
+
const RESERVED_HOST_SUFFIXES = RESERVED_NAMES.map((name) => `.${name}`);
|
|
14
|
+
/**
|
|
15
|
+
* True when `host` is a reserved placeholder name that can never resolve.
|
|
16
|
+
*
|
|
17
|
+
* Matches the bare `invalid`/`example` and any subdomain of those namespaces
|
|
18
|
+
* (`mail.example`, `foo.invalid`, …). Suffix matching is anchored on the dot so
|
|
19
|
+
* real hosts like `invalid.com` or `imap.gmail.com` are NOT treated as reserved.
|
|
20
|
+
*/
|
|
21
|
+
export const isReservedHost = (host) => {
|
|
22
|
+
const normalized = host.trim().toLowerCase();
|
|
23
|
+
if (RESERVED_NAMES.includes(normalized)) {
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
return RESERVED_HOST_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Check if an account's IMAP host can never resolve (reserved TLD).
|
|
30
|
+
* Returns true if the account should be skipped cleanly — no connection
|
|
31
|
+
* attempt, no thrown error, so the event is acked rather than dead-lettered.
|
|
32
|
+
*/
|
|
33
|
+
export const isUnsyncableHost = (account, log) => {
|
|
34
|
+
if (isReservedHost(account.imapHost)) {
|
|
35
|
+
log.warn({ accountId: account.accountId, imapHost: account.imapHost }, "Skipping account: IMAP host is a reserved, never-resolvable name");
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
return false;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Check if an account is deleted (tombstone pattern).
|
|
42
|
+
* Returns true if the account should be skipped.
|
|
43
|
+
*/
|
|
44
|
+
export const isAccountDeleted = (account, log) => {
|
|
45
|
+
if (account.deletedAt) {
|
|
46
|
+
log.info({ accountId: account.accountId, deletedAt: account.deletedAt }, "Skipping deleted account");
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
return false;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Check if an account requires re-authentication (e.g. OAuth token revoked).
|
|
53
|
+
* Returns true if the account should be skipped until the user re-auths.
|
|
54
|
+
*/
|
|
55
|
+
export const isAccountReauthRequired = (account, log) => {
|
|
56
|
+
if (account.connectionState === "reauth_required") {
|
|
57
|
+
log.info({
|
|
58
|
+
accountId: account.accountId,
|
|
59
|
+
connectionState: account.connectionState,
|
|
60
|
+
}, "Skipping account: reauth required");
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
return false;
|
|
64
|
+
};
|
|
65
|
+
export const isMailSyncDisabled = (account, log) => {
|
|
66
|
+
if (account.syncedServices.includes(AccountService.Mail))
|
|
67
|
+
return false;
|
|
68
|
+
log.info({ accountId: account.accountId, syncedServices: account.syncedServices }, "Skipping account: mail sync is off");
|
|
69
|
+
return true;
|
|
70
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { SSMClient } from "@aws-sdk/client-ssm";
|
|
2
|
+
import type { Logger } from "@remit/logger";
|
|
3
|
+
/**
|
|
4
|
+
* Test seam — clears the module-scope TTL cache between unit tests.
|
|
5
|
+
*/
|
|
6
|
+
export declare const resetBodySyncGateCache: () => void;
|
|
7
|
+
/**
|
|
8
|
+
* Reads the `/{stage}/Remit/bodySyncEnabled` SSM parameter, cached at module
|
|
9
|
+
* scope for a short TTL. Returns `true` (enabled) when the parameter is absent
|
|
10
|
+
* or unreadable — body sync fails open so a missing toggle never halts prod.
|
|
11
|
+
*/
|
|
12
|
+
export declare const isBodySyncEnabled: (parameterName: string, log: Logger, ssm?: SSMClient) => Promise<boolean>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
|
|
2
|
+
const CACHE_TTL_MS = 30_000;
|
|
3
|
+
let cache;
|
|
4
|
+
let inFlight;
|
|
5
|
+
let defaultClient;
|
|
6
|
+
const getDefaultClient = () => {
|
|
7
|
+
if (!defaultClient) {
|
|
8
|
+
defaultClient = new SSMClient({});
|
|
9
|
+
}
|
|
10
|
+
return defaultClient;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Test seam — clears the module-scope TTL cache between unit tests.
|
|
14
|
+
*/
|
|
15
|
+
export const resetBodySyncGateCache = () => {
|
|
16
|
+
cache = undefined;
|
|
17
|
+
inFlight = undefined;
|
|
18
|
+
};
|
|
19
|
+
const readParameter = async (ssm, parameterName, log) => {
|
|
20
|
+
const result = await ssm.send(new GetParameterCommand({ Name: parameterName }));
|
|
21
|
+
const value = result.Parameter?.Value;
|
|
22
|
+
if (value === undefined) {
|
|
23
|
+
log.warn({ parameterName }, "Body-sync toggle parameter missing a value, failing open (enabled)");
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
return value.trim().toLowerCase() !== "false";
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Reads the `/{stage}/Remit/bodySyncEnabled` SSM parameter, cached at module
|
|
30
|
+
* scope for a short TTL. Returns `true` (enabled) when the parameter is absent
|
|
31
|
+
* or unreadable — body sync fails open so a missing toggle never halts prod.
|
|
32
|
+
*/
|
|
33
|
+
export const isBodySyncEnabled = async (parameterName, log, ssm = getDefaultClient()) => {
|
|
34
|
+
if (cache && cache.expiresAt > Date.now()) {
|
|
35
|
+
return cache.enabled;
|
|
36
|
+
}
|
|
37
|
+
if (inFlight) {
|
|
38
|
+
return inFlight;
|
|
39
|
+
}
|
|
40
|
+
inFlight = readParameter(ssm, parameterName, log)
|
|
41
|
+
.catch((error) => {
|
|
42
|
+
log.warn({ parameterName, error }, "Failed to read body-sync toggle parameter, failing open (enabled)");
|
|
43
|
+
return true;
|
|
44
|
+
})
|
|
45
|
+
.then((enabled) => {
|
|
46
|
+
cache = { enabled, expiresAt: Date.now() + CACHE_TTL_MS };
|
|
47
|
+
return enabled;
|
|
48
|
+
})
|
|
49
|
+
.finally(() => {
|
|
50
|
+
inFlight = undefined;
|
|
51
|
+
});
|
|
52
|
+
return inFlight;
|
|
53
|
+
};
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Plain text on purpose, and the one deliberate exception to the log field
|
|
3
|
+
// contract in deploy/vps/README.md ("Logs"). That contract covers what a
|
|
4
|
+
// container's log driver ships; this file is the `remit-worker` bin, run by hand
|
|
5
|
+
// against a terminal, and is not an entrypoint of any image (see the target list
|
|
6
|
+
// in npm-scripts/docker-bundle.mjs). Do not route it through the logger — usage
|
|
7
|
+
// text and argument errors are for the person who typed the command.
|
|
8
|
+
import { parseArgs } from "node:util";
|
|
9
|
+
import { getClient } from "@remit/backend/client";
|
|
10
|
+
import { lambdaEmitEvent } from "./lambda-context.js";
|
|
11
|
+
import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "./sqs-queues.js";
|
|
12
|
+
const HELP = `
|
|
13
|
+
remit-imap-worker - Process IMAP sync events
|
|
14
|
+
|
|
15
|
+
USAGE:
|
|
16
|
+
remit-worker -t <type> -a <accountId> [options]
|
|
17
|
+
|
|
18
|
+
OPTIONS:
|
|
19
|
+
-t, --type <type> Event type (required)
|
|
20
|
+
-a, --accountId <id> Account ID (required)
|
|
21
|
+
-m, --mailboxId <id> Mailbox ID (required for some event types)
|
|
22
|
+
--messageIds <ids> Comma-separated message IDs (required for SYNC_MESSAGE_BODY)
|
|
23
|
+
--fullSync Force full sync, ignoring lastSyncUid (SYNC_MESSAGES only)
|
|
24
|
+
--path <path> Mailbox path (required for MAILBOX_CREATE, MAILBOX_DELETE)
|
|
25
|
+
--oldPath <path> Old mailbox path (required for MAILBOX_RENAME)
|
|
26
|
+
--newPath <path> New mailbox path (required for MAILBOX_RENAME)
|
|
27
|
+
--subscribe Subscribe to mailbox after creation (MAILBOX_CREATE only)
|
|
28
|
+
-h, --help Show this help message
|
|
29
|
+
|
|
30
|
+
EVENT TYPES:
|
|
31
|
+
SYNC_MAILBOXES Sync all mailboxes for an account
|
|
32
|
+
SYNC_MESSAGES Sync messages in a specific mailbox
|
|
33
|
+
SYNC_MESSAGE_BODY Fetch and store message bodies in batch
|
|
34
|
+
MAILBOX_CREATE Create a new mailbox
|
|
35
|
+
MAILBOX_RENAME Rename a mailbox
|
|
36
|
+
MAILBOX_DELETE Delete a mailbox
|
|
37
|
+
|
|
38
|
+
EXAMPLES:
|
|
39
|
+
# Sync all mailboxes for an account
|
|
40
|
+
remit-worker -t SYNC_MAILBOXES -a account-123
|
|
41
|
+
|
|
42
|
+
# Sync messages in a mailbox
|
|
43
|
+
remit-worker -t SYNC_MESSAGES -a account-123 -m mailbox-456
|
|
44
|
+
|
|
45
|
+
# Force a full sync of messages (ignore lastSyncUid)
|
|
46
|
+
remit-worker -t SYNC_MESSAGES -a account-123 -m mailbox-456 --fullSync
|
|
47
|
+
|
|
48
|
+
# Sync message bodies for specific messages
|
|
49
|
+
remit-worker -t SYNC_MESSAGE_BODY -a account-123 -m mailbox-456 --messageIds id1,id2,id3
|
|
50
|
+
|
|
51
|
+
# Create a new mailbox
|
|
52
|
+
remit-worker -t MAILBOX_CREATE -a account-123 -m mailbox-id --path Work/Projects --subscribe
|
|
53
|
+
|
|
54
|
+
# Rename a mailbox
|
|
55
|
+
remit-worker -t MAILBOX_RENAME -a account-123 -m mailbox-id --oldPath Work/Projects --newPath Archive/Projects
|
|
56
|
+
|
|
57
|
+
# Delete a mailbox
|
|
58
|
+
remit-worker -t MAILBOX_DELETE -a account-123 -m mailbox-id --path Work/Projects
|
|
59
|
+
`;
|
|
60
|
+
const { values } = parseArgs({
|
|
61
|
+
options: {
|
|
62
|
+
type: { type: "string", short: "t" },
|
|
63
|
+
accountId: { type: "string", short: "a" },
|
|
64
|
+
mailboxId: { type: "string", short: "m" },
|
|
65
|
+
messageIds: { type: "string" },
|
|
66
|
+
fullSync: { type: "boolean", default: false },
|
|
67
|
+
path: { type: "string" },
|
|
68
|
+
oldPath: { type: "string" },
|
|
69
|
+
newPath: { type: "string" },
|
|
70
|
+
subscribe: { type: "boolean", default: false },
|
|
71
|
+
help: { type: "boolean", short: "h", default: false },
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
if (values.help) {
|
|
75
|
+
console.log(HELP);
|
|
76
|
+
process.exit(0);
|
|
77
|
+
}
|
|
78
|
+
if (!values.type || !values.accountId) {
|
|
79
|
+
console.error("Error: --type and --accountId are required\n");
|
|
80
|
+
console.log(HELP);
|
|
81
|
+
process.exit(0);
|
|
82
|
+
}
|
|
83
|
+
const validTypes = [
|
|
84
|
+
"SYNC_MAILBOXES",
|
|
85
|
+
"SYNC_MESSAGES",
|
|
86
|
+
"SYNC_MESSAGE_BODY",
|
|
87
|
+
"MAILBOX_CREATE",
|
|
88
|
+
"MAILBOX_RENAME",
|
|
89
|
+
"MAILBOX_DELETE",
|
|
90
|
+
];
|
|
91
|
+
if (!validTypes.includes(values.type)) {
|
|
92
|
+
console.error(`Error: Invalid type "${values.type}". Must be one of: ${validTypes.join(", ")}\n`);
|
|
93
|
+
process.exit(1);
|
|
94
|
+
}
|
|
95
|
+
if (["SYNC_MESSAGES", "SYNC_MESSAGE_BODY"].includes(values.type) &&
|
|
96
|
+
!values.mailboxId) {
|
|
97
|
+
console.error(`Error: --mailboxId is required for ${values.type}\n`);
|
|
98
|
+
process.exit(1);
|
|
99
|
+
}
|
|
100
|
+
if (values.type === "SYNC_MESSAGE_BODY" && !values.messageIds) {
|
|
101
|
+
console.error(`Error: --messageIds is required for ${values.type}\n`);
|
|
102
|
+
process.exit(1);
|
|
103
|
+
}
|
|
104
|
+
// Validation for mailbox management events
|
|
105
|
+
if (["MAILBOX_CREATE", "MAILBOX_DELETE"].includes(values.type) &&
|
|
106
|
+
!values.path) {
|
|
107
|
+
console.error(`Error: --path is required for ${values.type}\n`);
|
|
108
|
+
process.exit(1);
|
|
109
|
+
}
|
|
110
|
+
if (["MAILBOX_CREATE", "MAILBOX_RENAME", "MAILBOX_DELETE"].includes(values.type) &&
|
|
111
|
+
!values.mailboxId) {
|
|
112
|
+
console.error(`Error: --mailboxId is required for ${values.type}\n`);
|
|
113
|
+
process.exit(1);
|
|
114
|
+
}
|
|
115
|
+
if (values.type === "MAILBOX_RENAME" && (!values.oldPath || !values.newPath)) {
|
|
116
|
+
console.error(`Error: --oldPath and --newPath are required for ${values.type}\n`);
|
|
117
|
+
process.exit(1);
|
|
118
|
+
}
|
|
119
|
+
// Build the event based on type
|
|
120
|
+
const buildEvent = () => {
|
|
121
|
+
switch (values.type) {
|
|
122
|
+
case "SYNC_MAILBOXES":
|
|
123
|
+
return {
|
|
124
|
+
type: "SYNC_MAILBOXES",
|
|
125
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
126
|
+
accountId: values.accountId,
|
|
127
|
+
};
|
|
128
|
+
case "SYNC_MESSAGES":
|
|
129
|
+
return {
|
|
130
|
+
type: "SYNC_MESSAGES",
|
|
131
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
132
|
+
accountId: values.accountId,
|
|
133
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
134
|
+
mailboxId: values.mailboxId,
|
|
135
|
+
fullSync: values.fullSync,
|
|
136
|
+
};
|
|
137
|
+
case "SYNC_MESSAGE_BODY":
|
|
138
|
+
return {
|
|
139
|
+
type: "SYNC_MESSAGE_BODY",
|
|
140
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
141
|
+
accountId: values.accountId,
|
|
142
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
143
|
+
mailboxId: values.mailboxId,
|
|
144
|
+
messageIds: values.messageIds?.split(",").map((id) => id.trim()),
|
|
145
|
+
};
|
|
146
|
+
case "MAILBOX_CREATE":
|
|
147
|
+
return {
|
|
148
|
+
type: "MAILBOX_CREATE",
|
|
149
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
150
|
+
accountId: values.accountId,
|
|
151
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
152
|
+
mailboxId: values.mailboxId,
|
|
153
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
154
|
+
path: values.path,
|
|
155
|
+
subscribe: values.subscribe,
|
|
156
|
+
};
|
|
157
|
+
case "MAILBOX_RENAME":
|
|
158
|
+
return {
|
|
159
|
+
type: "MAILBOX_RENAME",
|
|
160
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
161
|
+
accountId: values.accountId,
|
|
162
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
163
|
+
mailboxId: values.mailboxId,
|
|
164
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
165
|
+
oldPath: values.oldPath,
|
|
166
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
167
|
+
newPath: values.newPath,
|
|
168
|
+
};
|
|
169
|
+
case "MAILBOX_DELETE":
|
|
170
|
+
return {
|
|
171
|
+
type: "MAILBOX_DELETE",
|
|
172
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
173
|
+
accountId: values.accountId,
|
|
174
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
175
|
+
mailboxId: values.mailboxId,
|
|
176
|
+
// biome-ignore lint/style/noNonNullAssertion: value is guaranteed by caller contract
|
|
177
|
+
path: values.path,
|
|
178
|
+
};
|
|
179
|
+
default:
|
|
180
|
+
throw new Error(`Unknown event type: ${values.type}`);
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
const event = buildEvent();
|
|
184
|
+
console.log(`Enqueueing ${event.type} event for account ${event.accountId}...`);
|
|
185
|
+
lambdaEmitEvent(sqsWorkerQueues(workerQueueUrlsFromEnv()), (await getClient()).mailboxLock)(event)
|
|
186
|
+
.then(() => {
|
|
187
|
+
console.log("Event enqueued successfully");
|
|
188
|
+
process.exit(0);
|
|
189
|
+
})
|
|
190
|
+
.catch((error) => {
|
|
191
|
+
console.error("Failed to enqueue event:", error);
|
|
192
|
+
process.exit(1);
|
|
193
|
+
});
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ConnectionPoolOptions } from "./connection-pool.js";
|
|
2
|
+
import { type ImapWorkerConfig } from "./context.js";
|
|
3
|
+
export declare const resolveMailboxFreshnessMs: (env: NodeJS.ProcessEnv) => number;
|
|
4
|
+
export declare const imapWorkerConfigFromEnv: (env: NodeJS.ProcessEnv) => ImapWorkerConfig;
|
|
5
|
+
export declare const poolOptionsFromEnv: (env: NodeJS.ProcessEnv) => ConnectionPoolOptions;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { attemptBudget } from "@remit/sqs-client/attempt-budget";
|
|
2
|
+
import { DEFAULT_MAILBOX_FRESHNESS_MS, defaultImapWorkerConfig, } from "./context.js";
|
|
3
|
+
import { workerVersion } from "./worker-version.js";
|
|
4
|
+
const DEFAULT_CONNECTIONS_PER_ACCOUNT = 2;
|
|
5
|
+
// Deployments that drive sync from a test harness set MAILBOX_FRESHNESS_MS low
|
|
6
|
+
// so their explicit nudges are never no-oped by the freshness gate.
|
|
7
|
+
export const resolveMailboxFreshnessMs = (env) => {
|
|
8
|
+
const value = env.MAILBOX_FRESHNESS_MS?.trim();
|
|
9
|
+
if (!value)
|
|
10
|
+
return DEFAULT_MAILBOX_FRESHNESS_MS;
|
|
11
|
+
const raw = Number(value);
|
|
12
|
+
return Number.isInteger(raw) && raw >= 0 ? raw : DEFAULT_MAILBOX_FRESHNESS_MS;
|
|
13
|
+
};
|
|
14
|
+
export const imapWorkerConfigFromEnv = (env) => {
|
|
15
|
+
const { attempts } = defaultImapWorkerConfig;
|
|
16
|
+
return {
|
|
17
|
+
attempts: {
|
|
18
|
+
body: attemptBudget("BODY_SYNC_MAX_ATTEMPTS", attempts.body, env),
|
|
19
|
+
placementMove: attemptBudget("PLACEMENT_MOVE_MAX_ATTEMPTS", attempts.placementMove, env),
|
|
20
|
+
flagPush: attemptBudget("FLAG_PUSH_MAX_ATTEMPTS", attempts.flagPush, env),
|
|
21
|
+
appendSent: attemptBudget("APPEND_SENT_MAX_ATTEMPTS", attempts.appendSent, env),
|
|
22
|
+
messageMove: attemptBudget("MESSAGE_MOVE_MAX_ATTEMPTS", attempts.messageMove, env),
|
|
23
|
+
messageDelete: attemptBudget("MESSAGE_DELETE_MAX_ATTEMPTS", attempts.messageDelete, env),
|
|
24
|
+
messageCopy: attemptBudget("MESSAGE_COPY_MAX_ATTEMPTS", attempts.messageCopy, env),
|
|
25
|
+
},
|
|
26
|
+
bodyBatchSize: defaultImapWorkerConfig.bodyBatchSize,
|
|
27
|
+
mailboxFreshnessMs: resolveMailboxFreshnessMs(env),
|
|
28
|
+
workerVersion: workerVersion(),
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
export const poolOptionsFromEnv = (env) => {
|
|
32
|
+
const raw = Number(env.CONNECTIONS_PER_ACCOUNT);
|
|
33
|
+
return {
|
|
34
|
+
size: Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_CONNECTIONS_PER_ACCOUNT,
|
|
35
|
+
overflow: "Open",
|
|
36
|
+
};
|
|
37
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ConnectionScope, IImapConnection, MailCredentials } from "@remit/mailbox-service";
|
|
2
|
+
export type PoolOverflow = "Wait" | "Open";
|
|
3
|
+
export interface ConnectionPoolOptions {
|
|
4
|
+
readonly size: number;
|
|
5
|
+
readonly overflow: PoolOverflow;
|
|
6
|
+
}
|
|
7
|
+
export interface PoolAccount {
|
|
8
|
+
readonly accountId: string;
|
|
9
|
+
readonly username: string;
|
|
10
|
+
readonly imapHost: string;
|
|
11
|
+
readonly imapPort: number;
|
|
12
|
+
readonly imapTls: boolean;
|
|
13
|
+
}
|
|
14
|
+
export type ConnectImap = (account: PoolAccount, credentials: MailCredentials) => ConnectionScope;
|
|
15
|
+
export interface BorrowedConnection {
|
|
16
|
+
getConnection: () => Promise<IImapConnection>;
|
|
17
|
+
release: () => Promise<void>;
|
|
18
|
+
}
|
|
19
|
+
export interface ConnectionPool {
|
|
20
|
+
borrow: (account: PoolAccount, credentials: MailCredentials) => BorrowedConnection;
|
|
21
|
+
}
|
|
22
|
+
export declare const createConnectionPool: (options: ConnectionPoolOptions, connect: ConnectImap) => ConnectionPool;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
const isLive = (slot) => slot.connection?.isConnected ?? false;
|
|
2
|
+
const disconnectQuietly = async (slot) => {
|
|
3
|
+
await slot.scope.disconnect().catch(() => undefined);
|
|
4
|
+
};
|
|
5
|
+
export const createConnectionPool = (options, connect) => {
|
|
6
|
+
const accounts = new Map();
|
|
7
|
+
const slotsFor = (accountId) => {
|
|
8
|
+
const existing = accounts.get(accountId);
|
|
9
|
+
if (existing)
|
|
10
|
+
return existing;
|
|
11
|
+
const created = { slots: [], waiters: [] };
|
|
12
|
+
accounts.set(accountId, created);
|
|
13
|
+
return created;
|
|
14
|
+
};
|
|
15
|
+
const claim = (pool, createScope) => {
|
|
16
|
+
const live = pool.slots.find((slot) => !slot.busy && isLive(slot));
|
|
17
|
+
if (live) {
|
|
18
|
+
live.busy = true;
|
|
19
|
+
return { kind: "Pooled", slot: live, state: "Live" };
|
|
20
|
+
}
|
|
21
|
+
const stale = pool.slots.find((slot) => !slot.busy);
|
|
22
|
+
if (stale) {
|
|
23
|
+
stale.busy = true;
|
|
24
|
+
return { kind: "Pooled", slot: stale, state: "Stale" };
|
|
25
|
+
}
|
|
26
|
+
if (pool.slots.length < options.size) {
|
|
27
|
+
const slot = { scope: createScope(), connection: null, busy: true };
|
|
28
|
+
pool.slots.push(slot);
|
|
29
|
+
return { kind: "Pooled", slot, state: "New" };
|
|
30
|
+
}
|
|
31
|
+
return undefined;
|
|
32
|
+
};
|
|
33
|
+
const giveBack = (pool, slot) => {
|
|
34
|
+
const next = pool.waiters.shift();
|
|
35
|
+
if (next) {
|
|
36
|
+
next(slot);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
slot.busy = false;
|
|
40
|
+
};
|
|
41
|
+
const acquire = async (pool, createScope) => {
|
|
42
|
+
const claimed = claim(pool, createScope);
|
|
43
|
+
if (claimed)
|
|
44
|
+
return claimed;
|
|
45
|
+
if (options.overflow === "Open") {
|
|
46
|
+
return {
|
|
47
|
+
kind: "Overflow",
|
|
48
|
+
slot: { scope: createScope(), connection: null, busy: true },
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
const slot = await new Promise((resolve) => {
|
|
52
|
+
pool.waiters.push(resolve);
|
|
53
|
+
});
|
|
54
|
+
return { kind: "Pooled", slot, state: isLive(slot) ? "Live" : "Stale" };
|
|
55
|
+
};
|
|
56
|
+
const borrow = (account, credentials) => {
|
|
57
|
+
const pool = slotsFor(account.accountId);
|
|
58
|
+
const createScope = () => connect(account, credentials);
|
|
59
|
+
let leased;
|
|
60
|
+
const getConnection = async () => {
|
|
61
|
+
leased ??= acquire(pool, createScope).then(async (lease) => {
|
|
62
|
+
if (lease.kind === "Pooled" && lease.state === "Stale") {
|
|
63
|
+
await disconnectQuietly(lease.slot);
|
|
64
|
+
lease.slot.connection = null;
|
|
65
|
+
try {
|
|
66
|
+
lease.slot.scope = createScope();
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
giveBack(pool, lease.slot);
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return lease;
|
|
74
|
+
});
|
|
75
|
+
const { slot } = await leased;
|
|
76
|
+
const connection = await slot.scope.getConnection();
|
|
77
|
+
slot.connection = connection;
|
|
78
|
+
return connection;
|
|
79
|
+
};
|
|
80
|
+
const release = async () => {
|
|
81
|
+
if (!leased)
|
|
82
|
+
return;
|
|
83
|
+
const pending = leased;
|
|
84
|
+
leased = undefined;
|
|
85
|
+
const lease = await pending.catch(() => undefined);
|
|
86
|
+
if (!lease)
|
|
87
|
+
return;
|
|
88
|
+
if (lease.kind === "Overflow") {
|
|
89
|
+
await disconnectQuietly(lease.slot);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
giveBack(pool, lease.slot);
|
|
93
|
+
};
|
|
94
|
+
return { getConnection, release };
|
|
95
|
+
};
|
|
96
|
+
return { borrow };
|
|
97
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type ConnectionScope, type MailCredentials } from "@remit/mailbox-service";
|
|
2
|
+
export type { ConnectionScope };
|
|
3
|
+
export declare const createConnectionScopeWithCredentials: (account: {
|
|
4
|
+
username: string;
|
|
5
|
+
imapHost: string;
|
|
6
|
+
imapPort: number;
|
|
7
|
+
imapTls: boolean;
|
|
8
|
+
}, credentials: MailCredentials) => ConnectionScope;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { connectionScope, } from "@remit/mailbox-service";
|
|
2
|
+
import { nodeTransport } from "@remit/mailbox-service/node-transport";
|
|
3
|
+
export const createConnectionScopeWithCredentials = (account, credentials) => connectionScope({
|
|
4
|
+
host: account.imapHost,
|
|
5
|
+
port: account.imapPort,
|
|
6
|
+
tls: account.imapTls,
|
|
7
|
+
user: account.username,
|
|
8
|
+
credentials,
|
|
9
|
+
transport: nodeTransport(),
|
|
10
|
+
});
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { RemitClient } from "@remit/backend/client";
|
|
2
|
+
import type { Logger, MailFailureKind } from "@remit/logger";
|
|
3
|
+
import type { MailOAuthService } from "@remit/mail-oauth-service";
|
|
4
|
+
import type { ConnectionPool } from "./connection-pool.js";
|
|
5
|
+
import type { WorkerQueues } from "./queues.js";
|
|
6
|
+
export type ImapWorkerQueueName = "sync";
|
|
7
|
+
export type TokenService = Pick<MailOAuthService, "getAccessToken">;
|
|
8
|
+
export type ImapFailureKind = MailFailureKind;
|
|
9
|
+
export interface WorkerMetrics {
|
|
10
|
+
imapFailure: (operation: string, kind: ImapFailureKind, count: number) => void;
|
|
11
|
+
}
|
|
12
|
+
export interface AccountObjectPage {
|
|
13
|
+
keys: string[];
|
|
14
|
+
nextContinuationToken: string | undefined;
|
|
15
|
+
}
|
|
16
|
+
export interface AccountObjectStore {
|
|
17
|
+
list: (prefix: string, continuationToken: string | undefined) => Promise<AccountObjectPage>;
|
|
18
|
+
delete: (keys: string[]) => Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
export interface AttemptBudgets {
|
|
21
|
+
readonly body: number;
|
|
22
|
+
readonly placementMove: number;
|
|
23
|
+
readonly flagPush: number;
|
|
24
|
+
readonly appendSent: number;
|
|
25
|
+
readonly messageMove: number;
|
|
26
|
+
readonly messageDelete: number;
|
|
27
|
+
readonly messageCopy: number;
|
|
28
|
+
}
|
|
29
|
+
export interface ImapWorkerConfig {
|
|
30
|
+
readonly attempts: AttemptBudgets;
|
|
31
|
+
readonly bodyBatchSize: number;
|
|
32
|
+
readonly mailboxFreshnessMs: number;
|
|
33
|
+
readonly workerVersion: string;
|
|
34
|
+
}
|
|
35
|
+
export interface ProcessContext {
|
|
36
|
+
readonly client: RemitClient<ImapWorkerQueueName>;
|
|
37
|
+
readonly queues: WorkerQueues;
|
|
38
|
+
readonly pool: ConnectionPool;
|
|
39
|
+
readonly metrics: WorkerMetrics;
|
|
40
|
+
readonly config: ImapWorkerConfig;
|
|
41
|
+
readonly log: Logger;
|
|
42
|
+
readonly oauth: TokenService;
|
|
43
|
+
readonly accountObjects: AccountObjectStore;
|
|
44
|
+
}
|
|
45
|
+
export declare const DEFAULT_MAILBOX_FRESHNESS_MS = 60000;
|
|
46
|
+
export declare const defaultImapWorkerConfig: Omit<ImapWorkerConfig, "workerVersion">;
|
package/dist/context.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { SYNC_WARNING_SURFACE_ATTEMPTS } from "@remit/data-ports/sync-warning-state";
|
|
2
|
+
export const DEFAULT_MAILBOX_FRESHNESS_MS = 60_000;
|
|
3
|
+
export const defaultImapWorkerConfig = {
|
|
4
|
+
attempts: {
|
|
5
|
+
body: SYNC_WARNING_SURFACE_ATTEMPTS,
|
|
6
|
+
placementMove: 3,
|
|
7
|
+
flagPush: 3,
|
|
8
|
+
appendSent: 3,
|
|
9
|
+
messageMove: 3,
|
|
10
|
+
messageDelete: 3,
|
|
11
|
+
messageCopy: 3,
|
|
12
|
+
},
|
|
13
|
+
bodyBatchSize: 200,
|
|
14
|
+
mailboxFreshnessMs: DEFAULT_MAILBOX_FRESHNESS_MS,
|
|
15
|
+
};
|