@remit/imap-worker 0.0.1

package/README.md

@@ -0,0 +1,100 @@
+# @remit/imap-worker
+
+SQS-driven Lambda worker for handling IMAP lifecycle events in the Remit system.
+
+## Features
+
+- **Event Driven**: Processes events from SQS (`SYNC_MAILBOXES`, `SYNC_MESSAGES`, etc.)
+- **CLI Tool**: Includes a CLI for manually triggering events during development
+- **Bundled**: Built with esbuild for optimal Lambda performance
+
+## Installation
+
+```bash
+npm install
+npm run bundle
+```
+
+## Usage
+
+### Lambda Handler
+
+The package exports a standard AWS Lambda SQS handler:
+
+```typescript
+import { handler } from "@remit/imap-worker";
+```
+
+### CLI (Local Development)
+
+Trigger events manually using the included CLI:
+
+```bash
+# Sync all mailboxes for an account
+npm run cli -- -t SYNC_MAILBOXES -a <accountId>
+
+# Sync messages in a mailbox
+npm run cli -- -t SYNC_MESSAGES -a <accountId> -m <mailboxId>
+
+# Force full sync (ignore lastSyncUid)
+npm run cli -- -t SYNC_MESSAGES -a <accountId> -m <mailboxId> --fullSync
+
+# Sync message bodies for specific messages
+npm run cli -- -t SYNC_MESSAGE_BODY -a <accountId> -m <mailboxId> --messageIds id1,id2,id3
+```
+
+## Environment Variables
+
+| Variable                      | Required | Description                                              |
+| ----------------------------- | -------- | -------------------------------------------------------- |
+| `DYNAMODB_TABLE_NAME`         | Yes      | Name of the DynamoDB table for Remit data                |
+| `SQS_QUEUE_URL_MAILBOXES`     | Yes      | FIFO queue URL for `SYNC_MAILBOXES` events               |
+| `SQS_QUEUE_URL_MESSAGES`      | Yes      | FIFO queue URL for `SYNC_MESSAGES` events                |
+| `SQS_QUEUE_URL_BODY`          | Yes      | FIFO queue URL for `SYNC_MESSAGE_BODY` events            |
+| `SQS_QUEUE_URL_FLAGS`         | Yes      | FIFO queue URL for `SYNC_FLAGS` events                   |
+| `SQS_QUEUE_URL_MAILBOX_MGMT`  | Yes      | Standard queue URL for mailbox management events         |
+| `SQS_QUEUE_URL_MESSAGE_MGMT`  | Yes      | Standard queue URL for message management events         |
+| `S3_BUCKET`                   | Yes      | S3 bucket for storing raw message content                |
+| `NODE_ENV`                    | No       | Set to `development` for local execution                 |
+| `LOG_LEVEL`                   | No       | Logging level (default: `info`)                          |
+
+## Event Types
+
+| Event               | Description                                    | Required Fields                         |
+| ------------------- | ---------------------------------------------- | --------------------------------------- |
+| `SYNC_MAILBOXES`    | Discovers and syncs mailboxes for an account   | `accountId`                             |
+| `SYNC_MESSAGES`     | Fetches new messages for a mailbox (one batch) | `accountId`, `mailboxId`                |
+| `SYNC_MESSAGE_BODY` | Fetches and stores message bodies in batch     | `accountId`, `mailboxId`, `messageIds`  |
+
+### Event Schema
+
+All events share a base schema:
+
+```typescript
+interface BaseEvent {
+  accountId: string;
+  eventId: string;    // Idempotency key
+  timestamp: number;  // Unix timestamp
+}
+```
+
+## Architecture
+
+```
+SQS Queue
+    │
+    ▼
+┌───────────────┐
+│  Lambda/CLI   │
+│   (index.ts)  │
+└───────┬───────┘
+        │
+        ▼
+┌───────────────┐
+│   processor   │  ─── Routes events to handlers
+└───────┬───────┘
+        │
+        ├──► syncMailboxes
+        ├──► syncMessages
+        └──► syncMessageBody
+```

package/build.mjs

@@ -0,0 +1,17 @@
+import * as esbuild from "esbuild";
+
+await esbuild.build({
+	entryPoints: ["src/index.ts", "src/cli.ts"],
+	bundle: true,
+	platform: "node",
+	target: "node20",
+	format: "esm",
+	outdir: "dist",
+	sourcemap: true,
+	external: [
+		"@aws-sdk/*", // Use Lambda-provided SDK
+	],
+	banner: {
+		js: "import { createRequire } from 'module'; const require = createRequire(import.meta.url);",
+	},
+});

package/package.json

@@ -0,0 +1,50 @@
+{
+	"name": "@remit/imap-worker",
+	"version": "0.0.1",
+	"type": "module",
+	"main": "dist/index.js",
+	"bin": {
+		"remit-worker": "./dist/cli.js"
+	},
+	"scripts": {
+		"bundle": "esbuild src/index.ts --sourcemap --bundle --platform=node --format=esm --outfile=dist/index.js",
+		"cli": "node --env-file=../../localhost-dev-aws.env src/cli.ts",
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node --env-file=../../localhost-test-unit.env --test 'src/**/*.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"devDependencies": {
+		"@aws-sdk/client-sqs": "*",
+		"@aws-sdk/client-ssm": "*",
+		"@aws-sdk/core": "*",
+		"@remit/backend": "*",
+		"@remit/data-ports": "*",
+		"@remit/domain-enums": "*",
+		"@remit/logger-lambda": "*",
+		"@remit/mail-oauth-service": "*",
+		"@remit/mailbox-service": "*",
+		"@remit/search-index-worker": "*",
+		"@remit/secrets-service": "*",
+		"@remit/sqs-client": "*",
+		"@remit/storage-service": "*",
+		"@types/aws-lambda": "*",
+		"@types/nodemailer": "*",
+		"aws-sdk-client-mock": "*",
+		"esbuild": "^0.27.7",
+		"expect-env": "*",
+		"p-map": "*"
+	},
+	"dependencies": {
+		"@aws-sdk/client-s3": "^3.1055.0",
+		"nodemailer": "^9.0.3"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/remit.git",
+		"directory": "packages/imap-worker"
+	}
+}

package/src/account-check.test.ts

@@ -0,0 +1,122 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { AccountItem } from "@remit/data-ports";
+import {
+	isAccountDeleted,
+	isAccountReauthRequired,
+	isReservedHost,
+	isUnsyncableHost,
+} from "./account-check.js";
+
+const log = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+} as unknown as Parameters<typeof isAccountDeleted>[1];
+
+const buildAccount = (overrides: Partial<AccountItem> = {}): AccountItem =>
+	({
+		accountId: "acct-1",
+		connectionState: "authenticated",
+		...overrides,
+	}) as unknown as AccountItem;
+
+describe("isAccountDeleted", () => {
+	it("returns false when deletedAt is absent", () => {
+		assert.equal(isAccountDeleted(buildAccount(), log), false);
+	});
+
+	it("returns true when deletedAt is set", () => {
+		assert.equal(
+			isAccountDeleted(buildAccount({ deletedAt: Date.now() }), log),
+			true,
+		);
+	});
+});
+
+describe("isReservedHost", () => {
+	const reserved = [
+		"invalid",
+		"example",
+		"imap.invalid",
+		"foo.invalid",
+		"server.invalid",
+		"SERVER.INVALID",
+		"mail.example",
+		"foo.example",
+		"  imap.invalid  ",
+	];
+	for (const host of reserved) {
+		it(`returns true for reserved host ${JSON.stringify(host)}`, () => {
+			assert.equal(isReservedHost(host), true);
+		});
+	}
+
+	// localhost / .localhost / .test are legitimately used by local & e2e envs,
+	// so they must NOT be skipped (issue #835 follow-up).
+	const real = [
+		"imap.hostnet.nl",
+		"imap.gmail.com",
+		"localhost",
+		"foo.localhost",
+		"foo.test",
+		"notlocalhost",
+		"invalid.com",
+		"example.com",
+		"test.org",
+	];
+	for (const host of real) {
+		it(`returns false for real host ${JSON.stringify(host)}`, () => {
+			assert.equal(isReservedHost(host), false);
+		});
+	}
+});
+
+describe("isUnsyncableHost", () => {
+	it("returns false for a resolvable host", () => {
+		assert.equal(
+			isUnsyncableHost(buildAccount({ imapHost: "imap.gmail.com" }), log),
+			false,
+		);
+	});
+
+	it("returns true for a reserved host", () => {
+		assert.equal(
+			isUnsyncableHost(buildAccount({ imapHost: "imap.invalid" }), log),
+			true,
+		);
+	});
+});
+
+describe("isAccountReauthRequired", () => {
+	it("returns false for authenticated accounts", () => {
+		assert.equal(
+			isAccountReauthRequired(
+				buildAccount({ connectionState: "authenticated" }),
+				log,
+			),
+			false,
+		);
+	});
+
+	it("returns false for not_authenticated accounts", () => {
+		assert.equal(
+			isAccountReauthRequired(
+				buildAccount({ connectionState: "not_authenticated" }),
+				log,
+			),
+			false,
+		);
+	});
+
+	it("returns true when connectionState is reauth_required", () => {
+		assert.equal(
+			isAccountReauthRequired(
+				buildAccount({ connectionState: "reauth_required" }),
+				log,
+			),
+			true,
+		);
+	});
+});

package/src/account-check.ts

@@ -0,0 +1,88 @@
+import type { AccountItem } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+
+/**
+ * RFC 2606 reserved placeholder namespaces that are guaranteed never to resolve
+ * and are not used by any local/dev/e2e environment: `.invalid` (the actual
+ * smoke-test placeholder) and `.example`. An account pointed at one of these can
+ * never connect, so syncing it would retry and dead-letter forever — we skip it.
+ *
+ * `localhost`/`.localhost` and `.test` are deliberately NOT skipped: the e2e and
+ * mailfuzz suites run their IMAP server on `localhost`, and `.test` is commonly
+ * used for local testing.
+ */
+const RESERVED_NAMES = ["invalid", "example"] as const;
+const RESERVED_HOST_SUFFIXES = RESERVED_NAMES.map((name) => `.${name}`);
+
+/**
+ * True when `host` is a reserved placeholder name that can never resolve.
+ *
+ * Matches the bare `invalid`/`example` and any subdomain of those namespaces
+ * (`mail.example`, `foo.invalid`, …). Suffix matching is anchored on the dot so
+ * real hosts like `invalid.com` or `imap.gmail.com` are NOT treated as reserved.
+ */
+export const isReservedHost = (host: string): boolean => {
+	const normalized = host.trim().toLowerCase();
+	if (RESERVED_NAMES.includes(normalized as (typeof RESERVED_NAMES)[number])) {
+		return true;
+	}
+	return RESERVED_HOST_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
+};
+
+/**
+ * Check if an account's IMAP host can never resolve (reserved TLD).
+ * Returns true if the account should be skipped cleanly — no connection
+ * attempt, no thrown error, so the event is acked rather than dead-lettered.
+ */
+export const isUnsyncableHost = (
+	account: AccountItem,
+	log: Logger,
+): boolean => {
+	if (isReservedHost(account.imapHost)) {
+		log.warn(
+			{ accountId: account.accountId, imapHost: account.imapHost },
+			"Skipping account: IMAP host is a reserved, never-resolvable name",
+		);
+		return true;
+	}
+	return false;
+};
+
+/**
+ * Check if an account is deleted (tombstone pattern).
+ * Returns true if the account should be skipped.
+ */
+export const isAccountDeleted = (
+	account: AccountItem,
+	log: Logger,
+): boolean => {
+	if (account.deletedAt) {
+		log.info(
+			{ accountId: account.accountId, deletedAt: account.deletedAt },
+			"Skipping deleted account",
+		);
+		return true;
+	}
+	return false;
+};
+
+/**
+ * Check if an account requires re-authentication (e.g. OAuth token revoked).
+ * Returns true if the account should be skipped until the user re-auths.
+ */
+export const isAccountReauthRequired = (
+	account: AccountItem,
+	log: Logger,
+): boolean => {
+	if (account.connectionState === "reauth_required") {
+		log.info(
+			{
+				accountId: account.accountId,
+				connectionState: account.connectionState,
+			},
+			"Skipping account: reauth required",
+		);
+		return true;
+	}
+	return false;
+};

package/src/body-sync-gate.test.ts

@@ -0,0 +1,185 @@
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import {
+	GetParameterCommand,
+	ParameterNotFound,
+	SSMClient,
+} from "@aws-sdk/client-ssm";
+import type { Logger } from "@remit/logger-lambda";
+import { mockClient } from "aws-sdk-client-mock";
+import { isBodySyncEnabled, resetBodySyncGateCache } from "./body-sync-gate.js";
+
+const parameterName = "/dev/Remit/bodySyncEnabled";
+
+interface CapturedWarn {
+	args: unknown[];
+}
+
+const createCapturingLogger = (): {
+	log: Logger;
+	warnCalls: CapturedWarn[];
+} => {
+	const warnCalls: CapturedWarn[] = [];
+	const noop = () => {};
+	const log = {
+		info: noop,
+		warn: (...args: unknown[]) => warnCalls.push({ args }),
+		error: noop,
+		debug: noop,
+		fatal: noop,
+		trace: noop,
+		child: () => log,
+	} as unknown as Logger;
+	return { log, warnCalls };
+};
+
+const warnMessage = (warn: CapturedWarn): string => String(warn.args[1]);
+
+describe("isBodySyncEnabled", () => {
+	beforeEach(() => {
+		resetBodySyncGateCache();
+	});
+
+	afterEach(() => {
+		mockClient(SSMClient).reset();
+		resetBodySyncGateCache();
+	});
+
+	it("returns true when the parameter is 'true'", async () => {
+		const ssmMock = mockClient(SSMClient);
+		ssmMock
+			.on(GetParameterCommand, { Name: parameterName })
+			.resolves({ Parameter: { Value: "true" } });
+		const { log } = createCapturingLogger();
+
+		const enabled = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.equal(enabled, true);
+	});
+
+	it("returns false when the parameter is 'false'", async () => {
+		const ssmMock = mockClient(SSMClient);
+		ssmMock
+			.on(GetParameterCommand, { Name: parameterName })
+			.resolves({ Parameter: { Value: "false" } });
+		const { log } = createCapturingLogger();
+
+		const enabled = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.equal(enabled, false);
+	});
+
+	it("treats casing and whitespace as disabled for ' FALSE '", async () => {
+		const ssmMock = mockClient(SSMClient);
+		ssmMock
+			.on(GetParameterCommand, { Name: parameterName })
+			.resolves({ Parameter: { Value: " FALSE " } });
+		const { log } = createCapturingLogger();
+
+		const enabled = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.equal(enabled, false);
+	});
+
+	it("fails open with a warning when the parameter does not exist", async () => {
+		const ssmMock = mockClient(SSMClient);
+		ssmMock
+			.on(GetParameterCommand, { Name: parameterName })
+			.rejects(new ParameterNotFound({ message: "not found", $metadata: {} }));
+		const { log, warnCalls } = createCapturingLogger();
+
+		const enabled = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.equal(enabled, true);
+		assert.equal(warnCalls.length, 1);
+		assert.match(warnMessage(warnCalls[0]), /failing open/i);
+	});
+
+	it("fails open with a warning when the parameter has no value", async () => {
+		const ssmMock = mockClient(SSMClient);
+		ssmMock
+			.on(GetParameterCommand, { Name: parameterName })
+			.resolves({ Parameter: {} });
+		const { log, warnCalls } = createCapturingLogger();
+
+		const enabled = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.equal(enabled, true);
+		assert.equal(warnCalls.length, 1);
+		assert.match(warnMessage(warnCalls[0]), /missing a value/i);
+	});
+
+	it("caches the result within the TTL — second call does not re-fetch", async () => {
+		const ssmMock = mockClient(SSMClient);
+		ssmMock
+			.on(GetParameterCommand, { Name: parameterName })
+			.resolves({ Parameter: { Value: "false" } });
+		const { log } = createCapturingLogger();
+
+		const first = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+		const second = await isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.equal(first, false);
+		assert.equal(second, false);
+		assert.equal(ssmMock.commandCalls(GetParameterCommand).length, 1);
+	});
+
+	it("coalesces concurrent cold-cache calls into a single GetParameter", async () => {
+		const ssmMock = mockClient(SSMClient);
+		let resolveSend:
+			| ((value: { Parameter: { Value: string } }) => void)
+			| undefined;
+		ssmMock.on(GetParameterCommand, { Name: parameterName }).callsFake(
+			() =>
+				new Promise((resolve) => {
+					resolveSend = resolve;
+				}),
+		);
+		const { log } = createCapturingLogger();
+
+		const a = isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+		const b = isBodySyncEnabled(
+			parameterName,
+			log,
+			ssmMock as unknown as SSMClient,
+		);
+
+		assert.ok(resolveSend, "GetParameter should have been invoked");
+		resolveSend?.({ Parameter: { Value: "true" } });
+
+		assert.deepEqual(await Promise.all([a, b]), [true, true]);
+		assert.equal(ssmMock.commandCalls(GetParameterCommand).length, 1);
+	});
+});

package/src/body-sync-gate.ts

@@ -0,0 +1,86 @@
+import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
+import type { Logger } from "@remit/logger-lambda";
+
+const CACHE_TTL_MS = 30_000;
+
+interface CacheEntry {
+	readonly enabled: boolean;
+	readonly expiresAt: number;
+}
+
+let cache: CacheEntry | undefined;
+let inFlight: Promise<boolean> | undefined;
+let defaultClient: SSMClient | undefined;
+
+const getDefaultClient = (): SSMClient => {
+	if (!defaultClient) {
+		defaultClient = new SSMClient({});
+	}
+	return defaultClient;
+};
+
+/**
+ * Test seam — clears the module-scope TTL cache between unit tests.
+ */
+export const resetBodySyncGateCache = (): void => {
+	cache = undefined;
+	inFlight = undefined;
+};
+
+const readParameter = async (
+	ssm: SSMClient,
+	parameterName: string,
+	log: Logger,
+): Promise<boolean> => {
+	const result = await ssm.send(
+		new GetParameterCommand({ Name: parameterName }),
+	);
+	const value = result.Parameter?.Value;
+
+	if (value === undefined) {
+		log.warn(
+			{ parameterName },
+			"Body-sync toggle parameter missing a value, failing open (enabled)",
+		);
+		return true;
+	}
+
+	return value.trim().toLowerCase() !== "false";
+};
+
+/**
+ * Reads the `/{stage}/Remit/bodySyncEnabled` SSM parameter, cached at module
+ * scope for a short TTL. Returns `true` (enabled) when the parameter is absent
+ * or unreadable — body sync fails open so a missing toggle never halts prod.
+ */
+export const isBodySyncEnabled = async (
+	parameterName: string,
+	log: Logger,
+	ssm: SSMClient = getDefaultClient(),
+): Promise<boolean> => {
+	if (cache && cache.expiresAt > Date.now()) {
+		return cache.enabled;
+	}
+
+	if (inFlight) {
+		return inFlight;
+	}
+
+	inFlight = readParameter(ssm, parameterName, log)
+		.catch((error) => {
+			log.warn(
+				{ parameterName, error },
+				"Failed to read body-sync toggle parameter, failing open (enabled)",
+			);
+			return true;
+		})
+		.then((enabled) => {
+			cache = { enabled, expiresAt: Date.now() + CACHE_TTL_MS };
+			return enabled;
+		})
+		.finally(() => {
+			inFlight = undefined;
+		});
+
+	return inFlight;
+};