@remit/backend 0.0.122 → 0.0.124

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.122",
+  "version": "0.0.124",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/scripts/backfill-classification.ts

@@ -0,0 +1,89 @@
+import { existsSync } from "node:fs";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { logger } from "@remit/logger-lambda";
+import {
+	backfillClassifications,
+	type ClassificationBackfillCheckpoint,
+	type ClassificationBackfillCheckpointStore,
+} from "@remit/mailbox-service";
+import { getClient } from "../src/service/data-client.js";
+
+/**
+ * One-time, full-corpus classification backfill over the stored-body cohort
+ * the classifier never examined (issue #1197): rows whose bodies were synced
+ * before the classifier reached them, which carry `NotExamined` on
+ * `Message.classificationState` and so stay `uncategorized` in the list. The
+ * same alternate-entrypoint shape `backfill-list-id.mjs` uses: baked into the
+ * backend image, never wired into a compose one-shot, run by hand once per
+ * install:
+ *
+ *   docker compose -f docker-compose.sqlite.yml run --rm backend \
+ *     node backfill-classification.mjs
+ *
+ * Safe to interrupt: progress is checkpointed to disk after every page and
+ * picked back up on the next run, every already-examined row reads as
+ * `alreadyExamined` on a rerun, and a row whose category was already decided
+ * is recorded without re-deriving it, so a partial pass resumes rather than
+ * repeats.
+ */
+
+const CHECKPOINT_PATH =
+	process.env.CLASSIFICATION_BACKFILL_CHECKPOINT_PATH ??
+	"/data/sqlite/classification-backfill-checkpoint.json";
+
+const isCheckpoint = (
+	value: unknown,
+): value is ClassificationBackfillCheckpoint => {
+	if (typeof value !== "object" || value === null) return false;
+	if (!("accountConfigId" in value)) return false;
+	const candidate = value as Record<string, unknown>;
+	return (
+		typeof candidate.accountConfigId === "string" &&
+		(candidate.continuationToken === undefined ||
+			typeof candidate.continuationToken === "string")
+	);
+};
+
+const fileCheckpointStore: ClassificationBackfillCheckpointStore = {
+	load: async () => {
+		if (!existsSync(CHECKPOINT_PATH)) return undefined;
+		const raw = await readFile(CHECKPOINT_PATH, "utf8");
+		const parsed: unknown = JSON.parse(raw);
+		return isCheckpoint(parsed) ? parsed : undefined;
+	},
+	save: async (checkpoint) => {
+		await mkdir(dirname(CHECKPOINT_PATH), { recursive: true });
+		const staging = `${CHECKPOINT_PATH}.writing`;
+		await writeFile(staging, JSON.stringify(checkpoint));
+		await rename(staging, CHECKPOINT_PATH);
+	},
+	clear: async () => {
+		await rm(CHECKPOINT_PATH, { force: true });
+	},
+};
+
+const run = async (): Promise<void> => {
+	const client = await getClient();
+
+	const result = await backfillClassifications(
+		{
+			accountConfigService: client.accountConfig,
+			addressService: client.address,
+			threadMessageService: client.threadMessage,
+			messageService: client.message,
+			storageService: client.storage,
+		},
+		{ checkpointStore: fileCheckpointStore, logger },
+	);
+
+	// biome-ignore lint/plugin/no-logger-info: a completed full-corpus backfill is an audit-grade signal
+	logger.info({ result }, "backfill done");
+};
+
+run()
+	.then(() => process.exit(0))
+	.catch((error: unknown) => {
+		logger.error({ error }, "backfill failed");
+		process.exit(1);
+	});

package/src/handlers/account-guards.ts

@@ -1,10 +1,11 @@
 import type {
 	AccountResponse,
+	AccountService as AccountServiceName,
 	FolderAppointment,
 } from "@remit/api-openapi-types";
 import type { AccountItem } from "@remit/data-ports";
 import { BadRequestError, ConflictError } from "@remit/data-ports/errors";
-import { AccountAuthType } from "@remit/domain-enums";
+import { AccountAuthType, AccountService } from "@remit/domain-enums";
 import type { AccountOverrides } from "./account-overrides.js";
 import type { AccountSignature } from "./account-signature.js";
 
@@ -84,6 +85,41 @@ export const assertPasswordProvided = (
 	}
 };
 
+const MICROSOFT_GRANT_SERVICES: readonly AccountServiceName[] = [
+	AccountService.Mail,
+];
+
+export const resolveSyncedServices = (
+	authType: string,
+	requested: readonly AccountServiceName[],
+): AccountServiceName[] => {
+	const services = Object.values(AccountService).filter((service) =>
+		requested.includes(service),
+	);
+	if (services.length === 0) {
+		throw new BadRequestError(
+			"syncedServices must name at least one service. To stop syncing an account entirely, delete it with DELETE /accounts/{accountId}.",
+		);
+	}
+	if (authType !== AccountAuthType.OauthMicrosoft) {
+		if (services.some((service) => service !== AccountService.Mail)) {
+			throw new BadRequestError(
+				`A password account syncs mail only, so syncedServices must be ["${AccountService.Mail}"].`,
+			);
+		}
+		return services;
+	}
+	const uncovered = services.filter(
+		(service) => !MICROSOFT_GRANT_SERVICES.includes(service),
+	);
+	if (uncovered.length > 0) {
+		throw new BadRequestError(
+			`This account's Microsoft consent does not cover ${uncovered.join(", ")}. Grant it through POST /accounts/oauth/microsoft/start.`,
+		);
+	}
+	return services;
+};
+
 // SECURITY: passwordHash, oauthRefreshTokenHash, and smtpPasswordHash are
 // intentionally omitted — never expose token material in API responses.
 // Display name, mute flag, and signatures live in per-account AccountSetting
@@ -103,6 +139,7 @@ export const toAccountResponse = (
 	username: account.username,
 	email: account.email,
 	authType: account.authType ?? AccountAuthType.Password,
+	syncedServices: account.syncedServices,
 	imapHost: account.imapHost,
 	imapPort: account.imapPort,
 	imapTls: account.imapTls,

package/src/handlers/account-synced-services.test.ts

@@ -0,0 +1,217 @@
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type { AccountResponse } from "@remit/api-openapi-types";
+import type { AccountItem } from "@remit/data-ports";
+import {
+	AccountAuthType,
+	AccountService,
+	ConnectionState,
+} from "@remit/domain-enums";
+import {
+	AccountConfigRepo,
+	AccountRepo,
+	AccountSettingRepo,
+	ConfigImportRepo,
+	MailboxRepo,
+} from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
+import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
+import type { Context } from "openapi-backend";
+import { deriveAccountConfigId } from "../auth.js";
+import { handleError } from "../error.js";
+import { formatResponse } from "../response.js";
+import {
+	_resetForTest,
+	type RemitClient,
+	setClient,
+} from "../service/data-client.js";
+import { AccountDetailOperations, AccountOperations } from "./account.js";
+import { ConfigOperations } from "./config.js";
+
+const SUB = "cognito-sub-1180";
+const ACCOUNT_CONFIG_ID = deriveAccountConfigId(SUB);
+
+type Handler = (
+	context: Context,
+	event: APIGatewayProxyEvent,
+) => Promise<unknown>;
+
+const getConfig = ConfigOperations.ConfigOperations_getConfig as Handler;
+const createAccount =
+	AccountOperations.AccountOperations_createAccount as Handler;
+const updateAccount =
+	AccountDetailOperations.AccountDetailOperations_updateAccount as Handler;
+
+const event = (body?: unknown): APIGatewayProxyEvent =>
+	({
+		body: body === undefined ? null : JSON.stringify(body),
+		requestContext: { authorizer: { claims: { sub: SUB } } },
+	}) as unknown as APIGatewayProxyEvent;
+
+const context = (params: Record<string, string> = {}): Context =>
+	({ request: { params } }) as unknown as Context;
+
+const respond = async (
+	run: () => Promise<unknown>,
+): Promise<APIGatewayProxyResult> =>
+	run().then(
+		(body) => formatResponse(body as Record<string, unknown>),
+		(error: unknown) => handleError(error),
+	);
+
+const bodyOf = <T>(response: APIGatewayProxyResult): T =>
+	JSON.parse(response.body) as T;
+
+let accounts: AccountRepo;
+let close: () => void;
+
+const seedAccount = (authType: AccountItem["authType"]) =>
+	accounts.create({
+		accountConfigId: ACCOUNT_CONFIG_ID,
+		username: `${authType}@example.com`,
+		email: `${authType}@example.com`,
+		authType,
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		isActive: true,
+		connectionState: ConnectionState.NotAuthenticated,
+	});
+
+const readServices = async (accountId: string): Promise<string[]> => {
+	const config = bodyOf<{ accounts: AccountResponse[] }>(
+		await respond(() => getConfig(context(), event())),
+	);
+	const account = config.accounts.find((item) => item.accountId === accountId);
+	assert.ok(account, `GET /config lists ${accountId}`);
+	return account.syncedServices;
+};
+
+const patchServices = (accountId: string, syncedServices: string[]) =>
+	respond(() =>
+		updateAccount(context({ accountId }), event({ syncedServices })),
+	);
+
+beforeEach(async () => {
+	const store = createShippedSqliteDb();
+	close = store.close;
+	accounts = new AccountRepo(store.db);
+	const accountConfig = new AccountConfigRepo(store.db);
+	await accountConfig.create({
+		accountConfigId: ACCOUNT_CONFIG_ID,
+		userId: SUB,
+	});
+	setClient({
+		accountConfig,
+		account: accounts,
+		accountSetting: new AccountSettingRepo(store.db),
+		mailbox: new MailboxRepo(store.db),
+		configImport: new ConfigImportRepo(store.db),
+	} as unknown as RemitClient);
+});
+
+afterEach(() => {
+	_resetForTest();
+	close();
+});
+
+describe("the services an account syncs", () => {
+	it("reads mail for an account that never chose", async () => {
+		const account = await seedAccount(AccountAuthType.Password);
+
+		assert.deepEqual(await readServices(account.accountId), [
+			AccountService.Mail,
+		]);
+	});
+
+	it("writes a selection the account can serve and reads it back", async () => {
+		const account = await seedAccount(AccountAuthType.OauthMicrosoft);
+
+		const response = await patchServices(account.accountId, [
+			AccountService.Mail,
+			AccountService.Mail,
+		]);
+
+		assert.equal(response.statusCode, 200);
+		assert.deepEqual(bodyOf<AccountResponse>(response).syncedServices, [
+			AccountService.Mail,
+		]);
+		assert.deepEqual(await readServices(account.accountId), [
+			AccountService.Mail,
+		]);
+	});
+
+	it("refuses an empty selection and points at deleting the account", async () => {
+		const account = await seedAccount(AccountAuthType.OauthMicrosoft);
+
+		const response = await patchServices(account.accountId, []);
+
+		assert.equal(response.statusCode, 400);
+		assert.match(
+			bodyOf<{ message: string }>(response).message,
+			/DELETE \/accounts\/\{accountId\}/,
+		);
+		assert.deepEqual(await readServices(account.accountId), [
+			AccountService.Mail,
+		]);
+	});
+
+	it("refuses calendar on a password account", async () => {
+		const account = await seedAccount(AccountAuthType.Password);
+
+		const response = await patchServices(account.accountId, [
+			AccountService.Mail,
+			AccountService.Calendar,
+		]);
+
+		assert.equal(response.statusCode, 400);
+		assert.match(
+			bodyOf<{ message: string }>(response).message,
+			/password account syncs mail only/,
+		);
+		assert.deepEqual(await readServices(account.accountId), [
+			AccountService.Mail,
+		]);
+	});
+
+	it("refuses calendar the Microsoft consent never covered and names the consent route", async () => {
+		const account = await seedAccount(AccountAuthType.OauthMicrosoft);
+
+		const response = await patchServices(account.accountId, [
+			AccountService.Calendar,
+		]);
+
+		assert.equal(response.statusCode, 400);
+		assert.match(
+			bodyOf<{ message: string }>(response).message,
+			/POST \/accounts\/oauth\/microsoft\/start/,
+		);
+		assert.deepEqual(await readServices(account.accountId), [
+			AccountService.Mail,
+		]);
+	});
+
+	it("refuses to create a password account that syncs calendar", async () => {
+		const response = await respond(() =>
+			createAccount(
+				context(),
+				event({
+					email: "person@example.com",
+					password: "hunter2",
+					imapHost: "imap.example.com",
+					imapPort: 993,
+					imapTls: true,
+					imapStartTls: false,
+					syncedServices: [AccountService.Calendar],
+				}),
+			),
+		);
+
+		assert.equal(response.statusCode, 400);
+		assert.deepEqual(
+			await accounts.listAllByAccountConfig(ACCOUNT_CONFIG_ID),
+			[],
+		);
+	});
+});

package/src/handlers/account.ts

@@ -8,7 +8,11 @@ import type {
 	UpdateAccountInput,
 } from "@remit/api-openapi-types";
 import type { AccountItem, IAccountRepository } from "@remit/data-ports";
-import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
+import {
+	AccountAuthType,
+	AccountService,
+	ConnectionState,
+} from "@remit/domain-enums";
 import { logger } from "@remit/logger-lambda";
 import {
 	createMailOAuthService,
@@ -42,6 +46,7 @@ import {
 	assertNoDuplicateMailbox,
 	assertNotOAuthCreate,
 	assertPasswordProvided,
+	resolveSyncedServices,
 	toAccountResponse,
 } from "./account-guards.js";
 import {
@@ -203,6 +208,11 @@ export const AccountOperations: Record<
 		// Password accounts require a non-empty password
 		assertPasswordProvided(input.authType, input.password);
 
+		const syncedServices = resolveSyncedServices(
+			AccountAuthType.Password,
+			input.syncedServices ?? [AccountService.Mail],
+		);
+
 		const { account, accountConfig, accountSetting, mailbox, secrets } =
 			await getClient();
 
@@ -235,6 +245,7 @@ export const AccountOperations: Record<
 			email: input.email,
 			username: input.username ?? input.email,
 			authType: AccountAuthType.Password,
+			syncedServices,
 			passwordHash,
 			imapHost: input.imapHost,
 			imapPort: input.imapPort,
@@ -405,6 +416,13 @@ export const AccountDetailOperations: Record<
 			);
 		}
 
+		if (input.syncedServices !== undefined) {
+			updates.syncedServices = resolveSyncedServices(
+				existing.authType,
+				input.syncedServices,
+			);
+		}
+
 		// Copy non-password fields
 		const fields = [
 			"imapHost",