@remit/backend 0.0.121 → 0.0.123

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.121",
+  "version": "0.0.123",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/scripts/backfill-classification.ts

@@ -0,0 +1,89 @@
+import { existsSync } from "node:fs";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { logger } from "@remit/logger-lambda";
+import {
+	backfillClassifications,
+	type ClassificationBackfillCheckpoint,
+	type ClassificationBackfillCheckpointStore,
+} from "@remit/mailbox-service";
+import { getClient } from "../src/service/data-client.js";
+
+/**
+ * One-time, full-corpus classification backfill over the stored-body cohort
+ * the classifier never examined (issue #1197): rows whose bodies were synced
+ * before the classifier reached them, which carry `NotExamined` on
+ * `Message.classificationState` and so stay `uncategorized` in the list. The
+ * same alternate-entrypoint shape `backfill-list-id.mjs` uses: baked into the
+ * backend image, never wired into a compose one-shot, run by hand once per
+ * install:
+ *
+ *   docker compose -f docker-compose.sqlite.yml run --rm backend \
+ *     node backfill-classification.mjs
+ *
+ * Safe to interrupt: progress is checkpointed to disk after every page and
+ * picked back up on the next run, every already-examined row reads as
+ * `alreadyExamined` on a rerun, and a row whose category was already decided
+ * is recorded without re-deriving it, so a partial pass resumes rather than
+ * repeats.
+ */
+
+const CHECKPOINT_PATH =
+	process.env.CLASSIFICATION_BACKFILL_CHECKPOINT_PATH ??
+	"/data/sqlite/classification-backfill-checkpoint.json";
+
+const isCheckpoint = (
+	value: unknown,
+): value is ClassificationBackfillCheckpoint => {
+	if (typeof value !== "object" || value === null) return false;
+	if (!("accountConfigId" in value)) return false;
+	const candidate = value as Record<string, unknown>;
+	return (
+		typeof candidate.accountConfigId === "string" &&
+		(candidate.continuationToken === undefined ||
+			typeof candidate.continuationToken === "string")
+	);
+};
+
+const fileCheckpointStore: ClassificationBackfillCheckpointStore = {
+	load: async () => {
+		if (!existsSync(CHECKPOINT_PATH)) return undefined;
+		const raw = await readFile(CHECKPOINT_PATH, "utf8");
+		const parsed: unknown = JSON.parse(raw);
+		return isCheckpoint(parsed) ? parsed : undefined;
+	},
+	save: async (checkpoint) => {
+		await mkdir(dirname(CHECKPOINT_PATH), { recursive: true });
+		const staging = `${CHECKPOINT_PATH}.writing`;
+		await writeFile(staging, JSON.stringify(checkpoint));
+		await rename(staging, CHECKPOINT_PATH);
+	},
+	clear: async () => {
+		await rm(CHECKPOINT_PATH, { force: true });
+	},
+};
+
+const run = async (): Promise<void> => {
+	const client = await getClient();
+
+	const result = await backfillClassifications(
+		{
+			accountConfigService: client.accountConfig,
+			addressService: client.address,
+			threadMessageService: client.threadMessage,
+			messageService: client.message,
+			storageService: client.storage,
+		},
+		{ checkpointStore: fileCheckpointStore, logger },
+	);
+
+	// biome-ignore lint/plugin/no-logger-info: a completed full-corpus backfill is an audit-grade signal
+	logger.info({ result }, "backfill done");
+};
+
+run()
+	.then(() => process.exit(0))
+	.catch((error: unknown) => {
+		logger.error({ error }, "backfill failed");
+		process.exit(1);
+	});

package/src/index.test.ts

@@ -0,0 +1,83 @@
+/**
+ * Issue #1054: the backend must log one structured access line per request
+ * (method, path, status, duration, correlation id) at a level that is on in
+ * production. The access log lives in `rawHandler` in `src/index.ts`, wrapping
+ * the authenticated request path — so a 401 denial from edge-auth also gets an
+ * access line (once it flows through `formatResponse` with the correlation id).
+ *
+ * The 401 response shape (correlation id header + CORS on the denial) is tested
+ * directly in `jwt-auth.test.ts` via `runWithRequestContext`; this file tests
+ * that the handler emits the access log line on every request by spying on the
+ * logger's info method.
+ */
+import assert from "node:assert/strict";
+import { describe, it, mock } from "node:test";
+import { logger } from "@remit/logger-lambda";
+import type { APIGatewayProxyEvent, Context } from "aws-lambda";
+import { handler } from "./index.js";
+
+const buildEvent = (
+	overrides: Partial<APIGatewayProxyEvent> = {},
+): APIGatewayProxyEvent =>
+	({
+		path: "/api/test",
+		httpMethod: "GET",
+		pathParameters: {},
+		queryStringParameters: {},
+		headers: {},
+		requestContext: {},
+		...overrides,
+	}) as unknown as APIGatewayProxyEvent;
+
+const buildContext = (correlationId = "ctx-test-1"): Context =>
+	({
+		awsRequestId: correlationId,
+		functionName: "test-function",
+		getRemainingTimeInMillis: () => 30000,
+	}) as unknown as Context;
+
+describe("issue #1054: per-request access log", () => {
+	it('emits one "Request completed" line at info level with method, path, status, duration and correlation id', async () => {
+		delete process.env.LOCAL_ACCOUNT_CONFIG_ID;
+
+		// Spy on logger.info — the real logger passes (fields, message) in
+		// pino order, so the spy captures the structured fields and message.
+		const infoSpy = mock.method(logger, "info");
+
+		try {
+			await handler(
+				buildEvent({ path: "/api/__nonexistent_for_test__" }),
+				buildContext("ctx-access-log"),
+			).catch(() => {
+				// 404s are expected; we only care about the log line.
+			});
+
+			// `logger.info` is overloaded ((obj, msg?) and (msg, obj?)), so the
+			// mock types `call.arguments` by a single overload. Read the
+			// arguments positionally through an unknown[] view instead.
+			const accessCalls = infoSpy.mock.calls.filter((call) => {
+				// pino calls logger.info(fields, message)
+				const args = call.arguments as unknown[];
+				return args[1] === "Request completed";
+			});
+
+			assert.ok(accessCalls.length >= 1, "expected an access log line");
+			const fields = (accessCalls[0].arguments as unknown[])[0] as Record<
+				string,
+				unknown
+			>;
+
+			// infoSpy was called on logger.info, so the level is "info" by
+			// construction — pino's adapter routes .info() calls to the info
+			// level. No separate level field needed.
+
+			assert.equal(fields.method, "GET");
+			assert.equal(fields.path, "/api/__nonexistent_for_test__");
+			assert.equal(fields.correlationId, "ctx-access-log");
+			assert.equal(typeof fields.statusCode, "number");
+			assert.equal(typeof fields.durationMs, "number");
+		} finally {
+			infoSpy.mock.restore();
+		}
+	});
+});

package/src/index.ts

@@ -180,9 +180,17 @@ const rawHandler = async (event: APIGatewayProxyEvent, context: Context) => {
 			logger.debug({ method: event.httpMethod, path }, "Request received");
 
 			const origin = readOriginHeader(event.headers);
-
-			return runWithRequestContext(
-				{ origin, correlationId: context.awsRequestId },
+			const startedAt = Date.now();
+			// Capture the correlation id here: it is seeded by
+			// runWithRequestContext below (from context.awsRequestId), but that
+			// AsyncLocalStorage scope does not reliably survive the
+			// openapi-backend handleRequest call chain, so we read it from
+			// `context.awsRequestId` directly rather than via
+			// getRequestCorrelationId at log time.
+			const correlationId = context.awsRequestId;
+
+			const result = await runWithRequestContext(
+				{ origin, correlationId },
 				async () => {
 					if (usesBetterAuthJwt()) {
 						const denied = await authenticateSelfHostRequest(event);
@@ -194,6 +202,25 @@ const rawHandler = async (event: APIGatewayProxyEvent, context: Context) => {
 						.catch(handleError);
 				},
 			);
+
+			// One structured access line per request: method, path, status, and
+			// duration. The correlation id is inlined as a top-level field so a bug
+			// report quoting the id can grep logs by it directly, and emitted at
+			// info so it is on by default in production (the debug "Request
+			// received" above already filters out under normal log levels).
+			// biome-ignore lint/plugin/no-logger-info: one access line per request is an audit-grade signal (issue #1054)
+			logger.info(
+				{
+					method: event.httpMethod,
+					path,
+					statusCode: result.statusCode,
+					durationMs: Date.now() - startedAt,
+					correlationId,
+				},
+				"Request completed",
+			);
+
+			return result;
 		},
 	);
 };

package/src/jwt-auth.test.ts

@@ -5,6 +5,7 @@ import {
 	_setVerifierForTest,
 	authenticateSelfHostRequest,
 } from "./jwt-auth.js";
+import { runWithRequestContext } from "./request-context.js";
 
 const buildEvent = (
 	overrides: Partial<APIGatewayProxyEvent> = {},
@@ -41,7 +42,7 @@ test("valid token injects verified sub into authorizer claims", async () => {
 	assert.equal(event.requestContext.authorizer?.claims?.email, "a@b.com");
 });
 
-test("invalid token returns 401 and does not inject claims", async () => {
+test("invalid token returns 401 with correlation id and CORS headers", async () => {
 	_setVerifierForTest(async () => {
 		throw new Error("bad signature");
 	});
@@ -49,9 +50,29 @@ test("invalid token returns 401 and does not inject claims", async () => {
 		headers: { authorization: "Bearer bad.token" },
 	});
 
-	const result = await authenticateSelfHostRequest(event);
+	const result = await runWithRequestContext(
+		{ correlationId: "req-invalid-token" },
+		async () => authenticateSelfHostRequest(event),
+	);
 
 	assert.equal(result?.statusCode, 401);
+	assert.equal(
+		result?.headers?.["x-correlation-id"],
+		"req-invalid-token",
+		"edge-auth 401 must carry the correlation id",
+	);
+	assert.equal(
+		result?.headers?.["Access-Control-Allow-Origin"],
+		"*",
+		"edge-auth 401 must carry CORS headers",
+	);
+	assert.equal(
+		result?.body,
+		JSON.stringify({
+			code: "unauthorized",
+			message: "Invalid or expired token",
+		}),
+	);
 	assert.equal(event.requestContext.authorizer, undefined);
 });
 
@@ -64,12 +85,25 @@ test("no token with a local bypass configured is allowed", async () => {
 	assert.equal(result, null);
 });
 
-test("no token and no bypass returns 401", async () => {
+test("no token and no bypass returns 401 with correlation id", async () => {
 	const event = buildEvent();
 
-	const result = await authenticateSelfHostRequest(event);
+	const result = await runWithRequestContext(
+		{ correlationId: "req-no-token" },
+		async () => authenticateSelfHostRequest(event),
+	);
 
 	assert.equal(result?.statusCode, 401);
+	assert.equal(
+		result?.headers?.["x-correlation-id"],
+		"req-no-token",
+		"edge-auth 401 must carry the correlation id",
+	);
+	assert.equal(
+		result?.headers?.["Access-Control-Allow-Origin"],
+		"*",
+		"edge-auth 401 must carry CORS headers",
+	);
 });
 
 test("a tokenless GET to the Microsoft OAuth callback is admitted", async () => {

package/src/jwt-auth.ts

@@ -8,6 +8,7 @@ import { readCalendarFeedToken } from "@remit/calendar-service";
 import { logger } from "@remit/logger-lambda";
 import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
 import { defaultErrorCode } from "./error.js";
+import { formatResponse } from "./response.js";
 
 let verifier: JwtVerifier | null = null;
 
@@ -31,11 +32,8 @@ const readHeader = (
 	return undefined;
 };
 
-const unauthorized = (message: string): APIGatewayProxyResult => ({
-	statusCode: 401,
-	headers: { "Content-Type": "application/json" },
-	body: JSON.stringify({ code: defaultErrorCode(401), message }),
-});
+const unauthorized = (message: string): APIGatewayProxyResult =>
+	formatResponse({ code: defaultErrorCode(401), message }, 401);
 
 const injectClaims = (
 	event: APIGatewayProxyEvent,