@remit/backend 0.0.44 → 0.0.46

package/dev-server/log-output.test.ts

@@ -0,0 +1,173 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { createServer } from "node:net";
+import { dirname, resolve } from "node:path";
+import { after, before, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+
+// This file is the backend image's entrypoint, so what it writes IS the
+// container's log stream. deploy/vps/README.md ("Logs") promises one JSON object
+// per line, and a log-shipping pipeline written against that contract breaks on
+// the first line it cannot parse — which is why this runs the real server rather
+// than asserting against a mocked writer.
+//
+// Both streams are held to it. The container log driver merges stdout and
+// stderr into one log and `remit logs` shows both, so a raw line on stderr
+// breaks a Vector parser exactly like a raw line on stdout.
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+
+const freePort = (): Promise<number> =>
+	new Promise((resolveWith, reject) => {
+		const probe = createServer();
+		probe.on("error", reject);
+		probe.listen(0, "127.0.0.1", () => {
+			const address = probe.address();
+			if (address === null || typeof address === "string") {
+				probe.close();
+				reject(new Error("could not reserve a port"));
+				return;
+			}
+			probe.close(() => resolveWith(address.port));
+		});
+	});
+
+type Line = Record<string, unknown>;
+type Stream = "stdout" | "stderr";
+
+const captured: Record<Stream, string> = { stdout: "", stderr: "" };
+const listeners = new Set<() => void>();
+let port = 0;
+
+// The child's output reaches this process asynchronously, so a request that has
+// already been answered is not yet a line here. Wait for the line rather than
+// for a duration, and only once the stream ends on a newline, so no assertion
+// ever runs against half of one.
+const waitForStdout = (contains: string): Promise<void> =>
+	new Promise((resolveWith, reject) => {
+		const settled = () =>
+			captured.stdout.includes(contains) && captured.stdout.endsWith("\n");
+		if (settled()) {
+			resolveWith();
+			return;
+		}
+		const timer = setTimeout(() => {
+			stopWaiting();
+			reject(new Error(`stdout never carried ${contains}: ${captured.stdout}`));
+		}, 10_000);
+		const onData = () => {
+			if (!settled()) return;
+			stopWaiting();
+			resolveWith();
+		};
+		const stopWaiting = () => {
+			clearTimeout(timer);
+			listeners.delete(onData);
+		};
+		listeners.add(onData);
+	});
+
+const parsedLines = (stream: Stream): Line[] =>
+	captured[stream]
+		.split("\n")
+		.filter((line) => line.trim().length > 0)
+		.map((line, index) => {
+			try {
+				return JSON.parse(line) as Line;
+			} catch {
+				throw new assert.AssertionError({
+					message: `${stream} line ${index + 1} is not JSON, so a log pipeline drops it: ${line}`,
+				});
+			}
+		});
+
+describe("the backend entrypoint's log output", () => {
+	let server: ReturnType<typeof spawn>;
+
+	before(async () => {
+		port = await freePort();
+
+		server = spawn(
+			process.execPath,
+			["--import", "tsx", "dev-server/server.ts"],
+			{
+				cwd: packageRoot,
+				stdio: ["ignore", "pipe", "pipe", "ipc"],
+				env: {
+					...process.env,
+					SERVER_PORT: String(port),
+					LOG_LEVEL: "debug",
+					REMIT_SERVICE_NAME: "backend",
+					// tsx's own loader warnings are an artifact of running the
+					// TypeScript source; the image runs a bundle and never emits them.
+					NODE_NO_WARNINGS: "1",
+					NODE_OPTIONS: "",
+				},
+			},
+		);
+
+		for (const stream of ["stdout", "stderr"] as const) {
+			server[stream]?.setEncoding("utf8");
+			server[stream]?.on("data", (chunk: string) => {
+				captured[stream] += chunk;
+				for (const listener of [...listeners]) listener();
+			});
+		}
+
+		await new Promise<void>((resolveWith, reject) => {
+			server.on("message", (message) => {
+				if (message === "ready") resolveWith();
+			});
+			// Carry the child's stderr into the failure: a server that cannot boot
+			// otherwise reports only an exit code, and the reason is in that buffer.
+			server.on("exit", (code) =>
+				reject(
+					new Error(
+						`server exited before listening (code ${code})\n${captured.stderr}`,
+					),
+				),
+			);
+		});
+
+		// A request the router answers without touching a data backend, so the
+		// invocation's own lines — the ones withTelemetry writes around the
+		// handler — are captured by the time this suite reads them.
+		await fetch(`http://127.0.0.1:${port}/no-such-route`);
+		await waitForStdout("Request received");
+	});
+
+	after(() => {
+		server.kill("SIGKILL");
+	});
+
+	it("is JSON on every line of both streams", () => {
+		const lines = [...parsedLines("stdout"), ...parsedLines("stderr")];
+		assert.ok(lines.length > 0, "expected the server to have written a line");
+		for (const line of lines) {
+			assert.equal(typeof line.level, "string");
+			assert.equal(typeof line.time, "string");
+			assert.equal(line.service, "backend");
+			assert.equal(typeof line.msg, "string");
+		}
+	});
+
+	it("reports what it is listening on as fields, not a banner", () => {
+		const listening = parsedLines("stdout").find(
+			(line) => line.msg === "Backend listening",
+		);
+		assert.ok(listening, "expected a startup line");
+		assert.equal(listening.port, port);
+		assert.equal(listening.url, `http://localhost:${port}`);
+		assert.equal(listening.level, "info");
+	});
+
+	it("correlates the whole invocation under one requestId", () => {
+		const lines = parsedLines("stdout").filter(
+			(line) => line.msg === "Lambda invocation started" || line.path,
+		);
+		assert.ok(lines.length >= 2, "expected the invocation to have logged");
+		const requestIds = new Set(lines.map((line) => line.requestId));
+		assert.equal(requestIds.size, 1);
+		assert.equal([...requestIds][0] === undefined, false);
+	});
+});

package/dev-server/server.ts

@@ -288,7 +288,7 @@ app.all(/(.*)/, async (req: Request, res: Response) => {
 	) {
 		const parsed = await safeJsonParse<unknown>(body).catch(() => undefined);
 		if (parsed === undefined) {
-			console.error("[dev-server] Failed to parse JSON body");
+			logger.error("Failed to parse JSON body");
 		} else if (
 			parsed &&
 			typeof parsed === "object" &&
@@ -313,19 +313,30 @@ app.all(/(.*)/, async (req: Request, res: Response) => {
 
 const port = env.SERVER_PORT;
 
+// This file is the backend image's entrypoint, so its startup output is the
+// first thing a log collector reads from the container. It goes through the
+// logger for the same reason every other line does: one JSON object per line is
+// the contract in deploy/vps/README.md, and a banner printed alongside it is a
+// line the pipeline cannot parse.
+//
+// This is also the everyday `npm run dev` server, so the addresses stay whole
+// and clickable — `url`, not a port a developer has to assemble one themselves.
 app.listen(Number(port), "0.0.0.0", () => {
-	console.log(`Remit Backend running on http://localhost:${port}`);
-	console.log(
-		`OpenAPI documentation available at http://localhost:${port}/api-docs`,
+	// biome-ignore lint/plugin/no-logger-info: the configuration a container came up on is an audit-grade signal
+	logger.info(
+		{
+			port: Number(port),
+			url: `http://localhost:${port}`,
+			dynamodbPort: env.DYNAMODB_PORT,
+			dynamodbTable: env.DYNAMODB_TABLE_NAME,
+			nodeEnv: env.NODE_ENV,
+			...(isSelfHostBackend
+				? {}
+				: { apiDocsUrl: `http://localhost:${port}/api-docs` }),
+		},
+		"Backend listening",
 	);
 
-	console.table({
-		SERVER_PORT: port,
-		DYNAMODB_PORT: env.DYNAMODB_PORT,
-		DYNAMODB_TABLE: env.DYNAMODB_TABLE_NAME,
-		NODE_ENV: env.NODE_ENV,
-	});
-
 	process.send?.("ready");
 });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.44",
+  "version": "0.0.46",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/scripts/backfill-list-id.ts

@@ -57,12 +57,13 @@ const run = async (): Promise<void> => {
 		{ checkpointStore: fileCheckpointStore, logger },
 	);
 
-	console.log("[backfill-list-id] done", result);
+	// biome-ignore lint/plugin/no-logger-info: a completed full-corpus backfill is an audit-grade signal
+	logger.info({ result }, "backfill done");
 };
 
 run()
 	.then(() => process.exit(0))
 	.catch((error: unknown) => {
-		console.error("[backfill-list-id] failed", error);
+		logger.error({ error }, "backfill failed");
 		process.exit(1);
 	});

package/src/derive/deriveMuted.test.ts

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { AddressFlags } from "@remit/api-openapi-types";
+import { deriveMuted } from "./deriveMuted.js";
+
+const SET_AT = 1_700_000_000_000;
+
+describe("deriveMuted", () => {
+	it("returns false when flags is undefined", () => {
+		assert.equal(deriveMuted(undefined), false);
+	});
+
+	it("returns false for an empty flags object", () => {
+		assert.equal(deriveMuted({}), false);
+	});
+
+	it("returns false when muted.value is false", () => {
+		const flags: AddressFlags = { muted: { value: false, setAt: SET_AT } };
+		assert.equal(deriveMuted(flags), false);
+	});
+
+	it("returns true when muted.value is true", () => {
+		const flags: AddressFlags = { muted: { value: true, setAt: SET_AT } };
+		assert.equal(deriveMuted(flags), true);
+	});
+
+	it("ignores other flags (orthogonal axis)", () => {
+		const flags: AddressFlags = {
+			vip: { value: true, setAt: SET_AT },
+			wellknown: { value: true, setAt: SET_AT },
+		};
+		assert.equal(deriveMuted(flags), false);
+	});
+});

package/src/derive/deriveMuted.ts

@@ -0,0 +1,10 @@
+import type { AddressFlags } from "@remit/api-openapi-types";
+
+/**
+ * Derive whether the From address is muted from an Address's flags map.
+ *
+ * Pure function, no I/O. Frontend never derives this — single source of
+ * truth for filtering the daily brief.
+ */
+export const deriveMuted = (flags: AddressFlags | undefined): boolean =>
+	flags?.muted?.value === true;

package/src/derive/enrichThreadRows.test.ts

@@ -7,11 +7,13 @@ import type {
 	MessageLabelItem,
 	ThreadMessageItem,
 } from "@remit/data-ports";
+import { deriveAddressId } from "@remit/data-ports/id";
 import { type EnrichClient, enrichThreadRows } from "./enrichThreadRows.js";
 
 const threadRow = (
 	threadMessageId: string,
 	messageId: string,
+	fromEmail?: string,
 ): ThreadMessageItem =>
 	({
 		threadMessageId,
@@ -19,6 +21,7 @@ const threadRow = (
 		messageId,
 		accountConfigId: "acc-1",
 		mailboxId: "mbx-1",
+		fromEmail,
 		sentDate: 1,
 		isRead: true,
 		hasAttachment: false,
@@ -31,9 +34,10 @@ const threadRow = (
 const buildClient = (
 	messageLabels: MessageLabelItem[],
 	labels: LabelItem[],
+	addresses: AddressItem[] = [],
 ): EnrichClient => ({
 	message: { get: async () => [] as MessageItem[] },
-	address: { getAddress: async () => [] as AddressItem[] },
+	address: { getAddress: async () => addresses },
 	messageLabel: {
 		listByMessageIds: async (messageIds: string[]) =>
 			messageLabels.filter((row) => messageIds.includes(row.messageId)),
@@ -161,3 +165,59 @@ describe("enrichThreadRows — labels", () => {
 		assert.equal(second?.labels, undefined);
 	});
 });
+
+describe("enrichThreadRows — muted", () => {
+	const SET_AT = 1_700_000_000_000;
+
+	test("sets muted true from the batch-fetched Address's flags, no extra query", async () => {
+		const fromEmail = "muted@example.com";
+		const addressId = deriveAddressId("acc-1", fromEmail);
+		const rows = [threadRow("tm-1", "msg-1", fromEmail)];
+		const addresses = [
+			{
+				addressId,
+				accountConfigId: "acc-1",
+				flags: { muted: { value: true, setAt: SET_AT } },
+			},
+		] as unknown as AddressItem[];
+
+		let addressCalls = 0;
+		const client: EnrichClient = {
+			message: { get: async () => [] as MessageItem[] },
+			address: {
+				getAddress: async () => {
+					addressCalls += 1;
+					return addresses;
+				},
+			},
+			messageLabel: { listByMessageIds: async () => [] },
+			label: { listByAccountConfig: async () => [] },
+		};
+
+		const [result] = await enrichThreadRows(rows, client, "acc-1");
+		assert.equal(result?.muted, true);
+		assert.equal(addressCalls, 1);
+	});
+
+	test("defaults muted to false when the Address has no muted flag", async () => {
+		const fromEmail = "not-muted@example.com";
+		const addressId = deriveAddressId("acc-1", fromEmail);
+		const rows = [threadRow("tm-1", "msg-1", fromEmail)];
+		const addresses = [
+			{ addressId, accountConfigId: "acc-1", flags: {} },
+		] as unknown as AddressItem[];
+
+		const [result] = await enrichThreadRows(
+			rows,
+			buildClient([], [], addresses),
+			"acc-1",
+		);
+		assert.equal(result?.muted, false);
+	});
+
+	test("defaults muted to false when no Address row resolves", async () => {
+		const rows = [threadRow("tm-1", "msg-1")];
+		const [result] = await enrichThreadRows(rows, buildClient([], []), "acc-1");
+		assert.equal(result?.muted, false);
+	});
+});

package/src/derive/enrichThreadRows.ts

@@ -9,6 +9,7 @@ import type {
 import { deriveAddressId } from "@remit/data-ports/id";
 import { SenderTrust, StarColor } from "@remit/domain-enums";
 import { deriveAutoMoved } from "./autoMoved.js";
+import { deriveMuted } from "./deriveMuted.js";
 import { deriveSenderTrust } from "./senderTrust.js";
 
 /**
@@ -54,6 +55,7 @@ const toResponse = (item: ThreadMessageItem): ThreadMessageResponse => ({
 	createdAt: item.createdAt,
 	updatedAt: item.updatedAt,
 	senderTrust: SenderTrust.Unknown,
+	muted: false,
 });
 
 /**
@@ -98,20 +100,22 @@ export const planBatchFetch = (rows: ThreadMessageItem[]): BatchPlan => {
 };
 
 /**
- * Enrich a page of ThreadMessage rows with `senderTrust` (derived from the From
- * Address's flags map), `authenticity` and `autoMoved` (both projected from the
- * Message row, see `deriveAutoMoved`).
+ * Enrich a page of ThreadMessage rows with `senderTrust` and `muted` (both
+ * derived from the From Address's flags map), `authenticity` and `autoMoved`
+ * (both projected from the Message row, see `deriveAutoMoved`).
  *
- * `category` is not enriched: it is denormalized onto the ThreadMessage row and
- * carried straight through by `toResponse`, so the value a client renders is the
- * value the category filter matched.
+ * `category` is not enriched: it is denormalized onto the ThreadMessage row
+ * (shared with `Message.category`'s write-once value, see body-sync.ts) and
+ * carried straight through by `toResponse`, so the value a client renders is
+ * the value the category filter matched.
  *
  * Two BatchGetItem calls per page, regardless of page size — see
  * `planBatchFetch` for the dedup contract.
  *
- * Missing rows fall back gracefully: `senderTrust` defaults to `"unknown"`, and
- * `authenticity` / `autoMoved` are omitted whenever the Message row is absent or
- * the move isn't a real, in-effect auto-move.
+ * Missing rows fall back gracefully: `senderTrust` defaults to `"unknown"`,
+ * `muted` defaults to `false`, and `authenticity` / `autoMoved` are omitted
+ * whenever the Message row is absent or the move isn't a real, in-effect
+ * auto-move.
  *
  * Not annotated `Promise<ThreadMessageResponse[]>`: `labels` is a new field on
  * it in this same PR, and that package publishes separately from this repo —
@@ -169,6 +173,9 @@ export const enrichThreadRows = async (
 	const trustByAddressId = new Map(
 		addresses.map((a) => [a.addressId, deriveSenderTrust(a.flags)]),
 	);
+	const mutedByAddressId = new Map(
+		addresses.map((a) => [a.addressId, deriveMuted(a.flags)]),
+	);
 
 	return rows.map((row) => {
 		const base = toResponse(row);
@@ -178,6 +185,9 @@ export const enrichThreadRows = async (
 		const senderTrust = addressId
 			? (trustByAddressId.get(addressId) ?? SenderTrust.Unknown)
 			: SenderTrust.Unknown;
+		const muted = addressId
+			? (mutedByAddressId.get(addressId) ?? false)
+			: false;
 		const labels = labelsByMessageId.get(row.messageId);
 		return {
 			...base,
@@ -185,6 +195,7 @@ export const enrichThreadRows = async (
 			...(autoMoved !== undefined ? { autoMoved } : {}),
 			...(labels !== undefined ? { labels } : {}),
 			senderTrust,
+			muted,
 		};
 	});
 };

package/src/derive/filterThreadCriteria.test.ts

@@ -29,6 +29,7 @@ const row = (
 	createdAt: 0,
 	updatedAt: 0,
 	senderTrust: SenderTrust.Unknown,
+	muted: false,
 	...overrides,
 });
 

package/src/index.ts

@@ -144,11 +144,11 @@ const readOriginHeader = (
 // A scope, not `logger.setBindings`: this process serves requests concurrently,
 // and bindings on the shared logger belong to whichever request wrote them last,
 // so a line gets attributed to the wrong request. The scope follows the request
-// through its own async continuations and nothing else.
+// through its own async continuations and nothing else. It nests inside the one
+// `withTelemetry` opens, which is where `requestId` comes from.
 const rawHandler = async (event: APIGatewayProxyEvent, context: Context) =>
 	withLogContext(
 		{
-			requestId: context.awsRequestId,
 			path: event.path,
 			method: event.httpMethod,
 		},

package/tsconfig.json

@@ -3,5 +3,16 @@
 	"compilerOptions": {
 		"outDir": "dist"
 	},
-	"include": ["src/**/*.ts", "dev-server/**/*.ts", "scripts/**/*.ts"]
+	"include": [
+		"src/**/*.ts",
+		"dev-server/**/*.ts",
+		"scripts/**/*.ts",
+		// The backend image's third entrypoint. `migrate.mjs` and
+		// `backfill-list-id.mjs` ship inside this image alongside `server.mjs`
+		// (npm-scripts/docker-bundle.mjs), so they belong to the same typecheck —
+		// living under deploy/ is where it sits in the tree, not a different
+		// deliverable. Unchecked, it shipped a container entrypoint with two hard
+		// type errors.
+		"../../deploy/vps/migrate/**/*.ts"
+	]
 }