@remit/logger-lambda 0.0.9 → 0.0.11

package/README.md

@@ -1,35 +1,26 @@
 # @remit/logger-lambda
 
-Shared logging package for Remit Lambda workers. Wraps [Pino](https://getpino.io/) with Lambda-specific configuration and environment-aware formatting.
+The logging seam every Remit service writes through. One JSON object per line on
+stdout, via [pino](https://getpino.io/). The exported `Logger` interface is what
+consuming code imports; the writer behind it is an implementation detail.
 
-## Features
-
-- **Structured Logging**: JSON output for CloudWatch ingestion in production.
-- **Pretty Print**: Human-readable output for local development.
-- **Request Context**: Automatically correlates logs with Lambda request IDs.
-- **Zero Configuration**: Sensible defaults based on `NODE_ENV`.
-
-## Installation
-
-```bash
-npm install @remit/logger-lambda
-```
+The field names on each line are a contract an operator's log pipeline parses.
+They are documented, with the reserved names and the personal-data caveat, in
+[`deploy/vps/README.md`](../../deploy/vps/README.md) under "Logs".
 
 ## Usage
 
 ```typescript
 import { createLogger } from "@remit/logger-lambda";
-import type { SQSEvent, Context } from "aws-lambda";
+import type { SQSEvent } from "aws-lambda";
 
-export const handler = async (event: SQSEvent, context: Context) => {
-  // Initialize logger with Lambda context
-  const log = createLogger(context);
+const log = createLogger().child({ queue: "remit-imap-sync" });
 
-  log.info({ event }, "Processing SQS event");
+export const handler = async (event: SQSEvent) => {
+  log.debug({ records: event.Records.length }, "Batch received");
 
   try {
     // ... business logic
-    log.info("Success");
   } catch (error) {
     log.error({ error }, "Processing failed");
     throw error;
@@ -37,9 +28,69 @@ export const handler = async (event: SQSEvent, context: Context) => {
 };
 ```
 
-## Environment Variables
+## Metrics
+
+`@remit/logger-lambda/metrics` owns the process-wide Prometheus registry every
+service renders at `/metrics`, and the recorders that write to it. See
+[docs/design/standalone-observability.md](../../docs/design/standalone-observability.md)
+for the signal set and why it is pulled rather than pushed.
+
+```typescript
+import {
+  onScrape,
+  recordImapFailure,
+  startMetricsServer,
+} from "@remit/logger-lambda/metrics";
+
+// A service with no listener of its own gets one for /metrics alone. A port it
+// cannot bind is reported and then dropped: the process keeps doing its work
+// and serves no metrics.
+startMetricsServer();
+
+// Counters and histograms are written where the work happens.
+recordImapFailure("SYNC_MESSAGES", "auth");
+
+// A gauge whose value is a read is collected when a scrape arrives. A collector
+// that throws fails the scrape rather than rendering a stale or zero value.
+onScrape(async () => setBacklog(await countUndrainedRows()));
+```
+
+A signal only one service can answer for is declared by that service against the
+exported `registry` — see `search-index-worker/src/metrics.ts` and
+`queue-sidecar/src/metrics.ts`. Declaring it here would render it in every
+process that imports this module, including the four that cannot know its value.
+
+`withTelemetry` records handler duration and outcome into the same registry.
+
+Both argument orders work: `(bindings, message)` and `(message, bindings)`.
+Bindings land at the top level of the line, never nested. A value under `error`
+that is an `Error` is expanded to `type`, `message` and `stack`; anything else is
+written as it is.
+
+`child(bindings)` returns a logger that adds those bindings to every line, and
+`setBindings(bindings)` adds them to an existing one. Neither is per-request:
+they belong to the logger instance, so on a process serving requests
+concurrently use `withLogContext` instead.
+
+```typescript
+import { logger, withLogContext } from "@remit/logger-lambda";
+
+const handler = (request: Request) =>
+  withLogContext({ requestId: request.id }, async () => {
+    logger.debug("Request received"); // carries requestId
+    return respond(request);
+  });
+```
+
+The scope follows the work through its asynchronous continuations, so two
+overlapping requests never see each other's fields. Scopes nest, and an inner
+one adds to the bindings of the one it runs inside.
+
+## Environment variables
 
-| Variable    | Default      | Description                                             |
-| ----------- | ------------ | ------------------------------------------------------- |
-| `LOG_LEVEL` | `info`       | Pino log level (trace, debug, info, warn, error, fatal) |
-| `NODE_ENV`  | `production` | If `development`, enables pretty printing               |
+| Variable             | Default | Description                                                                                                                          |
+| -------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------ |
+| `LOG_LEVEL`          | `info`  | `trace`, `debug`, `info`, `warn`, `error`, `fatal` or `silent`. An unrecognised value logs one warning and falls back to the default. |
+| `REMIT_SERVICE_NAME` | `remit` | The `service` field on every line. Stamped into each service bundle at build time by `npm-scripts/docker-bundle.mjs`.                 |
+| `METRICS_PORT`       | `9464`  | Port `startMetricsServer` binds. |
+| `METRICS_HOST`       | `0.0.0.0` | Interface `startMetricsServer` binds. Set `127.0.0.1` when the service runs as a host process rather than a container. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/logger-lambda",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -8,6 +8,10 @@
     ".": {
       "types": "./src/index.ts",
       "default": "./src/index.ts"
+    },
+    "./metrics": {
+      "types": "./src/metrics.ts",
+      "default": "./src/metrics.ts"
     }
   },
   "scripts": {
@@ -17,9 +21,9 @@
     "build": "npm run test:typecheck"
   },
   "dependencies": {
-    "@aws-lambda-powertools/logger": "^2",
-    "@aws-lambda-powertools/metrics": "^2",
-    "@types/aws-lambda": "*"
+    "@types/aws-lambda": "*",
+    "pino": "^10",
+    "prom-client": "^15.1.3"
   },
   "license": "MIT",
   "publishConfig": {

package/src/index.ts

@@ -1,2 +1,2 @@
-export { MetricUnit } from "@aws-lambda-powertools/metrics";
 export * from "./logger.js";
+export * from "./metrics.js";

package/src/log-level.test.ts

@@ -0,0 +1,77 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+
+process.env.LOG_LEVEL = "verbose";
+delete process.env.REMIT_SERVICE_NAME;
+
+type Line = Record<string, unknown>;
+
+const originalWrite = process.stdout.write.bind(process.stdout);
+const written: string[] = [];
+let capturing = true;
+
+process.stdout.write = ((
+	chunk: string | Uint8Array,
+	...rest: unknown[]
+): boolean => {
+	if (capturing && typeof chunk === "string") {
+		written.push(chunk);
+		return true;
+	}
+	return (originalWrite as (...args: unknown[]) => boolean)(chunk, ...rest);
+}) as typeof process.stdout.write;
+
+// Capturing starts before the import: an unusable LOG_LEVEL is reported while
+// the module is evaluated, not on the first call.
+const { createLogger } = await import("./logger.js");
+capturing = false;
+
+const startup = written
+	.join("")
+	.split("\n")
+	.filter((line) => line.length > 0)
+	.map((line) => JSON.parse(line) as Line);
+
+const capture = (emit: () => void): Line[] => {
+	written.length = 0;
+	capturing = true;
+	try {
+		emit();
+	} finally {
+		capturing = false;
+	}
+	return written
+		.join("")
+		.split("\n")
+		.filter((line) => line.length > 0)
+		.map((line) => JSON.parse(line) as Line);
+};
+
+describe("log level", () => {
+	it("reports an unusable LOG_LEVEL instead of failing or ignoring it", () => {
+		assert.equal(startup.length, 1);
+		const [line] = startup;
+		assert.equal(line.level, "warn");
+		assert.match(String(line.msg), /LOG_LEVEL/);
+		assert.equal(line.configured, "verbose");
+		assert.match(String(line.expected), /trace/);
+	});
+
+	it("falls back to info, so debug and trace are dropped", () => {
+		const log = createLogger();
+		const lines = capture(() => {
+			log.trace("dropped");
+			log.debug("dropped");
+			log.warn("kept");
+		});
+		assert.deepEqual(
+			lines.map((line) => line.msg),
+			["kept"],
+		);
+	});
+
+	it("defaults the service name when none is stamped into the build", () => {
+		const [line] = capture(() => createLogger().warn("unnamed"));
+		assert.equal(line.service, "remit");
+	});
+});

package/src/log-output.test.ts

@@ -0,0 +1,355 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+
+// The field contract these tests pin down is documented for operators in
+// deploy/vps/README.md ("Logs"). A change here is a change to what every
+// log-shipping rule against this deployment parses.
+process.env.LOG_LEVEL = "trace";
+process.env.REMIT_SERVICE_NAME = "test-service";
+
+type Line = Record<string, unknown>;
+
+const originalWrite = process.stdout.write.bind(process.stdout);
+const written: string[] = [];
+let capturing = false;
+
+// pino uses `process.stdout` directly when its `write` has been replaced, and
+// the real fd otherwise — so the hook has to be in place before the import
+// below, and it passes writes through whenever a test is not capturing so the
+// test runner's own output still reaches the terminal.
+process.stdout.write = ((
+	chunk: string | Uint8Array,
+	...rest: unknown[]
+): boolean => {
+	if (capturing && typeof chunk === "string") {
+		written.push(chunk);
+		return true;
+	}
+	return (originalWrite as (...args: unknown[]) => boolean)(chunk, ...rest);
+}) as typeof process.stdout.write;
+
+const { createLogger, logger, withLogContext } = await import("./logger.js");
+
+const parse = (): Line[] =>
+	written
+		.join("")
+		.split("\n")
+		.filter((line) => line.length > 0)
+		.map((line) => JSON.parse(line) as Line);
+
+const capture = (emit: () => void): Line[] => {
+	written.length = 0;
+	capturing = true;
+	try {
+		emit();
+	} finally {
+		capturing = false;
+	}
+	return parse();
+};
+
+const captureAsync = async (emit: () => Promise<void>): Promise<Line[]> => {
+	written.length = 0;
+	capturing = true;
+	try {
+		await emit();
+	} finally {
+		capturing = false;
+	}
+	return parse();
+};
+
+const one = (emit: () => void): Line => {
+	const lines = capture(emit);
+	assert.equal(lines.length, 1, "expected exactly one log line");
+	return lines[0];
+};
+
+describe("log output", () => {
+	it("writes one JSON object per line on stdout", () => {
+		const log = createLogger();
+		const lines = capture(() => {
+			log.warn("first");
+			log.warn("second");
+		});
+		assert.deepEqual(
+			lines.map((line) => line.msg),
+			["first", "second"],
+		);
+	});
+
+	it("carries level, time, service and msg on every line", () => {
+		const line = one(() => createLogger().warn("hello"));
+		assert.equal(line.level, "warn");
+		assert.equal(line.service, "test-service");
+		assert.equal(line.msg, "hello");
+		assert.equal(typeof line.time, "string");
+		assert.equal(
+			new Date(line.time as string).toISOString(),
+			line.time,
+			"time is RFC 3339 in UTC",
+		);
+	});
+
+	it("names each level in lowercase, with fatal as its own name", () => {
+		const log = createLogger();
+		const levels = capture(() => {
+			log.trace("t");
+			log.debug("d");
+			log.info("i");
+			log.warn("w");
+			log.error("e");
+			log.fatal("f");
+		}).map((line) => line.level);
+		assert.deepEqual(levels, [
+			"trace",
+			"debug",
+			"info",
+			"warn",
+			"error",
+			"fatal",
+		]);
+	});
+
+	it("puts bindings at the top level, message first", () => {
+		const line = one(() =>
+			createLogger().error(
+				{ error: "boom", messageId: "m1" },
+				"Failed to parse message",
+			),
+		);
+		assert.equal(line.msg, "Failed to parse message");
+		assert.equal(line.error, "boom");
+		assert.equal(line.messageId, "m1");
+	});
+
+	it("emits an empty msg when only bindings are given", () => {
+		const line = one(() => createLogger().warn({ count: 3 }));
+		assert.equal(line.msg, "");
+		assert.equal(line.count, 3);
+	});
+
+	it("accepts bindings after a message", () => {
+		const line = one(() =>
+			createLogger().warn("watch out", { reason: "slow" }),
+		);
+		assert.equal(line.msg, "watch out");
+		assert.equal(line.reason, "slow");
+	});
+
+	it("ignores a non-object second argument after a message", () => {
+		const line = one(() =>
+			(createLogger().warn as (msg: string, obj?: unknown) => void)(
+				"plain",
+				"not-bindings",
+			),
+		);
+		assert.equal(line.msg, "plain");
+	});
+
+	it("child bindings appear on every line the child writes", () => {
+		const child = createLogger().child({ queue: "imap" });
+		const line = one(() => child.warn({ done: true }, "child log"));
+		assert.equal(line.queue, "imap");
+		assert.equal(line.done, true);
+		assert.equal(line.msg, "child log");
+	});
+
+	it("a call-site field shadowing a child binding is written once", () => {
+		const child = createLogger().child({ queue: "from-child" });
+		const [line] = capture(() => child.warn({ queue: "from-callsite" }, "x"));
+		assert.equal(line.queue, "from-callsite");
+		const repeats = written.join("").match(/"queue"/g) ?? [];
+		assert.equal(repeats.length, 1, "one key per line, not two");
+	});
+
+	it("a child inherits the bindings its parent had when it was created", () => {
+		const parent = createLogger();
+		parent.setBindings({ accountId: "a1" });
+		const child = parent.child({ queue: "smtp" });
+		const line = one(() => child.warn("nested"));
+		assert.equal(line.accountId, "a1");
+		assert.equal(line.queue, "smtp");
+	});
+
+	it("setBindings applies to later lines and replaces a key without repeating it", () => {
+		const log = createLogger();
+		log.setBindings({ requestId: "r1", path: "/one" });
+		const first = one(() => log.warn("first request"));
+		assert.equal(first.requestId, "r1");
+		assert.equal(first.path, "/one");
+
+		log.setBindings({ requestId: "r2", path: "/two" });
+		const [raw] = capture(() => log.warn("second request"));
+		assert.equal(raw.requestId, "r2");
+		assert.equal(raw.path, "/two");
+
+		const repeats = written.join("").match(/"requestId"/g) ?? [];
+		assert.equal(repeats.length, 1, "a rebound key is written once, not twice");
+	});
+
+	it("bindings set on one logger do not leak into another", () => {
+		const first = createLogger();
+		const second = createLogger();
+		first.setBindings({ owner: "first" });
+		const line = one(() => second.warn("independent"));
+		assert.equal(line.owner, undefined);
+	});
+
+	it("the shared logger writes the same shape", () => {
+		const line = one(() => logger.warn("shared"));
+		assert.equal(line.service, "test-service");
+		assert.equal(line.msg, "shared");
+	});
+
+	it("carries no pid or hostname", () => {
+		const line = one(() => createLogger().warn("lean"));
+		assert.equal(line.pid, undefined);
+		assert.equal(line.hostname, undefined);
+	});
+
+	it("drops a call-site field using a reserved name", () => {
+		const [line] = capture(() =>
+			createLogger().warn(
+				{ level: "trace", time: "1999", service: "other", msg: "hijack" },
+				"collide",
+			),
+		);
+		assert.equal(line.level, "warn");
+		assert.equal(line.service, "test-service");
+		assert.equal(line.msg, "collide");
+		for (const key of ['"level"', '"time"', '"service"', '"msg"']) {
+			const repeats = written.join("").match(new RegExp(key, "g")) ?? [];
+			assert.equal(repeats.length, 1, `${key} is written once`);
+		}
+	});
+});
+
+// The shapes the worker and backend failure paths actually produce. A contract
+// test that asserts a key no call site writes is how a serialisation regression
+// stays green, so each case below names the call site it mirrors.
+describe("error serialisation", () => {
+	it("expands an Error logged under error — imap-worker/src/index.ts", () => {
+		const line = one(() =>
+			createLogger().error(
+				{ error: new Error("boom"), messageId: "m1" },
+				"Event processing failed",
+			),
+		);
+		const error = line.error as Record<string, unknown>;
+		assert.equal(error.type, "Error");
+		assert.equal(error.message, "boom");
+		assert.match(String(error.stack), /^Error: boom\n\s+at /);
+		assert.equal(line.messageId, "m1");
+	});
+
+	it("keeps a subclass name and its own properties", () => {
+		class UpstreamError extends Error {
+			readonly statusCode = 502;
+		}
+		const line = one(() =>
+			createLogger().error({ error: new UpstreamError("gone") }, "failed"),
+		);
+		const error = line.error as Record<string, unknown>;
+		assert.equal(error.type, "UpstreamError");
+		assert.equal(error.statusCode, 502);
+	});
+
+	it("leaves a string alone — backend/src/error.ts", () => {
+		const failure = new Error("ElectroError: bad key");
+		const line = one(() =>
+			createLogger().error(
+				{ error: failure.message, name: failure.name, stack: failure.stack },
+				"Unhandled Error",
+			),
+		);
+		assert.equal(line.error, "ElectroError: bad key");
+		assert.equal(line.name, "Error");
+		assert.match(String(line.stack), /^Error: ElectroError: bad key\n\s+at /);
+	});
+
+	it("leaves a plain object alone — backend/src/response.ts", () => {
+		const line = one(() =>
+			createLogger().error(
+				{ "problematicValue_/items": { path: "/items", error: "too long" } },
+				"Response validation failed",
+			),
+		);
+		assert.deepEqual(line["problematicValue_/items"], {
+			path: "/items",
+			error: "too long",
+		});
+	});
+
+	it("expands an Error under err too", () => {
+		const line = one(() =>
+			createLogger().error({ err: new Error("kaboom") }, "handler failed"),
+		);
+		const err = line.err as Record<string, unknown>;
+		assert.equal(err.type, "Error");
+		assert.equal(err.message, "kaboom");
+	});
+});
+
+describe("withLogContext", () => {
+	it("adds its bindings to every line written inside it", () => {
+		const line = one(() =>
+			withLogContext({ requestId: "r1", path: "/one" }, () => {
+				logger.warn("inside");
+			}),
+		);
+		assert.equal(line.requestId, "r1");
+		assert.equal(line.path, "/one");
+	});
+
+	it("does not leak past the scope", () => {
+		withLogContext({ requestId: "r1" }, () => {});
+		const line = one(() => logger.warn("outside"));
+		assert.equal(line.requestId, undefined);
+	});
+
+	it("keeps overlapping scopes apart across awaits", async () => {
+		const request = (id: string, delayMs: number) =>
+			withLogContext({ requestId: id }, async () => {
+				await new Promise((resolve) => setTimeout(resolve, delayMs));
+				logger.warn(`handled ${id}`);
+			});
+
+		// The slow request opens its scope first and logs last, which is exactly
+		// the interleaving a shared mutable binding gets wrong.
+		const lines = await captureAsync(async () => {
+			await Promise.all([request("slow", 10), request("fast", 1)]);
+		});
+
+		assert.deepEqual(
+			lines.map((line) => [line.msg, line.requestId]),
+			[
+				["handled fast", "fast"],
+				["handled slow", "slow"],
+			],
+		);
+	});
+
+	it("nests, adding to the scope it runs inside", () => {
+		const line = one(() =>
+			withLogContext({ requestId: "r1" }, () =>
+				withLogContext({ accountId: "a1" }, () => {
+					logger.warn("nested scope");
+				}),
+			),
+		);
+		assert.equal(line.requestId, "r1");
+		assert.equal(line.accountId, "a1");
+	});
+
+	it("a call-site field wins over a scope binding, and is written once", () => {
+		const [line] = capture(() =>
+			withLogContext({ requestId: "from-scope" }, () => {
+				logger.warn({ requestId: "from-callsite" }, "x");
+			}),
+		);
+		assert.equal(line.requestId, "from-callsite");
+		const repeats = written.join("").match(/"requestId"/g) ?? [];
+		assert.equal(repeats.length, 1);
+	});
+});