@remit/queue-sidecar 0.0.7 → 0.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/queue-sidecar",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -20,8 +20,10 @@
     "test": "npm run test:typecheck && npm run test:run"
   },
   "dependencies": {
+    "@remit/logger-lambda": "*",
     "better-sqlite3": "^12.11.1",
-    "@types/better-sqlite3": "^7.6.13"
+    "@types/better-sqlite3": "^7.6.13",
+    "prom-client": "^15.1.3"
   },
   "devDependencies": {
     "@aws-sdk/client-sqs": "*",

package/src/metrics.test.ts

@@ -0,0 +1,197 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import type { Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { after, before, beforeEach, describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import { resetMetrics } from "@remit/logger-lambda/metrics";
+import { collectQueueDepths, queueRoles } from "./metrics.js";
+import { bootstrapQueues, parseQueuesConfig } from "./queues-config.js";
+import { createSidecarServer } from "./server.js";
+import { QueueStore } from "./store.js";
+
+const tmpRoot = join(
+	fileURLToPath(new URL(".", import.meta.url)),
+	"..",
+	".tmp",
+	"metrics",
+);
+
+const config = parseQueuesConfig({
+	queues: [
+		{
+			name: "work",
+			visibilityTimeoutSeconds: 30,
+			deadLetter: { name: "work-dlq", maxReceiveCount: 3 },
+		},
+		{ name: "work-dlq", visibilityTimeoutSeconds: 30 },
+	],
+});
+
+const sample = (text: string, labels: string[]): number => {
+	const line = text
+		.split("\n")
+		.find(
+			(candidate) =>
+				candidate.startsWith("remit_queue_messages{") &&
+				labels.every((label) => candidate.includes(label)),
+		);
+	assert.ok(line, `expected a sample for ${labels.join("")} in:\n${text}`);
+	return Number(line.slice(line.lastIndexOf(" ") + 1));
+};
+
+describe("queue depth roles", () => {
+	it("names a redrive target a dead-letter queue and everything else work", () => {
+		const roles = queueRoles([
+			{
+				name: "work",
+				fifo: false,
+				visibilityTimeoutSeconds: 30,
+				deadLetterTargetName: "work-dlq",
+				maxReceiveCount: 3,
+				contentBasedDeduplication: false,
+			},
+			{
+				name: "work-dlq",
+				fifo: false,
+				visibilityTimeoutSeconds: 30,
+				deadLetterTargetName: null,
+				maxReceiveCount: null,
+				contentBasedDeduplication: false,
+			},
+		]);
+		assert.equal(roles.get("work"), "work");
+		assert.equal(roles.get("work-dlq"), "dead_letter");
+	});
+});
+
+describe("the sidecar /metrics endpoint", () => {
+	let dir: string;
+	let store: QueueStore;
+	let server: Server;
+	let port: number;
+
+	const get = async (
+		path: string,
+	): Promise<{ status: number; contentType: string; body: string }> => {
+		const response = await fetch(`http://127.0.0.1:${port}${path}`);
+		return {
+			status: response.status,
+			contentType: response.headers.get("content-type") ?? "",
+			body: await response.text(),
+		};
+	};
+
+	before(async () => {
+		mkdirSync(tmpRoot, { recursive: true });
+		dir = mkdtempSync(join(tmpRoot, "run-"));
+		store = new QueueStore(join(dir, "queue.db"));
+		bootstrapQueues(store, config);
+		server = createSidecarServer({ store });
+		port = await new Promise<number>((resolve) => {
+			server.listen(0, "127.0.0.1", () =>
+				resolve((server.address() as AddressInfo).port),
+			);
+		});
+	});
+
+	after(async () => {
+		await new Promise<void>((resolve) => server.close(() => resolve()));
+		store.close();
+		rmSync(dir, { recursive: true, force: true });
+	});
+
+	beforeEach(() => resetMetrics());
+
+	it("reports every queue at zero on an idle stack", async () => {
+		const response = await get("/metrics");
+		assert.equal(response.status, 200);
+		assert.match(response.contentType, /^text\/plain/);
+		assert.match(response.body, /^# TYPE remit_queue_messages gauge$/m);
+		assert.equal(sample(response.body, ['queue="work"', 'role="work"']), 0);
+		assert.equal(
+			sample(response.body, ['queue="work-dlq"', 'role="dead_letter"']),
+			0,
+		);
+	});
+
+	it("reports work depth and dead-letter depth separately", async () => {
+		store.sendMessage({ queueName: "work", body: "a" });
+		store.sendMessage({ queueName: "work", body: "b" });
+		store.sendMessage({ queueName: "work-dlq", body: "poison" });
+		const { body } = await get("/metrics");
+		assert.equal(sample(body, ['queue="work"', 'role="work"']), 2);
+		assert.equal(sample(body, ['queue="work-dlq"']), 1);
+	});
+
+	it("re-reads depth on every scrape", async () => {
+		store.purgeQueue("work");
+		store.purgeQueue("work-dlq");
+		const { body } = await get("/metrics");
+		assert.equal(sample(body, ['queue="work"', 'role="work"']), 0);
+	});
+
+	it("keeps the SQS protocol and the health route unchanged", async () => {
+		assert.equal((await get("/health")).status, 200);
+		const post = await fetch(`http://127.0.0.1:${port}/metrics`, {
+			method: "POST",
+			body: "Action=ListQueues",
+		});
+		assert.equal(post.status, 400);
+	});
+
+	it("counts only visible messages as depth", () => {
+		store.sendMessage({ queueName: "work", body: "c" });
+		store.receiveMessages({ queueName: "work", maxMessages: 1 });
+		collectQueueDepths(store);
+		assert.equal(
+			store.getQueueAttributes("work").approximateNumberOfMessages,
+			0,
+		);
+	});
+});
+
+describe("a store that cannot answer for any queue", () => {
+	// A deployment always holds the queues queues.json declares. An empty table
+	// means the sidecar is reading the wrong file, and rendering that as a series
+	// with no samples is the same blindness this endpoint exists to remove: a
+	// dead-letter alert written on `> 0` would never fire, behind a 200.
+	it("throws rather than reporting no queues", () => {
+		assert.throws(
+			() =>
+				collectQueueDepths({
+					listQueues: () => [],
+					getQueueAttributes: () => {
+						throw new Error("never reached");
+					},
+				}),
+			/holds no queues/,
+		);
+	});
+
+	it("fails the scrape with a non-200 and no depth series", async () => {
+		mkdirSync(tmpRoot, { recursive: true });
+		const dir = mkdtempSync(join(tmpRoot, "empty-"));
+		const store = new QueueStore(join(dir, "queue.db"));
+		const server = createSidecarServer({ store });
+		const port = await new Promise<number>((resolve) => {
+			server.listen(0, "127.0.0.1", () =>
+				resolve((server.address() as AddressInfo).port),
+			);
+		});
+
+		const response = await fetch(`http://127.0.0.1:${port}/metrics`);
+		const body = await response.text();
+		assert.equal(response.status, 500);
+		assert.match(body, /holds no queues/);
+		assert.ok(
+			!body.includes("remit_queue_messages"),
+			"a failed scrape must render no depth series at all",
+		);
+
+		await new Promise<void>((resolve) => server.close(() => resolve()));
+		store.close();
+		rmSync(dir, { recursive: true, force: true });
+	});
+});

package/src/metrics.ts

@@ -0,0 +1,69 @@
+import { registry } from "@remit/logger-lambda/metrics";
+import { Gauge } from "prom-client";
+import type { QueueRecord, QueueStore } from "./store.js";
+
+/**
+ * Whether a queue is where work is sent or where it is quarantined. A
+ * dead-letter queue is not a different kind of queue — it is a queue some other
+ * queue redrives into — so the depth series is one metric and the role is a
+ * label, derived from the redrive policies the store already holds.
+ */
+export type QueueRole = "work" | "dead_letter";
+
+const queueMessages = new Gauge({
+	name: "remit_queue_messages",
+	help: "Messages currently on a queue, by queue and role.",
+	labelNames: ["queue", "role"],
+	registers: [registry],
+});
+
+export type QueueDepthSource = Pick<
+	QueueStore,
+	"listQueues" | "getQueueAttributes"
+>;
+
+export const queueRoles = (
+	queues: readonly QueueRecord[],
+): Map<string, QueueRole> => {
+	const deadLetterTargets = new Set(
+		queues.flatMap((queue) =>
+			queue.deadLetterTargetName === null ? [] : [queue.deadLetterTargetName],
+		),
+	);
+	return new Map(
+		queues.map((queue) => [
+			queue.name,
+			deadLetterTargets.has(queue.name) ? "dead_letter" : "work",
+		]),
+	);
+};
+
+/**
+ * Re-read every queue's depth. The whole series is replaced rather than added
+ * to, so a purged or deleted queue stops reporting its last known depth instead
+ * of pinning it forever.
+ *
+ * A store with no queues in it throws. A deployment always has the queues
+ * `queues.json` declares, so an empty table means the sidecar is pointed at the
+ * wrong `QUEUE_SIDECAR_DB`, or at a file `bootstrapQueues` never ran against.
+ * Rendering that as a series with no samples is the same failure this endpoint
+ * exists to remove: `remit_queue_messages{role="dead_letter"} > 0` never fires
+ * and the operator is as blind as before, with a 200 saying otherwise.
+ */
+export const collectQueueDepths = (store: QueueDepthSource): void => {
+	const queues = store.listQueues();
+	if (queues.length === 0) {
+		throw new Error(
+			"queue store holds no queues — the sidecar is reading the wrong database, or none was bootstrapped",
+		);
+	}
+	const roles = queueRoles(queues);
+	queueMessages.reset();
+	for (const queue of queues) {
+		const attributes = store.getQueueAttributes(queue.name);
+		queueMessages.set(
+			{ queue: queue.name, role: roles.get(queue.name) ?? "work" },
+			attributes.approximateNumberOfMessages,
+		);
+	}
+};

package/src/server.ts

@@ -1,6 +1,11 @@
 import { randomUUID } from "node:crypto";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import { createServer, type Server } from "node:http";
+import {
+	metricsContentType,
+	renderMetrics,
+} from "@remit/logger-lambda/metrics";
+import { collectQueueDepths } from "./metrics.js";
 import {
 	type BatchErrorEntry,
 	type BatchSuccessEntry,
@@ -384,6 +389,26 @@ export const createSidecarServer = (options: CreateSidecarOptions): Server => {
 			res.end("ok");
 			return;
 		}
+		// Queue and dead-letter depth (standalone-observability D2/D3), on the port
+		// the SQS protocol already answers on. The depths are read here rather than
+		// counted as messages move: the store is the only thing that knows them,
+		// and each is two counts over a `(queue_name, id)` index range with a
+		// `visible_at` filter per row — a table that is empty in steady state.
+		if (req.method === "GET" && req.url === "/metrics") {
+			Promise.resolve()
+				.then(() => collectQueueDepths(store))
+				.then(renderMetrics)
+				.then((body) => {
+					res.writeHead(200, { "content-type": metricsContentType });
+					res.end(body);
+				})
+				.catch((error: unknown) => {
+					log.error({ error: String(error) }, "sidecar: metrics failed");
+					res.writeHead(500, { "content-type": "text/plain" });
+					res.end(`metrics collection failed: ${String(error)}\n`);
+				});
+			return;
+		}
 		if (req.method !== "POST") {
 			respondError(res, new InvalidRequestError("only POST is supported"));
 			return;