@remit/queue-sidecar 0.0.9 → 0.0.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/queue-sidecar",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/metrics.test.ts

@@ -141,6 +141,23 @@ describe("the sidecar /metrics endpoint", () => {
 		assert.equal(post.status, 400);
 	});
 
+	it("exports how long each queue has gone without handing out a message", async () => {
+		store.purgeQueue("work");
+		store.sendMessage({ queueName: "work", body: "waiting" });
+		const idle = await get("/metrics");
+		assert.match(
+			idle.body,
+			/^remit_queue_last_receive_age_seconds\{queue="work",role="work"\} \d+$/m,
+		);
+
+		const quiet = store.lastReceiveAgeSeconds("work", Date.now() + 901_000);
+		assert.ok(quiet >= 900);
+
+		store.receiveMessages({ queueName: "work", maxMessages: 1 });
+		assert.equal(store.lastReceiveAgeSeconds("work"), 0);
+		store.purgeQueue("work");
+	});
+
 	it("counts only visible messages as depth", () => {
 		store.sendMessage({ queueName: "work", body: "c" });
 		store.receiveMessages({ queueName: "work", maxMessages: 1 });
@@ -165,6 +182,7 @@ describe("a store that cannot answer for any queue", () => {
 					getQueueAttributes: () => {
 						throw new Error("never reached");
 					},
+					lastReceiveAgeSeconds: () => 0,
 				}),
 			/holds no queues/,
 		);

package/src/metrics.ts

@@ -17,9 +17,16 @@ const queueMessages = new Gauge({
 	registers: [registry],
 });
 
+const queueLastReceiveAge = new Gauge({
+	name: "remit_queue_last_receive_age_seconds",
+	help: "Seconds since a queue last handed a message to a consumer, or since the sidecar started if it has not.",
+	labelNames: ["queue", "role"],
+	registers: [registry],
+});
+
 export type QueueDepthSource = Pick<
 	QueueStore,
-	"listQueues" | "getQueueAttributes"
+	"listQueues" | "getQueueAttributes" | "lastReceiveAgeSeconds"
 >;
 
 export const queueRoles = (
@@ -59,11 +66,13 @@ export const collectQueueDepths = (store: QueueDepthSource): void => {
 	}
 	const roles = queueRoles(queues);
 	queueMessages.reset();
+	queueLastReceiveAge.reset();
 	for (const queue of queues) {
 		const attributes = store.getQueueAttributes(queue.name);
-		queueMessages.set(
-			{ queue: queue.name, role: roles.get(queue.name) ?? "work" },
-			attributes.approximateNumberOfMessages,
-		);
+		const labels = { queue: queue.name, role: roles.get(queue.name) ?? "work" };
+		queueMessages.set(labels, attributes.approximateNumberOfMessages);
+		if (labels.role === "work") {
+			queueLastReceiveAge.set(labels, store.lastReceiveAgeSeconds(queue.name));
+		}
 	}
 };

package/src/store.test.ts

@@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { after, beforeEach, describe, it } from "node:test";
 import { fileURLToPath } from "node:url";
+import Database from "better-sqlite3";
 import { bootstrapQueues, parseQueuesConfig } from "./queues-config.js";
 import {
 	InvalidParameterValueError,
@@ -317,4 +318,57 @@ describe("QueueStore", () => {
 			0,
 		);
 	});
+
+	const seedBacklog = (rows: number, groups: number): void => {
+		const path = join(dir, "queue.db");
+		store.close();
+		const seed = new Database(path);
+		seed
+			.prepare(
+				`WITH RECURSIVE n(i) AS (
+					SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ?
+				)
+				INSERT INTO messages (
+					message_id, queue_name, body, md5_body, group_id, dedup_id,
+					sequence_number, receive_count, visible_at, receipt_handle,
+					sent_at, first_received_at
+				)
+				SELECT 'm' || i, 'orders.fifo', 'x', 'x', 'g' || CAST(i % CAST(? AS INTEGER) AS INTEGER), NULL,
+					i, 0, 0, NULL, 0, NULL
+				FROM n`,
+			)
+			.run(rows, groups);
+		seed.close();
+		store = new QueueStore(path);
+	};
+
+	it("receives only the head of each group from a FIFO backlog", () => {
+		seedBacklog(30, 3);
+		const received = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			now: 1_000,
+		});
+		assert.deepEqual(
+			received.map((m) => [m.messageId, m.groupId]),
+			[
+				["m1", "g1"],
+				["m2", "g2"],
+				["m3", "g0"],
+			],
+		);
+	});
+
+	it("receives from a 200k-row FIFO backlog within a fixed time budget", () => {
+		seedBacklog(200_000, 5);
+		const started = performance.now();
+		const received = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			now: 1_000,
+		});
+		const elapsed = performance.now() - started;
+		assert.equal(received.length, 5);
+		assert.ok(elapsed < 500, `receive took ${elapsed.toFixed(1)} ms`);
+	});
 });

package/src/store.ts

@@ -97,6 +97,8 @@ const toQueueRecord = (row: QueueRow): QueueRecord => ({
 export class QueueStore {
 	private readonly db: Database.Database;
 	private sequenceCounter: bigint;
+	private readonly startedAt = Date.now();
+	private readonly lastReceiveAt = new Map<string, number>();
 
 	constructor(filename: string) {
 		this.db = new Database(filename);
@@ -139,6 +141,8 @@ export class QueueStore {
 			);
 			CREATE INDEX IF NOT EXISTS messages_queue_order
 				ON messages (queue_name, id);
+			CREATE INDEX IF NOT EXISTS messages_queue_group
+				ON messages (queue_name, group_id, id);
 			CREATE TABLE IF NOT EXISTS dedup (
 				queue_name TEXT NOT NULL,
 				dedup_id TEXT NOT NULL,
@@ -305,32 +309,13 @@ export class QueueStore {
 			input.visibilityTimeoutSeconds ?? queue.visibilityTimeoutSeconds;
 
 		const receive = this.db.transaction((): ReceivedMessage[] => {
-			const inFlightGroups = new Set(
-				(
-					this.db
-						.prepare(
-							`SELECT DISTINCT group_id FROM messages
-							WHERE queue_name = ? AND group_id IS NOT NULL
-							AND visible_at > ?`,
-						)
-						.all(input.queueName, now) as { group_id: string }[]
-				).map((r) => r.group_id),
-			);
-
-			const candidates = this.db
-				.prepare(
-					`SELECT * FROM messages
-					WHERE queue_name = ? AND visible_at <= ?
-					ORDER BY id ASC`,
-				)
-				.all(input.queueName, now) as MessageRow[];
-
 			const picked: ReceivedMessage[] = [];
-			const claimedGroups = new Set<string>();
-
-			for (const candidate of candidates) {
-				if (picked.length >= input.maxMessages) break;
 
+			for (const candidate of this.nextCandidates(
+				queue,
+				now,
+				input.maxMessages,
+			)) {
 				if (
 					queue.maxReceiveCount !== null &&
 					queue.deadLetterTargetName !== null &&
@@ -340,12 +325,6 @@ export class QueueStore {
 					continue;
 				}
 
-				if (queue.fifo && candidate.group_id !== null) {
-					if (inFlightGroups.has(candidate.group_id)) continue;
-					if (claimedGroups.has(candidate.group_id)) continue;
-					claimedGroups.add(candidate.group_id);
-				}
-
 				const receiptHandle = randomUUID();
 				const receiveCount = candidate.receive_count + 1;
 				const firstReceived = candidate.first_received_at ?? now;
@@ -380,7 +359,15 @@ export class QueueStore {
 			return picked;
 		});
 
-		return receive();
+		const received = receive();
+		if (received.length > 0) this.lastReceiveAt.set(input.queueName, now);
+		return received;
+	}
+
+	lastReceiveAgeSeconds(queueName: string, now = Date.now()): number {
+		this.requireQueue(queueName);
+		const since = this.lastReceiveAt.get(queueName) ?? this.startedAt;
+		return Math.max(0, Math.floor((now - since) / 1000));
 	}
 
 	deleteMessage(queueName: string, receiptHandle: string): void {
@@ -425,19 +412,67 @@ export class QueueStore {
 		};
 	}
 
+	private nextCandidates(
+		queue: QueueRecord,
+		now: number,
+		limit: number,
+	): MessageRow[] {
+		if (!queue.fifo) {
+			return this.db
+				.prepare(
+					`SELECT * FROM messages
+					WHERE queue_name = ? AND visible_at <= ?
+					ORDER BY id ASC LIMIT ?`,
+				)
+				.all(queue.name, now, limit) as MessageRow[];
+		}
+
+		return this.db
+			.prepare(
+				`WITH RECURSIVE g(gid) AS (
+					SELECT MIN(group_id) FROM messages WHERE queue_name = @queue
+					UNION ALL
+					SELECT (
+						SELECT MIN(group_id) FROM messages
+						WHERE queue_name = @queue AND group_id > g.gid
+					)
+					FROM g WHERE g.gid IS NOT NULL
+				)
+				SELECT m.* FROM g
+				JOIN messages m ON m.id = (
+					SELECT MIN(id) FROM messages
+					WHERE queue_name = @queue AND group_id = g.gid
+				)
+				WHERE m.visible_at <= @now
+				ORDER BY m.id ASC LIMIT @limit`,
+			)
+			.all({ queue: queue.name, now, limit }) as MessageRow[];
+	}
+
 	private moveToDeadLetter(
 		message: MessageRow,
 		deadLetterName: string,
 		now: number,
 	): void {
+		this.db.prepare("DELETE FROM messages WHERE id = ?").run(message.id);
 		this.db
 			.prepare(
-				`UPDATE messages SET
-					queue_name = ?, receive_count = 0, visible_at = 0,
-					receipt_handle = NULL, first_received_at = NULL, sent_at = ?
-				WHERE id = ?`,
+				`INSERT INTO messages (
+					message_id, queue_name, body, md5_body, group_id, dedup_id,
+					sequence_number, receive_count, visible_at, receipt_handle,
+					sent_at, first_received_at
+				) VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, NULL, ?, NULL)`,
 			)
-			.run(deadLetterName, now, message.id);
+			.run(
+				message.message_id,
+				deadLetterName,
+				message.body,
+				message.md5_body,
+				message.group_id,
+				message.dedup_id,
+				message.sequence_number,
+				now,
+			);
 	}
 
 	private pruneDedup(queueName: string, now: number): void {