@remit/queue-sidecar 0.0.8 → 0.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/queue-sidecar",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -16,7 +16,7 @@
   "scripts": {
     "start": "node --import tsx src/main.ts",
     "test:typecheck": "tsgo --noEmit",
-    "test:run": "node --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=93 --test 'src/**/*.test.ts'",
+    "test:run": "node $NODE_TEST_FLAGS --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=93 --test 'src/**/*.test.ts'",
     "test": "npm run test:typecheck && npm run test:run"
   },
   "dependencies": {

package/src/store.test.ts

@@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { after, beforeEach, describe, it } from "node:test";
 import { fileURLToPath } from "node:url";
+import Database from "better-sqlite3";
 import { bootstrapQueues, parseQueuesConfig } from "./queues-config.js";
 import {
 	InvalidParameterValueError,
@@ -317,4 +318,57 @@ describe("QueueStore", () => {
 			0,
 		);
 	});
+
+	const seedBacklog = (rows: number, groups: number): void => {
+		const path = join(dir, "queue.db");
+		store.close();
+		const seed = new Database(path);
+		seed
+			.prepare(
+				`WITH RECURSIVE n(i) AS (
+					SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ?
+				)
+				INSERT INTO messages (
+					message_id, queue_name, body, md5_body, group_id, dedup_id,
+					sequence_number, receive_count, visible_at, receipt_handle,
+					sent_at, first_received_at
+				)
+				SELECT 'm' || i, 'orders.fifo', 'x', 'x', 'g' || CAST(i % CAST(? AS INTEGER) AS INTEGER), NULL,
+					i, 0, 0, NULL, 0, NULL
+				FROM n`,
+			)
+			.run(rows, groups);
+		seed.close();
+		store = new QueueStore(path);
+	};
+
+	it("receives only the head of each group from a FIFO backlog", () => {
+		seedBacklog(30, 3);
+		const received = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			now: 1_000,
+		});
+		assert.deepEqual(
+			received.map((m) => [m.messageId, m.groupId]),
+			[
+				["m1", "g1"],
+				["m2", "g2"],
+				["m3", "g0"],
+			],
+		);
+	});
+
+	it("receives from a 200k-row FIFO backlog within a fixed time budget", () => {
+		seedBacklog(200_000, 5);
+		const started = performance.now();
+		const received = store.receiveMessages({
+			queueName: "orders.fifo",
+			maxMessages: 10,
+			now: 1_000,
+		});
+		const elapsed = performance.now() - started;
+		assert.equal(received.length, 5);
+		assert.ok(elapsed < 500, `receive took ${elapsed.toFixed(1)} ms`);
+	});
 });

package/src/store.ts

@@ -139,6 +139,8 @@ export class QueueStore {
 			);
 			CREATE INDEX IF NOT EXISTS messages_queue_order
 				ON messages (queue_name, id);
+			CREATE INDEX IF NOT EXISTS messages_queue_group
+				ON messages (queue_name, group_id, id);
 			CREATE TABLE IF NOT EXISTS dedup (
 				queue_name TEXT NOT NULL,
 				dedup_id TEXT NOT NULL,
@@ -305,32 +307,13 @@ export class QueueStore {
 			input.visibilityTimeoutSeconds ?? queue.visibilityTimeoutSeconds;
 
 		const receive = this.db.transaction((): ReceivedMessage[] => {
-			const inFlightGroups = new Set(
-				(
-					this.db
-						.prepare(
-							`SELECT DISTINCT group_id FROM messages
-							WHERE queue_name = ? AND group_id IS NOT NULL
-							AND visible_at > ?`,
-						)
-						.all(input.queueName, now) as { group_id: string }[]
-				).map((r) => r.group_id),
-			);
-
-			const candidates = this.db
-				.prepare(
-					`SELECT * FROM messages
-					WHERE queue_name = ? AND visible_at <= ?
-					ORDER BY id ASC`,
-				)
-				.all(input.queueName, now) as MessageRow[];
-
 			const picked: ReceivedMessage[] = [];
-			const claimedGroups = new Set<string>();
-
-			for (const candidate of candidates) {
-				if (picked.length >= input.maxMessages) break;
 
+			for (const candidate of this.nextCandidates(
+				queue,
+				now,
+				input.maxMessages,
+			)) {
 				if (
 					queue.maxReceiveCount !== null &&
 					queue.deadLetterTargetName !== null &&
@@ -340,12 +323,6 @@ export class QueueStore {
 					continue;
 				}
 
-				if (queue.fifo && candidate.group_id !== null) {
-					if (inFlightGroups.has(candidate.group_id)) continue;
-					if (claimedGroups.has(candidate.group_id)) continue;
-					claimedGroups.add(candidate.group_id);
-				}
-
 				const receiptHandle = randomUUID();
 				const receiveCount = candidate.receive_count + 1;
 				const firstReceived = candidate.first_received_at ?? now;
@@ -425,19 +402,67 @@ export class QueueStore {
 		};
 	}
 
+	private nextCandidates(
+		queue: QueueRecord,
+		now: number,
+		limit: number,
+	): MessageRow[] {
+		if (!queue.fifo) {
+			return this.db
+				.prepare(
+					`SELECT * FROM messages
+					WHERE queue_name = ? AND visible_at <= ?
+					ORDER BY id ASC LIMIT ?`,
+				)
+				.all(queue.name, now, limit) as MessageRow[];
+		}
+
+		return this.db
+			.prepare(
+				`WITH RECURSIVE g(gid) AS (
+					SELECT MIN(group_id) FROM messages WHERE queue_name = @queue
+					UNION ALL
+					SELECT (
+						SELECT MIN(group_id) FROM messages
+						WHERE queue_name = @queue AND group_id > g.gid
+					)
+					FROM g WHERE g.gid IS NOT NULL
+				)
+				SELECT m.* FROM g
+				JOIN messages m ON m.id = (
+					SELECT MIN(id) FROM messages
+					WHERE queue_name = @queue AND group_id = g.gid
+				)
+				WHERE m.visible_at <= @now
+				ORDER BY m.id ASC LIMIT @limit`,
+			)
+			.all({ queue: queue.name, now, limit }) as MessageRow[];
+	}
+
 	private moveToDeadLetter(
 		message: MessageRow,
 		deadLetterName: string,
 		now: number,
 	): void {
+		this.db.prepare("DELETE FROM messages WHERE id = ?").run(message.id);
 		this.db
 			.prepare(
-				`UPDATE messages SET
-					queue_name = ?, receive_count = 0, visible_at = 0,
-					receipt_handle = NULL, first_received_at = NULL, sent_at = ?
-				WHERE id = ?`,
+				`INSERT INTO messages (
+					message_id, queue_name, body, md5_body, group_id, dedup_id,
+					sequence_number, receive_count, visible_at, receipt_handle,
+					sent_at, first_received_at
+				) VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, NULL, ?, NULL)`,
 			)
-			.run(deadLetterName, now, message.id);
+			.run(
+				message.message_id,
+				deadLetterName,
+				message.body,
+				message.md5_body,
+				message.group_id,
+				message.dedup_id,
+				message.sequence_number,
+				now,
+			);
 	}
 
 	private pruneDedup(queueName: string, now: number): void {