@remit/search-index-worker 0.0.11 → 0.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-index-worker",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -34,7 +34,8 @@
     "drizzle-orm": "^0.45.2",
     "expect-env": "*",
     "@types/aws-lambda": "*",
-    "@types/pg": "^8.0.0"
+    "@types/pg": "^8.0.0",
+    "prom-client": "^15.1.3"
   },
   "devDependencies": {
     "@aws-sdk/client-dynamodb": "*",

package/src/handler.ts

@@ -3,8 +3,8 @@ import { NotFoundError } from "@remit/data-ports/errors";
 import {
 	createLogger,
 	type Logger,
-	MetricUnit,
-	metrics,
+	queueNameFromEventSource,
+	recordQueueEvent,
 	withTelemetry,
 } from "@remit/logger-lambda";
 import type { VectorRecord } from "@remit/search-service";
@@ -38,27 +38,28 @@ export const processBatch = async (
 	log: Logger,
 ): Promise<SQSBatchResponse> => {
 	const batchItemFailures: { itemIdentifier: string }[] = [];
-	const processingStart = Date.now();
 
 	for (const record of records) {
 		const message = parseQueueMessage(record.body);
+		const start = Date.now();
 
 		const failed =
 			message.kind === "delete"
 				? await deleteMessage(message, services, log)
 				: await upsertMessage(message, services, log);
 
+		recordQueueEvent({
+			queue: queueNameFromEventSource(record.eventSourceARN),
+			eventType: message.kind,
+			outcome: failed ? "failure" : "success",
+			durationMs: Date.now() - start,
+		});
+
 		if (failed) {
 			batchItemFailures.push({ itemIdentifier: record.messageId });
 		}
 	}
 
-	metrics.addMetric(
-		"searchIndexProcessingDuration",
-		MetricUnit.Milliseconds,
-		Date.now() - processingStart,
-	);
-
 	return { batchItemFailures };
 };
 
@@ -71,7 +72,6 @@ const deleteMessage = async (
 		.delete(message.messageId)
 		.then(() => {
 			log.info("Deleted search vectors", { messageId: message.messageId });
-			metrics.addMetric("searchIndexProcessed", MetricUnit.Count, 1);
 			return false;
 		})
 		.catch((error) => {
@@ -79,7 +79,6 @@ const deleteMessage = async (
 				error: inspect(error),
 				messageId: message.messageId,
 			});
-			metrics.addMetric("searchIndexFailures", MetricUnit.Count, 1);
 			return true;
 		});
 
@@ -105,7 +104,6 @@ export const upsertMessage = async (
 				accountId: message.accountId,
 				messageId: message.messageId,
 			});
-			metrics.addMetric("searchIndexFailures", MetricUnit.Count, 1);
 			return true;
 		});
 
@@ -152,8 +150,6 @@ const indexMessage = async (
 		upserted,
 		skipped,
 	});
-	metrics.addMetric("searchIndexProcessed", MetricUnit.Count, upserted);
-	metrics.addMetric("searchIndexSkipped", MetricUnit.Count, skipped);
 	services.onIndexOutcome?.({ status: "indexed", upserted, skipped });
 };
 
@@ -280,6 +276,7 @@ const prepareUpsert = async (
 			fromName: threadMessage.fromName ?? null,
 			subject: threadMessage.subject ?? "",
 			category: threadMessage.category,
+			...(threadMessage.listId ? { listId: threadMessage.listId } : {}),
 		},
 	});
 };

package/src/metrics.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import { beforeEach, describe, it } from "node:test";
+import { renderMetrics, resetMetrics } from "@remit/logger-lambda/metrics";
+import { registerSearchIndexBacklog } from "./metrics.js";
+
+const backlogLines = (text: string): string[] =>
+	text
+		.split("\n")
+		.filter(
+			(line) =>
+				line.startsWith("remit_search_index_backlog") && line.length > 0,
+		);
+
+describe("the search index backlog series", () => {
+	beforeEach(() => resetMetrics());
+
+	// The reason this gauge is declared inside a function, in this package, and
+	// not at module scope in the shared registry: it carries no labels, so it
+	// renders from the moment it exists. Declared where every service can import
+	// it, four processes that cannot count an outbox would each publish a
+	// confident 0.
+	it("renders nothing in a process that never registered it", async () => {
+		assert.deepEqual(backlogLines(await renderMetrics()), []);
+	});
+
+	it("reports the count the registered reader returns", async () => {
+		registerSearchIndexBacklog(async () => 7);
+		const text = await renderMetrics();
+		assert.match(text, /^# TYPE remit_search_index_backlog_rows gauge$/m);
+		assert.match(text, /^remit_search_index_backlog_rows 7$/m);
+	});
+
+	it("re-reads on every scrape", async () => {
+		let rows = 3;
+		registerSearchIndexBacklog(async () => rows);
+		assert.match(await renderMetrics(), /^remit_search_index_backlog_rows 3$/m);
+		rows = 0;
+		assert.match(await renderMetrics(), /^remit_search_index_backlog_rows 0$/m);
+	});
+
+	it("fails the scrape when the outbox cannot be counted", async () => {
+		registerSearchIndexBacklog(async () => {
+			throw new Error("database is locked");
+		});
+		await assert.rejects(renderMetrics(), /database is locked/);
+	});
+});

package/src/metrics.ts

@@ -0,0 +1,31 @@
+import { onScrape, registry } from "@remit/logger-lambda/metrics";
+import { Gauge } from "prom-client";
+
+/**
+ * Put the search index backlog (standalone-observability D3) on this process's
+ * registry, read from `count` when a scrape arrives.
+ *
+ * The gauge is constructed here rather than at module scope, and this module is
+ * not the shared one, for the same reason: it carries no labels, so prom-client
+ * renders it from the moment it exists. Declared in the shared registry module
+ * it would be a confident `0` in the backend, the queue sidecar and three
+ * workers that cannot know the answer — five series separated only by
+ * `instance`, four of them permanently wrong. Declared at this module's scope it
+ * would be a `0` on every backend that has no outbox to count. The series exists
+ * only where something computes it.
+ */
+const NAME = "remit_search_index_backlog_rows";
+
+export const registerSearchIndexBacklog = (
+	count: () => Promise<number>,
+): void => {
+	// Registering twice replaces the reader rather than throwing on the duplicate
+	// name — the caller is saying where the count comes from now.
+	registry.removeSingleMetric(NAME);
+	const backlogRows = new Gauge({
+		name: NAME,
+		help: "Search-index outbox rows that have not been relayed yet.",
+		registers: [registry],
+	});
+	onScrape(async () => backlogRows.set(await count()));
+};

package/src/poller.ts

@@ -1,7 +1,8 @@
-import { createLogger } from "@remit/logger-lambda";
+import { createLogger, startMetricsServer } from "@remit/logger-lambda";
 import { runQueuePoller } from "@remit/sqs-client/poller";
 import { env } from "expect-env";
 import { handler } from "./index.js";
+import { registerSearchIndexBacklog } from "./metrics.js";
 import { maybeStartSqliteOutboxDrain } from "./sqlite-outbox-drain.js";
 
 /** Production queue poller — no e2e shim exists for this queue today (the
@@ -15,6 +16,18 @@ const log = createLogger();
 // loop, which blocks until shutdown, then stopped after it returns.
 const drain = await maybeStartSqliteOutboxDrain(log);
 
+// The backlog is a count of rows in the shared file, so it is read when a
+// scrape arrives rather than tracked as work moves. Registered only where there
+// is an outbox to count, so on every other backend the series is absent rather
+// than a zero nothing computed.
+if (drain) {
+	registerSearchIndexBacklog(() => drain.countBacklog());
+}
+
+// /metrics and nothing else, on the compose network (D2). Started before the
+// poll loop, which blocks until shutdown.
+startMetricsServer();
+
 try {
 	await runQueuePoller({
 		log,

package/src/sqlite-outbox-drain.test.ts

@@ -87,4 +87,29 @@ describe("SqliteOutboxStore", () => {
 		assert.deepEqual(pending, ["r2"], "the mid-flight row stays pending");
 		db.close();
 	});
+
+	test("counts undrained rows as the exported search index backlog", async () => {
+		const db = makeOutboxDb();
+		const store = new SqliteOutboxStore(db as unknown as never);
+		assert.equal(await store.countUnprocessedRows(), 0);
+
+		// Two rows for one message count as two: the backlog is outstanding work,
+		// not the number of messages it concerns.
+		insertRow(db, "r1", "m1", "message.body_synced");
+		insertRow(db, "r2", "m1", "message.body_synced");
+		insertRow(db, "r3", "m2", "message.moved");
+		assert.equal(await store.countUnprocessedRows(), 3);
+
+		await store.markRowsProcessed(["r1", "r2"]);
+		assert.equal(await store.countUnprocessedRows(), 1);
+		db.close();
+	});
+
+	test("ignores an event the drain does not relay", async () => {
+		const db = makeOutboxDb();
+		const store = new SqliteOutboxStore(db as unknown as never);
+		insertRow(db, "r1", "m1", "message.something_else");
+		assert.equal(await store.countUnprocessedRows(), 0);
+		db.close();
+	});
 });

package/src/sqlite-outbox-drain.ts

@@ -29,6 +29,7 @@ const DRAIN_INTERVAL_MS = 2_000;
 // out of the Lambda bundle).
 interface SqliteStatement {
 	all(...params: unknown[]): unknown[];
+	get(...params: unknown[]): unknown;
 	run(...params: unknown[]): unknown;
 }
 interface SqliteDatabase {
@@ -66,6 +67,23 @@ export class SqliteOutboxStore implements OutboxStore {
 		return rows.map((row) => row.id);
 	}
 
+	/**
+	 * The search index backlog (standalone-observability D3): rows committed but
+	 * not yet relayed onto the queue. Counted per row rather than per distinct
+	 * event, so it is the work outstanding and not the number of messages it
+	 * concerns.
+	 */
+	async countUnprocessedRows(): Promise<number> {
+		const placeholders = DRAIN_EVENTS.map(() => "?").join(", ");
+		const row = this.db
+			.prepare(
+				`SELECT COUNT(*) AS n FROM outbox
+				 WHERE event IN (${placeholders}) AND processed_at IS NULL`,
+			)
+			.get(...DRAIN_EVENTS) as { n: number };
+		return row.n;
+	}
+
 	async markRowsProcessed(ids: string[]): Promise<void> {
 		if (ids.length === 0) return;
 		const placeholders = ids.map(() => "?").join(", ");
@@ -80,6 +98,8 @@ export class SqliteOutboxStore implements OutboxStore {
 
 export interface RunningDrain {
 	stop(): Promise<void>;
+	/** Undrained outbox rows, read fresh — the D3 search index backlog. */
+	countBacklog(): Promise<number>;
 }
 
 export interface SqliteOutboxDrainConfig {
@@ -111,11 +131,8 @@ export const startSqliteOutboxDrain = async (
 	sqlite.pragma("synchronous = NORMAL");
 
 	const sqs = createSqsClient(queueUrl);
-	const relay = new OutboxRelay({
-		store: new SqliteOutboxStore(sqlite),
-		sqs,
-		queueUrl,
-	});
+	const store = new SqliteOutboxStore(sqlite);
+	const relay = new OutboxRelay({ store, sqs, queueUrl });
 
 	let draining = false;
 	let inFlight: Promise<unknown> = Promise.resolve();
@@ -139,6 +156,7 @@ export const startSqliteOutboxDrain = async (
 	timer.unref();
 
 	return {
+		countBacklog: () => store.countUnprocessedRows(),
 		stop: async () => {
 			clearInterval(timer);
 			await inFlight;