@remit/search-index-worker 0.0.18 → 0.0.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-index-worker",
-  "version": "0.0.18",
+  "version": "0.0.20",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -20,7 +20,6 @@
   },
   "dependencies": {
     "@remit/outbox-relay": "*",
-    "p-map": "*",
     "@remit/logger-lambda": "*",
     "@remit/sqs-client": "*",
     "@remit/data-ports": "*",
@@ -34,11 +33,7 @@
     "prom-client": "^15.1.3"
   },
   "devDependencies": {
-    "@aws-sdk/client-dynamodb": "*",
-    "@aws-sdk/core": "*",
-    "@aws-sdk/lib-dynamodb": "*",
-    "tsx": "*",
-    "zod": "*"
+    "tsx": "*"
   },
   "license": "MIT",
   "publishConfig": {

package/src/consumer.ts

@@ -1,154 +0,0 @@
-import {
-	DeleteMessageCommand,
-	ReceiveMessageCommand,
-	type ReceiveMessageCommandOutput,
-	type SQSClient,
-} from "@aws-sdk/client-sqs";
-import type { Logger } from "@remit/logger-lambda";
-import { createQueueProducer } from "@remit/sqs-client/producer";
-import type { SQSRecord } from "aws-lambda";
-import { type IndexOutcome, processBatch } from "./handler.js";
-import { createIndexWorkStats, type IndexWorkStats } from "./index-stats.js";
-import { parseQueueMessage } from "./parse.js";
-import type { Services } from "./services.js";
-import type { RunningConsumer } from "./shutdown.js";
-
-// `processBatch` takes a real `SQSRecord[]` (the Lambda event shape); the
-// long-poll `ReceiveMessageCommand` result carries the same message id/body
-// plus SDK-specific fields, so this fills in the rest with inert placeholders
-// — `processBatch` only reads `record.body` and `record.messageId`.
-const toSqsRecord = (
-	sqsMessage: NonNullable<ReceiveMessageCommandOutput["Messages"]>[number],
-): SQSRecord =>
-	({
-		messageId: sqsMessage.MessageId ?? sqsMessage.ReceiptHandle ?? "",
-		body: sqsMessage.Body ?? "",
-		receiptHandle: sqsMessage.ReceiptHandle ?? "",
-		attributes: {} as SQSRecord["attributes"],
-		messageAttributes: {},
-		md5OfBody: sqsMessage.MD5OfBody ?? "",
-		eventSource: "aws:sqs",
-		eventSourceARN: "",
-		awsRegion: "",
-	}) satisfies SQSRecord;
-
-const createSqsClient = (queueUrl: string): SQSClient =>
-	createQueueProducer({
-		queueUrl,
-		localCredentials: { accessKeyId: "local", secretAccessKey: "local" },
-	});
-
-export interface SqsConsumerConfig {
-	/** Search-index queue URL; defaults to `SQS_QUEUE_URL_SEARCH_INDEX`. */
-	queueUrl?: string;
-	services: Services;
-	logger: Logger;
-}
-
-/**
- * Long-polls the search-index SQS queue and processes one message at a time
- * via `processBatch` (a batch of one) — the same per-message logic the AWS
- * Lambda handler runs, reused so the two deployment shapes (Lambda event
- * source mapping vs. a long-running container/pm2 process) share one
- * indexing implementation. Used by the self-host stack, where the search-index
- * queue has no Lambda event source: the outbox drain relays committed events
- * onto this queue (the producer side) and this consumer takes them off it.
- *
- * A message is deleted only when `processBatch` reports no failure for it;
- * a failure leaves it on the queue so its visibility timeout lapses and SQS
- * redelivers (and eventually dead-letters) it — SQS owns the retry, same as
- * the Lambda path's `batchItemFailures`.
- *
- * Every outcome is fed to `IndexWorkStats` (via `Services.onIndexOutcome`)
- * and flushed on a lazy interval — the pg-only "index work summary" signal
- * that surfaces over-triggering (#1082) CloudWatch alarms can't see.
- */
-export const startSqsConsumer = (
-	config: SqsConsumerConfig,
-): RunningConsumer => {
-	const { logger: log } = config;
-	const queueUrl = config.queueUrl ?? process.env.SQS_QUEUE_URL_SEARCH_INDEX;
-	if (!queueUrl) throw new Error("SQS_QUEUE_URL_SEARCH_INDEX is required");
-
-	const sqs = createSqsClient(queueUrl);
-
-	const stats: IndexWorkStats = createIndexWorkStats();
-	const flushStats = (): void => {
-		const summary = stats.drain();
-		if (summary) log.info("index work summary", { ...summary });
-	};
-	const SUMMARY_INTERVAL_MS = 60_000;
-	const summaryTimer = setInterval(flushStats, SUMMARY_INTERVAL_MS);
-	summaryTimer.unref();
-
-	const controller = new AbortController();
-	const consume = async (): Promise<void> => {
-		while (!controller.signal.aborted) {
-			let response: ReceiveMessageCommandOutput;
-			try {
-				response = await sqs.send(
-					new ReceiveMessageCommand({
-						QueueUrl: queueUrl,
-						MaxNumberOfMessages: 10,
-						WaitTimeSeconds: 20,
-						VisibilityTimeout: 300,
-					}),
-					{ abortSignal: controller.signal },
-				);
-			} catch (error) {
-				if (controller.signal.aborted) return;
-				throw error;
-			}
-
-			for (const sqsMessage of response.Messages ?? []) {
-				if (!sqsMessage.Body || !sqsMessage.ReceiptHandle) continue;
-
-				const body = sqsMessage.Body;
-				const parsed = await Promise.resolve()
-					.then(() => parseQueueMessage(body))
-					.catch((error: unknown) => {
-						log.error("parse failed", { body, error: String(error) });
-						return null;
-					});
-				if (!parsed) continue;
-				const force = parsed.kind === "upsert" ? parsed.force : false;
-
-				let lastOutcome: IndexOutcome | undefined;
-				const services: Services = {
-					...config.services,
-					onIndexOutcome: (outcome) => {
-						lastOutcome = outcome;
-					},
-				};
-
-				const { batchItemFailures } = await processBatch(
-					[toSqsRecord(sqsMessage)],
-					services,
-					log,
-				);
-
-				if (lastOutcome) stats.record(lastOutcome, force);
-
-				if (batchItemFailures.length === 0) {
-					await sqs.send(
-						new DeleteMessageCommand({
-							QueueUrl: queueUrl,
-							ReceiptHandle: sqsMessage.ReceiptHandle,
-						}),
-					);
-				}
-			}
-		}
-	};
-	const consumer = consume();
-
-	return {
-		stop: async () => {
-			clearInterval(summaryTimer);
-			flushStats();
-			controller.abort();
-			await consumer;
-			sqs.destroy();
-		},
-	};
-};

package/src/index-stats.test.ts

@@ -1,56 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, test } from "node:test";
-import { createIndexWorkStats } from "./index-stats.js";
-
-describe("createIndexWorkStats", () => {
-	test("counts an upserted:0 indexed outcome as noop (the over-trigger signal)", () => {
-		const stats = createIndexWorkStats();
-		stats.record({ status: "indexed", upserted: 0, skipped: 4 }, false);
-		stats.record({ status: "indexed", upserted: 2, skipped: 1 }, false);
-		assert.deepStrictEqual(stats.drain(), {
-			processed: 2,
-			embedded: 1,
-			noop: 1,
-			deferred: 0,
-			dropped: 0,
-			forced: 0,
-		});
-	});
-
-	test("separates transient (deferred) from terminal (dropped) skips", () => {
-		const stats = createIndexWorkStats();
-		stats.record(
-			{ status: "skipped", reason: "parsed-body-not-found", retryable: true },
-			false,
-		);
-		stats.record(
-			{ status: "skipped", reason: "no-indexable-content", retryable: false },
-			false,
-		);
-		assert.deepStrictEqual(stats.drain(), {
-			processed: 2,
-			embedded: 0,
-			noop: 0,
-			deferred: 1,
-			dropped: 1,
-			forced: 0,
-		});
-	});
-
-	test("counts force re-indexes (moves) separately", () => {
-		const stats = createIndexWorkStats();
-		stats.record({ status: "indexed", upserted: 3, skipped: 0 }, true);
-		stats.record({ status: "indexed", upserted: 1, skipped: 0 }, false);
-		const summary = stats.drain();
-		assert.equal(summary?.forced, 1);
-		assert.equal(summary?.processed, 2);
-	});
-
-	test("drain resets the window and returns null when empty", () => {
-		const stats = createIndexWorkStats();
-		assert.equal(stats.drain(), null, "nothing recorded yet");
-		stats.record({ status: "indexed", upserted: 1, skipped: 0 }, false);
-		assert.ok(stats.drain());
-		assert.equal(stats.drain(), null, "drained window is empty again");
-	});
-});

package/src/index-stats.ts

@@ -1,62 +0,0 @@
-import type { IndexOutcome } from "./handler.js";
-
-/**
- * A window of index outcomes. `noop` is the proxy metric for over-triggering:
- * an indexed message that embedded nothing (`upserted: 0`) means an event fired
- * for content already in the store. A high noop rate is the signal that error
- * and DLQ alarms can't see — "too much successful work" — and is what let the
- * AWS search-index cost blowup ramp unnoticed (#1082). Pg-only: the Lambda path
- * never wires `Services.onIndexOutcome`, so this only accumulates in the
- * long-running Postgres consumer (`consumer.ts`).
- */
-export interface IndexWorkSummary {
-	processed: number;
-	/** Indexed and wrote vectors — real work. */
-	embedded: number;
-	/** Indexed but wrote nothing (`upserted: 0`) — an event for unchanged content. */
-	noop: number;
-	/** Transient skip (thread/body not visible yet); left undrained for retry. */
-	deferred: number;
-	/** Terminal skip; drained, will never index. */
-	dropped: number;
-	/** Of the above, how many were force re-indexes (moves) — always re-embedded. */
-	forced: number;
-}
-
-export interface IndexWorkStats {
-	record(outcome: IndexOutcome, force: boolean): void;
-	/** Return the accumulated window and reset it; null if nothing was recorded. */
-	drain(): IndexWorkSummary | null;
-}
-
-const empty = (): IndexWorkSummary => ({
-	processed: 0,
-	embedded: 0,
-	noop: 0,
-	deferred: 0,
-	dropped: 0,
-	forced: 0,
-});
-
-export const createIndexWorkStats = (): IndexWorkStats => {
-	let window = empty();
-	return {
-		record: (outcome, force) => {
-			window.processed += 1;
-			if (force) window.forced += 1;
-			if (outcome.status === "indexed") {
-				if (outcome.upserted > 0) window.embedded += 1;
-				else window.noop += 1;
-				return;
-			}
-			if (outcome.retryable) window.deferred += 1;
-			else window.dropped += 1;
-		},
-		drain: () => {
-			if (window.processed === 0) return null;
-			const summary = window;
-			window = empty();
-			return summary;
-		},
-	};
-};

package/src/run-worker.ts

@@ -1,26 +0,0 @@
-import { createLogger } from "@remit/logger-lambda";
-import { startSqsConsumer } from "./consumer.js";
-import { getServices } from "./services.js";
-import { runShutdown } from "./shutdown.js";
-
-const SHUTDOWN_TIMEOUT_MS = 10_000;
-const log = createLogger();
-
-const main = async (): Promise<void> => {
-	const services = await getServices();
-	const consumer = startSqsConsumer({ services, logger: log });
-
-	const shutdown = (): void =>
-		runShutdown(consumer, {
-			timeoutMs: SHUTDOWN_TIMEOUT_MS,
-			exit: (code) => process.exit(code),
-			onError: (error) =>
-				log.error("shutdown failed", { error: String(error) }),
-		});
-	process.on("SIGINT", shutdown);
-	process.on("SIGTERM", shutdown);
-
-	log.info("search-index-worker consumer started");
-};
-
-await main();

package/src/shutdown.test.ts

@@ -1,54 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, test } from "node:test";
-import type { RunningConsumer } from "./shutdown.js";
-import { runShutdown } from "./shutdown.js";
-
-const nextTick = (): Promise<void> =>
-	new Promise((resolve) => setImmediate(resolve));
-
-describe("runShutdown", () => {
-	test("force-exits with code 1 when stop() never settles", async () => {
-		const hanging: RunningConsumer = {
-			stop: () => new Promise<void>(() => {}),
-		};
-		const codes: number[] = [];
-
-		runShutdown(hanging, {
-			timeoutMs: 5,
-			exit: (code) => codes.push(code),
-		});
-
-		await new Promise((resolve) => setTimeout(resolve, 25));
-		assert.deepStrictEqual(codes, [1], "timeout forces a code-1 exit");
-	});
-
-	test("exits 0 when stop() resolves before the deadline", async () => {
-		const clean: RunningConsumer = { stop: async () => {} };
-		const codes: number[] = [];
-
-		runShutdown(clean, { timeoutMs: 10_000, exit: (code) => codes.push(code) });
-
-		await nextTick();
-		assert.deepStrictEqual(codes, [0], "clean stop exits 0");
-	});
-
-	test("exits 1 and reports when stop() rejects", async () => {
-		const failing: RunningConsumer = {
-			stop: async () => {
-				throw new Error("consumer stop failed");
-			},
-		};
-		const codes: number[] = [];
-		const errors: unknown[] = [];
-
-		runShutdown(failing, {
-			timeoutMs: 10_000,
-			exit: (code) => codes.push(code),
-			onError: (error) => errors.push(error),
-		});
-
-		await nextTick();
-		assert.deepStrictEqual(codes, [1], "a failed stop exits 1");
-		assert.equal(errors.length, 1);
-	});
-});

package/src/shutdown.ts

@@ -1,39 +0,0 @@
-export interface RunningConsumer {
-	stop(): Promise<void>;
-}
-
-export interface ShutdownOptions {
-	timeoutMs: number;
-	exit: (code: number) => void;
-	onError?: (error: unknown) => void;
-}
-
-/**
- * Race a clean `consumer.stop()` against a hard deadline, then exit regardless.
- *
- * `stop()` awaits the SQS long-poll loop; under a saturated event loop that can
- * hang. A shutdown that never exits leaves the process ignoring SIGTERM and
- * orphaning to init still burning CPU (issue #1171), so the deadline
- * force-exits even when `stop()` never settles.
- */
-export const runShutdown = (
-	consumer: RunningConsumer,
-	options: ShutdownOptions,
-): void => {
-	const { timeoutMs, exit, onError } = options;
-
-	const forceExit = setTimeout(() => exit(1), timeoutMs);
-	forceExit.unref();
-
-	consumer
-		.stop()
-		.then(() => {
-			clearTimeout(forceExit);
-			exit(0);
-		})
-		.catch((error: unknown) => {
-			clearTimeout(forceExit);
-			onError?.(error);
-			exit(1);
-		});
-};