@remit/search-index-worker 0.0.1

package/package.json

@@ -0,0 +1,50 @@
+{
+	"name": "@remit/search-index-worker",
+	"version": "0.0.1",
+	"type": "module",
+	"main": "dist/index.js",
+	"exports": {
+		".": {
+			"types": "./src/index.ts",
+			"default": "./src/index.ts"
+		}
+	},
+	"scripts": {
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.test.ts'",
+		"test:integ": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.integ.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"dependencies": {
+		"@remit/outbox-relay": "*",
+		"p-map": "*",
+		"pg": "^8.0.0"
+	},
+	"devDependencies": {
+		"@aws-sdk/client-dynamodb": "*",
+		"@aws-sdk/client-sqs": "*",
+		"@aws-sdk/core": "*",
+		"@aws-sdk/lib-dynamodb": "*",
+		"@remit/data-ports": "*",
+		"@remit/drizzle-service": "*",
+		"better-sqlite3": "^12.11.1",
+		"@remit/logger-lambda": "*",
+		"@remit/search-service": "*",
+		"@remit/sqs-client": "*",
+		"@remit/storage-service": "*",
+		"@types/aws-lambda": "*",
+		"@types/pg": "^8.0.0",
+		"expect-env": "*",
+		"tsx": "*",
+		"zod": "*"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/remit.git",
+		"directory": "packages/search-index-worker"
+	}
+}

package/src/consumer.ts

@@ -0,0 +1,163 @@
+import {
+	DeleteMessageCommand,
+	ReceiveMessageCommand,
+	type ReceiveMessageCommandOutput,
+	SQSClient,
+} from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import type { Logger } from "@remit/logger-lambda";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import type { SQSRecord } from "aws-lambda";
+import { type IndexOutcome, processBatch } from "./handler.js";
+import { createIndexWorkStats, type IndexWorkStats } from "./index-stats.js";
+import { parseQueueMessage } from "./parse.js";
+import type { Services } from "./services.js";
+import type { RunningConsumer } from "./shutdown.js";
+
+// `processBatch` takes a real `SQSRecord[]` (the Lambda event shape); the
+// long-poll `ReceiveMessageCommand` result carries the same message id/body
+// plus SDK-specific fields, so this fills in the rest with inert placeholders
+// — `processBatch` only reads `record.body` and `record.messageId`.
+const toSqsRecord = (
+	sqsMessage: NonNullable<ReceiveMessageCommandOutput["Messages"]>[number],
+): SQSRecord =>
+	({
+		messageId: sqsMessage.MessageId ?? sqsMessage.ReceiptHandle ?? "",
+		body: sqsMessage.Body ?? "",
+		receiptHandle: sqsMessage.ReceiptHandle ?? "",
+		attributes: {} as SQSRecord["attributes"],
+		messageAttributes: {},
+		md5OfBody: sqsMessage.MD5OfBody ?? "",
+		eventSource: "aws:sqs",
+		eventSourceARN: "",
+		awsRegion: "",
+	}) satisfies SQSRecord;
+
+const createSqsClient = (queueUrl: string): SQSClient => {
+	const isLocal = queueUrl.startsWith("http://localhost");
+	return new SQSClient({
+		endpoint: isLocal ? new URL(queueUrl).origin : undefined,
+		...(isLocal
+			? {
+					protocol: AwsQueryProtocol,
+					credentials: { accessKeyId: "local", secretAccessKey: "local" },
+				}
+			: { credentials: resolveSqsCredentials() }),
+	});
+};
+
+export interface SqsConsumerConfig {
+	/** Search-index queue URL; defaults to `SQS_QUEUE_URL_SEARCH_INDEX`. */
+	queueUrl?: string;
+	services: Services;
+	logger: Logger;
+}
+
+/**
+ * Long-polls the search-index SQS queue and processes one message at a time
+ * via `processBatch` (a batch of one) — the same per-message logic the AWS
+ * Lambda handler runs, reused so the two deployment shapes (Lambda event
+ * source mapping vs. a long-running container/pm2 process) share one
+ * indexing implementation. Used by the Postgres-parity stack, where the
+ * search-index queue has no Lambda event source; `remit-pg-index-worker`
+ * only relays committed outbox events onto this queue (the producer side —
+ * see its `worker.ts`), it does not consume them.
+ *
+ * A message is deleted only when `processBatch` reports no failure for it;
+ * a failure leaves it on the queue so its visibility timeout lapses and SQS
+ * redelivers (and eventually dead-letters) it — SQS owns the retry, same as
+ * the Lambda path's `batchItemFailures`.
+ *
+ * Every outcome is fed to `IndexWorkStats` (via `Services.onIndexOutcome`)
+ * and flushed on a lazy interval — the pg-only "index work summary" signal
+ * that surfaces over-triggering (#1082) CloudWatch alarms can't see.
+ */
+export const startSqsConsumer = (
+	config: SqsConsumerConfig,
+): RunningConsumer => {
+	const { logger: log } = config;
+	const queueUrl = config.queueUrl ?? process.env.SQS_QUEUE_URL_SEARCH_INDEX;
+	if (!queueUrl) throw new Error("SQS_QUEUE_URL_SEARCH_INDEX is required");
+
+	const sqs = createSqsClient(queueUrl);
+
+	const stats: IndexWorkStats = createIndexWorkStats();
+	const flushStats = (): void => {
+		const summary = stats.drain();
+		if (summary) log.info("index work summary", { ...summary });
+	};
+	const SUMMARY_INTERVAL_MS = 60_000;
+	const summaryTimer = setInterval(flushStats, SUMMARY_INTERVAL_MS);
+	summaryTimer.unref();
+
+	const controller = new AbortController();
+	const consume = async (): Promise<void> => {
+		while (!controller.signal.aborted) {
+			let response: ReceiveMessageCommandOutput;
+			try {
+				response = await sqs.send(
+					new ReceiveMessageCommand({
+						QueueUrl: queueUrl,
+						MaxNumberOfMessages: 10,
+						WaitTimeSeconds: 20,
+						VisibilityTimeout: 300,
+					}),
+					{ abortSignal: controller.signal },
+				);
+			} catch (error) {
+				if (controller.signal.aborted) return;
+				throw error;
+			}
+
+			for (const sqsMessage of response.Messages ?? []) {
+				if (!sqsMessage.Body || !sqsMessage.ReceiptHandle) continue;
+
+				const body = sqsMessage.Body;
+				const parsed = await Promise.resolve()
+					.then(() => parseQueueMessage(body))
+					.catch((error: unknown) => {
+						log.error("parse failed", { body, error: String(error) });
+						return null;
+					});
+				if (!parsed) continue;
+				const force = parsed.kind === "upsert" ? parsed.force : false;
+
+				let lastOutcome: IndexOutcome | undefined;
+				const services: Services = {
+					...config.services,
+					onIndexOutcome: (outcome) => {
+						lastOutcome = outcome;
+					},
+				};
+
+				const { batchItemFailures } = await processBatch(
+					[toSqsRecord(sqsMessage)],
+					services,
+					log,
+				);
+
+				if (lastOutcome) stats.record(lastOutcome, force);
+
+				if (batchItemFailures.length === 0) {
+					await sqs.send(
+						new DeleteMessageCommand({
+							QueueUrl: queueUrl,
+							ReceiptHandle: sqsMessage.ReceiptHandle,
+						}),
+					);
+				}
+			}
+		}
+	};
+	const consumer = consume();
+
+	return {
+		stop: async () => {
+			clearInterval(summaryTimer);
+			flushStats();
+			controller.abort();
+			await consumer;
+			sqs.destroy();
+		},
+	};
+};

package/src/data-ports.test.ts

@@ -0,0 +1,96 @@
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { afterEach, test } from "node:test";
+import { fileURLToPath } from "node:url";
+import { buildDataPortsFromEnv } from "./data-ports.js";
+
+// The DynamoDB composition module is stripped from the open-core tree; the
+// DynamoDB-path cases skip there and run where the module ships.
+const hasDynamoComposition = existsSync(
+	fileURLToPath(new URL("./compose-dynamodb.ts", import.meta.url)),
+);
+
+const withEnv = async (
+	overrides: Record<string, string | undefined>,
+	fn: () => Promise<void>,
+): Promise<void> => {
+	const saved: Record<string, string | undefined> = {};
+	for (const key of Object.keys(overrides)) saved[key] = process.env[key];
+	for (const [key, value] of Object.entries(overrides)) {
+		if (value === undefined) delete process.env[key];
+		else process.env[key] = value;
+	}
+	try {
+		await fn();
+	} finally {
+		for (const [key, value] of Object.entries(saved)) {
+			if (value === undefined) delete process.env[key];
+			else process.env[key] = value;
+		}
+	}
+};
+
+afterEach(() => {
+	delete process.env.DATA_BACKEND;
+});
+
+test(
+	"without DATA_BACKEND builds DynamoDB (electrodb) ports and no resolveAccountId hook",
+	{ skip: !hasDynamoComposition },
+	async () => {
+		await withEnv(
+			{ DATA_BACKEND: undefined, DYNAMODB_TABLE_NAME: "remit-test" },
+			async () => {
+				const ports = await buildDataPortsFromEnv();
+				assert.ok(ports.account, "account port must be defined");
+				assert.ok(ports.threadMessage, "threadMessage port must be defined");
+				assert.equal(
+					ports.resolveAccountId,
+					undefined,
+					"DynamoDB messages already carry a real accountId — no resolution hook",
+				);
+			},
+		);
+	},
+);
+
+test(
+	"without DATA_BACKEND and no DYNAMODB_TABLE_NAME throws",
+	{ skip: !hasDynamoComposition },
+	async () => {
+		await withEnv(
+			{ DATA_BACKEND: undefined, DYNAMODB_TABLE_NAME: undefined },
+			async () => {
+				await assert.rejects(() => buildDataPortsFromEnv());
+			},
+		);
+	},
+);
+
+test("DATA_BACKEND=postgres builds Drizzle ports with a resolveAccountId hook, without connecting", async () => {
+	await withEnv(
+		{
+			DATA_BACKEND: "postgres",
+			PG_CONNECTION_URL: "postgresql://remit:remit@localhost:5432/remit_test",
+		},
+		async () => {
+			const ports = await buildDataPortsFromEnv();
+			assert.ok(ports.account, "account port must be defined");
+			assert.ok(ports.threadMessage, "threadMessage port must be defined");
+			assert.equal(
+				typeof ports.resolveAccountId,
+				"function",
+				"the pg outbox relay carries no accountId, so the consumer must derive it",
+			);
+		},
+	);
+});
+
+test("DATA_BACKEND=postgres without PG_CONNECTION_URL throws", async () => {
+	await withEnv(
+		{ DATA_BACKEND: "postgres", PG_CONNECTION_URL: undefined },
+		async () => {
+			await assert.rejects(() => buildDataPortsFromEnv());
+		},
+	);
+});

package/src/data-ports.ts

@@ -0,0 +1,142 @@
+import type {
+	IAccountRepository,
+	IThreadMessageRepository,
+} from "@remit/data-ports";
+// Type-only: erased at build, so it carries no runtime dependency on
+// drizzle-orm. The value import lives inside `buildPostgresDataPorts` as a
+// dynamic `import()` — see the comment there for why.
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+
+export interface SearchIndexDataPorts {
+	account: IAccountRepository;
+	threadMessage: IThreadMessageRepository;
+	/**
+	 * Resolve the owning accountId for a message whose queue envelope carries no
+	 * accountId of its own. The DynamoDB stream bridge resolves accountId at
+	 * publish time, so every AWS search-index message already carries a real
+	 * one — this hook is `undefined` there, and the handler uses the message's
+	 * own `accountId` unchanged (see `prepareUpsert` in handler.ts).
+	 *
+	 * The Postgres outbox trigger fires from a plain `message_id`, so the pg
+	 * relay (`remit-pg-index-worker`) has no accountId to attach; this hook
+	 * derives it from the message's mailbox at consume time instead. Returns
+	 * null when the mailbox can't be resolved (the message is skipped, not
+	 * retried — see handler.ts).
+	 */
+	resolveAccountId?: (messageId: string) => Promise<string | null>;
+}
+
+// `@remit/drizzle-service` and `drizzle-orm/node-postgres` are loaded
+// lazily, inside this function, instead of as static top-level imports. This
+// branch only ever runs when `DATA_BACKEND === "postgres"` — true for the
+// local Postgres-parity dev stack, never on the deployed Lambda — but a static
+// import is bundled (and evaluated at module load) regardless of whether the
+// branch that uses it ever runs. `@remit/drizzle-service` and
+// `drizzle-orm` are marked `external` for the Lambda esbuild build (see
+// LAMBDA_ESBUILD_OPTIONS), so esbuild leaves this `import()` unresolved in the
+// bundle; it is only ever reached — and only ever needs to resolve — on the
+// Postgres path, which runs via `tsx` (no bundling, real module resolution)
+// and always has both packages installed. Mirrors
+// `packages/backend/src/service/dynamodb.ts`'s `buildPostgresClient`.
+const buildPostgresDataPorts = async (): Promise<SearchIndexDataPorts> => {
+	const pgConnectionUrl = process.env.PG_CONNECTION_URL;
+	if (!pgConnectionUrl) throw new Error("PG_CONNECTION_URL is required");
+
+	const {
+		AccountRepo,
+		DrizzleMessageRepository,
+		DrizzleThreadMessageRepository,
+		MailboxRepo,
+		messageDataSchema,
+	} = await import("@remit/drizzle-service");
+	const { drizzle } = await import("drizzle-orm/node-postgres");
+
+	const db = drizzle(pgConnectionUrl, { schema: messageDataSchema });
+	const genericDb = db as unknown as NodePgDatabase<Record<string, unknown>>;
+	const messageDataDb = db as unknown as NodePgDatabase<
+		typeof messageDataSchema
+	>;
+
+	const message = new DrizzleMessageRepository(messageDataDb);
+	const mailbox = new MailboxRepo(genericDb);
+
+	return {
+		account: new AccountRepo(genericDb),
+		threadMessage: new DrizzleThreadMessageRepository(pgConnectionUrl),
+		resolveAccountId: async (messageId) => {
+			const row = await message.get(messageId);
+			return mailbox.resolveAccountId(row.mailboxId);
+		},
+	};
+};
+
+// The SQLite twin of `buildPostgresDataPorts` (RFC 036): the same Drizzle repos
+// over the one shared SQLite file instead of a Postgres connection string.
+// `createSqliteDatabase` (and better-sqlite3 behind it) is kept external from
+// this worker's DynamoDB Lambda bundle by the same dynamic-import treatment.
+const buildSqliteDataPorts = async (): Promise<SearchIndexDataPorts> => {
+	const sqliteDbPath = process.env.SQLITE_DB_PATH;
+	if (!sqliteDbPath) throw new Error("SQLITE_DB_PATH is required");
+
+	const {
+		AccountRepo,
+		createSqliteDatabase,
+		DrizzleMessageRepository,
+		DrizzleThreadMessageRepository,
+		MailboxRepo,
+		messageDataSchema,
+	} = await import("@remit/drizzle-service");
+
+	const { db } = await createSqliteDatabase(messageDataSchema, {
+		filename: sqliteDbPath,
+	});
+	const genericDb = db as unknown as NodePgDatabase<Record<string, unknown>>;
+	const messageDataDb = db as unknown as NodePgDatabase<
+		typeof messageDataSchema
+	>;
+
+	const message = new DrizzleMessageRepository(messageDataDb);
+	const mailbox = new MailboxRepo(genericDb);
+
+	return {
+		account: new AccountRepo(genericDb),
+		threadMessage: new DrizzleThreadMessageRepository(genericDb),
+		resolveAccountId: async (messageId) => {
+			const row = await message.get(messageId);
+			return mailbox.resolveAccountId(row.mailboxId);
+		},
+	};
+};
+
+/**
+ * Select the account + threadMessage data ports from the environment, mirroring
+ * `buildVectorStoreFromEnv`'s `DATA_BACKEND` selection so one handler serves
+ * the DynamoDB (AWS, production), Postgres (pg-parity), and SQLite
+ * (single-box, RFC 036) stacks:
+ *
+ * - `DATA_BACKEND=postgres` → Drizzle repos over `PG_CONNECTION_URL`.
+ * - `DATA_BACKEND=sqlite` → Drizzle repos over the shared `SQLITE_DB_PATH` file.
+ * - otherwise → ElectroDB services over `DYNAMODB_TABLE_NAME` (unchanged from
+ *   the pre-convergence worker — this is the production path).
+ *
+ * Each branch loads its composition module through a dynamic `import()`, so the
+ * DynamoDB path — the sole importer of the closed `@remit/remit-electrodb-
+ * service` (`./compose-dynamodb.js`) — is never loaded on the relational
+ * (open-core) tree, where that module is not shipped. esbuild still bundles it
+ * into the Lambda because the specifier is a relative import, not an `external`.
+ */
+export const buildDataPortsFromEnv =
+	async (): Promise<SearchIndexDataPorts> => {
+		if (process.env.DATA_BACKEND === "postgres")
+			return buildPostgresDataPorts();
+		if (process.env.DATA_BACKEND === "sqlite") return buildSqliteDataPorts();
+		// `as string` stops tsgo resolving the module in the open-core tree, which
+		// strips it; the named contract keeps the call site typed rather than `any`.
+		type ComposeDynamoDBModule = {
+			buildDynamoDBDataPorts: () => SearchIndexDataPorts;
+		};
+		const { buildDynamoDBDataPorts } = (await import(
+			"./compose-dynamodb.js" as string
+		)) as ComposeDynamoDBModule;
+		return buildDynamoDBDataPorts();
+	};