@remit/search-index-worker 0.0.1

package/src/handler.ts

@@ -0,0 +1,285 @@
+import { inspect } from "node:util";
+import { NotFoundError } from "@remit/data-ports/errors";
+import {
+	createLogger,
+	type Logger,
+	MetricUnit,
+	metrics,
+	withTelemetry,
+} from "@remit/logger-lambda";
+import type { VectorRecord } from "@remit/search-service";
+import type { SQSBatchResponse, SQSEvent, SQSRecord } from "aws-lambda";
+import { type ParsedQueueMessage, parseQueueMessage } from "./parse.js";
+import { getServices, type Services } from "./services.js";
+
+const log = createLogger();
+
+export const handler = withTelemetry(
+	async (event: SQSEvent): Promise<SQSBatchResponse> => {
+		const services = await getServices();
+		return processBatch(event.Records, services, log);
+	},
+);
+
+/**
+ * A single message's indexing outcome — the pg-only work-summary signal
+ * (`remit-search-index-worker/consumer.ts`'s periodic noop/deferred/dropped
+ * log) hooks in via `Services.onIndexOutcome`, which is `undefined` on the
+ * Lambda path and so never fires there. Purely observational: nothing here
+ * feeds back into what gets logged, metriced, or returned to SQS.
+ */
+export type IndexOutcome =
+	| { status: "indexed"; upserted: number; skipped: number }
+	| { status: "skipped"; reason: string; retryable: boolean };
+
+export const processBatch = async (
+	records: SQSRecord[],
+	services: Services,
+	log: Logger,
+): Promise<SQSBatchResponse> => {
+	const batchItemFailures: { itemIdentifier: string }[] = [];
+	const processingStart = Date.now();
+
+	for (const record of records) {
+		const message = parseQueueMessage(record.body);
+
+		const failed =
+			message.kind === "delete"
+				? await deleteMessage(message, services, log)
+				: await upsertMessage(message, services, log);
+
+		if (failed) {
+			batchItemFailures.push({ itemIdentifier: record.messageId });
+		}
+	}
+
+	metrics.addMetric(
+		"searchIndexProcessingDuration",
+		MetricUnit.Milliseconds,
+		Date.now() - processingStart,
+	);
+
+	return { batchItemFailures };
+};
+
+const deleteMessage = async (
+	message: Extract<ParsedQueueMessage, { kind: "delete" }>,
+	services: Services,
+	log: Logger,
+): Promise<boolean> =>
+	services.searchService
+		.delete(message.messageId)
+		.then(() => {
+			log.info("Deleted search vectors", { messageId: message.messageId });
+			metrics.addMetric("searchIndexProcessed", MetricUnit.Count, 1);
+			return false;
+		})
+		.catch((error) => {
+			log.error("Delete failed", {
+				error: inspect(error),
+				messageId: message.messageId,
+			});
+			metrics.addMetric("searchIndexFailures", MetricUnit.Count, 1);
+			return true;
+		});
+
+// One SQS message = one email's chunks = one upsert. Isolating the
+// upsert per message means a record S3 Vectors rejects (e.g. a metadata
+// ValidationException) dead-letters on its own — its siblings in the
+// batch still index instead of retrying forever behind a poison record.
+// Exported so the long-running Postgres consumer (`consumer.ts`) and the
+// bulk reindex script (`reindex.ts`) can process one message at a time
+// outside the Lambda batch shape, reusing this exact logic.
+export const upsertMessage = async (
+	message: Extract<ParsedQueueMessage, { kind: "upsert" }>,
+	services: Services,
+	log: Logger,
+): Promise<boolean> =>
+	indexMessage(message, services, log)
+		.then(() => false)
+		.catch((error) => {
+			// Name the messageId so the dead-letter is diagnosable per message
+			// (#910); a transient/whole-call error retries this message alone.
+			log.error("Upsert failed", {
+				error: inspect(error),
+				accountId: message.accountId,
+				messageId: message.messageId,
+			});
+			metrics.addMetric("searchIndexFailures", MetricUnit.Count, 1);
+			return true;
+		});
+
+const indexMessage = async (
+	message: Extract<ParsedQueueMessage, { kind: "upsert" }>,
+	services: Services,
+	log: Logger,
+): Promise<void> => {
+	const vectorRecords = await prepareUpsert(
+		message.accountId,
+		message.messageId,
+		services,
+		log,
+	);
+	if (vectorRecords === null) return;
+	if (vectorRecords.length === 0) {
+		log.info("No indexable content, skipping", {
+			messageId: message.messageId,
+		});
+		services.onIndexOutcome?.({
+			status: "skipped",
+			reason: "no-indexable-content",
+			retryable: false,
+		});
+		return;
+	}
+
+	// Dedup by chunkId within this single message. The same email can
+	// emit a repeated chunkId (deterministic keys); duplicate keys in one
+	// PutVectors call are rejected by S3 Vectors.
+	const deduped = new Map<string, VectorRecord>();
+	for (const vectorRecord of vectorRecords) {
+		deduped.set(vectorRecord.chunkId, vectorRecord);
+	}
+	const upsertRecords = [...deduped.values()];
+
+	const { upserted, skipped } = await services.searchService.upsertVectors(
+		upsertRecords,
+		{ force: message.force },
+	);
+	log.info("Upsert complete", {
+		accountId: message.accountId,
+		messageId: message.messageId,
+		upserted,
+		skipped,
+	});
+	metrics.addMetric("searchIndexProcessed", MetricUnit.Count, upserted);
+	metrics.addMetric("searchIndexSkipped", MetricUnit.Count, skipped);
+	services.onIndexOutcome?.({ status: "indexed", upserted, skipped });
+};
+
+const prepareUpsert = async (
+	accountIdFromMessage: string,
+	messageId: string,
+	services: Services,
+	log: Logger,
+): Promise<VectorRecord[] | null> => {
+	const {
+		accountService,
+		threadMessageService,
+		storageService,
+		searchService,
+		resolveAccountId,
+	} = services;
+
+	// On DynamoDB `resolveAccountId` is undefined (the stream bridge already
+	// resolved and attached a real accountId to the queue message), so this is
+	// exactly `accountIdFromMessage` — unchanged from before this hook existed.
+	// On Postgres the queue message carries no real accountId (the outbox
+	// trigger fires from a bare message id), so this derives it from the
+	// message's mailbox instead (see `data-ports.ts`).
+	const accountId = resolveAccountId
+		? await resolveAccountId(messageId)
+		: accountIdFromMessage;
+	if (!accountId) {
+		log.info("Account not found for message, skipping", { messageId });
+		services.onIndexOutcome?.({
+			status: "skipped",
+			reason: "account-not-found",
+			retryable: false,
+		});
+		return null;
+	}
+
+	let accountConfigId: string;
+	try {
+		const account = await accountService.get(accountId);
+		if (account.deletedAt) {
+			log.info("Account deleted, skipping", {
+				accountId,
+				messageId,
+				deletedAt: account.deletedAt,
+			});
+			services.onIndexOutcome?.({
+				status: "skipped",
+				reason: "account-deleted",
+				retryable: false,
+			});
+			return null;
+		}
+		accountConfigId = account.accountConfigId;
+	} catch (error) {
+		// Only a genuine missing account is skippable. AccessDenied, throttling,
+		// and network errors must surface so the record retries (and reaches the
+		// DLQ) instead of being silently dropped as if it were a 404.
+		if (error instanceof NotFoundError) {
+			log.info("Account not found, skipping", { accountId, messageId });
+			services.onIndexOutcome?.({
+				status: "skipped",
+				reason: "account-not-found",
+				retryable: false,
+			});
+			return null;
+		}
+		throw error;
+	}
+
+	const threadMessage = await threadMessageService.findByMessageId(
+		accountConfigId,
+		messageId,
+	);
+	if (!threadMessage) {
+		log.info("ThreadMessage not found, skipping", { messageId });
+		services.onIndexOutcome?.({
+			status: "skipped",
+			reason: "thread-message-not-found",
+			retryable: true,
+		});
+		return null;
+	}
+
+	const parsedBody = await storageService.retrieveParsedBody(
+		threadMessage.accountConfigId,
+		accountId,
+		messageId,
+	);
+	if (!parsedBody) {
+		log.info("Parsed body not found in S3, skipping", { messageId });
+		services.onIndexOutcome?.({
+			status: "skipped",
+			reason: "parsed-body-not-found",
+			retryable: true,
+		});
+		return null;
+	}
+
+	return searchService.prepareVectors({
+		envelope: {
+			from: {
+				name: threadMessage.fromName ?? null,
+				email: threadMessage.fromEmail ?? "",
+			},
+			to: [],
+			cc: [],
+			bcc: [],
+			subject: threadMessage.subject ?? "",
+			attachments: [],
+		},
+		parsedBody: {
+			text: parsedBody.text,
+			html: parsedBody.html,
+		},
+		metadata: {
+			messageId,
+			threadId: threadMessage.threadId,
+			accountConfigId: threadMessage.accountConfigId,
+			mailboxIds: [threadMessage.mailboxId],
+			sentDate: threadMessage.sentDate,
+			isRead: threadMessage.isRead,
+			hasAttachment: threadMessage.hasAttachment,
+			hasStars: threadMessage.hasStars,
+			fromName: threadMessage.fromName ?? null,
+			subject: threadMessage.subject ?? "",
+			category: threadMessage.category,
+		},
+	});
+};

package/src/index-stats.test.ts

@@ -0,0 +1,56 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import { createIndexWorkStats } from "./index-stats.js";
+
+describe("createIndexWorkStats", () => {
+	test("counts an upserted:0 indexed outcome as noop (the over-trigger signal)", () => {
+		const stats = createIndexWorkStats();
+		stats.record({ status: "indexed", upserted: 0, skipped: 4 }, false);
+		stats.record({ status: "indexed", upserted: 2, skipped: 1 }, false);
+		assert.deepStrictEqual(stats.drain(), {
+			processed: 2,
+			embedded: 1,
+			noop: 1,
+			deferred: 0,
+			dropped: 0,
+			forced: 0,
+		});
+	});
+
+	test("separates transient (deferred) from terminal (dropped) skips", () => {
+		const stats = createIndexWorkStats();
+		stats.record(
+			{ status: "skipped", reason: "parsed-body-not-found", retryable: true },
+			false,
+		);
+		stats.record(
+			{ status: "skipped", reason: "no-indexable-content", retryable: false },
+			false,
+		);
+		assert.deepStrictEqual(stats.drain(), {
+			processed: 2,
+			embedded: 0,
+			noop: 0,
+			deferred: 1,
+			dropped: 1,
+			forced: 0,
+		});
+	});
+
+	test("counts force re-indexes (moves) separately", () => {
+		const stats = createIndexWorkStats();
+		stats.record({ status: "indexed", upserted: 3, skipped: 0 }, true);
+		stats.record({ status: "indexed", upserted: 1, skipped: 0 }, false);
+		const summary = stats.drain();
+		assert.equal(summary?.forced, 1);
+		assert.equal(summary?.processed, 2);
+	});
+
+	test("drain resets the window and returns null when empty", () => {
+		const stats = createIndexWorkStats();
+		assert.equal(stats.drain(), null, "nothing recorded yet");
+		stats.record({ status: "indexed", upserted: 1, skipped: 0 }, false);
+		assert.ok(stats.drain());
+		assert.equal(stats.drain(), null, "drained window is empty again");
+	});
+});

package/src/index-stats.ts

@@ -0,0 +1,62 @@
+import type { IndexOutcome } from "./handler.js";
+
+/**
+ * A window of index outcomes. `noop` is the proxy metric for over-triggering:
+ * an indexed message that embedded nothing (`upserted: 0`) means an event fired
+ * for content already in the store. A high noop rate is the signal that error
+ * and DLQ alarms can't see — "too much successful work" — and is what let the
+ * AWS search-index cost blowup ramp unnoticed (#1082). Pg-only: the Lambda path
+ * never wires `Services.onIndexOutcome`, so this only accumulates in the
+ * long-running Postgres consumer (`consumer.ts`).
+ */
+export interface IndexWorkSummary {
+	processed: number;
+	/** Indexed and wrote vectors — real work. */
+	embedded: number;
+	/** Indexed but wrote nothing (`upserted: 0`) — an event for unchanged content. */
+	noop: number;
+	/** Transient skip (thread/body not visible yet); left undrained for retry. */
+	deferred: number;
+	/** Terminal skip; drained, will never index. */
+	dropped: number;
+	/** Of the above, how many were force re-indexes (moves) — always re-embedded. */
+	forced: number;
+}
+
+export interface IndexWorkStats {
+	record(outcome: IndexOutcome, force: boolean): void;
+	/** Return the accumulated window and reset it; null if nothing was recorded. */
+	drain(): IndexWorkSummary | null;
+}
+
+const empty = (): IndexWorkSummary => ({
+	processed: 0,
+	embedded: 0,
+	noop: 0,
+	deferred: 0,
+	dropped: 0,
+	forced: 0,
+});
+
+export const createIndexWorkStats = (): IndexWorkStats => {
+	let window = empty();
+	return {
+		record: (outcome, force) => {
+			window.processed += 1;
+			if (force) window.forced += 1;
+			if (outcome.status === "indexed") {
+				if (outcome.upserted > 0) window.embedded += 1;
+				else window.noop += 1;
+				return;
+			}
+			if (outcome.retryable) window.deferred += 1;
+			else window.dropped += 1;
+		},
+		drain: () => {
+			if (window.processed === 0) return null;
+			const summary = window;
+			window = empty();
+			return summary;
+		},
+	};
+};

package/src/index.ts

@@ -0,0 +1,2 @@
+export { handler } from "./handler.js";
+export type { SearchIndexMessage } from "./search-index-message.js";

package/src/parse.ts

@@ -0,0 +1,19 @@
+import { searchIndexMessageSchema } from "@remit/search-service";
+
+export type ParsedQueueMessage =
+	| { kind: "upsert"; accountId: string; messageId: string; force: boolean }
+	| { kind: "delete"; messageId: string };
+
+export const parseQueueMessage = (body: string): ParsedQueueMessage => {
+	const parsed = searchIndexMessageSchema.parse(JSON.parse(body));
+
+	if (parsed.eventName === "REMOVE") {
+		return { kind: "delete", messageId: parsed.messageId };
+	}
+	return {
+		kind: "upsert",
+		accountId: parsed.accountId,
+		messageId: parsed.messageId,
+		force: parsed.force ?? false,
+	};
+};

package/src/poller.ts

@@ -0,0 +1,31 @@
+import { createLogger } from "@remit/logger-lambda";
+import { runQueuePoller } from "@remit/sqs-client/poller";
+import { env } from "expect-env";
+import { handler } from "./index.js";
+import { maybeStartSqliteOutboxDrain } from "./sqlite-outbox-drain.js";
+
+/** Production queue poller — no e2e shim exists for this queue today (the
+ * e2e/CI stack piggybacks it onto the imap-worker shim); this is the
+ * standalone production entrypoint for the dedicated image. */
+const log = createLogger();
+
+// On the SQLite backend (RFC 036 D2) this container also owns the outbox wake:
+// a 2-second poll relaying committed rows onto the search-index queue this same
+// process consumes. A no-op on every other backend. Started before the poll
+// loop, which blocks until shutdown, then stopped after it returns.
+const drain = await maybeStartSqliteOutboxDrain(log);
+
+try {
+	await runQueuePoller({
+		log,
+		targets: [
+			{
+				queueUrl: env.SQS_QUEUE_URL_SEARCH_INDEX,
+				handler,
+				functionName: "search-index-worker",
+			},
+		],
+	});
+} finally {
+	await drain?.stop();
+}

package/src/reindex.ts

@@ -0,0 +1,63 @@
+import type { Logger } from "@remit/logger-lambda";
+import pMap from "p-map";
+import type { Pool } from "pg";
+import { type IndexOutcome, upsertMessage } from "./handler.js";
+import type { Services } from "./services.js";
+
+export interface ReindexResult {
+	total: number;
+	indexed: number;
+	skipped: number;
+}
+
+const NOT_APPLICABLE_ACCOUNT_ID = "reindex-all";
+
+/**
+ * Re-embed every body-synced Postgres message. Concurrent (pMap) and
+ * force-upserting so a model change or a repair repopulates the whole store;
+ * keys-only scan keeps the hot path off `describe()`. Postgres-only — the
+ * `accountId` on the synthetic upsert message is a placeholder: `services`
+ * must carry `resolveAccountId` (true whenever `DATA_BACKEND=postgres`; see
+ * `data-ports.ts`), which derives the real one from each message's mailbox.
+ */
+export const reindexAll = async (
+	pool: Pool,
+	services: Services,
+	log: Logger,
+	options?: { concurrency?: number },
+): Promise<ReindexResult> => {
+	const rows = await pool.query<{ message_id: string }>(
+		"SELECT message_id FROM message WHERE body_storage_key IS NOT NULL",
+	);
+	const messageIds = rows.rows.map((row) => row.message_id);
+
+	let indexed = 0;
+	let skipped = 0;
+	await pMap(
+		messageIds,
+		async (messageId) => {
+			let outcome: IndexOutcome | undefined;
+			const taskServices: Services = {
+				...services,
+				onIndexOutcome: (o) => {
+					outcome = o;
+				},
+			};
+			await upsertMessage(
+				{
+					kind: "upsert",
+					accountId: NOT_APPLICABLE_ACCOUNT_ID,
+					messageId,
+					force: true,
+				},
+				taskServices,
+				log,
+			);
+			if (outcome?.status === "indexed") indexed += 1;
+			else skipped += 1;
+		},
+		{ concurrency: options?.concurrency ?? 8 },
+	);
+
+	return { total: messageIds.length, indexed, skipped };
+};

package/src/run-reindex.ts

@@ -0,0 +1,22 @@
+import { createLogger } from "@remit/logger-lambda";
+import pg from "pg";
+import { reindexAll } from "./reindex.js";
+import { getServices } from "./services.js";
+
+const log = createLogger();
+
+const main = async (): Promise<void> => {
+	const connectionString = process.env.PG_CONNECTION_URL;
+	if (!connectionString) throw new Error("PG_CONNECTION_URL is required");
+
+	const pool = new pg.Pool({ connectionString });
+	const services = await getServices();
+
+	const result = await reindexAll(pool, services, log);
+	log.info("reindex complete", { ...result });
+
+	await pool.end();
+	process.exit(0);
+};
+
+await main();

package/src/run-worker.ts

@@ -0,0 +1,26 @@
+import { createLogger } from "@remit/logger-lambda";
+import { startSqsConsumer } from "./consumer.js";
+import { getServices } from "./services.js";
+import { runShutdown } from "./shutdown.js";
+
+const SHUTDOWN_TIMEOUT_MS = 10_000;
+const log = createLogger();
+
+const main = async (): Promise<void> => {
+	const services = await getServices();
+	const consumer = startSqsConsumer({ services, logger: log });
+
+	const shutdown = (): void =>
+		runShutdown(consumer, {
+			timeoutMs: SHUTDOWN_TIMEOUT_MS,
+			exit: (code) => process.exit(code),
+			onError: (error) =>
+				log.error("shutdown failed", { error: String(error) }),
+		});
+	process.on("SIGINT", shutdown);
+	process.on("SIGTERM", shutdown);
+
+	log.info("search-index-worker consumer started");
+};
+
+await main();

package/src/search-index-message.ts

@@ -0,0 +1 @@
+export type { SearchIndexMessage } from "@remit/search-service";

package/src/services.ts

@@ -0,0 +1,81 @@
+import { createSearchService, type SearchService } from "@remit/search-service";
+import {
+	buildEmbeddingServiceFromEnv,
+	buildVectorStoreFromEnv,
+} from "@remit/search-service/from-env";
+import type { StorageService } from "@remit/storage-service";
+import { createStorageService } from "@remit/storage-service/s3";
+import {
+	buildDataPortsFromEnv,
+	type SearchIndexDataPorts,
+} from "./data-ports.js";
+import type { IndexOutcome } from "./handler.js";
+
+export interface Services {
+	accountService: SearchIndexDataPorts["account"];
+	threadMessageService: SearchIndexDataPorts["threadMessage"];
+	storageService: StorageService;
+	searchService: SearchService;
+	resolveAccountId?: SearchIndexDataPorts["resolveAccountId"];
+	/**
+	 * Fired once per upsert outcome — the pg-only work-summary signal
+	 * (`consumer.ts` wires this to `IndexWorkStats`). `undefined` on the Lambda
+	 * path, where it never fires and so never affects behavior.
+	 */
+	onIndexOutcome?: (outcome: IndexOutcome) => void;
+}
+
+let cached: Services | undefined;
+
+export const getServices = async (): Promise<Services> => {
+	if (cached) return cached;
+
+	const dataPorts = await buildDataPortsFromEnv();
+
+	const storageService = createStorageService();
+
+	// The worker must have a durable vector store — a typo'd or missing S3 (or,
+	// on Postgres, PG_CONNECTION_URL) env var must not silently succeed by
+	// falling back to the throwaway in-memory store (which emits success
+	// metrics but drops every vector).
+	const isPostgres = process.env.DATA_BACKEND === "postgres";
+	const pgConnectionUrl = process.env.PG_CONNECTION_URL;
+	const localPath = process.env.LOCAL_VECTORDB_PATH;
+	const bucket = process.env.S3_VECTORS_BUCKET_NAME;
+	const indexName = process.env.S3_VECTORS_INDEX_NAME;
+	if (
+		!(isPostgres && pgConnectionUrl) &&
+		!localPath &&
+		!(bucket && indexName)
+	) {
+		throw new Error(
+			"Vector store is not configured: set PG_CONNECTION_URL for the Postgres " +
+				"backend, LOCAL_VECTORDB_PATH for local dev, or both " +
+				"S3_VECTORS_BUCKET_NAME and S3_VECTORS_INDEX_NAME for production.",
+		);
+	}
+
+	// Build the embedder first so we can pass its dimension count to the
+	// sqlite-vec store — the vec0 table dimension must match the embedder.
+	const embedder = buildEmbeddingServiceFromEnv();
+	const searchService = createSearchService({
+		store: buildVectorStoreFromEnv(embedder.dimensions),
+		embedder,
+	});
+
+	cached = {
+		accountService: dataPorts.account,
+		threadMessageService: dataPorts.threadMessage,
+		resolveAccountId: dataPorts.resolveAccountId,
+		storageService,
+		searchService,
+	};
+	return cached;
+};
+
+export const createServices = (overrides: Services): Services => overrides;
+
+/** Reset the singleton — test use only. */
+export const _resetForTest = (): void => {
+	cached = undefined;
+};