@remit/search-index-worker 0.0.1

package/src/shutdown.test.ts

@@ -0,0 +1,54 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import type { RunningConsumer } from "./shutdown.js";
+import { runShutdown } from "./shutdown.js";
+
+const nextTick = (): Promise<void> =>
+	new Promise((resolve) => setImmediate(resolve));
+
+describe("runShutdown", () => {
+	test("force-exits with code 1 when stop() never settles", async () => {
+		const hanging: RunningConsumer = {
+			stop: () => new Promise<void>(() => {}),
+		};
+		const codes: number[] = [];
+
+		runShutdown(hanging, {
+			timeoutMs: 5,
+			exit: (code) => codes.push(code),
+		});
+
+		await new Promise((resolve) => setTimeout(resolve, 25));
+		assert.deepStrictEqual(codes, [1], "timeout forces a code-1 exit");
+	});
+
+	test("exits 0 when stop() resolves before the deadline", async () => {
+		const clean: RunningConsumer = { stop: async () => {} };
+		const codes: number[] = [];
+
+		runShutdown(clean, { timeoutMs: 10_000, exit: (code) => codes.push(code) });
+
+		await nextTick();
+		assert.deepStrictEqual(codes, [0], "clean stop exits 0");
+	});
+
+	test("exits 1 and reports when stop() rejects", async () => {
+		const failing: RunningConsumer = {
+			stop: async () => {
+				throw new Error("consumer stop failed");
+			},
+		};
+		const codes: number[] = [];
+		const errors: unknown[] = [];
+
+		runShutdown(failing, {
+			timeoutMs: 10_000,
+			exit: (code) => codes.push(code),
+			onError: (error) => errors.push(error),
+		});
+
+		await nextTick();
+		assert.deepStrictEqual(codes, [1], "a failed stop exits 1");
+		assert.equal(errors.length, 1);
+	});
+});

package/src/shutdown.ts

@@ -0,0 +1,39 @@
+export interface RunningConsumer {
+	stop(): Promise<void>;
+}
+
+export interface ShutdownOptions {
+	timeoutMs: number;
+	exit: (code: number) => void;
+	onError?: (error: unknown) => void;
+}
+
+/**
+ * Race a clean `consumer.stop()` against a hard deadline, then exit regardless.
+ *
+ * `stop()` awaits the SQS long-poll loop; under a saturated event loop that can
+ * hang. A shutdown that never exits leaves the process ignoring SIGTERM and
+ * orphaning to init still burning CPU (issue #1171), so the deadline
+ * force-exits even when `stop()` never settles.
+ */
+export const runShutdown = (
+	consumer: RunningConsumer,
+	options: ShutdownOptions,
+): void => {
+	const { timeoutMs, exit, onError } = options;
+
+	const forceExit = setTimeout(() => exit(1), timeoutMs);
+	forceExit.unref();
+
+	consumer
+		.stop()
+		.then(() => {
+			clearTimeout(forceExit);
+			exit(0);
+		})
+		.catch((error: unknown) => {
+			clearTimeout(forceExit);
+			onError?.(error);
+			exit(1);
+		});
+};

package/src/sqlite-outbox-drain.test.ts

@@ -0,0 +1,90 @@
+import assert from "node:assert/strict";
+import { describe, test } from "node:test";
+import type { SendMessageCommand } from "@aws-sdk/client-sqs";
+import { OutboxRelay } from "@remit/outbox-relay";
+import Database from "better-sqlite3";
+import { SqliteOutboxStore } from "./sqlite-outbox-drain.js";
+
+// The SQLite half of the outbox drain (RFC 036 D2): the raw row access against
+// the shared file. The relay/enqueue logic on top of it is the shared
+// OutboxRelay, already covered in remit-outbox-relay; here a real SQLite outbox
+// table proves the SQL — read undrained rows, mark exactly the drained ids.
+
+const makeOutboxDb = () => {
+	const db = new Database(":memory:");
+	db.exec(`CREATE TABLE outbox (
+		id TEXT PRIMARY KEY,
+		message_id TEXT NOT NULL,
+		event TEXT NOT NULL,
+		payload TEXT NOT NULL,
+		created_at INTEGER NOT NULL,
+		processed_at INTEGER
+	)`);
+	return db;
+};
+
+const insertRow = (
+	db: Database.Database,
+	id: string,
+	messageId: string,
+	event: string,
+) =>
+	db
+		.prepare(
+			`INSERT INTO outbox (id, message_id, event, payload, created_at)
+			 VALUES (?, ?, ?, ?, ?)`,
+		)
+		.run(id, messageId, event, JSON.stringify({ messageId }), Date.now());
+
+const fakeSqs = (sent: string[]) =>
+	({
+		send: async (cmd: SendMessageCommand) => {
+			sent.push(String(cmd.input.MessageBody));
+			return {};
+		},
+	}) as unknown as ConstructorParameters<typeof OutboxRelay>[0]["sqs"];
+
+describe("SqliteOutboxStore", () => {
+	test("drains unprocessed rows and marks them processed", async () => {
+		const db = makeOutboxDb();
+		insertRow(db, "r1", "m1", "message.body_synced");
+		insertRow(db, "r2", "m2", "message.moved");
+		insertRow(db, "r3", "m3", "message.removed");
+
+		const sent: string[] = [];
+		const relay = new OutboxRelay({
+			store: new SqliteOutboxStore(db as unknown as never),
+			sqs: fakeSqs(sent),
+			queueUrl: "q",
+		});
+
+		const count = await relay.drainPending();
+		assert.equal(count, 3);
+		assert.equal(sent.length, 3);
+
+		const stillPending = db
+			.prepare("SELECT count(*) AS n FROM outbox WHERE processed_at IS NULL")
+			.get() as { n: number };
+		assert.equal(stillPending.n, 0, "every relayed row is marked processed");
+
+		// A second pass has nothing left to do.
+		assert.equal(await relay.drainPending(), 0);
+		db.close();
+	});
+
+	test("a row appended after the id capture is not swallowed", async () => {
+		const db = makeOutboxDb();
+		insertRow(db, "r1", "m1", "message.body_synced");
+		const store = new SqliteOutboxStore(db as unknown as never);
+
+		// Capture the pending ids for m1 (mirrors the relay's pre-send capture),
+		// then a second row for the same message+event lands before marking.
+		const captured = await store.listPendingRowIds("m1", "message.body_synced");
+		insertRow(db, "r2", "m1", "message.body_synced");
+		await store.markRowsProcessed(captured);
+
+		const pending = await store.listPendingRowIds("m1", "message.body_synced");
+		assert.deepEqual(pending, ["r2"], "the mid-flight row stays pending");
+		db.close();
+	});
+});

package/src/sqlite-outbox-drain.ts

@@ -0,0 +1,161 @@
+import type { Logger } from "@remit/logger-lambda";
+import {
+	createSqsClient,
+	DRAIN_EVENTS,
+	isForceEvent,
+	isRemoveEvent,
+	OutboxRelay,
+	type OutboxStore,
+	type PendingIndexEvent,
+} from "@remit/outbox-relay";
+
+// The SQLite outbox wake (RFC 036 D2). SQLite has no cross-process NOTIFY, so
+// the durable outbox drain that backstops Postgres becomes the primary
+// mechanism here: a 2-second poll inside the search-index-worker — already
+// resident, already the queue's consuming side — relays undrained rows onto the
+// search-index queue. Against a local file with the outbox's partial index on
+// unprocessed rows the query is microseconds, and a file has none of the
+// scale-to-zero concern that forbade constant polling on Postgres.
+//
+// This process opens its own connection for the drain: it issues only single
+// UPDATE statements in autocommit (no transactions), so it never opens a
+// savepoint and needs no in-process write serialization — cross-process safety
+// with the writers is WAL plus busy_timeout, the same as every other writer.
+
+const DRAIN_INTERVAL_MS = 2_000;
+
+// A minimal view of the better-sqlite3 surface used here, so the module carries
+// no static type dependency on the native package (imported dynamically to stay
+// out of the Lambda bundle).
+interface SqliteStatement {
+	all(...params: unknown[]): unknown[];
+	run(...params: unknown[]): unknown;
+}
+interface SqliteDatabase {
+	pragma(source: string): unknown;
+	prepare(sql: string): SqliteStatement;
+	close(): void;
+}
+
+export class SqliteOutboxStore implements OutboxStore {
+	constructor(private readonly db: SqliteDatabase) {}
+
+	async listUnprocessedEvents(): Promise<PendingIndexEvent[]> {
+		const placeholders = DRAIN_EVENTS.map(() => "?").join(", ");
+		const rows = this.db
+			.prepare(
+				`SELECT DISTINCT message_id, event FROM outbox
+				 WHERE event IN (${placeholders}) AND processed_at IS NULL`,
+			)
+			.all(...DRAIN_EVENTS) as Array<{ message_id: string; event: string }>;
+		return rows.map((row) => ({
+			messageId: row.message_id,
+			event: row.event,
+			force: isForceEvent(row.event),
+			remove: isRemoveEvent(row.event),
+		}));
+	}
+
+	async listPendingRowIds(messageId: string, event: string): Promise<string[]> {
+		const rows = this.db
+			.prepare(
+				`SELECT id FROM outbox
+				 WHERE message_id = ? AND event = ? AND processed_at IS NULL`,
+			)
+			.all(messageId, event) as Array<{ id: string }>;
+		return rows.map((row) => row.id);
+	}
+
+	async markRowsProcessed(ids: string[]): Promise<void> {
+		if (ids.length === 0) return;
+		const placeholders = ids.map(() => "?").join(", ");
+		this.db
+			.prepare(
+				`UPDATE outbox SET processed_at = ?
+				 WHERE id IN (${placeholders}) AND processed_at IS NULL`,
+			)
+			.run(Date.now(), ...ids);
+	}
+}
+
+export interface RunningDrain {
+	stop(): Promise<void>;
+}
+
+export interface SqliteOutboxDrainConfig {
+	dbPath?: string;
+	queueUrl?: string;
+	logger: Logger;
+	intervalMs?: number;
+}
+
+/**
+ * Start the SQLite outbox drain: a boot drain to catch anything pending while
+ * the worker was down, then a poll every `intervalMs` (default 2 s). Overlapping
+ * ticks are skipped so a slow drain never stacks; the timer is unref'd so it
+ * never keeps the process alive on its own.
+ */
+export const startSqliteOutboxDrain = async (
+	config: SqliteOutboxDrainConfig,
+): Promise<RunningDrain> => {
+	const dbPath = config.dbPath ?? process.env.SQLITE_DB_PATH;
+	if (!dbPath) throw new Error("SQLITE_DB_PATH is required");
+	const queueUrl = config.queueUrl ?? process.env.SQS_QUEUE_URL_SEARCH_INDEX;
+	if (!queueUrl) throw new Error("SQS_QUEUE_URL_SEARCH_INDEX is required");
+	const log = config.logger;
+
+	const { default: Database } = await import("better-sqlite3");
+	const sqlite = new Database(dbPath) as unknown as SqliteDatabase;
+	sqlite.pragma("journal_mode = WAL");
+	sqlite.pragma("busy_timeout = 5000");
+	sqlite.pragma("synchronous = NORMAL");
+
+	const sqs = createSqsClient(queueUrl);
+	const relay = new OutboxRelay({
+		store: new SqliteOutboxStore(sqlite),
+		sqs,
+		queueUrl,
+	});
+
+	let draining = false;
+	let inFlight: Promise<unknown> = Promise.resolve();
+	const drainOnce = (): void => {
+		if (draining) return;
+		draining = true;
+		inFlight = relay
+			.drainPending()
+			.catch((error: unknown) =>
+				log.error("sqlite outbox drain failed", { error: String(error) }),
+			)
+			.finally(() => {
+				draining = false;
+			});
+	};
+
+	const bootCount = await relay.drainPending();
+	log.info("sqlite outbox boot drain enqueued", { count: bootCount });
+
+	const timer = setInterval(drainOnce, config.intervalMs ?? DRAIN_INTERVAL_MS);
+	timer.unref();
+
+	return {
+		stop: async () => {
+			clearInterval(timer);
+			await inFlight;
+			sqs.destroy();
+			sqlite.close();
+		},
+	};
+};
+
+/**
+ * Start the drain only on the SQLite backend; a no-op returning `undefined` on
+ * every other backend, so the shared poller entrypoint can call it
+ * unconditionally.
+ */
+export const maybeStartSqliteOutboxDrain = async (
+	logger: Logger,
+): Promise<RunningDrain | undefined> => {
+	if (process.env.DATA_BACKEND !== "sqlite") return undefined;
+	return startSqliteOutboxDrain({ logger });
+};

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"outDir": "dist",
+		"rootDir": "src"
+	},
+	"include": ["src/**/*.ts"]
+}