@remit/search-index-worker 0.0.15 → 0.0.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-index-worker",
-  "version": "0.0.15",
+  "version": "0.0.17",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -21,7 +21,6 @@
   "dependencies": {
     "@remit/outbox-relay": "*",
     "p-map": "*",
-    "pg": "^8.0.0",
     "@remit/logger-lambda": "*",
     "@remit/sqs-client": "*",
     "@remit/data-ports": "*",
@@ -30,10 +29,8 @@
     "@remit/storage-service": "*",
     "@aws-sdk/client-sqs": "*",
     "better-sqlite3": "^12.11.1",
-    "drizzle-orm": "^0.45.2",
     "expect-env": "*",
     "@types/aws-lambda": "*",
-    "@types/pg": "^8.0.0",
     "prom-client": "^15.1.3"
   },
   "devDependencies": {

package/src/consumer.ts

@@ -50,10 +50,9 @@ export interface SqsConsumerConfig {
  * via `processBatch` (a batch of one) — the same per-message logic the AWS
  * Lambda handler runs, reused so the two deployment shapes (Lambda event
  * source mapping vs. a long-running container/pm2 process) share one
- * indexing implementation. Used by the Postgres-parity stack, where the
- * search-index queue has no Lambda event source; `remit-pg-index-worker`
- * only relays committed outbox events onto this queue (the producer side —
- * see its `worker.ts`), it does not consume them.
+ * indexing implementation. Used by the self-host stack, where the search-index
+ * queue has no Lambda event source: the outbox drain relays committed events
+ * onto this queue (the producer side) and this consumer takes them off it.
  *
  * A message is deleted only when `processBatch` reports no failure for it;
  * a failure leaves it on the queue so its visibility timeout lapses and SQS

package/src/data-ports.test.ts

@@ -1,5 +1,8 @@
 import assert from "node:assert/strict";
-import { afterEach, test } from "node:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
 import {
 	buildDataPortsFromEnv,
 	type SearchIndexDataPorts,
@@ -26,55 +29,41 @@ const withEnv = async (
 	}
 };
 
-afterEach(() => {
-	delete process.env.DATA_BACKEND;
-});
-
-test("without DATA_BACKEND and no registered ports, throws (DynamoDB path is injected)", async () => {
-	await withEnv({ DATA_BACKEND: undefined }, async () => {
+test("with neither a registration nor a database, the error names setSearchIndexDataPorts", async () => {
+	await withEnv({ SQLITE_DB_PATH: undefined }, async () => {
 		await assert.rejects(
 			() => buildDataPortsFromEnv(),
-			/no DynamoDB search-index data ports registered/,
+			/no search-index data ports registered/,
 		);
 	});
 });
 
-test("without DATA_BACKEND returns the injected DynamoDB ports", async () => {
+test("SQLITE_DB_PATH builds Drizzle ports with a resolveAccountId hook", async () => {
+	const dir = mkdtempSync(join(tmpdir(), "remit-search-ports-"));
+	await withEnv({ SQLITE_DB_PATH: join(dir, "remit.db") }, async () => {
+		const ports = await buildDataPortsFromEnv();
+		assert.ok(ports.account, "account port must be defined");
+		assert.ok(ports.threadMessage, "threadMessage port must be defined");
+		assert.equal(
+			typeof ports.resolveAccountId,
+			"function",
+			"the outbox relay carries no accountId, so the consumer must derive it",
+		);
+	});
+	rmSync(dir, { recursive: true, force: true });
+});
+
+// Last: the registration is process-wide and takes precedence over everything
+// below it, so a test registering ports cannot run before the ones that must
+// reach the fallback.
+test("a registered set of ports is returned without reading the environment", async () => {
 	const injected = {
 		account: {},
 		threadMessage: {},
 	} as unknown as SearchIndexDataPorts;
 	setSearchIndexDataPorts(injected);
-	await withEnv({ DATA_BACKEND: undefined }, async () => {
+	await withEnv({ SQLITE_DB_PATH: undefined }, async () => {
 		const ports = await buildDataPortsFromEnv();
 		assert.equal(ports, injected);
 	});
 });
-
-test("DATA_BACKEND=postgres builds Drizzle ports with a resolveAccountId hook, without connecting", async () => {
-	await withEnv(
-		{
-			DATA_BACKEND: "postgres",
-			PG_CONNECTION_URL: "postgresql://remit:remit@localhost:5432/remit_test",
-		},
-		async () => {
-			const ports = await buildDataPortsFromEnv();
-			assert.ok(ports.account, "account port must be defined");
-			assert.ok(ports.threadMessage, "threadMessage port must be defined");
-			assert.equal(
-				typeof ports.resolveAccountId,
-				"function",
-				"the pg outbox relay carries no accountId, so the consumer must derive it",
-			);
-		},
-	);
-});
-
-test("DATA_BACKEND=postgres without PG_CONNECTION_URL throws", async () => {
-	await withEnv(
-		{ DATA_BACKEND: "postgres", PG_CONNECTION_URL: undefined },
-		async () => {
-			await assert.rejects(() => buildDataPortsFromEnv());
-		},
-	);
-});

package/src/data-ports.ts

@@ -2,10 +2,6 @@ import type {
 	IAccountRepository,
 	IThreadMessageRepository,
 } from "@remit/data-ports";
-// Type-only: erased at build, so it carries no runtime dependency on
-// drizzle-orm. The value import lives inside `buildPostgresDataPorts` as a
-// dynamic `import()` — see the comment there for why.
-import type { NodePgDatabase } from "drizzle-orm/node-postgres";
 
 export interface SearchIndexDataPorts {
 	account: IAccountRepository;
@@ -17,63 +13,21 @@ export interface SearchIndexDataPorts {
 	 * one — this hook is `undefined` there, and the handler uses the message's
 	 * own `accountId` unchanged (see `prepareUpsert` in handler.ts).
 	 *
-	 * The Postgres outbox trigger fires from a plain `message_id`, so the pg
-	 * relay (`remit-pg-index-worker`) has no accountId to attach; this hook
-	 * derives it from the message's mailbox at consume time instead. Returns
-	 * null when the mailbox can't be resolved (the message is skipped, not
-	 * retried — see handler.ts).
+	 * The outbox carries a plain `message_id`, so the relational drain has no
+	 * accountId to attach; this hook derives it from the message's mailbox at
+	 * consume time instead. Returns null when the mailbox can't be resolved (the
+	 * message is skipped, not retried — see handler.ts).
 	 */
 	resolveAccountId?: (messageId: string) => Promise<string | null>;
 }
 
-// `@remit/drizzle-service` and `drizzle-orm/node-postgres` are loaded
-// lazily, inside this function, instead of as static top-level imports. This
-// branch only ever runs when `DATA_BACKEND === "postgres"` — true for the
-// local Postgres-parity dev stack, never on the deployed Lambda — but a static
-// import is bundled (and evaluated at module load) regardless of whether the
-// branch that uses it ever runs. `@remit/drizzle-service` and
-// `drizzle-orm` are marked `external` for the Lambda esbuild build (see
-// LAMBDA_ESBUILD_OPTIONS), so esbuild leaves this `import()` unresolved in the
-// bundle; it is only ever reached — and only ever needs to resolve — on the
-// Postgres path, which runs via `tsx` (no bundling, real module resolution)
-// and always has both packages installed. Mirrors
-// `packages/backend/src/service/dynamodb.ts`'s `buildPostgresClient`.
-const buildPostgresDataPorts = async (): Promise<SearchIndexDataPorts> => {
-	const pgConnectionUrl = process.env.PG_CONNECTION_URL;
-	if (!pgConnectionUrl) throw new Error("PG_CONNECTION_URL is required");
-
-	const {
-		AccountRepo,
-		DrizzleMessageRepository,
-		DrizzleThreadMessageRepository,
-		MailboxRepo,
-		messageDataSchema,
-	} = await import("@remit/drizzle-service");
-	const { drizzle } = await import("drizzle-orm/node-postgres");
-
-	const db = drizzle(pgConnectionUrl, { schema: messageDataSchema });
-	const genericDb = db as unknown as NodePgDatabase<Record<string, unknown>>;
-	const messageDataDb = db as unknown as NodePgDatabase<
-		typeof messageDataSchema
-	>;
-
-	const message = new DrizzleMessageRepository(messageDataDb);
-	const mailbox = new MailboxRepo(genericDb);
-
-	return {
-		account: new AccountRepo(genericDb),
-		threadMessage: new DrizzleThreadMessageRepository(pgConnectionUrl),
-		resolveAccountId: async (messageId) => {
-			const row = await message.get(messageId);
-			return mailbox.resolveAccountId(row.mailboxId);
-		},
-	};
-};
-
-// The SQLite twin of `buildPostgresDataPorts` (RFC 036): the same Drizzle repos
-// over the one shared SQLite file instead of a Postgres connection string.
-// `createSqliteDatabase` (and better-sqlite3 behind it) is kept external from
-// this worker's DynamoDB Lambda bundle by the same dynamic-import treatment.
+// `@remit/drizzle-service` is loaded lazily, inside this function, instead of as
+// a static top-level import. A static import is bundled (and evaluated at module
+// load) regardless of whether the branch that uses it ever runs; the package is
+// marked `external` for the Lambda esbuild build (see LAMBDA_ESBUILD_OPTIONS),
+// so esbuild leaves this `import()` unresolved in the bundle. It resolves only
+// where the relational composition actually runs, which has the package
+// installed. Mirrors `packages/backend/src/service/data-client.ts`.
 const buildSqliteDataPorts = async (): Promise<SearchIndexDataPorts> => {
 	const sqliteDbPath = process.env.SQLITE_DB_PATH;
 	if (!sqliteDbPath) throw new Error("SQLITE_DB_PATH is required");
@@ -90,17 +44,13 @@ const buildSqliteDataPorts = async (): Promise<SearchIndexDataPorts> => {
 	const { db } = await createSqliteDatabase(messageDataSchema, {
 		filename: sqliteDbPath,
 	});
-	const genericDb = db as unknown as NodePgDatabase<Record<string, unknown>>;
-	const messageDataDb = db as unknown as NodePgDatabase<
-		typeof messageDataSchema
-	>;
 
-	const message = new DrizzleMessageRepository(messageDataDb);
-	const mailbox = new MailboxRepo(genericDb);
+	const message = new DrizzleMessageRepository(db);
+	const mailbox = new MailboxRepo(db);
 
 	return {
-		account: new AccountRepo(genericDb),
-		threadMessage: new DrizzleThreadMessageRepository(genericDb),
+		account: new AccountRepo(db),
+		threadMessage: new DrizzleThreadMessageRepository(db),
 		resolveAccountId: async (messageId) => {
 			const row = await message.get(messageId);
 			return mailbox.resolveAccountId(row.mailboxId);
@@ -108,40 +58,28 @@ const buildSqliteDataPorts = async (): Promise<SearchIndexDataPorts> => {
 	};
 };
 
-/**
- * Select the account + threadMessage data ports from the environment, mirroring
- * `buildVectorStoreFromEnv`'s `DATA_BACKEND` selection so one handler serves
- * the DynamoDB (AWS, production), Postgres (pg-parity), and SQLite
- * (single-box, RFC 036) stacks:
- *
- * - `DATA_BACKEND=postgres` → Drizzle repos over `PG_CONNECTION_URL`.
- * - `DATA_BACKEND=sqlite` → Drizzle repos over the shared `SQLITE_DB_PATH` file.
- * - otherwise → ElectroDB services over `DYNAMODB_TABLE_NAME` (unchanged from
- *   the pre-convergence worker — this is the production path).
- *
- * The DynamoDB ports are injected by the composition root, which lives outside
- * this shared, open-core module and is never imported here.
- */
 let injectedDataPorts: SearchIndexDataPorts | null = null;
 
 /**
  * Register the DynamoDB-backed search-index data ports from the composition
- * root. The relational backends compose in-package above and never touch this
+ * root. The relational backend composes in-package above and never touches this
  * seam.
  */
 export const setSearchIndexDataPorts = (ports: SearchIndexDataPorts): void => {
 	injectedDataPorts = ports;
 };
 
+/**
+ * The search-index data ports for this process: the ones a composition root
+ * registered, or — for a process that registered none — the SQLite composition
+ * this build contains. `SQLITE_DB_PATH` is that composition's precondition, not
+ * a backend selection.
+ */
 export const buildDataPortsFromEnv =
 	async (): Promise<SearchIndexDataPorts> => {
-		if (process.env.DATA_BACKEND === "postgres")
-			return buildPostgresDataPorts();
-		if (process.env.DATA_BACKEND === "sqlite") return buildSqliteDataPorts();
-		if (!injectedDataPorts) {
-			throw new Error(
-				"no DynamoDB search-index data ports registered — register them with setSearchIndexDataPorts() from your composition root",
-			);
-		}
-		return injectedDataPorts;
+		if (injectedDataPorts) return injectedDataPorts;
+		if (process.env.SQLITE_DB_PATH) return buildSqliteDataPorts();
+		throw new Error(
+			"no search-index data ports registered — register them with setSearchIndexDataPorts() from your composition root",
+		);
 	};

package/src/handler.ts

@@ -86,9 +86,8 @@ const deleteMessage = async (
 // upsert per message means a record S3 Vectors rejects (e.g. a metadata
 // ValidationException) dead-letters on its own — its siblings in the
 // batch still index instead of retrying forever behind a poison record.
-// Exported so the long-running Postgres consumer (`consumer.ts`) can
-// process one message at a time outside the Lambda batch shape, reusing
-// this exact logic.
+// Exported so the long-running consumer (`consumer.ts`) can process one
+// message at a time outside the Lambda batch shape, reusing this exact logic.
 export const upsertMessage = async (
 	message: Extract<ParsedQueueMessage, { kind: "upsert" }>,
 	services: Services,

package/src/services.ts

@@ -18,7 +18,7 @@ export interface Services {
 	searchService: SearchService;
 	resolveAccountId?: SearchIndexDataPorts["resolveAccountId"];
 	/**
-	 * Fired once per upsert outcome — the pg-only work-summary signal
+	 * Fired once per upsert outcome — the relational work-summary signal
 	 * (`consumer.ts` wires this to `IndexWorkStats`). `undefined` on the Lambda
 	 * path, where it never fires and so never affects behavior.
 	 */
@@ -34,24 +34,17 @@ export const getServices = async (): Promise<Services> => {
 
 	const storageService = createStorageService();
 
-	// The worker must have a durable vector store — a typo'd or missing S3 (or,
-	// on Postgres, PG_CONNECTION_URL) env var must not silently succeed by
-	// falling back to the throwaway in-memory store (which emits success
-	// metrics but drops every vector).
-	const isPostgres = process.env.DATA_BACKEND === "postgres";
-	const pgConnectionUrl = process.env.PG_CONNECTION_URL;
+	// The worker must have a durable vector store — a typo'd or missing env var
+	// must not silently succeed by falling back to the throwaway in-memory store
+	// (which emits success metrics but drops every vector).
 	const localPath = process.env.LOCAL_VECTORDB_PATH;
 	const bucket = process.env.S3_VECTORS_BUCKET_NAME;
 	const indexName = process.env.S3_VECTORS_INDEX_NAME;
-	if (
-		!(isPostgres && pgConnectionUrl) &&
-		!localPath &&
-		!(bucket && indexName)
-	) {
+	if (!localPath && !(bucket && indexName)) {
 		throw new Error(
-			"Vector store is not configured: set PG_CONNECTION_URL for the Postgres " +
-				"backend, LOCAL_VECTORDB_PATH for local dev, or both " +
-				"S3_VECTORS_BUCKET_NAME and S3_VECTORS_INDEX_NAME for production.",
+			"Vector store is not configured: set LOCAL_VECTORDB_PATH for local dev, " +
+				"or both S3_VECTORS_BUCKET_NAME and S3_VECTORS_INDEX_NAME for " +
+				"production.",
 		);
 	}