@remit/search-index-worker 0.0.22 → 0.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-index-worker",
-  "version": "0.0.22",
+  "version": "0.0.23",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/services.ts

@@ -8,6 +8,8 @@ import {
 import {
 	buildEmbeddingServiceFromEnv,
 	buildVectorStoreFromEnv,
+	EMBEDDING_PROVIDER_OFF,
+	readEmbeddingProviderFromEnv,
 } from "@remit/search-service/from-env";
 import { createHeartbeat } from "@remit/sqs-client/heartbeat";
 import type { StorageService } from "@remit/storage-service";
@@ -74,7 +76,7 @@ let governorResolved = false;
 export const getMemoryGovernor = (): MemoryGovernor | undefined => {
 	if (governorResolved) return governor;
 	governorResolved = true;
-	if (process.env.SEARCH_EMBEDDING_PROVIDER !== "local") return undefined;
+	if (readEmbeddingProviderFromEnv() !== "local") return undefined;
 
 	const config = readAdaptiveEmbeddingConfigFromEnv();
 	const metrics = registerAdaptiveEmbedding({
@@ -118,6 +120,19 @@ const governed = (embedder: EmbeddingService): EmbeddingService => {
 export const getServices = async (): Promise<Services> => {
 	if (cached) return cached;
 
+	// This process exists to embed. `off` is the self-host default and holds the
+	// container down behind the `semantic` compose profile
+	// (deploy/vps/docker-compose.sqlite.yml), so reaching here with it set means
+	// the worker was started against a deployment that asked for no embedding:
+	// every message it took off the queue would fail one at a time, forever. Say
+	// so once, at startup, and name the command that settles it.
+	if (readEmbeddingProviderFromEnv() === EMBEDDING_PROVIDER_OFF) {
+		throw new Error(
+			"SEARCH_EMBEDDING_PROVIDER is off, so there is nothing for this worker to embed. " +
+				"Turn semantic search on with 'remit semantic on', or leave this service down.",
+		);
+	}
+
 	const dataPorts = await buildDataPortsFromEnv();
 
 	const storageService = createStorageService();

package/src/sqlite-outbox-drain.test.ts

@@ -113,3 +113,38 @@ describe("SqliteOutboxStore", () => {
 		db.close();
 	});
 });
+
+// `remit semantic on` against an existing mailbox makes the first drain pass the
+// whole back catalogue, and the pass re-runs every 2 s. The read is bounded so
+// one tick relays a batch rather than tens of thousands of rows; what it does not
+// take stays unprocessed and is what the next tick selects.
+describe("SqliteOutboxStore, bounded", () => {
+	test("reads at most one batch, and the rest on the next pass", async () => {
+		const db = makeOutboxDb();
+		const total = 620;
+		for (let i = 0; i < total; i++) {
+			insertRow(db, `r${i}`, `m${i}`, "message.body_synced");
+		}
+		const store = new SqliteOutboxStore(db as unknown as never);
+
+		const first = await store.listUnprocessedEvents();
+		assert.equal(first.length, 500);
+
+		const sent: string[] = [];
+		const relay = new OutboxRelay({
+			store,
+			sqs: fakeSqs(sent),
+			queueUrl: "q",
+		});
+
+		assert.equal(await relay.drainPending(), 500);
+		assert.equal(await relay.drainPending(), total - 500);
+		assert.equal(await relay.drainPending(), 0);
+
+		const stillPending = db
+			.prepare("SELECT count(*) AS n FROM outbox WHERE processed_at IS NULL")
+			.get() as { n: number };
+		assert.equal(stillPending.n, 0, "every row was relayed, none skipped");
+		db.close();
+	});
+});

package/src/sqlite-outbox-drain.ts

@@ -24,6 +24,13 @@ import {
 
 const DRAIN_INTERVAL_MS = 2_000;
 
+// How many distinct messages one drain pass relays. `remit semantic on` on an
+// existing mailbox makes the first pass the whole back catalogue — tens of
+// thousands of rows read, enqueued and marked in one tick — so the pass is
+// bounded and the next tick, 2 s later, takes the following batch. Nothing is
+// dropped: unmarked rows stay unprocessed and are what the next pass selects.
+const DRAIN_BATCH_SIZE = 500;
+
 // A minimal view of the better-sqlite3 surface used here, so the module carries
 // no static type dependency on the native package (imported dynamically to stay
 // out of the Lambda bundle).
@@ -46,7 +53,9 @@ export class SqliteOutboxStore implements OutboxStore {
 		const rows = this.db
 			.prepare(
 				`SELECT DISTINCT message_id, event FROM outbox
-				 WHERE event IN (${placeholders}) AND processed_at IS NULL`,
+				 WHERE event IN (${placeholders}) AND processed_at IS NULL
+				 ORDER BY message_id
+				 LIMIT ${DRAIN_BATCH_SIZE}`,
 			)
 			.all(...DRAIN_EVENTS) as Array<{ message_id: string; event: string }>;
 		return rows.map((row) => ({