@remit/search-service 0.0.13 → 0.0.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-service",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -24,17 +24,12 @@
     "./sqlite-vec": {
       "types": "./src/sqlite-vec.ts",
       "default": "./src/sqlite-vec.ts"
-    },
-    "./pgvector": {
-      "types": "./src/pgvector.ts",
-      "default": "./src/pgvector.ts"
     }
   },
   "scripts": {
     "test:typecheck": "tsgo --noEmit",
-    "test:run": "node --env-file=../../localhost-test-unit.env --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=82 --test 'src/**/*.test.ts'",
+    "test:run": "node $NODE_TEST_FLAGS --env-file=../../localhost-test-unit.env --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=90 --test 'src/**/*.test.ts'",
     "test:integ:local": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/backends/sqlite-vec.integ.test.ts'",
-    "test:integ:pg": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/backends/pgvector.integ.test.ts'",
     "test:integ:model": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/semantic-search.integ.test.ts'",
     "test": "npm run test:typecheck && npm run test:run"
   },
@@ -44,14 +39,12 @@
     "@remit/api-openapi-types": "*",
     "better-sqlite3": "^12.11.1",
     "p-limit": "^6.2.0",
-    "pg": "^8.0.0",
     "sqlite-vec": "^0.1.9",
     "zod": "*",
     "@aws-sdk/client-bedrock-runtime": "*",
     "@aws-sdk/client-s3vectors": "*",
     "@smithy/types": "^4.16.1",
-    "@types/better-sqlite3": "^7.6.13",
-    "@types/pg": "^8.20.0"
+    "@types/better-sqlite3": "^7.6.13"
   },
   "devDependencies": {
     "@remit/storage-service": "*",

package/src/backends/memory.ts

@@ -24,8 +24,8 @@ export interface VectorStoreService {
 	getByMessage(messageId: string): Promise<VectorRecord[]>;
 	/**
 	 * Release any held connections (e.g. a pooled database client). Optional — the
-	 * in-memory and file backends hold nothing; the pgvector backend closes its
-	 * pool so a short-lived process (a test, a one-shot reindex) can exit cleanly.
+	 * in-memory backend holds nothing; a pooled backend closes its pool so a
+	 * short-lived process (a test, a one-shot reindex) can exit cleanly.
 	 */
 	close?(): Promise<void>;
 }

package/src/backends/sqlite-vec.integ.test.ts

@@ -5,7 +5,7 @@
  * native extension.
  *
  * Gated behind RUN_INTEG_TESTS because it loads the native better-sqlite3 and
- * sqlite-vec binaries, matching the pgvector integration suite.
+ * sqlite-vec binaries.
  *
  *   npm run test:integ:local -w packages/search-service
  */

package/src/from-env.ts

@@ -2,7 +2,6 @@ import type { DataType } from "@huggingface/transformers";
 import { BedrockEmbeddingService } from "./backends/bedrock.js";
 import type { VectorStoreService } from "./backends/memory.js";
 import { createMemoryVectorStore } from "./backends/memory.js";
-import { createPgVectorStore } from "./backends/pgvector.js";
 import { createS3VectorsBackend } from "./backends/s3-vectors.js";
 import { createSqliteVectorStore } from "./backends/sqlite-vec.js";
 import {
@@ -55,27 +54,19 @@ const parseDtype = (): DataType | undefined => {
  * composes a SearchService (the API, the search-index worker, the local
  * indexing shim) so the selection rule lives in one place:
  *
- * - `DATA_BACKEND=postgres` + `PG_CONNECTION_URL` set → pgvector (Postgres parity).
  * - `LOCAL_VECTORDB_PATH` set → persistent sqlite-vec (local dev).
  * - `S3_VECTORS_BUCKET_NAME` + `S3_VECTORS_INDEX_NAME` set → S3 Vectors (prod).
  * - otherwise → in-memory store (unit tests / default).
  *
  * `dimensions` should be the embedding service's dimension count. When a
- * dimension-typed store is selected (sqlite-vec's vec0 table, pgvector's
- * `VECTOR(n)` column), it is created with that dimension so the store and
+ * dimension-typed store is selected (sqlite-vec's vec0 table), it is created
+ * with that dimension so the store and
  * embedder always agree instead of failing confusingly at insert time (e.g. a
  * 64-dim deterministic embedder writing into a 384-wide column).
  */
 export const buildVectorStoreFromEnv = (
 	dimensions?: number,
 ): VectorStoreService => {
-	const pgConnectionUrl = process.env.PG_CONNECTION_URL;
-	if (process.env.DATA_BACKEND === "postgres" && pgConnectionUrl) {
-		return createPgVectorStore({
-			connectionString: pgConnectionUrl,
-			dimensions,
-		});
-	}
 	const localPath = process.env.LOCAL_VECTORDB_PATH;
 	if (localPath) {
 		return createSqliteVectorStore({ path: localPath, dimensions });
@@ -96,9 +87,9 @@ export const buildVectorStoreFromEnv = (
  * Select an embedder from the environment, mirroring `buildVectorStoreFromEnv`:
  *
  * - `SEARCH_EMBEDDING_PROVIDER=local` → Transformers.js model (local dev). The
- *   model is `SEARCH_EMBEDDING_MODEL_ID` (default MiniLM); the Postgres-parity
- *   stack points it at a multilingual MiniLM so the ~50% non-English mail corpus
- *   embeds well. Both models are 384-dim, so the pgvector column is stable.
+ *   model is `SEARCH_EMBEDDING_MODEL_ID` (default MiniLM); the self-host stack
+ *   points it at a multilingual MiniLM so the ~50% non-English mail corpus
+ *   embeds well. Both models are 384-dim, so the vector column is stable.
  * - `SEARCH_EMBEDDING_PROVIDER=bedrock` → Bedrock Titan (prod).
  * - otherwise → deterministic bag-of-words embedder (unit tests / default).
  *

package/src/backends/pgvector.integ.test.ts

@@ -1,174 +0,0 @@
-/**
- * Exercises the pgvector store against a real local Postgres with the `vector`
- * extension (the pg-parity container). Proves upsert, cosine ranking,
- * multi-condition scoped filtering, content-hash lookup, and delete — the
- * multi-condition filter is the case that silently emptied results on S3
- * Vectors, so it is tested explicitly here.
- *
- * Gated behind RUN_INTEG_TESTS. Point PG_CONNECTION_URL at a database whose
- * `vector` extension is enabled (default: local remit_test).
- *
- *   npm run test:integ:pg -w packages/search-service
- */
-import assert from "node:assert";
-import { randomUUID } from "node:crypto";
-import { after, before, describe, test } from "node:test";
-import pg from "pg";
-import type { ChunkMetadata, VectorRecord } from "../types.js";
-import type { VectorStoreService } from "./memory.js";
-import { createPgVectorStore } from "./pgvector.js";
-
-const RUN = process.env.RUN_INTEG_TESTS === "1";
-const CONNECTION_STRING =
-	process.env.PG_CONNECTION_URL ??
-	"postgresql://remit:remit@localhost:5432/remit_test";
-
-const DIMENSIONS = 4;
-
-const meta = (
-	over: Partial<ChunkMetadata> & { messageId: string },
-): ChunkMetadata => ({
-	threadId: "t-1",
-	accountConfigId: "acc-1",
-	mailboxIds: ["mb-1"],
-	chunkType: "body",
-	sentDate: 1000,
-	isRead: false,
-	hasAttachment: false,
-	hasStars: false,
-	...over,
-});
-
-const record = (
-	chunkId: string,
-	vector: number[],
-	over: Partial<ChunkMetadata> & { messageId: string },
-): VectorRecord => ({ chunkId, vector, metadata: meta(over) });
-
-describe("pgvector store (integration)", { skip: !RUN }, () => {
-	const table = `message_embedding_test_${randomUUID().replace(/-/g, "")}`;
-	let store: VectorStoreService;
-	let adminPool: pg.Pool;
-
-	before(() => {
-		adminPool = new pg.Pool({ connectionString: CONNECTION_STRING });
-		store = createPgVectorStore({
-			connectionString: CONNECTION_STRING,
-			dimensions: DIMENSIONS,
-			tableName: table,
-		});
-	});
-
-	after(async () => {
-		await adminPool.query(`DROP TABLE IF EXISTS ${table}`);
-		await adminPool.end();
-	});
-
-	test("upsert then query ranks by cosine similarity", async () => {
-		await store.upsert([
-			record("c-x", [1, 0, 0, 0], { messageId: "m-x", contentHash: "hx" }),
-			record("c-y", [0, 1, 0, 0], { messageId: "m-y", contentHash: "hy" }),
-			record("c-z", [0.9, 0.1, 0, 0], { messageId: "m-z", contentHash: "hz" }),
-		]);
-
-		const matches = await store.query({ vector: [1, 0, 0, 0], topK: 3 });
-
-		assert.equal(matches[0].chunkId, "c-x");
-		assert.equal(matches[1].chunkId, "c-z");
-		assert.equal(matches[2].chunkId, "c-y");
-		assert.ok(matches[0].score > matches[1].score);
-		assert.ok(matches[0].score > 0.99);
-	});
-
-	test("multi-condition scoped filter narrows to the matching partition", async () => {
-		await store.upsert([
-			record("s-a", [1, 0, 0, 0], {
-				messageId: "m-a",
-				accountConfigId: "acc-A",
-				mailboxIds: ["inbox"],
-				isRead: true,
-			}),
-			record("s-b", [1, 0, 0, 0], {
-				messageId: "m-b",
-				accountConfigId: "acc-A",
-				mailboxIds: ["archive"],
-				isRead: true,
-			}),
-			record("s-c", [1, 0, 0, 0], {
-				messageId: "m-c",
-				accountConfigId: "acc-B",
-				mailboxIds: ["inbox"],
-				isRead: true,
-			}),
-			record("s-d", [1, 0, 0, 0], {
-				messageId: "m-d",
-				accountConfigId: "acc-A",
-				mailboxIds: ["inbox"],
-				isRead: false,
-			}),
-		]);
-
-		const matches = await store.query({
-			vector: [1, 0, 0, 0],
-			topK: 10,
-			filter: { accountConfigId: "acc-A", mailboxId: "inbox", isRead: true },
-		});
-
-		const ids = matches.map((m) => m.chunkId);
-		assert.deepEqual(ids, ["s-a"]);
-	});
-
-	test("existingContentHashes returns stored hashes only for known keys", async () => {
-		const hashes = await store.existingContentHashes(["c-x", "c-y", "missing"]);
-		assert.equal(hashes.get("c-x"), "hx");
-		assert.equal(hashes.get("c-y"), "hy");
-		assert.equal(hashes.has("missing"), false);
-	});
-
-	test("upsert overwrites an existing chunk in place", async () => {
-		await store.upsert([
-			record("c-x", [0, 0, 0, 1], { messageId: "m-x", contentHash: "hx2" }),
-		]);
-		const hashes = await store.existingContentHashes(["c-x"]);
-		assert.equal(hashes.get("c-x"), "hx2");
-	});
-
-	test("getByMessage returns every chunk of a message with its vector and metadata", async () => {
-		await store.upsert([
-			record("g-sub", [1, 0, 0, 0], {
-				messageId: "m-get",
-				chunkType: "subject",
-				contentHash: "hg1",
-			}),
-			record("g-body", [0, 1, 0, 0], {
-				messageId: "m-get",
-				chunkType: "body",
-				contentHash: "hg2",
-			}),
-			record("g-other", [0, 0, 1, 0], { messageId: "m-other" }),
-		]);
-
-		const records = await store.getByMessage("m-get");
-
-		const byId = new Map(records.map((r) => [r.chunkId, r]));
-		assert.equal(records.length, 2, "only the message's own chunks");
-		assert.deepEqual(byId.get("g-sub")?.vector, [1, 0, 0, 0]);
-		assert.deepEqual(byId.get("g-body")?.vector, [0, 1, 0, 0]);
-		assert.equal(byId.get("g-sub")?.metadata.chunkType, "subject");
-		assert.equal(byId.get("g-body")?.metadata.messageId, "m-get");
-	});
-
-	test("getByMessage returns an empty array for an unknown message", async () => {
-		assert.deepEqual(await store.getByMessage("m-absent"), []);
-	});
-
-	test("delete removes every chunk of a message", async () => {
-		await store.upsert([
-			record("d-1", [1, 0, 0, 0], { messageId: "m-del" }),
-			record("d-2", [0, 1, 0, 0], { messageId: "m-del" }),
-		]);
-		await store.delete({ messageId: "m-del" });
-		const hashes = await store.existingContentHashes(["d-1", "d-2"]);
-		assert.equal(hashes.size, 0);
-	});
-});

package/src/backends/pgvector.ts

@@ -1,306 +0,0 @@
-import type { Pool, PoolClient } from "pg";
-import type {
-	ChunkMetadata,
-	VectorMatch,
-	VectorQuery,
-	VectorQueryFilter,
-	VectorRecord,
-} from "../types.js";
-import type { VectorStoreService } from "./memory.js";
-import { runtimeImport } from "./runtime-import.js";
-
-type PgModule = {
-	default: { Pool: new (config: { connectionString: string }) => Pool };
-};
-
-/**
- * pgvector-backed vector store for the Postgres-parity stack. Vectors live in
- * the same Postgres as their message rows (one row per chunk), so there is no
- * write amplification and scoped queries are plain SQL WHERE clauses combined
- * with the ANN ordering — the equality/range filters run in the same query as
- * the KNN instead of post-filtering a fixed top-k (which silently drops recall).
- *
- * `mailboxId` is a single-value scalar because the indexing pipeline always
- * writes a one-element `mailboxIds`; membership reduces to equality — same as
- * the sqlite-vec backend.
- *
- * The store self-provisions its table and indexes on first use (the `vector`
- * extension must already be enabled on the database — the local pg container
- * does this at startup). This mirrors the sqlite-vec backend, which owns its
- * own CREATE, and keeps the embedding table out of the drizzle schema that the
- * extension-less embedded-postgres test harness pushes.
- */
-const DEFAULT_DIMENSIONS = 384;
-const DEFAULT_TABLE = "message_embedding";
-
-const createTableSql = (table: string, dimensions: number): string => `
-	CREATE TABLE IF NOT EXISTS ${table} (
-		chunk_id TEXT PRIMARY KEY,
-		message_id TEXT NOT NULL,
-		account_config_id TEXT NOT NULL,
-		mailbox_id TEXT NOT NULL,
-		chunk_type TEXT NOT NULL,
-		sent_date BIGINT NOT NULL,
-		is_read BOOLEAN NOT NULL,
-		has_attachment BOOLEAN NOT NULL,
-		has_stars BOOLEAN NOT NULL,
-		embedding VECTOR(${dimensions}) NOT NULL,
-		metadata JSONB NOT NULL
-	);
-`;
-
-const createIndexesSql = (table: string): string[] => [
-	`CREATE INDEX IF NOT EXISTS ${table}_message_id_idx ON ${table} (message_id);`,
-	`CREATE INDEX IF NOT EXISTS ${table}_account_config_id_idx ON ${table} (account_config_id);`,
-	`CREATE INDEX IF NOT EXISTS ${table}_embedding_hnsw_idx ON ${table} USING hnsw (embedding vector_cosine_ops);`,
-];
-
-const toVectorLiteral = (vector: number[]): string => `[${vector.join(",")}]`;
-
-// pgvector renders a `vector` column as `[1,2,3]`; parse it back to a number[]
-// for the anchor pooling read path (getByMessage).
-const parseVectorLiteral = (literal: string): number[] => {
-	const inner = literal.trim().replace(/^\[/, "").replace(/\]$/, "");
-	if (inner.length === 0) return [];
-	return inner.split(",").map((n) => Number(n));
-};
-
-interface WhereClause {
-	sql: string;
-	params: unknown[];
-}
-
-const buildFilterClause = (
-	filter: VectorQueryFilter | undefined,
-	nextParamIndex: number,
-): WhereClause => {
-	const conditions: string[] = [];
-	const params: unknown[] = [];
-	let i = nextParamIndex;
-	if (!filter) return { sql: "", params };
-
-	if (filter.accountConfigId !== undefined) {
-		conditions.push(`account_config_id = $${i++}`);
-		params.push(filter.accountConfigId);
-	}
-	if (filter.mailboxId !== undefined) {
-		conditions.push(`mailbox_id = $${i++}`);
-		params.push(filter.mailboxId);
-	}
-	if (filter.chunkType !== undefined) {
-		conditions.push(`chunk_type = $${i++}`);
-		params.push(filter.chunkType);
-	}
-	if (filter.category !== undefined) {
-		conditions.push(`metadata->>'category' = $${i++}`);
-		params.push(filter.category);
-	}
-	if (filter.hasAttachment !== undefined) {
-		conditions.push(`has_attachment = $${i++}`);
-		params.push(filter.hasAttachment);
-	}
-	if (filter.hasStars !== undefined) {
-		conditions.push(`has_stars = $${i++}`);
-		params.push(filter.hasStars);
-	}
-	if (filter.isRead !== undefined) {
-		conditions.push(`is_read = $${i++}`);
-		params.push(filter.isRead);
-	}
-	if (filter.sentDateRange?.from !== undefined) {
-		conditions.push(`sent_date >= $${i++}`);
-		params.push(Math.trunc(filter.sentDateRange.from));
-	}
-	if (filter.sentDateRange?.to !== undefined) {
-		conditions.push(`sent_date <= $${i++}`);
-		params.push(Math.trunc(filter.sentDateRange.to));
-	}
-
-	return {
-		sql: conditions.length > 0 ? ` AND ${conditions.join(" AND ")}` : "",
-		params,
-	};
-};
-
-export interface PgVectorStoreConfig {
-	connectionString: string;
-	dimensions?: number;
-	tableName?: string;
-}
-
-export const createPgVectorStore = (
-	config: PgVectorStoreConfig,
-): VectorStoreService => {
-	const dimensions = config.dimensions ?? DEFAULT_DIMENSIONS;
-	const table = config.tableName ?? DEFAULT_TABLE;
-	let poolPromise: Promise<Pool> | null = null;
-
-	const getPool = async (): Promise<Pool> => {
-		if (poolPromise) return poolPromise;
-		poolPromise = (async () => {
-			const pg = await runtimeImport<PgModule>("pg");
-			const pool = new pg.default.Pool({
-				connectionString: config.connectionString,
-			});
-			await pool.query(createTableSql(table, dimensions));
-			for (const sql of createIndexesSql(table)) {
-				await pool.query(sql);
-			}
-			return pool;
-		})();
-		return poolPromise;
-	};
-
-	const upsert = async (vectors: VectorRecord[]): Promise<void> => {
-		if (vectors.length === 0) return;
-		const pool = await getPool();
-		const client = await pool.connect();
-		try {
-			await client.query("BEGIN");
-			for (const record of vectors) {
-				const m = record.metadata;
-				await client.query(
-					`
-					INSERT INTO ${table} (
-						chunk_id, message_id, account_config_id, mailbox_id, chunk_type,
-						sent_date, is_read, has_attachment, has_stars, embedding, metadata
-					) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::vector, $11::jsonb)
-					ON CONFLICT (chunk_id) DO UPDATE SET
-						message_id = EXCLUDED.message_id,
-						account_config_id = EXCLUDED.account_config_id,
-						mailbox_id = EXCLUDED.mailbox_id,
-						chunk_type = EXCLUDED.chunk_type,
-						sent_date = EXCLUDED.sent_date,
-						is_read = EXCLUDED.is_read,
-						has_attachment = EXCLUDED.has_attachment,
-						has_stars = EXCLUDED.has_stars,
-						embedding = EXCLUDED.embedding,
-						metadata = EXCLUDED.metadata
-					`,
-					[
-						record.chunkId,
-						m.messageId,
-						m.accountConfigId,
-						m.mailboxIds[0] ?? "",
-						m.chunkType,
-						Math.trunc(m.sentDate),
-						m.isRead,
-						m.hasAttachment,
-						m.hasStars,
-						toVectorLiteral(record.vector),
-						JSON.stringify(m),
-					],
-				);
-			}
-			await client.query("COMMIT");
-		} catch (error) {
-			await client.query("ROLLBACK");
-			throw error;
-		} finally {
-			client.release();
-		}
-	};
-
-	const query = async (params: VectorQuery): Promise<VectorMatch[]> => {
-		const pool = await getPool();
-		const vectorLiteral = toVectorLiteral(params.vector);
-		const { sql: filterSql, params: filterParams } = buildFilterClause(
-			params.filter,
-			3,
-		);
-		// Iterative index scan keeps a selective metadata filter from under-filling
-		// the top-k: pgvector re-scans the HNSW graph until topK matches pass the
-		// WHERE instead of stopping at the first ef_search candidates.
-		const client: PoolClient = await pool.connect();
-		try {
-			await client.query("BEGIN");
-			await client.query("SET LOCAL hnsw.iterative_scan = relaxed_order");
-			const result = await client.query<{
-				chunk_id: string;
-				score: number;
-				metadata: ChunkMetadata;
-			}>(
-				`
-				SELECT chunk_id, metadata, 1 - (embedding <=> $1::vector) AS score
-				FROM ${table}
-				WHERE TRUE${filterSql}
-				ORDER BY embedding <=> $1::vector
-				LIMIT $2
-				`,
-				[vectorLiteral, params.topK, ...filterParams],
-			);
-			await client.query("COMMIT");
-			return result.rows.map((row) => ({
-				chunkId: row.chunk_id,
-				score: Number(row.score),
-				metadata: row.metadata,
-			}));
-		} catch (error) {
-			await client.query("ROLLBACK");
-			throw error;
-		} finally {
-			client.release();
-		}
-	};
-
-	const existingContentHashes = async (
-		chunkIds: string[],
-	): Promise<Map<string, string>> => {
-		const out = new Map<string, string>();
-		if (chunkIds.length === 0) return out;
-		const pool = await getPool();
-		const result = await pool.query<{
-			chunk_id: string;
-			content_hash: string | null;
-		}>(
-			`SELECT chunk_id, metadata->>'contentHash' AS content_hash
-			 FROM ${table} WHERE chunk_id = ANY($1::text[])`,
-			[chunkIds],
-		);
-		for (const row of result.rows) {
-			if (row.content_hash !== null) out.set(row.chunk_id, row.content_hash);
-		}
-		return out;
-	};
-
-	const getByMessage = async (messageId: string): Promise<VectorRecord[]> => {
-		const pool = await getPool();
-		const result = await pool.query<{
-			chunk_id: string;
-			embedding: string;
-			metadata: ChunkMetadata;
-		}>(
-			`SELECT chunk_id, embedding::text AS embedding, metadata
-			 FROM ${table} WHERE message_id = $1`,
-			[messageId],
-		);
-		return result.rows.map((row) => ({
-			chunkId: row.chunk_id,
-			vector: parseVectorLiteral(row.embedding),
-			metadata: row.metadata,
-		}));
-	};
-
-	const del = async (filter: { messageId: string }): Promise<void> => {
-		const pool = await getPool();
-		await pool.query(`DELETE FROM ${table} WHERE message_id = $1`, [
-			filter.messageId,
-		]);
-	};
-
-	const close = async (): Promise<void> => {
-		if (!poolPromise) return;
-		const pool = await poolPromise;
-		poolPromise = null;
-		await pool.end();
-	};
-
-	return {
-		upsert,
-		query,
-		existingContentHashes,
-		getByMessage,
-		delete: del,
-		close,
-	};
-};

package/src/pgvector.ts

@@ -1,4 +0,0 @@
-export {
-	createPgVectorStore,
-	type PgVectorStoreConfig,
-} from "./backends/pgvector.js";