@remit/search-service 0.0.1

package/src/backends/memory.test.ts

@@ -0,0 +1,168 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import type { ChunkMetadata, VectorRecord } from "../types.js";
+import { MemoryVectorStore } from "./memory.js";
+
+const buildMetadata = (
+	overrides: Partial<ChunkMetadata> = {},
+): ChunkMetadata => ({
+	messageId: "msg-1",
+	threadId: "thread-1",
+	accountConfigId: "acct-1",
+	mailboxIds: ["mb-inbox"],
+	chunkType: "body",
+	sentDate: 1_700_000_000,
+	isRead: false,
+	hasAttachment: false,
+	hasStars: false,
+	...overrides,
+});
+
+const record = (
+	chunkId: string,
+	vector: number[],
+	overrides: Partial<ChunkMetadata> = {},
+): VectorRecord => ({
+	chunkId,
+	vector,
+	metadata: buildMetadata({ ...overrides }),
+});
+
+describe("MemoryVectorStore", () => {
+	it("returns nearest matches in score-descending order", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0]),
+			record("b", [0, 1, 0]),
+			record("c", [0.7, 0.7, 0]),
+		]);
+
+		const matches = await store.query({ vector: [1, 0, 0], topK: 3 });
+		assert.strictEqual(matches.length, 3);
+		assert.strictEqual(matches[0].chunkId, "a");
+		assert.ok(matches[0].score > matches[1].score);
+		assert.ok(matches[1].score >= matches[2].score);
+	});
+
+	it("respects topK", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0]),
+			record("b", [0, 1, 0]),
+			record("c", [0, 0, 1]),
+		]);
+
+		const matches = await store.query({ vector: [1, 0, 0], topK: 1 });
+		assert.strictEqual(matches.length, 1);
+	});
+
+	it("filters by mailboxId", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0], { mailboxIds: ["mb-inbox"] }),
+			record("b", [1, 0, 0], { mailboxIds: ["mb-archive"] }),
+		]);
+
+		const matches = await store.query({
+			vector: [1, 0, 0],
+			topK: 5,
+			filter: { mailboxId: "mb-archive" },
+		});
+		assert.strictEqual(matches.length, 1);
+		assert.strictEqual(matches[0].chunkId, "b");
+	});
+
+	it("filters by accountConfigId", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0], { accountConfigId: "acct-1" }),
+			record("b", [1, 0, 0], { accountConfigId: "acct-2" }),
+		]);
+
+		const matches = await store.query({
+			vector: [1, 0, 0],
+			topK: 5,
+			filter: { accountConfigId: "acct-1" },
+		});
+		assert.strictEqual(matches.length, 1);
+		assert.strictEqual(matches[0].chunkId, "a");
+	});
+
+	it("filters by category", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0], { category: "newsletter" }),
+			record("b", [1, 0, 0], { category: "personal" }),
+		]);
+
+		const matches = await store.query({
+			vector: [1, 0, 0],
+			topK: 5,
+			filter: { category: "newsletter" },
+		});
+		assert.strictEqual(matches.length, 1);
+		assert.strictEqual(matches[0].chunkId, "a");
+	});
+
+	it("filters by sentDateRange", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("old", [1, 0, 0], { sentDate: 100 }),
+			record("mid", [1, 0, 0], { sentDate: 500 }),
+			record("new", [1, 0, 0], { sentDate: 1000 }),
+		]);
+
+		const matches = await store.query({
+			vector: [1, 0, 0],
+			topK: 5,
+			filter: { sentDateRange: { from: 200, to: 800 } },
+		});
+		const ids = matches.map((m) => m.chunkId).sort();
+		assert.deepStrictEqual(ids, ["mid"]);
+	});
+
+	it("deletes all chunks for a given messageId", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0], { messageId: "msg-1" }),
+			record("b", [1, 0, 0], { messageId: "msg-1" }),
+			record("c", [1, 0, 0], { messageId: "msg-2" }),
+		]);
+
+		await store.delete({ messageId: "msg-1" });
+
+		const matches = await store.query({ vector: [1, 0, 0], topK: 5 });
+		assert.strictEqual(matches.length, 1);
+		assert.strictEqual(matches[0].chunkId, "c");
+	});
+
+	it("returns all chunk records for a messageId via getByMessage", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([
+			record("a", [1, 0, 0], { messageId: "msg-1", chunkType: "subject" }),
+			record("b", [0, 1, 0], { messageId: "msg-1", chunkType: "body" }),
+			record("c", [0, 0, 1], { messageId: "msg-2" }),
+		]);
+
+		const records = await store.getByMessage("msg-1");
+		const ids = records.map((r) => r.chunkId).sort();
+		assert.deepStrictEqual(ids, ["a", "b"]);
+		const subject = records.find((r) => r.chunkId === "a");
+		assert.deepStrictEqual(subject?.vector, [1, 0, 0]);
+	});
+
+	it("returns an empty array from getByMessage for an unknown message", async () => {
+		const store = new MemoryVectorStore();
+		assert.deepStrictEqual(await store.getByMessage("absent"), []);
+	});
+
+	it("upsert overwrites an existing record by chunkId", async () => {
+		const store = new MemoryVectorStore();
+		await store.upsert([record("a", [1, 0, 0])]);
+		await store.upsert([record("a", [0, 1, 0])]);
+		assert.strictEqual(store.size(), 1);
+
+		const matches = await store.query({ vector: [0, 1, 0], topK: 1 });
+		assert.ok(matches[0].score > 0.9);
+	});
+});

package/src/backends/memory.ts

@@ -0,0 +1,152 @@
+import type {
+	VectorMatch,
+	VectorQuery,
+	VectorQueryFilter,
+	VectorRecord,
+} from "../types.js";
+
+export interface VectorStoreService {
+	upsert(vectors: VectorRecord[]): Promise<void>;
+	query(params: VectorQuery): Promise<VectorMatch[]>;
+	delete(filter: { messageId: string }): Promise<void>;
+	/**
+	 * Read the stored content hash for each of the given deterministic chunk keys.
+	 * Keys with no stored vector, or a vector with no contentHash, are absent from
+	 * the map. Addresses vectors by key only — never an index-wide scan.
+	 */
+	existingContentHashes(chunkIds: string[]): Promise<Map<string, string>>;
+	/**
+	 * Read every stored chunk vector (data + metadata) for a message, addressed by
+	 * the message's deterministic chunk keys — never an index-wide scan. Empty when
+	 * the message has no indexed chunks. Backs the filter-anchor build (RFC 034
+	 * Decision 2.1), which pools a message's chunk vectors into a single anchor.
+	 */
+	getByMessage(messageId: string): Promise<VectorRecord[]>;
+	/**
+	 * Release any held connections (e.g. a pooled database client). Optional — the
+	 * in-memory and file backends hold nothing; the pgvector backend closes its
+	 * pool so a short-lived process (a test, a one-shot reindex) can exit cleanly.
+	 */
+	close?(): Promise<void>;
+}
+
+const cosineSimilarity = (a: number[], b: number[]): number => {
+	if (a.length !== b.length) {
+		throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+	}
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	for (let i = 0; i < a.length; i++) {
+		dot += a[i] * b[i];
+		normA += a[i] * a[i];
+		normB += b[i] * b[i];
+	}
+	if (normA === 0 || normB === 0) return 0;
+	return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+};
+
+const matchesFilter = (
+	record: VectorRecord,
+	filter: VectorQueryFilter | undefined,
+): boolean => {
+	if (!filter) return true;
+	const m = record.metadata;
+	if (
+		filter.accountConfigId !== undefined &&
+		m.accountConfigId !== filter.accountConfigId
+	) {
+		return false;
+	}
+	if (
+		filter.mailboxId !== undefined &&
+		!m.mailboxIds.includes(filter.mailboxId)
+	) {
+		return false;
+	}
+	if (filter.chunkType !== undefined && m.chunkType !== filter.chunkType) {
+		return false;
+	}
+	if (filter.category !== undefined && m.category !== filter.category) {
+		return false;
+	}
+	if (
+		filter.hasAttachment !== undefined &&
+		m.hasAttachment !== filter.hasAttachment
+	) {
+		return false;
+	}
+	if (filter.hasStars !== undefined && m.hasStars !== filter.hasStars) {
+		return false;
+	}
+	if (filter.isRead !== undefined && m.isRead !== filter.isRead) {
+		return false;
+	}
+	if (filter.sentDateRange) {
+		const { from, to } = filter.sentDateRange;
+		if (from !== undefined && m.sentDate < from) return false;
+		if (to !== undefined && m.sentDate > to) return false;
+	}
+	return true;
+};
+
+export class MemoryVectorStore implements VectorStoreService {
+	private store = new Map<string, VectorRecord>();
+
+	upsert = async (vectors: VectorRecord[]): Promise<void> => {
+		for (const v of vectors) {
+			this.store.set(v.chunkId, v);
+		}
+	};
+
+	query = async (params: VectorQuery): Promise<VectorMatch[]> => {
+		const matches: VectorMatch[] = [];
+		for (const record of this.store.values()) {
+			if (!matchesFilter(record, params.filter)) continue;
+			const score = cosineSimilarity(params.vector, record.vector);
+			matches.push({
+				chunkId: record.chunkId,
+				score,
+				metadata: record.metadata,
+			});
+		}
+		matches.sort((a, b) => b.score - a.score);
+		return matches.slice(0, params.topK);
+	};
+
+	existingContentHashes = async (
+		chunkIds: string[],
+	): Promise<Map<string, string>> => {
+		const out = new Map<string, string>();
+		for (const chunkId of chunkIds) {
+			const hash = this.store.get(chunkId)?.metadata.contentHash;
+			if (typeof hash === "string") out.set(chunkId, hash);
+		}
+		return out;
+	};
+
+	getByMessage = async (messageId: string): Promise<VectorRecord[]> => {
+		const out: VectorRecord[] = [];
+		for (const record of this.store.values()) {
+			if (record.metadata.messageId === messageId) out.push(record);
+		}
+		return out;
+	};
+
+	delete = async (filter: { messageId: string }): Promise<void> => {
+		const toDelete: string[] = [];
+		for (const [id, record] of this.store.entries()) {
+			if (record.metadata.messageId === filter.messageId) {
+				toDelete.push(id);
+			}
+		}
+		for (const id of toDelete) {
+			this.store.delete(id);
+		}
+	};
+
+	size = (): number => this.store.size;
+}
+
+export const createMemoryVectorStore = (): MemoryVectorStore =>
+	new MemoryVectorStore();

package/src/backends/pgvector.integ.test.ts

@@ -0,0 +1,174 @@
+/**
+ * Exercises the pgvector store against a real local Postgres with the `vector`
+ * extension (the pg-parity container). Proves upsert, cosine ranking,
+ * multi-condition scoped filtering, content-hash lookup, and delete — the
+ * multi-condition filter is the case that silently emptied results on S3
+ * Vectors, so it is tested explicitly here.
+ *
+ * Gated behind RUN_INTEG_TESTS. Point PG_CONNECTION_URL at a database whose
+ * `vector` extension is enabled (default: local remit_test).
+ *
+ *   npm run test:integ -w packages/search-service
+ */
+import assert from "node:assert";
+import { randomUUID } from "node:crypto";
+import { after, before, describe, test } from "node:test";
+import pg from "pg";
+import type { ChunkMetadata, VectorRecord } from "../types.js";
+import type { VectorStoreService } from "./memory.js";
+import { createPgVectorStore } from "./pgvector.js";
+
+const RUN = process.env.RUN_INTEG_TESTS === "1";
+const CONNECTION_STRING =
+	process.env.PG_CONNECTION_URL ??
+	"postgresql://remit:remit@localhost:5432/remit_test";
+
+const DIMENSIONS = 4;
+
+const meta = (
+	over: Partial<ChunkMetadata> & { messageId: string },
+): ChunkMetadata => ({
+	threadId: "t-1",
+	accountConfigId: "acc-1",
+	mailboxIds: ["mb-1"],
+	chunkType: "body",
+	sentDate: 1000,
+	isRead: false,
+	hasAttachment: false,
+	hasStars: false,
+	...over,
+});
+
+const record = (
+	chunkId: string,
+	vector: number[],
+	over: Partial<ChunkMetadata> & { messageId: string },
+): VectorRecord => ({ chunkId, vector, metadata: meta(over) });
+
+describe("pgvector store (integration)", { skip: !RUN }, () => {
+	const table = `message_embedding_test_${randomUUID().replace(/-/g, "")}`;
+	let store: VectorStoreService;
+	let adminPool: pg.Pool;
+
+	before(() => {
+		adminPool = new pg.Pool({ connectionString: CONNECTION_STRING });
+		store = createPgVectorStore({
+			connectionString: CONNECTION_STRING,
+			dimensions: DIMENSIONS,
+			tableName: table,
+		});
+	});
+
+	after(async () => {
+		await adminPool.query(`DROP TABLE IF EXISTS ${table}`);
+		await adminPool.end();
+	});
+
+	test("upsert then query ranks by cosine similarity", async () => {
+		await store.upsert([
+			record("c-x", [1, 0, 0, 0], { messageId: "m-x", contentHash: "hx" }),
+			record("c-y", [0, 1, 0, 0], { messageId: "m-y", contentHash: "hy" }),
+			record("c-z", [0.9, 0.1, 0, 0], { messageId: "m-z", contentHash: "hz" }),
+		]);
+
+		const matches = await store.query({ vector: [1, 0, 0, 0], topK: 3 });
+
+		assert.equal(matches[0].chunkId, "c-x");
+		assert.equal(matches[1].chunkId, "c-z");
+		assert.equal(matches[2].chunkId, "c-y");
+		assert.ok(matches[0].score > matches[1].score);
+		assert.ok(matches[0].score > 0.99);
+	});
+
+	test("multi-condition scoped filter narrows to the matching partition", async () => {
+		await store.upsert([
+			record("s-a", [1, 0, 0, 0], {
+				messageId: "m-a",
+				accountConfigId: "acc-A",
+				mailboxIds: ["inbox"],
+				isRead: true,
+			}),
+			record("s-b", [1, 0, 0, 0], {
+				messageId: "m-b",
+				accountConfigId: "acc-A",
+				mailboxIds: ["archive"],
+				isRead: true,
+			}),
+			record("s-c", [1, 0, 0, 0], {
+				messageId: "m-c",
+				accountConfigId: "acc-B",
+				mailboxIds: ["inbox"],
+				isRead: true,
+			}),
+			record("s-d", [1, 0, 0, 0], {
+				messageId: "m-d",
+				accountConfigId: "acc-A",
+				mailboxIds: ["inbox"],
+				isRead: false,
+			}),
+		]);
+
+		const matches = await store.query({
+			vector: [1, 0, 0, 0],
+			topK: 10,
+			filter: { accountConfigId: "acc-A", mailboxId: "inbox", isRead: true },
+		});
+
+		const ids = matches.map((m) => m.chunkId);
+		assert.deepEqual(ids, ["s-a"]);
+	});
+
+	test("existingContentHashes returns stored hashes only for known keys", async () => {
+		const hashes = await store.existingContentHashes(["c-x", "c-y", "missing"]);
+		assert.equal(hashes.get("c-x"), "hx");
+		assert.equal(hashes.get("c-y"), "hy");
+		assert.equal(hashes.has("missing"), false);
+	});
+
+	test("upsert overwrites an existing chunk in place", async () => {
+		await store.upsert([
+			record("c-x", [0, 0, 0, 1], { messageId: "m-x", contentHash: "hx2" }),
+		]);
+		const hashes = await store.existingContentHashes(["c-x"]);
+		assert.equal(hashes.get("c-x"), "hx2");
+	});
+
+	test("getByMessage returns every chunk of a message with its vector and metadata", async () => {
+		await store.upsert([
+			record("g-sub", [1, 0, 0, 0], {
+				messageId: "m-get",
+				chunkType: "subject",
+				contentHash: "hg1",
+			}),
+			record("g-body", [0, 1, 0, 0], {
+				messageId: "m-get",
+				chunkType: "body",
+				contentHash: "hg2",
+			}),
+			record("g-other", [0, 0, 1, 0], { messageId: "m-other" }),
+		]);
+
+		const records = await store.getByMessage("m-get");
+
+		const byId = new Map(records.map((r) => [r.chunkId, r]));
+		assert.equal(records.length, 2, "only the message's own chunks");
+		assert.deepEqual(byId.get("g-sub")?.vector, [1, 0, 0, 0]);
+		assert.deepEqual(byId.get("g-body")?.vector, [0, 1, 0, 0]);
+		assert.equal(byId.get("g-sub")?.metadata.chunkType, "subject");
+		assert.equal(byId.get("g-body")?.metadata.messageId, "m-get");
+	});
+
+	test("getByMessage returns an empty array for an unknown message", async () => {
+		assert.deepEqual(await store.getByMessage("m-absent"), []);
+	});
+
+	test("delete removes every chunk of a message", async () => {
+		await store.upsert([
+			record("d-1", [1, 0, 0, 0], { messageId: "m-del" }),
+			record("d-2", [0, 1, 0, 0], { messageId: "m-del" }),
+		]);
+		await store.delete({ messageId: "m-del" });
+		const hashes = await store.existingContentHashes(["d-1", "d-2"]);
+		assert.equal(hashes.size, 0);
+	});
+});