@remit/search-service 0.0.11 → 0.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-service",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -33,7 +33,9 @@
   "scripts": {
     "test:typecheck": "tsgo --noEmit",
     "test:run": "node --env-file=../../localhost-test-unit.env --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=82 --test 'src/**/*.test.ts'",
-    "test:integ": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.integ.test.ts'",
+    "test:integ:local": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/backends/sqlite-vec.integ.test.ts'",
+    "test:integ:pg": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/backends/pgvector.integ.test.ts'",
+    "test:integ:model": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/semantic-search.integ.test.ts'",
     "test": "npm run test:typecheck && npm run test:run"
   },
   "dependencies": {

package/src/anchor.test.ts

@@ -3,15 +3,18 @@ import { describe, it } from "node:test";
 import {
 	buildAnchorSourceText,
 	buildMessageAnchor,
+	deriveAnchorEmbeddingId,
 	poolChunkVectors,
+	UNKNOWN_CHUNK_EMBEDDING_ID,
 } from "./anchor.js";
 import { createMemoryVectorStore } from "./backends/memory.js";
-import { createDeterministicEmbeddingService } from "./embeddings.js";
 import type { ChunkMetadata, VectorRecord } from "./types.js";
 
 const l2Norm = (vector: number[]): number =>
 	Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
 
+const CURRENT_MODEL = "deterministic@8";
+
 const baseMetadata = (
 	overrides: Partial<ChunkMetadata> = {},
 ): ChunkMetadata => ({
@@ -24,6 +27,7 @@ const baseMetadata = (
 	isRead: false,
 	hasAttachment: false,
 	hasStars: false,
+	embeddingId: CURRENT_MODEL,
 	...overrides,
 });
 
@@ -93,8 +97,6 @@ describe("buildAnchorSourceText", () => {
 });
 
 describe("buildMessageAnchor", () => {
-	const embedder = createDeterministicEmbeddingService({ dimensions: 8 });
-
 	const putRecord = async (
 		store: ReturnType<typeof createMemoryVectorStore>,
 		record: VectorRecord,
@@ -120,12 +122,12 @@ describe("buildMessageAnchor", () => {
 		});
 
 		const anchor = await buildMessageAnchor(
-			{ store, embedder },
+			{ store },
 			{ accountConfigId: "acct-1", anchorMessageId: "msg-1" },
 		);
 
 		assert.ok(anchor);
-		assert.equal(anchor.anchorEmbeddingId, embedder.embeddingId);
+		assert.equal(anchor.anchorEmbeddingId, CURRENT_MODEL);
 		assert.equal(anchor.anchorEmbedding.length, 8);
 		assert.ok(Math.abs(l2Norm(anchor.anchorEmbedding) - 1) < 1e-9);
 		assert.equal(
@@ -137,7 +139,7 @@ describe("buildMessageAnchor", () => {
 	it("returns null when the message has no indexed chunks", async () => {
 		const store = createMemoryVectorStore();
 		const anchor = await buildMessageAnchor(
-			{ store, embedder },
+			{ store },
 			{ accountConfigId: "acct-1", anchorMessageId: "absent" },
 		);
 		assert.equal(anchor, null);
@@ -156,9 +158,104 @@ describe("buildMessageAnchor", () => {
 		});
 
 		const anchor = await buildMessageAnchor(
-			{ store, embedder },
+			{ store },
 			{ accountConfigId: "acct-1", anchorMessageId: "msg-1" },
 		);
 		assert.equal(anchor, null);
 	});
+
+	it("stamps the model the pooled chunks were actually embedded under, not a caller-supplied current model (#349)", async () => {
+		const store = createMemoryVectorStore();
+		await putRecord(store, {
+			chunkId: "msg-1::subject",
+			vector: [1, 0, 0, 0, 0, 0, 0, 0],
+			metadata: baseMetadata({
+				chunkType: "subject",
+				embeddingId: "amazon.titan-embed-text-v1@1024",
+				textPreview: "booking confirmed",
+			}),
+		});
+
+		const anchor = await buildMessageAnchor(
+			{ store },
+			{ accountConfigId: "acct-1", anchorMessageId: "msg-1" },
+		);
+
+		assert.ok(anchor);
+		assert.equal(anchor.anchorEmbeddingId, "amazon.titan-embed-text-v1@1024");
+		assert.notEqual(anchor.anchorEmbeddingId, CURRENT_MODEL);
+	});
+
+	it("throws when a message's chunks disagree on which model produced them", async () => {
+		const store = createMemoryVectorStore();
+		await putRecord(store, {
+			chunkId: "msg-1::subject",
+			vector: [1, 0, 0, 0, 0, 0, 0, 0],
+			metadata: baseMetadata({
+				chunkType: "subject",
+				embeddingId: "model-a@8",
+				textPreview: "booking confirmed",
+			}),
+		});
+		await putRecord(store, {
+			chunkId: "msg-1::body-0",
+			vector: [0, 1, 0, 0, 0, 0, 0, 0],
+			metadata: baseMetadata({
+				chunkType: "body",
+				embeddingId: "model-b@8",
+				textPreview: "your trip is booked",
+			}),
+		});
+
+		await assert.rejects(
+			buildMessageAnchor(
+				{ store },
+				{ accountConfigId: "acct-1", anchorMessageId: "msg-1" },
+			),
+			/mismatch/,
+		);
+	});
+
+	it("treats chunks with no recorded embeddingId as an unknown model, never the current one", async () => {
+		const store = createMemoryVectorStore();
+		await putRecord(store, {
+			chunkId: "msg-1::subject",
+			vector: [1, 0, 0, 0, 0, 0, 0, 0],
+			metadata: baseMetadata({
+				chunkType: "subject",
+				embeddingId: undefined,
+				textPreview: "booking confirmed",
+			}),
+		});
+
+		const anchor = await buildMessageAnchor(
+			{ store },
+			{ accountConfigId: "acct-1", anchorMessageId: "msg-1" },
+		);
+
+		assert.ok(anchor);
+		assert.equal(anchor.anchorEmbeddingId, UNKNOWN_CHUNK_EMBEDDING_ID);
+	});
+});
+
+describe("deriveAnchorEmbeddingId", () => {
+	const record = (embeddingId: string | undefined): VectorRecord => ({
+		chunkId: "c",
+		vector: [1],
+		metadata: baseMetadata({ embeddingId }),
+	});
+
+	it("returns the shared id when every record agrees", () => {
+		assert.equal(
+			deriveAnchorEmbeddingId([record("a@1"), record("a@1")]),
+			"a@1",
+		);
+	});
+
+	it("throws on disagreement between a known id and an unknown one", () => {
+		assert.throws(
+			() => deriveAnchorEmbeddingId([record("a@1"), record(undefined)]),
+			/mismatch/,
+		);
+	});
 });

package/src/anchor.ts

@@ -1,5 +1,4 @@
 import type { VectorStoreService } from "./backends/memory.js";
-import type { EmbeddingService } from "./embeddings.js";
 import { buildTextPreview } from "./search.js";
 import type { ChunkType, VectorRecord } from "./types.js";
 
@@ -46,6 +45,40 @@ export const poolChunkVectors = (vectors: number[][]): number[] => {
 	return sum.map((value) => value / norm);
 };
 
+/**
+ * Stamped when none of a message's pooled chunks carry an `embeddingId` (RFC
+ * 039 / #349) — vectors written before that metadata field existed. Distinct
+ * from every real `EmbeddingService.embeddingId` value, so a drift check that
+ * compares against the current model ("does this anchor match what's
+ * configured now?") never mistakes an unknown-provenance vector for current;
+ * it is always treated as needing a fresh anchor.
+ */
+export const UNKNOWN_CHUNK_EMBEDDING_ID = "unknown";
+
+/**
+ * Derive the single `embeddingId` the pooled chunk records were actually
+ * embedded under (#349) — never the caller's currently-configured embedder,
+ * which may have changed since these chunks were last (re-)indexed. All
+ * chunks for one message are written by the same index-time pass, so they
+ * must agree; a chunk with no recorded `embeddingId` (pre-dating that field)
+ * counts as `UNKNOWN_CHUNK_EMBEDDING_ID`. Disagreement — including a mix of a
+ * known id and an unknown one — throws rather than silently picking one,
+ * consistent with `poolChunkVectors`'s "let it crash" stance on a dimension
+ * mismatch.
+ */
+export const deriveAnchorEmbeddingId = (records: VectorRecord[]): string => {
+	const ids = new Set(
+		records.map((r) => r.metadata.embeddingId ?? UNKNOWN_CHUNK_EMBEDDING_ID),
+	);
+	if (ids.size > 1) {
+		throw new Error(
+			`Chunk embeddingId mismatch pooling anchor from ${records.length} records: ${[...ids].join(", ")}`,
+		);
+	}
+	const [id] = ids;
+	return id;
+};
+
 // The chunk types whose text carries the semantic meaning of "messages like
 // this" — subject and body. Structured chunks (sender, recipient, attachment,
 // entities) are excluded from the re-embeddable source text; they add no signal
@@ -80,7 +113,6 @@ export const buildAnchorSourceText = (
 
 export interface AnchorBuildDeps {
 	store: Pick<VectorStoreService, "getByMessage">;
-	embedder: Pick<EmbeddingService, "embeddingId">;
 }
 
 export interface AnchorBuildParams {
@@ -97,8 +129,13 @@ export interface AnchorBuildParams {
  * `FilterAnchor` row (and leave `Filter.hasAnchor` false) rather than persist an
  * empty anchor.
  *
- * `anchorEmbeddingId` is the current embedder's identifier — the model the
- * chunk vectors were embedded under, which the indexing pipeline keeps current.
+ * `anchorEmbeddingId` is derived from the pooled records' own
+ * `metadata.embeddingId` (#349) — the model that actually produced those
+ * vectors, which is not necessarily the currently-configured embedder. A
+ * message indexed under an older model keeps its old-space vectors until
+ * something re-indexes it; stamping the current embedder's id there would
+ * mislabel a stale vector as current and permanently defeat any drift check
+ * that trusts the stamp.
  */
 export const buildMessageAnchor = async (
 	deps: AnchorBuildDeps,
@@ -116,7 +153,7 @@ export const buildMessageAnchor = async (
 
 	return {
 		anchorEmbedding: poolChunkVectors(records.map((r) => r.vector)),
-		anchorEmbeddingId: deps.embedder.embeddingId,
+		anchorEmbeddingId: deriveAnchorEmbeddingId(records),
 		anchorSourceText: buildAnchorSourceText(
 			records.map((r) => ({
 				chunkType: r.metadata.chunkType,

package/src/backends/pgvector.integ.test.ts

@@ -8,7 +8,7 @@
  * Gated behind RUN_INTEG_TESTS. Point PG_CONNECTION_URL at a database whose
  * `vector` extension is enabled (default: local remit_test).
  *
- *   npm run test:integ -w packages/search-service
+ *   npm run test:integ:pg -w packages/search-service
  */
 import assert from "node:assert";
 import { randomUUID } from "node:crypto";

package/src/backends/s3-vectors.test.ts

@@ -247,6 +247,58 @@ describe("S3VectorsBackend.getByMessage (GetVectors by deterministic key, no sca
 		);
 	});
 
+	it("round-trips a chunk's embeddingId (#349) so a pooling anchor can recover which model produced it", async () => {
+		const subjectMetadata = {
+			messageId: MESSAGE_ID,
+			threadId: "thread-1",
+			accountConfigId: "acct-1",
+			mailboxIds: ["mb-inbox"],
+			chunkType: "subject",
+			sentDate: 1_700_000_000,
+			isRead: false,
+			hasAttachment: false,
+			hasStars: false,
+			embeddingId: "amazon.titan-embed-text-v1@1024",
+		};
+		const bodyMetadata = {
+			messageId: MESSAGE_ID,
+			threadId: "thread-1",
+			accountConfigId: "acct-1",
+			mailboxIds: ["mb-inbox"],
+			chunkType: "body",
+			sentDate: 1_700_000_000,
+			isRead: false,
+			hasAttachment: false,
+			hasStars: false,
+		};
+		s3vMock.on(GetVectorsCommand).resolves({
+			vectors: [
+				{
+					key: `${MESSAGE_ID}::subject`,
+					data: { float32: [1, 0, 0] },
+					metadata: subjectMetadata,
+				},
+				{
+					key: `${MESSAGE_ID}::body-0`,
+					data: { float32: [0, 1, 0] },
+					metadata: bodyMetadata,
+				},
+			],
+		});
+
+		const records = await buildBackend().getByMessage(MESSAGE_ID);
+		const byId = new Map(records.map((r) => [r.chunkId, r]));
+
+		assert.equal(
+			byId.get(`${MESSAGE_ID}::subject`)?.metadata.embeddingId,
+			"amazon.titan-embed-text-v1@1024",
+		);
+		assert.equal(
+			byId.get(`${MESSAGE_ID}::body-0`)?.metadata.embeddingId,
+			undefined,
+		);
+	});
+
 	it("requests vector data and metadata, addressing vectors by candidate key", async () => {
 		s3vMock.on(GetVectorsCommand).resolves({ vectors: [] });
 

package/src/backends/s3-vectors.ts

@@ -165,6 +165,12 @@ const toMetadata = (raw: unknown): ChunkMetadata => {
 	// treat as absent (score-neutral re-rank), same as the other display fields.
 	const textPreview =
 		typeof obj.textPreview === "string" ? obj.textPreview : undefined;
+	// embeddingId is absent on vectors written before this provenance field
+	// existed (#349); treat as absent so a pooling consumer (buildMessageAnchor)
+	// can tell "unknown model" apart from a real id, rather than losing the real
+	// id this backend actually stored.
+	const embeddingId =
+		typeof obj.embeddingId === "string" ? obj.embeddingId : undefined;
 	return {
 		messageId: obj.messageId,
 		threadId: obj.threadId,
@@ -180,6 +186,7 @@ const toMetadata = (raw: unknown): ChunkMetadata => {
 		...(subject !== undefined ? { subject } : {}),
 		...(category !== undefined ? { category } : {}),
 		...(textPreview !== undefined ? { textPreview } : {}),
+		...(embeddingId !== undefined ? { embeddingId } : {}),
 	};
 };
 

package/src/backends/sqlite-vec.integ.test.ts

@@ -7,7 +7,7 @@
  * Gated behind RUN_INTEG_TESTS because it loads the native better-sqlite3 and
  * sqlite-vec binaries, matching the pgvector integration suite.
  *
- *   npm run test:integ -w packages/search-service
+ *   npm run test:integ:local -w packages/search-service
  */
 import assert from "node:assert";
 import { after, before, describe, test } from "node:test";

package/src/search.test.ts

@@ -459,6 +459,37 @@ describe("DefaultSearchService idempotent indexing", () => {
 	});
 });
 
+describe("chunk embeddingId provenance (#349)", () => {
+	it("prepareVectors stamps every chunk with the embedder that produced its vector", async () => {
+		const embedder = createDeterministicEmbeddingService({ dimensions: 128 });
+		const service = new DefaultSearchService({
+			embedder,
+			store: new MemoryVectorStore(),
+		});
+
+		const records = await service.prepareVectors(aliceParams(invoiceBody));
+
+		assert.ok(records.length > 0);
+		for (const record of records) {
+			assert.equal(record.metadata.embeddingId, embedder.embeddingId);
+		}
+	});
+
+	it("indexIncremental stamps every re-embedded chunk with the embedder that produced its vector", async () => {
+		const embedder = createDeterministicEmbeddingService({ dimensions: 128 });
+		const store = new MemoryVectorStore();
+		const service = new DefaultSearchService({ embedder, store });
+
+		await service.indexIncremental(aliceParams(invoiceBody));
+		const stored = await store.getByMessage("msg-alice");
+
+		assert.ok(stored.length > 0);
+		for (const record of stored) {
+			assert.equal(record.metadata.embeddingId, embedder.embeddingId);
+		}
+	});
+});
+
 class CountingEmbedder implements EmbeddingService {
 	readonly embeddingId: string;
 	readonly dimensions: number;

package/src/search.ts

@@ -240,6 +240,7 @@ export class DefaultSearchService implements SearchService {
 				...metadata,
 				chunkType: chunk.chunkType,
 				contentHash: computeContentHash(embeddingId, chunk.text),
+				embeddingId,
 				textPreview: buildTextPreview(chunk.text),
 				...(chunk.chunkType === "attachment" && fileTypes.length > 0
 					? { fileTypes }
@@ -335,6 +336,7 @@ export class DefaultSearchService implements SearchService {
 				...metadata,
 				chunkType: h.chunk.chunkType,
 				contentHash: h.contentHash,
+				embeddingId,
 				textPreview: buildTextPreview(h.chunk.text),
 				...(h.chunk.chunkType === "attachment" && fileTypes.length > 0
 					? { fileTypes }

package/src/semantic-search.integ.test.ts

@@ -8,7 +8,7 @@
  * Downloads the MiniLM model on first run, so it is gated behind RUN_INTEG_TESTS
  * and excluded from the default unit-test path.
  *
- *   npm run test:integ -w packages/search-service
+ *   npm run test:integ:model -w packages/search-service
  */
 import assert from "node:assert";
 import { mkdtempSync, rmSync } from "node:fs";

package/src/types.ts

@@ -48,6 +48,16 @@ export interface ChunkMetadata {
 	 * embedding model changes. Absent on pre-hash vectors (re-PUT once to populate).
 	 */
 	contentHash?: string;
+	/**
+	 * `<modelId>@<dimensions>` id of the embedder that produced this chunk's
+	 * vector, stored at index time (see `EmbeddingService.embeddingId`). Lets a
+	 * consumer that pools chunk vectors — `buildMessageAnchor` — recover which
+	 * model actually produced them instead of assuming the currently configured
+	 * embedder. Absent on vectors written before this field existed; re-embedded
+	 * organically on next touch like `contentHash`/`textPreview` above, no
+	 * backfill.
+	 */
+	embeddingId?: string;
 	/**
 	 * Prefix of the chunk's embeddable text, stored at index time and used for the
 	 * literal-match re-rank in search.ts. Bounded independently by char count