@remit/search-service 0.0.20 → 0.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-service",
-  "version": "0.0.20",
+  "version": "0.0.21",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/backends/sqlite-vec.integ.test.ts

@@ -10,10 +10,16 @@
  *   npm run test:integ:local -w packages/search-service
  */
 import assert from "node:assert";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { after, before, describe, test } from "node:test";
 import type { ChunkMetadata, VectorRecord } from "../types.js";
 import type { VectorStoreService } from "./memory.js";
-import { createSqliteVectorStore } from "./sqlite-vec.js";
+import {
+	createSqliteVectorStore,
+	readSqliteIndexProvenance,
+} from "./sqlite-vec.js";
 
 const RUN = process.env.RUN_INTEG_TESTS === "1";
 const DIMENSIONS = 4;
@@ -197,3 +203,63 @@ describe("sqlite-vec store — SQLITE_VEC_EXTENSION_PATH override (integration)"
 		assert.ok(matches[0].score > 0.99);
 	});
 });
+
+// #455. The report reads the stored index rather than the store's query path,
+// so it has to be proved against a real vec0 table on disk: an in-memory store
+// would never show that a report can read a file it did not create.
+describe("index provenance (integration)", { skip: !RUN }, () => {
+	const dir = mkdtempSync(join(tmpdir(), "remit-index-report-"));
+	const path = join(dir, "vec.db");
+	let store: VectorStoreService;
+
+	before(async () => {
+		store = createSqliteVectorStore({ path, dimensions: DIMENSIONS });
+		await store.upsert([
+			record("p-1", [1, 0, 0, 0], {
+				messageId: "m-1",
+				embeddingId: "current@4",
+			}),
+			record("p-2", [0, 1, 0, 0], {
+				messageId: "m-1",
+				embeddingId: "current@4",
+			}),
+			record("p-3", [0, 0, 1, 0], {
+				messageId: "m-2",
+				embeddingId: "older@4",
+			}),
+			record("p-4", [0, 0, 0, 1], { messageId: "m-3" }),
+		]);
+	});
+
+	after(async () => {
+		await store.close?.();
+		rmSync(dir, { recursive: true, force: true });
+	});
+
+	test("counts the stored vectors by the embedder that wrote them", async () => {
+		const report = await readSqliteIndexProvenance({
+			path,
+			configuredEmbeddingId: "current@4",
+		});
+		assert.equal(report.chunks, 4);
+		assert.equal(report.messages, 3);
+		assert.deepEqual(report.groups, [
+			{ embeddingId: "current@4", chunks: 2, messages: 1, current: true },
+			{ embeddingId: "older@4", chunks: 1, messages: 1, current: false },
+			{ embeddingId: "unknown", chunks: 1, messages: 1, current: false },
+		]);
+	});
+
+	// A box that has never indexed anything must not grow a vector database from
+	// being asked about one.
+	test("reports an empty index without creating the database", async () => {
+		const absent = join(dir, "missing.db");
+		const report = await readSqliteIndexProvenance({
+			path: absent,
+			configuredEmbeddingId: "current@4",
+		});
+		assert.deepEqual(report.groups, []);
+		assert.equal(report.chunks, 0);
+		assert.equal(existsSync(absent), false);
+	});
+});

package/src/backends/sqlite-vec.ts

@@ -1,6 +1,11 @@
-import { mkdirSync } from "node:fs";
+import { existsSync, mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import type SqliteDatabase from "better-sqlite3";
+import {
+	type IndexedChunkProvenance,
+	type IndexProvenance,
+	summarizeIndexProvenance,
+} from "../index-report.js";
 import type {
 	ChunkMetadata,
 	VectorMatch,
@@ -14,7 +19,7 @@ import { runtimeImport } from "./runtime-import.js";
 type Database = SqliteDatabase.Database;
 
 type BetterSqlite3Module = {
-	default: new (path: string) => Database;
+	default: new (path: string, options?: SqliteDatabase.Options) => Database;
 };
 
 type SqliteVecModule = {
@@ -127,6 +132,63 @@ const buildFilterClause = (
 	return { sql: sql.length > 0 ? ` AND ${sql.join(" AND ")}` : "", params };
 };
 
+/**
+ * Count the stored index by the embedder that wrote each vector (#455), read
+ * straight off the vec0 table rather than through the store: a report is not a
+ * search, and the caller has no query to run.
+ *
+ * Reads only. `fileMustExist` keeps a report on a box that has never indexed
+ * anything from creating the vector database as a side effect of asking about
+ * it, and an absent vec0 table — the window between the first boot and the
+ * first upsert — is an empty index rather than an error. The connection is not
+ * opened read-only: these files are WAL, and a read-only connection cannot
+ * initialize the shared-memory index when no writer is attached.
+ *
+ * The rows are streamed into the summary. One row per chunk means several per
+ * message, and a full mailbox's worth of metadata JSON must not be materialized
+ * to be counted.
+ */
+export const readSqliteIndexProvenance = async (config: {
+	path: string;
+	configuredEmbeddingId: string;
+}): Promise<IndexProvenance> => {
+	if (!existsSync(config.path)) {
+		return summarizeIndexProvenance(config.configuredEmbeddingId, []);
+	}
+	const { default: Database } =
+		await runtimeImport<BetterSqlite3Module>("better-sqlite3");
+	const db = new Database(config.path, { fileMustExist: true });
+	try {
+		await loadSqliteVec(db);
+		const table = db
+			.prepare(
+				"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'vec_chunks'",
+			)
+			.get();
+		if (!table) {
+			return summarizeIndexProvenance(config.configuredEmbeddingId, []);
+		}
+		const rows = db
+			.prepare("SELECT message_id AS messageId, meta FROM vec_chunks")
+			.iterate() as IterableIterator<{ messageId: string; meta: string }>;
+		return summarizeIndexProvenance(
+			config.configuredEmbeddingId,
+			provenanceOf(rows),
+		);
+	} finally {
+		db.close();
+	}
+};
+
+function* provenanceOf(
+	rows: Iterable<{ messageId: string; meta: string }>,
+): Generator<IndexedChunkProvenance> {
+	for (const row of rows) {
+		const metadata = JSON.parse(row.meta) as ChunkMetadata;
+		yield { messageId: row.messageId, embeddingId: metadata.embeddingId };
+	}
+}
+
 export interface SqliteVectorStoreConfig {
 	path: string;
 	dimensions?: number;

package/src/index-report.test.ts

@@ -0,0 +1,129 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import {
+	formatIndexProvenance,
+	type IndexedChunkProvenance,
+	summarizeIndexProvenance,
+} from "./index-report.js";
+
+const CURRENT = "local:multilingual-MiniLM:q8@384";
+const OLDER = "local:MiniLM@384";
+
+const chunk = (
+	messageId: string,
+	embeddingId: string | undefined,
+): IndexedChunkProvenance => ({ messageId, embeddingId });
+
+describe("summarizeIndexProvenance", () => {
+	it("counts vectors and the messages they belong to per embedder", () => {
+		const report = summarizeIndexProvenance(CURRENT, [
+			chunk("m1", CURRENT),
+			chunk("m1", CURRENT),
+			chunk("m2", OLDER),
+		]);
+		assert.equal(report.chunks, 3);
+		assert.equal(report.messages, 2);
+		assert.deepEqual(report.groups, [
+			{ embeddingId: CURRENT, chunks: 2, messages: 1, current: true },
+			{ embeddingId: OLDER, chunks: 1, messages: 1, current: false },
+		]);
+	});
+
+	// The state the report exists to make visible: a switched embedder leaves the
+	// index holding two vector spaces, and every query compares them as one.
+	it("separates the configured embedder from every other one", () => {
+		const report = summarizeIndexProvenance(CURRENT, [
+			chunk("m1", OLDER),
+			chunk("m2", OLDER),
+			chunk("m3", CURRENT),
+		]);
+		assert.deepEqual(
+			report.groups.map((group) => [group.embeddingId, group.current]),
+			[
+				[OLDER, false],
+				[CURRENT, true],
+			],
+		);
+	});
+
+	// Vectors written before metadata carried an embeddingId are a third state,
+	// not "the current model": nothing recorded what wrote them.
+	it("buckets vectors with no recorded embedder as unknown", () => {
+		const report = summarizeIndexProvenance(CURRENT, [
+			chunk("m1", undefined),
+			chunk("m2", CURRENT),
+		]);
+		const unknown = report.groups.find(
+			(group) => group.embeddingId === "unknown",
+		);
+		assert.deepEqual(unknown, {
+			embeddingId: "unknown",
+			chunks: 1,
+			messages: 1,
+			current: false,
+		});
+	});
+
+	// Two runs over an unchanged index have to print the same report, or a diff
+	// between them is noise rather than a change.
+	it("orders groups by size and then by id", () => {
+		const report = summarizeIndexProvenance(CURRENT, [
+			chunk("m1", "b"),
+			chunk("m2", "a"),
+			chunk("m3", "c"),
+			chunk("m4", "c"),
+		]);
+		assert.deepEqual(
+			report.groups.map((group) => group.embeddingId),
+			["c", "a", "b"],
+		);
+	});
+
+	it("reports an empty index as empty rather than as unknown", () => {
+		const report = summarizeIndexProvenance(CURRENT, []);
+		assert.equal(report.chunks, 0);
+		assert.equal(report.messages, 0);
+		assert.deepEqual(report.groups, []);
+	});
+});
+
+describe("formatIndexProvenance", () => {
+	it("names the configured embedder and every one in the index", () => {
+		const text = formatIndexProvenance(
+			summarizeIndexProvenance(CURRENT, [
+				chunk("m1", CURRENT),
+				chunk("m2", OLDER),
+			]),
+		);
+		assert.match(text, new RegExp(`Configured embedder: ${CURRENT}`));
+		assert.match(text, /2 vectors over 2 messages/);
+		assert.match(text, /\(current\)/);
+		assert.match(text, /\(older model\)/);
+	});
+
+	// An empty index and an index on a stale model both answer every query with
+	// nothing, so the report has to say which one this is.
+	it("says an empty index is empty", () => {
+		const text = formatIndexProvenance(summarizeIndexProvenance(CURRENT, []));
+		assert.match(text, /Nothing is indexed/);
+	});
+
+	it("counts the messages a switched embedder left behind", () => {
+		const text = formatIndexProvenance(
+			summarizeIndexProvenance(CURRENT, [
+				chunk("m1", OLDER),
+				chunk("m2", OLDER),
+				chunk("m3", CURRENT),
+			]),
+		);
+		assert.match(text, /2 messages carry vectors from an embedder/);
+	});
+
+	it("says nothing about older models when every vector is current", () => {
+		const text = formatIndexProvenance(
+			summarizeIndexProvenance(CURRENT, [chunk("m1", CURRENT)]),
+		);
+		assert.ok(!text.includes("older model"), text);
+		assert.ok(!text.includes("no longer configured"), text);
+	});
+});

package/src/index-report.ts

@@ -0,0 +1,129 @@
+import { UNKNOWN_CHUNK_EMBEDDING_ID } from "./anchor.js";
+
+/**
+ * One indexed chunk, reduced to the two facts a provenance report is about: the
+ * message it belongs to, and the embedder that produced its vector. Vectors
+ * written before `metadata.embeddingId` existed carry no id at all (RFC 039 /
+ * #349), which is a third state and not "the current model".
+ */
+export interface IndexedChunkProvenance {
+	messageId: string;
+	embeddingId: string | undefined;
+}
+
+export interface IndexProvenanceGroup {
+	embeddingId: string;
+	chunks: number;
+	messages: number;
+	/** Whether this group's vectors came from the embedder configured now. */
+	current: boolean;
+}
+
+export interface IndexProvenance {
+	configuredEmbeddingId: string;
+	chunks: number;
+	messages: number;
+	groups: IndexProvenanceGroup[];
+}
+
+/**
+ * Count an index by the embedder its vectors came from, against the one this
+ * deployment is configured with (#455). Switching the embedder re-embeds a
+ * message only when something touches it, so an index that has seen a switch
+ * holds two vector spaces whose distances are not comparable — a silent ranking
+ * loss that no query result distinguishes from having nothing indexed.
+ *
+ * The chunks are an `Iterable` rather than an array so a caller can stream a
+ * cursor through it: an index is one row per chunk, several per message, and a
+ * corpus-sized report must not materialize all of them to count them.
+ */
+export const summarizeIndexProvenance = (
+	configuredEmbeddingId: string,
+	chunks: Iterable<IndexedChunkProvenance>,
+): IndexProvenance => {
+	const groups = new Map<string, { chunks: number; messages: Set<string> }>();
+	const messages = new Set<string>();
+	let total = 0;
+	for (const chunk of chunks) {
+		const embeddingId = chunk.embeddingId ?? UNKNOWN_CHUNK_EMBEDDING_ID;
+		const group = groups.get(embeddingId) ?? {
+			chunks: 0,
+			messages: new Set<string>(),
+		};
+		group.chunks += 1;
+		group.messages.add(chunk.messageId);
+		groups.set(embeddingId, group);
+		messages.add(chunk.messageId);
+		total += 1;
+	}
+	return {
+		configuredEmbeddingId,
+		chunks: total,
+		messages: messages.size,
+		groups: [...groups]
+			.map(([embeddingId, group]) => ({
+				embeddingId,
+				chunks: group.chunks,
+				messages: group.messages.size,
+				current: embeddingId === configuredEmbeddingId,
+			}))
+			// Largest first, then by id, so two runs over an unchanged index print
+			// the same report and a diff between two runs is a real change.
+			.sort(
+				(a, b) =>
+					b.chunks - a.chunks || a.embeddingId.localeCompare(b.embeddingId),
+			),
+	};
+};
+
+const plural = (count: number, noun: string): string =>
+	`${count} ${noun}${count === 1 ? "" : "s"}`;
+
+/**
+ * The report as an operator reads it, one fact per line. Every line states what
+ * it means for search rather than leaving a number to be interpreted: a count
+ * under an older model is vectors that still answer queries, from a space of
+ * their own.
+ */
+export const formatIndexProvenance = (report: IndexProvenance): string => {
+	const lines = [
+		`Configured embedder: ${report.configuredEmbeddingId}`,
+		`Indexed:             ${plural(report.chunks, "vector")} over ${plural(
+			report.messages,
+			"message",
+		)}`,
+	];
+	if (report.groups.length === 0) {
+		lines.push(
+			"",
+			"Nothing is indexed. Semantic filters and the Organize semantic widen match",
+			"nothing until the worker has embedded this mailbox.",
+		);
+		return `${lines.join("\n")}\n`;
+	}
+	lines.push("", "By the embedder that wrote them:");
+	for (const group of report.groups) {
+		const mark = group.current ? "current" : "older model";
+		lines.push(
+			`  ${group.embeddingId}  ${plural(group.chunks, "vector")} over ${plural(
+				group.messages,
+				"message",
+			)} (${mark})`,
+		);
+	}
+	const stale = report.groups.filter((group) => !group.current);
+	if (stale.length > 0) {
+		const staleMessages = stale.reduce(
+			(total, group) => total + group.messages,
+			0,
+		);
+		lines.push(
+			"",
+			`${plural(staleMessages, "message")} carry vectors from an embedder this`,
+			"deployment is no longer configured with. They are compared against current",
+			"vectors as though they shared a vector space, which they do not, and they",
+			"are re-embedded only when something touches the message.",
+		);
+	}
+	return `${lines.join("\n")}\n`;
+};

package/src/index.ts

@@ -44,6 +44,13 @@ export {
 	type LocalEmbeddingConfig,
 	LocalEmbeddingService,
 } from "./embeddings.js";
+export {
+	formatIndexProvenance,
+	type IndexedChunkProvenance,
+	type IndexProvenance,
+	type IndexProvenanceGroup,
+	summarizeIndexProvenance,
+} from "./index-report.js";
 export {
 	createSearchService,
 	DefaultSearchService,

package/src/sqlite-vec.ts

@@ -1,4 +1,5 @@
 export {
 	createSqliteVectorStore,
+	readSqliteIndexProvenance,
 	type SqliteVectorStoreConfig,
 } from "./backends/sqlite-vec.js";