@remit/search-service 0.0.9 → 0.0.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/search-service",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",

package/src/backends/sqlite-vec.integ.test.ts

@@ -142,3 +142,58 @@ describe("sqlite-vec store (integration)", { skip: !RUN }, () => {
 		assert.equal(hashes.size, 0);
 	});
 });
+
+/**
+ * The SQLITE_VEC_EXTENSION_PATH override branch — how the Alpine/musl backend
+ * image loads its own vec0.so instead of the npm package's glibc prebuilt. This
+ * suite runs on the glibc dev host, so it points the override at the npm
+ * package's own resolved loadable path: it proves the loader short-circuits
+ * getLoadablePath() and loads the extension via db.loadExtension(path), then
+ * drives the same vec0 read path (create table, upsert, cosine KNN) through it.
+ * The musl build of that .so is asserted to dlopen at image-build time in the
+ * Dockerfile's sqlite-vec-musl stage.
+ */
+describe("sqlite-vec store — SQLITE_VEC_EXTENSION_PATH override (integration)", {
+	skip: !RUN,
+}, () => {
+	let store: VectorStoreService;
+	let previous: string | undefined;
+
+	before(async () => {
+		previous = process.env.SQLITE_VEC_EXTENSION_PATH;
+		const { getLoadablePath } = await import("sqlite-vec");
+		process.env.SQLITE_VEC_EXTENSION_PATH = getLoadablePath();
+		store = createSqliteVectorStore({
+			path: ":memory:",
+			dimensions: DIMENSIONS,
+		});
+	});
+
+	after(async () => {
+		await store.close?.();
+		if (previous === undefined) {
+			delete process.env.SQLITE_VEC_EXTENSION_PATH;
+		} else {
+			process.env.SQLITE_VEC_EXTENSION_PATH = previous;
+		}
+	});
+
+	test("loads the extension from the override path and ranks by cosine", async () => {
+		await store.upsert([
+			record("o-x", [1, 0, 0, 0], { messageId: "m-ox", contentHash: "hx" }),
+			record("o-z", [0.9, 0.1, 0, 0], {
+				messageId: "m-oz",
+				contentHash: "hz",
+			}),
+			record("o-y", [0, 1, 0, 0], { messageId: "m-oy", contentHash: "hy" }),
+		]);
+
+		const matches = await store.query({ vector: [1, 0, 0, 0], topK: 3 });
+
+		assert.deepEqual(
+			matches.map((m) => m.chunkId),
+			["o-x", "o-z", "o-y"],
+		);
+		assert.ok(matches[0].score > 0.99);
+	});
+});

package/src/backends/sqlite-vec.ts

@@ -21,6 +21,24 @@ type SqliteVecModule = {
 	load: (db: Database) => void;
 };
 
+// `SQLITE_VEC_EXTENSION_PATH` overrides the npm `sqlite-vec` package's
+// getLoadablePath() with a pre-built loadable extension. That package resolves a
+// glibc-only prebuilt (`sqlite-vec-<platform>/vec0.so`) which cannot dlopen on
+// the Alpine/musl backend image, so that image bakes a musl-compiled `vec0.so`
+// and points this at it. better-sqlite3 derives the entry point from the
+// filename, and the `vec0` basename resolves to `sqlite3_vec_init` (SQLite drops
+// the digit). Unset keeps the npm resolution — the glibc search-index-worker
+// image sets nothing and loads the package unchanged.
+const loadSqliteVec = async (db: Database): Promise<void> => {
+	const overridePath = process.env.SQLITE_VEC_EXTENSION_PATH;
+	if (overridePath) {
+		db.loadExtension(overridePath);
+		return;
+	}
+	const sqliteVec = await runtimeImport<SqliteVecModule>("sqlite-vec");
+	sqliteVec.load(db);
+};
+
 /**
  * vec0 stores each chunk's vector alongside the scalar fields the query path
  * filters on, so equality / range filters are pushed into the KNN instead of
@@ -127,13 +145,12 @@ export const createSqliteVectorStore = (
 		dbPromise = (async () => {
 			const { default: Database } =
 				await runtimeImport<BetterSqlite3Module>("better-sqlite3");
-			const sqliteVec = await runtimeImport<SqliteVecModule>("sqlite-vec");
 			if (config.path !== ":memory:") {
 				mkdirSync(dirname(config.path), { recursive: true });
 			}
 			const db = new Database(config.path);
 			db.pragma("journal_mode = WAL");
-			sqliteVec.load(db);
+			await loadSqliteVec(db);
 			db.exec(CREATE_TABLE(dimensions));
 			return db;
 		})();

package/src/types.ts

@@ -40,6 +40,8 @@ export interface ChunkMetadata {
 	subject?: string;
 	/** Header-derived category. Stored at index time; absent for pre-enrichment vectors. */
 	category?: MessageCategory;
+	/** Normalized `List-Id` header value. Stored at index time so the semantic-arm literal refine can match a `ListId` clause; absent for pre-enrichment vectors or non-list mail. */
+	listId?: string;
 	/**
 	 * sha256 over the embedding model/version id and the chunk's embeddable text.
 	 * Lets a re-index skip an unchanged chunk and re-embed only when content or the