@camstack/system 1.2.47 → 1.2.48

package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs

@@ -1,4 +1,4 @@
-import { C as dataStoreProviderCapability, ct as BaseAddon, kt as parseJsonUnknown, l as RUNTIME_DEFAULTS, st as errMsg, vt as asJsonObject } from "../../dist-BRBfh1_9.mjs";
+import { C as cosineSimilarity, Nt as parseJsonUnknown, St as asJsonObject, T as decodeVectorBase64, dt as errMsg, ft as BaseAddon, l as RUNTIME_DEFAULTS, lt as vectorDimFromBase64, ut as vectorStoreCapability, w as dataStoreProviderCapability } from "../../dist-C7rPTQLR.mjs";
 import { randomUUID } from "node:crypto";
 import Database from "better-sqlite3";
 //#region src/builtins/sqlite-storage/filter-compiler.ts
@@ -641,6 +641,484 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 	}
 };
 //#endregion
+//#region src/builtins/sqlite-storage/vector-index.ts
+/**
+* SQLite-backed vector index — compact rows, exhaustive cosine ranking.
+*
+* Lives INSIDE the sqlite-settings builtin, against `SqliteSettingsBackend`
+* directly, and that placement is the point. It first shipped as its own addon
+* and that cost two round trips per write — caller → the `settings-store` door
+* → the engine ([D44](../../../../../docs/decisions/adr-0044.md)) — plus a
+* whole runner process, to reach the very database this addon already holds
+* open. Here an upsert is a function call.
+*
+* Replaceability is unaffected: it is the CAPABILITY that makes the backend
+* swappable, not the package. An ANN provider registers `vector-store` from
+* wherever it likes and `setActiveSingleton` picks it.
+*
+* This is the DEFAULT `vector-store` provider, not the final one. It exists to
+* make a row cheap; replacing exhaustive scan with a real ANN index is a
+* different provider registered against the same capability.
+*
+* What it changes versus the per-store JSON columns it replaces:
+*
+*  - the vector is a base64 Float32 TEXT column (~2.7 KB for 512 dims) instead
+*    of a JSON array of 512 numbers (~11.7 KB measured on the live hub), and
+*    decoding is a `readFloatLE` loop instead of `JSON.parse` of 512 tokens;
+*  - filterable metadata is promoted to real columns, so `deviceId` /
+*    `timestamp` / `className` prefilters are pushed into SQL and indexed
+*    rather than applied after loading everything;
+*  - a truncated scan is REPORTED (`truncated`), because a search that silently
+*    considered a subset reads as "nothing else matched".
+*
+* Metadata that is not one of the promoted columns rides in a JSON blob and is
+* returned verbatim; it cannot be filtered on. That is deliberate — a filter a
+* backend cannot push down is a filter that gets forgotten.
+*/
+/** Metadata keys promoted to real, indexable columns. */
+var PROMOTED_COLUMNS = [
+	"deviceId",
+	"timestamp",
+	"className",
+	"modelId"
+];
+/** Ceiling on rows compared in one query, so a pathological index cannot wedge the runner. */
+var MAX_SCAN_ROWS = 2e5;
+var collectionFor = (index) => `vector-store:${index}`;
+/**
+* Where index declarations live.
+*
+* They MUST be durable. The rows survive a restart in SQLite but the `dim` and
+* `metric` that make them queryable were only ever in memory, so after a hub
+* restart `query` and `stats` both threw "index was never declared" until some
+* caller happened to write — which meant semantic search was dead from boot
+* until the next track closed. Callers declare lazily (the dimension comes from
+* the active encoder, unknown at construction), so nothing re-declared on boot.
+*/
+var INDEX_REGISTRY_COLLECTION = "vector-store:__indexes";
+var SqliteVectorIndex = class {
+	store;
+	logger;
+	specs = /* @__PURE__ */ new Map();
+	constructor(deps) {
+		this.store = deps.store;
+		this.logger = deps.logger;
+	}
+	/**
+	* Load persisted index declarations. Call once at startup, BEFORE serving:
+	* without it every method on an existing index throws until a write
+	* re-declares it.
+	*/
+	async loadIndexRegistry() {
+		try {
+			await this.store.declareCollection({
+				collection: INDEX_REGISTRY_COLLECTION,
+				columns: [
+					{
+						name: "id",
+						type: "TEXT",
+						primaryKey: true,
+						notNull: true
+					},
+					{
+						name: "dim",
+						type: "INTEGER",
+						notNull: true
+					},
+					{
+						name: "metric",
+						type: "TEXT",
+						notNull: true
+					}
+				]
+			});
+			const rows = await this.store.query({ collection: INDEX_REGISTRY_COLLECTION });
+			for (const row of rows) {
+				const dim = row.data["dim"];
+				const metric = row.data["metric"];
+				if (typeof dim !== "number") continue;
+				this.specs.set(row.id, {
+					dim,
+					metric: isMetric(metric) ? metric : "cosine"
+				});
+			}
+			this.logger.info("vector-store index registry loaded", { meta: { indexes: this.specs.size } });
+		} catch (err) {
+			this.logger.error("vector-store index registry failed to load", { meta: { error: String(err) } });
+		}
+	}
+	/**
+	* Idempotent. Re-declaring an index with a DIFFERENT dim throws rather than
+	* wiping: silently dropping a populated index because a caller shipped a new
+	* model would destroy history that cannot be rebuilt.
+	*/
+	async declareIndex(index, dim, metric) {
+		const existing = this.specs.get(index);
+		if (existing !== void 0 && existing.dim !== dim) throw new Error(`vector-store: index "${index}" is declared with dim ${existing.dim}; refusing to redeclare as ${dim}`);
+		await this.store.declareCollection({
+			collection: collectionFor(index),
+			columns: [
+				{
+					name: "id",
+					type: "TEXT",
+					primaryKey: true,
+					notNull: true
+				},
+				{
+					name: "vector",
+					type: "TEXT",
+					notNull: true
+				},
+				{
+					name: "deviceId",
+					type: "INTEGER"
+				},
+				{
+					name: "timestamp",
+					type: "INTEGER"
+				},
+				{
+					name: "className",
+					type: "TEXT"
+				},
+				{
+					name: "modelId",
+					type: "TEXT"
+				},
+				{
+					name: "extra",
+					type: "JSON"
+				}
+			],
+			indexes: [
+				{
+					name: `idx_vec_${index}_dev_ts`,
+					columns: ["deviceId", "timestamp"]
+				},
+				{
+					name: `idx_vec_${index}_ts`,
+					columns: ["timestamp"]
+				},
+				{
+					name: `idx_vec_${index}_class`,
+					columns: ["className"]
+				}
+			]
+		});
+		this.specs.set(index, {
+			dim,
+			metric
+		});
+		try {
+			await this.store.set({
+				collection: INDEX_REGISTRY_COLLECTION,
+				key: index,
+				value: {
+					dim,
+					metric
+				}
+			});
+		} catch (err) {
+			this.logger.warn("vector-store index declaration not persisted", { meta: {
+				index,
+				dim,
+				error: String(err)
+			} });
+		}
+	}
+	/**
+	* Resolve an index's spec, recovering it from stored rows when the registry
+	* has no entry.
+	*
+	* The recovery is not belt-and-braces, it is required: indexes created before
+	* the registry existed have rows and no declaration, and so would be
+	* permanently unqueryable — which is exactly how this was found, with 8,494
+	* live vectors that `stats` refused to look at. The dimension is derivable
+	* from any stored vector, so derive it and persist it rather than demanding
+	* the caller redeclare. The metric is NOT derivable and falls back to
+	* `cosine`, which is what every current caller declares.
+	*/
+	async resolveSpec(index) {
+		const known = this.specs.get(index);
+		if (known !== void 0) return known;
+		const sample = (await this.store.query({
+			collection: collectionFor(index),
+			filter: { limit: 1 }
+		}))[0];
+		const vector = sample === void 0 ? void 0 : readRow(sample.data)?.vector;
+		if (vector === void 0) throw new Error(`vector-store: index "${index}" was never declared`);
+		const dim = vectorDimFromBase64(vector);
+		this.logger.warn("vector-store recovered an undeclared index from its rows", { meta: {
+			index,
+			dim,
+			metric: "cosine"
+		} });
+		await this.declareIndex(index, dim, "cosine");
+		return {
+			dim,
+			metric: "cosine"
+		};
+	}
+	/** Upsert items, refusing any whose vector length disagrees with the index. */
+	async upsert(index, items) {
+		const spec = await this.resolveSpec(index);
+		const collection = collectionFor(index);
+		let upserted = 0;
+		let rejected = 0;
+		for (const item of items) {
+			if (vectorDimFromBase64(item.vector) !== spec.dim) {
+				rejected += 1;
+				continue;
+			}
+			const { deviceId, timestamp, className, modelId, extra } = splitMetadata(item.metadata);
+			try {
+				await this.store.set({
+					collection,
+					key: item.id,
+					value: {
+						vector: item.vector,
+						deviceId,
+						timestamp,
+						className,
+						modelId,
+						extra
+					}
+				});
+				upserted += 1;
+			} catch (err) {
+				rejected += 1;
+				this.logger.warn("vector-store upsert failed", {
+					...deviceId !== null ? { tags: { deviceId } } : {},
+					meta: {
+						index,
+						id: item.id,
+						error: String(err)
+					}
+				});
+			}
+		}
+		if (rejected > 0) this.logger.warn("vector-store rejected items on upsert", { meta: {
+			index,
+			rejected,
+			expectedDim: spec.dim
+		} });
+		return {
+			upserted,
+			rejected
+		};
+	}
+	/** Exhaustive ranked search over the rows a prefilter admits. */
+	async query(params) {
+		const spec = await this.resolveSpec(params.index);
+		const queryVec = decodeVectorBase64(params.vector);
+		if (queryVec.length !== spec.dim) throw new Error(`vector-store: query vector has ${queryVec.length} dims, index "${params.index}" has ${spec.dim}`);
+		const rows = await this.store.query({
+			collection: collectionFor(params.index),
+			filter: {
+				...buildSqlFilter(params.filter),
+				orderBy: {
+					field: "timestamp",
+					direction: "desc"
+				},
+				limit: MAX_SCAN_ROWS
+			}
+		});
+		const minScore = params.minScore ?? Number.NEGATIVE_INFINITY;
+		const scored = [];
+		for (const row of rows) {
+			const stored = readRow(row.data);
+			if (stored === null) continue;
+			if (!passesUnpushedFilter(stored, params.filter)) continue;
+			const vec = decodeVectorBase64(stored.vector);
+			if (vec.length !== spec.dim) continue;
+			const score = scoreFor(spec.metric, queryVec, vec);
+			if (score < minScore) continue;
+			scored.push({
+				id: row.id,
+				score,
+				metadata: rebuildMetadata(stored)
+			});
+		}
+		scored.sort((a, b) => b.score - a.score);
+		return {
+			matches: scored.slice(0, params.topK),
+			scanned: rows.length,
+			truncated: rows.length >= MAX_SCAN_ROWS
+		};
+	}
+	/**
+	* Metadata for the given ids, WITHOUT decoding their vectors.
+	*
+	* The caller is a best-of gate comparing confidences, so the vector is dead
+	* weight on that path — reading it would reintroduce exactly the per-row cost
+	* this index exists to remove. Missing ids are absent from the result rather
+	* than present with an empty metadata object.
+	*/
+	async getByIds(index, ids) {
+		await this.resolveSpec(index);
+		if (ids.length === 0) return [];
+		const rows = await this.store.query({
+			collection: collectionFor(index),
+			filter: { limit: MAX_SCAN_ROWS }
+		});
+		const wanted = new Set(ids);
+		const out = [];
+		for (const row of rows) {
+			if (!wanted.has(row.id)) continue;
+			const stored = readRow(row.data);
+			if (stored === null) continue;
+			out.push({
+				id: row.id,
+				metadata: rebuildMetadata(stored)
+			});
+		}
+		return out;
+	}
+	async deleteByIds(index, ids) {
+		const collection = collectionFor(index);
+		let deleted = 0;
+		for (const id of ids) try {
+			await this.store.delete({
+				collection,
+				key: id
+			});
+			deleted += 1;
+		} catch (err) {
+			this.logger.warn("vector-store delete failed", { meta: {
+				index,
+				id,
+				error: String(err)
+			} });
+		}
+		return deleted;
+	}
+	async deleteByFilter(index, filter) {
+		const ids = (await this.store.query({
+			collection: collectionFor(index),
+			filter: {
+				...buildSqlFilter(filter),
+				limit: MAX_SCAN_ROWS
+			}
+		})).filter((r) => {
+			const stored = readRow(r.data);
+			return stored !== null && passesUnpushedFilter(stored, filter);
+		}).map((r) => r.id);
+		return this.deleteByIds(index, ids);
+	}
+	async stats(index) {
+		const spec = await this.resolveSpec(index);
+		return {
+			backend: "sqlite-bruteforce",
+			count: (await this.store.query({
+				collection: collectionFor(index),
+				filter: { limit: MAX_SCAN_ROWS }
+			})).length,
+			dim: spec.dim,
+			metric: spec.metric,
+			exact: true
+		};
+	}
+};
+/** Split caller metadata into promoted columns plus a verbatim remainder. */
+function splitMetadata(metadata) {
+	const extra = {};
+	for (const [key, value] of Object.entries(metadata)) if (!isPromoted(key)) extra[key] = value;
+	return {
+		deviceId: numberOrNull(metadata["deviceId"]),
+		timestamp: numberOrNull(metadata["timestamp"]),
+		className: stringOrNull(metadata["className"]),
+		modelId: stringOrNull(metadata["modelId"]),
+		extra
+	};
+}
+function isPromoted(key) {
+	for (const promoted of PROMOTED_COLUMNS) if (promoted === key) return true;
+	return false;
+}
+function numberOrNull(value) {
+	return typeof value === "number" ? value : null;
+}
+function stringOrNull(value) {
+	return typeof value === "string" ? value : null;
+}
+/** Rebuild the caller's metadata from the promoted columns plus the remainder. */
+function rebuildMetadata(row) {
+	const out = { ...row.extra };
+	if (row.deviceId !== null) out["deviceId"] = row.deviceId;
+	if (row.timestamp !== null) out["timestamp"] = row.timestamp;
+	if (row.className !== null) out["className"] = row.className;
+	if (row.modelId !== null) out["modelId"] = row.modelId;
+	return out;
+}
+/** The part of a filter SQLite can apply: promoted columns only. */
+function buildSqlFilter(filter) {
+	if (filter === void 0) return {};
+	const where = {};
+	const whereBetween = {};
+	for (const [key, value] of Object.entries(filter.equals ?? {})) if (isPromoted(key)) where[key] = value;
+	for (const [key, range] of Object.entries(filter.between ?? {})) if (isPromoted(key)) whereBetween[key] = range;
+	return {
+		...Object.keys(where).length > 0 ? { where } : {},
+		...Object.keys(whereBetween).length > 0 ? { whereBetween } : {}
+	};
+}
+/**
+* Apply the clauses SQL could not.
+*
+* A filter key that is not a promoted column lives inside the `extra` JSON
+* blob, so it is checked here in JS. It is applied rather than ignored — an
+* unapplied filter returns rows the caller will treat as matches.
+*/
+function passesUnpushedFilter(row, filter) {
+	if (filter === void 0) return true;
+	for (const [key, expected] of Object.entries(filter.equals ?? {})) {
+		if (isPromoted(key)) continue;
+		if (row.extra[key] !== expected) return false;
+	}
+	for (const [key, [lo, hi]] of Object.entries(filter.between ?? {})) {
+		if (isPromoted(key)) continue;
+		const actual = row.extra[key];
+		if (typeof actual !== "number" || actual < lo || actual > hi) return false;
+	}
+	return true;
+}
+function readRow(data) {
+	const vector = data["vector"];
+	if (typeof vector !== "string" || vector.length === 0) return null;
+	const extra = data["extra"];
+	return {
+		vector,
+		deviceId: numberOrNull(data["deviceId"]),
+		timestamp: numberOrNull(data["timestamp"]),
+		className: stringOrNull(data["className"]),
+		modelId: stringOrNull(data["modelId"]),
+		extra: isMetadata(extra) ? extra : {}
+	};
+}
+function isMetadata(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/** Narrow a stored metric string back to the enum. */
+function isMetric(value) {
+	return value === "cosine" || value === "dot" || value === "euclidean";
+}
+/** Score two vectors under the index's metric. Higher is nearer in every case. */
+function scoreFor(metric, a, b) {
+	if (metric === "cosine") return cosineSimilarity(a, b);
+	if (metric === "dot") {
+		let sum = 0;
+		for (let i = 0; i < a.length; i += 1) sum += (a[i] ?? 0) * (b[i] ?? 0);
+		return sum;
+	}
+	let sum = 0;
+	for (let i = 0; i < a.length; i += 1) {
+		const d = (a[i] ?? 0) - (b[i] ?? 0);
+		sum += d * d;
+	}
+	const distance = Math.sqrt(sum);
+	return distance === 0 ? 0 : -distance;
+}
+//#endregion
 //#region src/builtins/sqlite-storage/sqlite-settings.addon.ts
 /**
 * SQLite Settings addon — the relational ENGINE behind the data door.
@@ -651,9 +1129,16 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 * ([D44](../../../../../docs/decisions/adr-0044.md)). Callers reach this
 * engine only through the door — which is the point, because it is what
 * makes the engine replaceable.
+*
+* It ALSO serves `vector-store` (singleton, internal), from the same database
+* file and the same open handle. That capability first shipped as its own addon
+* and the separation cost two round trips per write — caller → door → engine —
+* plus a runner process, to reach a database this addon already has open. The
+* capability is what keeps the backend swappable; the package never was.
 */
 var SqliteSettingsAddon = class extends BaseAddon {
 	backend = null;
+	vectorIndex = null;
 	constructor() {
 		super({});
 	}
@@ -677,10 +1162,36 @@ var SqliteSettingsAddon = class extends BaseAddon {
 		this.ctx.logger.info("DB file status", { meta: { dbExists } });
 		this.backend = new SqliteSettingsBackend(dbPath, { ...RUNTIME_DEFAULTS });
 		await this.backend.initialize();
+		const vectorIndex = new SqliteVectorIndex({
+			store: this.backend,
+			logger: this.ctx.logger.child("VectorIndex")
+		});
+		await vectorIndex.loadIndexRegistry();
+		this.vectorIndex = vectorIndex;
+		const vectorProvider = {
+			declareIndex: async (input) => {
+				await vectorIndex.declareIndex(input.index, input.dim, input.metric);
+			},
+			upsert: async (input) => vectorIndex.upsert(input.index, input.items),
+			query: async (input) => vectorIndex.query({
+				index: input.index,
+				vector: input.vector,
+				topK: input.topK,
+				...input.minScore !== void 0 ? { minScore: input.minScore } : {},
+				...input.filter !== void 0 ? { filter: input.filter } : {}
+			}),
+			getByIds: async (input) => ({ items: await vectorIndex.getByIds(input.index, input.ids) }),
+			deleteByIds: async (input) => ({ deleted: await vectorIndex.deleteByIds(input.index, input.ids) }),
+			deleteByFilter: async (input) => ({ deleted: await vectorIndex.deleteByFilter(input.index, input.filter) }),
+			stats: async (input) => vectorIndex.stats(input.index)
+		};
 		this.ctx.logger.info("Initialized successfully");
 		return [{
 			capability: dataStoreProviderCapability,
 			provider: this.backend
+		}, {
+			capability: vectorStoreCapability,
+			provider: vectorProvider
 		}];
 	}
 	async onShutdown() {
@@ -689,6 +1200,10 @@ var SqliteSettingsAddon = class extends BaseAddon {
 	getBackend() {
 		return this.backend;
 	}
+	/** Exposed for tests; null before `onInitialize`. */
+	getVectorIndex() {
+		return this.vectorIndex;
+	}
 };
 //#endregion
 export { SqliteSettingsAddon, SqliteSettingsAddon as default, SqliteSettingsBackend as t };