@camstack/system 1.2.389 → 1.2.391

package/dist/builtins/sqlite-storage/table-sizes.d.ts

@@ -1,92 +1,92 @@
 /**
- * Which collection IS the database.
- *
- * ## Why this exists
- *
- * The boot line already says how big the file is (`dbSizeMb`) and how much of
- * it is real (`freePct`). Measured on the live hub 2026-08-25 those read
- * `dbSizeMb=858 liveMb=840 freeMb=18 freePct=2` — so the growth from 153 MB
- * (08-08) is not a high-water mark waiting for a VACUUM, it is live rows. That
- * turned "should we VACUUM" into "which table", and nothing in this system
- * could answer it.
- *
- * Nothing could, structurally: per-addon scoped collections
- * (`<addonId>:<canonical>`) are created on the first write and declared in no
- * source file, so the real set only exists on the running node. A reference
- * page cannot list them and a grep cannot find them.
- *
- * ## Why `dbstat` and not `COUNT(*)`
- *
- * `dbstat` reports the pages a table actually occupies, which is the question —
- * a table with few enormous rows and a table with many small ones are the same
- * problem when the file is the thing that grew. It walks the b-tree structure
- * without decoding row payloads, so it costs a page traversal rather than a
- * full table scan. It is still real work on an 840 MB file, which is why the
- * caller runs it ONCE, well after boot, off the critical path — never on the
- * path a config read waits behind.
- */
+* Which collection IS the database.
+*
+* ## Why this exists
+*
+* The boot line already says how big the file is (`dbSizeMb`) and how much of
+* it is real (`freePct`). Measured on the live hub 2026-08-25 those read
+* `dbSizeMb=858 liveMb=840 freeMb=18 freePct=2` — so the growth from 153 MB
+* (08-08) is not a high-water mark waiting for a VACUUM, it is live rows. That
+* turned "should we VACUUM" into "which table", and nothing in this system
+* could answer it.
+*
+* Nothing could, structurally: per-addon scoped collections
+* (`<addonId>:<canonical>`) are created on the first write and declared in no
+* source file, so the real set only exists on the running node. A reference
+* page cannot list them and a grep cannot find them.
+*
+* ## Why `dbstat` and not `COUNT(*)`
+*
+* `dbstat` reports the pages a table actually occupies, which is the question —
+* a table with few enormous rows and a table with many small ones are the same
+* problem when the file is the thing that grew. It walks the b-tree structure
+* without decoding row payloads, so it costs a page traversal rather than a
+* full table scan. It is still real work on an 840 MB file, which is why the
+* caller runs it ONCE, well after boot, off the critical path — never on the
+* path a config read waits behind.
+*/
 /** What kind of object a `dbstat` group is. An index is an object too. */
-export type SqliteObjectKind = 'table' | 'index' | 'unknown';
+export type SqliteObjectKind = "table" | "index" | "unknown";
 /** One `dbstat` group: an object and the space it occupies. */
 export interface TableSizeRow {
-    readonly name: string;
-    /**
-     * `sqlite_master.tbl_name` — for a table this IS the table, for an index it
-     * is the table the index belongs to. Ownership is read from the relation
-     * SQLite already stores; an index name carries none (`idx_b_outbox` only
-     * looks parseable, and `sqlite_autoindex_a:media_1` would parse to nonsense).
-     */
-    readonly tableName: string;
-    readonly bytes: number;
-    readonly pages: number;
-    readonly kind: SqliteObjectKind;
+	readonly name: string;
+	/**
+	* `sqlite_master.tbl_name` — for a table this IS the table, for an index it
+	* is the table the index belongs to. Ownership is read from the relation
+	* SQLite already stores; an index name carries none (`idx_b_outbox` only
+	* looks parseable, and `sqlite_autoindex_a:media_1` would parse to nonsense).
+	*/
+	readonly tableName: string;
+	readonly bytes: number;
+	readonly pages: number;
+	readonly kind: SqliteObjectKind;
 }
 /** One line of the report. */
 export interface TableSizeEntry {
-    readonly name: string;
-    readonly sizeMb: number;
-    readonly pages: number;
-    /** Share of the whole file, percent. 0 when the file size is unknown. */
-    readonly pct: number;
-    readonly kind: SqliteObjectKind;
+	readonly name: string;
+	readonly sizeMb: number;
+	readonly pages: number;
+	/** Share of the whole file, percent. 0 when the file size is unknown. */
+	readonly pct: number;
+	readonly kind: SqliteObjectKind;
 }
 /** Everything one addon prefix owns, tables and their indexes together. */
 export interface OwnerRollup {
-    readonly owner: string;
-    readonly sizeMb: number;
-    readonly objects: number;
+	readonly owner: string;
+	readonly sizeMb: number;
+	readonly objects: number;
 }
 export interface TableSizeReport {
-    readonly tables: readonly TableSizeEntry[];
-    /** Megabytes held by everything below the cut. */
-    readonly otherMb: number;
-    /** How many OBJECTS are behind `otherMb` — tables and indexes alike. */
-    readonly otherObjects: number;
-    readonly tableCount: number;
-    readonly indexCount: number;
-    /** Every owner, biggest first. */
-    readonly byOwner: readonly OwnerRollup[];
+	readonly tables: readonly TableSizeEntry[];
+	/** Megabytes held by everything below the cut. */
+	readonly otherMb: number;
+	/** How many OBJECTS are behind `otherMb` — tables and indexes alike. */
+	readonly otherObjects: number;
+	readonly tableCount: number;
+	readonly indexCount: number;
+	/** Every owner, biggest first. */
+	readonly byOwner: readonly OwnerRollup[];
 }
 /** The label for anything that is not a per-addon scoped collection. */
 export declare const UNSCOPED_OWNER = "(unscoped)";
 /**
- * Rank tables by the space they hold and keep the top `limit`.
- *
- * The tail is REPORTED, never dropped: a top-N that silently discards the rest
- * reads as "these N are the database", which is exactly the wrong conclusion
- * when the remainder is a third of the file.
- */
+* Rank tables by the space they hold and keep the top `limit`.
+*
+* The tail is REPORTED, never dropped: a top-N that silently discards the rest
+* reads as "these N are the database", which is exactly the wrong conclusion
+* when the remainder is a third of the file.
+*/
 export declare function summariseTableSizes(rows: readonly TableSizeRow[], totalBytes: number, limit: number): TableSizeReport;
 /**
- * How long after opening the connection the report runs.
- *
- * Not zero and not "on every boot forever at startup": boot is already the
- * busiest window this engine sees (`ensureTable` x9, the retired-key purge, the
- * vector registry), and a page traversal of an 840 MB file placed there would
- * be indistinguishable from the stall this whole subsystem was just fixed for.
- * Sixty seconds puts it after the storm, once per process.
- */
-export declare const STORAGE_REPORT_DELAY_MS = 60000;
+* How long after opening the connection the report runs.
+*
+* Not zero and not "on every boot forever at startup": boot is already the
+* busiest window this engine sees (`ensureTable` x9, the retired-key purge, the
+* vector registry), and a page traversal of an 840 MB file placed there would
+* be indistinguishable from the stall this whole subsystem was just fixed for.
+* Sixty seconds puts it after the storm, once per process.
+*/
+export declare const STORAGE_REPORT_DELAY_MS = 6e4;
 /** Top-N tables named individually; the remainder is reported as a tail. */
 export declare const STORAGE_REPORT_TOP_N = 10;
 /** Owners named individually in the log line. */

package/dist/builtins/sqlite-storage/vector-index-shared.d.ts

@@ -1,72 +1,81 @@
-import { VectorFilter, VectorMetadata, VectorMetric } from '@camstack/types';
+/**
+* Shapes shared by the vector index and its provider.
+*
+* There is ONE index implementation on purpose (`SqliteVecVectorIndex`). A
+* second one behind a flag would mean two code paths that can disagree and a
+* silent fallback that turns "the extension is missing" into "search is
+* mysteriously slow". Replaceability lives at the CAPABILITY — another provider
+* takes over wholesale — not inside this module.
+*/
+import type { VectorFilter, VectorMetadata, VectorMetric } from "@camstack/types";
 /** Metadata keys promoted to real, indexable columns in every backend. */
 export declare const PROMOTED_COLUMNS: readonly ["deviceId", "timestamp", "className", "modelId"];
 /** A ranked hit as a backend returns it. */
 export interface VectorMatchRow {
-    readonly id: string;
-    /** Similarity, higher is nearer — NOT a distance. Backends convert. */
-    readonly score: number;
-    readonly metadata: VectorMetadata;
+	readonly id: string;
+	/** Similarity, higher is nearer — NOT a distance. Backends convert. */
+	readonly score: number;
+	readonly metadata: VectorMetadata;
 }
 /**
- * What one ranked search returned, INCLUDING the `k` it really ran with.
- *
- * `effectiveTopK` is part of the answer rather than a log line for the reason
- * `vector-store.cap.ts` records: a ceiling the caller learns about only from
- * the provider's own log is a number truncated in silence.
- */
+* What one ranked search returned, INCLUDING the `k` it really ran with.
+*
+* `effectiveTopK` is part of the answer rather than a log line for the reason
+* `vector-store.cap.ts` records: a ceiling the caller learns about only from
+* the provider's own log is a number truncated in silence.
+*/
 export interface VectorQueryOutcome {
-    readonly matches: VectorMatchRow[];
-    readonly scanned: number;
-    readonly truncated: boolean;
-    readonly effectiveTopK: number;
+	readonly matches: VectorMatchRow[];
+	readonly scanned: number;
+	readonly truncated: boolean;
+	readonly effectiveTopK: number;
 }
 /** One row read back WITH its vector, base64 Float32LE. */
 export interface VectorFetchRow {
-    readonly id: string;
-    readonly vector: string;
-    readonly metadata: VectorMetadata;
+	readonly id: string;
+	readonly vector: string;
+	readonly metadata: VectorMetadata;
 }
 /** One page of an unranked walk. `nextCursor: null` means end of index. */
 export interface VectorScanPage {
-    readonly items: Array<{
-        id: string;
-        metadata: VectorMetadata;
-    }>;
-    readonly nextCursor: number | null;
+	readonly items: Array<{
+		id: string;
+		metadata: VectorMetadata;
+	}>;
+	readonly nextCursor: number | null;
 }
 export interface VectorIndexStats {
-    readonly backend: string;
-    readonly count: number;
-    readonly dim: number;
-    readonly metric: VectorMetric;
-    readonly exact: boolean;
+	readonly backend: string;
+	readonly count: number;
+	readonly dim: number;
+	readonly metric: VectorMetric;
+	readonly exact: boolean;
 }
 export interface VectorIndexBackend {
-    loadIndexRegistry(): Promise<void>;
-    declareIndex(index: string, dim: number, metric: VectorMetric): Promise<void>;
-    upsert(index: string, items: readonly {
-        id: string;
-        vector: string;
-        metadata: VectorMetadata;
-    }[]): Promise<{
-        upserted: number;
-        rejected: number;
-    }>;
-    query(params: {
-        index: string;
-        vector: string;
-        topK: number;
-        minScore?: number;
-        filter?: VectorFilter;
-    }): Promise<VectorQueryOutcome>;
-    getByIds(index: string, ids: readonly string[]): Promise<Array<{
-        id: string;
-        metadata: VectorMetadata;
-    }>>;
-    fetchByIds(index: string, ids: readonly string[]): Promise<VectorFetchRow[]>;
-    scan(index: string, cursor: number, limit: number): Promise<VectorScanPage>;
-    deleteByIds(index: string, ids: readonly string[]): Promise<number>;
-    deleteByFilter(index: string, filter: VectorFilter): Promise<number>;
-    stats(index: string): Promise<VectorIndexStats>;
+	loadIndexRegistry(): Promise<void>;
+	declareIndex(index: string, dim: number, metric: VectorMetric): Promise<void>;
+	upsert(index: string, items: readonly {
+		id: string;
+		vector: string;
+		metadata: VectorMetadata;
+	}[]): Promise<{
+		upserted: number;
+		rejected: number;
+	}>;
+	query(params: {
+		index: string;
+		vector: string;
+		topK: number;
+		minScore?: number;
+		filter?: VectorFilter;
+	}): Promise<VectorQueryOutcome>;
+	getByIds(index: string, ids: readonly string[]): Promise<Array<{
+		id: string;
+		metadata: VectorMetadata;
+	}>>;
+	fetchByIds(index: string, ids: readonly string[]): Promise<VectorFetchRow[]>;
+	scan(index: string, cursor: number, limit: number): Promise<VectorScanPage>;
+	deleteByIds(index: string, ids: readonly string[]): Promise<number>;
+	deleteByFilter(index: string, filter: VectorFilter): Promise<number>;
+	stats(index: string): Promise<VectorIndexStats>;
 }

package/dist/builtins/sqlite-storage/vector-index-vec.d.ts

@@ -1,106 +1,143 @@
-import { IScopedLogger, VectorFilter, VectorMetadata, VectorMetric, encodeVectorBase64 } from '@camstack/types';
-import { SqliteOpProfiler } from './query-profiler.js';
-import { VectorFetchRow, VectorIndexStats, VectorQueryOutcome, VectorScanPage } from './vector-index-shared.js';
+/**
+* sqlite-vec vector index — the ANN-capable `vector-store` backend.
+*
+* Same database file, same open handle, same process as the brute-force index
+* it replaces: `sqlite-vec` is an EXTENSION, not a service. That is why it was
+* chosen over Qdrant/Milvus/LanceDB — at this scale a second thing to run, back
+* up and version buys nothing that a `vec0` virtual table does not.
+*
+* ## What the extension gives us, and what it costs
+*
+* A `vec0` table stores the vector as a packed Float32 blob and does the
+* distance in C, with the metadata prefilters compiled into the SAME statement
+* — so `deviceId = ? AND ts BETWEEN ? AND ?` narrows the candidate set before
+* any distance is computed, instead of after loading every row into JS.
+*
+* It is not free of sharp edges, and all three below were found by running it,
+* not by reading about it:
+*
+*  1. **The primary key must be declared `TEXT PRIMARY KEY`.** The default
+*     `rowid` form rejects our ids with "Only integers are allowed for primary
+*     key values" — and our ids are track UUIDs.
+*  2. **Integer metadata must be bound as BigInt.** `better-sqlite3` sends a
+*     plain JS number as a FLOAT, and `vec0` rejects it with "Expected integer
+*     for INTEGER metadata column". A silent `Number` → `BigInt` conversion at
+*     the boundary is therefore mandatory, not defensive.
+*  3. **`distance_metric=cosine` returns a DISTANCE**, 0 for identical and 2
+*     for opposite. The capability's contract is a SCORE where higher is
+*     nearer, so it is converted — `score = 1 - distance` — exactly once, here.
+*
+* ## Switching backends is not free
+*
+* The `vec0` table is a different table from the brute-force one. Turning this
+* on gives an EMPTY index until something repopulates it; for object CLIP that
+* is `pipelineAnalytics.rebuildObjectEmbeddings`. Off by default for that
+* reason.
+*/
+import type { IScopedLogger, VectorFilter, VectorMetadata, VectorMetric } from "@camstack/types";
+import { encodeVectorBase64 } from "@camstack/types";
+import type { SqliteOpProfiler } from "./query-profiler.js";
+import { type VectorFetchRow, type VectorIndexStats, type VectorQueryOutcome, type VectorScanPage } from "./vector-index-shared.js";
 /** The raw statement surface this index needs from better-sqlite3. */
 export interface RawSqliteDatabase {
-    exec(sql: string): unknown;
-    prepare(sql: string): {
-        run(...params: unknown[]): {
-            changes: number;
-        };
-        all(...params: unknown[]): unknown[];
-        get(...params: unknown[]): unknown;
-    };
-    loadExtension?(path: string): void;
+	exec(sql: string): unknown;
+	prepare(sql: string): {
+		run(...params: unknown[]): {
+			changes: number;
+		};
+		all(...params: unknown[]): unknown[];
+		get(...params: unknown[]): unknown;
+	};
+	loadExtension?(path: string): void;
 }
 /**
- * The largest `k` sqlite-vec will accept in a KNN query.
- *
- * This is the EXTENSION's own compile-time limit, not a number chosen here —
- * measured against the real extension, which answers a larger `k` with
- * `SqliteError: k value in knn query too large, provided 20000 and the limit
- * is 4096` rather than with rows.
- *
- * It matters that this is the extension's number. The vector door looked like
- * a bypass of {@link SETTINGS_QUERY_HARD_ROW_CAP}: `vector-store.cap.ts`
- * declares `topK: z.number().int().positive()` with no `.max()`, and it is
- * bound straight into `k = ?` on the same connection, in the same process, as
- * the capped `data-store-provider.query`. It is not a bypass — sqlite-vec was
- * holding the line the whole time, which caps a KNN's heap cost at 4 096 rows
- * of metadata. What was actually broken is that it held the line by THROWING,
- * so an over-large `topK` reached the caller as a raw SQLite error instead of
- * as the bounded answer it could always have been served.
- */
+* The largest `k` sqlite-vec will accept in a KNN query.
+*
+* This is the EXTENSION's own compile-time limit, not a number chosen here —
+* measured against the real extension, which answers a larger `k` with
+* `SqliteError: k value in knn query too large, provided 20000 and the limit
+* is 4096` rather than with rows.
+*
+* It matters that this is the extension's number. The vector door looked like
+* a bypass of {@link SETTINGS_QUERY_HARD_ROW_CAP}: `vector-store.cap.ts`
+* declares `topK: z.number().int().positive()` with no `.max()`, and it is
+* bound straight into `k = ?` on the same connection, in the same process, as
+* the capped `data-store-provider.query`. It is not a bypass — sqlite-vec was
+* holding the line the whole time, which caps a KNN's heap cost at 4 096 rows
+* of metadata. What was actually broken is that it held the line by THROWING,
+* so an over-large `topK` reached the caller as a raw SQLite error instead of
+* as the bounded answer it could always have been served.
+*/
 export declare const VEC_KNN_MAX_K = 4096;
 export interface SqliteVecIndexDeps {
-    readonly db: RawSqliteDatabase;
-    readonly logger: IScopedLogger;
-    /**
-     * The relational engine's profiler, so a KNN lands in the SAME report
-     * window as the queries it competes with. Optional because the specs
-     * construct this index against a raw handle with no engine behind it.
-     */
-    readonly profiler?: SqliteOpProfiler;
+	readonly db: RawSqliteDatabase;
+	readonly logger: IScopedLogger;
+	/**
+	* The relational engine's profiler, so a KNN lands in the SAME report
+	* window as the queries it competes with. Optional because the specs
+	* construct this index against a raw handle with no engine behind it.
+	*/
+	readonly profiler?: SqliteOpProfiler;
 }
 export declare class SqliteVecVectorIndex {
-    private readonly db;
-    private readonly logger;
-    private readonly profiler;
-    private readonly specs;
-    constructor(deps: SqliteVecIndexDeps);
-    /** Run one synchronous statement under the shared profiler, if there is one. */
-    private measured;
-    /** Recover index declarations so an existing table is usable from boot. */
-    loadIndexRegistry(): Promise<void>;
-    declareIndex(index: string, dim: number, metric: VectorMetric): Promise<void>;
-    private specOf;
-    upsert(index: string, items: readonly {
-        id: string;
-        vector: string;
-        metadata: VectorMetadata;
-    }[]): Promise<{
-        upserted: number;
-        rejected: number;
-    }>;
-    query(params: {
-        index: string;
-        vector: string;
-        topK: number;
-        minScore?: number;
-        filter?: VectorFilter;
-    }): Promise<VectorQueryOutcome>;
-    /**
-     * Ids read back WITH their vectors.
-     *
-     * `vec0` returns a vector column as the packed Float32 blob it stores, which
-     * is byte-for-byte what `upsert` was given — so the round trip is a base64
-     * encode and nothing else. No re-normalisation, no float re-ordering: a
-     * gallery loaded through here ranks identically to one loaded from the JSON
-     * column it replaced.
-     */
-    fetchByIds(index: string, ids: readonly string[]): Promise<VectorFetchRow[]>;
-    /**
-     * One page of the whole index, unranked.
-     *
-     * `LIMIT ? OFFSET ?` over the `vec0` table with NO `MATCH` clause: a plain
-     * scan, so the extension's KNN `k` ceiling ({@link VEC_KNN_MAX_K}) does not
-     * apply and no distance is computed at all. Ordered by `id` so a cursor means
-     * the same thing across calls — `rowid` order is not stable under the
-     * DELETE-then-INSERT upsert this class performs.
-     *
-     * The cursor is an OFFSET, so rows deleted behind the walk shift the window.
-     * That is acceptable and deliberate for the one caller: a reconcile that
-     * misses a row this pass sees it next pass, and the alternative — a keyset
-     * cursor on a virtual table whose ordering the extension owns — buys nothing
-     * for a walk that is idempotent by construction.
-     */
-    scan(index: string, cursor: number, limit: number): Promise<VectorScanPage>;
-    getByIds(index: string, ids: readonly string[]): Promise<Array<{
-        id: string;
-        metadata: VectorMetadata;
-    }>>;
-    deleteByIds(index: string, ids: readonly string[]): Promise<number>;
-    deleteByFilter(index: string, filter: VectorFilter): Promise<number>;
-    stats(index: string): Promise<VectorIndexStats>;
+	private readonly db;
+	private readonly logger;
+	private readonly profiler;
+	private readonly specs;
+	constructor(deps: SqliteVecIndexDeps);
+	/** Run one synchronous statement under the shared profiler, if there is one. */
+	private measured;
+	/** Recover index declarations so an existing table is usable from boot. */
+	loadIndexRegistry(): Promise<void>;
+	declareIndex(index: string, dim: number, metric: VectorMetric): Promise<void>;
+	private specOf;
+	upsert(index: string, items: readonly {
+		id: string;
+		vector: string;
+		metadata: VectorMetadata;
+	}[]): Promise<{
+		upserted: number;
+		rejected: number;
+	}>;
+	query(params: {
+		index: string;
+		vector: string;
+		topK: number;
+		minScore?: number;
+		filter?: VectorFilter;
+	}): Promise<VectorQueryOutcome>;
+	/**
+	* Ids read back WITH their vectors.
+	*
+	* `vec0` returns a vector column as the packed Float32 blob it stores, which
+	* is byte-for-byte what `upsert` was given — so the round trip is a base64
+	* encode and nothing else. No re-normalisation, no float re-ordering: a
+	* gallery loaded through here ranks identically to one loaded from the JSON
+	* column it replaced.
+	*/
+	fetchByIds(index: string, ids: readonly string[]): Promise<VectorFetchRow[]>;
+	/**
+	* One page of the whole index, unranked.
+	*
+	* `LIMIT ? OFFSET ?` over the `vec0` table with NO `MATCH` clause: a plain
+	* scan, so the extension's KNN `k` ceiling ({@link VEC_KNN_MAX_K}) does not
+	* apply and no distance is computed at all. Ordered by `id` so a cursor means
+	* the same thing across calls — `rowid` order is not stable under the
+	* DELETE-then-INSERT upsert this class performs.
+	*
+	* The cursor is an OFFSET, so rows deleted behind the walk shift the window.
+	* That is acceptable and deliberate for the one caller: a reconcile that
+	* misses a row this pass sees it next pass, and the alternative — a keyset
+	* cursor on a virtual table whose ordering the extension owns — buys nothing
+	* for a walk that is idempotent by construction.
+	*/
+	scan(index: string, cursor: number, limit: number): Promise<VectorScanPage>;
+	getByIds(index: string, ids: readonly string[]): Promise<Array<{
+		id: string;
+		metadata: VectorMetadata;
+	}>>;
+	deleteByIds(index: string, ids: readonly string[]): Promise<number>;
+	deleteByFilter(index: string, filter: VectorFilter): Promise<number>;
+	stats(index: string): Promise<VectorIndexStats>;
 }
 export { encodeVectorBase64 };