crewly 1.3.31 → 1.4.1

package/dist/backend/backend/src/services/knowledge/vector-store.service.d.ts

@@ -1,9 +1,18 @@
 /**
- * Vector Store Service
+ * Vector Store Service — v2
  *
- * Provides local, on-device vector storage using SQLite (better-sqlite3).
- * Stores document embeddings and performs cosine-similarity search entirely
- * offline — no external vector database or cloud API is needed at query time.
+ * Local, on-device vector storage using SQLite (better-sqlite3) with optional
+ * sqlite-vec extension for native KNN search. Falls back gracefully to
+ * JavaScript cosine similarity when the extension is unavailable.
+ *
+ * v2 improvements over v1:
+ * - Binary BLOB storage (Float32Array) instead of JSON TEXT (~4× smaller, faster)
+ * - Optional sqlite-vec integration for native SQL-based KNN search (cosine metric)
+ * - Dimension validation ensures all vectors in a scope share the same width
+ * - Prepared statement caching for repeated operations
+ * - Input sanitization (NaN, Infinity rejection)
+ * - Batch upsert with transaction wrapping
+ * - Automatic schema migration from v1 (JSON TEXT) databases
  *
  * Storage locations:
  * - Global scope: ~/.crewly/vector-store.db
@@ -11,6 +20,8 @@
  *
  * @module services/knowledge/vector-store.service
  */
+/** Reset the sqlite-vec loader state (for testing). */
+export declare function resetSqliteVecLoader(): void;
 /** Constants for the vector store */
 export declare const VECTOR_STORE_CONSTANTS: {
     /** SQLite database filename */
@@ -21,6 +32,12 @@ export declare const VECTOR_STORE_CONSTANTS: {
     readonly DEFAULT_SIMILARITY_THRESHOLD: 0.1;
     /** Maximum metadata JSON size (bytes) */
     readonly MAX_METADATA_SIZE: 65536;
+    /** Current schema version */
+    readonly SCHEMA_VERSION: 2;
+    /** Maximum supported embedding dimensions */
+    readonly MAX_DIMENSIONS: 4096;
+    /** Maximum items in a single batch upsert */
+    readonly MAX_BATCH_SIZE: 500;
 };
 /**
  * A stored embedding record with metadata.
@@ -48,6 +65,20 @@ export interface VectorSearchResult {
     /** Associated metadata */
     metadata: Record<string, unknown>;
 }
+/**
+ * Convert a number array to a Float32Array Buffer for efficient BLOB storage.
+ *
+ * @param embedding - The embedding vector
+ * @returns Buffer containing the Float32Array binary data
+ */
+export declare function embeddingToBlob(embedding: number[]): Buffer;
+/**
+ * Convert a BLOB Buffer back to a number array.
+ *
+ * @param blob - Buffer containing Float32Array binary data
+ * @returns The embedding as a number array
+ */
+export declare function blobToEmbedding(blob: Buffer): number[];
 /**
  * Compute cosine similarity between two vectors.
  *
@@ -57,20 +88,32 @@ export interface VectorSearchResult {
  */
 export declare function cosineSimilarity(a: number[], b: number[]): number;
 /**
- * VectorStoreService provides local SQLite-based vector storage.
+ * Validate an embedding vector for storage. Rejects empty arrays, values
+ * that are not finite numbers (NaN, Infinity), and dimensions exceeding
+ * the maximum.
+ *
+ * @param embedding - The embedding to validate
+ * @throws Error if the embedding is invalid
+ */
+export declare function validateEmbedding(embedding: number[]): void;
+/**
+ * VectorStoreService provides local SQLite-based vector storage with optional
+ * native KNN search via the sqlite-vec extension.
  *
- * Uses better-sqlite3 for synchronous, high-performance SQLite access.
- * Embeddings are stored as JSON arrays and cosine similarity is computed
- * in JavaScript for portability (no native vector extension required).
+ * When sqlite-vec is available, search uses a `vec0` virtual table with cosine
+ * distance metric for hardware-accelerated nearest-neighbor queries. When it is
+ * not available, search falls back to JavaScript cosine similarity (fully
+ * portable, no native extension required).
  *
- * Each scope (global / project) gets its own database file so project
- * data stays isolated and portable.
+ * Each scope (global / project) gets its own database file so project data
+ * stays isolated and portable.
  *
  * @example
  * ```typescript
  * const store = VectorStoreService.getInstance();
  * store.upsert('doc-123', embedding, { title: 'My Doc' }, 'global');
- * const results = store.search(queryEmbedding, 'global', 5);
+ * const results = store.search(queryEmbedding, 'global');
+ * console.log(store.isVecEnabled('global')); // true if sqlite-vec loaded
  * ```
  */
 export declare class VectorStoreService {
@@ -99,16 +142,69 @@ export declare class VectorStoreService {
      */
     private dbPath;
     /**
-     * Open or retrieve a cached SQLite database connection.
-     * Creates the parent directory and schema if they don't exist.
+     * Check whether the sqlite-vec native KNN search is available for a scope.
+     *
+     * @param scope - Storage scope
+     * @param projectPath - Required when scope is 'project'
+     * @returns true if vec0 virtual table is active
+     */
+    isVecEnabled(scope: 'global' | 'project', projectPath?: string): boolean;
+    /**
+     * Get (or prepare + cache) a statement for the given database state.
+     *
+     * @param state - Database state to prepare on
+     * @param key - Unique cache key
+     * @param sql - SQL to prepare if not yet cached
+     * @returns A prepared Statement
+     */
+    private getStmt;
+    /**
+     * Open or retrieve a cached database connection. Creates the parent
+     * directory, schema, and vec0 table (if sqlite-vec is available) on first
+     * access. Handles automatic migration from v1 databases.
      *
      * @param scope - Document scope
      * @param projectPath - Project path for project-scoped databases
-     * @returns Open Database instance
+     * @returns Internal DatabaseState
+     */
+    private getDbState;
+    /**
+     * Migrate database schema to v2.
+     * - Creates vec_config table for tracking schema version and dimensions
+     * - Adds dimensions column if missing
+     * - Converts JSON TEXT embeddings to BLOB if needed
+     * - Creates vec0 virtual table if sqlite-vec is available and dimensions are known
+     *
+     * @param db - The database to migrate
+     * @param vecEnabled - Whether sqlite-vec extension is loaded
+     */
+    private migrateSchema;
+    /**
+     * Create the vec0 virtual table if it doesn't already exist and dimensions
+     * are known. Returns true if the table is ready for use.
+     *
+     * @param db - The database
+     * @param dimensions - Explicit dimensions; if omitted, reads from vec_config
+     * @returns true if vec0 table exists and is ready
+     */
+    private ensureVecTable;
+    /**
+     * Copy all existing embeddings matching the target dimensions into the vec0
+     * virtual table.
+     *
+     * @param db - The database
+     * @param dimensions - Target embedding dimensions
+     */
+    private backfillVecTable;
+    /**
+     * Read the stored embedding dimensions from the vec_config table.
+     *
+     * @param db - The database
+     * @returns Stored dimensions, or null if not yet determined
      */
-    private getDb;
+    private getStoredDimensions;
     /**
-     * Close all open database connections.
+     * Close all open database connections and clear caches.
      */
     closeAll(): void;
     /**
@@ -119,9 +215,38 @@ export declare class VectorStoreService {
      * @param metadata - Optional metadata to associate with the embedding
      * @param scope - Storage scope ('global' or 'project')
      * @param projectPath - Required when scope is 'project'
-     * @throws Error if embedding is empty or metadata exceeds size limit
+     * @throws Error if id is empty, embedding is invalid, metadata exceeds size,
+     *         or dimensions don't match previously stored embeddings
      */
     upsert(id: string, embedding: number[], metadata: Record<string, unknown>, scope: 'global' | 'project', projectPath?: string): void;
+    /**
+     * Sync a single embedding to the vec0 virtual table.
+     * Uses CAST(? AS INTEGER) for rowid because sqlite-vec's vec0 virtual
+     * table does not accept JS number params directly for primary keys.
+     *
+     * @param state - Database state
+     * @param id - Document identifier
+     * @param blob - Embedding as a Float32Array buffer
+     */
+    private syncToVec;
+    /**
+     * Batch upsert multiple embeddings in a single transaction. Significantly
+     * faster than individual upserts for bulk imports.
+     *
+     * @param items - Array of { id, embedding, metadata } objects
+     * @param scope - Storage scope
+     * @param projectPath - Required when scope is 'project'
+     * @returns Count of succeeded and failed inserts
+     * @throws Error if batch size exceeds MAX_BATCH_SIZE
+     */
+    batchUpsert(items: Array<{
+        id: string;
+        embedding: number[];
+        metadata: Record<string, unknown>;
+    }>, scope: 'global' | 'project', projectPath?: string): {
+        succeeded: number;
+        failed: number;
+    };
     /**
      * Retrieve a stored embedding by document ID.
      *
@@ -158,11 +283,9 @@ export declare class VectorStoreService {
      */
     count(scope: 'global' | 'project', projectPath?: string): number;
     /**
-     * Search for the most similar embeddings using cosine similarity.
-     *
-     * Loads all embeddings from the store, computes cosine similarity
-     * against the query vector, and returns the top results above the
-     * similarity threshold.
+     * Search for the most similar embeddings. Uses native vec0 KNN search when
+     * sqlite-vec is available, otherwise falls back to JavaScript cosine
+     * similarity.
      *
      * @param queryEmbedding - The query embedding vector
      * @param scope - Storage scope
@@ -173,7 +296,31 @@ export declare class VectorStoreService {
      */
     search(queryEmbedding: number[], scope: 'global' | 'project', projectPath?: string, limit?: number, threshold?: number): VectorSearchResult[];
     /**
-     * Delete all embeddings from the store.
+     * Native KNN search using sqlite-vec's vec0 virtual table with cosine
+     * distance metric. Results are converted from cosine distance (0 = identical,
+     * 1 = orthogonal) to cosine similarity (1 = identical, 0 = orthogonal).
+     *
+     * @param state - Database state
+     * @param queryEmbedding - The query vector
+     * @param limit - Max results
+     * @param threshold - Minimum cosine similarity
+     * @returns Sorted search results
+     */
+    private vecSearch;
+    /**
+     * JavaScript fallback search using cosine similarity on BLOB embeddings.
+     * Used when sqlite-vec is not available.
+     *
+     * @param state - Database state
+     * @param queryEmbedding - The query vector
+     * @param limit - Max results
+     * @param threshold - Minimum cosine similarity
+     * @returns Sorted search results
+     */
+    private jsFallbackSearch;
+    /**
+     * Delete all embeddings from the store. Also resets the stored dimension
+     * constraint so that subsequent inserts may use a different embedding width.
      *
      * @param scope - Storage scope
      * @param projectPath - Required when scope is 'project'

package/dist/backend/backend/src/services/knowledge/vector-store.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vector-store.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/knowledge/vector-store.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,qCAAqC;AACrC,eAAO,MAAM,sBAAsB;IACjC,+BAA+B;;IAE/B,sDAAsD;;IAEtD,6DAA6D;;IAE7D,yCAAyC;;CAEjC,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,gDAAgD;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,8CAA8C;IAC9C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAaD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAcjE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAmC;IAC1D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAwC;IAElE,OAAO;IAIP;;;;OAIG;IACH,MAAM,CAAC,WAAW,IAAI,kBAAkB;IAOxC;;OAEG;IACH,MAAM,CAAC,aAAa,IAAI,IAAI;IAW5B;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM;IAed;;;;;;;OAOG;IACH,OAAO,CAAC,KAAK;IAoCb;;OAEG;IACH,QAAQ,IAAI,IAAI;IAmBhB;;;;;;;;;OASG;IACH,MAAM,CACJ,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EAAE,EACnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAC3B,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI;IA8BP;;;;;;;OAOG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAiBvF;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO;IAM9E;;;;;;;OAOG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO;IAM3E;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM;IAMhE;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,cAAc,EAAE,MAAM,EAAE,EACxB,KAAK,EAAE,QAAQ,GAAG,SAAS,EAC3B,WAAW,CAAC,EAAE,MAAM,EACpB,KAAK,GAAE,MAAoD,EAC3D,SAAS,GAAE,MAA4D,GACtE,kBAAkB,EAAE;IA2BvB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM;CAKjE"}
+{"version":3,"file":"vector-store.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/knowledge/vector-store.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AA2DH,uDAAuD;AACvD,wBAAgB,oBAAoB,IAAI,IAAI,CAG3C;AAWD,qCAAqC;AACrC,eAAO,MAAM,sBAAsB;IACjC,+BAA+B;;IAE/B,sDAAsD;;IAEtD,6DAA6D;;IAE7D,yCAAyC;;IAEzC,6BAA6B;;IAE7B,6CAA6C;;IAE7C,6CAA6C;;CAErC,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,gDAAgD;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,8CAA8C;IAC9C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAoCD;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAG3D;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOtD;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAcjE;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAgB3D;AAMD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAmC;IAC1D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyC;IAEnE,OAAO;IAIP;;;;OAIG;IACH,MAAM,CAAC,WAAW,IAAI,kBAAkB;IAOxC;;OAEG;IACH,MAAM,CAAC,aAAa,IAAI,IAAI;IAW5B;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM;IAed;;;;;;OAMG;IACH,YAAY,CAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO;IAKxE;;;;;;;OAOG;IACH,OAAO,CAAC,OAAO;IASf;;;;;;;;OAQG;IACH,OAAO,CAAC,UAAU;IAgDlB;;;;;;;;;OASG;IACH,OAAO,CAAC,aAAa;IAsFrB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IA+BtB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IA0BxB;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAW3B;;OAEG;IACH,QAAQ,IAAI,IAAI;IAoBhB;;;;;;;;;;OAUG;IACH,MAAM,CACJ,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EAAE,EACnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAC3B,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI;IAiEP;;;;;;;;OAQG;IACH,OAAO,CAAC,SAAS;IAqBjB;;;;;;;;;OASG;IACH,WAAW,CACT,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,EACpF,KAAK,EAAE,QAAQ,GAAG,SAAS,EAC3B,WAAW,CAAC,EAAE,MAAM,GACnB;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IA8BxC;;;;;;;OAOG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAoBvF;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO;IAwB9E;;;;;;;OAOG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO;IAM3E;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM;IAMhE;;;;;;;;;;;OAWG;IACH,MAAM,CACJ,cAAc,EAAE,MAAM,EAAE,EACxB,KAAK,EAAE,QAAQ,GAAG,SAAS,EAC3B,WAAW,CAAC,EAAE,MAAM,EACpB,KAAK,GAAE,MAAoD,EAC3D,SAAS,GAAE,MAA4D,GACtE,kBAAkB,EAAE;IAsBvB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,SAAS;IAgDjB;;;;;;;;;OASG;IACH,OAAO,CAAC,gBAAgB;IA+BxB;;;;;;;OAOG;IACH,KAAK,CAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM;CAyBjE"}