@jagilber-org/index-server 1.22.0 → 1.26.1

package/dist/services/storage/jsonFileStore.d.ts

@@ -19,7 +19,9 @@ export declare class JsonFileStore implements IInstructionStore {
     load(): LoadResult;
     close(): void;
     get(id: string): InstructionEntry | null;
-    write(entry: InstructionEntry): void;
+    write(entry: InstructionEntry, opts?: {
+        createOnly?: boolean;
+    }): void;
     remove(id: string): void;
     list(opts?: ListOptions): InstructionEntry[];
     query(opts: QueryOptions): InstructionEntry[];

package/dist/services/storage/jsonFileStore.js

@@ -14,8 +14,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.JsonFileStore = void 0;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
-const crypto_1 = __importDefault(require("crypto"));
 const hashUtils_js_1 = require("./hashUtils.js");
+const atomicFs_js_1 = require("../atomicFs.js");
 class JsonFileStore {
     dir;
     cache = new Map();
@@ -87,12 +87,14 @@ class JsonFileStore {
         this.ensureLoaded();
         return this.cache.get(id) ?? null;
     }
-    write(entry) {
-        // Write to disk
+    write(entry, opts) {
         const filePath = path_1.default.join(this.dir, `${entry.id}.json`);
-        const tmpPath = `${filePath}.${crypto_1.default.randomBytes(4).toString('hex')}.tmp`;
-        fs_1.default.writeFileSync(tmpPath, JSON.stringify(entry, null, 2), 'utf-8');
-        fs_1.default.renameSync(tmpPath, filePath);
+        if (opts?.createOnly) {
+            (0, atomicFs_js_1.atomicCreateJson)(filePath, entry);
+        }
+        else {
+            (0, atomicFs_js_1.atomicWriteJson)(filePath, entry);
+        }
         // Update in-memory cache
         this.cache.set(entry.id, entry);
         this.loaded = true;

package/dist/services/storage/migrationEngine.d.ts

@@ -3,6 +3,7 @@
  *
  * Uses Node.js built-in node:sqlite. Zero third-party dependencies.
  */
+import type { IEmbeddingStore } from './types.js';
 export interface MigrationOptions {
     onProgress?: (current: number, total: number) => void;
 }
@@ -33,3 +34,15 @@ export declare function migrateJsonToSqlite(jsonDir: string, dbPath: string, opt
  * Each entry becomes a separate .json file named by ID.
  */
 export declare function migrateSqliteToJson(dbPath: string, jsonDir: string, opts?: MigrationOptions): ExportResult;
+export interface EmbeddingMigrationResult {
+    migrated: number;
+    skipped: number;
+    error?: string;
+}
+/**
+ * Migrate embeddings from a JSON file to an IEmbeddingStore (e.g. SqliteEmbeddingStore).
+ *
+ * Reads the JSON embedding cache and saves it into the target store.
+ * Idempotent: calling again with the same data overwrites safely.
+ */
+export declare function migrateJsonEmbeddingsToStore(jsonEmbeddingPath: string, targetStore: IEmbeddingStore): EmbeddingMigrationResult;

package/dist/services/storage/migrationEngine.js

@@ -10,9 +10,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.migrateJsonToSqlite = migrateJsonToSqlite;
 exports.migrateSqliteToJson = migrateSqliteToJson;
+exports.migrateJsonEmbeddingsToStore = migrateJsonEmbeddingsToStore;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const jsonFileStore_js_1 = require("./jsonFileStore.js");
+const jsonEmbeddingStore_js_1 = require("./jsonEmbeddingStore.js");
 const sqliteStore_js_1 = require("./sqliteStore.js");
 /**
  * Migrate all instructions from JSON files to a SQLite database.
@@ -91,3 +93,32 @@ function migrateSqliteToJson(dbPath, jsonDir, opts) {
     }
     return { exported, errors };
 }
+/**
+ * Migrate embeddings from a JSON file to an IEmbeddingStore (e.g. SqliteEmbeddingStore).
+ *
+ * Reads the JSON embedding cache and saves it into the target store.
+ * Idempotent: calling again with the same data overwrites safely.
+ */
+function migrateJsonEmbeddingsToStore(jsonEmbeddingPath, targetStore) {
+    try {
+        const jsonStore = new jsonEmbeddingStore_js_1.JsonEmbeddingStore(jsonEmbeddingPath);
+        const data = jsonStore.load();
+        jsonStore.close();
+        if (!data) {
+            return { migrated: 0, skipped: 0, error: 'No embedding data in JSON file' };
+        }
+        const count = Object.keys(data.embeddings).length;
+        if (count === 0) {
+            return { migrated: 0, skipped: 0 };
+        }
+        targetStore.save(data);
+        return { migrated: count, skipped: 0 };
+    }
+    catch (err) {
+        return {
+            migrated: 0,
+            skipped: 0,
+            error: err instanceof Error ? err.message : 'Migration failed',
+        };
+    }
+}

package/dist/services/storage/sqliteEmbeddingStore.d.ts

@@ -0,0 +1,30 @@
+/**
+ * SqliteEmbeddingStore — IEmbeddingStore backed by sqlite-vec (vec0 virtual table).
+ *
+ * Uses node:sqlite (DatabaseSync) with sqlite-vec extension for KNN vector search.
+ * Vectors stored as BLOB (Float32Array buffer), metadata in a companion table.
+ *
+ * Requires: Node.js ≥22.13.0, sqlite-vec npm package.
+ */
+import type { IEmbeddingStore, EmbeddingCacheData, EmbeddingSearchResult } from './types.js';
+export declare class SqliteEmbeddingStore implements IEmbeddingStore {
+    private db;
+    private readonly dims;
+    private readonly dbPath;
+    constructor(dbPath: string, dims?: number);
+    private initExtension;
+    /**
+     * Run PRAGMA integrity_check to detect database corruption.
+     * @returns true if the database is intact, false if corrupt.
+     */
+    private verifyIntegrity;
+    /**
+     * Rebuild a corrupt database by closing, deleting, and recreating it.
+     */
+    private rebuildDatabase;
+    private initSchema;
+    load(): EmbeddingCacheData | null;
+    save(data: EmbeddingCacheData): void;
+    search(queryVector: Float32Array, limit: number): EmbeddingSearchResult[];
+    close(): void;
+}

package/dist/services/storage/sqliteEmbeddingStore.js

@@ -0,0 +1,222 @@
+"use strict";
+/**
+ * SqliteEmbeddingStore — IEmbeddingStore backed by sqlite-vec (vec0 virtual table).
+ *
+ * Uses node:sqlite (DatabaseSync) with sqlite-vec extension for KNN vector search.
+ * Vectors stored as BLOB (Float32Array buffer), metadata in a companion table.
+ *
+ * Requires: Node.js ≥22.13.0, sqlite-vec npm package.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SqliteEmbeddingStore = void 0;
+const node_sqlite_1 = require("node:sqlite");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const sqlite_vec_1 = require("sqlite-vec");
+const logger_js_1 = require("../logger.js");
+const DEFAULT_VECTOR_DIMS = 384;
+const MAX_VECTOR_DIMS = 65536;
+const EMBEDDING_META_DDL = `CREATE TABLE IF NOT EXISTS embedding_meta (
+  instruction_id TEXT PRIMARY KEY,
+  model_name TEXT NOT NULL,
+  source_hash TEXT NOT NULL,
+  index_hash TEXT NOT NULL,
+  computed_at TEXT NOT NULL
+)`;
+/** Cache metadata stored in the SQLite metadata table. */
+const META_INDEX_HASH_KEY = 'embedding_index_hash';
+const META_MODEL_NAME_KEY = 'embedding_model_name';
+class SqliteEmbeddingStore {
+    db;
+    dims;
+    dbPath;
+    constructor(dbPath, dims = DEFAULT_VECTOR_DIMS) {
+        if (!Number.isInteger(dims) || dims < 1 || dims > MAX_VECTOR_DIMS) {
+            throw new Error(`Invalid vector dimensions: ${dims}. Must be an integer between 1 and ${MAX_VECTOR_DIMS}.`);
+        }
+        this.dims = dims;
+        this.dbPath = dbPath;
+        this.db = new node_sqlite_1.DatabaseSync(dbPath, { allowExtension: true });
+        this.initExtension();
+        if (!this.verifyIntegrity()) {
+            this.rebuildDatabase();
+        }
+        this.initSchema();
+    }
+    initExtension() {
+        const extPath = (0, sqlite_vec_1.getLoadablePath)();
+        // Security: validate the extension path resolves within node_modules
+        const resolved = path_1.default.resolve(extPath);
+        const nmDir = path_1.default.resolve(__dirname, '..', '..', '..', 'node_modules');
+        if (!resolved.startsWith(nmDir + path_1.default.sep) && !resolved.startsWith(nmDir + '/')) {
+            throw new Error(`sqlite-vec extension path escapes node_modules: ${resolved}`);
+        }
+        this.db.loadExtension(extPath);
+    }
+    /**
+     * Run PRAGMA integrity_check to detect database corruption.
+     * @returns true if the database is intact, false if corrupt.
+     */
+    verifyIntegrity() {
+        try {
+            const result = this.db.prepare('PRAGMA integrity_check').all();
+            if (result.length === 1 && result[0].integrity_check === 'ok')
+                return true;
+            const issues = result.map(r => r.integrity_check).join('; ');
+            (0, logger_js_1.logWarn)(`[embedding-store] Integrity check failed: ${issues}`);
+            return false;
+        }
+        catch (err) {
+            (0, logger_js_1.logWarn)(`[embedding-store] Integrity check error: ${err instanceof Error ? err.message : 'unknown'}`);
+            return false;
+        }
+    }
+    /**
+     * Rebuild a corrupt database by closing, deleting, and recreating it.
+     */
+    rebuildDatabase() {
+        (0, logger_js_1.logWarn)(`[embedding-store] Rebuilding corrupt database: ${this.dbPath}`);
+        try {
+            this.db.close();
+        }
+        catch { /* may already be unusable */ }
+        // Remove the corrupt DB and WAL/SHM files
+        for (const suffix of ['', '-wal', '-shm']) {
+            try {
+                fs_1.default.unlinkSync(this.dbPath + suffix);
+            }
+            catch { /* file may not exist */ }
+        }
+        this.db = new node_sqlite_1.DatabaseSync(this.dbPath, { allowExtension: true });
+        this.initExtension();
+        (0, logger_js_1.logInfo)(`[embedding-store] Database rebuilt successfully`);
+    }
+    initSchema() {
+        this.db.exec('PRAGMA journal_mode = WAL');
+        this.db.exec('PRAGMA busy_timeout = 5000');
+        // Ensure metadata table exists (may already exist from SqliteStore)
+        this.db.exec('CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT)');
+        this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
+      instruction_id TEXT PRIMARY KEY,
+      embedding float[${this.dims}]
+    )`);
+        this.db.exec(EMBEDDING_META_DDL);
+    }
+    load() {
+        // Read cache metadata
+        const hashRow = this.db.prepare('SELECT value FROM metadata WHERE key = ?').get(META_INDEX_HASH_KEY);
+        const modelRow = this.db.prepare('SELECT value FROM metadata WHERE key = ?').get(META_MODEL_NAME_KEY);
+        if (!hashRow)
+            return null;
+        // Read all embeddings
+        const rows = this.db.prepare('SELECT instruction_id, embedding FROM embeddings').all();
+        if (rows.length === 0) {
+            // CodeQL flagged the original `&& !hashRow` as dead code (already
+            // null-guarded at L115). We cannot upgrade to `if (rows.length === 0) return null`
+            // because the public contract (see embeddingStore.contract.spec.ts
+            // "save() with empty embeddings map succeeds") requires a non-null cache
+            // when hashRow exists but no embeddings have been persisted. Fall through
+            // to return { indexHash, entryHashes, embeddings: {} } so subsequent saves
+            // can layer onto the same metadata.
+        }
+        const embeddings = {};
+        for (const row of rows) {
+            // Validate buffer bounds before creating Float32Array view
+            const expectedBytes = this.dims * 4;
+            if (row.embedding.byteLength !== expectedBytes) {
+                (0, logger_js_1.logWarn)(`[embedding-store] Skipping entry '${row.instruction_id}': buffer size ${row.embedding.byteLength} bytes, expected ${expectedBytes} (${this.dims} dims × 4 bytes)`);
+                continue;
+            }
+            embeddings[row.instruction_id] = Array.from(new Float32Array(row.embedding.buffer, row.embedding.byteOffset, this.dims));
+        }
+        // Read entry hashes from embedding_meta
+        const metaRows = this.db.prepare('SELECT instruction_id, source_hash FROM embedding_meta').all();
+        const entryHashes = {};
+        for (const m of metaRows) {
+            entryHashes[m.instruction_id] = m.source_hash;
+        }
+        return {
+            indexHash: hashRow.value,
+            modelName: modelRow?.value,
+            entryHashes,
+            embeddings,
+        };
+    }
+    save(data) {
+        // TODO: Replace full-table clear+reinsert with incremental upsert for large datasets.
+        // Use a diff of entryHashes to INSERT OR REPLACE only changed entries and DELETE removed ones.
+        // Use a transaction for atomicity
+        const tx = this.db.prepare('BEGIN');
+        const commit = this.db.prepare('COMMIT');
+        const rollback = this.db.prepare('ROLLBACK');
+        try {
+            tx.run();
+            // Clear existing data
+            this.db.exec('DELETE FROM embeddings');
+            this.db.exec('DELETE FROM embedding_meta');
+            // Upsert cache metadata
+            const upsertMeta = this.db.prepare('INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)');
+            upsertMeta.run(META_INDEX_HASH_KEY, data.indexHash);
+            if (data.modelName) {
+                upsertMeta.run(META_MODEL_NAME_KEY, data.modelName);
+            }
+            // Insert embeddings
+            const insertVec = this.db.prepare('INSERT INTO embeddings (instruction_id, embedding) VALUES (?, ?)');
+            const insertMeta = this.db.prepare('INSERT INTO embedding_meta (instruction_id, model_name, source_hash, index_hash, computed_at) VALUES (?, ?, ?, ?, ?)');
+            const now = new Date().toISOString();
+            for (const [id, vec] of Object.entries(data.embeddings)) {
+                if (vec.length !== this.dims) {
+                    throw new Error(`Vector dimension mismatch for '${id}': expected ${this.dims}, got ${vec.length}`);
+                }
+                for (let i = 0; i < vec.length; i++) {
+                    if (!Number.isFinite(vec[i])) {
+                        throw new Error(`Invalid vector value for '${id}' at index ${i}: ${vec[i]}. Vectors must contain only finite numbers.`);
+                    }
+                }
+                const buf = Buffer.from(new Float32Array(vec).buffer);
+                insertVec.run(id, buf);
+                insertMeta.run(id, data.modelName ?? 'unknown', data.entryHashes?.[id] ?? '', data.indexHash, now);
+            }
+            commit.run();
+        }
+        catch (err) {
+            try {
+                rollback.run();
+            }
+            catch { /* rollback may fail if tx already aborted */ }
+            throw err;
+        }
+    }
+    search(queryVector, limit) {
+        if (limit <= 0)
+            return [];
+        if (queryVector.length !== this.dims) {
+            (0, logger_js_1.logWarn)(`[embedding-store] Search query dimension mismatch: expected ${this.dims}, got ${queryVector.length}`);
+            return [];
+        }
+        try {
+            const buf = Buffer.from(queryVector.buffer, queryVector.byteOffset, queryVector.byteLength);
+            const rows = this.db.prepare('SELECT instruction_id, distance FROM embeddings WHERE embedding MATCH ? ORDER BY distance LIMIT ?').all(buf, limit);
+            return rows.map(r => ({
+                id: r.instruction_id,
+                distance: r.distance,
+            }));
+        }
+        catch (err) {
+            (0, logger_js_1.logWarn)(`[embedding-store] search failed: ${err instanceof Error ? err.message : 'unknown'}`);
+            return [];
+        }
+    }
+    close() {
+        try {
+            this.db.close();
+        }
+        catch {
+            // Already closed
+        }
+    }
+}
+exports.SqliteEmbeddingStore = SqliteEmbeddingStore;

package/dist/services/storage/sqliteStore.d.ts

@@ -23,7 +23,9 @@ export declare class SqliteStore implements IInstructionStore {
     load(): LoadResult;
     close(): void;
     get(id: string): InstructionEntry | null;
-    write(entry: InstructionEntry): void;
+    write(entry: InstructionEntry, opts?: {
+        createOnly?: boolean;
+    }): void;
     remove(id: string): void;
     list(opts?: ListOptions): InstructionEntry[];
     query(opts: QueryOptions): InstructionEntry[];

package/dist/services/storage/sqliteStore.js

@@ -162,8 +162,8 @@ class SqliteStore {
         this.ensureLoaded();
         return this.cache.get(id) ?? null;
     }
-    write(entry) {
-        const sql = `INSERT OR REPLACE INTO instructions (
+    write(entry, opts) {
+        const sql = `${opts?.createOnly ? 'INSERT INTO' : 'INSERT OR REPLACE INTO'} instructions (
       id, title, body, rationale, priority, audience, requirement,
       categories, content_type, primary_category, source_hash,
       schema_version, deprecated_by, created_at, updated_at,

package/dist/services/storage/types.d.ts

@@ -123,7 +123,9 @@ export interface IInstructionStore {
      * Write (create or update) an instruction.
      * @param entry - The instruction to persist.
      */
-    write(entry: InstructionEntry): void;
+    write(entry: InstructionEntry, opts?: {
+        createOnly?: boolean;
+    }): void;
     /**
      * Remove an instruction by ID. No-op if not found.
      * @param id - Instruction ID to remove.
@@ -169,3 +171,48 @@ export interface IInstructionStore {
      */
     count(): number;
 }
+/**
+ * Persisted embedding cache format (backwards-compatible: entryHashes is optional).
+ */
+export interface EmbeddingCacheData {
+    indexHash: string;
+    modelName?: string;
+    /** Per-entry content hashes for incremental invalidation (required for v2+). */
+    entryHashes?: Record<string, string>;
+    embeddings: Record<string, number[]>;
+}
+/** A single embedding search result with distance score. */
+export interface EmbeddingSearchResult {
+    id: string;
+    distance: number;
+}
+/**
+ * Storage backend interface for embedding vectors.
+ *
+ * Implementations:
+ * - JsonEmbeddingStore: Flat JSON file (default, current behavior)
+ * - SqliteEmbeddingStore: sqlite-vec vec0 virtual table
+ */
+export interface IEmbeddingStore {
+    /**
+     * Load cached embedding data from the backing store.
+     * @returns The cached data, or null if no data exists.
+     */
+    load(): EmbeddingCacheData | null;
+    /**
+     * Save embedding data to the backing store.
+     * @param data - The embedding cache data to persist.
+     */
+    save(data: EmbeddingCacheData): void;
+    /**
+     * Search for the nearest vectors to a query vector (KNN).
+     * @param queryVector - The query embedding vector.
+     * @param limit - Maximum number of results to return.
+     * @returns Array of { id, distance } sorted by distance ascending.
+     */
+    search(queryVector: Float32Array, limit: number): EmbeddingSearchResult[];
+    /**
+     * Close the store and release resources.
+     */
+    close(): void;
+}