@softerist/heuristic-mcp 3.0.12 → 3.0.14

package/lib/vector-store-binary.js

@@ -433,7 +433,18 @@ export class BinaryVectorStore {
     return map;
   }
 
-  static async write(cacheDir, chunks, { contentCacheEntries, getContent, preRename } = {}) {
+  static async write(
+    cacheDir,
+    chunks,
+    {
+      contentCacheEntries,
+      vectorCacheEntries,
+      vectorLoadMode,
+      getContent,
+      getVector,
+      preRename,
+    } = {}
+  ) {
     ensureLittleEndian();
     const { vectorsPath, recordsPath, contentPath, filesPath } =
       BinaryVectorStore.getPaths(cacheDir);
@@ -446,8 +457,6 @@ export class BinaryVectorStore {
 
     const fileIds = new Map();
     const files = [];
-    let dim = null;
-
     const denseChunks = [];
     const denseSourceIndices = [];
     for (let i = 0; i < chunks.length; i += 1) {
@@ -457,43 +466,56 @@ export class BinaryVectorStore {
       denseSourceIndices.push(i);
     }
 
-    const recordEntries = new Array(denseChunks.length);
-    let contentOffset = 0;
-
-    for (let i = 0; i < denseChunks.length; i += 1) {
-      const chunk = denseChunks[i];
-      const sourceIndex = denseSourceIndices[i];
-
-      const file = chunk.file;
-      if (!fileIds.has(file)) {
-        fileIds.set(file, files.length);
-        files.push(file);
+    const resolveVector = async (chunk, sourceIndex) => {
+      let vectorSource = chunk.vector;
+      if (
+        (vectorSource === undefined || vectorSource === null) &&
+        typeof getVector === 'function'
+      ) {
+        vectorSource = getVector(chunk, sourceIndex);
+        if (vectorSource && typeof vectorSource.then === 'function') {
+          vectorSource = await vectorSource;
+        }
       }
-
-      if (chunk.vector === undefined || chunk.vector === null) {
+      if (vectorSource === undefined || vectorSource === null) {
         throw new Error(`Missing vector data for binary cache write at index ${sourceIndex}`);
       }
       const vector =
-        chunk.vector instanceof Float32Array ? chunk.vector : new Float32Array(chunk.vector);
-      if (!vector) {
-        throw new Error('Missing vector data for binary cache write');
-      }
-      if (vector.length === 0) {
+        vectorSource instanceof Float32Array
+          ? vectorSource
+          : ArrayBuffer.isView(vectorSource)
+            ? Float32Array.from(vectorSource)
+            : new Float32Array(vectorSource);
+      if (!vector || vector.length === 0) {
         throw new Error(`Empty vector data for binary cache write at index ${sourceIndex}`);
       }
-      if (dim === null) {
-        dim = vector.length;
-      } else if (vector.length !== dim) {
-        throw new Error('Vector dimension mismatch in binary cache write');
-      }
+      return vector;
+    };
 
+    const resolveContent = async (chunk, sourceIndex) => {
       const contentSource =
         chunk.content !== undefined && chunk.content !== null
           ? chunk.content
           : getContent
             ? await getContent(chunk, sourceIndex)
             : '';
-      const contentValue = normalizeContent(contentSource);
+      return normalizeContent(contentSource);
+    };
+
+    const recordEntries = new Array(denseChunks.length);
+    let contentOffset = 0;
+
+    for (let i = 0; i < denseChunks.length; i += 1) {
+      const chunk = denseChunks[i];
+      const sourceIndex = denseSourceIndices[i];
+
+      const file = chunk.file;
+      if (!fileIds.has(file)) {
+        fileIds.set(file, files.length);
+        files.push(file);
+      }
+
+      const contentValue = await resolveContent(chunk, sourceIndex);
       const contentLength = Buffer.byteLength(contentValue, 'utf-8');
 
       recordEntries[i] = {
@@ -502,14 +524,14 @@ export class BinaryVectorStore {
         endLine: chunk.endLine ?? 0,
         contentOffset,
         contentLength,
-        vector,
       };
 
       contentOffset += contentLength;
     }
 
-    if (!dim) dim = 0;
     const count = denseChunks.length;
+    const dim =
+      count > 0 ? (await resolveVector(denseChunks[0], denseSourceIndices[0])).length : 0;
 
     await fs.writeFile(filesTmp, JSON.stringify(files));
 
@@ -555,25 +577,23 @@ export class BinaryVectorStore {
         await recordsHandle.write(recordBuffer, 0, recordBuffer.length, recordPos);
         recordPos += recordBuffer.length;
 
+        const chunk = denseChunks[i];
+        const sourceIndex = denseSourceIndices[i];
+        const vector = await resolveVector(chunk, sourceIndex);
+        if (vector.length !== dim) {
+          throw new Error('Vector dimension mismatch in binary cache write');
+        }
         const vectorBuffer = Buffer.from(
-          entry.vector.buffer,
-          entry.vector.byteOffset,
-          entry.vector.byteLength
+          vector.buffer,
+          vector.byteOffset,
+          vector.byteLength
         );
         await vectorsHandle.write(vectorBuffer, 0, vectorBuffer.length, vectorPos);
         vectorPos += vectorBuffer.length;
 
         if (entry.contentLength > 0) {
           // Re-fetch content to avoid holding all strings in memory
-          const chunk = denseChunks[i];
-          const sourceIndex = denseSourceIndices[i];
-          const contentSource =
-            chunk.content !== undefined && chunk.content !== null
-              ? chunk.content
-              : getContent
-                ? await getContent(chunk, sourceIndex)
-                : '';
-          const val = normalizeContent(contentSource);
+          const val = await resolveContent(chunk, sourceIndex);
           const contentBuffer = Buffer.from(val, 'utf-8');
           await contentHandle.write(contentBuffer, 0, contentBuffer.length, contentPos);
           contentPos += contentBuffer.length;
@@ -598,21 +618,10 @@ export class BinaryVectorStore {
       renameWithRetry(filesTmp, filesPath),
     ]);
 
-    const [vectorsBuffer, recordsBuffer] = await Promise.all([
-      fs.readFile(vectorsPath),
-      fs.readFile(recordsPath),
-    ]);
-    const contentReadHandle = await fs.open(contentPath, 'r');
-
-    return new BinaryVectorStore({
-      vectorsBuffer,
-      recordsBuffer,
-      contentHandle: contentReadHandle,
-      contentSize: contentOffset,
-      files,
-      dim,
-      count,
+    return BinaryVectorStore.load(cacheDir, {
       contentCacheEntries,
+      vectorCacheEntries,
+      vectorLoadMode,
     });
   }
 }

package/lib/vector-store-sqlite.js

@@ -264,7 +264,7 @@ export class SqliteVectorStore {
    * @param {Array} chunks - Array of chunk objects with vector, file, startLine, endLine, content
    * @param {Object} options - { getContent, preRename }
    */
-  static async write(cacheDir, chunks, { getContent, preRename } = {}) {
+  static async write(cacheDir, chunks, { getContent, getVector, preRename } = {}) {
     if (!chunks || chunks.length === 0) {
       return null;
     }
@@ -286,10 +286,10 @@ export class SqliteVectorStore {
       denseSourceIndices.push(i);
     }
 
-    const resolveContent = async (chunk, sourceIndex) => {
-      if (chunk.content !== undefined && chunk.content !== null) {
-        return chunk.content;
-      }
+    const resolveContent = async (chunk, sourceIndex) => {
+      if (chunk.content !== undefined && chunk.content !== null) {
+        return chunk.content;
+      }
       if (typeof getContent === 'function') {
         const value = getContent(chunk, sourceIndex);
         if (value && typeof value.then === 'function') {
@@ -297,46 +297,39 @@ export class SqliteVectorStore {
         }
         return value;
       }
-      return null;
-    };
-
-    let dim = null;
-    const resolvedData = new Array(denseChunks.length);
-    for (let i = 0; i < denseChunks.length; i += 1) {
-      const chunk = denseChunks[i];
-      const sourceIndex = denseSourceIndices[i];
-
-      if (chunk.vector === undefined || chunk.vector === null) {
-        throw new Error(`Missing vector data for sqlite cache write at index ${sourceIndex}`);
-      }
-
-      const vector =
-        chunk.vector instanceof Float32Array ? chunk.vector : Float32Array.from(chunk.vector);
-      if (!vector || vector.length === 0) {
-        throw new Error(`Empty vector data for sqlite cache write at index ${sourceIndex}`);
-      }
-      if (dim === null) {
-        dim = vector.length;
-      } else if (vector.length !== dim) {
-        throw new Error('Vector dimension mismatch in sqlite cache write');
-      }
-
-      let content = await resolveContent(chunk, sourceIndex);
-      if (content === undefined) content = null;
-      if (content !== null && typeof content !== 'string') {
-        content = String(content);
-      }
-
-      resolvedData[i] = {
-        file: chunk.file,
-        startLine: chunk.startLine ?? 0,
-        endLine: chunk.endLine ?? 0,
-        content,
-        vectorBlob: Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength),
-      };
-    }
-
-    if (!dim) dim = 0;
+      return null;
+    };
+
+    const resolveVector = async (chunk, sourceIndex) => {
+      let vectorSource = chunk.vector;
+      if (
+        (vectorSource === undefined || vectorSource === null) &&
+        typeof getVector === 'function'
+      ) {
+        vectorSource = getVector(chunk, sourceIndex);
+        if (vectorSource && typeof vectorSource.then === 'function') {
+          vectorSource = await vectorSource;
+        }
+      }
+      if (vectorSource === undefined || vectorSource === null) {
+        throw new Error(`Missing vector data for sqlite cache write at index ${sourceIndex}`);
+      }
+      const vector =
+        vectorSource instanceof Float32Array
+          ? vectorSource
+          : ArrayBuffer.isView(vectorSource)
+            ? Float32Array.from(vectorSource)
+            : Float32Array.from(vectorSource);
+      if (!vector || vector.length === 0) {
+        throw new Error(`Empty vector data for sqlite cache write at index ${sourceIndex}`);
+      }
+      return vector;
+    };
+
+    const dim =
+      denseChunks.length > 0
+        ? (await resolveVector(denseChunks[0], denseSourceIndices[0])).length
+        : 0;
 
     // Create new database
     const db = new Database(writePath);
@@ -371,34 +364,51 @@ export class SqliteVectorStore {
       CREATE INDEX idx_chunks_file ON chunks(file);
     `);
 
-    // Insert metadata
-    const insertMeta = db.prepare(`INSERT INTO metadata (key, value) VALUES (?, ?)`);
-    insertMeta.run('version', String(STORE_VERSION));
-    insertMeta.run('dim', String(dim));
-    insertMeta.run('count', String(resolvedData.length));
-    insertMeta.run('createdAt', new Date().toISOString());
-
-    // Insert chunks in a transaction for speed
-    const insertChunk = db.prepare(`
-      INSERT INTO chunks (id, file, startLine, endLine, content, vector)
-      VALUES (?, ?, ?, ?, ?, ?)
-    `);
-
-    const insertMany = db.transaction((items) => {
-      for (let i = 0; i < items.length; i++) {
-        const item = items[i];
-        insertChunk.run(
-          i,
-          item.file,
-          item.startLine,
-          item.endLine,
-          item.content,
-          item.vectorBlob
-        );
-      }
-    });
-
-    insertMany(resolvedData);
+    // Insert metadata
+    const insertMeta = db.prepare(`INSERT INTO metadata (key, value) VALUES (?, ?)`);
+    insertMeta.run('version', String(STORE_VERSION));
+    insertMeta.run('dim', String(dim));
+    insertMeta.run('count', String(denseChunks.length));
+    insertMeta.run('createdAt', new Date().toISOString());
+
+    // Insert chunks in a transaction for speed without pre-materializing all vectors/content.
+    const insertChunk = db.prepare(`
+      INSERT INTO chunks (id, file, startLine, endLine, content, vector)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+
+    db.exec('BEGIN');
+    try {
+      for (let i = 0; i < denseChunks.length; i += 1) {
+        const chunk = denseChunks[i];
+        const sourceIndex = denseSourceIndices[i];
+        const vector = await resolveVector(chunk, sourceIndex);
+        if (vector.length !== dim) {
+          throw new Error('Vector dimension mismatch in sqlite cache write');
+        }
+        let content = await resolveContent(chunk, sourceIndex);
+        if (content === undefined) content = null;
+        if (content !== null && typeof content !== 'string') {
+          content = String(content);
+        }
+        insertChunk.run(
+          i,
+          chunk.file,
+          chunk.startLine ?? 0,
+          chunk.endLine ?? 0,
+          content,
+          Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength)
+        );
+      }
+      db.exec('COMMIT');
+    } catch (error) {
+      try {
+        db.exec('ROLLBACK');
+      } catch {
+        // ignore rollback errors
+      }
+      throw error;
+    }
 
     // Optimize the database
     db.exec('ANALYZE');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@softerist/heuristic-mcp",
-  "version": "3.0.12",
+  "version": "3.0.14",
   "description": "An enhanced MCP server providing intelligent semantic code search with find-similar-code, recency ranking, and improved chunking. Fork of smart-coding-mcp.",
   "type": "module",
   "main": "index.js",