@yamo/memory-mesh 3.2.4 → 3.2.5

package/lib/memory/adapters/client.d.ts

@@ -85,6 +85,11 @@ export declare class LanceDBClient {
      * @throws {QueryError} If stats query fails
      */
     getStats(): Promise<any>;
+    /**
+     * Compact old data files and prune versions older than 7 days.
+     * Best-effort — never throws.
+     */
+    optimize(): Promise<void>;
     /**
      * Sanitize an ID to prevent SQL injection
      * Removes any characters that aren't alphanumeric, underscore, or hyphen

package/lib/memory/adapters/client.js

@@ -201,7 +201,7 @@ export class LanceDBClient {
             await this.connect();
         }
         this._validateVector(vector);
-        const { limit = 10, nprobes = 20, filter = null } = options;
+        const { limit = 10, nprobes = 20, filter = null, refineFactor, timeoutMs } = options;
         return this._retryOperation(async () => {
             if (!this.table) {
                 throw new StorageError("Table not initialized");
@@ -217,12 +217,21 @@ export class LanceDBClient {
                     // ignore
                 }
             }
+            // Apply refineFactor for improved ANN recall (fetches N×candidates, reranks)
+            if (refineFactor && typeof refineFactor === "number") {
+                try {
+                    query = query.refineFactor(refineFactor);
+                }
+                catch (_e) {
+                    // ignore if not supported
+                }
+            }
             // Apply filter if provided
             if (filter) {
                 query = query.where(filter);
             }
-            // Execute search with limit
-            const resultsArray = await query.limit(limit).toArray();
+            // Execute search with limit (and optional timeout)
+            const resultsArray = await query.limit(limit).toArray(timeoutMs ? { timeoutMs } : undefined);
             return resultsArray.map((row) => ({
                 id: row.id,
                 content: row.content,
@@ -385,6 +394,22 @@ export class LanceDBClient {
             };
         });
     }
+    /**
+     * Compact old data files and prune versions older than 7 days.
+     * Best-effort — never throws.
+     */
+    async optimize() {
+        if (!this.isConnected || !this.table)
+            return;
+        try {
+            await this.table.optimize({
+                cleanupOlderThan: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
+            });
+        }
+        catch (_e) {
+            // Best-effort — never block normal operations
+        }
+    }
     /**
      * Sanitize an ID to prevent SQL injection
      * Removes any characters that aren't alphanumeric, underscore, or hyphen

package/lib/memory/adapters/client.ts

@@ -201,7 +201,7 @@ export class LanceDBClient {
             await this.connect();
         }
         this._validateVector(vector);
-        const { limit = 10, nprobes = 20, filter = null } = options;
+        const { limit = 10, nprobes = 20, filter = null, refineFactor, timeoutMs } = options;
         return this._retryOperation(async () => {
             if (!this.table) {
                 throw new StorageError("Table not initialized");
@@ -217,12 +217,23 @@ export class LanceDBClient {
                     // ignore
                 }
             }
+            // Apply refineFactor for improved ANN recall (fetches N×candidates, reranks)
+            if (refineFactor && typeof refineFactor === "number") {
+                try {
+                    query = query.refineFactor(refineFactor);
+                }
+                catch (_e) {
+                    // ignore if not supported
+                }
+            }
             // Apply filter if provided
             if (filter) {
                 query = query.where(filter);
             }
-            // Execute search with limit
-            const resultsArray = await query.limit(limit).toArray();
+            // Execute search with limit (and optional timeout)
+            const resultsArray = await query.limit(limit).toArray(
+                timeoutMs ? { timeoutMs } : undefined,
+            );
             return resultsArray.map((row) => ({
                 id: row.id,
                 content: row.content,
@@ -385,6 +396,21 @@ export class LanceDBClient {
             };
         });
     }
+    /**
+     * Compact old data files and prune versions older than 7 days.
+     * Best-effort — never throws.
+     */
+    async optimize() {
+        if (!this.isConnected || !this.table) return;
+        try {
+            await this.table.optimize({
+                cleanupOlderThan: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
+            });
+        }
+        catch (_e) {
+            // Best-effort — never block normal operations
+        }
+    }
     /**
      * Sanitize an ID to prevent SQL injection
      * Removes any characters that aren't alphanumeric, underscore, or hyphen

package/lib/memory/memory-mesh.d.ts

@@ -444,6 +444,11 @@ export declare class MemoryMesh {
      * await mesh.close(); // Clean up
      * ```
      */
+    /**
+     * Compact old data files and prune versions older than 7 days.
+     * Best-effort — delegates to LanceDBClient.optimize().
+     */
+    optimize(): Promise<any>;
     close(): Promise<void>;
 }
 /**

package/lib/memory/memory-mesh.js

@@ -1714,6 +1714,13 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
      * await mesh.close(); // Clean up
      * ```
      */
+    /**
+     * Compact old data files and prune versions older than 7 days.
+     * Best-effort — delegates to LanceDBClient.optimize().
+     */
+    async optimize() {
+        return this.client?.optimize?.();
+    }
     // eslint-disable-next-line @typescript-eslint/require-await
     async close() {
         try {

package/lib/memory/memory-mesh.ts

@@ -1801,6 +1801,13 @@ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
      * await mesh.close(); // Clean up
      * ```
      */
+    /**
+     * Compact old data files and prune versions older than 7 days.
+     * Best-effort — delegates to LanceDBClient.optimize().
+     */
+    async optimize() {
+        return this.client?.optimize?.();
+    }
     // eslint-disable-next-line @typescript-eslint/require-await
     async close() {
         try {

package/lib/memory/schema.d.ts

@@ -65,6 +65,12 @@ export declare function isSchemaV2(schema: any): any;
  * Safe to call on any table — non-memory tables skip the schema column additions.
  */
 export declare function migrateTableV2(table: any): Promise<void>;
+/**
+ * Ensure the vector column has an IVF_PQ index.
+ * Skipped when: table has too few rows, index already exists, or table is a mock.
+ * Called automatically by createMemoryTableWithDimension after migration.
+ */
+export declare function ensureVectorIndex(table: any): Promise<void>;
 /**
  * Memory table schema using Apache Arrow format (default 384 dimensions)
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -122,6 +128,7 @@ declare const _default: {
     createMemorySchemaV2: typeof createMemorySchemaV2;
     isSchemaV2: typeof isSchemaV2;
     migrateTableV2: typeof migrateTableV2;
+    ensureVectorIndex: typeof ensureVectorIndex;
     getEmbeddingDimension: typeof getEmbeddingDimension;
     DEFAULT_VECTOR_DIMENSION: number;
     EMBEDDING_DIMENSIONS: {

package/lib/memory/schema.js

@@ -9,6 +9,7 @@
  * - text-embedding-3-small: 1536 dimensions
  */
 import * as arrow from "apache-arrow";
+import { Index } from "@lancedb/lancedb";
 /**
  * Default vector dimension (all-MiniLM-L6-v2)
  */
@@ -153,6 +154,34 @@ export async function migrateTableV2(table) {
         { name: "last_accessed", valueSql: "cast(null as timestamp)" },
     ]);
 }
+/**
+ * Ensure the vector column has an IVF_PQ index.
+ * Skipped when: table has too few rows, index already exists, or table is a mock.
+ * Called automatically by createMemoryTableWithDimension after migration.
+ */
+export async function ensureVectorIndex(table) {
+    if (typeof table.listIndices !== "function")
+        return;
+    try {
+        const indices = await table.listIndices();
+        if (indices.some((i) => i.columns.includes("vector")))
+            return;
+        const rowCount = await table.countRows();
+        if (rowCount < INDEX_CONFIG.vector.num_partitions)
+            return;
+        await table.createIndex("vector", {
+            config: Index.ivfPq({
+                numPartitions: INDEX_CONFIG.vector.num_partitions,
+                numSubVectors: INDEX_CONFIG.vector.num_sub_vectors,
+                distanceType: INDEX_CONFIG.vector.metric,
+            }),
+            replace: false,
+        });
+    }
+    catch {
+        // Index creation is best-effort — never block table access
+    }
+}
 /**
  * Memory table schema using Apache Arrow format (default 384 dimensions)
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -209,6 +238,8 @@ export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
         }
         // Migrate existing tables to V2 (manifest paths + schema columns, idempotent)
         await migrateTableV2(table);
+        // Ensure vector index exists (no-op if already present or insufficient rows)
+        await ensureVectorIndex(table);
         return table;
     }
     catch (error) {
@@ -225,6 +256,7 @@ export default {
     createMemorySchemaV2,
     isSchemaV2,
     migrateTableV2,
+    ensureVectorIndex,
     getEmbeddingDimension,
     DEFAULT_VECTOR_DIMENSION,
     EMBEDDING_DIMENSIONS,

package/lib/memory/schema.ts

@@ -9,6 +9,8 @@
  * - text-embedding-3-small: 1536 dimensions
  */
 import * as arrow from "apache-arrow";
+import * as lancedb from "@lancedb/lancedb";
+import { Index } from "@lancedb/lancedb";
 /**
  * Default vector dimension (all-MiniLM-L6-v2)
  */
@@ -150,6 +152,31 @@ export async function migrateTableV2(table) {
         { name: "last_accessed", valueSql: "cast(null as timestamp)" },
     ]);
 }
+/**
+ * Ensure the vector column has an IVF_PQ index.
+ * Skipped when: table has too few rows, index already exists, or table is a mock.
+ * Called automatically by createMemoryTableWithDimension after migration.
+ */
+export async function ensureVectorIndex(table) {
+    if (typeof table.listIndices !== "function") return;
+    try {
+        const indices = await table.listIndices();
+        if (indices.some((i) => i.columns.includes("vector"))) return;
+        const rowCount = await table.countRows();
+        if (rowCount < INDEX_CONFIG.vector.num_partitions) return;
+        await table.createIndex("vector", {
+            config: Index.ivfPq({
+                numPartitions: INDEX_CONFIG.vector.num_partitions,
+                numSubVectors: INDEX_CONFIG.vector.num_sub_vectors,
+                distanceType: INDEX_CONFIG.vector.metric,
+            }),
+            replace: false,
+        });
+    }
+    catch {
+        // Index creation is best-effort — never block table access
+    }
+}
 /**
  * Memory table schema using Apache Arrow format (default 384 dimensions)
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -206,6 +233,8 @@ export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
         }
         // Migrate existing tables to V2 (manifest paths + schema columns, idempotent)
         await migrateTableV2(table);
+        // Ensure vector index exists (no-op if already present or insufficient rows)
+        await ensureVectorIndex(table);
         return table;
     }
     catch (error) {
@@ -222,6 +251,7 @@ export default {
     createMemorySchemaV2,
     isSchemaV2,
     migrateTableV2,
+    ensureVectorIndex,
     getEmbeddingDimension,
     DEFAULT_VECTOR_DIMENSION,
     EMBEDDING_DIMENSIONS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yamo/memory-mesh",
-  "version": "3.2.4",
+  "version": "3.2.5",
   "description": "Portable semantic memory system with Layer 0 Scrubber for YAMO agents (v3 Singularity Edition)",
   "repository": {
     "type": "git",