@yamo/memory-mesh 3.2.2 → 3.2.4

package/README.md

@@ -47,14 +47,46 @@ The installer will:
 
 ### CLI
 
+The `memory-mesh` CLI provides seven commands for full subconscious CRUD and recall:
+
 ```bash
 # Store a memory (automatically scrubbed & embedded)
-node tools/memory_mesh.mjs store --content "My important memory" --type "insight"
+memory-mesh store --content "My important memory" --type insight
+
+# Store with full provenance metadata
+memory-mesh store -c "Insight text" -t decision -r "Improves latency" -h "Caching reduces p95"
+
+# Bulk-ingest a directory (recursive, by extension)
+memory-mesh pull ./docs --extension ".md,.yamo" --type documentation
+
+# Semantic search
+memory-mesh search "query about orchestration" --limit 5
+
+# Retrieve a specific record by ID
+memory-mesh get --id mem_abc123
 
-# Search memories
-node tools/memory_mesh.mjs search "query" --limit 5
+# Delete a record by ID
+memory-mesh delete --id mem_abc123
+
+# Synthesize insights from recent memories
+memory-mesh reflect --topic "bugs" --lookback 10
+
+# Database health and statistics
+memory-mesh stats
 ```
 
+**Command Reference:**
+
+| Command | Key Options | Description |
+|---------|-------------|-------------|
+| `store` | `-c/--content` (required), `-t/--type`, `-r/--rationale`, `-h/--hypothesis` | Persist a semantic memory |
+| `pull` | `<path>` (required), `-e/--extension`, `-t/--type` | Bulk-ingest a directory |
+| `search` | `<query>` (required), `-l/--limit` | Semantic recall |
+| `get` | `-i/--id` (required) | Fetch a record by ID |
+| `delete` | `-i/--id` (required) | Remove a record by ID |
+| `reflect` | `-t/--topic`, `-l/--lookback` | Synthesize insights from memories |
+| `stats` | — | DB health, count, embedding model |
+
 ### Node.js API
 
 ```javascript
@@ -173,7 +205,7 @@ docker run -v $(pwd)/data:/app/runtime/data \
 
 ## About YAMO Protocol
 
-Memory Mesh is built on the **YAMO (Yet Another Markup for Orchestration) Protocol** - a structured language for transparent AI agent collaboration with immutable provenance tracking.
+Memory Mesh is built on the **YAMO (Yet Another Model Ontology) Protocol** - a structured language for transparent AI agent collaboration with immutable provenance tracking.
 
 **YAMO Protocol Features:**
 - **Structured Agent Workflows**: Semicolon-terminated constraints, explicit handoff chains

package/bin/memory_mesh.js

@@ -21,7 +21,7 @@ const program = new Command();
 program
   .name('memory-mesh')
   .description('YAMO Semantic Subconscious - Protocol-Native CLI')
-  .version('3.2.0');
+  .version('3.2.3');
 
 // Helper for beautiful logging
 const ui = {
@@ -194,4 +194,85 @@ program
     }
   });
 
+// 5. Get Command
+program
+  .command('get')
+  .description('Retrieve a memory record by ID')
+  .requiredOption('-i, --id <id>', 'Memory record ID')
+  .action(async (options) => {
+    const mesh = new MemoryMesh();
+    try {
+      const record = await mesh.get(options.id);
+      if (!record) {
+        ui.warn(`Record not found: ${options.id}`);
+        process.exit(1);
+      }
+      const meta = typeof record.metadata === 'string' ? JSON.parse(record.metadata) : (record.metadata || {});
+      ui.header(`Memory ${options.id}`);
+      console.log(`${pc.bold('ID:')}       ${pc.dim(record.id)}`);
+      console.log(`${pc.bold('Type:')}     ${pc.dim(meta.type || 'event')}`);
+      console.log(`${pc.bold('Created:')} ${pc.dim(record.created_at)}`);
+      console.log(`\n${pc.white(record.content)}`);
+    } catch (err) {
+      ui.error(`Get failed: ${err.message}`);
+      process.exit(1);
+    } finally {
+      await mesh.close();
+    }
+  });
+
+// 6. Delete Command
+program
+  .command('delete')
+  .description('Permanently remove a memory record by ID')
+  .requiredOption('-i, --id <id>', 'Memory record ID to delete')
+  .action(async (options) => {
+    const mesh = new MemoryMesh();
+    try {
+      await mesh.delete(options.id);
+      ui.success(`Deleted record ${pc.bold(options.id)}`);
+    } catch (err) {
+      ui.error(`Delete failed: ${err.message}`);
+      process.exit(1);
+    } finally {
+      await mesh.close();
+    }
+  });
+
+// 7. Reflect Command
+program
+  .command('reflect')
+  .description('Synthesize insights from stored memories')
+  .option('-t, --topic <topic>', 'Focus the reflection on a specific topic')
+  .option('-l, --lookback <number>', 'Number of memories to review', '10')
+  .action(async (options) => {
+    const mesh = new MemoryMesh();
+    try {
+      ui.info(`Reflecting on ${options.topic ? `"${pc.italic(options.topic)}"` : 'recent memories'}...`);
+      const result = await mesh.reflect({
+        topic: options.topic,
+        lookback: parseInt(options.lookback),
+      });
+      ui.header('Reflection');
+      if (result.reflection) {
+        console.log(pc.white(result.reflection));
+        console.log(`\n${pc.bold('Confidence:')} ${pc.cyan((result.confidence * 100).toFixed(0))}%`);
+      } else {
+        console.log(pc.dim(`Reviewed ${result.count} memories${result.topic ? ` on topic: ${result.topic}` : ''}`));
+        console.log(`\n${pc.bold('Prompt for LLM:')}\n${pc.white(result.prompt)}`);
+        if (result.context?.length) {
+          console.log('');
+          result.context.forEach((m, i) => {
+            console.log(`${pc.cyan(`Memory ${i + 1}:`)} ${pc.white(m.content.substring(0, 200))}${m.content.length > 200 ? '...' : ''}`);
+          });
+        }
+      }
+    } catch (err) {
+      ui.error(`Reflect failed: ${err.message}`);
+      process.exit(1);
+    } finally {
+      await mesh.close();
+    }
+  });
+
 program.parse();

package/lib/memory/memory-mesh.js

@@ -288,8 +288,16 @@ export class MemoryMesh {
                         const skillSchema = createSynthesizedSkillSchema(this.vectorDimension);
                         this.skillTable = await this.client.db.createTable("synthesized_skills", [], {
                             schema: skillSchema,
+                            storageOptions: { new_table_data_storage_version: "stable" },
                         });
                     }
+                    // Migrate manifest paths to V2 layout (idempotent)
+                    try {
+                        await this.skillTable.migrateManifestPathsV2();
+                    }
+                    catch {
+                        // Already migrated or not a local table — ignore
+                    }
                     if (process.env.YAMO_DEBUG === "true") {
                         logger.debug("YAMO blocks and synthesized skills tables initialized");
                     }
@@ -1808,6 +1816,23 @@ export async function run() {
         else if (action === "stats") {
             process.stdout.write(`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}\n`);
         }
+        else if (action === "get") {
+            const record = await mesh.get(input.id);
+            if (!record) {
+                process.stdout.write(`[MemoryMesh] Record not found: ${input.id}\n${JSON.stringify({ status: "not_found", id: input.id })}\n`);
+            }
+            else {
+                process.stdout.write(`[MemoryMesh] Record ${record.id}\n${JSON.stringify({ status: "ok", record }, null, 2)}\n`);
+            }
+        }
+        else if (action === "delete") {
+            await mesh.delete(input.id);
+            process.stdout.write(`[MemoryMesh] Deleted record ${input.id}\n${JSON.stringify({ status: "ok", id: input.id })}\n`);
+        }
+        else if (action === "reflect") {
+            const result = await mesh.reflect({ topic: input.topic, lookback: input.lookback });
+            process.stdout.write(`[MemoryMesh] Reflection complete.\n${JSON.stringify({ status: "ok", result }, null, 2)}\n`);
+        }
         else {
             logger.error({ action }, "Unknown action");
             process.exit(1);

package/lib/memory/memory-mesh.ts

@@ -320,8 +320,16 @@ export class MemoryMesh {
                         const skillSchema = createSynthesizedSkillSchema(this.vectorDimension);
                         this.skillTable = await this.client.db.createTable("synthesized_skills", [], {
                             schema: skillSchema,
+                            storageOptions: { new_table_data_storage_version: "stable" },
                         });
                     }
+                    // Migrate manifest paths to V2 layout (idempotent)
+                    try {
+                        await this.skillTable.migrateManifestPathsV2();
+                    }
+                    catch {
+                        // Already migrated or not a local table — ignore
+                    }
                     if (process.env.YAMO_DEBUG === "true") {
                         logger.debug("YAMO blocks and synthesized skills tables initialized");
                     }
@@ -1895,6 +1903,22 @@ export async function run() {
         else if (action === "stats") {
             process.stdout.write(`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}\n`);
         }
+        else if (action === "get") {
+            const record = await mesh.get(input.id);
+            if (!record) {
+                process.stdout.write(`[MemoryMesh] Record not found: ${input.id}\n${JSON.stringify({ status: "not_found", id: input.id })}\n`);
+            } else {
+                process.stdout.write(`[MemoryMesh] Record ${record.id}\n${JSON.stringify({ status: "ok", record }, null, 2)}\n`);
+            }
+        }
+        else if (action === "delete") {
+            await mesh.delete(input.id);
+            process.stdout.write(`[MemoryMesh] Deleted record ${input.id}\n${JSON.stringify({ status: "ok", id: input.id })}\n`);
+        }
+        else if (action === "reflect") {
+            const result = await mesh.reflect({ topic: input.topic, lookback: input.lookback });
+            process.stdout.write(`[MemoryMesh] Reflection complete.\n${JSON.stringify({ status: "ok", result }, null, 2)}\n`);
+        }
         else {
             logger.error({ action }, "Unknown action");
             process.exit(1);

package/lib/memory/schema.d.ts

@@ -57,6 +57,14 @@ export declare function createSynthesizedSkillSchema(vectorDim?: number): arrow.
  * @returns {boolean} True if V2 schema detected
  */
 export declare function isSchemaV2(schema: any): any;
+/**
+ * Migrate an existing table to V2:
+ * 1. Migrate manifest paths to V2 layout (efficient versioning, idempotent)
+ * 2. Add nullable V2 columns to memory_entries-style tables if not already present
+ *
+ * Safe to call on any table — non-memory tables skip the schema column additions.
+ */
+export declare function migrateTableV2(table: any): Promise<void>;
 /**
  * Memory table schema using Apache Arrow format (default 384 dimensions)
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -113,6 +121,7 @@ declare const _default: {
     createMemorySchema: typeof createMemorySchema;
     createMemorySchemaV2: typeof createMemorySchemaV2;
     isSchemaV2: typeof isSchemaV2;
+    migrateTableV2: typeof migrateTableV2;
     getEmbeddingDimension: typeof getEmbeddingDimension;
     DEFAULT_VECTOR_DIMENSION: number;
     EMBEDDING_DIMENSIONS: {

package/lib/memory/schema.js

@@ -112,6 +112,47 @@ export function createSynthesizedSkillSchema(vectorDim = DEFAULT_VECTOR_DIMENSIO
 export function isSchemaV2(schema) {
     return schema.fields.some((f) => f.name === "session_id");
 }
+/**
+ * Migrate an existing table to V2:
+ * 1. Migrate manifest paths to V2 layout (efficient versioning, idempotent)
+ * 2. Add nullable V2 columns to memory_entries-style tables if not already present
+ *
+ * Safe to call on any table — non-memory tables skip the schema column additions.
+ */
+export async function migrateTableV2(table) {
+    // Step 1: manifest path migration (idempotent on already-migrated tables)
+    try {
+        await table.migrateManifestPathsV2();
+    }
+    catch {
+        // Already migrated or not a local table — ignore
+    }
+    // Step 2: add V2 schema columns if this is a memory_entries-style table (V1)
+    // Guard: schema() may not exist on mock tables in tests
+    if (typeof table.schema !== "function")
+        return;
+    let schema;
+    try {
+        schema = await table.schema();
+    }
+    catch {
+        return; // Can't inspect schema — skip
+    }
+    if (isSchemaV2(schema))
+        return;
+    // Only add V2 columns if the table has the V1 memory_entries shape
+    const fieldNames = schema.fields.map((f) => f.name);
+    if (!fieldNames.includes("content") || !fieldNames.includes("vector"))
+        return;
+    await table.addColumns([
+        { name: "session_id", valueSql: "cast(null as string)" },
+        { name: "agent_id", valueSql: "cast(null as string)" },
+        { name: "memory_type", valueSql: "cast(null as string)" },
+        { name: "importance_score", valueSql: "cast(null as float)" },
+        { name: "access_count", valueSql: "cast(null as int)" },
+        { name: "last_accessed", valueSql: "cast(null as timestamp)" },
+    ]);
+}
 /**
  * Memory table schema using Apache Arrow format (default 384 dimensions)
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -153,16 +194,21 @@ export async function createMemoryTable(db, tableName = "memory_entries") {
  */
 export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
     try {
-        // Check if table already exists
         const existingTables = await db.tableNames();
+        let table;
         if (existingTables.includes(tableName)) {
-            return await db.openTable(tableName);
+            table = await db.openTable(tableName);
+        }
+        else {
+            // New tables use V2 schema and stable storage format
+            const schema = createMemorySchemaV2(vectorDim);
+            table = await db.createTable(tableName, [], {
+                schema,
+                storageOptions: { new_table_data_storage_version: "stable" },
+            });
         }
-        // Create schema with specified dimension
-        const schema = createMemorySchema(vectorDim);
-        // Create table with schema
-        // LanceDB v0.23.0+ accepts empty array as initial data with schema option
-        const table = await db.createTable(tableName, [], { schema }); // Cast to any because lancedb types might be strict about options
+        // Migrate existing tables to V2 (manifest paths + schema columns, idempotent)
+        await migrateTableV2(table);
         return table;
     }
     catch (error) {
@@ -178,6 +224,7 @@ export default {
     createMemorySchema,
     createMemorySchemaV2,
     isSchemaV2,
+    migrateTableV2,
     getEmbeddingDimension,
     DEFAULT_VECTOR_DIMENSION,
     EMBEDDING_DIMENSIONS,

package/lib/memory/schema.ts

@@ -112,6 +112,44 @@ export function createSynthesizedSkillSchema(vectorDim = DEFAULT_VECTOR_DIMENSIO
 export function isSchemaV2(schema) {
     return schema.fields.some((f) => f.name === "session_id");
 }
+/**
+ * Migrate an existing table to V2:
+ * 1. Migrate manifest paths to V2 layout (efficient versioning, idempotent)
+ * 2. Add nullable V2 columns to memory_entries-style tables if not already present
+ *
+ * Safe to call on any table — non-memory tables skip the schema column additions.
+ */
+export async function migrateTableV2(table) {
+    // Step 1: manifest path migration (idempotent on already-migrated tables)
+    try {
+        await table.migrateManifestPathsV2();
+    }
+    catch {
+        // Already migrated or not a local table — ignore
+    }
+    // Step 2: add V2 schema columns if this is a memory_entries-style table (V1)
+    // Guard: schema() may not exist on mock tables in tests
+    if (typeof table.schema !== "function") return;
+    let schema;
+    try {
+        schema = await table.schema();
+    }
+    catch {
+        return; // Can't inspect schema — skip
+    }
+    if (isSchemaV2(schema)) return;
+    // Only add V2 columns if the table has the V1 memory_entries shape
+    const fieldNames = schema.fields.map((f) => f.name);
+    if (!fieldNames.includes("content") || !fieldNames.includes("vector")) return;
+    await table.addColumns([
+        { name: "session_id", valueSql: "cast(null as string)" },
+        { name: "agent_id", valueSql: "cast(null as string)" },
+        { name: "memory_type", valueSql: "cast(null as string)" },
+        { name: "importance_score", valueSql: "cast(null as float)" },
+        { name: "access_count", valueSql: "cast(null as int)" },
+        { name: "last_accessed", valueSql: "cast(null as timestamp)" },
+    ]);
+}
 /**
  * Memory table schema using Apache Arrow format (default 384 dimensions)
  * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
@@ -153,16 +191,21 @@ export async function createMemoryTable(db, tableName = "memory_entries") {
  */
 export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
     try {
-        // Check if table already exists
         const existingTables = await db.tableNames();
+        let table;
         if (existingTables.includes(tableName)) {
-            return await db.openTable(tableName);
+            table = await db.openTable(tableName);
+        }
+        else {
+            // New tables use V2 schema and stable storage format
+            const schema = createMemorySchemaV2(vectorDim);
+            table = await db.createTable(tableName, [], {
+                schema,
+                storageOptions: { new_table_data_storage_version: "stable" },
+            });
         }
-        // Create schema with specified dimension
-        const schema = createMemorySchema(vectorDim);
-        // Create table with schema
-        // LanceDB v0.23.0+ accepts empty array as initial data with schema option
-        const table = await db.createTable(tableName, [], { schema }); // Cast to any because lancedb types might be strict about options
+        // Migrate existing tables to V2 (manifest paths + schema columns, idempotent)
+        await migrateTableV2(table);
         return table;
     }
     catch (error) {
@@ -178,6 +221,7 @@ export default {
     createMemorySchema,
     createMemorySchemaV2,
     isSchemaV2,
+    migrateTableV2,
     getEmbeddingDimension,
     DEFAULT_VECTOR_DIMENSION,
     EMBEDDING_DIMENSIONS,

package/lib/yamo/schema.js

@@ -40,15 +40,25 @@ export function createYamoSchema() {
  */
 export async function createYamoTable(db, tableName = "yamo_blocks") {
     try {
-        // Check if table already exists
         const existingTables = await db.tableNames();
+        let table;
         if (existingTables.includes(tableName)) {
-            // Table exists, open it
-            return await db.openTable(tableName);
+            table = await db.openTable(tableName);
+        }
+        else {
+            const schema = createYamoSchema();
+            table = await db.createTable(tableName, [], {
+                schema,
+                storageOptions: { new_table_data_storage_version: "stable" },
+            });
+        }
+        // Migrate manifest paths to V2 layout (idempotent)
+        try {
+            await table.migrateManifestPathsV2();
+        }
+        catch {
+            // Already migrated or not a local table — ignore
         }
-        // Create new table with YAMO schema
-        const schema = createYamoSchema();
-        const table = await db.createTable(tableName, [], { schema });
         return table;
     }
     catch (error) {

package/lib/yamo/schema.ts

@@ -40,15 +40,25 @@ export function createYamoSchema() {
  */
 export async function createYamoTable(db, tableName = "yamo_blocks") {
     try {
-        // Check if table already exists
         const existingTables = await db.tableNames();
+        let table;
         if (existingTables.includes(tableName)) {
-            // Table exists, open it
-            return await db.openTable(tableName);
+            table = await db.openTable(tableName);
+        }
+        else {
+            const schema = createYamoSchema();
+            table = await db.createTable(tableName, [], {
+                schema,
+                storageOptions: { new_table_data_storage_version: "stable" },
+            });
+        }
+        // Migrate manifest paths to V2 layout (idempotent)
+        try {
+            await table.migrateManifestPathsV2();
+        }
+        catch {
+            // Already migrated or not a local table — ignore
         }
-        // Create new table with YAMO schema
-        const schema = createYamoSchema();
-        const table = await db.createTable(tableName, [], { schema });
         return table;
     }
     catch (error) {

package/package.json

@@ -1,7 +1,11 @@
 {
   "name": "@yamo/memory-mesh",
-  "version": "3.2.2",
+  "version": "3.2.4",
   "description": "Portable semantic memory system with Layer 0 Scrubber for YAMO agents (v3 Singularity Edition)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/yamo-protocol/yamo-memory-mesh"
+  },
   "type": "module",
   "main": "lib/memory/index.js",
   "types": "lib/memory/index.d.ts",
@@ -23,16 +27,16 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@lancedb/lancedb": "^0.23.0",
-    "@xenova/transformers": "^2.17.0",
-    "apache-arrow": "^17.0.0",
+    "@lancedb/lancedb": "^0.26.2",
+    "@xenova/transformers": "^2.17.2",
+    "apache-arrow": "^18.1.0",
     "commander": "^14.0.3",
-    "onnxruntime-node": "^1.18.0",
+    "onnxruntime-node": "^1.24.3",
     "pino": "^10.3.1",
     "pino-pretty": "^13.1.3",
     "cli-progress": "^3.12.0",
     "picocolors": "^1.1.1",
-    "glob": "^13.0.5"
+    "glob": "^13.0.6"
   },
   "author": "Soverane Labs",
   "license": "MIT",
@@ -40,9 +44,9 @@
     "node": ">=18.0.0"
   },
   "devDependencies": {
-    "@types/node": "^25.0.9",
+    "@types/node": "^25.4.0",
     "cohere-ai": "^7.20.0",
-    "openai": "^6.16.0",
+    "openai": "^6.27.0",
     "typescript": "^5.9.3"
   }
 }