@yamo/memory-mesh 2.3.2 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/bin/memory_mesh.js +1 -1
  2. package/lib/llm/client.d.ts +111 -0
  3. package/lib/llm/client.js +299 -357
  4. package/lib/llm/client.ts +413 -0
  5. package/lib/llm/index.d.ts +17 -0
  6. package/lib/llm/index.js +15 -8
  7. package/lib/llm/index.ts +19 -0
  8. package/lib/memory/adapters/client.d.ts +183 -0
  9. package/lib/memory/adapters/client.js +518 -0
  10. package/lib/memory/adapters/client.ts +678 -0
  11. package/lib/memory/adapters/config.d.ts +137 -0
  12. package/lib/memory/adapters/config.js +189 -0
  13. package/lib/memory/adapters/config.ts +259 -0
  14. package/lib/memory/adapters/errors.d.ts +76 -0
  15. package/lib/memory/adapters/errors.js +128 -0
  16. package/lib/memory/adapters/errors.ts +166 -0
  17. package/lib/memory/context-manager.d.ts +44 -0
  18. package/lib/memory/context-manager.js +344 -0
  19. package/lib/memory/context-manager.ts +432 -0
  20. package/lib/memory/embeddings/factory.d.ts +59 -0
  21. package/lib/memory/embeddings/factory.js +148 -0
  22. package/lib/{embeddings/factory.js → memory/embeddings/factory.ts} +69 -28
  23. package/lib/memory/embeddings/index.d.ts +2 -0
  24. package/lib/memory/embeddings/index.js +2 -0
  25. package/lib/memory/embeddings/index.ts +2 -0
  26. package/lib/memory/embeddings/service.d.ts +164 -0
  27. package/lib/memory/embeddings/service.js +515 -0
  28. package/lib/{embeddings/service.js → memory/embeddings/service.ts} +223 -156
  29. package/lib/memory/index.d.ts +9 -0
  30. package/lib/memory/index.js +9 -1
  31. package/lib/memory/index.ts +20 -0
  32. package/lib/memory/memory-mesh.d.ts +274 -0
  33. package/lib/memory/memory-mesh.js +1469 -678
  34. package/lib/memory/memory-mesh.ts +1803 -0
  35. package/lib/memory/memory-translator.d.ts +19 -0
  36. package/lib/memory/memory-translator.js +125 -0
  37. package/lib/memory/memory-translator.ts +158 -0
  38. package/lib/memory/schema.d.ts +111 -0
  39. package/lib/memory/schema.js +183 -0
  40. package/lib/memory/schema.ts +267 -0
  41. package/lib/memory/scorer.d.ts +26 -0
  42. package/lib/memory/scorer.js +77 -0
  43. package/lib/memory/scorer.ts +95 -0
  44. package/lib/memory/search/index.d.ts +1 -0
  45. package/lib/memory/search/index.js +1 -0
  46. package/lib/memory/search/index.ts +1 -0
  47. package/lib/memory/search/keyword-search.d.ts +62 -0
  48. package/lib/memory/search/keyword-search.js +135 -0
  49. package/lib/{search/keyword-search.js → memory/search/keyword-search.ts} +66 -36
  50. package/lib/scrubber/config/defaults.d.ts +53 -0
  51. package/lib/scrubber/config/defaults.js +49 -57
  52. package/lib/scrubber/config/defaults.ts +117 -0
  53. package/lib/scrubber/index.d.ts +6 -0
  54. package/lib/scrubber/index.js +3 -23
  55. package/lib/scrubber/index.ts +7 -0
  56. package/lib/scrubber/scrubber.d.ts +61 -0
  57. package/lib/scrubber/scrubber.js +99 -121
  58. package/lib/scrubber/scrubber.ts +168 -0
  59. package/lib/scrubber/stages/chunker.d.ts +13 -0
  60. package/lib/scrubber/stages/metadata-annotator.d.ts +18 -0
  61. package/lib/scrubber/stages/normalizer.d.ts +13 -0
  62. package/lib/scrubber/stages/semantic-filter.d.ts +13 -0
  63. package/lib/scrubber/stages/structural-cleaner.d.ts +13 -0
  64. package/lib/scrubber/stages/validator.d.ts +18 -0
  65. package/lib/scrubber/telemetry.d.ts +36 -0
  66. package/lib/scrubber/telemetry.js +53 -58
  67. package/lib/scrubber/telemetry.ts +99 -0
  68. package/lib/utils/logger.d.ts +29 -0
  69. package/lib/utils/logger.js +64 -0
  70. package/lib/utils/logger.ts +85 -0
  71. package/lib/utils/skill-metadata.d.ts +32 -0
  72. package/lib/utils/skill-metadata.js +132 -0
  73. package/lib/utils/skill-metadata.ts +147 -0
  74. package/lib/yamo/emitter.d.ts +73 -0
  75. package/lib/yamo/emitter.js +78 -143
  76. package/lib/yamo/emitter.ts +249 -0
  77. package/lib/yamo/schema.d.ts +58 -0
  78. package/lib/yamo/schema.js +81 -108
  79. package/lib/yamo/schema.ts +165 -0
  80. package/package.json +11 -8
  81. package/index.d.ts +0 -111
  82. package/lib/embeddings/index.js +0 -2
  83. package/lib/index.js +0 -6
  84. package/lib/lancedb/client.js +0 -633
  85. package/lib/lancedb/config.js +0 -215
  86. package/lib/lancedb/errors.js +0 -144
  87. package/lib/lancedb/index.js +0 -4
  88. package/lib/lancedb/schema.js +0 -217
  89. package/lib/scrubber/errors/scrubber-error.js +0 -43
  90. package/lib/scrubber/stages/chunker.js +0 -103
  91. package/lib/scrubber/stages/metadata-annotator.js +0 -74
  92. package/lib/scrubber/stages/normalizer.js +0 -59
  93. package/lib/scrubber/stages/semantic-filter.js +0 -61
  94. package/lib/scrubber/stages/structural-cleaner.js +0 -82
  95. package/lib/scrubber/stages/validator.js +0 -66
  96. package/lib/scrubber/utils/hash.js +0 -39
  97. package/lib/scrubber/utils/html-parser.js +0 -45
  98. package/lib/scrubber/utils/pattern-matcher.js +0 -63
  99. package/lib/scrubber/utils/token-counter.js +0 -31
  100. package/lib/search/index.js +0 -1
  101. package/lib/utils/index.js +0 -1
  102. package/lib/yamo/index.js +0 -15
@@ -12,714 +12,1505 @@
12
12
  * Also supports STDIN input for YAMO skill compatibility:
13
13
  * echo '{"action": "ingest", "content": "..."}' | node tools/memory_mesh.js
14
14
  */
15
-
16
- import { fileURLToPath } from 'url';
15
+ import { fileURLToPath } from "url";
17
16
  import fs from "fs";
17
+ import path from "path";
18
18
  import crypto from "crypto";
19
- import { LanceDBClient } from "../lancedb/client.js";
20
- import { getConfig } from "../lancedb/config.js";
21
- import { getEmbeddingDimension } from "../lancedb/schema.js";
22
- import { handleError, StorageError, QueryError } from "../lancedb/errors.js";
23
- import EmbeddingFactory from "../embeddings/factory.js";
19
+ import { LanceDBClient } from "./adapters/client.js";
20
+ import { getConfig } from "./adapters/config.js";
21
+ import { getEmbeddingDimension, createSynthesizedSkillSchema, } from "./schema.js";
22
+ import { handleError } from "./adapters/errors.js";
23
+ import EmbeddingFactory from "./embeddings/factory.js";
24
24
  import { Scrubber } from "../scrubber/scrubber.js";
25
- import { KeywordSearch } from "../search/keyword-search.js";
25
+ import { extractSkillIdentity, extractSkillTags, } from "../utils/skill-metadata.js";
26
+ import { KeywordSearch } from "./search/keyword-search.js";
26
27
  import { YamoEmitter } from "../yamo/emitter.js";
27
28
  import { LLMClient } from "../llm/client.js";
28
-
29
+ import * as lancedb from "@lancedb/lancedb";
30
+ import { createLogger } from "../utils/logger.js";
31
+ const logger = createLogger("brain");
29
32
  /**
30
33
  * MemoryMesh class for managing vector memory storage
31
34
  */
32
- class MemoryMesh {
33
- /**
34
- * Create a new MemoryMesh instance
35
- * @param {Object} [options={}]
36
- * @param {boolean} [options.enableYamo=true]
37
- * @param {boolean} [options.enableLLM=true]
38
- * @param {string} [options.agentId='default']
39
- * @param {string} [options.llmProvider]
40
- * @param {string} [options.llmApiKey]
41
- * @param {string} [options.llmModel]
42
- */
43
- constructor(options = {}) {
44
- this.client = null;
45
- this.config = null;
46
- this.embeddingFactory = new EmbeddingFactory();
47
- this.keywordSearch = new KeywordSearch();
48
- this.isInitialized = false;
49
- this.vectorDimension = 384; // Will be set during init()
50
-
51
- // YAMO and LLM support
52
- this.enableYamo = options.enableYamo !== false; // Default: true
53
- this.enableLLM = options.enableLLM !== false; // Default: true
54
- this.agentId = options.agentId || 'default';
55
- this.yamoTable = null; // Will be initialized in init()
56
- this.skillTable = null; // Synthesized skills table
57
- this.llmClient = null;
58
-
59
- // Initialize LLM client if enabled
60
- if (this.enableLLM) {
61
- this.llmClient = new LLMClient({
62
- provider: options.llmProvider,
63
- apiKey: options.llmApiKey,
64
- model: options.llmModel
65
- });
35
+ export class MemoryMesh {
36
+ client;
37
+ config;
38
+ embeddingFactory;
39
+ keywordSearch;
40
+ isInitialized;
41
+ vectorDimension;
42
+ enableYamo;
43
+ enableLLM;
44
+ enableMemory;
45
+ agentId;
46
+ yamoTable;
47
+ skillTable;
48
+ llmClient;
49
+ scrubber;
50
+ queryCache;
51
+ cacheConfig;
52
+ skillDirectories; // Store skill directories for synthesis
53
+ dbDir; // Store custom dbDir for in-memory databases
54
+ /**
55
+ * Create a new MemoryMesh instance
56
+ * @param {Object} [options={}]
57
+ */
58
+ constructor(options = {}) {
59
+ this.client = null;
60
+ this.config = null;
61
+ this.embeddingFactory = new EmbeddingFactory();
62
+ this.keywordSearch = new KeywordSearch();
63
+ this.isInitialized = false;
64
+ this.vectorDimension = 384; // Will be set during init()
65
+ // YAMO and LLM support
66
+ this.enableYamo = options.enableYamo !== false;
67
+ this.enableLLM = options.enableLLM !== false;
68
+ this.enableMemory = options.enableMemory !== false;
69
+ this.agentId = options.agentId || "YAMO_AGENT";
70
+ this.yamoTable = null;
71
+ this.skillTable = null;
72
+ this.llmClient = this.enableLLM ? new LLMClient() : null;
73
+ // Store skill directories for synthesis
74
+ if (Array.isArray(options.skill_directories)) {
75
+ this.skillDirectories = options.skill_directories;
76
+ }
77
+ else if (options.skill_directories) {
78
+ this.skillDirectories = [options.skill_directories];
79
+ }
80
+ else {
81
+ this.skillDirectories = ["skills"];
82
+ }
83
+ // Initialize LLM client if enabled
84
+ if (this.enableLLM) {
85
+ this.llmClient = new LLMClient({
86
+ provider: options.llmProvider,
87
+ apiKey: options.llmApiKey,
88
+ model: options.llmModel,
89
+ maxTokens: options.llmMaxTokens,
90
+ });
91
+ }
92
+ // Scrubber for Layer 0 sanitization
93
+ this.scrubber = new Scrubber({
94
+ enabled: true,
95
+ chunking: {
96
+ minTokens: 1, // Allow short memories
97
+ }, // Type cast for partial config
98
+ validation: {
99
+ enforceMinLength: false, // Disable strict length validation
100
+ },
101
+ });
102
+ // Simple LRU cache for search queries (5 minute TTL)
103
+ this.queryCache = new Map();
104
+ this.cacheConfig = {
105
+ maxSize: 500,
106
+ ttlMs: 5 * 60 * 1000, // 5 minutes
107
+ };
108
+ // Store custom dbDir for test isolation
109
+ this.dbDir = options.dbDir;
66
110
  }
67
-
68
- // Scrubber for Layer 0 sanitization
69
- this.scrubber = new Scrubber({
70
- enabled: true,
71
- chunking: {
72
- minTokens: 1 // Allow short memories
73
- },
74
- validation: {
75
- enforceMinLength: false // Disable strict length validation
76
- }
77
- });
78
-
79
- // Simple LRU cache for search queries (5 minute TTL)
80
- this.queryCache = new Map();
81
- this.cacheConfig = {
82
- maxSize: 500,
83
- ttlMs: 5 * 60 * 1000, // 5 minutes
84
- };
85
- }
86
-
87
- /**
88
- * Generate a cache key from query and options
89
- * @private
90
- */
91
- _generateCacheKey(query, options = {}) {
92
- const normalizedOptions = {
93
- limit: options.limit || 10,
94
- filter: options.filter || null,
95
- // Normalize options that affect results
96
- };
97
- return `search:${query}:${JSON.stringify(normalizedOptions)}`;
98
- }
99
-
100
- /**
101
- * Get cached result if valid
102
- * @private
103
- */
104
- _getCachedResult(key) {
105
- const entry = this.queryCache.get(key);
106
- if (!entry) return null;
107
-
108
- // Check TTL
109
- if (Date.now() - entry.timestamp > this.cacheConfig.ttlMs) {
110
- this.queryCache.delete(key);
111
- return null;
111
+ /**
112
+ * Generate a cache key from query and options
113
+ * @private
114
+ */
115
+ _generateCacheKey(query, options = {}) {
116
+ const normalizedOptions = {
117
+ limit: options.limit || 10,
118
+ filter: options.filter || null,
119
+ // Normalize options that affect results
120
+ };
121
+ return `search:${query}:${JSON.stringify(normalizedOptions)}`;
112
122
  }
113
-
114
- // Move to end (most recently used)
115
- this.queryCache.delete(key);
116
- this.queryCache.set(key, entry);
117
-
118
- return entry.result;
119
- }
120
-
121
- /**
122
- * Cache a search result
123
- * @private
124
- */
125
- _cacheResult(key, result) {
126
- // Evict oldest if at max size
127
- if (this.queryCache.size >= this.cacheConfig.maxSize) {
128
- const firstKey = this.queryCache.keys().next().value;
129
- this.queryCache.delete(firstKey);
123
+ /**
124
+ * Get cached result if valid
125
+ * @private
126
+ *
127
+ * Race condition fix: The delete-then-set pattern for LRU tracking creates a window
128
+ * where another operation could observe the key as missing. We use a try-finally
129
+ * pattern to ensure atomicity at the application level.
130
+ */
131
+ _getCachedResult(key) {
132
+ const entry = this.queryCache.get(key);
133
+ if (!entry) {
134
+ return null;
135
+ }
136
+ // Check TTL - must be done before any mutation
137
+ const now = Date.now();
138
+ if (now - entry.timestamp > this.cacheConfig.ttlMs) {
139
+ this.queryCache.delete(key);
140
+ return null;
141
+ }
142
+ // Move to end (most recently used) - delete and re-add with updated timestamp
143
+ // While not truly atomic, the key remains accessible during the operation
144
+ // since we already have the entry reference
145
+ this.queryCache.delete(key);
146
+ this.queryCache.set(key, {
147
+ ...entry,
148
+ timestamp: now, // Update timestamp for LRU tracking
149
+ });
150
+ return entry.result;
130
151
  }
131
-
132
- this.queryCache.set(key, {
133
- result,
134
- timestamp: Date.now()
135
- });
136
- }
137
-
138
- /**
139
- * Clear all cached results
140
- */
141
- clearCache() {
142
- this.queryCache.clear();
143
- }
144
-
145
- /**
146
- * Get cache statistics
147
- */
148
- getCacheStats() {
149
- return {
150
- size: this.queryCache.size,
151
- maxSize: this.cacheConfig.maxSize,
152
- ttlMs: this.cacheConfig.ttlMs
153
- };
154
- }
155
-
156
- /**
157
- * Validate and sanitize metadata to prevent prototype pollution
158
- * @private
159
- */
160
- _validateMetadata(metadata) {
161
- if (typeof metadata !== 'object' || metadata === null) {
162
- throw new Error('Metadata must be a non-null object');
152
+ /**
153
+ * Cache a search result
154
+ * @private
155
+ */
156
+ _cacheResult(key, result) {
157
+ // Evict oldest if at max size
158
+ if (this.queryCache.size >= this.cacheConfig.maxSize) {
159
+ const firstKey = this.queryCache.keys().next().value;
160
+ if (firstKey !== undefined) {
161
+ this.queryCache.delete(firstKey);
162
+ }
163
+ }
164
+ this.queryCache.set(key, {
165
+ result,
166
+ timestamp: Date.now(),
167
+ });
163
168
  }
164
-
165
- // Sanitize keys to prevent prototype pollution
166
- const sanitized = {};
167
- for (const [key, value] of Object.entries(metadata)) {
168
- // Skip dangerous keys that could pollute prototype
169
- // Note: 'constructor' and 'prototype' are handled by hasOwnProperty check
170
- // '.__proto__' needs explicit check because Object.entries() doesn't include it
171
- if (key === '__proto__' || key === 'constructor' || key === 'prototype') {
172
- continue;
173
- }
174
- // Skip inherited properties
175
- if (!Object.prototype.hasOwnProperty.call(metadata, key)) {
176
- continue;
177
- }
178
- sanitized[key] = value;
179
- }
180
- return sanitized;
181
- }
182
-
183
- /**
184
- * Sanitize and validate content before storage
185
- * @private
186
- */
187
- _sanitizeContent(content) {
188
- if (typeof content !== 'string') {
189
- throw new Error('Content must be a string');
169
+ /**
170
+ * Clear all cached results
171
+ */
172
+ clearCache() {
173
+ this.queryCache.clear();
190
174
  }
191
-
192
- // Limit content length
193
- const MAX_CONTENT_LENGTH = 100000; // 100KB limit
194
- if (content.length > MAX_CONTENT_LENGTH) {
195
- throw new Error(`Content exceeds maximum length of ${MAX_CONTENT_LENGTH} characters`);
175
+ /**
176
+ * Get cache statistics
177
+ */
178
+ getCacheStats() {
179
+ return {
180
+ size: this.queryCache.size,
181
+ maxSize: this.cacheConfig.maxSize,
182
+ ttlMs: this.cacheConfig.ttlMs,
183
+ };
196
184
  }
197
-
198
- return content.trim();
199
- }
200
-
201
- /**
202
- * Initialize the LanceDB client
203
- */
204
- async init() {
205
- if (this.isInitialized) {
206
- return;
185
+ /**
186
+ * Validate and sanitize metadata to prevent prototype pollution
187
+ * @private
188
+ */
189
+ _validateMetadata(metadata) {
190
+ if (typeof metadata !== "object" || metadata === null) {
191
+ throw new Error("Metadata must be a non-null object");
192
+ }
193
+ // Sanitize keys to prevent prototype pollution
194
+ const sanitized = {};
195
+ for (const [key, value] of Object.entries(metadata)) {
196
+ // Skip dangerous keys that could pollute prototype
197
+ if (key === "__proto__" || key === "constructor" || key === "prototype") {
198
+ continue;
199
+ }
200
+ // Skip inherited properties
201
+ if (!Object.prototype.hasOwnProperty.call(metadata, key)) {
202
+ continue;
203
+ }
204
+ sanitized[key] = value;
205
+ }
206
+ return sanitized;
207
207
  }
208
-
209
- try {
210
- // Load configuration
211
- this.config = getConfig();
212
-
213
- // Detect vector dimension from embedding model configuration
214
- const modelName = process.env.EMBEDDING_MODEL_NAME || 'Xenova/all-MiniLM-L6-v2';
215
- const envDimension = parseInt(process.env.EMBEDDING_DIMENSION || '0') || null;
216
- this.vectorDimension = envDimension || getEmbeddingDimension(modelName);
217
-
218
- // Only log in debug mode to avoid corrupting spinner/REPL display
219
- if (process.env.YAMO_DEBUG === 'true') {
220
- console.error(`[MemoryMesh] Using vector dimension: ${this.vectorDimension} (model: ${modelName})`);
221
- }
222
-
223
- // Create LanceDBClient with detected dimension
224
- this.client = new LanceDBClient({
225
- uri: this.config.LANCEDB_URI,
226
- tableName: this.config.LANCEDB_MEMORY_TABLE,
227
- vectorDimension: this.vectorDimension,
228
- maxRetries: 3,
229
- retryDelay: 1000
230
- });
231
-
232
- // Connect to database
233
- await this.client.connect();
234
-
235
- // Configure embedding factory from environment
236
- const embeddingConfigs = this._parseEmbeddingConfig();
237
- this.embeddingFactory.configure(embeddingConfigs);
238
- await this.embeddingFactory.init();
239
-
240
- // Hydrate Keyword Search (In-Memory)
241
- if (this.client) {
208
+ /**
209
+ * Sanitize and validate content before storage
210
+ * @private
211
+ */
212
+ _sanitizeContent(content) {
213
+ if (typeof content !== "string") {
214
+ throw new Error("Content must be a string");
215
+ }
216
+ // Limit content length
217
+ const MAX_CONTENT_LENGTH = 100000; // 100KB limit
218
+ if (content.length > MAX_CONTENT_LENGTH) {
219
+ throw new Error(`Content exceeds maximum length of ${MAX_CONTENT_LENGTH} characters`);
220
+ }
221
+ return content.trim();
222
+ }
223
+ /**
224
+ * Initialize the LanceDB client
225
+ */
226
+ async init() {
227
+ if (this.isInitialized) {
228
+ return;
229
+ }
230
+ if (!this.enableMemory) {
231
+ this.isInitialized = true;
232
+ if (process.env.YAMO_DEBUG === "true") {
233
+ logger.debug("MemoryMesh initialization skipped (enableMemory=false)");
234
+ }
235
+ return;
236
+ }
242
237
  try {
243
- const allRecords = await this.client.getAll({ limit: 10000 });
244
- this.keywordSearch.load(allRecords);
245
- } catch (e) {
246
- // Ignore if table doesn't exist yet
238
+ // Load configuration
239
+ this.config = getConfig();
240
+ // Detect vector dimension from embedding model configuration
241
+ const modelName = process.env.EMBEDDING_MODEL_NAME || "Xenova/all-MiniLM-L6-v2";
242
+ const envDimension = parseInt(process.env.EMBEDDING_DIMENSION || "0") || null;
243
+ this.vectorDimension = envDimension || getEmbeddingDimension(modelName);
244
+ // Only log in debug mode to avoid corrupting spinner/REPL display
245
+ if (process.env.YAMO_DEBUG === "true") {
246
+ logger.debug({ dimension: this.vectorDimension, model: modelName }, "Using vector dimension");
247
+ }
248
+ // Use custom dbDir if provided (for test isolation), otherwise use config
249
+ const dbUri = this.dbDir || this.config.LANCEDB_URI;
250
+ // Create LanceDBClient with detected dimension
251
+ this.client = new LanceDBClient({
252
+ uri: dbUri,
253
+ tableName: this.config.LANCEDB_MEMORY_TABLE,
254
+ vectorDimension: this.vectorDimension,
255
+ maxRetries: 3,
256
+ retryDelay: 1000,
257
+ });
258
+ // Connect to database
259
+ await this.client.connect();
260
+ // Configure embedding factory from environment
261
+ const embeddingConfigs = this._parseEmbeddingConfig();
262
+ this.embeddingFactory.configure(embeddingConfigs);
263
+ await this.embeddingFactory.init();
264
+ // Hydrate Keyword Search (In-Memory)
265
+ if (this.client) {
266
+ try {
267
+ const allRecords = await this.client.getAll({ limit: 10000 });
268
+ this.keywordSearch.load(allRecords);
269
+ }
270
+ catch (_e) {
271
+ // Ignore if table doesn't exist yet
272
+ }
273
+ }
274
+ // Initialize extension tables if enabled
275
+ if (this.enableYamo && this.client && this.client.db) {
276
+ try {
277
+ const { createYamoTable } = await import("../yamo/schema.js");
278
+ this.yamoTable = await createYamoTable(this.client.db, "yamo_blocks");
279
+ // Initialize synthesized skills table (Recursive Skill Synthesis)
280
+ // const { createSynthesizedSkillSchema } = await import('./schema'); // Imported statically now
281
+ const existingTables = await this.client.db.tableNames();
282
+ if (existingTables.includes("synthesized_skills")) {
283
+ this.skillTable =
284
+ await this.client.db.openTable("synthesized_skills");
285
+ }
286
+ else {
287
+ const skillSchema = createSynthesizedSkillSchema(this.vectorDimension);
288
+ this.skillTable = await this.client.db.createTable("synthesized_skills", [], {
289
+ schema: skillSchema,
290
+ });
291
+ }
292
+ if (process.env.YAMO_DEBUG === "true") {
293
+ logger.debug("YAMO blocks and synthesized skills tables initialized");
294
+ }
295
+ }
296
+ catch (e) {
297
+ logger.warn({ err: e }, "Failed to initialize extension tables");
298
+ }
299
+ }
300
+ this.isInitialized = true;
247
301
  }
248
- }
249
-
250
- // Initialize extension tables if enabled
251
- if (this.enableYamo && this.client && this.client.db) {
302
+ catch (error) {
303
+ const e = error instanceof Error ? error : new Error(String(error));
304
+ throw e;
305
+ }
306
+ }
307
+ /**
308
+ * Add content to memory with auto-generated embedding and scrubbing.
309
+ *
310
+ * This is the primary method for storing information in the memory mesh.
311
+ * The content goes through several processing steps:
312
+ *
313
+ * 1. **Scrubbing**: PII and sensitive data are sanitized (if enabled)
314
+ * 2. **Validation**: Content length and metadata are validated
315
+ * 3. **Embedding**: Content is converted to a vector representation
316
+ * 4. **Storage**: Record is stored in LanceDB with metadata
317
+ * 5. **Emission**: Optional YAMO block emitted for provenance tracking
318
+ *
319
+ * @param content - The text content to store in memory
320
+ * @param metadata - Optional metadata (type, source, tags, etc.)
321
+ * @returns Promise with memory record containing id, content, metadata, created_at
322
+ *
323
+ * @example
324
+ * ```typescript
325
+ * const memory = await mesh.add("User likes TypeScript", {
326
+ * type: "preference",
327
+ * source: "chat",
328
+ * tags: ["programming", "languages"]
329
+ * });
330
+ * ```
331
+ *
332
+ * @throws {Error} If content exceeds max length (100KB)
333
+ * @throws {Error} If embedding generation fails
334
+ * @throws {Error} If database client is not initialized
335
+ */
336
+ async add(content, metadata = {}) {
337
+ await this.init();
338
+ const type = metadata.type || "event";
339
+ const enrichedMetadata = { ...metadata, type };
252
340
  try {
253
- const { createYamoTable } = await import('../yamo/schema.js');
254
- this.yamoTable = await createYamoTable(this.client.db, 'yamo_blocks');
255
-
256
- // Initialize synthesized skills table (Recursive Skill Synthesis)
257
- const { createSynthesizedSkillSchema } = await import('../lancedb/schema.js');
258
- const existingTables = await this.client.db.tableNames();
259
-
260
- if (existingTables.includes('synthesized_skills')) {
261
- this.skillTable = await this.client.db.openTable('synthesized_skills');
262
- } else {
263
- const skillSchema = createSynthesizedSkillSchema(this.vectorDimension);
264
- this.skillTable = await this.client.db.createTable('synthesized_skills', [], { schema: skillSchema });
265
- }
266
-
267
- if (process.env.YAMO_DEBUG === 'true') {
268
- console.error('[MemoryMesh] YAMO blocks and synthesized skills tables initialized');
269
- }
270
- } catch (e) {
271
- console.warn('[MemoryMesh] Failed to initialize extension tables:', e instanceof Error ? e.message : String(e));
341
+ let processedContent = content;
342
+ let scrubbedMetadata = {};
343
+ try {
344
+ const scrubbedResult = await this.scrubber.process({
345
+ content: content,
346
+ source: "memory-api",
347
+ type: "txt",
348
+ });
349
+ if (scrubbedResult.success && scrubbedResult.chunks.length > 0) {
350
+ processedContent = scrubbedResult.chunks
351
+ .map((c) => c.text)
352
+ .join("\n\n");
353
+ if (scrubbedResult.metadata) {
354
+ scrubbedMetadata = {
355
+ ...scrubbedResult.metadata,
356
+ scrubber_telemetry: JSON.stringify(scrubbedResult.telemetry),
357
+ };
358
+ }
359
+ }
360
+ }
361
+ catch (scrubError) {
362
+ if (process.env.YAMO_DEBUG === "true") {
363
+ logger.error({ err: scrubError }, "Scrubber failed");
364
+ }
365
+ }
366
+ const sanitizedContent = this._sanitizeContent(processedContent);
367
+ const sanitizedMetadata = this._validateMetadata({
368
+ ...scrubbedMetadata,
369
+ ...enrichedMetadata,
370
+ });
371
+ if (process.env.YAMO_DEBUG === "true") {
372
+ console.error("[DEBUG] brain.add() scrubbedMetadata.type:", scrubbedMetadata.type);
373
+ console.error("[DEBUG] brain.add() enrichedMetadata.type:", enrichedMetadata.type);
374
+ console.error("[DEBUG] brain.add() sanitizedMetadata.type:", sanitizedMetadata.type);
375
+ }
376
+ const vector = await this.embeddingFactory.embed(sanitizedContent);
377
+ // Dedup: search by the already-computed vector before inserting.
378
+ // Catches exact duplicates regardless of which write path is used,
379
+ // protecting callers that bypass captureInteraction()'s dedup guard.
380
+ if (this.client) {
381
+ const nearest = await this.client.search(vector, { limit: 1 });
382
+ if (nearest.length > 0 && nearest[0].content === sanitizedContent) {
383
+ return {
384
+ id: nearest[0].id,
385
+ content: sanitizedContent,
386
+ metadata: sanitizedMetadata,
387
+ created_at: new Date().toISOString(),
388
+ };
389
+ }
390
+ }
391
+ const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
392
+ const record = {
393
+ id,
394
+ vector,
395
+ content: sanitizedContent,
396
+ metadata: JSON.stringify(sanitizedMetadata),
397
+ };
398
+ if (process.env.YAMO_DEBUG === "true") {
399
+ console.error("[DEBUG] record.metadata.type:", JSON.parse(record.metadata).type);
400
+ }
401
+ if (!this.client) {
402
+ throw new Error("Database client not initialized");
403
+ }
404
+ const result = await this.client.add(record);
405
+ if (process.env.YAMO_DEBUG === "true") {
406
+ try {
407
+ console.error("[DEBUG] result.metadata.type:", JSON.parse(result.metadata).type);
408
+ }
409
+ catch {
410
+ console.error("[DEBUG] result.metadata:", result.metadata);
411
+ }
412
+ }
413
+ this.keywordSearch.add(record.id, record.content, sanitizedMetadata);
414
+ if (this.enableYamo) {
415
+ this._emitYamoBlock("retain", result.id, YamoEmitter.buildRetainBlock({
416
+ content: sanitizedContent,
417
+ metadata: sanitizedMetadata,
418
+ id: result.id,
419
+ agentId: this.agentId,
420
+ memoryType: sanitizedMetadata.type || "event",
421
+ })).catch((error) => {
422
+ // Log emission failures in debug mode but don't throw
423
+ if (process.env.YAMO_DEBUG === "true") {
424
+ logger.warn({ err: error }, "Failed to emit YAMO block (retain)");
425
+ }
426
+ });
427
+ }
428
+ return {
429
+ id: result.id,
430
+ content: sanitizedContent,
431
+ metadata: sanitizedMetadata,
432
+ created_at: new Date().toISOString(),
433
+ };
434
+ }
435
+ catch (error) {
436
+ throw error instanceof Error ? error : new Error(String(error));
272
437
  }
273
- }
274
-
275
- this.isInitialized = true;
276
-
277
- } catch (error) {
278
- const e = error instanceof Error ? error : new Error(String(error));
279
- throw e;
280
438
  }
281
- }
282
-
283
- /**
284
- * Add content to memory with auto-generated embedding
285
- */
286
- async add(content, metadata = {}) {
287
- await this.init();
288
-
289
- const type = metadata.type || 'event';
290
- const enrichedMetadata = { ...metadata, type };
291
-
292
- try {
293
- let processedContent = content;
294
- let scrubbedMetadata = {};
295
-
296
- try {
297
- const scrubbedResult = await this.scrubber.process({
298
- content: content,
299
- source: 'memory-api',
300
- type: 'txt'
439
+ /**
440
+ * Reflect on recent memories
441
+ */
442
+ async reflect(options = {}) {
443
+ await this.init();
444
+ const lookback = options.lookback || 10;
445
+ const topic = options.topic;
446
+ const generate = options.generate !== false;
447
+ let memories = [];
448
+ if (topic) {
449
+ memories = await this.search(topic, { limit: lookback });
450
+ }
451
+ else {
452
+ const all = await this.getAll();
453
+ memories = all
454
+ .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
455
+ .slice(0, lookback);
456
+ }
457
+ const prompt = `Review these memories. Synthesize a high-level "belief" or "observation".`;
458
+ if (!generate || !this.enableLLM || !this.llmClient) {
459
+ return {
460
+ topic,
461
+ count: memories.length,
462
+ context: memories.map((m) => ({
463
+ content: m.content,
464
+ type: m.metadata?.type || "event",
465
+ id: m.id,
466
+ })),
467
+ prompt,
468
+ };
469
+ }
470
+ let reflection = "";
471
+ let confidence = 0;
472
+ try {
473
+ const result = await this.llmClient.reflect(prompt, memories);
474
+ reflection = result.reflection;
475
+ confidence = result.confidence;
476
+ }
477
+ catch (_error) {
478
+ reflection = `Aggregated from ${memories.length} memories on topic: ${topic || "general"}`;
479
+ confidence = 0.5;
480
+ }
481
+ const reflectionId = `reflect_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
482
+ await this.add(reflection, {
483
+ type: "reflection",
484
+ topic: topic || "general",
485
+ source_memory_count: memories.length,
486
+ confidence,
487
+ generated_at: new Date().toISOString(),
301
488
  });
302
-
303
- if (scrubbedResult.success && scrubbedResult.chunks.length > 0) {
304
- processedContent = scrubbedResult.chunks.map(c => c.text).join('\n\n');
305
- if (scrubbedResult.metadata) {
306
- scrubbedMetadata = {
307
- ...scrubbedResult.metadata,
308
- scrubber_telemetry: JSON.stringify(scrubbedResult.telemetry)
309
- };
310
- }
311
- }
312
- } catch (scrubError) {
313
- if (process.env.YAMO_DEBUG === 'true') {
314
- console.error(`[MemoryMesh] Scrubber failed: ${scrubError.message}`);
315
- }
316
- }
317
-
318
- const sanitizedContent = this._sanitizeContent(processedContent);
319
- const sanitizedMetadata = this._validateMetadata({ ...enrichedMetadata, ...scrubbedMetadata });
320
-
321
- const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
322
- const vector = await this.embeddingFactory.embed(sanitizedContent);
323
-
324
- const record = {
325
- id,
326
- vector,
327
- content: sanitizedContent,
328
- metadata: JSON.stringify(sanitizedMetadata)
329
- };
330
-
331
- if (!this.client) throw new Error('Database client not initialized');
332
- const result = await this.client.add(record);
333
- this.keywordSearch.add(record.id, record.content, sanitizedMetadata);
334
-
335
- if (this.enableYamo) {
336
- this._emitYamoBlock('retain', result.id, YamoEmitter.buildRetainBlock({
337
- content: sanitizedContent,
338
- metadata: sanitizedMetadata,
339
- id: result.id,
340
- agentId: this.agentId,
341
- memoryType: sanitizedMetadata.type || 'event'
342
- })).catch(() => {});
343
- }
344
-
345
- return {
346
- id: result.id,
347
- content: sanitizedContent,
348
- metadata: sanitizedMetadata,
349
- created_at: new Date().toISOString()
350
- };
351
- } catch (error) {
352
- throw error instanceof Error ? error : new Error(String(error));
353
- }
354
- }
355
-
356
- /**
357
- * Reflect on recent memories
358
- */
359
- async reflect(options = {}) {
360
- await this.init();
361
- const lookback = options.lookback || 10;
362
- const topic = options.topic;
363
- const generate = options.generate !== false;
364
-
365
- let memories = [];
366
- if (topic) {
367
- memories = await this.search(topic, { limit: lookback });
368
- } else {
369
- const all = await this.getAll();
370
- memories = all
371
- .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
372
- .slice(0, lookback);
489
+ let yamoBlock = null;
490
+ if (this.enableYamo) {
491
+ yamoBlock = YamoEmitter.buildReflectBlock({
492
+ topic: topic || "general",
493
+ memoryCount: memories.length,
494
+ agentId: this.agentId,
495
+ reflection,
496
+ confidence,
497
+ });
498
+ await this._emitYamoBlock("reflect", reflectionId, yamoBlock);
499
+ }
500
+ return {
501
+ id: reflectionId,
502
+ topic: topic || "general",
503
+ reflection,
504
+ confidence,
505
+ sourceMemoryCount: memories.length,
506
+ yamoBlock,
507
+ createdAt: new Date().toISOString(),
508
+ };
373
509
  }
374
-
375
- const prompt = `Review these memories. Synthesize a high-level "belief" or "observation".`;
376
-
377
- if (!generate || !this.enableLLM || !this.llmClient) {
378
- return { topic, count: memories.length, context: memories.map(m => ({ content: m.content, type: m.metadata?.type || 'event', id: m.id })), prompt };
510
+ /**
511
+ * Ingest synthesized skill
512
+ * @param sourceFilePath - If provided, skip file write (file already exists)
513
+ */
514
+ async ingestSkill(yamoText, metadata = {}, sourceFilePath) {
515
+ await this.init();
516
+ if (!this.skillTable) {
517
+ throw new Error("Skill table not initialized");
518
+ }
519
+ // DEBUG: Trace sourceFilePath parameter
520
+ if (process.env.YAMO_DEBUG_PATHS === "true") {
521
+ console.error(`[BRAIN.ingestSkill] sourceFilePath parameter: ${sourceFilePath || "undefined"}`);
522
+ }
523
+ try {
524
+ const identity = extractSkillIdentity(yamoText);
525
+ const name = metadata.name || identity.name;
526
+ const intent = identity.intent;
527
+ const description = identity.description;
528
+ // RECURSION DETECTION: Check for recursive naming patterns
529
+ // Patterns like "SkillSkill", "SkillSkillSkill" indicate filename-derived names
530
+ const recursivePattern = /^(Skill|skill){2,}/;
531
+ if (recursivePattern.test(name)) {
532
+ logger.warn({ originalName: name }, "Detected recursive naming pattern, rejecting ingestion to prevent loop");
533
+ throw new Error(`Recursive naming pattern detected: ${name}. Skills must have proper name: field.`);
534
+ }
535
+ // Extract tags for tag-aware embeddings (improves semantic search)
536
+ const tags = extractSkillTags(yamoText);
537
+ const tagText = tags.length > 0 ? `\nTags: ${tags.join(", ")}` : "";
538
+ const embeddingText = `Skill: ${name}\nIntent: ${intent}${tagText}\nDescription: ${description}`;
539
+ const vector = await this.embeddingFactory.embed(embeddingText);
540
+ const id = `skill_${Date.now()}_${crypto.randomBytes(2).toString("hex")}`;
541
+ const skillMetadata = {
542
+ reliability: 0.5,
543
+ use_count: 0,
544
+ source: "manual",
545
+ ...metadata,
546
+ // Store source file path for policy loading and parent discovery
547
+ ...(sourceFilePath && { source_file: sourceFilePath }),
548
+ };
549
+ const record = {
550
+ id,
551
+ name,
552
+ intent,
553
+ yamo_text: yamoText,
554
+ vector,
555
+ metadata: JSON.stringify(skillMetadata),
556
+ created_at: new Date(),
557
+ };
558
+ await this.skillTable.add([record]);
559
+ // NEW: Persist to filesystem for longevity and visibility
560
+ // Skip if sourceFilePath provided (file already exists from SkillCreator)
561
+ // Skip if using in-memory database (:memory:)
562
+ if (!sourceFilePath && this.dbDir !== ":memory:") {
563
+ try {
564
+ const skillsDir = path.resolve(process.cwd(), this.skillDirectories[0] || "skills");
565
+ if (!fs.existsSync(skillsDir)) {
566
+ fs.mkdirSync(skillsDir, { recursive: true });
567
+ }
568
+ // Robust filename with length limit to prevent ENAMETOOLONG
569
+ const safeName = name
570
+ .toLowerCase()
571
+ .replace(/[^a-z0-9]/g, "-")
572
+ .replace(/-+/g, "-")
573
+ .substring(0, 50);
574
+ const fileName = `skill-${safeName}.yamo`;
575
+ const filePath = path.join(skillsDir, fileName);
576
+ // Only write if file doesn't already exist to prevent duplicates
577
+ if (!fs.existsSync(filePath)) {
578
+ fs.writeFileSync(filePath, yamoText, "utf8");
579
+ if (process.env.YAMO_DEBUG === "true") {
580
+ logger.debug({ filePath }, "Skill persisted to file");
581
+ }
582
+ }
583
+ }
584
+ catch (fileError) {
585
+ logger.warn({ err: fileError }, "Failed to persist skill to file");
586
+ }
587
+ }
588
+ return { id, name, intent };
589
+ }
590
+ catch (error) {
591
+ throw new Error(`Skill ingestion failed: ${error.message}`);
592
+ }
379
593
  }
380
-
381
- let reflection = null;
382
- let confidence = 0;
383
-
384
- try {
385
- const result = await this.llmClient.reflect(prompt, memories);
386
- reflection = result.reflection;
387
- confidence = result.confidence;
388
- } catch (error) {
389
- reflection = `Aggregated from ${memories.length} memories on topic: ${topic || 'general'}`;
390
- confidence = 0.5;
594
+ /**
595
+ * Recursive Skill Synthesis
596
+ */
597
+ async synthesize(options = {}) {
598
+ await this.init();
599
+ const topic = options.topic || "general_improvement";
600
+ const enrichedPrompt = options.enrichedPrompt || topic; // PHASE 4: Use enriched prompt
601
+ // const lookback = options.lookback || 20;
602
+ logger.info({ topic, enrichedPrompt }, "Synthesizing logic");
603
+ // OPTIMIZATION: If we have an execution engine (kernel), use SkillCreator!
604
+ if (this._kernel_execute) {
605
+ logger.info("Dispatching to SkillCreator agent...");
606
+ try {
607
+ // Use stored skill directories
608
+ const skillDirs = this.skillDirectories;
609
+ // Track existing .yamo files before SkillCreator runs
610
+ const filesBefore = new Set();
611
+ for (const dir of skillDirs) {
612
+ if (fs.existsSync(dir)) {
613
+ const walk = (currentDir) => {
614
+ try {
615
+ const entries = fs.readdirSync(currentDir, {
616
+ withFileTypes: true,
617
+ });
618
+ for (const entry of entries) {
619
+ const fullPath = path.join(currentDir, entry.name);
620
+ if (entry.isDirectory()) {
621
+ walk(fullPath);
622
+ }
623
+ else if (entry.isFile() && entry.name.endsWith(".yamo")) {
624
+ filesBefore.add(fullPath);
625
+ }
626
+ }
627
+ }
628
+ catch (e) {
629
+ // Skip directories we can't read
630
+ logger.debug({ dir, error: e }, "Could not read directory");
631
+ }
632
+ };
633
+ walk(dir);
634
+ }
635
+ }
636
+ // PHASE 4: Use enriched prompt for SkillCreator
637
+ await this._kernel_execute(`SkillCreator: design a new skill to handle ${enrichedPrompt}`, {
638
+ v1_1_enabled: true,
639
+ });
640
+ // Find newly created .yamo file
641
+ let newSkillFile;
642
+ for (const dir of skillDirs) {
643
+ if (fs.existsSync(dir)) {
644
+ const walk = (currentDir) => {
645
+ try {
646
+ const entries = fs.readdirSync(currentDir, {
647
+ withFileTypes: true,
648
+ });
649
+ for (const entry of entries) {
650
+ const fullPath = path.join(currentDir, entry.name);
651
+ if (entry.isDirectory()) {
652
+ walk(fullPath);
653
+ }
654
+ else if (entry.isFile() && entry.name.endsWith(".yamo")) {
655
+ if (!filesBefore.has(fullPath)) {
656
+ newSkillFile = fullPath;
657
+ }
658
+ }
659
+ }
660
+ }
661
+ catch (e) {
662
+ logger.debug({ dir, error: e }, "Could not read directory");
663
+ }
664
+ };
665
+ walk(dir);
666
+ }
667
+ }
668
+ // Ingest the newly created skill file
669
+ if (newSkillFile) {
670
+ logger.info({ skillFile: newSkillFile }, "Ingesting newly synthesized skill");
671
+ let skillContent = fs.readFileSync(newSkillFile, "utf8");
672
+ // PHASE 4: Expand compressed → canonical for disk storage
673
+ // Skills created by evolution are typically compressed; expand to canonical for readability
674
+ // Skip expansion in test environment or when disabled
675
+ const expansionEnabled = process.env.YAMO_EXPANSION_ENABLED !== "false";
676
+ const isCompressed = !skillContent.includes("---") ||
677
+ (skillContent.includes("---") &&
678
+ skillContent.split("---").length <= 1);
679
+ if (expansionEnabled && isCompressed) {
680
+ logger.info({ skillFile: newSkillFile }, "Expanding compressed skill to canonical format");
681
+ try {
682
+ const expanded = await this._kernel_execute("skill-expansion-system-prompt.yamo", {
683
+ input_yamo: skillContent,
684
+ });
685
+ if (expanded && expanded.canonical_yamo) {
686
+ skillContent = expanded.canonical_yamo;
687
+ // Write expanded canonical format back to disk
688
+ fs.writeFileSync(newSkillFile, skillContent, "utf8");
689
+ logger.info({ skillFile: newSkillFile }, "Skill expanded to canonical format on disk");
690
+ }
691
+ }
692
+ catch (e) {
693
+ logger.warn({ err: e }, "Failed to expand skill to canonical, using compressed format");
694
+ }
695
+ }
696
+ // ENSURE: Synthesized skills always have proper metadata with meaningful name
697
+ // This prevents duplicate skill-agent-{timestamp}.yamo files
698
+ const synIdentity = extractSkillIdentity(skillContent);
699
+ const hasName = !synIdentity.name.startsWith("Unnamed_");
700
+ if (!skillContent.includes("---") || !hasName) {
701
+ logger.info({ skillFile: newSkillFile }, "Adding metadata block to synthesized skill");
702
+ const intent = synIdentity.intent !== "general_procedure"
703
+ ? synIdentity.intent.replace(/[^a-zA-Z0-9]/g, "")
704
+ : "Synthesized";
705
+ const PascalCase = intent.charAt(0).toUpperCase() + intent.slice(1);
706
+ const skillName = `${PascalCase}_${Date.now().toString(36)}`;
707
+ const metadata = `---
708
+ name: ${skillName}
709
+ version: 1.0.0
710
+ author: YAMO Evolution
711
+ license: MIT
712
+ tags: synthesized, evolution, auto-generated
713
+ description: Auto-generated skill to handle: ${enrichedPrompt || topic}
714
+ ---
715
+ `;
716
+ // Prepend metadata if skill doesn't have it
717
+ if (!skillContent.startsWith("---")) {
718
+ skillContent = metadata + skillContent;
719
+ // Write back to disk with proper metadata
720
+ fs.writeFileSync(newSkillFile, skillContent, "utf8");
721
+ logger.info({ skillFile: newSkillFile, skillName }, "Added metadata block to synthesized skill");
722
+ }
723
+ }
724
+ const skill = await this.ingestSkill(skillContent, {
725
+ source: "synthesized",
726
+ trigger_topic: topic,
727
+ }, newSkillFile);
728
+ return {
729
+ status: "success",
730
+ analysis: "SkillCreator orchestrated evolution",
731
+ skill_id: skill.id,
732
+ skill_name: skill.name,
733
+ yamo_text: skillContent,
734
+ };
735
+ }
736
+ // Fallback if no new file found
737
+ return {
738
+ status: "success",
739
+ analysis: "SkillCreator orchestrated evolution (no file detected)",
740
+ skill_name: topic.split(" ")[0],
741
+ };
742
+ }
743
+ catch (e) {
744
+ logger.error({ err: e }, "SkillCreator agent failed");
745
+ return {
746
+ status: "error",
747
+ error: e.message,
748
+ analysis: "SkillCreator agent failed",
749
+ };
750
+ }
751
+ }
752
+ // SkillCreator is required for synthesis
753
+ if (!this._kernel_execute) {
754
+ throw new Error("Kernel execution (_kernel_execute) is required for synthesis. Use YamoKernel instead of MemoryMesh directly.");
755
+ }
756
+ // Should never reach here
757
+ return {
758
+ status: "error",
759
+ analysis: "Unexpected state in synthesis",
760
+ };
391
761
  }
392
-
393
- const reflectionId = `reflect_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
394
- await this.add(reflection, { type: 'reflection', topic: topic || 'general', source_memory_count: memories.length, confidence, generated_at: new Date().toISOString() });
395
-
396
- let yamoBlock = null;
397
- if (this.enableYamo) {
398
- yamoBlock = YamoEmitter.buildReflectBlock({ topic: topic || 'general', memoryCount: memories.length, agentId: this.agentId, reflection, confidence });
399
- await this._emitYamoBlock('reflect', reflectionId, yamoBlock);
762
+ /**
763
+ * Update reliability
764
+ */
765
+ async updateSkillReliability(id, success) {
766
+ await this.init();
767
+ if (!this.skillTable) {
768
+ throw new Error("Skill table not initialized");
769
+ }
770
+ try {
771
+ const results = await this.skillTable
772
+ .query()
773
+ .filter(`id == '${id}'`)
774
+ .toArray();
775
+ if (results.length === 0) {
776
+ throw new Error(`Skill ${id} not found`);
777
+ }
778
+ const record = results[0];
779
+ const metadata = JSON.parse(record.metadata);
780
+ const adjustment = success ? 0.1 : -0.2;
781
+ metadata.reliability = Math.max(0, Math.min(1.0, (metadata.reliability || 0.5) + adjustment));
782
+ metadata.use_count = (metadata.use_count || 0) + 1;
783
+ metadata.last_used = new Date().toISOString();
784
+ await this.skillTable.update({
785
+ where: `id == '${id}'`,
786
+ values: { metadata: JSON.stringify(metadata) },
787
+ });
788
+ return {
789
+ id,
790
+ reliability: metadata.reliability,
791
+ use_count: metadata.use_count,
792
+ };
793
+ }
794
+ catch (error) {
795
+ throw new Error(`Failed to update skill reliability: ${error.message}`);
796
+ }
400
797
  }
401
-
402
- return { id: reflectionId, topic: topic || 'general', reflection, confidence, sourceMemoryCount: memories.length, yamoBlock, createdAt: new Date().toISOString() };
403
- }
404
-
405
- /**
406
- * Ingest synthesized skill
407
- */
408
- async ingestSkill(yamoText, metadata = {}) {
409
- await this.init();
410
- if (!this.skillTable) throw new Error('Skill table not initialized');
411
-
412
- try {
413
- const nameMatch = yamoText.match(/name;([^;]+);/);
414
- const intentMatch = yamoText.match(/intent;([^;]+);/);
415
- const name = nameMatch ? nameMatch[1].trim() : `SynthesizedAgent_${Date.now()}`;
416
- const intent = intentMatch ? intentMatch[1].trim() : "general_procedure";
417
- const vector = await this.embeddingFactory.embed(intent);
418
- const id = `skill_${Date.now()}_${crypto.randomBytes(2).toString('hex')}`;
419
- const skillMetadata = { reliability: 0.5, use_count: 0, source: 'synthesis', ...metadata };
420
- const record = { id, name, intent, yamo_text: yamoText, vector, metadata: JSON.stringify(skillMetadata), created_at: new Date() };
421
- await this.skillTable.add([record]);
422
- return { id, name, intent };
423
- } catch (error) {
424
- throw new Error(`Skill ingestion failed: ${error.message}`);
425
- }
426
- }
427
-
428
- /**
429
- * Recursive Skill Synthesis
430
- */
431
- async synthesize(options = {}) {
432
- await this.init();
433
- if (!this.llmClient) throw new Error('LLM required for synthesis');
434
- const lookback = options.lookback || 20;
435
- const topic = options.topic;
436
- const memories = topic ? await this.search(topic, { limit: lookback }) : await this.getAll({ limit: lookback });
437
-
438
- const prompt = `Analyze these memories for RECURRING PROCEDURAL PATTERNS.
439
- If a pattern exists, synthesize an EXECUTABLE YAMO SKILL to handle it.
440
- Output MUST be a JSON object: {"analysis": "...", "pattern_detected": true/false, "proposed_skill": "name;...;agent: ... intent: ..."}`;
441
-
442
- try {
443
- const result = await this.llmClient.reflect(prompt, memories);
444
- let synthesis;
445
-
446
- try {
447
- synthesis = JSON.parse(result.reflection);
448
- } catch (e) {
449
- // YAMO v0.5: Self-Healing Syntax Bridge
450
- if (result.reflection.toLowerCase().includes('environment') || result.reflection.toLowerCase().includes('human')) {
451
- synthesis = {
452
- pattern_detected: true,
453
- analysis: "Detected critical environmental impact patterns in memory mesh.",
454
- proposed_skill: "name;EnvironmentalImpactAuditor;\nagent: SustainabilityAgent;\nintent: audit_human_environmental_impact;\nconstraints: - must_prioritize_carbon_metrics; - analyze_biodiversity_loss; - identify_resource_depletion;\nhandoff: End;"
455
- };
456
- } else {
457
- throw e;
458
- }
459
- }
460
-
461
- if (synthesis.pattern_detected && synthesis.proposed_skill) {
462
- const skill = await this.ingestSkill(synthesis.proposed_skill, { analysis: synthesis.analysis, trigger_topic: topic });
463
- return { status: 'success', analysis: synthesis.analysis, skill_id: skill.id, skill_name: skill.name, yamo_text: synthesis.proposed_skill };
464
- }
465
- return { status: 'no_pattern', analysis: synthesis.analysis || "No procedural patterns identified." };
466
- } catch (error) {
467
- throw new Error(`Synthesis failed: ${error.message}`);
468
- }
469
- }
470
-
471
- /**
472
- * Update reliability
473
- */
474
- async updateSkillReliability(id, success) {
475
- await this.init();
476
- if (!this.skillTable) throw new Error('Skill table not initialized');
477
- try {
478
- const results = await this.skillTable.query().filter(`id == '${id}'`).toArray();
479
- if (results.length === 0) throw new Error(`Skill ${id} not found`);
480
- const record = results[0];
481
- const metadata = JSON.parse(record.metadata);
482
- const adjustment = success ? 0.1 : -0.2;
483
- metadata.reliability = Math.max(0, Math.min(1.0, (metadata.reliability || 0.5) + adjustment));
484
- metadata.use_count = (metadata.use_count || 0) + 1;
485
- metadata.last_used = new Date().toISOString();
486
- await this.skillTable.update(`id == '${id}'`, { metadata: JSON.stringify(metadata) });
487
- return { id, reliability: metadata.reliability, use_count: metadata.use_count };
488
- } catch (error) {
489
- throw new Error(`Failed to update skill reliability: ${error.message}`);
490
- }
491
- }
492
-
493
- /**
494
- * Prune skills
495
- */
496
- async pruneSkills(threshold = 0.3) {
497
- await this.init();
498
- if (!this.skillTable) throw new Error('Skill table not initialized');
499
- try {
500
- const allSkills = await this.skillTable.query().toArray();
501
- let prunedCount = 0;
502
- for (const skill of allSkills) {
503
- const metadata = JSON.parse(skill.metadata);
504
- if (metadata.reliability < threshold) {
505
- await this.skillTable.delete(`id == '${skill.id}'`);
506
- prunedCount++;
507
- }
508
- }
509
- return { pruned_count: prunedCount, total_remaining: allSkills.length - prunedCount };
510
- } catch (error) {
511
- throw new Error(`Pruning failed: ${error.message}`);
512
- }
513
- }
514
-
515
- /**
516
- * Search for synthesized skills by semantic intent
517
- * @param {string} query - Search query (intent description)
518
- * @param {Object} [options={}] - Search options
519
- * @returns {Promise<Array>} Normalized skill results
520
- */
521
- async searchSkills(query, options = {}) {
522
- await this.init();
523
- if (!this.skillTable) return [];
524
-
525
- try {
526
- const vector = await this.embeddingFactory.embed(query);
527
- const results = await this.skillTable.search(vector).limit(options.limit || 5).toArray();
528
-
529
- // Normalize scores using the same Bayesian-lite logic if applicable,
530
- // but here we just use the vector distance normalization.
531
- return this._normalizeScores(results.map(r => ({
532
- ...r,
533
- score: r._distance !== undefined ? 1 - r._distance : 0.5
534
- })));
535
- } catch (error) {
536
- if (process.env.YAMO_DEBUG === 'true') {
537
- console.error(`[MemoryMesh] Skill search failed: ${error.message}`);
538
- }
539
- return [];
540
- }
541
- }
542
-
543
- /**
544
- * Emit a YAMO block to the YAMO blocks table
545
-
546
- async _emitYamoBlock(operationType, memoryId, yamoText) {
547
- if (!this.yamoTable) return;
548
- const yamoId = `yamo_${operationType}_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
549
- try {
550
- await this.yamoTable.add([{
551
- id: yamoId, agent_id: this.agentId, operation_type: operationType, yamo_text: yamoText,
552
- timestamp: new Date(), block_hash: null, prev_hash: null,
553
- metadata: JSON.stringify({ memory_id: memoryId || null, timestamp: new Date().toISOString() })
554
- }]);
555
- } catch (error) {}
556
- }
557
-
558
- /**
559
- * Search memory
560
- */
561
- async search(query, options = {}) {
562
- await this.init();
563
- try {
564
- const limit = options.limit || 10;
565
- const filter = options.filter || null;
566
- const useCache = options.useCache !== undefined ? options.useCache : true;
567
-
568
- if (useCache) {
569
- const cacheKey = this._generateCacheKey(query, { limit, filter });
570
- const cached = this._getCachedResult(cacheKey);
571
- if (cached) return cached;
572
- }
573
-
574
- const vector = await this.embeddingFactory.embed(query);
575
- if (!this.client) throw new Error('Database client not initialized');
576
- const vectorResults = await this.client.search(vector, { limit: limit * 2, metric: 'cosine', filter });
577
- const keywordResults = this.keywordSearch.search(query, { limit: limit * 2 });
578
-
579
- const k = 60;
580
- const scores = new Map();
581
- const docMap = new Map();
582
-
583
- vectorResults.forEach((doc, rank) => {
584
- const rrf = 1 / (k + rank + 1);
585
- scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
586
- docMap.set(doc.id, doc);
587
- });
588
-
589
- keywordResults.forEach((doc, rank) => {
590
- const rrf = 1 / (k + rank + 1);
591
- scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
592
- if (!docMap.has(doc.id)) docMap.set(doc.id, { id: doc.id, content: doc.content, metadata: doc.metadata, score: 0, created_at: new Date().toISOString() });
593
- });
594
-
595
- const mergedResults = Array.from(scores.entries())
596
- .sort((a, b) => b[1] - a[1])
597
- .slice(0, limit)
598
- .map(([id, score]) => {
599
- const doc = docMap.get(id);
600
- return doc ? { ...doc, score } : null;
601
- })
602
- .filter(d => d !== null);
603
-
604
- const normalizedResults = this._normalizeScores(mergedResults);
605
- if (useCache) {
606
- const cacheKey = this._generateCacheKey(query, { limit, filter });
607
- this._cacheResult(cacheKey, normalizedResults);
608
- }
609
-
610
- if (this.enableYamo) {
611
- this._emitYamoBlock('recall', undefined, YamoEmitter.buildRecallBlock({ query, resultCount: normalizedResults.length, limit, agentId: this.agentId, searchType: 'hybrid' })).catch(() => {});
612
- }
613
-
614
- return normalizedResults;
615
- } catch (error) {
616
- throw error instanceof Error ? error : new Error(String(error));
798
+ /**
799
+ * Prune skills
800
+ */
801
+ async pruneSkills(threshold = 0.3) {
802
+ await this.init();
803
+ if (!this.skillTable) {
804
+ throw new Error("Skill table not initialized");
805
+ }
806
+ try {
807
+ const allSkills = await this.skillTable.query().toArray();
808
+ let prunedCount = 0;
809
+ for (const skill of allSkills) {
810
+ const metadata = JSON.parse(skill.metadata);
811
+ if (metadata.reliability < threshold) {
812
+ await this.skillTable.delete(`id == '${skill.id}'`);
813
+ prunedCount++;
814
+ }
815
+ }
816
+ return {
817
+ pruned_count: prunedCount,
818
+ total_remaining: allSkills.length - prunedCount,
819
+ };
820
+ }
821
+ catch (error) {
822
+ throw new Error(`Pruning failed: ${error.message}`);
823
+ }
824
+ }
825
+ /**
826
+ * List all synthesized skills
827
+ * @param {Object} [options={}] - Search options
828
+ * @returns {Promise<Array>} Normalized skill results
829
+ */
830
+ async listSkills(options = {}) {
831
+ await this.init();
832
+ if (!this.skillTable) {
833
+ return [];
834
+ }
835
+ try {
836
+ const limit = options.limit || 10;
837
+ const results = await this.skillTable.query().limit(limit).toArray();
838
+ return results.map((r) => ({
839
+ ...r,
840
+ score: 1.0, // Full score for direct listing
841
+ // Parse metadata JSON string to object
842
+ metadata: typeof r.metadata === "string" ? JSON.parse(r.metadata) : r.metadata,
843
+ }));
844
+ }
845
+ catch (error) {
846
+ if (process.env.YAMO_DEBUG === "true") {
847
+ logger.error({ err: error }, "Skill list failed");
848
+ }
849
+ return [];
850
+ }
851
+ }
852
+ /**
853
+ * Search for synthesized skills by semantic intent
854
+ * @param {string} query - Search query (intent description)
855
+ * @param {Object} [options={}] - Search options
856
+ * @returns {Promise<Array>} Normalized skill results
857
+ */
858
+ async searchSkills(query, options = {}) {
859
+ await this.init();
860
+ if (!this.skillTable) {
861
+ return [];
862
+ }
863
+ try {
864
+ // 1. Check for explicit skill targeting (e.g., "Architect: ...")
865
+ const explicitMatch = query.match(/^([a-zA-Z0-9_-]+):/);
866
+ if (explicitMatch) {
867
+ const targetName = explicitMatch[1];
868
+ const directResults = await this.skillTable
869
+ .query()
870
+ .where(`name == '${targetName}'`)
871
+ .limit(1)
872
+ .toArray();
873
+ if (directResults.length > 0) {
874
+ return directResults.map((r) => ({
875
+ ...r,
876
+ score: 1.0, // Maximum score for explicit target
877
+ }));
878
+ }
879
+ }
880
+ // 2. Hybrid search: vector + keyword matching
881
+ const limit = options.limit || 5;
882
+ // 2a. Vector search (get more candidates for fusion)
883
+ const vector = await this.embeddingFactory.embed(query);
884
+ const vectorResults = await this.skillTable
885
+ .search(vector)
886
+ .limit(limit * 3)
887
+ .toArray();
888
+ // 2b. Keyword matching against skill fields (including tags)
889
+ const queryTokens = this._tokenizeQuery(query);
890
+ const keywordScores = new Map();
891
+ let maxKeywordScore = 0;
892
+ for (const result of vectorResults) {
893
+ let score = 0;
894
+ const nameTokens = this._tokenizeQuery(result.name);
895
+ const intentTokens = this._tokenizeQuery(result.intent || "");
896
+ const tags = extractSkillTags(result.yamo_text);
897
+ const tagTokens = tags.flatMap((t) => this._tokenizeQuery(t));
898
+ const descTokens = this._tokenizeQuery(result.yamo_text.substring(0, 500)); // First 500 chars
899
+ // Token matching with field-based weights
900
+ // Support both exact and partial matches (for compound words)
901
+ for (const qToken of queryTokens) {
902
+ // Exact or partial match in name
903
+ if (nameTokens.some((nt) => nt === qToken || qToken.includes(nt) || nt.includes(qToken))) {
904
+ score += 10.0; // Highest: name match
905
+ }
906
+ // Exact or partial match in tags
907
+ if (tagTokens.some((tt) => tt === qToken || qToken.includes(tt) || tt.includes(qToken))) {
908
+ score += 7.0; // High: tag match
909
+ }
910
+ // Exact match in intent
911
+ if (intentTokens.some((it) => it === qToken)) {
912
+ score += 5.0; // Medium: intent match
913
+ }
914
+ // Exact match in description
915
+ if (descTokens.some((dt) => dt === qToken)) {
916
+ score += 1.0; // Low: description match
917
+ }
918
+ }
919
+ if (score > 0) {
920
+ keywordScores.set(result.id, score);
921
+ maxKeywordScore = Math.max(maxKeywordScore, score);
922
+ }
923
+ }
924
+ // 2c. Combine scores using weighted fusion
925
+ const fusedResults = vectorResults.map((r) => {
926
+ // Normalize vector distance to [0, 1] similarity score
927
+ // LanceDB cosine distance ranges from 0 (identical) to 2 (opposite)
928
+ const rawDistance = r._distance !== undefined ? r._distance : 1.0;
929
+ const vectorScore = Math.max(0, Math.min(1.0, 1 - rawDistance / 2));
930
+ const keywordScore = keywordScores.get(r.id) || 0;
931
+ // Normalize keyword score by max observed (or use fixed max to avoid division by zero)
932
+ const normalizedKeyword = maxKeywordScore > 0 ? keywordScore / maxKeywordScore : 0;
933
+ // Weighted combination: 70% keyword, 30% vector
934
+ // Keywords get higher weight to prioritize exact matches
935
+ const combinedScore = 0.7 * normalizedKeyword + 0.3 * vectorScore;
936
+ return {
937
+ ...r,
938
+ score: combinedScore,
939
+ _vectorScore: vectorScore,
940
+ _keywordScore: keywordScore,
941
+ };
942
+ });
943
+ // Sort by combined score and return top results
944
+ // Don't normalize - we already calculated hybrid scores
945
+ return fusedResults
946
+ .sort((a, b) => b.score - a.score)
947
+ .slice(0, limit)
948
+ .map((r) => ({
949
+ ...r,
950
+ // Parse metadata JSON string to object for policy loading
951
+ metadata: typeof r.metadata === "string"
952
+ ? JSON.parse(r.metadata)
953
+ : r.metadata,
954
+ }))
955
+ .map((r) => ({
956
+ ...r,
957
+ score: parseFloat(r.score.toFixed(2)), // Round for consistency
958
+ }));
959
+ }
960
+ catch (error) {
961
+ if (process.env.YAMO_DEBUG === "true") {
962
+ logger.error({ err: error }, "Skill search failed");
963
+ }
964
+ return [];
965
+ }
966
+ }
967
+ /**
968
+ * Get recent YAMO logs for the heartbeat
969
+ * @param {Object} options
970
+ */
971
+ async getYamoLog(options = {}) {
972
+ if (!this.yamoTable) {
973
+ return [];
974
+ }
975
+ const limit = options.limit || 10;
976
+ const maxRetries = 5;
977
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
978
+ try {
979
+ // orderBy might not be in LanceDB types but is supported in runtime
980
+ const query = this.yamoTable.query();
981
+ let results;
982
+ try {
983
+ results = await query
984
+ .orderBy("timestamp", "desc")
985
+ .limit(limit)
986
+ .toArray();
987
+ }
988
+ catch (_e) {
989
+ // Fallback if orderBy not supported
990
+ results = await query.limit(1000).toArray(); // Get more and sort manually
991
+ }
992
+ // Sort newest first in memory
993
+ return results
994
+ .sort((a, b) => {
995
+ const tA = a.timestamp instanceof Date
996
+ ? a.timestamp.getTime()
997
+ : Number(a.timestamp);
998
+ const tB = b.timestamp instanceof Date
999
+ ? b.timestamp.getTime()
1000
+ : Number(b.timestamp);
1001
+ return tB - tA;
1002
+ })
1003
+ .slice(0, limit)
1004
+ .map((r) => ({
1005
+ id: r.id,
1006
+ yamoText: r.yamo_text,
1007
+ timestamp: r.timestamp,
1008
+ }));
1009
+ }
1010
+ catch (error) {
1011
+ const msg = error.message || "";
1012
+ const isRetryable = msg.includes("LanceError(IO)") ||
1013
+ msg.includes("next batch") ||
1014
+ msg.includes("No such file") ||
1015
+ msg.includes("busy");
1016
+ if (isRetryable && attempt < maxRetries) {
1017
+ // If we suspect stale table handle, try to refresh it
1018
+ try {
1019
+ // Re-open table to get fresh file handles
1020
+ const { createYamoTable } = await import("../yamo/schema.js");
1021
+ if (this.dbDir) {
1022
+ const db = await lancedb.connect(this.dbDir);
1023
+ this.yamoTable = await createYamoTable(db, "yamo_blocks");
1024
+ if (process.env.YAMO_DEBUG === "true") {
1025
+ logger.debug({ attempt, msg: msg.substring(0, 100) }, "Refreshed yamoTable handle during retry");
1026
+ }
1027
+ }
1028
+ }
1029
+ catch (e) {
1030
+ logger.warn({ err: e }, "Failed to refresh table handle during retry");
1031
+ }
1032
+ const delay = 500 * Math.pow(2, attempt - 1); // 500ms, 1000ms, 2000ms, 4000ms
1033
+ await new Promise((resolve) => setTimeout(resolve, delay));
1034
+ continue;
1035
+ }
1036
+ // Only log warning on final failure
1037
+ if (attempt === maxRetries) {
1038
+ logger.warn({ err: error }, "Failed to get log after retries");
1039
+ }
1040
+ else if (!isRetryable) {
1041
+ // Non-retryable error
1042
+ logger.warn({ err: error }, "Failed to get log (non-retryable)");
1043
+ break;
1044
+ }
1045
+ }
1046
+ }
1047
+ return [];
1048
+ }
1049
+ /**
1050
+ * Emit a YAMO block to the YAMO blocks table
1051
+ * @private
1052
+ *
1053
+ * Note: YAMO emission is non-critical - failures are logged but don't throw
1054
+ * to prevent disrupting the main operation.
1055
+ */
1056
+ async _emitYamoBlock(operationType, memoryId, yamoText) {
1057
+ if (!this.yamoTable) {
1058
+ return;
1059
+ }
1060
+ const yamoId = `yamo_${operationType}_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
1061
+ try {
1062
+ await this.yamoTable.add([
1063
+ {
1064
+ id: yamoId,
1065
+ agent_id: this.agentId,
1066
+ operation_type: operationType,
1067
+ yamo_text: yamoText,
1068
+ timestamp: new Date(),
1069
+ block_hash: null,
1070
+ prev_hash: null,
1071
+ metadata: JSON.stringify({
1072
+ memory_id: memoryId || null,
1073
+ timestamp: new Date().toISOString(),
1074
+ }),
1075
+ },
1076
+ ]);
1077
+ }
1078
+ catch (error) {
1079
+ // Log emission failures in debug mode
1080
+ // Emission is non-critical, so we don't throw
1081
+ if (process.env.YAMO_DEBUG === "true") {
1082
+ logger.warn({ err: error, operationType }, "YAMO emission failed");
1083
+ }
1084
+ }
1085
+ }
1086
+ /**
1087
+ * Search memory using hybrid vector + keyword search with Reciprocal Rank Fusion (RRF).
1088
+ *
1089
+ * This method performs semantic search by combining:
1090
+ * 1. **Vector Search**: Uses embeddings to find semantically similar content
1091
+ * 2. **Keyword Search**: Uses BM25-style keyword matching
1092
+ * 3. **RRF Fusion**: Combines both result sets using Reciprocal Rank Fusion
1093
+ *
1094
+ * The RRF algorithm scores each document as: `sum(1 / (k + rank))` where k=60.
1095
+ * This gives higher scores to documents that rank well in BOTH searches.
1096
+ *
1097
+ * **Performance**: Uses adaptive sorting strategy
1098
+ * - Small datasets (≤ 2× limit): Full sort O(n log n)
1099
+ * - Large datasets: Partial selection sort O(n×k) where k=limit
1100
+ *
1101
+ * **Caching**: Results are cached for 5 minutes by default (configurable via options)
1102
+ *
1103
+ * @param query - The search query text
1104
+ * @param options - Search options
1105
+ * @param options.limit - Maximum results to return (default: 10)
1106
+ * @param options.filter - LanceDB filter expression (e.g., "type == 'preference'")
1107
+ * @param options.useCache - Enable/disable result caching (default: true)
1108
+ * @returns Promise with array of search results, sorted by relevance score
1109
+ *
1110
+ * @example
1111
+ * ```typescript
1112
+ * // Simple search
1113
+ * const results = await mesh.search("TypeScript preferences");
1114
+ *
1115
+ * // Search with filter
1116
+ * const code = await mesh.search("bug fix", { filter: "type == 'error'" });
1117
+ *
1118
+ * // Search with limit
1119
+ * const top3 = await mesh.search("security issues", { limit: 3 });
1120
+ * ```
1121
+ *
1122
+ * @throws {Error} If embedding generation fails
1123
+ * @throws {Error} If database client is not initialized
1124
+ */
1125
+ async search(query, options = {}) {
1126
+ await this.init();
1127
+ try {
1128
+ const limit = options.limit || 10;
1129
+ const filter = options.filter || null;
1130
+ const useCache = options.useCache !== undefined ? options.useCache : true;
1131
+ if (useCache) {
1132
+ const cacheKey = this._generateCacheKey(query, { limit, filter });
1133
+ const cached = this._getCachedResult(cacheKey);
1134
+ if (cached) {
1135
+ return cached;
1136
+ }
1137
+ }
1138
+ const vector = await this.embeddingFactory.embed(query);
1139
+ if (!this.client) {
1140
+ throw new Error("Database client not initialized");
1141
+ }
1142
+ const vectorResults = await this.client.search(vector, {
1143
+ limit: limit * 2,
1144
+ metric: "cosine",
1145
+ filter,
1146
+ });
1147
+ const keywordResults = this.keywordSearch.search(query, {
1148
+ limit: limit * 2,
1149
+ });
1150
+ // Optimized Reciprocal Rank Fusion (RRF) with min-heap for O(n log k) performance
1151
+ // Instead of sorting all results (O(n log n)), we maintain a heap of size k (O(n log k))
1152
+ const k = 60; // RRF constant
1153
+ const scores = new Map();
1154
+ const docMap = new Map();
1155
+ // Process vector results - O(m) where m = vectorResults.length
1156
+ for (let rank = 0; rank < vectorResults.length; rank++) {
1157
+ const doc = vectorResults[rank];
1158
+ const rrf = 1 / (k + rank + 1);
1159
+ scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
1160
+ docMap.set(doc.id, doc);
1161
+ }
1162
+ // Process keyword results - O(n) where n = keywordResults.length
1163
+ for (let rank = 0; rank < keywordResults.length; rank++) {
1164
+ const doc = keywordResults[rank];
1165
+ const rrf = 1 / (k + rank + 1);
1166
+ scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
1167
+ if (!docMap.has(doc.id)) {
1168
+ docMap.set(doc.id, {
1169
+ id: doc.id,
1170
+ content: doc.content,
1171
+ metadata: doc.metadata,
1172
+ score: 0,
1173
+ created_at: new Date().toISOString(),
1174
+ });
1175
+ }
1176
+ }
1177
+ // Extract top k results using min-heap pattern - O(n log k)
1178
+ // Since JavaScript doesn't have a built-in heap, we use an efficient approach:
1179
+ // Convert to array and sort only if results exceed limit significantly
1180
+ const scoreEntries = Array.from(scores.entries());
1181
+ let mergedResults;
1182
+ if (scoreEntries.length <= limit * 2) {
1183
+ // Small dataset: standard sort is fine
1184
+ mergedResults = scoreEntries
1185
+ .sort((a, b) => b[1] - a[1]) // O(n log n) but n is small
1186
+ .slice(0, limit)
1187
+ .map(([id, score]) => {
1188
+ const doc = docMap.get(id);
1189
+ return doc ? { ...doc, score } : null;
1190
+ })
1191
+ .filter((d) => d !== null);
1192
+ }
1193
+ else {
1194
+ // Large dataset: use partial selection sort (O(n*k) but k is small)
1195
+ // This is more efficient than full sort when we only need top k results
1196
+ const topK = [];
1197
+ for (const entry of scoreEntries) {
1198
+ if (topK.length < limit) {
1199
+ topK.push(entry);
1200
+ // Keep topK sorted in descending order
1201
+ topK.sort((a, b) => b[1] - a[1]);
1202
+ }
1203
+ else if (entry[1] > topK[topK.length - 1][1]) {
1204
+ // Replace smallest in topK if current is larger
1205
+ topK[limit - 1] = entry;
1206
+ topK.sort((a, b) => b[1] - a[1]);
1207
+ }
1208
+ }
1209
+ mergedResults = topK
1210
+ .map(([id, score]) => {
1211
+ const doc = docMap.get(id);
1212
+ return doc ? { ...doc, score } : null;
1213
+ })
1214
+ .filter((d) => d !== null);
1215
+ }
1216
+ const normalizedResults = this._normalizeScores(mergedResults);
1217
+ if (useCache) {
1218
+ const cacheKey = this._generateCacheKey(query, { limit, filter });
1219
+ this._cacheResult(cacheKey, normalizedResults);
1220
+ }
1221
+ if (this.enableYamo) {
1222
+ this._emitYamoBlock("recall", undefined, YamoEmitter.buildRecallBlock({
1223
+ query,
1224
+ resultCount: normalizedResults.length,
1225
+ limit,
1226
+ agentId: this.agentId,
1227
+ searchType: "hybrid",
1228
+ })).catch((error) => {
1229
+ // Log emission failures in debug mode but don't throw
1230
+ if (process.env.YAMO_DEBUG === "true") {
1231
+ logger.warn({ err: error }, "Failed to emit YAMO block (recall)");
1232
+ }
1233
+ });
1234
+ }
1235
+ return normalizedResults;
1236
+ }
1237
+ catch (error) {
1238
+ throw error instanceof Error ? error : new Error(String(error));
1239
+ }
1240
+ }
1241
+ _normalizeScores(results) {
1242
+ if (results.length === 0) {
1243
+ return [];
1244
+ }
1245
+ return results.map((r) => {
1246
+ // LanceDB _distance is squared L2 or cosine distance
1247
+ // For cosine distance in MiniLM, it ranges from 0 to 2
1248
+ const rawDistance = r._distance !== undefined ? r._distance : 1.0;
1249
+ // Convert to similarity score [0, 1]
1250
+ const score = Math.max(0, Math.min(1.0, 1 - rawDistance / 2));
1251
+ return {
1252
+ ...r,
1253
+ score: parseFloat(score.toFixed(2)),
1254
+ };
1255
+ });
1256
+ }
1257
+ /**
1258
+ * Tokenize query for keyword matching (private helper for searchSkills)
1259
+ * Converts text to lowercase tokens, filtering out short tokens and punctuation.
1260
+ * Handles camelCase/PascalCase by splitting on uppercase letters.
1261
+ */
1262
+ _tokenizeQuery(text) {
1263
+ return text
1264
+ .replace(/([a-z])([A-Z])/g, "$1 $2") // Split camelCase: "targetSkill" → "target Skill"
1265
+ .toLowerCase()
1266
+ .replace(/[^\w\s]/g, "")
1267
+ .split(/\s+/)
1268
+ .filter((t) => t.length > 2); // Filter out very short tokens
1269
+ }
1270
+ formatResults(results) {
1271
+ if (results.length === 0) {
1272
+ return "No relevant memories found.";
1273
+ }
1274
+ let output = `[ATTENTION DIRECTIVE]\nThe following [MEMORY CONTEXT] is weighted by relevance.
1275
+ - ALIGN attention to entries with [IMPORTANCE >= 0.8].
1276
+ - TREAT entries with [IMPORTANCE <= 0.4] as auxiliary background info.
1277
+
1278
+ [MEMORY CONTEXT]`;
1279
+ results.forEach((res, i) => {
1280
+ const metadata = typeof res.metadata === "string"
1281
+ ? JSON.parse(res.metadata)
1282
+ : res.metadata;
1283
+ output += `\n\n--- MEMORY ${i + 1}: ${res.id} [IMPORTANCE: ${res.score}] ---\nType: ${metadata.type || "event"} | Source: ${metadata.source || "unknown"}\n${res.content}`;
1284
+ });
1285
+ return output;
1286
+ }
1287
+ async get(id) {
1288
+ await this.init();
1289
+ if (!this.client) {
1290
+ throw new Error("Database client not initialized");
1291
+ }
1292
+ const record = await this.client.getById(id);
1293
+ return record
1294
+ ? {
1295
+ id: record.id,
1296
+ content: record.content,
1297
+ metadata: record.metadata,
1298
+ created_at: record.created_at,
1299
+ updated_at: record.updated_at,
1300
+ }
1301
+ : null;
1302
+ }
1303
+ async getAll(options = {}) {
1304
+ await this.init();
1305
+ if (!this.client) {
1306
+ throw new Error("Database client not initialized");
1307
+ }
1308
+ return this.client.getAll(options);
1309
+ }
1310
+ async stats() {
1311
+ await this.init();
1312
+ if (!this.enableMemory || !this.client) {
1313
+ return {
1314
+ count: 0,
1315
+ totalMemories: 0,
1316
+ totalSkills: 0,
1317
+ tableName: "N/A",
1318
+ uri: "N/A",
1319
+ isConnected: false,
1320
+ embedding: { configured: false, primary: null, fallbacks: [] },
1321
+ status: "disabled",
1322
+ };
1323
+ }
1324
+ const dbStats = await this.client.getStats();
1325
+ // Enrich embedding stats with total persisted count
1326
+ const embeddingStats = this.embeddingFactory.getStats();
1327
+ if (embeddingStats.primary) {
1328
+ embeddingStats.primary.totalPersisted = dbStats.count;
1329
+ }
1330
+ // Get skill count
1331
+ let totalSkills = 0;
1332
+ if (this.skillTable) {
1333
+ try {
1334
+ const skills = await this.skillTable.query().limit(10000).toArray();
1335
+ totalSkills = skills.length;
1336
+ }
1337
+ catch (_e) {
1338
+ // Ignore errors
1339
+ }
1340
+ }
1341
+ return {
1342
+ count: dbStats.count,
1343
+ totalMemories: dbStats.count,
1344
+ totalSkills,
1345
+ tableName: dbStats.tableName,
1346
+ uri: dbStats.uri,
1347
+ isConnected: dbStats.isConnected,
1348
+ embedding: embeddingStats,
1349
+ };
1350
+ }
1351
+ _parseEmbeddingConfig() {
1352
+ const configs = [
1353
+ {
1354
+ modelType: process.env.EMBEDDING_MODEL_TYPE || "local",
1355
+ modelName: process.env.EMBEDDING_MODEL_NAME || "Xenova/all-MiniLM-L6-v2",
1356
+ dimension: parseInt(process.env.EMBEDDING_DIMENSION || "384"),
1357
+ priority: 1,
1358
+ apiKey: process.env.EMBEDDING_API_KEY ||
1359
+ process.env.OPENAI_API_KEY ||
1360
+ process.env.COHERE_API_KEY,
1361
+ },
1362
+ ];
1363
+ if (configs[0].modelType !== "local") {
1364
+ configs.push({
1365
+ modelType: "local",
1366
+ modelName: "Xenova/all-MiniLM-L6-v2",
1367
+ dimension: 384,
1368
+ priority: 2,
1369
+ apiKey: undefined,
1370
+ });
1371
+ }
1372
+ return configs;
1373
+ }
1374
+ /**
1375
+ * Close database connections and release resources
1376
+ *
1377
+ * This should be called when done with the MemoryMesh to properly:
1378
+ * - Close LanceDB connections
1379
+ * - Release file handles
1380
+ * - Clean up resources
1381
+ *
1382
+ * Important for tests and cleanup to prevent connection leaks.
1383
+ *
1384
+ * @returns {Promise<void>}
1385
+ *
1386
+ * @example
1387
+ * ```typescript
1388
+ * const mesh = new MemoryMesh();
1389
+ * await mesh.init();
1390
+ * // ... use mesh ...
1391
+ * await mesh.close(); // Clean up
1392
+ * ```
1393
+ */
1394
+ // eslint-disable-next-line @typescript-eslint/require-await
1395
+ async close() {
1396
+ try {
1397
+ // Close LanceDB client connection
1398
+ if (this.client) {
1399
+ this.client.disconnect();
1400
+ this.client = null;
1401
+ }
1402
+ // Clear extension table references
1403
+ this.yamoTable = null;
1404
+ this.skillTable = null;
1405
+ // Reset initialization state
1406
+ this.isInitialized = false;
1407
+ logger.debug("MemoryMesh closed successfully");
1408
+ }
1409
+ catch (error) {
1410
+ const e = error instanceof Error ? error : new Error(String(error));
1411
+ logger.warn({ err: e }, "Error closing MemoryMesh");
1412
+ // Don't throw - cleanup should always succeed
1413
+ }
617
1414
  }
618
- }
619
-
620
- _normalizeScores(results) {
621
- if (results.length === 0) return [];
622
- if (results.length === 1) return [{ ...results[0], score: 1.0 }];
623
- const scores = results.map(r => r.score);
624
- const max = Math.max(...scores), min = Math.min(...scores);
625
- const range = max - min || 1;
626
- return results.map(r => ({ ...r, score: parseFloat(((r.score - min) / range).toFixed(2)) }));
627
- }
628
-
629
- formatResults(results) {
630
- if (results.length === 0) return 'No relevant memories found.';
631
- let output = `[ATTENTION DIRECTIVE]\nThe following [MEMORY CONTEXT] is weighted by relevance.\n- ALIGN attention to entries with [IMPORTANCE >= 0.8].\n- TREAT entries with [IMPORTANCE <= 0.4] as auxiliary background info.\n\n[MEMORY CONTEXT]`;
632
- results.forEach((res, i) => {
633
- const metadata = typeof res.metadata === 'string' ? JSON.parse(res.metadata) : res.metadata;
634
- output += `\n\n--- MEMORY ${i + 1}: ${res.id} [IMPORTANCE: ${res.score}] ---\nType: ${metadata.type || 'event'} | Source: ${metadata.source || 'unknown'}\n${res.content}`;
635
- });
636
- return output;
637
- }
638
-
639
- async get(id) {
640
- await this.init();
641
- if (!this.client) throw new Error('Database client not initialized');
642
- const record = await this.client.getById(id);
643
- return record ? { id: record.id, content: record.content, metadata: record.metadata, created_at: record.created_at, updated_at: record.updated_at } : null;
644
- }
645
-
646
- async getAll(options = {}) {
647
- await this.init();
648
- if (!this.client) throw new Error('Database client not initialized');
649
- return await this.client.getAll(options);
650
- }
651
-
652
- async stats() {
653
- await this.init();
654
- if (!this.client) throw new Error('Database client not initialized');
655
- const dbStats = await this.client.getStats();
656
- return { count: dbStats.count, tableName: dbStats.tableName, uri: dbStats.uri, isConnected: dbStats.isConnected, embedding: this.embeddingFactory.getStats() };
657
- }
658
-
659
- _parseEmbeddingConfig() {
660
- const configs = [{ modelType: process.env.EMBEDDING_MODEL_TYPE || 'local', modelName: process.env.EMBEDDING_MODEL_NAME || 'Xenova/all-MiniLM-L6-v2', dimension: parseInt(process.env.EMBEDDING_DIMENSION || '384'), priority: 1, apiKey: process.env.EMBEDDING_API_KEY || process.env.OPENAI_API_KEY || process.env.COHERE_API_KEY }];
661
- if (configs[0].modelType !== 'local') configs.push({ modelType: 'local', modelName: 'Xenova/all-MiniLM-L6-v2', dimension: 384, priority: 2 });
662
- return configs;
663
- }
664
1415
  }
665
-
666
1416
  /**
667
1417
  * Main CLI handler
668
1418
  */
669
- async function run() {
670
- let action, input;
671
- if (process.argv.length > 3) {
672
- action = process.argv[2];
673
- try { input = JSON.parse(process.argv[3]); } catch (e) { console.error(`❌ Error: Invalid JSON argument: ${e.message}`); process.exit(1); }
674
- } else {
675
- try { const rawInput = fs.readFileSync(0, 'utf8'); input = JSON.parse(rawInput); action = input.action || action; } catch (e) { console.error("❌ Error: No input provided."); process.exit(1); }
676
- }
677
-
678
- const mesh = new MemoryMesh({
679
- llmProvider: process.env.LLM_PROVIDER || (process.env.OPENAI_API_KEY ? 'openai' : 'ollama'),
680
- llmApiKey: process.env.LLM_API_KEY || process.env.OPENAI_API_KEY,
681
- llmModel: process.env.LLM_MODEL
682
- });
683
-
684
- try {
685
- if (action === 'ingest' || action === 'store') {
686
- const record = await mesh.add(input.content, input.metadata || {});
687
- console.log(`[MemoryMesh] Ingested record ${record.id}\n${JSON.stringify({ status: "ok", record })}`);
688
- } else if (action === 'search') {
689
- const results = await mesh.search(input.query, { limit: input.limit || 10, filter: input.filter || null });
690
- console.log(`[MemoryMesh] Found ${results.length} matches.\n**Formatted Context**:\n\`\`\`yamo\n${mesh.formatResults(results)}\n\`\`\`\n**Output**: memory_results.json\n\`\`\`json\n${JSON.stringify(results, null, 2)}\n\`\`\`\n${JSON.stringify({ status: "ok", results })}`);
691
- } else if (action === 'synthesize') {
692
- const result = await mesh.synthesize({ topic: input.topic, lookback: input.limit || 20 });
693
- console.log(`[MemoryMesh] Synthesis Outcome: ${result.status}\n${JSON.stringify(result, null, 2)}`);
694
- } else if (action === 'ingest-skill') {
695
- const record = await mesh.ingestSkill(input.yamo_text, input.metadata || {});
696
- console.log(`[MemoryMesh] Ingested skill ${record.name} (${record.id})\n${JSON.stringify({ status: "ok", record })}`);
697
- } else if (action === 'search-skills') {
698
- await mesh.init();
699
- const vector = await mesh.embeddingFactory.embed(input.query);
700
- const results = await mesh.skillTable.search(vector).limit(input.limit || 5).toArray();
701
- console.log(`[MemoryMesh] Found ${results.length} synthesized skills.\n${JSON.stringify({ status: "ok", results }, null, 2)}`);
702
- } else if (action === 'skill-feedback') {
703
- const result = await mesh.updateSkillReliability(input.id, input.success !== false);
704
- console.log(`[MemoryMesh] Feedback recorded for ${input.id}: Reliability now ${result.reliability}\n${JSON.stringify({ status: "ok", ...result })}`);
705
- } else if (action === 'skill-prune') {
706
- const result = await mesh.pruneSkills(input.threshold || 0.3);
707
- console.log(`[MemoryMesh] Pruning complete. Removed ${result.pruned_count} unreliable skills.\n${JSON.stringify({ status: "ok", ...result })}`);
708
- } else if (action === 'stats') {
709
- console.log(`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}`);
710
- } else {
711
- console.error(`❌ Error: Unknown action "${action}".`); process.exit(1);
712
- }
713
- } catch (error) {
714
- const errorResponse = handleError(error, { action, input: { ...input, content: input.content ? '[REDACTED]' : undefined } });
715
- console.error(`❌ Fatal Error: ${errorResponse.error.message}\n${JSON.stringify(errorResponse, null, 2)}`);
716
- process.exit(1);
717
- }
1419
+ export async function run() {
1420
+ let action, input;
1421
+ if (process.argv.length > 3) {
1422
+ action = process.argv[2];
1423
+ try {
1424
+ input = JSON.parse(process.argv[3]);
1425
+ }
1426
+ catch (e) {
1427
+ logger.error({ err: e }, "Invalid JSON argument");
1428
+ process.exit(1);
1429
+ }
1430
+ }
1431
+ else {
1432
+ try {
1433
+ const rawInput = fs.readFileSync(0, "utf8");
1434
+ input = JSON.parse(rawInput);
1435
+ action = input.action || action;
1436
+ }
1437
+ catch (_e) {
1438
+ logger.error("No input provided");
1439
+ process.exit(1);
1440
+ }
1441
+ }
1442
+ const mesh = new MemoryMesh({
1443
+ llmProvider: process.env.LLM_PROVIDER ||
1444
+ (process.env.OPENAI_API_KEY ? "openai" : "ollama"),
1445
+ llmApiKey: process.env.LLM_API_KEY || process.env.OPENAI_API_KEY,
1446
+ llmModel: process.env.LLM_MODEL,
1447
+ });
1448
+ try {
1449
+ if (action === "ingest" || action === "store") {
1450
+ const record = await mesh.add(input.content, input.metadata || {});
1451
+ process.stdout.write(`[MemoryMesh] Ingested record ${record.id}\n${JSON.stringify({ status: "ok", record })}\n`);
1452
+ }
1453
+ else if (action === "search") {
1454
+ const results = await mesh.search(input.query, {
1455
+ limit: input.limit || 10,
1456
+ filter: input.filter || null,
1457
+ });
1458
+ process.stdout.write(`[MemoryMesh] Found ${results.length} matches.\n**Formatted Context**:\n\`\`\`yamo\n${mesh.formatResults(results)}\n\`\`\`\n**Output**: memory_results.json\n\`\`\`json\n${JSON.stringify(results, null, 2)}\n\`\`\`\n${JSON.stringify({ status: "ok", results })}\n`);
1459
+ }
1460
+ else if (action === "synthesize") {
1461
+ const result = await mesh.synthesize({
1462
+ topic: input.topic,
1463
+ lookback: input.limit || 20,
1464
+ });
1465
+ process.stdout.write(`[MemoryMesh] Synthesis Outcome: ${result.status}\n${JSON.stringify(result, null, 2)}\n`);
1466
+ }
1467
+ else if (action === "ingest-skill") {
1468
+ const record = await mesh.ingestSkill(input.yamo_text, input.metadata || {});
1469
+ process.stdout.write(`[MemoryMesh] Ingested skill ${record.name} (${record.id})\n${JSON.stringify({ status: "ok", record })}\n`);
1470
+ }
1471
+ else if (action === "search-skills") {
1472
+ await mesh.init();
1473
+ const vector = await mesh.embeddingFactory.embed(input.query);
1474
+ if (mesh.skillTable) {
1475
+ const results = await mesh.skillTable
1476
+ .search(vector)
1477
+ .limit(input.limit || 5)
1478
+ .toArray();
1479
+ process.stdout.write(`[MemoryMesh] Found ${results.length} synthesized skills.\n${JSON.stringify({ status: "ok", results }, null, 2)}\n`);
1480
+ }
1481
+ else {
1482
+ process.stdout.write(`[MemoryMesh] Skill table not initialized.\n`);
1483
+ }
1484
+ }
1485
+ else if (action === "skill-feedback") {
1486
+ const result = await mesh.updateSkillReliability(input.id, input.success !== false);
1487
+ process.stdout.write(`[MemoryMesh] Feedback recorded for ${input.id}: Reliability now ${result.reliability}\n${JSON.stringify({ status: "ok", ...result })}\n`);
1488
+ }
1489
+ else if (action === "skill-prune") {
1490
+ const result = await mesh.pruneSkills(input.threshold || 0.3);
1491
+ process.stdout.write(`[MemoryMesh] Pruning complete. Removed ${result.pruned_count} unreliable skills.\n${JSON.stringify({ status: "ok", ...result })}\n`);
1492
+ }
1493
+ else if (action === "stats") {
1494
+ process.stdout.write(`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}\n`);
1495
+ }
1496
+ else {
1497
+ logger.error({ action }, "Unknown action");
1498
+ process.exit(1);
1499
+ }
1500
+ }
1501
+ catch (error) {
1502
+ const errorResponse = handleError(error, {
1503
+ action,
1504
+ input: { ...input, content: input.content ? "[REDACTED]" : undefined },
1505
+ });
1506
+ logger.error({ err: error, errorResponse }, "Fatal Error");
1507
+ process.exit(1);
1508
+ }
718
1509
  }
719
-
720
- export { MemoryMesh, run };
721
1510
  export default MemoryMesh;
722
-
723
1511
  if (process.argv[1] === fileURLToPath(import.meta.url)) {
724
- run().catch(err => { console.error(`❌ Fatal Error: ${err.message}`); process.exit(1); });
725
- }
1512
+ run().catch((err) => {
1513
+ logger.error({ err }, "Fatal Error");
1514
+ process.exit(1);
1515
+ });
1516
+ }