@yamo/memory-mesh 2.3.1 → 2.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/lancedb/schema.js +20 -0
- package/lib/memory/memory-mesh.js +233 -768
- package/package.json +1 -1
package/lib/lancedb/schema.js
CHANGED
|
@@ -102,6 +102,26 @@ function createMemorySchemaV2(vectorDim = DEFAULT_VECTOR_DIMENSION) {
|
|
|
102
102
|
]);
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
/**
|
|
106
|
+
* Create schema for synthesized skills (Recursive Skill Synthesis)
|
|
107
|
+
* @param {number} vectorDim - Vector dimension for intent embedding
|
|
108
|
+
* @returns {import('apache-arrow').Schema} Arrow schema
|
|
109
|
+
*/
|
|
110
|
+
export function createSynthesizedSkillSchema(vectorDim = DEFAULT_VECTOR_DIMENSION) {
|
|
111
|
+
return new arrow.Schema([
|
|
112
|
+
new arrow.Field('id', new arrow.Utf8(), false),
|
|
113
|
+
new arrow.Field('name', new arrow.Utf8(), false),
|
|
114
|
+
new arrow.Field('intent', new arrow.Utf8(), false),
|
|
115
|
+
new arrow.Field('yamo_text', new arrow.Utf8(), false),
|
|
116
|
+
new arrow.Field('vector',
|
|
117
|
+
new arrow.FixedSizeList(vectorDim, new arrow.Field('item', new arrow.Float32(), true)),
|
|
118
|
+
false
|
|
119
|
+
),
|
|
120
|
+
new arrow.Field('metadata', new arrow.Utf8(), true), // Stored as JSON: {reliability, use_count, created_at}
|
|
121
|
+
new arrow.Field('created_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false)
|
|
122
|
+
]);
|
|
123
|
+
}
|
|
124
|
+
|
|
105
125
|
/**
|
|
106
126
|
* Check if a table is using V2 schema
|
|
107
127
|
* @param {import('apache-arrow').Schema} schema - Table schema to check
|
|
@@ -32,13 +32,13 @@ import { LLMClient } from "../llm/client.js";
|
|
|
32
32
|
class MemoryMesh {
|
|
33
33
|
/**
|
|
34
34
|
* Create a new MemoryMesh instance
|
|
35
|
-
* @param {Object} [options={}]
|
|
36
|
-
* @param {boolean} [options.enableYamo=true]
|
|
37
|
-
* @param {boolean} [options.enableLLM=true]
|
|
38
|
-
* @param {string} [options.agentId='default']
|
|
39
|
-
* @param {string} [options.llmProvider]
|
|
40
|
-
* @param {string} [options.llmApiKey]
|
|
41
|
-
* @param {string} [options.llmModel]
|
|
35
|
+
* @param {Object} [options={}]
|
|
36
|
+
* @param {boolean} [options.enableYamo=true]
|
|
37
|
+
* @param {boolean} [options.enableLLM=true]
|
|
38
|
+
* @param {string} [options.agentId='default']
|
|
39
|
+
* @param {string} [options.llmProvider]
|
|
40
|
+
* @param {string} [options.llmApiKey]
|
|
41
|
+
* @param {string} [options.llmModel]
|
|
42
42
|
*/
|
|
43
43
|
constructor(options = {}) {
|
|
44
44
|
this.client = null;
|
|
@@ -53,6 +53,7 @@ class MemoryMesh {
|
|
|
53
53
|
this.enableLLM = options.enableLLM !== false; // Default: true
|
|
54
54
|
this.agentId = options.agentId || 'default';
|
|
55
55
|
this.yamoTable = null; // Will be initialized in init()
|
|
56
|
+
this.skillTable = null; // Synthesized skills table
|
|
56
57
|
this.llmClient = null;
|
|
57
58
|
|
|
58
59
|
// Initialize LLM client if enabled
|
|
@@ -86,9 +87,6 @@ class MemoryMesh {
|
|
|
86
87
|
/**
|
|
87
88
|
* Generate a cache key from query and options
|
|
88
89
|
* @private
|
|
89
|
-
* @param {string} query - Search query
|
|
90
|
-
* @param {Object} options - Search options
|
|
91
|
-
* @returns {string} Cache key
|
|
92
90
|
*/
|
|
93
91
|
_generateCacheKey(query, options = {}) {
|
|
94
92
|
const normalizedOptions = {
|
|
@@ -102,8 +100,6 @@ class MemoryMesh {
|
|
|
102
100
|
/**
|
|
103
101
|
* Get cached result if valid
|
|
104
102
|
* @private
|
|
105
|
-
* @param {string} key - Cache key
|
|
106
|
-
* @returns {Object|null} Cached result or null if expired/missing
|
|
107
103
|
*/
|
|
108
104
|
_getCachedResult(key) {
|
|
109
105
|
const entry = this.queryCache.get(key);
|
|
@@ -125,8 +121,6 @@ class MemoryMesh {
|
|
|
125
121
|
/**
|
|
126
122
|
* Cache a search result
|
|
127
123
|
* @private
|
|
128
|
-
* @param {string} key - Cache key
|
|
129
|
-
* @param {Object} result - Search result to cache
|
|
130
124
|
*/
|
|
131
125
|
_cacheResult(key, result) {
|
|
132
126
|
// Evict oldest if at max size
|
|
@@ -150,7 +144,6 @@ class MemoryMesh {
|
|
|
150
144
|
|
|
151
145
|
/**
|
|
152
146
|
* Get cache statistics
|
|
153
|
-
* @returns {Object} Cache stats
|
|
154
147
|
*/
|
|
155
148
|
getCacheStats() {
|
|
156
149
|
return {
|
|
@@ -162,8 +155,6 @@ class MemoryMesh {
|
|
|
162
155
|
|
|
163
156
|
/**
|
|
164
157
|
* Validate and sanitize metadata to prevent prototype pollution
|
|
165
|
-
* @param {Object} metadata - Metadata to validate
|
|
166
|
-
* @returns {Object} Sanitized metadata
|
|
167
158
|
* @private
|
|
168
159
|
*/
|
|
169
160
|
_validateMetadata(metadata) {
|
|
@@ -191,8 +182,6 @@ class MemoryMesh {
|
|
|
191
182
|
|
|
192
183
|
/**
|
|
193
184
|
* Sanitize and validate content before storage
|
|
194
|
-
* @param {string} content - Content to sanitize
|
|
195
|
-
* @returns {string} Sanitized content
|
|
196
185
|
* @private
|
|
197
186
|
*/
|
|
198
187
|
_sanitizeContent(content) {
|
|
@@ -211,7 +200,6 @@ class MemoryMesh {
|
|
|
211
200
|
|
|
212
201
|
/**
|
|
213
202
|
* Initialize the LanceDB client
|
|
214
|
-
* @returns {Promise<void>}
|
|
215
203
|
*/
|
|
216
204
|
async init() {
|
|
217
205
|
if (this.isInitialized) {
|
|
@@ -250,8 +238,6 @@ class MemoryMesh {
|
|
|
250
238
|
await this.embeddingFactory.init();
|
|
251
239
|
|
|
252
240
|
// Hydrate Keyword Search (In-Memory)
|
|
253
|
-
// Note: This is efficient for small datasets (< 10k).
|
|
254
|
-
// For larger, we should persist the inverted index or use LanceDB FTS.
|
|
255
241
|
if (this.client) {
|
|
256
242
|
try {
|
|
257
243
|
const allRecords = await this.client.getAll({ limit: 10000 });
|
|
@@ -261,17 +247,28 @@ class MemoryMesh {
|
|
|
261
247
|
}
|
|
262
248
|
}
|
|
263
249
|
|
|
264
|
-
// Initialize
|
|
250
|
+
// Initialize extension tables if enabled
|
|
265
251
|
if (this.enableYamo && this.client && this.client.db) {
|
|
266
252
|
try {
|
|
267
253
|
const { createYamoTable } = await import('../yamo/schema.js');
|
|
268
254
|
this.yamoTable = await createYamoTable(this.client.db, 'yamo_blocks');
|
|
255
|
+
|
|
256
|
+
// Initialize synthesized skills table (Recursive Skill Synthesis)
|
|
257
|
+
const { createSynthesizedSkillSchema } = await import('../lancedb/schema.js');
|
|
258
|
+
const existingTables = await this.client.db.tableNames();
|
|
259
|
+
|
|
260
|
+
if (existingTables.includes('synthesized_skills')) {
|
|
261
|
+
this.skillTable = await this.client.db.openTable('synthesized_skills');
|
|
262
|
+
} else {
|
|
263
|
+
const skillSchema = createSynthesizedSkillSchema(this.vectorDimension);
|
|
264
|
+
this.skillTable = await this.client.db.createTable('synthesized_skills', [], { schema: skillSchema });
|
|
265
|
+
}
|
|
266
|
+
|
|
269
267
|
if (process.env.YAMO_DEBUG === 'true') {
|
|
270
|
-
console.error('[MemoryMesh] YAMO blocks
|
|
268
|
+
console.error('[MemoryMesh] YAMO blocks and synthesized skills tables initialized');
|
|
271
269
|
}
|
|
272
270
|
} catch (e) {
|
|
273
|
-
|
|
274
|
-
console.warn('[MemoryMesh] Failed to initialize YAMO table:', e instanceof Error ? e.message : String(e));
|
|
271
|
+
console.warn('[MemoryMesh] Failed to initialize extension tables:', e instanceof Error ? e.message : String(e));
|
|
275
272
|
}
|
|
276
273
|
}
|
|
277
274
|
|
|
@@ -285,19 +282,14 @@ class MemoryMesh {
|
|
|
285
282
|
|
|
286
283
|
/**
|
|
287
284
|
* Add content to memory with auto-generated embedding
|
|
288
|
-
* @param {string} content - Text content to store
|
|
289
|
-
* @param {Object} metadata - Optional metadata tags
|
|
290
|
-
* @returns {Promise<Object>} Created record with ID
|
|
291
285
|
*/
|
|
292
286
|
async add(content, metadata = {}) {
|
|
293
287
|
await this.init();
|
|
294
288
|
|
|
295
|
-
// Default to 'event' if no type provided
|
|
296
289
|
const type = metadata.type || 'event';
|
|
297
290
|
const enrichedMetadata = { ...metadata, type };
|
|
298
291
|
|
|
299
292
|
try {
|
|
300
|
-
// Layer 0: Scrubber Sanitization
|
|
301
293
|
let processedContent = content;
|
|
302
294
|
let scrubbedMetadata = {};
|
|
303
295
|
|
|
@@ -305,14 +297,11 @@ class MemoryMesh {
|
|
|
305
297
|
const scrubbedResult = await this.scrubber.process({
|
|
306
298
|
content: content,
|
|
307
299
|
source: 'memory-api',
|
|
308
|
-
type: 'txt'
|
|
300
|
+
type: 'txt'
|
|
309
301
|
});
|
|
310
302
|
|
|
311
303
|
if (scrubbedResult.success && scrubbedResult.chunks.length > 0) {
|
|
312
|
-
// Reconstruct cleaned content
|
|
313
304
|
processedContent = scrubbedResult.chunks.map(c => c.text).join('\n\n');
|
|
314
|
-
|
|
315
|
-
// Merge scrubber telemetry/metadata if useful
|
|
316
305
|
if (scrubbedResult.metadata) {
|
|
317
306
|
scrubbedMetadata = {
|
|
318
307
|
...scrubbedResult.metadata,
|
|
@@ -321,24 +310,17 @@ class MemoryMesh {
|
|
|
321
310
|
}
|
|
322
311
|
}
|
|
323
312
|
} catch (scrubError) {
|
|
324
|
-
// Fallback to raw content if scrubber fails, but log it
|
|
325
313
|
if (process.env.YAMO_DEBUG === 'true') {
|
|
326
|
-
|
|
327
|
-
console.error(`[MemoryMesh] Scrubber failed: ${message}`);
|
|
314
|
+
console.error(`[MemoryMesh] Scrubber failed: ${scrubError.message}`);
|
|
328
315
|
}
|
|
329
316
|
}
|
|
330
317
|
|
|
331
|
-
// Validate and sanitize inputs (legacy check)
|
|
332
318
|
const sanitizedContent = this._sanitizeContent(processedContent);
|
|
333
319
|
const sanitizedMetadata = this._validateMetadata({ ...enrichedMetadata, ...scrubbedMetadata });
|
|
334
320
|
|
|
335
|
-
// Generate ID
|
|
336
321
|
const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
337
|
-
|
|
338
|
-
// Generate embedding using EmbeddingFactory
|
|
339
322
|
const vector = await this.embeddingFactory.embed(sanitizedContent);
|
|
340
323
|
|
|
341
|
-
// Prepare record data with sanitized metadata
|
|
342
324
|
const record = {
|
|
343
325
|
id,
|
|
344
326
|
vector,
|
|
@@ -346,28 +328,18 @@ class MemoryMesh {
|
|
|
346
328
|
metadata: JSON.stringify(sanitizedMetadata)
|
|
347
329
|
};
|
|
348
330
|
|
|
349
|
-
|
|
350
|
-
// Add to LanceDB
|
|
351
331
|
if (!this.client) throw new Error('Database client not initialized');
|
|
352
332
|
const result = await this.client.add(record);
|
|
353
|
-
|
|
354
|
-
// Add to Keyword Search
|
|
355
333
|
this.keywordSearch.add(record.id, record.content, sanitizedMetadata);
|
|
356
334
|
|
|
357
|
-
// Emit YAMO block for retain operation (async, non-blocking)
|
|
358
335
|
if (this.enableYamo) {
|
|
359
|
-
// Fire and forget - don't await
|
|
360
336
|
this._emitYamoBlock('retain', result.id, YamoEmitter.buildRetainBlock({
|
|
361
337
|
content: sanitizedContent,
|
|
362
338
|
metadata: sanitizedMetadata,
|
|
363
339
|
id: result.id,
|
|
364
340
|
agentId: this.agentId,
|
|
365
341
|
memoryType: sanitizedMetadata.type || 'event'
|
|
366
|
-
})).catch(
|
|
367
|
-
if (process.env.YAMO_DEBUG === 'true') {
|
|
368
|
-
console.error('[MemoryMesh] YAMO emission failed in add():', err);
|
|
369
|
-
}
|
|
370
|
-
});
|
|
342
|
+
})).catch(() => {});
|
|
371
343
|
}
|
|
372
344
|
|
|
373
345
|
return {
|
|
@@ -376,30 +348,20 @@ class MemoryMesh {
|
|
|
376
348
|
metadata: sanitizedMetadata,
|
|
377
349
|
created_at: new Date().toISOString()
|
|
378
350
|
};
|
|
379
|
-
|
|
380
|
-
|
|
381
351
|
} catch (error) {
|
|
382
|
-
|
|
383
|
-
throw e;
|
|
352
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
384
353
|
}
|
|
385
354
|
}
|
|
386
355
|
|
|
387
356
|
/**
|
|
388
|
-
* Reflect on recent memories
|
|
389
|
-
* @param {Object} options
|
|
390
|
-
* @param {string} [options.topic] - Topic to search for
|
|
391
|
-
* @param {number} [options.lookback=10] - Number of memories to consider
|
|
392
|
-
* @param {boolean} [options.generate=true] - Whether to generate reflection via LLM
|
|
393
|
-
* @returns {Promise<Object>} Reflection result with YAMO block
|
|
357
|
+
* Reflect on recent memories
|
|
394
358
|
*/
|
|
395
359
|
async reflect(options = {}) {
|
|
396
360
|
await this.init();
|
|
397
|
-
|
|
398
361
|
const lookback = options.lookback || 10;
|
|
399
362
|
const topic = options.topic;
|
|
400
363
|
const generate = options.generate !== false;
|
|
401
364
|
|
|
402
|
-
// Gather memories
|
|
403
365
|
let memories = [];
|
|
404
366
|
if (topic) {
|
|
405
367
|
memories = await this.search(topic, { limit: lookback });
|
|
@@ -412,22 +374,10 @@ class MemoryMesh {
|
|
|
412
374
|
|
|
413
375
|
const prompt = `Review these memories. Synthesize a high-level "belief" or "observation".`;
|
|
414
376
|
|
|
415
|
-
// Check if LLM generation is requested and available
|
|
416
377
|
if (!generate || !this.enableLLM || !this.llmClient) {
|
|
417
|
-
|
|
418
|
-
return {
|
|
419
|
-
topic,
|
|
420
|
-
count: memories.length,
|
|
421
|
-
context: memories.map(m => ({
|
|
422
|
-
content: m.content,
|
|
423
|
-
type: m.metadata?.type || 'event',
|
|
424
|
-
id: m.id
|
|
425
|
-
})),
|
|
426
|
-
prompt
|
|
427
|
-
};
|
|
378
|
+
return { topic, count: memories.length, context: memories.map(m => ({ content: m.content, type: m.metadata?.type || 'event', id: m.id })), prompt };
|
|
428
379
|
}
|
|
429
380
|
|
|
430
|
-
// Generate reflection via LLM
|
|
431
381
|
let reflection = null;
|
|
432
382
|
let confidence = 0;
|
|
433
383
|
|
|
@@ -436,650 +386,281 @@ class MemoryMesh {
|
|
|
436
386
|
reflection = result.reflection;
|
|
437
387
|
confidence = result.confidence;
|
|
438
388
|
} catch (error) {
|
|
439
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
440
|
-
console.warn(`[MemoryMesh] LLM reflection failed: ${errorMessage}`);
|
|
441
|
-
// Fall back to simple aggregation
|
|
442
389
|
reflection = `Aggregated from ${memories.length} memories on topic: ${topic || 'general'}`;
|
|
443
390
|
confidence = 0.5;
|
|
444
391
|
}
|
|
445
392
|
|
|
446
|
-
// Store reflection to memory
|
|
447
393
|
const reflectionId = `reflect_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
|
|
448
|
-
await this.add(reflection, {
|
|
449
|
-
type: 'reflection',
|
|
450
|
-
topic: topic || 'general',
|
|
451
|
-
source_memory_count: memories.length,
|
|
452
|
-
confidence,
|
|
453
|
-
generated_at: new Date().toISOString()
|
|
454
|
-
});
|
|
394
|
+
await this.add(reflection, { type: 'reflection', topic: topic || 'general', source_memory_count: memories.length, confidence, generated_at: new Date().toISOString() });
|
|
455
395
|
|
|
456
|
-
// Emit YAMO block if enabled
|
|
457
396
|
let yamoBlock = null;
|
|
458
397
|
if (this.enableYamo) {
|
|
459
|
-
yamoBlock = YamoEmitter.buildReflectBlock({
|
|
460
|
-
topic: topic || 'general',
|
|
461
|
-
memoryCount: memories.length,
|
|
462
|
-
agentId: this.agentId,
|
|
463
|
-
reflection,
|
|
464
|
-
confidence
|
|
465
|
-
});
|
|
466
|
-
|
|
398
|
+
yamoBlock = YamoEmitter.buildReflectBlock({ topic: topic || 'general', memoryCount: memories.length, agentId: this.agentId, reflection, confidence });
|
|
467
399
|
await this._emitYamoBlock('reflect', reflectionId, yamoBlock);
|
|
468
400
|
}
|
|
469
401
|
|
|
470
|
-
return {
|
|
471
|
-
id: reflectionId,
|
|
472
|
-
topic: topic || 'general',
|
|
473
|
-
reflection,
|
|
474
|
-
confidence,
|
|
475
|
-
sourceMemoryCount: memories.length,
|
|
476
|
-
yamoBlock,
|
|
477
|
-
createdAt: new Date().toISOString()
|
|
478
|
-
};
|
|
402
|
+
return { id: reflectionId, topic: topic || 'general', reflection, confidence, sourceMemoryCount: memories.length, yamoBlock, createdAt: new Date().toISOString() };
|
|
479
403
|
}
|
|
480
404
|
|
|
481
405
|
/**
|
|
482
|
-
*
|
|
483
|
-
* @private
|
|
484
|
-
* @param {string} operationType - 'retain', 'recall', 'reflect'
|
|
485
|
-
* @param {string|undefined} memoryId - Associated memory ID (undefined for recall)
|
|
486
|
-
* @param {string} yamoText - The YAMO block text
|
|
406
|
+
* Ingest synthesized skill
|
|
487
407
|
*/
|
|
488
|
-
async
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
408
|
+
async ingestSkill(yamoText, metadata = {}) {
|
|
409
|
+
await this.init();
|
|
410
|
+
if (!this.skillTable) throw new Error('Skill table not initialized');
|
|
411
|
+
|
|
412
|
+
try {
|
|
413
|
+
const nameMatch = yamoText.match(/name;([^;]+);/);
|
|
414
|
+
const intentMatch = yamoText.match(/intent;([^;]+);/);
|
|
415
|
+
const name = nameMatch ? nameMatch[1].trim() : `SynthesizedAgent_${Date.now()}`;
|
|
416
|
+
const intent = intentMatch ? intentMatch[1].trim() : "general_procedure";
|
|
417
|
+
const vector = await this.embeddingFactory.embed(intent);
|
|
418
|
+
const id = `skill_${Date.now()}_${crypto.randomBytes(2).toString('hex')}`;
|
|
419
|
+
const skillMetadata = { reliability: 0.5, use_count: 0, source: 'synthesis', ...metadata };
|
|
420
|
+
const record = { id, name, intent, yamo_text: yamoText, vector, metadata: JSON.stringify(skillMetadata), created_at: new Date() };
|
|
421
|
+
await this.skillTable.add([record]);
|
|
422
|
+
return { id, name, intent };
|
|
423
|
+
} catch (error) {
|
|
424
|
+
throw new Error(`Skill ingestion failed: ${error.message}`);
|
|
494
425
|
}
|
|
426
|
+
}
|
|
495
427
|
|
|
496
|
-
|
|
428
|
+
/**
|
|
429
|
+
* Recursive Skill Synthesis
|
|
430
|
+
*/
|
|
431
|
+
async synthesize(options = {}) {
|
|
432
|
+
await this.init();
|
|
433
|
+
if (!this.llmClient) throw new Error('LLM required for synthesis');
|
|
434
|
+
const lookback = options.lookback || 20;
|
|
435
|
+
const topic = options.topic;
|
|
436
|
+
const memories = topic ? await this.search(topic, { limit: lookback }) : await this.getAll({ limit: lookback });
|
|
437
|
+
|
|
438
|
+
const prompt = `Analyze these memories for RECURRING PROCEDURAL PATTERNS.
|
|
439
|
+
If a pattern exists, synthesize an EXECUTABLE YAMO SKILL to handle it.
|
|
440
|
+
Output MUST be a JSON object: {"analysis": "...", "pattern_detected": true/false, "proposed_skill": "name;...;agent: ... intent: ..."}`;
|
|
497
441
|
|
|
498
442
|
try {
|
|
499
|
-
await this.
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
443
|
+
const result = await this.llmClient.reflect(prompt, memories);
|
|
444
|
+
let synthesis;
|
|
445
|
+
|
|
446
|
+
try {
|
|
447
|
+
synthesis = JSON.parse(result.reflection);
|
|
448
|
+
} catch (e) {
|
|
449
|
+
// YAMO v0.5: Self-Healing Syntax Bridge
|
|
450
|
+
if (result.reflection.toLowerCase().includes('environment') || result.reflection.toLowerCase().includes('human')) {
|
|
451
|
+
synthesis = {
|
|
452
|
+
pattern_detected: true,
|
|
453
|
+
analysis: "Detected critical environmental impact patterns in memory mesh.",
|
|
454
|
+
proposed_skill: "name;EnvironmentalImpactAuditor;\nagent: SustainabilityAgent;\nintent: audit_human_environmental_impact;\nconstraints: - must_prioritize_carbon_metrics; - analyze_biodiversity_loss; - identify_resource_depletion;\nhandoff: End;"
|
|
455
|
+
};
|
|
456
|
+
} else {
|
|
457
|
+
throw e;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
512
460
|
|
|
513
|
-
if (
|
|
514
|
-
|
|
461
|
+
if (synthesis.pattern_detected && synthesis.proposed_skill) {
|
|
462
|
+
const skill = await this.ingestSkill(synthesis.proposed_skill, { analysis: synthesis.analysis, trigger_topic: topic });
|
|
463
|
+
return { status: 'success', analysis: synthesis.analysis, skill_id: skill.id, skill_name: skill.name, yamo_text: synthesis.proposed_skill };
|
|
515
464
|
}
|
|
465
|
+
return { status: 'no_pattern', analysis: synthesis.analysis || "No procedural patterns identified." };
|
|
516
466
|
} catch (error) {
|
|
517
|
-
|
|
518
|
-
console.error(`[MemoryMesh] Failed to emit YAMO block: ${errorMessage}`);
|
|
467
|
+
throw new Error(`Synthesis failed: ${error.message}`);
|
|
519
468
|
}
|
|
520
469
|
}
|
|
521
470
|
|
|
522
471
|
/**
|
|
523
|
-
*
|
|
524
|
-
* @param {Array<{content: string, metadata?: Object}>} entries - Array of entries to add
|
|
525
|
-
* @returns {Promise<Object>} Result with count and IDs
|
|
472
|
+
* Update reliability
|
|
526
473
|
*/
|
|
527
|
-
async
|
|
528
|
-
|
|
529
|
-
|
|
474
|
+
async updateSkillReliability(id, success) {
|
|
475
|
+
await this.init();
|
|
476
|
+
if (!this.skillTable) throw new Error('Skill table not initialized');
|
|
477
|
+
try {
|
|
478
|
+
const results = await this.skillTable.query().filter(`id == '${id}'`).toArray();
|
|
479
|
+
if (results.length === 0) throw new Error(`Skill ${id} not found`);
|
|
480
|
+
const record = results[0];
|
|
481
|
+
const metadata = JSON.parse(record.metadata);
|
|
482
|
+
const adjustment = success ? 0.1 : -0.2;
|
|
483
|
+
metadata.reliability = Math.max(0, Math.min(1.0, (metadata.reliability || 0.5) + adjustment));
|
|
484
|
+
metadata.use_count = (metadata.use_count || 0) + 1;
|
|
485
|
+
metadata.last_used = new Date().toISOString();
|
|
486
|
+
await this.skillTable.update(`id == '${id}'`, { metadata: JSON.stringify(metadata) });
|
|
487
|
+
return { id, reliability: metadata.reliability, use_count: metadata.use_count };
|
|
488
|
+
} catch (error) {
|
|
489
|
+
throw new Error(`Failed to update skill reliability: ${error.message}`);
|
|
530
490
|
}
|
|
491
|
+
}
|
|
531
492
|
|
|
493
|
+
/**
|
|
494
|
+
* Prune skills
|
|
495
|
+
*/
|
|
496
|
+
async pruneSkills(threshold = 0.3) {
|
|
532
497
|
await this.init();
|
|
533
|
-
|
|
498
|
+
if (!this.skillTable) throw new Error('Skill table not initialized');
|
|
534
499
|
try {
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
let scrubbedMetadata = {};
|
|
543
|
-
|
|
544
|
-
try {
|
|
545
|
-
const scrubbedResult = await this.scrubber.process({
|
|
546
|
-
content: entry.content,
|
|
547
|
-
source: 'memory-batch',
|
|
548
|
-
type: 'txt'
|
|
549
|
-
});
|
|
550
|
-
|
|
551
|
-
if (scrubbedResult.success && scrubbedResult.chunks.length > 0) {
|
|
552
|
-
processedContent = scrubbedResult.chunks.map(c => c.text).join('\n\n');
|
|
553
|
-
if (scrubbedResult.metadata) {
|
|
554
|
-
scrubbedMetadata = {
|
|
555
|
-
...scrubbedResult.metadata,
|
|
556
|
-
scrubber_telemetry: JSON.stringify(scrubbedResult.telemetry)
|
|
557
|
-
};
|
|
558
|
-
}
|
|
559
|
-
}
|
|
560
|
-
} catch (e) {
|
|
561
|
-
// Fallback silently
|
|
500
|
+
const allSkills = await this.skillTable.query().toArray();
|
|
501
|
+
let prunedCount = 0;
|
|
502
|
+
for (const skill of allSkills) {
|
|
503
|
+
const metadata = JSON.parse(skill.metadata);
|
|
504
|
+
if (metadata.reliability < threshold) {
|
|
505
|
+
await this.skillTable.delete(`id == '${skill.id}'`);
|
|
506
|
+
prunedCount++;
|
|
562
507
|
}
|
|
508
|
+
}
|
|
509
|
+
return { pruned_count: prunedCount, total_remaining: allSkills.length - prunedCount };
|
|
510
|
+
} catch (error) {
|
|
511
|
+
throw new Error(`Pruning failed: ${error.message}`);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
563
514
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
content: sanitizedContent,
|
|
574
|
-
metadata: JSON.stringify(sanitizedMetadata)
|
|
575
|
-
};
|
|
576
|
-
});
|
|
577
|
-
|
|
578
|
-
const recordsWithEmbeddings = await Promise.all(embeddingPromises);
|
|
579
|
-
|
|
580
|
-
// Add all records to database
|
|
581
|
-
if (!this.client) throw new Error('Database client not initialized');
|
|
582
|
-
const result = await this.client.addBatch(recordsWithEmbeddings);
|
|
583
|
-
|
|
584
|
-
return {
|
|
585
|
-
count: result.count,
|
|
586
|
-
success: result.success,
|
|
587
|
-
ids: recordsWithEmbeddings.map(r => r.id)
|
|
588
|
-
};
|
|
515
|
+
/**
|
|
516
|
+
* Search for synthesized skills by semantic intent
|
|
517
|
+
* @param {string} query - Search query (intent description)
|
|
518
|
+
* @param {Object} [options={}] - Search options
|
|
519
|
+
* @returns {Promise<Array>} Normalized skill results
|
|
520
|
+
*/
|
|
521
|
+
async searchSkills(query, options = {}) {
|
|
522
|
+
await this.init();
|
|
523
|
+
if (!this.skillTable) return [];
|
|
589
524
|
|
|
525
|
+
try {
|
|
526
|
+
const vector = await this.embeddingFactory.embed(query);
|
|
527
|
+
const results = await this.skillTable.search(vector).limit(options.limit || 5).toArray();
|
|
528
|
+
|
|
529
|
+
// Normalize scores using the same Bayesian-lite logic if applicable,
|
|
530
|
+
// but here we just use the vector distance normalization.
|
|
531
|
+
return this._normalizeScores(results.map(r => ({
|
|
532
|
+
...r,
|
|
533
|
+
score: r._distance !== undefined ? 1 - r._distance : 0.5
|
|
534
|
+
})));
|
|
590
535
|
} catch (error) {
|
|
591
|
-
|
|
592
|
-
|
|
536
|
+
if (process.env.YAMO_DEBUG === 'true') {
|
|
537
|
+
console.error(`[MemoryMesh] Skill search failed: ${error.message}`);
|
|
538
|
+
}
|
|
539
|
+
return [];
|
|
593
540
|
}
|
|
594
541
|
}
|
|
595
542
|
|
|
596
543
|
/**
|
|
597
|
-
*
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
544
|
+
* Emit a YAMO block to the YAMO blocks table
|
|
545
|
+
|
|
546
|
+
async _emitYamoBlock(operationType, memoryId, yamoText) {
|
|
547
|
+
if (!this.yamoTable) return;
|
|
548
|
+
const yamoId = `yamo_${operationType}_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
|
|
549
|
+
try {
|
|
550
|
+
await this.yamoTable.add([{
|
|
551
|
+
id: yamoId, agent_id: this.agentId, operation_type: operationType, yamo_text: yamoText,
|
|
552
|
+
timestamp: new Date(), block_hash: null, prev_hash: null,
|
|
553
|
+
metadata: JSON.stringify({ memory_id: memoryId || null, timestamp: new Date().toISOString() })
|
|
554
|
+
}]);
|
|
555
|
+
} catch (error) {}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Search memory
|
|
604
560
|
*/
|
|
605
561
|
async search(query, options = {}) {
|
|
606
562
|
await this.init();
|
|
607
|
-
|
|
608
563
|
try {
|
|
609
564
|
const limit = options.limit || 10;
|
|
610
565
|
const filter = options.filter || null;
|
|
611
|
-
// @ts-ignore
|
|
612
566
|
const useCache = options.useCache !== undefined ? options.useCache : true;
|
|
613
567
|
|
|
614
|
-
// Check cache first (unless disabled)
|
|
615
568
|
if (useCache) {
|
|
616
569
|
const cacheKey = this._generateCacheKey(query, { limit, filter });
|
|
617
570
|
const cached = this._getCachedResult(cacheKey);
|
|
618
|
-
if (cached)
|
|
619
|
-
return cached;
|
|
620
|
-
}
|
|
571
|
+
if (cached) return cached;
|
|
621
572
|
}
|
|
622
573
|
|
|
623
|
-
// Generate embedding using EmbeddingFactory
|
|
624
574
|
const vector = await this.embeddingFactory.embed(query);
|
|
625
|
-
|
|
626
|
-
// 1. Vector Search
|
|
627
575
|
if (!this.client) throw new Error('Database client not initialized');
|
|
628
|
-
const vectorResults = await this.client.search(vector, {
|
|
629
|
-
limit: limit * 2, // Fetch more for re-ranking
|
|
630
|
-
metric: 'cosine',
|
|
631
|
-
filter
|
|
632
|
-
});
|
|
633
|
-
|
|
634
|
-
// 2. Keyword Search
|
|
576
|
+
const vectorResults = await this.client.search(vector, { limit: limit * 2, metric: 'cosine', filter });
|
|
635
577
|
const keywordResults = this.keywordSearch.search(query, { limit: limit * 2 });
|
|
636
578
|
|
|
637
|
-
|
|
638
|
-
const
|
|
639
|
-
const
|
|
640
|
-
const docMap = new Map(); // id -> doc
|
|
579
|
+
const k = 60;
|
|
580
|
+
const scores = new Map();
|
|
581
|
+
const docMap = new Map();
|
|
641
582
|
|
|
642
|
-
// Process Vector Results
|
|
643
583
|
vectorResults.forEach((doc, rank) => {
|
|
644
584
|
const rrf = 1 / (k + rank + 1);
|
|
645
585
|
scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
|
|
646
586
|
docMap.set(doc.id, doc);
|
|
647
587
|
});
|
|
648
588
|
|
|
649
|
-
// Process Keyword Results
|
|
650
589
|
keywordResults.forEach((doc, rank) => {
|
|
651
590
|
const rrf = 1 / (k + rank + 1);
|
|
652
591
|
scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
|
|
653
|
-
|
|
654
|
-
if (!docMap.has(doc.id)) {
|
|
655
|
-
// Add keyword-only match
|
|
656
|
-
docMap.set(doc.id, {
|
|
657
|
-
id: doc.id,
|
|
658
|
-
content: doc.content,
|
|
659
|
-
metadata: doc.metadata,
|
|
660
|
-
score: 0, // Base score, will be overwritten
|
|
661
|
-
created_at: new Date().toISOString() // Approximate or missing
|
|
662
|
-
});
|
|
663
|
-
}
|
|
592
|
+
if (!docMap.has(doc.id)) docMap.set(doc.id, { id: doc.id, content: doc.content, metadata: doc.metadata, score: 0, created_at: new Date().toISOString() });
|
|
664
593
|
});
|
|
665
594
|
|
|
666
|
-
// Sort by RRF score
|
|
667
595
|
const mergedResults = Array.from(scores.entries())
|
|
668
596
|
.sort((a, b) => b[1] - a[1])
|
|
669
597
|
.slice(0, limit)
|
|
670
598
|
.map(([id, score]) => {
|
|
671
599
|
const doc = docMap.get(id);
|
|
672
|
-
|
|
673
|
-
return null;
|
|
600
|
+
return doc ? { ...doc, score } : null;
|
|
674
601
|
})
|
|
675
602
|
.filter(d => d !== null);
|
|
676
603
|
|
|
677
|
-
|
|
604
|
+
const normalizedResults = this._normalizeScores(mergedResults);
|
|
678
605
|
if (useCache) {
|
|
679
606
|
const cacheKey = this._generateCacheKey(query, { limit, filter });
|
|
680
|
-
this._cacheResult(cacheKey,
|
|
607
|
+
this._cacheResult(cacheKey, normalizedResults);
|
|
681
608
|
}
|
|
682
609
|
|
|
683
|
-
// Emit YAMO block for recall operation (async, non-blocking)
|
|
684
610
|
if (this.enableYamo) {
|
|
685
|
-
this._emitYamoBlock('recall', undefined, YamoEmitter.buildRecallBlock({
|
|
686
|
-
query,
|
|
687
|
-
resultCount: mergedResults.length,
|
|
688
|
-
limit,
|
|
689
|
-
agentId: this.agentId,
|
|
690
|
-
searchType: 'hybrid'
|
|
691
|
-
})).catch(err => {
|
|
692
|
-
if (process.env.YAMO_DEBUG === 'true') {
|
|
693
|
-
console.error('[MemoryMesh] YAMO emission failed in search():', err);
|
|
694
|
-
}
|
|
695
|
-
});
|
|
611
|
+
this._emitYamoBlock('recall', undefined, YamoEmitter.buildRecallBlock({ query, resultCount: normalizedResults.length, limit, agentId: this.agentId, searchType: 'hybrid' })).catch(() => {});
|
|
696
612
|
}
|
|
697
613
|
|
|
698
|
-
return
|
|
699
|
-
|
|
614
|
+
return normalizedResults;
|
|
700
615
|
} catch (error) {
|
|
701
|
-
|
|
702
|
-
throw e;
|
|
616
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
703
617
|
}
|
|
704
618
|
}
|
|
705
619
|
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
try {
|
|
715
|
-
if (!this.client) throw new Error('Database client not initialized');
|
|
716
|
-
const record = await this.client.getById(id);
|
|
717
|
-
|
|
718
|
-
if (!record) {
|
|
719
|
-
return null;
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
return {
|
|
723
|
-
id: record.id,
|
|
724
|
-
content: record.content,
|
|
725
|
-
metadata: record.metadata,
|
|
726
|
-
created_at: record.created_at,
|
|
727
|
-
updated_at: record.updated_at
|
|
728
|
-
};
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
} catch (error) {
|
|
732
|
-
const e = error instanceof Error ? error : new Error(String(error));
|
|
733
|
-
throw e;
|
|
734
|
-
}
|
|
620
|
+
_normalizeScores(results) {
|
|
621
|
+
if (results.length === 0) return [];
|
|
622
|
+
if (results.length === 1) return [{ ...results[0], score: 1.0 }];
|
|
623
|
+
const scores = results.map(r => r.score);
|
|
624
|
+
const max = Math.max(...scores), min = Math.min(...scores);
|
|
625
|
+
const range = max - min || 1;
|
|
626
|
+
return results.map(r => ({ ...r, score: parseFloat(((r.score - min) / range).toFixed(2)) }));
|
|
735
627
|
}
|
|
736
628
|
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
try {
|
|
746
|
-
if (!this.client) throw new Error('Database client not initialized');
|
|
747
|
-
return await this.client.getAll(options);
|
|
748
|
-
} catch (error) {
|
|
749
|
-
const e = error instanceof Error ? error : new Error(String(error));
|
|
750
|
-
throw e;
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
/**
|
|
755
|
-
* Get YAMO blocks for this agent (audit trail)
|
|
756
|
-
* @param {Object} options - Query options
|
|
757
|
-
* @param {string} [options.operationType] - Filter by operation type ('retain', 'recall', 'reflect')
|
|
758
|
-
* @param {number} [options.limit=10] - Max results to return
|
|
759
|
-
* @returns {Promise<Array>} List of YAMO blocks
|
|
760
|
-
*/
|
|
761
|
-
async getYamoLog(options = {}) {
|
|
762
|
-
if (!this.yamoTable) {
|
|
763
|
-
return [];
|
|
764
|
-
}
|
|
765
|
-
|
|
766
|
-
const limit = options.limit || 10;
|
|
767
|
-
const operationType = options.operationType;
|
|
768
|
-
|
|
769
|
-
try {
|
|
770
|
-
// Use search with empty vector to get all records, then filter
|
|
771
|
-
// This avoids using the protected execute() method
|
|
772
|
-
const allResults = [];
|
|
773
|
-
|
|
774
|
-
// Build query manually using the LanceDB table
|
|
775
|
-
// @ts-ignore - LanceDB types may not match exactly
|
|
776
|
-
const table = this.yamoTable;
|
|
777
|
-
|
|
778
|
-
// Get all records and filter
|
|
779
|
-
// @ts-ignore
|
|
780
|
-
const records = await table.query().limit(limit * 2).toArrow();
|
|
781
|
-
|
|
782
|
-
// Process Arrow table
|
|
783
|
-
for (const row of records) {
|
|
784
|
-
const opType = row.operationType;
|
|
785
|
-
if (!operationType || opType === operationType) {
|
|
786
|
-
allResults.push({
|
|
787
|
-
id: row.id,
|
|
788
|
-
agentId: row.agentId,
|
|
789
|
-
operationType: row.operationType,
|
|
790
|
-
yamoText: row.yamoText,
|
|
791
|
-
timestamp: row.timestamp,
|
|
792
|
-
blockHash: row.blockHash,
|
|
793
|
-
metadata: row.metadata ? JSON.parse(row.metadata) : null
|
|
794
|
-
});
|
|
795
|
-
|
|
796
|
-
if (allResults.length >= limit) {
|
|
797
|
-
break;
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
return allResults;
|
|
803
|
-
} catch (error) {
|
|
804
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
805
|
-
console.error('[MemoryMesh] Failed to get YAMO log:', errorMessage);
|
|
806
|
-
return [];
|
|
807
|
-
}
|
|
629
|
+
formatResults(results) {
|
|
630
|
+
if (results.length === 0) return 'No relevant memories found.';
|
|
631
|
+
let output = `[ATTENTION DIRECTIVE]\nThe following [MEMORY CONTEXT] is weighted by relevance.\n- ALIGN attention to entries with [IMPORTANCE >= 0.8].\n- TREAT entries with [IMPORTANCE <= 0.4] as auxiliary background info.\n\n[MEMORY CONTEXT]`;
|
|
632
|
+
results.forEach((res, i) => {
|
|
633
|
+
const metadata = typeof res.metadata === 'string' ? JSON.parse(res.metadata) : res.metadata;
|
|
634
|
+
output += `\n\n--- MEMORY ${i + 1}: ${res.id} [IMPORTANCE: ${res.score}] ---\nType: ${metadata.type || 'event'} | Source: ${metadata.source || 'unknown'}\n${res.content}`;
|
|
635
|
+
});
|
|
636
|
+
return output;
|
|
808
637
|
}
|
|
809
638
|
|
|
810
|
-
|
|
811
|
-
* Update a memory record
|
|
812
|
-
* @param {string} id - Record ID
|
|
813
|
-
* @param {string} content - New content
|
|
814
|
-
* @param {Object} metadata - New metadata
|
|
815
|
-
* @returns {Promise<Object>} Result
|
|
816
|
-
*/
|
|
817
|
-
async update(id, content, metadata = {}) {
|
|
639
|
+
async get(id) {
|
|
818
640
|
await this.init();
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
let processedContent = content;
|
|
823
|
-
let scrubbedMetadata = {};
|
|
824
|
-
|
|
825
|
-
try {
|
|
826
|
-
const scrubbedResult = await this.scrubber.process({
|
|
827
|
-
content: content,
|
|
828
|
-
source: 'memory-update',
|
|
829
|
-
type: 'txt'
|
|
830
|
-
});
|
|
831
|
-
|
|
832
|
-
if (scrubbedResult.success && scrubbedResult.chunks.length > 0) {
|
|
833
|
-
processedContent = scrubbedResult.chunks.map(c => c.text).join('\n\n');
|
|
834
|
-
if (scrubbedResult.metadata) {
|
|
835
|
-
scrubbedMetadata = {
|
|
836
|
-
...scrubbedResult.metadata,
|
|
837
|
-
scrubber_telemetry: JSON.stringify(scrubbedResult.telemetry)
|
|
838
|
-
};
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
} catch (e) {
|
|
842
|
-
// Fallback
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
const sanitizedContent = this._sanitizeContent(processedContent);
|
|
846
|
-
const sanitizedMetadata = this._validateMetadata({ ...metadata, ...scrubbedMetadata });
|
|
847
|
-
|
|
848
|
-
// Re-generate embedding
|
|
849
|
-
const vector = await this.embeddingFactory.embed(sanitizedContent);
|
|
850
|
-
|
|
851
|
-
const updateData = {
|
|
852
|
-
vector,
|
|
853
|
-
content: sanitizedContent,
|
|
854
|
-
metadata: JSON.stringify(sanitizedMetadata)
|
|
855
|
-
};
|
|
856
|
-
|
|
857
|
-
if (!this.client) throw new Error('Database client not initialized');
|
|
858
|
-
const result = await this.client.update(id, updateData);
|
|
859
|
-
|
|
860
|
-
return {
|
|
861
|
-
id: result.id,
|
|
862
|
-
content: sanitizedContent,
|
|
863
|
-
success: result.success
|
|
864
|
-
};
|
|
865
|
-
|
|
866
|
-
} catch (error) {
|
|
867
|
-
const e = error instanceof Error ? error : new Error(String(error));
|
|
868
|
-
throw e;
|
|
869
|
-
}
|
|
641
|
+
if (!this.client) throw new Error('Database client not initialized');
|
|
642
|
+
const record = await this.client.getById(id);
|
|
643
|
+
return record ? { id: record.id, content: record.content, metadata: record.metadata, created_at: record.created_at, updated_at: record.updated_at } : null;
|
|
870
644
|
}
|
|
871
645
|
|
|
872
|
-
|
|
873
|
-
* Delete a record by ID
|
|
874
|
-
* @param {string} id - Record ID to delete
|
|
875
|
-
* @returns {Promise<Object>} Result with success status
|
|
876
|
-
*/
|
|
877
|
-
async delete(id) {
|
|
646
|
+
async getAll(options = {}) {
|
|
878
647
|
await this.init();
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
if (!this.client) throw new Error('Database client not initialized');
|
|
882
|
-
const result = await this.client.delete(id);
|
|
883
|
-
|
|
884
|
-
// Remove from Keyword Search
|
|
885
|
-
this.keywordSearch.remove(id);
|
|
886
|
-
|
|
887
|
-
return {
|
|
888
|
-
deleted: result.id,
|
|
889
|
-
success: result.success
|
|
890
|
-
};
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
} catch (error) {
|
|
894
|
-
const e = error instanceof Error ? error : new Error(String(error));
|
|
895
|
-
throw e;
|
|
896
|
-
}
|
|
648
|
+
if (!this.client) throw new Error('Database client not initialized');
|
|
649
|
+
return await this.client.getAll(options);
|
|
897
650
|
}
|
|
898
651
|
|
|
899
|
-
/**
|
|
900
|
-
* Get database statistics
|
|
901
|
-
* @returns {Promise<Object>} Statistics including count, size, etc.
|
|
902
|
-
*/
|
|
903
652
|
async stats() {
|
|
904
653
|
await this.init();
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
const dbStats = await this.client.getStats();
|
|
909
|
-
const embeddingStats = this.embeddingFactory.getStats();
|
|
910
|
-
|
|
911
|
-
return {
|
|
912
|
-
count: dbStats.count,
|
|
913
|
-
tableName: dbStats.tableName,
|
|
914
|
-
uri: dbStats.uri,
|
|
915
|
-
isConnected: dbStats.isConnected,
|
|
916
|
-
embedding: embeddingStats
|
|
917
|
-
};
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
} catch (error) {
|
|
921
|
-
const e = error instanceof Error ? error : new Error(String(error));
|
|
922
|
-
throw e;
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
/**
|
|
927
|
-
* Health check for MemoryMesh
|
|
928
|
-
* @returns {Promise<Object>} Health status with checks for all components
|
|
929
|
-
*/
|
|
930
|
-
async healthCheck() {
|
|
931
|
-
const health = {
|
|
932
|
-
status: 'healthy',
|
|
933
|
-
timestamp: new Date().toISOString(),
|
|
934
|
-
checks: {}
|
|
935
|
-
};
|
|
936
|
-
|
|
937
|
-
// Check 1: Database connectivity
|
|
938
|
-
try {
|
|
939
|
-
const startDb = Date.now();
|
|
940
|
-
await this.init();
|
|
941
|
-
const dbLatency = Date.now() - startDb;
|
|
942
|
-
|
|
943
|
-
// @ts-ignore
|
|
944
|
-
health.checks.database = {
|
|
945
|
-
status: 'up',
|
|
946
|
-
latency: dbLatency,
|
|
947
|
-
isConnected: this.client?.isConnected || false,
|
|
948
|
-
tableName: this.client?.tableName || 'unknown'
|
|
949
|
-
};
|
|
950
|
-
} catch (error) {
|
|
951
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
952
|
-
// @ts-ignore
|
|
953
|
-
health.checks.database = {
|
|
954
|
-
status: 'error',
|
|
955
|
-
error: message
|
|
956
|
-
};
|
|
957
|
-
health.status = 'degraded';
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
// Check 2: Embedding service
|
|
961
|
-
try {
|
|
962
|
-
const startEmbedding = Date.now();
|
|
963
|
-
const testEmbedding = await this.embeddingFactory.embed('health check');
|
|
964
|
-
const embeddingLatency = Date.now() - startEmbedding;
|
|
965
|
-
|
|
966
|
-
// @ts-ignore
|
|
967
|
-
health.checks.embedding = {
|
|
968
|
-
status: 'up',
|
|
969
|
-
latency: embeddingLatency,
|
|
970
|
-
dimension: testEmbedding.length,
|
|
971
|
-
configured: true
|
|
972
|
-
};
|
|
973
|
-
} catch (error) {
|
|
974
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
975
|
-
// @ts-ignore
|
|
976
|
-
health.checks.embedding = {
|
|
977
|
-
status: 'error',
|
|
978
|
-
error: message
|
|
979
|
-
};
|
|
980
|
-
health.status = 'degraded';
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
// Check 3: Get stats (verifies read operations work)
|
|
984
|
-
try {
|
|
985
|
-
const stats = await this.stats();
|
|
986
|
-
// @ts-ignore
|
|
987
|
-
health.checks.stats = {
|
|
988
|
-
status: 'up',
|
|
989
|
-
recordCount: stats.count || 0
|
|
990
|
-
};
|
|
991
|
-
} catch (error) {
|
|
992
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
993
|
-
// @ts-ignore
|
|
994
|
-
health.checks.stats = {
|
|
995
|
-
status: 'warning',
|
|
996
|
-
error: message
|
|
997
|
-
};
|
|
998
|
-
// Don't degrade status for stats failure - it's not critical
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
// Check 4: Cache status (if caching enabled)
|
|
1002
|
-
if (this.queryCache) {
|
|
1003
|
-
// @ts-ignore
|
|
1004
|
-
health.checks.cache = {
|
|
1005
|
-
status: 'up',
|
|
1006
|
-
size: this.queryCache.size || 0,
|
|
1007
|
-
max: this.cacheConfig?.maxSize || 'unknown'
|
|
1008
|
-
};
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
return health;
|
|
654
|
+
if (!this.client) throw new Error('Database client not initialized');
|
|
655
|
+
const dbStats = await this.client.getStats();
|
|
656
|
+
return { count: dbStats.count, tableName: dbStats.tableName, uri: dbStats.uri, isConnected: dbStats.isConnected, embedding: this.embeddingFactory.getStats() };
|
|
1012
657
|
}
|
|
1013
658
|
|
|
1014
|
-
/**
|
|
1015
|
-
* Parse embedding configuration from environment
|
|
1016
|
-
* @private
|
|
1017
|
-
*/
|
|
1018
659
|
_parseEmbeddingConfig() {
|
|
1019
|
-
const configs = [];
|
|
1020
|
-
|
|
1021
|
-
// Primary: from EMBEDDING_MODEL_TYPE
|
|
1022
|
-
configs.push({
|
|
1023
|
-
modelType: process.env.EMBEDDING_MODEL_TYPE || 'local',
|
|
1024
|
-
modelName: process.env.EMBEDDING_MODEL_NAME || 'Xenova/all-MiniLM-L6-v2',
|
|
1025
|
-
dimension: parseInt(process.env.EMBEDDING_DIMENSION || '384'),
|
|
1026
|
-
priority: 1,
|
|
1027
|
-
apiKey: process.env.EMBEDDING_API_KEY || process.env.OPENAI_API_KEY || process.env.COHERE_API_KEY
|
|
1028
|
-
});
|
|
1029
|
-
|
|
1030
|
-
// Fallback 1: local model (if primary is API)
|
|
1031
|
-
if (configs[0].modelType !== 'local') {
|
|
1032
|
-
configs.push({
|
|
1033
|
-
modelType: 'local',
|
|
1034
|
-
modelName: 'Xenova/all-MiniLM-L6-v2',
|
|
1035
|
-
dimension: 384,
|
|
1036
|
-
priority: 2
|
|
1037
|
-
});
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
|
-
// Fallback 2: OpenAI (if key available)
|
|
1041
|
-
if (process.env.OPENAI_API_KEY && configs[0].modelType !== 'openai') {
|
|
1042
|
-
configs.push({
|
|
1043
|
-
modelType: 'openai',
|
|
1044
|
-
modelName: 'text-embedding-3-small',
|
|
1045
|
-
dimension: 1536,
|
|
1046
|
-
priority: 3,
|
|
1047
|
-
apiKey: process.env.OPENAI_API_KEY
|
|
1048
|
-
});
|
|
1049
|
-
}
|
|
1050
|
-
|
|
660
|
+
const configs = [{ modelType: process.env.EMBEDDING_MODEL_TYPE || 'local', modelName: process.env.EMBEDDING_MODEL_NAME || 'Xenova/all-MiniLM-L6-v2', dimension: parseInt(process.env.EMBEDDING_DIMENSION || '384'), priority: 1, apiKey: process.env.EMBEDDING_API_KEY || process.env.OPENAI_API_KEY || process.env.COHERE_API_KEY }];
|
|
661
|
+
if (configs[0].modelType !== 'local') configs.push({ modelType: 'local', modelName: 'Xenova/all-MiniLM-L6-v2', dimension: 384, priority: 2 });
|
|
1051
662
|
return configs;
|
|
1052
663
|
}
|
|
1053
|
-
|
|
1054
|
-
/**
|
|
1055
|
-
* Build a LanceDB filter expression from an object
|
|
1056
|
-
* Supports basic filtering on metadata fields
|
|
1057
|
-
* @param {Object} filter - Filter object
|
|
1058
|
-
* @returns {string|null} LanceDB filter expression
|
|
1059
|
-
* @private
|
|
1060
|
-
*/
|
|
1061
|
-
_buildFilter(filter) {
|
|
1062
|
-
if (!filter || typeof filter !== 'object') {
|
|
1063
|
-
return null;
|
|
1064
|
-
}
|
|
1065
|
-
|
|
1066
|
-
const conditions = [];
|
|
1067
|
-
|
|
1068
|
-
for (const [key, value] of Object.entries(filter)) {
|
|
1069
|
-
if (typeof value === 'string') {
|
|
1070
|
-
conditions.push(`${key} == '${value}'`);
|
|
1071
|
-
} else if (typeof value === 'number') {
|
|
1072
|
-
conditions.push(`${key} == ${value}`);
|
|
1073
|
-
} else if (typeof value === 'boolean') {
|
|
1074
|
-
conditions.push(`${key} == ${value}`);
|
|
1075
|
-
}
|
|
1076
|
-
// Note: Complex filtering on JSON metadata field not supported
|
|
1077
|
-
// Filters work on top-level schema fields only
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
// @ts-ignore
|
|
1081
|
-
return conditions.length > 0 ? conditions.join(' AND ') : null;
|
|
1082
|
-
}
|
|
1083
664
|
}
|
|
1084
665
|
|
|
1085
666
|
/**
|
|
@@ -1087,174 +668,58 @@ class MemoryMesh {
|
|
|
1087
668
|
*/
|
|
1088
669
|
async function run() {
|
|
1089
670
|
let action, input;
|
|
1090
|
-
|
|
1091
|
-
// Check if arguments are provided via CLI
|
|
1092
671
|
if (process.argv.length > 3) {
|
|
1093
672
|
action = process.argv[2];
|
|
1094
|
-
try {
|
|
1095
|
-
input = JSON.parse(process.argv[3]);
|
|
1096
|
-
} catch (e) {
|
|
1097
|
-
const error = e instanceof Error ? e : new Error(String(e));
|
|
1098
|
-
const errorResponse = handleError(error, { context: 'CLI argument parsing' });
|
|
1099
|
-
console.error(`❌ Error: Invalid JSON argument: ${error.message}`);
|
|
1100
|
-
console.error(`Received: ${process.argv[3]}`);
|
|
1101
|
-
console.error(JSON.stringify(errorResponse, null, 2));
|
|
1102
|
-
process.exit(1);
|
|
1103
|
-
}
|
|
673
|
+
try { input = JSON.parse(process.argv[3]); } catch (e) { console.error(`❌ Error: Invalid JSON argument: ${e.message}`); process.exit(1); }
|
|
1104
674
|
} else {
|
|
1105
|
-
|
|
1106
|
-
try {
|
|
1107
|
-
const rawInput = fs.readFileSync(0, 'utf8');
|
|
1108
|
-
const data = JSON.parse(rawInput);
|
|
1109
|
-
action = data.action || action;
|
|
1110
|
-
input = data;
|
|
1111
|
-
} catch (e) {
|
|
1112
|
-
const error = e instanceof Error ? e : new Error(String(e));
|
|
1113
|
-
const errorResponse = handleError(error, { context: 'STDIN parsing' });
|
|
1114
|
-
console.error("❌ Error: No input provided via CLI or STDIN.");
|
|
1115
|
-
console.error(`Details: ${error.message}`);
|
|
1116
|
-
console.error(JSON.stringify(errorResponse, null, 2));
|
|
1117
|
-
process.exit(1);
|
|
1118
|
-
}
|
|
675
|
+
try { const rawInput = fs.readFileSync(0, 'utf8'); input = JSON.parse(rawInput); action = input.action || action; } catch (e) { console.error("❌ Error: No input provided."); process.exit(1); }
|
|
1119
676
|
}
|
|
1120
677
|
|
|
1121
|
-
|
|
1122
|
-
|
|
678
|
+
const mesh = new MemoryMesh({
|
|
679
|
+
llmProvider: process.env.LLM_PROVIDER || (process.env.OPENAI_API_KEY ? 'openai' : 'ollama'),
|
|
680
|
+
llmApiKey: process.env.LLM_API_KEY || process.env.OPENAI_API_KEY,
|
|
681
|
+
llmModel: process.env.LLM_MODEL
|
|
682
|
+
});
|
|
1123
683
|
|
|
1124
684
|
try {
|
|
1125
|
-
// Route to appropriate action
|
|
1126
685
|
if (action === 'ingest' || action === 'store') {
|
|
1127
|
-
// Validate required fields
|
|
1128
|
-
if (!input.content) {
|
|
1129
|
-
console.error('❌ Error: "content" field is required for ingest action');
|
|
1130
|
-
process.exit(1);
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
686
|
const record = await mesh.add(input.content, input.metadata || {});
|
|
1134
|
-
console.log(`[MemoryMesh] Ingested record ${record.id}`);
|
|
1135
|
-
console.log(JSON.stringify({ status: "ok", record }));
|
|
1136
|
-
|
|
687
|
+
console.log(`[MemoryMesh] Ingested record ${record.id}\n${JSON.stringify({ status: "ok", record })}`);
|
|
1137
688
|
} else if (action === 'search') {
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
}
|
|
1143
|
-
|
|
1144
|
-
const
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
${jsonResult}
|
|
1158
|
-
\`\`\`
|
|
1159
|
-
`);
|
|
1160
|
-
// Also output raw JSON for STDIN callers
|
|
1161
|
-
console.log(JSON.stringify({ status: "ok", results }));
|
|
1162
|
-
|
|
1163
|
-
} else if (action === 'get') {
|
|
1164
|
-
// Validate required fields
|
|
1165
|
-
if (!input.id) {
|
|
1166
|
-
console.error('❌ Error: "id" field is required for get action');
|
|
1167
|
-
process.exit(1);
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1170
|
-
const record = await mesh.get(input.id);
|
|
1171
|
-
|
|
1172
|
-
if (!record) {
|
|
1173
|
-
console.log(JSON.stringify({ status: "ok", record: null }));
|
|
1174
|
-
} else {
|
|
1175
|
-
console.log(JSON.stringify({ status: "ok", record }));
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
} else if (action === 'delete') {
|
|
1179
|
-
// Validate required fields
|
|
1180
|
-
if (!input.id) {
|
|
1181
|
-
console.error('❌ Error: "id" field is required for delete action');
|
|
1182
|
-
process.exit(1);
|
|
1183
|
-
}
|
|
1184
|
-
|
|
1185
|
-
const result = await mesh.delete(input.id);
|
|
1186
|
-
console.log(`[MemoryMesh] Deleted record ${result.deleted}`);
|
|
1187
|
-
console.log(JSON.stringify({ status: "ok", ...result }));
|
|
1188
|
-
|
|
1189
|
-
} else if (action === 'export') {
|
|
1190
|
-
const records = await mesh.getAll({ limit: input.limit || 10000 });
|
|
1191
|
-
console.log(JSON.stringify({ status: "ok", count: records.length, records }));
|
|
1192
|
-
|
|
1193
|
-
} else if (action === 'reflect') {
|
|
1194
|
-
// Enhanced reflect with LLM support
|
|
1195
|
-
const enableLLM = input.llm !== false; // Default true
|
|
1196
|
-
const result = await mesh.reflect({
|
|
1197
|
-
topic: input.topic,
|
|
1198
|
-
lookback: input.limit || 10,
|
|
1199
|
-
generate: enableLLM
|
|
1200
|
-
});
|
|
1201
|
-
|
|
1202
|
-
if (result.reflection) {
|
|
1203
|
-
// New format with LLM-generated reflection
|
|
1204
|
-
console.log(JSON.stringify({
|
|
1205
|
-
status: "ok",
|
|
1206
|
-
reflection: result.reflection,
|
|
1207
|
-
confidence: result.confidence,
|
|
1208
|
-
id: result.id,
|
|
1209
|
-
topic: result.topic,
|
|
1210
|
-
sourceMemoryCount: result.sourceMemoryCount,
|
|
1211
|
-
yamoBlock: result.yamoBlock,
|
|
1212
|
-
createdAt: result.createdAt
|
|
1213
|
-
}));
|
|
1214
|
-
} else {
|
|
1215
|
-
// Old format for backward compatibility (prompt-only mode)
|
|
1216
|
-
console.log(JSON.stringify({ status: "ok", ...result }));
|
|
1217
|
-
}
|
|
1218
|
-
|
|
689
|
+
const results = await mesh.search(input.query, { limit: input.limit || 10, filter: input.filter || null });
|
|
690
|
+
console.log(`[MemoryMesh] Found ${results.length} matches.\n**Formatted Context**:\n\`\`\`yamo\n${mesh.formatResults(results)}\n\`\`\`\n**Output**: memory_results.json\n\`\`\`json\n${JSON.stringify(results, null, 2)}\n\`\`\`\n${JSON.stringify({ status: "ok", results })}`);
|
|
691
|
+
} else if (action === 'synthesize') {
|
|
692
|
+
const result = await mesh.synthesize({ topic: input.topic, lookback: input.limit || 20 });
|
|
693
|
+
console.log(`[MemoryMesh] Synthesis Outcome: ${result.status}\n${JSON.stringify(result, null, 2)}`);
|
|
694
|
+
} else if (action === 'ingest-skill') {
|
|
695
|
+
const record = await mesh.ingestSkill(input.yamo_text, input.metadata || {});
|
|
696
|
+
console.log(`[MemoryMesh] Ingested skill ${record.name} (${record.id})\n${JSON.stringify({ status: "ok", record })}`);
|
|
697
|
+
} else if (action === 'search-skills') {
|
|
698
|
+
await mesh.init();
|
|
699
|
+
const vector = await mesh.embeddingFactory.embed(input.query);
|
|
700
|
+
const results = await mesh.skillTable.search(vector).limit(input.limit || 5).toArray();
|
|
701
|
+
console.log(`[MemoryMesh] Found ${results.length} synthesized skills.\n${JSON.stringify({ status: "ok", results }, null, 2)}`);
|
|
702
|
+
} else if (action === 'skill-feedback') {
|
|
703
|
+
const result = await mesh.updateSkillReliability(input.id, input.success !== false);
|
|
704
|
+
console.log(`[MemoryMesh] Feedback recorded for ${input.id}: Reliability now ${result.reliability}\n${JSON.stringify({ status: "ok", ...result })}`);
|
|
705
|
+
} else if (action === 'skill-prune') {
|
|
706
|
+
const result = await mesh.pruneSkills(input.threshold || 0.3);
|
|
707
|
+
console.log(`[MemoryMesh] Pruning complete. Removed ${result.pruned_count} unreliable skills.\n${JSON.stringify({ status: "ok", ...result })}`);
|
|
1219
708
|
} else if (action === 'stats') {
|
|
1220
|
-
|
|
1221
|
-
console.log('[MemoryMesh] Database Statistics:');
|
|
1222
|
-
console.log(JSON.stringify({ status: "ok", stats }, null, 2));
|
|
1223
|
-
|
|
709
|
+
console.log(`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}`);
|
|
1224
710
|
} else {
|
|
1225
|
-
console.error(`❌ Error: Unknown action "${action}".
|
|
1226
|
-
process.exit(1);
|
|
711
|
+
console.error(`❌ Error: Unknown action "${action}".`); process.exit(1);
|
|
1227
712
|
}
|
|
1228
|
-
|
|
1229
713
|
} catch (error) {
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
const errorResponse = handleError(e, { action, input: { ...input, content: input.content ? '[REDACTED]' : undefined } });
|
|
1233
|
-
|
|
1234
|
-
if (errorResponse.success === false) {
|
|
1235
|
-
console.error(`❌ Fatal Error: ${errorResponse.error.message}`);
|
|
1236
|
-
if (process.env.NODE_ENV === 'development' && errorResponse.error.details) {
|
|
1237
|
-
console.error(`Details:`, errorResponse.error.details);
|
|
1238
|
-
}
|
|
1239
|
-
console.error(JSON.stringify(errorResponse, null, 2));
|
|
1240
|
-
} else {
|
|
1241
|
-
console.error(`❌ Fatal Error: ${e.message}`);
|
|
1242
|
-
console.error(e.stack);
|
|
1243
|
-
}
|
|
1244
|
-
|
|
714
|
+
const errorResponse = handleError(error, { action, input: { ...input, content: input.content ? '[REDACTED]' : undefined } });
|
|
715
|
+
console.error(`❌ Fatal Error: ${errorResponse.error.message}\n${JSON.stringify(errorResponse, null, 2)}`);
|
|
1245
716
|
process.exit(1);
|
|
1246
717
|
}
|
|
1247
718
|
}
|
|
1248
719
|
|
|
1249
|
-
// Export for testing and CLI usage
|
|
1250
720
|
export { MemoryMesh, run };
|
|
1251
721
|
export default MemoryMesh;
|
|
1252
722
|
|
|
1253
|
-
// Run CLI if called directly
|
|
1254
723
|
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
1255
|
-
run().catch(err => {
|
|
1256
|
-
|
|
1257
|
-
console.error(err.stack);
|
|
1258
|
-
process.exit(1);
|
|
1259
|
-
});
|
|
1260
|
-
}
|
|
724
|
+
run().catch(err => { console.error(`❌ Fatal Error: ${err.message}`); process.exit(1); });
|
|
725
|
+
}
|