@yamo/memory-mesh 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/bin/memory_mesh.js +95 -8
- package/lib/llm/client.d.ts +23 -48
- package/lib/llm/client.js +1 -0
- package/lib/llm/client.ts +298 -377
- package/lib/llm/index.js +1 -0
- package/lib/llm/index.ts +1 -2
- package/lib/memory/adapters/client.d.ts +22 -85
- package/lib/memory/adapters/client.js +1 -0
- package/lib/memory/adapters/client.ts +474 -633
- package/lib/memory/adapters/config.d.ts +82 -89
- package/lib/memory/adapters/config.js +1 -0
- package/lib/memory/adapters/config.ts +156 -225
- package/lib/memory/adapters/errors.d.ts +28 -20
- package/lib/memory/adapters/errors.js +1 -0
- package/lib/memory/adapters/errors.ts +83 -120
- package/lib/memory/context-manager.d.ts +15 -18
- package/lib/memory/context-manager.js +1 -0
- package/lib/memory/context-manager.ts +314 -401
- package/lib/memory/embeddings/factory.d.ts +18 -20
- package/lib/memory/embeddings/factory.js +1 -0
- package/lib/memory/embeddings/factory.ts +130 -173
- package/lib/memory/embeddings/index.js +1 -0
- package/lib/memory/embeddings/index.ts +1 -0
- package/lib/memory/embeddings/service.d.ts +36 -66
- package/lib/memory/embeddings/service.js +1 -0
- package/lib/memory/embeddings/service.ts +479 -616
- package/lib/memory/index.d.ts +2 -2
- package/lib/memory/index.js +1 -0
- package/lib/memory/index.ts +3 -13
- package/lib/memory/memory-mesh.d.ts +151 -93
- package/lib/memory/memory-mesh.js +1 -0
- package/lib/memory/memory-mesh.ts +1406 -1692
- package/lib/memory/memory-translator.d.ts +1 -6
- package/lib/memory/memory-translator.js +1 -0
- package/lib/memory/memory-translator.ts +96 -128
- package/lib/memory/schema.d.ts +29 -10
- package/lib/memory/schema.js +1 -0
- package/lib/memory/schema.ts +102 -185
- package/lib/memory/scorer.d.ts +3 -4
- package/lib/memory/scorer.js +1 -0
- package/lib/memory/scorer.ts +69 -86
- package/lib/memory/search/index.js +1 -0
- package/lib/memory/search/index.ts +1 -0
- package/lib/memory/search/keyword-search.d.ts +10 -26
- package/lib/memory/search/keyword-search.js +1 -0
- package/lib/memory/search/keyword-search.ts +123 -161
- package/lib/scrubber/config/defaults.d.ts +39 -46
- package/lib/scrubber/config/defaults.js +1 -0
- package/lib/scrubber/config/defaults.ts +50 -112
- package/lib/scrubber/errors/scrubber-error.d.ts +22 -0
- package/lib/scrubber/errors/scrubber-error.js +39 -0
- package/lib/scrubber/errors/scrubber-error.ts +44 -0
- package/lib/scrubber/index.d.ts +0 -1
- package/lib/scrubber/index.js +1 -0
- package/lib/scrubber/index.ts +1 -2
- package/lib/scrubber/scrubber.d.ts +14 -31
- package/lib/scrubber/scrubber.js +1 -0
- package/lib/scrubber/scrubber.ts +93 -152
- package/lib/scrubber/stages/chunker.d.ts +22 -10
- package/lib/scrubber/stages/chunker.js +86 -0
- package/lib/scrubber/stages/chunker.ts +104 -0
- package/lib/scrubber/stages/metadata-annotator.d.ts +14 -15
- package/lib/scrubber/stages/metadata-annotator.js +64 -0
- package/lib/scrubber/stages/metadata-annotator.ts +75 -0
- package/lib/scrubber/stages/normalizer.d.ts +13 -10
- package/lib/scrubber/stages/normalizer.js +51 -0
- package/lib/scrubber/stages/normalizer.ts +60 -0
- package/lib/scrubber/stages/semantic-filter.d.ts +13 -10
- package/lib/scrubber/stages/semantic-filter.js +51 -0
- package/lib/scrubber/stages/semantic-filter.ts +62 -0
- package/lib/scrubber/stages/structural-cleaner.d.ts +15 -10
- package/lib/scrubber/stages/structural-cleaner.js +73 -0
- package/lib/scrubber/stages/structural-cleaner.ts +83 -0
- package/lib/scrubber/stages/validator.d.ts +14 -15
- package/lib/scrubber/stages/validator.js +56 -0
- package/lib/scrubber/stages/validator.ts +67 -0
- package/lib/scrubber/telemetry.d.ts +20 -27
- package/lib/scrubber/telemetry.js +1 -0
- package/lib/scrubber/telemetry.ts +53 -90
- package/lib/scrubber/utils/hash.d.ts +14 -0
- package/lib/scrubber/utils/hash.js +37 -0
- package/lib/scrubber/utils/hash.ts +40 -0
- package/lib/scrubber/utils/html-parser.d.ts +14 -0
- package/lib/scrubber/utils/html-parser.js +38 -0
- package/lib/scrubber/utils/html-parser.ts +46 -0
- package/lib/scrubber/utils/pattern-matcher.d.ts +12 -0
- package/lib/scrubber/utils/pattern-matcher.js +54 -0
- package/lib/scrubber/utils/pattern-matcher.ts +64 -0
- package/lib/scrubber/utils/token-counter.d.ts +18 -0
- package/lib/scrubber/utils/token-counter.js +30 -0
- package/lib/scrubber/utils/token-counter.ts +32 -0
- package/lib/utils/logger.d.ts +1 -11
- package/lib/utils/logger.js +1 -0
- package/lib/utils/logger.ts +43 -63
- package/lib/utils/skill-metadata.d.ts +6 -14
- package/lib/utils/skill-metadata.js +1 -0
- package/lib/utils/skill-metadata.ts +89 -103
- package/lib/yamo/emitter.d.ts +8 -35
- package/lib/yamo/emitter.js +1 -0
- package/lib/yamo/emitter.ts +77 -155
- package/lib/yamo/index.d.ts +14 -0
- package/lib/yamo/index.js +14 -0
- package/lib/yamo/index.ts +16 -0
- package/lib/yamo/schema.d.ts +8 -10
- package/lib/yamo/schema.js +1 -0
- package/lib/yamo/schema.ts +82 -114
- package/package.json +5 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
1
2
|
/**
|
|
2
3
|
* Memory Mesh - Vector Memory Storage with LanceDB
|
|
3
4
|
* Provides persistent semantic memory for YAMO OS using LanceDB backend
|
|
@@ -12,880 +13,699 @@
|
|
|
12
13
|
* Also supports STDIN input for YAMO skill compatibility:
|
|
13
14
|
* echo '{"action": "ingest", "content": "..."}' | node tools/memory_mesh.js
|
|
14
15
|
*/
|
|
15
|
-
|
|
16
16
|
import { fileURLToPath } from "url";
|
|
17
17
|
import fs from "fs";
|
|
18
18
|
import path from "path";
|
|
19
19
|
import crypto from "crypto";
|
|
20
20
|
import { LanceDBClient } from "./adapters/client.js";
|
|
21
|
-
import { getConfig
|
|
22
|
-
import {
|
|
23
|
-
getEmbeddingDimension,
|
|
24
|
-
createSynthesizedSkillSchema,
|
|
25
|
-
} from "./schema.js";
|
|
21
|
+
import { getConfig } from "./adapters/config.js";
|
|
22
|
+
import { getEmbeddingDimension, createSynthesizedSkillSchema, } from "./schema.js";
|
|
26
23
|
import { handleError } from "./adapters/errors.js";
|
|
27
24
|
import EmbeddingFactory from "./embeddings/factory.js";
|
|
28
25
|
import { Scrubber } from "../scrubber/scrubber.js";
|
|
29
|
-
import {
|
|
30
|
-
extractSkillIdentity,
|
|
31
|
-
extractSkillTags,
|
|
32
|
-
} from "../utils/skill-metadata.js";
|
|
26
|
+
import { extractSkillIdentity, extractSkillTags, } from "../utils/skill-metadata.js";
|
|
33
27
|
import { KeywordSearch } from "./search/keyword-search.js";
|
|
34
28
|
import { YamoEmitter } from "../yamo/emitter.js";
|
|
35
29
|
import { LLMClient } from "../llm/client.js";
|
|
36
30
|
import * as lancedb from "@lancedb/lancedb";
|
|
37
31
|
import { createLogger } from "../utils/logger.js";
|
|
38
|
-
|
|
39
32
|
const logger = createLogger("brain");
|
|
40
|
-
|
|
41
|
-
export interface MemoryMeshOptions {
|
|
42
|
-
enableYamo?: boolean;
|
|
43
|
-
enableLLM?: boolean;
|
|
44
|
-
enableMemory?: boolean;
|
|
45
|
-
agentId?: string;
|
|
46
|
-
llmProvider?: string;
|
|
47
|
-
llmApiKey?: string;
|
|
48
|
-
llmModel?: string;
|
|
49
|
-
llmMaxTokens?: number;
|
|
50
|
-
skill_directories?: string | string[];
|
|
51
|
-
dbDir?: string;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export interface MemoryEntry {
|
|
55
|
-
id: string;
|
|
56
|
-
content: string;
|
|
57
|
-
vector: number[];
|
|
58
|
-
metadata: string;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export interface SearchResult extends MemoryEntry {
|
|
62
|
-
score: number;
|
|
63
|
-
[key: string]: any;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export interface CacheEntry {
|
|
67
|
-
result: SearchResult[];
|
|
68
|
-
timestamp: number;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
33
|
/**
|
|
72
34
|
* MemoryMesh class for managing vector memory storage
|
|
73
35
|
*/
|
|
74
36
|
export class MemoryMesh {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
// Simple LRU cache for search queries (5 minute TTL)
|
|
150
|
-
this.queryCache = new Map();
|
|
151
|
-
this.cacheConfig = {
|
|
152
|
-
maxSize: 500,
|
|
153
|
-
ttlMs: 5 * 60 * 1000, // 5 minutes
|
|
154
|
-
};
|
|
155
|
-
|
|
156
|
-
// Store custom dbDir for test isolation
|
|
157
|
-
this.dbDir = options.dbDir;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/**
|
|
161
|
-
* Generate a cache key from query and options
|
|
162
|
-
* @private
|
|
163
|
-
*/
|
|
164
|
-
_generateCacheKey(query: string, options: any = {}): string {
|
|
165
|
-
const normalizedOptions = {
|
|
166
|
-
limit: options.limit || 10,
|
|
167
|
-
filter: options.filter || null,
|
|
168
|
-
// Normalize options that affect results
|
|
169
|
-
};
|
|
170
|
-
return `search:${query}:${JSON.stringify(normalizedOptions)}`;
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
/**
|
|
174
|
-
* Get cached result if valid
|
|
175
|
-
* @private
|
|
176
|
-
*
|
|
177
|
-
* Race condition fix: The delete-then-set pattern for LRU tracking creates a window
|
|
178
|
-
* where another operation could observe the key as missing. We use a try-finally
|
|
179
|
-
* pattern to ensure atomicity at the application level.
|
|
180
|
-
*/
|
|
181
|
-
_getCachedResult(key: string): SearchResult[] | null {
|
|
182
|
-
const entry = this.queryCache.get(key);
|
|
183
|
-
if (!entry) {
|
|
184
|
-
return null;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// Check TTL - must be done before any mutation
|
|
188
|
-
const now = Date.now();
|
|
189
|
-
if (now - entry.timestamp > this.cacheConfig.ttlMs) {
|
|
190
|
-
this.queryCache.delete(key);
|
|
191
|
-
return null;
|
|
37
|
+
client;
|
|
38
|
+
config;
|
|
39
|
+
embeddingFactory;
|
|
40
|
+
keywordSearch;
|
|
41
|
+
isInitialized;
|
|
42
|
+
vectorDimension;
|
|
43
|
+
enableYamo;
|
|
44
|
+
enableLLM;
|
|
45
|
+
enableMemory;
|
|
46
|
+
agentId;
|
|
47
|
+
yamoTable;
|
|
48
|
+
skillTable;
|
|
49
|
+
llmClient;
|
|
50
|
+
scrubber;
|
|
51
|
+
queryCache;
|
|
52
|
+
cacheConfig;
|
|
53
|
+
skillDirectories; // Store skill directories for synthesis
|
|
54
|
+
dbDir; // Store custom dbDir for in-memory databases
|
|
55
|
+
/**
|
|
56
|
+
* Create a new MemoryMesh instance
|
|
57
|
+
* @param {Object} [options={}]
|
|
58
|
+
*/
|
|
59
|
+
constructor(options = {}) {
|
|
60
|
+
this.client = null;
|
|
61
|
+
this.config = null;
|
|
62
|
+
this.embeddingFactory = new EmbeddingFactory();
|
|
63
|
+
this.keywordSearch = new KeywordSearch();
|
|
64
|
+
this.isInitialized = false;
|
|
65
|
+
this.vectorDimension = 384; // Will be set during init()
|
|
66
|
+
// YAMO and LLM support
|
|
67
|
+
this.enableYamo = options.enableYamo !== false;
|
|
68
|
+
this.enableLLM = options.enableLLM !== false;
|
|
69
|
+
this.enableMemory = options.enableMemory !== false;
|
|
70
|
+
this.agentId = options.agentId || "YAMO_AGENT";
|
|
71
|
+
this.yamoTable = null;
|
|
72
|
+
this.skillTable = null;
|
|
73
|
+
this.llmClient = this.enableLLM ? new LLMClient() : null;
|
|
74
|
+
// Store skill directories for synthesis
|
|
75
|
+
if (Array.isArray(options.skill_directories)) {
|
|
76
|
+
this.skillDirectories = options.skill_directories;
|
|
77
|
+
}
|
|
78
|
+
else if (options.skill_directories) {
|
|
79
|
+
this.skillDirectories = [options.skill_directories];
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
this.skillDirectories = ["skills"];
|
|
83
|
+
}
|
|
84
|
+
// Initialize LLM client if enabled
|
|
85
|
+
if (this.enableLLM) {
|
|
86
|
+
this.llmClient = new LLMClient({
|
|
87
|
+
provider: options.llmProvider,
|
|
88
|
+
apiKey: options.llmApiKey,
|
|
89
|
+
model: options.llmModel,
|
|
90
|
+
maxTokens: options.llmMaxTokens,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
// Scrubber for Layer 0 sanitization
|
|
94
|
+
this.scrubber = new Scrubber({
|
|
95
|
+
enabled: true,
|
|
96
|
+
chunking: {
|
|
97
|
+
minTokens: 1, // Allow short memories
|
|
98
|
+
}, // Type cast for partial config
|
|
99
|
+
validation: {
|
|
100
|
+
enforceMinLength: false, // Disable strict length validation
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
// Simple LRU cache for search queries (5 minute TTL)
|
|
104
|
+
this.queryCache = new Map();
|
|
105
|
+
this.cacheConfig = {
|
|
106
|
+
maxSize: 500,
|
|
107
|
+
ttlMs: 5 * 60 * 1000, // 5 minutes
|
|
108
|
+
};
|
|
109
|
+
// Store custom dbDir for test isolation
|
|
110
|
+
this.dbDir = options.dbDir;
|
|
192
111
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* Cache a search result
|
|
208
|
-
* @private
|
|
209
|
-
*/
|
|
210
|
-
_cacheResult(key: string, result: SearchResult[]): void {
|
|
211
|
-
// Evict oldest if at max size
|
|
212
|
-
if (this.queryCache.size >= this.cacheConfig.maxSize) {
|
|
213
|
-
const firstKey = this.queryCache.keys().next().value;
|
|
214
|
-
if (firstKey !== undefined) {
|
|
215
|
-
this.queryCache.delete(firstKey);
|
|
216
|
-
}
|
|
112
|
+
/**
|
|
113
|
+
* Generate a cache key from query and options
|
|
114
|
+
* @private
|
|
115
|
+
*/
|
|
116
|
+
_generateCacheKey(query, options = {}) {
|
|
117
|
+
const normalizedOptions = {
|
|
118
|
+
limit: options.limit || 10,
|
|
119
|
+
filter: options.filter || null,
|
|
120
|
+
// Normalize options that affect results
|
|
121
|
+
};
|
|
122
|
+
return `search:${query}:${JSON.stringify(normalizedOptions)}`;
|
|
217
123
|
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
*/
|
|
247
|
-
_validateMetadata(metadata: any): Record<string, any> {
|
|
248
|
-
if (typeof metadata !== "object" || metadata === null) {
|
|
249
|
-
throw new Error("Metadata must be a non-null object");
|
|
124
|
+
/**
|
|
125
|
+
* Get cached result if valid
|
|
126
|
+
* @private
|
|
127
|
+
*
|
|
128
|
+
* Race condition fix: The delete-then-set pattern for LRU tracking creates a window
|
|
129
|
+
* where another operation could observe the key as missing. We use a try-finally
|
|
130
|
+
* pattern to ensure atomicity at the application level.
|
|
131
|
+
*/
|
|
132
|
+
_getCachedResult(key) {
|
|
133
|
+
const entry = this.queryCache.get(key);
|
|
134
|
+
if (!entry) {
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
// Check TTL - must be done before any mutation
|
|
138
|
+
const now = Date.now();
|
|
139
|
+
if (now - entry.timestamp > this.cacheConfig.ttlMs) {
|
|
140
|
+
this.queryCache.delete(key);
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
// Move to end (most recently used) - delete and re-add with updated timestamp
|
|
144
|
+
// While not truly atomic, the key remains accessible during the operation
|
|
145
|
+
// since we already have the entry reference
|
|
146
|
+
this.queryCache.delete(key);
|
|
147
|
+
this.queryCache.set(key, {
|
|
148
|
+
...entry,
|
|
149
|
+
timestamp: now, // Update timestamp for LRU tracking
|
|
150
|
+
});
|
|
151
|
+
return entry.result;
|
|
250
152
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
153
|
+
/**
|
|
154
|
+
* Cache a search result
|
|
155
|
+
* @private
|
|
156
|
+
*/
|
|
157
|
+
_cacheResult(key, result) {
|
|
158
|
+
// Evict oldest if at max size
|
|
159
|
+
if (this.queryCache.size >= this.cacheConfig.maxSize) {
|
|
160
|
+
const firstKey = this.queryCache.keys().next().value;
|
|
161
|
+
if (firstKey !== undefined) {
|
|
162
|
+
this.queryCache.delete(firstKey);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
this.queryCache.set(key, {
|
|
166
|
+
result,
|
|
167
|
+
timestamp: Date.now(),
|
|
168
|
+
});
|
|
264
169
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
* @private
|
|
271
|
-
*/
|
|
272
|
-
_sanitizeContent(content: string): string {
|
|
273
|
-
if (typeof content !== "string") {
|
|
274
|
-
throw new Error("Content must be a string");
|
|
170
|
+
/**
|
|
171
|
+
* Clear all cached results
|
|
172
|
+
*/
|
|
173
|
+
clearCache() {
|
|
174
|
+
this.queryCache.clear();
|
|
275
175
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
176
|
+
/**
|
|
177
|
+
* Get cache statistics
|
|
178
|
+
*/
|
|
179
|
+
getCacheStats() {
|
|
180
|
+
return {
|
|
181
|
+
size: this.queryCache.size,
|
|
182
|
+
maxSize: this.cacheConfig.maxSize,
|
|
183
|
+
ttlMs: this.cacheConfig.ttlMs,
|
|
184
|
+
};
|
|
283
185
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
186
|
+
/**
|
|
187
|
+
* Validate and sanitize metadata to prevent prototype pollution
|
|
188
|
+
* @private
|
|
189
|
+
*/
|
|
190
|
+
_validateMetadata(metadata) {
|
|
191
|
+
if (typeof metadata !== "object" || metadata === null) {
|
|
192
|
+
throw new Error("Metadata must be a non-null object");
|
|
193
|
+
}
|
|
194
|
+
// Sanitize keys to prevent prototype pollution
|
|
195
|
+
const sanitized = {};
|
|
196
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
197
|
+
// Skip dangerous keys that could pollute prototype
|
|
198
|
+
if (key === "__proto__" || key === "constructor" || key === "prototype") {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
// Skip inherited properties
|
|
202
|
+
if (!Object.prototype.hasOwnProperty.call(metadata, key)) {
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
sanitized[key] = value;
|
|
206
|
+
}
|
|
207
|
+
return sanitized;
|
|
294
208
|
}
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
209
|
+
/**
|
|
210
|
+
* Sanitize and validate content before storage
|
|
211
|
+
* @private
|
|
212
|
+
*/
|
|
213
|
+
_sanitizeContent(content) {
|
|
214
|
+
if (typeof content !== "string") {
|
|
215
|
+
throw new Error("Content must be a string");
|
|
216
|
+
}
|
|
217
|
+
// Limit content length
|
|
218
|
+
const MAX_CONTENT_LENGTH = 100000; // 100KB limit
|
|
219
|
+
if (content.length > MAX_CONTENT_LENGTH) {
|
|
220
|
+
throw new Error(`Content exceeds maximum length of ${MAX_CONTENT_LENGTH} characters`);
|
|
221
|
+
}
|
|
222
|
+
return content.trim();
|
|
302
223
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
317
|
-
logger.debug(
|
|
318
|
-
{ dimension: this.vectorDimension, model: modelName },
|
|
319
|
-
"Using vector dimension",
|
|
320
|
-
);
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
// Use custom dbDir if provided (for test isolation), otherwise use config
|
|
324
|
-
const dbUri = this.dbDir || this.config.LANCEDB_URI;
|
|
325
|
-
|
|
326
|
-
// Create LanceDBClient with detected dimension
|
|
327
|
-
this.client = new LanceDBClient({
|
|
328
|
-
uri: dbUri,
|
|
329
|
-
tableName: this.config.LANCEDB_MEMORY_TABLE,
|
|
330
|
-
vectorDimension: this.vectorDimension,
|
|
331
|
-
maxRetries: 3,
|
|
332
|
-
retryDelay: 1000,
|
|
333
|
-
});
|
|
334
|
-
|
|
335
|
-
// Connect to database
|
|
336
|
-
await this.client.connect();
|
|
337
|
-
|
|
338
|
-
// Configure embedding factory from environment
|
|
339
|
-
const embeddingConfigs = this._parseEmbeddingConfig();
|
|
340
|
-
this.embeddingFactory.configure(embeddingConfigs);
|
|
341
|
-
await this.embeddingFactory.init();
|
|
342
|
-
|
|
343
|
-
// Hydrate Keyword Search (In-Memory)
|
|
344
|
-
if (this.client) {
|
|
345
|
-
try {
|
|
346
|
-
const allRecords = await this.client.getAll({ limit: 10000 });
|
|
347
|
-
this.keywordSearch.load(allRecords as any);
|
|
348
|
-
} catch (_e) {
|
|
349
|
-
// Ignore if table doesn't exist yet
|
|
224
|
+
/**
|
|
225
|
+
* Initialize the LanceDB client
|
|
226
|
+
*/
|
|
227
|
+
async init() {
|
|
228
|
+
if (this.isInitialized) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
if (!this.enableMemory) {
|
|
232
|
+
this.isInitialized = true;
|
|
233
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
234
|
+
logger.debug("MemoryMesh initialization skipped (enableMemory=false)");
|
|
235
|
+
}
|
|
236
|
+
return;
|
|
350
237
|
}
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
// Initialize extension tables if enabled
|
|
354
|
-
if (this.enableYamo && this.client && this.client.db) {
|
|
355
238
|
try {
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
const
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
239
|
+
// Load configuration
|
|
240
|
+
this.config = getConfig();
|
|
241
|
+
// Detect vector dimension from embedding model configuration
|
|
242
|
+
const modelName = process.env.EMBEDDING_MODEL_NAME || "Xenova/all-MiniLM-L6-v2";
|
|
243
|
+
const envDimension = parseInt(process.env.EMBEDDING_DIMENSION || "0") || null;
|
|
244
|
+
this.vectorDimension = envDimension || getEmbeddingDimension(modelName);
|
|
245
|
+
// Only log in debug mode to avoid corrupting spinner/REPL display
|
|
246
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
247
|
+
logger.debug({ dimension: this.vectorDimension, model: modelName }, "Using vector dimension");
|
|
248
|
+
}
|
|
249
|
+
// Use custom dbDir if provided (for test isolation), otherwise use config
|
|
250
|
+
const dbUri = this.dbDir || this.config.LANCEDB_URI;
|
|
251
|
+
// Create LanceDBClient with detected dimension
|
|
252
|
+
this.client = new LanceDBClient({
|
|
253
|
+
uri: dbUri,
|
|
254
|
+
tableName: this.config.LANCEDB_MEMORY_TABLE,
|
|
255
|
+
vectorDimension: this.vectorDimension,
|
|
256
|
+
maxRetries: 3,
|
|
257
|
+
retryDelay: 1000,
|
|
258
|
+
});
|
|
259
|
+
// Connect to database
|
|
260
|
+
await this.client.connect();
|
|
261
|
+
// Configure embedding factory from environment
|
|
262
|
+
const embeddingConfigs = this._parseEmbeddingConfig();
|
|
263
|
+
this.embeddingFactory.configure(embeddingConfigs);
|
|
264
|
+
await this.embeddingFactory.init();
|
|
265
|
+
// Hydrate Keyword Search (In-Memory)
|
|
266
|
+
if (this.client) {
|
|
267
|
+
try {
|
|
268
|
+
const allRecords = await this.client.getAll({ limit: 10000 });
|
|
269
|
+
this.keywordSearch.load(allRecords);
|
|
270
|
+
}
|
|
271
|
+
catch (_e) {
|
|
272
|
+
// Ignore if table doesn't exist yet
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// Initialize extension tables if enabled
|
|
276
|
+
if (this.enableYamo && this.client && this.client.db) {
|
|
277
|
+
try {
|
|
278
|
+
const { createYamoTable } = await import("../yamo/schema.js");
|
|
279
|
+
this.yamoTable = await createYamoTable(this.client.db, "yamo_blocks");
|
|
280
|
+
// Initialize synthesized skills table (Recursive Skill Synthesis)
|
|
281
|
+
// const { createSynthesizedSkillSchema } = await import('./schema'); // Imported statically now
|
|
282
|
+
const existingTables = await this.client.db.tableNames();
|
|
283
|
+
if (existingTables.includes("synthesized_skills")) {
|
|
284
|
+
this.skillTable =
|
|
285
|
+
await this.client.db.openTable("synthesized_skills");
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
const skillSchema = createSynthesizedSkillSchema(this.vectorDimension);
|
|
289
|
+
this.skillTable = await this.client.db.createTable("synthesized_skills", [], {
|
|
290
|
+
schema: skillSchema,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
294
|
+
logger.debug("YAMO blocks and synthesized skills tables initialized");
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
catch (e) {
|
|
298
|
+
logger.warn({ err: e }, "Failed to initialize extension tables");
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
this.isInitialized = true;
|
|
302
|
+
}
|
|
303
|
+
catch (error) {
|
|
304
|
+
const e = error instanceof Error ? error : new Error(String(error));
|
|
305
|
+
throw e;
|
|
306
|
+
}
|
|
393
307
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
308
|
+
/**
|
|
309
|
+
* Add content to memory with auto-generated embedding and scrubbing.
|
|
310
|
+
*
|
|
311
|
+
* This is the primary method for storing information in the memory mesh.
|
|
312
|
+
* The content goes through several processing steps:
|
|
313
|
+
*
|
|
314
|
+
* 1. **Scrubbing**: PII and sensitive data are sanitized (if enabled)
|
|
315
|
+
* 2. **Validation**: Content length and metadata are validated
|
|
316
|
+
* 3. **Embedding**: Content is converted to a vector representation
|
|
317
|
+
* 4. **Storage**: Record is stored in LanceDB with metadata
|
|
318
|
+
* 5. **Emission**: Optional YAMO block emitted for provenance tracking
|
|
319
|
+
*
|
|
320
|
+
* @param content - The text content to store in memory
|
|
321
|
+
* @param metadata - Optional metadata (type, source, tags, etc.)
|
|
322
|
+
* @returns Promise with memory record containing id, content, metadata, created_at
|
|
323
|
+
*
|
|
324
|
+
* @example
|
|
325
|
+
* ```typescript
|
|
326
|
+
* const memory = await mesh.add("User likes TypeScript", {
|
|
327
|
+
* type: "preference",
|
|
328
|
+
* source: "chat",
|
|
329
|
+
* tags: ["programming", "languages"]
|
|
330
|
+
* });
|
|
331
|
+
* ```
|
|
332
|
+
*
|
|
333
|
+
* @throws {Error} If content exceeds max length (100KB)
|
|
334
|
+
* @throws {Error} If embedding generation fails
|
|
335
|
+
* @throws {Error} If database client is not initialized
|
|
336
|
+
*/
|
|
337
|
+
async add(content, metadata = {}) {
|
|
338
|
+
await this.init();
|
|
339
|
+
const type = metadata.type || "event";
|
|
340
|
+
const enrichedMetadata = { ...metadata, type };
|
|
341
|
+
try {
|
|
342
|
+
let processedContent = content;
|
|
343
|
+
let scrubbedMetadata = {};
|
|
344
|
+
try {
|
|
345
|
+
const scrubbedResult = await this.scrubber.process({
|
|
346
|
+
content: content,
|
|
347
|
+
source: "memory-api",
|
|
348
|
+
type: "txt",
|
|
349
|
+
});
|
|
350
|
+
if (scrubbedResult.success && scrubbedResult.chunks.length > 0) {
|
|
351
|
+
processedContent = scrubbedResult.chunks
|
|
352
|
+
.map((c) => c.text)
|
|
353
|
+
.join("\n\n");
|
|
354
|
+
if (scrubbedResult.metadata) {
|
|
355
|
+
scrubbedMetadata = {
|
|
356
|
+
...scrubbedResult.metadata,
|
|
357
|
+
scrubber_telemetry: JSON.stringify(scrubbedResult.telemetry),
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
catch (scrubError) {
|
|
363
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
364
|
+
logger.error({ err: scrubError }, "Scrubber failed");
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
const sanitizedContent = this._sanitizeContent(processedContent);
|
|
368
|
+
const sanitizedMetadata = this._validateMetadata({
|
|
369
|
+
...scrubbedMetadata,
|
|
370
|
+
...enrichedMetadata,
|
|
371
|
+
});
|
|
372
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
373
|
+
console.error("[DEBUG] brain.add() scrubbedMetadata.type:", scrubbedMetadata.type);
|
|
374
|
+
console.error("[DEBUG] brain.add() enrichedMetadata.type:", enrichedMetadata.type);
|
|
375
|
+
console.error("[DEBUG] brain.add() sanitizedMetadata.type:", sanitizedMetadata.type);
|
|
376
|
+
}
|
|
377
|
+
const vector = await this.embeddingFactory.embed(sanitizedContent);
|
|
378
|
+
// Dedup: search by the already-computed vector before inserting.
|
|
379
|
+
// Catches exact duplicates regardless of which write path is used,
|
|
380
|
+
// protecting callers that bypass captureInteraction()'s dedup guard.
|
|
381
|
+
if (this.client) {
|
|
382
|
+
const nearest = await this.client.search(vector, { limit: 1 });
|
|
383
|
+
if (nearest.length > 0 && nearest[0].content === sanitizedContent) {
|
|
384
|
+
return {
|
|
385
|
+
id: nearest[0].id,
|
|
386
|
+
content: sanitizedContent,
|
|
387
|
+
metadata: sanitizedMetadata,
|
|
388
|
+
created_at: new Date().toISOString(),
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
393
|
+
const record = {
|
|
394
|
+
id,
|
|
395
|
+
vector,
|
|
396
|
+
content: sanitizedContent,
|
|
397
|
+
metadata: JSON.stringify(sanitizedMetadata),
|
|
398
|
+
};
|
|
399
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
400
|
+
console.error("[DEBUG] record.metadata.type:", JSON.parse(record.metadata).type);
|
|
401
|
+
}
|
|
402
|
+
if (!this.client) {
|
|
403
|
+
throw new Error("Database client not initialized");
|
|
404
|
+
}
|
|
405
|
+
const result = await this.client.add(record);
|
|
406
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
407
|
+
try {
|
|
408
|
+
console.error("[DEBUG] result.metadata.type:", JSON.parse(result.metadata).type);
|
|
409
|
+
}
|
|
410
|
+
catch {
|
|
411
|
+
console.error("[DEBUG] result.metadata:", result.metadata);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
this.keywordSearch.add(record.id, record.content, sanitizedMetadata);
|
|
415
|
+
if (this.enableYamo) {
|
|
416
|
+
this._emitYamoBlock("retain", result.id, YamoEmitter.buildRetainBlock({
|
|
417
|
+
content: sanitizedContent,
|
|
418
|
+
metadata: sanitizedMetadata,
|
|
419
|
+
id: result.id,
|
|
420
|
+
agentId: this.agentId,
|
|
421
|
+
memoryType: sanitizedMetadata.type || "event",
|
|
422
|
+
})).catch((error) => {
|
|
423
|
+
// Log emission failures in debug mode but don't throw
|
|
424
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
425
|
+
logger.warn({ err: error }, "Failed to emit YAMO block (retain)");
|
|
426
|
+
}
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
return {
|
|
430
|
+
id: result.id,
|
|
431
|
+
content: sanitizedContent,
|
|
432
|
+
metadata: sanitizedMetadata,
|
|
433
|
+
created_at: new Date().toISOString(),
|
|
450
434
|
};
|
|
451
|
-
}
|
|
452
435
|
}
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
logger.error({ err: scrubError }, "Scrubber failed");
|
|
436
|
+
catch (error) {
|
|
437
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
456
438
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
)
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
metadata: sanitizedMetadata,
|
|
492
|
-
created_at: new Date().toISOString(),
|
|
493
|
-
};
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
498
|
-
|
|
499
|
-
const record: MemoryEntry = {
|
|
500
|
-
id,
|
|
501
|
-
vector,
|
|
502
|
-
content: sanitizedContent,
|
|
503
|
-
metadata: JSON.stringify(sanitizedMetadata),
|
|
504
|
-
};
|
|
505
|
-
|
|
506
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
507
|
-
console.error(
|
|
508
|
-
"[DEBUG] record.metadata.type:",
|
|
509
|
-
JSON.parse(record.metadata).type,
|
|
510
|
-
);
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
if (!this.client) {
|
|
514
|
-
throw new Error("Database client not initialized");
|
|
515
|
-
}
|
|
516
|
-
const result = await this.client.add(record);
|
|
517
|
-
|
|
518
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Reflect on recent memories
|
|
442
|
+
*/
|
|
443
|
+
async reflect(options = {}) {
|
|
444
|
+
await this.init();
|
|
445
|
+
const lookback = options.lookback || 10;
|
|
446
|
+
const topic = options.topic;
|
|
447
|
+
const generate = options.generate !== false;
|
|
448
|
+
let memories = [];
|
|
449
|
+
if (topic) {
|
|
450
|
+
memories = await this.search(topic, { limit: lookback });
|
|
451
|
+
}
|
|
452
|
+
else {
|
|
453
|
+
const all = await this.getAll();
|
|
454
|
+
memories = all
|
|
455
|
+
.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
|
|
456
|
+
.slice(0, lookback);
|
|
457
|
+
}
|
|
458
|
+
const prompt = `Review these memories. Synthesize a high-level "belief" or "observation".`;
|
|
459
|
+
if (!generate || !this.enableLLM || !this.llmClient) {
|
|
460
|
+
return {
|
|
461
|
+
topic,
|
|
462
|
+
count: memories.length,
|
|
463
|
+
context: memories.map((m) => ({
|
|
464
|
+
content: m.content,
|
|
465
|
+
type: m.metadata?.type || "event",
|
|
466
|
+
id: m.id,
|
|
467
|
+
})),
|
|
468
|
+
prompt,
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
let reflection = "";
|
|
472
|
+
let confidence = 0;
|
|
519
473
|
try {
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
YamoEmitter.buildRetainBlock({
|
|
536
|
-
content: sanitizedContent,
|
|
537
|
-
metadata: sanitizedMetadata,
|
|
538
|
-
id: result.id,
|
|
539
|
-
agentId: this.agentId,
|
|
540
|
-
memoryType: sanitizedMetadata.type || "event",
|
|
541
|
-
}),
|
|
542
|
-
).catch((error) => {
|
|
543
|
-
// Log emission failures in debug mode but don't throw
|
|
544
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
545
|
-
logger.warn({ err: error }, "Failed to emit YAMO block (retain)");
|
|
546
|
-
}
|
|
474
|
+
const result = await this.llmClient.reflect(prompt, memories);
|
|
475
|
+
reflection = result.reflection;
|
|
476
|
+
confidence = result.confidence;
|
|
477
|
+
}
|
|
478
|
+
catch (_error) {
|
|
479
|
+
reflection = `Aggregated from ${memories.length} memories on topic: ${topic || "general"}`;
|
|
480
|
+
confidence = 0.5;
|
|
481
|
+
}
|
|
482
|
+
const reflectionId = `reflect_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
|
|
483
|
+
await this.add(reflection, {
|
|
484
|
+
type: "reflection",
|
|
485
|
+
topic: topic || "general",
|
|
486
|
+
source_memory_count: memories.length,
|
|
487
|
+
confidence,
|
|
488
|
+
generated_at: new Date().toISOString(),
|
|
547
489
|
});
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
const generate = options.generate !== false;
|
|
569
|
-
|
|
570
|
-
let memories: any[] = [];
|
|
571
|
-
if (topic) {
|
|
572
|
-
memories = await this.search(topic, { limit: lookback });
|
|
573
|
-
} else {
|
|
574
|
-
const all = await this.getAll();
|
|
575
|
-
memories = all
|
|
576
|
-
.sort(
|
|
577
|
-
(a: any, b: any) =>
|
|
578
|
-
new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
|
|
579
|
-
)
|
|
580
|
-
.slice(0, lookback);
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
const prompt = `Review these memories. Synthesize a high-level "belief" or "observation".`;
|
|
584
|
-
|
|
585
|
-
if (!generate || !this.enableLLM || !this.llmClient) {
|
|
586
|
-
return {
|
|
587
|
-
topic,
|
|
588
|
-
count: memories.length,
|
|
589
|
-
context: memories.map((m) => ({
|
|
590
|
-
content: m.content,
|
|
591
|
-
type: m.metadata?.type || "event",
|
|
592
|
-
id: m.id,
|
|
593
|
-
})),
|
|
594
|
-
prompt,
|
|
595
|
-
};
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
let reflection: string = "";
|
|
599
|
-
let confidence = 0;
|
|
600
|
-
|
|
601
|
-
try {
|
|
602
|
-
const result = await this.llmClient!.reflect(prompt, memories);
|
|
603
|
-
reflection = result.reflection;
|
|
604
|
-
confidence = result.confidence;
|
|
605
|
-
} catch (_error) {
|
|
606
|
-
reflection = `Aggregated from ${memories.length} memories on topic: ${topic || "general"}`;
|
|
607
|
-
confidence = 0.5;
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
const reflectionId = `reflect_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
|
|
611
|
-
await this.add(reflection, {
|
|
612
|
-
type: "reflection",
|
|
613
|
-
topic: topic || "general",
|
|
614
|
-
source_memory_count: memories.length,
|
|
615
|
-
confidence,
|
|
616
|
-
generated_at: new Date().toISOString(),
|
|
617
|
-
});
|
|
618
|
-
|
|
619
|
-
let yamoBlock: string | null = null;
|
|
620
|
-
if (this.enableYamo) {
|
|
621
|
-
yamoBlock = YamoEmitter.buildReflectBlock({
|
|
622
|
-
topic: topic || "general",
|
|
623
|
-
memoryCount: memories.length,
|
|
624
|
-
agentId: this.agentId,
|
|
625
|
-
reflection,
|
|
626
|
-
confidence,
|
|
627
|
-
});
|
|
628
|
-
await this._emitYamoBlock("reflect", reflectionId, yamoBlock);
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
return {
|
|
632
|
-
id: reflectionId,
|
|
633
|
-
topic: topic || "general",
|
|
634
|
-
reflection,
|
|
635
|
-
confidence,
|
|
636
|
-
sourceMemoryCount: memories.length,
|
|
637
|
-
yamoBlock,
|
|
638
|
-
createdAt: new Date().toISOString(),
|
|
639
|
-
};
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
/**
|
|
643
|
-
* Ingest synthesized skill
|
|
644
|
-
* @param sourceFilePath - If provided, skip file write (file already exists)
|
|
645
|
-
*/
|
|
646
|
-
async ingestSkill(
|
|
647
|
-
yamoText: string,
|
|
648
|
-
metadata: any = {},
|
|
649
|
-
sourceFilePath?: string,
|
|
650
|
-
): Promise<any> {
|
|
651
|
-
await this.init();
|
|
652
|
-
if (!this.skillTable) {
|
|
653
|
-
throw new Error("Skill table not initialized");
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
// DEBUG: Trace sourceFilePath parameter
|
|
657
|
-
if (process.env.YAMO_DEBUG_PATHS === "true") {
|
|
658
|
-
console.error(
|
|
659
|
-
`[BRAIN.ingestSkill] sourceFilePath parameter: ${sourceFilePath || "undefined"}`,
|
|
660
|
-
);
|
|
490
|
+
let yamoBlock = null;
|
|
491
|
+
if (this.enableYamo) {
|
|
492
|
+
yamoBlock = YamoEmitter.buildReflectBlock({
|
|
493
|
+
topic: topic || "general",
|
|
494
|
+
memoryCount: memories.length,
|
|
495
|
+
agentId: this.agentId,
|
|
496
|
+
reflection,
|
|
497
|
+
confidence,
|
|
498
|
+
});
|
|
499
|
+
await this._emitYamoBlock("reflect", reflectionId, yamoBlock);
|
|
500
|
+
}
|
|
501
|
+
return {
|
|
502
|
+
id: reflectionId,
|
|
503
|
+
topic: topic || "general",
|
|
504
|
+
reflection,
|
|
505
|
+
confidence,
|
|
506
|
+
sourceMemoryCount: memories.length,
|
|
507
|
+
yamoBlock,
|
|
508
|
+
createdAt: new Date().toISOString(),
|
|
509
|
+
};
|
|
661
510
|
}
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
"Detected recursive naming pattern, rejecting ingestion to prevent loop",
|
|
676
|
-
);
|
|
677
|
-
throw new Error(
|
|
678
|
-
`Recursive naming pattern detected: ${name}. Skills must have proper name: field.`,
|
|
679
|
-
);
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
// Extract tags for tag-aware embeddings (improves semantic search)
|
|
683
|
-
const tags = extractSkillTags(yamoText);
|
|
684
|
-
const tagText = tags.length > 0 ? `\nTags: ${tags.join(", ")}` : "";
|
|
685
|
-
|
|
686
|
-
const embeddingText = `Skill: ${name}\nIntent: ${intent}${tagText}\nDescription: ${description}`;
|
|
687
|
-
const vector = await this.embeddingFactory.embed(embeddingText);
|
|
688
|
-
|
|
689
|
-
const id = `skill_${Date.now()}_${crypto.randomBytes(2).toString("hex")}`;
|
|
690
|
-
const skillMetadata = {
|
|
691
|
-
reliability: 0.5,
|
|
692
|
-
use_count: 0,
|
|
693
|
-
source: "manual",
|
|
694
|
-
...metadata,
|
|
695
|
-
// Store source file path for policy loading and parent discovery
|
|
696
|
-
...(sourceFilePath && { source_file: sourceFilePath }),
|
|
697
|
-
};
|
|
698
|
-
const record = {
|
|
699
|
-
id,
|
|
700
|
-
name,
|
|
701
|
-
intent,
|
|
702
|
-
yamo_text: yamoText,
|
|
703
|
-
vector,
|
|
704
|
-
metadata: JSON.stringify(skillMetadata),
|
|
705
|
-
created_at: new Date(),
|
|
706
|
-
};
|
|
707
|
-
await this.skillTable.add([record]);
|
|
708
|
-
|
|
709
|
-
// NEW: Persist to filesystem for longevity and visibility
|
|
710
|
-
// Skip if sourceFilePath provided (file already exists from SkillCreator)
|
|
711
|
-
// Skip if using in-memory database (:memory:)
|
|
712
|
-
if (!sourceFilePath && this.dbDir !== ":memory:") {
|
|
511
|
+
/**
|
|
512
|
+
* Ingest synthesized skill
|
|
513
|
+
* @param sourceFilePath - If provided, skip file write (file already exists)
|
|
514
|
+
*/
|
|
515
|
+
async ingestSkill(yamoText, metadata = {}, sourceFilePath) {
|
|
516
|
+
await this.init();
|
|
517
|
+
if (!this.skillTable) {
|
|
518
|
+
throw new Error("Skill table not initialized");
|
|
519
|
+
}
|
|
520
|
+
// DEBUG: Trace sourceFilePath parameter
|
|
521
|
+
if (process.env.YAMO_DEBUG_PATHS === "true") {
|
|
522
|
+
console.error(`[BRAIN.ingestSkill] sourceFilePath parameter: ${sourceFilePath || "undefined"}`);
|
|
523
|
+
}
|
|
713
524
|
try {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
.replace(/[^a-z0-9]/g, "-")
|
|
725
|
-
.replace(/-+/g, "-")
|
|
726
|
-
.substring(0, 50);
|
|
727
|
-
const fileName = `skill-${safeName}.yamo`;
|
|
728
|
-
const filePath = path.join(skillsDir, fileName);
|
|
729
|
-
// Only write if file doesn't already exist to prevent duplicates
|
|
730
|
-
if (!fs.existsSync(filePath)) {
|
|
731
|
-
fs.writeFileSync(filePath, yamoText, "utf8");
|
|
732
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
733
|
-
logger.debug({ filePath }, "Skill persisted to file");
|
|
525
|
+
const identity = extractSkillIdentity(yamoText);
|
|
526
|
+
const name = metadata.name || identity.name;
|
|
527
|
+
const intent = identity.intent;
|
|
528
|
+
const description = identity.description;
|
|
529
|
+
// RECURSION DETECTION: Check for recursive naming patterns
|
|
530
|
+
// Patterns like "SkillSkill", "SkillSkillSkill" indicate filename-derived names
|
|
531
|
+
const recursivePattern = /^(Skill|skill){2,}/;
|
|
532
|
+
if (recursivePattern.test(name)) {
|
|
533
|
+
logger.warn({ originalName: name }, "Detected recursive naming pattern, rejecting ingestion to prevent loop");
|
|
534
|
+
throw new Error(`Recursive naming pattern detected: ${name}. Skills must have proper name: field.`);
|
|
734
535
|
}
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
536
|
+
// Extract tags for tag-aware embeddings (improves semantic search)
|
|
537
|
+
const tags = extractSkillTags(yamoText);
|
|
538
|
+
const tagText = tags.length > 0 ? `\nTags: ${tags.join(", ")}` : "";
|
|
539
|
+
const embeddingText = `Skill: ${name}\nIntent: ${intent}${tagText}\nDescription: ${description}`;
|
|
540
|
+
const vector = await this.embeddingFactory.embed(embeddingText);
|
|
541
|
+
const id = `skill_${Date.now()}_${crypto.randomBytes(2).toString("hex")}`;
|
|
542
|
+
const skillMetadata = {
|
|
543
|
+
reliability: 0.5,
|
|
544
|
+
use_count: 0,
|
|
545
|
+
source: "manual",
|
|
546
|
+
...metadata,
|
|
547
|
+
// Store source file path for policy loading and parent discovery
|
|
548
|
+
...(sourceFilePath && { source_file: sourceFilePath }),
|
|
549
|
+
};
|
|
550
|
+
const record = {
|
|
551
|
+
id,
|
|
552
|
+
name,
|
|
553
|
+
intent,
|
|
554
|
+
yamo_text: yamoText,
|
|
555
|
+
vector,
|
|
556
|
+
metadata: JSON.stringify(skillMetadata),
|
|
557
|
+
created_at: new Date(),
|
|
558
|
+
};
|
|
559
|
+
await this.skillTable.add([record]);
|
|
560
|
+
// NEW: Persist to filesystem for longevity and visibility
|
|
561
|
+
// Skip if sourceFilePath provided (file already exists from SkillCreator)
|
|
562
|
+
// Skip if using in-memory database (:memory:)
|
|
563
|
+
if (!sourceFilePath && this.dbDir !== ":memory:") {
|
|
564
|
+
try {
|
|
565
|
+
const skillsDir = path.resolve(process.cwd(), this.skillDirectories[0] || "skills");
|
|
566
|
+
if (!fs.existsSync(skillsDir)) {
|
|
567
|
+
fs.mkdirSync(skillsDir, { recursive: true });
|
|
568
|
+
}
|
|
569
|
+
// Robust filename with length limit to prevent ENAMETOOLONG
|
|
570
|
+
const safeName = name
|
|
571
|
+
.toLowerCase()
|
|
572
|
+
.replace(/[^a-z0-9]/g, "-")
|
|
573
|
+
.replace(/-+/g, "-")
|
|
574
|
+
.substring(0, 50);
|
|
575
|
+
const fileName = `skill-${safeName}.yamo`;
|
|
576
|
+
const filePath = path.join(skillsDir, fileName);
|
|
577
|
+
// Only write if file doesn't already exist to prevent duplicates
|
|
578
|
+
if (!fs.existsSync(filePath)) {
|
|
579
|
+
fs.writeFileSync(filePath, yamoText, "utf8");
|
|
580
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
581
|
+
logger.debug({ filePath }, "Skill persisted to file");
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
catch (fileError) {
|
|
586
|
+
logger.warn({ err: fileError }, "Failed to persist skill to file");
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
return { id, name, intent };
|
|
590
|
+
}
|
|
591
|
+
catch (error) {
|
|
592
|
+
throw new Error(`Skill ingestion failed: ${error.message}`);
|
|
738
593
|
}
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
return { id, name, intent };
|
|
742
|
-
} catch (error: any) {
|
|
743
|
-
throw new Error(`Skill ingestion failed: ${error.message}`);
|
|
744
594
|
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
595
|
+
/**
|
|
596
|
+
* Recursive Skill Synthesis
|
|
597
|
+
*/
|
|
598
|
+
async synthesize(options = {}) {
|
|
599
|
+
await this.init();
|
|
600
|
+
const topic = options.topic || "general_improvement";
|
|
601
|
+
const enrichedPrompt = options.enrichedPrompt || topic; // PHASE 4: Use enriched prompt
|
|
602
|
+
// const lookback = options.lookback || 20;
|
|
603
|
+
logger.info({ topic, enrichedPrompt }, "Synthesizing logic");
|
|
604
|
+
// OPTIMIZATION: If we have an execution engine (kernel), use SkillCreator!
|
|
605
|
+
if (this._kernel_execute) {
|
|
606
|
+
logger.info("Dispatching to SkillCreator agent...");
|
|
607
|
+
try {
|
|
608
|
+
// Use stored skill directories
|
|
609
|
+
const skillDirs = this.skillDirectories;
|
|
610
|
+
// Track existing .yamo files before SkillCreator runs
|
|
611
|
+
const filesBefore = new Set();
|
|
612
|
+
for (const dir of skillDirs) {
|
|
613
|
+
if (fs.existsSync(dir)) {
|
|
614
|
+
const walk = (currentDir) => {
|
|
615
|
+
try {
|
|
616
|
+
const entries = fs.readdirSync(currentDir, {
|
|
617
|
+
withFileTypes: true,
|
|
618
|
+
});
|
|
619
|
+
for (const entry of entries) {
|
|
620
|
+
const fullPath = path.join(currentDir, entry.name);
|
|
621
|
+
if (entry.isDirectory()) {
|
|
622
|
+
walk(fullPath);
|
|
623
|
+
}
|
|
624
|
+
else if (entry.isFile() && entry.name.endsWith(".yamo")) {
|
|
625
|
+
filesBefore.add(fullPath);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
catch (e) {
|
|
630
|
+
// Skip directories we can't read
|
|
631
|
+
logger.debug({ dir, error: e }, "Could not read directory");
|
|
632
|
+
}
|
|
633
|
+
};
|
|
634
|
+
walk(dir);
|
|
635
|
+
}
|
|
781
636
|
}
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
}
|
|
786
|
-
};
|
|
787
|
-
walk(dir);
|
|
788
|
-
}
|
|
789
|
-
}
|
|
790
|
-
|
|
791
|
-
// PHASE 4: Use enriched prompt for SkillCreator
|
|
792
|
-
await (this as any)._kernel_execute(
|
|
793
|
-
`SkillCreator: design a new skill to handle ${enrichedPrompt}`,
|
|
794
|
-
{
|
|
795
|
-
v1_1_enabled: true,
|
|
796
|
-
},
|
|
797
|
-
);
|
|
798
|
-
|
|
799
|
-
// Find newly created .yamo file
|
|
800
|
-
let newSkillFile: string | undefined;
|
|
801
|
-
for (const dir of skillDirs) {
|
|
802
|
-
if (fs.existsSync(dir)) {
|
|
803
|
-
const walk = (currentDir: string) => {
|
|
804
|
-
try {
|
|
805
|
-
const entries = fs.readdirSync(currentDir, {
|
|
806
|
-
withFileTypes: true,
|
|
637
|
+
// PHASE 4: Use enriched prompt for SkillCreator
|
|
638
|
+
await this._kernel_execute(`SkillCreator: design a new skill to handle ${enrichedPrompt}`, {
|
|
639
|
+
v1_1_enabled: true,
|
|
807
640
|
});
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
641
|
+
// Find newly created .yamo file
|
|
642
|
+
let newSkillFile;
|
|
643
|
+
for (const dir of skillDirs) {
|
|
644
|
+
if (fs.existsSync(dir)) {
|
|
645
|
+
const walk = (currentDir) => {
|
|
646
|
+
try {
|
|
647
|
+
const entries = fs.readdirSync(currentDir, {
|
|
648
|
+
withFileTypes: true,
|
|
649
|
+
});
|
|
650
|
+
for (const entry of entries) {
|
|
651
|
+
const fullPath = path.join(currentDir, entry.name);
|
|
652
|
+
if (entry.isDirectory()) {
|
|
653
|
+
walk(fullPath);
|
|
654
|
+
}
|
|
655
|
+
else if (entry.isFile() && entry.name.endsWith(".yamo")) {
|
|
656
|
+
if (!filesBefore.has(fullPath)) {
|
|
657
|
+
newSkillFile = fullPath;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
catch (e) {
|
|
663
|
+
logger.debug({ dir, error: e }, "Could not read directory");
|
|
664
|
+
}
|
|
665
|
+
};
|
|
666
|
+
walk(dir);
|
|
815
667
|
}
|
|
816
|
-
}
|
|
817
668
|
}
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
fs.writeFileSync(newSkillFile, skillContent, "utf8");
|
|
859
|
-
logger.info(
|
|
860
|
-
{ skillFile: newSkillFile },
|
|
861
|
-
"Skill expanded to canonical format on disk",
|
|
862
|
-
);
|
|
863
|
-
}
|
|
864
|
-
} catch (e) {
|
|
865
|
-
logger.warn(
|
|
866
|
-
{ err: e },
|
|
867
|
-
"Failed to expand skill to canonical, using compressed format",
|
|
868
|
-
);
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
// ENSURE: Synthesized skills always have proper metadata with meaningful name
|
|
873
|
-
// This prevents duplicate skill-agent-{timestamp}.yamo files
|
|
874
|
-
const synIdentity = extractSkillIdentity(skillContent);
|
|
875
|
-
const hasName = !synIdentity.name.startsWith("Unnamed_");
|
|
876
|
-
if (!skillContent.includes("---") || !hasName) {
|
|
877
|
-
logger.info(
|
|
878
|
-
{ skillFile: newSkillFile },
|
|
879
|
-
"Adding metadata block to synthesized skill",
|
|
880
|
-
);
|
|
881
|
-
const intent =
|
|
882
|
-
synIdentity.intent !== "general_procedure"
|
|
883
|
-
? synIdentity.intent.replace(/[^a-zA-Z0-9]/g, "")
|
|
884
|
-
: "Synthesized";
|
|
885
|
-
const PascalCase = intent.charAt(0).toUpperCase() + intent.slice(1);
|
|
886
|
-
const skillName = `${PascalCase}_${Date.now().toString(36)}`;
|
|
887
|
-
|
|
888
|
-
const metadata = `---
|
|
669
|
+
// Ingest the newly created skill file
|
|
670
|
+
if (newSkillFile) {
|
|
671
|
+
logger.info({ skillFile: newSkillFile }, "Ingesting newly synthesized skill");
|
|
672
|
+
let skillContent = fs.readFileSync(newSkillFile, "utf8");
|
|
673
|
+
// PHASE 4: Expand compressed → canonical for disk storage
|
|
674
|
+
// Skills created by evolution are typically compressed; expand to canonical for readability
|
|
675
|
+
// Skip expansion in test environment or when disabled
|
|
676
|
+
const expansionEnabled = process.env.YAMO_EXPANSION_ENABLED !== "false";
|
|
677
|
+
const isCompressed = !skillContent.includes("---") ||
|
|
678
|
+
(skillContent.includes("---") &&
|
|
679
|
+
skillContent.split("---").length <= 1);
|
|
680
|
+
if (expansionEnabled && isCompressed) {
|
|
681
|
+
logger.info({ skillFile: newSkillFile }, "Expanding compressed skill to canonical format");
|
|
682
|
+
try {
|
|
683
|
+
const expanded = await this._kernel_execute("skill-expansion-system-prompt.yamo", {
|
|
684
|
+
input_yamo: skillContent,
|
|
685
|
+
});
|
|
686
|
+
if (expanded && expanded.canonical_yamo) {
|
|
687
|
+
skillContent = expanded.canonical_yamo;
|
|
688
|
+
// Write expanded canonical format back to disk
|
|
689
|
+
fs.writeFileSync(newSkillFile, skillContent, "utf8");
|
|
690
|
+
logger.info({ skillFile: newSkillFile }, "Skill expanded to canonical format on disk");
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
catch (e) {
|
|
694
|
+
logger.warn({ err: e }, "Failed to expand skill to canonical, using compressed format");
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
// ENSURE: Synthesized skills always have proper metadata with meaningful name
|
|
698
|
+
// This prevents duplicate skill-agent-{timestamp}.yamo files
|
|
699
|
+
const synIdentity = extractSkillIdentity(skillContent);
|
|
700
|
+
const hasName = !synIdentity.name.startsWith("Unnamed_");
|
|
701
|
+
if (!skillContent.includes("---") || !hasName) {
|
|
702
|
+
logger.info({ skillFile: newSkillFile }, "Adding metadata block to synthesized skill");
|
|
703
|
+
const intent = synIdentity.intent !== "general_procedure"
|
|
704
|
+
? synIdentity.intent.replace(/[^a-zA-Z0-9]/g, "")
|
|
705
|
+
: "Synthesized";
|
|
706
|
+
const PascalCase = intent.charAt(0).toUpperCase() + intent.slice(1);
|
|
707
|
+
const skillName = `${PascalCase}_${Date.now().toString(36)}`;
|
|
708
|
+
const metadata = `---
|
|
889
709
|
name: ${skillName}
|
|
890
710
|
version: 1.0.0
|
|
891
711
|
author: YAMO Evolution
|
|
@@ -894,910 +714,804 @@ tags: synthesized, evolution, auto-generated
|
|
|
894
714
|
description: Auto-generated skill to handle: ${enrichedPrompt || topic}
|
|
895
715
|
---
|
|
896
716
|
`;
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
717
|
+
// Prepend metadata if skill doesn't have it
|
|
718
|
+
if (!skillContent.startsWith("---")) {
|
|
719
|
+
skillContent = metadata + skillContent;
|
|
720
|
+
// Write back to disk with proper metadata
|
|
721
|
+
fs.writeFileSync(newSkillFile, skillContent, "utf8");
|
|
722
|
+
logger.info({ skillFile: newSkillFile, skillName }, "Added metadata block to synthesized skill");
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
const skill = await this.ingestSkill(skillContent, {
|
|
726
|
+
source: "synthesized",
|
|
727
|
+
trigger_topic: topic,
|
|
728
|
+
}, newSkillFile);
|
|
729
|
+
return {
|
|
730
|
+
status: "success",
|
|
731
|
+
analysis: "SkillCreator orchestrated evolution",
|
|
732
|
+
skill_id: skill.id,
|
|
733
|
+
skill_name: skill.name,
|
|
734
|
+
yamo_text: skillContent,
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
// Fallback if no new file found
|
|
738
|
+
return {
|
|
739
|
+
status: "success",
|
|
740
|
+
analysis: "SkillCreator orchestrated evolution (no file detected)",
|
|
741
|
+
skill_name: topic.split(" ")[0],
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
catch (e) {
|
|
745
|
+
logger.error({ err: e }, "SkillCreator agent failed");
|
|
746
|
+
return {
|
|
747
|
+
status: "error",
|
|
748
|
+
error: e.message,
|
|
749
|
+
analysis: "SkillCreator agent failed",
|
|
750
|
+
};
|
|
907
751
|
}
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
const skill = await this.ingestSkill(
|
|
911
|
-
skillContent,
|
|
912
|
-
{
|
|
913
|
-
source: "synthesized",
|
|
914
|
-
trigger_topic: topic,
|
|
915
|
-
},
|
|
916
|
-
newSkillFile,
|
|
917
|
-
);
|
|
918
|
-
return {
|
|
919
|
-
status: "success",
|
|
920
|
-
analysis: "SkillCreator orchestrated evolution",
|
|
921
|
-
skill_id: skill.id,
|
|
922
|
-
skill_name: skill.name,
|
|
923
|
-
yamo_text: skillContent,
|
|
924
|
-
};
|
|
925
752
|
}
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
skill_name: topic.split(" ")[0],
|
|
932
|
-
};
|
|
933
|
-
} catch (e: any) {
|
|
934
|
-
logger.error({ err: e }, "SkillCreator agent failed");
|
|
753
|
+
// SkillCreator is required for synthesis
|
|
754
|
+
if (!this._kernel_execute) {
|
|
755
|
+
throw new Error("Kernel execution (_kernel_execute) is required for synthesis. Use YamoKernel instead of MemoryMesh directly.");
|
|
756
|
+
}
|
|
757
|
+
// Should never reach here
|
|
935
758
|
return {
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
analysis: "SkillCreator agent failed",
|
|
759
|
+
status: "error",
|
|
760
|
+
analysis: "Unexpected state in synthesis",
|
|
939
761
|
};
|
|
940
|
-
}
|
|
941
762
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
0,
|
|
978
|
-
Math.min(1.0, (metadata.reliability || 0.5) + adjustment),
|
|
979
|
-
);
|
|
980
|
-
metadata.use_count = (metadata.use_count || 0) + 1;
|
|
981
|
-
metadata.last_used = new Date().toISOString();
|
|
982
|
-
await this.skillTable.update({
|
|
983
|
-
where: `id == '${id}'`,
|
|
984
|
-
values: { metadata: JSON.stringify(metadata) },
|
|
985
|
-
} as any);
|
|
986
|
-
return {
|
|
987
|
-
id,
|
|
988
|
-
reliability: metadata.reliability,
|
|
989
|
-
use_count: metadata.use_count,
|
|
990
|
-
};
|
|
991
|
-
} catch (error: any) {
|
|
992
|
-
throw new Error(`Failed to update skill reliability: ${error.message}`);
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
/**
|
|
997
|
-
* Prune skills
|
|
998
|
-
*/
|
|
999
|
-
async pruneSkills(threshold: number = 0.3): Promise<any> {
|
|
1000
|
-
await this.init();
|
|
1001
|
-
if (!this.skillTable) {
|
|
1002
|
-
throw new Error("Skill table not initialized");
|
|
1003
|
-
}
|
|
1004
|
-
try {
|
|
1005
|
-
const allSkills = await this.skillTable.query().toArray();
|
|
1006
|
-
let prunedCount = 0;
|
|
1007
|
-
for (const skill of allSkills) {
|
|
1008
|
-
const metadata = JSON.parse(skill.metadata);
|
|
1009
|
-
if (metadata.reliability < threshold) {
|
|
1010
|
-
await this.skillTable.delete(`id == '${skill.id}'`);
|
|
1011
|
-
prunedCount++;
|
|
1012
|
-
}
|
|
1013
|
-
}
|
|
1014
|
-
return {
|
|
1015
|
-
pruned_count: prunedCount,
|
|
1016
|
-
total_remaining: allSkills.length - prunedCount,
|
|
1017
|
-
};
|
|
1018
|
-
} catch (error: any) {
|
|
1019
|
-
throw new Error(`Pruning failed: ${error.message}`);
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
/**
|
|
1024
|
-
* List all synthesized skills
|
|
1025
|
-
* @param {Object} [options={}] - Search options
|
|
1026
|
-
* @returns {Promise<Array>} Normalized skill results
|
|
1027
|
-
*/
|
|
1028
|
-
async listSkills(options: any = {}): Promise<any[]> {
|
|
1029
|
-
await this.init();
|
|
1030
|
-
if (!this.skillTable) {
|
|
1031
|
-
return [];
|
|
763
|
+
/**
|
|
764
|
+
* Update reliability
|
|
765
|
+
*/
|
|
766
|
+
async updateSkillReliability(id, success) {
|
|
767
|
+
await this.init();
|
|
768
|
+
if (!this.skillTable) {
|
|
769
|
+
throw new Error("Skill table not initialized");
|
|
770
|
+
}
|
|
771
|
+
try {
|
|
772
|
+
const results = await this.skillTable
|
|
773
|
+
.query()
|
|
774
|
+
.filter(`id == '${id}'`)
|
|
775
|
+
.toArray();
|
|
776
|
+
if (results.length === 0) {
|
|
777
|
+
throw new Error(`Skill ${id} not found`);
|
|
778
|
+
}
|
|
779
|
+
const record = results[0];
|
|
780
|
+
const metadata = JSON.parse(record.metadata);
|
|
781
|
+
const adjustment = success ? 0.1 : -0.2;
|
|
782
|
+
metadata.reliability = Math.max(0, Math.min(1.0, (metadata.reliability || 0.5) + adjustment));
|
|
783
|
+
metadata.use_count = (metadata.use_count || 0) + 1;
|
|
784
|
+
metadata.last_used = new Date().toISOString();
|
|
785
|
+
await this.skillTable.update({
|
|
786
|
+
where: `id == '${id}'`,
|
|
787
|
+
values: { metadata: JSON.stringify(metadata) },
|
|
788
|
+
});
|
|
789
|
+
return {
|
|
790
|
+
id,
|
|
791
|
+
reliability: metadata.reliability,
|
|
792
|
+
use_count: metadata.use_count,
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
catch (error) {
|
|
796
|
+
throw new Error(`Failed to update skill reliability: ${error.message}`);
|
|
797
|
+
}
|
|
1032
798
|
}
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
799
|
+
/**
|
|
800
|
+
* Prune skills
|
|
801
|
+
*/
|
|
802
|
+
async pruneSkills(threshold = 0.3) {
|
|
803
|
+
await this.init();
|
|
804
|
+
if (!this.skillTable) {
|
|
805
|
+
throw new Error("Skill table not initialized");
|
|
806
|
+
}
|
|
807
|
+
try {
|
|
808
|
+
const allSkills = await this.skillTable.query().toArray();
|
|
809
|
+
let prunedCount = 0;
|
|
810
|
+
for (const skill of allSkills) {
|
|
811
|
+
const metadata = JSON.parse(skill.metadata);
|
|
812
|
+
if (metadata.reliability < threshold) {
|
|
813
|
+
await this.skillTable.delete(`id == '${skill.id}'`);
|
|
814
|
+
prunedCount++;
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
return {
|
|
818
|
+
pruned_count: prunedCount,
|
|
819
|
+
total_remaining: allSkills.length - prunedCount,
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
catch (error) {
|
|
823
|
+
throw new Error(`Pruning failed: ${error.message}`);
|
|
824
|
+
}
|
|
1050
825
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
826
|
+
/**
|
|
827
|
+
* List all synthesized skills
|
|
828
|
+
* @param {Object} [options={}] - Search options
|
|
829
|
+
* @returns {Promise<Array>} Normalized skill results
|
|
830
|
+
*/
|
|
831
|
+
async listSkills(options = {}) {
|
|
832
|
+
await this.init();
|
|
833
|
+
if (!this.skillTable) {
|
|
834
|
+
return [];
|
|
835
|
+
}
|
|
836
|
+
try {
|
|
837
|
+
const limit = options.limit || 10;
|
|
838
|
+
const results = await this.skillTable.query().limit(limit).toArray();
|
|
839
|
+
return results.map((r) => ({
|
|
840
|
+
...r,
|
|
841
|
+
score: 1.0, // Full score for direct listing
|
|
842
|
+
// Parse metadata JSON string to object
|
|
843
|
+
metadata: typeof r.metadata === "string" ? JSON.parse(r.metadata) : r.metadata,
|
|
844
|
+
}));
|
|
845
|
+
}
|
|
846
|
+
catch (error) {
|
|
847
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
848
|
+
logger.error({ err: error }, "Skill list failed");
|
|
849
|
+
}
|
|
850
|
+
return [];
|
|
851
|
+
}
|
|
1063
852
|
}
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
.toArray();
|
|
1075
|
-
|
|
1076
|
-
if (directResults.length > 0) {
|
|
1077
|
-
return directResults.map((r) => ({
|
|
1078
|
-
...r,
|
|
1079
|
-
score: 1.0, // Maximum score for explicit target
|
|
1080
|
-
}));
|
|
853
|
+
/**
|
|
854
|
+
* Search for synthesized skills by semantic intent
|
|
855
|
+
* @param {string} query - Search query (intent description)
|
|
856
|
+
* @param {Object} [options={}] - Search options
|
|
857
|
+
* @returns {Promise<Array>} Normalized skill results
|
|
858
|
+
*/
|
|
859
|
+
async searchSkills(query, options = {}) {
|
|
860
|
+
await this.init();
|
|
861
|
+
if (!this.skillTable) {
|
|
862
|
+
return [];
|
|
1081
863
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
864
|
+
try {
|
|
865
|
+
// 1. Check for explicit skill targeting (e.g., "Architect: ...")
|
|
866
|
+
const explicitMatch = query.match(/^([a-zA-Z0-9_-]+):/);
|
|
867
|
+
if (explicitMatch) {
|
|
868
|
+
const targetName = explicitMatch[1];
|
|
869
|
+
const directResults = await this.skillTable
|
|
870
|
+
.query()
|
|
871
|
+
.where(`name == '${targetName}'`)
|
|
872
|
+
.limit(1)
|
|
873
|
+
.toArray();
|
|
874
|
+
if (directResults.length > 0) {
|
|
875
|
+
return directResults.map((r) => ({
|
|
876
|
+
...r,
|
|
877
|
+
score: 1.0, // Maximum score for explicit target
|
|
878
|
+
}));
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
// 2. Hybrid search: vector + keyword matching
|
|
882
|
+
const limit = options.limit || 5;
|
|
883
|
+
// 2a. Vector search (get more candidates for fusion)
|
|
884
|
+
const vector = await this.embeddingFactory.embed(query);
|
|
885
|
+
const vectorResults = await this.skillTable
|
|
886
|
+
.search(vector)
|
|
887
|
+
.limit(limit * 3)
|
|
888
|
+
.toArray();
|
|
889
|
+
// 2b. Keyword matching against skill fields (including tags)
|
|
890
|
+
const queryTokens = this._tokenizeQuery(query);
|
|
891
|
+
const keywordScores = new Map();
|
|
892
|
+
let maxKeywordScore = 0;
|
|
893
|
+
for (const result of vectorResults) {
|
|
894
|
+
let score = 0;
|
|
895
|
+
const nameTokens = this._tokenizeQuery(result.name);
|
|
896
|
+
const intentTokens = this._tokenizeQuery(result.intent || "");
|
|
897
|
+
const tags = extractSkillTags(result.yamo_text);
|
|
898
|
+
const tagTokens = tags.flatMap((t) => this._tokenizeQuery(t));
|
|
899
|
+
const descTokens = this._tokenizeQuery(result.yamo_text.substring(0, 500)); // First 500 chars
|
|
900
|
+
// Token matching with field-based weights
|
|
901
|
+
// Support both exact and partial matches (for compound words)
|
|
902
|
+
for (const qToken of queryTokens) {
|
|
903
|
+
// Exact or partial match in name
|
|
904
|
+
if (nameTokens.some((nt) => nt === qToken || qToken.includes(nt) || nt.includes(qToken))) {
|
|
905
|
+
score += 10.0; // Highest: name match
|
|
906
|
+
}
|
|
907
|
+
// Exact or partial match in tags
|
|
908
|
+
if (tagTokens.some((tt) => tt === qToken || qToken.includes(tt) || tt.includes(qToken))) {
|
|
909
|
+
score += 7.0; // High: tag match
|
|
910
|
+
}
|
|
911
|
+
// Exact match in intent
|
|
912
|
+
if (intentTokens.some((it) => it === qToken)) {
|
|
913
|
+
score += 5.0; // Medium: intent match
|
|
914
|
+
}
|
|
915
|
+
// Exact match in description
|
|
916
|
+
if (descTokens.some((dt) => dt === qToken)) {
|
|
917
|
+
score += 1.0; // Low: description match
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
if (score > 0) {
|
|
921
|
+
keywordScores.set(result.id, score);
|
|
922
|
+
maxKeywordScore = Math.max(maxKeywordScore, score);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
// 2c. Combine scores using weighted fusion
|
|
926
|
+
const fusedResults = vectorResults.map((r) => {
|
|
927
|
+
// Normalize vector distance to [0, 1] similarity score
|
|
928
|
+
// LanceDB cosine distance ranges from 0 (identical) to 2 (opposite)
|
|
929
|
+
const rawDistance = r._distance !== undefined ? r._distance : 1.0;
|
|
930
|
+
const vectorScore = Math.max(0, Math.min(1.0, 1 - rawDistance / 2));
|
|
931
|
+
const keywordScore = keywordScores.get(r.id) || 0;
|
|
932
|
+
// Normalize keyword score by max observed (or use fixed max to avoid division by zero)
|
|
933
|
+
const normalizedKeyword = maxKeywordScore > 0 ? keywordScore / maxKeywordScore : 0;
|
|
934
|
+
// Weighted combination: 70% keyword, 30% vector
|
|
935
|
+
// Keywords get higher weight to prioritize exact matches
|
|
936
|
+
const combinedScore = 0.7 * normalizedKeyword + 0.3 * vectorScore;
|
|
937
|
+
return {
|
|
938
|
+
...r,
|
|
939
|
+
score: combinedScore,
|
|
940
|
+
_vectorScore: vectorScore,
|
|
941
|
+
_keywordScore: keywordScore,
|
|
942
|
+
};
|
|
943
|
+
});
|
|
944
|
+
// Sort by combined score and return top results
|
|
945
|
+
// Don't normalize - we already calculated hybrid scores
|
|
946
|
+
return fusedResults
|
|
947
|
+
.sort((a, b) => b.score - a.score)
|
|
948
|
+
.slice(0, limit)
|
|
949
|
+
.map((r) => ({
|
|
950
|
+
...r,
|
|
951
|
+
// Parse metadata JSON string to object for policy loading
|
|
952
|
+
metadata: typeof r.metadata === "string"
|
|
953
|
+
? JSON.parse(r.metadata)
|
|
954
|
+
: r.metadata,
|
|
955
|
+
}))
|
|
956
|
+
.map((r) => ({
|
|
957
|
+
...r,
|
|
958
|
+
score: parseFloat(r.score.toFixed(2)), // Round for consistency
|
|
959
|
+
}));
|
|
1138
960
|
}
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
961
|
+
catch (error) {
|
|
962
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
963
|
+
logger.error({ err: error }, "Skill search failed");
|
|
964
|
+
}
|
|
965
|
+
return [];
|
|
1143
966
|
}
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
// 2c. Combine scores using weighted fusion
|
|
1147
|
-
const fusedResults = vectorResults.map((r) => {
|
|
1148
|
-
// Normalize vector distance to [0, 1] similarity score
|
|
1149
|
-
// LanceDB cosine distance ranges from 0 (identical) to 2 (opposite)
|
|
1150
|
-
const rawDistance = r._distance !== undefined ? r._distance : 1.0;
|
|
1151
|
-
const vectorScore = Math.max(0, Math.min(1.0, 1 - rawDistance / 2));
|
|
1152
|
-
|
|
1153
|
-
const keywordScore = keywordScores.get(r.id) || 0;
|
|
1154
|
-
|
|
1155
|
-
// Normalize keyword score by max observed (or use fixed max to avoid division by zero)
|
|
1156
|
-
const normalizedKeyword =
|
|
1157
|
-
maxKeywordScore > 0 ? keywordScore / maxKeywordScore : 0;
|
|
1158
|
-
|
|
1159
|
-
// Weighted combination: 70% keyword, 30% vector
|
|
1160
|
-
// Keywords get higher weight to prioritize exact matches
|
|
1161
|
-
const combinedScore = 0.7 * normalizedKeyword + 0.3 * vectorScore;
|
|
1162
|
-
|
|
1163
|
-
return {
|
|
1164
|
-
...r,
|
|
1165
|
-
score: combinedScore,
|
|
1166
|
-
_vectorScore: vectorScore,
|
|
1167
|
-
_keywordScore: keywordScore,
|
|
1168
|
-
};
|
|
1169
|
-
});
|
|
1170
|
-
|
|
1171
|
-
// Sort by combined score and return top results
|
|
1172
|
-
// Don't normalize - we already calculated hybrid scores
|
|
1173
|
-
return fusedResults
|
|
1174
|
-
.sort((a, b) => b.score - a.score)
|
|
1175
|
-
.slice(0, limit)
|
|
1176
|
-
.map((r) => ({
|
|
1177
|
-
...r,
|
|
1178
|
-
// Parse metadata JSON string to object for policy loading
|
|
1179
|
-
metadata:
|
|
1180
|
-
typeof r.metadata === "string"
|
|
1181
|
-
? JSON.parse(r.metadata)
|
|
1182
|
-
: r.metadata,
|
|
1183
|
-
}))
|
|
1184
|
-
.map((r) => ({
|
|
1185
|
-
...r,
|
|
1186
|
-
score: parseFloat(r.score.toFixed(2)), // Round for consistency
|
|
1187
|
-
}));
|
|
1188
|
-
} catch (error: any) {
|
|
1189
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
1190
|
-
logger.error({ err: error }, "Skill search failed");
|
|
1191
|
-
}
|
|
1192
|
-
return [];
|
|
1193
967
|
}
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
968
|
+
/**
|
|
969
|
+
* Get recent YAMO logs for the heartbeat
|
|
970
|
+
* @param {Object} options
|
|
971
|
+
*/
|
|
972
|
+
async getYamoLog(options = {}) {
|
|
973
|
+
if (!this.yamoTable) {
|
|
974
|
+
return [];
|
|
975
|
+
}
|
|
976
|
+
const limit = options.limit || 10;
|
|
977
|
+
const maxRetries = 5;
|
|
978
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
979
|
+
try {
|
|
980
|
+
// orderBy might not be in LanceDB types but is supported in runtime
|
|
981
|
+
const query = this.yamoTable.query();
|
|
982
|
+
let results;
|
|
983
|
+
try {
|
|
984
|
+
results = await query
|
|
985
|
+
.orderBy("timestamp", "desc")
|
|
986
|
+
.limit(limit)
|
|
987
|
+
.toArray();
|
|
988
|
+
}
|
|
989
|
+
catch (_e) {
|
|
990
|
+
// Fallback if orderBy not supported
|
|
991
|
+
results = await query.limit(1000).toArray(); // Get more and sort manually
|
|
992
|
+
}
|
|
993
|
+
// Sort newest first in memory
|
|
994
|
+
return results
|
|
995
|
+
.sort((a, b) => {
|
|
996
|
+
const tA = a.timestamp instanceof Date
|
|
997
|
+
? a.timestamp.getTime()
|
|
998
|
+
: Number(a.timestamp);
|
|
999
|
+
const tB = b.timestamp instanceof Date
|
|
1000
|
+
? b.timestamp.getTime()
|
|
1001
|
+
: Number(b.timestamp);
|
|
1002
|
+
return tB - tA;
|
|
1003
|
+
})
|
|
1004
|
+
.slice(0, limit)
|
|
1005
|
+
.map((r) => ({
|
|
1006
|
+
id: r.id,
|
|
1007
|
+
yamoText: r.yamo_text,
|
|
1008
|
+
timestamp: r.timestamp,
|
|
1009
|
+
}));
|
|
1010
|
+
}
|
|
1011
|
+
catch (error) {
|
|
1012
|
+
const msg = error.message || "";
|
|
1013
|
+
const isRetryable = msg.includes("LanceError(IO)") ||
|
|
1014
|
+
msg.includes("next batch") ||
|
|
1015
|
+
msg.includes("No such file") ||
|
|
1016
|
+
msg.includes("busy");
|
|
1017
|
+
if (isRetryable && attempt < maxRetries) {
|
|
1018
|
+
// If we suspect stale table handle, try to refresh it
|
|
1019
|
+
try {
|
|
1020
|
+
// Re-open table to get fresh file handles
|
|
1021
|
+
const { createYamoTable } = await import("../yamo/schema.js");
|
|
1022
|
+
if (this.dbDir) {
|
|
1023
|
+
const db = await lancedb.connect(this.dbDir);
|
|
1024
|
+
this.yamoTable = await createYamoTable(db, "yamo_blocks");
|
|
1025
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
1026
|
+
logger.debug({ attempt, msg: msg.substring(0, 100) }, "Refreshed yamoTable handle during retry");
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
catch (e) {
|
|
1031
|
+
logger.warn({ err: e }, "Failed to refresh table handle during retry");
|
|
1032
|
+
}
|
|
1033
|
+
const delay = 500 * Math.pow(2, attempt - 1); // 500ms, 1000ms, 2000ms, 4000ms
|
|
1034
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1035
|
+
continue;
|
|
1036
|
+
}
|
|
1037
|
+
// Only log warning on final failure
|
|
1038
|
+
if (attempt === maxRetries) {
|
|
1039
|
+
logger.warn({ err: error }, "Failed to get log after retries");
|
|
1040
|
+
}
|
|
1041
|
+
else if (!isRetryable) {
|
|
1042
|
+
// Non-retryable error
|
|
1043
|
+
logger.warn({ err: error }, "Failed to get log (non-retryable)");
|
|
1044
|
+
break;
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
return [];
|
|
1203
1049
|
}
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1050
|
+
/**
|
|
1051
|
+
* Emit a YAMO block to the YAMO blocks table
|
|
1052
|
+
* @private
|
|
1053
|
+
*
|
|
1054
|
+
* Note: YAMO emission is non-critical - failures are logged but don't throw
|
|
1055
|
+
* to prevent disrupting the main operation.
|
|
1056
|
+
*/
|
|
1057
|
+
async _emitYamoBlock(operationType, memoryId, yamoText) {
|
|
1058
|
+
if (!this.yamoTable) {
|
|
1059
|
+
return;
|
|
1060
|
+
}
|
|
1061
|
+
const yamoId = `yamo_${operationType}_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
|
|
1212
1062
|
try {
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1063
|
+
await this.yamoTable.add([
|
|
1064
|
+
{
|
|
1065
|
+
id: yamoId,
|
|
1066
|
+
agent_id: this.agentId,
|
|
1067
|
+
operation_type: operationType,
|
|
1068
|
+
yamo_text: yamoText,
|
|
1069
|
+
timestamp: new Date(),
|
|
1070
|
+
block_hash: null,
|
|
1071
|
+
prev_hash: null,
|
|
1072
|
+
metadata: JSON.stringify({
|
|
1073
|
+
memory_id: memoryId || null,
|
|
1074
|
+
timestamp: new Date().toISOString(),
|
|
1075
|
+
}),
|
|
1076
|
+
},
|
|
1077
|
+
]);
|
|
1220
1078
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
a.timestamp instanceof Date
|
|
1227
|
-
? a.timestamp.getTime()
|
|
1228
|
-
: Number(a.timestamp);
|
|
1229
|
-
const tB =
|
|
1230
|
-
b.timestamp instanceof Date
|
|
1231
|
-
? b.timestamp.getTime()
|
|
1232
|
-
: Number(b.timestamp);
|
|
1233
|
-
return tB - tA;
|
|
1234
|
-
})
|
|
1235
|
-
.slice(0, limit)
|
|
1236
|
-
.map((r: any) => ({
|
|
1237
|
-
id: r.id,
|
|
1238
|
-
yamoText: r.yamo_text,
|
|
1239
|
-
timestamp: r.timestamp,
|
|
1240
|
-
}));
|
|
1241
|
-
} catch (error: any) {
|
|
1242
|
-
const msg = error.message || "";
|
|
1243
|
-
const isRetryable =
|
|
1244
|
-
msg.includes("LanceError(IO)") ||
|
|
1245
|
-
msg.includes("next batch") ||
|
|
1246
|
-
msg.includes("No such file") ||
|
|
1247
|
-
msg.includes("busy");
|
|
1248
|
-
|
|
1249
|
-
if (isRetryable && attempt < maxRetries) {
|
|
1250
|
-
// If we suspect stale table handle, try to refresh it
|
|
1251
|
-
try {
|
|
1252
|
-
// Re-open table to get fresh file handles
|
|
1253
|
-
const { createYamoTable } = await import("../yamo/schema.js");
|
|
1254
|
-
|
|
1255
|
-
if (this.dbDir) {
|
|
1256
|
-
const db = await lancedb.connect(this.dbDir);
|
|
1257
|
-
this.yamoTable = await createYamoTable(db, "yamo_blocks");
|
|
1258
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
1259
|
-
logger.debug(
|
|
1260
|
-
{ attempt, msg: msg.substring(0, 100) },
|
|
1261
|
-
"Refreshed yamoTable handle during retry",
|
|
1262
|
-
);
|
|
1263
|
-
}
|
|
1079
|
+
catch (error) {
|
|
1080
|
+
// Log emission failures in debug mode
|
|
1081
|
+
// Emission is non-critical, so we don't throw
|
|
1082
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
1083
|
+
logger.warn({ err: error, operationType }, "YAMO emission failed");
|
|
1264
1084
|
}
|
|
1265
|
-
} catch (e) {
|
|
1266
|
-
logger.warn(
|
|
1267
|
-
{ err: e },
|
|
1268
|
-
"Failed to refresh table handle during retry",
|
|
1269
|
-
);
|
|
1270
|
-
}
|
|
1271
|
-
|
|
1272
|
-
const delay = 500 * Math.pow(2, attempt - 1); // 500ms, 1000ms, 2000ms, 4000ms
|
|
1273
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1274
|
-
continue;
|
|
1275
1085
|
}
|
|
1276
|
-
|
|
1277
|
-
// Only log warning on final failure
|
|
1278
|
-
if (attempt === maxRetries) {
|
|
1279
|
-
logger.warn({ err: error }, "Failed to get log after retries");
|
|
1280
|
-
} else if (!isRetryable) {
|
|
1281
|
-
// Non-retryable error
|
|
1282
|
-
logger.warn({ err: error }, "Failed to get log (non-retryable)");
|
|
1283
|
-
break;
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
1086
|
}
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1087
|
+
/**
|
|
1088
|
+
* Search memory using hybrid vector + keyword search with Reciprocal Rank Fusion (RRF).
|
|
1089
|
+
*
|
|
1090
|
+
* This method performs semantic search by combining:
|
|
1091
|
+
* 1. **Vector Search**: Uses embeddings to find semantically similar content
|
|
1092
|
+
* 2. **Keyword Search**: Uses BM25-style keyword matching
|
|
1093
|
+
* 3. **RRF Fusion**: Combines both result sets using Reciprocal Rank Fusion
|
|
1094
|
+
*
|
|
1095
|
+
* The RRF algorithm scores each document as: `sum(1 / (k + rank))` where k=60.
|
|
1096
|
+
* This gives higher scores to documents that rank well in BOTH searches.
|
|
1097
|
+
*
|
|
1098
|
+
* **Performance**: Uses adaptive sorting strategy
|
|
1099
|
+
* - Small datasets (≤ 2× limit): Full sort O(n log n)
|
|
1100
|
+
* - Large datasets: Partial selection sort O(n×k) where k=limit
|
|
1101
|
+
*
|
|
1102
|
+
* **Caching**: Results are cached for 5 minutes by default (configurable via options)
|
|
1103
|
+
*
|
|
1104
|
+
* @param query - The search query text
|
|
1105
|
+
* @param options - Search options
|
|
1106
|
+
* @param options.limit - Maximum results to return (default: 10)
|
|
1107
|
+
* @param options.filter - LanceDB filter expression (e.g., "type == 'preference'")
|
|
1108
|
+
* @param options.useCache - Enable/disable result caching (default: true)
|
|
1109
|
+
* @returns Promise with array of search results, sorted by relevance score
|
|
1110
|
+
*
|
|
1111
|
+
* @example
|
|
1112
|
+
* ```typescript
|
|
1113
|
+
* // Simple search
|
|
1114
|
+
* const results = await mesh.search("TypeScript preferences");
|
|
1115
|
+
*
|
|
1116
|
+
* // Search with filter
|
|
1117
|
+
* const code = await mesh.search("bug fix", { filter: "type == 'error'" });
|
|
1118
|
+
*
|
|
1119
|
+
* // Search with limit
|
|
1120
|
+
* const top3 = await mesh.search("security issues", { limit: 3 });
|
|
1121
|
+
* ```
|
|
1122
|
+
*
|
|
1123
|
+
* @throws {Error} If embedding generation fails
|
|
1124
|
+
* @throws {Error} If database client is not initialized
|
|
1125
|
+
*/
|
|
1126
|
+
async search(query, options = {}) {
|
|
1127
|
+
await this.init();
|
|
1128
|
+
try {
|
|
1129
|
+
const limit = options.limit || 10;
|
|
1130
|
+
const filter = options.filter || null;
|
|
1131
|
+
const useCache = options.useCache !== undefined ? options.useCache : true;
|
|
1132
|
+
if (useCache) {
|
|
1133
|
+
const cacheKey = this._generateCacheKey(query, { limit, filter });
|
|
1134
|
+
const cached = this._getCachedResult(cacheKey);
|
|
1135
|
+
if (cached) {
|
|
1136
|
+
return cached;
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
const vector = await this.embeddingFactory.embed(query);
|
|
1140
|
+
if (!this.client) {
|
|
1141
|
+
throw new Error("Database client not initialized");
|
|
1142
|
+
}
|
|
1143
|
+
const vectorResults = await this.client.search(vector, {
|
|
1144
|
+
limit: limit * 2,
|
|
1145
|
+
metric: "cosine",
|
|
1146
|
+
filter,
|
|
1147
|
+
});
|
|
1148
|
+
const keywordResults = this.keywordSearch.search(query, {
|
|
1149
|
+
limit: limit * 2,
|
|
1150
|
+
});
|
|
1151
|
+
// Optimized Reciprocal Rank Fusion (RRF) with min-heap for O(n log k) performance
|
|
1152
|
+
// Instead of sorting all results (O(n log n)), we maintain a heap of size k (O(n log k))
|
|
1153
|
+
const k = 60; // RRF constant
|
|
1154
|
+
const scores = new Map();
|
|
1155
|
+
const docMap = new Map();
|
|
1156
|
+
// Process vector results - O(m) where m = vectorResults.length
|
|
1157
|
+
for (let rank = 0; rank < vectorResults.length; rank++) {
|
|
1158
|
+
const doc = vectorResults[rank];
|
|
1159
|
+
const rrf = 1 / (k + rank + 1);
|
|
1160
|
+
scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
|
|
1161
|
+
docMap.set(doc.id, doc);
|
|
1162
|
+
}
|
|
1163
|
+
// Process keyword results - O(n) where n = keywordResults.length
|
|
1164
|
+
for (let rank = 0; rank < keywordResults.length; rank++) {
|
|
1165
|
+
const doc = keywordResults[rank];
|
|
1166
|
+
const rrf = 1 / (k + rank + 1);
|
|
1167
|
+
scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
|
|
1168
|
+
if (!docMap.has(doc.id)) {
|
|
1169
|
+
docMap.set(doc.id, {
|
|
1170
|
+
id: doc.id,
|
|
1171
|
+
content: doc.content,
|
|
1172
|
+
metadata: doc.metadata,
|
|
1173
|
+
score: 0,
|
|
1174
|
+
created_at: new Date().toISOString(),
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
// Extract top k results using min-heap pattern - O(n log k)
|
|
1179
|
+
// Since JavaScript doesn't have a built-in heap, we use an efficient approach:
|
|
1180
|
+
// Convert to array and sort only if results exceed limit significantly
|
|
1181
|
+
const scoreEntries = Array.from(scores.entries());
|
|
1182
|
+
let mergedResults;
|
|
1183
|
+
if (scoreEntries.length <= limit * 2) {
|
|
1184
|
+
// Small dataset: standard sort is fine
|
|
1185
|
+
mergedResults = scoreEntries
|
|
1186
|
+
.sort((a, b) => b[1] - a[1]) // O(n log n) but n is small
|
|
1187
|
+
.slice(0, limit)
|
|
1188
|
+
.map(([id, score]) => {
|
|
1189
|
+
const doc = docMap.get(id);
|
|
1190
|
+
return doc ? { ...doc, score } : null;
|
|
1191
|
+
})
|
|
1192
|
+
.filter((d) => d !== null);
|
|
1193
|
+
}
|
|
1194
|
+
else {
|
|
1195
|
+
// Large dataset: use partial selection sort (O(n*k) but k is small)
|
|
1196
|
+
// This is more efficient than full sort when we only need top k results
|
|
1197
|
+
const topK = [];
|
|
1198
|
+
for (const entry of scoreEntries) {
|
|
1199
|
+
if (topK.length < limit) {
|
|
1200
|
+
topK.push(entry);
|
|
1201
|
+
// Keep topK sorted in descending order
|
|
1202
|
+
topK.sort((a, b) => b[1] - a[1]);
|
|
1203
|
+
}
|
|
1204
|
+
else if (entry[1] > topK[topK.length - 1][1]) {
|
|
1205
|
+
// Replace smallest in topK if current is larger
|
|
1206
|
+
topK[limit - 1] = entry;
|
|
1207
|
+
topK.sort((a, b) => b[1] - a[1]);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
mergedResults = topK
|
|
1211
|
+
.map(([id, score]) => {
|
|
1212
|
+
const doc = docMap.get(id);
|
|
1213
|
+
return doc ? { ...doc, score } : null;
|
|
1214
|
+
})
|
|
1215
|
+
.filter((d) => d !== null);
|
|
1216
|
+
}
|
|
1217
|
+
const normalizedResults = this._normalizeScores(mergedResults);
|
|
1218
|
+
if (useCache) {
|
|
1219
|
+
const cacheKey = this._generateCacheKey(query, { limit, filter });
|
|
1220
|
+
this._cacheResult(cacheKey, normalizedResults);
|
|
1221
|
+
}
|
|
1222
|
+
if (this.enableYamo) {
|
|
1223
|
+
this._emitYamoBlock("recall", undefined, YamoEmitter.buildRecallBlock({
|
|
1224
|
+
query,
|
|
1225
|
+
resultCount: normalizedResults.length,
|
|
1226
|
+
limit,
|
|
1227
|
+
agentId: this.agentId,
|
|
1228
|
+
searchType: "hybrid",
|
|
1229
|
+
})).catch((error) => {
|
|
1230
|
+
// Log emission failures in debug mode but don't throw
|
|
1231
|
+
if (process.env.YAMO_DEBUG === "true") {
|
|
1232
|
+
logger.warn({ err: error }, "Failed to emit YAMO block (recall)");
|
|
1233
|
+
}
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
return normalizedResults;
|
|
1237
|
+
}
|
|
1238
|
+
catch (error) {
|
|
1239
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
1240
|
+
}
|
|
1328
1241
|
}
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
* Search memory using hybrid vector + keyword search with Reciprocal Rank Fusion (RRF).
|
|
1333
|
-
*
|
|
1334
|
-
* This method performs semantic search by combining:
|
|
1335
|
-
* 1. **Vector Search**: Uses embeddings to find semantically similar content
|
|
1336
|
-
* 2. **Keyword Search**: Uses BM25-style keyword matching
|
|
1337
|
-
* 3. **RRF Fusion**: Combines both result sets using Reciprocal Rank Fusion
|
|
1338
|
-
*
|
|
1339
|
-
* The RRF algorithm scores each document as: `sum(1 / (k + rank))` where k=60.
|
|
1340
|
-
* This gives higher scores to documents that rank well in BOTH searches.
|
|
1341
|
-
*
|
|
1342
|
-
* **Performance**: Uses adaptive sorting strategy
|
|
1343
|
-
* - Small datasets (≤ 2× limit): Full sort O(n log n)
|
|
1344
|
-
* - Large datasets: Partial selection sort O(n×k) where k=limit
|
|
1345
|
-
*
|
|
1346
|
-
* **Caching**: Results are cached for 5 minutes by default (configurable via options)
|
|
1347
|
-
*
|
|
1348
|
-
* @param query - The search query text
|
|
1349
|
-
* @param options - Search options
|
|
1350
|
-
* @param options.limit - Maximum results to return (default: 10)
|
|
1351
|
-
* @param options.filter - LanceDB filter expression (e.g., "type == 'preference'")
|
|
1352
|
-
* @param options.useCache - Enable/disable result caching (default: true)
|
|
1353
|
-
* @returns Promise with array of search results, sorted by relevance score
|
|
1354
|
-
*
|
|
1355
|
-
* @example
|
|
1356
|
-
* ```typescript
|
|
1357
|
-
* // Simple search
|
|
1358
|
-
* const results = await mesh.search("TypeScript preferences");
|
|
1359
|
-
*
|
|
1360
|
-
* // Search with filter
|
|
1361
|
-
* const code = await mesh.search("bug fix", { filter: "type == 'error'" });
|
|
1362
|
-
*
|
|
1363
|
-
* // Search with limit
|
|
1364
|
-
* const top3 = await mesh.search("security issues", { limit: 3 });
|
|
1365
|
-
* ```
|
|
1366
|
-
*
|
|
1367
|
-
* @throws {Error} If embedding generation fails
|
|
1368
|
-
* @throws {Error} If database client is not initialized
|
|
1369
|
-
*/
|
|
1370
|
-
async search(query: string, options: any = {}): Promise<SearchResult[]> {
|
|
1371
|
-
await this.init();
|
|
1372
|
-
try {
|
|
1373
|
-
const limit = options.limit || 10;
|
|
1374
|
-
const filter = options.filter || null;
|
|
1375
|
-
const useCache = options.useCache !== undefined ? options.useCache : true;
|
|
1376
|
-
|
|
1377
|
-
if (useCache) {
|
|
1378
|
-
const cacheKey = this._generateCacheKey(query, { limit, filter });
|
|
1379
|
-
const cached = this._getCachedResult(cacheKey);
|
|
1380
|
-
if (cached) {
|
|
1381
|
-
return cached;
|
|
1242
|
+
_normalizeScores(results) {
|
|
1243
|
+
if (results.length === 0) {
|
|
1244
|
+
return [];
|
|
1382
1245
|
}
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
});
|
|
1394
|
-
const keywordResults = this.keywordSearch.search(query, {
|
|
1395
|
-
limit: limit * 2,
|
|
1396
|
-
});
|
|
1397
|
-
|
|
1398
|
-
// Optimized Reciprocal Rank Fusion (RRF) with min-heap for O(n log k) performance
|
|
1399
|
-
// Instead of sorting all results (O(n log n)), we maintain a heap of size k (O(n log k))
|
|
1400
|
-
const k = 60; // RRF constant
|
|
1401
|
-
const scores = new Map<string, number>();
|
|
1402
|
-
const docMap = new Map<string, any>();
|
|
1403
|
-
|
|
1404
|
-
// Process vector results - O(m) where m = vectorResults.length
|
|
1405
|
-
for (let rank = 0; rank < vectorResults.length; rank++) {
|
|
1406
|
-
const doc = vectorResults[rank];
|
|
1407
|
-
const rrf = 1 / (k + rank + 1);
|
|
1408
|
-
scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
|
|
1409
|
-
docMap.set(doc.id, doc);
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
|
-
// Process keyword results - O(n) where n = keywordResults.length
|
|
1413
|
-
for (let rank = 0; rank < keywordResults.length; rank++) {
|
|
1414
|
-
const doc = keywordResults[rank];
|
|
1415
|
-
const rrf = 1 / (k + rank + 1);
|
|
1416
|
-
scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
|
|
1417
|
-
if (!docMap.has(doc.id)) {
|
|
1418
|
-
docMap.set(doc.id, {
|
|
1419
|
-
id: doc.id,
|
|
1420
|
-
content: doc.content,
|
|
1421
|
-
metadata: doc.metadata,
|
|
1422
|
-
score: 0,
|
|
1423
|
-
created_at: new Date().toISOString(),
|
|
1424
|
-
});
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
// Extract top k results using min-heap pattern - O(n log k)
|
|
1429
|
-
// Since JavaScript doesn't have a built-in heap, we use an efficient approach:
|
|
1430
|
-
// Convert to array and sort only if results exceed limit significantly
|
|
1431
|
-
const scoreEntries = Array.from(scores.entries());
|
|
1432
|
-
|
|
1433
|
-
let mergedResults: SearchResult[];
|
|
1434
|
-
if (scoreEntries.length <= limit * 2) {
|
|
1435
|
-
// Small dataset: standard sort is fine
|
|
1436
|
-
mergedResults = scoreEntries
|
|
1437
|
-
.sort((a, b) => b[1] - a[1]) // O(n log n) but n is small
|
|
1438
|
-
.slice(0, limit)
|
|
1439
|
-
.map(([id, score]) => {
|
|
1440
|
-
const doc = docMap.get(id);
|
|
1441
|
-
return doc ? { ...doc, score } : null;
|
|
1442
|
-
})
|
|
1443
|
-
.filter((d): d is SearchResult => d !== null);
|
|
1444
|
-
} else {
|
|
1445
|
-
// Large dataset: use partial selection sort (O(n*k) but k is small)
|
|
1446
|
-
// This is more efficient than full sort when we only need top k results
|
|
1447
|
-
const topK: [string, number][] = [];
|
|
1448
|
-
for (const entry of scoreEntries) {
|
|
1449
|
-
if (topK.length < limit) {
|
|
1450
|
-
topK.push(entry);
|
|
1451
|
-
// Keep topK sorted in descending order
|
|
1452
|
-
topK.sort((a, b) => b[1] - a[1]);
|
|
1453
|
-
} else if (entry[1] > topK[topK.length - 1][1]) {
|
|
1454
|
-
// Replace smallest in topK if current is larger
|
|
1455
|
-
topK[limit - 1] = entry;
|
|
1456
|
-
topK.sort((a, b) => b[1] - a[1]);
|
|
1457
|
-
}
|
|
1458
|
-
}
|
|
1459
|
-
mergedResults = topK
|
|
1460
|
-
.map(([id, score]) => {
|
|
1461
|
-
const doc = docMap.get(id);
|
|
1462
|
-
return doc ? { ...doc, score } : null;
|
|
1463
|
-
})
|
|
1464
|
-
.filter((d): d is SearchResult => d !== null);
|
|
1465
|
-
}
|
|
1466
|
-
|
|
1467
|
-
const normalizedResults = this._normalizeScores(mergedResults);
|
|
1468
|
-
if (useCache) {
|
|
1469
|
-
const cacheKey = this._generateCacheKey(query, { limit, filter });
|
|
1470
|
-
this._cacheResult(cacheKey, normalizedResults);
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
if (this.enableYamo) {
|
|
1474
|
-
this._emitYamoBlock(
|
|
1475
|
-
"recall",
|
|
1476
|
-
undefined,
|
|
1477
|
-
YamoEmitter.buildRecallBlock({
|
|
1478
|
-
query,
|
|
1479
|
-
resultCount: normalizedResults.length,
|
|
1480
|
-
limit,
|
|
1481
|
-
agentId: this.agentId,
|
|
1482
|
-
searchType: "hybrid",
|
|
1483
|
-
}),
|
|
1484
|
-
).catch((error) => {
|
|
1485
|
-
// Log emission failures in debug mode but don't throw
|
|
1486
|
-
if (process.env.YAMO_DEBUG === "true") {
|
|
1487
|
-
logger.warn({ err: error }, "Failed to emit YAMO block (recall)");
|
|
1488
|
-
}
|
|
1246
|
+
return results.map((r) => {
|
|
1247
|
+
// LanceDB _distance is squared L2 or cosine distance
|
|
1248
|
+
// For cosine distance in MiniLM, it ranges from 0 to 2
|
|
1249
|
+
const rawDistance = r._distance !== undefined ? r._distance : 1.0;
|
|
1250
|
+
// Convert to similarity score [0, 1]
|
|
1251
|
+
const score = Math.max(0, Math.min(1.0, 1 - rawDistance / 2));
|
|
1252
|
+
return {
|
|
1253
|
+
...r,
|
|
1254
|
+
score: parseFloat(score.toFixed(2)),
|
|
1255
|
+
};
|
|
1489
1256
|
});
|
|
1490
|
-
}
|
|
1491
|
-
|
|
1492
|
-
return normalizedResults;
|
|
1493
|
-
} catch (error) {
|
|
1494
|
-
throw error instanceof Error ? error : new Error(String(error));
|
|
1495
|
-
}
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
_normalizeScores(results: SearchResult[]): SearchResult[] {
|
|
1499
|
-
if (results.length === 0) {
|
|
1500
|
-
return [];
|
|
1501
1257
|
}
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
/**
|
|
1517
|
-
* Tokenize query for keyword matching (private helper for searchSkills)
|
|
1518
|
-
* Converts text to lowercase tokens, filtering out short tokens and punctuation.
|
|
1519
|
-
* Handles camelCase/PascalCase by splitting on uppercase letters.
|
|
1520
|
-
*/
|
|
1521
|
-
private _tokenizeQuery(text: string): string[] {
|
|
1522
|
-
return text
|
|
1523
|
-
.replace(/([a-z])([A-Z])/g, "$1 $2") // Split camelCase: "targetSkill" → "target Skill"
|
|
1524
|
-
.toLowerCase()
|
|
1525
|
-
.replace(/[^\w\s]/g, "")
|
|
1526
|
-
.split(/\s+/)
|
|
1527
|
-
.filter((t) => t.length > 2); // Filter out very short tokens
|
|
1528
|
-
}
|
|
1529
|
-
|
|
1530
|
-
formatResults(results: SearchResult[]): string {
|
|
1531
|
-
if (results.length === 0) {
|
|
1532
|
-
return "No relevant memories found.";
|
|
1258
|
+
/**
|
|
1259
|
+
* Tokenize query for keyword matching (private helper for searchSkills)
|
|
1260
|
+
* Converts text to lowercase tokens, filtering out short tokens and punctuation.
|
|
1261
|
+
* Handles camelCase/PascalCase by splitting on uppercase letters.
|
|
1262
|
+
*/
|
|
1263
|
+
_tokenizeQuery(text) {
|
|
1264
|
+
return text
|
|
1265
|
+
.replace(/([a-z])([A-Z])/g, "$1 $2") // Split camelCase: "targetSkill" → "target Skill"
|
|
1266
|
+
.toLowerCase()
|
|
1267
|
+
.replace(/[^\w\s]/g, "")
|
|
1268
|
+
.split(/\s+/)
|
|
1269
|
+
.filter((t) => t.length > 2); // Filter out very short tokens
|
|
1533
1270
|
}
|
|
1534
|
-
|
|
1271
|
+
formatResults(results) {
|
|
1272
|
+
if (results.length === 0) {
|
|
1273
|
+
return "No relevant memories found.";
|
|
1274
|
+
}
|
|
1275
|
+
let output = `[ATTENTION DIRECTIVE]\nThe following [MEMORY CONTEXT] is weighted by relevance.
|
|
1535
1276
|
- ALIGN attention to entries with [IMPORTANCE >= 0.8].
|
|
1536
1277
|
- TREAT entries with [IMPORTANCE <= 0.4] as auxiliary background info.
|
|
1537
1278
|
|
|
1538
1279
|
[MEMORY CONTEXT]`;
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
return output;
|
|
1547
|
-
}
|
|
1548
|
-
|
|
1549
|
-
async get(id: string): Promise<any> {
|
|
1550
|
-
await this.init();
|
|
1551
|
-
if (!this.client) {
|
|
1552
|
-
throw new Error("Database client not initialized");
|
|
1553
|
-
}
|
|
1554
|
-
const record = await this.client.getById(id);
|
|
1555
|
-
return record
|
|
1556
|
-
? {
|
|
1557
|
-
id: record.id,
|
|
1558
|
-
content: record.content,
|
|
1559
|
-
metadata: record.metadata,
|
|
1560
|
-
created_at: record.created_at,
|
|
1561
|
-
updated_at: record.updated_at,
|
|
1562
|
-
}
|
|
1563
|
-
: null;
|
|
1564
|
-
}
|
|
1565
|
-
|
|
1566
|
-
async getAll(options: any = {}): Promise<any> {
|
|
1567
|
-
await this.init();
|
|
1568
|
-
if (!this.client) {
|
|
1569
|
-
throw new Error("Database client not initialized");
|
|
1280
|
+
results.forEach((res, i) => {
|
|
1281
|
+
const metadata = typeof res.metadata === "string"
|
|
1282
|
+
? JSON.parse(res.metadata)
|
|
1283
|
+
: res.metadata;
|
|
1284
|
+
output += `\n\n--- MEMORY ${i + 1}: ${res.id} [IMPORTANCE: ${res.score}] ---\nType: ${metadata.type || "event"} | Source: ${metadata.source || "unknown"}\n${res.content}`;
|
|
1285
|
+
});
|
|
1286
|
+
return output;
|
|
1570
1287
|
}
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
};
|
|
1288
|
+
async get(id) {
|
|
1289
|
+
await this.init();
|
|
1290
|
+
if (!this.client) {
|
|
1291
|
+
throw new Error("Database client not initialized");
|
|
1292
|
+
}
|
|
1293
|
+
const record = await this.client.getById(id);
|
|
1294
|
+
return record
|
|
1295
|
+
? {
|
|
1296
|
+
id: record.id,
|
|
1297
|
+
content: record.content,
|
|
1298
|
+
metadata: record.metadata,
|
|
1299
|
+
created_at: record.created_at,
|
|
1300
|
+
updated_at: record.updated_at,
|
|
1301
|
+
}
|
|
1302
|
+
: null;
|
|
1587
1303
|
}
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1304
|
+
async getAll(options = {}) {
|
|
1305
|
+
await this.init();
|
|
1306
|
+
if (!this.client) {
|
|
1307
|
+
throw new Error("Database client not initialized");
|
|
1308
|
+
}
|
|
1309
|
+
return this.client.getAll(options);
|
|
1594
1310
|
}
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1311
|
+
async stats() {
|
|
1312
|
+
await this.init();
|
|
1313
|
+
if (!this.enableMemory || !this.client) {
|
|
1314
|
+
return {
|
|
1315
|
+
count: 0,
|
|
1316
|
+
totalMemories: 0,
|
|
1317
|
+
totalSkills: 0,
|
|
1318
|
+
tableName: "N/A",
|
|
1319
|
+
uri: "N/A",
|
|
1320
|
+
isConnected: false,
|
|
1321
|
+
embedding: { configured: false, primary: null, fallbacks: [] },
|
|
1322
|
+
status: "disabled",
|
|
1323
|
+
};
|
|
1324
|
+
}
|
|
1325
|
+
const dbStats = await this.client.getStats();
|
|
1326
|
+
// Enrich embedding stats with total persisted count
|
|
1327
|
+
const embeddingStats = this.embeddingFactory.getStats();
|
|
1328
|
+
if (embeddingStats.primary) {
|
|
1329
|
+
embeddingStats.primary.totalPersisted = dbStats.count;
|
|
1330
|
+
}
|
|
1331
|
+
// Get skill count
|
|
1332
|
+
let totalSkills = 0;
|
|
1333
|
+
if (this.skillTable) {
|
|
1334
|
+
try {
|
|
1335
|
+
const skills = await this.skillTable.query().limit(10000).toArray();
|
|
1336
|
+
totalSkills = skills.length;
|
|
1337
|
+
}
|
|
1338
|
+
catch (_e) {
|
|
1339
|
+
// Ignore errors
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
return {
|
|
1343
|
+
count: dbStats.count,
|
|
1344
|
+
totalMemories: dbStats.count,
|
|
1345
|
+
totalSkills,
|
|
1346
|
+
tableName: dbStats.tableName,
|
|
1347
|
+
uri: dbStats.uri,
|
|
1348
|
+
isConnected: dbStats.isConnected,
|
|
1349
|
+
embedding: embeddingStats,
|
|
1350
|
+
};
|
|
1605
1351
|
}
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
process.env.OPENAI_API_KEY ||
|
|
1629
|
-
process.env.COHERE_API_KEY,
|
|
1630
|
-
},
|
|
1631
|
-
];
|
|
1632
|
-
if (configs[0].modelType !== "local") {
|
|
1633
|
-
configs.push({
|
|
1634
|
-
modelType: "local",
|
|
1635
|
-
modelName: "Xenova/all-MiniLM-L6-v2",
|
|
1636
|
-
dimension: 384,
|
|
1637
|
-
priority: 2,
|
|
1638
|
-
apiKey: undefined,
|
|
1639
|
-
});
|
|
1352
|
+
_parseEmbeddingConfig() {
|
|
1353
|
+
const configs = [
|
|
1354
|
+
{
|
|
1355
|
+
modelType: process.env.EMBEDDING_MODEL_TYPE || "local",
|
|
1356
|
+
modelName: process.env.EMBEDDING_MODEL_NAME || "Xenova/all-MiniLM-L6-v2",
|
|
1357
|
+
dimension: parseInt(process.env.EMBEDDING_DIMENSION || "384"),
|
|
1358
|
+
priority: 1,
|
|
1359
|
+
apiKey: process.env.EMBEDDING_API_KEY ||
|
|
1360
|
+
process.env.OPENAI_API_KEY ||
|
|
1361
|
+
process.env.COHERE_API_KEY,
|
|
1362
|
+
},
|
|
1363
|
+
];
|
|
1364
|
+
if (configs[0].modelType !== "local") {
|
|
1365
|
+
configs.push({
|
|
1366
|
+
modelType: "local",
|
|
1367
|
+
modelName: "Xenova/all-MiniLM-L6-v2",
|
|
1368
|
+
dimension: 384,
|
|
1369
|
+
priority: 2,
|
|
1370
|
+
apiKey: undefined,
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
1373
|
+
return configs;
|
|
1640
1374
|
}
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
} catch (error) {
|
|
1682
|
-
const e = error instanceof Error ? error : new Error(String(error));
|
|
1683
|
-
logger.warn({ err: e }, "Error closing MemoryMesh");
|
|
1684
|
-
// Don't throw - cleanup should always succeed
|
|
1375
|
+
/**
|
|
1376
|
+
* Close database connections and release resources
|
|
1377
|
+
*
|
|
1378
|
+
* This should be called when done with the MemoryMesh to properly:
|
|
1379
|
+
* - Close LanceDB connections
|
|
1380
|
+
* - Release file handles
|
|
1381
|
+
* - Clean up resources
|
|
1382
|
+
*
|
|
1383
|
+
* Important for tests and cleanup to prevent connection leaks.
|
|
1384
|
+
*
|
|
1385
|
+
* @returns {Promise<void>}
|
|
1386
|
+
*
|
|
1387
|
+
* @example
|
|
1388
|
+
* ```typescript
|
|
1389
|
+
* const mesh = new MemoryMesh();
|
|
1390
|
+
* await mesh.init();
|
|
1391
|
+
* // ... use mesh ...
|
|
1392
|
+
* await mesh.close(); // Clean up
|
|
1393
|
+
* ```
|
|
1394
|
+
*/
|
|
1395
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
1396
|
+
async close() {
|
|
1397
|
+
try {
|
|
1398
|
+
// Close LanceDB client connection
|
|
1399
|
+
if (this.client) {
|
|
1400
|
+
this.client.disconnect();
|
|
1401
|
+
this.client = null;
|
|
1402
|
+
}
|
|
1403
|
+
// Clear extension table references
|
|
1404
|
+
this.yamoTable = null;
|
|
1405
|
+
this.skillTable = null;
|
|
1406
|
+
// Reset initialization state
|
|
1407
|
+
this.isInitialized = false;
|
|
1408
|
+
logger.debug("MemoryMesh closed successfully");
|
|
1409
|
+
}
|
|
1410
|
+
catch (error) {
|
|
1411
|
+
const e = error instanceof Error ? error : new Error(String(error));
|
|
1412
|
+
logger.warn({ err: e }, "Error closing MemoryMesh");
|
|
1413
|
+
// Don't throw - cleanup should always succeed
|
|
1414
|
+
}
|
|
1685
1415
|
}
|
|
1686
|
-
}
|
|
1687
1416
|
}
|
|
1688
|
-
|
|
1689
1417
|
/**
|
|
1690
1418
|
* Main CLI handler
|
|
1691
1419
|
*/
|
|
1692
1420
|
export async function run() {
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1421
|
+
let action, input;
|
|
1422
|
+
if (process.argv.length > 3) {
|
|
1423
|
+
action = process.argv[2];
|
|
1424
|
+
try {
|
|
1425
|
+
input = JSON.parse(process.argv[3]);
|
|
1426
|
+
}
|
|
1427
|
+
catch (e) {
|
|
1428
|
+
logger.error({ err: e }, "Invalid JSON argument");
|
|
1429
|
+
process.exit(1);
|
|
1430
|
+
}
|
|
1701
1431
|
}
|
|
1702
|
-
|
|
1432
|
+
else {
|
|
1433
|
+
try {
|
|
1434
|
+
const rawInput = fs.readFileSync(0, "utf8");
|
|
1435
|
+
input = JSON.parse(rawInput);
|
|
1436
|
+
action = input.action || action;
|
|
1437
|
+
}
|
|
1438
|
+
catch (_e) {
|
|
1439
|
+
logger.error("No input provided");
|
|
1440
|
+
process.exit(1);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
const mesh = new MemoryMesh({
|
|
1444
|
+
llmProvider: process.env.LLM_PROVIDER ||
|
|
1445
|
+
(process.env.OPENAI_API_KEY ? "openai" : "ollama"),
|
|
1446
|
+
llmApiKey: process.env.LLM_API_KEY || process.env.OPENAI_API_KEY,
|
|
1447
|
+
llmModel: process.env.LLM_MODEL,
|
|
1448
|
+
});
|
|
1703
1449
|
try {
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1450
|
+
if (action === "ingest" || action === "store") {
|
|
1451
|
+
const record = await mesh.add(input.content, input.metadata || {});
|
|
1452
|
+
process.stdout.write(`[MemoryMesh] Ingested record ${record.id}\n${JSON.stringify({ status: "ok", record })}\n`);
|
|
1453
|
+
}
|
|
1454
|
+
else if (action === "search") {
|
|
1455
|
+
const results = await mesh.search(input.query, {
|
|
1456
|
+
limit: input.limit || 10,
|
|
1457
|
+
filter: input.filter || null,
|
|
1458
|
+
});
|
|
1459
|
+
process.stdout.write(`[MemoryMesh] Found ${results.length} matches.\n**Formatted Context**:\n\`\`\`yamo\n${mesh.formatResults(results)}\n\`\`\`\n**Output**: memory_results.json\n\`\`\`json\n${JSON.stringify(results, null, 2)}\n\`\`\`\n${JSON.stringify({ status: "ok", results })}\n`);
|
|
1460
|
+
}
|
|
1461
|
+
else if (action === "synthesize") {
|
|
1462
|
+
const result = await mesh.synthesize({
|
|
1463
|
+
topic: input.topic,
|
|
1464
|
+
lookback: input.limit || 20,
|
|
1465
|
+
});
|
|
1466
|
+
process.stdout.write(`[MemoryMesh] Synthesis Outcome: ${result.status}\n${JSON.stringify(result, null, 2)}\n`);
|
|
1467
|
+
}
|
|
1468
|
+
else if (action === "ingest-skill") {
|
|
1469
|
+
const record = await mesh.ingestSkill(input.yamo_text, input.metadata || {});
|
|
1470
|
+
process.stdout.write(`[MemoryMesh] Ingested skill ${record.name} (${record.id})\n${JSON.stringify({ status: "ok", record })}\n`);
|
|
1471
|
+
}
|
|
1472
|
+
else if (action === "search-skills") {
|
|
1473
|
+
await mesh.init();
|
|
1474
|
+
const vector = await mesh.embeddingFactory.embed(input.query);
|
|
1475
|
+
if (mesh.skillTable) {
|
|
1476
|
+
const results = await mesh.skillTable
|
|
1477
|
+
.search(vector)
|
|
1478
|
+
.limit(input.limit || 5)
|
|
1479
|
+
.toArray();
|
|
1480
|
+
process.stdout.write(`[MemoryMesh] Found ${results.length} synthesized skills.\n${JSON.stringify({ status: "ok", results }, null, 2)}\n`);
|
|
1481
|
+
}
|
|
1482
|
+
else {
|
|
1483
|
+
process.stdout.write(`[MemoryMesh] Skill table not initialized.\n`);
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
else if (action === "skill-feedback") {
|
|
1487
|
+
const result = await mesh.updateSkillReliability(input.id, input.success !== false);
|
|
1488
|
+
process.stdout.write(`[MemoryMesh] Feedback recorded for ${input.id}: Reliability now ${result.reliability}\n${JSON.stringify({ status: "ok", ...result })}\n`);
|
|
1489
|
+
}
|
|
1490
|
+
else if (action === "skill-prune") {
|
|
1491
|
+
const result = await mesh.pruneSkills(input.threshold || 0.3);
|
|
1492
|
+
process.stdout.write(`[MemoryMesh] Pruning complete. Removed ${result.pruned_count} unreliable skills.\n${JSON.stringify({ status: "ok", ...result })}\n`);
|
|
1493
|
+
}
|
|
1494
|
+
else if (action === "stats") {
|
|
1495
|
+
process.stdout.write(`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}\n`);
|
|
1496
|
+
}
|
|
1497
|
+
else {
|
|
1498
|
+
logger.error({ action }, "Unknown action");
|
|
1499
|
+
process.exit(1);
|
|
1500
|
+
}
|
|
1710
1501
|
}
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
llmModel: process.env.LLM_MODEL,
|
|
1719
|
-
});
|
|
1720
|
-
|
|
1721
|
-
try {
|
|
1722
|
-
if (action === "ingest" || action === "store") {
|
|
1723
|
-
const record = await mesh.add(input.content, input.metadata || {});
|
|
1724
|
-
process.stdout.write(
|
|
1725
|
-
`[MemoryMesh] Ingested record ${record.id}\n${JSON.stringify({ status: "ok", record })}\n`,
|
|
1726
|
-
);
|
|
1727
|
-
} else if (action === "search") {
|
|
1728
|
-
const results = await mesh.search(input.query, {
|
|
1729
|
-
limit: input.limit || 10,
|
|
1730
|
-
filter: input.filter || null,
|
|
1731
|
-
});
|
|
1732
|
-
process.stdout.write(
|
|
1733
|
-
`[MemoryMesh] Found ${results.length} matches.\n**Formatted Context**:\n\`\`\`yamo\n${mesh.formatResults(results)}\n\`\`\`\n**Output**: memory_results.json\n\`\`\`json\n${JSON.stringify(results, null, 2)}\n\`\`\`\n${JSON.stringify({ status: "ok", results })}\n`,
|
|
1734
|
-
);
|
|
1735
|
-
} else if (action === "synthesize") {
|
|
1736
|
-
const result = await mesh.synthesize({
|
|
1737
|
-
topic: input.topic,
|
|
1738
|
-
lookback: input.limit || 20,
|
|
1739
|
-
});
|
|
1740
|
-
process.stdout.write(
|
|
1741
|
-
`[MemoryMesh] Synthesis Outcome: ${result.status}\n${JSON.stringify(result, null, 2)}\n`,
|
|
1742
|
-
);
|
|
1743
|
-
} else if (action === "ingest-skill") {
|
|
1744
|
-
const record = await mesh.ingestSkill(
|
|
1745
|
-
input.yamo_text,
|
|
1746
|
-
input.metadata || {},
|
|
1747
|
-
);
|
|
1748
|
-
process.stdout.write(
|
|
1749
|
-
`[MemoryMesh] Ingested skill ${record.name} (${record.id})\n${JSON.stringify({ status: "ok", record })}\n`,
|
|
1750
|
-
);
|
|
1751
|
-
} else if (action === "search-skills") {
|
|
1752
|
-
await mesh.init();
|
|
1753
|
-
const vector = await mesh.embeddingFactory.embed(input.query);
|
|
1754
|
-
if (mesh.skillTable) {
|
|
1755
|
-
const results = await mesh.skillTable
|
|
1756
|
-
.search(vector)
|
|
1757
|
-
.limit(input.limit || 5)
|
|
1758
|
-
.toArray();
|
|
1759
|
-
process.stdout.write(
|
|
1760
|
-
`[MemoryMesh] Found ${results.length} synthesized skills.\n${JSON.stringify({ status: "ok", results }, null, 2)}\n`,
|
|
1761
|
-
);
|
|
1762
|
-
} else {
|
|
1763
|
-
process.stdout.write(`[MemoryMesh] Skill table not initialized.\n`);
|
|
1764
|
-
}
|
|
1765
|
-
} else if (action === "skill-feedback") {
|
|
1766
|
-
const result = await mesh.updateSkillReliability(
|
|
1767
|
-
input.id,
|
|
1768
|
-
input.success !== false,
|
|
1769
|
-
);
|
|
1770
|
-
process.stdout.write(
|
|
1771
|
-
`[MemoryMesh] Feedback recorded for ${input.id}: Reliability now ${result.reliability}\n${JSON.stringify({ status: "ok", ...result })}\n`,
|
|
1772
|
-
);
|
|
1773
|
-
} else if (action === "skill-prune") {
|
|
1774
|
-
const result = await mesh.pruneSkills(input.threshold || 0.3);
|
|
1775
|
-
process.stdout.write(
|
|
1776
|
-
`[MemoryMesh] Pruning complete. Removed ${result.pruned_count} unreliable skills.\n${JSON.stringify({ status: "ok", ...result })}\n`,
|
|
1777
|
-
);
|
|
1778
|
-
} else if (action === "stats") {
|
|
1779
|
-
process.stdout.write(
|
|
1780
|
-
`[MemoryMesh] Database Statistics:\n${JSON.stringify({ status: "ok", stats: await mesh.stats() }, null, 2)}\n`,
|
|
1781
|
-
);
|
|
1782
|
-
} else {
|
|
1783
|
-
logger.error({ action }, "Unknown action");
|
|
1784
|
-
process.exit(1);
|
|
1502
|
+
catch (error) {
|
|
1503
|
+
const errorResponse = handleError(error, {
|
|
1504
|
+
action,
|
|
1505
|
+
input: { ...input, content: input.content ? "[REDACTED]" : undefined },
|
|
1506
|
+
});
|
|
1507
|
+
logger.error({ err: error, errorResponse }, "Fatal Error");
|
|
1508
|
+
process.exit(1);
|
|
1785
1509
|
}
|
|
1786
|
-
} catch (error) {
|
|
1787
|
-
const errorResponse = handleError(error, {
|
|
1788
|
-
action,
|
|
1789
|
-
input: { ...input, content: input.content ? "[REDACTED]" : undefined },
|
|
1790
|
-
});
|
|
1791
|
-
logger.error({ err: error, errorResponse }, "Fatal Error");
|
|
1792
|
-
process.exit(1);
|
|
1793
|
-
}
|
|
1794
1510
|
}
|
|
1795
|
-
|
|
1796
1511
|
export default MemoryMesh;
|
|
1797
|
-
|
|
1798
1512
|
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1513
|
+
run().catch((err) => {
|
|
1514
|
+
logger.error({ err }, "Fatal Error");
|
|
1515
|
+
process.exit(1);
|
|
1516
|
+
});
|
|
1803
1517
|
}
|