@yamo/memory-mesh 2.3.2 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -2
- package/bin/memory_mesh.js +1 -1
- package/lib/llm/client.d.ts +86 -0
- package/lib/llm/client.js +300 -357
- package/lib/llm/client.ts +334 -0
- package/lib/llm/index.d.ts +17 -0
- package/lib/llm/index.js +16 -8
- package/lib/llm/index.ts +18 -0
- package/lib/memory/adapters/client.d.ts +120 -0
- package/lib/memory/adapters/client.js +519 -0
- package/lib/memory/adapters/client.ts +519 -0
- package/lib/memory/adapters/config.d.ts +130 -0
- package/lib/memory/adapters/config.js +190 -0
- package/lib/memory/adapters/config.ts +190 -0
- package/lib/memory/adapters/errors.d.ts +84 -0
- package/lib/memory/adapters/errors.js +129 -0
- package/lib/memory/adapters/errors.ts +129 -0
- package/lib/memory/context-manager.d.ts +41 -0
- package/lib/memory/context-manager.js +345 -0
- package/lib/memory/context-manager.ts +345 -0
- package/lib/memory/embeddings/factory.d.ts +57 -0
- package/lib/memory/embeddings/factory.js +149 -0
- package/lib/memory/embeddings/factory.ts +149 -0
- package/lib/memory/embeddings/index.d.ts +2 -0
- package/lib/memory/embeddings/index.js +3 -0
- package/lib/memory/embeddings/index.ts +3 -0
- package/lib/memory/embeddings/service.d.ts +134 -0
- package/lib/memory/embeddings/service.js +516 -0
- package/lib/memory/embeddings/service.ts +516 -0
- package/lib/memory/index.d.ts +9 -0
- package/lib/memory/index.js +10 -1
- package/lib/memory/index.ts +10 -0
- package/lib/memory/memory-mesh.d.ts +332 -0
- package/lib/memory/memory-mesh.js +1470 -678
- package/lib/memory/memory-mesh.ts +1517 -0
- package/lib/memory/memory-translator.d.ts +14 -0
- package/lib/memory/memory-translator.js +126 -0
- package/lib/memory/memory-translator.ts +126 -0
- package/lib/memory/schema.d.ts +130 -0
- package/lib/memory/schema.js +184 -0
- package/lib/memory/schema.ts +184 -0
- package/lib/memory/scorer.d.ts +25 -0
- package/lib/memory/scorer.js +78 -0
- package/lib/memory/scorer.ts +78 -0
- package/lib/memory/search/index.d.ts +1 -0
- package/lib/memory/search/index.js +2 -0
- package/lib/memory/search/index.ts +2 -0
- package/lib/memory/search/keyword-search.d.ts +46 -0
- package/lib/memory/search/keyword-search.js +136 -0
- package/lib/memory/search/keyword-search.ts +136 -0
- package/lib/scrubber/config/defaults.d.ts +46 -0
- package/lib/scrubber/config/defaults.js +50 -57
- package/lib/scrubber/config/defaults.ts +55 -0
- package/lib/scrubber/errors/scrubber-error.d.ts +22 -0
- package/lib/scrubber/errors/scrubber-error.js +28 -32
- package/lib/scrubber/errors/scrubber-error.ts +44 -0
- package/lib/scrubber/index.d.ts +5 -0
- package/lib/scrubber/index.js +4 -23
- package/lib/scrubber/index.ts +6 -0
- package/lib/scrubber/scrubber.d.ts +44 -0
- package/lib/scrubber/scrubber.js +100 -121
- package/lib/scrubber/scrubber.ts +109 -0
- package/lib/scrubber/stages/chunker.d.ts +25 -0
- package/lib/scrubber/stages/chunker.js +74 -91
- package/lib/scrubber/stages/chunker.ts +104 -0
- package/lib/scrubber/stages/metadata-annotator.d.ts +17 -0
- package/lib/scrubber/stages/metadata-annotator.js +55 -65
- package/lib/scrubber/stages/metadata-annotator.ts +75 -0
- package/lib/scrubber/stages/normalizer.d.ts +16 -0
- package/lib/scrubber/stages/normalizer.js +42 -50
- package/lib/scrubber/stages/normalizer.ts +60 -0
- package/lib/scrubber/stages/semantic-filter.d.ts +16 -0
- package/lib/scrubber/stages/semantic-filter.js +42 -52
- package/lib/scrubber/stages/semantic-filter.ts +62 -0
- package/lib/scrubber/stages/structural-cleaner.d.ts +18 -0
- package/lib/scrubber/stages/structural-cleaner.js +66 -75
- package/lib/scrubber/stages/structural-cleaner.ts +83 -0
- package/lib/scrubber/stages/validator.d.ts +17 -0
- package/lib/scrubber/stages/validator.js +46 -56
- package/lib/scrubber/stages/validator.ts +67 -0
- package/lib/scrubber/telemetry.d.ts +29 -0
- package/lib/scrubber/telemetry.js +54 -58
- package/lib/scrubber/telemetry.ts +62 -0
- package/lib/scrubber/utils/hash.d.ts +14 -0
- package/lib/scrubber/utils/hash.js +30 -32
- package/lib/scrubber/utils/hash.ts +40 -0
- package/lib/scrubber/utils/html-parser.d.ts +14 -0
- package/lib/scrubber/utils/html-parser.js +32 -39
- package/lib/scrubber/utils/html-parser.ts +46 -0
- package/lib/scrubber/utils/pattern-matcher.d.ts +12 -0
- package/lib/scrubber/utils/pattern-matcher.js +48 -57
- package/lib/scrubber/utils/pattern-matcher.ts +64 -0
- package/lib/scrubber/utils/token-counter.d.ts +18 -0
- package/lib/scrubber/utils/token-counter.js +24 -25
- package/lib/scrubber/utils/token-counter.ts +32 -0
- package/lib/utils/logger.d.ts +19 -0
- package/lib/utils/logger.js +65 -0
- package/lib/utils/logger.ts +65 -0
- package/lib/utils/skill-metadata.d.ts +24 -0
- package/lib/utils/skill-metadata.js +133 -0
- package/lib/utils/skill-metadata.ts +133 -0
- package/lib/yamo/emitter.d.ts +46 -0
- package/lib/yamo/emitter.js +79 -143
- package/lib/yamo/emitter.ts +171 -0
- package/lib/yamo/index.d.ts +14 -0
- package/lib/yamo/index.js +6 -7
- package/lib/yamo/index.ts +16 -0
- package/lib/yamo/schema.d.ts +56 -0
- package/lib/yamo/schema.js +82 -108
- package/lib/yamo/schema.ts +133 -0
- package/package.json +13 -8
- package/index.d.ts +0 -111
- package/lib/embeddings/factory.js +0 -151
- package/lib/embeddings/index.js +0 -2
- package/lib/embeddings/service.js +0 -586
- package/lib/index.js +0 -6
- package/lib/lancedb/client.js +0 -633
- package/lib/lancedb/config.js +0 -215
- package/lib/lancedb/errors.js +0 -144
- package/lib/lancedb/index.js +0 -4
- package/lib/lancedb/schema.js +0 -217
- package/lib/search/index.js +0 -1
- package/lib/search/keyword-search.js +0 -144
- package/lib/utils/index.js +0 -1
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
/**
|
|
3
|
+
* LanceDB Configuration Loader
|
|
4
|
+
* Loads and validates configuration from environment variables
|
|
5
|
+
*/
|
|
6
|
+
import path from "path";
|
|
7
|
+
/**
|
|
8
|
+
* Default configuration values
|
|
9
|
+
*/
|
|
10
|
+
export const DEFAULTS = {
|
|
11
|
+
// LanceDB Configuration
|
|
12
|
+
LANCEDB_URI: "./runtime/data/lancedb",
|
|
13
|
+
LANCEDB_MEMORY_TABLE: "memory_entries",
|
|
14
|
+
LANCEDB_MAX_CACHE_SIZE: "2GB",
|
|
15
|
+
// Embedding Model Configuration
|
|
16
|
+
EMBEDDING_MODEL_TYPE: "local",
|
|
17
|
+
EMBEDDING_MODEL_NAME: "Xenova/all-MiniLM-L6-v2",
|
|
18
|
+
EMBEDDING_DIMENSION: "384",
|
|
19
|
+
EMBEDDING_BATCH_SIZE: "32",
|
|
20
|
+
EMBEDDING_NORMALIZE: "true",
|
|
21
|
+
// API-based Embeddings
|
|
22
|
+
OPENAI_EMBEDDING_MODEL: "text-embedding-3-small",
|
|
23
|
+
// Search Configuration
|
|
24
|
+
DEFAULT_TOP_K: "10",
|
|
25
|
+
DEFAULT_SIMILARITY_THRESHOLD: "0.7",
|
|
26
|
+
ENABLE_HYBRID_SEARCH: "true",
|
|
27
|
+
HYBRID_SEARCH_ALPHA: "0.5",
|
|
28
|
+
// Performance Tuning
|
|
29
|
+
VECTOR_INDEX_TYPE: "ivf_pq",
|
|
30
|
+
IVF_PARTITIONS: "256",
|
|
31
|
+
PQ_BITS: "8",
|
|
32
|
+
ENABLE_QUERY_CACHE: "true",
|
|
33
|
+
QUERY_CACHE_TTL: "300",
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Memory system configuration defaults
|
|
37
|
+
*/
|
|
38
|
+
export const MEMORY_DEFAULTS = {
|
|
39
|
+
// Feature flags
|
|
40
|
+
MEMORY_ENABLED: "true",
|
|
41
|
+
MEMORY_AUTO_CAPTURE: "true",
|
|
42
|
+
MEMORY_AUTO_RECALL: "true",
|
|
43
|
+
// Recall settings
|
|
44
|
+
MEMORY_MAX_CONTEXT: "5",
|
|
45
|
+
MEMORY_RELEVANCE_THRESHOLD: "0.7",
|
|
46
|
+
MEMORY_IMPORTANCE_BOOST: "1.5",
|
|
47
|
+
MEMORY_RECENCY_WEIGHT: "0.3",
|
|
48
|
+
// Capture settings
|
|
49
|
+
MEMORY_MIN_IMPORTANCE: "0.3",
|
|
50
|
+
MEMORY_DEDUP_THRESHOLD: "0.9",
|
|
51
|
+
MEMORY_CAPTURE_TOOL_RESULTS: "true",
|
|
52
|
+
MEMORY_CAPTURE_FILE_OPS: "true",
|
|
53
|
+
// Retention settings
|
|
54
|
+
MEMORY_RETENTION_ENABLED: "true",
|
|
55
|
+
MEMORY_RETENTION_DAYS: "90",
|
|
56
|
+
MEMORY_MAX_PER_SESSION: "100",
|
|
57
|
+
MEMORY_MIN_IMPORTANCE_TO_KEEP: "0.5",
|
|
58
|
+
// Privacy settings
|
|
59
|
+
MEMORY_REDACT_PII: "false",
|
|
60
|
+
MEMORY_ENCRYPTION_ENABLED: "false",
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Load configuration with validation
|
|
64
|
+
*/
|
|
65
|
+
export function loadConfig() {
|
|
66
|
+
const config = {};
|
|
67
|
+
for (const [key, defaultValue] of Object.entries(DEFAULTS)) {
|
|
68
|
+
config[key] = process.env[key] || defaultValue;
|
|
69
|
+
}
|
|
70
|
+
// Resolve relative paths to absolute relative to package root
|
|
71
|
+
if (config.LANCEDB_URI &&
|
|
72
|
+
(config.LANCEDB_URI.startsWith("./") ||
|
|
73
|
+
config.LANCEDB_URI.startsWith("../"))) {
|
|
74
|
+
const currentFileUrl = import.meta.url;
|
|
75
|
+
const currentFilePath = new URL(currentFileUrl).pathname;
|
|
76
|
+
// config.ts is in lib/brain/adapters, so package root is ../../../
|
|
77
|
+
const packageRoot = path.resolve(path.dirname(currentFilePath), "../../../");
|
|
78
|
+
config.LANCEDB_URI = path.resolve(packageRoot, config.LANCEDB_URI);
|
|
79
|
+
}
|
|
80
|
+
return config;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Load memory-specific configuration
|
|
84
|
+
* @returns {Object} Memory configuration object
|
|
85
|
+
*/
|
|
86
|
+
export function loadMemoryConfig() {
|
|
87
|
+
return {
|
|
88
|
+
enabled: process.env.MEMORY_ENABLED !== "false",
|
|
89
|
+
autoCapture: process.env.MEMORY_AUTO_CAPTURE !== "false",
|
|
90
|
+
autoRecall: process.env.MEMORY_AUTO_RECALL !== "false",
|
|
91
|
+
maxContext: parseInt(process.env.MEMORY_MAX_CONTEXT || "5"),
|
|
92
|
+
relevanceThreshold: parseFloat(process.env.MEMORY_RELEVANCE_THRESHOLD || "0.7"),
|
|
93
|
+
importanceBoost: parseFloat(process.env.MEMORY_IMPORTANCE_BOOST || "1.5"),
|
|
94
|
+
recencyWeight: parseFloat(process.env.MEMORY_RECENCY_WEIGHT || "0.3"),
|
|
95
|
+
minImportance: parseFloat(process.env.MEMORY_MIN_IMPORTANCE || "0.3"),
|
|
96
|
+
dedupThreshold: parseFloat(process.env.MEMORY_DEDUP_THRESHOLD || "0.9"),
|
|
97
|
+
captureToolResults: process.env.MEMORY_CAPTURE_TOOL_RESULTS !== "false",
|
|
98
|
+
captureFileOps: process.env.MEMORY_CAPTURE_FILE_OPS !== "false",
|
|
99
|
+
retention: {
|
|
100
|
+
enabled: process.env.MEMORY_RETENTION_ENABLED !== "false",
|
|
101
|
+
days: parseInt(process.env.MEMORY_RETENTION_DAYS || "90"),
|
|
102
|
+
maxPerSession: parseInt(process.env.MEMORY_MAX_PER_SESSION || "100"),
|
|
103
|
+
minImportanceToKeep: parseFloat(process.env.MEMORY_MIN_IMPORTANCE_TO_KEEP || "0.5"),
|
|
104
|
+
},
|
|
105
|
+
privacy: {
|
|
106
|
+
redactPii: process.env.MEMORY_REDACT_PII === "true",
|
|
107
|
+
encryptionEnabled: process.env.MEMORY_ENCRYPTION_ENABLED === "true",
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Validate configuration
|
|
113
|
+
*/
|
|
114
|
+
export function validateConfig(config) {
|
|
115
|
+
const errors = [];
|
|
116
|
+
// Validate embedding model type
|
|
117
|
+
const validModelTypes = ["local", "openai", "cohere", "voyage"];
|
|
118
|
+
if (!validModelTypes.includes(config.EMBEDDING_MODEL_TYPE)) {
|
|
119
|
+
errors.push(`Invalid EMBEDDING_MODEL_TYPE: ${config.EMBEDDING_MODEL_TYPE}`);
|
|
120
|
+
}
|
|
121
|
+
// Validate numeric values
|
|
122
|
+
const dimension = parseInt(config.EMBEDDING_DIMENSION);
|
|
123
|
+
if (isNaN(dimension) || dimension <= 0) {
|
|
124
|
+
errors.push(`Invalid EMBEDDING_DIMENSION: ${config.EMBEDDING_DIMENSION}`);
|
|
125
|
+
}
|
|
126
|
+
const topK = parseInt(config.DEFAULT_TOP_K);
|
|
127
|
+
if (isNaN(topK) || topK <= 0) {
|
|
128
|
+
errors.push(`Invalid DEFAULT_TOP_K: ${config.DEFAULT_TOP_K}`);
|
|
129
|
+
}
|
|
130
|
+
// Validate boolean strings
|
|
131
|
+
const boolFields = [
|
|
132
|
+
"EMBEDDING_NORMALIZE",
|
|
133
|
+
"ENABLE_HYBRID_SEARCH",
|
|
134
|
+
"ENABLE_QUERY_CACHE",
|
|
135
|
+
];
|
|
136
|
+
for (const field of boolFields) {
|
|
137
|
+
const value = config[field].toLowerCase();
|
|
138
|
+
if (value !== "true" && value !== "false") {
|
|
139
|
+
errors.push(`Invalid ${field}: must be 'true' or 'false'`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// Validate similarity threshold (0-1 range)
|
|
143
|
+
const threshold = parseFloat(config.DEFAULT_SIMILARITY_THRESHOLD);
|
|
144
|
+
if (isNaN(threshold) || threshold < 0 || threshold > 1) {
|
|
145
|
+
errors.push(`Invalid DEFAULT_SIMILARITY_THRESHOLD: must be between 0 and 1`);
|
|
146
|
+
}
|
|
147
|
+
// Validate hybrid search alpha (0-1 range)
|
|
148
|
+
const alpha = parseFloat(config.HYBRID_SEARCH_ALPHA);
|
|
149
|
+
if (isNaN(alpha) || alpha < 0 || alpha > 1) {
|
|
150
|
+
errors.push(`Invalid HYBRID_SEARCH_ALPHA: must be between 0 and 1`);
|
|
151
|
+
}
|
|
152
|
+
// Validate positive integers
|
|
153
|
+
const positiveIntFields = [
|
|
154
|
+
"EMBEDDING_BATCH_SIZE",
|
|
155
|
+
"IVF_PARTITIONS",
|
|
156
|
+
"PQ_BITS",
|
|
157
|
+
"QUERY_CACHE_TTL",
|
|
158
|
+
];
|
|
159
|
+
for (const field of positiveIntFields) {
|
|
160
|
+
const value = parseInt(config[field]);
|
|
161
|
+
if (isNaN(value) || value <= 0) {
|
|
162
|
+
errors.push(`Invalid ${field}: must be a positive integer`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// Validate cache size format (e.g., "2GB", "500MB")
|
|
166
|
+
const cacheSizePattern = /^\d+(\.\d+)?(KB|MB|GB|TB)$/;
|
|
167
|
+
if (!cacheSizePattern.test(config.LANCEDB_MAX_CACHE_SIZE)) {
|
|
168
|
+
errors.push(`Invalid LANCEDB_MAX_CACHE_SIZE: must match pattern like "2GB", "500MB"`);
|
|
169
|
+
}
|
|
170
|
+
return errors;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Get validated configuration
|
|
174
|
+
*/
|
|
175
|
+
export function getConfig() {
|
|
176
|
+
const config = loadConfig();
|
|
177
|
+
const errors = validateConfig(config);
|
|
178
|
+
if (errors.length > 0) {
|
|
179
|
+
throw new Error(`Configuration validation failed:\n${errors.join("\n")}`);
|
|
180
|
+
}
|
|
181
|
+
return config;
|
|
182
|
+
}
|
|
183
|
+
export default {
|
|
184
|
+
loadConfig,
|
|
185
|
+
validateConfig,
|
|
186
|
+
getConfig,
|
|
187
|
+
loadMemoryConfig,
|
|
188
|
+
DEFAULTS,
|
|
189
|
+
MEMORY_DEFAULTS,
|
|
190
|
+
};
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
/**
|
|
3
|
+
* LanceDB Configuration Loader
|
|
4
|
+
* Loads and validates configuration from environment variables
|
|
5
|
+
*/
|
|
6
|
+
import path from "path";
|
|
7
|
+
/**
|
|
8
|
+
* Default configuration values
|
|
9
|
+
*/
|
|
10
|
+
export const DEFAULTS = {
|
|
11
|
+
// LanceDB Configuration
|
|
12
|
+
LANCEDB_URI: "./runtime/data/lancedb",
|
|
13
|
+
LANCEDB_MEMORY_TABLE: "memory_entries",
|
|
14
|
+
LANCEDB_MAX_CACHE_SIZE: "2GB",
|
|
15
|
+
// Embedding Model Configuration
|
|
16
|
+
EMBEDDING_MODEL_TYPE: "local",
|
|
17
|
+
EMBEDDING_MODEL_NAME: "Xenova/all-MiniLM-L6-v2",
|
|
18
|
+
EMBEDDING_DIMENSION: "384",
|
|
19
|
+
EMBEDDING_BATCH_SIZE: "32",
|
|
20
|
+
EMBEDDING_NORMALIZE: "true",
|
|
21
|
+
// API-based Embeddings
|
|
22
|
+
OPENAI_EMBEDDING_MODEL: "text-embedding-3-small",
|
|
23
|
+
// Search Configuration
|
|
24
|
+
DEFAULT_TOP_K: "10",
|
|
25
|
+
DEFAULT_SIMILARITY_THRESHOLD: "0.7",
|
|
26
|
+
ENABLE_HYBRID_SEARCH: "true",
|
|
27
|
+
HYBRID_SEARCH_ALPHA: "0.5",
|
|
28
|
+
// Performance Tuning
|
|
29
|
+
VECTOR_INDEX_TYPE: "ivf_pq",
|
|
30
|
+
IVF_PARTITIONS: "256",
|
|
31
|
+
PQ_BITS: "8",
|
|
32
|
+
ENABLE_QUERY_CACHE: "true",
|
|
33
|
+
QUERY_CACHE_TTL: "300",
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Memory system configuration defaults
|
|
37
|
+
*/
|
|
38
|
+
export const MEMORY_DEFAULTS = {
|
|
39
|
+
// Feature flags
|
|
40
|
+
MEMORY_ENABLED: "true",
|
|
41
|
+
MEMORY_AUTO_CAPTURE: "true",
|
|
42
|
+
MEMORY_AUTO_RECALL: "true",
|
|
43
|
+
// Recall settings
|
|
44
|
+
MEMORY_MAX_CONTEXT: "5",
|
|
45
|
+
MEMORY_RELEVANCE_THRESHOLD: "0.7",
|
|
46
|
+
MEMORY_IMPORTANCE_BOOST: "1.5",
|
|
47
|
+
MEMORY_RECENCY_WEIGHT: "0.3",
|
|
48
|
+
// Capture settings
|
|
49
|
+
MEMORY_MIN_IMPORTANCE: "0.3",
|
|
50
|
+
MEMORY_DEDUP_THRESHOLD: "0.9",
|
|
51
|
+
MEMORY_CAPTURE_TOOL_RESULTS: "true",
|
|
52
|
+
MEMORY_CAPTURE_FILE_OPS: "true",
|
|
53
|
+
// Retention settings
|
|
54
|
+
MEMORY_RETENTION_ENABLED: "true",
|
|
55
|
+
MEMORY_RETENTION_DAYS: "90",
|
|
56
|
+
MEMORY_MAX_PER_SESSION: "100",
|
|
57
|
+
MEMORY_MIN_IMPORTANCE_TO_KEEP: "0.5",
|
|
58
|
+
// Privacy settings
|
|
59
|
+
MEMORY_REDACT_PII: "false",
|
|
60
|
+
MEMORY_ENCRYPTION_ENABLED: "false",
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Load configuration with validation
|
|
64
|
+
*/
|
|
65
|
+
export function loadConfig() {
|
|
66
|
+
const config = {};
|
|
67
|
+
for (const [key, defaultValue] of Object.entries(DEFAULTS)) {
|
|
68
|
+
config[key] = process.env[key] || defaultValue;
|
|
69
|
+
}
|
|
70
|
+
// Resolve relative paths to absolute relative to package root
|
|
71
|
+
if (config.LANCEDB_URI &&
|
|
72
|
+
(config.LANCEDB_URI.startsWith("./") ||
|
|
73
|
+
config.LANCEDB_URI.startsWith("../"))) {
|
|
74
|
+
const currentFileUrl = import.meta.url;
|
|
75
|
+
const currentFilePath = new URL(currentFileUrl).pathname;
|
|
76
|
+
// config.ts is in lib/brain/adapters, so package root is ../../../
|
|
77
|
+
const packageRoot = path.resolve(path.dirname(currentFilePath), "../../../");
|
|
78
|
+
config.LANCEDB_URI = path.resolve(packageRoot, config.LANCEDB_URI);
|
|
79
|
+
}
|
|
80
|
+
return config;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Load memory-specific configuration
|
|
84
|
+
* @returns {Object} Memory configuration object
|
|
85
|
+
*/
|
|
86
|
+
export function loadMemoryConfig() {
|
|
87
|
+
return {
|
|
88
|
+
enabled: process.env.MEMORY_ENABLED !== "false",
|
|
89
|
+
autoCapture: process.env.MEMORY_AUTO_CAPTURE !== "false",
|
|
90
|
+
autoRecall: process.env.MEMORY_AUTO_RECALL !== "false",
|
|
91
|
+
maxContext: parseInt(process.env.MEMORY_MAX_CONTEXT || "5"),
|
|
92
|
+
relevanceThreshold: parseFloat(process.env.MEMORY_RELEVANCE_THRESHOLD || "0.7"),
|
|
93
|
+
importanceBoost: parseFloat(process.env.MEMORY_IMPORTANCE_BOOST || "1.5"),
|
|
94
|
+
recencyWeight: parseFloat(process.env.MEMORY_RECENCY_WEIGHT || "0.3"),
|
|
95
|
+
minImportance: parseFloat(process.env.MEMORY_MIN_IMPORTANCE || "0.3"),
|
|
96
|
+
dedupThreshold: parseFloat(process.env.MEMORY_DEDUP_THRESHOLD || "0.9"),
|
|
97
|
+
captureToolResults: process.env.MEMORY_CAPTURE_TOOL_RESULTS !== "false",
|
|
98
|
+
captureFileOps: process.env.MEMORY_CAPTURE_FILE_OPS !== "false",
|
|
99
|
+
retention: {
|
|
100
|
+
enabled: process.env.MEMORY_RETENTION_ENABLED !== "false",
|
|
101
|
+
days: parseInt(process.env.MEMORY_RETENTION_DAYS || "90"),
|
|
102
|
+
maxPerSession: parseInt(process.env.MEMORY_MAX_PER_SESSION || "100"),
|
|
103
|
+
minImportanceToKeep: parseFloat(process.env.MEMORY_MIN_IMPORTANCE_TO_KEEP || "0.5"),
|
|
104
|
+
},
|
|
105
|
+
privacy: {
|
|
106
|
+
redactPii: process.env.MEMORY_REDACT_PII === "true",
|
|
107
|
+
encryptionEnabled: process.env.MEMORY_ENCRYPTION_ENABLED === "true",
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Validate configuration
|
|
113
|
+
*/
|
|
114
|
+
export function validateConfig(config) {
|
|
115
|
+
const errors = [];
|
|
116
|
+
// Validate embedding model type
|
|
117
|
+
const validModelTypes = ["local", "openai", "cohere", "voyage"];
|
|
118
|
+
if (!validModelTypes.includes(config.EMBEDDING_MODEL_TYPE)) {
|
|
119
|
+
errors.push(`Invalid EMBEDDING_MODEL_TYPE: ${config.EMBEDDING_MODEL_TYPE}`);
|
|
120
|
+
}
|
|
121
|
+
// Validate numeric values
|
|
122
|
+
const dimension = parseInt(config.EMBEDDING_DIMENSION);
|
|
123
|
+
if (isNaN(dimension) || dimension <= 0) {
|
|
124
|
+
errors.push(`Invalid EMBEDDING_DIMENSION: ${config.EMBEDDING_DIMENSION}`);
|
|
125
|
+
}
|
|
126
|
+
const topK = parseInt(config.DEFAULT_TOP_K);
|
|
127
|
+
if (isNaN(topK) || topK <= 0) {
|
|
128
|
+
errors.push(`Invalid DEFAULT_TOP_K: ${config.DEFAULT_TOP_K}`);
|
|
129
|
+
}
|
|
130
|
+
// Validate boolean strings
|
|
131
|
+
const boolFields = [
|
|
132
|
+
"EMBEDDING_NORMALIZE",
|
|
133
|
+
"ENABLE_HYBRID_SEARCH",
|
|
134
|
+
"ENABLE_QUERY_CACHE",
|
|
135
|
+
];
|
|
136
|
+
for (const field of boolFields) {
|
|
137
|
+
const value = config[field].toLowerCase();
|
|
138
|
+
if (value !== "true" && value !== "false") {
|
|
139
|
+
errors.push(`Invalid ${field}: must be 'true' or 'false'`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// Validate similarity threshold (0-1 range)
|
|
143
|
+
const threshold = parseFloat(config.DEFAULT_SIMILARITY_THRESHOLD);
|
|
144
|
+
if (isNaN(threshold) || threshold < 0 || threshold > 1) {
|
|
145
|
+
errors.push(`Invalid DEFAULT_SIMILARITY_THRESHOLD: must be between 0 and 1`);
|
|
146
|
+
}
|
|
147
|
+
// Validate hybrid search alpha (0-1 range)
|
|
148
|
+
const alpha = parseFloat(config.HYBRID_SEARCH_ALPHA);
|
|
149
|
+
if (isNaN(alpha) || alpha < 0 || alpha > 1) {
|
|
150
|
+
errors.push(`Invalid HYBRID_SEARCH_ALPHA: must be between 0 and 1`);
|
|
151
|
+
}
|
|
152
|
+
// Validate positive integers
|
|
153
|
+
const positiveIntFields = [
|
|
154
|
+
"EMBEDDING_BATCH_SIZE",
|
|
155
|
+
"IVF_PARTITIONS",
|
|
156
|
+
"PQ_BITS",
|
|
157
|
+
"QUERY_CACHE_TTL",
|
|
158
|
+
];
|
|
159
|
+
for (const field of positiveIntFields) {
|
|
160
|
+
const value = parseInt(config[field]);
|
|
161
|
+
if (isNaN(value) || value <= 0) {
|
|
162
|
+
errors.push(`Invalid ${field}: must be a positive integer`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// Validate cache size format (e.g., "2GB", "500MB")
|
|
166
|
+
const cacheSizePattern = /^\d+(\.\d+)?(KB|MB|GB|TB)$/;
|
|
167
|
+
if (!cacheSizePattern.test(config.LANCEDB_MAX_CACHE_SIZE)) {
|
|
168
|
+
errors.push(`Invalid LANCEDB_MAX_CACHE_SIZE: must match pattern like "2GB", "500MB"`);
|
|
169
|
+
}
|
|
170
|
+
return errors;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Get validated configuration
|
|
174
|
+
*/
|
|
175
|
+
export function getConfig() {
|
|
176
|
+
const config = loadConfig();
|
|
177
|
+
const errors = validateConfig(config);
|
|
178
|
+
if (errors.length > 0) {
|
|
179
|
+
throw new Error(`Configuration validation failed:\n${errors.join("\n")}`);
|
|
180
|
+
}
|
|
181
|
+
return config;
|
|
182
|
+
}
|
|
183
|
+
export default {
|
|
184
|
+
loadConfig,
|
|
185
|
+
validateConfig,
|
|
186
|
+
getConfig,
|
|
187
|
+
loadMemoryConfig,
|
|
188
|
+
DEFAULTS,
|
|
189
|
+
MEMORY_DEFAULTS,
|
|
190
|
+
};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Custom error classes for LanceDB operations
|
|
3
|
+
*
|
|
4
|
+
* Base error class for all LanceDB-related errors. Captures proper stack traces
|
|
5
|
+
* to ensure debugging information points to where errors are thrown, not to the
|
|
6
|
+
* error constructor.
|
|
7
|
+
*/
|
|
8
|
+
export declare class LanceDBError extends Error {
|
|
9
|
+
code: any;
|
|
10
|
+
details: any;
|
|
11
|
+
timestamp: any;
|
|
12
|
+
/**
|
|
13
|
+
* Create a new LanceDBError
|
|
14
|
+
* @param {string} message - Human-readable error message
|
|
15
|
+
* @param {string} code - Machine-readable error code (e.g., 'EMBEDDING_ERROR')
|
|
16
|
+
* @param {Object} details - Additional error context and metadata
|
|
17
|
+
*/
|
|
18
|
+
constructor(message: any, code: any, details?: {});
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Error raised when embedding generation or comparison fails
|
|
22
|
+
*/
|
|
23
|
+
export declare class EmbeddingError extends LanceDBError {
|
|
24
|
+
constructor(message: any, details: any);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Error raised when storage operations (read/write/delete) fail
|
|
28
|
+
*/
|
|
29
|
+
export declare class StorageError extends LanceDBError {
|
|
30
|
+
constructor(message: any, details: any);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Error raised when database queries fail or return invalid results
|
|
34
|
+
*/
|
|
35
|
+
export declare class QueryError extends LanceDBError {
|
|
36
|
+
constructor(message: any, details: any);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Error raised when configuration is missing or invalid
|
|
40
|
+
*/
|
|
41
|
+
export declare class ConfigurationError extends LanceDBError {
|
|
42
|
+
constructor(message: any, details: any);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Sanitize error messages by redacting sensitive information
|
|
46
|
+
* @param {string} message - Error message to sanitize
|
|
47
|
+
* @returns {string} Sanitized error message
|
|
48
|
+
*/
|
|
49
|
+
export declare function sanitizeErrorMessage(message: any): string;
|
|
50
|
+
/**
|
|
51
|
+
* Normalize errors into a consistent response format
|
|
52
|
+
* @param {Error} error - The error to handle
|
|
53
|
+
* @param {Object} context - Additional context about where/when the error occurred
|
|
54
|
+
* @returns {Object} Formatted error response with success: false
|
|
55
|
+
*/
|
|
56
|
+
export declare function handleError(error: any, context?: {}): {
|
|
57
|
+
success: boolean;
|
|
58
|
+
error: {
|
|
59
|
+
code: any;
|
|
60
|
+
message: string;
|
|
61
|
+
details: any;
|
|
62
|
+
context: {};
|
|
63
|
+
stack?: undefined;
|
|
64
|
+
};
|
|
65
|
+
} | {
|
|
66
|
+
success: boolean;
|
|
67
|
+
error: {
|
|
68
|
+
code: string;
|
|
69
|
+
message: string;
|
|
70
|
+
stack: string;
|
|
71
|
+
context: {};
|
|
72
|
+
details?: undefined;
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
declare const _default: {
|
|
76
|
+
LanceDBError: typeof LanceDBError;
|
|
77
|
+
EmbeddingError: typeof EmbeddingError;
|
|
78
|
+
StorageError: typeof StorageError;
|
|
79
|
+
QueryError: typeof QueryError;
|
|
80
|
+
ConfigurationError: typeof ConfigurationError;
|
|
81
|
+
handleError: typeof handleError;
|
|
82
|
+
sanitizeErrorMessage: typeof sanitizeErrorMessage;
|
|
83
|
+
};
|
|
84
|
+
export default _default;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
/**
|
|
3
|
+
* Custom error classes for LanceDB operations
|
|
4
|
+
*
|
|
5
|
+
* Base error class for all LanceDB-related errors. Captures proper stack traces
|
|
6
|
+
* to ensure debugging information points to where errors are thrown, not to the
|
|
7
|
+
* error constructor.
|
|
8
|
+
*/
|
|
9
|
+
export class LanceDBError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
details;
|
|
12
|
+
timestamp;
|
|
13
|
+
/**
|
|
14
|
+
* Create a new LanceDBError
|
|
15
|
+
* @param {string} message - Human-readable error message
|
|
16
|
+
* @param {string} code - Machine-readable error code (e.g., 'EMBEDDING_ERROR')
|
|
17
|
+
* @param {Object} details - Additional error context and metadata
|
|
18
|
+
*/
|
|
19
|
+
constructor(message, code, details = {}) {
|
|
20
|
+
super(message);
|
|
21
|
+
this.name = "LanceDBError";
|
|
22
|
+
this.code = code;
|
|
23
|
+
this.details = details;
|
|
24
|
+
this.timestamp = new Date().toISOString();
|
|
25
|
+
// Capture stack trace for proper debugging (Node.js best practice)
|
|
26
|
+
// This ensures stack traces point to where the error was thrown,
|
|
27
|
+
// not to the error constructor itself
|
|
28
|
+
Error.captureStackTrace(this, this.constructor);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Error raised when embedding generation or comparison fails
|
|
33
|
+
*/
|
|
34
|
+
export class EmbeddingError extends LanceDBError {
|
|
35
|
+
constructor(message, details) {
|
|
36
|
+
super(message, "EMBEDDING_ERROR", details);
|
|
37
|
+
this.name = "EmbeddingError";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Error raised when storage operations (read/write/delete) fail
|
|
42
|
+
*/
|
|
43
|
+
export class StorageError extends LanceDBError {
|
|
44
|
+
constructor(message, details) {
|
|
45
|
+
super(message, "STORAGE_ERROR", details);
|
|
46
|
+
this.name = "StorageError";
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Error raised when database queries fail or return invalid results
|
|
51
|
+
*/
|
|
52
|
+
export class QueryError extends LanceDBError {
|
|
53
|
+
constructor(message, details) {
|
|
54
|
+
super(message, "QUERY_ERROR", details);
|
|
55
|
+
this.name = "QueryError";
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Error raised when configuration is missing or invalid
|
|
60
|
+
*/
|
|
61
|
+
export class ConfigurationError extends LanceDBError {
|
|
62
|
+
constructor(message, details) {
|
|
63
|
+
super(message, "CONFIGURATION_ERROR", details);
|
|
64
|
+
this.name = "ConfigurationError";
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Sanitize error messages by redacting sensitive information
|
|
69
|
+
* @param {string} message - Error message to sanitize
|
|
70
|
+
* @returns {string} Sanitized error message
|
|
71
|
+
*/
|
|
72
|
+
export function sanitizeErrorMessage(message) {
|
|
73
|
+
if (typeof message !== "string") {
|
|
74
|
+
return "[Non-string error message]";
|
|
75
|
+
}
|
|
76
|
+
// Redact common sensitive patterns
|
|
77
|
+
return (message
|
|
78
|
+
// Redact Bearer tokens
|
|
79
|
+
.replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, "Bearer [REDACTED]")
|
|
80
|
+
// Redact OpenAI API keys (sk- followed by 32+ chars)
|
|
81
|
+
.replace(/sk-[A-Za-z0-9]{32,}/g, "sk-[REDACTED]")
|
|
82
|
+
// Redact generic API keys (20+ alphanumeric chars after api_key)
|
|
83
|
+
.replace(/api_key["\s:]+[A-Za-z0-9]{20,}/gi, "api_key: [REDACTED]")
|
|
84
|
+
// Redact environment variable patterns that might contain secrets
|
|
85
|
+
.replace(/(OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY)["='\s]+[A-Za-z0-9\-_]+/gi, "$1=[REDACTED]")
|
|
86
|
+
// Redact Authorization headers
|
|
87
|
+
.replace(/Authorization:\s*[^"\r\n]+/gi, "Authorization: [REDACTED]")
|
|
88
|
+
// Redact potential JWT tokens
|
|
89
|
+
.replace(/eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g, "[JWT_REDACTED]"));
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Normalize errors into a consistent response format
|
|
93
|
+
* @param {Error} error - The error to handle
|
|
94
|
+
* @param {Object} context - Additional context about where/when the error occurred
|
|
95
|
+
* @returns {Object} Formatted error response with success: false
|
|
96
|
+
*/
|
|
97
|
+
export function handleError(error, context = {}) {
|
|
98
|
+
if (error instanceof LanceDBError) {
|
|
99
|
+
return {
|
|
100
|
+
success: false,
|
|
101
|
+
error: {
|
|
102
|
+
code: error.code,
|
|
103
|
+
message: sanitizeErrorMessage(error.message),
|
|
104
|
+
details: error.details,
|
|
105
|
+
context,
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
110
|
+
// Wrap unknown errors
|
|
111
|
+
return {
|
|
112
|
+
success: false,
|
|
113
|
+
error: {
|
|
114
|
+
code: "UNKNOWN_ERROR",
|
|
115
|
+
message: sanitizeErrorMessage(err.message),
|
|
116
|
+
stack: process.env.NODE_ENV === "development" ? err.stack : undefined,
|
|
117
|
+
context,
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
export default {
|
|
122
|
+
LanceDBError,
|
|
123
|
+
EmbeddingError,
|
|
124
|
+
StorageError,
|
|
125
|
+
QueryError,
|
|
126
|
+
ConfigurationError,
|
|
127
|
+
handleError,
|
|
128
|
+
sanitizeErrorMessage,
|
|
129
|
+
};
|