@danielsimonjr/memory-mcp 12.2.3 → 12.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -102
- package/dist/index.js +6 -0
- package/dist/server/toolDefinitions.d.ts.map +1 -1
- package/dist/server/toolDefinitions.js +745 -3
- package/dist/server/toolHandlers.d.ts +1 -9
- package/dist/server/toolHandlers.d.ts.map +1 -1
- package/dist/server/toolHandlers.js +859 -11
- package/package.json +9 -6
- package/dist/core/EntityManager.d.ts +0 -263
- package/dist/core/EntityManager.d.ts.map +0 -1
- package/dist/core/EntityManager.js +0 -478
- package/dist/core/GraphStorage.d.ts +0 -344
- package/dist/core/GraphStorage.d.ts.map +0 -1
- package/dist/core/GraphStorage.js +0 -708
- package/dist/core/GraphTraversal.d.ts +0 -136
- package/dist/core/GraphTraversal.d.ts.map +0 -1
- package/dist/core/GraphTraversal.js +0 -556
- package/dist/core/HierarchyManager.d.ts +0 -111
- package/dist/core/HierarchyManager.d.ts.map +0 -1
- package/dist/core/HierarchyManager.js +0 -224
- package/dist/core/ManagerContext.d.ts +0 -72
- package/dist/core/ManagerContext.d.ts.map +0 -1
- package/dist/core/ManagerContext.js +0 -118
- package/dist/core/ObservationManager.d.ts +0 -85
- package/dist/core/ObservationManager.d.ts.map +0 -1
- package/dist/core/ObservationManager.js +0 -123
- package/dist/core/RelationManager.d.ts +0 -131
- package/dist/core/RelationManager.d.ts.map +0 -1
- package/dist/core/RelationManager.js +0 -210
- package/dist/core/SQLiteStorage.d.ts +0 -354
- package/dist/core/SQLiteStorage.d.ts.map +0 -1
- package/dist/core/SQLiteStorage.js +0 -917
- package/dist/core/StorageFactory.d.ts +0 -45
- package/dist/core/StorageFactory.d.ts.map +0 -1
- package/dist/core/StorageFactory.js +0 -64
- package/dist/core/TransactionManager.d.ts +0 -259
- package/dist/core/TransactionManager.d.ts.map +0 -1
- package/dist/core/TransactionManager.js +0 -392
- package/dist/core/index.d.ts +0 -16
- package/dist/core/index.d.ts.map +0 -1
- package/dist/core/index.js +0 -17
- package/dist/features/AnalyticsManager.d.ts +0 -44
- package/dist/features/AnalyticsManager.d.ts.map +0 -1
- package/dist/features/AnalyticsManager.js +0 -223
- package/dist/features/ArchiveManager.d.ts +0 -121
- package/dist/features/ArchiveManager.d.ts.map +0 -1
- package/dist/features/ArchiveManager.js +0 -240
- package/dist/features/CompressionManager.d.ts +0 -109
- package/dist/features/CompressionManager.d.ts.map +0 -1
- package/dist/features/CompressionManager.js +0 -404
- package/dist/features/IOManager.d.ts +0 -221
- package/dist/features/IOManager.d.ts.map +0 -1
- package/dist/features/IOManager.js +0 -997
- package/dist/features/StreamingExporter.d.ts +0 -105
- package/dist/features/StreamingExporter.d.ts.map +0 -1
- package/dist/features/StreamingExporter.js +0 -150
- package/dist/features/TagManager.d.ts +0 -147
- package/dist/features/TagManager.d.ts.map +0 -1
- package/dist/features/TagManager.js +0 -210
- package/dist/features/index.d.ts +0 -12
- package/dist/features/index.d.ts.map +0 -1
- package/dist/features/index.js +0 -11
- package/dist/search/BasicSearch.d.ts +0 -51
- package/dist/search/BasicSearch.d.ts.map +0 -1
- package/dist/search/BasicSearch.js +0 -137
- package/dist/search/BooleanSearch.d.ts +0 -98
- package/dist/search/BooleanSearch.d.ts.map +0 -1
- package/dist/search/BooleanSearch.js +0 -430
- package/dist/search/EmbeddingService.d.ts +0 -178
- package/dist/search/EmbeddingService.d.ts.map +0 -1
- package/dist/search/EmbeddingService.js +0 -358
- package/dist/search/FuzzySearch.d.ts +0 -118
- package/dist/search/FuzzySearch.d.ts.map +0 -1
- package/dist/search/FuzzySearch.js +0 -312
- package/dist/search/RankedSearch.d.ts +0 -71
- package/dist/search/RankedSearch.d.ts.map +0 -1
- package/dist/search/RankedSearch.js +0 -238
- package/dist/search/SavedSearchManager.d.ts +0 -79
- package/dist/search/SavedSearchManager.d.ts.map +0 -1
- package/dist/search/SavedSearchManager.js +0 -145
- package/dist/search/SearchFilterChain.d.ts +0 -120
- package/dist/search/SearchFilterChain.d.ts.map +0 -1
- package/dist/search/SearchFilterChain.js +0 -185
- package/dist/search/SearchManager.d.ts +0 -285
- package/dist/search/SearchManager.d.ts.map +0 -1
- package/dist/search/SearchManager.js +0 -338
- package/dist/search/SearchSuggestions.d.ts +0 -27
- package/dist/search/SearchSuggestions.d.ts.map +0 -1
- package/dist/search/SearchSuggestions.js +0 -57
- package/dist/search/SemanticSearch.d.ts +0 -146
- package/dist/search/SemanticSearch.d.ts.map +0 -1
- package/dist/search/SemanticSearch.js +0 -313
- package/dist/search/TFIDFIndexManager.d.ts +0 -63
- package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
- package/dist/search/TFIDFIndexManager.js +0 -215
- package/dist/search/VectorStore.d.ts +0 -235
- package/dist/search/VectorStore.d.ts.map +0 -1
- package/dist/search/VectorStore.js +0 -311
- package/dist/search/index.d.ts +0 -18
- package/dist/search/index.d.ts.map +0 -1
- package/dist/search/index.js +0 -21
- package/dist/types/index.d.ts +0 -13
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -12
- package/dist/types/types.d.ts +0 -1234
- package/dist/types/types.d.ts.map +0 -1
- package/dist/types/types.js +0 -9
- package/dist/utils/compressedCache.d.ts +0 -192
- package/dist/utils/compressedCache.d.ts.map +0 -1
- package/dist/utils/compressedCache.js +0 -309
- package/dist/utils/compressionUtil.d.ts +0 -214
- package/dist/utils/compressionUtil.d.ts.map +0 -1
- package/dist/utils/compressionUtil.js +0 -247
- package/dist/utils/constants.d.ts +0 -245
- package/dist/utils/constants.d.ts.map +0 -1
- package/dist/utils/constants.js +0 -252
- package/dist/utils/entityUtils.d.ts +0 -321
- package/dist/utils/entityUtils.d.ts.map +0 -1
- package/dist/utils/entityUtils.js +0 -538
- package/dist/utils/errors.d.ts +0 -77
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/errors.js +0 -121
- package/dist/utils/formatters.d.ts +0 -145
- package/dist/utils/formatters.d.ts.map +0 -1
- package/dist/utils/formatters.js +0 -132
- package/dist/utils/index.d.ts +0 -22
- package/dist/utils/index.d.ts.map +0 -1
- package/dist/utils/index.js +0 -75
- package/dist/utils/indexes.d.ts +0 -270
- package/dist/utils/indexes.d.ts.map +0 -1
- package/dist/utils/indexes.js +0 -526
- package/dist/utils/logger.d.ts +0 -24
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -33
- package/dist/utils/parallelUtils.d.ts +0 -72
- package/dist/utils/parallelUtils.d.ts.map +0 -1
- package/dist/utils/parallelUtils.js +0 -169
- package/dist/utils/schemas.d.ts +0 -374
- package/dist/utils/schemas.d.ts.map +0 -1
- package/dist/utils/schemas.js +0 -484
- package/dist/utils/searchAlgorithms.d.ts +0 -99
- package/dist/utils/searchAlgorithms.d.ts.map +0 -1
- package/dist/utils/searchAlgorithms.js +0 -167
- package/dist/utils/searchCache.d.ts +0 -108
- package/dist/utils/searchCache.d.ts.map +0 -1
- package/dist/utils/searchCache.js +0 -209
- package/dist/utils/taskScheduler.d.ts +0 -290
- package/dist/utils/taskScheduler.d.ts.map +0 -1
- package/dist/utils/taskScheduler.js +0 -466
- package/dist/workers/WorkerPool.d.ts +0 -81
- package/dist/workers/WorkerPool.d.ts.map +0 -1
- package/dist/workers/WorkerPool.js +0 -121
- package/dist/workers/index.d.ts +0 -12
- package/dist/workers/index.d.ts.map +0 -1
- package/dist/workers/index.js +0 -9
- package/dist/workers/levenshteinWorker.d.ts +0 -60
- package/dist/workers/levenshteinWorker.d.ts.map +0 -1
- package/dist/workers/levenshteinWorker.js +0 -98
|
@@ -1,430 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Boolean Search
|
|
3
|
-
*
|
|
4
|
-
* Advanced search with boolean operators (AND, OR, NOT) and field-specific queries.
|
|
5
|
-
*
|
|
6
|
-
* @module search/BooleanSearch
|
|
7
|
-
*/
|
|
8
|
-
import { SEARCH_LIMITS, QUERY_LIMITS } from '../utils/constants.js';
|
|
9
|
-
import { ValidationError } from '../utils/errors.js';
|
|
10
|
-
import { SearchFilterChain } from './SearchFilterChain.js';
|
|
11
|
-
/**
|
|
12
|
-
* Phase 4 Sprint 4: Maximum AST cache size.
|
|
13
|
-
*/
|
|
14
|
-
const AST_CACHE_MAX_SIZE = 50;
|
|
15
|
-
/**
|
|
16
|
-
* Phase 4 Sprint 4: Result cache max size.
|
|
17
|
-
*/
|
|
18
|
-
const RESULT_CACHE_MAX_SIZE = 100;
|
|
19
|
-
/**
|
|
20
|
-
* Phase 4 Sprint 4: Cache TTL in milliseconds (5 minutes).
|
|
21
|
-
*/
|
|
22
|
-
const BOOLEAN_CACHE_TTL_MS = 5 * 60 * 1000;
|
|
23
|
-
/**
|
|
24
|
-
* Performs boolean search with query parsing and AST evaluation.
|
|
25
|
-
*/
|
|
26
|
-
export class BooleanSearch {
|
|
27
|
-
storage;
|
|
28
|
-
/**
|
|
29
|
-
* Phase 4 Sprint 4: AST cache to avoid re-parsing queries.
|
|
30
|
-
* Maps query string -> parsed AST.
|
|
31
|
-
*/
|
|
32
|
-
astCache = new Map();
|
|
33
|
-
/**
|
|
34
|
-
* Phase 4 Sprint 4: Result cache for boolean search.
|
|
35
|
-
* Maps cache key -> cached results.
|
|
36
|
-
*/
|
|
37
|
-
resultCache = new Map();
|
|
38
|
-
constructor(storage) {
|
|
39
|
-
this.storage = storage;
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* Phase 4 Sprint 4: Generate cache key for boolean search.
|
|
43
|
-
*/
|
|
44
|
-
generateCacheKey(query, tags, minImportance, maxImportance, offset, limit) {
|
|
45
|
-
return JSON.stringify({
|
|
46
|
-
q: query,
|
|
47
|
-
tags: tags?.sort().join(',') ?? '',
|
|
48
|
-
min: minImportance,
|
|
49
|
-
max: maxImportance,
|
|
50
|
-
off: offset,
|
|
51
|
-
lim: limit,
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Phase 4 Sprint 4: Clear all caches.
|
|
56
|
-
*/
|
|
57
|
-
clearCache() {
|
|
58
|
-
this.astCache.clear();
|
|
59
|
-
this.resultCache.clear();
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Phase 4 Sprint 4: Cleanup old cache entries.
|
|
63
|
-
*/
|
|
64
|
-
cleanupResultCache() {
|
|
65
|
-
const now = Date.now();
|
|
66
|
-
const entries = Array.from(this.resultCache.entries());
|
|
67
|
-
// Remove expired entries
|
|
68
|
-
for (const [key, entry] of entries) {
|
|
69
|
-
if (now - entry.timestamp > BOOLEAN_CACHE_TTL_MS) {
|
|
70
|
-
this.resultCache.delete(key);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
// If still over limit, remove oldest entries
|
|
74
|
-
if (this.resultCache.size > RESULT_CACHE_MAX_SIZE) {
|
|
75
|
-
const sortedEntries = entries
|
|
76
|
-
.filter(([k]) => this.resultCache.has(k))
|
|
77
|
-
.sort((a, b) => a[1].timestamp - b[1].timestamp);
|
|
78
|
-
const toRemove = sortedEntries.slice(0, this.resultCache.size - RESULT_CACHE_MAX_SIZE);
|
|
79
|
-
for (const [key] of toRemove) {
|
|
80
|
-
this.resultCache.delete(key);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Phase 4 Sprint 4: Get or parse AST for a query.
|
|
86
|
-
*/
|
|
87
|
-
getOrParseAST(query) {
|
|
88
|
-
// Check AST cache
|
|
89
|
-
const cached = this.astCache.get(query);
|
|
90
|
-
if (cached) {
|
|
91
|
-
return cached;
|
|
92
|
-
}
|
|
93
|
-
// Parse and cache
|
|
94
|
-
const ast = this.parseBooleanQuery(query);
|
|
95
|
-
// Enforce cache size limit
|
|
96
|
-
if (this.astCache.size >= AST_CACHE_MAX_SIZE) {
|
|
97
|
-
// Remove first entry (oldest)
|
|
98
|
-
const firstKey = this.astCache.keys().next().value;
|
|
99
|
-
if (firstKey)
|
|
100
|
-
this.astCache.delete(firstKey);
|
|
101
|
-
}
|
|
102
|
-
this.astCache.set(query, ast);
|
|
103
|
-
return ast;
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Boolean search with support for AND, OR, NOT operators, field-specific queries, and pagination.
|
|
107
|
-
*
|
|
108
|
-
* Phase 4 Sprint 4: Implements AST caching and result caching for repeated queries.
|
|
109
|
-
*
|
|
110
|
-
* Query syntax examples:
|
|
111
|
-
* - "alice AND programming" - Both terms must match
|
|
112
|
-
* - "type:person OR type:organization" - Either type matches
|
|
113
|
-
* - "NOT archived" - Exclude archived items
|
|
114
|
-
* - "name:alice AND (observation:coding OR observation:teaching)"
|
|
115
|
-
*
|
|
116
|
-
* @param query - Boolean query string
|
|
117
|
-
* @param tags - Optional tags filter
|
|
118
|
-
* @param minImportance - Optional minimum importance
|
|
119
|
-
* @param maxImportance - Optional maximum importance
|
|
120
|
-
* @param offset - Number of results to skip (default: 0)
|
|
121
|
-
* @param limit - Maximum number of results (default: 50, max: 200)
|
|
122
|
-
* @returns Filtered knowledge graph matching the boolean query with pagination applied
|
|
123
|
-
*/
|
|
124
|
-
async booleanSearch(query, tags, minImportance, maxImportance, offset = 0, limit = SEARCH_LIMITS.DEFAULT) {
|
|
125
|
-
// Validate query length
|
|
126
|
-
if (query.length > QUERY_LIMITS.MAX_QUERY_LENGTH) {
|
|
127
|
-
throw new ValidationError('Query too long', [`Query length ${query.length} exceeds maximum of ${QUERY_LIMITS.MAX_QUERY_LENGTH} characters`]);
|
|
128
|
-
}
|
|
129
|
-
const graph = await this.storage.loadGraph();
|
|
130
|
-
// Phase 4 Sprint 4: Check result cache
|
|
131
|
-
const cacheKey = this.generateCacheKey(query, tags, minImportance, maxImportance, offset, limit);
|
|
132
|
-
const cached = this.resultCache.get(cacheKey);
|
|
133
|
-
if (cached && cached.entityCount === graph.entities.length) {
|
|
134
|
-
const now = Date.now();
|
|
135
|
-
if (now - cached.timestamp < BOOLEAN_CACHE_TTL_MS) {
|
|
136
|
-
// Return cached results
|
|
137
|
-
const cachedNameSet = new Set(cached.entityNames);
|
|
138
|
-
const cachedEntities = graph.entities.filter(e => cachedNameSet.has(e.name));
|
|
139
|
-
const cachedRelations = graph.relations.filter(r => cachedNameSet.has(r.from) && cachedNameSet.has(r.to));
|
|
140
|
-
return { entities: cachedEntities, relations: cachedRelations };
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
// Phase 4 Sprint 4: Use cached AST or parse new one
|
|
144
|
-
let queryAst;
|
|
145
|
-
try {
|
|
146
|
-
queryAst = this.getOrParseAST(query);
|
|
147
|
-
}
|
|
148
|
-
catch (error) {
|
|
149
|
-
throw new Error(`Failed to parse boolean query: ${error instanceof Error ? error.message : String(error)}`);
|
|
150
|
-
}
|
|
151
|
-
// Validate query complexity
|
|
152
|
-
this.validateQueryComplexity(queryAst);
|
|
153
|
-
// First filter by boolean query evaluation (search-specific)
|
|
154
|
-
const booleanMatched = graph.entities.filter(e => this.evaluateBooleanQuery(queryAst, e));
|
|
155
|
-
// Apply tag and importance filters using SearchFilterChain
|
|
156
|
-
const filters = { tags, minImportance, maxImportance };
|
|
157
|
-
const filteredEntities = SearchFilterChain.applyFilters(booleanMatched, filters);
|
|
158
|
-
// Apply pagination using SearchFilterChain
|
|
159
|
-
const pagination = SearchFilterChain.validatePagination(offset, limit);
|
|
160
|
-
const paginatedEntities = SearchFilterChain.paginate(filteredEntities, pagination);
|
|
161
|
-
// Phase 4 Sprint 4: Cache the results
|
|
162
|
-
this.resultCache.set(cacheKey, {
|
|
163
|
-
ast: queryAst,
|
|
164
|
-
entityNames: paginatedEntities.map(e => e.name),
|
|
165
|
-
entityCount: graph.entities.length,
|
|
166
|
-
timestamp: Date.now(),
|
|
167
|
-
});
|
|
168
|
-
// Cleanup old cache entries periodically
|
|
169
|
-
if (this.resultCache.size > RESULT_CACHE_MAX_SIZE / 2) {
|
|
170
|
-
this.cleanupResultCache();
|
|
171
|
-
}
|
|
172
|
-
const filteredEntityNames = new Set(paginatedEntities.map(e => e.name));
|
|
173
|
-
const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to));
|
|
174
|
-
return { entities: paginatedEntities, relations: filteredRelations };
|
|
175
|
-
}
|
|
176
|
-
/**
|
|
177
|
-
* Tokenize a boolean query into tokens.
|
|
178
|
-
*
|
|
179
|
-
* Handles quoted strings, parentheses, and operators.
|
|
180
|
-
*/
|
|
181
|
-
tokenizeBooleanQuery(query) {
|
|
182
|
-
const tokens = [];
|
|
183
|
-
let current = '';
|
|
184
|
-
let inQuotes = false;
|
|
185
|
-
for (let i = 0; i < query.length; i++) {
|
|
186
|
-
const char = query[i];
|
|
187
|
-
if (char === '"') {
|
|
188
|
-
if (inQuotes) {
|
|
189
|
-
// End of quoted string
|
|
190
|
-
tokens.push(current);
|
|
191
|
-
current = '';
|
|
192
|
-
inQuotes = false;
|
|
193
|
-
}
|
|
194
|
-
else {
|
|
195
|
-
// Start of quoted string
|
|
196
|
-
if (current.trim()) {
|
|
197
|
-
tokens.push(current.trim());
|
|
198
|
-
current = '';
|
|
199
|
-
}
|
|
200
|
-
inQuotes = true;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
else if (!inQuotes && (char === '(' || char === ')')) {
|
|
204
|
-
// Parentheses are separate tokens
|
|
205
|
-
if (current.trim()) {
|
|
206
|
-
tokens.push(current.trim());
|
|
207
|
-
current = '';
|
|
208
|
-
}
|
|
209
|
-
tokens.push(char);
|
|
210
|
-
}
|
|
211
|
-
else if (!inQuotes && /\s/.test(char)) {
|
|
212
|
-
// Whitespace outside quotes
|
|
213
|
-
if (current.trim()) {
|
|
214
|
-
tokens.push(current.trim());
|
|
215
|
-
current = '';
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
else {
|
|
219
|
-
current += char;
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
if (current.trim()) {
|
|
223
|
-
tokens.push(current.trim());
|
|
224
|
-
}
|
|
225
|
-
return tokens;
|
|
226
|
-
}
|
|
227
|
-
/**
|
|
228
|
-
* Parse a boolean search query into an AST.
|
|
229
|
-
*
|
|
230
|
-
* Supports: AND, OR, NOT, parentheses, field-specific queries (field:value)
|
|
231
|
-
*/
|
|
232
|
-
parseBooleanQuery(query) {
|
|
233
|
-
const tokens = this.tokenizeBooleanQuery(query);
|
|
234
|
-
let position = 0;
|
|
235
|
-
const peek = () => tokens[position];
|
|
236
|
-
const consume = () => tokens[position++];
|
|
237
|
-
// Parse OR expressions (lowest precedence)
|
|
238
|
-
const parseOr = () => {
|
|
239
|
-
let left = parseAnd();
|
|
240
|
-
while (peek()?.toUpperCase() === 'OR') {
|
|
241
|
-
consume(); // consume 'OR'
|
|
242
|
-
const right = parseAnd();
|
|
243
|
-
left = { type: 'OR', children: [left, right] };
|
|
244
|
-
}
|
|
245
|
-
return left;
|
|
246
|
-
};
|
|
247
|
-
// Parse AND expressions
|
|
248
|
-
const parseAnd = () => {
|
|
249
|
-
let left = parseNot();
|
|
250
|
-
while (peek() && peek()?.toUpperCase() !== 'OR' && peek() !== ')') {
|
|
251
|
-
// Implicit AND if next token is not OR or )
|
|
252
|
-
if (peek()?.toUpperCase() === 'AND') {
|
|
253
|
-
consume(); // consume 'AND'
|
|
254
|
-
}
|
|
255
|
-
const right = parseNot();
|
|
256
|
-
left = { type: 'AND', children: [left, right] };
|
|
257
|
-
}
|
|
258
|
-
return left;
|
|
259
|
-
};
|
|
260
|
-
// Parse NOT expressions
|
|
261
|
-
const parseNot = () => {
|
|
262
|
-
if (peek()?.toUpperCase() === 'NOT') {
|
|
263
|
-
consume(); // consume 'NOT'
|
|
264
|
-
const child = parseNot();
|
|
265
|
-
return { type: 'NOT', child };
|
|
266
|
-
}
|
|
267
|
-
return parsePrimary();
|
|
268
|
-
};
|
|
269
|
-
// Parse primary expressions (terms, field queries, parentheses)
|
|
270
|
-
const parsePrimary = () => {
|
|
271
|
-
const token = peek();
|
|
272
|
-
if (!token) {
|
|
273
|
-
throw new Error('Unexpected end of query');
|
|
274
|
-
}
|
|
275
|
-
// Parentheses
|
|
276
|
-
if (token === '(') {
|
|
277
|
-
consume(); // consume '('
|
|
278
|
-
const node = parseOr();
|
|
279
|
-
if (consume() !== ')') {
|
|
280
|
-
throw new Error('Expected closing parenthesis');
|
|
281
|
-
}
|
|
282
|
-
return node;
|
|
283
|
-
}
|
|
284
|
-
// Field-specific query (field:value)
|
|
285
|
-
if (token.includes(':')) {
|
|
286
|
-
consume();
|
|
287
|
-
const [field, ...valueParts] = token.split(':');
|
|
288
|
-
const value = valueParts.join(':'); // Handle colons in value
|
|
289
|
-
return { type: 'TERM', field: field.toLowerCase(), value: value.toLowerCase() };
|
|
290
|
-
}
|
|
291
|
-
// Regular term
|
|
292
|
-
consume();
|
|
293
|
-
return { type: 'TERM', value: token.toLowerCase() };
|
|
294
|
-
};
|
|
295
|
-
const result = parseOr();
|
|
296
|
-
// Check for unconsumed tokens
|
|
297
|
-
if (position < tokens.length) {
|
|
298
|
-
throw new Error(`Unexpected token: ${tokens[position]}`);
|
|
299
|
-
}
|
|
300
|
-
return result;
|
|
301
|
-
}
|
|
302
|
-
/**
|
|
303
|
-
* Evaluate a boolean query AST against an entity.
|
|
304
|
-
*/
|
|
305
|
-
evaluateBooleanQuery(node, entity) {
|
|
306
|
-
switch (node.type) {
|
|
307
|
-
case 'AND':
|
|
308
|
-
return node.children.every(child => this.evaluateBooleanQuery(child, entity));
|
|
309
|
-
case 'OR':
|
|
310
|
-
return node.children.some(child => this.evaluateBooleanQuery(child, entity));
|
|
311
|
-
case 'NOT':
|
|
312
|
-
return !this.evaluateBooleanQuery(node.child, entity);
|
|
313
|
-
case 'TERM': {
|
|
314
|
-
const value = node.value;
|
|
315
|
-
// OPTIMIZED: Use pre-computed lowercase cache
|
|
316
|
-
const lowercased = this.storage.getLowercased(entity.name);
|
|
317
|
-
// Field-specific search
|
|
318
|
-
if (node.field) {
|
|
319
|
-
switch (node.field) {
|
|
320
|
-
case 'name':
|
|
321
|
-
return lowercased ? lowercased.name.includes(value) : entity.name.toLowerCase().includes(value);
|
|
322
|
-
case 'type':
|
|
323
|
-
case 'entitytype':
|
|
324
|
-
return lowercased ? lowercased.entityType.includes(value) : entity.entityType.toLowerCase().includes(value);
|
|
325
|
-
case 'observation':
|
|
326
|
-
case 'observations':
|
|
327
|
-
// OPTIMIZED: Use observation index for simple single-word terms (O(1) vs O(n))
|
|
328
|
-
// The index only matches complete words, not substrings, so we can only
|
|
329
|
-
// use it as a quick positive check. If not found in index, fall through
|
|
330
|
-
// to substring matching for compatibility.
|
|
331
|
-
if (this.isSimpleTerm(value) && !value.includes(' ')) {
|
|
332
|
-
const candidateNames = this.storage.getEntitiesByObservationWord(value);
|
|
333
|
-
if (candidateNames.has(entity.name)) {
|
|
334
|
-
return true; // O(1) positive match
|
|
335
|
-
}
|
|
336
|
-
// Not found in index - entity doesn't have this complete word,
|
|
337
|
-
// but might contain it as substring - fall through to check
|
|
338
|
-
}
|
|
339
|
-
// Linear scan for substring matches, phrases, and patterns
|
|
340
|
-
return lowercased
|
|
341
|
-
? lowercased.observations.some(obs => obs.includes(value))
|
|
342
|
-
: entity.observations.some(obs => obs.toLowerCase().includes(value));
|
|
343
|
-
case 'tag':
|
|
344
|
-
case 'tags':
|
|
345
|
-
return lowercased
|
|
346
|
-
? lowercased.tags.some(tag => tag.includes(value))
|
|
347
|
-
: (entity.tags?.some(tag => tag.toLowerCase().includes(value)) || false);
|
|
348
|
-
default:
|
|
349
|
-
// Unknown field, search all text fields
|
|
350
|
-
return this.entityMatchesTerm(entity, value, lowercased);
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
// General search across all fields
|
|
354
|
-
return this.entityMatchesTerm(entity, value, lowercased);
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
/**
|
|
359
|
-
* Check if a search term is simple (no regex or wildcards).
|
|
360
|
-
* Simple terms can use the O(1) observation index.
|
|
361
|
-
*/
|
|
362
|
-
isSimpleTerm(term) {
|
|
363
|
-
const specialChars = /[.*+?^${}()|\\[\]]/;
|
|
364
|
-
return !specialChars.test(term);
|
|
365
|
-
}
|
|
366
|
-
/**
|
|
367
|
-
* Check if entity matches a search term in any text field.
|
|
368
|
-
* OPTIMIZED: Uses pre-computed lowercase data when available.
|
|
369
|
-
*/
|
|
370
|
-
entityMatchesTerm(entity, term, lowercased) {
|
|
371
|
-
if (lowercased) {
|
|
372
|
-
return (lowercased.name.includes(term) ||
|
|
373
|
-
lowercased.entityType.includes(term) ||
|
|
374
|
-
lowercased.observations.some(obs => obs.includes(term)) ||
|
|
375
|
-
lowercased.tags.some(tag => tag.includes(term)));
|
|
376
|
-
}
|
|
377
|
-
// Fallback for entities not in cache
|
|
378
|
-
const termLower = term.toLowerCase();
|
|
379
|
-
return (entity.name.toLowerCase().includes(termLower) ||
|
|
380
|
-
entity.entityType.toLowerCase().includes(termLower) ||
|
|
381
|
-
entity.observations.some(obs => obs.toLowerCase().includes(termLower)) ||
|
|
382
|
-
(entity.tags?.some(tag => tag.toLowerCase().includes(termLower)) || false));
|
|
383
|
-
}
|
|
384
|
-
/**
|
|
385
|
-
* Validate query complexity to prevent resource exhaustion.
|
|
386
|
-
* Checks nesting depth, term count, and operator count against configured limits.
|
|
387
|
-
*/
|
|
388
|
-
validateQueryComplexity(node, depth = 0) {
|
|
389
|
-
// Check nesting depth
|
|
390
|
-
if (depth > QUERY_LIMITS.MAX_DEPTH) {
|
|
391
|
-
throw new ValidationError('Query too complex', [`Query nesting depth ${depth} exceeds maximum of ${QUERY_LIMITS.MAX_DEPTH}`]);
|
|
392
|
-
}
|
|
393
|
-
// Count terms and operators recursively
|
|
394
|
-
const complexity = this.calculateQueryComplexity(node);
|
|
395
|
-
if (complexity.terms > QUERY_LIMITS.MAX_TERMS) {
|
|
396
|
-
throw new ValidationError('Query too complex', [`Query has ${complexity.terms} terms, exceeds maximum of ${QUERY_LIMITS.MAX_TERMS}`]);
|
|
397
|
-
}
|
|
398
|
-
if (complexity.operators > QUERY_LIMITS.MAX_OPERATORS) {
|
|
399
|
-
throw new ValidationError('Query too complex', [`Query has ${complexity.operators} operators, exceeds maximum of ${QUERY_LIMITS.MAX_OPERATORS}`]);
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
/**
|
|
403
|
-
* Calculate query complexity metrics.
|
|
404
|
-
*/
|
|
405
|
-
calculateQueryComplexity(node, depth = 0) {
|
|
406
|
-
switch (node.type) {
|
|
407
|
-
case 'AND':
|
|
408
|
-
case 'OR':
|
|
409
|
-
const childResults = node.children.map(child => this.calculateQueryComplexity(child, depth + 1));
|
|
410
|
-
return {
|
|
411
|
-
terms: childResults.reduce((sum, r) => sum + r.terms, 0),
|
|
412
|
-
operators: childResults.reduce((sum, r) => sum + r.operators, 1), // +1 for current operator
|
|
413
|
-
maxDepth: Math.max(depth, ...childResults.map(r => r.maxDepth)),
|
|
414
|
-
};
|
|
415
|
-
case 'NOT':
|
|
416
|
-
const notResult = this.calculateQueryComplexity(node.child, depth + 1);
|
|
417
|
-
return {
|
|
418
|
-
terms: notResult.terms,
|
|
419
|
-
operators: notResult.operators + 1,
|
|
420
|
-
maxDepth: Math.max(depth, notResult.maxDepth),
|
|
421
|
-
};
|
|
422
|
-
case 'TERM':
|
|
423
|
-
return {
|
|
424
|
-
terms: 1,
|
|
425
|
-
operators: 0,
|
|
426
|
-
maxDepth: depth,
|
|
427
|
-
};
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
}
|
|
@@ -1,178 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Embedding Service
|
|
3
|
-
*
|
|
4
|
-
* Phase 4 Sprint 10: Provides embedding abstractions for semantic search.
|
|
5
|
-
* Supports multiple providers: OpenAI (cloud) and local (transformers.js).
|
|
6
|
-
*
|
|
7
|
-
* @module search/EmbeddingService
|
|
8
|
-
*/
|
|
9
|
-
import type { EmbeddingService, EmbeddingConfig } from '../types/index.js';
|
|
10
|
-
/**
|
|
11
|
-
* OpenAI Embedding Service
|
|
12
|
-
*
|
|
13
|
-
* Uses OpenAI's text-embedding-3-small model for generating embeddings.
|
|
14
|
-
* Supports single and batch embedding with rate limit handling.
|
|
15
|
-
*
|
|
16
|
-
* @example
|
|
17
|
-
* ```typescript
|
|
18
|
-
* const service = new OpenAIEmbeddingService('sk-...');
|
|
19
|
-
* const embedding = await service.embed("Hello world");
|
|
20
|
-
* console.log(`Generated ${embedding.length} dimensions`);
|
|
21
|
-
* ```
|
|
22
|
-
*/
|
|
23
|
-
export declare class OpenAIEmbeddingService implements EmbeddingService {
|
|
24
|
-
readonly dimensions: number;
|
|
25
|
-
readonly provider = "openai";
|
|
26
|
-
readonly model: string;
|
|
27
|
-
private apiKey;
|
|
28
|
-
/**
|
|
29
|
-
* Create an OpenAI embedding service.
|
|
30
|
-
*
|
|
31
|
-
* @param apiKey - OpenAI API key
|
|
32
|
-
* @param model - Optional model override (default: text-embedding-3-small)
|
|
33
|
-
*/
|
|
34
|
-
constructor(apiKey: string, model?: string);
|
|
35
|
-
/**
|
|
36
|
-
* Check if the service is ready.
|
|
37
|
-
*/
|
|
38
|
-
isReady(): Promise<boolean>;
|
|
39
|
-
/**
|
|
40
|
-
* Generate embedding for a single text.
|
|
41
|
-
*
|
|
42
|
-
* @param text - Text to embed
|
|
43
|
-
* @returns Embedding vector
|
|
44
|
-
*/
|
|
45
|
-
embed(text: string): Promise<number[]>;
|
|
46
|
-
/**
|
|
47
|
-
* Generate embeddings for multiple texts in batch.
|
|
48
|
-
*
|
|
49
|
-
* @param texts - Array of texts to embed
|
|
50
|
-
* @returns Array of embedding vectors
|
|
51
|
-
*/
|
|
52
|
-
embedBatch(texts: string[]): Promise<number[][]>;
|
|
53
|
-
/**
|
|
54
|
-
* Internal batch embedding with retry logic.
|
|
55
|
-
*/
|
|
56
|
-
private embedBatchInternal;
|
|
57
|
-
/**
|
|
58
|
-
* Check if an error is retryable.
|
|
59
|
-
*/
|
|
60
|
-
private isRetryableError;
|
|
61
|
-
/**
|
|
62
|
-
* Sleep for a given duration.
|
|
63
|
-
*/
|
|
64
|
-
private sleep;
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Local Embedding Service
|
|
68
|
-
*
|
|
69
|
-
* Uses @xenova/transformers for local embedding generation.
|
|
70
|
-
* No API calls needed - runs entirely offline after initial model download.
|
|
71
|
-
*
|
|
72
|
-
* Note: Requires @xenova/transformers to be installed as an optional dependency.
|
|
73
|
-
* If not available, initialization will fail gracefully.
|
|
74
|
-
*
|
|
75
|
-
* @example
|
|
76
|
-
* ```typescript
|
|
77
|
-
* const service = new LocalEmbeddingService();
|
|
78
|
-
* await service.initialize();
|
|
79
|
-
* const embedding = await service.embed("Hello world");
|
|
80
|
-
* ```
|
|
81
|
-
*/
|
|
82
|
-
export declare class LocalEmbeddingService implements EmbeddingService {
|
|
83
|
-
readonly dimensions: number;
|
|
84
|
-
readonly provider = "local";
|
|
85
|
-
readonly model: string;
|
|
86
|
-
private pipeline;
|
|
87
|
-
private initialized;
|
|
88
|
-
private initPromise;
|
|
89
|
-
/**
|
|
90
|
-
* Create a local embedding service.
|
|
91
|
-
*
|
|
92
|
-
* @param model - Optional model override (default: Xenova/all-MiniLM-L6-v2)
|
|
93
|
-
*/
|
|
94
|
-
constructor(model?: string);
|
|
95
|
-
/**
|
|
96
|
-
* Initialize the model pipeline.
|
|
97
|
-
* Must be called before using embed/embedBatch.
|
|
98
|
-
*/
|
|
99
|
-
initialize(): Promise<void>;
|
|
100
|
-
/**
|
|
101
|
-
* Internal initialization.
|
|
102
|
-
*/
|
|
103
|
-
private initializeInternal;
|
|
104
|
-
/**
|
|
105
|
-
* Check if the service is ready.
|
|
106
|
-
*/
|
|
107
|
-
isReady(): Promise<boolean>;
|
|
108
|
-
/**
|
|
109
|
-
* Generate embedding for a single text.
|
|
110
|
-
*
|
|
111
|
-
* @param text - Text to embed
|
|
112
|
-
* @returns Embedding vector
|
|
113
|
-
*/
|
|
114
|
-
embed(text: string): Promise<number[]>;
|
|
115
|
-
/**
|
|
116
|
-
* Generate embeddings for multiple texts in batch.
|
|
117
|
-
* Note: Local processing is done sequentially to avoid memory issues.
|
|
118
|
-
*
|
|
119
|
-
* @param texts - Array of texts to embed
|
|
120
|
-
* @returns Array of embedding vectors
|
|
121
|
-
*/
|
|
122
|
-
embedBatch(texts: string[]): Promise<number[][]>;
|
|
123
|
-
/**
|
|
124
|
-
* Ensure the service is initialized.
|
|
125
|
-
*/
|
|
126
|
-
private ensureInitialized;
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* Mock Embedding Service for testing
|
|
130
|
-
*
|
|
131
|
-
* Generates deterministic mock embeddings for testing purposes.
|
|
132
|
-
* Useful for unit tests that don't need real embeddings.
|
|
133
|
-
*/
|
|
134
|
-
export declare class MockEmbeddingService implements EmbeddingService {
|
|
135
|
-
readonly dimensions: number;
|
|
136
|
-
readonly provider = "mock";
|
|
137
|
-
readonly model = "mock-model";
|
|
138
|
-
/**
|
|
139
|
-
* Create a mock embedding service.
|
|
140
|
-
*
|
|
141
|
-
* @param dimensions - Number of dimensions for mock embeddings
|
|
142
|
-
*/
|
|
143
|
-
constructor(dimensions?: number);
|
|
144
|
-
/**
|
|
145
|
-
* Check if the service is ready.
|
|
146
|
-
*/
|
|
147
|
-
isReady(): Promise<boolean>;
|
|
148
|
-
/**
|
|
149
|
-
* Generate a deterministic mock embedding for a text.
|
|
150
|
-
*
|
|
151
|
-
* @param text - Text to embed
|
|
152
|
-
* @returns Mock embedding vector
|
|
153
|
-
*/
|
|
154
|
-
embed(text: string): Promise<number[]>;
|
|
155
|
-
/**
|
|
156
|
-
* Generate mock embeddings for multiple texts.
|
|
157
|
-
*
|
|
158
|
-
* @param texts - Array of texts to embed
|
|
159
|
-
* @returns Array of mock embedding vectors
|
|
160
|
-
*/
|
|
161
|
-
embedBatch(texts: string[]): Promise<number[][]>;
|
|
162
|
-
/**
|
|
163
|
-
* Simple string hash function.
|
|
164
|
-
*/
|
|
165
|
-
private hashString;
|
|
166
|
-
/**
|
|
167
|
-
* Normalize a vector to unit length.
|
|
168
|
-
*/
|
|
169
|
-
private normalize;
|
|
170
|
-
}
|
|
171
|
-
/**
|
|
172
|
-
* Create an embedding service based on configuration.
|
|
173
|
-
*
|
|
174
|
-
* @param config - Optional configuration override
|
|
175
|
-
* @returns Embedding service instance, or null if provider is 'none'
|
|
176
|
-
*/
|
|
177
|
-
export declare function createEmbeddingService(config?: Partial<EmbeddingConfig>): EmbeddingService | null;
|
|
178
|
-
//# sourceMappingURL=EmbeddingService.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"EmbeddingService.d.ts","sourceRoot":"","sources":["../../src/search/EmbeddingService.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAO3E;;;;;;;;;;;;GAYG;AACH,qBAAa,sBAAuB,YAAW,gBAAgB;IAC7D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,YAAY;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,MAAM,CAAS;IAEvB;;;;;OAKG;gBACS,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM;IAS1C;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAK5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAkBtD;;OAEG;YACW,kBAAkB;IA0DhC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd;AAmBD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAC5D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAuC;IAClE,QAAQ,CAAC,QAAQ,WAAW;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAA8B;IAEjD;;;;OAIG;gBACS,KAAK,CAAC,EAAE,MAAM;IAI1B;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAWjC;;OAEG;YACW,kBAAkB;IAkBhC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAWjC;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAS5C;;;;;;OAMG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAWtD;;OAEG;YACW,iBAAiB;CAKhC;AAED;;;;;GAKG;AACH,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,UAAU;IAC3B,QAAQ,CAAC,KAAK,gBAAgB;IAE9B;;;;OAIG;gBACS,UAAU,GAAE,MAAY;IAIpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAe5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAItD;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;OAEG;IACH,OAAO,CAAC,SAAS;CAalB;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,gBAAgB,GAAG,IAAI,CAoBjG"}
|