@danielsimonjr/memory-mcp 11.0.1 → 11.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -22
- package/dist/core/EntityManager.d.ts +10 -15
- package/dist/core/EntityManager.d.ts.map +1 -1
- package/dist/core/EntityManager.js +21 -54
- package/dist/core/GraphStorage.d.ts +0 -51
- package/dist/core/GraphStorage.d.ts.map +1 -1
- package/dist/core/GraphStorage.js +2 -79
- package/dist/core/GraphTraversal.d.ts +2 -7
- package/dist/core/GraphTraversal.d.ts.map +1 -1
- package/dist/core/GraphTraversal.js +2 -19
- package/dist/core/ManagerContext.d.ts +0 -4
- package/dist/core/ManagerContext.d.ts.map +1 -1
- package/dist/core/ManagerContext.js +2 -12
- package/dist/core/RelationManager.d.ts.map +1 -1
- package/dist/core/RelationManager.js +4 -5
- package/dist/core/SQLiteStorage.d.ts.map +1 -1
- package/dist/core/SQLiteStorage.js +2 -3
- package/dist/core/TransactionManager.d.ts +2 -207
- package/dist/core/TransactionManager.d.ts.map +1 -1
- package/dist/core/TransactionManager.js +6 -482
- package/dist/core/index.d.ts +1 -2
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -3
- package/dist/features/ArchiveManager.d.ts +2 -14
- package/dist/features/ArchiveManager.d.ts.map +1 -1
- package/dist/features/ArchiveManager.js +3 -44
- package/dist/features/CompressionManager.d.ts +4 -14
- package/dist/features/CompressionManager.d.ts.map +1 -1
- package/dist/features/CompressionManager.js +9 -74
- package/dist/features/IOManager.d.ts +2 -6
- package/dist/features/IOManager.d.ts.map +1 -1
- package/dist/features/IOManager.js +10 -105
- package/dist/features/StreamingExporter.d.ts +4 -27
- package/dist/features/StreamingExporter.d.ts.map +1 -1
- package/dist/features/StreamingExporter.js +4 -65
- package/dist/features/index.d.ts +0 -2
- package/dist/features/index.d.ts.map +1 -1
- package/dist/features/index.js +0 -3
- package/dist/search/EmbeddingService.d.ts +9 -108
- package/dist/search/EmbeddingService.d.ts.map +1 -1
- package/dist/search/EmbeddingService.js +15 -187
- package/dist/search/FuzzySearch.js +1 -1
- package/dist/search/SavedSearchManager.d.ts.map +1 -1
- package/dist/search/SavedSearchManager.js +2 -3
- package/dist/search/SearchManager.d.ts +1 -42
- package/dist/search/SearchManager.d.ts.map +1 -1
- package/dist/search/SearchManager.js +0 -115
- package/dist/search/SemanticSearch.d.ts +1 -4
- package/dist/search/SemanticSearch.d.ts.map +1 -1
- package/dist/search/SemanticSearch.js +2 -12
- package/dist/search/TFIDFIndexManager.d.ts +0 -88
- package/dist/search/TFIDFIndexManager.d.ts.map +1 -1
- package/dist/search/TFIDFIndexManager.js +0 -217
- package/dist/search/index.d.ts +1 -18
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +1 -32
- package/dist/server/MCPServer.d.ts.map +1 -1
- package/dist/server/MCPServer.js +4 -1
- package/dist/server/responseCompressor.js +5 -5
- package/dist/server/toolDefinitions.d.ts.map +1 -1
- package/dist/server/toolDefinitions.js +5 -1
- package/dist/server/toolHandlers.d.ts +9 -5
- package/dist/server/toolHandlers.d.ts.map +1 -1
- package/dist/server/toolHandlers.js +23 -8
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/types.d.ts +2 -579
- package/dist/types/types.d.ts.map +1 -1
- package/dist/utils/compressedCache.d.ts +0 -29
- package/dist/utils/compressedCache.d.ts.map +1 -1
- package/dist/utils/compressedCache.js +0 -39
- package/dist/utils/entityUtils.d.ts +1 -59
- package/dist/utils/entityUtils.d.ts.map +1 -1
- package/dist/utils/entityUtils.js +3 -113
- package/dist/utils/errors.d.ts +0 -18
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +0 -24
- package/dist/utils/index.d.ts +2 -6
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -14
- package/dist/utils/logger.d.ts +0 -7
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/logger.js +2 -9
- package/dist/utils/parallelUtils.d.ts +1 -5
- package/dist/utils/parallelUtils.d.ts.map +1 -1
- package/dist/utils/parallelUtils.js +1 -23
- package/dist/utils/schemas.d.ts +16 -16
- package/dist/utils/schemas.d.ts.map +1 -1
- package/dist/utils/schemas.js +12 -12
- package/dist/utils/taskScheduler.d.ts +0 -4
- package/dist/utils/taskScheduler.d.ts.map +1 -1
- package/dist/utils/taskScheduler.js +1 -21
- package/dist/workers/WorkerPool.d.ts +81 -0
- package/dist/workers/WorkerPool.d.ts.map +1 -0
- package/dist/workers/WorkerPool.js +121 -0
- package/dist/workers/index.d.ts +1 -1
- package/dist/workers/index.d.ts.map +1 -1
- package/dist/workers/levenshteinWorker.js +1 -1
- package/package.json +1 -4
- package/dist/__tests__/file-path.test.js +0 -119
- package/dist/__tests__/knowledge-graph.test.js +0 -318
- package/dist/core/GraphEventEmitter.d.ts +0 -202
- package/dist/core/GraphEventEmitter.d.ts.map +0 -1
- package/dist/core/GraphEventEmitter.js +0 -346
- package/dist/features/KeywordExtractor.d.ts +0 -61
- package/dist/features/KeywordExtractor.d.ts.map +0 -1
- package/dist/features/KeywordExtractor.js +0 -126
- package/dist/features/ObservationNormalizer.d.ts +0 -90
- package/dist/features/ObservationNormalizer.d.ts.map +0 -1
- package/dist/features/ObservationNormalizer.js +0 -193
- package/dist/memory.jsonl +0 -1
- package/dist/search/BM25Search.d.ts +0 -148
- package/dist/search/BM25Search.d.ts.map +0 -1
- package/dist/search/BM25Search.js +0 -339
- package/dist/search/EarlyTerminationManager.d.ts +0 -140
- package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
- package/dist/search/EarlyTerminationManager.js +0 -279
- package/dist/search/EmbeddingCache.d.ts +0 -175
- package/dist/search/EmbeddingCache.d.ts.map +0 -1
- package/dist/search/EmbeddingCache.js +0 -246
- package/dist/search/HybridScorer.d.ts +0 -181
- package/dist/search/HybridScorer.d.ts.map +0 -1
- package/dist/search/HybridScorer.js +0 -257
- package/dist/search/HybridSearchManager.d.ts +0 -80
- package/dist/search/HybridSearchManager.d.ts.map +0 -1
- package/dist/search/HybridSearchManager.js +0 -187
- package/dist/search/IncrementalIndexer.d.ts +0 -201
- package/dist/search/IncrementalIndexer.d.ts.map +0 -1
- package/dist/search/IncrementalIndexer.js +0 -342
- package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
- package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
- package/dist/search/OptimizedInvertedIndex.js +0 -358
- package/dist/search/ParallelSearchExecutor.d.ts +0 -172
- package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
- package/dist/search/ParallelSearchExecutor.js +0 -309
- package/dist/search/QuantizedVectorStore.d.ts +0 -171
- package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
- package/dist/search/QuantizedVectorStore.js +0 -307
- package/dist/search/QueryAnalyzer.d.ts +0 -76
- package/dist/search/QueryAnalyzer.d.ts.map +0 -1
- package/dist/search/QueryAnalyzer.js +0 -227
- package/dist/search/QueryCostEstimator.d.ts +0 -244
- package/dist/search/QueryCostEstimator.d.ts.map +0 -1
- package/dist/search/QueryCostEstimator.js +0 -652
- package/dist/search/QueryPlanCache.d.ts +0 -220
- package/dist/search/QueryPlanCache.d.ts.map +0 -1
- package/dist/search/QueryPlanCache.js +0 -379
- package/dist/search/QueryPlanner.d.ts +0 -58
- package/dist/search/QueryPlanner.d.ts.map +0 -1
- package/dist/search/QueryPlanner.js +0 -137
- package/dist/search/ReflectionManager.d.ts +0 -120
- package/dist/search/ReflectionManager.d.ts.map +0 -1
- package/dist/search/ReflectionManager.js +0 -231
- package/dist/search/SymbolicSearch.d.ts +0 -61
- package/dist/search/SymbolicSearch.d.ts.map +0 -1
- package/dist/search/SymbolicSearch.js +0 -163
- package/dist/search/TFIDFEventSync.d.ts +0 -85
- package/dist/search/TFIDFEventSync.d.ts.map +0 -1
- package/dist/search/TFIDFEventSync.js +0 -133
- package/dist/utils/BatchProcessor.d.ts +0 -271
- package/dist/utils/BatchProcessor.d.ts.map +0 -1
- package/dist/utils/BatchProcessor.js +0 -376
- package/dist/utils/MemoryMonitor.d.ts +0 -176
- package/dist/utils/MemoryMonitor.d.ts.map +0 -1
- package/dist/utils/MemoryMonitor.js +0 -305
- package/dist/utils/WorkerPoolManager.d.ts +0 -233
- package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
- package/dist/utils/WorkerPoolManager.js +0 -420
- package/dist/utils/operationUtils.d.ts +0 -124
- package/dist/utils/operationUtils.d.ts.map +0 -1
- package/dist/utils/operationUtils.js +0 -175
- package/dist/vitest.config.js +0 -13
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Hybrid Search Manager
|
|
3
|
-
*
|
|
4
|
-
* Phase 11: Orchestrates three-layer hybrid search combining
|
|
5
|
-
* semantic, lexical, and symbolic signals.
|
|
6
|
-
*
|
|
7
|
-
* @module search/HybridSearchManager
|
|
8
|
-
*/
|
|
9
|
-
import type { HybridSearchOptions, HybridSearchResult, ReadonlyKnowledgeGraph } from '../types/index.js';
|
|
10
|
-
import type { SemanticSearch } from './SemanticSearch.js';
|
|
11
|
-
import type { RankedSearch } from './RankedSearch.js';
|
|
12
|
-
import { SymbolicSearch } from './SymbolicSearch.js';
|
|
13
|
-
/**
|
|
14
|
-
* Default weights for hybrid search layers.
|
|
15
|
-
*/
|
|
16
|
-
export declare const DEFAULT_HYBRID_WEIGHTS: {
|
|
17
|
-
semantic: number;
|
|
18
|
-
lexical: number;
|
|
19
|
-
symbolic: number;
|
|
20
|
-
};
|
|
21
|
-
/**
|
|
22
|
-
* Hybrid Search Manager
|
|
23
|
-
*
|
|
24
|
-
* Combines three search layers:
|
|
25
|
-
* 1. Semantic: Vector similarity via embeddings
|
|
26
|
-
* 2. Lexical: Keyword matching via TF-IDF/BM25
|
|
27
|
-
* 3. Symbolic: Structured metadata filtering
|
|
28
|
-
*
|
|
29
|
-
* @example
|
|
30
|
-
* ```typescript
|
|
31
|
-
* const hybrid = new HybridSearchManager(semanticSearch, rankedSearch);
|
|
32
|
-
* const results = await hybrid.search(graph, 'machine learning', {
|
|
33
|
-
* semanticWeight: 0.5,
|
|
34
|
-
* lexicalWeight: 0.3,
|
|
35
|
-
* symbolicWeight: 0.2,
|
|
36
|
-
* symbolic: { tags: ['ai'] }
|
|
37
|
-
* });
|
|
38
|
-
* ```
|
|
39
|
-
*/
|
|
40
|
-
export declare class HybridSearchManager {
|
|
41
|
-
private semanticSearch;
|
|
42
|
-
private rankedSearch;
|
|
43
|
-
private symbolicSearch;
|
|
44
|
-
constructor(semanticSearch: SemanticSearch | null, rankedSearch: RankedSearch);
|
|
45
|
-
/**
|
|
46
|
-
* Perform hybrid search combining all three layers.
|
|
47
|
-
*
|
|
48
|
-
* @param graph - Knowledge graph to search
|
|
49
|
-
* @param query - Search query text
|
|
50
|
-
* @param options - Hybrid search options with weights
|
|
51
|
-
* @returns Combined and ranked results
|
|
52
|
-
*/
|
|
53
|
-
search(graph: ReadonlyKnowledgeGraph, query: string, options?: Partial<HybridSearchOptions>): Promise<HybridSearchResult[]>;
|
|
54
|
-
/**
|
|
55
|
-
* Execute semantic search layer.
|
|
56
|
-
*/
|
|
57
|
-
private executeSemanticSearch;
|
|
58
|
-
/**
|
|
59
|
-
* Execute lexical search layer (TF-IDF/BM25).
|
|
60
|
-
*/
|
|
61
|
-
private executeLexicalSearch;
|
|
62
|
-
/**
|
|
63
|
-
* Execute symbolic search layer.
|
|
64
|
-
*/
|
|
65
|
-
private executeSymbolicSearch;
|
|
66
|
-
/**
|
|
67
|
-
* Merge results from all three layers.
|
|
68
|
-
*/
|
|
69
|
-
private mergeResults;
|
|
70
|
-
/**
|
|
71
|
-
* Search with full entity resolution.
|
|
72
|
-
* Alias for search() since we now always resolve entities.
|
|
73
|
-
*/
|
|
74
|
-
searchWithEntities(graph: ReadonlyKnowledgeGraph, query: string, options?: Partial<HybridSearchOptions>): Promise<HybridSearchResult[]>;
|
|
75
|
-
/**
|
|
76
|
-
* Get the symbolic search instance for direct access.
|
|
77
|
-
*/
|
|
78
|
-
getSymbolicSearch(): SymbolicSearch;
|
|
79
|
-
}
|
|
80
|
-
//# sourceMappingURL=HybridSearchManager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"HybridSearchManager.d.ts","sourceRoot":"","sources":["../../src/search/HybridSearchManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAEV,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EAEvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAGrD;;GAEG;AACH,eAAO,MAAM,sBAAsB;;;;CAIlC,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,mBAAmB;IAI5B,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,YAAY;IAJtB,OAAO,CAAC,cAAc,CAAiB;gBAG7B,cAAc,EAAE,cAAc,GAAG,IAAI,EACrC,YAAY,EAAE,YAAY;IAKpC;;;;;;;OAOG;IACG,MAAM,CACV,KAAK,EAAE,sBAAsB,EAC7B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM,GACzC,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAuChC;;OAEG;YACW,qBAAqB;IA8BnC;;OAEG;YACW,oBAAoB;IA4BlC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsB7B;;OAEG;IACH,OAAO,CAAC,YAAY;IAkDpB;;;OAGG;IACG,kBAAkB,CACtB,KAAK,EAAE,sBAAsB,EAC7B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM,GACzC,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAIhC;;OAEG;IACH,iBAAiB,IAAI,cAAc;CAGpC"}
|
|
@@ -1,187 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Hybrid Search Manager
|
|
3
|
-
*
|
|
4
|
-
* Phase 11: Orchestrates three-layer hybrid search combining
|
|
5
|
-
* semantic, lexical, and symbolic signals.
|
|
6
|
-
*
|
|
7
|
-
* @module search/HybridSearchManager
|
|
8
|
-
*/
|
|
9
|
-
import { SymbolicSearch } from './SymbolicSearch.js';
|
|
10
|
-
import { SEMANTIC_SEARCH_LIMITS } from '../utils/constants.js';
|
|
11
|
-
/**
|
|
12
|
-
* Default weights for hybrid search layers.
|
|
13
|
-
*/
|
|
14
|
-
export const DEFAULT_HYBRID_WEIGHTS = {
|
|
15
|
-
semantic: 0.5,
|
|
16
|
-
lexical: 0.3,
|
|
17
|
-
symbolic: 0.2,
|
|
18
|
-
};
|
|
19
|
-
/**
|
|
20
|
-
* Hybrid Search Manager
|
|
21
|
-
*
|
|
22
|
-
* Combines three search layers:
|
|
23
|
-
* 1. Semantic: Vector similarity via embeddings
|
|
24
|
-
* 2. Lexical: Keyword matching via TF-IDF/BM25
|
|
25
|
-
* 3. Symbolic: Structured metadata filtering
|
|
26
|
-
*
|
|
27
|
-
* @example
|
|
28
|
-
* ```typescript
|
|
29
|
-
* const hybrid = new HybridSearchManager(semanticSearch, rankedSearch);
|
|
30
|
-
* const results = await hybrid.search(graph, 'machine learning', {
|
|
31
|
-
* semanticWeight: 0.5,
|
|
32
|
-
* lexicalWeight: 0.3,
|
|
33
|
-
* symbolicWeight: 0.2,
|
|
34
|
-
* symbolic: { tags: ['ai'] }
|
|
35
|
-
* });
|
|
36
|
-
* ```
|
|
37
|
-
*/
|
|
38
|
-
export class HybridSearchManager {
|
|
39
|
-
semanticSearch;
|
|
40
|
-
rankedSearch;
|
|
41
|
-
symbolicSearch;
|
|
42
|
-
constructor(semanticSearch, rankedSearch) {
|
|
43
|
-
this.semanticSearch = semanticSearch;
|
|
44
|
-
this.rankedSearch = rankedSearch;
|
|
45
|
-
this.symbolicSearch = new SymbolicSearch();
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Perform hybrid search combining all three layers.
|
|
49
|
-
*
|
|
50
|
-
* @param graph - Knowledge graph to search
|
|
51
|
-
* @param query - Search query text
|
|
52
|
-
* @param options - Hybrid search options with weights
|
|
53
|
-
* @returns Combined and ranked results
|
|
54
|
-
*/
|
|
55
|
-
async search(graph, query, options = {}) {
|
|
56
|
-
const { semanticWeight = DEFAULT_HYBRID_WEIGHTS.semantic, lexicalWeight = DEFAULT_HYBRID_WEIGHTS.lexical, symbolicWeight = DEFAULT_HYBRID_WEIGHTS.symbolic, semantic = {}, lexical = {}, symbolic = {}, limit = SEMANTIC_SEARCH_LIMITS.DEFAULT_LIMIT, } = options;
|
|
57
|
-
// Normalize weights
|
|
58
|
-
const totalWeight = semanticWeight + lexicalWeight + symbolicWeight;
|
|
59
|
-
const normSemantic = semanticWeight / totalWeight;
|
|
60
|
-
const normLexical = lexicalWeight / totalWeight;
|
|
61
|
-
const normSymbolic = symbolicWeight / totalWeight;
|
|
62
|
-
// Execute searches in parallel
|
|
63
|
-
const [semanticResults, lexicalResults, symbolicResults] = await Promise.all([
|
|
64
|
-
this.executeSemanticSearch(graph, query, semantic, limit * 2),
|
|
65
|
-
this.executeLexicalSearch(query, lexical, limit * 2),
|
|
66
|
-
this.executeSymbolicSearch(graph.entities, symbolic),
|
|
67
|
-
]);
|
|
68
|
-
// Merge results
|
|
69
|
-
const merged = this.mergeResults(graph.entities, semanticResults, lexicalResults, symbolicResults, { semantic: normSemantic, lexical: normLexical, symbolic: normSymbolic });
|
|
70
|
-
// Sort by combined score and limit
|
|
71
|
-
return merged
|
|
72
|
-
.sort((a, b) => b.scores.combined - a.scores.combined)
|
|
73
|
-
.slice(0, limit);
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Execute semantic search layer.
|
|
77
|
-
*/
|
|
78
|
-
async executeSemanticSearch(graph, query, options, limit) {
|
|
79
|
-
const results = new Map();
|
|
80
|
-
if (!this.semanticSearch) {
|
|
81
|
-
return results; // Semantic search not available
|
|
82
|
-
}
|
|
83
|
-
try {
|
|
84
|
-
const semanticResults = await this.semanticSearch.search(graph, query, options.topK ?? limit, options.minSimilarity ?? 0);
|
|
85
|
-
for (const result of semanticResults) {
|
|
86
|
-
results.set(result.entity.name, result.similarity);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
catch {
|
|
90
|
-
// Semantic search may fail if not indexed
|
|
91
|
-
}
|
|
92
|
-
return results;
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Execute lexical search layer (TF-IDF/BM25).
|
|
96
|
-
*/
|
|
97
|
-
async executeLexicalSearch(query, _options, limit) {
|
|
98
|
-
const results = new Map();
|
|
99
|
-
try {
|
|
100
|
-
const lexicalResults = await this.rankedSearch.searchNodesRanked(query, undefined, // tags
|
|
101
|
-
undefined, // minImportance
|
|
102
|
-
undefined, // maxImportance
|
|
103
|
-
limit);
|
|
104
|
-
// Normalize scores to 0-1 range
|
|
105
|
-
const maxScore = Math.max(...lexicalResults.map(r => r.score), 1);
|
|
106
|
-
for (const result of lexicalResults) {
|
|
107
|
-
results.set(result.entity.name, result.score / maxScore);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
catch {
|
|
111
|
-
// Lexical search may fail
|
|
112
|
-
}
|
|
113
|
-
return results;
|
|
114
|
-
}
|
|
115
|
-
/**
|
|
116
|
-
* Execute symbolic search layer.
|
|
117
|
-
*/
|
|
118
|
-
executeSymbolicSearch(entities, filters) {
|
|
119
|
-
const results = new Map();
|
|
120
|
-
if (!filters || Object.keys(filters).length === 0) {
|
|
121
|
-
// No symbolic filters, give all entities base score
|
|
122
|
-
for (const entity of entities) {
|
|
123
|
-
results.set(entity.name, 0.5);
|
|
124
|
-
}
|
|
125
|
-
return results;
|
|
126
|
-
}
|
|
127
|
-
const symbolicResults = this.symbolicSearch.search(entities, filters);
|
|
128
|
-
for (const result of symbolicResults) {
|
|
129
|
-
results.set(result.entity.name, result.score);
|
|
130
|
-
}
|
|
131
|
-
return results;
|
|
132
|
-
}
|
|
133
|
-
/**
|
|
134
|
-
* Merge results from all three layers.
|
|
135
|
-
*/
|
|
136
|
-
mergeResults(entities, semanticScores, lexicalScores, symbolicScores, weights) {
|
|
137
|
-
// Collect all unique entity names that have at least one non-zero score
|
|
138
|
-
const allNames = new Set([
|
|
139
|
-
...semanticScores.keys(),
|
|
140
|
-
...lexicalScores.keys(),
|
|
141
|
-
...symbolicScores.keys(),
|
|
142
|
-
]);
|
|
143
|
-
// Create entity lookup map
|
|
144
|
-
const entityMap = new Map(entities.map(e => [e.name, e]));
|
|
145
|
-
const results = [];
|
|
146
|
-
for (const name of allNames) {
|
|
147
|
-
const entity = entityMap.get(name);
|
|
148
|
-
if (!entity)
|
|
149
|
-
continue;
|
|
150
|
-
const semantic = semanticScores.get(name) ?? 0;
|
|
151
|
-
const lexical = lexicalScores.get(name) ?? 0;
|
|
152
|
-
const symbolic = symbolicScores.get(name) ?? 0;
|
|
153
|
-
const combined = semantic * weights.semantic +
|
|
154
|
-
lexical * weights.lexical +
|
|
155
|
-
symbolic * weights.symbolic;
|
|
156
|
-
const matchedLayers = [];
|
|
157
|
-
if (semantic > 0)
|
|
158
|
-
matchedLayers.push('semantic');
|
|
159
|
-
if (lexical > 0)
|
|
160
|
-
matchedLayers.push('lexical');
|
|
161
|
-
if (symbolic > 0)
|
|
162
|
-
matchedLayers.push('symbolic');
|
|
163
|
-
// Skip if no layers matched meaningfully
|
|
164
|
-
if (matchedLayers.length === 0)
|
|
165
|
-
continue;
|
|
166
|
-
results.push({
|
|
167
|
-
entity,
|
|
168
|
-
scores: { semantic, lexical, symbolic, combined },
|
|
169
|
-
matchedLayers,
|
|
170
|
-
});
|
|
171
|
-
}
|
|
172
|
-
return results;
|
|
173
|
-
}
|
|
174
|
-
/**
|
|
175
|
-
* Search with full entity resolution.
|
|
176
|
-
* Alias for search() since we now always resolve entities.
|
|
177
|
-
*/
|
|
178
|
-
async searchWithEntities(graph, query, options = {}) {
|
|
179
|
-
return this.search(graph, query, options);
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Get the symbolic search instance for direct access.
|
|
183
|
-
*/
|
|
184
|
-
getSymbolicSearch() {
|
|
185
|
-
return this.symbolicSearch;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
@@ -1,201 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Incremental Indexer
|
|
3
|
-
*
|
|
4
|
-
* Phase 12 Sprint 5: Queues embedding index updates and batch-processes them
|
|
5
|
-
* efficiently using flush thresholds and timer-based flushing.
|
|
6
|
-
*
|
|
7
|
-
* @module search/IncrementalIndexer
|
|
8
|
-
*/
|
|
9
|
-
import type { EmbeddingService, EmbeddingMode } from '../types/index.js';
|
|
10
|
-
import type { IVectorStore } from '../types/index.js';
|
|
11
|
-
import type { EmbeddingProgressCallback } from './EmbeddingService.js';
|
|
12
|
-
/**
|
|
13
|
-
* Types of index operations.
|
|
14
|
-
*/
|
|
15
|
-
export type IndexOperationType = 'create' | 'update' | 'delete';
|
|
16
|
-
/**
|
|
17
|
-
* Queued index operation.
|
|
18
|
-
*/
|
|
19
|
-
export interface IndexOperation {
|
|
20
|
-
/** Type of operation */
|
|
21
|
-
type: IndexOperationType;
|
|
22
|
-
/** Entity name */
|
|
23
|
-
entityName: string;
|
|
24
|
-
/** Text to embed (for create/update) */
|
|
25
|
-
text?: string;
|
|
26
|
-
/** Timestamp when operation was queued */
|
|
27
|
-
queuedAt: number;
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Options for IncrementalIndexer.
|
|
31
|
-
*/
|
|
32
|
-
export interface IncrementalIndexerOptions {
|
|
33
|
-
/** Flush threshold - number of operations to queue before auto-flush (default: 50) */
|
|
34
|
-
flushThreshold?: number;
|
|
35
|
-
/** Timer-based flush interval in milliseconds (default: 5000ms) */
|
|
36
|
-
flushIntervalMs?: number;
|
|
37
|
-
/** Embedding mode for new embeddings (default: 'document') */
|
|
38
|
-
embeddingMode?: EmbeddingMode;
|
|
39
|
-
/** Progress callback for batch operations */
|
|
40
|
-
onProgress?: EmbeddingProgressCallback;
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Default indexer options.
|
|
44
|
-
*/
|
|
45
|
-
export declare const DEFAULT_INDEXER_OPTIONS: Required<Omit<IncrementalIndexerOptions, 'onProgress'>>;
|
|
46
|
-
/**
|
|
47
|
-
* Result of a flush operation.
|
|
48
|
-
*/
|
|
49
|
-
export interface FlushResult {
|
|
50
|
-
/** Number of operations processed */
|
|
51
|
-
processed: number;
|
|
52
|
-
/** Number of successful operations */
|
|
53
|
-
succeeded: number;
|
|
54
|
-
/** Number of failed operations */
|
|
55
|
-
failed: number;
|
|
56
|
-
/** Errors encountered during flush */
|
|
57
|
-
errors: Array<{
|
|
58
|
-
entityName: string;
|
|
59
|
-
error: string;
|
|
60
|
-
}>;
|
|
61
|
-
/** Duration of flush in milliseconds */
|
|
62
|
-
durationMs: number;
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Incremental indexer for embedding vectors.
|
|
66
|
-
*
|
|
67
|
-
* Queues index updates and batch-processes them efficiently:
|
|
68
|
-
* - Auto-flush when queue reaches threshold
|
|
69
|
-
* - Timer-based flush for time-sensitive updates
|
|
70
|
-
* - Supports create, update, and delete operations
|
|
71
|
-
* - Graceful shutdown with final flush
|
|
72
|
-
*
|
|
73
|
-
* @example
|
|
74
|
-
* ```typescript
|
|
75
|
-
* const indexer = new IncrementalIndexer(embeddingService, vectorStore, {
|
|
76
|
-
* flushThreshold: 100,
|
|
77
|
-
* flushIntervalMs: 10000,
|
|
78
|
-
* });
|
|
79
|
-
*
|
|
80
|
-
* // Queue operations
|
|
81
|
-
* indexer.queueCreate('entity1', 'Entity text content');
|
|
82
|
-
* indexer.queueUpdate('entity2', 'Updated text content');
|
|
83
|
-
* indexer.queueDelete('entity3');
|
|
84
|
-
*
|
|
85
|
-
* // Manual flush
|
|
86
|
-
* const result = await indexer.flush();
|
|
87
|
-
* console.log(`Processed ${result.processed} operations`);
|
|
88
|
-
*
|
|
89
|
-
* // Graceful shutdown
|
|
90
|
-
* await indexer.shutdown();
|
|
91
|
-
* ```
|
|
92
|
-
*/
|
|
93
|
-
export declare class IncrementalIndexer {
|
|
94
|
-
private embeddingService;
|
|
95
|
-
private vectorStore;
|
|
96
|
-
private options;
|
|
97
|
-
private onProgress?;
|
|
98
|
-
private queue;
|
|
99
|
-
private flushTimer;
|
|
100
|
-
private isFlushing;
|
|
101
|
-
private isShutdown;
|
|
102
|
-
/**
|
|
103
|
-
* Create a new incremental indexer.
|
|
104
|
-
*
|
|
105
|
-
* @param embeddingService - Service for generating embeddings
|
|
106
|
-
* @param vectorStore - Store for embedding vectors
|
|
107
|
-
* @param options - Indexer configuration options
|
|
108
|
-
*/
|
|
109
|
-
constructor(embeddingService: EmbeddingService, vectorStore: IVectorStore, options?: IncrementalIndexerOptions);
|
|
110
|
-
/**
|
|
111
|
-
* Start the timer-based flush interval.
|
|
112
|
-
*/
|
|
113
|
-
private startFlushTimer;
|
|
114
|
-
/**
|
|
115
|
-
* Stop the flush timer.
|
|
116
|
-
*/
|
|
117
|
-
private stopFlushTimer;
|
|
118
|
-
/**
|
|
119
|
-
* Queue a create operation.
|
|
120
|
-
*
|
|
121
|
-
* @param entityName - Name of the entity to create
|
|
122
|
-
* @param text - Text content to embed
|
|
123
|
-
*/
|
|
124
|
-
queueCreate(entityName: string, text: string): void;
|
|
125
|
-
/**
|
|
126
|
-
* Queue an update operation.
|
|
127
|
-
*
|
|
128
|
-
* @param entityName - Name of the entity to update
|
|
129
|
-
* @param text - Updated text content to embed
|
|
130
|
-
*/
|
|
131
|
-
queueUpdate(entityName: string, text: string): void;
|
|
132
|
-
/**
|
|
133
|
-
* Queue a delete operation.
|
|
134
|
-
*
|
|
135
|
-
* @param entityName - Name of the entity to delete
|
|
136
|
-
*/
|
|
137
|
-
queueDelete(entityName: string): void;
|
|
138
|
-
/**
|
|
139
|
-
* Remove all queued operations for an entity.
|
|
140
|
-
*
|
|
141
|
-
* @param entityName - Entity name to remove from queue
|
|
142
|
-
*/
|
|
143
|
-
private removeFromQueue;
|
|
144
|
-
/**
|
|
145
|
-
* Check if auto-flush threshold is reached.
|
|
146
|
-
*/
|
|
147
|
-
private checkAutoFlush;
|
|
148
|
-
/**
|
|
149
|
-
* Flush the queue and process all pending operations.
|
|
150
|
-
*
|
|
151
|
-
* @returns Result of the flush operation
|
|
152
|
-
*/
|
|
153
|
-
flush(): Promise<FlushResult>;
|
|
154
|
-
/**
|
|
155
|
-
* Get the current queue size.
|
|
156
|
-
*
|
|
157
|
-
* @returns Number of operations in the queue
|
|
158
|
-
*/
|
|
159
|
-
getQueueSize(): number;
|
|
160
|
-
/**
|
|
161
|
-
* Get the current queue contents (for debugging/monitoring).
|
|
162
|
-
*
|
|
163
|
-
* @returns Copy of the current queue
|
|
164
|
-
*/
|
|
165
|
-
getQueue(): IndexOperation[];
|
|
166
|
-
/**
|
|
167
|
-
* Check if the indexer is currently flushing.
|
|
168
|
-
*
|
|
169
|
-
* @returns True if flushing
|
|
170
|
-
*/
|
|
171
|
-
isBusy(): boolean;
|
|
172
|
-
/**
|
|
173
|
-
* Check if the indexer is shutdown.
|
|
174
|
-
*
|
|
175
|
-
* @returns True if shutdown
|
|
176
|
-
*/
|
|
177
|
-
isShutdownComplete(): boolean;
|
|
178
|
-
/**
|
|
179
|
-
* Graceful shutdown with final flush.
|
|
180
|
-
*
|
|
181
|
-
* Stops the timer and flushes any remaining operations.
|
|
182
|
-
*
|
|
183
|
-
* @returns Result of the final flush
|
|
184
|
-
*/
|
|
185
|
-
shutdown(): Promise<FlushResult>;
|
|
186
|
-
/**
|
|
187
|
-
* Update indexer options.
|
|
188
|
-
*
|
|
189
|
-
* Note: Changes to flushIntervalMs will take effect on the next interval.
|
|
190
|
-
*
|
|
191
|
-
* @param options - New options to apply
|
|
192
|
-
*/
|
|
193
|
-
updateOptions(options: Partial<IncrementalIndexerOptions>): void;
|
|
194
|
-
/**
|
|
195
|
-
* Clear the queue without processing.
|
|
196
|
-
*
|
|
197
|
-
* @returns Number of operations cleared
|
|
198
|
-
*/
|
|
199
|
-
clearQueue(): number;
|
|
200
|
-
}
|
|
201
|
-
//# sourceMappingURL=IncrementalIndexer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"IncrementalIndexer.d.ts","sourceRoot":"","sources":["../../src/search/IncrementalIndexer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,wBAAwB;IACxB,IAAI,EAAE,kBAAkB,CAAC;IACzB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,wCAAwC;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,sFAAsF;IACtF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,6CAA6C;IAC7C,UAAU,CAAC,EAAE,yBAAyB,CAAC;CACxC;AAED;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,QAAQ,CAAC,IAAI,CAAC,yBAAyB,EAAE,YAAY,CAAC,CAI3F,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrD,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,OAAO,CAA0D;IACzE,OAAO,CAAC,UAAU,CAAC,CAA4B;IAE/C,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAS;IAE3B;;;;;;OAMG;gBAED,gBAAgB,EAAE,gBAAgB,EAClC,WAAW,EAAE,YAAY,EACzB,OAAO,CAAC,EAAE,yBAAyB;IAWrC;;OAEG;IACH,OAAO,CAAC,eAAe;IAYvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;;;;OAKG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAkBnD;;;;;OAKG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAkBnD;;;;OAIG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAiBrC;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAIvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,WAAW,CAAC;IA+FnC;;;;OAIG;IACH,YAAY,IAAI,MAAM;IAItB;;;;OAIG;IACH,QAAQ,IAAI,cAAc,EAAE;IAI5B;;;;OAIG;IACH,MAAM,IAAI,OAAO;IAIjB;;;;OAIG;IACH,kBAAkB,IAAI,OAAO;IAI7B;;;;;;OAMG;IACG,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC;IAatC;;;;;;OAMG;IACH,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,GAAG,IAAI;IAkBhE;;;;OAIG;IACH,UAAU,IAAI,MAAM;CAKrB"}
|