causantic 0.10.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -7
- package/config.schema.json +66 -0
- package/dist/cli/commands/init/ingest.js +2 -2
- package/dist/cli/commands/init/ingest.js.map +1 -1
- package/dist/cli/skill-templates.js +2 -2
- package/dist/clusters/cluster-manager.d.ts.map +1 -1
- package/dist/clusters/cluster-manager.js.map +1 -1
- package/dist/config/bootstrap.d.ts +20 -0
- package/dist/config/bootstrap.d.ts.map +1 -0
- package/dist/config/bootstrap.js +24 -0
- package/dist/config/bootstrap.js.map +1 -0
- package/dist/config/index.d.ts +1 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +1 -0
- package/dist/config/index.js.map +1 -1
- package/dist/config/loader.d.ts +16 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +194 -176
- package/dist/config/loader.js.map +1 -1
- package/dist/config/memory-config.d.ts +12 -0
- package/dist/config/memory-config.d.ts.map +1 -1
- package/dist/config/memory-config.js +64 -6
- package/dist/config/memory-config.js.map +1 -1
- package/dist/dashboard/server.d.ts.map +1 -1
- package/dist/dashboard/server.js +3 -1
- package/dist/dashboard/server.js.map +1 -1
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.js.map +1 -1
- package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts.map +1 -1
- package/dist/eval/experiments/index-differentiation/alignment-analysis.js +3 -9
- package/dist/eval/experiments/index-differentiation/alignment-analysis.js.map +1 -1
- package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts.map +1 -1
- package/dist/eval/experiments/index-differentiation/discrimination-test.js.map +1 -1
- package/dist/eval/experiments/index-differentiation/refinement-test.d.ts.map +1 -1
- package/dist/eval/experiments/index-differentiation/refinement-test.js +1 -3
- package/dist/eval/experiments/index-differentiation/refinement-test.js.map +1 -1
- package/dist/eval/experiments/index-differentiation/run-experiment.js +5 -7
- package/dist/eval/experiments/index-differentiation/run-experiment.js.map +1 -1
- package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts.map +1 -1
- package/dist/eval/experiments/index-differentiation/similarity-analysis.js.map +1 -1
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js +1 -3
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js.map +1 -1
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js.map +1 -1
- package/dist/eval/experiments/index-vs-chunk/query-generator.js.map +1 -1
- package/dist/eval/experiments/index-vs-chunk/run-experiment.js +6 -16
- package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map +1 -1
- package/dist/eval/experiments/pipeline-dropout/run-experiment.js +12 -4
- package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map +1 -1
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js.map +1 -1
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js +26 -12
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js.map +1 -1
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.js +1 -1
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.js.map +1 -1
- package/dist/hooks/hook-utils.d.ts +1 -1
- package/dist/hooks/hook-utils.d.ts.map +1 -1
- package/dist/hooks/hook-utils.js +4 -2
- package/dist/hooks/hook-utils.js.map +1 -1
- package/dist/hooks/session-start.d.ts.map +1 -1
- package/dist/hooks/session-start.js +4 -1
- package/dist/hooks/session-start.js.map +1 -1
- package/dist/index-entries/index-generator.d.ts.map +1 -1
- package/dist/index-entries/index-generator.js +1 -3
- package/dist/index-entries/index-generator.js.map +1 -1
- package/dist/index-entries/index-refresher.d.ts.map +1 -1
- package/dist/index-entries/index-refresher.js.map +1 -1
- package/dist/index-entries/index.d.ts +1 -1
- package/dist/index-entries/index.d.ts.map +1 -1
- package/dist/index-entries/index.js +1 -1
- package/dist/index-entries/index.js.map +1 -1
- package/dist/ingest/brief-debrief-detector.d.ts.map +1 -1
- package/dist/ingest/brief-debrief-detector.js +6 -5
- package/dist/ingest/brief-debrief-detector.js.map +1 -1
- package/dist/ingest/ingest-session.d.ts.map +1 -1
- package/dist/ingest/ingest-session.js +109 -37
- package/dist/ingest/ingest-session.js.map +1 -1
- package/dist/ingest/session-state.d.ts.map +1 -1
- package/dist/ingest/session-state.js +6 -18
- package/dist/ingest/session-state.js.map +1 -1
- package/dist/mcp/server.d.ts +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +15 -5
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/services.d.ts.map +1 -1
- package/dist/mcp/services.js +9 -0
- package/dist/mcp/services.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +36 -47
- package/dist/mcp/tools.js.map +1 -1
- package/dist/models/embedder.d.ts.map +1 -1
- package/dist/models/embedder.js +1 -0
- package/dist/models/embedder.js.map +1 -1
- package/dist/repomap/parser.d.ts.map +1 -1
- package/dist/repomap/parser.js +71 -22
- package/dist/repomap/parser.js.map +1 -1
- package/dist/repomap/regex-parser.d.ts.map +1 -1
- package/dist/repomap/regex-parser.js +30 -6
- package/dist/repomap/regex-parser.js.map +1 -1
- package/dist/repomap/renderer.d.ts.map +1 -1
- package/dist/repomap/renderer.js.map +1 -1
- package/dist/repomap/scanner.d.ts.map +1 -1
- package/dist/repomap/scanner.js +30 -11
- package/dist/repomap/scanner.js.map +1 -1
- package/dist/retrieval/chain-walker.d.ts.map +1 -1
- package/dist/retrieval/chain-walker.js +6 -2
- package/dist/retrieval/chain-walker.js.map +1 -1
- package/dist/retrieval/context-assembler.d.ts +1 -1
- package/dist/retrieval/context-assembler.d.ts.map +1 -1
- package/dist/retrieval/rrf.d.ts +1 -1
- package/dist/retrieval/rrf.d.ts.map +1 -1
- package/dist/retrieval/rrf.js +1 -1
- package/dist/retrieval/rrf.js.map +1 -1
- package/dist/retrieval/search-assembler.d.ts +1 -1
- package/dist/retrieval/search-assembler.d.ts.map +1 -1
- package/dist/retrieval/search-assembler.js +324 -227
- package/dist/retrieval/search-assembler.js.map +1 -1
- package/dist/retrieval/session-reconstructor.d.ts.map +1 -1
- package/dist/retrieval/session-reconstructor.js +7 -5
- package/dist/retrieval/session-reconstructor.js.map +1 -1
- package/dist/storage/chunk-store.d.ts.map +1 -1
- package/dist/storage/chunk-store.js +2 -0
- package/dist/storage/chunk-store.js.map +1 -1
- package/dist/storage/cluster-store.d.ts.map +1 -1
- package/dist/storage/cluster-store.js +3 -11
- package/dist/storage/cluster-store.js.map +1 -1
- package/dist/storage/db.d.ts +7 -0
- package/dist/storage/db.d.ts.map +1 -1
- package/dist/storage/db.js +25 -4
- package/dist/storage/db.js.map +1 -1
- package/dist/storage/entity-store.d.ts +48 -0
- package/dist/storage/entity-store.d.ts.map +1 -0
- package/dist/storage/entity-store.js +111 -0
- package/dist/storage/entity-store.js.map +1 -0
- package/dist/storage/index-entry-store.d.ts.map +1 -1
- package/dist/storage/index-entry-store.js +39 -40
- package/dist/storage/index-entry-store.js.map +1 -1
- package/dist/storage/keyword-store.d.ts +5 -0
- package/dist/storage/keyword-store.d.ts.map +1 -1
- package/dist/storage/keyword-store.js +1 -1
- package/dist/storage/keyword-store.js.map +1 -1
- package/dist/storage/migrations.d.ts.map +1 -1
- package/dist/storage/migrations.js +45 -0
- package/dist/storage/migrations.js.map +1 -1
- package/dist/storage/schema.sql +38 -2
- package/dist/storage/session-state-store.d.ts.map +1 -1
- package/dist/storage/session-state-store.js +46 -8
- package/dist/storage/session-state-store.js.map +1 -1
- package/dist/storage/types.d.ts +4 -2
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/vector-store-cleanup.d.ts +47 -0
- package/dist/storage/vector-store-cleanup.d.ts.map +1 -0
- package/dist/storage/vector-store-cleanup.js +101 -0
- package/dist/storage/vector-store-cleanup.js.map +1 -0
- package/dist/storage/vector-store.d.ts +13 -1
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +56 -111
- package/dist/storage/vector-store.js.map +1 -1
- package/dist/utils/entity-extractor.d.ts +23 -0
- package/dist/utils/entity-extractor.d.ts.map +1 -0
- package/dist/utils/entity-extractor.js +233 -0
- package/dist/utils/entity-extractor.js.map +1 -0
- package/package.json +3 -2
|
@@ -17,9 +17,13 @@ import { KeywordStore } from '../storage/keyword-store.js';
|
|
|
17
17
|
import { fuseRRF } from './rrf.js';
|
|
18
18
|
import { expandViaClusters } from './cluster-expander.js';
|
|
19
19
|
import { reorderWithMMR } from './mmr.js';
|
|
20
|
+
import { extractEntities } from '../utils/entity-extractor.js';
|
|
21
|
+
import { findEntitiesByAlias, getChunkIdsForEntity } from '../storage/entity-store.js';
|
|
20
22
|
import { createLogger } from '../utils/logger.js';
|
|
21
23
|
import { formatSearchChunk } from './formatting.js';
|
|
22
24
|
const log = createLogger('search-assembler');
|
|
25
|
+
/** RRF weight for entity-boosted results. */
|
|
26
|
+
const ENTITY_RRF_BOOST = 1.5;
|
|
23
27
|
/**
|
|
24
28
|
* Shared embedder instance.
|
|
25
29
|
*/
|
|
@@ -50,243 +54,258 @@ function getKeywordStore() {
|
|
|
50
54
|
}
|
|
51
55
|
return sharedKeywordStore;
|
|
52
56
|
}
|
|
57
|
+
// ── Extracted pipeline stages ────────────────────────────────────────────────
|
|
53
58
|
/**
|
|
54
|
-
*
|
|
59
|
+
* Filter items by agent when agent filtering is active but project filtering is not.
|
|
55
60
|
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
61
|
+
* When projectFilter is set, agent filtering is handled by the storage layer.
|
|
62
|
+
* This function handles the post-filter case where no project scope was provided.
|
|
58
63
|
*/
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
// Post-filter by agent when no project filter was used
|
|
83
|
-
if (agentFilter && !projectFilter) {
|
|
84
|
-
keywordResults = keywordResults.filter((r) => {
|
|
85
|
-
const chunk = getChunkById(r.id);
|
|
86
|
-
return chunk?.agentId === agentFilter;
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
fusedResults = keywordResults.map((r) => ({
|
|
90
|
-
chunkId: r.id,
|
|
91
|
-
score: r.score,
|
|
92
|
-
source: 'keyword',
|
|
93
|
-
}));
|
|
94
|
-
// Optional vector enrichment: merge vector results via RRF
|
|
95
|
-
if (config.vectorEnrichment) {
|
|
96
|
-
try {
|
|
97
|
-
vectorStore.setModelId(embeddingModel);
|
|
98
|
-
const embedder = await getEmbedder(embeddingModel);
|
|
99
|
-
const queryResult = await embedder.embed(query, true);
|
|
100
|
-
queryEmbedding = queryResult.embedding;
|
|
101
|
-
let vectorResults = await (projectFilter
|
|
102
|
-
? vectorStore.searchByProject(queryResult.embedding, projectFilter, vectorSearchLimit, agentFilter)
|
|
103
|
-
: vectorStore.search(queryResult.embedding, vectorSearchLimit));
|
|
104
|
-
if (agentFilter && !projectFilter) {
|
|
105
|
-
vectorResults = vectorResults.filter((s) => {
|
|
106
|
-
const chunk = getChunkById(s.id);
|
|
107
|
-
return chunk?.agentId === agentFilter;
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
if (vectorResults.length > 0) {
|
|
111
|
-
const vectorItems = vectorResults.map((s) => ({
|
|
112
|
-
chunkId: s.id,
|
|
113
|
-
score: Math.max(0, 1 - s.distance),
|
|
114
|
-
source: 'vector',
|
|
115
|
-
}));
|
|
116
|
-
fusedResults = fuseRRF([
|
|
117
|
-
{ items: fusedResults, weight: hybridSearch.keywordWeight },
|
|
118
|
-
{ items: vectorItems, weight: hybridSearch.vectorWeight },
|
|
119
|
-
], hybridSearch.rrfK);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
catch (error) {
|
|
123
|
-
log.warn('Vector enrichment failed, using keyword results only', {
|
|
124
|
-
error: error.message,
|
|
125
|
-
});
|
|
64
|
+
function filterByAgent(items, agentFilter, projectFilter, getAgent) {
|
|
65
|
+
if (!agentFilter || projectFilter)
|
|
66
|
+
return items;
|
|
67
|
+
return items.filter((item) => getAgent(item.id) === agentFilter);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Extract entity mentions from the query and find matching chunks.
|
|
71
|
+
* Returns ranked items suitable for RRF fusion.
|
|
72
|
+
*/
|
|
73
|
+
function getEntityResults(query, projectFilter) {
|
|
74
|
+
const mentions = extractEntities(query);
|
|
75
|
+
if (mentions.length === 0)
|
|
76
|
+
return [];
|
|
77
|
+
const project = typeof projectFilter === 'string' ? projectFilter : undefined;
|
|
78
|
+
if (!project)
|
|
79
|
+
return []; // entity lookup requires project scope
|
|
80
|
+
const chunkIds = new Set();
|
|
81
|
+
for (const mention of mentions) {
|
|
82
|
+
const entities = findEntitiesByAlias(mention.normalizedName, mention.entityType, project);
|
|
83
|
+
for (const entity of entities) {
|
|
84
|
+
for (const cid of getChunkIdsForEntity(entity.id, 100)) {
|
|
85
|
+
chunkIds.add(cid);
|
|
126
86
|
}
|
|
127
87
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
88
|
+
}
|
|
89
|
+
return [...chunkIds].map((id, i) => ({
|
|
90
|
+
chunkId: id,
|
|
91
|
+
score: 1.0 / (i + 1),
|
|
92
|
+
source: 'entity',
|
|
93
|
+
}));
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Merge entity-boosted results into fused results via RRF.
|
|
97
|
+
*/
|
|
98
|
+
function applyEntityBoost(fusedResults, query, projectFilter, rrfK) {
|
|
99
|
+
try {
|
|
100
|
+
const entityItems = getEntityResults(query, projectFilter);
|
|
101
|
+
if (entityItems.length > 0) {
|
|
102
|
+
return fuseRRF([
|
|
103
|
+
{ items: fusedResults, weight: 1.0 },
|
|
104
|
+
{ items: entityItems, weight: ENTITY_RRF_BOOST },
|
|
105
|
+
], rrfK);
|
|
138
106
|
}
|
|
139
|
-
// Skip cluster expansion for keyword-primary mode
|
|
140
107
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
score: r.score,
|
|
200
|
-
source: 'keyword',
|
|
201
|
-
}));
|
|
202
|
-
const indexFused = fuseRRF([
|
|
203
|
-
{ items: indexVectorItems, weight: hybridSearch.vectorWeight },
|
|
204
|
-
...(indexKeywordItems.length > 0
|
|
205
|
-
? [{ items: indexKeywordItems, weight: hybridSearch.keywordWeight }]
|
|
206
|
-
: []),
|
|
207
|
-
], hybridSearch.rrfK);
|
|
208
|
-
const indexEntryIds = indexFused.map((r) => r.chunkId);
|
|
209
|
-
const chunkIds = dereferenceToChunkIds(indexEntryIds);
|
|
210
|
-
const chunkScoreMap = new Map();
|
|
211
|
-
for (const item of indexFused) {
|
|
212
|
-
const entryChunkIds = dereferenceToChunkIds([item.chunkId]);
|
|
213
|
-
for (const cid of entryChunkIds) {
|
|
214
|
-
const existing = chunkScoreMap.get(cid);
|
|
215
|
-
if (!existing || item.score > existing.score) {
|
|
216
|
-
chunkScoreMap.set(cid, { score: item.score, source: item.source });
|
|
217
|
-
}
|
|
218
|
-
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
log.warn('Entity search failed', { error: error.message });
|
|
110
|
+
}
|
|
111
|
+
return fusedResults;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Keyword-primary retrieval path.
|
|
115
|
+
*
|
|
116
|
+
* keyword → [optional vector enrichment] → entity boost
|
|
117
|
+
* No cluster expansion.
|
|
118
|
+
*/
|
|
119
|
+
async function keywordPrimarySearch(query, projectFilter, agentFilter, vectorSearchLimit, config) {
|
|
120
|
+
const { hybridSearch, embeddingModel } = config;
|
|
121
|
+
let queryEmbedding = [];
|
|
122
|
+
let keywordResults = [];
|
|
123
|
+
try {
|
|
124
|
+
const keywordStore = getKeywordStore();
|
|
125
|
+
keywordResults = projectFilter
|
|
126
|
+
? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit, agentFilter)
|
|
127
|
+
: keywordStore.search(query, hybridSearch.keywordSearchLimit);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
log.warn('Keyword search failed', { error: error.message });
|
|
131
|
+
}
|
|
132
|
+
// Post-filter by agent when no project filter was used
|
|
133
|
+
keywordResults = filterByAgent(keywordResults, agentFilter, projectFilter, (id) => {
|
|
134
|
+
const chunk = getChunkById(id);
|
|
135
|
+
return chunk?.agentId;
|
|
136
|
+
});
|
|
137
|
+
let fusedResults = keywordResults.map((r) => ({
|
|
138
|
+
chunkId: r.id,
|
|
139
|
+
score: r.score,
|
|
140
|
+
source: 'keyword',
|
|
141
|
+
}));
|
|
142
|
+
// Optional vector enrichment: merge vector results via RRF
|
|
143
|
+
if (config.vectorEnrichment) {
|
|
144
|
+
try {
|
|
145
|
+
vectorStore.setModelId(embeddingModel);
|
|
146
|
+
const embedder = await getEmbedder(embeddingModel);
|
|
147
|
+
const queryResult = await embedder.embed(query, true);
|
|
148
|
+
queryEmbedding = queryResult.embedding;
|
|
149
|
+
let vectorResults = await (projectFilter
|
|
150
|
+
? vectorStore.searchByProject(queryResult.embedding, projectFilter, vectorSearchLimit, agentFilter)
|
|
151
|
+
: vectorStore.search(queryResult.embedding, vectorSearchLimit));
|
|
152
|
+
vectorResults = filterByAgent(vectorResults, agentFilter, projectFilter, (id) => {
|
|
153
|
+
const chunk = getChunkById(id);
|
|
154
|
+
return chunk?.agentId;
|
|
155
|
+
});
|
|
156
|
+
if (vectorResults.length > 0) {
|
|
157
|
+
const vectorItems = vectorResults.map((s) => ({
|
|
158
|
+
chunkId: s.id,
|
|
159
|
+
score: Math.max(0, 1 - s.distance),
|
|
160
|
+
source: 'vector',
|
|
161
|
+
}));
|
|
162
|
+
fusedResults = fuseRRF([
|
|
163
|
+
{ items: fusedResults, weight: hybridSearch.keywordWeight },
|
|
164
|
+
{ items: vectorItems, weight: hybridSearch.vectorWeight },
|
|
165
|
+
], hybridSearch.rrfK);
|
|
219
166
|
}
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
score: entry?.score ?? 0,
|
|
225
|
-
source: entry?.source,
|
|
226
|
-
};
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
log.warn('Vector enrichment failed, using keyword results only', {
|
|
170
|
+
error: error.message,
|
|
227
171
|
});
|
|
228
172
|
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
173
|
+
}
|
|
174
|
+
// Entity boost
|
|
175
|
+
fusedResults = applyEntityBoost(fusedResults, query, projectFilter, hybridSearch.rrfK);
|
|
176
|
+
if (fusedResults.length === 0) {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
return { fusedResults, queryEmbedding, useIndexSearch: false };
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Index-based hybrid retrieval path.
|
|
183
|
+
*
|
|
184
|
+
* Uses semantic index entries (vector + keyword) → RRF → dereference to chunks.
|
|
185
|
+
*/
|
|
186
|
+
async function indexBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config) {
|
|
187
|
+
const { hybridSearch, embeddingModel } = config;
|
|
188
|
+
indexVectorStore.setModelId(embeddingModel);
|
|
189
|
+
const entryCount = getIndexEntryCount();
|
|
190
|
+
const indexedChunks = getIndexedChunkCount();
|
|
191
|
+
const entriesPerChunk = indexedChunks > 0 ? entryCount / indexedChunks : 1;
|
|
192
|
+
const indexSearchLimit = Math.ceil(vectorSearchLimit * entriesPerChunk);
|
|
193
|
+
const indexVectorPromise = projectFilter
|
|
194
|
+
? indexVectorStore.searchByProject(queryEmbedding, projectFilter, indexSearchLimit, agentFilter)
|
|
195
|
+
: indexVectorStore.search(queryEmbedding, indexSearchLimit);
|
|
196
|
+
let indexKeywordResults = [];
|
|
197
|
+
try {
|
|
198
|
+
indexKeywordResults = searchIndexEntriesByKeyword(query, hybridSearch.keywordSearchLimit, projectFilter, agentFilter);
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
log.warn('Index keyword search unavailable', {
|
|
202
|
+
error: error.message,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
let indexSimilar = await indexVectorPromise;
|
|
206
|
+
indexSimilar = filterByAgent(indexSimilar, agentFilter, projectFilter, (id) => indexVectorStore.getChunkAgent(id));
|
|
207
|
+
indexKeywordResults = filterByAgent(indexKeywordResults, agentFilter, projectFilter, (id) => indexVectorStore.getChunkAgent(id));
|
|
208
|
+
if (indexSimilar.length === 0 && indexKeywordResults.length === 0) {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
const indexVectorItems = indexSimilar.map((s) => ({
|
|
212
|
+
chunkId: s.id,
|
|
213
|
+
score: Math.max(0, 1 - s.distance),
|
|
214
|
+
source: 'vector',
|
|
215
|
+
}));
|
|
216
|
+
const indexKeywordItems = indexKeywordResults.map((r) => ({
|
|
217
|
+
chunkId: r.id,
|
|
218
|
+
score: r.score,
|
|
219
|
+
source: 'keyword',
|
|
220
|
+
}));
|
|
221
|
+
const indexFused = fuseRRF([
|
|
222
|
+
{ items: indexVectorItems, weight: hybridSearch.vectorWeight },
|
|
223
|
+
...(indexKeywordItems.length > 0
|
|
224
|
+
? [{ items: indexKeywordItems, weight: hybridSearch.keywordWeight }]
|
|
225
|
+
: []),
|
|
226
|
+
], hybridSearch.rrfK);
|
|
227
|
+
const indexEntryIds = indexFused.map((r) => r.chunkId);
|
|
228
|
+
const chunkIds = dereferenceToChunkIds(indexEntryIds);
|
|
229
|
+
const chunkScoreMap = new Map();
|
|
230
|
+
for (const item of indexFused) {
|
|
231
|
+
const entryChunkIds = dereferenceToChunkIds([item.chunkId]);
|
|
232
|
+
for (const cid of entryChunkIds) {
|
|
233
|
+
const existing = chunkScoreMap.get(cid);
|
|
234
|
+
if (!existing || item.score > existing.score) {
|
|
235
|
+
chunkScoreMap.set(cid, { score: item.score, source: item.source });
|
|
267
236
|
}
|
|
268
|
-
const vectorItems = similar.map((s) => ({
|
|
269
|
-
chunkId: s.id,
|
|
270
|
-
score: Math.max(0, 1 - s.distance),
|
|
271
|
-
source: 'vector',
|
|
272
|
-
}));
|
|
273
|
-
const keywordItems = keywordResults.map((r) => ({
|
|
274
|
-
chunkId: r.id,
|
|
275
|
-
score: r.score,
|
|
276
|
-
source: 'keyword',
|
|
277
|
-
}));
|
|
278
|
-
fusedResults = fuseRRF([
|
|
279
|
-
{ items: vectorItems, weight: hybridSearch.vectorWeight },
|
|
280
|
-
...(keywordItems.length > 0
|
|
281
|
-
? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
|
|
282
|
-
: []),
|
|
283
|
-
], hybridSearch.rrfK);
|
|
284
|
-
}
|
|
285
|
-
// Cluster expansion (hybrid/vector path only)
|
|
286
|
-
if (!skipClusters) {
|
|
287
|
-
fusedResults = expandViaClusters(fusedResults, clusterExpansion, projectFilter, agentFilter, config.feedbackWeight);
|
|
288
237
|
}
|
|
289
238
|
}
|
|
239
|
+
const fusedResults = chunkIds.map((cid) => {
|
|
240
|
+
const entry = chunkScoreMap.get(cid);
|
|
241
|
+
return {
|
|
242
|
+
chunkId: cid,
|
|
243
|
+
score: entry?.score ?? 0,
|
|
244
|
+
source: entry?.source,
|
|
245
|
+
};
|
|
246
|
+
});
|
|
247
|
+
return { fusedResults, queryEmbedding, useIndexSearch: true };
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Chunk-based hybrid retrieval path (fallback when no semantic index).
|
|
251
|
+
*
|
|
252
|
+
* vector + keyword → RRF
|
|
253
|
+
*/
|
|
254
|
+
async function chunkBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config) {
|
|
255
|
+
const { hybridSearch } = config;
|
|
256
|
+
const vectorSearchPromise = projectFilter
|
|
257
|
+
? vectorStore.searchByProject(queryEmbedding, projectFilter, vectorSearchLimit, agentFilter)
|
|
258
|
+
: vectorStore.search(queryEmbedding, vectorSearchLimit);
|
|
259
|
+
let keywordResults = [];
|
|
260
|
+
try {
|
|
261
|
+
const keywordStore = getKeywordStore();
|
|
262
|
+
keywordResults = projectFilter
|
|
263
|
+
? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit, agentFilter)
|
|
264
|
+
: keywordStore.search(query, hybridSearch.keywordSearchLimit);
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
log.warn('Keyword search unavailable, falling back to vector-only', {
|
|
268
|
+
error: error.message,
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
let similar = await vectorSearchPromise;
|
|
272
|
+
similar = filterByAgent(similar, agentFilter, projectFilter, (id) => {
|
|
273
|
+
const chunk = getChunkById(id);
|
|
274
|
+
return chunk?.agentId;
|
|
275
|
+
});
|
|
276
|
+
keywordResults = filterByAgent(keywordResults, agentFilter, projectFilter, (id) => {
|
|
277
|
+
const chunk = getChunkById(id);
|
|
278
|
+
return chunk?.agentId;
|
|
279
|
+
});
|
|
280
|
+
if (similar.length === 0 && keywordResults.length === 0) {
|
|
281
|
+
return null;
|
|
282
|
+
}
|
|
283
|
+
const vectorItems = similar.map((s) => ({
|
|
284
|
+
chunkId: s.id,
|
|
285
|
+
score: Math.max(0, 1 - s.distance),
|
|
286
|
+
source: 'vector',
|
|
287
|
+
}));
|
|
288
|
+
const keywordItems = keywordResults.map((r) => ({
|
|
289
|
+
chunkId: r.id,
|
|
290
|
+
score: r.score,
|
|
291
|
+
source: 'keyword',
|
|
292
|
+
}));
|
|
293
|
+
const fusedResults = fuseRRF([
|
|
294
|
+
{ items: vectorItems, weight: hybridSearch.vectorWeight },
|
|
295
|
+
...(keywordItems.length > 0
|
|
296
|
+
? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
|
|
297
|
+
: []),
|
|
298
|
+
], hybridSearch.rrfK);
|
|
299
|
+
return { fusedResults, queryEmbedding, useIndexSearch: false };
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Shared post-processing pipeline that all retrieval paths converge on.
|
|
303
|
+
*
|
|
304
|
+
* source tracking → seed extraction → dedupe → recency boost + length penalty →
|
|
305
|
+
* size bounding → MMR reranking → score normalization → budget assembly
|
|
306
|
+
*/
|
|
307
|
+
async function postProcessResults(fusedResults, opts) {
|
|
308
|
+
const { queryEmbedding, maxTokens, currentSessionId, config, useIndexSearch } = opts;
|
|
290
309
|
const sourceMap = new Map();
|
|
291
310
|
for (const item of fusedResults) {
|
|
292
311
|
if (item.source && !sourceMap.has(item.chunkId)) {
|
|
@@ -336,7 +355,7 @@ export async function searchContext(request) {
|
|
|
336
355
|
return tokens !== undefined && tokens <= maxTokens;
|
|
337
356
|
});
|
|
338
357
|
// MMR reranking (diversity-aware, budget-aware ordering)
|
|
339
|
-
const reordered = await reorderWithMMR(sizeBounded, queryEmbedding, mmrReranking, {
|
|
358
|
+
const reordered = await reorderWithMMR(sizeBounded, queryEmbedding, config.mmrReranking, {
|
|
340
359
|
tokenBudget: maxTokens,
|
|
341
360
|
chunkTokenCounts: chunkTokenMap,
|
|
342
361
|
});
|
|
@@ -356,11 +375,89 @@ export async function searchContext(request) {
|
|
|
356
375
|
tokenCount: assembled.tokenCount,
|
|
357
376
|
chunks: assembled.includedChunks,
|
|
358
377
|
totalConsidered: deduped.length,
|
|
359
|
-
durationMs: Date.now() - startTime,
|
|
360
|
-
queryEmbedding,
|
|
361
378
|
seedIds,
|
|
362
379
|
};
|
|
363
380
|
}
|
|
381
|
+
// ── Main orchestrator ────────────────────────────────────────────────────────
|
|
382
|
+
/**
|
|
383
|
+
* Run the search pipeline.
|
|
384
|
+
*
|
|
385
|
+
* Keyword-primary mode: keyword → [optional vector enrichment] → recency → MMR → budget
|
|
386
|
+
* Hybrid mode: embed → [vector, keyword] → RRF → cluster expand → recency → MMR → budget
|
|
387
|
+
*/
|
|
388
|
+
export async function searchContext(request) {
|
|
389
|
+
const startTime = Date.now();
|
|
390
|
+
const externalConfig = loadConfig();
|
|
391
|
+
const config = toRuntimeConfig(externalConfig);
|
|
392
|
+
const { query, currentSessionId, projectFilter, maxTokens = config.mcpMaxResponseTokens, vectorSearchLimit = 20, skipClusters = false, agentFilter, } = request;
|
|
393
|
+
const { embeddingModel } = config;
|
|
394
|
+
const retrievalMode = config.retrievalPrimary;
|
|
395
|
+
const emptyResponse = {
|
|
396
|
+
text: '',
|
|
397
|
+
tokenCount: 0,
|
|
398
|
+
chunks: [],
|
|
399
|
+
totalConsidered: 0,
|
|
400
|
+
durationMs: Date.now() - startTime,
|
|
401
|
+
queryEmbedding: [],
|
|
402
|
+
seedIds: [],
|
|
403
|
+
};
|
|
404
|
+
let result;
|
|
405
|
+
if (retrievalMode === 'keyword') {
|
|
406
|
+
// ── Keyword-primary search path ──────────────────────────────────────
|
|
407
|
+
result = await keywordPrimarySearch(query, projectFilter, agentFilter, vectorSearchLimit, config);
|
|
408
|
+
if (!result) {
|
|
409
|
+
emptyResponse.durationMs = Date.now() - startTime;
|
|
410
|
+
return emptyResponse;
|
|
411
|
+
}
|
|
412
|
+
// Skip cluster expansion for keyword-primary mode
|
|
413
|
+
}
|
|
414
|
+
else {
|
|
415
|
+
// ── Hybrid/vector search path ────────────────────────────────────────
|
|
416
|
+
// Configure vector store for current model
|
|
417
|
+
vectorStore.setModelId(embeddingModel);
|
|
418
|
+
// 1. Embed query
|
|
419
|
+
const embedder = await getEmbedder(embeddingModel);
|
|
420
|
+
const queryResult = await embedder.embed(query, true);
|
|
421
|
+
const queryEmbedding = queryResult.embedding;
|
|
422
|
+
// Determine whether to use index-based search
|
|
423
|
+
const useIndexSearch = config.semanticIndex.useForSearch && getIndexEntryCount() > 0;
|
|
424
|
+
if (useIndexSearch) {
|
|
425
|
+
result = await indexBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config);
|
|
426
|
+
}
|
|
427
|
+
else {
|
|
428
|
+
result = await chunkBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config);
|
|
429
|
+
}
|
|
430
|
+
if (!result) {
|
|
431
|
+
emptyResponse.queryEmbedding = queryEmbedding;
|
|
432
|
+
emptyResponse.durationMs = Date.now() - startTime;
|
|
433
|
+
return emptyResponse;
|
|
434
|
+
}
|
|
435
|
+
// Entity boost (hybrid/vector path)
|
|
436
|
+
result.fusedResults = applyEntityBoost(result.fusedResults, query, projectFilter, config.hybridSearch.rrfK);
|
|
437
|
+
// Cluster expansion (hybrid/vector path only)
|
|
438
|
+
if (!skipClusters) {
|
|
439
|
+
result.fusedResults = expandViaClusters(result.fusedResults, config.clusterExpansion, projectFilter, agentFilter, config.feedbackWeight);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
// ── Shared post-processing ───────────────────────────────────────────
|
|
443
|
+
const processed = await postProcessResults(result.fusedResults, {
|
|
444
|
+
queryEmbedding: result.queryEmbedding,
|
|
445
|
+
maxTokens,
|
|
446
|
+
currentSessionId,
|
|
447
|
+
config,
|
|
448
|
+
useIndexSearch: result.useIndexSearch,
|
|
449
|
+
});
|
|
450
|
+
return {
|
|
451
|
+
text: processed.text,
|
|
452
|
+
tokenCount: processed.tokenCount,
|
|
453
|
+
chunks: processed.chunks,
|
|
454
|
+
totalConsidered: processed.totalConsidered,
|
|
455
|
+
durationMs: Date.now() - startTime,
|
|
456
|
+
queryEmbedding: result.queryEmbedding,
|
|
457
|
+
seedIds: processed.seedIds,
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
// ── Budget assembly ──────────────────────────────────────────────────────────
|
|
364
461
|
/**
|
|
365
462
|
* Formatting overhead constants.
|
|
366
463
|
*
|