causantic 0.10.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +2 -7
  2. package/config.schema.json +66 -0
  3. package/dist/cli/commands/init/ingest.js +2 -2
  4. package/dist/cli/commands/init/ingest.js.map +1 -1
  5. package/dist/cli/skill-templates.js +2 -2
  6. package/dist/clusters/cluster-manager.d.ts.map +1 -1
  7. package/dist/clusters/cluster-manager.js.map +1 -1
  8. package/dist/config/bootstrap.d.ts +20 -0
  9. package/dist/config/bootstrap.d.ts.map +1 -0
  10. package/dist/config/bootstrap.js +24 -0
  11. package/dist/config/bootstrap.js.map +1 -0
  12. package/dist/config/index.d.ts +1 -0
  13. package/dist/config/index.d.ts.map +1 -1
  14. package/dist/config/index.js +1 -0
  15. package/dist/config/index.js.map +1 -1
  16. package/dist/config/loader.d.ts +16 -1
  17. package/dist/config/loader.d.ts.map +1 -1
  18. package/dist/config/loader.js +194 -176
  19. package/dist/config/loader.js.map +1 -1
  20. package/dist/config/memory-config.d.ts +12 -0
  21. package/dist/config/memory-config.d.ts.map +1 -1
  22. package/dist/config/memory-config.js +64 -6
  23. package/dist/config/memory-config.js.map +1 -1
  24. package/dist/dashboard/server.d.ts.map +1 -1
  25. package/dist/dashboard/server.js +3 -1
  26. package/dist/dashboard/server.js.map +1 -1
  27. package/dist/eval/experiments/embedding-model-comparison/run-experiment.js.map +1 -1
  28. package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts.map +1 -1
  29. package/dist/eval/experiments/index-differentiation/alignment-analysis.js +3 -9
  30. package/dist/eval/experiments/index-differentiation/alignment-analysis.js.map +1 -1
  31. package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts.map +1 -1
  32. package/dist/eval/experiments/index-differentiation/discrimination-test.js.map +1 -1
  33. package/dist/eval/experiments/index-differentiation/refinement-test.d.ts.map +1 -1
  34. package/dist/eval/experiments/index-differentiation/refinement-test.js +1 -3
  35. package/dist/eval/experiments/index-differentiation/refinement-test.js.map +1 -1
  36. package/dist/eval/experiments/index-differentiation/run-experiment.js +5 -7
  37. package/dist/eval/experiments/index-differentiation/run-experiment.js.map +1 -1
  38. package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts.map +1 -1
  39. package/dist/eval/experiments/index-differentiation/similarity-analysis.js.map +1 -1
  40. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js +1 -3
  41. package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js.map +1 -1
  42. package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js.map +1 -1
  43. package/dist/eval/experiments/index-vs-chunk/query-generator.js.map +1 -1
  44. package/dist/eval/experiments/index-vs-chunk/run-experiment.js +6 -16
  45. package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map +1 -1
  46. package/dist/eval/experiments/pipeline-dropout/run-experiment.js +12 -4
  47. package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map +1 -1
  48. package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js.map +1 -1
  49. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js +26 -12
  50. package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js.map +1 -1
  51. package/dist/eval/experiments/rescorer-ceiling/run-experiment.js +1 -1
  52. package/dist/eval/experiments/rescorer-ceiling/run-experiment.js.map +1 -1
  53. package/dist/hooks/hook-utils.d.ts +1 -1
  54. package/dist/hooks/hook-utils.d.ts.map +1 -1
  55. package/dist/hooks/hook-utils.js +4 -2
  56. package/dist/hooks/hook-utils.js.map +1 -1
  57. package/dist/hooks/session-start.d.ts.map +1 -1
  58. package/dist/hooks/session-start.js +4 -1
  59. package/dist/hooks/session-start.js.map +1 -1
  60. package/dist/index-entries/index-generator.d.ts.map +1 -1
  61. package/dist/index-entries/index-generator.js +1 -3
  62. package/dist/index-entries/index-generator.js.map +1 -1
  63. package/dist/index-entries/index-refresher.d.ts.map +1 -1
  64. package/dist/index-entries/index-refresher.js.map +1 -1
  65. package/dist/index-entries/index.d.ts +1 -1
  66. package/dist/index-entries/index.d.ts.map +1 -1
  67. package/dist/index-entries/index.js +1 -1
  68. package/dist/index-entries/index.js.map +1 -1
  69. package/dist/ingest/brief-debrief-detector.d.ts.map +1 -1
  70. package/dist/ingest/brief-debrief-detector.js +6 -5
  71. package/dist/ingest/brief-debrief-detector.js.map +1 -1
  72. package/dist/ingest/ingest-session.d.ts.map +1 -1
  73. package/dist/ingest/ingest-session.js +109 -37
  74. package/dist/ingest/ingest-session.js.map +1 -1
  75. package/dist/ingest/session-state.d.ts.map +1 -1
  76. package/dist/ingest/session-state.js +6 -18
  77. package/dist/ingest/session-state.js.map +1 -1
  78. package/dist/mcp/server.d.ts +1 -1
  79. package/dist/mcp/server.d.ts.map +1 -1
  80. package/dist/mcp/server.js +15 -5
  81. package/dist/mcp/server.js.map +1 -1
  82. package/dist/mcp/services.d.ts.map +1 -1
  83. package/dist/mcp/services.js +9 -0
  84. package/dist/mcp/services.js.map +1 -1
  85. package/dist/mcp/tools.d.ts.map +1 -1
  86. package/dist/mcp/tools.js +36 -47
  87. package/dist/mcp/tools.js.map +1 -1
  88. package/dist/models/embedder.d.ts.map +1 -1
  89. package/dist/models/embedder.js +1 -0
  90. package/dist/models/embedder.js.map +1 -1
  91. package/dist/repomap/parser.d.ts.map +1 -1
  92. package/dist/repomap/parser.js +71 -22
  93. package/dist/repomap/parser.js.map +1 -1
  94. package/dist/repomap/regex-parser.d.ts.map +1 -1
  95. package/dist/repomap/regex-parser.js +30 -6
  96. package/dist/repomap/regex-parser.js.map +1 -1
  97. package/dist/repomap/renderer.d.ts.map +1 -1
  98. package/dist/repomap/renderer.js.map +1 -1
  99. package/dist/repomap/scanner.d.ts.map +1 -1
  100. package/dist/repomap/scanner.js +30 -11
  101. package/dist/repomap/scanner.js.map +1 -1
  102. package/dist/retrieval/chain-walker.d.ts.map +1 -1
  103. package/dist/retrieval/chain-walker.js +6 -2
  104. package/dist/retrieval/chain-walker.js.map +1 -1
  105. package/dist/retrieval/context-assembler.d.ts +1 -1
  106. package/dist/retrieval/context-assembler.d.ts.map +1 -1
  107. package/dist/retrieval/rrf.d.ts +1 -1
  108. package/dist/retrieval/rrf.d.ts.map +1 -1
  109. package/dist/retrieval/rrf.js +1 -1
  110. package/dist/retrieval/rrf.js.map +1 -1
  111. package/dist/retrieval/search-assembler.d.ts +1 -1
  112. package/dist/retrieval/search-assembler.d.ts.map +1 -1
  113. package/dist/retrieval/search-assembler.js +324 -227
  114. package/dist/retrieval/search-assembler.js.map +1 -1
  115. package/dist/retrieval/session-reconstructor.d.ts.map +1 -1
  116. package/dist/retrieval/session-reconstructor.js +7 -5
  117. package/dist/retrieval/session-reconstructor.js.map +1 -1
  118. package/dist/storage/chunk-store.d.ts.map +1 -1
  119. package/dist/storage/chunk-store.js +2 -0
  120. package/dist/storage/chunk-store.js.map +1 -1
  121. package/dist/storage/cluster-store.d.ts.map +1 -1
  122. package/dist/storage/cluster-store.js +3 -11
  123. package/dist/storage/cluster-store.js.map +1 -1
  124. package/dist/storage/db.d.ts +7 -0
  125. package/dist/storage/db.d.ts.map +1 -1
  126. package/dist/storage/db.js +25 -4
  127. package/dist/storage/db.js.map +1 -1
  128. package/dist/storage/entity-store.d.ts +48 -0
  129. package/dist/storage/entity-store.d.ts.map +1 -0
  130. package/dist/storage/entity-store.js +111 -0
  131. package/dist/storage/entity-store.js.map +1 -0
  132. package/dist/storage/index-entry-store.d.ts.map +1 -1
  133. package/dist/storage/index-entry-store.js +39 -40
  134. package/dist/storage/index-entry-store.js.map +1 -1
  135. package/dist/storage/keyword-store.d.ts +5 -0
  136. package/dist/storage/keyword-store.d.ts.map +1 -1
  137. package/dist/storage/keyword-store.js +1 -1
  138. package/dist/storage/keyword-store.js.map +1 -1
  139. package/dist/storage/migrations.d.ts.map +1 -1
  140. package/dist/storage/migrations.js +45 -0
  141. package/dist/storage/migrations.js.map +1 -1
  142. package/dist/storage/schema.sql +38 -2
  143. package/dist/storage/session-state-store.d.ts.map +1 -1
  144. package/dist/storage/session-state-store.js +46 -8
  145. package/dist/storage/session-state-store.js.map +1 -1
  146. package/dist/storage/types.d.ts +4 -2
  147. package/dist/storage/types.d.ts.map +1 -1
  148. package/dist/storage/vector-store-cleanup.d.ts +47 -0
  149. package/dist/storage/vector-store-cleanup.d.ts.map +1 -0
  150. package/dist/storage/vector-store-cleanup.js +101 -0
  151. package/dist/storage/vector-store-cleanup.js.map +1 -0
  152. package/dist/storage/vector-store.d.ts +13 -1
  153. package/dist/storage/vector-store.d.ts.map +1 -1
  154. package/dist/storage/vector-store.js +56 -111
  155. package/dist/storage/vector-store.js.map +1 -1
  156. package/dist/utils/entity-extractor.d.ts +23 -0
  157. package/dist/utils/entity-extractor.d.ts.map +1 -0
  158. package/dist/utils/entity-extractor.js +233 -0
  159. package/dist/utils/entity-extractor.js.map +1 -0
  160. package/package.json +3 -2
@@ -17,9 +17,13 @@ import { KeywordStore } from '../storage/keyword-store.js';
17
17
  import { fuseRRF } from './rrf.js';
18
18
  import { expandViaClusters } from './cluster-expander.js';
19
19
  import { reorderWithMMR } from './mmr.js';
20
+ import { extractEntities } from '../utils/entity-extractor.js';
21
+ import { findEntitiesByAlias, getChunkIdsForEntity } from '../storage/entity-store.js';
20
22
  import { createLogger } from '../utils/logger.js';
21
23
  import { formatSearchChunk } from './formatting.js';
22
24
  const log = createLogger('search-assembler');
25
+ /** RRF weight for entity-boosted results. */
26
+ const ENTITY_RRF_BOOST = 1.5;
23
27
  /**
24
28
  * Shared embedder instance.
25
29
  */
@@ -50,243 +54,258 @@ function getKeywordStore() {
50
54
  }
51
55
  return sharedKeywordStore;
52
56
  }
57
+ // ── Extracted pipeline stages ────────────────────────────────────────────────
53
58
  /**
54
- * Run the search pipeline.
59
+ * Filter items by agent when agent filtering is active but project filtering is not.
55
60
  *
56
- * Keyword-primary mode: keyword [optional vector enrichment] recency MMR → budget
57
- * Hybrid mode: embed [vector, keyword] RRF cluster expand recency → MMR → budget
61
+ * When projectFilter is set, agent filtering is handled by the storage layer.
62
+ * This function handles the post-filter case where no project scope was provided.
58
63
  */
59
- export async function searchContext(request) {
60
- const startTime = Date.now();
61
- const externalConfig = loadConfig();
62
- const config = toRuntimeConfig(externalConfig);
63
- const { query, currentSessionId, projectFilter, maxTokens = config.mcpMaxResponseTokens, vectorSearchLimit = 20, skipClusters = false, agentFilter, } = request;
64
- const { hybridSearch, clusterExpansion, mmrReranking, embeddingModel } = config;
65
- const retrievalMode = config.retrievalPrimary;
66
- let fusedResults;
67
- let queryEmbedding = [];
68
- let useIndexSearch = false;
69
- if (retrievalMode === 'keyword') {
70
- // ── Keyword-primary search path ──────────────────────────────────────
71
- // No embedding needed unless vector enrichment is enabled.
72
- let keywordResults = [];
73
- try {
74
- const keywordStore = getKeywordStore();
75
- keywordResults = projectFilter
76
- ? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit, agentFilter)
77
- : keywordStore.search(query, hybridSearch.keywordSearchLimit);
78
- }
79
- catch (error) {
80
- log.warn('Keyword search failed', { error: error.message });
81
- }
82
- // Post-filter by agent when no project filter was used
83
- if (agentFilter && !projectFilter) {
84
- keywordResults = keywordResults.filter((r) => {
85
- const chunk = getChunkById(r.id);
86
- return chunk?.agentId === agentFilter;
87
- });
88
- }
89
- fusedResults = keywordResults.map((r) => ({
90
- chunkId: r.id,
91
- score: r.score,
92
- source: 'keyword',
93
- }));
94
- // Optional vector enrichment: merge vector results via RRF
95
- if (config.vectorEnrichment) {
96
- try {
97
- vectorStore.setModelId(embeddingModel);
98
- const embedder = await getEmbedder(embeddingModel);
99
- const queryResult = await embedder.embed(query, true);
100
- queryEmbedding = queryResult.embedding;
101
- let vectorResults = await (projectFilter
102
- ? vectorStore.searchByProject(queryResult.embedding, projectFilter, vectorSearchLimit, agentFilter)
103
- : vectorStore.search(queryResult.embedding, vectorSearchLimit));
104
- if (agentFilter && !projectFilter) {
105
- vectorResults = vectorResults.filter((s) => {
106
- const chunk = getChunkById(s.id);
107
- return chunk?.agentId === agentFilter;
108
- });
109
- }
110
- if (vectorResults.length > 0) {
111
- const vectorItems = vectorResults.map((s) => ({
112
- chunkId: s.id,
113
- score: Math.max(0, 1 - s.distance),
114
- source: 'vector',
115
- }));
116
- fusedResults = fuseRRF([
117
- { items: fusedResults, weight: hybridSearch.keywordWeight },
118
- { items: vectorItems, weight: hybridSearch.vectorWeight },
119
- ], hybridSearch.rrfK);
120
- }
121
- }
122
- catch (error) {
123
- log.warn('Vector enrichment failed, using keyword results only', {
124
- error: error.message,
125
- });
64
+ function filterByAgent(items, agentFilter, projectFilter, getAgent) {
65
+ if (!agentFilter || projectFilter)
66
+ return items;
67
+ return items.filter((item) => getAgent(item.id) === agentFilter);
68
+ }
69
+ /**
70
+ * Extract entity mentions from the query and find matching chunks.
71
+ * Returns ranked items suitable for RRF fusion.
72
+ */
73
+ function getEntityResults(query, projectFilter) {
74
+ const mentions = extractEntities(query);
75
+ if (mentions.length === 0)
76
+ return [];
77
+ const project = typeof projectFilter === 'string' ? projectFilter : undefined;
78
+ if (!project)
79
+ return []; // entity lookup requires project scope
80
+ const chunkIds = new Set();
81
+ for (const mention of mentions) {
82
+ const entities = findEntitiesByAlias(mention.normalizedName, mention.entityType, project);
83
+ for (const entity of entities) {
84
+ for (const cid of getChunkIdsForEntity(entity.id, 100)) {
85
+ chunkIds.add(cid);
126
86
  }
127
87
  }
128
- if (fusedResults.length === 0) {
129
- return {
130
- text: '',
131
- tokenCount: 0,
132
- chunks: [],
133
- totalConsidered: 0,
134
- durationMs: Date.now() - startTime,
135
- queryEmbedding,
136
- seedIds: [],
137
- };
88
+ }
89
+ return [...chunkIds].map((id, i) => ({
90
+ chunkId: id,
91
+ score: 1.0 / (i + 1),
92
+ source: 'entity',
93
+ }));
94
+ }
95
+ /**
96
+ * Merge entity-boosted results into fused results via RRF.
97
+ */
98
+ function applyEntityBoost(fusedResults, query, projectFilter, rrfK) {
99
+ try {
100
+ const entityItems = getEntityResults(query, projectFilter);
101
+ if (entityItems.length > 0) {
102
+ return fuseRRF([
103
+ { items: fusedResults, weight: 1.0 },
104
+ { items: entityItems, weight: ENTITY_RRF_BOOST },
105
+ ], rrfK);
138
106
  }
139
- // Skip cluster expansion for keyword-primary mode
140
107
  }
141
- else {
142
- // ── Hybrid/vector search path ────────────────────────────────────────
143
- // Configure vector store for current model
144
- vectorStore.setModelId(embeddingModel);
145
- // 1. Embed query
146
- const embedder = await getEmbedder(embeddingModel);
147
- const queryResult = await embedder.embed(query, true);
148
- queryEmbedding = queryResult.embedding;
149
- // Determine whether to use index-based search
150
- useIndexSearch = config.semanticIndex.useForSearch && getIndexEntryCount() > 0;
151
- if (useIndexSearch) {
152
- // ── Index-based search path ──────────────────────────────────────
153
- indexVectorStore.setModelId(embeddingModel);
154
- const entryCount = getIndexEntryCount();
155
- const indexedChunks = getIndexedChunkCount();
156
- const entriesPerChunk = indexedChunks > 0 ? entryCount / indexedChunks : 1;
157
- const indexSearchLimit = Math.ceil(vectorSearchLimit * entriesPerChunk);
158
- const indexVectorPromise = projectFilter
159
- ? indexVectorStore.searchByProject(queryResult.embedding, projectFilter, indexSearchLimit, agentFilter)
160
- : indexVectorStore.search(queryResult.embedding, indexSearchLimit);
161
- let indexKeywordResults = [];
162
- try {
163
- indexKeywordResults = searchIndexEntriesByKeyword(query, hybridSearch.keywordSearchLimit, projectFilter, agentFilter);
164
- }
165
- catch (error) {
166
- log.warn('Index keyword search unavailable', {
167
- error: error.message,
168
- });
169
- }
170
- let indexSimilar = await indexVectorPromise;
171
- if (agentFilter && !projectFilter) {
172
- indexSimilar = indexSimilar.filter((s) => {
173
- const agent = indexVectorStore.getChunkAgent(s.id);
174
- return agent === agentFilter;
175
- });
176
- indexKeywordResults = indexKeywordResults.filter((r) => {
177
- const agent = indexVectorStore.getChunkAgent(r.id);
178
- return agent === agentFilter;
179
- });
180
- }
181
- if (indexSimilar.length === 0 && indexKeywordResults.length === 0) {
182
- return {
183
- text: '',
184
- tokenCount: 0,
185
- chunks: [],
186
- totalConsidered: 0,
187
- durationMs: Date.now() - startTime,
188
- queryEmbedding,
189
- seedIds: [],
190
- };
191
- }
192
- const indexVectorItems = indexSimilar.map((s) => ({
193
- chunkId: s.id,
194
- score: Math.max(0, 1 - s.distance),
195
- source: 'vector',
196
- }));
197
- const indexKeywordItems = indexKeywordResults.map((r) => ({
198
- chunkId: r.id,
199
- score: r.score,
200
- source: 'keyword',
201
- }));
202
- const indexFused = fuseRRF([
203
- { items: indexVectorItems, weight: hybridSearch.vectorWeight },
204
- ...(indexKeywordItems.length > 0
205
- ? [{ items: indexKeywordItems, weight: hybridSearch.keywordWeight }]
206
- : []),
207
- ], hybridSearch.rrfK);
208
- const indexEntryIds = indexFused.map((r) => r.chunkId);
209
- const chunkIds = dereferenceToChunkIds(indexEntryIds);
210
- const chunkScoreMap = new Map();
211
- for (const item of indexFused) {
212
- const entryChunkIds = dereferenceToChunkIds([item.chunkId]);
213
- for (const cid of entryChunkIds) {
214
- const existing = chunkScoreMap.get(cid);
215
- if (!existing || item.score > existing.score) {
216
- chunkScoreMap.set(cid, { score: item.score, source: item.source });
217
- }
218
- }
108
+ catch (error) {
109
+ log.warn('Entity search failed', { error: error.message });
110
+ }
111
+ return fusedResults;
112
+ }
113
+ /**
114
+ * Keyword-primary retrieval path.
115
+ *
116
+ * keyword [optional vector enrichment] → entity boost
117
+ * No cluster expansion.
118
+ */
119
+ async function keywordPrimarySearch(query, projectFilter, agentFilter, vectorSearchLimit, config) {
120
+ const { hybridSearch, embeddingModel } = config;
121
+ let queryEmbedding = [];
122
+ let keywordResults = [];
123
+ try {
124
+ const keywordStore = getKeywordStore();
125
+ keywordResults = projectFilter
126
+ ? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit, agentFilter)
127
+ : keywordStore.search(query, hybridSearch.keywordSearchLimit);
128
+ }
129
+ catch (error) {
130
+ log.warn('Keyword search failed', { error: error.message });
131
+ }
132
+ // Post-filter by agent when no project filter was used
133
+ keywordResults = filterByAgent(keywordResults, agentFilter, projectFilter, (id) => {
134
+ const chunk = getChunkById(id);
135
+ return chunk?.agentId;
136
+ });
137
+ let fusedResults = keywordResults.map((r) => ({
138
+ chunkId: r.id,
139
+ score: r.score,
140
+ source: 'keyword',
141
+ }));
142
+ // Optional vector enrichment: merge vector results via RRF
143
+ if (config.vectorEnrichment) {
144
+ try {
145
+ vectorStore.setModelId(embeddingModel);
146
+ const embedder = await getEmbedder(embeddingModel);
147
+ const queryResult = await embedder.embed(query, true);
148
+ queryEmbedding = queryResult.embedding;
149
+ let vectorResults = await (projectFilter
150
+ ? vectorStore.searchByProject(queryResult.embedding, projectFilter, vectorSearchLimit, agentFilter)
151
+ : vectorStore.search(queryResult.embedding, vectorSearchLimit));
152
+ vectorResults = filterByAgent(vectorResults, agentFilter, projectFilter, (id) => {
153
+ const chunk = getChunkById(id);
154
+ return chunk?.agentId;
155
+ });
156
+ if (vectorResults.length > 0) {
157
+ const vectorItems = vectorResults.map((s) => ({
158
+ chunkId: s.id,
159
+ score: Math.max(0, 1 - s.distance),
160
+ source: 'vector',
161
+ }));
162
+ fusedResults = fuseRRF([
163
+ { items: fusedResults, weight: hybridSearch.keywordWeight },
164
+ { items: vectorItems, weight: hybridSearch.vectorWeight },
165
+ ], hybridSearch.rrfK);
219
166
  }
220
- fusedResults = chunkIds.map((cid) => {
221
- const entry = chunkScoreMap.get(cid);
222
- return {
223
- chunkId: cid,
224
- score: entry?.score ?? 0,
225
- source: entry?.source,
226
- };
167
+ }
168
+ catch (error) {
169
+ log.warn('Vector enrichment failed, using keyword results only', {
170
+ error: error.message,
227
171
  });
228
172
  }
229
- else {
230
- // ── Chunk-based search path (fallback) ─────────────────────────────
231
- const vectorSearchPromise = projectFilter
232
- ? vectorStore.searchByProject(queryResult.embedding, projectFilter, vectorSearchLimit, agentFilter)
233
- : vectorStore.search(queryResult.embedding, vectorSearchLimit);
234
- let keywordResults = [];
235
- try {
236
- const keywordStore = getKeywordStore();
237
- keywordResults = projectFilter
238
- ? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit, agentFilter)
239
- : keywordStore.search(query, hybridSearch.keywordSearchLimit);
240
- }
241
- catch (error) {
242
- log.warn('Keyword search unavailable, falling back to vector-only', {
243
- error: error.message,
244
- });
245
- }
246
- let similar = await vectorSearchPromise;
247
- if (agentFilter && !projectFilter) {
248
- similar = similar.filter((s) => {
249
- const chunk = getChunkById(s.id);
250
- return chunk?.agentId === agentFilter;
251
- });
252
- keywordResults = keywordResults.filter((r) => {
253
- const chunk = getChunkById(r.id);
254
- return chunk?.agentId === agentFilter;
255
- });
256
- }
257
- if (similar.length === 0 && keywordResults.length === 0) {
258
- return {
259
- text: '',
260
- tokenCount: 0,
261
- chunks: [],
262
- totalConsidered: 0,
263
- durationMs: Date.now() - startTime,
264
- queryEmbedding,
265
- seedIds: [],
266
- };
173
+ }
174
+ // Entity boost
175
+ fusedResults = applyEntityBoost(fusedResults, query, projectFilter, hybridSearch.rrfK);
176
+ if (fusedResults.length === 0) {
177
+ return null;
178
+ }
179
+ return { fusedResults, queryEmbedding, useIndexSearch: false };
180
+ }
181
+ /**
182
+ * Index-based hybrid retrieval path.
183
+ *
184
+ * Uses semantic index entries (vector + keyword) → RRF → dereference to chunks.
185
+ */
186
+ async function indexBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config) {
187
+ const { hybridSearch, embeddingModel } = config;
188
+ indexVectorStore.setModelId(embeddingModel);
189
+ const entryCount = getIndexEntryCount();
190
+ const indexedChunks = getIndexedChunkCount();
191
+ const entriesPerChunk = indexedChunks > 0 ? entryCount / indexedChunks : 1;
192
+ const indexSearchLimit = Math.ceil(vectorSearchLimit * entriesPerChunk);
193
+ const indexVectorPromise = projectFilter
194
+ ? indexVectorStore.searchByProject(queryEmbedding, projectFilter, indexSearchLimit, agentFilter)
195
+ : indexVectorStore.search(queryEmbedding, indexSearchLimit);
196
+ let indexKeywordResults = [];
197
+ try {
198
+ indexKeywordResults = searchIndexEntriesByKeyword(query, hybridSearch.keywordSearchLimit, projectFilter, agentFilter);
199
+ }
200
+ catch (error) {
201
+ log.warn('Index keyword search unavailable', {
202
+ error: error.message,
203
+ });
204
+ }
205
+ let indexSimilar = await indexVectorPromise;
206
+ indexSimilar = filterByAgent(indexSimilar, agentFilter, projectFilter, (id) => indexVectorStore.getChunkAgent(id));
207
+ indexKeywordResults = filterByAgent(indexKeywordResults, agentFilter, projectFilter, (id) => indexVectorStore.getChunkAgent(id));
208
+ if (indexSimilar.length === 0 && indexKeywordResults.length === 0) {
209
+ return null;
210
+ }
211
+ const indexVectorItems = indexSimilar.map((s) => ({
212
+ chunkId: s.id,
213
+ score: Math.max(0, 1 - s.distance),
214
+ source: 'vector',
215
+ }));
216
+ const indexKeywordItems = indexKeywordResults.map((r) => ({
217
+ chunkId: r.id,
218
+ score: r.score,
219
+ source: 'keyword',
220
+ }));
221
+ const indexFused = fuseRRF([
222
+ { items: indexVectorItems, weight: hybridSearch.vectorWeight },
223
+ ...(indexKeywordItems.length > 0
224
+ ? [{ items: indexKeywordItems, weight: hybridSearch.keywordWeight }]
225
+ : []),
226
+ ], hybridSearch.rrfK);
227
+ const indexEntryIds = indexFused.map((r) => r.chunkId);
228
+ const chunkIds = dereferenceToChunkIds(indexEntryIds);
229
+ const chunkScoreMap = new Map();
230
+ for (const item of indexFused) {
231
+ const entryChunkIds = dereferenceToChunkIds([item.chunkId]);
232
+ for (const cid of entryChunkIds) {
233
+ const existing = chunkScoreMap.get(cid);
234
+ if (!existing || item.score > existing.score) {
235
+ chunkScoreMap.set(cid, { score: item.score, source: item.source });
267
236
  }
268
- const vectorItems = similar.map((s) => ({
269
- chunkId: s.id,
270
- score: Math.max(0, 1 - s.distance),
271
- source: 'vector',
272
- }));
273
- const keywordItems = keywordResults.map((r) => ({
274
- chunkId: r.id,
275
- score: r.score,
276
- source: 'keyword',
277
- }));
278
- fusedResults = fuseRRF([
279
- { items: vectorItems, weight: hybridSearch.vectorWeight },
280
- ...(keywordItems.length > 0
281
- ? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
282
- : []),
283
- ], hybridSearch.rrfK);
284
- }
285
- // Cluster expansion (hybrid/vector path only)
286
- if (!skipClusters) {
287
- fusedResults = expandViaClusters(fusedResults, clusterExpansion, projectFilter, agentFilter, config.feedbackWeight);
288
237
  }
289
238
  }
239
+ const fusedResults = chunkIds.map((cid) => {
240
+ const entry = chunkScoreMap.get(cid);
241
+ return {
242
+ chunkId: cid,
243
+ score: entry?.score ?? 0,
244
+ source: entry?.source,
245
+ };
246
+ });
247
+ return { fusedResults, queryEmbedding, useIndexSearch: true };
248
+ }
249
+ /**
250
+ * Chunk-based hybrid retrieval path (fallback when no semantic index).
251
+ *
252
+ * vector + keyword → RRF
253
+ */
254
+ async function chunkBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config) {
255
+ const { hybridSearch } = config;
256
+ const vectorSearchPromise = projectFilter
257
+ ? vectorStore.searchByProject(queryEmbedding, projectFilter, vectorSearchLimit, agentFilter)
258
+ : vectorStore.search(queryEmbedding, vectorSearchLimit);
259
+ let keywordResults = [];
260
+ try {
261
+ const keywordStore = getKeywordStore();
262
+ keywordResults = projectFilter
263
+ ? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit, agentFilter)
264
+ : keywordStore.search(query, hybridSearch.keywordSearchLimit);
265
+ }
266
+ catch (error) {
267
+ log.warn('Keyword search unavailable, falling back to vector-only', {
268
+ error: error.message,
269
+ });
270
+ }
271
+ let similar = await vectorSearchPromise;
272
+ similar = filterByAgent(similar, agentFilter, projectFilter, (id) => {
273
+ const chunk = getChunkById(id);
274
+ return chunk?.agentId;
275
+ });
276
+ keywordResults = filterByAgent(keywordResults, agentFilter, projectFilter, (id) => {
277
+ const chunk = getChunkById(id);
278
+ return chunk?.agentId;
279
+ });
280
+ if (similar.length === 0 && keywordResults.length === 0) {
281
+ return null;
282
+ }
283
+ const vectorItems = similar.map((s) => ({
284
+ chunkId: s.id,
285
+ score: Math.max(0, 1 - s.distance),
286
+ source: 'vector',
287
+ }));
288
+ const keywordItems = keywordResults.map((r) => ({
289
+ chunkId: r.id,
290
+ score: r.score,
291
+ source: 'keyword',
292
+ }));
293
+ const fusedResults = fuseRRF([
294
+ { items: vectorItems, weight: hybridSearch.vectorWeight },
295
+ ...(keywordItems.length > 0
296
+ ? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
297
+ : []),
298
+ ], hybridSearch.rrfK);
299
+ return { fusedResults, queryEmbedding, useIndexSearch: false };
300
+ }
301
+ /**
302
+ * Shared post-processing pipeline that all retrieval paths converge on.
303
+ *
304
+ * source tracking → seed extraction → dedupe → recency boost + length penalty →
305
+ * size bounding → MMR reranking → score normalization → budget assembly
306
+ */
307
+ async function postProcessResults(fusedResults, opts) {
308
+ const { queryEmbedding, maxTokens, currentSessionId, config, useIndexSearch } = opts;
290
309
  const sourceMap = new Map();
291
310
  for (const item of fusedResults) {
292
311
  if (item.source && !sourceMap.has(item.chunkId)) {
@@ -336,7 +355,7 @@ export async function searchContext(request) {
336
355
  return tokens !== undefined && tokens <= maxTokens;
337
356
  });
338
357
  // MMR reranking (diversity-aware, budget-aware ordering)
339
- const reordered = await reorderWithMMR(sizeBounded, queryEmbedding, mmrReranking, {
358
+ const reordered = await reorderWithMMR(sizeBounded, queryEmbedding, config.mmrReranking, {
340
359
  tokenBudget: maxTokens,
341
360
  chunkTokenCounts: chunkTokenMap,
342
361
  });
@@ -356,11 +375,89 @@ export async function searchContext(request) {
356
375
  tokenCount: assembled.tokenCount,
357
376
  chunks: assembled.includedChunks,
358
377
  totalConsidered: deduped.length,
359
- durationMs: Date.now() - startTime,
360
- queryEmbedding,
361
378
  seedIds,
362
379
  };
363
380
  }
381
+ // ── Main orchestrator ────────────────────────────────────────────────────────
382
+ /**
383
+ * Run the search pipeline.
384
+ *
385
+ * Keyword-primary mode: keyword → [optional vector enrichment] → recency → MMR → budget
386
+ * Hybrid mode: embed → [vector, keyword] → RRF → cluster expand → recency → MMR → budget
387
+ */
388
+ export async function searchContext(request) {
389
+ const startTime = Date.now();
390
+ const externalConfig = loadConfig();
391
+ const config = toRuntimeConfig(externalConfig);
392
+ const { query, currentSessionId, projectFilter, maxTokens = config.mcpMaxResponseTokens, vectorSearchLimit = 20, skipClusters = false, agentFilter, } = request;
393
+ const { embeddingModel } = config;
394
+ const retrievalMode = config.retrievalPrimary;
395
+ const emptyResponse = {
396
+ text: '',
397
+ tokenCount: 0,
398
+ chunks: [],
399
+ totalConsidered: 0,
400
+ durationMs: Date.now() - startTime,
401
+ queryEmbedding: [],
402
+ seedIds: [],
403
+ };
404
+ let result;
405
+ if (retrievalMode === 'keyword') {
406
+ // ── Keyword-primary search path ──────────────────────────────────────
407
+ result = await keywordPrimarySearch(query, projectFilter, agentFilter, vectorSearchLimit, config);
408
+ if (!result) {
409
+ emptyResponse.durationMs = Date.now() - startTime;
410
+ return emptyResponse;
411
+ }
412
+ // Skip cluster expansion for keyword-primary mode
413
+ }
414
+ else {
415
+ // ── Hybrid/vector search path ────────────────────────────────────────
416
+ // Configure vector store for current model
417
+ vectorStore.setModelId(embeddingModel);
418
+ // 1. Embed query
419
+ const embedder = await getEmbedder(embeddingModel);
420
+ const queryResult = await embedder.embed(query, true);
421
+ const queryEmbedding = queryResult.embedding;
422
+ // Determine whether to use index-based search
423
+ const useIndexSearch = config.semanticIndex.useForSearch && getIndexEntryCount() > 0;
424
+ if (useIndexSearch) {
425
+ result = await indexBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config);
426
+ }
427
+ else {
428
+ result = await chunkBasedSearch(queryEmbedding, query, projectFilter, agentFilter, vectorSearchLimit, config);
429
+ }
430
+ if (!result) {
431
+ emptyResponse.queryEmbedding = queryEmbedding;
432
+ emptyResponse.durationMs = Date.now() - startTime;
433
+ return emptyResponse;
434
+ }
435
+ // Entity boost (hybrid/vector path)
436
+ result.fusedResults = applyEntityBoost(result.fusedResults, query, projectFilter, config.hybridSearch.rrfK);
437
+ // Cluster expansion (hybrid/vector path only)
438
+ if (!skipClusters) {
439
+ result.fusedResults = expandViaClusters(result.fusedResults, config.clusterExpansion, projectFilter, agentFilter, config.feedbackWeight);
440
+ }
441
+ }
442
+ // ── Shared post-processing ───────────────────────────────────────────
443
+ const processed = await postProcessResults(result.fusedResults, {
444
+ queryEmbedding: result.queryEmbedding,
445
+ maxTokens,
446
+ currentSessionId,
447
+ config,
448
+ useIndexSearch: result.useIndexSearch,
449
+ });
450
+ return {
451
+ text: processed.text,
452
+ tokenCount: processed.tokenCount,
453
+ chunks: processed.chunks,
454
+ totalConsidered: processed.totalConsidered,
455
+ durationMs: Date.now() - startTime,
456
+ queryEmbedding: result.queryEmbedding,
457
+ seedIds: processed.seedIds,
458
+ };
459
+ }
460
+ // ── Budget assembly ──────────────────────────────────────────────────────────
364
461
  /**
365
462
  * Formatting overhead constants.
366
463
  *