opencode-rag-plugin 1.23.1 → 1.23.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.js +3 -0
- package/dist/chunker/factory.d.ts +4 -0
- package/dist/chunker/factory.js +36 -7
- package/dist/cli/commands/init-helpers.js +0 -3
- package/dist/core/chunk-role.d.ts +22 -0
- package/dist/core/chunk-role.js +47 -0
- package/dist/core/config.d.ts +19 -3
- package/dist/core/config.js +24 -4
- package/dist/core/interfaces.d.ts +10 -0
- package/dist/indexer/pipeline.js +6 -0
- package/dist/indexer/worker.d.ts +2 -0
- package/dist/indexer/worker.js +2 -0
- package/dist/plugin.js +9 -7
- package/dist/retriever/context-optimizer.js +13 -1
- package/dist/retriever/keyword-index.d.ts +15 -0
- package/dist/retriever/keyword-index.js +119 -7
- package/dist/retriever/retriever.d.ts +7 -0
- package/dist/retriever/retriever.js +56 -4
- package/dist/tui.js +3 -3
- package/dist/vectorstore/lancedb.d.ts +4 -2
- package/dist/vectorstore/lancedb.js +26 -7
- package/dist/vectorstore/memory.js +1 -0
- package/package.json +1 -1
package/dist/api.js
CHANGED
|
@@ -50,6 +50,9 @@ export async function search(query, options = {}) {
|
|
|
50
50
|
minScore: options.minScore ?? ctx.config.retrieval.minScore,
|
|
51
51
|
keywordIndex: ctx.keywordIndex,
|
|
52
52
|
keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
|
|
53
|
+
symbolKeywordWeight: ctx.config.retrieval.hybridSearch?.symbolKeywordWeight,
|
|
54
|
+
docKeywordDemotion: ctx.config.retrieval.hybridSearch?.docKeywordDemotion,
|
|
55
|
+
testKeywordDemotion: ctx.config.retrieval.hybridSearch?.testKeywordDemotion,
|
|
53
56
|
hybridEnabled: ctx.config.retrieval.hybridSearch?.enabled,
|
|
54
57
|
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
55
58
|
explain: options.explain,
|
|
@@ -64,5 +64,9 @@ export declare function getRegisteredExtensions(): string[];
|
|
|
64
64
|
*/
|
|
65
65
|
export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>, options?: {
|
|
66
66
|
maxSvgSizeBytes?: number;
|
|
67
|
+
/** Maximum lines per chunk; oversized chunks are split into windows. */
|
|
68
|
+
maxChunkSize?: number;
|
|
69
|
+
/** Overlapping lines shared between adjacent/split chunks. */
|
|
70
|
+
chunkOverlap?: number;
|
|
67
71
|
}): Promise<Chunk[]>;
|
|
68
72
|
export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
|
package/dist/chunker/factory.js
CHANGED
|
@@ -120,11 +120,34 @@ export function getRegisteredExtensions() {
|
|
|
120
120
|
}
|
|
121
121
|
const MAX_CHUNK_LINES = 100;
|
|
122
122
|
const MAX_CHUNK_CHARS = 8000;
|
|
123
|
-
|
|
123
|
+
/** Prepend the tail of the previous chunk as context to the next one, so a
|
|
124
|
+
* construct split across a chunk boundary is still retrievable whole. */
|
|
125
|
+
function applyChunkOverlap(chunks, overlap) {
|
|
126
|
+
if (overlap <= 0 || chunks.length <= 1)
|
|
127
|
+
return chunks;
|
|
124
128
|
const result = [];
|
|
129
|
+
let prevTail = null;
|
|
125
130
|
for (const chunk of chunks) {
|
|
126
131
|
const lines = chunk.content.split("\n");
|
|
127
|
-
if (
|
|
132
|
+
if (prevTail && prevTail.length > 0) {
|
|
133
|
+
result.push({
|
|
134
|
+
...chunk,
|
|
135
|
+
id: uuid(),
|
|
136
|
+
content: [...prevTail, ...lines].join("\n"),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
result.push(chunk);
|
|
141
|
+
}
|
|
142
|
+
prevTail = lines.slice(Math.max(0, lines.length - overlap));
|
|
143
|
+
}
|
|
144
|
+
return result;
|
|
145
|
+
}
|
|
146
|
+
function splitOversized(chunks, filePath, maxLines = MAX_CHUNK_LINES, maxChars = MAX_CHUNK_CHARS, overlap = 0) {
|
|
147
|
+
const result = [];
|
|
148
|
+
for (const chunk of chunks) {
|
|
149
|
+
const lines = chunk.content.split("\n");
|
|
150
|
+
if (lines.length <= maxLines && chunk.content.length <= maxChars) {
|
|
128
151
|
result.push(chunk);
|
|
129
152
|
continue;
|
|
130
153
|
}
|
|
@@ -136,7 +159,7 @@ function splitOversized(chunks, filePath) {
|
|
|
136
159
|
const line = lines[i];
|
|
137
160
|
const lineLen = line.length + 1;
|
|
138
161
|
if (currentLines.length > 0 &&
|
|
139
|
-
(currentLines.length >=
|
|
162
|
+
(currentLines.length >= maxLines || currentCharCount + lineLen > maxChars)) {
|
|
140
163
|
subChunks.push({
|
|
141
164
|
id: uuid(),
|
|
142
165
|
content: currentLines.join("\n"),
|
|
@@ -147,9 +170,12 @@ function splitOversized(chunks, filePath) {
|
|
|
147
170
|
language: chunk.metadata.language,
|
|
148
171
|
},
|
|
149
172
|
});
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
173
|
+
// Seed the next window with the last `overlap` lines so split chunks
|
|
174
|
+
// still share their boundary context.
|
|
175
|
+
const tail = overlap > 0 ? currentLines.slice(Math.max(0, currentLines.length - overlap)) : [];
|
|
176
|
+
currentLines = [...tail];
|
|
177
|
+
currentCharCount = tail.reduce((sum, l) => sum + l.length + 1, 0);
|
|
178
|
+
lineOffset = i - tail.length;
|
|
153
179
|
}
|
|
154
180
|
currentLines.push(line);
|
|
155
181
|
currentCharCount += lineLen;
|
|
@@ -226,7 +252,10 @@ export async function chunkFile(filePath, content, nodeTypesOverrides, options)
|
|
|
226
252
|
if (chunks.length === 0) {
|
|
227
253
|
return fallbackChunker.chunk(filePath, content);
|
|
228
254
|
}
|
|
229
|
-
|
|
255
|
+
const overlap = Math.max(0, options?.chunkOverlap ?? 0);
|
|
256
|
+
const maxLines = options?.maxChunkSize && options.maxChunkSize > 0 ? options.maxChunkSize : MAX_CHUNK_LINES;
|
|
257
|
+
const split = splitOversized(chunks, filePath, maxLines, MAX_CHUNK_CHARS, overlap);
|
|
258
|
+
return applyChunkOverlap(split, overlap);
|
|
230
259
|
}
|
|
231
260
|
export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
|
|
232
261
|
//# sourceMappingURL=factory.js.map
|
|
@@ -513,9 +513,6 @@ export function generateDefaultConfigJson(tuning) {
|
|
|
513
513
|
level: DEFAULT_CONFIG.logging.level,
|
|
514
514
|
logFilePath: DEFAULT_CONFIG.logging.logFilePath,
|
|
515
515
|
},
|
|
516
|
-
chunking: {
|
|
517
|
-
nodeTypes: {},
|
|
518
|
-
},
|
|
519
516
|
}, null, 2) + "\n";
|
|
520
517
|
}
|
|
521
518
|
//# sourceMappingURL=init-helpers.js.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Classifies a chunk's provenance (source / test / doc) from its file path.
|
|
3
|
+
*
|
|
4
|
+
* The retrieval fusion uses this to demote documentation and test chunks when they
|
|
5
|
+
* compete on keyword matches — natural-language queries about "how X works" match
|
|
6
|
+
* doc and test files literally (their prose names the concepts), which otherwise
|
|
7
|
+
* lets them outrank the implementation the query is really after.
|
|
8
|
+
*/
|
|
9
|
+
import type { Chunk } from "./interfaces.js";
|
|
10
|
+
export type ChunkRole = "source" | "test" | "doc";
|
|
11
|
+
/**
|
|
12
|
+
* Classify the role of a file from its path. Test takes precedence over doc
|
|
13
|
+
* (a `*.test.md` is a test artifact), and everything else is source.
|
|
14
|
+
*/
|
|
15
|
+
export declare function classifyFilePath(filePath: string): ChunkRole;
|
|
16
|
+
/**
|
|
17
|
+
* Tag a chunk with its derived role. Idempotent — a chunk that already carries a
|
|
18
|
+
* role is returned unchanged.
|
|
19
|
+
*/
|
|
20
|
+
export declare function tagChunkRole(chunk: Chunk): Chunk;
|
|
21
|
+
/** Tag all chunks produced for a file with their derived role. */
|
|
22
|
+
export declare function tagChunksRole(chunks: Chunk[]): Chunk[];
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Classifies a chunk's provenance (source / test / doc) from its file path.
|
|
3
|
+
*
|
|
4
|
+
* The retrieval fusion uses this to demote documentation and test chunks when they
|
|
5
|
+
* compete on keyword matches — natural-language queries about "how X works" match
|
|
6
|
+
* doc and test files literally (their prose names the concepts), which otherwise
|
|
7
|
+
* lets them outrank the implementation the query is really after.
|
|
8
|
+
*/
|
|
9
|
+
const TEST_PATH_SEGMENTS = ["__tests__", "test", "tests", "spec", "specs", "fixtures"];
|
|
10
|
+
const TEST_BASENAME = /\.(?:test|spec|bench)\.[a-z0-9]+$/i;
|
|
11
|
+
const DOC_PATH_SEGMENTS = ["doc", "docs", "documentation"];
|
|
12
|
+
const DOC_EXTENSIONS = [".md", ".mdx", ".rst", ".adoc"];
|
|
13
|
+
/**
|
|
14
|
+
* Classify the role of a file from its path. Test takes precedence over doc
|
|
15
|
+
* (a `*.test.md` is a test artifact), and everything else is source.
|
|
16
|
+
*/
|
|
17
|
+
export function classifyFilePath(filePath) {
|
|
18
|
+
const normalized = filePath.replace(/\\/g, "/");
|
|
19
|
+
const segments = normalized.split("/");
|
|
20
|
+
const basename = segments[segments.length - 1] ?? "";
|
|
21
|
+
if (TEST_BASENAME.test(basename))
|
|
22
|
+
return "test";
|
|
23
|
+
if (segments.some((s) => TEST_PATH_SEGMENTS.includes(s.toLowerCase())))
|
|
24
|
+
return "test";
|
|
25
|
+
if (DOC_EXTENSIONS.some((ext) => basename.toLowerCase().endsWith(ext)))
|
|
26
|
+
return "doc";
|
|
27
|
+
if (segments.some((s) => DOC_PATH_SEGMENTS.includes(s.toLowerCase())))
|
|
28
|
+
return "doc";
|
|
29
|
+
return "source";
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Tag a chunk with its derived role. Idempotent — a chunk that already carries a
|
|
33
|
+
* role is returned unchanged.
|
|
34
|
+
*/
|
|
35
|
+
export function tagChunkRole(chunk) {
|
|
36
|
+
if (chunk.metadata.role)
|
|
37
|
+
return chunk;
|
|
38
|
+
chunk.metadata.role = classifyFilePath(chunk.metadata.filePath);
|
|
39
|
+
return chunk;
|
|
40
|
+
}
|
|
41
|
+
/** Tag all chunks produced for a file with their derived role. */
|
|
42
|
+
export function tagChunksRole(chunks) {
|
|
43
|
+
for (const chunk of chunks)
|
|
44
|
+
tagChunkRole(chunk);
|
|
45
|
+
return chunks;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=chunk-role.js.map
|
package/dist/core/config.d.ts
CHANGED
|
@@ -181,16 +181,20 @@ export interface MemoryConfig {
|
|
|
181
181
|
minConfidence: number;
|
|
182
182
|
/** Minimum query-relevance score (0-1) for a quirk to be recalled. */
|
|
183
183
|
recallMinScore: number;
|
|
184
|
-
/** Minimum relevance score for
|
|
184
|
+
/** Minimum relevance score for quirks to be AUTO-injected into the prompt
|
|
185
|
+
* (both the system-prompt transform and the user message). Deliberately a
|
|
186
|
+
* high bar: injected quirks occupy context on every turn, so only strongly
|
|
187
|
+
* relevant ones should reach the prompt. Manual `recall_quirks` calls use
|
|
188
|
+
* the lower `recallMinScore`. */
|
|
185
189
|
autoInjectMinScore: number;
|
|
186
190
|
/** Maximum latency budget (ms) for auto-inject quirk recall. If exceeded, injection is skipped. */
|
|
187
191
|
autoInjectLatencyBudgetMs: number;
|
|
188
|
-
/** Max quirks to auto-inject per turn (default
|
|
192
|
+
/** Max quirks to auto-inject per turn (default 1). */
|
|
189
193
|
autoInjectTopK: number;
|
|
190
194
|
/**
|
|
191
195
|
* Minimum number of shared word tokens (≥3 chars) between a candidate quirk's
|
|
192
196
|
* content and the user's *current* message for the quirk to be auto-injected
|
|
193
|
-
* (default
|
|
197
|
+
* (default 2). Acts as a relevance gate against meta-quirks that match only
|
|
194
198
|
* the prior assistant text in the combined recall query. Set to `0` to disable.
|
|
195
199
|
*/
|
|
196
200
|
autoInjectMinTokenOverlap: number;
|
|
@@ -319,6 +323,9 @@ export interface RagConfig {
|
|
|
319
323
|
includeDirs?: string[];
|
|
320
324
|
/** Number of overlapping lines between adjacent chunks. */
|
|
321
325
|
chunkOverlap: number;
|
|
326
|
+
/** Maximum lines per chunk — oversized chunks are split into windows with
|
|
327
|
+
* `chunkOverlap` lines of context between them. 0 = no limit (whole AST node). */
|
|
328
|
+
maxChunkSize?: number;
|
|
322
329
|
/** Minimum file size in bytes to index (0 = no minimum). */
|
|
323
330
|
minFileSizeBytes?: number;
|
|
324
331
|
/** Maximum concurrent file processing tasks. */
|
|
@@ -378,6 +385,15 @@ export interface RagConfig {
|
|
|
378
385
|
enabled: boolean;
|
|
379
386
|
/** Weight for keyword scores in fusion (0 = vector only, 1 = keyword only). */
|
|
380
387
|
keywordWeight: number;
|
|
388
|
+
/** Keyword weight for symbol-style queries (a bare identifier like
|
|
389
|
+
* `cosineSimilarity`). Keyword search nails exact symbol matches, so these
|
|
390
|
+
* get a higher weight than natural-language queries. Defaults to keywordWeight. */
|
|
391
|
+
symbolKeywordWeight?: number;
|
|
392
|
+
/** Multiplier (0-1) on the keyword contribution of documentation chunks.
|
|
393
|
+
* Docs name concepts literally and otherwise outrank the implementation. */
|
|
394
|
+
docKeywordDemotion?: number;
|
|
395
|
+
/** Multiplier (0-1) on the keyword contribution of test chunks. */
|
|
396
|
+
testKeywordDemotion?: number;
|
|
381
397
|
};
|
|
382
398
|
/** Context window optimization settings for post-retrieval quality filtering. */
|
|
383
399
|
contextOptimization?: ContextOptimizationConfig;
|
package/dist/core/config.js
CHANGED
|
@@ -111,6 +111,7 @@ export const DEFAULT_CONFIG = {
|
|
|
111
111
|
"package-lock.json",
|
|
112
112
|
],
|
|
113
113
|
chunkOverlap: 0,
|
|
114
|
+
maxChunkSize: 0,
|
|
114
115
|
minFileSizeBytes: 0,
|
|
115
116
|
concurrency: 8,
|
|
116
117
|
embedBatchSize: 100,
|
|
@@ -131,6 +132,9 @@ export const DEFAULT_CONFIG = {
|
|
|
131
132
|
hybridSearch: {
|
|
132
133
|
enabled: true,
|
|
133
134
|
keywordWeight: 0.4,
|
|
135
|
+
symbolKeywordWeight: 0.6,
|
|
136
|
+
docKeywordDemotion: 0.5,
|
|
137
|
+
testKeywordDemotion: 0.6,
|
|
134
138
|
},
|
|
135
139
|
contextOptimization: {
|
|
136
140
|
enabled: true,
|
|
@@ -273,11 +277,11 @@ export const DEFAULT_CONFIG = {
|
|
|
273
277
|
enabled: true,
|
|
274
278
|
autoInject: true,
|
|
275
279
|
minConfidence: 0.5,
|
|
276
|
-
recallMinScore: 0.
|
|
277
|
-
autoInjectMinScore: 0.
|
|
280
|
+
recallMinScore: 0.72,
|
|
281
|
+
autoInjectMinScore: 0.75,
|
|
278
282
|
autoInjectLatencyBudgetMs: 2000,
|
|
279
|
-
autoInjectTopK:
|
|
280
|
-
autoInjectMinTokenOverlap:
|
|
283
|
+
autoInjectTopK: 1,
|
|
284
|
+
autoInjectMinTokenOverlap: 2,
|
|
281
285
|
passiveCapture: true,
|
|
282
286
|
promptEnforcement: true,
|
|
283
287
|
sessionEndExtraction: true,
|
|
@@ -365,6 +369,22 @@ export function validateConfig(config) {
|
|
|
365
369
|
if (kw < 0 || kw > 1) {
|
|
366
370
|
warnings.push("retrieval.hybridSearch.keywordWeight must be between 0 and 1");
|
|
367
371
|
}
|
|
372
|
+
const hs = config.retrieval.hybridSearch;
|
|
373
|
+
if (hs.symbolKeywordWeight != null && (hs.symbolKeywordWeight < 0 || hs.symbolKeywordWeight > 1)) {
|
|
374
|
+
warnings.push("retrieval.hybridSearch.symbolKeywordWeight must be between 0 and 1");
|
|
375
|
+
}
|
|
376
|
+
if (hs.docKeywordDemotion != null && (hs.docKeywordDemotion < 0 || hs.docKeywordDemotion > 1)) {
|
|
377
|
+
warnings.push("retrieval.hybridSearch.docKeywordDemotion must be between 0 and 1");
|
|
378
|
+
}
|
|
379
|
+
if (hs.testKeywordDemotion != null && (hs.testKeywordDemotion < 0 || hs.testKeywordDemotion > 1)) {
|
|
380
|
+
warnings.push("retrieval.hybridSearch.testKeywordDemotion must be between 0 and 1");
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
if (config.indexing.maxChunkSize != null && config.indexing.maxChunkSize < 0) {
|
|
384
|
+
warnings.push("indexing.maxChunkSize must be >= 0");
|
|
385
|
+
}
|
|
386
|
+
if (config.indexing.maxChunkSize && config.indexing.maxChunkSize <= config.indexing.chunkOverlap) {
|
|
387
|
+
warnings.push("indexing.maxChunkSize must be greater than chunkOverlap (otherwise windows make no progress)");
|
|
368
388
|
}
|
|
369
389
|
if (config.memory?.recallMinScore != null) {
|
|
370
390
|
const r = config.memory.recallMinScore;
|
|
@@ -16,6 +16,9 @@ export interface Chunk {
|
|
|
16
16
|
contentType?: string;
|
|
17
17
|
/** Synthetic chunk kind — e.g. "quirk" for experiential memory entries. */
|
|
18
18
|
kind?: string;
|
|
19
|
+
/** Provenance of the file, derived from its path: source / test / doc. Used by
|
|
20
|
+
* hybrid fusion to demote doc/test chunks that match query terms literally. */
|
|
21
|
+
role?: "source" | "test" | "doc";
|
|
19
22
|
/** Quirk sub-type when kind === "quirk": gotcha, preference, decision, environment-constraint. */
|
|
20
23
|
quirkType?: string;
|
|
21
24
|
/** Arbitrary tags for filtering/queries. */
|
|
@@ -79,6 +82,12 @@ export interface SearchExplanation {
|
|
|
79
82
|
/** Rank (0-indexed) in the keyword index results. */
|
|
80
83
|
keywordRank?: number;
|
|
81
84
|
};
|
|
85
|
+
/** Calibrated relevance in [0,1] derived from the raw vector-score spread of the
|
|
86
|
+
* candidate set — unlike the fused `score` (RRF-normalized so rank 0 is always
|
|
87
|
+
* ~1.0), this reflects how confident the match actually is. */
|
|
88
|
+
confidence?: number;
|
|
89
|
+
/** Shape of the query that produced this result ("symbol" vs "natural-language"). */
|
|
90
|
+
queryShape?: "symbol" | "natural-language";
|
|
82
91
|
/** Query terms that matched in the keyword index, if hybrid search was used. */
|
|
83
92
|
matchedTerms?: string[];
|
|
84
93
|
}
|
|
@@ -148,6 +157,7 @@ export interface ChunkSummary {
|
|
|
148
157
|
content: string;
|
|
149
158
|
description: string;
|
|
150
159
|
kind: string;
|
|
160
|
+
role: string;
|
|
151
161
|
quirkType: string;
|
|
152
162
|
tags: string;
|
|
153
163
|
}
|
package/dist/indexer/pipeline.js
CHANGED
|
@@ -12,6 +12,7 @@ import { createVectorStore } from "../vectorstore/factory.js";
|
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { createIndexStats } from "./stats.js";
|
|
14
14
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
15
|
+
import { tagChunksRole } from "../core/chunk-role.js";
|
|
15
16
|
import { buildFallbackDescription } from "./description-stage.js";
|
|
16
17
|
import { getCurrentCommit, getChangedFilesSince, getUntrackedFiles, getRepoRoot } from "./git-diff.js";
|
|
17
18
|
export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
|
|
@@ -410,6 +411,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
410
411
|
options.progress?.startFile(fileLabel);
|
|
411
412
|
}
|
|
412
413
|
const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions, descHash);
|
|
414
|
+
// Stamp every chunk with its file provenance (source/test/doc) so
|
|
415
|
+
// hybrid fusion can demote docs/tests that match query terms literally.
|
|
416
|
+
if (prep.chunks && prep.chunks.length > 0) {
|
|
417
|
+
tagChunksRole(prep.chunks);
|
|
418
|
+
}
|
|
413
419
|
if (isActive) {
|
|
414
420
|
chunkedDone++;
|
|
415
421
|
if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
|
package/dist/indexer/worker.d.ts
CHANGED
|
@@ -121,6 +121,8 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
|
|
|
121
121
|
indexing?: {
|
|
122
122
|
maxSvgSizeBytes?: number;
|
|
123
123
|
embedDescriptions?: boolean;
|
|
124
|
+
maxChunkSize?: number;
|
|
125
|
+
chunkOverlap?: number;
|
|
124
126
|
};
|
|
125
127
|
}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
|
|
126
128
|
/**
|
package/dist/indexer/worker.js
CHANGED
|
@@ -117,6 +117,8 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
|
|
|
117
117
|
else {
|
|
118
118
|
chunks = await chunkFile(file.filePath, file.content, config.chunking?.nodeTypes, {
|
|
119
119
|
maxSvgSizeBytes: config.indexing?.maxSvgSizeBytes,
|
|
120
|
+
maxChunkSize: config.indexing?.maxChunkSize,
|
|
121
|
+
chunkOverlap: config.indexing?.chunkOverlap,
|
|
120
122
|
}).catch((err) => {
|
|
121
123
|
logger.warn(` ${fileLabel} (chunking failed: ${err.message})`);
|
|
122
124
|
return null;
|
package/dist/plugin.js
CHANGED
|
@@ -839,7 +839,7 @@ export function createRagHooks(options) {
|
|
|
839
839
|
const budgetMs = memCfg.autoInjectLatencyBudgetMs ?? 2000;
|
|
840
840
|
let timer;
|
|
841
841
|
let quirkResults = await Promise.race([
|
|
842
|
-
recallQuirks(quirkDeps, query, { topK: memCfg.autoInjectTopK ??
|
|
842
|
+
recallQuirks(quirkDeps, query, { topK: memCfg.autoInjectTopK ?? 1, minScore: memCfg.autoInjectMinScore })
|
|
843
843
|
.then((r) => {
|
|
844
844
|
if (timer)
|
|
845
845
|
clearTimeout(timer);
|
|
@@ -855,7 +855,7 @@ export function createRagHooks(options) {
|
|
|
855
855
|
// Lexical gate: drop quirks that match only the prior assistant text (no token
|
|
856
856
|
// overlap with the user's actual request) — prevents meta-quirks (quirks about
|
|
857
857
|
// quirks) from being injected into unrelated tasks. Disabled when autoInjectMinTokenOverlap=0.
|
|
858
|
-
const minTokenOverlap = memCfg.autoInjectMinTokenOverlap ??
|
|
858
|
+
const minTokenOverlap = memCfg.autoInjectMinTokenOverlap ?? 2;
|
|
859
859
|
if (minTokenOverlap > 0 && userReq.length > 0) {
|
|
860
860
|
quirkResults = quirkResults.filter((qr) => sharedWords(qr.chunk.content, userReq) >= minTokenOverlap);
|
|
861
861
|
}
|
|
@@ -1140,12 +1140,14 @@ export function createRagHooks(options) {
|
|
|
1140
1140
|
if (quirkQuery.length > 0) {
|
|
1141
1141
|
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1142
1142
|
const budgetMs = quirkMemoryCfg.autoInjectLatencyBudgetMs ?? 2000;
|
|
1143
|
-
//
|
|
1144
|
-
//
|
|
1145
|
-
|
|
1143
|
+
// Both auto-inject paths (system prompt transform and this user-message
|
|
1144
|
+
// injection) use the same high autoInjectMinScore bar — injected quirks
|
|
1145
|
+
// occupy context every turn, so only strongly relevant ones qualify.
|
|
1146
|
+
// Manual recall_quirks calls use the lower recallMinScore.
|
|
1147
|
+
const minScore = quirkMemoryCfg.autoInjectMinScore ?? 0.75;
|
|
1146
1148
|
let timer;
|
|
1147
1149
|
let quirkResults = await Promise.race([
|
|
1148
|
-
recallQuirks(quirkDeps, quirkQuery, { topK: quirkMemoryCfg.autoInjectTopK ??
|
|
1150
|
+
recallQuirks(quirkDeps, quirkQuery, { topK: quirkMemoryCfg.autoInjectTopK ?? 1, minScore })
|
|
1149
1151
|
.then((r) => {
|
|
1150
1152
|
if (timer)
|
|
1151
1153
|
clearTimeout(timer);
|
|
@@ -1161,7 +1163,7 @@ export function createRagHooks(options) {
|
|
|
1161
1163
|
// Lexical gate: drop quirks that match only the prior assistant text (no token
|
|
1162
1164
|
// overlap with the user's current message) — prevents meta-quirks (quirks about
|
|
1163
1165
|
// quirks) from being injected into unrelated tasks. Disabled when autoInjectMinTokenOverlap=0.
|
|
1164
|
-
const minTokenOverlap = quirkMemoryCfg.autoInjectMinTokenOverlap ??
|
|
1166
|
+
const minTokenOverlap = quirkMemoryCfg.autoInjectMinTokenOverlap ?? 2;
|
|
1165
1167
|
if (minTokenOverlap > 0 && text.length > 0) {
|
|
1166
1168
|
quirkResults = quirkResults.filter((qr) => sharedWords(qr.chunk.content, text) >= minTokenOverlap);
|
|
1167
1169
|
}
|
|
@@ -63,6 +63,10 @@ function mergeAdjacentChunks(results, gapThreshold) {
|
|
|
63
63
|
const sorted = [...results].sort((a, b) => a.chunk.metadata.startLine - b.chunk.metadata.startLine);
|
|
64
64
|
const merged = [];
|
|
65
65
|
let pending = toOptimized(sorted[0]);
|
|
66
|
+
// The explanation (score breakdown, ranks) of the best-scoring constituent —
|
|
67
|
+
// carried onto the merged result so `--explain` stays accurate after merges.
|
|
68
|
+
let pendingExplanation = sorted[0].explanation;
|
|
69
|
+
let pendingBestScore = sorted[0].score;
|
|
66
70
|
for (let i = 1; i < sorted.length; i++) {
|
|
67
71
|
const current = sorted[i];
|
|
68
72
|
const gap = current.chunk.metadata.startLine - pending.chunk.metadata.endLine;
|
|
@@ -83,15 +87,23 @@ function mergeAdjacentChunks(results, gapThreshold) {
|
|
|
83
87
|
endLine: current.chunk.metadata.endLine,
|
|
84
88
|
},
|
|
85
89
|
};
|
|
90
|
+
const bestScore = Math.max(pending.score, current.score);
|
|
91
|
+
if (current.score > pendingBestScore) {
|
|
92
|
+
pendingBestScore = current.score;
|
|
93
|
+
pendingExplanation = current.explanation;
|
|
94
|
+
}
|
|
86
95
|
pending = {
|
|
87
96
|
chunk: mergedChunk,
|
|
88
|
-
score:
|
|
97
|
+
score: bestScore,
|
|
98
|
+
explanation: pendingExplanation,
|
|
89
99
|
optimized: { mergedFrom: sourceIds },
|
|
90
100
|
};
|
|
91
101
|
}
|
|
92
102
|
else {
|
|
93
103
|
merged.push(pending);
|
|
94
104
|
pending = toOptimized(current);
|
|
105
|
+
pendingExplanation = current.explanation;
|
|
106
|
+
pendingBestScore = current.score;
|
|
95
107
|
}
|
|
96
108
|
}
|
|
97
109
|
if (pending)
|
|
@@ -20,11 +20,26 @@ export declare class KeywordIndex {
|
|
|
20
20
|
private readonly storePath?;
|
|
21
21
|
/** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
|
|
22
22
|
private fileToIds;
|
|
23
|
+
/** Indexed-token length per chunk, and the running total — drives BM25 length
|
|
24
|
+
* normalization so long chunks can't dominate purely on term frequency. */
|
|
25
|
+
private docLengths;
|
|
26
|
+
private totalTokens;
|
|
27
|
+
/** The exact set of tokens indexed per chunk id, recorded at add time.
|
|
28
|
+
* Removal deletes precisely these entries instead of re-tokenizing the
|
|
29
|
+
* chunk: re-derivation would leave dangling inverted-index references
|
|
30
|
+
* whenever the chunk's text drifted between indexing and removal (the
|
|
31
|
+
* description stage mutates `chunk.description` after addChunks, or the
|
|
32
|
+
* persisted index was written by a build whose tokenizer differed). */
|
|
33
|
+
private chunkTokens;
|
|
23
34
|
constructor(storePath?: string);
|
|
24
35
|
addChunks(chunks: Chunk[]): void;
|
|
25
36
|
getMatchedTerms(query: string, chunkId: string): string[];
|
|
26
37
|
removeByFilePath(filePath: string): void;
|
|
27
38
|
search(query: string, topK: number, filter?: MetadataFilter): SearchResult[];
|
|
39
|
+
/** Remove every inverted-index reference to chunk ids that no longer have a
|
|
40
|
+
* chunkMap entry. Uses the recorded token set when available (fast path,
|
|
41
|
+
* always taken for loaded indexes) and falls back to a full scan. */
|
|
42
|
+
private pruneDangling;
|
|
28
43
|
close(): void;
|
|
29
44
|
clear(): void;
|
|
30
45
|
count(): number;
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { normalizeFilePath } from "../core/manifest.js";
|
|
2
2
|
import { normalizeFileExtensions, matchesFileExtension } from "../core/filters.js";
|
|
3
|
-
const INDEX_VERSION =
|
|
3
|
+
const INDEX_VERSION = 3;
|
|
4
|
+
/**
|
|
5
|
+
* Text indexed for keyword search: chunk content plus its LLM description when
|
|
6
|
+
* one exists. Without this, keyword and vector search would see different text
|
|
7
|
+
* spaces — a description can name a concept the raw code never spells out.
|
|
8
|
+
*/
|
|
9
|
+
function indexableText(chunk) {
|
|
10
|
+
return chunk.description ? `${chunk.content}\n${chunk.description}` : chunk.content;
|
|
11
|
+
}
|
|
4
12
|
/** Suffix-stripping stemmer. Only stems words >= 6 characters to reduce false positives. */
|
|
5
13
|
function stem(word) {
|
|
6
14
|
if (word.length < 6)
|
|
@@ -75,6 +83,17 @@ export class KeywordIndex {
|
|
|
75
83
|
storePath;
|
|
76
84
|
/** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
|
|
77
85
|
fileToIds = new Map();
|
|
86
|
+
/** Indexed-token length per chunk, and the running total — drives BM25 length
|
|
87
|
+
* normalization so long chunks can't dominate purely on term frequency. */
|
|
88
|
+
docLengths = new Map();
|
|
89
|
+
totalTokens = 0;
|
|
90
|
+
/** The exact set of tokens indexed per chunk id, recorded at add time.
|
|
91
|
+
* Removal deletes precisely these entries instead of re-tokenizing the
|
|
92
|
+
* chunk: re-derivation would leave dangling inverted-index references
|
|
93
|
+
* whenever the chunk's text drifted between indexing and removal (the
|
|
94
|
+
* description stage mutates `chunk.description` after addChunks, or the
|
|
95
|
+
* persisted index was written by a build whose tokenizer differed). */
|
|
96
|
+
chunkTokens = new Map();
|
|
78
97
|
constructor(storePath) {
|
|
79
98
|
this.storePath = storePath;
|
|
80
99
|
}
|
|
@@ -89,7 +108,10 @@ export class KeywordIndex {
|
|
|
89
108
|
this.fileToIds.set(fileKey, ids);
|
|
90
109
|
}
|
|
91
110
|
ids.add(id);
|
|
92
|
-
const tokens = tokenize(chunk
|
|
111
|
+
const tokens = tokenize(indexableText(chunk));
|
|
112
|
+
this.docLengths.set(id, tokens.length);
|
|
113
|
+
this.totalTokens += tokens.length;
|
|
114
|
+
this.chunkTokens.set(id, new Set(tokens));
|
|
93
115
|
for (const token of tokens) {
|
|
94
116
|
let docs = this.invertedIndex.get(token);
|
|
95
117
|
if (!docs) {
|
|
@@ -132,7 +154,10 @@ export class KeywordIndex {
|
|
|
132
154
|
for (const id of idsToRemove) {
|
|
133
155
|
const chunk = this.chunkMap.get(id);
|
|
134
156
|
if (chunk) {
|
|
135
|
-
|
|
157
|
+
// Delete exactly the entries addChunks created. The recorded token set
|
|
158
|
+
// is authoritative; re-derive only when the entry predates exact
|
|
159
|
+
// tracking (loaded indexes always rebuild chunkTokens in load()).
|
|
160
|
+
const tokens = this.chunkTokens.get(id) ?? new Set(tokenize(indexableText(chunk)));
|
|
136
161
|
for (const token of tokens) {
|
|
137
162
|
const docs = this.invertedIndex.get(token);
|
|
138
163
|
if (docs) {
|
|
@@ -143,6 +168,12 @@ export class KeywordIndex {
|
|
|
143
168
|
}
|
|
144
169
|
}
|
|
145
170
|
this.chunkMap.delete(id);
|
|
171
|
+
this.chunkTokens.delete(id);
|
|
172
|
+
const len = this.docLengths.get(id);
|
|
173
|
+
if (len !== undefined) {
|
|
174
|
+
this.totalTokens -= len;
|
|
175
|
+
this.docLengths.delete(id);
|
|
176
|
+
}
|
|
146
177
|
}
|
|
147
178
|
}
|
|
148
179
|
}
|
|
@@ -153,36 +184,91 @@ export class KeywordIndex {
|
|
|
153
184
|
if (queryTokens.length === 0)
|
|
154
185
|
return [];
|
|
155
186
|
const totalChunks = this.chunkMap.size;
|
|
187
|
+
const avgDocLength = this.totalTokens > 0 ? this.totalTokens / totalChunks : 0;
|
|
156
188
|
const scores = new Map();
|
|
189
|
+
// BM25: term-frequency saturation plus length normalization, so a chunk that
|
|
190
|
+
// merely repeats a term (or is very long) can't outrank a focused match.
|
|
191
|
+
const K1 = 1.2;
|
|
192
|
+
const B = 0.75;
|
|
157
193
|
for (const token of queryTokens) {
|
|
158
194
|
const docs = this.invertedIndex.get(token);
|
|
159
195
|
if (!docs)
|
|
160
196
|
continue;
|
|
161
197
|
const df = docs.size;
|
|
162
|
-
const idf = Math.log(1 + totalChunks / (df
|
|
198
|
+
const idf = Math.log(1 + (totalChunks - df + 0.5) / (df + 0.5));
|
|
163
199
|
for (const [chunkId, freq] of docs) {
|
|
164
|
-
const
|
|
165
|
-
|
|
200
|
+
const docLength = this.docLengths.get(chunkId) ?? avgDocLength;
|
|
201
|
+
const lengthNorm = avgDocLength > 0
|
|
202
|
+
? 1 - B + B * (docLength / avgDocLength)
|
|
203
|
+
: 1;
|
|
204
|
+
const tf = (freq * (K1 + 1)) / (freq + K1 * lengthNorm);
|
|
205
|
+
scores.set(chunkId, (scores.get(chunkId) ?? 0) + idf * tf);
|
|
166
206
|
}
|
|
167
207
|
}
|
|
208
|
+
const dangling = [];
|
|
168
209
|
const sorted = [...scores.entries()]
|
|
169
|
-
.filter(([id]) =>
|
|
210
|
+
.filter(([id]) => {
|
|
211
|
+
const chunk = this.chunkMap.get(id);
|
|
212
|
+
if (!chunk) {
|
|
213
|
+
// The inverted index references a chunk id that no longer exists.
|
|
214
|
+
// This is the "undefined is not an object (evaluating
|
|
215
|
+
// 'chunk.metadata')" crash site: a removal that missed tokens
|
|
216
|
+
// (index persisted by an older build, interrupted re-index) leaves
|
|
217
|
+
// phantom ids behind. Skip and prune them so a corrupt index
|
|
218
|
+
// self-heals instead of failing every filtered search.
|
|
219
|
+
dangling.push(id);
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
return matchesFilter(chunk, filter);
|
|
223
|
+
})
|
|
170
224
|
.sort((a, b) => b[1] - a[1])
|
|
171
225
|
.slice(0, topK);
|
|
226
|
+
if (dangling.length > 0)
|
|
227
|
+
this.pruneDangling(dangling);
|
|
172
228
|
return sorted.map(([chunkId, score]) => {
|
|
173
229
|
const chunk = this.chunkMap.get(chunkId);
|
|
174
230
|
return { chunk, score };
|
|
175
231
|
});
|
|
176
232
|
}
|
|
233
|
+
/** Remove every inverted-index reference to chunk ids that no longer have a
|
|
234
|
+
* chunkMap entry. Uses the recorded token set when available (fast path,
|
|
235
|
+
* always taken for loaded indexes) and falls back to a full scan. */
|
|
236
|
+
pruneDangling(ids) {
|
|
237
|
+
for (const id of ids) {
|
|
238
|
+
const tokens = this.chunkTokens.get(id);
|
|
239
|
+
if (tokens) {
|
|
240
|
+
for (const token of tokens) {
|
|
241
|
+
const docs = this.invertedIndex.get(token);
|
|
242
|
+
if (docs) {
|
|
243
|
+
docs.delete(id);
|
|
244
|
+
if (docs.size === 0)
|
|
245
|
+
this.invertedIndex.delete(token);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
else {
|
|
250
|
+
for (const [token, docs] of this.invertedIndex) {
|
|
251
|
+
if (docs.delete(id) && docs.size === 0)
|
|
252
|
+
this.invertedIndex.delete(token);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
this.chunkTokens.delete(id);
|
|
256
|
+
this.docLengths.delete(id);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
177
259
|
close() {
|
|
178
260
|
this.invertedIndex.clear();
|
|
179
261
|
this.chunkMap.clear();
|
|
180
262
|
this.fileToIds.clear();
|
|
263
|
+
this.chunkTokens.clear();
|
|
181
264
|
}
|
|
182
265
|
clear() {
|
|
183
266
|
this.invertedIndex.clear();
|
|
184
267
|
this.chunkMap.clear();
|
|
185
268
|
this.fileToIds.clear();
|
|
269
|
+
this.chunkTokens.clear();
|
|
270
|
+
this.docLengths.clear();
|
|
271
|
+
this.totalTokens = 0;
|
|
186
272
|
}
|
|
187
273
|
count() {
|
|
188
274
|
return this.chunkMap.size;
|
|
@@ -209,6 +295,7 @@ export class KeywordIndex {
|
|
|
209
295
|
endLine: chunk.metadata.endLine,
|
|
210
296
|
language: chunk.metadata.language,
|
|
211
297
|
kind: chunk.metadata.kind ?? "",
|
|
298
|
+
role: chunk.metadata.role ?? "",
|
|
212
299
|
quirkType: chunk.metadata.quirkType ?? "",
|
|
213
300
|
tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
|
|
214
301
|
},
|
|
@@ -241,6 +328,19 @@ export class KeywordIndex {
|
|
|
241
328
|
const docMap = new Map(docs);
|
|
242
329
|
index.invertedIndex.set(token, docMap);
|
|
243
330
|
}
|
|
331
|
+
// Rebuild the exact per-chunk token sets from the persisted inverted
|
|
332
|
+
// index, so removal is exact even for indexes written by older builds
|
|
333
|
+
// (whose tokenizer or indexableText may have differed).
|
|
334
|
+
for (const [token, docs] of index.invertedIndex) {
|
|
335
|
+
for (const id of docs.keys()) {
|
|
336
|
+
let tokens = index.chunkTokens.get(id);
|
|
337
|
+
if (!tokens) {
|
|
338
|
+
tokens = new Set();
|
|
339
|
+
index.chunkTokens.set(id, tokens);
|
|
340
|
+
}
|
|
341
|
+
tokens.add(token);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
244
344
|
for (const [id, data] of Object.entries(parsed.chunkMap)) {
|
|
245
345
|
let tags;
|
|
246
346
|
try {
|
|
@@ -260,11 +360,23 @@ export class KeywordIndex {
|
|
|
260
360
|
endLine: data.endLine,
|
|
261
361
|
language: data.language,
|
|
262
362
|
kind: data.kind || undefined,
|
|
363
|
+
role: data.role === "source" || data.role === "test" || data.role === "doc" ? data.role : undefined,
|
|
263
364
|
quirkType: data.quirkType || undefined,
|
|
264
365
|
tags,
|
|
265
366
|
},
|
|
266
367
|
});
|
|
267
368
|
}
|
|
369
|
+
// Recompute BM25 length statistics from the loaded chunk map. The persisted
|
|
370
|
+
// inverted index is authoritative for term frequencies; lengths are cheap
|
|
371
|
+
// to derive and this keeps the serialized format lean.
|
|
372
|
+
for (const [id, chunk] of index.chunkMap) {
|
|
373
|
+
// Prefer the exact token count from the inverted index over
|
|
374
|
+
// re-tokenizing — re-tokenization can drift from what was indexed.
|
|
375
|
+
const recorded = index.chunkTokens.get(id);
|
|
376
|
+
const len = recorded ? recorded.size : tokenize(indexableText(chunk)).length;
|
|
377
|
+
index.docLengths.set(id, len);
|
|
378
|
+
index.totalTokens += len;
|
|
379
|
+
}
|
|
268
380
|
// Rebuild the per-file id index from the loaded chunk map so removals
|
|
269
381
|
// work even for entries persisted by older versions.
|
|
270
382
|
for (const [id, chunk] of index.chunkMap) {
|
|
@@ -8,6 +8,13 @@ export interface RetrieveOptions {
|
|
|
8
8
|
minScore?: number;
|
|
9
9
|
keywordIndex?: KeywordIndex;
|
|
10
10
|
keywordWeight?: number;
|
|
11
|
+
/** Keyword weight for symbol-style queries (a bare identifier like `cosineSimilarity`).
|
|
12
|
+
* Keyword search nails exact symbol matches, so these queries get a higher weight. */
|
|
13
|
+
symbolKeywordWeight?: number;
|
|
14
|
+
/** Multiplier applied to the keyword contribution of documentation chunks (0-1). */
|
|
15
|
+
docKeywordDemotion?: number;
|
|
16
|
+
/** Multiplier applied to the keyword contribution of test chunks (0-1). */
|
|
17
|
+
testKeywordDemotion?: number;
|
|
11
18
|
/** Whether hybrid search is enabled. When false, keyword index is ignored. */
|
|
12
19
|
hybridEnabled?: boolean;
|
|
13
20
|
queryPrefix?: string;
|
|
@@ -5,6 +5,41 @@ const FETCH_OVERFETCH_FACTOR = 3;
|
|
|
5
5
|
const RRF_K = 60;
|
|
6
6
|
/** Multiply raw RRF scores by (K+1) to normalize to ~[0,1]. */
|
|
7
7
|
const RRF_NORMALIZE = RRF_K + 1;
|
|
8
|
+
/** Classify a query as a bare symbol lookup vs. natural-language prose.
|
|
9
|
+
* Symbol queries have no whitespace and look like one (possibly dotted) identifier. */
|
|
10
|
+
function detectQueryShape(query) {
|
|
11
|
+
const trimmed = query.trim();
|
|
12
|
+
if (trimmed.length === 0 || trimmed.length > 60)
|
|
13
|
+
return "natural-language";
|
|
14
|
+
if (/\s/.test(trimmed))
|
|
15
|
+
return "natural-language";
|
|
16
|
+
return /^[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)*$/.test(trimmed)
|
|
17
|
+
? "symbol"
|
|
18
|
+
: "natural-language";
|
|
19
|
+
}
|
|
20
|
+
/** Demotion applied to a chunk's keyword contribution based on its file's provenance.
|
|
21
|
+
* Docs and tests name concepts literally, so their keyword matches are discounted. */
|
|
22
|
+
function keywordDemotion(chunk, docDemotion, testDemotion) {
|
|
23
|
+
switch (chunk.metadata.role) {
|
|
24
|
+
case "doc": return docDemotion;
|
|
25
|
+
case "test": return testDemotion;
|
|
26
|
+
default: return 1;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** Calibrate a per-result confidence from the raw vector-score spread of the
|
|
30
|
+
* candidate set. Unlike the fused RRF score (rank 0 is always ~1.0), this
|
|
31
|
+
* reflects how separable the top match actually was. */
|
|
32
|
+
function attachConfidence(results) {
|
|
33
|
+
const raws = results.map((r) => r.explanation?.scoreBreakdown?.rawVectorScore ?? 0);
|
|
34
|
+
const max = Math.max(...raws);
|
|
35
|
+
const min = Math.min(...raws);
|
|
36
|
+
for (const r of results) {
|
|
37
|
+
if (!r.explanation)
|
|
38
|
+
continue;
|
|
39
|
+
const raw = r.explanation.scoreBreakdown?.rawVectorScore ?? 0;
|
|
40
|
+
r.explanation.confidence = max > min ? (raw - min) / (max - min) : 1;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
8
43
|
/**
|
|
9
44
|
* Perform hybrid vector-keyword retrieval with configurable scoring.
|
|
10
45
|
*
|
|
@@ -22,6 +57,15 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
22
57
|
try {
|
|
23
58
|
const topK = options.topK ?? 10;
|
|
24
59
|
const minScore = options.minScore ?? 0;
|
|
60
|
+
const queryShape = detectQueryShape(query);
|
|
61
|
+
const baseKw = options.keywordWeight ?? 0.4;
|
|
62
|
+
// Symbol queries are exact-identifier lookups — keyword search is the strong
|
|
63
|
+
// signal there, so use the (typically higher) symbol weight.
|
|
64
|
+
const kw = queryShape === "symbol"
|
|
65
|
+
? options.symbolKeywordWeight ?? baseKw
|
|
66
|
+
: baseKw;
|
|
67
|
+
const docDemotion = options.docKeywordDemotion ?? 0.5;
|
|
68
|
+
const testDemotion = options.testKeywordDemotion ?? 0.6;
|
|
25
69
|
const prefixedQuery = (options.queryPrefix ?? "") + query;
|
|
26
70
|
const embeddings = await embedder.embed([prefixedQuery], "query");
|
|
27
71
|
const embedding = embeddings[0];
|
|
@@ -39,7 +83,6 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
39
83
|
if (keywordResults.length === 0) {
|
|
40
84
|
const filtered = vectorResults.filter((r) => r.score >= minScore).slice(0, topK);
|
|
41
85
|
if (options.explain) {
|
|
42
|
-
const kw = options.keywordWeight ?? 0.4;
|
|
43
86
|
for (const r of filtered) {
|
|
44
87
|
r.explanation = {
|
|
45
88
|
scoreBreakdown: {
|
|
@@ -49,8 +92,10 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
49
92
|
rawKeywordScore: 0,
|
|
50
93
|
keywordWeight: kw,
|
|
51
94
|
},
|
|
95
|
+
queryShape,
|
|
52
96
|
};
|
|
53
97
|
}
|
|
98
|
+
attachConfidence(filtered);
|
|
54
99
|
}
|
|
55
100
|
return filtered;
|
|
56
101
|
}
|
|
@@ -62,15 +107,18 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
62
107
|
for (const r of keywordResults)
|
|
63
108
|
if (!chunkById.has(r.chunk.id))
|
|
64
109
|
chunkById.set(r.chunk.id, r);
|
|
65
|
-
const kw = options.keywordWeight ?? 0.4;
|
|
66
110
|
const allIds = new Set([...vRank.keys(), ...kRank.keys()]);
|
|
67
111
|
const combinedResults = [...allIds].map((id) => {
|
|
68
112
|
const vR = vRank.get(id);
|
|
69
113
|
const kR = kRank.get(id);
|
|
114
|
+
const chunk = chunkById.get(id)?.chunk;
|
|
115
|
+
if (!chunk)
|
|
116
|
+
return null;
|
|
117
|
+
const kMultiplier = kR !== undefined ? keywordDemotion(chunk, docDemotion, testDemotion) : 1;
|
|
70
118
|
const vContrib = vR !== undefined ? ((1 - kw) * RRF_NORMALIZE) / (RRF_K + vR + 1) : 0;
|
|
71
|
-
const kContrib = kR !== undefined ? (kw * RRF_NORMALIZE) / (RRF_K + kR + 1) : 0;
|
|
119
|
+
const kContrib = kR !== undefined ? (kw * RRF_NORMALIZE) / (RRF_K + kR + 1) * kMultiplier : 0;
|
|
72
120
|
const score = vContrib + kContrib;
|
|
73
|
-
const result = { chunk
|
|
121
|
+
const result = { chunk, score };
|
|
74
122
|
if (options.explain) {
|
|
75
123
|
result.explanation = {
|
|
76
124
|
scoreBreakdown: {
|
|
@@ -82,6 +130,7 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
82
130
|
vectorRank: vR,
|
|
83
131
|
keywordRank: kR,
|
|
84
132
|
},
|
|
133
|
+
queryShape,
|
|
85
134
|
};
|
|
86
135
|
if (options.keywordIndex && kR !== undefined) {
|
|
87
136
|
const terms = options.keywordIndex.getMatchedTerms(query, id);
|
|
@@ -91,9 +140,12 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
91
140
|
}
|
|
92
141
|
return result;
|
|
93
142
|
})
|
|
143
|
+
.filter((r) => r !== null)
|
|
94
144
|
.filter((r) => r.score >= minScore)
|
|
95
145
|
.sort((a, b) => b.score - a.score)
|
|
96
146
|
.slice(0, topK);
|
|
147
|
+
if (options.explain)
|
|
148
|
+
attachConfidence(combinedResults);
|
|
97
149
|
return combinedResults;
|
|
98
150
|
}
|
|
99
151
|
catch (err) {
|
package/dist/tui.js
CHANGED
|
@@ -543,15 +543,15 @@ function buildSettingCategories(cfg, ro, providers) {
|
|
|
543
543
|
},
|
|
544
544
|
{
|
|
545
545
|
path: ["memory", "recallMinScore"],
|
|
546
|
-
label: "
|
|
546
|
+
label: "Manual recall min score",
|
|
547
547
|
type: "number",
|
|
548
548
|
currentValue: memoryRo.recallMinScore ?? memoryCfg.recallMinScore ?? 0.72,
|
|
549
549
|
},
|
|
550
550
|
{
|
|
551
551
|
path: ["memory", "autoInjectMinScore"],
|
|
552
|
-
label: "
|
|
552
|
+
label: "Auto-inject min score",
|
|
553
553
|
type: "number",
|
|
554
|
-
currentValue: memoryRo.autoInjectMinScore ?? memoryCfg.autoInjectMinScore ?? 0.
|
|
554
|
+
currentValue: memoryRo.autoInjectMinScore ?? memoryCfg.autoInjectMinScore ?? 0.75,
|
|
555
555
|
},
|
|
556
556
|
{
|
|
557
557
|
path: ["memory", "autoInjectLatencyBudgetMs"],
|
|
@@ -180,7 +180,7 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
180
180
|
*/
|
|
181
181
|
private assertEmbeddingDimensions;
|
|
182
182
|
private hasColumn;
|
|
183
|
-
/** Add kind/quirkType/tags columns if missing from an existing table. */
|
|
183
|
+
/** Add kind/role/quirkType/tags columns if missing from an existing table. */
|
|
184
184
|
private migrateNewColumns;
|
|
185
185
|
/**
|
|
186
186
|
* Ensure that kind/quirkType/tags columns are nullable so that queries
|
|
@@ -315,7 +315,9 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
315
315
|
private ensureCosineIndex;
|
|
316
316
|
/**
|
|
317
317
|
* Perform a single index-metric repair pass. Skips stores that have no
|
|
318
|
-
* index and fewer than
|
|
318
|
+
* index and fewer than `MIN_ROWS_FOR_IVF_INDEX` rows (brute-force is
|
|
319
|
+
* optimal there, and an IVF index would only trip Lance's "dataset is too
|
|
320
|
+
* small" KMeans warning). Uses a
|
|
319
321
|
* single `createIndex` with `replace: true` — a dropIndex + createIndex
|
|
320
322
|
* sequence races in LanceDB ("Retryable commit conflict") and leaves the
|
|
321
323
|
* stale index in place. On failure the memo is cleared so the next
|
|
@@ -8,7 +8,7 @@ import path from "node:path";
|
|
|
8
8
|
import { normalizeFilePath, manifestPathFor } from "../core/manifest.js";
|
|
9
9
|
import { normalizeFileExtensions, matchesFileExtension } from "../core/filters.js";
|
|
10
10
|
const TABLE_NAME = "chunks";
|
|
11
|
-
const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language", "kind", "quirkType", "tags"];
|
|
11
|
+
const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language", "kind", "role", "quirkType", "tags"];
|
|
12
12
|
/**
|
|
13
13
|
* Upper bound for the number of failed index-creation attempts per process.
|
|
14
14
|
* Beyond this the repair gives up for the process lifetime instead of
|
|
@@ -24,6 +24,15 @@ const MAX_INDEX_REPAIR_ATTEMPTS = 3;
|
|
|
24
24
|
* instead tells the user to rebuild the store.
|
|
25
25
|
*/
|
|
26
26
|
const MAX_STALE_INDEX_VERSIONS = 40;
|
|
27
|
+
/**
|
|
28
|
+
* Minimum row count for an IVF index to be worth building. Below this,
|
|
29
|
+
* brute-force flat scan is optimal and Lance's own KMeans trainer warns
|
|
30
|
+
* "dataset is too small to have a meaningful index" (it names this 4096
|
|
31
|
+
* floor) when too many of its 16 base partitions end up empty. Skipping
|
|
32
|
+
* index creation here keeps small stores warning-free and avoids a
|
|
33
|
+
* meaningless 9 MB index file.
|
|
34
|
+
*/
|
|
35
|
+
const MIN_ROWS_FOR_IVF_INDEX = 4096;
|
|
27
36
|
/**
|
|
28
37
|
* Count index-version directories in a LanceDB table directory. Each
|
|
29
38
|
* `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
|
|
@@ -444,6 +453,7 @@ export class LanceDbStore {
|
|
|
444
453
|
endLine: 0,
|
|
445
454
|
language: "",
|
|
446
455
|
kind: "",
|
|
456
|
+
role: "",
|
|
447
457
|
quirkType: "",
|
|
448
458
|
tags: "",
|
|
449
459
|
};
|
|
@@ -534,10 +544,10 @@ export class LanceDbStore {
|
|
|
534
544
|
return false;
|
|
535
545
|
}
|
|
536
546
|
}
|
|
537
|
-
/** Add kind/quirkType/tags columns if missing from an existing table. */
|
|
547
|
+
/** Add kind/role/quirkType/tags columns if missing from an existing table. */
|
|
538
548
|
async migrateNewColumns() {
|
|
539
549
|
const missing = [];
|
|
540
|
-
for (const col of ["kind", "quirkType", "tags"]) {
|
|
550
|
+
for (const col of ["kind", "role", "quirkType", "tags"]) {
|
|
541
551
|
if (!(await this.hasColumn(col))) {
|
|
542
552
|
missing.push({ name: col, valueSql: "''" });
|
|
543
553
|
}
|
|
@@ -553,7 +563,7 @@ export class LanceDbStore {
|
|
|
553
563
|
return;
|
|
554
564
|
}
|
|
555
565
|
}
|
|
556
|
-
await this.ensureColumnsNullable(["kind", "quirkType", "tags"]);
|
|
566
|
+
await this.ensureColumnsNullable(["kind", "role", "quirkType", "tags"]);
|
|
557
567
|
}
|
|
558
568
|
/**
|
|
559
569
|
* Ensure that kind/quirkType/tags columns are nullable so that queries
|
|
@@ -647,6 +657,7 @@ export class LanceDbStore {
|
|
|
647
657
|
endLine: c.metadata.endLine,
|
|
648
658
|
language: c.metadata.language,
|
|
649
659
|
kind: c.metadata.kind ?? "",
|
|
660
|
+
role: c.metadata.role ?? "",
|
|
650
661
|
quirkType: c.metadata.quirkType ?? "",
|
|
651
662
|
tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
|
|
652
663
|
};
|
|
@@ -792,6 +803,7 @@ export class LanceDbStore {
|
|
|
792
803
|
endLine: row.endLine,
|
|
793
804
|
language: row.language,
|
|
794
805
|
kind: row.kind || undefined,
|
|
806
|
+
role: row.role === "source" || row.role === "test" || row.role === "doc" ? row.role : undefined,
|
|
795
807
|
quirkType: row.quirkType || undefined,
|
|
796
808
|
tags,
|
|
797
809
|
},
|
|
@@ -860,6 +872,7 @@ export class LanceDbStore {
|
|
|
860
872
|
endLine: row.endLine,
|
|
861
873
|
language: row.language,
|
|
862
874
|
kind: row.kind || undefined,
|
|
875
|
+
role: row.role === "source" || row.role === "test" || row.role === "doc" ? row.role : undefined,
|
|
863
876
|
quirkType: row.quirkType || undefined,
|
|
864
877
|
tags,
|
|
865
878
|
},
|
|
@@ -891,6 +904,7 @@ export class LanceDbStore {
|
|
|
891
904
|
content: row.content,
|
|
892
905
|
description: row.description ?? "",
|
|
893
906
|
kind: row.kind ?? "",
|
|
907
|
+
role: row.role ?? "",
|
|
894
908
|
quirkType: row.quirkType ?? "",
|
|
895
909
|
tags: row.tags ?? "",
|
|
896
910
|
}));
|
|
@@ -935,6 +949,7 @@ export class LanceDbStore {
|
|
|
935
949
|
content: row.content,
|
|
936
950
|
description: row.description ?? "",
|
|
937
951
|
kind: row.kind ?? "",
|
|
952
|
+
role: row.role ?? "",
|
|
938
953
|
quirkType: row.quirkType ?? "",
|
|
939
954
|
tags: row.tags ?? "",
|
|
940
955
|
})),
|
|
@@ -963,6 +978,7 @@ export class LanceDbStore {
|
|
|
963
978
|
content: row.content,
|
|
964
979
|
description: row.description ?? "",
|
|
965
980
|
kind: row.kind ?? "",
|
|
981
|
+
role: row.role ?? "",
|
|
966
982
|
quirkType: row.quirkType ?? "",
|
|
967
983
|
tags: row.tags ?? "",
|
|
968
984
|
};
|
|
@@ -989,6 +1005,7 @@ export class LanceDbStore {
|
|
|
989
1005
|
content: row.content,
|
|
990
1006
|
description: row.description ?? "",
|
|
991
1007
|
kind: row.kind ?? "",
|
|
1008
|
+
role: row.role ?? "",
|
|
992
1009
|
quirkType: row.quirkType ?? "",
|
|
993
1010
|
tags: row.tags ?? "",
|
|
994
1011
|
}));
|
|
@@ -1115,7 +1132,9 @@ export class LanceDbStore {
|
|
|
1115
1132
|
}
|
|
1116
1133
|
/**
|
|
1117
1134
|
* Perform a single index-metric repair pass. Skips stores that have no
|
|
1118
|
-
* index and fewer than
|
|
1135
|
+
* index and fewer than `MIN_ROWS_FOR_IVF_INDEX` rows (brute-force is
|
|
1136
|
+
* optimal there, and an IVF index would only trip Lance's "dataset is too
|
|
1137
|
+
* small" KMeans warning). Uses a
|
|
1119
1138
|
* single `createIndex` with `replace: true` — a dropIndex + createIndex
|
|
1120
1139
|
* sequence races in LanceDB ("Retryable commit conflict") and leaves the
|
|
1121
1140
|
* stale index in place. On failure the memo is cleared so the next
|
|
@@ -1148,8 +1167,8 @@ export class LanceDbStore {
|
|
|
1148
1167
|
const stats = await table.indexStats(idxName);
|
|
1149
1168
|
if (stats && stats.distanceType === "cosine")
|
|
1150
1169
|
return; // already healthy
|
|
1151
|
-
if (!stats && count <
|
|
1152
|
-
return; //
|
|
1170
|
+
if (!stats && count < MIN_ROWS_FOR_IVF_INDEX)
|
|
1171
|
+
return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
|
|
1153
1172
|
const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
|
|
1154
1173
|
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
1155
1174
|
try {
|
|
@@ -70,6 +70,7 @@ export class InMemoryVectorStore {
|
|
|
70
70
|
content: c.content,
|
|
71
71
|
description: c.description ?? "",
|
|
72
72
|
kind: c.metadata.kind ?? "",
|
|
73
|
+
role: c.metadata.role ?? "",
|
|
73
74
|
quirkType: c.metadata.quirkType ?? "",
|
|
74
75
|
tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
|
|
75
76
|
}));
|