opencode-rag-plugin 1.23.1 → 1.23.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/api.js CHANGED
@@ -50,6 +50,9 @@ export async function search(query, options = {}) {
50
50
  minScore: options.minScore ?? ctx.config.retrieval.minScore,
51
51
  keywordIndex: ctx.keywordIndex,
52
52
  keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
53
+ symbolKeywordWeight: ctx.config.retrieval.hybridSearch?.symbolKeywordWeight,
54
+ docKeywordDemotion: ctx.config.retrieval.hybridSearch?.docKeywordDemotion,
55
+ testKeywordDemotion: ctx.config.retrieval.hybridSearch?.testKeywordDemotion,
53
56
  hybridEnabled: ctx.config.retrieval.hybridSearch?.enabled,
54
57
  queryPrefix: ctx.config.embedding.queryPrefix,
55
58
  explain: options.explain,
@@ -64,5 +64,9 @@ export declare function getRegisteredExtensions(): string[];
64
64
  */
65
65
  export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>, options?: {
66
66
  maxSvgSizeBytes?: number;
67
+ /** Maximum lines per chunk; oversized chunks are split into windows. */
68
+ maxChunkSize?: number;
69
+ /** Overlapping lines shared between adjacent/split chunks. */
70
+ chunkOverlap?: number;
67
71
  }): Promise<Chunk[]>;
68
72
  export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
@@ -120,11 +120,34 @@ export function getRegisteredExtensions() {
120
120
  }
121
121
  const MAX_CHUNK_LINES = 100;
122
122
  const MAX_CHUNK_CHARS = 8000;
123
- function splitOversized(chunks, filePath) {
123
+ /** Prepend the tail of the previous chunk as context to the next one, so a
124
+ * construct split across a chunk boundary is still retrievable whole. */
125
+ function applyChunkOverlap(chunks, overlap) {
126
+ if (overlap <= 0 || chunks.length <= 1)
127
+ return chunks;
124
128
  const result = [];
129
+ let prevTail = null;
125
130
  for (const chunk of chunks) {
126
131
  const lines = chunk.content.split("\n");
127
- if (lines.length <= MAX_CHUNK_LINES && chunk.content.length <= MAX_CHUNK_CHARS) {
132
+ if (prevTail && prevTail.length > 0) {
133
+ result.push({
134
+ ...chunk,
135
+ id: uuid(),
136
+ content: [...prevTail, ...lines].join("\n"),
137
+ });
138
+ }
139
+ else {
140
+ result.push(chunk);
141
+ }
142
+ prevTail = lines.slice(Math.max(0, lines.length - overlap));
143
+ }
144
+ return result;
145
+ }
146
+ function splitOversized(chunks, filePath, maxLines = MAX_CHUNK_LINES, maxChars = MAX_CHUNK_CHARS, overlap = 0) {
147
+ const result = [];
148
+ for (const chunk of chunks) {
149
+ const lines = chunk.content.split("\n");
150
+ if (lines.length <= maxLines && chunk.content.length <= maxChars) {
128
151
  result.push(chunk);
129
152
  continue;
130
153
  }
@@ -136,7 +159,7 @@ function splitOversized(chunks, filePath) {
136
159
  const line = lines[i];
137
160
  const lineLen = line.length + 1;
138
161
  if (currentLines.length > 0 &&
139
- (currentLines.length >= MAX_CHUNK_LINES || currentCharCount + lineLen > MAX_CHUNK_CHARS)) {
162
+ (currentLines.length >= maxLines || currentCharCount + lineLen > maxChars)) {
140
163
  subChunks.push({
141
164
  id: uuid(),
142
165
  content: currentLines.join("\n"),
@@ -147,9 +170,12 @@ function splitOversized(chunks, filePath) {
147
170
  language: chunk.metadata.language,
148
171
  },
149
172
  });
150
- currentLines = [];
151
- currentCharCount = 0;
152
- lineOffset = i;
173
+ // Seed the next window with the last `overlap` lines so split chunks
174
+ // still share their boundary context.
175
+ const tail = overlap > 0 ? currentLines.slice(Math.max(0, currentLines.length - overlap)) : [];
176
+ currentLines = [...tail];
177
+ currentCharCount = tail.reduce((sum, l) => sum + l.length + 1, 0);
178
+ lineOffset = i - tail.length;
153
179
  }
154
180
  currentLines.push(line);
155
181
  currentCharCount += lineLen;
@@ -226,7 +252,10 @@ export async function chunkFile(filePath, content, nodeTypesOverrides, options)
226
252
  if (chunks.length === 0) {
227
253
  return fallbackChunker.chunk(filePath, content);
228
254
  }
229
- return splitOversized(chunks, filePath);
255
+ const overlap = Math.max(0, options?.chunkOverlap ?? 0);
256
+ const maxLines = options?.maxChunkSize && options.maxChunkSize > 0 ? options.maxChunkSize : MAX_CHUNK_LINES;
257
+ const split = splitOversized(chunks, filePath, maxLines, MAX_CHUNK_CHARS, overlap);
258
+ return applyChunkOverlap(split, overlap);
230
259
  }
231
260
  export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
232
261
  //# sourceMappingURL=factory.js.map
@@ -513,9 +513,6 @@ export function generateDefaultConfigJson(tuning) {
513
513
  level: DEFAULT_CONFIG.logging.level,
514
514
  logFilePath: DEFAULT_CONFIG.logging.logFilePath,
515
515
  },
516
- chunking: {
517
- nodeTypes: {},
518
- },
519
516
  }, null, 2) + "\n";
520
517
  }
521
518
  //# sourceMappingURL=init-helpers.js.map
@@ -0,0 +1,22 @@
1
+ /**
2
+ * @fileoverview Classifies a chunk's provenance (source / test / doc) from its file path.
3
+ *
4
+ * The retrieval fusion uses this to demote documentation and test chunks when they
5
+ * compete on keyword matches — natural-language queries about "how X works" match
6
+ * doc and test files literally (their prose names the concepts), which otherwise
7
+ * lets them outrank the implementation the query is really after.
8
+ */
9
+ import type { Chunk } from "./interfaces.js";
10
+ export type ChunkRole = "source" | "test" | "doc";
11
+ /**
12
+ * Classify the role of a file from its path. Test takes precedence over doc
13
+ * (a `*.test.md` is a test artifact), and everything else is source.
14
+ */
15
+ export declare function classifyFilePath(filePath: string): ChunkRole;
16
+ /**
17
+ * Tag a chunk with its derived role. Idempotent — a chunk that already carries a
18
+ * role is returned unchanged.
19
+ */
20
+ export declare function tagChunkRole(chunk: Chunk): Chunk;
21
+ /** Tag all chunks produced for a file with their derived role. */
22
+ export declare function tagChunksRole(chunks: Chunk[]): Chunk[];
@@ -0,0 +1,47 @@
1
+ /**
2
+ * @fileoverview Classifies a chunk's provenance (source / test / doc) from its file path.
3
+ *
4
+ * The retrieval fusion uses this to demote documentation and test chunks when they
5
+ * compete on keyword matches — natural-language queries about "how X works" match
6
+ * doc and test files literally (their prose names the concepts), which otherwise
7
+ * lets them outrank the implementation the query is really after.
8
+ */
9
+ const TEST_PATH_SEGMENTS = ["__tests__", "test", "tests", "spec", "specs", "fixtures"];
10
+ const TEST_BASENAME = /\.(?:test|spec|bench)\.[a-z0-9]+$/i;
11
+ const DOC_PATH_SEGMENTS = ["doc", "docs", "documentation"];
12
+ const DOC_EXTENSIONS = [".md", ".mdx", ".rst", ".adoc"];
13
+ /**
14
+ * Classify the role of a file from its path. Test takes precedence over doc
15
+ * (a `*.test.md` is a test artifact), and everything else is source.
16
+ */
17
+ export function classifyFilePath(filePath) {
18
+ const normalized = filePath.replace(/\\/g, "/");
19
+ const segments = normalized.split("/");
20
+ const basename = segments[segments.length - 1] ?? "";
21
+ if (TEST_BASENAME.test(basename))
22
+ return "test";
23
+ if (segments.some((s) => TEST_PATH_SEGMENTS.includes(s.toLowerCase())))
24
+ return "test";
25
+ if (DOC_EXTENSIONS.some((ext) => basename.toLowerCase().endsWith(ext)))
26
+ return "doc";
27
+ if (segments.some((s) => DOC_PATH_SEGMENTS.includes(s.toLowerCase())))
28
+ return "doc";
29
+ return "source";
30
+ }
31
+ /**
32
+ * Tag a chunk with its derived role. Idempotent — a chunk that already carries a
33
+ * role is returned unchanged.
34
+ */
35
+ export function tagChunkRole(chunk) {
36
+ if (chunk.metadata.role)
37
+ return chunk;
38
+ chunk.metadata.role = classifyFilePath(chunk.metadata.filePath);
39
+ return chunk;
40
+ }
41
+ /** Tag all chunks produced for a file with their derived role. */
42
+ export function tagChunksRole(chunks) {
43
+ for (const chunk of chunks)
44
+ tagChunkRole(chunk);
45
+ return chunks;
46
+ }
47
+ //# sourceMappingURL=chunk-role.js.map
@@ -181,16 +181,20 @@ export interface MemoryConfig {
181
181
  minConfidence: number;
182
182
  /** Minimum query-relevance score (0-1) for a quirk to be recalled. */
183
183
  recallMinScore: number;
184
- /** Minimum relevance score for auto-injected quirks (lower than recallMinScore for manual calls). */
184
+ /** Minimum relevance score for quirks to be AUTO-injected into the prompt
185
+ * (both the system-prompt transform and the user message). Deliberately a
186
+ * high bar: injected quirks occupy context on every turn, so only strongly
187
+ * relevant ones should reach the prompt. Manual `recall_quirks` calls use
188
+ * the lower `recallMinScore`. */
185
189
  autoInjectMinScore: number;
186
190
  /** Maximum latency budget (ms) for auto-inject quirk recall. If exceeded, injection is skipped. */
187
191
  autoInjectLatencyBudgetMs: number;
188
- /** Max quirks to auto-inject per turn (default 2). */
192
+ /** Max quirks to auto-inject per turn (default 1). */
189
193
  autoInjectTopK: number;
190
194
  /**
191
195
  * Minimum number of shared word tokens (≥3 chars) between a candidate quirk's
192
196
  * content and the user's *current* message for the quirk to be auto-injected
193
- * (default 1). Acts as a relevance gate against meta-quirks that match only
197
+ * (default 2). Acts as a relevance gate against meta-quirks that match only
194
198
  * the prior assistant text in the combined recall query. Set to `0` to disable.
195
199
  */
196
200
  autoInjectMinTokenOverlap: number;
@@ -319,6 +323,9 @@ export interface RagConfig {
319
323
  includeDirs?: string[];
320
324
  /** Number of overlapping lines between adjacent chunks. */
321
325
  chunkOverlap: number;
326
+ /** Maximum lines per chunk — oversized chunks are split into windows with
327
+ * `chunkOverlap` lines of context between them. 0 = no limit (whole AST node). */
328
+ maxChunkSize?: number;
322
329
  /** Minimum file size in bytes to index (0 = no minimum). */
323
330
  minFileSizeBytes?: number;
324
331
  /** Maximum concurrent file processing tasks. */
@@ -378,6 +385,15 @@ export interface RagConfig {
378
385
  enabled: boolean;
379
386
  /** Weight for keyword scores in fusion (0 = vector only, 1 = keyword only). */
380
387
  keywordWeight: number;
388
+ /** Keyword weight for symbol-style queries (a bare identifier like
389
+ * `cosineSimilarity`). Keyword search nails exact symbol matches, so these
390
+ * get a higher weight than natural-language queries. Defaults to keywordWeight. */
391
+ symbolKeywordWeight?: number;
392
+ /** Multiplier (0-1) on the keyword contribution of documentation chunks.
393
+ * Docs name concepts literally and otherwise outrank the implementation. */
394
+ docKeywordDemotion?: number;
395
+ /** Multiplier (0-1) on the keyword contribution of test chunks. */
396
+ testKeywordDemotion?: number;
381
397
  };
382
398
  /** Context window optimization settings for post-retrieval quality filtering. */
383
399
  contextOptimization?: ContextOptimizationConfig;
@@ -111,6 +111,7 @@ export const DEFAULT_CONFIG = {
111
111
  "package-lock.json",
112
112
  ],
113
113
  chunkOverlap: 0,
114
+ maxChunkSize: 0,
114
115
  minFileSizeBytes: 0,
115
116
  concurrency: 8,
116
117
  embedBatchSize: 100,
@@ -131,6 +132,9 @@ export const DEFAULT_CONFIG = {
131
132
  hybridSearch: {
132
133
  enabled: true,
133
134
  keywordWeight: 0.4,
135
+ symbolKeywordWeight: 0.6,
136
+ docKeywordDemotion: 0.5,
137
+ testKeywordDemotion: 0.6,
134
138
  },
135
139
  contextOptimization: {
136
140
  enabled: true,
@@ -273,11 +277,11 @@ export const DEFAULT_CONFIG = {
273
277
  enabled: true,
274
278
  autoInject: true,
275
279
  minConfidence: 0.5,
276
- recallMinScore: 0.6,
277
- autoInjectMinScore: 0.5,
280
+ recallMinScore: 0.72,
281
+ autoInjectMinScore: 0.75,
278
282
  autoInjectLatencyBudgetMs: 2000,
279
- autoInjectTopK: 2,
280
- autoInjectMinTokenOverlap: 1,
283
+ autoInjectTopK: 1,
284
+ autoInjectMinTokenOverlap: 2,
281
285
  passiveCapture: true,
282
286
  promptEnforcement: true,
283
287
  sessionEndExtraction: true,
@@ -365,6 +369,22 @@ export function validateConfig(config) {
365
369
  if (kw < 0 || kw > 1) {
366
370
  warnings.push("retrieval.hybridSearch.keywordWeight must be between 0 and 1");
367
371
  }
372
+ const hs = config.retrieval.hybridSearch;
373
+ if (hs.symbolKeywordWeight != null && (hs.symbolKeywordWeight < 0 || hs.symbolKeywordWeight > 1)) {
374
+ warnings.push("retrieval.hybridSearch.symbolKeywordWeight must be between 0 and 1");
375
+ }
376
+ if (hs.docKeywordDemotion != null && (hs.docKeywordDemotion < 0 || hs.docKeywordDemotion > 1)) {
377
+ warnings.push("retrieval.hybridSearch.docKeywordDemotion must be between 0 and 1");
378
+ }
379
+ if (hs.testKeywordDemotion != null && (hs.testKeywordDemotion < 0 || hs.testKeywordDemotion > 1)) {
380
+ warnings.push("retrieval.hybridSearch.testKeywordDemotion must be between 0 and 1");
381
+ }
382
+ }
383
+ if (config.indexing.maxChunkSize != null && config.indexing.maxChunkSize < 0) {
384
+ warnings.push("indexing.maxChunkSize must be >= 0");
385
+ }
386
+ if (config.indexing.maxChunkSize && config.indexing.maxChunkSize <= config.indexing.chunkOverlap) {
387
+ warnings.push("indexing.maxChunkSize must be greater than chunkOverlap (otherwise windows make no progress)");
368
388
  }
369
389
  if (config.memory?.recallMinScore != null) {
370
390
  const r = config.memory.recallMinScore;
@@ -16,6 +16,9 @@ export interface Chunk {
16
16
  contentType?: string;
17
17
  /** Synthetic chunk kind — e.g. "quirk" for experiential memory entries. */
18
18
  kind?: string;
19
+ /** Provenance of the file, derived from its path: source / test / doc. Used by
20
+ * hybrid fusion to demote doc/test chunks that match query terms literally. */
21
+ role?: "source" | "test" | "doc";
19
22
  /** Quirk sub-type when kind === "quirk": gotcha, preference, decision, environment-constraint. */
20
23
  quirkType?: string;
21
24
  /** Arbitrary tags for filtering/queries. */
@@ -79,6 +82,12 @@ export interface SearchExplanation {
79
82
  /** Rank (0-indexed) in the keyword index results. */
80
83
  keywordRank?: number;
81
84
  };
85
+ /** Calibrated relevance in [0,1] derived from the raw vector-score spread of the
86
+ * candidate set — unlike the fused `score` (RRF-normalized so rank 0 is always
87
+ * ~1.0), this reflects how confident the match actually is. */
88
+ confidence?: number;
89
+ /** Shape of the query that produced this result ("symbol" vs "natural-language"). */
90
+ queryShape?: "symbol" | "natural-language";
82
91
  /** Query terms that matched in the keyword index, if hybrid search was used. */
83
92
  matchedTerms?: string[];
84
93
  }
@@ -148,6 +157,7 @@ export interface ChunkSummary {
148
157
  content: string;
149
158
  description: string;
150
159
  kind: string;
160
+ role: string;
151
161
  quirkType: string;
152
162
  tags: string;
153
163
  }
@@ -12,6 +12,7 @@ import { createVectorStore } from "../vectorstore/factory.js";
12
12
  import { swapStoreDirectories } from "../vectorstore/lancedb.js";
13
13
  import { createIndexStats } from "./stats.js";
14
14
  import { prepareFile, buildTextsToEmbed } from "./worker.js";
15
+ import { tagChunksRole } from "../core/chunk-role.js";
15
16
  import { buildFallbackDescription } from "./description-stage.js";
16
17
  import { getCurrentCommit, getChangedFilesSince, getUntrackedFiles, getRepoRoot } from "./git-diff.js";
17
18
  export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
@@ -410,6 +411,11 @@ async function runIndexPassInner(options, logger) {
410
411
  options.progress?.startFile(fileLabel);
411
412
  }
412
413
  const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger, deferDescriptions, descHash);
414
+ // Stamp every chunk with its file provenance (source/test/doc) so
415
+ // hybrid fusion can demote docs/tests that match query terms literally.
416
+ if (prep.chunks && prep.chunks.length > 0) {
417
+ tagChunksRole(prep.chunks);
418
+ }
413
419
  if (isActive) {
414
420
  chunkedDone++;
415
421
  if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
@@ -121,6 +121,8 @@ export declare function prepareFile(file: WorkspaceFile, cwd: string, previous:
121
121
  indexing?: {
122
122
  maxSvgSizeBytes?: number;
123
123
  embedDescriptions?: boolean;
124
+ maxChunkSize?: number;
125
+ chunkOverlap?: number;
124
126
  };
125
127
  }, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger, deferDescriptions?: boolean, descHash?: string): Promise<PreparedFile>;
126
128
  /**
@@ -117,6 +117,8 @@ export async function prepareFile(file, cwd, previous, config, keywordIndex, des
117
117
  else {
118
118
  chunks = await chunkFile(file.filePath, file.content, config.chunking?.nodeTypes, {
119
119
  maxSvgSizeBytes: config.indexing?.maxSvgSizeBytes,
120
+ maxChunkSize: config.indexing?.maxChunkSize,
121
+ chunkOverlap: config.indexing?.chunkOverlap,
120
122
  }).catch((err) => {
121
123
  logger.warn(` ${fileLabel} (chunking failed: ${err.message})`);
122
124
  return null;
package/dist/plugin.js CHANGED
@@ -839,7 +839,7 @@ export function createRagHooks(options) {
839
839
  const budgetMs = memCfg.autoInjectLatencyBudgetMs ?? 2000;
840
840
  let timer;
841
841
  let quirkResults = await Promise.race([
842
- recallQuirks(quirkDeps, query, { topK: memCfg.autoInjectTopK ?? 2, minScore: memCfg.autoInjectMinScore })
842
+ recallQuirks(quirkDeps, query, { topK: memCfg.autoInjectTopK ?? 1, minScore: memCfg.autoInjectMinScore })
843
843
  .then((r) => {
844
844
  if (timer)
845
845
  clearTimeout(timer);
@@ -855,7 +855,7 @@ export function createRagHooks(options) {
855
855
  // Lexical gate: drop quirks that match only the prior assistant text (no token
856
856
  // overlap with the user's actual request) — prevents meta-quirks (quirks about
857
857
  // quirks) from being injected into unrelated tasks. Disabled when autoInjectMinTokenOverlap=0.
858
- const minTokenOverlap = memCfg.autoInjectMinTokenOverlap ?? 1;
858
+ const minTokenOverlap = memCfg.autoInjectMinTokenOverlap ?? 2;
859
859
  if (minTokenOverlap > 0 && userReq.length > 0) {
860
860
  quirkResults = quirkResults.filter((qr) => sharedWords(qr.chunk.content, userReq) >= minTokenOverlap);
861
861
  }
@@ -1140,12 +1140,14 @@ export function createRagHooks(options) {
1140
1140
  if (quirkQuery.length > 0) {
1141
1141
  const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1142
1142
  const budgetMs = quirkMemoryCfg.autoInjectLatencyBudgetMs ?? 2000;
1143
- // Use the stricter recallMinScore for user-prompt injection
1144
- // (system prompt injection uses the lower autoInjectMinScore)
1145
- const minScore = quirkMemoryCfg.recallMinScore ?? 0.72;
1143
+ // Both auto-inject paths (system prompt transform and this user-message
1144
+ // injection) use the same high autoInjectMinScore bar — injected quirks
1145
+ // occupy context every turn, so only strongly relevant ones qualify.
1146
+ // Manual recall_quirks calls use the lower recallMinScore.
1147
+ const minScore = quirkMemoryCfg.autoInjectMinScore ?? 0.75;
1146
1148
  let timer;
1147
1149
  let quirkResults = await Promise.race([
1148
- recallQuirks(quirkDeps, quirkQuery, { topK: quirkMemoryCfg.autoInjectTopK ?? 2, minScore })
1150
+ recallQuirks(quirkDeps, quirkQuery, { topK: quirkMemoryCfg.autoInjectTopK ?? 1, minScore })
1149
1151
  .then((r) => {
1150
1152
  if (timer)
1151
1153
  clearTimeout(timer);
@@ -1161,7 +1163,7 @@ export function createRagHooks(options) {
1161
1163
  // Lexical gate: drop quirks that match only the prior assistant text (no token
1162
1164
  // overlap with the user's current message) — prevents meta-quirks (quirks about
1163
1165
  // quirks) from being injected into unrelated tasks. Disabled when autoInjectMinTokenOverlap=0.
1164
- const minTokenOverlap = quirkMemoryCfg.autoInjectMinTokenOverlap ?? 1;
1166
+ const minTokenOverlap = quirkMemoryCfg.autoInjectMinTokenOverlap ?? 2;
1165
1167
  if (minTokenOverlap > 0 && text.length > 0) {
1166
1168
  quirkResults = quirkResults.filter((qr) => sharedWords(qr.chunk.content, text) >= minTokenOverlap);
1167
1169
  }
@@ -63,6 +63,10 @@ function mergeAdjacentChunks(results, gapThreshold) {
63
63
  const sorted = [...results].sort((a, b) => a.chunk.metadata.startLine - b.chunk.metadata.startLine);
64
64
  const merged = [];
65
65
  let pending = toOptimized(sorted[0]);
66
+ // The explanation (score breakdown, ranks) of the best-scoring constituent —
67
+ // carried onto the merged result so `--explain` stays accurate after merges.
68
+ let pendingExplanation = sorted[0].explanation;
69
+ let pendingBestScore = sorted[0].score;
66
70
  for (let i = 1; i < sorted.length; i++) {
67
71
  const current = sorted[i];
68
72
  const gap = current.chunk.metadata.startLine - pending.chunk.metadata.endLine;
@@ -83,15 +87,23 @@ function mergeAdjacentChunks(results, gapThreshold) {
83
87
  endLine: current.chunk.metadata.endLine,
84
88
  },
85
89
  };
90
+ const bestScore = Math.max(pending.score, current.score);
91
+ if (current.score > pendingBestScore) {
92
+ pendingBestScore = current.score;
93
+ pendingExplanation = current.explanation;
94
+ }
86
95
  pending = {
87
96
  chunk: mergedChunk,
88
- score: Math.max(pending.score, current.score),
97
+ score: bestScore,
98
+ explanation: pendingExplanation,
89
99
  optimized: { mergedFrom: sourceIds },
90
100
  };
91
101
  }
92
102
  else {
93
103
  merged.push(pending);
94
104
  pending = toOptimized(current);
105
+ pendingExplanation = current.explanation;
106
+ pendingBestScore = current.score;
95
107
  }
96
108
  }
97
109
  if (pending)
@@ -20,11 +20,26 @@ export declare class KeywordIndex {
20
20
  private readonly storePath?;
21
21
  /** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
22
22
  private fileToIds;
23
+ /** Indexed-token length per chunk, and the running total — drives BM25 length
24
+ * normalization so long chunks can't dominate purely on term frequency. */
25
+ private docLengths;
26
+ private totalTokens;
27
+ /** The exact set of tokens indexed per chunk id, recorded at add time.
28
+ * Removal deletes precisely these entries instead of re-tokenizing the
29
+ * chunk: re-derivation would leave dangling inverted-index references
30
+ * whenever the chunk's text drifted between indexing and removal (the
31
+ * description stage mutates `chunk.description` after addChunks, or the
32
+ * persisted index was written by a build whose tokenizer differed). */
33
+ private chunkTokens;
23
34
  constructor(storePath?: string);
24
35
  addChunks(chunks: Chunk[]): void;
25
36
  getMatchedTerms(query: string, chunkId: string): string[];
26
37
  removeByFilePath(filePath: string): void;
27
38
  search(query: string, topK: number, filter?: MetadataFilter): SearchResult[];
39
+ /** Remove every inverted-index reference to chunk ids that no longer have a
40
+ * chunkMap entry. Uses the recorded token set when available (fast path,
41
+ * always taken for loaded indexes) and falls back to a full scan. */
42
+ private pruneDangling;
28
43
  close(): void;
29
44
  clear(): void;
30
45
  count(): number;
@@ -1,6 +1,14 @@
1
1
  import { normalizeFilePath } from "../core/manifest.js";
2
2
  import { normalizeFileExtensions, matchesFileExtension } from "../core/filters.js";
3
- const INDEX_VERSION = 2;
3
+ const INDEX_VERSION = 3;
4
+ /**
5
+ * Text indexed for keyword search: chunk content plus its LLM description when
6
+ * one exists. Without this, keyword and vector search would see different text
7
+ * spaces — a description can name a concept the raw code never spells out.
8
+ */
9
+ function indexableText(chunk) {
10
+ return chunk.description ? `${chunk.content}\n${chunk.description}` : chunk.content;
11
+ }
4
12
  /** Suffix-stripping stemmer. Only stems words >= 6 characters to reduce false positives. */
5
13
  function stem(word) {
6
14
  if (word.length < 6)
@@ -75,6 +83,17 @@ export class KeywordIndex {
75
83
  storePath;
76
84
  /** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
77
85
  fileToIds = new Map();
86
+ /** Indexed-token length per chunk, and the running total — drives BM25 length
87
+ * normalization so long chunks can't dominate purely on term frequency. */
88
+ docLengths = new Map();
89
+ totalTokens = 0;
90
+ /** The exact set of tokens indexed per chunk id, recorded at add time.
91
+ * Removal deletes precisely these entries instead of re-tokenizing the
92
+ * chunk: re-derivation would leave dangling inverted-index references
93
+ * whenever the chunk's text drifted between indexing and removal (the
94
+ * description stage mutates `chunk.description` after addChunks, or the
95
+ * persisted index was written by a build whose tokenizer differed). */
96
+ chunkTokens = new Map();
78
97
  constructor(storePath) {
79
98
  this.storePath = storePath;
80
99
  }
@@ -89,7 +108,10 @@ export class KeywordIndex {
89
108
  this.fileToIds.set(fileKey, ids);
90
109
  }
91
110
  ids.add(id);
92
- const tokens = tokenize(chunk.content);
111
+ const tokens = tokenize(indexableText(chunk));
112
+ this.docLengths.set(id, tokens.length);
113
+ this.totalTokens += tokens.length;
114
+ this.chunkTokens.set(id, new Set(tokens));
93
115
  for (const token of tokens) {
94
116
  let docs = this.invertedIndex.get(token);
95
117
  if (!docs) {
@@ -132,7 +154,10 @@ export class KeywordIndex {
132
154
  for (const id of idsToRemove) {
133
155
  const chunk = this.chunkMap.get(id);
134
156
  if (chunk) {
135
- const tokens = tokenize(chunk.content);
157
+ // Delete exactly the entries addChunks created. The recorded token set
158
+ // is authoritative; re-derive only when the entry predates exact
159
+ // tracking (loaded indexes always rebuild chunkTokens in load()).
160
+ const tokens = this.chunkTokens.get(id) ?? new Set(tokenize(indexableText(chunk)));
136
161
  for (const token of tokens) {
137
162
  const docs = this.invertedIndex.get(token);
138
163
  if (docs) {
@@ -143,6 +168,12 @@ export class KeywordIndex {
143
168
  }
144
169
  }
145
170
  this.chunkMap.delete(id);
171
+ this.chunkTokens.delete(id);
172
+ const len = this.docLengths.get(id);
173
+ if (len !== undefined) {
174
+ this.totalTokens -= len;
175
+ this.docLengths.delete(id);
176
+ }
146
177
  }
147
178
  }
148
179
  }
@@ -153,36 +184,91 @@ export class KeywordIndex {
153
184
  if (queryTokens.length === 0)
154
185
  return [];
155
186
  const totalChunks = this.chunkMap.size;
187
+ const avgDocLength = this.totalTokens > 0 ? this.totalTokens / totalChunks : 0;
156
188
  const scores = new Map();
189
+ // BM25: term-frequency saturation plus length normalization, so a chunk that
190
+ // merely repeats a term (or is very long) can't outrank a focused match.
191
+ const K1 = 1.2;
192
+ const B = 0.75;
157
193
  for (const token of queryTokens) {
158
194
  const docs = this.invertedIndex.get(token);
159
195
  if (!docs)
160
196
  continue;
161
197
  const df = docs.size;
162
- const idf = Math.log(1 + totalChunks / (df || 1));
198
+ const idf = Math.log(1 + (totalChunks - df + 0.5) / (df + 0.5));
163
199
  for (const [chunkId, freq] of docs) {
164
- const score = freq * idf;
165
- scores.set(chunkId, (scores.get(chunkId) ?? 0) + score);
200
+ const docLength = this.docLengths.get(chunkId) ?? avgDocLength;
201
+ const lengthNorm = avgDocLength > 0
202
+ ? 1 - B + B * (docLength / avgDocLength)
203
+ : 1;
204
+ const tf = (freq * (K1 + 1)) / (freq + K1 * lengthNorm);
205
+ scores.set(chunkId, (scores.get(chunkId) ?? 0) + idf * tf);
166
206
  }
167
207
  }
208
+ const dangling = [];
168
209
  const sorted = [...scores.entries()]
169
- .filter(([id]) => matchesFilter(this.chunkMap.get(id), filter))
210
+ .filter(([id]) => {
211
+ const chunk = this.chunkMap.get(id);
212
+ if (!chunk) {
213
+ // The inverted index references a chunk id that no longer exists.
214
+ // This is the "undefined is not an object (evaluating
215
+ // 'chunk.metadata')" crash site: a removal that missed tokens
216
+ // (index persisted by an older build, interrupted re-index) leaves
217
+ // phantom ids behind. Skip and prune them so a corrupt index
218
+ // self-heals instead of failing every filtered search.
219
+ dangling.push(id);
220
+ return false;
221
+ }
222
+ return matchesFilter(chunk, filter);
223
+ })
170
224
  .sort((a, b) => b[1] - a[1])
171
225
  .slice(0, topK);
226
+ if (dangling.length > 0)
227
+ this.pruneDangling(dangling);
172
228
  return sorted.map(([chunkId, score]) => {
173
229
  const chunk = this.chunkMap.get(chunkId);
174
230
  return { chunk, score };
175
231
  });
176
232
  }
233
+ /** Remove every inverted-index reference to chunk ids that no longer have a
234
+ * chunkMap entry. Uses the recorded token set when available (fast path,
235
+ * always taken for loaded indexes) and falls back to a full scan. */
236
+ pruneDangling(ids) {
237
+ for (const id of ids) {
238
+ const tokens = this.chunkTokens.get(id);
239
+ if (tokens) {
240
+ for (const token of tokens) {
241
+ const docs = this.invertedIndex.get(token);
242
+ if (docs) {
243
+ docs.delete(id);
244
+ if (docs.size === 0)
245
+ this.invertedIndex.delete(token);
246
+ }
247
+ }
248
+ }
249
+ else {
250
+ for (const [token, docs] of this.invertedIndex) {
251
+ if (docs.delete(id) && docs.size === 0)
252
+ this.invertedIndex.delete(token);
253
+ }
254
+ }
255
+ this.chunkTokens.delete(id);
256
+ this.docLengths.delete(id);
257
+ }
258
+ }
177
259
  close() {
178
260
  this.invertedIndex.clear();
179
261
  this.chunkMap.clear();
180
262
  this.fileToIds.clear();
263
+ this.chunkTokens.clear();
181
264
  }
182
265
  clear() {
183
266
  this.invertedIndex.clear();
184
267
  this.chunkMap.clear();
185
268
  this.fileToIds.clear();
269
+ this.chunkTokens.clear();
270
+ this.docLengths.clear();
271
+ this.totalTokens = 0;
186
272
  }
187
273
  count() {
188
274
  return this.chunkMap.size;
@@ -209,6 +295,7 @@ export class KeywordIndex {
209
295
  endLine: chunk.metadata.endLine,
210
296
  language: chunk.metadata.language,
211
297
  kind: chunk.metadata.kind ?? "",
298
+ role: chunk.metadata.role ?? "",
212
299
  quirkType: chunk.metadata.quirkType ?? "",
213
300
  tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
214
301
  },
@@ -241,6 +328,19 @@ export class KeywordIndex {
241
328
  const docMap = new Map(docs);
242
329
  index.invertedIndex.set(token, docMap);
243
330
  }
331
+ // Rebuild the exact per-chunk token sets from the persisted inverted
332
+ // index, so removal is exact even for indexes written by older builds
333
+ // (whose tokenizer or indexableText may have differed).
334
+ for (const [token, docs] of index.invertedIndex) {
335
+ for (const id of docs.keys()) {
336
+ let tokens = index.chunkTokens.get(id);
337
+ if (!tokens) {
338
+ tokens = new Set();
339
+ index.chunkTokens.set(id, tokens);
340
+ }
341
+ tokens.add(token);
342
+ }
343
+ }
244
344
  for (const [id, data] of Object.entries(parsed.chunkMap)) {
245
345
  let tags;
246
346
  try {
@@ -260,11 +360,23 @@ export class KeywordIndex {
260
360
  endLine: data.endLine,
261
361
  language: data.language,
262
362
  kind: data.kind || undefined,
363
+ role: data.role === "source" || data.role === "test" || data.role === "doc" ? data.role : undefined,
263
364
  quirkType: data.quirkType || undefined,
264
365
  tags,
265
366
  },
266
367
  });
267
368
  }
369
+ // Recompute BM25 length statistics from the loaded chunk map. The persisted
370
+ // inverted index is authoritative for term frequencies; lengths are cheap
371
+ // to derive and this keeps the serialized format lean.
372
+ for (const [id, chunk] of index.chunkMap) {
373
+ // Prefer the exact token count from the inverted index over
374
+ // re-tokenizing — re-tokenization can drift from what was indexed.
375
+ const recorded = index.chunkTokens.get(id);
376
+ const len = recorded ? recorded.size : tokenize(indexableText(chunk)).length;
377
+ index.docLengths.set(id, len);
378
+ index.totalTokens += len;
379
+ }
268
380
  // Rebuild the per-file id index from the loaded chunk map so removals
269
381
  // work even for entries persisted by older versions.
270
382
  for (const [id, chunk] of index.chunkMap) {
@@ -8,6 +8,13 @@ export interface RetrieveOptions {
8
8
  minScore?: number;
9
9
  keywordIndex?: KeywordIndex;
10
10
  keywordWeight?: number;
11
+ /** Keyword weight for symbol-style queries (a bare identifier like `cosineSimilarity`).
12
+ * Keyword search nails exact symbol matches, so these queries get a higher weight. */
13
+ symbolKeywordWeight?: number;
14
+ /** Multiplier applied to the keyword contribution of documentation chunks (0-1). */
15
+ docKeywordDemotion?: number;
16
+ /** Multiplier applied to the keyword contribution of test chunks (0-1). */
17
+ testKeywordDemotion?: number;
11
18
  /** Whether hybrid search is enabled. When false, keyword index is ignored. */
12
19
  hybridEnabled?: boolean;
13
20
  queryPrefix?: string;
@@ -5,6 +5,41 @@ const FETCH_OVERFETCH_FACTOR = 3;
5
5
  const RRF_K = 60;
6
6
  /** Multiply raw RRF scores by (K+1) to normalize to ~[0,1]. */
7
7
  const RRF_NORMALIZE = RRF_K + 1;
8
+ /** Classify a query as a bare symbol lookup vs. natural-language prose.
9
+ * Symbol queries have no whitespace and look like one (possibly dotted) identifier. */
10
+ function detectQueryShape(query) {
11
+ const trimmed = query.trim();
12
+ if (trimmed.length === 0 || trimmed.length > 60)
13
+ return "natural-language";
14
+ if (/\s/.test(trimmed))
15
+ return "natural-language";
16
+ return /^[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)*$/.test(trimmed)
17
+ ? "symbol"
18
+ : "natural-language";
19
+ }
20
+ /** Demotion applied to a chunk's keyword contribution based on its file's provenance.
21
+ * Docs and tests name concepts literally, so their keyword matches are discounted. */
22
+ function keywordDemotion(chunk, docDemotion, testDemotion) {
23
+ switch (chunk.metadata.role) {
24
+ case "doc": return docDemotion;
25
+ case "test": return testDemotion;
26
+ default: return 1;
27
+ }
28
+ }
29
+ /** Calibrate a per-result confidence from the raw vector-score spread of the
30
+ * candidate set. Unlike the fused RRF score (rank 0 is always ~1.0), this
31
+ * reflects how separable the top match actually was. */
32
+ function attachConfidence(results) {
33
+ const raws = results.map((r) => r.explanation?.scoreBreakdown?.rawVectorScore ?? 0);
34
+ const max = Math.max(...raws);
35
+ const min = Math.min(...raws);
36
+ for (const r of results) {
37
+ if (!r.explanation)
38
+ continue;
39
+ const raw = r.explanation.scoreBreakdown?.rawVectorScore ?? 0;
40
+ r.explanation.confidence = max > min ? (raw - min) / (max - min) : 1;
41
+ }
42
+ }
8
43
  /**
9
44
  * Perform hybrid vector-keyword retrieval with configurable scoring.
10
45
  *
@@ -22,6 +57,15 @@ export async function retrieve(query, embedder, store, options = {}) {
22
57
  try {
23
58
  const topK = options.topK ?? 10;
24
59
  const minScore = options.minScore ?? 0;
60
+ const queryShape = detectQueryShape(query);
61
+ const baseKw = options.keywordWeight ?? 0.4;
62
+ // Symbol queries are exact-identifier lookups — keyword search is the strong
63
+ // signal there, so use the (typically higher) symbol weight.
64
+ const kw = queryShape === "symbol"
65
+ ? options.symbolKeywordWeight ?? baseKw
66
+ : baseKw;
67
+ const docDemotion = options.docKeywordDemotion ?? 0.5;
68
+ const testDemotion = options.testKeywordDemotion ?? 0.6;
25
69
  const prefixedQuery = (options.queryPrefix ?? "") + query;
26
70
  const embeddings = await embedder.embed([prefixedQuery], "query");
27
71
  const embedding = embeddings[0];
@@ -39,7 +83,6 @@ export async function retrieve(query, embedder, store, options = {}) {
39
83
  if (keywordResults.length === 0) {
40
84
  const filtered = vectorResults.filter((r) => r.score >= minScore).slice(0, topK);
41
85
  if (options.explain) {
42
- const kw = options.keywordWeight ?? 0.4;
43
86
  for (const r of filtered) {
44
87
  r.explanation = {
45
88
  scoreBreakdown: {
@@ -49,8 +92,10 @@ export async function retrieve(query, embedder, store, options = {}) {
49
92
  rawKeywordScore: 0,
50
93
  keywordWeight: kw,
51
94
  },
95
+ queryShape,
52
96
  };
53
97
  }
98
+ attachConfidence(filtered);
54
99
  }
55
100
  return filtered;
56
101
  }
@@ -62,15 +107,18 @@ export async function retrieve(query, embedder, store, options = {}) {
62
107
  for (const r of keywordResults)
63
108
  if (!chunkById.has(r.chunk.id))
64
109
  chunkById.set(r.chunk.id, r);
65
- const kw = options.keywordWeight ?? 0.4;
66
110
  const allIds = new Set([...vRank.keys(), ...kRank.keys()]);
67
111
  const combinedResults = [...allIds].map((id) => {
68
112
  const vR = vRank.get(id);
69
113
  const kR = kRank.get(id);
114
+ const chunk = chunkById.get(id)?.chunk;
115
+ if (!chunk)
116
+ return null;
117
+ const kMultiplier = kR !== undefined ? keywordDemotion(chunk, docDemotion, testDemotion) : 1;
70
118
  const vContrib = vR !== undefined ? ((1 - kw) * RRF_NORMALIZE) / (RRF_K + vR + 1) : 0;
71
- const kContrib = kR !== undefined ? (kw * RRF_NORMALIZE) / (RRF_K + kR + 1) : 0;
119
+ const kContrib = kR !== undefined ? (kw * RRF_NORMALIZE) / (RRF_K + kR + 1) * kMultiplier : 0;
72
120
  const score = vContrib + kContrib;
73
- const result = { chunk: chunkById.get(id).chunk, score };
121
+ const result = { chunk, score };
74
122
  if (options.explain) {
75
123
  result.explanation = {
76
124
  scoreBreakdown: {
@@ -82,6 +130,7 @@ export async function retrieve(query, embedder, store, options = {}) {
82
130
  vectorRank: vR,
83
131
  keywordRank: kR,
84
132
  },
133
+ queryShape,
85
134
  };
86
135
  if (options.keywordIndex && kR !== undefined) {
87
136
  const terms = options.keywordIndex.getMatchedTerms(query, id);
@@ -91,9 +140,12 @@ export async function retrieve(query, embedder, store, options = {}) {
91
140
  }
92
141
  return result;
93
142
  })
143
+ .filter((r) => r !== null)
94
144
  .filter((r) => r.score >= minScore)
95
145
  .sort((a, b) => b.score - a.score)
96
146
  .slice(0, topK);
147
+ if (options.explain)
148
+ attachConfidence(combinedResults);
97
149
  return combinedResults;
98
150
  }
99
151
  catch (err) {
package/dist/tui.js CHANGED
@@ -543,15 +543,15 @@ function buildSettingCategories(cfg, ro, providers) {
543
543
  },
544
544
  {
545
545
  path: ["memory", "recallMinScore"],
546
- label: "User prompt min score",
546
+ label: "Manual recall min score",
547
547
  type: "number",
548
548
  currentValue: memoryRo.recallMinScore ?? memoryCfg.recallMinScore ?? 0.72,
549
549
  },
550
550
  {
551
551
  path: ["memory", "autoInjectMinScore"],
552
- label: "System prompt min score",
552
+ label: "Auto-inject min score",
553
553
  type: "number",
554
- currentValue: memoryRo.autoInjectMinScore ?? memoryCfg.autoInjectMinScore ?? 0.6,
554
+ currentValue: memoryRo.autoInjectMinScore ?? memoryCfg.autoInjectMinScore ?? 0.75,
555
555
  },
556
556
  {
557
557
  path: ["memory", "autoInjectLatencyBudgetMs"],
@@ -180,7 +180,7 @@ export declare class LanceDbStore implements VectorStore {
180
180
  */
181
181
  private assertEmbeddingDimensions;
182
182
  private hasColumn;
183
- /** Add kind/quirkType/tags columns if missing from an existing table. */
183
+ /** Add kind/role/quirkType/tags columns if missing from an existing table. */
184
184
  private migrateNewColumns;
185
185
  /**
186
186
  * Ensure that kind/quirkType/tags columns are nullable so that queries
@@ -315,7 +315,9 @@ export declare class LanceDbStore implements VectorStore {
315
315
  private ensureCosineIndex;
316
316
  /**
317
317
  * Perform a single index-metric repair pass. Skips stores that have no
318
- * index and fewer than 1000 rows (brute-force is optimal there). Uses a
318
+ * index and fewer than `MIN_ROWS_FOR_IVF_INDEX` rows (brute-force is
319
+ * optimal there, and an IVF index would only trip Lance's "dataset is too
320
+ * small" KMeans warning). Uses a
319
321
  * single `createIndex` with `replace: true` — a dropIndex + createIndex
320
322
  * sequence races in LanceDB ("Retryable commit conflict") and leaves the
321
323
  * stale index in place. On failure the memo is cleared so the next
@@ -8,7 +8,7 @@ import path from "node:path";
8
8
  import { normalizeFilePath, manifestPathFor } from "../core/manifest.js";
9
9
  import { normalizeFileExtensions, matchesFileExtension } from "../core/filters.js";
10
10
  const TABLE_NAME = "chunks";
11
- const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language", "kind", "quirkType", "tags"];
11
+ const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language", "kind", "role", "quirkType", "tags"];
12
12
  /**
13
13
  * Upper bound for the number of failed index-creation attempts per process.
14
14
  * Beyond this the repair gives up for the process lifetime instead of
@@ -24,6 +24,15 @@ const MAX_INDEX_REPAIR_ATTEMPTS = 3;
24
24
  * instead tells the user to rebuild the store.
25
25
  */
26
26
  const MAX_STALE_INDEX_VERSIONS = 40;
27
+ /**
28
+ * Minimum row count for an IVF index to be worth building. Below this,
29
+ * brute-force flat scan is optimal and Lance's own KMeans trainer warns
30
+ * "dataset is too small to have a meaningful index" (it names this 4096
31
+ * floor) when too many of its 16 base partitions end up empty. Skipping
32
+ * index creation here keeps small stores warning-free and avoids a
33
+ * meaningless 9 MB index file.
34
+ */
35
+ const MIN_ROWS_FOR_IVF_INDEX = 4096;
27
36
  /**
28
37
  * Count index-version directories in a LanceDB table directory. Each
29
38
  * `createIndex` writes a new `<uuid>/` directory under `_indices`; a healthy
@@ -444,6 +453,7 @@ export class LanceDbStore {
444
453
  endLine: 0,
445
454
  language: "",
446
455
  kind: "",
456
+ role: "",
447
457
  quirkType: "",
448
458
  tags: "",
449
459
  };
@@ -534,10 +544,10 @@ export class LanceDbStore {
534
544
  return false;
535
545
  }
536
546
  }
537
- /** Add kind/quirkType/tags columns if missing from an existing table. */
547
+ /** Add kind/role/quirkType/tags columns if missing from an existing table. */
538
548
  async migrateNewColumns() {
539
549
  const missing = [];
540
- for (const col of ["kind", "quirkType", "tags"]) {
550
+ for (const col of ["kind", "role", "quirkType", "tags"]) {
541
551
  if (!(await this.hasColumn(col))) {
542
552
  missing.push({ name: col, valueSql: "''" });
543
553
  }
@@ -553,7 +563,7 @@ export class LanceDbStore {
553
563
  return;
554
564
  }
555
565
  }
556
- await this.ensureColumnsNullable(["kind", "quirkType", "tags"]);
566
+ await this.ensureColumnsNullable(["kind", "role", "quirkType", "tags"]);
557
567
  }
558
568
  /**
559
569
  * Ensure that kind/quirkType/tags columns are nullable so that queries
@@ -647,6 +657,7 @@ export class LanceDbStore {
647
657
  endLine: c.metadata.endLine,
648
658
  language: c.metadata.language,
649
659
  kind: c.metadata.kind ?? "",
660
+ role: c.metadata.role ?? "",
650
661
  quirkType: c.metadata.quirkType ?? "",
651
662
  tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
652
663
  };
@@ -792,6 +803,7 @@ export class LanceDbStore {
792
803
  endLine: row.endLine,
793
804
  language: row.language,
794
805
  kind: row.kind || undefined,
806
+ role: row.role === "source" || row.role === "test" || row.role === "doc" ? row.role : undefined,
795
807
  quirkType: row.quirkType || undefined,
796
808
  tags,
797
809
  },
@@ -860,6 +872,7 @@ export class LanceDbStore {
860
872
  endLine: row.endLine,
861
873
  language: row.language,
862
874
  kind: row.kind || undefined,
875
+ role: row.role === "source" || row.role === "test" || row.role === "doc" ? row.role : undefined,
863
876
  quirkType: row.quirkType || undefined,
864
877
  tags,
865
878
  },
@@ -891,6 +904,7 @@ export class LanceDbStore {
891
904
  content: row.content,
892
905
  description: row.description ?? "",
893
906
  kind: row.kind ?? "",
907
+ role: row.role ?? "",
894
908
  quirkType: row.quirkType ?? "",
895
909
  tags: row.tags ?? "",
896
910
  }));
@@ -935,6 +949,7 @@ export class LanceDbStore {
935
949
  content: row.content,
936
950
  description: row.description ?? "",
937
951
  kind: row.kind ?? "",
952
+ role: row.role ?? "",
938
953
  quirkType: row.quirkType ?? "",
939
954
  tags: row.tags ?? "",
940
955
  })),
@@ -963,6 +978,7 @@ export class LanceDbStore {
963
978
  content: row.content,
964
979
  description: row.description ?? "",
965
980
  kind: row.kind ?? "",
981
+ role: row.role ?? "",
966
982
  quirkType: row.quirkType ?? "",
967
983
  tags: row.tags ?? "",
968
984
  };
@@ -989,6 +1005,7 @@ export class LanceDbStore {
989
1005
  content: row.content,
990
1006
  description: row.description ?? "",
991
1007
  kind: row.kind ?? "",
1008
+ role: row.role ?? "",
992
1009
  quirkType: row.quirkType ?? "",
993
1010
  tags: row.tags ?? "",
994
1011
  }));
@@ -1115,7 +1132,9 @@ export class LanceDbStore {
1115
1132
  }
1116
1133
  /**
1117
1134
  * Perform a single index-metric repair pass. Skips stores that have no
1118
- * index and fewer than 1000 rows (brute-force is optimal there). Uses a
1135
+ * index and fewer than `MIN_ROWS_FOR_IVF_INDEX` rows (brute-force is
1136
+ * optimal there, and an IVF index would only trip Lance's "dataset is too
1137
+ * small" KMeans warning). Uses a
1119
1138
  * single `createIndex` with `replace: true` — a dropIndex + createIndex
1120
1139
  * sequence races in LanceDB ("Retryable commit conflict") and leaves the
1121
1140
  * stale index in place. On failure the memo is cleared so the next
@@ -1148,8 +1167,8 @@ export class LanceDbStore {
1148
1167
  const stats = await table.indexStats(idxName);
1149
1168
  if (stats && stats.distanceType === "cosine")
1150
1169
  return; // already healthy
1151
- if (!stats && count < 1000)
1152
- return; // tiny store, no index — leave brute-force
1170
+ if (!stats && count < MIN_ROWS_FOR_IVF_INDEX)
1171
+ return; // small store — brute-force is optimal, and IVF would only trip Lance's "too small for a meaningful index" KMeans warning
1153
1172
  const numPartitions = Math.max(16, Math.min(256, Math.floor(count / 256)));
1154
1173
  for (let attempt = 1; attempt <= 3; attempt++) {
1155
1174
  try {
@@ -70,6 +70,7 @@ export class InMemoryVectorStore {
70
70
  content: c.content,
71
71
  description: c.description ?? "",
72
72
  kind: c.metadata.kind ?? "",
73
+ role: c.metadata.role ?? "",
73
74
  quirkType: c.metadata.quirkType ?? "",
74
75
  tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
75
76
  }));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-rag-plugin",
3
- "version": "1.23.1",
3
+ "version": "1.23.3",
4
4
  "description": "OpenCode plugin for local-first RAG-based semantic code search",
5
5
  "type": "module",
6
6
  "main": "./dist/plugin-entry.js",