@javierkaiser/opencode-indexing 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -1
- package/README.md +167 -6
- package/index.ts +78 -4
- package/install.sh +0 -0
- package/package.json +1 -1
- package/src/config.ts +31 -5
- package/src/embedder.ts +339 -99
- package/src/indexer.ts +12 -1
- package/src/lancedb-store.ts +2 -3
- package/src/lancedb-support.ts +83 -0
- package/src/registry.ts +105 -19
- package/src/rpc.ts +1 -2
- package/src/search.ts +41 -2
- package/src/settings-commands.ts +6 -4
- package/src/settings.ts +28 -13
- package/src/status.ts +53 -5
- package/src/types.ts +41 -0
- package/tui/settings-page-data.ts +7 -8
- package/tui.ts +50 -9
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { settingsFilePath } from "./settings.ts"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Which platforms the LanceDB binding can actually run on.
|
|
5
|
+
*
|
|
6
|
+
* `@lancedb/lancedb` is a JavaScript wrapper that `require()`s one of seven
|
|
7
|
+
* per-platform native binaries, each published as its own optional dependency.
|
|
8
|
+
* There is no WASM build, so a platform outside that list cannot use LanceDB at
|
|
9
|
+
* all, and installing it again will not change that.
|
|
10
|
+
*
|
|
11
|
+
* macOS on Intel is the one that bites: `lancedb-darwin-x64` is not published,
|
|
12
|
+
* so LanceDB works on Apple Silicon only. The list is here so the plugin can say
|
|
13
|
+
* so plainly instead of surfacing a module-resolution stack trace; the import
|
|
14
|
+
* itself stays the authority on whether the binary loaded.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export const LANCEDB_VERSION = "0.26.2"
|
|
18
|
+
|
|
19
|
+
/** `platform-arch`, the way a native package would be named. */
|
|
20
|
+
export function platformId(platform: string = process.platform, arch: string = process.arch): string {
|
|
21
|
+
return `${platform}-${arch}`
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const SUPPORTED = new Set([
|
|
25
|
+
"win32-x64",
|
|
26
|
+
"win32-arm64",
|
|
27
|
+
"darwin-arm64",
|
|
28
|
+
"linux-x64",
|
|
29
|
+
"linux-arm64",
|
|
30
|
+
])
|
|
31
|
+
|
|
32
|
+
export interface LanceDbSupport {
|
|
33
|
+
/** `platform-arch` of the current process. */
|
|
34
|
+
id: string
|
|
35
|
+
/** Whether a native binary is published for it. */
|
|
36
|
+
supported: boolean
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function lanceDbSupport(
|
|
40
|
+
platform: string = process.platform,
|
|
41
|
+
arch: string = process.arch,
|
|
42
|
+
): LanceDbSupport {
|
|
43
|
+
const id = platformId(platform, arch)
|
|
44
|
+
return { id, supported: SUPPORTED.has(id) }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Node appends the resolution chain to a MODULE_NOT_FOUND message, which buries
|
|
49
|
+
* the one line that says what went wrong under seven lines of internal files.
|
|
50
|
+
* The cause is still there; it is just what a reader acts on that is kept.
|
|
51
|
+
*/
|
|
52
|
+
function causeSummary(cause: unknown): string {
|
|
53
|
+
const message = cause instanceof Error ? cause.message : String(cause)
|
|
54
|
+
const atRequireStack = message.indexOf("Require stack:")
|
|
55
|
+
return (atRequireStack === -1 ? message : message.slice(0, atRequireStack)).trim()
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The message shown when LanceDB cannot be loaded.
|
|
60
|
+
*
|
|
61
|
+
* The two cases need different advice, so they get different words: a platform
|
|
62
|
+
* with no binary cannot be fixed by installing anything, while a supported
|
|
63
|
+
* platform that failed to load usually means the optional dependency was
|
|
64
|
+
* skipped.
|
|
65
|
+
*/
|
|
66
|
+
export function lanceDbUnavailableMessage(
|
|
67
|
+
cause: unknown,
|
|
68
|
+
support: LanceDbSupport = lanceDbSupport(),
|
|
69
|
+
): string {
|
|
70
|
+
const { id, supported } = support
|
|
71
|
+
if (!supported) {
|
|
72
|
+
return (
|
|
73
|
+
`LanceDB is the default store for the own index but has no native binary for ${id}. ` +
|
|
74
|
+
`It cannot run on this platform and reinstalling will not change that. ` +
|
|
75
|
+
`Set vectorStore=qdrant explicitly in ${settingsFilePath()} to use a Qdrant server instead.`
|
|
76
|
+
)
|
|
77
|
+
}
|
|
78
|
+
return (
|
|
79
|
+
`LanceDB could not be loaded for ${id}. It ships as an optional dependency: ` +
|
|
80
|
+
`install @lancedb/lancedb@${LANCEDB_VERSION}, or set vectorStore=qdrant. ` +
|
|
81
|
+
`Cause: ${causeSummary(cause)}`
|
|
82
|
+
)
|
|
83
|
+
}
|
package/src/registry.ts
CHANGED
|
@@ -4,42 +4,71 @@ import { createHash } from "node:crypto"
|
|
|
4
4
|
* Embedding model catalog, mirrored from Kilo Code / Roo Code
|
|
5
5
|
* (`packages/kilo-indexing/src/indexing/model-registry.ts`) so that query
|
|
6
6
|
* embeddings are compatible with existing Kilo indexes.
|
|
7
|
+
*
|
|
8
|
+
* The mirror is the compatibility contract: an import is accepted only when
|
|
9
|
+
* provider, model id and dimension match byte for byte, so an entry here that
|
|
10
|
+
* drifts from Kilo's silently makes an index un-importable. Entries marked
|
|
11
|
+
* `kiloMirror: false` are ours alone and can never be imported.
|
|
7
12
|
*/
|
|
8
13
|
|
|
9
14
|
import type { EmbedderProvider, ModelProfile } from "./types.ts"
|
|
10
15
|
|
|
16
|
+
/** Input ceilings and batch limits are from each provider's own documentation. */
|
|
11
17
|
const profiles: Partial<Record<EmbedderProvider, Record<string, ModelProfile>>> = {
|
|
12
18
|
openai: {
|
|
13
|
-
"text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4 },
|
|
14
|
-
"text-embedding-3-large": { dimension: 3072, scoreThreshold: 0.4 },
|
|
15
|
-
"text-embedding-ada-002": { dimension: 1536, scoreThreshold: 0.4 },
|
|
19
|
+
"text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4, maxInputTokens: 8192, maxBatchItems: 2048, maxBatchTokens: 300_000 },
|
|
20
|
+
"text-embedding-3-large": { dimension: 3072, scoreThreshold: 0.4, maxInputTokens: 8192, maxBatchItems: 2048, maxBatchTokens: 300_000 },
|
|
21
|
+
"text-embedding-ada-002": { dimension: 1536, scoreThreshold: 0.4, maxInputTokens: 8192, maxBatchItems: 2048, maxBatchTokens: 300_000 },
|
|
16
22
|
},
|
|
17
23
|
ollama: {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
"
|
|
24
|
+
// 512 tokens is the smallest input in the catalog by a wide margin. A chunk of
|
|
25
|
+
// real code is routinely longer, and Ollama truncates by default, so this one
|
|
26
|
+
// is also the reason the Ollama adapter refuses to truncate silently.
|
|
27
|
+
"mxbai-embed-large": { dimension: 1024, scoreThreshold: 0.3, maxInputTokens: 512, normalized: true },
|
|
28
|
+
"nomic-embed-text": { dimension: 768, scoreThreshold: 0.3, queryPrefix: "search_query: ", maxInputTokens: 2048, normalized: true },
|
|
29
|
+
// Ollama reports the tag, but Kilo stores whatever its config names, so an
|
|
30
|
+
// index carrying this id was written by us and cannot be imported.
|
|
31
|
+
"nomic-embed-text:latest": { dimension: 768, scoreThreshold: 0.3, queryPrefix: "search_query: ", maxInputTokens: 2048, normalized: true, kiloMirror: false },
|
|
32
|
+
"all-minilm": { dimension: 384, scoreThreshold: 0.3, maxInputTokens: 512, normalized: true },
|
|
33
|
+
"bge-m3": { dimension: 1024, scoreThreshold: 0.3, maxInputTokens: 8192, normalized: true, kiloMirror: false },
|
|
22
34
|
},
|
|
23
35
|
gemini: {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
"embedding-001": { dimension:
|
|
36
|
+
// gemini-embedding-001 accepts only 2048 tokens, the tightest cloud ceiling here.
|
|
37
|
+
// gemini-embedding-2 takes 8192 and is the better default despite costing more.
|
|
38
|
+
"gemini-embedding-001": { dimension: 3072, scoreThreshold: 0.35, maxInputTokens: 2048 },
|
|
39
|
+
"gemini-embedding-2": { dimension: 3072, scoreThreshold: 0.35, maxInputTokens: 8192, normalized: true, kiloMirror: false },
|
|
40
|
+
"text-embedding-004": { dimension: 768, scoreThreshold: 0.35, maxInputTokens: 2048 },
|
|
41
|
+
"embedding-001": { dimension: 768, scoreThreshold: 0.35, maxInputTokens: 2048 },
|
|
27
42
|
},
|
|
28
43
|
mistral: {
|
|
29
|
-
"codestral-embed-2505": { dimension: 1536, scoreThreshold: 0.35 },
|
|
30
|
-
"codestral-embed": { dimension: 1536, scoreThreshold: 0.35 },
|
|
31
|
-
"mistral-embed": { dimension: 1024, scoreThreshold: 0.35 },
|
|
44
|
+
"codestral-embed-2505": { dimension: 1536, scoreThreshold: 0.35, maxInputTokens: 8192, normalized: true },
|
|
45
|
+
"codestral-embed": { dimension: 1536, scoreThreshold: 0.35, maxInputTokens: 8192, normalized: true },
|
|
46
|
+
"mistral-embed": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 8192, normalized: true },
|
|
32
47
|
},
|
|
33
48
|
voyage: {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
"voyage-
|
|
49
|
+
// 32k of context on a code-specific model: four times any other entry here, and
|
|
50
|
+
// the reason voyage is a better default than codestral for code chunks.
|
|
51
|
+
"voyage-code-4": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, maxBatchTokens: 120_000, normalized: true, kiloMirror: false },
|
|
52
|
+
"voyage-code-3": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, maxBatchTokens: 120_000, normalized: true },
|
|
53
|
+
"voyage-4-large": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, maxBatchTokens: 120_000, normalized: true, kiloMirror: false },
|
|
54
|
+
"voyage-4": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, maxBatchTokens: 320_000, normalized: true, kiloMirror: false },
|
|
55
|
+
"voyage-4-lite": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, maxBatchTokens: 1_000_000, normalized: true, kiloMirror: false },
|
|
56
|
+
"voyage-3": { dimension: 1024, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, normalized: true },
|
|
57
|
+
"voyage-3-lite": { dimension: 512, scoreThreshold: 0.35, maxInputTokens: 32_000, maxBatchItems: 1000, normalized: true },
|
|
37
58
|
},
|
|
38
59
|
openrouter: {
|
|
39
|
-
"openai/text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4 },
|
|
40
|
-
"openai/text-embedding-3-large": { dimension: 3072, scoreThreshold: 0.4 },
|
|
41
|
-
|
|
60
|
+
"openai/text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4, maxInputTokens: 8192, maxBatchTokens: 300_000 },
|
|
61
|
+
"openai/text-embedding-3-large": { dimension: 3072, scoreThreshold: 0.4, maxInputTokens: 8192, maxBatchTokens: 300_000 },
|
|
62
|
+
// Kilo's registry names this `google/gemini-embedding-2-preview`. Ours said
|
|
63
|
+
// `google/gemini-embedding-001`, which can never match a Kilo index: the
|
|
64
|
+
// profile comparison is exact string equality, so the drift was invisible and
|
|
65
|
+
// simply made every Gemini-via-OpenRouter index un-importable.
|
|
66
|
+
"google/gemini-embedding-2-preview": { dimension: 3072, scoreThreshold: 0.35, maxInputTokens: 8192 },
|
|
42
67
|
},
|
|
68
|
+
// Kilo's registry also lists `bedrock` and `vercel-ai-gateway`. Both are
|
|
69
|
+
// deliberately absent: bedrock needs SigV4 request signing, and neither is a
|
|
70
|
+
// provider this plugin offers. A catalog entry with no implemented adapter is
|
|
71
|
+
// worse than a missing model, because it looks selectable and then fails.
|
|
43
72
|
"openai-compatible": {},
|
|
44
73
|
}
|
|
45
74
|
|
|
@@ -49,6 +78,7 @@ const defaults: Record<string, string> = {
|
|
|
49
78
|
gemini: "gemini-embedding-001",
|
|
50
79
|
mistral: "codestral-embed-2505",
|
|
51
80
|
voyage: "voyage-code-3",
|
|
81
|
+
bedrock: "amazon.titan-embed-text-v2:0",
|
|
52
82
|
openrouter: "openai/text-embedding-3-small",
|
|
53
83
|
"openai-compatible": "",
|
|
54
84
|
}
|
|
@@ -59,6 +89,15 @@ export const DEFAULT_EMBEDDING_BATCH_SIZE = 60
|
|
|
59
89
|
export const DEFAULT_MAX_FILE_SIZE_BYTES = 1024 * 1024
|
|
60
90
|
export const MAX_SEARCH_RESULTS = 200
|
|
61
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Ceiling used when a model is not in the catalog.
|
|
94
|
+
*
|
|
95
|
+
* `mxbai-embed-large` takes 512 tokens, and Ollama truncates anything longer
|
|
96
|
+
* without saying so. For an unknown model the safe assumption is the tightest
|
|
97
|
+
* documented one: a shorter index beat a silently wrong index.
|
|
98
|
+
*/
|
|
99
|
+
export const FALLBACK_MAX_INPUT_TOKENS = 512
|
|
100
|
+
|
|
62
101
|
/** Scanner extension allowlist mirrored from Kilo's tree-sitter index. */
|
|
63
102
|
export const DEFAULT_SCANNER_EXTENSIONS: readonly string[] = [
|
|
64
103
|
".bash",
|
|
@@ -145,6 +184,41 @@ export function getDefaultModelId(provider: EmbedderProvider): string {
|
|
|
145
184
|
return defaults[provider] ?? ""
|
|
146
185
|
}
|
|
147
186
|
|
|
187
|
+
/**
|
|
188
|
+
* Providers whose adapter has been exercised against the real service.
|
|
189
|
+
*
|
|
190
|
+
* Not a quality judgement, a statement of fact. The others were written from
|
|
191
|
+
* each provider's documentation and tested against fixtures written alongside
|
|
192
|
+
* them, so a wrong assumption about a field name or a response shape would pass
|
|
193
|
+
* every test and fail on first use. Surfacing that where a provider is chosen is
|
|
194
|
+
* cheaper than letting a user discover it from a failed indexing run.
|
|
195
|
+
*/
|
|
196
|
+
export const LIVE_TESTED_PROVIDERS: ReadonlySet<string> = new Set(["mistral"])
|
|
197
|
+
|
|
198
|
+
export function isLiveTested(provider: EmbedderProvider): boolean {
|
|
199
|
+
return LIVE_TESTED_PROVIDERS.has(provider)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Every model id the catalog knows for a provider, in catalog order.
|
|
204
|
+
*/
|
|
205
|
+
export function listModelIds(provider: EmbedderProvider): string[] {
|
|
206
|
+
return Object.keys(profiles[provider] ?? {})
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Whether an index built with this profile could ever be imported from Kilo.
|
|
211
|
+
*
|
|
212
|
+
* `undefined` means the profile is unknown to us, which is a different thing
|
|
213
|
+
* from "not importable": a model we have never heard of may still be exactly
|
|
214
|
+
* what a Kilo index holds.
|
|
215
|
+
*/
|
|
216
|
+
export function isKiloMirrored(provider: EmbedderProvider, modelId: string): boolean | undefined {
|
|
217
|
+
const profile = getModelProfile(provider, modelId)
|
|
218
|
+
if (!profile) return undefined
|
|
219
|
+
return profile.kiloMirror !== false
|
|
220
|
+
}
|
|
221
|
+
|
|
148
222
|
export function getModelProfile(provider: EmbedderProvider, modelId: string): ModelProfile | undefined {
|
|
149
223
|
return profiles[provider]?.[modelId]
|
|
150
224
|
}
|
|
@@ -161,6 +235,18 @@ export function getModelQueryPrefix(provider: EmbedderProvider, modelId: string)
|
|
|
161
235
|
return getModelProfile(provider, modelId)?.queryPrefix
|
|
162
236
|
}
|
|
163
237
|
|
|
238
|
+
/**
|
|
239
|
+
* Largest input the model accepts, in tokens.
|
|
240
|
+
*
|
|
241
|
+
* Falls back to the most conservative ceiling any catalog model has, rather than
|
|
242
|
+
* the most generous: guessing high means sending a 32,000-character chunk to a
|
|
243
|
+
* model that accepts 512, which comes back silently truncated and gets indexed
|
|
244
|
+
* as if it covered the whole function.
|
|
245
|
+
*/
|
|
246
|
+
export function getModelMaxInputTokens(provider: EmbedderProvider, modelId: string): number {
|
|
247
|
+
return getModelProfile(provider, modelId)?.maxInputTokens ?? FALLBACK_MAX_INPUT_TOKENS
|
|
248
|
+
}
|
|
249
|
+
|
|
164
250
|
/** sha256 hex of a workspace path (no normalization, mirrors Kilo). */
|
|
165
251
|
export function workspaceHash(root: string): string {
|
|
166
252
|
return createHash("sha256").update(root).digest("hex")
|
package/src/rpc.ts
CHANGED
|
@@ -32,8 +32,7 @@ const settingsShape = {
|
|
|
32
32
|
importFromKilo: { type: "boolean" },
|
|
33
33
|
autoRefresh: { type: "boolean" },
|
|
34
34
|
searchMaxResults: { type: "number" },
|
|
35
|
-
|
|
36
|
-
hasOpenAiKey: { type: "boolean" },
|
|
35
|
+
configuredProviders: { type: "array", items: { type: "string" } },
|
|
37
36
|
settingsFile: { type: "string" },
|
|
38
37
|
},
|
|
39
38
|
required: ["vectorStore", "qdrantUrl", "provider", "importFromKilo", "settingsFile"],
|
package/src/search.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { getModelScoreThreshold, profileKey, workspaceHash } from "./registry.ts
|
|
|
8
8
|
import { withProfile } from "./config.ts"
|
|
9
9
|
import type { EmbedderProvider, IndexingSettings, KVStore, QueryHit, SearchHit } from "./types.ts"
|
|
10
10
|
import type { Qdrant } from "./qdrant.ts"
|
|
11
|
+
import type { VectorStoreAdapter } from "./vector-store.ts"
|
|
11
12
|
import { describeError } from "./errors.ts"
|
|
12
13
|
|
|
13
14
|
export interface SearchDeps {
|
|
@@ -15,6 +16,13 @@ export interface SearchDeps {
|
|
|
15
16
|
kv: KVStore
|
|
16
17
|
root: string
|
|
17
18
|
settings: IndexingSettings
|
|
19
|
+
/**
|
|
20
|
+
* The own-index store. Built from the settings when omitted.
|
|
21
|
+
*
|
|
22
|
+
* Injected so a caller that already holds one does not build a second
|
|
23
|
+
* identical client, and so the search can be exercised without a server.
|
|
24
|
+
*/
|
|
25
|
+
ownStore?: VectorStoreAdapter
|
|
18
26
|
}
|
|
19
27
|
|
|
20
28
|
export interface SearchMeta {
|
|
@@ -80,7 +88,7 @@ export async function searchCode(deps: SearchDeps, query: string, pathPrefix?: s
|
|
|
80
88
|
const errors: string[] = []
|
|
81
89
|
const limit = Math.max(1, Math.min(overrideLimit ?? settings.searchMaxResults, 200))
|
|
82
90
|
|
|
83
|
-
const ownStore = createOwnStore(settings, root)
|
|
91
|
+
const ownStore = deps.ownStore ?? createOwnStore(settings, root)
|
|
84
92
|
const qdrantClient = settings.vectorStore === "qdrant" ? qdrant : createSettingsQdrant(settings)
|
|
85
93
|
const [ownInfo, kiloCollection] = await Promise.all([
|
|
86
94
|
ownStore.info().catch(() => null),
|
|
@@ -102,6 +110,7 @@ export async function searchCode(deps: SearchDeps, query: string, pathPrefix?: s
|
|
|
102
110
|
if (!profiles.has(profileKey(baseProfile))) profiles.set(profileKey(baseProfile), baseProfile)
|
|
103
111
|
|
|
104
112
|
const vectors = new Map<string, number[]>()
|
|
113
|
+
const embedFailures = new Map<string, string>()
|
|
105
114
|
for (const [key, profile] of profiles) {
|
|
106
115
|
try {
|
|
107
116
|
const embedder = createEmbedder(withProfile(settings, profile))
|
|
@@ -109,10 +118,32 @@ export async function searchCode(deps: SearchDeps, query: string, pathPrefix?: s
|
|
|
109
118
|
if (!vector) throw new Error("empty embedding")
|
|
110
119
|
vectors.set(key, vector)
|
|
111
120
|
} catch (error) {
|
|
112
|
-
|
|
121
|
+
embedFailures.set(key, describeError(error))
|
|
113
122
|
}
|
|
114
123
|
}
|
|
115
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Refuse to answer from part of the corpus.
|
|
127
|
+
*
|
|
128
|
+
* A stored index whose profile cannot be embedded is unreachable, and
|
|
129
|
+
* returning only the half that worked looks exactly like a complete answer
|
|
130
|
+
* whose other half ranked lower. The caller asked what matches, not what
|
|
131
|
+
* matches in whichever part happened to be readable, so this is an error.
|
|
132
|
+
* An index with no points needs no query vector and is not a problem.
|
|
133
|
+
*/
|
|
134
|
+
const unreachable = (label: string, profile: ProfileRef | undefined, points: number | null | undefined): void => {
|
|
135
|
+
if (!profile || (points ?? 0) <= 0) return
|
|
136
|
+
const key = profileKey(profile)
|
|
137
|
+
if (vectors.has(key)) return
|
|
138
|
+
const cause = embedFailures.get(key)
|
|
139
|
+
throw new Error(
|
|
140
|
+
`Cannot search ${label}: it holds ${points} points built with ${key}, and that profile could not be embedded` +
|
|
141
|
+
`${cause ? ` (${cause})` : ""}. Searching only the other index would return an incomplete result set as if it were complete.`,
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
unreachable("the own index", ownInfo?.profile, ownInfo?.pointsCount)
|
|
145
|
+
unreachable("Kilo's index", kiloInfo?.profile, kiloInfo?.pointsCount)
|
|
146
|
+
|
|
116
147
|
const minScore = settings.searchMinScore
|
|
117
148
|
|
|
118
149
|
const kiloHits = (async (): Promise<SearchHit[]> => {
|
|
@@ -129,6 +160,11 @@ export async function searchCode(deps: SearchDeps, query: string, pathPrefix?: s
|
|
|
129
160
|
})
|
|
130
161
|
return hits.map((hit) => toSearchHit(hit, "kilo")).filter((hit): hit is SearchHit => hit !== undefined)
|
|
131
162
|
})().catch((error) => {
|
|
163
|
+
// Kilo's index exists and holds points; failing to read it would drop those
|
|
164
|
+
// results from an answer that still looks complete.
|
|
165
|
+
if ((kiloInfo?.pointsCount ?? 0) > 0) {
|
|
166
|
+
throw new Error(`Cannot read Kilo's index (${kiloCollection}): ${describeError(error)}`)
|
|
167
|
+
}
|
|
132
168
|
errors.push(`kilo search: ${describeError(error)}`)
|
|
133
169
|
return [] as SearchHit[]
|
|
134
170
|
})
|
|
@@ -147,6 +183,9 @@ export async function searchCode(deps: SearchDeps, query: string, pathPrefix?: s
|
|
|
147
183
|
})
|
|
148
184
|
return hits.map((hit) => toSearchHit(hit, "own")).filter((hit): hit is SearchHit => hit !== undefined)
|
|
149
185
|
})().catch((error) => {
|
|
186
|
+
if ((ownInfo?.pointsCount ?? 0) > 0) {
|
|
187
|
+
throw new Error(`Cannot read the own index (${ownStore.name}): ${describeError(error)}`)
|
|
188
|
+
}
|
|
150
189
|
errors.push(`own search: ${describeError(error)}`)
|
|
151
190
|
return [] as SearchHit[]
|
|
152
191
|
})
|
package/src/settings-commands.ts
CHANGED
|
@@ -172,10 +172,12 @@ const KEY_SPECS: Record<string, KeySpec> = {
|
|
|
172
172
|
description: "Embedding provider",
|
|
173
173
|
toPatch: (value) => {
|
|
174
174
|
const provider = value as (typeof PROVIDERS)[number]
|
|
175
|
-
//
|
|
176
|
-
// one
|
|
177
|
-
|
|
178
|
-
|
|
175
|
+
// Clear the model so the new provider's default applies. Model ids are
|
|
176
|
+
// namespaced per provider, so keeping the old one is never right: it
|
|
177
|
+
// turned a switch to Voyage into `voyage` + `codestral-embed-2505`. The
|
|
178
|
+
// previous code claimed to do this and returned only the provider, which
|
|
179
|
+
// left the stale model in the file and outranked the default.
|
|
180
|
+
return { provider: provider as SettingsFile["provider"], model: "" }
|
|
179
181
|
},
|
|
180
182
|
display: (settings) => settings.provider,
|
|
181
183
|
},
|
package/src/settings.ts
CHANGED
|
@@ -93,6 +93,18 @@ function compact<T extends Record<string, unknown>>(input: T): T {
|
|
|
93
93
|
return out as T
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/**
|
|
97
|
+
* Merge the two objects a level deep.
|
|
98
|
+
*
|
|
99
|
+
* Derived from the incoming keys rather than listing the provider names here.
|
|
100
|
+
* The hand-written list this replaced silently dropped any credential added to
|
|
101
|
+
* the shape but not to the list: the field existed, the resolver read it, and
|
|
102
|
+
* the value never reached the file.
|
|
103
|
+
*/
|
|
104
|
+
function mergeNested<T extends Record<string, unknown>>(current: T | undefined, incoming: T): T {
|
|
105
|
+
return compact({ ...(current ?? {}), ...incoming }) as T
|
|
106
|
+
}
|
|
107
|
+
|
|
96
108
|
function deepMerge(base: SettingsFile, patch: SettingsFile): SettingsFile {
|
|
97
109
|
const out: SettingsFile = { ...base }
|
|
98
110
|
for (const [key, value] of Object.entries(patch)) {
|
|
@@ -108,19 +120,22 @@ function deepMerge(base: SettingsFile, patch: SettingsFile): SettingsFile {
|
|
|
108
120
|
if (key === "apiKeys") {
|
|
109
121
|
const current = out.apiKeys ?? {}
|
|
110
122
|
const incoming = value as NonNullable<SettingsFile["apiKeys"]>
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
123
|
+
const merged: Record<string, unknown> = { ...current }
|
|
124
|
+
for (const [provider, providerValue] of Object.entries(incoming)) {
|
|
125
|
+
if (providerValue === undefined) continue
|
|
126
|
+
// ollama and openai-compatible are objects; the rest are plain keys.
|
|
127
|
+
// Both shapes merge the same way, so no per-provider branch is needed.
|
|
128
|
+
merged[provider] =
|
|
129
|
+
typeof providerValue === "object" && providerValue !== null
|
|
130
|
+
? mergeNested(
|
|
131
|
+
typeof merged[provider] === "object" && merged[provider] !== null
|
|
132
|
+
? (merged[provider] as Record<string, unknown>)
|
|
133
|
+
: undefined,
|
|
134
|
+
providerValue as Record<string, unknown>,
|
|
135
|
+
)
|
|
136
|
+
: providerValue
|
|
137
|
+
}
|
|
138
|
+
out.apiKeys = compact(merged) as SettingsFile["apiKeys"]
|
|
124
139
|
continue
|
|
125
140
|
}
|
|
126
141
|
;(out as Record<string, unknown>)[key] = value
|
package/src/status.ts
CHANGED
|
@@ -5,10 +5,11 @@ import { getKiloStoreInfo, resolveKiloCollection } from "./kilo-store.ts"
|
|
|
5
5
|
import { discoverKiloSources, type KiloSourceInfo } from "./import.ts"
|
|
6
6
|
import { createOwnStore, createSettingsQdrant } from "./store-factory.ts"
|
|
7
7
|
import { diffFiles, loadManifest } from "./manifest.ts"
|
|
8
|
-
import { workspaceHash } from "./registry.ts"
|
|
8
|
+
import { getModelDimension, profileKey, workspaceHash } from "./registry.ts"
|
|
9
9
|
import { scanWorkspace } from "./scanner.ts"
|
|
10
10
|
import type { IndexingSettings, KVStore } from "./types.ts"
|
|
11
11
|
import type { Qdrant } from "./qdrant.ts"
|
|
12
|
+
import type { VectorStoreAdapter } from "./vector-store.ts"
|
|
12
13
|
import { describeError } from "./errors.ts"
|
|
13
14
|
|
|
14
15
|
export interface StatusDeps {
|
|
@@ -16,6 +17,8 @@ export interface StatusDeps {
|
|
|
16
17
|
kv: KVStore
|
|
17
18
|
root: string
|
|
18
19
|
settings: IndexingSettings
|
|
20
|
+
/** The own-index store. Built from the settings when omitted. */
|
|
21
|
+
ownStore?: VectorStoreAdapter
|
|
19
22
|
}
|
|
20
23
|
|
|
21
24
|
export interface IndexStatus {
|
|
@@ -56,16 +59,29 @@ const KILO_CACHE_PREFIX = "kiloCollection/"
|
|
|
56
59
|
|
|
57
60
|
export async function getStatus(deps: StatusDeps, options?: { checkFreshness?: boolean }): Promise<IndexStatus> {
|
|
58
61
|
const { qdrant, kv, root, settings } = deps
|
|
59
|
-
|
|
62
|
+
// Configuration warnings are produced once per resolution and were previously
|
|
63
|
+
// never shown, so a store that cannot work on this platform looked simply
|
|
64
|
+
// absent rather than misconfigured.
|
|
65
|
+
const warnings: string[] = [...settings.warnings]
|
|
60
66
|
let reachable = true
|
|
61
67
|
try {
|
|
62
68
|
await qdrant.listCollections()
|
|
63
69
|
} catch (error) {
|
|
64
70
|
reachable = false
|
|
65
|
-
|
|
71
|
+
// Only worth reporting when Qdrant is actually part of this setup. With the
|
|
72
|
+
// own index on LanceDB and no Qdrant-backed Kilo index, a dead Qdrant is
|
|
73
|
+
// not a problem the user has, and saying so would train them to ignore it.
|
|
74
|
+
const qdrantIsOurs = settings.vectorStore === "qdrant" || settings.kiloVectorStore === "qdrant"
|
|
75
|
+
if (qdrantIsOurs) {
|
|
76
|
+
warnings.push(
|
|
77
|
+
settings.vectorStore === "qdrant"
|
|
78
|
+
? `Qdrant unreachable at ${settings.qdrantUrl}: ${describeError(error)}`
|
|
79
|
+
: `Qdrant unreachable at ${settings.qdrantUrl}, so a Kilo index stored there cannot be imported: ${describeError(error)}`,
|
|
80
|
+
)
|
|
81
|
+
}
|
|
66
82
|
}
|
|
67
83
|
|
|
68
|
-
const ownStore = createOwnStore(settings, root)
|
|
84
|
+
const ownStore = deps.ownStore ?? createOwnStore(settings, root)
|
|
69
85
|
const cacheKey = KILO_CACHE_PREFIX + workspaceHash(root).slice(0, 16)
|
|
70
86
|
const cachedKilo = await kv.get(cacheKey)
|
|
71
87
|
const kiloCollection = reachable
|
|
@@ -110,8 +126,40 @@ export async function getStatus(deps: StatusDeps, options?: { checkFreshness?: b
|
|
|
110
126
|
: null
|
|
111
127
|
|
|
112
128
|
const compatibleSource = kiloSources.find((source) => source.compatible)
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* A profile change destroys the own index and re-embeds everything.
|
|
132
|
+
*
|
|
133
|
+
* Saying so here is the point: the rebuild happens on the next build or
|
|
134
|
+
* refresh, so a user who changes provider to save money can spend the whole
|
|
135
|
+
* repository on embeddings before seeing any indication that it happened.
|
|
136
|
+
* The count is reported because it is the only estimate of the cost.
|
|
137
|
+
*/
|
|
138
|
+
const configuredProfile = {
|
|
139
|
+
provider: settings.provider,
|
|
140
|
+
modelId: settings.modelId,
|
|
141
|
+
dimension: settings.dimension || getModelDimension(settings.provider, settings.modelId) || 0,
|
|
142
|
+
}
|
|
143
|
+
const storedProfile = ownInfo?.profile
|
|
144
|
+
const profileChanged =
|
|
145
|
+
storedProfile !== undefined &&
|
|
146
|
+
(storedProfile.provider !== configuredProfile.provider ||
|
|
147
|
+
storedProfile.modelId !== configuredProfile.modelId ||
|
|
148
|
+
(configuredProfile.dimension > 0 && storedProfile.dimension !== configuredProfile.dimension))
|
|
149
|
+
if (profileChanged && (ownInfo?.pointsCount ?? 0) > 0) {
|
|
150
|
+
warnings.push(
|
|
151
|
+
`The own index holds ${ownInfo?.pointsCount} points built with ${profileKey(storedProfile)} but the configured ` +
|
|
152
|
+
`model is ${profileKey(configuredProfile)}. The next build or refresh will delete them and re-embed the whole ` +
|
|
153
|
+
`workspace. Set the model back to reuse them, or re-import from Kilo if a matching source exists.`,
|
|
154
|
+
)
|
|
155
|
+
}
|
|
156
|
+
|
|
113
157
|
let recommendation: string
|
|
114
|
-
if (
|
|
158
|
+
if (profileChanged) {
|
|
159
|
+
recommendation =
|
|
160
|
+
`The embedding model changed since the own index was built (${profileKey(storedProfile!)} -> ` +
|
|
161
|
+
`${profileKey(configuredProfile)}). Run indexing_build to rebuild it, which re-embeds every file.`
|
|
162
|
+
} else if (!reachable && settings.vectorStore === "qdrant") {
|
|
115
163
|
recommendation = "Start Qdrant (e.g. `docker run -p 6333:6333 qdrant/qdrant` or Podman equivalent), then re-run indexing_status."
|
|
116
164
|
} else if (!kiloCollection && !ownInfo?.exists && kiloSources.length === 0) {
|
|
117
165
|
recommendation = "No index found. Run indexing_build to create an independent index (works without Kilo Code)."
|
package/src/types.ts
CHANGED
|
@@ -22,6 +22,38 @@ export interface ModelProfile {
|
|
|
22
22
|
dimension: number
|
|
23
23
|
scoreThreshold?: number
|
|
24
24
|
queryPrefix?: string
|
|
25
|
+
/**
|
|
26
|
+
* Largest input the model accepts, in tokens.
|
|
27
|
+
*
|
|
28
|
+
* Not the same for every model, and the spread is wide: `mxbai-embed-large`
|
|
29
|
+
* takes 512 while `voyage-code-4` takes 32,000. Sizing every request by one
|
|
30
|
+
* global constant silently truncates every chunk past the smallest model's
|
|
31
|
+
* limit, and that truncated vector is stored as if it covered the whole
|
|
32
|
+
* function.
|
|
33
|
+
*/
|
|
34
|
+
maxInputTokens?: number
|
|
35
|
+
/** Documented ceiling on items in one request, where the provider publishes one. */
|
|
36
|
+
maxBatchItems?: number
|
|
37
|
+
/** Documented ceiling on tokens in one request, where the provider publishes one. */
|
|
38
|
+
maxBatchTokens?: number
|
|
39
|
+
/**
|
|
40
|
+
* Whether the provider documents its output as already L2-normalised.
|
|
41
|
+
*
|
|
42
|
+
* The provider's claim, not a measurement we rely on. Mistral documents its
|
|
43
|
+
* embeddings as "norm 1" and a live call measured 0.9913, which is why every
|
|
44
|
+
* comparison here uses cosine regardless: it is correct either way, and this
|
|
45
|
+
* field exists only so a future fast path knows whether a bare dot product
|
|
46
|
+
* would be equivalent.
|
|
47
|
+
*/
|
|
48
|
+
normalized?: boolean
|
|
49
|
+
/**
|
|
50
|
+
* Marks a model we know about that Kilo's registry does not.
|
|
51
|
+
*
|
|
52
|
+
* The catalog is a mirror, so that query embeddings stay interchangeable with
|
|
53
|
+
* a Kilo index. An entry with no Kilo counterpart is still useful, but it can
|
|
54
|
+
* never be import-compatible, and the status report has to be able to say so.
|
|
55
|
+
*/
|
|
56
|
+
kiloMirror?: boolean
|
|
25
57
|
}
|
|
26
58
|
|
|
27
59
|
export interface ProviderCredentials {
|
|
@@ -39,6 +71,15 @@ export interface ProviderCredentials {
|
|
|
39
71
|
export interface IndexingSettings extends EmbeddingProfile {
|
|
40
72
|
/** Backend for the plugin's OWN index. */
|
|
41
73
|
vectorStore: "qdrant" | "lancedb"
|
|
74
|
+
/**
|
|
75
|
+
* Store Kilo keeps its own index in, when kilo.jsonc says so.
|
|
76
|
+
*
|
|
77
|
+
* Kept alongside `vectorStore` rather than folded into it: Kilo decides where
|
|
78
|
+
* an import reads from, the own index decides where it writes, and the status
|
|
79
|
+
* report needs both to know whether an unreachable Qdrant is a problem here or
|
|
80
|
+
* simply not this user's business.
|
|
81
|
+
*/
|
|
82
|
+
kiloVectorStore: "qdrant" | "lancedb" | null
|
|
42
83
|
qdrantUrl: string
|
|
43
84
|
qdrantApiKey?: string
|
|
44
85
|
/** Base directory for the plugin's own LanceDB stores. */
|
|
@@ -24,8 +24,8 @@ export interface SettingsView {
|
|
|
24
24
|
/** Master switch for writing the own index. */
|
|
25
25
|
enabled: boolean
|
|
26
26
|
searchMaxResults: number
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
/** Providers that have a usable credential or base URL configured. */
|
|
28
|
+
configuredProviders: string[]
|
|
29
29
|
settingsFile: string
|
|
30
30
|
}
|
|
31
31
|
|
|
@@ -101,8 +101,7 @@ export const UNKNOWN_SETTINGS: SettingsView = {
|
|
|
101
101
|
enabled: false,
|
|
102
102
|
autoRefresh: false,
|
|
103
103
|
searchMaxResults: 0,
|
|
104
|
-
|
|
105
|
-
hasOpenAiKey: false,
|
|
104
|
+
configuredProviders: [],
|
|
106
105
|
settingsFile: "(unavailable)",
|
|
107
106
|
}
|
|
108
107
|
|
|
@@ -149,10 +148,10 @@ export function configRows(settings: SettingsView): ConfigRow[] {
|
|
|
149
148
|
{ label: "Max results", value: String(settings.searchMaxResults) },
|
|
150
149
|
{ label: "Settings file", value: settings.settingsFile },
|
|
151
150
|
]
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
rows.push({ label: "
|
|
151
|
+
// Derived from the server's list rather than a pair of booleans, so a
|
|
152
|
+
// provider added anywhere else shows up here without a change to this file.
|
|
153
|
+
const keys = settings.configuredProviders
|
|
154
|
+
rows.push({ label: "Credentials", value: keys.length > 0 ? `${keys.join(", ")} configured` : "none configured" })
|
|
156
155
|
return rows
|
|
157
156
|
}
|
|
158
157
|
|