opencode-codebase-index 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +6 -5
- package/commands/peek.md +2 -1
- package/commands/search.md +2 -1
- package/dist/cli.cjs +1020 -260
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1020 -260
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +788 -256
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +788 -256
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +5440 -2157
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +5429 -2146
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +3 -1
- package/skill/SKILL.md +12 -16
package/dist/cli.js
CHANGED
|
@@ -716,6 +716,17 @@ var EMBEDDING_MODELS = {
|
|
|
716
716
|
maxTokens: 2048,
|
|
717
717
|
costPer1MTokens: 0.15,
|
|
718
718
|
taskAble: true
|
|
719
|
+
},
|
|
720
|
+
"gemini-embedding-2": {
|
|
721
|
+
provider: "google",
|
|
722
|
+
model: "gemini-embedding-2",
|
|
723
|
+
// Keep a conservative, testable default embedding dimension. Gemini Embedding 2 supports
|
|
724
|
+
// flexible dimensions via outputDimensionality.
|
|
725
|
+
dimensions: 1536,
|
|
726
|
+
maxTokens: 8192,
|
|
727
|
+
costPer1MTokens: 0.15,
|
|
728
|
+
taskAble: false,
|
|
729
|
+
promptStyle: "embedding-2"
|
|
719
730
|
}
|
|
720
731
|
},
|
|
721
732
|
"openai": {
|
|
@@ -749,26 +760,15 @@ var EMBEDDING_MODELS = {
|
|
|
749
760
|
maxTokens: 512,
|
|
750
761
|
costPer1MTokens: 0
|
|
751
762
|
}
|
|
752
|
-
},
|
|
753
|
-
"github-copilot": {
|
|
754
|
-
"text-embedding-3-small": {
|
|
755
|
-
provider: "github-copilot",
|
|
756
|
-
model: "text-embedding-3-small",
|
|
757
|
-
dimensions: 1536,
|
|
758
|
-
maxTokens: 8191,
|
|
759
|
-
costPer1MTokens: 0
|
|
760
|
-
}
|
|
761
763
|
}
|
|
762
764
|
};
|
|
763
765
|
var DEFAULT_PROVIDER_MODELS = {
|
|
764
|
-
"github-copilot": "text-embedding-3-small",
|
|
765
766
|
"openai": "text-embedding-3-small",
|
|
766
767
|
"google": "gemini-embedding-001",
|
|
767
768
|
"ollama": "nomic-embed-text"
|
|
768
769
|
};
|
|
769
770
|
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
770
771
|
"ollama",
|
|
771
|
-
"github-copilot",
|
|
772
772
|
"openai",
|
|
773
773
|
"google"
|
|
774
774
|
];
|
|
@@ -794,6 +794,9 @@ function getDefaultIndexingConfig() {
|
|
|
794
794
|
maxDepth: 5,
|
|
795
795
|
maxFilesPerDirectory: 100,
|
|
796
796
|
fallbackToTextOnMaxChunks: true,
|
|
797
|
+
// Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi
|
|
798
|
+
// fallback used when a native caller omits the argument).
|
|
799
|
+
linesPerChunk: 30,
|
|
797
800
|
gitBlame: { enabled: false }
|
|
798
801
|
};
|
|
799
802
|
}
|
|
@@ -927,6 +930,7 @@ function parseConfig(raw) {
|
|
|
927
930
|
maxDepth: typeof rawIndexing.maxDepth === "number" ? rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth : defaultIndexing.maxDepth,
|
|
928
931
|
maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
|
|
929
932
|
fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
|
|
933
|
+
linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk,
|
|
930
934
|
gitBlame: {
|
|
931
935
|
enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof rawIndexing.gitBlame.enabled === "boolean" ? rawIndexing.gitBlame.enabled : defaultIndexing.gitBlame.enabled
|
|
932
936
|
}
|
|
@@ -969,6 +973,7 @@ function parseConfig(raw) {
|
|
|
969
973
|
let embeddingModel;
|
|
970
974
|
let customProvider;
|
|
971
975
|
let reranker;
|
|
976
|
+
const githubCopilotDeprecationMessage = '`embeddingProvider: "github-copilot"` is deprecated and no longer available. Migrate existing configs to `embeddingProvider: "google"` and select an explicit Google model. For existing indexes, run `index_codebase` with `force: true` after changing to `gemini-embedding-001` or `gemini-embedding-2` to rebuild embeddings. See docs/configuration.md for details.';
|
|
972
977
|
if (embeddingProviderValue === "custom") {
|
|
973
978
|
embeddingProvider = "custom";
|
|
974
979
|
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
@@ -1008,6 +1013,8 @@ function parseConfig(raw) {
|
|
|
1008
1013
|
} else if (rawEmbeddingModel) {
|
|
1009
1014
|
embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
1010
1015
|
}
|
|
1016
|
+
} else if (embeddingProviderValue === "github-copilot") {
|
|
1017
|
+
throw new Error(githubCopilotDeprecationMessage);
|
|
1011
1018
|
} else {
|
|
1012
1019
|
embeddingProvider = "auto";
|
|
1013
1020
|
}
|
|
@@ -1038,10 +1045,21 @@ function parseConfig(raw) {
|
|
|
1038
1045
|
timeoutMs: typeof rawReranker.timeoutMs === "number" ? Math.max(1e3, Math.floor(rawReranker.timeoutMs)) : 1e4
|
|
1039
1046
|
};
|
|
1040
1047
|
}
|
|
1048
|
+
const rawEmbedding = input.embedding && typeof input.embedding === "object" ? input.embedding : {};
|
|
1049
|
+
const rawEmbeddingBatch = rawEmbedding.batch && typeof rawEmbedding.batch === "object" ? rawEmbedding.batch : null;
|
|
1050
|
+
const embeddingMaxBatchItems = typeof rawEmbeddingBatch?.maxBatchItems === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchItems) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchItems)) : void 0;
|
|
1051
|
+
const embeddingMaxBatchTokens = typeof rawEmbeddingBatch?.maxBatchTokens === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchTokens) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchTokens)) : void 0;
|
|
1052
|
+
const embedding = embeddingMaxBatchItems !== void 0 || embeddingMaxBatchTokens !== void 0 ? {
|
|
1053
|
+
batch: {
|
|
1054
|
+
...embeddingMaxBatchItems !== void 0 ? { maxBatchItems: embeddingMaxBatchItems } : {},
|
|
1055
|
+
...embeddingMaxBatchTokens !== void 0 ? { maxBatchTokens: embeddingMaxBatchTokens } : {}
|
|
1056
|
+
}
|
|
1057
|
+
} : {};
|
|
1041
1058
|
return {
|
|
1042
1059
|
embeddingProvider,
|
|
1043
1060
|
embeddingModel,
|
|
1044
1061
|
customProvider,
|
|
1062
|
+
embedding,
|
|
1045
1063
|
scope: isValidScope(scopeValue) ? scopeValue : "project",
|
|
1046
1064
|
include: includeValue ?? DEFAULT_INCLUDE,
|
|
1047
1065
|
exclude: excludeValue ?? DEFAULT_EXCLUDE,
|
|
@@ -2537,8 +2555,6 @@ async function tryDetectProvider() {
|
|
|
2537
2555
|
}
|
|
2538
2556
|
async function getProviderCredentials(provider) {
|
|
2539
2557
|
switch (provider) {
|
|
2540
|
-
case "github-copilot":
|
|
2541
|
-
return getGitHubCopilotCredentials();
|
|
2542
2558
|
case "openai":
|
|
2543
2559
|
return getOpenAICredentials();
|
|
2544
2560
|
case "google":
|
|
@@ -2549,22 +2565,6 @@ async function getProviderCredentials(provider) {
|
|
|
2549
2565
|
return null;
|
|
2550
2566
|
}
|
|
2551
2567
|
}
|
|
2552
|
-
function getGitHubCopilotCredentials() {
|
|
2553
|
-
const authData = loadOpenCodeAuth();
|
|
2554
|
-
const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
|
|
2555
|
-
if (!copilotAuth || copilotAuth.type !== "oauth") {
|
|
2556
|
-
return null;
|
|
2557
|
-
}
|
|
2558
|
-
const auth = copilotAuth;
|
|
2559
|
-
const baseUrl = auth.enterpriseUrl ? `https://copilot-api.${auth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
|
|
2560
|
-
return {
|
|
2561
|
-
provider: "github-copilot",
|
|
2562
|
-
baseUrl,
|
|
2563
|
-
refreshToken: copilotAuth.refresh,
|
|
2564
|
-
accessToken: copilotAuth.access,
|
|
2565
|
-
tokenExpires: copilotAuth.expires
|
|
2566
|
-
};
|
|
2567
|
-
}
|
|
2568
2568
|
function getOpenAICredentials() {
|
|
2569
2569
|
const authData = loadOpenCodeAuth();
|
|
2570
2570
|
const openaiAuth = authData["openai"];
|
|
@@ -2690,8 +2690,6 @@ async function tryDetectOllamaProvider() {
|
|
|
2690
2690
|
}
|
|
2691
2691
|
function getProviderDisplayName(provider) {
|
|
2692
2692
|
switch (provider) {
|
|
2693
|
-
case "github-copilot":
|
|
2694
|
-
return "GitHub Copilot";
|
|
2695
2693
|
case "openai":
|
|
2696
2694
|
return "OpenAI";
|
|
2697
2695
|
case "google":
|
|
@@ -2916,44 +2914,6 @@ var CustomEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
2916
2914
|
}
|
|
2917
2915
|
};
|
|
2918
2916
|
|
|
2919
|
-
// src/embeddings/providers/github-copilot.ts
|
|
2920
|
-
var GitHubCopilotEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
2921
|
-
constructor(credentials, modelInfo) {
|
|
2922
|
-
super(credentials, modelInfo);
|
|
2923
|
-
}
|
|
2924
|
-
getToken() {
|
|
2925
|
-
if (!this.credentials.refreshToken) {
|
|
2926
|
-
throw new Error("No OAuth token available for GitHub");
|
|
2927
|
-
}
|
|
2928
|
-
return this.credentials.refreshToken;
|
|
2929
|
-
}
|
|
2930
|
-
async embedBatch(texts) {
|
|
2931
|
-
const token = this.getToken();
|
|
2932
|
-
const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
|
|
2933
|
-
method: "POST",
|
|
2934
|
-
headers: {
|
|
2935
|
-
Authorization: `Bearer ${token}`,
|
|
2936
|
-
"Content-Type": "application/json",
|
|
2937
|
-
Accept: "application/vnd.github+json",
|
|
2938
|
-
"X-GitHub-Api-Version": "2022-11-28"
|
|
2939
|
-
},
|
|
2940
|
-
body: JSON.stringify({
|
|
2941
|
-
model: `openai/${this.modelInfo.model}`,
|
|
2942
|
-
input: texts
|
|
2943
|
-
})
|
|
2944
|
-
});
|
|
2945
|
-
if (!response.ok) {
|
|
2946
|
-
const error = (await response.text()).slice(0, 500);
|
|
2947
|
-
throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
|
|
2948
|
-
}
|
|
2949
|
-
const data = await response.json();
|
|
2950
|
-
return {
|
|
2951
|
-
embeddings: data.data.map((d) => d.embedding),
|
|
2952
|
-
totalTokensUsed: data.usage.total_tokens
|
|
2953
|
-
};
|
|
2954
|
-
}
|
|
2955
|
-
};
|
|
2956
|
-
|
|
2957
2917
|
// src/embeddings/providers/google.ts
|
|
2958
2918
|
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddingProvider {
|
|
2959
2919
|
static BATCH_SIZE = 20;
|
|
@@ -2961,24 +2921,30 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
2961
2921
|
super(credentials, modelInfo);
|
|
2962
2922
|
}
|
|
2963
2923
|
async embedQuery(query) {
|
|
2964
|
-
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2965
|
-
const
|
|
2924
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2925
|
+
const texts = [
|
|
2926
|
+
this.modelInfo.model === "gemini-embedding-2" ? `task: code retrieval | query: ${query}` : query
|
|
2927
|
+
];
|
|
2928
|
+
const result = await this.embedWithTaskType(texts, taskType);
|
|
2966
2929
|
return {
|
|
2967
2930
|
embedding: result.embeddings[0],
|
|
2968
2931
|
tokensUsed: result.totalTokensUsed
|
|
2969
2932
|
};
|
|
2970
2933
|
}
|
|
2971
2934
|
async embedDocument(document) {
|
|
2972
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2973
|
-
const result = await this.embedWithTaskType([
|
|
2935
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2936
|
+
const result = await this.embedWithTaskType([
|
|
2937
|
+
this.modelInfo.model === "gemini-embedding-2" ? `title: none | text: ${document}` : document
|
|
2938
|
+
], taskType);
|
|
2974
2939
|
return {
|
|
2975
2940
|
embedding: result.embeddings[0],
|
|
2976
2941
|
tokensUsed: result.totalTokensUsed
|
|
2977
2942
|
};
|
|
2978
2943
|
}
|
|
2979
2944
|
async embedBatch(texts) {
|
|
2980
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2981
|
-
|
|
2945
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2946
|
+
const formattedTexts = this.modelInfo.model === "gemini-embedding-2" ? texts.map((text) => `title: none | text: ${text}`) : texts;
|
|
2947
|
+
return this.embedWithTaskType(formattedTexts, taskType);
|
|
2982
2948
|
}
|
|
2983
2949
|
async embedWithTaskType(texts, taskType) {
|
|
2984
2950
|
const batches = [];
|
|
@@ -3028,6 +2994,10 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
3028
2994
|
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddingProvider {
|
|
3029
2995
|
static MIN_TRUNCATION_CHARS = 512;
|
|
3030
2996
|
static REQUEST_TIMEOUT_MS = 12e4;
|
|
2997
|
+
// Set when /api/embed returns 404 so subsequent multi-text batches skip the
|
|
2998
|
+
// batched endpoint and go straight to the legacy per-text path (one probe per
|
|
2999
|
+
// old ollama install, not one probe per batch).
|
|
3000
|
+
batchEndpointUnavailable = false;
|
|
3031
3001
|
constructor(credentials, modelInfo) {
|
|
3032
3002
|
super(credentials, modelInfo);
|
|
3033
3003
|
}
|
|
@@ -3045,6 +3015,21 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3045
3015
|
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
3046
3016
|
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
3047
3017
|
}
|
|
3018
|
+
// True for a 404 from the newer /api/embed endpoint, i.e. an ollama version that
|
|
3019
|
+
// does not provide it. embedBatch uses this to fall back to the legacy per-text
|
|
3020
|
+
// /api/embeddings path so old ollama installs do not regress.
|
|
3021
|
+
isBatchEndpointUnavailableError(error) {
|
|
3022
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3023
|
+
return message.includes("Ollama /api/embed not available");
|
|
3024
|
+
}
|
|
3025
|
+
// True for a malformed /api/embed response (wrong vector count or a bad vector).
|
|
3026
|
+
// embedBatch falls back to the per-text path on this so a bad batch response
|
|
3027
|
+
// re-embeds each text cleanly. A text that then fails per-text is not isolated
|
|
3028
|
+
// here; it is isolated on the recovery run, which re-embeds one text per request.
|
|
3029
|
+
isBatchValidationError(error) {
|
|
3030
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3031
|
+
return message.includes("invalid embedding batch");
|
|
3032
|
+
}
|
|
3048
3033
|
buildTruncationCandidates(text) {
|
|
3049
3034
|
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
3050
3035
|
const candidateLimits = /* @__PURE__ */ new Set();
|
|
@@ -3146,7 +3131,74 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3146
3131
|
tokensUsed: this.estimateTokens(text)
|
|
3147
3132
|
};
|
|
3148
3133
|
}
|
|
3149
|
-
|
|
3134
|
+
// Embeds many texts in one POST /api/embed request (input: string[]). Ollama
|
|
3135
|
+
// encodes each input independently, so the model context length applies per input
|
|
3136
|
+
// (the upstream splitter already bounds each input), not over the batch. This
|
|
3137
|
+
// amortizes N HTTP round-trips into one.
|
|
3138
|
+
async embedMany(texts) {
|
|
3139
|
+
const controller = new AbortController();
|
|
3140
|
+
const timeout = setTimeout(
|
|
3141
|
+
() => controller.abort(),
|
|
3142
|
+
_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS
|
|
3143
|
+
);
|
|
3144
|
+
let response;
|
|
3145
|
+
try {
|
|
3146
|
+
response = await fetch(`${this.credentials.baseUrl}/api/embed`, {
|
|
3147
|
+
method: "POST",
|
|
3148
|
+
headers: {
|
|
3149
|
+
"Content-Type": "application/json"
|
|
3150
|
+
},
|
|
3151
|
+
body: JSON.stringify({
|
|
3152
|
+
model: this.modelInfo.model,
|
|
3153
|
+
input: texts,
|
|
3154
|
+
truncate: false
|
|
3155
|
+
}),
|
|
3156
|
+
signal: controller.signal
|
|
3157
|
+
});
|
|
3158
|
+
} catch (error) {
|
|
3159
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
3160
|
+
throw new Error(
|
|
3161
|
+
`Ollama embedding request timed out after ${_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS}ms`
|
|
3162
|
+
);
|
|
3163
|
+
}
|
|
3164
|
+
throw error;
|
|
3165
|
+
} finally {
|
|
3166
|
+
clearTimeout(timeout);
|
|
3167
|
+
}
|
|
3168
|
+
if (!response.ok) {
|
|
3169
|
+
const error = (await response.text()).slice(0, 500);
|
|
3170
|
+
if (response.status === 404) {
|
|
3171
|
+
throw new Error(`Ollama /api/embed not available: ${response.status} - ${error}`);
|
|
3172
|
+
}
|
|
3173
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
3174
|
+
}
|
|
3175
|
+
let parsed;
|
|
3176
|
+
try {
|
|
3177
|
+
parsed = await response.json();
|
|
3178
|
+
} catch {
|
|
3179
|
+
throw new Error(
|
|
3180
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
3181
|
+
);
|
|
3182
|
+
}
|
|
3183
|
+
const data = parsed && typeof parsed === "object" ? parsed : {};
|
|
3184
|
+
if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length || data.embeddings.some(
|
|
3185
|
+
(value) => !Array.isArray(value) || value.length !== this.modelInfo.dimensions || value.some((v) => typeof v !== "number" || !Number.isFinite(v))
|
|
3186
|
+
)) {
|
|
3187
|
+
throw new Error(
|
|
3188
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
3189
|
+
);
|
|
3190
|
+
}
|
|
3191
|
+
return {
|
|
3192
|
+
embeddings: data.embeddings,
|
|
3193
|
+
totalTokensUsed: texts.reduce((sum, text) => sum + this.estimateTokens(text), 0)
|
|
3194
|
+
};
|
|
3195
|
+
}
|
|
3196
|
+
// Per-text /api/embeddings path shared by the single-text fast path and the
|
|
3197
|
+
// batch fallback. Uses the legacy endpoint one text at a time, so each text gets
|
|
3198
|
+
// its own truncation safety net and a vector validated on its own. A text that
|
|
3199
|
+
// hard-fails per-text throws here and fails the whole request batch; the recovery
|
|
3200
|
+
// run re-embeds one text per request to isolate it.
|
|
3201
|
+
async embedOneByOne(texts) {
|
|
3150
3202
|
const results = [];
|
|
3151
3203
|
for (const text of texts) {
|
|
3152
3204
|
results.push(await this.embedSingleWithFallback(text));
|
|
@@ -3156,6 +3208,26 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3156
3208
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
3157
3209
|
};
|
|
3158
3210
|
}
|
|
3211
|
+
async embedBatch(texts) {
|
|
3212
|
+
if (texts.length === 0) {
|
|
3213
|
+
return { embeddings: [], totalTokensUsed: 0 };
|
|
3214
|
+
}
|
|
3215
|
+
if (texts.length === 1 || this.batchEndpointUnavailable) {
|
|
3216
|
+
return this.embedOneByOne(texts);
|
|
3217
|
+
}
|
|
3218
|
+
try {
|
|
3219
|
+
return await this.embedMany(texts);
|
|
3220
|
+
} catch (error) {
|
|
3221
|
+
if (this.isBatchEndpointUnavailableError(error)) {
|
|
3222
|
+
this.batchEndpointUnavailable = true;
|
|
3223
|
+
return this.embedOneByOne(texts);
|
|
3224
|
+
}
|
|
3225
|
+
if (!this.isContextLengthError(error) && !this.isBatchValidationError(error)) {
|
|
3226
|
+
throw error;
|
|
3227
|
+
}
|
|
3228
|
+
return this.embedOneByOne(texts);
|
|
3229
|
+
}
|
|
3230
|
+
}
|
|
3159
3231
|
};
|
|
3160
3232
|
|
|
3161
3233
|
// src/embeddings/providers/openai.ts
|
|
@@ -3190,8 +3262,6 @@ var OpenAIEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
3190
3262
|
// src/embeddings/provider.ts
|
|
3191
3263
|
function createEmbeddingProvider(configuredProviderInfo) {
|
|
3192
3264
|
switch (configuredProviderInfo.provider) {
|
|
3193
|
-
case "github-copilot":
|
|
3194
|
-
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
3195
3265
|
case "openai":
|
|
3196
3266
|
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
3197
3267
|
case "google":
|
|
@@ -4209,12 +4279,12 @@ try {
|
|
|
4209
4279
|
}
|
|
4210
4280
|
|
|
4211
4281
|
// src/native/parsing.ts
|
|
4212
|
-
function parseFileAsText(filePath, content) {
|
|
4213
|
-
const result = native.parseFileAsText(filePath, content);
|
|
4282
|
+
function parseFileAsText(filePath, content, linesPerChunk) {
|
|
4283
|
+
const result = native.parseFileAsText(filePath, content, linesPerChunk);
|
|
4214
4284
|
return result.map(mapChunk);
|
|
4215
4285
|
}
|
|
4216
|
-
function parseFiles(files) {
|
|
4217
|
-
const result = native.parseFiles(files);
|
|
4286
|
+
function parseFiles(files, linesPerChunk) {
|
|
4287
|
+
const result = native.parseFiles(files, linesPerChunk);
|
|
4218
4288
|
return result.map((f) => ({
|
|
4219
4289
|
path: f.path,
|
|
4220
4290
|
chunks: f.chunks.map(mapChunk),
|
|
@@ -4291,13 +4361,13 @@ var VectorStore = class {
|
|
|
4291
4361
|
const metadata = items.map((i) => JSON.stringify(i.metadata));
|
|
4292
4362
|
this.inner.addBatch(ids, vectors, metadata);
|
|
4293
4363
|
}
|
|
4294
|
-
search(queryVector, limit = 10) {
|
|
4364
|
+
search(queryVector, limit = 10, allowedIds) {
|
|
4295
4365
|
if (queryVector.length !== this.dimensions) {
|
|
4296
4366
|
throw new Error(
|
|
4297
4367
|
`Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
|
|
4298
4368
|
);
|
|
4299
4369
|
}
|
|
4300
|
-
const results = this.inner.search(queryVector, limit);
|
|
4370
|
+
const results = allowedIds === void 0 ? this.inner.search(queryVector, limit) : this.inner.searchFiltered(queryVector, limit, allowedIds);
|
|
4301
4371
|
return results.map((r) => ({
|
|
4302
4372
|
id: r.id,
|
|
4303
4373
|
score: r.score,
|
|
@@ -4525,6 +4595,10 @@ var Database = class _Database {
|
|
|
4525
4595
|
this.throwIfClosed();
|
|
4526
4596
|
return this.inner.getBranchChunkIds(branch);
|
|
4527
4597
|
}
|
|
4598
|
+
getChunkIdsByBlameDate(since, until) {
|
|
4599
|
+
this.throwIfClosed();
|
|
4600
|
+
return this.inner.getChunkIdsByBlameDate(since, until);
|
|
4601
|
+
}
|
|
4528
4602
|
getBranchDelta(branch, baseBranch) {
|
|
4529
4603
|
this.throwIfClosed();
|
|
4530
4604
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
@@ -5486,6 +5560,9 @@ function resolveWorktreeFallbackPath(projectRoot, relativePath) {
|
|
|
5486
5560
|
const fallbackPath = path10.join(mainRepoRoot, relativePath);
|
|
5487
5561
|
return existsSync5(fallbackPath) ? fallbackPath : null;
|
|
5488
5562
|
}
|
|
5563
|
+
function getHostProjectConfigRelativePath(host) {
|
|
5564
|
+
return getProjectConfigRelativePath(host);
|
|
5565
|
+
}
|
|
5489
5566
|
function getProjectConfigCandidatePaths(projectRoot, host) {
|
|
5490
5567
|
const candidates = [path10.join(projectRoot, getProjectConfigRelativePath(host))];
|
|
5491
5568
|
if (host !== "opencode") {
|
|
@@ -5577,6 +5654,9 @@ function resolveProjectConfigPath(projectRoot, host) {
|
|
|
5577
5654
|
const candidates = getProjectConfigCandidatePaths(projectRoot, host);
|
|
5578
5655
|
return candidates.find((candidate) => existsSync5(candidate)) ?? path10.join(projectRoot, getProjectConfigRelativePath(host));
|
|
5579
5656
|
}
|
|
5657
|
+
function resolveWritableProjectConfigPath(projectRoot, host) {
|
|
5658
|
+
return path10.join(projectRoot, getProjectConfigRelativePath(host));
|
|
5659
|
+
}
|
|
5580
5660
|
function resolveProjectIndexPath(projectRoot, scope, host) {
|
|
5581
5661
|
if (scope === "global") {
|
|
5582
5662
|
return resolveGlobalIndexPath(host);
|
|
@@ -6282,6 +6362,10 @@ function scoreCandidate(query, intent, candidate, originalIndex) {
|
|
|
6282
6362
|
let boost = 0;
|
|
6283
6363
|
if (intent.primary === "conceptual") {
|
|
6284
6364
|
boost += Math.min(0.14, overlap * 0.14);
|
|
6365
|
+
if (intent.preferSourcePaths) {
|
|
6366
|
+
boost += implementationPath ? 0.32 : 0;
|
|
6367
|
+
if (testPath || fixturePath || docsPath) boost -= 0.35;
|
|
6368
|
+
}
|
|
6285
6369
|
if (generatedOrVendor) boost -= 0.18;
|
|
6286
6370
|
if (importChunk || weakContainer) boost -= 0.04;
|
|
6287
6371
|
} else if (intent.primary === "test") {
|
|
@@ -7206,6 +7290,19 @@ function parseOwner(value) {
|
|
|
7206
7290
|
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
7207
7291
|
if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
|
|
7208
7292
|
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
7293
|
+
if (candidate.recoveryProtocolVersion !== void 0 && candidate.recoveryProtocolVersion !== 1) return null;
|
|
7294
|
+
if (candidate.projectRoot !== void 0 && typeof candidate.projectRoot !== "string") return null;
|
|
7295
|
+
if (candidate.scopedRoots !== void 0) {
|
|
7296
|
+
if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) {
|
|
7297
|
+
return null;
|
|
7298
|
+
}
|
|
7299
|
+
}
|
|
7300
|
+
if (candidate.clearRecovery !== void 0) {
|
|
7301
|
+
const recovery = candidate.clearRecovery;
|
|
7302
|
+
if (typeof recovery !== "object" || recovery === null || recovery.phase !== "clearing" || typeof recovery.embeddingProvider !== "string" || recovery.embeddingProvider.length === 0 || typeof recovery.embeddingModel !== "string" || recovery.embeddingModel.length === 0 || !Number.isInteger(recovery.embeddingDimensions) || (recovery.embeddingDimensions ?? 0) <= 0 || typeof recovery.embeddingStrategyVersion !== "string" || recovery.embeddingStrategyVersion.length === 0 || recovery.compatibilityDecision !== "compatible" && recovery.compatibilityDecision !== "embedding-strategy-mismatch" && recovery.compatibilityDecision !== "incompatible" || candidate.operation !== "clear" && candidate.operation !== "force-index") {
|
|
7303
|
+
return null;
|
|
7304
|
+
}
|
|
7305
|
+
}
|
|
7209
7306
|
return candidate;
|
|
7210
7307
|
}
|
|
7211
7308
|
function parseReclaimOwner(value) {
|
|
@@ -7446,13 +7543,18 @@ function isTransientIndexLockContention(error) {
|
|
|
7446
7543
|
if (!isIndexLockContentionError(error) || !("reason" in error)) return false;
|
|
7447
7544
|
return error.reason === "active" || error.reason === "reclaiming";
|
|
7448
7545
|
}
|
|
7449
|
-
function acquireIndexLock(indexPath, operation) {
|
|
7546
|
+
function acquireIndexLock(indexPath, operation, recoveryScope) {
|
|
7450
7547
|
mkdirSync2(indexPath, { recursive: true });
|
|
7451
7548
|
const canonicalIndexPath = realpathSync3.native(indexPath);
|
|
7452
7549
|
const lockPath = path13.join(canonicalIndexPath, "indexing.lock");
|
|
7453
7550
|
cleanupDeadPublicationCandidates(canonicalIndexPath);
|
|
7454
7551
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
7455
|
-
const owner = createOwner(operation)
|
|
7552
|
+
const owner = recoveryScope === void 0 ? createOwner(operation) : {
|
|
7553
|
+
...createOwner(operation),
|
|
7554
|
+
recoveryProtocolVersion: 1,
|
|
7555
|
+
projectRoot: recoveryScope.projectRoot,
|
|
7556
|
+
scopedRoots: recoveryScope.scopedRoots
|
|
7557
|
+
};
|
|
7456
7558
|
if (publishJsonDirectory(lockPath, owner)) {
|
|
7457
7559
|
const lease = {
|
|
7458
7560
|
canonicalIndexPath,
|
|
@@ -7517,6 +7619,33 @@ function releaseIndexLock(lease) {
|
|
|
7517
7619
|
}
|
|
7518
7620
|
return true;
|
|
7519
7621
|
}
|
|
7622
|
+
function setIndexLockClearRecoveryState(lease, clearRecovery) {
|
|
7623
|
+
const currentOwner = readDirectoryOwner(lease.lockPath);
|
|
7624
|
+
if (!currentOwner || !sameOwner(currentOwner, lease.owner)) {
|
|
7625
|
+
throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
7626
|
+
}
|
|
7627
|
+
const nextOwner = { ...currentOwner };
|
|
7628
|
+
if (clearRecovery === null) {
|
|
7629
|
+
delete nextOwner.clearRecovery;
|
|
7630
|
+
} else {
|
|
7631
|
+
nextOwner.clearRecovery = clearRecovery;
|
|
7632
|
+
}
|
|
7633
|
+
const ownerPath = path13.join(lease.lockPath, OWNER_FILE_NAME);
|
|
7634
|
+
const temporaryPath = path13.join(
|
|
7635
|
+
lease.lockPath,
|
|
7636
|
+
`${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${randomUUID()}`
|
|
7637
|
+
);
|
|
7638
|
+
try {
|
|
7639
|
+
writeFileSync2(temporaryPath, JSON.stringify(nextOwner), {
|
|
7640
|
+
encoding: "utf-8",
|
|
7641
|
+
flag: "wx",
|
|
7642
|
+
mode: 384
|
|
7643
|
+
});
|
|
7644
|
+
retryTransientFilesystemOperation(() => renameSync(temporaryPath, ownerPath));
|
|
7645
|
+
} finally {
|
|
7646
|
+
if (existsSync6(temporaryPath)) rmSync(temporaryPath, { force: true });
|
|
7647
|
+
}
|
|
7648
|
+
}
|
|
7520
7649
|
function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
|
|
7521
7650
|
if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
|
|
7522
7651
|
temporaryCounter += 1;
|
|
@@ -7754,6 +7883,18 @@ function createFailedBatchWriter(targetPath) {
|
|
|
7754
7883
|
temporaryPath
|
|
7755
7884
|
};
|
|
7756
7885
|
}
|
|
7886
|
+
function writeFailedBatchRecords(targetPath, records) {
|
|
7887
|
+
const writer = createFailedBatchWriter(targetPath);
|
|
7888
|
+
try {
|
|
7889
|
+
for (const record of records) {
|
|
7890
|
+
writer.write(record);
|
|
7891
|
+
}
|
|
7892
|
+
writer.commit();
|
|
7893
|
+
} catch (error) {
|
|
7894
|
+
writer.cleanup();
|
|
7895
|
+
throw error;
|
|
7896
|
+
}
|
|
7897
|
+
}
|
|
7757
7898
|
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
7758
7899
|
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
7759
7900
|
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
@@ -8036,14 +8177,18 @@ function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
|
8036
8177
|
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
8037
8178
|
return Math.min(2e3, maxChunkTokens);
|
|
8038
8179
|
}
|
|
8039
|
-
|
|
8040
|
-
|
|
8041
|
-
|
|
8042
|
-
|
|
8043
|
-
|
|
8044
|
-
};
|
|
8180
|
+
var DEFAULT_OLLAMA_MAX_BATCH_ITEMS = 16;
|
|
8181
|
+
var DEFAULT_OLLAMA_MAX_BATCH_TOKENS = 65536;
|
|
8182
|
+
function getDynamicBatchOptions(provider, embeddingBatch) {
|
|
8183
|
+
if (provider.provider !== "ollama") {
|
|
8184
|
+
return {};
|
|
8045
8185
|
}
|
|
8046
|
-
|
|
8186
|
+
const base = { maxBatchTokens: DEFAULT_OLLAMA_MAX_BATCH_TOKENS, maxBatchItems: DEFAULT_OLLAMA_MAX_BATCH_ITEMS };
|
|
8187
|
+
return {
|
|
8188
|
+
...base,
|
|
8189
|
+
...typeof embeddingBatch?.maxBatchTokens === "number" && Number.isFinite(embeddingBatch.maxBatchTokens) ? { maxBatchTokens: embeddingBatch.maxBatchTokens } : {},
|
|
8190
|
+
...typeof embeddingBatch?.maxBatchItems === "number" && Number.isFinite(embeddingBatch.maxBatchItems) ? { maxBatchItems: embeddingBatch.maxBatchItems } : {}
|
|
8191
|
+
};
|
|
8047
8192
|
}
|
|
8048
8193
|
function isSqliteCorruptionError(error) {
|
|
8049
8194
|
const message = getErrorMessage3(error).toLowerCase();
|
|
@@ -8061,6 +8206,14 @@ function getPendingChunkId(rawChunk) {
|
|
|
8061
8206
|
const id = rawChunk.id;
|
|
8062
8207
|
return typeof id === "string" ? id : null;
|
|
8063
8208
|
}
|
|
8209
|
+
function parseBlameTimestamp(value, endOfDay) {
|
|
8210
|
+
let timestampMs = Date.parse(value);
|
|
8211
|
+
if (Number.isNaN(timestampMs)) return null;
|
|
8212
|
+
if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(value.trim())) {
|
|
8213
|
+
timestampMs += 24 * 60 * 60 * 1e3 - 1;
|
|
8214
|
+
}
|
|
8215
|
+
return Math.floor(timestampMs / 1e3);
|
|
8216
|
+
}
|
|
8064
8217
|
function metadataFromBlame(blame) {
|
|
8065
8218
|
if (!blame) {
|
|
8066
8219
|
return {};
|
|
@@ -8207,7 +8360,7 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
|
|
|
8207
8360
|
const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
|
|
8208
8361
|
return [...promoted, ...remainder];
|
|
8209
8362
|
}
|
|
8210
|
-
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
8363
|
+
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source", allowNonSourcePaths = false) {
|
|
8211
8364
|
if (!prioritizeSourcePaths) {
|
|
8212
8365
|
return [];
|
|
8213
8366
|
}
|
|
@@ -8227,7 +8380,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8227
8380
|
if (!isImplementationChunkType(chunkType)) {
|
|
8228
8381
|
return false;
|
|
8229
8382
|
}
|
|
8230
|
-
if (!isLikelyImplementationPath2(chunk.filePath)) {
|
|
8383
|
+
if (!allowNonSourcePaths && !isLikelyImplementationPath2(chunk.filePath)) {
|
|
8231
8384
|
return false;
|
|
8232
8385
|
}
|
|
8233
8386
|
const nameLower = (chunk.name ?? "").toLowerCase();
|
|
@@ -8291,7 +8444,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8291
8444
|
}
|
|
8292
8445
|
foundCoveringChunk = upsertChunkCandidate(chunk, identifier, normalizedIdentifier) || foundCoveringChunk;
|
|
8293
8446
|
}
|
|
8294
|
-
if (foundCoveringChunk || !isLikelyImplementationPath2(symbol.filePath)) {
|
|
8447
|
+
if (foundCoveringChunk || !allowNonSourcePaths && !isLikelyImplementationPath2(symbol.filePath)) {
|
|
8295
8448
|
continue;
|
|
8296
8449
|
}
|
|
8297
8450
|
const symbolName = symbol.name.toLowerCase();
|
|
@@ -8345,7 +8498,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8345
8498
|
const ranked = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
8346
8499
|
if (ranked.length === 0) {
|
|
8347
8500
|
const implementationFallback = fallbackCandidates.filter(
|
|
8348
|
-
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && isLikelyImplementationPath2(candidate.metadata.filePath)
|
|
8501
|
+
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && (allowNonSourcePaths || isLikelyImplementationPath2(candidate.metadata.filePath))
|
|
8349
8502
|
);
|
|
8350
8503
|
for (const candidate of implementationFallback) {
|
|
8351
8504
|
const nameLower = (candidate.metadata.name ?? "").toLowerCase();
|
|
@@ -8461,10 +8614,16 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
8461
8614
|
return false;
|
|
8462
8615
|
}
|
|
8463
8616
|
if (options?.blameSince) {
|
|
8464
|
-
const
|
|
8465
|
-
if (
|
|
8617
|
+
const since = parseBlameTimestamp(options.blameSince, false);
|
|
8618
|
+
if (since === null) return false;
|
|
8466
8619
|
const committedAt = candidate.metadata.blameCommittedAt;
|
|
8467
|
-
if (committedAt === void 0 || committedAt <
|
|
8620
|
+
if (committedAt === void 0 || committedAt < since) return false;
|
|
8621
|
+
}
|
|
8622
|
+
if (options?.blameUntil) {
|
|
8623
|
+
const until = parseBlameTimestamp(options.blameUntil, true);
|
|
8624
|
+
if (until === null) return false;
|
|
8625
|
+
const committedAt = candidate.metadata.blameCommittedAt;
|
|
8626
|
+
if (committedAt === void 0 || committedAt > until) return false;
|
|
8468
8627
|
}
|
|
8469
8628
|
return true;
|
|
8470
8629
|
}
|
|
@@ -8520,9 +8679,10 @@ var Indexer = class _Indexer {
|
|
|
8520
8679
|
writerArtifactFingerprint = null;
|
|
8521
8680
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
8522
8681
|
fileBatchLimits;
|
|
8682
|
+
checkpointIntervalChunks;
|
|
8523
8683
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
8524
8684
|
this.projectRoot = projectRoot;
|
|
8525
|
-
this.projectIdentityHash =
|
|
8685
|
+
this.projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
8526
8686
|
this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot;
|
|
8527
8687
|
this.branchNameOverride = runtimeOptions.branchName;
|
|
8528
8688
|
this.catalogIdentityOverride = runtimeOptions.catalogIdentity;
|
|
@@ -8532,6 +8692,7 @@ var Indexer = class _Indexer {
|
|
|
8532
8692
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
8533
8693
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
8534
8694
|
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
8695
|
+
this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks;
|
|
8535
8696
|
this.config = config;
|
|
8536
8697
|
this.host = host;
|
|
8537
8698
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -8643,6 +8804,9 @@ var Indexer = class _Indexer {
|
|
|
8643
8804
|
return path15.resolve(targetPath);
|
|
8644
8805
|
}
|
|
8645
8806
|
}
|
|
8807
|
+
getProjectIdentityHash(projectRoot) {
|
|
8808
|
+
return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
8809
|
+
}
|
|
8646
8810
|
isProjectOwnedIndexPath() {
|
|
8647
8811
|
return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host);
|
|
8648
8812
|
}
|
|
@@ -8679,7 +8843,10 @@ var Indexer = class _Indexer {
|
|
|
8679
8843
|
}
|
|
8680
8844
|
async withIndexMutationLease(operation, callback) {
|
|
8681
8845
|
this.refreshBranchInfo();
|
|
8682
|
-
const lease = acquireIndexLock(this.indexPath, operation
|
|
8846
|
+
const lease = acquireIndexLock(this.indexPath, operation, {
|
|
8847
|
+
projectRoot: this.projectRoot,
|
|
8848
|
+
scopedRoots: this.getScopedRoots()
|
|
8849
|
+
});
|
|
8683
8850
|
this.indexPath = lease.canonicalIndexPath;
|
|
8684
8851
|
this.refreshRuntimeArtifactPaths();
|
|
8685
8852
|
this.activeIndexLease = lease;
|
|
@@ -8734,6 +8901,7 @@ var Indexer = class _Indexer {
|
|
|
8734
8901
|
}
|
|
8735
8902
|
loadFileHashCache() {
|
|
8736
8903
|
if (!existsSync8(this.fileHashCachePath)) {
|
|
8904
|
+
this.fileHashCache = /* @__PURE__ */ new Map();
|
|
8737
8905
|
return;
|
|
8738
8906
|
}
|
|
8739
8907
|
try {
|
|
@@ -8773,10 +8941,10 @@ var Indexer = class _Indexer {
|
|
|
8773
8941
|
invertedIndex.serialize()
|
|
8774
8942
|
);
|
|
8775
8943
|
}
|
|
8776
|
-
getScopedRoots() {
|
|
8777
|
-
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(
|
|
8944
|
+
getScopedRoots(projectRoot = this.projectRoot) {
|
|
8945
|
+
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(projectRoot)]);
|
|
8778
8946
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
8779
|
-
roots.add(this.getCanonicalPath(path15.resolve(
|
|
8947
|
+
roots.add(this.getCanonicalPath(path15.resolve(projectRoot, kbRoot)));
|
|
8780
8948
|
}
|
|
8781
8949
|
return Array.from(roots);
|
|
8782
8950
|
}
|
|
@@ -8847,14 +9015,17 @@ var Indexer = class _Indexer {
|
|
|
8847
9015
|
getLegacyBranchCatalogKey() {
|
|
8848
9016
|
return this.currentBranch || "default";
|
|
8849
9017
|
}
|
|
8850
|
-
getLegacyMigrationMetadataKey() {
|
|
8851
|
-
return `index.globalBranchMigration.${
|
|
9018
|
+
getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9019
|
+
return `index.globalBranchMigration.${projectIdentityHash}`;
|
|
9020
|
+
}
|
|
9021
|
+
getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9022
|
+
return `index.embeddingStrategyVersion.${projectIdentityHash}`;
|
|
8852
9023
|
}
|
|
8853
|
-
|
|
8854
|
-
return `index.
|
|
9024
|
+
getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9025
|
+
return `index.forceReembed.${projectIdentityHash}`;
|
|
8855
9026
|
}
|
|
8856
|
-
|
|
8857
|
-
return `index.
|
|
9027
|
+
getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9028
|
+
return `index.migrationFinalized.${projectIdentityHash}`;
|
|
8858
9029
|
}
|
|
8859
9030
|
getBranchMigrationMetadataKey(prefix, catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
8860
9031
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
@@ -8960,7 +9131,7 @@ var Indexer = class _Indexer {
|
|
|
8960
9131
|
const legacy = this.getLegacyBranchCatalogKey();
|
|
8961
9132
|
return primary === legacy ? [primary] : [primary, legacy];
|
|
8962
9133
|
}
|
|
8963
|
-
getProjectLocalScopedOwnershipIds(roots) {
|
|
9134
|
+
getProjectLocalScopedOwnershipIds(roots, projectRoot = this.projectRoot) {
|
|
8964
9135
|
const chunkIds = /* @__PURE__ */ new Set();
|
|
8965
9136
|
const symbolIds = /* @__PURE__ */ new Set();
|
|
8966
9137
|
if (!this.database) {
|
|
@@ -8968,10 +9139,10 @@ var Indexer = class _Indexer {
|
|
|
8968
9139
|
}
|
|
8969
9140
|
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
8970
9141
|
...Array.from(this.fileHashCache.keys()).filter(
|
|
8971
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
9142
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
8972
9143
|
),
|
|
8973
9144
|
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
8974
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
9145
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
8975
9146
|
)
|
|
8976
9147
|
]);
|
|
8977
9148
|
for (const filePath of projectLocalFilePaths) {
|
|
@@ -8984,15 +9155,16 @@ var Indexer = class _Indexer {
|
|
|
8984
9155
|
}
|
|
8985
9156
|
return { chunkIds, symbolIds };
|
|
8986
9157
|
}
|
|
8987
|
-
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
9158
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds, projectRoot = this.projectRoot) {
|
|
8988
9159
|
if (this.config.scope !== "global") {
|
|
8989
9160
|
return this.getBranchCatalogCleanupKeys();
|
|
8990
9161
|
}
|
|
8991
9162
|
const keys = /* @__PURE__ */ new Set();
|
|
8992
9163
|
const projectChunkIdSet = new Set(projectChunkIds);
|
|
8993
9164
|
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
9165
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
8994
9166
|
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
8995
|
-
if (branchKey.startsWith(`${
|
|
9167
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
8996
9168
|
keys.add(branchKey);
|
|
8997
9169
|
continue;
|
|
8998
9170
|
}
|
|
@@ -9002,8 +9174,10 @@ var Indexer = class _Indexer {
|
|
|
9002
9174
|
keys.add(branchKey);
|
|
9003
9175
|
}
|
|
9004
9176
|
}
|
|
9005
|
-
|
|
9006
|
-
|
|
9177
|
+
if (projectRoot === this.projectRoot) {
|
|
9178
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
9179
|
+
keys.add(branchKey);
|
|
9180
|
+
}
|
|
9007
9181
|
}
|
|
9008
9182
|
return Array.from(keys);
|
|
9009
9183
|
}
|
|
@@ -9011,10 +9185,10 @@ var Indexer = class _Indexer {
|
|
|
9011
9185
|
const canonicalFilePath = this.getCanonicalStoredFilePath(filePath);
|
|
9012
9186
|
return roots.some((root) => isPathWithinRoot2(canonicalFilePath, root));
|
|
9013
9187
|
}
|
|
9014
|
-
isFileInProjectRoot(filePath) {
|
|
9188
|
+
isFileInProjectRoot(filePath, projectRoot = this.projectRoot) {
|
|
9015
9189
|
return isPathWithinRoot2(
|
|
9016
9190
|
this.getCanonicalStoredFilePath(filePath),
|
|
9017
|
-
this.getCanonicalPath(
|
|
9191
|
+
this.getCanonicalPath(projectRoot)
|
|
9018
9192
|
);
|
|
9019
9193
|
}
|
|
9020
9194
|
clearScopedFileHashCache(roots) {
|
|
@@ -9056,12 +9230,12 @@ var Indexer = class _Indexer {
|
|
|
9056
9230
|
}
|
|
9057
9231
|
return false;
|
|
9058
9232
|
}
|
|
9059
|
-
hasForeignScopedBranchData() {
|
|
9233
|
+
hasForeignScopedBranchData(projectRoot = this.projectRoot, roots = this.getScopedRoots(projectRoot)) {
|
|
9060
9234
|
if (!this.database || this.config.scope !== "global") {
|
|
9061
9235
|
return false;
|
|
9062
9236
|
}
|
|
9063
|
-
const
|
|
9064
|
-
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
9237
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
9238
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot);
|
|
9065
9239
|
return this.database.getAllBranches().some(
|
|
9066
9240
|
(branchKey) => {
|
|
9067
9241
|
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
@@ -9070,7 +9244,7 @@ var Indexer = class _Indexer {
|
|
|
9070
9244
|
if (!hasBranchData) {
|
|
9071
9245
|
return false;
|
|
9072
9246
|
}
|
|
9073
|
-
if (branchKey.startsWith(`${
|
|
9247
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
9074
9248
|
return false;
|
|
9075
9249
|
}
|
|
9076
9250
|
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
@@ -9079,7 +9253,7 @@ var Indexer = class _Indexer {
|
|
|
9079
9253
|
}
|
|
9080
9254
|
);
|
|
9081
9255
|
}
|
|
9082
|
-
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
9256
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot = this.projectRoot) {
|
|
9083
9257
|
const allMetadata = store.getAllMetadata();
|
|
9084
9258
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
9085
9259
|
const filePaths = /* @__PURE__ */ new Set([
|
|
@@ -9087,7 +9261,7 @@ var Indexer = class _Indexer {
|
|
|
9087
9261
|
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
9088
9262
|
]);
|
|
9089
9263
|
const projectLocalFilePaths = new Set(
|
|
9090
|
-
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath))
|
|
9264
|
+
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot))
|
|
9091
9265
|
);
|
|
9092
9266
|
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
9093
9267
|
for (const filePath of filePaths) {
|
|
@@ -9097,7 +9271,7 @@ var Indexer = class _Indexer {
|
|
|
9097
9271
|
}
|
|
9098
9272
|
const removedChunkIdList = Array.from(removedChunkIds);
|
|
9099
9273
|
const projectLocalChunkIds = new Set(
|
|
9100
|
-
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)).map(({ key }) => key)
|
|
9274
|
+
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)).map(({ key }) => key)
|
|
9101
9275
|
);
|
|
9102
9276
|
for (const filePath of projectLocalFilePaths) {
|
|
9103
9277
|
for (const chunk of database.getChunksByFile(filePath)) {
|
|
@@ -9116,7 +9290,8 @@ var Indexer = class _Indexer {
|
|
|
9116
9290
|
}
|
|
9117
9291
|
const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys(
|
|
9118
9292
|
Array.from(projectLocalChunkIds),
|
|
9119
|
-
Array.from(projectLocalSymbolIds)
|
|
9293
|
+
Array.from(projectLocalSymbolIds),
|
|
9294
|
+
projectRoot
|
|
9120
9295
|
);
|
|
9121
9296
|
for (const branchKey of branchCleanupKeys) {
|
|
9122
9297
|
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
@@ -9151,29 +9326,96 @@ var Indexer = class _Indexer {
|
|
|
9151
9326
|
database.gcOrphanSymbols();
|
|
9152
9327
|
database.gcOrphanEmbeddings();
|
|
9153
9328
|
database.gcOrphanChunks();
|
|
9154
|
-
store.save();
|
|
9155
9329
|
this.saveInvertedIndex(invertedIndex);
|
|
9330
|
+
store.save();
|
|
9156
9331
|
return {
|
|
9157
9332
|
removedChunkIds: removedChunkIdList,
|
|
9158
9333
|
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
9159
9334
|
};
|
|
9160
9335
|
}
|
|
9336
|
+
getCurrentClearRecoveryState() {
|
|
9337
|
+
if (!this.configuredProviderInfo) {
|
|
9338
|
+
throw new Error("Cannot persist clear recovery state before the embedding provider is initialized");
|
|
9339
|
+
}
|
|
9340
|
+
const compatibility = this.checkCompatibility();
|
|
9341
|
+
const compatibilityDecision = compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible";
|
|
9342
|
+
return {
|
|
9343
|
+
phase: "clearing",
|
|
9344
|
+
embeddingProvider: this.configuredProviderInfo.provider,
|
|
9345
|
+
embeddingModel: this.configuredProviderInfo.modelInfo.model,
|
|
9346
|
+
embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions,
|
|
9347
|
+
embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION,
|
|
9348
|
+
compatibilityDecision
|
|
9349
|
+
};
|
|
9350
|
+
}
|
|
9351
|
+
beginClearRecoveryState() {
|
|
9352
|
+
const recovery = this.getCurrentClearRecoveryState();
|
|
9353
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), recovery);
|
|
9354
|
+
return recovery;
|
|
9355
|
+
}
|
|
9356
|
+
finishClearRecoveryState() {
|
|
9357
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), null);
|
|
9358
|
+
}
|
|
9359
|
+
matchesCurrentClearRecoveryConfiguration(recovery) {
|
|
9360
|
+
const configuredProviderInfo = this.configuredProviderInfo;
|
|
9361
|
+
return configuredProviderInfo !== null && recovery.embeddingProvider === configuredProviderInfo.provider && recovery.embeddingModel === configuredProviderInfo.modelInfo.model && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION;
|
|
9362
|
+
}
|
|
9363
|
+
hasUnknownLegacyForceIndexClear(owner) {
|
|
9364
|
+
return owner.operation === "force-index" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1 && existsSync8(path15.join(this.indexPath, "force-index-phase"));
|
|
9365
|
+
}
|
|
9161
9366
|
async recoverFromInterruptedIndexingUnlocked(owners) {
|
|
9162
9367
|
for (const owner of owners) {
|
|
9163
9368
|
this.logger.warn("Detected interrupted indexing session, recovering...", {
|
|
9164
9369
|
pid: owner.pid,
|
|
9165
9370
|
hostname: owner.hostname,
|
|
9166
9371
|
operation: owner.operation,
|
|
9167
|
-
startedAt: owner.startedAt
|
|
9372
|
+
startedAt: owner.startedAt,
|
|
9373
|
+
projectRoot: owner.projectRoot
|
|
9168
9374
|
});
|
|
9169
9375
|
}
|
|
9170
9376
|
if (this.config.scope === "global") {
|
|
9171
|
-
|
|
9172
|
-
|
|
9377
|
+
const clearScopes = [];
|
|
9378
|
+
for (const owner of owners) {
|
|
9379
|
+
if (this.hasUnknownLegacyForceIndexClear(owner)) {
|
|
9380
|
+
throw new Error(
|
|
9381
|
+
`Cannot automatically recover interrupted force-index ${owner.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
9382
|
+
);
|
|
9383
|
+
}
|
|
9384
|
+
if (owner.operation === "clear" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1) {
|
|
9385
|
+
throw new Error(
|
|
9386
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating recovery state is unknown. The recovery marker was retained for manual inspection.`
|
|
9387
|
+
);
|
|
9388
|
+
}
|
|
9389
|
+
if (owner.clearRecovery === void 0) continue;
|
|
9390
|
+
if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) {
|
|
9391
|
+
throw new Error(
|
|
9392
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating project scope is unknown. The recovery marker was retained for manual inspection.`
|
|
9393
|
+
);
|
|
9394
|
+
}
|
|
9395
|
+
if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) {
|
|
9396
|
+
throw new Error(
|
|
9397
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the current embedding configuration does not match the originating lease. The recovery marker was retained; retry from the originating project with matching settings.`
|
|
9398
|
+
);
|
|
9399
|
+
}
|
|
9400
|
+
clearScopes.push({
|
|
9401
|
+
projectRoot: owner.projectRoot,
|
|
9402
|
+
scopedRoots: owner.scopedRoots,
|
|
9403
|
+
compatibilityDecision: owner.clearRecovery.compatibilityDecision
|
|
9404
|
+
});
|
|
9405
|
+
}
|
|
9406
|
+
if (clearScopes.length > 0) {
|
|
9407
|
+
this.loadFileHashCache();
|
|
9408
|
+
}
|
|
9409
|
+
for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) {
|
|
9410
|
+
this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision);
|
|
9173
9411
|
}
|
|
9174
9412
|
await this.healthCheckUnlocked();
|
|
9413
|
+
this.logger.info(
|
|
9414
|
+
clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint"
|
|
9415
|
+
);
|
|
9416
|
+
return;
|
|
9175
9417
|
}
|
|
9176
|
-
this.logger.info("Recovery complete, next index will
|
|
9418
|
+
this.logger.info("Recovery complete, next index will resume from the last checkpoint");
|
|
9177
9419
|
}
|
|
9178
9420
|
*loadSerializedFailedBatches() {
|
|
9179
9421
|
let warned = false;
|
|
@@ -9211,14 +9453,99 @@ var Indexer = class _Indexer {
|
|
|
9211
9453
|
state.writer.write(record);
|
|
9212
9454
|
state.recordsWritten += record.chunks.length;
|
|
9213
9455
|
}
|
|
9214
|
-
finalizeFailedBatchWriteState(state) {
|
|
9456
|
+
finalizeFailedBatchWriteState(state, resolvedChunkIds = /* @__PURE__ */ new Set()) {
|
|
9215
9457
|
if (state.recordsWritten > 0) {
|
|
9216
|
-
|
|
9458
|
+
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
9459
|
+
const retained = [];
|
|
9460
|
+
const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath));
|
|
9461
|
+
for (let i = records.length - 1; i >= 0; i--) {
|
|
9462
|
+
const chunks = records[i].chunks.filter((rawChunk) => {
|
|
9463
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9464
|
+
if (chunkId !== null) {
|
|
9465
|
+
if (resolvedChunkIds.has(chunkId)) return false;
|
|
9466
|
+
if (seenChunkIds.has(chunkId)) return false;
|
|
9467
|
+
seenChunkIds.add(chunkId);
|
|
9468
|
+
}
|
|
9469
|
+
return true;
|
|
9470
|
+
});
|
|
9471
|
+
if (chunks.length > 0) {
|
|
9472
|
+
retained.unshift({ ...records[i], chunks });
|
|
9473
|
+
}
|
|
9474
|
+
}
|
|
9475
|
+
state.writer.cleanup();
|
|
9476
|
+
if (retained.length > 0) {
|
|
9477
|
+
writeFailedBatchRecords(this.failedBatchesPath, retained);
|
|
9478
|
+
} else {
|
|
9479
|
+
writeFailedBatchRecords(this.failedBatchesPath, []);
|
|
9480
|
+
this.clearFailedBatchState();
|
|
9481
|
+
}
|
|
9217
9482
|
return;
|
|
9218
9483
|
}
|
|
9219
|
-
state.writer.
|
|
9484
|
+
state.writer.commit();
|
|
9220
9485
|
this.clearFailedBatchState();
|
|
9221
9486
|
}
|
|
9487
|
+
getCheckpointIntervalChunks(totalChunks) {
|
|
9488
|
+
return Math.max(
|
|
9489
|
+
this.checkpointIntervalChunks ?? 2e3,
|
|
9490
|
+
Math.floor(totalChunks / 10)
|
|
9491
|
+
);
|
|
9492
|
+
}
|
|
9493
|
+
checkpointIndexRun(database, store, invertedIndex, failedProcessing, resolvedRetryChunkIds, currentFileHashes, committedFilePaths, scopedRoots, configuredProviderInfo) {
|
|
9494
|
+
if (!this.hasProjectForceReembedPending()) {
|
|
9495
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
9496
|
+
this.indexCompatibility = { compatible: true };
|
|
9497
|
+
}
|
|
9498
|
+
database.commitWriteTransaction();
|
|
9499
|
+
database.beginWriteTransaction();
|
|
9500
|
+
this.saveInvertedIndex(invertedIndex);
|
|
9501
|
+
store.save();
|
|
9502
|
+
if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0 || failedProcessing.discardedExistingRecords) {
|
|
9503
|
+
for (const metadata of failedProcessing.latestById.values()) {
|
|
9504
|
+
const alreadyMaterialized = metadata.chunks.some((rawChunk) => {
|
|
9505
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9506
|
+
return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId);
|
|
9507
|
+
});
|
|
9508
|
+
if (alreadyMaterialized) continue;
|
|
9509
|
+
this.writeFailedBatchRecord(failedProcessing.state, {
|
|
9510
|
+
chunks: metadata.chunks,
|
|
9511
|
+
attemptCount: metadata.attemptCount,
|
|
9512
|
+
error: metadata.error,
|
|
9513
|
+
lastAttempt: metadata.lastAttempt
|
|
9514
|
+
});
|
|
9515
|
+
for (const rawChunk of metadata.chunks) {
|
|
9516
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9517
|
+
if (chunkId !== null) {
|
|
9518
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
9519
|
+
}
|
|
9520
|
+
}
|
|
9521
|
+
}
|
|
9522
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
9523
|
+
failedProcessing.state = this.createFailedBatchWriteState();
|
|
9524
|
+
failedProcessing.discardedExistingRecords = false;
|
|
9525
|
+
for (const record of this.loadSerializedFailedBatches()) {
|
|
9526
|
+
for (const rawChunk of record.chunks) {
|
|
9527
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9528
|
+
this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] });
|
|
9529
|
+
if (chunkId !== null) {
|
|
9530
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
9531
|
+
}
|
|
9532
|
+
}
|
|
9533
|
+
}
|
|
9534
|
+
}
|
|
9535
|
+
const partialHashes = /* @__PURE__ */ new Map();
|
|
9536
|
+
for (const filePath of committedFilePaths) {
|
|
9537
|
+
const hash = currentFileHashes.get(filePath);
|
|
9538
|
+
if (hash !== void 0) {
|
|
9539
|
+
partialHashes.set(filePath, hash);
|
|
9540
|
+
}
|
|
9541
|
+
}
|
|
9542
|
+
if (scopedRoots) {
|
|
9543
|
+
this.replaceScopedFileHashCache(partialHashes, scopedRoots);
|
|
9544
|
+
} else {
|
|
9545
|
+
this.fileHashCache = partialHashes;
|
|
9546
|
+
this.saveFileHashCache();
|
|
9547
|
+
}
|
|
9548
|
+
}
|
|
9222
9549
|
clearFailedBatchState() {
|
|
9223
9550
|
if (existsSync8(this.failedBatchesPath)) {
|
|
9224
9551
|
try {
|
|
@@ -9245,6 +9572,7 @@ var Indexer = class _Indexer {
|
|
|
9245
9572
|
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
9246
9573
|
const state = this.createFailedBatchWriteState();
|
|
9247
9574
|
const latestById = /* @__PURE__ */ new Map();
|
|
9575
|
+
let discardedExistingRecords = false;
|
|
9248
9576
|
try {
|
|
9249
9577
|
for (const batch of this.loadSerializedFailedBatches()) {
|
|
9250
9578
|
for (const rawChunk of batch.chunks) {
|
|
@@ -9255,10 +9583,12 @@ var Indexer = class _Indexer {
|
|
|
9255
9583
|
continue;
|
|
9256
9584
|
}
|
|
9257
9585
|
if (!shouldProcess(filePath)) {
|
|
9586
|
+
discardedExistingRecords = true;
|
|
9258
9587
|
continue;
|
|
9259
9588
|
}
|
|
9260
9589
|
const chunkId = getPendingChunkId(rawChunk);
|
|
9261
9590
|
if (!chunkId) {
|
|
9591
|
+
discardedExistingRecords = true;
|
|
9262
9592
|
continue;
|
|
9263
9593
|
}
|
|
9264
9594
|
const existing = latestById.get(chunkId);
|
|
@@ -9266,12 +9596,18 @@ var Indexer = class _Indexer {
|
|
|
9266
9596
|
latestById.set(chunkId, {
|
|
9267
9597
|
attemptCount: batch.attemptCount,
|
|
9268
9598
|
error: batch.error,
|
|
9269
|
-
lastAttempt: batch.lastAttempt
|
|
9599
|
+
lastAttempt: batch.lastAttempt,
|
|
9600
|
+
chunks: [rawChunk]
|
|
9270
9601
|
});
|
|
9271
9602
|
}
|
|
9272
9603
|
}
|
|
9273
9604
|
}
|
|
9274
|
-
return {
|
|
9605
|
+
return {
|
|
9606
|
+
state,
|
|
9607
|
+
latestById,
|
|
9608
|
+
materializedRetryIds: /* @__PURE__ */ new Set(),
|
|
9609
|
+
discardedExistingRecords
|
|
9610
|
+
};
|
|
9275
9611
|
} catch (error) {
|
|
9276
9612
|
state.writer.cleanup();
|
|
9277
9613
|
throw error;
|
|
@@ -9307,10 +9643,34 @@ var Indexer = class _Indexer {
|
|
|
9307
9643
|
}
|
|
9308
9644
|
}
|
|
9309
9645
|
}
|
|
9646
|
+
restoreMissingChunkRows(database, chunks) {
|
|
9647
|
+
const missing = [];
|
|
9648
|
+
for (const chunk of chunks) {
|
|
9649
|
+
if (database.getChunk(chunk.id)) {
|
|
9650
|
+
continue;
|
|
9651
|
+
}
|
|
9652
|
+
missing.push({
|
|
9653
|
+
chunkId: chunk.id,
|
|
9654
|
+
contentHash: chunk.contentHash,
|
|
9655
|
+
filePath: chunk.metadata.filePath,
|
|
9656
|
+
startLine: chunk.metadata.startLine,
|
|
9657
|
+
endLine: chunk.metadata.endLine,
|
|
9658
|
+
nodeType: chunk.metadata.chunkType,
|
|
9659
|
+
name: chunk.metadata.name,
|
|
9660
|
+
language: chunk.metadata.language,
|
|
9661
|
+
blameSha: chunk.metadata.blameSha,
|
|
9662
|
+
blameAuthor: chunk.metadata.blameAuthor,
|
|
9663
|
+
blameAuthorEmail: chunk.metadata.blameAuthorEmail,
|
|
9664
|
+
blameCommittedAt: chunk.metadata.blameCommittedAt,
|
|
9665
|
+
blameSummary: chunk.metadata.blameSummary
|
|
9666
|
+
});
|
|
9667
|
+
}
|
|
9668
|
+
if (missing.length > 0) {
|
|
9669
|
+
database.upsertChunksBatch(missing);
|
|
9670
|
+
}
|
|
9671
|
+
}
|
|
9310
9672
|
getProviderRateLimits(provider) {
|
|
9311
9673
|
switch (provider) {
|
|
9312
|
-
case "github-copilot":
|
|
9313
|
-
return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
|
|
9314
9674
|
case "openai":
|
|
9315
9675
|
return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
9316
9676
|
case "google":
|
|
@@ -9379,10 +9739,11 @@ var Indexer = class _Indexer {
|
|
|
9379
9739
|
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
9380
9740
|
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
9381
9741
|
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
9382
|
-
const
|
|
9383
|
-
|
|
9384
|
-
|
|
9385
|
-
|
|
9742
|
+
const batchOptions = getDynamicBatchOptions(options.configuredProviderInfo, this.config.embedding?.batch);
|
|
9743
|
+
if (options.forceSingleItemBatches && options.configuredProviderInfo.provider === "ollama") {
|
|
9744
|
+
batchOptions.maxBatchItems = 1;
|
|
9745
|
+
}
|
|
9746
|
+
const requestBatches = createPendingEmbeddingRequestBatches(chunksNeedingEmbedding, batchOptions);
|
|
9386
9747
|
let fatalError;
|
|
9387
9748
|
for (const requestBatch of requestBatches) {
|
|
9388
9749
|
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
@@ -9945,7 +10306,7 @@ var Indexer = class _Indexer {
|
|
|
9945
10306
|
}
|
|
9946
10307
|
if (!this.configuredProviderInfo) {
|
|
9947
10308
|
throw new Error(
|
|
9948
|
-
"No embedding provider available. Configure
|
|
10309
|
+
"No embedding provider available. Configure OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
9949
10310
|
);
|
|
9950
10311
|
}
|
|
9951
10312
|
this.logger.info("Initializing indexer", {
|
|
@@ -9976,7 +10337,20 @@ var Indexer = class _Indexer {
|
|
|
9976
10337
|
]);
|
|
9977
10338
|
}
|
|
9978
10339
|
if (recoveredOwners.length > 0 && this.config.scope === "project") {
|
|
9979
|
-
|
|
10340
|
+
const unknownLegacyForceIndex = recoveredOwners.find(
|
|
10341
|
+
(owner) => this.hasUnknownLegacyForceIndexClear(owner)
|
|
10342
|
+
);
|
|
10343
|
+
if (unknownLegacyForceIndex) {
|
|
10344
|
+
throw new Error(
|
|
10345
|
+
`Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
10346
|
+
);
|
|
10347
|
+
}
|
|
10348
|
+
const shouldReset = recoveredOwners.some(
|
|
10349
|
+
(owner) => owner.clearRecovery !== void 0 || owner.operation === "clear" && owner.recoveryProtocolVersion !== 1
|
|
10350
|
+
);
|
|
10351
|
+
if (shouldReset) {
|
|
10352
|
+
await this.resetLocalIndexArtifacts();
|
|
10353
|
+
}
|
|
9980
10354
|
}
|
|
9981
10355
|
this.store = new VectorStore(storePath, dimensions);
|
|
9982
10356
|
if (existsSync8(storePath) || existsSync8(vectorMetadataPath)) {
|
|
@@ -10612,7 +10986,17 @@ var Indexer = class _Indexer {
|
|
|
10612
10986
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
10613
10987
|
for (const file of files) {
|
|
10614
10988
|
const storedPath = this.toStoredFilePath(file.path);
|
|
10615
|
-
|
|
10989
|
+
let currentHash;
|
|
10990
|
+
try {
|
|
10991
|
+
currentHash = hashFile(file.path);
|
|
10992
|
+
} catch (error) {
|
|
10993
|
+
stats.skippedFiles.push({ path: this.toCanonicalFilePath(file.path), reason: "unreadable" });
|
|
10994
|
+
this.logger.warn("Skipped unreadable file during indexing", {
|
|
10995
|
+
path: file.path,
|
|
10996
|
+
error: getErrorMessage3(error)
|
|
10997
|
+
});
|
|
10998
|
+
continue;
|
|
10999
|
+
}
|
|
10616
11000
|
currentFileHashes.set(storedPath, currentHash);
|
|
10617
11001
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
10618
11002
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
@@ -10620,7 +11004,8 @@ var Indexer = class _Indexer {
|
|
|
10620
11004
|
);
|
|
10621
11005
|
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path15.extname(storedPath).toLowerCase() === ".swift";
|
|
10622
11006
|
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path15.extname(storedPath).toLowerCase() === ".metal";
|
|
10623
|
-
|
|
11007
|
+
const inMigrationScope = forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots);
|
|
11008
|
+
if (cachedHashMatches && !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
10624
11009
|
unchangedFilePaths.add(storedPath);
|
|
10625
11010
|
this.logger.recordCacheHit();
|
|
10626
11011
|
} else {
|
|
@@ -10746,6 +11131,9 @@ var Indexer = class _Indexer {
|
|
|
10746
11131
|
}
|
|
10747
11132
|
}
|
|
10748
11133
|
let processedChangedFiles = 0;
|
|
11134
|
+
let lastCheckpointChunks = 0;
|
|
11135
|
+
const committedFilePaths = new Set(unchangedFilePaths);
|
|
11136
|
+
const resolvedRetryChunkIds = /* @__PURE__ */ new Set();
|
|
10749
11137
|
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
10750
11138
|
changedFileDescriptors,
|
|
10751
11139
|
(descriptor) => descriptor.sourceBytes,
|
|
@@ -10759,7 +11147,7 @@ var Indexer = class _Indexer {
|
|
|
10759
11147
|
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
10760
11148
|
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
10761
11149
|
const parseStartTime = performance2.now();
|
|
10762
|
-
const parsedFiles = parseFiles(loadedFiles);
|
|
11150
|
+
const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk);
|
|
10763
11151
|
const parseMs = performance2.now() - parseStartTime;
|
|
10764
11152
|
this.logger.recordFilesParsed(parsedFiles.length);
|
|
10765
11153
|
this.logger.recordParseDuration(parseMs);
|
|
@@ -10782,7 +11170,7 @@ var Indexer = class _Indexer {
|
|
|
10782
11170
|
}
|
|
10783
11171
|
let chunksToProcess = parsed.chunks;
|
|
10784
11172
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
10785
|
-
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
11173
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk);
|
|
10786
11174
|
}
|
|
10787
11175
|
chunksToProcess = selectIndexableChunks(
|
|
10788
11176
|
chunksToProcess,
|
|
@@ -10916,6 +11304,10 @@ var Indexer = class _Indexer {
|
|
|
10916
11304
|
}
|
|
10917
11305
|
if (symbolBatch.length > 0) {
|
|
10918
11306
|
database.upsertSymbolsBatch(symbolBatch);
|
|
11307
|
+
database.addSymbolsToBranchBatch(
|
|
11308
|
+
this.getBranchCatalogKey(),
|
|
11309
|
+
symbolBatch.map((symbol) => symbol.id)
|
|
11310
|
+
);
|
|
10919
11311
|
}
|
|
10920
11312
|
if (edgeBatch.length > 0) {
|
|
10921
11313
|
database.upsertCallEdgesBatch(edgeBatch);
|
|
@@ -10951,6 +11343,12 @@ var Indexer = class _Indexer {
|
|
|
10951
11343
|
forceReembed: forceScopedReembed,
|
|
10952
11344
|
reuseCachedEmbeddings: true,
|
|
10953
11345
|
incrementRepeatedFailures: true,
|
|
11346
|
+
onSucceeded: (succeededChunks) => {
|
|
11347
|
+
database.addChunksToBranchBatch(
|
|
11348
|
+
this.getBranchCatalogKey(),
|
|
11349
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
11350
|
+
);
|
|
11351
|
+
},
|
|
10954
11352
|
onProgress: (batchProgress) => onProgress?.({
|
|
10955
11353
|
phase: "embedding",
|
|
10956
11354
|
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
@@ -10969,6 +11367,27 @@ var Indexer = class _Indexer {
|
|
|
10969
11367
|
}
|
|
10970
11368
|
}
|
|
10971
11369
|
}
|
|
11370
|
+
for (const descriptor of descriptorBatch) {
|
|
11371
|
+
const existingFileChunks = existingChunksByFile.get(descriptor.storedPath);
|
|
11372
|
+
if (!existingFileChunks || existingFileChunks.size === 0) {
|
|
11373
|
+
committedFilePaths.add(descriptor.storedPath);
|
|
11374
|
+
}
|
|
11375
|
+
}
|
|
11376
|
+
const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks);
|
|
11377
|
+
if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) {
|
|
11378
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
11379
|
+
this.checkpointIndexRun(
|
|
11380
|
+
database,
|
|
11381
|
+
store,
|
|
11382
|
+
invertedIndex,
|
|
11383
|
+
failedProcessing,
|
|
11384
|
+
resolvedRetryChunkIds,
|
|
11385
|
+
currentFileHashes,
|
|
11386
|
+
committedFilePaths,
|
|
11387
|
+
scopedRoots,
|
|
11388
|
+
configuredProviderInfo
|
|
11389
|
+
);
|
|
11390
|
+
}
|
|
10972
11391
|
}
|
|
10973
11392
|
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
10974
11393
|
failedProcessing.latestById,
|
|
@@ -10989,6 +11408,7 @@ var Indexer = class _Indexer {
|
|
|
10989
11408
|
retryableChunksWithExistingData.add(chunk.id);
|
|
10990
11409
|
}
|
|
10991
11410
|
}
|
|
11411
|
+
this.restoreMissingChunkRows(database, pendingChunks);
|
|
10992
11412
|
stats.totalChunks += pendingChunks.length;
|
|
10993
11413
|
onProgress?.({
|
|
10994
11414
|
phase: "embedding",
|
|
@@ -11011,6 +11431,17 @@ var Indexer = class _Indexer {
|
|
|
11011
11431
|
forceReembed: forceScopedReembed,
|
|
11012
11432
|
reuseCachedEmbeddings: true,
|
|
11013
11433
|
incrementRepeatedFailures: true,
|
|
11434
|
+
forceSingleItemBatches: true,
|
|
11435
|
+
onSucceeded: (succeededChunks) => {
|
|
11436
|
+
database.addChunksToBranchBatch(
|
|
11437
|
+
this.getBranchCatalogKey(),
|
|
11438
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
11439
|
+
);
|
|
11440
|
+
for (const chunk of succeededChunks) {
|
|
11441
|
+
failedProcessing.latestById.delete(chunk.id);
|
|
11442
|
+
resolvedRetryChunkIds.add(chunk.id);
|
|
11443
|
+
}
|
|
11444
|
+
},
|
|
11014
11445
|
onProgress: (batchProgress) => onProgress?.({
|
|
11015
11446
|
phase: "embedding",
|
|
11016
11447
|
filesProcessed: files.length,
|
|
@@ -11028,6 +11459,20 @@ var Indexer = class _Indexer {
|
|
|
11028
11459
|
failedForcedChunkIds.add(chunkId);
|
|
11029
11460
|
}
|
|
11030
11461
|
}
|
|
11462
|
+
if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) {
|
|
11463
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
11464
|
+
this.checkpointIndexRun(
|
|
11465
|
+
database,
|
|
11466
|
+
store,
|
|
11467
|
+
invertedIndex,
|
|
11468
|
+
failedProcessing,
|
|
11469
|
+
resolvedRetryChunkIds,
|
|
11470
|
+
currentFileHashes,
|
|
11471
|
+
committedFilePaths,
|
|
11472
|
+
scopedRoots,
|
|
11473
|
+
configuredProviderInfo
|
|
11474
|
+
);
|
|
11475
|
+
}
|
|
11031
11476
|
}
|
|
11032
11477
|
const removedChunkIds = [];
|
|
11033
11478
|
for (const [chunkId] of existingChunks) {
|
|
@@ -11064,13 +11509,6 @@ var Indexer = class _Indexer {
|
|
|
11064
11509
|
if (removedStoredChunks) {
|
|
11065
11510
|
this.saveInvertedIndex(invertedIndex);
|
|
11066
11511
|
}
|
|
11067
|
-
if (scopedRoots) {
|
|
11068
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11069
|
-
} else {
|
|
11070
|
-
this.fileHashCache = currentFileHashes;
|
|
11071
|
-
this.saveFileHashCache();
|
|
11072
|
-
}
|
|
11073
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11074
11512
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11075
11513
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11076
11514
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11079,6 +11517,13 @@ var Indexer = class _Indexer {
|
|
|
11079
11517
|
this.indexCompatibility = { compatible: true };
|
|
11080
11518
|
database.commitWriteTransaction();
|
|
11081
11519
|
writeTransactionActive = false;
|
|
11520
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11521
|
+
if (scopedRoots) {
|
|
11522
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11523
|
+
} else {
|
|
11524
|
+
this.fileHashCache = currentFileHashes;
|
|
11525
|
+
this.saveFileHashCache();
|
|
11526
|
+
}
|
|
11082
11527
|
stats.durationMs = Date.now() - startTime;
|
|
11083
11528
|
onProgress?.({
|
|
11084
11529
|
phase: "complete",
|
|
@@ -11102,13 +11547,6 @@ var Indexer = class _Indexer {
|
|
|
11102
11547
|
);
|
|
11103
11548
|
store.save();
|
|
11104
11549
|
this.saveInvertedIndex(invertedIndex);
|
|
11105
|
-
if (scopedRoots) {
|
|
11106
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11107
|
-
} else {
|
|
11108
|
-
this.fileHashCache = currentFileHashes;
|
|
11109
|
-
this.saveFileHashCache();
|
|
11110
|
-
}
|
|
11111
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11112
11550
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11113
11551
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11114
11552
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11117,6 +11555,13 @@ var Indexer = class _Indexer {
|
|
|
11117
11555
|
this.indexCompatibility = { compatible: true };
|
|
11118
11556
|
database.commitWriteTransaction();
|
|
11119
11557
|
writeTransactionActive = false;
|
|
11558
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11559
|
+
if (scopedRoots) {
|
|
11560
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11561
|
+
} else {
|
|
11562
|
+
this.fileHashCache = currentFileHashes;
|
|
11563
|
+
this.saveFileHashCache();
|
|
11564
|
+
}
|
|
11120
11565
|
stats.durationMs = Date.now() - startTime;
|
|
11121
11566
|
onProgress?.({
|
|
11122
11567
|
phase: "complete",
|
|
@@ -11151,15 +11596,15 @@ var Indexer = class _Indexer {
|
|
|
11151
11596
|
);
|
|
11152
11597
|
store.save();
|
|
11153
11598
|
this.saveInvertedIndex(invertedIndex);
|
|
11599
|
+
database.commitWriteTransaction();
|
|
11600
|
+
writeTransactionActive = false;
|
|
11601
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11154
11602
|
if (scopedRoots) {
|
|
11155
11603
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11156
11604
|
} else {
|
|
11157
11605
|
this.fileHashCache = currentFileHashes;
|
|
11158
11606
|
this.saveFileHashCache();
|
|
11159
11607
|
}
|
|
11160
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11161
|
-
database.commitWriteTransaction();
|
|
11162
|
-
writeTransactionActive = false;
|
|
11163
11608
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
11164
11609
|
const gcReset = await this.maybeRunOrphanGc();
|
|
11165
11610
|
if (gcReset) {
|
|
@@ -11183,6 +11628,9 @@ var Indexer = class _Indexer {
|
|
|
11183
11628
|
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
11184
11629
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11185
11630
|
}
|
|
11631
|
+
if (forceScopedReembed) {
|
|
11632
|
+
database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true");
|
|
11633
|
+
}
|
|
11186
11634
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11187
11635
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11188
11636
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11293,26 +11741,41 @@ var Indexer = class _Indexer {
|
|
|
11293
11741
|
shouldPrefilterByBranch: branchChunkIds !== null && (this.config.scope === "global" || hasInitializedBranchCatalog)
|
|
11294
11742
|
};
|
|
11295
11743
|
}
|
|
11296
|
-
|
|
11744
|
+
searchCandidatesWithAllowedIds(initialLimit, totalCount, allowedChunkIds, shouldPrefilter, search, getChunkId) {
|
|
11297
11745
|
const normalizedLimit = Math.max(0, Math.floor(initialLimit));
|
|
11298
11746
|
if (normalizedLimit === 0) return [];
|
|
11299
|
-
if (!
|
|
11747
|
+
if (!shouldPrefilter || !allowedChunkIds) {
|
|
11300
11748
|
return search(normalizedLimit);
|
|
11301
11749
|
}
|
|
11302
|
-
const targetCount = Math.min(normalizedLimit,
|
|
11750
|
+
const targetCount = Math.min(normalizedLimit, allowedChunkIds.size);
|
|
11303
11751
|
if (targetCount === 0 || totalCount === 0) return [];
|
|
11304
11752
|
let requestedLimit = Math.min(normalizedLimit, totalCount);
|
|
11305
11753
|
while (true) {
|
|
11306
11754
|
const results = search(requestedLimit);
|
|
11307
|
-
const
|
|
11308
|
-
if (
|
|
11309
|
-
return
|
|
11755
|
+
const allowedResults = results.filter((candidate) => allowedChunkIds.has(getChunkId(candidate)));
|
|
11756
|
+
if (allowedResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
|
|
11757
|
+
return allowedResults;
|
|
11310
11758
|
}
|
|
11311
11759
|
const nextLimit = Math.min(totalCount, Math.max(requestedLimit + 1, requestedLimit * 2));
|
|
11312
|
-
if (nextLimit === requestedLimit) return
|
|
11760
|
+
if (nextLimit === requestedLimit) return allowedResults;
|
|
11313
11761
|
requestedLimit = nextLimit;
|
|
11314
11762
|
}
|
|
11315
11763
|
}
|
|
11764
|
+
getTemporalChunkIds(database, options) {
|
|
11765
|
+
if (!options?.blameSince && !options?.blameUntil) return null;
|
|
11766
|
+
const since = options.blameSince ? parseBlameTimestamp(options.blameSince, false) : void 0;
|
|
11767
|
+
const until = options.blameUntil ? parseBlameTimestamp(options.blameUntil, true) : void 0;
|
|
11768
|
+
if (since === null || until === null) {
|
|
11769
|
+
return /* @__PURE__ */ new Set();
|
|
11770
|
+
}
|
|
11771
|
+
return new Set(database.getChunkIdsByBlameDate(since, until));
|
|
11772
|
+
}
|
|
11773
|
+
intersectChunkIdSets(first, second) {
|
|
11774
|
+
if (first === null) return second;
|
|
11775
|
+
if (second === null) return first;
|
|
11776
|
+
const [smaller, larger] = first.size <= second.size ? [first, second] : [second, first];
|
|
11777
|
+
return new Set(Array.from(smaller).filter((chunkId) => larger.has(chunkId)));
|
|
11778
|
+
}
|
|
11316
11779
|
buildCandidateSnapshot(candidate) {
|
|
11317
11780
|
return {
|
|
11318
11781
|
id: candidate.id,
|
|
@@ -11327,13 +11790,16 @@ var Indexer = class _Indexer {
|
|
|
11327
11790
|
buildCandidateSnapshotList(candidates) {
|
|
11328
11791
|
return candidates.map((candidate) => this.buildCandidateSnapshot(candidate));
|
|
11329
11792
|
}
|
|
11330
|
-
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch) {
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
|
|
11793
|
+
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch, temporalChunkIds) {
|
|
11794
|
+
const availableCount = temporalChunkIds?.size ?? store.count();
|
|
11795
|
+
if (availableCount === 0) return [];
|
|
11796
|
+
const allowedIds = temporalChunkIds === null ? void 0 : Array.from(temporalChunkIds);
|
|
11797
|
+
return this.searchCandidatesWithAllowedIds(
|
|
11798
|
+
Math.min(initialLimit, availableCount),
|
|
11799
|
+
availableCount,
|
|
11334
11800
|
branchChunkIds,
|
|
11335
11801
|
shouldPrefilterByBranch,
|
|
11336
|
-
(requestedLimit) => store.search(embedding, requestedLimit),
|
|
11802
|
+
(requestedLimit) => store.search(embedding, requestedLimit, allowedIds),
|
|
11337
11803
|
(candidate) => candidate.id
|
|
11338
11804
|
);
|
|
11339
11805
|
}
|
|
@@ -11358,8 +11824,9 @@ var Indexer = class _Indexer {
|
|
|
11358
11824
|
const rerankTopN = this.config.search.rerankTopN;
|
|
11359
11825
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
11360
11826
|
const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
|
|
11827
|
+
const prioritizeSourcePaths = sourceIntent || options?.prioritizeSourcePaths === true;
|
|
11361
11828
|
const identifierHints = extractIdentifierHints(query);
|
|
11362
|
-
const candidateLimit = maxResults * (
|
|
11829
|
+
const candidateLimit = maxResults * (prioritizeSourcePaths ? 12 : 4);
|
|
11363
11830
|
this.logger.search("debug", "Starting search", {
|
|
11364
11831
|
query,
|
|
11365
11832
|
maxResults,
|
|
@@ -11390,6 +11857,7 @@ var Indexer = class _Indexer {
|
|
|
11390
11857
|
branchChunkIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchChunkIds(branchKey)));
|
|
11391
11858
|
branchSymbolIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchSymbolIds(branchKey)));
|
|
11392
11859
|
}
|
|
11860
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
11393
11861
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
11394
11862
|
const prefilterMs = performance2.now() - prefilterStartTime;
|
|
11395
11863
|
const vectorStartTime = performance2.now();
|
|
@@ -11398,7 +11866,8 @@ var Indexer = class _Indexer {
|
|
|
11398
11866
|
embedding,
|
|
11399
11867
|
candidateLimit,
|
|
11400
11868
|
branchChunkIds,
|
|
11401
|
-
shouldPrefilterByBranch
|
|
11869
|
+
shouldPrefilterByBranch,
|
|
11870
|
+
temporalChunkIds
|
|
11402
11871
|
) : [];
|
|
11403
11872
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
11404
11873
|
const keywordStartTime = performance2.now();
|
|
@@ -11408,7 +11877,8 @@ var Indexer = class _Indexer {
|
|
|
11408
11877
|
store,
|
|
11409
11878
|
invertedIndex,
|
|
11410
11879
|
branchChunkIds,
|
|
11411
|
-
shouldPrefilterByBranch
|
|
11880
|
+
shouldPrefilterByBranch,
|
|
11881
|
+
temporalChunkIds
|
|
11412
11882
|
);
|
|
11413
11883
|
const keywordMs = performance2.now() - keywordStartTime;
|
|
11414
11884
|
const scopedSemanticCandidates = semanticCandidates.filter(
|
|
@@ -11430,7 +11900,7 @@ var Indexer = class _Indexer {
|
|
|
11430
11900
|
rerankTopN,
|
|
11431
11901
|
limit: maxResults,
|
|
11432
11902
|
hybridWeight: rankingHybridWeight,
|
|
11433
|
-
prioritizeSourcePaths
|
|
11903
|
+
prioritizeSourcePaths
|
|
11434
11904
|
});
|
|
11435
11905
|
const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
|
|
11436
11906
|
definitionIntent: options?.definitionIntent === true,
|
|
@@ -11466,10 +11936,11 @@ var Indexer = class _Indexer {
|
|
|
11466
11936
|
branchSymbolIds,
|
|
11467
11937
|
maxResults,
|
|
11468
11938
|
union,
|
|
11469
|
-
sourceIntent
|
|
11939
|
+
sourceIntent,
|
|
11940
|
+
options?.definitionIntent === true && ((options.directory?.trim().length ?? 0) > 0 || (options.fileType?.trim().length ?? 0) > 0)
|
|
11470
11941
|
);
|
|
11471
11942
|
const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
|
|
11472
|
-
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
11943
|
+
const primaryLane = options?.definitionIntent === true ? mergeTieredResults(symbolLane, prePrimaryLane, maxResults * 4) : mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
11473
11944
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
11474
11945
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
11475
11946
|
const baseFiltered = tiered.filter(
|
|
@@ -11564,14 +12035,18 @@ var Indexer = class _Indexer {
|
|
|
11564
12035
|
})
|
|
11565
12036
|
);
|
|
11566
12037
|
}
|
|
11567
|
-
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false) {
|
|
12038
|
+
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false, temporalChunkIds = null) {
|
|
11568
12039
|
const normalizedLimit = Math.max(0, Math.floor(limit));
|
|
11569
12040
|
if (normalizedLimit === 0) return [];
|
|
11570
|
-
const
|
|
12041
|
+
const allowedChunkIds = this.intersectChunkIdSets(
|
|
12042
|
+
shouldPrefilterByBranch ? branchChunkIds : null,
|
|
12043
|
+
temporalChunkIds
|
|
12044
|
+
);
|
|
12045
|
+
const scoreEntries = this.searchCandidatesWithAllowedIds(
|
|
11571
12046
|
normalizedLimit,
|
|
11572
12047
|
invertedIndex.getDocumentCount(),
|
|
11573
|
-
|
|
11574
|
-
|
|
12048
|
+
allowedChunkIds,
|
|
12049
|
+
allowedChunkIds !== null,
|
|
11575
12050
|
(requestedLimit) => Array.from(invertedIndex.search(query, requestedLimit)),
|
|
11576
12051
|
([chunkId]) => chunkId
|
|
11577
12052
|
);
|
|
@@ -11656,7 +12131,17 @@ var Indexer = class _Indexer {
|
|
|
11656
12131
|
);
|
|
11657
12132
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
11658
12133
|
for (const file of files) {
|
|
11659
|
-
|
|
12134
|
+
let hash;
|
|
12135
|
+
try {
|
|
12136
|
+
hash = hashFile(file.path);
|
|
12137
|
+
} catch (error) {
|
|
12138
|
+
this.logger.warn("Skipped unreadable file during freshness check", {
|
|
12139
|
+
path: file.path,
|
|
12140
|
+
error: getErrorMessage3(error)
|
|
12141
|
+
});
|
|
12142
|
+
return { readable: false, current: false, reason: "unreadable" };
|
|
12143
|
+
}
|
|
12144
|
+
currentFileHashes.set(this.toStoredFilePath(file.path), hash);
|
|
11660
12145
|
}
|
|
11661
12146
|
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
11662
12147
|
const cachedFileHashes = scopedRoots ? new Map(Array.from(this.fileHashCache).filter(([filePath]) => this.isFileInCurrentScope(filePath, scopedRoots))) : this.fileHashCache;
|
|
@@ -11682,69 +12167,87 @@ var Indexer = class _Indexer {
|
|
|
11682
12167
|
async forceIndex(onProgress) {
|
|
11683
12168
|
return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
|
|
11684
12169
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
11685
|
-
|
|
12170
|
+
const recovery = this.beginClearRecoveryState();
|
|
12171
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
12172
|
+
this.finishClearRecoveryState();
|
|
11686
12173
|
return this.indexUnlocked(onProgress, [], true);
|
|
11687
12174
|
});
|
|
11688
12175
|
}
|
|
11689
12176
|
async clearIndex() {
|
|
11690
12177
|
await this.withIndexMutationLease("clear", async (recoveredOwners) => {
|
|
11691
12178
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
11692
|
-
|
|
12179
|
+
const recovery = this.beginClearRecoveryState();
|
|
12180
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
11693
12181
|
});
|
|
11694
12182
|
}
|
|
11695
|
-
|
|
12183
|
+
clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot) {
|
|
11696
12184
|
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
11697
|
-
|
|
11698
|
-
|
|
11699
|
-
|
|
11700
|
-
|
|
11701
|
-
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
|
|
12185
|
+
const clearedBranchKeys = database.getAllBranches();
|
|
12186
|
+
store.clear();
|
|
12187
|
+
store.save();
|
|
12188
|
+
invertedIndex.clear();
|
|
12189
|
+
this.saveInvertedIndex(invertedIndex);
|
|
12190
|
+
this.fileHashCache.clear();
|
|
12191
|
+
this.saveFileHashCache();
|
|
12192
|
+
database.clearAllIndexedData();
|
|
12193
|
+
this.deleteBranchCommitMetadata(database, clearedBranchKeys);
|
|
12194
|
+
this.clearFailedBatchState();
|
|
12195
|
+
database.deleteMetadata("index.version");
|
|
12196
|
+
database.deleteMetadata("index.pathStorageVersion");
|
|
12197
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
12198
|
+
database.deleteMetadata("index.embeddingModel");
|
|
12199
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
12200
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
12201
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
12202
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
12203
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash));
|
|
12204
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash));
|
|
12205
|
+
database.deleteMetadata("index.createdAt");
|
|
12206
|
+
database.deleteMetadata("index.updatedAt");
|
|
12207
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
12208
|
+
}
|
|
12209
|
+
clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots(), recoveryDecision) {
|
|
12210
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
12211
|
+
store.load();
|
|
12212
|
+
invertedIndex.load();
|
|
12213
|
+
this.loadFileHashCache();
|
|
12214
|
+
const compatibility = this.checkCompatibility();
|
|
12215
|
+
const compatibilityDecision = recoveryDecision ?? (compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible");
|
|
12216
|
+
const allMetadata = store.getAllMetadata();
|
|
12217
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
12218
|
+
if (compatibilityDecision !== "compatible" && hasForeignData) {
|
|
12219
|
+
if (compatibilityDecision === "embedding-strategy-mismatch") {
|
|
12220
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
12221
|
+
this.clearScopedFileHashCache(roots);
|
|
12222
|
+
this.clearScopedFailedBatches(roots);
|
|
12223
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
12224
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true");
|
|
12225
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
12226
|
+
database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash));
|
|
12227
|
+
if (projectRoot === this.projectRoot) {
|
|
11712
12228
|
this.indexCompatibility = { compatible: true };
|
|
11713
|
-
return;
|
|
11714
12229
|
}
|
|
11715
|
-
throw new Error(
|
|
11716
|
-
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
11717
|
-
);
|
|
11718
|
-
}
|
|
11719
|
-
if (!hasForeignData) {
|
|
11720
|
-
const clearedBranchKeys2 = database.getAllBranches();
|
|
11721
|
-
store.clear();
|
|
11722
|
-
store.save();
|
|
11723
|
-
invertedIndex.clear();
|
|
11724
|
-
this.saveInvertedIndex(invertedIndex);
|
|
11725
|
-
this.fileHashCache.clear();
|
|
11726
|
-
this.saveFileHashCache();
|
|
11727
|
-
database.clearAllIndexedData();
|
|
11728
|
-
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
11729
|
-
this.clearFailedBatchState();
|
|
11730
|
-
database.deleteMetadata("index.version");
|
|
11731
|
-
database.deleteMetadata("index.pathStorageVersion");
|
|
11732
|
-
database.deleteMetadata("index.embeddingProvider");
|
|
11733
|
-
database.deleteMetadata("index.embeddingModel");
|
|
11734
|
-
database.deleteMetadata("index.embeddingDimensions");
|
|
11735
|
-
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
11736
|
-
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
11737
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11738
|
-
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
11739
|
-
database.deleteMetadata("index.createdAt");
|
|
11740
|
-
database.deleteMetadata("index.updatedAt");
|
|
11741
|
-
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
11742
12230
|
return;
|
|
11743
12231
|
}
|
|
11744
|
-
|
|
11745
|
-
|
|
11746
|
-
|
|
12232
|
+
throw new Error(
|
|
12233
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
12234
|
+
);
|
|
12235
|
+
}
|
|
12236
|
+
if (!hasForeignData) {
|
|
12237
|
+
this.clearGlobalIndexDataUnlocked(projectRoot);
|
|
12238
|
+
return;
|
|
12239
|
+
}
|
|
12240
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
12241
|
+
this.clearScopedFileHashCache(roots);
|
|
12242
|
+
this.clearScopedFailedBatches(roots);
|
|
12243
|
+
if (projectRoot === this.projectRoot) {
|
|
11747
12244
|
this.indexCompatibility = compatibility;
|
|
12245
|
+
}
|
|
12246
|
+
}
|
|
12247
|
+
async clearIndexUnlocked(recoveryDecision) {
|
|
12248
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
12249
|
+
if (this.config.scope === "global") {
|
|
12250
|
+
this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision);
|
|
11748
12251
|
return;
|
|
11749
12252
|
}
|
|
11750
12253
|
if (!this.isProjectOwnedIndexPath()) {
|
|
@@ -11910,6 +12413,7 @@ var Indexer = class _Indexer {
|
|
|
11910
12413
|
)) {
|
|
11911
12414
|
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
11912
12415
|
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
12416
|
+
this.restoreMissingChunkRows(database, chunks);
|
|
11913
12417
|
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
11914
12418
|
store,
|
|
11915
12419
|
provider,
|
|
@@ -11924,6 +12428,7 @@ var Indexer = class _Indexer {
|
|
|
11924
12428
|
forceReembed: false,
|
|
11925
12429
|
reuseCachedEmbeddings: false,
|
|
11926
12430
|
incrementRepeatedFailures: false,
|
|
12431
|
+
forceSingleItemBatches: true,
|
|
11927
12432
|
onSucceeded: (succeededChunks) => {
|
|
11928
12433
|
database.addChunksToBranchBatch(
|
|
11929
12434
|
this.getBranchCatalogKey(),
|
|
@@ -11945,9 +12450,12 @@ var Indexer = class _Indexer {
|
|
|
11945
12450
|
this.saveInvertedIndex(invertedIndex);
|
|
11946
12451
|
}
|
|
11947
12452
|
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
11948
|
-
database.
|
|
11949
|
-
|
|
11950
|
-
|
|
12453
|
+
const migrationFinalized = database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true";
|
|
12454
|
+
if (migrationFinalized) {
|
|
12455
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12456
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12457
|
+
this.indexCompatibility = { compatible: true };
|
|
12458
|
+
}
|
|
11951
12459
|
}
|
|
11952
12460
|
return { succeeded, failed, remaining };
|
|
11953
12461
|
}
|
|
@@ -11969,7 +12477,8 @@ var Indexer = class _Indexer {
|
|
|
11969
12477
|
latestById.set(chunkId, {
|
|
11970
12478
|
attemptCount: batch.attemptCount,
|
|
11971
12479
|
error: batch.error,
|
|
11972
|
-
lastAttempt: batch.lastAttempt
|
|
12480
|
+
lastAttempt: batch.lastAttempt,
|
|
12481
|
+
chunks: [rawChunk]
|
|
11973
12482
|
});
|
|
11974
12483
|
}
|
|
11975
12484
|
}
|
|
@@ -12036,6 +12545,7 @@ var Indexer = class _Indexer {
|
|
|
12036
12545
|
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
12037
12546
|
);
|
|
12038
12547
|
}
|
|
12548
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
12039
12549
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
12040
12550
|
const prefilterMs = performance2.now() - prefilterStartTime;
|
|
12041
12551
|
const vectorStartTime = performance2.now();
|
|
@@ -12044,7 +12554,8 @@ var Indexer = class _Indexer {
|
|
|
12044
12554
|
embedding,
|
|
12045
12555
|
limit * 2,
|
|
12046
12556
|
branchChunkIds,
|
|
12047
|
-
shouldPrefilterByBranch
|
|
12557
|
+
shouldPrefilterByBranch,
|
|
12558
|
+
temporalChunkIds
|
|
12048
12559
|
);
|
|
12049
12560
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
12050
12561
|
if (this.config.scope !== "global" && branchChunkIds && !hasInitializedBranchCatalog) {
|
|
@@ -12626,6 +13137,36 @@ function resolveConfigPathValue(value, baseDir) {
|
|
|
12626
13137
|
const absolutePath = path16.isAbsolute(trimmed) ? trimmed : path16.resolve(baseDir, trimmed);
|
|
12627
13138
|
return path16.normalize(absolutePath);
|
|
12628
13139
|
}
|
|
13140
|
+
function serializeConfigPathValue(value, baseDir) {
|
|
13141
|
+
const trimmed = value.trim();
|
|
13142
|
+
if (!trimmed) {
|
|
13143
|
+
return trimmed;
|
|
13144
|
+
}
|
|
13145
|
+
if (!path16.isAbsolute(trimmed)) {
|
|
13146
|
+
return normalizePathSeparators(path16.normalize(trimmed));
|
|
13147
|
+
}
|
|
13148
|
+
const relativePath = path16.relative(baseDir, trimmed);
|
|
13149
|
+
if (!relativePath || !relativePath.startsWith("..") && !path16.isAbsolute(relativePath)) {
|
|
13150
|
+
return normalizePathSeparators(path16.normalize(relativePath || "."));
|
|
13151
|
+
}
|
|
13152
|
+
return path16.normalize(trimmed);
|
|
13153
|
+
}
|
|
13154
|
+
function resolveKnowledgeBasePath(value, projectRoot) {
|
|
13155
|
+
return path16.isAbsolute(value) ? value : path16.resolve(projectRoot, value);
|
|
13156
|
+
}
|
|
13157
|
+
function normalizeKnowledgeBasePath(value, projectRoot) {
|
|
13158
|
+
return path16.normalize(resolveKnowledgeBasePath(value, projectRoot));
|
|
13159
|
+
}
|
|
13160
|
+
function hasMatchingKnowledgeBasePath(knowledgeBases, inputPath, projectRoot) {
|
|
13161
|
+
const normalizedInput = path16.normalize(inputPath);
|
|
13162
|
+
return knowledgeBases.some((kb) => normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput);
|
|
13163
|
+
}
|
|
13164
|
+
function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot) {
|
|
13165
|
+
const normalizedInput = path16.normalize(inputPath);
|
|
13166
|
+
return knowledgeBases.findIndex(
|
|
13167
|
+
(kb) => path16.normalize(kb) === normalizedInput || normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput
|
|
13168
|
+
);
|
|
13169
|
+
}
|
|
12629
13170
|
|
|
12630
13171
|
// src/tools/format-communities.ts
|
|
12631
13172
|
function compareText(left, right) {
|
|
@@ -14332,7 +14873,7 @@ function applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key
|
|
|
14332
14873
|
function mergeUniqueStringArray(values) {
|
|
14333
14874
|
return [...new Set(values.map((value) => String(value).trim()))];
|
|
14334
14875
|
}
|
|
14335
|
-
function
|
|
14876
|
+
function normalizeKnowledgeBasePath2(value) {
|
|
14336
14877
|
let normalized = path19.normalize(String(value).trim());
|
|
14337
14878
|
const root = path19.parse(normalized).root;
|
|
14338
14879
|
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
@@ -14341,7 +14882,7 @@ function normalizeKnowledgeBasePath(value) {
|
|
|
14341
14882
|
return normalized;
|
|
14342
14883
|
}
|
|
14343
14884
|
function mergeKnowledgeBasePaths(values) {
|
|
14344
|
-
return [...new Set(values.map((value) =>
|
|
14885
|
+
return [...new Set(values.map((value) => normalizeKnowledgeBasePath2(value)).filter((value) => value.length > 0))];
|
|
14345
14886
|
}
|
|
14346
14887
|
function validateConfigLayerShape(rawConfig, filePath) {
|
|
14347
14888
|
if (!isRecord(rawConfig)) {
|
|
@@ -14471,9 +15012,30 @@ function toConfigRecord(rawConfig) {
|
|
|
14471
15012
|
}
|
|
14472
15013
|
return { ...rawConfig };
|
|
14473
15014
|
}
|
|
15015
|
+
function getConfigPath(projectRoot, host) {
|
|
15016
|
+
return resolveWritableProjectConfigPath(projectRoot, host);
|
|
15017
|
+
}
|
|
14474
15018
|
function loadRuntimeConfig(projectRoot, host) {
|
|
14475
15019
|
return normalizeKnowledgeBasePaths(toConfigRecord(loadMergedConfig(projectRoot, host)), projectRoot);
|
|
14476
15020
|
}
|
|
15021
|
+
function loadEditableConfig(projectRoot, host) {
|
|
15022
|
+
return normalizeKnowledgeBasePaths(toConfigRecord(loadProjectConfigLayer(projectRoot, host)), projectRoot);
|
|
15023
|
+
}
|
|
15024
|
+
function saveConfig(projectRoot, config, host) {
|
|
15025
|
+
const configPath = getConfigPath(projectRoot, host);
|
|
15026
|
+
const configDir = path20.dirname(configPath);
|
|
15027
|
+
const configBaseDir = path20.dirname(configDir);
|
|
15028
|
+
if (!existsSync11(configDir)) {
|
|
15029
|
+
mkdirSync5(configDir, { recursive: true });
|
|
15030
|
+
}
|
|
15031
|
+
const serializableConfig = { ...config };
|
|
15032
|
+
if (Array.isArray(serializableConfig.knowledgeBases)) {
|
|
15033
|
+
serializableConfig.knowledgeBases = serializableConfig.knowledgeBases.map(
|
|
15034
|
+
(kb) => serializeConfigPathValue(kb, configBaseDir)
|
|
15035
|
+
);
|
|
15036
|
+
}
|
|
15037
|
+
writeFileSync4(configPath, JSON.stringify(serializableConfig, null, 2) + "\n", "utf-8");
|
|
15038
|
+
}
|
|
14477
15039
|
|
|
14478
15040
|
// src/tools/operation-runtime.ts
|
|
14479
15041
|
var indexerCache = /* @__PURE__ */ new Map();
|
|
@@ -14692,9 +15254,11 @@ async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
|
14692
15254
|
contextLines: options.contextLines,
|
|
14693
15255
|
metadataOnly: options.metadataOnly,
|
|
14694
15256
|
definitionIntent: options.definitionIntent,
|
|
15257
|
+
prioritizeSourcePaths: options.prioritizeSourcePaths,
|
|
14695
15258
|
blameAuthor: options.blameAuthor,
|
|
14696
15259
|
blameSha: options.blameSha,
|
|
14697
15260
|
blameSince: options.blameSince,
|
|
15261
|
+
blameUntil: options.blameUntil,
|
|
14698
15262
|
trace: options.trace
|
|
14699
15263
|
});
|
|
14700
15264
|
}
|
|
@@ -14740,7 +15304,9 @@ async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
|
14740
15304
|
fileType: options.fileType,
|
|
14741
15305
|
directory: options.directory,
|
|
14742
15306
|
chunkType: options.chunkType,
|
|
14743
|
-
excludeFile: options.excludeFile
|
|
15307
|
+
excludeFile: options.excludeFile,
|
|
15308
|
+
blameSince: options.blameSince,
|
|
15309
|
+
blameUntil: options.blameUntil
|
|
14744
15310
|
});
|
|
14745
15311
|
}
|
|
14746
15312
|
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
@@ -14986,6 +15552,141 @@ async function getIndexLogs(projectRoot, host, args) {
|
|
|
14986
15552
|
}).join("\n");
|
|
14987
15553
|
return { kind: "entries", text };
|
|
14988
15554
|
}
|
|
15555
|
+
function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
15556
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15557
|
+
const inputPath = knowledgeBasePath.trim();
|
|
15558
|
+
const normalizedPath3 = path21.resolve(
|
|
15559
|
+
path21.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
|
|
15560
|
+
);
|
|
15561
|
+
if (!existsSync12(normalizedPath3)) {
|
|
15562
|
+
return `Error: Directory does not exist: ${normalizedPath3}`;
|
|
15563
|
+
}
|
|
15564
|
+
let realPath;
|
|
15565
|
+
try {
|
|
15566
|
+
realPath = realpathSync5(normalizedPath3);
|
|
15567
|
+
} catch {
|
|
15568
|
+
return `Error: Cannot resolve path: ${normalizedPath3}`;
|
|
15569
|
+
}
|
|
15570
|
+
const blockedPrefixes = [
|
|
15571
|
+
"/etc",
|
|
15572
|
+
"/proc",
|
|
15573
|
+
"/sys",
|
|
15574
|
+
"/dev",
|
|
15575
|
+
"/boot",
|
|
15576
|
+
"/root",
|
|
15577
|
+
"/var/run",
|
|
15578
|
+
"/var/log"
|
|
15579
|
+
];
|
|
15580
|
+
const homeDir = process.platform === "win32" ? process.env.USERPROFILE ?? "" : process.env.HOME ?? "";
|
|
15581
|
+
const sensitiveDotDirs = [
|
|
15582
|
+
".ssh",
|
|
15583
|
+
".gnupg",
|
|
15584
|
+
".aws",
|
|
15585
|
+
".config/gcloud",
|
|
15586
|
+
".docker",
|
|
15587
|
+
".kube"
|
|
15588
|
+
];
|
|
15589
|
+
for (const prefix of blockedPrefixes) {
|
|
15590
|
+
if (realPath === prefix || realPath.startsWith(`${prefix}/`)) {
|
|
15591
|
+
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
|
|
15592
|
+
}
|
|
15593
|
+
}
|
|
15594
|
+
for (const dotDir of sensitiveDotDirs) {
|
|
15595
|
+
const sensitiveDir = path21.join(homeDir, dotDir);
|
|
15596
|
+
if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
|
|
15597
|
+
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
|
|
15598
|
+
}
|
|
15599
|
+
}
|
|
15600
|
+
try {
|
|
15601
|
+
const stat5 = statSync5(normalizedPath3);
|
|
15602
|
+
if (!stat5.isDirectory()) {
|
|
15603
|
+
return `Error: Path is not a directory: ${normalizedPath3}`;
|
|
15604
|
+
}
|
|
15605
|
+
} catch (error) {
|
|
15606
|
+
return `Error: Cannot access directory: ${normalizedPath3} - ${error instanceof Error ? error.message : String(error)}`;
|
|
15607
|
+
}
|
|
15608
|
+
const config = loadEditableConfig(root, host);
|
|
15609
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15610
|
+
const alreadyExists = hasMatchingKnowledgeBasePath(knowledgeBases, normalizedPath3, root);
|
|
15611
|
+
if (alreadyExists) {
|
|
15612
|
+
return `Knowledge base already configured: ${normalizedPath3}`;
|
|
15613
|
+
}
|
|
15614
|
+
knowledgeBases.push(normalizedPath3);
|
|
15615
|
+
config.knowledgeBases = knowledgeBases;
|
|
15616
|
+
saveConfig(root, config, host);
|
|
15617
|
+
refreshIndexerForDirectory(root, host);
|
|
15618
|
+
let result = `${normalizedPath3}
|
|
15619
|
+
`;
|
|
15620
|
+
result += `Total knowledge bases: ${knowledgeBases.length}
|
|
15621
|
+
`;
|
|
15622
|
+
result += `Config path: ${getConfigPath(root, host)}
|
|
15623
|
+
`;
|
|
15624
|
+
result += `
|
|
15625
|
+
Run /index to rebuild the index with the new knowledge base.`;
|
|
15626
|
+
return result;
|
|
15627
|
+
}
|
|
15628
|
+
function listKnowledgeBases(projectRoot, host) {
|
|
15629
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15630
|
+
const config = loadRuntimeConfig(root, host);
|
|
15631
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15632
|
+
if (knowledgeBases.length === 0) {
|
|
15633
|
+
return "No knowledge bases configured. Use add_knowledge_base to add folders.";
|
|
15634
|
+
}
|
|
15635
|
+
let result = `Knowledge Bases (${knowledgeBases.length}):
|
|
15636
|
+
|
|
15637
|
+
`;
|
|
15638
|
+
for (let i = 0; i < knowledgeBases.length; i++) {
|
|
15639
|
+
const kb = knowledgeBases[i];
|
|
15640
|
+
const resolvedPath = resolveKnowledgeBasePath(kb, root);
|
|
15641
|
+
const exists = existsSync12(resolvedPath);
|
|
15642
|
+
result += `[${i + 1}] ${kb}
|
|
15643
|
+
`;
|
|
15644
|
+
result += ` Resolved: ${resolvedPath}
|
|
15645
|
+
`;
|
|
15646
|
+
result += ` Status: ${exists ? "Exists" : "NOT FOUND"}
|
|
15647
|
+
`;
|
|
15648
|
+
if (exists) {
|
|
15649
|
+
try {
|
|
15650
|
+
const stat5 = statSync5(resolvedPath);
|
|
15651
|
+
result += ` Type: ${stat5.isDirectory() ? "Directory" : "File"}
|
|
15652
|
+
`;
|
|
15653
|
+
} catch {
|
|
15654
|
+
}
|
|
15655
|
+
}
|
|
15656
|
+
result += "\n";
|
|
15657
|
+
}
|
|
15658
|
+
const hasHostConfig = existsSync12(path21.join(root, getHostProjectConfigRelativePath(host)));
|
|
15659
|
+
if (hasHostConfig) {
|
|
15660
|
+
result += `
|
|
15661
|
+
Config sources: 1 file(s).`;
|
|
15662
|
+
}
|
|
15663
|
+
result += `
|
|
15664
|
+
Config file: ${getConfigPath(root, host)}`;
|
|
15665
|
+
return result;
|
|
15666
|
+
}
|
|
15667
|
+
function removeKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
15668
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15669
|
+
const config = loadEditableConfig(root, host);
|
|
15670
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15671
|
+
const index = findKnowledgeBasePathIndex(knowledgeBases, knowledgeBasePath, root);
|
|
15672
|
+
if (index === -1) {
|
|
15673
|
+
return `Knowledge base not found: ${knowledgeBasePath}`;
|
|
15674
|
+
}
|
|
15675
|
+
const removed = knowledgeBases.splice(index, 1)[0];
|
|
15676
|
+
config.knowledgeBases = knowledgeBases;
|
|
15677
|
+
saveConfig(root, config, host);
|
|
15678
|
+
refreshIndexerForDirectory(root, host);
|
|
15679
|
+
let result = `Removed: ${removed}
|
|
15680
|
+
|
|
15681
|
+
`;
|
|
15682
|
+
result += `Remaining knowledge bases: ${knowledgeBases.length}
|
|
15683
|
+
`;
|
|
15684
|
+
result += `Config saved to: ${getConfigPath(root, host)}
|
|
15685
|
+
`;
|
|
15686
|
+
result += `
|
|
15687
|
+
Run /index to rebuild the index without the removed knowledge base.`;
|
|
15688
|
+
return result;
|
|
15689
|
+
}
|
|
14989
15690
|
|
|
14990
15691
|
// src/tools/context-search.ts
|
|
14991
15692
|
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
@@ -15214,13 +15915,19 @@ async function resolveSearchContext(input, operations) {
|
|
|
15214
15915
|
(trace) => operations.lookup(symbol, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
|
|
15215
15916
|
);
|
|
15216
15917
|
};
|
|
15217
|
-
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt) => {
|
|
15918
|
+
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt, prioritizeSourcePaths) => {
|
|
15218
15919
|
return recordAttempt(
|
|
15219
15920
|
"conceptual",
|
|
15220
15921
|
searchQuery,
|
|
15221
15922
|
scope,
|
|
15222
15923
|
relaxedFieldsForAttempt,
|
|
15223
|
-
(trace) => operations.search(
|
|
15924
|
+
(trace) => operations.search(
|
|
15925
|
+
searchQuery,
|
|
15926
|
+
MAX_CONTEXT_RESULT_LIMIT,
|
|
15927
|
+
scope,
|
|
15928
|
+
input.diagnostic ? trace : void 0,
|
|
15929
|
+
{ prioritizeSourcePaths }
|
|
15930
|
+
)
|
|
15224
15931
|
);
|
|
15225
15932
|
};
|
|
15226
15933
|
const findSuccessfulAttemptState = (route) => {
|
|
@@ -15348,10 +16055,12 @@ Explicit symbol lookup only; conceptual search was not attempted.`
|
|
|
15348
16055
|
}
|
|
15349
16056
|
}
|
|
15350
16057
|
for (const attempt of conceptualAttemptPlan) {
|
|
16058
|
+
const attemptIntent = analyzeQueryIntent(attempt.queryText);
|
|
16059
|
+
const prioritizeSourcePaths = attemptIntent.primary !== "docs" && attemptIntent.primary !== "test";
|
|
15351
16060
|
if (inferredSymbol && attempt.queryText === inferredSymbol && attempt.queryText !== query) {
|
|
15352
16061
|
decisions.fallbackFromOriginalConceptualToInferred = true;
|
|
15353
16062
|
}
|
|
15354
|
-
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed);
|
|
16063
|
+
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed, prioritizeSourcePaths);
|
|
15355
16064
|
if (results.length > 0) {
|
|
15356
16065
|
const heading = buildPackHeading("conceptual", decisions);
|
|
15357
16066
|
const intent = analyzeQueryIntent(attempt.queryText);
|
|
@@ -15488,12 +16197,13 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
15488
16197
|
directory: scope.directory,
|
|
15489
16198
|
trace
|
|
15490
16199
|
}),
|
|
15491
|
-
search: (queryText, retrievalLimit, scope, trace) => searchCodebase(projectRoot, host, queryText, {
|
|
16200
|
+
search: (queryText, retrievalLimit, scope, trace, searchOptions) => searchCodebase(projectRoot, host, queryText, {
|
|
15492
16201
|
limit: retrievalLimit,
|
|
15493
16202
|
fileType: scope.fileType,
|
|
15494
16203
|
directory: scope.directory,
|
|
15495
16204
|
metadataOnly: true,
|
|
15496
|
-
trace
|
|
16205
|
+
trace,
|
|
16206
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
15497
16207
|
})
|
|
15498
16208
|
});
|
|
15499
16209
|
}
|
|
@@ -16994,12 +17704,13 @@ async function runEvaluation(options) {
|
|
|
16994
17704
|
fileType: scope.fileType,
|
|
16995
17705
|
directory: scope.directory
|
|
16996
17706
|
}),
|
|
16997
|
-
search: (searchQuery, limit, scope) => indexer.search(searchQuery, limit, {
|
|
17707
|
+
search: (searchQuery, limit, scope, _trace, searchOptions) => indexer.search(searchQuery, limit, {
|
|
16998
17708
|
metadataOnly: true,
|
|
16999
17709
|
filterByBranch: !!query.expected.branch,
|
|
17000
17710
|
definitionIntent: false,
|
|
17001
17711
|
fileType: scope.fileType,
|
|
17002
|
-
directory: scope.directory
|
|
17712
|
+
directory: scope.directory,
|
|
17713
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
17003
17714
|
})
|
|
17004
17715
|
}) : void 0;
|
|
17005
17716
|
const result = editContextResult?.results ?? contextResult?.details?.results ?? await indexer.search(query.query, 10, {
|
|
@@ -17775,11 +18486,21 @@ var PI_TOOL_NAMES = [
|
|
|
17775
18486
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
17776
18487
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
17777
18488
|
];
|
|
18489
|
+
var MCP_TOOL_NAMES = [
|
|
18490
|
+
...PORTABLE_TOOL_NAMES,
|
|
18491
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
18492
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18493
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE
|
|
18494
|
+
];
|
|
17778
18495
|
|
|
17779
18496
|
// src/adapters/mcp/register-tools.ts
|
|
17780
18497
|
function allowNullAsUndefined(schema) {
|
|
17781
18498
|
return z2.preprocess((value) => value === null ? void 0 : value, schema);
|
|
17782
18499
|
}
|
|
18500
|
+
function knowledgeBaseResult(text) {
|
|
18501
|
+
const content = [{ type: "text", text }];
|
|
18502
|
+
return text.startsWith("Error: ") ? { content, isError: true } : { content };
|
|
18503
|
+
}
|
|
17783
18504
|
function registerMcpTools(server, runtime) {
|
|
17784
18505
|
server.tool(
|
|
17785
18506
|
TOOL_NAME.CODEBASE_CONTEXT,
|
|
@@ -17840,7 +18561,8 @@ function registerMcpTools(server, runtime) {
|
|
|
17840
18561
|
contextLines: allowNullAsUndefined(z2.number().optional()).describe("Number of extra lines to include before/after each match (default: 0)"),
|
|
17841
18562
|
blameAuthor: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame author name or email"),
|
|
17842
18563
|
blameSha: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame commit SHA or prefix"),
|
|
17843
|
-
blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date")
|
|
18564
|
+
blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18565
|
+
blameUntil: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17844
18566
|
},
|
|
17845
18567
|
async (args) => {
|
|
17846
18568
|
return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "search", args.query, {
|
|
@@ -17851,7 +18573,8 @@ function registerMcpTools(server, runtime) {
|
|
|
17851
18573
|
contextLines: args.contextLines,
|
|
17852
18574
|
blameAuthor: args.blameAuthor,
|
|
17853
18575
|
blameSha: args.blameSha,
|
|
17854
|
-
blameSince: args.blameSince
|
|
18576
|
+
blameSince: args.blameSince,
|
|
18577
|
+
blameUntil: args.blameUntil
|
|
17855
18578
|
}, (results) => {
|
|
17856
18579
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} results for "${args.query}":
|
|
17857
18580
|
|
|
@@ -17871,7 +18594,8 @@ ${formatSearchResults(results, "score")}`;
|
|
|
17871
18594
|
chunkType: allowNullAsUndefined(z2.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
|
|
17872
18595
|
blameAuthor: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame author name or email"),
|
|
17873
18596
|
blameSha: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame commit SHA or prefix"),
|
|
17874
|
-
blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date")
|
|
18597
|
+
blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18598
|
+
blameUntil: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17875
18599
|
},
|
|
17876
18600
|
async (args) => {
|
|
17877
18601
|
return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "peek", args.query, {
|
|
@@ -17882,7 +18606,8 @@ ${formatSearchResults(results, "score")}`;
|
|
|
17882
18606
|
metadataOnly: true,
|
|
17883
18607
|
blameAuthor: args.blameAuthor,
|
|
17884
18608
|
blameSha: args.blameSha,
|
|
17885
|
-
blameSince: args.blameSince
|
|
18609
|
+
blameSince: args.blameSince,
|
|
18610
|
+
blameUntil: args.blameUntil
|
|
17886
18611
|
}, (results) => {
|
|
17887
18612
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} locations for "${args.query}":
|
|
17888
18613
|
|
|
@@ -17959,7 +18684,9 @@ ${formatCodebasePeek(results)}`;
|
|
|
17959
18684
|
fileType: allowNullAsUndefined(z2.string().optional()).describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
17960
18685
|
directory: allowNullAsUndefined(z2.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
17961
18686
|
chunkType: allowNullAsUndefined(z2.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
|
|
17962
|
-
excludeFile: allowNullAsUndefined(z2.string().optional()).describe("Exclude results from this file path")
|
|
18687
|
+
excludeFile: allowNullAsUndefined(z2.string().optional()).describe("Exclude results from this file path"),
|
|
18688
|
+
blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18689
|
+
blameUntil: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17963
18690
|
},
|
|
17964
18691
|
async (args) => {
|
|
17965
18692
|
const results = await findSimilarCode(runtime.projectRoot, runtime.host, args.code, {
|
|
@@ -17967,7 +18694,9 @@ ${formatCodebasePeek(results)}`;
|
|
|
17967
18694
|
fileType: args.fileType,
|
|
17968
18695
|
directory: args.directory,
|
|
17969
18696
|
chunkType: args.chunkType,
|
|
17970
|
-
excludeFile: args.excludeFile
|
|
18697
|
+
excludeFile: args.excludeFile,
|
|
18698
|
+
blameSince: args.blameSince,
|
|
18699
|
+
blameUntil: args.blameUntil
|
|
17971
18700
|
});
|
|
17972
18701
|
if (results.length === 0) {
|
|
17973
18702
|
return { content: [{ type: "text", text: "No similar code found. Try a different snippet or run index_codebase first." }] };
|
|
@@ -18071,6 +18800,37 @@ ${formatSearchResults(results)}` }] };
|
|
|
18071
18800
|
return { content: [{ type: "text", text: result.text }] };
|
|
18072
18801
|
}
|
|
18073
18802
|
);
|
|
18803
|
+
server.tool(
|
|
18804
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
18805
|
+
"Add a folder as a knowledge base to the semantic search index. The folder is indexed alongside the project code on the next index run. Provide an absolute path or a path relative to the project root. The path is written to the project-local host config of this MCP server (under the server project root), not to a user-global config, and the index is refreshed. Git blame metadata is collected only for files in the project git repo; knowledge-base files outside the repo remain searchable by content but have no blame. A knowledge base that appears in list_knowledge_bases but was inherited from a global config cannot be removed by this tool.",
|
|
18806
|
+
{
|
|
18807
|
+
path: z2.string().describe("Path to the folder to add as a knowledge base (absolute or relative to the project root)")
|
|
18808
|
+
},
|
|
18809
|
+
async (args) => {
|
|
18810
|
+
const result = addKnowledgeBase(runtime.projectRoot, runtime.host, args.path);
|
|
18811
|
+
return knowledgeBaseResult(result);
|
|
18812
|
+
}
|
|
18813
|
+
);
|
|
18814
|
+
server.tool(
|
|
18815
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18816
|
+
"List the configured knowledge base folders that the index includes alongside the project code. The list is the union of project-local and user-global knowledge bases; each entry shows the resolved path and whether it exists.",
|
|
18817
|
+
{},
|
|
18818
|
+
async () => {
|
|
18819
|
+
const result = listKnowledgeBases(runtime.projectRoot, runtime.host);
|
|
18820
|
+
return knowledgeBaseResult(result);
|
|
18821
|
+
}
|
|
18822
|
+
);
|
|
18823
|
+
server.tool(
|
|
18824
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE,
|
|
18825
|
+
"Remove a knowledge base folder from the semantic search index and refresh the index. The path must match a project-local configured path exactly. Knowledge bases inherited from a user-global config are not removable by this tool.",
|
|
18826
|
+
{
|
|
18827
|
+
path: z2.string().describe("Path of the knowledge base to remove (must match a project-local configured path exactly)")
|
|
18828
|
+
},
|
|
18829
|
+
async (args) => {
|
|
18830
|
+
const result = removeKnowledgeBase(runtime.projectRoot, runtime.host, args.path.trim());
|
|
18831
|
+
return knowledgeBaseResult(result);
|
|
18832
|
+
}
|
|
18833
|
+
);
|
|
18074
18834
|
}
|
|
18075
18835
|
|
|
18076
18836
|
// src/adapters/mcp/server.ts
|