opencode-codebase-index 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +6 -5
- package/commands/peek.md +2 -1
- package/commands/search.md +2 -1
- package/dist/cli.cjs +1020 -260
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1020 -260
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +788 -256
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +788 -256
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +5440 -2157
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +5429 -2146
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +3 -1
- package/skill/SKILL.md +12 -16
package/dist/cli.cjs
CHANGED
|
@@ -730,6 +730,17 @@ var EMBEDDING_MODELS = {
|
|
|
730
730
|
maxTokens: 2048,
|
|
731
731
|
costPer1MTokens: 0.15,
|
|
732
732
|
taskAble: true
|
|
733
|
+
},
|
|
734
|
+
"gemini-embedding-2": {
|
|
735
|
+
provider: "google",
|
|
736
|
+
model: "gemini-embedding-2",
|
|
737
|
+
// Keep a conservative, testable default embedding dimension. Gemini Embedding 2 supports
|
|
738
|
+
// flexible dimensions via outputDimensionality.
|
|
739
|
+
dimensions: 1536,
|
|
740
|
+
maxTokens: 8192,
|
|
741
|
+
costPer1MTokens: 0.15,
|
|
742
|
+
taskAble: false,
|
|
743
|
+
promptStyle: "embedding-2"
|
|
733
744
|
}
|
|
734
745
|
},
|
|
735
746
|
"openai": {
|
|
@@ -763,26 +774,15 @@ var EMBEDDING_MODELS = {
|
|
|
763
774
|
maxTokens: 512,
|
|
764
775
|
costPer1MTokens: 0
|
|
765
776
|
}
|
|
766
|
-
},
|
|
767
|
-
"github-copilot": {
|
|
768
|
-
"text-embedding-3-small": {
|
|
769
|
-
provider: "github-copilot",
|
|
770
|
-
model: "text-embedding-3-small",
|
|
771
|
-
dimensions: 1536,
|
|
772
|
-
maxTokens: 8191,
|
|
773
|
-
costPer1MTokens: 0
|
|
774
|
-
}
|
|
775
777
|
}
|
|
776
778
|
};
|
|
777
779
|
var DEFAULT_PROVIDER_MODELS = {
|
|
778
|
-
"github-copilot": "text-embedding-3-small",
|
|
779
780
|
"openai": "text-embedding-3-small",
|
|
780
781
|
"google": "gemini-embedding-001",
|
|
781
782
|
"ollama": "nomic-embed-text"
|
|
782
783
|
};
|
|
783
784
|
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
784
785
|
"ollama",
|
|
785
|
-
"github-copilot",
|
|
786
786
|
"openai",
|
|
787
787
|
"google"
|
|
788
788
|
];
|
|
@@ -808,6 +808,9 @@ function getDefaultIndexingConfig() {
|
|
|
808
808
|
maxDepth: 5,
|
|
809
809
|
maxFilesPerDirectory: 100,
|
|
810
810
|
fallbackToTextOnMaxChunks: true,
|
|
811
|
+
// Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi
|
|
812
|
+
// fallback used when a native caller omits the argument).
|
|
813
|
+
linesPerChunk: 30,
|
|
811
814
|
gitBlame: { enabled: false }
|
|
812
815
|
};
|
|
813
816
|
}
|
|
@@ -941,6 +944,7 @@ function parseConfig(raw) {
|
|
|
941
944
|
maxDepth: typeof rawIndexing.maxDepth === "number" ? rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth : defaultIndexing.maxDepth,
|
|
942
945
|
maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
|
|
943
946
|
fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
|
|
947
|
+
linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk,
|
|
944
948
|
gitBlame: {
|
|
945
949
|
enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof rawIndexing.gitBlame.enabled === "boolean" ? rawIndexing.gitBlame.enabled : defaultIndexing.gitBlame.enabled
|
|
946
950
|
}
|
|
@@ -983,6 +987,7 @@ function parseConfig(raw) {
|
|
|
983
987
|
let embeddingModel;
|
|
984
988
|
let customProvider;
|
|
985
989
|
let reranker;
|
|
990
|
+
const githubCopilotDeprecationMessage = '`embeddingProvider: "github-copilot"` is deprecated and no longer available. Migrate existing configs to `embeddingProvider: "google"` and select an explicit Google model. For existing indexes, run `index_codebase` with `force: true` after changing to `gemini-embedding-001` or `gemini-embedding-2` to rebuild embeddings. See docs/configuration.md for details.';
|
|
986
991
|
if (embeddingProviderValue === "custom") {
|
|
987
992
|
embeddingProvider = "custom";
|
|
988
993
|
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
@@ -1022,6 +1027,8 @@ function parseConfig(raw) {
|
|
|
1022
1027
|
} else if (rawEmbeddingModel) {
|
|
1023
1028
|
embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
1024
1029
|
}
|
|
1030
|
+
} else if (embeddingProviderValue === "github-copilot") {
|
|
1031
|
+
throw new Error(githubCopilotDeprecationMessage);
|
|
1025
1032
|
} else {
|
|
1026
1033
|
embeddingProvider = "auto";
|
|
1027
1034
|
}
|
|
@@ -1052,10 +1059,21 @@ function parseConfig(raw) {
|
|
|
1052
1059
|
timeoutMs: typeof rawReranker.timeoutMs === "number" ? Math.max(1e3, Math.floor(rawReranker.timeoutMs)) : 1e4
|
|
1053
1060
|
};
|
|
1054
1061
|
}
|
|
1062
|
+
const rawEmbedding = input.embedding && typeof input.embedding === "object" ? input.embedding : {};
|
|
1063
|
+
const rawEmbeddingBatch = rawEmbedding.batch && typeof rawEmbedding.batch === "object" ? rawEmbedding.batch : null;
|
|
1064
|
+
const embeddingMaxBatchItems = typeof rawEmbeddingBatch?.maxBatchItems === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchItems) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchItems)) : void 0;
|
|
1065
|
+
const embeddingMaxBatchTokens = typeof rawEmbeddingBatch?.maxBatchTokens === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchTokens) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchTokens)) : void 0;
|
|
1066
|
+
const embedding = embeddingMaxBatchItems !== void 0 || embeddingMaxBatchTokens !== void 0 ? {
|
|
1067
|
+
batch: {
|
|
1068
|
+
...embeddingMaxBatchItems !== void 0 ? { maxBatchItems: embeddingMaxBatchItems } : {},
|
|
1069
|
+
...embeddingMaxBatchTokens !== void 0 ? { maxBatchTokens: embeddingMaxBatchTokens } : {}
|
|
1070
|
+
}
|
|
1071
|
+
} : {};
|
|
1055
1072
|
return {
|
|
1056
1073
|
embeddingProvider,
|
|
1057
1074
|
embeddingModel,
|
|
1058
1075
|
customProvider,
|
|
1076
|
+
embedding,
|
|
1059
1077
|
scope: isValidScope(scopeValue) ? scopeValue : "project",
|
|
1060
1078
|
include: includeValue ?? DEFAULT_INCLUDE,
|
|
1061
1079
|
exclude: excludeValue ?? DEFAULT_EXCLUDE,
|
|
@@ -2551,8 +2569,6 @@ async function tryDetectProvider() {
|
|
|
2551
2569
|
}
|
|
2552
2570
|
async function getProviderCredentials(provider) {
|
|
2553
2571
|
switch (provider) {
|
|
2554
|
-
case "github-copilot":
|
|
2555
|
-
return getGitHubCopilotCredentials();
|
|
2556
2572
|
case "openai":
|
|
2557
2573
|
return getOpenAICredentials();
|
|
2558
2574
|
case "google":
|
|
@@ -2563,22 +2579,6 @@ async function getProviderCredentials(provider) {
|
|
|
2563
2579
|
return null;
|
|
2564
2580
|
}
|
|
2565
2581
|
}
|
|
2566
|
-
function getGitHubCopilotCredentials() {
|
|
2567
|
-
const authData = loadOpenCodeAuth();
|
|
2568
|
-
const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
|
|
2569
|
-
if (!copilotAuth || copilotAuth.type !== "oauth") {
|
|
2570
|
-
return null;
|
|
2571
|
-
}
|
|
2572
|
-
const auth = copilotAuth;
|
|
2573
|
-
const baseUrl = auth.enterpriseUrl ? `https://copilot-api.${auth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
|
|
2574
|
-
return {
|
|
2575
|
-
provider: "github-copilot",
|
|
2576
|
-
baseUrl,
|
|
2577
|
-
refreshToken: copilotAuth.refresh,
|
|
2578
|
-
accessToken: copilotAuth.access,
|
|
2579
|
-
tokenExpires: copilotAuth.expires
|
|
2580
|
-
};
|
|
2581
|
-
}
|
|
2582
2582
|
function getOpenAICredentials() {
|
|
2583
2583
|
const authData = loadOpenCodeAuth();
|
|
2584
2584
|
const openaiAuth = authData["openai"];
|
|
@@ -2704,8 +2704,6 @@ async function tryDetectOllamaProvider() {
|
|
|
2704
2704
|
}
|
|
2705
2705
|
function getProviderDisplayName(provider) {
|
|
2706
2706
|
switch (provider) {
|
|
2707
|
-
case "github-copilot":
|
|
2708
|
-
return "GitHub Copilot";
|
|
2709
2707
|
case "openai":
|
|
2710
2708
|
return "OpenAI";
|
|
2711
2709
|
case "google":
|
|
@@ -2930,44 +2928,6 @@ var CustomEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
2930
2928
|
}
|
|
2931
2929
|
};
|
|
2932
2930
|
|
|
2933
|
-
// src/embeddings/providers/github-copilot.ts
|
|
2934
|
-
var GitHubCopilotEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
2935
|
-
constructor(credentials, modelInfo) {
|
|
2936
|
-
super(credentials, modelInfo);
|
|
2937
|
-
}
|
|
2938
|
-
getToken() {
|
|
2939
|
-
if (!this.credentials.refreshToken) {
|
|
2940
|
-
throw new Error("No OAuth token available for GitHub");
|
|
2941
|
-
}
|
|
2942
|
-
return this.credentials.refreshToken;
|
|
2943
|
-
}
|
|
2944
|
-
async embedBatch(texts) {
|
|
2945
|
-
const token = this.getToken();
|
|
2946
|
-
const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
|
|
2947
|
-
method: "POST",
|
|
2948
|
-
headers: {
|
|
2949
|
-
Authorization: `Bearer ${token}`,
|
|
2950
|
-
"Content-Type": "application/json",
|
|
2951
|
-
Accept: "application/vnd.github+json",
|
|
2952
|
-
"X-GitHub-Api-Version": "2022-11-28"
|
|
2953
|
-
},
|
|
2954
|
-
body: JSON.stringify({
|
|
2955
|
-
model: `openai/${this.modelInfo.model}`,
|
|
2956
|
-
input: texts
|
|
2957
|
-
})
|
|
2958
|
-
});
|
|
2959
|
-
if (!response.ok) {
|
|
2960
|
-
const error = (await response.text()).slice(0, 500);
|
|
2961
|
-
throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
|
|
2962
|
-
}
|
|
2963
|
-
const data = await response.json();
|
|
2964
|
-
return {
|
|
2965
|
-
embeddings: data.data.map((d) => d.embedding),
|
|
2966
|
-
totalTokensUsed: data.usage.total_tokens
|
|
2967
|
-
};
|
|
2968
|
-
}
|
|
2969
|
-
};
|
|
2970
|
-
|
|
2971
2931
|
// src/embeddings/providers/google.ts
|
|
2972
2932
|
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddingProvider {
|
|
2973
2933
|
static BATCH_SIZE = 20;
|
|
@@ -2975,24 +2935,30 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
2975
2935
|
super(credentials, modelInfo);
|
|
2976
2936
|
}
|
|
2977
2937
|
async embedQuery(query) {
|
|
2978
|
-
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2979
|
-
const
|
|
2938
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2939
|
+
const texts = [
|
|
2940
|
+
this.modelInfo.model === "gemini-embedding-2" ? `task: code retrieval | query: ${query}` : query
|
|
2941
|
+
];
|
|
2942
|
+
const result = await this.embedWithTaskType(texts, taskType);
|
|
2980
2943
|
return {
|
|
2981
2944
|
embedding: result.embeddings[0],
|
|
2982
2945
|
tokensUsed: result.totalTokensUsed
|
|
2983
2946
|
};
|
|
2984
2947
|
}
|
|
2985
2948
|
async embedDocument(document) {
|
|
2986
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2987
|
-
const result = await this.embedWithTaskType([
|
|
2949
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2950
|
+
const result = await this.embedWithTaskType([
|
|
2951
|
+
this.modelInfo.model === "gemini-embedding-2" ? `title: none | text: ${document}` : document
|
|
2952
|
+
], taskType);
|
|
2988
2953
|
return {
|
|
2989
2954
|
embedding: result.embeddings[0],
|
|
2990
2955
|
tokensUsed: result.totalTokensUsed
|
|
2991
2956
|
};
|
|
2992
2957
|
}
|
|
2993
2958
|
async embedBatch(texts) {
|
|
2994
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2995
|
-
|
|
2959
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2960
|
+
const formattedTexts = this.modelInfo.model === "gemini-embedding-2" ? texts.map((text) => `title: none | text: ${text}`) : texts;
|
|
2961
|
+
return this.embedWithTaskType(formattedTexts, taskType);
|
|
2996
2962
|
}
|
|
2997
2963
|
async embedWithTaskType(texts, taskType) {
|
|
2998
2964
|
const batches = [];
|
|
@@ -3042,6 +3008,10 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
3042
3008
|
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddingProvider {
|
|
3043
3009
|
static MIN_TRUNCATION_CHARS = 512;
|
|
3044
3010
|
static REQUEST_TIMEOUT_MS = 12e4;
|
|
3011
|
+
// Set when /api/embed returns 404 so subsequent multi-text batches skip the
|
|
3012
|
+
// batched endpoint and go straight to the legacy per-text path (one probe per
|
|
3013
|
+
// old ollama install, not one probe per batch).
|
|
3014
|
+
batchEndpointUnavailable = false;
|
|
3045
3015
|
constructor(credentials, modelInfo) {
|
|
3046
3016
|
super(credentials, modelInfo);
|
|
3047
3017
|
}
|
|
@@ -3059,6 +3029,21 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3059
3029
|
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
3060
3030
|
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
3061
3031
|
}
|
|
3032
|
+
// True for a 404 from the newer /api/embed endpoint, i.e. an ollama version that
|
|
3033
|
+
// does not provide it. embedBatch uses this to fall back to the legacy per-text
|
|
3034
|
+
// /api/embeddings path so old ollama installs do not regress.
|
|
3035
|
+
isBatchEndpointUnavailableError(error) {
|
|
3036
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3037
|
+
return message.includes("Ollama /api/embed not available");
|
|
3038
|
+
}
|
|
3039
|
+
// True for a malformed /api/embed response (wrong vector count or a bad vector).
|
|
3040
|
+
// embedBatch falls back to the per-text path on this so a bad batch response
|
|
3041
|
+
// re-embeds each text cleanly. A text that then fails per-text is not isolated
|
|
3042
|
+
// here; it is isolated on the recovery run, which re-embeds one text per request.
|
|
3043
|
+
isBatchValidationError(error) {
|
|
3044
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3045
|
+
return message.includes("invalid embedding batch");
|
|
3046
|
+
}
|
|
3062
3047
|
buildTruncationCandidates(text) {
|
|
3063
3048
|
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
3064
3049
|
const candidateLimits = /* @__PURE__ */ new Set();
|
|
@@ -3160,7 +3145,74 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3160
3145
|
tokensUsed: this.estimateTokens(text)
|
|
3161
3146
|
};
|
|
3162
3147
|
}
|
|
3163
|
-
|
|
3148
|
+
// Embeds many texts in one POST /api/embed request (input: string[]). Ollama
|
|
3149
|
+
// encodes each input independently, so the model context length applies per input
|
|
3150
|
+
// (the upstream splitter already bounds each input), not over the batch. This
|
|
3151
|
+
// amortizes N HTTP round-trips into one.
|
|
3152
|
+
async embedMany(texts) {
|
|
3153
|
+
const controller = new AbortController();
|
|
3154
|
+
const timeout = setTimeout(
|
|
3155
|
+
() => controller.abort(),
|
|
3156
|
+
_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS
|
|
3157
|
+
);
|
|
3158
|
+
let response;
|
|
3159
|
+
try {
|
|
3160
|
+
response = await fetch(`${this.credentials.baseUrl}/api/embed`, {
|
|
3161
|
+
method: "POST",
|
|
3162
|
+
headers: {
|
|
3163
|
+
"Content-Type": "application/json"
|
|
3164
|
+
},
|
|
3165
|
+
body: JSON.stringify({
|
|
3166
|
+
model: this.modelInfo.model,
|
|
3167
|
+
input: texts,
|
|
3168
|
+
truncate: false
|
|
3169
|
+
}),
|
|
3170
|
+
signal: controller.signal
|
|
3171
|
+
});
|
|
3172
|
+
} catch (error) {
|
|
3173
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
3174
|
+
throw new Error(
|
|
3175
|
+
`Ollama embedding request timed out after ${_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS}ms`
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
throw error;
|
|
3179
|
+
} finally {
|
|
3180
|
+
clearTimeout(timeout);
|
|
3181
|
+
}
|
|
3182
|
+
if (!response.ok) {
|
|
3183
|
+
const error = (await response.text()).slice(0, 500);
|
|
3184
|
+
if (response.status === 404) {
|
|
3185
|
+
throw new Error(`Ollama /api/embed not available: ${response.status} - ${error}`);
|
|
3186
|
+
}
|
|
3187
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
3188
|
+
}
|
|
3189
|
+
let parsed;
|
|
3190
|
+
try {
|
|
3191
|
+
parsed = await response.json();
|
|
3192
|
+
} catch {
|
|
3193
|
+
throw new Error(
|
|
3194
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
3195
|
+
);
|
|
3196
|
+
}
|
|
3197
|
+
const data = parsed && typeof parsed === "object" ? parsed : {};
|
|
3198
|
+
if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length || data.embeddings.some(
|
|
3199
|
+
(value) => !Array.isArray(value) || value.length !== this.modelInfo.dimensions || value.some((v) => typeof v !== "number" || !Number.isFinite(v))
|
|
3200
|
+
)) {
|
|
3201
|
+
throw new Error(
|
|
3202
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
3203
|
+
);
|
|
3204
|
+
}
|
|
3205
|
+
return {
|
|
3206
|
+
embeddings: data.embeddings,
|
|
3207
|
+
totalTokensUsed: texts.reduce((sum, text) => sum + this.estimateTokens(text), 0)
|
|
3208
|
+
};
|
|
3209
|
+
}
|
|
3210
|
+
// Per-text /api/embeddings path shared by the single-text fast path and the
|
|
3211
|
+
// batch fallback. Uses the legacy endpoint one text at a time, so each text gets
|
|
3212
|
+
// its own truncation safety net and a vector validated on its own. A text that
|
|
3213
|
+
// hard-fails per-text throws here and fails the whole request batch; the recovery
|
|
3214
|
+
// run re-embeds one text per request to isolate it.
|
|
3215
|
+
async embedOneByOne(texts) {
|
|
3164
3216
|
const results = [];
|
|
3165
3217
|
for (const text of texts) {
|
|
3166
3218
|
results.push(await this.embedSingleWithFallback(text));
|
|
@@ -3170,6 +3222,26 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3170
3222
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
3171
3223
|
};
|
|
3172
3224
|
}
|
|
3225
|
+
async embedBatch(texts) {
|
|
3226
|
+
if (texts.length === 0) {
|
|
3227
|
+
return { embeddings: [], totalTokensUsed: 0 };
|
|
3228
|
+
}
|
|
3229
|
+
if (texts.length === 1 || this.batchEndpointUnavailable) {
|
|
3230
|
+
return this.embedOneByOne(texts);
|
|
3231
|
+
}
|
|
3232
|
+
try {
|
|
3233
|
+
return await this.embedMany(texts);
|
|
3234
|
+
} catch (error) {
|
|
3235
|
+
if (this.isBatchEndpointUnavailableError(error)) {
|
|
3236
|
+
this.batchEndpointUnavailable = true;
|
|
3237
|
+
return this.embedOneByOne(texts);
|
|
3238
|
+
}
|
|
3239
|
+
if (!this.isContextLengthError(error) && !this.isBatchValidationError(error)) {
|
|
3240
|
+
throw error;
|
|
3241
|
+
}
|
|
3242
|
+
return this.embedOneByOne(texts);
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3173
3245
|
};
|
|
3174
3246
|
|
|
3175
3247
|
// src/embeddings/providers/openai.ts
|
|
@@ -3204,8 +3276,6 @@ var OpenAIEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
3204
3276
|
// src/embeddings/provider.ts
|
|
3205
3277
|
function createEmbeddingProvider(configuredProviderInfo) {
|
|
3206
3278
|
switch (configuredProviderInfo.provider) {
|
|
3207
|
-
case "github-copilot":
|
|
3208
|
-
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
3209
3279
|
case "openai":
|
|
3210
3280
|
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
3211
3281
|
case "google":
|
|
@@ -4224,12 +4294,12 @@ try {
|
|
|
4224
4294
|
}
|
|
4225
4295
|
|
|
4226
4296
|
// src/native/parsing.ts
|
|
4227
|
-
function parseFileAsText(filePath, content) {
|
|
4228
|
-
const result = native.parseFileAsText(filePath, content);
|
|
4297
|
+
function parseFileAsText(filePath, content, linesPerChunk) {
|
|
4298
|
+
const result = native.parseFileAsText(filePath, content, linesPerChunk);
|
|
4229
4299
|
return result.map(mapChunk);
|
|
4230
4300
|
}
|
|
4231
|
-
function parseFiles(files) {
|
|
4232
|
-
const result = native.parseFiles(files);
|
|
4301
|
+
function parseFiles(files, linesPerChunk) {
|
|
4302
|
+
const result = native.parseFiles(files, linesPerChunk);
|
|
4233
4303
|
return result.map((f) => ({
|
|
4234
4304
|
path: f.path,
|
|
4235
4305
|
chunks: f.chunks.map(mapChunk),
|
|
@@ -4306,13 +4376,13 @@ var VectorStore = class {
|
|
|
4306
4376
|
const metadata = items.map((i) => JSON.stringify(i.metadata));
|
|
4307
4377
|
this.inner.addBatch(ids, vectors, metadata);
|
|
4308
4378
|
}
|
|
4309
|
-
search(queryVector, limit = 10) {
|
|
4379
|
+
search(queryVector, limit = 10, allowedIds) {
|
|
4310
4380
|
if (queryVector.length !== this.dimensions) {
|
|
4311
4381
|
throw new Error(
|
|
4312
4382
|
`Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
|
|
4313
4383
|
);
|
|
4314
4384
|
}
|
|
4315
|
-
const results = this.inner.search(queryVector, limit);
|
|
4385
|
+
const results = allowedIds === void 0 ? this.inner.search(queryVector, limit) : this.inner.searchFiltered(queryVector, limit, allowedIds);
|
|
4316
4386
|
return results.map((r) => ({
|
|
4317
4387
|
id: r.id,
|
|
4318
4388
|
score: r.score,
|
|
@@ -4540,6 +4610,10 @@ var Database = class _Database {
|
|
|
4540
4610
|
this.throwIfClosed();
|
|
4541
4611
|
return this.inner.getBranchChunkIds(branch);
|
|
4542
4612
|
}
|
|
4613
|
+
getChunkIdsByBlameDate(since, until) {
|
|
4614
|
+
this.throwIfClosed();
|
|
4615
|
+
return this.inner.getChunkIdsByBlameDate(since, until);
|
|
4616
|
+
}
|
|
4543
4617
|
getBranchDelta(branch, baseBranch) {
|
|
4544
4618
|
this.throwIfClosed();
|
|
4545
4619
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
@@ -5501,6 +5575,9 @@ function resolveWorktreeFallbackPath(projectRoot, relativePath) {
|
|
|
5501
5575
|
const fallbackPath = path10.join(mainRepoRoot, relativePath);
|
|
5502
5576
|
return (0, import_fs7.existsSync)(fallbackPath) ? fallbackPath : null;
|
|
5503
5577
|
}
|
|
5578
|
+
function getHostProjectConfigRelativePath(host) {
|
|
5579
|
+
return getProjectConfigRelativePath(host);
|
|
5580
|
+
}
|
|
5504
5581
|
function getProjectConfigCandidatePaths(projectRoot, host) {
|
|
5505
5582
|
const candidates = [path10.join(projectRoot, getProjectConfigRelativePath(host))];
|
|
5506
5583
|
if (host !== "opencode") {
|
|
@@ -5592,6 +5669,9 @@ function resolveProjectConfigPath(projectRoot, host) {
|
|
|
5592
5669
|
const candidates = getProjectConfigCandidatePaths(projectRoot, host);
|
|
5593
5670
|
return candidates.find((candidate) => (0, import_fs7.existsSync)(candidate)) ?? path10.join(projectRoot, getProjectConfigRelativePath(host));
|
|
5594
5671
|
}
|
|
5672
|
+
function resolveWritableProjectConfigPath(projectRoot, host) {
|
|
5673
|
+
return path10.join(projectRoot, getProjectConfigRelativePath(host));
|
|
5674
|
+
}
|
|
5595
5675
|
function resolveProjectIndexPath(projectRoot, scope, host) {
|
|
5596
5676
|
if (scope === "global") {
|
|
5597
5677
|
return resolveGlobalIndexPath(host);
|
|
@@ -6297,6 +6377,10 @@ function scoreCandidate(query, intent, candidate, originalIndex) {
|
|
|
6297
6377
|
let boost = 0;
|
|
6298
6378
|
if (intent.primary === "conceptual") {
|
|
6299
6379
|
boost += Math.min(0.14, overlap * 0.14);
|
|
6380
|
+
if (intent.preferSourcePaths) {
|
|
6381
|
+
boost += implementationPath ? 0.32 : 0;
|
|
6382
|
+
if (testPath || fixturePath || docsPath) boost -= 0.35;
|
|
6383
|
+
}
|
|
6300
6384
|
if (generatedOrVendor) boost -= 0.18;
|
|
6301
6385
|
if (importChunk || weakContainer) boost -= 0.04;
|
|
6302
6386
|
} else if (intent.primary === "test") {
|
|
@@ -7211,6 +7295,19 @@ function parseOwner(value) {
|
|
|
7211
7295
|
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
7212
7296
|
if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
|
|
7213
7297
|
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
7298
|
+
if (candidate.recoveryProtocolVersion !== void 0 && candidate.recoveryProtocolVersion !== 1) return null;
|
|
7299
|
+
if (candidate.projectRoot !== void 0 && typeof candidate.projectRoot !== "string") return null;
|
|
7300
|
+
if (candidate.scopedRoots !== void 0) {
|
|
7301
|
+
if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) {
|
|
7302
|
+
return null;
|
|
7303
|
+
}
|
|
7304
|
+
}
|
|
7305
|
+
if (candidate.clearRecovery !== void 0) {
|
|
7306
|
+
const recovery = candidate.clearRecovery;
|
|
7307
|
+
if (typeof recovery !== "object" || recovery === null || recovery.phase !== "clearing" || typeof recovery.embeddingProvider !== "string" || recovery.embeddingProvider.length === 0 || typeof recovery.embeddingModel !== "string" || recovery.embeddingModel.length === 0 || !Number.isInteger(recovery.embeddingDimensions) || (recovery.embeddingDimensions ?? 0) <= 0 || typeof recovery.embeddingStrategyVersion !== "string" || recovery.embeddingStrategyVersion.length === 0 || recovery.compatibilityDecision !== "compatible" && recovery.compatibilityDecision !== "embedding-strategy-mismatch" && recovery.compatibilityDecision !== "incompatible" || candidate.operation !== "clear" && candidate.operation !== "force-index") {
|
|
7308
|
+
return null;
|
|
7309
|
+
}
|
|
7310
|
+
}
|
|
7214
7311
|
return candidate;
|
|
7215
7312
|
}
|
|
7216
7313
|
function parseReclaimOwner(value) {
|
|
@@ -7451,13 +7548,18 @@ function isTransientIndexLockContention(error) {
|
|
|
7451
7548
|
if (!isIndexLockContentionError(error) || !("reason" in error)) return false;
|
|
7452
7549
|
return error.reason === "active" || error.reason === "reclaiming";
|
|
7453
7550
|
}
|
|
7454
|
-
function acquireIndexLock(indexPath, operation) {
|
|
7551
|
+
function acquireIndexLock(indexPath, operation, recoveryScope) {
|
|
7455
7552
|
(0, import_fs9.mkdirSync)(indexPath, { recursive: true });
|
|
7456
7553
|
const canonicalIndexPath = import_fs9.realpathSync.native(indexPath);
|
|
7457
7554
|
const lockPath = path13.join(canonicalIndexPath, "indexing.lock");
|
|
7458
7555
|
cleanupDeadPublicationCandidates(canonicalIndexPath);
|
|
7459
7556
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
7460
|
-
const owner = createOwner(operation)
|
|
7557
|
+
const owner = recoveryScope === void 0 ? createOwner(operation) : {
|
|
7558
|
+
...createOwner(operation),
|
|
7559
|
+
recoveryProtocolVersion: 1,
|
|
7560
|
+
projectRoot: recoveryScope.projectRoot,
|
|
7561
|
+
scopedRoots: recoveryScope.scopedRoots
|
|
7562
|
+
};
|
|
7461
7563
|
if (publishJsonDirectory(lockPath, owner)) {
|
|
7462
7564
|
const lease = {
|
|
7463
7565
|
canonicalIndexPath,
|
|
@@ -7522,6 +7624,33 @@ function releaseIndexLock(lease) {
|
|
|
7522
7624
|
}
|
|
7523
7625
|
return true;
|
|
7524
7626
|
}
|
|
7627
|
+
function setIndexLockClearRecoveryState(lease, clearRecovery) {
|
|
7628
|
+
const currentOwner = readDirectoryOwner(lease.lockPath);
|
|
7629
|
+
if (!currentOwner || !sameOwner(currentOwner, lease.owner)) {
|
|
7630
|
+
throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
7631
|
+
}
|
|
7632
|
+
const nextOwner = { ...currentOwner };
|
|
7633
|
+
if (clearRecovery === null) {
|
|
7634
|
+
delete nextOwner.clearRecovery;
|
|
7635
|
+
} else {
|
|
7636
|
+
nextOwner.clearRecovery = clearRecovery;
|
|
7637
|
+
}
|
|
7638
|
+
const ownerPath = path13.join(lease.lockPath, OWNER_FILE_NAME);
|
|
7639
|
+
const temporaryPath = path13.join(
|
|
7640
|
+
lease.lockPath,
|
|
7641
|
+
`${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${(0, import_crypto2.randomUUID)()}`
|
|
7642
|
+
);
|
|
7643
|
+
try {
|
|
7644
|
+
(0, import_fs9.writeFileSync)(temporaryPath, JSON.stringify(nextOwner), {
|
|
7645
|
+
encoding: "utf-8",
|
|
7646
|
+
flag: "wx",
|
|
7647
|
+
mode: 384
|
|
7648
|
+
});
|
|
7649
|
+
retryTransientFilesystemOperation(() => (0, import_fs9.renameSync)(temporaryPath, ownerPath));
|
|
7650
|
+
} finally {
|
|
7651
|
+
if ((0, import_fs9.existsSync)(temporaryPath)) (0, import_fs9.rmSync)(temporaryPath, { force: true });
|
|
7652
|
+
}
|
|
7653
|
+
}
|
|
7525
7654
|
function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
|
|
7526
7655
|
if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
|
|
7527
7656
|
temporaryCounter += 1;
|
|
@@ -7759,6 +7888,18 @@ function createFailedBatchWriter(targetPath) {
|
|
|
7759
7888
|
temporaryPath
|
|
7760
7889
|
};
|
|
7761
7890
|
}
|
|
7891
|
+
function writeFailedBatchRecords(targetPath, records) {
|
|
7892
|
+
const writer = createFailedBatchWriter(targetPath);
|
|
7893
|
+
try {
|
|
7894
|
+
for (const record of records) {
|
|
7895
|
+
writer.write(record);
|
|
7896
|
+
}
|
|
7897
|
+
writer.commit();
|
|
7898
|
+
} catch (error) {
|
|
7899
|
+
writer.cleanup();
|
|
7900
|
+
throw error;
|
|
7901
|
+
}
|
|
7902
|
+
}
|
|
7762
7903
|
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
7763
7904
|
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
7764
7905
|
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
@@ -8041,14 +8182,18 @@ function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
|
8041
8182
|
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
8042
8183
|
return Math.min(2e3, maxChunkTokens);
|
|
8043
8184
|
}
|
|
8044
|
-
|
|
8045
|
-
|
|
8046
|
-
|
|
8047
|
-
|
|
8048
|
-
|
|
8049
|
-
};
|
|
8185
|
+
var DEFAULT_OLLAMA_MAX_BATCH_ITEMS = 16;
|
|
8186
|
+
var DEFAULT_OLLAMA_MAX_BATCH_TOKENS = 65536;
|
|
8187
|
+
function getDynamicBatchOptions(provider, embeddingBatch) {
|
|
8188
|
+
if (provider.provider !== "ollama") {
|
|
8189
|
+
return {};
|
|
8050
8190
|
}
|
|
8051
|
-
|
|
8191
|
+
const base = { maxBatchTokens: DEFAULT_OLLAMA_MAX_BATCH_TOKENS, maxBatchItems: DEFAULT_OLLAMA_MAX_BATCH_ITEMS };
|
|
8192
|
+
return {
|
|
8193
|
+
...base,
|
|
8194
|
+
...typeof embeddingBatch?.maxBatchTokens === "number" && Number.isFinite(embeddingBatch.maxBatchTokens) ? { maxBatchTokens: embeddingBatch.maxBatchTokens } : {},
|
|
8195
|
+
...typeof embeddingBatch?.maxBatchItems === "number" && Number.isFinite(embeddingBatch.maxBatchItems) ? { maxBatchItems: embeddingBatch.maxBatchItems } : {}
|
|
8196
|
+
};
|
|
8052
8197
|
}
|
|
8053
8198
|
function isSqliteCorruptionError(error) {
|
|
8054
8199
|
const message = getErrorMessage3(error).toLowerCase();
|
|
@@ -8066,6 +8211,14 @@ function getPendingChunkId(rawChunk) {
|
|
|
8066
8211
|
const id = rawChunk.id;
|
|
8067
8212
|
return typeof id === "string" ? id : null;
|
|
8068
8213
|
}
|
|
8214
|
+
function parseBlameTimestamp(value, endOfDay) {
|
|
8215
|
+
let timestampMs = Date.parse(value);
|
|
8216
|
+
if (Number.isNaN(timestampMs)) return null;
|
|
8217
|
+
if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(value.trim())) {
|
|
8218
|
+
timestampMs += 24 * 60 * 60 * 1e3 - 1;
|
|
8219
|
+
}
|
|
8220
|
+
return Math.floor(timestampMs / 1e3);
|
|
8221
|
+
}
|
|
8069
8222
|
function metadataFromBlame(blame) {
|
|
8070
8223
|
if (!blame) {
|
|
8071
8224
|
return {};
|
|
@@ -8212,7 +8365,7 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
|
|
|
8212
8365
|
const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
|
|
8213
8366
|
return [...promoted, ...remainder];
|
|
8214
8367
|
}
|
|
8215
|
-
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
8368
|
+
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source", allowNonSourcePaths = false) {
|
|
8216
8369
|
if (!prioritizeSourcePaths) {
|
|
8217
8370
|
return [];
|
|
8218
8371
|
}
|
|
@@ -8232,7 +8385,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8232
8385
|
if (!isImplementationChunkType(chunkType)) {
|
|
8233
8386
|
return false;
|
|
8234
8387
|
}
|
|
8235
|
-
if (!isLikelyImplementationPath2(chunk.filePath)) {
|
|
8388
|
+
if (!allowNonSourcePaths && !isLikelyImplementationPath2(chunk.filePath)) {
|
|
8236
8389
|
return false;
|
|
8237
8390
|
}
|
|
8238
8391
|
const nameLower = (chunk.name ?? "").toLowerCase();
|
|
@@ -8296,7 +8449,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8296
8449
|
}
|
|
8297
8450
|
foundCoveringChunk = upsertChunkCandidate(chunk, identifier, normalizedIdentifier) || foundCoveringChunk;
|
|
8298
8451
|
}
|
|
8299
|
-
if (foundCoveringChunk || !isLikelyImplementationPath2(symbol.filePath)) {
|
|
8452
|
+
if (foundCoveringChunk || !allowNonSourcePaths && !isLikelyImplementationPath2(symbol.filePath)) {
|
|
8300
8453
|
continue;
|
|
8301
8454
|
}
|
|
8302
8455
|
const symbolName = symbol.name.toLowerCase();
|
|
@@ -8350,7 +8503,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8350
8503
|
const ranked = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
8351
8504
|
if (ranked.length === 0) {
|
|
8352
8505
|
const implementationFallback = fallbackCandidates.filter(
|
|
8353
|
-
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && isLikelyImplementationPath2(candidate.metadata.filePath)
|
|
8506
|
+
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && (allowNonSourcePaths || isLikelyImplementationPath2(candidate.metadata.filePath))
|
|
8354
8507
|
);
|
|
8355
8508
|
for (const candidate of implementationFallback) {
|
|
8356
8509
|
const nameLower = (candidate.metadata.name ?? "").toLowerCase();
|
|
@@ -8466,10 +8619,16 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
8466
8619
|
return false;
|
|
8467
8620
|
}
|
|
8468
8621
|
if (options?.blameSince) {
|
|
8469
|
-
const
|
|
8470
|
-
if (
|
|
8622
|
+
const since = parseBlameTimestamp(options.blameSince, false);
|
|
8623
|
+
if (since === null) return false;
|
|
8471
8624
|
const committedAt = candidate.metadata.blameCommittedAt;
|
|
8472
|
-
if (committedAt === void 0 || committedAt <
|
|
8625
|
+
if (committedAt === void 0 || committedAt < since) return false;
|
|
8626
|
+
}
|
|
8627
|
+
if (options?.blameUntil) {
|
|
8628
|
+
const until = parseBlameTimestamp(options.blameUntil, true);
|
|
8629
|
+
if (until === null) return false;
|
|
8630
|
+
const committedAt = candidate.metadata.blameCommittedAt;
|
|
8631
|
+
if (committedAt === void 0 || committedAt > until) return false;
|
|
8473
8632
|
}
|
|
8474
8633
|
return true;
|
|
8475
8634
|
}
|
|
@@ -8525,9 +8684,10 @@ var Indexer = class _Indexer {
|
|
|
8525
8684
|
writerArtifactFingerprint = null;
|
|
8526
8685
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
8527
8686
|
fileBatchLimits;
|
|
8687
|
+
checkpointIntervalChunks;
|
|
8528
8688
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
8529
8689
|
this.projectRoot = projectRoot;
|
|
8530
|
-
this.projectIdentityHash =
|
|
8690
|
+
this.projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
8531
8691
|
this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot;
|
|
8532
8692
|
this.branchNameOverride = runtimeOptions.branchName;
|
|
8533
8693
|
this.catalogIdentityOverride = runtimeOptions.catalogIdentity;
|
|
@@ -8537,6 +8697,7 @@ var Indexer = class _Indexer {
|
|
|
8537
8697
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
8538
8698
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
8539
8699
|
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
8700
|
+
this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks;
|
|
8540
8701
|
this.config = config;
|
|
8541
8702
|
this.host = host;
|
|
8542
8703
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -8648,6 +8809,9 @@ var Indexer = class _Indexer {
|
|
|
8648
8809
|
return path15.resolve(targetPath);
|
|
8649
8810
|
}
|
|
8650
8811
|
}
|
|
8812
|
+
getProjectIdentityHash(projectRoot) {
|
|
8813
|
+
return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
8814
|
+
}
|
|
8651
8815
|
isProjectOwnedIndexPath() {
|
|
8652
8816
|
return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host);
|
|
8653
8817
|
}
|
|
@@ -8684,7 +8848,10 @@ var Indexer = class _Indexer {
|
|
|
8684
8848
|
}
|
|
8685
8849
|
async withIndexMutationLease(operation, callback) {
|
|
8686
8850
|
this.refreshBranchInfo();
|
|
8687
|
-
const lease = acquireIndexLock(this.indexPath, operation
|
|
8851
|
+
const lease = acquireIndexLock(this.indexPath, operation, {
|
|
8852
|
+
projectRoot: this.projectRoot,
|
|
8853
|
+
scopedRoots: this.getScopedRoots()
|
|
8854
|
+
});
|
|
8688
8855
|
this.indexPath = lease.canonicalIndexPath;
|
|
8689
8856
|
this.refreshRuntimeArtifactPaths();
|
|
8690
8857
|
this.activeIndexLease = lease;
|
|
@@ -8739,6 +8906,7 @@ var Indexer = class _Indexer {
|
|
|
8739
8906
|
}
|
|
8740
8907
|
loadFileHashCache() {
|
|
8741
8908
|
if (!(0, import_fs10.existsSync)(this.fileHashCachePath)) {
|
|
8909
|
+
this.fileHashCache = /* @__PURE__ */ new Map();
|
|
8742
8910
|
return;
|
|
8743
8911
|
}
|
|
8744
8912
|
try {
|
|
@@ -8778,10 +8946,10 @@ var Indexer = class _Indexer {
|
|
|
8778
8946
|
invertedIndex.serialize()
|
|
8779
8947
|
);
|
|
8780
8948
|
}
|
|
8781
|
-
getScopedRoots() {
|
|
8782
|
-
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(
|
|
8949
|
+
getScopedRoots(projectRoot = this.projectRoot) {
|
|
8950
|
+
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(projectRoot)]);
|
|
8783
8951
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
8784
|
-
roots.add(this.getCanonicalPath(path15.resolve(
|
|
8952
|
+
roots.add(this.getCanonicalPath(path15.resolve(projectRoot, kbRoot)));
|
|
8785
8953
|
}
|
|
8786
8954
|
return Array.from(roots);
|
|
8787
8955
|
}
|
|
@@ -8852,14 +9020,17 @@ var Indexer = class _Indexer {
|
|
|
8852
9020
|
getLegacyBranchCatalogKey() {
|
|
8853
9021
|
return this.currentBranch || "default";
|
|
8854
9022
|
}
|
|
8855
|
-
getLegacyMigrationMetadataKey() {
|
|
8856
|
-
return `index.globalBranchMigration.${
|
|
9023
|
+
getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9024
|
+
return `index.globalBranchMigration.${projectIdentityHash}`;
|
|
9025
|
+
}
|
|
9026
|
+
getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9027
|
+
return `index.embeddingStrategyVersion.${projectIdentityHash}`;
|
|
8857
9028
|
}
|
|
8858
|
-
|
|
8859
|
-
return `index.
|
|
9029
|
+
getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9030
|
+
return `index.forceReembed.${projectIdentityHash}`;
|
|
8860
9031
|
}
|
|
8861
|
-
|
|
8862
|
-
return `index.
|
|
9032
|
+
getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9033
|
+
return `index.migrationFinalized.${projectIdentityHash}`;
|
|
8863
9034
|
}
|
|
8864
9035
|
getBranchMigrationMetadataKey(prefix, catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
8865
9036
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
@@ -8965,7 +9136,7 @@ var Indexer = class _Indexer {
|
|
|
8965
9136
|
const legacy = this.getLegacyBranchCatalogKey();
|
|
8966
9137
|
return primary === legacy ? [primary] : [primary, legacy];
|
|
8967
9138
|
}
|
|
8968
|
-
getProjectLocalScopedOwnershipIds(roots) {
|
|
9139
|
+
getProjectLocalScopedOwnershipIds(roots, projectRoot = this.projectRoot) {
|
|
8969
9140
|
const chunkIds = /* @__PURE__ */ new Set();
|
|
8970
9141
|
const symbolIds = /* @__PURE__ */ new Set();
|
|
8971
9142
|
if (!this.database) {
|
|
@@ -8973,10 +9144,10 @@ var Indexer = class _Indexer {
|
|
|
8973
9144
|
}
|
|
8974
9145
|
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
8975
9146
|
...Array.from(this.fileHashCache.keys()).filter(
|
|
8976
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
9147
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
8977
9148
|
),
|
|
8978
9149
|
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
8979
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
9150
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
8980
9151
|
)
|
|
8981
9152
|
]);
|
|
8982
9153
|
for (const filePath of projectLocalFilePaths) {
|
|
@@ -8989,15 +9160,16 @@ var Indexer = class _Indexer {
|
|
|
8989
9160
|
}
|
|
8990
9161
|
return { chunkIds, symbolIds };
|
|
8991
9162
|
}
|
|
8992
|
-
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
9163
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds, projectRoot = this.projectRoot) {
|
|
8993
9164
|
if (this.config.scope !== "global") {
|
|
8994
9165
|
return this.getBranchCatalogCleanupKeys();
|
|
8995
9166
|
}
|
|
8996
9167
|
const keys = /* @__PURE__ */ new Set();
|
|
8997
9168
|
const projectChunkIdSet = new Set(projectChunkIds);
|
|
8998
9169
|
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
9170
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
8999
9171
|
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
9000
|
-
if (branchKey.startsWith(`${
|
|
9172
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
9001
9173
|
keys.add(branchKey);
|
|
9002
9174
|
continue;
|
|
9003
9175
|
}
|
|
@@ -9007,8 +9179,10 @@ var Indexer = class _Indexer {
|
|
|
9007
9179
|
keys.add(branchKey);
|
|
9008
9180
|
}
|
|
9009
9181
|
}
|
|
9010
|
-
|
|
9011
|
-
|
|
9182
|
+
if (projectRoot === this.projectRoot) {
|
|
9183
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
9184
|
+
keys.add(branchKey);
|
|
9185
|
+
}
|
|
9012
9186
|
}
|
|
9013
9187
|
return Array.from(keys);
|
|
9014
9188
|
}
|
|
@@ -9016,10 +9190,10 @@ var Indexer = class _Indexer {
|
|
|
9016
9190
|
const canonicalFilePath = this.getCanonicalStoredFilePath(filePath);
|
|
9017
9191
|
return roots.some((root) => isPathWithinRoot2(canonicalFilePath, root));
|
|
9018
9192
|
}
|
|
9019
|
-
isFileInProjectRoot(filePath) {
|
|
9193
|
+
isFileInProjectRoot(filePath, projectRoot = this.projectRoot) {
|
|
9020
9194
|
return isPathWithinRoot2(
|
|
9021
9195
|
this.getCanonicalStoredFilePath(filePath),
|
|
9022
|
-
this.getCanonicalPath(
|
|
9196
|
+
this.getCanonicalPath(projectRoot)
|
|
9023
9197
|
);
|
|
9024
9198
|
}
|
|
9025
9199
|
clearScopedFileHashCache(roots) {
|
|
@@ -9061,12 +9235,12 @@ var Indexer = class _Indexer {
|
|
|
9061
9235
|
}
|
|
9062
9236
|
return false;
|
|
9063
9237
|
}
|
|
9064
|
-
hasForeignScopedBranchData() {
|
|
9238
|
+
hasForeignScopedBranchData(projectRoot = this.projectRoot, roots = this.getScopedRoots(projectRoot)) {
|
|
9065
9239
|
if (!this.database || this.config.scope !== "global") {
|
|
9066
9240
|
return false;
|
|
9067
9241
|
}
|
|
9068
|
-
const
|
|
9069
|
-
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
9242
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
9243
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot);
|
|
9070
9244
|
return this.database.getAllBranches().some(
|
|
9071
9245
|
(branchKey) => {
|
|
9072
9246
|
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
@@ -9075,7 +9249,7 @@ var Indexer = class _Indexer {
|
|
|
9075
9249
|
if (!hasBranchData) {
|
|
9076
9250
|
return false;
|
|
9077
9251
|
}
|
|
9078
|
-
if (branchKey.startsWith(`${
|
|
9252
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
9079
9253
|
return false;
|
|
9080
9254
|
}
|
|
9081
9255
|
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
@@ -9084,7 +9258,7 @@ var Indexer = class _Indexer {
|
|
|
9084
9258
|
}
|
|
9085
9259
|
);
|
|
9086
9260
|
}
|
|
9087
|
-
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
9261
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot = this.projectRoot) {
|
|
9088
9262
|
const allMetadata = store.getAllMetadata();
|
|
9089
9263
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
9090
9264
|
const filePaths = /* @__PURE__ */ new Set([
|
|
@@ -9092,7 +9266,7 @@ var Indexer = class _Indexer {
|
|
|
9092
9266
|
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
9093
9267
|
]);
|
|
9094
9268
|
const projectLocalFilePaths = new Set(
|
|
9095
|
-
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath))
|
|
9269
|
+
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot))
|
|
9096
9270
|
);
|
|
9097
9271
|
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
9098
9272
|
for (const filePath of filePaths) {
|
|
@@ -9102,7 +9276,7 @@ var Indexer = class _Indexer {
|
|
|
9102
9276
|
}
|
|
9103
9277
|
const removedChunkIdList = Array.from(removedChunkIds);
|
|
9104
9278
|
const projectLocalChunkIds = new Set(
|
|
9105
|
-
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)).map(({ key }) => key)
|
|
9279
|
+
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)).map(({ key }) => key)
|
|
9106
9280
|
);
|
|
9107
9281
|
for (const filePath of projectLocalFilePaths) {
|
|
9108
9282
|
for (const chunk of database.getChunksByFile(filePath)) {
|
|
@@ -9121,7 +9295,8 @@ var Indexer = class _Indexer {
|
|
|
9121
9295
|
}
|
|
9122
9296
|
const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys(
|
|
9123
9297
|
Array.from(projectLocalChunkIds),
|
|
9124
|
-
Array.from(projectLocalSymbolIds)
|
|
9298
|
+
Array.from(projectLocalSymbolIds),
|
|
9299
|
+
projectRoot
|
|
9125
9300
|
);
|
|
9126
9301
|
for (const branchKey of branchCleanupKeys) {
|
|
9127
9302
|
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
@@ -9156,29 +9331,96 @@ var Indexer = class _Indexer {
|
|
|
9156
9331
|
database.gcOrphanSymbols();
|
|
9157
9332
|
database.gcOrphanEmbeddings();
|
|
9158
9333
|
database.gcOrphanChunks();
|
|
9159
|
-
store.save();
|
|
9160
9334
|
this.saveInvertedIndex(invertedIndex);
|
|
9335
|
+
store.save();
|
|
9161
9336
|
return {
|
|
9162
9337
|
removedChunkIds: removedChunkIdList,
|
|
9163
9338
|
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
9164
9339
|
};
|
|
9165
9340
|
}
|
|
9341
|
+
getCurrentClearRecoveryState() {
|
|
9342
|
+
if (!this.configuredProviderInfo) {
|
|
9343
|
+
throw new Error("Cannot persist clear recovery state before the embedding provider is initialized");
|
|
9344
|
+
}
|
|
9345
|
+
const compatibility = this.checkCompatibility();
|
|
9346
|
+
const compatibilityDecision = compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible";
|
|
9347
|
+
return {
|
|
9348
|
+
phase: "clearing",
|
|
9349
|
+
embeddingProvider: this.configuredProviderInfo.provider,
|
|
9350
|
+
embeddingModel: this.configuredProviderInfo.modelInfo.model,
|
|
9351
|
+
embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions,
|
|
9352
|
+
embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION,
|
|
9353
|
+
compatibilityDecision
|
|
9354
|
+
};
|
|
9355
|
+
}
|
|
9356
|
+
beginClearRecoveryState() {
|
|
9357
|
+
const recovery = this.getCurrentClearRecoveryState();
|
|
9358
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), recovery);
|
|
9359
|
+
return recovery;
|
|
9360
|
+
}
|
|
9361
|
+
finishClearRecoveryState() {
|
|
9362
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), null);
|
|
9363
|
+
}
|
|
9364
|
+
matchesCurrentClearRecoveryConfiguration(recovery) {
|
|
9365
|
+
const configuredProviderInfo = this.configuredProviderInfo;
|
|
9366
|
+
return configuredProviderInfo !== null && recovery.embeddingProvider === configuredProviderInfo.provider && recovery.embeddingModel === configuredProviderInfo.modelInfo.model && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION;
|
|
9367
|
+
}
|
|
9368
|
+
hasUnknownLegacyForceIndexClear(owner) {
|
|
9369
|
+
return owner.operation === "force-index" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1 && (0, import_fs10.existsSync)(path15.join(this.indexPath, "force-index-phase"));
|
|
9370
|
+
}
|
|
9166
9371
|
async recoverFromInterruptedIndexingUnlocked(owners) {
|
|
9167
9372
|
for (const owner of owners) {
|
|
9168
9373
|
this.logger.warn("Detected interrupted indexing session, recovering...", {
|
|
9169
9374
|
pid: owner.pid,
|
|
9170
9375
|
hostname: owner.hostname,
|
|
9171
9376
|
operation: owner.operation,
|
|
9172
|
-
startedAt: owner.startedAt
|
|
9377
|
+
startedAt: owner.startedAt,
|
|
9378
|
+
projectRoot: owner.projectRoot
|
|
9173
9379
|
});
|
|
9174
9380
|
}
|
|
9175
9381
|
if (this.config.scope === "global") {
|
|
9176
|
-
|
|
9177
|
-
|
|
9382
|
+
const clearScopes = [];
|
|
9383
|
+
for (const owner of owners) {
|
|
9384
|
+
if (this.hasUnknownLegacyForceIndexClear(owner)) {
|
|
9385
|
+
throw new Error(
|
|
9386
|
+
`Cannot automatically recover interrupted force-index ${owner.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
9387
|
+
);
|
|
9388
|
+
}
|
|
9389
|
+
if (owner.operation === "clear" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1) {
|
|
9390
|
+
throw new Error(
|
|
9391
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating recovery state is unknown. The recovery marker was retained for manual inspection.`
|
|
9392
|
+
);
|
|
9393
|
+
}
|
|
9394
|
+
if (owner.clearRecovery === void 0) continue;
|
|
9395
|
+
if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) {
|
|
9396
|
+
throw new Error(
|
|
9397
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating project scope is unknown. The recovery marker was retained for manual inspection.`
|
|
9398
|
+
);
|
|
9399
|
+
}
|
|
9400
|
+
if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) {
|
|
9401
|
+
throw new Error(
|
|
9402
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the current embedding configuration does not match the originating lease. The recovery marker was retained; retry from the originating project with matching settings.`
|
|
9403
|
+
);
|
|
9404
|
+
}
|
|
9405
|
+
clearScopes.push({
|
|
9406
|
+
projectRoot: owner.projectRoot,
|
|
9407
|
+
scopedRoots: owner.scopedRoots,
|
|
9408
|
+
compatibilityDecision: owner.clearRecovery.compatibilityDecision
|
|
9409
|
+
});
|
|
9410
|
+
}
|
|
9411
|
+
if (clearScopes.length > 0) {
|
|
9412
|
+
this.loadFileHashCache();
|
|
9413
|
+
}
|
|
9414
|
+
for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) {
|
|
9415
|
+
this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision);
|
|
9178
9416
|
}
|
|
9179
9417
|
await this.healthCheckUnlocked();
|
|
9418
|
+
this.logger.info(
|
|
9419
|
+
clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint"
|
|
9420
|
+
);
|
|
9421
|
+
return;
|
|
9180
9422
|
}
|
|
9181
|
-
this.logger.info("Recovery complete, next index will
|
|
9423
|
+
this.logger.info("Recovery complete, next index will resume from the last checkpoint");
|
|
9182
9424
|
}
|
|
9183
9425
|
*loadSerializedFailedBatches() {
|
|
9184
9426
|
let warned = false;
|
|
@@ -9216,14 +9458,99 @@ var Indexer = class _Indexer {
|
|
|
9216
9458
|
state.writer.write(record);
|
|
9217
9459
|
state.recordsWritten += record.chunks.length;
|
|
9218
9460
|
}
|
|
9219
|
-
finalizeFailedBatchWriteState(state) {
|
|
9461
|
+
finalizeFailedBatchWriteState(state, resolvedChunkIds = /* @__PURE__ */ new Set()) {
|
|
9220
9462
|
if (state.recordsWritten > 0) {
|
|
9221
|
-
|
|
9463
|
+
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
9464
|
+
const retained = [];
|
|
9465
|
+
const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath));
|
|
9466
|
+
for (let i = records.length - 1; i >= 0; i--) {
|
|
9467
|
+
const chunks = records[i].chunks.filter((rawChunk) => {
|
|
9468
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9469
|
+
if (chunkId !== null) {
|
|
9470
|
+
if (resolvedChunkIds.has(chunkId)) return false;
|
|
9471
|
+
if (seenChunkIds.has(chunkId)) return false;
|
|
9472
|
+
seenChunkIds.add(chunkId);
|
|
9473
|
+
}
|
|
9474
|
+
return true;
|
|
9475
|
+
});
|
|
9476
|
+
if (chunks.length > 0) {
|
|
9477
|
+
retained.unshift({ ...records[i], chunks });
|
|
9478
|
+
}
|
|
9479
|
+
}
|
|
9480
|
+
state.writer.cleanup();
|
|
9481
|
+
if (retained.length > 0) {
|
|
9482
|
+
writeFailedBatchRecords(this.failedBatchesPath, retained);
|
|
9483
|
+
} else {
|
|
9484
|
+
writeFailedBatchRecords(this.failedBatchesPath, []);
|
|
9485
|
+
this.clearFailedBatchState();
|
|
9486
|
+
}
|
|
9222
9487
|
return;
|
|
9223
9488
|
}
|
|
9224
|
-
state.writer.
|
|
9489
|
+
state.writer.commit();
|
|
9225
9490
|
this.clearFailedBatchState();
|
|
9226
9491
|
}
|
|
9492
|
+
getCheckpointIntervalChunks(totalChunks) {
|
|
9493
|
+
return Math.max(
|
|
9494
|
+
this.checkpointIntervalChunks ?? 2e3,
|
|
9495
|
+
Math.floor(totalChunks / 10)
|
|
9496
|
+
);
|
|
9497
|
+
}
|
|
9498
|
+
checkpointIndexRun(database, store, invertedIndex, failedProcessing, resolvedRetryChunkIds, currentFileHashes, committedFilePaths, scopedRoots, configuredProviderInfo) {
|
|
9499
|
+
if (!this.hasProjectForceReembedPending()) {
|
|
9500
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
9501
|
+
this.indexCompatibility = { compatible: true };
|
|
9502
|
+
}
|
|
9503
|
+
database.commitWriteTransaction();
|
|
9504
|
+
database.beginWriteTransaction();
|
|
9505
|
+
this.saveInvertedIndex(invertedIndex);
|
|
9506
|
+
store.save();
|
|
9507
|
+
if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0 || failedProcessing.discardedExistingRecords) {
|
|
9508
|
+
for (const metadata of failedProcessing.latestById.values()) {
|
|
9509
|
+
const alreadyMaterialized = metadata.chunks.some((rawChunk) => {
|
|
9510
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9511
|
+
return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId);
|
|
9512
|
+
});
|
|
9513
|
+
if (alreadyMaterialized) continue;
|
|
9514
|
+
this.writeFailedBatchRecord(failedProcessing.state, {
|
|
9515
|
+
chunks: metadata.chunks,
|
|
9516
|
+
attemptCount: metadata.attemptCount,
|
|
9517
|
+
error: metadata.error,
|
|
9518
|
+
lastAttempt: metadata.lastAttempt
|
|
9519
|
+
});
|
|
9520
|
+
for (const rawChunk of metadata.chunks) {
|
|
9521
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9522
|
+
if (chunkId !== null) {
|
|
9523
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
9524
|
+
}
|
|
9525
|
+
}
|
|
9526
|
+
}
|
|
9527
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
9528
|
+
failedProcessing.state = this.createFailedBatchWriteState();
|
|
9529
|
+
failedProcessing.discardedExistingRecords = false;
|
|
9530
|
+
for (const record of this.loadSerializedFailedBatches()) {
|
|
9531
|
+
for (const rawChunk of record.chunks) {
|
|
9532
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9533
|
+
this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] });
|
|
9534
|
+
if (chunkId !== null) {
|
|
9535
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
9536
|
+
}
|
|
9537
|
+
}
|
|
9538
|
+
}
|
|
9539
|
+
}
|
|
9540
|
+
const partialHashes = /* @__PURE__ */ new Map();
|
|
9541
|
+
for (const filePath of committedFilePaths) {
|
|
9542
|
+
const hash = currentFileHashes.get(filePath);
|
|
9543
|
+
if (hash !== void 0) {
|
|
9544
|
+
partialHashes.set(filePath, hash);
|
|
9545
|
+
}
|
|
9546
|
+
}
|
|
9547
|
+
if (scopedRoots) {
|
|
9548
|
+
this.replaceScopedFileHashCache(partialHashes, scopedRoots);
|
|
9549
|
+
} else {
|
|
9550
|
+
this.fileHashCache = partialHashes;
|
|
9551
|
+
this.saveFileHashCache();
|
|
9552
|
+
}
|
|
9553
|
+
}
|
|
9227
9554
|
clearFailedBatchState() {
|
|
9228
9555
|
if ((0, import_fs10.existsSync)(this.failedBatchesPath)) {
|
|
9229
9556
|
try {
|
|
@@ -9250,6 +9577,7 @@ var Indexer = class _Indexer {
|
|
|
9250
9577
|
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
9251
9578
|
const state = this.createFailedBatchWriteState();
|
|
9252
9579
|
const latestById = /* @__PURE__ */ new Map();
|
|
9580
|
+
let discardedExistingRecords = false;
|
|
9253
9581
|
try {
|
|
9254
9582
|
for (const batch of this.loadSerializedFailedBatches()) {
|
|
9255
9583
|
for (const rawChunk of batch.chunks) {
|
|
@@ -9260,10 +9588,12 @@ var Indexer = class _Indexer {
|
|
|
9260
9588
|
continue;
|
|
9261
9589
|
}
|
|
9262
9590
|
if (!shouldProcess(filePath)) {
|
|
9591
|
+
discardedExistingRecords = true;
|
|
9263
9592
|
continue;
|
|
9264
9593
|
}
|
|
9265
9594
|
const chunkId = getPendingChunkId(rawChunk);
|
|
9266
9595
|
if (!chunkId) {
|
|
9596
|
+
discardedExistingRecords = true;
|
|
9267
9597
|
continue;
|
|
9268
9598
|
}
|
|
9269
9599
|
const existing = latestById.get(chunkId);
|
|
@@ -9271,12 +9601,18 @@ var Indexer = class _Indexer {
|
|
|
9271
9601
|
latestById.set(chunkId, {
|
|
9272
9602
|
attemptCount: batch.attemptCount,
|
|
9273
9603
|
error: batch.error,
|
|
9274
|
-
lastAttempt: batch.lastAttempt
|
|
9604
|
+
lastAttempt: batch.lastAttempt,
|
|
9605
|
+
chunks: [rawChunk]
|
|
9275
9606
|
});
|
|
9276
9607
|
}
|
|
9277
9608
|
}
|
|
9278
9609
|
}
|
|
9279
|
-
return {
|
|
9610
|
+
return {
|
|
9611
|
+
state,
|
|
9612
|
+
latestById,
|
|
9613
|
+
materializedRetryIds: /* @__PURE__ */ new Set(),
|
|
9614
|
+
discardedExistingRecords
|
|
9615
|
+
};
|
|
9280
9616
|
} catch (error) {
|
|
9281
9617
|
state.writer.cleanup();
|
|
9282
9618
|
throw error;
|
|
@@ -9312,10 +9648,34 @@ var Indexer = class _Indexer {
|
|
|
9312
9648
|
}
|
|
9313
9649
|
}
|
|
9314
9650
|
}
|
|
9651
|
+
restoreMissingChunkRows(database, chunks) {
|
|
9652
|
+
const missing = [];
|
|
9653
|
+
for (const chunk of chunks) {
|
|
9654
|
+
if (database.getChunk(chunk.id)) {
|
|
9655
|
+
continue;
|
|
9656
|
+
}
|
|
9657
|
+
missing.push({
|
|
9658
|
+
chunkId: chunk.id,
|
|
9659
|
+
contentHash: chunk.contentHash,
|
|
9660
|
+
filePath: chunk.metadata.filePath,
|
|
9661
|
+
startLine: chunk.metadata.startLine,
|
|
9662
|
+
endLine: chunk.metadata.endLine,
|
|
9663
|
+
nodeType: chunk.metadata.chunkType,
|
|
9664
|
+
name: chunk.metadata.name,
|
|
9665
|
+
language: chunk.metadata.language,
|
|
9666
|
+
blameSha: chunk.metadata.blameSha,
|
|
9667
|
+
blameAuthor: chunk.metadata.blameAuthor,
|
|
9668
|
+
blameAuthorEmail: chunk.metadata.blameAuthorEmail,
|
|
9669
|
+
blameCommittedAt: chunk.metadata.blameCommittedAt,
|
|
9670
|
+
blameSummary: chunk.metadata.blameSummary
|
|
9671
|
+
});
|
|
9672
|
+
}
|
|
9673
|
+
if (missing.length > 0) {
|
|
9674
|
+
database.upsertChunksBatch(missing);
|
|
9675
|
+
}
|
|
9676
|
+
}
|
|
9315
9677
|
getProviderRateLimits(provider) {
|
|
9316
9678
|
switch (provider) {
|
|
9317
|
-
case "github-copilot":
|
|
9318
|
-
return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
|
|
9319
9679
|
case "openai":
|
|
9320
9680
|
return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
9321
9681
|
case "google":
|
|
@@ -9384,10 +9744,11 @@ var Indexer = class _Indexer {
|
|
|
9384
9744
|
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
9385
9745
|
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
9386
9746
|
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
9387
|
-
const
|
|
9388
|
-
|
|
9389
|
-
|
|
9390
|
-
|
|
9747
|
+
const batchOptions = getDynamicBatchOptions(options.configuredProviderInfo, this.config.embedding?.batch);
|
|
9748
|
+
if (options.forceSingleItemBatches && options.configuredProviderInfo.provider === "ollama") {
|
|
9749
|
+
batchOptions.maxBatchItems = 1;
|
|
9750
|
+
}
|
|
9751
|
+
const requestBatches = createPendingEmbeddingRequestBatches(chunksNeedingEmbedding, batchOptions);
|
|
9391
9752
|
let fatalError;
|
|
9392
9753
|
for (const requestBatch of requestBatches) {
|
|
9393
9754
|
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
@@ -9950,7 +10311,7 @@ var Indexer = class _Indexer {
|
|
|
9950
10311
|
}
|
|
9951
10312
|
if (!this.configuredProviderInfo) {
|
|
9952
10313
|
throw new Error(
|
|
9953
|
-
"No embedding provider available. Configure
|
|
10314
|
+
"No embedding provider available. Configure OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
9954
10315
|
);
|
|
9955
10316
|
}
|
|
9956
10317
|
this.logger.info("Initializing indexer", {
|
|
@@ -9981,7 +10342,20 @@ var Indexer = class _Indexer {
|
|
|
9981
10342
|
]);
|
|
9982
10343
|
}
|
|
9983
10344
|
if (recoveredOwners.length > 0 && this.config.scope === "project") {
|
|
9984
|
-
|
|
10345
|
+
const unknownLegacyForceIndex = recoveredOwners.find(
|
|
10346
|
+
(owner) => this.hasUnknownLegacyForceIndexClear(owner)
|
|
10347
|
+
);
|
|
10348
|
+
if (unknownLegacyForceIndex) {
|
|
10349
|
+
throw new Error(
|
|
10350
|
+
`Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
10351
|
+
);
|
|
10352
|
+
}
|
|
10353
|
+
const shouldReset = recoveredOwners.some(
|
|
10354
|
+
(owner) => owner.clearRecovery !== void 0 || owner.operation === "clear" && owner.recoveryProtocolVersion !== 1
|
|
10355
|
+
);
|
|
10356
|
+
if (shouldReset) {
|
|
10357
|
+
await this.resetLocalIndexArtifacts();
|
|
10358
|
+
}
|
|
9985
10359
|
}
|
|
9986
10360
|
this.store = new VectorStore(storePath, dimensions);
|
|
9987
10361
|
if ((0, import_fs10.existsSync)(storePath) || (0, import_fs10.existsSync)(vectorMetadataPath)) {
|
|
@@ -10617,7 +10991,17 @@ var Indexer = class _Indexer {
|
|
|
10617
10991
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
10618
10992
|
for (const file of files) {
|
|
10619
10993
|
const storedPath = this.toStoredFilePath(file.path);
|
|
10620
|
-
|
|
10994
|
+
let currentHash;
|
|
10995
|
+
try {
|
|
10996
|
+
currentHash = hashFile(file.path);
|
|
10997
|
+
} catch (error) {
|
|
10998
|
+
stats.skippedFiles.push({ path: this.toCanonicalFilePath(file.path), reason: "unreadable" });
|
|
10999
|
+
this.logger.warn("Skipped unreadable file during indexing", {
|
|
11000
|
+
path: file.path,
|
|
11001
|
+
error: getErrorMessage3(error)
|
|
11002
|
+
});
|
|
11003
|
+
continue;
|
|
11004
|
+
}
|
|
10621
11005
|
currentFileHashes.set(storedPath, currentHash);
|
|
10622
11006
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
10623
11007
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
@@ -10625,7 +11009,8 @@ var Indexer = class _Indexer {
|
|
|
10625
11009
|
);
|
|
10626
11010
|
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path15.extname(storedPath).toLowerCase() === ".swift";
|
|
10627
11011
|
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path15.extname(storedPath).toLowerCase() === ".metal";
|
|
10628
|
-
|
|
11012
|
+
const inMigrationScope = forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots);
|
|
11013
|
+
if (cachedHashMatches && !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
10629
11014
|
unchangedFilePaths.add(storedPath);
|
|
10630
11015
|
this.logger.recordCacheHit();
|
|
10631
11016
|
} else {
|
|
@@ -10751,6 +11136,9 @@ var Indexer = class _Indexer {
|
|
|
10751
11136
|
}
|
|
10752
11137
|
}
|
|
10753
11138
|
let processedChangedFiles = 0;
|
|
11139
|
+
let lastCheckpointChunks = 0;
|
|
11140
|
+
const committedFilePaths = new Set(unchangedFilePaths);
|
|
11141
|
+
const resolvedRetryChunkIds = /* @__PURE__ */ new Set();
|
|
10754
11142
|
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
10755
11143
|
changedFileDescriptors,
|
|
10756
11144
|
(descriptor) => descriptor.sourceBytes,
|
|
@@ -10764,7 +11152,7 @@ var Indexer = class _Indexer {
|
|
|
10764
11152
|
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
10765
11153
|
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
10766
11154
|
const parseStartTime = import_perf_hooks.performance.now();
|
|
10767
|
-
const parsedFiles = parseFiles(loadedFiles);
|
|
11155
|
+
const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk);
|
|
10768
11156
|
const parseMs = import_perf_hooks.performance.now() - parseStartTime;
|
|
10769
11157
|
this.logger.recordFilesParsed(parsedFiles.length);
|
|
10770
11158
|
this.logger.recordParseDuration(parseMs);
|
|
@@ -10787,7 +11175,7 @@ var Indexer = class _Indexer {
|
|
|
10787
11175
|
}
|
|
10788
11176
|
let chunksToProcess = parsed.chunks;
|
|
10789
11177
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
10790
|
-
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
11178
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk);
|
|
10791
11179
|
}
|
|
10792
11180
|
chunksToProcess = selectIndexableChunks(
|
|
10793
11181
|
chunksToProcess,
|
|
@@ -10921,6 +11309,10 @@ var Indexer = class _Indexer {
|
|
|
10921
11309
|
}
|
|
10922
11310
|
if (symbolBatch.length > 0) {
|
|
10923
11311
|
database.upsertSymbolsBatch(symbolBatch);
|
|
11312
|
+
database.addSymbolsToBranchBatch(
|
|
11313
|
+
this.getBranchCatalogKey(),
|
|
11314
|
+
symbolBatch.map((symbol) => symbol.id)
|
|
11315
|
+
);
|
|
10924
11316
|
}
|
|
10925
11317
|
if (edgeBatch.length > 0) {
|
|
10926
11318
|
database.upsertCallEdgesBatch(edgeBatch);
|
|
@@ -10956,6 +11348,12 @@ var Indexer = class _Indexer {
|
|
|
10956
11348
|
forceReembed: forceScopedReembed,
|
|
10957
11349
|
reuseCachedEmbeddings: true,
|
|
10958
11350
|
incrementRepeatedFailures: true,
|
|
11351
|
+
onSucceeded: (succeededChunks) => {
|
|
11352
|
+
database.addChunksToBranchBatch(
|
|
11353
|
+
this.getBranchCatalogKey(),
|
|
11354
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
11355
|
+
);
|
|
11356
|
+
},
|
|
10959
11357
|
onProgress: (batchProgress) => onProgress?.({
|
|
10960
11358
|
phase: "embedding",
|
|
10961
11359
|
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
@@ -10974,6 +11372,27 @@ var Indexer = class _Indexer {
|
|
|
10974
11372
|
}
|
|
10975
11373
|
}
|
|
10976
11374
|
}
|
|
11375
|
+
for (const descriptor of descriptorBatch) {
|
|
11376
|
+
const existingFileChunks = existingChunksByFile.get(descriptor.storedPath);
|
|
11377
|
+
if (!existingFileChunks || existingFileChunks.size === 0) {
|
|
11378
|
+
committedFilePaths.add(descriptor.storedPath);
|
|
11379
|
+
}
|
|
11380
|
+
}
|
|
11381
|
+
const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks);
|
|
11382
|
+
if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) {
|
|
11383
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
11384
|
+
this.checkpointIndexRun(
|
|
11385
|
+
database,
|
|
11386
|
+
store,
|
|
11387
|
+
invertedIndex,
|
|
11388
|
+
failedProcessing,
|
|
11389
|
+
resolvedRetryChunkIds,
|
|
11390
|
+
currentFileHashes,
|
|
11391
|
+
committedFilePaths,
|
|
11392
|
+
scopedRoots,
|
|
11393
|
+
configuredProviderInfo
|
|
11394
|
+
);
|
|
11395
|
+
}
|
|
10977
11396
|
}
|
|
10978
11397
|
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
10979
11398
|
failedProcessing.latestById,
|
|
@@ -10994,6 +11413,7 @@ var Indexer = class _Indexer {
|
|
|
10994
11413
|
retryableChunksWithExistingData.add(chunk.id);
|
|
10995
11414
|
}
|
|
10996
11415
|
}
|
|
11416
|
+
this.restoreMissingChunkRows(database, pendingChunks);
|
|
10997
11417
|
stats.totalChunks += pendingChunks.length;
|
|
10998
11418
|
onProgress?.({
|
|
10999
11419
|
phase: "embedding",
|
|
@@ -11016,6 +11436,17 @@ var Indexer = class _Indexer {
|
|
|
11016
11436
|
forceReembed: forceScopedReembed,
|
|
11017
11437
|
reuseCachedEmbeddings: true,
|
|
11018
11438
|
incrementRepeatedFailures: true,
|
|
11439
|
+
forceSingleItemBatches: true,
|
|
11440
|
+
onSucceeded: (succeededChunks) => {
|
|
11441
|
+
database.addChunksToBranchBatch(
|
|
11442
|
+
this.getBranchCatalogKey(),
|
|
11443
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
11444
|
+
);
|
|
11445
|
+
for (const chunk of succeededChunks) {
|
|
11446
|
+
failedProcessing.latestById.delete(chunk.id);
|
|
11447
|
+
resolvedRetryChunkIds.add(chunk.id);
|
|
11448
|
+
}
|
|
11449
|
+
},
|
|
11019
11450
|
onProgress: (batchProgress) => onProgress?.({
|
|
11020
11451
|
phase: "embedding",
|
|
11021
11452
|
filesProcessed: files.length,
|
|
@@ -11033,6 +11464,20 @@ var Indexer = class _Indexer {
|
|
|
11033
11464
|
failedForcedChunkIds.add(chunkId);
|
|
11034
11465
|
}
|
|
11035
11466
|
}
|
|
11467
|
+
if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) {
|
|
11468
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
11469
|
+
this.checkpointIndexRun(
|
|
11470
|
+
database,
|
|
11471
|
+
store,
|
|
11472
|
+
invertedIndex,
|
|
11473
|
+
failedProcessing,
|
|
11474
|
+
resolvedRetryChunkIds,
|
|
11475
|
+
currentFileHashes,
|
|
11476
|
+
committedFilePaths,
|
|
11477
|
+
scopedRoots,
|
|
11478
|
+
configuredProviderInfo
|
|
11479
|
+
);
|
|
11480
|
+
}
|
|
11036
11481
|
}
|
|
11037
11482
|
const removedChunkIds = [];
|
|
11038
11483
|
for (const [chunkId] of existingChunks) {
|
|
@@ -11069,13 +11514,6 @@ var Indexer = class _Indexer {
|
|
|
11069
11514
|
if (removedStoredChunks) {
|
|
11070
11515
|
this.saveInvertedIndex(invertedIndex);
|
|
11071
11516
|
}
|
|
11072
|
-
if (scopedRoots) {
|
|
11073
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11074
|
-
} else {
|
|
11075
|
-
this.fileHashCache = currentFileHashes;
|
|
11076
|
-
this.saveFileHashCache();
|
|
11077
|
-
}
|
|
11078
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11079
11517
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11080
11518
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11081
11519
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11084,6 +11522,13 @@ var Indexer = class _Indexer {
|
|
|
11084
11522
|
this.indexCompatibility = { compatible: true };
|
|
11085
11523
|
database.commitWriteTransaction();
|
|
11086
11524
|
writeTransactionActive = false;
|
|
11525
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11526
|
+
if (scopedRoots) {
|
|
11527
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11528
|
+
} else {
|
|
11529
|
+
this.fileHashCache = currentFileHashes;
|
|
11530
|
+
this.saveFileHashCache();
|
|
11531
|
+
}
|
|
11087
11532
|
stats.durationMs = Date.now() - startTime;
|
|
11088
11533
|
onProgress?.({
|
|
11089
11534
|
phase: "complete",
|
|
@@ -11107,13 +11552,6 @@ var Indexer = class _Indexer {
|
|
|
11107
11552
|
);
|
|
11108
11553
|
store.save();
|
|
11109
11554
|
this.saveInvertedIndex(invertedIndex);
|
|
11110
|
-
if (scopedRoots) {
|
|
11111
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11112
|
-
} else {
|
|
11113
|
-
this.fileHashCache = currentFileHashes;
|
|
11114
|
-
this.saveFileHashCache();
|
|
11115
|
-
}
|
|
11116
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11117
11555
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11118
11556
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11119
11557
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11122,6 +11560,13 @@ var Indexer = class _Indexer {
|
|
|
11122
11560
|
this.indexCompatibility = { compatible: true };
|
|
11123
11561
|
database.commitWriteTransaction();
|
|
11124
11562
|
writeTransactionActive = false;
|
|
11563
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11564
|
+
if (scopedRoots) {
|
|
11565
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11566
|
+
} else {
|
|
11567
|
+
this.fileHashCache = currentFileHashes;
|
|
11568
|
+
this.saveFileHashCache();
|
|
11569
|
+
}
|
|
11125
11570
|
stats.durationMs = Date.now() - startTime;
|
|
11126
11571
|
onProgress?.({
|
|
11127
11572
|
phase: "complete",
|
|
@@ -11156,15 +11601,15 @@ var Indexer = class _Indexer {
|
|
|
11156
11601
|
);
|
|
11157
11602
|
store.save();
|
|
11158
11603
|
this.saveInvertedIndex(invertedIndex);
|
|
11604
|
+
database.commitWriteTransaction();
|
|
11605
|
+
writeTransactionActive = false;
|
|
11606
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11159
11607
|
if (scopedRoots) {
|
|
11160
11608
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11161
11609
|
} else {
|
|
11162
11610
|
this.fileHashCache = currentFileHashes;
|
|
11163
11611
|
this.saveFileHashCache();
|
|
11164
11612
|
}
|
|
11165
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11166
|
-
database.commitWriteTransaction();
|
|
11167
|
-
writeTransactionActive = false;
|
|
11168
11613
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
11169
11614
|
const gcReset = await this.maybeRunOrphanGc();
|
|
11170
11615
|
if (gcReset) {
|
|
@@ -11188,6 +11633,9 @@ var Indexer = class _Indexer {
|
|
|
11188
11633
|
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
11189
11634
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11190
11635
|
}
|
|
11636
|
+
if (forceScopedReembed) {
|
|
11637
|
+
database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true");
|
|
11638
|
+
}
|
|
11191
11639
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11192
11640
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11193
11641
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11298,26 +11746,41 @@ var Indexer = class _Indexer {
|
|
|
11298
11746
|
shouldPrefilterByBranch: branchChunkIds !== null && (this.config.scope === "global" || hasInitializedBranchCatalog)
|
|
11299
11747
|
};
|
|
11300
11748
|
}
|
|
11301
|
-
|
|
11749
|
+
searchCandidatesWithAllowedIds(initialLimit, totalCount, allowedChunkIds, shouldPrefilter, search, getChunkId) {
|
|
11302
11750
|
const normalizedLimit = Math.max(0, Math.floor(initialLimit));
|
|
11303
11751
|
if (normalizedLimit === 0) return [];
|
|
11304
|
-
if (!
|
|
11752
|
+
if (!shouldPrefilter || !allowedChunkIds) {
|
|
11305
11753
|
return search(normalizedLimit);
|
|
11306
11754
|
}
|
|
11307
|
-
const targetCount = Math.min(normalizedLimit,
|
|
11755
|
+
const targetCount = Math.min(normalizedLimit, allowedChunkIds.size);
|
|
11308
11756
|
if (targetCount === 0 || totalCount === 0) return [];
|
|
11309
11757
|
let requestedLimit = Math.min(normalizedLimit, totalCount);
|
|
11310
11758
|
while (true) {
|
|
11311
11759
|
const results = search(requestedLimit);
|
|
11312
|
-
const
|
|
11313
|
-
if (
|
|
11314
|
-
return
|
|
11760
|
+
const allowedResults = results.filter((candidate) => allowedChunkIds.has(getChunkId(candidate)));
|
|
11761
|
+
if (allowedResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
|
|
11762
|
+
return allowedResults;
|
|
11315
11763
|
}
|
|
11316
11764
|
const nextLimit = Math.min(totalCount, Math.max(requestedLimit + 1, requestedLimit * 2));
|
|
11317
|
-
if (nextLimit === requestedLimit) return
|
|
11765
|
+
if (nextLimit === requestedLimit) return allowedResults;
|
|
11318
11766
|
requestedLimit = nextLimit;
|
|
11319
11767
|
}
|
|
11320
11768
|
}
|
|
11769
|
+
getTemporalChunkIds(database, options) {
|
|
11770
|
+
if (!options?.blameSince && !options?.blameUntil) return null;
|
|
11771
|
+
const since = options.blameSince ? parseBlameTimestamp(options.blameSince, false) : void 0;
|
|
11772
|
+
const until = options.blameUntil ? parseBlameTimestamp(options.blameUntil, true) : void 0;
|
|
11773
|
+
if (since === null || until === null) {
|
|
11774
|
+
return /* @__PURE__ */ new Set();
|
|
11775
|
+
}
|
|
11776
|
+
return new Set(database.getChunkIdsByBlameDate(since, until));
|
|
11777
|
+
}
|
|
11778
|
+
intersectChunkIdSets(first, second) {
|
|
11779
|
+
if (first === null) return second;
|
|
11780
|
+
if (second === null) return first;
|
|
11781
|
+
const [smaller, larger] = first.size <= second.size ? [first, second] : [second, first];
|
|
11782
|
+
return new Set(Array.from(smaller).filter((chunkId) => larger.has(chunkId)));
|
|
11783
|
+
}
|
|
11321
11784
|
buildCandidateSnapshot(candidate) {
|
|
11322
11785
|
return {
|
|
11323
11786
|
id: candidate.id,
|
|
@@ -11332,13 +11795,16 @@ var Indexer = class _Indexer {
|
|
|
11332
11795
|
buildCandidateSnapshotList(candidates) {
|
|
11333
11796
|
return candidates.map((candidate) => this.buildCandidateSnapshot(candidate));
|
|
11334
11797
|
}
|
|
11335
|
-
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch) {
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11798
|
+
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch, temporalChunkIds) {
|
|
11799
|
+
const availableCount = temporalChunkIds?.size ?? store.count();
|
|
11800
|
+
if (availableCount === 0) return [];
|
|
11801
|
+
const allowedIds = temporalChunkIds === null ? void 0 : Array.from(temporalChunkIds);
|
|
11802
|
+
return this.searchCandidatesWithAllowedIds(
|
|
11803
|
+
Math.min(initialLimit, availableCount),
|
|
11804
|
+
availableCount,
|
|
11339
11805
|
branchChunkIds,
|
|
11340
11806
|
shouldPrefilterByBranch,
|
|
11341
|
-
(requestedLimit) => store.search(embedding, requestedLimit),
|
|
11807
|
+
(requestedLimit) => store.search(embedding, requestedLimit, allowedIds),
|
|
11342
11808
|
(candidate) => candidate.id
|
|
11343
11809
|
);
|
|
11344
11810
|
}
|
|
@@ -11363,8 +11829,9 @@ var Indexer = class _Indexer {
|
|
|
11363
11829
|
const rerankTopN = this.config.search.rerankTopN;
|
|
11364
11830
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
11365
11831
|
const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
|
|
11832
|
+
const prioritizeSourcePaths = sourceIntent || options?.prioritizeSourcePaths === true;
|
|
11366
11833
|
const identifierHints = extractIdentifierHints(query);
|
|
11367
|
-
const candidateLimit = maxResults * (
|
|
11834
|
+
const candidateLimit = maxResults * (prioritizeSourcePaths ? 12 : 4);
|
|
11368
11835
|
this.logger.search("debug", "Starting search", {
|
|
11369
11836
|
query,
|
|
11370
11837
|
maxResults,
|
|
@@ -11395,6 +11862,7 @@ var Indexer = class _Indexer {
|
|
|
11395
11862
|
branchChunkIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchChunkIds(branchKey)));
|
|
11396
11863
|
branchSymbolIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchSymbolIds(branchKey)));
|
|
11397
11864
|
}
|
|
11865
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
11398
11866
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
11399
11867
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
11400
11868
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
@@ -11403,7 +11871,8 @@ var Indexer = class _Indexer {
|
|
|
11403
11871
|
embedding,
|
|
11404
11872
|
candidateLimit,
|
|
11405
11873
|
branchChunkIds,
|
|
11406
|
-
shouldPrefilterByBranch
|
|
11874
|
+
shouldPrefilterByBranch,
|
|
11875
|
+
temporalChunkIds
|
|
11407
11876
|
) : [];
|
|
11408
11877
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
11409
11878
|
const keywordStartTime = import_perf_hooks.performance.now();
|
|
@@ -11413,7 +11882,8 @@ var Indexer = class _Indexer {
|
|
|
11413
11882
|
store,
|
|
11414
11883
|
invertedIndex,
|
|
11415
11884
|
branchChunkIds,
|
|
11416
|
-
shouldPrefilterByBranch
|
|
11885
|
+
shouldPrefilterByBranch,
|
|
11886
|
+
temporalChunkIds
|
|
11417
11887
|
);
|
|
11418
11888
|
const keywordMs = import_perf_hooks.performance.now() - keywordStartTime;
|
|
11419
11889
|
const scopedSemanticCandidates = semanticCandidates.filter(
|
|
@@ -11435,7 +11905,7 @@ var Indexer = class _Indexer {
|
|
|
11435
11905
|
rerankTopN,
|
|
11436
11906
|
limit: maxResults,
|
|
11437
11907
|
hybridWeight: rankingHybridWeight,
|
|
11438
|
-
prioritizeSourcePaths
|
|
11908
|
+
prioritizeSourcePaths
|
|
11439
11909
|
});
|
|
11440
11910
|
const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
|
|
11441
11911
|
definitionIntent: options?.definitionIntent === true,
|
|
@@ -11471,10 +11941,11 @@ var Indexer = class _Indexer {
|
|
|
11471
11941
|
branchSymbolIds,
|
|
11472
11942
|
maxResults,
|
|
11473
11943
|
union,
|
|
11474
|
-
sourceIntent
|
|
11944
|
+
sourceIntent,
|
|
11945
|
+
options?.definitionIntent === true && ((options.directory?.trim().length ?? 0) > 0 || (options.fileType?.trim().length ?? 0) > 0)
|
|
11475
11946
|
);
|
|
11476
11947
|
const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
|
|
11477
|
-
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
11948
|
+
const primaryLane = options?.definitionIntent === true ? mergeTieredResults(symbolLane, prePrimaryLane, maxResults * 4) : mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
11478
11949
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
11479
11950
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
11480
11951
|
const baseFiltered = tiered.filter(
|
|
@@ -11569,14 +12040,18 @@ var Indexer = class _Indexer {
|
|
|
11569
12040
|
})
|
|
11570
12041
|
);
|
|
11571
12042
|
}
|
|
11572
|
-
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false) {
|
|
12043
|
+
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false, temporalChunkIds = null) {
|
|
11573
12044
|
const normalizedLimit = Math.max(0, Math.floor(limit));
|
|
11574
12045
|
if (normalizedLimit === 0) return [];
|
|
11575
|
-
const
|
|
12046
|
+
const allowedChunkIds = this.intersectChunkIdSets(
|
|
12047
|
+
shouldPrefilterByBranch ? branchChunkIds : null,
|
|
12048
|
+
temporalChunkIds
|
|
12049
|
+
);
|
|
12050
|
+
const scoreEntries = this.searchCandidatesWithAllowedIds(
|
|
11576
12051
|
normalizedLimit,
|
|
11577
12052
|
invertedIndex.getDocumentCount(),
|
|
11578
|
-
|
|
11579
|
-
|
|
12053
|
+
allowedChunkIds,
|
|
12054
|
+
allowedChunkIds !== null,
|
|
11580
12055
|
(requestedLimit) => Array.from(invertedIndex.search(query, requestedLimit)),
|
|
11581
12056
|
([chunkId]) => chunkId
|
|
11582
12057
|
);
|
|
@@ -11661,7 +12136,17 @@ var Indexer = class _Indexer {
|
|
|
11661
12136
|
);
|
|
11662
12137
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
11663
12138
|
for (const file of files) {
|
|
11664
|
-
|
|
12139
|
+
let hash;
|
|
12140
|
+
try {
|
|
12141
|
+
hash = hashFile(file.path);
|
|
12142
|
+
} catch (error) {
|
|
12143
|
+
this.logger.warn("Skipped unreadable file during freshness check", {
|
|
12144
|
+
path: file.path,
|
|
12145
|
+
error: getErrorMessage3(error)
|
|
12146
|
+
});
|
|
12147
|
+
return { readable: false, current: false, reason: "unreadable" };
|
|
12148
|
+
}
|
|
12149
|
+
currentFileHashes.set(this.toStoredFilePath(file.path), hash);
|
|
11665
12150
|
}
|
|
11666
12151
|
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
11667
12152
|
const cachedFileHashes = scopedRoots ? new Map(Array.from(this.fileHashCache).filter(([filePath]) => this.isFileInCurrentScope(filePath, scopedRoots))) : this.fileHashCache;
|
|
@@ -11687,69 +12172,87 @@ var Indexer = class _Indexer {
|
|
|
11687
12172
|
async forceIndex(onProgress) {
|
|
11688
12173
|
return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
|
|
11689
12174
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
11690
|
-
|
|
12175
|
+
const recovery = this.beginClearRecoveryState();
|
|
12176
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
12177
|
+
this.finishClearRecoveryState();
|
|
11691
12178
|
return this.indexUnlocked(onProgress, [], true);
|
|
11692
12179
|
});
|
|
11693
12180
|
}
|
|
11694
12181
|
async clearIndex() {
|
|
11695
12182
|
await this.withIndexMutationLease("clear", async (recoveredOwners) => {
|
|
11696
12183
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
11697
|
-
|
|
12184
|
+
const recovery = this.beginClearRecoveryState();
|
|
12185
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
11698
12186
|
});
|
|
11699
12187
|
}
|
|
11700
|
-
|
|
12188
|
+
clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot) {
|
|
11701
12189
|
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
|
|
11712
|
-
|
|
11713
|
-
|
|
11714
|
-
|
|
11715
|
-
|
|
11716
|
-
|
|
12190
|
+
const clearedBranchKeys = database.getAllBranches();
|
|
12191
|
+
store.clear();
|
|
12192
|
+
store.save();
|
|
12193
|
+
invertedIndex.clear();
|
|
12194
|
+
this.saveInvertedIndex(invertedIndex);
|
|
12195
|
+
this.fileHashCache.clear();
|
|
12196
|
+
this.saveFileHashCache();
|
|
12197
|
+
database.clearAllIndexedData();
|
|
12198
|
+
this.deleteBranchCommitMetadata(database, clearedBranchKeys);
|
|
12199
|
+
this.clearFailedBatchState();
|
|
12200
|
+
database.deleteMetadata("index.version");
|
|
12201
|
+
database.deleteMetadata("index.pathStorageVersion");
|
|
12202
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
12203
|
+
database.deleteMetadata("index.embeddingModel");
|
|
12204
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
12205
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
12206
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
12207
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
12208
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash));
|
|
12209
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash));
|
|
12210
|
+
database.deleteMetadata("index.createdAt");
|
|
12211
|
+
database.deleteMetadata("index.updatedAt");
|
|
12212
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
12213
|
+
}
|
|
12214
|
+
clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots(), recoveryDecision) {
|
|
12215
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
12216
|
+
store.load();
|
|
12217
|
+
invertedIndex.load();
|
|
12218
|
+
this.loadFileHashCache();
|
|
12219
|
+
const compatibility = this.checkCompatibility();
|
|
12220
|
+
const compatibilityDecision = recoveryDecision ?? (compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible");
|
|
12221
|
+
const allMetadata = store.getAllMetadata();
|
|
12222
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
12223
|
+
if (compatibilityDecision !== "compatible" && hasForeignData) {
|
|
12224
|
+
if (compatibilityDecision === "embedding-strategy-mismatch") {
|
|
12225
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
12226
|
+
this.clearScopedFileHashCache(roots);
|
|
12227
|
+
this.clearScopedFailedBatches(roots);
|
|
12228
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
12229
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true");
|
|
12230
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
12231
|
+
database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash));
|
|
12232
|
+
if (projectRoot === this.projectRoot) {
|
|
11717
12233
|
this.indexCompatibility = { compatible: true };
|
|
11718
|
-
return;
|
|
11719
12234
|
}
|
|
11720
|
-
throw new Error(
|
|
11721
|
-
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
11722
|
-
);
|
|
11723
|
-
}
|
|
11724
|
-
if (!hasForeignData) {
|
|
11725
|
-
const clearedBranchKeys2 = database.getAllBranches();
|
|
11726
|
-
store.clear();
|
|
11727
|
-
store.save();
|
|
11728
|
-
invertedIndex.clear();
|
|
11729
|
-
this.saveInvertedIndex(invertedIndex);
|
|
11730
|
-
this.fileHashCache.clear();
|
|
11731
|
-
this.saveFileHashCache();
|
|
11732
|
-
database.clearAllIndexedData();
|
|
11733
|
-
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
11734
|
-
this.clearFailedBatchState();
|
|
11735
|
-
database.deleteMetadata("index.version");
|
|
11736
|
-
database.deleteMetadata("index.pathStorageVersion");
|
|
11737
|
-
database.deleteMetadata("index.embeddingProvider");
|
|
11738
|
-
database.deleteMetadata("index.embeddingModel");
|
|
11739
|
-
database.deleteMetadata("index.embeddingDimensions");
|
|
11740
|
-
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
11741
|
-
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
11742
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11743
|
-
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
11744
|
-
database.deleteMetadata("index.createdAt");
|
|
11745
|
-
database.deleteMetadata("index.updatedAt");
|
|
11746
|
-
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
11747
12235
|
return;
|
|
11748
12236
|
}
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
12237
|
+
throw new Error(
|
|
12238
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
12239
|
+
);
|
|
12240
|
+
}
|
|
12241
|
+
if (!hasForeignData) {
|
|
12242
|
+
this.clearGlobalIndexDataUnlocked(projectRoot);
|
|
12243
|
+
return;
|
|
12244
|
+
}
|
|
12245
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
12246
|
+
this.clearScopedFileHashCache(roots);
|
|
12247
|
+
this.clearScopedFailedBatches(roots);
|
|
12248
|
+
if (projectRoot === this.projectRoot) {
|
|
11752
12249
|
this.indexCompatibility = compatibility;
|
|
12250
|
+
}
|
|
12251
|
+
}
|
|
12252
|
+
async clearIndexUnlocked(recoveryDecision) {
|
|
12253
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
12254
|
+
if (this.config.scope === "global") {
|
|
12255
|
+
this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision);
|
|
11753
12256
|
return;
|
|
11754
12257
|
}
|
|
11755
12258
|
if (!this.isProjectOwnedIndexPath()) {
|
|
@@ -11915,6 +12418,7 @@ var Indexer = class _Indexer {
|
|
|
11915
12418
|
)) {
|
|
11916
12419
|
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
11917
12420
|
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
12421
|
+
this.restoreMissingChunkRows(database, chunks);
|
|
11918
12422
|
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
11919
12423
|
store,
|
|
11920
12424
|
provider,
|
|
@@ -11929,6 +12433,7 @@ var Indexer = class _Indexer {
|
|
|
11929
12433
|
forceReembed: false,
|
|
11930
12434
|
reuseCachedEmbeddings: false,
|
|
11931
12435
|
incrementRepeatedFailures: false,
|
|
12436
|
+
forceSingleItemBatches: true,
|
|
11932
12437
|
onSucceeded: (succeededChunks) => {
|
|
11933
12438
|
database.addChunksToBranchBatch(
|
|
11934
12439
|
this.getBranchCatalogKey(),
|
|
@@ -11950,9 +12455,12 @@ var Indexer = class _Indexer {
|
|
|
11950
12455
|
this.saveInvertedIndex(invertedIndex);
|
|
11951
12456
|
}
|
|
11952
12457
|
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
11953
|
-
database.
|
|
11954
|
-
|
|
11955
|
-
|
|
12458
|
+
const migrationFinalized = database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true";
|
|
12459
|
+
if (migrationFinalized) {
|
|
12460
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12461
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12462
|
+
this.indexCompatibility = { compatible: true };
|
|
12463
|
+
}
|
|
11956
12464
|
}
|
|
11957
12465
|
return { succeeded, failed, remaining };
|
|
11958
12466
|
}
|
|
@@ -11974,7 +12482,8 @@ var Indexer = class _Indexer {
|
|
|
11974
12482
|
latestById.set(chunkId, {
|
|
11975
12483
|
attemptCount: batch.attemptCount,
|
|
11976
12484
|
error: batch.error,
|
|
11977
|
-
lastAttempt: batch.lastAttempt
|
|
12485
|
+
lastAttempt: batch.lastAttempt,
|
|
12486
|
+
chunks: [rawChunk]
|
|
11978
12487
|
});
|
|
11979
12488
|
}
|
|
11980
12489
|
}
|
|
@@ -12041,6 +12550,7 @@ var Indexer = class _Indexer {
|
|
|
12041
12550
|
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
12042
12551
|
);
|
|
12043
12552
|
}
|
|
12553
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
12044
12554
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
12045
12555
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
12046
12556
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
@@ -12049,7 +12559,8 @@ var Indexer = class _Indexer {
|
|
|
12049
12559
|
embedding,
|
|
12050
12560
|
limit * 2,
|
|
12051
12561
|
branchChunkIds,
|
|
12052
|
-
shouldPrefilterByBranch
|
|
12562
|
+
shouldPrefilterByBranch,
|
|
12563
|
+
temporalChunkIds
|
|
12053
12564
|
);
|
|
12054
12565
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
12055
12566
|
if (this.config.scope !== "global" && branchChunkIds && !hasInitializedBranchCatalog) {
|
|
@@ -12631,6 +13142,36 @@ function resolveConfigPathValue(value, baseDir) {
|
|
|
12631
13142
|
const absolutePath = path16.isAbsolute(trimmed) ? trimmed : path16.resolve(baseDir, trimmed);
|
|
12632
13143
|
return path16.normalize(absolutePath);
|
|
12633
13144
|
}
|
|
13145
|
+
function serializeConfigPathValue(value, baseDir) {
|
|
13146
|
+
const trimmed = value.trim();
|
|
13147
|
+
if (!trimmed) {
|
|
13148
|
+
return trimmed;
|
|
13149
|
+
}
|
|
13150
|
+
if (!path16.isAbsolute(trimmed)) {
|
|
13151
|
+
return normalizePathSeparators(path16.normalize(trimmed));
|
|
13152
|
+
}
|
|
13153
|
+
const relativePath = path16.relative(baseDir, trimmed);
|
|
13154
|
+
if (!relativePath || !relativePath.startsWith("..") && !path16.isAbsolute(relativePath)) {
|
|
13155
|
+
return normalizePathSeparators(path16.normalize(relativePath || "."));
|
|
13156
|
+
}
|
|
13157
|
+
return path16.normalize(trimmed);
|
|
13158
|
+
}
|
|
13159
|
+
function resolveKnowledgeBasePath(value, projectRoot) {
|
|
13160
|
+
return path16.isAbsolute(value) ? value : path16.resolve(projectRoot, value);
|
|
13161
|
+
}
|
|
13162
|
+
function normalizeKnowledgeBasePath(value, projectRoot) {
|
|
13163
|
+
return path16.normalize(resolveKnowledgeBasePath(value, projectRoot));
|
|
13164
|
+
}
|
|
13165
|
+
function hasMatchingKnowledgeBasePath(knowledgeBases, inputPath, projectRoot) {
|
|
13166
|
+
const normalizedInput = path16.normalize(inputPath);
|
|
13167
|
+
return knowledgeBases.some((kb) => normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput);
|
|
13168
|
+
}
|
|
13169
|
+
function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot) {
|
|
13170
|
+
const normalizedInput = path16.normalize(inputPath);
|
|
13171
|
+
return knowledgeBases.findIndex(
|
|
13172
|
+
(kb) => path16.normalize(kb) === normalizedInput || normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput
|
|
13173
|
+
);
|
|
13174
|
+
}
|
|
12634
13175
|
|
|
12635
13176
|
// src/tools/format-communities.ts
|
|
12636
13177
|
function compareText(left, right) {
|
|
@@ -14337,7 +14878,7 @@ function applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key
|
|
|
14337
14878
|
function mergeUniqueStringArray(values) {
|
|
14338
14879
|
return [...new Set(values.map((value) => String(value).trim()))];
|
|
14339
14880
|
}
|
|
14340
|
-
function
|
|
14881
|
+
function normalizeKnowledgeBasePath2(value) {
|
|
14341
14882
|
let normalized = path19.normalize(String(value).trim());
|
|
14342
14883
|
const root = path19.parse(normalized).root;
|
|
14343
14884
|
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
@@ -14346,7 +14887,7 @@ function normalizeKnowledgeBasePath(value) {
|
|
|
14346
14887
|
return normalized;
|
|
14347
14888
|
}
|
|
14348
14889
|
function mergeKnowledgeBasePaths(values) {
|
|
14349
|
-
return [...new Set(values.map((value) =>
|
|
14890
|
+
return [...new Set(values.map((value) => normalizeKnowledgeBasePath2(value)).filter((value) => value.length > 0))];
|
|
14350
14891
|
}
|
|
14351
14892
|
function validateConfigLayerShape(rawConfig, filePath) {
|
|
14352
14893
|
if (!isRecord(rawConfig)) {
|
|
@@ -14476,9 +15017,30 @@ function toConfigRecord(rawConfig) {
|
|
|
14476
15017
|
}
|
|
14477
15018
|
return { ...rawConfig };
|
|
14478
15019
|
}
|
|
15020
|
+
function getConfigPath(projectRoot, host) {
|
|
15021
|
+
return resolveWritableProjectConfigPath(projectRoot, host);
|
|
15022
|
+
}
|
|
14479
15023
|
function loadRuntimeConfig(projectRoot, host) {
|
|
14480
15024
|
return normalizeKnowledgeBasePaths(toConfigRecord(loadMergedConfig(projectRoot, host)), projectRoot);
|
|
14481
15025
|
}
|
|
15026
|
+
function loadEditableConfig(projectRoot, host) {
|
|
15027
|
+
return normalizeKnowledgeBasePaths(toConfigRecord(loadProjectConfigLayer(projectRoot, host)), projectRoot);
|
|
15028
|
+
}
|
|
15029
|
+
function saveConfig(projectRoot, config, host) {
|
|
15030
|
+
const configPath = getConfigPath(projectRoot, host);
|
|
15031
|
+
const configDir = path20.dirname(configPath);
|
|
15032
|
+
const configBaseDir = path20.dirname(configDir);
|
|
15033
|
+
if (!(0, import_fs13.existsSync)(configDir)) {
|
|
15034
|
+
(0, import_fs13.mkdirSync)(configDir, { recursive: true });
|
|
15035
|
+
}
|
|
15036
|
+
const serializableConfig = { ...config };
|
|
15037
|
+
if (Array.isArray(serializableConfig.knowledgeBases)) {
|
|
15038
|
+
serializableConfig.knowledgeBases = serializableConfig.knowledgeBases.map(
|
|
15039
|
+
(kb) => serializeConfigPathValue(kb, configBaseDir)
|
|
15040
|
+
);
|
|
15041
|
+
}
|
|
15042
|
+
(0, import_fs13.writeFileSync)(configPath, JSON.stringify(serializableConfig, null, 2) + "\n", "utf-8");
|
|
15043
|
+
}
|
|
14482
15044
|
|
|
14483
15045
|
// src/tools/operation-runtime.ts
|
|
14484
15046
|
var indexerCache = /* @__PURE__ */ new Map();
|
|
@@ -14697,9 +15259,11 @@ async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
|
14697
15259
|
contextLines: options.contextLines,
|
|
14698
15260
|
metadataOnly: options.metadataOnly,
|
|
14699
15261
|
definitionIntent: options.definitionIntent,
|
|
15262
|
+
prioritizeSourcePaths: options.prioritizeSourcePaths,
|
|
14700
15263
|
blameAuthor: options.blameAuthor,
|
|
14701
15264
|
blameSha: options.blameSha,
|
|
14702
15265
|
blameSince: options.blameSince,
|
|
15266
|
+
blameUntil: options.blameUntil,
|
|
14703
15267
|
trace: options.trace
|
|
14704
15268
|
});
|
|
14705
15269
|
}
|
|
@@ -14745,7 +15309,9 @@ async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
|
14745
15309
|
fileType: options.fileType,
|
|
14746
15310
|
directory: options.directory,
|
|
14747
15311
|
chunkType: options.chunkType,
|
|
14748
|
-
excludeFile: options.excludeFile
|
|
15312
|
+
excludeFile: options.excludeFile,
|
|
15313
|
+
blameSince: options.blameSince,
|
|
15314
|
+
blameUntil: options.blameUntil
|
|
14749
15315
|
});
|
|
14750
15316
|
}
|
|
14751
15317
|
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
@@ -14991,6 +15557,141 @@ async function getIndexLogs(projectRoot, host, args) {
|
|
|
14991
15557
|
}).join("\n");
|
|
14992
15558
|
return { kind: "entries", text };
|
|
14993
15559
|
}
|
|
15560
|
+
function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
15561
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15562
|
+
const inputPath = knowledgeBasePath.trim();
|
|
15563
|
+
const normalizedPath3 = path21.resolve(
|
|
15564
|
+
path21.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
|
|
15565
|
+
);
|
|
15566
|
+
if (!(0, import_fs14.existsSync)(normalizedPath3)) {
|
|
15567
|
+
return `Error: Directory does not exist: ${normalizedPath3}`;
|
|
15568
|
+
}
|
|
15569
|
+
let realPath;
|
|
15570
|
+
try {
|
|
15571
|
+
realPath = (0, import_fs14.realpathSync)(normalizedPath3);
|
|
15572
|
+
} catch {
|
|
15573
|
+
return `Error: Cannot resolve path: ${normalizedPath3}`;
|
|
15574
|
+
}
|
|
15575
|
+
const blockedPrefixes = [
|
|
15576
|
+
"/etc",
|
|
15577
|
+
"/proc",
|
|
15578
|
+
"/sys",
|
|
15579
|
+
"/dev",
|
|
15580
|
+
"/boot",
|
|
15581
|
+
"/root",
|
|
15582
|
+
"/var/run",
|
|
15583
|
+
"/var/log"
|
|
15584
|
+
];
|
|
15585
|
+
const homeDir = process.platform === "win32" ? process.env.USERPROFILE ?? "" : process.env.HOME ?? "";
|
|
15586
|
+
const sensitiveDotDirs = [
|
|
15587
|
+
".ssh",
|
|
15588
|
+
".gnupg",
|
|
15589
|
+
".aws",
|
|
15590
|
+
".config/gcloud",
|
|
15591
|
+
".docker",
|
|
15592
|
+
".kube"
|
|
15593
|
+
];
|
|
15594
|
+
for (const prefix of blockedPrefixes) {
|
|
15595
|
+
if (realPath === prefix || realPath.startsWith(`${prefix}/`)) {
|
|
15596
|
+
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
|
|
15597
|
+
}
|
|
15598
|
+
}
|
|
15599
|
+
for (const dotDir of sensitiveDotDirs) {
|
|
15600
|
+
const sensitiveDir = path21.join(homeDir, dotDir);
|
|
15601
|
+
if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
|
|
15602
|
+
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
|
|
15603
|
+
}
|
|
15604
|
+
}
|
|
15605
|
+
try {
|
|
15606
|
+
const stat5 = (0, import_fs14.statSync)(normalizedPath3);
|
|
15607
|
+
if (!stat5.isDirectory()) {
|
|
15608
|
+
return `Error: Path is not a directory: ${normalizedPath3}`;
|
|
15609
|
+
}
|
|
15610
|
+
} catch (error) {
|
|
15611
|
+
return `Error: Cannot access directory: ${normalizedPath3} - ${error instanceof Error ? error.message : String(error)}`;
|
|
15612
|
+
}
|
|
15613
|
+
const config = loadEditableConfig(root, host);
|
|
15614
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15615
|
+
const alreadyExists = hasMatchingKnowledgeBasePath(knowledgeBases, normalizedPath3, root);
|
|
15616
|
+
if (alreadyExists) {
|
|
15617
|
+
return `Knowledge base already configured: ${normalizedPath3}`;
|
|
15618
|
+
}
|
|
15619
|
+
knowledgeBases.push(normalizedPath3);
|
|
15620
|
+
config.knowledgeBases = knowledgeBases;
|
|
15621
|
+
saveConfig(root, config, host);
|
|
15622
|
+
refreshIndexerForDirectory(root, host);
|
|
15623
|
+
let result = `${normalizedPath3}
|
|
15624
|
+
`;
|
|
15625
|
+
result += `Total knowledge bases: ${knowledgeBases.length}
|
|
15626
|
+
`;
|
|
15627
|
+
result += `Config path: ${getConfigPath(root, host)}
|
|
15628
|
+
`;
|
|
15629
|
+
result += `
|
|
15630
|
+
Run /index to rebuild the index with the new knowledge base.`;
|
|
15631
|
+
return result;
|
|
15632
|
+
}
|
|
15633
|
+
function listKnowledgeBases(projectRoot, host) {
|
|
15634
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15635
|
+
const config = loadRuntimeConfig(root, host);
|
|
15636
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15637
|
+
if (knowledgeBases.length === 0) {
|
|
15638
|
+
return "No knowledge bases configured. Use add_knowledge_base to add folders.";
|
|
15639
|
+
}
|
|
15640
|
+
let result = `Knowledge Bases (${knowledgeBases.length}):
|
|
15641
|
+
|
|
15642
|
+
`;
|
|
15643
|
+
for (let i = 0; i < knowledgeBases.length; i++) {
|
|
15644
|
+
const kb = knowledgeBases[i];
|
|
15645
|
+
const resolvedPath = resolveKnowledgeBasePath(kb, root);
|
|
15646
|
+
const exists = (0, import_fs14.existsSync)(resolvedPath);
|
|
15647
|
+
result += `[${i + 1}] ${kb}
|
|
15648
|
+
`;
|
|
15649
|
+
result += ` Resolved: ${resolvedPath}
|
|
15650
|
+
`;
|
|
15651
|
+
result += ` Status: ${exists ? "Exists" : "NOT FOUND"}
|
|
15652
|
+
`;
|
|
15653
|
+
if (exists) {
|
|
15654
|
+
try {
|
|
15655
|
+
const stat5 = (0, import_fs14.statSync)(resolvedPath);
|
|
15656
|
+
result += ` Type: ${stat5.isDirectory() ? "Directory" : "File"}
|
|
15657
|
+
`;
|
|
15658
|
+
} catch {
|
|
15659
|
+
}
|
|
15660
|
+
}
|
|
15661
|
+
result += "\n";
|
|
15662
|
+
}
|
|
15663
|
+
const hasHostConfig = (0, import_fs14.existsSync)(path21.join(root, getHostProjectConfigRelativePath(host)));
|
|
15664
|
+
if (hasHostConfig) {
|
|
15665
|
+
result += `
|
|
15666
|
+
Config sources: 1 file(s).`;
|
|
15667
|
+
}
|
|
15668
|
+
result += `
|
|
15669
|
+
Config file: ${getConfigPath(root, host)}`;
|
|
15670
|
+
return result;
|
|
15671
|
+
}
|
|
15672
|
+
function removeKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
15673
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15674
|
+
const config = loadEditableConfig(root, host);
|
|
15675
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15676
|
+
const index = findKnowledgeBasePathIndex(knowledgeBases, knowledgeBasePath, root);
|
|
15677
|
+
if (index === -1) {
|
|
15678
|
+
return `Knowledge base not found: ${knowledgeBasePath}`;
|
|
15679
|
+
}
|
|
15680
|
+
const removed = knowledgeBases.splice(index, 1)[0];
|
|
15681
|
+
config.knowledgeBases = knowledgeBases;
|
|
15682
|
+
saveConfig(root, config, host);
|
|
15683
|
+
refreshIndexerForDirectory(root, host);
|
|
15684
|
+
let result = `Removed: ${removed}
|
|
15685
|
+
|
|
15686
|
+
`;
|
|
15687
|
+
result += `Remaining knowledge bases: ${knowledgeBases.length}
|
|
15688
|
+
`;
|
|
15689
|
+
result += `Config saved to: ${getConfigPath(root, host)}
|
|
15690
|
+
`;
|
|
15691
|
+
result += `
|
|
15692
|
+
Run /index to rebuild the index without the removed knowledge base.`;
|
|
15693
|
+
return result;
|
|
15694
|
+
}
|
|
14994
15695
|
|
|
14995
15696
|
// src/tools/context-search.ts
|
|
14996
15697
|
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
@@ -15219,13 +15920,19 @@ async function resolveSearchContext(input, operations) {
|
|
|
15219
15920
|
(trace) => operations.lookup(symbol, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
|
|
15220
15921
|
);
|
|
15221
15922
|
};
|
|
15222
|
-
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt) => {
|
|
15923
|
+
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt, prioritizeSourcePaths) => {
|
|
15223
15924
|
return recordAttempt(
|
|
15224
15925
|
"conceptual",
|
|
15225
15926
|
searchQuery,
|
|
15226
15927
|
scope,
|
|
15227
15928
|
relaxedFieldsForAttempt,
|
|
15228
|
-
(trace) => operations.search(
|
|
15929
|
+
(trace) => operations.search(
|
|
15930
|
+
searchQuery,
|
|
15931
|
+
MAX_CONTEXT_RESULT_LIMIT,
|
|
15932
|
+
scope,
|
|
15933
|
+
input.diagnostic ? trace : void 0,
|
|
15934
|
+
{ prioritizeSourcePaths }
|
|
15935
|
+
)
|
|
15229
15936
|
);
|
|
15230
15937
|
};
|
|
15231
15938
|
const findSuccessfulAttemptState = (route) => {
|
|
@@ -15353,10 +16060,12 @@ Explicit symbol lookup only; conceptual search was not attempted.`
|
|
|
15353
16060
|
}
|
|
15354
16061
|
}
|
|
15355
16062
|
for (const attempt of conceptualAttemptPlan) {
|
|
16063
|
+
const attemptIntent = analyzeQueryIntent(attempt.queryText);
|
|
16064
|
+
const prioritizeSourcePaths = attemptIntent.primary !== "docs" && attemptIntent.primary !== "test";
|
|
15356
16065
|
if (inferredSymbol && attempt.queryText === inferredSymbol && attempt.queryText !== query) {
|
|
15357
16066
|
decisions.fallbackFromOriginalConceptualToInferred = true;
|
|
15358
16067
|
}
|
|
15359
|
-
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed);
|
|
16068
|
+
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed, prioritizeSourcePaths);
|
|
15360
16069
|
if (results.length > 0) {
|
|
15361
16070
|
const heading = buildPackHeading("conceptual", decisions);
|
|
15362
16071
|
const intent = analyzeQueryIntent(attempt.queryText);
|
|
@@ -15493,12 +16202,13 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
15493
16202
|
directory: scope.directory,
|
|
15494
16203
|
trace
|
|
15495
16204
|
}),
|
|
15496
|
-
search: (queryText, retrievalLimit, scope, trace) => searchCodebase(projectRoot, host, queryText, {
|
|
16205
|
+
search: (queryText, retrievalLimit, scope, trace, searchOptions) => searchCodebase(projectRoot, host, queryText, {
|
|
15497
16206
|
limit: retrievalLimit,
|
|
15498
16207
|
fileType: scope.fileType,
|
|
15499
16208
|
directory: scope.directory,
|
|
15500
16209
|
metadataOnly: true,
|
|
15501
|
-
trace
|
|
16210
|
+
trace,
|
|
16211
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
15502
16212
|
})
|
|
15503
16213
|
});
|
|
15504
16214
|
}
|
|
@@ -16999,12 +17709,13 @@ async function runEvaluation(options) {
|
|
|
16999
17709
|
fileType: scope.fileType,
|
|
17000
17710
|
directory: scope.directory
|
|
17001
17711
|
}),
|
|
17002
|
-
search: (searchQuery, limit, scope) => indexer.search(searchQuery, limit, {
|
|
17712
|
+
search: (searchQuery, limit, scope, _trace, searchOptions) => indexer.search(searchQuery, limit, {
|
|
17003
17713
|
metadataOnly: true,
|
|
17004
17714
|
filterByBranch: !!query.expected.branch,
|
|
17005
17715
|
definitionIntent: false,
|
|
17006
17716
|
fileType: scope.fileType,
|
|
17007
|
-
directory: scope.directory
|
|
17717
|
+
directory: scope.directory,
|
|
17718
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
17008
17719
|
})
|
|
17009
17720
|
}) : void 0;
|
|
17010
17721
|
const result = editContextResult?.results ?? contextResult?.details?.results ?? await indexer.search(query.query, 10, {
|
|
@@ -17781,11 +18492,21 @@ var PI_TOOL_NAMES = [
|
|
|
17781
18492
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
17782
18493
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
17783
18494
|
];
|
|
18495
|
+
var MCP_TOOL_NAMES = [
|
|
18496
|
+
...PORTABLE_TOOL_NAMES,
|
|
18497
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
18498
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18499
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE
|
|
18500
|
+
];
|
|
17784
18501
|
|
|
17785
18502
|
// src/adapters/mcp/register-tools.ts
|
|
17786
18503
|
function allowNullAsUndefined(schema) {
|
|
17787
18504
|
return import_zod2.z.preprocess((value) => value === null ? void 0 : value, schema);
|
|
17788
18505
|
}
|
|
18506
|
+
function knowledgeBaseResult(text) {
|
|
18507
|
+
const content = [{ type: "text", text }];
|
|
18508
|
+
return text.startsWith("Error: ") ? { content, isError: true } : { content };
|
|
18509
|
+
}
|
|
17789
18510
|
function registerMcpTools(server, runtime) {
|
|
17790
18511
|
server.tool(
|
|
17791
18512
|
TOOL_NAME.CODEBASE_CONTEXT,
|
|
@@ -17846,7 +18567,8 @@ function registerMcpTools(server, runtime) {
|
|
|
17846
18567
|
contextLines: allowNullAsUndefined(import_zod2.z.number().optional()).describe("Number of extra lines to include before/after each match (default: 0)"),
|
|
17847
18568
|
blameAuthor: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame author name or email"),
|
|
17848
18569
|
blameSha: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame commit SHA or prefix"),
|
|
17849
|
-
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date")
|
|
18570
|
+
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18571
|
+
blameUntil: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17850
18572
|
},
|
|
17851
18573
|
async (args) => {
|
|
17852
18574
|
return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "search", args.query, {
|
|
@@ -17857,7 +18579,8 @@ function registerMcpTools(server, runtime) {
|
|
|
17857
18579
|
contextLines: args.contextLines,
|
|
17858
18580
|
blameAuthor: args.blameAuthor,
|
|
17859
18581
|
blameSha: args.blameSha,
|
|
17860
|
-
blameSince: args.blameSince
|
|
18582
|
+
blameSince: args.blameSince,
|
|
18583
|
+
blameUntil: args.blameUntil
|
|
17861
18584
|
}, (results) => {
|
|
17862
18585
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} results for "${args.query}":
|
|
17863
18586
|
|
|
@@ -17877,7 +18600,8 @@ ${formatSearchResults(results, "score")}`;
|
|
|
17877
18600
|
chunkType: allowNullAsUndefined(import_zod2.z.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
|
|
17878
18601
|
blameAuthor: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame author name or email"),
|
|
17879
18602
|
blameSha: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame commit SHA or prefix"),
|
|
17880
|
-
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date")
|
|
18603
|
+
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18604
|
+
blameUntil: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17881
18605
|
},
|
|
17882
18606
|
async (args) => {
|
|
17883
18607
|
return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "peek", args.query, {
|
|
@@ -17888,7 +18612,8 @@ ${formatSearchResults(results, "score")}`;
|
|
|
17888
18612
|
metadataOnly: true,
|
|
17889
18613
|
blameAuthor: args.blameAuthor,
|
|
17890
18614
|
blameSha: args.blameSha,
|
|
17891
|
-
blameSince: args.blameSince
|
|
18615
|
+
blameSince: args.blameSince,
|
|
18616
|
+
blameUntil: args.blameUntil
|
|
17892
18617
|
}, (results) => {
|
|
17893
18618
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} locations for "${args.query}":
|
|
17894
18619
|
|
|
@@ -17965,7 +18690,9 @@ ${formatCodebasePeek(results)}`;
|
|
|
17965
18690
|
fileType: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
17966
18691
|
directory: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
17967
18692
|
chunkType: allowNullAsUndefined(import_zod2.z.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
|
|
17968
|
-
excludeFile: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Exclude results from this file path")
|
|
18693
|
+
excludeFile: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Exclude results from this file path"),
|
|
18694
|
+
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18695
|
+
blameUntil: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17969
18696
|
},
|
|
17970
18697
|
async (args) => {
|
|
17971
18698
|
const results = await findSimilarCode(runtime.projectRoot, runtime.host, args.code, {
|
|
@@ -17973,7 +18700,9 @@ ${formatCodebasePeek(results)}`;
|
|
|
17973
18700
|
fileType: args.fileType,
|
|
17974
18701
|
directory: args.directory,
|
|
17975
18702
|
chunkType: args.chunkType,
|
|
17976
|
-
excludeFile: args.excludeFile
|
|
18703
|
+
excludeFile: args.excludeFile,
|
|
18704
|
+
blameSince: args.blameSince,
|
|
18705
|
+
blameUntil: args.blameUntil
|
|
17977
18706
|
});
|
|
17978
18707
|
if (results.length === 0) {
|
|
17979
18708
|
return { content: [{ type: "text", text: "No similar code found. Try a different snippet or run index_codebase first." }] };
|
|
@@ -18077,6 +18806,37 @@ ${formatSearchResults(results)}` }] };
|
|
|
18077
18806
|
return { content: [{ type: "text", text: result.text }] };
|
|
18078
18807
|
}
|
|
18079
18808
|
);
|
|
18809
|
+
server.tool(
|
|
18810
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
18811
|
+
"Add a folder as a knowledge base to the semantic search index. The folder is indexed alongside the project code on the next index run. Provide an absolute path or a path relative to the project root. The path is written to the project-local host config of this MCP server (under the server project root), not to a user-global config, and the index is refreshed. Git blame metadata is collected only for files in the project git repo; knowledge-base files outside the repo remain searchable by content but have no blame. A knowledge base that appears in list_knowledge_bases but was inherited from a global config cannot be removed by this tool.",
|
|
18812
|
+
{
|
|
18813
|
+
path: import_zod2.z.string().describe("Path to the folder to add as a knowledge base (absolute or relative to the project root)")
|
|
18814
|
+
},
|
|
18815
|
+
async (args) => {
|
|
18816
|
+
const result = addKnowledgeBase(runtime.projectRoot, runtime.host, args.path);
|
|
18817
|
+
return knowledgeBaseResult(result);
|
|
18818
|
+
}
|
|
18819
|
+
);
|
|
18820
|
+
server.tool(
|
|
18821
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18822
|
+
"List the configured knowledge base folders that the index includes alongside the project code. The list is the union of project-local and user-global knowledge bases; each entry shows the resolved path and whether it exists.",
|
|
18823
|
+
{},
|
|
18824
|
+
async () => {
|
|
18825
|
+
const result = listKnowledgeBases(runtime.projectRoot, runtime.host);
|
|
18826
|
+
return knowledgeBaseResult(result);
|
|
18827
|
+
}
|
|
18828
|
+
);
|
|
18829
|
+
server.tool(
|
|
18830
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE,
|
|
18831
|
+
"Remove a knowledge base folder from the semantic search index and refresh the index. The path must match a project-local configured path exactly. Knowledge bases inherited from a user-global config are not removable by this tool.",
|
|
18832
|
+
{
|
|
18833
|
+
path: import_zod2.z.string().describe("Path of the knowledge base to remove (must match a project-local configured path exactly)")
|
|
18834
|
+
},
|
|
18835
|
+
async (args) => {
|
|
18836
|
+
const result = removeKnowledgeBase(runtime.projectRoot, runtime.host, args.path.trim());
|
|
18837
|
+
return knowledgeBaseResult(result);
|
|
18838
|
+
}
|
|
18839
|
+
);
|
|
18080
18840
|
}
|
|
18081
18841
|
|
|
18082
18842
|
// src/adapters/mcp/server.ts
|