opencode-codebase-index 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +16 -5
- package/commands/peek.md +2 -1
- package/commands/search.md +2 -1
- package/dist/cbi.cjs +14792 -0
- package/dist/cbi.cjs.map +1 -0
- package/dist/cbi.js +14786 -0
- package/dist/cbi.js.map +1 -0
- package/dist/cli.cjs +1121 -263
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1121 -263
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +885 -259
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +885 -259
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +5529 -2154
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +5517 -2142
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +5 -2
- package/skill/SKILL.md +12 -16
package/dist/cli.cjs
CHANGED
|
@@ -730,6 +730,17 @@ var EMBEDDING_MODELS = {
|
|
|
730
730
|
maxTokens: 2048,
|
|
731
731
|
costPer1MTokens: 0.15,
|
|
732
732
|
taskAble: true
|
|
733
|
+
},
|
|
734
|
+
"gemini-embedding-2": {
|
|
735
|
+
provider: "google",
|
|
736
|
+
model: "gemini-embedding-2",
|
|
737
|
+
// Keep a conservative, testable default embedding dimension. Gemini Embedding 2 supports
|
|
738
|
+
// flexible dimensions via outputDimensionality.
|
|
739
|
+
dimensions: 1536,
|
|
740
|
+
maxTokens: 8192,
|
|
741
|
+
costPer1MTokens: 0.15,
|
|
742
|
+
taskAble: false,
|
|
743
|
+
promptStyle: "embedding-2"
|
|
733
744
|
}
|
|
734
745
|
},
|
|
735
746
|
"openai": {
|
|
@@ -763,26 +774,15 @@ var EMBEDDING_MODELS = {
|
|
|
763
774
|
maxTokens: 512,
|
|
764
775
|
costPer1MTokens: 0
|
|
765
776
|
}
|
|
766
|
-
},
|
|
767
|
-
"github-copilot": {
|
|
768
|
-
"text-embedding-3-small": {
|
|
769
|
-
provider: "github-copilot",
|
|
770
|
-
model: "text-embedding-3-small",
|
|
771
|
-
dimensions: 1536,
|
|
772
|
-
maxTokens: 8191,
|
|
773
|
-
costPer1MTokens: 0
|
|
774
|
-
}
|
|
775
777
|
}
|
|
776
778
|
};
|
|
777
779
|
var DEFAULT_PROVIDER_MODELS = {
|
|
778
|
-
"github-copilot": "text-embedding-3-small",
|
|
779
780
|
"openai": "text-embedding-3-small",
|
|
780
781
|
"google": "gemini-embedding-001",
|
|
781
782
|
"ollama": "nomic-embed-text"
|
|
782
783
|
};
|
|
783
784
|
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
784
785
|
"ollama",
|
|
785
|
-
"github-copilot",
|
|
786
786
|
"openai",
|
|
787
787
|
"google"
|
|
788
788
|
];
|
|
@@ -808,6 +808,9 @@ function getDefaultIndexingConfig() {
|
|
|
808
808
|
maxDepth: 5,
|
|
809
809
|
maxFilesPerDirectory: 100,
|
|
810
810
|
fallbackToTextOnMaxChunks: true,
|
|
811
|
+
// Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi
|
|
812
|
+
// fallback used when a native caller omits the argument).
|
|
813
|
+
linesPerChunk: 30,
|
|
811
814
|
gitBlame: { enabled: false }
|
|
812
815
|
};
|
|
813
816
|
}
|
|
@@ -941,6 +944,7 @@ function parseConfig(raw) {
|
|
|
941
944
|
maxDepth: typeof rawIndexing.maxDepth === "number" ? rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth : defaultIndexing.maxDepth,
|
|
942
945
|
maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
|
|
943
946
|
fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
|
|
947
|
+
linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk,
|
|
944
948
|
gitBlame: {
|
|
945
949
|
enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof rawIndexing.gitBlame.enabled === "boolean" ? rawIndexing.gitBlame.enabled : defaultIndexing.gitBlame.enabled
|
|
946
950
|
}
|
|
@@ -983,6 +987,7 @@ function parseConfig(raw) {
|
|
|
983
987
|
let embeddingModel;
|
|
984
988
|
let customProvider;
|
|
985
989
|
let reranker;
|
|
990
|
+
const githubCopilotDeprecationMessage = '`embeddingProvider: "github-copilot"` is deprecated and no longer available. Migrate existing configs to `embeddingProvider: "google"` and select an explicit Google model. For existing indexes, run `index_codebase` with `force: true` after changing to `gemini-embedding-001` or `gemini-embedding-2` to rebuild embeddings. See docs/configuration.md for details.';
|
|
986
991
|
if (embeddingProviderValue === "custom") {
|
|
987
992
|
embeddingProvider = "custom";
|
|
988
993
|
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
@@ -1022,6 +1027,8 @@ function parseConfig(raw) {
|
|
|
1022
1027
|
} else if (rawEmbeddingModel) {
|
|
1023
1028
|
embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
1024
1029
|
}
|
|
1030
|
+
} else if (embeddingProviderValue === "github-copilot") {
|
|
1031
|
+
throw new Error(githubCopilotDeprecationMessage);
|
|
1025
1032
|
} else {
|
|
1026
1033
|
embeddingProvider = "auto";
|
|
1027
1034
|
}
|
|
@@ -1052,10 +1059,21 @@ function parseConfig(raw) {
|
|
|
1052
1059
|
timeoutMs: typeof rawReranker.timeoutMs === "number" ? Math.max(1e3, Math.floor(rawReranker.timeoutMs)) : 1e4
|
|
1053
1060
|
};
|
|
1054
1061
|
}
|
|
1062
|
+
const rawEmbedding = input.embedding && typeof input.embedding === "object" ? input.embedding : {};
|
|
1063
|
+
const rawEmbeddingBatch = rawEmbedding.batch && typeof rawEmbedding.batch === "object" ? rawEmbedding.batch : null;
|
|
1064
|
+
const embeddingMaxBatchItems = typeof rawEmbeddingBatch?.maxBatchItems === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchItems) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchItems)) : void 0;
|
|
1065
|
+
const embeddingMaxBatchTokens = typeof rawEmbeddingBatch?.maxBatchTokens === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchTokens) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchTokens)) : void 0;
|
|
1066
|
+
const embedding = embeddingMaxBatchItems !== void 0 || embeddingMaxBatchTokens !== void 0 ? {
|
|
1067
|
+
batch: {
|
|
1068
|
+
...embeddingMaxBatchItems !== void 0 ? { maxBatchItems: embeddingMaxBatchItems } : {},
|
|
1069
|
+
...embeddingMaxBatchTokens !== void 0 ? { maxBatchTokens: embeddingMaxBatchTokens } : {}
|
|
1070
|
+
}
|
|
1071
|
+
} : {};
|
|
1055
1072
|
return {
|
|
1056
1073
|
embeddingProvider,
|
|
1057
1074
|
embeddingModel,
|
|
1058
1075
|
customProvider,
|
|
1076
|
+
embedding,
|
|
1059
1077
|
scope: isValidScope(scopeValue) ? scopeValue : "project",
|
|
1060
1078
|
include: includeValue ?? DEFAULT_INCLUDE,
|
|
1061
1079
|
exclude: excludeValue ?? DEFAULT_EXCLUDE,
|
|
@@ -2551,8 +2569,6 @@ async function tryDetectProvider() {
|
|
|
2551
2569
|
}
|
|
2552
2570
|
async function getProviderCredentials(provider) {
|
|
2553
2571
|
switch (provider) {
|
|
2554
|
-
case "github-copilot":
|
|
2555
|
-
return getGitHubCopilotCredentials();
|
|
2556
2572
|
case "openai":
|
|
2557
2573
|
return getOpenAICredentials();
|
|
2558
2574
|
case "google":
|
|
@@ -2563,22 +2579,6 @@ async function getProviderCredentials(provider) {
|
|
|
2563
2579
|
return null;
|
|
2564
2580
|
}
|
|
2565
2581
|
}
|
|
2566
|
-
function getGitHubCopilotCredentials() {
|
|
2567
|
-
const authData = loadOpenCodeAuth();
|
|
2568
|
-
const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
|
|
2569
|
-
if (!copilotAuth || copilotAuth.type !== "oauth") {
|
|
2570
|
-
return null;
|
|
2571
|
-
}
|
|
2572
|
-
const auth = copilotAuth;
|
|
2573
|
-
const baseUrl = auth.enterpriseUrl ? `https://copilot-api.${auth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
|
|
2574
|
-
return {
|
|
2575
|
-
provider: "github-copilot",
|
|
2576
|
-
baseUrl,
|
|
2577
|
-
refreshToken: copilotAuth.refresh,
|
|
2578
|
-
accessToken: copilotAuth.access,
|
|
2579
|
-
tokenExpires: copilotAuth.expires
|
|
2580
|
-
};
|
|
2581
|
-
}
|
|
2582
2582
|
function getOpenAICredentials() {
|
|
2583
2583
|
const authData = loadOpenCodeAuth();
|
|
2584
2584
|
const openaiAuth = authData["openai"];
|
|
@@ -2704,8 +2704,6 @@ async function tryDetectOllamaProvider() {
|
|
|
2704
2704
|
}
|
|
2705
2705
|
function getProviderDisplayName(provider) {
|
|
2706
2706
|
switch (provider) {
|
|
2707
|
-
case "github-copilot":
|
|
2708
|
-
return "GitHub Copilot";
|
|
2709
2707
|
case "openai":
|
|
2710
2708
|
return "OpenAI";
|
|
2711
2709
|
case "google":
|
|
@@ -2930,44 +2928,6 @@ var CustomEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
2930
2928
|
}
|
|
2931
2929
|
};
|
|
2932
2930
|
|
|
2933
|
-
// src/embeddings/providers/github-copilot.ts
|
|
2934
|
-
var GitHubCopilotEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
2935
|
-
constructor(credentials, modelInfo) {
|
|
2936
|
-
super(credentials, modelInfo);
|
|
2937
|
-
}
|
|
2938
|
-
getToken() {
|
|
2939
|
-
if (!this.credentials.refreshToken) {
|
|
2940
|
-
throw new Error("No OAuth token available for GitHub");
|
|
2941
|
-
}
|
|
2942
|
-
return this.credentials.refreshToken;
|
|
2943
|
-
}
|
|
2944
|
-
async embedBatch(texts) {
|
|
2945
|
-
const token = this.getToken();
|
|
2946
|
-
const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
|
|
2947
|
-
method: "POST",
|
|
2948
|
-
headers: {
|
|
2949
|
-
Authorization: `Bearer ${token}`,
|
|
2950
|
-
"Content-Type": "application/json",
|
|
2951
|
-
Accept: "application/vnd.github+json",
|
|
2952
|
-
"X-GitHub-Api-Version": "2022-11-28"
|
|
2953
|
-
},
|
|
2954
|
-
body: JSON.stringify({
|
|
2955
|
-
model: `openai/${this.modelInfo.model}`,
|
|
2956
|
-
input: texts
|
|
2957
|
-
})
|
|
2958
|
-
});
|
|
2959
|
-
if (!response.ok) {
|
|
2960
|
-
const error = (await response.text()).slice(0, 500);
|
|
2961
|
-
throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
|
|
2962
|
-
}
|
|
2963
|
-
const data = await response.json();
|
|
2964
|
-
return {
|
|
2965
|
-
embeddings: data.data.map((d) => d.embedding),
|
|
2966
|
-
totalTokensUsed: data.usage.total_tokens
|
|
2967
|
-
};
|
|
2968
|
-
}
|
|
2969
|
-
};
|
|
2970
|
-
|
|
2971
2931
|
// src/embeddings/providers/google.ts
|
|
2972
2932
|
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddingProvider {
|
|
2973
2933
|
static BATCH_SIZE = 20;
|
|
@@ -2975,24 +2935,30 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
2975
2935
|
super(credentials, modelInfo);
|
|
2976
2936
|
}
|
|
2977
2937
|
async embedQuery(query) {
|
|
2978
|
-
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2979
|
-
const
|
|
2938
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2939
|
+
const texts = [
|
|
2940
|
+
this.modelInfo.model === "gemini-embedding-2" ? `task: code retrieval | query: ${query}` : query
|
|
2941
|
+
];
|
|
2942
|
+
const result = await this.embedWithTaskType(texts, taskType);
|
|
2980
2943
|
return {
|
|
2981
2944
|
embedding: result.embeddings[0],
|
|
2982
2945
|
tokensUsed: result.totalTokensUsed
|
|
2983
2946
|
};
|
|
2984
2947
|
}
|
|
2985
2948
|
async embedDocument(document) {
|
|
2986
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2987
|
-
const result = await this.embedWithTaskType([
|
|
2949
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2950
|
+
const result = await this.embedWithTaskType([
|
|
2951
|
+
this.modelInfo.model === "gemini-embedding-2" ? `title: none | text: ${document}` : document
|
|
2952
|
+
], taskType);
|
|
2988
2953
|
return {
|
|
2989
2954
|
embedding: result.embeddings[0],
|
|
2990
2955
|
tokensUsed: result.totalTokensUsed
|
|
2991
2956
|
};
|
|
2992
2957
|
}
|
|
2993
2958
|
async embedBatch(texts) {
|
|
2994
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2995
|
-
|
|
2959
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2960
|
+
const formattedTexts = this.modelInfo.model === "gemini-embedding-2" ? texts.map((text) => `title: none | text: ${text}`) : texts;
|
|
2961
|
+
return this.embedWithTaskType(formattedTexts, taskType);
|
|
2996
2962
|
}
|
|
2997
2963
|
async embedWithTaskType(texts, taskType) {
|
|
2998
2964
|
const batches = [];
|
|
@@ -3042,6 +3008,10 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
3042
3008
|
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddingProvider {
|
|
3043
3009
|
static MIN_TRUNCATION_CHARS = 512;
|
|
3044
3010
|
static REQUEST_TIMEOUT_MS = 12e4;
|
|
3011
|
+
// Set when /api/embed returns 404 so subsequent multi-text batches skip the
|
|
3012
|
+
// batched endpoint and go straight to the legacy per-text path (one probe per
|
|
3013
|
+
// old ollama install, not one probe per batch).
|
|
3014
|
+
batchEndpointUnavailable = false;
|
|
3045
3015
|
constructor(credentials, modelInfo) {
|
|
3046
3016
|
super(credentials, modelInfo);
|
|
3047
3017
|
}
|
|
@@ -3059,6 +3029,21 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3059
3029
|
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
3060
3030
|
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
3061
3031
|
}
|
|
3032
|
+
// True for a 404 from the newer /api/embed endpoint, i.e. an ollama version that
|
|
3033
|
+
// does not provide it. embedBatch uses this to fall back to the legacy per-text
|
|
3034
|
+
// /api/embeddings path so old ollama installs do not regress.
|
|
3035
|
+
isBatchEndpointUnavailableError(error) {
|
|
3036
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3037
|
+
return message.includes("Ollama /api/embed not available");
|
|
3038
|
+
}
|
|
3039
|
+
// True for a malformed /api/embed response (wrong vector count or a bad vector).
|
|
3040
|
+
// embedBatch falls back to the per-text path on this so a bad batch response
|
|
3041
|
+
// re-embeds each text cleanly. A text that then fails per-text is not isolated
|
|
3042
|
+
// here; it is isolated on the recovery run, which re-embeds one text per request.
|
|
3043
|
+
isBatchValidationError(error) {
|
|
3044
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3045
|
+
return message.includes("invalid embedding batch");
|
|
3046
|
+
}
|
|
3062
3047
|
buildTruncationCandidates(text) {
|
|
3063
3048
|
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
3064
3049
|
const candidateLimits = /* @__PURE__ */ new Set();
|
|
@@ -3160,7 +3145,74 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3160
3145
|
tokensUsed: this.estimateTokens(text)
|
|
3161
3146
|
};
|
|
3162
3147
|
}
|
|
3163
|
-
|
|
3148
|
+
// Embeds many texts in one POST /api/embed request (input: string[]). Ollama
|
|
3149
|
+
// encodes each input independently, so the model context length applies per input
|
|
3150
|
+
// (the upstream splitter already bounds each input), not over the batch. This
|
|
3151
|
+
// amortizes N HTTP round-trips into one.
|
|
3152
|
+
async embedMany(texts) {
|
|
3153
|
+
const controller = new AbortController();
|
|
3154
|
+
const timeout = setTimeout(
|
|
3155
|
+
() => controller.abort(),
|
|
3156
|
+
_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS
|
|
3157
|
+
);
|
|
3158
|
+
let response;
|
|
3159
|
+
try {
|
|
3160
|
+
response = await fetch(`${this.credentials.baseUrl}/api/embed`, {
|
|
3161
|
+
method: "POST",
|
|
3162
|
+
headers: {
|
|
3163
|
+
"Content-Type": "application/json"
|
|
3164
|
+
},
|
|
3165
|
+
body: JSON.stringify({
|
|
3166
|
+
model: this.modelInfo.model,
|
|
3167
|
+
input: texts,
|
|
3168
|
+
truncate: false
|
|
3169
|
+
}),
|
|
3170
|
+
signal: controller.signal
|
|
3171
|
+
});
|
|
3172
|
+
} catch (error) {
|
|
3173
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
3174
|
+
throw new Error(
|
|
3175
|
+
`Ollama embedding request timed out after ${_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS}ms`
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
throw error;
|
|
3179
|
+
} finally {
|
|
3180
|
+
clearTimeout(timeout);
|
|
3181
|
+
}
|
|
3182
|
+
if (!response.ok) {
|
|
3183
|
+
const error = (await response.text()).slice(0, 500);
|
|
3184
|
+
if (response.status === 404) {
|
|
3185
|
+
throw new Error(`Ollama /api/embed not available: ${response.status} - ${error}`);
|
|
3186
|
+
}
|
|
3187
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
3188
|
+
}
|
|
3189
|
+
let parsed;
|
|
3190
|
+
try {
|
|
3191
|
+
parsed = await response.json();
|
|
3192
|
+
} catch {
|
|
3193
|
+
throw new Error(
|
|
3194
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
3195
|
+
);
|
|
3196
|
+
}
|
|
3197
|
+
const data = parsed && typeof parsed === "object" ? parsed : {};
|
|
3198
|
+
if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length || data.embeddings.some(
|
|
3199
|
+
(value) => !Array.isArray(value) || value.length !== this.modelInfo.dimensions || value.some((v) => typeof v !== "number" || !Number.isFinite(v))
|
|
3200
|
+
)) {
|
|
3201
|
+
throw new Error(
|
|
3202
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
3203
|
+
);
|
|
3204
|
+
}
|
|
3205
|
+
return {
|
|
3206
|
+
embeddings: data.embeddings,
|
|
3207
|
+
totalTokensUsed: texts.reduce((sum, text) => sum + this.estimateTokens(text), 0)
|
|
3208
|
+
};
|
|
3209
|
+
}
|
|
3210
|
+
// Per-text /api/embeddings path shared by the single-text fast path and the
|
|
3211
|
+
// batch fallback. Uses the legacy endpoint one text at a time, so each text gets
|
|
3212
|
+
// its own truncation safety net and a vector validated on its own. A text that
|
|
3213
|
+
// hard-fails per-text throws here and fails the whole request batch; the recovery
|
|
3214
|
+
// run re-embeds one text per request to isolate it.
|
|
3215
|
+
async embedOneByOne(texts) {
|
|
3164
3216
|
const results = [];
|
|
3165
3217
|
for (const text of texts) {
|
|
3166
3218
|
results.push(await this.embedSingleWithFallback(text));
|
|
@@ -3170,6 +3222,26 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
3170
3222
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
3171
3223
|
};
|
|
3172
3224
|
}
|
|
3225
|
+
async embedBatch(texts) {
|
|
3226
|
+
if (texts.length === 0) {
|
|
3227
|
+
return { embeddings: [], totalTokensUsed: 0 };
|
|
3228
|
+
}
|
|
3229
|
+
if (texts.length === 1 || this.batchEndpointUnavailable) {
|
|
3230
|
+
return this.embedOneByOne(texts);
|
|
3231
|
+
}
|
|
3232
|
+
try {
|
|
3233
|
+
return await this.embedMany(texts);
|
|
3234
|
+
} catch (error) {
|
|
3235
|
+
if (this.isBatchEndpointUnavailableError(error)) {
|
|
3236
|
+
this.batchEndpointUnavailable = true;
|
|
3237
|
+
return this.embedOneByOne(texts);
|
|
3238
|
+
}
|
|
3239
|
+
if (!this.isContextLengthError(error) && !this.isBatchValidationError(error)) {
|
|
3240
|
+
throw error;
|
|
3241
|
+
}
|
|
3242
|
+
return this.embedOneByOne(texts);
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3173
3245
|
};
|
|
3174
3246
|
|
|
3175
3247
|
// src/embeddings/providers/openai.ts
|
|
@@ -3204,8 +3276,6 @@ var OpenAIEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
3204
3276
|
// src/embeddings/provider.ts
|
|
3205
3277
|
function createEmbeddingProvider(configuredProviderInfo) {
|
|
3206
3278
|
switch (configuredProviderInfo.provider) {
|
|
3207
|
-
case "github-copilot":
|
|
3208
|
-
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
3209
3279
|
case "openai":
|
|
3210
3280
|
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
3211
3281
|
case "google":
|
|
@@ -3540,6 +3610,26 @@ function formatCostEstimate(estimate) {
|
|
|
3540
3610
|
\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
|
|
3541
3611
|
`;
|
|
3542
3612
|
}
|
|
3613
|
+
function formatDryRunEstimate(estimate) {
|
|
3614
|
+
return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed.
|
|
3615
|
+
|
|
3616
|
+
Files to embed: ${estimate.filesCount.toLocaleString()}
|
|
3617
|
+
Chunks to embed: ${estimate.chunksCount.toLocaleString()}
|
|
3618
|
+
Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()}
|
|
3619
|
+
|
|
3620
|
+
The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It
|
|
3621
|
+
matches the live "Tokens used" counter only for providers that report usage on the
|
|
3622
|
+
same basis (ollama); for providers that report a server tokenizer count (OpenAI,
|
|
3623
|
+
Gemini, custom) it is only an estimate.
|
|
3624
|
+
|
|
3625
|
+
For a matching provider and a project-scoped force index, the force pass clears its
|
|
3626
|
+
own cached embeddings, so the live counter climbs to this number. A force index on a
|
|
3627
|
+
shared global index can reuse cached embeddings from other projects, and an
|
|
3628
|
+
incremental index counts cached chunks that are not re-embedded; in both cases this
|
|
3629
|
+
number is an upper bound on the live counter, so a progress percent against this
|
|
3630
|
+
total tops out below 100%.
|
|
3631
|
+
`;
|
|
3632
|
+
}
|
|
3543
3633
|
function formatBytes(bytes) {
|
|
3544
3634
|
if (bytes === 0) return "0 B";
|
|
3545
3635
|
const k = 1024;
|
|
@@ -4224,12 +4314,12 @@ try {
|
|
|
4224
4314
|
}
|
|
4225
4315
|
|
|
4226
4316
|
// src/native/parsing.ts
|
|
4227
|
-
function parseFileAsText(filePath, content) {
|
|
4228
|
-
const result = native.parseFileAsText(filePath, content);
|
|
4317
|
+
function parseFileAsText(filePath, content, linesPerChunk) {
|
|
4318
|
+
const result = native.parseFileAsText(filePath, content, linesPerChunk);
|
|
4229
4319
|
return result.map(mapChunk);
|
|
4230
4320
|
}
|
|
4231
|
-
function parseFiles(files) {
|
|
4232
|
-
const result = native.parseFiles(files);
|
|
4321
|
+
function parseFiles(files, linesPerChunk) {
|
|
4322
|
+
const result = native.parseFiles(files, linesPerChunk);
|
|
4233
4323
|
return result.map((f) => ({
|
|
4234
4324
|
path: f.path,
|
|
4235
4325
|
chunks: f.chunks.map(mapChunk),
|
|
@@ -4306,13 +4396,13 @@ var VectorStore = class {
|
|
|
4306
4396
|
const metadata = items.map((i) => JSON.stringify(i.metadata));
|
|
4307
4397
|
this.inner.addBatch(ids, vectors, metadata);
|
|
4308
4398
|
}
|
|
4309
|
-
search(queryVector, limit = 10) {
|
|
4399
|
+
search(queryVector, limit = 10, allowedIds) {
|
|
4310
4400
|
if (queryVector.length !== this.dimensions) {
|
|
4311
4401
|
throw new Error(
|
|
4312
4402
|
`Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
|
|
4313
4403
|
);
|
|
4314
4404
|
}
|
|
4315
|
-
const results = this.inner.search(queryVector, limit);
|
|
4405
|
+
const results = allowedIds === void 0 ? this.inner.search(queryVector, limit) : this.inner.searchFiltered(queryVector, limit, allowedIds);
|
|
4316
4406
|
return results.map((r) => ({
|
|
4317
4407
|
id: r.id,
|
|
4318
4408
|
score: r.score,
|
|
@@ -4540,6 +4630,10 @@ var Database = class _Database {
|
|
|
4540
4630
|
this.throwIfClosed();
|
|
4541
4631
|
return this.inner.getBranchChunkIds(branch);
|
|
4542
4632
|
}
|
|
4633
|
+
getChunkIdsByBlameDate(since, until) {
|
|
4634
|
+
this.throwIfClosed();
|
|
4635
|
+
return this.inner.getChunkIdsByBlameDate(since, until);
|
|
4636
|
+
}
|
|
4543
4637
|
getBranchDelta(branch, baseBranch) {
|
|
4544
4638
|
this.throwIfClosed();
|
|
4545
4639
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
@@ -5501,6 +5595,9 @@ function resolveWorktreeFallbackPath(projectRoot, relativePath) {
|
|
|
5501
5595
|
const fallbackPath = path10.join(mainRepoRoot, relativePath);
|
|
5502
5596
|
return (0, import_fs7.existsSync)(fallbackPath) ? fallbackPath : null;
|
|
5503
5597
|
}
|
|
5598
|
+
function getHostProjectConfigRelativePath(host) {
|
|
5599
|
+
return getProjectConfigRelativePath(host);
|
|
5600
|
+
}
|
|
5504
5601
|
function getProjectConfigCandidatePaths(projectRoot, host) {
|
|
5505
5602
|
const candidates = [path10.join(projectRoot, getProjectConfigRelativePath(host))];
|
|
5506
5603
|
if (host !== "opencode") {
|
|
@@ -5592,6 +5689,9 @@ function resolveProjectConfigPath(projectRoot, host) {
|
|
|
5592
5689
|
const candidates = getProjectConfigCandidatePaths(projectRoot, host);
|
|
5593
5690
|
return candidates.find((candidate) => (0, import_fs7.existsSync)(candidate)) ?? path10.join(projectRoot, getProjectConfigRelativePath(host));
|
|
5594
5691
|
}
|
|
5692
|
+
function resolveWritableProjectConfigPath(projectRoot, host) {
|
|
5693
|
+
return path10.join(projectRoot, getProjectConfigRelativePath(host));
|
|
5694
|
+
}
|
|
5595
5695
|
function resolveProjectIndexPath(projectRoot, scope, host) {
|
|
5596
5696
|
if (scope === "global") {
|
|
5597
5697
|
return resolveGlobalIndexPath(host);
|
|
@@ -6297,6 +6397,10 @@ function scoreCandidate(query, intent, candidate, originalIndex) {
|
|
|
6297
6397
|
let boost = 0;
|
|
6298
6398
|
if (intent.primary === "conceptual") {
|
|
6299
6399
|
boost += Math.min(0.14, overlap * 0.14);
|
|
6400
|
+
if (intent.preferSourcePaths) {
|
|
6401
|
+
boost += implementationPath ? 0.32 : 0;
|
|
6402
|
+
if (testPath || fixturePath || docsPath) boost -= 0.35;
|
|
6403
|
+
}
|
|
6300
6404
|
if (generatedOrVendor) boost -= 0.18;
|
|
6301
6405
|
if (importChunk || weakContainer) boost -= 0.04;
|
|
6302
6406
|
} else if (intent.primary === "test") {
|
|
@@ -6751,6 +6855,9 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
|
6751
6855
|
"enum_declaration",
|
|
6752
6856
|
"function_definition",
|
|
6753
6857
|
"class_definition",
|
|
6858
|
+
// Ruby module/class symbols that are declaration-bearing and navigable.
|
|
6859
|
+
"class",
|
|
6860
|
+
"module",
|
|
6754
6861
|
"class_specifier",
|
|
6755
6862
|
"struct_specifier",
|
|
6756
6863
|
"namespace_definition",
|
|
@@ -7211,6 +7318,19 @@ function parseOwner(value) {
|
|
|
7211
7318
|
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
7212
7319
|
if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
|
|
7213
7320
|
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
7321
|
+
if (candidate.recoveryProtocolVersion !== void 0 && candidate.recoveryProtocolVersion !== 1) return null;
|
|
7322
|
+
if (candidate.projectRoot !== void 0 && typeof candidate.projectRoot !== "string") return null;
|
|
7323
|
+
if (candidate.scopedRoots !== void 0) {
|
|
7324
|
+
if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) {
|
|
7325
|
+
return null;
|
|
7326
|
+
}
|
|
7327
|
+
}
|
|
7328
|
+
if (candidate.clearRecovery !== void 0) {
|
|
7329
|
+
const recovery = candidate.clearRecovery;
|
|
7330
|
+
if (typeof recovery !== "object" || recovery === null || recovery.phase !== "clearing" || typeof recovery.embeddingProvider !== "string" || recovery.embeddingProvider.length === 0 || typeof recovery.embeddingModel !== "string" || recovery.embeddingModel.length === 0 || !Number.isInteger(recovery.embeddingDimensions) || (recovery.embeddingDimensions ?? 0) <= 0 || typeof recovery.embeddingStrategyVersion !== "string" || recovery.embeddingStrategyVersion.length === 0 || recovery.compatibilityDecision !== "compatible" && recovery.compatibilityDecision !== "embedding-strategy-mismatch" && recovery.compatibilityDecision !== "incompatible" || candidate.operation !== "clear" && candidate.operation !== "force-index") {
|
|
7331
|
+
return null;
|
|
7332
|
+
}
|
|
7333
|
+
}
|
|
7214
7334
|
return candidate;
|
|
7215
7335
|
}
|
|
7216
7336
|
function parseReclaimOwner(value) {
|
|
@@ -7451,13 +7571,18 @@ function isTransientIndexLockContention(error) {
|
|
|
7451
7571
|
if (!isIndexLockContentionError(error) || !("reason" in error)) return false;
|
|
7452
7572
|
return error.reason === "active" || error.reason === "reclaiming";
|
|
7453
7573
|
}
|
|
7454
|
-
function acquireIndexLock(indexPath, operation) {
|
|
7574
|
+
function acquireIndexLock(indexPath, operation, recoveryScope) {
|
|
7455
7575
|
(0, import_fs9.mkdirSync)(indexPath, { recursive: true });
|
|
7456
7576
|
const canonicalIndexPath = import_fs9.realpathSync.native(indexPath);
|
|
7457
7577
|
const lockPath = path13.join(canonicalIndexPath, "indexing.lock");
|
|
7458
7578
|
cleanupDeadPublicationCandidates(canonicalIndexPath);
|
|
7459
7579
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
7460
|
-
const owner = createOwner(operation)
|
|
7580
|
+
const owner = recoveryScope === void 0 ? createOwner(operation) : {
|
|
7581
|
+
...createOwner(operation),
|
|
7582
|
+
recoveryProtocolVersion: 1,
|
|
7583
|
+
projectRoot: recoveryScope.projectRoot,
|
|
7584
|
+
scopedRoots: recoveryScope.scopedRoots
|
|
7585
|
+
};
|
|
7461
7586
|
if (publishJsonDirectory(lockPath, owner)) {
|
|
7462
7587
|
const lease = {
|
|
7463
7588
|
canonicalIndexPath,
|
|
@@ -7522,6 +7647,33 @@ function releaseIndexLock(lease) {
|
|
|
7522
7647
|
}
|
|
7523
7648
|
return true;
|
|
7524
7649
|
}
|
|
7650
|
+
function setIndexLockClearRecoveryState(lease, clearRecovery) {
|
|
7651
|
+
const currentOwner = readDirectoryOwner(lease.lockPath);
|
|
7652
|
+
if (!currentOwner || !sameOwner(currentOwner, lease.owner)) {
|
|
7653
|
+
throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
7654
|
+
}
|
|
7655
|
+
const nextOwner = { ...currentOwner };
|
|
7656
|
+
if (clearRecovery === null) {
|
|
7657
|
+
delete nextOwner.clearRecovery;
|
|
7658
|
+
} else {
|
|
7659
|
+
nextOwner.clearRecovery = clearRecovery;
|
|
7660
|
+
}
|
|
7661
|
+
const ownerPath = path13.join(lease.lockPath, OWNER_FILE_NAME);
|
|
7662
|
+
const temporaryPath = path13.join(
|
|
7663
|
+
lease.lockPath,
|
|
7664
|
+
`${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${(0, import_crypto2.randomUUID)()}`
|
|
7665
|
+
);
|
|
7666
|
+
try {
|
|
7667
|
+
(0, import_fs9.writeFileSync)(temporaryPath, JSON.stringify(nextOwner), {
|
|
7668
|
+
encoding: "utf-8",
|
|
7669
|
+
flag: "wx",
|
|
7670
|
+
mode: 384
|
|
7671
|
+
});
|
|
7672
|
+
retryTransientFilesystemOperation(() => (0, import_fs9.renameSync)(temporaryPath, ownerPath));
|
|
7673
|
+
} finally {
|
|
7674
|
+
if ((0, import_fs9.existsSync)(temporaryPath)) (0, import_fs9.rmSync)(temporaryPath, { force: true });
|
|
7675
|
+
}
|
|
7676
|
+
}
|
|
7525
7677
|
function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
|
|
7526
7678
|
if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
|
|
7527
7679
|
temporaryCounter += 1;
|
|
@@ -7759,6 +7911,18 @@ function createFailedBatchWriter(targetPath) {
|
|
|
7759
7911
|
temporaryPath
|
|
7760
7912
|
};
|
|
7761
7913
|
}
|
|
7914
|
+
function writeFailedBatchRecords(targetPath, records) {
|
|
7915
|
+
const writer = createFailedBatchWriter(targetPath);
|
|
7916
|
+
try {
|
|
7917
|
+
for (const record of records) {
|
|
7918
|
+
writer.write(record);
|
|
7919
|
+
}
|
|
7920
|
+
writer.commit();
|
|
7921
|
+
} catch (error) {
|
|
7922
|
+
writer.cleanup();
|
|
7923
|
+
throw error;
|
|
7924
|
+
}
|
|
7925
|
+
}
|
|
7762
7926
|
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
7763
7927
|
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
7764
7928
|
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
@@ -8041,14 +8205,18 @@ function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
|
8041
8205
|
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
8042
8206
|
return Math.min(2e3, maxChunkTokens);
|
|
8043
8207
|
}
|
|
8044
|
-
|
|
8045
|
-
|
|
8046
|
-
|
|
8047
|
-
|
|
8048
|
-
|
|
8049
|
-
};
|
|
8208
|
+
var DEFAULT_OLLAMA_MAX_BATCH_ITEMS = 16;
|
|
8209
|
+
var DEFAULT_OLLAMA_MAX_BATCH_TOKENS = 65536;
|
|
8210
|
+
function getDynamicBatchOptions(provider, embeddingBatch) {
|
|
8211
|
+
if (provider.provider !== "ollama") {
|
|
8212
|
+
return {};
|
|
8050
8213
|
}
|
|
8051
|
-
|
|
8214
|
+
const base = { maxBatchTokens: DEFAULT_OLLAMA_MAX_BATCH_TOKENS, maxBatchItems: DEFAULT_OLLAMA_MAX_BATCH_ITEMS };
|
|
8215
|
+
return {
|
|
8216
|
+
...base,
|
|
8217
|
+
...typeof embeddingBatch?.maxBatchTokens === "number" && Number.isFinite(embeddingBatch.maxBatchTokens) ? { maxBatchTokens: embeddingBatch.maxBatchTokens } : {},
|
|
8218
|
+
...typeof embeddingBatch?.maxBatchItems === "number" && Number.isFinite(embeddingBatch.maxBatchItems) ? { maxBatchItems: embeddingBatch.maxBatchItems } : {}
|
|
8219
|
+
};
|
|
8052
8220
|
}
|
|
8053
8221
|
function isSqliteCorruptionError(error) {
|
|
8054
8222
|
const message = getErrorMessage3(error).toLowerCase();
|
|
@@ -8066,6 +8234,14 @@ function getPendingChunkId(rawChunk) {
|
|
|
8066
8234
|
const id = rawChunk.id;
|
|
8067
8235
|
return typeof id === "string" ? id : null;
|
|
8068
8236
|
}
|
|
8237
|
+
function parseBlameTimestamp(value, endOfDay) {
|
|
8238
|
+
let timestampMs = Date.parse(value);
|
|
8239
|
+
if (Number.isNaN(timestampMs)) return null;
|
|
8240
|
+
if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(value.trim())) {
|
|
8241
|
+
timestampMs += 24 * 60 * 60 * 1e3 - 1;
|
|
8242
|
+
}
|
|
8243
|
+
return Math.floor(timestampMs / 1e3);
|
|
8244
|
+
}
|
|
8069
8245
|
function metadataFromBlame(blame) {
|
|
8070
8246
|
if (!blame) {
|
|
8071
8247
|
return {};
|
|
@@ -8212,7 +8388,7 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
|
|
|
8212
8388
|
const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
|
|
8213
8389
|
return [...promoted, ...remainder];
|
|
8214
8390
|
}
|
|
8215
|
-
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
8391
|
+
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source", allowNonSourcePaths = false) {
|
|
8216
8392
|
if (!prioritizeSourcePaths) {
|
|
8217
8393
|
return [];
|
|
8218
8394
|
}
|
|
@@ -8232,7 +8408,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8232
8408
|
if (!isImplementationChunkType(chunkType)) {
|
|
8233
8409
|
return false;
|
|
8234
8410
|
}
|
|
8235
|
-
if (!isLikelyImplementationPath2(chunk.filePath)) {
|
|
8411
|
+
if (!allowNonSourcePaths && !isLikelyImplementationPath2(chunk.filePath)) {
|
|
8236
8412
|
return false;
|
|
8237
8413
|
}
|
|
8238
8414
|
const nameLower = (chunk.name ?? "").toLowerCase();
|
|
@@ -8296,7 +8472,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8296
8472
|
}
|
|
8297
8473
|
foundCoveringChunk = upsertChunkCandidate(chunk, identifier, normalizedIdentifier) || foundCoveringChunk;
|
|
8298
8474
|
}
|
|
8299
|
-
if (foundCoveringChunk || !isLikelyImplementationPath2(symbol.filePath)) {
|
|
8475
|
+
if (foundCoveringChunk || !allowNonSourcePaths && !isLikelyImplementationPath2(symbol.filePath)) {
|
|
8300
8476
|
continue;
|
|
8301
8477
|
}
|
|
8302
8478
|
const symbolName = symbol.name.toLowerCase();
|
|
@@ -8350,7 +8526,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
8350
8526
|
const ranked = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
8351
8527
|
if (ranked.length === 0) {
|
|
8352
8528
|
const implementationFallback = fallbackCandidates.filter(
|
|
8353
|
-
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && isLikelyImplementationPath2(candidate.metadata.filePath)
|
|
8529
|
+
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && (allowNonSourcePaths || isLikelyImplementationPath2(candidate.metadata.filePath))
|
|
8354
8530
|
);
|
|
8355
8531
|
for (const candidate of implementationFallback) {
|
|
8356
8532
|
const nameLower = (candidate.metadata.name ?? "").toLowerCase();
|
|
@@ -8466,10 +8642,16 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
8466
8642
|
return false;
|
|
8467
8643
|
}
|
|
8468
8644
|
if (options?.blameSince) {
|
|
8469
|
-
const
|
|
8470
|
-
if (
|
|
8645
|
+
const since = parseBlameTimestamp(options.blameSince, false);
|
|
8646
|
+
if (since === null) return false;
|
|
8471
8647
|
const committedAt = candidate.metadata.blameCommittedAt;
|
|
8472
|
-
if (committedAt === void 0 || committedAt <
|
|
8648
|
+
if (committedAt === void 0 || committedAt < since) return false;
|
|
8649
|
+
}
|
|
8650
|
+
if (options?.blameUntil) {
|
|
8651
|
+
const until = parseBlameTimestamp(options.blameUntil, true);
|
|
8652
|
+
if (until === null) return false;
|
|
8653
|
+
const committedAt = candidate.metadata.blameCommittedAt;
|
|
8654
|
+
if (committedAt === void 0 || committedAt > until) return false;
|
|
8473
8655
|
}
|
|
8474
8656
|
return true;
|
|
8475
8657
|
}
|
|
@@ -8525,9 +8707,10 @@ var Indexer = class _Indexer {
|
|
|
8525
8707
|
writerArtifactFingerprint = null;
|
|
8526
8708
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
8527
8709
|
fileBatchLimits;
|
|
8710
|
+
checkpointIntervalChunks;
|
|
8528
8711
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
8529
8712
|
this.projectRoot = projectRoot;
|
|
8530
|
-
this.projectIdentityHash =
|
|
8713
|
+
this.projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
8531
8714
|
this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot;
|
|
8532
8715
|
this.branchNameOverride = runtimeOptions.branchName;
|
|
8533
8716
|
this.catalogIdentityOverride = runtimeOptions.catalogIdentity;
|
|
@@ -8537,6 +8720,7 @@ var Indexer = class _Indexer {
|
|
|
8537
8720
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
8538
8721
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
8539
8722
|
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
8723
|
+
this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks;
|
|
8540
8724
|
this.config = config;
|
|
8541
8725
|
this.host = host;
|
|
8542
8726
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -8648,6 +8832,9 @@ var Indexer = class _Indexer {
|
|
|
8648
8832
|
return path15.resolve(targetPath);
|
|
8649
8833
|
}
|
|
8650
8834
|
}
|
|
8835
|
+
getProjectIdentityHash(projectRoot) {
|
|
8836
|
+
return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
8837
|
+
}
|
|
8651
8838
|
isProjectOwnedIndexPath() {
|
|
8652
8839
|
return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host);
|
|
8653
8840
|
}
|
|
@@ -8684,7 +8871,10 @@ var Indexer = class _Indexer {
|
|
|
8684
8871
|
}
|
|
8685
8872
|
async withIndexMutationLease(operation, callback) {
|
|
8686
8873
|
this.refreshBranchInfo();
|
|
8687
|
-
const lease = acquireIndexLock(this.indexPath, operation
|
|
8874
|
+
const lease = acquireIndexLock(this.indexPath, operation, {
|
|
8875
|
+
projectRoot: this.projectRoot,
|
|
8876
|
+
scopedRoots: this.getScopedRoots()
|
|
8877
|
+
});
|
|
8688
8878
|
this.indexPath = lease.canonicalIndexPath;
|
|
8689
8879
|
this.refreshRuntimeArtifactPaths();
|
|
8690
8880
|
this.activeIndexLease = lease;
|
|
@@ -8739,6 +8929,7 @@ var Indexer = class _Indexer {
|
|
|
8739
8929
|
}
|
|
8740
8930
|
loadFileHashCache() {
|
|
8741
8931
|
if (!(0, import_fs10.existsSync)(this.fileHashCachePath)) {
|
|
8932
|
+
this.fileHashCache = /* @__PURE__ */ new Map();
|
|
8742
8933
|
return;
|
|
8743
8934
|
}
|
|
8744
8935
|
try {
|
|
@@ -8778,10 +8969,10 @@ var Indexer = class _Indexer {
|
|
|
8778
8969
|
invertedIndex.serialize()
|
|
8779
8970
|
);
|
|
8780
8971
|
}
|
|
8781
|
-
getScopedRoots() {
|
|
8782
|
-
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(
|
|
8972
|
+
getScopedRoots(projectRoot = this.projectRoot) {
|
|
8973
|
+
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(projectRoot)]);
|
|
8783
8974
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
8784
|
-
roots.add(this.getCanonicalPath(path15.resolve(
|
|
8975
|
+
roots.add(this.getCanonicalPath(path15.resolve(projectRoot, kbRoot)));
|
|
8785
8976
|
}
|
|
8786
8977
|
return Array.from(roots);
|
|
8787
8978
|
}
|
|
@@ -8852,14 +9043,17 @@ var Indexer = class _Indexer {
|
|
|
8852
9043
|
getLegacyBranchCatalogKey() {
|
|
8853
9044
|
return this.currentBranch || "default";
|
|
8854
9045
|
}
|
|
8855
|
-
getLegacyMigrationMetadataKey() {
|
|
8856
|
-
return `index.globalBranchMigration.${
|
|
9046
|
+
getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9047
|
+
return `index.globalBranchMigration.${projectIdentityHash}`;
|
|
8857
9048
|
}
|
|
8858
|
-
getProjectEmbeddingStrategyMetadataKey() {
|
|
8859
|
-
return `index.embeddingStrategyVersion.${
|
|
9049
|
+
getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9050
|
+
return `index.embeddingStrategyVersion.${projectIdentityHash}`;
|
|
8860
9051
|
}
|
|
8861
|
-
getProjectForceReembedMetadataKey() {
|
|
8862
|
-
return `index.forceReembed.${
|
|
9052
|
+
getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9053
|
+
return `index.forceReembed.${projectIdentityHash}`;
|
|
9054
|
+
}
|
|
9055
|
+
getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
9056
|
+
return `index.migrationFinalized.${projectIdentityHash}`;
|
|
8863
9057
|
}
|
|
8864
9058
|
getBranchMigrationMetadataKey(prefix, catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
8865
9059
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
@@ -8965,7 +9159,7 @@ var Indexer = class _Indexer {
|
|
|
8965
9159
|
const legacy = this.getLegacyBranchCatalogKey();
|
|
8966
9160
|
return primary === legacy ? [primary] : [primary, legacy];
|
|
8967
9161
|
}
|
|
8968
|
-
getProjectLocalScopedOwnershipIds(roots) {
|
|
9162
|
+
getProjectLocalScopedOwnershipIds(roots, projectRoot = this.projectRoot) {
|
|
8969
9163
|
const chunkIds = /* @__PURE__ */ new Set();
|
|
8970
9164
|
const symbolIds = /* @__PURE__ */ new Set();
|
|
8971
9165
|
if (!this.database) {
|
|
@@ -8973,10 +9167,10 @@ var Indexer = class _Indexer {
|
|
|
8973
9167
|
}
|
|
8974
9168
|
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
8975
9169
|
...Array.from(this.fileHashCache.keys()).filter(
|
|
8976
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
9170
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
8977
9171
|
),
|
|
8978
9172
|
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
8979
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
9173
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
8980
9174
|
)
|
|
8981
9175
|
]);
|
|
8982
9176
|
for (const filePath of projectLocalFilePaths) {
|
|
@@ -8989,15 +9183,16 @@ var Indexer = class _Indexer {
|
|
|
8989
9183
|
}
|
|
8990
9184
|
return { chunkIds, symbolIds };
|
|
8991
9185
|
}
|
|
8992
|
-
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
9186
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds, projectRoot = this.projectRoot) {
|
|
8993
9187
|
if (this.config.scope !== "global") {
|
|
8994
9188
|
return this.getBranchCatalogCleanupKeys();
|
|
8995
9189
|
}
|
|
8996
9190
|
const keys = /* @__PURE__ */ new Set();
|
|
8997
9191
|
const projectChunkIdSet = new Set(projectChunkIds);
|
|
8998
9192
|
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
9193
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
8999
9194
|
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
9000
|
-
if (branchKey.startsWith(`${
|
|
9195
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
9001
9196
|
keys.add(branchKey);
|
|
9002
9197
|
continue;
|
|
9003
9198
|
}
|
|
@@ -9007,8 +9202,10 @@ var Indexer = class _Indexer {
|
|
|
9007
9202
|
keys.add(branchKey);
|
|
9008
9203
|
}
|
|
9009
9204
|
}
|
|
9010
|
-
|
|
9011
|
-
|
|
9205
|
+
if (projectRoot === this.projectRoot) {
|
|
9206
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
9207
|
+
keys.add(branchKey);
|
|
9208
|
+
}
|
|
9012
9209
|
}
|
|
9013
9210
|
return Array.from(keys);
|
|
9014
9211
|
}
|
|
@@ -9016,10 +9213,10 @@ var Indexer = class _Indexer {
|
|
|
9016
9213
|
const canonicalFilePath = this.getCanonicalStoredFilePath(filePath);
|
|
9017
9214
|
return roots.some((root) => isPathWithinRoot2(canonicalFilePath, root));
|
|
9018
9215
|
}
|
|
9019
|
-
isFileInProjectRoot(filePath) {
|
|
9216
|
+
isFileInProjectRoot(filePath, projectRoot = this.projectRoot) {
|
|
9020
9217
|
return isPathWithinRoot2(
|
|
9021
9218
|
this.getCanonicalStoredFilePath(filePath),
|
|
9022
|
-
this.getCanonicalPath(
|
|
9219
|
+
this.getCanonicalPath(projectRoot)
|
|
9023
9220
|
);
|
|
9024
9221
|
}
|
|
9025
9222
|
clearScopedFileHashCache(roots) {
|
|
@@ -9061,12 +9258,12 @@ var Indexer = class _Indexer {
|
|
|
9061
9258
|
}
|
|
9062
9259
|
return false;
|
|
9063
9260
|
}
|
|
9064
|
-
hasForeignScopedBranchData() {
|
|
9261
|
+
hasForeignScopedBranchData(projectRoot = this.projectRoot, roots = this.getScopedRoots(projectRoot)) {
|
|
9065
9262
|
if (!this.database || this.config.scope !== "global") {
|
|
9066
9263
|
return false;
|
|
9067
9264
|
}
|
|
9068
|
-
const
|
|
9069
|
-
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
9265
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
9266
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot);
|
|
9070
9267
|
return this.database.getAllBranches().some(
|
|
9071
9268
|
(branchKey) => {
|
|
9072
9269
|
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
@@ -9075,7 +9272,7 @@ var Indexer = class _Indexer {
|
|
|
9075
9272
|
if (!hasBranchData) {
|
|
9076
9273
|
return false;
|
|
9077
9274
|
}
|
|
9078
|
-
if (branchKey.startsWith(`${
|
|
9275
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
9079
9276
|
return false;
|
|
9080
9277
|
}
|
|
9081
9278
|
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
@@ -9084,7 +9281,7 @@ var Indexer = class _Indexer {
|
|
|
9084
9281
|
}
|
|
9085
9282
|
);
|
|
9086
9283
|
}
|
|
9087
|
-
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
9284
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot = this.projectRoot) {
|
|
9088
9285
|
const allMetadata = store.getAllMetadata();
|
|
9089
9286
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
9090
9287
|
const filePaths = /* @__PURE__ */ new Set([
|
|
@@ -9092,7 +9289,7 @@ var Indexer = class _Indexer {
|
|
|
9092
9289
|
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
9093
9290
|
]);
|
|
9094
9291
|
const projectLocalFilePaths = new Set(
|
|
9095
|
-
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath))
|
|
9292
|
+
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot))
|
|
9096
9293
|
);
|
|
9097
9294
|
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
9098
9295
|
for (const filePath of filePaths) {
|
|
@@ -9102,7 +9299,7 @@ var Indexer = class _Indexer {
|
|
|
9102
9299
|
}
|
|
9103
9300
|
const removedChunkIdList = Array.from(removedChunkIds);
|
|
9104
9301
|
const projectLocalChunkIds = new Set(
|
|
9105
|
-
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)).map(({ key }) => key)
|
|
9302
|
+
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)).map(({ key }) => key)
|
|
9106
9303
|
);
|
|
9107
9304
|
for (const filePath of projectLocalFilePaths) {
|
|
9108
9305
|
for (const chunk of database.getChunksByFile(filePath)) {
|
|
@@ -9121,7 +9318,8 @@ var Indexer = class _Indexer {
|
|
|
9121
9318
|
}
|
|
9122
9319
|
const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys(
|
|
9123
9320
|
Array.from(projectLocalChunkIds),
|
|
9124
|
-
Array.from(projectLocalSymbolIds)
|
|
9321
|
+
Array.from(projectLocalSymbolIds),
|
|
9322
|
+
projectRoot
|
|
9125
9323
|
);
|
|
9126
9324
|
for (const branchKey of branchCleanupKeys) {
|
|
9127
9325
|
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
@@ -9156,29 +9354,96 @@ var Indexer = class _Indexer {
|
|
|
9156
9354
|
database.gcOrphanSymbols();
|
|
9157
9355
|
database.gcOrphanEmbeddings();
|
|
9158
9356
|
database.gcOrphanChunks();
|
|
9159
|
-
store.save();
|
|
9160
9357
|
this.saveInvertedIndex(invertedIndex);
|
|
9358
|
+
store.save();
|
|
9161
9359
|
return {
|
|
9162
9360
|
removedChunkIds: removedChunkIdList,
|
|
9163
9361
|
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
9164
9362
|
};
|
|
9165
9363
|
}
|
|
9364
|
+
getCurrentClearRecoveryState() {
|
|
9365
|
+
if (!this.configuredProviderInfo) {
|
|
9366
|
+
throw new Error("Cannot persist clear recovery state before the embedding provider is initialized");
|
|
9367
|
+
}
|
|
9368
|
+
const compatibility = this.checkCompatibility();
|
|
9369
|
+
const compatibilityDecision = compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible";
|
|
9370
|
+
return {
|
|
9371
|
+
phase: "clearing",
|
|
9372
|
+
embeddingProvider: this.configuredProviderInfo.provider,
|
|
9373
|
+
embeddingModel: this.configuredProviderInfo.modelInfo.model,
|
|
9374
|
+
embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions,
|
|
9375
|
+
embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION,
|
|
9376
|
+
compatibilityDecision
|
|
9377
|
+
};
|
|
9378
|
+
}
|
|
9379
|
+
beginClearRecoveryState() {
|
|
9380
|
+
const recovery = this.getCurrentClearRecoveryState();
|
|
9381
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), recovery);
|
|
9382
|
+
return recovery;
|
|
9383
|
+
}
|
|
9384
|
+
finishClearRecoveryState() {
|
|
9385
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), null);
|
|
9386
|
+
}
|
|
9387
|
+
matchesCurrentClearRecoveryConfiguration(recovery) {
|
|
9388
|
+
const configuredProviderInfo = this.configuredProviderInfo;
|
|
9389
|
+
return configuredProviderInfo !== null && recovery.embeddingProvider === configuredProviderInfo.provider && recovery.embeddingModel === configuredProviderInfo.modelInfo.model && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION;
|
|
9390
|
+
}
|
|
9391
|
+
hasUnknownLegacyForceIndexClear(owner) {
|
|
9392
|
+
return owner.operation === "force-index" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1 && (0, import_fs10.existsSync)(path15.join(this.indexPath, "force-index-phase"));
|
|
9393
|
+
}
|
|
9166
9394
|
async recoverFromInterruptedIndexingUnlocked(owners) {
|
|
9167
9395
|
for (const owner of owners) {
|
|
9168
9396
|
this.logger.warn("Detected interrupted indexing session, recovering...", {
|
|
9169
9397
|
pid: owner.pid,
|
|
9170
9398
|
hostname: owner.hostname,
|
|
9171
9399
|
operation: owner.operation,
|
|
9172
|
-
startedAt: owner.startedAt
|
|
9400
|
+
startedAt: owner.startedAt,
|
|
9401
|
+
projectRoot: owner.projectRoot
|
|
9173
9402
|
});
|
|
9174
9403
|
}
|
|
9175
9404
|
if (this.config.scope === "global") {
|
|
9176
|
-
|
|
9177
|
-
|
|
9405
|
+
const clearScopes = [];
|
|
9406
|
+
for (const owner of owners) {
|
|
9407
|
+
if (this.hasUnknownLegacyForceIndexClear(owner)) {
|
|
9408
|
+
throw new Error(
|
|
9409
|
+
`Cannot automatically recover interrupted force-index ${owner.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
9410
|
+
);
|
|
9411
|
+
}
|
|
9412
|
+
if (owner.operation === "clear" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1) {
|
|
9413
|
+
throw new Error(
|
|
9414
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating recovery state is unknown. The recovery marker was retained for manual inspection.`
|
|
9415
|
+
);
|
|
9416
|
+
}
|
|
9417
|
+
if (owner.clearRecovery === void 0) continue;
|
|
9418
|
+
if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) {
|
|
9419
|
+
throw new Error(
|
|
9420
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating project scope is unknown. The recovery marker was retained for manual inspection.`
|
|
9421
|
+
);
|
|
9422
|
+
}
|
|
9423
|
+
if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) {
|
|
9424
|
+
throw new Error(
|
|
9425
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the current embedding configuration does not match the originating lease. The recovery marker was retained; retry from the originating project with matching settings.`
|
|
9426
|
+
);
|
|
9427
|
+
}
|
|
9428
|
+
clearScopes.push({
|
|
9429
|
+
projectRoot: owner.projectRoot,
|
|
9430
|
+
scopedRoots: owner.scopedRoots,
|
|
9431
|
+
compatibilityDecision: owner.clearRecovery.compatibilityDecision
|
|
9432
|
+
});
|
|
9433
|
+
}
|
|
9434
|
+
if (clearScopes.length > 0) {
|
|
9435
|
+
this.loadFileHashCache();
|
|
9436
|
+
}
|
|
9437
|
+
for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) {
|
|
9438
|
+
this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision);
|
|
9178
9439
|
}
|
|
9179
9440
|
await this.healthCheckUnlocked();
|
|
9441
|
+
this.logger.info(
|
|
9442
|
+
clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint"
|
|
9443
|
+
);
|
|
9444
|
+
return;
|
|
9180
9445
|
}
|
|
9181
|
-
this.logger.info("Recovery complete, next index will
|
|
9446
|
+
this.logger.info("Recovery complete, next index will resume from the last checkpoint");
|
|
9182
9447
|
}
|
|
9183
9448
|
*loadSerializedFailedBatches() {
|
|
9184
9449
|
let warned = false;
|
|
@@ -9216,14 +9481,99 @@ var Indexer = class _Indexer {
|
|
|
9216
9481
|
state.writer.write(record);
|
|
9217
9482
|
state.recordsWritten += record.chunks.length;
|
|
9218
9483
|
}
|
|
9219
|
-
finalizeFailedBatchWriteState(state) {
|
|
9484
|
+
finalizeFailedBatchWriteState(state, resolvedChunkIds = /* @__PURE__ */ new Set()) {
|
|
9220
9485
|
if (state.recordsWritten > 0) {
|
|
9221
|
-
|
|
9486
|
+
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
9487
|
+
const retained = [];
|
|
9488
|
+
const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath));
|
|
9489
|
+
for (let i = records.length - 1; i >= 0; i--) {
|
|
9490
|
+
const chunks = records[i].chunks.filter((rawChunk) => {
|
|
9491
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9492
|
+
if (chunkId !== null) {
|
|
9493
|
+
if (resolvedChunkIds.has(chunkId)) return false;
|
|
9494
|
+
if (seenChunkIds.has(chunkId)) return false;
|
|
9495
|
+
seenChunkIds.add(chunkId);
|
|
9496
|
+
}
|
|
9497
|
+
return true;
|
|
9498
|
+
});
|
|
9499
|
+
if (chunks.length > 0) {
|
|
9500
|
+
retained.unshift({ ...records[i], chunks });
|
|
9501
|
+
}
|
|
9502
|
+
}
|
|
9503
|
+
state.writer.cleanup();
|
|
9504
|
+
if (retained.length > 0) {
|
|
9505
|
+
writeFailedBatchRecords(this.failedBatchesPath, retained);
|
|
9506
|
+
} else {
|
|
9507
|
+
writeFailedBatchRecords(this.failedBatchesPath, []);
|
|
9508
|
+
this.clearFailedBatchState();
|
|
9509
|
+
}
|
|
9222
9510
|
return;
|
|
9223
9511
|
}
|
|
9224
|
-
state.writer.
|
|
9512
|
+
state.writer.commit();
|
|
9225
9513
|
this.clearFailedBatchState();
|
|
9226
9514
|
}
|
|
9515
|
+
getCheckpointIntervalChunks(totalChunks) {
|
|
9516
|
+
return Math.max(
|
|
9517
|
+
this.checkpointIntervalChunks ?? 2e3,
|
|
9518
|
+
Math.floor(totalChunks / 10)
|
|
9519
|
+
);
|
|
9520
|
+
}
|
|
9521
|
+
checkpointIndexRun(database, store, invertedIndex, failedProcessing, resolvedRetryChunkIds, currentFileHashes, committedFilePaths, scopedRoots, configuredProviderInfo) {
|
|
9522
|
+
if (!this.hasProjectForceReembedPending()) {
|
|
9523
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
9524
|
+
this.indexCompatibility = { compatible: true };
|
|
9525
|
+
}
|
|
9526
|
+
database.commitWriteTransaction();
|
|
9527
|
+
database.beginWriteTransaction();
|
|
9528
|
+
this.saveInvertedIndex(invertedIndex);
|
|
9529
|
+
store.save();
|
|
9530
|
+
if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0 || failedProcessing.discardedExistingRecords) {
|
|
9531
|
+
for (const metadata of failedProcessing.latestById.values()) {
|
|
9532
|
+
const alreadyMaterialized = metadata.chunks.some((rawChunk) => {
|
|
9533
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9534
|
+
return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId);
|
|
9535
|
+
});
|
|
9536
|
+
if (alreadyMaterialized) continue;
|
|
9537
|
+
this.writeFailedBatchRecord(failedProcessing.state, {
|
|
9538
|
+
chunks: metadata.chunks,
|
|
9539
|
+
attemptCount: metadata.attemptCount,
|
|
9540
|
+
error: metadata.error,
|
|
9541
|
+
lastAttempt: metadata.lastAttempt
|
|
9542
|
+
});
|
|
9543
|
+
for (const rawChunk of metadata.chunks) {
|
|
9544
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9545
|
+
if (chunkId !== null) {
|
|
9546
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
9547
|
+
}
|
|
9548
|
+
}
|
|
9549
|
+
}
|
|
9550
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
9551
|
+
failedProcessing.state = this.createFailedBatchWriteState();
|
|
9552
|
+
failedProcessing.discardedExistingRecords = false;
|
|
9553
|
+
for (const record of this.loadSerializedFailedBatches()) {
|
|
9554
|
+
for (const rawChunk of record.chunks) {
|
|
9555
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
9556
|
+
this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] });
|
|
9557
|
+
if (chunkId !== null) {
|
|
9558
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
9559
|
+
}
|
|
9560
|
+
}
|
|
9561
|
+
}
|
|
9562
|
+
}
|
|
9563
|
+
const partialHashes = /* @__PURE__ */ new Map();
|
|
9564
|
+
for (const filePath of committedFilePaths) {
|
|
9565
|
+
const hash = currentFileHashes.get(filePath);
|
|
9566
|
+
if (hash !== void 0) {
|
|
9567
|
+
partialHashes.set(filePath, hash);
|
|
9568
|
+
}
|
|
9569
|
+
}
|
|
9570
|
+
if (scopedRoots) {
|
|
9571
|
+
this.replaceScopedFileHashCache(partialHashes, scopedRoots);
|
|
9572
|
+
} else {
|
|
9573
|
+
this.fileHashCache = partialHashes;
|
|
9574
|
+
this.saveFileHashCache();
|
|
9575
|
+
}
|
|
9576
|
+
}
|
|
9227
9577
|
clearFailedBatchState() {
|
|
9228
9578
|
if ((0, import_fs10.existsSync)(this.failedBatchesPath)) {
|
|
9229
9579
|
try {
|
|
@@ -9250,6 +9600,7 @@ var Indexer = class _Indexer {
|
|
|
9250
9600
|
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
9251
9601
|
const state = this.createFailedBatchWriteState();
|
|
9252
9602
|
const latestById = /* @__PURE__ */ new Map();
|
|
9603
|
+
let discardedExistingRecords = false;
|
|
9253
9604
|
try {
|
|
9254
9605
|
for (const batch of this.loadSerializedFailedBatches()) {
|
|
9255
9606
|
for (const rawChunk of batch.chunks) {
|
|
@@ -9260,10 +9611,12 @@ var Indexer = class _Indexer {
|
|
|
9260
9611
|
continue;
|
|
9261
9612
|
}
|
|
9262
9613
|
if (!shouldProcess(filePath)) {
|
|
9614
|
+
discardedExistingRecords = true;
|
|
9263
9615
|
continue;
|
|
9264
9616
|
}
|
|
9265
9617
|
const chunkId = getPendingChunkId(rawChunk);
|
|
9266
9618
|
if (!chunkId) {
|
|
9619
|
+
discardedExistingRecords = true;
|
|
9267
9620
|
continue;
|
|
9268
9621
|
}
|
|
9269
9622
|
const existing = latestById.get(chunkId);
|
|
@@ -9271,12 +9624,18 @@ var Indexer = class _Indexer {
|
|
|
9271
9624
|
latestById.set(chunkId, {
|
|
9272
9625
|
attemptCount: batch.attemptCount,
|
|
9273
9626
|
error: batch.error,
|
|
9274
|
-
lastAttempt: batch.lastAttempt
|
|
9627
|
+
lastAttempt: batch.lastAttempt,
|
|
9628
|
+
chunks: [rawChunk]
|
|
9275
9629
|
});
|
|
9276
9630
|
}
|
|
9277
9631
|
}
|
|
9278
9632
|
}
|
|
9279
|
-
return {
|
|
9633
|
+
return {
|
|
9634
|
+
state,
|
|
9635
|
+
latestById,
|
|
9636
|
+
materializedRetryIds: /* @__PURE__ */ new Set(),
|
|
9637
|
+
discardedExistingRecords
|
|
9638
|
+
};
|
|
9280
9639
|
} catch (error) {
|
|
9281
9640
|
state.writer.cleanup();
|
|
9282
9641
|
throw error;
|
|
@@ -9312,10 +9671,34 @@ var Indexer = class _Indexer {
|
|
|
9312
9671
|
}
|
|
9313
9672
|
}
|
|
9314
9673
|
}
|
|
9674
|
+
restoreMissingChunkRows(database, chunks) {
|
|
9675
|
+
const missing = [];
|
|
9676
|
+
for (const chunk of chunks) {
|
|
9677
|
+
if (database.getChunk(chunk.id)) {
|
|
9678
|
+
continue;
|
|
9679
|
+
}
|
|
9680
|
+
missing.push({
|
|
9681
|
+
chunkId: chunk.id,
|
|
9682
|
+
contentHash: chunk.contentHash,
|
|
9683
|
+
filePath: chunk.metadata.filePath,
|
|
9684
|
+
startLine: chunk.metadata.startLine,
|
|
9685
|
+
endLine: chunk.metadata.endLine,
|
|
9686
|
+
nodeType: chunk.metadata.chunkType,
|
|
9687
|
+
name: chunk.metadata.name,
|
|
9688
|
+
language: chunk.metadata.language,
|
|
9689
|
+
blameSha: chunk.metadata.blameSha,
|
|
9690
|
+
blameAuthor: chunk.metadata.blameAuthor,
|
|
9691
|
+
blameAuthorEmail: chunk.metadata.blameAuthorEmail,
|
|
9692
|
+
blameCommittedAt: chunk.metadata.blameCommittedAt,
|
|
9693
|
+
blameSummary: chunk.metadata.blameSummary
|
|
9694
|
+
});
|
|
9695
|
+
}
|
|
9696
|
+
if (missing.length > 0) {
|
|
9697
|
+
database.upsertChunksBatch(missing);
|
|
9698
|
+
}
|
|
9699
|
+
}
|
|
9315
9700
|
getProviderRateLimits(provider) {
|
|
9316
9701
|
switch (provider) {
|
|
9317
|
-
case "github-copilot":
|
|
9318
|
-
return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
|
|
9319
9702
|
case "openai":
|
|
9320
9703
|
return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
9321
9704
|
case "google":
|
|
@@ -9384,10 +9767,11 @@ var Indexer = class _Indexer {
|
|
|
9384
9767
|
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
9385
9768
|
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
9386
9769
|
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
9387
|
-
const
|
|
9388
|
-
|
|
9389
|
-
|
|
9390
|
-
|
|
9770
|
+
const batchOptions = getDynamicBatchOptions(options.configuredProviderInfo, this.config.embedding?.batch);
|
|
9771
|
+
if (options.forceSingleItemBatches && options.configuredProviderInfo.provider === "ollama") {
|
|
9772
|
+
batchOptions.maxBatchItems = 1;
|
|
9773
|
+
}
|
|
9774
|
+
const requestBatches = createPendingEmbeddingRequestBatches(chunksNeedingEmbedding, batchOptions);
|
|
9391
9775
|
let fatalError;
|
|
9392
9776
|
for (const requestBatch of requestBatches) {
|
|
9393
9777
|
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
@@ -9950,7 +10334,7 @@ var Indexer = class _Indexer {
|
|
|
9950
10334
|
}
|
|
9951
10335
|
if (!this.configuredProviderInfo) {
|
|
9952
10336
|
throw new Error(
|
|
9953
|
-
"No embedding provider available. Configure
|
|
10337
|
+
"No embedding provider available. Configure OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
9954
10338
|
);
|
|
9955
10339
|
}
|
|
9956
10340
|
this.logger.info("Initializing indexer", {
|
|
@@ -9981,7 +10365,20 @@ var Indexer = class _Indexer {
|
|
|
9981
10365
|
]);
|
|
9982
10366
|
}
|
|
9983
10367
|
if (recoveredOwners.length > 0 && this.config.scope === "project") {
|
|
9984
|
-
|
|
10368
|
+
const unknownLegacyForceIndex = recoveredOwners.find(
|
|
10369
|
+
(owner) => this.hasUnknownLegacyForceIndexClear(owner)
|
|
10370
|
+
);
|
|
10371
|
+
if (unknownLegacyForceIndex) {
|
|
10372
|
+
throw new Error(
|
|
10373
|
+
`Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
10374
|
+
);
|
|
10375
|
+
}
|
|
10376
|
+
const shouldReset = recoveredOwners.some(
|
|
10377
|
+
(owner) => owner.clearRecovery !== void 0 || owner.operation === "clear" && owner.recoveryProtocolVersion !== 1
|
|
10378
|
+
);
|
|
10379
|
+
if (shouldReset) {
|
|
10380
|
+
await this.resetLocalIndexArtifacts();
|
|
10381
|
+
}
|
|
9985
10382
|
}
|
|
9986
10383
|
this.store = new VectorStore(storePath, dimensions);
|
|
9987
10384
|
if ((0, import_fs10.existsSync)(storePath) || (0, import_fs10.existsSync)(vectorMetadataPath)) {
|
|
@@ -10497,6 +10894,70 @@ var Indexer = class _Indexer {
|
|
|
10497
10894
|
);
|
|
10498
10895
|
return createCostEstimate(files, configuredProviderInfo);
|
|
10499
10896
|
}
|
|
10897
|
+
// Dry-run counterpart to index()/forceIndex(): parse the real file set and sum
|
|
10898
|
+
// estimateTokens over the embedding text of every indexable chunk, without
|
|
10899
|
+
// calling the embedding provider or writing to the index. Read-only and
|
|
10900
|
+
// lock-free (mirrors estimateCost). The token sum is the exact value "Tokens
|
|
10901
|
+
// used" climbs to for a force index (cache bypassed); for an incremental it is
|
|
10902
|
+
// an upper bound because cached chunks are counted here but not re-embedded.
|
|
10903
|
+
// Used by index_codebase(dryRun:true) to give a stable, monotonic progress
|
|
10904
|
+
// denominator that matches the live "Tokens used" basis.
|
|
10905
|
+
async dryRunCost() {
|
|
10906
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
10907
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
10908
|
+
const includePatterns = [...this.config.include, ...this.config.additionalInclude];
|
|
10909
|
+
const { files } = await collectFiles(
|
|
10910
|
+
this.materializedProjectRoot,
|
|
10911
|
+
includePatterns,
|
|
10912
|
+
this.config.exclude,
|
|
10913
|
+
this.config.indexing.maxFileSize,
|
|
10914
|
+
this.getMaterializedKnowledgeBases(),
|
|
10915
|
+
{ maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }
|
|
10916
|
+
);
|
|
10917
|
+
let filesCount = 0;
|
|
10918
|
+
let chunksCount = 0;
|
|
10919
|
+
let tokensToEmbed = 0;
|
|
10920
|
+
for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) {
|
|
10921
|
+
const loadedFiles = await Promise.all(batch.map(async (f) => {
|
|
10922
|
+
try {
|
|
10923
|
+
return {
|
|
10924
|
+
path: this.toStoredFilePath(f.path),
|
|
10925
|
+
content: await import_fs10.promises.readFile(f.path, "utf-8")
|
|
10926
|
+
};
|
|
10927
|
+
} catch {
|
|
10928
|
+
return null;
|
|
10929
|
+
}
|
|
10930
|
+
}));
|
|
10931
|
+
const readable = loadedFiles.filter(
|
|
10932
|
+
(f) => f !== null
|
|
10933
|
+
);
|
|
10934
|
+
filesCount += readable.length;
|
|
10935
|
+
const contentByPath = new Map(readable.map((f) => [f.path, f.content]));
|
|
10936
|
+
const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk);
|
|
10937
|
+
for (const parsed of parsedFiles) {
|
|
10938
|
+
let chunksToProcess = parsed.chunks;
|
|
10939
|
+
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
10940
|
+
const content = contentByPath.get(parsed.path);
|
|
10941
|
+
if (content !== void 0) {
|
|
10942
|
+
chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk);
|
|
10943
|
+
}
|
|
10944
|
+
}
|
|
10945
|
+
chunksToProcess = selectIndexableChunks(
|
|
10946
|
+
chunksToProcess,
|
|
10947
|
+
this.config.indexing.maxChunksPerFile,
|
|
10948
|
+
this.config.indexing.semanticOnly
|
|
10949
|
+
);
|
|
10950
|
+
for (const chunk of chunksToProcess) {
|
|
10951
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens);
|
|
10952
|
+
chunksCount += 1;
|
|
10953
|
+
for (const text of texts) {
|
|
10954
|
+
tokensToEmbed += estimateTokens2(text);
|
|
10955
|
+
}
|
|
10956
|
+
}
|
|
10957
|
+
}
|
|
10958
|
+
}
|
|
10959
|
+
return { filesCount, chunksCount, tokensToEmbed };
|
|
10960
|
+
}
|
|
10500
10961
|
async index(onProgress) {
|
|
10501
10962
|
return this.withIndexMutationLease("index", async (recoveredOwners) => {
|
|
10502
10963
|
return this.indexUnlocked(onProgress, recoveredOwners);
|
|
@@ -10617,7 +11078,17 @@ var Indexer = class _Indexer {
|
|
|
10617
11078
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
10618
11079
|
for (const file of files) {
|
|
10619
11080
|
const storedPath = this.toStoredFilePath(file.path);
|
|
10620
|
-
|
|
11081
|
+
let currentHash;
|
|
11082
|
+
try {
|
|
11083
|
+
currentHash = hashFile(file.path);
|
|
11084
|
+
} catch (error) {
|
|
11085
|
+
stats.skippedFiles.push({ path: this.toCanonicalFilePath(file.path), reason: "unreadable" });
|
|
11086
|
+
this.logger.warn("Skipped unreadable file during indexing", {
|
|
11087
|
+
path: file.path,
|
|
11088
|
+
error: getErrorMessage3(error)
|
|
11089
|
+
});
|
|
11090
|
+
continue;
|
|
11091
|
+
}
|
|
10621
11092
|
currentFileHashes.set(storedPath, currentHash);
|
|
10622
11093
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
10623
11094
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
@@ -10625,7 +11096,8 @@ var Indexer = class _Indexer {
|
|
|
10625
11096
|
);
|
|
10626
11097
|
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path15.extname(storedPath).toLowerCase() === ".swift";
|
|
10627
11098
|
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path15.extname(storedPath).toLowerCase() === ".metal";
|
|
10628
|
-
|
|
11099
|
+
const inMigrationScope = forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots);
|
|
11100
|
+
if (cachedHashMatches && !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
10629
11101
|
unchangedFilePaths.add(storedPath);
|
|
10630
11102
|
this.logger.recordCacheHit();
|
|
10631
11103
|
} else {
|
|
@@ -10751,6 +11223,9 @@ var Indexer = class _Indexer {
|
|
|
10751
11223
|
}
|
|
10752
11224
|
}
|
|
10753
11225
|
let processedChangedFiles = 0;
|
|
11226
|
+
let lastCheckpointChunks = 0;
|
|
11227
|
+
const committedFilePaths = new Set(unchangedFilePaths);
|
|
11228
|
+
const resolvedRetryChunkIds = /* @__PURE__ */ new Set();
|
|
10754
11229
|
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
10755
11230
|
changedFileDescriptors,
|
|
10756
11231
|
(descriptor) => descriptor.sourceBytes,
|
|
@@ -10764,7 +11239,7 @@ var Indexer = class _Indexer {
|
|
|
10764
11239
|
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
10765
11240
|
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
10766
11241
|
const parseStartTime = import_perf_hooks.performance.now();
|
|
10767
|
-
const parsedFiles = parseFiles(loadedFiles);
|
|
11242
|
+
const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk);
|
|
10768
11243
|
const parseMs = import_perf_hooks.performance.now() - parseStartTime;
|
|
10769
11244
|
this.logger.recordFilesParsed(parsedFiles.length);
|
|
10770
11245
|
this.logger.recordParseDuration(parseMs);
|
|
@@ -10787,7 +11262,7 @@ var Indexer = class _Indexer {
|
|
|
10787
11262
|
}
|
|
10788
11263
|
let chunksToProcess = parsed.chunks;
|
|
10789
11264
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
10790
|
-
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
11265
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk);
|
|
10791
11266
|
}
|
|
10792
11267
|
chunksToProcess = selectIndexableChunks(
|
|
10793
11268
|
chunksToProcess,
|
|
@@ -10921,6 +11396,10 @@ var Indexer = class _Indexer {
|
|
|
10921
11396
|
}
|
|
10922
11397
|
if (symbolBatch.length > 0) {
|
|
10923
11398
|
database.upsertSymbolsBatch(symbolBatch);
|
|
11399
|
+
database.addSymbolsToBranchBatch(
|
|
11400
|
+
this.getBranchCatalogKey(),
|
|
11401
|
+
symbolBatch.map((symbol) => symbol.id)
|
|
11402
|
+
);
|
|
10924
11403
|
}
|
|
10925
11404
|
if (edgeBatch.length > 0) {
|
|
10926
11405
|
database.upsertCallEdgesBatch(edgeBatch);
|
|
@@ -10956,6 +11435,12 @@ var Indexer = class _Indexer {
|
|
|
10956
11435
|
forceReembed: forceScopedReembed,
|
|
10957
11436
|
reuseCachedEmbeddings: true,
|
|
10958
11437
|
incrementRepeatedFailures: true,
|
|
11438
|
+
onSucceeded: (succeededChunks) => {
|
|
11439
|
+
database.addChunksToBranchBatch(
|
|
11440
|
+
this.getBranchCatalogKey(),
|
|
11441
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
11442
|
+
);
|
|
11443
|
+
},
|
|
10959
11444
|
onProgress: (batchProgress) => onProgress?.({
|
|
10960
11445
|
phase: "embedding",
|
|
10961
11446
|
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
@@ -10974,6 +11459,27 @@ var Indexer = class _Indexer {
|
|
|
10974
11459
|
}
|
|
10975
11460
|
}
|
|
10976
11461
|
}
|
|
11462
|
+
for (const descriptor of descriptorBatch) {
|
|
11463
|
+
const existingFileChunks = existingChunksByFile.get(descriptor.storedPath);
|
|
11464
|
+
if (!existingFileChunks || existingFileChunks.size === 0) {
|
|
11465
|
+
committedFilePaths.add(descriptor.storedPath);
|
|
11466
|
+
}
|
|
11467
|
+
}
|
|
11468
|
+
const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks);
|
|
11469
|
+
if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) {
|
|
11470
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
11471
|
+
this.checkpointIndexRun(
|
|
11472
|
+
database,
|
|
11473
|
+
store,
|
|
11474
|
+
invertedIndex,
|
|
11475
|
+
failedProcessing,
|
|
11476
|
+
resolvedRetryChunkIds,
|
|
11477
|
+
currentFileHashes,
|
|
11478
|
+
committedFilePaths,
|
|
11479
|
+
scopedRoots,
|
|
11480
|
+
configuredProviderInfo
|
|
11481
|
+
);
|
|
11482
|
+
}
|
|
10977
11483
|
}
|
|
10978
11484
|
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
10979
11485
|
failedProcessing.latestById,
|
|
@@ -10994,6 +11500,7 @@ var Indexer = class _Indexer {
|
|
|
10994
11500
|
retryableChunksWithExistingData.add(chunk.id);
|
|
10995
11501
|
}
|
|
10996
11502
|
}
|
|
11503
|
+
this.restoreMissingChunkRows(database, pendingChunks);
|
|
10997
11504
|
stats.totalChunks += pendingChunks.length;
|
|
10998
11505
|
onProgress?.({
|
|
10999
11506
|
phase: "embedding",
|
|
@@ -11016,6 +11523,17 @@ var Indexer = class _Indexer {
|
|
|
11016
11523
|
forceReembed: forceScopedReembed,
|
|
11017
11524
|
reuseCachedEmbeddings: true,
|
|
11018
11525
|
incrementRepeatedFailures: true,
|
|
11526
|
+
forceSingleItemBatches: true,
|
|
11527
|
+
onSucceeded: (succeededChunks) => {
|
|
11528
|
+
database.addChunksToBranchBatch(
|
|
11529
|
+
this.getBranchCatalogKey(),
|
|
11530
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
11531
|
+
);
|
|
11532
|
+
for (const chunk of succeededChunks) {
|
|
11533
|
+
failedProcessing.latestById.delete(chunk.id);
|
|
11534
|
+
resolvedRetryChunkIds.add(chunk.id);
|
|
11535
|
+
}
|
|
11536
|
+
},
|
|
11019
11537
|
onProgress: (batchProgress) => onProgress?.({
|
|
11020
11538
|
phase: "embedding",
|
|
11021
11539
|
filesProcessed: files.length,
|
|
@@ -11033,6 +11551,20 @@ var Indexer = class _Indexer {
|
|
|
11033
11551
|
failedForcedChunkIds.add(chunkId);
|
|
11034
11552
|
}
|
|
11035
11553
|
}
|
|
11554
|
+
if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) {
|
|
11555
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
11556
|
+
this.checkpointIndexRun(
|
|
11557
|
+
database,
|
|
11558
|
+
store,
|
|
11559
|
+
invertedIndex,
|
|
11560
|
+
failedProcessing,
|
|
11561
|
+
resolvedRetryChunkIds,
|
|
11562
|
+
currentFileHashes,
|
|
11563
|
+
committedFilePaths,
|
|
11564
|
+
scopedRoots,
|
|
11565
|
+
configuredProviderInfo
|
|
11566
|
+
);
|
|
11567
|
+
}
|
|
11036
11568
|
}
|
|
11037
11569
|
const removedChunkIds = [];
|
|
11038
11570
|
for (const [chunkId] of existingChunks) {
|
|
@@ -11069,13 +11601,6 @@ var Indexer = class _Indexer {
|
|
|
11069
11601
|
if (removedStoredChunks) {
|
|
11070
11602
|
this.saveInvertedIndex(invertedIndex);
|
|
11071
11603
|
}
|
|
11072
|
-
if (scopedRoots) {
|
|
11073
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11074
|
-
} else {
|
|
11075
|
-
this.fileHashCache = currentFileHashes;
|
|
11076
|
-
this.saveFileHashCache();
|
|
11077
|
-
}
|
|
11078
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11079
11604
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11080
11605
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11081
11606
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11084,6 +11609,13 @@ var Indexer = class _Indexer {
|
|
|
11084
11609
|
this.indexCompatibility = { compatible: true };
|
|
11085
11610
|
database.commitWriteTransaction();
|
|
11086
11611
|
writeTransactionActive = false;
|
|
11612
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11613
|
+
if (scopedRoots) {
|
|
11614
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11615
|
+
} else {
|
|
11616
|
+
this.fileHashCache = currentFileHashes;
|
|
11617
|
+
this.saveFileHashCache();
|
|
11618
|
+
}
|
|
11087
11619
|
stats.durationMs = Date.now() - startTime;
|
|
11088
11620
|
onProgress?.({
|
|
11089
11621
|
phase: "complete",
|
|
@@ -11107,13 +11639,6 @@ var Indexer = class _Indexer {
|
|
|
11107
11639
|
);
|
|
11108
11640
|
store.save();
|
|
11109
11641
|
this.saveInvertedIndex(invertedIndex);
|
|
11110
|
-
if (scopedRoots) {
|
|
11111
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11112
|
-
} else {
|
|
11113
|
-
this.fileHashCache = currentFileHashes;
|
|
11114
|
-
this.saveFileHashCache();
|
|
11115
|
-
}
|
|
11116
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11117
11642
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11118
11643
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11119
11644
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11122,6 +11647,13 @@ var Indexer = class _Indexer {
|
|
|
11122
11647
|
this.indexCompatibility = { compatible: true };
|
|
11123
11648
|
database.commitWriteTransaction();
|
|
11124
11649
|
writeTransactionActive = false;
|
|
11650
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11651
|
+
if (scopedRoots) {
|
|
11652
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11653
|
+
} else {
|
|
11654
|
+
this.fileHashCache = currentFileHashes;
|
|
11655
|
+
this.saveFileHashCache();
|
|
11656
|
+
}
|
|
11125
11657
|
stats.durationMs = Date.now() - startTime;
|
|
11126
11658
|
onProgress?.({
|
|
11127
11659
|
phase: "complete",
|
|
@@ -11156,15 +11688,15 @@ var Indexer = class _Indexer {
|
|
|
11156
11688
|
);
|
|
11157
11689
|
store.save();
|
|
11158
11690
|
this.saveInvertedIndex(invertedIndex);
|
|
11691
|
+
database.commitWriteTransaction();
|
|
11692
|
+
writeTransactionActive = false;
|
|
11693
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11159
11694
|
if (scopedRoots) {
|
|
11160
11695
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11161
11696
|
} else {
|
|
11162
11697
|
this.fileHashCache = currentFileHashes;
|
|
11163
11698
|
this.saveFileHashCache();
|
|
11164
11699
|
}
|
|
11165
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
11166
|
-
database.commitWriteTransaction();
|
|
11167
|
-
writeTransactionActive = false;
|
|
11168
11700
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
11169
11701
|
const gcReset = await this.maybeRunOrphanGc();
|
|
11170
11702
|
if (gcReset) {
|
|
@@ -11188,6 +11720,9 @@ var Indexer = class _Indexer {
|
|
|
11188
11720
|
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
11189
11721
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11190
11722
|
}
|
|
11723
|
+
if (forceScopedReembed) {
|
|
11724
|
+
database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true");
|
|
11725
|
+
}
|
|
11191
11726
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11192
11727
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11193
11728
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -11298,26 +11833,41 @@ var Indexer = class _Indexer {
|
|
|
11298
11833
|
shouldPrefilterByBranch: branchChunkIds !== null && (this.config.scope === "global" || hasInitializedBranchCatalog)
|
|
11299
11834
|
};
|
|
11300
11835
|
}
|
|
11301
|
-
|
|
11836
|
+
searchCandidatesWithAllowedIds(initialLimit, totalCount, allowedChunkIds, shouldPrefilter, search, getChunkId) {
|
|
11302
11837
|
const normalizedLimit = Math.max(0, Math.floor(initialLimit));
|
|
11303
11838
|
if (normalizedLimit === 0) return [];
|
|
11304
|
-
if (!
|
|
11839
|
+
if (!shouldPrefilter || !allowedChunkIds) {
|
|
11305
11840
|
return search(normalizedLimit);
|
|
11306
11841
|
}
|
|
11307
|
-
const targetCount = Math.min(normalizedLimit,
|
|
11842
|
+
const targetCount = Math.min(normalizedLimit, allowedChunkIds.size);
|
|
11308
11843
|
if (targetCount === 0 || totalCount === 0) return [];
|
|
11309
11844
|
let requestedLimit = Math.min(normalizedLimit, totalCount);
|
|
11310
11845
|
while (true) {
|
|
11311
11846
|
const results = search(requestedLimit);
|
|
11312
|
-
const
|
|
11313
|
-
if (
|
|
11314
|
-
return
|
|
11847
|
+
const allowedResults = results.filter((candidate) => allowedChunkIds.has(getChunkId(candidate)));
|
|
11848
|
+
if (allowedResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
|
|
11849
|
+
return allowedResults;
|
|
11315
11850
|
}
|
|
11316
11851
|
const nextLimit = Math.min(totalCount, Math.max(requestedLimit + 1, requestedLimit * 2));
|
|
11317
|
-
if (nextLimit === requestedLimit) return
|
|
11852
|
+
if (nextLimit === requestedLimit) return allowedResults;
|
|
11318
11853
|
requestedLimit = nextLimit;
|
|
11319
11854
|
}
|
|
11320
11855
|
}
|
|
11856
|
+
getTemporalChunkIds(database, options) {
|
|
11857
|
+
if (!options?.blameSince && !options?.blameUntil) return null;
|
|
11858
|
+
const since = options.blameSince ? parseBlameTimestamp(options.blameSince, false) : void 0;
|
|
11859
|
+
const until = options.blameUntil ? parseBlameTimestamp(options.blameUntil, true) : void 0;
|
|
11860
|
+
if (since === null || until === null) {
|
|
11861
|
+
return /* @__PURE__ */ new Set();
|
|
11862
|
+
}
|
|
11863
|
+
return new Set(database.getChunkIdsByBlameDate(since, until));
|
|
11864
|
+
}
|
|
11865
|
+
intersectChunkIdSets(first, second) {
|
|
11866
|
+
if (first === null) return second;
|
|
11867
|
+
if (second === null) return first;
|
|
11868
|
+
const [smaller, larger] = first.size <= second.size ? [first, second] : [second, first];
|
|
11869
|
+
return new Set(Array.from(smaller).filter((chunkId) => larger.has(chunkId)));
|
|
11870
|
+
}
|
|
11321
11871
|
buildCandidateSnapshot(candidate) {
|
|
11322
11872
|
return {
|
|
11323
11873
|
id: candidate.id,
|
|
@@ -11332,13 +11882,16 @@ var Indexer = class _Indexer {
|
|
|
11332
11882
|
buildCandidateSnapshotList(candidates) {
|
|
11333
11883
|
return candidates.map((candidate) => this.buildCandidateSnapshot(candidate));
|
|
11334
11884
|
}
|
|
11335
|
-
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch) {
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11885
|
+
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch, temporalChunkIds) {
|
|
11886
|
+
const availableCount = temporalChunkIds?.size ?? store.count();
|
|
11887
|
+
if (availableCount === 0) return [];
|
|
11888
|
+
const allowedIds = temporalChunkIds === null ? void 0 : Array.from(temporalChunkIds);
|
|
11889
|
+
return this.searchCandidatesWithAllowedIds(
|
|
11890
|
+
Math.min(initialLimit, availableCount),
|
|
11891
|
+
availableCount,
|
|
11339
11892
|
branchChunkIds,
|
|
11340
11893
|
shouldPrefilterByBranch,
|
|
11341
|
-
(requestedLimit) => store.search(embedding, requestedLimit),
|
|
11894
|
+
(requestedLimit) => store.search(embedding, requestedLimit, allowedIds),
|
|
11342
11895
|
(candidate) => candidate.id
|
|
11343
11896
|
);
|
|
11344
11897
|
}
|
|
@@ -11363,8 +11916,9 @@ var Indexer = class _Indexer {
|
|
|
11363
11916
|
const rerankTopN = this.config.search.rerankTopN;
|
|
11364
11917
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
11365
11918
|
const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
|
|
11919
|
+
const prioritizeSourcePaths = sourceIntent || options?.prioritizeSourcePaths === true;
|
|
11366
11920
|
const identifierHints = extractIdentifierHints(query);
|
|
11367
|
-
const candidateLimit = maxResults * (
|
|
11921
|
+
const candidateLimit = maxResults * (prioritizeSourcePaths ? 12 : 4);
|
|
11368
11922
|
this.logger.search("debug", "Starting search", {
|
|
11369
11923
|
query,
|
|
11370
11924
|
maxResults,
|
|
@@ -11395,6 +11949,7 @@ var Indexer = class _Indexer {
|
|
|
11395
11949
|
branchChunkIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchChunkIds(branchKey)));
|
|
11396
11950
|
branchSymbolIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchSymbolIds(branchKey)));
|
|
11397
11951
|
}
|
|
11952
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
11398
11953
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
11399
11954
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
11400
11955
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
@@ -11403,7 +11958,8 @@ var Indexer = class _Indexer {
|
|
|
11403
11958
|
embedding,
|
|
11404
11959
|
candidateLimit,
|
|
11405
11960
|
branchChunkIds,
|
|
11406
|
-
shouldPrefilterByBranch
|
|
11961
|
+
shouldPrefilterByBranch,
|
|
11962
|
+
temporalChunkIds
|
|
11407
11963
|
) : [];
|
|
11408
11964
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
11409
11965
|
const keywordStartTime = import_perf_hooks.performance.now();
|
|
@@ -11413,7 +11969,8 @@ var Indexer = class _Indexer {
|
|
|
11413
11969
|
store,
|
|
11414
11970
|
invertedIndex,
|
|
11415
11971
|
branchChunkIds,
|
|
11416
|
-
shouldPrefilterByBranch
|
|
11972
|
+
shouldPrefilterByBranch,
|
|
11973
|
+
temporalChunkIds
|
|
11417
11974
|
);
|
|
11418
11975
|
const keywordMs = import_perf_hooks.performance.now() - keywordStartTime;
|
|
11419
11976
|
const scopedSemanticCandidates = semanticCandidates.filter(
|
|
@@ -11435,7 +11992,7 @@ var Indexer = class _Indexer {
|
|
|
11435
11992
|
rerankTopN,
|
|
11436
11993
|
limit: maxResults,
|
|
11437
11994
|
hybridWeight: rankingHybridWeight,
|
|
11438
|
-
prioritizeSourcePaths
|
|
11995
|
+
prioritizeSourcePaths
|
|
11439
11996
|
});
|
|
11440
11997
|
const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
|
|
11441
11998
|
definitionIntent: options?.definitionIntent === true,
|
|
@@ -11471,10 +12028,11 @@ var Indexer = class _Indexer {
|
|
|
11471
12028
|
branchSymbolIds,
|
|
11472
12029
|
maxResults,
|
|
11473
12030
|
union,
|
|
11474
|
-
sourceIntent
|
|
12031
|
+
sourceIntent,
|
|
12032
|
+
options?.definitionIntent === true && ((options.directory?.trim().length ?? 0) > 0 || (options.fileType?.trim().length ?? 0) > 0)
|
|
11475
12033
|
);
|
|
11476
12034
|
const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
|
|
11477
|
-
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
12035
|
+
const primaryLane = options?.definitionIntent === true ? mergeTieredResults(symbolLane, prePrimaryLane, maxResults * 4) : mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
11478
12036
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
11479
12037
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
11480
12038
|
const baseFiltered = tiered.filter(
|
|
@@ -11569,14 +12127,18 @@ var Indexer = class _Indexer {
|
|
|
11569
12127
|
})
|
|
11570
12128
|
);
|
|
11571
12129
|
}
|
|
11572
|
-
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false) {
|
|
12130
|
+
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false, temporalChunkIds = null) {
|
|
11573
12131
|
const normalizedLimit = Math.max(0, Math.floor(limit));
|
|
11574
12132
|
if (normalizedLimit === 0) return [];
|
|
11575
|
-
const
|
|
12133
|
+
const allowedChunkIds = this.intersectChunkIdSets(
|
|
12134
|
+
shouldPrefilterByBranch ? branchChunkIds : null,
|
|
12135
|
+
temporalChunkIds
|
|
12136
|
+
);
|
|
12137
|
+
const scoreEntries = this.searchCandidatesWithAllowedIds(
|
|
11576
12138
|
normalizedLimit,
|
|
11577
12139
|
invertedIndex.getDocumentCount(),
|
|
11578
|
-
|
|
11579
|
-
|
|
12140
|
+
allowedChunkIds,
|
|
12141
|
+
allowedChunkIds !== null,
|
|
11580
12142
|
(requestedLimit) => Array.from(invertedIndex.search(query, requestedLimit)),
|
|
11581
12143
|
([chunkId]) => chunkId
|
|
11582
12144
|
);
|
|
@@ -11661,7 +12223,17 @@ var Indexer = class _Indexer {
|
|
|
11661
12223
|
);
|
|
11662
12224
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
11663
12225
|
for (const file of files) {
|
|
11664
|
-
|
|
12226
|
+
let hash;
|
|
12227
|
+
try {
|
|
12228
|
+
hash = hashFile(file.path);
|
|
12229
|
+
} catch (error) {
|
|
12230
|
+
this.logger.warn("Skipped unreadable file during freshness check", {
|
|
12231
|
+
path: file.path,
|
|
12232
|
+
error: getErrorMessage3(error)
|
|
12233
|
+
});
|
|
12234
|
+
return { readable: false, current: false, reason: "unreadable" };
|
|
12235
|
+
}
|
|
12236
|
+
currentFileHashes.set(this.toStoredFilePath(file.path), hash);
|
|
11665
12237
|
}
|
|
11666
12238
|
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
11667
12239
|
const cachedFileHashes = scopedRoots ? new Map(Array.from(this.fileHashCache).filter(([filePath]) => this.isFileInCurrentScope(filePath, scopedRoots))) : this.fileHashCache;
|
|
@@ -11687,69 +12259,87 @@ var Indexer = class _Indexer {
|
|
|
11687
12259
|
async forceIndex(onProgress) {
|
|
11688
12260
|
return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
|
|
11689
12261
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
11690
|
-
|
|
12262
|
+
const recovery = this.beginClearRecoveryState();
|
|
12263
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
12264
|
+
this.finishClearRecoveryState();
|
|
11691
12265
|
return this.indexUnlocked(onProgress, [], true);
|
|
11692
12266
|
});
|
|
11693
12267
|
}
|
|
11694
12268
|
async clearIndex() {
|
|
11695
12269
|
await this.withIndexMutationLease("clear", async (recoveredOwners) => {
|
|
11696
12270
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
11697
|
-
|
|
12271
|
+
const recovery = this.beginClearRecoveryState();
|
|
12272
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
11698
12273
|
});
|
|
11699
12274
|
}
|
|
11700
|
-
|
|
12275
|
+
clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot) {
|
|
11701
12276
|
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
|
|
11712
|
-
|
|
11713
|
-
|
|
11714
|
-
|
|
11715
|
-
|
|
11716
|
-
|
|
12277
|
+
const clearedBranchKeys = database.getAllBranches();
|
|
12278
|
+
store.clear();
|
|
12279
|
+
store.save();
|
|
12280
|
+
invertedIndex.clear();
|
|
12281
|
+
this.saveInvertedIndex(invertedIndex);
|
|
12282
|
+
this.fileHashCache.clear();
|
|
12283
|
+
this.saveFileHashCache();
|
|
12284
|
+
database.clearAllIndexedData();
|
|
12285
|
+
this.deleteBranchCommitMetadata(database, clearedBranchKeys);
|
|
12286
|
+
this.clearFailedBatchState();
|
|
12287
|
+
database.deleteMetadata("index.version");
|
|
12288
|
+
database.deleteMetadata("index.pathStorageVersion");
|
|
12289
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
12290
|
+
database.deleteMetadata("index.embeddingModel");
|
|
12291
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
12292
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
12293
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
12294
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
12295
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash));
|
|
12296
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash));
|
|
12297
|
+
database.deleteMetadata("index.createdAt");
|
|
12298
|
+
database.deleteMetadata("index.updatedAt");
|
|
12299
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
12300
|
+
}
|
|
12301
|
+
clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots(), recoveryDecision) {
|
|
12302
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
12303
|
+
store.load();
|
|
12304
|
+
invertedIndex.load();
|
|
12305
|
+
this.loadFileHashCache();
|
|
12306
|
+
const compatibility = this.checkCompatibility();
|
|
12307
|
+
const compatibilityDecision = recoveryDecision ?? (compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible");
|
|
12308
|
+
const allMetadata = store.getAllMetadata();
|
|
12309
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
12310
|
+
if (compatibilityDecision !== "compatible" && hasForeignData) {
|
|
12311
|
+
if (compatibilityDecision === "embedding-strategy-mismatch") {
|
|
12312
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
12313
|
+
this.clearScopedFileHashCache(roots);
|
|
12314
|
+
this.clearScopedFailedBatches(roots);
|
|
12315
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
12316
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true");
|
|
12317
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
12318
|
+
database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash));
|
|
12319
|
+
if (projectRoot === this.projectRoot) {
|
|
11717
12320
|
this.indexCompatibility = { compatible: true };
|
|
11718
|
-
return;
|
|
11719
12321
|
}
|
|
11720
|
-
throw new Error(
|
|
11721
|
-
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
11722
|
-
);
|
|
11723
|
-
}
|
|
11724
|
-
if (!hasForeignData) {
|
|
11725
|
-
const clearedBranchKeys2 = database.getAllBranches();
|
|
11726
|
-
store.clear();
|
|
11727
|
-
store.save();
|
|
11728
|
-
invertedIndex.clear();
|
|
11729
|
-
this.saveInvertedIndex(invertedIndex);
|
|
11730
|
-
this.fileHashCache.clear();
|
|
11731
|
-
this.saveFileHashCache();
|
|
11732
|
-
database.clearAllIndexedData();
|
|
11733
|
-
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
11734
|
-
this.clearFailedBatchState();
|
|
11735
|
-
database.deleteMetadata("index.version");
|
|
11736
|
-
database.deleteMetadata("index.pathStorageVersion");
|
|
11737
|
-
database.deleteMetadata("index.embeddingProvider");
|
|
11738
|
-
database.deleteMetadata("index.embeddingModel");
|
|
11739
|
-
database.deleteMetadata("index.embeddingDimensions");
|
|
11740
|
-
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
11741
|
-
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
11742
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11743
|
-
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
11744
|
-
database.deleteMetadata("index.createdAt");
|
|
11745
|
-
database.deleteMetadata("index.updatedAt");
|
|
11746
|
-
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
11747
12322
|
return;
|
|
11748
12323
|
}
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
12324
|
+
throw new Error(
|
|
12325
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
12326
|
+
);
|
|
12327
|
+
}
|
|
12328
|
+
if (!hasForeignData) {
|
|
12329
|
+
this.clearGlobalIndexDataUnlocked(projectRoot);
|
|
12330
|
+
return;
|
|
12331
|
+
}
|
|
12332
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
12333
|
+
this.clearScopedFileHashCache(roots);
|
|
12334
|
+
this.clearScopedFailedBatches(roots);
|
|
12335
|
+
if (projectRoot === this.projectRoot) {
|
|
11752
12336
|
this.indexCompatibility = compatibility;
|
|
12337
|
+
}
|
|
12338
|
+
}
|
|
12339
|
+
async clearIndexUnlocked(recoveryDecision) {
|
|
12340
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
12341
|
+
if (this.config.scope === "global") {
|
|
12342
|
+
this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision);
|
|
11753
12343
|
return;
|
|
11754
12344
|
}
|
|
11755
12345
|
if (!this.isProjectOwnedIndexPath()) {
|
|
@@ -11915,6 +12505,7 @@ var Indexer = class _Indexer {
|
|
|
11915
12505
|
)) {
|
|
11916
12506
|
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
11917
12507
|
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
12508
|
+
this.restoreMissingChunkRows(database, chunks);
|
|
11918
12509
|
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
11919
12510
|
store,
|
|
11920
12511
|
provider,
|
|
@@ -11929,6 +12520,7 @@ var Indexer = class _Indexer {
|
|
|
11929
12520
|
forceReembed: false,
|
|
11930
12521
|
reuseCachedEmbeddings: false,
|
|
11931
12522
|
incrementRepeatedFailures: false,
|
|
12523
|
+
forceSingleItemBatches: true,
|
|
11932
12524
|
onSucceeded: (succeededChunks) => {
|
|
11933
12525
|
database.addChunksToBranchBatch(
|
|
11934
12526
|
this.getBranchCatalogKey(),
|
|
@@ -11950,9 +12542,12 @@ var Indexer = class _Indexer {
|
|
|
11950
12542
|
this.saveInvertedIndex(invertedIndex);
|
|
11951
12543
|
}
|
|
11952
12544
|
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
11953
|
-
database.
|
|
11954
|
-
|
|
11955
|
-
|
|
12545
|
+
const migrationFinalized = database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true";
|
|
12546
|
+
if (migrationFinalized) {
|
|
12547
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12548
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12549
|
+
this.indexCompatibility = { compatible: true };
|
|
12550
|
+
}
|
|
11956
12551
|
}
|
|
11957
12552
|
return { succeeded, failed, remaining };
|
|
11958
12553
|
}
|
|
@@ -11974,7 +12569,8 @@ var Indexer = class _Indexer {
|
|
|
11974
12569
|
latestById.set(chunkId, {
|
|
11975
12570
|
attemptCount: batch.attemptCount,
|
|
11976
12571
|
error: batch.error,
|
|
11977
|
-
lastAttempt: batch.lastAttempt
|
|
12572
|
+
lastAttempt: batch.lastAttempt,
|
|
12573
|
+
chunks: [rawChunk]
|
|
11978
12574
|
});
|
|
11979
12575
|
}
|
|
11980
12576
|
}
|
|
@@ -12041,6 +12637,7 @@ var Indexer = class _Indexer {
|
|
|
12041
12637
|
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
12042
12638
|
);
|
|
12043
12639
|
}
|
|
12640
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
12044
12641
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
12045
12642
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
12046
12643
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
@@ -12049,7 +12646,8 @@ var Indexer = class _Indexer {
|
|
|
12049
12646
|
embedding,
|
|
12050
12647
|
limit * 2,
|
|
12051
12648
|
branchChunkIds,
|
|
12052
|
-
shouldPrefilterByBranch
|
|
12649
|
+
shouldPrefilterByBranch,
|
|
12650
|
+
temporalChunkIds
|
|
12053
12651
|
);
|
|
12054
12652
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
12055
12653
|
if (this.config.scope !== "global" && branchChunkIds && !hasInitializedBranchCatalog) {
|
|
@@ -12631,6 +13229,36 @@ function resolveConfigPathValue(value, baseDir) {
|
|
|
12631
13229
|
const absolutePath = path16.isAbsolute(trimmed) ? trimmed : path16.resolve(baseDir, trimmed);
|
|
12632
13230
|
return path16.normalize(absolutePath);
|
|
12633
13231
|
}
|
|
13232
|
+
function serializeConfigPathValue(value, baseDir) {
|
|
13233
|
+
const trimmed = value.trim();
|
|
13234
|
+
if (!trimmed) {
|
|
13235
|
+
return trimmed;
|
|
13236
|
+
}
|
|
13237
|
+
if (!path16.isAbsolute(trimmed)) {
|
|
13238
|
+
return normalizePathSeparators(path16.normalize(trimmed));
|
|
13239
|
+
}
|
|
13240
|
+
const relativePath = path16.relative(baseDir, trimmed);
|
|
13241
|
+
if (!relativePath || !relativePath.startsWith("..") && !path16.isAbsolute(relativePath)) {
|
|
13242
|
+
return normalizePathSeparators(path16.normalize(relativePath || "."));
|
|
13243
|
+
}
|
|
13244
|
+
return path16.normalize(trimmed);
|
|
13245
|
+
}
|
|
13246
|
+
function resolveKnowledgeBasePath(value, projectRoot) {
|
|
13247
|
+
return path16.isAbsolute(value) ? value : path16.resolve(projectRoot, value);
|
|
13248
|
+
}
|
|
13249
|
+
function normalizeKnowledgeBasePath(value, projectRoot) {
|
|
13250
|
+
return path16.normalize(resolveKnowledgeBasePath(value, projectRoot));
|
|
13251
|
+
}
|
|
13252
|
+
function hasMatchingKnowledgeBasePath(knowledgeBases, inputPath, projectRoot) {
|
|
13253
|
+
const normalizedInput = path16.normalize(inputPath);
|
|
13254
|
+
return knowledgeBases.some((kb) => normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput);
|
|
13255
|
+
}
|
|
13256
|
+
function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot) {
|
|
13257
|
+
const normalizedInput = path16.normalize(inputPath);
|
|
13258
|
+
return knowledgeBases.findIndex(
|
|
13259
|
+
(kb) => path16.normalize(kb) === normalizedInput || normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput
|
|
13260
|
+
);
|
|
13261
|
+
}
|
|
12634
13262
|
|
|
12635
13263
|
// src/tools/format-communities.ts
|
|
12636
13264
|
function compareText(left, right) {
|
|
@@ -14337,7 +14965,7 @@ function applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key
|
|
|
14337
14965
|
function mergeUniqueStringArray(values) {
|
|
14338
14966
|
return [...new Set(values.map((value) => String(value).trim()))];
|
|
14339
14967
|
}
|
|
14340
|
-
function
|
|
14968
|
+
function normalizeKnowledgeBasePath2(value) {
|
|
14341
14969
|
let normalized = path19.normalize(String(value).trim());
|
|
14342
14970
|
const root = path19.parse(normalized).root;
|
|
14343
14971
|
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
@@ -14346,7 +14974,7 @@ function normalizeKnowledgeBasePath(value) {
|
|
|
14346
14974
|
return normalized;
|
|
14347
14975
|
}
|
|
14348
14976
|
function mergeKnowledgeBasePaths(values) {
|
|
14349
|
-
return [...new Set(values.map((value) =>
|
|
14977
|
+
return [...new Set(values.map((value) => normalizeKnowledgeBasePath2(value)).filter((value) => value.length > 0))];
|
|
14350
14978
|
}
|
|
14351
14979
|
function validateConfigLayerShape(rawConfig, filePath) {
|
|
14352
14980
|
if (!isRecord(rawConfig)) {
|
|
@@ -14476,9 +15104,30 @@ function toConfigRecord(rawConfig) {
|
|
|
14476
15104
|
}
|
|
14477
15105
|
return { ...rawConfig };
|
|
14478
15106
|
}
|
|
15107
|
+
function getConfigPath(projectRoot, host) {
|
|
15108
|
+
return resolveWritableProjectConfigPath(projectRoot, host);
|
|
15109
|
+
}
|
|
14479
15110
|
function loadRuntimeConfig(projectRoot, host) {
|
|
14480
15111
|
return normalizeKnowledgeBasePaths(toConfigRecord(loadMergedConfig(projectRoot, host)), projectRoot);
|
|
14481
15112
|
}
|
|
15113
|
+
function loadEditableConfig(projectRoot, host) {
|
|
15114
|
+
return normalizeKnowledgeBasePaths(toConfigRecord(loadProjectConfigLayer(projectRoot, host)), projectRoot);
|
|
15115
|
+
}
|
|
15116
|
+
function saveConfig(projectRoot, config, host) {
|
|
15117
|
+
const configPath = getConfigPath(projectRoot, host);
|
|
15118
|
+
const configDir = path20.dirname(configPath);
|
|
15119
|
+
const configBaseDir = path20.dirname(configDir);
|
|
15120
|
+
if (!(0, import_fs13.existsSync)(configDir)) {
|
|
15121
|
+
(0, import_fs13.mkdirSync)(configDir, { recursive: true });
|
|
15122
|
+
}
|
|
15123
|
+
const serializableConfig = { ...config };
|
|
15124
|
+
if (Array.isArray(serializableConfig.knowledgeBases)) {
|
|
15125
|
+
serializableConfig.knowledgeBases = serializableConfig.knowledgeBases.map(
|
|
15126
|
+
(kb) => serializeConfigPathValue(kb, configBaseDir)
|
|
15127
|
+
);
|
|
15128
|
+
}
|
|
15129
|
+
(0, import_fs13.writeFileSync)(configPath, JSON.stringify(serializableConfig, null, 2) + "\n", "utf-8");
|
|
15130
|
+
}
|
|
14482
15131
|
|
|
14483
15132
|
// src/tools/operation-runtime.ts
|
|
14484
15133
|
var indexerCache = /* @__PURE__ */ new Map();
|
|
@@ -14697,9 +15346,11 @@ async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
|
14697
15346
|
contextLines: options.contextLines,
|
|
14698
15347
|
metadataOnly: options.metadataOnly,
|
|
14699
15348
|
definitionIntent: options.definitionIntent,
|
|
15349
|
+
prioritizeSourcePaths: options.prioritizeSourcePaths,
|
|
14700
15350
|
blameAuthor: options.blameAuthor,
|
|
14701
15351
|
blameSha: options.blameSha,
|
|
14702
15352
|
blameSince: options.blameSince,
|
|
15353
|
+
blameUntil: options.blameUntil,
|
|
14703
15354
|
trace: options.trace
|
|
14704
15355
|
});
|
|
14705
15356
|
}
|
|
@@ -14745,7 +15396,9 @@ async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
|
14745
15396
|
fileType: options.fileType,
|
|
14746
15397
|
directory: options.directory,
|
|
14747
15398
|
chunkType: options.chunkType,
|
|
14748
|
-
excludeFile: options.excludeFile
|
|
15399
|
+
excludeFile: options.excludeFile,
|
|
15400
|
+
blameSince: options.blameSince,
|
|
15401
|
+
blameUntil: options.blameUntil
|
|
14749
15402
|
});
|
|
14750
15403
|
}
|
|
14751
15404
|
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
@@ -14808,6 +15461,9 @@ async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
|
14808
15461
|
if (args.estimateOnly) {
|
|
14809
15462
|
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
14810
15463
|
}
|
|
15464
|
+
if (args.dryRun) {
|
|
15465
|
+
return { kind: "dryrun", dryrun: await indexer.dryRunCost() };
|
|
15466
|
+
}
|
|
14811
15467
|
const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => {
|
|
14812
15468
|
if (onProgress) {
|
|
14813
15469
|
void onProgress(formatProgressTitle(progress), {
|
|
@@ -14991,6 +15647,141 @@ async function getIndexLogs(projectRoot, host, args) {
|
|
|
14991
15647
|
}).join("\n");
|
|
14992
15648
|
return { kind: "entries", text };
|
|
14993
15649
|
}
|
|
15650
|
+
function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
15651
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15652
|
+
const inputPath = knowledgeBasePath.trim();
|
|
15653
|
+
const normalizedPath3 = path21.resolve(
|
|
15654
|
+
path21.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
|
|
15655
|
+
);
|
|
15656
|
+
if (!(0, import_fs14.existsSync)(normalizedPath3)) {
|
|
15657
|
+
return `Error: Directory does not exist: ${normalizedPath3}`;
|
|
15658
|
+
}
|
|
15659
|
+
let realPath;
|
|
15660
|
+
try {
|
|
15661
|
+
realPath = (0, import_fs14.realpathSync)(normalizedPath3);
|
|
15662
|
+
} catch {
|
|
15663
|
+
return `Error: Cannot resolve path: ${normalizedPath3}`;
|
|
15664
|
+
}
|
|
15665
|
+
const blockedPrefixes = [
|
|
15666
|
+
"/etc",
|
|
15667
|
+
"/proc",
|
|
15668
|
+
"/sys",
|
|
15669
|
+
"/dev",
|
|
15670
|
+
"/boot",
|
|
15671
|
+
"/root",
|
|
15672
|
+
"/var/run",
|
|
15673
|
+
"/var/log"
|
|
15674
|
+
];
|
|
15675
|
+
const homeDir = process.platform === "win32" ? process.env.USERPROFILE ?? "" : process.env.HOME ?? "";
|
|
15676
|
+
const sensitiveDotDirs = [
|
|
15677
|
+
".ssh",
|
|
15678
|
+
".gnupg",
|
|
15679
|
+
".aws",
|
|
15680
|
+
".config/gcloud",
|
|
15681
|
+
".docker",
|
|
15682
|
+
".kube"
|
|
15683
|
+
];
|
|
15684
|
+
for (const prefix of blockedPrefixes) {
|
|
15685
|
+
if (realPath === prefix || realPath.startsWith(`${prefix}/`)) {
|
|
15686
|
+
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
|
|
15687
|
+
}
|
|
15688
|
+
}
|
|
15689
|
+
for (const dotDir of sensitiveDotDirs) {
|
|
15690
|
+
const sensitiveDir = path21.join(homeDir, dotDir);
|
|
15691
|
+
if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
|
|
15692
|
+
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
|
|
15693
|
+
}
|
|
15694
|
+
}
|
|
15695
|
+
try {
|
|
15696
|
+
const stat5 = (0, import_fs14.statSync)(normalizedPath3);
|
|
15697
|
+
if (!stat5.isDirectory()) {
|
|
15698
|
+
return `Error: Path is not a directory: ${normalizedPath3}`;
|
|
15699
|
+
}
|
|
15700
|
+
} catch (error) {
|
|
15701
|
+
return `Error: Cannot access directory: ${normalizedPath3} - ${error instanceof Error ? error.message : String(error)}`;
|
|
15702
|
+
}
|
|
15703
|
+
const config = loadEditableConfig(root, host);
|
|
15704
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15705
|
+
const alreadyExists = hasMatchingKnowledgeBasePath(knowledgeBases, normalizedPath3, root);
|
|
15706
|
+
if (alreadyExists) {
|
|
15707
|
+
return `Knowledge base already configured: ${normalizedPath3}`;
|
|
15708
|
+
}
|
|
15709
|
+
knowledgeBases.push(normalizedPath3);
|
|
15710
|
+
config.knowledgeBases = knowledgeBases;
|
|
15711
|
+
saveConfig(root, config, host);
|
|
15712
|
+
refreshIndexerForDirectory(root, host);
|
|
15713
|
+
let result = `${normalizedPath3}
|
|
15714
|
+
`;
|
|
15715
|
+
result += `Total knowledge bases: ${knowledgeBases.length}
|
|
15716
|
+
`;
|
|
15717
|
+
result += `Config path: ${getConfigPath(root, host)}
|
|
15718
|
+
`;
|
|
15719
|
+
result += `
|
|
15720
|
+
Run /index to rebuild the index with the new knowledge base.`;
|
|
15721
|
+
return result;
|
|
15722
|
+
}
|
|
15723
|
+
function listKnowledgeBases(projectRoot, host) {
|
|
15724
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15725
|
+
const config = loadRuntimeConfig(root, host);
|
|
15726
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15727
|
+
if (knowledgeBases.length === 0) {
|
|
15728
|
+
return "No knowledge bases configured. Use add_knowledge_base to add folders.";
|
|
15729
|
+
}
|
|
15730
|
+
let result = `Knowledge Bases (${knowledgeBases.length}):
|
|
15731
|
+
|
|
15732
|
+
`;
|
|
15733
|
+
for (let i = 0; i < knowledgeBases.length; i++) {
|
|
15734
|
+
const kb = knowledgeBases[i];
|
|
15735
|
+
const resolvedPath = resolveKnowledgeBasePath(kb, root);
|
|
15736
|
+
const exists = (0, import_fs14.existsSync)(resolvedPath);
|
|
15737
|
+
result += `[${i + 1}] ${kb}
|
|
15738
|
+
`;
|
|
15739
|
+
result += ` Resolved: ${resolvedPath}
|
|
15740
|
+
`;
|
|
15741
|
+
result += ` Status: ${exists ? "Exists" : "NOT FOUND"}
|
|
15742
|
+
`;
|
|
15743
|
+
if (exists) {
|
|
15744
|
+
try {
|
|
15745
|
+
const stat5 = (0, import_fs14.statSync)(resolvedPath);
|
|
15746
|
+
result += ` Type: ${stat5.isDirectory() ? "Directory" : "File"}
|
|
15747
|
+
`;
|
|
15748
|
+
} catch {
|
|
15749
|
+
}
|
|
15750
|
+
}
|
|
15751
|
+
result += "\n";
|
|
15752
|
+
}
|
|
15753
|
+
const hasHostConfig = (0, import_fs14.existsSync)(path21.join(root, getHostProjectConfigRelativePath(host)));
|
|
15754
|
+
if (hasHostConfig) {
|
|
15755
|
+
result += `
|
|
15756
|
+
Config sources: 1 file(s).`;
|
|
15757
|
+
}
|
|
15758
|
+
result += `
|
|
15759
|
+
Config file: ${getConfigPath(root, host)}`;
|
|
15760
|
+
return result;
|
|
15761
|
+
}
|
|
15762
|
+
function removeKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
15763
|
+
const root = getProjectRoot(projectRoot, host);
|
|
15764
|
+
const config = loadEditableConfig(root, host);
|
|
15765
|
+
const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
|
|
15766
|
+
const index = findKnowledgeBasePathIndex(knowledgeBases, knowledgeBasePath, root);
|
|
15767
|
+
if (index === -1) {
|
|
15768
|
+
return `Knowledge base not found: ${knowledgeBasePath}`;
|
|
15769
|
+
}
|
|
15770
|
+
const removed = knowledgeBases.splice(index, 1)[0];
|
|
15771
|
+
config.knowledgeBases = knowledgeBases;
|
|
15772
|
+
saveConfig(root, config, host);
|
|
15773
|
+
refreshIndexerForDirectory(root, host);
|
|
15774
|
+
let result = `Removed: ${removed}
|
|
15775
|
+
|
|
15776
|
+
`;
|
|
15777
|
+
result += `Remaining knowledge bases: ${knowledgeBases.length}
|
|
15778
|
+
`;
|
|
15779
|
+
result += `Config saved to: ${getConfigPath(root, host)}
|
|
15780
|
+
`;
|
|
15781
|
+
result += `
|
|
15782
|
+
Run /index to rebuild the index without the removed knowledge base.`;
|
|
15783
|
+
return result;
|
|
15784
|
+
}
|
|
14994
15785
|
|
|
14995
15786
|
// src/tools/context-search.ts
|
|
14996
15787
|
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
@@ -15219,13 +16010,19 @@ async function resolveSearchContext(input, operations) {
|
|
|
15219
16010
|
(trace) => operations.lookup(symbol, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
|
|
15220
16011
|
);
|
|
15221
16012
|
};
|
|
15222
|
-
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt) => {
|
|
16013
|
+
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt, prioritizeSourcePaths) => {
|
|
15223
16014
|
return recordAttempt(
|
|
15224
16015
|
"conceptual",
|
|
15225
16016
|
searchQuery,
|
|
15226
16017
|
scope,
|
|
15227
16018
|
relaxedFieldsForAttempt,
|
|
15228
|
-
(trace) => operations.search(
|
|
16019
|
+
(trace) => operations.search(
|
|
16020
|
+
searchQuery,
|
|
16021
|
+
MAX_CONTEXT_RESULT_LIMIT,
|
|
16022
|
+
scope,
|
|
16023
|
+
input.diagnostic ? trace : void 0,
|
|
16024
|
+
{ prioritizeSourcePaths }
|
|
16025
|
+
)
|
|
15229
16026
|
);
|
|
15230
16027
|
};
|
|
15231
16028
|
const findSuccessfulAttemptState = (route) => {
|
|
@@ -15353,10 +16150,12 @@ Explicit symbol lookup only; conceptual search was not attempted.`
|
|
|
15353
16150
|
}
|
|
15354
16151
|
}
|
|
15355
16152
|
for (const attempt of conceptualAttemptPlan) {
|
|
16153
|
+
const attemptIntent = analyzeQueryIntent(attempt.queryText);
|
|
16154
|
+
const prioritizeSourcePaths = attemptIntent.primary !== "docs" && attemptIntent.primary !== "test";
|
|
15356
16155
|
if (inferredSymbol && attempt.queryText === inferredSymbol && attempt.queryText !== query) {
|
|
15357
16156
|
decisions.fallbackFromOriginalConceptualToInferred = true;
|
|
15358
16157
|
}
|
|
15359
|
-
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed);
|
|
16158
|
+
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed, prioritizeSourcePaths);
|
|
15360
16159
|
if (results.length > 0) {
|
|
15361
16160
|
const heading = buildPackHeading("conceptual", decisions);
|
|
15362
16161
|
const intent = analyzeQueryIntent(attempt.queryText);
|
|
@@ -15493,12 +16292,13 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
15493
16292
|
directory: scope.directory,
|
|
15494
16293
|
trace
|
|
15495
16294
|
}),
|
|
15496
|
-
search: (queryText, retrievalLimit, scope, trace) => searchCodebase(projectRoot, host, queryText, {
|
|
16295
|
+
search: (queryText, retrievalLimit, scope, trace, searchOptions) => searchCodebase(projectRoot, host, queryText, {
|
|
15497
16296
|
limit: retrievalLimit,
|
|
15498
16297
|
fileType: scope.fileType,
|
|
15499
16298
|
directory: scope.directory,
|
|
15500
16299
|
metadataOnly: true,
|
|
15501
|
-
trace
|
|
16300
|
+
trace,
|
|
16301
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
15502
16302
|
})
|
|
15503
16303
|
});
|
|
15504
16304
|
}
|
|
@@ -16999,12 +17799,13 @@ async function runEvaluation(options) {
|
|
|
16999
17799
|
fileType: scope.fileType,
|
|
17000
17800
|
directory: scope.directory
|
|
17001
17801
|
}),
|
|
17002
|
-
search: (searchQuery, limit, scope) => indexer.search(searchQuery, limit, {
|
|
17802
|
+
search: (searchQuery, limit, scope, _trace, searchOptions) => indexer.search(searchQuery, limit, {
|
|
17003
17803
|
metadataOnly: true,
|
|
17004
17804
|
filterByBranch: !!query.expected.branch,
|
|
17005
17805
|
definitionIntent: false,
|
|
17006
17806
|
fileType: scope.fileType,
|
|
17007
|
-
directory: scope.directory
|
|
17807
|
+
directory: scope.directory,
|
|
17808
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
17008
17809
|
})
|
|
17009
17810
|
}) : void 0;
|
|
17010
17811
|
const result = editContextResult?.results ?? contextResult?.details?.results ?? await indexer.search(query.query, 10, {
|
|
@@ -17590,6 +18391,7 @@ async function executeCodebaseEditContext(projectRoot, host, args) {
|
|
|
17590
18391
|
async function executeIndexCodebase(projectRoot, host, args, onProgress) {
|
|
17591
18392
|
const result = await runIndexCodebase(projectRoot, host, args, onProgress);
|
|
17592
18393
|
if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) };
|
|
18394
|
+
if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) };
|
|
17593
18395
|
if (result.kind === "busy") return { text: result.text, isError: true };
|
|
17594
18396
|
if (result.kind === "message") return { text: result.text };
|
|
17595
18397
|
return { text: formatIndexStats(result.stats, args.verbose ?? false) };
|
|
@@ -17781,11 +18583,21 @@ var PI_TOOL_NAMES = [
|
|
|
17781
18583
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
17782
18584
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
17783
18585
|
];
|
|
18586
|
+
var MCP_TOOL_NAMES = [
|
|
18587
|
+
...PORTABLE_TOOL_NAMES,
|
|
18588
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
18589
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18590
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE
|
|
18591
|
+
];
|
|
17784
18592
|
|
|
17785
18593
|
// src/adapters/mcp/register-tools.ts
|
|
17786
18594
|
function allowNullAsUndefined(schema) {
|
|
17787
18595
|
return import_zod2.z.preprocess((value) => value === null ? void 0 : value, schema);
|
|
17788
18596
|
}
|
|
18597
|
+
function knowledgeBaseResult(text) {
|
|
18598
|
+
const content = [{ type: "text", text }];
|
|
18599
|
+
return text.startsWith("Error: ") ? { content, isError: true } : { content };
|
|
18600
|
+
}
|
|
17789
18601
|
function registerMcpTools(server, runtime) {
|
|
17790
18602
|
server.tool(
|
|
17791
18603
|
TOOL_NAME.CODEBASE_CONTEXT,
|
|
@@ -17846,7 +18658,8 @@ function registerMcpTools(server, runtime) {
|
|
|
17846
18658
|
contextLines: allowNullAsUndefined(import_zod2.z.number().optional()).describe("Number of extra lines to include before/after each match (default: 0)"),
|
|
17847
18659
|
blameAuthor: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame author name or email"),
|
|
17848
18660
|
blameSha: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame commit SHA or prefix"),
|
|
17849
|
-
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date")
|
|
18661
|
+
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18662
|
+
blameUntil: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17850
18663
|
},
|
|
17851
18664
|
async (args) => {
|
|
17852
18665
|
return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "search", args.query, {
|
|
@@ -17857,7 +18670,8 @@ function registerMcpTools(server, runtime) {
|
|
|
17857
18670
|
contextLines: args.contextLines,
|
|
17858
18671
|
blameAuthor: args.blameAuthor,
|
|
17859
18672
|
blameSha: args.blameSha,
|
|
17860
|
-
blameSince: args.blameSince
|
|
18673
|
+
blameSince: args.blameSince,
|
|
18674
|
+
blameUntil: args.blameUntil
|
|
17861
18675
|
}, (results) => {
|
|
17862
18676
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} results for "${args.query}":
|
|
17863
18677
|
|
|
@@ -17877,7 +18691,8 @@ ${formatSearchResults(results, "score")}`;
|
|
|
17877
18691
|
chunkType: allowNullAsUndefined(import_zod2.z.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
|
|
17878
18692
|
blameAuthor: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame author name or email"),
|
|
17879
18693
|
blameSha: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by git blame commit SHA or prefix"),
|
|
17880
|
-
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date")
|
|
18694
|
+
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18695
|
+
blameUntil: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17881
18696
|
},
|
|
17882
18697
|
async (args) => {
|
|
17883
18698
|
return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "peek", args.query, {
|
|
@@ -17888,7 +18703,8 @@ ${formatSearchResults(results, "score")}`;
|
|
|
17888
18703
|
metadataOnly: true,
|
|
17889
18704
|
blameAuthor: args.blameAuthor,
|
|
17890
18705
|
blameSha: args.blameSha,
|
|
17891
|
-
blameSince: args.blameSince
|
|
18706
|
+
blameSince: args.blameSince,
|
|
18707
|
+
blameUntil: args.blameUntil
|
|
17892
18708
|
}, (results) => {
|
|
17893
18709
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} locations for "${args.query}":
|
|
17894
18710
|
|
|
@@ -17903,6 +18719,7 @@ ${formatCodebasePeek(results)}`;
|
|
|
17903
18719
|
{
|
|
17904
18720
|
force: allowNullAsUndefined(import_zod2.z.boolean().optional().default(false)).describe("Force reindex even if already indexed"),
|
|
17905
18721
|
estimateOnly: allowNullAsUndefined(import_zod2.z.boolean().optional().default(false)).describe("Only show cost estimate without indexing"),
|
|
18722
|
+
dryRun: allowNullAsUndefined(import_zod2.z.boolean().optional().default(false)).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."),
|
|
17906
18723
|
verbose: allowNullAsUndefined(import_zod2.z.boolean().optional().default(false)).describe("Show detailed info about skipped files and parsing failures")
|
|
17907
18724
|
},
|
|
17908
18725
|
async (args) => {
|
|
@@ -17965,7 +18782,9 @@ ${formatCodebasePeek(results)}`;
|
|
|
17965
18782
|
fileType: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
17966
18783
|
directory: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
17967
18784
|
chunkType: allowNullAsUndefined(import_zod2.z.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
|
|
17968
|
-
excludeFile: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Exclude results from this file path")
|
|
18785
|
+
excludeFile: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Exclude results from this file path"),
|
|
18786
|
+
blameSince: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or after this date"),
|
|
18787
|
+
blameUntil: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter to chunks last changed on or before this date")
|
|
17969
18788
|
},
|
|
17970
18789
|
async (args) => {
|
|
17971
18790
|
const results = await findSimilarCode(runtime.projectRoot, runtime.host, args.code, {
|
|
@@ -17973,7 +18792,9 @@ ${formatCodebasePeek(results)}`;
|
|
|
17973
18792
|
fileType: args.fileType,
|
|
17974
18793
|
directory: args.directory,
|
|
17975
18794
|
chunkType: args.chunkType,
|
|
17976
|
-
excludeFile: args.excludeFile
|
|
18795
|
+
excludeFile: args.excludeFile,
|
|
18796
|
+
blameSince: args.blameSince,
|
|
18797
|
+
blameUntil: args.blameUntil
|
|
17977
18798
|
});
|
|
17978
18799
|
if (results.length === 0) {
|
|
17979
18800
|
return { content: [{ type: "text", text: "No similar code found. Try a different snippet or run index_codebase first." }] };
|
|
@@ -18077,12 +18898,43 @@ ${formatSearchResults(results)}` }] };
|
|
|
18077
18898
|
return { content: [{ type: "text", text: result.text }] };
|
|
18078
18899
|
}
|
|
18079
18900
|
);
|
|
18901
|
+
server.tool(
|
|
18902
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
18903
|
+
"Add a folder as a knowledge base to the semantic search index. The folder is indexed alongside the project code on the next index run. Provide an absolute path or a path relative to the project root. The path is written to the project-local host config of this MCP server (under the server project root), not to a user-global config, and the index is refreshed. Git blame metadata is collected only for files in the project git repo; knowledge-base files outside the repo remain searchable by content but have no blame. A knowledge base that appears in list_knowledge_bases but was inherited from a global config cannot be removed by this tool.",
|
|
18904
|
+
{
|
|
18905
|
+
path: import_zod2.z.string().describe("Path to the folder to add as a knowledge base (absolute or relative to the project root)")
|
|
18906
|
+
},
|
|
18907
|
+
async (args) => {
|
|
18908
|
+
const result = addKnowledgeBase(runtime.projectRoot, runtime.host, args.path);
|
|
18909
|
+
return knowledgeBaseResult(result);
|
|
18910
|
+
}
|
|
18911
|
+
);
|
|
18912
|
+
server.tool(
|
|
18913
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18914
|
+
"List the configured knowledge base folders that the index includes alongside the project code. The list is the union of project-local and user-global knowledge bases; each entry shows the resolved path and whether it exists.",
|
|
18915
|
+
{},
|
|
18916
|
+
async () => {
|
|
18917
|
+
const result = listKnowledgeBases(runtime.projectRoot, runtime.host);
|
|
18918
|
+
return knowledgeBaseResult(result);
|
|
18919
|
+
}
|
|
18920
|
+
);
|
|
18921
|
+
server.tool(
|
|
18922
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE,
|
|
18923
|
+
"Remove a knowledge base folder from the semantic search index and refresh the index. The path must match a project-local configured path exactly. Knowledge bases inherited from a user-global config are not removable by this tool.",
|
|
18924
|
+
{
|
|
18925
|
+
path: import_zod2.z.string().describe("Path of the knowledge base to remove (must match a project-local configured path exactly)")
|
|
18926
|
+
},
|
|
18927
|
+
async (args) => {
|
|
18928
|
+
const result = removeKnowledgeBase(runtime.projectRoot, runtime.host, args.path.trim());
|
|
18929
|
+
return knowledgeBaseResult(result);
|
|
18930
|
+
}
|
|
18931
|
+
);
|
|
18080
18932
|
}
|
|
18081
18933
|
|
|
18082
18934
|
// src/adapters/mcp/server.ts
|
|
18083
18935
|
function getServerInstructions(host) {
|
|
18084
18936
|
const hostText = `host ${host}`;
|
|
18085
|
-
return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns a token-budgeted location pack and routes to definitions or call-graph helpers when symbol intent is present. Keep the default tokenBudget for normal discovery, then use implementation_lookup, codebase_search, or a targeted file read only for selected locations that need source content. Use codebase_peek for direct conceptual location lookup. For exact identifiers or exhaustive matches, use grep. After identifying symbols, use call_graph or call_graph_path to trace dependencies. If the index is unavailable, run index_codebase, then retry the retrieval tool.`;
|
|
18937
|
+
return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns a token-budgeted location pack and routes to definitions or call-graph helpers when symbol intent is present. For code changes with a known or suspected symbol target, optionally call codebase_edit_context as a compact pre-edit step for bounded source plus direct callers and callees before broad file reads. Keep the default tokenBudget for normal discovery, then use implementation_lookup, codebase_search, or a targeted file read only for selected locations that need source content. Use codebase_peek for direct conceptual location lookup. For exact identifiers or exhaustive matches, use grep. After identifying symbols, use call_graph or call_graph_path to trace dependencies. If the index is unavailable, run index_codebase, then retry the retrieval tool.`;
|
|
18086
18938
|
}
|
|
18087
18939
|
function createMcpServer(projectRoot, config, host) {
|
|
18088
18940
|
const server = new import_mcp.McpServer({
|
|
@@ -21500,6 +22352,7 @@ function parseIndexArgs(argv, cwd) {
|
|
|
21500
22352
|
let config;
|
|
21501
22353
|
let force = false;
|
|
21502
22354
|
let estimateOnly = false;
|
|
22355
|
+
let dryRun = false;
|
|
21503
22356
|
let verbose = false;
|
|
21504
22357
|
for (let i = 0; i < argv.length; i += 1) {
|
|
21505
22358
|
const arg = argv[i];
|
|
@@ -21537,13 +22390,16 @@ function parseIndexArgs(argv, cwd) {
|
|
|
21537
22390
|
host = parseHostMode(value);
|
|
21538
22391
|
continue;
|
|
21539
22392
|
}
|
|
21540
|
-
if (arg === "--force" || arg === "--estimate-only" || arg === "--verbose") {
|
|
22393
|
+
if (arg === "--force" || arg === "--estimate-only" || arg === "--dry-run" || arg === "--verbose") {
|
|
21541
22394
|
if (arg === "--force") {
|
|
21542
22395
|
force = true;
|
|
21543
22396
|
}
|
|
21544
22397
|
if (arg === "--estimate-only") {
|
|
21545
22398
|
estimateOnly = true;
|
|
21546
22399
|
}
|
|
22400
|
+
if (arg === "--dry-run") {
|
|
22401
|
+
dryRun = true;
|
|
22402
|
+
}
|
|
21547
22403
|
if (arg === "--verbose") {
|
|
21548
22404
|
verbose = true;
|
|
21549
22405
|
}
|
|
@@ -21554,7 +22410,7 @@ function parseIndexArgs(argv, cwd) {
|
|
|
21554
22410
|
}
|
|
21555
22411
|
throw new Error(`Unknown index option: ${arg}`);
|
|
21556
22412
|
}
|
|
21557
|
-
return { project, host, config, force, estimateOnly, verbose };
|
|
22413
|
+
return { project, host, config, force, estimateOnly, dryRun, verbose };
|
|
21558
22414
|
}
|
|
21559
22415
|
function loadCliRawConfig(args) {
|
|
21560
22416
|
return args.config ? loadConfigFile(args.config) : loadMergedConfig(args.project, args.host);
|
|
@@ -21571,6 +22427,7 @@ Options:
|
|
|
21571
22427
|
--config <path> Explicit JSON config path
|
|
21572
22428
|
--force Rebuild index even if already up to date
|
|
21573
22429
|
--estimate-only Estimate indexing cost only
|
|
22430
|
+
--dry-run Parse only; report the exact embedding token total without indexing
|
|
21574
22431
|
--verbose Include detailed final index statistics
|
|
21575
22432
|
--help Show this message
|
|
21576
22433
|
|
|
@@ -21766,6 +22623,7 @@ async function handleIndexCommand(argv, cwd, deps = {}) {
|
|
|
21766
22623
|
const indexArgs = {
|
|
21767
22624
|
force: parsedArgs.force,
|
|
21768
22625
|
estimateOnly: parsedArgs.estimateOnly,
|
|
22626
|
+
dryRun: parsedArgs.dryRun,
|
|
21769
22627
|
verbose: parsedArgs.verbose
|
|
21770
22628
|
};
|
|
21771
22629
|
const result = await runIndex(parsedArgs.project, parsedArgs.host, indexArgs, (title, metadata) => {
|