opencode-codebase-index 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +16 -5
- package/commands/peek.md +2 -1
- package/commands/search.md +2 -1
- package/dist/cbi.cjs +14792 -0
- package/dist/cbi.cjs.map +1 -0
- package/dist/cbi.js +14786 -0
- package/dist/cbi.js.map +1 -0
- package/dist/cli.cjs +1121 -263
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1121 -263
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +885 -259
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +885 -259
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +5529 -2154
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +5517 -2142
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +5 -2
- package/skill/SKILL.md +12 -16
package/dist/index.cjs
CHANGED
|
@@ -724,6 +724,17 @@ var EMBEDDING_MODELS = {
|
|
|
724
724
|
maxTokens: 2048,
|
|
725
725
|
costPer1MTokens: 0.15,
|
|
726
726
|
taskAble: true
|
|
727
|
+
},
|
|
728
|
+
"gemini-embedding-2": {
|
|
729
|
+
provider: "google",
|
|
730
|
+
model: "gemini-embedding-2",
|
|
731
|
+
// Keep a conservative, testable default embedding dimension. Gemini Embedding 2 supports
|
|
732
|
+
// flexible dimensions via outputDimensionality.
|
|
733
|
+
dimensions: 1536,
|
|
734
|
+
maxTokens: 8192,
|
|
735
|
+
costPer1MTokens: 0.15,
|
|
736
|
+
taskAble: false,
|
|
737
|
+
promptStyle: "embedding-2"
|
|
727
738
|
}
|
|
728
739
|
},
|
|
729
740
|
"openai": {
|
|
@@ -757,26 +768,15 @@ var EMBEDDING_MODELS = {
|
|
|
757
768
|
maxTokens: 512,
|
|
758
769
|
costPer1MTokens: 0
|
|
759
770
|
}
|
|
760
|
-
},
|
|
761
|
-
"github-copilot": {
|
|
762
|
-
"text-embedding-3-small": {
|
|
763
|
-
provider: "github-copilot",
|
|
764
|
-
model: "text-embedding-3-small",
|
|
765
|
-
dimensions: 1536,
|
|
766
|
-
maxTokens: 8191,
|
|
767
|
-
costPer1MTokens: 0
|
|
768
|
-
}
|
|
769
771
|
}
|
|
770
772
|
};
|
|
771
773
|
var DEFAULT_PROVIDER_MODELS = {
|
|
772
|
-
"github-copilot": "text-embedding-3-small",
|
|
773
774
|
"openai": "text-embedding-3-small",
|
|
774
775
|
"google": "gemini-embedding-001",
|
|
775
776
|
"ollama": "nomic-embed-text"
|
|
776
777
|
};
|
|
777
778
|
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
778
779
|
"ollama",
|
|
779
|
-
"github-copilot",
|
|
780
780
|
"openai",
|
|
781
781
|
"google"
|
|
782
782
|
];
|
|
@@ -802,6 +802,9 @@ function getDefaultIndexingConfig() {
|
|
|
802
802
|
maxDepth: 5,
|
|
803
803
|
maxFilesPerDirectory: 100,
|
|
804
804
|
fallbackToTextOnMaxChunks: true,
|
|
805
|
+
// Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi
|
|
806
|
+
// fallback used when a native caller omits the argument).
|
|
807
|
+
linesPerChunk: 30,
|
|
805
808
|
gitBlame: { enabled: false }
|
|
806
809
|
};
|
|
807
810
|
}
|
|
@@ -935,6 +938,7 @@ function parseConfig(raw) {
|
|
|
935
938
|
maxDepth: typeof rawIndexing.maxDepth === "number" ? rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth : defaultIndexing.maxDepth,
|
|
936
939
|
maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
|
|
937
940
|
fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
|
|
941
|
+
linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk,
|
|
938
942
|
gitBlame: {
|
|
939
943
|
enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof rawIndexing.gitBlame.enabled === "boolean" ? rawIndexing.gitBlame.enabled : defaultIndexing.gitBlame.enabled
|
|
940
944
|
}
|
|
@@ -977,6 +981,7 @@ function parseConfig(raw) {
|
|
|
977
981
|
let embeddingModel;
|
|
978
982
|
let customProvider;
|
|
979
983
|
let reranker;
|
|
984
|
+
const githubCopilotDeprecationMessage = '`embeddingProvider: "github-copilot"` is deprecated and no longer available. Migrate existing configs to `embeddingProvider: "google"` and select an explicit Google model. For existing indexes, run `index_codebase` with `force: true` after changing to `gemini-embedding-001` or `gemini-embedding-2` to rebuild embeddings. See docs/configuration.md for details.';
|
|
980
985
|
if (embeddingProviderValue === "custom") {
|
|
981
986
|
embeddingProvider = "custom";
|
|
982
987
|
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
@@ -1016,6 +1021,8 @@ function parseConfig(raw) {
|
|
|
1016
1021
|
} else if (rawEmbeddingModel) {
|
|
1017
1022
|
embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
1018
1023
|
}
|
|
1024
|
+
} else if (embeddingProviderValue === "github-copilot") {
|
|
1025
|
+
throw new Error(githubCopilotDeprecationMessage);
|
|
1019
1026
|
} else {
|
|
1020
1027
|
embeddingProvider = "auto";
|
|
1021
1028
|
}
|
|
@@ -1046,10 +1053,21 @@ function parseConfig(raw) {
|
|
|
1046
1053
|
timeoutMs: typeof rawReranker.timeoutMs === "number" ? Math.max(1e3, Math.floor(rawReranker.timeoutMs)) : 1e4
|
|
1047
1054
|
};
|
|
1048
1055
|
}
|
|
1056
|
+
const rawEmbedding = input.embedding && typeof input.embedding === "object" ? input.embedding : {};
|
|
1057
|
+
const rawEmbeddingBatch = rawEmbedding.batch && typeof rawEmbedding.batch === "object" ? rawEmbedding.batch : null;
|
|
1058
|
+
const embeddingMaxBatchItems = typeof rawEmbeddingBatch?.maxBatchItems === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchItems) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchItems)) : void 0;
|
|
1059
|
+
const embeddingMaxBatchTokens = typeof rawEmbeddingBatch?.maxBatchTokens === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchTokens) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchTokens)) : void 0;
|
|
1060
|
+
const embedding = embeddingMaxBatchItems !== void 0 || embeddingMaxBatchTokens !== void 0 ? {
|
|
1061
|
+
batch: {
|
|
1062
|
+
...embeddingMaxBatchItems !== void 0 ? { maxBatchItems: embeddingMaxBatchItems } : {},
|
|
1063
|
+
...embeddingMaxBatchTokens !== void 0 ? { maxBatchTokens: embeddingMaxBatchTokens } : {}
|
|
1064
|
+
}
|
|
1065
|
+
} : {};
|
|
1049
1066
|
return {
|
|
1050
1067
|
embeddingProvider,
|
|
1051
1068
|
embeddingModel,
|
|
1052
1069
|
customProvider,
|
|
1070
|
+
embedding,
|
|
1053
1071
|
scope: isValidScope(scopeValue) ? scopeValue : "project",
|
|
1054
1072
|
include: includeValue ?? DEFAULT_INCLUDE,
|
|
1055
1073
|
exclude: excludeValue ?? DEFAULT_EXCLUDE,
|
|
@@ -2316,6 +2334,10 @@ function scoreCandidate(query, intent, candidate, originalIndex) {
|
|
|
2316
2334
|
let boost = 0;
|
|
2317
2335
|
if (intent.primary === "conceptual") {
|
|
2318
2336
|
boost += Math.min(0.14, overlap * 0.14);
|
|
2337
|
+
if (intent.preferSourcePaths) {
|
|
2338
|
+
boost += implementationPath ? 0.32 : 0;
|
|
2339
|
+
if (testPath || fixturePath || docsPath) boost -= 0.35;
|
|
2340
|
+
}
|
|
2319
2341
|
if (generatedOrVendor) boost -= 0.18;
|
|
2320
2342
|
if (importChunk || weakContainer) boost -= 0.04;
|
|
2321
2343
|
} else if (intent.primary === "test") {
|
|
@@ -3283,6 +3305,19 @@ function parseOwner(value) {
|
|
|
3283
3305
|
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
3284
3306
|
if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
|
|
3285
3307
|
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
3308
|
+
if (candidate.recoveryProtocolVersion !== void 0 && candidate.recoveryProtocolVersion !== 1) return null;
|
|
3309
|
+
if (candidate.projectRoot !== void 0 && typeof candidate.projectRoot !== "string") return null;
|
|
3310
|
+
if (candidate.scopedRoots !== void 0) {
|
|
3311
|
+
if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) {
|
|
3312
|
+
return null;
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
if (candidate.clearRecovery !== void 0) {
|
|
3316
|
+
const recovery = candidate.clearRecovery;
|
|
3317
|
+
if (typeof recovery !== "object" || recovery === null || recovery.phase !== "clearing" || typeof recovery.embeddingProvider !== "string" || recovery.embeddingProvider.length === 0 || typeof recovery.embeddingModel !== "string" || recovery.embeddingModel.length === 0 || !Number.isInteger(recovery.embeddingDimensions) || (recovery.embeddingDimensions ?? 0) <= 0 || typeof recovery.embeddingStrategyVersion !== "string" || recovery.embeddingStrategyVersion.length === 0 || recovery.compatibilityDecision !== "compatible" && recovery.compatibilityDecision !== "embedding-strategy-mismatch" && recovery.compatibilityDecision !== "incompatible" || candidate.operation !== "clear" && candidate.operation !== "force-index") {
|
|
3318
|
+
return null;
|
|
3319
|
+
}
|
|
3320
|
+
}
|
|
3286
3321
|
return candidate;
|
|
3287
3322
|
}
|
|
3288
3323
|
function parseReclaimOwner(value) {
|
|
@@ -3523,13 +3558,18 @@ function isTransientIndexLockContention(error) {
|
|
|
3523
3558
|
if (!isIndexLockContentionError(error) || !("reason" in error)) return false;
|
|
3524
3559
|
return error.reason === "active" || error.reason === "reclaiming";
|
|
3525
3560
|
}
|
|
3526
|
-
function acquireIndexLock(indexPath, operation) {
|
|
3561
|
+
function acquireIndexLock(indexPath, operation, recoveryScope) {
|
|
3527
3562
|
(0, import_fs5.mkdirSync)(indexPath, { recursive: true });
|
|
3528
3563
|
const canonicalIndexPath = import_fs5.realpathSync.native(indexPath);
|
|
3529
3564
|
const lockPath = path9.join(canonicalIndexPath, "indexing.lock");
|
|
3530
3565
|
cleanupDeadPublicationCandidates(canonicalIndexPath);
|
|
3531
3566
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
3532
|
-
const owner = createOwner(operation)
|
|
3567
|
+
const owner = recoveryScope === void 0 ? createOwner(operation) : {
|
|
3568
|
+
...createOwner(operation),
|
|
3569
|
+
recoveryProtocolVersion: 1,
|
|
3570
|
+
projectRoot: recoveryScope.projectRoot,
|
|
3571
|
+
scopedRoots: recoveryScope.scopedRoots
|
|
3572
|
+
};
|
|
3533
3573
|
if (publishJsonDirectory(lockPath, owner)) {
|
|
3534
3574
|
const lease = {
|
|
3535
3575
|
canonicalIndexPath,
|
|
@@ -3594,6 +3634,33 @@ function releaseIndexLock(lease) {
|
|
|
3594
3634
|
}
|
|
3595
3635
|
return true;
|
|
3596
3636
|
}
|
|
3637
|
+
function setIndexLockClearRecoveryState(lease, clearRecovery) {
|
|
3638
|
+
const currentOwner = readDirectoryOwner(lease.lockPath);
|
|
3639
|
+
if (!currentOwner || !sameOwner(currentOwner, lease.owner)) {
|
|
3640
|
+
throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
3641
|
+
}
|
|
3642
|
+
const nextOwner = { ...currentOwner };
|
|
3643
|
+
if (clearRecovery === null) {
|
|
3644
|
+
delete nextOwner.clearRecovery;
|
|
3645
|
+
} else {
|
|
3646
|
+
nextOwner.clearRecovery = clearRecovery;
|
|
3647
|
+
}
|
|
3648
|
+
const ownerPath = path9.join(lease.lockPath, OWNER_FILE_NAME);
|
|
3649
|
+
const temporaryPath = path9.join(
|
|
3650
|
+
lease.lockPath,
|
|
3651
|
+
`${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${(0, import_crypto.randomUUID)()}`
|
|
3652
|
+
);
|
|
3653
|
+
try {
|
|
3654
|
+
(0, import_fs5.writeFileSync)(temporaryPath, JSON.stringify(nextOwner), {
|
|
3655
|
+
encoding: "utf-8",
|
|
3656
|
+
flag: "wx",
|
|
3657
|
+
mode: 384
|
|
3658
|
+
});
|
|
3659
|
+
retryTransientFilesystemOperation(() => (0, import_fs5.renameSync)(temporaryPath, ownerPath));
|
|
3660
|
+
} finally {
|
|
3661
|
+
if ((0, import_fs5.existsSync)(temporaryPath)) (0, import_fs5.rmSync)(temporaryPath, { force: true });
|
|
3662
|
+
}
|
|
3663
|
+
}
|
|
3597
3664
|
function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
|
|
3598
3665
|
if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
|
|
3599
3666
|
temporaryCounter += 1;
|
|
@@ -5737,8 +5804,6 @@ async function tryDetectProvider() {
|
|
|
5737
5804
|
}
|
|
5738
5805
|
async function getProviderCredentials(provider) {
|
|
5739
5806
|
switch (provider) {
|
|
5740
|
-
case "github-copilot":
|
|
5741
|
-
return getGitHubCopilotCredentials();
|
|
5742
5807
|
case "openai":
|
|
5743
5808
|
return getOpenAICredentials();
|
|
5744
5809
|
case "google":
|
|
@@ -5749,22 +5814,6 @@ async function getProviderCredentials(provider) {
|
|
|
5749
5814
|
return null;
|
|
5750
5815
|
}
|
|
5751
5816
|
}
|
|
5752
|
-
function getGitHubCopilotCredentials() {
|
|
5753
|
-
const authData = loadOpenCodeAuth();
|
|
5754
|
-
const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
|
|
5755
|
-
if (!copilotAuth || copilotAuth.type !== "oauth") {
|
|
5756
|
-
return null;
|
|
5757
|
-
}
|
|
5758
|
-
const auth = copilotAuth;
|
|
5759
|
-
const baseUrl = auth.enterpriseUrl ? `https://copilot-api.${auth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
|
|
5760
|
-
return {
|
|
5761
|
-
provider: "github-copilot",
|
|
5762
|
-
baseUrl,
|
|
5763
|
-
refreshToken: copilotAuth.refresh,
|
|
5764
|
-
accessToken: copilotAuth.access,
|
|
5765
|
-
tokenExpires: copilotAuth.expires
|
|
5766
|
-
};
|
|
5767
|
-
}
|
|
5768
5817
|
function getOpenAICredentials() {
|
|
5769
5818
|
const authData = loadOpenCodeAuth();
|
|
5770
5819
|
const openaiAuth = authData["openai"];
|
|
@@ -5890,8 +5939,6 @@ async function tryDetectOllamaProvider() {
|
|
|
5890
5939
|
}
|
|
5891
5940
|
function getProviderDisplayName(provider) {
|
|
5892
5941
|
switch (provider) {
|
|
5893
|
-
case "github-copilot":
|
|
5894
|
-
return "GitHub Copilot";
|
|
5895
5942
|
case "openai":
|
|
5896
5943
|
return "OpenAI";
|
|
5897
5944
|
case "google":
|
|
@@ -6116,44 +6163,6 @@ var CustomEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
6116
6163
|
}
|
|
6117
6164
|
};
|
|
6118
6165
|
|
|
6119
|
-
// src/embeddings/providers/github-copilot.ts
|
|
6120
|
-
var GitHubCopilotEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
6121
|
-
constructor(credentials, modelInfo) {
|
|
6122
|
-
super(credentials, modelInfo);
|
|
6123
|
-
}
|
|
6124
|
-
getToken() {
|
|
6125
|
-
if (!this.credentials.refreshToken) {
|
|
6126
|
-
throw new Error("No OAuth token available for GitHub");
|
|
6127
|
-
}
|
|
6128
|
-
return this.credentials.refreshToken;
|
|
6129
|
-
}
|
|
6130
|
-
async embedBatch(texts) {
|
|
6131
|
-
const token = this.getToken();
|
|
6132
|
-
const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
|
|
6133
|
-
method: "POST",
|
|
6134
|
-
headers: {
|
|
6135
|
-
Authorization: `Bearer ${token}`,
|
|
6136
|
-
"Content-Type": "application/json",
|
|
6137
|
-
Accept: "application/vnd.github+json",
|
|
6138
|
-
"X-GitHub-Api-Version": "2022-11-28"
|
|
6139
|
-
},
|
|
6140
|
-
body: JSON.stringify({
|
|
6141
|
-
model: `openai/${this.modelInfo.model}`,
|
|
6142
|
-
input: texts
|
|
6143
|
-
})
|
|
6144
|
-
});
|
|
6145
|
-
if (!response.ok) {
|
|
6146
|
-
const error = (await response.text()).slice(0, 500);
|
|
6147
|
-
throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
|
|
6148
|
-
}
|
|
6149
|
-
const data = await response.json();
|
|
6150
|
-
return {
|
|
6151
|
-
embeddings: data.data.map((d) => d.embedding),
|
|
6152
|
-
totalTokensUsed: data.usage.total_tokens
|
|
6153
|
-
};
|
|
6154
|
-
}
|
|
6155
|
-
};
|
|
6156
|
-
|
|
6157
6166
|
// src/embeddings/providers/google.ts
|
|
6158
6167
|
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddingProvider {
|
|
6159
6168
|
static BATCH_SIZE = 20;
|
|
@@ -6161,24 +6170,30 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
6161
6170
|
super(credentials, modelInfo);
|
|
6162
6171
|
}
|
|
6163
6172
|
async embedQuery(query) {
|
|
6164
|
-
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
6165
|
-
const
|
|
6173
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
6174
|
+
const texts = [
|
|
6175
|
+
this.modelInfo.model === "gemini-embedding-2" ? `task: code retrieval | query: ${query}` : query
|
|
6176
|
+
];
|
|
6177
|
+
const result = await this.embedWithTaskType(texts, taskType);
|
|
6166
6178
|
return {
|
|
6167
6179
|
embedding: result.embeddings[0],
|
|
6168
6180
|
tokensUsed: result.totalTokensUsed
|
|
6169
6181
|
};
|
|
6170
6182
|
}
|
|
6171
6183
|
async embedDocument(document) {
|
|
6172
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6173
|
-
const result = await this.embedWithTaskType([
|
|
6184
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6185
|
+
const result = await this.embedWithTaskType([
|
|
6186
|
+
this.modelInfo.model === "gemini-embedding-2" ? `title: none | text: ${document}` : document
|
|
6187
|
+
], taskType);
|
|
6174
6188
|
return {
|
|
6175
6189
|
embedding: result.embeddings[0],
|
|
6176
6190
|
tokensUsed: result.totalTokensUsed
|
|
6177
6191
|
};
|
|
6178
6192
|
}
|
|
6179
6193
|
async embedBatch(texts) {
|
|
6180
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6181
|
-
|
|
6194
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6195
|
+
const formattedTexts = this.modelInfo.model === "gemini-embedding-2" ? texts.map((text) => `title: none | text: ${text}`) : texts;
|
|
6196
|
+
return this.embedWithTaskType(formattedTexts, taskType);
|
|
6182
6197
|
}
|
|
6183
6198
|
async embedWithTaskType(texts, taskType) {
|
|
6184
6199
|
const batches = [];
|
|
@@ -6228,6 +6243,10 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
6228
6243
|
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddingProvider {
|
|
6229
6244
|
static MIN_TRUNCATION_CHARS = 512;
|
|
6230
6245
|
static REQUEST_TIMEOUT_MS = 12e4;
|
|
6246
|
+
// Set when /api/embed returns 404 so subsequent multi-text batches skip the
|
|
6247
|
+
// batched endpoint and go straight to the legacy per-text path (one probe per
|
|
6248
|
+
// old ollama install, not one probe per batch).
|
|
6249
|
+
batchEndpointUnavailable = false;
|
|
6231
6250
|
constructor(credentials, modelInfo) {
|
|
6232
6251
|
super(credentials, modelInfo);
|
|
6233
6252
|
}
|
|
@@ -6245,6 +6264,21 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
6245
6264
|
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
6246
6265
|
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
6247
6266
|
}
|
|
6267
|
+
// True for a 404 from the newer /api/embed endpoint, i.e. an ollama version that
|
|
6268
|
+
// does not provide it. embedBatch uses this to fall back to the legacy per-text
|
|
6269
|
+
// /api/embeddings path so old ollama installs do not regress.
|
|
6270
|
+
isBatchEndpointUnavailableError(error) {
|
|
6271
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6272
|
+
return message.includes("Ollama /api/embed not available");
|
|
6273
|
+
}
|
|
6274
|
+
// True for a malformed /api/embed response (wrong vector count or a bad vector).
|
|
6275
|
+
// embedBatch falls back to the per-text path on this so a bad batch response
|
|
6276
|
+
// re-embeds each text cleanly. A text that then fails per-text is not isolated
|
|
6277
|
+
// here; it is isolated on the recovery run, which re-embeds one text per request.
|
|
6278
|
+
isBatchValidationError(error) {
|
|
6279
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6280
|
+
return message.includes("invalid embedding batch");
|
|
6281
|
+
}
|
|
6248
6282
|
buildTruncationCandidates(text) {
|
|
6249
6283
|
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
6250
6284
|
const candidateLimits = /* @__PURE__ */ new Set();
|
|
@@ -6346,7 +6380,74 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
6346
6380
|
tokensUsed: this.estimateTokens(text)
|
|
6347
6381
|
};
|
|
6348
6382
|
}
|
|
6349
|
-
|
|
6383
|
+
// Embeds many texts in one POST /api/embed request (input: string[]). Ollama
|
|
6384
|
+
// encodes each input independently, so the model context length applies per input
|
|
6385
|
+
// (the upstream splitter already bounds each input), not over the batch. This
|
|
6386
|
+
// amortizes N HTTP round-trips into one.
|
|
6387
|
+
async embedMany(texts) {
|
|
6388
|
+
const controller = new AbortController();
|
|
6389
|
+
const timeout = setTimeout(
|
|
6390
|
+
() => controller.abort(),
|
|
6391
|
+
_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS
|
|
6392
|
+
);
|
|
6393
|
+
let response;
|
|
6394
|
+
try {
|
|
6395
|
+
response = await fetch(`${this.credentials.baseUrl}/api/embed`, {
|
|
6396
|
+
method: "POST",
|
|
6397
|
+
headers: {
|
|
6398
|
+
"Content-Type": "application/json"
|
|
6399
|
+
},
|
|
6400
|
+
body: JSON.stringify({
|
|
6401
|
+
model: this.modelInfo.model,
|
|
6402
|
+
input: texts,
|
|
6403
|
+
truncate: false
|
|
6404
|
+
}),
|
|
6405
|
+
signal: controller.signal
|
|
6406
|
+
});
|
|
6407
|
+
} catch (error) {
|
|
6408
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
6409
|
+
throw new Error(
|
|
6410
|
+
`Ollama embedding request timed out after ${_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS}ms`
|
|
6411
|
+
);
|
|
6412
|
+
}
|
|
6413
|
+
throw error;
|
|
6414
|
+
} finally {
|
|
6415
|
+
clearTimeout(timeout);
|
|
6416
|
+
}
|
|
6417
|
+
if (!response.ok) {
|
|
6418
|
+
const error = (await response.text()).slice(0, 500);
|
|
6419
|
+
if (response.status === 404) {
|
|
6420
|
+
throw new Error(`Ollama /api/embed not available: ${response.status} - ${error}`);
|
|
6421
|
+
}
|
|
6422
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
6423
|
+
}
|
|
6424
|
+
let parsed;
|
|
6425
|
+
try {
|
|
6426
|
+
parsed = await response.json();
|
|
6427
|
+
} catch {
|
|
6428
|
+
throw new Error(
|
|
6429
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
6430
|
+
);
|
|
6431
|
+
}
|
|
6432
|
+
const data = parsed && typeof parsed === "object" ? parsed : {};
|
|
6433
|
+
if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length || data.embeddings.some(
|
|
6434
|
+
(value) => !Array.isArray(value) || value.length !== this.modelInfo.dimensions || value.some((v) => typeof v !== "number" || !Number.isFinite(v))
|
|
6435
|
+
)) {
|
|
6436
|
+
throw new Error(
|
|
6437
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
6438
|
+
);
|
|
6439
|
+
}
|
|
6440
|
+
return {
|
|
6441
|
+
embeddings: data.embeddings,
|
|
6442
|
+
totalTokensUsed: texts.reduce((sum, text) => sum + this.estimateTokens(text), 0)
|
|
6443
|
+
};
|
|
6444
|
+
}
|
|
6445
|
+
// Per-text /api/embeddings path shared by the single-text fast path and the
|
|
6446
|
+
// batch fallback. Uses the legacy endpoint one text at a time, so each text gets
|
|
6447
|
+
// its own truncation safety net and a vector validated on its own. A text that
|
|
6448
|
+
// hard-fails per-text throws here and fails the whole request batch; the recovery
|
|
6449
|
+
// run re-embeds one text per request to isolate it.
|
|
6450
|
+
async embedOneByOne(texts) {
|
|
6350
6451
|
const results = [];
|
|
6351
6452
|
for (const text of texts) {
|
|
6352
6453
|
results.push(await this.embedSingleWithFallback(text));
|
|
@@ -6356,6 +6457,26 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
6356
6457
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
6357
6458
|
};
|
|
6358
6459
|
}
|
|
6460
|
+
async embedBatch(texts) {
|
|
6461
|
+
if (texts.length === 0) {
|
|
6462
|
+
return { embeddings: [], totalTokensUsed: 0 };
|
|
6463
|
+
}
|
|
6464
|
+
if (texts.length === 1 || this.batchEndpointUnavailable) {
|
|
6465
|
+
return this.embedOneByOne(texts);
|
|
6466
|
+
}
|
|
6467
|
+
try {
|
|
6468
|
+
return await this.embedMany(texts);
|
|
6469
|
+
} catch (error) {
|
|
6470
|
+
if (this.isBatchEndpointUnavailableError(error)) {
|
|
6471
|
+
this.batchEndpointUnavailable = true;
|
|
6472
|
+
return this.embedOneByOne(texts);
|
|
6473
|
+
}
|
|
6474
|
+
if (!this.isContextLengthError(error) && !this.isBatchValidationError(error)) {
|
|
6475
|
+
throw error;
|
|
6476
|
+
}
|
|
6477
|
+
return this.embedOneByOne(texts);
|
|
6478
|
+
}
|
|
6479
|
+
}
|
|
6359
6480
|
};
|
|
6360
6481
|
|
|
6361
6482
|
// src/embeddings/providers/openai.ts
|
|
@@ -6390,8 +6511,6 @@ var OpenAIEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
6390
6511
|
// src/embeddings/provider.ts
|
|
6391
6512
|
function createEmbeddingProvider(configuredProviderInfo) {
|
|
6392
6513
|
switch (configuredProviderInfo.provider) {
|
|
6393
|
-
case "github-copilot":
|
|
6394
|
-
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
6395
6514
|
case "openai":
|
|
6396
6515
|
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
6397
6516
|
case "google":
|
|
@@ -6459,6 +6578,26 @@ function formatCostEstimate(estimate) {
|
|
|
6459
6578
|
\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
|
|
6460
6579
|
`;
|
|
6461
6580
|
}
|
|
6581
|
+
function formatDryRunEstimate(estimate) {
|
|
6582
|
+
return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed.
|
|
6583
|
+
|
|
6584
|
+
Files to embed: ${estimate.filesCount.toLocaleString()}
|
|
6585
|
+
Chunks to embed: ${estimate.chunksCount.toLocaleString()}
|
|
6586
|
+
Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()}
|
|
6587
|
+
|
|
6588
|
+
The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It
|
|
6589
|
+
matches the live "Tokens used" counter only for providers that report usage on the
|
|
6590
|
+
same basis (ollama); for providers that report a server tokenizer count (OpenAI,
|
|
6591
|
+
Gemini, custom) it is only an estimate.
|
|
6592
|
+
|
|
6593
|
+
For a matching provider and a project-scoped force index, the force pass clears its
|
|
6594
|
+
own cached embeddings, so the live counter climbs to this number. A force index on a
|
|
6595
|
+
shared global index can reuse cached embeddings from other projects, and an
|
|
6596
|
+
incremental index counts cached chunks that are not re-embedded; in both cases this
|
|
6597
|
+
number is an upper bound on the live counter, so a progress percent against this
|
|
6598
|
+
total tops out below 100%.
|
|
6599
|
+
`;
|
|
6600
|
+
}
|
|
6462
6601
|
function formatBytes(bytes) {
|
|
6463
6602
|
if (bytes === 0) return "0 B";
|
|
6464
6603
|
const k = 1024;
|
|
@@ -7143,12 +7282,12 @@ try {
|
|
|
7143
7282
|
}
|
|
7144
7283
|
|
|
7145
7284
|
// src/native/parsing.ts
|
|
7146
|
-
function parseFileAsText(filePath, content) {
|
|
7147
|
-
const result = native.parseFileAsText(filePath, content);
|
|
7285
|
+
function parseFileAsText(filePath, content, linesPerChunk) {
|
|
7286
|
+
const result = native.parseFileAsText(filePath, content, linesPerChunk);
|
|
7148
7287
|
return result.map(mapChunk);
|
|
7149
7288
|
}
|
|
7150
|
-
function parseFiles(files) {
|
|
7151
|
-
const result = native.parseFiles(files);
|
|
7289
|
+
function parseFiles(files, linesPerChunk) {
|
|
7290
|
+
const result = native.parseFiles(files, linesPerChunk);
|
|
7152
7291
|
return result.map((f) => ({
|
|
7153
7292
|
path: f.path,
|
|
7154
7293
|
chunks: f.chunks.map(mapChunk),
|
|
@@ -7225,13 +7364,13 @@ var VectorStore = class {
|
|
|
7225
7364
|
const metadata = items.map((i) => JSON.stringify(i.metadata));
|
|
7226
7365
|
this.inner.addBatch(ids, vectors, metadata);
|
|
7227
7366
|
}
|
|
7228
|
-
search(queryVector, limit = 10) {
|
|
7367
|
+
search(queryVector, limit = 10, allowedIds) {
|
|
7229
7368
|
if (queryVector.length !== this.dimensions) {
|
|
7230
7369
|
throw new Error(
|
|
7231
7370
|
`Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
|
|
7232
7371
|
);
|
|
7233
7372
|
}
|
|
7234
|
-
const results = this.inner.search(queryVector, limit);
|
|
7373
|
+
const results = allowedIds === void 0 ? this.inner.search(queryVector, limit) : this.inner.searchFiltered(queryVector, limit, allowedIds);
|
|
7235
7374
|
return results.map((r) => ({
|
|
7236
7375
|
id: r.id,
|
|
7237
7376
|
score: r.score,
|
|
@@ -7459,6 +7598,10 @@ var Database = class _Database {
|
|
|
7459
7598
|
this.throwIfClosed();
|
|
7460
7599
|
return this.inner.getBranchChunkIds(branch);
|
|
7461
7600
|
}
|
|
7601
|
+
getChunkIdsByBlameDate(since, until) {
|
|
7602
|
+
this.throwIfClosed();
|
|
7603
|
+
return this.inner.getChunkIdsByBlameDate(since, until);
|
|
7604
|
+
}
|
|
7462
7605
|
getBranchDelta(branch, baseBranch) {
|
|
7463
7606
|
this.throwIfClosed();
|
|
7464
7607
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
@@ -8730,6 +8873,9 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
|
8730
8873
|
"enum_declaration",
|
|
8731
8874
|
"function_definition",
|
|
8732
8875
|
"class_definition",
|
|
8876
|
+
// Ruby module/class symbols that are declaration-bearing and navigable.
|
|
8877
|
+
"class",
|
|
8878
|
+
"module",
|
|
8733
8879
|
"class_specifier",
|
|
8734
8880
|
"struct_specifier",
|
|
8735
8881
|
"namespace_definition",
|
|
@@ -9347,6 +9493,18 @@ function createFailedBatchWriter(targetPath) {
|
|
|
9347
9493
|
temporaryPath
|
|
9348
9494
|
};
|
|
9349
9495
|
}
|
|
9496
|
+
function writeFailedBatchRecords(targetPath, records) {
|
|
9497
|
+
const writer = createFailedBatchWriter(targetPath);
|
|
9498
|
+
try {
|
|
9499
|
+
for (const record of records) {
|
|
9500
|
+
writer.write(record);
|
|
9501
|
+
}
|
|
9502
|
+
writer.commit();
|
|
9503
|
+
} catch (error) {
|
|
9504
|
+
writer.cleanup();
|
|
9505
|
+
throw error;
|
|
9506
|
+
}
|
|
9507
|
+
}
|
|
9350
9508
|
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
9351
9509
|
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
9352
9510
|
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
@@ -9629,14 +9787,18 @@ function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
|
9629
9787
|
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
9630
9788
|
return Math.min(2e3, maxChunkTokens);
|
|
9631
9789
|
}
|
|
9632
|
-
|
|
9633
|
-
|
|
9634
|
-
|
|
9635
|
-
|
|
9636
|
-
|
|
9637
|
-
};
|
|
9790
|
+
var DEFAULT_OLLAMA_MAX_BATCH_ITEMS = 16;
|
|
9791
|
+
var DEFAULT_OLLAMA_MAX_BATCH_TOKENS = 65536;
|
|
9792
|
+
function getDynamicBatchOptions(provider, embeddingBatch) {
|
|
9793
|
+
if (provider.provider !== "ollama") {
|
|
9794
|
+
return {};
|
|
9638
9795
|
}
|
|
9639
|
-
|
|
9796
|
+
const base = { maxBatchTokens: DEFAULT_OLLAMA_MAX_BATCH_TOKENS, maxBatchItems: DEFAULT_OLLAMA_MAX_BATCH_ITEMS };
|
|
9797
|
+
return {
|
|
9798
|
+
...base,
|
|
9799
|
+
...typeof embeddingBatch?.maxBatchTokens === "number" && Number.isFinite(embeddingBatch.maxBatchTokens) ? { maxBatchTokens: embeddingBatch.maxBatchTokens } : {},
|
|
9800
|
+
...typeof embeddingBatch?.maxBatchItems === "number" && Number.isFinite(embeddingBatch.maxBatchItems) ? { maxBatchItems: embeddingBatch.maxBatchItems } : {}
|
|
9801
|
+
};
|
|
9640
9802
|
}
|
|
9641
9803
|
function isSqliteCorruptionError(error) {
|
|
9642
9804
|
const message = getErrorMessage4(error).toLowerCase();
|
|
@@ -9654,6 +9816,14 @@ function getPendingChunkId(rawChunk) {
|
|
|
9654
9816
|
const id = rawChunk.id;
|
|
9655
9817
|
return typeof id === "string" ? id : null;
|
|
9656
9818
|
}
|
|
9819
|
+
function parseBlameTimestamp(value, endOfDay) {
|
|
9820
|
+
let timestampMs = Date.parse(value);
|
|
9821
|
+
if (Number.isNaN(timestampMs)) return null;
|
|
9822
|
+
if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(value.trim())) {
|
|
9823
|
+
timestampMs += 24 * 60 * 60 * 1e3 - 1;
|
|
9824
|
+
}
|
|
9825
|
+
return Math.floor(timestampMs / 1e3);
|
|
9826
|
+
}
|
|
9657
9827
|
function metadataFromBlame(blame) {
|
|
9658
9828
|
if (!blame) {
|
|
9659
9829
|
return {};
|
|
@@ -9800,7 +9970,7 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
|
|
|
9800
9970
|
const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
|
|
9801
9971
|
return [...promoted, ...remainder];
|
|
9802
9972
|
}
|
|
9803
|
-
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
9973
|
+
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source", allowNonSourcePaths = false) {
|
|
9804
9974
|
if (!prioritizeSourcePaths) {
|
|
9805
9975
|
return [];
|
|
9806
9976
|
}
|
|
@@ -9820,7 +9990,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
9820
9990
|
if (!isImplementationChunkType(chunkType)) {
|
|
9821
9991
|
return false;
|
|
9822
9992
|
}
|
|
9823
|
-
if (!isLikelyImplementationPath2(chunk.filePath)) {
|
|
9993
|
+
if (!allowNonSourcePaths && !isLikelyImplementationPath2(chunk.filePath)) {
|
|
9824
9994
|
return false;
|
|
9825
9995
|
}
|
|
9826
9996
|
const nameLower = (chunk.name ?? "").toLowerCase();
|
|
@@ -9884,7 +10054,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
9884
10054
|
}
|
|
9885
10055
|
foundCoveringChunk = upsertChunkCandidate(chunk, identifier, normalizedIdentifier) || foundCoveringChunk;
|
|
9886
10056
|
}
|
|
9887
|
-
if (foundCoveringChunk || !isLikelyImplementationPath2(symbol.filePath)) {
|
|
10057
|
+
if (foundCoveringChunk || !allowNonSourcePaths && !isLikelyImplementationPath2(symbol.filePath)) {
|
|
9888
10058
|
continue;
|
|
9889
10059
|
}
|
|
9890
10060
|
const symbolName = symbol.name.toLowerCase();
|
|
@@ -9938,7 +10108,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
9938
10108
|
const ranked = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
9939
10109
|
if (ranked.length === 0) {
|
|
9940
10110
|
const implementationFallback = fallbackCandidates.filter(
|
|
9941
|
-
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && isLikelyImplementationPath2(candidate.metadata.filePath)
|
|
10111
|
+
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && (allowNonSourcePaths || isLikelyImplementationPath2(candidate.metadata.filePath))
|
|
9942
10112
|
);
|
|
9943
10113
|
for (const candidate of implementationFallback) {
|
|
9944
10114
|
const nameLower = (candidate.metadata.name ?? "").toLowerCase();
|
|
@@ -10054,10 +10224,16 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
10054
10224
|
return false;
|
|
10055
10225
|
}
|
|
10056
10226
|
if (options?.blameSince) {
|
|
10057
|
-
const
|
|
10058
|
-
if (
|
|
10227
|
+
const since = parseBlameTimestamp(options.blameSince, false);
|
|
10228
|
+
if (since === null) return false;
|
|
10059
10229
|
const committedAt = candidate.metadata.blameCommittedAt;
|
|
10060
|
-
if (committedAt === void 0 || committedAt <
|
|
10230
|
+
if (committedAt === void 0 || committedAt < since) return false;
|
|
10231
|
+
}
|
|
10232
|
+
if (options?.blameUntil) {
|
|
10233
|
+
const until = parseBlameTimestamp(options.blameUntil, true);
|
|
10234
|
+
if (until === null) return false;
|
|
10235
|
+
const committedAt = candidate.metadata.blameCommittedAt;
|
|
10236
|
+
if (committedAt === void 0 || committedAt > until) return false;
|
|
10061
10237
|
}
|
|
10062
10238
|
return true;
|
|
10063
10239
|
}
|
|
@@ -10113,9 +10289,10 @@ var Indexer = class _Indexer {
|
|
|
10113
10289
|
writerArtifactFingerprint = null;
|
|
10114
10290
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
10115
10291
|
fileBatchLimits;
|
|
10292
|
+
checkpointIntervalChunks;
|
|
10116
10293
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
10117
10294
|
this.projectRoot = projectRoot;
|
|
10118
|
-
this.projectIdentityHash =
|
|
10295
|
+
this.projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
10119
10296
|
this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot;
|
|
10120
10297
|
this.branchNameOverride = runtimeOptions.branchName;
|
|
10121
10298
|
this.catalogIdentityOverride = runtimeOptions.catalogIdentity;
|
|
@@ -10125,6 +10302,7 @@ var Indexer = class _Indexer {
|
|
|
10125
10302
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
10126
10303
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
10127
10304
|
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
10305
|
+
this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks;
|
|
10128
10306
|
this.config = config;
|
|
10129
10307
|
this.host = host;
|
|
10130
10308
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -10236,6 +10414,9 @@ var Indexer = class _Indexer {
|
|
|
10236
10414
|
return path19.resolve(targetPath);
|
|
10237
10415
|
}
|
|
10238
10416
|
}
|
|
10417
|
+
getProjectIdentityHash(projectRoot) {
|
|
10418
|
+
return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
10419
|
+
}
|
|
10239
10420
|
isProjectOwnedIndexPath() {
|
|
10240
10421
|
return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host);
|
|
10241
10422
|
}
|
|
@@ -10272,7 +10453,10 @@ var Indexer = class _Indexer {
|
|
|
10272
10453
|
}
|
|
10273
10454
|
async withIndexMutationLease(operation, callback) {
|
|
10274
10455
|
this.refreshBranchInfo();
|
|
10275
|
-
const lease = acquireIndexLock(this.indexPath, operation
|
|
10456
|
+
const lease = acquireIndexLock(this.indexPath, operation, {
|
|
10457
|
+
projectRoot: this.projectRoot,
|
|
10458
|
+
scopedRoots: this.getScopedRoots()
|
|
10459
|
+
});
|
|
10276
10460
|
this.indexPath = lease.canonicalIndexPath;
|
|
10277
10461
|
this.refreshRuntimeArtifactPaths();
|
|
10278
10462
|
this.activeIndexLease = lease;
|
|
@@ -10327,6 +10511,7 @@ var Indexer = class _Indexer {
|
|
|
10327
10511
|
}
|
|
10328
10512
|
loadFileHashCache() {
|
|
10329
10513
|
if (!(0, import_fs12.existsSync)(this.fileHashCachePath)) {
|
|
10514
|
+
this.fileHashCache = /* @__PURE__ */ new Map();
|
|
10330
10515
|
return;
|
|
10331
10516
|
}
|
|
10332
10517
|
try {
|
|
@@ -10366,10 +10551,10 @@ var Indexer = class _Indexer {
|
|
|
10366
10551
|
invertedIndex.serialize()
|
|
10367
10552
|
);
|
|
10368
10553
|
}
|
|
10369
|
-
getScopedRoots() {
|
|
10370
|
-
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(
|
|
10554
|
+
getScopedRoots(projectRoot = this.projectRoot) {
|
|
10555
|
+
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(projectRoot)]);
|
|
10371
10556
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
10372
|
-
roots.add(this.getCanonicalPath(path19.resolve(
|
|
10557
|
+
roots.add(this.getCanonicalPath(path19.resolve(projectRoot, kbRoot)));
|
|
10373
10558
|
}
|
|
10374
10559
|
return Array.from(roots);
|
|
10375
10560
|
}
|
|
@@ -10440,14 +10625,17 @@ var Indexer = class _Indexer {
|
|
|
10440
10625
|
getLegacyBranchCatalogKey() {
|
|
10441
10626
|
return this.currentBranch || "default";
|
|
10442
10627
|
}
|
|
10443
|
-
getLegacyMigrationMetadataKey() {
|
|
10444
|
-
return `index.globalBranchMigration.${
|
|
10628
|
+
getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10629
|
+
return `index.globalBranchMigration.${projectIdentityHash}`;
|
|
10630
|
+
}
|
|
10631
|
+
getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10632
|
+
return `index.embeddingStrategyVersion.${projectIdentityHash}`;
|
|
10445
10633
|
}
|
|
10446
|
-
|
|
10447
|
-
return `index.
|
|
10634
|
+
getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10635
|
+
return `index.forceReembed.${projectIdentityHash}`;
|
|
10448
10636
|
}
|
|
10449
|
-
|
|
10450
|
-
return `index.
|
|
10637
|
+
getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10638
|
+
return `index.migrationFinalized.${projectIdentityHash}`;
|
|
10451
10639
|
}
|
|
10452
10640
|
getBranchMigrationMetadataKey(prefix, catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
10453
10641
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
@@ -10553,7 +10741,7 @@ var Indexer = class _Indexer {
|
|
|
10553
10741
|
const legacy = this.getLegacyBranchCatalogKey();
|
|
10554
10742
|
return primary === legacy ? [primary] : [primary, legacy];
|
|
10555
10743
|
}
|
|
10556
|
-
getProjectLocalScopedOwnershipIds(roots) {
|
|
10744
|
+
getProjectLocalScopedOwnershipIds(roots, projectRoot = this.projectRoot) {
|
|
10557
10745
|
const chunkIds = /* @__PURE__ */ new Set();
|
|
10558
10746
|
const symbolIds = /* @__PURE__ */ new Set();
|
|
10559
10747
|
if (!this.database) {
|
|
@@ -10561,10 +10749,10 @@ var Indexer = class _Indexer {
|
|
|
10561
10749
|
}
|
|
10562
10750
|
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
10563
10751
|
...Array.from(this.fileHashCache.keys()).filter(
|
|
10564
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
10752
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
10565
10753
|
),
|
|
10566
10754
|
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
10567
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
10755
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
10568
10756
|
)
|
|
10569
10757
|
]);
|
|
10570
10758
|
for (const filePath of projectLocalFilePaths) {
|
|
@@ -10577,15 +10765,16 @@ var Indexer = class _Indexer {
|
|
|
10577
10765
|
}
|
|
10578
10766
|
return { chunkIds, symbolIds };
|
|
10579
10767
|
}
|
|
10580
|
-
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
10768
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds, projectRoot = this.projectRoot) {
|
|
10581
10769
|
if (this.config.scope !== "global") {
|
|
10582
10770
|
return this.getBranchCatalogCleanupKeys();
|
|
10583
10771
|
}
|
|
10584
10772
|
const keys = /* @__PURE__ */ new Set();
|
|
10585
10773
|
const projectChunkIdSet = new Set(projectChunkIds);
|
|
10586
10774
|
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
10775
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
10587
10776
|
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
10588
|
-
if (branchKey.startsWith(`${
|
|
10777
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
10589
10778
|
keys.add(branchKey);
|
|
10590
10779
|
continue;
|
|
10591
10780
|
}
|
|
@@ -10595,8 +10784,10 @@ var Indexer = class _Indexer {
|
|
|
10595
10784
|
keys.add(branchKey);
|
|
10596
10785
|
}
|
|
10597
10786
|
}
|
|
10598
|
-
|
|
10599
|
-
|
|
10787
|
+
if (projectRoot === this.projectRoot) {
|
|
10788
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
10789
|
+
keys.add(branchKey);
|
|
10790
|
+
}
|
|
10600
10791
|
}
|
|
10601
10792
|
return Array.from(keys);
|
|
10602
10793
|
}
|
|
@@ -10604,10 +10795,10 @@ var Indexer = class _Indexer {
|
|
|
10604
10795
|
const canonicalFilePath = this.getCanonicalStoredFilePath(filePath);
|
|
10605
10796
|
return roots.some((root) => isPathWithinRoot2(canonicalFilePath, root));
|
|
10606
10797
|
}
|
|
10607
|
-
isFileInProjectRoot(filePath) {
|
|
10798
|
+
isFileInProjectRoot(filePath, projectRoot = this.projectRoot) {
|
|
10608
10799
|
return isPathWithinRoot2(
|
|
10609
10800
|
this.getCanonicalStoredFilePath(filePath),
|
|
10610
|
-
this.getCanonicalPath(
|
|
10801
|
+
this.getCanonicalPath(projectRoot)
|
|
10611
10802
|
);
|
|
10612
10803
|
}
|
|
10613
10804
|
clearScopedFileHashCache(roots) {
|
|
@@ -10649,12 +10840,12 @@ var Indexer = class _Indexer {
|
|
|
10649
10840
|
}
|
|
10650
10841
|
return false;
|
|
10651
10842
|
}
|
|
10652
|
-
hasForeignScopedBranchData() {
|
|
10843
|
+
hasForeignScopedBranchData(projectRoot = this.projectRoot, roots = this.getScopedRoots(projectRoot)) {
|
|
10653
10844
|
if (!this.database || this.config.scope !== "global") {
|
|
10654
10845
|
return false;
|
|
10655
10846
|
}
|
|
10656
|
-
const
|
|
10657
|
-
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
10847
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
10848
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot);
|
|
10658
10849
|
return this.database.getAllBranches().some(
|
|
10659
10850
|
(branchKey) => {
|
|
10660
10851
|
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
@@ -10663,7 +10854,7 @@ var Indexer = class _Indexer {
|
|
|
10663
10854
|
if (!hasBranchData) {
|
|
10664
10855
|
return false;
|
|
10665
10856
|
}
|
|
10666
|
-
if (branchKey.startsWith(`${
|
|
10857
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
10667
10858
|
return false;
|
|
10668
10859
|
}
|
|
10669
10860
|
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
@@ -10672,7 +10863,7 @@ var Indexer = class _Indexer {
|
|
|
10672
10863
|
}
|
|
10673
10864
|
);
|
|
10674
10865
|
}
|
|
10675
|
-
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
10866
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot = this.projectRoot) {
|
|
10676
10867
|
const allMetadata = store.getAllMetadata();
|
|
10677
10868
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
10678
10869
|
const filePaths = /* @__PURE__ */ new Set([
|
|
@@ -10680,7 +10871,7 @@ var Indexer = class _Indexer {
|
|
|
10680
10871
|
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
10681
10872
|
]);
|
|
10682
10873
|
const projectLocalFilePaths = new Set(
|
|
10683
|
-
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath))
|
|
10874
|
+
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot))
|
|
10684
10875
|
);
|
|
10685
10876
|
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
10686
10877
|
for (const filePath of filePaths) {
|
|
@@ -10690,7 +10881,7 @@ var Indexer = class _Indexer {
|
|
|
10690
10881
|
}
|
|
10691
10882
|
const removedChunkIdList = Array.from(removedChunkIds);
|
|
10692
10883
|
const projectLocalChunkIds = new Set(
|
|
10693
|
-
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)).map(({ key }) => key)
|
|
10884
|
+
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)).map(({ key }) => key)
|
|
10694
10885
|
);
|
|
10695
10886
|
for (const filePath of projectLocalFilePaths) {
|
|
10696
10887
|
for (const chunk of database.getChunksByFile(filePath)) {
|
|
@@ -10709,7 +10900,8 @@ var Indexer = class _Indexer {
|
|
|
10709
10900
|
}
|
|
10710
10901
|
const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys(
|
|
10711
10902
|
Array.from(projectLocalChunkIds),
|
|
10712
|
-
Array.from(projectLocalSymbolIds)
|
|
10903
|
+
Array.from(projectLocalSymbolIds),
|
|
10904
|
+
projectRoot
|
|
10713
10905
|
);
|
|
10714
10906
|
for (const branchKey of branchCleanupKeys) {
|
|
10715
10907
|
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
@@ -10744,29 +10936,96 @@ var Indexer = class _Indexer {
|
|
|
10744
10936
|
database.gcOrphanSymbols();
|
|
10745
10937
|
database.gcOrphanEmbeddings();
|
|
10746
10938
|
database.gcOrphanChunks();
|
|
10747
|
-
store.save();
|
|
10748
10939
|
this.saveInvertedIndex(invertedIndex);
|
|
10940
|
+
store.save();
|
|
10749
10941
|
return {
|
|
10750
10942
|
removedChunkIds: removedChunkIdList,
|
|
10751
10943
|
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
10752
10944
|
};
|
|
10753
10945
|
}
|
|
10946
|
+
getCurrentClearRecoveryState() {
|
|
10947
|
+
if (!this.configuredProviderInfo) {
|
|
10948
|
+
throw new Error("Cannot persist clear recovery state before the embedding provider is initialized");
|
|
10949
|
+
}
|
|
10950
|
+
const compatibility = this.checkCompatibility();
|
|
10951
|
+
const compatibilityDecision = compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible";
|
|
10952
|
+
return {
|
|
10953
|
+
phase: "clearing",
|
|
10954
|
+
embeddingProvider: this.configuredProviderInfo.provider,
|
|
10955
|
+
embeddingModel: this.configuredProviderInfo.modelInfo.model,
|
|
10956
|
+
embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions,
|
|
10957
|
+
embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION,
|
|
10958
|
+
compatibilityDecision
|
|
10959
|
+
};
|
|
10960
|
+
}
|
|
10961
|
+
beginClearRecoveryState() {
|
|
10962
|
+
const recovery = this.getCurrentClearRecoveryState();
|
|
10963
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), recovery);
|
|
10964
|
+
return recovery;
|
|
10965
|
+
}
|
|
10966
|
+
finishClearRecoveryState() {
|
|
10967
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), null);
|
|
10968
|
+
}
|
|
10969
|
+
matchesCurrentClearRecoveryConfiguration(recovery) {
|
|
10970
|
+
const configuredProviderInfo = this.configuredProviderInfo;
|
|
10971
|
+
return configuredProviderInfo !== null && recovery.embeddingProvider === configuredProviderInfo.provider && recovery.embeddingModel === configuredProviderInfo.modelInfo.model && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION;
|
|
10972
|
+
}
|
|
10973
|
+
hasUnknownLegacyForceIndexClear(owner) {
|
|
10974
|
+
return owner.operation === "force-index" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1 && (0, import_fs12.existsSync)(path19.join(this.indexPath, "force-index-phase"));
|
|
10975
|
+
}
|
|
10754
10976
|
async recoverFromInterruptedIndexingUnlocked(owners) {
|
|
10755
10977
|
for (const owner of owners) {
|
|
10756
10978
|
this.logger.warn("Detected interrupted indexing session, recovering...", {
|
|
10757
10979
|
pid: owner.pid,
|
|
10758
10980
|
hostname: owner.hostname,
|
|
10759
10981
|
operation: owner.operation,
|
|
10760
|
-
startedAt: owner.startedAt
|
|
10982
|
+
startedAt: owner.startedAt,
|
|
10983
|
+
projectRoot: owner.projectRoot
|
|
10761
10984
|
});
|
|
10762
10985
|
}
|
|
10763
10986
|
if (this.config.scope === "global") {
|
|
10764
|
-
|
|
10765
|
-
|
|
10987
|
+
const clearScopes = [];
|
|
10988
|
+
for (const owner of owners) {
|
|
10989
|
+
if (this.hasUnknownLegacyForceIndexClear(owner)) {
|
|
10990
|
+
throw new Error(
|
|
10991
|
+
`Cannot automatically recover interrupted force-index ${owner.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
10992
|
+
);
|
|
10993
|
+
}
|
|
10994
|
+
if (owner.operation === "clear" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1) {
|
|
10995
|
+
throw new Error(
|
|
10996
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating recovery state is unknown. The recovery marker was retained for manual inspection.`
|
|
10997
|
+
);
|
|
10998
|
+
}
|
|
10999
|
+
if (owner.clearRecovery === void 0) continue;
|
|
11000
|
+
if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) {
|
|
11001
|
+
throw new Error(
|
|
11002
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating project scope is unknown. The recovery marker was retained for manual inspection.`
|
|
11003
|
+
);
|
|
11004
|
+
}
|
|
11005
|
+
if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) {
|
|
11006
|
+
throw new Error(
|
|
11007
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the current embedding configuration does not match the originating lease. The recovery marker was retained; retry from the originating project with matching settings.`
|
|
11008
|
+
);
|
|
11009
|
+
}
|
|
11010
|
+
clearScopes.push({
|
|
11011
|
+
projectRoot: owner.projectRoot,
|
|
11012
|
+
scopedRoots: owner.scopedRoots,
|
|
11013
|
+
compatibilityDecision: owner.clearRecovery.compatibilityDecision
|
|
11014
|
+
});
|
|
11015
|
+
}
|
|
11016
|
+
if (clearScopes.length > 0) {
|
|
11017
|
+
this.loadFileHashCache();
|
|
11018
|
+
}
|
|
11019
|
+
for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) {
|
|
11020
|
+
this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision);
|
|
10766
11021
|
}
|
|
10767
11022
|
await this.healthCheckUnlocked();
|
|
11023
|
+
this.logger.info(
|
|
11024
|
+
clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint"
|
|
11025
|
+
);
|
|
11026
|
+
return;
|
|
10768
11027
|
}
|
|
10769
|
-
this.logger.info("Recovery complete, next index will
|
|
11028
|
+
this.logger.info("Recovery complete, next index will resume from the last checkpoint");
|
|
10770
11029
|
}
|
|
10771
11030
|
*loadSerializedFailedBatches() {
|
|
10772
11031
|
let warned = false;
|
|
@@ -10804,14 +11063,99 @@ var Indexer = class _Indexer {
|
|
|
10804
11063
|
state.writer.write(record);
|
|
10805
11064
|
state.recordsWritten += record.chunks.length;
|
|
10806
11065
|
}
|
|
10807
|
-
finalizeFailedBatchWriteState(state) {
|
|
11066
|
+
finalizeFailedBatchWriteState(state, resolvedChunkIds = /* @__PURE__ */ new Set()) {
|
|
10808
11067
|
if (state.recordsWritten > 0) {
|
|
10809
|
-
|
|
11068
|
+
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
11069
|
+
const retained = [];
|
|
11070
|
+
const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath));
|
|
11071
|
+
for (let i = records.length - 1; i >= 0; i--) {
|
|
11072
|
+
const chunks = records[i].chunks.filter((rawChunk) => {
|
|
11073
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11074
|
+
if (chunkId !== null) {
|
|
11075
|
+
if (resolvedChunkIds.has(chunkId)) return false;
|
|
11076
|
+
if (seenChunkIds.has(chunkId)) return false;
|
|
11077
|
+
seenChunkIds.add(chunkId);
|
|
11078
|
+
}
|
|
11079
|
+
return true;
|
|
11080
|
+
});
|
|
11081
|
+
if (chunks.length > 0) {
|
|
11082
|
+
retained.unshift({ ...records[i], chunks });
|
|
11083
|
+
}
|
|
11084
|
+
}
|
|
11085
|
+
state.writer.cleanup();
|
|
11086
|
+
if (retained.length > 0) {
|
|
11087
|
+
writeFailedBatchRecords(this.failedBatchesPath, retained);
|
|
11088
|
+
} else {
|
|
11089
|
+
writeFailedBatchRecords(this.failedBatchesPath, []);
|
|
11090
|
+
this.clearFailedBatchState();
|
|
11091
|
+
}
|
|
10810
11092
|
return;
|
|
10811
11093
|
}
|
|
10812
|
-
state.writer.
|
|
11094
|
+
state.writer.commit();
|
|
10813
11095
|
this.clearFailedBatchState();
|
|
10814
11096
|
}
|
|
11097
|
+
getCheckpointIntervalChunks(totalChunks) {
|
|
11098
|
+
return Math.max(
|
|
11099
|
+
this.checkpointIntervalChunks ?? 2e3,
|
|
11100
|
+
Math.floor(totalChunks / 10)
|
|
11101
|
+
);
|
|
11102
|
+
}
|
|
11103
|
+
checkpointIndexRun(database, store, invertedIndex, failedProcessing, resolvedRetryChunkIds, currentFileHashes, committedFilePaths, scopedRoots, configuredProviderInfo) {
|
|
11104
|
+
if (!this.hasProjectForceReembedPending()) {
|
|
11105
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
11106
|
+
this.indexCompatibility = { compatible: true };
|
|
11107
|
+
}
|
|
11108
|
+
database.commitWriteTransaction();
|
|
11109
|
+
database.beginWriteTransaction();
|
|
11110
|
+
this.saveInvertedIndex(invertedIndex);
|
|
11111
|
+
store.save();
|
|
11112
|
+
if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0 || failedProcessing.discardedExistingRecords) {
|
|
11113
|
+
for (const metadata of failedProcessing.latestById.values()) {
|
|
11114
|
+
const alreadyMaterialized = metadata.chunks.some((rawChunk) => {
|
|
11115
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11116
|
+
return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId);
|
|
11117
|
+
});
|
|
11118
|
+
if (alreadyMaterialized) continue;
|
|
11119
|
+
this.writeFailedBatchRecord(failedProcessing.state, {
|
|
11120
|
+
chunks: metadata.chunks,
|
|
11121
|
+
attemptCount: metadata.attemptCount,
|
|
11122
|
+
error: metadata.error,
|
|
11123
|
+
lastAttempt: metadata.lastAttempt
|
|
11124
|
+
});
|
|
11125
|
+
for (const rawChunk of metadata.chunks) {
|
|
11126
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11127
|
+
if (chunkId !== null) {
|
|
11128
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
11129
|
+
}
|
|
11130
|
+
}
|
|
11131
|
+
}
|
|
11132
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11133
|
+
failedProcessing.state = this.createFailedBatchWriteState();
|
|
11134
|
+
failedProcessing.discardedExistingRecords = false;
|
|
11135
|
+
for (const record of this.loadSerializedFailedBatches()) {
|
|
11136
|
+
for (const rawChunk of record.chunks) {
|
|
11137
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11138
|
+
this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] });
|
|
11139
|
+
if (chunkId !== null) {
|
|
11140
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
11141
|
+
}
|
|
11142
|
+
}
|
|
11143
|
+
}
|
|
11144
|
+
}
|
|
11145
|
+
const partialHashes = /* @__PURE__ */ new Map();
|
|
11146
|
+
for (const filePath of committedFilePaths) {
|
|
11147
|
+
const hash = currentFileHashes.get(filePath);
|
|
11148
|
+
if (hash !== void 0) {
|
|
11149
|
+
partialHashes.set(filePath, hash);
|
|
11150
|
+
}
|
|
11151
|
+
}
|
|
11152
|
+
if (scopedRoots) {
|
|
11153
|
+
this.replaceScopedFileHashCache(partialHashes, scopedRoots);
|
|
11154
|
+
} else {
|
|
11155
|
+
this.fileHashCache = partialHashes;
|
|
11156
|
+
this.saveFileHashCache();
|
|
11157
|
+
}
|
|
11158
|
+
}
|
|
10815
11159
|
clearFailedBatchState() {
|
|
10816
11160
|
if ((0, import_fs12.existsSync)(this.failedBatchesPath)) {
|
|
10817
11161
|
try {
|
|
@@ -10838,6 +11182,7 @@ var Indexer = class _Indexer {
|
|
|
10838
11182
|
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
10839
11183
|
const state = this.createFailedBatchWriteState();
|
|
10840
11184
|
const latestById = /* @__PURE__ */ new Map();
|
|
11185
|
+
let discardedExistingRecords = false;
|
|
10841
11186
|
try {
|
|
10842
11187
|
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10843
11188
|
for (const rawChunk of batch.chunks) {
|
|
@@ -10848,10 +11193,12 @@ var Indexer = class _Indexer {
|
|
|
10848
11193
|
continue;
|
|
10849
11194
|
}
|
|
10850
11195
|
if (!shouldProcess(filePath)) {
|
|
11196
|
+
discardedExistingRecords = true;
|
|
10851
11197
|
continue;
|
|
10852
11198
|
}
|
|
10853
11199
|
const chunkId = getPendingChunkId(rawChunk);
|
|
10854
11200
|
if (!chunkId) {
|
|
11201
|
+
discardedExistingRecords = true;
|
|
10855
11202
|
continue;
|
|
10856
11203
|
}
|
|
10857
11204
|
const existing = latestById.get(chunkId);
|
|
@@ -10859,12 +11206,18 @@ var Indexer = class _Indexer {
|
|
|
10859
11206
|
latestById.set(chunkId, {
|
|
10860
11207
|
attemptCount: batch.attemptCount,
|
|
10861
11208
|
error: batch.error,
|
|
10862
|
-
lastAttempt: batch.lastAttempt
|
|
11209
|
+
lastAttempt: batch.lastAttempt,
|
|
11210
|
+
chunks: [rawChunk]
|
|
10863
11211
|
});
|
|
10864
11212
|
}
|
|
10865
11213
|
}
|
|
10866
11214
|
}
|
|
10867
|
-
return {
|
|
11215
|
+
return {
|
|
11216
|
+
state,
|
|
11217
|
+
latestById,
|
|
11218
|
+
materializedRetryIds: /* @__PURE__ */ new Set(),
|
|
11219
|
+
discardedExistingRecords
|
|
11220
|
+
};
|
|
10868
11221
|
} catch (error) {
|
|
10869
11222
|
state.writer.cleanup();
|
|
10870
11223
|
throw error;
|
|
@@ -10900,10 +11253,34 @@ var Indexer = class _Indexer {
|
|
|
10900
11253
|
}
|
|
10901
11254
|
}
|
|
10902
11255
|
}
|
|
11256
|
+
restoreMissingChunkRows(database, chunks) {
|
|
11257
|
+
const missing = [];
|
|
11258
|
+
for (const chunk of chunks) {
|
|
11259
|
+
if (database.getChunk(chunk.id)) {
|
|
11260
|
+
continue;
|
|
11261
|
+
}
|
|
11262
|
+
missing.push({
|
|
11263
|
+
chunkId: chunk.id,
|
|
11264
|
+
contentHash: chunk.contentHash,
|
|
11265
|
+
filePath: chunk.metadata.filePath,
|
|
11266
|
+
startLine: chunk.metadata.startLine,
|
|
11267
|
+
endLine: chunk.metadata.endLine,
|
|
11268
|
+
nodeType: chunk.metadata.chunkType,
|
|
11269
|
+
name: chunk.metadata.name,
|
|
11270
|
+
language: chunk.metadata.language,
|
|
11271
|
+
blameSha: chunk.metadata.blameSha,
|
|
11272
|
+
blameAuthor: chunk.metadata.blameAuthor,
|
|
11273
|
+
blameAuthorEmail: chunk.metadata.blameAuthorEmail,
|
|
11274
|
+
blameCommittedAt: chunk.metadata.blameCommittedAt,
|
|
11275
|
+
blameSummary: chunk.metadata.blameSummary
|
|
11276
|
+
});
|
|
11277
|
+
}
|
|
11278
|
+
if (missing.length > 0) {
|
|
11279
|
+
database.upsertChunksBatch(missing);
|
|
11280
|
+
}
|
|
11281
|
+
}
|
|
10903
11282
|
getProviderRateLimits(provider) {
|
|
10904
11283
|
switch (provider) {
|
|
10905
|
-
case "github-copilot":
|
|
10906
|
-
return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
|
|
10907
11284
|
case "openai":
|
|
10908
11285
|
return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
10909
11286
|
case "google":
|
|
@@ -10972,10 +11349,11 @@ var Indexer = class _Indexer {
|
|
|
10972
11349
|
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
10973
11350
|
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
10974
11351
|
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
10975
|
-
const
|
|
10976
|
-
|
|
10977
|
-
|
|
10978
|
-
|
|
11352
|
+
const batchOptions = getDynamicBatchOptions(options.configuredProviderInfo, this.config.embedding?.batch);
|
|
11353
|
+
if (options.forceSingleItemBatches && options.configuredProviderInfo.provider === "ollama") {
|
|
11354
|
+
batchOptions.maxBatchItems = 1;
|
|
11355
|
+
}
|
|
11356
|
+
const requestBatches = createPendingEmbeddingRequestBatches(chunksNeedingEmbedding, batchOptions);
|
|
10979
11357
|
let fatalError;
|
|
10980
11358
|
for (const requestBatch of requestBatches) {
|
|
10981
11359
|
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
@@ -11538,7 +11916,7 @@ var Indexer = class _Indexer {
|
|
|
11538
11916
|
}
|
|
11539
11917
|
if (!this.configuredProviderInfo) {
|
|
11540
11918
|
throw new Error(
|
|
11541
|
-
"No embedding provider available. Configure
|
|
11919
|
+
"No embedding provider available. Configure OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
11542
11920
|
);
|
|
11543
11921
|
}
|
|
11544
11922
|
this.logger.info("Initializing indexer", {
|
|
@@ -11569,7 +11947,20 @@ var Indexer = class _Indexer {
|
|
|
11569
11947
|
]);
|
|
11570
11948
|
}
|
|
11571
11949
|
if (recoveredOwners.length > 0 && this.config.scope === "project") {
|
|
11572
|
-
|
|
11950
|
+
const unknownLegacyForceIndex = recoveredOwners.find(
|
|
11951
|
+
(owner) => this.hasUnknownLegacyForceIndexClear(owner)
|
|
11952
|
+
);
|
|
11953
|
+
if (unknownLegacyForceIndex) {
|
|
11954
|
+
throw new Error(
|
|
11955
|
+
`Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
11956
|
+
);
|
|
11957
|
+
}
|
|
11958
|
+
const shouldReset = recoveredOwners.some(
|
|
11959
|
+
(owner) => owner.clearRecovery !== void 0 || owner.operation === "clear" && owner.recoveryProtocolVersion !== 1
|
|
11960
|
+
);
|
|
11961
|
+
if (shouldReset) {
|
|
11962
|
+
await this.resetLocalIndexArtifacts();
|
|
11963
|
+
}
|
|
11573
11964
|
}
|
|
11574
11965
|
this.store = new VectorStore(storePath, dimensions);
|
|
11575
11966
|
if ((0, import_fs12.existsSync)(storePath) || (0, import_fs12.existsSync)(vectorMetadataPath)) {
|
|
@@ -12085,6 +12476,70 @@ var Indexer = class _Indexer {
|
|
|
12085
12476
|
);
|
|
12086
12477
|
return createCostEstimate(files, configuredProviderInfo);
|
|
12087
12478
|
}
|
|
12479
|
+
// Dry-run counterpart to index()/forceIndex(): parse the real file set and sum
|
|
12480
|
+
// estimateTokens over the embedding text of every indexable chunk, without
|
|
12481
|
+
// calling the embedding provider or writing to the index. Read-only and
|
|
12482
|
+
// lock-free (mirrors estimateCost). The token sum is the exact value "Tokens
|
|
12483
|
+
// used" climbs to for a force index (cache bypassed); for an incremental it is
|
|
12484
|
+
// an upper bound because cached chunks are counted here but not re-embedded.
|
|
12485
|
+
// Used by index_codebase(dryRun:true) to give a stable, monotonic progress
|
|
12486
|
+
// denominator that matches the live "Tokens used" basis.
|
|
12487
|
+
async dryRunCost() {
|
|
12488
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
12489
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
12490
|
+
const includePatterns = [...this.config.include, ...this.config.additionalInclude];
|
|
12491
|
+
const { files } = await collectFiles(
|
|
12492
|
+
this.materializedProjectRoot,
|
|
12493
|
+
includePatterns,
|
|
12494
|
+
this.config.exclude,
|
|
12495
|
+
this.config.indexing.maxFileSize,
|
|
12496
|
+
this.getMaterializedKnowledgeBases(),
|
|
12497
|
+
{ maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }
|
|
12498
|
+
);
|
|
12499
|
+
let filesCount = 0;
|
|
12500
|
+
let chunksCount = 0;
|
|
12501
|
+
let tokensToEmbed = 0;
|
|
12502
|
+
for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) {
|
|
12503
|
+
const loadedFiles = await Promise.all(batch.map(async (f) => {
|
|
12504
|
+
try {
|
|
12505
|
+
return {
|
|
12506
|
+
path: this.toStoredFilePath(f.path),
|
|
12507
|
+
content: await import_fs12.promises.readFile(f.path, "utf-8")
|
|
12508
|
+
};
|
|
12509
|
+
} catch {
|
|
12510
|
+
return null;
|
|
12511
|
+
}
|
|
12512
|
+
}));
|
|
12513
|
+
const readable = loadedFiles.filter(
|
|
12514
|
+
(f) => f !== null
|
|
12515
|
+
);
|
|
12516
|
+
filesCount += readable.length;
|
|
12517
|
+
const contentByPath = new Map(readable.map((f) => [f.path, f.content]));
|
|
12518
|
+
const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk);
|
|
12519
|
+
for (const parsed of parsedFiles) {
|
|
12520
|
+
let chunksToProcess = parsed.chunks;
|
|
12521
|
+
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
12522
|
+
const content = contentByPath.get(parsed.path);
|
|
12523
|
+
if (content !== void 0) {
|
|
12524
|
+
chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk);
|
|
12525
|
+
}
|
|
12526
|
+
}
|
|
12527
|
+
chunksToProcess = selectIndexableChunks(
|
|
12528
|
+
chunksToProcess,
|
|
12529
|
+
this.config.indexing.maxChunksPerFile,
|
|
12530
|
+
this.config.indexing.semanticOnly
|
|
12531
|
+
);
|
|
12532
|
+
for (const chunk of chunksToProcess) {
|
|
12533
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens);
|
|
12534
|
+
chunksCount += 1;
|
|
12535
|
+
for (const text of texts) {
|
|
12536
|
+
tokensToEmbed += estimateTokens(text);
|
|
12537
|
+
}
|
|
12538
|
+
}
|
|
12539
|
+
}
|
|
12540
|
+
}
|
|
12541
|
+
return { filesCount, chunksCount, tokensToEmbed };
|
|
12542
|
+
}
|
|
12088
12543
|
async index(onProgress) {
|
|
12089
12544
|
return this.withIndexMutationLease("index", async (recoveredOwners) => {
|
|
12090
12545
|
return this.indexUnlocked(onProgress, recoveredOwners);
|
|
@@ -12205,7 +12660,17 @@ var Indexer = class _Indexer {
|
|
|
12205
12660
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
12206
12661
|
for (const file of files) {
|
|
12207
12662
|
const storedPath = this.toStoredFilePath(file.path);
|
|
12208
|
-
|
|
12663
|
+
let currentHash;
|
|
12664
|
+
try {
|
|
12665
|
+
currentHash = hashFile(file.path);
|
|
12666
|
+
} catch (error) {
|
|
12667
|
+
stats.skippedFiles.push({ path: this.toCanonicalFilePath(file.path), reason: "unreadable" });
|
|
12668
|
+
this.logger.warn("Skipped unreadable file during indexing", {
|
|
12669
|
+
path: file.path,
|
|
12670
|
+
error: getErrorMessage4(error)
|
|
12671
|
+
});
|
|
12672
|
+
continue;
|
|
12673
|
+
}
|
|
12209
12674
|
currentFileHashes.set(storedPath, currentHash);
|
|
12210
12675
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
12211
12676
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
@@ -12213,7 +12678,8 @@ var Indexer = class _Indexer {
|
|
|
12213
12678
|
);
|
|
12214
12679
|
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path19.extname(storedPath).toLowerCase() === ".swift";
|
|
12215
12680
|
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path19.extname(storedPath).toLowerCase() === ".metal";
|
|
12216
|
-
|
|
12681
|
+
const inMigrationScope = forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots);
|
|
12682
|
+
if (cachedHashMatches && !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
12217
12683
|
unchangedFilePaths.add(storedPath);
|
|
12218
12684
|
this.logger.recordCacheHit();
|
|
12219
12685
|
} else {
|
|
@@ -12339,6 +12805,9 @@ var Indexer = class _Indexer {
|
|
|
12339
12805
|
}
|
|
12340
12806
|
}
|
|
12341
12807
|
let processedChangedFiles = 0;
|
|
12808
|
+
let lastCheckpointChunks = 0;
|
|
12809
|
+
const committedFilePaths = new Set(unchangedFilePaths);
|
|
12810
|
+
const resolvedRetryChunkIds = /* @__PURE__ */ new Set();
|
|
12342
12811
|
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
12343
12812
|
changedFileDescriptors,
|
|
12344
12813
|
(descriptor) => descriptor.sourceBytes,
|
|
@@ -12352,7 +12821,7 @@ var Indexer = class _Indexer {
|
|
|
12352
12821
|
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
12353
12822
|
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
12354
12823
|
const parseStartTime = import_perf_hooks.performance.now();
|
|
12355
|
-
const parsedFiles = parseFiles(loadedFiles);
|
|
12824
|
+
const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk);
|
|
12356
12825
|
const parseMs = import_perf_hooks.performance.now() - parseStartTime;
|
|
12357
12826
|
this.logger.recordFilesParsed(parsedFiles.length);
|
|
12358
12827
|
this.logger.recordParseDuration(parseMs);
|
|
@@ -12375,7 +12844,7 @@ var Indexer = class _Indexer {
|
|
|
12375
12844
|
}
|
|
12376
12845
|
let chunksToProcess = parsed.chunks;
|
|
12377
12846
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
12378
|
-
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
12847
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk);
|
|
12379
12848
|
}
|
|
12380
12849
|
chunksToProcess = selectIndexableChunks(
|
|
12381
12850
|
chunksToProcess,
|
|
@@ -12509,6 +12978,10 @@ var Indexer = class _Indexer {
|
|
|
12509
12978
|
}
|
|
12510
12979
|
if (symbolBatch.length > 0) {
|
|
12511
12980
|
database.upsertSymbolsBatch(symbolBatch);
|
|
12981
|
+
database.addSymbolsToBranchBatch(
|
|
12982
|
+
this.getBranchCatalogKey(),
|
|
12983
|
+
symbolBatch.map((symbol) => symbol.id)
|
|
12984
|
+
);
|
|
12512
12985
|
}
|
|
12513
12986
|
if (edgeBatch.length > 0) {
|
|
12514
12987
|
database.upsertCallEdgesBatch(edgeBatch);
|
|
@@ -12544,6 +13017,12 @@ var Indexer = class _Indexer {
|
|
|
12544
13017
|
forceReembed: forceScopedReembed,
|
|
12545
13018
|
reuseCachedEmbeddings: true,
|
|
12546
13019
|
incrementRepeatedFailures: true,
|
|
13020
|
+
onSucceeded: (succeededChunks) => {
|
|
13021
|
+
database.addChunksToBranchBatch(
|
|
13022
|
+
this.getBranchCatalogKey(),
|
|
13023
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
13024
|
+
);
|
|
13025
|
+
},
|
|
12547
13026
|
onProgress: (batchProgress) => onProgress?.({
|
|
12548
13027
|
phase: "embedding",
|
|
12549
13028
|
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
@@ -12562,6 +13041,27 @@ var Indexer = class _Indexer {
|
|
|
12562
13041
|
}
|
|
12563
13042
|
}
|
|
12564
13043
|
}
|
|
13044
|
+
for (const descriptor of descriptorBatch) {
|
|
13045
|
+
const existingFileChunks = existingChunksByFile.get(descriptor.storedPath);
|
|
13046
|
+
if (!existingFileChunks || existingFileChunks.size === 0) {
|
|
13047
|
+
committedFilePaths.add(descriptor.storedPath);
|
|
13048
|
+
}
|
|
13049
|
+
}
|
|
13050
|
+
const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks);
|
|
13051
|
+
if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) {
|
|
13052
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
13053
|
+
this.checkpointIndexRun(
|
|
13054
|
+
database,
|
|
13055
|
+
store,
|
|
13056
|
+
invertedIndex,
|
|
13057
|
+
failedProcessing,
|
|
13058
|
+
resolvedRetryChunkIds,
|
|
13059
|
+
currentFileHashes,
|
|
13060
|
+
committedFilePaths,
|
|
13061
|
+
scopedRoots,
|
|
13062
|
+
configuredProviderInfo
|
|
13063
|
+
);
|
|
13064
|
+
}
|
|
12565
13065
|
}
|
|
12566
13066
|
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
12567
13067
|
failedProcessing.latestById,
|
|
@@ -12582,6 +13082,7 @@ var Indexer = class _Indexer {
|
|
|
12582
13082
|
retryableChunksWithExistingData.add(chunk.id);
|
|
12583
13083
|
}
|
|
12584
13084
|
}
|
|
13085
|
+
this.restoreMissingChunkRows(database, pendingChunks);
|
|
12585
13086
|
stats.totalChunks += pendingChunks.length;
|
|
12586
13087
|
onProgress?.({
|
|
12587
13088
|
phase: "embedding",
|
|
@@ -12604,6 +13105,17 @@ var Indexer = class _Indexer {
|
|
|
12604
13105
|
forceReembed: forceScopedReembed,
|
|
12605
13106
|
reuseCachedEmbeddings: true,
|
|
12606
13107
|
incrementRepeatedFailures: true,
|
|
13108
|
+
forceSingleItemBatches: true,
|
|
13109
|
+
onSucceeded: (succeededChunks) => {
|
|
13110
|
+
database.addChunksToBranchBatch(
|
|
13111
|
+
this.getBranchCatalogKey(),
|
|
13112
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
13113
|
+
);
|
|
13114
|
+
for (const chunk of succeededChunks) {
|
|
13115
|
+
failedProcessing.latestById.delete(chunk.id);
|
|
13116
|
+
resolvedRetryChunkIds.add(chunk.id);
|
|
13117
|
+
}
|
|
13118
|
+
},
|
|
12607
13119
|
onProgress: (batchProgress) => onProgress?.({
|
|
12608
13120
|
phase: "embedding",
|
|
12609
13121
|
filesProcessed: files.length,
|
|
@@ -12621,6 +13133,20 @@ var Indexer = class _Indexer {
|
|
|
12621
13133
|
failedForcedChunkIds.add(chunkId);
|
|
12622
13134
|
}
|
|
12623
13135
|
}
|
|
13136
|
+
if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) {
|
|
13137
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
13138
|
+
this.checkpointIndexRun(
|
|
13139
|
+
database,
|
|
13140
|
+
store,
|
|
13141
|
+
invertedIndex,
|
|
13142
|
+
failedProcessing,
|
|
13143
|
+
resolvedRetryChunkIds,
|
|
13144
|
+
currentFileHashes,
|
|
13145
|
+
committedFilePaths,
|
|
13146
|
+
scopedRoots,
|
|
13147
|
+
configuredProviderInfo
|
|
13148
|
+
);
|
|
13149
|
+
}
|
|
12624
13150
|
}
|
|
12625
13151
|
const removedChunkIds = [];
|
|
12626
13152
|
for (const [chunkId] of existingChunks) {
|
|
@@ -12657,13 +13183,6 @@ var Indexer = class _Indexer {
|
|
|
12657
13183
|
if (removedStoredChunks) {
|
|
12658
13184
|
this.saveInvertedIndex(invertedIndex);
|
|
12659
13185
|
}
|
|
12660
|
-
if (scopedRoots) {
|
|
12661
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12662
|
-
} else {
|
|
12663
|
-
this.fileHashCache = currentFileHashes;
|
|
12664
|
-
this.saveFileHashCache();
|
|
12665
|
-
}
|
|
12666
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12667
13186
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12668
13187
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12669
13188
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -12672,6 +13191,13 @@ var Indexer = class _Indexer {
|
|
|
12672
13191
|
this.indexCompatibility = { compatible: true };
|
|
12673
13192
|
database.commitWriteTransaction();
|
|
12674
13193
|
writeTransactionActive = false;
|
|
13194
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
13195
|
+
if (scopedRoots) {
|
|
13196
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
13197
|
+
} else {
|
|
13198
|
+
this.fileHashCache = currentFileHashes;
|
|
13199
|
+
this.saveFileHashCache();
|
|
13200
|
+
}
|
|
12675
13201
|
stats.durationMs = Date.now() - startTime;
|
|
12676
13202
|
onProgress?.({
|
|
12677
13203
|
phase: "complete",
|
|
@@ -12695,13 +13221,6 @@ var Indexer = class _Indexer {
|
|
|
12695
13221
|
);
|
|
12696
13222
|
store.save();
|
|
12697
13223
|
this.saveInvertedIndex(invertedIndex);
|
|
12698
|
-
if (scopedRoots) {
|
|
12699
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12700
|
-
} else {
|
|
12701
|
-
this.fileHashCache = currentFileHashes;
|
|
12702
|
-
this.saveFileHashCache();
|
|
12703
|
-
}
|
|
12704
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12705
13224
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12706
13225
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12707
13226
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -12710,6 +13229,13 @@ var Indexer = class _Indexer {
|
|
|
12710
13229
|
this.indexCompatibility = { compatible: true };
|
|
12711
13230
|
database.commitWriteTransaction();
|
|
12712
13231
|
writeTransactionActive = false;
|
|
13232
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
13233
|
+
if (scopedRoots) {
|
|
13234
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
13235
|
+
} else {
|
|
13236
|
+
this.fileHashCache = currentFileHashes;
|
|
13237
|
+
this.saveFileHashCache();
|
|
13238
|
+
}
|
|
12713
13239
|
stats.durationMs = Date.now() - startTime;
|
|
12714
13240
|
onProgress?.({
|
|
12715
13241
|
phase: "complete",
|
|
@@ -12744,15 +13270,15 @@ var Indexer = class _Indexer {
|
|
|
12744
13270
|
);
|
|
12745
13271
|
store.save();
|
|
12746
13272
|
this.saveInvertedIndex(invertedIndex);
|
|
13273
|
+
database.commitWriteTransaction();
|
|
13274
|
+
writeTransactionActive = false;
|
|
13275
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
12747
13276
|
if (scopedRoots) {
|
|
12748
13277
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12749
13278
|
} else {
|
|
12750
13279
|
this.fileHashCache = currentFileHashes;
|
|
12751
13280
|
this.saveFileHashCache();
|
|
12752
13281
|
}
|
|
12753
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12754
|
-
database.commitWriteTransaction();
|
|
12755
|
-
writeTransactionActive = false;
|
|
12756
13282
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
12757
13283
|
const gcReset = await this.maybeRunOrphanGc();
|
|
12758
13284
|
if (gcReset) {
|
|
@@ -12776,6 +13302,9 @@ var Indexer = class _Indexer {
|
|
|
12776
13302
|
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
12777
13303
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12778
13304
|
}
|
|
13305
|
+
if (forceScopedReembed) {
|
|
13306
|
+
database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true");
|
|
13307
|
+
}
|
|
12779
13308
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12780
13309
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12781
13310
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -12886,26 +13415,41 @@ var Indexer = class _Indexer {
|
|
|
12886
13415
|
shouldPrefilterByBranch: branchChunkIds !== null && (this.config.scope === "global" || hasInitializedBranchCatalog)
|
|
12887
13416
|
};
|
|
12888
13417
|
}
|
|
12889
|
-
|
|
13418
|
+
searchCandidatesWithAllowedIds(initialLimit, totalCount, allowedChunkIds, shouldPrefilter, search, getChunkId) {
|
|
12890
13419
|
const normalizedLimit = Math.max(0, Math.floor(initialLimit));
|
|
12891
13420
|
if (normalizedLimit === 0) return [];
|
|
12892
|
-
if (!
|
|
13421
|
+
if (!shouldPrefilter || !allowedChunkIds) {
|
|
12893
13422
|
return search(normalizedLimit);
|
|
12894
13423
|
}
|
|
12895
|
-
const targetCount = Math.min(normalizedLimit,
|
|
13424
|
+
const targetCount = Math.min(normalizedLimit, allowedChunkIds.size);
|
|
12896
13425
|
if (targetCount === 0 || totalCount === 0) return [];
|
|
12897
13426
|
let requestedLimit = Math.min(normalizedLimit, totalCount);
|
|
12898
13427
|
while (true) {
|
|
12899
13428
|
const results = search(requestedLimit);
|
|
12900
|
-
const
|
|
12901
|
-
if (
|
|
12902
|
-
return
|
|
13429
|
+
const allowedResults = results.filter((candidate) => allowedChunkIds.has(getChunkId(candidate)));
|
|
13430
|
+
if (allowedResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
|
|
13431
|
+
return allowedResults;
|
|
12903
13432
|
}
|
|
12904
13433
|
const nextLimit = Math.min(totalCount, Math.max(requestedLimit + 1, requestedLimit * 2));
|
|
12905
|
-
if (nextLimit === requestedLimit) return
|
|
13434
|
+
if (nextLimit === requestedLimit) return allowedResults;
|
|
12906
13435
|
requestedLimit = nextLimit;
|
|
12907
13436
|
}
|
|
12908
13437
|
}
|
|
13438
|
+
getTemporalChunkIds(database, options) {
|
|
13439
|
+
if (!options?.blameSince && !options?.blameUntil) return null;
|
|
13440
|
+
const since = options.blameSince ? parseBlameTimestamp(options.blameSince, false) : void 0;
|
|
13441
|
+
const until = options.blameUntil ? parseBlameTimestamp(options.blameUntil, true) : void 0;
|
|
13442
|
+
if (since === null || until === null) {
|
|
13443
|
+
return /* @__PURE__ */ new Set();
|
|
13444
|
+
}
|
|
13445
|
+
return new Set(database.getChunkIdsByBlameDate(since, until));
|
|
13446
|
+
}
|
|
13447
|
+
intersectChunkIdSets(first, second) {
|
|
13448
|
+
if (first === null) return second;
|
|
13449
|
+
if (second === null) return first;
|
|
13450
|
+
const [smaller, larger] = first.size <= second.size ? [first, second] : [second, first];
|
|
13451
|
+
return new Set(Array.from(smaller).filter((chunkId) => larger.has(chunkId)));
|
|
13452
|
+
}
|
|
12909
13453
|
buildCandidateSnapshot(candidate) {
|
|
12910
13454
|
return {
|
|
12911
13455
|
id: candidate.id,
|
|
@@ -12920,13 +13464,16 @@ var Indexer = class _Indexer {
|
|
|
12920
13464
|
buildCandidateSnapshotList(candidates) {
|
|
12921
13465
|
return candidates.map((candidate) => this.buildCandidateSnapshot(candidate));
|
|
12922
13466
|
}
|
|
12923
|
-
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch) {
|
|
12924
|
-
|
|
12925
|
-
|
|
12926
|
-
|
|
13467
|
+
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch, temporalChunkIds) {
|
|
13468
|
+
const availableCount = temporalChunkIds?.size ?? store.count();
|
|
13469
|
+
if (availableCount === 0) return [];
|
|
13470
|
+
const allowedIds = temporalChunkIds === null ? void 0 : Array.from(temporalChunkIds);
|
|
13471
|
+
return this.searchCandidatesWithAllowedIds(
|
|
13472
|
+
Math.min(initialLimit, availableCount),
|
|
13473
|
+
availableCount,
|
|
12927
13474
|
branchChunkIds,
|
|
12928
13475
|
shouldPrefilterByBranch,
|
|
12929
|
-
(requestedLimit) => store.search(embedding, requestedLimit),
|
|
13476
|
+
(requestedLimit) => store.search(embedding, requestedLimit, allowedIds),
|
|
12930
13477
|
(candidate) => candidate.id
|
|
12931
13478
|
);
|
|
12932
13479
|
}
|
|
@@ -12951,8 +13498,9 @@ var Indexer = class _Indexer {
|
|
|
12951
13498
|
const rerankTopN = this.config.search.rerankTopN;
|
|
12952
13499
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
12953
13500
|
const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
|
|
13501
|
+
const prioritizeSourcePaths = sourceIntent || options?.prioritizeSourcePaths === true;
|
|
12954
13502
|
const identifierHints = extractIdentifierHints(query);
|
|
12955
|
-
const candidateLimit = maxResults * (
|
|
13503
|
+
const candidateLimit = maxResults * (prioritizeSourcePaths ? 12 : 4);
|
|
12956
13504
|
this.logger.search("debug", "Starting search", {
|
|
12957
13505
|
query,
|
|
12958
13506
|
maxResults,
|
|
@@ -12983,6 +13531,7 @@ var Indexer = class _Indexer {
|
|
|
12983
13531
|
branchChunkIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchChunkIds(branchKey)));
|
|
12984
13532
|
branchSymbolIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchSymbolIds(branchKey)));
|
|
12985
13533
|
}
|
|
13534
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
12986
13535
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
12987
13536
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
12988
13537
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
@@ -12991,7 +13540,8 @@ var Indexer = class _Indexer {
|
|
|
12991
13540
|
embedding,
|
|
12992
13541
|
candidateLimit,
|
|
12993
13542
|
branchChunkIds,
|
|
12994
|
-
shouldPrefilterByBranch
|
|
13543
|
+
shouldPrefilterByBranch,
|
|
13544
|
+
temporalChunkIds
|
|
12995
13545
|
) : [];
|
|
12996
13546
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
12997
13547
|
const keywordStartTime = import_perf_hooks.performance.now();
|
|
@@ -13001,7 +13551,8 @@ var Indexer = class _Indexer {
|
|
|
13001
13551
|
store,
|
|
13002
13552
|
invertedIndex,
|
|
13003
13553
|
branchChunkIds,
|
|
13004
|
-
shouldPrefilterByBranch
|
|
13554
|
+
shouldPrefilterByBranch,
|
|
13555
|
+
temporalChunkIds
|
|
13005
13556
|
);
|
|
13006
13557
|
const keywordMs = import_perf_hooks.performance.now() - keywordStartTime;
|
|
13007
13558
|
const scopedSemanticCandidates = semanticCandidates.filter(
|
|
@@ -13023,7 +13574,7 @@ var Indexer = class _Indexer {
|
|
|
13023
13574
|
rerankTopN,
|
|
13024
13575
|
limit: maxResults,
|
|
13025
13576
|
hybridWeight: rankingHybridWeight,
|
|
13026
|
-
prioritizeSourcePaths
|
|
13577
|
+
prioritizeSourcePaths
|
|
13027
13578
|
});
|
|
13028
13579
|
const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
|
|
13029
13580
|
definitionIntent: options?.definitionIntent === true,
|
|
@@ -13059,10 +13610,11 @@ var Indexer = class _Indexer {
|
|
|
13059
13610
|
branchSymbolIds,
|
|
13060
13611
|
maxResults,
|
|
13061
13612
|
union,
|
|
13062
|
-
sourceIntent
|
|
13613
|
+
sourceIntent,
|
|
13614
|
+
options?.definitionIntent === true && ((options.directory?.trim().length ?? 0) > 0 || (options.fileType?.trim().length ?? 0) > 0)
|
|
13063
13615
|
);
|
|
13064
13616
|
const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
|
|
13065
|
-
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
13617
|
+
const primaryLane = options?.definitionIntent === true ? mergeTieredResults(symbolLane, prePrimaryLane, maxResults * 4) : mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
13066
13618
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
13067
13619
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
13068
13620
|
const baseFiltered = tiered.filter(
|
|
@@ -13157,14 +13709,18 @@ var Indexer = class _Indexer {
|
|
|
13157
13709
|
})
|
|
13158
13710
|
);
|
|
13159
13711
|
}
|
|
13160
|
-
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false) {
|
|
13712
|
+
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false, temporalChunkIds = null) {
|
|
13161
13713
|
const normalizedLimit = Math.max(0, Math.floor(limit));
|
|
13162
13714
|
if (normalizedLimit === 0) return [];
|
|
13163
|
-
const
|
|
13715
|
+
const allowedChunkIds = this.intersectChunkIdSets(
|
|
13716
|
+
shouldPrefilterByBranch ? branchChunkIds : null,
|
|
13717
|
+
temporalChunkIds
|
|
13718
|
+
);
|
|
13719
|
+
const scoreEntries = this.searchCandidatesWithAllowedIds(
|
|
13164
13720
|
normalizedLimit,
|
|
13165
13721
|
invertedIndex.getDocumentCount(),
|
|
13166
|
-
|
|
13167
|
-
|
|
13722
|
+
allowedChunkIds,
|
|
13723
|
+
allowedChunkIds !== null,
|
|
13168
13724
|
(requestedLimit) => Array.from(invertedIndex.search(query, requestedLimit)),
|
|
13169
13725
|
([chunkId]) => chunkId
|
|
13170
13726
|
);
|
|
@@ -13249,7 +13805,17 @@ var Indexer = class _Indexer {
|
|
|
13249
13805
|
);
|
|
13250
13806
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
13251
13807
|
for (const file of files) {
|
|
13252
|
-
|
|
13808
|
+
let hash;
|
|
13809
|
+
try {
|
|
13810
|
+
hash = hashFile(file.path);
|
|
13811
|
+
} catch (error) {
|
|
13812
|
+
this.logger.warn("Skipped unreadable file during freshness check", {
|
|
13813
|
+
path: file.path,
|
|
13814
|
+
error: getErrorMessage4(error)
|
|
13815
|
+
});
|
|
13816
|
+
return { readable: false, current: false, reason: "unreadable" };
|
|
13817
|
+
}
|
|
13818
|
+
currentFileHashes.set(this.toStoredFilePath(file.path), hash);
|
|
13253
13819
|
}
|
|
13254
13820
|
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
13255
13821
|
const cachedFileHashes = scopedRoots ? new Map(Array.from(this.fileHashCache).filter(([filePath]) => this.isFileInCurrentScope(filePath, scopedRoots))) : this.fileHashCache;
|
|
@@ -13275,69 +13841,87 @@ var Indexer = class _Indexer {
|
|
|
13275
13841
|
async forceIndex(onProgress) {
|
|
13276
13842
|
return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
|
|
13277
13843
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
13278
|
-
|
|
13844
|
+
const recovery = this.beginClearRecoveryState();
|
|
13845
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
13846
|
+
this.finishClearRecoveryState();
|
|
13279
13847
|
return this.indexUnlocked(onProgress, [], true);
|
|
13280
13848
|
});
|
|
13281
13849
|
}
|
|
13282
13850
|
async clearIndex() {
|
|
13283
13851
|
await this.withIndexMutationLease("clear", async (recoveredOwners) => {
|
|
13284
13852
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
13285
|
-
|
|
13853
|
+
const recovery = this.beginClearRecoveryState();
|
|
13854
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
13286
13855
|
});
|
|
13287
13856
|
}
|
|
13288
|
-
|
|
13857
|
+
clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot) {
|
|
13289
13858
|
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
13290
|
-
|
|
13291
|
-
|
|
13292
|
-
|
|
13293
|
-
|
|
13294
|
-
|
|
13295
|
-
|
|
13296
|
-
|
|
13297
|
-
|
|
13298
|
-
|
|
13299
|
-
|
|
13300
|
-
|
|
13301
|
-
|
|
13302
|
-
|
|
13303
|
-
|
|
13304
|
-
|
|
13859
|
+
const clearedBranchKeys = database.getAllBranches();
|
|
13860
|
+
store.clear();
|
|
13861
|
+
store.save();
|
|
13862
|
+
invertedIndex.clear();
|
|
13863
|
+
this.saveInvertedIndex(invertedIndex);
|
|
13864
|
+
this.fileHashCache.clear();
|
|
13865
|
+
this.saveFileHashCache();
|
|
13866
|
+
database.clearAllIndexedData();
|
|
13867
|
+
this.deleteBranchCommitMetadata(database, clearedBranchKeys);
|
|
13868
|
+
this.clearFailedBatchState();
|
|
13869
|
+
database.deleteMetadata("index.version");
|
|
13870
|
+
database.deleteMetadata("index.pathStorageVersion");
|
|
13871
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
13872
|
+
database.deleteMetadata("index.embeddingModel");
|
|
13873
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
13874
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
13875
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
13876
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
13877
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash));
|
|
13878
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash));
|
|
13879
|
+
database.deleteMetadata("index.createdAt");
|
|
13880
|
+
database.deleteMetadata("index.updatedAt");
|
|
13881
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
13882
|
+
}
|
|
13883
|
+
clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots(), recoveryDecision) {
|
|
13884
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
13885
|
+
store.load();
|
|
13886
|
+
invertedIndex.load();
|
|
13887
|
+
this.loadFileHashCache();
|
|
13888
|
+
const compatibility = this.checkCompatibility();
|
|
13889
|
+
const compatibilityDecision = recoveryDecision ?? (compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible");
|
|
13890
|
+
const allMetadata = store.getAllMetadata();
|
|
13891
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
13892
|
+
if (compatibilityDecision !== "compatible" && hasForeignData) {
|
|
13893
|
+
if (compatibilityDecision === "embedding-strategy-mismatch") {
|
|
13894
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
13895
|
+
this.clearScopedFileHashCache(roots);
|
|
13896
|
+
this.clearScopedFailedBatches(roots);
|
|
13897
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
13898
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true");
|
|
13899
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
13900
|
+
database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash));
|
|
13901
|
+
if (projectRoot === this.projectRoot) {
|
|
13305
13902
|
this.indexCompatibility = { compatible: true };
|
|
13306
|
-
return;
|
|
13307
13903
|
}
|
|
13308
|
-
throw new Error(
|
|
13309
|
-
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
13310
|
-
);
|
|
13311
|
-
}
|
|
13312
|
-
if (!hasForeignData) {
|
|
13313
|
-
const clearedBranchKeys2 = database.getAllBranches();
|
|
13314
|
-
store.clear();
|
|
13315
|
-
store.save();
|
|
13316
|
-
invertedIndex.clear();
|
|
13317
|
-
this.saveInvertedIndex(invertedIndex);
|
|
13318
|
-
this.fileHashCache.clear();
|
|
13319
|
-
this.saveFileHashCache();
|
|
13320
|
-
database.clearAllIndexedData();
|
|
13321
|
-
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
13322
|
-
this.clearFailedBatchState();
|
|
13323
|
-
database.deleteMetadata("index.version");
|
|
13324
|
-
database.deleteMetadata("index.pathStorageVersion");
|
|
13325
|
-
database.deleteMetadata("index.embeddingProvider");
|
|
13326
|
-
database.deleteMetadata("index.embeddingModel");
|
|
13327
|
-
database.deleteMetadata("index.embeddingDimensions");
|
|
13328
|
-
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
13329
|
-
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
13330
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
13331
|
-
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
13332
|
-
database.deleteMetadata("index.createdAt");
|
|
13333
|
-
database.deleteMetadata("index.updatedAt");
|
|
13334
|
-
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
13335
13904
|
return;
|
|
13336
13905
|
}
|
|
13337
|
-
|
|
13338
|
-
|
|
13339
|
-
|
|
13906
|
+
throw new Error(
|
|
13907
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
13908
|
+
);
|
|
13909
|
+
}
|
|
13910
|
+
if (!hasForeignData) {
|
|
13911
|
+
this.clearGlobalIndexDataUnlocked(projectRoot);
|
|
13912
|
+
return;
|
|
13913
|
+
}
|
|
13914
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
13915
|
+
this.clearScopedFileHashCache(roots);
|
|
13916
|
+
this.clearScopedFailedBatches(roots);
|
|
13917
|
+
if (projectRoot === this.projectRoot) {
|
|
13340
13918
|
this.indexCompatibility = compatibility;
|
|
13919
|
+
}
|
|
13920
|
+
}
|
|
13921
|
+
async clearIndexUnlocked(recoveryDecision) {
|
|
13922
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
13923
|
+
if (this.config.scope === "global") {
|
|
13924
|
+
this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision);
|
|
13341
13925
|
return;
|
|
13342
13926
|
}
|
|
13343
13927
|
if (!this.isProjectOwnedIndexPath()) {
|
|
@@ -13503,6 +14087,7 @@ var Indexer = class _Indexer {
|
|
|
13503
14087
|
)) {
|
|
13504
14088
|
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
13505
14089
|
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
14090
|
+
this.restoreMissingChunkRows(database, chunks);
|
|
13506
14091
|
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
13507
14092
|
store,
|
|
13508
14093
|
provider,
|
|
@@ -13517,6 +14102,7 @@ var Indexer = class _Indexer {
|
|
|
13517
14102
|
forceReembed: false,
|
|
13518
14103
|
reuseCachedEmbeddings: false,
|
|
13519
14104
|
incrementRepeatedFailures: false,
|
|
14105
|
+
forceSingleItemBatches: true,
|
|
13520
14106
|
onSucceeded: (succeededChunks) => {
|
|
13521
14107
|
database.addChunksToBranchBatch(
|
|
13522
14108
|
this.getBranchCatalogKey(),
|
|
@@ -13538,9 +14124,12 @@ var Indexer = class _Indexer {
|
|
|
13538
14124
|
this.saveInvertedIndex(invertedIndex);
|
|
13539
14125
|
}
|
|
13540
14126
|
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
13541
|
-
database.
|
|
13542
|
-
|
|
13543
|
-
|
|
14127
|
+
const migrationFinalized = database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true";
|
|
14128
|
+
if (migrationFinalized) {
|
|
14129
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
14130
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
14131
|
+
this.indexCompatibility = { compatible: true };
|
|
14132
|
+
}
|
|
13544
14133
|
}
|
|
13545
14134
|
return { succeeded, failed, remaining };
|
|
13546
14135
|
}
|
|
@@ -13562,7 +14151,8 @@ var Indexer = class _Indexer {
|
|
|
13562
14151
|
latestById.set(chunkId, {
|
|
13563
14152
|
attemptCount: batch.attemptCount,
|
|
13564
14153
|
error: batch.error,
|
|
13565
|
-
lastAttempt: batch.lastAttempt
|
|
14154
|
+
lastAttempt: batch.lastAttempt,
|
|
14155
|
+
chunks: [rawChunk]
|
|
13566
14156
|
});
|
|
13567
14157
|
}
|
|
13568
14158
|
}
|
|
@@ -13629,6 +14219,7 @@ var Indexer = class _Indexer {
|
|
|
13629
14219
|
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
13630
14220
|
);
|
|
13631
14221
|
}
|
|
14222
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
13632
14223
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
13633
14224
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
13634
14225
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
@@ -13637,7 +14228,8 @@ var Indexer = class _Indexer {
|
|
|
13637
14228
|
embedding,
|
|
13638
14229
|
limit * 2,
|
|
13639
14230
|
branchChunkIds,
|
|
13640
|
-
shouldPrefilterByBranch
|
|
14231
|
+
shouldPrefilterByBranch,
|
|
14232
|
+
temporalChunkIds
|
|
13641
14233
|
);
|
|
13642
14234
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
13643
14235
|
if (this.config.scope !== "global" && branchChunkIds && !hasInitializedBranchCatalog) {
|
|
@@ -14386,9 +14978,11 @@ async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
|
14386
14978
|
contextLines: options.contextLines,
|
|
14387
14979
|
metadataOnly: options.metadataOnly,
|
|
14388
14980
|
definitionIntent: options.definitionIntent,
|
|
14981
|
+
prioritizeSourcePaths: options.prioritizeSourcePaths,
|
|
14389
14982
|
blameAuthor: options.blameAuthor,
|
|
14390
14983
|
blameSha: options.blameSha,
|
|
14391
14984
|
blameSince: options.blameSince,
|
|
14985
|
+
blameUntil: options.blameUntil,
|
|
14392
14986
|
trace: options.trace
|
|
14393
14987
|
});
|
|
14394
14988
|
}
|
|
@@ -14434,7 +15028,9 @@ async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
|
14434
15028
|
fileType: options.fileType,
|
|
14435
15029
|
directory: options.directory,
|
|
14436
15030
|
chunkType: options.chunkType,
|
|
14437
|
-
excludeFile: options.excludeFile
|
|
15031
|
+
excludeFile: options.excludeFile,
|
|
15032
|
+
blameSince: options.blameSince,
|
|
15033
|
+
blameUntil: options.blameUntil
|
|
14438
15034
|
});
|
|
14439
15035
|
}
|
|
14440
15036
|
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
@@ -14497,6 +15093,9 @@ async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
|
14497
15093
|
if (args.estimateOnly) {
|
|
14498
15094
|
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
14499
15095
|
}
|
|
15096
|
+
if (args.dryRun) {
|
|
15097
|
+
return { kind: "dryrun", dryrun: await indexer.dryRunCost() };
|
|
15098
|
+
}
|
|
14500
15099
|
const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => {
|
|
14501
15100
|
if (onProgress) {
|
|
14502
15101
|
void onProgress(formatProgressTitle(progress), {
|
|
@@ -17973,13 +18572,19 @@ async function resolveSearchContext(input, operations) {
|
|
|
17973
18572
|
(trace) => operations.lookup(symbol, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
|
|
17974
18573
|
);
|
|
17975
18574
|
};
|
|
17976
|
-
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt) => {
|
|
18575
|
+
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt, prioritizeSourcePaths) => {
|
|
17977
18576
|
return recordAttempt(
|
|
17978
18577
|
"conceptual",
|
|
17979
18578
|
searchQuery,
|
|
17980
18579
|
scope,
|
|
17981
18580
|
relaxedFieldsForAttempt,
|
|
17982
|
-
(trace) => operations.search(
|
|
18581
|
+
(trace) => operations.search(
|
|
18582
|
+
searchQuery,
|
|
18583
|
+
MAX_CONTEXT_RESULT_LIMIT,
|
|
18584
|
+
scope,
|
|
18585
|
+
input.diagnostic ? trace : void 0,
|
|
18586
|
+
{ prioritizeSourcePaths }
|
|
18587
|
+
)
|
|
17983
18588
|
);
|
|
17984
18589
|
};
|
|
17985
18590
|
const findSuccessfulAttemptState = (route) => {
|
|
@@ -18107,10 +18712,12 @@ Explicit symbol lookup only; conceptual search was not attempted.`
|
|
|
18107
18712
|
}
|
|
18108
18713
|
}
|
|
18109
18714
|
for (const attempt of conceptualAttemptPlan) {
|
|
18715
|
+
const attemptIntent = analyzeQueryIntent(attempt.queryText);
|
|
18716
|
+
const prioritizeSourcePaths = attemptIntent.primary !== "docs" && attemptIntent.primary !== "test";
|
|
18110
18717
|
if (inferredSymbol && attempt.queryText === inferredSymbol && attempt.queryText !== query) {
|
|
18111
18718
|
decisions.fallbackFromOriginalConceptualToInferred = true;
|
|
18112
18719
|
}
|
|
18113
|
-
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed);
|
|
18720
|
+
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed, prioritizeSourcePaths);
|
|
18114
18721
|
if (results.length > 0) {
|
|
18115
18722
|
const heading = buildPackHeading("conceptual", decisions);
|
|
18116
18723
|
const intent = analyzeQueryIntent(attempt.queryText);
|
|
@@ -18247,12 +18854,13 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
18247
18854
|
directory: scope.directory,
|
|
18248
18855
|
trace
|
|
18249
18856
|
}),
|
|
18250
|
-
search: (queryText, retrievalLimit, scope, trace) => searchCodebase(projectRoot, host, queryText, {
|
|
18857
|
+
search: (queryText, retrievalLimit, scope, trace, searchOptions) => searchCodebase(projectRoot, host, queryText, {
|
|
18251
18858
|
limit: retrievalLimit,
|
|
18252
18859
|
fileType: scope.fileType,
|
|
18253
18860
|
directory: scope.directory,
|
|
18254
18861
|
metadataOnly: true,
|
|
18255
|
-
trace
|
|
18862
|
+
trace,
|
|
18863
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
18256
18864
|
})
|
|
18257
18865
|
});
|
|
18258
18866
|
}
|
|
@@ -18328,6 +18936,7 @@ async function executeCodebaseEditContext(projectRoot, host, args) {
|
|
|
18328
18936
|
async function executeIndexCodebase(projectRoot, host, args, onProgress) {
|
|
18329
18937
|
const result = await runIndexCodebase(projectRoot, host, args, onProgress);
|
|
18330
18938
|
if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) };
|
|
18939
|
+
if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) };
|
|
18331
18940
|
if (result.kind === "busy") return { text: result.text, isError: true };
|
|
18332
18941
|
if (result.kind === "message") return { text: result.text };
|
|
18333
18942
|
return { text: formatIndexStats(result.stats, args.verbose ?? false) };
|
|
@@ -19167,7 +19776,8 @@ var codebase_peek = tool({
|
|
|
19167
19776
|
chunkType: z3.enum(CHUNK_TYPE_VALUES).optional().describe("Filter by code chunk type"),
|
|
19168
19777
|
blameAuthor: z3.string().optional().describe("Filter by git blame author name or email"),
|
|
19169
19778
|
blameSha: z3.string().optional().describe("Filter by git blame commit SHA or prefix"),
|
|
19170
|
-
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)")
|
|
19779
|
+
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)"),
|
|
19780
|
+
blameUntil: z3.string().optional().describe("Filter to chunks last changed on or before this date (e.g., 2025-01-31)")
|
|
19171
19781
|
},
|
|
19172
19782
|
async execute(args, context) {
|
|
19173
19783
|
return searchCodebaseWithEffectiveness(context?.worktree, DEFAULT_HOST, "peek", args.query, {
|
|
@@ -19178,7 +19788,8 @@ var codebase_peek = tool({
|
|
|
19178
19788
|
metadataOnly: true,
|
|
19179
19789
|
blameAuthor: args.blameAuthor,
|
|
19180
19790
|
blameSha: args.blameSha,
|
|
19181
|
-
blameSince: args.blameSince
|
|
19791
|
+
blameSince: args.blameSince,
|
|
19792
|
+
blameUntil: args.blameUntil
|
|
19182
19793
|
}, (results) => {
|
|
19183
19794
|
const text = formatCodebasePeek(results);
|
|
19184
19795
|
return { output: text, text };
|
|
@@ -19190,6 +19801,7 @@ var index_codebase = tool({
|
|
|
19190
19801
|
args: {
|
|
19191
19802
|
force: z3.boolean().optional().default(false).describe("Force reindex even if already indexed"),
|
|
19192
19803
|
estimateOnly: z3.boolean().optional().default(false).describe("Only show cost estimate without indexing"),
|
|
19804
|
+
dryRun: z3.boolean().optional().default(false).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."),
|
|
19193
19805
|
verbose: z3.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures")
|
|
19194
19806
|
},
|
|
19195
19807
|
async execute(args, context) {
|
|
@@ -19240,7 +19852,9 @@ var find_similar = tool({
|
|
|
19240
19852
|
fileType: z3.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
19241
19853
|
directory: z3.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
19242
19854
|
chunkType: z3.enum(CHUNK_TYPE_VALUES).optional().describe("Filter by code chunk type"),
|
|
19243
|
-
excludeFile: z3.string().optional().describe("Exclude results from this file path (useful when searching for duplicates of code from a specific file)")
|
|
19855
|
+
excludeFile: z3.string().optional().describe("Exclude results from this file path (useful when searching for duplicates of code from a specific file)"),
|
|
19856
|
+
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)"),
|
|
19857
|
+
blameUntil: z3.string().optional().describe("Filter to chunks last changed on or before this date (e.g., 2025-01-31)")
|
|
19244
19858
|
},
|
|
19245
19859
|
async execute(args, context) {
|
|
19246
19860
|
const results = await findSimilarCode(context?.worktree, DEFAULT_HOST, args.code, {
|
|
@@ -19248,7 +19862,9 @@ var find_similar = tool({
|
|
|
19248
19862
|
fileType: args.fileType,
|
|
19249
19863
|
directory: args.directory,
|
|
19250
19864
|
chunkType: args.chunkType,
|
|
19251
|
-
excludeFile: args.excludeFile
|
|
19865
|
+
excludeFile: args.excludeFile,
|
|
19866
|
+
blameSince: args.blameSince,
|
|
19867
|
+
blameUntil: args.blameUntil
|
|
19252
19868
|
});
|
|
19253
19869
|
if (results.length === 0) {
|
|
19254
19870
|
return "No similar code found. Try a different snippet or run index_codebase first.";
|
|
@@ -19267,7 +19883,8 @@ var codebase_search = tool({
|
|
|
19267
19883
|
contextLines: z3.number().optional().describe("Number of extra lines to include before/after each match (default: 0)"),
|
|
19268
19884
|
blameAuthor: z3.string().optional().describe("Filter by git blame author name or email"),
|
|
19269
19885
|
blameSha: z3.string().optional().describe("Filter by git blame commit SHA or prefix"),
|
|
19270
|
-
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)")
|
|
19886
|
+
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)"),
|
|
19887
|
+
blameUntil: z3.string().optional().describe("Filter to chunks last changed on or before this date (e.g., 2025-01-31)")
|
|
19271
19888
|
},
|
|
19272
19889
|
async execute(args, context) {
|
|
19273
19890
|
return searchCodebaseWithEffectiveness(context?.worktree, DEFAULT_HOST, "search", args.query, {
|
|
@@ -19278,7 +19895,8 @@ var codebase_search = tool({
|
|
|
19278
19895
|
contextLines: args.contextLines,
|
|
19279
19896
|
blameAuthor: args.blameAuthor,
|
|
19280
19897
|
blameSha: args.blameSha,
|
|
19281
|
-
blameSince: args.blameSince
|
|
19898
|
+
blameSince: args.blameSince,
|
|
19899
|
+
blameUntil: args.blameUntil
|
|
19282
19900
|
}, (results) => {
|
|
19283
19901
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : formatSearchResults(results, "score");
|
|
19284
19902
|
return { output: text, text };
|
|
@@ -19491,6 +20109,12 @@ var PI_TOOL_NAMES = [
|
|
|
19491
20109
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
19492
20110
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
19493
20111
|
];
|
|
20112
|
+
var MCP_TOOL_NAMES = [
|
|
20113
|
+
...PORTABLE_TOOL_NAMES,
|
|
20114
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
20115
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
20116
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE
|
|
20117
|
+
];
|
|
19494
20118
|
|
|
19495
20119
|
// src/commands/loader.ts
|
|
19496
20120
|
var import_fs16 = require("fs");
|
|
@@ -19778,6 +20402,7 @@ function assessRoutingIntent(text) {
|
|
|
19778
20402
|
};
|
|
19779
20403
|
}
|
|
19780
20404
|
function buildRoutingHint(assessment, status, includeGraphHandoff = false) {
|
|
20405
|
+
const hasSymbolCue = hasIdentifierShape(assessment.text) || containsQuotedIdentifier(assessment.text);
|
|
19781
20406
|
if (assessment.intent === "definition_lookup") {
|
|
19782
20407
|
if (!status || !status.indexed || status.compatibility?.compatible === false) {
|
|
19783
20408
|
return "For this turn, if you need a symbol definition, check `index_status` first and run `index_codebase` if the index is missing or incompatible. Then use `implementation_lookup` for the definition site. Use `grep` for exhaustive literal matches.";
|
|
@@ -19787,12 +20412,13 @@ function buildRoutingHint(assessment, status, includeGraphHandoff = false) {
|
|
|
19787
20412
|
if (assessment.intent !== "local_conceptual" && assessment.intent !== "local_broad_task") {
|
|
19788
20413
|
return null;
|
|
19789
20414
|
}
|
|
20415
|
+
const preEditHint = assessment.intent === "local_broad_task" && hasSymbolCue ? " If a likely target symbol is already known or strongly suspected, consider optional `codebase_edit_context` as a compact pre-edit next step for bounded source plus direct callers and callees." : "";
|
|
19790
20416
|
if (!status || !status.indexed || status.compatibility?.compatible === false) {
|
|
19791
20417
|
const graphHandoff2 = includeGraphHandoff ? " Use graph tools after semantic discovery identifies relevant symbols." : "";
|
|
19792
|
-
return `For this turn, if local code discovery by behavior is needed, check \`index_status\` first and run \`index_codebase\` if the index is missing or incompatible.${graphHandoff2} Then use \`codebase_context\` as the first local repository lookup. Use \`grep\` for exact identifiers or exhaustive matches
|
|
20418
|
+
return `For this turn, if local code discovery by behavior is needed, check \`index_status\` first and run \`index_codebase\` if the index is missing or incompatible.${graphHandoff2} Then use \`codebase_context\` as the first local repository lookup. Use \`grep\` for exact identifiers or exhaustive matches.${preEditHint}`;
|
|
19793
20419
|
}
|
|
19794
20420
|
const graphHandoff = includeGraphHandoff ? " before graph tools such as `call_graph`, `call_graph_path`, `pr_impact`, or OMO CodeGraph" : "";
|
|
19795
|
-
return `For this turn, prefer \`codebase_context\` for local code discovery, then use \`codebase_peek\` for metadata and \`codebase_search\` when you need implementation content${graphHandoff}. Use \`grep\` for exact identifiers or exhaustive matches
|
|
20421
|
+
return `For this turn, prefer \`codebase_context\` for local code discovery, then use \`codebase_peek\` for metadata and \`codebase_search\` when you need implementation content${graphHandoff}. Use \`grep\` for exact identifiers or exhaustive matches.${preEditHint}`;
|
|
19796
20422
|
}
|
|
19797
20423
|
var RoutingHintController = class {
|
|
19798
20424
|
constructor(getStatus, maxSessions = 200, includeGraphHandoff = false) {
|
|
@@ -19834,7 +20460,7 @@ var RoutingHintController = class {
|
|
|
19834
20460
|
if (!state || !state.pendingHint) {
|
|
19835
20461
|
return;
|
|
19836
20462
|
}
|
|
19837
|
-
if (toolName === "codebase_context" || toolName === "codebase_peek" || toolName === "codebase_search" || toolName === "implementation_lookup" || toolName === "index_status" || toolName === "index_codebase") {
|
|
20463
|
+
if (toolName === "codebase_context" || toolName === "codebase_edit_context" || toolName === "codebase_peek" || toolName === "codebase_search" || toolName === "implementation_lookup" || toolName === "index_status" || toolName === "index_codebase") {
|
|
19838
20464
|
state.pendingHint = false;
|
|
19839
20465
|
state.updatedAt = Date.now();
|
|
19840
20466
|
this.sessionState.set(sessionID, state);
|