opencode-codebase-index 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +16 -5
- package/commands/peek.md +2 -1
- package/commands/search.md +2 -1
- package/dist/cbi.cjs +14792 -0
- package/dist/cbi.cjs.map +1 -0
- package/dist/cbi.js +14786 -0
- package/dist/cbi.js.map +1 -0
- package/dist/cli.cjs +1121 -263
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1121 -263
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +885 -259
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +885 -259
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +5529 -2154
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +5517 -2142
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +5 -2
- package/skill/SKILL.md +12 -16
package/dist/index.js
CHANGED
|
@@ -712,6 +712,17 @@ var EMBEDDING_MODELS = {
|
|
|
712
712
|
maxTokens: 2048,
|
|
713
713
|
costPer1MTokens: 0.15,
|
|
714
714
|
taskAble: true
|
|
715
|
+
},
|
|
716
|
+
"gemini-embedding-2": {
|
|
717
|
+
provider: "google",
|
|
718
|
+
model: "gemini-embedding-2",
|
|
719
|
+
// Keep a conservative, testable default embedding dimension. Gemini Embedding 2 supports
|
|
720
|
+
// flexible dimensions via outputDimensionality.
|
|
721
|
+
dimensions: 1536,
|
|
722
|
+
maxTokens: 8192,
|
|
723
|
+
costPer1MTokens: 0.15,
|
|
724
|
+
taskAble: false,
|
|
725
|
+
promptStyle: "embedding-2"
|
|
715
726
|
}
|
|
716
727
|
},
|
|
717
728
|
"openai": {
|
|
@@ -745,26 +756,15 @@ var EMBEDDING_MODELS = {
|
|
|
745
756
|
maxTokens: 512,
|
|
746
757
|
costPer1MTokens: 0
|
|
747
758
|
}
|
|
748
|
-
},
|
|
749
|
-
"github-copilot": {
|
|
750
|
-
"text-embedding-3-small": {
|
|
751
|
-
provider: "github-copilot",
|
|
752
|
-
model: "text-embedding-3-small",
|
|
753
|
-
dimensions: 1536,
|
|
754
|
-
maxTokens: 8191,
|
|
755
|
-
costPer1MTokens: 0
|
|
756
|
-
}
|
|
757
759
|
}
|
|
758
760
|
};
|
|
759
761
|
var DEFAULT_PROVIDER_MODELS = {
|
|
760
|
-
"github-copilot": "text-embedding-3-small",
|
|
761
762
|
"openai": "text-embedding-3-small",
|
|
762
763
|
"google": "gemini-embedding-001",
|
|
763
764
|
"ollama": "nomic-embed-text"
|
|
764
765
|
};
|
|
765
766
|
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
766
767
|
"ollama",
|
|
767
|
-
"github-copilot",
|
|
768
768
|
"openai",
|
|
769
769
|
"google"
|
|
770
770
|
];
|
|
@@ -790,6 +790,9 @@ function getDefaultIndexingConfig() {
|
|
|
790
790
|
maxDepth: 5,
|
|
791
791
|
maxFilesPerDirectory: 100,
|
|
792
792
|
fallbackToTextOnMaxChunks: true,
|
|
793
|
+
// Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi
|
|
794
|
+
// fallback used when a native caller omits the argument).
|
|
795
|
+
linesPerChunk: 30,
|
|
793
796
|
gitBlame: { enabled: false }
|
|
794
797
|
};
|
|
795
798
|
}
|
|
@@ -923,6 +926,7 @@ function parseConfig(raw) {
|
|
|
923
926
|
maxDepth: typeof rawIndexing.maxDepth === "number" ? rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth : defaultIndexing.maxDepth,
|
|
924
927
|
maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
|
|
925
928
|
fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
|
|
929
|
+
linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk,
|
|
926
930
|
gitBlame: {
|
|
927
931
|
enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof rawIndexing.gitBlame.enabled === "boolean" ? rawIndexing.gitBlame.enabled : defaultIndexing.gitBlame.enabled
|
|
928
932
|
}
|
|
@@ -965,6 +969,7 @@ function parseConfig(raw) {
|
|
|
965
969
|
let embeddingModel;
|
|
966
970
|
let customProvider;
|
|
967
971
|
let reranker;
|
|
972
|
+
const githubCopilotDeprecationMessage = '`embeddingProvider: "github-copilot"` is deprecated and no longer available. Migrate existing configs to `embeddingProvider: "google"` and select an explicit Google model. For existing indexes, run `index_codebase` with `force: true` after changing to `gemini-embedding-001` or `gemini-embedding-2` to rebuild embeddings. See docs/configuration.md for details.';
|
|
968
973
|
if (embeddingProviderValue === "custom") {
|
|
969
974
|
embeddingProvider = "custom";
|
|
970
975
|
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
@@ -1004,6 +1009,8 @@ function parseConfig(raw) {
|
|
|
1004
1009
|
} else if (rawEmbeddingModel) {
|
|
1005
1010
|
embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
1006
1011
|
}
|
|
1012
|
+
} else if (embeddingProviderValue === "github-copilot") {
|
|
1013
|
+
throw new Error(githubCopilotDeprecationMessage);
|
|
1007
1014
|
} else {
|
|
1008
1015
|
embeddingProvider = "auto";
|
|
1009
1016
|
}
|
|
@@ -1034,10 +1041,21 @@ function parseConfig(raw) {
|
|
|
1034
1041
|
timeoutMs: typeof rawReranker.timeoutMs === "number" ? Math.max(1e3, Math.floor(rawReranker.timeoutMs)) : 1e4
|
|
1035
1042
|
};
|
|
1036
1043
|
}
|
|
1044
|
+
const rawEmbedding = input.embedding && typeof input.embedding === "object" ? input.embedding : {};
|
|
1045
|
+
const rawEmbeddingBatch = rawEmbedding.batch && typeof rawEmbedding.batch === "object" ? rawEmbedding.batch : null;
|
|
1046
|
+
const embeddingMaxBatchItems = typeof rawEmbeddingBatch?.maxBatchItems === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchItems) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchItems)) : void 0;
|
|
1047
|
+
const embeddingMaxBatchTokens = typeof rawEmbeddingBatch?.maxBatchTokens === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchTokens) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchTokens)) : void 0;
|
|
1048
|
+
const embedding = embeddingMaxBatchItems !== void 0 || embeddingMaxBatchTokens !== void 0 ? {
|
|
1049
|
+
batch: {
|
|
1050
|
+
...embeddingMaxBatchItems !== void 0 ? { maxBatchItems: embeddingMaxBatchItems } : {},
|
|
1051
|
+
...embeddingMaxBatchTokens !== void 0 ? { maxBatchTokens: embeddingMaxBatchTokens } : {}
|
|
1052
|
+
}
|
|
1053
|
+
} : {};
|
|
1037
1054
|
return {
|
|
1038
1055
|
embeddingProvider,
|
|
1039
1056
|
embeddingModel,
|
|
1040
1057
|
customProvider,
|
|
1058
|
+
embedding,
|
|
1041
1059
|
scope: isValidScope(scopeValue) ? scopeValue : "project",
|
|
1042
1060
|
include: includeValue ?? DEFAULT_INCLUDE,
|
|
1043
1061
|
exclude: excludeValue ?? DEFAULT_EXCLUDE,
|
|
@@ -2304,6 +2322,10 @@ function scoreCandidate(query, intent, candidate, originalIndex) {
|
|
|
2304
2322
|
let boost = 0;
|
|
2305
2323
|
if (intent.primary === "conceptual") {
|
|
2306
2324
|
boost += Math.min(0.14, overlap * 0.14);
|
|
2325
|
+
if (intent.preferSourcePaths) {
|
|
2326
|
+
boost += implementationPath ? 0.32 : 0;
|
|
2327
|
+
if (testPath || fixturePath || docsPath) boost -= 0.35;
|
|
2328
|
+
}
|
|
2307
2329
|
if (generatedOrVendor) boost -= 0.18;
|
|
2308
2330
|
if (importChunk || weakContainer) boost -= 0.04;
|
|
2309
2331
|
} else if (intent.primary === "test") {
|
|
@@ -3281,6 +3303,19 @@ function parseOwner(value) {
|
|
|
3281
3303
|
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
3282
3304
|
if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
|
|
3283
3305
|
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
3306
|
+
if (candidate.recoveryProtocolVersion !== void 0 && candidate.recoveryProtocolVersion !== 1) return null;
|
|
3307
|
+
if (candidate.projectRoot !== void 0 && typeof candidate.projectRoot !== "string") return null;
|
|
3308
|
+
if (candidate.scopedRoots !== void 0) {
|
|
3309
|
+
if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) {
|
|
3310
|
+
return null;
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
if (candidate.clearRecovery !== void 0) {
|
|
3314
|
+
const recovery = candidate.clearRecovery;
|
|
3315
|
+
if (typeof recovery !== "object" || recovery === null || recovery.phase !== "clearing" || typeof recovery.embeddingProvider !== "string" || recovery.embeddingProvider.length === 0 || typeof recovery.embeddingModel !== "string" || recovery.embeddingModel.length === 0 || !Number.isInteger(recovery.embeddingDimensions) || (recovery.embeddingDimensions ?? 0) <= 0 || typeof recovery.embeddingStrategyVersion !== "string" || recovery.embeddingStrategyVersion.length === 0 || recovery.compatibilityDecision !== "compatible" && recovery.compatibilityDecision !== "embedding-strategy-mismatch" && recovery.compatibilityDecision !== "incompatible" || candidate.operation !== "clear" && candidate.operation !== "force-index") {
|
|
3316
|
+
return null;
|
|
3317
|
+
}
|
|
3318
|
+
}
|
|
3284
3319
|
return candidate;
|
|
3285
3320
|
}
|
|
3286
3321
|
function parseReclaimOwner(value) {
|
|
@@ -3521,13 +3556,18 @@ function isTransientIndexLockContention(error) {
|
|
|
3521
3556
|
if (!isIndexLockContentionError(error) || !("reason" in error)) return false;
|
|
3522
3557
|
return error.reason === "active" || error.reason === "reclaiming";
|
|
3523
3558
|
}
|
|
3524
|
-
function acquireIndexLock(indexPath, operation) {
|
|
3559
|
+
function acquireIndexLock(indexPath, operation, recoveryScope) {
|
|
3525
3560
|
mkdirSync(indexPath, { recursive: true });
|
|
3526
3561
|
const canonicalIndexPath = realpathSync2.native(indexPath);
|
|
3527
3562
|
const lockPath = path9.join(canonicalIndexPath, "indexing.lock");
|
|
3528
3563
|
cleanupDeadPublicationCandidates(canonicalIndexPath);
|
|
3529
3564
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
3530
|
-
const owner = createOwner(operation)
|
|
3565
|
+
const owner = recoveryScope === void 0 ? createOwner(operation) : {
|
|
3566
|
+
...createOwner(operation),
|
|
3567
|
+
recoveryProtocolVersion: 1,
|
|
3568
|
+
projectRoot: recoveryScope.projectRoot,
|
|
3569
|
+
scopedRoots: recoveryScope.scopedRoots
|
|
3570
|
+
};
|
|
3531
3571
|
if (publishJsonDirectory(lockPath, owner)) {
|
|
3532
3572
|
const lease = {
|
|
3533
3573
|
canonicalIndexPath,
|
|
@@ -3592,6 +3632,33 @@ function releaseIndexLock(lease) {
|
|
|
3592
3632
|
}
|
|
3593
3633
|
return true;
|
|
3594
3634
|
}
|
|
3635
|
+
function setIndexLockClearRecoveryState(lease, clearRecovery) {
|
|
3636
|
+
const currentOwner = readDirectoryOwner(lease.lockPath);
|
|
3637
|
+
if (!currentOwner || !sameOwner(currentOwner, lease.owner)) {
|
|
3638
|
+
throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
3639
|
+
}
|
|
3640
|
+
const nextOwner = { ...currentOwner };
|
|
3641
|
+
if (clearRecovery === null) {
|
|
3642
|
+
delete nextOwner.clearRecovery;
|
|
3643
|
+
} else {
|
|
3644
|
+
nextOwner.clearRecovery = clearRecovery;
|
|
3645
|
+
}
|
|
3646
|
+
const ownerPath = path9.join(lease.lockPath, OWNER_FILE_NAME);
|
|
3647
|
+
const temporaryPath = path9.join(
|
|
3648
|
+
lease.lockPath,
|
|
3649
|
+
`${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${randomUUID()}`
|
|
3650
|
+
);
|
|
3651
|
+
try {
|
|
3652
|
+
writeFileSync(temporaryPath, JSON.stringify(nextOwner), {
|
|
3653
|
+
encoding: "utf-8",
|
|
3654
|
+
flag: "wx",
|
|
3655
|
+
mode: 384
|
|
3656
|
+
});
|
|
3657
|
+
retryTransientFilesystemOperation(() => renameSync(temporaryPath, ownerPath));
|
|
3658
|
+
} finally {
|
|
3659
|
+
if (existsSync5(temporaryPath)) rmSync(temporaryPath, { force: true });
|
|
3660
|
+
}
|
|
3661
|
+
}
|
|
3595
3662
|
function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
|
|
3596
3663
|
if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
|
|
3597
3664
|
temporaryCounter += 1;
|
|
@@ -5735,8 +5802,6 @@ async function tryDetectProvider() {
|
|
|
5735
5802
|
}
|
|
5736
5803
|
async function getProviderCredentials(provider) {
|
|
5737
5804
|
switch (provider) {
|
|
5738
|
-
case "github-copilot":
|
|
5739
|
-
return getGitHubCopilotCredentials();
|
|
5740
5805
|
case "openai":
|
|
5741
5806
|
return getOpenAICredentials();
|
|
5742
5807
|
case "google":
|
|
@@ -5747,22 +5812,6 @@ async function getProviderCredentials(provider) {
|
|
|
5747
5812
|
return null;
|
|
5748
5813
|
}
|
|
5749
5814
|
}
|
|
5750
|
-
function getGitHubCopilotCredentials() {
|
|
5751
|
-
const authData = loadOpenCodeAuth();
|
|
5752
|
-
const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
|
|
5753
|
-
if (!copilotAuth || copilotAuth.type !== "oauth") {
|
|
5754
|
-
return null;
|
|
5755
|
-
}
|
|
5756
|
-
const auth = copilotAuth;
|
|
5757
|
-
const baseUrl = auth.enterpriseUrl ? `https://copilot-api.${auth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
|
|
5758
|
-
return {
|
|
5759
|
-
provider: "github-copilot",
|
|
5760
|
-
baseUrl,
|
|
5761
|
-
refreshToken: copilotAuth.refresh,
|
|
5762
|
-
accessToken: copilotAuth.access,
|
|
5763
|
-
tokenExpires: copilotAuth.expires
|
|
5764
|
-
};
|
|
5765
|
-
}
|
|
5766
5815
|
function getOpenAICredentials() {
|
|
5767
5816
|
const authData = loadOpenCodeAuth();
|
|
5768
5817
|
const openaiAuth = authData["openai"];
|
|
@@ -5888,8 +5937,6 @@ async function tryDetectOllamaProvider() {
|
|
|
5888
5937
|
}
|
|
5889
5938
|
function getProviderDisplayName(provider) {
|
|
5890
5939
|
switch (provider) {
|
|
5891
|
-
case "github-copilot":
|
|
5892
|
-
return "GitHub Copilot";
|
|
5893
5940
|
case "openai":
|
|
5894
5941
|
return "OpenAI";
|
|
5895
5942
|
case "google":
|
|
@@ -6114,44 +6161,6 @@ var CustomEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
6114
6161
|
}
|
|
6115
6162
|
};
|
|
6116
6163
|
|
|
6117
|
-
// src/embeddings/providers/github-copilot.ts
|
|
6118
|
-
var GitHubCopilotEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
6119
|
-
constructor(credentials, modelInfo) {
|
|
6120
|
-
super(credentials, modelInfo);
|
|
6121
|
-
}
|
|
6122
|
-
getToken() {
|
|
6123
|
-
if (!this.credentials.refreshToken) {
|
|
6124
|
-
throw new Error("No OAuth token available for GitHub");
|
|
6125
|
-
}
|
|
6126
|
-
return this.credentials.refreshToken;
|
|
6127
|
-
}
|
|
6128
|
-
async embedBatch(texts) {
|
|
6129
|
-
const token = this.getToken();
|
|
6130
|
-
const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
|
|
6131
|
-
method: "POST",
|
|
6132
|
-
headers: {
|
|
6133
|
-
Authorization: `Bearer ${token}`,
|
|
6134
|
-
"Content-Type": "application/json",
|
|
6135
|
-
Accept: "application/vnd.github+json",
|
|
6136
|
-
"X-GitHub-Api-Version": "2022-11-28"
|
|
6137
|
-
},
|
|
6138
|
-
body: JSON.stringify({
|
|
6139
|
-
model: `openai/${this.modelInfo.model}`,
|
|
6140
|
-
input: texts
|
|
6141
|
-
})
|
|
6142
|
-
});
|
|
6143
|
-
if (!response.ok) {
|
|
6144
|
-
const error = (await response.text()).slice(0, 500);
|
|
6145
|
-
throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
|
|
6146
|
-
}
|
|
6147
|
-
const data = await response.json();
|
|
6148
|
-
return {
|
|
6149
|
-
embeddings: data.data.map((d) => d.embedding),
|
|
6150
|
-
totalTokensUsed: data.usage.total_tokens
|
|
6151
|
-
};
|
|
6152
|
-
}
|
|
6153
|
-
};
|
|
6154
|
-
|
|
6155
6164
|
// src/embeddings/providers/google.ts
|
|
6156
6165
|
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddingProvider {
|
|
6157
6166
|
static BATCH_SIZE = 20;
|
|
@@ -6159,24 +6168,30 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
6159
6168
|
super(credentials, modelInfo);
|
|
6160
6169
|
}
|
|
6161
6170
|
async embedQuery(query) {
|
|
6162
|
-
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
6163
|
-
const
|
|
6171
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
6172
|
+
const texts = [
|
|
6173
|
+
this.modelInfo.model === "gemini-embedding-2" ? `task: code retrieval | query: ${query}` : query
|
|
6174
|
+
];
|
|
6175
|
+
const result = await this.embedWithTaskType(texts, taskType);
|
|
6164
6176
|
return {
|
|
6165
6177
|
embedding: result.embeddings[0],
|
|
6166
6178
|
tokensUsed: result.totalTokensUsed
|
|
6167
6179
|
};
|
|
6168
6180
|
}
|
|
6169
6181
|
async embedDocument(document) {
|
|
6170
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6171
|
-
const result = await this.embedWithTaskType([
|
|
6182
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6183
|
+
const result = await this.embedWithTaskType([
|
|
6184
|
+
this.modelInfo.model === "gemini-embedding-2" ? `title: none | text: ${document}` : document
|
|
6185
|
+
], taskType);
|
|
6172
6186
|
return {
|
|
6173
6187
|
embedding: result.embeddings[0],
|
|
6174
6188
|
tokensUsed: result.totalTokensUsed
|
|
6175
6189
|
};
|
|
6176
6190
|
}
|
|
6177
6191
|
async embedBatch(texts) {
|
|
6178
|
-
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6179
|
-
|
|
6192
|
+
const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
6193
|
+
const formattedTexts = this.modelInfo.model === "gemini-embedding-2" ? texts.map((text) => `title: none | text: ${text}`) : texts;
|
|
6194
|
+
return this.embedWithTaskType(formattedTexts, taskType);
|
|
6180
6195
|
}
|
|
6181
6196
|
async embedWithTaskType(texts, taskType) {
|
|
6182
6197
|
const batches = [];
|
|
@@ -6226,6 +6241,10 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
|
|
|
6226
6241
|
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddingProvider {
|
|
6227
6242
|
static MIN_TRUNCATION_CHARS = 512;
|
|
6228
6243
|
static REQUEST_TIMEOUT_MS = 12e4;
|
|
6244
|
+
// Set when /api/embed returns 404 so subsequent multi-text batches skip the
|
|
6245
|
+
// batched endpoint and go straight to the legacy per-text path (one probe per
|
|
6246
|
+
// old ollama install, not one probe per batch).
|
|
6247
|
+
batchEndpointUnavailable = false;
|
|
6229
6248
|
constructor(credentials, modelInfo) {
|
|
6230
6249
|
super(credentials, modelInfo);
|
|
6231
6250
|
}
|
|
@@ -6243,6 +6262,21 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
6243
6262
|
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
6244
6263
|
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
6245
6264
|
}
|
|
6265
|
+
// True for a 404 from the newer /api/embed endpoint, i.e. an ollama version that
|
|
6266
|
+
// does not provide it. embedBatch uses this to fall back to the legacy per-text
|
|
6267
|
+
// /api/embeddings path so old ollama installs do not regress.
|
|
6268
|
+
isBatchEndpointUnavailableError(error) {
|
|
6269
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6270
|
+
return message.includes("Ollama /api/embed not available");
|
|
6271
|
+
}
|
|
6272
|
+
// True for a malformed /api/embed response (wrong vector count or a bad vector).
|
|
6273
|
+
// embedBatch falls back to the per-text path on this so a bad batch response
|
|
6274
|
+
// re-embeds each text cleanly. A text that then fails per-text is not isolated
|
|
6275
|
+
// here; it is isolated on the recovery run, which re-embeds one text per request.
|
|
6276
|
+
isBatchValidationError(error) {
|
|
6277
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6278
|
+
return message.includes("invalid embedding batch");
|
|
6279
|
+
}
|
|
6246
6280
|
buildTruncationCandidates(text) {
|
|
6247
6281
|
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
6248
6282
|
const candidateLimits = /* @__PURE__ */ new Set();
|
|
@@ -6344,7 +6378,74 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
6344
6378
|
tokensUsed: this.estimateTokens(text)
|
|
6345
6379
|
};
|
|
6346
6380
|
}
|
|
6347
|
-
|
|
6381
|
+
// Embeds many texts in one POST /api/embed request (input: string[]). Ollama
|
|
6382
|
+
// encodes each input independently, so the model context length applies per input
|
|
6383
|
+
// (the upstream splitter already bounds each input), not over the batch. This
|
|
6384
|
+
// amortizes N HTTP round-trips into one.
|
|
6385
|
+
async embedMany(texts) {
|
|
6386
|
+
const controller = new AbortController();
|
|
6387
|
+
const timeout = setTimeout(
|
|
6388
|
+
() => controller.abort(),
|
|
6389
|
+
_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS
|
|
6390
|
+
);
|
|
6391
|
+
let response;
|
|
6392
|
+
try {
|
|
6393
|
+
response = await fetch(`${this.credentials.baseUrl}/api/embed`, {
|
|
6394
|
+
method: "POST",
|
|
6395
|
+
headers: {
|
|
6396
|
+
"Content-Type": "application/json"
|
|
6397
|
+
},
|
|
6398
|
+
body: JSON.stringify({
|
|
6399
|
+
model: this.modelInfo.model,
|
|
6400
|
+
input: texts,
|
|
6401
|
+
truncate: false
|
|
6402
|
+
}),
|
|
6403
|
+
signal: controller.signal
|
|
6404
|
+
});
|
|
6405
|
+
} catch (error) {
|
|
6406
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
6407
|
+
throw new Error(
|
|
6408
|
+
`Ollama embedding request timed out after ${_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS}ms`
|
|
6409
|
+
);
|
|
6410
|
+
}
|
|
6411
|
+
throw error;
|
|
6412
|
+
} finally {
|
|
6413
|
+
clearTimeout(timeout);
|
|
6414
|
+
}
|
|
6415
|
+
if (!response.ok) {
|
|
6416
|
+
const error = (await response.text()).slice(0, 500);
|
|
6417
|
+
if (response.status === 404) {
|
|
6418
|
+
throw new Error(`Ollama /api/embed not available: ${response.status} - ${error}`);
|
|
6419
|
+
}
|
|
6420
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
6421
|
+
}
|
|
6422
|
+
let parsed;
|
|
6423
|
+
try {
|
|
6424
|
+
parsed = await response.json();
|
|
6425
|
+
} catch {
|
|
6426
|
+
throw new Error(
|
|
6427
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
6428
|
+
);
|
|
6429
|
+
}
|
|
6430
|
+
const data = parsed && typeof parsed === "object" ? parsed : {};
|
|
6431
|
+
if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length || data.embeddings.some(
|
|
6432
|
+
(value) => !Array.isArray(value) || value.length !== this.modelInfo.dimensions || value.some((v) => typeof v !== "number" || !Number.isFinite(v))
|
|
6433
|
+
)) {
|
|
6434
|
+
throw new Error(
|
|
6435
|
+
`Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
|
|
6436
|
+
);
|
|
6437
|
+
}
|
|
6438
|
+
return {
|
|
6439
|
+
embeddings: data.embeddings,
|
|
6440
|
+
totalTokensUsed: texts.reduce((sum, text) => sum + this.estimateTokens(text), 0)
|
|
6441
|
+
};
|
|
6442
|
+
}
|
|
6443
|
+
// Per-text /api/embeddings path shared by the single-text fast path and the
|
|
6444
|
+
// batch fallback. Uses the legacy endpoint one text at a time, so each text gets
|
|
6445
|
+
// its own truncation safety net and a vector validated on its own. A text that
|
|
6446
|
+
// hard-fails per-text throws here and fails the whole request batch; the recovery
|
|
6447
|
+
// run re-embeds one text per request to isolate it.
|
|
6448
|
+
async embedOneByOne(texts) {
|
|
6348
6449
|
const results = [];
|
|
6349
6450
|
for (const text of texts) {
|
|
6350
6451
|
results.push(await this.embedSingleWithFallback(text));
|
|
@@ -6354,6 +6455,26 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
|
|
|
6354
6455
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
6355
6456
|
};
|
|
6356
6457
|
}
|
|
6458
|
+
async embedBatch(texts) {
|
|
6459
|
+
if (texts.length === 0) {
|
|
6460
|
+
return { embeddings: [], totalTokensUsed: 0 };
|
|
6461
|
+
}
|
|
6462
|
+
if (texts.length === 1 || this.batchEndpointUnavailable) {
|
|
6463
|
+
return this.embedOneByOne(texts);
|
|
6464
|
+
}
|
|
6465
|
+
try {
|
|
6466
|
+
return await this.embedMany(texts);
|
|
6467
|
+
} catch (error) {
|
|
6468
|
+
if (this.isBatchEndpointUnavailableError(error)) {
|
|
6469
|
+
this.batchEndpointUnavailable = true;
|
|
6470
|
+
return this.embedOneByOne(texts);
|
|
6471
|
+
}
|
|
6472
|
+
if (!this.isContextLengthError(error) && !this.isBatchValidationError(error)) {
|
|
6473
|
+
throw error;
|
|
6474
|
+
}
|
|
6475
|
+
return this.embedOneByOne(texts);
|
|
6476
|
+
}
|
|
6477
|
+
}
|
|
6357
6478
|
};
|
|
6358
6479
|
|
|
6359
6480
|
// src/embeddings/providers/openai.ts
|
|
@@ -6388,8 +6509,6 @@ var OpenAIEmbeddingProvider = class extends BaseEmbeddingProvider {
|
|
|
6388
6509
|
// src/embeddings/provider.ts
|
|
6389
6510
|
function createEmbeddingProvider(configuredProviderInfo) {
|
|
6390
6511
|
switch (configuredProviderInfo.provider) {
|
|
6391
|
-
case "github-copilot":
|
|
6392
|
-
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
6393
6512
|
case "openai":
|
|
6394
6513
|
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
6395
6514
|
case "google":
|
|
@@ -6457,6 +6576,26 @@ function formatCostEstimate(estimate) {
|
|
|
6457
6576
|
\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
|
|
6458
6577
|
`;
|
|
6459
6578
|
}
|
|
6579
|
+
function formatDryRunEstimate(estimate) {
|
|
6580
|
+
return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed.
|
|
6581
|
+
|
|
6582
|
+
Files to embed: ${estimate.filesCount.toLocaleString()}
|
|
6583
|
+
Chunks to embed: ${estimate.chunksCount.toLocaleString()}
|
|
6584
|
+
Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()}
|
|
6585
|
+
|
|
6586
|
+
The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It
|
|
6587
|
+
matches the live "Tokens used" counter only for providers that report usage on the
|
|
6588
|
+
same basis (ollama); for providers that report a server tokenizer count (OpenAI,
|
|
6589
|
+
Gemini, custom) it is only an estimate.
|
|
6590
|
+
|
|
6591
|
+
For a matching provider and a project-scoped force index, the force pass clears its
|
|
6592
|
+
own cached embeddings, so the live counter climbs to this number. A force index on a
|
|
6593
|
+
shared global index can reuse cached embeddings from other projects, and an
|
|
6594
|
+
incremental index counts cached chunks that are not re-embedded; in both cases this
|
|
6595
|
+
number is an upper bound on the live counter, so a progress percent against this
|
|
6596
|
+
total tops out below 100%.
|
|
6597
|
+
`;
|
|
6598
|
+
}
|
|
6460
6599
|
function formatBytes(bytes) {
|
|
6461
6600
|
if (bytes === 0) return "0 B";
|
|
6462
6601
|
const k = 1024;
|
|
@@ -7140,12 +7279,12 @@ try {
|
|
|
7140
7279
|
}
|
|
7141
7280
|
|
|
7142
7281
|
// src/native/parsing.ts
|
|
7143
|
-
function parseFileAsText(filePath, content) {
|
|
7144
|
-
const result = native.parseFileAsText(filePath, content);
|
|
7282
|
+
function parseFileAsText(filePath, content, linesPerChunk) {
|
|
7283
|
+
const result = native.parseFileAsText(filePath, content, linesPerChunk);
|
|
7145
7284
|
return result.map(mapChunk);
|
|
7146
7285
|
}
|
|
7147
|
-
function parseFiles(files) {
|
|
7148
|
-
const result = native.parseFiles(files);
|
|
7286
|
+
function parseFiles(files, linesPerChunk) {
|
|
7287
|
+
const result = native.parseFiles(files, linesPerChunk);
|
|
7149
7288
|
return result.map((f) => ({
|
|
7150
7289
|
path: f.path,
|
|
7151
7290
|
chunks: f.chunks.map(mapChunk),
|
|
@@ -7222,13 +7361,13 @@ var VectorStore = class {
|
|
|
7222
7361
|
const metadata = items.map((i) => JSON.stringify(i.metadata));
|
|
7223
7362
|
this.inner.addBatch(ids, vectors, metadata);
|
|
7224
7363
|
}
|
|
7225
|
-
search(queryVector, limit = 10) {
|
|
7364
|
+
search(queryVector, limit = 10, allowedIds) {
|
|
7226
7365
|
if (queryVector.length !== this.dimensions) {
|
|
7227
7366
|
throw new Error(
|
|
7228
7367
|
`Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
|
|
7229
7368
|
);
|
|
7230
7369
|
}
|
|
7231
|
-
const results = this.inner.search(queryVector, limit);
|
|
7370
|
+
const results = allowedIds === void 0 ? this.inner.search(queryVector, limit) : this.inner.searchFiltered(queryVector, limit, allowedIds);
|
|
7232
7371
|
return results.map((r) => ({
|
|
7233
7372
|
id: r.id,
|
|
7234
7373
|
score: r.score,
|
|
@@ -7456,6 +7595,10 @@ var Database = class _Database {
|
|
|
7456
7595
|
this.throwIfClosed();
|
|
7457
7596
|
return this.inner.getBranchChunkIds(branch);
|
|
7458
7597
|
}
|
|
7598
|
+
getChunkIdsByBlameDate(since, until) {
|
|
7599
|
+
this.throwIfClosed();
|
|
7600
|
+
return this.inner.getChunkIdsByBlameDate(since, until);
|
|
7601
|
+
}
|
|
7459
7602
|
getBranchDelta(branch, baseBranch) {
|
|
7460
7603
|
this.throwIfClosed();
|
|
7461
7604
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
@@ -8727,6 +8870,9 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
|
8727
8870
|
"enum_declaration",
|
|
8728
8871
|
"function_definition",
|
|
8729
8872
|
"class_definition",
|
|
8873
|
+
// Ruby module/class symbols that are declaration-bearing and navigable.
|
|
8874
|
+
"class",
|
|
8875
|
+
"module",
|
|
8730
8876
|
"class_specifier",
|
|
8731
8877
|
"struct_specifier",
|
|
8732
8878
|
"namespace_definition",
|
|
@@ -9344,6 +9490,18 @@ function createFailedBatchWriter(targetPath) {
|
|
|
9344
9490
|
temporaryPath
|
|
9345
9491
|
};
|
|
9346
9492
|
}
|
|
9493
|
+
function writeFailedBatchRecords(targetPath, records) {
|
|
9494
|
+
const writer = createFailedBatchWriter(targetPath);
|
|
9495
|
+
try {
|
|
9496
|
+
for (const record of records) {
|
|
9497
|
+
writer.write(record);
|
|
9498
|
+
}
|
|
9499
|
+
writer.commit();
|
|
9500
|
+
} catch (error) {
|
|
9501
|
+
writer.cleanup();
|
|
9502
|
+
throw error;
|
|
9503
|
+
}
|
|
9504
|
+
}
|
|
9347
9505
|
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
9348
9506
|
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
9349
9507
|
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
@@ -9626,14 +9784,18 @@ function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
|
9626
9784
|
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
9627
9785
|
return Math.min(2e3, maxChunkTokens);
|
|
9628
9786
|
}
|
|
9629
|
-
|
|
9630
|
-
|
|
9631
|
-
|
|
9632
|
-
|
|
9633
|
-
|
|
9634
|
-
};
|
|
9787
|
+
var DEFAULT_OLLAMA_MAX_BATCH_ITEMS = 16;
|
|
9788
|
+
var DEFAULT_OLLAMA_MAX_BATCH_TOKENS = 65536;
|
|
9789
|
+
function getDynamicBatchOptions(provider, embeddingBatch) {
|
|
9790
|
+
if (provider.provider !== "ollama") {
|
|
9791
|
+
return {};
|
|
9635
9792
|
}
|
|
9636
|
-
|
|
9793
|
+
const base = { maxBatchTokens: DEFAULT_OLLAMA_MAX_BATCH_TOKENS, maxBatchItems: DEFAULT_OLLAMA_MAX_BATCH_ITEMS };
|
|
9794
|
+
return {
|
|
9795
|
+
...base,
|
|
9796
|
+
...typeof embeddingBatch?.maxBatchTokens === "number" && Number.isFinite(embeddingBatch.maxBatchTokens) ? { maxBatchTokens: embeddingBatch.maxBatchTokens } : {},
|
|
9797
|
+
...typeof embeddingBatch?.maxBatchItems === "number" && Number.isFinite(embeddingBatch.maxBatchItems) ? { maxBatchItems: embeddingBatch.maxBatchItems } : {}
|
|
9798
|
+
};
|
|
9637
9799
|
}
|
|
9638
9800
|
function isSqliteCorruptionError(error) {
|
|
9639
9801
|
const message = getErrorMessage4(error).toLowerCase();
|
|
@@ -9651,6 +9813,14 @@ function getPendingChunkId(rawChunk) {
|
|
|
9651
9813
|
const id = rawChunk.id;
|
|
9652
9814
|
return typeof id === "string" ? id : null;
|
|
9653
9815
|
}
|
|
9816
|
+
function parseBlameTimestamp(value, endOfDay) {
|
|
9817
|
+
let timestampMs = Date.parse(value);
|
|
9818
|
+
if (Number.isNaN(timestampMs)) return null;
|
|
9819
|
+
if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(value.trim())) {
|
|
9820
|
+
timestampMs += 24 * 60 * 60 * 1e3 - 1;
|
|
9821
|
+
}
|
|
9822
|
+
return Math.floor(timestampMs / 1e3);
|
|
9823
|
+
}
|
|
9654
9824
|
function metadataFromBlame(blame) {
|
|
9655
9825
|
if (!blame) {
|
|
9656
9826
|
return {};
|
|
@@ -9797,7 +9967,7 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
|
|
|
9797
9967
|
const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
|
|
9798
9968
|
return [...promoted, ...remainder];
|
|
9799
9969
|
}
|
|
9800
|
-
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
9970
|
+
function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source", allowNonSourcePaths = false) {
|
|
9801
9971
|
if (!prioritizeSourcePaths) {
|
|
9802
9972
|
return [];
|
|
9803
9973
|
}
|
|
@@ -9817,7 +9987,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
9817
9987
|
if (!isImplementationChunkType(chunkType)) {
|
|
9818
9988
|
return false;
|
|
9819
9989
|
}
|
|
9820
|
-
if (!isLikelyImplementationPath2(chunk.filePath)) {
|
|
9990
|
+
if (!allowNonSourcePaths && !isLikelyImplementationPath2(chunk.filePath)) {
|
|
9821
9991
|
return false;
|
|
9822
9992
|
}
|
|
9823
9993
|
const nameLower = (chunk.name ?? "").toLowerCase();
|
|
@@ -9881,7 +10051,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
9881
10051
|
}
|
|
9882
10052
|
foundCoveringChunk = upsertChunkCandidate(chunk, identifier, normalizedIdentifier) || foundCoveringChunk;
|
|
9883
10053
|
}
|
|
9884
|
-
if (foundCoveringChunk || !isLikelyImplementationPath2(symbol.filePath)) {
|
|
10054
|
+
if (foundCoveringChunk || !allowNonSourcePaths && !isLikelyImplementationPath2(symbol.filePath)) {
|
|
9885
10055
|
continue;
|
|
9886
10056
|
}
|
|
9887
10057
|
const symbolName = symbol.name.toLowerCase();
|
|
@@ -9935,7 +10105,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
|
|
|
9935
10105
|
const ranked = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
9936
10106
|
if (ranked.length === 0) {
|
|
9937
10107
|
const implementationFallback = fallbackCandidates.filter(
|
|
9938
|
-
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && isLikelyImplementationPath2(candidate.metadata.filePath)
|
|
10108
|
+
(candidate) => isImplementationChunkType(candidate.metadata.chunkType) && (allowNonSourcePaths || isLikelyImplementationPath2(candidate.metadata.filePath))
|
|
9939
10109
|
);
|
|
9940
10110
|
for (const candidate of implementationFallback) {
|
|
9941
10111
|
const nameLower = (candidate.metadata.name ?? "").toLowerCase();
|
|
@@ -10051,10 +10221,16 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
10051
10221
|
return false;
|
|
10052
10222
|
}
|
|
10053
10223
|
if (options?.blameSince) {
|
|
10054
|
-
const
|
|
10055
|
-
if (
|
|
10224
|
+
const since = parseBlameTimestamp(options.blameSince, false);
|
|
10225
|
+
if (since === null) return false;
|
|
10056
10226
|
const committedAt = candidate.metadata.blameCommittedAt;
|
|
10057
|
-
if (committedAt === void 0 || committedAt <
|
|
10227
|
+
if (committedAt === void 0 || committedAt < since) return false;
|
|
10228
|
+
}
|
|
10229
|
+
if (options?.blameUntil) {
|
|
10230
|
+
const until = parseBlameTimestamp(options.blameUntil, true);
|
|
10231
|
+
if (until === null) return false;
|
|
10232
|
+
const committedAt = candidate.metadata.blameCommittedAt;
|
|
10233
|
+
if (committedAt === void 0 || committedAt > until) return false;
|
|
10058
10234
|
}
|
|
10059
10235
|
return true;
|
|
10060
10236
|
}
|
|
@@ -10110,9 +10286,10 @@ var Indexer = class _Indexer {
|
|
|
10110
10286
|
writerArtifactFingerprint = null;
|
|
10111
10287
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
10112
10288
|
fileBatchLimits;
|
|
10289
|
+
checkpointIntervalChunks;
|
|
10113
10290
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
10114
10291
|
this.projectRoot = projectRoot;
|
|
10115
|
-
this.projectIdentityHash =
|
|
10292
|
+
this.projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
10116
10293
|
this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot;
|
|
10117
10294
|
this.branchNameOverride = runtimeOptions.branchName;
|
|
10118
10295
|
this.catalogIdentityOverride = runtimeOptions.catalogIdentity;
|
|
@@ -10122,6 +10299,7 @@ var Indexer = class _Indexer {
|
|
|
10122
10299
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
10123
10300
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
10124
10301
|
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
10302
|
+
this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks;
|
|
10125
10303
|
this.config = config;
|
|
10126
10304
|
this.host = host;
|
|
10127
10305
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -10233,6 +10411,9 @@ var Indexer = class _Indexer {
|
|
|
10233
10411
|
return path19.resolve(targetPath);
|
|
10234
10412
|
}
|
|
10235
10413
|
}
|
|
10414
|
+
getProjectIdentityHash(projectRoot) {
|
|
10415
|
+
return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
10416
|
+
}
|
|
10236
10417
|
isProjectOwnedIndexPath() {
|
|
10237
10418
|
return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host);
|
|
10238
10419
|
}
|
|
@@ -10269,7 +10450,10 @@ var Indexer = class _Indexer {
|
|
|
10269
10450
|
}
|
|
10270
10451
|
async withIndexMutationLease(operation, callback) {
|
|
10271
10452
|
this.refreshBranchInfo();
|
|
10272
|
-
const lease = acquireIndexLock(this.indexPath, operation
|
|
10453
|
+
const lease = acquireIndexLock(this.indexPath, operation, {
|
|
10454
|
+
projectRoot: this.projectRoot,
|
|
10455
|
+
scopedRoots: this.getScopedRoots()
|
|
10456
|
+
});
|
|
10273
10457
|
this.indexPath = lease.canonicalIndexPath;
|
|
10274
10458
|
this.refreshRuntimeArtifactPaths();
|
|
10275
10459
|
this.activeIndexLease = lease;
|
|
@@ -10324,6 +10508,7 @@ var Indexer = class _Indexer {
|
|
|
10324
10508
|
}
|
|
10325
10509
|
loadFileHashCache() {
|
|
10326
10510
|
if (!existsSync11(this.fileHashCachePath)) {
|
|
10511
|
+
this.fileHashCache = /* @__PURE__ */ new Map();
|
|
10327
10512
|
return;
|
|
10328
10513
|
}
|
|
10329
10514
|
try {
|
|
@@ -10363,10 +10548,10 @@ var Indexer = class _Indexer {
|
|
|
10363
10548
|
invertedIndex.serialize()
|
|
10364
10549
|
);
|
|
10365
10550
|
}
|
|
10366
|
-
getScopedRoots() {
|
|
10367
|
-
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(
|
|
10551
|
+
getScopedRoots(projectRoot = this.projectRoot) {
|
|
10552
|
+
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(projectRoot)]);
|
|
10368
10553
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
10369
|
-
roots.add(this.getCanonicalPath(path19.resolve(
|
|
10554
|
+
roots.add(this.getCanonicalPath(path19.resolve(projectRoot, kbRoot)));
|
|
10370
10555
|
}
|
|
10371
10556
|
return Array.from(roots);
|
|
10372
10557
|
}
|
|
@@ -10437,14 +10622,17 @@ var Indexer = class _Indexer {
|
|
|
10437
10622
|
getLegacyBranchCatalogKey() {
|
|
10438
10623
|
return this.currentBranch || "default";
|
|
10439
10624
|
}
|
|
10440
|
-
getLegacyMigrationMetadataKey() {
|
|
10441
|
-
return `index.globalBranchMigration.${
|
|
10625
|
+
getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10626
|
+
return `index.globalBranchMigration.${projectIdentityHash}`;
|
|
10627
|
+
}
|
|
10628
|
+
getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10629
|
+
return `index.embeddingStrategyVersion.${projectIdentityHash}`;
|
|
10442
10630
|
}
|
|
10443
|
-
|
|
10444
|
-
return `index.
|
|
10631
|
+
getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10632
|
+
return `index.forceReembed.${projectIdentityHash}`;
|
|
10445
10633
|
}
|
|
10446
|
-
|
|
10447
|
-
return `index.
|
|
10634
|
+
getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
|
|
10635
|
+
return `index.migrationFinalized.${projectIdentityHash}`;
|
|
10448
10636
|
}
|
|
10449
10637
|
getBranchMigrationMetadataKey(prefix, catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
10450
10638
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
@@ -10550,7 +10738,7 @@ var Indexer = class _Indexer {
|
|
|
10550
10738
|
const legacy = this.getLegacyBranchCatalogKey();
|
|
10551
10739
|
return primary === legacy ? [primary] : [primary, legacy];
|
|
10552
10740
|
}
|
|
10553
|
-
getProjectLocalScopedOwnershipIds(roots) {
|
|
10741
|
+
getProjectLocalScopedOwnershipIds(roots, projectRoot = this.projectRoot) {
|
|
10554
10742
|
const chunkIds = /* @__PURE__ */ new Set();
|
|
10555
10743
|
const symbolIds = /* @__PURE__ */ new Set();
|
|
10556
10744
|
if (!this.database) {
|
|
@@ -10558,10 +10746,10 @@ var Indexer = class _Indexer {
|
|
|
10558
10746
|
}
|
|
10559
10747
|
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
10560
10748
|
...Array.from(this.fileHashCache.keys()).filter(
|
|
10561
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
10749
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
10562
10750
|
),
|
|
10563
10751
|
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
10564
|
-
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
|
|
10752
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
|
|
10565
10753
|
)
|
|
10566
10754
|
]);
|
|
10567
10755
|
for (const filePath of projectLocalFilePaths) {
|
|
@@ -10574,15 +10762,16 @@ var Indexer = class _Indexer {
|
|
|
10574
10762
|
}
|
|
10575
10763
|
return { chunkIds, symbolIds };
|
|
10576
10764
|
}
|
|
10577
|
-
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
10765
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds, projectRoot = this.projectRoot) {
|
|
10578
10766
|
if (this.config.scope !== "global") {
|
|
10579
10767
|
return this.getBranchCatalogCleanupKeys();
|
|
10580
10768
|
}
|
|
10581
10769
|
const keys = /* @__PURE__ */ new Set();
|
|
10582
10770
|
const projectChunkIdSet = new Set(projectChunkIds);
|
|
10583
10771
|
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
10772
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
10584
10773
|
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
10585
|
-
if (branchKey.startsWith(`${
|
|
10774
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
10586
10775
|
keys.add(branchKey);
|
|
10587
10776
|
continue;
|
|
10588
10777
|
}
|
|
@@ -10592,8 +10781,10 @@ var Indexer = class _Indexer {
|
|
|
10592
10781
|
keys.add(branchKey);
|
|
10593
10782
|
}
|
|
10594
10783
|
}
|
|
10595
|
-
|
|
10596
|
-
|
|
10784
|
+
if (projectRoot === this.projectRoot) {
|
|
10785
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
10786
|
+
keys.add(branchKey);
|
|
10787
|
+
}
|
|
10597
10788
|
}
|
|
10598
10789
|
return Array.from(keys);
|
|
10599
10790
|
}
|
|
@@ -10601,10 +10792,10 @@ var Indexer = class _Indexer {
|
|
|
10601
10792
|
const canonicalFilePath = this.getCanonicalStoredFilePath(filePath);
|
|
10602
10793
|
return roots.some((root) => isPathWithinRoot2(canonicalFilePath, root));
|
|
10603
10794
|
}
|
|
10604
|
-
isFileInProjectRoot(filePath) {
|
|
10795
|
+
isFileInProjectRoot(filePath, projectRoot = this.projectRoot) {
|
|
10605
10796
|
return isPathWithinRoot2(
|
|
10606
10797
|
this.getCanonicalStoredFilePath(filePath),
|
|
10607
|
-
this.getCanonicalPath(
|
|
10798
|
+
this.getCanonicalPath(projectRoot)
|
|
10608
10799
|
);
|
|
10609
10800
|
}
|
|
10610
10801
|
clearScopedFileHashCache(roots) {
|
|
@@ -10646,12 +10837,12 @@ var Indexer = class _Indexer {
|
|
|
10646
10837
|
}
|
|
10647
10838
|
return false;
|
|
10648
10839
|
}
|
|
10649
|
-
hasForeignScopedBranchData() {
|
|
10840
|
+
hasForeignScopedBranchData(projectRoot = this.projectRoot, roots = this.getScopedRoots(projectRoot)) {
|
|
10650
10841
|
if (!this.database || this.config.scope !== "global") {
|
|
10651
10842
|
return false;
|
|
10652
10843
|
}
|
|
10653
|
-
const
|
|
10654
|
-
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
10844
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
10845
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot);
|
|
10655
10846
|
return this.database.getAllBranches().some(
|
|
10656
10847
|
(branchKey) => {
|
|
10657
10848
|
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
@@ -10660,7 +10851,7 @@ var Indexer = class _Indexer {
|
|
|
10660
10851
|
if (!hasBranchData) {
|
|
10661
10852
|
return false;
|
|
10662
10853
|
}
|
|
10663
|
-
if (branchKey.startsWith(`${
|
|
10854
|
+
if (branchKey.startsWith(`${projectIdentityHash}:`)) {
|
|
10664
10855
|
return false;
|
|
10665
10856
|
}
|
|
10666
10857
|
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
@@ -10669,7 +10860,7 @@ var Indexer = class _Indexer {
|
|
|
10669
10860
|
}
|
|
10670
10861
|
);
|
|
10671
10862
|
}
|
|
10672
|
-
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
10863
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot = this.projectRoot) {
|
|
10673
10864
|
const allMetadata = store.getAllMetadata();
|
|
10674
10865
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
10675
10866
|
const filePaths = /* @__PURE__ */ new Set([
|
|
@@ -10677,7 +10868,7 @@ var Indexer = class _Indexer {
|
|
|
10677
10868
|
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
10678
10869
|
]);
|
|
10679
10870
|
const projectLocalFilePaths = new Set(
|
|
10680
|
-
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath))
|
|
10871
|
+
Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot))
|
|
10681
10872
|
);
|
|
10682
10873
|
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
10683
10874
|
for (const filePath of filePaths) {
|
|
@@ -10687,7 +10878,7 @@ var Indexer = class _Indexer {
|
|
|
10687
10878
|
}
|
|
10688
10879
|
const removedChunkIdList = Array.from(removedChunkIds);
|
|
10689
10880
|
const projectLocalChunkIds = new Set(
|
|
10690
|
-
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)).map(({ key }) => key)
|
|
10881
|
+
scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)).map(({ key }) => key)
|
|
10691
10882
|
);
|
|
10692
10883
|
for (const filePath of projectLocalFilePaths) {
|
|
10693
10884
|
for (const chunk of database.getChunksByFile(filePath)) {
|
|
@@ -10706,7 +10897,8 @@ var Indexer = class _Indexer {
|
|
|
10706
10897
|
}
|
|
10707
10898
|
const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys(
|
|
10708
10899
|
Array.from(projectLocalChunkIds),
|
|
10709
|
-
Array.from(projectLocalSymbolIds)
|
|
10900
|
+
Array.from(projectLocalSymbolIds),
|
|
10901
|
+
projectRoot
|
|
10710
10902
|
);
|
|
10711
10903
|
for (const branchKey of branchCleanupKeys) {
|
|
10712
10904
|
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
@@ -10741,29 +10933,96 @@ var Indexer = class _Indexer {
|
|
|
10741
10933
|
database.gcOrphanSymbols();
|
|
10742
10934
|
database.gcOrphanEmbeddings();
|
|
10743
10935
|
database.gcOrphanChunks();
|
|
10744
|
-
store.save();
|
|
10745
10936
|
this.saveInvertedIndex(invertedIndex);
|
|
10937
|
+
store.save();
|
|
10746
10938
|
return {
|
|
10747
10939
|
removedChunkIds: removedChunkIdList,
|
|
10748
10940
|
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
10749
10941
|
};
|
|
10750
10942
|
}
|
|
10943
|
+
getCurrentClearRecoveryState() {
|
|
10944
|
+
if (!this.configuredProviderInfo) {
|
|
10945
|
+
throw new Error("Cannot persist clear recovery state before the embedding provider is initialized");
|
|
10946
|
+
}
|
|
10947
|
+
const compatibility = this.checkCompatibility();
|
|
10948
|
+
const compatibilityDecision = compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible";
|
|
10949
|
+
return {
|
|
10950
|
+
phase: "clearing",
|
|
10951
|
+
embeddingProvider: this.configuredProviderInfo.provider,
|
|
10952
|
+
embeddingModel: this.configuredProviderInfo.modelInfo.model,
|
|
10953
|
+
embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions,
|
|
10954
|
+
embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION,
|
|
10955
|
+
compatibilityDecision
|
|
10956
|
+
};
|
|
10957
|
+
}
|
|
10958
|
+
beginClearRecoveryState() {
|
|
10959
|
+
const recovery = this.getCurrentClearRecoveryState();
|
|
10960
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), recovery);
|
|
10961
|
+
return recovery;
|
|
10962
|
+
}
|
|
10963
|
+
finishClearRecoveryState() {
|
|
10964
|
+
setIndexLockClearRecoveryState(this.requireActiveLease(), null);
|
|
10965
|
+
}
|
|
10966
|
+
matchesCurrentClearRecoveryConfiguration(recovery) {
|
|
10967
|
+
const configuredProviderInfo = this.configuredProviderInfo;
|
|
10968
|
+
return configuredProviderInfo !== null && recovery.embeddingProvider === configuredProviderInfo.provider && recovery.embeddingModel === configuredProviderInfo.modelInfo.model && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION;
|
|
10969
|
+
}
|
|
10970
|
+
hasUnknownLegacyForceIndexClear(owner) {
|
|
10971
|
+
return owner.operation === "force-index" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1 && existsSync11(path19.join(this.indexPath, "force-index-phase"));
|
|
10972
|
+
}
|
|
10751
10973
|
async recoverFromInterruptedIndexingUnlocked(owners) {
|
|
10752
10974
|
for (const owner of owners) {
|
|
10753
10975
|
this.logger.warn("Detected interrupted indexing session, recovering...", {
|
|
10754
10976
|
pid: owner.pid,
|
|
10755
10977
|
hostname: owner.hostname,
|
|
10756
10978
|
operation: owner.operation,
|
|
10757
|
-
startedAt: owner.startedAt
|
|
10979
|
+
startedAt: owner.startedAt,
|
|
10980
|
+
projectRoot: owner.projectRoot
|
|
10758
10981
|
});
|
|
10759
10982
|
}
|
|
10760
10983
|
if (this.config.scope === "global") {
|
|
10761
|
-
|
|
10762
|
-
|
|
10984
|
+
const clearScopes = [];
|
|
10985
|
+
for (const owner of owners) {
|
|
10986
|
+
if (this.hasUnknownLegacyForceIndexClear(owner)) {
|
|
10987
|
+
throw new Error(
|
|
10988
|
+
`Cannot automatically recover interrupted force-index ${owner.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
10989
|
+
);
|
|
10990
|
+
}
|
|
10991
|
+
if (owner.operation === "clear" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1) {
|
|
10992
|
+
throw new Error(
|
|
10993
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating recovery state is unknown. The recovery marker was retained for manual inspection.`
|
|
10994
|
+
);
|
|
10995
|
+
}
|
|
10996
|
+
if (owner.clearRecovery === void 0) continue;
|
|
10997
|
+
if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) {
|
|
10998
|
+
throw new Error(
|
|
10999
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the originating project scope is unknown. The recovery marker was retained for manual inspection.`
|
|
11000
|
+
);
|
|
11001
|
+
}
|
|
11002
|
+
if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) {
|
|
11003
|
+
throw new Error(
|
|
11004
|
+
`Cannot automatically recover interrupted global clear ${owner.token}: the current embedding configuration does not match the originating lease. The recovery marker was retained; retry from the originating project with matching settings.`
|
|
11005
|
+
);
|
|
11006
|
+
}
|
|
11007
|
+
clearScopes.push({
|
|
11008
|
+
projectRoot: owner.projectRoot,
|
|
11009
|
+
scopedRoots: owner.scopedRoots,
|
|
11010
|
+
compatibilityDecision: owner.clearRecovery.compatibilityDecision
|
|
11011
|
+
});
|
|
11012
|
+
}
|
|
11013
|
+
if (clearScopes.length > 0) {
|
|
11014
|
+
this.loadFileHashCache();
|
|
11015
|
+
}
|
|
11016
|
+
for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) {
|
|
11017
|
+
this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision);
|
|
10763
11018
|
}
|
|
10764
11019
|
await this.healthCheckUnlocked();
|
|
11020
|
+
this.logger.info(
|
|
11021
|
+
clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint"
|
|
11022
|
+
);
|
|
11023
|
+
return;
|
|
10765
11024
|
}
|
|
10766
|
-
this.logger.info("Recovery complete, next index will
|
|
11025
|
+
this.logger.info("Recovery complete, next index will resume from the last checkpoint");
|
|
10767
11026
|
}
|
|
10768
11027
|
*loadSerializedFailedBatches() {
|
|
10769
11028
|
let warned = false;
|
|
@@ -10801,14 +11060,99 @@ var Indexer = class _Indexer {
|
|
|
10801
11060
|
state.writer.write(record);
|
|
10802
11061
|
state.recordsWritten += record.chunks.length;
|
|
10803
11062
|
}
|
|
10804
|
-
finalizeFailedBatchWriteState(state) {
|
|
11063
|
+
finalizeFailedBatchWriteState(state, resolvedChunkIds = /* @__PURE__ */ new Set()) {
|
|
10805
11064
|
if (state.recordsWritten > 0) {
|
|
10806
|
-
|
|
11065
|
+
const seenChunkIds = /* @__PURE__ */ new Set();
|
|
11066
|
+
const retained = [];
|
|
11067
|
+
const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath));
|
|
11068
|
+
for (let i = records.length - 1; i >= 0; i--) {
|
|
11069
|
+
const chunks = records[i].chunks.filter((rawChunk) => {
|
|
11070
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11071
|
+
if (chunkId !== null) {
|
|
11072
|
+
if (resolvedChunkIds.has(chunkId)) return false;
|
|
11073
|
+
if (seenChunkIds.has(chunkId)) return false;
|
|
11074
|
+
seenChunkIds.add(chunkId);
|
|
11075
|
+
}
|
|
11076
|
+
return true;
|
|
11077
|
+
});
|
|
11078
|
+
if (chunks.length > 0) {
|
|
11079
|
+
retained.unshift({ ...records[i], chunks });
|
|
11080
|
+
}
|
|
11081
|
+
}
|
|
11082
|
+
state.writer.cleanup();
|
|
11083
|
+
if (retained.length > 0) {
|
|
11084
|
+
writeFailedBatchRecords(this.failedBatchesPath, retained);
|
|
11085
|
+
} else {
|
|
11086
|
+
writeFailedBatchRecords(this.failedBatchesPath, []);
|
|
11087
|
+
this.clearFailedBatchState();
|
|
11088
|
+
}
|
|
10807
11089
|
return;
|
|
10808
11090
|
}
|
|
10809
|
-
state.writer.
|
|
11091
|
+
state.writer.commit();
|
|
10810
11092
|
this.clearFailedBatchState();
|
|
10811
11093
|
}
|
|
11094
|
+
getCheckpointIntervalChunks(totalChunks) {
|
|
11095
|
+
return Math.max(
|
|
11096
|
+
this.checkpointIntervalChunks ?? 2e3,
|
|
11097
|
+
Math.floor(totalChunks / 10)
|
|
11098
|
+
);
|
|
11099
|
+
}
|
|
11100
|
+
checkpointIndexRun(database, store, invertedIndex, failedProcessing, resolvedRetryChunkIds, currentFileHashes, committedFilePaths, scopedRoots, configuredProviderInfo) {
|
|
11101
|
+
if (!this.hasProjectForceReembedPending()) {
|
|
11102
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
11103
|
+
this.indexCompatibility = { compatible: true };
|
|
11104
|
+
}
|
|
11105
|
+
database.commitWriteTransaction();
|
|
11106
|
+
database.beginWriteTransaction();
|
|
11107
|
+
this.saveInvertedIndex(invertedIndex);
|
|
11108
|
+
store.save();
|
|
11109
|
+
if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0 || failedProcessing.discardedExistingRecords) {
|
|
11110
|
+
for (const metadata of failedProcessing.latestById.values()) {
|
|
11111
|
+
const alreadyMaterialized = metadata.chunks.some((rawChunk) => {
|
|
11112
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11113
|
+
return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId);
|
|
11114
|
+
});
|
|
11115
|
+
if (alreadyMaterialized) continue;
|
|
11116
|
+
this.writeFailedBatchRecord(failedProcessing.state, {
|
|
11117
|
+
chunks: metadata.chunks,
|
|
11118
|
+
attemptCount: metadata.attemptCount,
|
|
11119
|
+
error: metadata.error,
|
|
11120
|
+
lastAttempt: metadata.lastAttempt
|
|
11121
|
+
});
|
|
11122
|
+
for (const rawChunk of metadata.chunks) {
|
|
11123
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11124
|
+
if (chunkId !== null) {
|
|
11125
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
11126
|
+
}
|
|
11127
|
+
}
|
|
11128
|
+
}
|
|
11129
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
11130
|
+
failedProcessing.state = this.createFailedBatchWriteState();
|
|
11131
|
+
failedProcessing.discardedExistingRecords = false;
|
|
11132
|
+
for (const record of this.loadSerializedFailedBatches()) {
|
|
11133
|
+
for (const rawChunk of record.chunks) {
|
|
11134
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
11135
|
+
this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] });
|
|
11136
|
+
if (chunkId !== null) {
|
|
11137
|
+
failedProcessing.materializedRetryIds.add(chunkId);
|
|
11138
|
+
}
|
|
11139
|
+
}
|
|
11140
|
+
}
|
|
11141
|
+
}
|
|
11142
|
+
const partialHashes = /* @__PURE__ */ new Map();
|
|
11143
|
+
for (const filePath of committedFilePaths) {
|
|
11144
|
+
const hash = currentFileHashes.get(filePath);
|
|
11145
|
+
if (hash !== void 0) {
|
|
11146
|
+
partialHashes.set(filePath, hash);
|
|
11147
|
+
}
|
|
11148
|
+
}
|
|
11149
|
+
if (scopedRoots) {
|
|
11150
|
+
this.replaceScopedFileHashCache(partialHashes, scopedRoots);
|
|
11151
|
+
} else {
|
|
11152
|
+
this.fileHashCache = partialHashes;
|
|
11153
|
+
this.saveFileHashCache();
|
|
11154
|
+
}
|
|
11155
|
+
}
|
|
10812
11156
|
clearFailedBatchState() {
|
|
10813
11157
|
if (existsSync11(this.failedBatchesPath)) {
|
|
10814
11158
|
try {
|
|
@@ -10835,6 +11179,7 @@ var Indexer = class _Indexer {
|
|
|
10835
11179
|
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
10836
11180
|
const state = this.createFailedBatchWriteState();
|
|
10837
11181
|
const latestById = /* @__PURE__ */ new Map();
|
|
11182
|
+
let discardedExistingRecords = false;
|
|
10838
11183
|
try {
|
|
10839
11184
|
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10840
11185
|
for (const rawChunk of batch.chunks) {
|
|
@@ -10845,10 +11190,12 @@ var Indexer = class _Indexer {
|
|
|
10845
11190
|
continue;
|
|
10846
11191
|
}
|
|
10847
11192
|
if (!shouldProcess(filePath)) {
|
|
11193
|
+
discardedExistingRecords = true;
|
|
10848
11194
|
continue;
|
|
10849
11195
|
}
|
|
10850
11196
|
const chunkId = getPendingChunkId(rawChunk);
|
|
10851
11197
|
if (!chunkId) {
|
|
11198
|
+
discardedExistingRecords = true;
|
|
10852
11199
|
continue;
|
|
10853
11200
|
}
|
|
10854
11201
|
const existing = latestById.get(chunkId);
|
|
@@ -10856,12 +11203,18 @@ var Indexer = class _Indexer {
|
|
|
10856
11203
|
latestById.set(chunkId, {
|
|
10857
11204
|
attemptCount: batch.attemptCount,
|
|
10858
11205
|
error: batch.error,
|
|
10859
|
-
lastAttempt: batch.lastAttempt
|
|
11206
|
+
lastAttempt: batch.lastAttempt,
|
|
11207
|
+
chunks: [rawChunk]
|
|
10860
11208
|
});
|
|
10861
11209
|
}
|
|
10862
11210
|
}
|
|
10863
11211
|
}
|
|
10864
|
-
return {
|
|
11212
|
+
return {
|
|
11213
|
+
state,
|
|
11214
|
+
latestById,
|
|
11215
|
+
materializedRetryIds: /* @__PURE__ */ new Set(),
|
|
11216
|
+
discardedExistingRecords
|
|
11217
|
+
};
|
|
10865
11218
|
} catch (error) {
|
|
10866
11219
|
state.writer.cleanup();
|
|
10867
11220
|
throw error;
|
|
@@ -10897,10 +11250,34 @@ var Indexer = class _Indexer {
|
|
|
10897
11250
|
}
|
|
10898
11251
|
}
|
|
10899
11252
|
}
|
|
11253
|
+
restoreMissingChunkRows(database, chunks) {
|
|
11254
|
+
const missing = [];
|
|
11255
|
+
for (const chunk of chunks) {
|
|
11256
|
+
if (database.getChunk(chunk.id)) {
|
|
11257
|
+
continue;
|
|
11258
|
+
}
|
|
11259
|
+
missing.push({
|
|
11260
|
+
chunkId: chunk.id,
|
|
11261
|
+
contentHash: chunk.contentHash,
|
|
11262
|
+
filePath: chunk.metadata.filePath,
|
|
11263
|
+
startLine: chunk.metadata.startLine,
|
|
11264
|
+
endLine: chunk.metadata.endLine,
|
|
11265
|
+
nodeType: chunk.metadata.chunkType,
|
|
11266
|
+
name: chunk.metadata.name,
|
|
11267
|
+
language: chunk.metadata.language,
|
|
11268
|
+
blameSha: chunk.metadata.blameSha,
|
|
11269
|
+
blameAuthor: chunk.metadata.blameAuthor,
|
|
11270
|
+
blameAuthorEmail: chunk.metadata.blameAuthorEmail,
|
|
11271
|
+
blameCommittedAt: chunk.metadata.blameCommittedAt,
|
|
11272
|
+
blameSummary: chunk.metadata.blameSummary
|
|
11273
|
+
});
|
|
11274
|
+
}
|
|
11275
|
+
if (missing.length > 0) {
|
|
11276
|
+
database.upsertChunksBatch(missing);
|
|
11277
|
+
}
|
|
11278
|
+
}
|
|
10900
11279
|
getProviderRateLimits(provider) {
|
|
10901
11280
|
switch (provider) {
|
|
10902
|
-
case "github-copilot":
|
|
10903
|
-
return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
|
|
10904
11281
|
case "openai":
|
|
10905
11282
|
return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
10906
11283
|
case "google":
|
|
@@ -10969,10 +11346,11 @@ var Indexer = class _Indexer {
|
|
|
10969
11346
|
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
10970
11347
|
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
10971
11348
|
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
10972
|
-
const
|
|
10973
|
-
|
|
10974
|
-
|
|
10975
|
-
|
|
11349
|
+
const batchOptions = getDynamicBatchOptions(options.configuredProviderInfo, this.config.embedding?.batch);
|
|
11350
|
+
if (options.forceSingleItemBatches && options.configuredProviderInfo.provider === "ollama") {
|
|
11351
|
+
batchOptions.maxBatchItems = 1;
|
|
11352
|
+
}
|
|
11353
|
+
const requestBatches = createPendingEmbeddingRequestBatches(chunksNeedingEmbedding, batchOptions);
|
|
10976
11354
|
let fatalError;
|
|
10977
11355
|
for (const requestBatch of requestBatches) {
|
|
10978
11356
|
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
@@ -11535,7 +11913,7 @@ var Indexer = class _Indexer {
|
|
|
11535
11913
|
}
|
|
11536
11914
|
if (!this.configuredProviderInfo) {
|
|
11537
11915
|
throw new Error(
|
|
11538
|
-
"No embedding provider available. Configure
|
|
11916
|
+
"No embedding provider available. Configure OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
11539
11917
|
);
|
|
11540
11918
|
}
|
|
11541
11919
|
this.logger.info("Initializing indexer", {
|
|
@@ -11566,7 +11944,20 @@ var Indexer = class _Indexer {
|
|
|
11566
11944
|
]);
|
|
11567
11945
|
}
|
|
11568
11946
|
if (recoveredOwners.length > 0 && this.config.scope === "project") {
|
|
11569
|
-
|
|
11947
|
+
const unknownLegacyForceIndex = recoveredOwners.find(
|
|
11948
|
+
(owner) => this.hasUnknownLegacyForceIndexClear(owner)
|
|
11949
|
+
);
|
|
11950
|
+
if (unknownLegacyForceIndex) {
|
|
11951
|
+
throw new Error(
|
|
11952
|
+
`Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
|
|
11953
|
+
);
|
|
11954
|
+
}
|
|
11955
|
+
const shouldReset = recoveredOwners.some(
|
|
11956
|
+
(owner) => owner.clearRecovery !== void 0 || owner.operation === "clear" && owner.recoveryProtocolVersion !== 1
|
|
11957
|
+
);
|
|
11958
|
+
if (shouldReset) {
|
|
11959
|
+
await this.resetLocalIndexArtifacts();
|
|
11960
|
+
}
|
|
11570
11961
|
}
|
|
11571
11962
|
this.store = new VectorStore(storePath, dimensions);
|
|
11572
11963
|
if (existsSync11(storePath) || existsSync11(vectorMetadataPath)) {
|
|
@@ -12082,6 +12473,70 @@ var Indexer = class _Indexer {
|
|
|
12082
12473
|
);
|
|
12083
12474
|
return createCostEstimate(files, configuredProviderInfo);
|
|
12084
12475
|
}
|
|
12476
|
+
// Dry-run counterpart to index()/forceIndex(): parse the real file set and sum
|
|
12477
|
+
// estimateTokens over the embedding text of every indexable chunk, without
|
|
12478
|
+
// calling the embedding provider or writing to the index. Read-only and
|
|
12479
|
+
// lock-free (mirrors estimateCost). The token sum is the exact value "Tokens
|
|
12480
|
+
// used" climbs to for a force index (cache bypassed); for an incremental it is
|
|
12481
|
+
// an upper bound because cached chunks are counted here but not re-embedded.
|
|
12482
|
+
// Used by index_codebase(dryRun:true) to give a stable, monotonic progress
|
|
12483
|
+
// denominator that matches the live "Tokens used" basis.
|
|
12484
|
+
async dryRunCost() {
|
|
12485
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
12486
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
12487
|
+
const includePatterns = [...this.config.include, ...this.config.additionalInclude];
|
|
12488
|
+
const { files } = await collectFiles(
|
|
12489
|
+
this.materializedProjectRoot,
|
|
12490
|
+
includePatterns,
|
|
12491
|
+
this.config.exclude,
|
|
12492
|
+
this.config.indexing.maxFileSize,
|
|
12493
|
+
this.getMaterializedKnowledgeBases(),
|
|
12494
|
+
{ maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }
|
|
12495
|
+
);
|
|
12496
|
+
let filesCount = 0;
|
|
12497
|
+
let chunksCount = 0;
|
|
12498
|
+
let tokensToEmbed = 0;
|
|
12499
|
+
for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) {
|
|
12500
|
+
const loadedFiles = await Promise.all(batch.map(async (f) => {
|
|
12501
|
+
try {
|
|
12502
|
+
return {
|
|
12503
|
+
path: this.toStoredFilePath(f.path),
|
|
12504
|
+
content: await fsPromises3.readFile(f.path, "utf-8")
|
|
12505
|
+
};
|
|
12506
|
+
} catch {
|
|
12507
|
+
return null;
|
|
12508
|
+
}
|
|
12509
|
+
}));
|
|
12510
|
+
const readable = loadedFiles.filter(
|
|
12511
|
+
(f) => f !== null
|
|
12512
|
+
);
|
|
12513
|
+
filesCount += readable.length;
|
|
12514
|
+
const contentByPath = new Map(readable.map((f) => [f.path, f.content]));
|
|
12515
|
+
const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk);
|
|
12516
|
+
for (const parsed of parsedFiles) {
|
|
12517
|
+
let chunksToProcess = parsed.chunks;
|
|
12518
|
+
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
12519
|
+
const content = contentByPath.get(parsed.path);
|
|
12520
|
+
if (content !== void 0) {
|
|
12521
|
+
chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk);
|
|
12522
|
+
}
|
|
12523
|
+
}
|
|
12524
|
+
chunksToProcess = selectIndexableChunks(
|
|
12525
|
+
chunksToProcess,
|
|
12526
|
+
this.config.indexing.maxChunksPerFile,
|
|
12527
|
+
this.config.indexing.semanticOnly
|
|
12528
|
+
);
|
|
12529
|
+
for (const chunk of chunksToProcess) {
|
|
12530
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens);
|
|
12531
|
+
chunksCount += 1;
|
|
12532
|
+
for (const text of texts) {
|
|
12533
|
+
tokensToEmbed += estimateTokens(text);
|
|
12534
|
+
}
|
|
12535
|
+
}
|
|
12536
|
+
}
|
|
12537
|
+
}
|
|
12538
|
+
return { filesCount, chunksCount, tokensToEmbed };
|
|
12539
|
+
}
|
|
12085
12540
|
async index(onProgress) {
|
|
12086
12541
|
return this.withIndexMutationLease("index", async (recoveredOwners) => {
|
|
12087
12542
|
return this.indexUnlocked(onProgress, recoveredOwners);
|
|
@@ -12202,7 +12657,17 @@ var Indexer = class _Indexer {
|
|
|
12202
12657
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
12203
12658
|
for (const file of files) {
|
|
12204
12659
|
const storedPath = this.toStoredFilePath(file.path);
|
|
12205
|
-
|
|
12660
|
+
let currentHash;
|
|
12661
|
+
try {
|
|
12662
|
+
currentHash = hashFile(file.path);
|
|
12663
|
+
} catch (error) {
|
|
12664
|
+
stats.skippedFiles.push({ path: this.toCanonicalFilePath(file.path), reason: "unreadable" });
|
|
12665
|
+
this.logger.warn("Skipped unreadable file during indexing", {
|
|
12666
|
+
path: file.path,
|
|
12667
|
+
error: getErrorMessage4(error)
|
|
12668
|
+
});
|
|
12669
|
+
continue;
|
|
12670
|
+
}
|
|
12206
12671
|
currentFileHashes.set(storedPath, currentHash);
|
|
12207
12672
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
12208
12673
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
@@ -12210,7 +12675,8 @@ var Indexer = class _Indexer {
|
|
|
12210
12675
|
);
|
|
12211
12676
|
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path19.extname(storedPath).toLowerCase() === ".swift";
|
|
12212
12677
|
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path19.extname(storedPath).toLowerCase() === ".metal";
|
|
12213
|
-
|
|
12678
|
+
const inMigrationScope = forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots);
|
|
12679
|
+
if (cachedHashMatches && !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
12214
12680
|
unchangedFilePaths.add(storedPath);
|
|
12215
12681
|
this.logger.recordCacheHit();
|
|
12216
12682
|
} else {
|
|
@@ -12336,6 +12802,9 @@ var Indexer = class _Indexer {
|
|
|
12336
12802
|
}
|
|
12337
12803
|
}
|
|
12338
12804
|
let processedChangedFiles = 0;
|
|
12805
|
+
let lastCheckpointChunks = 0;
|
|
12806
|
+
const committedFilePaths = new Set(unchangedFilePaths);
|
|
12807
|
+
const resolvedRetryChunkIds = /* @__PURE__ */ new Set();
|
|
12339
12808
|
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
12340
12809
|
changedFileDescriptors,
|
|
12341
12810
|
(descriptor) => descriptor.sourceBytes,
|
|
@@ -12349,7 +12818,7 @@ var Indexer = class _Indexer {
|
|
|
12349
12818
|
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
12350
12819
|
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
12351
12820
|
const parseStartTime = performance2.now();
|
|
12352
|
-
const parsedFiles = parseFiles(loadedFiles);
|
|
12821
|
+
const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk);
|
|
12353
12822
|
const parseMs = performance2.now() - parseStartTime;
|
|
12354
12823
|
this.logger.recordFilesParsed(parsedFiles.length);
|
|
12355
12824
|
this.logger.recordParseDuration(parseMs);
|
|
@@ -12372,7 +12841,7 @@ var Indexer = class _Indexer {
|
|
|
12372
12841
|
}
|
|
12373
12842
|
let chunksToProcess = parsed.chunks;
|
|
12374
12843
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
12375
|
-
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
12844
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk);
|
|
12376
12845
|
}
|
|
12377
12846
|
chunksToProcess = selectIndexableChunks(
|
|
12378
12847
|
chunksToProcess,
|
|
@@ -12506,6 +12975,10 @@ var Indexer = class _Indexer {
|
|
|
12506
12975
|
}
|
|
12507
12976
|
if (symbolBatch.length > 0) {
|
|
12508
12977
|
database.upsertSymbolsBatch(symbolBatch);
|
|
12978
|
+
database.addSymbolsToBranchBatch(
|
|
12979
|
+
this.getBranchCatalogKey(),
|
|
12980
|
+
symbolBatch.map((symbol) => symbol.id)
|
|
12981
|
+
);
|
|
12509
12982
|
}
|
|
12510
12983
|
if (edgeBatch.length > 0) {
|
|
12511
12984
|
database.upsertCallEdgesBatch(edgeBatch);
|
|
@@ -12541,6 +13014,12 @@ var Indexer = class _Indexer {
|
|
|
12541
13014
|
forceReembed: forceScopedReembed,
|
|
12542
13015
|
reuseCachedEmbeddings: true,
|
|
12543
13016
|
incrementRepeatedFailures: true,
|
|
13017
|
+
onSucceeded: (succeededChunks) => {
|
|
13018
|
+
database.addChunksToBranchBatch(
|
|
13019
|
+
this.getBranchCatalogKey(),
|
|
13020
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
13021
|
+
);
|
|
13022
|
+
},
|
|
12544
13023
|
onProgress: (batchProgress) => onProgress?.({
|
|
12545
13024
|
phase: "embedding",
|
|
12546
13025
|
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
@@ -12559,6 +13038,27 @@ var Indexer = class _Indexer {
|
|
|
12559
13038
|
}
|
|
12560
13039
|
}
|
|
12561
13040
|
}
|
|
13041
|
+
for (const descriptor of descriptorBatch) {
|
|
13042
|
+
const existingFileChunks = existingChunksByFile.get(descriptor.storedPath);
|
|
13043
|
+
if (!existingFileChunks || existingFileChunks.size === 0) {
|
|
13044
|
+
committedFilePaths.add(descriptor.storedPath);
|
|
13045
|
+
}
|
|
13046
|
+
}
|
|
13047
|
+
const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks);
|
|
13048
|
+
if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) {
|
|
13049
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
13050
|
+
this.checkpointIndexRun(
|
|
13051
|
+
database,
|
|
13052
|
+
store,
|
|
13053
|
+
invertedIndex,
|
|
13054
|
+
failedProcessing,
|
|
13055
|
+
resolvedRetryChunkIds,
|
|
13056
|
+
currentFileHashes,
|
|
13057
|
+
committedFilePaths,
|
|
13058
|
+
scopedRoots,
|
|
13059
|
+
configuredProviderInfo
|
|
13060
|
+
);
|
|
13061
|
+
}
|
|
12562
13062
|
}
|
|
12563
13063
|
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
12564
13064
|
failedProcessing.latestById,
|
|
@@ -12579,6 +13079,7 @@ var Indexer = class _Indexer {
|
|
|
12579
13079
|
retryableChunksWithExistingData.add(chunk.id);
|
|
12580
13080
|
}
|
|
12581
13081
|
}
|
|
13082
|
+
this.restoreMissingChunkRows(database, pendingChunks);
|
|
12582
13083
|
stats.totalChunks += pendingChunks.length;
|
|
12583
13084
|
onProgress?.({
|
|
12584
13085
|
phase: "embedding",
|
|
@@ -12601,6 +13102,17 @@ var Indexer = class _Indexer {
|
|
|
12601
13102
|
forceReembed: forceScopedReembed,
|
|
12602
13103
|
reuseCachedEmbeddings: true,
|
|
12603
13104
|
incrementRepeatedFailures: true,
|
|
13105
|
+
forceSingleItemBatches: true,
|
|
13106
|
+
onSucceeded: (succeededChunks) => {
|
|
13107
|
+
database.addChunksToBranchBatch(
|
|
13108
|
+
this.getBranchCatalogKey(),
|
|
13109
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
13110
|
+
);
|
|
13111
|
+
for (const chunk of succeededChunks) {
|
|
13112
|
+
failedProcessing.latestById.delete(chunk.id);
|
|
13113
|
+
resolvedRetryChunkIds.add(chunk.id);
|
|
13114
|
+
}
|
|
13115
|
+
},
|
|
12604
13116
|
onProgress: (batchProgress) => onProgress?.({
|
|
12605
13117
|
phase: "embedding",
|
|
12606
13118
|
filesProcessed: files.length,
|
|
@@ -12618,6 +13130,20 @@ var Indexer = class _Indexer {
|
|
|
12618
13130
|
failedForcedChunkIds.add(chunkId);
|
|
12619
13131
|
}
|
|
12620
13132
|
}
|
|
13133
|
+
if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) {
|
|
13134
|
+
lastCheckpointChunks = stats.totalChunks;
|
|
13135
|
+
this.checkpointIndexRun(
|
|
13136
|
+
database,
|
|
13137
|
+
store,
|
|
13138
|
+
invertedIndex,
|
|
13139
|
+
failedProcessing,
|
|
13140
|
+
resolvedRetryChunkIds,
|
|
13141
|
+
currentFileHashes,
|
|
13142
|
+
committedFilePaths,
|
|
13143
|
+
scopedRoots,
|
|
13144
|
+
configuredProviderInfo
|
|
13145
|
+
);
|
|
13146
|
+
}
|
|
12621
13147
|
}
|
|
12622
13148
|
const removedChunkIds = [];
|
|
12623
13149
|
for (const [chunkId] of existingChunks) {
|
|
@@ -12654,13 +13180,6 @@ var Indexer = class _Indexer {
|
|
|
12654
13180
|
if (removedStoredChunks) {
|
|
12655
13181
|
this.saveInvertedIndex(invertedIndex);
|
|
12656
13182
|
}
|
|
12657
|
-
if (scopedRoots) {
|
|
12658
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12659
|
-
} else {
|
|
12660
|
-
this.fileHashCache = currentFileHashes;
|
|
12661
|
-
this.saveFileHashCache();
|
|
12662
|
-
}
|
|
12663
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12664
13183
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12665
13184
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12666
13185
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -12669,6 +13188,13 @@ var Indexer = class _Indexer {
|
|
|
12669
13188
|
this.indexCompatibility = { compatible: true };
|
|
12670
13189
|
database.commitWriteTransaction();
|
|
12671
13190
|
writeTransactionActive = false;
|
|
13191
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
13192
|
+
if (scopedRoots) {
|
|
13193
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
13194
|
+
} else {
|
|
13195
|
+
this.fileHashCache = currentFileHashes;
|
|
13196
|
+
this.saveFileHashCache();
|
|
13197
|
+
}
|
|
12672
13198
|
stats.durationMs = Date.now() - startTime;
|
|
12673
13199
|
onProgress?.({
|
|
12674
13200
|
phase: "complete",
|
|
@@ -12692,13 +13218,6 @@ var Indexer = class _Indexer {
|
|
|
12692
13218
|
);
|
|
12693
13219
|
store.save();
|
|
12694
13220
|
this.saveInvertedIndex(invertedIndex);
|
|
12695
|
-
if (scopedRoots) {
|
|
12696
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12697
|
-
} else {
|
|
12698
|
-
this.fileHashCache = currentFileHashes;
|
|
12699
|
-
this.saveFileHashCache();
|
|
12700
|
-
}
|
|
12701
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12702
13221
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12703
13222
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12704
13223
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -12707,6 +13226,13 @@ var Indexer = class _Indexer {
|
|
|
12707
13226
|
this.indexCompatibility = { compatible: true };
|
|
12708
13227
|
database.commitWriteTransaction();
|
|
12709
13228
|
writeTransactionActive = false;
|
|
13229
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
13230
|
+
if (scopedRoots) {
|
|
13231
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
13232
|
+
} else {
|
|
13233
|
+
this.fileHashCache = currentFileHashes;
|
|
13234
|
+
this.saveFileHashCache();
|
|
13235
|
+
}
|
|
12710
13236
|
stats.durationMs = Date.now() - startTime;
|
|
12711
13237
|
onProgress?.({
|
|
12712
13238
|
phase: "complete",
|
|
@@ -12741,15 +13267,15 @@ var Indexer = class _Indexer {
|
|
|
12741
13267
|
);
|
|
12742
13268
|
store.save();
|
|
12743
13269
|
this.saveInvertedIndex(invertedIndex);
|
|
13270
|
+
database.commitWriteTransaction();
|
|
13271
|
+
writeTransactionActive = false;
|
|
13272
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
|
|
12744
13273
|
if (scopedRoots) {
|
|
12745
13274
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12746
13275
|
} else {
|
|
12747
13276
|
this.fileHashCache = currentFileHashes;
|
|
12748
13277
|
this.saveFileHashCache();
|
|
12749
13278
|
}
|
|
12750
|
-
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12751
|
-
database.commitWriteTransaction();
|
|
12752
|
-
writeTransactionActive = false;
|
|
12753
13279
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
12754
13280
|
const gcReset = await this.maybeRunOrphanGc();
|
|
12755
13281
|
if (gcReset) {
|
|
@@ -12773,6 +13299,9 @@ var Indexer = class _Indexer {
|
|
|
12773
13299
|
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
12774
13300
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12775
13301
|
}
|
|
13302
|
+
if (forceScopedReembed) {
|
|
13303
|
+
database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true");
|
|
13304
|
+
}
|
|
12776
13305
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12777
13306
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12778
13307
|
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
@@ -12883,26 +13412,41 @@ var Indexer = class _Indexer {
|
|
|
12883
13412
|
shouldPrefilterByBranch: branchChunkIds !== null && (this.config.scope === "global" || hasInitializedBranchCatalog)
|
|
12884
13413
|
};
|
|
12885
13414
|
}
|
|
12886
|
-
|
|
13415
|
+
searchCandidatesWithAllowedIds(initialLimit, totalCount, allowedChunkIds, shouldPrefilter, search, getChunkId) {
|
|
12887
13416
|
const normalizedLimit = Math.max(0, Math.floor(initialLimit));
|
|
12888
13417
|
if (normalizedLimit === 0) return [];
|
|
12889
|
-
if (!
|
|
13418
|
+
if (!shouldPrefilter || !allowedChunkIds) {
|
|
12890
13419
|
return search(normalizedLimit);
|
|
12891
13420
|
}
|
|
12892
|
-
const targetCount = Math.min(normalizedLimit,
|
|
13421
|
+
const targetCount = Math.min(normalizedLimit, allowedChunkIds.size);
|
|
12893
13422
|
if (targetCount === 0 || totalCount === 0) return [];
|
|
12894
13423
|
let requestedLimit = Math.min(normalizedLimit, totalCount);
|
|
12895
13424
|
while (true) {
|
|
12896
13425
|
const results = search(requestedLimit);
|
|
12897
|
-
const
|
|
12898
|
-
if (
|
|
12899
|
-
return
|
|
13426
|
+
const allowedResults = results.filter((candidate) => allowedChunkIds.has(getChunkId(candidate)));
|
|
13427
|
+
if (allowedResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
|
|
13428
|
+
return allowedResults;
|
|
12900
13429
|
}
|
|
12901
13430
|
const nextLimit = Math.min(totalCount, Math.max(requestedLimit + 1, requestedLimit * 2));
|
|
12902
|
-
if (nextLimit === requestedLimit) return
|
|
13431
|
+
if (nextLimit === requestedLimit) return allowedResults;
|
|
12903
13432
|
requestedLimit = nextLimit;
|
|
12904
13433
|
}
|
|
12905
13434
|
}
|
|
13435
|
+
getTemporalChunkIds(database, options) {
|
|
13436
|
+
if (!options?.blameSince && !options?.blameUntil) return null;
|
|
13437
|
+
const since = options.blameSince ? parseBlameTimestamp(options.blameSince, false) : void 0;
|
|
13438
|
+
const until = options.blameUntil ? parseBlameTimestamp(options.blameUntil, true) : void 0;
|
|
13439
|
+
if (since === null || until === null) {
|
|
13440
|
+
return /* @__PURE__ */ new Set();
|
|
13441
|
+
}
|
|
13442
|
+
return new Set(database.getChunkIdsByBlameDate(since, until));
|
|
13443
|
+
}
|
|
13444
|
+
intersectChunkIdSets(first, second) {
|
|
13445
|
+
if (first === null) return second;
|
|
13446
|
+
if (second === null) return first;
|
|
13447
|
+
const [smaller, larger] = first.size <= second.size ? [first, second] : [second, first];
|
|
13448
|
+
return new Set(Array.from(smaller).filter((chunkId) => larger.has(chunkId)));
|
|
13449
|
+
}
|
|
12906
13450
|
buildCandidateSnapshot(candidate) {
|
|
12907
13451
|
return {
|
|
12908
13452
|
id: candidate.id,
|
|
@@ -12917,13 +13461,16 @@ var Indexer = class _Indexer {
|
|
|
12917
13461
|
buildCandidateSnapshotList(candidates) {
|
|
12918
13462
|
return candidates.map((candidate) => this.buildCandidateSnapshot(candidate));
|
|
12919
13463
|
}
|
|
12920
|
-
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch) {
|
|
12921
|
-
|
|
12922
|
-
|
|
12923
|
-
|
|
13464
|
+
searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch, temporalChunkIds) {
|
|
13465
|
+
const availableCount = temporalChunkIds?.size ?? store.count();
|
|
13466
|
+
if (availableCount === 0) return [];
|
|
13467
|
+
const allowedIds = temporalChunkIds === null ? void 0 : Array.from(temporalChunkIds);
|
|
13468
|
+
return this.searchCandidatesWithAllowedIds(
|
|
13469
|
+
Math.min(initialLimit, availableCount),
|
|
13470
|
+
availableCount,
|
|
12924
13471
|
branchChunkIds,
|
|
12925
13472
|
shouldPrefilterByBranch,
|
|
12926
|
-
(requestedLimit) => store.search(embedding, requestedLimit),
|
|
13473
|
+
(requestedLimit) => store.search(embedding, requestedLimit, allowedIds),
|
|
12927
13474
|
(candidate) => candidate.id
|
|
12928
13475
|
);
|
|
12929
13476
|
}
|
|
@@ -12948,8 +13495,9 @@ var Indexer = class _Indexer {
|
|
|
12948
13495
|
const rerankTopN = this.config.search.rerankTopN;
|
|
12949
13496
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
12950
13497
|
const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
|
|
13498
|
+
const prioritizeSourcePaths = sourceIntent || options?.prioritizeSourcePaths === true;
|
|
12951
13499
|
const identifierHints = extractIdentifierHints(query);
|
|
12952
|
-
const candidateLimit = maxResults * (
|
|
13500
|
+
const candidateLimit = maxResults * (prioritizeSourcePaths ? 12 : 4);
|
|
12953
13501
|
this.logger.search("debug", "Starting search", {
|
|
12954
13502
|
query,
|
|
12955
13503
|
maxResults,
|
|
@@ -12980,6 +13528,7 @@ var Indexer = class _Indexer {
|
|
|
12980
13528
|
branchChunkIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchChunkIds(branchKey)));
|
|
12981
13529
|
branchSymbolIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchSymbolIds(branchKey)));
|
|
12982
13530
|
}
|
|
13531
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
12983
13532
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
12984
13533
|
const prefilterMs = performance2.now() - prefilterStartTime;
|
|
12985
13534
|
const vectorStartTime = performance2.now();
|
|
@@ -12988,7 +13537,8 @@ var Indexer = class _Indexer {
|
|
|
12988
13537
|
embedding,
|
|
12989
13538
|
candidateLimit,
|
|
12990
13539
|
branchChunkIds,
|
|
12991
|
-
shouldPrefilterByBranch
|
|
13540
|
+
shouldPrefilterByBranch,
|
|
13541
|
+
temporalChunkIds
|
|
12992
13542
|
) : [];
|
|
12993
13543
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
12994
13544
|
const keywordStartTime = performance2.now();
|
|
@@ -12998,7 +13548,8 @@ var Indexer = class _Indexer {
|
|
|
12998
13548
|
store,
|
|
12999
13549
|
invertedIndex,
|
|
13000
13550
|
branchChunkIds,
|
|
13001
|
-
shouldPrefilterByBranch
|
|
13551
|
+
shouldPrefilterByBranch,
|
|
13552
|
+
temporalChunkIds
|
|
13002
13553
|
);
|
|
13003
13554
|
const keywordMs = performance2.now() - keywordStartTime;
|
|
13004
13555
|
const scopedSemanticCandidates = semanticCandidates.filter(
|
|
@@ -13020,7 +13571,7 @@ var Indexer = class _Indexer {
|
|
|
13020
13571
|
rerankTopN,
|
|
13021
13572
|
limit: maxResults,
|
|
13022
13573
|
hybridWeight: rankingHybridWeight,
|
|
13023
|
-
prioritizeSourcePaths
|
|
13574
|
+
prioritizeSourcePaths
|
|
13024
13575
|
});
|
|
13025
13576
|
const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
|
|
13026
13577
|
definitionIntent: options?.definitionIntent === true,
|
|
@@ -13056,10 +13607,11 @@ var Indexer = class _Indexer {
|
|
|
13056
13607
|
branchSymbolIds,
|
|
13057
13608
|
maxResults,
|
|
13058
13609
|
union,
|
|
13059
|
-
sourceIntent
|
|
13610
|
+
sourceIntent,
|
|
13611
|
+
options?.definitionIntent === true && ((options.directory?.trim().length ?? 0) > 0 || (options.fileType?.trim().length ?? 0) > 0)
|
|
13060
13612
|
);
|
|
13061
13613
|
const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
|
|
13062
|
-
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
13614
|
+
const primaryLane = options?.definitionIntent === true ? mergeTieredResults(symbolLane, prePrimaryLane, maxResults * 4) : mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
13063
13615
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
13064
13616
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
13065
13617
|
const baseFiltered = tiered.filter(
|
|
@@ -13154,14 +13706,18 @@ var Indexer = class _Indexer {
|
|
|
13154
13706
|
})
|
|
13155
13707
|
);
|
|
13156
13708
|
}
|
|
13157
|
-
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false) {
|
|
13709
|
+
async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false, temporalChunkIds = null) {
|
|
13158
13710
|
const normalizedLimit = Math.max(0, Math.floor(limit));
|
|
13159
13711
|
if (normalizedLimit === 0) return [];
|
|
13160
|
-
const
|
|
13712
|
+
const allowedChunkIds = this.intersectChunkIdSets(
|
|
13713
|
+
shouldPrefilterByBranch ? branchChunkIds : null,
|
|
13714
|
+
temporalChunkIds
|
|
13715
|
+
);
|
|
13716
|
+
const scoreEntries = this.searchCandidatesWithAllowedIds(
|
|
13161
13717
|
normalizedLimit,
|
|
13162
13718
|
invertedIndex.getDocumentCount(),
|
|
13163
|
-
|
|
13164
|
-
|
|
13719
|
+
allowedChunkIds,
|
|
13720
|
+
allowedChunkIds !== null,
|
|
13165
13721
|
(requestedLimit) => Array.from(invertedIndex.search(query, requestedLimit)),
|
|
13166
13722
|
([chunkId]) => chunkId
|
|
13167
13723
|
);
|
|
@@ -13246,7 +13802,17 @@ var Indexer = class _Indexer {
|
|
|
13246
13802
|
);
|
|
13247
13803
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
13248
13804
|
for (const file of files) {
|
|
13249
|
-
|
|
13805
|
+
let hash;
|
|
13806
|
+
try {
|
|
13807
|
+
hash = hashFile(file.path);
|
|
13808
|
+
} catch (error) {
|
|
13809
|
+
this.logger.warn("Skipped unreadable file during freshness check", {
|
|
13810
|
+
path: file.path,
|
|
13811
|
+
error: getErrorMessage4(error)
|
|
13812
|
+
});
|
|
13813
|
+
return { readable: false, current: false, reason: "unreadable" };
|
|
13814
|
+
}
|
|
13815
|
+
currentFileHashes.set(this.toStoredFilePath(file.path), hash);
|
|
13250
13816
|
}
|
|
13251
13817
|
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
13252
13818
|
const cachedFileHashes = scopedRoots ? new Map(Array.from(this.fileHashCache).filter(([filePath]) => this.isFileInCurrentScope(filePath, scopedRoots))) : this.fileHashCache;
|
|
@@ -13272,69 +13838,87 @@ var Indexer = class _Indexer {
|
|
|
13272
13838
|
async forceIndex(onProgress) {
|
|
13273
13839
|
return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
|
|
13274
13840
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
13275
|
-
|
|
13841
|
+
const recovery = this.beginClearRecoveryState();
|
|
13842
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
13843
|
+
this.finishClearRecoveryState();
|
|
13276
13844
|
return this.indexUnlocked(onProgress, [], true);
|
|
13277
13845
|
});
|
|
13278
13846
|
}
|
|
13279
13847
|
async clearIndex() {
|
|
13280
13848
|
await this.withIndexMutationLease("clear", async (recoveredOwners) => {
|
|
13281
13849
|
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
13282
|
-
|
|
13850
|
+
const recovery = this.beginClearRecoveryState();
|
|
13851
|
+
await this.clearIndexUnlocked(recovery.compatibilityDecision);
|
|
13283
13852
|
});
|
|
13284
13853
|
}
|
|
13285
|
-
|
|
13854
|
+
clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot) {
|
|
13286
13855
|
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
13287
|
-
|
|
13288
|
-
|
|
13289
|
-
|
|
13290
|
-
|
|
13291
|
-
|
|
13292
|
-
|
|
13293
|
-
|
|
13294
|
-
|
|
13295
|
-
|
|
13296
|
-
|
|
13297
|
-
|
|
13298
|
-
|
|
13299
|
-
|
|
13300
|
-
|
|
13301
|
-
|
|
13856
|
+
const clearedBranchKeys = database.getAllBranches();
|
|
13857
|
+
store.clear();
|
|
13858
|
+
store.save();
|
|
13859
|
+
invertedIndex.clear();
|
|
13860
|
+
this.saveInvertedIndex(invertedIndex);
|
|
13861
|
+
this.fileHashCache.clear();
|
|
13862
|
+
this.saveFileHashCache();
|
|
13863
|
+
database.clearAllIndexedData();
|
|
13864
|
+
this.deleteBranchCommitMetadata(database, clearedBranchKeys);
|
|
13865
|
+
this.clearFailedBatchState();
|
|
13866
|
+
database.deleteMetadata("index.version");
|
|
13867
|
+
database.deleteMetadata("index.pathStorageVersion");
|
|
13868
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
13869
|
+
database.deleteMetadata("index.embeddingModel");
|
|
13870
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
13871
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
13872
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
13873
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
13874
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash));
|
|
13875
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash));
|
|
13876
|
+
database.deleteMetadata("index.createdAt");
|
|
13877
|
+
database.deleteMetadata("index.updatedAt");
|
|
13878
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
13879
|
+
}
|
|
13880
|
+
clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots(), recoveryDecision) {
|
|
13881
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
13882
|
+
store.load();
|
|
13883
|
+
invertedIndex.load();
|
|
13884
|
+
this.loadFileHashCache();
|
|
13885
|
+
const compatibility = this.checkCompatibility();
|
|
13886
|
+
const compatibilityDecision = recoveryDecision ?? (compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible");
|
|
13887
|
+
const allMetadata = store.getAllMetadata();
|
|
13888
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
13889
|
+
if (compatibilityDecision !== "compatible" && hasForeignData) {
|
|
13890
|
+
if (compatibilityDecision === "embedding-strategy-mismatch") {
|
|
13891
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
13892
|
+
this.clearScopedFileHashCache(roots);
|
|
13893
|
+
this.clearScopedFailedBatches(roots);
|
|
13894
|
+
const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
|
|
13895
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true");
|
|
13896
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
|
|
13897
|
+
database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash));
|
|
13898
|
+
if (projectRoot === this.projectRoot) {
|
|
13302
13899
|
this.indexCompatibility = { compatible: true };
|
|
13303
|
-
return;
|
|
13304
13900
|
}
|
|
13305
|
-
throw new Error(
|
|
13306
|
-
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
13307
|
-
);
|
|
13308
|
-
}
|
|
13309
|
-
if (!hasForeignData) {
|
|
13310
|
-
const clearedBranchKeys2 = database.getAllBranches();
|
|
13311
|
-
store.clear();
|
|
13312
|
-
store.save();
|
|
13313
|
-
invertedIndex.clear();
|
|
13314
|
-
this.saveInvertedIndex(invertedIndex);
|
|
13315
|
-
this.fileHashCache.clear();
|
|
13316
|
-
this.saveFileHashCache();
|
|
13317
|
-
database.clearAllIndexedData();
|
|
13318
|
-
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
13319
|
-
this.clearFailedBatchState();
|
|
13320
|
-
database.deleteMetadata("index.version");
|
|
13321
|
-
database.deleteMetadata("index.pathStorageVersion");
|
|
13322
|
-
database.deleteMetadata("index.embeddingProvider");
|
|
13323
|
-
database.deleteMetadata("index.embeddingModel");
|
|
13324
|
-
database.deleteMetadata("index.embeddingDimensions");
|
|
13325
|
-
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
13326
|
-
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
13327
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
13328
|
-
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
13329
|
-
database.deleteMetadata("index.createdAt");
|
|
13330
|
-
database.deleteMetadata("index.updatedAt");
|
|
13331
|
-
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
13332
13901
|
return;
|
|
13333
13902
|
}
|
|
13334
|
-
|
|
13335
|
-
|
|
13336
|
-
|
|
13903
|
+
throw new Error(
|
|
13904
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
13905
|
+
);
|
|
13906
|
+
}
|
|
13907
|
+
if (!hasForeignData) {
|
|
13908
|
+
this.clearGlobalIndexDataUnlocked(projectRoot);
|
|
13909
|
+
return;
|
|
13910
|
+
}
|
|
13911
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
|
|
13912
|
+
this.clearScopedFileHashCache(roots);
|
|
13913
|
+
this.clearScopedFailedBatches(roots);
|
|
13914
|
+
if (projectRoot === this.projectRoot) {
|
|
13337
13915
|
this.indexCompatibility = compatibility;
|
|
13916
|
+
}
|
|
13917
|
+
}
|
|
13918
|
+
async clearIndexUnlocked(recoveryDecision) {
|
|
13919
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
13920
|
+
if (this.config.scope === "global") {
|
|
13921
|
+
this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision);
|
|
13338
13922
|
return;
|
|
13339
13923
|
}
|
|
13340
13924
|
if (!this.isProjectOwnedIndexPath()) {
|
|
@@ -13500,6 +14084,7 @@ var Indexer = class _Indexer {
|
|
|
13500
14084
|
)) {
|
|
13501
14085
|
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
13502
14086
|
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
14087
|
+
this.restoreMissingChunkRows(database, chunks);
|
|
13503
14088
|
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
13504
14089
|
store,
|
|
13505
14090
|
provider,
|
|
@@ -13514,6 +14099,7 @@ var Indexer = class _Indexer {
|
|
|
13514
14099
|
forceReembed: false,
|
|
13515
14100
|
reuseCachedEmbeddings: false,
|
|
13516
14101
|
incrementRepeatedFailures: false,
|
|
14102
|
+
forceSingleItemBatches: true,
|
|
13517
14103
|
onSucceeded: (succeededChunks) => {
|
|
13518
14104
|
database.addChunksToBranchBatch(
|
|
13519
14105
|
this.getBranchCatalogKey(),
|
|
@@ -13535,9 +14121,12 @@ var Indexer = class _Indexer {
|
|
|
13535
14121
|
this.saveInvertedIndex(invertedIndex);
|
|
13536
14122
|
}
|
|
13537
14123
|
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
13538
|
-
database.
|
|
13539
|
-
|
|
13540
|
-
|
|
14124
|
+
const migrationFinalized = database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true";
|
|
14125
|
+
if (migrationFinalized) {
|
|
14126
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
14127
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
14128
|
+
this.indexCompatibility = { compatible: true };
|
|
14129
|
+
}
|
|
13541
14130
|
}
|
|
13542
14131
|
return { succeeded, failed, remaining };
|
|
13543
14132
|
}
|
|
@@ -13559,7 +14148,8 @@ var Indexer = class _Indexer {
|
|
|
13559
14148
|
latestById.set(chunkId, {
|
|
13560
14149
|
attemptCount: batch.attemptCount,
|
|
13561
14150
|
error: batch.error,
|
|
13562
|
-
lastAttempt: batch.lastAttempt
|
|
14151
|
+
lastAttempt: batch.lastAttempt,
|
|
14152
|
+
chunks: [rawChunk]
|
|
13563
14153
|
});
|
|
13564
14154
|
}
|
|
13565
14155
|
}
|
|
@@ -13626,6 +14216,7 @@ var Indexer = class _Indexer {
|
|
|
13626
14216
|
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
13627
14217
|
);
|
|
13628
14218
|
}
|
|
14219
|
+
const temporalChunkIds = this.getTemporalChunkIds(database, options);
|
|
13629
14220
|
const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
|
|
13630
14221
|
const prefilterMs = performance2.now() - prefilterStartTime;
|
|
13631
14222
|
const vectorStartTime = performance2.now();
|
|
@@ -13634,7 +14225,8 @@ var Indexer = class _Indexer {
|
|
|
13634
14225
|
embedding,
|
|
13635
14226
|
limit * 2,
|
|
13636
14227
|
branchChunkIds,
|
|
13637
|
-
shouldPrefilterByBranch
|
|
14228
|
+
shouldPrefilterByBranch,
|
|
14229
|
+
temporalChunkIds
|
|
13638
14230
|
);
|
|
13639
14231
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
13640
14232
|
if (this.config.scope !== "global" && branchChunkIds && !hasInitializedBranchCatalog) {
|
|
@@ -14383,9 +14975,11 @@ async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
|
14383
14975
|
contextLines: options.contextLines,
|
|
14384
14976
|
metadataOnly: options.metadataOnly,
|
|
14385
14977
|
definitionIntent: options.definitionIntent,
|
|
14978
|
+
prioritizeSourcePaths: options.prioritizeSourcePaths,
|
|
14386
14979
|
blameAuthor: options.blameAuthor,
|
|
14387
14980
|
blameSha: options.blameSha,
|
|
14388
14981
|
blameSince: options.blameSince,
|
|
14982
|
+
blameUntil: options.blameUntil,
|
|
14389
14983
|
trace: options.trace
|
|
14390
14984
|
});
|
|
14391
14985
|
}
|
|
@@ -14431,7 +15025,9 @@ async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
|
14431
15025
|
fileType: options.fileType,
|
|
14432
15026
|
directory: options.directory,
|
|
14433
15027
|
chunkType: options.chunkType,
|
|
14434
|
-
excludeFile: options.excludeFile
|
|
15028
|
+
excludeFile: options.excludeFile,
|
|
15029
|
+
blameSince: options.blameSince,
|
|
15030
|
+
blameUntil: options.blameUntil
|
|
14435
15031
|
});
|
|
14436
15032
|
}
|
|
14437
15033
|
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
@@ -14494,6 +15090,9 @@ async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
|
14494
15090
|
if (args.estimateOnly) {
|
|
14495
15091
|
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
14496
15092
|
}
|
|
15093
|
+
if (args.dryRun) {
|
|
15094
|
+
return { kind: "dryrun", dryrun: await indexer.dryRunCost() };
|
|
15095
|
+
}
|
|
14497
15096
|
const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => {
|
|
14498
15097
|
if (onProgress) {
|
|
14499
15098
|
void onProgress(formatProgressTitle(progress), {
|
|
@@ -17970,13 +18569,19 @@ async function resolveSearchContext(input, operations) {
|
|
|
17970
18569
|
(trace) => operations.lookup(symbol, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
|
|
17971
18570
|
);
|
|
17972
18571
|
};
|
|
17973
|
-
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt) => {
|
|
18572
|
+
const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt, prioritizeSourcePaths) => {
|
|
17974
18573
|
return recordAttempt(
|
|
17975
18574
|
"conceptual",
|
|
17976
18575
|
searchQuery,
|
|
17977
18576
|
scope,
|
|
17978
18577
|
relaxedFieldsForAttempt,
|
|
17979
|
-
(trace) => operations.search(
|
|
18578
|
+
(trace) => operations.search(
|
|
18579
|
+
searchQuery,
|
|
18580
|
+
MAX_CONTEXT_RESULT_LIMIT,
|
|
18581
|
+
scope,
|
|
18582
|
+
input.diagnostic ? trace : void 0,
|
|
18583
|
+
{ prioritizeSourcePaths }
|
|
18584
|
+
)
|
|
17980
18585
|
);
|
|
17981
18586
|
};
|
|
17982
18587
|
const findSuccessfulAttemptState = (route) => {
|
|
@@ -18104,10 +18709,12 @@ Explicit symbol lookup only; conceptual search was not attempted.`
|
|
|
18104
18709
|
}
|
|
18105
18710
|
}
|
|
18106
18711
|
for (const attempt of conceptualAttemptPlan) {
|
|
18712
|
+
const attemptIntent = analyzeQueryIntent(attempt.queryText);
|
|
18713
|
+
const prioritizeSourcePaths = attemptIntent.primary !== "docs" && attemptIntent.primary !== "test";
|
|
18107
18714
|
if (inferredSymbol && attempt.queryText === inferredSymbol && attempt.queryText !== query) {
|
|
18108
18715
|
decisions.fallbackFromOriginalConceptualToInferred = true;
|
|
18109
18716
|
}
|
|
18110
|
-
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed);
|
|
18717
|
+
const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed, prioritizeSourcePaths);
|
|
18111
18718
|
if (results.length > 0) {
|
|
18112
18719
|
const heading = buildPackHeading("conceptual", decisions);
|
|
18113
18720
|
const intent = analyzeQueryIntent(attempt.queryText);
|
|
@@ -18244,12 +18851,13 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
18244
18851
|
directory: scope.directory,
|
|
18245
18852
|
trace
|
|
18246
18853
|
}),
|
|
18247
|
-
search: (queryText, retrievalLimit, scope, trace) => searchCodebase(projectRoot, host, queryText, {
|
|
18854
|
+
search: (queryText, retrievalLimit, scope, trace, searchOptions) => searchCodebase(projectRoot, host, queryText, {
|
|
18248
18855
|
limit: retrievalLimit,
|
|
18249
18856
|
fileType: scope.fileType,
|
|
18250
18857
|
directory: scope.directory,
|
|
18251
18858
|
metadataOnly: true,
|
|
18252
|
-
trace
|
|
18859
|
+
trace,
|
|
18860
|
+
prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
|
|
18253
18861
|
})
|
|
18254
18862
|
});
|
|
18255
18863
|
}
|
|
@@ -18325,6 +18933,7 @@ async function executeCodebaseEditContext(projectRoot, host, args) {
|
|
|
18325
18933
|
async function executeIndexCodebase(projectRoot, host, args, onProgress) {
|
|
18326
18934
|
const result = await runIndexCodebase(projectRoot, host, args, onProgress);
|
|
18327
18935
|
if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) };
|
|
18936
|
+
if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) };
|
|
18328
18937
|
if (result.kind === "busy") return { text: result.text, isError: true };
|
|
18329
18938
|
if (result.kind === "message") return { text: result.text };
|
|
18330
18939
|
return { text: formatIndexStats(result.stats, args.verbose ?? false) };
|
|
@@ -19164,7 +19773,8 @@ var codebase_peek = tool({
|
|
|
19164
19773
|
chunkType: z3.enum(CHUNK_TYPE_VALUES).optional().describe("Filter by code chunk type"),
|
|
19165
19774
|
blameAuthor: z3.string().optional().describe("Filter by git blame author name or email"),
|
|
19166
19775
|
blameSha: z3.string().optional().describe("Filter by git blame commit SHA or prefix"),
|
|
19167
|
-
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)")
|
|
19776
|
+
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)"),
|
|
19777
|
+
blameUntil: z3.string().optional().describe("Filter to chunks last changed on or before this date (e.g., 2025-01-31)")
|
|
19168
19778
|
},
|
|
19169
19779
|
async execute(args, context) {
|
|
19170
19780
|
return searchCodebaseWithEffectiveness(context?.worktree, DEFAULT_HOST, "peek", args.query, {
|
|
@@ -19175,7 +19785,8 @@ var codebase_peek = tool({
|
|
|
19175
19785
|
metadataOnly: true,
|
|
19176
19786
|
blameAuthor: args.blameAuthor,
|
|
19177
19787
|
blameSha: args.blameSha,
|
|
19178
|
-
blameSince: args.blameSince
|
|
19788
|
+
blameSince: args.blameSince,
|
|
19789
|
+
blameUntil: args.blameUntil
|
|
19179
19790
|
}, (results) => {
|
|
19180
19791
|
const text = formatCodebasePeek(results);
|
|
19181
19792
|
return { output: text, text };
|
|
@@ -19187,6 +19798,7 @@ var index_codebase = tool({
|
|
|
19187
19798
|
args: {
|
|
19188
19799
|
force: z3.boolean().optional().default(false).describe("Force reindex even if already indexed"),
|
|
19189
19800
|
estimateOnly: z3.boolean().optional().default(false).describe("Only show cost estimate without indexing"),
|
|
19801
|
+
dryRun: z3.boolean().optional().default(false).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."),
|
|
19190
19802
|
verbose: z3.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures")
|
|
19191
19803
|
},
|
|
19192
19804
|
async execute(args, context) {
|
|
@@ -19237,7 +19849,9 @@ var find_similar = tool({
|
|
|
19237
19849
|
fileType: z3.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
19238
19850
|
directory: z3.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
19239
19851
|
chunkType: z3.enum(CHUNK_TYPE_VALUES).optional().describe("Filter by code chunk type"),
|
|
19240
|
-
excludeFile: z3.string().optional().describe("Exclude results from this file path (useful when searching for duplicates of code from a specific file)")
|
|
19852
|
+
excludeFile: z3.string().optional().describe("Exclude results from this file path (useful when searching for duplicates of code from a specific file)"),
|
|
19853
|
+
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)"),
|
|
19854
|
+
blameUntil: z3.string().optional().describe("Filter to chunks last changed on or before this date (e.g., 2025-01-31)")
|
|
19241
19855
|
},
|
|
19242
19856
|
async execute(args, context) {
|
|
19243
19857
|
const results = await findSimilarCode(context?.worktree, DEFAULT_HOST, args.code, {
|
|
@@ -19245,7 +19859,9 @@ var find_similar = tool({
|
|
|
19245
19859
|
fileType: args.fileType,
|
|
19246
19860
|
directory: args.directory,
|
|
19247
19861
|
chunkType: args.chunkType,
|
|
19248
|
-
excludeFile: args.excludeFile
|
|
19862
|
+
excludeFile: args.excludeFile,
|
|
19863
|
+
blameSince: args.blameSince,
|
|
19864
|
+
blameUntil: args.blameUntil
|
|
19249
19865
|
});
|
|
19250
19866
|
if (results.length === 0) {
|
|
19251
19867
|
return "No similar code found. Try a different snippet or run index_codebase first.";
|
|
@@ -19264,7 +19880,8 @@ var codebase_search = tool({
|
|
|
19264
19880
|
contextLines: z3.number().optional().describe("Number of extra lines to include before/after each match (default: 0)"),
|
|
19265
19881
|
blameAuthor: z3.string().optional().describe("Filter by git blame author name or email"),
|
|
19266
19882
|
blameSha: z3.string().optional().describe("Filter by git blame commit SHA or prefix"),
|
|
19267
|
-
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)")
|
|
19883
|
+
blameSince: z3.string().optional().describe("Filter to chunks last changed on or after this date (e.g., 2025-01-01)"),
|
|
19884
|
+
blameUntil: z3.string().optional().describe("Filter to chunks last changed on or before this date (e.g., 2025-01-31)")
|
|
19268
19885
|
},
|
|
19269
19886
|
async execute(args, context) {
|
|
19270
19887
|
return searchCodebaseWithEffectiveness(context?.worktree, DEFAULT_HOST, "search", args.query, {
|
|
@@ -19275,7 +19892,8 @@ var codebase_search = tool({
|
|
|
19275
19892
|
contextLines: args.contextLines,
|
|
19276
19893
|
blameAuthor: args.blameAuthor,
|
|
19277
19894
|
blameSha: args.blameSha,
|
|
19278
|
-
blameSince: args.blameSince
|
|
19895
|
+
blameSince: args.blameSince,
|
|
19896
|
+
blameUntil: args.blameUntil
|
|
19279
19897
|
}, (results) => {
|
|
19280
19898
|
const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : formatSearchResults(results, "score");
|
|
19281
19899
|
return { output: text, text };
|
|
@@ -19488,6 +20106,12 @@ var PI_TOOL_NAMES = [
|
|
|
19488
20106
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
19489
20107
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
19490
20108
|
];
|
|
20109
|
+
var MCP_TOOL_NAMES = [
|
|
20110
|
+
...PORTABLE_TOOL_NAMES,
|
|
20111
|
+
TOOL_NAME.ADD_KNOWLEDGE_BASE,
|
|
20112
|
+
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
20113
|
+
TOOL_NAME.REMOVE_KNOWLEDGE_BASE
|
|
20114
|
+
];
|
|
19491
20115
|
|
|
19492
20116
|
// src/commands/loader.ts
|
|
19493
20117
|
import { existsSync as existsSync14, readdirSync as readdirSync3, readFileSync as readFileSync9 } from "fs";
|
|
@@ -19775,6 +20399,7 @@ function assessRoutingIntent(text) {
|
|
|
19775
20399
|
};
|
|
19776
20400
|
}
|
|
19777
20401
|
function buildRoutingHint(assessment, status, includeGraphHandoff = false) {
|
|
20402
|
+
const hasSymbolCue = hasIdentifierShape(assessment.text) || containsQuotedIdentifier(assessment.text);
|
|
19778
20403
|
if (assessment.intent === "definition_lookup") {
|
|
19779
20404
|
if (!status || !status.indexed || status.compatibility?.compatible === false) {
|
|
19780
20405
|
return "For this turn, if you need a symbol definition, check `index_status` first and run `index_codebase` if the index is missing or incompatible. Then use `implementation_lookup` for the definition site. Use `grep` for exhaustive literal matches.";
|
|
@@ -19784,12 +20409,13 @@ function buildRoutingHint(assessment, status, includeGraphHandoff = false) {
|
|
|
19784
20409
|
if (assessment.intent !== "local_conceptual" && assessment.intent !== "local_broad_task") {
|
|
19785
20410
|
return null;
|
|
19786
20411
|
}
|
|
20412
|
+
const preEditHint = assessment.intent === "local_broad_task" && hasSymbolCue ? " If a likely target symbol is already known or strongly suspected, consider optional `codebase_edit_context` as a compact pre-edit next step for bounded source plus direct callers and callees." : "";
|
|
19787
20413
|
if (!status || !status.indexed || status.compatibility?.compatible === false) {
|
|
19788
20414
|
const graphHandoff2 = includeGraphHandoff ? " Use graph tools after semantic discovery identifies relevant symbols." : "";
|
|
19789
|
-
return `For this turn, if local code discovery by behavior is needed, check \`index_status\` first and run \`index_codebase\` if the index is missing or incompatible.${graphHandoff2} Then use \`codebase_context\` as the first local repository lookup. Use \`grep\` for exact identifiers or exhaustive matches
|
|
20415
|
+
return `For this turn, if local code discovery by behavior is needed, check \`index_status\` first and run \`index_codebase\` if the index is missing or incompatible.${graphHandoff2} Then use \`codebase_context\` as the first local repository lookup. Use \`grep\` for exact identifiers or exhaustive matches.${preEditHint}`;
|
|
19790
20416
|
}
|
|
19791
20417
|
const graphHandoff = includeGraphHandoff ? " before graph tools such as `call_graph`, `call_graph_path`, `pr_impact`, or OMO CodeGraph" : "";
|
|
19792
|
-
return `For this turn, prefer \`codebase_context\` for local code discovery, then use \`codebase_peek\` for metadata and \`codebase_search\` when you need implementation content${graphHandoff}. Use \`grep\` for exact identifiers or exhaustive matches
|
|
20418
|
+
return `For this turn, prefer \`codebase_context\` for local code discovery, then use \`codebase_peek\` for metadata and \`codebase_search\` when you need implementation content${graphHandoff}. Use \`grep\` for exact identifiers or exhaustive matches.${preEditHint}`;
|
|
19793
20419
|
}
|
|
19794
20420
|
var RoutingHintController = class {
|
|
19795
20421
|
constructor(getStatus, maxSessions = 200, includeGraphHandoff = false) {
|
|
@@ -19831,7 +20457,7 @@ var RoutingHintController = class {
|
|
|
19831
20457
|
if (!state || !state.pendingHint) {
|
|
19832
20458
|
return;
|
|
19833
20459
|
}
|
|
19834
|
-
if (toolName === "codebase_context" || toolName === "codebase_peek" || toolName === "codebase_search" || toolName === "implementation_lookup" || toolName === "index_status" || toolName === "index_codebase") {
|
|
20460
|
+
if (toolName === "codebase_context" || toolName === "codebase_edit_context" || toolName === "codebase_peek" || toolName === "codebase_search" || toolName === "implementation_lookup" || toolName === "index_status" || toolName === "index_codebase") {
|
|
19835
20461
|
state.pendingHint = false;
|
|
19836
20462
|
state.updatedAt = Date.now();
|
|
19837
20463
|
this.sessionState.set(sessionID, state);
|