opencode-codebase-index 0.23.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -716,6 +716,17 @@ var EMBEDDING_MODELS = {
716
716
  maxTokens: 2048,
717
717
  costPer1MTokens: 0.15,
718
718
  taskAble: true
719
+ },
720
+ "gemini-embedding-2": {
721
+ provider: "google",
722
+ model: "gemini-embedding-2",
723
+ // Keep a conservative, testable default embedding dimension. Gemini Embedding 2 supports
724
+ // flexible dimensions via outputDimensionality.
725
+ dimensions: 1536,
726
+ maxTokens: 8192,
727
+ costPer1MTokens: 0.15,
728
+ taskAble: false,
729
+ promptStyle: "embedding-2"
719
730
  }
720
731
  },
721
732
  "openai": {
@@ -749,26 +760,15 @@ var EMBEDDING_MODELS = {
749
760
  maxTokens: 512,
750
761
  costPer1MTokens: 0
751
762
  }
752
- },
753
- "github-copilot": {
754
- "text-embedding-3-small": {
755
- provider: "github-copilot",
756
- model: "text-embedding-3-small",
757
- dimensions: 1536,
758
- maxTokens: 8191,
759
- costPer1MTokens: 0
760
- }
761
763
  }
762
764
  };
763
765
  var DEFAULT_PROVIDER_MODELS = {
764
- "github-copilot": "text-embedding-3-small",
765
766
  "openai": "text-embedding-3-small",
766
767
  "google": "gemini-embedding-001",
767
768
  "ollama": "nomic-embed-text"
768
769
  };
769
770
  var AUTO_DETECT_PROVIDER_ORDER = [
770
771
  "ollama",
771
- "github-copilot",
772
772
  "openai",
773
773
  "google"
774
774
  ];
@@ -794,6 +794,9 @@ function getDefaultIndexingConfig() {
794
794
  maxDepth: 5,
795
795
  maxFilesPerDirectory: 100,
796
796
  fallbackToTextOnMaxChunks: true,
797
+ // Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi
798
+ // fallback used when a native caller omits the argument).
799
+ linesPerChunk: 30,
797
800
  gitBlame: { enabled: false }
798
801
  };
799
802
  }
@@ -927,6 +930,7 @@ function parseConfig(raw) {
927
930
  maxDepth: typeof rawIndexing.maxDepth === "number" ? rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth : defaultIndexing.maxDepth,
928
931
  maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory,
929
932
  fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks,
933
+ linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk,
930
934
  gitBlame: {
931
935
  enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof rawIndexing.gitBlame.enabled === "boolean" ? rawIndexing.gitBlame.enabled : defaultIndexing.gitBlame.enabled
932
936
  }
@@ -969,6 +973,7 @@ function parseConfig(raw) {
969
973
  let embeddingModel;
970
974
  let customProvider;
971
975
  let reranker;
976
+ const githubCopilotDeprecationMessage = '`embeddingProvider: "github-copilot"` is deprecated and no longer available. Migrate existing configs to `embeddingProvider: "google"` and select an explicit Google model. For existing indexes, run `index_codebase` with `force: true` after changing to `gemini-embedding-001` or `gemini-embedding-2` to rebuild embeddings. See docs/configuration.md for details.';
972
977
  if (embeddingProviderValue === "custom") {
973
978
  embeddingProvider = "custom";
974
979
  const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
@@ -1008,6 +1013,8 @@ function parseConfig(raw) {
1008
1013
  } else if (rawEmbeddingModel) {
1009
1014
  embeddingModel = DEFAULT_PROVIDER_MODELS[embeddingProvider];
1010
1015
  }
1016
+ } else if (embeddingProviderValue === "github-copilot") {
1017
+ throw new Error(githubCopilotDeprecationMessage);
1011
1018
  } else {
1012
1019
  embeddingProvider = "auto";
1013
1020
  }
@@ -1038,10 +1045,21 @@ function parseConfig(raw) {
1038
1045
  timeoutMs: typeof rawReranker.timeoutMs === "number" ? Math.max(1e3, Math.floor(rawReranker.timeoutMs)) : 1e4
1039
1046
  };
1040
1047
  }
1048
+ const rawEmbedding = input.embedding && typeof input.embedding === "object" ? input.embedding : {};
1049
+ const rawEmbeddingBatch = rawEmbedding.batch && typeof rawEmbedding.batch === "object" ? rawEmbedding.batch : null;
1050
+ const embeddingMaxBatchItems = typeof rawEmbeddingBatch?.maxBatchItems === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchItems) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchItems)) : void 0;
1051
+ const embeddingMaxBatchTokens = typeof rawEmbeddingBatch?.maxBatchTokens === "number" && Number.isFinite(rawEmbeddingBatch.maxBatchTokens) ? Math.max(1, Math.floor(rawEmbeddingBatch.maxBatchTokens)) : void 0;
1052
+ const embedding = embeddingMaxBatchItems !== void 0 || embeddingMaxBatchTokens !== void 0 ? {
1053
+ batch: {
1054
+ ...embeddingMaxBatchItems !== void 0 ? { maxBatchItems: embeddingMaxBatchItems } : {},
1055
+ ...embeddingMaxBatchTokens !== void 0 ? { maxBatchTokens: embeddingMaxBatchTokens } : {}
1056
+ }
1057
+ } : {};
1041
1058
  return {
1042
1059
  embeddingProvider,
1043
1060
  embeddingModel,
1044
1061
  customProvider,
1062
+ embedding,
1045
1063
  scope: isValidScope(scopeValue) ? scopeValue : "project",
1046
1064
  include: includeValue ?? DEFAULT_INCLUDE,
1047
1065
  exclude: excludeValue ?? DEFAULT_EXCLUDE,
@@ -2537,8 +2555,6 @@ async function tryDetectProvider() {
2537
2555
  }
2538
2556
  async function getProviderCredentials(provider) {
2539
2557
  switch (provider) {
2540
- case "github-copilot":
2541
- return getGitHubCopilotCredentials();
2542
2558
  case "openai":
2543
2559
  return getOpenAICredentials();
2544
2560
  case "google":
@@ -2549,22 +2565,6 @@ async function getProviderCredentials(provider) {
2549
2565
  return null;
2550
2566
  }
2551
2567
  }
2552
- function getGitHubCopilotCredentials() {
2553
- const authData = loadOpenCodeAuth();
2554
- const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
2555
- if (!copilotAuth || copilotAuth.type !== "oauth") {
2556
- return null;
2557
- }
2558
- const auth = copilotAuth;
2559
- const baseUrl = auth.enterpriseUrl ? `https://copilot-api.${auth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
2560
- return {
2561
- provider: "github-copilot",
2562
- baseUrl,
2563
- refreshToken: copilotAuth.refresh,
2564
- accessToken: copilotAuth.access,
2565
- tokenExpires: copilotAuth.expires
2566
- };
2567
- }
2568
2568
  function getOpenAICredentials() {
2569
2569
  const authData = loadOpenCodeAuth();
2570
2570
  const openaiAuth = authData["openai"];
@@ -2690,8 +2690,6 @@ async function tryDetectOllamaProvider() {
2690
2690
  }
2691
2691
  function getProviderDisplayName(provider) {
2692
2692
  switch (provider) {
2693
- case "github-copilot":
2694
- return "GitHub Copilot";
2695
2693
  case "openai":
2696
2694
  return "OpenAI";
2697
2695
  case "google":
@@ -2916,44 +2914,6 @@ var CustomEmbeddingProvider = class extends BaseEmbeddingProvider {
2916
2914
  }
2917
2915
  };
2918
2916
 
2919
- // src/embeddings/providers/github-copilot.ts
2920
- var GitHubCopilotEmbeddingProvider = class extends BaseEmbeddingProvider {
2921
- constructor(credentials, modelInfo) {
2922
- super(credentials, modelInfo);
2923
- }
2924
- getToken() {
2925
- if (!this.credentials.refreshToken) {
2926
- throw new Error("No OAuth token available for GitHub");
2927
- }
2928
- return this.credentials.refreshToken;
2929
- }
2930
- async embedBatch(texts) {
2931
- const token = this.getToken();
2932
- const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
2933
- method: "POST",
2934
- headers: {
2935
- Authorization: `Bearer ${token}`,
2936
- "Content-Type": "application/json",
2937
- Accept: "application/vnd.github+json",
2938
- "X-GitHub-Api-Version": "2022-11-28"
2939
- },
2940
- body: JSON.stringify({
2941
- model: `openai/${this.modelInfo.model}`,
2942
- input: texts
2943
- })
2944
- });
2945
- if (!response.ok) {
2946
- const error = (await response.text()).slice(0, 500);
2947
- throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
2948
- }
2949
- const data = await response.json();
2950
- return {
2951
- embeddings: data.data.map((d) => d.embedding),
2952
- totalTokensUsed: data.usage.total_tokens
2953
- };
2954
- }
2955
- };
2956
-
2957
2917
  // src/embeddings/providers/google.ts
2958
2918
  var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddingProvider {
2959
2919
  static BATCH_SIZE = 20;
@@ -2961,24 +2921,30 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
2961
2921
  super(credentials, modelInfo);
2962
2922
  }
2963
2923
  async embedQuery(query) {
2964
- const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
2965
- const result = await this.embedWithTaskType([query], taskType);
2924
+ const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
2925
+ const texts = [
2926
+ this.modelInfo.model === "gemini-embedding-2" ? `task: code retrieval | query: ${query}` : query
2927
+ ];
2928
+ const result = await this.embedWithTaskType(texts, taskType);
2966
2929
  return {
2967
2930
  embedding: result.embeddings[0],
2968
2931
  tokensUsed: result.totalTokensUsed
2969
2932
  };
2970
2933
  }
2971
2934
  async embedDocument(document) {
2972
- const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
2973
- const result = await this.embedWithTaskType([document], taskType);
2935
+ const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
2936
+ const result = await this.embedWithTaskType([
2937
+ this.modelInfo.model === "gemini-embedding-2" ? `title: none | text: ${document}` : document
2938
+ ], taskType);
2974
2939
  return {
2975
2940
  embedding: result.embeddings[0],
2976
2941
  tokensUsed: result.totalTokensUsed
2977
2942
  };
2978
2943
  }
2979
2944
  async embedBatch(texts) {
2980
- const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
2981
- return this.embedWithTaskType(texts, taskType);
2945
+ const taskType = this.modelInfo.model === "gemini-embedding-001" && this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
2946
+ const formattedTexts = this.modelInfo.model === "gemini-embedding-2" ? texts.map((text) => `title: none | text: ${text}`) : texts;
2947
+ return this.embedWithTaskType(formattedTexts, taskType);
2982
2948
  }
2983
2949
  async embedWithTaskType(texts, taskType) {
2984
2950
  const batches = [];
@@ -3028,6 +2994,10 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider extends BaseEmbeddi
3028
2994
  var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddingProvider {
3029
2995
  static MIN_TRUNCATION_CHARS = 512;
3030
2996
  static REQUEST_TIMEOUT_MS = 12e4;
2997
+ // Set when /api/embed returns 404 so subsequent multi-text batches skip the
2998
+ // batched endpoint and go straight to the legacy per-text path (one probe per
2999
+ // old ollama install, not one probe per batch).
3000
+ batchEndpointUnavailable = false;
3031
3001
  constructor(credentials, modelInfo) {
3032
3002
  super(credentials, modelInfo);
3033
3003
  }
@@ -3045,6 +3015,21 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
3045
3015
  const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
3046
3016
  return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
3047
3017
  }
3018
+ // True for a 404 from the newer /api/embed endpoint, i.e. an ollama version that
3019
+ // does not provide it. embedBatch uses this to fall back to the legacy per-text
3020
+ // /api/embeddings path so old ollama installs do not regress.
3021
+ isBatchEndpointUnavailableError(error) {
3022
+ const message = error instanceof Error ? error.message : String(error);
3023
+ return message.includes("Ollama /api/embed not available");
3024
+ }
3025
+ // True for a malformed /api/embed response (wrong vector count or a bad vector).
3026
+ // embedBatch falls back to the per-text path on this so a bad batch response
3027
+ // re-embeds each text cleanly. A text that then fails per-text is not isolated
3028
+ // here; it is isolated on the recovery run, which re-embeds one text per request.
3029
+ isBatchValidationError(error) {
3030
+ const message = error instanceof Error ? error.message : String(error);
3031
+ return message.includes("invalid embedding batch");
3032
+ }
3048
3033
  buildTruncationCandidates(text) {
3049
3034
  const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
3050
3035
  const candidateLimits = /* @__PURE__ */ new Set();
@@ -3146,7 +3131,74 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
3146
3131
  tokensUsed: this.estimateTokens(text)
3147
3132
  };
3148
3133
  }
3149
- async embedBatch(texts) {
3134
+ // Embeds many texts in one POST /api/embed request (input: string[]). Ollama
3135
+ // encodes each input independently, so the model context length applies per input
3136
+ // (the upstream splitter already bounds each input), not over the batch. This
3137
+ // amortizes N HTTP round-trips into one.
3138
+ async embedMany(texts) {
3139
+ const controller = new AbortController();
3140
+ const timeout = setTimeout(
3141
+ () => controller.abort(),
3142
+ _OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS
3143
+ );
3144
+ let response;
3145
+ try {
3146
+ response = await fetch(`${this.credentials.baseUrl}/api/embed`, {
3147
+ method: "POST",
3148
+ headers: {
3149
+ "Content-Type": "application/json"
3150
+ },
3151
+ body: JSON.stringify({
3152
+ model: this.modelInfo.model,
3153
+ input: texts,
3154
+ truncate: false
3155
+ }),
3156
+ signal: controller.signal
3157
+ });
3158
+ } catch (error) {
3159
+ if (error instanceof Error && error.name === "AbortError") {
3160
+ throw new Error(
3161
+ `Ollama embedding request timed out after ${_OllamaEmbeddingProvider.REQUEST_TIMEOUT_MS}ms`
3162
+ );
3163
+ }
3164
+ throw error;
3165
+ } finally {
3166
+ clearTimeout(timeout);
3167
+ }
3168
+ if (!response.ok) {
3169
+ const error = (await response.text()).slice(0, 500);
3170
+ if (response.status === 404) {
3171
+ throw new Error(`Ollama /api/embed not available: ${response.status} - ${error}`);
3172
+ }
3173
+ throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
3174
+ }
3175
+ let parsed;
3176
+ try {
3177
+ parsed = await response.json();
3178
+ } catch {
3179
+ throw new Error(
3180
+ `Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
3181
+ );
3182
+ }
3183
+ const data = parsed && typeof parsed === "object" ? parsed : {};
3184
+ if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length || data.embeddings.some(
3185
+ (value) => !Array.isArray(value) || value.length !== this.modelInfo.dimensions || value.some((v) => typeof v !== "number" || !Number.isFinite(v))
3186
+ )) {
3187
+ throw new Error(
3188
+ `Ollama returned an invalid embedding batch; expected ${texts.length} vectors of ${this.modelInfo.dimensions} finite dimensions`
3189
+ );
3190
+ }
3191
+ return {
3192
+ embeddings: data.embeddings,
3193
+ totalTokensUsed: texts.reduce((sum, text) => sum + this.estimateTokens(text), 0)
3194
+ };
3195
+ }
3196
+ // Per-text /api/embeddings path shared by the single-text fast path and the
3197
+ // batch fallback. Uses the legacy endpoint one text at a time, so each text gets
3198
+ // its own truncation safety net and a vector validated on its own. A text that
3199
+ // hard-fails per-text throws here and fails the whole request batch; the recovery
3200
+ // run re-embeds one text per request to isolate it.
3201
+ async embedOneByOne(texts) {
3150
3202
  const results = [];
3151
3203
  for (const text of texts) {
3152
3204
  results.push(await this.embedSingleWithFallback(text));
@@ -3156,6 +3208,26 @@ var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider extends BaseEmbeddi
3156
3208
  totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
3157
3209
  };
3158
3210
  }
3211
+ async embedBatch(texts) {
3212
+ if (texts.length === 0) {
3213
+ return { embeddings: [], totalTokensUsed: 0 };
3214
+ }
3215
+ if (texts.length === 1 || this.batchEndpointUnavailable) {
3216
+ return this.embedOneByOne(texts);
3217
+ }
3218
+ try {
3219
+ return await this.embedMany(texts);
3220
+ } catch (error) {
3221
+ if (this.isBatchEndpointUnavailableError(error)) {
3222
+ this.batchEndpointUnavailable = true;
3223
+ return this.embedOneByOne(texts);
3224
+ }
3225
+ if (!this.isContextLengthError(error) && !this.isBatchValidationError(error)) {
3226
+ throw error;
3227
+ }
3228
+ return this.embedOneByOne(texts);
3229
+ }
3230
+ }
3159
3231
  };
3160
3232
 
3161
3233
  // src/embeddings/providers/openai.ts
@@ -3190,8 +3262,6 @@ var OpenAIEmbeddingProvider = class extends BaseEmbeddingProvider {
3190
3262
  // src/embeddings/provider.ts
3191
3263
  function createEmbeddingProvider(configuredProviderInfo) {
3192
3264
  switch (configuredProviderInfo.provider) {
3193
- case "github-copilot":
3194
- return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
3195
3265
  case "openai":
3196
3266
  return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
3197
3267
  case "google":
@@ -3526,6 +3596,26 @@ function formatCostEstimate(estimate) {
3526
3596
  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
3527
3597
  `;
3528
3598
  }
3599
+ function formatDryRunEstimate(estimate) {
3600
+ return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed.
3601
+
3602
+ Files to embed: ${estimate.filesCount.toLocaleString()}
3603
+ Chunks to embed: ${estimate.chunksCount.toLocaleString()}
3604
+ Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()}
3605
+
3606
+ The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It
3607
+ matches the live "Tokens used" counter only for providers that report usage on the
3608
+ same basis (ollama); for providers that report a server tokenizer count (OpenAI,
3609
+ Gemini, custom) it is only an estimate.
3610
+
3611
+ For a matching provider and a project-scoped force index, the force pass clears its
3612
+ own cached embeddings, so the live counter climbs to this number. A force index on a
3613
+ shared global index can reuse cached embeddings from other projects, and an
3614
+ incremental index counts cached chunks that are not re-embedded; in both cases this
3615
+ number is an upper bound on the live counter, so a progress percent against this
3616
+ total tops out below 100%.
3617
+ `;
3618
+ }
3529
3619
  function formatBytes(bytes) {
3530
3620
  if (bytes === 0) return "0 B";
3531
3621
  const k = 1024;
@@ -4209,12 +4299,12 @@ try {
4209
4299
  }
4210
4300
 
4211
4301
  // src/native/parsing.ts
4212
- function parseFileAsText(filePath, content) {
4213
- const result = native.parseFileAsText(filePath, content);
4302
+ function parseFileAsText(filePath, content, linesPerChunk) {
4303
+ const result = native.parseFileAsText(filePath, content, linesPerChunk);
4214
4304
  return result.map(mapChunk);
4215
4305
  }
4216
- function parseFiles(files) {
4217
- const result = native.parseFiles(files);
4306
+ function parseFiles(files, linesPerChunk) {
4307
+ const result = native.parseFiles(files, linesPerChunk);
4218
4308
  return result.map((f) => ({
4219
4309
  path: f.path,
4220
4310
  chunks: f.chunks.map(mapChunk),
@@ -4291,13 +4381,13 @@ var VectorStore = class {
4291
4381
  const metadata = items.map((i) => JSON.stringify(i.metadata));
4292
4382
  this.inner.addBatch(ids, vectors, metadata);
4293
4383
  }
4294
- search(queryVector, limit = 10) {
4384
+ search(queryVector, limit = 10, allowedIds) {
4295
4385
  if (queryVector.length !== this.dimensions) {
4296
4386
  throw new Error(
4297
4387
  `Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
4298
4388
  );
4299
4389
  }
4300
- const results = this.inner.search(queryVector, limit);
4390
+ const results = allowedIds === void 0 ? this.inner.search(queryVector, limit) : this.inner.searchFiltered(queryVector, limit, allowedIds);
4301
4391
  return results.map((r) => ({
4302
4392
  id: r.id,
4303
4393
  score: r.score,
@@ -4525,6 +4615,10 @@ var Database = class _Database {
4525
4615
  this.throwIfClosed();
4526
4616
  return this.inner.getBranchChunkIds(branch);
4527
4617
  }
4618
+ getChunkIdsByBlameDate(since, until) {
4619
+ this.throwIfClosed();
4620
+ return this.inner.getChunkIdsByBlameDate(since, until);
4621
+ }
4528
4622
  getBranchDelta(branch, baseBranch) {
4529
4623
  this.throwIfClosed();
4530
4624
  return this.inner.getBranchDelta(branch, baseBranch);
@@ -5486,6 +5580,9 @@ function resolveWorktreeFallbackPath(projectRoot, relativePath) {
5486
5580
  const fallbackPath = path10.join(mainRepoRoot, relativePath);
5487
5581
  return existsSync5(fallbackPath) ? fallbackPath : null;
5488
5582
  }
5583
+ function getHostProjectConfigRelativePath(host) {
5584
+ return getProjectConfigRelativePath(host);
5585
+ }
5489
5586
  function getProjectConfigCandidatePaths(projectRoot, host) {
5490
5587
  const candidates = [path10.join(projectRoot, getProjectConfigRelativePath(host))];
5491
5588
  if (host !== "opencode") {
@@ -5577,6 +5674,9 @@ function resolveProjectConfigPath(projectRoot, host) {
5577
5674
  const candidates = getProjectConfigCandidatePaths(projectRoot, host);
5578
5675
  return candidates.find((candidate) => existsSync5(candidate)) ?? path10.join(projectRoot, getProjectConfigRelativePath(host));
5579
5676
  }
5677
+ function resolveWritableProjectConfigPath(projectRoot, host) {
5678
+ return path10.join(projectRoot, getProjectConfigRelativePath(host));
5679
+ }
5580
5680
  function resolveProjectIndexPath(projectRoot, scope, host) {
5581
5681
  if (scope === "global") {
5582
5682
  return resolveGlobalIndexPath(host);
@@ -6282,6 +6382,10 @@ function scoreCandidate(query, intent, candidate, originalIndex) {
6282
6382
  let boost = 0;
6283
6383
  if (intent.primary === "conceptual") {
6284
6384
  boost += Math.min(0.14, overlap * 0.14);
6385
+ if (intent.preferSourcePaths) {
6386
+ boost += implementationPath ? 0.32 : 0;
6387
+ if (testPath || fixturePath || docsPath) boost -= 0.35;
6388
+ }
6285
6389
  if (generatedOrVendor) boost -= 0.18;
6286
6390
  if (importChunk || weakContainer) boost -= 0.04;
6287
6391
  } else if (intent.primary === "test") {
@@ -6736,6 +6840,9 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
6736
6840
  "enum_declaration",
6737
6841
  "function_definition",
6738
6842
  "class_definition",
6843
+ // Ruby module/class symbols that are declaration-bearing and navigable.
6844
+ "class",
6845
+ "module",
6739
6846
  "class_specifier",
6740
6847
  "struct_specifier",
6741
6848
  "namespace_definition",
@@ -7206,6 +7313,19 @@ function parseOwner(value) {
7206
7313
  if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
7207
7314
  if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
7208
7315
  if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
7316
+ if (candidate.recoveryProtocolVersion !== void 0 && candidate.recoveryProtocolVersion !== 1) return null;
7317
+ if (candidate.projectRoot !== void 0 && typeof candidate.projectRoot !== "string") return null;
7318
+ if (candidate.scopedRoots !== void 0) {
7319
+ if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) {
7320
+ return null;
7321
+ }
7322
+ }
7323
+ if (candidate.clearRecovery !== void 0) {
7324
+ const recovery = candidate.clearRecovery;
7325
+ if (typeof recovery !== "object" || recovery === null || recovery.phase !== "clearing" || typeof recovery.embeddingProvider !== "string" || recovery.embeddingProvider.length === 0 || typeof recovery.embeddingModel !== "string" || recovery.embeddingModel.length === 0 || !Number.isInteger(recovery.embeddingDimensions) || (recovery.embeddingDimensions ?? 0) <= 0 || typeof recovery.embeddingStrategyVersion !== "string" || recovery.embeddingStrategyVersion.length === 0 || recovery.compatibilityDecision !== "compatible" && recovery.compatibilityDecision !== "embedding-strategy-mismatch" && recovery.compatibilityDecision !== "incompatible" || candidate.operation !== "clear" && candidate.operation !== "force-index") {
7326
+ return null;
7327
+ }
7328
+ }
7209
7329
  return candidate;
7210
7330
  }
7211
7331
  function parseReclaimOwner(value) {
@@ -7446,13 +7566,18 @@ function isTransientIndexLockContention(error) {
7446
7566
  if (!isIndexLockContentionError(error) || !("reason" in error)) return false;
7447
7567
  return error.reason === "active" || error.reason === "reclaiming";
7448
7568
  }
7449
- function acquireIndexLock(indexPath, operation) {
7569
+ function acquireIndexLock(indexPath, operation, recoveryScope) {
7450
7570
  mkdirSync2(indexPath, { recursive: true });
7451
7571
  const canonicalIndexPath = realpathSync3.native(indexPath);
7452
7572
  const lockPath = path13.join(canonicalIndexPath, "indexing.lock");
7453
7573
  cleanupDeadPublicationCandidates(canonicalIndexPath);
7454
7574
  for (let attempt = 0; attempt < 6; attempt += 1) {
7455
- const owner = createOwner(operation);
7575
+ const owner = recoveryScope === void 0 ? createOwner(operation) : {
7576
+ ...createOwner(operation),
7577
+ recoveryProtocolVersion: 1,
7578
+ projectRoot: recoveryScope.projectRoot,
7579
+ scopedRoots: recoveryScope.scopedRoots
7580
+ };
7456
7581
  if (publishJsonDirectory(lockPath, owner)) {
7457
7582
  const lease = {
7458
7583
  canonicalIndexPath,
@@ -7517,6 +7642,33 @@ function releaseIndexLock(lease) {
7517
7642
  }
7518
7643
  return true;
7519
7644
  }
7645
+ function setIndexLockClearRecoveryState(lease, clearRecovery) {
7646
+ const currentOwner = readDirectoryOwner(lease.lockPath);
7647
+ if (!currentOwner || !sameOwner(currentOwner, lease.owner)) {
7648
+ throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
7649
+ }
7650
+ const nextOwner = { ...currentOwner };
7651
+ if (clearRecovery === null) {
7652
+ delete nextOwner.clearRecovery;
7653
+ } else {
7654
+ nextOwner.clearRecovery = clearRecovery;
7655
+ }
7656
+ const ownerPath = path13.join(lease.lockPath, OWNER_FILE_NAME);
7657
+ const temporaryPath = path13.join(
7658
+ lease.lockPath,
7659
+ `${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${randomUUID()}`
7660
+ );
7661
+ try {
7662
+ writeFileSync2(temporaryPath, JSON.stringify(nextOwner), {
7663
+ encoding: "utf-8",
7664
+ flag: "wx",
7665
+ mode: 384
7666
+ });
7667
+ retryTransientFilesystemOperation(() => renameSync(temporaryPath, ownerPath));
7668
+ } finally {
7669
+ if (existsSync6(temporaryPath)) rmSync(temporaryPath, { force: true });
7670
+ }
7671
+ }
7520
7672
  function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
7521
7673
  if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
7522
7674
  temporaryCounter += 1;
@@ -7754,6 +7906,18 @@ function createFailedBatchWriter(targetPath) {
7754
7906
  temporaryPath
7755
7907
  };
7756
7908
  }
7909
+ function writeFailedBatchRecords(targetPath, records) {
7910
+ const writer = createFailedBatchWriter(targetPath);
7911
+ try {
7912
+ for (const record of records) {
7913
+ writer.write(record);
7914
+ }
7915
+ writer.commit();
7916
+ } catch (error) {
7917
+ writer.cleanup();
7918
+ throw error;
7919
+ }
7920
+ }
7757
7921
  function* readLegacyFailedBatchRecords(filePath, options) {
7758
7922
  const rawData = fs2.readFileSync(filePath, "utf-8");
7759
7923
  const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
@@ -8036,14 +8200,18 @@ function getSafeEmbeddingChunkTokenLimit(provider) {
8036
8200
  const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
8037
8201
  return Math.min(2e3, maxChunkTokens);
8038
8202
  }
8039
- function getDynamicBatchOptions(provider) {
8040
- if (provider.provider === "ollama") {
8041
- return {
8042
- maxBatchTokens: provider.modelInfo.maxTokens,
8043
- maxBatchItems: 1
8044
- };
8203
+ var DEFAULT_OLLAMA_MAX_BATCH_ITEMS = 16;
8204
+ var DEFAULT_OLLAMA_MAX_BATCH_TOKENS = 65536;
8205
+ function getDynamicBatchOptions(provider, embeddingBatch) {
8206
+ if (provider.provider !== "ollama") {
8207
+ return {};
8045
8208
  }
8046
- return {};
8209
+ const base = { maxBatchTokens: DEFAULT_OLLAMA_MAX_BATCH_TOKENS, maxBatchItems: DEFAULT_OLLAMA_MAX_BATCH_ITEMS };
8210
+ return {
8211
+ ...base,
8212
+ ...typeof embeddingBatch?.maxBatchTokens === "number" && Number.isFinite(embeddingBatch.maxBatchTokens) ? { maxBatchTokens: embeddingBatch.maxBatchTokens } : {},
8213
+ ...typeof embeddingBatch?.maxBatchItems === "number" && Number.isFinite(embeddingBatch.maxBatchItems) ? { maxBatchItems: embeddingBatch.maxBatchItems } : {}
8214
+ };
8047
8215
  }
8048
8216
  function isSqliteCorruptionError(error) {
8049
8217
  const message = getErrorMessage3(error).toLowerCase();
@@ -8061,6 +8229,14 @@ function getPendingChunkId(rawChunk) {
8061
8229
  const id = rawChunk.id;
8062
8230
  return typeof id === "string" ? id : null;
8063
8231
  }
8232
+ function parseBlameTimestamp(value, endOfDay) {
8233
+ let timestampMs = Date.parse(value);
8234
+ if (Number.isNaN(timestampMs)) return null;
8235
+ if (endOfDay && /^\d{4}-\d{2}-\d{2}$/.test(value.trim())) {
8236
+ timestampMs += 24 * 60 * 60 * 1e3 - 1;
8237
+ }
8238
+ return Math.floor(timestampMs / 1e3);
8239
+ }
8064
8240
  function metadataFromBlame(blame) {
8065
8241
  if (!blame) {
8066
8242
  return {};
@@ -8207,7 +8383,7 @@ function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCa
8207
8383
  const remainder = combined.filter((candidate) => !promotedIds.has(candidate.id));
8208
8384
  return [...promoted, ...remainder];
8209
8385
  }
8210
- function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
8386
+ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, limit, fallbackCandidates, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source", allowNonSourcePaths = false) {
8211
8387
  if (!prioritizeSourcePaths) {
8212
8388
  return [];
8213
8389
  }
@@ -8227,7 +8403,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
8227
8403
  if (!isImplementationChunkType(chunkType)) {
8228
8404
  return false;
8229
8405
  }
8230
- if (!isLikelyImplementationPath2(chunk.filePath)) {
8406
+ if (!allowNonSourcePaths && !isLikelyImplementationPath2(chunk.filePath)) {
8231
8407
  return false;
8232
8408
  }
8233
8409
  const nameLower = (chunk.name ?? "").toLowerCase();
@@ -8291,7 +8467,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
8291
8467
  }
8292
8468
  foundCoveringChunk = upsertChunkCandidate(chunk, identifier, normalizedIdentifier) || foundCoveringChunk;
8293
8469
  }
8294
- if (foundCoveringChunk || !isLikelyImplementationPath2(symbol.filePath)) {
8470
+ if (foundCoveringChunk || !allowNonSourcePaths && !isLikelyImplementationPath2(symbol.filePath)) {
8295
8471
  continue;
8296
8472
  }
8297
8473
  const symbolName = symbol.name.toLowerCase();
@@ -8345,7 +8521,7 @@ function buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbol
8345
8521
  const ranked = Array.from(symbolCandidates.values()).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
8346
8522
  if (ranked.length === 0) {
8347
8523
  const implementationFallback = fallbackCandidates.filter(
8348
- (candidate) => isImplementationChunkType(candidate.metadata.chunkType) && isLikelyImplementationPath2(candidate.metadata.filePath)
8524
+ (candidate) => isImplementationChunkType(candidate.metadata.chunkType) && (allowNonSourcePaths || isLikelyImplementationPath2(candidate.metadata.filePath))
8349
8525
  );
8350
8526
  for (const candidate of implementationFallback) {
8351
8527
  const nameLower = (candidate.metadata.name ?? "").toLowerCase();
@@ -8461,10 +8637,16 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
8461
8637
  return false;
8462
8638
  }
8463
8639
  if (options?.blameSince) {
8464
- const sinceMs = Date.parse(options.blameSince);
8465
- if (Number.isNaN(sinceMs)) return false;
8640
+ const since = parseBlameTimestamp(options.blameSince, false);
8641
+ if (since === null) return false;
8466
8642
  const committedAt = candidate.metadata.blameCommittedAt;
8467
- if (committedAt === void 0 || committedAt < Math.floor(sinceMs / 1e3)) return false;
8643
+ if (committedAt === void 0 || committedAt < since) return false;
8644
+ }
8645
+ if (options?.blameUntil) {
8646
+ const until = parseBlameTimestamp(options.blameUntil, true);
8647
+ if (until === null) return false;
8648
+ const committedAt = candidate.metadata.blameCommittedAt;
8649
+ if (committedAt === void 0 || committedAt > until) return false;
8468
8650
  }
8469
8651
  return true;
8470
8652
  }
@@ -8520,9 +8702,10 @@ var Indexer = class _Indexer {
8520
8702
  writerArtifactFingerprint = null;
8521
8703
  readerArtifactRetryAfter = /* @__PURE__ */ new Map();
8522
8704
  fileBatchLimits;
8705
+ checkpointIntervalChunks;
8523
8706
  constructor(projectRoot, config, host, runtimeOptions = {}) {
8524
8707
  this.projectRoot = projectRoot;
8525
- this.projectIdentityHash = hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
8708
+ this.projectIdentityHash = this.getProjectIdentityHash(projectRoot);
8526
8709
  this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot;
8527
8710
  this.branchNameOverride = runtimeOptions.branchName;
8528
8711
  this.catalogIdentityOverride = runtimeOptions.catalogIdentity;
@@ -8532,6 +8715,7 @@ var Indexer = class _Indexer {
8532
8715
  this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
8533
8716
  this.indexPathOverride = runtimeOptions.indexPath;
8534
8717
  this.fileBatchLimits = runtimeOptions.fileBatchLimits;
8718
+ this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks;
8535
8719
  this.config = config;
8536
8720
  this.host = host;
8537
8721
  if (isGitRepo(this.materializedProjectRoot)) {
@@ -8643,6 +8827,9 @@ var Indexer = class _Indexer {
8643
8827
  return path15.resolve(targetPath);
8644
8828
  }
8645
8829
  }
8830
+ getProjectIdentityHash(projectRoot) {
8831
+ return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
8832
+ }
8646
8833
  isProjectOwnedIndexPath() {
8647
8834
  return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host);
8648
8835
  }
@@ -8679,7 +8866,10 @@ var Indexer = class _Indexer {
8679
8866
  }
8680
8867
  async withIndexMutationLease(operation, callback) {
8681
8868
  this.refreshBranchInfo();
8682
- const lease = acquireIndexLock(this.indexPath, operation);
8869
+ const lease = acquireIndexLock(this.indexPath, operation, {
8870
+ projectRoot: this.projectRoot,
8871
+ scopedRoots: this.getScopedRoots()
8872
+ });
8683
8873
  this.indexPath = lease.canonicalIndexPath;
8684
8874
  this.refreshRuntimeArtifactPaths();
8685
8875
  this.activeIndexLease = lease;
@@ -8734,6 +8924,7 @@ var Indexer = class _Indexer {
8734
8924
  }
8735
8925
  loadFileHashCache() {
8736
8926
  if (!existsSync8(this.fileHashCachePath)) {
8927
+ this.fileHashCache = /* @__PURE__ */ new Map();
8737
8928
  return;
8738
8929
  }
8739
8930
  try {
@@ -8773,10 +8964,10 @@ var Indexer = class _Indexer {
8773
8964
  invertedIndex.serialize()
8774
8965
  );
8775
8966
  }
8776
- getScopedRoots() {
8777
- const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(this.projectRoot)]);
8967
+ getScopedRoots(projectRoot = this.projectRoot) {
8968
+ const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(projectRoot)]);
8778
8969
  for (const kbRoot of this.config.knowledgeBases) {
8779
- roots.add(this.getCanonicalPath(path15.resolve(this.projectRoot, kbRoot)));
8970
+ roots.add(this.getCanonicalPath(path15.resolve(projectRoot, kbRoot)));
8780
8971
  }
8781
8972
  return Array.from(roots);
8782
8973
  }
@@ -8847,14 +9038,17 @@ var Indexer = class _Indexer {
8847
9038
  getLegacyBranchCatalogKey() {
8848
9039
  return this.currentBranch || "default";
8849
9040
  }
8850
- getLegacyMigrationMetadataKey() {
8851
- return `index.globalBranchMigration.${this.projectIdentityHash}`;
9041
+ getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash) {
9042
+ return `index.globalBranchMigration.${projectIdentityHash}`;
8852
9043
  }
8853
- getProjectEmbeddingStrategyMetadataKey() {
8854
- return `index.embeddingStrategyVersion.${this.projectIdentityHash}`;
9044
+ getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash) {
9045
+ return `index.embeddingStrategyVersion.${projectIdentityHash}`;
8855
9046
  }
8856
- getProjectForceReembedMetadataKey() {
8857
- return `index.forceReembed.${this.projectIdentityHash}`;
9047
+ getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
9048
+ return `index.forceReembed.${projectIdentityHash}`;
9049
+ }
9050
+ getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash) {
9051
+ return `index.migrationFinalized.${projectIdentityHash}`;
8858
9052
  }
8859
9053
  getBranchMigrationMetadataKey(prefix, catalogIdentity = this.getBranchCatalogIdentity()) {
8860
9054
  const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
@@ -8960,7 +9154,7 @@ var Indexer = class _Indexer {
8960
9154
  const legacy = this.getLegacyBranchCatalogKey();
8961
9155
  return primary === legacy ? [primary] : [primary, legacy];
8962
9156
  }
8963
- getProjectLocalScopedOwnershipIds(roots) {
9157
+ getProjectLocalScopedOwnershipIds(roots, projectRoot = this.projectRoot) {
8964
9158
  const chunkIds = /* @__PURE__ */ new Set();
8965
9159
  const symbolIds = /* @__PURE__ */ new Set();
8966
9160
  if (!this.database) {
@@ -8968,10 +9162,10 @@ var Indexer = class _Indexer {
8968
9162
  }
8969
9163
  const projectLocalFilePaths = /* @__PURE__ */ new Set([
8970
9164
  ...Array.from(this.fileHashCache.keys()).filter(
8971
- (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
9165
+ (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
8972
9166
  ),
8973
9167
  ...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
8974
- (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath)
9168
+ (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot)
8975
9169
  )
8976
9170
  ]);
8977
9171
  for (const filePath of projectLocalFilePaths) {
@@ -8984,15 +9178,16 @@ var Indexer = class _Indexer {
8984
9178
  }
8985
9179
  return { chunkIds, symbolIds };
8986
9180
  }
8987
- getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
9181
+ getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds, projectRoot = this.projectRoot) {
8988
9182
  if (this.config.scope !== "global") {
8989
9183
  return this.getBranchCatalogCleanupKeys();
8990
9184
  }
8991
9185
  const keys = /* @__PURE__ */ new Set();
8992
9186
  const projectChunkIdSet = new Set(projectChunkIds);
8993
9187
  const projectSymbolIdSet = new Set(projectSymbolIds);
9188
+ const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
8994
9189
  for (const branchKey of this.database?.getAllBranches() ?? []) {
8995
- if (branchKey.startsWith(`${this.projectIdentityHash}:`)) {
9190
+ if (branchKey.startsWith(`${projectIdentityHash}:`)) {
8996
9191
  keys.add(branchKey);
8997
9192
  continue;
8998
9193
  }
@@ -9002,8 +9197,10 @@ var Indexer = class _Indexer {
9002
9197
  keys.add(branchKey);
9003
9198
  }
9004
9199
  }
9005
- for (const branchKey of this.getBranchCatalogCleanupKeys()) {
9006
- keys.add(branchKey);
9200
+ if (projectRoot === this.projectRoot) {
9201
+ for (const branchKey of this.getBranchCatalogCleanupKeys()) {
9202
+ keys.add(branchKey);
9203
+ }
9007
9204
  }
9008
9205
  return Array.from(keys);
9009
9206
  }
@@ -9011,10 +9208,10 @@ var Indexer = class _Indexer {
9011
9208
  const canonicalFilePath = this.getCanonicalStoredFilePath(filePath);
9012
9209
  return roots.some((root) => isPathWithinRoot2(canonicalFilePath, root));
9013
9210
  }
9014
- isFileInProjectRoot(filePath) {
9211
+ isFileInProjectRoot(filePath, projectRoot = this.projectRoot) {
9015
9212
  return isPathWithinRoot2(
9016
9213
  this.getCanonicalStoredFilePath(filePath),
9017
- this.getCanonicalPath(this.projectRoot)
9214
+ this.getCanonicalPath(projectRoot)
9018
9215
  );
9019
9216
  }
9020
9217
  clearScopedFileHashCache(roots) {
@@ -9056,12 +9253,12 @@ var Indexer = class _Indexer {
9056
9253
  }
9057
9254
  return false;
9058
9255
  }
9059
- hasForeignScopedBranchData() {
9256
+ hasForeignScopedBranchData(projectRoot = this.projectRoot, roots = this.getScopedRoots(projectRoot)) {
9060
9257
  if (!this.database || this.config.scope !== "global") {
9061
9258
  return false;
9062
9259
  }
9063
- const roots = this.getScopedRoots();
9064
- const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
9260
+ const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
9261
+ const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot);
9065
9262
  return this.database.getAllBranches().some(
9066
9263
  (branchKey) => {
9067
9264
  const branchChunkIds = this.database.getBranchChunkIds(branchKey);
@@ -9070,7 +9267,7 @@ var Indexer = class _Indexer {
9070
9267
  if (!hasBranchData) {
9071
9268
  return false;
9072
9269
  }
9073
- if (branchKey.startsWith(`${this.projectIdentityHash}:`)) {
9270
+ if (branchKey.startsWith(`${projectIdentityHash}:`)) {
9074
9271
  return false;
9075
9272
  }
9076
9273
  const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
@@ -9079,7 +9276,7 @@ var Indexer = class _Indexer {
9079
9276
  }
9080
9277
  );
9081
9278
  }
9082
- clearSharedIndexProjectData(store, invertedIndex, database, roots) {
9279
+ clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot = this.projectRoot) {
9083
9280
  const allMetadata = store.getAllMetadata();
9084
9281
  const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
9085
9282
  const filePaths = /* @__PURE__ */ new Set([
@@ -9087,7 +9284,7 @@ var Indexer = class _Indexer {
9087
9284
  ...scopedEntries.map(({ metadata }) => metadata.filePath)
9088
9285
  ]);
9089
9286
  const projectLocalFilePaths = new Set(
9090
- Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath))
9287
+ Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot))
9091
9288
  );
9092
9289
  const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
9093
9290
  for (const filePath of filePaths) {
@@ -9097,7 +9294,7 @@ var Indexer = class _Indexer {
9097
9294
  }
9098
9295
  const removedChunkIdList = Array.from(removedChunkIds);
9099
9296
  const projectLocalChunkIds = new Set(
9100
- scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)).map(({ key }) => key)
9297
+ scopedEntries.filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)).map(({ key }) => key)
9101
9298
  );
9102
9299
  for (const filePath of projectLocalFilePaths) {
9103
9300
  for (const chunk of database.getChunksByFile(filePath)) {
@@ -9116,7 +9313,8 @@ var Indexer = class _Indexer {
9116
9313
  }
9117
9314
  const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys(
9118
9315
  Array.from(projectLocalChunkIds),
9119
- Array.from(projectLocalSymbolIds)
9316
+ Array.from(projectLocalSymbolIds),
9317
+ projectRoot
9120
9318
  );
9121
9319
  for (const branchKey of branchCleanupKeys) {
9122
9320
  database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
@@ -9151,29 +9349,96 @@ var Indexer = class _Indexer {
9151
9349
  database.gcOrphanSymbols();
9152
9350
  database.gcOrphanEmbeddings();
9153
9351
  database.gcOrphanChunks();
9154
- store.save();
9155
9352
  this.saveInvertedIndex(invertedIndex);
9353
+ store.save();
9156
9354
  return {
9157
9355
  removedChunkIds: removedChunkIdList,
9158
9356
  hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
9159
9357
  };
9160
9358
  }
9359
+ getCurrentClearRecoveryState() {
9360
+ if (!this.configuredProviderInfo) {
9361
+ throw new Error("Cannot persist clear recovery state before the embedding provider is initialized");
9362
+ }
9363
+ const compatibility = this.checkCompatibility();
9364
+ const compatibilityDecision = compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible";
9365
+ return {
9366
+ phase: "clearing",
9367
+ embeddingProvider: this.configuredProviderInfo.provider,
9368
+ embeddingModel: this.configuredProviderInfo.modelInfo.model,
9369
+ embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions,
9370
+ embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION,
9371
+ compatibilityDecision
9372
+ };
9373
+ }
9374
+ beginClearRecoveryState() {
9375
+ const recovery = this.getCurrentClearRecoveryState();
9376
+ setIndexLockClearRecoveryState(this.requireActiveLease(), recovery);
9377
+ return recovery;
9378
+ }
9379
+ finishClearRecoveryState() {
9380
+ setIndexLockClearRecoveryState(this.requireActiveLease(), null);
9381
+ }
9382
+ matchesCurrentClearRecoveryConfiguration(recovery) {
9383
+ const configuredProviderInfo = this.configuredProviderInfo;
9384
+ return configuredProviderInfo !== null && recovery.embeddingProvider === configuredProviderInfo.provider && recovery.embeddingModel === configuredProviderInfo.modelInfo.model && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION;
9385
+ }
9386
+ hasUnknownLegacyForceIndexClear(owner) {
9387
+ return owner.operation === "force-index" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1 && existsSync8(path15.join(this.indexPath, "force-index-phase"));
9388
+ }
9161
9389
  async recoverFromInterruptedIndexingUnlocked(owners) {
9162
9390
  for (const owner of owners) {
9163
9391
  this.logger.warn("Detected interrupted indexing session, recovering...", {
9164
9392
  pid: owner.pid,
9165
9393
  hostname: owner.hostname,
9166
9394
  operation: owner.operation,
9167
- startedAt: owner.startedAt
9395
+ startedAt: owner.startedAt,
9396
+ projectRoot: owner.projectRoot
9168
9397
  });
9169
9398
  }
9170
9399
  if (this.config.scope === "global") {
9171
- if (existsSync8(this.fileHashCachePath)) {
9172
- unlinkSync2(this.fileHashCachePath);
9400
+ const clearScopes = [];
9401
+ for (const owner of owners) {
9402
+ if (this.hasUnknownLegacyForceIndexClear(owner)) {
9403
+ throw new Error(
9404
+ `Cannot automatically recover interrupted force-index ${owner.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
9405
+ );
9406
+ }
9407
+ if (owner.operation === "clear" && owner.clearRecovery === void 0 && owner.recoveryProtocolVersion !== 1) {
9408
+ throw new Error(
9409
+ `Cannot automatically recover interrupted global clear ${owner.token}: the originating recovery state is unknown. The recovery marker was retained for manual inspection.`
9410
+ );
9411
+ }
9412
+ if (owner.clearRecovery === void 0) continue;
9413
+ if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) {
9414
+ throw new Error(
9415
+ `Cannot automatically recover interrupted global clear ${owner.token}: the originating project scope is unknown. The recovery marker was retained for manual inspection.`
9416
+ );
9417
+ }
9418
+ if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) {
9419
+ throw new Error(
9420
+ `Cannot automatically recover interrupted global clear ${owner.token}: the current embedding configuration does not match the originating lease. The recovery marker was retained; retry from the originating project with matching settings.`
9421
+ );
9422
+ }
9423
+ clearScopes.push({
9424
+ projectRoot: owner.projectRoot,
9425
+ scopedRoots: owner.scopedRoots,
9426
+ compatibilityDecision: owner.clearRecovery.compatibilityDecision
9427
+ });
9428
+ }
9429
+ if (clearScopes.length > 0) {
9430
+ this.loadFileHashCache();
9431
+ }
9432
+ for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) {
9433
+ this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision);
9173
9434
  }
9174
9435
  await this.healthCheckUnlocked();
9436
+ this.logger.info(
9437
+ clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint"
9438
+ );
9439
+ return;
9175
9440
  }
9176
- this.logger.info("Recovery complete, next index will re-process all files");
9441
+ this.logger.info("Recovery complete, next index will resume from the last checkpoint");
9177
9442
  }
9178
9443
  *loadSerializedFailedBatches() {
9179
9444
  let warned = false;
@@ -9211,14 +9476,99 @@ var Indexer = class _Indexer {
9211
9476
  state.writer.write(record);
9212
9477
  state.recordsWritten += record.chunks.length;
9213
9478
  }
9214
- finalizeFailedBatchWriteState(state) {
9479
+ finalizeFailedBatchWriteState(state, resolvedChunkIds = /* @__PURE__ */ new Set()) {
9215
9480
  if (state.recordsWritten > 0) {
9216
- state.writer.commit();
9481
+ const seenChunkIds = /* @__PURE__ */ new Set();
9482
+ const retained = [];
9483
+ const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath));
9484
+ for (let i = records.length - 1; i >= 0; i--) {
9485
+ const chunks = records[i].chunks.filter((rawChunk) => {
9486
+ const chunkId = getPendingChunkId(rawChunk);
9487
+ if (chunkId !== null) {
9488
+ if (resolvedChunkIds.has(chunkId)) return false;
9489
+ if (seenChunkIds.has(chunkId)) return false;
9490
+ seenChunkIds.add(chunkId);
9491
+ }
9492
+ return true;
9493
+ });
9494
+ if (chunks.length > 0) {
9495
+ retained.unshift({ ...records[i], chunks });
9496
+ }
9497
+ }
9498
+ state.writer.cleanup();
9499
+ if (retained.length > 0) {
9500
+ writeFailedBatchRecords(this.failedBatchesPath, retained);
9501
+ } else {
9502
+ writeFailedBatchRecords(this.failedBatchesPath, []);
9503
+ this.clearFailedBatchState();
9504
+ }
9217
9505
  return;
9218
9506
  }
9219
- state.writer.cleanup();
9507
+ state.writer.commit();
9220
9508
  this.clearFailedBatchState();
9221
9509
  }
9510
+ getCheckpointIntervalChunks(totalChunks) {
9511
+ return Math.max(
9512
+ this.checkpointIntervalChunks ?? 2e3,
9513
+ Math.floor(totalChunks / 10)
9514
+ );
9515
+ }
9516
+ checkpointIndexRun(database, store, invertedIndex, failedProcessing, resolvedRetryChunkIds, currentFileHashes, committedFilePaths, scopedRoots, configuredProviderInfo) {
9517
+ if (!this.hasProjectForceReembedPending()) {
9518
+ this.saveIndexMetadata(configuredProviderInfo);
9519
+ this.indexCompatibility = { compatible: true };
9520
+ }
9521
+ database.commitWriteTransaction();
9522
+ database.beginWriteTransaction();
9523
+ this.saveInvertedIndex(invertedIndex);
9524
+ store.save();
9525
+ if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0 || failedProcessing.discardedExistingRecords) {
9526
+ for (const metadata of failedProcessing.latestById.values()) {
9527
+ const alreadyMaterialized = metadata.chunks.some((rawChunk) => {
9528
+ const chunkId = getPendingChunkId(rawChunk);
9529
+ return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId);
9530
+ });
9531
+ if (alreadyMaterialized) continue;
9532
+ this.writeFailedBatchRecord(failedProcessing.state, {
9533
+ chunks: metadata.chunks,
9534
+ attemptCount: metadata.attemptCount,
9535
+ error: metadata.error,
9536
+ lastAttempt: metadata.lastAttempt
9537
+ });
9538
+ for (const rawChunk of metadata.chunks) {
9539
+ const chunkId = getPendingChunkId(rawChunk);
9540
+ if (chunkId !== null) {
9541
+ failedProcessing.materializedRetryIds.add(chunkId);
9542
+ }
9543
+ }
9544
+ }
9545
+ this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
9546
+ failedProcessing.state = this.createFailedBatchWriteState();
9547
+ failedProcessing.discardedExistingRecords = false;
9548
+ for (const record of this.loadSerializedFailedBatches()) {
9549
+ for (const rawChunk of record.chunks) {
9550
+ const chunkId = getPendingChunkId(rawChunk);
9551
+ this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] });
9552
+ if (chunkId !== null) {
9553
+ failedProcessing.materializedRetryIds.add(chunkId);
9554
+ }
9555
+ }
9556
+ }
9557
+ }
9558
+ const partialHashes = /* @__PURE__ */ new Map();
9559
+ for (const filePath of committedFilePaths) {
9560
+ const hash = currentFileHashes.get(filePath);
9561
+ if (hash !== void 0) {
9562
+ partialHashes.set(filePath, hash);
9563
+ }
9564
+ }
9565
+ if (scopedRoots) {
9566
+ this.replaceScopedFileHashCache(partialHashes, scopedRoots);
9567
+ } else {
9568
+ this.fileHashCache = partialHashes;
9569
+ this.saveFileHashCache();
9570
+ }
9571
+ }
9222
9572
  clearFailedBatchState() {
9223
9573
  if (existsSync8(this.failedBatchesPath)) {
9224
9574
  try {
@@ -9245,6 +9595,7 @@ var Indexer = class _Indexer {
9245
9595
  prepareFailedBatchProcessing(roots, shouldProcess) {
9246
9596
  const state = this.createFailedBatchWriteState();
9247
9597
  const latestById = /* @__PURE__ */ new Map();
9598
+ let discardedExistingRecords = false;
9248
9599
  try {
9249
9600
  for (const batch of this.loadSerializedFailedBatches()) {
9250
9601
  for (const rawChunk of batch.chunks) {
@@ -9255,10 +9606,12 @@ var Indexer = class _Indexer {
9255
9606
  continue;
9256
9607
  }
9257
9608
  if (!shouldProcess(filePath)) {
9609
+ discardedExistingRecords = true;
9258
9610
  continue;
9259
9611
  }
9260
9612
  const chunkId = getPendingChunkId(rawChunk);
9261
9613
  if (!chunkId) {
9614
+ discardedExistingRecords = true;
9262
9615
  continue;
9263
9616
  }
9264
9617
  const existing = latestById.get(chunkId);
@@ -9266,12 +9619,18 @@ var Indexer = class _Indexer {
9266
9619
  latestById.set(chunkId, {
9267
9620
  attemptCount: batch.attemptCount,
9268
9621
  error: batch.error,
9269
- lastAttempt: batch.lastAttempt
9622
+ lastAttempt: batch.lastAttempt,
9623
+ chunks: [rawChunk]
9270
9624
  });
9271
9625
  }
9272
9626
  }
9273
9627
  }
9274
- return { state, latestById };
9628
+ return {
9629
+ state,
9630
+ latestById,
9631
+ materializedRetryIds: /* @__PURE__ */ new Set(),
9632
+ discardedExistingRecords
9633
+ };
9275
9634
  } catch (error) {
9276
9635
  state.writer.cleanup();
9277
9636
  throw error;
@@ -9307,10 +9666,34 @@ var Indexer = class _Indexer {
9307
9666
  }
9308
9667
  }
9309
9668
  }
9669
+ restoreMissingChunkRows(database, chunks) {
9670
+ const missing = [];
9671
+ for (const chunk of chunks) {
9672
+ if (database.getChunk(chunk.id)) {
9673
+ continue;
9674
+ }
9675
+ missing.push({
9676
+ chunkId: chunk.id,
9677
+ contentHash: chunk.contentHash,
9678
+ filePath: chunk.metadata.filePath,
9679
+ startLine: chunk.metadata.startLine,
9680
+ endLine: chunk.metadata.endLine,
9681
+ nodeType: chunk.metadata.chunkType,
9682
+ name: chunk.metadata.name,
9683
+ language: chunk.metadata.language,
9684
+ blameSha: chunk.metadata.blameSha,
9685
+ blameAuthor: chunk.metadata.blameAuthor,
9686
+ blameAuthorEmail: chunk.metadata.blameAuthorEmail,
9687
+ blameCommittedAt: chunk.metadata.blameCommittedAt,
9688
+ blameSummary: chunk.metadata.blameSummary
9689
+ });
9690
+ }
9691
+ if (missing.length > 0) {
9692
+ database.upsertChunksBatch(missing);
9693
+ }
9694
+ }
9310
9695
  getProviderRateLimits(provider) {
9311
9696
  switch (provider) {
9312
- case "github-copilot":
9313
- return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
9314
9697
  case "openai":
9315
9698
  return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
9316
9699
  case "google":
@@ -9379,10 +9762,11 @@ var Indexer = class _Indexer {
9379
9762
  const embeddingPartsByChunk = /* @__PURE__ */ new Map();
9380
9763
  const completedVectorsByChunkId = /* @__PURE__ */ new Map();
9381
9764
  const completedChunkIds = /* @__PURE__ */ new Set();
9382
- const requestBatches = createPendingEmbeddingRequestBatches(
9383
- chunksNeedingEmbedding,
9384
- getDynamicBatchOptions(options.configuredProviderInfo)
9385
- );
9765
+ const batchOptions = getDynamicBatchOptions(options.configuredProviderInfo, this.config.embedding?.batch);
9766
+ if (options.forceSingleItemBatches && options.configuredProviderInfo.provider === "ollama") {
9767
+ batchOptions.maxBatchItems = 1;
9768
+ }
9769
+ const requestBatches = createPendingEmbeddingRequestBatches(chunksNeedingEmbedding, batchOptions);
9386
9770
  let fatalError;
9387
9771
  for (const requestBatch of requestBatches) {
9388
9772
  await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
@@ -9945,7 +10329,7 @@ var Indexer = class _Indexer {
9945
10329
  }
9946
10330
  if (!this.configuredProviderInfo) {
9947
10331
  throw new Error(
9948
- "No embedding provider available. Configure GitHub Copilot, OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
10332
+ "No embedding provider available. Configure OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
9949
10333
  );
9950
10334
  }
9951
10335
  this.logger.info("Initializing indexer", {
@@ -9976,7 +10360,20 @@ var Indexer = class _Indexer {
9976
10360
  ]);
9977
10361
  }
9978
10362
  if (recoveredOwners.length > 0 && this.config.scope === "project") {
9979
- await this.resetLocalIndexArtifacts();
10363
+ const unknownLegacyForceIndex = recoveredOwners.find(
10364
+ (owner) => this.hasUnknownLegacyForceIndexClear(owner)
10365
+ );
10366
+ if (unknownLegacyForceIndex) {
10367
+ throw new Error(
10368
+ `Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection.`
10369
+ );
10370
+ }
10371
+ const shouldReset = recoveredOwners.some(
10372
+ (owner) => owner.clearRecovery !== void 0 || owner.operation === "clear" && owner.recoveryProtocolVersion !== 1
10373
+ );
10374
+ if (shouldReset) {
10375
+ await this.resetLocalIndexArtifacts();
10376
+ }
9980
10377
  }
9981
10378
  this.store = new VectorStore(storePath, dimensions);
9982
10379
  if (existsSync8(storePath) || existsSync8(vectorMetadataPath)) {
@@ -10492,6 +10889,70 @@ var Indexer = class _Indexer {
10492
10889
  );
10493
10890
  return createCostEstimate(files, configuredProviderInfo);
10494
10891
  }
10892
+ // Dry-run counterpart to index()/forceIndex(): parse the real file set and sum
10893
+ // estimateTokens over the embedding text of every indexable chunk, without
10894
+ // calling the embedding provider or writing to the index. Read-only and
10895
+ // lock-free (mirrors estimateCost). The token sum is the exact value "Tokens
10896
+ // used" climbs to for a force index (cache bypassed); for an incremental it is
10897
+ // an upper bound because cached chunks are counted here but not re-embedded.
10898
+ // Used by index_codebase(dryRun:true) to give a stable, monotonic progress
10899
+ // denominator that matches the live "Tokens used" basis.
10900
+ async dryRunCost() {
10901
+ const { configuredProviderInfo } = await this.ensureInitialized();
10902
+ const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
10903
+ const includePatterns = [...this.config.include, ...this.config.additionalInclude];
10904
+ const { files } = await collectFiles(
10905
+ this.materializedProjectRoot,
10906
+ includePatterns,
10907
+ this.config.exclude,
10908
+ this.config.indexing.maxFileSize,
10909
+ this.getMaterializedKnowledgeBases(),
10910
+ { maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }
10911
+ );
10912
+ let filesCount = 0;
10913
+ let chunksCount = 0;
10914
+ let tokensToEmbed = 0;
10915
+ for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) {
10916
+ const loadedFiles = await Promise.all(batch.map(async (f) => {
10917
+ try {
10918
+ return {
10919
+ path: this.toStoredFilePath(f.path),
10920
+ content: await fsPromises3.readFile(f.path, "utf-8")
10921
+ };
10922
+ } catch {
10923
+ return null;
10924
+ }
10925
+ }));
10926
+ const readable = loadedFiles.filter(
10927
+ (f) => f !== null
10928
+ );
10929
+ filesCount += readable.length;
10930
+ const contentByPath = new Map(readable.map((f) => [f.path, f.content]));
10931
+ const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk);
10932
+ for (const parsed of parsedFiles) {
10933
+ let chunksToProcess = parsed.chunks;
10934
+ if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
10935
+ const content = contentByPath.get(parsed.path);
10936
+ if (content !== void 0) {
10937
+ chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk);
10938
+ }
10939
+ }
10940
+ chunksToProcess = selectIndexableChunks(
10941
+ chunksToProcess,
10942
+ this.config.indexing.maxChunksPerFile,
10943
+ this.config.indexing.semanticOnly
10944
+ );
10945
+ for (const chunk of chunksToProcess) {
10946
+ const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens);
10947
+ chunksCount += 1;
10948
+ for (const text of texts) {
10949
+ tokensToEmbed += estimateTokens2(text);
10950
+ }
10951
+ }
10952
+ }
10953
+ }
10954
+ return { filesCount, chunksCount, tokensToEmbed };
10955
+ }
10495
10956
  async index(onProgress) {
10496
10957
  return this.withIndexMutationLease("index", async (recoveredOwners) => {
10497
10958
  return this.indexUnlocked(onProgress, recoveredOwners);
@@ -10612,7 +11073,17 @@ var Indexer = class _Indexer {
10612
11073
  const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
10613
11074
  for (const file of files) {
10614
11075
  const storedPath = this.toStoredFilePath(file.path);
10615
- const currentHash = hashFile(file.path);
11076
+ let currentHash;
11077
+ try {
11078
+ currentHash = hashFile(file.path);
11079
+ } catch (error) {
11080
+ stats.skippedFiles.push({ path: this.toCanonicalFilePath(file.path), reason: "unreadable" });
11081
+ this.logger.warn("Skipped unreadable file during indexing", {
11082
+ path: file.path,
11083
+ error: getErrorMessage3(error)
11084
+ });
11085
+ continue;
11086
+ }
10616
11087
  currentFileHashes.set(storedPath, currentHash);
10617
11088
  const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
10618
11089
  const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
@@ -10620,7 +11091,8 @@ var Indexer = class _Indexer {
10620
11091
  );
10621
11092
  const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path15.extname(storedPath).toLowerCase() === ".swift";
10622
11093
  const requiresMetalParserUpgrade = reparseCachedMetalFiles && path15.extname(storedPath).toLowerCase() === ".metal";
10623
- if (cachedHashMatches && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
11094
+ const inMigrationScope = forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots);
11095
+ if (cachedHashMatches && !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
10624
11096
  unchangedFilePaths.add(storedPath);
10625
11097
  this.logger.recordCacheHit();
10626
11098
  } else {
@@ -10746,6 +11218,9 @@ var Indexer = class _Indexer {
10746
11218
  }
10747
11219
  }
10748
11220
  let processedChangedFiles = 0;
11221
+ let lastCheckpointChunks = 0;
11222
+ const committedFilePaths = new Set(unchangedFilePaths);
11223
+ const resolvedRetryChunkIds = /* @__PURE__ */ new Set();
10749
11224
  for (const descriptorBatch of iterateOrderedFileBatches(
10750
11225
  changedFileDescriptors,
10751
11226
  (descriptor) => descriptor.sourceBytes,
@@ -10759,7 +11234,7 @@ var Indexer = class _Indexer {
10759
11234
  const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
10760
11235
  const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
10761
11236
  const parseStartTime = performance2.now();
10762
- const parsedFiles = parseFiles(loadedFiles);
11237
+ const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk);
10763
11238
  const parseMs = performance2.now() - parseStartTime;
10764
11239
  this.logger.recordFilesParsed(parsedFiles.length);
10765
11240
  this.logger.recordParseDuration(parseMs);
@@ -10782,7 +11257,7 @@ var Indexer = class _Indexer {
10782
11257
  }
10783
11258
  let chunksToProcess = parsed.chunks;
10784
11259
  if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
10785
- chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
11260
+ chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk);
10786
11261
  }
10787
11262
  chunksToProcess = selectIndexableChunks(
10788
11263
  chunksToProcess,
@@ -10916,6 +11391,10 @@ var Indexer = class _Indexer {
10916
11391
  }
10917
11392
  if (symbolBatch.length > 0) {
10918
11393
  database.upsertSymbolsBatch(symbolBatch);
11394
+ database.addSymbolsToBranchBatch(
11395
+ this.getBranchCatalogKey(),
11396
+ symbolBatch.map((symbol) => symbol.id)
11397
+ );
10919
11398
  }
10920
11399
  if (edgeBatch.length > 0) {
10921
11400
  database.upsertCallEdgesBatch(edgeBatch);
@@ -10951,6 +11430,12 @@ var Indexer = class _Indexer {
10951
11430
  forceReembed: forceScopedReembed,
10952
11431
  reuseCachedEmbeddings: true,
10953
11432
  incrementRepeatedFailures: true,
11433
+ onSucceeded: (succeededChunks) => {
11434
+ database.addChunksToBranchBatch(
11435
+ this.getBranchCatalogKey(),
11436
+ succeededChunks.map((chunk) => chunk.id)
11437
+ );
11438
+ },
10954
11439
  onProgress: (batchProgress) => onProgress?.({
10955
11440
  phase: "embedding",
10956
11441
  filesProcessed: unchangedFilePaths.size + processedChangedFiles,
@@ -10969,6 +11454,27 @@ var Indexer = class _Indexer {
10969
11454
  }
10970
11455
  }
10971
11456
  }
11457
+ for (const descriptor of descriptorBatch) {
11458
+ const existingFileChunks = existingChunksByFile.get(descriptor.storedPath);
11459
+ if (!existingFileChunks || existingFileChunks.size === 0) {
11460
+ committedFilePaths.add(descriptor.storedPath);
11461
+ }
11462
+ }
11463
+ const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks);
11464
+ if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) {
11465
+ lastCheckpointChunks = stats.totalChunks;
11466
+ this.checkpointIndexRun(
11467
+ database,
11468
+ store,
11469
+ invertedIndex,
11470
+ failedProcessing,
11471
+ resolvedRetryChunkIds,
11472
+ currentFileHashes,
11473
+ committedFilePaths,
11474
+ scopedRoots,
11475
+ configuredProviderInfo
11476
+ );
11477
+ }
10972
11478
  }
10973
11479
  const retryableFailedChunks = this.iterateLatestFailedChunks(
10974
11480
  failedProcessing.latestById,
@@ -10989,6 +11495,7 @@ var Indexer = class _Indexer {
10989
11495
  retryableChunksWithExistingData.add(chunk.id);
10990
11496
  }
10991
11497
  }
11498
+ this.restoreMissingChunkRows(database, pendingChunks);
10992
11499
  stats.totalChunks += pendingChunks.length;
10993
11500
  onProgress?.({
10994
11501
  phase: "embedding",
@@ -11011,6 +11518,17 @@ var Indexer = class _Indexer {
11011
11518
  forceReembed: forceScopedReembed,
11012
11519
  reuseCachedEmbeddings: true,
11013
11520
  incrementRepeatedFailures: true,
11521
+ forceSingleItemBatches: true,
11522
+ onSucceeded: (succeededChunks) => {
11523
+ database.addChunksToBranchBatch(
11524
+ this.getBranchCatalogKey(),
11525
+ succeededChunks.map((chunk) => chunk.id)
11526
+ );
11527
+ for (const chunk of succeededChunks) {
11528
+ failedProcessing.latestById.delete(chunk.id);
11529
+ resolvedRetryChunkIds.add(chunk.id);
11530
+ }
11531
+ },
11014
11532
  onProgress: (batchProgress) => onProgress?.({
11015
11533
  phase: "embedding",
11016
11534
  filesProcessed: files.length,
@@ -11028,6 +11546,20 @@ var Indexer = class _Indexer {
11028
11546
  failedForcedChunkIds.add(chunkId);
11029
11547
  }
11030
11548
  }
11549
+ if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) {
11550
+ lastCheckpointChunks = stats.totalChunks;
11551
+ this.checkpointIndexRun(
11552
+ database,
11553
+ store,
11554
+ invertedIndex,
11555
+ failedProcessing,
11556
+ resolvedRetryChunkIds,
11557
+ currentFileHashes,
11558
+ committedFilePaths,
11559
+ scopedRoots,
11560
+ configuredProviderInfo
11561
+ );
11562
+ }
11031
11563
  }
11032
11564
  const removedChunkIds = [];
11033
11565
  for (const [chunkId] of existingChunks) {
@@ -11064,13 +11596,6 @@ var Indexer = class _Indexer {
11064
11596
  if (removedStoredChunks) {
11065
11597
  this.saveInvertedIndex(invertedIndex);
11066
11598
  }
11067
- if (scopedRoots) {
11068
- this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
11069
- } else {
11070
- this.fileHashCache = currentFileHashes;
11071
- this.saveFileHashCache();
11072
- }
11073
- this.finalizeFailedBatchWriteState(failedProcessing.state);
11074
11599
  database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
11075
11600
  database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
11076
11601
  database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
@@ -11079,6 +11604,13 @@ var Indexer = class _Indexer {
11079
11604
  this.indexCompatibility = { compatible: true };
11080
11605
  database.commitWriteTransaction();
11081
11606
  writeTransactionActive = false;
11607
+ this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
11608
+ if (scopedRoots) {
11609
+ this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
11610
+ } else {
11611
+ this.fileHashCache = currentFileHashes;
11612
+ this.saveFileHashCache();
11613
+ }
11082
11614
  stats.durationMs = Date.now() - startTime;
11083
11615
  onProgress?.({
11084
11616
  phase: "complete",
@@ -11102,13 +11634,6 @@ var Indexer = class _Indexer {
11102
11634
  );
11103
11635
  store.save();
11104
11636
  this.saveInvertedIndex(invertedIndex);
11105
- if (scopedRoots) {
11106
- this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
11107
- } else {
11108
- this.fileHashCache = currentFileHashes;
11109
- this.saveFileHashCache();
11110
- }
11111
- this.finalizeFailedBatchWriteState(failedProcessing.state);
11112
11637
  database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
11113
11638
  database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
11114
11639
  database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
@@ -11117,6 +11642,13 @@ var Indexer = class _Indexer {
11117
11642
  this.indexCompatibility = { compatible: true };
11118
11643
  database.commitWriteTransaction();
11119
11644
  writeTransactionActive = false;
11645
+ this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
11646
+ if (scopedRoots) {
11647
+ this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
11648
+ } else {
11649
+ this.fileHashCache = currentFileHashes;
11650
+ this.saveFileHashCache();
11651
+ }
11120
11652
  stats.durationMs = Date.now() - startTime;
11121
11653
  onProgress?.({
11122
11654
  phase: "complete",
@@ -11151,15 +11683,15 @@ var Indexer = class _Indexer {
11151
11683
  );
11152
11684
  store.save();
11153
11685
  this.saveInvertedIndex(invertedIndex);
11686
+ database.commitWriteTransaction();
11687
+ writeTransactionActive = false;
11688
+ this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds);
11154
11689
  if (scopedRoots) {
11155
11690
  this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
11156
11691
  } else {
11157
11692
  this.fileHashCache = currentFileHashes;
11158
11693
  this.saveFileHashCache();
11159
11694
  }
11160
- this.finalizeFailedBatchWriteState(failedProcessing.state);
11161
- database.commitWriteTransaction();
11162
- writeTransactionActive = false;
11163
11695
  if (this.config.indexing.autoGc && stats.removedChunks > 0) {
11164
11696
  const gcReset = await this.maybeRunOrphanGc();
11165
11697
  if (gcReset) {
@@ -11183,6 +11715,9 @@ var Indexer = class _Indexer {
11183
11715
  if (forceScopedReembed && failedForcedChunkIds.size === 0) {
11184
11716
  database.deleteMetadata(this.getProjectForceReembedMetadataKey());
11185
11717
  }
11718
+ if (forceScopedReembed) {
11719
+ database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true");
11720
+ }
11186
11721
  database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
11187
11722
  database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
11188
11723
  database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
@@ -11293,26 +11828,41 @@ var Indexer = class _Indexer {
11293
11828
  shouldPrefilterByBranch: branchChunkIds !== null && (this.config.scope === "global" || hasInitializedBranchCatalog)
11294
11829
  };
11295
11830
  }
11296
- searchCandidatesWithBranchPrefilter(initialLimit, totalCount, branchChunkIds, shouldPrefilterByBranch, search, getChunkId) {
11831
+ searchCandidatesWithAllowedIds(initialLimit, totalCount, allowedChunkIds, shouldPrefilter, search, getChunkId) {
11297
11832
  const normalizedLimit = Math.max(0, Math.floor(initialLimit));
11298
11833
  if (normalizedLimit === 0) return [];
11299
- if (!shouldPrefilterByBranch || !branchChunkIds) {
11834
+ if (!shouldPrefilter || !allowedChunkIds) {
11300
11835
  return search(normalizedLimit);
11301
11836
  }
11302
- const targetCount = Math.min(normalizedLimit, branchChunkIds.size);
11837
+ const targetCount = Math.min(normalizedLimit, allowedChunkIds.size);
11303
11838
  if (targetCount === 0 || totalCount === 0) return [];
11304
11839
  let requestedLimit = Math.min(normalizedLimit, totalCount);
11305
11840
  while (true) {
11306
11841
  const results = search(requestedLimit);
11307
- const branchResults = results.filter((candidate) => branchChunkIds.has(getChunkId(candidate)));
11308
- if (branchResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
11309
- return branchResults;
11842
+ const allowedResults = results.filter((candidate) => allowedChunkIds.has(getChunkId(candidate)));
11843
+ if (allowedResults.length >= targetCount || results.length < requestedLimit || requestedLimit >= totalCount) {
11844
+ return allowedResults;
11310
11845
  }
11311
11846
  const nextLimit = Math.min(totalCount, Math.max(requestedLimit + 1, requestedLimit * 2));
11312
- if (nextLimit === requestedLimit) return branchResults;
11847
+ if (nextLimit === requestedLimit) return allowedResults;
11313
11848
  requestedLimit = nextLimit;
11314
11849
  }
11315
11850
  }
11851
+ getTemporalChunkIds(database, options) {
11852
+ if (!options?.blameSince && !options?.blameUntil) return null;
11853
+ const since = options.blameSince ? parseBlameTimestamp(options.blameSince, false) : void 0;
11854
+ const until = options.blameUntil ? parseBlameTimestamp(options.blameUntil, true) : void 0;
11855
+ if (since === null || until === null) {
11856
+ return /* @__PURE__ */ new Set();
11857
+ }
11858
+ return new Set(database.getChunkIdsByBlameDate(since, until));
11859
+ }
11860
+ intersectChunkIdSets(first, second) {
11861
+ if (first === null) return second;
11862
+ if (second === null) return first;
11863
+ const [smaller, larger] = first.size <= second.size ? [first, second] : [second, first];
11864
+ return new Set(Array.from(smaller).filter((chunkId) => larger.has(chunkId)));
11865
+ }
11316
11866
  buildCandidateSnapshot(candidate) {
11317
11867
  return {
11318
11868
  id: candidate.id,
@@ -11327,13 +11877,16 @@ var Indexer = class _Indexer {
11327
11877
  buildCandidateSnapshotList(candidates) {
11328
11878
  return candidates.map((candidate) => this.buildCandidateSnapshot(candidate));
11329
11879
  }
11330
- searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch) {
11331
- return this.searchCandidatesWithBranchPrefilter(
11332
- initialLimit,
11333
- store.count(),
11880
+ searchSemanticCandidates(store, embedding, initialLimit, branchChunkIds, shouldPrefilterByBranch, temporalChunkIds) {
11881
+ const availableCount = temporalChunkIds?.size ?? store.count();
11882
+ if (availableCount === 0) return [];
11883
+ const allowedIds = temporalChunkIds === null ? void 0 : Array.from(temporalChunkIds);
11884
+ return this.searchCandidatesWithAllowedIds(
11885
+ Math.min(initialLimit, availableCount),
11886
+ availableCount,
11334
11887
  branchChunkIds,
11335
11888
  shouldPrefilterByBranch,
11336
- (requestedLimit) => store.search(embedding, requestedLimit),
11889
+ (requestedLimit) => store.search(embedding, requestedLimit, allowedIds),
11337
11890
  (candidate) => candidate.id
11338
11891
  );
11339
11892
  }
@@ -11358,8 +11911,9 @@ var Indexer = class _Indexer {
11358
11911
  const rerankTopN = this.config.search.rerankTopN;
11359
11912
  const filterByBranch = options?.filterByBranch ?? true;
11360
11913
  const sourceIntent = options?.definitionIntent === true || classifyQueryIntentRaw(query) === "source";
11914
+ const prioritizeSourcePaths = sourceIntent || options?.prioritizeSourcePaths === true;
11361
11915
  const identifierHints = extractIdentifierHints(query);
11362
- const candidateLimit = maxResults * (sourceIntent ? 12 : 4);
11916
+ const candidateLimit = maxResults * (prioritizeSourcePaths ? 12 : 4);
11363
11917
  this.logger.search("debug", "Starting search", {
11364
11918
  query,
11365
11919
  maxResults,
@@ -11390,6 +11944,7 @@ var Indexer = class _Indexer {
11390
11944
  branchChunkIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchChunkIds(branchKey)));
11391
11945
  branchSymbolIds = new Set(branchCatalogKeys.flatMap((branchKey) => database.getBranchSymbolIds(branchKey)));
11392
11946
  }
11947
+ const temporalChunkIds = this.getTemporalChunkIds(database, options);
11393
11948
  const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
11394
11949
  const prefilterMs = performance2.now() - prefilterStartTime;
11395
11950
  const vectorStartTime = performance2.now();
@@ -11398,7 +11953,8 @@ var Indexer = class _Indexer {
11398
11953
  embedding,
11399
11954
  candidateLimit,
11400
11955
  branchChunkIds,
11401
- shouldPrefilterByBranch
11956
+ shouldPrefilterByBranch,
11957
+ temporalChunkIds
11402
11958
  ) : [];
11403
11959
  const vectorMs = performance2.now() - vectorStartTime;
11404
11960
  const keywordStartTime = performance2.now();
@@ -11408,7 +11964,8 @@ var Indexer = class _Indexer {
11408
11964
  store,
11409
11965
  invertedIndex,
11410
11966
  branchChunkIds,
11411
- shouldPrefilterByBranch
11967
+ shouldPrefilterByBranch,
11968
+ temporalChunkIds
11412
11969
  );
11413
11970
  const keywordMs = performance2.now() - keywordStartTime;
11414
11971
  const scopedSemanticCandidates = semanticCandidates.filter(
@@ -11430,7 +11987,7 @@ var Indexer = class _Indexer {
11430
11987
  rerankTopN,
11431
11988
  limit: maxResults,
11432
11989
  hybridWeight: rankingHybridWeight,
11433
- prioritizeSourcePaths: sourceIntent
11990
+ prioritizeSourcePaths
11434
11991
  });
11435
11992
  const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
11436
11993
  definitionIntent: options?.definitionIntent === true,
@@ -11466,10 +12023,11 @@ var Indexer = class _Indexer {
11466
12023
  branchSymbolIds,
11467
12024
  maxResults,
11468
12025
  union,
11469
- sourceIntent
12026
+ sourceIntent,
12027
+ options?.definitionIntent === true && ((options.directory?.trim().length ?? 0) > 0 || (options.fileType?.trim().length ?? 0) > 0)
11470
12028
  );
11471
12029
  const prePrimaryLane = mergeTieredResults(deterministicIdentifierLane, identifierLane, maxResults * 4);
11472
- const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
12030
+ const primaryLane = options?.definitionIntent === true ? mergeTieredResults(symbolLane, prePrimaryLane, maxResults * 4) : mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
11473
12031
  const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
11474
12032
  const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
11475
12033
  const baseFiltered = tiered.filter(
@@ -11564,14 +12122,18 @@ var Indexer = class _Indexer {
11564
12122
  })
11565
12123
  );
11566
12124
  }
11567
- async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false) {
12125
+ async keywordSearch(query, limit, store, invertedIndex, branchChunkIds = null, shouldPrefilterByBranch = false, temporalChunkIds = null) {
11568
12126
  const normalizedLimit = Math.max(0, Math.floor(limit));
11569
12127
  if (normalizedLimit === 0) return [];
11570
- const scoreEntries = this.searchCandidatesWithBranchPrefilter(
12128
+ const allowedChunkIds = this.intersectChunkIdSets(
12129
+ shouldPrefilterByBranch ? branchChunkIds : null,
12130
+ temporalChunkIds
12131
+ );
12132
+ const scoreEntries = this.searchCandidatesWithAllowedIds(
11571
12133
  normalizedLimit,
11572
12134
  invertedIndex.getDocumentCount(),
11573
- branchChunkIds,
11574
- shouldPrefilterByBranch,
12135
+ allowedChunkIds,
12136
+ allowedChunkIds !== null,
11575
12137
  (requestedLimit) => Array.from(invertedIndex.search(query, requestedLimit)),
11576
12138
  ([chunkId]) => chunkId
11577
12139
  );
@@ -11656,7 +12218,17 @@ var Indexer = class _Indexer {
11656
12218
  );
11657
12219
  const currentFileHashes = /* @__PURE__ */ new Map();
11658
12220
  for (const file of files) {
11659
- currentFileHashes.set(this.toStoredFilePath(file.path), hashFile(file.path));
12221
+ let hash;
12222
+ try {
12223
+ hash = hashFile(file.path);
12224
+ } catch (error) {
12225
+ this.logger.warn("Skipped unreadable file during freshness check", {
12226
+ path: file.path,
12227
+ error: getErrorMessage3(error)
12228
+ });
12229
+ return { readable: false, current: false, reason: "unreadable" };
12230
+ }
12231
+ currentFileHashes.set(this.toStoredFilePath(file.path), hash);
11660
12232
  }
11661
12233
  const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
11662
12234
  const cachedFileHashes = scopedRoots ? new Map(Array.from(this.fileHashCache).filter(([filePath]) => this.isFileInCurrentScope(filePath, scopedRoots))) : this.fileHashCache;
@@ -11682,69 +12254,87 @@ var Indexer = class _Indexer {
11682
12254
  async forceIndex(onProgress) {
11683
12255
  return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
11684
12256
  await this.ensureInitializedUnlocked(recoveredOwners);
11685
- await this.clearIndexUnlocked();
12257
+ const recovery = this.beginClearRecoveryState();
12258
+ await this.clearIndexUnlocked(recovery.compatibilityDecision);
12259
+ this.finishClearRecoveryState();
11686
12260
  return this.indexUnlocked(onProgress, [], true);
11687
12261
  });
11688
12262
  }
11689
12263
  async clearIndex() {
11690
12264
  await this.withIndexMutationLease("clear", async (recoveredOwners) => {
11691
12265
  await this.ensureInitializedUnlocked(recoveredOwners);
11692
- await this.clearIndexUnlocked();
12266
+ const recovery = this.beginClearRecoveryState();
12267
+ await this.clearIndexUnlocked(recovery.compatibilityDecision);
11693
12268
  });
11694
12269
  }
11695
- async clearIndexUnlocked() {
12270
+ clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot) {
11696
12271
  const { store, invertedIndex, database } = this.requireLoadedIndexState();
11697
- if (this.config.scope === "global") {
11698
- store.load();
11699
- invertedIndex.load();
11700
- this.loadFileHashCache();
11701
- const roots = this.getScopedRoots();
11702
- const compatibility = this.checkCompatibility();
11703
- const allMetadata = store.getAllMetadata();
11704
- const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData() || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
11705
- if (!compatibility.compatible && hasForeignData) {
11706
- if (compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */) {
11707
- this.clearSharedIndexProjectData(store, invertedIndex, database, roots);
11708
- this.clearScopedFileHashCache(roots);
11709
- this.clearScopedFailedBatches(roots);
11710
- database.setMetadata(this.getProjectForceReembedMetadataKey(), "true");
11711
- database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
12272
+ const clearedBranchKeys = database.getAllBranches();
12273
+ store.clear();
12274
+ store.save();
12275
+ invertedIndex.clear();
12276
+ this.saveInvertedIndex(invertedIndex);
12277
+ this.fileHashCache.clear();
12278
+ this.saveFileHashCache();
12279
+ database.clearAllIndexedData();
12280
+ this.deleteBranchCommitMetadata(database, clearedBranchKeys);
12281
+ this.clearFailedBatchState();
12282
+ database.deleteMetadata("index.version");
12283
+ database.deleteMetadata("index.pathStorageVersion");
12284
+ database.deleteMetadata("index.embeddingProvider");
12285
+ database.deleteMetadata("index.embeddingModel");
12286
+ database.deleteMetadata("index.embeddingDimensions");
12287
+ database.deleteMetadata("index.embeddingStrategyVersion");
12288
+ const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
12289
+ database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
12290
+ database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash));
12291
+ database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash));
12292
+ database.deleteMetadata("index.createdAt");
12293
+ database.deleteMetadata("index.updatedAt");
12294
+ this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
12295
+ }
12296
+ clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots(), recoveryDecision) {
12297
+ const { store, invertedIndex, database } = this.requireLoadedIndexState();
12298
+ store.load();
12299
+ invertedIndex.load();
12300
+ this.loadFileHashCache();
12301
+ const compatibility = this.checkCompatibility();
12302
+ const compatibilityDecision = recoveryDecision ?? (compatibility.compatible ? "compatible" : compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */ ? "embedding-strategy-mismatch" : "incompatible");
12303
+ const allMetadata = store.getAllMetadata();
12304
+ const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
12305
+ if (compatibilityDecision !== "compatible" && hasForeignData) {
12306
+ if (compatibilityDecision === "embedding-strategy-mismatch") {
12307
+ this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
12308
+ this.clearScopedFileHashCache(roots);
12309
+ this.clearScopedFailedBatches(roots);
12310
+ const projectIdentityHash = this.getProjectIdentityHash(projectRoot);
12311
+ database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true");
12312
+ database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash));
12313
+ database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash));
12314
+ if (projectRoot === this.projectRoot) {
11712
12315
  this.indexCompatibility = { compatible: true };
11713
- return;
11714
12316
  }
11715
- throw new Error(
11716
- `Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
11717
- );
11718
- }
11719
- if (!hasForeignData) {
11720
- const clearedBranchKeys2 = database.getAllBranches();
11721
- store.clear();
11722
- store.save();
11723
- invertedIndex.clear();
11724
- this.saveInvertedIndex(invertedIndex);
11725
- this.fileHashCache.clear();
11726
- this.saveFileHashCache();
11727
- database.clearAllIndexedData();
11728
- this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
11729
- this.clearFailedBatchState();
11730
- database.deleteMetadata("index.version");
11731
- database.deleteMetadata("index.pathStorageVersion");
11732
- database.deleteMetadata("index.embeddingProvider");
11733
- database.deleteMetadata("index.embeddingModel");
11734
- database.deleteMetadata("index.embeddingDimensions");
11735
- database.deleteMetadata("index.embeddingStrategyVersion");
11736
- database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
11737
- database.deleteMetadata(this.getProjectForceReembedMetadataKey());
11738
- database.deleteMetadata(this.getLegacyMigrationMetadataKey());
11739
- database.deleteMetadata("index.createdAt");
11740
- database.deleteMetadata("index.updatedAt");
11741
- this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
11742
12317
  return;
11743
12318
  }
11744
- this.clearSharedIndexProjectData(store, invertedIndex, database, roots);
11745
- this.clearScopedFileHashCache(roots);
11746
- this.clearScopedFailedBatches(roots);
12319
+ throw new Error(
12320
+ `Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
12321
+ );
12322
+ }
12323
+ if (!hasForeignData) {
12324
+ this.clearGlobalIndexDataUnlocked(projectRoot);
12325
+ return;
12326
+ }
12327
+ this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot);
12328
+ this.clearScopedFileHashCache(roots);
12329
+ this.clearScopedFailedBatches(roots);
12330
+ if (projectRoot === this.projectRoot) {
11747
12331
  this.indexCompatibility = compatibility;
12332
+ }
12333
+ }
12334
+ async clearIndexUnlocked(recoveryDecision) {
12335
+ const { store, invertedIndex, database } = this.requireLoadedIndexState();
12336
+ if (this.config.scope === "global") {
12337
+ this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision);
11748
12338
  return;
11749
12339
  }
11750
12340
  if (!this.isProjectOwnedIndexPath()) {
@@ -11910,6 +12500,7 @@ var Indexer = class _Indexer {
11910
12500
  )) {
11911
12501
  const chunks = retryBatch.map(({ chunk }) => chunk);
11912
12502
  const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
12503
+ this.restoreMissingChunkRows(database, chunks);
11913
12504
  const batchResult = await this.processPendingChunkBatch(chunks, {
11914
12505
  store,
11915
12506
  provider,
@@ -11924,6 +12515,7 @@ var Indexer = class _Indexer {
11924
12515
  forceReembed: false,
11925
12516
  reuseCachedEmbeddings: false,
11926
12517
  incrementRepeatedFailures: false,
12518
+ forceSingleItemBatches: true,
11927
12519
  onSucceeded: (succeededChunks) => {
11928
12520
  database.addChunksToBranchBatch(
11929
12521
  this.getBranchCatalogKey(),
@@ -11945,9 +12537,12 @@ var Indexer = class _Indexer {
11945
12537
  this.saveInvertedIndex(invertedIndex);
11946
12538
  }
11947
12539
  if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
11948
- database.deleteMetadata(this.getProjectForceReembedMetadataKey());
11949
- this.saveIndexMetadata(configuredProviderInfo);
11950
- this.indexCompatibility = { compatible: true };
12540
+ const migrationFinalized = database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true";
12541
+ if (migrationFinalized) {
12542
+ database.deleteMetadata(this.getProjectForceReembedMetadataKey());
12543
+ this.saveIndexMetadata(configuredProviderInfo);
12544
+ this.indexCompatibility = { compatible: true };
12545
+ }
11951
12546
  }
11952
12547
  return { succeeded, failed, remaining };
11953
12548
  }
@@ -11969,7 +12564,8 @@ var Indexer = class _Indexer {
11969
12564
  latestById.set(chunkId, {
11970
12565
  attemptCount: batch.attemptCount,
11971
12566
  error: batch.error,
11972
- lastAttempt: batch.lastAttempt
12567
+ lastAttempt: batch.lastAttempt,
12568
+ chunks: [rawChunk]
11973
12569
  });
11974
12570
  }
11975
12571
  }
@@ -12036,6 +12632,7 @@ var Indexer = class _Indexer {
12036
12632
  this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
12037
12633
  );
12038
12634
  }
12635
+ const temporalChunkIds = this.getTemporalChunkIds(database, options);
12039
12636
  const { hasInitializedBranchCatalog, shouldPrefilterByBranch } = this.getBranchPrefilterState(database, branchChunkIds);
12040
12637
  const prefilterMs = performance2.now() - prefilterStartTime;
12041
12638
  const vectorStartTime = performance2.now();
@@ -12044,7 +12641,8 @@ var Indexer = class _Indexer {
12044
12641
  embedding,
12045
12642
  limit * 2,
12046
12643
  branchChunkIds,
12047
- shouldPrefilterByBranch
12644
+ shouldPrefilterByBranch,
12645
+ temporalChunkIds
12048
12646
  );
12049
12647
  const vectorMs = performance2.now() - vectorStartTime;
12050
12648
  if (this.config.scope !== "global" && branchChunkIds && !hasInitializedBranchCatalog) {
@@ -12626,6 +13224,36 @@ function resolveConfigPathValue(value, baseDir) {
12626
13224
  const absolutePath = path16.isAbsolute(trimmed) ? trimmed : path16.resolve(baseDir, trimmed);
12627
13225
  return path16.normalize(absolutePath);
12628
13226
  }
13227
+ function serializeConfigPathValue(value, baseDir) {
13228
+ const trimmed = value.trim();
13229
+ if (!trimmed) {
13230
+ return trimmed;
13231
+ }
13232
+ if (!path16.isAbsolute(trimmed)) {
13233
+ return normalizePathSeparators(path16.normalize(trimmed));
13234
+ }
13235
+ const relativePath = path16.relative(baseDir, trimmed);
13236
+ if (!relativePath || !relativePath.startsWith("..") && !path16.isAbsolute(relativePath)) {
13237
+ return normalizePathSeparators(path16.normalize(relativePath || "."));
13238
+ }
13239
+ return path16.normalize(trimmed);
13240
+ }
13241
+ function resolveKnowledgeBasePath(value, projectRoot) {
13242
+ return path16.isAbsolute(value) ? value : path16.resolve(projectRoot, value);
13243
+ }
13244
+ function normalizeKnowledgeBasePath(value, projectRoot) {
13245
+ return path16.normalize(resolveKnowledgeBasePath(value, projectRoot));
13246
+ }
13247
+ function hasMatchingKnowledgeBasePath(knowledgeBases, inputPath, projectRoot) {
13248
+ const normalizedInput = path16.normalize(inputPath);
13249
+ return knowledgeBases.some((kb) => normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput);
13250
+ }
13251
+ function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot) {
13252
+ const normalizedInput = path16.normalize(inputPath);
13253
+ return knowledgeBases.findIndex(
13254
+ (kb) => path16.normalize(kb) === normalizedInput || normalizeKnowledgeBasePath(kb, projectRoot) === normalizedInput
13255
+ );
13256
+ }
12629
13257
 
12630
13258
  // src/tools/format-communities.ts
12631
13259
  function compareText(left, right) {
@@ -14332,7 +14960,7 @@ function applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key
14332
14960
  function mergeUniqueStringArray(values) {
14333
14961
  return [...new Set(values.map((value) => String(value).trim()))];
14334
14962
  }
14335
- function normalizeKnowledgeBasePath(value) {
14963
+ function normalizeKnowledgeBasePath2(value) {
14336
14964
  let normalized = path19.normalize(String(value).trim());
14337
14965
  const root = path19.parse(normalized).root;
14338
14966
  while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
@@ -14341,7 +14969,7 @@ function normalizeKnowledgeBasePath(value) {
14341
14969
  return normalized;
14342
14970
  }
14343
14971
  function mergeKnowledgeBasePaths(values) {
14344
- return [...new Set(values.map((value) => normalizeKnowledgeBasePath(value)).filter((value) => value.length > 0))];
14972
+ return [...new Set(values.map((value) => normalizeKnowledgeBasePath2(value)).filter((value) => value.length > 0))];
14345
14973
  }
14346
14974
  function validateConfigLayerShape(rawConfig, filePath) {
14347
14975
  if (!isRecord(rawConfig)) {
@@ -14471,9 +15099,30 @@ function toConfigRecord(rawConfig) {
14471
15099
  }
14472
15100
  return { ...rawConfig };
14473
15101
  }
15102
+ function getConfigPath(projectRoot, host) {
15103
+ return resolveWritableProjectConfigPath(projectRoot, host);
15104
+ }
14474
15105
  function loadRuntimeConfig(projectRoot, host) {
14475
15106
  return normalizeKnowledgeBasePaths(toConfigRecord(loadMergedConfig(projectRoot, host)), projectRoot);
14476
15107
  }
15108
+ function loadEditableConfig(projectRoot, host) {
15109
+ return normalizeKnowledgeBasePaths(toConfigRecord(loadProjectConfigLayer(projectRoot, host)), projectRoot);
15110
+ }
15111
+ function saveConfig(projectRoot, config, host) {
15112
+ const configPath = getConfigPath(projectRoot, host);
15113
+ const configDir = path20.dirname(configPath);
15114
+ const configBaseDir = path20.dirname(configDir);
15115
+ if (!existsSync11(configDir)) {
15116
+ mkdirSync5(configDir, { recursive: true });
15117
+ }
15118
+ const serializableConfig = { ...config };
15119
+ if (Array.isArray(serializableConfig.knowledgeBases)) {
15120
+ serializableConfig.knowledgeBases = serializableConfig.knowledgeBases.map(
15121
+ (kb) => serializeConfigPathValue(kb, configBaseDir)
15122
+ );
15123
+ }
15124
+ writeFileSync4(configPath, JSON.stringify(serializableConfig, null, 2) + "\n", "utf-8");
15125
+ }
14477
15126
 
14478
15127
  // src/tools/operation-runtime.ts
14479
15128
  var indexerCache = /* @__PURE__ */ new Map();
@@ -14692,9 +15341,11 @@ async function searchCodebase(projectRoot, host, query, options = {}) {
14692
15341
  contextLines: options.contextLines,
14693
15342
  metadataOnly: options.metadataOnly,
14694
15343
  definitionIntent: options.definitionIntent,
15344
+ prioritizeSourcePaths: options.prioritizeSourcePaths,
14695
15345
  blameAuthor: options.blameAuthor,
14696
15346
  blameSha: options.blameSha,
14697
15347
  blameSince: options.blameSince,
15348
+ blameUntil: options.blameUntil,
14698
15349
  trace: options.trace
14699
15350
  });
14700
15351
  }
@@ -14740,7 +15391,9 @@ async function findSimilarCode(projectRoot, host, code, options = {}) {
14740
15391
  fileType: options.fileType,
14741
15392
  directory: options.directory,
14742
15393
  chunkType: options.chunkType,
14743
- excludeFile: options.excludeFile
15394
+ excludeFile: options.excludeFile,
15395
+ blameSince: options.blameSince,
15396
+ blameUntil: options.blameUntil
14744
15397
  });
14745
15398
  }
14746
15399
  async function implementationLookup(projectRoot, host, query, options = {}) {
@@ -14803,6 +15456,9 @@ async function runIndexCodebase(projectRoot, host, args, onProgress) {
14803
15456
  if (args.estimateOnly) {
14804
15457
  return { kind: "estimate", estimate: await indexer.estimateCost() };
14805
15458
  }
15459
+ if (args.dryRun) {
15460
+ return { kind: "dryrun", dryrun: await indexer.dryRunCost() };
15461
+ }
14806
15462
  const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => {
14807
15463
  if (onProgress) {
14808
15464
  void onProgress(formatProgressTitle(progress), {
@@ -14986,6 +15642,141 @@ async function getIndexLogs(projectRoot, host, args) {
14986
15642
  }).join("\n");
14987
15643
  return { kind: "entries", text };
14988
15644
  }
15645
+ function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
15646
+ const root = getProjectRoot(projectRoot, host);
15647
+ const inputPath = knowledgeBasePath.trim();
15648
+ const normalizedPath3 = path21.resolve(
15649
+ path21.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
15650
+ );
15651
+ if (!existsSync12(normalizedPath3)) {
15652
+ return `Error: Directory does not exist: ${normalizedPath3}`;
15653
+ }
15654
+ let realPath;
15655
+ try {
15656
+ realPath = realpathSync5(normalizedPath3);
15657
+ } catch {
15658
+ return `Error: Cannot resolve path: ${normalizedPath3}`;
15659
+ }
15660
+ const blockedPrefixes = [
15661
+ "/etc",
15662
+ "/proc",
15663
+ "/sys",
15664
+ "/dev",
15665
+ "/boot",
15666
+ "/root",
15667
+ "/var/run",
15668
+ "/var/log"
15669
+ ];
15670
+ const homeDir = process.platform === "win32" ? process.env.USERPROFILE ?? "" : process.env.HOME ?? "";
15671
+ const sensitiveDotDirs = [
15672
+ ".ssh",
15673
+ ".gnupg",
15674
+ ".aws",
15675
+ ".config/gcloud",
15676
+ ".docker",
15677
+ ".kube"
15678
+ ];
15679
+ for (const prefix of blockedPrefixes) {
15680
+ if (realPath === prefix || realPath.startsWith(`${prefix}/`)) {
15681
+ return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
15682
+ }
15683
+ }
15684
+ for (const dotDir of sensitiveDotDirs) {
15685
+ const sensitiveDir = path21.join(homeDir, dotDir);
15686
+ if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
15687
+ return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath3}`;
15688
+ }
15689
+ }
15690
+ try {
15691
+ const stat5 = statSync5(normalizedPath3);
15692
+ if (!stat5.isDirectory()) {
15693
+ return `Error: Path is not a directory: ${normalizedPath3}`;
15694
+ }
15695
+ } catch (error) {
15696
+ return `Error: Cannot access directory: ${normalizedPath3} - ${error instanceof Error ? error.message : String(error)}`;
15697
+ }
15698
+ const config = loadEditableConfig(root, host);
15699
+ const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
15700
+ const alreadyExists = hasMatchingKnowledgeBasePath(knowledgeBases, normalizedPath3, root);
15701
+ if (alreadyExists) {
15702
+ return `Knowledge base already configured: ${normalizedPath3}`;
15703
+ }
15704
+ knowledgeBases.push(normalizedPath3);
15705
+ config.knowledgeBases = knowledgeBases;
15706
+ saveConfig(root, config, host);
15707
+ refreshIndexerForDirectory(root, host);
15708
+ let result = `${normalizedPath3}
15709
+ `;
15710
+ result += `Total knowledge bases: ${knowledgeBases.length}
15711
+ `;
15712
+ result += `Config path: ${getConfigPath(root, host)}
15713
+ `;
15714
+ result += `
15715
+ Run /index to rebuild the index with the new knowledge base.`;
15716
+ return result;
15717
+ }
15718
+ function listKnowledgeBases(projectRoot, host) {
15719
+ const root = getProjectRoot(projectRoot, host);
15720
+ const config = loadRuntimeConfig(root, host);
15721
+ const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
15722
+ if (knowledgeBases.length === 0) {
15723
+ return "No knowledge bases configured. Use add_knowledge_base to add folders.";
15724
+ }
15725
+ let result = `Knowledge Bases (${knowledgeBases.length}):
15726
+
15727
+ `;
15728
+ for (let i = 0; i < knowledgeBases.length; i++) {
15729
+ const kb = knowledgeBases[i];
15730
+ const resolvedPath = resolveKnowledgeBasePath(kb, root);
15731
+ const exists = existsSync12(resolvedPath);
15732
+ result += `[${i + 1}] ${kb}
15733
+ `;
15734
+ result += ` Resolved: ${resolvedPath}
15735
+ `;
15736
+ result += ` Status: ${exists ? "Exists" : "NOT FOUND"}
15737
+ `;
15738
+ if (exists) {
15739
+ try {
15740
+ const stat5 = statSync5(resolvedPath);
15741
+ result += ` Type: ${stat5.isDirectory() ? "Directory" : "File"}
15742
+ `;
15743
+ } catch {
15744
+ }
15745
+ }
15746
+ result += "\n";
15747
+ }
15748
+ const hasHostConfig = existsSync12(path21.join(root, getHostProjectConfigRelativePath(host)));
15749
+ if (hasHostConfig) {
15750
+ result += `
15751
+ Config sources: 1 file(s).`;
15752
+ }
15753
+ result += `
15754
+ Config file: ${getConfigPath(root, host)}`;
15755
+ return result;
15756
+ }
15757
+ function removeKnowledgeBase(projectRoot, host, knowledgeBasePath) {
15758
+ const root = getProjectRoot(projectRoot, host);
15759
+ const config = loadEditableConfig(root, host);
15760
+ const knowledgeBases = Array.isArray(config.knowledgeBases) ? config.knowledgeBases : [];
15761
+ const index = findKnowledgeBasePathIndex(knowledgeBases, knowledgeBasePath, root);
15762
+ if (index === -1) {
15763
+ return `Knowledge base not found: ${knowledgeBasePath}`;
15764
+ }
15765
+ const removed = knowledgeBases.splice(index, 1)[0];
15766
+ config.knowledgeBases = knowledgeBases;
15767
+ saveConfig(root, config, host);
15768
+ refreshIndexerForDirectory(root, host);
15769
+ let result = `Removed: ${removed}
15770
+
15771
+ `;
15772
+ result += `Remaining knowledge bases: ${knowledgeBases.length}
15773
+ `;
15774
+ result += `Config saved to: ${getConfigPath(root, host)}
15775
+ `;
15776
+ result += `
15777
+ Run /index to rebuild the index without the removed knowledge base.`;
15778
+ return result;
15779
+ }
14989
15780
 
14990
15781
  // src/tools/context-search.ts
14991
15782
  var MIN_CONTEXT_RESULT_LIMIT = 1;
@@ -15214,13 +16005,19 @@ async function resolveSearchContext(input, operations) {
15214
16005
  (trace) => operations.lookup(symbol, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
15215
16006
  );
15216
16007
  };
15217
- const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt) => {
16008
+ const tryConceptualSearch = async (searchQuery, scope, relaxedFieldsForAttempt, prioritizeSourcePaths) => {
15218
16009
  return recordAttempt(
15219
16010
  "conceptual",
15220
16011
  searchQuery,
15221
16012
  scope,
15222
16013
  relaxedFieldsForAttempt,
15223
- (trace) => operations.search(searchQuery, MAX_CONTEXT_RESULT_LIMIT, scope, input.diagnostic ? trace : void 0)
16014
+ (trace) => operations.search(
16015
+ searchQuery,
16016
+ MAX_CONTEXT_RESULT_LIMIT,
16017
+ scope,
16018
+ input.diagnostic ? trace : void 0,
16019
+ { prioritizeSourcePaths }
16020
+ )
15224
16021
  );
15225
16022
  };
15226
16023
  const findSuccessfulAttemptState = (route) => {
@@ -15348,10 +16145,12 @@ Explicit symbol lookup only; conceptual search was not attempted.`
15348
16145
  }
15349
16146
  }
15350
16147
  for (const attempt of conceptualAttemptPlan) {
16148
+ const attemptIntent = analyzeQueryIntent(attempt.queryText);
16149
+ const prioritizeSourcePaths = attemptIntent.primary !== "docs" && attemptIntent.primary !== "test";
15351
16150
  if (inferredSymbol && attempt.queryText === inferredSymbol && attempt.queryText !== query) {
15352
16151
  decisions.fallbackFromOriginalConceptualToInferred = true;
15353
16152
  }
15354
- const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed);
16153
+ const results = await tryConceptualSearch(attempt.queryText, attempt.scope, attempt.relaxed, prioritizeSourcePaths);
15355
16154
  if (results.length > 0) {
15356
16155
  const heading = buildPackHeading("conceptual", decisions);
15357
16156
  const intent = analyzeQueryIntent(attempt.queryText);
@@ -15488,12 +16287,13 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
15488
16287
  directory: scope.directory,
15489
16288
  trace
15490
16289
  }),
15491
- search: (queryText, retrievalLimit, scope, trace) => searchCodebase(projectRoot, host, queryText, {
16290
+ search: (queryText, retrievalLimit, scope, trace, searchOptions) => searchCodebase(projectRoot, host, queryText, {
15492
16291
  limit: retrievalLimit,
15493
16292
  fileType: scope.fileType,
15494
16293
  directory: scope.directory,
15495
16294
  metadataOnly: true,
15496
- trace
16295
+ trace,
16296
+ prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
15497
16297
  })
15498
16298
  });
15499
16299
  }
@@ -16994,12 +17794,13 @@ async function runEvaluation(options) {
16994
17794
  fileType: scope.fileType,
16995
17795
  directory: scope.directory
16996
17796
  }),
16997
- search: (searchQuery, limit, scope) => indexer.search(searchQuery, limit, {
17797
+ search: (searchQuery, limit, scope, _trace, searchOptions) => indexer.search(searchQuery, limit, {
16998
17798
  metadataOnly: true,
16999
17799
  filterByBranch: !!query.expected.branch,
17000
17800
  definitionIntent: false,
17001
17801
  fileType: scope.fileType,
17002
- directory: scope.directory
17802
+ directory: scope.directory,
17803
+ prioritizeSourcePaths: searchOptions?.prioritizeSourcePaths
17003
17804
  })
17004
17805
  }) : void 0;
17005
17806
  const result = editContextResult?.results ?? contextResult?.details?.results ?? await indexer.search(query.query, 10, {
@@ -17584,6 +18385,7 @@ async function executeCodebaseEditContext(projectRoot, host, args) {
17584
18385
  async function executeIndexCodebase(projectRoot, host, args, onProgress) {
17585
18386
  const result = await runIndexCodebase(projectRoot, host, args, onProgress);
17586
18387
  if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) };
18388
+ if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) };
17587
18389
  if (result.kind === "busy") return { text: result.text, isError: true };
17588
18390
  if (result.kind === "message") return { text: result.text };
17589
18391
  return { text: formatIndexStats(result.stats, args.verbose ?? false) };
@@ -17775,11 +18577,21 @@ var PI_TOOL_NAMES = [
17775
18577
  TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
17776
18578
  TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
17777
18579
  ];
18580
+ var MCP_TOOL_NAMES = [
18581
+ ...PORTABLE_TOOL_NAMES,
18582
+ TOOL_NAME.ADD_KNOWLEDGE_BASE,
18583
+ TOOL_NAME.LIST_KNOWLEDGE_BASES,
18584
+ TOOL_NAME.REMOVE_KNOWLEDGE_BASE
18585
+ ];
17778
18586
 
17779
18587
  // src/adapters/mcp/register-tools.ts
17780
18588
  function allowNullAsUndefined(schema) {
17781
18589
  return z2.preprocess((value) => value === null ? void 0 : value, schema);
17782
18590
  }
18591
+ function knowledgeBaseResult(text) {
18592
+ const content = [{ type: "text", text }];
18593
+ return text.startsWith("Error: ") ? { content, isError: true } : { content };
18594
+ }
17783
18595
  function registerMcpTools(server, runtime) {
17784
18596
  server.tool(
17785
18597
  TOOL_NAME.CODEBASE_CONTEXT,
@@ -17840,7 +18652,8 @@ function registerMcpTools(server, runtime) {
17840
18652
  contextLines: allowNullAsUndefined(z2.number().optional()).describe("Number of extra lines to include before/after each match (default: 0)"),
17841
18653
  blameAuthor: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame author name or email"),
17842
18654
  blameSha: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame commit SHA or prefix"),
17843
- blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date")
18655
+ blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date"),
18656
+ blameUntil: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or before this date")
17844
18657
  },
17845
18658
  async (args) => {
17846
18659
  return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "search", args.query, {
@@ -17851,7 +18664,8 @@ function registerMcpTools(server, runtime) {
17851
18664
  contextLines: args.contextLines,
17852
18665
  blameAuthor: args.blameAuthor,
17853
18666
  blameSha: args.blameSha,
17854
- blameSince: args.blameSince
18667
+ blameSince: args.blameSince,
18668
+ blameUntil: args.blameUntil
17855
18669
  }, (results) => {
17856
18670
  const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} results for "${args.query}":
17857
18671
 
@@ -17871,7 +18685,8 @@ ${formatSearchResults(results, "score")}`;
17871
18685
  chunkType: allowNullAsUndefined(z2.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
17872
18686
  blameAuthor: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame author name or email"),
17873
18687
  blameSha: allowNullAsUndefined(z2.string().optional()).describe("Filter by git blame commit SHA or prefix"),
17874
- blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date")
18688
+ blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date"),
18689
+ blameUntil: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or before this date")
17875
18690
  },
17876
18691
  async (args) => {
17877
18692
  return searchCodebaseWithEffectiveness(runtime.projectRoot, runtime.host, "peek", args.query, {
@@ -17882,7 +18697,8 @@ ${formatSearchResults(results, "score")}`;
17882
18697
  metadataOnly: true,
17883
18698
  blameAuthor: args.blameAuthor,
17884
18699
  blameSha: args.blameSha,
17885
- blameSince: args.blameSince
18700
+ blameSince: args.blameSince,
18701
+ blameUntil: args.blameUntil
17886
18702
  }, (results) => {
17887
18703
  const text = results.length === 0 ? "No matching code found. Try a different query or run index_codebase first." : `Found ${results.length} locations for "${args.query}":
17888
18704
 
@@ -17897,6 +18713,7 @@ ${formatCodebasePeek(results)}`;
17897
18713
  {
17898
18714
  force: allowNullAsUndefined(z2.boolean().optional().default(false)).describe("Force reindex even if already indexed"),
17899
18715
  estimateOnly: allowNullAsUndefined(z2.boolean().optional().default(false)).describe("Only show cost estimate without indexing"),
18716
+ dryRun: allowNullAsUndefined(z2.boolean().optional().default(false)).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."),
17900
18717
  verbose: allowNullAsUndefined(z2.boolean().optional().default(false)).describe("Show detailed info about skipped files and parsing failures")
17901
18718
  },
17902
18719
  async (args) => {
@@ -17959,7 +18776,9 @@ ${formatCodebasePeek(results)}`;
17959
18776
  fileType: allowNullAsUndefined(z2.string().optional()).describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
17960
18777
  directory: allowNullAsUndefined(z2.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
17961
18778
  chunkType: allowNullAsUndefined(z2.enum(CHUNK_TYPES).optional()).describe("Filter by code chunk type"),
17962
- excludeFile: allowNullAsUndefined(z2.string().optional()).describe("Exclude results from this file path")
18779
+ excludeFile: allowNullAsUndefined(z2.string().optional()).describe("Exclude results from this file path"),
18780
+ blameSince: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or after this date"),
18781
+ blameUntil: allowNullAsUndefined(z2.string().optional()).describe("Filter to chunks last changed on or before this date")
17963
18782
  },
17964
18783
  async (args) => {
17965
18784
  const results = await findSimilarCode(runtime.projectRoot, runtime.host, args.code, {
@@ -17967,7 +18786,9 @@ ${formatCodebasePeek(results)}`;
17967
18786
  fileType: args.fileType,
17968
18787
  directory: args.directory,
17969
18788
  chunkType: args.chunkType,
17970
- excludeFile: args.excludeFile
18789
+ excludeFile: args.excludeFile,
18790
+ blameSince: args.blameSince,
18791
+ blameUntil: args.blameUntil
17971
18792
  });
17972
18793
  if (results.length === 0) {
17973
18794
  return { content: [{ type: "text", text: "No similar code found. Try a different snippet or run index_codebase first." }] };
@@ -18071,12 +18892,43 @@ ${formatSearchResults(results)}` }] };
18071
18892
  return { content: [{ type: "text", text: result.text }] };
18072
18893
  }
18073
18894
  );
18895
+ server.tool(
18896
+ TOOL_NAME.ADD_KNOWLEDGE_BASE,
18897
+ "Add a folder as a knowledge base to the semantic search index. The folder is indexed alongside the project code on the next index run. Provide an absolute path or a path relative to the project root. The path is written to the project-local host config of this MCP server (under the server project root), not to a user-global config, and the index is refreshed. Git blame metadata is collected only for files in the project git repo; knowledge-base files outside the repo remain searchable by content but have no blame. A knowledge base that appears in list_knowledge_bases but was inherited from a global config cannot be removed by this tool.",
18898
+ {
18899
+ path: z2.string().describe("Path to the folder to add as a knowledge base (absolute or relative to the project root)")
18900
+ },
18901
+ async (args) => {
18902
+ const result = addKnowledgeBase(runtime.projectRoot, runtime.host, args.path);
18903
+ return knowledgeBaseResult(result);
18904
+ }
18905
+ );
18906
+ server.tool(
18907
+ TOOL_NAME.LIST_KNOWLEDGE_BASES,
18908
+ "List the configured knowledge base folders that the index includes alongside the project code. The list is the union of project-local and user-global knowledge bases; each entry shows the resolved path and whether it exists.",
18909
+ {},
18910
+ async () => {
18911
+ const result = listKnowledgeBases(runtime.projectRoot, runtime.host);
18912
+ return knowledgeBaseResult(result);
18913
+ }
18914
+ );
18915
+ server.tool(
18916
+ TOOL_NAME.REMOVE_KNOWLEDGE_BASE,
18917
+ "Remove a knowledge base folder from the semantic search index and refresh the index. The path must match a project-local configured path exactly. Knowledge bases inherited from a user-global config are not removable by this tool.",
18918
+ {
18919
+ path: z2.string().describe("Path of the knowledge base to remove (must match a project-local configured path exactly)")
18920
+ },
18921
+ async (args) => {
18922
+ const result = removeKnowledgeBase(runtime.projectRoot, runtime.host, args.path.trim());
18923
+ return knowledgeBaseResult(result);
18924
+ }
18925
+ );
18074
18926
  }
18075
18927
 
18076
18928
  // src/adapters/mcp/server.ts
18077
18929
  function getServerInstructions(host) {
18078
18930
  const hostText = `host ${host}`;
18079
- return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns a token-budgeted location pack and routes to definitions or call-graph helpers when symbol intent is present. Keep the default tokenBudget for normal discovery, then use implementation_lookup, codebase_search, or a targeted file read only for selected locations that need source content. Use codebase_peek for direct conceptual location lookup. For exact identifiers or exhaustive matches, use grep. After identifying symbols, use call_graph or call_graph_path to trace dependencies. If the index is unavailable, run index_codebase, then retry the retrieval tool.`;
18931
+ return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns a token-budgeted location pack and routes to definitions or call-graph helpers when symbol intent is present. For code changes with a known or suspected symbol target, optionally call codebase_edit_context as a compact pre-edit step for bounded source plus direct callers and callees before broad file reads. Keep the default tokenBudget for normal discovery, then use implementation_lookup, codebase_search, or a targeted file read only for selected locations that need source content. Use codebase_peek for direct conceptual location lookup. For exact identifiers or exhaustive matches, use grep. After identifying symbols, use call_graph or call_graph_path to trace dependencies. If the index is unavailable, run index_codebase, then retry the retrieval tool.`;
18080
18932
  }
18081
18933
  function createMcpServer(projectRoot, config, host) {
18082
18934
  const server = new McpServer({
@@ -21494,6 +22346,7 @@ function parseIndexArgs(argv, cwd) {
21494
22346
  let config;
21495
22347
  let force = false;
21496
22348
  let estimateOnly = false;
22349
+ let dryRun = false;
21497
22350
  let verbose = false;
21498
22351
  for (let i = 0; i < argv.length; i += 1) {
21499
22352
  const arg = argv[i];
@@ -21531,13 +22384,16 @@ function parseIndexArgs(argv, cwd) {
21531
22384
  host = parseHostMode(value);
21532
22385
  continue;
21533
22386
  }
21534
- if (arg === "--force" || arg === "--estimate-only" || arg === "--verbose") {
22387
+ if (arg === "--force" || arg === "--estimate-only" || arg === "--dry-run" || arg === "--verbose") {
21535
22388
  if (arg === "--force") {
21536
22389
  force = true;
21537
22390
  }
21538
22391
  if (arg === "--estimate-only") {
21539
22392
  estimateOnly = true;
21540
22393
  }
22394
+ if (arg === "--dry-run") {
22395
+ dryRun = true;
22396
+ }
21541
22397
  if (arg === "--verbose") {
21542
22398
  verbose = true;
21543
22399
  }
@@ -21548,7 +22404,7 @@ function parseIndexArgs(argv, cwd) {
21548
22404
  }
21549
22405
  throw new Error(`Unknown index option: ${arg}`);
21550
22406
  }
21551
- return { project, host, config, force, estimateOnly, verbose };
22407
+ return { project, host, config, force, estimateOnly, dryRun, verbose };
21552
22408
  }
21553
22409
  function loadCliRawConfig(args) {
21554
22410
  return args.config ? loadConfigFile(args.config) : loadMergedConfig(args.project, args.host);
@@ -21565,6 +22421,7 @@ Options:
21565
22421
  --config <path> Explicit JSON config path
21566
22422
  --force Rebuild index even if already up to date
21567
22423
  --estimate-only Estimate indexing cost only
22424
+ --dry-run Parse only; report the exact embedding token total without indexing
21568
22425
  --verbose Include detailed final index statistics
21569
22426
  --help Show this message
21570
22427
 
@@ -21760,6 +22617,7 @@ async function handleIndexCommand(argv, cwd, deps = {}) {
21760
22617
  const indexArgs = {
21761
22618
  force: parsedArgs.force,
21762
22619
  estimateOnly: parsedArgs.estimateOnly,
22620
+ dryRun: parsedArgs.dryRun,
21763
22621
  verbose: parsedArgs.verbose
21764
22622
  };
21765
22623
  const result = await runIndex(parsedArgs.project, parsedArgs.host, indexArgs, (title, metadata) => {