opencode-codebase-index 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -2
- package/dist/cli.cjs +5016 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.js +5021 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +814 -357
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +816 -358
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +23 -4
package/dist/index.cjs
CHANGED
|
@@ -662,7 +662,7 @@ var path8 = __toESM(require("path"), 1);
|
|
|
662
662
|
var os3 = __toESM(require("os"), 1);
|
|
663
663
|
var import_url2 = require("url");
|
|
664
664
|
|
|
665
|
-
// src/config/
|
|
665
|
+
// src/config/constants.ts
|
|
666
666
|
var DEFAULT_INCLUDE = [
|
|
667
667
|
"**/*.{ts,tsx,js,jsx,mjs,cjs}",
|
|
668
668
|
"**/*.{py,pyi}",
|
|
@@ -690,6 +690,80 @@ var DEFAULT_EXCLUDE = [
|
|
|
690
690
|
"**/.nuxt/**",
|
|
691
691
|
"**/.opencode/**"
|
|
692
692
|
];
|
|
693
|
+
var EMBEDDING_MODELS = {
|
|
694
|
+
"google": {
|
|
695
|
+
// `text-embedding-004` is DEPRECATED - https://ai.google.dev/gemini-api/docs/deprecations
|
|
696
|
+
"text-embedding-005": {
|
|
697
|
+
provider: "google",
|
|
698
|
+
model: "text-embedding-005",
|
|
699
|
+
dimensions: 768,
|
|
700
|
+
maxTokens: 2048,
|
|
701
|
+
costPer1MTokens: 0.025,
|
|
702
|
+
taskAble: false
|
|
703
|
+
// Note: on reality, this model allows for task-specific embeddings. See: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/embeddings/task-types
|
|
704
|
+
},
|
|
705
|
+
"gemini-embedding-001": {
|
|
706
|
+
provider: "google",
|
|
707
|
+
model: "gemini-embedding-001",
|
|
708
|
+
// Native output is 3072D, but we use Matryoshka truncation via outputDimensionality
|
|
709
|
+
// to reduce to 1536D for better storage/search efficiency with minimal quality loss.
|
|
710
|
+
// Google recommends 768, 1536, or 3072. See: https://ai.google.dev/gemini-api/docs/embeddings
|
|
711
|
+
dimensions: 1536,
|
|
712
|
+
maxTokens: 2048,
|
|
713
|
+
costPer1MTokens: 0.15,
|
|
714
|
+
taskAble: true
|
|
715
|
+
}
|
|
716
|
+
},
|
|
717
|
+
"openai": {
|
|
718
|
+
"text-embedding-3-small": {
|
|
719
|
+
provider: "openai",
|
|
720
|
+
model: "text-embedding-3-small",
|
|
721
|
+
dimensions: 1536,
|
|
722
|
+
maxTokens: 8191,
|
|
723
|
+
costPer1MTokens: 0.02
|
|
724
|
+
},
|
|
725
|
+
"text-embedding-3-large": {
|
|
726
|
+
provider: "openai",
|
|
727
|
+
model: "text-embedding-3-large",
|
|
728
|
+
dimensions: 3072,
|
|
729
|
+
maxTokens: 8191,
|
|
730
|
+
costPer1MTokens: 0.13
|
|
731
|
+
}
|
|
732
|
+
},
|
|
733
|
+
"ollama": {
|
|
734
|
+
"nomic-embed-text": {
|
|
735
|
+
provider: "ollama",
|
|
736
|
+
model: "nomic-embed-text",
|
|
737
|
+
dimensions: 768,
|
|
738
|
+
maxTokens: 8192,
|
|
739
|
+
costPer1MTokens: 0
|
|
740
|
+
},
|
|
741
|
+
"mxbai-embed-large": {
|
|
742
|
+
provider: "ollama",
|
|
743
|
+
model: "mxbai-embed-large",
|
|
744
|
+
dimensions: 1024,
|
|
745
|
+
maxTokens: 512,
|
|
746
|
+
costPer1MTokens: 0
|
|
747
|
+
}
|
|
748
|
+
},
|
|
749
|
+
"github-copilot": {
|
|
750
|
+
"text-embedding-3-small": {
|
|
751
|
+
provider: "github-copilot",
|
|
752
|
+
model: "text-embedding-3-small",
|
|
753
|
+
dimensions: 1536,
|
|
754
|
+
maxTokens: 8191,
|
|
755
|
+
costPer1MTokens: 0
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
};
|
|
759
|
+
var DEFAULT_PROVIDER_MODELS = {
|
|
760
|
+
"github-copilot": "text-embedding-3-small",
|
|
761
|
+
"openai": "text-embedding-3-small",
|
|
762
|
+
"google": "text-embedding-005",
|
|
763
|
+
"ollama": "nomic-embed-text"
|
|
764
|
+
};
|
|
765
|
+
|
|
766
|
+
// src/config/schema.ts
|
|
693
767
|
function getDefaultIndexingConfig() {
|
|
694
768
|
return {
|
|
695
769
|
autoIndex: false,
|
|
@@ -726,11 +800,13 @@ function getDefaultDebugConfig() {
|
|
|
726
800
|
metrics: true
|
|
727
801
|
};
|
|
728
802
|
}
|
|
729
|
-
var VALID_PROVIDERS = ["auto", "github-copilot", "openai", "google", "ollama"];
|
|
730
803
|
var VALID_SCOPES = ["project", "global"];
|
|
731
804
|
var VALID_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
732
805
|
function isValidProvider(value) {
|
|
733
|
-
return typeof value === "string" &&
|
|
806
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS).includes(value);
|
|
807
|
+
}
|
|
808
|
+
function isValidModel(value, provider) {
|
|
809
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS[provider]).includes(value);
|
|
734
810
|
}
|
|
735
811
|
function isValidScope(value) {
|
|
736
812
|
return typeof value === "string" && VALID_SCOPES.includes(value);
|
|
@@ -779,9 +855,45 @@ function parseConfig(raw) {
|
|
|
779
855
|
logBranch: typeof rawDebug.logBranch === "boolean" ? rawDebug.logBranch : defaultDebug.logBranch,
|
|
780
856
|
metrics: typeof rawDebug.metrics === "boolean" ? rawDebug.metrics : defaultDebug.metrics
|
|
781
857
|
};
|
|
858
|
+
let embeddingProvider;
|
|
859
|
+
let embeddingModel = void 0;
|
|
860
|
+
let customProvider = void 0;
|
|
861
|
+
if (input.embeddingProvider === "custom") {
|
|
862
|
+
embeddingProvider = "custom";
|
|
863
|
+
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
864
|
+
if (rawCustom && typeof rawCustom.baseUrl === "string" && rawCustom.baseUrl.trim().length > 0 && typeof rawCustom.model === "string" && rawCustom.model.trim().length > 0 && typeof rawCustom.dimensions === "number" && Number.isInteger(rawCustom.dimensions) && rawCustom.dimensions > 0) {
|
|
865
|
+
customProvider = {
|
|
866
|
+
baseUrl: rawCustom.baseUrl.trim().replace(/\/+$/, ""),
|
|
867
|
+
model: rawCustom.model,
|
|
868
|
+
dimensions: rawCustom.dimensions,
|
|
869
|
+
apiKey: typeof rawCustom.apiKey === "string" ? rawCustom.apiKey : void 0,
|
|
870
|
+
maxTokens: typeof rawCustom.maxTokens === "number" ? rawCustom.maxTokens : void 0,
|
|
871
|
+
timeoutMs: typeof rawCustom.timeoutMs === "number" ? Math.max(1e3, rawCustom.timeoutMs) : void 0,
|
|
872
|
+
concurrency: typeof rawCustom.concurrency === "number" ? Math.max(1, Math.floor(rawCustom.concurrency)) : void 0,
|
|
873
|
+
requestIntervalMs: typeof rawCustom.requestIntervalMs === "number" ? Math.max(0, Math.floor(rawCustom.requestIntervalMs)) : void 0
|
|
874
|
+
};
|
|
875
|
+
if (!/\/v\d+\/?$/.test(customProvider.baseUrl)) {
|
|
876
|
+
console.warn(
|
|
877
|
+
`[codebase-index] Warning: customProvider.baseUrl ("${customProvider.baseUrl}") does not end with an API version path like /v1. The plugin appends /embeddings automatically, so the full URL will be "${customProvider.baseUrl}/embeddings". If your provider expects /v1/embeddings, set baseUrl to "${customProvider.baseUrl}/v1".`
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
} else {
|
|
881
|
+
throw new Error(
|
|
882
|
+
"embeddingProvider is 'custom' but customProvider config is missing or invalid. Required fields: baseUrl (string), model (string), dimensions (positive integer)."
|
|
883
|
+
);
|
|
884
|
+
}
|
|
885
|
+
} else if (isValidProvider(input.embeddingProvider)) {
|
|
886
|
+
embeddingProvider = input.embeddingProvider;
|
|
887
|
+
if (input.embeddingModel) {
|
|
888
|
+
embeddingModel = isValidModel(input.embeddingModel, embeddingProvider) ? input.embeddingModel : DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
889
|
+
}
|
|
890
|
+
} else {
|
|
891
|
+
embeddingProvider = "auto";
|
|
892
|
+
}
|
|
782
893
|
return {
|
|
783
|
-
embeddingProvider
|
|
784
|
-
embeddingModel
|
|
894
|
+
embeddingProvider,
|
|
895
|
+
embeddingModel,
|
|
896
|
+
customProvider,
|
|
785
897
|
scope: isValidScope(input.scope) ? input.scope : "project",
|
|
786
898
|
include: isStringArray(input.include) ? input.include : DEFAULT_INCLUDE,
|
|
787
899
|
exclude: isStringArray(input.exclude) ? input.exclude : DEFAULT_EXCLUDE,
|
|
@@ -790,64 +902,12 @@ function parseConfig(raw) {
|
|
|
790
902
|
debug
|
|
791
903
|
};
|
|
792
904
|
}
|
|
793
|
-
var EMBEDDING_MODELS = {
|
|
794
|
-
"github-copilot/text-embedding-3-small": {
|
|
795
|
-
provider: "github-copilot",
|
|
796
|
-
model: "text-embedding-3-small",
|
|
797
|
-
dimensions: 1536,
|
|
798
|
-
maxTokens: 8191,
|
|
799
|
-
costPer1MTokens: 0
|
|
800
|
-
},
|
|
801
|
-
"openai/text-embedding-3-small": {
|
|
802
|
-
provider: "openai",
|
|
803
|
-
model: "text-embedding-3-small",
|
|
804
|
-
dimensions: 1536,
|
|
805
|
-
maxTokens: 8191,
|
|
806
|
-
costPer1MTokens: 0.02
|
|
807
|
-
},
|
|
808
|
-
"openai/text-embedding-3-large": {
|
|
809
|
-
provider: "openai",
|
|
810
|
-
model: "text-embedding-3-large",
|
|
811
|
-
dimensions: 3072,
|
|
812
|
-
maxTokens: 8191,
|
|
813
|
-
costPer1MTokens: 0.13
|
|
814
|
-
},
|
|
815
|
-
"google/text-embedding-004": {
|
|
816
|
-
provider: "google",
|
|
817
|
-
model: "text-embedding-004",
|
|
818
|
-
dimensions: 768,
|
|
819
|
-
maxTokens: 2048,
|
|
820
|
-
costPer1MTokens: 0
|
|
821
|
-
},
|
|
822
|
-
"ollama/nomic-embed-text": {
|
|
823
|
-
provider: "ollama",
|
|
824
|
-
model: "nomic-embed-text",
|
|
825
|
-
dimensions: 768,
|
|
826
|
-
maxTokens: 8192,
|
|
827
|
-
costPer1MTokens: 0
|
|
828
|
-
},
|
|
829
|
-
"ollama/mxbai-embed-large": {
|
|
830
|
-
provider: "ollama",
|
|
831
|
-
model: "mxbai-embed-large",
|
|
832
|
-
dimensions: 1024,
|
|
833
|
-
maxTokens: 512,
|
|
834
|
-
costPer1MTokens: 0
|
|
835
|
-
}
|
|
836
|
-
};
|
|
837
905
|
function getDefaultModelForProvider(provider) {
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
case "openai":
|
|
842
|
-
return EMBEDDING_MODELS["openai/text-embedding-3-small"];
|
|
843
|
-
case "google":
|
|
844
|
-
return EMBEDDING_MODELS["google/text-embedding-004"];
|
|
845
|
-
case "ollama":
|
|
846
|
-
return EMBEDDING_MODELS["ollama/nomic-embed-text"];
|
|
847
|
-
default:
|
|
848
|
-
return EMBEDDING_MODELS["github-copilot/text-embedding-3-small"];
|
|
849
|
-
}
|
|
906
|
+
const models = EMBEDDING_MODELS[provider];
|
|
907
|
+
const providerDefault = DEFAULT_PROVIDER_MODELS[provider];
|
|
908
|
+
return models[providerDefault];
|
|
850
909
|
}
|
|
910
|
+
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
851
911
|
|
|
852
912
|
// src/indexer/index.ts
|
|
853
913
|
var import_fs4 = require("fs");
|
|
@@ -876,7 +936,7 @@ function pTimeout(promise, options) {
|
|
|
876
936
|
} = options;
|
|
877
937
|
let timer;
|
|
878
938
|
let abortHandler;
|
|
879
|
-
const wrappedPromise = new Promise((
|
|
939
|
+
const wrappedPromise = new Promise((resolve5, reject) => {
|
|
880
940
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
881
941
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
882
942
|
}
|
|
@@ -890,7 +950,7 @@ function pTimeout(promise, options) {
|
|
|
890
950
|
};
|
|
891
951
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
892
952
|
}
|
|
893
|
-
promise.then(
|
|
953
|
+
promise.then(resolve5, reject);
|
|
894
954
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
895
955
|
return;
|
|
896
956
|
}
|
|
@@ -898,7 +958,7 @@ function pTimeout(promise, options) {
|
|
|
898
958
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
899
959
|
if (fallback) {
|
|
900
960
|
try {
|
|
901
|
-
|
|
961
|
+
resolve5(fallback());
|
|
902
962
|
} catch (error) {
|
|
903
963
|
reject(error);
|
|
904
964
|
}
|
|
@@ -908,7 +968,7 @@ function pTimeout(promise, options) {
|
|
|
908
968
|
promise.cancel();
|
|
909
969
|
}
|
|
910
970
|
if (message === false) {
|
|
911
|
-
|
|
971
|
+
resolve5();
|
|
912
972
|
} else if (message instanceof Error) {
|
|
913
973
|
reject(message);
|
|
914
974
|
} else {
|
|
@@ -1298,7 +1358,7 @@ var PQueue = class extends import_index.default {
|
|
|
1298
1358
|
// Assign unique ID if not provided
|
|
1299
1359
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1300
1360
|
};
|
|
1301
|
-
return new Promise((
|
|
1361
|
+
return new Promise((resolve5, reject) => {
|
|
1302
1362
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1303
1363
|
this.#queue.enqueue(async () => {
|
|
1304
1364
|
this.#pending++;
|
|
@@ -1336,7 +1396,7 @@ var PQueue = class extends import_index.default {
|
|
|
1336
1396
|
})]);
|
|
1337
1397
|
}
|
|
1338
1398
|
const result = await operation;
|
|
1339
|
-
|
|
1399
|
+
resolve5(result);
|
|
1340
1400
|
this.emit("completed", result);
|
|
1341
1401
|
} catch (error) {
|
|
1342
1402
|
reject(error);
|
|
@@ -1493,13 +1553,13 @@ var PQueue = class extends import_index.default {
|
|
|
1493
1553
|
});
|
|
1494
1554
|
}
|
|
1495
1555
|
async #onEvent(event, filter) {
|
|
1496
|
-
return new Promise((
|
|
1556
|
+
return new Promise((resolve5) => {
|
|
1497
1557
|
const listener = () => {
|
|
1498
1558
|
if (filter && !filter()) {
|
|
1499
1559
|
return;
|
|
1500
1560
|
}
|
|
1501
1561
|
this.off(event, listener);
|
|
1502
|
-
|
|
1562
|
+
resolve5();
|
|
1503
1563
|
};
|
|
1504
1564
|
this.on(event, listener);
|
|
1505
1565
|
});
|
|
@@ -1784,7 +1844,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1784
1844
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
1785
1845
|
options.signal?.throwIfAborted();
|
|
1786
1846
|
if (finalDelay > 0) {
|
|
1787
|
-
await new Promise((
|
|
1847
|
+
await new Promise((resolve5, reject) => {
|
|
1788
1848
|
const onAbort = () => {
|
|
1789
1849
|
clearTimeout(timeoutToken);
|
|
1790
1850
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -1792,7 +1852,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1792
1852
|
};
|
|
1793
1853
|
const timeoutToken = setTimeout(() => {
|
|
1794
1854
|
options.signal?.removeEventListener("abort", onAbort);
|
|
1795
|
-
|
|
1855
|
+
resolve5();
|
|
1796
1856
|
}, finalDelay);
|
|
1797
1857
|
if (options.unref) {
|
|
1798
1858
|
timeoutToken.unref?.();
|
|
@@ -1856,12 +1916,6 @@ async function pRetry(input, options = {}) {
|
|
|
1856
1916
|
var import_fs = require("fs");
|
|
1857
1917
|
var path = __toESM(require("path"), 1);
|
|
1858
1918
|
var os = __toESM(require("os"), 1);
|
|
1859
|
-
var EMBEDDING_CAPABLE_PROVIDERS = [
|
|
1860
|
-
"github-copilot",
|
|
1861
|
-
"openai",
|
|
1862
|
-
"google",
|
|
1863
|
-
"ollama"
|
|
1864
|
-
];
|
|
1865
1919
|
function getOpenCodeAuthPath() {
|
|
1866
1920
|
return path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
|
|
1867
1921
|
}
|
|
@@ -1875,21 +1929,34 @@ function loadOpenCodeAuth() {
|
|
|
1875
1929
|
}
|
|
1876
1930
|
return {};
|
|
1877
1931
|
}
|
|
1878
|
-
async function detectEmbeddingProvider(preferredProvider) {
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
if (
|
|
1932
|
+
async function detectEmbeddingProvider(preferredProvider, model) {
|
|
1933
|
+
const credentials = await getProviderCredentials(preferredProvider);
|
|
1934
|
+
if (credentials) {
|
|
1935
|
+
if (!model) {
|
|
1882
1936
|
return {
|
|
1883
1937
|
provider: preferredProvider,
|
|
1884
1938
|
credentials,
|
|
1885
1939
|
modelInfo: getDefaultModelForProvider(preferredProvider)
|
|
1886
1940
|
};
|
|
1887
1941
|
}
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1942
|
+
if (!isValidModel(model, preferredProvider)) {
|
|
1943
|
+
throw new Error(
|
|
1944
|
+
`Model '${model}' is not supported by provider '${preferredProvider}'`
|
|
1945
|
+
);
|
|
1946
|
+
}
|
|
1947
|
+
const providerModels = EMBEDDING_MODELS[preferredProvider];
|
|
1948
|
+
return {
|
|
1949
|
+
provider: preferredProvider,
|
|
1950
|
+
credentials,
|
|
1951
|
+
modelInfo: providerModels[model]
|
|
1952
|
+
};
|
|
1891
1953
|
}
|
|
1892
|
-
|
|
1954
|
+
throw new Error(
|
|
1955
|
+
`Preferred provider '${preferredProvider}' is not configured or authenticated`
|
|
1956
|
+
);
|
|
1957
|
+
}
|
|
1958
|
+
async function tryDetectProvider() {
|
|
1959
|
+
for (const provider of availableProviders) {
|
|
1893
1960
|
const credentials = await getProviderCredentials(provider);
|
|
1894
1961
|
if (credentials) {
|
|
1895
1962
|
return {
|
|
@@ -1900,7 +1967,7 @@ async function detectEmbeddingProvider(preferredProvider) {
|
|
|
1900
1967
|
}
|
|
1901
1968
|
}
|
|
1902
1969
|
throw new Error(
|
|
1903
|
-
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${
|
|
1970
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${availableProviders.join(", ")}.`
|
|
1904
1971
|
);
|
|
1905
1972
|
}
|
|
1906
1973
|
async function getProviderCredentials(provider) {
|
|
@@ -1992,24 +2059,55 @@ function getProviderDisplayName(provider) {
|
|
|
1992
2059
|
return "Google (Gemini)";
|
|
1993
2060
|
case "ollama":
|
|
1994
2061
|
return "Ollama (Local)";
|
|
2062
|
+
case "custom":
|
|
2063
|
+
return "Custom (OpenAI-compatible)";
|
|
1995
2064
|
default:
|
|
1996
2065
|
return provider;
|
|
1997
2066
|
}
|
|
1998
2067
|
}
|
|
2068
|
+
function createCustomProviderInfo(config) {
|
|
2069
|
+
const baseUrl = config.baseUrl.replace(/\/+$/, "");
|
|
2070
|
+
return {
|
|
2071
|
+
provider: "custom",
|
|
2072
|
+
credentials: {
|
|
2073
|
+
provider: "custom",
|
|
2074
|
+
baseUrl,
|
|
2075
|
+
apiKey: config.apiKey
|
|
2076
|
+
},
|
|
2077
|
+
modelInfo: {
|
|
2078
|
+
provider: "custom",
|
|
2079
|
+
model: config.model,
|
|
2080
|
+
dimensions: config.dimensions,
|
|
2081
|
+
maxTokens: config.maxTokens ?? 8192,
|
|
2082
|
+
costPer1MTokens: 0,
|
|
2083
|
+
timeoutMs: config.timeoutMs ?? 3e4
|
|
2084
|
+
}
|
|
2085
|
+
};
|
|
2086
|
+
}
|
|
1999
2087
|
|
|
2000
2088
|
// src/embeddings/provider.ts
|
|
2001
|
-
|
|
2002
|
-
|
|
2089
|
+
var CustomProviderNonRetryableError = class extends Error {
|
|
2090
|
+
constructor(message) {
|
|
2091
|
+
super(message);
|
|
2092
|
+
this.name = "CustomProviderNonRetryableError";
|
|
2093
|
+
}
|
|
2094
|
+
};
|
|
2095
|
+
function createEmbeddingProvider(configuredProviderInfo) {
|
|
2096
|
+
switch (configuredProviderInfo.provider) {
|
|
2003
2097
|
case "github-copilot":
|
|
2004
|
-
return new GitHubCopilotEmbeddingProvider(credentials, modelInfo);
|
|
2098
|
+
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2005
2099
|
case "openai":
|
|
2006
|
-
return new OpenAIEmbeddingProvider(credentials, modelInfo);
|
|
2100
|
+
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2007
2101
|
case "google":
|
|
2008
|
-
return new GoogleEmbeddingProvider(credentials, modelInfo);
|
|
2102
|
+
return new GoogleEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2009
2103
|
case "ollama":
|
|
2010
|
-
return new OllamaEmbeddingProvider(credentials, modelInfo);
|
|
2011
|
-
|
|
2012
|
-
|
|
2104
|
+
return new OllamaEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2105
|
+
case "custom":
|
|
2106
|
+
return new CustomEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2107
|
+
default: {
|
|
2108
|
+
const _exhaustive = configuredProviderInfo;
|
|
2109
|
+
throw new Error(`Unsupported embedding provider: ${_exhaustive.provider}`);
|
|
2110
|
+
}
|
|
2013
2111
|
}
|
|
2014
2112
|
}
|
|
2015
2113
|
var GitHubCopilotEmbeddingProvider = class {
|
|
@@ -2023,8 +2121,15 @@ var GitHubCopilotEmbeddingProvider = class {
|
|
|
2023
2121
|
}
|
|
2024
2122
|
return this.credentials.refreshToken;
|
|
2025
2123
|
}
|
|
2026
|
-
async
|
|
2027
|
-
const result = await this.embedBatch([
|
|
2124
|
+
async embedQuery(query) {
|
|
2125
|
+
const result = await this.embedBatch([query]);
|
|
2126
|
+
return {
|
|
2127
|
+
embedding: result.embeddings[0],
|
|
2128
|
+
tokensUsed: result.totalTokensUsed
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
async embedDocument(document) {
|
|
2132
|
+
const result = await this.embedBatch([document]);
|
|
2028
2133
|
return {
|
|
2029
2134
|
embedding: result.embeddings[0],
|
|
2030
2135
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2064,8 +2169,15 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2064
2169
|
this.credentials = credentials;
|
|
2065
2170
|
this.modelInfo = modelInfo;
|
|
2066
2171
|
}
|
|
2067
|
-
async
|
|
2068
|
-
const result = await this.embedBatch([
|
|
2172
|
+
async embedQuery(query) {
|
|
2173
|
+
const result = await this.embedBatch([query]);
|
|
2174
|
+
return {
|
|
2175
|
+
embedding: result.embeddings[0],
|
|
2176
|
+
tokensUsed: result.totalTokensUsed
|
|
2177
|
+
};
|
|
2178
|
+
}
|
|
2179
|
+
async embedDocument(document) {
|
|
2180
|
+
const result = await this.embedBatch([document]);
|
|
2069
2181
|
return {
|
|
2070
2182
|
embedding: result.embeddings[0],
|
|
2071
2183
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2097,33 +2209,61 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2097
2209
|
return this.modelInfo;
|
|
2098
2210
|
}
|
|
2099
2211
|
};
|
|
2100
|
-
var GoogleEmbeddingProvider = class {
|
|
2212
|
+
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
2101
2213
|
constructor(credentials, modelInfo) {
|
|
2102
2214
|
this.credentials = credentials;
|
|
2103
2215
|
this.modelInfo = modelInfo;
|
|
2104
2216
|
}
|
|
2105
|
-
|
|
2106
|
-
|
|
2217
|
+
static BATCH_SIZE = 20;
|
|
2218
|
+
async embedQuery(query) {
|
|
2219
|
+
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2220
|
+
const result = await this.embedWithTaskType([query], taskType);
|
|
2221
|
+
return {
|
|
2222
|
+
embedding: result.embeddings[0],
|
|
2223
|
+
tokensUsed: result.totalTokensUsed
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
async embedDocument(document) {
|
|
2227
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2228
|
+
const result = await this.embedWithTaskType([document], taskType);
|
|
2107
2229
|
return {
|
|
2108
2230
|
embedding: result.embeddings[0],
|
|
2109
2231
|
tokensUsed: result.totalTokensUsed
|
|
2110
2232
|
};
|
|
2111
2233
|
}
|
|
2112
2234
|
async embedBatch(texts) {
|
|
2113
|
-
const
|
|
2114
|
-
|
|
2235
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2236
|
+
return this.embedWithTaskType(texts, taskType);
|
|
2237
|
+
}
|
|
2238
|
+
/**
|
|
2239
|
+
* Embeds texts using the Google embedContent API.
|
|
2240
|
+
* Sends multiple texts as parts in batched requests (up to BATCH_SIZE per call).
|
|
2241
|
+
* When taskType is provided (gemini-embedding-001), includes it in the request
|
|
2242
|
+
* for task-specific embedding optimization.
|
|
2243
|
+
*/
|
|
2244
|
+
async embedWithTaskType(texts, taskType) {
|
|
2245
|
+
const batches = [];
|
|
2246
|
+
for (let i = 0; i < texts.length; i += _GoogleEmbeddingProvider.BATCH_SIZE) {
|
|
2247
|
+
batches.push(texts.slice(i, i + _GoogleEmbeddingProvider.BATCH_SIZE));
|
|
2248
|
+
}
|
|
2249
|
+
const batchResults = await Promise.all(
|
|
2250
|
+
batches.map(async (batch) => {
|
|
2251
|
+
const requests = batch.map((text) => ({
|
|
2252
|
+
model: `models/${this.modelInfo.model}`,
|
|
2253
|
+
content: {
|
|
2254
|
+
parts: [{ text }]
|
|
2255
|
+
},
|
|
2256
|
+
taskType,
|
|
2257
|
+
outputDimensionality: this.modelInfo.dimensions
|
|
2258
|
+
}));
|
|
2115
2259
|
const response = await fetch(
|
|
2116
|
-
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:
|
|
2260
|
+
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:batchEmbedContents?key=${this.credentials.apiKey}`,
|
|
2117
2261
|
{
|
|
2118
2262
|
method: "POST",
|
|
2119
2263
|
headers: {
|
|
2120
2264
|
"Content-Type": "application/json"
|
|
2121
2265
|
},
|
|
2122
|
-
body: JSON.stringify({
|
|
2123
|
-
content: {
|
|
2124
|
-
parts: [{ text }]
|
|
2125
|
-
}
|
|
2126
|
-
})
|
|
2266
|
+
body: JSON.stringify({ requests })
|
|
2127
2267
|
}
|
|
2128
2268
|
);
|
|
2129
2269
|
if (!response.ok) {
|
|
@@ -2132,14 +2272,14 @@ var GoogleEmbeddingProvider = class {
|
|
|
2132
2272
|
}
|
|
2133
2273
|
const data = await response.json();
|
|
2134
2274
|
return {
|
|
2135
|
-
|
|
2136
|
-
tokensUsed: Math.ceil(text.length / 4)
|
|
2275
|
+
embeddings: data.embeddings.map((e) => e.values),
|
|
2276
|
+
tokensUsed: batch.reduce((sum, text) => sum + Math.ceil(text.length / 4), 0)
|
|
2137
2277
|
};
|
|
2138
2278
|
})
|
|
2139
2279
|
);
|
|
2140
2280
|
return {
|
|
2141
|
-
embeddings:
|
|
2142
|
-
totalTokensUsed:
|
|
2281
|
+
embeddings: batchResults.flatMap((r) => r.embeddings),
|
|
2282
|
+
totalTokensUsed: batchResults.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2143
2283
|
};
|
|
2144
2284
|
}
|
|
2145
2285
|
getModelInfo() {
|
|
@@ -2151,29 +2291,44 @@ var OllamaEmbeddingProvider = class {
|
|
|
2151
2291
|
this.credentials = credentials;
|
|
2152
2292
|
this.modelInfo = modelInfo;
|
|
2153
2293
|
}
|
|
2154
|
-
async
|
|
2155
|
-
const
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
})
|
|
2164
|
-
});
|
|
2165
|
-
if (!response.ok) {
|
|
2166
|
-
const error = await response.text();
|
|
2167
|
-
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2168
|
-
}
|
|
2169
|
-
const data = await response.json();
|
|
2294
|
+
async embedQuery(query) {
|
|
2295
|
+
const result = await this.embedBatch([query]);
|
|
2296
|
+
return {
|
|
2297
|
+
embedding: result.embeddings[0],
|
|
2298
|
+
tokensUsed: result.totalTokensUsed
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
async embedDocument(document) {
|
|
2302
|
+
const result = await this.embedBatch([document]);
|
|
2170
2303
|
return {
|
|
2171
|
-
embedding:
|
|
2172
|
-
tokensUsed:
|
|
2304
|
+
embedding: result.embeddings[0],
|
|
2305
|
+
tokensUsed: result.totalTokensUsed
|
|
2173
2306
|
};
|
|
2174
2307
|
}
|
|
2175
2308
|
async embedBatch(texts) {
|
|
2176
|
-
const results = await Promise.all(
|
|
2309
|
+
const results = await Promise.all(
|
|
2310
|
+
texts.map(async (text) => {
|
|
2311
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2312
|
+
method: "POST",
|
|
2313
|
+
headers: {
|
|
2314
|
+
"Content-Type": "application/json"
|
|
2315
|
+
},
|
|
2316
|
+
body: JSON.stringify({
|
|
2317
|
+
model: this.modelInfo.model,
|
|
2318
|
+
prompt: text
|
|
2319
|
+
})
|
|
2320
|
+
});
|
|
2321
|
+
if (!response.ok) {
|
|
2322
|
+
const error = await response.text();
|
|
2323
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2324
|
+
}
|
|
2325
|
+
const data = await response.json();
|
|
2326
|
+
return {
|
|
2327
|
+
embedding: data.embedding,
|
|
2328
|
+
tokensUsed: Math.ceil(text.length / 4)
|
|
2329
|
+
};
|
|
2330
|
+
})
|
|
2331
|
+
);
|
|
2177
2332
|
return {
|
|
2178
2333
|
embeddings: results.map((r) => r.embedding),
|
|
2179
2334
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
@@ -2183,6 +2338,90 @@ var OllamaEmbeddingProvider = class {
|
|
|
2183
2338
|
return this.modelInfo;
|
|
2184
2339
|
}
|
|
2185
2340
|
};
|
|
2341
|
+
var CustomEmbeddingProvider = class {
|
|
2342
|
+
constructor(credentials, modelInfo) {
|
|
2343
|
+
this.credentials = credentials;
|
|
2344
|
+
this.modelInfo = modelInfo;
|
|
2345
|
+
}
|
|
2346
|
+
async embedQuery(query) {
|
|
2347
|
+
const result = await this.embedBatch([query]);
|
|
2348
|
+
return {
|
|
2349
|
+
embedding: result.embeddings[0],
|
|
2350
|
+
tokensUsed: result.totalTokensUsed
|
|
2351
|
+
};
|
|
2352
|
+
}
|
|
2353
|
+
async embedDocument(document) {
|
|
2354
|
+
const result = await this.embedBatch([document]);
|
|
2355
|
+
return {
|
|
2356
|
+
embedding: result.embeddings[0],
|
|
2357
|
+
tokensUsed: result.totalTokensUsed
|
|
2358
|
+
};
|
|
2359
|
+
}
|
|
2360
|
+
async embedBatch(texts) {
|
|
2361
|
+
const headers = {
|
|
2362
|
+
"Content-Type": "application/json"
|
|
2363
|
+
};
|
|
2364
|
+
if (this.credentials.apiKey) {
|
|
2365
|
+
headers["Authorization"] = `Bearer ${this.credentials.apiKey}`;
|
|
2366
|
+
}
|
|
2367
|
+
const baseUrl = this.credentials.baseUrl ?? "";
|
|
2368
|
+
const timeoutMs = this.modelInfo.timeoutMs;
|
|
2369
|
+
const controller = new AbortController();
|
|
2370
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
2371
|
+
let response;
|
|
2372
|
+
try {
|
|
2373
|
+
response = await fetch(`${baseUrl}/embeddings`, {
|
|
2374
|
+
method: "POST",
|
|
2375
|
+
headers,
|
|
2376
|
+
body: JSON.stringify({
|
|
2377
|
+
model: this.modelInfo.model,
|
|
2378
|
+
input: texts
|
|
2379
|
+
}),
|
|
2380
|
+
signal: controller.signal
|
|
2381
|
+
});
|
|
2382
|
+
} catch (error) {
|
|
2383
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
2384
|
+
throw new Error(`Custom embedding API request timed out after ${timeoutMs}ms for ${baseUrl}/embeddings`);
|
|
2385
|
+
}
|
|
2386
|
+
throw error;
|
|
2387
|
+
} finally {
|
|
2388
|
+
clearTimeout(timeout);
|
|
2389
|
+
}
|
|
2390
|
+
if (!response.ok) {
|
|
2391
|
+
const errorText = await response.text();
|
|
2392
|
+
if (response.status >= 400 && response.status < 500 && response.status !== 429) {
|
|
2393
|
+
throw new CustomProviderNonRetryableError(`Custom embedding API error (non-retryable): ${response.status} - ${errorText}`);
|
|
2394
|
+
}
|
|
2395
|
+
throw new Error(`Custom embedding API error: ${response.status} - ${errorText}`);
|
|
2396
|
+
}
|
|
2397
|
+
const data = await response.json();
|
|
2398
|
+
if (data.data && Array.isArray(data.data)) {
|
|
2399
|
+
if (data.data.length > 0) {
|
|
2400
|
+
const actualDims = data.data[0].embedding.length;
|
|
2401
|
+
if (actualDims !== this.modelInfo.dimensions) {
|
|
2402
|
+
throw new Error(
|
|
2403
|
+
`Dimension mismatch: customProvider.dimensions is ${this.modelInfo.dimensions}, but the API returned vectors with ${actualDims} dimensions. Update your config to match the model's actual output dimensions.`
|
|
2404
|
+
);
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
if (data.data.length !== texts.length) {
|
|
2408
|
+
throw new Error(
|
|
2409
|
+
`Embedding count mismatch: sent ${texts.length} texts but received ${data.data.length} embeddings. The custom embedding server may not support batch input.`
|
|
2410
|
+
);
|
|
2411
|
+
}
|
|
2412
|
+
return {
|
|
2413
|
+
embeddings: data.data.map((d) => d.embedding),
|
|
2414
|
+
// Rough estimate: ~4 chars per token. Used as fallback when the server
|
|
2415
|
+
// doesn't return usage.total_tokens (e.g. llama.cpp, some vLLM configs).
|
|
2416
|
+
totalTokensUsed: data.usage?.total_tokens ?? texts.reduce((sum, t) => sum + Math.ceil(t.length / 4), 0)
|
|
2417
|
+
};
|
|
2418
|
+
}
|
|
2419
|
+
throw new Error("Custom embedding API returned unexpected response format. Expected OpenAI-compatible format with data[].embedding.");
|
|
2420
|
+
}
|
|
2421
|
+
getModelInfo() {
|
|
2422
|
+
return this.modelInfo;
|
|
2423
|
+
}
|
|
2424
|
+
};
|
|
2186
2425
|
|
|
2187
2426
|
// src/utils/files.ts
|
|
2188
2427
|
var import_ignore = __toESM(require_ignore(), 1);
|
|
@@ -3118,11 +3357,40 @@ var Database = class {
|
|
|
3118
3357
|
var import_fs3 = require("fs");
|
|
3119
3358
|
var path4 = __toESM(require("path"), 1);
|
|
3120
3359
|
var import_child_process = require("child_process");
|
|
3360
|
+
function resolveGitDir(repoRoot) {
|
|
3361
|
+
const gitPath = path4.join(repoRoot, ".git");
|
|
3362
|
+
if (!(0, import_fs3.existsSync)(gitPath)) {
|
|
3363
|
+
return null;
|
|
3364
|
+
}
|
|
3365
|
+
try {
|
|
3366
|
+
const stat4 = (0, import_fs3.statSync)(gitPath);
|
|
3367
|
+
if (stat4.isDirectory()) {
|
|
3368
|
+
return gitPath;
|
|
3369
|
+
}
|
|
3370
|
+
if (stat4.isFile()) {
|
|
3371
|
+
const content = (0, import_fs3.readFileSync)(gitPath, "utf-8").trim();
|
|
3372
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
3373
|
+
if (match) {
|
|
3374
|
+
const gitdir = match[1];
|
|
3375
|
+
const resolvedPath = path4.isAbsolute(gitdir) ? gitdir : path4.resolve(repoRoot, gitdir);
|
|
3376
|
+
if ((0, import_fs3.existsSync)(resolvedPath)) {
|
|
3377
|
+
return resolvedPath;
|
|
3378
|
+
}
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
} catch {
|
|
3382
|
+
}
|
|
3383
|
+
return null;
|
|
3384
|
+
}
|
|
3121
3385
|
function isGitRepo(dir) {
|
|
3122
|
-
return (
|
|
3386
|
+
return resolveGitDir(dir) !== null;
|
|
3123
3387
|
}
|
|
3124
3388
|
function getCurrentBranch(repoRoot) {
|
|
3125
|
-
const
|
|
3389
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3390
|
+
if (!gitDir) {
|
|
3391
|
+
return null;
|
|
3392
|
+
}
|
|
3393
|
+
const headPath = path4.join(gitDir, "HEAD");
|
|
3126
3394
|
if (!(0, import_fs3.existsSync)(headPath)) {
|
|
3127
3395
|
return null;
|
|
3128
3396
|
}
|
|
@@ -3141,20 +3409,23 @@ function getCurrentBranch(repoRoot) {
|
|
|
3141
3409
|
}
|
|
3142
3410
|
}
|
|
3143
3411
|
function getBaseBranch(repoRoot) {
|
|
3412
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3144
3413
|
const candidates = ["main", "master", "develop", "trunk"];
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3414
|
+
if (gitDir) {
|
|
3415
|
+
for (const candidate of candidates) {
|
|
3416
|
+
const refPath = path4.join(gitDir, "refs", "heads", candidate);
|
|
3417
|
+
if ((0, import_fs3.existsSync)(refPath)) {
|
|
3418
|
+
return candidate;
|
|
3419
|
+
}
|
|
3420
|
+
const packedRefsPath = path4.join(gitDir, "packed-refs");
|
|
3421
|
+
if ((0, import_fs3.existsSync)(packedRefsPath)) {
|
|
3422
|
+
try {
|
|
3423
|
+
const content = (0, import_fs3.readFileSync)(packedRefsPath, "utf-8");
|
|
3424
|
+
if (content.includes(`refs/heads/${candidate}`)) {
|
|
3425
|
+
return candidate;
|
|
3426
|
+
}
|
|
3427
|
+
} catch {
|
|
3156
3428
|
}
|
|
3157
|
-
} catch {
|
|
3158
3429
|
}
|
|
3159
3430
|
}
|
|
3160
3431
|
}
|
|
@@ -3179,6 +3450,10 @@ function getBranchOrDefault(repoRoot) {
|
|
|
3179
3450
|
return getCurrentBranch(repoRoot) ?? "default";
|
|
3180
3451
|
}
|
|
3181
3452
|
function getHeadPath(repoRoot) {
|
|
3453
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3454
|
+
if (gitDir) {
|
|
3455
|
+
return path4.join(gitDir, "HEAD");
|
|
3456
|
+
}
|
|
3182
3457
|
return path4.join(repoRoot, ".git", "HEAD");
|
|
3183
3458
|
}
|
|
3184
3459
|
|
|
@@ -3206,6 +3481,7 @@ function isRateLimitError(error) {
|
|
|
3206
3481
|
const message = getErrorMessage(error);
|
|
3207
3482
|
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
3208
3483
|
}
|
|
3484
|
+
var INDEX_METADATA_VERSION = "1";
|
|
3209
3485
|
var Indexer = class {
|
|
3210
3486
|
config;
|
|
3211
3487
|
projectRoot;
|
|
@@ -3214,7 +3490,7 @@ var Indexer = class {
|
|
|
3214
3490
|
invertedIndex = null;
|
|
3215
3491
|
database = null;
|
|
3216
3492
|
provider = null;
|
|
3217
|
-
|
|
3493
|
+
configuredProviderInfo = null;
|
|
3218
3494
|
fileHashCache = /* @__PURE__ */ new Map();
|
|
3219
3495
|
fileHashCachePath = "";
|
|
3220
3496
|
failedBatchesPath = "";
|
|
@@ -3225,12 +3501,15 @@ var Indexer = class {
|
|
|
3225
3501
|
maxQueryCacheSize = 100;
|
|
3226
3502
|
queryCacheTtlMs = 5 * 60 * 1e3;
|
|
3227
3503
|
querySimilarityThreshold = 0.85;
|
|
3504
|
+
indexCompatibility = null;
|
|
3505
|
+
indexingLockPath = "";
|
|
3228
3506
|
constructor(projectRoot, config) {
|
|
3229
3507
|
this.projectRoot = projectRoot;
|
|
3230
3508
|
this.config = config;
|
|
3231
3509
|
this.indexPath = this.getIndexPath();
|
|
3232
3510
|
this.fileHashCachePath = path5.join(this.indexPath, "file-hashes.json");
|
|
3233
3511
|
this.failedBatchesPath = path5.join(this.indexPath, "failed-batches.json");
|
|
3512
|
+
this.indexingLockPath = path5.join(this.indexPath, "indexing.lock");
|
|
3234
3513
|
this.logger = initializeLogger(config.debug);
|
|
3235
3514
|
}
|
|
3236
3515
|
getIndexPath() {
|
|
@@ -3256,7 +3535,36 @@ var Indexer = class {
|
|
|
3256
3535
|
for (const [k, v] of this.fileHashCache) {
|
|
3257
3536
|
obj[k] = v;
|
|
3258
3537
|
}
|
|
3259
|
-
|
|
3538
|
+
this.atomicWriteSync(this.fileHashCachePath, JSON.stringify(obj));
|
|
3539
|
+
}
|
|
3540
|
+
atomicWriteSync(targetPath, data) {
|
|
3541
|
+
const tempPath = `${targetPath}.tmp`;
|
|
3542
|
+
(0, import_fs4.writeFileSync)(tempPath, data);
|
|
3543
|
+
(0, import_fs4.renameSync)(tempPath, targetPath);
|
|
3544
|
+
}
|
|
3545
|
+
checkForInterruptedIndexing() {
|
|
3546
|
+
return (0, import_fs4.existsSync)(this.indexingLockPath);
|
|
3547
|
+
}
|
|
3548
|
+
acquireIndexingLock() {
|
|
3549
|
+
const lockData = {
|
|
3550
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3551
|
+
pid: process.pid
|
|
3552
|
+
};
|
|
3553
|
+
(0, import_fs4.writeFileSync)(this.indexingLockPath, JSON.stringify(lockData));
|
|
3554
|
+
}
|
|
3555
|
+
releaseIndexingLock() {
|
|
3556
|
+
if ((0, import_fs4.existsSync)(this.indexingLockPath)) {
|
|
3557
|
+
(0, import_fs4.unlinkSync)(this.indexingLockPath);
|
|
3558
|
+
}
|
|
3559
|
+
}
|
|
3560
|
+
async recoverFromInterruptedIndexing() {
|
|
3561
|
+
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
3562
|
+
if ((0, import_fs4.existsSync)(this.fileHashCachePath)) {
|
|
3563
|
+
(0, import_fs4.unlinkSync)(this.fileHashCachePath);
|
|
3564
|
+
}
|
|
3565
|
+
await this.healthCheck();
|
|
3566
|
+
this.releaseIndexingLock();
|
|
3567
|
+
this.logger.info("Recovery complete, next index will re-process all files");
|
|
3260
3568
|
}
|
|
3261
3569
|
loadFailedBatches() {
|
|
3262
3570
|
try {
|
|
@@ -3299,28 +3607,43 @@ var Indexer = class {
|
|
|
3299
3607
|
return { concurrency: 5, intervalMs: 200, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3300
3608
|
case "ollama":
|
|
3301
3609
|
return { concurrency: 5, intervalMs: 0, minRetryMs: 500, maxRetryMs: 5e3 };
|
|
3610
|
+
case "custom": {
|
|
3611
|
+
const customConfig = this.config.customProvider;
|
|
3612
|
+
return {
|
|
3613
|
+
concurrency: customConfig?.concurrency ?? 3,
|
|
3614
|
+
intervalMs: customConfig?.requestIntervalMs ?? 1e3,
|
|
3615
|
+
minRetryMs: 1e3,
|
|
3616
|
+
maxRetryMs: 3e4
|
|
3617
|
+
};
|
|
3618
|
+
}
|
|
3302
3619
|
default:
|
|
3303
3620
|
return { concurrency: 3, intervalMs: 1e3, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3304
3621
|
}
|
|
3305
3622
|
}
|
|
3306
3623
|
async initialize() {
|
|
3307
|
-
|
|
3308
|
-
|
|
3624
|
+
if (this.config.embeddingProvider === "custom") {
|
|
3625
|
+
if (!this.config.customProvider) {
|
|
3626
|
+
throw new Error("embeddingProvider is 'custom' but customProvider config is missing.");
|
|
3627
|
+
}
|
|
3628
|
+
this.configuredProviderInfo = createCustomProviderInfo(this.config.customProvider);
|
|
3629
|
+
} else if (this.config.embeddingProvider === "auto") {
|
|
3630
|
+
this.configuredProviderInfo = await tryDetectProvider();
|
|
3631
|
+
} else {
|
|
3632
|
+
this.configuredProviderInfo = await detectEmbeddingProvider(this.config.embeddingProvider, this.config.embeddingModel);
|
|
3633
|
+
}
|
|
3634
|
+
if (!this.configuredProviderInfo) {
|
|
3309
3635
|
throw new Error(
|
|
3310
|
-
"No embedding provider available. Configure GitHub, OpenAI, Google, or
|
|
3636
|
+
"No embedding provider available. Configure GitHub Copilot, OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
3311
3637
|
);
|
|
3312
3638
|
}
|
|
3313
3639
|
this.logger.info("Initializing indexer", {
|
|
3314
|
-
provider: this.
|
|
3315
|
-
model: this.
|
|
3640
|
+
provider: this.configuredProviderInfo.provider,
|
|
3641
|
+
model: this.configuredProviderInfo.modelInfo.model,
|
|
3316
3642
|
scope: this.config.scope
|
|
3317
3643
|
});
|
|
3318
|
-
this.provider = createEmbeddingProvider(
|
|
3319
|
-
this.detectedProvider.credentials,
|
|
3320
|
-
this.detectedProvider.modelInfo
|
|
3321
|
-
);
|
|
3644
|
+
this.provider = createEmbeddingProvider(this.configuredProviderInfo);
|
|
3322
3645
|
await import_fs4.promises.mkdir(this.indexPath, { recursive: true });
|
|
3323
|
-
const dimensions = this.
|
|
3646
|
+
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
3324
3647
|
const storePath = path5.join(this.indexPath, "vectors");
|
|
3325
3648
|
this.store = new VectorStore(storePath, dimensions);
|
|
3326
3649
|
const indexFilePath = path5.join(this.indexPath, "vectors.usearch");
|
|
@@ -3340,9 +3663,20 @@ var Indexer = class {
|
|
|
3340
3663
|
const dbPath = path5.join(this.indexPath, "codebase.db");
|
|
3341
3664
|
const dbIsNew = !(0, import_fs4.existsSync)(dbPath);
|
|
3342
3665
|
this.database = new Database(dbPath);
|
|
3666
|
+
if (this.checkForInterruptedIndexing()) {
|
|
3667
|
+
await this.recoverFromInterruptedIndexing();
|
|
3668
|
+
}
|
|
3343
3669
|
if (dbIsNew && this.store.count() > 0) {
|
|
3344
3670
|
this.migrateFromLegacyIndex();
|
|
3345
3671
|
}
|
|
3672
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3673
|
+
if (!this.indexCompatibility.compatible) {
|
|
3674
|
+
this.logger.warn("Index compatibility issue detected", {
|
|
3675
|
+
reason: this.indexCompatibility.reason,
|
|
3676
|
+
storedMetadata: this.indexCompatibility.storedMetadata,
|
|
3677
|
+
configuredProviderInfo: this.configuredProviderInfo
|
|
3678
|
+
});
|
|
3679
|
+
}
|
|
3346
3680
|
if (isGitRepo(this.projectRoot)) {
|
|
3347
3681
|
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
3348
3682
|
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
@@ -3413,30 +3747,106 @@ var Indexer = class {
|
|
|
3413
3747
|
}
|
|
3414
3748
|
this.database.addChunksToBranchBatch(this.currentBranch || "default", chunkIds);
|
|
3415
3749
|
}
|
|
3750
|
+
loadIndexMetadata() {
|
|
3751
|
+
if (!this.database) return null;
|
|
3752
|
+
const version = this.database.getMetadata("index.version");
|
|
3753
|
+
if (!version) return null;
|
|
3754
|
+
return {
|
|
3755
|
+
indexVersion: version,
|
|
3756
|
+
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
3757
|
+
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
3758
|
+
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
3759
|
+
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
3760
|
+
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
3761
|
+
};
|
|
3762
|
+
}
|
|
3763
|
+
saveIndexMetadata(provider) {
|
|
3764
|
+
if (!this.database) return;
|
|
3765
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3766
|
+
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
3767
|
+
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
3768
|
+
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
3769
|
+
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
3770
|
+
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
3771
|
+
this.database.setMetadata("index.updatedAt", now);
|
|
3772
|
+
if (!existingCreatedAt) {
|
|
3773
|
+
this.database.setMetadata("index.createdAt", now);
|
|
3774
|
+
}
|
|
3775
|
+
}
|
|
3776
|
+
validateIndexCompatibility(provider) {
|
|
3777
|
+
const storedMetadata = this.loadIndexMetadata();
|
|
3778
|
+
if (!storedMetadata) {
|
|
3779
|
+
return { compatible: true };
|
|
3780
|
+
}
|
|
3781
|
+
const currentProvider = provider.provider;
|
|
3782
|
+
const currentModel = provider.modelInfo.model;
|
|
3783
|
+
const currentDimensions = provider.modelInfo.dimensions;
|
|
3784
|
+
if (storedMetadata.embeddingDimensions !== currentDimensions) {
|
|
3785
|
+
return {
|
|
3786
|
+
compatible: false,
|
|
3787
|
+
code: "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
|
|
3788
|
+
reason: `Dimension mismatch: index has ${storedMetadata.embeddingDimensions}D vectors (${storedMetadata.embeddingProvider}/${storedMetadata.embeddingModel}), but current provider uses ${currentDimensions}D (${currentProvider}/${currentModel}). Run index_codebase with force=true to rebuild.`,
|
|
3789
|
+
storedMetadata
|
|
3790
|
+
};
|
|
3791
|
+
}
|
|
3792
|
+
if (storedMetadata.embeddingModel !== currentModel) {
|
|
3793
|
+
return {
|
|
3794
|
+
compatible: false,
|
|
3795
|
+
code: "MODEL_MISMATCH" /* MODEL_MISMATCH */,
|
|
3796
|
+
reason: `Model mismatch: index was built with "${storedMetadata.embeddingModel}", but current model is "${currentModel}". Embeddings are incompatible. Run index_codebase with force=true to rebuild.`,
|
|
3797
|
+
storedMetadata
|
|
3798
|
+
};
|
|
3799
|
+
}
|
|
3800
|
+
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
3801
|
+
this.logger.warn("Provider changed", {
|
|
3802
|
+
storedProvider: storedMetadata.embeddingProvider,
|
|
3803
|
+
currentProvider
|
|
3804
|
+
});
|
|
3805
|
+
}
|
|
3806
|
+
return {
|
|
3807
|
+
compatible: true,
|
|
3808
|
+
storedMetadata
|
|
3809
|
+
};
|
|
3810
|
+
}
|
|
3811
|
+
checkCompatibility() {
|
|
3812
|
+
if (!this.indexCompatibility) {
|
|
3813
|
+
if (!this.configuredProviderInfo) {
|
|
3814
|
+
throw new Error("No embedding provider info, you must initialize the indexer first.");
|
|
3815
|
+
}
|
|
3816
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3817
|
+
}
|
|
3818
|
+
return this.indexCompatibility;
|
|
3819
|
+
}
|
|
3416
3820
|
async ensureInitialized() {
|
|
3417
|
-
if (!this.store || !this.provider || !this.invertedIndex || !this.
|
|
3821
|
+
if (!this.store || !this.provider || !this.invertedIndex || !this.configuredProviderInfo || !this.database) {
|
|
3418
3822
|
await this.initialize();
|
|
3419
3823
|
}
|
|
3420
3824
|
return {
|
|
3421
3825
|
store: this.store,
|
|
3422
3826
|
provider: this.provider,
|
|
3423
3827
|
invertedIndex: this.invertedIndex,
|
|
3424
|
-
|
|
3828
|
+
configuredProviderInfo: this.configuredProviderInfo,
|
|
3425
3829
|
database: this.database
|
|
3426
3830
|
};
|
|
3427
3831
|
}
|
|
3428
3832
|
async estimateCost() {
|
|
3429
|
-
const {
|
|
3833
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
3430
3834
|
const { files } = await collectFiles(
|
|
3431
3835
|
this.projectRoot,
|
|
3432
3836
|
this.config.include,
|
|
3433
3837
|
this.config.exclude,
|
|
3434
3838
|
this.config.indexing.maxFileSize
|
|
3435
3839
|
);
|
|
3436
|
-
return createCostEstimate(files,
|
|
3840
|
+
return createCostEstimate(files, configuredProviderInfo);
|
|
3437
3841
|
}
|
|
3438
3842
|
async index(onProgress) {
|
|
3439
|
-
const { store, provider, invertedIndex, database,
|
|
3843
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
3844
|
+
if (!this.indexCompatibility?.compatible) {
|
|
3845
|
+
throw new Error(
|
|
3846
|
+
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
3847
|
+
);
|
|
3848
|
+
}
|
|
3849
|
+
this.acquireIndexingLock();
|
|
3440
3850
|
this.logger.recordIndexingStart();
|
|
3441
3851
|
this.logger.info("Starting indexing", { projectRoot: this.projectRoot });
|
|
3442
3852
|
const startTime = Date.now();
|
|
@@ -3543,7 +3953,7 @@ var Indexer = class {
|
|
|
3543
3953
|
const id = generateChunkId(parsed.path, chunk);
|
|
3544
3954
|
const contentHash = generateChunkHash(chunk);
|
|
3545
3955
|
currentChunkIds.add(id);
|
|
3546
|
-
|
|
3956
|
+
chunkDataBatch.push({
|
|
3547
3957
|
chunkId: id,
|
|
3548
3958
|
contentHash,
|
|
3549
3959
|
filePath: parsed.path,
|
|
@@ -3552,8 +3962,7 @@ var Indexer = class {
|
|
|
3552
3962
|
nodeType: chunk.chunkType,
|
|
3553
3963
|
name: chunk.name,
|
|
3554
3964
|
language: chunk.language
|
|
3555
|
-
};
|
|
3556
|
-
chunkDataBatch.push(chunkData);
|
|
3965
|
+
});
|
|
3557
3966
|
if (existingChunks.get(id) === contentHash) {
|
|
3558
3967
|
fileChunkCount++;
|
|
3559
3968
|
continue;
|
|
@@ -3606,6 +4015,7 @@ var Indexer = class {
|
|
|
3606
4015
|
chunksProcessed: 0,
|
|
3607
4016
|
totalChunks: 0
|
|
3608
4017
|
});
|
|
4018
|
+
this.releaseIndexingLock();
|
|
3609
4019
|
return stats;
|
|
3610
4020
|
}
|
|
3611
4021
|
if (pendingChunks.length === 0) {
|
|
@@ -3623,6 +4033,7 @@ var Indexer = class {
|
|
|
3623
4033
|
chunksProcessed: 0,
|
|
3624
4034
|
totalChunks: 0
|
|
3625
4035
|
});
|
|
4036
|
+
this.releaseIndexingLock();
|
|
3626
4037
|
return stats;
|
|
3627
4038
|
}
|
|
3628
4039
|
onProgress?.({
|
|
@@ -3651,7 +4062,7 @@ var Indexer = class {
|
|
|
3651
4062
|
stats.indexedChunks++;
|
|
3652
4063
|
}
|
|
3653
4064
|
}
|
|
3654
|
-
const providerRateLimits = this.getProviderRateLimits(
|
|
4065
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
3655
4066
|
const queue = new PQueue({
|
|
3656
4067
|
concurrency: providerRateLimits.concurrency,
|
|
3657
4068
|
interval: providerRateLimits.intervalMs,
|
|
@@ -3662,7 +4073,7 @@ var Indexer = class {
|
|
|
3662
4073
|
for (const batch of dynamicBatches) {
|
|
3663
4074
|
queue.add(async () => {
|
|
3664
4075
|
if (rateLimitBackoffMs > 0) {
|
|
3665
|
-
await new Promise((
|
|
4076
|
+
await new Promise((resolve5) => setTimeout(resolve5, rateLimitBackoffMs));
|
|
3666
4077
|
}
|
|
3667
4078
|
try {
|
|
3668
4079
|
const result = await pRetry(
|
|
@@ -3675,6 +4086,7 @@ var Indexer = class {
|
|
|
3675
4086
|
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
3676
4087
|
maxTimeout: providerRateLimits.maxRetryMs,
|
|
3677
4088
|
factor: 2,
|
|
4089
|
+
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError),
|
|
3678
4090
|
onFailedAttempt: (error) => {
|
|
3679
4091
|
const message = getErrorMessage(error);
|
|
3680
4092
|
if (isRateLimitError(error)) {
|
|
@@ -3706,7 +4118,7 @@ var Indexer = class {
|
|
|
3706
4118
|
contentHash: chunk.contentHash,
|
|
3707
4119
|
embedding: float32ArrayToBuffer(result.embeddings[i]),
|
|
3708
4120
|
chunkText: chunk.text,
|
|
3709
|
-
model:
|
|
4121
|
+
model: configuredProviderInfo.modelInfo.model
|
|
3710
4122
|
}));
|
|
3711
4123
|
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
3712
4124
|
for (const chunk of batch) {
|
|
@@ -3757,6 +4169,8 @@ var Indexer = class {
|
|
|
3757
4169
|
await this.maybeRunOrphanGc();
|
|
3758
4170
|
}
|
|
3759
4171
|
stats.durationMs = Date.now() - startTime;
|
|
4172
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
4173
|
+
this.indexCompatibility = { compatible: true };
|
|
3760
4174
|
this.logger.recordIndexingEnd();
|
|
3761
4175
|
this.logger.info("Indexing complete", {
|
|
3762
4176
|
files: stats.totalFiles,
|
|
@@ -3777,6 +4191,7 @@ var Indexer = class {
|
|
|
3777
4191
|
chunksProcessed: stats.indexedChunks,
|
|
3778
4192
|
totalChunks: pendingChunks.length
|
|
3779
4193
|
});
|
|
4194
|
+
this.releaseIndexingLock();
|
|
3780
4195
|
return stats;
|
|
3781
4196
|
}
|
|
3782
4197
|
async getQueryEmbedding(query, provider) {
|
|
@@ -3799,7 +4214,7 @@ var Indexer = class {
|
|
|
3799
4214
|
}
|
|
3800
4215
|
this.logger.cache("debug", "Query embedding cache miss", { query: query.slice(0, 50) });
|
|
3801
4216
|
this.logger.recordQueryCacheMiss();
|
|
3802
|
-
const { embedding, tokensUsed } = await provider.
|
|
4217
|
+
const { embedding, tokensUsed } = await provider.embedQuery(query);
|
|
3803
4218
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
3804
4219
|
if (this.queryEmbeddingCache.size >= this.maxQueryCacheSize) {
|
|
3805
4220
|
const oldestKey = this.queryEmbeddingCache.keys().next().value;
|
|
@@ -3842,8 +4257,14 @@ var Indexer = class {
|
|
|
3842
4257
|
return intersection / union;
|
|
3843
4258
|
}
|
|
3844
4259
|
async search(query, limit, options) {
|
|
3845
|
-
const searchStartTime = import_perf_hooks.performance.now();
|
|
3846
4260
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4261
|
+
const compatibility = this.checkCompatibility();
|
|
4262
|
+
if (!compatibility.compatible) {
|
|
4263
|
+
throw new Error(
|
|
4264
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} A possible solution is to run index_codebase with force=true to rebuild the index.`
|
|
4265
|
+
);
|
|
4266
|
+
}
|
|
4267
|
+
const searchStartTime = import_perf_hooks.performance.now();
|
|
3847
4268
|
if (store.count() === 0) {
|
|
3848
4269
|
this.logger.search("debug", "Search on empty index", { query });
|
|
3849
4270
|
return [];
|
|
@@ -3985,15 +4406,16 @@ var Indexer = class {
|
|
|
3985
4406
|
return results.slice(0, limit);
|
|
3986
4407
|
}
|
|
3987
4408
|
async getStatus() {
|
|
3988
|
-
const { store,
|
|
4409
|
+
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
3989
4410
|
return {
|
|
3990
4411
|
indexed: store.count() > 0,
|
|
3991
4412
|
vectorCount: store.count(),
|
|
3992
|
-
provider:
|
|
3993
|
-
model:
|
|
4413
|
+
provider: configuredProviderInfo.provider,
|
|
4414
|
+
model: configuredProviderInfo.modelInfo.model,
|
|
3994
4415
|
indexPath: this.indexPath,
|
|
3995
4416
|
currentBranch: this.currentBranch,
|
|
3996
|
-
baseBranch: this.baseBranch
|
|
4417
|
+
baseBranch: this.baseBranch,
|
|
4418
|
+
compatibility: this.indexCompatibility
|
|
3997
4419
|
};
|
|
3998
4420
|
}
|
|
3999
4421
|
async clearIndex() {
|
|
@@ -4005,6 +4427,13 @@ var Indexer = class {
|
|
|
4005
4427
|
this.fileHashCache.clear();
|
|
4006
4428
|
this.saveFileHashCache();
|
|
4007
4429
|
database.clearBranch(this.currentBranch);
|
|
4430
|
+
database.deleteMetadata("index.version");
|
|
4431
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
4432
|
+
database.deleteMetadata("index.embeddingModel");
|
|
4433
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
4434
|
+
database.deleteMetadata("index.createdAt");
|
|
4435
|
+
database.deleteMetadata("index.updatedAt");
|
|
4436
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
4008
4437
|
}
|
|
4009
4438
|
async healthCheck() {
|
|
4010
4439
|
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
@@ -4118,26 +4547,31 @@ var Indexer = class {
|
|
|
4118
4547
|
getLogger() {
|
|
4119
4548
|
return this.logger;
|
|
4120
4549
|
}
|
|
4121
|
-
async findSimilar(code, limit, options) {
|
|
4122
|
-
const searchStartTime = import_perf_hooks.performance.now();
|
|
4550
|
+
async findSimilar(code, limit = this.config.search.maxResults, options) {
|
|
4123
4551
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4552
|
+
const compatibility = this.checkCompatibility();
|
|
4553
|
+
if (!compatibility.compatible) {
|
|
4554
|
+
throw new Error(
|
|
4555
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} Run index_codebase with force=true to rebuild the index.`
|
|
4556
|
+
);
|
|
4557
|
+
}
|
|
4558
|
+
const searchStartTime = import_perf_hooks.performance.now();
|
|
4124
4559
|
if (store.count() === 0) {
|
|
4125
4560
|
this.logger.search("debug", "Find similar on empty index");
|
|
4126
4561
|
return [];
|
|
4127
4562
|
}
|
|
4128
|
-
const maxResults = limit ?? this.config.search.maxResults;
|
|
4129
4563
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
4130
4564
|
this.logger.search("debug", "Starting find similar", {
|
|
4131
4565
|
codeLength: code.length,
|
|
4132
|
-
|
|
4566
|
+
limit,
|
|
4133
4567
|
filterByBranch
|
|
4134
4568
|
});
|
|
4135
4569
|
const embeddingStartTime = import_perf_hooks.performance.now();
|
|
4136
|
-
const { embedding, tokensUsed } = await provider.
|
|
4570
|
+
const { embedding, tokensUsed } = await provider.embedDocument(code);
|
|
4137
4571
|
const embeddingMs = import_perf_hooks.performance.now() - embeddingStartTime;
|
|
4138
4572
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
4139
4573
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
4140
|
-
const semanticResults = store.search(embedding,
|
|
4574
|
+
const semanticResults = store.search(embedding, limit * 2);
|
|
4141
4575
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
4142
4576
|
let branchChunkIds = null;
|
|
4143
4577
|
if (filterByBranch && this.currentBranch !== "default") {
|
|
@@ -4161,7 +4595,7 @@ var Indexer = class {
|
|
|
4161
4595
|
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
4162
4596
|
}
|
|
4163
4597
|
return true;
|
|
4164
|
-
}).slice(0,
|
|
4598
|
+
}).slice(0, limit);
|
|
4165
4599
|
const totalSearchMs = import_perf_hooks.performance.now() - searchStartTime;
|
|
4166
4600
|
this.logger.recordSearch(totalSearchMs, {
|
|
4167
4601
|
embeddingMs,
|
|
@@ -5076,7 +5510,7 @@ var NodeFsHandler = class {
|
|
|
5076
5510
|
this._addToNodeFs(path9, initialAdd, wh, depth + 1);
|
|
5077
5511
|
}
|
|
5078
5512
|
}).on(EV.ERROR, this._boundHandleError);
|
|
5079
|
-
return new Promise((
|
|
5513
|
+
return new Promise((resolve5, reject) => {
|
|
5080
5514
|
if (!stream)
|
|
5081
5515
|
return reject();
|
|
5082
5516
|
stream.once(STR_END, () => {
|
|
@@ -5085,7 +5519,7 @@ var NodeFsHandler = class {
|
|
|
5085
5519
|
return;
|
|
5086
5520
|
}
|
|
5087
5521
|
const wasThrottled = throttler ? throttler.clear() : false;
|
|
5088
|
-
|
|
5522
|
+
resolve5(void 0);
|
|
5089
5523
|
previous.getChildren().filter((item) => {
|
|
5090
5524
|
return item !== directory && !current.has(item);
|
|
5091
5525
|
}).forEach((item) => {
|
|
@@ -6132,7 +6566,8 @@ function createWatcherWithIndexer(indexer, projectRoot, config) {
|
|
|
6132
6566
|
|
|
6133
6567
|
// src/tools/index.ts
|
|
6134
6568
|
var import_plugin = require("@opencode-ai/plugin");
|
|
6135
|
-
|
|
6569
|
+
|
|
6570
|
+
// src/tools/utils.ts
|
|
6136
6571
|
var MAX_CONTENT_LINES = 30;
|
|
6137
6572
|
function truncateContent(content) {
|
|
6138
6573
|
const lines = content.split("\n");
|
|
@@ -6140,49 +6575,177 @@ function truncateContent(content) {
|
|
|
6140
6575
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
6141
6576
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
6142
6577
|
}
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
}
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
}
|
|
6153
|
-
var codebase_search = (0, import_plugin.tool)({
|
|
6154
|
-
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6155
|
-
args: {
|
|
6156
|
-
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6157
|
-
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6158
|
-
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6159
|
-
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6160
|
-
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6161
|
-
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6162
|
-
},
|
|
6163
|
-
async execute(args) {
|
|
6164
|
-
const indexer = getIndexer();
|
|
6165
|
-
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6166
|
-
fileType: args.fileType,
|
|
6167
|
-
directory: args.directory,
|
|
6168
|
-
chunkType: args.chunkType,
|
|
6169
|
-
contextLines: args.contextLines
|
|
6170
|
-
});
|
|
6171
|
-
if (results.length === 0) {
|
|
6172
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6578
|
+
function formatIndexStats(stats, verbose = false) {
|
|
6579
|
+
const lines = [];
|
|
6580
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
6581
|
+
lines.push(`Indexed. ${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
6582
|
+
} else if (stats.indexedChunks === 0) {
|
|
6583
|
+
lines.push(`Indexed. ${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
6584
|
+
} else {
|
|
6585
|
+
let main = `Indexed. ${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
6586
|
+
if (stats.existingChunks > 0) {
|
|
6587
|
+
main += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
6173
6588
|
}
|
|
6174
|
-
|
|
6175
|
-
|
|
6176
|
-
|
|
6589
|
+
lines.push(main);
|
|
6590
|
+
if (stats.removedChunks > 0) {
|
|
6591
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
6592
|
+
}
|
|
6593
|
+
if (stats.failedChunks > 0) {
|
|
6594
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
6595
|
+
}
|
|
6596
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
6597
|
+
}
|
|
6598
|
+
if (verbose) {
|
|
6599
|
+
if (stats.skippedFiles.length > 0) {
|
|
6600
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6601
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6602
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6603
|
+
lines.push("");
|
|
6604
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6605
|
+
if (tooLarge.length > 0) {
|
|
6606
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6607
|
+
}
|
|
6608
|
+
if (excluded.length > 0) {
|
|
6609
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6610
|
+
}
|
|
6611
|
+
if (gitignored.length > 0) {
|
|
6612
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6613
|
+
}
|
|
6614
|
+
}
|
|
6615
|
+
if (stats.parseFailures.length > 0) {
|
|
6616
|
+
lines.push("");
|
|
6617
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6618
|
+
}
|
|
6619
|
+
}
|
|
6620
|
+
return lines.join("\n");
|
|
6621
|
+
}
|
|
6622
|
+
function formatStatus(status) {
|
|
6623
|
+
if (!status.indexed) {
|
|
6624
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6625
|
+
}
|
|
6626
|
+
const lines = [
|
|
6627
|
+
`Index status:`,
|
|
6628
|
+
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6629
|
+
` Provider: ${status.provider}`,
|
|
6630
|
+
` Model: ${status.model}`,
|
|
6631
|
+
` Location: ${status.indexPath}`
|
|
6632
|
+
];
|
|
6633
|
+
if (status.currentBranch !== "default") {
|
|
6634
|
+
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6635
|
+
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6636
|
+
}
|
|
6637
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
6638
|
+
lines.push("");
|
|
6639
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
6640
|
+
if (status.compatibility.storedMetadata) {
|
|
6641
|
+
const stored = status.compatibility.storedMetadata;
|
|
6642
|
+
lines.push(` Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
6643
|
+
lines.push(` Current config: ${status.provider}/${status.model}`);
|
|
6644
|
+
}
|
|
6645
|
+
} else if (!status.compatibility) {
|
|
6646
|
+
lines.push(` Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
6647
|
+
} else {
|
|
6648
|
+
lines.push(` Compatibility: Index is compatible with the current provider and model.`);
|
|
6649
|
+
}
|
|
6650
|
+
return lines.join("\n");
|
|
6651
|
+
}
|
|
6652
|
+
function formatProgressTitle(progress) {
|
|
6653
|
+
switch (progress.phase) {
|
|
6654
|
+
case "scanning":
|
|
6655
|
+
return "Scanning files...";
|
|
6656
|
+
case "parsing":
|
|
6657
|
+
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6658
|
+
case "embedding":
|
|
6659
|
+
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6660
|
+
case "storing":
|
|
6661
|
+
return "Storing index...";
|
|
6662
|
+
case "complete":
|
|
6663
|
+
return "Indexing complete";
|
|
6664
|
+
default:
|
|
6665
|
+
return "Indexing...";
|
|
6666
|
+
}
|
|
6667
|
+
}
|
|
6668
|
+
function calculatePercentage(progress) {
|
|
6669
|
+
if (progress.phase === "scanning") return 0;
|
|
6670
|
+
if (progress.phase === "complete") return 100;
|
|
6671
|
+
if (progress.phase === "parsing") {
|
|
6672
|
+
if (progress.totalFiles === 0) return 5;
|
|
6673
|
+
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6674
|
+
}
|
|
6675
|
+
if (progress.phase === "embedding") {
|
|
6676
|
+
if (progress.totalChunks === 0) return 20;
|
|
6677
|
+
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6678
|
+
}
|
|
6679
|
+
if (progress.phase === "storing") return 95;
|
|
6680
|
+
return 0;
|
|
6681
|
+
}
|
|
6682
|
+
function formatCodebasePeek(results, query) {
|
|
6683
|
+
if (results.length === 0) {
|
|
6684
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6685
|
+
}
|
|
6686
|
+
const formatted = results.map((r, idx) => {
|
|
6687
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6688
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6689
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6690
|
+
});
|
|
6691
|
+
return `Found ${results.length} locations for "${query}":
|
|
6692
|
+
|
|
6693
|
+
${formatted.join("\n")}
|
|
6694
|
+
|
|
6695
|
+
Use Read tool to examine specific files.`;
|
|
6696
|
+
}
|
|
6697
|
+
function formatHealthCheck(result) {
|
|
6698
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0) {
|
|
6699
|
+
return "Index is healthy. No stale entries found.";
|
|
6700
|
+
}
|
|
6701
|
+
const lines = [`Health check complete:`];
|
|
6702
|
+
if (result.removed > 0) {
|
|
6703
|
+
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6704
|
+
}
|
|
6705
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
6706
|
+
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6707
|
+
}
|
|
6708
|
+
if (result.gcOrphanChunks > 0) {
|
|
6709
|
+
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6710
|
+
}
|
|
6711
|
+
if (result.filePaths.length > 0) {
|
|
6712
|
+
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6713
|
+
}
|
|
6714
|
+
return lines.join("\n");
|
|
6715
|
+
}
|
|
6716
|
+
function formatLogs(logs) {
|
|
6717
|
+
if (logs.length === 0) {
|
|
6718
|
+
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6719
|
+
}
|
|
6720
|
+
return logs.map((l) => {
|
|
6721
|
+
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6722
|
+
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6723
|
+
}).join("\n");
|
|
6724
|
+
}
|
|
6725
|
+
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
6726
|
+
const formatted = results.map((r, idx) => {
|
|
6727
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6728
|
+
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
6729
|
+
return `${header} ${scoreLabel}
|
|
6177
6730
|
\`\`\`
|
|
6178
6731
|
${truncateContent(r.content)}
|
|
6179
6732
|
\`\`\``;
|
|
6180
|
-
|
|
6181
|
-
|
|
6733
|
+
});
|
|
6734
|
+
return formatted.join("\n\n");
|
|
6735
|
+
}
|
|
6182
6736
|
|
|
6183
|
-
|
|
6737
|
+
// src/tools/index.ts
|
|
6738
|
+
var z = import_plugin.tool.schema;
|
|
6739
|
+
var sharedIndexer = null;
|
|
6740
|
+
function initializeTools(projectRoot, config) {
|
|
6741
|
+
sharedIndexer = new Indexer(projectRoot, config);
|
|
6742
|
+
}
|
|
6743
|
+
function getIndexer() {
|
|
6744
|
+
if (!sharedIndexer) {
|
|
6745
|
+
throw new Error("Codebase index tools not initialized. Plugin may not be loaded correctly.");
|
|
6184
6746
|
}
|
|
6185
|
-
|
|
6747
|
+
return sharedIndexer;
|
|
6748
|
+
}
|
|
6186
6749
|
var codebase_peek = (0, import_plugin.tool)({
|
|
6187
6750
|
description: "Quick lookup of code locations by meaning. Returns only metadata (file, line, name, type) WITHOUT code content. Use this first to find WHERE code is, then use Read tool to examine specific files. Saves tokens by not returning full code blocks. Best for: discovery, navigation, finding multiple related locations.",
|
|
6188
6751
|
args: {
|
|
@@ -6200,19 +6763,7 @@ var codebase_peek = (0, import_plugin.tool)({
|
|
|
6200
6763
|
chunkType: args.chunkType,
|
|
6201
6764
|
metadataOnly: true
|
|
6202
6765
|
});
|
|
6203
|
-
|
|
6204
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6205
|
-
}
|
|
6206
|
-
const formatted = results.map((r, idx) => {
|
|
6207
|
-
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6208
|
-
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6209
|
-
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6210
|
-
});
|
|
6211
|
-
return `Found ${results.length} locations for "${args.query}":
|
|
6212
|
-
|
|
6213
|
-
${formatted.join("\n")}
|
|
6214
|
-
|
|
6215
|
-
Use Read tool to examine specific files.`;
|
|
6766
|
+
return formatCodebasePeek(results, args.query);
|
|
6216
6767
|
}
|
|
6217
6768
|
});
|
|
6218
6769
|
var index_codebase = (0, import_plugin.tool)({
|
|
@@ -6262,23 +6813,7 @@ var index_health_check = (0, import_plugin.tool)({
|
|
|
6262
6813
|
async execute() {
|
|
6263
6814
|
const indexer = getIndexer();
|
|
6264
6815
|
const result = await indexer.healthCheck();
|
|
6265
|
-
|
|
6266
|
-
return "Index is healthy. No stale entries found.";
|
|
6267
|
-
}
|
|
6268
|
-
const lines = [`Health check complete:`];
|
|
6269
|
-
if (result.removed > 0) {
|
|
6270
|
-
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6271
|
-
}
|
|
6272
|
-
if (result.gcOrphanEmbeddings > 0) {
|
|
6273
|
-
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6274
|
-
}
|
|
6275
|
-
if (result.gcOrphanChunks > 0) {
|
|
6276
|
-
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6277
|
-
}
|
|
6278
|
-
if (result.filePaths.length > 0) {
|
|
6279
|
-
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6280
|
-
}
|
|
6281
|
-
return lines.join("\n");
|
|
6816
|
+
return formatHealthCheck(result);
|
|
6282
6817
|
}
|
|
6283
6818
|
});
|
|
6284
6819
|
var index_metrics = (0, import_plugin.tool)({
|
|
@@ -6317,13 +6852,7 @@ var index_logs = (0, import_plugin.tool)({
|
|
|
6317
6852
|
} else {
|
|
6318
6853
|
logs = logger.getLogs(args.limit);
|
|
6319
6854
|
}
|
|
6320
|
-
|
|
6321
|
-
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6322
|
-
}
|
|
6323
|
-
return logs.map((l) => {
|
|
6324
|
-
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6325
|
-
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6326
|
-
}).join("\n");
|
|
6855
|
+
return formatLogs(logs);
|
|
6327
6856
|
}
|
|
6328
6857
|
});
|
|
6329
6858
|
var find_similar = (0, import_plugin.tool)({
|
|
@@ -6338,7 +6867,7 @@ var find_similar = (0, import_plugin.tool)({
|
|
|
6338
6867
|
},
|
|
6339
6868
|
async execute(args) {
|
|
6340
6869
|
const indexer = getIndexer();
|
|
6341
|
-
const results = await indexer.findSimilar(args.code, args.limit
|
|
6870
|
+
const results = await indexer.findSimilar(args.code, args.limit, {
|
|
6342
6871
|
fileType: args.fileType,
|
|
6343
6872
|
directory: args.directory,
|
|
6344
6873
|
chunkType: args.chunkType,
|
|
@@ -6347,109 +6876,37 @@ var find_similar = (0, import_plugin.tool)({
|
|
|
6347
6876
|
if (results.length === 0) {
|
|
6348
6877
|
return "No similar code found. Try a different snippet or run index_codebase first.";
|
|
6349
6878
|
}
|
|
6350
|
-
const formatted = results.map((r, idx) => {
|
|
6351
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6352
|
-
return `${header} (similarity: ${(r.score * 100).toFixed(1)}%)
|
|
6353
|
-
\`\`\`
|
|
6354
|
-
${truncateContent(r.content)}
|
|
6355
|
-
\`\`\``;
|
|
6356
|
-
});
|
|
6357
6879
|
return `Found ${results.length} similar code blocks:
|
|
6358
6880
|
|
|
6359
|
-
${
|
|
6881
|
+
${formatSearchResults(results)}`;
|
|
6360
6882
|
}
|
|
6361
6883
|
});
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
}
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
if (verbose) {
|
|
6383
|
-
if (stats.skippedFiles.length > 0) {
|
|
6384
|
-
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6385
|
-
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6386
|
-
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6387
|
-
lines.push("");
|
|
6388
|
-
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6389
|
-
if (tooLarge.length > 0) {
|
|
6390
|
-
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6391
|
-
}
|
|
6392
|
-
if (excluded.length > 0) {
|
|
6393
|
-
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6394
|
-
}
|
|
6395
|
-
if (gitignored.length > 0) {
|
|
6396
|
-
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6397
|
-
}
|
|
6398
|
-
}
|
|
6399
|
-
if (stats.parseFailures.length > 0) {
|
|
6400
|
-
lines.push("");
|
|
6401
|
-
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6884
|
+
var codebase_search = (0, import_plugin.tool)({
|
|
6885
|
+
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6886
|
+
args: {
|
|
6887
|
+
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6888
|
+
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6889
|
+
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6890
|
+
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6891
|
+
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6892
|
+
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6893
|
+
},
|
|
6894
|
+
async execute(args) {
|
|
6895
|
+
const indexer = getIndexer();
|
|
6896
|
+
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6897
|
+
fileType: args.fileType,
|
|
6898
|
+
directory: args.directory,
|
|
6899
|
+
chunkType: args.chunkType,
|
|
6900
|
+
contextLines: args.contextLines
|
|
6901
|
+
});
|
|
6902
|
+
if (results.length === 0) {
|
|
6903
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6402
6904
|
}
|
|
6905
|
+
return `Found ${results.length} results for "${args.query}":
|
|
6906
|
+
|
|
6907
|
+
${formatSearchResults(results, "score")}`;
|
|
6403
6908
|
}
|
|
6404
|
-
|
|
6405
|
-
}
|
|
6406
|
-
function formatStatus(status) {
|
|
6407
|
-
if (!status.indexed) {
|
|
6408
|
-
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6409
|
-
}
|
|
6410
|
-
const lines = [
|
|
6411
|
-
`Index status:`,
|
|
6412
|
-
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6413
|
-
` Provider: ${status.provider}`,
|
|
6414
|
-
` Model: ${status.model}`,
|
|
6415
|
-
` Location: ${status.indexPath}`
|
|
6416
|
-
];
|
|
6417
|
-
if (status.currentBranch !== "default") {
|
|
6418
|
-
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6419
|
-
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6420
|
-
}
|
|
6421
|
-
return lines.join("\n");
|
|
6422
|
-
}
|
|
6423
|
-
function formatProgressTitle(progress) {
|
|
6424
|
-
switch (progress.phase) {
|
|
6425
|
-
case "scanning":
|
|
6426
|
-
return "Scanning files...";
|
|
6427
|
-
case "parsing":
|
|
6428
|
-
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6429
|
-
case "embedding":
|
|
6430
|
-
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6431
|
-
case "storing":
|
|
6432
|
-
return "Storing index...";
|
|
6433
|
-
case "complete":
|
|
6434
|
-
return "Indexing complete";
|
|
6435
|
-
default:
|
|
6436
|
-
return "Indexing...";
|
|
6437
|
-
}
|
|
6438
|
-
}
|
|
6439
|
-
function calculatePercentage(progress) {
|
|
6440
|
-
if (progress.phase === "scanning") return 0;
|
|
6441
|
-
if (progress.phase === "complete") return 100;
|
|
6442
|
-
if (progress.phase === "parsing") {
|
|
6443
|
-
if (progress.totalFiles === 0) return 5;
|
|
6444
|
-
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6445
|
-
}
|
|
6446
|
-
if (progress.phase === "embedding") {
|
|
6447
|
-
if (progress.totalChunks === 0) return 20;
|
|
6448
|
-
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6449
|
-
}
|
|
6450
|
-
if (progress.phase === "storing") return 95;
|
|
6451
|
-
return 0;
|
|
6452
|
-
}
|
|
6909
|
+
});
|
|
6453
6910
|
|
|
6454
6911
|
// src/commands/loader.ts
|
|
6455
6912
|
var import_fs5 = require("fs");
|