opencode-codebase-index 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -2
- package/dist/cli.cjs +4862 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.js +4867 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +655 -352
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +657 -353
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +23 -4
package/dist/index.cjs
CHANGED
|
@@ -662,7 +662,7 @@ var path8 = __toESM(require("path"), 1);
|
|
|
662
662
|
var os3 = __toESM(require("os"), 1);
|
|
663
663
|
var import_url2 = require("url");
|
|
664
664
|
|
|
665
|
-
// src/config/
|
|
665
|
+
// src/config/constants.ts
|
|
666
666
|
var DEFAULT_INCLUDE = [
|
|
667
667
|
"**/*.{ts,tsx,js,jsx,mjs,cjs}",
|
|
668
668
|
"**/*.{py,pyi}",
|
|
@@ -690,6 +690,80 @@ var DEFAULT_EXCLUDE = [
|
|
|
690
690
|
"**/.nuxt/**",
|
|
691
691
|
"**/.opencode/**"
|
|
692
692
|
];
|
|
693
|
+
var EMBEDDING_MODELS = {
|
|
694
|
+
"google": {
|
|
695
|
+
// `text-embedding-004` is DEPRECATED - https://ai.google.dev/gemini-api/docs/deprecations
|
|
696
|
+
"text-embedding-005": {
|
|
697
|
+
provider: "google",
|
|
698
|
+
model: "text-embedding-005",
|
|
699
|
+
dimensions: 768,
|
|
700
|
+
maxTokens: 2048,
|
|
701
|
+
costPer1MTokens: 0.025,
|
|
702
|
+
taskAble: false
|
|
703
|
+
// Note: on reality, this model allows for task-specific embeddings. See: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/embeddings/task-types
|
|
704
|
+
},
|
|
705
|
+
"gemini-embedding-001": {
|
|
706
|
+
provider: "google",
|
|
707
|
+
model: "gemini-embedding-001",
|
|
708
|
+
// Native output is 3072D, but we use Matryoshka truncation via outputDimensionality
|
|
709
|
+
// to reduce to 1536D for better storage/search efficiency with minimal quality loss.
|
|
710
|
+
// Google recommends 768, 1536, or 3072. See: https://ai.google.dev/gemini-api/docs/embeddings
|
|
711
|
+
dimensions: 1536,
|
|
712
|
+
maxTokens: 2048,
|
|
713
|
+
costPer1MTokens: 0.15,
|
|
714
|
+
taskAble: true
|
|
715
|
+
}
|
|
716
|
+
},
|
|
717
|
+
"openai": {
|
|
718
|
+
"text-embedding-3-small": {
|
|
719
|
+
provider: "openai",
|
|
720
|
+
model: "text-embedding-3-small",
|
|
721
|
+
dimensions: 1536,
|
|
722
|
+
maxTokens: 8191,
|
|
723
|
+
costPer1MTokens: 0.02
|
|
724
|
+
},
|
|
725
|
+
"text-embedding-3-large": {
|
|
726
|
+
provider: "openai",
|
|
727
|
+
model: "text-embedding-3-large",
|
|
728
|
+
dimensions: 3072,
|
|
729
|
+
maxTokens: 8191,
|
|
730
|
+
costPer1MTokens: 0.13
|
|
731
|
+
}
|
|
732
|
+
},
|
|
733
|
+
"ollama": {
|
|
734
|
+
"nomic-embed-text": {
|
|
735
|
+
provider: "ollama",
|
|
736
|
+
model: "nomic-embed-text",
|
|
737
|
+
dimensions: 768,
|
|
738
|
+
maxTokens: 8192,
|
|
739
|
+
costPer1MTokens: 0
|
|
740
|
+
},
|
|
741
|
+
"mxbai-embed-large": {
|
|
742
|
+
provider: "ollama",
|
|
743
|
+
model: "mxbai-embed-large",
|
|
744
|
+
dimensions: 1024,
|
|
745
|
+
maxTokens: 512,
|
|
746
|
+
costPer1MTokens: 0
|
|
747
|
+
}
|
|
748
|
+
},
|
|
749
|
+
"github-copilot": {
|
|
750
|
+
"text-embedding-3-small": {
|
|
751
|
+
provider: "github-copilot",
|
|
752
|
+
model: "text-embedding-3-small",
|
|
753
|
+
dimensions: 1536,
|
|
754
|
+
maxTokens: 8191,
|
|
755
|
+
costPer1MTokens: 0
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
};
|
|
759
|
+
var DEFAULT_PROVIDER_MODELS = {
|
|
760
|
+
"github-copilot": "text-embedding-3-small",
|
|
761
|
+
"openai": "text-embedding-3-small",
|
|
762
|
+
"google": "text-embedding-005",
|
|
763
|
+
"ollama": "nomic-embed-text"
|
|
764
|
+
};
|
|
765
|
+
|
|
766
|
+
// src/config/schema.ts
|
|
693
767
|
function getDefaultIndexingConfig() {
|
|
694
768
|
return {
|
|
695
769
|
autoIndex: false,
|
|
@@ -726,11 +800,13 @@ function getDefaultDebugConfig() {
|
|
|
726
800
|
metrics: true
|
|
727
801
|
};
|
|
728
802
|
}
|
|
729
|
-
var VALID_PROVIDERS = ["auto", "github-copilot", "openai", "google", "ollama"];
|
|
730
803
|
var VALID_SCOPES = ["project", "global"];
|
|
731
804
|
var VALID_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
732
805
|
function isValidProvider(value) {
|
|
733
|
-
return typeof value === "string" &&
|
|
806
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS).includes(value);
|
|
807
|
+
}
|
|
808
|
+
function isValidModel(value, provider) {
|
|
809
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS[provider]).includes(value);
|
|
734
810
|
}
|
|
735
811
|
function isValidScope(value) {
|
|
736
812
|
return typeof value === "string" && VALID_SCOPES.includes(value);
|
|
@@ -779,9 +855,19 @@ function parseConfig(raw) {
|
|
|
779
855
|
logBranch: typeof rawDebug.logBranch === "boolean" ? rawDebug.logBranch : defaultDebug.logBranch,
|
|
780
856
|
metrics: typeof rawDebug.metrics === "boolean" ? rawDebug.metrics : defaultDebug.metrics
|
|
781
857
|
};
|
|
858
|
+
let embeddingProvider;
|
|
859
|
+
let embeddingModel = void 0;
|
|
860
|
+
if (isValidProvider(input.embeddingProvider)) {
|
|
861
|
+
embeddingProvider = input.embeddingProvider;
|
|
862
|
+
if (input.embeddingModel) {
|
|
863
|
+
embeddingModel = isValidModel(input.embeddingModel, embeddingProvider) ? input.embeddingModel : DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
864
|
+
}
|
|
865
|
+
} else {
|
|
866
|
+
embeddingProvider = "auto";
|
|
867
|
+
}
|
|
782
868
|
return {
|
|
783
|
-
embeddingProvider
|
|
784
|
-
embeddingModel
|
|
869
|
+
embeddingProvider,
|
|
870
|
+
embeddingModel,
|
|
785
871
|
scope: isValidScope(input.scope) ? input.scope : "project",
|
|
786
872
|
include: isStringArray(input.include) ? input.include : DEFAULT_INCLUDE,
|
|
787
873
|
exclude: isStringArray(input.exclude) ? input.exclude : DEFAULT_EXCLUDE,
|
|
@@ -790,64 +876,12 @@ function parseConfig(raw) {
|
|
|
790
876
|
debug
|
|
791
877
|
};
|
|
792
878
|
}
|
|
793
|
-
var EMBEDDING_MODELS = {
|
|
794
|
-
"github-copilot/text-embedding-3-small": {
|
|
795
|
-
provider: "github-copilot",
|
|
796
|
-
model: "text-embedding-3-small",
|
|
797
|
-
dimensions: 1536,
|
|
798
|
-
maxTokens: 8191,
|
|
799
|
-
costPer1MTokens: 0
|
|
800
|
-
},
|
|
801
|
-
"openai/text-embedding-3-small": {
|
|
802
|
-
provider: "openai",
|
|
803
|
-
model: "text-embedding-3-small",
|
|
804
|
-
dimensions: 1536,
|
|
805
|
-
maxTokens: 8191,
|
|
806
|
-
costPer1MTokens: 0.02
|
|
807
|
-
},
|
|
808
|
-
"openai/text-embedding-3-large": {
|
|
809
|
-
provider: "openai",
|
|
810
|
-
model: "text-embedding-3-large",
|
|
811
|
-
dimensions: 3072,
|
|
812
|
-
maxTokens: 8191,
|
|
813
|
-
costPer1MTokens: 0.13
|
|
814
|
-
},
|
|
815
|
-
"google/text-embedding-004": {
|
|
816
|
-
provider: "google",
|
|
817
|
-
model: "text-embedding-004",
|
|
818
|
-
dimensions: 768,
|
|
819
|
-
maxTokens: 2048,
|
|
820
|
-
costPer1MTokens: 0
|
|
821
|
-
},
|
|
822
|
-
"ollama/nomic-embed-text": {
|
|
823
|
-
provider: "ollama",
|
|
824
|
-
model: "nomic-embed-text",
|
|
825
|
-
dimensions: 768,
|
|
826
|
-
maxTokens: 8192,
|
|
827
|
-
costPer1MTokens: 0
|
|
828
|
-
},
|
|
829
|
-
"ollama/mxbai-embed-large": {
|
|
830
|
-
provider: "ollama",
|
|
831
|
-
model: "mxbai-embed-large",
|
|
832
|
-
dimensions: 1024,
|
|
833
|
-
maxTokens: 512,
|
|
834
|
-
costPer1MTokens: 0
|
|
835
|
-
}
|
|
836
|
-
};
|
|
837
879
|
function getDefaultModelForProvider(provider) {
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
case "openai":
|
|
842
|
-
return EMBEDDING_MODELS["openai/text-embedding-3-small"];
|
|
843
|
-
case "google":
|
|
844
|
-
return EMBEDDING_MODELS["google/text-embedding-004"];
|
|
845
|
-
case "ollama":
|
|
846
|
-
return EMBEDDING_MODELS["ollama/nomic-embed-text"];
|
|
847
|
-
default:
|
|
848
|
-
return EMBEDDING_MODELS["github-copilot/text-embedding-3-small"];
|
|
849
|
-
}
|
|
880
|
+
const models = EMBEDDING_MODELS[provider];
|
|
881
|
+
const providerDefault = DEFAULT_PROVIDER_MODELS[provider];
|
|
882
|
+
return models[providerDefault];
|
|
850
883
|
}
|
|
884
|
+
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
851
885
|
|
|
852
886
|
// src/indexer/index.ts
|
|
853
887
|
var import_fs4 = require("fs");
|
|
@@ -876,7 +910,7 @@ function pTimeout(promise, options) {
|
|
|
876
910
|
} = options;
|
|
877
911
|
let timer;
|
|
878
912
|
let abortHandler;
|
|
879
|
-
const wrappedPromise = new Promise((
|
|
913
|
+
const wrappedPromise = new Promise((resolve5, reject) => {
|
|
880
914
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
881
915
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
882
916
|
}
|
|
@@ -890,7 +924,7 @@ function pTimeout(promise, options) {
|
|
|
890
924
|
};
|
|
891
925
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
892
926
|
}
|
|
893
|
-
promise.then(
|
|
927
|
+
promise.then(resolve5, reject);
|
|
894
928
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
895
929
|
return;
|
|
896
930
|
}
|
|
@@ -898,7 +932,7 @@ function pTimeout(promise, options) {
|
|
|
898
932
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
899
933
|
if (fallback) {
|
|
900
934
|
try {
|
|
901
|
-
|
|
935
|
+
resolve5(fallback());
|
|
902
936
|
} catch (error) {
|
|
903
937
|
reject(error);
|
|
904
938
|
}
|
|
@@ -908,7 +942,7 @@ function pTimeout(promise, options) {
|
|
|
908
942
|
promise.cancel();
|
|
909
943
|
}
|
|
910
944
|
if (message === false) {
|
|
911
|
-
|
|
945
|
+
resolve5();
|
|
912
946
|
} else if (message instanceof Error) {
|
|
913
947
|
reject(message);
|
|
914
948
|
} else {
|
|
@@ -1298,7 +1332,7 @@ var PQueue = class extends import_index.default {
|
|
|
1298
1332
|
// Assign unique ID if not provided
|
|
1299
1333
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1300
1334
|
};
|
|
1301
|
-
return new Promise((
|
|
1335
|
+
return new Promise((resolve5, reject) => {
|
|
1302
1336
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1303
1337
|
this.#queue.enqueue(async () => {
|
|
1304
1338
|
this.#pending++;
|
|
@@ -1336,7 +1370,7 @@ var PQueue = class extends import_index.default {
|
|
|
1336
1370
|
})]);
|
|
1337
1371
|
}
|
|
1338
1372
|
const result = await operation;
|
|
1339
|
-
|
|
1373
|
+
resolve5(result);
|
|
1340
1374
|
this.emit("completed", result);
|
|
1341
1375
|
} catch (error) {
|
|
1342
1376
|
reject(error);
|
|
@@ -1493,13 +1527,13 @@ var PQueue = class extends import_index.default {
|
|
|
1493
1527
|
});
|
|
1494
1528
|
}
|
|
1495
1529
|
async #onEvent(event, filter) {
|
|
1496
|
-
return new Promise((
|
|
1530
|
+
return new Promise((resolve5) => {
|
|
1497
1531
|
const listener = () => {
|
|
1498
1532
|
if (filter && !filter()) {
|
|
1499
1533
|
return;
|
|
1500
1534
|
}
|
|
1501
1535
|
this.off(event, listener);
|
|
1502
|
-
|
|
1536
|
+
resolve5();
|
|
1503
1537
|
};
|
|
1504
1538
|
this.on(event, listener);
|
|
1505
1539
|
});
|
|
@@ -1784,7 +1818,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1784
1818
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
1785
1819
|
options.signal?.throwIfAborted();
|
|
1786
1820
|
if (finalDelay > 0) {
|
|
1787
|
-
await new Promise((
|
|
1821
|
+
await new Promise((resolve5, reject) => {
|
|
1788
1822
|
const onAbort = () => {
|
|
1789
1823
|
clearTimeout(timeoutToken);
|
|
1790
1824
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -1792,7 +1826,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1792
1826
|
};
|
|
1793
1827
|
const timeoutToken = setTimeout(() => {
|
|
1794
1828
|
options.signal?.removeEventListener("abort", onAbort);
|
|
1795
|
-
|
|
1829
|
+
resolve5();
|
|
1796
1830
|
}, finalDelay);
|
|
1797
1831
|
if (options.unref) {
|
|
1798
1832
|
timeoutToken.unref?.();
|
|
@@ -1856,12 +1890,6 @@ async function pRetry(input, options = {}) {
|
|
|
1856
1890
|
var import_fs = require("fs");
|
|
1857
1891
|
var path = __toESM(require("path"), 1);
|
|
1858
1892
|
var os = __toESM(require("os"), 1);
|
|
1859
|
-
var EMBEDDING_CAPABLE_PROVIDERS = [
|
|
1860
|
-
"github-copilot",
|
|
1861
|
-
"openai",
|
|
1862
|
-
"google",
|
|
1863
|
-
"ollama"
|
|
1864
|
-
];
|
|
1865
1893
|
function getOpenCodeAuthPath() {
|
|
1866
1894
|
return path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
|
|
1867
1895
|
}
|
|
@@ -1875,21 +1903,34 @@ function loadOpenCodeAuth() {
|
|
|
1875
1903
|
}
|
|
1876
1904
|
return {};
|
|
1877
1905
|
}
|
|
1878
|
-
async function detectEmbeddingProvider(preferredProvider) {
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
if (
|
|
1906
|
+
async function detectEmbeddingProvider(preferredProvider, model) {
|
|
1907
|
+
const credentials = await getProviderCredentials(preferredProvider);
|
|
1908
|
+
if (credentials) {
|
|
1909
|
+
if (!model) {
|
|
1882
1910
|
return {
|
|
1883
1911
|
provider: preferredProvider,
|
|
1884
1912
|
credentials,
|
|
1885
1913
|
modelInfo: getDefaultModelForProvider(preferredProvider)
|
|
1886
1914
|
};
|
|
1887
1915
|
}
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1916
|
+
if (!isValidModel(model, preferredProvider)) {
|
|
1917
|
+
throw new Error(
|
|
1918
|
+
`Model '${model}' is not supported by provider '${preferredProvider}'`
|
|
1919
|
+
);
|
|
1920
|
+
}
|
|
1921
|
+
const providerModels = EMBEDDING_MODELS[preferredProvider];
|
|
1922
|
+
return {
|
|
1923
|
+
provider: preferredProvider,
|
|
1924
|
+
credentials,
|
|
1925
|
+
modelInfo: providerModels[model]
|
|
1926
|
+
};
|
|
1891
1927
|
}
|
|
1892
|
-
|
|
1928
|
+
throw new Error(
|
|
1929
|
+
`Preferred provider '${preferredProvider}' is not configured or authenticated`
|
|
1930
|
+
);
|
|
1931
|
+
}
|
|
1932
|
+
async function tryDetectProvider() {
|
|
1933
|
+
for (const provider of availableProviders) {
|
|
1893
1934
|
const credentials = await getProviderCredentials(provider);
|
|
1894
1935
|
if (credentials) {
|
|
1895
1936
|
return {
|
|
@@ -1900,7 +1941,7 @@ async function detectEmbeddingProvider(preferredProvider) {
|
|
|
1900
1941
|
}
|
|
1901
1942
|
}
|
|
1902
1943
|
throw new Error(
|
|
1903
|
-
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${
|
|
1944
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${availableProviders.join(", ")}.`
|
|
1904
1945
|
);
|
|
1905
1946
|
}
|
|
1906
1947
|
async function getProviderCredentials(provider) {
|
|
@@ -1998,18 +2039,20 @@ function getProviderDisplayName(provider) {
|
|
|
1998
2039
|
}
|
|
1999
2040
|
|
|
2000
2041
|
// src/embeddings/provider.ts
|
|
2001
|
-
function createEmbeddingProvider(
|
|
2002
|
-
switch (
|
|
2042
|
+
function createEmbeddingProvider(configuredProviderInfo) {
|
|
2043
|
+
switch (configuredProviderInfo.provider) {
|
|
2003
2044
|
case "github-copilot":
|
|
2004
|
-
return new GitHubCopilotEmbeddingProvider(credentials, modelInfo);
|
|
2045
|
+
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2005
2046
|
case "openai":
|
|
2006
|
-
return new OpenAIEmbeddingProvider(credentials, modelInfo);
|
|
2047
|
+
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2007
2048
|
case "google":
|
|
2008
|
-
return new GoogleEmbeddingProvider(credentials, modelInfo);
|
|
2049
|
+
return new GoogleEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2009
2050
|
case "ollama":
|
|
2010
|
-
return new OllamaEmbeddingProvider(credentials, modelInfo);
|
|
2011
|
-
default:
|
|
2012
|
-
|
|
2051
|
+
return new OllamaEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2052
|
+
default: {
|
|
2053
|
+
const _exhaustive = configuredProviderInfo;
|
|
2054
|
+
throw new Error(`Unsupported embedding provider: ${_exhaustive.provider}`);
|
|
2055
|
+
}
|
|
2013
2056
|
}
|
|
2014
2057
|
}
|
|
2015
2058
|
var GitHubCopilotEmbeddingProvider = class {
|
|
@@ -2023,8 +2066,15 @@ var GitHubCopilotEmbeddingProvider = class {
|
|
|
2023
2066
|
}
|
|
2024
2067
|
return this.credentials.refreshToken;
|
|
2025
2068
|
}
|
|
2026
|
-
async
|
|
2027
|
-
const result = await this.embedBatch([
|
|
2069
|
+
async embedQuery(query) {
|
|
2070
|
+
const result = await this.embedBatch([query]);
|
|
2071
|
+
return {
|
|
2072
|
+
embedding: result.embeddings[0],
|
|
2073
|
+
tokensUsed: result.totalTokensUsed
|
|
2074
|
+
};
|
|
2075
|
+
}
|
|
2076
|
+
async embedDocument(document) {
|
|
2077
|
+
const result = await this.embedBatch([document]);
|
|
2028
2078
|
return {
|
|
2029
2079
|
embedding: result.embeddings[0],
|
|
2030
2080
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2064,8 +2114,15 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2064
2114
|
this.credentials = credentials;
|
|
2065
2115
|
this.modelInfo = modelInfo;
|
|
2066
2116
|
}
|
|
2067
|
-
async
|
|
2068
|
-
const result = await this.embedBatch([
|
|
2117
|
+
async embedQuery(query) {
|
|
2118
|
+
const result = await this.embedBatch([query]);
|
|
2119
|
+
return {
|
|
2120
|
+
embedding: result.embeddings[0],
|
|
2121
|
+
tokensUsed: result.totalTokensUsed
|
|
2122
|
+
};
|
|
2123
|
+
}
|
|
2124
|
+
async embedDocument(document) {
|
|
2125
|
+
const result = await this.embedBatch([document]);
|
|
2069
2126
|
return {
|
|
2070
2127
|
embedding: result.embeddings[0],
|
|
2071
2128
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2097,33 +2154,61 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2097
2154
|
return this.modelInfo;
|
|
2098
2155
|
}
|
|
2099
2156
|
};
|
|
2100
|
-
var GoogleEmbeddingProvider = class {
|
|
2157
|
+
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
2101
2158
|
constructor(credentials, modelInfo) {
|
|
2102
2159
|
this.credentials = credentials;
|
|
2103
2160
|
this.modelInfo = modelInfo;
|
|
2104
2161
|
}
|
|
2105
|
-
|
|
2106
|
-
|
|
2162
|
+
static BATCH_SIZE = 20;
|
|
2163
|
+
async embedQuery(query) {
|
|
2164
|
+
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2165
|
+
const result = await this.embedWithTaskType([query], taskType);
|
|
2166
|
+
return {
|
|
2167
|
+
embedding: result.embeddings[0],
|
|
2168
|
+
tokensUsed: result.totalTokensUsed
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
async embedDocument(document) {
|
|
2172
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2173
|
+
const result = await this.embedWithTaskType([document], taskType);
|
|
2107
2174
|
return {
|
|
2108
2175
|
embedding: result.embeddings[0],
|
|
2109
2176
|
tokensUsed: result.totalTokensUsed
|
|
2110
2177
|
};
|
|
2111
2178
|
}
|
|
2112
2179
|
async embedBatch(texts) {
|
|
2113
|
-
const
|
|
2114
|
-
|
|
2180
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2181
|
+
return this.embedWithTaskType(texts, taskType);
|
|
2182
|
+
}
|
|
2183
|
+
/**
|
|
2184
|
+
* Embeds texts using the Google embedContent API.
|
|
2185
|
+
* Sends multiple texts as parts in batched requests (up to BATCH_SIZE per call).
|
|
2186
|
+
* When taskType is provided (gemini-embedding-001), includes it in the request
|
|
2187
|
+
* for task-specific embedding optimization.
|
|
2188
|
+
*/
|
|
2189
|
+
async embedWithTaskType(texts, taskType) {
|
|
2190
|
+
const batches = [];
|
|
2191
|
+
for (let i = 0; i < texts.length; i += _GoogleEmbeddingProvider.BATCH_SIZE) {
|
|
2192
|
+
batches.push(texts.slice(i, i + _GoogleEmbeddingProvider.BATCH_SIZE));
|
|
2193
|
+
}
|
|
2194
|
+
const batchResults = await Promise.all(
|
|
2195
|
+
batches.map(async (batch) => {
|
|
2196
|
+
const requests = batch.map((text) => ({
|
|
2197
|
+
model: `models/${this.modelInfo.model}`,
|
|
2198
|
+
content: {
|
|
2199
|
+
parts: [{ text }]
|
|
2200
|
+
},
|
|
2201
|
+
taskType,
|
|
2202
|
+
outputDimensionality: this.modelInfo.dimensions
|
|
2203
|
+
}));
|
|
2115
2204
|
const response = await fetch(
|
|
2116
|
-
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:
|
|
2205
|
+
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:batchEmbedContents?key=${this.credentials.apiKey}`,
|
|
2117
2206
|
{
|
|
2118
2207
|
method: "POST",
|
|
2119
2208
|
headers: {
|
|
2120
2209
|
"Content-Type": "application/json"
|
|
2121
2210
|
},
|
|
2122
|
-
body: JSON.stringify({
|
|
2123
|
-
content: {
|
|
2124
|
-
parts: [{ text }]
|
|
2125
|
-
}
|
|
2126
|
-
})
|
|
2211
|
+
body: JSON.stringify({ requests })
|
|
2127
2212
|
}
|
|
2128
2213
|
);
|
|
2129
2214
|
if (!response.ok) {
|
|
@@ -2132,14 +2217,14 @@ var GoogleEmbeddingProvider = class {
|
|
|
2132
2217
|
}
|
|
2133
2218
|
const data = await response.json();
|
|
2134
2219
|
return {
|
|
2135
|
-
|
|
2136
|
-
tokensUsed: Math.ceil(text.length / 4)
|
|
2220
|
+
embeddings: data.embeddings.map((e) => e.values),
|
|
2221
|
+
tokensUsed: batch.reduce((sum, text) => sum + Math.ceil(text.length / 4), 0)
|
|
2137
2222
|
};
|
|
2138
2223
|
})
|
|
2139
2224
|
);
|
|
2140
2225
|
return {
|
|
2141
|
-
embeddings:
|
|
2142
|
-
totalTokensUsed:
|
|
2226
|
+
embeddings: batchResults.flatMap((r) => r.embeddings),
|
|
2227
|
+
totalTokensUsed: batchResults.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2143
2228
|
};
|
|
2144
2229
|
}
|
|
2145
2230
|
getModelInfo() {
|
|
@@ -2151,29 +2236,44 @@ var OllamaEmbeddingProvider = class {
|
|
|
2151
2236
|
this.credentials = credentials;
|
|
2152
2237
|
this.modelInfo = modelInfo;
|
|
2153
2238
|
}
|
|
2154
|
-
async
|
|
2155
|
-
const
|
|
2156
|
-
method: "POST",
|
|
2157
|
-
headers: {
|
|
2158
|
-
"Content-Type": "application/json"
|
|
2159
|
-
},
|
|
2160
|
-
body: JSON.stringify({
|
|
2161
|
-
model: this.modelInfo.model,
|
|
2162
|
-
prompt: text
|
|
2163
|
-
})
|
|
2164
|
-
});
|
|
2165
|
-
if (!response.ok) {
|
|
2166
|
-
const error = await response.text();
|
|
2167
|
-
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2168
|
-
}
|
|
2169
|
-
const data = await response.json();
|
|
2239
|
+
async embedQuery(query) {
|
|
2240
|
+
const result = await this.embedBatch([query]);
|
|
2170
2241
|
return {
|
|
2171
|
-
embedding:
|
|
2172
|
-
tokensUsed:
|
|
2242
|
+
embedding: result.embeddings[0],
|
|
2243
|
+
tokensUsed: result.totalTokensUsed
|
|
2244
|
+
};
|
|
2245
|
+
}
|
|
2246
|
+
async embedDocument(document) {
|
|
2247
|
+
const result = await this.embedBatch([document]);
|
|
2248
|
+
return {
|
|
2249
|
+
embedding: result.embeddings[0],
|
|
2250
|
+
tokensUsed: result.totalTokensUsed
|
|
2173
2251
|
};
|
|
2174
2252
|
}
|
|
2175
2253
|
async embedBatch(texts) {
|
|
2176
|
-
const results = await Promise.all(
|
|
2254
|
+
const results = await Promise.all(
|
|
2255
|
+
texts.map(async (text) => {
|
|
2256
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2257
|
+
method: "POST",
|
|
2258
|
+
headers: {
|
|
2259
|
+
"Content-Type": "application/json"
|
|
2260
|
+
},
|
|
2261
|
+
body: JSON.stringify({
|
|
2262
|
+
model: this.modelInfo.model,
|
|
2263
|
+
prompt: text
|
|
2264
|
+
})
|
|
2265
|
+
});
|
|
2266
|
+
if (!response.ok) {
|
|
2267
|
+
const error = await response.text();
|
|
2268
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2269
|
+
}
|
|
2270
|
+
const data = await response.json();
|
|
2271
|
+
return {
|
|
2272
|
+
embedding: data.embedding,
|
|
2273
|
+
tokensUsed: Math.ceil(text.length / 4)
|
|
2274
|
+
};
|
|
2275
|
+
})
|
|
2276
|
+
);
|
|
2177
2277
|
return {
|
|
2178
2278
|
embeddings: results.map((r) => r.embedding),
|
|
2179
2279
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
@@ -3118,11 +3218,40 @@ var Database = class {
|
|
|
3118
3218
|
var import_fs3 = require("fs");
|
|
3119
3219
|
var path4 = __toESM(require("path"), 1);
|
|
3120
3220
|
var import_child_process = require("child_process");
|
|
3221
|
+
function resolveGitDir(repoRoot) {
|
|
3222
|
+
const gitPath = path4.join(repoRoot, ".git");
|
|
3223
|
+
if (!(0, import_fs3.existsSync)(gitPath)) {
|
|
3224
|
+
return null;
|
|
3225
|
+
}
|
|
3226
|
+
try {
|
|
3227
|
+
const stat4 = (0, import_fs3.statSync)(gitPath);
|
|
3228
|
+
if (stat4.isDirectory()) {
|
|
3229
|
+
return gitPath;
|
|
3230
|
+
}
|
|
3231
|
+
if (stat4.isFile()) {
|
|
3232
|
+
const content = (0, import_fs3.readFileSync)(gitPath, "utf-8").trim();
|
|
3233
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
3234
|
+
if (match) {
|
|
3235
|
+
const gitdir = match[1];
|
|
3236
|
+
const resolvedPath = path4.isAbsolute(gitdir) ? gitdir : path4.resolve(repoRoot, gitdir);
|
|
3237
|
+
if ((0, import_fs3.existsSync)(resolvedPath)) {
|
|
3238
|
+
return resolvedPath;
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
}
|
|
3242
|
+
} catch {
|
|
3243
|
+
}
|
|
3244
|
+
return null;
|
|
3245
|
+
}
|
|
3121
3246
|
function isGitRepo(dir) {
|
|
3122
|
-
return (
|
|
3247
|
+
return resolveGitDir(dir) !== null;
|
|
3123
3248
|
}
|
|
3124
3249
|
function getCurrentBranch(repoRoot) {
|
|
3125
|
-
const
|
|
3250
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3251
|
+
if (!gitDir) {
|
|
3252
|
+
return null;
|
|
3253
|
+
}
|
|
3254
|
+
const headPath = path4.join(gitDir, "HEAD");
|
|
3126
3255
|
if (!(0, import_fs3.existsSync)(headPath)) {
|
|
3127
3256
|
return null;
|
|
3128
3257
|
}
|
|
@@ -3141,20 +3270,23 @@ function getCurrentBranch(repoRoot) {
|
|
|
3141
3270
|
}
|
|
3142
3271
|
}
|
|
3143
3272
|
function getBaseBranch(repoRoot) {
|
|
3273
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3144
3274
|
const candidates = ["main", "master", "develop", "trunk"];
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3275
|
+
if (gitDir) {
|
|
3276
|
+
for (const candidate of candidates) {
|
|
3277
|
+
const refPath = path4.join(gitDir, "refs", "heads", candidate);
|
|
3278
|
+
if ((0, import_fs3.existsSync)(refPath)) {
|
|
3279
|
+
return candidate;
|
|
3280
|
+
}
|
|
3281
|
+
const packedRefsPath = path4.join(gitDir, "packed-refs");
|
|
3282
|
+
if ((0, import_fs3.existsSync)(packedRefsPath)) {
|
|
3283
|
+
try {
|
|
3284
|
+
const content = (0, import_fs3.readFileSync)(packedRefsPath, "utf-8");
|
|
3285
|
+
if (content.includes(`refs/heads/${candidate}`)) {
|
|
3286
|
+
return candidate;
|
|
3287
|
+
}
|
|
3288
|
+
} catch {
|
|
3156
3289
|
}
|
|
3157
|
-
} catch {
|
|
3158
3290
|
}
|
|
3159
3291
|
}
|
|
3160
3292
|
}
|
|
@@ -3179,6 +3311,10 @@ function getBranchOrDefault(repoRoot) {
|
|
|
3179
3311
|
return getCurrentBranch(repoRoot) ?? "default";
|
|
3180
3312
|
}
|
|
3181
3313
|
function getHeadPath(repoRoot) {
|
|
3314
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3315
|
+
if (gitDir) {
|
|
3316
|
+
return path4.join(gitDir, "HEAD");
|
|
3317
|
+
}
|
|
3182
3318
|
return path4.join(repoRoot, ".git", "HEAD");
|
|
3183
3319
|
}
|
|
3184
3320
|
|
|
@@ -3206,6 +3342,7 @@ function isRateLimitError(error) {
|
|
|
3206
3342
|
const message = getErrorMessage(error);
|
|
3207
3343
|
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
3208
3344
|
}
|
|
3345
|
+
var INDEX_METADATA_VERSION = "1";
|
|
3209
3346
|
var Indexer = class {
|
|
3210
3347
|
config;
|
|
3211
3348
|
projectRoot;
|
|
@@ -3214,7 +3351,7 @@ var Indexer = class {
|
|
|
3214
3351
|
invertedIndex = null;
|
|
3215
3352
|
database = null;
|
|
3216
3353
|
provider = null;
|
|
3217
|
-
|
|
3354
|
+
configuredProviderInfo = null;
|
|
3218
3355
|
fileHashCache = /* @__PURE__ */ new Map();
|
|
3219
3356
|
fileHashCachePath = "";
|
|
3220
3357
|
failedBatchesPath = "";
|
|
@@ -3225,12 +3362,15 @@ var Indexer = class {
|
|
|
3225
3362
|
maxQueryCacheSize = 100;
|
|
3226
3363
|
queryCacheTtlMs = 5 * 60 * 1e3;
|
|
3227
3364
|
querySimilarityThreshold = 0.85;
|
|
3365
|
+
indexCompatibility = null;
|
|
3366
|
+
indexingLockPath = "";
|
|
3228
3367
|
constructor(projectRoot, config) {
|
|
3229
3368
|
this.projectRoot = projectRoot;
|
|
3230
3369
|
this.config = config;
|
|
3231
3370
|
this.indexPath = this.getIndexPath();
|
|
3232
3371
|
this.fileHashCachePath = path5.join(this.indexPath, "file-hashes.json");
|
|
3233
3372
|
this.failedBatchesPath = path5.join(this.indexPath, "failed-batches.json");
|
|
3373
|
+
this.indexingLockPath = path5.join(this.indexPath, "indexing.lock");
|
|
3234
3374
|
this.logger = initializeLogger(config.debug);
|
|
3235
3375
|
}
|
|
3236
3376
|
getIndexPath() {
|
|
@@ -3256,7 +3396,36 @@ var Indexer = class {
|
|
|
3256
3396
|
for (const [k, v] of this.fileHashCache) {
|
|
3257
3397
|
obj[k] = v;
|
|
3258
3398
|
}
|
|
3259
|
-
|
|
3399
|
+
this.atomicWriteSync(this.fileHashCachePath, JSON.stringify(obj));
|
|
3400
|
+
}
|
|
3401
|
+
atomicWriteSync(targetPath, data) {
|
|
3402
|
+
const tempPath = `${targetPath}.tmp`;
|
|
3403
|
+
(0, import_fs4.writeFileSync)(tempPath, data);
|
|
3404
|
+
(0, import_fs4.renameSync)(tempPath, targetPath);
|
|
3405
|
+
}
|
|
3406
|
+
checkForInterruptedIndexing() {
|
|
3407
|
+
return (0, import_fs4.existsSync)(this.indexingLockPath);
|
|
3408
|
+
}
|
|
3409
|
+
acquireIndexingLock() {
|
|
3410
|
+
const lockData = {
|
|
3411
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3412
|
+
pid: process.pid
|
|
3413
|
+
};
|
|
3414
|
+
(0, import_fs4.writeFileSync)(this.indexingLockPath, JSON.stringify(lockData));
|
|
3415
|
+
}
|
|
3416
|
+
releaseIndexingLock() {
|
|
3417
|
+
if ((0, import_fs4.existsSync)(this.indexingLockPath)) {
|
|
3418
|
+
(0, import_fs4.unlinkSync)(this.indexingLockPath);
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
async recoverFromInterruptedIndexing() {
|
|
3422
|
+
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
3423
|
+
if ((0, import_fs4.existsSync)(this.fileHashCachePath)) {
|
|
3424
|
+
(0, import_fs4.unlinkSync)(this.fileHashCachePath);
|
|
3425
|
+
}
|
|
3426
|
+
await this.healthCheck();
|
|
3427
|
+
this.releaseIndexingLock();
|
|
3428
|
+
this.logger.info("Recovery complete, next index will re-process all files");
|
|
3260
3429
|
}
|
|
3261
3430
|
loadFailedBatches() {
|
|
3262
3431
|
try {
|
|
@@ -3304,23 +3473,27 @@ var Indexer = class {
|
|
|
3304
3473
|
}
|
|
3305
3474
|
}
|
|
3306
3475
|
async initialize() {
|
|
3307
|
-
|
|
3308
|
-
|
|
3476
|
+
if (this.config.embeddingProvider === "auto") {
|
|
3477
|
+
this.configuredProviderInfo = await tryDetectProvider();
|
|
3478
|
+
} else {
|
|
3479
|
+
this.configuredProviderInfo = await detectEmbeddingProvider(this.config.embeddingProvider, this.config.embeddingModel);
|
|
3480
|
+
}
|
|
3481
|
+
if (!this.configuredProviderInfo) {
|
|
3309
3482
|
throw new Error(
|
|
3310
3483
|
"No embedding provider available. Configure GitHub, OpenAI, Google, or Ollama."
|
|
3311
3484
|
);
|
|
3312
3485
|
}
|
|
3313
3486
|
this.logger.info("Initializing indexer", {
|
|
3314
|
-
provider: this.
|
|
3315
|
-
model: this.
|
|
3487
|
+
provider: this.configuredProviderInfo.provider,
|
|
3488
|
+
model: this.configuredProviderInfo.modelInfo.model,
|
|
3316
3489
|
scope: this.config.scope
|
|
3317
3490
|
});
|
|
3318
|
-
this.provider = createEmbeddingProvider(
|
|
3319
|
-
this.detectedProvider.credentials,
|
|
3320
|
-
this.detectedProvider.modelInfo
|
|
3321
|
-
);
|
|
3491
|
+
this.provider = createEmbeddingProvider(this.configuredProviderInfo);
|
|
3322
3492
|
await import_fs4.promises.mkdir(this.indexPath, { recursive: true });
|
|
3323
|
-
|
|
3493
|
+
if (this.checkForInterruptedIndexing()) {
|
|
3494
|
+
await this.recoverFromInterruptedIndexing();
|
|
3495
|
+
}
|
|
3496
|
+
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
3324
3497
|
const storePath = path5.join(this.indexPath, "vectors");
|
|
3325
3498
|
this.store = new VectorStore(storePath, dimensions);
|
|
3326
3499
|
const indexFilePath = path5.join(this.indexPath, "vectors.usearch");
|
|
@@ -3343,6 +3516,14 @@ var Indexer = class {
|
|
|
3343
3516
|
if (dbIsNew && this.store.count() > 0) {
|
|
3344
3517
|
this.migrateFromLegacyIndex();
|
|
3345
3518
|
}
|
|
3519
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3520
|
+
if (!this.indexCompatibility.compatible) {
|
|
3521
|
+
this.logger.warn("Index compatibility issue detected", {
|
|
3522
|
+
reason: this.indexCompatibility.reason,
|
|
3523
|
+
storedMetadata: this.indexCompatibility.storedMetadata,
|
|
3524
|
+
configuredProviderInfo: this.configuredProviderInfo
|
|
3525
|
+
});
|
|
3526
|
+
}
|
|
3346
3527
|
if (isGitRepo(this.projectRoot)) {
|
|
3347
3528
|
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
3348
3529
|
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
@@ -3413,30 +3594,106 @@ var Indexer = class {
|
|
|
3413
3594
|
}
|
|
3414
3595
|
this.database.addChunksToBranchBatch(this.currentBranch || "default", chunkIds);
|
|
3415
3596
|
}
|
|
3597
|
+
loadIndexMetadata() {
|
|
3598
|
+
if (!this.database) return null;
|
|
3599
|
+
const version = this.database.getMetadata("index.version");
|
|
3600
|
+
if (!version) return null;
|
|
3601
|
+
return {
|
|
3602
|
+
indexVersion: version,
|
|
3603
|
+
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
3604
|
+
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
3605
|
+
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
3606
|
+
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
3607
|
+
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
3608
|
+
};
|
|
3609
|
+
}
|
|
3610
|
+
saveIndexMetadata(provider) {
|
|
3611
|
+
if (!this.database) return;
|
|
3612
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3613
|
+
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
3614
|
+
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
3615
|
+
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
3616
|
+
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
3617
|
+
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
3618
|
+
this.database.setMetadata("index.updatedAt", now);
|
|
3619
|
+
if (!existingCreatedAt) {
|
|
3620
|
+
this.database.setMetadata("index.createdAt", now);
|
|
3621
|
+
}
|
|
3622
|
+
}
|
|
3623
|
+
validateIndexCompatibility(provider) {
|
|
3624
|
+
const storedMetadata = this.loadIndexMetadata();
|
|
3625
|
+
if (!storedMetadata) {
|
|
3626
|
+
return { compatible: true };
|
|
3627
|
+
}
|
|
3628
|
+
const currentProvider = provider.provider;
|
|
3629
|
+
const currentModel = provider.modelInfo.model;
|
|
3630
|
+
const currentDimensions = provider.modelInfo.dimensions;
|
|
3631
|
+
if (storedMetadata.embeddingDimensions !== currentDimensions) {
|
|
3632
|
+
return {
|
|
3633
|
+
compatible: false,
|
|
3634
|
+
code: "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
|
|
3635
|
+
reason: `Dimension mismatch: index has ${storedMetadata.embeddingDimensions}D vectors (${storedMetadata.embeddingProvider}/${storedMetadata.embeddingModel}), but current provider uses ${currentDimensions}D (${currentProvider}/${currentModel}). Run index_codebase with force=true to rebuild.`,
|
|
3636
|
+
storedMetadata
|
|
3637
|
+
};
|
|
3638
|
+
}
|
|
3639
|
+
if (storedMetadata.embeddingModel !== currentModel) {
|
|
3640
|
+
return {
|
|
3641
|
+
compatible: false,
|
|
3642
|
+
code: "MODEL_MISMATCH" /* MODEL_MISMATCH */,
|
|
3643
|
+
reason: `Model mismatch: index was built with "${storedMetadata.embeddingModel}", but current model is "${currentModel}". Embeddings are incompatible. Run index_codebase with force=true to rebuild.`,
|
|
3644
|
+
storedMetadata
|
|
3645
|
+
};
|
|
3646
|
+
}
|
|
3647
|
+
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
3648
|
+
this.logger.warn("Provider changed", {
|
|
3649
|
+
storedProvider: storedMetadata.embeddingProvider,
|
|
3650
|
+
currentProvider
|
|
3651
|
+
});
|
|
3652
|
+
}
|
|
3653
|
+
return {
|
|
3654
|
+
compatible: true,
|
|
3655
|
+
storedMetadata
|
|
3656
|
+
};
|
|
3657
|
+
}
|
|
3658
|
+
checkCompatibility() {
|
|
3659
|
+
if (!this.indexCompatibility) {
|
|
3660
|
+
if (!this.configuredProviderInfo) {
|
|
3661
|
+
throw new Error("No embedding provider info, you must initialize the indexer first.");
|
|
3662
|
+
}
|
|
3663
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3664
|
+
}
|
|
3665
|
+
return this.indexCompatibility;
|
|
3666
|
+
}
|
|
3416
3667
|
async ensureInitialized() {
|
|
3417
|
-
if (!this.store || !this.provider || !this.invertedIndex || !this.
|
|
3668
|
+
if (!this.store || !this.provider || !this.invertedIndex || !this.configuredProviderInfo || !this.database) {
|
|
3418
3669
|
await this.initialize();
|
|
3419
3670
|
}
|
|
3420
3671
|
return {
|
|
3421
3672
|
store: this.store,
|
|
3422
3673
|
provider: this.provider,
|
|
3423
3674
|
invertedIndex: this.invertedIndex,
|
|
3424
|
-
|
|
3675
|
+
configuredProviderInfo: this.configuredProviderInfo,
|
|
3425
3676
|
database: this.database
|
|
3426
3677
|
};
|
|
3427
3678
|
}
|
|
3428
3679
|
async estimateCost() {
|
|
3429
|
-
const {
|
|
3680
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
3430
3681
|
const { files } = await collectFiles(
|
|
3431
3682
|
this.projectRoot,
|
|
3432
3683
|
this.config.include,
|
|
3433
3684
|
this.config.exclude,
|
|
3434
3685
|
this.config.indexing.maxFileSize
|
|
3435
3686
|
);
|
|
3436
|
-
return createCostEstimate(files,
|
|
3687
|
+
return createCostEstimate(files, configuredProviderInfo);
|
|
3437
3688
|
}
|
|
3438
3689
|
async index(onProgress) {
|
|
3439
|
-
const { store, provider, invertedIndex, database,
|
|
3690
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
3691
|
+
if (!this.indexCompatibility?.compatible) {
|
|
3692
|
+
throw new Error(
|
|
3693
|
+
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
3694
|
+
);
|
|
3695
|
+
}
|
|
3696
|
+
this.acquireIndexingLock();
|
|
3440
3697
|
this.logger.recordIndexingStart();
|
|
3441
3698
|
this.logger.info("Starting indexing", { projectRoot: this.projectRoot });
|
|
3442
3699
|
const startTime = Date.now();
|
|
@@ -3543,7 +3800,7 @@ var Indexer = class {
|
|
|
3543
3800
|
const id = generateChunkId(parsed.path, chunk);
|
|
3544
3801
|
const contentHash = generateChunkHash(chunk);
|
|
3545
3802
|
currentChunkIds.add(id);
|
|
3546
|
-
|
|
3803
|
+
chunkDataBatch.push({
|
|
3547
3804
|
chunkId: id,
|
|
3548
3805
|
contentHash,
|
|
3549
3806
|
filePath: parsed.path,
|
|
@@ -3552,8 +3809,7 @@ var Indexer = class {
|
|
|
3552
3809
|
nodeType: chunk.chunkType,
|
|
3553
3810
|
name: chunk.name,
|
|
3554
3811
|
language: chunk.language
|
|
3555
|
-
};
|
|
3556
|
-
chunkDataBatch.push(chunkData);
|
|
3812
|
+
});
|
|
3557
3813
|
if (existingChunks.get(id) === contentHash) {
|
|
3558
3814
|
fileChunkCount++;
|
|
3559
3815
|
continue;
|
|
@@ -3606,6 +3862,7 @@ var Indexer = class {
|
|
|
3606
3862
|
chunksProcessed: 0,
|
|
3607
3863
|
totalChunks: 0
|
|
3608
3864
|
});
|
|
3865
|
+
this.releaseIndexingLock();
|
|
3609
3866
|
return stats;
|
|
3610
3867
|
}
|
|
3611
3868
|
if (pendingChunks.length === 0) {
|
|
@@ -3623,6 +3880,7 @@ var Indexer = class {
|
|
|
3623
3880
|
chunksProcessed: 0,
|
|
3624
3881
|
totalChunks: 0
|
|
3625
3882
|
});
|
|
3883
|
+
this.releaseIndexingLock();
|
|
3626
3884
|
return stats;
|
|
3627
3885
|
}
|
|
3628
3886
|
onProgress?.({
|
|
@@ -3651,7 +3909,7 @@ var Indexer = class {
|
|
|
3651
3909
|
stats.indexedChunks++;
|
|
3652
3910
|
}
|
|
3653
3911
|
}
|
|
3654
|
-
const providerRateLimits = this.getProviderRateLimits(
|
|
3912
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
3655
3913
|
const queue = new PQueue({
|
|
3656
3914
|
concurrency: providerRateLimits.concurrency,
|
|
3657
3915
|
interval: providerRateLimits.intervalMs,
|
|
@@ -3662,7 +3920,7 @@ var Indexer = class {
|
|
|
3662
3920
|
for (const batch of dynamicBatches) {
|
|
3663
3921
|
queue.add(async () => {
|
|
3664
3922
|
if (rateLimitBackoffMs > 0) {
|
|
3665
|
-
await new Promise((
|
|
3923
|
+
await new Promise((resolve5) => setTimeout(resolve5, rateLimitBackoffMs));
|
|
3666
3924
|
}
|
|
3667
3925
|
try {
|
|
3668
3926
|
const result = await pRetry(
|
|
@@ -3706,7 +3964,7 @@ var Indexer = class {
|
|
|
3706
3964
|
contentHash: chunk.contentHash,
|
|
3707
3965
|
embedding: float32ArrayToBuffer(result.embeddings[i]),
|
|
3708
3966
|
chunkText: chunk.text,
|
|
3709
|
-
model:
|
|
3967
|
+
model: configuredProviderInfo.modelInfo.model
|
|
3710
3968
|
}));
|
|
3711
3969
|
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
3712
3970
|
for (const chunk of batch) {
|
|
@@ -3757,6 +4015,8 @@ var Indexer = class {
|
|
|
3757
4015
|
await this.maybeRunOrphanGc();
|
|
3758
4016
|
}
|
|
3759
4017
|
stats.durationMs = Date.now() - startTime;
|
|
4018
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
4019
|
+
this.indexCompatibility = { compatible: true };
|
|
3760
4020
|
this.logger.recordIndexingEnd();
|
|
3761
4021
|
this.logger.info("Indexing complete", {
|
|
3762
4022
|
files: stats.totalFiles,
|
|
@@ -3777,6 +4037,7 @@ var Indexer = class {
|
|
|
3777
4037
|
chunksProcessed: stats.indexedChunks,
|
|
3778
4038
|
totalChunks: pendingChunks.length
|
|
3779
4039
|
});
|
|
4040
|
+
this.releaseIndexingLock();
|
|
3780
4041
|
return stats;
|
|
3781
4042
|
}
|
|
3782
4043
|
async getQueryEmbedding(query, provider) {
|
|
@@ -3799,7 +4060,7 @@ var Indexer = class {
|
|
|
3799
4060
|
}
|
|
3800
4061
|
this.logger.cache("debug", "Query embedding cache miss", { query: query.slice(0, 50) });
|
|
3801
4062
|
this.logger.recordQueryCacheMiss();
|
|
3802
|
-
const { embedding, tokensUsed } = await provider.
|
|
4063
|
+
const { embedding, tokensUsed } = await provider.embedQuery(query);
|
|
3803
4064
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
3804
4065
|
if (this.queryEmbeddingCache.size >= this.maxQueryCacheSize) {
|
|
3805
4066
|
const oldestKey = this.queryEmbeddingCache.keys().next().value;
|
|
@@ -3842,8 +4103,14 @@ var Indexer = class {
|
|
|
3842
4103
|
return intersection / union;
|
|
3843
4104
|
}
|
|
3844
4105
|
async search(query, limit, options) {
|
|
3845
|
-
const searchStartTime = import_perf_hooks.performance.now();
|
|
3846
4106
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4107
|
+
const compatibility = this.checkCompatibility();
|
|
4108
|
+
if (!compatibility.compatible) {
|
|
4109
|
+
throw new Error(
|
|
4110
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} A possible solution is to run index_codebase with force=true to rebuild the index.`
|
|
4111
|
+
);
|
|
4112
|
+
}
|
|
4113
|
+
const searchStartTime = import_perf_hooks.performance.now();
|
|
3847
4114
|
if (store.count() === 0) {
|
|
3848
4115
|
this.logger.search("debug", "Search on empty index", { query });
|
|
3849
4116
|
return [];
|
|
@@ -3985,15 +4252,16 @@ var Indexer = class {
|
|
|
3985
4252
|
return results.slice(0, limit);
|
|
3986
4253
|
}
|
|
3987
4254
|
async getStatus() {
|
|
3988
|
-
const { store,
|
|
4255
|
+
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
3989
4256
|
return {
|
|
3990
4257
|
indexed: store.count() > 0,
|
|
3991
4258
|
vectorCount: store.count(),
|
|
3992
|
-
provider:
|
|
3993
|
-
model:
|
|
4259
|
+
provider: configuredProviderInfo.provider,
|
|
4260
|
+
model: configuredProviderInfo.modelInfo.model,
|
|
3994
4261
|
indexPath: this.indexPath,
|
|
3995
4262
|
currentBranch: this.currentBranch,
|
|
3996
|
-
baseBranch: this.baseBranch
|
|
4263
|
+
baseBranch: this.baseBranch,
|
|
4264
|
+
compatibility: this.indexCompatibility
|
|
3997
4265
|
};
|
|
3998
4266
|
}
|
|
3999
4267
|
async clearIndex() {
|
|
@@ -4005,6 +4273,13 @@ var Indexer = class {
|
|
|
4005
4273
|
this.fileHashCache.clear();
|
|
4006
4274
|
this.saveFileHashCache();
|
|
4007
4275
|
database.clearBranch(this.currentBranch);
|
|
4276
|
+
database.deleteMetadata("index.version");
|
|
4277
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
4278
|
+
database.deleteMetadata("index.embeddingModel");
|
|
4279
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
4280
|
+
database.deleteMetadata("index.createdAt");
|
|
4281
|
+
database.deleteMetadata("index.updatedAt");
|
|
4282
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
4008
4283
|
}
|
|
4009
4284
|
async healthCheck() {
|
|
4010
4285
|
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
@@ -4118,26 +4393,31 @@ var Indexer = class {
|
|
|
4118
4393
|
getLogger() {
|
|
4119
4394
|
return this.logger;
|
|
4120
4395
|
}
|
|
4121
|
-
async findSimilar(code, limit, options) {
|
|
4122
|
-
const searchStartTime = import_perf_hooks.performance.now();
|
|
4396
|
+
async findSimilar(code, limit = this.config.search.maxResults, options) {
|
|
4123
4397
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4398
|
+
const compatibility = this.checkCompatibility();
|
|
4399
|
+
if (!compatibility.compatible) {
|
|
4400
|
+
throw new Error(
|
|
4401
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} Run index_codebase with force=true to rebuild the index.`
|
|
4402
|
+
);
|
|
4403
|
+
}
|
|
4404
|
+
const searchStartTime = import_perf_hooks.performance.now();
|
|
4124
4405
|
if (store.count() === 0) {
|
|
4125
4406
|
this.logger.search("debug", "Find similar on empty index");
|
|
4126
4407
|
return [];
|
|
4127
4408
|
}
|
|
4128
|
-
const maxResults = limit ?? this.config.search.maxResults;
|
|
4129
4409
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
4130
4410
|
this.logger.search("debug", "Starting find similar", {
|
|
4131
4411
|
codeLength: code.length,
|
|
4132
|
-
|
|
4412
|
+
limit,
|
|
4133
4413
|
filterByBranch
|
|
4134
4414
|
});
|
|
4135
4415
|
const embeddingStartTime = import_perf_hooks.performance.now();
|
|
4136
|
-
const { embedding, tokensUsed } = await provider.
|
|
4416
|
+
const { embedding, tokensUsed } = await provider.embedDocument(code);
|
|
4137
4417
|
const embeddingMs = import_perf_hooks.performance.now() - embeddingStartTime;
|
|
4138
4418
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
4139
4419
|
const vectorStartTime = import_perf_hooks.performance.now();
|
|
4140
|
-
const semanticResults = store.search(embedding,
|
|
4420
|
+
const semanticResults = store.search(embedding, limit * 2);
|
|
4141
4421
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
4142
4422
|
let branchChunkIds = null;
|
|
4143
4423
|
if (filterByBranch && this.currentBranch !== "default") {
|
|
@@ -4161,7 +4441,7 @@ var Indexer = class {
|
|
|
4161
4441
|
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
4162
4442
|
}
|
|
4163
4443
|
return true;
|
|
4164
|
-
}).slice(0,
|
|
4444
|
+
}).slice(0, limit);
|
|
4165
4445
|
const totalSearchMs = import_perf_hooks.performance.now() - searchStartTime;
|
|
4166
4446
|
this.logger.recordSearch(totalSearchMs, {
|
|
4167
4447
|
embeddingMs,
|
|
@@ -5076,7 +5356,7 @@ var NodeFsHandler = class {
|
|
|
5076
5356
|
this._addToNodeFs(path9, initialAdd, wh, depth + 1);
|
|
5077
5357
|
}
|
|
5078
5358
|
}).on(EV.ERROR, this._boundHandleError);
|
|
5079
|
-
return new Promise((
|
|
5359
|
+
return new Promise((resolve5, reject) => {
|
|
5080
5360
|
if (!stream)
|
|
5081
5361
|
return reject();
|
|
5082
5362
|
stream.once(STR_END, () => {
|
|
@@ -5085,7 +5365,7 @@ var NodeFsHandler = class {
|
|
|
5085
5365
|
return;
|
|
5086
5366
|
}
|
|
5087
5367
|
const wasThrottled = throttler ? throttler.clear() : false;
|
|
5088
|
-
|
|
5368
|
+
resolve5(void 0);
|
|
5089
5369
|
previous.getChildren().filter((item) => {
|
|
5090
5370
|
return item !== directory && !current.has(item);
|
|
5091
5371
|
}).forEach((item) => {
|
|
@@ -6132,7 +6412,8 @@ function createWatcherWithIndexer(indexer, projectRoot, config) {
|
|
|
6132
6412
|
|
|
6133
6413
|
// src/tools/index.ts
|
|
6134
6414
|
var import_plugin = require("@opencode-ai/plugin");
|
|
6135
|
-
|
|
6415
|
+
|
|
6416
|
+
// src/tools/utils.ts
|
|
6136
6417
|
var MAX_CONTENT_LINES = 30;
|
|
6137
6418
|
function truncateContent(content) {
|
|
6138
6419
|
const lines = content.split("\n");
|
|
@@ -6140,6 +6421,167 @@ function truncateContent(content) {
|
|
|
6140
6421
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
6141
6422
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
6142
6423
|
}
|
|
6424
|
+
function formatIndexStats(stats, verbose = false) {
|
|
6425
|
+
const lines = [];
|
|
6426
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
6427
|
+
lines.push(`Indexed. ${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
6428
|
+
} else if (stats.indexedChunks === 0) {
|
|
6429
|
+
lines.push(`Indexed. ${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
6430
|
+
} else {
|
|
6431
|
+
let main = `Indexed. ${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
6432
|
+
if (stats.existingChunks > 0) {
|
|
6433
|
+
main += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
6434
|
+
}
|
|
6435
|
+
lines.push(main);
|
|
6436
|
+
if (stats.removedChunks > 0) {
|
|
6437
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
6438
|
+
}
|
|
6439
|
+
if (stats.failedChunks > 0) {
|
|
6440
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
6441
|
+
}
|
|
6442
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
6443
|
+
}
|
|
6444
|
+
if (verbose) {
|
|
6445
|
+
if (stats.skippedFiles.length > 0) {
|
|
6446
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6447
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6448
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6449
|
+
lines.push("");
|
|
6450
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6451
|
+
if (tooLarge.length > 0) {
|
|
6452
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6453
|
+
}
|
|
6454
|
+
if (excluded.length > 0) {
|
|
6455
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6456
|
+
}
|
|
6457
|
+
if (gitignored.length > 0) {
|
|
6458
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6459
|
+
}
|
|
6460
|
+
}
|
|
6461
|
+
if (stats.parseFailures.length > 0) {
|
|
6462
|
+
lines.push("");
|
|
6463
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6464
|
+
}
|
|
6465
|
+
}
|
|
6466
|
+
return lines.join("\n");
|
|
6467
|
+
}
|
|
6468
|
+
function formatStatus(status) {
|
|
6469
|
+
if (!status.indexed) {
|
|
6470
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6471
|
+
}
|
|
6472
|
+
const lines = [
|
|
6473
|
+
`Index status:`,
|
|
6474
|
+
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6475
|
+
` Provider: ${status.provider}`,
|
|
6476
|
+
` Model: ${status.model}`,
|
|
6477
|
+
` Location: ${status.indexPath}`
|
|
6478
|
+
];
|
|
6479
|
+
if (status.currentBranch !== "default") {
|
|
6480
|
+
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6481
|
+
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6482
|
+
}
|
|
6483
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
6484
|
+
lines.push("");
|
|
6485
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
6486
|
+
if (status.compatibility.storedMetadata) {
|
|
6487
|
+
const stored = status.compatibility.storedMetadata;
|
|
6488
|
+
lines.push(` Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
6489
|
+
lines.push(` Current config: ${status.provider}/${status.model}`);
|
|
6490
|
+
}
|
|
6491
|
+
} else if (!status.compatibility) {
|
|
6492
|
+
lines.push(` Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
6493
|
+
} else {
|
|
6494
|
+
lines.push(` Compatibility: Index is compatible with the current provider and model.`);
|
|
6495
|
+
}
|
|
6496
|
+
return lines.join("\n");
|
|
6497
|
+
}
|
|
6498
|
+
function formatProgressTitle(progress) {
|
|
6499
|
+
switch (progress.phase) {
|
|
6500
|
+
case "scanning":
|
|
6501
|
+
return "Scanning files...";
|
|
6502
|
+
case "parsing":
|
|
6503
|
+
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6504
|
+
case "embedding":
|
|
6505
|
+
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6506
|
+
case "storing":
|
|
6507
|
+
return "Storing index...";
|
|
6508
|
+
case "complete":
|
|
6509
|
+
return "Indexing complete";
|
|
6510
|
+
default:
|
|
6511
|
+
return "Indexing...";
|
|
6512
|
+
}
|
|
6513
|
+
}
|
|
6514
|
+
function calculatePercentage(progress) {
|
|
6515
|
+
if (progress.phase === "scanning") return 0;
|
|
6516
|
+
if (progress.phase === "complete") return 100;
|
|
6517
|
+
if (progress.phase === "parsing") {
|
|
6518
|
+
if (progress.totalFiles === 0) return 5;
|
|
6519
|
+
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6520
|
+
}
|
|
6521
|
+
if (progress.phase === "embedding") {
|
|
6522
|
+
if (progress.totalChunks === 0) return 20;
|
|
6523
|
+
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6524
|
+
}
|
|
6525
|
+
if (progress.phase === "storing") return 95;
|
|
6526
|
+
return 0;
|
|
6527
|
+
}
|
|
6528
|
+
function formatCodebasePeek(results, query) {
|
|
6529
|
+
if (results.length === 0) {
|
|
6530
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6531
|
+
}
|
|
6532
|
+
const formatted = results.map((r, idx) => {
|
|
6533
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6534
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6535
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6536
|
+
});
|
|
6537
|
+
return `Found ${results.length} locations for "${query}":
|
|
6538
|
+
|
|
6539
|
+
${formatted.join("\n")}
|
|
6540
|
+
|
|
6541
|
+
Use Read tool to examine specific files.`;
|
|
6542
|
+
}
|
|
6543
|
+
function formatHealthCheck(result) {
|
|
6544
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0) {
|
|
6545
|
+
return "Index is healthy. No stale entries found.";
|
|
6546
|
+
}
|
|
6547
|
+
const lines = [`Health check complete:`];
|
|
6548
|
+
if (result.removed > 0) {
|
|
6549
|
+
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6550
|
+
}
|
|
6551
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
6552
|
+
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6553
|
+
}
|
|
6554
|
+
if (result.gcOrphanChunks > 0) {
|
|
6555
|
+
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6556
|
+
}
|
|
6557
|
+
if (result.filePaths.length > 0) {
|
|
6558
|
+
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6559
|
+
}
|
|
6560
|
+
return lines.join("\n");
|
|
6561
|
+
}
|
|
6562
|
+
function formatLogs(logs) {
|
|
6563
|
+
if (logs.length === 0) {
|
|
6564
|
+
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6565
|
+
}
|
|
6566
|
+
return logs.map((l) => {
|
|
6567
|
+
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6568
|
+
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6569
|
+
}).join("\n");
|
|
6570
|
+
}
|
|
6571
|
+
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
6572
|
+
const formatted = results.map((r, idx) => {
|
|
6573
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6574
|
+
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
6575
|
+
return `${header} ${scoreLabel}
|
|
6576
|
+
\`\`\`
|
|
6577
|
+
${truncateContent(r.content)}
|
|
6578
|
+
\`\`\``;
|
|
6579
|
+
});
|
|
6580
|
+
return formatted.join("\n\n");
|
|
6581
|
+
}
|
|
6582
|
+
|
|
6583
|
+
// src/tools/index.ts
|
|
6584
|
+
var z = import_plugin.tool.schema;
|
|
6143
6585
|
var sharedIndexer = null;
|
|
6144
6586
|
function initializeTools(projectRoot, config) {
|
|
6145
6587
|
sharedIndexer = new Indexer(projectRoot, config);
|
|
@@ -6150,39 +6592,6 @@ function getIndexer() {
|
|
|
6150
6592
|
}
|
|
6151
6593
|
return sharedIndexer;
|
|
6152
6594
|
}
|
|
6153
|
-
var codebase_search = (0, import_plugin.tool)({
|
|
6154
|
-
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6155
|
-
args: {
|
|
6156
|
-
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6157
|
-
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6158
|
-
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6159
|
-
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6160
|
-
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6161
|
-
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6162
|
-
},
|
|
6163
|
-
async execute(args) {
|
|
6164
|
-
const indexer = getIndexer();
|
|
6165
|
-
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6166
|
-
fileType: args.fileType,
|
|
6167
|
-
directory: args.directory,
|
|
6168
|
-
chunkType: args.chunkType,
|
|
6169
|
-
contextLines: args.contextLines
|
|
6170
|
-
});
|
|
6171
|
-
if (results.length === 0) {
|
|
6172
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6173
|
-
}
|
|
6174
|
-
const formatted = results.map((r, idx) => {
|
|
6175
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6176
|
-
return `${header} (score: ${r.score.toFixed(2)})
|
|
6177
|
-
\`\`\`
|
|
6178
|
-
${truncateContent(r.content)}
|
|
6179
|
-
\`\`\``;
|
|
6180
|
-
});
|
|
6181
|
-
return `Found ${results.length} results for "${args.query}":
|
|
6182
|
-
|
|
6183
|
-
${formatted.join("\n\n")}`;
|
|
6184
|
-
}
|
|
6185
|
-
});
|
|
6186
6595
|
var codebase_peek = (0, import_plugin.tool)({
|
|
6187
6596
|
description: "Quick lookup of code locations by meaning. Returns only metadata (file, line, name, type) WITHOUT code content. Use this first to find WHERE code is, then use Read tool to examine specific files. Saves tokens by not returning full code blocks. Best for: discovery, navigation, finding multiple related locations.",
|
|
6188
6597
|
args: {
|
|
@@ -6200,19 +6609,7 @@ var codebase_peek = (0, import_plugin.tool)({
|
|
|
6200
6609
|
chunkType: args.chunkType,
|
|
6201
6610
|
metadataOnly: true
|
|
6202
6611
|
});
|
|
6203
|
-
|
|
6204
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6205
|
-
}
|
|
6206
|
-
const formatted = results.map((r, idx) => {
|
|
6207
|
-
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6208
|
-
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6209
|
-
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6210
|
-
});
|
|
6211
|
-
return `Found ${results.length} locations for "${args.query}":
|
|
6212
|
-
|
|
6213
|
-
${formatted.join("\n")}
|
|
6214
|
-
|
|
6215
|
-
Use Read tool to examine specific files.`;
|
|
6612
|
+
return formatCodebasePeek(results, args.query);
|
|
6216
6613
|
}
|
|
6217
6614
|
});
|
|
6218
6615
|
var index_codebase = (0, import_plugin.tool)({
|
|
@@ -6262,23 +6659,7 @@ var index_health_check = (0, import_plugin.tool)({
|
|
|
6262
6659
|
async execute() {
|
|
6263
6660
|
const indexer = getIndexer();
|
|
6264
6661
|
const result = await indexer.healthCheck();
|
|
6265
|
-
|
|
6266
|
-
return "Index is healthy. No stale entries found.";
|
|
6267
|
-
}
|
|
6268
|
-
const lines = [`Health check complete:`];
|
|
6269
|
-
if (result.removed > 0) {
|
|
6270
|
-
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6271
|
-
}
|
|
6272
|
-
if (result.gcOrphanEmbeddings > 0) {
|
|
6273
|
-
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6274
|
-
}
|
|
6275
|
-
if (result.gcOrphanChunks > 0) {
|
|
6276
|
-
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6277
|
-
}
|
|
6278
|
-
if (result.filePaths.length > 0) {
|
|
6279
|
-
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6280
|
-
}
|
|
6281
|
-
return lines.join("\n");
|
|
6662
|
+
return formatHealthCheck(result);
|
|
6282
6663
|
}
|
|
6283
6664
|
});
|
|
6284
6665
|
var index_metrics = (0, import_plugin.tool)({
|
|
@@ -6317,13 +6698,7 @@ var index_logs = (0, import_plugin.tool)({
|
|
|
6317
6698
|
} else {
|
|
6318
6699
|
logs = logger.getLogs(args.limit);
|
|
6319
6700
|
}
|
|
6320
|
-
|
|
6321
|
-
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6322
|
-
}
|
|
6323
|
-
return logs.map((l) => {
|
|
6324
|
-
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6325
|
-
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6326
|
-
}).join("\n");
|
|
6701
|
+
return formatLogs(logs);
|
|
6327
6702
|
}
|
|
6328
6703
|
});
|
|
6329
6704
|
var find_similar = (0, import_plugin.tool)({
|
|
@@ -6338,7 +6713,7 @@ var find_similar = (0, import_plugin.tool)({
|
|
|
6338
6713
|
},
|
|
6339
6714
|
async execute(args) {
|
|
6340
6715
|
const indexer = getIndexer();
|
|
6341
|
-
const results = await indexer.findSimilar(args.code, args.limit
|
|
6716
|
+
const results = await indexer.findSimilar(args.code, args.limit, {
|
|
6342
6717
|
fileType: args.fileType,
|
|
6343
6718
|
directory: args.directory,
|
|
6344
6719
|
chunkType: args.chunkType,
|
|
@@ -6347,109 +6722,37 @@ var find_similar = (0, import_plugin.tool)({
|
|
|
6347
6722
|
if (results.length === 0) {
|
|
6348
6723
|
return "No similar code found. Try a different snippet or run index_codebase first.";
|
|
6349
6724
|
}
|
|
6350
|
-
const formatted = results.map((r, idx) => {
|
|
6351
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6352
|
-
return `${header} (similarity: ${(r.score * 100).toFixed(1)}%)
|
|
6353
|
-
\`\`\`
|
|
6354
|
-
${truncateContent(r.content)}
|
|
6355
|
-
\`\`\``;
|
|
6356
|
-
});
|
|
6357
6725
|
return `Found ${results.length} similar code blocks:
|
|
6358
6726
|
|
|
6359
|
-
${
|
|
6727
|
+
${formatSearchResults(results)}`;
|
|
6360
6728
|
}
|
|
6361
6729
|
});
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
}
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
if (verbose) {
|
|
6383
|
-
if (stats.skippedFiles.length > 0) {
|
|
6384
|
-
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6385
|
-
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6386
|
-
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6387
|
-
lines.push("");
|
|
6388
|
-
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6389
|
-
if (tooLarge.length > 0) {
|
|
6390
|
-
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6391
|
-
}
|
|
6392
|
-
if (excluded.length > 0) {
|
|
6393
|
-
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6394
|
-
}
|
|
6395
|
-
if (gitignored.length > 0) {
|
|
6396
|
-
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6397
|
-
}
|
|
6398
|
-
}
|
|
6399
|
-
if (stats.parseFailures.length > 0) {
|
|
6400
|
-
lines.push("");
|
|
6401
|
-
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6730
|
+
var codebase_search = (0, import_plugin.tool)({
|
|
6731
|
+
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6732
|
+
args: {
|
|
6733
|
+
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6734
|
+
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6735
|
+
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6736
|
+
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6737
|
+
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6738
|
+
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6739
|
+
},
|
|
6740
|
+
async execute(args) {
|
|
6741
|
+
const indexer = getIndexer();
|
|
6742
|
+
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6743
|
+
fileType: args.fileType,
|
|
6744
|
+
directory: args.directory,
|
|
6745
|
+
chunkType: args.chunkType,
|
|
6746
|
+
contextLines: args.contextLines
|
|
6747
|
+
});
|
|
6748
|
+
if (results.length === 0) {
|
|
6749
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6402
6750
|
}
|
|
6751
|
+
return `Found ${results.length} results for "${args.query}":
|
|
6752
|
+
|
|
6753
|
+
${formatSearchResults(results, "score")}`;
|
|
6403
6754
|
}
|
|
6404
|
-
|
|
6405
|
-
}
|
|
6406
|
-
function formatStatus(status) {
|
|
6407
|
-
if (!status.indexed) {
|
|
6408
|
-
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6409
|
-
}
|
|
6410
|
-
const lines = [
|
|
6411
|
-
`Index status:`,
|
|
6412
|
-
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6413
|
-
` Provider: ${status.provider}`,
|
|
6414
|
-
` Model: ${status.model}`,
|
|
6415
|
-
` Location: ${status.indexPath}`
|
|
6416
|
-
];
|
|
6417
|
-
if (status.currentBranch !== "default") {
|
|
6418
|
-
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6419
|
-
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6420
|
-
}
|
|
6421
|
-
return lines.join("\n");
|
|
6422
|
-
}
|
|
6423
|
-
function formatProgressTitle(progress) {
|
|
6424
|
-
switch (progress.phase) {
|
|
6425
|
-
case "scanning":
|
|
6426
|
-
return "Scanning files...";
|
|
6427
|
-
case "parsing":
|
|
6428
|
-
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6429
|
-
case "embedding":
|
|
6430
|
-
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6431
|
-
case "storing":
|
|
6432
|
-
return "Storing index...";
|
|
6433
|
-
case "complete":
|
|
6434
|
-
return "Indexing complete";
|
|
6435
|
-
default:
|
|
6436
|
-
return "Indexing...";
|
|
6437
|
-
}
|
|
6438
|
-
}
|
|
6439
|
-
function calculatePercentage(progress) {
|
|
6440
|
-
if (progress.phase === "scanning") return 0;
|
|
6441
|
-
if (progress.phase === "complete") return 100;
|
|
6442
|
-
if (progress.phase === "parsing") {
|
|
6443
|
-
if (progress.totalFiles === 0) return 5;
|
|
6444
|
-
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6445
|
-
}
|
|
6446
|
-
if (progress.phase === "embedding") {
|
|
6447
|
-
if (progress.totalChunks === 0) return 20;
|
|
6448
|
-
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6449
|
-
}
|
|
6450
|
-
if (progress.phase === "storing") return 95;
|
|
6451
|
-
return 0;
|
|
6452
|
-
}
|
|
6755
|
+
});
|
|
6453
6756
|
|
|
6454
6757
|
// src/commands/loader.ts
|
|
6455
6758
|
var import_fs5 = require("fs");
|