opencode-codebase-index 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -2
- package/dist/cli.cjs +5016 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.js +5021 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +814 -357
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +816 -358
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +23 -4
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// opencode-codebase-index - Semantic codebase search for OpenCode
|
|
2
|
+
import { createRequire } from 'module'; const require = createRequire(import.meta.url);
|
|
2
3
|
var __create = Object.create;
|
|
3
4
|
var __defProp = Object.defineProperty;
|
|
4
5
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
@@ -657,7 +658,7 @@ import * as path8 from "path";
|
|
|
657
658
|
import * as os3 from "os";
|
|
658
659
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
659
660
|
|
|
660
|
-
// src/config/
|
|
661
|
+
// src/config/constants.ts
|
|
661
662
|
var DEFAULT_INCLUDE = [
|
|
662
663
|
"**/*.{ts,tsx,js,jsx,mjs,cjs}",
|
|
663
664
|
"**/*.{py,pyi}",
|
|
@@ -685,6 +686,80 @@ var DEFAULT_EXCLUDE = [
|
|
|
685
686
|
"**/.nuxt/**",
|
|
686
687
|
"**/.opencode/**"
|
|
687
688
|
];
|
|
689
|
+
var EMBEDDING_MODELS = {
|
|
690
|
+
"google": {
|
|
691
|
+
// `text-embedding-004` is DEPRECATED - https://ai.google.dev/gemini-api/docs/deprecations
|
|
692
|
+
"text-embedding-005": {
|
|
693
|
+
provider: "google",
|
|
694
|
+
model: "text-embedding-005",
|
|
695
|
+
dimensions: 768,
|
|
696
|
+
maxTokens: 2048,
|
|
697
|
+
costPer1MTokens: 0.025,
|
|
698
|
+
taskAble: false
|
|
699
|
+
// Note: on reality, this model allows for task-specific embeddings. See: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/embeddings/task-types
|
|
700
|
+
},
|
|
701
|
+
"gemini-embedding-001": {
|
|
702
|
+
provider: "google",
|
|
703
|
+
model: "gemini-embedding-001",
|
|
704
|
+
// Native output is 3072D, but we use Matryoshka truncation via outputDimensionality
|
|
705
|
+
// to reduce to 1536D for better storage/search efficiency with minimal quality loss.
|
|
706
|
+
// Google recommends 768, 1536, or 3072. See: https://ai.google.dev/gemini-api/docs/embeddings
|
|
707
|
+
dimensions: 1536,
|
|
708
|
+
maxTokens: 2048,
|
|
709
|
+
costPer1MTokens: 0.15,
|
|
710
|
+
taskAble: true
|
|
711
|
+
}
|
|
712
|
+
},
|
|
713
|
+
"openai": {
|
|
714
|
+
"text-embedding-3-small": {
|
|
715
|
+
provider: "openai",
|
|
716
|
+
model: "text-embedding-3-small",
|
|
717
|
+
dimensions: 1536,
|
|
718
|
+
maxTokens: 8191,
|
|
719
|
+
costPer1MTokens: 0.02
|
|
720
|
+
},
|
|
721
|
+
"text-embedding-3-large": {
|
|
722
|
+
provider: "openai",
|
|
723
|
+
model: "text-embedding-3-large",
|
|
724
|
+
dimensions: 3072,
|
|
725
|
+
maxTokens: 8191,
|
|
726
|
+
costPer1MTokens: 0.13
|
|
727
|
+
}
|
|
728
|
+
},
|
|
729
|
+
"ollama": {
|
|
730
|
+
"nomic-embed-text": {
|
|
731
|
+
provider: "ollama",
|
|
732
|
+
model: "nomic-embed-text",
|
|
733
|
+
dimensions: 768,
|
|
734
|
+
maxTokens: 8192,
|
|
735
|
+
costPer1MTokens: 0
|
|
736
|
+
},
|
|
737
|
+
"mxbai-embed-large": {
|
|
738
|
+
provider: "ollama",
|
|
739
|
+
model: "mxbai-embed-large",
|
|
740
|
+
dimensions: 1024,
|
|
741
|
+
maxTokens: 512,
|
|
742
|
+
costPer1MTokens: 0
|
|
743
|
+
}
|
|
744
|
+
},
|
|
745
|
+
"github-copilot": {
|
|
746
|
+
"text-embedding-3-small": {
|
|
747
|
+
provider: "github-copilot",
|
|
748
|
+
model: "text-embedding-3-small",
|
|
749
|
+
dimensions: 1536,
|
|
750
|
+
maxTokens: 8191,
|
|
751
|
+
costPer1MTokens: 0
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
};
|
|
755
|
+
var DEFAULT_PROVIDER_MODELS = {
|
|
756
|
+
"github-copilot": "text-embedding-3-small",
|
|
757
|
+
"openai": "text-embedding-3-small",
|
|
758
|
+
"google": "text-embedding-005",
|
|
759
|
+
"ollama": "nomic-embed-text"
|
|
760
|
+
};
|
|
761
|
+
|
|
762
|
+
// src/config/schema.ts
|
|
688
763
|
function getDefaultIndexingConfig() {
|
|
689
764
|
return {
|
|
690
765
|
autoIndex: false,
|
|
@@ -721,11 +796,13 @@ function getDefaultDebugConfig() {
|
|
|
721
796
|
metrics: true
|
|
722
797
|
};
|
|
723
798
|
}
|
|
724
|
-
var VALID_PROVIDERS = ["auto", "github-copilot", "openai", "google", "ollama"];
|
|
725
799
|
var VALID_SCOPES = ["project", "global"];
|
|
726
800
|
var VALID_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
727
801
|
function isValidProvider(value) {
|
|
728
|
-
return typeof value === "string" &&
|
|
802
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS).includes(value);
|
|
803
|
+
}
|
|
804
|
+
function isValidModel(value, provider) {
|
|
805
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS[provider]).includes(value);
|
|
729
806
|
}
|
|
730
807
|
function isValidScope(value) {
|
|
731
808
|
return typeof value === "string" && VALID_SCOPES.includes(value);
|
|
@@ -774,9 +851,45 @@ function parseConfig(raw) {
|
|
|
774
851
|
logBranch: typeof rawDebug.logBranch === "boolean" ? rawDebug.logBranch : defaultDebug.logBranch,
|
|
775
852
|
metrics: typeof rawDebug.metrics === "boolean" ? rawDebug.metrics : defaultDebug.metrics
|
|
776
853
|
};
|
|
854
|
+
let embeddingProvider;
|
|
855
|
+
let embeddingModel = void 0;
|
|
856
|
+
let customProvider = void 0;
|
|
857
|
+
if (input.embeddingProvider === "custom") {
|
|
858
|
+
embeddingProvider = "custom";
|
|
859
|
+
const rawCustom = input.customProvider && typeof input.customProvider === "object" ? input.customProvider : null;
|
|
860
|
+
if (rawCustom && typeof rawCustom.baseUrl === "string" && rawCustom.baseUrl.trim().length > 0 && typeof rawCustom.model === "string" && rawCustom.model.trim().length > 0 && typeof rawCustom.dimensions === "number" && Number.isInteger(rawCustom.dimensions) && rawCustom.dimensions > 0) {
|
|
861
|
+
customProvider = {
|
|
862
|
+
baseUrl: rawCustom.baseUrl.trim().replace(/\/+$/, ""),
|
|
863
|
+
model: rawCustom.model,
|
|
864
|
+
dimensions: rawCustom.dimensions,
|
|
865
|
+
apiKey: typeof rawCustom.apiKey === "string" ? rawCustom.apiKey : void 0,
|
|
866
|
+
maxTokens: typeof rawCustom.maxTokens === "number" ? rawCustom.maxTokens : void 0,
|
|
867
|
+
timeoutMs: typeof rawCustom.timeoutMs === "number" ? Math.max(1e3, rawCustom.timeoutMs) : void 0,
|
|
868
|
+
concurrency: typeof rawCustom.concurrency === "number" ? Math.max(1, Math.floor(rawCustom.concurrency)) : void 0,
|
|
869
|
+
requestIntervalMs: typeof rawCustom.requestIntervalMs === "number" ? Math.max(0, Math.floor(rawCustom.requestIntervalMs)) : void 0
|
|
870
|
+
};
|
|
871
|
+
if (!/\/v\d+\/?$/.test(customProvider.baseUrl)) {
|
|
872
|
+
console.warn(
|
|
873
|
+
`[codebase-index] Warning: customProvider.baseUrl ("${customProvider.baseUrl}") does not end with an API version path like /v1. The plugin appends /embeddings automatically, so the full URL will be "${customProvider.baseUrl}/embeddings". If your provider expects /v1/embeddings, set baseUrl to "${customProvider.baseUrl}/v1".`
|
|
874
|
+
);
|
|
875
|
+
}
|
|
876
|
+
} else {
|
|
877
|
+
throw new Error(
|
|
878
|
+
"embeddingProvider is 'custom' but customProvider config is missing or invalid. Required fields: baseUrl (string), model (string), dimensions (positive integer)."
|
|
879
|
+
);
|
|
880
|
+
}
|
|
881
|
+
} else if (isValidProvider(input.embeddingProvider)) {
|
|
882
|
+
embeddingProvider = input.embeddingProvider;
|
|
883
|
+
if (input.embeddingModel) {
|
|
884
|
+
embeddingModel = isValidModel(input.embeddingModel, embeddingProvider) ? input.embeddingModel : DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
885
|
+
}
|
|
886
|
+
} else {
|
|
887
|
+
embeddingProvider = "auto";
|
|
888
|
+
}
|
|
777
889
|
return {
|
|
778
|
-
embeddingProvider
|
|
779
|
-
embeddingModel
|
|
890
|
+
embeddingProvider,
|
|
891
|
+
embeddingModel,
|
|
892
|
+
customProvider,
|
|
780
893
|
scope: isValidScope(input.scope) ? input.scope : "project",
|
|
781
894
|
include: isStringArray(input.include) ? input.include : DEFAULT_INCLUDE,
|
|
782
895
|
exclude: isStringArray(input.exclude) ? input.exclude : DEFAULT_EXCLUDE,
|
|
@@ -785,67 +898,15 @@ function parseConfig(raw) {
|
|
|
785
898
|
debug
|
|
786
899
|
};
|
|
787
900
|
}
|
|
788
|
-
var EMBEDDING_MODELS = {
|
|
789
|
-
"github-copilot/text-embedding-3-small": {
|
|
790
|
-
provider: "github-copilot",
|
|
791
|
-
model: "text-embedding-3-small",
|
|
792
|
-
dimensions: 1536,
|
|
793
|
-
maxTokens: 8191,
|
|
794
|
-
costPer1MTokens: 0
|
|
795
|
-
},
|
|
796
|
-
"openai/text-embedding-3-small": {
|
|
797
|
-
provider: "openai",
|
|
798
|
-
model: "text-embedding-3-small",
|
|
799
|
-
dimensions: 1536,
|
|
800
|
-
maxTokens: 8191,
|
|
801
|
-
costPer1MTokens: 0.02
|
|
802
|
-
},
|
|
803
|
-
"openai/text-embedding-3-large": {
|
|
804
|
-
provider: "openai",
|
|
805
|
-
model: "text-embedding-3-large",
|
|
806
|
-
dimensions: 3072,
|
|
807
|
-
maxTokens: 8191,
|
|
808
|
-
costPer1MTokens: 0.13
|
|
809
|
-
},
|
|
810
|
-
"google/text-embedding-004": {
|
|
811
|
-
provider: "google",
|
|
812
|
-
model: "text-embedding-004",
|
|
813
|
-
dimensions: 768,
|
|
814
|
-
maxTokens: 2048,
|
|
815
|
-
costPer1MTokens: 0
|
|
816
|
-
},
|
|
817
|
-
"ollama/nomic-embed-text": {
|
|
818
|
-
provider: "ollama",
|
|
819
|
-
model: "nomic-embed-text",
|
|
820
|
-
dimensions: 768,
|
|
821
|
-
maxTokens: 8192,
|
|
822
|
-
costPer1MTokens: 0
|
|
823
|
-
},
|
|
824
|
-
"ollama/mxbai-embed-large": {
|
|
825
|
-
provider: "ollama",
|
|
826
|
-
model: "mxbai-embed-large",
|
|
827
|
-
dimensions: 1024,
|
|
828
|
-
maxTokens: 512,
|
|
829
|
-
costPer1MTokens: 0
|
|
830
|
-
}
|
|
831
|
-
};
|
|
832
901
|
function getDefaultModelForProvider(provider) {
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
case "openai":
|
|
837
|
-
return EMBEDDING_MODELS["openai/text-embedding-3-small"];
|
|
838
|
-
case "google":
|
|
839
|
-
return EMBEDDING_MODELS["google/text-embedding-004"];
|
|
840
|
-
case "ollama":
|
|
841
|
-
return EMBEDDING_MODELS["ollama/nomic-embed-text"];
|
|
842
|
-
default:
|
|
843
|
-
return EMBEDDING_MODELS["github-copilot/text-embedding-3-small"];
|
|
844
|
-
}
|
|
902
|
+
const models = EMBEDDING_MODELS[provider];
|
|
903
|
+
const providerDefault = DEFAULT_PROVIDER_MODELS[provider];
|
|
904
|
+
return models[providerDefault];
|
|
845
905
|
}
|
|
906
|
+
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
846
907
|
|
|
847
908
|
// src/indexer/index.ts
|
|
848
|
-
import { existsSync as existsSync4, readFileSync as readFileSync4, writeFileSync, promises as fsPromises2 } from "fs";
|
|
909
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4, writeFileSync, renameSync, unlinkSync, promises as fsPromises2 } from "fs";
|
|
849
910
|
import * as path5 from "path";
|
|
850
911
|
import { performance as performance2 } from "perf_hooks";
|
|
851
912
|
|
|
@@ -871,7 +932,7 @@ function pTimeout(promise, options) {
|
|
|
871
932
|
} = options;
|
|
872
933
|
let timer;
|
|
873
934
|
let abortHandler;
|
|
874
|
-
const wrappedPromise = new Promise((
|
|
935
|
+
const wrappedPromise = new Promise((resolve5, reject) => {
|
|
875
936
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
876
937
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
877
938
|
}
|
|
@@ -885,7 +946,7 @@ function pTimeout(promise, options) {
|
|
|
885
946
|
};
|
|
886
947
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
887
948
|
}
|
|
888
|
-
promise.then(
|
|
949
|
+
promise.then(resolve5, reject);
|
|
889
950
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
890
951
|
return;
|
|
891
952
|
}
|
|
@@ -893,7 +954,7 @@ function pTimeout(promise, options) {
|
|
|
893
954
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
894
955
|
if (fallback) {
|
|
895
956
|
try {
|
|
896
|
-
|
|
957
|
+
resolve5(fallback());
|
|
897
958
|
} catch (error) {
|
|
898
959
|
reject(error);
|
|
899
960
|
}
|
|
@@ -903,7 +964,7 @@ function pTimeout(promise, options) {
|
|
|
903
964
|
promise.cancel();
|
|
904
965
|
}
|
|
905
966
|
if (message === false) {
|
|
906
|
-
|
|
967
|
+
resolve5();
|
|
907
968
|
} else if (message instanceof Error) {
|
|
908
969
|
reject(message);
|
|
909
970
|
} else {
|
|
@@ -1293,7 +1354,7 @@ var PQueue = class extends import_index.default {
|
|
|
1293
1354
|
// Assign unique ID if not provided
|
|
1294
1355
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1295
1356
|
};
|
|
1296
|
-
return new Promise((
|
|
1357
|
+
return new Promise((resolve5, reject) => {
|
|
1297
1358
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1298
1359
|
this.#queue.enqueue(async () => {
|
|
1299
1360
|
this.#pending++;
|
|
@@ -1331,7 +1392,7 @@ var PQueue = class extends import_index.default {
|
|
|
1331
1392
|
})]);
|
|
1332
1393
|
}
|
|
1333
1394
|
const result = await operation;
|
|
1334
|
-
|
|
1395
|
+
resolve5(result);
|
|
1335
1396
|
this.emit("completed", result);
|
|
1336
1397
|
} catch (error) {
|
|
1337
1398
|
reject(error);
|
|
@@ -1488,13 +1549,13 @@ var PQueue = class extends import_index.default {
|
|
|
1488
1549
|
});
|
|
1489
1550
|
}
|
|
1490
1551
|
async #onEvent(event, filter) {
|
|
1491
|
-
return new Promise((
|
|
1552
|
+
return new Promise((resolve5) => {
|
|
1492
1553
|
const listener = () => {
|
|
1493
1554
|
if (filter && !filter()) {
|
|
1494
1555
|
return;
|
|
1495
1556
|
}
|
|
1496
1557
|
this.off(event, listener);
|
|
1497
|
-
|
|
1558
|
+
resolve5();
|
|
1498
1559
|
};
|
|
1499
1560
|
this.on(event, listener);
|
|
1500
1561
|
});
|
|
@@ -1779,7 +1840,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1779
1840
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
1780
1841
|
options.signal?.throwIfAborted();
|
|
1781
1842
|
if (finalDelay > 0) {
|
|
1782
|
-
await new Promise((
|
|
1843
|
+
await new Promise((resolve5, reject) => {
|
|
1783
1844
|
const onAbort = () => {
|
|
1784
1845
|
clearTimeout(timeoutToken);
|
|
1785
1846
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -1787,7 +1848,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1787
1848
|
};
|
|
1788
1849
|
const timeoutToken = setTimeout(() => {
|
|
1789
1850
|
options.signal?.removeEventListener("abort", onAbort);
|
|
1790
|
-
|
|
1851
|
+
resolve5();
|
|
1791
1852
|
}, finalDelay);
|
|
1792
1853
|
if (options.unref) {
|
|
1793
1854
|
timeoutToken.unref?.();
|
|
@@ -1851,12 +1912,6 @@ async function pRetry(input, options = {}) {
|
|
|
1851
1912
|
import { existsSync, readFileSync } from "fs";
|
|
1852
1913
|
import * as path from "path";
|
|
1853
1914
|
import * as os from "os";
|
|
1854
|
-
var EMBEDDING_CAPABLE_PROVIDERS = [
|
|
1855
|
-
"github-copilot",
|
|
1856
|
-
"openai",
|
|
1857
|
-
"google",
|
|
1858
|
-
"ollama"
|
|
1859
|
-
];
|
|
1860
1915
|
function getOpenCodeAuthPath() {
|
|
1861
1916
|
return path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
|
|
1862
1917
|
}
|
|
@@ -1870,21 +1925,34 @@ function loadOpenCodeAuth() {
|
|
|
1870
1925
|
}
|
|
1871
1926
|
return {};
|
|
1872
1927
|
}
|
|
1873
|
-
async function detectEmbeddingProvider(preferredProvider) {
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
if (
|
|
1928
|
+
async function detectEmbeddingProvider(preferredProvider, model) {
|
|
1929
|
+
const credentials = await getProviderCredentials(preferredProvider);
|
|
1930
|
+
if (credentials) {
|
|
1931
|
+
if (!model) {
|
|
1877
1932
|
return {
|
|
1878
1933
|
provider: preferredProvider,
|
|
1879
1934
|
credentials,
|
|
1880
1935
|
modelInfo: getDefaultModelForProvider(preferredProvider)
|
|
1881
1936
|
};
|
|
1882
1937
|
}
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1938
|
+
if (!isValidModel(model, preferredProvider)) {
|
|
1939
|
+
throw new Error(
|
|
1940
|
+
`Model '${model}' is not supported by provider '${preferredProvider}'`
|
|
1941
|
+
);
|
|
1942
|
+
}
|
|
1943
|
+
const providerModels = EMBEDDING_MODELS[preferredProvider];
|
|
1944
|
+
return {
|
|
1945
|
+
provider: preferredProvider,
|
|
1946
|
+
credentials,
|
|
1947
|
+
modelInfo: providerModels[model]
|
|
1948
|
+
};
|
|
1886
1949
|
}
|
|
1887
|
-
|
|
1950
|
+
throw new Error(
|
|
1951
|
+
`Preferred provider '${preferredProvider}' is not configured or authenticated`
|
|
1952
|
+
);
|
|
1953
|
+
}
|
|
1954
|
+
async function tryDetectProvider() {
|
|
1955
|
+
for (const provider of availableProviders) {
|
|
1888
1956
|
const credentials = await getProviderCredentials(provider);
|
|
1889
1957
|
if (credentials) {
|
|
1890
1958
|
return {
|
|
@@ -1895,7 +1963,7 @@ async function detectEmbeddingProvider(preferredProvider) {
|
|
|
1895
1963
|
}
|
|
1896
1964
|
}
|
|
1897
1965
|
throw new Error(
|
|
1898
|
-
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${
|
|
1966
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${availableProviders.join(", ")}.`
|
|
1899
1967
|
);
|
|
1900
1968
|
}
|
|
1901
1969
|
async function getProviderCredentials(provider) {
|
|
@@ -1987,24 +2055,55 @@ function getProviderDisplayName(provider) {
|
|
|
1987
2055
|
return "Google (Gemini)";
|
|
1988
2056
|
case "ollama":
|
|
1989
2057
|
return "Ollama (Local)";
|
|
2058
|
+
case "custom":
|
|
2059
|
+
return "Custom (OpenAI-compatible)";
|
|
1990
2060
|
default:
|
|
1991
2061
|
return provider;
|
|
1992
2062
|
}
|
|
1993
2063
|
}
|
|
2064
|
+
function createCustomProviderInfo(config) {
|
|
2065
|
+
const baseUrl = config.baseUrl.replace(/\/+$/, "");
|
|
2066
|
+
return {
|
|
2067
|
+
provider: "custom",
|
|
2068
|
+
credentials: {
|
|
2069
|
+
provider: "custom",
|
|
2070
|
+
baseUrl,
|
|
2071
|
+
apiKey: config.apiKey
|
|
2072
|
+
},
|
|
2073
|
+
modelInfo: {
|
|
2074
|
+
provider: "custom",
|
|
2075
|
+
model: config.model,
|
|
2076
|
+
dimensions: config.dimensions,
|
|
2077
|
+
maxTokens: config.maxTokens ?? 8192,
|
|
2078
|
+
costPer1MTokens: 0,
|
|
2079
|
+
timeoutMs: config.timeoutMs ?? 3e4
|
|
2080
|
+
}
|
|
2081
|
+
};
|
|
2082
|
+
}
|
|
1994
2083
|
|
|
1995
2084
|
// src/embeddings/provider.ts
|
|
1996
|
-
|
|
1997
|
-
|
|
2085
|
+
var CustomProviderNonRetryableError = class extends Error {
|
|
2086
|
+
constructor(message) {
|
|
2087
|
+
super(message);
|
|
2088
|
+
this.name = "CustomProviderNonRetryableError";
|
|
2089
|
+
}
|
|
2090
|
+
};
|
|
2091
|
+
function createEmbeddingProvider(configuredProviderInfo) {
|
|
2092
|
+
switch (configuredProviderInfo.provider) {
|
|
1998
2093
|
case "github-copilot":
|
|
1999
|
-
return new GitHubCopilotEmbeddingProvider(credentials, modelInfo);
|
|
2094
|
+
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2000
2095
|
case "openai":
|
|
2001
|
-
return new OpenAIEmbeddingProvider(credentials, modelInfo);
|
|
2096
|
+
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2002
2097
|
case "google":
|
|
2003
|
-
return new GoogleEmbeddingProvider(credentials, modelInfo);
|
|
2098
|
+
return new GoogleEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2004
2099
|
case "ollama":
|
|
2005
|
-
return new OllamaEmbeddingProvider(credentials, modelInfo);
|
|
2006
|
-
|
|
2007
|
-
|
|
2100
|
+
return new OllamaEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2101
|
+
case "custom":
|
|
2102
|
+
return new CustomEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2103
|
+
default: {
|
|
2104
|
+
const _exhaustive = configuredProviderInfo;
|
|
2105
|
+
throw new Error(`Unsupported embedding provider: ${_exhaustive.provider}`);
|
|
2106
|
+
}
|
|
2008
2107
|
}
|
|
2009
2108
|
}
|
|
2010
2109
|
var GitHubCopilotEmbeddingProvider = class {
|
|
@@ -2018,8 +2117,15 @@ var GitHubCopilotEmbeddingProvider = class {
|
|
|
2018
2117
|
}
|
|
2019
2118
|
return this.credentials.refreshToken;
|
|
2020
2119
|
}
|
|
2021
|
-
async
|
|
2022
|
-
const result = await this.embedBatch([
|
|
2120
|
+
async embedQuery(query) {
|
|
2121
|
+
const result = await this.embedBatch([query]);
|
|
2122
|
+
return {
|
|
2123
|
+
embedding: result.embeddings[0],
|
|
2124
|
+
tokensUsed: result.totalTokensUsed
|
|
2125
|
+
};
|
|
2126
|
+
}
|
|
2127
|
+
async embedDocument(document) {
|
|
2128
|
+
const result = await this.embedBatch([document]);
|
|
2023
2129
|
return {
|
|
2024
2130
|
embedding: result.embeddings[0],
|
|
2025
2131
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2059,8 +2165,15 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2059
2165
|
this.credentials = credentials;
|
|
2060
2166
|
this.modelInfo = modelInfo;
|
|
2061
2167
|
}
|
|
2062
|
-
async
|
|
2063
|
-
const result = await this.embedBatch([
|
|
2168
|
+
async embedQuery(query) {
|
|
2169
|
+
const result = await this.embedBatch([query]);
|
|
2170
|
+
return {
|
|
2171
|
+
embedding: result.embeddings[0],
|
|
2172
|
+
tokensUsed: result.totalTokensUsed
|
|
2173
|
+
};
|
|
2174
|
+
}
|
|
2175
|
+
async embedDocument(document) {
|
|
2176
|
+
const result = await this.embedBatch([document]);
|
|
2064
2177
|
return {
|
|
2065
2178
|
embedding: result.embeddings[0],
|
|
2066
2179
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2092,33 +2205,61 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2092
2205
|
return this.modelInfo;
|
|
2093
2206
|
}
|
|
2094
2207
|
};
|
|
2095
|
-
var GoogleEmbeddingProvider = class {
|
|
2208
|
+
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
2096
2209
|
constructor(credentials, modelInfo) {
|
|
2097
2210
|
this.credentials = credentials;
|
|
2098
2211
|
this.modelInfo = modelInfo;
|
|
2099
2212
|
}
|
|
2100
|
-
|
|
2101
|
-
|
|
2213
|
+
static BATCH_SIZE = 20;
|
|
2214
|
+
async embedQuery(query) {
|
|
2215
|
+
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2216
|
+
const result = await this.embedWithTaskType([query], taskType);
|
|
2217
|
+
return {
|
|
2218
|
+
embedding: result.embeddings[0],
|
|
2219
|
+
tokensUsed: result.totalTokensUsed
|
|
2220
|
+
};
|
|
2221
|
+
}
|
|
2222
|
+
async embedDocument(document) {
|
|
2223
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2224
|
+
const result = await this.embedWithTaskType([document], taskType);
|
|
2102
2225
|
return {
|
|
2103
2226
|
embedding: result.embeddings[0],
|
|
2104
2227
|
tokensUsed: result.totalTokensUsed
|
|
2105
2228
|
};
|
|
2106
2229
|
}
|
|
2107
2230
|
async embedBatch(texts) {
|
|
2108
|
-
const
|
|
2109
|
-
|
|
2231
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2232
|
+
return this.embedWithTaskType(texts, taskType);
|
|
2233
|
+
}
|
|
2234
|
+
/**
|
|
2235
|
+
* Embeds texts using the Google embedContent API.
|
|
2236
|
+
* Sends multiple texts as parts in batched requests (up to BATCH_SIZE per call).
|
|
2237
|
+
* When taskType is provided (gemini-embedding-001), includes it in the request
|
|
2238
|
+
* for task-specific embedding optimization.
|
|
2239
|
+
*/
|
|
2240
|
+
async embedWithTaskType(texts, taskType) {
|
|
2241
|
+
const batches = [];
|
|
2242
|
+
for (let i = 0; i < texts.length; i += _GoogleEmbeddingProvider.BATCH_SIZE) {
|
|
2243
|
+
batches.push(texts.slice(i, i + _GoogleEmbeddingProvider.BATCH_SIZE));
|
|
2244
|
+
}
|
|
2245
|
+
const batchResults = await Promise.all(
|
|
2246
|
+
batches.map(async (batch) => {
|
|
2247
|
+
const requests = batch.map((text) => ({
|
|
2248
|
+
model: `models/${this.modelInfo.model}`,
|
|
2249
|
+
content: {
|
|
2250
|
+
parts: [{ text }]
|
|
2251
|
+
},
|
|
2252
|
+
taskType,
|
|
2253
|
+
outputDimensionality: this.modelInfo.dimensions
|
|
2254
|
+
}));
|
|
2110
2255
|
const response = await fetch(
|
|
2111
|
-
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:
|
|
2256
|
+
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:batchEmbedContents?key=${this.credentials.apiKey}`,
|
|
2112
2257
|
{
|
|
2113
2258
|
method: "POST",
|
|
2114
2259
|
headers: {
|
|
2115
2260
|
"Content-Type": "application/json"
|
|
2116
2261
|
},
|
|
2117
|
-
body: JSON.stringify({
|
|
2118
|
-
content: {
|
|
2119
|
-
parts: [{ text }]
|
|
2120
|
-
}
|
|
2121
|
-
})
|
|
2262
|
+
body: JSON.stringify({ requests })
|
|
2122
2263
|
}
|
|
2123
2264
|
);
|
|
2124
2265
|
if (!response.ok) {
|
|
@@ -2127,14 +2268,14 @@ var GoogleEmbeddingProvider = class {
|
|
|
2127
2268
|
}
|
|
2128
2269
|
const data = await response.json();
|
|
2129
2270
|
return {
|
|
2130
|
-
|
|
2131
|
-
tokensUsed: Math.ceil(text.length / 4)
|
|
2271
|
+
embeddings: data.embeddings.map((e) => e.values),
|
|
2272
|
+
tokensUsed: batch.reduce((sum, text) => sum + Math.ceil(text.length / 4), 0)
|
|
2132
2273
|
};
|
|
2133
2274
|
})
|
|
2134
2275
|
);
|
|
2135
2276
|
return {
|
|
2136
|
-
embeddings:
|
|
2137
|
-
totalTokensUsed:
|
|
2277
|
+
embeddings: batchResults.flatMap((r) => r.embeddings),
|
|
2278
|
+
totalTokensUsed: batchResults.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2138
2279
|
};
|
|
2139
2280
|
}
|
|
2140
2281
|
getModelInfo() {
|
|
@@ -2146,29 +2287,44 @@ var OllamaEmbeddingProvider = class {
|
|
|
2146
2287
|
this.credentials = credentials;
|
|
2147
2288
|
this.modelInfo = modelInfo;
|
|
2148
2289
|
}
|
|
2149
|
-
async
|
|
2150
|
-
const
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
})
|
|
2159
|
-
});
|
|
2160
|
-
if (!response.ok) {
|
|
2161
|
-
const error = await response.text();
|
|
2162
|
-
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2163
|
-
}
|
|
2164
|
-
const data = await response.json();
|
|
2290
|
+
async embedQuery(query) {
|
|
2291
|
+
const result = await this.embedBatch([query]);
|
|
2292
|
+
return {
|
|
2293
|
+
embedding: result.embeddings[0],
|
|
2294
|
+
tokensUsed: result.totalTokensUsed
|
|
2295
|
+
};
|
|
2296
|
+
}
|
|
2297
|
+
async embedDocument(document) {
|
|
2298
|
+
const result = await this.embedBatch([document]);
|
|
2165
2299
|
return {
|
|
2166
|
-
embedding:
|
|
2167
|
-
tokensUsed:
|
|
2300
|
+
embedding: result.embeddings[0],
|
|
2301
|
+
tokensUsed: result.totalTokensUsed
|
|
2168
2302
|
};
|
|
2169
2303
|
}
|
|
2170
2304
|
async embedBatch(texts) {
|
|
2171
|
-
const results = await Promise.all(
|
|
2305
|
+
const results = await Promise.all(
|
|
2306
|
+
texts.map(async (text) => {
|
|
2307
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2308
|
+
method: "POST",
|
|
2309
|
+
headers: {
|
|
2310
|
+
"Content-Type": "application/json"
|
|
2311
|
+
},
|
|
2312
|
+
body: JSON.stringify({
|
|
2313
|
+
model: this.modelInfo.model,
|
|
2314
|
+
prompt: text
|
|
2315
|
+
})
|
|
2316
|
+
});
|
|
2317
|
+
if (!response.ok) {
|
|
2318
|
+
const error = await response.text();
|
|
2319
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2320
|
+
}
|
|
2321
|
+
const data = await response.json();
|
|
2322
|
+
return {
|
|
2323
|
+
embedding: data.embedding,
|
|
2324
|
+
tokensUsed: Math.ceil(text.length / 4)
|
|
2325
|
+
};
|
|
2326
|
+
})
|
|
2327
|
+
);
|
|
2172
2328
|
return {
|
|
2173
2329
|
embeddings: results.map((r) => r.embedding),
|
|
2174
2330
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
@@ -2178,6 +2334,90 @@ var OllamaEmbeddingProvider = class {
|
|
|
2178
2334
|
return this.modelInfo;
|
|
2179
2335
|
}
|
|
2180
2336
|
};
|
|
2337
|
+
var CustomEmbeddingProvider = class {
|
|
2338
|
+
constructor(credentials, modelInfo) {
|
|
2339
|
+
this.credentials = credentials;
|
|
2340
|
+
this.modelInfo = modelInfo;
|
|
2341
|
+
}
|
|
2342
|
+
async embedQuery(query) {
|
|
2343
|
+
const result = await this.embedBatch([query]);
|
|
2344
|
+
return {
|
|
2345
|
+
embedding: result.embeddings[0],
|
|
2346
|
+
tokensUsed: result.totalTokensUsed
|
|
2347
|
+
};
|
|
2348
|
+
}
|
|
2349
|
+
async embedDocument(document) {
|
|
2350
|
+
const result = await this.embedBatch([document]);
|
|
2351
|
+
return {
|
|
2352
|
+
embedding: result.embeddings[0],
|
|
2353
|
+
tokensUsed: result.totalTokensUsed
|
|
2354
|
+
};
|
|
2355
|
+
}
|
|
2356
|
+
async embedBatch(texts) {
|
|
2357
|
+
const headers = {
|
|
2358
|
+
"Content-Type": "application/json"
|
|
2359
|
+
};
|
|
2360
|
+
if (this.credentials.apiKey) {
|
|
2361
|
+
headers["Authorization"] = `Bearer ${this.credentials.apiKey}`;
|
|
2362
|
+
}
|
|
2363
|
+
const baseUrl = this.credentials.baseUrl ?? "";
|
|
2364
|
+
const timeoutMs = this.modelInfo.timeoutMs;
|
|
2365
|
+
const controller = new AbortController();
|
|
2366
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
2367
|
+
let response;
|
|
2368
|
+
try {
|
|
2369
|
+
response = await fetch(`${baseUrl}/embeddings`, {
|
|
2370
|
+
method: "POST",
|
|
2371
|
+
headers,
|
|
2372
|
+
body: JSON.stringify({
|
|
2373
|
+
model: this.modelInfo.model,
|
|
2374
|
+
input: texts
|
|
2375
|
+
}),
|
|
2376
|
+
signal: controller.signal
|
|
2377
|
+
});
|
|
2378
|
+
} catch (error) {
|
|
2379
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
2380
|
+
throw new Error(`Custom embedding API request timed out after ${timeoutMs}ms for ${baseUrl}/embeddings`);
|
|
2381
|
+
}
|
|
2382
|
+
throw error;
|
|
2383
|
+
} finally {
|
|
2384
|
+
clearTimeout(timeout);
|
|
2385
|
+
}
|
|
2386
|
+
if (!response.ok) {
|
|
2387
|
+
const errorText = await response.text();
|
|
2388
|
+
if (response.status >= 400 && response.status < 500 && response.status !== 429) {
|
|
2389
|
+
throw new CustomProviderNonRetryableError(`Custom embedding API error (non-retryable): ${response.status} - ${errorText}`);
|
|
2390
|
+
}
|
|
2391
|
+
throw new Error(`Custom embedding API error: ${response.status} - ${errorText}`);
|
|
2392
|
+
}
|
|
2393
|
+
const data = await response.json();
|
|
2394
|
+
if (data.data && Array.isArray(data.data)) {
|
|
2395
|
+
if (data.data.length > 0) {
|
|
2396
|
+
const actualDims = data.data[0].embedding.length;
|
|
2397
|
+
if (actualDims !== this.modelInfo.dimensions) {
|
|
2398
|
+
throw new Error(
|
|
2399
|
+
`Dimension mismatch: customProvider.dimensions is ${this.modelInfo.dimensions}, but the API returned vectors with ${actualDims} dimensions. Update your config to match the model's actual output dimensions.`
|
|
2400
|
+
);
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
if (data.data.length !== texts.length) {
|
|
2404
|
+
throw new Error(
|
|
2405
|
+
`Embedding count mismatch: sent ${texts.length} texts but received ${data.data.length} embeddings. The custom embedding server may not support batch input.`
|
|
2406
|
+
);
|
|
2407
|
+
}
|
|
2408
|
+
return {
|
|
2409
|
+
embeddings: data.data.map((d) => d.embedding),
|
|
2410
|
+
// Rough estimate: ~4 chars per token. Used as fallback when the server
|
|
2411
|
+
// doesn't return usage.total_tokens (e.g. llama.cpp, some vLLM configs).
|
|
2412
|
+
totalTokensUsed: data.usage?.total_tokens ?? texts.reduce((sum, t) => sum + Math.ceil(t.length / 4), 0)
|
|
2413
|
+
};
|
|
2414
|
+
}
|
|
2415
|
+
throw new Error("Custom embedding API returned unexpected response format. Expected OpenAI-compatible format with data[].embedding.");
|
|
2416
|
+
}
|
|
2417
|
+
getModelInfo() {
|
|
2418
|
+
return this.modelInfo;
|
|
2419
|
+
}
|
|
2420
|
+
};
|
|
2181
2421
|
|
|
2182
2422
|
// src/utils/files.ts
|
|
2183
2423
|
var import_ignore = __toESM(require_ignore(), 1);
|
|
@@ -3112,11 +3352,40 @@ var Database = class {
|
|
|
3112
3352
|
import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync, statSync } from "fs";
|
|
3113
3353
|
import * as path4 from "path";
|
|
3114
3354
|
import { execSync } from "child_process";
|
|
3355
|
+
function resolveGitDir(repoRoot) {
|
|
3356
|
+
const gitPath = path4.join(repoRoot, ".git");
|
|
3357
|
+
if (!existsSync3(gitPath)) {
|
|
3358
|
+
return null;
|
|
3359
|
+
}
|
|
3360
|
+
try {
|
|
3361
|
+
const stat4 = statSync(gitPath);
|
|
3362
|
+
if (stat4.isDirectory()) {
|
|
3363
|
+
return gitPath;
|
|
3364
|
+
}
|
|
3365
|
+
if (stat4.isFile()) {
|
|
3366
|
+
const content = readFileSync3(gitPath, "utf-8").trim();
|
|
3367
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
3368
|
+
if (match) {
|
|
3369
|
+
const gitdir = match[1];
|
|
3370
|
+
const resolvedPath = path4.isAbsolute(gitdir) ? gitdir : path4.resolve(repoRoot, gitdir);
|
|
3371
|
+
if (existsSync3(resolvedPath)) {
|
|
3372
|
+
return resolvedPath;
|
|
3373
|
+
}
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
3376
|
+
} catch {
|
|
3377
|
+
}
|
|
3378
|
+
return null;
|
|
3379
|
+
}
|
|
3115
3380
|
function isGitRepo(dir) {
|
|
3116
|
-
return
|
|
3381
|
+
return resolveGitDir(dir) !== null;
|
|
3117
3382
|
}
|
|
3118
3383
|
function getCurrentBranch(repoRoot) {
|
|
3119
|
-
const
|
|
3384
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3385
|
+
if (!gitDir) {
|
|
3386
|
+
return null;
|
|
3387
|
+
}
|
|
3388
|
+
const headPath = path4.join(gitDir, "HEAD");
|
|
3120
3389
|
if (!existsSync3(headPath)) {
|
|
3121
3390
|
return null;
|
|
3122
3391
|
}
|
|
@@ -3135,20 +3404,23 @@ function getCurrentBranch(repoRoot) {
|
|
|
3135
3404
|
}
|
|
3136
3405
|
}
|
|
3137
3406
|
function getBaseBranch(repoRoot) {
|
|
3407
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3138
3408
|
const candidates = ["main", "master", "develop", "trunk"];
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3409
|
+
if (gitDir) {
|
|
3410
|
+
for (const candidate of candidates) {
|
|
3411
|
+
const refPath = path4.join(gitDir, "refs", "heads", candidate);
|
|
3412
|
+
if (existsSync3(refPath)) {
|
|
3413
|
+
return candidate;
|
|
3414
|
+
}
|
|
3415
|
+
const packedRefsPath = path4.join(gitDir, "packed-refs");
|
|
3416
|
+
if (existsSync3(packedRefsPath)) {
|
|
3417
|
+
try {
|
|
3418
|
+
const content = readFileSync3(packedRefsPath, "utf-8");
|
|
3419
|
+
if (content.includes(`refs/heads/${candidate}`)) {
|
|
3420
|
+
return candidate;
|
|
3421
|
+
}
|
|
3422
|
+
} catch {
|
|
3150
3423
|
}
|
|
3151
|
-
} catch {
|
|
3152
3424
|
}
|
|
3153
3425
|
}
|
|
3154
3426
|
}
|
|
@@ -3173,6 +3445,10 @@ function getBranchOrDefault(repoRoot) {
|
|
|
3173
3445
|
return getCurrentBranch(repoRoot) ?? "default";
|
|
3174
3446
|
}
|
|
3175
3447
|
function getHeadPath(repoRoot) {
|
|
3448
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3449
|
+
if (gitDir) {
|
|
3450
|
+
return path4.join(gitDir, "HEAD");
|
|
3451
|
+
}
|
|
3176
3452
|
return path4.join(repoRoot, ".git", "HEAD");
|
|
3177
3453
|
}
|
|
3178
3454
|
|
|
@@ -3200,6 +3476,7 @@ function isRateLimitError(error) {
|
|
|
3200
3476
|
const message = getErrorMessage(error);
|
|
3201
3477
|
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
3202
3478
|
}
|
|
3479
|
+
var INDEX_METADATA_VERSION = "1";
|
|
3203
3480
|
var Indexer = class {
|
|
3204
3481
|
config;
|
|
3205
3482
|
projectRoot;
|
|
@@ -3208,7 +3485,7 @@ var Indexer = class {
|
|
|
3208
3485
|
invertedIndex = null;
|
|
3209
3486
|
database = null;
|
|
3210
3487
|
provider = null;
|
|
3211
|
-
|
|
3488
|
+
configuredProviderInfo = null;
|
|
3212
3489
|
fileHashCache = /* @__PURE__ */ new Map();
|
|
3213
3490
|
fileHashCachePath = "";
|
|
3214
3491
|
failedBatchesPath = "";
|
|
@@ -3219,12 +3496,15 @@ var Indexer = class {
|
|
|
3219
3496
|
maxQueryCacheSize = 100;
|
|
3220
3497
|
queryCacheTtlMs = 5 * 60 * 1e3;
|
|
3221
3498
|
querySimilarityThreshold = 0.85;
|
|
3499
|
+
indexCompatibility = null;
|
|
3500
|
+
indexingLockPath = "";
|
|
3222
3501
|
constructor(projectRoot, config) {
|
|
3223
3502
|
this.projectRoot = projectRoot;
|
|
3224
3503
|
this.config = config;
|
|
3225
3504
|
this.indexPath = this.getIndexPath();
|
|
3226
3505
|
this.fileHashCachePath = path5.join(this.indexPath, "file-hashes.json");
|
|
3227
3506
|
this.failedBatchesPath = path5.join(this.indexPath, "failed-batches.json");
|
|
3507
|
+
this.indexingLockPath = path5.join(this.indexPath, "indexing.lock");
|
|
3228
3508
|
this.logger = initializeLogger(config.debug);
|
|
3229
3509
|
}
|
|
3230
3510
|
getIndexPath() {
|
|
@@ -3250,7 +3530,36 @@ var Indexer = class {
|
|
|
3250
3530
|
for (const [k, v] of this.fileHashCache) {
|
|
3251
3531
|
obj[k] = v;
|
|
3252
3532
|
}
|
|
3253
|
-
|
|
3533
|
+
this.atomicWriteSync(this.fileHashCachePath, JSON.stringify(obj));
|
|
3534
|
+
}
|
|
3535
|
+
atomicWriteSync(targetPath, data) {
|
|
3536
|
+
const tempPath = `${targetPath}.tmp`;
|
|
3537
|
+
writeFileSync(tempPath, data);
|
|
3538
|
+
renameSync(tempPath, targetPath);
|
|
3539
|
+
}
|
|
3540
|
+
checkForInterruptedIndexing() {
|
|
3541
|
+
return existsSync4(this.indexingLockPath);
|
|
3542
|
+
}
|
|
3543
|
+
acquireIndexingLock() {
|
|
3544
|
+
const lockData = {
|
|
3545
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3546
|
+
pid: process.pid
|
|
3547
|
+
};
|
|
3548
|
+
writeFileSync(this.indexingLockPath, JSON.stringify(lockData));
|
|
3549
|
+
}
|
|
3550
|
+
releaseIndexingLock() {
|
|
3551
|
+
if (existsSync4(this.indexingLockPath)) {
|
|
3552
|
+
unlinkSync(this.indexingLockPath);
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
async recoverFromInterruptedIndexing() {
|
|
3556
|
+
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
3557
|
+
if (existsSync4(this.fileHashCachePath)) {
|
|
3558
|
+
unlinkSync(this.fileHashCachePath);
|
|
3559
|
+
}
|
|
3560
|
+
await this.healthCheck();
|
|
3561
|
+
this.releaseIndexingLock();
|
|
3562
|
+
this.logger.info("Recovery complete, next index will re-process all files");
|
|
3254
3563
|
}
|
|
3255
3564
|
loadFailedBatches() {
|
|
3256
3565
|
try {
|
|
@@ -3293,28 +3602,43 @@ var Indexer = class {
|
|
|
3293
3602
|
return { concurrency: 5, intervalMs: 200, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3294
3603
|
case "ollama":
|
|
3295
3604
|
return { concurrency: 5, intervalMs: 0, minRetryMs: 500, maxRetryMs: 5e3 };
|
|
3605
|
+
case "custom": {
|
|
3606
|
+
const customConfig = this.config.customProvider;
|
|
3607
|
+
return {
|
|
3608
|
+
concurrency: customConfig?.concurrency ?? 3,
|
|
3609
|
+
intervalMs: customConfig?.requestIntervalMs ?? 1e3,
|
|
3610
|
+
minRetryMs: 1e3,
|
|
3611
|
+
maxRetryMs: 3e4
|
|
3612
|
+
};
|
|
3613
|
+
}
|
|
3296
3614
|
default:
|
|
3297
3615
|
return { concurrency: 3, intervalMs: 1e3, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3298
3616
|
}
|
|
3299
3617
|
}
|
|
3300
3618
|
async initialize() {
|
|
3301
|
-
|
|
3302
|
-
|
|
3619
|
+
if (this.config.embeddingProvider === "custom") {
|
|
3620
|
+
if (!this.config.customProvider) {
|
|
3621
|
+
throw new Error("embeddingProvider is 'custom' but customProvider config is missing.");
|
|
3622
|
+
}
|
|
3623
|
+
this.configuredProviderInfo = createCustomProviderInfo(this.config.customProvider);
|
|
3624
|
+
} else if (this.config.embeddingProvider === "auto") {
|
|
3625
|
+
this.configuredProviderInfo = await tryDetectProvider();
|
|
3626
|
+
} else {
|
|
3627
|
+
this.configuredProviderInfo = await detectEmbeddingProvider(this.config.embeddingProvider, this.config.embeddingModel);
|
|
3628
|
+
}
|
|
3629
|
+
if (!this.configuredProviderInfo) {
|
|
3303
3630
|
throw new Error(
|
|
3304
|
-
"No embedding provider available. Configure GitHub, OpenAI, Google, or
|
|
3631
|
+
"No embedding provider available. Configure GitHub Copilot, OpenAI, Google, Ollama, or a custom OpenAI-compatible endpoint."
|
|
3305
3632
|
);
|
|
3306
3633
|
}
|
|
3307
3634
|
this.logger.info("Initializing indexer", {
|
|
3308
|
-
provider: this.
|
|
3309
|
-
model: this.
|
|
3635
|
+
provider: this.configuredProviderInfo.provider,
|
|
3636
|
+
model: this.configuredProviderInfo.modelInfo.model,
|
|
3310
3637
|
scope: this.config.scope
|
|
3311
3638
|
});
|
|
3312
|
-
this.provider = createEmbeddingProvider(
|
|
3313
|
-
this.detectedProvider.credentials,
|
|
3314
|
-
this.detectedProvider.modelInfo
|
|
3315
|
-
);
|
|
3639
|
+
this.provider = createEmbeddingProvider(this.configuredProviderInfo);
|
|
3316
3640
|
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
3317
|
-
const dimensions = this.
|
|
3641
|
+
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
3318
3642
|
const storePath = path5.join(this.indexPath, "vectors");
|
|
3319
3643
|
this.store = new VectorStore(storePath, dimensions);
|
|
3320
3644
|
const indexFilePath = path5.join(this.indexPath, "vectors.usearch");
|
|
@@ -3334,9 +3658,20 @@ var Indexer = class {
|
|
|
3334
3658
|
const dbPath = path5.join(this.indexPath, "codebase.db");
|
|
3335
3659
|
const dbIsNew = !existsSync4(dbPath);
|
|
3336
3660
|
this.database = new Database(dbPath);
|
|
3661
|
+
if (this.checkForInterruptedIndexing()) {
|
|
3662
|
+
await this.recoverFromInterruptedIndexing();
|
|
3663
|
+
}
|
|
3337
3664
|
if (dbIsNew && this.store.count() > 0) {
|
|
3338
3665
|
this.migrateFromLegacyIndex();
|
|
3339
3666
|
}
|
|
3667
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3668
|
+
if (!this.indexCompatibility.compatible) {
|
|
3669
|
+
this.logger.warn("Index compatibility issue detected", {
|
|
3670
|
+
reason: this.indexCompatibility.reason,
|
|
3671
|
+
storedMetadata: this.indexCompatibility.storedMetadata,
|
|
3672
|
+
configuredProviderInfo: this.configuredProviderInfo
|
|
3673
|
+
});
|
|
3674
|
+
}
|
|
3340
3675
|
if (isGitRepo(this.projectRoot)) {
|
|
3341
3676
|
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
3342
3677
|
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
@@ -3407,30 +3742,106 @@ var Indexer = class {
|
|
|
3407
3742
|
}
|
|
3408
3743
|
this.database.addChunksToBranchBatch(this.currentBranch || "default", chunkIds);
|
|
3409
3744
|
}
|
|
3745
|
+
loadIndexMetadata() {
|
|
3746
|
+
if (!this.database) return null;
|
|
3747
|
+
const version = this.database.getMetadata("index.version");
|
|
3748
|
+
if (!version) return null;
|
|
3749
|
+
return {
|
|
3750
|
+
indexVersion: version,
|
|
3751
|
+
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
3752
|
+
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
3753
|
+
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
3754
|
+
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
3755
|
+
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
3756
|
+
};
|
|
3757
|
+
}
|
|
3758
|
+
saveIndexMetadata(provider) {
|
|
3759
|
+
if (!this.database) return;
|
|
3760
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3761
|
+
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
3762
|
+
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
3763
|
+
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
3764
|
+
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
3765
|
+
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
3766
|
+
this.database.setMetadata("index.updatedAt", now);
|
|
3767
|
+
if (!existingCreatedAt) {
|
|
3768
|
+
this.database.setMetadata("index.createdAt", now);
|
|
3769
|
+
}
|
|
3770
|
+
}
|
|
3771
|
+
validateIndexCompatibility(provider) {
|
|
3772
|
+
const storedMetadata = this.loadIndexMetadata();
|
|
3773
|
+
if (!storedMetadata) {
|
|
3774
|
+
return { compatible: true };
|
|
3775
|
+
}
|
|
3776
|
+
const currentProvider = provider.provider;
|
|
3777
|
+
const currentModel = provider.modelInfo.model;
|
|
3778
|
+
const currentDimensions = provider.modelInfo.dimensions;
|
|
3779
|
+
if (storedMetadata.embeddingDimensions !== currentDimensions) {
|
|
3780
|
+
return {
|
|
3781
|
+
compatible: false,
|
|
3782
|
+
code: "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
|
|
3783
|
+
reason: `Dimension mismatch: index has ${storedMetadata.embeddingDimensions}D vectors (${storedMetadata.embeddingProvider}/${storedMetadata.embeddingModel}), but current provider uses ${currentDimensions}D (${currentProvider}/${currentModel}). Run index_codebase with force=true to rebuild.`,
|
|
3784
|
+
storedMetadata
|
|
3785
|
+
};
|
|
3786
|
+
}
|
|
3787
|
+
if (storedMetadata.embeddingModel !== currentModel) {
|
|
3788
|
+
return {
|
|
3789
|
+
compatible: false,
|
|
3790
|
+
code: "MODEL_MISMATCH" /* MODEL_MISMATCH */,
|
|
3791
|
+
reason: `Model mismatch: index was built with "${storedMetadata.embeddingModel}", but current model is "${currentModel}". Embeddings are incompatible. Run index_codebase with force=true to rebuild.`,
|
|
3792
|
+
storedMetadata
|
|
3793
|
+
};
|
|
3794
|
+
}
|
|
3795
|
+
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
3796
|
+
this.logger.warn("Provider changed", {
|
|
3797
|
+
storedProvider: storedMetadata.embeddingProvider,
|
|
3798
|
+
currentProvider
|
|
3799
|
+
});
|
|
3800
|
+
}
|
|
3801
|
+
return {
|
|
3802
|
+
compatible: true,
|
|
3803
|
+
storedMetadata
|
|
3804
|
+
};
|
|
3805
|
+
}
|
|
3806
|
+
checkCompatibility() {
|
|
3807
|
+
if (!this.indexCompatibility) {
|
|
3808
|
+
if (!this.configuredProviderInfo) {
|
|
3809
|
+
throw new Error("No embedding provider info, you must initialize the indexer first.");
|
|
3810
|
+
}
|
|
3811
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3812
|
+
}
|
|
3813
|
+
return this.indexCompatibility;
|
|
3814
|
+
}
|
|
3410
3815
|
async ensureInitialized() {
|
|
3411
|
-
if (!this.store || !this.provider || !this.invertedIndex || !this.
|
|
3816
|
+
if (!this.store || !this.provider || !this.invertedIndex || !this.configuredProviderInfo || !this.database) {
|
|
3412
3817
|
await this.initialize();
|
|
3413
3818
|
}
|
|
3414
3819
|
return {
|
|
3415
3820
|
store: this.store,
|
|
3416
3821
|
provider: this.provider,
|
|
3417
3822
|
invertedIndex: this.invertedIndex,
|
|
3418
|
-
|
|
3823
|
+
configuredProviderInfo: this.configuredProviderInfo,
|
|
3419
3824
|
database: this.database
|
|
3420
3825
|
};
|
|
3421
3826
|
}
|
|
3422
3827
|
async estimateCost() {
|
|
3423
|
-
const {
|
|
3828
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
3424
3829
|
const { files } = await collectFiles(
|
|
3425
3830
|
this.projectRoot,
|
|
3426
3831
|
this.config.include,
|
|
3427
3832
|
this.config.exclude,
|
|
3428
3833
|
this.config.indexing.maxFileSize
|
|
3429
3834
|
);
|
|
3430
|
-
return createCostEstimate(files,
|
|
3835
|
+
return createCostEstimate(files, configuredProviderInfo);
|
|
3431
3836
|
}
|
|
3432
3837
|
async index(onProgress) {
|
|
3433
|
-
const { store, provider, invertedIndex, database,
|
|
3838
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
3839
|
+
if (!this.indexCompatibility?.compatible) {
|
|
3840
|
+
throw new Error(
|
|
3841
|
+
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
3842
|
+
);
|
|
3843
|
+
}
|
|
3844
|
+
this.acquireIndexingLock();
|
|
3434
3845
|
this.logger.recordIndexingStart();
|
|
3435
3846
|
this.logger.info("Starting indexing", { projectRoot: this.projectRoot });
|
|
3436
3847
|
const startTime = Date.now();
|
|
@@ -3537,7 +3948,7 @@ var Indexer = class {
|
|
|
3537
3948
|
const id = generateChunkId(parsed.path, chunk);
|
|
3538
3949
|
const contentHash = generateChunkHash(chunk);
|
|
3539
3950
|
currentChunkIds.add(id);
|
|
3540
|
-
|
|
3951
|
+
chunkDataBatch.push({
|
|
3541
3952
|
chunkId: id,
|
|
3542
3953
|
contentHash,
|
|
3543
3954
|
filePath: parsed.path,
|
|
@@ -3546,8 +3957,7 @@ var Indexer = class {
|
|
|
3546
3957
|
nodeType: chunk.chunkType,
|
|
3547
3958
|
name: chunk.name,
|
|
3548
3959
|
language: chunk.language
|
|
3549
|
-
};
|
|
3550
|
-
chunkDataBatch.push(chunkData);
|
|
3960
|
+
});
|
|
3551
3961
|
if (existingChunks.get(id) === contentHash) {
|
|
3552
3962
|
fileChunkCount++;
|
|
3553
3963
|
continue;
|
|
@@ -3600,6 +4010,7 @@ var Indexer = class {
|
|
|
3600
4010
|
chunksProcessed: 0,
|
|
3601
4011
|
totalChunks: 0
|
|
3602
4012
|
});
|
|
4013
|
+
this.releaseIndexingLock();
|
|
3603
4014
|
return stats;
|
|
3604
4015
|
}
|
|
3605
4016
|
if (pendingChunks.length === 0) {
|
|
@@ -3617,6 +4028,7 @@ var Indexer = class {
|
|
|
3617
4028
|
chunksProcessed: 0,
|
|
3618
4029
|
totalChunks: 0
|
|
3619
4030
|
});
|
|
4031
|
+
this.releaseIndexingLock();
|
|
3620
4032
|
return stats;
|
|
3621
4033
|
}
|
|
3622
4034
|
onProgress?.({
|
|
@@ -3645,7 +4057,7 @@ var Indexer = class {
|
|
|
3645
4057
|
stats.indexedChunks++;
|
|
3646
4058
|
}
|
|
3647
4059
|
}
|
|
3648
|
-
const providerRateLimits = this.getProviderRateLimits(
|
|
4060
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
3649
4061
|
const queue = new PQueue({
|
|
3650
4062
|
concurrency: providerRateLimits.concurrency,
|
|
3651
4063
|
interval: providerRateLimits.intervalMs,
|
|
@@ -3656,7 +4068,7 @@ var Indexer = class {
|
|
|
3656
4068
|
for (const batch of dynamicBatches) {
|
|
3657
4069
|
queue.add(async () => {
|
|
3658
4070
|
if (rateLimitBackoffMs > 0) {
|
|
3659
|
-
await new Promise((
|
|
4071
|
+
await new Promise((resolve5) => setTimeout(resolve5, rateLimitBackoffMs));
|
|
3660
4072
|
}
|
|
3661
4073
|
try {
|
|
3662
4074
|
const result = await pRetry(
|
|
@@ -3669,6 +4081,7 @@ var Indexer = class {
|
|
|
3669
4081
|
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
3670
4082
|
maxTimeout: providerRateLimits.maxRetryMs,
|
|
3671
4083
|
factor: 2,
|
|
4084
|
+
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError),
|
|
3672
4085
|
onFailedAttempt: (error) => {
|
|
3673
4086
|
const message = getErrorMessage(error);
|
|
3674
4087
|
if (isRateLimitError(error)) {
|
|
@@ -3700,7 +4113,7 @@ var Indexer = class {
|
|
|
3700
4113
|
contentHash: chunk.contentHash,
|
|
3701
4114
|
embedding: float32ArrayToBuffer(result.embeddings[i]),
|
|
3702
4115
|
chunkText: chunk.text,
|
|
3703
|
-
model:
|
|
4116
|
+
model: configuredProviderInfo.modelInfo.model
|
|
3704
4117
|
}));
|
|
3705
4118
|
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
3706
4119
|
for (const chunk of batch) {
|
|
@@ -3751,6 +4164,8 @@ var Indexer = class {
|
|
|
3751
4164
|
await this.maybeRunOrphanGc();
|
|
3752
4165
|
}
|
|
3753
4166
|
stats.durationMs = Date.now() - startTime;
|
|
4167
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
4168
|
+
this.indexCompatibility = { compatible: true };
|
|
3754
4169
|
this.logger.recordIndexingEnd();
|
|
3755
4170
|
this.logger.info("Indexing complete", {
|
|
3756
4171
|
files: stats.totalFiles,
|
|
@@ -3771,6 +4186,7 @@ var Indexer = class {
|
|
|
3771
4186
|
chunksProcessed: stats.indexedChunks,
|
|
3772
4187
|
totalChunks: pendingChunks.length
|
|
3773
4188
|
});
|
|
4189
|
+
this.releaseIndexingLock();
|
|
3774
4190
|
return stats;
|
|
3775
4191
|
}
|
|
3776
4192
|
async getQueryEmbedding(query, provider) {
|
|
@@ -3793,7 +4209,7 @@ var Indexer = class {
|
|
|
3793
4209
|
}
|
|
3794
4210
|
this.logger.cache("debug", "Query embedding cache miss", { query: query.slice(0, 50) });
|
|
3795
4211
|
this.logger.recordQueryCacheMiss();
|
|
3796
|
-
const { embedding, tokensUsed } = await provider.
|
|
4212
|
+
const { embedding, tokensUsed } = await provider.embedQuery(query);
|
|
3797
4213
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
3798
4214
|
if (this.queryEmbeddingCache.size >= this.maxQueryCacheSize) {
|
|
3799
4215
|
const oldestKey = this.queryEmbeddingCache.keys().next().value;
|
|
@@ -3836,8 +4252,14 @@ var Indexer = class {
|
|
|
3836
4252
|
return intersection / union;
|
|
3837
4253
|
}
|
|
3838
4254
|
async search(query, limit, options) {
|
|
3839
|
-
const searchStartTime = performance2.now();
|
|
3840
4255
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4256
|
+
const compatibility = this.checkCompatibility();
|
|
4257
|
+
if (!compatibility.compatible) {
|
|
4258
|
+
throw new Error(
|
|
4259
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} A possible solution is to run index_codebase with force=true to rebuild the index.`
|
|
4260
|
+
);
|
|
4261
|
+
}
|
|
4262
|
+
const searchStartTime = performance2.now();
|
|
3841
4263
|
if (store.count() === 0) {
|
|
3842
4264
|
this.logger.search("debug", "Search on empty index", { query });
|
|
3843
4265
|
return [];
|
|
@@ -3979,15 +4401,16 @@ var Indexer = class {
|
|
|
3979
4401
|
return results.slice(0, limit);
|
|
3980
4402
|
}
|
|
3981
4403
|
async getStatus() {
|
|
3982
|
-
const { store,
|
|
4404
|
+
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
3983
4405
|
return {
|
|
3984
4406
|
indexed: store.count() > 0,
|
|
3985
4407
|
vectorCount: store.count(),
|
|
3986
|
-
provider:
|
|
3987
|
-
model:
|
|
4408
|
+
provider: configuredProviderInfo.provider,
|
|
4409
|
+
model: configuredProviderInfo.modelInfo.model,
|
|
3988
4410
|
indexPath: this.indexPath,
|
|
3989
4411
|
currentBranch: this.currentBranch,
|
|
3990
|
-
baseBranch: this.baseBranch
|
|
4412
|
+
baseBranch: this.baseBranch,
|
|
4413
|
+
compatibility: this.indexCompatibility
|
|
3991
4414
|
};
|
|
3992
4415
|
}
|
|
3993
4416
|
async clearIndex() {
|
|
@@ -3999,6 +4422,13 @@ var Indexer = class {
|
|
|
3999
4422
|
this.fileHashCache.clear();
|
|
4000
4423
|
this.saveFileHashCache();
|
|
4001
4424
|
database.clearBranch(this.currentBranch);
|
|
4425
|
+
database.deleteMetadata("index.version");
|
|
4426
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
4427
|
+
database.deleteMetadata("index.embeddingModel");
|
|
4428
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
4429
|
+
database.deleteMetadata("index.createdAt");
|
|
4430
|
+
database.deleteMetadata("index.updatedAt");
|
|
4431
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
4002
4432
|
}
|
|
4003
4433
|
async healthCheck() {
|
|
4004
4434
|
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
@@ -4112,26 +4542,31 @@ var Indexer = class {
|
|
|
4112
4542
|
getLogger() {
|
|
4113
4543
|
return this.logger;
|
|
4114
4544
|
}
|
|
4115
|
-
async findSimilar(code, limit, options) {
|
|
4116
|
-
const searchStartTime = performance2.now();
|
|
4545
|
+
async findSimilar(code, limit = this.config.search.maxResults, options) {
|
|
4117
4546
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4547
|
+
const compatibility = this.checkCompatibility();
|
|
4548
|
+
if (!compatibility.compatible) {
|
|
4549
|
+
throw new Error(
|
|
4550
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} Run index_codebase with force=true to rebuild the index.`
|
|
4551
|
+
);
|
|
4552
|
+
}
|
|
4553
|
+
const searchStartTime = performance2.now();
|
|
4118
4554
|
if (store.count() === 0) {
|
|
4119
4555
|
this.logger.search("debug", "Find similar on empty index");
|
|
4120
4556
|
return [];
|
|
4121
4557
|
}
|
|
4122
|
-
const maxResults = limit ?? this.config.search.maxResults;
|
|
4123
4558
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
4124
4559
|
this.logger.search("debug", "Starting find similar", {
|
|
4125
4560
|
codeLength: code.length,
|
|
4126
|
-
|
|
4561
|
+
limit,
|
|
4127
4562
|
filterByBranch
|
|
4128
4563
|
});
|
|
4129
4564
|
const embeddingStartTime = performance2.now();
|
|
4130
|
-
const { embedding, tokensUsed } = await provider.
|
|
4565
|
+
const { embedding, tokensUsed } = await provider.embedDocument(code);
|
|
4131
4566
|
const embeddingMs = performance2.now() - embeddingStartTime;
|
|
4132
4567
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
4133
4568
|
const vectorStartTime = performance2.now();
|
|
4134
|
-
const semanticResults = store.search(embedding,
|
|
4569
|
+
const semanticResults = store.search(embedding, limit * 2);
|
|
4135
4570
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
4136
4571
|
let branchChunkIds = null;
|
|
4137
4572
|
if (filterByBranch && this.currentBranch !== "default") {
|
|
@@ -4155,7 +4590,7 @@ var Indexer = class {
|
|
|
4155
4590
|
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
4156
4591
|
}
|
|
4157
4592
|
return true;
|
|
4158
|
-
}).slice(0,
|
|
4593
|
+
}).slice(0, limit);
|
|
4159
4594
|
const totalSearchMs = performance2.now() - searchStartTime;
|
|
4160
4595
|
this.logger.recordSearch(totalSearchMs, {
|
|
4161
4596
|
embeddingMs,
|
|
@@ -5070,7 +5505,7 @@ var NodeFsHandler = class {
|
|
|
5070
5505
|
this._addToNodeFs(path9, initialAdd, wh, depth + 1);
|
|
5071
5506
|
}
|
|
5072
5507
|
}).on(EV.ERROR, this._boundHandleError);
|
|
5073
|
-
return new Promise((
|
|
5508
|
+
return new Promise((resolve5, reject) => {
|
|
5074
5509
|
if (!stream)
|
|
5075
5510
|
return reject();
|
|
5076
5511
|
stream.once(STR_END, () => {
|
|
@@ -5079,7 +5514,7 @@ var NodeFsHandler = class {
|
|
|
5079
5514
|
return;
|
|
5080
5515
|
}
|
|
5081
5516
|
const wasThrottled = throttler ? throttler.clear() : false;
|
|
5082
|
-
|
|
5517
|
+
resolve5(void 0);
|
|
5083
5518
|
previous.getChildren().filter((item) => {
|
|
5084
5519
|
return item !== directory && !current.has(item);
|
|
5085
5520
|
}).forEach((item) => {
|
|
@@ -6126,7 +6561,8 @@ function createWatcherWithIndexer(indexer, projectRoot, config) {
|
|
|
6126
6561
|
|
|
6127
6562
|
// src/tools/index.ts
|
|
6128
6563
|
import { tool } from "@opencode-ai/plugin";
|
|
6129
|
-
|
|
6564
|
+
|
|
6565
|
+
// src/tools/utils.ts
|
|
6130
6566
|
var MAX_CONTENT_LINES = 30;
|
|
6131
6567
|
function truncateContent(content) {
|
|
6132
6568
|
const lines = content.split("\n");
|
|
@@ -6134,49 +6570,177 @@ function truncateContent(content) {
|
|
|
6134
6570
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
6135
6571
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
6136
6572
|
}
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
}
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
}
|
|
6147
|
-
var codebase_search = tool({
|
|
6148
|
-
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6149
|
-
args: {
|
|
6150
|
-
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6151
|
-
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6152
|
-
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6153
|
-
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6154
|
-
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6155
|
-
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6156
|
-
},
|
|
6157
|
-
async execute(args) {
|
|
6158
|
-
const indexer = getIndexer();
|
|
6159
|
-
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6160
|
-
fileType: args.fileType,
|
|
6161
|
-
directory: args.directory,
|
|
6162
|
-
chunkType: args.chunkType,
|
|
6163
|
-
contextLines: args.contextLines
|
|
6164
|
-
});
|
|
6165
|
-
if (results.length === 0) {
|
|
6166
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6573
|
+
function formatIndexStats(stats, verbose = false) {
|
|
6574
|
+
const lines = [];
|
|
6575
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
6576
|
+
lines.push(`Indexed. ${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
6577
|
+
} else if (stats.indexedChunks === 0) {
|
|
6578
|
+
lines.push(`Indexed. ${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
6579
|
+
} else {
|
|
6580
|
+
let main = `Indexed. ${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
6581
|
+
if (stats.existingChunks > 0) {
|
|
6582
|
+
main += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
6167
6583
|
}
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
|
|
6584
|
+
lines.push(main);
|
|
6585
|
+
if (stats.removedChunks > 0) {
|
|
6586
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
6587
|
+
}
|
|
6588
|
+
if (stats.failedChunks > 0) {
|
|
6589
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
6590
|
+
}
|
|
6591
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
6592
|
+
}
|
|
6593
|
+
if (verbose) {
|
|
6594
|
+
if (stats.skippedFiles.length > 0) {
|
|
6595
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6596
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6597
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6598
|
+
lines.push("");
|
|
6599
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6600
|
+
if (tooLarge.length > 0) {
|
|
6601
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6602
|
+
}
|
|
6603
|
+
if (excluded.length > 0) {
|
|
6604
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6605
|
+
}
|
|
6606
|
+
if (gitignored.length > 0) {
|
|
6607
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6608
|
+
}
|
|
6609
|
+
}
|
|
6610
|
+
if (stats.parseFailures.length > 0) {
|
|
6611
|
+
lines.push("");
|
|
6612
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6613
|
+
}
|
|
6614
|
+
}
|
|
6615
|
+
return lines.join("\n");
|
|
6616
|
+
}
|
|
6617
|
+
function formatStatus(status) {
|
|
6618
|
+
if (!status.indexed) {
|
|
6619
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6620
|
+
}
|
|
6621
|
+
const lines = [
|
|
6622
|
+
`Index status:`,
|
|
6623
|
+
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6624
|
+
` Provider: ${status.provider}`,
|
|
6625
|
+
` Model: ${status.model}`,
|
|
6626
|
+
` Location: ${status.indexPath}`
|
|
6627
|
+
];
|
|
6628
|
+
if (status.currentBranch !== "default") {
|
|
6629
|
+
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6630
|
+
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6631
|
+
}
|
|
6632
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
6633
|
+
lines.push("");
|
|
6634
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
6635
|
+
if (status.compatibility.storedMetadata) {
|
|
6636
|
+
const stored = status.compatibility.storedMetadata;
|
|
6637
|
+
lines.push(` Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
6638
|
+
lines.push(` Current config: ${status.provider}/${status.model}`);
|
|
6639
|
+
}
|
|
6640
|
+
} else if (!status.compatibility) {
|
|
6641
|
+
lines.push(` Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
6642
|
+
} else {
|
|
6643
|
+
lines.push(` Compatibility: Index is compatible with the current provider and model.`);
|
|
6644
|
+
}
|
|
6645
|
+
return lines.join("\n");
|
|
6646
|
+
}
|
|
6647
|
+
function formatProgressTitle(progress) {
|
|
6648
|
+
switch (progress.phase) {
|
|
6649
|
+
case "scanning":
|
|
6650
|
+
return "Scanning files...";
|
|
6651
|
+
case "parsing":
|
|
6652
|
+
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6653
|
+
case "embedding":
|
|
6654
|
+
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6655
|
+
case "storing":
|
|
6656
|
+
return "Storing index...";
|
|
6657
|
+
case "complete":
|
|
6658
|
+
return "Indexing complete";
|
|
6659
|
+
default:
|
|
6660
|
+
return "Indexing...";
|
|
6661
|
+
}
|
|
6662
|
+
}
|
|
6663
|
+
function calculatePercentage(progress) {
|
|
6664
|
+
if (progress.phase === "scanning") return 0;
|
|
6665
|
+
if (progress.phase === "complete") return 100;
|
|
6666
|
+
if (progress.phase === "parsing") {
|
|
6667
|
+
if (progress.totalFiles === 0) return 5;
|
|
6668
|
+
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6669
|
+
}
|
|
6670
|
+
if (progress.phase === "embedding") {
|
|
6671
|
+
if (progress.totalChunks === 0) return 20;
|
|
6672
|
+
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6673
|
+
}
|
|
6674
|
+
if (progress.phase === "storing") return 95;
|
|
6675
|
+
return 0;
|
|
6676
|
+
}
|
|
6677
|
+
function formatCodebasePeek(results, query) {
|
|
6678
|
+
if (results.length === 0) {
|
|
6679
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6680
|
+
}
|
|
6681
|
+
const formatted = results.map((r, idx) => {
|
|
6682
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6683
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6684
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6685
|
+
});
|
|
6686
|
+
return `Found ${results.length} locations for "${query}":
|
|
6687
|
+
|
|
6688
|
+
${formatted.join("\n")}
|
|
6689
|
+
|
|
6690
|
+
Use Read tool to examine specific files.`;
|
|
6691
|
+
}
|
|
6692
|
+
function formatHealthCheck(result) {
|
|
6693
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0) {
|
|
6694
|
+
return "Index is healthy. No stale entries found.";
|
|
6695
|
+
}
|
|
6696
|
+
const lines = [`Health check complete:`];
|
|
6697
|
+
if (result.removed > 0) {
|
|
6698
|
+
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6699
|
+
}
|
|
6700
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
6701
|
+
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6702
|
+
}
|
|
6703
|
+
if (result.gcOrphanChunks > 0) {
|
|
6704
|
+
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6705
|
+
}
|
|
6706
|
+
if (result.filePaths.length > 0) {
|
|
6707
|
+
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6708
|
+
}
|
|
6709
|
+
return lines.join("\n");
|
|
6710
|
+
}
|
|
6711
|
+
function formatLogs(logs) {
|
|
6712
|
+
if (logs.length === 0) {
|
|
6713
|
+
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6714
|
+
}
|
|
6715
|
+
return logs.map((l) => {
|
|
6716
|
+
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6717
|
+
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6718
|
+
}).join("\n");
|
|
6719
|
+
}
|
|
6720
|
+
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
6721
|
+
const formatted = results.map((r, idx) => {
|
|
6722
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6723
|
+
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
6724
|
+
return `${header} ${scoreLabel}
|
|
6171
6725
|
\`\`\`
|
|
6172
6726
|
${truncateContent(r.content)}
|
|
6173
6727
|
\`\`\``;
|
|
6174
|
-
|
|
6175
|
-
|
|
6728
|
+
});
|
|
6729
|
+
return formatted.join("\n\n");
|
|
6730
|
+
}
|
|
6176
6731
|
|
|
6177
|
-
|
|
6732
|
+
// src/tools/index.ts
|
|
6733
|
+
var z = tool.schema;
|
|
6734
|
+
var sharedIndexer = null;
|
|
6735
|
+
function initializeTools(projectRoot, config) {
|
|
6736
|
+
sharedIndexer = new Indexer(projectRoot, config);
|
|
6737
|
+
}
|
|
6738
|
+
function getIndexer() {
|
|
6739
|
+
if (!sharedIndexer) {
|
|
6740
|
+
throw new Error("Codebase index tools not initialized. Plugin may not be loaded correctly.");
|
|
6178
6741
|
}
|
|
6179
|
-
|
|
6742
|
+
return sharedIndexer;
|
|
6743
|
+
}
|
|
6180
6744
|
var codebase_peek = tool({
|
|
6181
6745
|
description: "Quick lookup of code locations by meaning. Returns only metadata (file, line, name, type) WITHOUT code content. Use this first to find WHERE code is, then use Read tool to examine specific files. Saves tokens by not returning full code blocks. Best for: discovery, navigation, finding multiple related locations.",
|
|
6182
6746
|
args: {
|
|
@@ -6194,19 +6758,7 @@ var codebase_peek = tool({
|
|
|
6194
6758
|
chunkType: args.chunkType,
|
|
6195
6759
|
metadataOnly: true
|
|
6196
6760
|
});
|
|
6197
|
-
|
|
6198
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6199
|
-
}
|
|
6200
|
-
const formatted = results.map((r, idx) => {
|
|
6201
|
-
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6202
|
-
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6203
|
-
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6204
|
-
});
|
|
6205
|
-
return `Found ${results.length} locations for "${args.query}":
|
|
6206
|
-
|
|
6207
|
-
${formatted.join("\n")}
|
|
6208
|
-
|
|
6209
|
-
Use Read tool to examine specific files.`;
|
|
6761
|
+
return formatCodebasePeek(results, args.query);
|
|
6210
6762
|
}
|
|
6211
6763
|
});
|
|
6212
6764
|
var index_codebase = tool({
|
|
@@ -6256,23 +6808,7 @@ var index_health_check = tool({
|
|
|
6256
6808
|
async execute() {
|
|
6257
6809
|
const indexer = getIndexer();
|
|
6258
6810
|
const result = await indexer.healthCheck();
|
|
6259
|
-
|
|
6260
|
-
return "Index is healthy. No stale entries found.";
|
|
6261
|
-
}
|
|
6262
|
-
const lines = [`Health check complete:`];
|
|
6263
|
-
if (result.removed > 0) {
|
|
6264
|
-
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6265
|
-
}
|
|
6266
|
-
if (result.gcOrphanEmbeddings > 0) {
|
|
6267
|
-
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6268
|
-
}
|
|
6269
|
-
if (result.gcOrphanChunks > 0) {
|
|
6270
|
-
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6271
|
-
}
|
|
6272
|
-
if (result.filePaths.length > 0) {
|
|
6273
|
-
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6274
|
-
}
|
|
6275
|
-
return lines.join("\n");
|
|
6811
|
+
return formatHealthCheck(result);
|
|
6276
6812
|
}
|
|
6277
6813
|
});
|
|
6278
6814
|
var index_metrics = tool({
|
|
@@ -6311,13 +6847,7 @@ var index_logs = tool({
|
|
|
6311
6847
|
} else {
|
|
6312
6848
|
logs = logger.getLogs(args.limit);
|
|
6313
6849
|
}
|
|
6314
|
-
|
|
6315
|
-
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6316
|
-
}
|
|
6317
|
-
return logs.map((l) => {
|
|
6318
|
-
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6319
|
-
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6320
|
-
}).join("\n");
|
|
6850
|
+
return formatLogs(logs);
|
|
6321
6851
|
}
|
|
6322
6852
|
});
|
|
6323
6853
|
var find_similar = tool({
|
|
@@ -6332,7 +6862,7 @@ var find_similar = tool({
|
|
|
6332
6862
|
},
|
|
6333
6863
|
async execute(args) {
|
|
6334
6864
|
const indexer = getIndexer();
|
|
6335
|
-
const results = await indexer.findSimilar(args.code, args.limit
|
|
6865
|
+
const results = await indexer.findSimilar(args.code, args.limit, {
|
|
6336
6866
|
fileType: args.fileType,
|
|
6337
6867
|
directory: args.directory,
|
|
6338
6868
|
chunkType: args.chunkType,
|
|
@@ -6341,109 +6871,37 @@ var find_similar = tool({
|
|
|
6341
6871
|
if (results.length === 0) {
|
|
6342
6872
|
return "No similar code found. Try a different snippet or run index_codebase first.";
|
|
6343
6873
|
}
|
|
6344
|
-
const formatted = results.map((r, idx) => {
|
|
6345
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6346
|
-
return `${header} (similarity: ${(r.score * 100).toFixed(1)}%)
|
|
6347
|
-
\`\`\`
|
|
6348
|
-
${truncateContent(r.content)}
|
|
6349
|
-
\`\`\``;
|
|
6350
|
-
});
|
|
6351
6874
|
return `Found ${results.length} similar code blocks:
|
|
6352
6875
|
|
|
6353
|
-
${
|
|
6876
|
+
${formatSearchResults(results)}`;
|
|
6354
6877
|
}
|
|
6355
6878
|
});
|
|
6356
|
-
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
}
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
if (verbose) {
|
|
6377
|
-
if (stats.skippedFiles.length > 0) {
|
|
6378
|
-
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6379
|
-
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6380
|
-
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6381
|
-
lines.push("");
|
|
6382
|
-
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6383
|
-
if (tooLarge.length > 0) {
|
|
6384
|
-
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6385
|
-
}
|
|
6386
|
-
if (excluded.length > 0) {
|
|
6387
|
-
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6388
|
-
}
|
|
6389
|
-
if (gitignored.length > 0) {
|
|
6390
|
-
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6391
|
-
}
|
|
6392
|
-
}
|
|
6393
|
-
if (stats.parseFailures.length > 0) {
|
|
6394
|
-
lines.push("");
|
|
6395
|
-
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6879
|
+
var codebase_search = tool({
|
|
6880
|
+
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6881
|
+
args: {
|
|
6882
|
+
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6883
|
+
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6884
|
+
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6885
|
+
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6886
|
+
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6887
|
+
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6888
|
+
},
|
|
6889
|
+
async execute(args) {
|
|
6890
|
+
const indexer = getIndexer();
|
|
6891
|
+
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6892
|
+
fileType: args.fileType,
|
|
6893
|
+
directory: args.directory,
|
|
6894
|
+
chunkType: args.chunkType,
|
|
6895
|
+
contextLines: args.contextLines
|
|
6896
|
+
});
|
|
6897
|
+
if (results.length === 0) {
|
|
6898
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6396
6899
|
}
|
|
6900
|
+
return `Found ${results.length} results for "${args.query}":
|
|
6901
|
+
|
|
6902
|
+
${formatSearchResults(results, "score")}`;
|
|
6397
6903
|
}
|
|
6398
|
-
|
|
6399
|
-
}
|
|
6400
|
-
function formatStatus(status) {
|
|
6401
|
-
if (!status.indexed) {
|
|
6402
|
-
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6403
|
-
}
|
|
6404
|
-
const lines = [
|
|
6405
|
-
`Index status:`,
|
|
6406
|
-
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6407
|
-
` Provider: ${status.provider}`,
|
|
6408
|
-
` Model: ${status.model}`,
|
|
6409
|
-
` Location: ${status.indexPath}`
|
|
6410
|
-
];
|
|
6411
|
-
if (status.currentBranch !== "default") {
|
|
6412
|
-
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6413
|
-
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6414
|
-
}
|
|
6415
|
-
return lines.join("\n");
|
|
6416
|
-
}
|
|
6417
|
-
function formatProgressTitle(progress) {
|
|
6418
|
-
switch (progress.phase) {
|
|
6419
|
-
case "scanning":
|
|
6420
|
-
return "Scanning files...";
|
|
6421
|
-
case "parsing":
|
|
6422
|
-
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6423
|
-
case "embedding":
|
|
6424
|
-
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6425
|
-
case "storing":
|
|
6426
|
-
return "Storing index...";
|
|
6427
|
-
case "complete":
|
|
6428
|
-
return "Indexing complete";
|
|
6429
|
-
default:
|
|
6430
|
-
return "Indexing...";
|
|
6431
|
-
}
|
|
6432
|
-
}
|
|
6433
|
-
function calculatePercentage(progress) {
|
|
6434
|
-
if (progress.phase === "scanning") return 0;
|
|
6435
|
-
if (progress.phase === "complete") return 100;
|
|
6436
|
-
if (progress.phase === "parsing") {
|
|
6437
|
-
if (progress.totalFiles === 0) return 5;
|
|
6438
|
-
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6439
|
-
}
|
|
6440
|
-
if (progress.phase === "embedding") {
|
|
6441
|
-
if (progress.totalChunks === 0) return 20;
|
|
6442
|
-
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6443
|
-
}
|
|
6444
|
-
if (progress.phase === "storing") return 95;
|
|
6445
|
-
return 0;
|
|
6446
|
-
}
|
|
6904
|
+
});
|
|
6447
6905
|
|
|
6448
6906
|
// src/commands/loader.ts
|
|
6449
6907
|
import { existsSync as existsSync5, readdirSync as readdirSync2, readFileSync as readFileSync5 } from "fs";
|