opencode-codebase-index 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -2
- package/dist/cli.cjs +4862 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.js +4867 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +655 -352
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +657 -353
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +23 -4
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// opencode-codebase-index - Semantic codebase search for OpenCode
|
|
2
|
+
import { createRequire } from 'module'; const require = createRequire(import.meta.url);
|
|
2
3
|
var __create = Object.create;
|
|
3
4
|
var __defProp = Object.defineProperty;
|
|
4
5
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
@@ -657,7 +658,7 @@ import * as path8 from "path";
|
|
|
657
658
|
import * as os3 from "os";
|
|
658
659
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
659
660
|
|
|
660
|
-
// src/config/
|
|
661
|
+
// src/config/constants.ts
|
|
661
662
|
var DEFAULT_INCLUDE = [
|
|
662
663
|
"**/*.{ts,tsx,js,jsx,mjs,cjs}",
|
|
663
664
|
"**/*.{py,pyi}",
|
|
@@ -685,6 +686,80 @@ var DEFAULT_EXCLUDE = [
|
|
|
685
686
|
"**/.nuxt/**",
|
|
686
687
|
"**/.opencode/**"
|
|
687
688
|
];
|
|
689
|
+
var EMBEDDING_MODELS = {
|
|
690
|
+
"google": {
|
|
691
|
+
// `text-embedding-004` is DEPRECATED - https://ai.google.dev/gemini-api/docs/deprecations
|
|
692
|
+
"text-embedding-005": {
|
|
693
|
+
provider: "google",
|
|
694
|
+
model: "text-embedding-005",
|
|
695
|
+
dimensions: 768,
|
|
696
|
+
maxTokens: 2048,
|
|
697
|
+
costPer1MTokens: 0.025,
|
|
698
|
+
taskAble: false
|
|
699
|
+
// Note: on reality, this model allows for task-specific embeddings. See: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/embeddings/task-types
|
|
700
|
+
},
|
|
701
|
+
"gemini-embedding-001": {
|
|
702
|
+
provider: "google",
|
|
703
|
+
model: "gemini-embedding-001",
|
|
704
|
+
// Native output is 3072D, but we use Matryoshka truncation via outputDimensionality
|
|
705
|
+
// to reduce to 1536D for better storage/search efficiency with minimal quality loss.
|
|
706
|
+
// Google recommends 768, 1536, or 3072. See: https://ai.google.dev/gemini-api/docs/embeddings
|
|
707
|
+
dimensions: 1536,
|
|
708
|
+
maxTokens: 2048,
|
|
709
|
+
costPer1MTokens: 0.15,
|
|
710
|
+
taskAble: true
|
|
711
|
+
}
|
|
712
|
+
},
|
|
713
|
+
"openai": {
|
|
714
|
+
"text-embedding-3-small": {
|
|
715
|
+
provider: "openai",
|
|
716
|
+
model: "text-embedding-3-small",
|
|
717
|
+
dimensions: 1536,
|
|
718
|
+
maxTokens: 8191,
|
|
719
|
+
costPer1MTokens: 0.02
|
|
720
|
+
},
|
|
721
|
+
"text-embedding-3-large": {
|
|
722
|
+
provider: "openai",
|
|
723
|
+
model: "text-embedding-3-large",
|
|
724
|
+
dimensions: 3072,
|
|
725
|
+
maxTokens: 8191,
|
|
726
|
+
costPer1MTokens: 0.13
|
|
727
|
+
}
|
|
728
|
+
},
|
|
729
|
+
"ollama": {
|
|
730
|
+
"nomic-embed-text": {
|
|
731
|
+
provider: "ollama",
|
|
732
|
+
model: "nomic-embed-text",
|
|
733
|
+
dimensions: 768,
|
|
734
|
+
maxTokens: 8192,
|
|
735
|
+
costPer1MTokens: 0
|
|
736
|
+
},
|
|
737
|
+
"mxbai-embed-large": {
|
|
738
|
+
provider: "ollama",
|
|
739
|
+
model: "mxbai-embed-large",
|
|
740
|
+
dimensions: 1024,
|
|
741
|
+
maxTokens: 512,
|
|
742
|
+
costPer1MTokens: 0
|
|
743
|
+
}
|
|
744
|
+
},
|
|
745
|
+
"github-copilot": {
|
|
746
|
+
"text-embedding-3-small": {
|
|
747
|
+
provider: "github-copilot",
|
|
748
|
+
model: "text-embedding-3-small",
|
|
749
|
+
dimensions: 1536,
|
|
750
|
+
maxTokens: 8191,
|
|
751
|
+
costPer1MTokens: 0
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
};
|
|
755
|
+
var DEFAULT_PROVIDER_MODELS = {
|
|
756
|
+
"github-copilot": "text-embedding-3-small",
|
|
757
|
+
"openai": "text-embedding-3-small",
|
|
758
|
+
"google": "text-embedding-005",
|
|
759
|
+
"ollama": "nomic-embed-text"
|
|
760
|
+
};
|
|
761
|
+
|
|
762
|
+
// src/config/schema.ts
|
|
688
763
|
function getDefaultIndexingConfig() {
|
|
689
764
|
return {
|
|
690
765
|
autoIndex: false,
|
|
@@ -721,11 +796,13 @@ function getDefaultDebugConfig() {
|
|
|
721
796
|
metrics: true
|
|
722
797
|
};
|
|
723
798
|
}
|
|
724
|
-
var VALID_PROVIDERS = ["auto", "github-copilot", "openai", "google", "ollama"];
|
|
725
799
|
var VALID_SCOPES = ["project", "global"];
|
|
726
800
|
var VALID_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
727
801
|
function isValidProvider(value) {
|
|
728
|
-
return typeof value === "string" &&
|
|
802
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS).includes(value);
|
|
803
|
+
}
|
|
804
|
+
function isValidModel(value, provider) {
|
|
805
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS[provider]).includes(value);
|
|
729
806
|
}
|
|
730
807
|
function isValidScope(value) {
|
|
731
808
|
return typeof value === "string" && VALID_SCOPES.includes(value);
|
|
@@ -774,9 +851,19 @@ function parseConfig(raw) {
|
|
|
774
851
|
logBranch: typeof rawDebug.logBranch === "boolean" ? rawDebug.logBranch : defaultDebug.logBranch,
|
|
775
852
|
metrics: typeof rawDebug.metrics === "boolean" ? rawDebug.metrics : defaultDebug.metrics
|
|
776
853
|
};
|
|
854
|
+
let embeddingProvider;
|
|
855
|
+
let embeddingModel = void 0;
|
|
856
|
+
if (isValidProvider(input.embeddingProvider)) {
|
|
857
|
+
embeddingProvider = input.embeddingProvider;
|
|
858
|
+
if (input.embeddingModel) {
|
|
859
|
+
embeddingModel = isValidModel(input.embeddingModel, embeddingProvider) ? input.embeddingModel : DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
860
|
+
}
|
|
861
|
+
} else {
|
|
862
|
+
embeddingProvider = "auto";
|
|
863
|
+
}
|
|
777
864
|
return {
|
|
778
|
-
embeddingProvider
|
|
779
|
-
embeddingModel
|
|
865
|
+
embeddingProvider,
|
|
866
|
+
embeddingModel,
|
|
780
867
|
scope: isValidScope(input.scope) ? input.scope : "project",
|
|
781
868
|
include: isStringArray(input.include) ? input.include : DEFAULT_INCLUDE,
|
|
782
869
|
exclude: isStringArray(input.exclude) ? input.exclude : DEFAULT_EXCLUDE,
|
|
@@ -785,67 +872,15 @@ function parseConfig(raw) {
|
|
|
785
872
|
debug
|
|
786
873
|
};
|
|
787
874
|
}
|
|
788
|
-
var EMBEDDING_MODELS = {
|
|
789
|
-
"github-copilot/text-embedding-3-small": {
|
|
790
|
-
provider: "github-copilot",
|
|
791
|
-
model: "text-embedding-3-small",
|
|
792
|
-
dimensions: 1536,
|
|
793
|
-
maxTokens: 8191,
|
|
794
|
-
costPer1MTokens: 0
|
|
795
|
-
},
|
|
796
|
-
"openai/text-embedding-3-small": {
|
|
797
|
-
provider: "openai",
|
|
798
|
-
model: "text-embedding-3-small",
|
|
799
|
-
dimensions: 1536,
|
|
800
|
-
maxTokens: 8191,
|
|
801
|
-
costPer1MTokens: 0.02
|
|
802
|
-
},
|
|
803
|
-
"openai/text-embedding-3-large": {
|
|
804
|
-
provider: "openai",
|
|
805
|
-
model: "text-embedding-3-large",
|
|
806
|
-
dimensions: 3072,
|
|
807
|
-
maxTokens: 8191,
|
|
808
|
-
costPer1MTokens: 0.13
|
|
809
|
-
},
|
|
810
|
-
"google/text-embedding-004": {
|
|
811
|
-
provider: "google",
|
|
812
|
-
model: "text-embedding-004",
|
|
813
|
-
dimensions: 768,
|
|
814
|
-
maxTokens: 2048,
|
|
815
|
-
costPer1MTokens: 0
|
|
816
|
-
},
|
|
817
|
-
"ollama/nomic-embed-text": {
|
|
818
|
-
provider: "ollama",
|
|
819
|
-
model: "nomic-embed-text",
|
|
820
|
-
dimensions: 768,
|
|
821
|
-
maxTokens: 8192,
|
|
822
|
-
costPer1MTokens: 0
|
|
823
|
-
},
|
|
824
|
-
"ollama/mxbai-embed-large": {
|
|
825
|
-
provider: "ollama",
|
|
826
|
-
model: "mxbai-embed-large",
|
|
827
|
-
dimensions: 1024,
|
|
828
|
-
maxTokens: 512,
|
|
829
|
-
costPer1MTokens: 0
|
|
830
|
-
}
|
|
831
|
-
};
|
|
832
875
|
function getDefaultModelForProvider(provider) {
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
case "openai":
|
|
837
|
-
return EMBEDDING_MODELS["openai/text-embedding-3-small"];
|
|
838
|
-
case "google":
|
|
839
|
-
return EMBEDDING_MODELS["google/text-embedding-004"];
|
|
840
|
-
case "ollama":
|
|
841
|
-
return EMBEDDING_MODELS["ollama/nomic-embed-text"];
|
|
842
|
-
default:
|
|
843
|
-
return EMBEDDING_MODELS["github-copilot/text-embedding-3-small"];
|
|
844
|
-
}
|
|
876
|
+
const models = EMBEDDING_MODELS[provider];
|
|
877
|
+
const providerDefault = DEFAULT_PROVIDER_MODELS[provider];
|
|
878
|
+
return models[providerDefault];
|
|
845
879
|
}
|
|
880
|
+
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
846
881
|
|
|
847
882
|
// src/indexer/index.ts
|
|
848
|
-
import { existsSync as existsSync4, readFileSync as readFileSync4, writeFileSync, promises as fsPromises2 } from "fs";
|
|
883
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4, writeFileSync, renameSync, unlinkSync, promises as fsPromises2 } from "fs";
|
|
849
884
|
import * as path5 from "path";
|
|
850
885
|
import { performance as performance2 } from "perf_hooks";
|
|
851
886
|
|
|
@@ -871,7 +906,7 @@ function pTimeout(promise, options) {
|
|
|
871
906
|
} = options;
|
|
872
907
|
let timer;
|
|
873
908
|
let abortHandler;
|
|
874
|
-
const wrappedPromise = new Promise((
|
|
909
|
+
const wrappedPromise = new Promise((resolve5, reject) => {
|
|
875
910
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
876
911
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
877
912
|
}
|
|
@@ -885,7 +920,7 @@ function pTimeout(promise, options) {
|
|
|
885
920
|
};
|
|
886
921
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
887
922
|
}
|
|
888
|
-
promise.then(
|
|
923
|
+
promise.then(resolve5, reject);
|
|
889
924
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
890
925
|
return;
|
|
891
926
|
}
|
|
@@ -893,7 +928,7 @@ function pTimeout(promise, options) {
|
|
|
893
928
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
894
929
|
if (fallback) {
|
|
895
930
|
try {
|
|
896
|
-
|
|
931
|
+
resolve5(fallback());
|
|
897
932
|
} catch (error) {
|
|
898
933
|
reject(error);
|
|
899
934
|
}
|
|
@@ -903,7 +938,7 @@ function pTimeout(promise, options) {
|
|
|
903
938
|
promise.cancel();
|
|
904
939
|
}
|
|
905
940
|
if (message === false) {
|
|
906
|
-
|
|
941
|
+
resolve5();
|
|
907
942
|
} else if (message instanceof Error) {
|
|
908
943
|
reject(message);
|
|
909
944
|
} else {
|
|
@@ -1293,7 +1328,7 @@ var PQueue = class extends import_index.default {
|
|
|
1293
1328
|
// Assign unique ID if not provided
|
|
1294
1329
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1295
1330
|
};
|
|
1296
|
-
return new Promise((
|
|
1331
|
+
return new Promise((resolve5, reject) => {
|
|
1297
1332
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1298
1333
|
this.#queue.enqueue(async () => {
|
|
1299
1334
|
this.#pending++;
|
|
@@ -1331,7 +1366,7 @@ var PQueue = class extends import_index.default {
|
|
|
1331
1366
|
})]);
|
|
1332
1367
|
}
|
|
1333
1368
|
const result = await operation;
|
|
1334
|
-
|
|
1369
|
+
resolve5(result);
|
|
1335
1370
|
this.emit("completed", result);
|
|
1336
1371
|
} catch (error) {
|
|
1337
1372
|
reject(error);
|
|
@@ -1488,13 +1523,13 @@ var PQueue = class extends import_index.default {
|
|
|
1488
1523
|
});
|
|
1489
1524
|
}
|
|
1490
1525
|
async #onEvent(event, filter) {
|
|
1491
|
-
return new Promise((
|
|
1526
|
+
return new Promise((resolve5) => {
|
|
1492
1527
|
const listener = () => {
|
|
1493
1528
|
if (filter && !filter()) {
|
|
1494
1529
|
return;
|
|
1495
1530
|
}
|
|
1496
1531
|
this.off(event, listener);
|
|
1497
|
-
|
|
1532
|
+
resolve5();
|
|
1498
1533
|
};
|
|
1499
1534
|
this.on(event, listener);
|
|
1500
1535
|
});
|
|
@@ -1779,7 +1814,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1779
1814
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
1780
1815
|
options.signal?.throwIfAborted();
|
|
1781
1816
|
if (finalDelay > 0) {
|
|
1782
|
-
await new Promise((
|
|
1817
|
+
await new Promise((resolve5, reject) => {
|
|
1783
1818
|
const onAbort = () => {
|
|
1784
1819
|
clearTimeout(timeoutToken);
|
|
1785
1820
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -1787,7 +1822,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
1787
1822
|
};
|
|
1788
1823
|
const timeoutToken = setTimeout(() => {
|
|
1789
1824
|
options.signal?.removeEventListener("abort", onAbort);
|
|
1790
|
-
|
|
1825
|
+
resolve5();
|
|
1791
1826
|
}, finalDelay);
|
|
1792
1827
|
if (options.unref) {
|
|
1793
1828
|
timeoutToken.unref?.();
|
|
@@ -1851,12 +1886,6 @@ async function pRetry(input, options = {}) {
|
|
|
1851
1886
|
import { existsSync, readFileSync } from "fs";
|
|
1852
1887
|
import * as path from "path";
|
|
1853
1888
|
import * as os from "os";
|
|
1854
|
-
var EMBEDDING_CAPABLE_PROVIDERS = [
|
|
1855
|
-
"github-copilot",
|
|
1856
|
-
"openai",
|
|
1857
|
-
"google",
|
|
1858
|
-
"ollama"
|
|
1859
|
-
];
|
|
1860
1889
|
function getOpenCodeAuthPath() {
|
|
1861
1890
|
return path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
|
|
1862
1891
|
}
|
|
@@ -1870,21 +1899,34 @@ function loadOpenCodeAuth() {
|
|
|
1870
1899
|
}
|
|
1871
1900
|
return {};
|
|
1872
1901
|
}
|
|
1873
|
-
async function detectEmbeddingProvider(preferredProvider) {
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
if (
|
|
1902
|
+
async function detectEmbeddingProvider(preferredProvider, model) {
|
|
1903
|
+
const credentials = await getProviderCredentials(preferredProvider);
|
|
1904
|
+
if (credentials) {
|
|
1905
|
+
if (!model) {
|
|
1877
1906
|
return {
|
|
1878
1907
|
provider: preferredProvider,
|
|
1879
1908
|
credentials,
|
|
1880
1909
|
modelInfo: getDefaultModelForProvider(preferredProvider)
|
|
1881
1910
|
};
|
|
1882
1911
|
}
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1912
|
+
if (!isValidModel(model, preferredProvider)) {
|
|
1913
|
+
throw new Error(
|
|
1914
|
+
`Model '${model}' is not supported by provider '${preferredProvider}'`
|
|
1915
|
+
);
|
|
1916
|
+
}
|
|
1917
|
+
const providerModels = EMBEDDING_MODELS[preferredProvider];
|
|
1918
|
+
return {
|
|
1919
|
+
provider: preferredProvider,
|
|
1920
|
+
credentials,
|
|
1921
|
+
modelInfo: providerModels[model]
|
|
1922
|
+
};
|
|
1886
1923
|
}
|
|
1887
|
-
|
|
1924
|
+
throw new Error(
|
|
1925
|
+
`Preferred provider '${preferredProvider}' is not configured or authenticated`
|
|
1926
|
+
);
|
|
1927
|
+
}
|
|
1928
|
+
async function tryDetectProvider() {
|
|
1929
|
+
for (const provider of availableProviders) {
|
|
1888
1930
|
const credentials = await getProviderCredentials(provider);
|
|
1889
1931
|
if (credentials) {
|
|
1890
1932
|
return {
|
|
@@ -1895,7 +1937,7 @@ async function detectEmbeddingProvider(preferredProvider) {
|
|
|
1895
1937
|
}
|
|
1896
1938
|
}
|
|
1897
1939
|
throw new Error(
|
|
1898
|
-
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${
|
|
1940
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${availableProviders.join(", ")}.`
|
|
1899
1941
|
);
|
|
1900
1942
|
}
|
|
1901
1943
|
async function getProviderCredentials(provider) {
|
|
@@ -1993,18 +2035,20 @@ function getProviderDisplayName(provider) {
|
|
|
1993
2035
|
}
|
|
1994
2036
|
|
|
1995
2037
|
// src/embeddings/provider.ts
|
|
1996
|
-
function createEmbeddingProvider(
|
|
1997
|
-
switch (
|
|
2038
|
+
function createEmbeddingProvider(configuredProviderInfo) {
|
|
2039
|
+
switch (configuredProviderInfo.provider) {
|
|
1998
2040
|
case "github-copilot":
|
|
1999
|
-
return new GitHubCopilotEmbeddingProvider(credentials, modelInfo);
|
|
2041
|
+
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2000
2042
|
case "openai":
|
|
2001
|
-
return new OpenAIEmbeddingProvider(credentials, modelInfo);
|
|
2043
|
+
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2002
2044
|
case "google":
|
|
2003
|
-
return new GoogleEmbeddingProvider(credentials, modelInfo);
|
|
2045
|
+
return new GoogleEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2004
2046
|
case "ollama":
|
|
2005
|
-
return new OllamaEmbeddingProvider(credentials, modelInfo);
|
|
2006
|
-
default:
|
|
2007
|
-
|
|
2047
|
+
return new OllamaEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2048
|
+
default: {
|
|
2049
|
+
const _exhaustive = configuredProviderInfo;
|
|
2050
|
+
throw new Error(`Unsupported embedding provider: ${_exhaustive.provider}`);
|
|
2051
|
+
}
|
|
2008
2052
|
}
|
|
2009
2053
|
}
|
|
2010
2054
|
var GitHubCopilotEmbeddingProvider = class {
|
|
@@ -2018,8 +2062,15 @@ var GitHubCopilotEmbeddingProvider = class {
|
|
|
2018
2062
|
}
|
|
2019
2063
|
return this.credentials.refreshToken;
|
|
2020
2064
|
}
|
|
2021
|
-
async
|
|
2022
|
-
const result = await this.embedBatch([
|
|
2065
|
+
async embedQuery(query) {
|
|
2066
|
+
const result = await this.embedBatch([query]);
|
|
2067
|
+
return {
|
|
2068
|
+
embedding: result.embeddings[0],
|
|
2069
|
+
tokensUsed: result.totalTokensUsed
|
|
2070
|
+
};
|
|
2071
|
+
}
|
|
2072
|
+
async embedDocument(document) {
|
|
2073
|
+
const result = await this.embedBatch([document]);
|
|
2023
2074
|
return {
|
|
2024
2075
|
embedding: result.embeddings[0],
|
|
2025
2076
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2059,8 +2110,15 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2059
2110
|
this.credentials = credentials;
|
|
2060
2111
|
this.modelInfo = modelInfo;
|
|
2061
2112
|
}
|
|
2062
|
-
async
|
|
2063
|
-
const result = await this.embedBatch([
|
|
2113
|
+
async embedQuery(query) {
|
|
2114
|
+
const result = await this.embedBatch([query]);
|
|
2115
|
+
return {
|
|
2116
|
+
embedding: result.embeddings[0],
|
|
2117
|
+
tokensUsed: result.totalTokensUsed
|
|
2118
|
+
};
|
|
2119
|
+
}
|
|
2120
|
+
async embedDocument(document) {
|
|
2121
|
+
const result = await this.embedBatch([document]);
|
|
2064
2122
|
return {
|
|
2065
2123
|
embedding: result.embeddings[0],
|
|
2066
2124
|
tokensUsed: result.totalTokensUsed
|
|
@@ -2092,33 +2150,61 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2092
2150
|
return this.modelInfo;
|
|
2093
2151
|
}
|
|
2094
2152
|
};
|
|
2095
|
-
var GoogleEmbeddingProvider = class {
|
|
2153
|
+
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
2096
2154
|
constructor(credentials, modelInfo) {
|
|
2097
2155
|
this.credentials = credentials;
|
|
2098
2156
|
this.modelInfo = modelInfo;
|
|
2099
2157
|
}
|
|
2100
|
-
|
|
2101
|
-
|
|
2158
|
+
static BATCH_SIZE = 20;
|
|
2159
|
+
async embedQuery(query) {
|
|
2160
|
+
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2161
|
+
const result = await this.embedWithTaskType([query], taskType);
|
|
2162
|
+
return {
|
|
2163
|
+
embedding: result.embeddings[0],
|
|
2164
|
+
tokensUsed: result.totalTokensUsed
|
|
2165
|
+
};
|
|
2166
|
+
}
|
|
2167
|
+
async embedDocument(document) {
|
|
2168
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2169
|
+
const result = await this.embedWithTaskType([document], taskType);
|
|
2102
2170
|
return {
|
|
2103
2171
|
embedding: result.embeddings[0],
|
|
2104
2172
|
tokensUsed: result.totalTokensUsed
|
|
2105
2173
|
};
|
|
2106
2174
|
}
|
|
2107
2175
|
async embedBatch(texts) {
|
|
2108
|
-
const
|
|
2109
|
-
|
|
2176
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2177
|
+
return this.embedWithTaskType(texts, taskType);
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Embeds texts using the Google embedContent API.
|
|
2181
|
+
* Sends multiple texts as parts in batched requests (up to BATCH_SIZE per call).
|
|
2182
|
+
* When taskType is provided (gemini-embedding-001), includes it in the request
|
|
2183
|
+
* for task-specific embedding optimization.
|
|
2184
|
+
*/
|
|
2185
|
+
async embedWithTaskType(texts, taskType) {
|
|
2186
|
+
const batches = [];
|
|
2187
|
+
for (let i = 0; i < texts.length; i += _GoogleEmbeddingProvider.BATCH_SIZE) {
|
|
2188
|
+
batches.push(texts.slice(i, i + _GoogleEmbeddingProvider.BATCH_SIZE));
|
|
2189
|
+
}
|
|
2190
|
+
const batchResults = await Promise.all(
|
|
2191
|
+
batches.map(async (batch) => {
|
|
2192
|
+
const requests = batch.map((text) => ({
|
|
2193
|
+
model: `models/${this.modelInfo.model}`,
|
|
2194
|
+
content: {
|
|
2195
|
+
parts: [{ text }]
|
|
2196
|
+
},
|
|
2197
|
+
taskType,
|
|
2198
|
+
outputDimensionality: this.modelInfo.dimensions
|
|
2199
|
+
}));
|
|
2110
2200
|
const response = await fetch(
|
|
2111
|
-
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:
|
|
2201
|
+
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:batchEmbedContents?key=${this.credentials.apiKey}`,
|
|
2112
2202
|
{
|
|
2113
2203
|
method: "POST",
|
|
2114
2204
|
headers: {
|
|
2115
2205
|
"Content-Type": "application/json"
|
|
2116
2206
|
},
|
|
2117
|
-
body: JSON.stringify({
|
|
2118
|
-
content: {
|
|
2119
|
-
parts: [{ text }]
|
|
2120
|
-
}
|
|
2121
|
-
})
|
|
2207
|
+
body: JSON.stringify({ requests })
|
|
2122
2208
|
}
|
|
2123
2209
|
);
|
|
2124
2210
|
if (!response.ok) {
|
|
@@ -2127,14 +2213,14 @@ var GoogleEmbeddingProvider = class {
|
|
|
2127
2213
|
}
|
|
2128
2214
|
const data = await response.json();
|
|
2129
2215
|
return {
|
|
2130
|
-
|
|
2131
|
-
tokensUsed: Math.ceil(text.length / 4)
|
|
2216
|
+
embeddings: data.embeddings.map((e) => e.values),
|
|
2217
|
+
tokensUsed: batch.reduce((sum, text) => sum + Math.ceil(text.length / 4), 0)
|
|
2132
2218
|
};
|
|
2133
2219
|
})
|
|
2134
2220
|
);
|
|
2135
2221
|
return {
|
|
2136
|
-
embeddings:
|
|
2137
|
-
totalTokensUsed:
|
|
2222
|
+
embeddings: batchResults.flatMap((r) => r.embeddings),
|
|
2223
|
+
totalTokensUsed: batchResults.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2138
2224
|
};
|
|
2139
2225
|
}
|
|
2140
2226
|
getModelInfo() {
|
|
@@ -2146,29 +2232,44 @@ var OllamaEmbeddingProvider = class {
|
|
|
2146
2232
|
this.credentials = credentials;
|
|
2147
2233
|
this.modelInfo = modelInfo;
|
|
2148
2234
|
}
|
|
2149
|
-
async
|
|
2150
|
-
const
|
|
2151
|
-
method: "POST",
|
|
2152
|
-
headers: {
|
|
2153
|
-
"Content-Type": "application/json"
|
|
2154
|
-
},
|
|
2155
|
-
body: JSON.stringify({
|
|
2156
|
-
model: this.modelInfo.model,
|
|
2157
|
-
prompt: text
|
|
2158
|
-
})
|
|
2159
|
-
});
|
|
2160
|
-
if (!response.ok) {
|
|
2161
|
-
const error = await response.text();
|
|
2162
|
-
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2163
|
-
}
|
|
2164
|
-
const data = await response.json();
|
|
2235
|
+
async embedQuery(query) {
|
|
2236
|
+
const result = await this.embedBatch([query]);
|
|
2165
2237
|
return {
|
|
2166
|
-
embedding:
|
|
2167
|
-
tokensUsed:
|
|
2238
|
+
embedding: result.embeddings[0],
|
|
2239
|
+
tokensUsed: result.totalTokensUsed
|
|
2240
|
+
};
|
|
2241
|
+
}
|
|
2242
|
+
async embedDocument(document) {
|
|
2243
|
+
const result = await this.embedBatch([document]);
|
|
2244
|
+
return {
|
|
2245
|
+
embedding: result.embeddings[0],
|
|
2246
|
+
tokensUsed: result.totalTokensUsed
|
|
2168
2247
|
};
|
|
2169
2248
|
}
|
|
2170
2249
|
async embedBatch(texts) {
|
|
2171
|
-
const results = await Promise.all(
|
|
2250
|
+
const results = await Promise.all(
|
|
2251
|
+
texts.map(async (text) => {
|
|
2252
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2253
|
+
method: "POST",
|
|
2254
|
+
headers: {
|
|
2255
|
+
"Content-Type": "application/json"
|
|
2256
|
+
},
|
|
2257
|
+
body: JSON.stringify({
|
|
2258
|
+
model: this.modelInfo.model,
|
|
2259
|
+
prompt: text
|
|
2260
|
+
})
|
|
2261
|
+
});
|
|
2262
|
+
if (!response.ok) {
|
|
2263
|
+
const error = await response.text();
|
|
2264
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2265
|
+
}
|
|
2266
|
+
const data = await response.json();
|
|
2267
|
+
return {
|
|
2268
|
+
embedding: data.embedding,
|
|
2269
|
+
tokensUsed: Math.ceil(text.length / 4)
|
|
2270
|
+
};
|
|
2271
|
+
})
|
|
2272
|
+
);
|
|
2172
2273
|
return {
|
|
2173
2274
|
embeddings: results.map((r) => r.embedding),
|
|
2174
2275
|
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
@@ -3112,11 +3213,40 @@ var Database = class {
|
|
|
3112
3213
|
import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync, statSync } from "fs";
|
|
3113
3214
|
import * as path4 from "path";
|
|
3114
3215
|
import { execSync } from "child_process";
|
|
3216
|
+
function resolveGitDir(repoRoot) {
|
|
3217
|
+
const gitPath = path4.join(repoRoot, ".git");
|
|
3218
|
+
if (!existsSync3(gitPath)) {
|
|
3219
|
+
return null;
|
|
3220
|
+
}
|
|
3221
|
+
try {
|
|
3222
|
+
const stat4 = statSync(gitPath);
|
|
3223
|
+
if (stat4.isDirectory()) {
|
|
3224
|
+
return gitPath;
|
|
3225
|
+
}
|
|
3226
|
+
if (stat4.isFile()) {
|
|
3227
|
+
const content = readFileSync3(gitPath, "utf-8").trim();
|
|
3228
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
3229
|
+
if (match) {
|
|
3230
|
+
const gitdir = match[1];
|
|
3231
|
+
const resolvedPath = path4.isAbsolute(gitdir) ? gitdir : path4.resolve(repoRoot, gitdir);
|
|
3232
|
+
if (existsSync3(resolvedPath)) {
|
|
3233
|
+
return resolvedPath;
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
}
|
|
3237
|
+
} catch {
|
|
3238
|
+
}
|
|
3239
|
+
return null;
|
|
3240
|
+
}
|
|
3115
3241
|
function isGitRepo(dir) {
|
|
3116
|
-
return
|
|
3242
|
+
return resolveGitDir(dir) !== null;
|
|
3117
3243
|
}
|
|
3118
3244
|
function getCurrentBranch(repoRoot) {
|
|
3119
|
-
const
|
|
3245
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3246
|
+
if (!gitDir) {
|
|
3247
|
+
return null;
|
|
3248
|
+
}
|
|
3249
|
+
const headPath = path4.join(gitDir, "HEAD");
|
|
3120
3250
|
if (!existsSync3(headPath)) {
|
|
3121
3251
|
return null;
|
|
3122
3252
|
}
|
|
@@ -3135,20 +3265,23 @@ function getCurrentBranch(repoRoot) {
|
|
|
3135
3265
|
}
|
|
3136
3266
|
}
|
|
3137
3267
|
function getBaseBranch(repoRoot) {
|
|
3268
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3138
3269
|
const candidates = ["main", "master", "develop", "trunk"];
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3270
|
+
if (gitDir) {
|
|
3271
|
+
for (const candidate of candidates) {
|
|
3272
|
+
const refPath = path4.join(gitDir, "refs", "heads", candidate);
|
|
3273
|
+
if (existsSync3(refPath)) {
|
|
3274
|
+
return candidate;
|
|
3275
|
+
}
|
|
3276
|
+
const packedRefsPath = path4.join(gitDir, "packed-refs");
|
|
3277
|
+
if (existsSync3(packedRefsPath)) {
|
|
3278
|
+
try {
|
|
3279
|
+
const content = readFileSync3(packedRefsPath, "utf-8");
|
|
3280
|
+
if (content.includes(`refs/heads/${candidate}`)) {
|
|
3281
|
+
return candidate;
|
|
3282
|
+
}
|
|
3283
|
+
} catch {
|
|
3150
3284
|
}
|
|
3151
|
-
} catch {
|
|
3152
3285
|
}
|
|
3153
3286
|
}
|
|
3154
3287
|
}
|
|
@@ -3173,6 +3306,10 @@ function getBranchOrDefault(repoRoot) {
|
|
|
3173
3306
|
return getCurrentBranch(repoRoot) ?? "default";
|
|
3174
3307
|
}
|
|
3175
3308
|
function getHeadPath(repoRoot) {
|
|
3309
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3310
|
+
if (gitDir) {
|
|
3311
|
+
return path4.join(gitDir, "HEAD");
|
|
3312
|
+
}
|
|
3176
3313
|
return path4.join(repoRoot, ".git", "HEAD");
|
|
3177
3314
|
}
|
|
3178
3315
|
|
|
@@ -3200,6 +3337,7 @@ function isRateLimitError(error) {
|
|
|
3200
3337
|
const message = getErrorMessage(error);
|
|
3201
3338
|
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
3202
3339
|
}
|
|
3340
|
+
var INDEX_METADATA_VERSION = "1";
|
|
3203
3341
|
var Indexer = class {
|
|
3204
3342
|
config;
|
|
3205
3343
|
projectRoot;
|
|
@@ -3208,7 +3346,7 @@ var Indexer = class {
|
|
|
3208
3346
|
invertedIndex = null;
|
|
3209
3347
|
database = null;
|
|
3210
3348
|
provider = null;
|
|
3211
|
-
|
|
3349
|
+
configuredProviderInfo = null;
|
|
3212
3350
|
fileHashCache = /* @__PURE__ */ new Map();
|
|
3213
3351
|
fileHashCachePath = "";
|
|
3214
3352
|
failedBatchesPath = "";
|
|
@@ -3219,12 +3357,15 @@ var Indexer = class {
|
|
|
3219
3357
|
maxQueryCacheSize = 100;
|
|
3220
3358
|
queryCacheTtlMs = 5 * 60 * 1e3;
|
|
3221
3359
|
querySimilarityThreshold = 0.85;
|
|
3360
|
+
indexCompatibility = null;
|
|
3361
|
+
indexingLockPath = "";
|
|
3222
3362
|
constructor(projectRoot, config) {
|
|
3223
3363
|
this.projectRoot = projectRoot;
|
|
3224
3364
|
this.config = config;
|
|
3225
3365
|
this.indexPath = this.getIndexPath();
|
|
3226
3366
|
this.fileHashCachePath = path5.join(this.indexPath, "file-hashes.json");
|
|
3227
3367
|
this.failedBatchesPath = path5.join(this.indexPath, "failed-batches.json");
|
|
3368
|
+
this.indexingLockPath = path5.join(this.indexPath, "indexing.lock");
|
|
3228
3369
|
this.logger = initializeLogger(config.debug);
|
|
3229
3370
|
}
|
|
3230
3371
|
getIndexPath() {
|
|
@@ -3250,7 +3391,36 @@ var Indexer = class {
|
|
|
3250
3391
|
for (const [k, v] of this.fileHashCache) {
|
|
3251
3392
|
obj[k] = v;
|
|
3252
3393
|
}
|
|
3253
|
-
|
|
3394
|
+
this.atomicWriteSync(this.fileHashCachePath, JSON.stringify(obj));
|
|
3395
|
+
}
|
|
3396
|
+
atomicWriteSync(targetPath, data) {
|
|
3397
|
+
const tempPath = `${targetPath}.tmp`;
|
|
3398
|
+
writeFileSync(tempPath, data);
|
|
3399
|
+
renameSync(tempPath, targetPath);
|
|
3400
|
+
}
|
|
3401
|
+
checkForInterruptedIndexing() {
|
|
3402
|
+
return existsSync4(this.indexingLockPath);
|
|
3403
|
+
}
|
|
3404
|
+
acquireIndexingLock() {
|
|
3405
|
+
const lockData = {
|
|
3406
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3407
|
+
pid: process.pid
|
|
3408
|
+
};
|
|
3409
|
+
writeFileSync(this.indexingLockPath, JSON.stringify(lockData));
|
|
3410
|
+
}
|
|
3411
|
+
releaseIndexingLock() {
|
|
3412
|
+
if (existsSync4(this.indexingLockPath)) {
|
|
3413
|
+
unlinkSync(this.indexingLockPath);
|
|
3414
|
+
}
|
|
3415
|
+
}
|
|
3416
|
+
async recoverFromInterruptedIndexing() {
|
|
3417
|
+
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
3418
|
+
if (existsSync4(this.fileHashCachePath)) {
|
|
3419
|
+
unlinkSync(this.fileHashCachePath);
|
|
3420
|
+
}
|
|
3421
|
+
await this.healthCheck();
|
|
3422
|
+
this.releaseIndexingLock();
|
|
3423
|
+
this.logger.info("Recovery complete, next index will re-process all files");
|
|
3254
3424
|
}
|
|
3255
3425
|
loadFailedBatches() {
|
|
3256
3426
|
try {
|
|
@@ -3298,23 +3468,27 @@ var Indexer = class {
|
|
|
3298
3468
|
}
|
|
3299
3469
|
}
|
|
3300
3470
|
async initialize() {
|
|
3301
|
-
|
|
3302
|
-
|
|
3471
|
+
if (this.config.embeddingProvider === "auto") {
|
|
3472
|
+
this.configuredProviderInfo = await tryDetectProvider();
|
|
3473
|
+
} else {
|
|
3474
|
+
this.configuredProviderInfo = await detectEmbeddingProvider(this.config.embeddingProvider, this.config.embeddingModel);
|
|
3475
|
+
}
|
|
3476
|
+
if (!this.configuredProviderInfo) {
|
|
3303
3477
|
throw new Error(
|
|
3304
3478
|
"No embedding provider available. Configure GitHub, OpenAI, Google, or Ollama."
|
|
3305
3479
|
);
|
|
3306
3480
|
}
|
|
3307
3481
|
this.logger.info("Initializing indexer", {
|
|
3308
|
-
provider: this.
|
|
3309
|
-
model: this.
|
|
3482
|
+
provider: this.configuredProviderInfo.provider,
|
|
3483
|
+
model: this.configuredProviderInfo.modelInfo.model,
|
|
3310
3484
|
scope: this.config.scope
|
|
3311
3485
|
});
|
|
3312
|
-
this.provider = createEmbeddingProvider(
|
|
3313
|
-
this.detectedProvider.credentials,
|
|
3314
|
-
this.detectedProvider.modelInfo
|
|
3315
|
-
);
|
|
3486
|
+
this.provider = createEmbeddingProvider(this.configuredProviderInfo);
|
|
3316
3487
|
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
3317
|
-
|
|
3488
|
+
if (this.checkForInterruptedIndexing()) {
|
|
3489
|
+
await this.recoverFromInterruptedIndexing();
|
|
3490
|
+
}
|
|
3491
|
+
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
3318
3492
|
const storePath = path5.join(this.indexPath, "vectors");
|
|
3319
3493
|
this.store = new VectorStore(storePath, dimensions);
|
|
3320
3494
|
const indexFilePath = path5.join(this.indexPath, "vectors.usearch");
|
|
@@ -3337,6 +3511,14 @@ var Indexer = class {
|
|
|
3337
3511
|
if (dbIsNew && this.store.count() > 0) {
|
|
3338
3512
|
this.migrateFromLegacyIndex();
|
|
3339
3513
|
}
|
|
3514
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3515
|
+
if (!this.indexCompatibility.compatible) {
|
|
3516
|
+
this.logger.warn("Index compatibility issue detected", {
|
|
3517
|
+
reason: this.indexCompatibility.reason,
|
|
3518
|
+
storedMetadata: this.indexCompatibility.storedMetadata,
|
|
3519
|
+
configuredProviderInfo: this.configuredProviderInfo
|
|
3520
|
+
});
|
|
3521
|
+
}
|
|
3340
3522
|
if (isGitRepo(this.projectRoot)) {
|
|
3341
3523
|
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
3342
3524
|
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
@@ -3407,30 +3589,106 @@ var Indexer = class {
|
|
|
3407
3589
|
}
|
|
3408
3590
|
this.database.addChunksToBranchBatch(this.currentBranch || "default", chunkIds);
|
|
3409
3591
|
}
|
|
3592
|
+
loadIndexMetadata() {
|
|
3593
|
+
if (!this.database) return null;
|
|
3594
|
+
const version = this.database.getMetadata("index.version");
|
|
3595
|
+
if (!version) return null;
|
|
3596
|
+
return {
|
|
3597
|
+
indexVersion: version,
|
|
3598
|
+
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
3599
|
+
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
3600
|
+
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
3601
|
+
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
3602
|
+
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
3603
|
+
};
|
|
3604
|
+
}
|
|
3605
|
+
saveIndexMetadata(provider) {
|
|
3606
|
+
if (!this.database) return;
|
|
3607
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3608
|
+
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
3609
|
+
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
3610
|
+
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
3611
|
+
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
3612
|
+
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
3613
|
+
this.database.setMetadata("index.updatedAt", now);
|
|
3614
|
+
if (!existingCreatedAt) {
|
|
3615
|
+
this.database.setMetadata("index.createdAt", now);
|
|
3616
|
+
}
|
|
3617
|
+
}
|
|
3618
|
+
validateIndexCompatibility(provider) {
|
|
3619
|
+
const storedMetadata = this.loadIndexMetadata();
|
|
3620
|
+
if (!storedMetadata) {
|
|
3621
|
+
return { compatible: true };
|
|
3622
|
+
}
|
|
3623
|
+
const currentProvider = provider.provider;
|
|
3624
|
+
const currentModel = provider.modelInfo.model;
|
|
3625
|
+
const currentDimensions = provider.modelInfo.dimensions;
|
|
3626
|
+
if (storedMetadata.embeddingDimensions !== currentDimensions) {
|
|
3627
|
+
return {
|
|
3628
|
+
compatible: false,
|
|
3629
|
+
code: "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
|
|
3630
|
+
reason: `Dimension mismatch: index has ${storedMetadata.embeddingDimensions}D vectors (${storedMetadata.embeddingProvider}/${storedMetadata.embeddingModel}), but current provider uses ${currentDimensions}D (${currentProvider}/${currentModel}). Run index_codebase with force=true to rebuild.`,
|
|
3631
|
+
storedMetadata
|
|
3632
|
+
};
|
|
3633
|
+
}
|
|
3634
|
+
if (storedMetadata.embeddingModel !== currentModel) {
|
|
3635
|
+
return {
|
|
3636
|
+
compatible: false,
|
|
3637
|
+
code: "MODEL_MISMATCH" /* MODEL_MISMATCH */,
|
|
3638
|
+
reason: `Model mismatch: index was built with "${storedMetadata.embeddingModel}", but current model is "${currentModel}". Embeddings are incompatible. Run index_codebase with force=true to rebuild.`,
|
|
3639
|
+
storedMetadata
|
|
3640
|
+
};
|
|
3641
|
+
}
|
|
3642
|
+
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
3643
|
+
this.logger.warn("Provider changed", {
|
|
3644
|
+
storedProvider: storedMetadata.embeddingProvider,
|
|
3645
|
+
currentProvider
|
|
3646
|
+
});
|
|
3647
|
+
}
|
|
3648
|
+
return {
|
|
3649
|
+
compatible: true,
|
|
3650
|
+
storedMetadata
|
|
3651
|
+
};
|
|
3652
|
+
}
|
|
3653
|
+
checkCompatibility() {
|
|
3654
|
+
if (!this.indexCompatibility) {
|
|
3655
|
+
if (!this.configuredProviderInfo) {
|
|
3656
|
+
throw new Error("No embedding provider info, you must initialize the indexer first.");
|
|
3657
|
+
}
|
|
3658
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3659
|
+
}
|
|
3660
|
+
return this.indexCompatibility;
|
|
3661
|
+
}
|
|
3410
3662
|
async ensureInitialized() {
|
|
3411
|
-
if (!this.store || !this.provider || !this.invertedIndex || !this.
|
|
3663
|
+
if (!this.store || !this.provider || !this.invertedIndex || !this.configuredProviderInfo || !this.database) {
|
|
3412
3664
|
await this.initialize();
|
|
3413
3665
|
}
|
|
3414
3666
|
return {
|
|
3415
3667
|
store: this.store,
|
|
3416
3668
|
provider: this.provider,
|
|
3417
3669
|
invertedIndex: this.invertedIndex,
|
|
3418
|
-
|
|
3670
|
+
configuredProviderInfo: this.configuredProviderInfo,
|
|
3419
3671
|
database: this.database
|
|
3420
3672
|
};
|
|
3421
3673
|
}
|
|
3422
3674
|
async estimateCost() {
|
|
3423
|
-
const {
|
|
3675
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
3424
3676
|
const { files } = await collectFiles(
|
|
3425
3677
|
this.projectRoot,
|
|
3426
3678
|
this.config.include,
|
|
3427
3679
|
this.config.exclude,
|
|
3428
3680
|
this.config.indexing.maxFileSize
|
|
3429
3681
|
);
|
|
3430
|
-
return createCostEstimate(files,
|
|
3682
|
+
return createCostEstimate(files, configuredProviderInfo);
|
|
3431
3683
|
}
|
|
3432
3684
|
async index(onProgress) {
|
|
3433
|
-
const { store, provider, invertedIndex, database,
|
|
3685
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
3686
|
+
if (!this.indexCompatibility?.compatible) {
|
|
3687
|
+
throw new Error(
|
|
3688
|
+
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
3689
|
+
);
|
|
3690
|
+
}
|
|
3691
|
+
this.acquireIndexingLock();
|
|
3434
3692
|
this.logger.recordIndexingStart();
|
|
3435
3693
|
this.logger.info("Starting indexing", { projectRoot: this.projectRoot });
|
|
3436
3694
|
const startTime = Date.now();
|
|
@@ -3537,7 +3795,7 @@ var Indexer = class {
|
|
|
3537
3795
|
const id = generateChunkId(parsed.path, chunk);
|
|
3538
3796
|
const contentHash = generateChunkHash(chunk);
|
|
3539
3797
|
currentChunkIds.add(id);
|
|
3540
|
-
|
|
3798
|
+
chunkDataBatch.push({
|
|
3541
3799
|
chunkId: id,
|
|
3542
3800
|
contentHash,
|
|
3543
3801
|
filePath: parsed.path,
|
|
@@ -3546,8 +3804,7 @@ var Indexer = class {
|
|
|
3546
3804
|
nodeType: chunk.chunkType,
|
|
3547
3805
|
name: chunk.name,
|
|
3548
3806
|
language: chunk.language
|
|
3549
|
-
};
|
|
3550
|
-
chunkDataBatch.push(chunkData);
|
|
3807
|
+
});
|
|
3551
3808
|
if (existingChunks.get(id) === contentHash) {
|
|
3552
3809
|
fileChunkCount++;
|
|
3553
3810
|
continue;
|
|
@@ -3600,6 +3857,7 @@ var Indexer = class {
|
|
|
3600
3857
|
chunksProcessed: 0,
|
|
3601
3858
|
totalChunks: 0
|
|
3602
3859
|
});
|
|
3860
|
+
this.releaseIndexingLock();
|
|
3603
3861
|
return stats;
|
|
3604
3862
|
}
|
|
3605
3863
|
if (pendingChunks.length === 0) {
|
|
@@ -3617,6 +3875,7 @@ var Indexer = class {
|
|
|
3617
3875
|
chunksProcessed: 0,
|
|
3618
3876
|
totalChunks: 0
|
|
3619
3877
|
});
|
|
3878
|
+
this.releaseIndexingLock();
|
|
3620
3879
|
return stats;
|
|
3621
3880
|
}
|
|
3622
3881
|
onProgress?.({
|
|
@@ -3645,7 +3904,7 @@ var Indexer = class {
|
|
|
3645
3904
|
stats.indexedChunks++;
|
|
3646
3905
|
}
|
|
3647
3906
|
}
|
|
3648
|
-
const providerRateLimits = this.getProviderRateLimits(
|
|
3907
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
3649
3908
|
const queue = new PQueue({
|
|
3650
3909
|
concurrency: providerRateLimits.concurrency,
|
|
3651
3910
|
interval: providerRateLimits.intervalMs,
|
|
@@ -3656,7 +3915,7 @@ var Indexer = class {
|
|
|
3656
3915
|
for (const batch of dynamicBatches) {
|
|
3657
3916
|
queue.add(async () => {
|
|
3658
3917
|
if (rateLimitBackoffMs > 0) {
|
|
3659
|
-
await new Promise((
|
|
3918
|
+
await new Promise((resolve5) => setTimeout(resolve5, rateLimitBackoffMs));
|
|
3660
3919
|
}
|
|
3661
3920
|
try {
|
|
3662
3921
|
const result = await pRetry(
|
|
@@ -3700,7 +3959,7 @@ var Indexer = class {
|
|
|
3700
3959
|
contentHash: chunk.contentHash,
|
|
3701
3960
|
embedding: float32ArrayToBuffer(result.embeddings[i]),
|
|
3702
3961
|
chunkText: chunk.text,
|
|
3703
|
-
model:
|
|
3962
|
+
model: configuredProviderInfo.modelInfo.model
|
|
3704
3963
|
}));
|
|
3705
3964
|
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
3706
3965
|
for (const chunk of batch) {
|
|
@@ -3751,6 +4010,8 @@ var Indexer = class {
|
|
|
3751
4010
|
await this.maybeRunOrphanGc();
|
|
3752
4011
|
}
|
|
3753
4012
|
stats.durationMs = Date.now() - startTime;
|
|
4013
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
4014
|
+
this.indexCompatibility = { compatible: true };
|
|
3754
4015
|
this.logger.recordIndexingEnd();
|
|
3755
4016
|
this.logger.info("Indexing complete", {
|
|
3756
4017
|
files: stats.totalFiles,
|
|
@@ -3771,6 +4032,7 @@ var Indexer = class {
|
|
|
3771
4032
|
chunksProcessed: stats.indexedChunks,
|
|
3772
4033
|
totalChunks: pendingChunks.length
|
|
3773
4034
|
});
|
|
4035
|
+
this.releaseIndexingLock();
|
|
3774
4036
|
return stats;
|
|
3775
4037
|
}
|
|
3776
4038
|
async getQueryEmbedding(query, provider) {
|
|
@@ -3793,7 +4055,7 @@ var Indexer = class {
|
|
|
3793
4055
|
}
|
|
3794
4056
|
this.logger.cache("debug", "Query embedding cache miss", { query: query.slice(0, 50) });
|
|
3795
4057
|
this.logger.recordQueryCacheMiss();
|
|
3796
|
-
const { embedding, tokensUsed } = await provider.
|
|
4058
|
+
const { embedding, tokensUsed } = await provider.embedQuery(query);
|
|
3797
4059
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
3798
4060
|
if (this.queryEmbeddingCache.size >= this.maxQueryCacheSize) {
|
|
3799
4061
|
const oldestKey = this.queryEmbeddingCache.keys().next().value;
|
|
@@ -3836,8 +4098,14 @@ var Indexer = class {
|
|
|
3836
4098
|
return intersection / union;
|
|
3837
4099
|
}
|
|
3838
4100
|
async search(query, limit, options) {
|
|
3839
|
-
const searchStartTime = performance2.now();
|
|
3840
4101
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4102
|
+
const compatibility = this.checkCompatibility();
|
|
4103
|
+
if (!compatibility.compatible) {
|
|
4104
|
+
throw new Error(
|
|
4105
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} A possible solution is to run index_codebase with force=true to rebuild the index.`
|
|
4106
|
+
);
|
|
4107
|
+
}
|
|
4108
|
+
const searchStartTime = performance2.now();
|
|
3841
4109
|
if (store.count() === 0) {
|
|
3842
4110
|
this.logger.search("debug", "Search on empty index", { query });
|
|
3843
4111
|
return [];
|
|
@@ -3979,15 +4247,16 @@ var Indexer = class {
|
|
|
3979
4247
|
return results.slice(0, limit);
|
|
3980
4248
|
}
|
|
3981
4249
|
async getStatus() {
|
|
3982
|
-
const { store,
|
|
4250
|
+
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
3983
4251
|
return {
|
|
3984
4252
|
indexed: store.count() > 0,
|
|
3985
4253
|
vectorCount: store.count(),
|
|
3986
|
-
provider:
|
|
3987
|
-
model:
|
|
4254
|
+
provider: configuredProviderInfo.provider,
|
|
4255
|
+
model: configuredProviderInfo.modelInfo.model,
|
|
3988
4256
|
indexPath: this.indexPath,
|
|
3989
4257
|
currentBranch: this.currentBranch,
|
|
3990
|
-
baseBranch: this.baseBranch
|
|
4258
|
+
baseBranch: this.baseBranch,
|
|
4259
|
+
compatibility: this.indexCompatibility
|
|
3991
4260
|
};
|
|
3992
4261
|
}
|
|
3993
4262
|
async clearIndex() {
|
|
@@ -3999,6 +4268,13 @@ var Indexer = class {
|
|
|
3999
4268
|
this.fileHashCache.clear();
|
|
4000
4269
|
this.saveFileHashCache();
|
|
4001
4270
|
database.clearBranch(this.currentBranch);
|
|
4271
|
+
database.deleteMetadata("index.version");
|
|
4272
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
4273
|
+
database.deleteMetadata("index.embeddingModel");
|
|
4274
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
4275
|
+
database.deleteMetadata("index.createdAt");
|
|
4276
|
+
database.deleteMetadata("index.updatedAt");
|
|
4277
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
4002
4278
|
}
|
|
4003
4279
|
async healthCheck() {
|
|
4004
4280
|
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
@@ -4112,26 +4388,31 @@ var Indexer = class {
|
|
|
4112
4388
|
getLogger() {
|
|
4113
4389
|
return this.logger;
|
|
4114
4390
|
}
|
|
4115
|
-
async findSimilar(code, limit, options) {
|
|
4116
|
-
const searchStartTime = performance2.now();
|
|
4391
|
+
async findSimilar(code, limit = this.config.search.maxResults, options) {
|
|
4117
4392
|
const { store, provider, database } = await this.ensureInitialized();
|
|
4393
|
+
const compatibility = this.checkCompatibility();
|
|
4394
|
+
if (!compatibility.compatible) {
|
|
4395
|
+
throw new Error(
|
|
4396
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} Run index_codebase with force=true to rebuild the index.`
|
|
4397
|
+
);
|
|
4398
|
+
}
|
|
4399
|
+
const searchStartTime = performance2.now();
|
|
4118
4400
|
if (store.count() === 0) {
|
|
4119
4401
|
this.logger.search("debug", "Find similar on empty index");
|
|
4120
4402
|
return [];
|
|
4121
4403
|
}
|
|
4122
|
-
const maxResults = limit ?? this.config.search.maxResults;
|
|
4123
4404
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
4124
4405
|
this.logger.search("debug", "Starting find similar", {
|
|
4125
4406
|
codeLength: code.length,
|
|
4126
|
-
|
|
4407
|
+
limit,
|
|
4127
4408
|
filterByBranch
|
|
4128
4409
|
});
|
|
4129
4410
|
const embeddingStartTime = performance2.now();
|
|
4130
|
-
const { embedding, tokensUsed } = await provider.
|
|
4411
|
+
const { embedding, tokensUsed } = await provider.embedDocument(code);
|
|
4131
4412
|
const embeddingMs = performance2.now() - embeddingStartTime;
|
|
4132
4413
|
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
4133
4414
|
const vectorStartTime = performance2.now();
|
|
4134
|
-
const semanticResults = store.search(embedding,
|
|
4415
|
+
const semanticResults = store.search(embedding, limit * 2);
|
|
4135
4416
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
4136
4417
|
let branchChunkIds = null;
|
|
4137
4418
|
if (filterByBranch && this.currentBranch !== "default") {
|
|
@@ -4155,7 +4436,7 @@ var Indexer = class {
|
|
|
4155
4436
|
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
4156
4437
|
}
|
|
4157
4438
|
return true;
|
|
4158
|
-
}).slice(0,
|
|
4439
|
+
}).slice(0, limit);
|
|
4159
4440
|
const totalSearchMs = performance2.now() - searchStartTime;
|
|
4160
4441
|
this.logger.recordSearch(totalSearchMs, {
|
|
4161
4442
|
embeddingMs,
|
|
@@ -5070,7 +5351,7 @@ var NodeFsHandler = class {
|
|
|
5070
5351
|
this._addToNodeFs(path9, initialAdd, wh, depth + 1);
|
|
5071
5352
|
}
|
|
5072
5353
|
}).on(EV.ERROR, this._boundHandleError);
|
|
5073
|
-
return new Promise((
|
|
5354
|
+
return new Promise((resolve5, reject) => {
|
|
5074
5355
|
if (!stream)
|
|
5075
5356
|
return reject();
|
|
5076
5357
|
stream.once(STR_END, () => {
|
|
@@ -5079,7 +5360,7 @@ var NodeFsHandler = class {
|
|
|
5079
5360
|
return;
|
|
5080
5361
|
}
|
|
5081
5362
|
const wasThrottled = throttler ? throttler.clear() : false;
|
|
5082
|
-
|
|
5363
|
+
resolve5(void 0);
|
|
5083
5364
|
previous.getChildren().filter((item) => {
|
|
5084
5365
|
return item !== directory && !current.has(item);
|
|
5085
5366
|
}).forEach((item) => {
|
|
@@ -6126,7 +6407,8 @@ function createWatcherWithIndexer(indexer, projectRoot, config) {
|
|
|
6126
6407
|
|
|
6127
6408
|
// src/tools/index.ts
|
|
6128
6409
|
import { tool } from "@opencode-ai/plugin";
|
|
6129
|
-
|
|
6410
|
+
|
|
6411
|
+
// src/tools/utils.ts
|
|
6130
6412
|
var MAX_CONTENT_LINES = 30;
|
|
6131
6413
|
function truncateContent(content) {
|
|
6132
6414
|
const lines = content.split("\n");
|
|
@@ -6134,6 +6416,167 @@ function truncateContent(content) {
|
|
|
6134
6416
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
6135
6417
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
6136
6418
|
}
|
|
6419
|
+
function formatIndexStats(stats, verbose = false) {
|
|
6420
|
+
const lines = [];
|
|
6421
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
6422
|
+
lines.push(`Indexed. ${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
6423
|
+
} else if (stats.indexedChunks === 0) {
|
|
6424
|
+
lines.push(`Indexed. ${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
6425
|
+
} else {
|
|
6426
|
+
let main = `Indexed. ${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
6427
|
+
if (stats.existingChunks > 0) {
|
|
6428
|
+
main += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
6429
|
+
}
|
|
6430
|
+
lines.push(main);
|
|
6431
|
+
if (stats.removedChunks > 0) {
|
|
6432
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
6433
|
+
}
|
|
6434
|
+
if (stats.failedChunks > 0) {
|
|
6435
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
6436
|
+
}
|
|
6437
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
6438
|
+
}
|
|
6439
|
+
if (verbose) {
|
|
6440
|
+
if (stats.skippedFiles.length > 0) {
|
|
6441
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6442
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6443
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6444
|
+
lines.push("");
|
|
6445
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6446
|
+
if (tooLarge.length > 0) {
|
|
6447
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6448
|
+
}
|
|
6449
|
+
if (excluded.length > 0) {
|
|
6450
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6451
|
+
}
|
|
6452
|
+
if (gitignored.length > 0) {
|
|
6453
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6454
|
+
}
|
|
6455
|
+
}
|
|
6456
|
+
if (stats.parseFailures.length > 0) {
|
|
6457
|
+
lines.push("");
|
|
6458
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6459
|
+
}
|
|
6460
|
+
}
|
|
6461
|
+
return lines.join("\n");
|
|
6462
|
+
}
|
|
6463
|
+
function formatStatus(status) {
|
|
6464
|
+
if (!status.indexed) {
|
|
6465
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6466
|
+
}
|
|
6467
|
+
const lines = [
|
|
6468
|
+
`Index status:`,
|
|
6469
|
+
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6470
|
+
` Provider: ${status.provider}`,
|
|
6471
|
+
` Model: ${status.model}`,
|
|
6472
|
+
` Location: ${status.indexPath}`
|
|
6473
|
+
];
|
|
6474
|
+
if (status.currentBranch !== "default") {
|
|
6475
|
+
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6476
|
+
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6477
|
+
}
|
|
6478
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
6479
|
+
lines.push("");
|
|
6480
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
6481
|
+
if (status.compatibility.storedMetadata) {
|
|
6482
|
+
const stored = status.compatibility.storedMetadata;
|
|
6483
|
+
lines.push(` Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
6484
|
+
lines.push(` Current config: ${status.provider}/${status.model}`);
|
|
6485
|
+
}
|
|
6486
|
+
} else if (!status.compatibility) {
|
|
6487
|
+
lines.push(` Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
6488
|
+
} else {
|
|
6489
|
+
lines.push(` Compatibility: Index is compatible with the current provider and model.`);
|
|
6490
|
+
}
|
|
6491
|
+
return lines.join("\n");
|
|
6492
|
+
}
|
|
6493
|
+
function formatProgressTitle(progress) {
|
|
6494
|
+
switch (progress.phase) {
|
|
6495
|
+
case "scanning":
|
|
6496
|
+
return "Scanning files...";
|
|
6497
|
+
case "parsing":
|
|
6498
|
+
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6499
|
+
case "embedding":
|
|
6500
|
+
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6501
|
+
case "storing":
|
|
6502
|
+
return "Storing index...";
|
|
6503
|
+
case "complete":
|
|
6504
|
+
return "Indexing complete";
|
|
6505
|
+
default:
|
|
6506
|
+
return "Indexing...";
|
|
6507
|
+
}
|
|
6508
|
+
}
|
|
6509
|
+
function calculatePercentage(progress) {
|
|
6510
|
+
if (progress.phase === "scanning") return 0;
|
|
6511
|
+
if (progress.phase === "complete") return 100;
|
|
6512
|
+
if (progress.phase === "parsing") {
|
|
6513
|
+
if (progress.totalFiles === 0) return 5;
|
|
6514
|
+
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6515
|
+
}
|
|
6516
|
+
if (progress.phase === "embedding") {
|
|
6517
|
+
if (progress.totalChunks === 0) return 20;
|
|
6518
|
+
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6519
|
+
}
|
|
6520
|
+
if (progress.phase === "storing") return 95;
|
|
6521
|
+
return 0;
|
|
6522
|
+
}
|
|
6523
|
+
function formatCodebasePeek(results, query) {
|
|
6524
|
+
if (results.length === 0) {
|
|
6525
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6526
|
+
}
|
|
6527
|
+
const formatted = results.map((r, idx) => {
|
|
6528
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6529
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6530
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6531
|
+
});
|
|
6532
|
+
return `Found ${results.length} locations for "${query}":
|
|
6533
|
+
|
|
6534
|
+
${formatted.join("\n")}
|
|
6535
|
+
|
|
6536
|
+
Use Read tool to examine specific files.`;
|
|
6537
|
+
}
|
|
6538
|
+
function formatHealthCheck(result) {
|
|
6539
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0) {
|
|
6540
|
+
return "Index is healthy. No stale entries found.";
|
|
6541
|
+
}
|
|
6542
|
+
const lines = [`Health check complete:`];
|
|
6543
|
+
if (result.removed > 0) {
|
|
6544
|
+
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6545
|
+
}
|
|
6546
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
6547
|
+
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6548
|
+
}
|
|
6549
|
+
if (result.gcOrphanChunks > 0) {
|
|
6550
|
+
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6551
|
+
}
|
|
6552
|
+
if (result.filePaths.length > 0) {
|
|
6553
|
+
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6554
|
+
}
|
|
6555
|
+
return lines.join("\n");
|
|
6556
|
+
}
|
|
6557
|
+
function formatLogs(logs) {
|
|
6558
|
+
if (logs.length === 0) {
|
|
6559
|
+
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6560
|
+
}
|
|
6561
|
+
return logs.map((l) => {
|
|
6562
|
+
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6563
|
+
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6564
|
+
}).join("\n");
|
|
6565
|
+
}
|
|
6566
|
+
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
6567
|
+
const formatted = results.map((r, idx) => {
|
|
6568
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6569
|
+
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
6570
|
+
return `${header} ${scoreLabel}
|
|
6571
|
+
\`\`\`
|
|
6572
|
+
${truncateContent(r.content)}
|
|
6573
|
+
\`\`\``;
|
|
6574
|
+
});
|
|
6575
|
+
return formatted.join("\n\n");
|
|
6576
|
+
}
|
|
6577
|
+
|
|
6578
|
+
// src/tools/index.ts
|
|
6579
|
+
var z = tool.schema;
|
|
6137
6580
|
var sharedIndexer = null;
|
|
6138
6581
|
function initializeTools(projectRoot, config) {
|
|
6139
6582
|
sharedIndexer = new Indexer(projectRoot, config);
|
|
@@ -6144,39 +6587,6 @@ function getIndexer() {
|
|
|
6144
6587
|
}
|
|
6145
6588
|
return sharedIndexer;
|
|
6146
6589
|
}
|
|
6147
|
-
var codebase_search = tool({
|
|
6148
|
-
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6149
|
-
args: {
|
|
6150
|
-
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6151
|
-
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6152
|
-
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6153
|
-
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6154
|
-
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6155
|
-
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6156
|
-
},
|
|
6157
|
-
async execute(args) {
|
|
6158
|
-
const indexer = getIndexer();
|
|
6159
|
-
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6160
|
-
fileType: args.fileType,
|
|
6161
|
-
directory: args.directory,
|
|
6162
|
-
chunkType: args.chunkType,
|
|
6163
|
-
contextLines: args.contextLines
|
|
6164
|
-
});
|
|
6165
|
-
if (results.length === 0) {
|
|
6166
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6167
|
-
}
|
|
6168
|
-
const formatted = results.map((r, idx) => {
|
|
6169
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6170
|
-
return `${header} (score: ${r.score.toFixed(2)})
|
|
6171
|
-
\`\`\`
|
|
6172
|
-
${truncateContent(r.content)}
|
|
6173
|
-
\`\`\``;
|
|
6174
|
-
});
|
|
6175
|
-
return `Found ${results.length} results for "${args.query}":
|
|
6176
|
-
|
|
6177
|
-
${formatted.join("\n\n")}`;
|
|
6178
|
-
}
|
|
6179
|
-
});
|
|
6180
6590
|
var codebase_peek = tool({
|
|
6181
6591
|
description: "Quick lookup of code locations by meaning. Returns only metadata (file, line, name, type) WITHOUT code content. Use this first to find WHERE code is, then use Read tool to examine specific files. Saves tokens by not returning full code blocks. Best for: discovery, navigation, finding multiple related locations.",
|
|
6182
6592
|
args: {
|
|
@@ -6194,19 +6604,7 @@ var codebase_peek = tool({
|
|
|
6194
6604
|
chunkType: args.chunkType,
|
|
6195
6605
|
metadataOnly: true
|
|
6196
6606
|
});
|
|
6197
|
-
|
|
6198
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6199
|
-
}
|
|
6200
|
-
const formatted = results.map((r, idx) => {
|
|
6201
|
-
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6202
|
-
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
6203
|
-
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
6204
|
-
});
|
|
6205
|
-
return `Found ${results.length} locations for "${args.query}":
|
|
6206
|
-
|
|
6207
|
-
${formatted.join("\n")}
|
|
6208
|
-
|
|
6209
|
-
Use Read tool to examine specific files.`;
|
|
6607
|
+
return formatCodebasePeek(results, args.query);
|
|
6210
6608
|
}
|
|
6211
6609
|
});
|
|
6212
6610
|
var index_codebase = tool({
|
|
@@ -6256,23 +6654,7 @@ var index_health_check = tool({
|
|
|
6256
6654
|
async execute() {
|
|
6257
6655
|
const indexer = getIndexer();
|
|
6258
6656
|
const result = await indexer.healthCheck();
|
|
6259
|
-
|
|
6260
|
-
return "Index is healthy. No stale entries found.";
|
|
6261
|
-
}
|
|
6262
|
-
const lines = [`Health check complete:`];
|
|
6263
|
-
if (result.removed > 0) {
|
|
6264
|
-
lines.push(` Removed stale entries: ${result.removed}`);
|
|
6265
|
-
}
|
|
6266
|
-
if (result.gcOrphanEmbeddings > 0) {
|
|
6267
|
-
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
6268
|
-
}
|
|
6269
|
-
if (result.gcOrphanChunks > 0) {
|
|
6270
|
-
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
6271
|
-
}
|
|
6272
|
-
if (result.filePaths.length > 0) {
|
|
6273
|
-
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
6274
|
-
}
|
|
6275
|
-
return lines.join("\n");
|
|
6657
|
+
return formatHealthCheck(result);
|
|
6276
6658
|
}
|
|
6277
6659
|
});
|
|
6278
6660
|
var index_metrics = tool({
|
|
@@ -6311,13 +6693,7 @@ var index_logs = tool({
|
|
|
6311
6693
|
} else {
|
|
6312
6694
|
logs = logger.getLogs(args.limit);
|
|
6313
6695
|
}
|
|
6314
|
-
|
|
6315
|
-
return "No logs recorded yet. Logs are captured during indexing and search operations.";
|
|
6316
|
-
}
|
|
6317
|
-
return logs.map((l) => {
|
|
6318
|
-
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
6319
|
-
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
6320
|
-
}).join("\n");
|
|
6696
|
+
return formatLogs(logs);
|
|
6321
6697
|
}
|
|
6322
6698
|
});
|
|
6323
6699
|
var find_similar = tool({
|
|
@@ -6332,7 +6708,7 @@ var find_similar = tool({
|
|
|
6332
6708
|
},
|
|
6333
6709
|
async execute(args) {
|
|
6334
6710
|
const indexer = getIndexer();
|
|
6335
|
-
const results = await indexer.findSimilar(args.code, args.limit
|
|
6711
|
+
const results = await indexer.findSimilar(args.code, args.limit, {
|
|
6336
6712
|
fileType: args.fileType,
|
|
6337
6713
|
directory: args.directory,
|
|
6338
6714
|
chunkType: args.chunkType,
|
|
@@ -6341,109 +6717,37 @@ var find_similar = tool({
|
|
|
6341
6717
|
if (results.length === 0) {
|
|
6342
6718
|
return "No similar code found. Try a different snippet or run index_codebase first.";
|
|
6343
6719
|
}
|
|
6344
|
-
const formatted = results.map((r, idx) => {
|
|
6345
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
6346
|
-
return `${header} (similarity: ${(r.score * 100).toFixed(1)}%)
|
|
6347
|
-
\`\`\`
|
|
6348
|
-
${truncateContent(r.content)}
|
|
6349
|
-
\`\`\``;
|
|
6350
|
-
});
|
|
6351
6720
|
return `Found ${results.length} similar code blocks:
|
|
6352
6721
|
|
|
6353
|
-
${
|
|
6722
|
+
${formatSearchResults(results)}`;
|
|
6354
6723
|
}
|
|
6355
6724
|
});
|
|
6356
|
-
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
}
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
if (verbose) {
|
|
6377
|
-
if (stats.skippedFiles.length > 0) {
|
|
6378
|
-
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
6379
|
-
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
6380
|
-
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
6381
|
-
lines.push("");
|
|
6382
|
-
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
6383
|
-
if (tooLarge.length > 0) {
|
|
6384
|
-
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
6385
|
-
}
|
|
6386
|
-
if (excluded.length > 0) {
|
|
6387
|
-
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
6388
|
-
}
|
|
6389
|
-
if (gitignored.length > 0) {
|
|
6390
|
-
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
6391
|
-
}
|
|
6392
|
-
}
|
|
6393
|
-
if (stats.parseFailures.length > 0) {
|
|
6394
|
-
lines.push("");
|
|
6395
|
-
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
6725
|
+
var codebase_search = tool({
|
|
6726
|
+
description: "Search codebase by MEANING, not keywords. Returns full code content. Use when you need to see actual implementation. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead. For known identifiers like 'validateToken', use grep - it's faster.",
|
|
6727
|
+
args: {
|
|
6728
|
+
query: z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
6729
|
+
limit: z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
6730
|
+
fileType: z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
6731
|
+
directory: z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
6732
|
+
chunkType: z.enum(["function", "class", "method", "interface", "type", "enum", "struct", "impl", "trait", "module", "other"]).optional().describe("Filter by code chunk type"),
|
|
6733
|
+
contextLines: z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
6734
|
+
},
|
|
6735
|
+
async execute(args) {
|
|
6736
|
+
const indexer = getIndexer();
|
|
6737
|
+
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
6738
|
+
fileType: args.fileType,
|
|
6739
|
+
directory: args.directory,
|
|
6740
|
+
chunkType: args.chunkType,
|
|
6741
|
+
contextLines: args.contextLines
|
|
6742
|
+
});
|
|
6743
|
+
if (results.length === 0) {
|
|
6744
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
6396
6745
|
}
|
|
6746
|
+
return `Found ${results.length} results for "${args.query}":
|
|
6747
|
+
|
|
6748
|
+
${formatSearchResults(results, "score")}`;
|
|
6397
6749
|
}
|
|
6398
|
-
|
|
6399
|
-
}
|
|
6400
|
-
function formatStatus(status) {
|
|
6401
|
-
if (!status.indexed) {
|
|
6402
|
-
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
6403
|
-
}
|
|
6404
|
-
const lines = [
|
|
6405
|
-
`Index status:`,
|
|
6406
|
-
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
6407
|
-
` Provider: ${status.provider}`,
|
|
6408
|
-
` Model: ${status.model}`,
|
|
6409
|
-
` Location: ${status.indexPath}`
|
|
6410
|
-
];
|
|
6411
|
-
if (status.currentBranch !== "default") {
|
|
6412
|
-
lines.push(` Current branch: ${status.currentBranch}`);
|
|
6413
|
-
lines.push(` Base branch: ${status.baseBranch}`);
|
|
6414
|
-
}
|
|
6415
|
-
return lines.join("\n");
|
|
6416
|
-
}
|
|
6417
|
-
function formatProgressTitle(progress) {
|
|
6418
|
-
switch (progress.phase) {
|
|
6419
|
-
case "scanning":
|
|
6420
|
-
return "Scanning files...";
|
|
6421
|
-
case "parsing":
|
|
6422
|
-
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
6423
|
-
case "embedding":
|
|
6424
|
-
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
6425
|
-
case "storing":
|
|
6426
|
-
return "Storing index...";
|
|
6427
|
-
case "complete":
|
|
6428
|
-
return "Indexing complete";
|
|
6429
|
-
default:
|
|
6430
|
-
return "Indexing...";
|
|
6431
|
-
}
|
|
6432
|
-
}
|
|
6433
|
-
function calculatePercentage(progress) {
|
|
6434
|
-
if (progress.phase === "scanning") return 0;
|
|
6435
|
-
if (progress.phase === "complete") return 100;
|
|
6436
|
-
if (progress.phase === "parsing") {
|
|
6437
|
-
if (progress.totalFiles === 0) return 5;
|
|
6438
|
-
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
6439
|
-
}
|
|
6440
|
-
if (progress.phase === "embedding") {
|
|
6441
|
-
if (progress.totalChunks === 0) return 20;
|
|
6442
|
-
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
6443
|
-
}
|
|
6444
|
-
if (progress.phase === "storing") return 95;
|
|
6445
|
-
return 0;
|
|
6446
|
-
}
|
|
6750
|
+
});
|
|
6447
6751
|
|
|
6448
6752
|
// src/commands/loader.ts
|
|
6449
6753
|
import { existsSync as existsSync5, readdirSync as readdirSync2, readFileSync as readFileSync5 } from "fs";
|