opencode-codebase-index 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -7
- package/commands/index.md +4 -0
- package/commands/peek.md +24 -0
- package/commands/reindex.md +25 -0
- package/commands/status.md +5 -1
- package/dist/cli.cjs +2423 -730
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2416 -723
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +2352 -711
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2341 -700
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +3 -3
package/dist/cli.cjs
CHANGED
|
@@ -487,7 +487,7 @@ var require_ignore = __commonJS({
|
|
|
487
487
|
// path matching.
|
|
488
488
|
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
489
489
|
// @returns {TestResult} true if a file is ignored
|
|
490
|
-
test(
|
|
490
|
+
test(path13, checkUnignored, mode) {
|
|
491
491
|
let ignored = false;
|
|
492
492
|
let unignored = false;
|
|
493
493
|
let matchedRule;
|
|
@@ -496,7 +496,7 @@ var require_ignore = __commonJS({
|
|
|
496
496
|
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
497
497
|
return;
|
|
498
498
|
}
|
|
499
|
-
const matched = rule[mode].test(
|
|
499
|
+
const matched = rule[mode].test(path13);
|
|
500
500
|
if (!matched) {
|
|
501
501
|
return;
|
|
502
502
|
}
|
|
@@ -517,17 +517,17 @@ var require_ignore = __commonJS({
|
|
|
517
517
|
var throwError = (message, Ctor) => {
|
|
518
518
|
throw new Ctor(message);
|
|
519
519
|
};
|
|
520
|
-
var checkPath = (
|
|
521
|
-
if (!isString(
|
|
520
|
+
var checkPath = (path13, originalPath, doThrow) => {
|
|
521
|
+
if (!isString(path13)) {
|
|
522
522
|
return doThrow(
|
|
523
523
|
`path must be a string, but got \`${originalPath}\``,
|
|
524
524
|
TypeError
|
|
525
525
|
);
|
|
526
526
|
}
|
|
527
|
-
if (!
|
|
527
|
+
if (!path13) {
|
|
528
528
|
return doThrow(`path must not be empty`, TypeError);
|
|
529
529
|
}
|
|
530
|
-
if (checkPath.isNotRelative(
|
|
530
|
+
if (checkPath.isNotRelative(path13)) {
|
|
531
531
|
const r = "`path.relative()`d";
|
|
532
532
|
return doThrow(
|
|
533
533
|
`path should be a ${r} string, but got "${originalPath}"`,
|
|
@@ -536,7 +536,7 @@ var require_ignore = __commonJS({
|
|
|
536
536
|
}
|
|
537
537
|
return true;
|
|
538
538
|
};
|
|
539
|
-
var isNotRelative = (
|
|
539
|
+
var isNotRelative = (path13) => REGEX_TEST_INVALID_PATH.test(path13);
|
|
540
540
|
checkPath.isNotRelative = isNotRelative;
|
|
541
541
|
checkPath.convert = (p) => p;
|
|
542
542
|
var Ignore2 = class {
|
|
@@ -566,19 +566,19 @@ var require_ignore = __commonJS({
|
|
|
566
566
|
}
|
|
567
567
|
// @returns {TestResult}
|
|
568
568
|
_test(originalPath, cache, checkUnignored, slices) {
|
|
569
|
-
const
|
|
569
|
+
const path13 = originalPath && checkPath.convert(originalPath);
|
|
570
570
|
checkPath(
|
|
571
|
-
|
|
571
|
+
path13,
|
|
572
572
|
originalPath,
|
|
573
573
|
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
574
574
|
);
|
|
575
|
-
return this._t(
|
|
575
|
+
return this._t(path13, cache, checkUnignored, slices);
|
|
576
576
|
}
|
|
577
|
-
checkIgnore(
|
|
578
|
-
if (!REGEX_TEST_TRAILING_SLASH.test(
|
|
579
|
-
return this.test(
|
|
577
|
+
checkIgnore(path13) {
|
|
578
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path13)) {
|
|
579
|
+
return this.test(path13);
|
|
580
580
|
}
|
|
581
|
-
const slices =
|
|
581
|
+
const slices = path13.split(SLASH).filter(Boolean);
|
|
582
582
|
slices.pop();
|
|
583
583
|
if (slices.length) {
|
|
584
584
|
const parent = this._t(
|
|
@@ -591,18 +591,18 @@ var require_ignore = __commonJS({
|
|
|
591
591
|
return parent;
|
|
592
592
|
}
|
|
593
593
|
}
|
|
594
|
-
return this._rules.test(
|
|
594
|
+
return this._rules.test(path13, false, MODE_CHECK_IGNORE);
|
|
595
595
|
}
|
|
596
|
-
_t(
|
|
597
|
-
if (
|
|
598
|
-
return cache[
|
|
596
|
+
_t(path13, cache, checkUnignored, slices) {
|
|
597
|
+
if (path13 in cache) {
|
|
598
|
+
return cache[path13];
|
|
599
599
|
}
|
|
600
600
|
if (!slices) {
|
|
601
|
-
slices =
|
|
601
|
+
slices = path13.split(SLASH).filter(Boolean);
|
|
602
602
|
}
|
|
603
603
|
slices.pop();
|
|
604
604
|
if (!slices.length) {
|
|
605
|
-
return cache[
|
|
605
|
+
return cache[path13] = this._rules.test(path13, checkUnignored, MODE_IGNORE);
|
|
606
606
|
}
|
|
607
607
|
const parent = this._t(
|
|
608
608
|
slices.join(SLASH) + SLASH,
|
|
@@ -610,29 +610,29 @@ var require_ignore = __commonJS({
|
|
|
610
610
|
checkUnignored,
|
|
611
611
|
slices
|
|
612
612
|
);
|
|
613
|
-
return cache[
|
|
613
|
+
return cache[path13] = parent.ignored ? parent : this._rules.test(path13, checkUnignored, MODE_IGNORE);
|
|
614
614
|
}
|
|
615
|
-
ignores(
|
|
616
|
-
return this._test(
|
|
615
|
+
ignores(path13) {
|
|
616
|
+
return this._test(path13, this._ignoreCache, false).ignored;
|
|
617
617
|
}
|
|
618
618
|
createFilter() {
|
|
619
|
-
return (
|
|
619
|
+
return (path13) => !this.ignores(path13);
|
|
620
620
|
}
|
|
621
621
|
filter(paths) {
|
|
622
622
|
return makeArray(paths).filter(this.createFilter());
|
|
623
623
|
}
|
|
624
624
|
// @returns {TestResult}
|
|
625
|
-
test(
|
|
626
|
-
return this._test(
|
|
625
|
+
test(path13) {
|
|
626
|
+
return this._test(path13, this._testCache, true);
|
|
627
627
|
}
|
|
628
628
|
};
|
|
629
629
|
var factory = (options) => new Ignore2(options);
|
|
630
|
-
var isPathValid = (
|
|
630
|
+
var isPathValid = (path13) => checkPath(path13 && checkPath.convert(path13), path13, RETURN_FALSE);
|
|
631
631
|
var setupWindows = () => {
|
|
632
632
|
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
633
633
|
checkPath.convert = makePosix;
|
|
634
634
|
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
635
|
-
checkPath.isNotRelative = (
|
|
635
|
+
checkPath.isNotRelative = (path13) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path13) || isNotRelative(path13);
|
|
636
636
|
};
|
|
637
637
|
if (
|
|
638
638
|
// Detect `process` so that it can run in browsers.
|
|
@@ -649,7 +649,7 @@ var require_ignore = __commonJS({
|
|
|
649
649
|
|
|
650
650
|
// src/cli.ts
|
|
651
651
|
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
652
|
-
var
|
|
652
|
+
var path12 = __toESM(require("path"), 1);
|
|
653
653
|
|
|
654
654
|
// src/config/constants.ts
|
|
655
655
|
var DEFAULT_INCLUDE = [
|
|
@@ -658,12 +658,14 @@ var DEFAULT_INCLUDE = [
|
|
|
658
658
|
"**/*.{go,rs,java,kt,scala}",
|
|
659
659
|
"**/*.{c,cpp,cc,h,hpp}",
|
|
660
660
|
"**/*.{rb,php,inc,swift}",
|
|
661
|
+
"**/*.{cls,trigger}",
|
|
661
662
|
"**/*.{vue,svelte,astro}",
|
|
662
663
|
"**/*.{sql,graphql,proto}",
|
|
663
664
|
"**/*.{yaml,yml,toml}",
|
|
664
665
|
"**/*.{md,mdx}",
|
|
665
666
|
"**/*.{sh,bash,zsh}",
|
|
666
|
-
"**/*.{txt,html,htm}"
|
|
667
|
+
"**/*.{txt,html,htm}",
|
|
668
|
+
"**/*.zig"
|
|
667
669
|
];
|
|
668
670
|
var DEFAULT_EXCLUDE = [
|
|
669
671
|
"**/node_modules/**",
|
|
@@ -728,7 +730,7 @@ var EMBEDDING_MODELS = {
|
|
|
728
730
|
provider: "ollama",
|
|
729
731
|
model: "nomic-embed-text",
|
|
730
732
|
dimensions: 768,
|
|
731
|
-
maxTokens:
|
|
733
|
+
maxTokens: 2048,
|
|
732
734
|
costPer1MTokens: 0
|
|
733
735
|
},
|
|
734
736
|
"mxbai-embed-large": {
|
|
@@ -752,9 +754,15 @@ var EMBEDDING_MODELS = {
|
|
|
752
754
|
var DEFAULT_PROVIDER_MODELS = {
|
|
753
755
|
"github-copilot": "text-embedding-3-small",
|
|
754
756
|
"openai": "text-embedding-3-small",
|
|
755
|
-
"google": "
|
|
757
|
+
"google": "gemini-embedding-001",
|
|
756
758
|
"ollama": "nomic-embed-text"
|
|
757
759
|
};
|
|
760
|
+
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
761
|
+
"ollama",
|
|
762
|
+
"github-copilot",
|
|
763
|
+
"openai",
|
|
764
|
+
"google"
|
|
765
|
+
];
|
|
758
766
|
|
|
759
767
|
// src/config/env-substitution.ts
|
|
760
768
|
var ENV_REFERENCE_PATTERN = /^\{env:([A-Z_][A-Z0-9_]*)\}$/;
|
|
@@ -1013,9 +1021,12 @@ function getDefaultModelForProvider(provider) {
|
|
|
1013
1021
|
return models[providerDefault];
|
|
1014
1022
|
}
|
|
1015
1023
|
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
1024
|
+
var autoDetectProviders = AUTO_DETECT_PROVIDER_ORDER.filter(
|
|
1025
|
+
(provider) => provider in EMBEDDING_MODELS
|
|
1026
|
+
);
|
|
1016
1027
|
|
|
1017
1028
|
// src/eval/cli.ts
|
|
1018
|
-
var
|
|
1029
|
+
var path10 = __toESM(require("path"), 1);
|
|
1019
1030
|
|
|
1020
1031
|
// src/eval/compare.ts
|
|
1021
1032
|
function metricDelta(current, baseline) {
|
|
@@ -1058,9 +1069,9 @@ function compareSummaries(current, baseline, againstPath) {
|
|
|
1058
1069
|
// src/eval/reports.ts
|
|
1059
1070
|
var import_fs = require("fs");
|
|
1060
1071
|
var path = __toESM(require("path"), 1);
|
|
1061
|
-
function assertFiniteNumber(value,
|
|
1072
|
+
function assertFiniteNumber(value, path13) {
|
|
1062
1073
|
if (typeof value !== "number" || Number.isNaN(value) || !Number.isFinite(value)) {
|
|
1063
|
-
throw new Error(`${
|
|
1074
|
+
throw new Error(`${path13} must be a finite number`);
|
|
1064
1075
|
}
|
|
1065
1076
|
return value;
|
|
1066
1077
|
}
|
|
@@ -1234,16 +1245,352 @@ function buildPerQueryArtifact(perQuery) {
|
|
|
1234
1245
|
}
|
|
1235
1246
|
|
|
1236
1247
|
// src/eval/runner.ts
|
|
1237
|
-
var import_fs7 = require("fs");
|
|
1238
|
-
var import_fs8 = require("fs");
|
|
1239
1248
|
var import_fs9 = require("fs");
|
|
1240
|
-
var
|
|
1241
|
-
var
|
|
1249
|
+
var import_fs10 = require("fs");
|
|
1250
|
+
var import_fs11 = require("fs");
|
|
1251
|
+
var import_fs12 = require("fs");
|
|
1252
|
+
var import_fs13 = require("fs");
|
|
1253
|
+
var os5 = __toESM(require("os"), 1);
|
|
1254
|
+
var path9 = __toESM(require("path"), 1);
|
|
1242
1255
|
var import_perf_hooks2 = require("perf_hooks");
|
|
1243
1256
|
|
|
1257
|
+
// src/config/merger.ts
|
|
1258
|
+
var import_fs4 = require("fs");
|
|
1259
|
+
var os2 = __toESM(require("os"), 1);
|
|
1260
|
+
var path4 = __toESM(require("path"), 1);
|
|
1261
|
+
|
|
1262
|
+
// src/config/paths.ts
|
|
1263
|
+
var import_fs3 = require("fs");
|
|
1264
|
+
var os = __toESM(require("os"), 1);
|
|
1265
|
+
var path3 = __toESM(require("path"), 1);
|
|
1266
|
+
|
|
1267
|
+
// src/git/index.ts
|
|
1268
|
+
var import_fs2 = require("fs");
|
|
1269
|
+
var path2 = __toESM(require("path"), 1);
|
|
1270
|
+
function readPackedRefs(gitDir) {
|
|
1271
|
+
const packedRefsPath = path2.join(gitDir, "packed-refs");
|
|
1272
|
+
if (!(0, import_fs2.existsSync)(packedRefsPath)) {
|
|
1273
|
+
return [];
|
|
1274
|
+
}
|
|
1275
|
+
try {
|
|
1276
|
+
return (0, import_fs2.readFileSync)(packedRefsPath, "utf-8").split("\n").map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("^"));
|
|
1277
|
+
} catch {
|
|
1278
|
+
return [];
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
function resolveCommonGitDir(gitDir) {
|
|
1282
|
+
const commonDirPath = path2.join(gitDir, "commondir");
|
|
1283
|
+
if (!(0, import_fs2.existsSync)(commonDirPath)) {
|
|
1284
|
+
return gitDir;
|
|
1285
|
+
}
|
|
1286
|
+
try {
|
|
1287
|
+
const raw = (0, import_fs2.readFileSync)(commonDirPath, "utf-8").trim();
|
|
1288
|
+
if (!raw) {
|
|
1289
|
+
return gitDir;
|
|
1290
|
+
}
|
|
1291
|
+
const resolved = path2.isAbsolute(raw) ? raw : path2.resolve(gitDir, raw);
|
|
1292
|
+
if ((0, import_fs2.existsSync)(resolved)) {
|
|
1293
|
+
return resolved;
|
|
1294
|
+
}
|
|
1295
|
+
} catch {
|
|
1296
|
+
return gitDir;
|
|
1297
|
+
}
|
|
1298
|
+
return gitDir;
|
|
1299
|
+
}
|
|
1300
|
+
function resolveWorktreeMainRepoRoot(repoRoot) {
|
|
1301
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
1302
|
+
if (!gitDir) {
|
|
1303
|
+
return null;
|
|
1304
|
+
}
|
|
1305
|
+
const commonGitDir = resolveCommonGitDir(gitDir);
|
|
1306
|
+
if (commonGitDir === gitDir || path2.basename(commonGitDir) !== ".git") {
|
|
1307
|
+
return null;
|
|
1308
|
+
}
|
|
1309
|
+
const mainRepoRoot = path2.dirname(commonGitDir);
|
|
1310
|
+
if (!(0, import_fs2.existsSync)(mainRepoRoot)) {
|
|
1311
|
+
return null;
|
|
1312
|
+
}
|
|
1313
|
+
return path2.resolve(mainRepoRoot) === path2.resolve(repoRoot) ? null : mainRepoRoot;
|
|
1314
|
+
}
|
|
1315
|
+
function resolveGitDir(repoRoot) {
|
|
1316
|
+
const gitPath = path2.join(repoRoot, ".git");
|
|
1317
|
+
if (!(0, import_fs2.existsSync)(gitPath)) {
|
|
1318
|
+
return null;
|
|
1319
|
+
}
|
|
1320
|
+
try {
|
|
1321
|
+
const stat = (0, import_fs2.statSync)(gitPath);
|
|
1322
|
+
if (stat.isDirectory()) {
|
|
1323
|
+
return gitPath;
|
|
1324
|
+
}
|
|
1325
|
+
if (stat.isFile()) {
|
|
1326
|
+
const content = (0, import_fs2.readFileSync)(gitPath, "utf-8").trim();
|
|
1327
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
1328
|
+
if (match) {
|
|
1329
|
+
const gitdir = match[1];
|
|
1330
|
+
const resolvedPath = path2.isAbsolute(gitdir) ? gitdir : path2.resolve(repoRoot, gitdir);
|
|
1331
|
+
if ((0, import_fs2.existsSync)(resolvedPath)) {
|
|
1332
|
+
return resolvedPath;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
} catch {
|
|
1337
|
+
}
|
|
1338
|
+
return null;
|
|
1339
|
+
}
|
|
1340
|
+
function isGitRepo(dir) {
|
|
1341
|
+
return resolveGitDir(dir) !== null;
|
|
1342
|
+
}
|
|
1343
|
+
function getCurrentBranch(repoRoot) {
|
|
1344
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
1345
|
+
if (!gitDir) {
|
|
1346
|
+
return null;
|
|
1347
|
+
}
|
|
1348
|
+
const headPath = path2.join(gitDir, "HEAD");
|
|
1349
|
+
if (!(0, import_fs2.existsSync)(headPath)) {
|
|
1350
|
+
return null;
|
|
1351
|
+
}
|
|
1352
|
+
try {
|
|
1353
|
+
const headContent = (0, import_fs2.readFileSync)(headPath, "utf-8").trim();
|
|
1354
|
+
const match = headContent.match(/^ref: refs\/heads\/(.+)$/);
|
|
1355
|
+
if (match) {
|
|
1356
|
+
return match[1];
|
|
1357
|
+
}
|
|
1358
|
+
if (/^[0-9a-f]{40}$/i.test(headContent)) {
|
|
1359
|
+
return headContent.slice(0, 7);
|
|
1360
|
+
}
|
|
1361
|
+
return null;
|
|
1362
|
+
} catch {
|
|
1363
|
+
return null;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
function getBaseBranch(repoRoot) {
|
|
1367
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
1368
|
+
const refStoreDir = gitDir ? resolveCommonGitDir(gitDir) : null;
|
|
1369
|
+
const candidates = ["main", "master", "develop", "trunk"];
|
|
1370
|
+
if (refStoreDir) {
|
|
1371
|
+
for (const candidate of candidates) {
|
|
1372
|
+
const refPath = path2.join(refStoreDir, "refs", "heads", candidate);
|
|
1373
|
+
if ((0, import_fs2.existsSync)(refPath)) {
|
|
1374
|
+
return candidate;
|
|
1375
|
+
}
|
|
1376
|
+
const packedRefs = readPackedRefs(refStoreDir);
|
|
1377
|
+
if (packedRefs.some((line) => line.endsWith(` refs/heads/${candidate}`))) {
|
|
1378
|
+
return candidate;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
return getCurrentBranch(repoRoot) ?? "main";
|
|
1383
|
+
}
|
|
1384
|
+
function getBranchOrDefault(repoRoot) {
|
|
1385
|
+
if (!isGitRepo(repoRoot)) {
|
|
1386
|
+
return "default";
|
|
1387
|
+
}
|
|
1388
|
+
return getCurrentBranch(repoRoot) ?? "default";
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
// src/config/paths.ts
|
|
1392
|
+
var PROJECT_CONFIG_RELATIVE_PATH = path3.join(".opencode", "codebase-index.json");
|
|
1393
|
+
var PROJECT_INDEX_RELATIVE_PATH = path3.join(".opencode", "index");
|
|
1394
|
+
function resolveWorktreeFallbackPath(projectRoot, relativePath) {
|
|
1395
|
+
const mainRepoRoot = resolveWorktreeMainRepoRoot(projectRoot);
|
|
1396
|
+
if (!mainRepoRoot) {
|
|
1397
|
+
return null;
|
|
1398
|
+
}
|
|
1399
|
+
const fallbackPath = path3.join(mainRepoRoot, relativePath);
|
|
1400
|
+
return (0, import_fs3.existsSync)(fallbackPath) ? fallbackPath : null;
|
|
1401
|
+
}
|
|
1402
|
+
function hasProjectConfig(projectRoot) {
|
|
1403
|
+
return (0, import_fs3.existsSync)(path3.join(projectRoot, PROJECT_CONFIG_RELATIVE_PATH));
|
|
1404
|
+
}
|
|
1405
|
+
function getGlobalIndexPath() {
|
|
1406
|
+
return path3.join(os.homedir(), ".opencode", "global-index");
|
|
1407
|
+
}
|
|
1408
|
+
function resolveProjectConfigPath(projectRoot) {
|
|
1409
|
+
const localConfigPath = path3.join(projectRoot, PROJECT_CONFIG_RELATIVE_PATH);
|
|
1410
|
+
if ((0, import_fs3.existsSync)(localConfigPath)) {
|
|
1411
|
+
return localConfigPath;
|
|
1412
|
+
}
|
|
1413
|
+
return resolveWorktreeFallbackPath(projectRoot, PROJECT_CONFIG_RELATIVE_PATH) ?? localConfigPath;
|
|
1414
|
+
}
|
|
1415
|
+
function resolveProjectIndexPath(projectRoot, scope) {
|
|
1416
|
+
if (scope === "global") {
|
|
1417
|
+
return getGlobalIndexPath();
|
|
1418
|
+
}
|
|
1419
|
+
const localIndexPath = path3.join(projectRoot, PROJECT_INDEX_RELATIVE_PATH);
|
|
1420
|
+
if ((0, import_fs3.existsSync)(localIndexPath)) {
|
|
1421
|
+
return localIndexPath;
|
|
1422
|
+
}
|
|
1423
|
+
if (hasProjectConfig(projectRoot)) {
|
|
1424
|
+
return localIndexPath;
|
|
1425
|
+
}
|
|
1426
|
+
return resolveWorktreeFallbackPath(projectRoot, PROJECT_INDEX_RELATIVE_PATH) ?? localIndexPath;
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
// src/config/merger.ts
|
|
1430
|
+
function loadJsonFile(filePath) {
|
|
1431
|
+
try {
|
|
1432
|
+
if ((0, import_fs4.existsSync)(filePath)) {
|
|
1433
|
+
const content = (0, import_fs4.readFileSync)(filePath, "utf-8");
|
|
1434
|
+
return JSON.parse(content);
|
|
1435
|
+
}
|
|
1436
|
+
} catch {
|
|
1437
|
+
}
|
|
1438
|
+
return null;
|
|
1439
|
+
}
|
|
1440
|
+
function normalizeRelativeConfigPath(candidate) {
|
|
1441
|
+
return candidate.replace(/\\/g, "/");
|
|
1442
|
+
}
|
|
1443
|
+
function rebasePathEntries(values, fromDir, toDir) {
|
|
1444
|
+
if (!Array.isArray(values)) {
|
|
1445
|
+
return [];
|
|
1446
|
+
}
|
|
1447
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
1448
|
+
const trimmed = value.trim();
|
|
1449
|
+
if (!trimmed || path4.isAbsolute(trimmed)) {
|
|
1450
|
+
return trimmed;
|
|
1451
|
+
}
|
|
1452
|
+
return normalizeRelativeConfigPath(path4.normalize(path4.relative(toDir, path4.resolve(fromDir, trimmed))));
|
|
1453
|
+
}).filter(Boolean);
|
|
1454
|
+
}
|
|
1455
|
+
function isWithinRoot(rootDir, targetPath) {
|
|
1456
|
+
const relativePath = path4.relative(rootDir, targetPath);
|
|
1457
|
+
return relativePath === "" || !relativePath.startsWith("..") && !path4.isAbsolute(relativePath);
|
|
1458
|
+
}
|
|
1459
|
+
function resolveInheritedKnowledgeBaseEntries(values, sourceRoot, targetRoot) {
|
|
1460
|
+
if (!Array.isArray(values)) {
|
|
1461
|
+
return [];
|
|
1462
|
+
}
|
|
1463
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
1464
|
+
const trimmed = value.trim();
|
|
1465
|
+
if (!trimmed) {
|
|
1466
|
+
return trimmed;
|
|
1467
|
+
}
|
|
1468
|
+
if (path4.isAbsolute(trimmed)) {
|
|
1469
|
+
if (isWithinRoot(sourceRoot, trimmed)) {
|
|
1470
|
+
return normalizeRelativeConfigPath(path4.normalize(path4.relative(sourceRoot, trimmed) || "."));
|
|
1471
|
+
}
|
|
1472
|
+
return path4.normalize(trimmed);
|
|
1473
|
+
}
|
|
1474
|
+
const resolvedFromSource = path4.resolve(sourceRoot, trimmed);
|
|
1475
|
+
if (isWithinRoot(sourceRoot, resolvedFromSource)) {
|
|
1476
|
+
return normalizeRelativeConfigPath(path4.normalize(trimmed));
|
|
1477
|
+
}
|
|
1478
|
+
return normalizeRelativeConfigPath(path4.normalize(path4.relative(targetRoot, resolvedFromSource)));
|
|
1479
|
+
}).filter(Boolean);
|
|
1480
|
+
}
|
|
1481
|
+
function materializeLocalProjectConfig(projectRoot, config) {
|
|
1482
|
+
const localConfigPath = path4.join(projectRoot, ".opencode", "codebase-index.json");
|
|
1483
|
+
(0, import_fs4.mkdirSync)(path4.dirname(localConfigPath), { recursive: true });
|
|
1484
|
+
(0, import_fs4.writeFileSync)(localConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
1485
|
+
return localConfigPath;
|
|
1486
|
+
}
|
|
1487
|
+
function loadProjectConfigLayer(projectRoot) {
|
|
1488
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot);
|
|
1489
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
1490
|
+
if (!projectConfig) {
|
|
1491
|
+
return {};
|
|
1492
|
+
}
|
|
1493
|
+
const normalizedConfig = { ...projectConfig };
|
|
1494
|
+
const projectConfigBaseDir = path4.dirname(path4.dirname(projectConfigPath));
|
|
1495
|
+
if (Array.isArray(normalizedConfig.knowledgeBases)) {
|
|
1496
|
+
normalizedConfig.knowledgeBases = resolveInheritedKnowledgeBaseEntries(
|
|
1497
|
+
normalizedConfig.knowledgeBases,
|
|
1498
|
+
projectConfigBaseDir,
|
|
1499
|
+
projectRoot
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
return normalizedConfig;
|
|
1503
|
+
}
|
|
1504
|
+
function loadMergedConfig(projectRoot) {
|
|
1505
|
+
const globalConfigPath = os2.homedir() + "/.config/opencode/codebase-index.json";
|
|
1506
|
+
const globalConfig = loadJsonFile(globalConfigPath);
|
|
1507
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot);
|
|
1508
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
1509
|
+
const normalizedProjectConfig = loadProjectConfigLayer(projectRoot);
|
|
1510
|
+
if (!globalConfig && !projectConfig) {
|
|
1511
|
+
return {};
|
|
1512
|
+
}
|
|
1513
|
+
if (!projectConfig && globalConfig) {
|
|
1514
|
+
return globalConfig;
|
|
1515
|
+
}
|
|
1516
|
+
if (!globalConfig && projectConfig) {
|
|
1517
|
+
return normalizedProjectConfig;
|
|
1518
|
+
}
|
|
1519
|
+
const merged = { ...globalConfig };
|
|
1520
|
+
if (projectConfig && "embeddingProvider" in normalizedProjectConfig) {
|
|
1521
|
+
merged.embeddingProvider = normalizedProjectConfig.embeddingProvider;
|
|
1522
|
+
} else if (globalConfig && globalConfig.embeddingProvider) {
|
|
1523
|
+
merged.embeddingProvider = globalConfig.embeddingProvider;
|
|
1524
|
+
}
|
|
1525
|
+
if (projectConfig && "customProvider" in normalizedProjectConfig) {
|
|
1526
|
+
merged.customProvider = normalizedProjectConfig.customProvider;
|
|
1527
|
+
} else if (globalConfig && globalConfig.customProvider) {
|
|
1528
|
+
merged.customProvider = globalConfig.customProvider;
|
|
1529
|
+
}
|
|
1530
|
+
if (projectConfig && "embeddingModel" in normalizedProjectConfig) {
|
|
1531
|
+
merged.embeddingModel = normalizedProjectConfig.embeddingModel;
|
|
1532
|
+
} else if (globalConfig && globalConfig.embeddingModel) {
|
|
1533
|
+
merged.embeddingModel = globalConfig.embeddingModel;
|
|
1534
|
+
}
|
|
1535
|
+
if (projectConfig && "reranker" in normalizedProjectConfig) {
|
|
1536
|
+
merged.reranker = normalizedProjectConfig.reranker;
|
|
1537
|
+
} else if (globalConfig && globalConfig.reranker) {
|
|
1538
|
+
merged.reranker = globalConfig.reranker;
|
|
1539
|
+
}
|
|
1540
|
+
if (projectConfig && "include" in normalizedProjectConfig) {
|
|
1541
|
+
merged.include = normalizedProjectConfig.include;
|
|
1542
|
+
} else if (globalConfig && globalConfig.include) {
|
|
1543
|
+
merged.include = globalConfig.include;
|
|
1544
|
+
}
|
|
1545
|
+
if (projectConfig && "exclude" in normalizedProjectConfig) {
|
|
1546
|
+
merged.exclude = normalizedProjectConfig.exclude;
|
|
1547
|
+
} else if (globalConfig && globalConfig.exclude) {
|
|
1548
|
+
merged.exclude = globalConfig.exclude;
|
|
1549
|
+
}
|
|
1550
|
+
if (projectConfig && "indexing" in normalizedProjectConfig) {
|
|
1551
|
+
merged.indexing = normalizedProjectConfig.indexing;
|
|
1552
|
+
} else if (globalConfig && globalConfig.indexing) {
|
|
1553
|
+
merged.indexing = globalConfig.indexing;
|
|
1554
|
+
}
|
|
1555
|
+
if (projectConfig && "search" in normalizedProjectConfig) {
|
|
1556
|
+
merged.search = normalizedProjectConfig.search;
|
|
1557
|
+
} else if (globalConfig && globalConfig.search) {
|
|
1558
|
+
merged.search = globalConfig.search;
|
|
1559
|
+
}
|
|
1560
|
+
if (projectConfig && "debug" in normalizedProjectConfig) {
|
|
1561
|
+
merged.debug = normalizedProjectConfig.debug;
|
|
1562
|
+
} else if (globalConfig && globalConfig.debug) {
|
|
1563
|
+
merged.debug = globalConfig.debug;
|
|
1564
|
+
}
|
|
1565
|
+
if (projectConfig && "scope" in normalizedProjectConfig) {
|
|
1566
|
+
merged.scope = normalizedProjectConfig.scope;
|
|
1567
|
+
} else if (globalConfig && "scope" in globalConfig) {
|
|
1568
|
+
merged.scope = globalConfig.scope;
|
|
1569
|
+
}
|
|
1570
|
+
if (projectConfig) {
|
|
1571
|
+
for (const key of Object.keys(projectConfig)) {
|
|
1572
|
+
if (key === "embeddingProvider" || key === "customProvider" || key === "embeddingModel" || key === "reranker" || key === "include" || key === "exclude" || key === "indexing" || key === "search" || key === "debug" || key === "scope" || key === "knowledgeBases" || key === "additionalInclude") {
|
|
1573
|
+
continue;
|
|
1574
|
+
}
|
|
1575
|
+
merged[key] = normalizedProjectConfig[key];
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
1579
|
+
const projectKbs = projectConfig ? Array.isArray(normalizedProjectConfig.knowledgeBases) ? normalizedProjectConfig.knowledgeBases : [] : [];
|
|
1580
|
+
const allKbs = [...globalKbs, ...projectKbs];
|
|
1581
|
+
const uniqueKbs = [...new Set(allKbs.map((p) => String(p).trim()))];
|
|
1582
|
+
merged.knowledgeBases = uniqueKbs;
|
|
1583
|
+
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
1584
|
+
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
1585
|
+
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
1586
|
+
const uniqueAdditional = [...new Set(allAdditional.map((p) => String(p).trim()))];
|
|
1587
|
+
merged.additionalInclude = uniqueAdditional;
|
|
1588
|
+
return merged;
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1244
1591
|
// src/indexer/index.ts
|
|
1245
|
-
var
|
|
1246
|
-
var
|
|
1592
|
+
var import_fs7 = require("fs");
|
|
1593
|
+
var path8 = __toESM(require("path"), 1);
|
|
1247
1594
|
var import_perf_hooks = require("perf_hooks");
|
|
1248
1595
|
|
|
1249
1596
|
// node_modules/eventemitter3/index.mjs
|
|
@@ -1268,7 +1615,7 @@ function pTimeout(promise, options) {
|
|
|
1268
1615
|
} = options;
|
|
1269
1616
|
let timer;
|
|
1270
1617
|
let abortHandler;
|
|
1271
|
-
const wrappedPromise = new Promise((
|
|
1618
|
+
const wrappedPromise = new Promise((resolve8, reject) => {
|
|
1272
1619
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
1273
1620
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
1274
1621
|
}
|
|
@@ -1282,7 +1629,7 @@ function pTimeout(promise, options) {
|
|
|
1282
1629
|
};
|
|
1283
1630
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
1284
1631
|
}
|
|
1285
|
-
promise.then(
|
|
1632
|
+
promise.then(resolve8, reject);
|
|
1286
1633
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
1287
1634
|
return;
|
|
1288
1635
|
}
|
|
@@ -1290,7 +1637,7 @@ function pTimeout(promise, options) {
|
|
|
1290
1637
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
1291
1638
|
if (fallback) {
|
|
1292
1639
|
try {
|
|
1293
|
-
|
|
1640
|
+
resolve8(fallback());
|
|
1294
1641
|
} catch (error) {
|
|
1295
1642
|
reject(error);
|
|
1296
1643
|
}
|
|
@@ -1300,7 +1647,7 @@ function pTimeout(promise, options) {
|
|
|
1300
1647
|
promise.cancel();
|
|
1301
1648
|
}
|
|
1302
1649
|
if (message === false) {
|
|
1303
|
-
|
|
1650
|
+
resolve8();
|
|
1304
1651
|
} else if (message instanceof Error) {
|
|
1305
1652
|
reject(message);
|
|
1306
1653
|
} else {
|
|
@@ -1702,7 +2049,7 @@ var PQueue = class extends import_index.default {
|
|
|
1702
2049
|
// Assign unique ID if not provided
|
|
1703
2050
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1704
2051
|
};
|
|
1705
|
-
return new Promise((
|
|
2052
|
+
return new Promise((resolve8, reject) => {
|
|
1706
2053
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1707
2054
|
let cleanupQueueAbortHandler = () => void 0;
|
|
1708
2055
|
const run = async () => {
|
|
@@ -1742,7 +2089,7 @@ var PQueue = class extends import_index.default {
|
|
|
1742
2089
|
})]);
|
|
1743
2090
|
}
|
|
1744
2091
|
const result = await operation;
|
|
1745
|
-
|
|
2092
|
+
resolve8(result);
|
|
1746
2093
|
this.emit("completed", result);
|
|
1747
2094
|
} catch (error) {
|
|
1748
2095
|
reject(error);
|
|
@@ -1930,13 +2277,13 @@ var PQueue = class extends import_index.default {
|
|
|
1930
2277
|
});
|
|
1931
2278
|
}
|
|
1932
2279
|
async #onEvent(event, filter) {
|
|
1933
|
-
return new Promise((
|
|
2280
|
+
return new Promise((resolve8) => {
|
|
1934
2281
|
const listener = () => {
|
|
1935
2282
|
if (filter && !filter()) {
|
|
1936
2283
|
return;
|
|
1937
2284
|
}
|
|
1938
2285
|
this.off(event, listener);
|
|
1939
|
-
|
|
2286
|
+
resolve8();
|
|
1940
2287
|
};
|
|
1941
2288
|
this.on(event, listener);
|
|
1942
2289
|
});
|
|
@@ -2222,7 +2569,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
2222
2569
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
2223
2570
|
options.signal?.throwIfAborted();
|
|
2224
2571
|
if (finalDelay > 0) {
|
|
2225
|
-
await new Promise((
|
|
2572
|
+
await new Promise((resolve8, reject) => {
|
|
2226
2573
|
const onAbort = () => {
|
|
2227
2574
|
clearTimeout(timeoutToken);
|
|
2228
2575
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -2230,7 +2577,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
2230
2577
|
};
|
|
2231
2578
|
const timeoutToken = setTimeout(() => {
|
|
2232
2579
|
options.signal?.removeEventListener("abort", onAbort);
|
|
2233
|
-
|
|
2580
|
+
resolve8();
|
|
2234
2581
|
}, finalDelay);
|
|
2235
2582
|
if (options.unref) {
|
|
2236
2583
|
timeoutToken.unref?.();
|
|
@@ -2291,17 +2638,17 @@ async function pRetry(input, options = {}) {
|
|
|
2291
2638
|
}
|
|
2292
2639
|
|
|
2293
2640
|
// src/embeddings/detector.ts
|
|
2294
|
-
var
|
|
2295
|
-
var
|
|
2296
|
-
var
|
|
2641
|
+
var import_fs5 = require("fs");
|
|
2642
|
+
var path5 = __toESM(require("path"), 1);
|
|
2643
|
+
var os3 = __toESM(require("os"), 1);
|
|
2297
2644
|
function getOpenCodeAuthPath() {
|
|
2298
|
-
return
|
|
2645
|
+
return path5.join(os3.homedir(), ".local", "share", "opencode", "auth.json");
|
|
2299
2646
|
}
|
|
2300
2647
|
function loadOpenCodeAuth() {
|
|
2301
2648
|
const authPath = getOpenCodeAuthPath();
|
|
2302
2649
|
try {
|
|
2303
|
-
if ((0,
|
|
2304
|
-
return JSON.parse((0,
|
|
2650
|
+
if ((0, import_fs5.existsSync)(authPath)) {
|
|
2651
|
+
return JSON.parse((0, import_fs5.readFileSync)(authPath, "utf-8"));
|
|
2305
2652
|
}
|
|
2306
2653
|
} catch {
|
|
2307
2654
|
}
|
|
@@ -2334,7 +2681,7 @@ async function detectEmbeddingProvider(preferredProvider, model) {
|
|
|
2334
2681
|
);
|
|
2335
2682
|
}
|
|
2336
2683
|
async function tryDetectProvider() {
|
|
2337
|
-
for (const provider of
|
|
2684
|
+
for (const provider of autoDetectProviders) {
|
|
2338
2685
|
const credentials = await getProviderCredentials(provider);
|
|
2339
2686
|
if (credentials) {
|
|
2340
2687
|
return {
|
|
@@ -2345,7 +2692,7 @@ async function tryDetectProvider() {
|
|
|
2345
2692
|
}
|
|
2346
2693
|
}
|
|
2347
2694
|
throw new Error(
|
|
2348
|
-
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${
|
|
2695
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${autoDetectProviders.join(", ")}.`
|
|
2349
2696
|
);
|
|
2350
2697
|
}
|
|
2351
2698
|
async function getProviderCredentials(provider) {
|
|
@@ -2665,11 +3012,12 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
|
2665
3012
|
return this.modelInfo;
|
|
2666
3013
|
}
|
|
2667
3014
|
};
|
|
2668
|
-
var OllamaEmbeddingProvider = class {
|
|
3015
|
+
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider {
|
|
2669
3016
|
constructor(credentials, modelInfo) {
|
|
2670
3017
|
this.credentials = credentials;
|
|
2671
3018
|
this.modelInfo = modelInfo;
|
|
2672
3019
|
}
|
|
3020
|
+
static MIN_TRUNCATION_CHARS = 512;
|
|
2673
3021
|
async embedQuery(query) {
|
|
2674
3022
|
const result = await this.embedBatch([query]);
|
|
2675
3023
|
return {
|
|
@@ -2684,43 +3032,116 @@ var OllamaEmbeddingProvider = class {
|
|
|
2684
3032
|
tokensUsed: result.totalTokensUsed
|
|
2685
3033
|
};
|
|
2686
3034
|
}
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
texts.map(async (text) => {
|
|
2690
|
-
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2691
|
-
method: "POST",
|
|
2692
|
-
headers: {
|
|
2693
|
-
"Content-Type": "application/json"
|
|
2694
|
-
},
|
|
2695
|
-
body: JSON.stringify({
|
|
2696
|
-
model: this.modelInfo.model,
|
|
2697
|
-
prompt: text
|
|
2698
|
-
})
|
|
2699
|
-
});
|
|
2700
|
-
if (!response.ok) {
|
|
2701
|
-
const error = await response.text();
|
|
2702
|
-
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2703
|
-
}
|
|
2704
|
-
const data = await response.json();
|
|
2705
|
-
return {
|
|
2706
|
-
embedding: data.embedding,
|
|
2707
|
-
tokensUsed: Math.ceil(text.length / 4)
|
|
2708
|
-
};
|
|
2709
|
-
})
|
|
2710
|
-
);
|
|
2711
|
-
return {
|
|
2712
|
-
embeddings: results.map((r) => r.embedding),
|
|
2713
|
-
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2714
|
-
};
|
|
3035
|
+
estimateTokens(text) {
|
|
3036
|
+
return Math.ceil(text.length / 4);
|
|
2715
3037
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
3038
|
+
truncateToCharLimit(text, maxChars) {
|
|
3039
|
+
if (text.length <= maxChars) {
|
|
3040
|
+
return text;
|
|
3041
|
+
}
|
|
3042
|
+
return `${text.slice(0, Math.max(0, maxChars - 17))}
|
|
3043
|
+
... [truncated]`;
|
|
2718
3044
|
}
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
3045
|
+
isContextLengthError(error) {
|
|
3046
|
+
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
3047
|
+
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
3048
|
+
}
|
|
3049
|
+
buildTruncationCandidates(text) {
|
|
3050
|
+
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
3051
|
+
const candidateLimits = /* @__PURE__ */ new Set();
|
|
3052
|
+
const baselineLimit = text.length > baseMaxChars ? baseMaxChars : Math.max(
|
|
3053
|
+
_OllamaEmbeddingProvider.MIN_TRUNCATION_CHARS,
|
|
3054
|
+
Math.floor(text.length * 0.9)
|
|
3055
|
+
);
|
|
3056
|
+
if (baselineLimit < text.length) {
|
|
3057
|
+
candidateLimits.add(baselineLimit);
|
|
3058
|
+
}
|
|
3059
|
+
for (const factor of [0.75, 0.6, 0.45, 0.35, 0.25]) {
|
|
3060
|
+
const scaledLimit = Math.max(
|
|
3061
|
+
_OllamaEmbeddingProvider.MIN_TRUNCATION_CHARS,
|
|
3062
|
+
Math.floor(baselineLimit * factor)
|
|
3063
|
+
);
|
|
3064
|
+
if (scaledLimit < text.length) {
|
|
3065
|
+
candidateLimits.add(scaledLimit);
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
candidateLimits.add(Math.min(text.length - 1, _OllamaEmbeddingProvider.MIN_TRUNCATION_CHARS));
|
|
3069
|
+
const candidates = [];
|
|
3070
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3071
|
+
for (const limit of [...candidateLimits].sort((a, b) => b - a)) {
|
|
3072
|
+
if (limit <= 0 || limit >= text.length) {
|
|
3073
|
+
continue;
|
|
3074
|
+
}
|
|
3075
|
+
const truncated = this.truncateToCharLimit(text, limit);
|
|
3076
|
+
if (truncated === text || seen.has(truncated)) {
|
|
3077
|
+
continue;
|
|
3078
|
+
}
|
|
3079
|
+
seen.add(truncated);
|
|
3080
|
+
candidates.push(truncated);
|
|
3081
|
+
}
|
|
3082
|
+
return candidates;
|
|
3083
|
+
}
|
|
3084
|
+
async embedSingleWithFallback(text) {
|
|
3085
|
+
try {
|
|
3086
|
+
return await this.embedSingle(text);
|
|
3087
|
+
} catch (error) {
|
|
3088
|
+
if (!this.isContextLengthError(error)) {
|
|
3089
|
+
throw error;
|
|
3090
|
+
}
|
|
3091
|
+
let lastError = error;
|
|
3092
|
+
for (const truncated of this.buildTruncationCandidates(text)) {
|
|
3093
|
+
try {
|
|
3094
|
+
return await this.embedSingle(truncated);
|
|
3095
|
+
} catch (retryError) {
|
|
3096
|
+
if (!this.isContextLengthError(retryError)) {
|
|
3097
|
+
throw retryError;
|
|
3098
|
+
}
|
|
3099
|
+
lastError = retryError;
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
throw lastError;
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
async embedSingle(text) {
|
|
3106
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
3107
|
+
method: "POST",
|
|
3108
|
+
headers: {
|
|
3109
|
+
"Content-Type": "application/json"
|
|
3110
|
+
},
|
|
3111
|
+
body: JSON.stringify({
|
|
3112
|
+
model: this.modelInfo.model,
|
|
3113
|
+
prompt: text,
|
|
3114
|
+
truncate: false
|
|
3115
|
+
})
|
|
3116
|
+
});
|
|
3117
|
+
if (!response.ok) {
|
|
3118
|
+
const error = await response.text();
|
|
3119
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
3120
|
+
}
|
|
3121
|
+
const data = await response.json();
|
|
3122
|
+
return {
|
|
3123
|
+
embedding: data.embedding,
|
|
3124
|
+
tokensUsed: this.estimateTokens(text)
|
|
3125
|
+
};
|
|
3126
|
+
}
|
|
3127
|
+
async embedBatch(texts) {
|
|
3128
|
+
const results = [];
|
|
3129
|
+
for (const text of texts) {
|
|
3130
|
+
results.push(await this.embedSingleWithFallback(text));
|
|
3131
|
+
}
|
|
3132
|
+
return {
|
|
3133
|
+
embeddings: results.map((r) => r.embedding),
|
|
3134
|
+
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
3135
|
+
};
|
|
3136
|
+
}
|
|
3137
|
+
getModelInfo() {
|
|
3138
|
+
return this.modelInfo;
|
|
3139
|
+
}
|
|
3140
|
+
};
|
|
3141
|
+
var CustomEmbeddingProvider = class {
|
|
3142
|
+
constructor(credentials, modelInfo) {
|
|
3143
|
+
this.credentials = credentials;
|
|
3144
|
+
this.modelInfo = modelInfo;
|
|
2724
3145
|
}
|
|
2725
3146
|
splitIntoRequestBatches(texts) {
|
|
2726
3147
|
const maxBatchSize = this.modelInfo.maxBatchSize;
|
|
@@ -2911,8 +3332,8 @@ var SiliconFlowReranker = class {
|
|
|
2911
3332
|
|
|
2912
3333
|
// src/utils/files.ts
|
|
2913
3334
|
var import_ignore = __toESM(require_ignore(), 1);
|
|
2914
|
-
var
|
|
2915
|
-
var
|
|
3335
|
+
var import_fs6 = require("fs");
|
|
3336
|
+
var path6 = __toESM(require("path"), 1);
|
|
2916
3337
|
function createIgnoreFilter(projectRoot) {
|
|
2917
3338
|
const ig = (0, import_ignore.default)();
|
|
2918
3339
|
const defaultIgnores = [
|
|
@@ -2933,9 +3354,9 @@ function createIgnoreFilter(projectRoot) {
|
|
|
2933
3354
|
"**/*build*/**"
|
|
2934
3355
|
];
|
|
2935
3356
|
ig.add(defaultIgnores);
|
|
2936
|
-
const gitignorePath =
|
|
2937
|
-
if ((0,
|
|
2938
|
-
const gitignoreContent = (0,
|
|
3357
|
+
const gitignorePath = path6.join(projectRoot, ".gitignore");
|
|
3358
|
+
if ((0, import_fs6.existsSync)(gitignorePath)) {
|
|
3359
|
+
const gitignoreContent = (0, import_fs6.readFileSync)(gitignorePath, "utf-8");
|
|
2939
3360
|
ig.add(gitignoreContent);
|
|
2940
3361
|
}
|
|
2941
3362
|
return ig;
|
|
@@ -2956,12 +3377,12 @@ function matchGlob(filePath, pattern) {
|
|
|
2956
3377
|
return regex.test(filePath);
|
|
2957
3378
|
}
|
|
2958
3379
|
async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns, ignoreFilter, maxFileSize, skipped, options, currentDepth = 0) {
|
|
2959
|
-
const entries = await
|
|
3380
|
+
const entries = await import_fs6.promises.readdir(dir, { withFileTypes: true });
|
|
2960
3381
|
const filesInDir = [];
|
|
2961
3382
|
const subdirs = [];
|
|
2962
3383
|
for (const entry of entries) {
|
|
2963
|
-
const fullPath =
|
|
2964
|
-
const relativePath =
|
|
3384
|
+
const fullPath = path6.join(dir, entry.name);
|
|
3385
|
+
const relativePath = path6.relative(projectRoot, fullPath);
|
|
2965
3386
|
if (entry.name.startsWith(".") && entry.name !== "." && entry.name !== "..") {
|
|
2966
3387
|
if (entry.isDirectory()) {
|
|
2967
3388
|
skipped.push({ path: relativePath, reason: "excluded" });
|
|
@@ -2981,7 +3402,7 @@ async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns
|
|
|
2981
3402
|
if (entry.isDirectory()) {
|
|
2982
3403
|
subdirs.push({ fullPath, relativePath });
|
|
2983
3404
|
} else if (entry.isFile()) {
|
|
2984
|
-
const stat = await
|
|
3405
|
+
const stat = await import_fs6.promises.stat(fullPath);
|
|
2985
3406
|
if (stat.size > maxFileSize) {
|
|
2986
3407
|
skipped.push({ path: relativePath, reason: "too_large" });
|
|
2987
3408
|
continue;
|
|
@@ -3010,7 +3431,7 @@ async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns
|
|
|
3010
3431
|
yield f;
|
|
3011
3432
|
}
|
|
3012
3433
|
for (let i = options.maxFilesPerDirectory; i < filesInDir.length; i++) {
|
|
3013
|
-
skipped.push({ path:
|
|
3434
|
+
skipped.push({ path: path6.relative(projectRoot, filesInDir[i].path), reason: "excluded" });
|
|
3014
3435
|
}
|
|
3015
3436
|
const canRecurse = options.maxDepth === -1 || currentDepth < options.maxDepth;
|
|
3016
3437
|
if (canRecurse) {
|
|
@@ -3050,14 +3471,14 @@ async function collectFiles(projectRoot, includePatterns, excludePatterns, maxFi
|
|
|
3050
3471
|
if (additionalRoots && additionalRoots.length > 0) {
|
|
3051
3472
|
const normalizedRoots = /* @__PURE__ */ new Set();
|
|
3052
3473
|
for (const kbRoot of additionalRoots) {
|
|
3053
|
-
const resolved =
|
|
3054
|
-
|
|
3474
|
+
const resolved = path6.normalize(
|
|
3475
|
+
path6.isAbsolute(kbRoot) ? kbRoot : path6.resolve(projectRoot, kbRoot)
|
|
3055
3476
|
);
|
|
3056
3477
|
normalizedRoots.add(resolved);
|
|
3057
3478
|
}
|
|
3058
3479
|
for (const resolvedKbRoot of normalizedRoots) {
|
|
3059
3480
|
try {
|
|
3060
|
-
const stat = await
|
|
3481
|
+
const stat = await import_fs6.promises.stat(resolvedKbRoot);
|
|
3061
3482
|
if (!stat.isDirectory()) {
|
|
3062
3483
|
skipped.push({ path: resolvedKbRoot, reason: "excluded" });
|
|
3063
3484
|
continue;
|
|
@@ -3441,14 +3862,14 @@ function initializeLogger(config) {
|
|
|
3441
3862
|
}
|
|
3442
3863
|
|
|
3443
3864
|
// src/native/index.ts
|
|
3444
|
-
var
|
|
3445
|
-
var
|
|
3865
|
+
var path7 = __toESM(require("path"), 1);
|
|
3866
|
+
var os4 = __toESM(require("os"), 1);
|
|
3446
3867
|
var module2 = __toESM(require("module"), 1);
|
|
3447
3868
|
var import_url = require("url");
|
|
3448
3869
|
var import_meta = {};
|
|
3449
3870
|
function getNativeBinding() {
|
|
3450
|
-
const platform2 =
|
|
3451
|
-
const arch2 =
|
|
3871
|
+
const platform2 = os4.platform();
|
|
3872
|
+
const arch2 = os4.arch();
|
|
3452
3873
|
let bindingName;
|
|
3453
3874
|
if (platform2 === "darwin" && arch2 === "arm64") {
|
|
3454
3875
|
bindingName = "codebase-index-native.darwin-arm64.node";
|
|
@@ -3466,18 +3887,19 @@ function getNativeBinding() {
|
|
|
3466
3887
|
let currentDir;
|
|
3467
3888
|
let requireTarget;
|
|
3468
3889
|
if (typeof import_meta !== "undefined" && import_meta.url) {
|
|
3469
|
-
currentDir =
|
|
3890
|
+
currentDir = path7.dirname((0, import_url.fileURLToPath)(import_meta.url));
|
|
3470
3891
|
requireTarget = import_meta.url;
|
|
3471
3892
|
} else if (typeof __dirname !== "undefined") {
|
|
3472
3893
|
currentDir = __dirname;
|
|
3473
3894
|
requireTarget = __filename;
|
|
3474
3895
|
} else {
|
|
3475
3896
|
currentDir = process.cwd();
|
|
3476
|
-
requireTarget =
|
|
3897
|
+
requireTarget = path7.join(currentDir, "index.js");
|
|
3477
3898
|
}
|
|
3478
|
-
const
|
|
3479
|
-
const
|
|
3480
|
-
const
|
|
3899
|
+
const normalizedDir = currentDir.replace(/\\/g, "/");
|
|
3900
|
+
const isDevMode = normalizedDir.includes("/src/native") || currentDir.includes(path7.join("src", "native"));
|
|
3901
|
+
const packageRoot = isDevMode ? path7.resolve(currentDir, "../..") : path7.resolve(currentDir, "..");
|
|
3902
|
+
const nativePath = path7.join(packageRoot, "native", bindingName);
|
|
3481
3903
|
const require2 = module2.createRequire(requireTarget);
|
|
3482
3904
|
return require2(nativePath);
|
|
3483
3905
|
}
|
|
@@ -3508,11 +3930,20 @@ function createMockNativeBinding() {
|
|
|
3508
3930
|
constructor() {
|
|
3509
3931
|
throw error;
|
|
3510
3932
|
}
|
|
3933
|
+
serialize() {
|
|
3934
|
+
throw error;
|
|
3935
|
+
}
|
|
3936
|
+
deserialize() {
|
|
3937
|
+
throw error;
|
|
3938
|
+
}
|
|
3511
3939
|
},
|
|
3512
3940
|
Database: class {
|
|
3513
3941
|
constructor() {
|
|
3514
3942
|
throw error;
|
|
3515
3943
|
}
|
|
3944
|
+
close() {
|
|
3945
|
+
throw error;
|
|
3946
|
+
}
|
|
3516
3947
|
}
|
|
3517
3948
|
};
|
|
3518
3949
|
}
|
|
@@ -3645,7 +4076,7 @@ var MAX_SINGLE_CHUNK_TOKENS = 2e3;
|
|
|
3645
4076
|
function estimateTokens2(text) {
|
|
3646
4077
|
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
3647
4078
|
}
|
|
3648
|
-
function
|
|
4079
|
+
function getEmbeddingHeaderParts(chunk, filePath) {
|
|
3649
4080
|
const parts = [];
|
|
3650
4081
|
const fileName = filePath.split("/").pop() || filePath;
|
|
3651
4082
|
const dirPath = filePath.split("/").slice(-3, -1).join("/");
|
|
@@ -3692,23 +4123,52 @@ function createEmbeddingText(chunk, filePath) {
|
|
|
3692
4123
|
if (semanticHints.length > 0) {
|
|
3693
4124
|
parts.push(`Purpose: ${semanticHints.join(", ")}`);
|
|
3694
4125
|
}
|
|
3695
|
-
parts
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
const
|
|
3699
|
-
if (
|
|
3700
|
-
|
|
4126
|
+
return parts;
|
|
4127
|
+
}
|
|
4128
|
+
function buildEmbeddingText(headerParts, content, partIndex, partCount) {
|
|
4129
|
+
const parts = [...headerParts];
|
|
4130
|
+
if (partCount && partCount > 1 && partIndex) {
|
|
4131
|
+
parts.push(`Part ${partIndex}/${partCount}`);
|
|
3701
4132
|
}
|
|
4133
|
+
parts.push("");
|
|
3702
4134
|
parts.push(content);
|
|
3703
4135
|
return parts.join("\n");
|
|
3704
4136
|
}
|
|
3705
|
-
function
|
|
4137
|
+
function splitOversizedContent(content, maxContentChars) {
|
|
4138
|
+
if (content.length <= maxContentChars) {
|
|
4139
|
+
return [content];
|
|
4140
|
+
}
|
|
4141
|
+
const overlapChars = Math.max(CHARS_PER_TOKEN * 32, Math.min(Math.floor(maxContentChars * 0.15), CHARS_PER_TOKEN * 128));
|
|
4142
|
+
const stepChars = Math.max(1, maxContentChars - overlapChars);
|
|
4143
|
+
const segments = [];
|
|
4144
|
+
for (let start = 0; start < content.length; start += stepChars) {
|
|
4145
|
+
const end = Math.min(content.length, start + maxContentChars);
|
|
4146
|
+
segments.push(content.slice(start, end));
|
|
4147
|
+
if (end >= content.length) {
|
|
4148
|
+
break;
|
|
4149
|
+
}
|
|
4150
|
+
}
|
|
4151
|
+
return segments;
|
|
4152
|
+
}
|
|
4153
|
+
function createEmbeddingTexts(chunk, filePath, maxChunkTokens = MAX_SINGLE_CHUNK_TOKENS) {
|
|
4154
|
+
const headerParts = getEmbeddingHeaderParts(chunk, filePath);
|
|
4155
|
+
const headerLength = buildEmbeddingText(headerParts, "", 1, 9).length;
|
|
4156
|
+
const maxContentChars = Math.max(1, maxChunkTokens * CHARS_PER_TOKEN - headerLength);
|
|
4157
|
+
const segments = splitOversizedContent(chunk.content, maxContentChars);
|
|
4158
|
+
if (segments.length === 1) {
|
|
4159
|
+
return [buildEmbeddingText(headerParts, segments[0])];
|
|
4160
|
+
}
|
|
4161
|
+
return segments.map((segment, index) => buildEmbeddingText(headerParts, segment, index + 1, segments.length));
|
|
4162
|
+
}
|
|
4163
|
+
function createDynamicBatches(chunks, options = {}) {
|
|
3706
4164
|
const batches = [];
|
|
3707
4165
|
let currentBatch = [];
|
|
3708
4166
|
let currentTokens = 0;
|
|
4167
|
+
const maxBatchTokens = Math.max(1, options.maxBatchTokens ?? MAX_BATCH_TOKENS);
|
|
4168
|
+
const maxBatchItems = Math.max(1, options.maxBatchItems ?? Number.MAX_SAFE_INTEGER);
|
|
3709
4169
|
for (const chunk of chunks) {
|
|
3710
|
-
const chunkTokens = estimateTokens2(chunk.text);
|
|
3711
|
-
if (currentBatch.length > 0 && currentTokens + chunkTokens >
|
|
4170
|
+
const chunkTokens = chunk.tokenCount ?? estimateTokens2(chunk.text);
|
|
4171
|
+
if (currentBatch.length > 0 && (currentTokens + chunkTokens > maxBatchTokens || currentBatch.length >= maxBatchItems)) {
|
|
3712
4172
|
batches.push(currentBatch);
|
|
3713
4173
|
currentBatch = [];
|
|
3714
4174
|
currentTokens = 0;
|
|
@@ -3827,6 +4287,12 @@ var InvertedIndex = class {
|
|
|
3827
4287
|
save() {
|
|
3828
4288
|
this.inner.save();
|
|
3829
4289
|
}
|
|
4290
|
+
serialize() {
|
|
4291
|
+
return this.inner.serialize();
|
|
4292
|
+
}
|
|
4293
|
+
deserialize(json) {
|
|
4294
|
+
this.inner.deserialize(json);
|
|
4295
|
+
}
|
|
3830
4296
|
addChunk(chunkId, content) {
|
|
3831
4297
|
this.inner.addChunk(chunkId, content);
|
|
3832
4298
|
}
|
|
@@ -3853,267 +4319,263 @@ var InvertedIndex = class {
|
|
|
3853
4319
|
};
|
|
3854
4320
|
var Database = class {
|
|
3855
4321
|
inner;
|
|
4322
|
+
closed = false;
|
|
3856
4323
|
constructor(dbPath) {
|
|
3857
4324
|
this.inner = new native.Database(dbPath);
|
|
3858
4325
|
}
|
|
4326
|
+
throwIfClosed() {
|
|
4327
|
+
if (this.closed) {
|
|
4328
|
+
throw new Error("Database is closed");
|
|
4329
|
+
}
|
|
4330
|
+
}
|
|
4331
|
+
close() {
|
|
4332
|
+
if (this.closed) {
|
|
4333
|
+
return;
|
|
4334
|
+
}
|
|
4335
|
+
if (typeof this.inner.close === "function") {
|
|
4336
|
+
this.inner.close();
|
|
4337
|
+
}
|
|
4338
|
+
this.closed = true;
|
|
4339
|
+
}
|
|
3859
4340
|
embeddingExists(contentHash) {
|
|
4341
|
+
this.throwIfClosed();
|
|
3860
4342
|
return this.inner.embeddingExists(contentHash);
|
|
3861
4343
|
}
|
|
3862
4344
|
getEmbedding(contentHash) {
|
|
4345
|
+
this.throwIfClosed();
|
|
3863
4346
|
return this.inner.getEmbedding(contentHash) ?? null;
|
|
3864
4347
|
}
|
|
3865
4348
|
upsertEmbedding(contentHash, embedding, chunkText, model) {
|
|
4349
|
+
this.throwIfClosed();
|
|
3866
4350
|
this.inner.upsertEmbedding(contentHash, embedding, chunkText, model);
|
|
3867
4351
|
}
|
|
3868
4352
|
upsertEmbeddingsBatch(items) {
|
|
4353
|
+
this.throwIfClosed();
|
|
3869
4354
|
if (items.length === 0) return;
|
|
3870
4355
|
this.inner.upsertEmbeddingsBatch(items);
|
|
3871
4356
|
}
|
|
3872
4357
|
getMissingEmbeddings(contentHashes) {
|
|
4358
|
+
this.throwIfClosed();
|
|
3873
4359
|
return this.inner.getMissingEmbeddings(contentHashes);
|
|
3874
4360
|
}
|
|
3875
4361
|
upsertChunk(chunk) {
|
|
4362
|
+
this.throwIfClosed();
|
|
3876
4363
|
this.inner.upsertChunk(chunk);
|
|
3877
4364
|
}
|
|
3878
4365
|
upsertChunksBatch(chunks) {
|
|
4366
|
+
this.throwIfClosed();
|
|
3879
4367
|
if (chunks.length === 0) return;
|
|
3880
4368
|
this.inner.upsertChunksBatch(chunks);
|
|
3881
4369
|
}
|
|
3882
4370
|
getChunk(chunkId) {
|
|
4371
|
+
this.throwIfClosed();
|
|
3883
4372
|
return this.inner.getChunk(chunkId) ?? null;
|
|
3884
4373
|
}
|
|
3885
4374
|
getChunksByFile(filePath) {
|
|
4375
|
+
this.throwIfClosed();
|
|
3886
4376
|
return this.inner.getChunksByFile(filePath);
|
|
3887
4377
|
}
|
|
3888
4378
|
getChunksByName(name) {
|
|
4379
|
+
this.throwIfClosed();
|
|
3889
4380
|
return this.inner.getChunksByName(name);
|
|
3890
4381
|
}
|
|
3891
4382
|
getChunksByNameCi(name) {
|
|
4383
|
+
this.throwIfClosed();
|
|
3892
4384
|
return this.inner.getChunksByNameCi(name);
|
|
3893
4385
|
}
|
|
3894
4386
|
deleteChunksByFile(filePath) {
|
|
4387
|
+
this.throwIfClosed();
|
|
3895
4388
|
return this.inner.deleteChunksByFile(filePath);
|
|
3896
4389
|
}
|
|
4390
|
+
deleteChunksByIds(chunkIds) {
|
|
4391
|
+
this.throwIfClosed();
|
|
4392
|
+
if (chunkIds.length === 0) return 0;
|
|
4393
|
+
return this.inner.deleteChunksByIds(chunkIds);
|
|
4394
|
+
}
|
|
3897
4395
|
addChunksToBranch(branch, chunkIds) {
|
|
4396
|
+
this.throwIfClosed();
|
|
3898
4397
|
this.inner.addChunksToBranch(branch, chunkIds);
|
|
3899
4398
|
}
|
|
3900
4399
|
addChunksToBranchBatch(branch, chunkIds) {
|
|
4400
|
+
this.throwIfClosed();
|
|
3901
4401
|
if (chunkIds.length === 0) return;
|
|
3902
4402
|
this.inner.addChunksToBranchBatch(branch, chunkIds);
|
|
3903
4403
|
}
|
|
3904
4404
|
clearBranch(branch) {
|
|
4405
|
+
this.throwIfClosed();
|
|
3905
4406
|
return this.inner.clearBranch(branch);
|
|
3906
4407
|
}
|
|
4408
|
+
deleteBranchChunksByChunkIds(chunkIds) {
|
|
4409
|
+
this.throwIfClosed();
|
|
4410
|
+
if (chunkIds.length === 0) return 0;
|
|
4411
|
+
return this.inner.deleteBranchChunksByChunkIds(chunkIds);
|
|
4412
|
+
}
|
|
4413
|
+
deleteBranchChunksForBranch(branch, chunkIds) {
|
|
4414
|
+
this.throwIfClosed();
|
|
4415
|
+
if (chunkIds.length === 0) return 0;
|
|
4416
|
+
return this.inner.deleteBranchChunksForBranch(branch, chunkIds);
|
|
4417
|
+
}
|
|
3907
4418
|
getBranchChunkIds(branch) {
|
|
4419
|
+
this.throwIfClosed();
|
|
3908
4420
|
return this.inner.getBranchChunkIds(branch);
|
|
3909
4421
|
}
|
|
3910
4422
|
getBranchDelta(branch, baseBranch) {
|
|
4423
|
+
this.throwIfClosed();
|
|
3911
4424
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
3912
4425
|
}
|
|
4426
|
+
getReferencedChunkIds(chunkIds) {
|
|
4427
|
+
this.throwIfClosed();
|
|
4428
|
+
if (chunkIds.length === 0) return [];
|
|
4429
|
+
return this.inner.getReferencedChunkIds(chunkIds);
|
|
4430
|
+
}
|
|
3913
4431
|
chunkExistsOnBranch(branch, chunkId) {
|
|
4432
|
+
this.throwIfClosed();
|
|
3914
4433
|
return this.inner.chunkExistsOnBranch(branch, chunkId);
|
|
3915
4434
|
}
|
|
3916
4435
|
getAllBranches() {
|
|
4436
|
+
this.throwIfClosed();
|
|
3917
4437
|
return this.inner.getAllBranches();
|
|
3918
4438
|
}
|
|
3919
4439
|
getMetadata(key) {
|
|
4440
|
+
this.throwIfClosed();
|
|
3920
4441
|
return this.inner.getMetadata(key) ?? null;
|
|
3921
4442
|
}
|
|
3922
4443
|
setMetadata(key, value) {
|
|
4444
|
+
this.throwIfClosed();
|
|
3923
4445
|
this.inner.setMetadata(key, value);
|
|
3924
4446
|
}
|
|
3925
4447
|
deleteMetadata(key) {
|
|
4448
|
+
this.throwIfClosed();
|
|
3926
4449
|
return this.inner.deleteMetadata(key);
|
|
3927
4450
|
}
|
|
4451
|
+
clearAllIndexedData() {
|
|
4452
|
+
this.throwIfClosed();
|
|
4453
|
+
this.inner.clearAllIndexedData();
|
|
4454
|
+
}
|
|
4455
|
+
clearCallEdgeTargetsForSymbols(symbolIds) {
|
|
4456
|
+
this.throwIfClosed();
|
|
4457
|
+
if (symbolIds.length === 0) return 0;
|
|
4458
|
+
return this.inner.clearCallEdgeTargetsForSymbols(symbolIds);
|
|
4459
|
+
}
|
|
3928
4460
|
gcOrphanEmbeddings() {
|
|
4461
|
+
this.throwIfClosed();
|
|
3929
4462
|
return this.inner.gcOrphanEmbeddings();
|
|
3930
4463
|
}
|
|
3931
4464
|
gcOrphanChunks() {
|
|
4465
|
+
this.throwIfClosed();
|
|
3932
4466
|
return this.inner.gcOrphanChunks();
|
|
3933
4467
|
}
|
|
3934
4468
|
getStats() {
|
|
4469
|
+
this.throwIfClosed();
|
|
3935
4470
|
return this.inner.getStats();
|
|
3936
4471
|
}
|
|
3937
4472
|
// ── Symbol methods ──────────────────────────────────────────────
|
|
3938
4473
|
upsertSymbol(symbol) {
|
|
4474
|
+
this.throwIfClosed();
|
|
3939
4475
|
this.inner.upsertSymbol(symbol);
|
|
3940
4476
|
}
|
|
3941
4477
|
upsertSymbolsBatch(symbols) {
|
|
4478
|
+
this.throwIfClosed();
|
|
3942
4479
|
if (symbols.length === 0) return;
|
|
3943
4480
|
this.inner.upsertSymbolsBatch(symbols);
|
|
3944
4481
|
}
|
|
3945
4482
|
getSymbolsByFile(filePath) {
|
|
4483
|
+
this.throwIfClosed();
|
|
3946
4484
|
return this.inner.getSymbolsByFile(filePath);
|
|
3947
4485
|
}
|
|
3948
4486
|
getSymbolByName(name, filePath) {
|
|
4487
|
+
this.throwIfClosed();
|
|
3949
4488
|
return this.inner.getSymbolByName(name, filePath) ?? null;
|
|
3950
4489
|
}
|
|
3951
4490
|
getSymbolsByName(name) {
|
|
4491
|
+
this.throwIfClosed();
|
|
3952
4492
|
return this.inner.getSymbolsByName(name);
|
|
3953
4493
|
}
|
|
3954
4494
|
getSymbolsByNameCi(name) {
|
|
4495
|
+
this.throwIfClosed();
|
|
3955
4496
|
return this.inner.getSymbolsByNameCi(name);
|
|
3956
4497
|
}
|
|
3957
4498
|
deleteSymbolsByFile(filePath) {
|
|
4499
|
+
this.throwIfClosed();
|
|
3958
4500
|
return this.inner.deleteSymbolsByFile(filePath);
|
|
3959
4501
|
}
|
|
3960
4502
|
// ── Call Edge methods ────────────────────────────────────────────
|
|
3961
4503
|
upsertCallEdge(edge) {
|
|
4504
|
+
this.throwIfClosed();
|
|
3962
4505
|
this.inner.upsertCallEdge(edge);
|
|
3963
4506
|
}
|
|
3964
4507
|
upsertCallEdgesBatch(edges) {
|
|
4508
|
+
this.throwIfClosed();
|
|
3965
4509
|
if (edges.length === 0) return;
|
|
3966
4510
|
this.inner.upsertCallEdgesBatch(edges);
|
|
3967
4511
|
}
|
|
3968
4512
|
getCallers(targetName, branch) {
|
|
4513
|
+
this.throwIfClosed();
|
|
3969
4514
|
return this.inner.getCallers(targetName, branch);
|
|
3970
4515
|
}
|
|
3971
4516
|
getCallersWithContext(targetName, branch) {
|
|
4517
|
+
this.throwIfClosed();
|
|
3972
4518
|
return this.inner.getCallersWithContext(targetName, branch);
|
|
3973
4519
|
}
|
|
3974
4520
|
getCallees(symbolId, branch) {
|
|
4521
|
+
this.throwIfClosed();
|
|
3975
4522
|
return this.inner.getCallees(symbolId, branch);
|
|
3976
4523
|
}
|
|
3977
4524
|
deleteCallEdgesByFile(filePath) {
|
|
4525
|
+
this.throwIfClosed();
|
|
3978
4526
|
return this.inner.deleteCallEdgesByFile(filePath);
|
|
3979
4527
|
}
|
|
3980
4528
|
resolveCallEdge(edgeId, toSymbolId) {
|
|
4529
|
+
this.throwIfClosed();
|
|
3981
4530
|
this.inner.resolveCallEdge(edgeId, toSymbolId);
|
|
3982
4531
|
}
|
|
3983
4532
|
// ── Branch Symbol methods ────────────────────────────────────────
|
|
3984
4533
|
addSymbolsToBranch(branch, symbolIds) {
|
|
4534
|
+
this.throwIfClosed();
|
|
3985
4535
|
this.inner.addSymbolsToBranch(branch, symbolIds);
|
|
3986
4536
|
}
|
|
3987
4537
|
addSymbolsToBranchBatch(branch, symbolIds) {
|
|
4538
|
+
this.throwIfClosed();
|
|
3988
4539
|
if (symbolIds.length === 0) return;
|
|
3989
4540
|
this.inner.addSymbolsToBranchBatch(branch, symbolIds);
|
|
3990
4541
|
}
|
|
3991
4542
|
getBranchSymbolIds(branch) {
|
|
4543
|
+
this.throwIfClosed();
|
|
3992
4544
|
return this.inner.getBranchSymbolIds(branch);
|
|
3993
4545
|
}
|
|
3994
4546
|
clearBranchSymbols(branch) {
|
|
4547
|
+
this.throwIfClosed();
|
|
3995
4548
|
return this.inner.clearBranchSymbols(branch);
|
|
3996
4549
|
}
|
|
4550
|
+
getReferencedSymbolIds(symbolIds) {
|
|
4551
|
+
this.throwIfClosed();
|
|
4552
|
+
if (symbolIds.length === 0) return [];
|
|
4553
|
+
return this.inner.getReferencedSymbolIds(symbolIds);
|
|
4554
|
+
}
|
|
4555
|
+
deleteBranchSymbolsBySymbolIds(symbolIds) {
|
|
4556
|
+
this.throwIfClosed();
|
|
4557
|
+
if (symbolIds.length === 0) return 0;
|
|
4558
|
+
return this.inner.deleteBranchSymbolsBySymbolIds(symbolIds);
|
|
4559
|
+
}
|
|
4560
|
+
deleteBranchSymbolsForBranch(branch, symbolIds) {
|
|
4561
|
+
this.throwIfClosed();
|
|
4562
|
+
if (symbolIds.length === 0) return 0;
|
|
4563
|
+
return this.inner.deleteBranchSymbolsForBranch(branch, symbolIds);
|
|
4564
|
+
}
|
|
3997
4565
|
// ── GC methods for symbols/edges ─────────────────────────────────
|
|
3998
4566
|
gcOrphanSymbols() {
|
|
4567
|
+
this.throwIfClosed();
|
|
3999
4568
|
return this.inner.gcOrphanSymbols();
|
|
4000
4569
|
}
|
|
4001
4570
|
gcOrphanCallEdges() {
|
|
4571
|
+
this.throwIfClosed();
|
|
4002
4572
|
return this.inner.gcOrphanCallEdges();
|
|
4003
4573
|
}
|
|
4004
4574
|
};
|
|
4005
4575
|
|
|
4006
|
-
// src/git/index.ts
|
|
4007
|
-
var import_fs4 = require("fs");
|
|
4008
|
-
var path5 = __toESM(require("path"), 1);
|
|
4009
|
-
function readPackedRefs(gitDir) {
|
|
4010
|
-
const packedRefsPath = path5.join(gitDir, "packed-refs");
|
|
4011
|
-
if (!(0, import_fs4.existsSync)(packedRefsPath)) {
|
|
4012
|
-
return [];
|
|
4013
|
-
}
|
|
4014
|
-
try {
|
|
4015
|
-
return (0, import_fs4.readFileSync)(packedRefsPath, "utf-8").split("\n").map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("^"));
|
|
4016
|
-
} catch {
|
|
4017
|
-
return [];
|
|
4018
|
-
}
|
|
4019
|
-
}
|
|
4020
|
-
function resolveCommonGitDir(gitDir) {
|
|
4021
|
-
const commonDirPath = path5.join(gitDir, "commondir");
|
|
4022
|
-
if (!(0, import_fs4.existsSync)(commonDirPath)) {
|
|
4023
|
-
return gitDir;
|
|
4024
|
-
}
|
|
4025
|
-
try {
|
|
4026
|
-
const raw = (0, import_fs4.readFileSync)(commonDirPath, "utf-8").trim();
|
|
4027
|
-
if (!raw) {
|
|
4028
|
-
return gitDir;
|
|
4029
|
-
}
|
|
4030
|
-
const resolved = path5.isAbsolute(raw) ? raw : path5.resolve(gitDir, raw);
|
|
4031
|
-
if ((0, import_fs4.existsSync)(resolved)) {
|
|
4032
|
-
return resolved;
|
|
4033
|
-
}
|
|
4034
|
-
} catch {
|
|
4035
|
-
return gitDir;
|
|
4036
|
-
}
|
|
4037
|
-
return gitDir;
|
|
4038
|
-
}
|
|
4039
|
-
function resolveGitDir(repoRoot) {
|
|
4040
|
-
const gitPath = path5.join(repoRoot, ".git");
|
|
4041
|
-
if (!(0, import_fs4.existsSync)(gitPath)) {
|
|
4042
|
-
return null;
|
|
4043
|
-
}
|
|
4044
|
-
try {
|
|
4045
|
-
const stat = (0, import_fs4.statSync)(gitPath);
|
|
4046
|
-
if (stat.isDirectory()) {
|
|
4047
|
-
return gitPath;
|
|
4048
|
-
}
|
|
4049
|
-
if (stat.isFile()) {
|
|
4050
|
-
const content = (0, import_fs4.readFileSync)(gitPath, "utf-8").trim();
|
|
4051
|
-
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
4052
|
-
if (match) {
|
|
4053
|
-
const gitdir = match[1];
|
|
4054
|
-
const resolvedPath = path5.isAbsolute(gitdir) ? gitdir : path5.resolve(repoRoot, gitdir);
|
|
4055
|
-
if ((0, import_fs4.existsSync)(resolvedPath)) {
|
|
4056
|
-
return resolvedPath;
|
|
4057
|
-
}
|
|
4058
|
-
}
|
|
4059
|
-
}
|
|
4060
|
-
} catch {
|
|
4061
|
-
}
|
|
4062
|
-
return null;
|
|
4063
|
-
}
|
|
4064
|
-
function isGitRepo(dir) {
|
|
4065
|
-
return resolveGitDir(dir) !== null;
|
|
4066
|
-
}
|
|
4067
|
-
function getCurrentBranch(repoRoot) {
|
|
4068
|
-
const gitDir = resolveGitDir(repoRoot);
|
|
4069
|
-
if (!gitDir) {
|
|
4070
|
-
return null;
|
|
4071
|
-
}
|
|
4072
|
-
const headPath = path5.join(gitDir, "HEAD");
|
|
4073
|
-
if (!(0, import_fs4.existsSync)(headPath)) {
|
|
4074
|
-
return null;
|
|
4075
|
-
}
|
|
4076
|
-
try {
|
|
4077
|
-
const headContent = (0, import_fs4.readFileSync)(headPath, "utf-8").trim();
|
|
4078
|
-
const match = headContent.match(/^ref: refs\/heads\/(.+)$/);
|
|
4079
|
-
if (match) {
|
|
4080
|
-
return match[1];
|
|
4081
|
-
}
|
|
4082
|
-
if (/^[0-9a-f]{40}$/i.test(headContent)) {
|
|
4083
|
-
return headContent.slice(0, 7);
|
|
4084
|
-
}
|
|
4085
|
-
return null;
|
|
4086
|
-
} catch {
|
|
4087
|
-
return null;
|
|
4088
|
-
}
|
|
4089
|
-
}
|
|
4090
|
-
function getBaseBranch(repoRoot) {
|
|
4091
|
-
const gitDir = resolveGitDir(repoRoot);
|
|
4092
|
-
const refStoreDir = gitDir ? resolveCommonGitDir(gitDir) : null;
|
|
4093
|
-
const candidates = ["main", "master", "develop", "trunk"];
|
|
4094
|
-
if (refStoreDir) {
|
|
4095
|
-
for (const candidate of candidates) {
|
|
4096
|
-
const refPath = path5.join(refStoreDir, "refs", "heads", candidate);
|
|
4097
|
-
if ((0, import_fs4.existsSync)(refPath)) {
|
|
4098
|
-
return candidate;
|
|
4099
|
-
}
|
|
4100
|
-
const packedRefs = readPackedRefs(refStoreDir);
|
|
4101
|
-
if (packedRefs.some((line) => line.endsWith(` refs/heads/${candidate}`))) {
|
|
4102
|
-
return candidate;
|
|
4103
|
-
}
|
|
4104
|
-
}
|
|
4105
|
-
}
|
|
4106
|
-
return getCurrentBranch(repoRoot) ?? "main";
|
|
4107
|
-
}
|
|
4108
|
-
function getBranchOrDefault(repoRoot) {
|
|
4109
|
-
if (!isGitRepo(repoRoot)) {
|
|
4110
|
-
return "default";
|
|
4111
|
-
}
|
|
4112
|
-
return getCurrentBranch(repoRoot) ?? "default";
|
|
4113
|
-
}
|
|
4114
|
-
|
|
4115
4576
|
// src/indexer/index.ts
|
|
4116
|
-
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "php"]);
|
|
4577
|
+
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "php", "apex", "zig"]);
|
|
4578
|
+
var CASE_INSENSITIVE_LANGUAGES = /* @__PURE__ */ new Set(["apex"]);
|
|
4117
4579
|
var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
4118
4580
|
"function_declaration",
|
|
4119
4581
|
"function",
|
|
@@ -4135,7 +4597,11 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
|
4135
4597
|
"enum_item",
|
|
4136
4598
|
"trait_item",
|
|
4137
4599
|
"mod_item",
|
|
4138
|
-
"trait_declaration"
|
|
4600
|
+
"trait_declaration",
|
|
4601
|
+
"trigger_declaration",
|
|
4602
|
+
"test_declaration",
|
|
4603
|
+
"struct_declaration",
|
|
4604
|
+
"union_declaration"
|
|
4139
4605
|
]);
|
|
4140
4606
|
function float32ArrayToBuffer(arr) {
|
|
4141
4607
|
const float32 = new Float32Array(arr);
|
|
@@ -4160,12 +4626,192 @@ function isRateLimitError(error) {
|
|
|
4160
4626
|
const message = getErrorMessage(error);
|
|
4161
4627
|
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
4162
4628
|
}
|
|
4629
|
+
function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
4630
|
+
const providerMaxTokens = provider.modelInfo.maxTokens;
|
|
4631
|
+
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
4632
|
+
return Math.min(2e3, maxChunkTokens);
|
|
4633
|
+
}
|
|
4634
|
+
function getDynamicBatchOptions(provider) {
|
|
4635
|
+
if (provider.provider === "ollama") {
|
|
4636
|
+
return {
|
|
4637
|
+
maxBatchTokens: provider.modelInfo.maxTokens,
|
|
4638
|
+
maxBatchItems: 1
|
|
4639
|
+
};
|
|
4640
|
+
}
|
|
4641
|
+
return {};
|
|
4642
|
+
}
|
|
4643
|
+
function isSqliteCorruptionError(error) {
|
|
4644
|
+
const message = getErrorMessage(error).toLowerCase();
|
|
4645
|
+
return message.includes("database disk image is malformed") || message.includes("file is not a database") || message.includes("database schema is corrupt") || message.includes("sqlite_corrupt");
|
|
4646
|
+
}
|
|
4647
|
+
var STARTUP_WARNING_METADATA_KEY = "index.startupWarning";
|
|
4163
4648
|
var INDEX_METADATA_VERSION = "1";
|
|
4649
|
+
var EMBEDDING_STRATEGY_VERSION = "2";
|
|
4164
4650
|
var RANKING_TOKEN_CACHE_LIMIT = 4096;
|
|
4651
|
+
var RANK_HYBRID_CACHE_LIMIT = 256;
|
|
4652
|
+
function createPendingChunkStorageText(texts) {
|
|
4653
|
+
const primaryText = texts[0]?.text ?? "";
|
|
4654
|
+
if (texts.length <= 1) {
|
|
4655
|
+
return primaryText;
|
|
4656
|
+
}
|
|
4657
|
+
return `${primaryText}
|
|
4658
|
+
|
|
4659
|
+
... [split into ${texts.length} parts for embedding]`;
|
|
4660
|
+
}
|
|
4661
|
+
function normalizePendingChunk(rawChunk, maxChunkTokens) {
|
|
4662
|
+
if (!rawChunk || typeof rawChunk !== "object") {
|
|
4663
|
+
return null;
|
|
4664
|
+
}
|
|
4665
|
+
const chunk = rawChunk;
|
|
4666
|
+
if (typeof chunk.id !== "string" || typeof chunk.contentHash !== "string" || !chunk.metadata || typeof chunk.metadata !== "object") {
|
|
4667
|
+
return null;
|
|
4668
|
+
}
|
|
4669
|
+
const texts = Array.isArray(chunk.texts) ? chunk.texts.map((entry) => {
|
|
4670
|
+
if (!entry || typeof entry.text !== "string") {
|
|
4671
|
+
return null;
|
|
4672
|
+
}
|
|
4673
|
+
return {
|
|
4674
|
+
text: entry.text,
|
|
4675
|
+
tokenCount: typeof entry.tokenCount === "number" && Number.isFinite(entry.tokenCount) ? entry.tokenCount : estimateTokens2(entry.text)
|
|
4676
|
+
};
|
|
4677
|
+
}).filter((entry) => entry !== null) : [];
|
|
4678
|
+
if (texts.length === 0 && typeof chunk.text === "string") {
|
|
4679
|
+
if (typeof chunk.content === "string" && chunk.content.length > 0 && chunk.metadata && typeof chunk.metadata === "object") {
|
|
4680
|
+
const metadata = chunk.metadata;
|
|
4681
|
+
const rebuiltChunk = {
|
|
4682
|
+
content: chunk.content,
|
|
4683
|
+
startLine: typeof metadata.startLine === "number" ? metadata.startLine : 1,
|
|
4684
|
+
endLine: typeof metadata.endLine === "number" ? metadata.endLine : 1,
|
|
4685
|
+
chunkType: typeof metadata.chunkType === "string" ? metadata.chunkType : "other",
|
|
4686
|
+
name: typeof metadata.name === "string" ? metadata.name : void 0,
|
|
4687
|
+
language: typeof metadata.language === "string" ? metadata.language : "text"
|
|
4688
|
+
};
|
|
4689
|
+
const filePath = typeof metadata.filePath === "string" ? metadata.filePath : "unknown";
|
|
4690
|
+
texts.push(
|
|
4691
|
+
...createEmbeddingTexts(rebuiltChunk, filePath, maxChunkTokens).map((text) => ({
|
|
4692
|
+
text,
|
|
4693
|
+
tokenCount: estimateTokens2(text)
|
|
4694
|
+
}))
|
|
4695
|
+
);
|
|
4696
|
+
} else {
|
|
4697
|
+
texts.push({
|
|
4698
|
+
text: chunk.text,
|
|
4699
|
+
tokenCount: estimateTokens2(chunk.text)
|
|
4700
|
+
});
|
|
4701
|
+
}
|
|
4702
|
+
}
|
|
4703
|
+
if (texts.length === 0) {
|
|
4704
|
+
return null;
|
|
4705
|
+
}
|
|
4706
|
+
return {
|
|
4707
|
+
id: chunk.id,
|
|
4708
|
+
texts,
|
|
4709
|
+
storageText: typeof chunk.storageText === "string" ? chunk.storageText : createPendingChunkStorageText(texts),
|
|
4710
|
+
content: typeof chunk.content === "string" ? chunk.content : "",
|
|
4711
|
+
contentHash: chunk.contentHash,
|
|
4712
|
+
metadata: chunk.metadata
|
|
4713
|
+
};
|
|
4714
|
+
}
|
|
4715
|
+
function getPendingChunkFilePath(rawChunk) {
|
|
4716
|
+
if (!rawChunk || typeof rawChunk !== "object") {
|
|
4717
|
+
return null;
|
|
4718
|
+
}
|
|
4719
|
+
const chunk = rawChunk;
|
|
4720
|
+
if (!chunk.metadata || typeof chunk.metadata !== "object") {
|
|
4721
|
+
return null;
|
|
4722
|
+
}
|
|
4723
|
+
const metadata = chunk.metadata;
|
|
4724
|
+
return typeof metadata.filePath === "string" ? metadata.filePath : null;
|
|
4725
|
+
}
|
|
4726
|
+
function normalizeFailedBatch(batch, maxChunkTokens) {
|
|
4727
|
+
const chunks = batch.chunks.map((chunk) => normalizePendingChunk(chunk, maxChunkTokens)).filter((chunk) => chunk !== null);
|
|
4728
|
+
if (chunks.length === 0) {
|
|
4729
|
+
return null;
|
|
4730
|
+
}
|
|
4731
|
+
return {
|
|
4732
|
+
chunks,
|
|
4733
|
+
error: batch.error,
|
|
4734
|
+
attemptCount: batch.attemptCount,
|
|
4735
|
+
lastAttempt: batch.lastAttempt
|
|
4736
|
+
};
|
|
4737
|
+
}
|
|
4738
|
+
function createPendingEmbeddingRequests(chunks) {
|
|
4739
|
+
return chunks.flatMap(
|
|
4740
|
+
(chunk) => chunk.texts.map((textPart, partIndex) => ({
|
|
4741
|
+
chunk,
|
|
4742
|
+
partIndex,
|
|
4743
|
+
text: textPart.text,
|
|
4744
|
+
tokenCount: textPart.tokenCount
|
|
4745
|
+
}))
|
|
4746
|
+
);
|
|
4747
|
+
}
|
|
4748
|
+
function createPendingEmbeddingRequestBatches(chunks, options = {}) {
|
|
4749
|
+
return createDynamicBatches(createPendingEmbeddingRequests(chunks), options);
|
|
4750
|
+
}
|
|
4751
|
+
function getUniquePendingChunksFromRequests(requests) {
|
|
4752
|
+
const uniqueChunks = /* @__PURE__ */ new Map();
|
|
4753
|
+
for (const request of requests) {
|
|
4754
|
+
uniqueChunks.set(request.chunk.id, request.chunk);
|
|
4755
|
+
}
|
|
4756
|
+
return Array.from(uniqueChunks.values());
|
|
4757
|
+
}
|
|
4758
|
+
function coalesceFailedBatches(batches) {
|
|
4759
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
4760
|
+
for (const batch of batches) {
|
|
4761
|
+
const key = `${batch.attemptCount}:${batch.lastAttempt}:${batch.error}`;
|
|
4762
|
+
const existing = grouped.get(key);
|
|
4763
|
+
if (!existing) {
|
|
4764
|
+
grouped.set(key, {
|
|
4765
|
+
...batch,
|
|
4766
|
+
chunks: [...batch.chunks]
|
|
4767
|
+
});
|
|
4768
|
+
continue;
|
|
4769
|
+
}
|
|
4770
|
+
existing.chunks.push(...batch.chunks);
|
|
4771
|
+
}
|
|
4772
|
+
return Array.from(grouped.values());
|
|
4773
|
+
}
|
|
4774
|
+
function poolEmbeddingVectors(vectors, weights) {
|
|
4775
|
+
const firstVector = vectors[0];
|
|
4776
|
+
if (!firstVector) {
|
|
4777
|
+
return [];
|
|
4778
|
+
}
|
|
4779
|
+
const pooled = new Array(firstVector.length).fill(0);
|
|
4780
|
+
let totalWeight = 0;
|
|
4781
|
+
for (let index = 0; index < vectors.length; index++) {
|
|
4782
|
+
const vector = vectors[index];
|
|
4783
|
+
const weight = Math.max(1, weights[index] ?? 1);
|
|
4784
|
+
totalWeight += weight;
|
|
4785
|
+
for (let dimension = 0; dimension < vector.length; dimension++) {
|
|
4786
|
+
pooled[dimension] += vector[dimension] * weight;
|
|
4787
|
+
}
|
|
4788
|
+
}
|
|
4789
|
+
if (totalWeight === 0) {
|
|
4790
|
+
return firstVector;
|
|
4791
|
+
}
|
|
4792
|
+
return pooled.map((value) => value / totalWeight);
|
|
4793
|
+
}
|
|
4794
|
+
function hasAllEmbeddingParts(parts, expectedPartCount) {
|
|
4795
|
+
if (parts.length !== expectedPartCount) {
|
|
4796
|
+
return false;
|
|
4797
|
+
}
|
|
4798
|
+
for (let index = 0; index < expectedPartCount; index++) {
|
|
4799
|
+
if (parts[index] === void 0) {
|
|
4800
|
+
return false;
|
|
4801
|
+
}
|
|
4802
|
+
}
|
|
4803
|
+
return true;
|
|
4804
|
+
}
|
|
4805
|
+
function isPathWithinRoot(filePath, rootPath) {
|
|
4806
|
+
const normalizedFilePath = path8.resolve(filePath);
|
|
4807
|
+
const normalizedRoot = path8.resolve(rootPath);
|
|
4808
|
+
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${path8.sep}`);
|
|
4809
|
+
}
|
|
4165
4810
|
var rankingQueryTokenCache = /* @__PURE__ */ new Map();
|
|
4166
4811
|
var rankingNameTokenCache = /* @__PURE__ */ new Map();
|
|
4167
4812
|
var rankingPathTokenCache = /* @__PURE__ */ new Map();
|
|
4168
4813
|
var rankingTextTokenCache = /* @__PURE__ */ new Map();
|
|
4814
|
+
var rankHybridResultsCache = /* @__PURE__ */ new WeakMap();
|
|
4169
4815
|
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
4170
4816
|
"the",
|
|
4171
4817
|
"and",
|
|
@@ -4381,7 +5027,16 @@ function classifyExternalRerankBand(candidate, preferSourcePaths, docIntent) {
|
|
|
4381
5027
|
function classifyQueryIntent(tokens) {
|
|
4382
5028
|
const sourceIntentHits = tokens.filter((t) => SOURCE_INTENT_HINTS.has(t)).length;
|
|
4383
5029
|
const docTestIntentHits = tokens.filter((t) => DOC_TEST_INTENT_HINTS.has(t)).length;
|
|
4384
|
-
|
|
5030
|
+
if (sourceIntentHits === 0 && docTestIntentHits === 0) {
|
|
5031
|
+
return "neutral";
|
|
5032
|
+
}
|
|
5033
|
+
if (sourceIntentHits > docTestIntentHits) {
|
|
5034
|
+
return "source";
|
|
5035
|
+
}
|
|
5036
|
+
if (docTestIntentHits > sourceIntentHits) {
|
|
5037
|
+
return "doc_test";
|
|
5038
|
+
}
|
|
5039
|
+
return "neutral";
|
|
4385
5040
|
}
|
|
4386
5041
|
function classifyQueryIntentRaw(query) {
|
|
4387
5042
|
const lowerQuery = query.toLowerCase();
|
|
@@ -4403,7 +5058,7 @@ function classifyQueryIntentRaw(query) {
|
|
|
4403
5058
|
if (docTestRawHits > sourceRawHits) {
|
|
4404
5059
|
return "doc_test";
|
|
4405
5060
|
}
|
|
4406
|
-
if (sourceRawHits > docTestRawHits) {
|
|
5061
|
+
if (sourceRawHits > docTestRawHits && sourceRawHits > 0) {
|
|
4407
5062
|
return "source";
|
|
4408
5063
|
}
|
|
4409
5064
|
const hasWhereIsPattern = /\bwhere\s+is\b/.test(lowerQuery);
|
|
@@ -4670,9 +5325,8 @@ function rerankResults(query, candidates, rerankTopN, options) {
|
|
|
4670
5325
|
return candidates;
|
|
4671
5326
|
}
|
|
4672
5327
|
const queryTokenList = Array.from(queryTokens);
|
|
4673
|
-
const intent = classifyQueryIntentRaw(query);
|
|
4674
5328
|
const docIntent = classifyDocIntent(queryTokenList);
|
|
4675
|
-
const preferSourcePaths = options?.prioritizeSourcePaths ??
|
|
5329
|
+
const preferSourcePaths = options?.prioritizeSourcePaths ?? classifyQueryIntentRaw(query) === "source";
|
|
4676
5330
|
const identifierHints = extractIdentifierHints(query);
|
|
4677
5331
|
const head = candidates.slice(0, topN).map((candidate, idx) => {
|
|
4678
5332
|
const pathTokens = splitPathTokens(candidate.metadata.filePath);
|
|
@@ -4822,13 +5476,38 @@ function buildDiversityKey(metadata) {
|
|
|
4822
5476
|
return normalizedPath;
|
|
4823
5477
|
}
|
|
4824
5478
|
function rankHybridResults(query, semanticResults, keywordResults, options) {
|
|
5479
|
+
const prioritizeSourcePaths = options.prioritizeSourcePaths ?? classifyQueryIntentRaw(query) === "source";
|
|
5480
|
+
const cacheKey = `${query}${options.fusionStrategy}|${options.rrfK}|${options.hybridWeight}|${options.rerankTopN}|${options.limit}|${prioritizeSourcePaths ? 1 : 0}`;
|
|
5481
|
+
let byKeyword = rankHybridResultsCache.get(semanticResults);
|
|
5482
|
+
if (!byKeyword) {
|
|
5483
|
+
byKeyword = /* @__PURE__ */ new WeakMap();
|
|
5484
|
+
rankHybridResultsCache.set(semanticResults, byKeyword);
|
|
5485
|
+
}
|
|
5486
|
+
let bucket = byKeyword.get(keywordResults);
|
|
5487
|
+
if (!bucket) {
|
|
5488
|
+
bucket = /* @__PURE__ */ new Map();
|
|
5489
|
+
byKeyword.set(keywordResults, bucket);
|
|
5490
|
+
} else {
|
|
5491
|
+
const cached = bucket.get(cacheKey);
|
|
5492
|
+
if (cached) {
|
|
5493
|
+
return cached;
|
|
5494
|
+
}
|
|
5495
|
+
}
|
|
4825
5496
|
const overfetchLimit = Math.max(options.limit * 4, options.limit);
|
|
4826
5497
|
const fused = options.fusionStrategy === "rrf" ? fuseResultsRrf(semanticResults, keywordResults, options.rrfK, overfetchLimit) : fuseResultsWeighted(semanticResults, keywordResults, options.hybridWeight, overfetchLimit);
|
|
4827
5498
|
const rerankPoolLimit = Math.max(overfetchLimit, options.rerankTopN * 3, options.limit * 6);
|
|
4828
5499
|
const rerankPool = fused.slice(0, rerankPoolLimit);
|
|
4829
|
-
|
|
4830
|
-
prioritizeSourcePaths
|
|
5500
|
+
const ranked = rerankResults(query, rerankPool, options.rerankTopN, {
|
|
5501
|
+
prioritizeSourcePaths
|
|
4831
5502
|
});
|
|
5503
|
+
if (bucket.size >= RANK_HYBRID_CACHE_LIMIT) {
|
|
5504
|
+
const oldest = bucket.keys().next().value;
|
|
5505
|
+
if (oldest !== void 0) {
|
|
5506
|
+
bucket.delete(oldest);
|
|
5507
|
+
}
|
|
5508
|
+
}
|
|
5509
|
+
bucket.set(cacheKey, ranked);
|
|
5510
|
+
return ranked;
|
|
4832
5511
|
}
|
|
4833
5512
|
function rankSemanticOnlyResults(query, semanticResults, options) {
|
|
4834
5513
|
const overfetchLimit = Math.max(options.limit * 4, options.limit);
|
|
@@ -5125,7 +5804,22 @@ function mergeTieredResults(symbolLane, hybridLane, limit) {
|
|
|
5125
5804
|
}
|
|
5126
5805
|
return out;
|
|
5127
5806
|
}
|
|
5128
|
-
function
|
|
5807
|
+
function matchesSearchFilters(candidate, options, minScore) {
|
|
5808
|
+
if (candidate.score < minScore) return false;
|
|
5809
|
+
if (options?.fileType) {
|
|
5810
|
+
const ext = candidate.metadata.filePath.split(".").pop()?.toLowerCase();
|
|
5811
|
+
if (ext !== options.fileType.toLowerCase().replace(/^\./, "")) return false;
|
|
5812
|
+
}
|
|
5813
|
+
if (options?.directory) {
|
|
5814
|
+
const normalizedDir = options.directory.replace(/^\/|\/$/g, "");
|
|
5815
|
+
if (!candidate.metadata.filePath.includes(`/${normalizedDir}/`) && !candidate.metadata.filePath.includes(`${normalizedDir}/`)) return false;
|
|
5816
|
+
}
|
|
5817
|
+
if (options?.chunkType && candidate.metadata.chunkType !== options.chunkType) {
|
|
5818
|
+
return false;
|
|
5819
|
+
}
|
|
5820
|
+
return true;
|
|
5821
|
+
}
|
|
5822
|
+
function unionCandidates(semanticCandidates, keywordCandidates) {
|
|
5129
5823
|
const byId = /* @__PURE__ */ new Map();
|
|
5130
5824
|
for (const candidate of semanticCandidates) {
|
|
5131
5825
|
byId.set(candidate.id, candidate);
|
|
@@ -5164,22 +5858,18 @@ var Indexer = class {
|
|
|
5164
5858
|
this.projectRoot = projectRoot;
|
|
5165
5859
|
this.config = config;
|
|
5166
5860
|
this.indexPath = this.getIndexPath();
|
|
5167
|
-
this.fileHashCachePath =
|
|
5168
|
-
this.failedBatchesPath =
|
|
5169
|
-
this.indexingLockPath =
|
|
5861
|
+
this.fileHashCachePath = path8.join(this.indexPath, "file-hashes.json");
|
|
5862
|
+
this.failedBatchesPath = path8.join(this.indexPath, "failed-batches.json");
|
|
5863
|
+
this.indexingLockPath = path8.join(this.indexPath, "indexing.lock");
|
|
5170
5864
|
this.logger = initializeLogger(config.debug);
|
|
5171
5865
|
}
|
|
5172
5866
|
getIndexPath() {
|
|
5173
|
-
|
|
5174
|
-
const homeDir = process.env.HOME || process.env.USERPROFILE || "";
|
|
5175
|
-
return path6.join(homeDir, ".opencode", "global-index");
|
|
5176
|
-
}
|
|
5177
|
-
return path6.join(this.projectRoot, ".opencode", "index");
|
|
5867
|
+
return resolveProjectIndexPath(this.projectRoot, this.config.scope);
|
|
5178
5868
|
}
|
|
5179
5869
|
loadFileHashCache() {
|
|
5180
5870
|
try {
|
|
5181
|
-
if ((0,
|
|
5182
|
-
const data = (0,
|
|
5871
|
+
if ((0, import_fs7.existsSync)(this.fileHashCachePath)) {
|
|
5872
|
+
const data = (0, import_fs7.readFileSync)(this.fileHashCachePath, "utf-8");
|
|
5183
5873
|
const parsed = JSON.parse(data);
|
|
5184
5874
|
this.fileHashCache = new Map(Object.entries(parsed));
|
|
5185
5875
|
}
|
|
@@ -5196,63 +5886,423 @@ var Indexer = class {
|
|
|
5196
5886
|
}
|
|
5197
5887
|
atomicWriteSync(targetPath, data) {
|
|
5198
5888
|
const tempPath = `${targetPath}.tmp`;
|
|
5199
|
-
(0,
|
|
5200
|
-
(0,
|
|
5889
|
+
(0, import_fs7.mkdirSync)(path8.dirname(targetPath), { recursive: true });
|
|
5890
|
+
(0, import_fs7.writeFileSync)(tempPath, data);
|
|
5891
|
+
(0, import_fs7.renameSync)(tempPath, targetPath);
|
|
5892
|
+
}
|
|
5893
|
+
getScopedRoots() {
|
|
5894
|
+
const roots = /* @__PURE__ */ new Set([path8.resolve(this.projectRoot)]);
|
|
5895
|
+
for (const kbRoot of this.config.knowledgeBases) {
|
|
5896
|
+
roots.add(path8.resolve(this.projectRoot, kbRoot));
|
|
5897
|
+
}
|
|
5898
|
+
return Array.from(roots);
|
|
5899
|
+
}
|
|
5900
|
+
getBranchCatalogKey() {
|
|
5901
|
+
const branchName = this.currentBranch || "default";
|
|
5902
|
+
if (this.config.scope !== "global") {
|
|
5903
|
+
return branchName;
|
|
5904
|
+
}
|
|
5905
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5906
|
+
return `${projectHash}:${branchName}`;
|
|
5907
|
+
}
|
|
5908
|
+
getLegacyBranchCatalogKey() {
|
|
5909
|
+
return this.currentBranch || "default";
|
|
5910
|
+
}
|
|
5911
|
+
getLegacyMigrationMetadataKey() {
|
|
5912
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5913
|
+
return `index.globalBranchMigration.${projectHash}`;
|
|
5914
|
+
}
|
|
5915
|
+
getProjectEmbeddingStrategyMetadataKey() {
|
|
5916
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5917
|
+
return `index.embeddingStrategyVersion.${projectHash}`;
|
|
5918
|
+
}
|
|
5919
|
+
getProjectForceReembedMetadataKey() {
|
|
5920
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5921
|
+
return `index.forceReembed.${projectHash}`;
|
|
5922
|
+
}
|
|
5923
|
+
hasProjectForceReembedPending() {
|
|
5924
|
+
return this.config.scope === "global" && this.database?.getMetadata(this.getProjectForceReembedMetadataKey()) === "true";
|
|
5925
|
+
}
|
|
5926
|
+
hasScopedIndexedData() {
|
|
5927
|
+
if (!this.store || this.config.scope !== "global") {
|
|
5928
|
+
return false;
|
|
5929
|
+
}
|
|
5930
|
+
if (this.hasProjectForceReembedPending()) {
|
|
5931
|
+
return false;
|
|
5932
|
+
}
|
|
5933
|
+
const roots = this.getScopedRoots();
|
|
5934
|
+
if (Array.from(this.fileHashCache.keys()).some((filePath) => this.isFileInCurrentScope(filePath, roots))) {
|
|
5935
|
+
return true;
|
|
5936
|
+
}
|
|
5937
|
+
if (this.loadSerializedFailedBatches().some(
|
|
5938
|
+
(batch) => batch.chunks.some((chunk) => {
|
|
5939
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
5940
|
+
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
5941
|
+
})
|
|
5942
|
+
)) {
|
|
5943
|
+
return true;
|
|
5944
|
+
}
|
|
5945
|
+
if (!this.database) {
|
|
5946
|
+
return false;
|
|
5947
|
+
}
|
|
5948
|
+
if (this.getBranchCatalogKeys().some((branchKey) => {
|
|
5949
|
+
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
5950
|
+
if (branchChunkIds.length > 0) {
|
|
5951
|
+
return true;
|
|
5952
|
+
}
|
|
5953
|
+
return this.database.getBranchSymbolIds(branchKey).length > 0;
|
|
5954
|
+
})) {
|
|
5955
|
+
return true;
|
|
5956
|
+
}
|
|
5957
|
+
const hasAnyBranchRows = this.database.getAllBranches().some((branchKey) => {
|
|
5958
|
+
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
5959
|
+
if (branchChunkIds.length > 0) {
|
|
5960
|
+
return true;
|
|
5961
|
+
}
|
|
5962
|
+
return this.database.getBranchSymbolIds(branchKey).length > 0;
|
|
5963
|
+
});
|
|
5964
|
+
if (hasAnyBranchRows) {
|
|
5965
|
+
return false;
|
|
5966
|
+
}
|
|
5967
|
+
return this.store.getAllMetadata().some(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
5968
|
+
}
|
|
5969
|
+
loadStoredEmbeddingStrategyVersion() {
|
|
5970
|
+
if (!this.database) {
|
|
5971
|
+
return null;
|
|
5972
|
+
}
|
|
5973
|
+
if (this.hasProjectForceReembedPending()) {
|
|
5974
|
+
return null;
|
|
5975
|
+
}
|
|
5976
|
+
if (this.config.scope !== "global") {
|
|
5977
|
+
return this.database.getMetadata("index.embeddingStrategyVersion") ?? "1";
|
|
5978
|
+
}
|
|
5979
|
+
const projectVersion = this.database.getMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
5980
|
+
if (projectVersion) {
|
|
5981
|
+
return projectVersion;
|
|
5982
|
+
}
|
|
5983
|
+
const legacySharedVersion = this.database.getMetadata("index.embeddingStrategyVersion");
|
|
5984
|
+
if (legacySharedVersion && this.hasScopedIndexedData()) {
|
|
5985
|
+
return legacySharedVersion;
|
|
5986
|
+
}
|
|
5987
|
+
return null;
|
|
5988
|
+
}
|
|
5989
|
+
getBranchCatalogKeys() {
|
|
5990
|
+
const primary = this.getBranchCatalogKey();
|
|
5991
|
+
if (this.config.scope !== "global") {
|
|
5992
|
+
return [primary];
|
|
5993
|
+
}
|
|
5994
|
+
if (this.database?.getMetadata(this.getLegacyMigrationMetadataKey()) === "done") {
|
|
5995
|
+
return [primary];
|
|
5996
|
+
}
|
|
5997
|
+
const legacy = this.getLegacyBranchCatalogKey();
|
|
5998
|
+
return primary === legacy ? [primary] : [primary, legacy];
|
|
5999
|
+
}
|
|
6000
|
+
getBranchCatalogCleanupKeys() {
|
|
6001
|
+
const primary = this.getBranchCatalogKey();
|
|
6002
|
+
if (this.config.scope !== "global") {
|
|
6003
|
+
return [primary];
|
|
6004
|
+
}
|
|
6005
|
+
const legacy = this.getLegacyBranchCatalogKey();
|
|
6006
|
+
return primary === legacy ? [primary] : [primary, legacy];
|
|
6007
|
+
}
|
|
6008
|
+
getProjectLocalScopedOwnershipIds(roots) {
|
|
6009
|
+
const chunkIds = /* @__PURE__ */ new Set();
|
|
6010
|
+
const symbolIds = /* @__PURE__ */ new Set();
|
|
6011
|
+
if (!this.database) {
|
|
6012
|
+
return { chunkIds, symbolIds };
|
|
6013
|
+
}
|
|
6014
|
+
const projectRootPath = path8.resolve(this.projectRoot);
|
|
6015
|
+
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
6016
|
+
...Array.from(this.fileHashCache.keys()).filter(
|
|
6017
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath)
|
|
6018
|
+
),
|
|
6019
|
+
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
6020
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath)
|
|
6021
|
+
)
|
|
6022
|
+
]);
|
|
6023
|
+
for (const filePath of projectLocalFilePaths) {
|
|
6024
|
+
for (const chunk of this.database.getChunksByFile(filePath)) {
|
|
6025
|
+
chunkIds.add(chunk.chunkId);
|
|
6026
|
+
}
|
|
6027
|
+
for (const symbol of this.database.getSymbolsByFile(filePath)) {
|
|
6028
|
+
symbolIds.add(symbol.id);
|
|
6029
|
+
}
|
|
6030
|
+
}
|
|
6031
|
+
return { chunkIds, symbolIds };
|
|
6032
|
+
}
|
|
6033
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
6034
|
+
if (this.config.scope !== "global") {
|
|
6035
|
+
return this.getBranchCatalogCleanupKeys();
|
|
6036
|
+
}
|
|
6037
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
6038
|
+
const keys = /* @__PURE__ */ new Set();
|
|
6039
|
+
const projectChunkIdSet = new Set(projectChunkIds);
|
|
6040
|
+
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
6041
|
+
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
6042
|
+
if (branchKey.startsWith(`${projectHash}:`)) {
|
|
6043
|
+
keys.add(branchKey);
|
|
6044
|
+
continue;
|
|
6045
|
+
}
|
|
6046
|
+
if (branchKey.includes(":")) {
|
|
6047
|
+
continue;
|
|
6048
|
+
}
|
|
6049
|
+
const referencesProjectChunks = this.database?.getBranchChunkIds(branchKey).some((chunkId) => projectChunkIdSet.has(chunkId)) ?? false;
|
|
6050
|
+
const referencesProjectSymbols = this.database?.getBranchSymbolIds(branchKey).some((symbolId) => projectSymbolIdSet.has(symbolId)) ?? false;
|
|
6051
|
+
if (referencesProjectChunks || referencesProjectSymbols) {
|
|
6052
|
+
keys.add(branchKey);
|
|
6053
|
+
}
|
|
6054
|
+
}
|
|
6055
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
6056
|
+
keys.add(branchKey);
|
|
6057
|
+
}
|
|
6058
|
+
return Array.from(keys);
|
|
6059
|
+
}
|
|
6060
|
+
isFileInCurrentScope(filePath, roots) {
|
|
6061
|
+
return roots.some((root) => isPathWithinRoot(filePath, root));
|
|
6062
|
+
}
|
|
6063
|
+
clearScopedFileHashCache(roots) {
|
|
6064
|
+
for (const filePath of Array.from(this.fileHashCache.keys())) {
|
|
6065
|
+
if (this.isFileInCurrentScope(filePath, roots)) {
|
|
6066
|
+
this.fileHashCache.delete(filePath);
|
|
6067
|
+
}
|
|
6068
|
+
}
|
|
6069
|
+
this.saveFileHashCache();
|
|
6070
|
+
}
|
|
6071
|
+
replaceScopedFileHashCache(currentFileHashes, roots) {
|
|
6072
|
+
for (const filePath of Array.from(this.fileHashCache.keys())) {
|
|
6073
|
+
if (this.isFileInCurrentScope(filePath, roots)) {
|
|
6074
|
+
this.fileHashCache.delete(filePath);
|
|
6075
|
+
}
|
|
6076
|
+
}
|
|
6077
|
+
for (const [filePath, hash] of currentFileHashes) {
|
|
6078
|
+
this.fileHashCache.set(filePath, hash);
|
|
6079
|
+
}
|
|
6080
|
+
this.saveFileHashCache();
|
|
6081
|
+
}
|
|
6082
|
+
partitionFailedBatches(roots, maxChunkTokens) {
|
|
6083
|
+
const scoped = [];
|
|
6084
|
+
const retained = [];
|
|
6085
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
6086
|
+
const scopedChunks = batch.chunks.filter((chunk) => {
|
|
6087
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
6088
|
+
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
6089
|
+
});
|
|
6090
|
+
const retainedChunks = batch.chunks.filter((chunk) => {
|
|
6091
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
6092
|
+
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
6093
|
+
});
|
|
6094
|
+
if (scopedChunks.length > 0) {
|
|
6095
|
+
const normalizedBatch = normalizeFailedBatch({ ...batch, chunks: scopedChunks }, maxChunkTokens);
|
|
6096
|
+
if (normalizedBatch) {
|
|
6097
|
+
scoped.push(normalizedBatch);
|
|
6098
|
+
}
|
|
6099
|
+
}
|
|
6100
|
+
if (retainedChunks.length > 0) {
|
|
6101
|
+
retained.push({ ...batch, chunks: retainedChunks });
|
|
6102
|
+
}
|
|
6103
|
+
}
|
|
6104
|
+
return { scoped, retained };
|
|
6105
|
+
}
|
|
6106
|
+
clearScopedFailedBatches(roots) {
|
|
6107
|
+
const { retained: retainedBatches } = this.partitionFailedBatches(roots);
|
|
6108
|
+
this.saveFailedBatches(retainedBatches);
|
|
6109
|
+
}
|
|
6110
|
+
hasForeignScopedFileHashData(roots) {
|
|
6111
|
+
return Array.from(this.fileHashCache.keys()).some((filePath) => !this.isFileInCurrentScope(filePath, roots));
|
|
6112
|
+
}
|
|
6113
|
+
hasForeignScopedFailedBatches(roots) {
|
|
6114
|
+
const { retained } = this.partitionFailedBatches(roots);
|
|
6115
|
+
return retained.length > 0;
|
|
6116
|
+
}
|
|
6117
|
+
hasForeignScopedBranchData() {
|
|
6118
|
+
if (!this.database || this.config.scope !== "global") {
|
|
6119
|
+
return false;
|
|
6120
|
+
}
|
|
6121
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
6122
|
+
const roots = this.getScopedRoots();
|
|
6123
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
6124
|
+
return this.database.getAllBranches().some(
|
|
6125
|
+
(branchKey) => {
|
|
6126
|
+
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
6127
|
+
const branchSymbolIds = this.database.getBranchSymbolIds(branchKey);
|
|
6128
|
+
const hasBranchData = branchChunkIds.length > 0 || branchSymbolIds.length > 0;
|
|
6129
|
+
if (!hasBranchData) {
|
|
6130
|
+
return false;
|
|
6131
|
+
}
|
|
6132
|
+
if (branchKey.startsWith(`${projectHash}:`)) {
|
|
6133
|
+
return false;
|
|
6134
|
+
}
|
|
6135
|
+
if (!branchKey.includes(":")) {
|
|
6136
|
+
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
6137
|
+
const referencesCurrentProjectSymbols = branchSymbolIds.some((symbolId) => projectLocalSymbolIds.has(symbolId));
|
|
6138
|
+
return !(referencesCurrentProjectChunks || referencesCurrentProjectSymbols);
|
|
6139
|
+
}
|
|
6140
|
+
return true;
|
|
6141
|
+
}
|
|
6142
|
+
);
|
|
6143
|
+
}
|
|
6144
|
+
saveScopedFailedBatches(batches, roots) {
|
|
6145
|
+
const { retained } = this.partitionFailedBatches(roots);
|
|
6146
|
+
this.saveFailedBatches([...retained, ...batches]);
|
|
6147
|
+
}
|
|
6148
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
6149
|
+
const allMetadata = store.getAllMetadata();
|
|
6150
|
+
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
6151
|
+
const filePaths = /* @__PURE__ */ new Set([
|
|
6152
|
+
...Array.from(this.fileHashCache.keys()).filter((filePath) => this.isFileInCurrentScope(filePath, roots)),
|
|
6153
|
+
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
6154
|
+
]);
|
|
6155
|
+
const projectRootPath = path8.resolve(this.projectRoot);
|
|
6156
|
+
const projectLocalFilePaths = new Set(
|
|
6157
|
+
Array.from(filePaths).filter((filePath) => isPathWithinRoot(filePath, projectRootPath))
|
|
6158
|
+
);
|
|
6159
|
+
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
6160
|
+
for (const filePath of filePaths) {
|
|
6161
|
+
for (const chunk of database.getChunksByFile(filePath)) {
|
|
6162
|
+
removedChunkIds.add(chunk.chunkId);
|
|
6163
|
+
}
|
|
6164
|
+
}
|
|
6165
|
+
const removedChunkIdList = Array.from(removedChunkIds);
|
|
6166
|
+
const projectLocalChunkIds = new Set(
|
|
6167
|
+
scopedEntries.filter(({ metadata }) => isPathWithinRoot(metadata.filePath, projectRootPath)).map(({ key }) => key)
|
|
6168
|
+
);
|
|
6169
|
+
for (const filePath of projectLocalFilePaths) {
|
|
6170
|
+
for (const chunk of database.getChunksByFile(filePath)) {
|
|
6171
|
+
projectLocalChunkIds.add(chunk.chunkId);
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
const symbolIds = [];
|
|
6175
|
+
const projectLocalSymbolIds = /* @__PURE__ */ new Set();
|
|
6176
|
+
for (const filePath of filePaths) {
|
|
6177
|
+
for (const symbol of database.getSymbolsByFile(filePath)) {
|
|
6178
|
+
symbolIds.push(symbol.id);
|
|
6179
|
+
if (projectLocalFilePaths.has(filePath)) {
|
|
6180
|
+
projectLocalSymbolIds.add(symbol.id);
|
|
6181
|
+
}
|
|
6182
|
+
}
|
|
6183
|
+
}
|
|
6184
|
+
for (const branchKey of this.getProjectScopedBranchCatalogCleanupKeys(Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds))) {
|
|
6185
|
+
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
6186
|
+
}
|
|
6187
|
+
const sharedChunkIds = new Set(database.getReferencedChunkIds(removedChunkIdList));
|
|
6188
|
+
const removableChunkIds = removedChunkIdList.filter((chunkId) => !sharedChunkIds.has(chunkId));
|
|
6189
|
+
if (removableChunkIds.length > 0) {
|
|
6190
|
+
this.rebuildVectorStoreExcludingChunkIds(store, database, removableChunkIds);
|
|
6191
|
+
for (const chunkId of removableChunkIds) {
|
|
6192
|
+
invertedIndex.removeChunk(chunkId);
|
|
6193
|
+
}
|
|
6194
|
+
}
|
|
6195
|
+
for (const branchKey of this.getProjectScopedBranchCatalogCleanupKeys(Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds))) {
|
|
6196
|
+
database.deleteBranchSymbolsForBranch(branchKey, symbolIds);
|
|
6197
|
+
}
|
|
6198
|
+
const sharedSymbolIds = new Set(database.getReferencedSymbolIds(symbolIds));
|
|
6199
|
+
const removableSymbolIds = symbolIds.filter((symbolId) => !sharedSymbolIds.has(symbolId));
|
|
6200
|
+
database.clearCallEdgeTargetsForSymbols(removableSymbolIds);
|
|
6201
|
+
for (const filePath of filePaths) {
|
|
6202
|
+
const fileChunkIds = database.getChunksByFile(filePath).map((chunk) => chunk.chunkId);
|
|
6203
|
+
const fileSymbols = database.getSymbolsByFile(filePath);
|
|
6204
|
+
if (fileChunkIds.every((chunkId) => !sharedChunkIds.has(chunkId))) {
|
|
6205
|
+
database.deleteChunksByFile(filePath);
|
|
6206
|
+
}
|
|
6207
|
+
if (fileSymbols.every((symbol) => !sharedSymbolIds.has(symbol.id))) {
|
|
6208
|
+
database.deleteCallEdgesByFile(filePath);
|
|
6209
|
+
database.deleteSymbolsByFile(filePath);
|
|
6210
|
+
}
|
|
6211
|
+
}
|
|
6212
|
+
database.gcOrphanCallEdges();
|
|
6213
|
+
database.gcOrphanSymbols();
|
|
6214
|
+
database.gcOrphanEmbeddings();
|
|
6215
|
+
database.gcOrphanChunks();
|
|
6216
|
+
store.save();
|
|
6217
|
+
invertedIndex.save();
|
|
6218
|
+
return {
|
|
6219
|
+
removedChunkIds: removedChunkIdList,
|
|
6220
|
+
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
6221
|
+
};
|
|
5201
6222
|
}
|
|
5202
6223
|
checkForInterruptedIndexing() {
|
|
5203
|
-
return (0,
|
|
6224
|
+
return (0, import_fs7.existsSync)(this.indexingLockPath);
|
|
5204
6225
|
}
|
|
5205
6226
|
acquireIndexingLock() {
|
|
5206
6227
|
const lockData = {
|
|
5207
6228
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5208
6229
|
pid: process.pid
|
|
5209
6230
|
};
|
|
5210
|
-
(0,
|
|
6231
|
+
(0, import_fs7.writeFileSync)(this.indexingLockPath, JSON.stringify(lockData));
|
|
5211
6232
|
}
|
|
5212
6233
|
releaseIndexingLock() {
|
|
5213
|
-
if ((0,
|
|
5214
|
-
(0,
|
|
6234
|
+
if ((0, import_fs7.existsSync)(this.indexingLockPath)) {
|
|
6235
|
+
(0, import_fs7.unlinkSync)(this.indexingLockPath);
|
|
5215
6236
|
}
|
|
5216
6237
|
}
|
|
5217
6238
|
async recoverFromInterruptedIndexing() {
|
|
5218
6239
|
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
5219
|
-
if ((0,
|
|
5220
|
-
(0,
|
|
6240
|
+
if ((0, import_fs7.existsSync)(this.fileHashCachePath)) {
|
|
6241
|
+
(0, import_fs7.unlinkSync)(this.fileHashCachePath);
|
|
5221
6242
|
}
|
|
5222
6243
|
await this.healthCheck();
|
|
5223
6244
|
this.releaseIndexingLock();
|
|
5224
6245
|
this.logger.info("Recovery complete, next index will re-process all files");
|
|
5225
6246
|
}
|
|
5226
|
-
loadFailedBatches() {
|
|
6247
|
+
loadFailedBatches(maxChunkTokens) {
|
|
5227
6248
|
try {
|
|
5228
|
-
|
|
5229
|
-
const data = (0, import_fs5.readFileSync)(this.failedBatchesPath, "utf-8");
|
|
5230
|
-
return JSON.parse(data);
|
|
5231
|
-
}
|
|
6249
|
+
return this.loadSerializedFailedBatches().map((batch) => normalizeFailedBatch(batch, maxChunkTokens)).filter((batch) => batch !== null);
|
|
5232
6250
|
} catch {
|
|
5233
6251
|
return [];
|
|
5234
6252
|
}
|
|
5235
|
-
|
|
6253
|
+
}
|
|
6254
|
+
loadSerializedFailedBatches() {
|
|
6255
|
+
if (!(0, import_fs7.existsSync)(this.failedBatchesPath)) {
|
|
6256
|
+
return [];
|
|
6257
|
+
}
|
|
6258
|
+
const data = (0, import_fs7.readFileSync)(this.failedBatchesPath, "utf-8");
|
|
6259
|
+
const parsed = JSON.parse(data);
|
|
6260
|
+
return parsed.map((batch) => {
|
|
6261
|
+
const chunks = Array.isArray(batch.chunks) ? batch.chunks : [];
|
|
6262
|
+
if (chunks.length === 0) {
|
|
6263
|
+
return null;
|
|
6264
|
+
}
|
|
6265
|
+
return {
|
|
6266
|
+
chunks,
|
|
6267
|
+
error: typeof batch.error === "string" ? batch.error : "Unknown embedding error",
|
|
6268
|
+
attemptCount: typeof batch.attemptCount === "number" ? batch.attemptCount : 1,
|
|
6269
|
+
lastAttempt: typeof batch.lastAttempt === "string" ? batch.lastAttempt : (/* @__PURE__ */ new Date()).toISOString()
|
|
6270
|
+
};
|
|
6271
|
+
}).filter((batch) => batch !== null);
|
|
5236
6272
|
}
|
|
5237
6273
|
saveFailedBatches(batches) {
|
|
5238
6274
|
if (batches.length === 0) {
|
|
5239
|
-
if ((0,
|
|
5240
|
-
|
|
5241
|
-
|
|
6275
|
+
if ((0, import_fs7.existsSync)(this.failedBatchesPath)) {
|
|
6276
|
+
try {
|
|
6277
|
+
(0, import_fs7.unlinkSync)(this.failedBatchesPath);
|
|
6278
|
+
} catch {
|
|
6279
|
+
}
|
|
5242
6280
|
}
|
|
5243
6281
|
return;
|
|
5244
6282
|
}
|
|
5245
|
-
(0,
|
|
6283
|
+
(0, import_fs7.writeFileSync)(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
5246
6284
|
}
|
|
5247
|
-
|
|
5248
|
-
const
|
|
5249
|
-
|
|
5250
|
-
chunks
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
6285
|
+
collectRetryableFailedChunks(currentFileHashes, unchangedFilePaths, maxChunkTokens) {
|
|
6286
|
+
const retryableById = /* @__PURE__ */ new Map();
|
|
6287
|
+
for (const batch of this.loadFailedBatches(maxChunkTokens)) {
|
|
6288
|
+
for (const chunk of batch.chunks) {
|
|
6289
|
+
const filePath = chunk.metadata.filePath;
|
|
6290
|
+
if (!currentFileHashes.has(filePath)) {
|
|
6291
|
+
continue;
|
|
6292
|
+
}
|
|
6293
|
+
if (!unchangedFilePaths.has(filePath)) {
|
|
6294
|
+
continue;
|
|
6295
|
+
}
|
|
6296
|
+
const existing = retryableById.get(chunk.id);
|
|
6297
|
+
if (!existing || batch.attemptCount > existing.attemptCount) {
|
|
6298
|
+
retryableById.set(chunk.id, {
|
|
6299
|
+
chunk,
|
|
6300
|
+
attemptCount: batch.attemptCount
|
|
6301
|
+
});
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
}
|
|
6305
|
+
return Array.from(retryableById.values());
|
|
5256
6306
|
}
|
|
5257
6307
|
getProviderRateLimits(provider) {
|
|
5258
6308
|
switch (provider) {
|
|
@@ -5410,7 +6460,7 @@ var Indexer = class {
|
|
|
5410
6460
|
const intent = isLikelyImplementationPath(candidate.metadata.filePath) ? "implementation" : "doc_or_test";
|
|
5411
6461
|
parts.push(`intent_hint: ${intent}`);
|
|
5412
6462
|
try {
|
|
5413
|
-
const fileContent = await
|
|
6463
|
+
const fileContent = await import_fs7.promises.readFile(candidate.metadata.filePath, "utf-8");
|
|
5414
6464
|
const lines = fileContent.split("\n");
|
|
5415
6465
|
const snippetStartLine = Math.max(1, candidate.metadata.startLine - 2);
|
|
5416
6466
|
const snippetEndLine = Math.min(lines.length, candidate.metadata.endLine + 2);
|
|
@@ -5455,33 +6505,56 @@ var Indexer = class {
|
|
|
5455
6505
|
});
|
|
5456
6506
|
}
|
|
5457
6507
|
}
|
|
5458
|
-
await
|
|
6508
|
+
await import_fs7.promises.mkdir(this.indexPath, { recursive: true });
|
|
5459
6509
|
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
5460
|
-
const storePath =
|
|
6510
|
+
const storePath = path8.join(this.indexPath, "vectors");
|
|
5461
6511
|
this.store = new VectorStore(storePath, dimensions);
|
|
5462
|
-
const indexFilePath =
|
|
5463
|
-
if ((0,
|
|
6512
|
+
const indexFilePath = path8.join(this.indexPath, "vectors.usearch");
|
|
6513
|
+
if ((0, import_fs7.existsSync)(indexFilePath)) {
|
|
5464
6514
|
this.store.load();
|
|
5465
6515
|
}
|
|
5466
|
-
const invertedIndexPath =
|
|
6516
|
+
const invertedIndexPath = path8.join(this.indexPath, "inverted-index.json");
|
|
5467
6517
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
5468
6518
|
try {
|
|
5469
6519
|
this.invertedIndex.load();
|
|
5470
6520
|
} catch {
|
|
5471
|
-
if ((0,
|
|
5472
|
-
await
|
|
6521
|
+
if ((0, import_fs7.existsSync)(invertedIndexPath)) {
|
|
6522
|
+
await import_fs7.promises.unlink(invertedIndexPath);
|
|
5473
6523
|
}
|
|
5474
6524
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
5475
6525
|
}
|
|
5476
|
-
const dbPath =
|
|
5477
|
-
|
|
5478
|
-
|
|
6526
|
+
const dbPath = path8.join(this.indexPath, "codebase.db");
|
|
6527
|
+
let dbIsNew = !(0, import_fs7.existsSync)(dbPath);
|
|
6528
|
+
try {
|
|
6529
|
+
this.database = new Database(dbPath);
|
|
6530
|
+
} catch (error) {
|
|
6531
|
+
if (!await this.tryResetCorruptedIndex("initializing index database", error)) {
|
|
6532
|
+
throw error;
|
|
6533
|
+
}
|
|
6534
|
+
this.store = new VectorStore(storePath, dimensions);
|
|
6535
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
6536
|
+
this.database = new Database(dbPath);
|
|
6537
|
+
dbIsNew = true;
|
|
6538
|
+
}
|
|
6539
|
+
if (isGitRepo(this.projectRoot)) {
|
|
6540
|
+
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
6541
|
+
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
6542
|
+
this.logger.branch("info", "Detected git repository", {
|
|
6543
|
+
currentBranch: this.currentBranch,
|
|
6544
|
+
baseBranch: this.baseBranch
|
|
6545
|
+
});
|
|
6546
|
+
} else {
|
|
6547
|
+
this.currentBranch = "default";
|
|
6548
|
+
this.baseBranch = "default";
|
|
6549
|
+
this.logger.branch("debug", "Not a git repository, using default branch");
|
|
6550
|
+
}
|
|
5479
6551
|
if (this.checkForInterruptedIndexing()) {
|
|
5480
6552
|
await this.recoverFromInterruptedIndexing();
|
|
5481
6553
|
}
|
|
5482
6554
|
if (dbIsNew && this.store.count() > 0) {
|
|
5483
6555
|
this.migrateFromLegacyIndex();
|
|
5484
6556
|
}
|
|
6557
|
+
this.loadFileHashCache();
|
|
5485
6558
|
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
5486
6559
|
if (!this.indexCompatibility.compatible) {
|
|
5487
6560
|
this.logger.warn("Index compatibility issue detected", {
|
|
@@ -5490,18 +6563,6 @@ var Indexer = class {
|
|
|
5490
6563
|
configuredProviderInfo: this.configuredProviderInfo
|
|
5491
6564
|
});
|
|
5492
6565
|
}
|
|
5493
|
-
if (isGitRepo(this.projectRoot)) {
|
|
5494
|
-
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
5495
|
-
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
5496
|
-
this.logger.branch("info", "Detected git repository", {
|
|
5497
|
-
currentBranch: this.currentBranch,
|
|
5498
|
-
baseBranch: this.baseBranch
|
|
5499
|
-
});
|
|
5500
|
-
} else {
|
|
5501
|
-
this.currentBranch = "default";
|
|
5502
|
-
this.baseBranch = "default";
|
|
5503
|
-
this.logger.branch("debug", "Not a git repository, using default branch");
|
|
5504
|
-
}
|
|
5505
6566
|
if (this.config.indexing.autoGc) {
|
|
5506
6567
|
await this.maybeRunAutoGc();
|
|
5507
6568
|
}
|
|
@@ -5521,20 +6582,169 @@ var Indexer = class {
|
|
|
5521
6582
|
}
|
|
5522
6583
|
}
|
|
5523
6584
|
if (shouldRunGc) {
|
|
5524
|
-
await this.healthCheck();
|
|
6585
|
+
const result = await this.healthCheck();
|
|
6586
|
+
if (result.warning) {
|
|
6587
|
+
this.database.setMetadata(STARTUP_WARNING_METADATA_KEY, result.warning);
|
|
6588
|
+
} else {
|
|
6589
|
+
this.database.deleteMetadata(STARTUP_WARNING_METADATA_KEY);
|
|
6590
|
+
}
|
|
5525
6591
|
this.database.setMetadata("lastGcTimestamp", now.toString());
|
|
5526
6592
|
}
|
|
5527
6593
|
}
|
|
5528
6594
|
async maybeRunOrphanGc() {
|
|
5529
|
-
if (!this.database) return;
|
|
6595
|
+
if (!this.database) return null;
|
|
5530
6596
|
const stats = this.database.getStats();
|
|
5531
|
-
if (!stats) return;
|
|
6597
|
+
if (!stats) return null;
|
|
5532
6598
|
const orphanCount = stats.embeddingCount - stats.chunkCount;
|
|
5533
6599
|
if (orphanCount > this.config.indexing.gcOrphanThreshold) {
|
|
5534
|
-
|
|
5535
|
-
|
|
6600
|
+
try {
|
|
6601
|
+
this.database.gcOrphanEmbeddings();
|
|
6602
|
+
this.database.gcOrphanChunks();
|
|
6603
|
+
} catch (error) {
|
|
6604
|
+
if (await this.tryResetCorruptedIndex("running automatic orphan garbage collection", error)) {
|
|
6605
|
+
return {
|
|
6606
|
+
resetCorruptedIndex: true,
|
|
6607
|
+
warning: this.getCorruptedIndexWarning(path8.join(this.indexPath, "codebase.db"))
|
|
6608
|
+
};
|
|
6609
|
+
}
|
|
6610
|
+
throw error;
|
|
6611
|
+
}
|
|
5536
6612
|
this.database.setMetadata("lastGcTimestamp", Date.now().toString());
|
|
5537
6613
|
}
|
|
6614
|
+
return null;
|
|
6615
|
+
}
|
|
6616
|
+
rebuildVectorStoreExcludingChunkIds(store, database, excludedChunkIds) {
|
|
6617
|
+
const excludedSet = new Set(excludedChunkIds);
|
|
6618
|
+
if (excludedSet.size === 0) {
|
|
6619
|
+
return;
|
|
6620
|
+
}
|
|
6621
|
+
const retainedEntries = store.getAllMetadata().filter(({ key }) => !excludedSet.has(key));
|
|
6622
|
+
const storeBasePath = path8.join(this.indexPath, "vectors");
|
|
6623
|
+
const storeIndexPath = `${storeBasePath}.usearch`;
|
|
6624
|
+
const storeMetadataPath = `${storeBasePath}.meta.json`;
|
|
6625
|
+
const backupIndexPath = `${storeIndexPath}.bak`;
|
|
6626
|
+
const backupMetadataPath = `${storeMetadataPath}.bak`;
|
|
6627
|
+
let backedUpIndex = false;
|
|
6628
|
+
let backedUpMetadata = false;
|
|
6629
|
+
let rebuiltCount = 0;
|
|
6630
|
+
let skippedCount = 0;
|
|
6631
|
+
if ((0, import_fs7.existsSync)(backupIndexPath)) {
|
|
6632
|
+
(0, import_fs7.unlinkSync)(backupIndexPath);
|
|
6633
|
+
}
|
|
6634
|
+
if ((0, import_fs7.existsSync)(backupMetadataPath)) {
|
|
6635
|
+
(0, import_fs7.unlinkSync)(backupMetadataPath);
|
|
6636
|
+
}
|
|
6637
|
+
try {
|
|
6638
|
+
if ((0, import_fs7.existsSync)(storeIndexPath)) {
|
|
6639
|
+
(0, import_fs7.renameSync)(storeIndexPath, backupIndexPath);
|
|
6640
|
+
backedUpIndex = true;
|
|
6641
|
+
}
|
|
6642
|
+
if ((0, import_fs7.existsSync)(storeMetadataPath)) {
|
|
6643
|
+
(0, import_fs7.renameSync)(storeMetadataPath, backupMetadataPath);
|
|
6644
|
+
backedUpMetadata = true;
|
|
6645
|
+
}
|
|
6646
|
+
store.clear();
|
|
6647
|
+
for (const { key, metadata } of retainedEntries) {
|
|
6648
|
+
const chunk = database.getChunk(key);
|
|
6649
|
+
if (!chunk) {
|
|
6650
|
+
skippedCount += 1;
|
|
6651
|
+
continue;
|
|
6652
|
+
}
|
|
6653
|
+
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
6654
|
+
if (!embeddingBuffer) {
|
|
6655
|
+
skippedCount += 1;
|
|
6656
|
+
continue;
|
|
6657
|
+
}
|
|
6658
|
+
const vector = bufferToFloat32Array(embeddingBuffer);
|
|
6659
|
+
store.add(key, Array.from(vector), metadata);
|
|
6660
|
+
rebuiltCount += 1;
|
|
6661
|
+
}
|
|
6662
|
+
store.save();
|
|
6663
|
+
if (backedUpIndex && (0, import_fs7.existsSync)(backupIndexPath)) {
|
|
6664
|
+
(0, import_fs7.unlinkSync)(backupIndexPath);
|
|
6665
|
+
}
|
|
6666
|
+
if (backedUpMetadata && (0, import_fs7.existsSync)(backupMetadataPath)) {
|
|
6667
|
+
(0, import_fs7.unlinkSync)(backupMetadataPath);
|
|
6668
|
+
}
|
|
6669
|
+
this.logger.gc("info", "Rebuilt vector store to avoid native remove", {
|
|
6670
|
+
excludedChunks: excludedSet.size,
|
|
6671
|
+
rebuiltChunks: rebuiltCount,
|
|
6672
|
+
skippedChunks: skippedCount
|
|
6673
|
+
});
|
|
6674
|
+
} catch (error) {
|
|
6675
|
+
try {
|
|
6676
|
+
store.clear();
|
|
6677
|
+
} catch {
|
|
6678
|
+
}
|
|
6679
|
+
if ((0, import_fs7.existsSync)(storeIndexPath)) {
|
|
6680
|
+
(0, import_fs7.unlinkSync)(storeIndexPath);
|
|
6681
|
+
}
|
|
6682
|
+
if ((0, import_fs7.existsSync)(storeMetadataPath)) {
|
|
6683
|
+
(0, import_fs7.unlinkSync)(storeMetadataPath);
|
|
6684
|
+
}
|
|
6685
|
+
if (backedUpIndex && (0, import_fs7.existsSync)(backupIndexPath)) {
|
|
6686
|
+
(0, import_fs7.renameSync)(backupIndexPath, storeIndexPath);
|
|
6687
|
+
}
|
|
6688
|
+
if (backedUpMetadata && (0, import_fs7.existsSync)(backupMetadataPath)) {
|
|
6689
|
+
(0, import_fs7.renameSync)(backupMetadataPath, storeMetadataPath);
|
|
6690
|
+
}
|
|
6691
|
+
if (backedUpIndex || backedUpMetadata) {
|
|
6692
|
+
store.load();
|
|
6693
|
+
}
|
|
6694
|
+
throw error;
|
|
6695
|
+
}
|
|
6696
|
+
}
|
|
6697
|
+
getCorruptedIndexWarning(dbPath) {
|
|
6698
|
+
if (this.config.scope === "global") {
|
|
6699
|
+
return `Detected a corrupted shared global SQLite index at ${dbPath}. Automatic repair is disabled for global scope because it may delete other projects' index data. Remove or repair the shared index manually, then rerun index_codebase with force=true.`;
|
|
6700
|
+
}
|
|
6701
|
+
return `Detected a corrupted local SQLite index at ${dbPath} and reset the local index. Run index_codebase to rebuild search data.`;
|
|
6702
|
+
}
|
|
6703
|
+
async tryResetCorruptedIndex(stage, error) {
|
|
6704
|
+
if (!isSqliteCorruptionError(error)) {
|
|
6705
|
+
return false;
|
|
6706
|
+
}
|
|
6707
|
+
const dbPath = path8.join(this.indexPath, "codebase.db");
|
|
6708
|
+
const warning = this.getCorruptedIndexWarning(dbPath);
|
|
6709
|
+
const errorMessage = getErrorMessage(error);
|
|
6710
|
+
if (this.config.scope === "global") {
|
|
6711
|
+
this.logger.error("Detected corrupted shared global index database", {
|
|
6712
|
+
stage,
|
|
6713
|
+
dbPath,
|
|
6714
|
+
error: errorMessage
|
|
6715
|
+
});
|
|
6716
|
+
throw new Error(`${warning} Original SQLite error: ${errorMessage}`);
|
|
6717
|
+
}
|
|
6718
|
+
this.logger.warn("Detected corrupted local index database, resetting local index", {
|
|
6719
|
+
stage,
|
|
6720
|
+
dbPath,
|
|
6721
|
+
error: errorMessage
|
|
6722
|
+
});
|
|
6723
|
+
this.store = null;
|
|
6724
|
+
this.invertedIndex = null;
|
|
6725
|
+
this.database?.close();
|
|
6726
|
+
this.database = null;
|
|
6727
|
+
this.indexCompatibility = null;
|
|
6728
|
+
this.fileHashCache.clear();
|
|
6729
|
+
const resetPaths = [
|
|
6730
|
+
path8.join(this.indexPath, "codebase.db"),
|
|
6731
|
+
path8.join(this.indexPath, "codebase.db-shm"),
|
|
6732
|
+
path8.join(this.indexPath, "codebase.db-wal"),
|
|
6733
|
+
path8.join(this.indexPath, "vectors.usearch"),
|
|
6734
|
+
path8.join(this.indexPath, "inverted-index.json"),
|
|
6735
|
+
path8.join(this.indexPath, "file-hashes.json"),
|
|
6736
|
+
path8.join(this.indexPath, "failed-batches.json"),
|
|
6737
|
+
path8.join(this.indexPath, "indexing.lock"),
|
|
6738
|
+
path8.join(this.indexPath, "vectors")
|
|
6739
|
+
];
|
|
6740
|
+
await Promise.all(resetPaths.map(async (targetPath) => {
|
|
6741
|
+
try {
|
|
6742
|
+
await import_fs7.promises.rm(targetPath, { recursive: true, force: true });
|
|
6743
|
+
} catch {
|
|
6744
|
+
}
|
|
6745
|
+
}));
|
|
6746
|
+
await import_fs7.promises.mkdir(this.indexPath, { recursive: true });
|
|
6747
|
+
return true;
|
|
5538
6748
|
}
|
|
5539
6749
|
migrateFromLegacyIndex() {
|
|
5540
6750
|
if (!this.store || !this.database) return;
|
|
@@ -5558,7 +6768,7 @@ var Indexer = class {
|
|
|
5558
6768
|
if (chunkDataBatch.length > 0) {
|
|
5559
6769
|
this.database.upsertChunksBatch(chunkDataBatch);
|
|
5560
6770
|
}
|
|
5561
|
-
this.database.addChunksToBranchBatch(this.
|
|
6771
|
+
this.database.addChunksToBranchBatch(this.getBranchCatalogKey(), chunkIds);
|
|
5562
6772
|
}
|
|
5563
6773
|
loadIndexMetadata() {
|
|
5564
6774
|
if (!this.database) return null;
|
|
@@ -5569,6 +6779,7 @@ var Indexer = class {
|
|
|
5569
6779
|
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
5570
6780
|
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
5571
6781
|
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
6782
|
+
embeddingStrategyVersion: this.loadStoredEmbeddingStrategyVersion() ?? EMBEDDING_STRATEGY_VERSION,
|
|
5572
6783
|
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
5573
6784
|
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
5574
6785
|
};
|
|
@@ -5577,10 +6788,22 @@ var Indexer = class {
|
|
|
5577
6788
|
if (!this.database) return;
|
|
5578
6789
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5579
6790
|
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
6791
|
+
const completeProjectEmbeddingStrategyReset = !this.hasProjectForceReembedPending();
|
|
5580
6792
|
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
5581
6793
|
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
5582
6794
|
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
5583
6795
|
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
6796
|
+
if (this.config.scope === "global") {
|
|
6797
|
+
if (completeProjectEmbeddingStrategyReset) {
|
|
6798
|
+
this.database.setMetadata(this.getProjectEmbeddingStrategyMetadataKey(), EMBEDDING_STRATEGY_VERSION);
|
|
6799
|
+
}
|
|
6800
|
+
this.database.setMetadata(this.getLegacyMigrationMetadataKey(), "done");
|
|
6801
|
+
if (completeProjectEmbeddingStrategyReset) {
|
|
6802
|
+
this.database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
6803
|
+
}
|
|
6804
|
+
} else {
|
|
6805
|
+
this.database.setMetadata("index.embeddingStrategyVersion", EMBEDDING_STRATEGY_VERSION);
|
|
6806
|
+
}
|
|
5584
6807
|
this.database.setMetadata("index.updatedAt", now);
|
|
5585
6808
|
if (!existingCreatedAt) {
|
|
5586
6809
|
this.database.setMetadata("index.createdAt", now);
|
|
@@ -5610,6 +6833,14 @@ var Indexer = class {
|
|
|
5610
6833
|
storedMetadata
|
|
5611
6834
|
};
|
|
5612
6835
|
}
|
|
6836
|
+
if (storedMetadata.embeddingStrategyVersion !== EMBEDDING_STRATEGY_VERSION) {
|
|
6837
|
+
return {
|
|
6838
|
+
compatible: false,
|
|
6839
|
+
code: "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */,
|
|
6840
|
+
reason: `Embedding strategy mismatch: index was built with embedding strategy v${storedMetadata.embeddingStrategyVersion}, but the current code requires v${EMBEDDING_STRATEGY_VERSION}. Run index_codebase with force=true to rebuild cached embeddings.`,
|
|
6841
|
+
storedMetadata
|
|
6842
|
+
};
|
|
6843
|
+
}
|
|
5613
6844
|
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
5614
6845
|
this.logger.warn("Provider changed", {
|
|
5615
6846
|
storedProvider: storedMetadata.embeddingProvider,
|
|
@@ -5657,6 +6888,10 @@ var Indexer = class {
|
|
|
5657
6888
|
}
|
|
5658
6889
|
async index(onProgress) {
|
|
5659
6890
|
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
6891
|
+
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
6892
|
+
const branchCatalogKey = this.getBranchCatalogKey();
|
|
6893
|
+
const forceScopedReembed = scopedRoots !== null && database.getMetadata(this.getProjectForceReembedMetadataKey()) === "true";
|
|
6894
|
+
const failedForcedChunkIds = /* @__PURE__ */ new Set();
|
|
5660
6895
|
if (!this.indexCompatibility?.compatible) {
|
|
5661
6896
|
throw new Error(
|
|
5662
6897
|
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
@@ -5678,6 +6913,7 @@ var Indexer = class {
|
|
|
5678
6913
|
skippedFiles: [],
|
|
5679
6914
|
parseFailures: []
|
|
5680
6915
|
};
|
|
6916
|
+
const failedBatchesForCurrentRun = [];
|
|
5681
6917
|
onProgress?.({
|
|
5682
6918
|
phase: "scanning",
|
|
5683
6919
|
filesProcessed: 0,
|
|
@@ -5712,7 +6948,7 @@ var Indexer = class {
|
|
|
5712
6948
|
unchangedFilePaths.add(f.path);
|
|
5713
6949
|
this.logger.recordCacheHit();
|
|
5714
6950
|
} else {
|
|
5715
|
-
const content = await
|
|
6951
|
+
const content = await import_fs7.promises.readFile(f.path, "utf-8");
|
|
5716
6952
|
changedFiles.push({ path: f.path, content, hash: currentHash });
|
|
5717
6953
|
this.logger.recordCacheMiss();
|
|
5718
6954
|
}
|
|
@@ -5737,6 +6973,12 @@ var Indexer = class {
|
|
|
5737
6973
|
const existingChunks = /* @__PURE__ */ new Map();
|
|
5738
6974
|
const existingChunksByFile = /* @__PURE__ */ new Map();
|
|
5739
6975
|
for (const { key, metadata } of store.getAllMetadata()) {
|
|
6976
|
+
if (scopedRoots && !this.isFileInCurrentScope(metadata.filePath, scopedRoots)) {
|
|
6977
|
+
continue;
|
|
6978
|
+
}
|
|
6979
|
+
if (forceScopedReembed && scopedRoots && this.isFileInCurrentScope(metadata.filePath, scopedRoots)) {
|
|
6980
|
+
continue;
|
|
6981
|
+
}
|
|
5740
6982
|
existingChunks.set(key, metadata.hash);
|
|
5741
6983
|
const fileChunks = existingChunksByFile.get(metadata.filePath) || /* @__PURE__ */ new Set();
|
|
5742
6984
|
fileChunks.add(key);
|
|
@@ -5758,7 +7000,7 @@ var Indexer = class {
|
|
|
5758
7000
|
for (const parsed of parsedFiles) {
|
|
5759
7001
|
currentFilePaths.add(parsed.path);
|
|
5760
7002
|
if (parsed.chunks.length === 0) {
|
|
5761
|
-
const relativePath =
|
|
7003
|
+
const relativePath = path8.relative(this.projectRoot, parsed.path);
|
|
5762
7004
|
stats.parseFailures.push(relativePath);
|
|
5763
7005
|
}
|
|
5764
7006
|
let fileChunkCount = 0;
|
|
@@ -5794,7 +7036,10 @@ var Indexer = class {
|
|
|
5794
7036
|
fileChunkCount++;
|
|
5795
7037
|
continue;
|
|
5796
7038
|
}
|
|
5797
|
-
const
|
|
7039
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)).map((text) => ({
|
|
7040
|
+
text,
|
|
7041
|
+
tokenCount: estimateTokens2(text)
|
|
7042
|
+
}));
|
|
5798
7043
|
const metadata = {
|
|
5799
7044
|
filePath: parsed.path,
|
|
5800
7045
|
startLine: chunk.startLine,
|
|
@@ -5804,10 +7049,38 @@ var Indexer = class {
|
|
|
5804
7049
|
language: chunk.language,
|
|
5805
7050
|
hash: contentHash
|
|
5806
7051
|
};
|
|
5807
|
-
pendingChunks.push({
|
|
7052
|
+
pendingChunks.push({
|
|
7053
|
+
id,
|
|
7054
|
+
texts,
|
|
7055
|
+
storageText: createPendingChunkStorageText(texts),
|
|
7056
|
+
content: chunk.content,
|
|
7057
|
+
contentHash,
|
|
7058
|
+
metadata
|
|
7059
|
+
});
|
|
5808
7060
|
fileChunkCount++;
|
|
5809
7061
|
}
|
|
5810
7062
|
}
|
|
7063
|
+
const retryableFailedChunks = this.collectRetryableFailedChunks(
|
|
7064
|
+
currentFileHashes,
|
|
7065
|
+
unchangedFilePaths,
|
|
7066
|
+
getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)
|
|
7067
|
+
);
|
|
7068
|
+
const retryableFailedAttemptCounts = /* @__PURE__ */ new Map();
|
|
7069
|
+
const retryableChunksWithExistingData = /* @__PURE__ */ new Set();
|
|
7070
|
+
if (retryableFailedChunks.length > 0) {
|
|
7071
|
+
const pendingChunkIds = new Set(pendingChunks.map((chunk) => chunk.id));
|
|
7072
|
+
for (const { chunk, attemptCount } of retryableFailedChunks) {
|
|
7073
|
+
retryableFailedAttemptCounts.set(chunk.id, attemptCount);
|
|
7074
|
+
if (existingChunks.has(chunk.id)) {
|
|
7075
|
+
retryableChunksWithExistingData.add(chunk.id);
|
|
7076
|
+
}
|
|
7077
|
+
if (!pendingChunkIds.has(chunk.id)) {
|
|
7078
|
+
pendingChunks.push(chunk);
|
|
7079
|
+
pendingChunkIds.add(chunk.id);
|
|
7080
|
+
currentChunkIds.add(chunk.id);
|
|
7081
|
+
}
|
|
7082
|
+
}
|
|
7083
|
+
}
|
|
5811
7084
|
if (chunkDataBatch.length > 0) {
|
|
5812
7085
|
database.upsertChunksBatch(chunkDataBatch);
|
|
5813
7086
|
}
|
|
@@ -5836,17 +7109,20 @@ var Indexer = class {
|
|
|
5836
7109
|
fileSymbols.push(symbol);
|
|
5837
7110
|
allSymbolIds.add(symbolId);
|
|
5838
7111
|
}
|
|
7112
|
+
const fileLanguage = parsed.chunks[0]?.language;
|
|
7113
|
+
const isCaseInsensitiveLanguage = !!fileLanguage && CASE_INSENSITIVE_LANGUAGES.has(fileLanguage);
|
|
7114
|
+
const normalizeSymbolKey = (name) => isCaseInsensitiveLanguage ? name.toLowerCase() : name;
|
|
5839
7115
|
const symbolsByName = /* @__PURE__ */ new Map();
|
|
5840
7116
|
for (const symbol of fileSymbols) {
|
|
5841
|
-
const
|
|
7117
|
+
const key = normalizeSymbolKey(symbol.name);
|
|
7118
|
+
const existing = symbolsByName.get(key) ?? [];
|
|
5842
7119
|
existing.push(symbol);
|
|
5843
|
-
symbolsByName.set(
|
|
7120
|
+
symbolsByName.set(key, existing);
|
|
5844
7121
|
}
|
|
5845
7122
|
if (fileSymbols.length > 0) {
|
|
5846
7123
|
database.upsertSymbolsBatch(fileSymbols);
|
|
5847
7124
|
symbolsByFile.set(parsed.path, fileSymbols);
|
|
5848
7125
|
}
|
|
5849
|
-
const fileLanguage = parsed.chunks[0]?.language;
|
|
5850
7126
|
if (!fileLanguage || !CALL_GRAPH_LANGUAGES.has(fileLanguage)) continue;
|
|
5851
7127
|
const callSites = extractCalls(changedFile.content, fileLanguage);
|
|
5852
7128
|
if (callSites.length === 0) continue;
|
|
@@ -5871,7 +7147,7 @@ var Indexer = class {
|
|
|
5871
7147
|
if (edges.length > 0) {
|
|
5872
7148
|
database.upsertCallEdgesBatch(edges);
|
|
5873
7149
|
for (const edge of edges) {
|
|
5874
|
-
const candidates = symbolsByName.get(edge.targetName);
|
|
7150
|
+
const candidates = symbolsByName.get(normalizeSymbolKey(edge.targetName));
|
|
5875
7151
|
if (candidates && candidates.length === 1) {
|
|
5876
7152
|
database.resolveCallEdge(edge.id, candidates[0].id);
|
|
5877
7153
|
}
|
|
@@ -5884,14 +7160,20 @@ var Indexer = class {
|
|
|
5884
7160
|
allSymbolIds.add(sym.id);
|
|
5885
7161
|
}
|
|
5886
7162
|
}
|
|
5887
|
-
|
|
7163
|
+
const removedChunkIds = [];
|
|
5888
7164
|
for (const [chunkId] of existingChunks) {
|
|
5889
7165
|
if (!currentChunkIds.has(chunkId)) {
|
|
5890
|
-
|
|
7166
|
+
removedChunkIds.push(chunkId);
|
|
7167
|
+
}
|
|
7168
|
+
}
|
|
7169
|
+
if (removedChunkIds.length > 0) {
|
|
7170
|
+
this.rebuildVectorStoreExcludingChunkIds(store, database, removedChunkIds);
|
|
7171
|
+
for (const chunkId of removedChunkIds) {
|
|
5891
7172
|
invertedIndex.removeChunk(chunkId);
|
|
5892
|
-
removedCount++;
|
|
5893
7173
|
}
|
|
7174
|
+
database.deleteChunksByIds(removedChunkIds);
|
|
5894
7175
|
}
|
|
7176
|
+
const removedCount = removedChunkIds.length;
|
|
5895
7177
|
stats.totalChunks = pendingChunks.length;
|
|
5896
7178
|
stats.existingChunks = currentChunkIds.size - pendingChunks.length;
|
|
5897
7179
|
stats.removedChunks = removedCount;
|
|
@@ -5903,12 +7185,20 @@ var Indexer = class {
|
|
|
5903
7185
|
removed: removedCount
|
|
5904
7186
|
});
|
|
5905
7187
|
if (pendingChunks.length === 0 && removedCount === 0) {
|
|
5906
|
-
database.clearBranch(
|
|
5907
|
-
database.addChunksToBranchBatch(
|
|
5908
|
-
database.clearBranchSymbols(
|
|
5909
|
-
database.addSymbolsToBranchBatch(
|
|
5910
|
-
|
|
5911
|
-
|
|
7188
|
+
database.clearBranch(branchCatalogKey);
|
|
7189
|
+
database.addChunksToBranchBatch(branchCatalogKey, Array.from(currentChunkIds));
|
|
7190
|
+
database.clearBranchSymbols(branchCatalogKey);
|
|
7191
|
+
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
7192
|
+
if (scopedRoots) {
|
|
7193
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7194
|
+
this.clearScopedFailedBatches(scopedRoots);
|
|
7195
|
+
} else {
|
|
7196
|
+
this.fileHashCache = currentFileHashes;
|
|
7197
|
+
this.saveFileHashCache();
|
|
7198
|
+
this.saveFailedBatches([]);
|
|
7199
|
+
}
|
|
7200
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
7201
|
+
this.indexCompatibility = { compatible: true };
|
|
5912
7202
|
stats.durationMs = Date.now() - startTime;
|
|
5913
7203
|
onProgress?.({
|
|
5914
7204
|
phase: "complete",
|
|
@@ -5921,14 +7211,22 @@ var Indexer = class {
|
|
|
5921
7211
|
return stats;
|
|
5922
7212
|
}
|
|
5923
7213
|
if (pendingChunks.length === 0) {
|
|
5924
|
-
database.clearBranch(
|
|
5925
|
-
database.addChunksToBranchBatch(
|
|
5926
|
-
database.clearBranchSymbols(
|
|
5927
|
-
database.addSymbolsToBranchBatch(
|
|
7214
|
+
database.clearBranch(branchCatalogKey);
|
|
7215
|
+
database.addChunksToBranchBatch(branchCatalogKey, Array.from(currentChunkIds));
|
|
7216
|
+
database.clearBranchSymbols(branchCatalogKey);
|
|
7217
|
+
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
5928
7218
|
store.save();
|
|
5929
7219
|
invertedIndex.save();
|
|
5930
|
-
|
|
5931
|
-
|
|
7220
|
+
if (scopedRoots) {
|
|
7221
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7222
|
+
this.clearScopedFailedBatches(scopedRoots);
|
|
7223
|
+
} else {
|
|
7224
|
+
this.fileHashCache = currentFileHashes;
|
|
7225
|
+
this.saveFileHashCache();
|
|
7226
|
+
this.saveFailedBatches([]);
|
|
7227
|
+
}
|
|
7228
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
7229
|
+
this.indexCompatibility = { compatible: true };
|
|
5932
7230
|
stats.durationMs = Date.now() - startTime;
|
|
5933
7231
|
onProgress?.({
|
|
5934
7232
|
phase: "complete",
|
|
@@ -5949,8 +7247,9 @@ var Indexer = class {
|
|
|
5949
7247
|
});
|
|
5950
7248
|
const allContentHashes = pendingChunks.map((c) => c.contentHash);
|
|
5951
7249
|
const missingHashes = new Set(database.getMissingEmbeddings(allContentHashes));
|
|
5952
|
-
const
|
|
5953
|
-
const
|
|
7250
|
+
const forcedReembedChunkIds = forceScopedReembed ? new Set(pendingChunks.map((chunk) => chunk.id)) : /* @__PURE__ */ new Set();
|
|
7251
|
+
const chunksNeedingEmbedding = pendingChunks.filter((c) => forcedReembedChunkIds.has(c.id) || missingHashes.has(c.contentHash));
|
|
7252
|
+
const chunksWithExistingEmbedding = pendingChunks.filter((c) => !forcedReembedChunkIds.has(c.id) && !missingHashes.has(c.contentHash));
|
|
5954
7253
|
this.logger.cache("info", "Embedding cache lookup", {
|
|
5955
7254
|
needsEmbedding: chunksNeedingEmbedding.length,
|
|
5956
7255
|
fromCache: chunksWithExistingEmbedding.length
|
|
@@ -5972,17 +7271,24 @@ var Indexer = class {
|
|
|
5972
7271
|
interval: providerRateLimits.intervalMs,
|
|
5973
7272
|
intervalCap: providerRateLimits.concurrency
|
|
5974
7273
|
});
|
|
5975
|
-
const
|
|
7274
|
+
const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk]));
|
|
7275
|
+
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
7276
|
+
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
7277
|
+
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
7278
|
+
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
7279
|
+
chunksNeedingEmbedding,
|
|
7280
|
+
getDynamicBatchOptions(configuredProviderInfo)
|
|
7281
|
+
);
|
|
5976
7282
|
let rateLimitBackoffMs = 0;
|
|
5977
|
-
for (const
|
|
7283
|
+
for (const requestBatch of requestBatches) {
|
|
5978
7284
|
queue.add(async () => {
|
|
5979
7285
|
if (rateLimitBackoffMs > 0) {
|
|
5980
|
-
await new Promise((
|
|
7286
|
+
await new Promise((resolve8) => setTimeout(resolve8, rateLimitBackoffMs));
|
|
5981
7287
|
}
|
|
5982
7288
|
try {
|
|
5983
7289
|
const result = await pRetry(
|
|
5984
7290
|
async () => {
|
|
5985
|
-
const texts =
|
|
7291
|
+
const texts = requestBatch.map((request) => request.text);
|
|
5986
7292
|
return provider.embedBatch(texts);
|
|
5987
7293
|
},
|
|
5988
7294
|
{
|
|
@@ -6012,29 +7318,82 @@ var Indexer = class {
|
|
|
6012
7318
|
if (rateLimitBackoffMs > 0) {
|
|
6013
7319
|
rateLimitBackoffMs = Math.max(0, rateLimitBackoffMs - 2e3);
|
|
6014
7320
|
}
|
|
6015
|
-
const
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
|
|
6029
|
-
|
|
6030
|
-
|
|
7321
|
+
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
7322
|
+
requestBatch.forEach((request, idx) => {
|
|
7323
|
+
if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
7324
|
+
return;
|
|
7325
|
+
}
|
|
7326
|
+
const vector = result.embeddings[idx];
|
|
7327
|
+
if (!vector) {
|
|
7328
|
+
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
7329
|
+
}
|
|
7330
|
+
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
7331
|
+
parts[request.partIndex] = {
|
|
7332
|
+
vector,
|
|
7333
|
+
tokenCount: request.tokenCount
|
|
7334
|
+
};
|
|
7335
|
+
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
7336
|
+
touchedChunkIds.add(request.chunk.id);
|
|
7337
|
+
});
|
|
7338
|
+
const pooledResults = [];
|
|
7339
|
+
for (const chunkId of touchedChunkIds) {
|
|
7340
|
+
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
7341
|
+
continue;
|
|
7342
|
+
}
|
|
7343
|
+
const chunk = pendingChunksById.get(chunkId);
|
|
7344
|
+
if (!chunk) {
|
|
7345
|
+
continue;
|
|
7346
|
+
}
|
|
7347
|
+
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
7348
|
+
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
7349
|
+
continue;
|
|
7350
|
+
}
|
|
7351
|
+
const orderedParts = parts;
|
|
7352
|
+
pooledResults.push({
|
|
7353
|
+
chunk,
|
|
7354
|
+
vector: poolEmbeddingVectors(
|
|
7355
|
+
orderedParts.map((part) => part.vector),
|
|
7356
|
+
orderedParts.map((part) => part.tokenCount)
|
|
7357
|
+
)
|
|
7358
|
+
});
|
|
7359
|
+
}
|
|
7360
|
+
if (pooledResults.length > 0) {
|
|
7361
|
+
const items = pooledResults.map(({ chunk, vector }) => ({
|
|
7362
|
+
id: chunk.id,
|
|
7363
|
+
vector,
|
|
7364
|
+
metadata: chunk.metadata
|
|
7365
|
+
}));
|
|
7366
|
+
store.addBatch(items);
|
|
7367
|
+
const embeddingBatchItems = pooledResults.map(({ chunk, vector }) => ({
|
|
7368
|
+
contentHash: chunk.contentHash,
|
|
7369
|
+
embedding: float32ArrayToBuffer(vector),
|
|
7370
|
+
chunkText: chunk.storageText,
|
|
7371
|
+
model: configuredProviderInfo.modelInfo.model
|
|
7372
|
+
}));
|
|
7373
|
+
try {
|
|
7374
|
+
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
7375
|
+
} catch (dbError) {
|
|
7376
|
+
this.rebuildVectorStoreExcludingChunkIds(
|
|
7377
|
+
store,
|
|
7378
|
+
database,
|
|
7379
|
+
pooledResults.map(({ chunk }) => chunk.id)
|
|
7380
|
+
);
|
|
7381
|
+
throw dbError;
|
|
7382
|
+
}
|
|
7383
|
+
for (const { chunk } of pooledResults) {
|
|
7384
|
+
invertedIndex.removeChunk(chunk.id);
|
|
7385
|
+
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
7386
|
+
completedChunkIds.add(chunk.id);
|
|
7387
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
7388
|
+
}
|
|
7389
|
+
stats.indexedChunks += pooledResults.length;
|
|
7390
|
+
this.logger.recordChunksEmbedded(pooledResults.length);
|
|
6031
7391
|
}
|
|
6032
|
-
stats.indexedChunks += batch.length;
|
|
6033
7392
|
stats.tokensUsed += result.totalTokensUsed;
|
|
6034
|
-
this.logger.recordChunksEmbedded(batch.length);
|
|
6035
7393
|
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
6036
7394
|
this.logger.embedding("debug", `Embedded batch`, {
|
|
6037
|
-
batchSize:
|
|
7395
|
+
batchSize: pooledResults.length,
|
|
7396
|
+
requestCount: requestBatch.length,
|
|
6038
7397
|
tokens: result.totalTokensUsed
|
|
6039
7398
|
});
|
|
6040
7399
|
onProgress?.({
|
|
@@ -6045,17 +7404,49 @@ var Indexer = class {
|
|
|
6045
7404
|
totalChunks: pendingChunks.length
|
|
6046
7405
|
});
|
|
6047
7406
|
} catch (error) {
|
|
6048
|
-
|
|
6049
|
-
|
|
7407
|
+
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id));
|
|
7408
|
+
const failureMessage = getErrorMessage(error);
|
|
7409
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
7410
|
+
for (const chunk of failedChunks) {
|
|
7411
|
+
if (!failedChunkIds.has(chunk.id)) {
|
|
7412
|
+
failedChunkIds.add(chunk.id);
|
|
7413
|
+
stats.failedChunks += 1;
|
|
7414
|
+
}
|
|
7415
|
+
if (forceScopedReembed) {
|
|
7416
|
+
failedForcedChunkIds.add(chunk.id);
|
|
7417
|
+
}
|
|
7418
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
7419
|
+
const existingFailedBatchIndex = failedBatchesForCurrentRun.findIndex(
|
|
7420
|
+
(failedBatch2) => failedBatch2.chunks[0]?.id === chunk.id
|
|
7421
|
+
);
|
|
7422
|
+
const existingFailedBatch = existingFailedBatchIndex === -1 ? void 0 : failedBatchesForCurrentRun[existingFailedBatchIndex];
|
|
7423
|
+
const failedBatch = {
|
|
7424
|
+
chunks: [chunk],
|
|
7425
|
+
error: failureMessage,
|
|
7426
|
+
attemptCount: (existingFailedBatch?.attemptCount ?? retryableFailedAttemptCounts.get(chunk.id) ?? 0) + 1,
|
|
7427
|
+
lastAttempt: failureTimestamp
|
|
7428
|
+
};
|
|
7429
|
+
if (existingFailedBatchIndex === -1) {
|
|
7430
|
+
failedBatchesForCurrentRun.push(failedBatch);
|
|
7431
|
+
} else {
|
|
7432
|
+
failedBatchesForCurrentRun[existingFailedBatchIndex] = failedBatch;
|
|
7433
|
+
}
|
|
7434
|
+
}
|
|
6050
7435
|
this.logger.recordEmbeddingError();
|
|
6051
7436
|
this.logger.embedding("error", `Failed to embed batch after retries`, {
|
|
6052
|
-
batchSize:
|
|
6053
|
-
|
|
7437
|
+
batchSize: failedChunks.length,
|
|
7438
|
+
requestCount: requestBatch.length,
|
|
7439
|
+
error: failureMessage
|
|
6054
7440
|
});
|
|
6055
7441
|
}
|
|
6056
7442
|
});
|
|
6057
7443
|
}
|
|
6058
7444
|
await queue.onIdle();
|
|
7445
|
+
if (scopedRoots) {
|
|
7446
|
+
this.saveScopedFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun), scopedRoots);
|
|
7447
|
+
} else {
|
|
7448
|
+
this.saveFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun));
|
|
7449
|
+
}
|
|
6059
7450
|
onProgress?.({
|
|
6060
7451
|
phase: "storing",
|
|
6061
7452
|
filesProcessed: files.length,
|
|
@@ -6063,18 +7454,48 @@ var Indexer = class {
|
|
|
6063
7454
|
chunksProcessed: stats.indexedChunks,
|
|
6064
7455
|
totalChunks: pendingChunks.length
|
|
6065
7456
|
});
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
7457
|
+
const branchChunkIds = Array.from(currentChunkIds).filter(
|
|
7458
|
+
(chunkId) => {
|
|
7459
|
+
const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId);
|
|
7460
|
+
const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId);
|
|
7461
|
+
return !isNewlyFailed && !isForcedFailed;
|
|
7462
|
+
}
|
|
7463
|
+
);
|
|
7464
|
+
database.clearBranch(branchCatalogKey);
|
|
7465
|
+
database.addChunksToBranchBatch(branchCatalogKey, branchChunkIds);
|
|
7466
|
+
database.clearBranchSymbols(branchCatalogKey);
|
|
7467
|
+
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
6070
7468
|
store.save();
|
|
6071
7469
|
invertedIndex.save();
|
|
6072
|
-
|
|
6073
|
-
|
|
7470
|
+
if (scopedRoots) {
|
|
7471
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7472
|
+
} else {
|
|
7473
|
+
this.fileHashCache = currentFileHashes;
|
|
7474
|
+
this.saveFileHashCache();
|
|
7475
|
+
}
|
|
6074
7476
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
6075
|
-
await this.maybeRunOrphanGc();
|
|
7477
|
+
const gcReset = await this.maybeRunOrphanGc();
|
|
7478
|
+
if (gcReset) {
|
|
7479
|
+
stats.durationMs = Date.now() - startTime;
|
|
7480
|
+
stats.warning = gcReset.warning;
|
|
7481
|
+
stats.resetCorruptedIndex = true;
|
|
7482
|
+
this.logger.recordIndexingEnd();
|
|
7483
|
+
this.logger.warn("Indexing ended after resetting corrupted local index during automatic GC", {
|
|
7484
|
+
files: stats.totalFiles,
|
|
7485
|
+
indexed: stats.indexedChunks,
|
|
7486
|
+
existing: stats.existingChunks,
|
|
7487
|
+
removed: stats.removedChunks,
|
|
7488
|
+
failed: stats.failedChunks,
|
|
7489
|
+
tokens: stats.tokensUsed,
|
|
7490
|
+
durationMs: stats.durationMs
|
|
7491
|
+
});
|
|
7492
|
+
return stats;
|
|
7493
|
+
}
|
|
6076
7494
|
}
|
|
6077
7495
|
stats.durationMs = Date.now() - startTime;
|
|
7496
|
+
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
7497
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
7498
|
+
}
|
|
6078
7499
|
this.saveIndexMetadata(configuredProviderInfo);
|
|
6079
7500
|
this.indexCompatibility = { compatible: true };
|
|
6080
7501
|
this.logger.recordIndexingEnd();
|
|
@@ -6203,27 +7624,30 @@ var Indexer = class {
|
|
|
6203
7624
|
const keywordResults = await this.keywordSearch(query, maxResults * 4);
|
|
6204
7625
|
const keywordMs = import_perf_hooks.performance.now() - keywordStartTime;
|
|
6205
7626
|
let branchChunkIds = null;
|
|
6206
|
-
if (filterByBranch && this.currentBranch !== "default") {
|
|
6207
|
-
branchChunkIds = new Set(
|
|
7627
|
+
if (filterByBranch && (this.config.scope === "global" || this.currentBranch !== "default")) {
|
|
7628
|
+
branchChunkIds = new Set(
|
|
7629
|
+
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
7630
|
+
);
|
|
6208
7631
|
}
|
|
6209
7632
|
const prefilterStartTime = import_perf_hooks.performance.now();
|
|
6210
|
-
const shouldPrefilterByBranch = branchChunkIds !== null && branchChunkIds.size > 0;
|
|
7633
|
+
const shouldPrefilterByBranch = branchChunkIds !== null && (this.config.scope === "global" || branchChunkIds.size > 0);
|
|
7634
|
+
const allowBranchPrefilterFallback = this.config.scope !== "global";
|
|
6211
7635
|
const prefilteredSemantic = shouldPrefilterByBranch && branchChunkIds ? semanticResults.filter((r) => branchChunkIds.has(r.id)) : semanticResults;
|
|
6212
7636
|
const prefilteredKeyword = shouldPrefilterByBranch && branchChunkIds ? keywordResults.filter((r) => branchChunkIds.has(r.id)) : keywordResults;
|
|
6213
|
-
const semanticCandidates = shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
6214
|
-
const keywordCandidates = shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0 ? keywordResults : prefilteredKeyword;
|
|
7637
|
+
const semanticCandidates = allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
7638
|
+
const keywordCandidates = allowBranchPrefilterFallback && shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0 ? keywordResults : prefilteredKeyword;
|
|
6215
7639
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
6216
|
-
if (branchChunkIds && branchChunkIds.size === 0) {
|
|
7640
|
+
if (this.config.scope !== "global" && branchChunkIds && branchChunkIds.size === 0) {
|
|
6217
7641
|
this.logger.search("warn", "Branch prefilter skipped because branch catalog is empty", {
|
|
6218
7642
|
branch: this.currentBranch
|
|
6219
7643
|
});
|
|
6220
7644
|
}
|
|
6221
|
-
if (shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
7645
|
+
if (allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
6222
7646
|
this.logger.search("warn", "Branch prefilter produced no semantic overlap, using unfiltered semantic candidates", {
|
|
6223
7647
|
branch: this.currentBranch
|
|
6224
7648
|
});
|
|
6225
7649
|
}
|
|
6226
|
-
if (shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0) {
|
|
7650
|
+
if (allowBranchPrefilterFallback && shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0) {
|
|
6227
7651
|
this.logger.search("warn", "Branch prefilter produced no keyword overlap, using unfiltered keyword candidates", {
|
|
6228
7652
|
branch: this.currentBranch
|
|
6229
7653
|
});
|
|
@@ -6276,26 +7700,13 @@ var Indexer = class {
|
|
|
6276
7700
|
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
6277
7701
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
6278
7702
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
6279
|
-
const baseFiltered = tiered.filter((r) =>
|
|
6280
|
-
if (r.score < this.config.search.minScore) return false;
|
|
6281
|
-
if (options?.fileType) {
|
|
6282
|
-
const ext = r.metadata.filePath.split(".").pop()?.toLowerCase();
|
|
6283
|
-
if (ext !== options.fileType.toLowerCase().replace(/^\./, "")) return false;
|
|
6284
|
-
}
|
|
6285
|
-
if (options?.directory) {
|
|
6286
|
-
const normalizedDir = options.directory.replace(/^\/|\/$/g, "");
|
|
6287
|
-
if (!r.metadata.filePath.includes(`/${normalizedDir}/`) && !r.metadata.filePath.includes(`${normalizedDir}/`)) return false;
|
|
6288
|
-
}
|
|
6289
|
-
if (options?.chunkType) {
|
|
6290
|
-
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
6291
|
-
}
|
|
6292
|
-
return true;
|
|
6293
|
-
});
|
|
7703
|
+
const baseFiltered = tiered.filter((r) => matchesSearchFilters(r, options, this.config.search.minScore));
|
|
6294
7704
|
const implementationOnly = baseFiltered.filter(
|
|
6295
7705
|
(r) => isLikelyImplementationPath(r.metadata.filePath) && isImplementationChunkType(r.metadata.chunkType)
|
|
6296
7706
|
);
|
|
6297
7707
|
const filtered = (sourceIntent && hasCodeHints && implementationOnly.length > 0 ? implementationOnly : baseFiltered).slice(0, maxResults);
|
|
6298
|
-
const
|
|
7708
|
+
const identifierFallback = !options?.definitionIntent && filtered.length === 0 && identifierHints.length > 0 ? buildSymbolDefinitionLane(query, database, branchChunkIds, maxResults, union, true).filter((r) => matchesSearchFilters(r, options, this.config.search.minScore)).slice(0, maxResults) : [];
|
|
7709
|
+
const finalResults = filtered.length > 0 ? filtered : identifierFallback;
|
|
6299
7710
|
const totalSearchMs = import_perf_hooks.performance.now() - searchStartTime;
|
|
6300
7711
|
this.logger.recordSearch(totalSearchMs, {
|
|
6301
7712
|
embeddingMs,
|
|
@@ -6321,7 +7732,7 @@ var Indexer = class {
|
|
|
6321
7732
|
let contextEndLine = r.metadata.endLine;
|
|
6322
7733
|
if (!metadataOnly && this.config.search.includeContext) {
|
|
6323
7734
|
try {
|
|
6324
|
-
const fileContent = await
|
|
7735
|
+
const fileContent = await import_fs7.promises.readFile(
|
|
6325
7736
|
r.metadata.filePath,
|
|
6326
7737
|
"utf-8"
|
|
6327
7738
|
);
|
|
@@ -6365,7 +7776,8 @@ var Indexer = class {
|
|
|
6365
7776
|
return results.slice(0, limit);
|
|
6366
7777
|
}
|
|
6367
7778
|
async getStatus() {
|
|
6368
|
-
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
7779
|
+
const { store, configuredProviderInfo, database } = await this.ensureInitialized();
|
|
7780
|
+
const failedBatchesCount = this.getFailedBatchesCount();
|
|
6369
7781
|
return {
|
|
6370
7782
|
indexed: store.count() > 0,
|
|
6371
7783
|
vectorCount: store.count(),
|
|
@@ -6374,23 +7786,86 @@ var Indexer = class {
|
|
|
6374
7786
|
indexPath: this.indexPath,
|
|
6375
7787
|
currentBranch: this.currentBranch,
|
|
6376
7788
|
baseBranch: this.baseBranch,
|
|
6377
|
-
compatibility: this.indexCompatibility
|
|
7789
|
+
compatibility: this.indexCompatibility,
|
|
7790
|
+
failedBatchesCount,
|
|
7791
|
+
failedBatchesPath: failedBatchesCount > 0 ? this.failedBatchesPath : void 0,
|
|
7792
|
+
warning: database.getMetadata(STARTUP_WARNING_METADATA_KEY) ?? void 0
|
|
6378
7793
|
};
|
|
6379
7794
|
}
|
|
6380
7795
|
async clearIndex() {
|
|
6381
7796
|
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
7797
|
+
if (this.config.scope === "global") {
|
|
7798
|
+
store.load();
|
|
7799
|
+
invertedIndex.load();
|
|
7800
|
+
this.loadFileHashCache();
|
|
7801
|
+
const roots = this.getScopedRoots();
|
|
7802
|
+
const compatibility = this.checkCompatibility();
|
|
7803
|
+
const allMetadata = store.getAllMetadata();
|
|
7804
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData() || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
7805
|
+
if (!compatibility.compatible && hasForeignData) {
|
|
7806
|
+
if (compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */) {
|
|
7807
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots);
|
|
7808
|
+
this.clearScopedFileHashCache(roots);
|
|
7809
|
+
this.clearScopedFailedBatches(roots);
|
|
7810
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(), "true");
|
|
7811
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
7812
|
+
this.indexCompatibility = { compatible: true };
|
|
7813
|
+
return;
|
|
7814
|
+
}
|
|
7815
|
+
throw new Error(
|
|
7816
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
7817
|
+
);
|
|
7818
|
+
}
|
|
7819
|
+
if (!hasForeignData) {
|
|
7820
|
+
store.clear();
|
|
7821
|
+
store.save();
|
|
7822
|
+
invertedIndex.clear();
|
|
7823
|
+
invertedIndex.save();
|
|
7824
|
+
this.fileHashCache.clear();
|
|
7825
|
+
this.saveFileHashCache();
|
|
7826
|
+
database.clearAllIndexedData();
|
|
7827
|
+
this.saveFailedBatches([]);
|
|
7828
|
+
database.deleteMetadata("index.version");
|
|
7829
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
7830
|
+
database.deleteMetadata("index.embeddingModel");
|
|
7831
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
7832
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
7833
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
7834
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
7835
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
7836
|
+
database.deleteMetadata("index.createdAt");
|
|
7837
|
+
database.deleteMetadata("index.updatedAt");
|
|
7838
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
7839
|
+
return;
|
|
7840
|
+
}
|
|
7841
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots);
|
|
7842
|
+
this.clearScopedFileHashCache(roots);
|
|
7843
|
+
this.clearScopedFailedBatches(roots);
|
|
7844
|
+
this.indexCompatibility = compatibility;
|
|
7845
|
+
return;
|
|
7846
|
+
}
|
|
7847
|
+
const localProjectIndexPath = path8.join(this.projectRoot, ".opencode", "index");
|
|
7848
|
+
if (path8.resolve(this.indexPath) !== path8.resolve(localProjectIndexPath)) {
|
|
7849
|
+
throw new Error(
|
|
7850
|
+
"Project-scoped force rebuild is unsafe while using an inherited worktree index. Create a local project config boundary before clearing the index."
|
|
7851
|
+
);
|
|
7852
|
+
}
|
|
6382
7853
|
store.clear();
|
|
6383
7854
|
store.save();
|
|
6384
7855
|
invertedIndex.clear();
|
|
6385
7856
|
invertedIndex.save();
|
|
6386
7857
|
this.fileHashCache.clear();
|
|
6387
7858
|
this.saveFileHashCache();
|
|
6388
|
-
database.
|
|
6389
|
-
|
|
7859
|
+
database.clearAllIndexedData();
|
|
7860
|
+
this.saveFailedBatches([]);
|
|
6390
7861
|
database.deleteMetadata("index.version");
|
|
6391
7862
|
database.deleteMetadata("index.embeddingProvider");
|
|
6392
7863
|
database.deleteMetadata("index.embeddingModel");
|
|
6393
7864
|
database.deleteMetadata("index.embeddingDimensions");
|
|
7865
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
7866
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
7867
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
7868
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
6394
7869
|
database.deleteMetadata("index.createdAt");
|
|
6395
7870
|
database.deleteMetadata("index.updatedAt");
|
|
6396
7871
|
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
@@ -6406,28 +7881,61 @@ var Indexer = class {
|
|
|
6406
7881
|
filePathsToChunkKeys.set(metadata.filePath, existing);
|
|
6407
7882
|
}
|
|
6408
7883
|
const removedFilePaths = [];
|
|
6409
|
-
|
|
7884
|
+
const removedChunkKeys = [];
|
|
7885
|
+
const chunkKeysByRemovedFile = /* @__PURE__ */ new Map();
|
|
6410
7886
|
for (const [filePath, chunkKeys] of filePathsToChunkKeys) {
|
|
6411
|
-
if (!(0,
|
|
7887
|
+
if (!(0, import_fs7.existsSync)(filePath)) {
|
|
7888
|
+
chunkKeysByRemovedFile.set(filePath, chunkKeys);
|
|
6412
7889
|
for (const key of chunkKeys) {
|
|
6413
|
-
|
|
6414
|
-
invertedIndex.removeChunk(key);
|
|
6415
|
-
removedCount++;
|
|
7890
|
+
removedChunkKeys.push(key);
|
|
6416
7891
|
}
|
|
6417
|
-
database.deleteChunksByFile(filePath);
|
|
6418
|
-
database.deleteCallEdgesByFile(filePath);
|
|
6419
|
-
database.deleteSymbolsByFile(filePath);
|
|
6420
7892
|
removedFilePaths.push(filePath);
|
|
6421
7893
|
}
|
|
6422
7894
|
}
|
|
7895
|
+
if (removedChunkKeys.length > 0) {
|
|
7896
|
+
this.rebuildVectorStoreExcludingChunkIds(store, database, removedChunkKeys);
|
|
7897
|
+
for (const key of removedChunkKeys) {
|
|
7898
|
+
invertedIndex.removeChunk(key);
|
|
7899
|
+
}
|
|
7900
|
+
}
|
|
7901
|
+
for (const filePath of removedFilePaths) {
|
|
7902
|
+
const fileChunkKeys = chunkKeysByRemovedFile.get(filePath) ?? [];
|
|
7903
|
+
if (fileChunkKeys.length > 0) {
|
|
7904
|
+
database.deleteChunksByIds(fileChunkKeys);
|
|
7905
|
+
}
|
|
7906
|
+
database.deleteCallEdgesByFile(filePath);
|
|
7907
|
+
database.deleteSymbolsByFile(filePath);
|
|
7908
|
+
}
|
|
7909
|
+
const removedCount = removedChunkKeys.length;
|
|
6423
7910
|
if (removedCount > 0) {
|
|
6424
7911
|
store.save();
|
|
6425
7912
|
invertedIndex.save();
|
|
6426
7913
|
}
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
|
|
7914
|
+
let gcOrphanEmbeddings;
|
|
7915
|
+
let gcOrphanChunks;
|
|
7916
|
+
let gcOrphanSymbols;
|
|
7917
|
+
let gcOrphanCallEdges;
|
|
7918
|
+
try {
|
|
7919
|
+
gcOrphanEmbeddings = database.gcOrphanEmbeddings();
|
|
7920
|
+
gcOrphanChunks = database.gcOrphanChunks();
|
|
7921
|
+
gcOrphanSymbols = database.gcOrphanSymbols();
|
|
7922
|
+
gcOrphanCallEdges = database.gcOrphanCallEdges();
|
|
7923
|
+
} catch (error) {
|
|
7924
|
+
if (!await this.tryResetCorruptedIndex("running index health check", error)) {
|
|
7925
|
+
throw error;
|
|
7926
|
+
}
|
|
7927
|
+
await this.ensureInitialized();
|
|
7928
|
+
return {
|
|
7929
|
+
removed: 0,
|
|
7930
|
+
filePaths: [],
|
|
7931
|
+
gcOrphanEmbeddings: 0,
|
|
7932
|
+
gcOrphanChunks: 0,
|
|
7933
|
+
gcOrphanSymbols: 0,
|
|
7934
|
+
gcOrphanCallEdges: 0,
|
|
7935
|
+
resetCorruptedIndex: true,
|
|
7936
|
+
warning: this.getCorruptedIndexWarning(path8.join(this.indexPath, "codebase.db"))
|
|
7937
|
+
};
|
|
7938
|
+
}
|
|
6431
7939
|
this.logger.recordGc(removedCount, gcOrphanChunks, gcOrphanEmbeddings);
|
|
6432
7940
|
this.logger.gc("info", "Health check complete", {
|
|
6433
7941
|
removedStale: removedCount,
|
|
@@ -6438,8 +7946,12 @@ var Indexer = class {
|
|
|
6438
7946
|
return { removed: removedCount, filePaths: removedFilePaths, gcOrphanEmbeddings, gcOrphanChunks, gcOrphanSymbols, gcOrphanCallEdges };
|
|
6439
7947
|
}
|
|
6440
7948
|
async retryFailedBatches() {
|
|
6441
|
-
const { store, provider, invertedIndex } = await this.ensureInitialized();
|
|
6442
|
-
const
|
|
7949
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
7950
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
7951
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
7952
|
+
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
7953
|
+
const { scoped: scopedFailedBatches, retained: retainedFailedBatches } = roots ? this.partitionFailedBatches(roots, maxChunkTokens) : { scoped: this.loadFailedBatches(maxChunkTokens), retained: [] };
|
|
7954
|
+
const failedBatches = scopedFailedBatches;
|
|
6443
7955
|
if (failedBatches.length === 0) {
|
|
6444
7956
|
return { succeeded: 0, failed: 0, remaining: 0 };
|
|
6445
7957
|
}
|
|
@@ -6447,49 +7959,170 @@ var Indexer = class {
|
|
|
6447
7959
|
let failed = 0;
|
|
6448
7960
|
const stillFailing = [];
|
|
6449
7961
|
for (const batch of failedBatches) {
|
|
7962
|
+
const batchChunksById = new Map(batch.chunks.map((chunk) => [chunk.id, chunk]));
|
|
7963
|
+
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
7964
|
+
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
7965
|
+
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
7966
|
+
const failedChunksForBatch = /* @__PURE__ */ new Map();
|
|
7967
|
+
const pooledResults = [];
|
|
6450
7968
|
try {
|
|
6451
|
-
const
|
|
6452
|
-
|
|
6453
|
-
|
|
6454
|
-
return provider.embedBatch(texts);
|
|
6455
|
-
},
|
|
6456
|
-
{
|
|
6457
|
-
retries: this.config.indexing.retries,
|
|
6458
|
-
minTimeout: this.config.indexing.retryDelayMs
|
|
6459
|
-
}
|
|
7969
|
+
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
7970
|
+
batch.chunks,
|
|
7971
|
+
getDynamicBatchOptions(configuredProviderInfo)
|
|
6460
7972
|
);
|
|
6461
|
-
const
|
|
7973
|
+
for (const requestBatch of requestBatches) {
|
|
7974
|
+
try {
|
|
7975
|
+
const result = await pRetry(
|
|
7976
|
+
async () => {
|
|
7977
|
+
const texts = requestBatch.map((request) => request.text);
|
|
7978
|
+
return provider.embedBatch(texts);
|
|
7979
|
+
},
|
|
7980
|
+
{
|
|
7981
|
+
retries: this.config.indexing.retries,
|
|
7982
|
+
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
7983
|
+
maxTimeout: providerRateLimits.maxRetryMs,
|
|
7984
|
+
factor: 2,
|
|
7985
|
+
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError)
|
|
7986
|
+
}
|
|
7987
|
+
);
|
|
7988
|
+
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
7989
|
+
requestBatch.forEach((request, idx) => {
|
|
7990
|
+
if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
7991
|
+
return;
|
|
7992
|
+
}
|
|
7993
|
+
const vector = result.embeddings[idx];
|
|
7994
|
+
if (!vector) {
|
|
7995
|
+
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
7996
|
+
}
|
|
7997
|
+
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
7998
|
+
parts[request.partIndex] = {
|
|
7999
|
+
vector,
|
|
8000
|
+
tokenCount: request.tokenCount
|
|
8001
|
+
};
|
|
8002
|
+
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
8003
|
+
touchedChunkIds.add(request.chunk.id);
|
|
8004
|
+
});
|
|
8005
|
+
for (const chunkId of touchedChunkIds) {
|
|
8006
|
+
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
8007
|
+
continue;
|
|
8008
|
+
}
|
|
8009
|
+
const chunk = batchChunksById.get(chunkId);
|
|
8010
|
+
if (!chunk) {
|
|
8011
|
+
continue;
|
|
8012
|
+
}
|
|
8013
|
+
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
8014
|
+
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
8015
|
+
continue;
|
|
8016
|
+
}
|
|
8017
|
+
const orderedParts = parts;
|
|
8018
|
+
pooledResults.push({
|
|
8019
|
+
chunk,
|
|
8020
|
+
vector: poolEmbeddingVectors(
|
|
8021
|
+
orderedParts.map((part) => part.vector),
|
|
8022
|
+
orderedParts.map((part) => part.tokenCount)
|
|
8023
|
+
)
|
|
8024
|
+
});
|
|
8025
|
+
}
|
|
8026
|
+
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
8027
|
+
} catch (error) {
|
|
8028
|
+
const failureMessage = String(error);
|
|
8029
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
8030
|
+
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id) && !failedChunkIds.has(chunk.id));
|
|
8031
|
+
for (const chunk of failedChunks) {
|
|
8032
|
+
failedChunkIds.add(chunk.id);
|
|
8033
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
8034
|
+
failedChunksForBatch.set(chunk.id, {
|
|
8035
|
+
chunks: [chunk],
|
|
8036
|
+
attemptCount: batch.attemptCount + 1,
|
|
8037
|
+
lastAttempt: failureTimestamp,
|
|
8038
|
+
error: failureMessage
|
|
8039
|
+
});
|
|
8040
|
+
}
|
|
8041
|
+
failed += failedChunks.length;
|
|
8042
|
+
this.logger.recordEmbeddingError();
|
|
8043
|
+
}
|
|
8044
|
+
}
|
|
8045
|
+
const successfulResults = pooledResults.filter(({ chunk }) => !failedChunkIds.has(chunk.id));
|
|
8046
|
+
const items = successfulResults.map(({ chunk, vector }) => ({
|
|
6462
8047
|
id: chunk.id,
|
|
6463
|
-
vector
|
|
8048
|
+
vector,
|
|
6464
8049
|
metadata: chunk.metadata
|
|
6465
8050
|
}));
|
|
6466
|
-
|
|
6467
|
-
|
|
8051
|
+
if (items.length > 0) {
|
|
8052
|
+
store.addBatch(items);
|
|
8053
|
+
}
|
|
8054
|
+
if (successfulResults.length > 0) {
|
|
8055
|
+
try {
|
|
8056
|
+
database.upsertEmbeddingsBatch(
|
|
8057
|
+
successfulResults.map(({ chunk, vector }) => ({
|
|
8058
|
+
contentHash: chunk.contentHash,
|
|
8059
|
+
embedding: float32ArrayToBuffer(vector),
|
|
8060
|
+
chunkText: chunk.storageText,
|
|
8061
|
+
model: configuredProviderInfo.modelInfo.model
|
|
8062
|
+
}))
|
|
8063
|
+
);
|
|
8064
|
+
} catch (dbError) {
|
|
8065
|
+
this.rebuildVectorStoreExcludingChunkIds(
|
|
8066
|
+
store,
|
|
8067
|
+
database,
|
|
8068
|
+
successfulResults.map(({ chunk }) => chunk.id)
|
|
8069
|
+
);
|
|
8070
|
+
throw dbError;
|
|
8071
|
+
}
|
|
8072
|
+
}
|
|
8073
|
+
for (const { chunk } of successfulResults) {
|
|
6468
8074
|
invertedIndex.removeChunk(chunk.id);
|
|
6469
8075
|
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
8076
|
+
completedChunkIds.add(chunk.id);
|
|
8077
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
6470
8078
|
}
|
|
6471
|
-
|
|
6472
|
-
|
|
6473
|
-
|
|
8079
|
+
database.addChunksToBranchBatch(
|
|
8080
|
+
this.getBranchCatalogKey(),
|
|
8081
|
+
successfulResults.map(({ chunk }) => chunk.id)
|
|
8082
|
+
);
|
|
8083
|
+
this.logger.recordChunksEmbedded(successfulResults.length);
|
|
8084
|
+
succeeded += successfulResults.length;
|
|
8085
|
+
stillFailing.push(...failedChunksForBatch.values());
|
|
6474
8086
|
} catch (error) {
|
|
6475
|
-
|
|
8087
|
+
const failureMessage = getErrorMessage(error);
|
|
8088
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
8089
|
+
const unaccountedChunks = batch.chunks.filter(
|
|
8090
|
+
(chunk) => !failedChunksForBatch.has(chunk.id) && !completedChunkIds.has(chunk.id)
|
|
8091
|
+
);
|
|
8092
|
+
for (const chunk of unaccountedChunks) {
|
|
8093
|
+
failedChunksForBatch.set(chunk.id, {
|
|
8094
|
+
chunks: [chunk],
|
|
8095
|
+
attemptCount: batch.attemptCount + 1,
|
|
8096
|
+
lastAttempt: failureTimestamp,
|
|
8097
|
+
error: failureMessage
|
|
8098
|
+
});
|
|
8099
|
+
}
|
|
8100
|
+
failed += unaccountedChunks.length;
|
|
6476
8101
|
this.logger.recordEmbeddingError();
|
|
6477
|
-
stillFailing.push(
|
|
6478
|
-
...batch,
|
|
6479
|
-
attemptCount: batch.attemptCount + 1,
|
|
6480
|
-
lastAttempt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6481
|
-
error: String(error)
|
|
6482
|
-
});
|
|
8102
|
+
stillFailing.push(...coalesceFailedBatches(Array.from(failedChunksForBatch.values())));
|
|
6483
8103
|
}
|
|
6484
8104
|
}
|
|
6485
|
-
|
|
8105
|
+
const persistedStillFailing = coalesceFailedBatches(stillFailing);
|
|
8106
|
+
if (roots) {
|
|
8107
|
+
this.saveFailedBatches([...retainedFailedBatches, ...persistedStillFailing]);
|
|
8108
|
+
} else {
|
|
8109
|
+
this.saveFailedBatches(persistedStillFailing);
|
|
8110
|
+
}
|
|
6486
8111
|
if (succeeded > 0) {
|
|
6487
8112
|
store.save();
|
|
6488
8113
|
invertedIndex.save();
|
|
6489
8114
|
}
|
|
6490
|
-
|
|
8115
|
+
if (roots && succeeded > 0 && persistedStillFailing.length === 0 && this.hasProjectForceReembedPending()) {
|
|
8116
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
8117
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
8118
|
+
this.indexCompatibility = { compatible: true };
|
|
8119
|
+
}
|
|
8120
|
+
return { succeeded, failed, remaining: persistedStillFailing.length };
|
|
6491
8121
|
}
|
|
6492
8122
|
getFailedBatchesCount() {
|
|
8123
|
+
if (this.config.scope === "global") {
|
|
8124
|
+
return this.partitionFailedBatches(this.getScopedRoots()).scoped.length;
|
|
8125
|
+
}
|
|
6493
8126
|
return this.loadFailedBatches().length;
|
|
6494
8127
|
}
|
|
6495
8128
|
getCurrentBranch() {
|
|
@@ -6538,20 +8171,23 @@ var Indexer = class {
|
|
|
6538
8171
|
const semanticResults = store.search(embedding, limit * 2);
|
|
6539
8172
|
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
6540
8173
|
let branchChunkIds = null;
|
|
6541
|
-
if (filterByBranch && this.currentBranch !== "default") {
|
|
6542
|
-
branchChunkIds = new Set(
|
|
8174
|
+
if (filterByBranch && (this.config.scope === "global" || this.currentBranch !== "default")) {
|
|
8175
|
+
branchChunkIds = new Set(
|
|
8176
|
+
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
8177
|
+
);
|
|
6543
8178
|
}
|
|
6544
8179
|
const prefilterStartTime = import_perf_hooks.performance.now();
|
|
6545
|
-
const shouldPrefilterByBranch = branchChunkIds !== null && branchChunkIds.size > 0;
|
|
8180
|
+
const shouldPrefilterByBranch = branchChunkIds !== null && (this.config.scope === "global" || branchChunkIds.size > 0);
|
|
8181
|
+
const allowBranchPrefilterFallback = this.config.scope !== "global";
|
|
6546
8182
|
const prefilteredSemantic = shouldPrefilterByBranch && branchChunkIds ? semanticResults.filter((r) => branchChunkIds.has(r.id)) : semanticResults;
|
|
6547
|
-
const semanticCandidates = shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
8183
|
+
const semanticCandidates = allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
6548
8184
|
const prefilterMs = import_perf_hooks.performance.now() - prefilterStartTime;
|
|
6549
|
-
if (branchChunkIds && branchChunkIds.size === 0) {
|
|
8185
|
+
if (this.config.scope !== "global" && branchChunkIds && branchChunkIds.size === 0) {
|
|
6550
8186
|
this.logger.search("warn", "Branch prefilter skipped because branch catalog is empty", {
|
|
6551
8187
|
branch: this.currentBranch
|
|
6552
8188
|
});
|
|
6553
8189
|
}
|
|
6554
|
-
if (shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
8190
|
+
if (allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
6555
8191
|
this.logger.search("warn", "Branch prefilter produced no semantic overlap, using unfiltered semantic candidates", {
|
|
6556
8192
|
branch: this.currentBranch
|
|
6557
8193
|
});
|
|
@@ -6600,7 +8236,7 @@ var Indexer = class {
|
|
|
6600
8236
|
let content = "";
|
|
6601
8237
|
if (this.config.search.includeContext) {
|
|
6602
8238
|
try {
|
|
6603
|
-
const fileContent = await
|
|
8239
|
+
const fileContent = await import_fs7.promises.readFile(
|
|
6604
8240
|
r.metadata.filePath,
|
|
6605
8241
|
"utf-8"
|
|
6606
8242
|
);
|
|
@@ -6624,11 +8260,39 @@ var Indexer = class {
|
|
|
6624
8260
|
}
|
|
6625
8261
|
async getCallers(targetName) {
|
|
6626
8262
|
const { database } = await this.ensureInitialized();
|
|
6627
|
-
|
|
8263
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8264
|
+
const results = [];
|
|
8265
|
+
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
8266
|
+
for (const edge of database.getCallersWithContext(targetName, branchKey)) {
|
|
8267
|
+
if (!seen.has(edge.id)) {
|
|
8268
|
+
seen.add(edge.id);
|
|
8269
|
+
results.push(edge);
|
|
8270
|
+
}
|
|
8271
|
+
}
|
|
8272
|
+
}
|
|
8273
|
+
return results;
|
|
6628
8274
|
}
|
|
6629
8275
|
async getCallees(symbolId) {
|
|
6630
8276
|
const { database } = await this.ensureInitialized();
|
|
6631
|
-
|
|
8277
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8278
|
+
const results = [];
|
|
8279
|
+
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
8280
|
+
for (const edge of database.getCallees(symbolId, branchKey)) {
|
|
8281
|
+
if (!seen.has(edge.id)) {
|
|
8282
|
+
seen.add(edge.id);
|
|
8283
|
+
results.push(edge);
|
|
8284
|
+
}
|
|
8285
|
+
}
|
|
8286
|
+
}
|
|
8287
|
+
return results;
|
|
8288
|
+
}
|
|
8289
|
+
async close() {
|
|
8290
|
+
await this.database?.close();
|
|
8291
|
+
this.database = null;
|
|
8292
|
+
this.store = null;
|
|
8293
|
+
this.invertedIndex = null;
|
|
8294
|
+
this.provider = null;
|
|
8295
|
+
this.reranker = null;
|
|
6632
8296
|
}
|
|
6633
8297
|
};
|
|
6634
8298
|
|
|
@@ -6879,9 +8543,9 @@ function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingToke
|
|
|
6879
8543
|
}
|
|
6880
8544
|
|
|
6881
8545
|
// src/eval/schema.ts
|
|
6882
|
-
var
|
|
8546
|
+
var import_fs8 = require("fs");
|
|
6883
8547
|
function parseJsonFile(filePath) {
|
|
6884
|
-
const content = (0,
|
|
8548
|
+
const content = (0, import_fs8.readFileSync)(filePath, "utf-8");
|
|
6885
8549
|
return JSON.parse(content);
|
|
6886
8550
|
}
|
|
6887
8551
|
function isRecord(value) {
|
|
@@ -6890,23 +8554,23 @@ function isRecord(value) {
|
|
|
6890
8554
|
function isStringArray2(value) {
|
|
6891
8555
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
6892
8556
|
}
|
|
6893
|
-
function asPositiveNumber(value,
|
|
8557
|
+
function asPositiveNumber(value, path13) {
|
|
6894
8558
|
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
6895
|
-
throw new Error(`${
|
|
8559
|
+
throw new Error(`${path13} must be a non-negative number`);
|
|
6896
8560
|
}
|
|
6897
8561
|
return value;
|
|
6898
8562
|
}
|
|
6899
|
-
function parseQueryType(value,
|
|
8563
|
+
function parseQueryType(value, path13) {
|
|
6900
8564
|
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy") {
|
|
6901
8565
|
return value;
|
|
6902
8566
|
}
|
|
6903
8567
|
throw new Error(
|
|
6904
|
-
`${
|
|
8568
|
+
`${path13} must be one of: definition, implementation-intent, similarity, keyword-heavy`
|
|
6905
8569
|
);
|
|
6906
8570
|
}
|
|
6907
|
-
function parseExpected(input,
|
|
8571
|
+
function parseExpected(input, path13) {
|
|
6908
8572
|
if (!isRecord(input)) {
|
|
6909
|
-
throw new Error(`${
|
|
8573
|
+
throw new Error(`${path13} must be an object`);
|
|
6910
8574
|
}
|
|
6911
8575
|
const filePathRaw = input.filePath;
|
|
6912
8576
|
const acceptableFilesRaw = input.acceptableFiles;
|
|
@@ -6915,16 +8579,16 @@ function parseExpected(input, path11) {
|
|
|
6915
8579
|
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
6916
8580
|
const acceptableFiles = isStringArray2(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
6917
8581
|
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
6918
|
-
throw new Error(`${
|
|
8582
|
+
throw new Error(`${path13} must include either expected.filePath or expected.acceptableFiles`);
|
|
6919
8583
|
}
|
|
6920
8584
|
if (acceptableFilesRaw !== void 0 && !isStringArray2(acceptableFilesRaw)) {
|
|
6921
|
-
throw new Error(`${
|
|
8585
|
+
throw new Error(`${path13}.acceptableFiles must be an array of strings`);
|
|
6922
8586
|
}
|
|
6923
8587
|
if (symbolRaw !== void 0 && typeof symbolRaw !== "string") {
|
|
6924
|
-
throw new Error(`${
|
|
8588
|
+
throw new Error(`${path13}.symbol must be a string when provided`);
|
|
6925
8589
|
}
|
|
6926
8590
|
if (branchRaw !== void 0 && typeof branchRaw !== "string") {
|
|
6927
|
-
throw new Error(`${
|
|
8591
|
+
throw new Error(`${path13}.branch must be a string when provided`);
|
|
6928
8592
|
}
|
|
6929
8593
|
return {
|
|
6930
8594
|
filePath,
|
|
@@ -6934,25 +8598,25 @@ function parseExpected(input, path11) {
|
|
|
6934
8598
|
};
|
|
6935
8599
|
}
|
|
6936
8600
|
function parseQuery(input, index) {
|
|
6937
|
-
const
|
|
8601
|
+
const path13 = `queries[${index}]`;
|
|
6938
8602
|
if (!isRecord(input)) {
|
|
6939
|
-
throw new Error(`${
|
|
8603
|
+
throw new Error(`${path13} must be an object`);
|
|
6940
8604
|
}
|
|
6941
8605
|
const id = input.id;
|
|
6942
8606
|
const query = input.query;
|
|
6943
8607
|
const queryType = input.queryType;
|
|
6944
8608
|
const expected = input.expected;
|
|
6945
8609
|
if (typeof id !== "string" || id.trim().length === 0) {
|
|
6946
|
-
throw new Error(`${
|
|
8610
|
+
throw new Error(`${path13}.id must be a non-empty string`);
|
|
6947
8611
|
}
|
|
6948
8612
|
if (typeof query !== "string" || query.trim().length === 0) {
|
|
6949
|
-
throw new Error(`${
|
|
8613
|
+
throw new Error(`${path13}.query must be a non-empty string`);
|
|
6950
8614
|
}
|
|
6951
8615
|
return {
|
|
6952
8616
|
id,
|
|
6953
8617
|
query,
|
|
6954
|
-
queryType: parseQueryType(queryType, `${
|
|
6955
|
-
expected: parseExpected(expected, `${
|
|
8618
|
+
queryType: parseQueryType(queryType, `${path13}.queryType`),
|
|
8619
|
+
expected: parseExpected(expected, `${path13}.expected`)
|
|
6956
8620
|
};
|
|
6957
8621
|
}
|
|
6958
8622
|
function parseGoldenDataset(raw, sourceLabel) {
|
|
@@ -7049,34 +8713,82 @@ function loadBudget(budgetPath) {
|
|
|
7049
8713
|
|
|
7050
8714
|
// src/eval/runner.ts
|
|
7051
8715
|
function toAbsolute(projectRoot, maybeRelative) {
|
|
7052
|
-
return
|
|
8716
|
+
return path9.isAbsolute(maybeRelative) ? maybeRelative : path9.join(projectRoot, maybeRelative);
|
|
8717
|
+
}
|
|
8718
|
+
function isProjectScopedConfigPath(configPath) {
|
|
8719
|
+
return path9.basename(configPath) === "codebase-index.json" && path9.basename(path9.dirname(configPath)) === ".opencode";
|
|
8720
|
+
}
|
|
8721
|
+
function normalizeEvalConfigKnowledgeBases(rawConfig, projectRoot, resolvedConfigPath) {
|
|
8722
|
+
const config = rawConfig && typeof rawConfig === "object" ? { ...rawConfig } : {};
|
|
8723
|
+
if (!Array.isArray(config.knowledgeBases)) {
|
|
8724
|
+
return config;
|
|
8725
|
+
}
|
|
8726
|
+
config.knowledgeBases = isProjectScopedConfigPath(resolvedConfigPath) ? resolveInheritedKnowledgeBaseEntries(
|
|
8727
|
+
config.knowledgeBases,
|
|
8728
|
+
path9.dirname(path9.dirname(resolvedConfigPath)),
|
|
8729
|
+
projectRoot
|
|
8730
|
+
) : rebasePathEntries(
|
|
8731
|
+
config.knowledgeBases,
|
|
8732
|
+
path9.dirname(resolvedConfigPath),
|
|
8733
|
+
projectRoot
|
|
8734
|
+
);
|
|
8735
|
+
return config;
|
|
7053
8736
|
}
|
|
7054
8737
|
function loadRawConfig(projectRoot, configPath) {
|
|
7055
8738
|
const fromPath = configPath ? toAbsolute(projectRoot, configPath) : null;
|
|
7056
|
-
if (fromPath && (0,
|
|
7057
|
-
return
|
|
8739
|
+
if (fromPath && (0, import_fs9.existsSync)(fromPath)) {
|
|
8740
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
8741
|
+
JSON.parse((0, import_fs11.readFileSync)(fromPath, "utf-8")),
|
|
8742
|
+
projectRoot,
|
|
8743
|
+
fromPath
|
|
8744
|
+
);
|
|
7058
8745
|
}
|
|
7059
|
-
const projectConfig =
|
|
7060
|
-
if ((0,
|
|
7061
|
-
return
|
|
8746
|
+
const projectConfig = resolveProjectConfigPath(projectRoot);
|
|
8747
|
+
if ((0, import_fs9.existsSync)(projectConfig)) {
|
|
8748
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
8749
|
+
JSON.parse((0, import_fs11.readFileSync)(projectConfig, "utf-8")),
|
|
8750
|
+
projectRoot,
|
|
8751
|
+
projectConfig
|
|
8752
|
+
);
|
|
7062
8753
|
}
|
|
7063
|
-
const globalConfig =
|
|
7064
|
-
if ((0,
|
|
7065
|
-
return JSON.parse((0,
|
|
8754
|
+
const globalConfig = path9.join(os5.homedir(), ".config", "opencode", "codebase-index.json");
|
|
8755
|
+
if ((0, import_fs9.existsSync)(globalConfig)) {
|
|
8756
|
+
return JSON.parse((0, import_fs11.readFileSync)(globalConfig, "utf-8"));
|
|
7066
8757
|
}
|
|
7067
8758
|
return {};
|
|
7068
8759
|
}
|
|
7069
8760
|
function getIndexRootPath(projectRoot, scope) {
|
|
7070
|
-
|
|
7071
|
-
|
|
7072
|
-
|
|
7073
|
-
return
|
|
8761
|
+
return scope === "global" ? getGlobalIndexPath() : resolveProjectIndexPath(projectRoot, scope);
|
|
8762
|
+
}
|
|
8763
|
+
function getLocalProjectIndexRoot(projectRoot) {
|
|
8764
|
+
return path9.join(projectRoot, ".opencode", "index");
|
|
8765
|
+
}
|
|
8766
|
+
function getLocalProjectConfigPath(projectRoot) {
|
|
8767
|
+
return path9.join(projectRoot, ".opencode", "codebase-index.json");
|
|
7074
8768
|
}
|
|
7075
8769
|
function clearIndexRoot(projectRoot, scope) {
|
|
7076
|
-
const indexRoot = getIndexRootPath(projectRoot, scope);
|
|
7077
|
-
if ((0,
|
|
7078
|
-
(0,
|
|
8770
|
+
const indexRoot = scope === "global" ? getIndexRootPath(projectRoot, scope) : getLocalProjectIndexRoot(projectRoot);
|
|
8771
|
+
if ((0, import_fs9.existsSync)(indexRoot)) {
|
|
8772
|
+
(0, import_fs12.rmSync)(indexRoot, { recursive: true, force: true });
|
|
8773
|
+
}
|
|
8774
|
+
}
|
|
8775
|
+
function ensureLocalEvalProjectConfig(projectRoot, configPath) {
|
|
8776
|
+
const localConfigPath = getLocalProjectConfigPath(projectRoot);
|
|
8777
|
+
const resolvedConfigPath = configPath ? toAbsolute(projectRoot, configPath) : resolveProjectConfigPath(projectRoot);
|
|
8778
|
+
if (!configPath && (0, import_fs9.existsSync)(localConfigPath)) {
|
|
8779
|
+
return localConfigPath;
|
|
7079
8780
|
}
|
|
8781
|
+
if (!(0, import_fs9.existsSync)(resolvedConfigPath) || resolvedConfigPath === localConfigPath) {
|
|
8782
|
+
return resolvedConfigPath;
|
|
8783
|
+
}
|
|
8784
|
+
const sourceConfig = normalizeEvalConfigKnowledgeBases(
|
|
8785
|
+
JSON.parse((0, import_fs11.readFileSync)(resolvedConfigPath, "utf-8")),
|
|
8786
|
+
projectRoot,
|
|
8787
|
+
resolvedConfigPath
|
|
8788
|
+
);
|
|
8789
|
+
(0, import_fs10.mkdirSync)(path9.dirname(localConfigPath), { recursive: true });
|
|
8790
|
+
(0, import_fs13.writeFileSync)(localConfigPath, JSON.stringify(sourceConfig, null, 2), "utf-8");
|
|
8791
|
+
return localConfigPath;
|
|
7080
8792
|
}
|
|
7081
8793
|
function loadParsedConfig(projectRoot, configPath) {
|
|
7082
8794
|
const raw = loadRawConfig(projectRoot, configPath);
|
|
@@ -7108,94 +8820,99 @@ async function runEvaluation(options) {
|
|
|
7108
8820
|
const againstPath = options.againstPath ? toAbsolute(options.projectRoot, options.againstPath) : void 0;
|
|
7109
8821
|
const budgetPath = options.budgetPath ? toAbsolute(options.projectRoot, options.budgetPath) : void 0;
|
|
7110
8822
|
const dataset = loadGoldenDataset(datasetPath);
|
|
7111
|
-
const
|
|
8823
|
+
const resolvedEvalConfigPath = options.reindex ? ensureLocalEvalProjectConfig(options.projectRoot, options.configPath) : options.configPath;
|
|
8824
|
+
const parsedConfig = loadParsedConfig(options.projectRoot, resolvedEvalConfigPath);
|
|
7112
8825
|
const effectiveConfig = resolveSearchConfig(parsedConfig, options.searchOverrides);
|
|
7113
8826
|
if (options.reindex) {
|
|
7114
8827
|
clearIndexRoot(options.projectRoot, effectiveConfig.scope);
|
|
7115
8828
|
}
|
|
7116
8829
|
const indexer = new Indexer(options.projectRoot, effectiveConfig);
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
|
|
7120
|
-
|
|
7121
|
-
|
|
7122
|
-
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
7123
|
-
);
|
|
7124
|
-
}
|
|
7125
|
-
const start = import_perf_hooks2.performance.now();
|
|
7126
|
-
const result = await indexer.search(query.query, 10, {
|
|
7127
|
-
metadataOnly: true,
|
|
7128
|
-
filterByBranch: query.expected.branch ? true : false
|
|
7129
|
-
});
|
|
7130
|
-
const elapsed = import_perf_hooks2.performance.now() - start;
|
|
7131
|
-
const materialized = result.map((item) => ({
|
|
7132
|
-
filePath: item.filePath,
|
|
7133
|
-
startLine: item.startLine,
|
|
7134
|
-
endLine: item.endLine,
|
|
7135
|
-
score: item.score,
|
|
7136
|
-
chunkType: item.chunkType,
|
|
7137
|
-
name: item.name
|
|
7138
|
-
}));
|
|
7139
|
-
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
7140
|
-
}
|
|
7141
|
-
const logger = indexer.getLogger();
|
|
7142
|
-
const metricSnapshot = logger.getMetrics();
|
|
7143
|
-
const costPer1MTokensUsd = effectiveConfig.embeddingProvider === "custom" || effectiveConfig.embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(effectiveConfig.embeddingProvider).costPer1MTokens;
|
|
7144
|
-
const summary = {
|
|
7145
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7146
|
-
projectRoot: options.projectRoot,
|
|
7147
|
-
datasetPath,
|
|
7148
|
-
datasetName: dataset.name,
|
|
7149
|
-
datasetVersion: dataset.version,
|
|
7150
|
-
queryCount: dataset.queries.length,
|
|
7151
|
-
topK: 10,
|
|
7152
|
-
searchConfig: {
|
|
7153
|
-
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
7154
|
-
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
7155
|
-
rrfK: effectiveConfig.search.rrfK,
|
|
7156
|
-
rerankTopN: effectiveConfig.search.rerankTopN
|
|
7157
|
-
},
|
|
7158
|
-
metrics: computeEvalMetrics(
|
|
7159
|
-
dataset.queries,
|
|
7160
|
-
perQuery,
|
|
7161
|
-
metricSnapshot.embeddingApiCalls,
|
|
7162
|
-
metricSnapshot.embeddingTokensUsed,
|
|
7163
|
-
costPer1MTokensUsd
|
|
7164
|
-
)
|
|
7165
|
-
};
|
|
7166
|
-
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
7167
|
-
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
7168
|
-
writeJson(path7.join(outputDir, "summary.json"), summary);
|
|
7169
|
-
writeJson(path7.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
7170
|
-
let comparison;
|
|
7171
|
-
if (againstPath) {
|
|
7172
|
-
const baseline = loadSummary(againstPath);
|
|
7173
|
-
comparison = compareSummaries(summary, baseline, againstPath);
|
|
7174
|
-
writeJson(path7.join(outputDir, "compare.json"), comparison);
|
|
7175
|
-
}
|
|
7176
|
-
let gate;
|
|
7177
|
-
if (options.ciMode) {
|
|
7178
|
-
if (!budgetPath) {
|
|
7179
|
-
throw new Error("CI mode requires --budget path");
|
|
7180
|
-
}
|
|
7181
|
-
const budget = loadBudget(budgetPath);
|
|
7182
|
-
if (!comparison && budget.baselinePath) {
|
|
7183
|
-
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
7184
|
-
if ((0, import_fs7.existsSync)(resolvedBaseline)) {
|
|
7185
|
-
const baselineSummary = loadSummary(resolvedBaseline);
|
|
7186
|
-
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
7187
|
-
writeJson(path7.join(outputDir, "compare.json"), comparison);
|
|
7188
|
-
} else if (budget.failOnMissingBaseline) {
|
|
8830
|
+
try {
|
|
8831
|
+
await indexer.index();
|
|
8832
|
+
const perQuery = [];
|
|
8833
|
+
for (const query of dataset.queries) {
|
|
8834
|
+
if (query.expected.branch && query.expected.branch !== indexer.getCurrentBranch()) {
|
|
7189
8835
|
throw new Error(
|
|
7190
|
-
`
|
|
8836
|
+
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
7191
8837
|
);
|
|
7192
8838
|
}
|
|
8839
|
+
const start = import_perf_hooks2.performance.now();
|
|
8840
|
+
const result = await indexer.search(query.query, 10, {
|
|
8841
|
+
metadataOnly: true,
|
|
8842
|
+
filterByBranch: query.expected.branch ? true : false
|
|
8843
|
+
});
|
|
8844
|
+
const elapsed = import_perf_hooks2.performance.now() - start;
|
|
8845
|
+
const materialized = result.map((item) => ({
|
|
8846
|
+
filePath: item.filePath,
|
|
8847
|
+
startLine: item.startLine,
|
|
8848
|
+
endLine: item.endLine,
|
|
8849
|
+
score: item.score,
|
|
8850
|
+
chunkType: item.chunkType,
|
|
8851
|
+
name: item.name
|
|
8852
|
+
}));
|
|
8853
|
+
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
8854
|
+
}
|
|
8855
|
+
const logger = indexer.getLogger();
|
|
8856
|
+
const metricSnapshot = logger.getMetrics();
|
|
8857
|
+
const costPer1MTokensUsd = effectiveConfig.embeddingProvider === "custom" || effectiveConfig.embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(effectiveConfig.embeddingProvider).costPer1MTokens;
|
|
8858
|
+
const summary = {
|
|
8859
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8860
|
+
projectRoot: options.projectRoot,
|
|
8861
|
+
datasetPath,
|
|
8862
|
+
datasetName: dataset.name,
|
|
8863
|
+
datasetVersion: dataset.version,
|
|
8864
|
+
queryCount: dataset.queries.length,
|
|
8865
|
+
topK: 10,
|
|
8866
|
+
searchConfig: {
|
|
8867
|
+
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
8868
|
+
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
8869
|
+
rrfK: effectiveConfig.search.rrfK,
|
|
8870
|
+
rerankTopN: effectiveConfig.search.rerankTopN
|
|
8871
|
+
},
|
|
8872
|
+
metrics: computeEvalMetrics(
|
|
8873
|
+
dataset.queries,
|
|
8874
|
+
perQuery,
|
|
8875
|
+
metricSnapshot.embeddingApiCalls,
|
|
8876
|
+
metricSnapshot.embeddingTokensUsed,
|
|
8877
|
+
costPer1MTokensUsd
|
|
8878
|
+
)
|
|
8879
|
+
};
|
|
8880
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
8881
|
+
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
8882
|
+
writeJson(path9.join(outputDir, "summary.json"), summary);
|
|
8883
|
+
writeJson(path9.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
8884
|
+
let comparison;
|
|
8885
|
+
if (againstPath) {
|
|
8886
|
+
const baseline = loadSummary(againstPath);
|
|
8887
|
+
comparison = compareSummaries(summary, baseline, againstPath);
|
|
8888
|
+
writeJson(path9.join(outputDir, "compare.json"), comparison);
|
|
8889
|
+
}
|
|
8890
|
+
let gate;
|
|
8891
|
+
if (options.ciMode) {
|
|
8892
|
+
if (!budgetPath) {
|
|
8893
|
+
throw new Error("CI mode requires --budget path");
|
|
8894
|
+
}
|
|
8895
|
+
const budget = loadBudget(budgetPath);
|
|
8896
|
+
if (!comparison && budget.baselinePath) {
|
|
8897
|
+
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
8898
|
+
if ((0, import_fs9.existsSync)(resolvedBaseline)) {
|
|
8899
|
+
const baselineSummary = loadSummary(resolvedBaseline);
|
|
8900
|
+
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
8901
|
+
writeJson(path9.join(outputDir, "compare.json"), comparison);
|
|
8902
|
+
} else if (budget.failOnMissingBaseline) {
|
|
8903
|
+
throw new Error(
|
|
8904
|
+
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
8905
|
+
);
|
|
8906
|
+
}
|
|
8907
|
+
}
|
|
8908
|
+
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
7193
8909
|
}
|
|
7194
|
-
|
|
8910
|
+
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
8911
|
+
writeText(path9.join(outputDir, "summary.md"), markdown);
|
|
8912
|
+
return { outputDir, summary, perQuery, comparison, gate };
|
|
8913
|
+
} finally {
|
|
8914
|
+
await indexer.close();
|
|
7195
8915
|
}
|
|
7196
|
-
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
7197
|
-
writeText(path7.join(outputDir, "summary.md"), markdown);
|
|
7198
|
-
return { outputDir, summary, perQuery, comparison, gate };
|
|
7199
8916
|
}
|
|
7200
8917
|
async function runSweep(options, sweep) {
|
|
7201
8918
|
const fusionValues = sweep.fusionStrategy && sweep.fusionStrategy.length > 0 ? [...sweep.fusionStrategy] : [void 0];
|
|
@@ -7249,15 +8966,15 @@ async function runSweep(options, sweep) {
|
|
|
7249
8966
|
bestByMrrAt10,
|
|
7250
8967
|
bestByP95Latency
|
|
7251
8968
|
};
|
|
7252
|
-
writeJson(
|
|
8969
|
+
writeJson(path9.join(outputDir, "compare.json"), aggregate);
|
|
7253
8970
|
const md = createSummaryMarkdown(
|
|
7254
8971
|
bestByHitAt5?.summary ?? runs[0].summary,
|
|
7255
8972
|
bestByHitAt5?.comparison,
|
|
7256
8973
|
void 0,
|
|
7257
8974
|
aggregate
|
|
7258
8975
|
);
|
|
7259
|
-
writeText(
|
|
7260
|
-
writeJson(
|
|
8976
|
+
writeText(path9.join(outputDir, "summary.md"), md);
|
|
8977
|
+
writeJson(path9.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
7261
8978
|
return { outputDir, aggregate };
|
|
7262
8979
|
}
|
|
7263
8980
|
|
|
@@ -7333,12 +9050,12 @@ function parseEvalArgs(argv, cwd) {
|
|
|
7333
9050
|
const arg = argv[i];
|
|
7334
9051
|
const next = argv[i + 1];
|
|
7335
9052
|
if (arg === "--project" && next) {
|
|
7336
|
-
parsed.projectRoot =
|
|
9053
|
+
parsed.projectRoot = path10.resolve(cwd, next);
|
|
7337
9054
|
i += 1;
|
|
7338
9055
|
continue;
|
|
7339
9056
|
}
|
|
7340
9057
|
if (arg === "--config" && next) {
|
|
7341
|
-
parsed.configPath =
|
|
9058
|
+
parsed.configPath = path10.resolve(cwd, next);
|
|
7342
9059
|
i += 1;
|
|
7343
9060
|
continue;
|
|
7344
9061
|
}
|
|
@@ -7532,22 +9249,22 @@ async function handleEvalCommand(args, cwd) {
|
|
|
7532
9249
|
if (!parsed.againstPath.endsWith(".json")) {
|
|
7533
9250
|
throw new Error("eval diff --against must point to a summary JSON file");
|
|
7534
9251
|
}
|
|
7535
|
-
const currentSummary = loadSummary(
|
|
9252
|
+
const currentSummary = loadSummary(path10.resolve(parsed.projectRoot, currentPath), {
|
|
7536
9253
|
allowLegacyDiversityMetrics: true
|
|
7537
9254
|
});
|
|
7538
|
-
const baselineSummary = loadSummary(
|
|
9255
|
+
const baselineSummary = loadSummary(path10.resolve(parsed.projectRoot, parsed.againstPath), {
|
|
7539
9256
|
allowLegacyDiversityMetrics: true
|
|
7540
9257
|
});
|
|
7541
9258
|
const comparison = compareSummaries(
|
|
7542
9259
|
currentSummary,
|
|
7543
9260
|
baselineSummary,
|
|
7544
|
-
|
|
9261
|
+
path10.resolve(parsed.projectRoot, parsed.againstPath)
|
|
7545
9262
|
);
|
|
7546
|
-
const outputDir = createRunDirectory(
|
|
9263
|
+
const outputDir = createRunDirectory(path10.resolve(parsed.projectRoot, parsed.outputRoot));
|
|
7547
9264
|
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
7548
|
-
writeJson(
|
|
7549
|
-
writeText(
|
|
7550
|
-
writeJson(
|
|
9265
|
+
writeJson(path10.join(outputDir, "compare.json"), comparison);
|
|
9266
|
+
writeText(path10.join(outputDir, "summary.md"), summaryMd);
|
|
9267
|
+
writeJson(path10.join(outputDir, "summary.json"), currentSummary);
|
|
7551
9268
|
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
7552
9269
|
return 0;
|
|
7553
9270
|
}
|
|
@@ -7557,6 +9274,8 @@ async function handleEvalCommand(args, cwd) {
|
|
|
7557
9274
|
// src/mcp-server.ts
|
|
7558
9275
|
var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
7559
9276
|
var import_zod = require("zod");
|
|
9277
|
+
var path11 = __toESM(require("path"), 1);
|
|
9278
|
+
var import_fs14 = require("fs");
|
|
7560
9279
|
|
|
7561
9280
|
// src/tools/utils.ts
|
|
7562
9281
|
var MAX_CONTENT_LINES = 30;
|
|
@@ -7566,36 +9285,24 @@ function truncateContent(content) {
|
|
|
7566
9285
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
7567
9286
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
7568
9287
|
}
|
|
7569
|
-
function formatDefinitionLookup(results, query) {
|
|
7570
|
-
if (results.length === 0) {
|
|
7571
|
-
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
7572
|
-
}
|
|
7573
|
-
const formatted = results.map((r, idx) => {
|
|
7574
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
7575
|
-
return `${header} (score: ${r.score.toFixed(2)})
|
|
7576
|
-
\`\`\`
|
|
7577
|
-
${truncateContent(r.content)}
|
|
7578
|
-
\`\`\``;
|
|
7579
|
-
});
|
|
7580
|
-
return formatted.join("\n\n");
|
|
7581
|
-
}
|
|
7582
|
-
|
|
7583
|
-
// src/mcp-server.ts
|
|
7584
|
-
var MAX_CONTENT_LINES2 = 30;
|
|
7585
|
-
function truncateContent2(content) {
|
|
7586
|
-
const lines = content.split("\n");
|
|
7587
|
-
if (lines.length <= MAX_CONTENT_LINES2) return content;
|
|
7588
|
-
return lines.slice(0, MAX_CONTENT_LINES2).join("\n") + `
|
|
7589
|
-
// ... (${lines.length - MAX_CONTENT_LINES2} more lines)`;
|
|
7590
|
-
}
|
|
7591
9288
|
function formatIndexStats(stats, verbose = false) {
|
|
9289
|
+
if (stats.resetCorruptedIndex) {
|
|
9290
|
+
return stats.warning ?? "Detected a corrupted local index and reset it during indexing. Run index_codebase again to rebuild search data.";
|
|
9291
|
+
}
|
|
7592
9292
|
const lines = [];
|
|
9293
|
+
if (stats.failedChunks > 0) {
|
|
9294
|
+
lines.push(`INDEXING WARNING: ${stats.failedChunks} chunks failed to embed.`);
|
|
9295
|
+
if (stats.failedBatchesPath) {
|
|
9296
|
+
lines.push(`Inspect failed batches at: ${stats.failedBatchesPath}`);
|
|
9297
|
+
}
|
|
9298
|
+
lines.push("");
|
|
9299
|
+
}
|
|
7593
9300
|
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
7594
|
-
lines.push(
|
|
9301
|
+
lines.push(`${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
7595
9302
|
} else if (stats.indexedChunks === 0) {
|
|
7596
|
-
lines.push(
|
|
9303
|
+
lines.push(`${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
7597
9304
|
} else {
|
|
7598
|
-
let main2 =
|
|
9305
|
+
let main2 = `${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
7599
9306
|
if (stats.existingChunks > 0) {
|
|
7600
9307
|
main2 += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
7601
9308
|
}
|
|
@@ -7634,21 +9341,103 @@ function formatIndexStats(stats, verbose = false) {
|
|
|
7634
9341
|
}
|
|
7635
9342
|
function formatStatus(status) {
|
|
7636
9343
|
if (!status.indexed) {
|
|
9344
|
+
if (status.warning) {
|
|
9345
|
+
return status.warning;
|
|
9346
|
+
}
|
|
9347
|
+
if (status.failedBatchesCount > 0) {
|
|
9348
|
+
const lines2 = [
|
|
9349
|
+
"Codebase is not indexed. The last indexing run left failed embedding batches.",
|
|
9350
|
+
"Fix the provider/model configuration, then rerun index_codebase normally to retry the saved failed batches. Use force=true only for a full rebuild or compatibility reset."
|
|
9351
|
+
];
|
|
9352
|
+
if (status.failedBatchesPath) {
|
|
9353
|
+
lines2.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
9354
|
+
}
|
|
9355
|
+
return lines2.join("\n");
|
|
9356
|
+
}
|
|
7637
9357
|
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
7638
9358
|
}
|
|
7639
9359
|
const lines = [
|
|
7640
|
-
`
|
|
7641
|
-
`
|
|
7642
|
-
`
|
|
7643
|
-
`
|
|
7644
|
-
` Location: ${status.indexPath}`
|
|
9360
|
+
`Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
9361
|
+
`Provider: ${status.provider}`,
|
|
9362
|
+
`Model: ${status.model}`,
|
|
9363
|
+
`Location: ${status.indexPath}`
|
|
7645
9364
|
];
|
|
7646
9365
|
if (status.currentBranch !== "default") {
|
|
7647
|
-
lines.push(`
|
|
7648
|
-
lines.push(`
|
|
9366
|
+
lines.push(`Current branch: ${status.currentBranch}`);
|
|
9367
|
+
lines.push(`Base branch: ${status.baseBranch}`);
|
|
9368
|
+
}
|
|
9369
|
+
if (status.failedBatchesCount > 0) {
|
|
9370
|
+
lines.push("");
|
|
9371
|
+
lines.push(`INDEXING WARNING: ${status.failedBatchesCount} failed embedding batch${status.failedBatchesCount === 1 ? " remains" : "es remain"}.`);
|
|
9372
|
+
if (status.failedBatchesPath) {
|
|
9373
|
+
lines.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
9374
|
+
}
|
|
9375
|
+
}
|
|
9376
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
9377
|
+
lines.push("");
|
|
9378
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
9379
|
+
if (status.compatibility.storedMetadata) {
|
|
9380
|
+
const stored = status.compatibility.storedMetadata;
|
|
9381
|
+
lines.push(`Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
9382
|
+
lines.push(`Current config: ${status.provider}/${status.model}`);
|
|
9383
|
+
}
|
|
9384
|
+
} else if (!status.compatibility) {
|
|
9385
|
+
lines.push(`Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
9386
|
+
} else {
|
|
9387
|
+
lines.push(`Compatibility: Index is compatible with the current provider and model.`);
|
|
9388
|
+
}
|
|
9389
|
+
return lines.join("\n");
|
|
9390
|
+
}
|
|
9391
|
+
function formatHealthCheck(result) {
|
|
9392
|
+
if (result.resetCorruptedIndex) {
|
|
9393
|
+
return result.warning ?? "Detected a corrupted local index and reset it. Run index_codebase to rebuild search data.";
|
|
9394
|
+
}
|
|
9395
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0 && result.gcOrphanSymbols === 0 && result.gcOrphanCallEdges === 0) {
|
|
9396
|
+
return "Index is healthy. No stale entries found.";
|
|
9397
|
+
}
|
|
9398
|
+
const lines = [];
|
|
9399
|
+
if (result.removed > 0) {
|
|
9400
|
+
lines.push(`Removed stale entries: ${result.removed}`);
|
|
9401
|
+
}
|
|
9402
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
9403
|
+
lines.push(`Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
9404
|
+
}
|
|
9405
|
+
if (result.gcOrphanChunks > 0) {
|
|
9406
|
+
lines.push(`Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
9407
|
+
}
|
|
9408
|
+
if (result.gcOrphanSymbols > 0) {
|
|
9409
|
+
lines.push(`Garbage collected orphan symbols: ${result.gcOrphanSymbols}`);
|
|
9410
|
+
}
|
|
9411
|
+
if (result.gcOrphanCallEdges > 0) {
|
|
9412
|
+
lines.push(`Garbage collected orphan call edges: ${result.gcOrphanCallEdges}`);
|
|
9413
|
+
}
|
|
9414
|
+
if (result.filePaths.length > 0) {
|
|
9415
|
+
lines.push(`Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
7649
9416
|
}
|
|
7650
9417
|
return lines.join("\n");
|
|
7651
9418
|
}
|
|
9419
|
+
function formatDefinitionLookup(results, query) {
|
|
9420
|
+
if (results.length === 0) {
|
|
9421
|
+
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
9422
|
+
}
|
|
9423
|
+
const formatted = results.map((r, idx) => {
|
|
9424
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
9425
|
+
return `${header} (score: ${r.score.toFixed(2)})
|
|
9426
|
+
\`\`\`
|
|
9427
|
+
${truncateContent(r.content)}
|
|
9428
|
+
\`\`\``;
|
|
9429
|
+
});
|
|
9430
|
+
return formatted.join("\n\n");
|
|
9431
|
+
}
|
|
9432
|
+
|
|
9433
|
+
// src/mcp-server.ts
|
|
9434
|
+
var MAX_CONTENT_LINES2 = 30;
|
|
9435
|
+
function truncateContent2(content) {
|
|
9436
|
+
const lines = content.split("\n");
|
|
9437
|
+
if (lines.length <= MAX_CONTENT_LINES2) return content;
|
|
9438
|
+
return lines.slice(0, MAX_CONTENT_LINES2).join("\n") + `
|
|
9439
|
+
// ... (${lines.length - MAX_CONTENT_LINES2} more lines)`;
|
|
9440
|
+
}
|
|
7652
9441
|
var CHUNK_TYPE_ENUM = [
|
|
7653
9442
|
"function",
|
|
7654
9443
|
"class",
|
|
@@ -7667,8 +9456,25 @@ function createMcpServer(projectRoot, config) {
|
|
|
7667
9456
|
name: "opencode-codebase-index",
|
|
7668
9457
|
version: "0.5.1"
|
|
7669
9458
|
});
|
|
7670
|
-
const
|
|
9459
|
+
const runtimeConfig = config;
|
|
9460
|
+
let indexer = new Indexer(projectRoot, runtimeConfig);
|
|
7671
9461
|
let initialized = false;
|
|
9462
|
+
function refreshIndexerFromConfig() {
|
|
9463
|
+
indexer = new Indexer(projectRoot, runtimeConfig);
|
|
9464
|
+
initialized = false;
|
|
9465
|
+
}
|
|
9466
|
+
function shouldForceLocalizeProjectIndex() {
|
|
9467
|
+
if (runtimeConfig.scope !== "project") {
|
|
9468
|
+
return false;
|
|
9469
|
+
}
|
|
9470
|
+
const localIndexPath = path11.join(projectRoot, ".opencode", "index");
|
|
9471
|
+
const mainRepoRoot = resolveWorktreeMainRepoRoot(projectRoot);
|
|
9472
|
+
if (!mainRepoRoot) {
|
|
9473
|
+
return false;
|
|
9474
|
+
}
|
|
9475
|
+
const inheritedIndexPath = path11.join(mainRepoRoot, ".opencode", "index");
|
|
9476
|
+
return !(0, import_fs14.existsSync)(localIndexPath) && (0, import_fs14.existsSync)(inheritedIndexPath);
|
|
9477
|
+
}
|
|
7672
9478
|
async function ensureInitialized() {
|
|
7673
9479
|
if (!initialized) {
|
|
7674
9480
|
await indexer.initialize();
|
|
@@ -7751,13 +9557,22 @@ Use Read tool to examine specific files.` }] };
|
|
|
7751
9557
|
verbose: import_zod.z.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures")
|
|
7752
9558
|
},
|
|
7753
9559
|
async (args) => {
|
|
7754
|
-
await ensureInitialized();
|
|
7755
9560
|
if (args.estimateOnly) {
|
|
9561
|
+
await ensureInitialized();
|
|
7756
9562
|
const estimate = await indexer.estimateCost();
|
|
7757
9563
|
return { content: [{ type: "text", text: formatCostEstimate(estimate) }] };
|
|
7758
9564
|
}
|
|
7759
9565
|
if (args.force) {
|
|
9566
|
+
if (shouldForceLocalizeProjectIndex()) {
|
|
9567
|
+
materializeLocalProjectConfig(projectRoot, loadProjectConfigLayer(projectRoot));
|
|
9568
|
+
refreshIndexerFromConfig();
|
|
9569
|
+
}
|
|
9570
|
+
await ensureInitialized();
|
|
7760
9571
|
await indexer.clearIndex();
|
|
9572
|
+
refreshIndexerFromConfig();
|
|
9573
|
+
await ensureInitialized();
|
|
9574
|
+
} else {
|
|
9575
|
+
await ensureInitialized();
|
|
7761
9576
|
}
|
|
7762
9577
|
const stats = await indexer.index();
|
|
7763
9578
|
return { content: [{ type: "text", text: formatIndexStats(stats, args.verbose ?? false) }] };
|
|
@@ -7780,29 +9595,7 @@ Use Read tool to examine specific files.` }] };
|
|
|
7780
9595
|
async () => {
|
|
7781
9596
|
await ensureInitialized();
|
|
7782
9597
|
const result = await indexer.healthCheck();
|
|
7783
|
-
|
|
7784
|
-
return { content: [{ type: "text", text: "Index is healthy. No stale entries found." }] };
|
|
7785
|
-
}
|
|
7786
|
-
const lines = [`Health check complete:`];
|
|
7787
|
-
if (result.removed > 0) {
|
|
7788
|
-
lines.push(` Removed stale entries: ${result.removed}`);
|
|
7789
|
-
}
|
|
7790
|
-
if (result.gcOrphanEmbeddings > 0) {
|
|
7791
|
-
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
7792
|
-
}
|
|
7793
|
-
if (result.gcOrphanChunks > 0) {
|
|
7794
|
-
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
7795
|
-
}
|
|
7796
|
-
if (result.gcOrphanSymbols > 0) {
|
|
7797
|
-
lines.push(` Garbage collected orphan symbols: ${result.gcOrphanSymbols}`);
|
|
7798
|
-
}
|
|
7799
|
-
if (result.gcOrphanCallEdges > 0) {
|
|
7800
|
-
lines.push(` Garbage collected orphan call edges: ${result.gcOrphanCallEdges}`);
|
|
7801
|
-
}
|
|
7802
|
-
if (result.filePaths.length > 0) {
|
|
7803
|
-
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
7804
|
-
}
|
|
7805
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
9598
|
+
return { content: [{ type: "text", text: formatHealthCheck(result) }] };
|
|
7806
9599
|
}
|
|
7807
9600
|
);
|
|
7808
9601
|
server.tool(
|
|
@@ -8031,115 +9824,15 @@ Use the implementation_lookup tool to find where this symbol is defined. This pr
|
|
|
8031
9824
|
return server;
|
|
8032
9825
|
}
|
|
8033
9826
|
|
|
8034
|
-
// src/config/merger.ts
|
|
8035
|
-
var import_fs10 = require("fs");
|
|
8036
|
-
var path9 = __toESM(require("path"), 1);
|
|
8037
|
-
var os4 = __toESM(require("os"), 1);
|
|
8038
|
-
function loadJsonFile(filePath) {
|
|
8039
|
-
try {
|
|
8040
|
-
if ((0, import_fs10.existsSync)(filePath)) {
|
|
8041
|
-
const content = (0, import_fs10.readFileSync)(filePath, "utf-8");
|
|
8042
|
-
return JSON.parse(content);
|
|
8043
|
-
}
|
|
8044
|
-
} catch {
|
|
8045
|
-
}
|
|
8046
|
-
return null;
|
|
8047
|
-
}
|
|
8048
|
-
function loadMergedConfig(projectRoot) {
|
|
8049
|
-
const globalConfigPath = path9.join(os4.homedir(), ".config", "opencode", "codebase-index.json");
|
|
8050
|
-
const globalConfig = loadJsonFile(globalConfigPath);
|
|
8051
|
-
const projectConfigPath = path9.join(projectRoot, ".opencode", "codebase-index.json");
|
|
8052
|
-
const projectConfig = loadJsonFile(projectConfigPath);
|
|
8053
|
-
if (!globalConfig && !projectConfig) {
|
|
8054
|
-
return {};
|
|
8055
|
-
}
|
|
8056
|
-
if (!projectConfig && globalConfig) {
|
|
8057
|
-
return globalConfig;
|
|
8058
|
-
}
|
|
8059
|
-
if (!globalConfig && projectConfig) {
|
|
8060
|
-
return projectConfig;
|
|
8061
|
-
}
|
|
8062
|
-
const merged = { ...globalConfig };
|
|
8063
|
-
if (projectConfig && "embeddingProvider" in projectConfig) {
|
|
8064
|
-
merged.embeddingProvider = projectConfig.embeddingProvider;
|
|
8065
|
-
} else if (globalConfig && globalConfig.embeddingProvider) {
|
|
8066
|
-
merged.embeddingProvider = globalConfig.embeddingProvider;
|
|
8067
|
-
}
|
|
8068
|
-
if (projectConfig && "customProvider" in projectConfig) {
|
|
8069
|
-
merged.customProvider = projectConfig.customProvider;
|
|
8070
|
-
} else if (globalConfig && globalConfig.customProvider) {
|
|
8071
|
-
merged.customProvider = globalConfig.customProvider;
|
|
8072
|
-
}
|
|
8073
|
-
if (projectConfig && "embeddingModel" in projectConfig) {
|
|
8074
|
-
merged.embeddingModel = projectConfig.embeddingModel;
|
|
8075
|
-
} else if (globalConfig && globalConfig.embeddingModel) {
|
|
8076
|
-
merged.embeddingModel = globalConfig.embeddingModel;
|
|
8077
|
-
}
|
|
8078
|
-
if (projectConfig && "reranker" in projectConfig) {
|
|
8079
|
-
merged.reranker = projectConfig.reranker;
|
|
8080
|
-
} else if (globalConfig && globalConfig.reranker) {
|
|
8081
|
-
merged.reranker = globalConfig.reranker;
|
|
8082
|
-
}
|
|
8083
|
-
if (projectConfig && "include" in projectConfig) {
|
|
8084
|
-
merged.include = projectConfig.include;
|
|
8085
|
-
} else if (globalConfig && globalConfig.include) {
|
|
8086
|
-
merged.include = globalConfig.include;
|
|
8087
|
-
}
|
|
8088
|
-
if (projectConfig && "exclude" in projectConfig) {
|
|
8089
|
-
merged.exclude = projectConfig.exclude;
|
|
8090
|
-
} else if (globalConfig && globalConfig.exclude) {
|
|
8091
|
-
merged.exclude = globalConfig.exclude;
|
|
8092
|
-
}
|
|
8093
|
-
if (projectConfig && "indexing" in projectConfig) {
|
|
8094
|
-
merged.indexing = projectConfig.indexing;
|
|
8095
|
-
} else if (globalConfig && globalConfig.indexing) {
|
|
8096
|
-
merged.indexing = globalConfig.indexing;
|
|
8097
|
-
}
|
|
8098
|
-
if (projectConfig && "search" in projectConfig) {
|
|
8099
|
-
merged.search = projectConfig.search;
|
|
8100
|
-
} else if (globalConfig && globalConfig.search) {
|
|
8101
|
-
merged.search = globalConfig.search;
|
|
8102
|
-
}
|
|
8103
|
-
if (projectConfig && "debug" in projectConfig) {
|
|
8104
|
-
merged.debug = projectConfig.debug;
|
|
8105
|
-
} else if (globalConfig && globalConfig.debug) {
|
|
8106
|
-
merged.debug = globalConfig.debug;
|
|
8107
|
-
}
|
|
8108
|
-
if (projectConfig && "scope" in projectConfig) {
|
|
8109
|
-
merged.scope = projectConfig.scope;
|
|
8110
|
-
} else if (globalConfig && "scope" in globalConfig) {
|
|
8111
|
-
merged.scope = globalConfig.scope;
|
|
8112
|
-
}
|
|
8113
|
-
if (projectConfig) {
|
|
8114
|
-
for (const key of Object.keys(projectConfig)) {
|
|
8115
|
-
if (key === "embeddingProvider" || key === "customProvider" || key === "embeddingModel" || key === "reranker" || key === "include" || key === "exclude" || key === "indexing" || key === "search" || key === "debug" || key === "scope" || key === "knowledgeBases" || key === "additionalInclude") {
|
|
8116
|
-
continue;
|
|
8117
|
-
}
|
|
8118
|
-
merged[key] = projectConfig[key];
|
|
8119
|
-
}
|
|
8120
|
-
}
|
|
8121
|
-
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
8122
|
-
const projectKbs = projectConfig && Array.isArray(projectConfig.knowledgeBases) ? projectConfig.knowledgeBases : [];
|
|
8123
|
-
const allKbs = [...globalKbs, ...projectKbs];
|
|
8124
|
-
const uniqueKbs = [...new Set(allKbs.map((p) => String(p).trim()))];
|
|
8125
|
-
merged.knowledgeBases = uniqueKbs;
|
|
8126
|
-
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
8127
|
-
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
8128
|
-
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
8129
|
-
const uniqueAdditional = [...new Set(allAdditional.map((p) => String(p).trim()))];
|
|
8130
|
-
merged.additionalInclude = uniqueAdditional;
|
|
8131
|
-
return merged;
|
|
8132
|
-
}
|
|
8133
|
-
|
|
8134
9827
|
// src/cli.ts
|
|
8135
9828
|
function parseArgs(argv) {
|
|
8136
9829
|
let project = process.cwd();
|
|
8137
9830
|
let config;
|
|
8138
9831
|
for (let i = 2; i < argv.length; i++) {
|
|
8139
9832
|
if (argv[i] === "--project" && argv[i + 1]) {
|
|
8140
|
-
project =
|
|
9833
|
+
project = path12.resolve(argv[++i]);
|
|
8141
9834
|
} else if (argv[i] === "--config" && argv[i + 1]) {
|
|
8142
|
-
config =
|
|
9835
|
+
config = path12.resolve(argv[++i]);
|
|
8143
9836
|
}
|
|
8144
9837
|
}
|
|
8145
9838
|
return { project, config };
|