opencode-codebase-index 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -7
- package/commands/index.md +4 -0
- package/commands/peek.md +24 -0
- package/commands/reindex.md +25 -0
- package/commands/status.md +5 -1
- package/dist/cli.cjs +2422 -730
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2415 -723
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +2351 -711
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2340 -700
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +3 -3
package/dist/cli.js
CHANGED
|
@@ -487,7 +487,7 @@ var require_ignore = __commonJS({
|
|
|
487
487
|
// path matching.
|
|
488
488
|
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
489
489
|
// @returns {TestResult} true if a file is ignored
|
|
490
|
-
test(
|
|
490
|
+
test(path13, checkUnignored, mode) {
|
|
491
491
|
let ignored = false;
|
|
492
492
|
let unignored = false;
|
|
493
493
|
let matchedRule;
|
|
@@ -496,7 +496,7 @@ var require_ignore = __commonJS({
|
|
|
496
496
|
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
497
497
|
return;
|
|
498
498
|
}
|
|
499
|
-
const matched = rule[mode].test(
|
|
499
|
+
const matched = rule[mode].test(path13);
|
|
500
500
|
if (!matched) {
|
|
501
501
|
return;
|
|
502
502
|
}
|
|
@@ -517,17 +517,17 @@ var require_ignore = __commonJS({
|
|
|
517
517
|
var throwError = (message, Ctor) => {
|
|
518
518
|
throw new Ctor(message);
|
|
519
519
|
};
|
|
520
|
-
var checkPath = (
|
|
521
|
-
if (!isString(
|
|
520
|
+
var checkPath = (path13, originalPath, doThrow) => {
|
|
521
|
+
if (!isString(path13)) {
|
|
522
522
|
return doThrow(
|
|
523
523
|
`path must be a string, but got \`${originalPath}\``,
|
|
524
524
|
TypeError
|
|
525
525
|
);
|
|
526
526
|
}
|
|
527
|
-
if (!
|
|
527
|
+
if (!path13) {
|
|
528
528
|
return doThrow(`path must not be empty`, TypeError);
|
|
529
529
|
}
|
|
530
|
-
if (checkPath.isNotRelative(
|
|
530
|
+
if (checkPath.isNotRelative(path13)) {
|
|
531
531
|
const r = "`path.relative()`d";
|
|
532
532
|
return doThrow(
|
|
533
533
|
`path should be a ${r} string, but got "${originalPath}"`,
|
|
@@ -536,7 +536,7 @@ var require_ignore = __commonJS({
|
|
|
536
536
|
}
|
|
537
537
|
return true;
|
|
538
538
|
};
|
|
539
|
-
var isNotRelative = (
|
|
539
|
+
var isNotRelative = (path13) => REGEX_TEST_INVALID_PATH.test(path13);
|
|
540
540
|
checkPath.isNotRelative = isNotRelative;
|
|
541
541
|
checkPath.convert = (p) => p;
|
|
542
542
|
var Ignore2 = class {
|
|
@@ -566,19 +566,19 @@ var require_ignore = __commonJS({
|
|
|
566
566
|
}
|
|
567
567
|
// @returns {TestResult}
|
|
568
568
|
_test(originalPath, cache, checkUnignored, slices) {
|
|
569
|
-
const
|
|
569
|
+
const path13 = originalPath && checkPath.convert(originalPath);
|
|
570
570
|
checkPath(
|
|
571
|
-
|
|
571
|
+
path13,
|
|
572
572
|
originalPath,
|
|
573
573
|
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
574
574
|
);
|
|
575
|
-
return this._t(
|
|
575
|
+
return this._t(path13, cache, checkUnignored, slices);
|
|
576
576
|
}
|
|
577
|
-
checkIgnore(
|
|
578
|
-
if (!REGEX_TEST_TRAILING_SLASH.test(
|
|
579
|
-
return this.test(
|
|
577
|
+
checkIgnore(path13) {
|
|
578
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path13)) {
|
|
579
|
+
return this.test(path13);
|
|
580
580
|
}
|
|
581
|
-
const slices =
|
|
581
|
+
const slices = path13.split(SLASH).filter(Boolean);
|
|
582
582
|
slices.pop();
|
|
583
583
|
if (slices.length) {
|
|
584
584
|
const parent = this._t(
|
|
@@ -591,18 +591,18 @@ var require_ignore = __commonJS({
|
|
|
591
591
|
return parent;
|
|
592
592
|
}
|
|
593
593
|
}
|
|
594
|
-
return this._rules.test(
|
|
594
|
+
return this._rules.test(path13, false, MODE_CHECK_IGNORE);
|
|
595
595
|
}
|
|
596
|
-
_t(
|
|
597
|
-
if (
|
|
598
|
-
return cache[
|
|
596
|
+
_t(path13, cache, checkUnignored, slices) {
|
|
597
|
+
if (path13 in cache) {
|
|
598
|
+
return cache[path13];
|
|
599
599
|
}
|
|
600
600
|
if (!slices) {
|
|
601
|
-
slices =
|
|
601
|
+
slices = path13.split(SLASH).filter(Boolean);
|
|
602
602
|
}
|
|
603
603
|
slices.pop();
|
|
604
604
|
if (!slices.length) {
|
|
605
|
-
return cache[
|
|
605
|
+
return cache[path13] = this._rules.test(path13, checkUnignored, MODE_IGNORE);
|
|
606
606
|
}
|
|
607
607
|
const parent = this._t(
|
|
608
608
|
slices.join(SLASH) + SLASH,
|
|
@@ -610,29 +610,29 @@ var require_ignore = __commonJS({
|
|
|
610
610
|
checkUnignored,
|
|
611
611
|
slices
|
|
612
612
|
);
|
|
613
|
-
return cache[
|
|
613
|
+
return cache[path13] = parent.ignored ? parent : this._rules.test(path13, checkUnignored, MODE_IGNORE);
|
|
614
614
|
}
|
|
615
|
-
ignores(
|
|
616
|
-
return this._test(
|
|
615
|
+
ignores(path13) {
|
|
616
|
+
return this._test(path13, this._ignoreCache, false).ignored;
|
|
617
617
|
}
|
|
618
618
|
createFilter() {
|
|
619
|
-
return (
|
|
619
|
+
return (path13) => !this.ignores(path13);
|
|
620
620
|
}
|
|
621
621
|
filter(paths) {
|
|
622
622
|
return makeArray(paths).filter(this.createFilter());
|
|
623
623
|
}
|
|
624
624
|
// @returns {TestResult}
|
|
625
|
-
test(
|
|
626
|
-
return this._test(
|
|
625
|
+
test(path13) {
|
|
626
|
+
return this._test(path13, this._testCache, true);
|
|
627
627
|
}
|
|
628
628
|
};
|
|
629
629
|
var factory = (options) => new Ignore2(options);
|
|
630
|
-
var isPathValid = (
|
|
630
|
+
var isPathValid = (path13) => checkPath(path13 && checkPath.convert(path13), path13, RETURN_FALSE);
|
|
631
631
|
var setupWindows = () => {
|
|
632
632
|
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
633
633
|
checkPath.convert = makePosix;
|
|
634
634
|
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
635
|
-
checkPath.isNotRelative = (
|
|
635
|
+
checkPath.isNotRelative = (path13) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path13) || isNotRelative(path13);
|
|
636
636
|
};
|
|
637
637
|
if (
|
|
638
638
|
// Detect `process` so that it can run in browsers.
|
|
@@ -649,7 +649,7 @@ var require_ignore = __commonJS({
|
|
|
649
649
|
|
|
650
650
|
// src/cli.ts
|
|
651
651
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
652
|
-
import * as
|
|
652
|
+
import * as path12 from "path";
|
|
653
653
|
|
|
654
654
|
// src/config/constants.ts
|
|
655
655
|
var DEFAULT_INCLUDE = [
|
|
@@ -658,12 +658,14 @@ var DEFAULT_INCLUDE = [
|
|
|
658
658
|
"**/*.{go,rs,java,kt,scala}",
|
|
659
659
|
"**/*.{c,cpp,cc,h,hpp}",
|
|
660
660
|
"**/*.{rb,php,inc,swift}",
|
|
661
|
+
"**/*.{cls,trigger}",
|
|
661
662
|
"**/*.{vue,svelte,astro}",
|
|
662
663
|
"**/*.{sql,graphql,proto}",
|
|
663
664
|
"**/*.{yaml,yml,toml}",
|
|
664
665
|
"**/*.{md,mdx}",
|
|
665
666
|
"**/*.{sh,bash,zsh}",
|
|
666
|
-
"**/*.{txt,html,htm}"
|
|
667
|
+
"**/*.{txt,html,htm}",
|
|
668
|
+
"**/*.zig"
|
|
667
669
|
];
|
|
668
670
|
var DEFAULT_EXCLUDE = [
|
|
669
671
|
"**/node_modules/**",
|
|
@@ -728,7 +730,7 @@ var EMBEDDING_MODELS = {
|
|
|
728
730
|
provider: "ollama",
|
|
729
731
|
model: "nomic-embed-text",
|
|
730
732
|
dimensions: 768,
|
|
731
|
-
maxTokens:
|
|
733
|
+
maxTokens: 2048,
|
|
732
734
|
costPer1MTokens: 0
|
|
733
735
|
},
|
|
734
736
|
"mxbai-embed-large": {
|
|
@@ -752,9 +754,15 @@ var EMBEDDING_MODELS = {
|
|
|
752
754
|
var DEFAULT_PROVIDER_MODELS = {
|
|
753
755
|
"github-copilot": "text-embedding-3-small",
|
|
754
756
|
"openai": "text-embedding-3-small",
|
|
755
|
-
"google": "
|
|
757
|
+
"google": "gemini-embedding-001",
|
|
756
758
|
"ollama": "nomic-embed-text"
|
|
757
759
|
};
|
|
760
|
+
var AUTO_DETECT_PROVIDER_ORDER = [
|
|
761
|
+
"ollama",
|
|
762
|
+
"github-copilot",
|
|
763
|
+
"openai",
|
|
764
|
+
"google"
|
|
765
|
+
];
|
|
758
766
|
|
|
759
767
|
// src/config/env-substitution.ts
|
|
760
768
|
var ENV_REFERENCE_PATTERN = /^\{env:([A-Z_][A-Z0-9_]*)\}$/;
|
|
@@ -1013,9 +1021,12 @@ function getDefaultModelForProvider(provider) {
|
|
|
1013
1021
|
return models[providerDefault];
|
|
1014
1022
|
}
|
|
1015
1023
|
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
1024
|
+
var autoDetectProviders = AUTO_DETECT_PROVIDER_ORDER.filter(
|
|
1025
|
+
(provider) => provider in EMBEDDING_MODELS
|
|
1026
|
+
);
|
|
1016
1027
|
|
|
1017
1028
|
// src/eval/cli.ts
|
|
1018
|
-
import * as
|
|
1029
|
+
import * as path10 from "path";
|
|
1019
1030
|
|
|
1020
1031
|
// src/eval/compare.ts
|
|
1021
1032
|
function metricDelta(current, baseline) {
|
|
@@ -1058,9 +1069,9 @@ function compareSummaries(current, baseline, againstPath) {
|
|
|
1058
1069
|
// src/eval/reports.ts
|
|
1059
1070
|
import { mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
1060
1071
|
import * as path from "path";
|
|
1061
|
-
function assertFiniteNumber(value,
|
|
1072
|
+
function assertFiniteNumber(value, path13) {
|
|
1062
1073
|
if (typeof value !== "number" || Number.isNaN(value) || !Number.isFinite(value)) {
|
|
1063
|
-
throw new Error(`${
|
|
1074
|
+
throw new Error(`${path13} must be a finite number`);
|
|
1064
1075
|
}
|
|
1065
1076
|
return value;
|
|
1066
1077
|
}
|
|
@@ -1234,16 +1245,352 @@ function buildPerQueryArtifact(perQuery) {
|
|
|
1234
1245
|
}
|
|
1235
1246
|
|
|
1236
1247
|
// src/eval/runner.ts
|
|
1237
|
-
import { existsSync as
|
|
1238
|
-
import {
|
|
1248
|
+
import { existsSync as existsSync7 } from "fs";
|
|
1249
|
+
import { mkdirSync as mkdirSync3 } from "fs";
|
|
1250
|
+
import { readFileSync as readFileSync8 } from "fs";
|
|
1239
1251
|
import { rmSync } from "fs";
|
|
1240
|
-
import
|
|
1241
|
-
import * as
|
|
1252
|
+
import { writeFileSync as writeFileSync4 } from "fs";
|
|
1253
|
+
import * as os5 from "os";
|
|
1254
|
+
import * as path9 from "path";
|
|
1242
1255
|
import { performance as performance3 } from "perf_hooks";
|
|
1243
1256
|
|
|
1257
|
+
// src/config/merger.ts
|
|
1258
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
1259
|
+
import * as os2 from "os";
|
|
1260
|
+
import * as path4 from "path";
|
|
1261
|
+
|
|
1262
|
+
// src/config/paths.ts
|
|
1263
|
+
import { existsSync as existsSync2 } from "fs";
|
|
1264
|
+
import * as os from "os";
|
|
1265
|
+
import * as path3 from "path";
|
|
1266
|
+
|
|
1267
|
+
// src/git/index.ts
|
|
1268
|
+
import { existsSync, readFileSync as readFileSync2, readdirSync, statSync } from "fs";
|
|
1269
|
+
import * as path2 from "path";
|
|
1270
|
+
function readPackedRefs(gitDir) {
|
|
1271
|
+
const packedRefsPath = path2.join(gitDir, "packed-refs");
|
|
1272
|
+
if (!existsSync(packedRefsPath)) {
|
|
1273
|
+
return [];
|
|
1274
|
+
}
|
|
1275
|
+
try {
|
|
1276
|
+
return readFileSync2(packedRefsPath, "utf-8").split("\n").map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("^"));
|
|
1277
|
+
} catch {
|
|
1278
|
+
return [];
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
function resolveCommonGitDir(gitDir) {
|
|
1282
|
+
const commonDirPath = path2.join(gitDir, "commondir");
|
|
1283
|
+
if (!existsSync(commonDirPath)) {
|
|
1284
|
+
return gitDir;
|
|
1285
|
+
}
|
|
1286
|
+
try {
|
|
1287
|
+
const raw = readFileSync2(commonDirPath, "utf-8").trim();
|
|
1288
|
+
if (!raw) {
|
|
1289
|
+
return gitDir;
|
|
1290
|
+
}
|
|
1291
|
+
const resolved = path2.isAbsolute(raw) ? raw : path2.resolve(gitDir, raw);
|
|
1292
|
+
if (existsSync(resolved)) {
|
|
1293
|
+
return resolved;
|
|
1294
|
+
}
|
|
1295
|
+
} catch {
|
|
1296
|
+
return gitDir;
|
|
1297
|
+
}
|
|
1298
|
+
return gitDir;
|
|
1299
|
+
}
|
|
1300
|
+
function resolveWorktreeMainRepoRoot(repoRoot) {
|
|
1301
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
1302
|
+
if (!gitDir) {
|
|
1303
|
+
return null;
|
|
1304
|
+
}
|
|
1305
|
+
const commonGitDir = resolveCommonGitDir(gitDir);
|
|
1306
|
+
if (commonGitDir === gitDir || path2.basename(commonGitDir) !== ".git") {
|
|
1307
|
+
return null;
|
|
1308
|
+
}
|
|
1309
|
+
const mainRepoRoot = path2.dirname(commonGitDir);
|
|
1310
|
+
if (!existsSync(mainRepoRoot)) {
|
|
1311
|
+
return null;
|
|
1312
|
+
}
|
|
1313
|
+
return path2.resolve(mainRepoRoot) === path2.resolve(repoRoot) ? null : mainRepoRoot;
|
|
1314
|
+
}
|
|
1315
|
+
function resolveGitDir(repoRoot) {
|
|
1316
|
+
const gitPath = path2.join(repoRoot, ".git");
|
|
1317
|
+
if (!existsSync(gitPath)) {
|
|
1318
|
+
return null;
|
|
1319
|
+
}
|
|
1320
|
+
try {
|
|
1321
|
+
const stat = statSync(gitPath);
|
|
1322
|
+
if (stat.isDirectory()) {
|
|
1323
|
+
return gitPath;
|
|
1324
|
+
}
|
|
1325
|
+
if (stat.isFile()) {
|
|
1326
|
+
const content = readFileSync2(gitPath, "utf-8").trim();
|
|
1327
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
1328
|
+
if (match) {
|
|
1329
|
+
const gitdir = match[1];
|
|
1330
|
+
const resolvedPath = path2.isAbsolute(gitdir) ? gitdir : path2.resolve(repoRoot, gitdir);
|
|
1331
|
+
if (existsSync(resolvedPath)) {
|
|
1332
|
+
return resolvedPath;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
} catch {
|
|
1337
|
+
}
|
|
1338
|
+
return null;
|
|
1339
|
+
}
|
|
1340
|
+
function isGitRepo(dir) {
|
|
1341
|
+
return resolveGitDir(dir) !== null;
|
|
1342
|
+
}
|
|
1343
|
+
function getCurrentBranch(repoRoot) {
|
|
1344
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
1345
|
+
if (!gitDir) {
|
|
1346
|
+
return null;
|
|
1347
|
+
}
|
|
1348
|
+
const headPath = path2.join(gitDir, "HEAD");
|
|
1349
|
+
if (!existsSync(headPath)) {
|
|
1350
|
+
return null;
|
|
1351
|
+
}
|
|
1352
|
+
try {
|
|
1353
|
+
const headContent = readFileSync2(headPath, "utf-8").trim();
|
|
1354
|
+
const match = headContent.match(/^ref: refs\/heads\/(.+)$/);
|
|
1355
|
+
if (match) {
|
|
1356
|
+
return match[1];
|
|
1357
|
+
}
|
|
1358
|
+
if (/^[0-9a-f]{40}$/i.test(headContent)) {
|
|
1359
|
+
return headContent.slice(0, 7);
|
|
1360
|
+
}
|
|
1361
|
+
return null;
|
|
1362
|
+
} catch {
|
|
1363
|
+
return null;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
function getBaseBranch(repoRoot) {
|
|
1367
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
1368
|
+
const refStoreDir = gitDir ? resolveCommonGitDir(gitDir) : null;
|
|
1369
|
+
const candidates = ["main", "master", "develop", "trunk"];
|
|
1370
|
+
if (refStoreDir) {
|
|
1371
|
+
for (const candidate of candidates) {
|
|
1372
|
+
const refPath = path2.join(refStoreDir, "refs", "heads", candidate);
|
|
1373
|
+
if (existsSync(refPath)) {
|
|
1374
|
+
return candidate;
|
|
1375
|
+
}
|
|
1376
|
+
const packedRefs = readPackedRefs(refStoreDir);
|
|
1377
|
+
if (packedRefs.some((line) => line.endsWith(` refs/heads/${candidate}`))) {
|
|
1378
|
+
return candidate;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
return getCurrentBranch(repoRoot) ?? "main";
|
|
1383
|
+
}
|
|
1384
|
+
function getBranchOrDefault(repoRoot) {
|
|
1385
|
+
if (!isGitRepo(repoRoot)) {
|
|
1386
|
+
return "default";
|
|
1387
|
+
}
|
|
1388
|
+
return getCurrentBranch(repoRoot) ?? "default";
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
// src/config/paths.ts
|
|
1392
|
+
var PROJECT_CONFIG_RELATIVE_PATH = path3.join(".opencode", "codebase-index.json");
|
|
1393
|
+
var PROJECT_INDEX_RELATIVE_PATH = path3.join(".opencode", "index");
|
|
1394
|
+
function resolveWorktreeFallbackPath(projectRoot, relativePath) {
|
|
1395
|
+
const mainRepoRoot = resolveWorktreeMainRepoRoot(projectRoot);
|
|
1396
|
+
if (!mainRepoRoot) {
|
|
1397
|
+
return null;
|
|
1398
|
+
}
|
|
1399
|
+
const fallbackPath = path3.join(mainRepoRoot, relativePath);
|
|
1400
|
+
return existsSync2(fallbackPath) ? fallbackPath : null;
|
|
1401
|
+
}
|
|
1402
|
+
function hasProjectConfig(projectRoot) {
|
|
1403
|
+
return existsSync2(path3.join(projectRoot, PROJECT_CONFIG_RELATIVE_PATH));
|
|
1404
|
+
}
|
|
1405
|
+
function getGlobalIndexPath() {
|
|
1406
|
+
return path3.join(os.homedir(), ".opencode", "global-index");
|
|
1407
|
+
}
|
|
1408
|
+
function resolveProjectConfigPath(projectRoot) {
|
|
1409
|
+
const localConfigPath = path3.join(projectRoot, PROJECT_CONFIG_RELATIVE_PATH);
|
|
1410
|
+
if (existsSync2(localConfigPath)) {
|
|
1411
|
+
return localConfigPath;
|
|
1412
|
+
}
|
|
1413
|
+
return resolveWorktreeFallbackPath(projectRoot, PROJECT_CONFIG_RELATIVE_PATH) ?? localConfigPath;
|
|
1414
|
+
}
|
|
1415
|
+
function resolveProjectIndexPath(projectRoot, scope) {
|
|
1416
|
+
if (scope === "global") {
|
|
1417
|
+
return getGlobalIndexPath();
|
|
1418
|
+
}
|
|
1419
|
+
const localIndexPath = path3.join(projectRoot, PROJECT_INDEX_RELATIVE_PATH);
|
|
1420
|
+
if (existsSync2(localIndexPath)) {
|
|
1421
|
+
return localIndexPath;
|
|
1422
|
+
}
|
|
1423
|
+
if (hasProjectConfig(projectRoot)) {
|
|
1424
|
+
return localIndexPath;
|
|
1425
|
+
}
|
|
1426
|
+
return resolveWorktreeFallbackPath(projectRoot, PROJECT_INDEX_RELATIVE_PATH) ?? localIndexPath;
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
// src/config/merger.ts
|
|
1430
|
+
function loadJsonFile(filePath) {
|
|
1431
|
+
try {
|
|
1432
|
+
if (existsSync3(filePath)) {
|
|
1433
|
+
const content = readFileSync3(filePath, "utf-8");
|
|
1434
|
+
return JSON.parse(content);
|
|
1435
|
+
}
|
|
1436
|
+
} catch {
|
|
1437
|
+
}
|
|
1438
|
+
return null;
|
|
1439
|
+
}
|
|
1440
|
+
function normalizeRelativeConfigPath(candidate) {
|
|
1441
|
+
return candidate.replace(/\\/g, "/");
|
|
1442
|
+
}
|
|
1443
|
+
function rebasePathEntries(values, fromDir, toDir) {
|
|
1444
|
+
if (!Array.isArray(values)) {
|
|
1445
|
+
return [];
|
|
1446
|
+
}
|
|
1447
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
1448
|
+
const trimmed = value.trim();
|
|
1449
|
+
if (!trimmed || path4.isAbsolute(trimmed)) {
|
|
1450
|
+
return trimmed;
|
|
1451
|
+
}
|
|
1452
|
+
return normalizeRelativeConfigPath(path4.normalize(path4.relative(toDir, path4.resolve(fromDir, trimmed))));
|
|
1453
|
+
}).filter(Boolean);
|
|
1454
|
+
}
|
|
1455
|
+
function isWithinRoot(rootDir, targetPath) {
|
|
1456
|
+
const relativePath = path4.relative(rootDir, targetPath);
|
|
1457
|
+
return relativePath === "" || !relativePath.startsWith("..") && !path4.isAbsolute(relativePath);
|
|
1458
|
+
}
|
|
1459
|
+
function resolveInheritedKnowledgeBaseEntries(values, sourceRoot, targetRoot) {
|
|
1460
|
+
if (!Array.isArray(values)) {
|
|
1461
|
+
return [];
|
|
1462
|
+
}
|
|
1463
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
1464
|
+
const trimmed = value.trim();
|
|
1465
|
+
if (!trimmed) {
|
|
1466
|
+
return trimmed;
|
|
1467
|
+
}
|
|
1468
|
+
if (path4.isAbsolute(trimmed)) {
|
|
1469
|
+
if (isWithinRoot(sourceRoot, trimmed)) {
|
|
1470
|
+
return normalizeRelativeConfigPath(path4.normalize(path4.relative(sourceRoot, trimmed) || "."));
|
|
1471
|
+
}
|
|
1472
|
+
return path4.normalize(trimmed);
|
|
1473
|
+
}
|
|
1474
|
+
const resolvedFromSource = path4.resolve(sourceRoot, trimmed);
|
|
1475
|
+
if (isWithinRoot(sourceRoot, resolvedFromSource)) {
|
|
1476
|
+
return normalizeRelativeConfigPath(path4.normalize(trimmed));
|
|
1477
|
+
}
|
|
1478
|
+
return normalizeRelativeConfigPath(path4.normalize(path4.relative(targetRoot, resolvedFromSource)));
|
|
1479
|
+
}).filter(Boolean);
|
|
1480
|
+
}
|
|
1481
|
+
function materializeLocalProjectConfig(projectRoot, config) {
|
|
1482
|
+
const localConfigPath = path4.join(projectRoot, ".opencode", "codebase-index.json");
|
|
1483
|
+
mkdirSync2(path4.dirname(localConfigPath), { recursive: true });
|
|
1484
|
+
writeFileSync2(localConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
1485
|
+
return localConfigPath;
|
|
1486
|
+
}
|
|
1487
|
+
function loadProjectConfigLayer(projectRoot) {
|
|
1488
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot);
|
|
1489
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
1490
|
+
if (!projectConfig) {
|
|
1491
|
+
return {};
|
|
1492
|
+
}
|
|
1493
|
+
const normalizedConfig = { ...projectConfig };
|
|
1494
|
+
const projectConfigBaseDir = path4.dirname(path4.dirname(projectConfigPath));
|
|
1495
|
+
if (Array.isArray(normalizedConfig.knowledgeBases)) {
|
|
1496
|
+
normalizedConfig.knowledgeBases = resolveInheritedKnowledgeBaseEntries(
|
|
1497
|
+
normalizedConfig.knowledgeBases,
|
|
1498
|
+
projectConfigBaseDir,
|
|
1499
|
+
projectRoot
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
return normalizedConfig;
|
|
1503
|
+
}
|
|
1504
|
+
function loadMergedConfig(projectRoot) {
|
|
1505
|
+
const globalConfigPath = os2.homedir() + "/.config/opencode/codebase-index.json";
|
|
1506
|
+
const globalConfig = loadJsonFile(globalConfigPath);
|
|
1507
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot);
|
|
1508
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
1509
|
+
const normalizedProjectConfig = loadProjectConfigLayer(projectRoot);
|
|
1510
|
+
if (!globalConfig && !projectConfig) {
|
|
1511
|
+
return {};
|
|
1512
|
+
}
|
|
1513
|
+
if (!projectConfig && globalConfig) {
|
|
1514
|
+
return globalConfig;
|
|
1515
|
+
}
|
|
1516
|
+
if (!globalConfig && projectConfig) {
|
|
1517
|
+
return normalizedProjectConfig;
|
|
1518
|
+
}
|
|
1519
|
+
const merged = { ...globalConfig };
|
|
1520
|
+
if (projectConfig && "embeddingProvider" in normalizedProjectConfig) {
|
|
1521
|
+
merged.embeddingProvider = normalizedProjectConfig.embeddingProvider;
|
|
1522
|
+
} else if (globalConfig && globalConfig.embeddingProvider) {
|
|
1523
|
+
merged.embeddingProvider = globalConfig.embeddingProvider;
|
|
1524
|
+
}
|
|
1525
|
+
if (projectConfig && "customProvider" in normalizedProjectConfig) {
|
|
1526
|
+
merged.customProvider = normalizedProjectConfig.customProvider;
|
|
1527
|
+
} else if (globalConfig && globalConfig.customProvider) {
|
|
1528
|
+
merged.customProvider = globalConfig.customProvider;
|
|
1529
|
+
}
|
|
1530
|
+
if (projectConfig && "embeddingModel" in normalizedProjectConfig) {
|
|
1531
|
+
merged.embeddingModel = normalizedProjectConfig.embeddingModel;
|
|
1532
|
+
} else if (globalConfig && globalConfig.embeddingModel) {
|
|
1533
|
+
merged.embeddingModel = globalConfig.embeddingModel;
|
|
1534
|
+
}
|
|
1535
|
+
if (projectConfig && "reranker" in normalizedProjectConfig) {
|
|
1536
|
+
merged.reranker = normalizedProjectConfig.reranker;
|
|
1537
|
+
} else if (globalConfig && globalConfig.reranker) {
|
|
1538
|
+
merged.reranker = globalConfig.reranker;
|
|
1539
|
+
}
|
|
1540
|
+
if (projectConfig && "include" in normalizedProjectConfig) {
|
|
1541
|
+
merged.include = normalizedProjectConfig.include;
|
|
1542
|
+
} else if (globalConfig && globalConfig.include) {
|
|
1543
|
+
merged.include = globalConfig.include;
|
|
1544
|
+
}
|
|
1545
|
+
if (projectConfig && "exclude" in normalizedProjectConfig) {
|
|
1546
|
+
merged.exclude = normalizedProjectConfig.exclude;
|
|
1547
|
+
} else if (globalConfig && globalConfig.exclude) {
|
|
1548
|
+
merged.exclude = globalConfig.exclude;
|
|
1549
|
+
}
|
|
1550
|
+
if (projectConfig && "indexing" in normalizedProjectConfig) {
|
|
1551
|
+
merged.indexing = normalizedProjectConfig.indexing;
|
|
1552
|
+
} else if (globalConfig && globalConfig.indexing) {
|
|
1553
|
+
merged.indexing = globalConfig.indexing;
|
|
1554
|
+
}
|
|
1555
|
+
if (projectConfig && "search" in normalizedProjectConfig) {
|
|
1556
|
+
merged.search = normalizedProjectConfig.search;
|
|
1557
|
+
} else if (globalConfig && globalConfig.search) {
|
|
1558
|
+
merged.search = globalConfig.search;
|
|
1559
|
+
}
|
|
1560
|
+
if (projectConfig && "debug" in normalizedProjectConfig) {
|
|
1561
|
+
merged.debug = normalizedProjectConfig.debug;
|
|
1562
|
+
} else if (globalConfig && globalConfig.debug) {
|
|
1563
|
+
merged.debug = globalConfig.debug;
|
|
1564
|
+
}
|
|
1565
|
+
if (projectConfig && "scope" in normalizedProjectConfig) {
|
|
1566
|
+
merged.scope = normalizedProjectConfig.scope;
|
|
1567
|
+
} else if (globalConfig && "scope" in globalConfig) {
|
|
1568
|
+
merged.scope = globalConfig.scope;
|
|
1569
|
+
}
|
|
1570
|
+
if (projectConfig) {
|
|
1571
|
+
for (const key of Object.keys(projectConfig)) {
|
|
1572
|
+
if (key === "embeddingProvider" || key === "customProvider" || key === "embeddingModel" || key === "reranker" || key === "include" || key === "exclude" || key === "indexing" || key === "search" || key === "debug" || key === "scope" || key === "knowledgeBases" || key === "additionalInclude") {
|
|
1573
|
+
continue;
|
|
1574
|
+
}
|
|
1575
|
+
merged[key] = normalizedProjectConfig[key];
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
1579
|
+
const projectKbs = projectConfig ? Array.isArray(normalizedProjectConfig.knowledgeBases) ? normalizedProjectConfig.knowledgeBases : [] : [];
|
|
1580
|
+
const allKbs = [...globalKbs, ...projectKbs];
|
|
1581
|
+
const uniqueKbs = [...new Set(allKbs.map((p) => String(p).trim()))];
|
|
1582
|
+
merged.knowledgeBases = uniqueKbs;
|
|
1583
|
+
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
1584
|
+
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
1585
|
+
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
1586
|
+
const uniqueAdditional = [...new Set(allAdditional.map((p) => String(p).trim()))];
|
|
1587
|
+
merged.additionalInclude = uniqueAdditional;
|
|
1588
|
+
return merged;
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1244
1591
|
// src/indexer/index.ts
|
|
1245
|
-
import { existsSync as
|
|
1246
|
-
import * as
|
|
1592
|
+
import { existsSync as existsSync6, readFileSync as readFileSync6, writeFileSync as writeFileSync3, renameSync, unlinkSync, promises as fsPromises2 } from "fs";
|
|
1593
|
+
import * as path8 from "path";
|
|
1247
1594
|
import { performance as performance2 } from "perf_hooks";
|
|
1248
1595
|
|
|
1249
1596
|
// node_modules/eventemitter3/index.mjs
|
|
@@ -1268,7 +1615,7 @@ function pTimeout(promise, options) {
|
|
|
1268
1615
|
} = options;
|
|
1269
1616
|
let timer;
|
|
1270
1617
|
let abortHandler;
|
|
1271
|
-
const wrappedPromise = new Promise((
|
|
1618
|
+
const wrappedPromise = new Promise((resolve8, reject) => {
|
|
1272
1619
|
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
1273
1620
|
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
1274
1621
|
}
|
|
@@ -1282,7 +1629,7 @@ function pTimeout(promise, options) {
|
|
|
1282
1629
|
};
|
|
1283
1630
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
1284
1631
|
}
|
|
1285
|
-
promise.then(
|
|
1632
|
+
promise.then(resolve8, reject);
|
|
1286
1633
|
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
1287
1634
|
return;
|
|
1288
1635
|
}
|
|
@@ -1290,7 +1637,7 @@ function pTimeout(promise, options) {
|
|
|
1290
1637
|
timer = customTimers.setTimeout.call(void 0, () => {
|
|
1291
1638
|
if (fallback) {
|
|
1292
1639
|
try {
|
|
1293
|
-
|
|
1640
|
+
resolve8(fallback());
|
|
1294
1641
|
} catch (error) {
|
|
1295
1642
|
reject(error);
|
|
1296
1643
|
}
|
|
@@ -1300,7 +1647,7 @@ function pTimeout(promise, options) {
|
|
|
1300
1647
|
promise.cancel();
|
|
1301
1648
|
}
|
|
1302
1649
|
if (message === false) {
|
|
1303
|
-
|
|
1650
|
+
resolve8();
|
|
1304
1651
|
} else if (message instanceof Error) {
|
|
1305
1652
|
reject(message);
|
|
1306
1653
|
} else {
|
|
@@ -1702,7 +2049,7 @@ var PQueue = class extends import_index.default {
|
|
|
1702
2049
|
// Assign unique ID if not provided
|
|
1703
2050
|
id: options.id ?? (this.#idAssigner++).toString()
|
|
1704
2051
|
};
|
|
1705
|
-
return new Promise((
|
|
2052
|
+
return new Promise((resolve8, reject) => {
|
|
1706
2053
|
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1707
2054
|
let cleanupQueueAbortHandler = () => void 0;
|
|
1708
2055
|
const run = async () => {
|
|
@@ -1742,7 +2089,7 @@ var PQueue = class extends import_index.default {
|
|
|
1742
2089
|
})]);
|
|
1743
2090
|
}
|
|
1744
2091
|
const result = await operation;
|
|
1745
|
-
|
|
2092
|
+
resolve8(result);
|
|
1746
2093
|
this.emit("completed", result);
|
|
1747
2094
|
} catch (error) {
|
|
1748
2095
|
reject(error);
|
|
@@ -1930,13 +2277,13 @@ var PQueue = class extends import_index.default {
|
|
|
1930
2277
|
});
|
|
1931
2278
|
}
|
|
1932
2279
|
async #onEvent(event, filter) {
|
|
1933
|
-
return new Promise((
|
|
2280
|
+
return new Promise((resolve8) => {
|
|
1934
2281
|
const listener = () => {
|
|
1935
2282
|
if (filter && !filter()) {
|
|
1936
2283
|
return;
|
|
1937
2284
|
}
|
|
1938
2285
|
this.off(event, listener);
|
|
1939
|
-
|
|
2286
|
+
resolve8();
|
|
1940
2287
|
};
|
|
1941
2288
|
this.on(event, listener);
|
|
1942
2289
|
});
|
|
@@ -2222,7 +2569,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
2222
2569
|
const finalDelay = Math.min(delayTime, remainingTime);
|
|
2223
2570
|
options.signal?.throwIfAborted();
|
|
2224
2571
|
if (finalDelay > 0) {
|
|
2225
|
-
await new Promise((
|
|
2572
|
+
await new Promise((resolve8, reject) => {
|
|
2226
2573
|
const onAbort = () => {
|
|
2227
2574
|
clearTimeout(timeoutToken);
|
|
2228
2575
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -2230,7 +2577,7 @@ async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTi
|
|
|
2230
2577
|
};
|
|
2231
2578
|
const timeoutToken = setTimeout(() => {
|
|
2232
2579
|
options.signal?.removeEventListener("abort", onAbort);
|
|
2233
|
-
|
|
2580
|
+
resolve8();
|
|
2234
2581
|
}, finalDelay);
|
|
2235
2582
|
if (options.unref) {
|
|
2236
2583
|
timeoutToken.unref?.();
|
|
@@ -2291,17 +2638,17 @@ async function pRetry(input, options = {}) {
|
|
|
2291
2638
|
}
|
|
2292
2639
|
|
|
2293
2640
|
// src/embeddings/detector.ts
|
|
2294
|
-
import { existsSync, readFileSync as
|
|
2295
|
-
import * as
|
|
2296
|
-
import * as
|
|
2641
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
|
|
2642
|
+
import * as path5 from "path";
|
|
2643
|
+
import * as os3 from "os";
|
|
2297
2644
|
function getOpenCodeAuthPath() {
|
|
2298
|
-
return
|
|
2645
|
+
return path5.join(os3.homedir(), ".local", "share", "opencode", "auth.json");
|
|
2299
2646
|
}
|
|
2300
2647
|
function loadOpenCodeAuth() {
|
|
2301
2648
|
const authPath = getOpenCodeAuthPath();
|
|
2302
2649
|
try {
|
|
2303
|
-
if (
|
|
2304
|
-
return JSON.parse(
|
|
2650
|
+
if (existsSync4(authPath)) {
|
|
2651
|
+
return JSON.parse(readFileSync4(authPath, "utf-8"));
|
|
2305
2652
|
}
|
|
2306
2653
|
} catch {
|
|
2307
2654
|
}
|
|
@@ -2334,7 +2681,7 @@ async function detectEmbeddingProvider(preferredProvider, model) {
|
|
|
2334
2681
|
);
|
|
2335
2682
|
}
|
|
2336
2683
|
async function tryDetectProvider() {
|
|
2337
|
-
for (const provider of
|
|
2684
|
+
for (const provider of autoDetectProviders) {
|
|
2338
2685
|
const credentials = await getProviderCredentials(provider);
|
|
2339
2686
|
if (credentials) {
|
|
2340
2687
|
return {
|
|
@@ -2345,7 +2692,7 @@ async function tryDetectProvider() {
|
|
|
2345
2692
|
}
|
|
2346
2693
|
}
|
|
2347
2694
|
throw new Error(
|
|
2348
|
-
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${
|
|
2695
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${autoDetectProviders.join(", ")}.`
|
|
2349
2696
|
);
|
|
2350
2697
|
}
|
|
2351
2698
|
async function getProviderCredentials(provider) {
|
|
@@ -2665,11 +3012,12 @@ var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
|
2665
3012
|
return this.modelInfo;
|
|
2666
3013
|
}
|
|
2667
3014
|
};
|
|
2668
|
-
var OllamaEmbeddingProvider = class {
|
|
3015
|
+
var OllamaEmbeddingProvider = class _OllamaEmbeddingProvider {
|
|
2669
3016
|
constructor(credentials, modelInfo) {
|
|
2670
3017
|
this.credentials = credentials;
|
|
2671
3018
|
this.modelInfo = modelInfo;
|
|
2672
3019
|
}
|
|
3020
|
+
static MIN_TRUNCATION_CHARS = 512;
|
|
2673
3021
|
async embedQuery(query) {
|
|
2674
3022
|
const result = await this.embedBatch([query]);
|
|
2675
3023
|
return {
|
|
@@ -2684,52 +3032,125 @@ var OllamaEmbeddingProvider = class {
|
|
|
2684
3032
|
tokensUsed: result.totalTokensUsed
|
|
2685
3033
|
};
|
|
2686
3034
|
}
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
texts.map(async (text) => {
|
|
2690
|
-
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2691
|
-
method: "POST",
|
|
2692
|
-
headers: {
|
|
2693
|
-
"Content-Type": "application/json"
|
|
2694
|
-
},
|
|
2695
|
-
body: JSON.stringify({
|
|
2696
|
-
model: this.modelInfo.model,
|
|
2697
|
-
prompt: text
|
|
2698
|
-
})
|
|
2699
|
-
});
|
|
2700
|
-
if (!response.ok) {
|
|
2701
|
-
const error = await response.text();
|
|
2702
|
-
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2703
|
-
}
|
|
2704
|
-
const data = await response.json();
|
|
2705
|
-
return {
|
|
2706
|
-
embedding: data.embedding,
|
|
2707
|
-
tokensUsed: Math.ceil(text.length / 4)
|
|
2708
|
-
};
|
|
2709
|
-
})
|
|
2710
|
-
);
|
|
2711
|
-
return {
|
|
2712
|
-
embeddings: results.map((r) => r.embedding),
|
|
2713
|
-
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2714
|
-
};
|
|
3035
|
+
estimateTokens(text) {
|
|
3036
|
+
return Math.ceil(text.length / 4);
|
|
2715
3037
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
3038
|
+
truncateToCharLimit(text, maxChars) {
|
|
3039
|
+
if (text.length <= maxChars) {
|
|
3040
|
+
return text;
|
|
3041
|
+
}
|
|
3042
|
+
return `${text.slice(0, Math.max(0, maxChars - 17))}
|
|
3043
|
+
... [truncated]`;
|
|
2718
3044
|
}
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
this.credentials = credentials;
|
|
2723
|
-
this.modelInfo = modelInfo;
|
|
3045
|
+
isContextLengthError(error) {
|
|
3046
|
+
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
3047
|
+
return message.includes("context length") && (message.includes("exceed") || message.includes("exceeded") || message.includes("too long")) || message.includes("input length exceeds the context length") || message.includes("context length exceeded");
|
|
2724
3048
|
}
|
|
2725
|
-
|
|
2726
|
-
const
|
|
2727
|
-
|
|
2728
|
-
|
|
3049
|
+
buildTruncationCandidates(text) {
|
|
3050
|
+
const baseMaxChars = Math.max(1, this.modelInfo.maxTokens * 4);
|
|
3051
|
+
const candidateLimits = /* @__PURE__ */ new Set();
|
|
3052
|
+
const baselineLimit = text.length > baseMaxChars ? baseMaxChars : Math.max(
|
|
3053
|
+
_OllamaEmbeddingProvider.MIN_TRUNCATION_CHARS,
|
|
3054
|
+
Math.floor(text.length * 0.9)
|
|
3055
|
+
);
|
|
3056
|
+
if (baselineLimit < text.length) {
|
|
3057
|
+
candidateLimits.add(baselineLimit);
|
|
2729
3058
|
}
|
|
2730
|
-
const
|
|
2731
|
-
|
|
2732
|
-
|
|
3059
|
+
for (const factor of [0.75, 0.6, 0.45, 0.35, 0.25]) {
|
|
3060
|
+
const scaledLimit = Math.max(
|
|
3061
|
+
_OllamaEmbeddingProvider.MIN_TRUNCATION_CHARS,
|
|
3062
|
+
Math.floor(baselineLimit * factor)
|
|
3063
|
+
);
|
|
3064
|
+
if (scaledLimit < text.length) {
|
|
3065
|
+
candidateLimits.add(scaledLimit);
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
candidateLimits.add(Math.min(text.length - 1, _OllamaEmbeddingProvider.MIN_TRUNCATION_CHARS));
|
|
3069
|
+
const candidates = [];
|
|
3070
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3071
|
+
for (const limit of [...candidateLimits].sort((a, b) => b - a)) {
|
|
3072
|
+
if (limit <= 0 || limit >= text.length) {
|
|
3073
|
+
continue;
|
|
3074
|
+
}
|
|
3075
|
+
const truncated = this.truncateToCharLimit(text, limit);
|
|
3076
|
+
if (truncated === text || seen.has(truncated)) {
|
|
3077
|
+
continue;
|
|
3078
|
+
}
|
|
3079
|
+
seen.add(truncated);
|
|
3080
|
+
candidates.push(truncated);
|
|
3081
|
+
}
|
|
3082
|
+
return candidates;
|
|
3083
|
+
}
|
|
3084
|
+
async embedSingleWithFallback(text) {
|
|
3085
|
+
try {
|
|
3086
|
+
return await this.embedSingle(text);
|
|
3087
|
+
} catch (error) {
|
|
3088
|
+
if (!this.isContextLengthError(error)) {
|
|
3089
|
+
throw error;
|
|
3090
|
+
}
|
|
3091
|
+
let lastError = error;
|
|
3092
|
+
for (const truncated of this.buildTruncationCandidates(text)) {
|
|
3093
|
+
try {
|
|
3094
|
+
return await this.embedSingle(truncated);
|
|
3095
|
+
} catch (retryError) {
|
|
3096
|
+
if (!this.isContextLengthError(retryError)) {
|
|
3097
|
+
throw retryError;
|
|
3098
|
+
}
|
|
3099
|
+
lastError = retryError;
|
|
3100
|
+
}
|
|
3101
|
+
}
|
|
3102
|
+
throw lastError;
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
async embedSingle(text) {
|
|
3106
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
3107
|
+
method: "POST",
|
|
3108
|
+
headers: {
|
|
3109
|
+
"Content-Type": "application/json"
|
|
3110
|
+
},
|
|
3111
|
+
body: JSON.stringify({
|
|
3112
|
+
model: this.modelInfo.model,
|
|
3113
|
+
prompt: text,
|
|
3114
|
+
truncate: false
|
|
3115
|
+
})
|
|
3116
|
+
});
|
|
3117
|
+
if (!response.ok) {
|
|
3118
|
+
const error = await response.text();
|
|
3119
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
3120
|
+
}
|
|
3121
|
+
const data = await response.json();
|
|
3122
|
+
return {
|
|
3123
|
+
embedding: data.embedding,
|
|
3124
|
+
tokensUsed: this.estimateTokens(text)
|
|
3125
|
+
};
|
|
3126
|
+
}
|
|
3127
|
+
async embedBatch(texts) {
|
|
3128
|
+
const results = [];
|
|
3129
|
+
for (const text of texts) {
|
|
3130
|
+
results.push(await this.embedSingleWithFallback(text));
|
|
3131
|
+
}
|
|
3132
|
+
return {
|
|
3133
|
+
embeddings: results.map((r) => r.embedding),
|
|
3134
|
+
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
3135
|
+
};
|
|
3136
|
+
}
|
|
3137
|
+
getModelInfo() {
|
|
3138
|
+
return this.modelInfo;
|
|
3139
|
+
}
|
|
3140
|
+
};
|
|
3141
|
+
var CustomEmbeddingProvider = class {
|
|
3142
|
+
constructor(credentials, modelInfo) {
|
|
3143
|
+
this.credentials = credentials;
|
|
3144
|
+
this.modelInfo = modelInfo;
|
|
3145
|
+
}
|
|
3146
|
+
splitIntoRequestBatches(texts) {
|
|
3147
|
+
const maxBatchSize = this.modelInfo.maxBatchSize;
|
|
3148
|
+
if (!maxBatchSize || texts.length <= maxBatchSize) {
|
|
3149
|
+
return [texts];
|
|
3150
|
+
}
|
|
3151
|
+
const batches = [];
|
|
3152
|
+
for (let i = 0; i < texts.length; i += maxBatchSize) {
|
|
3153
|
+
batches.push(texts.slice(i, i + maxBatchSize));
|
|
2733
3154
|
}
|
|
2734
3155
|
return batches;
|
|
2735
3156
|
}
|
|
@@ -2911,8 +3332,8 @@ var SiliconFlowReranker = class {
|
|
|
2911
3332
|
|
|
2912
3333
|
// src/utils/files.ts
|
|
2913
3334
|
var import_ignore = __toESM(require_ignore(), 1);
|
|
2914
|
-
import { existsSync as
|
|
2915
|
-
import * as
|
|
3335
|
+
import { existsSync as existsSync5, readFileSync as readFileSync5, promises as fsPromises } from "fs";
|
|
3336
|
+
import * as path6 from "path";
|
|
2916
3337
|
function createIgnoreFilter(projectRoot) {
|
|
2917
3338
|
const ig = (0, import_ignore.default)();
|
|
2918
3339
|
const defaultIgnores = [
|
|
@@ -2933,9 +3354,9 @@ function createIgnoreFilter(projectRoot) {
|
|
|
2933
3354
|
"**/*build*/**"
|
|
2934
3355
|
];
|
|
2935
3356
|
ig.add(defaultIgnores);
|
|
2936
|
-
const gitignorePath =
|
|
2937
|
-
if (
|
|
2938
|
-
const gitignoreContent =
|
|
3357
|
+
const gitignorePath = path6.join(projectRoot, ".gitignore");
|
|
3358
|
+
if (existsSync5(gitignorePath)) {
|
|
3359
|
+
const gitignoreContent = readFileSync5(gitignorePath, "utf-8");
|
|
2939
3360
|
ig.add(gitignoreContent);
|
|
2940
3361
|
}
|
|
2941
3362
|
return ig;
|
|
@@ -2960,8 +3381,8 @@ async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns
|
|
|
2960
3381
|
const filesInDir = [];
|
|
2961
3382
|
const subdirs = [];
|
|
2962
3383
|
for (const entry of entries) {
|
|
2963
|
-
const fullPath =
|
|
2964
|
-
const relativePath =
|
|
3384
|
+
const fullPath = path6.join(dir, entry.name);
|
|
3385
|
+
const relativePath = path6.relative(projectRoot, fullPath);
|
|
2965
3386
|
if (entry.name.startsWith(".") && entry.name !== "." && entry.name !== "..") {
|
|
2966
3387
|
if (entry.isDirectory()) {
|
|
2967
3388
|
skipped.push({ path: relativePath, reason: "excluded" });
|
|
@@ -3010,7 +3431,7 @@ async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns
|
|
|
3010
3431
|
yield f;
|
|
3011
3432
|
}
|
|
3012
3433
|
for (let i = options.maxFilesPerDirectory; i < filesInDir.length; i++) {
|
|
3013
|
-
skipped.push({ path:
|
|
3434
|
+
skipped.push({ path: path6.relative(projectRoot, filesInDir[i].path), reason: "excluded" });
|
|
3014
3435
|
}
|
|
3015
3436
|
const canRecurse = options.maxDepth === -1 || currentDepth < options.maxDepth;
|
|
3016
3437
|
if (canRecurse) {
|
|
@@ -3050,8 +3471,8 @@ async function collectFiles(projectRoot, includePatterns, excludePatterns, maxFi
|
|
|
3050
3471
|
if (additionalRoots && additionalRoots.length > 0) {
|
|
3051
3472
|
const normalizedRoots = /* @__PURE__ */ new Set();
|
|
3052
3473
|
for (const kbRoot of additionalRoots) {
|
|
3053
|
-
const resolved =
|
|
3054
|
-
|
|
3474
|
+
const resolved = path6.normalize(
|
|
3475
|
+
path6.isAbsolute(kbRoot) ? kbRoot : path6.resolve(projectRoot, kbRoot)
|
|
3055
3476
|
);
|
|
3056
3477
|
normalizedRoots.add(resolved);
|
|
3057
3478
|
}
|
|
@@ -3441,13 +3862,13 @@ function initializeLogger(config) {
|
|
|
3441
3862
|
}
|
|
3442
3863
|
|
|
3443
3864
|
// src/native/index.ts
|
|
3444
|
-
import * as
|
|
3445
|
-
import * as
|
|
3865
|
+
import * as path7 from "path";
|
|
3866
|
+
import * as os4 from "os";
|
|
3446
3867
|
import * as module from "module";
|
|
3447
3868
|
import { fileURLToPath } from "url";
|
|
3448
3869
|
function getNativeBinding() {
|
|
3449
|
-
const platform2 =
|
|
3450
|
-
const arch2 =
|
|
3870
|
+
const platform2 = os4.platform();
|
|
3871
|
+
const arch2 = os4.arch();
|
|
3451
3872
|
let bindingName;
|
|
3452
3873
|
if (platform2 === "darwin" && arch2 === "arm64") {
|
|
3453
3874
|
bindingName = "codebase-index-native.darwin-arm64.node";
|
|
@@ -3465,18 +3886,19 @@ function getNativeBinding() {
|
|
|
3465
3886
|
let currentDir;
|
|
3466
3887
|
let requireTarget;
|
|
3467
3888
|
if (typeof import.meta !== "undefined" && import.meta.url) {
|
|
3468
|
-
currentDir =
|
|
3889
|
+
currentDir = path7.dirname(fileURLToPath(import.meta.url));
|
|
3469
3890
|
requireTarget = import.meta.url;
|
|
3470
3891
|
} else if (typeof __dirname !== "undefined") {
|
|
3471
3892
|
currentDir = __dirname;
|
|
3472
3893
|
requireTarget = __filename;
|
|
3473
3894
|
} else {
|
|
3474
3895
|
currentDir = process.cwd();
|
|
3475
|
-
requireTarget =
|
|
3896
|
+
requireTarget = path7.join(currentDir, "index.js");
|
|
3476
3897
|
}
|
|
3477
|
-
const
|
|
3478
|
-
const
|
|
3479
|
-
const
|
|
3898
|
+
const normalizedDir = currentDir.replace(/\\/g, "/");
|
|
3899
|
+
const isDevMode = normalizedDir.includes("/src/native") || currentDir.includes(path7.join("src", "native"));
|
|
3900
|
+
const packageRoot = isDevMode ? path7.resolve(currentDir, "../..") : path7.resolve(currentDir, "..");
|
|
3901
|
+
const nativePath = path7.join(packageRoot, "native", bindingName);
|
|
3480
3902
|
const require2 = module.createRequire(requireTarget);
|
|
3481
3903
|
return require2(nativePath);
|
|
3482
3904
|
}
|
|
@@ -3507,11 +3929,20 @@ function createMockNativeBinding() {
|
|
|
3507
3929
|
constructor() {
|
|
3508
3930
|
throw error;
|
|
3509
3931
|
}
|
|
3932
|
+
serialize() {
|
|
3933
|
+
throw error;
|
|
3934
|
+
}
|
|
3935
|
+
deserialize() {
|
|
3936
|
+
throw error;
|
|
3937
|
+
}
|
|
3510
3938
|
},
|
|
3511
3939
|
Database: class {
|
|
3512
3940
|
constructor() {
|
|
3513
3941
|
throw error;
|
|
3514
3942
|
}
|
|
3943
|
+
close() {
|
|
3944
|
+
throw error;
|
|
3945
|
+
}
|
|
3515
3946
|
}
|
|
3516
3947
|
};
|
|
3517
3948
|
}
|
|
@@ -3644,7 +4075,7 @@ var MAX_SINGLE_CHUNK_TOKENS = 2e3;
|
|
|
3644
4075
|
function estimateTokens2(text) {
|
|
3645
4076
|
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
3646
4077
|
}
|
|
3647
|
-
function
|
|
4078
|
+
function getEmbeddingHeaderParts(chunk, filePath) {
|
|
3648
4079
|
const parts = [];
|
|
3649
4080
|
const fileName = filePath.split("/").pop() || filePath;
|
|
3650
4081
|
const dirPath = filePath.split("/").slice(-3, -1).join("/");
|
|
@@ -3691,23 +4122,52 @@ function createEmbeddingText(chunk, filePath) {
|
|
|
3691
4122
|
if (semanticHints.length > 0) {
|
|
3692
4123
|
parts.push(`Purpose: ${semanticHints.join(", ")}`);
|
|
3693
4124
|
}
|
|
3694
|
-
parts
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
const
|
|
3698
|
-
if (
|
|
3699
|
-
|
|
4125
|
+
return parts;
|
|
4126
|
+
}
|
|
4127
|
+
function buildEmbeddingText(headerParts, content, partIndex, partCount) {
|
|
4128
|
+
const parts = [...headerParts];
|
|
4129
|
+
if (partCount && partCount > 1 && partIndex) {
|
|
4130
|
+
parts.push(`Part ${partIndex}/${partCount}`);
|
|
3700
4131
|
}
|
|
4132
|
+
parts.push("");
|
|
3701
4133
|
parts.push(content);
|
|
3702
4134
|
return parts.join("\n");
|
|
3703
4135
|
}
|
|
3704
|
-
function
|
|
4136
|
+
function splitOversizedContent(content, maxContentChars) {
|
|
4137
|
+
if (content.length <= maxContentChars) {
|
|
4138
|
+
return [content];
|
|
4139
|
+
}
|
|
4140
|
+
const overlapChars = Math.max(CHARS_PER_TOKEN * 32, Math.min(Math.floor(maxContentChars * 0.15), CHARS_PER_TOKEN * 128));
|
|
4141
|
+
const stepChars = Math.max(1, maxContentChars - overlapChars);
|
|
4142
|
+
const segments = [];
|
|
4143
|
+
for (let start = 0; start < content.length; start += stepChars) {
|
|
4144
|
+
const end = Math.min(content.length, start + maxContentChars);
|
|
4145
|
+
segments.push(content.slice(start, end));
|
|
4146
|
+
if (end >= content.length) {
|
|
4147
|
+
break;
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
return segments;
|
|
4151
|
+
}
|
|
4152
|
+
function createEmbeddingTexts(chunk, filePath, maxChunkTokens = MAX_SINGLE_CHUNK_TOKENS) {
|
|
4153
|
+
const headerParts = getEmbeddingHeaderParts(chunk, filePath);
|
|
4154
|
+
const headerLength = buildEmbeddingText(headerParts, "", 1, 9).length;
|
|
4155
|
+
const maxContentChars = Math.max(1, maxChunkTokens * CHARS_PER_TOKEN - headerLength);
|
|
4156
|
+
const segments = splitOversizedContent(chunk.content, maxContentChars);
|
|
4157
|
+
if (segments.length === 1) {
|
|
4158
|
+
return [buildEmbeddingText(headerParts, segments[0])];
|
|
4159
|
+
}
|
|
4160
|
+
return segments.map((segment, index) => buildEmbeddingText(headerParts, segment, index + 1, segments.length));
|
|
4161
|
+
}
|
|
4162
|
+
function createDynamicBatches(chunks, options = {}) {
|
|
3705
4163
|
const batches = [];
|
|
3706
4164
|
let currentBatch = [];
|
|
3707
4165
|
let currentTokens = 0;
|
|
4166
|
+
const maxBatchTokens = Math.max(1, options.maxBatchTokens ?? MAX_BATCH_TOKENS);
|
|
4167
|
+
const maxBatchItems = Math.max(1, options.maxBatchItems ?? Number.MAX_SAFE_INTEGER);
|
|
3708
4168
|
for (const chunk of chunks) {
|
|
3709
|
-
const chunkTokens = estimateTokens2(chunk.text);
|
|
3710
|
-
if (currentBatch.length > 0 && currentTokens + chunkTokens >
|
|
4169
|
+
const chunkTokens = chunk.tokenCount ?? estimateTokens2(chunk.text);
|
|
4170
|
+
if (currentBatch.length > 0 && (currentTokens + chunkTokens > maxBatchTokens || currentBatch.length >= maxBatchItems)) {
|
|
3711
4171
|
batches.push(currentBatch);
|
|
3712
4172
|
currentBatch = [];
|
|
3713
4173
|
currentTokens = 0;
|
|
@@ -3826,6 +4286,12 @@ var InvertedIndex = class {
|
|
|
3826
4286
|
save() {
|
|
3827
4287
|
this.inner.save();
|
|
3828
4288
|
}
|
|
4289
|
+
serialize() {
|
|
4290
|
+
return this.inner.serialize();
|
|
4291
|
+
}
|
|
4292
|
+
deserialize(json) {
|
|
4293
|
+
this.inner.deserialize(json);
|
|
4294
|
+
}
|
|
3829
4295
|
addChunk(chunkId, content) {
|
|
3830
4296
|
this.inner.addChunk(chunkId, content);
|
|
3831
4297
|
}
|
|
@@ -3852,267 +4318,263 @@ var InvertedIndex = class {
|
|
|
3852
4318
|
};
|
|
3853
4319
|
var Database = class {
|
|
3854
4320
|
inner;
|
|
4321
|
+
closed = false;
|
|
3855
4322
|
constructor(dbPath) {
|
|
3856
4323
|
this.inner = new native.Database(dbPath);
|
|
3857
4324
|
}
|
|
4325
|
+
throwIfClosed() {
|
|
4326
|
+
if (this.closed) {
|
|
4327
|
+
throw new Error("Database is closed");
|
|
4328
|
+
}
|
|
4329
|
+
}
|
|
4330
|
+
close() {
|
|
4331
|
+
if (this.closed) {
|
|
4332
|
+
return;
|
|
4333
|
+
}
|
|
4334
|
+
if (typeof this.inner.close === "function") {
|
|
4335
|
+
this.inner.close();
|
|
4336
|
+
}
|
|
4337
|
+
this.closed = true;
|
|
4338
|
+
}
|
|
3858
4339
|
embeddingExists(contentHash) {
|
|
4340
|
+
this.throwIfClosed();
|
|
3859
4341
|
return this.inner.embeddingExists(contentHash);
|
|
3860
4342
|
}
|
|
3861
4343
|
getEmbedding(contentHash) {
|
|
4344
|
+
this.throwIfClosed();
|
|
3862
4345
|
return this.inner.getEmbedding(contentHash) ?? null;
|
|
3863
4346
|
}
|
|
3864
4347
|
upsertEmbedding(contentHash, embedding, chunkText, model) {
|
|
4348
|
+
this.throwIfClosed();
|
|
3865
4349
|
this.inner.upsertEmbedding(contentHash, embedding, chunkText, model);
|
|
3866
4350
|
}
|
|
3867
4351
|
upsertEmbeddingsBatch(items) {
|
|
4352
|
+
this.throwIfClosed();
|
|
3868
4353
|
if (items.length === 0) return;
|
|
3869
4354
|
this.inner.upsertEmbeddingsBatch(items);
|
|
3870
4355
|
}
|
|
3871
4356
|
getMissingEmbeddings(contentHashes) {
|
|
4357
|
+
this.throwIfClosed();
|
|
3872
4358
|
return this.inner.getMissingEmbeddings(contentHashes);
|
|
3873
4359
|
}
|
|
3874
4360
|
upsertChunk(chunk) {
|
|
4361
|
+
this.throwIfClosed();
|
|
3875
4362
|
this.inner.upsertChunk(chunk);
|
|
3876
4363
|
}
|
|
3877
4364
|
upsertChunksBatch(chunks) {
|
|
4365
|
+
this.throwIfClosed();
|
|
3878
4366
|
if (chunks.length === 0) return;
|
|
3879
4367
|
this.inner.upsertChunksBatch(chunks);
|
|
3880
4368
|
}
|
|
3881
4369
|
getChunk(chunkId) {
|
|
4370
|
+
this.throwIfClosed();
|
|
3882
4371
|
return this.inner.getChunk(chunkId) ?? null;
|
|
3883
4372
|
}
|
|
3884
4373
|
getChunksByFile(filePath) {
|
|
4374
|
+
this.throwIfClosed();
|
|
3885
4375
|
return this.inner.getChunksByFile(filePath);
|
|
3886
4376
|
}
|
|
3887
4377
|
getChunksByName(name) {
|
|
4378
|
+
this.throwIfClosed();
|
|
3888
4379
|
return this.inner.getChunksByName(name);
|
|
3889
4380
|
}
|
|
3890
4381
|
getChunksByNameCi(name) {
|
|
4382
|
+
this.throwIfClosed();
|
|
3891
4383
|
return this.inner.getChunksByNameCi(name);
|
|
3892
4384
|
}
|
|
3893
4385
|
deleteChunksByFile(filePath) {
|
|
4386
|
+
this.throwIfClosed();
|
|
3894
4387
|
return this.inner.deleteChunksByFile(filePath);
|
|
3895
4388
|
}
|
|
4389
|
+
deleteChunksByIds(chunkIds) {
|
|
4390
|
+
this.throwIfClosed();
|
|
4391
|
+
if (chunkIds.length === 0) return 0;
|
|
4392
|
+
return this.inner.deleteChunksByIds(chunkIds);
|
|
4393
|
+
}
|
|
3896
4394
|
addChunksToBranch(branch, chunkIds) {
|
|
4395
|
+
this.throwIfClosed();
|
|
3897
4396
|
this.inner.addChunksToBranch(branch, chunkIds);
|
|
3898
4397
|
}
|
|
3899
4398
|
addChunksToBranchBatch(branch, chunkIds) {
|
|
4399
|
+
this.throwIfClosed();
|
|
3900
4400
|
if (chunkIds.length === 0) return;
|
|
3901
4401
|
this.inner.addChunksToBranchBatch(branch, chunkIds);
|
|
3902
4402
|
}
|
|
3903
4403
|
clearBranch(branch) {
|
|
4404
|
+
this.throwIfClosed();
|
|
3904
4405
|
return this.inner.clearBranch(branch);
|
|
3905
4406
|
}
|
|
4407
|
+
deleteBranchChunksByChunkIds(chunkIds) {
|
|
4408
|
+
this.throwIfClosed();
|
|
4409
|
+
if (chunkIds.length === 0) return 0;
|
|
4410
|
+
return this.inner.deleteBranchChunksByChunkIds(chunkIds);
|
|
4411
|
+
}
|
|
4412
|
+
deleteBranchChunksForBranch(branch, chunkIds) {
|
|
4413
|
+
this.throwIfClosed();
|
|
4414
|
+
if (chunkIds.length === 0) return 0;
|
|
4415
|
+
return this.inner.deleteBranchChunksForBranch(branch, chunkIds);
|
|
4416
|
+
}
|
|
3906
4417
|
getBranchChunkIds(branch) {
|
|
4418
|
+
this.throwIfClosed();
|
|
3907
4419
|
return this.inner.getBranchChunkIds(branch);
|
|
3908
4420
|
}
|
|
3909
4421
|
getBranchDelta(branch, baseBranch) {
|
|
4422
|
+
this.throwIfClosed();
|
|
3910
4423
|
return this.inner.getBranchDelta(branch, baseBranch);
|
|
3911
4424
|
}
|
|
4425
|
+
getReferencedChunkIds(chunkIds) {
|
|
4426
|
+
this.throwIfClosed();
|
|
4427
|
+
if (chunkIds.length === 0) return [];
|
|
4428
|
+
return this.inner.getReferencedChunkIds(chunkIds);
|
|
4429
|
+
}
|
|
3912
4430
|
chunkExistsOnBranch(branch, chunkId) {
|
|
4431
|
+
this.throwIfClosed();
|
|
3913
4432
|
return this.inner.chunkExistsOnBranch(branch, chunkId);
|
|
3914
4433
|
}
|
|
3915
4434
|
getAllBranches() {
|
|
4435
|
+
this.throwIfClosed();
|
|
3916
4436
|
return this.inner.getAllBranches();
|
|
3917
4437
|
}
|
|
3918
4438
|
getMetadata(key) {
|
|
4439
|
+
this.throwIfClosed();
|
|
3919
4440
|
return this.inner.getMetadata(key) ?? null;
|
|
3920
4441
|
}
|
|
3921
4442
|
setMetadata(key, value) {
|
|
4443
|
+
this.throwIfClosed();
|
|
3922
4444
|
this.inner.setMetadata(key, value);
|
|
3923
4445
|
}
|
|
3924
4446
|
deleteMetadata(key) {
|
|
4447
|
+
this.throwIfClosed();
|
|
3925
4448
|
return this.inner.deleteMetadata(key);
|
|
3926
4449
|
}
|
|
4450
|
+
clearAllIndexedData() {
|
|
4451
|
+
this.throwIfClosed();
|
|
4452
|
+
this.inner.clearAllIndexedData();
|
|
4453
|
+
}
|
|
4454
|
+
clearCallEdgeTargetsForSymbols(symbolIds) {
|
|
4455
|
+
this.throwIfClosed();
|
|
4456
|
+
if (symbolIds.length === 0) return 0;
|
|
4457
|
+
return this.inner.clearCallEdgeTargetsForSymbols(symbolIds);
|
|
4458
|
+
}
|
|
3927
4459
|
gcOrphanEmbeddings() {
|
|
4460
|
+
this.throwIfClosed();
|
|
3928
4461
|
return this.inner.gcOrphanEmbeddings();
|
|
3929
4462
|
}
|
|
3930
4463
|
gcOrphanChunks() {
|
|
4464
|
+
this.throwIfClosed();
|
|
3931
4465
|
return this.inner.gcOrphanChunks();
|
|
3932
4466
|
}
|
|
3933
4467
|
getStats() {
|
|
4468
|
+
this.throwIfClosed();
|
|
3934
4469
|
return this.inner.getStats();
|
|
3935
4470
|
}
|
|
3936
4471
|
// ── Symbol methods ──────────────────────────────────────────────
|
|
3937
4472
|
upsertSymbol(symbol) {
|
|
4473
|
+
this.throwIfClosed();
|
|
3938
4474
|
this.inner.upsertSymbol(symbol);
|
|
3939
4475
|
}
|
|
3940
4476
|
upsertSymbolsBatch(symbols) {
|
|
4477
|
+
this.throwIfClosed();
|
|
3941
4478
|
if (symbols.length === 0) return;
|
|
3942
4479
|
this.inner.upsertSymbolsBatch(symbols);
|
|
3943
4480
|
}
|
|
3944
4481
|
getSymbolsByFile(filePath) {
|
|
4482
|
+
this.throwIfClosed();
|
|
3945
4483
|
return this.inner.getSymbolsByFile(filePath);
|
|
3946
4484
|
}
|
|
3947
4485
|
getSymbolByName(name, filePath) {
|
|
4486
|
+
this.throwIfClosed();
|
|
3948
4487
|
return this.inner.getSymbolByName(name, filePath) ?? null;
|
|
3949
4488
|
}
|
|
3950
4489
|
getSymbolsByName(name) {
|
|
4490
|
+
this.throwIfClosed();
|
|
3951
4491
|
return this.inner.getSymbolsByName(name);
|
|
3952
4492
|
}
|
|
3953
4493
|
getSymbolsByNameCi(name) {
|
|
4494
|
+
this.throwIfClosed();
|
|
3954
4495
|
return this.inner.getSymbolsByNameCi(name);
|
|
3955
4496
|
}
|
|
3956
4497
|
deleteSymbolsByFile(filePath) {
|
|
4498
|
+
this.throwIfClosed();
|
|
3957
4499
|
return this.inner.deleteSymbolsByFile(filePath);
|
|
3958
4500
|
}
|
|
3959
4501
|
// ── Call Edge methods ────────────────────────────────────────────
|
|
3960
4502
|
upsertCallEdge(edge) {
|
|
4503
|
+
this.throwIfClosed();
|
|
3961
4504
|
this.inner.upsertCallEdge(edge);
|
|
3962
4505
|
}
|
|
3963
4506
|
upsertCallEdgesBatch(edges) {
|
|
4507
|
+
this.throwIfClosed();
|
|
3964
4508
|
if (edges.length === 0) return;
|
|
3965
4509
|
this.inner.upsertCallEdgesBatch(edges);
|
|
3966
4510
|
}
|
|
3967
4511
|
getCallers(targetName, branch) {
|
|
4512
|
+
this.throwIfClosed();
|
|
3968
4513
|
return this.inner.getCallers(targetName, branch);
|
|
3969
4514
|
}
|
|
3970
4515
|
getCallersWithContext(targetName, branch) {
|
|
4516
|
+
this.throwIfClosed();
|
|
3971
4517
|
return this.inner.getCallersWithContext(targetName, branch);
|
|
3972
4518
|
}
|
|
3973
4519
|
getCallees(symbolId, branch) {
|
|
4520
|
+
this.throwIfClosed();
|
|
3974
4521
|
return this.inner.getCallees(symbolId, branch);
|
|
3975
4522
|
}
|
|
3976
4523
|
deleteCallEdgesByFile(filePath) {
|
|
4524
|
+
this.throwIfClosed();
|
|
3977
4525
|
return this.inner.deleteCallEdgesByFile(filePath);
|
|
3978
4526
|
}
|
|
3979
4527
|
resolveCallEdge(edgeId, toSymbolId) {
|
|
4528
|
+
this.throwIfClosed();
|
|
3980
4529
|
this.inner.resolveCallEdge(edgeId, toSymbolId);
|
|
3981
4530
|
}
|
|
3982
4531
|
// ── Branch Symbol methods ────────────────────────────────────────
|
|
3983
4532
|
addSymbolsToBranch(branch, symbolIds) {
|
|
4533
|
+
this.throwIfClosed();
|
|
3984
4534
|
this.inner.addSymbolsToBranch(branch, symbolIds);
|
|
3985
4535
|
}
|
|
3986
4536
|
addSymbolsToBranchBatch(branch, symbolIds) {
|
|
4537
|
+
this.throwIfClosed();
|
|
3987
4538
|
if (symbolIds.length === 0) return;
|
|
3988
4539
|
this.inner.addSymbolsToBranchBatch(branch, symbolIds);
|
|
3989
4540
|
}
|
|
3990
4541
|
getBranchSymbolIds(branch) {
|
|
4542
|
+
this.throwIfClosed();
|
|
3991
4543
|
return this.inner.getBranchSymbolIds(branch);
|
|
3992
4544
|
}
|
|
3993
4545
|
clearBranchSymbols(branch) {
|
|
4546
|
+
this.throwIfClosed();
|
|
3994
4547
|
return this.inner.clearBranchSymbols(branch);
|
|
3995
4548
|
}
|
|
4549
|
+
getReferencedSymbolIds(symbolIds) {
|
|
4550
|
+
this.throwIfClosed();
|
|
4551
|
+
if (symbolIds.length === 0) return [];
|
|
4552
|
+
return this.inner.getReferencedSymbolIds(symbolIds);
|
|
4553
|
+
}
|
|
4554
|
+
deleteBranchSymbolsBySymbolIds(symbolIds) {
|
|
4555
|
+
this.throwIfClosed();
|
|
4556
|
+
if (symbolIds.length === 0) return 0;
|
|
4557
|
+
return this.inner.deleteBranchSymbolsBySymbolIds(symbolIds);
|
|
4558
|
+
}
|
|
4559
|
+
deleteBranchSymbolsForBranch(branch, symbolIds) {
|
|
4560
|
+
this.throwIfClosed();
|
|
4561
|
+
if (symbolIds.length === 0) return 0;
|
|
4562
|
+
return this.inner.deleteBranchSymbolsForBranch(branch, symbolIds);
|
|
4563
|
+
}
|
|
3996
4564
|
// ── GC methods for symbols/edges ─────────────────────────────────
|
|
3997
4565
|
gcOrphanSymbols() {
|
|
4566
|
+
this.throwIfClosed();
|
|
3998
4567
|
return this.inner.gcOrphanSymbols();
|
|
3999
4568
|
}
|
|
4000
4569
|
gcOrphanCallEdges() {
|
|
4570
|
+
this.throwIfClosed();
|
|
4001
4571
|
return this.inner.gcOrphanCallEdges();
|
|
4002
4572
|
}
|
|
4003
4573
|
};
|
|
4004
4574
|
|
|
4005
|
-
// src/git/index.ts
|
|
4006
|
-
import { existsSync as existsSync3, readFileSync as readFileSync4, readdirSync, statSync } from "fs";
|
|
4007
|
-
import * as path5 from "path";
|
|
4008
|
-
function readPackedRefs(gitDir) {
|
|
4009
|
-
const packedRefsPath = path5.join(gitDir, "packed-refs");
|
|
4010
|
-
if (!existsSync3(packedRefsPath)) {
|
|
4011
|
-
return [];
|
|
4012
|
-
}
|
|
4013
|
-
try {
|
|
4014
|
-
return readFileSync4(packedRefsPath, "utf-8").split("\n").map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("^"));
|
|
4015
|
-
} catch {
|
|
4016
|
-
return [];
|
|
4017
|
-
}
|
|
4018
|
-
}
|
|
4019
|
-
function resolveCommonGitDir(gitDir) {
|
|
4020
|
-
const commonDirPath = path5.join(gitDir, "commondir");
|
|
4021
|
-
if (!existsSync3(commonDirPath)) {
|
|
4022
|
-
return gitDir;
|
|
4023
|
-
}
|
|
4024
|
-
try {
|
|
4025
|
-
const raw = readFileSync4(commonDirPath, "utf-8").trim();
|
|
4026
|
-
if (!raw) {
|
|
4027
|
-
return gitDir;
|
|
4028
|
-
}
|
|
4029
|
-
const resolved = path5.isAbsolute(raw) ? raw : path5.resolve(gitDir, raw);
|
|
4030
|
-
if (existsSync3(resolved)) {
|
|
4031
|
-
return resolved;
|
|
4032
|
-
}
|
|
4033
|
-
} catch {
|
|
4034
|
-
return gitDir;
|
|
4035
|
-
}
|
|
4036
|
-
return gitDir;
|
|
4037
|
-
}
|
|
4038
|
-
function resolveGitDir(repoRoot) {
|
|
4039
|
-
const gitPath = path5.join(repoRoot, ".git");
|
|
4040
|
-
if (!existsSync3(gitPath)) {
|
|
4041
|
-
return null;
|
|
4042
|
-
}
|
|
4043
|
-
try {
|
|
4044
|
-
const stat = statSync(gitPath);
|
|
4045
|
-
if (stat.isDirectory()) {
|
|
4046
|
-
return gitPath;
|
|
4047
|
-
}
|
|
4048
|
-
if (stat.isFile()) {
|
|
4049
|
-
const content = readFileSync4(gitPath, "utf-8").trim();
|
|
4050
|
-
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
4051
|
-
if (match) {
|
|
4052
|
-
const gitdir = match[1];
|
|
4053
|
-
const resolvedPath = path5.isAbsolute(gitdir) ? gitdir : path5.resolve(repoRoot, gitdir);
|
|
4054
|
-
if (existsSync3(resolvedPath)) {
|
|
4055
|
-
return resolvedPath;
|
|
4056
|
-
}
|
|
4057
|
-
}
|
|
4058
|
-
}
|
|
4059
|
-
} catch {
|
|
4060
|
-
}
|
|
4061
|
-
return null;
|
|
4062
|
-
}
|
|
4063
|
-
function isGitRepo(dir) {
|
|
4064
|
-
return resolveGitDir(dir) !== null;
|
|
4065
|
-
}
|
|
4066
|
-
function getCurrentBranch(repoRoot) {
|
|
4067
|
-
const gitDir = resolveGitDir(repoRoot);
|
|
4068
|
-
if (!gitDir) {
|
|
4069
|
-
return null;
|
|
4070
|
-
}
|
|
4071
|
-
const headPath = path5.join(gitDir, "HEAD");
|
|
4072
|
-
if (!existsSync3(headPath)) {
|
|
4073
|
-
return null;
|
|
4074
|
-
}
|
|
4075
|
-
try {
|
|
4076
|
-
const headContent = readFileSync4(headPath, "utf-8").trim();
|
|
4077
|
-
const match = headContent.match(/^ref: refs\/heads\/(.+)$/);
|
|
4078
|
-
if (match) {
|
|
4079
|
-
return match[1];
|
|
4080
|
-
}
|
|
4081
|
-
if (/^[0-9a-f]{40}$/i.test(headContent)) {
|
|
4082
|
-
return headContent.slice(0, 7);
|
|
4083
|
-
}
|
|
4084
|
-
return null;
|
|
4085
|
-
} catch {
|
|
4086
|
-
return null;
|
|
4087
|
-
}
|
|
4088
|
-
}
|
|
4089
|
-
function getBaseBranch(repoRoot) {
|
|
4090
|
-
const gitDir = resolveGitDir(repoRoot);
|
|
4091
|
-
const refStoreDir = gitDir ? resolveCommonGitDir(gitDir) : null;
|
|
4092
|
-
const candidates = ["main", "master", "develop", "trunk"];
|
|
4093
|
-
if (refStoreDir) {
|
|
4094
|
-
for (const candidate of candidates) {
|
|
4095
|
-
const refPath = path5.join(refStoreDir, "refs", "heads", candidate);
|
|
4096
|
-
if (existsSync3(refPath)) {
|
|
4097
|
-
return candidate;
|
|
4098
|
-
}
|
|
4099
|
-
const packedRefs = readPackedRefs(refStoreDir);
|
|
4100
|
-
if (packedRefs.some((line) => line.endsWith(` refs/heads/${candidate}`))) {
|
|
4101
|
-
return candidate;
|
|
4102
|
-
}
|
|
4103
|
-
}
|
|
4104
|
-
}
|
|
4105
|
-
return getCurrentBranch(repoRoot) ?? "main";
|
|
4106
|
-
}
|
|
4107
|
-
function getBranchOrDefault(repoRoot) {
|
|
4108
|
-
if (!isGitRepo(repoRoot)) {
|
|
4109
|
-
return "default";
|
|
4110
|
-
}
|
|
4111
|
-
return getCurrentBranch(repoRoot) ?? "default";
|
|
4112
|
-
}
|
|
4113
|
-
|
|
4114
4575
|
// src/indexer/index.ts
|
|
4115
|
-
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "php"]);
|
|
4576
|
+
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "php", "apex", "zig"]);
|
|
4577
|
+
var CASE_INSENSITIVE_LANGUAGES = /* @__PURE__ */ new Set(["apex"]);
|
|
4116
4578
|
var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
4117
4579
|
"function_declaration",
|
|
4118
4580
|
"function",
|
|
@@ -4134,7 +4596,11 @@ var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
|
4134
4596
|
"enum_item",
|
|
4135
4597
|
"trait_item",
|
|
4136
4598
|
"mod_item",
|
|
4137
|
-
"trait_declaration"
|
|
4599
|
+
"trait_declaration",
|
|
4600
|
+
"trigger_declaration",
|
|
4601
|
+
"test_declaration",
|
|
4602
|
+
"struct_declaration",
|
|
4603
|
+
"union_declaration"
|
|
4138
4604
|
]);
|
|
4139
4605
|
function float32ArrayToBuffer(arr) {
|
|
4140
4606
|
const float32 = new Float32Array(arr);
|
|
@@ -4159,12 +4625,192 @@ function isRateLimitError(error) {
|
|
|
4159
4625
|
const message = getErrorMessage(error);
|
|
4160
4626
|
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
4161
4627
|
}
|
|
4628
|
+
function getSafeEmbeddingChunkTokenLimit(provider) {
|
|
4629
|
+
const providerMaxTokens = provider.modelInfo.maxTokens;
|
|
4630
|
+
const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75));
|
|
4631
|
+
return Math.min(2e3, maxChunkTokens);
|
|
4632
|
+
}
|
|
4633
|
+
function getDynamicBatchOptions(provider) {
|
|
4634
|
+
if (provider.provider === "ollama") {
|
|
4635
|
+
return {
|
|
4636
|
+
maxBatchTokens: provider.modelInfo.maxTokens,
|
|
4637
|
+
maxBatchItems: 1
|
|
4638
|
+
};
|
|
4639
|
+
}
|
|
4640
|
+
return {};
|
|
4641
|
+
}
|
|
4642
|
+
function isSqliteCorruptionError(error) {
|
|
4643
|
+
const message = getErrorMessage(error).toLowerCase();
|
|
4644
|
+
return message.includes("database disk image is malformed") || message.includes("file is not a database") || message.includes("database schema is corrupt") || message.includes("sqlite_corrupt");
|
|
4645
|
+
}
|
|
4646
|
+
var STARTUP_WARNING_METADATA_KEY = "index.startupWarning";
|
|
4162
4647
|
var INDEX_METADATA_VERSION = "1";
|
|
4648
|
+
var EMBEDDING_STRATEGY_VERSION = "2";
|
|
4163
4649
|
var RANKING_TOKEN_CACHE_LIMIT = 4096;
|
|
4650
|
+
var RANK_HYBRID_CACHE_LIMIT = 256;
|
|
4651
|
+
function createPendingChunkStorageText(texts) {
|
|
4652
|
+
const primaryText = texts[0]?.text ?? "";
|
|
4653
|
+
if (texts.length <= 1) {
|
|
4654
|
+
return primaryText;
|
|
4655
|
+
}
|
|
4656
|
+
return `${primaryText}
|
|
4657
|
+
|
|
4658
|
+
... [split into ${texts.length} parts for embedding]`;
|
|
4659
|
+
}
|
|
4660
|
+
function normalizePendingChunk(rawChunk, maxChunkTokens) {
|
|
4661
|
+
if (!rawChunk || typeof rawChunk !== "object") {
|
|
4662
|
+
return null;
|
|
4663
|
+
}
|
|
4664
|
+
const chunk = rawChunk;
|
|
4665
|
+
if (typeof chunk.id !== "string" || typeof chunk.contentHash !== "string" || !chunk.metadata || typeof chunk.metadata !== "object") {
|
|
4666
|
+
return null;
|
|
4667
|
+
}
|
|
4668
|
+
const texts = Array.isArray(chunk.texts) ? chunk.texts.map((entry) => {
|
|
4669
|
+
if (!entry || typeof entry.text !== "string") {
|
|
4670
|
+
return null;
|
|
4671
|
+
}
|
|
4672
|
+
return {
|
|
4673
|
+
text: entry.text,
|
|
4674
|
+
tokenCount: typeof entry.tokenCount === "number" && Number.isFinite(entry.tokenCount) ? entry.tokenCount : estimateTokens2(entry.text)
|
|
4675
|
+
};
|
|
4676
|
+
}).filter((entry) => entry !== null) : [];
|
|
4677
|
+
if (texts.length === 0 && typeof chunk.text === "string") {
|
|
4678
|
+
if (typeof chunk.content === "string" && chunk.content.length > 0 && chunk.metadata && typeof chunk.metadata === "object") {
|
|
4679
|
+
const metadata = chunk.metadata;
|
|
4680
|
+
const rebuiltChunk = {
|
|
4681
|
+
content: chunk.content,
|
|
4682
|
+
startLine: typeof metadata.startLine === "number" ? metadata.startLine : 1,
|
|
4683
|
+
endLine: typeof metadata.endLine === "number" ? metadata.endLine : 1,
|
|
4684
|
+
chunkType: typeof metadata.chunkType === "string" ? metadata.chunkType : "other",
|
|
4685
|
+
name: typeof metadata.name === "string" ? metadata.name : void 0,
|
|
4686
|
+
language: typeof metadata.language === "string" ? metadata.language : "text"
|
|
4687
|
+
};
|
|
4688
|
+
const filePath = typeof metadata.filePath === "string" ? metadata.filePath : "unknown";
|
|
4689
|
+
texts.push(
|
|
4690
|
+
...createEmbeddingTexts(rebuiltChunk, filePath, maxChunkTokens).map((text) => ({
|
|
4691
|
+
text,
|
|
4692
|
+
tokenCount: estimateTokens2(text)
|
|
4693
|
+
}))
|
|
4694
|
+
);
|
|
4695
|
+
} else {
|
|
4696
|
+
texts.push({
|
|
4697
|
+
text: chunk.text,
|
|
4698
|
+
tokenCount: estimateTokens2(chunk.text)
|
|
4699
|
+
});
|
|
4700
|
+
}
|
|
4701
|
+
}
|
|
4702
|
+
if (texts.length === 0) {
|
|
4703
|
+
return null;
|
|
4704
|
+
}
|
|
4705
|
+
return {
|
|
4706
|
+
id: chunk.id,
|
|
4707
|
+
texts,
|
|
4708
|
+
storageText: typeof chunk.storageText === "string" ? chunk.storageText : createPendingChunkStorageText(texts),
|
|
4709
|
+
content: typeof chunk.content === "string" ? chunk.content : "",
|
|
4710
|
+
contentHash: chunk.contentHash,
|
|
4711
|
+
metadata: chunk.metadata
|
|
4712
|
+
};
|
|
4713
|
+
}
|
|
4714
|
+
function getPendingChunkFilePath(rawChunk) {
|
|
4715
|
+
if (!rawChunk || typeof rawChunk !== "object") {
|
|
4716
|
+
return null;
|
|
4717
|
+
}
|
|
4718
|
+
const chunk = rawChunk;
|
|
4719
|
+
if (!chunk.metadata || typeof chunk.metadata !== "object") {
|
|
4720
|
+
return null;
|
|
4721
|
+
}
|
|
4722
|
+
const metadata = chunk.metadata;
|
|
4723
|
+
return typeof metadata.filePath === "string" ? metadata.filePath : null;
|
|
4724
|
+
}
|
|
4725
|
+
function normalizeFailedBatch(batch, maxChunkTokens) {
|
|
4726
|
+
const chunks = batch.chunks.map((chunk) => normalizePendingChunk(chunk, maxChunkTokens)).filter((chunk) => chunk !== null);
|
|
4727
|
+
if (chunks.length === 0) {
|
|
4728
|
+
return null;
|
|
4729
|
+
}
|
|
4730
|
+
return {
|
|
4731
|
+
chunks,
|
|
4732
|
+
error: batch.error,
|
|
4733
|
+
attemptCount: batch.attemptCount,
|
|
4734
|
+
lastAttempt: batch.lastAttempt
|
|
4735
|
+
};
|
|
4736
|
+
}
|
|
4737
|
+
function createPendingEmbeddingRequests(chunks) {
|
|
4738
|
+
return chunks.flatMap(
|
|
4739
|
+
(chunk) => chunk.texts.map((textPart, partIndex) => ({
|
|
4740
|
+
chunk,
|
|
4741
|
+
partIndex,
|
|
4742
|
+
text: textPart.text,
|
|
4743
|
+
tokenCount: textPart.tokenCount
|
|
4744
|
+
}))
|
|
4745
|
+
);
|
|
4746
|
+
}
|
|
4747
|
+
function createPendingEmbeddingRequestBatches(chunks, options = {}) {
|
|
4748
|
+
return createDynamicBatches(createPendingEmbeddingRequests(chunks), options);
|
|
4749
|
+
}
|
|
4750
|
+
function getUniquePendingChunksFromRequests(requests) {
|
|
4751
|
+
const uniqueChunks = /* @__PURE__ */ new Map();
|
|
4752
|
+
for (const request of requests) {
|
|
4753
|
+
uniqueChunks.set(request.chunk.id, request.chunk);
|
|
4754
|
+
}
|
|
4755
|
+
return Array.from(uniqueChunks.values());
|
|
4756
|
+
}
|
|
4757
|
+
function coalesceFailedBatches(batches) {
|
|
4758
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
4759
|
+
for (const batch of batches) {
|
|
4760
|
+
const key = `${batch.attemptCount}:${batch.lastAttempt}:${batch.error}`;
|
|
4761
|
+
const existing = grouped.get(key);
|
|
4762
|
+
if (!existing) {
|
|
4763
|
+
grouped.set(key, {
|
|
4764
|
+
...batch,
|
|
4765
|
+
chunks: [...batch.chunks]
|
|
4766
|
+
});
|
|
4767
|
+
continue;
|
|
4768
|
+
}
|
|
4769
|
+
existing.chunks.push(...batch.chunks);
|
|
4770
|
+
}
|
|
4771
|
+
return Array.from(grouped.values());
|
|
4772
|
+
}
|
|
4773
|
+
function poolEmbeddingVectors(vectors, weights) {
|
|
4774
|
+
const firstVector = vectors[0];
|
|
4775
|
+
if (!firstVector) {
|
|
4776
|
+
return [];
|
|
4777
|
+
}
|
|
4778
|
+
const pooled = new Array(firstVector.length).fill(0);
|
|
4779
|
+
let totalWeight = 0;
|
|
4780
|
+
for (let index = 0; index < vectors.length; index++) {
|
|
4781
|
+
const vector = vectors[index];
|
|
4782
|
+
const weight = Math.max(1, weights[index] ?? 1);
|
|
4783
|
+
totalWeight += weight;
|
|
4784
|
+
for (let dimension = 0; dimension < vector.length; dimension++) {
|
|
4785
|
+
pooled[dimension] += vector[dimension] * weight;
|
|
4786
|
+
}
|
|
4787
|
+
}
|
|
4788
|
+
if (totalWeight === 0) {
|
|
4789
|
+
return firstVector;
|
|
4790
|
+
}
|
|
4791
|
+
return pooled.map((value) => value / totalWeight);
|
|
4792
|
+
}
|
|
4793
|
+
function hasAllEmbeddingParts(parts, expectedPartCount) {
|
|
4794
|
+
if (parts.length !== expectedPartCount) {
|
|
4795
|
+
return false;
|
|
4796
|
+
}
|
|
4797
|
+
for (let index = 0; index < expectedPartCount; index++) {
|
|
4798
|
+
if (parts[index] === void 0) {
|
|
4799
|
+
return false;
|
|
4800
|
+
}
|
|
4801
|
+
}
|
|
4802
|
+
return true;
|
|
4803
|
+
}
|
|
4804
|
+
function isPathWithinRoot(filePath, rootPath) {
|
|
4805
|
+
const normalizedFilePath = path8.resolve(filePath);
|
|
4806
|
+
const normalizedRoot = path8.resolve(rootPath);
|
|
4807
|
+
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${path8.sep}`);
|
|
4808
|
+
}
|
|
4164
4809
|
var rankingQueryTokenCache = /* @__PURE__ */ new Map();
|
|
4165
4810
|
var rankingNameTokenCache = /* @__PURE__ */ new Map();
|
|
4166
4811
|
var rankingPathTokenCache = /* @__PURE__ */ new Map();
|
|
4167
4812
|
var rankingTextTokenCache = /* @__PURE__ */ new Map();
|
|
4813
|
+
var rankHybridResultsCache = /* @__PURE__ */ new WeakMap();
|
|
4168
4814
|
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
4169
4815
|
"the",
|
|
4170
4816
|
"and",
|
|
@@ -4380,7 +5026,16 @@ function classifyExternalRerankBand(candidate, preferSourcePaths, docIntent) {
|
|
|
4380
5026
|
function classifyQueryIntent(tokens) {
|
|
4381
5027
|
const sourceIntentHits = tokens.filter((t) => SOURCE_INTENT_HINTS.has(t)).length;
|
|
4382
5028
|
const docTestIntentHits = tokens.filter((t) => DOC_TEST_INTENT_HINTS.has(t)).length;
|
|
4383
|
-
|
|
5029
|
+
if (sourceIntentHits === 0 && docTestIntentHits === 0) {
|
|
5030
|
+
return "neutral";
|
|
5031
|
+
}
|
|
5032
|
+
if (sourceIntentHits > docTestIntentHits) {
|
|
5033
|
+
return "source";
|
|
5034
|
+
}
|
|
5035
|
+
if (docTestIntentHits > sourceIntentHits) {
|
|
5036
|
+
return "doc_test";
|
|
5037
|
+
}
|
|
5038
|
+
return "neutral";
|
|
4384
5039
|
}
|
|
4385
5040
|
function classifyQueryIntentRaw(query) {
|
|
4386
5041
|
const lowerQuery = query.toLowerCase();
|
|
@@ -4402,7 +5057,7 @@ function classifyQueryIntentRaw(query) {
|
|
|
4402
5057
|
if (docTestRawHits > sourceRawHits) {
|
|
4403
5058
|
return "doc_test";
|
|
4404
5059
|
}
|
|
4405
|
-
if (sourceRawHits > docTestRawHits) {
|
|
5060
|
+
if (sourceRawHits > docTestRawHits && sourceRawHits > 0) {
|
|
4406
5061
|
return "source";
|
|
4407
5062
|
}
|
|
4408
5063
|
const hasWhereIsPattern = /\bwhere\s+is\b/.test(lowerQuery);
|
|
@@ -4669,9 +5324,8 @@ function rerankResults(query, candidates, rerankTopN, options) {
|
|
|
4669
5324
|
return candidates;
|
|
4670
5325
|
}
|
|
4671
5326
|
const queryTokenList = Array.from(queryTokens);
|
|
4672
|
-
const intent = classifyQueryIntentRaw(query);
|
|
4673
5327
|
const docIntent = classifyDocIntent(queryTokenList);
|
|
4674
|
-
const preferSourcePaths = options?.prioritizeSourcePaths ??
|
|
5328
|
+
const preferSourcePaths = options?.prioritizeSourcePaths ?? classifyQueryIntentRaw(query) === "source";
|
|
4675
5329
|
const identifierHints = extractIdentifierHints(query);
|
|
4676
5330
|
const head = candidates.slice(0, topN).map((candidate, idx) => {
|
|
4677
5331
|
const pathTokens = splitPathTokens(candidate.metadata.filePath);
|
|
@@ -4821,13 +5475,38 @@ function buildDiversityKey(metadata) {
|
|
|
4821
5475
|
return normalizedPath;
|
|
4822
5476
|
}
|
|
4823
5477
|
function rankHybridResults(query, semanticResults, keywordResults, options) {
|
|
5478
|
+
const prioritizeSourcePaths = options.prioritizeSourcePaths ?? classifyQueryIntentRaw(query) === "source";
|
|
5479
|
+
const cacheKey = `${query}${options.fusionStrategy}|${options.rrfK}|${options.hybridWeight}|${options.rerankTopN}|${options.limit}|${prioritizeSourcePaths ? 1 : 0}`;
|
|
5480
|
+
let byKeyword = rankHybridResultsCache.get(semanticResults);
|
|
5481
|
+
if (!byKeyword) {
|
|
5482
|
+
byKeyword = /* @__PURE__ */ new WeakMap();
|
|
5483
|
+
rankHybridResultsCache.set(semanticResults, byKeyword);
|
|
5484
|
+
}
|
|
5485
|
+
let bucket = byKeyword.get(keywordResults);
|
|
5486
|
+
if (!bucket) {
|
|
5487
|
+
bucket = /* @__PURE__ */ new Map();
|
|
5488
|
+
byKeyword.set(keywordResults, bucket);
|
|
5489
|
+
} else {
|
|
5490
|
+
const cached = bucket.get(cacheKey);
|
|
5491
|
+
if (cached) {
|
|
5492
|
+
return cached;
|
|
5493
|
+
}
|
|
5494
|
+
}
|
|
4824
5495
|
const overfetchLimit = Math.max(options.limit * 4, options.limit);
|
|
4825
5496
|
const fused = options.fusionStrategy === "rrf" ? fuseResultsRrf(semanticResults, keywordResults, options.rrfK, overfetchLimit) : fuseResultsWeighted(semanticResults, keywordResults, options.hybridWeight, overfetchLimit);
|
|
4826
5497
|
const rerankPoolLimit = Math.max(overfetchLimit, options.rerankTopN * 3, options.limit * 6);
|
|
4827
5498
|
const rerankPool = fused.slice(0, rerankPoolLimit);
|
|
4828
|
-
|
|
4829
|
-
prioritizeSourcePaths
|
|
5499
|
+
const ranked = rerankResults(query, rerankPool, options.rerankTopN, {
|
|
5500
|
+
prioritizeSourcePaths
|
|
4830
5501
|
});
|
|
5502
|
+
if (bucket.size >= RANK_HYBRID_CACHE_LIMIT) {
|
|
5503
|
+
const oldest = bucket.keys().next().value;
|
|
5504
|
+
if (oldest !== void 0) {
|
|
5505
|
+
bucket.delete(oldest);
|
|
5506
|
+
}
|
|
5507
|
+
}
|
|
5508
|
+
bucket.set(cacheKey, ranked);
|
|
5509
|
+
return ranked;
|
|
4831
5510
|
}
|
|
4832
5511
|
function rankSemanticOnlyResults(query, semanticResults, options) {
|
|
4833
5512
|
const overfetchLimit = Math.max(options.limit * 4, options.limit);
|
|
@@ -5124,6 +5803,21 @@ function mergeTieredResults(symbolLane, hybridLane, limit) {
|
|
|
5124
5803
|
}
|
|
5125
5804
|
return out;
|
|
5126
5805
|
}
|
|
5806
|
+
function matchesSearchFilters(candidate, options, minScore) {
|
|
5807
|
+
if (candidate.score < minScore) return false;
|
|
5808
|
+
if (options?.fileType) {
|
|
5809
|
+
const ext = candidate.metadata.filePath.split(".").pop()?.toLowerCase();
|
|
5810
|
+
if (ext !== options.fileType.toLowerCase().replace(/^\./, "")) return false;
|
|
5811
|
+
}
|
|
5812
|
+
if (options?.directory) {
|
|
5813
|
+
const normalizedDir = options.directory.replace(/^\/|\/$/g, "");
|
|
5814
|
+
if (!candidate.metadata.filePath.includes(`/${normalizedDir}/`) && !candidate.metadata.filePath.includes(`${normalizedDir}/`)) return false;
|
|
5815
|
+
}
|
|
5816
|
+
if (options?.chunkType && candidate.metadata.chunkType !== options.chunkType) {
|
|
5817
|
+
return false;
|
|
5818
|
+
}
|
|
5819
|
+
return true;
|
|
5820
|
+
}
|
|
5127
5821
|
function unionCandidates(semanticCandidates, keywordCandidates) {
|
|
5128
5822
|
const byId = /* @__PURE__ */ new Map();
|
|
5129
5823
|
for (const candidate of semanticCandidates) {
|
|
@@ -5163,22 +5857,18 @@ var Indexer = class {
|
|
|
5163
5857
|
this.projectRoot = projectRoot;
|
|
5164
5858
|
this.config = config;
|
|
5165
5859
|
this.indexPath = this.getIndexPath();
|
|
5166
|
-
this.fileHashCachePath =
|
|
5167
|
-
this.failedBatchesPath =
|
|
5168
|
-
this.indexingLockPath =
|
|
5860
|
+
this.fileHashCachePath = path8.join(this.indexPath, "file-hashes.json");
|
|
5861
|
+
this.failedBatchesPath = path8.join(this.indexPath, "failed-batches.json");
|
|
5862
|
+
this.indexingLockPath = path8.join(this.indexPath, "indexing.lock");
|
|
5169
5863
|
this.logger = initializeLogger(config.debug);
|
|
5170
5864
|
}
|
|
5171
5865
|
getIndexPath() {
|
|
5172
|
-
|
|
5173
|
-
const homeDir = process.env.HOME || process.env.USERPROFILE || "";
|
|
5174
|
-
return path6.join(homeDir, ".opencode", "global-index");
|
|
5175
|
-
}
|
|
5176
|
-
return path6.join(this.projectRoot, ".opencode", "index");
|
|
5866
|
+
return resolveProjectIndexPath(this.projectRoot, this.config.scope);
|
|
5177
5867
|
}
|
|
5178
5868
|
loadFileHashCache() {
|
|
5179
5869
|
try {
|
|
5180
|
-
if (
|
|
5181
|
-
const data =
|
|
5870
|
+
if (existsSync6(this.fileHashCachePath)) {
|
|
5871
|
+
const data = readFileSync6(this.fileHashCachePath, "utf-8");
|
|
5182
5872
|
const parsed = JSON.parse(data);
|
|
5183
5873
|
this.fileHashCache = new Map(Object.entries(parsed));
|
|
5184
5874
|
}
|
|
@@ -5195,63 +5885,422 @@ var Indexer = class {
|
|
|
5195
5885
|
}
|
|
5196
5886
|
atomicWriteSync(targetPath, data) {
|
|
5197
5887
|
const tempPath = `${targetPath}.tmp`;
|
|
5198
|
-
|
|
5888
|
+
writeFileSync3(tempPath, data);
|
|
5199
5889
|
renameSync(tempPath, targetPath);
|
|
5200
5890
|
}
|
|
5891
|
+
getScopedRoots() {
|
|
5892
|
+
const roots = /* @__PURE__ */ new Set([path8.resolve(this.projectRoot)]);
|
|
5893
|
+
for (const kbRoot of this.config.knowledgeBases) {
|
|
5894
|
+
roots.add(path8.resolve(this.projectRoot, kbRoot));
|
|
5895
|
+
}
|
|
5896
|
+
return Array.from(roots);
|
|
5897
|
+
}
|
|
5898
|
+
getBranchCatalogKey() {
|
|
5899
|
+
const branchName = this.currentBranch || "default";
|
|
5900
|
+
if (this.config.scope !== "global") {
|
|
5901
|
+
return branchName;
|
|
5902
|
+
}
|
|
5903
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5904
|
+
return `${projectHash}:${branchName}`;
|
|
5905
|
+
}
|
|
5906
|
+
getLegacyBranchCatalogKey() {
|
|
5907
|
+
return this.currentBranch || "default";
|
|
5908
|
+
}
|
|
5909
|
+
getLegacyMigrationMetadataKey() {
|
|
5910
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5911
|
+
return `index.globalBranchMigration.${projectHash}`;
|
|
5912
|
+
}
|
|
5913
|
+
getProjectEmbeddingStrategyMetadataKey() {
|
|
5914
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5915
|
+
return `index.embeddingStrategyVersion.${projectHash}`;
|
|
5916
|
+
}
|
|
5917
|
+
getProjectForceReembedMetadataKey() {
|
|
5918
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
5919
|
+
return `index.forceReembed.${projectHash}`;
|
|
5920
|
+
}
|
|
5921
|
+
hasProjectForceReembedPending() {
|
|
5922
|
+
return this.config.scope === "global" && this.database?.getMetadata(this.getProjectForceReembedMetadataKey()) === "true";
|
|
5923
|
+
}
|
|
5924
|
+
hasScopedIndexedData() {
|
|
5925
|
+
if (!this.store || this.config.scope !== "global") {
|
|
5926
|
+
return false;
|
|
5927
|
+
}
|
|
5928
|
+
if (this.hasProjectForceReembedPending()) {
|
|
5929
|
+
return false;
|
|
5930
|
+
}
|
|
5931
|
+
const roots = this.getScopedRoots();
|
|
5932
|
+
if (Array.from(this.fileHashCache.keys()).some((filePath) => this.isFileInCurrentScope(filePath, roots))) {
|
|
5933
|
+
return true;
|
|
5934
|
+
}
|
|
5935
|
+
if (this.loadSerializedFailedBatches().some(
|
|
5936
|
+
(batch) => batch.chunks.some((chunk) => {
|
|
5937
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
5938
|
+
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
5939
|
+
})
|
|
5940
|
+
)) {
|
|
5941
|
+
return true;
|
|
5942
|
+
}
|
|
5943
|
+
if (!this.database) {
|
|
5944
|
+
return false;
|
|
5945
|
+
}
|
|
5946
|
+
if (this.getBranchCatalogKeys().some((branchKey) => {
|
|
5947
|
+
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
5948
|
+
if (branchChunkIds.length > 0) {
|
|
5949
|
+
return true;
|
|
5950
|
+
}
|
|
5951
|
+
return this.database.getBranchSymbolIds(branchKey).length > 0;
|
|
5952
|
+
})) {
|
|
5953
|
+
return true;
|
|
5954
|
+
}
|
|
5955
|
+
const hasAnyBranchRows = this.database.getAllBranches().some((branchKey) => {
|
|
5956
|
+
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
5957
|
+
if (branchChunkIds.length > 0) {
|
|
5958
|
+
return true;
|
|
5959
|
+
}
|
|
5960
|
+
return this.database.getBranchSymbolIds(branchKey).length > 0;
|
|
5961
|
+
});
|
|
5962
|
+
if (hasAnyBranchRows) {
|
|
5963
|
+
return false;
|
|
5964
|
+
}
|
|
5965
|
+
return this.store.getAllMetadata().some(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
5966
|
+
}
|
|
5967
|
+
loadStoredEmbeddingStrategyVersion() {
|
|
5968
|
+
if (!this.database) {
|
|
5969
|
+
return null;
|
|
5970
|
+
}
|
|
5971
|
+
if (this.hasProjectForceReembedPending()) {
|
|
5972
|
+
return null;
|
|
5973
|
+
}
|
|
5974
|
+
if (this.config.scope !== "global") {
|
|
5975
|
+
return this.database.getMetadata("index.embeddingStrategyVersion") ?? "1";
|
|
5976
|
+
}
|
|
5977
|
+
const projectVersion = this.database.getMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
5978
|
+
if (projectVersion) {
|
|
5979
|
+
return projectVersion;
|
|
5980
|
+
}
|
|
5981
|
+
const legacySharedVersion = this.database.getMetadata("index.embeddingStrategyVersion");
|
|
5982
|
+
if (legacySharedVersion && this.hasScopedIndexedData()) {
|
|
5983
|
+
return legacySharedVersion;
|
|
5984
|
+
}
|
|
5985
|
+
return null;
|
|
5986
|
+
}
|
|
5987
|
+
getBranchCatalogKeys() {
|
|
5988
|
+
const primary = this.getBranchCatalogKey();
|
|
5989
|
+
if (this.config.scope !== "global") {
|
|
5990
|
+
return [primary];
|
|
5991
|
+
}
|
|
5992
|
+
if (this.database?.getMetadata(this.getLegacyMigrationMetadataKey()) === "done") {
|
|
5993
|
+
return [primary];
|
|
5994
|
+
}
|
|
5995
|
+
const legacy = this.getLegacyBranchCatalogKey();
|
|
5996
|
+
return primary === legacy ? [primary] : [primary, legacy];
|
|
5997
|
+
}
|
|
5998
|
+
getBranchCatalogCleanupKeys() {
|
|
5999
|
+
const primary = this.getBranchCatalogKey();
|
|
6000
|
+
if (this.config.scope !== "global") {
|
|
6001
|
+
return [primary];
|
|
6002
|
+
}
|
|
6003
|
+
const legacy = this.getLegacyBranchCatalogKey();
|
|
6004
|
+
return primary === legacy ? [primary] : [primary, legacy];
|
|
6005
|
+
}
|
|
6006
|
+
getProjectLocalScopedOwnershipIds(roots) {
|
|
6007
|
+
const chunkIds = /* @__PURE__ */ new Set();
|
|
6008
|
+
const symbolIds = /* @__PURE__ */ new Set();
|
|
6009
|
+
if (!this.database) {
|
|
6010
|
+
return { chunkIds, symbolIds };
|
|
6011
|
+
}
|
|
6012
|
+
const projectRootPath = path8.resolve(this.projectRoot);
|
|
6013
|
+
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
6014
|
+
...Array.from(this.fileHashCache.keys()).filter(
|
|
6015
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath)
|
|
6016
|
+
),
|
|
6017
|
+
...(this.store?.getAllMetadata() ?? []).map(({ metadata }) => metadata.filePath).filter(
|
|
6018
|
+
(filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath)
|
|
6019
|
+
)
|
|
6020
|
+
]);
|
|
6021
|
+
for (const filePath of projectLocalFilePaths) {
|
|
6022
|
+
for (const chunk of this.database.getChunksByFile(filePath)) {
|
|
6023
|
+
chunkIds.add(chunk.chunkId);
|
|
6024
|
+
}
|
|
6025
|
+
for (const symbol of this.database.getSymbolsByFile(filePath)) {
|
|
6026
|
+
symbolIds.add(symbol.id);
|
|
6027
|
+
}
|
|
6028
|
+
}
|
|
6029
|
+
return { chunkIds, symbolIds };
|
|
6030
|
+
}
|
|
6031
|
+
getProjectScopedBranchCatalogCleanupKeys(projectChunkIds, projectSymbolIds) {
|
|
6032
|
+
if (this.config.scope !== "global") {
|
|
6033
|
+
return this.getBranchCatalogCleanupKeys();
|
|
6034
|
+
}
|
|
6035
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
6036
|
+
const keys = /* @__PURE__ */ new Set();
|
|
6037
|
+
const projectChunkIdSet = new Set(projectChunkIds);
|
|
6038
|
+
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
6039
|
+
for (const branchKey of this.database?.getAllBranches() ?? []) {
|
|
6040
|
+
if (branchKey.startsWith(`${projectHash}:`)) {
|
|
6041
|
+
keys.add(branchKey);
|
|
6042
|
+
continue;
|
|
6043
|
+
}
|
|
6044
|
+
if (branchKey.includes(":")) {
|
|
6045
|
+
continue;
|
|
6046
|
+
}
|
|
6047
|
+
const referencesProjectChunks = this.database?.getBranchChunkIds(branchKey).some((chunkId) => projectChunkIdSet.has(chunkId)) ?? false;
|
|
6048
|
+
const referencesProjectSymbols = this.database?.getBranchSymbolIds(branchKey).some((symbolId) => projectSymbolIdSet.has(symbolId)) ?? false;
|
|
6049
|
+
if (referencesProjectChunks || referencesProjectSymbols) {
|
|
6050
|
+
keys.add(branchKey);
|
|
6051
|
+
}
|
|
6052
|
+
}
|
|
6053
|
+
for (const branchKey of this.getBranchCatalogCleanupKeys()) {
|
|
6054
|
+
keys.add(branchKey);
|
|
6055
|
+
}
|
|
6056
|
+
return Array.from(keys);
|
|
6057
|
+
}
|
|
6058
|
+
isFileInCurrentScope(filePath, roots) {
|
|
6059
|
+
return roots.some((root) => isPathWithinRoot(filePath, root));
|
|
6060
|
+
}
|
|
6061
|
+
clearScopedFileHashCache(roots) {
|
|
6062
|
+
for (const filePath of Array.from(this.fileHashCache.keys())) {
|
|
6063
|
+
if (this.isFileInCurrentScope(filePath, roots)) {
|
|
6064
|
+
this.fileHashCache.delete(filePath);
|
|
6065
|
+
}
|
|
6066
|
+
}
|
|
6067
|
+
this.saveFileHashCache();
|
|
6068
|
+
}
|
|
6069
|
+
replaceScopedFileHashCache(currentFileHashes, roots) {
|
|
6070
|
+
for (const filePath of Array.from(this.fileHashCache.keys())) {
|
|
6071
|
+
if (this.isFileInCurrentScope(filePath, roots)) {
|
|
6072
|
+
this.fileHashCache.delete(filePath);
|
|
6073
|
+
}
|
|
6074
|
+
}
|
|
6075
|
+
for (const [filePath, hash] of currentFileHashes) {
|
|
6076
|
+
this.fileHashCache.set(filePath, hash);
|
|
6077
|
+
}
|
|
6078
|
+
this.saveFileHashCache();
|
|
6079
|
+
}
|
|
6080
|
+
partitionFailedBatches(roots, maxChunkTokens) {
|
|
6081
|
+
const scoped = [];
|
|
6082
|
+
const retained = [];
|
|
6083
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
6084
|
+
const scopedChunks = batch.chunks.filter((chunk) => {
|
|
6085
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
6086
|
+
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
6087
|
+
});
|
|
6088
|
+
const retainedChunks = batch.chunks.filter((chunk) => {
|
|
6089
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
6090
|
+
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
6091
|
+
});
|
|
6092
|
+
if (scopedChunks.length > 0) {
|
|
6093
|
+
const normalizedBatch = normalizeFailedBatch({ ...batch, chunks: scopedChunks }, maxChunkTokens);
|
|
6094
|
+
if (normalizedBatch) {
|
|
6095
|
+
scoped.push(normalizedBatch);
|
|
6096
|
+
}
|
|
6097
|
+
}
|
|
6098
|
+
if (retainedChunks.length > 0) {
|
|
6099
|
+
retained.push({ ...batch, chunks: retainedChunks });
|
|
6100
|
+
}
|
|
6101
|
+
}
|
|
6102
|
+
return { scoped, retained };
|
|
6103
|
+
}
|
|
6104
|
+
clearScopedFailedBatches(roots) {
|
|
6105
|
+
const { retained: retainedBatches } = this.partitionFailedBatches(roots);
|
|
6106
|
+
this.saveFailedBatches(retainedBatches);
|
|
6107
|
+
}
|
|
6108
|
+
hasForeignScopedFileHashData(roots) {
|
|
6109
|
+
return Array.from(this.fileHashCache.keys()).some((filePath) => !this.isFileInCurrentScope(filePath, roots));
|
|
6110
|
+
}
|
|
6111
|
+
hasForeignScopedFailedBatches(roots) {
|
|
6112
|
+
const { retained } = this.partitionFailedBatches(roots);
|
|
6113
|
+
return retained.length > 0;
|
|
6114
|
+
}
|
|
6115
|
+
hasForeignScopedBranchData() {
|
|
6116
|
+
if (!this.database || this.config.scope !== "global") {
|
|
6117
|
+
return false;
|
|
6118
|
+
}
|
|
6119
|
+
const projectHash = hashContent(path8.resolve(this.projectRoot)).slice(0, 16);
|
|
6120
|
+
const roots = this.getScopedRoots();
|
|
6121
|
+
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
6122
|
+
return this.database.getAllBranches().some(
|
|
6123
|
+
(branchKey) => {
|
|
6124
|
+
const branchChunkIds = this.database.getBranchChunkIds(branchKey);
|
|
6125
|
+
const branchSymbolIds = this.database.getBranchSymbolIds(branchKey);
|
|
6126
|
+
const hasBranchData = branchChunkIds.length > 0 || branchSymbolIds.length > 0;
|
|
6127
|
+
if (!hasBranchData) {
|
|
6128
|
+
return false;
|
|
6129
|
+
}
|
|
6130
|
+
if (branchKey.startsWith(`${projectHash}:`)) {
|
|
6131
|
+
return false;
|
|
6132
|
+
}
|
|
6133
|
+
if (!branchKey.includes(":")) {
|
|
6134
|
+
const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId));
|
|
6135
|
+
const referencesCurrentProjectSymbols = branchSymbolIds.some((symbolId) => projectLocalSymbolIds.has(symbolId));
|
|
6136
|
+
return !(referencesCurrentProjectChunks || referencesCurrentProjectSymbols);
|
|
6137
|
+
}
|
|
6138
|
+
return true;
|
|
6139
|
+
}
|
|
6140
|
+
);
|
|
6141
|
+
}
|
|
6142
|
+
saveScopedFailedBatches(batches, roots) {
|
|
6143
|
+
const { retained } = this.partitionFailedBatches(roots);
|
|
6144
|
+
this.saveFailedBatches([...retained, ...batches]);
|
|
6145
|
+
}
|
|
6146
|
+
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
6147
|
+
const allMetadata = store.getAllMetadata();
|
|
6148
|
+
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
6149
|
+
const filePaths = /* @__PURE__ */ new Set([
|
|
6150
|
+
...Array.from(this.fileHashCache.keys()).filter((filePath) => this.isFileInCurrentScope(filePath, roots)),
|
|
6151
|
+
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
6152
|
+
]);
|
|
6153
|
+
const projectRootPath = path8.resolve(this.projectRoot);
|
|
6154
|
+
const projectLocalFilePaths = new Set(
|
|
6155
|
+
Array.from(filePaths).filter((filePath) => isPathWithinRoot(filePath, projectRootPath))
|
|
6156
|
+
);
|
|
6157
|
+
const removedChunkIds = new Set(scopedEntries.map(({ key }) => key));
|
|
6158
|
+
for (const filePath of filePaths) {
|
|
6159
|
+
for (const chunk of database.getChunksByFile(filePath)) {
|
|
6160
|
+
removedChunkIds.add(chunk.chunkId);
|
|
6161
|
+
}
|
|
6162
|
+
}
|
|
6163
|
+
const removedChunkIdList = Array.from(removedChunkIds);
|
|
6164
|
+
const projectLocalChunkIds = new Set(
|
|
6165
|
+
scopedEntries.filter(({ metadata }) => isPathWithinRoot(metadata.filePath, projectRootPath)).map(({ key }) => key)
|
|
6166
|
+
);
|
|
6167
|
+
for (const filePath of projectLocalFilePaths) {
|
|
6168
|
+
for (const chunk of database.getChunksByFile(filePath)) {
|
|
6169
|
+
projectLocalChunkIds.add(chunk.chunkId);
|
|
6170
|
+
}
|
|
6171
|
+
}
|
|
6172
|
+
const symbolIds = [];
|
|
6173
|
+
const projectLocalSymbolIds = /* @__PURE__ */ new Set();
|
|
6174
|
+
for (const filePath of filePaths) {
|
|
6175
|
+
for (const symbol of database.getSymbolsByFile(filePath)) {
|
|
6176
|
+
symbolIds.push(symbol.id);
|
|
6177
|
+
if (projectLocalFilePaths.has(filePath)) {
|
|
6178
|
+
projectLocalSymbolIds.add(symbol.id);
|
|
6179
|
+
}
|
|
6180
|
+
}
|
|
6181
|
+
}
|
|
6182
|
+
for (const branchKey of this.getProjectScopedBranchCatalogCleanupKeys(Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds))) {
|
|
6183
|
+
database.deleteBranchChunksForBranch(branchKey, removedChunkIdList);
|
|
6184
|
+
}
|
|
6185
|
+
const sharedChunkIds = new Set(database.getReferencedChunkIds(removedChunkIdList));
|
|
6186
|
+
const removableChunkIds = removedChunkIdList.filter((chunkId) => !sharedChunkIds.has(chunkId));
|
|
6187
|
+
if (removableChunkIds.length > 0) {
|
|
6188
|
+
this.rebuildVectorStoreExcludingChunkIds(store, database, removableChunkIds);
|
|
6189
|
+
for (const chunkId of removableChunkIds) {
|
|
6190
|
+
invertedIndex.removeChunk(chunkId);
|
|
6191
|
+
}
|
|
6192
|
+
}
|
|
6193
|
+
for (const branchKey of this.getProjectScopedBranchCatalogCleanupKeys(Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds))) {
|
|
6194
|
+
database.deleteBranchSymbolsForBranch(branchKey, symbolIds);
|
|
6195
|
+
}
|
|
6196
|
+
const sharedSymbolIds = new Set(database.getReferencedSymbolIds(symbolIds));
|
|
6197
|
+
const removableSymbolIds = symbolIds.filter((symbolId) => !sharedSymbolIds.has(symbolId));
|
|
6198
|
+
database.clearCallEdgeTargetsForSymbols(removableSymbolIds);
|
|
6199
|
+
for (const filePath of filePaths) {
|
|
6200
|
+
const fileChunkIds = database.getChunksByFile(filePath).map((chunk) => chunk.chunkId);
|
|
6201
|
+
const fileSymbols = database.getSymbolsByFile(filePath);
|
|
6202
|
+
if (fileChunkIds.every((chunkId) => !sharedChunkIds.has(chunkId))) {
|
|
6203
|
+
database.deleteChunksByFile(filePath);
|
|
6204
|
+
}
|
|
6205
|
+
if (fileSymbols.every((symbol) => !sharedSymbolIds.has(symbol.id))) {
|
|
6206
|
+
database.deleteCallEdgesByFile(filePath);
|
|
6207
|
+
database.deleteSymbolsByFile(filePath);
|
|
6208
|
+
}
|
|
6209
|
+
}
|
|
6210
|
+
database.gcOrphanCallEdges();
|
|
6211
|
+
database.gcOrphanSymbols();
|
|
6212
|
+
database.gcOrphanEmbeddings();
|
|
6213
|
+
database.gcOrphanChunks();
|
|
6214
|
+
store.save();
|
|
6215
|
+
invertedIndex.save();
|
|
6216
|
+
return {
|
|
6217
|
+
removedChunkIds: removedChunkIdList,
|
|
6218
|
+
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
6219
|
+
};
|
|
6220
|
+
}
|
|
5201
6221
|
checkForInterruptedIndexing() {
|
|
5202
|
-
return
|
|
6222
|
+
return existsSync6(this.indexingLockPath);
|
|
5203
6223
|
}
|
|
5204
6224
|
acquireIndexingLock() {
|
|
5205
6225
|
const lockData = {
|
|
5206
6226
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5207
6227
|
pid: process.pid
|
|
5208
6228
|
};
|
|
5209
|
-
|
|
6229
|
+
writeFileSync3(this.indexingLockPath, JSON.stringify(lockData));
|
|
5210
6230
|
}
|
|
5211
6231
|
releaseIndexingLock() {
|
|
5212
|
-
if (
|
|
6232
|
+
if (existsSync6(this.indexingLockPath)) {
|
|
5213
6233
|
unlinkSync(this.indexingLockPath);
|
|
5214
6234
|
}
|
|
5215
6235
|
}
|
|
5216
6236
|
async recoverFromInterruptedIndexing() {
|
|
5217
6237
|
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
5218
|
-
if (
|
|
6238
|
+
if (existsSync6(this.fileHashCachePath)) {
|
|
5219
6239
|
unlinkSync(this.fileHashCachePath);
|
|
5220
6240
|
}
|
|
5221
6241
|
await this.healthCheck();
|
|
5222
6242
|
this.releaseIndexingLock();
|
|
5223
6243
|
this.logger.info("Recovery complete, next index will re-process all files");
|
|
5224
6244
|
}
|
|
5225
|
-
loadFailedBatches() {
|
|
6245
|
+
loadFailedBatches(maxChunkTokens) {
|
|
5226
6246
|
try {
|
|
5227
|
-
|
|
5228
|
-
const data = readFileSync5(this.failedBatchesPath, "utf-8");
|
|
5229
|
-
return JSON.parse(data);
|
|
5230
|
-
}
|
|
6247
|
+
return this.loadSerializedFailedBatches().map((batch) => normalizeFailedBatch(batch, maxChunkTokens)).filter((batch) => batch !== null);
|
|
5231
6248
|
} catch {
|
|
5232
6249
|
return [];
|
|
5233
6250
|
}
|
|
5234
|
-
|
|
6251
|
+
}
|
|
6252
|
+
loadSerializedFailedBatches() {
|
|
6253
|
+
if (!existsSync6(this.failedBatchesPath)) {
|
|
6254
|
+
return [];
|
|
6255
|
+
}
|
|
6256
|
+
const data = readFileSync6(this.failedBatchesPath, "utf-8");
|
|
6257
|
+
const parsed = JSON.parse(data);
|
|
6258
|
+
return parsed.map((batch) => {
|
|
6259
|
+
const chunks = Array.isArray(batch.chunks) ? batch.chunks : [];
|
|
6260
|
+
if (chunks.length === 0) {
|
|
6261
|
+
return null;
|
|
6262
|
+
}
|
|
6263
|
+
return {
|
|
6264
|
+
chunks,
|
|
6265
|
+
error: typeof batch.error === "string" ? batch.error : "Unknown embedding error",
|
|
6266
|
+
attemptCount: typeof batch.attemptCount === "number" ? batch.attemptCount : 1,
|
|
6267
|
+
lastAttempt: typeof batch.lastAttempt === "string" ? batch.lastAttempt : (/* @__PURE__ */ new Date()).toISOString()
|
|
6268
|
+
};
|
|
6269
|
+
}).filter((batch) => batch !== null);
|
|
5235
6270
|
}
|
|
5236
6271
|
saveFailedBatches(batches) {
|
|
5237
6272
|
if (batches.length === 0) {
|
|
5238
|
-
if (
|
|
5239
|
-
|
|
5240
|
-
|
|
6273
|
+
if (existsSync6(this.failedBatchesPath)) {
|
|
6274
|
+
try {
|
|
6275
|
+
unlinkSync(this.failedBatchesPath);
|
|
6276
|
+
} catch {
|
|
6277
|
+
}
|
|
5241
6278
|
}
|
|
5242
6279
|
return;
|
|
5243
6280
|
}
|
|
5244
|
-
|
|
6281
|
+
writeFileSync3(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
5245
6282
|
}
|
|
5246
|
-
|
|
5247
|
-
const
|
|
5248
|
-
|
|
5249
|
-
chunks
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
6283
|
+
collectRetryableFailedChunks(currentFileHashes, unchangedFilePaths, maxChunkTokens) {
|
|
6284
|
+
const retryableById = /* @__PURE__ */ new Map();
|
|
6285
|
+
for (const batch of this.loadFailedBatches(maxChunkTokens)) {
|
|
6286
|
+
for (const chunk of batch.chunks) {
|
|
6287
|
+
const filePath = chunk.metadata.filePath;
|
|
6288
|
+
if (!currentFileHashes.has(filePath)) {
|
|
6289
|
+
continue;
|
|
6290
|
+
}
|
|
6291
|
+
if (!unchangedFilePaths.has(filePath)) {
|
|
6292
|
+
continue;
|
|
6293
|
+
}
|
|
6294
|
+
const existing = retryableById.get(chunk.id);
|
|
6295
|
+
if (!existing || batch.attemptCount > existing.attemptCount) {
|
|
6296
|
+
retryableById.set(chunk.id, {
|
|
6297
|
+
chunk,
|
|
6298
|
+
attemptCount: batch.attemptCount
|
|
6299
|
+
});
|
|
6300
|
+
}
|
|
6301
|
+
}
|
|
6302
|
+
}
|
|
6303
|
+
return Array.from(retryableById.values());
|
|
5255
6304
|
}
|
|
5256
6305
|
getProviderRateLimits(provider) {
|
|
5257
6306
|
switch (provider) {
|
|
@@ -5456,31 +6505,54 @@ var Indexer = class {
|
|
|
5456
6505
|
}
|
|
5457
6506
|
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
5458
6507
|
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
5459
|
-
const storePath =
|
|
6508
|
+
const storePath = path8.join(this.indexPath, "vectors");
|
|
5460
6509
|
this.store = new VectorStore(storePath, dimensions);
|
|
5461
|
-
const indexFilePath =
|
|
5462
|
-
if (
|
|
6510
|
+
const indexFilePath = path8.join(this.indexPath, "vectors.usearch");
|
|
6511
|
+
if (existsSync6(indexFilePath)) {
|
|
5463
6512
|
this.store.load();
|
|
5464
6513
|
}
|
|
5465
|
-
const invertedIndexPath =
|
|
6514
|
+
const invertedIndexPath = path8.join(this.indexPath, "inverted-index.json");
|
|
5466
6515
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
5467
6516
|
try {
|
|
5468
6517
|
this.invertedIndex.load();
|
|
5469
6518
|
} catch {
|
|
5470
|
-
if (
|
|
6519
|
+
if (existsSync6(invertedIndexPath)) {
|
|
5471
6520
|
await fsPromises2.unlink(invertedIndexPath);
|
|
5472
6521
|
}
|
|
5473
6522
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
5474
6523
|
}
|
|
5475
|
-
const dbPath =
|
|
5476
|
-
|
|
5477
|
-
|
|
6524
|
+
const dbPath = path8.join(this.indexPath, "codebase.db");
|
|
6525
|
+
let dbIsNew = !existsSync6(dbPath);
|
|
6526
|
+
try {
|
|
6527
|
+
this.database = new Database(dbPath);
|
|
6528
|
+
} catch (error) {
|
|
6529
|
+
if (!await this.tryResetCorruptedIndex("initializing index database", error)) {
|
|
6530
|
+
throw error;
|
|
6531
|
+
}
|
|
6532
|
+
this.store = new VectorStore(storePath, dimensions);
|
|
6533
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
6534
|
+
this.database = new Database(dbPath);
|
|
6535
|
+
dbIsNew = true;
|
|
6536
|
+
}
|
|
6537
|
+
if (isGitRepo(this.projectRoot)) {
|
|
6538
|
+
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
6539
|
+
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
6540
|
+
this.logger.branch("info", "Detected git repository", {
|
|
6541
|
+
currentBranch: this.currentBranch,
|
|
6542
|
+
baseBranch: this.baseBranch
|
|
6543
|
+
});
|
|
6544
|
+
} else {
|
|
6545
|
+
this.currentBranch = "default";
|
|
6546
|
+
this.baseBranch = "default";
|
|
6547
|
+
this.logger.branch("debug", "Not a git repository, using default branch");
|
|
6548
|
+
}
|
|
5478
6549
|
if (this.checkForInterruptedIndexing()) {
|
|
5479
6550
|
await this.recoverFromInterruptedIndexing();
|
|
5480
6551
|
}
|
|
5481
6552
|
if (dbIsNew && this.store.count() > 0) {
|
|
5482
6553
|
this.migrateFromLegacyIndex();
|
|
5483
6554
|
}
|
|
6555
|
+
this.loadFileHashCache();
|
|
5484
6556
|
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
5485
6557
|
if (!this.indexCompatibility.compatible) {
|
|
5486
6558
|
this.logger.warn("Index compatibility issue detected", {
|
|
@@ -5489,18 +6561,6 @@ var Indexer = class {
|
|
|
5489
6561
|
configuredProviderInfo: this.configuredProviderInfo
|
|
5490
6562
|
});
|
|
5491
6563
|
}
|
|
5492
|
-
if (isGitRepo(this.projectRoot)) {
|
|
5493
|
-
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
5494
|
-
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
5495
|
-
this.logger.branch("info", "Detected git repository", {
|
|
5496
|
-
currentBranch: this.currentBranch,
|
|
5497
|
-
baseBranch: this.baseBranch
|
|
5498
|
-
});
|
|
5499
|
-
} else {
|
|
5500
|
-
this.currentBranch = "default";
|
|
5501
|
-
this.baseBranch = "default";
|
|
5502
|
-
this.logger.branch("debug", "Not a git repository, using default branch");
|
|
5503
|
-
}
|
|
5504
6564
|
if (this.config.indexing.autoGc) {
|
|
5505
6565
|
await this.maybeRunAutoGc();
|
|
5506
6566
|
}
|
|
@@ -5520,20 +6580,169 @@ var Indexer = class {
|
|
|
5520
6580
|
}
|
|
5521
6581
|
}
|
|
5522
6582
|
if (shouldRunGc) {
|
|
5523
|
-
await this.healthCheck();
|
|
6583
|
+
const result = await this.healthCheck();
|
|
6584
|
+
if (result.warning) {
|
|
6585
|
+
this.database.setMetadata(STARTUP_WARNING_METADATA_KEY, result.warning);
|
|
6586
|
+
} else {
|
|
6587
|
+
this.database.deleteMetadata(STARTUP_WARNING_METADATA_KEY);
|
|
6588
|
+
}
|
|
5524
6589
|
this.database.setMetadata("lastGcTimestamp", now.toString());
|
|
5525
6590
|
}
|
|
5526
6591
|
}
|
|
5527
6592
|
async maybeRunOrphanGc() {
|
|
5528
|
-
if (!this.database) return;
|
|
6593
|
+
if (!this.database) return null;
|
|
5529
6594
|
const stats = this.database.getStats();
|
|
5530
|
-
if (!stats) return;
|
|
6595
|
+
if (!stats) return null;
|
|
5531
6596
|
const orphanCount = stats.embeddingCount - stats.chunkCount;
|
|
5532
6597
|
if (orphanCount > this.config.indexing.gcOrphanThreshold) {
|
|
5533
|
-
|
|
5534
|
-
|
|
6598
|
+
try {
|
|
6599
|
+
this.database.gcOrphanEmbeddings();
|
|
6600
|
+
this.database.gcOrphanChunks();
|
|
6601
|
+
} catch (error) {
|
|
6602
|
+
if (await this.tryResetCorruptedIndex("running automatic orphan garbage collection", error)) {
|
|
6603
|
+
return {
|
|
6604
|
+
resetCorruptedIndex: true,
|
|
6605
|
+
warning: this.getCorruptedIndexWarning(path8.join(this.indexPath, "codebase.db"))
|
|
6606
|
+
};
|
|
6607
|
+
}
|
|
6608
|
+
throw error;
|
|
6609
|
+
}
|
|
5535
6610
|
this.database.setMetadata("lastGcTimestamp", Date.now().toString());
|
|
5536
6611
|
}
|
|
6612
|
+
return null;
|
|
6613
|
+
}
|
|
6614
|
+
rebuildVectorStoreExcludingChunkIds(store, database, excludedChunkIds) {
|
|
6615
|
+
const excludedSet = new Set(excludedChunkIds);
|
|
6616
|
+
if (excludedSet.size === 0) {
|
|
6617
|
+
return;
|
|
6618
|
+
}
|
|
6619
|
+
const retainedEntries = store.getAllMetadata().filter(({ key }) => !excludedSet.has(key));
|
|
6620
|
+
const storeBasePath = path8.join(this.indexPath, "vectors");
|
|
6621
|
+
const storeIndexPath = `${storeBasePath}.usearch`;
|
|
6622
|
+
const storeMetadataPath = `${storeBasePath}.meta.json`;
|
|
6623
|
+
const backupIndexPath = `${storeIndexPath}.bak`;
|
|
6624
|
+
const backupMetadataPath = `${storeMetadataPath}.bak`;
|
|
6625
|
+
let backedUpIndex = false;
|
|
6626
|
+
let backedUpMetadata = false;
|
|
6627
|
+
let rebuiltCount = 0;
|
|
6628
|
+
let skippedCount = 0;
|
|
6629
|
+
if (existsSync6(backupIndexPath)) {
|
|
6630
|
+
unlinkSync(backupIndexPath);
|
|
6631
|
+
}
|
|
6632
|
+
if (existsSync6(backupMetadataPath)) {
|
|
6633
|
+
unlinkSync(backupMetadataPath);
|
|
6634
|
+
}
|
|
6635
|
+
try {
|
|
6636
|
+
if (existsSync6(storeIndexPath)) {
|
|
6637
|
+
renameSync(storeIndexPath, backupIndexPath);
|
|
6638
|
+
backedUpIndex = true;
|
|
6639
|
+
}
|
|
6640
|
+
if (existsSync6(storeMetadataPath)) {
|
|
6641
|
+
renameSync(storeMetadataPath, backupMetadataPath);
|
|
6642
|
+
backedUpMetadata = true;
|
|
6643
|
+
}
|
|
6644
|
+
store.clear();
|
|
6645
|
+
for (const { key, metadata } of retainedEntries) {
|
|
6646
|
+
const chunk = database.getChunk(key);
|
|
6647
|
+
if (!chunk) {
|
|
6648
|
+
skippedCount += 1;
|
|
6649
|
+
continue;
|
|
6650
|
+
}
|
|
6651
|
+
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
6652
|
+
if (!embeddingBuffer) {
|
|
6653
|
+
skippedCount += 1;
|
|
6654
|
+
continue;
|
|
6655
|
+
}
|
|
6656
|
+
const vector = bufferToFloat32Array(embeddingBuffer);
|
|
6657
|
+
store.add(key, Array.from(vector), metadata);
|
|
6658
|
+
rebuiltCount += 1;
|
|
6659
|
+
}
|
|
6660
|
+
store.save();
|
|
6661
|
+
if (backedUpIndex && existsSync6(backupIndexPath)) {
|
|
6662
|
+
unlinkSync(backupIndexPath);
|
|
6663
|
+
}
|
|
6664
|
+
if (backedUpMetadata && existsSync6(backupMetadataPath)) {
|
|
6665
|
+
unlinkSync(backupMetadataPath);
|
|
6666
|
+
}
|
|
6667
|
+
this.logger.gc("info", "Rebuilt vector store to avoid native remove", {
|
|
6668
|
+
excludedChunks: excludedSet.size,
|
|
6669
|
+
rebuiltChunks: rebuiltCount,
|
|
6670
|
+
skippedChunks: skippedCount
|
|
6671
|
+
});
|
|
6672
|
+
} catch (error) {
|
|
6673
|
+
try {
|
|
6674
|
+
store.clear();
|
|
6675
|
+
} catch {
|
|
6676
|
+
}
|
|
6677
|
+
if (existsSync6(storeIndexPath)) {
|
|
6678
|
+
unlinkSync(storeIndexPath);
|
|
6679
|
+
}
|
|
6680
|
+
if (existsSync6(storeMetadataPath)) {
|
|
6681
|
+
unlinkSync(storeMetadataPath);
|
|
6682
|
+
}
|
|
6683
|
+
if (backedUpIndex && existsSync6(backupIndexPath)) {
|
|
6684
|
+
renameSync(backupIndexPath, storeIndexPath);
|
|
6685
|
+
}
|
|
6686
|
+
if (backedUpMetadata && existsSync6(backupMetadataPath)) {
|
|
6687
|
+
renameSync(backupMetadataPath, storeMetadataPath);
|
|
6688
|
+
}
|
|
6689
|
+
if (backedUpIndex || backedUpMetadata) {
|
|
6690
|
+
store.load();
|
|
6691
|
+
}
|
|
6692
|
+
throw error;
|
|
6693
|
+
}
|
|
6694
|
+
}
|
|
6695
|
+
getCorruptedIndexWarning(dbPath) {
|
|
6696
|
+
if (this.config.scope === "global") {
|
|
6697
|
+
return `Detected a corrupted shared global SQLite index at ${dbPath}. Automatic repair is disabled for global scope because it may delete other projects' index data. Remove or repair the shared index manually, then rerun index_codebase with force=true.`;
|
|
6698
|
+
}
|
|
6699
|
+
return `Detected a corrupted local SQLite index at ${dbPath} and reset the local index. Run index_codebase to rebuild search data.`;
|
|
6700
|
+
}
|
|
6701
|
+
async tryResetCorruptedIndex(stage, error) {
|
|
6702
|
+
if (!isSqliteCorruptionError(error)) {
|
|
6703
|
+
return false;
|
|
6704
|
+
}
|
|
6705
|
+
const dbPath = path8.join(this.indexPath, "codebase.db");
|
|
6706
|
+
const warning = this.getCorruptedIndexWarning(dbPath);
|
|
6707
|
+
const errorMessage = getErrorMessage(error);
|
|
6708
|
+
if (this.config.scope === "global") {
|
|
6709
|
+
this.logger.error("Detected corrupted shared global index database", {
|
|
6710
|
+
stage,
|
|
6711
|
+
dbPath,
|
|
6712
|
+
error: errorMessage
|
|
6713
|
+
});
|
|
6714
|
+
throw new Error(`${warning} Original SQLite error: ${errorMessage}`);
|
|
6715
|
+
}
|
|
6716
|
+
this.logger.warn("Detected corrupted local index database, resetting local index", {
|
|
6717
|
+
stage,
|
|
6718
|
+
dbPath,
|
|
6719
|
+
error: errorMessage
|
|
6720
|
+
});
|
|
6721
|
+
this.store = null;
|
|
6722
|
+
this.invertedIndex = null;
|
|
6723
|
+
this.database?.close();
|
|
6724
|
+
this.database = null;
|
|
6725
|
+
this.indexCompatibility = null;
|
|
6726
|
+
this.fileHashCache.clear();
|
|
6727
|
+
const resetPaths = [
|
|
6728
|
+
path8.join(this.indexPath, "codebase.db"),
|
|
6729
|
+
path8.join(this.indexPath, "codebase.db-shm"),
|
|
6730
|
+
path8.join(this.indexPath, "codebase.db-wal"),
|
|
6731
|
+
path8.join(this.indexPath, "vectors.usearch"),
|
|
6732
|
+
path8.join(this.indexPath, "inverted-index.json"),
|
|
6733
|
+
path8.join(this.indexPath, "file-hashes.json"),
|
|
6734
|
+
path8.join(this.indexPath, "failed-batches.json"),
|
|
6735
|
+
path8.join(this.indexPath, "indexing.lock"),
|
|
6736
|
+
path8.join(this.indexPath, "vectors")
|
|
6737
|
+
];
|
|
6738
|
+
await Promise.all(resetPaths.map(async (targetPath) => {
|
|
6739
|
+
try {
|
|
6740
|
+
await fsPromises2.rm(targetPath, { recursive: true, force: true });
|
|
6741
|
+
} catch {
|
|
6742
|
+
}
|
|
6743
|
+
}));
|
|
6744
|
+
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
6745
|
+
return true;
|
|
5537
6746
|
}
|
|
5538
6747
|
migrateFromLegacyIndex() {
|
|
5539
6748
|
if (!this.store || !this.database) return;
|
|
@@ -5557,7 +6766,7 @@ var Indexer = class {
|
|
|
5557
6766
|
if (chunkDataBatch.length > 0) {
|
|
5558
6767
|
this.database.upsertChunksBatch(chunkDataBatch);
|
|
5559
6768
|
}
|
|
5560
|
-
this.database.addChunksToBranchBatch(this.
|
|
6769
|
+
this.database.addChunksToBranchBatch(this.getBranchCatalogKey(), chunkIds);
|
|
5561
6770
|
}
|
|
5562
6771
|
loadIndexMetadata() {
|
|
5563
6772
|
if (!this.database) return null;
|
|
@@ -5568,6 +6777,7 @@ var Indexer = class {
|
|
|
5568
6777
|
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
5569
6778
|
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
5570
6779
|
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
6780
|
+
embeddingStrategyVersion: this.loadStoredEmbeddingStrategyVersion() ?? EMBEDDING_STRATEGY_VERSION,
|
|
5571
6781
|
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
5572
6782
|
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
5573
6783
|
};
|
|
@@ -5576,10 +6786,22 @@ var Indexer = class {
|
|
|
5576
6786
|
if (!this.database) return;
|
|
5577
6787
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5578
6788
|
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
6789
|
+
const completeProjectEmbeddingStrategyReset = !this.hasProjectForceReembedPending();
|
|
5579
6790
|
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
5580
6791
|
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
5581
6792
|
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
5582
6793
|
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
6794
|
+
if (this.config.scope === "global") {
|
|
6795
|
+
if (completeProjectEmbeddingStrategyReset) {
|
|
6796
|
+
this.database.setMetadata(this.getProjectEmbeddingStrategyMetadataKey(), EMBEDDING_STRATEGY_VERSION);
|
|
6797
|
+
}
|
|
6798
|
+
this.database.setMetadata(this.getLegacyMigrationMetadataKey(), "done");
|
|
6799
|
+
if (completeProjectEmbeddingStrategyReset) {
|
|
6800
|
+
this.database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
6801
|
+
}
|
|
6802
|
+
} else {
|
|
6803
|
+
this.database.setMetadata("index.embeddingStrategyVersion", EMBEDDING_STRATEGY_VERSION);
|
|
6804
|
+
}
|
|
5583
6805
|
this.database.setMetadata("index.updatedAt", now);
|
|
5584
6806
|
if (!existingCreatedAt) {
|
|
5585
6807
|
this.database.setMetadata("index.createdAt", now);
|
|
@@ -5609,6 +6831,14 @@ var Indexer = class {
|
|
|
5609
6831
|
storedMetadata
|
|
5610
6832
|
};
|
|
5611
6833
|
}
|
|
6834
|
+
if (storedMetadata.embeddingStrategyVersion !== EMBEDDING_STRATEGY_VERSION) {
|
|
6835
|
+
return {
|
|
6836
|
+
compatible: false,
|
|
6837
|
+
code: "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */,
|
|
6838
|
+
reason: `Embedding strategy mismatch: index was built with embedding strategy v${storedMetadata.embeddingStrategyVersion}, but the current code requires v${EMBEDDING_STRATEGY_VERSION}. Run index_codebase with force=true to rebuild cached embeddings.`,
|
|
6839
|
+
storedMetadata
|
|
6840
|
+
};
|
|
6841
|
+
}
|
|
5612
6842
|
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
5613
6843
|
this.logger.warn("Provider changed", {
|
|
5614
6844
|
storedProvider: storedMetadata.embeddingProvider,
|
|
@@ -5656,6 +6886,10 @@ var Indexer = class {
|
|
|
5656
6886
|
}
|
|
5657
6887
|
async index(onProgress) {
|
|
5658
6888
|
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
6889
|
+
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
6890
|
+
const branchCatalogKey = this.getBranchCatalogKey();
|
|
6891
|
+
const forceScopedReembed = scopedRoots !== null && database.getMetadata(this.getProjectForceReembedMetadataKey()) === "true";
|
|
6892
|
+
const failedForcedChunkIds = /* @__PURE__ */ new Set();
|
|
5659
6893
|
if (!this.indexCompatibility?.compatible) {
|
|
5660
6894
|
throw new Error(
|
|
5661
6895
|
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
@@ -5677,6 +6911,7 @@ var Indexer = class {
|
|
|
5677
6911
|
skippedFiles: [],
|
|
5678
6912
|
parseFailures: []
|
|
5679
6913
|
};
|
|
6914
|
+
const failedBatchesForCurrentRun = [];
|
|
5680
6915
|
onProgress?.({
|
|
5681
6916
|
phase: "scanning",
|
|
5682
6917
|
filesProcessed: 0,
|
|
@@ -5736,6 +6971,12 @@ var Indexer = class {
|
|
|
5736
6971
|
const existingChunks = /* @__PURE__ */ new Map();
|
|
5737
6972
|
const existingChunksByFile = /* @__PURE__ */ new Map();
|
|
5738
6973
|
for (const { key, metadata } of store.getAllMetadata()) {
|
|
6974
|
+
if (scopedRoots && !this.isFileInCurrentScope(metadata.filePath, scopedRoots)) {
|
|
6975
|
+
continue;
|
|
6976
|
+
}
|
|
6977
|
+
if (forceScopedReembed && scopedRoots && this.isFileInCurrentScope(metadata.filePath, scopedRoots)) {
|
|
6978
|
+
continue;
|
|
6979
|
+
}
|
|
5739
6980
|
existingChunks.set(key, metadata.hash);
|
|
5740
6981
|
const fileChunks = existingChunksByFile.get(metadata.filePath) || /* @__PURE__ */ new Set();
|
|
5741
6982
|
fileChunks.add(key);
|
|
@@ -5757,7 +6998,7 @@ var Indexer = class {
|
|
|
5757
6998
|
for (const parsed of parsedFiles) {
|
|
5758
6999
|
currentFilePaths.add(parsed.path);
|
|
5759
7000
|
if (parsed.chunks.length === 0) {
|
|
5760
|
-
const relativePath =
|
|
7001
|
+
const relativePath = path8.relative(this.projectRoot, parsed.path);
|
|
5761
7002
|
stats.parseFailures.push(relativePath);
|
|
5762
7003
|
}
|
|
5763
7004
|
let fileChunkCount = 0;
|
|
@@ -5793,7 +7034,10 @@ var Indexer = class {
|
|
|
5793
7034
|
fileChunkCount++;
|
|
5794
7035
|
continue;
|
|
5795
7036
|
}
|
|
5796
|
-
const
|
|
7037
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)).map((text) => ({
|
|
7038
|
+
text,
|
|
7039
|
+
tokenCount: estimateTokens2(text)
|
|
7040
|
+
}));
|
|
5797
7041
|
const metadata = {
|
|
5798
7042
|
filePath: parsed.path,
|
|
5799
7043
|
startLine: chunk.startLine,
|
|
@@ -5803,10 +7047,38 @@ var Indexer = class {
|
|
|
5803
7047
|
language: chunk.language,
|
|
5804
7048
|
hash: contentHash
|
|
5805
7049
|
};
|
|
5806
|
-
pendingChunks.push({
|
|
7050
|
+
pendingChunks.push({
|
|
7051
|
+
id,
|
|
7052
|
+
texts,
|
|
7053
|
+
storageText: createPendingChunkStorageText(texts),
|
|
7054
|
+
content: chunk.content,
|
|
7055
|
+
contentHash,
|
|
7056
|
+
metadata
|
|
7057
|
+
});
|
|
5807
7058
|
fileChunkCount++;
|
|
5808
7059
|
}
|
|
5809
7060
|
}
|
|
7061
|
+
const retryableFailedChunks = this.collectRetryableFailedChunks(
|
|
7062
|
+
currentFileHashes,
|
|
7063
|
+
unchangedFilePaths,
|
|
7064
|
+
getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)
|
|
7065
|
+
);
|
|
7066
|
+
const retryableFailedAttemptCounts = /* @__PURE__ */ new Map();
|
|
7067
|
+
const retryableChunksWithExistingData = /* @__PURE__ */ new Set();
|
|
7068
|
+
if (retryableFailedChunks.length > 0) {
|
|
7069
|
+
const pendingChunkIds = new Set(pendingChunks.map((chunk) => chunk.id));
|
|
7070
|
+
for (const { chunk, attemptCount } of retryableFailedChunks) {
|
|
7071
|
+
retryableFailedAttemptCounts.set(chunk.id, attemptCount);
|
|
7072
|
+
if (existingChunks.has(chunk.id)) {
|
|
7073
|
+
retryableChunksWithExistingData.add(chunk.id);
|
|
7074
|
+
}
|
|
7075
|
+
if (!pendingChunkIds.has(chunk.id)) {
|
|
7076
|
+
pendingChunks.push(chunk);
|
|
7077
|
+
pendingChunkIds.add(chunk.id);
|
|
7078
|
+
currentChunkIds.add(chunk.id);
|
|
7079
|
+
}
|
|
7080
|
+
}
|
|
7081
|
+
}
|
|
5810
7082
|
if (chunkDataBatch.length > 0) {
|
|
5811
7083
|
database.upsertChunksBatch(chunkDataBatch);
|
|
5812
7084
|
}
|
|
@@ -5835,17 +7107,20 @@ var Indexer = class {
|
|
|
5835
7107
|
fileSymbols.push(symbol);
|
|
5836
7108
|
allSymbolIds.add(symbolId);
|
|
5837
7109
|
}
|
|
7110
|
+
const fileLanguage = parsed.chunks[0]?.language;
|
|
7111
|
+
const isCaseInsensitiveLanguage = !!fileLanguage && CASE_INSENSITIVE_LANGUAGES.has(fileLanguage);
|
|
7112
|
+
const normalizeSymbolKey = (name) => isCaseInsensitiveLanguage ? name.toLowerCase() : name;
|
|
5838
7113
|
const symbolsByName = /* @__PURE__ */ new Map();
|
|
5839
7114
|
for (const symbol of fileSymbols) {
|
|
5840
|
-
const
|
|
7115
|
+
const key = normalizeSymbolKey(symbol.name);
|
|
7116
|
+
const existing = symbolsByName.get(key) ?? [];
|
|
5841
7117
|
existing.push(symbol);
|
|
5842
|
-
symbolsByName.set(
|
|
7118
|
+
symbolsByName.set(key, existing);
|
|
5843
7119
|
}
|
|
5844
7120
|
if (fileSymbols.length > 0) {
|
|
5845
7121
|
database.upsertSymbolsBatch(fileSymbols);
|
|
5846
7122
|
symbolsByFile.set(parsed.path, fileSymbols);
|
|
5847
7123
|
}
|
|
5848
|
-
const fileLanguage = parsed.chunks[0]?.language;
|
|
5849
7124
|
if (!fileLanguage || !CALL_GRAPH_LANGUAGES.has(fileLanguage)) continue;
|
|
5850
7125
|
const callSites = extractCalls(changedFile.content, fileLanguage);
|
|
5851
7126
|
if (callSites.length === 0) continue;
|
|
@@ -5870,7 +7145,7 @@ var Indexer = class {
|
|
|
5870
7145
|
if (edges.length > 0) {
|
|
5871
7146
|
database.upsertCallEdgesBatch(edges);
|
|
5872
7147
|
for (const edge of edges) {
|
|
5873
|
-
const candidates = symbolsByName.get(edge.targetName);
|
|
7148
|
+
const candidates = symbolsByName.get(normalizeSymbolKey(edge.targetName));
|
|
5874
7149
|
if (candidates && candidates.length === 1) {
|
|
5875
7150
|
database.resolveCallEdge(edge.id, candidates[0].id);
|
|
5876
7151
|
}
|
|
@@ -5883,14 +7158,20 @@ var Indexer = class {
|
|
|
5883
7158
|
allSymbolIds.add(sym.id);
|
|
5884
7159
|
}
|
|
5885
7160
|
}
|
|
5886
|
-
|
|
7161
|
+
const removedChunkIds = [];
|
|
5887
7162
|
for (const [chunkId] of existingChunks) {
|
|
5888
7163
|
if (!currentChunkIds.has(chunkId)) {
|
|
5889
|
-
|
|
7164
|
+
removedChunkIds.push(chunkId);
|
|
7165
|
+
}
|
|
7166
|
+
}
|
|
7167
|
+
if (removedChunkIds.length > 0) {
|
|
7168
|
+
this.rebuildVectorStoreExcludingChunkIds(store, database, removedChunkIds);
|
|
7169
|
+
for (const chunkId of removedChunkIds) {
|
|
5890
7170
|
invertedIndex.removeChunk(chunkId);
|
|
5891
|
-
removedCount++;
|
|
5892
7171
|
}
|
|
7172
|
+
database.deleteChunksByIds(removedChunkIds);
|
|
5893
7173
|
}
|
|
7174
|
+
const removedCount = removedChunkIds.length;
|
|
5894
7175
|
stats.totalChunks = pendingChunks.length;
|
|
5895
7176
|
stats.existingChunks = currentChunkIds.size - pendingChunks.length;
|
|
5896
7177
|
stats.removedChunks = removedCount;
|
|
@@ -5902,12 +7183,20 @@ var Indexer = class {
|
|
|
5902
7183
|
removed: removedCount
|
|
5903
7184
|
});
|
|
5904
7185
|
if (pendingChunks.length === 0 && removedCount === 0) {
|
|
5905
|
-
database.clearBranch(
|
|
5906
|
-
database.addChunksToBranchBatch(
|
|
5907
|
-
database.clearBranchSymbols(
|
|
5908
|
-
database.addSymbolsToBranchBatch(
|
|
5909
|
-
|
|
5910
|
-
|
|
7186
|
+
database.clearBranch(branchCatalogKey);
|
|
7187
|
+
database.addChunksToBranchBatch(branchCatalogKey, Array.from(currentChunkIds));
|
|
7188
|
+
database.clearBranchSymbols(branchCatalogKey);
|
|
7189
|
+
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
7190
|
+
if (scopedRoots) {
|
|
7191
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7192
|
+
this.clearScopedFailedBatches(scopedRoots);
|
|
7193
|
+
} else {
|
|
7194
|
+
this.fileHashCache = currentFileHashes;
|
|
7195
|
+
this.saveFileHashCache();
|
|
7196
|
+
this.saveFailedBatches([]);
|
|
7197
|
+
}
|
|
7198
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
7199
|
+
this.indexCompatibility = { compatible: true };
|
|
5911
7200
|
stats.durationMs = Date.now() - startTime;
|
|
5912
7201
|
onProgress?.({
|
|
5913
7202
|
phase: "complete",
|
|
@@ -5920,14 +7209,22 @@ var Indexer = class {
|
|
|
5920
7209
|
return stats;
|
|
5921
7210
|
}
|
|
5922
7211
|
if (pendingChunks.length === 0) {
|
|
5923
|
-
database.clearBranch(
|
|
5924
|
-
database.addChunksToBranchBatch(
|
|
5925
|
-
database.clearBranchSymbols(
|
|
5926
|
-
database.addSymbolsToBranchBatch(
|
|
7212
|
+
database.clearBranch(branchCatalogKey);
|
|
7213
|
+
database.addChunksToBranchBatch(branchCatalogKey, Array.from(currentChunkIds));
|
|
7214
|
+
database.clearBranchSymbols(branchCatalogKey);
|
|
7215
|
+
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
5927
7216
|
store.save();
|
|
5928
7217
|
invertedIndex.save();
|
|
5929
|
-
|
|
5930
|
-
|
|
7218
|
+
if (scopedRoots) {
|
|
7219
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7220
|
+
this.clearScopedFailedBatches(scopedRoots);
|
|
7221
|
+
} else {
|
|
7222
|
+
this.fileHashCache = currentFileHashes;
|
|
7223
|
+
this.saveFileHashCache();
|
|
7224
|
+
this.saveFailedBatches([]);
|
|
7225
|
+
}
|
|
7226
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
7227
|
+
this.indexCompatibility = { compatible: true };
|
|
5931
7228
|
stats.durationMs = Date.now() - startTime;
|
|
5932
7229
|
onProgress?.({
|
|
5933
7230
|
phase: "complete",
|
|
@@ -5948,8 +7245,9 @@ var Indexer = class {
|
|
|
5948
7245
|
});
|
|
5949
7246
|
const allContentHashes = pendingChunks.map((c) => c.contentHash);
|
|
5950
7247
|
const missingHashes = new Set(database.getMissingEmbeddings(allContentHashes));
|
|
5951
|
-
const
|
|
5952
|
-
const
|
|
7248
|
+
const forcedReembedChunkIds = forceScopedReembed ? new Set(pendingChunks.map((chunk) => chunk.id)) : /* @__PURE__ */ new Set();
|
|
7249
|
+
const chunksNeedingEmbedding = pendingChunks.filter((c) => forcedReembedChunkIds.has(c.id) || missingHashes.has(c.contentHash));
|
|
7250
|
+
const chunksWithExistingEmbedding = pendingChunks.filter((c) => !forcedReembedChunkIds.has(c.id) && !missingHashes.has(c.contentHash));
|
|
5953
7251
|
this.logger.cache("info", "Embedding cache lookup", {
|
|
5954
7252
|
needsEmbedding: chunksNeedingEmbedding.length,
|
|
5955
7253
|
fromCache: chunksWithExistingEmbedding.length
|
|
@@ -5971,17 +7269,24 @@ var Indexer = class {
|
|
|
5971
7269
|
interval: providerRateLimits.intervalMs,
|
|
5972
7270
|
intervalCap: providerRateLimits.concurrency
|
|
5973
7271
|
});
|
|
5974
|
-
const
|
|
7272
|
+
const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk]));
|
|
7273
|
+
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
7274
|
+
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
7275
|
+
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
7276
|
+
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
7277
|
+
chunksNeedingEmbedding,
|
|
7278
|
+
getDynamicBatchOptions(configuredProviderInfo)
|
|
7279
|
+
);
|
|
5975
7280
|
let rateLimitBackoffMs = 0;
|
|
5976
|
-
for (const
|
|
7281
|
+
for (const requestBatch of requestBatches) {
|
|
5977
7282
|
queue.add(async () => {
|
|
5978
7283
|
if (rateLimitBackoffMs > 0) {
|
|
5979
|
-
await new Promise((
|
|
7284
|
+
await new Promise((resolve8) => setTimeout(resolve8, rateLimitBackoffMs));
|
|
5980
7285
|
}
|
|
5981
7286
|
try {
|
|
5982
7287
|
const result = await pRetry(
|
|
5983
7288
|
async () => {
|
|
5984
|
-
const texts =
|
|
7289
|
+
const texts = requestBatch.map((request) => request.text);
|
|
5985
7290
|
return provider.embedBatch(texts);
|
|
5986
7291
|
},
|
|
5987
7292
|
{
|
|
@@ -6011,29 +7316,82 @@ var Indexer = class {
|
|
|
6011
7316
|
if (rateLimitBackoffMs > 0) {
|
|
6012
7317
|
rateLimitBackoffMs = Math.max(0, rateLimitBackoffMs - 2e3);
|
|
6013
7318
|
}
|
|
6014
|
-
const
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
|
|
6029
|
-
|
|
7319
|
+
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
7320
|
+
requestBatch.forEach((request, idx) => {
|
|
7321
|
+
if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
7322
|
+
return;
|
|
7323
|
+
}
|
|
7324
|
+
const vector = result.embeddings[idx];
|
|
7325
|
+
if (!vector) {
|
|
7326
|
+
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
7327
|
+
}
|
|
7328
|
+
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
7329
|
+
parts[request.partIndex] = {
|
|
7330
|
+
vector,
|
|
7331
|
+
tokenCount: request.tokenCount
|
|
7332
|
+
};
|
|
7333
|
+
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
7334
|
+
touchedChunkIds.add(request.chunk.id);
|
|
7335
|
+
});
|
|
7336
|
+
const pooledResults = [];
|
|
7337
|
+
for (const chunkId of touchedChunkIds) {
|
|
7338
|
+
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
7339
|
+
continue;
|
|
7340
|
+
}
|
|
7341
|
+
const chunk = pendingChunksById.get(chunkId);
|
|
7342
|
+
if (!chunk) {
|
|
7343
|
+
continue;
|
|
7344
|
+
}
|
|
7345
|
+
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
7346
|
+
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
7347
|
+
continue;
|
|
7348
|
+
}
|
|
7349
|
+
const orderedParts = parts;
|
|
7350
|
+
pooledResults.push({
|
|
7351
|
+
chunk,
|
|
7352
|
+
vector: poolEmbeddingVectors(
|
|
7353
|
+
orderedParts.map((part) => part.vector),
|
|
7354
|
+
orderedParts.map((part) => part.tokenCount)
|
|
7355
|
+
)
|
|
7356
|
+
});
|
|
7357
|
+
}
|
|
7358
|
+
if (pooledResults.length > 0) {
|
|
7359
|
+
const items = pooledResults.map(({ chunk, vector }) => ({
|
|
7360
|
+
id: chunk.id,
|
|
7361
|
+
vector,
|
|
7362
|
+
metadata: chunk.metadata
|
|
7363
|
+
}));
|
|
7364
|
+
store.addBatch(items);
|
|
7365
|
+
const embeddingBatchItems = pooledResults.map(({ chunk, vector }) => ({
|
|
7366
|
+
contentHash: chunk.contentHash,
|
|
7367
|
+
embedding: float32ArrayToBuffer(vector),
|
|
7368
|
+
chunkText: chunk.storageText,
|
|
7369
|
+
model: configuredProviderInfo.modelInfo.model
|
|
7370
|
+
}));
|
|
7371
|
+
try {
|
|
7372
|
+
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
7373
|
+
} catch (dbError) {
|
|
7374
|
+
this.rebuildVectorStoreExcludingChunkIds(
|
|
7375
|
+
store,
|
|
7376
|
+
database,
|
|
7377
|
+
pooledResults.map(({ chunk }) => chunk.id)
|
|
7378
|
+
);
|
|
7379
|
+
throw dbError;
|
|
7380
|
+
}
|
|
7381
|
+
for (const { chunk } of pooledResults) {
|
|
7382
|
+
invertedIndex.removeChunk(chunk.id);
|
|
7383
|
+
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
7384
|
+
completedChunkIds.add(chunk.id);
|
|
7385
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
7386
|
+
}
|
|
7387
|
+
stats.indexedChunks += pooledResults.length;
|
|
7388
|
+
this.logger.recordChunksEmbedded(pooledResults.length);
|
|
6030
7389
|
}
|
|
6031
|
-
stats.indexedChunks += batch.length;
|
|
6032
7390
|
stats.tokensUsed += result.totalTokensUsed;
|
|
6033
|
-
this.logger.recordChunksEmbedded(batch.length);
|
|
6034
7391
|
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
6035
7392
|
this.logger.embedding("debug", `Embedded batch`, {
|
|
6036
|
-
batchSize:
|
|
7393
|
+
batchSize: pooledResults.length,
|
|
7394
|
+
requestCount: requestBatch.length,
|
|
6037
7395
|
tokens: result.totalTokensUsed
|
|
6038
7396
|
});
|
|
6039
7397
|
onProgress?.({
|
|
@@ -6044,17 +7402,49 @@ var Indexer = class {
|
|
|
6044
7402
|
totalChunks: pendingChunks.length
|
|
6045
7403
|
});
|
|
6046
7404
|
} catch (error) {
|
|
6047
|
-
|
|
6048
|
-
|
|
7405
|
+
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id));
|
|
7406
|
+
const failureMessage = getErrorMessage(error);
|
|
7407
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
7408
|
+
for (const chunk of failedChunks) {
|
|
7409
|
+
if (!failedChunkIds.has(chunk.id)) {
|
|
7410
|
+
failedChunkIds.add(chunk.id);
|
|
7411
|
+
stats.failedChunks += 1;
|
|
7412
|
+
}
|
|
7413
|
+
if (forceScopedReembed) {
|
|
7414
|
+
failedForcedChunkIds.add(chunk.id);
|
|
7415
|
+
}
|
|
7416
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
7417
|
+
const existingFailedBatchIndex = failedBatchesForCurrentRun.findIndex(
|
|
7418
|
+
(failedBatch2) => failedBatch2.chunks[0]?.id === chunk.id
|
|
7419
|
+
);
|
|
7420
|
+
const existingFailedBatch = existingFailedBatchIndex === -1 ? void 0 : failedBatchesForCurrentRun[existingFailedBatchIndex];
|
|
7421
|
+
const failedBatch = {
|
|
7422
|
+
chunks: [chunk],
|
|
7423
|
+
error: failureMessage,
|
|
7424
|
+
attemptCount: (existingFailedBatch?.attemptCount ?? retryableFailedAttemptCounts.get(chunk.id) ?? 0) + 1,
|
|
7425
|
+
lastAttempt: failureTimestamp
|
|
7426
|
+
};
|
|
7427
|
+
if (existingFailedBatchIndex === -1) {
|
|
7428
|
+
failedBatchesForCurrentRun.push(failedBatch);
|
|
7429
|
+
} else {
|
|
7430
|
+
failedBatchesForCurrentRun[existingFailedBatchIndex] = failedBatch;
|
|
7431
|
+
}
|
|
7432
|
+
}
|
|
6049
7433
|
this.logger.recordEmbeddingError();
|
|
6050
7434
|
this.logger.embedding("error", `Failed to embed batch after retries`, {
|
|
6051
|
-
batchSize:
|
|
6052
|
-
|
|
7435
|
+
batchSize: failedChunks.length,
|
|
7436
|
+
requestCount: requestBatch.length,
|
|
7437
|
+
error: failureMessage
|
|
6053
7438
|
});
|
|
6054
7439
|
}
|
|
6055
7440
|
});
|
|
6056
7441
|
}
|
|
6057
7442
|
await queue.onIdle();
|
|
7443
|
+
if (scopedRoots) {
|
|
7444
|
+
this.saveScopedFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun), scopedRoots);
|
|
7445
|
+
} else {
|
|
7446
|
+
this.saveFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun));
|
|
7447
|
+
}
|
|
6058
7448
|
onProgress?.({
|
|
6059
7449
|
phase: "storing",
|
|
6060
7450
|
filesProcessed: files.length,
|
|
@@ -6062,18 +7452,48 @@ var Indexer = class {
|
|
|
6062
7452
|
chunksProcessed: stats.indexedChunks,
|
|
6063
7453
|
totalChunks: pendingChunks.length
|
|
6064
7454
|
});
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
7455
|
+
const branchChunkIds = Array.from(currentChunkIds).filter(
|
|
7456
|
+
(chunkId) => {
|
|
7457
|
+
const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId);
|
|
7458
|
+
const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId);
|
|
7459
|
+
return !isNewlyFailed && !isForcedFailed;
|
|
7460
|
+
}
|
|
7461
|
+
);
|
|
7462
|
+
database.clearBranch(branchCatalogKey);
|
|
7463
|
+
database.addChunksToBranchBatch(branchCatalogKey, branchChunkIds);
|
|
7464
|
+
database.clearBranchSymbols(branchCatalogKey);
|
|
7465
|
+
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
6069
7466
|
store.save();
|
|
6070
7467
|
invertedIndex.save();
|
|
6071
|
-
|
|
6072
|
-
|
|
7468
|
+
if (scopedRoots) {
|
|
7469
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7470
|
+
} else {
|
|
7471
|
+
this.fileHashCache = currentFileHashes;
|
|
7472
|
+
this.saveFileHashCache();
|
|
7473
|
+
}
|
|
6073
7474
|
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
6074
|
-
await this.maybeRunOrphanGc();
|
|
7475
|
+
const gcReset = await this.maybeRunOrphanGc();
|
|
7476
|
+
if (gcReset) {
|
|
7477
|
+
stats.durationMs = Date.now() - startTime;
|
|
7478
|
+
stats.warning = gcReset.warning;
|
|
7479
|
+
stats.resetCorruptedIndex = true;
|
|
7480
|
+
this.logger.recordIndexingEnd();
|
|
7481
|
+
this.logger.warn("Indexing ended after resetting corrupted local index during automatic GC", {
|
|
7482
|
+
files: stats.totalFiles,
|
|
7483
|
+
indexed: stats.indexedChunks,
|
|
7484
|
+
existing: stats.existingChunks,
|
|
7485
|
+
removed: stats.removedChunks,
|
|
7486
|
+
failed: stats.failedChunks,
|
|
7487
|
+
tokens: stats.tokensUsed,
|
|
7488
|
+
durationMs: stats.durationMs
|
|
7489
|
+
});
|
|
7490
|
+
return stats;
|
|
7491
|
+
}
|
|
6075
7492
|
}
|
|
6076
7493
|
stats.durationMs = Date.now() - startTime;
|
|
7494
|
+
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
7495
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
7496
|
+
}
|
|
6077
7497
|
this.saveIndexMetadata(configuredProviderInfo);
|
|
6078
7498
|
this.indexCompatibility = { compatible: true };
|
|
6079
7499
|
this.logger.recordIndexingEnd();
|
|
@@ -6202,27 +7622,30 @@ var Indexer = class {
|
|
|
6202
7622
|
const keywordResults = await this.keywordSearch(query, maxResults * 4);
|
|
6203
7623
|
const keywordMs = performance2.now() - keywordStartTime;
|
|
6204
7624
|
let branchChunkIds = null;
|
|
6205
|
-
if (filterByBranch && this.currentBranch !== "default") {
|
|
6206
|
-
branchChunkIds = new Set(
|
|
7625
|
+
if (filterByBranch && (this.config.scope === "global" || this.currentBranch !== "default")) {
|
|
7626
|
+
branchChunkIds = new Set(
|
|
7627
|
+
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
7628
|
+
);
|
|
6207
7629
|
}
|
|
6208
7630
|
const prefilterStartTime = performance2.now();
|
|
6209
|
-
const shouldPrefilterByBranch = branchChunkIds !== null && branchChunkIds.size > 0;
|
|
7631
|
+
const shouldPrefilterByBranch = branchChunkIds !== null && (this.config.scope === "global" || branchChunkIds.size > 0);
|
|
7632
|
+
const allowBranchPrefilterFallback = this.config.scope !== "global";
|
|
6210
7633
|
const prefilteredSemantic = shouldPrefilterByBranch && branchChunkIds ? semanticResults.filter((r) => branchChunkIds.has(r.id)) : semanticResults;
|
|
6211
7634
|
const prefilteredKeyword = shouldPrefilterByBranch && branchChunkIds ? keywordResults.filter((r) => branchChunkIds.has(r.id)) : keywordResults;
|
|
6212
|
-
const semanticCandidates = shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
6213
|
-
const keywordCandidates = shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0 ? keywordResults : prefilteredKeyword;
|
|
7635
|
+
const semanticCandidates = allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
7636
|
+
const keywordCandidates = allowBranchPrefilterFallback && shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0 ? keywordResults : prefilteredKeyword;
|
|
6214
7637
|
const prefilterMs = performance2.now() - prefilterStartTime;
|
|
6215
|
-
if (branchChunkIds && branchChunkIds.size === 0) {
|
|
7638
|
+
if (this.config.scope !== "global" && branchChunkIds && branchChunkIds.size === 0) {
|
|
6216
7639
|
this.logger.search("warn", "Branch prefilter skipped because branch catalog is empty", {
|
|
6217
7640
|
branch: this.currentBranch
|
|
6218
7641
|
});
|
|
6219
7642
|
}
|
|
6220
|
-
if (shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
7643
|
+
if (allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
6221
7644
|
this.logger.search("warn", "Branch prefilter produced no semantic overlap, using unfiltered semantic candidates", {
|
|
6222
7645
|
branch: this.currentBranch
|
|
6223
7646
|
});
|
|
6224
7647
|
}
|
|
6225
|
-
if (shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0) {
|
|
7648
|
+
if (allowBranchPrefilterFallback && shouldPrefilterByBranch && keywordResults.length > 0 && prefilteredKeyword.length === 0) {
|
|
6226
7649
|
this.logger.search("warn", "Branch prefilter produced no keyword overlap, using unfiltered keyword candidates", {
|
|
6227
7650
|
branch: this.currentBranch
|
|
6228
7651
|
});
|
|
@@ -6275,26 +7698,13 @@ var Indexer = class {
|
|
|
6275
7698
|
const primaryLane = mergeTieredResults(prePrimaryLane, symbolLane, maxResults * 4);
|
|
6276
7699
|
const tiered = mergeTieredResults(primaryLane, rescued, maxResults * 4);
|
|
6277
7700
|
const hasCodeHints = extractCodeTermHints(query).length > 0 || identifierHints.length > 0;
|
|
6278
|
-
const baseFiltered = tiered.filter((r) =>
|
|
6279
|
-
if (r.score < this.config.search.minScore) return false;
|
|
6280
|
-
if (options?.fileType) {
|
|
6281
|
-
const ext = r.metadata.filePath.split(".").pop()?.toLowerCase();
|
|
6282
|
-
if (ext !== options.fileType.toLowerCase().replace(/^\./, "")) return false;
|
|
6283
|
-
}
|
|
6284
|
-
if (options?.directory) {
|
|
6285
|
-
const normalizedDir = options.directory.replace(/^\/|\/$/g, "");
|
|
6286
|
-
if (!r.metadata.filePath.includes(`/${normalizedDir}/`) && !r.metadata.filePath.includes(`${normalizedDir}/`)) return false;
|
|
6287
|
-
}
|
|
6288
|
-
if (options?.chunkType) {
|
|
6289
|
-
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
6290
|
-
}
|
|
6291
|
-
return true;
|
|
6292
|
-
});
|
|
7701
|
+
const baseFiltered = tiered.filter((r) => matchesSearchFilters(r, options, this.config.search.minScore));
|
|
6293
7702
|
const implementationOnly = baseFiltered.filter(
|
|
6294
7703
|
(r) => isLikelyImplementationPath(r.metadata.filePath) && isImplementationChunkType(r.metadata.chunkType)
|
|
6295
7704
|
);
|
|
6296
7705
|
const filtered = (sourceIntent && hasCodeHints && implementationOnly.length > 0 ? implementationOnly : baseFiltered).slice(0, maxResults);
|
|
6297
|
-
const
|
|
7706
|
+
const identifierFallback = !options?.definitionIntent && filtered.length === 0 && identifierHints.length > 0 ? buildSymbolDefinitionLane(query, database, branchChunkIds, maxResults, union, true).filter((r) => matchesSearchFilters(r, options, this.config.search.minScore)).slice(0, maxResults) : [];
|
|
7707
|
+
const finalResults = filtered.length > 0 ? filtered : identifierFallback;
|
|
6298
7708
|
const totalSearchMs = performance2.now() - searchStartTime;
|
|
6299
7709
|
this.logger.recordSearch(totalSearchMs, {
|
|
6300
7710
|
embeddingMs,
|
|
@@ -6364,7 +7774,8 @@ var Indexer = class {
|
|
|
6364
7774
|
return results.slice(0, limit);
|
|
6365
7775
|
}
|
|
6366
7776
|
async getStatus() {
|
|
6367
|
-
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
7777
|
+
const { store, configuredProviderInfo, database } = await this.ensureInitialized();
|
|
7778
|
+
const failedBatchesCount = this.getFailedBatchesCount();
|
|
6368
7779
|
return {
|
|
6369
7780
|
indexed: store.count() > 0,
|
|
6370
7781
|
vectorCount: store.count(),
|
|
@@ -6373,23 +7784,86 @@ var Indexer = class {
|
|
|
6373
7784
|
indexPath: this.indexPath,
|
|
6374
7785
|
currentBranch: this.currentBranch,
|
|
6375
7786
|
baseBranch: this.baseBranch,
|
|
6376
|
-
compatibility: this.indexCompatibility
|
|
7787
|
+
compatibility: this.indexCompatibility,
|
|
7788
|
+
failedBatchesCount,
|
|
7789
|
+
failedBatchesPath: failedBatchesCount > 0 ? this.failedBatchesPath : void 0,
|
|
7790
|
+
warning: database.getMetadata(STARTUP_WARNING_METADATA_KEY) ?? void 0
|
|
6377
7791
|
};
|
|
6378
7792
|
}
|
|
6379
7793
|
async clearIndex() {
|
|
6380
7794
|
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
7795
|
+
if (this.config.scope === "global") {
|
|
7796
|
+
store.load();
|
|
7797
|
+
invertedIndex.load();
|
|
7798
|
+
this.loadFileHashCache();
|
|
7799
|
+
const roots = this.getScopedRoots();
|
|
7800
|
+
const compatibility = this.checkCompatibility();
|
|
7801
|
+
const allMetadata = store.getAllMetadata();
|
|
7802
|
+
const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || this.hasForeignScopedBranchData() || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots);
|
|
7803
|
+
if (!compatibility.compatible && hasForeignData) {
|
|
7804
|
+
if (compatibility.code === "EMBEDDING_STRATEGY_MISMATCH" /* EMBEDDING_STRATEGY_MISMATCH */) {
|
|
7805
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots);
|
|
7806
|
+
this.clearScopedFileHashCache(roots);
|
|
7807
|
+
this.clearScopedFailedBatches(roots);
|
|
7808
|
+
database.setMetadata(this.getProjectForceReembedMetadataKey(), "true");
|
|
7809
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
7810
|
+
this.indexCompatibility = { compatible: true };
|
|
7811
|
+
return;
|
|
7812
|
+
}
|
|
7813
|
+
throw new Error(
|
|
7814
|
+
`Global index compatibility reset is unsafe because the shared index contains files from other projects. The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.`
|
|
7815
|
+
);
|
|
7816
|
+
}
|
|
7817
|
+
if (!hasForeignData) {
|
|
7818
|
+
store.clear();
|
|
7819
|
+
store.save();
|
|
7820
|
+
invertedIndex.clear();
|
|
7821
|
+
invertedIndex.save();
|
|
7822
|
+
this.fileHashCache.clear();
|
|
7823
|
+
this.saveFileHashCache();
|
|
7824
|
+
database.clearAllIndexedData();
|
|
7825
|
+
this.saveFailedBatches([]);
|
|
7826
|
+
database.deleteMetadata("index.version");
|
|
7827
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
7828
|
+
database.deleteMetadata("index.embeddingModel");
|
|
7829
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
7830
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
7831
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
7832
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
7833
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
7834
|
+
database.deleteMetadata("index.createdAt");
|
|
7835
|
+
database.deleteMetadata("index.updatedAt");
|
|
7836
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
7837
|
+
return;
|
|
7838
|
+
}
|
|
7839
|
+
this.clearSharedIndexProjectData(store, invertedIndex, database, roots);
|
|
7840
|
+
this.clearScopedFileHashCache(roots);
|
|
7841
|
+
this.clearScopedFailedBatches(roots);
|
|
7842
|
+
this.indexCompatibility = compatibility;
|
|
7843
|
+
return;
|
|
7844
|
+
}
|
|
7845
|
+
const localProjectIndexPath = path8.join(this.projectRoot, ".opencode", "index");
|
|
7846
|
+
if (path8.resolve(this.indexPath) !== path8.resolve(localProjectIndexPath)) {
|
|
7847
|
+
throw new Error(
|
|
7848
|
+
"Project-scoped force rebuild is unsafe while using an inherited worktree index. Create a local project config boundary before clearing the index."
|
|
7849
|
+
);
|
|
7850
|
+
}
|
|
6381
7851
|
store.clear();
|
|
6382
7852
|
store.save();
|
|
6383
7853
|
invertedIndex.clear();
|
|
6384
7854
|
invertedIndex.save();
|
|
6385
7855
|
this.fileHashCache.clear();
|
|
6386
7856
|
this.saveFileHashCache();
|
|
6387
|
-
database.
|
|
6388
|
-
|
|
7857
|
+
database.clearAllIndexedData();
|
|
7858
|
+
this.saveFailedBatches([]);
|
|
6389
7859
|
database.deleteMetadata("index.version");
|
|
6390
7860
|
database.deleteMetadata("index.embeddingProvider");
|
|
6391
7861
|
database.deleteMetadata("index.embeddingModel");
|
|
6392
7862
|
database.deleteMetadata("index.embeddingDimensions");
|
|
7863
|
+
database.deleteMetadata("index.embeddingStrategyVersion");
|
|
7864
|
+
database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey());
|
|
7865
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
7866
|
+
database.deleteMetadata(this.getLegacyMigrationMetadataKey());
|
|
6393
7867
|
database.deleteMetadata("index.createdAt");
|
|
6394
7868
|
database.deleteMetadata("index.updatedAt");
|
|
6395
7869
|
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
@@ -6405,28 +7879,61 @@ var Indexer = class {
|
|
|
6405
7879
|
filePathsToChunkKeys.set(metadata.filePath, existing);
|
|
6406
7880
|
}
|
|
6407
7881
|
const removedFilePaths = [];
|
|
6408
|
-
|
|
7882
|
+
const removedChunkKeys = [];
|
|
7883
|
+
const chunkKeysByRemovedFile = /* @__PURE__ */ new Map();
|
|
6409
7884
|
for (const [filePath, chunkKeys] of filePathsToChunkKeys) {
|
|
6410
|
-
if (!
|
|
7885
|
+
if (!existsSync6(filePath)) {
|
|
7886
|
+
chunkKeysByRemovedFile.set(filePath, chunkKeys);
|
|
6411
7887
|
for (const key of chunkKeys) {
|
|
6412
|
-
|
|
6413
|
-
invertedIndex.removeChunk(key);
|
|
6414
|
-
removedCount++;
|
|
7888
|
+
removedChunkKeys.push(key);
|
|
6415
7889
|
}
|
|
6416
|
-
database.deleteChunksByFile(filePath);
|
|
6417
|
-
database.deleteCallEdgesByFile(filePath);
|
|
6418
|
-
database.deleteSymbolsByFile(filePath);
|
|
6419
7890
|
removedFilePaths.push(filePath);
|
|
6420
7891
|
}
|
|
6421
7892
|
}
|
|
7893
|
+
if (removedChunkKeys.length > 0) {
|
|
7894
|
+
this.rebuildVectorStoreExcludingChunkIds(store, database, removedChunkKeys);
|
|
7895
|
+
for (const key of removedChunkKeys) {
|
|
7896
|
+
invertedIndex.removeChunk(key);
|
|
7897
|
+
}
|
|
7898
|
+
}
|
|
7899
|
+
for (const filePath of removedFilePaths) {
|
|
7900
|
+
const fileChunkKeys = chunkKeysByRemovedFile.get(filePath) ?? [];
|
|
7901
|
+
if (fileChunkKeys.length > 0) {
|
|
7902
|
+
database.deleteChunksByIds(fileChunkKeys);
|
|
7903
|
+
}
|
|
7904
|
+
database.deleteCallEdgesByFile(filePath);
|
|
7905
|
+
database.deleteSymbolsByFile(filePath);
|
|
7906
|
+
}
|
|
7907
|
+
const removedCount = removedChunkKeys.length;
|
|
6422
7908
|
if (removedCount > 0) {
|
|
6423
7909
|
store.save();
|
|
6424
7910
|
invertedIndex.save();
|
|
6425
7911
|
}
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
7912
|
+
let gcOrphanEmbeddings;
|
|
7913
|
+
let gcOrphanChunks;
|
|
7914
|
+
let gcOrphanSymbols;
|
|
7915
|
+
let gcOrphanCallEdges;
|
|
7916
|
+
try {
|
|
7917
|
+
gcOrphanEmbeddings = database.gcOrphanEmbeddings();
|
|
7918
|
+
gcOrphanChunks = database.gcOrphanChunks();
|
|
7919
|
+
gcOrphanSymbols = database.gcOrphanSymbols();
|
|
7920
|
+
gcOrphanCallEdges = database.gcOrphanCallEdges();
|
|
7921
|
+
} catch (error) {
|
|
7922
|
+
if (!await this.tryResetCorruptedIndex("running index health check", error)) {
|
|
7923
|
+
throw error;
|
|
7924
|
+
}
|
|
7925
|
+
await this.ensureInitialized();
|
|
7926
|
+
return {
|
|
7927
|
+
removed: 0,
|
|
7928
|
+
filePaths: [],
|
|
7929
|
+
gcOrphanEmbeddings: 0,
|
|
7930
|
+
gcOrphanChunks: 0,
|
|
7931
|
+
gcOrphanSymbols: 0,
|
|
7932
|
+
gcOrphanCallEdges: 0,
|
|
7933
|
+
resetCorruptedIndex: true,
|
|
7934
|
+
warning: this.getCorruptedIndexWarning(path8.join(this.indexPath, "codebase.db"))
|
|
7935
|
+
};
|
|
7936
|
+
}
|
|
6430
7937
|
this.logger.recordGc(removedCount, gcOrphanChunks, gcOrphanEmbeddings);
|
|
6431
7938
|
this.logger.gc("info", "Health check complete", {
|
|
6432
7939
|
removedStale: removedCount,
|
|
@@ -6437,8 +7944,12 @@ var Indexer = class {
|
|
|
6437
7944
|
return { removed: removedCount, filePaths: removedFilePaths, gcOrphanEmbeddings, gcOrphanChunks, gcOrphanSymbols, gcOrphanCallEdges };
|
|
6438
7945
|
}
|
|
6439
7946
|
async retryFailedBatches() {
|
|
6440
|
-
const { store, provider, invertedIndex } = await this.ensureInitialized();
|
|
6441
|
-
const
|
|
7947
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
7948
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
7949
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
7950
|
+
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
7951
|
+
const { scoped: scopedFailedBatches, retained: retainedFailedBatches } = roots ? this.partitionFailedBatches(roots, maxChunkTokens) : { scoped: this.loadFailedBatches(maxChunkTokens), retained: [] };
|
|
7952
|
+
const failedBatches = scopedFailedBatches;
|
|
6442
7953
|
if (failedBatches.length === 0) {
|
|
6443
7954
|
return { succeeded: 0, failed: 0, remaining: 0 };
|
|
6444
7955
|
}
|
|
@@ -6446,49 +7957,170 @@ var Indexer = class {
|
|
|
6446
7957
|
let failed = 0;
|
|
6447
7958
|
const stillFailing = [];
|
|
6448
7959
|
for (const batch of failedBatches) {
|
|
7960
|
+
const batchChunksById = new Map(batch.chunks.map((chunk) => [chunk.id, chunk]));
|
|
7961
|
+
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
7962
|
+
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
7963
|
+
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
7964
|
+
const failedChunksForBatch = /* @__PURE__ */ new Map();
|
|
7965
|
+
const pooledResults = [];
|
|
6449
7966
|
try {
|
|
6450
|
-
const
|
|
6451
|
-
|
|
6452
|
-
|
|
6453
|
-
return provider.embedBatch(texts);
|
|
6454
|
-
},
|
|
6455
|
-
{
|
|
6456
|
-
retries: this.config.indexing.retries,
|
|
6457
|
-
minTimeout: this.config.indexing.retryDelayMs
|
|
6458
|
-
}
|
|
7967
|
+
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
7968
|
+
batch.chunks,
|
|
7969
|
+
getDynamicBatchOptions(configuredProviderInfo)
|
|
6459
7970
|
);
|
|
6460
|
-
const
|
|
7971
|
+
for (const requestBatch of requestBatches) {
|
|
7972
|
+
try {
|
|
7973
|
+
const result = await pRetry(
|
|
7974
|
+
async () => {
|
|
7975
|
+
const texts = requestBatch.map((request) => request.text);
|
|
7976
|
+
return provider.embedBatch(texts);
|
|
7977
|
+
},
|
|
7978
|
+
{
|
|
7979
|
+
retries: this.config.indexing.retries,
|
|
7980
|
+
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
7981
|
+
maxTimeout: providerRateLimits.maxRetryMs,
|
|
7982
|
+
factor: 2,
|
|
7983
|
+
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError)
|
|
7984
|
+
}
|
|
7985
|
+
);
|
|
7986
|
+
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
7987
|
+
requestBatch.forEach((request, idx) => {
|
|
7988
|
+
if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
7989
|
+
return;
|
|
7990
|
+
}
|
|
7991
|
+
const vector = result.embeddings[idx];
|
|
7992
|
+
if (!vector) {
|
|
7993
|
+
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
7994
|
+
}
|
|
7995
|
+
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
7996
|
+
parts[request.partIndex] = {
|
|
7997
|
+
vector,
|
|
7998
|
+
tokenCount: request.tokenCount
|
|
7999
|
+
};
|
|
8000
|
+
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
8001
|
+
touchedChunkIds.add(request.chunk.id);
|
|
8002
|
+
});
|
|
8003
|
+
for (const chunkId of touchedChunkIds) {
|
|
8004
|
+
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
8005
|
+
continue;
|
|
8006
|
+
}
|
|
8007
|
+
const chunk = batchChunksById.get(chunkId);
|
|
8008
|
+
if (!chunk) {
|
|
8009
|
+
continue;
|
|
8010
|
+
}
|
|
8011
|
+
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
8012
|
+
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
8013
|
+
continue;
|
|
8014
|
+
}
|
|
8015
|
+
const orderedParts = parts;
|
|
8016
|
+
pooledResults.push({
|
|
8017
|
+
chunk,
|
|
8018
|
+
vector: poolEmbeddingVectors(
|
|
8019
|
+
orderedParts.map((part) => part.vector),
|
|
8020
|
+
orderedParts.map((part) => part.tokenCount)
|
|
8021
|
+
)
|
|
8022
|
+
});
|
|
8023
|
+
}
|
|
8024
|
+
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
8025
|
+
} catch (error) {
|
|
8026
|
+
const failureMessage = String(error);
|
|
8027
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
8028
|
+
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id) && !failedChunkIds.has(chunk.id));
|
|
8029
|
+
for (const chunk of failedChunks) {
|
|
8030
|
+
failedChunkIds.add(chunk.id);
|
|
8031
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
8032
|
+
failedChunksForBatch.set(chunk.id, {
|
|
8033
|
+
chunks: [chunk],
|
|
8034
|
+
attemptCount: batch.attemptCount + 1,
|
|
8035
|
+
lastAttempt: failureTimestamp,
|
|
8036
|
+
error: failureMessage
|
|
8037
|
+
});
|
|
8038
|
+
}
|
|
8039
|
+
failed += failedChunks.length;
|
|
8040
|
+
this.logger.recordEmbeddingError();
|
|
8041
|
+
}
|
|
8042
|
+
}
|
|
8043
|
+
const successfulResults = pooledResults.filter(({ chunk }) => !failedChunkIds.has(chunk.id));
|
|
8044
|
+
const items = successfulResults.map(({ chunk, vector }) => ({
|
|
6461
8045
|
id: chunk.id,
|
|
6462
|
-
vector
|
|
8046
|
+
vector,
|
|
6463
8047
|
metadata: chunk.metadata
|
|
6464
8048
|
}));
|
|
6465
|
-
|
|
6466
|
-
|
|
8049
|
+
if (items.length > 0) {
|
|
8050
|
+
store.addBatch(items);
|
|
8051
|
+
}
|
|
8052
|
+
if (successfulResults.length > 0) {
|
|
8053
|
+
try {
|
|
8054
|
+
database.upsertEmbeddingsBatch(
|
|
8055
|
+
successfulResults.map(({ chunk, vector }) => ({
|
|
8056
|
+
contentHash: chunk.contentHash,
|
|
8057
|
+
embedding: float32ArrayToBuffer(vector),
|
|
8058
|
+
chunkText: chunk.storageText,
|
|
8059
|
+
model: configuredProviderInfo.modelInfo.model
|
|
8060
|
+
}))
|
|
8061
|
+
);
|
|
8062
|
+
} catch (dbError) {
|
|
8063
|
+
this.rebuildVectorStoreExcludingChunkIds(
|
|
8064
|
+
store,
|
|
8065
|
+
database,
|
|
8066
|
+
successfulResults.map(({ chunk }) => chunk.id)
|
|
8067
|
+
);
|
|
8068
|
+
throw dbError;
|
|
8069
|
+
}
|
|
8070
|
+
}
|
|
8071
|
+
for (const { chunk } of successfulResults) {
|
|
6467
8072
|
invertedIndex.removeChunk(chunk.id);
|
|
6468
8073
|
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
8074
|
+
completedChunkIds.add(chunk.id);
|
|
8075
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
6469
8076
|
}
|
|
6470
|
-
|
|
6471
|
-
|
|
6472
|
-
|
|
8077
|
+
database.addChunksToBranchBatch(
|
|
8078
|
+
this.getBranchCatalogKey(),
|
|
8079
|
+
successfulResults.map(({ chunk }) => chunk.id)
|
|
8080
|
+
);
|
|
8081
|
+
this.logger.recordChunksEmbedded(successfulResults.length);
|
|
8082
|
+
succeeded += successfulResults.length;
|
|
8083
|
+
stillFailing.push(...failedChunksForBatch.values());
|
|
6473
8084
|
} catch (error) {
|
|
6474
|
-
|
|
8085
|
+
const failureMessage = getErrorMessage(error);
|
|
8086
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
8087
|
+
const unaccountedChunks = batch.chunks.filter(
|
|
8088
|
+
(chunk) => !failedChunksForBatch.has(chunk.id) && !completedChunkIds.has(chunk.id)
|
|
8089
|
+
);
|
|
8090
|
+
for (const chunk of unaccountedChunks) {
|
|
8091
|
+
failedChunksForBatch.set(chunk.id, {
|
|
8092
|
+
chunks: [chunk],
|
|
8093
|
+
attemptCount: batch.attemptCount + 1,
|
|
8094
|
+
lastAttempt: failureTimestamp,
|
|
8095
|
+
error: failureMessage
|
|
8096
|
+
});
|
|
8097
|
+
}
|
|
8098
|
+
failed += unaccountedChunks.length;
|
|
6475
8099
|
this.logger.recordEmbeddingError();
|
|
6476
|
-
stillFailing.push(
|
|
6477
|
-
...batch,
|
|
6478
|
-
attemptCount: batch.attemptCount + 1,
|
|
6479
|
-
lastAttempt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6480
|
-
error: String(error)
|
|
6481
|
-
});
|
|
8100
|
+
stillFailing.push(...coalesceFailedBatches(Array.from(failedChunksForBatch.values())));
|
|
6482
8101
|
}
|
|
6483
8102
|
}
|
|
6484
|
-
|
|
8103
|
+
const persistedStillFailing = coalesceFailedBatches(stillFailing);
|
|
8104
|
+
if (roots) {
|
|
8105
|
+
this.saveFailedBatches([...retainedFailedBatches, ...persistedStillFailing]);
|
|
8106
|
+
} else {
|
|
8107
|
+
this.saveFailedBatches(persistedStillFailing);
|
|
8108
|
+
}
|
|
6485
8109
|
if (succeeded > 0) {
|
|
6486
8110
|
store.save();
|
|
6487
8111
|
invertedIndex.save();
|
|
6488
8112
|
}
|
|
6489
|
-
|
|
8113
|
+
if (roots && succeeded > 0 && persistedStillFailing.length === 0 && this.hasProjectForceReembedPending()) {
|
|
8114
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
8115
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
8116
|
+
this.indexCompatibility = { compatible: true };
|
|
8117
|
+
}
|
|
8118
|
+
return { succeeded, failed, remaining: persistedStillFailing.length };
|
|
6490
8119
|
}
|
|
6491
8120
|
getFailedBatchesCount() {
|
|
8121
|
+
if (this.config.scope === "global") {
|
|
8122
|
+
return this.partitionFailedBatches(this.getScopedRoots()).scoped.length;
|
|
8123
|
+
}
|
|
6492
8124
|
return this.loadFailedBatches().length;
|
|
6493
8125
|
}
|
|
6494
8126
|
getCurrentBranch() {
|
|
@@ -6537,20 +8169,23 @@ var Indexer = class {
|
|
|
6537
8169
|
const semanticResults = store.search(embedding, limit * 2);
|
|
6538
8170
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
6539
8171
|
let branchChunkIds = null;
|
|
6540
|
-
if (filterByBranch && this.currentBranch !== "default") {
|
|
6541
|
-
branchChunkIds = new Set(
|
|
8172
|
+
if (filterByBranch && (this.config.scope === "global" || this.currentBranch !== "default")) {
|
|
8173
|
+
branchChunkIds = new Set(
|
|
8174
|
+
this.getBranchCatalogKeys().flatMap((branchKey) => database.getBranchChunkIds(branchKey))
|
|
8175
|
+
);
|
|
6542
8176
|
}
|
|
6543
8177
|
const prefilterStartTime = performance2.now();
|
|
6544
|
-
const shouldPrefilterByBranch = branchChunkIds !== null && branchChunkIds.size > 0;
|
|
8178
|
+
const shouldPrefilterByBranch = branchChunkIds !== null && (this.config.scope === "global" || branchChunkIds.size > 0);
|
|
8179
|
+
const allowBranchPrefilterFallback = this.config.scope !== "global";
|
|
6545
8180
|
const prefilteredSemantic = shouldPrefilterByBranch && branchChunkIds ? semanticResults.filter((r) => branchChunkIds.has(r.id)) : semanticResults;
|
|
6546
|
-
const semanticCandidates = shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
8181
|
+
const semanticCandidates = allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0 ? semanticResults : prefilteredSemantic;
|
|
6547
8182
|
const prefilterMs = performance2.now() - prefilterStartTime;
|
|
6548
|
-
if (branchChunkIds && branchChunkIds.size === 0) {
|
|
8183
|
+
if (this.config.scope !== "global" && branchChunkIds && branchChunkIds.size === 0) {
|
|
6549
8184
|
this.logger.search("warn", "Branch prefilter skipped because branch catalog is empty", {
|
|
6550
8185
|
branch: this.currentBranch
|
|
6551
8186
|
});
|
|
6552
8187
|
}
|
|
6553
|
-
if (shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
8188
|
+
if (allowBranchPrefilterFallback && shouldPrefilterByBranch && semanticResults.length > 0 && prefilteredSemantic.length === 0) {
|
|
6554
8189
|
this.logger.search("warn", "Branch prefilter produced no semantic overlap, using unfiltered semantic candidates", {
|
|
6555
8190
|
branch: this.currentBranch
|
|
6556
8191
|
});
|
|
@@ -6623,11 +8258,39 @@ var Indexer = class {
|
|
|
6623
8258
|
}
|
|
6624
8259
|
async getCallers(targetName) {
|
|
6625
8260
|
const { database } = await this.ensureInitialized();
|
|
6626
|
-
|
|
8261
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8262
|
+
const results = [];
|
|
8263
|
+
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
8264
|
+
for (const edge of database.getCallersWithContext(targetName, branchKey)) {
|
|
8265
|
+
if (!seen.has(edge.id)) {
|
|
8266
|
+
seen.add(edge.id);
|
|
8267
|
+
results.push(edge);
|
|
8268
|
+
}
|
|
8269
|
+
}
|
|
8270
|
+
}
|
|
8271
|
+
return results;
|
|
6627
8272
|
}
|
|
6628
8273
|
async getCallees(symbolId) {
|
|
6629
8274
|
const { database } = await this.ensureInitialized();
|
|
6630
|
-
|
|
8275
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8276
|
+
const results = [];
|
|
8277
|
+
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
8278
|
+
for (const edge of database.getCallees(symbolId, branchKey)) {
|
|
8279
|
+
if (!seen.has(edge.id)) {
|
|
8280
|
+
seen.add(edge.id);
|
|
8281
|
+
results.push(edge);
|
|
8282
|
+
}
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8285
|
+
return results;
|
|
8286
|
+
}
|
|
8287
|
+
async close() {
|
|
8288
|
+
await this.database?.close();
|
|
8289
|
+
this.database = null;
|
|
8290
|
+
this.store = null;
|
|
8291
|
+
this.invertedIndex = null;
|
|
8292
|
+
this.provider = null;
|
|
8293
|
+
this.reranker = null;
|
|
6631
8294
|
}
|
|
6632
8295
|
};
|
|
6633
8296
|
|
|
@@ -6878,9 +8541,9 @@ function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingToke
|
|
|
6878
8541
|
}
|
|
6879
8542
|
|
|
6880
8543
|
// src/eval/schema.ts
|
|
6881
|
-
import { readFileSync as
|
|
8544
|
+
import { readFileSync as readFileSync7 } from "fs";
|
|
6882
8545
|
function parseJsonFile(filePath) {
|
|
6883
|
-
const content =
|
|
8546
|
+
const content = readFileSync7(filePath, "utf-8");
|
|
6884
8547
|
return JSON.parse(content);
|
|
6885
8548
|
}
|
|
6886
8549
|
function isRecord(value) {
|
|
@@ -6889,23 +8552,23 @@ function isRecord(value) {
|
|
|
6889
8552
|
function isStringArray2(value) {
|
|
6890
8553
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
6891
8554
|
}
|
|
6892
|
-
function asPositiveNumber(value,
|
|
8555
|
+
function asPositiveNumber(value, path13) {
|
|
6893
8556
|
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
6894
|
-
throw new Error(`${
|
|
8557
|
+
throw new Error(`${path13} must be a non-negative number`);
|
|
6895
8558
|
}
|
|
6896
8559
|
return value;
|
|
6897
8560
|
}
|
|
6898
|
-
function parseQueryType(value,
|
|
8561
|
+
function parseQueryType(value, path13) {
|
|
6899
8562
|
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy") {
|
|
6900
8563
|
return value;
|
|
6901
8564
|
}
|
|
6902
8565
|
throw new Error(
|
|
6903
|
-
`${
|
|
8566
|
+
`${path13} must be one of: definition, implementation-intent, similarity, keyword-heavy`
|
|
6904
8567
|
);
|
|
6905
8568
|
}
|
|
6906
|
-
function parseExpected(input,
|
|
8569
|
+
function parseExpected(input, path13) {
|
|
6907
8570
|
if (!isRecord(input)) {
|
|
6908
|
-
throw new Error(`${
|
|
8571
|
+
throw new Error(`${path13} must be an object`);
|
|
6909
8572
|
}
|
|
6910
8573
|
const filePathRaw = input.filePath;
|
|
6911
8574
|
const acceptableFilesRaw = input.acceptableFiles;
|
|
@@ -6914,16 +8577,16 @@ function parseExpected(input, path11) {
|
|
|
6914
8577
|
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
6915
8578
|
const acceptableFiles = isStringArray2(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
6916
8579
|
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
6917
|
-
throw new Error(`${
|
|
8580
|
+
throw new Error(`${path13} must include either expected.filePath or expected.acceptableFiles`);
|
|
6918
8581
|
}
|
|
6919
8582
|
if (acceptableFilesRaw !== void 0 && !isStringArray2(acceptableFilesRaw)) {
|
|
6920
|
-
throw new Error(`${
|
|
8583
|
+
throw new Error(`${path13}.acceptableFiles must be an array of strings`);
|
|
6921
8584
|
}
|
|
6922
8585
|
if (symbolRaw !== void 0 && typeof symbolRaw !== "string") {
|
|
6923
|
-
throw new Error(`${
|
|
8586
|
+
throw new Error(`${path13}.symbol must be a string when provided`);
|
|
6924
8587
|
}
|
|
6925
8588
|
if (branchRaw !== void 0 && typeof branchRaw !== "string") {
|
|
6926
|
-
throw new Error(`${
|
|
8589
|
+
throw new Error(`${path13}.branch must be a string when provided`);
|
|
6927
8590
|
}
|
|
6928
8591
|
return {
|
|
6929
8592
|
filePath,
|
|
@@ -6933,25 +8596,25 @@ function parseExpected(input, path11) {
|
|
|
6933
8596
|
};
|
|
6934
8597
|
}
|
|
6935
8598
|
function parseQuery(input, index) {
|
|
6936
|
-
const
|
|
8599
|
+
const path13 = `queries[${index}]`;
|
|
6937
8600
|
if (!isRecord(input)) {
|
|
6938
|
-
throw new Error(`${
|
|
8601
|
+
throw new Error(`${path13} must be an object`);
|
|
6939
8602
|
}
|
|
6940
8603
|
const id = input.id;
|
|
6941
8604
|
const query = input.query;
|
|
6942
8605
|
const queryType = input.queryType;
|
|
6943
8606
|
const expected = input.expected;
|
|
6944
8607
|
if (typeof id !== "string" || id.trim().length === 0) {
|
|
6945
|
-
throw new Error(`${
|
|
8608
|
+
throw new Error(`${path13}.id must be a non-empty string`);
|
|
6946
8609
|
}
|
|
6947
8610
|
if (typeof query !== "string" || query.trim().length === 0) {
|
|
6948
|
-
throw new Error(`${
|
|
8611
|
+
throw new Error(`${path13}.query must be a non-empty string`);
|
|
6949
8612
|
}
|
|
6950
8613
|
return {
|
|
6951
8614
|
id,
|
|
6952
8615
|
query,
|
|
6953
|
-
queryType: parseQueryType(queryType, `${
|
|
6954
|
-
expected: parseExpected(expected, `${
|
|
8616
|
+
queryType: parseQueryType(queryType, `${path13}.queryType`),
|
|
8617
|
+
expected: parseExpected(expected, `${path13}.expected`)
|
|
6955
8618
|
};
|
|
6956
8619
|
}
|
|
6957
8620
|
function parseGoldenDataset(raw, sourceLabel) {
|
|
@@ -7048,35 +8711,83 @@ function loadBudget(budgetPath) {
|
|
|
7048
8711
|
|
|
7049
8712
|
// src/eval/runner.ts
|
|
7050
8713
|
function toAbsolute(projectRoot, maybeRelative) {
|
|
7051
|
-
return
|
|
8714
|
+
return path9.isAbsolute(maybeRelative) ? maybeRelative : path9.join(projectRoot, maybeRelative);
|
|
8715
|
+
}
|
|
8716
|
+
function isProjectScopedConfigPath(configPath) {
|
|
8717
|
+
return path9.basename(configPath) === "codebase-index.json" && path9.basename(path9.dirname(configPath)) === ".opencode";
|
|
8718
|
+
}
|
|
8719
|
+
function normalizeEvalConfigKnowledgeBases(rawConfig, projectRoot, resolvedConfigPath) {
|
|
8720
|
+
const config = rawConfig && typeof rawConfig === "object" ? { ...rawConfig } : {};
|
|
8721
|
+
if (!Array.isArray(config.knowledgeBases)) {
|
|
8722
|
+
return config;
|
|
8723
|
+
}
|
|
8724
|
+
config.knowledgeBases = isProjectScopedConfigPath(resolvedConfigPath) ? resolveInheritedKnowledgeBaseEntries(
|
|
8725
|
+
config.knowledgeBases,
|
|
8726
|
+
path9.dirname(path9.dirname(resolvedConfigPath)),
|
|
8727
|
+
projectRoot
|
|
8728
|
+
) : rebasePathEntries(
|
|
8729
|
+
config.knowledgeBases,
|
|
8730
|
+
path9.dirname(resolvedConfigPath),
|
|
8731
|
+
projectRoot
|
|
8732
|
+
);
|
|
8733
|
+
return config;
|
|
7052
8734
|
}
|
|
7053
8735
|
function loadRawConfig(projectRoot, configPath) {
|
|
7054
8736
|
const fromPath = configPath ? toAbsolute(projectRoot, configPath) : null;
|
|
7055
|
-
if (fromPath &&
|
|
7056
|
-
return
|
|
8737
|
+
if (fromPath && existsSync7(fromPath)) {
|
|
8738
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
8739
|
+
JSON.parse(readFileSync8(fromPath, "utf-8")),
|
|
8740
|
+
projectRoot,
|
|
8741
|
+
fromPath
|
|
8742
|
+
);
|
|
7057
8743
|
}
|
|
7058
|
-
const projectConfig =
|
|
7059
|
-
if (
|
|
7060
|
-
return
|
|
8744
|
+
const projectConfig = resolveProjectConfigPath(projectRoot);
|
|
8745
|
+
if (existsSync7(projectConfig)) {
|
|
8746
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
8747
|
+
JSON.parse(readFileSync8(projectConfig, "utf-8")),
|
|
8748
|
+
projectRoot,
|
|
8749
|
+
projectConfig
|
|
8750
|
+
);
|
|
7061
8751
|
}
|
|
7062
|
-
const globalConfig =
|
|
7063
|
-
if (
|
|
7064
|
-
return JSON.parse(
|
|
8752
|
+
const globalConfig = path9.join(os5.homedir(), ".config", "opencode", "codebase-index.json");
|
|
8753
|
+
if (existsSync7(globalConfig)) {
|
|
8754
|
+
return JSON.parse(readFileSync8(globalConfig, "utf-8"));
|
|
7065
8755
|
}
|
|
7066
8756
|
return {};
|
|
7067
8757
|
}
|
|
7068
8758
|
function getIndexRootPath(projectRoot, scope) {
|
|
7069
|
-
|
|
7070
|
-
|
|
7071
|
-
|
|
7072
|
-
return
|
|
8759
|
+
return scope === "global" ? getGlobalIndexPath() : resolveProjectIndexPath(projectRoot, scope);
|
|
8760
|
+
}
|
|
8761
|
+
function getLocalProjectIndexRoot(projectRoot) {
|
|
8762
|
+
return path9.join(projectRoot, ".opencode", "index");
|
|
8763
|
+
}
|
|
8764
|
+
function getLocalProjectConfigPath(projectRoot) {
|
|
8765
|
+
return path9.join(projectRoot, ".opencode", "codebase-index.json");
|
|
7073
8766
|
}
|
|
7074
8767
|
function clearIndexRoot(projectRoot, scope) {
|
|
7075
|
-
const indexRoot = getIndexRootPath(projectRoot, scope);
|
|
7076
|
-
if (
|
|
8768
|
+
const indexRoot = scope === "global" ? getIndexRootPath(projectRoot, scope) : getLocalProjectIndexRoot(projectRoot);
|
|
8769
|
+
if (existsSync7(indexRoot)) {
|
|
7077
8770
|
rmSync(indexRoot, { recursive: true, force: true });
|
|
7078
8771
|
}
|
|
7079
8772
|
}
|
|
8773
|
+
function ensureLocalEvalProjectConfig(projectRoot, configPath) {
|
|
8774
|
+
const localConfigPath = getLocalProjectConfigPath(projectRoot);
|
|
8775
|
+
const resolvedConfigPath = configPath ? toAbsolute(projectRoot, configPath) : resolveProjectConfigPath(projectRoot);
|
|
8776
|
+
if (!configPath && existsSync7(localConfigPath)) {
|
|
8777
|
+
return localConfigPath;
|
|
8778
|
+
}
|
|
8779
|
+
if (!existsSync7(resolvedConfigPath) || resolvedConfigPath === localConfigPath) {
|
|
8780
|
+
return resolvedConfigPath;
|
|
8781
|
+
}
|
|
8782
|
+
const sourceConfig = normalizeEvalConfigKnowledgeBases(
|
|
8783
|
+
JSON.parse(readFileSync8(resolvedConfigPath, "utf-8")),
|
|
8784
|
+
projectRoot,
|
|
8785
|
+
resolvedConfigPath
|
|
8786
|
+
);
|
|
8787
|
+
mkdirSync3(path9.dirname(localConfigPath), { recursive: true });
|
|
8788
|
+
writeFileSync4(localConfigPath, JSON.stringify(sourceConfig, null, 2), "utf-8");
|
|
8789
|
+
return localConfigPath;
|
|
8790
|
+
}
|
|
7080
8791
|
function loadParsedConfig(projectRoot, configPath) {
|
|
7081
8792
|
const raw = loadRawConfig(projectRoot, configPath);
|
|
7082
8793
|
return parseConfig(raw);
|
|
@@ -7107,94 +8818,99 @@ async function runEvaluation(options) {
|
|
|
7107
8818
|
const againstPath = options.againstPath ? toAbsolute(options.projectRoot, options.againstPath) : void 0;
|
|
7108
8819
|
const budgetPath = options.budgetPath ? toAbsolute(options.projectRoot, options.budgetPath) : void 0;
|
|
7109
8820
|
const dataset = loadGoldenDataset(datasetPath);
|
|
7110
|
-
const
|
|
8821
|
+
const resolvedEvalConfigPath = options.reindex ? ensureLocalEvalProjectConfig(options.projectRoot, options.configPath) : options.configPath;
|
|
8822
|
+
const parsedConfig = loadParsedConfig(options.projectRoot, resolvedEvalConfigPath);
|
|
7111
8823
|
const effectiveConfig = resolveSearchConfig(parsedConfig, options.searchOverrides);
|
|
7112
8824
|
if (options.reindex) {
|
|
7113
8825
|
clearIndexRoot(options.projectRoot, effectiveConfig.scope);
|
|
7114
8826
|
}
|
|
7115
8827
|
const indexer = new Indexer(options.projectRoot, effectiveConfig);
|
|
7116
|
-
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
|
|
7120
|
-
|
|
7121
|
-
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
7122
|
-
);
|
|
7123
|
-
}
|
|
7124
|
-
const start = performance3.now();
|
|
7125
|
-
const result = await indexer.search(query.query, 10, {
|
|
7126
|
-
metadataOnly: true,
|
|
7127
|
-
filterByBranch: query.expected.branch ? true : false
|
|
7128
|
-
});
|
|
7129
|
-
const elapsed = performance3.now() - start;
|
|
7130
|
-
const materialized = result.map((item) => ({
|
|
7131
|
-
filePath: item.filePath,
|
|
7132
|
-
startLine: item.startLine,
|
|
7133
|
-
endLine: item.endLine,
|
|
7134
|
-
score: item.score,
|
|
7135
|
-
chunkType: item.chunkType,
|
|
7136
|
-
name: item.name
|
|
7137
|
-
}));
|
|
7138
|
-
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
7139
|
-
}
|
|
7140
|
-
const logger = indexer.getLogger();
|
|
7141
|
-
const metricSnapshot = logger.getMetrics();
|
|
7142
|
-
const costPer1MTokensUsd = effectiveConfig.embeddingProvider === "custom" || effectiveConfig.embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(effectiveConfig.embeddingProvider).costPer1MTokens;
|
|
7143
|
-
const summary = {
|
|
7144
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7145
|
-
projectRoot: options.projectRoot,
|
|
7146
|
-
datasetPath,
|
|
7147
|
-
datasetName: dataset.name,
|
|
7148
|
-
datasetVersion: dataset.version,
|
|
7149
|
-
queryCount: dataset.queries.length,
|
|
7150
|
-
topK: 10,
|
|
7151
|
-
searchConfig: {
|
|
7152
|
-
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
7153
|
-
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
7154
|
-
rrfK: effectiveConfig.search.rrfK,
|
|
7155
|
-
rerankTopN: effectiveConfig.search.rerankTopN
|
|
7156
|
-
},
|
|
7157
|
-
metrics: computeEvalMetrics(
|
|
7158
|
-
dataset.queries,
|
|
7159
|
-
perQuery,
|
|
7160
|
-
metricSnapshot.embeddingApiCalls,
|
|
7161
|
-
metricSnapshot.embeddingTokensUsed,
|
|
7162
|
-
costPer1MTokensUsd
|
|
7163
|
-
)
|
|
7164
|
-
};
|
|
7165
|
-
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
7166
|
-
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
7167
|
-
writeJson(path7.join(outputDir, "summary.json"), summary);
|
|
7168
|
-
writeJson(path7.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
7169
|
-
let comparison;
|
|
7170
|
-
if (againstPath) {
|
|
7171
|
-
const baseline = loadSummary(againstPath);
|
|
7172
|
-
comparison = compareSummaries(summary, baseline, againstPath);
|
|
7173
|
-
writeJson(path7.join(outputDir, "compare.json"), comparison);
|
|
7174
|
-
}
|
|
7175
|
-
let gate;
|
|
7176
|
-
if (options.ciMode) {
|
|
7177
|
-
if (!budgetPath) {
|
|
7178
|
-
throw new Error("CI mode requires --budget path");
|
|
7179
|
-
}
|
|
7180
|
-
const budget = loadBudget(budgetPath);
|
|
7181
|
-
if (!comparison && budget.baselinePath) {
|
|
7182
|
-
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
7183
|
-
if (existsSync5(resolvedBaseline)) {
|
|
7184
|
-
const baselineSummary = loadSummary(resolvedBaseline);
|
|
7185
|
-
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
7186
|
-
writeJson(path7.join(outputDir, "compare.json"), comparison);
|
|
7187
|
-
} else if (budget.failOnMissingBaseline) {
|
|
8828
|
+
try {
|
|
8829
|
+
await indexer.index();
|
|
8830
|
+
const perQuery = [];
|
|
8831
|
+
for (const query of dataset.queries) {
|
|
8832
|
+
if (query.expected.branch && query.expected.branch !== indexer.getCurrentBranch()) {
|
|
7188
8833
|
throw new Error(
|
|
7189
|
-
`
|
|
8834
|
+
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
7190
8835
|
);
|
|
7191
8836
|
}
|
|
8837
|
+
const start = performance3.now();
|
|
8838
|
+
const result = await indexer.search(query.query, 10, {
|
|
8839
|
+
metadataOnly: true,
|
|
8840
|
+
filterByBranch: query.expected.branch ? true : false
|
|
8841
|
+
});
|
|
8842
|
+
const elapsed = performance3.now() - start;
|
|
8843
|
+
const materialized = result.map((item) => ({
|
|
8844
|
+
filePath: item.filePath,
|
|
8845
|
+
startLine: item.startLine,
|
|
8846
|
+
endLine: item.endLine,
|
|
8847
|
+
score: item.score,
|
|
8848
|
+
chunkType: item.chunkType,
|
|
8849
|
+
name: item.name
|
|
8850
|
+
}));
|
|
8851
|
+
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
8852
|
+
}
|
|
8853
|
+
const logger = indexer.getLogger();
|
|
8854
|
+
const metricSnapshot = logger.getMetrics();
|
|
8855
|
+
const costPer1MTokensUsd = effectiveConfig.embeddingProvider === "custom" || effectiveConfig.embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(effectiveConfig.embeddingProvider).costPer1MTokens;
|
|
8856
|
+
const summary = {
|
|
8857
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8858
|
+
projectRoot: options.projectRoot,
|
|
8859
|
+
datasetPath,
|
|
8860
|
+
datasetName: dataset.name,
|
|
8861
|
+
datasetVersion: dataset.version,
|
|
8862
|
+
queryCount: dataset.queries.length,
|
|
8863
|
+
topK: 10,
|
|
8864
|
+
searchConfig: {
|
|
8865
|
+
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
8866
|
+
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
8867
|
+
rrfK: effectiveConfig.search.rrfK,
|
|
8868
|
+
rerankTopN: effectiveConfig.search.rerankTopN
|
|
8869
|
+
},
|
|
8870
|
+
metrics: computeEvalMetrics(
|
|
8871
|
+
dataset.queries,
|
|
8872
|
+
perQuery,
|
|
8873
|
+
metricSnapshot.embeddingApiCalls,
|
|
8874
|
+
metricSnapshot.embeddingTokensUsed,
|
|
8875
|
+
costPer1MTokensUsd
|
|
8876
|
+
)
|
|
8877
|
+
};
|
|
8878
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
8879
|
+
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
8880
|
+
writeJson(path9.join(outputDir, "summary.json"), summary);
|
|
8881
|
+
writeJson(path9.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
8882
|
+
let comparison;
|
|
8883
|
+
if (againstPath) {
|
|
8884
|
+
const baseline = loadSummary(againstPath);
|
|
8885
|
+
comparison = compareSummaries(summary, baseline, againstPath);
|
|
8886
|
+
writeJson(path9.join(outputDir, "compare.json"), comparison);
|
|
8887
|
+
}
|
|
8888
|
+
let gate;
|
|
8889
|
+
if (options.ciMode) {
|
|
8890
|
+
if (!budgetPath) {
|
|
8891
|
+
throw new Error("CI mode requires --budget path");
|
|
8892
|
+
}
|
|
8893
|
+
const budget = loadBudget(budgetPath);
|
|
8894
|
+
if (!comparison && budget.baselinePath) {
|
|
8895
|
+
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
8896
|
+
if (existsSync7(resolvedBaseline)) {
|
|
8897
|
+
const baselineSummary = loadSummary(resolvedBaseline);
|
|
8898
|
+
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
8899
|
+
writeJson(path9.join(outputDir, "compare.json"), comparison);
|
|
8900
|
+
} else if (budget.failOnMissingBaseline) {
|
|
8901
|
+
throw new Error(
|
|
8902
|
+
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
8903
|
+
);
|
|
8904
|
+
}
|
|
8905
|
+
}
|
|
8906
|
+
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
7192
8907
|
}
|
|
7193
|
-
|
|
8908
|
+
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
8909
|
+
writeText(path9.join(outputDir, "summary.md"), markdown);
|
|
8910
|
+
return { outputDir, summary, perQuery, comparison, gate };
|
|
8911
|
+
} finally {
|
|
8912
|
+
await indexer.close();
|
|
7194
8913
|
}
|
|
7195
|
-
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
7196
|
-
writeText(path7.join(outputDir, "summary.md"), markdown);
|
|
7197
|
-
return { outputDir, summary, perQuery, comparison, gate };
|
|
7198
8914
|
}
|
|
7199
8915
|
async function runSweep(options, sweep) {
|
|
7200
8916
|
const fusionValues = sweep.fusionStrategy && sweep.fusionStrategy.length > 0 ? [...sweep.fusionStrategy] : [void 0];
|
|
@@ -7248,15 +8964,15 @@ async function runSweep(options, sweep) {
|
|
|
7248
8964
|
bestByMrrAt10,
|
|
7249
8965
|
bestByP95Latency
|
|
7250
8966
|
};
|
|
7251
|
-
writeJson(
|
|
8967
|
+
writeJson(path9.join(outputDir, "compare.json"), aggregate);
|
|
7252
8968
|
const md = createSummaryMarkdown(
|
|
7253
8969
|
bestByHitAt5?.summary ?? runs[0].summary,
|
|
7254
8970
|
bestByHitAt5?.comparison,
|
|
7255
8971
|
void 0,
|
|
7256
8972
|
aggregate
|
|
7257
8973
|
);
|
|
7258
|
-
writeText(
|
|
7259
|
-
writeJson(
|
|
8974
|
+
writeText(path9.join(outputDir, "summary.md"), md);
|
|
8975
|
+
writeJson(path9.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
7260
8976
|
return { outputDir, aggregate };
|
|
7261
8977
|
}
|
|
7262
8978
|
|
|
@@ -7332,12 +9048,12 @@ function parseEvalArgs(argv, cwd) {
|
|
|
7332
9048
|
const arg = argv[i];
|
|
7333
9049
|
const next = argv[i + 1];
|
|
7334
9050
|
if (arg === "--project" && next) {
|
|
7335
|
-
parsed.projectRoot =
|
|
9051
|
+
parsed.projectRoot = path10.resolve(cwd, next);
|
|
7336
9052
|
i += 1;
|
|
7337
9053
|
continue;
|
|
7338
9054
|
}
|
|
7339
9055
|
if (arg === "--config" && next) {
|
|
7340
|
-
parsed.configPath =
|
|
9056
|
+
parsed.configPath = path10.resolve(cwd, next);
|
|
7341
9057
|
i += 1;
|
|
7342
9058
|
continue;
|
|
7343
9059
|
}
|
|
@@ -7531,22 +9247,22 @@ async function handleEvalCommand(args, cwd) {
|
|
|
7531
9247
|
if (!parsed.againstPath.endsWith(".json")) {
|
|
7532
9248
|
throw new Error("eval diff --against must point to a summary JSON file");
|
|
7533
9249
|
}
|
|
7534
|
-
const currentSummary = loadSummary(
|
|
9250
|
+
const currentSummary = loadSummary(path10.resolve(parsed.projectRoot, currentPath), {
|
|
7535
9251
|
allowLegacyDiversityMetrics: true
|
|
7536
9252
|
});
|
|
7537
|
-
const baselineSummary = loadSummary(
|
|
9253
|
+
const baselineSummary = loadSummary(path10.resolve(parsed.projectRoot, parsed.againstPath), {
|
|
7538
9254
|
allowLegacyDiversityMetrics: true
|
|
7539
9255
|
});
|
|
7540
9256
|
const comparison = compareSummaries(
|
|
7541
9257
|
currentSummary,
|
|
7542
9258
|
baselineSummary,
|
|
7543
|
-
|
|
9259
|
+
path10.resolve(parsed.projectRoot, parsed.againstPath)
|
|
7544
9260
|
);
|
|
7545
|
-
const outputDir = createRunDirectory(
|
|
9261
|
+
const outputDir = createRunDirectory(path10.resolve(parsed.projectRoot, parsed.outputRoot));
|
|
7546
9262
|
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
7547
|
-
writeJson(
|
|
7548
|
-
writeText(
|
|
7549
|
-
writeJson(
|
|
9263
|
+
writeJson(path10.join(outputDir, "compare.json"), comparison);
|
|
9264
|
+
writeText(path10.join(outputDir, "summary.md"), summaryMd);
|
|
9265
|
+
writeJson(path10.join(outputDir, "summary.json"), currentSummary);
|
|
7550
9266
|
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
7551
9267
|
return 0;
|
|
7552
9268
|
}
|
|
@@ -7556,6 +9272,8 @@ async function handleEvalCommand(args, cwd) {
|
|
|
7556
9272
|
// src/mcp-server.ts
|
|
7557
9273
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
7558
9274
|
import { z } from "zod";
|
|
9275
|
+
import * as path11 from "path";
|
|
9276
|
+
import { existsSync as existsSync8 } from "fs";
|
|
7559
9277
|
|
|
7560
9278
|
// src/tools/utils.ts
|
|
7561
9279
|
var MAX_CONTENT_LINES = 30;
|
|
@@ -7565,36 +9283,24 @@ function truncateContent(content) {
|
|
|
7565
9283
|
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
7566
9284
|
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
7567
9285
|
}
|
|
7568
|
-
function formatDefinitionLookup(results, query) {
|
|
7569
|
-
if (results.length === 0) {
|
|
7570
|
-
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
7571
|
-
}
|
|
7572
|
-
const formatted = results.map((r, idx) => {
|
|
7573
|
-
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
7574
|
-
return `${header} (score: ${r.score.toFixed(2)})
|
|
7575
|
-
\`\`\`
|
|
7576
|
-
${truncateContent(r.content)}
|
|
7577
|
-
\`\`\``;
|
|
7578
|
-
});
|
|
7579
|
-
return formatted.join("\n\n");
|
|
7580
|
-
}
|
|
7581
|
-
|
|
7582
|
-
// src/mcp-server.ts
|
|
7583
|
-
var MAX_CONTENT_LINES2 = 30;
|
|
7584
|
-
function truncateContent2(content) {
|
|
7585
|
-
const lines = content.split("\n");
|
|
7586
|
-
if (lines.length <= MAX_CONTENT_LINES2) return content;
|
|
7587
|
-
return lines.slice(0, MAX_CONTENT_LINES2).join("\n") + `
|
|
7588
|
-
// ... (${lines.length - MAX_CONTENT_LINES2} more lines)`;
|
|
7589
|
-
}
|
|
7590
9286
|
function formatIndexStats(stats, verbose = false) {
|
|
9287
|
+
if (stats.resetCorruptedIndex) {
|
|
9288
|
+
return stats.warning ?? "Detected a corrupted local index and reset it during indexing. Run index_codebase again to rebuild search data.";
|
|
9289
|
+
}
|
|
7591
9290
|
const lines = [];
|
|
9291
|
+
if (stats.failedChunks > 0) {
|
|
9292
|
+
lines.push(`INDEXING WARNING: ${stats.failedChunks} chunks failed to embed.`);
|
|
9293
|
+
if (stats.failedBatchesPath) {
|
|
9294
|
+
lines.push(`Inspect failed batches at: ${stats.failedBatchesPath}`);
|
|
9295
|
+
}
|
|
9296
|
+
lines.push("");
|
|
9297
|
+
}
|
|
7592
9298
|
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
7593
|
-
lines.push(
|
|
9299
|
+
lines.push(`${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
7594
9300
|
} else if (stats.indexedChunks === 0) {
|
|
7595
|
-
lines.push(
|
|
9301
|
+
lines.push(`${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
7596
9302
|
} else {
|
|
7597
|
-
let main2 =
|
|
9303
|
+
let main2 = `${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
7598
9304
|
if (stats.existingChunks > 0) {
|
|
7599
9305
|
main2 += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
7600
9306
|
}
|
|
@@ -7633,21 +9339,103 @@ function formatIndexStats(stats, verbose = false) {
|
|
|
7633
9339
|
}
|
|
7634
9340
|
function formatStatus(status) {
|
|
7635
9341
|
if (!status.indexed) {
|
|
9342
|
+
if (status.warning) {
|
|
9343
|
+
return status.warning;
|
|
9344
|
+
}
|
|
9345
|
+
if (status.failedBatchesCount > 0) {
|
|
9346
|
+
const lines2 = [
|
|
9347
|
+
"Codebase is not indexed. The last indexing run left failed embedding batches.",
|
|
9348
|
+
"Fix the provider/model configuration, then rerun index_codebase normally to retry the saved failed batches. Use force=true only for a full rebuild or compatibility reset."
|
|
9349
|
+
];
|
|
9350
|
+
if (status.failedBatchesPath) {
|
|
9351
|
+
lines2.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
9352
|
+
}
|
|
9353
|
+
return lines2.join("\n");
|
|
9354
|
+
}
|
|
7636
9355
|
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
7637
9356
|
}
|
|
7638
9357
|
const lines = [
|
|
7639
|
-
`
|
|
7640
|
-
`
|
|
7641
|
-
`
|
|
7642
|
-
`
|
|
7643
|
-
` Location: ${status.indexPath}`
|
|
9358
|
+
`Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
9359
|
+
`Provider: ${status.provider}`,
|
|
9360
|
+
`Model: ${status.model}`,
|
|
9361
|
+
`Location: ${status.indexPath}`
|
|
7644
9362
|
];
|
|
7645
9363
|
if (status.currentBranch !== "default") {
|
|
7646
|
-
lines.push(`
|
|
7647
|
-
lines.push(`
|
|
9364
|
+
lines.push(`Current branch: ${status.currentBranch}`);
|
|
9365
|
+
lines.push(`Base branch: ${status.baseBranch}`);
|
|
9366
|
+
}
|
|
9367
|
+
if (status.failedBatchesCount > 0) {
|
|
9368
|
+
lines.push("");
|
|
9369
|
+
lines.push(`INDEXING WARNING: ${status.failedBatchesCount} failed embedding batch${status.failedBatchesCount === 1 ? " remains" : "es remain"}.`);
|
|
9370
|
+
if (status.failedBatchesPath) {
|
|
9371
|
+
lines.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
9372
|
+
}
|
|
9373
|
+
}
|
|
9374
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
9375
|
+
lines.push("");
|
|
9376
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
9377
|
+
if (status.compatibility.storedMetadata) {
|
|
9378
|
+
const stored = status.compatibility.storedMetadata;
|
|
9379
|
+
lines.push(`Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
9380
|
+
lines.push(`Current config: ${status.provider}/${status.model}`);
|
|
9381
|
+
}
|
|
9382
|
+
} else if (!status.compatibility) {
|
|
9383
|
+
lines.push(`Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
9384
|
+
} else {
|
|
9385
|
+
lines.push(`Compatibility: Index is compatible with the current provider and model.`);
|
|
9386
|
+
}
|
|
9387
|
+
return lines.join("\n");
|
|
9388
|
+
}
|
|
9389
|
+
function formatHealthCheck(result) {
|
|
9390
|
+
if (result.resetCorruptedIndex) {
|
|
9391
|
+
return result.warning ?? "Detected a corrupted local index and reset it. Run index_codebase to rebuild search data.";
|
|
9392
|
+
}
|
|
9393
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0 && result.gcOrphanSymbols === 0 && result.gcOrphanCallEdges === 0) {
|
|
9394
|
+
return "Index is healthy. No stale entries found.";
|
|
9395
|
+
}
|
|
9396
|
+
const lines = [];
|
|
9397
|
+
if (result.removed > 0) {
|
|
9398
|
+
lines.push(`Removed stale entries: ${result.removed}`);
|
|
9399
|
+
}
|
|
9400
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
9401
|
+
lines.push(`Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
9402
|
+
}
|
|
9403
|
+
if (result.gcOrphanChunks > 0) {
|
|
9404
|
+
lines.push(`Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
9405
|
+
}
|
|
9406
|
+
if (result.gcOrphanSymbols > 0) {
|
|
9407
|
+
lines.push(`Garbage collected orphan symbols: ${result.gcOrphanSymbols}`);
|
|
9408
|
+
}
|
|
9409
|
+
if (result.gcOrphanCallEdges > 0) {
|
|
9410
|
+
lines.push(`Garbage collected orphan call edges: ${result.gcOrphanCallEdges}`);
|
|
9411
|
+
}
|
|
9412
|
+
if (result.filePaths.length > 0) {
|
|
9413
|
+
lines.push(`Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
7648
9414
|
}
|
|
7649
9415
|
return lines.join("\n");
|
|
7650
9416
|
}
|
|
9417
|
+
function formatDefinitionLookup(results, query) {
|
|
9418
|
+
if (results.length === 0) {
|
|
9419
|
+
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
9420
|
+
}
|
|
9421
|
+
const formatted = results.map((r, idx) => {
|
|
9422
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
9423
|
+
return `${header} (score: ${r.score.toFixed(2)})
|
|
9424
|
+
\`\`\`
|
|
9425
|
+
${truncateContent(r.content)}
|
|
9426
|
+
\`\`\``;
|
|
9427
|
+
});
|
|
9428
|
+
return formatted.join("\n\n");
|
|
9429
|
+
}
|
|
9430
|
+
|
|
9431
|
+
// src/mcp-server.ts
|
|
9432
|
+
var MAX_CONTENT_LINES2 = 30;
|
|
9433
|
+
function truncateContent2(content) {
|
|
9434
|
+
const lines = content.split("\n");
|
|
9435
|
+
if (lines.length <= MAX_CONTENT_LINES2) return content;
|
|
9436
|
+
return lines.slice(0, MAX_CONTENT_LINES2).join("\n") + `
|
|
9437
|
+
// ... (${lines.length - MAX_CONTENT_LINES2} more lines)`;
|
|
9438
|
+
}
|
|
7651
9439
|
var CHUNK_TYPE_ENUM = [
|
|
7652
9440
|
"function",
|
|
7653
9441
|
"class",
|
|
@@ -7666,8 +9454,25 @@ function createMcpServer(projectRoot, config) {
|
|
|
7666
9454
|
name: "opencode-codebase-index",
|
|
7667
9455
|
version: "0.5.1"
|
|
7668
9456
|
});
|
|
7669
|
-
const
|
|
9457
|
+
const runtimeConfig = config;
|
|
9458
|
+
let indexer = new Indexer(projectRoot, runtimeConfig);
|
|
7670
9459
|
let initialized = false;
|
|
9460
|
+
function refreshIndexerFromConfig() {
|
|
9461
|
+
indexer = new Indexer(projectRoot, runtimeConfig);
|
|
9462
|
+
initialized = false;
|
|
9463
|
+
}
|
|
9464
|
+
function shouldForceLocalizeProjectIndex() {
|
|
9465
|
+
if (runtimeConfig.scope !== "project") {
|
|
9466
|
+
return false;
|
|
9467
|
+
}
|
|
9468
|
+
const localIndexPath = path11.join(projectRoot, ".opencode", "index");
|
|
9469
|
+
const mainRepoRoot = resolveWorktreeMainRepoRoot(projectRoot);
|
|
9470
|
+
if (!mainRepoRoot) {
|
|
9471
|
+
return false;
|
|
9472
|
+
}
|
|
9473
|
+
const inheritedIndexPath = path11.join(mainRepoRoot, ".opencode", "index");
|
|
9474
|
+
return !existsSync8(localIndexPath) && existsSync8(inheritedIndexPath);
|
|
9475
|
+
}
|
|
7671
9476
|
async function ensureInitialized() {
|
|
7672
9477
|
if (!initialized) {
|
|
7673
9478
|
await indexer.initialize();
|
|
@@ -7750,13 +9555,22 @@ Use Read tool to examine specific files.` }] };
|
|
|
7750
9555
|
verbose: z.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures")
|
|
7751
9556
|
},
|
|
7752
9557
|
async (args) => {
|
|
7753
|
-
await ensureInitialized();
|
|
7754
9558
|
if (args.estimateOnly) {
|
|
9559
|
+
await ensureInitialized();
|
|
7755
9560
|
const estimate = await indexer.estimateCost();
|
|
7756
9561
|
return { content: [{ type: "text", text: formatCostEstimate(estimate) }] };
|
|
7757
9562
|
}
|
|
7758
9563
|
if (args.force) {
|
|
9564
|
+
if (shouldForceLocalizeProjectIndex()) {
|
|
9565
|
+
materializeLocalProjectConfig(projectRoot, loadProjectConfigLayer(projectRoot));
|
|
9566
|
+
refreshIndexerFromConfig();
|
|
9567
|
+
}
|
|
9568
|
+
await ensureInitialized();
|
|
7759
9569
|
await indexer.clearIndex();
|
|
9570
|
+
refreshIndexerFromConfig();
|
|
9571
|
+
await ensureInitialized();
|
|
9572
|
+
} else {
|
|
9573
|
+
await ensureInitialized();
|
|
7760
9574
|
}
|
|
7761
9575
|
const stats = await indexer.index();
|
|
7762
9576
|
return { content: [{ type: "text", text: formatIndexStats(stats, args.verbose ?? false) }] };
|
|
@@ -7779,29 +9593,7 @@ Use Read tool to examine specific files.` }] };
|
|
|
7779
9593
|
async () => {
|
|
7780
9594
|
await ensureInitialized();
|
|
7781
9595
|
const result = await indexer.healthCheck();
|
|
7782
|
-
|
|
7783
|
-
return { content: [{ type: "text", text: "Index is healthy. No stale entries found." }] };
|
|
7784
|
-
}
|
|
7785
|
-
const lines = [`Health check complete:`];
|
|
7786
|
-
if (result.removed > 0) {
|
|
7787
|
-
lines.push(` Removed stale entries: ${result.removed}`);
|
|
7788
|
-
}
|
|
7789
|
-
if (result.gcOrphanEmbeddings > 0) {
|
|
7790
|
-
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
7791
|
-
}
|
|
7792
|
-
if (result.gcOrphanChunks > 0) {
|
|
7793
|
-
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
7794
|
-
}
|
|
7795
|
-
if (result.gcOrphanSymbols > 0) {
|
|
7796
|
-
lines.push(` Garbage collected orphan symbols: ${result.gcOrphanSymbols}`);
|
|
7797
|
-
}
|
|
7798
|
-
if (result.gcOrphanCallEdges > 0) {
|
|
7799
|
-
lines.push(` Garbage collected orphan call edges: ${result.gcOrphanCallEdges}`);
|
|
7800
|
-
}
|
|
7801
|
-
if (result.filePaths.length > 0) {
|
|
7802
|
-
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
7803
|
-
}
|
|
7804
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
9596
|
+
return { content: [{ type: "text", text: formatHealthCheck(result) }] };
|
|
7805
9597
|
}
|
|
7806
9598
|
);
|
|
7807
9599
|
server.tool(
|
|
@@ -8030,115 +9822,15 @@ Use the implementation_lookup tool to find where this symbol is defined. This pr
|
|
|
8030
9822
|
return server;
|
|
8031
9823
|
}
|
|
8032
9824
|
|
|
8033
|
-
// src/config/merger.ts
|
|
8034
|
-
import { existsSync as existsSync6, readFileSync as readFileSync8 } from "fs";
|
|
8035
|
-
import * as path9 from "path";
|
|
8036
|
-
import * as os4 from "os";
|
|
8037
|
-
function loadJsonFile(filePath) {
|
|
8038
|
-
try {
|
|
8039
|
-
if (existsSync6(filePath)) {
|
|
8040
|
-
const content = readFileSync8(filePath, "utf-8");
|
|
8041
|
-
return JSON.parse(content);
|
|
8042
|
-
}
|
|
8043
|
-
} catch {
|
|
8044
|
-
}
|
|
8045
|
-
return null;
|
|
8046
|
-
}
|
|
8047
|
-
function loadMergedConfig(projectRoot) {
|
|
8048
|
-
const globalConfigPath = path9.join(os4.homedir(), ".config", "opencode", "codebase-index.json");
|
|
8049
|
-
const globalConfig = loadJsonFile(globalConfigPath);
|
|
8050
|
-
const projectConfigPath = path9.join(projectRoot, ".opencode", "codebase-index.json");
|
|
8051
|
-
const projectConfig = loadJsonFile(projectConfigPath);
|
|
8052
|
-
if (!globalConfig && !projectConfig) {
|
|
8053
|
-
return {};
|
|
8054
|
-
}
|
|
8055
|
-
if (!projectConfig && globalConfig) {
|
|
8056
|
-
return globalConfig;
|
|
8057
|
-
}
|
|
8058
|
-
if (!globalConfig && projectConfig) {
|
|
8059
|
-
return projectConfig;
|
|
8060
|
-
}
|
|
8061
|
-
const merged = { ...globalConfig };
|
|
8062
|
-
if (projectConfig && "embeddingProvider" in projectConfig) {
|
|
8063
|
-
merged.embeddingProvider = projectConfig.embeddingProvider;
|
|
8064
|
-
} else if (globalConfig && globalConfig.embeddingProvider) {
|
|
8065
|
-
merged.embeddingProvider = globalConfig.embeddingProvider;
|
|
8066
|
-
}
|
|
8067
|
-
if (projectConfig && "customProvider" in projectConfig) {
|
|
8068
|
-
merged.customProvider = projectConfig.customProvider;
|
|
8069
|
-
} else if (globalConfig && globalConfig.customProvider) {
|
|
8070
|
-
merged.customProvider = globalConfig.customProvider;
|
|
8071
|
-
}
|
|
8072
|
-
if (projectConfig && "embeddingModel" in projectConfig) {
|
|
8073
|
-
merged.embeddingModel = projectConfig.embeddingModel;
|
|
8074
|
-
} else if (globalConfig && globalConfig.embeddingModel) {
|
|
8075
|
-
merged.embeddingModel = globalConfig.embeddingModel;
|
|
8076
|
-
}
|
|
8077
|
-
if (projectConfig && "reranker" in projectConfig) {
|
|
8078
|
-
merged.reranker = projectConfig.reranker;
|
|
8079
|
-
} else if (globalConfig && globalConfig.reranker) {
|
|
8080
|
-
merged.reranker = globalConfig.reranker;
|
|
8081
|
-
}
|
|
8082
|
-
if (projectConfig && "include" in projectConfig) {
|
|
8083
|
-
merged.include = projectConfig.include;
|
|
8084
|
-
} else if (globalConfig && globalConfig.include) {
|
|
8085
|
-
merged.include = globalConfig.include;
|
|
8086
|
-
}
|
|
8087
|
-
if (projectConfig && "exclude" in projectConfig) {
|
|
8088
|
-
merged.exclude = projectConfig.exclude;
|
|
8089
|
-
} else if (globalConfig && globalConfig.exclude) {
|
|
8090
|
-
merged.exclude = globalConfig.exclude;
|
|
8091
|
-
}
|
|
8092
|
-
if (projectConfig && "indexing" in projectConfig) {
|
|
8093
|
-
merged.indexing = projectConfig.indexing;
|
|
8094
|
-
} else if (globalConfig && globalConfig.indexing) {
|
|
8095
|
-
merged.indexing = globalConfig.indexing;
|
|
8096
|
-
}
|
|
8097
|
-
if (projectConfig && "search" in projectConfig) {
|
|
8098
|
-
merged.search = projectConfig.search;
|
|
8099
|
-
} else if (globalConfig && globalConfig.search) {
|
|
8100
|
-
merged.search = globalConfig.search;
|
|
8101
|
-
}
|
|
8102
|
-
if (projectConfig && "debug" in projectConfig) {
|
|
8103
|
-
merged.debug = projectConfig.debug;
|
|
8104
|
-
} else if (globalConfig && globalConfig.debug) {
|
|
8105
|
-
merged.debug = globalConfig.debug;
|
|
8106
|
-
}
|
|
8107
|
-
if (projectConfig && "scope" in projectConfig) {
|
|
8108
|
-
merged.scope = projectConfig.scope;
|
|
8109
|
-
} else if (globalConfig && "scope" in globalConfig) {
|
|
8110
|
-
merged.scope = globalConfig.scope;
|
|
8111
|
-
}
|
|
8112
|
-
if (projectConfig) {
|
|
8113
|
-
for (const key of Object.keys(projectConfig)) {
|
|
8114
|
-
if (key === "embeddingProvider" || key === "customProvider" || key === "embeddingModel" || key === "reranker" || key === "include" || key === "exclude" || key === "indexing" || key === "search" || key === "debug" || key === "scope" || key === "knowledgeBases" || key === "additionalInclude") {
|
|
8115
|
-
continue;
|
|
8116
|
-
}
|
|
8117
|
-
merged[key] = projectConfig[key];
|
|
8118
|
-
}
|
|
8119
|
-
}
|
|
8120
|
-
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
8121
|
-
const projectKbs = projectConfig && Array.isArray(projectConfig.knowledgeBases) ? projectConfig.knowledgeBases : [];
|
|
8122
|
-
const allKbs = [...globalKbs, ...projectKbs];
|
|
8123
|
-
const uniqueKbs = [...new Set(allKbs.map((p) => String(p).trim()))];
|
|
8124
|
-
merged.knowledgeBases = uniqueKbs;
|
|
8125
|
-
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
8126
|
-
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
8127
|
-
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
8128
|
-
const uniqueAdditional = [...new Set(allAdditional.map((p) => String(p).trim()))];
|
|
8129
|
-
merged.additionalInclude = uniqueAdditional;
|
|
8130
|
-
return merged;
|
|
8131
|
-
}
|
|
8132
|
-
|
|
8133
9825
|
// src/cli.ts
|
|
8134
9826
|
function parseArgs(argv) {
|
|
8135
9827
|
let project = process.cwd();
|
|
8136
9828
|
let config;
|
|
8137
9829
|
for (let i = 2; i < argv.length; i++) {
|
|
8138
9830
|
if (argv[i] === "--project" && argv[i + 1]) {
|
|
8139
|
-
project =
|
|
9831
|
+
project = path12.resolve(argv[++i]);
|
|
8140
9832
|
} else if (argv[i] === "--config" && argv[i + 1]) {
|
|
8141
|
-
config =
|
|
9833
|
+
config = path12.resolve(argv[++i]);
|
|
8142
9834
|
}
|
|
8143
9835
|
}
|
|
8144
9836
|
return { project, config };
|