opencode-codebase-index 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +2 -2
- package/README.md +73 -12
- package/dist/cli.cjs +1782 -607
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1781 -596
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +1610 -533
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1606 -519
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +1396 -311
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +1395 -300
- package/dist/pi-extension.js.map +1 -1
- package/hooks/hooks.json +2 -2
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +11 -8
- package/skills/codebase-search/SKILL.md +9 -7
package/dist/pi-extension.js
CHANGED
|
@@ -490,7 +490,7 @@ var require_ignore = __commonJS({
|
|
|
490
490
|
// path matching.
|
|
491
491
|
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
492
492
|
// @returns {TestResult} true if a file is ignored
|
|
493
|
-
test(
|
|
493
|
+
test(path17, checkUnignored, mode) {
|
|
494
494
|
let ignored = false;
|
|
495
495
|
let unignored = false;
|
|
496
496
|
let matchedRule;
|
|
@@ -499,7 +499,7 @@ var require_ignore = __commonJS({
|
|
|
499
499
|
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
500
500
|
return;
|
|
501
501
|
}
|
|
502
|
-
const matched = rule[mode].test(
|
|
502
|
+
const matched = rule[mode].test(path17);
|
|
503
503
|
if (!matched) {
|
|
504
504
|
return;
|
|
505
505
|
}
|
|
@@ -520,17 +520,17 @@ var require_ignore = __commonJS({
|
|
|
520
520
|
var throwError = (message, Ctor) => {
|
|
521
521
|
throw new Ctor(message);
|
|
522
522
|
};
|
|
523
|
-
var checkPath = (
|
|
524
|
-
if (!isString(
|
|
523
|
+
var checkPath = (path17, originalPath, doThrow) => {
|
|
524
|
+
if (!isString(path17)) {
|
|
525
525
|
return doThrow(
|
|
526
526
|
`path must be a string, but got \`${originalPath}\``,
|
|
527
527
|
TypeError
|
|
528
528
|
);
|
|
529
529
|
}
|
|
530
|
-
if (!
|
|
530
|
+
if (!path17) {
|
|
531
531
|
return doThrow(`path must not be empty`, TypeError);
|
|
532
532
|
}
|
|
533
|
-
if (checkPath.isNotRelative(
|
|
533
|
+
if (checkPath.isNotRelative(path17)) {
|
|
534
534
|
const r = "`path.relative()`d";
|
|
535
535
|
return doThrow(
|
|
536
536
|
`path should be a ${r} string, but got "${originalPath}"`,
|
|
@@ -539,7 +539,7 @@ var require_ignore = __commonJS({
|
|
|
539
539
|
}
|
|
540
540
|
return true;
|
|
541
541
|
};
|
|
542
|
-
var isNotRelative = (
|
|
542
|
+
var isNotRelative = (path17) => REGEX_TEST_INVALID_PATH.test(path17);
|
|
543
543
|
checkPath.isNotRelative = isNotRelative;
|
|
544
544
|
checkPath.convert = (p) => p;
|
|
545
545
|
var Ignore2 = class {
|
|
@@ -569,19 +569,19 @@ var require_ignore = __commonJS({
|
|
|
569
569
|
}
|
|
570
570
|
// @returns {TestResult}
|
|
571
571
|
_test(originalPath, cache, checkUnignored, slices) {
|
|
572
|
-
const
|
|
572
|
+
const path17 = originalPath && checkPath.convert(originalPath);
|
|
573
573
|
checkPath(
|
|
574
|
-
|
|
574
|
+
path17,
|
|
575
575
|
originalPath,
|
|
576
576
|
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
577
577
|
);
|
|
578
|
-
return this._t(
|
|
578
|
+
return this._t(path17, cache, checkUnignored, slices);
|
|
579
579
|
}
|
|
580
|
-
checkIgnore(
|
|
581
|
-
if (!REGEX_TEST_TRAILING_SLASH.test(
|
|
582
|
-
return this.test(
|
|
580
|
+
checkIgnore(path17) {
|
|
581
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path17)) {
|
|
582
|
+
return this.test(path17);
|
|
583
583
|
}
|
|
584
|
-
const slices =
|
|
584
|
+
const slices = path17.split(SLASH).filter(Boolean);
|
|
585
585
|
slices.pop();
|
|
586
586
|
if (slices.length) {
|
|
587
587
|
const parent = this._t(
|
|
@@ -594,18 +594,18 @@ var require_ignore = __commonJS({
|
|
|
594
594
|
return parent;
|
|
595
595
|
}
|
|
596
596
|
}
|
|
597
|
-
return this._rules.test(
|
|
597
|
+
return this._rules.test(path17, false, MODE_CHECK_IGNORE);
|
|
598
598
|
}
|
|
599
|
-
_t(
|
|
600
|
-
if (
|
|
601
|
-
return cache[
|
|
599
|
+
_t(path17, cache, checkUnignored, slices) {
|
|
600
|
+
if (path17 in cache) {
|
|
601
|
+
return cache[path17];
|
|
602
602
|
}
|
|
603
603
|
if (!slices) {
|
|
604
|
-
slices =
|
|
604
|
+
slices = path17.split(SLASH).filter(Boolean);
|
|
605
605
|
}
|
|
606
606
|
slices.pop();
|
|
607
607
|
if (!slices.length) {
|
|
608
|
-
return cache[
|
|
608
|
+
return cache[path17] = this._rules.test(path17, checkUnignored, MODE_IGNORE);
|
|
609
609
|
}
|
|
610
610
|
const parent = this._t(
|
|
611
611
|
slices.join(SLASH) + SLASH,
|
|
@@ -613,29 +613,29 @@ var require_ignore = __commonJS({
|
|
|
613
613
|
checkUnignored,
|
|
614
614
|
slices
|
|
615
615
|
);
|
|
616
|
-
return cache[
|
|
616
|
+
return cache[path17] = parent.ignored ? parent : this._rules.test(path17, checkUnignored, MODE_IGNORE);
|
|
617
617
|
}
|
|
618
|
-
ignores(
|
|
619
|
-
return this._test(
|
|
618
|
+
ignores(path17) {
|
|
619
|
+
return this._test(path17, this._ignoreCache, false).ignored;
|
|
620
620
|
}
|
|
621
621
|
createFilter() {
|
|
622
|
-
return (
|
|
622
|
+
return (path17) => !this.ignores(path17);
|
|
623
623
|
}
|
|
624
624
|
filter(paths) {
|
|
625
625
|
return makeArray(paths).filter(this.createFilter());
|
|
626
626
|
}
|
|
627
627
|
// @returns {TestResult}
|
|
628
|
-
test(
|
|
629
|
-
return this._test(
|
|
628
|
+
test(path17) {
|
|
629
|
+
return this._test(path17, this._testCache, true);
|
|
630
630
|
}
|
|
631
631
|
};
|
|
632
632
|
var factory = (options) => new Ignore2(options);
|
|
633
|
-
var isPathValid = (
|
|
633
|
+
var isPathValid = (path17) => checkPath(path17 && checkPath.convert(path17), path17, RETURN_FALSE);
|
|
634
634
|
var setupWindows = () => {
|
|
635
635
|
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
636
636
|
checkPath.convert = makePosix;
|
|
637
637
|
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
638
|
-
checkPath.isNotRelative = (
|
|
638
|
+
checkPath.isNotRelative = (path17) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path17) || isNotRelative(path17);
|
|
639
639
|
};
|
|
640
640
|
if (
|
|
641
641
|
// Detect `process` so that it can run in browsers.
|
|
@@ -655,10 +655,10 @@ import { Type as Type2 } from "typebox";
|
|
|
655
655
|
|
|
656
656
|
// src/config/constants.ts
|
|
657
657
|
var DEFAULT_INCLUDE = [
|
|
658
|
-
"**/*.{ts,tsx,js,jsx,mjs,cjs}",
|
|
658
|
+
"**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}",
|
|
659
659
|
"**/*.{py,pyi}",
|
|
660
|
-
"**/*.{go,rs,java,kt,scala}",
|
|
661
|
-
"**/*.{c,cpp,cc,h,hpp}",
|
|
660
|
+
"**/*.{go,rs,java,cs,kt,scala}",
|
|
661
|
+
"**/*.{c,cpp,cc,cxx,h,hpp,hxx}",
|
|
662
662
|
"**/*.{rb,php,inc,swift}",
|
|
663
663
|
"**/*.{cls,trigger}",
|
|
664
664
|
"**/*.{vue,svelte,astro}",
|
|
@@ -1327,8 +1327,8 @@ function formatPrImpact(result) {
|
|
|
1327
1327
|
}
|
|
1328
1328
|
|
|
1329
1329
|
// src/tools/operations.ts
|
|
1330
|
-
import { existsSync as
|
|
1331
|
-
import * as
|
|
1330
|
+
import { existsSync as existsSync10, realpathSync as realpathSync2, statSync as statSync4 } from "fs";
|
|
1331
|
+
import * as path16 from "path";
|
|
1332
1332
|
|
|
1333
1333
|
// src/config/merger.ts
|
|
1334
1334
|
import { existsSync as existsSync5, mkdirSync, readFileSync as readFileSync4, writeFileSync } from "fs";
|
|
@@ -1525,6 +1525,9 @@ function getHostProjectIndexRelativePath(host) {
|
|
|
1525
1525
|
function hasHostProjectConfig(projectRoot3, host) {
|
|
1526
1526
|
return existsSync4(path4.join(projectRoot3, getProjectConfigRelativePath(host)));
|
|
1527
1527
|
}
|
|
1528
|
+
function hasHostGlobalConfig(host) {
|
|
1529
|
+
return existsSync4(getGlobalConfigPath(host));
|
|
1530
|
+
}
|
|
1528
1531
|
function getGlobalIndexPath(host = "opencode") {
|
|
1529
1532
|
switch (host) {
|
|
1530
1533
|
case "opencode":
|
|
@@ -1564,6 +1567,9 @@ function resolveGlobalIndexPath(host = "opencode") {
|
|
|
1564
1567
|
return hostIndexPath;
|
|
1565
1568
|
}
|
|
1566
1569
|
if (host !== "opencode") {
|
|
1570
|
+
if (hasHostGlobalConfig(host)) {
|
|
1571
|
+
return hostIndexPath;
|
|
1572
|
+
}
|
|
1567
1573
|
const legacyIndexPath = getGlobalIndexPath("opencode");
|
|
1568
1574
|
if (existsSync4(legacyIndexPath)) {
|
|
1569
1575
|
return legacyIndexPath;
|
|
@@ -1829,8 +1835,8 @@ function loadMergedConfig(projectRoot3, host = "opencode") {
|
|
|
1829
1835
|
}
|
|
1830
1836
|
|
|
1831
1837
|
// src/indexer/index.ts
|
|
1832
|
-
import { existsSync as
|
|
1833
|
-
import * as
|
|
1838
|
+
import { existsSync as existsSync8, readFileSync as readFileSync7, statSync as statSync3, writeFileSync as writeFileSync3, renameSync as renameSync2, unlinkSync, mkdirSync as mkdirSync3, promises as fsPromises2 } from "fs";
|
|
1839
|
+
import * as path13 from "path";
|
|
1834
1840
|
import { performance as performance2 } from "perf_hooks";
|
|
1835
1841
|
import { execFile as execFile3 } from "child_process";
|
|
1836
1842
|
import { promisify as promisify3 } from "util";
|
|
@@ -2937,17 +2943,17 @@ function validateExternalUrl(urlString) {
|
|
|
2937
2943
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
2938
2944
|
return { valid: false, reason: `Blocked protocol: ${parsed.protocol}` };
|
|
2939
2945
|
}
|
|
2940
|
-
const
|
|
2941
|
-
if (BLOCKED_HOSTNAMES.has(
|
|
2942
|
-
return { valid: false, reason: `Blocked: cloud metadata service (${
|
|
2946
|
+
const hostname2 = parsed.hostname.toLowerCase();
|
|
2947
|
+
if (BLOCKED_HOSTNAMES.has(hostname2)) {
|
|
2948
|
+
return { valid: false, reason: `Blocked: cloud metadata service (${hostname2})` };
|
|
2943
2949
|
}
|
|
2944
2950
|
for (const pattern of BLOCKED_METADATA_IPS) {
|
|
2945
|
-
if (pattern.test(
|
|
2946
|
-
return { valid: false, reason: `Blocked: cloud metadata IP (${
|
|
2951
|
+
if (pattern.test(hostname2)) {
|
|
2952
|
+
return { valid: false, reason: `Blocked: cloud metadata IP (${hostname2})` };
|
|
2947
2953
|
}
|
|
2948
2954
|
}
|
|
2949
|
-
if (/^169\.254\./.test(
|
|
2950
|
-
return { valid: false, reason: `Blocked: link-local address (${
|
|
2955
|
+
if (/^169\.254\./.test(hostname2)) {
|
|
2956
|
+
return { valid: false, reason: `Blocked: link-local address (${hostname2})` };
|
|
2951
2957
|
}
|
|
2952
2958
|
return { valid: true };
|
|
2953
2959
|
}
|
|
@@ -3984,6 +3990,12 @@ function createMockNativeBinding() {
|
|
|
3984
3990
|
constructor() {
|
|
3985
3991
|
throw error;
|
|
3986
3992
|
}
|
|
3993
|
+
static openReadOnly() {
|
|
3994
|
+
throw error;
|
|
3995
|
+
}
|
|
3996
|
+
static createEmptyReadOnly() {
|
|
3997
|
+
throw error;
|
|
3998
|
+
}
|
|
3987
3999
|
close() {
|
|
3988
4000
|
throw error;
|
|
3989
4001
|
}
|
|
@@ -4087,6 +4099,12 @@ var VectorStore = class {
|
|
|
4087
4099
|
load() {
|
|
4088
4100
|
this.inner.load();
|
|
4089
4101
|
}
|
|
4102
|
+
loadStrict() {
|
|
4103
|
+
this.inner.loadStrict();
|
|
4104
|
+
}
|
|
4105
|
+
hasFingerprint() {
|
|
4106
|
+
return this.inner.hasFingerprint();
|
|
4107
|
+
}
|
|
4090
4108
|
count() {
|
|
4091
4109
|
return this.inner.count();
|
|
4092
4110
|
}
|
|
@@ -4369,12 +4387,24 @@ var InvertedIndex = class {
|
|
|
4369
4387
|
return this.inner.documentCount();
|
|
4370
4388
|
}
|
|
4371
4389
|
};
|
|
4372
|
-
var Database = class {
|
|
4390
|
+
var Database = class _Database {
|
|
4373
4391
|
inner;
|
|
4374
4392
|
closed = false;
|
|
4375
4393
|
constructor(dbPath) {
|
|
4376
4394
|
this.inner = new native.Database(dbPath);
|
|
4377
4395
|
}
|
|
4396
|
+
static fromNative(inner) {
|
|
4397
|
+
const database = Object.create(_Database.prototype);
|
|
4398
|
+
database.inner = inner;
|
|
4399
|
+
database.closed = false;
|
|
4400
|
+
return database;
|
|
4401
|
+
}
|
|
4402
|
+
static openReadOnly(dbPath) {
|
|
4403
|
+
return _Database.fromNative(native.Database.openReadOnly(dbPath));
|
|
4404
|
+
}
|
|
4405
|
+
static createEmptyReadOnly() {
|
|
4406
|
+
return _Database.fromNative(native.Database.createEmptyReadOnly());
|
|
4407
|
+
}
|
|
4378
4408
|
throwIfClosed() {
|
|
4379
4409
|
if (this.closed) {
|
|
4380
4410
|
throw new Error("Database is closed");
|
|
@@ -4803,6 +4833,428 @@ async function getChunkGitBlame(projectRoot3, filePath, startLine, endLine) {
|
|
|
4803
4833
|
}
|
|
4804
4834
|
}
|
|
4805
4835
|
|
|
4836
|
+
// src/indexer/index-lock.ts
|
|
4837
|
+
import { randomUUID } from "crypto";
|
|
4838
|
+
import {
|
|
4839
|
+
existsSync as existsSync7,
|
|
4840
|
+
lstatSync,
|
|
4841
|
+
mkdirSync as mkdirSync2,
|
|
4842
|
+
readFileSync as readFileSync6,
|
|
4843
|
+
readdirSync as readdirSync2,
|
|
4844
|
+
realpathSync,
|
|
4845
|
+
renameSync,
|
|
4846
|
+
rmSync,
|
|
4847
|
+
writeFileSync as writeFileSync2
|
|
4848
|
+
} from "fs";
|
|
4849
|
+
import * as os4 from "os";
|
|
4850
|
+
import * as path12 from "path";
|
|
4851
|
+
var OWNER_FILE_NAME = "owner.json";
|
|
4852
|
+
var RECLAIM_DIRECTORY_NAME = "reclaiming";
|
|
4853
|
+
var RECOVERY_MARKER_PREFIX = "indexing.lock.recovery.";
|
|
4854
|
+
var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
4855
|
+
var VALID_OPERATIONS = /* @__PURE__ */ new Set([
|
|
4856
|
+
"initialize",
|
|
4857
|
+
"index",
|
|
4858
|
+
"force-index",
|
|
4859
|
+
"clear",
|
|
4860
|
+
"health-check",
|
|
4861
|
+
"retry-failed-batches",
|
|
4862
|
+
"recovery"
|
|
4863
|
+
]);
|
|
4864
|
+
var temporaryCounter = 0;
|
|
4865
|
+
function getErrorCode(error) {
|
|
4866
|
+
return typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
4867
|
+
}
|
|
4868
|
+
function retryTransientFilesystemOperation(operation) {
|
|
4869
|
+
let lastError;
|
|
4870
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4871
|
+
try {
|
|
4872
|
+
operation();
|
|
4873
|
+
return;
|
|
4874
|
+
} catch (error) {
|
|
4875
|
+
lastError = error;
|
|
4876
|
+
const code = getErrorCode(error);
|
|
4877
|
+
if (code !== "EBUSY" && code !== "EPERM") throw error;
|
|
4878
|
+
if (attempt < 2) {
|
|
4879
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, (attempt + 1) * 10);
|
|
4880
|
+
}
|
|
4881
|
+
}
|
|
4882
|
+
}
|
|
4883
|
+
throw lastError;
|
|
4884
|
+
}
|
|
4885
|
+
function parseOwner(value) {
|
|
4886
|
+
if (typeof value !== "object" || value === null) return null;
|
|
4887
|
+
const candidate = value;
|
|
4888
|
+
if (!Number.isInteger(candidate.pid) || (candidate.pid ?? 0) <= 0) return null;
|
|
4889
|
+
if (typeof candidate.hostname !== "string" || candidate.hostname.length === 0) return null;
|
|
4890
|
+
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
4891
|
+
if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation)) return null;
|
|
4892
|
+
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
4893
|
+
return candidate;
|
|
4894
|
+
}
|
|
4895
|
+
function parseReclaimOwner(value) {
|
|
4896
|
+
if (typeof value !== "object" || value === null) return null;
|
|
4897
|
+
const candidate = value;
|
|
4898
|
+
if (!Number.isInteger(candidate.pid) || (candidate.pid ?? 0) <= 0) return null;
|
|
4899
|
+
if (typeof candidate.hostname !== "string" || candidate.hostname.length === 0) return null;
|
|
4900
|
+
if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null;
|
|
4901
|
+
if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null;
|
|
4902
|
+
if (typeof candidate.expectedOwnerToken !== "string" || !UUID_PATTERN.test(candidate.expectedOwnerToken)) return null;
|
|
4903
|
+
return candidate;
|
|
4904
|
+
}
|
|
4905
|
+
function readJsonDirectory(directoryPath, parser) {
|
|
4906
|
+
try {
|
|
4907
|
+
return parser(JSON.parse(readFileSync6(path12.join(directoryPath, OWNER_FILE_NAME), "utf-8")));
|
|
4908
|
+
} catch {
|
|
4909
|
+
return null;
|
|
4910
|
+
}
|
|
4911
|
+
}
|
|
4912
|
+
function readDirectoryOwner(lockPath) {
|
|
4913
|
+
return readJsonDirectory(lockPath, parseOwner);
|
|
4914
|
+
}
|
|
4915
|
+
function readReclaimOwner(markerPath) {
|
|
4916
|
+
return readJsonDirectory(markerPath, parseReclaimOwner);
|
|
4917
|
+
}
|
|
4918
|
+
function readRecoveryOwner(markerPath) {
|
|
4919
|
+
return readDirectoryOwner(markerPath);
|
|
4920
|
+
}
|
|
4921
|
+
function readLegacyOwner(lockPath) {
|
|
4922
|
+
try {
|
|
4923
|
+
const parsed = JSON.parse(readFileSync6(lockPath, "utf-8"));
|
|
4924
|
+
if (!Number.isInteger(parsed.pid) || Number(parsed.pid) <= 0) return null;
|
|
4925
|
+
if (typeof parsed.startedAt !== "string" || Number.isNaN(Date.parse(parsed.startedAt))) return null;
|
|
4926
|
+
return {
|
|
4927
|
+
pid: Number(parsed.pid),
|
|
4928
|
+
hostname: typeof parsed.hostname === "string" ? parsed.hostname : os4.hostname(),
|
|
4929
|
+
startedAt: parsed.startedAt,
|
|
4930
|
+
operation: typeof parsed.operation === "string" && VALID_OPERATIONS.has(parsed.operation) ? parsed.operation : "index",
|
|
4931
|
+
token: typeof parsed.token === "string" ? parsed.token : "legacy-v0.14.0"
|
|
4932
|
+
};
|
|
4933
|
+
} catch {
|
|
4934
|
+
return null;
|
|
4935
|
+
}
|
|
4936
|
+
}
|
|
4937
|
+
function getOwnerLiveness(owner) {
|
|
4938
|
+
if (owner.hostname !== os4.hostname()) return "unknown";
|
|
4939
|
+
try {
|
|
4940
|
+
process.kill(owner.pid, 0);
|
|
4941
|
+
return "alive";
|
|
4942
|
+
} catch (error) {
|
|
4943
|
+
const code = getErrorCode(error);
|
|
4944
|
+
if (code === "ESRCH") return "dead";
|
|
4945
|
+
if (code === "EPERM") return "alive";
|
|
4946
|
+
return "unknown";
|
|
4947
|
+
}
|
|
4948
|
+
}
|
|
4949
|
+
function sameOwner(left, right) {
|
|
4950
|
+
return left.pid === right.pid && left.hostname === right.hostname && left.token === right.token;
|
|
4951
|
+
}
|
|
4952
|
+
function sameReclaimOwner(left, right) {
|
|
4953
|
+
return left.pid === right.pid && left.hostname === right.hostname && left.token === right.token && left.expectedOwnerToken === right.expectedOwnerToken;
|
|
4954
|
+
}
|
|
4955
|
+
function publishJsonDirectory(finalPath, value) {
|
|
4956
|
+
const candidatePath = `${finalPath}.candidate.${process.pid}.${randomUUID()}`;
|
|
4957
|
+
mkdirSync2(candidatePath, { mode: 448 });
|
|
4958
|
+
try {
|
|
4959
|
+
writeFileSync2(path12.join(candidatePath, OWNER_FILE_NAME), JSON.stringify(value), {
|
|
4960
|
+
encoding: "utf-8",
|
|
4961
|
+
flag: "wx",
|
|
4962
|
+
mode: 384
|
|
4963
|
+
});
|
|
4964
|
+
if (existsSync7(finalPath)) return false;
|
|
4965
|
+
try {
|
|
4966
|
+
renameSync(candidatePath, finalPath);
|
|
4967
|
+
return true;
|
|
4968
|
+
} catch (error) {
|
|
4969
|
+
if (existsSync7(finalPath)) return false;
|
|
4970
|
+
throw error;
|
|
4971
|
+
}
|
|
4972
|
+
} finally {
|
|
4973
|
+
if (existsSync7(candidatePath)) rmSync(candidatePath, { recursive: true, force: true });
|
|
4974
|
+
}
|
|
4975
|
+
}
|
|
4976
|
+
function createOwner(operation) {
|
|
4977
|
+
return {
|
|
4978
|
+
pid: process.pid,
|
|
4979
|
+
hostname: os4.hostname(),
|
|
4980
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4981
|
+
operation,
|
|
4982
|
+
token: randomUUID()
|
|
4983
|
+
};
|
|
4984
|
+
}
|
|
4985
|
+
function recoveryMarkerPath(indexPath, owner) {
|
|
4986
|
+
return path12.join(indexPath, `${RECOVERY_MARKER_PREFIX}${owner.token}`);
|
|
4987
|
+
}
|
|
4988
|
+
function publishRecoveryMarker(indexPath, owner) {
|
|
4989
|
+
const markerPath = recoveryMarkerPath(indexPath, owner);
|
|
4990
|
+
if (!publishJsonDirectory(markerPath, owner)) {
|
|
4991
|
+
const markerOwner = readRecoveryOwner(markerPath);
|
|
4992
|
+
if (!markerOwner || !sameOwner(markerOwner, owner)) {
|
|
4993
|
+
throw new IndexLockContentionError(markerPath, markerOwner, "unknown-owner");
|
|
4994
|
+
}
|
|
4995
|
+
}
|
|
4996
|
+
return markerPath;
|
|
4997
|
+
}
|
|
4998
|
+
function getPendingRecoveries(indexPath) {
|
|
4999
|
+
const recoveries = [];
|
|
5000
|
+
const markerNames = readdirSync2(indexPath).filter((name) => {
|
|
5001
|
+
if (!name.startsWith(RECOVERY_MARKER_PREFIX)) return false;
|
|
5002
|
+
return UUID_PATTERN.test(name.slice(RECOVERY_MARKER_PREFIX.length));
|
|
5003
|
+
}).sort();
|
|
5004
|
+
for (const markerName of markerNames) {
|
|
5005
|
+
const markerPath = path12.join(indexPath, markerName);
|
|
5006
|
+
const markerToken = markerName.slice(RECOVERY_MARKER_PREFIX.length);
|
|
5007
|
+
let markerStats;
|
|
5008
|
+
try {
|
|
5009
|
+
markerStats = lstatSync(markerPath);
|
|
5010
|
+
} catch (error) {
|
|
5011
|
+
if (getErrorCode(error) === "ENOENT") continue;
|
|
5012
|
+
throw error;
|
|
5013
|
+
}
|
|
5014
|
+
if (!markerStats.isDirectory()) {
|
|
5015
|
+
throw new IndexLockContentionError(markerPath, null, "unknown-owner");
|
|
5016
|
+
}
|
|
5017
|
+
const owner = readRecoveryOwner(markerPath);
|
|
5018
|
+
if (!owner || owner.token !== markerToken || getOwnerLiveness(owner) !== "dead") {
|
|
5019
|
+
throw new IndexLockContentionError(markerPath, owner, "unknown-owner");
|
|
5020
|
+
}
|
|
5021
|
+
recoveries.push({ owner, markerPath });
|
|
5022
|
+
}
|
|
5023
|
+
recoveries.sort((left, right) => {
|
|
5024
|
+
const byTimestamp = left.owner.startedAt.localeCompare(right.owner.startedAt);
|
|
5025
|
+
return byTimestamp !== 0 ? byTimestamp : left.markerPath.localeCompare(right.markerPath);
|
|
5026
|
+
});
|
|
5027
|
+
return recoveries;
|
|
5028
|
+
}
|
|
5029
|
+
function cleanupDeadPublicationCandidates(indexPath) {
|
|
5030
|
+
const candidatePattern = /^indexing\.lock(?:\.recovery\.[0-9a-f-]{36})?\.candidate\.(\d+)\.[0-9a-f-]{36}$/i;
|
|
5031
|
+
for (const entry of readdirSync2(indexPath)) {
|
|
5032
|
+
const match = candidatePattern.exec(entry);
|
|
5033
|
+
if (!match) continue;
|
|
5034
|
+
const pid = Number(match[1]);
|
|
5035
|
+
if (!Number.isInteger(pid) || pid <= 0) continue;
|
|
5036
|
+
if (getOwnerLiveness({ pid, hostname: os4.hostname() }) !== "dead") continue;
|
|
5037
|
+
rmSync(path12.join(indexPath, entry), { recursive: true, force: true });
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
function removeDeadReclaimMarker(lockPath, expectedOwner) {
|
|
5041
|
+
const markerPath = path12.join(lockPath, RECLAIM_DIRECTORY_NAME);
|
|
5042
|
+
const marker = readReclaimOwner(markerPath);
|
|
5043
|
+
if (!marker || marker.expectedOwnerToken !== expectedOwner.token || getOwnerLiveness(marker) !== "dead") {
|
|
5044
|
+
return false;
|
|
5045
|
+
}
|
|
5046
|
+
const currentOwner = readDirectoryOwner(lockPath);
|
|
5047
|
+
if (!currentOwner || !sameOwner(currentOwner, expectedOwner) || getOwnerLiveness(currentOwner) !== "dead") {
|
|
5048
|
+
return false;
|
|
5049
|
+
}
|
|
5050
|
+
const claimedMarkerPath = `${markerPath}.stale.${marker.pid}.${marker.token}.${randomUUID()}`;
|
|
5051
|
+
try {
|
|
5052
|
+
renameSync(markerPath, claimedMarkerPath);
|
|
5053
|
+
} catch (error) {
|
|
5054
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5055
|
+
throw error;
|
|
5056
|
+
}
|
|
5057
|
+
const claimedMarker = readReclaimOwner(claimedMarkerPath);
|
|
5058
|
+
const ownerAfterClaim = readDirectoryOwner(lockPath);
|
|
5059
|
+
if (!claimedMarker || !sameReclaimOwner(claimedMarker, marker) || !ownerAfterClaim || !sameOwner(ownerAfterClaim, expectedOwner) || getOwnerLiveness(ownerAfterClaim) !== "dead") {
|
|
5060
|
+
if (!existsSync7(markerPath) && existsSync7(claimedMarkerPath)) {
|
|
5061
|
+
renameSync(claimedMarkerPath, markerPath);
|
|
5062
|
+
}
|
|
5063
|
+
return false;
|
|
5064
|
+
}
|
|
5065
|
+
rmSync(claimedMarkerPath, { recursive: true, force: true });
|
|
5066
|
+
return true;
|
|
5067
|
+
}
|
|
5068
|
+
function reclaimDeadOwner(indexPath, lockPath, expectedOwner) {
|
|
5069
|
+
const reclaimPath = path12.join(lockPath, RECLAIM_DIRECTORY_NAME);
|
|
5070
|
+
const reclaimOwner = {
|
|
5071
|
+
pid: process.pid,
|
|
5072
|
+
hostname: os4.hostname(),
|
|
5073
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5074
|
+
token: randomUUID(),
|
|
5075
|
+
expectedOwnerToken: expectedOwner.token
|
|
5076
|
+
};
|
|
5077
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
5078
|
+
if (publishJsonDirectory(reclaimPath, reclaimOwner)) break;
|
|
5079
|
+
if (attempt === 0 && removeDeadReclaimMarker(lockPath, expectedOwner)) continue;
|
|
5080
|
+
return false;
|
|
5081
|
+
}
|
|
5082
|
+
try {
|
|
5083
|
+
const currentReclaimer = readReclaimOwner(reclaimPath);
|
|
5084
|
+
const currentOwner = readDirectoryOwner(lockPath);
|
|
5085
|
+
if (!currentReclaimer || !sameReclaimOwner(currentReclaimer, reclaimOwner) || !currentOwner || !sameOwner(currentOwner, expectedOwner) || getOwnerLiveness(currentOwner) !== "dead") {
|
|
5086
|
+
return false;
|
|
5087
|
+
}
|
|
5088
|
+
publishRecoveryMarker(indexPath, expectedOwner);
|
|
5089
|
+
const ownerBeforeQuarantine = readDirectoryOwner(lockPath);
|
|
5090
|
+
const reclaimerBeforeQuarantine = readReclaimOwner(reclaimPath);
|
|
5091
|
+
if (!ownerBeforeQuarantine || !sameOwner(ownerBeforeQuarantine, expectedOwner) || getOwnerLiveness(ownerBeforeQuarantine) !== "dead" || !reclaimerBeforeQuarantine || !sameReclaimOwner(reclaimerBeforeQuarantine, reclaimOwner)) {
|
|
5092
|
+
return false;
|
|
5093
|
+
}
|
|
5094
|
+
const quarantinePath = `${lockPath}.stale.${expectedOwner.token}.${reclaimOwner.token}`;
|
|
5095
|
+
renameSync(lockPath, quarantinePath);
|
|
5096
|
+
rmSync(quarantinePath, { recursive: true, force: true });
|
|
5097
|
+
return true;
|
|
5098
|
+
} catch (error) {
|
|
5099
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5100
|
+
throw error;
|
|
5101
|
+
}
|
|
5102
|
+
}
|
|
5103
|
+
var IndexLockContentionError = class extends Error {
|
|
5104
|
+
constructor(lockPath, owner, reason) {
|
|
5105
|
+
const ownerDescription = owner ? `PID ${owner.pid} on ${owner.hostname}, operation ${owner.operation}, since ${owner.startedAt}` : "an unreadable owner";
|
|
5106
|
+
super(`Index mutation already in progress: ${ownerDescription}`);
|
|
5107
|
+
this.lockPath = lockPath;
|
|
5108
|
+
this.owner = owner;
|
|
5109
|
+
this.reason = reason;
|
|
5110
|
+
this.name = "IndexLockContentionError";
|
|
5111
|
+
}
|
|
5112
|
+
lockPath;
|
|
5113
|
+
owner;
|
|
5114
|
+
reason;
|
|
5115
|
+
code = "INDEX_BUSY";
|
|
5116
|
+
};
|
|
5117
|
+
function isIndexLockContentionError(error) {
|
|
5118
|
+
return error instanceof IndexLockContentionError || typeof error === "object" && error !== null && "code" in error && error.code === "INDEX_BUSY";
|
|
5119
|
+
}
|
|
5120
|
+
function acquireIndexLock(indexPath, operation) {
|
|
5121
|
+
mkdirSync2(indexPath, { recursive: true });
|
|
5122
|
+
const canonicalIndexPath = realpathSync.native(indexPath);
|
|
5123
|
+
const lockPath = path12.join(canonicalIndexPath, "indexing.lock");
|
|
5124
|
+
cleanupDeadPublicationCandidates(canonicalIndexPath);
|
|
5125
|
+
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
5126
|
+
const owner = createOwner(operation);
|
|
5127
|
+
if (publishJsonDirectory(lockPath, owner)) {
|
|
5128
|
+
const lease = {
|
|
5129
|
+
canonicalIndexPath,
|
|
5130
|
+
lockPath,
|
|
5131
|
+
owner,
|
|
5132
|
+
recoveries: []
|
|
5133
|
+
};
|
|
5134
|
+
try {
|
|
5135
|
+
lease.recoveries = getPendingRecoveries(canonicalIndexPath);
|
|
5136
|
+
return lease;
|
|
5137
|
+
} catch (error) {
|
|
5138
|
+
releaseIndexLock(lease);
|
|
5139
|
+
throw error;
|
|
5140
|
+
}
|
|
5141
|
+
}
|
|
5142
|
+
let stats;
|
|
5143
|
+
try {
|
|
5144
|
+
stats = lstatSync(lockPath);
|
|
5145
|
+
} catch (error) {
|
|
5146
|
+
if (getErrorCode(error) === "ENOENT") continue;
|
|
5147
|
+
throw error;
|
|
5148
|
+
}
|
|
5149
|
+
if (!stats.isDirectory()) {
|
|
5150
|
+
const legacyOwner = readLegacyOwner(lockPath);
|
|
5151
|
+
throw new IndexLockContentionError(lockPath, legacyOwner, "legacy-lock");
|
|
5152
|
+
}
|
|
5153
|
+
const existingOwner = readDirectoryOwner(lockPath);
|
|
5154
|
+
if (!existingOwner) {
|
|
5155
|
+
throw new IndexLockContentionError(lockPath, null, "unknown-owner");
|
|
5156
|
+
}
|
|
5157
|
+
const liveness = getOwnerLiveness(existingOwner);
|
|
5158
|
+
if (liveness !== "dead") {
|
|
5159
|
+
throw new IndexLockContentionError(lockPath, existingOwner, liveness === "alive" ? "active" : "unknown-owner");
|
|
5160
|
+
}
|
|
5161
|
+
if (!reclaimDeadOwner(canonicalIndexPath, lockPath, existingOwner)) {
|
|
5162
|
+
throw new IndexLockContentionError(lockPath, existingOwner, "reclaiming");
|
|
5163
|
+
}
|
|
5164
|
+
}
|
|
5165
|
+
throw new IndexLockContentionError(lockPath, null, "reclaiming");
|
|
5166
|
+
}
|
|
5167
|
+
function releaseIndexLock(lease) {
|
|
5168
|
+
const currentOwner = readDirectoryOwner(lease.lockPath);
|
|
5169
|
+
if (!currentOwner || !sameOwner(currentOwner, lease.owner)) return false;
|
|
5170
|
+
const releasePath = `${lease.lockPath}.release.${lease.owner.pid}.${lease.owner.token}`;
|
|
5171
|
+
try {
|
|
5172
|
+
retryTransientFilesystemOperation(() => renameSync(lease.lockPath, releasePath));
|
|
5173
|
+
} catch (error) {
|
|
5174
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5175
|
+
throw error;
|
|
5176
|
+
}
|
|
5177
|
+
const claimedOwner = readDirectoryOwner(releasePath);
|
|
5178
|
+
if (!claimedOwner || !sameOwner(claimedOwner, lease.owner)) {
|
|
5179
|
+
if (!existsSync7(lease.lockPath) && existsSync7(releasePath)) {
|
|
5180
|
+
renameSync(releasePath, lease.lockPath);
|
|
5181
|
+
}
|
|
5182
|
+
return false;
|
|
5183
|
+
}
|
|
5184
|
+
try {
|
|
5185
|
+
retryTransientFilesystemOperation(() => rmSync(releasePath, { recursive: true, force: true }));
|
|
5186
|
+
} catch (error) {
|
|
5187
|
+
console.error(`[codebase-index] Lease released but tombstone cleanup failed: ${releasePath}`, error);
|
|
5188
|
+
}
|
|
5189
|
+
return true;
|
|
5190
|
+
}
|
|
5191
|
+
async function withIndexLock(indexPath, operation, callback, options = {}) {
|
|
5192
|
+
const lease = acquireIndexLock(indexPath, operation);
|
|
5193
|
+
let result;
|
|
5194
|
+
let callbackError;
|
|
5195
|
+
let callbackFailed = false;
|
|
5196
|
+
try {
|
|
5197
|
+
result = await callback(lease);
|
|
5198
|
+
} catch (error) {
|
|
5199
|
+
callbackFailed = true;
|
|
5200
|
+
callbackError = error;
|
|
5201
|
+
}
|
|
5202
|
+
if (!callbackFailed && options.completeRecoveries !== false) {
|
|
5203
|
+
try {
|
|
5204
|
+
completeLeaseRecovery(lease);
|
|
5205
|
+
} catch (error) {
|
|
5206
|
+
callbackFailed = true;
|
|
5207
|
+
callbackError = error;
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5210
|
+
let releaseError;
|
|
5211
|
+
try {
|
|
5212
|
+
if (!releaseIndexLock(lease)) {
|
|
5213
|
+
releaseError = new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
5214
|
+
}
|
|
5215
|
+
} catch (error) {
|
|
5216
|
+
releaseError = error;
|
|
5217
|
+
}
|
|
5218
|
+
if (releaseError !== void 0) {
|
|
5219
|
+
if (callbackFailed) throw new AggregateError([callbackError, releaseError], "Index mutation and lease release both failed");
|
|
5220
|
+
throw releaseError;
|
|
5221
|
+
}
|
|
5222
|
+
if (callbackFailed) throw callbackError;
|
|
5223
|
+
return result;
|
|
5224
|
+
}
|
|
5225
|
+
function createLeaseTemporaryPath(targetPath, owner, kind = "tmp") {
|
|
5226
|
+
if (kind === "bak") return `${targetPath}.bak.${owner.pid}.${owner.token}`;
|
|
5227
|
+
temporaryCounter += 1;
|
|
5228
|
+
return `${targetPath}.tmp.${owner.pid}.${owner.token}.${temporaryCounter}`;
|
|
5229
|
+
}
|
|
5230
|
+
function removeLeaseTemporaryPath(temporaryPath) {
|
|
5231
|
+
if (existsSync7(temporaryPath)) rmSync(temporaryPath, { recursive: true, force: true });
|
|
5232
|
+
}
|
|
5233
|
+
function recoverLeaseArtifacts(indexPath, owner, backupTargets) {
|
|
5234
|
+
for (const targetPath of backupTargets) {
|
|
5235
|
+
const backupPath = createLeaseTemporaryPath(targetPath, owner, "bak");
|
|
5236
|
+
if (!existsSync7(backupPath)) continue;
|
|
5237
|
+
if (existsSync7(targetPath)) rmSync(targetPath, { recursive: true, force: true });
|
|
5238
|
+
renameSync(backupPath, targetPath);
|
|
5239
|
+
}
|
|
5240
|
+
const temporaryOwnerMarker = `.tmp.${owner.pid}.${owner.token}.`;
|
|
5241
|
+
for (const entry of readdirSync2(indexPath)) {
|
|
5242
|
+
if (!entry.includes(temporaryOwnerMarker)) continue;
|
|
5243
|
+
rmSync(path12.join(indexPath, entry), { recursive: true, force: true });
|
|
5244
|
+
}
|
|
5245
|
+
}
|
|
5246
|
+
function completeLeaseRecovery(lease) {
|
|
5247
|
+
for (const recovery of lease.recoveries) {
|
|
5248
|
+
const markerOwner = readRecoveryOwner(recovery.markerPath);
|
|
5249
|
+
if (!markerOwner || !sameOwner(markerOwner, recovery.owner)) {
|
|
5250
|
+
throw new Error(`Recovery marker ownership changed: ${recovery.markerPath}`);
|
|
5251
|
+
}
|
|
5252
|
+
}
|
|
5253
|
+
for (const recovery of lease.recoveries) {
|
|
5254
|
+
rmSync(recovery.markerPath, { recursive: true, force: true });
|
|
5255
|
+
}
|
|
5256
|
+
}
|
|
5257
|
+
|
|
4806
5258
|
// src/indexer/index.ts
|
|
4807
5259
|
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "php", "apex", "zig", "gdscript", "matlab", "bash"]);
|
|
4808
5260
|
var CASE_INSENSITIVE_LANGUAGES = /* @__PURE__ */ new Set(["apex"]);
|
|
@@ -4884,6 +5336,7 @@ function isSqliteCorruptionError(error) {
|
|
|
4884
5336
|
return message.includes("database disk image is malformed") || message.includes("file is not a database") || message.includes("database schema is corrupt") || message.includes("sqlite_corrupt");
|
|
4885
5337
|
}
|
|
4886
5338
|
var STARTUP_WARNING_METADATA_KEY = "index.startupWarning";
|
|
5339
|
+
var READER_ARTIFACT_RETRY_INTERVAL_MS = 1e3;
|
|
4887
5340
|
function metadataFromBlame(blame) {
|
|
4888
5341
|
if (!blame) {
|
|
4889
5342
|
return {};
|
|
@@ -5081,9 +5534,9 @@ function hasAllEmbeddingParts(parts, expectedPartCount) {
|
|
|
5081
5534
|
return true;
|
|
5082
5535
|
}
|
|
5083
5536
|
function isPathWithinRoot(filePath, rootPath) {
|
|
5084
|
-
const normalizedFilePath =
|
|
5085
|
-
const normalizedRoot =
|
|
5086
|
-
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${
|
|
5537
|
+
const normalizedFilePath = path13.resolve(filePath);
|
|
5538
|
+
const normalizedRoot = path13.resolve(rootPath);
|
|
5539
|
+
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${path13.sep}`);
|
|
5087
5540
|
}
|
|
5088
5541
|
var rankingQueryTokenCache = /* @__PURE__ */ new Map();
|
|
5089
5542
|
var rankingNameTokenCache = /* @__PURE__ */ new Map();
|
|
@@ -6149,26 +6602,133 @@ var Indexer = class {
|
|
|
6149
6602
|
queryCacheTtlMs = 5 * 60 * 1e3;
|
|
6150
6603
|
querySimilarityThreshold = 0.85;
|
|
6151
6604
|
indexCompatibility = null;
|
|
6152
|
-
|
|
6605
|
+
activeIndexLease = null;
|
|
6606
|
+
initializationPromise = null;
|
|
6607
|
+
initializationMode = "none";
|
|
6608
|
+
readIssues = [];
|
|
6609
|
+
retiredDatabases = [];
|
|
6610
|
+
readerArtifactFingerprint = null;
|
|
6611
|
+
writerArtifactFingerprint = null;
|
|
6612
|
+
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
6153
6613
|
constructor(projectRoot3, config, host = "opencode") {
|
|
6154
6614
|
this.projectRoot = projectRoot3;
|
|
6155
6615
|
this.config = config;
|
|
6156
6616
|
this.host = host;
|
|
6157
6617
|
this.indexPath = this.getIndexPath();
|
|
6158
|
-
this.fileHashCachePath =
|
|
6159
|
-
this.failedBatchesPath =
|
|
6160
|
-
this.indexingLockPath = path12.join(this.indexPath, "indexing.lock");
|
|
6618
|
+
this.fileHashCachePath = path13.join(this.indexPath, "file-hashes.json");
|
|
6619
|
+
this.failedBatchesPath = path13.join(this.indexPath, "failed-batches.json");
|
|
6161
6620
|
this.logger = initializeLogger(config.debug);
|
|
6162
6621
|
}
|
|
6163
6622
|
getIndexPath() {
|
|
6164
6623
|
return resolveProjectIndexPath(this.projectRoot, this.config.scope, this.host);
|
|
6165
6624
|
}
|
|
6625
|
+
isLocalProjectIndexPath() {
|
|
6626
|
+
const localProjectIndexPaths = [path13.join(this.projectRoot, getHostProjectIndexRelativePath(this.host))];
|
|
6627
|
+
if (this.host !== "opencode") {
|
|
6628
|
+
localProjectIndexPaths.push(path13.join(this.projectRoot, getHostProjectIndexRelativePath("opencode")));
|
|
6629
|
+
}
|
|
6630
|
+
return localProjectIndexPaths.some((localPath) => {
|
|
6631
|
+
if (!existsSync8(localPath) || !existsSync8(this.indexPath)) {
|
|
6632
|
+
return path13.resolve(this.indexPath) === path13.resolve(localPath);
|
|
6633
|
+
}
|
|
6634
|
+
const indexStats = statSync3(this.indexPath);
|
|
6635
|
+
const localStats = statSync3(localPath);
|
|
6636
|
+
return indexStats.dev === localStats.dev && indexStats.ino === localStats.ino;
|
|
6637
|
+
});
|
|
6638
|
+
}
|
|
6639
|
+
resetLoadedIndexState(retireDatabase = false) {
|
|
6640
|
+
if (this.database) {
|
|
6641
|
+
if (retireDatabase) {
|
|
6642
|
+
this.retiredDatabases.push(this.database);
|
|
6643
|
+
} else {
|
|
6644
|
+
this.database.close();
|
|
6645
|
+
}
|
|
6646
|
+
}
|
|
6647
|
+
this.store = null;
|
|
6648
|
+
this.invertedIndex = null;
|
|
6649
|
+
this.database = null;
|
|
6650
|
+
this.provider = null;
|
|
6651
|
+
this.configuredProviderInfo = null;
|
|
6652
|
+
this.reranker = null;
|
|
6653
|
+
this.indexCompatibility = null;
|
|
6654
|
+
this.initializationMode = "none";
|
|
6655
|
+
this.readIssues = [];
|
|
6656
|
+
this.readerArtifactFingerprint = null;
|
|
6657
|
+
this.writerArtifactFingerprint = null;
|
|
6658
|
+
this.readerArtifactRetryAfter.clear();
|
|
6659
|
+
this.fileHashCache.clear();
|
|
6660
|
+
}
|
|
6661
|
+
refreshLoadedIndexState() {
|
|
6662
|
+
if (!this.store || !this.invertedIndex || !this.configuredProviderInfo) return;
|
|
6663
|
+
this.store.load();
|
|
6664
|
+
this.invertedIndex.load();
|
|
6665
|
+
this.fileHashCache.clear();
|
|
6666
|
+
this.loadFileHashCache();
|
|
6667
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
6668
|
+
this.readIssues = [];
|
|
6669
|
+
this.readerArtifactRetryAfter.clear();
|
|
6670
|
+
}
|
|
6671
|
+
async withIndexMutationLease(operation, callback) {
|
|
6672
|
+
const lease = acquireIndexLock(this.indexPath, operation);
|
|
6673
|
+
this.indexPath = lease.canonicalIndexPath;
|
|
6674
|
+
this.fileHashCachePath = path13.join(this.indexPath, "file-hashes.json");
|
|
6675
|
+
this.failedBatchesPath = path13.join(this.indexPath, "failed-batches.json");
|
|
6676
|
+
this.activeIndexLease = lease;
|
|
6677
|
+
let result;
|
|
6678
|
+
let callbackError;
|
|
6679
|
+
let callbackFailed = false;
|
|
6680
|
+
try {
|
|
6681
|
+
result = await callback(lease.recoveries.map(({ owner }) => owner));
|
|
6682
|
+
} catch (error) {
|
|
6683
|
+
callbackFailed = true;
|
|
6684
|
+
callbackError = error;
|
|
6685
|
+
}
|
|
6686
|
+
if (!callbackFailed) {
|
|
6687
|
+
try {
|
|
6688
|
+
completeLeaseRecovery(lease);
|
|
6689
|
+
this.writerArtifactFingerprint = this.captureReaderArtifactFingerprint();
|
|
6690
|
+
} catch (error) {
|
|
6691
|
+
callbackFailed = true;
|
|
6692
|
+
callbackError = error;
|
|
6693
|
+
}
|
|
6694
|
+
}
|
|
6695
|
+
let releaseError;
|
|
6696
|
+
try {
|
|
6697
|
+
if (!releaseIndexLock(lease)) {
|
|
6698
|
+
releaseError = new Error(`Lost ownership of index mutation lease ${lease.owner.token}`);
|
|
6699
|
+
this.writerArtifactFingerprint = null;
|
|
6700
|
+
if (this.activeIndexLease?.owner.token === lease.owner.token) {
|
|
6701
|
+
this.activeIndexLease = null;
|
|
6702
|
+
}
|
|
6703
|
+
} else if (this.activeIndexLease?.owner.token === lease.owner.token) {
|
|
6704
|
+
this.activeIndexLease = null;
|
|
6705
|
+
}
|
|
6706
|
+
} catch (error) {
|
|
6707
|
+
releaseError = error;
|
|
6708
|
+
this.writerArtifactFingerprint = null;
|
|
6709
|
+
if (!existsSync8(lease.lockPath) && this.activeIndexLease?.owner.token === lease.owner.token) {
|
|
6710
|
+
this.activeIndexLease = null;
|
|
6711
|
+
}
|
|
6712
|
+
}
|
|
6713
|
+
if (releaseError !== void 0) {
|
|
6714
|
+
if (callbackFailed) throw new AggregateError([callbackError, releaseError], "Index mutation and lease release both failed");
|
|
6715
|
+
throw releaseError;
|
|
6716
|
+
}
|
|
6717
|
+
if (callbackFailed) throw callbackError;
|
|
6718
|
+
return result;
|
|
6719
|
+
}
|
|
6720
|
+
requireActiveLease() {
|
|
6721
|
+
if (!this.activeIndexLease) {
|
|
6722
|
+
throw new Error("Index mutation attempted without an active interprocess lease");
|
|
6723
|
+
}
|
|
6724
|
+
return this.activeIndexLease;
|
|
6725
|
+
}
|
|
6166
6726
|
loadFileHashCache() {
|
|
6167
|
-
if (!
|
|
6727
|
+
if (!existsSync8(this.fileHashCachePath)) {
|
|
6168
6728
|
return;
|
|
6169
6729
|
}
|
|
6170
6730
|
try {
|
|
6171
|
-
const data =
|
|
6731
|
+
const data = readFileSync7(this.fileHashCachePath, "utf-8");
|
|
6172
6732
|
const parsed = JSON.parse(data);
|
|
6173
6733
|
this.fileHashCache = new Map(Object.entries(parsed));
|
|
6174
6734
|
} catch (error) {
|
|
@@ -6188,15 +6748,26 @@ var Indexer = class {
|
|
|
6188
6748
|
this.atomicWriteSync(this.fileHashCachePath, JSON.stringify(obj));
|
|
6189
6749
|
}
|
|
6190
6750
|
atomicWriteSync(targetPath, data) {
|
|
6191
|
-
const
|
|
6192
|
-
|
|
6193
|
-
|
|
6194
|
-
|
|
6751
|
+
const lease = this.requireActiveLease();
|
|
6752
|
+
const tempPath = createLeaseTemporaryPath(targetPath, lease.owner, "tmp");
|
|
6753
|
+
mkdirSync3(path13.dirname(targetPath), { recursive: true });
|
|
6754
|
+
try {
|
|
6755
|
+
writeFileSync3(tempPath, data);
|
|
6756
|
+
renameSync2(tempPath, targetPath);
|
|
6757
|
+
} finally {
|
|
6758
|
+
removeLeaseTemporaryPath(tempPath);
|
|
6759
|
+
}
|
|
6760
|
+
}
|
|
6761
|
+
saveInvertedIndex(invertedIndex) {
|
|
6762
|
+
this.atomicWriteSync(
|
|
6763
|
+
path13.join(this.indexPath, "inverted-index.json"),
|
|
6764
|
+
invertedIndex.serialize()
|
|
6765
|
+
);
|
|
6195
6766
|
}
|
|
6196
6767
|
getScopedRoots() {
|
|
6197
|
-
const roots = /* @__PURE__ */ new Set([
|
|
6768
|
+
const roots = /* @__PURE__ */ new Set([path13.resolve(this.projectRoot)]);
|
|
6198
6769
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
6199
|
-
roots.add(
|
|
6770
|
+
roots.add(path13.resolve(this.projectRoot, kbRoot));
|
|
6200
6771
|
}
|
|
6201
6772
|
return Array.from(roots);
|
|
6202
6773
|
}
|
|
@@ -6205,29 +6776,29 @@ var Indexer = class {
|
|
|
6205
6776
|
if (this.config.scope !== "global") {
|
|
6206
6777
|
return branchName;
|
|
6207
6778
|
}
|
|
6208
|
-
const projectHash = hashContent(
|
|
6779
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6209
6780
|
return `${projectHash}:${branchName}`;
|
|
6210
6781
|
}
|
|
6211
6782
|
getBranchCatalogKeyFor(branchName) {
|
|
6212
6783
|
if (this.config.scope !== "global") {
|
|
6213
6784
|
return branchName;
|
|
6214
6785
|
}
|
|
6215
|
-
const projectHash = hashContent(
|
|
6786
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6216
6787
|
return `${projectHash}:${branchName}`;
|
|
6217
6788
|
}
|
|
6218
6789
|
getLegacyBranchCatalogKey() {
|
|
6219
6790
|
return this.currentBranch || "default";
|
|
6220
6791
|
}
|
|
6221
6792
|
getLegacyMigrationMetadataKey() {
|
|
6222
|
-
const projectHash = hashContent(
|
|
6793
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6223
6794
|
return `index.globalBranchMigration.${projectHash}`;
|
|
6224
6795
|
}
|
|
6225
6796
|
getProjectEmbeddingStrategyMetadataKey() {
|
|
6226
|
-
const projectHash = hashContent(
|
|
6797
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6227
6798
|
return `index.embeddingStrategyVersion.${projectHash}`;
|
|
6228
6799
|
}
|
|
6229
6800
|
getProjectForceReembedMetadataKey() {
|
|
6230
|
-
const projectHash = hashContent(
|
|
6801
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6231
6802
|
return `index.forceReembed.${projectHash}`;
|
|
6232
6803
|
}
|
|
6233
6804
|
hasProjectForceReembedPending() {
|
|
@@ -6321,7 +6892,7 @@ var Indexer = class {
|
|
|
6321
6892
|
if (!this.database) {
|
|
6322
6893
|
return { chunkIds, symbolIds };
|
|
6323
6894
|
}
|
|
6324
|
-
const projectRootPath =
|
|
6895
|
+
const projectRootPath = path13.resolve(this.projectRoot);
|
|
6325
6896
|
const projectLocalFilePaths = /* @__PURE__ */ new Set([
|
|
6326
6897
|
...Array.from(this.fileHashCache.keys()).filter(
|
|
6327
6898
|
(filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath)
|
|
@@ -6344,7 +6915,7 @@ var Indexer = class {
|
|
|
6344
6915
|
if (this.config.scope !== "global") {
|
|
6345
6916
|
return this.getBranchCatalogCleanupKeys();
|
|
6346
6917
|
}
|
|
6347
|
-
const projectHash = hashContent(
|
|
6918
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6348
6919
|
const keys = /* @__PURE__ */ new Set();
|
|
6349
6920
|
const projectChunkIdSet = new Set(projectChunkIds);
|
|
6350
6921
|
const projectSymbolIdSet = new Set(projectSymbolIds);
|
|
@@ -6428,7 +6999,7 @@ var Indexer = class {
|
|
|
6428
6999
|
if (!this.database || this.config.scope !== "global") {
|
|
6429
7000
|
return false;
|
|
6430
7001
|
}
|
|
6431
|
-
const projectHash = hashContent(
|
|
7002
|
+
const projectHash = hashContent(path13.resolve(this.projectRoot)).slice(0, 16);
|
|
6432
7003
|
const roots = this.getScopedRoots();
|
|
6433
7004
|
const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots);
|
|
6434
7005
|
return this.database.getAllBranches().some(
|
|
@@ -6462,7 +7033,7 @@ var Indexer = class {
|
|
|
6462
7033
|
...Array.from(this.fileHashCache.keys()).filter((filePath) => this.isFileInCurrentScope(filePath, roots)),
|
|
6463
7034
|
...scopedEntries.map(({ metadata }) => metadata.filePath)
|
|
6464
7035
|
]);
|
|
6465
|
-
const projectRootPath =
|
|
7036
|
+
const projectRootPath = path13.resolve(this.projectRoot);
|
|
6466
7037
|
const projectLocalFilePaths = new Set(
|
|
6467
7038
|
Array.from(filePaths).filter((filePath) => isPathWithinRoot(filePath, projectRootPath))
|
|
6468
7039
|
);
|
|
@@ -6524,34 +7095,27 @@ var Indexer = class {
|
|
|
6524
7095
|
database.gcOrphanEmbeddings();
|
|
6525
7096
|
database.gcOrphanChunks();
|
|
6526
7097
|
store.save();
|
|
6527
|
-
|
|
7098
|
+
this.saveInvertedIndex(invertedIndex);
|
|
6528
7099
|
return {
|
|
6529
7100
|
removedChunkIds: removedChunkIdList,
|
|
6530
7101
|
hasForeignData: allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots))
|
|
6531
7102
|
};
|
|
6532
7103
|
}
|
|
6533
|
-
|
|
6534
|
-
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6539
|
-
|
|
6540
|
-
|
|
6541
|
-
writeFileSync2(this.indexingLockPath, JSON.stringify(lockData));
|
|
6542
|
-
}
|
|
6543
|
-
releaseIndexingLock() {
|
|
6544
|
-
if (existsSync7(this.indexingLockPath)) {
|
|
6545
|
-
unlinkSync(this.indexingLockPath);
|
|
7104
|
+
async recoverFromInterruptedIndexingUnlocked(owners) {
|
|
7105
|
+
for (const owner of owners) {
|
|
7106
|
+
this.logger.warn("Detected interrupted indexing session, recovering...", {
|
|
7107
|
+
pid: owner.pid,
|
|
7108
|
+
hostname: owner.hostname,
|
|
7109
|
+
operation: owner.operation,
|
|
7110
|
+
startedAt: owner.startedAt
|
|
7111
|
+
});
|
|
6546
7112
|
}
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
7113
|
+
if (this.config.scope === "global") {
|
|
7114
|
+
if (existsSync8(this.fileHashCachePath)) {
|
|
7115
|
+
unlinkSync(this.fileHashCachePath);
|
|
7116
|
+
}
|
|
7117
|
+
await this.healthCheckUnlocked();
|
|
6552
7118
|
}
|
|
6553
|
-
await this.healthCheck();
|
|
6554
|
-
this.releaseIndexingLock();
|
|
6555
7119
|
this.logger.info("Recovery complete, next index will re-process all files");
|
|
6556
7120
|
}
|
|
6557
7121
|
loadFailedBatches(maxChunkTokens) {
|
|
@@ -6567,10 +7131,10 @@ var Indexer = class {
|
|
|
6567
7131
|
}
|
|
6568
7132
|
}
|
|
6569
7133
|
loadSerializedFailedBatches() {
|
|
6570
|
-
if (!
|
|
7134
|
+
if (!existsSync8(this.failedBatchesPath)) {
|
|
6571
7135
|
return [];
|
|
6572
7136
|
}
|
|
6573
|
-
const data =
|
|
7137
|
+
const data = readFileSync7(this.failedBatchesPath, "utf-8");
|
|
6574
7138
|
const parsed = JSON.parse(data);
|
|
6575
7139
|
return parsed.map((batch) => {
|
|
6576
7140
|
const chunks = Array.isArray(batch.chunks) ? batch.chunks : [];
|
|
@@ -6587,7 +7151,7 @@ var Indexer = class {
|
|
|
6587
7151
|
}
|
|
6588
7152
|
saveFailedBatches(batches) {
|
|
6589
7153
|
if (batches.length === 0) {
|
|
6590
|
-
if (
|
|
7154
|
+
if (existsSync8(this.failedBatchesPath)) {
|
|
6591
7155
|
try {
|
|
6592
7156
|
unlinkSync(this.failedBatchesPath);
|
|
6593
7157
|
} catch {
|
|
@@ -6595,7 +7159,7 @@ var Indexer = class {
|
|
|
6595
7159
|
}
|
|
6596
7160
|
return;
|
|
6597
7161
|
}
|
|
6598
|
-
|
|
7162
|
+
this.atomicWriteSync(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
6599
7163
|
}
|
|
6600
7164
|
collectRetryableFailedChunks(currentFileHashes, unchangedFilePaths, maxChunkTokens) {
|
|
6601
7165
|
const retryableById = /* @__PURE__ */ new Map();
|
|
@@ -6789,6 +7353,222 @@ var Indexer = class {
|
|
|
6789
7353
|
return parts.join("\n");
|
|
6790
7354
|
}
|
|
6791
7355
|
async initialize() {
|
|
7356
|
+
if (this.initializationPromise) {
|
|
7357
|
+
await this.initializationPromise;
|
|
7358
|
+
}
|
|
7359
|
+
if (this.isInitializedFor("reader")) {
|
|
7360
|
+
return;
|
|
7361
|
+
}
|
|
7362
|
+
await this.initializeOnce("reader", [], { skipAutoGc: true });
|
|
7363
|
+
}
|
|
7364
|
+
async initializeOnce(mode, recoveredOwners, options) {
|
|
7365
|
+
if (this.initializationPromise) {
|
|
7366
|
+
await this.initializationPromise;
|
|
7367
|
+
if (this.isInitializedFor(mode)) {
|
|
7368
|
+
return;
|
|
7369
|
+
}
|
|
7370
|
+
return this.initializeOnce(mode, recoveredOwners, options);
|
|
7371
|
+
}
|
|
7372
|
+
if (this.isInitializedFor(mode)) {
|
|
7373
|
+
return;
|
|
7374
|
+
}
|
|
7375
|
+
const initialization = this.initializeUnlocked(mode, recoveredOwners, options).catch((error) => {
|
|
7376
|
+
this.resetLoadedIndexState();
|
|
7377
|
+
throw error;
|
|
7378
|
+
}).finally(() => {
|
|
7379
|
+
if (this.initializationPromise === initialization) {
|
|
7380
|
+
this.initializationPromise = null;
|
|
7381
|
+
}
|
|
7382
|
+
});
|
|
7383
|
+
this.initializationPromise = initialization;
|
|
7384
|
+
await initialization;
|
|
7385
|
+
}
|
|
7386
|
+
isInitializedFor(mode) {
|
|
7387
|
+
const hasState = Boolean(
|
|
7388
|
+
this.store && this.provider && this.invertedIndex && this.configuredProviderInfo && this.database
|
|
7389
|
+
);
|
|
7390
|
+
if (!hasState) {
|
|
7391
|
+
return false;
|
|
7392
|
+
}
|
|
7393
|
+
return mode === "reader" ? this.initializationMode !== "none" : this.initializationMode === "writer";
|
|
7394
|
+
}
|
|
7395
|
+
recordReadIssue(component, message, error) {
|
|
7396
|
+
this.readIssues.push(this.createReadIssue(component, message));
|
|
7397
|
+
this.readerArtifactRetryAfter.set(component, Date.now() + READER_ARTIFACT_RETRY_INTERVAL_MS);
|
|
7398
|
+
this.logger.warn(message, error === void 0 ? void 0 : { error: getErrorMessage2(error) });
|
|
7399
|
+
}
|
|
7400
|
+
createReadIssue(component, message) {
|
|
7401
|
+
return {
|
|
7402
|
+
component,
|
|
7403
|
+
message,
|
|
7404
|
+
blocking: component !== "keyword"
|
|
7405
|
+
};
|
|
7406
|
+
}
|
|
7407
|
+
getVectorReadIssueMessage() {
|
|
7408
|
+
if (this.config.scope === "global") {
|
|
7409
|
+
return "Shared vector index could not be read. Restore or repair the complete fingerprinted shared vector artifacts; automatic reset is disabled for global scope.";
|
|
7410
|
+
}
|
|
7411
|
+
if (!this.isLocalProjectIndexPath()) {
|
|
7412
|
+
return "Vector index could not be read from an inherited project index. Restore or fingerprint it from the checkout that owns the index; do not remove or rebuild it from this worktree.";
|
|
7413
|
+
}
|
|
7414
|
+
return "Vector index could not be read. Run index_codebase after the active writer finishes to fingerprint a structurally valid legacy pair, or remove this checkout's local index directory and run index_codebase to rebuild it.";
|
|
7415
|
+
}
|
|
7416
|
+
getKeywordReadIssueMessage() {
|
|
7417
|
+
if (this.config.scope === "global") {
|
|
7418
|
+
return "Shared keyword index could not be read; semantic search remains available. Restore or repair the shared keyword artifact; automatic reset is disabled for global scope.";
|
|
7419
|
+
}
|
|
7420
|
+
if (!this.isLocalProjectIndexPath()) {
|
|
7421
|
+
return "Keyword index could not be read from an inherited project index; semantic search remains available. Restore or repair it from the checkout that owns the index; do not rebuild it from this worktree.";
|
|
7422
|
+
}
|
|
7423
|
+
return "Keyword index could not be read; semantic search remains available. Restore a readable published keyword index, or run index_codebase with force=true after the active writer finishes.";
|
|
7424
|
+
}
|
|
7425
|
+
getDatabaseReadIssueMessage() {
|
|
7426
|
+
if (this.config.scope === "global") {
|
|
7427
|
+
return "Shared index database could not be read. Restore or repair the shared SQLite database; automatic reset is disabled for global scope.";
|
|
7428
|
+
}
|
|
7429
|
+
if (!this.isLocalProjectIndexPath()) {
|
|
7430
|
+
return "Index database could not be read from an inherited project index. Restore or repair it from the checkout that owns the index; do not migrate or rebuild it from this worktree.";
|
|
7431
|
+
}
|
|
7432
|
+
return "Index database could not be read. Run index_codebase after the active writer finishes to repair or migrate it under the writer lease.";
|
|
7433
|
+
}
|
|
7434
|
+
getReaderFileFingerprint(filePath, identityOnly = false) {
|
|
7435
|
+
try {
|
|
7436
|
+
const stats = statSync3(filePath);
|
|
7437
|
+
if (identityOnly) {
|
|
7438
|
+
return `${stats.dev}:${stats.ino}`;
|
|
7439
|
+
}
|
|
7440
|
+
return `${stats.dev}:${stats.ino}:${stats.size}:${stats.mtimeMs}:${stats.ctimeMs}`;
|
|
7441
|
+
} catch (error) {
|
|
7442
|
+
return `unavailable:${getErrorMessage2(error)}`;
|
|
7443
|
+
}
|
|
7444
|
+
}
|
|
7445
|
+
captureReaderArtifactFingerprint() {
|
|
7446
|
+
const storePath = path13.join(this.indexPath, "vectors");
|
|
7447
|
+
return {
|
|
7448
|
+
vectors: `${this.getReaderFileFingerprint(storePath)}|${this.getReaderFileFingerprint(`${storePath}.meta.json`)}`,
|
|
7449
|
+
keyword: this.getReaderFileFingerprint(path13.join(this.indexPath, "inverted-index.json")),
|
|
7450
|
+
database: this.getReaderFileFingerprint(path13.join(this.indexPath, "codebase.db")),
|
|
7451
|
+
databaseIdentity: this.getReaderFileFingerprint(path13.join(this.indexPath, "codebase.db"), true)
|
|
7452
|
+
};
|
|
7453
|
+
}
|
|
7454
|
+
refreshReaderArtifacts() {
|
|
7455
|
+
if (this.initializationMode !== "reader" || !this.configuredProviderInfo) {
|
|
7456
|
+
return;
|
|
7457
|
+
}
|
|
7458
|
+
const previousFingerprint = this.readerArtifactFingerprint;
|
|
7459
|
+
const currentFingerprint = this.captureReaderArtifactFingerprint();
|
|
7460
|
+
const issues = new Map(this.readIssues.map((issue) => [issue.component, issue]));
|
|
7461
|
+
const retryDue = (component) => issues.has(component) && Date.now() >= (this.readerArtifactRetryAfter.get(component) ?? 0);
|
|
7462
|
+
const vectorsChanged = !previousFingerprint || currentFingerprint.vectors !== previousFingerprint.vectors;
|
|
7463
|
+
const keywordChanged = !previousFingerprint || currentFingerprint.keyword !== previousFingerprint.keyword;
|
|
7464
|
+
const databaseChanged = !previousFingerprint || currentFingerprint.database !== previousFingerprint.database;
|
|
7465
|
+
const databaseReplaced = !previousFingerprint || currentFingerprint.databaseIdentity !== previousFingerprint.databaseIdentity;
|
|
7466
|
+
if (previousFingerprint && !vectorsChanged && !keywordChanged && !databaseChanged && !Array.from(issues.keys()).some(retryDue)) {
|
|
7467
|
+
return;
|
|
7468
|
+
}
|
|
7469
|
+
const setIssue = (component, message, error) => {
|
|
7470
|
+
if (!issues.has(component)) {
|
|
7471
|
+
this.logger.warn(message, error === void 0 ? void 0 : { error: getErrorMessage2(error) });
|
|
7472
|
+
}
|
|
7473
|
+
issues.set(component, this.createReadIssue(component, message));
|
|
7474
|
+
this.readerArtifactRetryAfter.set(component, Date.now() + READER_ARTIFACT_RETRY_INTERVAL_MS);
|
|
7475
|
+
};
|
|
7476
|
+
const storePath = path13.join(this.indexPath, "vectors");
|
|
7477
|
+
const vectorMetadataPath = `${storePath}.meta.json`;
|
|
7478
|
+
const invertedIndexPath = path13.join(this.indexPath, "inverted-index.json");
|
|
7479
|
+
const dbPath = path13.join(this.indexPath, "codebase.db");
|
|
7480
|
+
if (vectorsChanged || retryDue("vectors")) {
|
|
7481
|
+
const vectorStoreExists = existsSync8(storePath);
|
|
7482
|
+
const vectorMetadataExists = existsSync8(vectorMetadataPath);
|
|
7483
|
+
if (vectorStoreExists && vectorMetadataExists) {
|
|
7484
|
+
try {
|
|
7485
|
+
const store = new VectorStore(storePath, this.configuredProviderInfo.modelInfo.dimensions);
|
|
7486
|
+
store.loadStrict();
|
|
7487
|
+
this.store = store;
|
|
7488
|
+
issues.delete("vectors");
|
|
7489
|
+
this.readerArtifactRetryAfter.delete("vectors");
|
|
7490
|
+
} catch (error) {
|
|
7491
|
+
setIssue("vectors", this.getVectorReadIssueMessage(), error);
|
|
7492
|
+
}
|
|
7493
|
+
} else if (vectorStoreExists !== vectorMetadataExists || issues.has("vectors")) {
|
|
7494
|
+
setIssue("vectors", this.getVectorReadIssueMessage());
|
|
7495
|
+
}
|
|
7496
|
+
}
|
|
7497
|
+
if (keywordChanged || retryDue("keyword") || !existsSync8(invertedIndexPath) && (this.store?.count() ?? 0) > 0) {
|
|
7498
|
+
if (existsSync8(invertedIndexPath)) {
|
|
7499
|
+
try {
|
|
7500
|
+
const invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
7501
|
+
invertedIndex.load();
|
|
7502
|
+
this.invertedIndex = invertedIndex;
|
|
7503
|
+
issues.delete("keyword");
|
|
7504
|
+
this.readerArtifactRetryAfter.delete("keyword");
|
|
7505
|
+
} catch (error) {
|
|
7506
|
+
setIssue("keyword", this.getKeywordReadIssueMessage(), error);
|
|
7507
|
+
}
|
|
7508
|
+
} else if ((this.store?.count() ?? 0) > 0 || issues.has("keyword")) {
|
|
7509
|
+
setIssue("keyword", this.getKeywordReadIssueMessage());
|
|
7510
|
+
}
|
|
7511
|
+
}
|
|
7512
|
+
if (databaseReplaced || databaseChanged && issues.has("database") || retryDue("database")) {
|
|
7513
|
+
if (existsSync8(dbPath)) {
|
|
7514
|
+
try {
|
|
7515
|
+
const database = Database.openReadOnly(dbPath);
|
|
7516
|
+
if (this.database) {
|
|
7517
|
+
this.retiredDatabases.push(this.database);
|
|
7518
|
+
}
|
|
7519
|
+
this.database = database;
|
|
7520
|
+
issues.delete("database");
|
|
7521
|
+
this.readerArtifactRetryAfter.delete("database");
|
|
7522
|
+
} catch (error) {
|
|
7523
|
+
setIssue("database", this.getDatabaseReadIssueMessage(), error);
|
|
7524
|
+
}
|
|
7525
|
+
} else if ((this.store?.count() ?? 0) > 0 || issues.has("database")) {
|
|
7526
|
+
setIssue("database", this.getDatabaseReadIssueMessage());
|
|
7527
|
+
}
|
|
7528
|
+
}
|
|
7529
|
+
if (!issues.has("database")) {
|
|
7530
|
+
try {
|
|
7531
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
7532
|
+
} catch (error) {
|
|
7533
|
+
setIssue("database", this.getDatabaseReadIssueMessage(), error);
|
|
7534
|
+
}
|
|
7535
|
+
}
|
|
7536
|
+
this.readIssues = Array.from(issues.values());
|
|
7537
|
+
this.readerArtifactFingerprint = currentFingerprint;
|
|
7538
|
+
}
|
|
7539
|
+
refreshInactiveWriterArtifacts() {
|
|
7540
|
+
if (this.initializationMode !== "writer" || this.activeIndexLease) {
|
|
7541
|
+
return true;
|
|
7542
|
+
}
|
|
7543
|
+
const previousFingerprint = this.writerArtifactFingerprint;
|
|
7544
|
+
const currentFingerprint = this.captureReaderArtifactFingerprint();
|
|
7545
|
+
const retryDue = this.readIssues.some(
|
|
7546
|
+
(issue) => Date.now() >= (this.readerArtifactRetryAfter.get(issue.component) ?? 0)
|
|
7547
|
+
);
|
|
7548
|
+
const artifactsChanged = !previousFingerprint || currentFingerprint.vectors !== previousFingerprint.vectors || currentFingerprint.keyword !== previousFingerprint.keyword || currentFingerprint.database !== previousFingerprint.database || currentFingerprint.databaseIdentity !== previousFingerprint.databaseIdentity;
|
|
7549
|
+
if (!artifactsChanged && !retryDue) {
|
|
7550
|
+
return true;
|
|
7551
|
+
}
|
|
7552
|
+
if (!previousFingerprint || currentFingerprint.databaseIdentity !== previousFingerprint.databaseIdentity) {
|
|
7553
|
+
return false;
|
|
7554
|
+
}
|
|
7555
|
+
this.initializationMode = "reader";
|
|
7556
|
+
this.readerArtifactFingerprint = previousFingerprint;
|
|
7557
|
+
try {
|
|
7558
|
+
this.refreshReaderArtifacts();
|
|
7559
|
+
this.writerArtifactFingerprint = this.readerArtifactFingerprint ?? currentFingerprint;
|
|
7560
|
+
} finally {
|
|
7561
|
+
this.readerArtifactFingerprint = null;
|
|
7562
|
+
this.initializationMode = "writer";
|
|
7563
|
+
}
|
|
7564
|
+
return true;
|
|
7565
|
+
}
|
|
7566
|
+
async initializeUnlocked(mode, recoveredOwners = [], options = {}) {
|
|
7567
|
+
if (mode === "writer") {
|
|
7568
|
+
this.requireActiveLease();
|
|
7569
|
+
}
|
|
7570
|
+
this.readIssues = [];
|
|
7571
|
+
this.readerArtifactRetryAfter.clear();
|
|
6792
7572
|
if (this.config.embeddingProvider === "custom") {
|
|
6793
7573
|
if (!this.config.customProvider) {
|
|
6794
7574
|
throw new Error("embeddingProvider is 'custom' but customProvider config is missing.");
|
|
@@ -6820,36 +7600,103 @@ var Indexer = class {
|
|
|
6820
7600
|
});
|
|
6821
7601
|
}
|
|
6822
7602
|
}
|
|
6823
|
-
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
6824
7603
|
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
6825
|
-
const storePath =
|
|
6826
|
-
|
|
6827
|
-
const
|
|
6828
|
-
|
|
6829
|
-
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6833
|
-
|
|
6834
|
-
|
|
6835
|
-
|
|
6836
|
-
|
|
6837
|
-
|
|
7604
|
+
const storePath = path13.join(this.indexPath, "vectors");
|
|
7605
|
+
const vectorMetadataPath = `${storePath}.meta.json`;
|
|
7606
|
+
const invertedIndexPath = path13.join(this.indexPath, "inverted-index.json");
|
|
7607
|
+
const dbPath = path13.join(this.indexPath, "codebase.db");
|
|
7608
|
+
let dbIsNew = !existsSync8(dbPath);
|
|
7609
|
+
const readerArtifactFingerprint = mode === "reader" ? this.captureReaderArtifactFingerprint() : null;
|
|
7610
|
+
if (mode === "writer") {
|
|
7611
|
+
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
7612
|
+
if (recoveredOwners.length > 0 && this.config.scope === "project" && !this.isLocalProjectIndexPath()) {
|
|
7613
|
+
throw new Error(
|
|
7614
|
+
"Interrupted indexing recovery is unsafe while using an inherited worktree index. Run index_codebase with force=true to create a local project index boundary."
|
|
7615
|
+
);
|
|
7616
|
+
}
|
|
7617
|
+
for (const recoveredOwner of recoveredOwners) {
|
|
7618
|
+
recoverLeaseArtifacts(this.indexPath, recoveredOwner, [
|
|
7619
|
+
storePath,
|
|
7620
|
+
`${storePath}.meta.json`
|
|
7621
|
+
]);
|
|
7622
|
+
}
|
|
7623
|
+
if (recoveredOwners.length > 0 && this.config.scope === "project") {
|
|
7624
|
+
await this.resetLocalIndexArtifacts();
|
|
7625
|
+
}
|
|
7626
|
+
this.store = new VectorStore(storePath, dimensions);
|
|
7627
|
+
if (existsSync8(storePath) || existsSync8(vectorMetadataPath)) {
|
|
7628
|
+
this.store.load();
|
|
6838
7629
|
}
|
|
6839
7630
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
6840
|
-
|
|
6841
|
-
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
|
|
6845
|
-
|
|
6846
|
-
|
|
6847
|
-
throw error;
|
|
7631
|
+
try {
|
|
7632
|
+
this.invertedIndex.load();
|
|
7633
|
+
} catch {
|
|
7634
|
+
if (existsSync8(invertedIndexPath)) {
|
|
7635
|
+
await fsPromises2.unlink(invertedIndexPath);
|
|
7636
|
+
}
|
|
7637
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
6848
7638
|
}
|
|
7639
|
+
try {
|
|
7640
|
+
this.database = new Database(dbPath);
|
|
7641
|
+
} catch (error) {
|
|
7642
|
+
if (!await this.tryResetCorruptedIndex("initializing index database", error)) {
|
|
7643
|
+
throw error;
|
|
7644
|
+
}
|
|
7645
|
+
this.store = new VectorStore(storePath, dimensions);
|
|
7646
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
7647
|
+
this.database = new Database(dbPath);
|
|
7648
|
+
dbIsNew = true;
|
|
7649
|
+
}
|
|
7650
|
+
} else {
|
|
6849
7651
|
this.store = new VectorStore(storePath, dimensions);
|
|
7652
|
+
const vectorStoreExists = existsSync8(storePath);
|
|
7653
|
+
const vectorMetadataExists = existsSync8(vectorMetadataPath);
|
|
7654
|
+
const vectorReadFailureMessage = this.getVectorReadIssueMessage();
|
|
7655
|
+
if (vectorStoreExists !== vectorMetadataExists) {
|
|
7656
|
+
this.recordReadIssue("vectors", vectorReadFailureMessage);
|
|
7657
|
+
} else if (vectorStoreExists) {
|
|
7658
|
+
try {
|
|
7659
|
+
this.store.loadStrict();
|
|
7660
|
+
} catch (error) {
|
|
7661
|
+
this.recordReadIssue("vectors", vectorReadFailureMessage, error);
|
|
7662
|
+
this.store = new VectorStore(storePath, dimensions);
|
|
7663
|
+
}
|
|
7664
|
+
}
|
|
6850
7665
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
6851
|
-
|
|
6852
|
-
|
|
7666
|
+
if (existsSync8(invertedIndexPath)) {
|
|
7667
|
+
try {
|
|
7668
|
+
this.invertedIndex.load();
|
|
7669
|
+
} catch (error) {
|
|
7670
|
+
this.recordReadIssue(
|
|
7671
|
+
"keyword",
|
|
7672
|
+
this.getKeywordReadIssueMessage(),
|
|
7673
|
+
error
|
|
7674
|
+
);
|
|
7675
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
7676
|
+
}
|
|
7677
|
+
} else if (this.store.count() > 0) {
|
|
7678
|
+
this.recordReadIssue("keyword", this.getKeywordReadIssueMessage());
|
|
7679
|
+
}
|
|
7680
|
+
if (existsSync8(dbPath)) {
|
|
7681
|
+
try {
|
|
7682
|
+
this.database = Database.openReadOnly(dbPath);
|
|
7683
|
+
} catch (error) {
|
|
7684
|
+
this.recordReadIssue(
|
|
7685
|
+
"database",
|
|
7686
|
+
this.getDatabaseReadIssueMessage(),
|
|
7687
|
+
error
|
|
7688
|
+
);
|
|
7689
|
+
this.database = Database.createEmptyReadOnly();
|
|
7690
|
+
}
|
|
7691
|
+
} else {
|
|
7692
|
+
this.database = Database.createEmptyReadOnly();
|
|
7693
|
+
if (this.store.count() > 0) {
|
|
7694
|
+
this.recordReadIssue(
|
|
7695
|
+
"database",
|
|
7696
|
+
`Index database is missing for the published vectors. ${this.getDatabaseReadIssueMessage()}`
|
|
7697
|
+
);
|
|
7698
|
+
}
|
|
7699
|
+
}
|
|
6853
7700
|
}
|
|
6854
7701
|
if (isGitRepo(this.projectRoot)) {
|
|
6855
7702
|
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
@@ -6863,10 +7710,10 @@ var Indexer = class {
|
|
|
6863
7710
|
this.baseBranch = "default";
|
|
6864
7711
|
this.logger.branch("debug", "Not a git repository, using default branch");
|
|
6865
7712
|
}
|
|
6866
|
-
if (
|
|
6867
|
-
await this.
|
|
7713
|
+
if (mode === "writer" && recoveredOwners.length > 0) {
|
|
7714
|
+
await this.recoverFromInterruptedIndexingUnlocked(recoveredOwners);
|
|
6868
7715
|
}
|
|
6869
|
-
if (dbIsNew && this.store.count() > 0) {
|
|
7716
|
+
if (mode === "writer" && dbIsNew && this.store.count() > 0) {
|
|
6870
7717
|
this.migrateFromLegacyIndex();
|
|
6871
7718
|
}
|
|
6872
7719
|
this.loadFileHashCache();
|
|
@@ -6878,9 +7725,11 @@ var Indexer = class {
|
|
|
6878
7725
|
configuredProviderInfo: this.configuredProviderInfo
|
|
6879
7726
|
});
|
|
6880
7727
|
}
|
|
6881
|
-
if (this.config.indexing.autoGc) {
|
|
7728
|
+
if (mode === "writer" && this.config.indexing.autoGc && !options.skipAutoGc) {
|
|
6882
7729
|
await this.maybeRunAutoGc();
|
|
6883
7730
|
}
|
|
7731
|
+
this.initializationMode = mode;
|
|
7732
|
+
this.readerArtifactFingerprint = readerArtifactFingerprint;
|
|
6884
7733
|
}
|
|
6885
7734
|
async maybeRunAutoGc() {
|
|
6886
7735
|
if (!this.database) return;
|
|
@@ -6897,7 +7746,7 @@ var Indexer = class {
|
|
|
6897
7746
|
}
|
|
6898
7747
|
}
|
|
6899
7748
|
if (shouldRunGc) {
|
|
6900
|
-
const result = await this.
|
|
7749
|
+
const result = await this.healthCheckUnlocked();
|
|
6901
7750
|
if (result.warning) {
|
|
6902
7751
|
this.database.setMetadata(STARTUP_WARNING_METADATA_KEY, result.warning);
|
|
6903
7752
|
} else {
|
|
@@ -6919,7 +7768,7 @@ var Indexer = class {
|
|
|
6919
7768
|
if (await this.tryResetCorruptedIndex("running automatic orphan garbage collection", error)) {
|
|
6920
7769
|
return {
|
|
6921
7770
|
resetCorruptedIndex: true,
|
|
6922
|
-
warning: this.getCorruptedIndexWarning(
|
|
7771
|
+
warning: this.getCorruptedIndexWarning(path13.join(this.indexPath, "codebase.db"))
|
|
6923
7772
|
};
|
|
6924
7773
|
}
|
|
6925
7774
|
throw error;
|
|
@@ -6934,28 +7783,29 @@ var Indexer = class {
|
|
|
6934
7783
|
return;
|
|
6935
7784
|
}
|
|
6936
7785
|
const retainedEntries = store.getAllMetadata().filter(({ key }) => !excludedSet.has(key));
|
|
6937
|
-
const storeBasePath =
|
|
6938
|
-
const storeIndexPath =
|
|
7786
|
+
const storeBasePath = path13.join(this.indexPath, "vectors");
|
|
7787
|
+
const storeIndexPath = storeBasePath;
|
|
6939
7788
|
const storeMetadataPath = `${storeBasePath}.meta.json`;
|
|
6940
|
-
const
|
|
6941
|
-
const
|
|
7789
|
+
const lease = this.requireActiveLease();
|
|
7790
|
+
const backupIndexPath = createLeaseTemporaryPath(storeIndexPath, lease.owner, "bak");
|
|
7791
|
+
const backupMetadataPath = createLeaseTemporaryPath(storeMetadataPath, lease.owner, "bak");
|
|
6942
7792
|
let backedUpIndex = false;
|
|
6943
7793
|
let backedUpMetadata = false;
|
|
6944
7794
|
let rebuiltCount = 0;
|
|
6945
7795
|
let skippedCount = 0;
|
|
6946
|
-
if (
|
|
7796
|
+
if (existsSync8(backupIndexPath)) {
|
|
6947
7797
|
unlinkSync(backupIndexPath);
|
|
6948
7798
|
}
|
|
6949
|
-
if (
|
|
7799
|
+
if (existsSync8(backupMetadataPath)) {
|
|
6950
7800
|
unlinkSync(backupMetadataPath);
|
|
6951
7801
|
}
|
|
6952
7802
|
try {
|
|
6953
|
-
if (
|
|
6954
|
-
|
|
7803
|
+
if (existsSync8(storeIndexPath)) {
|
|
7804
|
+
renameSync2(storeIndexPath, backupIndexPath);
|
|
6955
7805
|
backedUpIndex = true;
|
|
6956
7806
|
}
|
|
6957
|
-
if (
|
|
6958
|
-
|
|
7807
|
+
if (existsSync8(storeMetadataPath)) {
|
|
7808
|
+
renameSync2(storeMetadataPath, backupMetadataPath);
|
|
6959
7809
|
backedUpMetadata = true;
|
|
6960
7810
|
}
|
|
6961
7811
|
store.clear();
|
|
@@ -6975,10 +7825,10 @@ var Indexer = class {
|
|
|
6975
7825
|
rebuiltCount += 1;
|
|
6976
7826
|
}
|
|
6977
7827
|
store.save();
|
|
6978
|
-
if (backedUpIndex &&
|
|
7828
|
+
if (backedUpIndex && existsSync8(backupIndexPath)) {
|
|
6979
7829
|
unlinkSync(backupIndexPath);
|
|
6980
7830
|
}
|
|
6981
|
-
if (backedUpMetadata &&
|
|
7831
|
+
if (backedUpMetadata && existsSync8(backupMetadataPath)) {
|
|
6982
7832
|
unlinkSync(backupMetadataPath);
|
|
6983
7833
|
}
|
|
6984
7834
|
this.logger.gc("info", "Rebuilt vector store to avoid native remove", {
|
|
@@ -6991,17 +7841,17 @@ var Indexer = class {
|
|
|
6991
7841
|
store.clear();
|
|
6992
7842
|
} catch {
|
|
6993
7843
|
}
|
|
6994
|
-
if (
|
|
7844
|
+
if (existsSync8(storeIndexPath)) {
|
|
6995
7845
|
unlinkSync(storeIndexPath);
|
|
6996
7846
|
}
|
|
6997
|
-
if (
|
|
7847
|
+
if (existsSync8(storeMetadataPath)) {
|
|
6998
7848
|
unlinkSync(storeMetadataPath);
|
|
6999
7849
|
}
|
|
7000
|
-
if (backedUpIndex &&
|
|
7001
|
-
|
|
7850
|
+
if (backedUpIndex && existsSync8(backupIndexPath)) {
|
|
7851
|
+
renameSync2(backupIndexPath, storeIndexPath);
|
|
7002
7852
|
}
|
|
7003
|
-
if (backedUpMetadata &&
|
|
7004
|
-
|
|
7853
|
+
if (backedUpMetadata && existsSync8(backupMetadataPath)) {
|
|
7854
|
+
renameSync2(backupMetadataPath, storeMetadataPath);
|
|
7005
7855
|
}
|
|
7006
7856
|
if (backedUpIndex || backedUpMetadata) {
|
|
7007
7857
|
store.load();
|
|
@@ -7015,11 +7865,37 @@ var Indexer = class {
|
|
|
7015
7865
|
}
|
|
7016
7866
|
return `Detected a corrupted local SQLite index at ${dbPath} and reset the local index. Run index_codebase to rebuild search data.`;
|
|
7017
7867
|
}
|
|
7868
|
+
async resetLocalIndexArtifacts() {
|
|
7869
|
+
this.store = null;
|
|
7870
|
+
this.invertedIndex = null;
|
|
7871
|
+
this.database?.close();
|
|
7872
|
+
this.database = null;
|
|
7873
|
+
this.indexCompatibility = null;
|
|
7874
|
+
this.initializationMode = "none";
|
|
7875
|
+
this.readIssues = [];
|
|
7876
|
+
this.readerArtifactFingerprint = null;
|
|
7877
|
+
this.writerArtifactFingerprint = null;
|
|
7878
|
+
this.readerArtifactRetryAfter.clear();
|
|
7879
|
+
this.fileHashCache.clear();
|
|
7880
|
+
const resetPaths = [
|
|
7881
|
+
path13.join(this.indexPath, "codebase.db"),
|
|
7882
|
+
path13.join(this.indexPath, "codebase.db-shm"),
|
|
7883
|
+
path13.join(this.indexPath, "codebase.db-wal"),
|
|
7884
|
+
path13.join(this.indexPath, "vectors"),
|
|
7885
|
+
path13.join(this.indexPath, "vectors.usearch"),
|
|
7886
|
+
path13.join(this.indexPath, "vectors.meta.json"),
|
|
7887
|
+
path13.join(this.indexPath, "inverted-index.json"),
|
|
7888
|
+
path13.join(this.indexPath, "file-hashes.json"),
|
|
7889
|
+
path13.join(this.indexPath, "failed-batches.json")
|
|
7890
|
+
];
|
|
7891
|
+
await Promise.all(resetPaths.map((targetPath) => fsPromises2.rm(targetPath, { recursive: true, force: true })));
|
|
7892
|
+
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
7893
|
+
}
|
|
7018
7894
|
async tryResetCorruptedIndex(stage, error) {
|
|
7019
7895
|
if (!isSqliteCorruptionError(error)) {
|
|
7020
7896
|
return false;
|
|
7021
7897
|
}
|
|
7022
|
-
const dbPath =
|
|
7898
|
+
const dbPath = path13.join(this.indexPath, "codebase.db");
|
|
7023
7899
|
const warning = this.getCorruptedIndexWarning(dbPath);
|
|
7024
7900
|
const errorMessage = getErrorMessage2(error);
|
|
7025
7901
|
if (this.config.scope === "global") {
|
|
@@ -7035,30 +7911,7 @@ var Indexer = class {
|
|
|
7035
7911
|
dbPath,
|
|
7036
7912
|
error: errorMessage
|
|
7037
7913
|
});
|
|
7038
|
-
this.
|
|
7039
|
-
this.invertedIndex = null;
|
|
7040
|
-
this.database?.close();
|
|
7041
|
-
this.database = null;
|
|
7042
|
-
this.indexCompatibility = null;
|
|
7043
|
-
this.fileHashCache.clear();
|
|
7044
|
-
const resetPaths = [
|
|
7045
|
-
path12.join(this.indexPath, "codebase.db"),
|
|
7046
|
-
path12.join(this.indexPath, "codebase.db-shm"),
|
|
7047
|
-
path12.join(this.indexPath, "codebase.db-wal"),
|
|
7048
|
-
path12.join(this.indexPath, "vectors.usearch"),
|
|
7049
|
-
path12.join(this.indexPath, "inverted-index.json"),
|
|
7050
|
-
path12.join(this.indexPath, "file-hashes.json"),
|
|
7051
|
-
path12.join(this.indexPath, "failed-batches.json"),
|
|
7052
|
-
path12.join(this.indexPath, "indexing.lock"),
|
|
7053
|
-
path12.join(this.indexPath, "vectors")
|
|
7054
|
-
];
|
|
7055
|
-
await Promise.all(resetPaths.map(async (targetPath) => {
|
|
7056
|
-
try {
|
|
7057
|
-
await fsPromises2.rm(targetPath, { recursive: true, force: true });
|
|
7058
|
-
} catch {
|
|
7059
|
-
}
|
|
7060
|
-
}));
|
|
7061
|
-
await fsPromises2.mkdir(this.indexPath, { recursive: true });
|
|
7914
|
+
await this.resetLocalIndexArtifacts();
|
|
7062
7915
|
return true;
|
|
7063
7916
|
}
|
|
7064
7917
|
migrateFromLegacyIndex() {
|
|
@@ -7177,8 +8030,62 @@ var Indexer = class {
|
|
|
7177
8030
|
return this.indexCompatibility;
|
|
7178
8031
|
}
|
|
7179
8032
|
async ensureInitialized() {
|
|
8033
|
+
let initializedReader = false;
|
|
8034
|
+
while (true) {
|
|
8035
|
+
if (this.initializationPromise) {
|
|
8036
|
+
await this.initializationPromise;
|
|
8037
|
+
}
|
|
8038
|
+
if (!this.isInitializedFor("reader")) {
|
|
8039
|
+
await this.initialize();
|
|
8040
|
+
initializedReader = true;
|
|
8041
|
+
continue;
|
|
8042
|
+
}
|
|
8043
|
+
if (this.initializationMode === "writer" && !this.activeIndexLease && !initializedReader) {
|
|
8044
|
+
if (!this.refreshInactiveWriterArtifacts()) {
|
|
8045
|
+
this.resetLoadedIndexState(true);
|
|
8046
|
+
await this.initialize();
|
|
8047
|
+
initializedReader = true;
|
|
8048
|
+
continue;
|
|
8049
|
+
}
|
|
8050
|
+
}
|
|
8051
|
+
if (this.initializationMode === "reader" && !initializedReader) {
|
|
8052
|
+
this.refreshReaderArtifacts();
|
|
8053
|
+
}
|
|
8054
|
+
const state = this.requireLoadedIndexState();
|
|
8055
|
+
return {
|
|
8056
|
+
...state,
|
|
8057
|
+
readIssues: [...this.readIssues],
|
|
8058
|
+
compatibility: this.indexCompatibility ?? this.validateIndexCompatibility(state.configuredProviderInfo)
|
|
8059
|
+
};
|
|
8060
|
+
}
|
|
8061
|
+
}
|
|
8062
|
+
async ensureInitializedUnlocked(recoveredOwners = []) {
|
|
8063
|
+
this.requireActiveLease();
|
|
8064
|
+
if (this.initializationPromise) {
|
|
8065
|
+
await this.initializationPromise;
|
|
8066
|
+
}
|
|
8067
|
+
if (recoveredOwners.length > 0 || !this.isInitializedFor("writer")) {
|
|
8068
|
+
const retireReaderDatabase = this.initializationMode === "reader";
|
|
8069
|
+
this.resetLoadedIndexState(retireReaderDatabase);
|
|
8070
|
+
await this.initializeOnce("writer", recoveredOwners, { skipAutoGc: true });
|
|
8071
|
+
} else {
|
|
8072
|
+
this.refreshLoadedIndexState();
|
|
8073
|
+
}
|
|
8074
|
+
if (this.config.indexing.autoGc) {
|
|
8075
|
+
await this.maybeRunAutoGc();
|
|
8076
|
+
}
|
|
8077
|
+
return this.requireLoadedIndexState();
|
|
8078
|
+
}
|
|
8079
|
+
requireReadableComponents(readIssues, ...components) {
|
|
8080
|
+
const componentSet = new Set(components);
|
|
8081
|
+
const issues = readIssues.filter((issue) => issue.blocking && componentSet.has(issue.component));
|
|
8082
|
+
if (issues.length > 0) {
|
|
8083
|
+
throw new Error(issues.map((issue) => issue.message).join(" "));
|
|
8084
|
+
}
|
|
8085
|
+
}
|
|
8086
|
+
requireLoadedIndexState() {
|
|
7180
8087
|
if (!this.store || !this.provider || !this.invertedIndex || !this.configuredProviderInfo || !this.database) {
|
|
7181
|
-
|
|
8088
|
+
throw new Error("Index state is not initialized");
|
|
7182
8089
|
}
|
|
7183
8090
|
return {
|
|
7184
8091
|
store: this.store,
|
|
@@ -7202,7 +8109,12 @@ var Indexer = class {
|
|
|
7202
8109
|
return createCostEstimate(files, configuredProviderInfo);
|
|
7203
8110
|
}
|
|
7204
8111
|
async index(onProgress) {
|
|
7205
|
-
|
|
8112
|
+
return this.withIndexMutationLease("index", async (recoveredOwners) => {
|
|
8113
|
+
return this.indexUnlocked(onProgress, recoveredOwners);
|
|
8114
|
+
});
|
|
8115
|
+
}
|
|
8116
|
+
async indexUnlocked(onProgress, recoveredOwners = [], stateReady = false) {
|
|
8117
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = stateReady ? this.requireLoadedIndexState() : await this.ensureInitializedUnlocked(recoveredOwners);
|
|
7206
8118
|
const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
7207
8119
|
const branchCatalogKey = this.getBranchCatalogKey();
|
|
7208
8120
|
const forceScopedReembed = scopedRoots !== null && database.getMetadata(this.getProjectForceReembedMetadataKey()) === "true";
|
|
@@ -7212,7 +8124,6 @@ var Indexer = class {
|
|
|
7212
8124
|
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
7213
8125
|
);
|
|
7214
8126
|
}
|
|
7215
|
-
this.acquireIndexingLock();
|
|
7216
8127
|
this.logger.recordIndexingStart();
|
|
7217
8128
|
this.logger.info("Starting indexing", { projectRoot: this.projectRoot });
|
|
7218
8129
|
const startTime = Date.now();
|
|
@@ -7361,7 +8272,7 @@ var Indexer = class {
|
|
|
7361
8272
|
for (const parsed of parsedFiles) {
|
|
7362
8273
|
currentFilePaths.add(parsed.path);
|
|
7363
8274
|
if (parsed.chunks.length === 0) {
|
|
7364
|
-
const relativePath =
|
|
8275
|
+
const relativePath = path13.relative(this.projectRoot, parsed.path);
|
|
7365
8276
|
stats.parseFailures.push(relativePath);
|
|
7366
8277
|
}
|
|
7367
8278
|
let fileChunkCount = 0;
|
|
@@ -7561,7 +8472,9 @@ var Indexer = class {
|
|
|
7561
8472
|
database.addChunksToBranchBatch(branchCatalogKey, Array.from(currentChunkIds));
|
|
7562
8473
|
database.clearBranchSymbols(branchCatalogKey);
|
|
7563
8474
|
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
7564
|
-
|
|
8475
|
+
const vectorPath = path13.join(this.indexPath, "vectors");
|
|
8476
|
+
const shouldFingerprintLegacyPair = !store.hasFingerprint() && existsSync8(vectorPath) && existsSync8(`${vectorPath}.meta.json`);
|
|
8477
|
+
if (backfilledBlameMetadata || shouldFingerprintLegacyPair) {
|
|
7565
8478
|
store.save();
|
|
7566
8479
|
}
|
|
7567
8480
|
if (scopedRoots) {
|
|
@@ -7582,7 +8495,6 @@ var Indexer = class {
|
|
|
7582
8495
|
chunksProcessed: 0,
|
|
7583
8496
|
totalChunks: 0
|
|
7584
8497
|
});
|
|
7585
|
-
this.releaseIndexingLock();
|
|
7586
8498
|
return stats;
|
|
7587
8499
|
}
|
|
7588
8500
|
if (pendingChunks.length === 0) {
|
|
@@ -7591,7 +8503,7 @@ var Indexer = class {
|
|
|
7591
8503
|
database.clearBranchSymbols(branchCatalogKey);
|
|
7592
8504
|
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
7593
8505
|
store.save();
|
|
7594
|
-
|
|
8506
|
+
this.saveInvertedIndex(invertedIndex);
|
|
7595
8507
|
if (scopedRoots) {
|
|
7596
8508
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7597
8509
|
this.clearScopedFailedBatches(scopedRoots);
|
|
@@ -7610,7 +8522,6 @@ var Indexer = class {
|
|
|
7610
8522
|
chunksProcessed: 0,
|
|
7611
8523
|
totalChunks: 0
|
|
7612
8524
|
});
|
|
7613
|
-
this.releaseIndexingLock();
|
|
7614
8525
|
return stats;
|
|
7615
8526
|
}
|
|
7616
8527
|
onProgress?.({
|
|
@@ -7841,7 +8752,7 @@ var Indexer = class {
|
|
|
7841
8752
|
database.clearBranchSymbols(branchCatalogKey);
|
|
7842
8753
|
database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds));
|
|
7843
8754
|
store.save();
|
|
7844
|
-
|
|
8755
|
+
this.saveInvertedIndex(invertedIndex);
|
|
7845
8756
|
if (scopedRoots) {
|
|
7846
8757
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
7847
8758
|
} else {
|
|
@@ -7893,7 +8804,6 @@ var Indexer = class {
|
|
|
7893
8804
|
chunksProcessed: stats.indexedChunks,
|
|
7894
8805
|
totalChunks: pendingChunks.length
|
|
7895
8806
|
});
|
|
7896
|
-
this.releaseIndexingLock();
|
|
7897
8807
|
return stats;
|
|
7898
8808
|
}
|
|
7899
8809
|
async getQueryEmbedding(query, provider) {
|
|
@@ -7959,8 +8869,8 @@ var Indexer = class {
|
|
|
7959
8869
|
return intersection / union;
|
|
7960
8870
|
}
|
|
7961
8871
|
async search(query, limit, options) {
|
|
7962
|
-
const { store, provider, database } = await this.ensureInitialized();
|
|
7963
|
-
|
|
8872
|
+
const { store, provider, invertedIndex, database, readIssues, compatibility } = await this.ensureInitialized();
|
|
8873
|
+
this.requireReadableComponents(readIssues, "vectors", "database");
|
|
7964
8874
|
if (!compatibility.compatible) {
|
|
7965
8875
|
throw new Error(
|
|
7966
8876
|
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} A possible solution is to run index_codebase with force=true to rebuild the index.`
|
|
@@ -7974,6 +8884,7 @@ var Indexer = class {
|
|
|
7974
8884
|
const maxResults = limit ?? this.config.search.maxResults;
|
|
7975
8885
|
const hybridWeight = options?.hybridWeight ?? this.config.search.hybridWeight;
|
|
7976
8886
|
const fusionStrategy = this.config.search.fusionStrategy;
|
|
8887
|
+
const effectiveHybridWeight = fusionStrategy === "weighted" && readIssues.some((issue) => issue.component === "keyword") ? 0 : hybridWeight;
|
|
7977
8888
|
const rrfK = this.config.search.rrfK;
|
|
7978
8889
|
const rerankTopN = this.config.search.rerankTopN;
|
|
7979
8890
|
const filterByBranch = options?.filterByBranch ?? true;
|
|
@@ -7982,7 +8893,7 @@ var Indexer = class {
|
|
|
7982
8893
|
this.logger.search("debug", "Starting search", {
|
|
7983
8894
|
query,
|
|
7984
8895
|
maxResults,
|
|
7985
|
-
hybridWeight,
|
|
8896
|
+
hybridWeight: effectiveHybridWeight,
|
|
7986
8897
|
fusionStrategy,
|
|
7987
8898
|
rrfK,
|
|
7988
8899
|
rerankTopN,
|
|
@@ -7990,13 +8901,22 @@ var Indexer = class {
|
|
|
7990
8901
|
});
|
|
7991
8902
|
const embeddingStartTime = performance2.now();
|
|
7992
8903
|
const embeddingQuery = stripFilePathHint(query);
|
|
7993
|
-
|
|
8904
|
+
let embedding;
|
|
8905
|
+
try {
|
|
8906
|
+
embedding = await this.getQueryEmbedding(embeddingQuery, provider);
|
|
8907
|
+
} catch (error) {
|
|
8908
|
+
this.logger.warn("Query embedding failed; falling back to keyword-only search", {
|
|
8909
|
+
query,
|
|
8910
|
+
error: getErrorMessage2(error),
|
|
8911
|
+
action: "Check the embedding provider configuration and retry search after restoring provider health."
|
|
8912
|
+
});
|
|
8913
|
+
}
|
|
7994
8914
|
const embeddingMs = performance2.now() - embeddingStartTime;
|
|
7995
8915
|
const vectorStartTime = performance2.now();
|
|
7996
|
-
const semanticResults = store.search(embedding, maxResults * 4);
|
|
8916
|
+
const semanticResults = embedding ? store.search(embedding, maxResults * 4) : [];
|
|
7997
8917
|
const vectorMs = performance2.now() - vectorStartTime;
|
|
7998
8918
|
const keywordStartTime = performance2.now();
|
|
7999
|
-
const keywordResults = await this.keywordSearch(query, maxResults * 4);
|
|
8919
|
+
const keywordResults = await this.keywordSearch(query, maxResults * 4, store, invertedIndex);
|
|
8000
8920
|
const keywordMs = performance2.now() - keywordStartTime;
|
|
8001
8921
|
let branchChunkIds = null;
|
|
8002
8922
|
if (filterByBranch && (this.config.scope === "global" || this.currentBranch !== "default")) {
|
|
@@ -8028,12 +8948,13 @@ var Indexer = class {
|
|
|
8028
8948
|
});
|
|
8029
8949
|
}
|
|
8030
8950
|
const fusionStartTime = performance2.now();
|
|
8951
|
+
const rankingHybridWeight = embedding === void 0 && fusionStrategy === "weighted" ? 1 : effectiveHybridWeight;
|
|
8031
8952
|
const combined = rankHybridResults(query, semanticCandidates, keywordCandidates, {
|
|
8032
8953
|
fusionStrategy,
|
|
8033
8954
|
rrfK,
|
|
8034
8955
|
rerankTopN,
|
|
8035
8956
|
limit: maxResults,
|
|
8036
|
-
hybridWeight,
|
|
8957
|
+
hybridWeight: rankingHybridWeight,
|
|
8037
8958
|
prioritizeSourcePaths: sourceIntent
|
|
8038
8959
|
});
|
|
8039
8960
|
const rerankedCombined = await this.rerankCandidatesWithApi(query, combined, {
|
|
@@ -8133,8 +9054,7 @@ var Indexer = class {
|
|
|
8133
9054
|
})
|
|
8134
9055
|
);
|
|
8135
9056
|
}
|
|
8136
|
-
async keywordSearch(query, limit) {
|
|
8137
|
-
const { store, invertedIndex } = await this.ensureInitialized();
|
|
9057
|
+
async keywordSearch(query, limit, store, invertedIndex) {
|
|
8138
9058
|
const scores = invertedIndex.search(query);
|
|
8139
9059
|
if (scores.size === 0) {
|
|
8140
9060
|
return [];
|
|
@@ -8152,24 +9072,54 @@ var Indexer = class {
|
|
|
8152
9072
|
return results.slice(0, limit);
|
|
8153
9073
|
}
|
|
8154
9074
|
async getStatus() {
|
|
8155
|
-
const { store, configuredProviderInfo, database } = await this.ensureInitialized();
|
|
9075
|
+
const { store, configuredProviderInfo, database, readIssues, compatibility } = await this.ensureInitialized();
|
|
8156
9076
|
const failedBatchesCount = this.getFailedBatchesCount();
|
|
9077
|
+
const vectorCount = store.count();
|
|
9078
|
+
const statusReadIssues = [...readIssues];
|
|
9079
|
+
let startupWarning = "";
|
|
9080
|
+
if (!statusReadIssues.some((issue) => issue.component === "database")) {
|
|
9081
|
+
try {
|
|
9082
|
+
startupWarning = database.getMetadata(STARTUP_WARNING_METADATA_KEY) ?? "";
|
|
9083
|
+
} catch (error) {
|
|
9084
|
+
const message = this.getDatabaseReadIssueMessage();
|
|
9085
|
+
statusReadIssues.push(this.createReadIssue("database", message));
|
|
9086
|
+
if (!this.readIssues.some((issue) => issue.component === "database")) {
|
|
9087
|
+
this.recordReadIssue("database", message, error);
|
|
9088
|
+
}
|
|
9089
|
+
}
|
|
9090
|
+
}
|
|
9091
|
+
const readWarning = statusReadIssues.map((issue) => issue.message).join(" ");
|
|
9092
|
+
const warning = [readWarning, startupWarning].filter((message) => message.length > 0).join(" ");
|
|
9093
|
+
const hasBlockingReadIssue = statusReadIssues.some((issue) => issue.blocking);
|
|
8157
9094
|
return {
|
|
8158
|
-
indexed:
|
|
8159
|
-
vectorCount
|
|
9095
|
+
indexed: vectorCount > 0 && !hasBlockingReadIssue,
|
|
9096
|
+
vectorCount,
|
|
8160
9097
|
provider: configuredProviderInfo.provider,
|
|
8161
9098
|
model: configuredProviderInfo.modelInfo.model,
|
|
8162
9099
|
indexPath: this.indexPath,
|
|
8163
9100
|
currentBranch: this.currentBranch,
|
|
8164
9101
|
baseBranch: this.baseBranch,
|
|
8165
|
-
compatibility
|
|
9102
|
+
compatibility,
|
|
8166
9103
|
failedBatchesCount,
|
|
8167
9104
|
failedBatchesPath: failedBatchesCount > 0 ? this.failedBatchesPath : void 0,
|
|
8168
|
-
warning:
|
|
9105
|
+
warning: warning || void 0
|
|
8169
9106
|
};
|
|
8170
9107
|
}
|
|
9108
|
+
async forceIndex(onProgress) {
|
|
9109
|
+
return this.withIndexMutationLease("force-index", async (recoveredOwners) => {
|
|
9110
|
+
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
9111
|
+
await this.clearIndexUnlocked();
|
|
9112
|
+
return this.indexUnlocked(onProgress, [], true);
|
|
9113
|
+
});
|
|
9114
|
+
}
|
|
8171
9115
|
async clearIndex() {
|
|
8172
|
-
|
|
9116
|
+
await this.withIndexMutationLease("clear", async (recoveredOwners) => {
|
|
9117
|
+
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
9118
|
+
await this.clearIndexUnlocked();
|
|
9119
|
+
});
|
|
9120
|
+
}
|
|
9121
|
+
async clearIndexUnlocked() {
|
|
9122
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
8173
9123
|
if (this.config.scope === "global") {
|
|
8174
9124
|
store.load();
|
|
8175
9125
|
invertedIndex.load();
|
|
@@ -8196,7 +9146,7 @@ var Indexer = class {
|
|
|
8196
9146
|
store.clear();
|
|
8197
9147
|
store.save();
|
|
8198
9148
|
invertedIndex.clear();
|
|
8199
|
-
|
|
9149
|
+
this.saveInvertedIndex(invertedIndex);
|
|
8200
9150
|
this.fileHashCache.clear();
|
|
8201
9151
|
this.saveFileHashCache();
|
|
8202
9152
|
database.clearAllIndexedData();
|
|
@@ -8220,14 +9170,7 @@ var Indexer = class {
|
|
|
8220
9170
|
this.indexCompatibility = compatibility;
|
|
8221
9171
|
return;
|
|
8222
9172
|
}
|
|
8223
|
-
|
|
8224
|
-
if (this.host !== "opencode") {
|
|
8225
|
-
localProjectIndexPaths.push(path12.join(this.projectRoot, getHostProjectIndexRelativePath("opencode")));
|
|
8226
|
-
}
|
|
8227
|
-
const isLocalProjectIndex = localProjectIndexPaths.some(
|
|
8228
|
-
(localPath) => path12.resolve(this.indexPath) === path12.resolve(localPath)
|
|
8229
|
-
);
|
|
8230
|
-
if (!isLocalProjectIndex) {
|
|
9173
|
+
if (!this.isLocalProjectIndexPath()) {
|
|
8231
9174
|
throw new Error(
|
|
8232
9175
|
"Project-scoped force rebuild is unsafe while using an inherited worktree index. Create a local project config boundary before clearing the index."
|
|
8233
9176
|
);
|
|
@@ -8235,7 +9178,7 @@ var Indexer = class {
|
|
|
8235
9178
|
store.clear();
|
|
8236
9179
|
store.save();
|
|
8237
9180
|
invertedIndex.clear();
|
|
8238
|
-
|
|
9181
|
+
this.saveInvertedIndex(invertedIndex);
|
|
8239
9182
|
this.fileHashCache.clear();
|
|
8240
9183
|
this.saveFileHashCache();
|
|
8241
9184
|
database.clearAllIndexedData();
|
|
@@ -8253,7 +9196,13 @@ var Indexer = class {
|
|
|
8253
9196
|
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
8254
9197
|
}
|
|
8255
9198
|
async healthCheck() {
|
|
8256
|
-
|
|
9199
|
+
return this.withIndexMutationLease("health-check", async (recoveredOwners) => {
|
|
9200
|
+
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
9201
|
+
return this.healthCheckUnlocked();
|
|
9202
|
+
});
|
|
9203
|
+
}
|
|
9204
|
+
async healthCheckUnlocked() {
|
|
9205
|
+
const { store, invertedIndex, database } = this.requireLoadedIndexState();
|
|
8257
9206
|
this.logger.gc("info", "Starting health check");
|
|
8258
9207
|
const allMetadata = store.getAllMetadata();
|
|
8259
9208
|
const filePathsToChunkKeys = /* @__PURE__ */ new Map();
|
|
@@ -8266,7 +9215,7 @@ var Indexer = class {
|
|
|
8266
9215
|
const removedChunkKeys = [];
|
|
8267
9216
|
const chunkKeysByRemovedFile = /* @__PURE__ */ new Map();
|
|
8268
9217
|
for (const [filePath, chunkKeys] of filePathsToChunkKeys) {
|
|
8269
|
-
if (!
|
|
9218
|
+
if (!existsSync8(filePath)) {
|
|
8270
9219
|
chunkKeysByRemovedFile.set(filePath, chunkKeys);
|
|
8271
9220
|
for (const key of chunkKeys) {
|
|
8272
9221
|
removedChunkKeys.push(key);
|
|
@@ -8291,7 +9240,7 @@ var Indexer = class {
|
|
|
8291
9240
|
const removedCount = removedChunkKeys.length;
|
|
8292
9241
|
if (removedCount > 0) {
|
|
8293
9242
|
store.save();
|
|
8294
|
-
|
|
9243
|
+
this.saveInvertedIndex(invertedIndex);
|
|
8295
9244
|
}
|
|
8296
9245
|
let gcOrphanEmbeddings;
|
|
8297
9246
|
let gcOrphanChunks;
|
|
@@ -8306,7 +9255,7 @@ var Indexer = class {
|
|
|
8306
9255
|
if (!await this.tryResetCorruptedIndex("running index health check", error)) {
|
|
8307
9256
|
throw error;
|
|
8308
9257
|
}
|
|
8309
|
-
await this.
|
|
9258
|
+
await this.initializeUnlocked("writer", [], { skipAutoGc: true });
|
|
8310
9259
|
return {
|
|
8311
9260
|
removed: 0,
|
|
8312
9261
|
filePaths: [],
|
|
@@ -8315,7 +9264,7 @@ var Indexer = class {
|
|
|
8315
9264
|
gcOrphanSymbols: 0,
|
|
8316
9265
|
gcOrphanCallEdges: 0,
|
|
8317
9266
|
resetCorruptedIndex: true,
|
|
8318
|
-
warning: this.getCorruptedIndexWarning(
|
|
9267
|
+
warning: this.getCorruptedIndexWarning(path13.join(this.indexPath, "codebase.db"))
|
|
8319
9268
|
};
|
|
8320
9269
|
}
|
|
8321
9270
|
this.logger.recordGc(removedCount, gcOrphanChunks, gcOrphanEmbeddings);
|
|
@@ -8328,7 +9277,13 @@ var Indexer = class {
|
|
|
8328
9277
|
return { removed: removedCount, filePaths: removedFilePaths, gcOrphanEmbeddings, gcOrphanChunks, gcOrphanSymbols, gcOrphanCallEdges };
|
|
8329
9278
|
}
|
|
8330
9279
|
async retryFailedBatches() {
|
|
8331
|
-
|
|
9280
|
+
return this.withIndexMutationLease("retry-failed-batches", async (recoveredOwners) => {
|
|
9281
|
+
await this.ensureInitializedUnlocked(recoveredOwners);
|
|
9282
|
+
return this.retryFailedBatchesUnlocked();
|
|
9283
|
+
});
|
|
9284
|
+
}
|
|
9285
|
+
async retryFailedBatchesUnlocked() {
|
|
9286
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = this.requireLoadedIndexState();
|
|
8332
9287
|
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
8333
9288
|
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
8334
9289
|
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
@@ -8492,7 +9447,7 @@ var Indexer = class {
|
|
|
8492
9447
|
}
|
|
8493
9448
|
if (succeeded > 0) {
|
|
8494
9449
|
store.save();
|
|
8495
|
-
|
|
9450
|
+
this.saveInvertedIndex(invertedIndex);
|
|
8496
9451
|
}
|
|
8497
9452
|
if (roots && succeeded > 0 && persistedStillFailing.length === 0 && this.hasProjectForceReembedPending()) {
|
|
8498
9453
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
@@ -8520,15 +9475,16 @@ var Indexer = class {
|
|
|
8520
9475
|
}
|
|
8521
9476
|
}
|
|
8522
9477
|
async getDatabaseStats() {
|
|
8523
|
-
const { database } = await this.ensureInitialized();
|
|
9478
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9479
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8524
9480
|
return database.getStats();
|
|
8525
9481
|
}
|
|
8526
9482
|
getLogger() {
|
|
8527
9483
|
return this.logger;
|
|
8528
9484
|
}
|
|
8529
9485
|
async findSimilar(code, limit = this.config.search.maxResults, options) {
|
|
8530
|
-
const { store, provider, database } = await this.ensureInitialized();
|
|
8531
|
-
|
|
9486
|
+
const { store, provider, database, readIssues, compatibility } = await this.ensureInitialized();
|
|
9487
|
+
this.requireReadableComponents(readIssues, "vectors", "database");
|
|
8532
9488
|
if (!compatibility.compatible) {
|
|
8533
9489
|
throw new Error(
|
|
8534
9490
|
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} Run index_codebase with force=true to rebuild the index.`
|
|
@@ -8642,7 +9598,8 @@ var Indexer = class {
|
|
|
8642
9598
|
);
|
|
8643
9599
|
}
|
|
8644
9600
|
async getCallers(targetName, callTypeFilter) {
|
|
8645
|
-
const { database } = await this.ensureInitialized();
|
|
9601
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9602
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8646
9603
|
const seen = /* @__PURE__ */ new Set();
|
|
8647
9604
|
const results = [];
|
|
8648
9605
|
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
@@ -8656,7 +9613,8 @@ var Indexer = class {
|
|
|
8656
9613
|
return results;
|
|
8657
9614
|
}
|
|
8658
9615
|
async getCallees(symbolId, callTypeFilter) {
|
|
8659
|
-
const { database } = await this.ensureInitialized();
|
|
9616
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9617
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8660
9618
|
const seen = /* @__PURE__ */ new Set();
|
|
8661
9619
|
const results = [];
|
|
8662
9620
|
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
@@ -8670,43 +9628,50 @@ var Indexer = class {
|
|
|
8670
9628
|
return results;
|
|
8671
9629
|
}
|
|
8672
9630
|
async findCallPath(fromName, toName, maxDepth) {
|
|
8673
|
-
const { database } = await this.ensureInitialized();
|
|
9631
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9632
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8674
9633
|
let shortest = [];
|
|
8675
9634
|
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
8676
|
-
const
|
|
8677
|
-
if (
|
|
8678
|
-
shortest =
|
|
9635
|
+
const path17 = database.findShortestPath(fromName, toName, branchKey, maxDepth);
|
|
9636
|
+
if (path17.length > 0 && (shortest.length === 0 || path17.length < shortest.length)) {
|
|
9637
|
+
shortest = path17;
|
|
8679
9638
|
}
|
|
8680
9639
|
}
|
|
8681
9640
|
return shortest;
|
|
8682
9641
|
}
|
|
8683
9642
|
async getSymbolsForBranch(branch) {
|
|
8684
|
-
const { database } = await this.ensureInitialized();
|
|
9643
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9644
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8685
9645
|
const resolvedBranch = branch ?? this.getBranchCatalogKey();
|
|
8686
9646
|
return database.getSymbolsForBranch(resolvedBranch);
|
|
8687
9647
|
}
|
|
8688
9648
|
async getSymbolsForFiles(filePaths, branch) {
|
|
8689
|
-
const { database } = await this.ensureInitialized();
|
|
9649
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9650
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8690
9651
|
const resolvedBranch = branch ?? this.getBranchCatalogKey();
|
|
8691
9652
|
return database.getSymbolsForFiles(filePaths, resolvedBranch);
|
|
8692
9653
|
}
|
|
8693
9654
|
async getTransitiveReachability(rootSymbolIds, direction, maxDepth) {
|
|
8694
|
-
const { database } = await this.ensureInitialized();
|
|
9655
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9656
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8695
9657
|
const branch = this.getBranchCatalogKey();
|
|
8696
9658
|
return database.getTransitiveReachability(rootSymbolIds, branch, direction, maxDepth);
|
|
8697
9659
|
}
|
|
8698
9660
|
async detectCommunities(branch, symbolIds) {
|
|
8699
|
-
const { database } = await this.ensureInitialized();
|
|
9661
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9662
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8700
9663
|
const resolvedBranch = branch ?? this.getBranchCatalogKey();
|
|
8701
9664
|
return database.detectCommunities(resolvedBranch, symbolIds);
|
|
8702
9665
|
}
|
|
8703
9666
|
async computeCentrality(branch) {
|
|
8704
|
-
const { database } = await this.ensureInitialized();
|
|
9667
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9668
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8705
9669
|
const resolvedBranch = branch ?? this.getBranchCatalogKey();
|
|
8706
9670
|
return database.computeCentrality(resolvedBranch);
|
|
8707
9671
|
}
|
|
8708
9672
|
async getPrImpact(opts) {
|
|
8709
|
-
const { database } = await this.ensureInitialized();
|
|
9673
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
9674
|
+
this.requireReadableComponents(readIssues, "database");
|
|
8710
9675
|
const execFileAsync3 = promisify3(execFile3);
|
|
8711
9676
|
const changedFilesResult = await getChangedFiles({
|
|
8712
9677
|
pr: opts.pr,
|
|
@@ -8729,7 +9694,7 @@ var Indexer = class {
|
|
|
8729
9694
|
"Run index_codebase first to build the call graph and symbol index for this branch."
|
|
8730
9695
|
);
|
|
8731
9696
|
}
|
|
8732
|
-
const absoluteChangedFiles = changedFiles.map((f) =>
|
|
9697
|
+
const absoluteChangedFiles = changedFiles.map((f) => path13.resolve(this.projectRoot, f));
|
|
8733
9698
|
const directSymbols = database.getSymbolsForFiles(absoluteChangedFiles, branchKey);
|
|
8734
9699
|
const directIds = directSymbols.map((s) => s.id);
|
|
8735
9700
|
const direction = opts.direction ?? "both";
|
|
@@ -8812,7 +9777,7 @@ var Indexer = class {
|
|
|
8812
9777
|
projectRoot: this.projectRoot,
|
|
8813
9778
|
baseBranch: this.baseBranch
|
|
8814
9779
|
});
|
|
8815
|
-
const otherAbsolute = otherChanged.files.map((f) =>
|
|
9780
|
+
const otherAbsolute = otherChanged.files.map((f) => path13.resolve(this.projectRoot, f));
|
|
8816
9781
|
const otherBranchKey = this.getBranchCatalogKeyFor(openPr.headRefName);
|
|
8817
9782
|
const otherSymbols = database.getSymbolsForFiles(otherAbsolute, otherBranchKey);
|
|
8818
9783
|
const otherLabels = /* @__PURE__ */ new Set();
|
|
@@ -8862,7 +9827,8 @@ var Indexer = class {
|
|
|
8862
9827
|
};
|
|
8863
9828
|
}
|
|
8864
9829
|
async getVisualizationData(options) {
|
|
8865
|
-
const { database, store } = await this.ensureInitialized();
|
|
9830
|
+
const { database, store, readIssues } = await this.ensureInitialized();
|
|
9831
|
+
this.requireReadableComponents(readIssues, "vectors", "database");
|
|
8866
9832
|
const seenSymbols = /* @__PURE__ */ new Map();
|
|
8867
9833
|
const seenEdges = /* @__PURE__ */ new Map();
|
|
8868
9834
|
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
@@ -8875,12 +9841,12 @@ var Indexer = class {
|
|
|
8875
9841
|
if (meta.filePath) filePaths.add(meta.filePath);
|
|
8876
9842
|
}
|
|
8877
9843
|
const directory = options?.directory?.replace(/\/$/, "");
|
|
8878
|
-
const absoluteDirectoryFilter = directory ?
|
|
9844
|
+
const absoluteDirectoryFilter = directory ? path13.resolve(this.projectRoot, directory) : void 0;
|
|
8879
9845
|
for (const filePath of filePaths) {
|
|
8880
9846
|
if (directory) {
|
|
8881
|
-
const absoluteFilePath =
|
|
9847
|
+
const absoluteFilePath = path13.resolve(filePath);
|
|
8882
9848
|
const matchesRelative = filePath === directory || filePath.startsWith(directory + "/");
|
|
8883
|
-
const matchesProjectRelative = absoluteDirectoryFilter !== void 0 && (absoluteFilePath === absoluteDirectoryFilter || absoluteFilePath.startsWith(absoluteDirectoryFilter +
|
|
9849
|
+
const matchesProjectRelative = absoluteDirectoryFilter !== void 0 && (absoluteFilePath === absoluteDirectoryFilter || absoluteFilePath.startsWith(absoluteDirectoryFilter + path13.sep));
|
|
8884
9850
|
if (!matchesRelative && !matchesProjectRelative) {
|
|
8885
9851
|
continue;
|
|
8886
9852
|
}
|
|
@@ -8902,53 +9868,64 @@ var Indexer = class {
|
|
|
8902
9868
|
return { symbols: [...seenSymbols.values()], edges: [...seenEdges.values()] };
|
|
8903
9869
|
}
|
|
8904
9870
|
async close() {
|
|
8905
|
-
|
|
9871
|
+
this.database?.close();
|
|
9872
|
+
for (const database of this.retiredDatabases) {
|
|
9873
|
+
database.close();
|
|
9874
|
+
}
|
|
9875
|
+
this.retiredDatabases = [];
|
|
8906
9876
|
this.database = null;
|
|
8907
9877
|
this.store = null;
|
|
8908
9878
|
this.invertedIndex = null;
|
|
8909
9879
|
this.provider = null;
|
|
8910
9880
|
this.reranker = null;
|
|
9881
|
+
this.configuredProviderInfo = null;
|
|
9882
|
+
this.indexCompatibility = null;
|
|
9883
|
+
this.initializationMode = "none";
|
|
9884
|
+
this.readIssues = [];
|
|
9885
|
+
this.readerArtifactFingerprint = null;
|
|
9886
|
+
this.writerArtifactFingerprint = null;
|
|
9887
|
+
this.readerArtifactRetryAfter.clear();
|
|
8911
9888
|
}
|
|
8912
9889
|
};
|
|
8913
9890
|
|
|
8914
9891
|
// src/tools/knowledge-base-paths.ts
|
|
8915
|
-
import * as
|
|
9892
|
+
import * as path14 from "path";
|
|
8916
9893
|
function resolveConfigPathValue(value, baseDir) {
|
|
8917
9894
|
const trimmed = value.trim();
|
|
8918
9895
|
if (!trimmed) {
|
|
8919
9896
|
return trimmed;
|
|
8920
9897
|
}
|
|
8921
|
-
const absolutePath =
|
|
8922
|
-
return
|
|
9898
|
+
const absolutePath = path14.isAbsolute(trimmed) ? trimmed : path14.resolve(baseDir, trimmed);
|
|
9899
|
+
return path14.normalize(absolutePath);
|
|
8923
9900
|
}
|
|
8924
9901
|
function serializeConfigPathValue(value, baseDir) {
|
|
8925
9902
|
const trimmed = value.trim();
|
|
8926
9903
|
if (!trimmed) {
|
|
8927
9904
|
return trimmed;
|
|
8928
9905
|
}
|
|
8929
|
-
if (!
|
|
8930
|
-
return normalizePathSeparators(
|
|
9906
|
+
if (!path14.isAbsolute(trimmed)) {
|
|
9907
|
+
return normalizePathSeparators(path14.normalize(trimmed));
|
|
8931
9908
|
}
|
|
8932
|
-
const relativePath =
|
|
8933
|
-
if (!relativePath || !relativePath.startsWith("..") && !
|
|
8934
|
-
return normalizePathSeparators(
|
|
9909
|
+
const relativePath = path14.relative(baseDir, trimmed);
|
|
9910
|
+
if (!relativePath || !relativePath.startsWith("..") && !path14.isAbsolute(relativePath)) {
|
|
9911
|
+
return normalizePathSeparators(path14.normalize(relativePath || "."));
|
|
8935
9912
|
}
|
|
8936
|
-
return
|
|
9913
|
+
return path14.normalize(trimmed);
|
|
8937
9914
|
}
|
|
8938
9915
|
function resolveKnowledgeBasePath(value, projectRoot3) {
|
|
8939
|
-
return
|
|
9916
|
+
return path14.isAbsolute(value) ? value : path14.resolve(projectRoot3, value);
|
|
8940
9917
|
}
|
|
8941
9918
|
function normalizeKnowledgeBasePath2(value, projectRoot3) {
|
|
8942
|
-
return
|
|
9919
|
+
return path14.normalize(resolveKnowledgeBasePath(value, projectRoot3));
|
|
8943
9920
|
}
|
|
8944
9921
|
function hasMatchingKnowledgeBasePath(knowledgeBases, inputPath, projectRoot3) {
|
|
8945
|
-
const normalizedInput =
|
|
9922
|
+
const normalizedInput = path14.normalize(inputPath);
|
|
8946
9923
|
return knowledgeBases.some((kb) => normalizeKnowledgeBasePath2(kb, projectRoot3) === normalizedInput);
|
|
8947
9924
|
}
|
|
8948
9925
|
function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot3) {
|
|
8949
|
-
const normalizedInput =
|
|
9926
|
+
const normalizedInput = path14.normalize(inputPath);
|
|
8950
9927
|
return knowledgeBases.findIndex(
|
|
8951
|
-
(kb) =>
|
|
9928
|
+
(kb) => path14.normalize(kb) === normalizedInput || normalizeKnowledgeBasePath2(kb, projectRoot3) === normalizedInput
|
|
8952
9929
|
);
|
|
8953
9930
|
}
|
|
8954
9931
|
|
|
@@ -9048,6 +10025,10 @@ function formatStatus(status) {
|
|
|
9048
10025
|
lines.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
9049
10026
|
}
|
|
9050
10027
|
}
|
|
10028
|
+
if (status.warning) {
|
|
10029
|
+
lines.push("");
|
|
10030
|
+
lines.push(`INDEX WARNING: ${status.warning}`);
|
|
10031
|
+
}
|
|
9051
10032
|
if (status.compatibility && !status.compatibility.compatible) {
|
|
9052
10033
|
lines.push("");
|
|
9053
10034
|
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
@@ -9093,6 +10074,17 @@ function calculatePercentage(progress) {
|
|
|
9093
10074
|
if (progress.phase === "storing") return 95;
|
|
9094
10075
|
return 0;
|
|
9095
10076
|
}
|
|
10077
|
+
function formatCodebasePeek(results) {
|
|
10078
|
+
if (results.length === 0) {
|
|
10079
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
10080
|
+
}
|
|
10081
|
+
const formatted = results.map((r, idx) => {
|
|
10082
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
10083
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
10084
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})${formatBlame(r)}`;
|
|
10085
|
+
});
|
|
10086
|
+
return formatted.join("\n");
|
|
10087
|
+
}
|
|
9096
10088
|
function formatHealthCheck(result) {
|
|
9097
10089
|
if (result.resetCorruptedIndex) {
|
|
9098
10090
|
return result.warning ?? "Detected a corrupted local index and reset it. Run index_codebase to rebuild search data.";
|
|
@@ -9142,16 +10134,16 @@ function formatCallGraphCallees(symbolId, callees, relationshipType) {
|
|
|
9142
10134
|
return `[${index + 1}] \u2192 ${edge.targetName} (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? ` [resolved: ${edge.toSymbolId}]` : " [unresolved]"}`;
|
|
9143
10135
|
}).join("\n");
|
|
9144
10136
|
}
|
|
9145
|
-
function formatCallGraphPath(from, to,
|
|
9146
|
-
if (
|
|
10137
|
+
function formatCallGraphPath(from, to, path17) {
|
|
10138
|
+
if (path17.length === 0) {
|
|
9147
10139
|
return `No path found between "${from}" and "${to}". They may be in disconnected components, or the call graph index needs updating.`;
|
|
9148
10140
|
}
|
|
9149
|
-
const formatted =
|
|
10141
|
+
const formatted = path17.map((hop, index) => {
|
|
9150
10142
|
const prefix = index === 0 ? "[start]" : `--${hop.callType}-->`;
|
|
9151
10143
|
const location = hop.filePath ? ` (${hop.filePath}:${hop.line})` : "";
|
|
9152
10144
|
return `${prefix} ${hop.symbolName}${location}`;
|
|
9153
10145
|
});
|
|
9154
|
-
return `Path (${
|
|
10146
|
+
return `Path (${path17.length} hops):
|
|
9155
10147
|
${formatted.join("\n")}`;
|
|
9156
10148
|
}
|
|
9157
10149
|
function formatResultHeader(result, index) {
|
|
@@ -9191,8 +10183,8 @@ ${truncateContent(r.content)}
|
|
|
9191
10183
|
}
|
|
9192
10184
|
|
|
9193
10185
|
// src/tools/config-state.ts
|
|
9194
|
-
import { existsSync as
|
|
9195
|
-
import * as
|
|
10186
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync4, writeFileSync as writeFileSync4 } from "fs";
|
|
10187
|
+
import * as path15 from "path";
|
|
9196
10188
|
function normalizeKnowledgeBasePaths(config, projectRoot3) {
|
|
9197
10189
|
const normalized = { ...config };
|
|
9198
10190
|
if (Array.isArray(normalized.knowledgeBases)) {
|
|
@@ -9219,10 +10211,10 @@ function loadEditableConfig(projectRoot3, host = "opencode") {
|
|
|
9219
10211
|
}
|
|
9220
10212
|
function saveConfig(projectRoot3, config, host = "opencode") {
|
|
9221
10213
|
const configPath = getConfigPath(projectRoot3, host);
|
|
9222
|
-
const configDir =
|
|
9223
|
-
const configBaseDir =
|
|
9224
|
-
if (!
|
|
9225
|
-
|
|
10214
|
+
const configDir = path15.dirname(configPath);
|
|
10215
|
+
const configBaseDir = path15.dirname(configDir);
|
|
10216
|
+
if (!existsSync9(configDir)) {
|
|
10217
|
+
mkdirSync4(configDir, { recursive: true });
|
|
9226
10218
|
}
|
|
9227
10219
|
const serializableConfig = { ...config };
|
|
9228
10220
|
if (Array.isArray(serializableConfig.knowledgeBases)) {
|
|
@@ -9230,13 +10222,31 @@ function saveConfig(projectRoot3, config, host = "opencode") {
|
|
|
9230
10222
|
(kb) => serializeConfigPathValue(kb, configBaseDir)
|
|
9231
10223
|
);
|
|
9232
10224
|
}
|
|
9233
|
-
|
|
10225
|
+
writeFileSync4(configPath, JSON.stringify(serializableConfig, null, 2) + "\n", "utf-8");
|
|
9234
10226
|
}
|
|
9235
10227
|
|
|
9236
10228
|
// src/tools/operations.ts
|
|
9237
10229
|
var indexerCache = /* @__PURE__ */ new Map();
|
|
9238
10230
|
var configCache = /* @__PURE__ */ new Map();
|
|
9239
10231
|
var defaultProjectRoots = /* @__PURE__ */ new Map();
|
|
10232
|
+
function getIndexBusyResult(error) {
|
|
10233
|
+
if (!isIndexLockContentionError(error)) return null;
|
|
10234
|
+
const owner = error.owner;
|
|
10235
|
+
const ownerText = owner ? `PID ${owner.pid}, operation ${owner.operation}, since ${owner.startedAt}` : "unreadable owner";
|
|
10236
|
+
if (error.reason === "legacy-lock") {
|
|
10237
|
+
return {
|
|
10238
|
+
kind: "busy",
|
|
10239
|
+
text: `INDEX_BUSY: legacy lock format detected (${ownerText}). Verify the PID and remove this lock manually only if it is stale.`
|
|
10240
|
+
};
|
|
10241
|
+
}
|
|
10242
|
+
if (error.reason === "unknown-owner") {
|
|
10243
|
+
return {
|
|
10244
|
+
kind: "busy",
|
|
10245
|
+
text: `INDEX_BUSY: unreadable or remote lock owner (${ownerText}). Automatic recovery was refused; manual verification is required.`
|
|
10246
|
+
};
|
|
10247
|
+
}
|
|
10248
|
+
return { kind: "busy", text: `INDEX_BUSY: another index operation is already in progress (${ownerText}).` };
|
|
10249
|
+
}
|
|
9240
10250
|
function getProjectRoot(projectRoot3, host) {
|
|
9241
10251
|
if (projectRoot3) {
|
|
9242
10252
|
return projectRoot3;
|
|
@@ -9274,8 +10284,8 @@ function refreshIndexerForDirectory(projectRoot3, host = "opencode", config = pa
|
|
|
9274
10284
|
}
|
|
9275
10285
|
function shouldForceLocalizeProjectIndex(projectRoot3, host = "opencode") {
|
|
9276
10286
|
const root = getProjectRoot(projectRoot3, host);
|
|
9277
|
-
const localIndexPath =
|
|
9278
|
-
if (
|
|
10287
|
+
const localIndexPath = path16.join(root, getHostProjectIndexRelativePath(host));
|
|
10288
|
+
if (existsSync10(localIndexPath)) {
|
|
9279
10289
|
return false;
|
|
9280
10290
|
}
|
|
9281
10291
|
const inheritedIndexPath = resolveWorktreeFallbackProjectIndexPath(root, host);
|
|
@@ -9331,35 +10341,46 @@ async function getCallGraphPath(projectRoot3, host, from, to, maxDepth) {
|
|
|
9331
10341
|
async function runIndexCodebase(projectRoot3, host, args, onProgress) {
|
|
9332
10342
|
const root = getProjectRoot(projectRoot3, host);
|
|
9333
10343
|
const key = getIndexerCacheKey(root, host);
|
|
9334
|
-
const cachedConfig = configCache.get(key);
|
|
9335
10344
|
let indexer = getIndexerForProject(root, host);
|
|
9336
|
-
|
|
9337
|
-
|
|
9338
|
-
|
|
9339
|
-
|
|
9340
|
-
|
|
9341
|
-
|
|
9342
|
-
|
|
9343
|
-
|
|
9344
|
-
|
|
9345
|
-
|
|
9346
|
-
|
|
9347
|
-
|
|
9348
|
-
|
|
9349
|
-
|
|
9350
|
-
|
|
9351
|
-
|
|
9352
|
-
|
|
9353
|
-
|
|
9354
|
-
|
|
9355
|
-
chunksProcessed: progress.chunksProcessed,
|
|
9356
|
-
totalChunks: progress.totalChunks,
|
|
9357
|
-
percentage: calculatePercentage(progress)
|
|
10345
|
+
const runtimeConfig = configCache.get(key);
|
|
10346
|
+
try {
|
|
10347
|
+
if (args.estimateOnly) {
|
|
10348
|
+
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
10349
|
+
}
|
|
10350
|
+
const runIndex = async (target) => {
|
|
10351
|
+
const operation = args.force ? target.forceIndex.bind(target) : target.index.bind(target);
|
|
10352
|
+
return operation((progress) => {
|
|
10353
|
+
if (onProgress) {
|
|
10354
|
+
return onProgress(formatProgressTitle(progress), {
|
|
10355
|
+
phase: progress.phase,
|
|
10356
|
+
filesProcessed: progress.filesProcessed,
|
|
10357
|
+
totalFiles: progress.totalFiles,
|
|
10358
|
+
chunksProcessed: progress.chunksProcessed,
|
|
10359
|
+
totalChunks: progress.totalChunks,
|
|
10360
|
+
percentage: calculatePercentage(progress)
|
|
10361
|
+
});
|
|
10362
|
+
}
|
|
10363
|
+
return Promise.resolve();
|
|
9358
10364
|
});
|
|
10365
|
+
};
|
|
10366
|
+
let stats;
|
|
10367
|
+
if (args.force && shouldForceLocalizeProjectIndex(root, host)) {
|
|
10368
|
+
const inheritedIndexPath = resolveProjectIndexPath(root, runtimeConfig.scope, host);
|
|
10369
|
+
stats = await withIndexLock(inheritedIndexPath, "force-index", async () => {
|
|
10370
|
+
materializeLocalProjectConfig(root, loadProjectConfigLayer(root, host), host);
|
|
10371
|
+
refreshIndexerForDirectory(root, host, runtimeConfig);
|
|
10372
|
+
indexer = getIndexerForProject(root, host);
|
|
10373
|
+
return runIndex(indexer);
|
|
10374
|
+
}, { completeRecoveries: false });
|
|
10375
|
+
} else {
|
|
10376
|
+
stats = await runIndex(indexer);
|
|
9359
10377
|
}
|
|
9360
|
-
return
|
|
9361
|
-
})
|
|
9362
|
-
|
|
10378
|
+
return { kind: "stats", stats };
|
|
10379
|
+
} catch (error) {
|
|
10380
|
+
const busyResult = getIndexBusyResult(error);
|
|
10381
|
+
if (!busyResult) throw error;
|
|
10382
|
+
return busyResult;
|
|
10383
|
+
}
|
|
9363
10384
|
}
|
|
9364
10385
|
async function getIndexStatus(projectRoot3, host) {
|
|
9365
10386
|
const indexer = getIndexerForProject(projectRoot3, host);
|
|
@@ -9369,6 +10390,15 @@ async function getIndexHealthCheck(projectRoot3, host) {
|
|
|
9369
10390
|
const indexer = getIndexerForProject(projectRoot3, host);
|
|
9370
10391
|
return indexer.healthCheck();
|
|
9371
10392
|
}
|
|
10393
|
+
async function runIndexHealthCheck(projectRoot3, host) {
|
|
10394
|
+
try {
|
|
10395
|
+
return { kind: "health", health: await getIndexHealthCheck(projectRoot3, host) };
|
|
10396
|
+
} catch (error) {
|
|
10397
|
+
const busyResult = getIndexBusyResult(error);
|
|
10398
|
+
if (!busyResult) throw error;
|
|
10399
|
+
return busyResult;
|
|
10400
|
+
}
|
|
10401
|
+
}
|
|
9372
10402
|
async function getPrImpact(projectRoot3, host, params) {
|
|
9373
10403
|
const indexer = getIndexerForProject(projectRoot3, host);
|
|
9374
10404
|
return indexer.getPrImpact({
|
|
@@ -9435,15 +10465,15 @@ async function getIndexLogs(projectRoot3, host, args) {
|
|
|
9435
10465
|
function addKnowledgeBase(projectRoot3, host, knowledgeBasePath) {
|
|
9436
10466
|
const root = getProjectRoot(projectRoot3, host);
|
|
9437
10467
|
const inputPath = knowledgeBasePath.trim();
|
|
9438
|
-
const normalizedPath =
|
|
9439
|
-
|
|
10468
|
+
const normalizedPath = path16.resolve(
|
|
10469
|
+
path16.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
|
|
9440
10470
|
);
|
|
9441
|
-
if (!
|
|
10471
|
+
if (!existsSync10(normalizedPath)) {
|
|
9442
10472
|
return `Error: Directory does not exist: ${normalizedPath}`;
|
|
9443
10473
|
}
|
|
9444
10474
|
let realPath;
|
|
9445
10475
|
try {
|
|
9446
|
-
realPath =
|
|
10476
|
+
realPath = realpathSync2(normalizedPath);
|
|
9447
10477
|
} catch {
|
|
9448
10478
|
return `Error: Cannot resolve path: ${normalizedPath}`;
|
|
9449
10479
|
}
|
|
@@ -9472,13 +10502,13 @@ function addKnowledgeBase(projectRoot3, host, knowledgeBasePath) {
|
|
|
9472
10502
|
}
|
|
9473
10503
|
}
|
|
9474
10504
|
for (const dotDir of sensitiveDotDirs) {
|
|
9475
|
-
const sensitiveDir =
|
|
10505
|
+
const sensitiveDir = path16.join(homeDir, dotDir);
|
|
9476
10506
|
if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
|
|
9477
10507
|
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath}`;
|
|
9478
10508
|
}
|
|
9479
10509
|
}
|
|
9480
10510
|
try {
|
|
9481
|
-
const stat =
|
|
10511
|
+
const stat = statSync4(normalizedPath);
|
|
9482
10512
|
if (!stat.isDirectory()) {
|
|
9483
10513
|
return `Error: Path is not a directory: ${normalizedPath}`;
|
|
9484
10514
|
}
|
|
@@ -9518,7 +10548,7 @@ function listKnowledgeBases(projectRoot3, host) {
|
|
|
9518
10548
|
for (let i = 0; i < knowledgeBases.length; i++) {
|
|
9519
10549
|
const kb = knowledgeBases[i];
|
|
9520
10550
|
const resolvedPath = resolveKnowledgeBasePath(kb, root);
|
|
9521
|
-
const exists =
|
|
10551
|
+
const exists = existsSync10(resolvedPath);
|
|
9522
10552
|
result += `[${i + 1}] ${kb}
|
|
9523
10553
|
`;
|
|
9524
10554
|
result += ` Resolved: ${resolvedPath}
|
|
@@ -9527,7 +10557,7 @@ function listKnowledgeBases(projectRoot3, host) {
|
|
|
9527
10557
|
`;
|
|
9528
10558
|
if (exists) {
|
|
9529
10559
|
try {
|
|
9530
|
-
const stat =
|
|
10560
|
+
const stat = statSync4(resolvedPath);
|
|
9531
10561
|
result += ` Type: ${stat.isDirectory() ? "Directory" : "File"}
|
|
9532
10562
|
`;
|
|
9533
10563
|
} catch {
|
|
@@ -9535,7 +10565,7 @@ function listKnowledgeBases(projectRoot3, host) {
|
|
|
9535
10565
|
}
|
|
9536
10566
|
result += "\n";
|
|
9537
10567
|
}
|
|
9538
|
-
const hasHostConfig =
|
|
10568
|
+
const hasHostConfig = existsSync10(path16.join(root, getHostProjectConfigRelativePath(host)));
|
|
9539
10569
|
if (hasHostConfig) {
|
|
9540
10570
|
result += `
|
|
9541
10571
|
Config sources: 1 file(s).`;
|
|
@@ -9641,10 +10671,67 @@ function projectRoot2(ctx) {
|
|
|
9641
10671
|
return ctx?.cwd ?? process.cwd();
|
|
9642
10672
|
}
|
|
9643
10673
|
function codebaseIndexPiExtension(pi) {
|
|
10674
|
+
pi.registerTool({
|
|
10675
|
+
name: "codebase_context",
|
|
10676
|
+
label: "Codebase Context",
|
|
10677
|
+
description: "PREFERRED FIRST TOOL for any repository question. Check index_status when freshness is unknown, then use this tool for low-token location discovery and dependency flow.",
|
|
10678
|
+
parameters: Type2.Object({
|
|
10679
|
+
query: Type2.String({ description: "Natural language description of what code you're trying to locate" }),
|
|
10680
|
+
from: Type2.Optional(Type2.String({ description: "Source symbol when asking for a dependency path." })),
|
|
10681
|
+
to: Type2.Optional(Type2.String({ description: "Target symbol when asking for a dependency path." })),
|
|
10682
|
+
symbol: Type2.Optional(Type2.String({ description: "Exact symbol name for an authoritative definition lookup." })),
|
|
10683
|
+
limit: Type2.Optional(Type2.Number({ description: "Maximum results (default: 10)" })),
|
|
10684
|
+
maxDepth: Type2.Optional(Type2.Number({ description: "Maximum call-graph traversal depth for from/to paths" })),
|
|
10685
|
+
fileType: Type2.Optional(Type2.String({ description: "Filter by file extension, e.g., ts, py, rs" })),
|
|
10686
|
+
directory: Type2.Optional(Type2.String({ description: "Filter by directory path" }))
|
|
10687
|
+
}),
|
|
10688
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
10689
|
+
const root = projectRoot2(ctx);
|
|
10690
|
+
if (params.from && params.to) {
|
|
10691
|
+
const path17 = await getCallGraphPath(root, HOST2, params.from, params.to, params.maxDepth);
|
|
10692
|
+
if (path17.length > 0) {
|
|
10693
|
+
return text2(formatCallGraphPath(params.from, params.to, path17), path17);
|
|
10694
|
+
}
|
|
10695
|
+
const { callers } = await getCallGraphData(root, HOST2, {
|
|
10696
|
+
name: params.to,
|
|
10697
|
+
direction: "callers"
|
|
10698
|
+
});
|
|
10699
|
+
const directEdge = callers.find((edge) => edge.fromSymbolName === params.from);
|
|
10700
|
+
if (directEdge) {
|
|
10701
|
+
const location = directEdge.fromSymbolFilePath ? ` at ${directEdge.fromSymbolFilePath}:${directEdge.line}` : "";
|
|
10702
|
+
return text2(
|
|
10703
|
+
`Direct path: ${params.from} --${directEdge.callType}--> ${params.to}${location} (edge is ${directEdge.isResolved ? "resolved" : "unresolved"}).`,
|
|
10704
|
+
path17
|
|
10705
|
+
);
|
|
10706
|
+
}
|
|
10707
|
+
return text2(formatCallGraphPath(params.from, params.to, path17), path17);
|
|
10708
|
+
}
|
|
10709
|
+
if (params.symbol) {
|
|
10710
|
+
const results2 = await implementationLookup(root, HOST2, params.symbol, {
|
|
10711
|
+
limit: params.limit ?? 10,
|
|
10712
|
+
fileType: params.fileType,
|
|
10713
|
+
directory: params.directory
|
|
10714
|
+
});
|
|
10715
|
+
return text2(formatDefinitionLookup(results2, params.symbol), results2);
|
|
10716
|
+
}
|
|
10717
|
+
const results = await searchCodebase(root, HOST2, params.query, {
|
|
10718
|
+
limit: params.limit ?? 10,
|
|
10719
|
+
fileType: params.fileType,
|
|
10720
|
+
directory: params.directory,
|
|
10721
|
+
metadataOnly: true
|
|
10722
|
+
});
|
|
10723
|
+
if (results.length === 0) {
|
|
10724
|
+
return text2("No matching code found. Try a different query or run index_status/index_codebase first.");
|
|
10725
|
+
}
|
|
10726
|
+
return text2(`Found ${results.length} locations for "${params.query}":
|
|
10727
|
+
|
|
10728
|
+
${formatCodebasePeek(results)}`, results);
|
|
10729
|
+
}
|
|
10730
|
+
});
|
|
9644
10731
|
pi.registerTool({
|
|
9645
10732
|
name: "codebase_search",
|
|
9646
10733
|
label: "Codebase Search",
|
|
9647
|
-
description: "
|
|
10734
|
+
description: "Use this after codebase_context when you need semantic content, not just locations. Describe behavior, not syntax.",
|
|
9648
10735
|
parameters: Type2.Object({
|
|
9649
10736
|
query: Type2.String({ description: "Natural language description of what code you're looking for" }),
|
|
9650
10737
|
limit: Type2.Optional(Type2.Number({ description: "Maximum results (default: 10)" })),
|
|
@@ -9664,7 +10751,7 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9664
10751
|
pi.registerTool({
|
|
9665
10752
|
name: "codebase_peek",
|
|
9666
10753
|
label: "Codebase Peek",
|
|
9667
|
-
description: "
|
|
10754
|
+
description: "LOW-TOKEN location-first retrieval. Prefer codebase_context first, then use this for cheap conceptual lookup.",
|
|
9668
10755
|
parameters: Type2.Object({
|
|
9669
10756
|
query: Type2.String(),
|
|
9670
10757
|
limit: Type2.Optional(Type2.Number()),
|
|
@@ -9683,7 +10770,7 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9683
10770
|
pi.registerTool({
|
|
9684
10771
|
name: "find_similar",
|
|
9685
10772
|
label: "Find Similar Code",
|
|
9686
|
-
description: "Find code similar to a snippet for duplicate detection and
|
|
10773
|
+
description: "Find code similar to a snippet for duplicate detection, pattern discovery, and refactor planning.",
|
|
9687
10774
|
parameters: Type2.Object({
|
|
9688
10775
|
code: Type2.String({ description: "Code snippet to compare" }),
|
|
9689
10776
|
limit: Type2.Optional(Type2.Number()),
|
|
@@ -9700,7 +10787,7 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9700
10787
|
pi.registerTool({
|
|
9701
10788
|
name: "implementation_lookup",
|
|
9702
10789
|
label: "Implementation Lookup",
|
|
9703
|
-
description: "Find likely symbol definitions or implementations
|
|
10790
|
+
description: "Find likely symbol definitions or implementations after codebase_context identifies a symbol.",
|
|
9704
10791
|
parameters: Type2.Object({
|
|
9705
10792
|
query: Type2.String(),
|
|
9706
10793
|
limit: Type2.Optional(Type2.Number()),
|
|
@@ -9715,7 +10802,7 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9715
10802
|
pi.registerTool({
|
|
9716
10803
|
name: "index_codebase",
|
|
9717
10804
|
label: "Index Codebase",
|
|
9718
|
-
description: "Build or refresh the semantic codebase index.",
|
|
10805
|
+
description: "Build or refresh the semantic codebase index. Run index_status when freshness is unknown.",
|
|
9719
10806
|
parameters: Type2.Object({
|
|
9720
10807
|
force: Type2.Optional(Type2.Boolean({ default: false })),
|
|
9721
10808
|
estimateOnly: Type2.Optional(Type2.Boolean({ default: false })),
|
|
@@ -9723,7 +10810,9 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9723
10810
|
}),
|
|
9724
10811
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
9725
10812
|
const result = await runIndexCodebase(projectRoot2(ctx), HOST2, params);
|
|
9726
|
-
|
|
10813
|
+
if (result.kind === "estimate") return text2(formatCostEstimate(result.estimate), result.estimate);
|
|
10814
|
+
if (result.kind === "busy") return text2(result.text, { code: "INDEX_BUSY" });
|
|
10815
|
+
return text2(formatIndexStats(result.stats, params.verbose ?? false), result.stats);
|
|
9727
10816
|
}
|
|
9728
10817
|
});
|
|
9729
10818
|
pi.registerTool({
|
|
@@ -9739,11 +10828,12 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9739
10828
|
pi.registerTool({
|
|
9740
10829
|
name: "index_health_check",
|
|
9741
10830
|
label: "Index Health Check",
|
|
9742
|
-
description: "Garbage collect orphaned embeddings/chunks and report health.",
|
|
10831
|
+
description: "Garbage collect orphaned embeddings/chunks and report index health status.",
|
|
9743
10832
|
parameters: Type2.Object({}),
|
|
9744
10833
|
async execute(_toolCallId, _params, _signal, _onUpdate, ctx) {
|
|
9745
|
-
const result = await
|
|
9746
|
-
return text2(
|
|
10834
|
+
const result = await runIndexHealthCheck(projectRoot2(ctx), HOST2);
|
|
10835
|
+
if (result.kind === "busy") return text2(result.text, { code: "INDEX_BUSY" });
|
|
10836
|
+
return text2(formatHealthCheck(result.health), result.health);
|
|
9747
10837
|
}
|
|
9748
10838
|
});
|
|
9749
10839
|
pi.registerTool({
|
|
@@ -9778,6 +10868,11 @@ function codebaseIndexPiExtension(pi) {
|
|
|
9778
10868
|
}
|
|
9779
10869
|
});
|
|
9780
10870
|
registerPiCallGraphTools(pi);
|
|
10871
|
+
pi.on("before_agent_start", (event) => ({
|
|
10872
|
+
systemPrompt: `${event.systemPrompt}
|
|
10873
|
+
|
|
10874
|
+
Check index_status first when index readiness is unknown. For repository questions, call codebase_context before search/grep/bash/read-style broad reads. Use implementation_lookup for known symbols and call_graph/call_graph_path after endpoints are identified for dependency flow. Avoid broad tool calls until semantic locations are known.`
|
|
10875
|
+
}));
|
|
9781
10876
|
pi.registerTool({
|
|
9782
10877
|
name: "pr_impact",
|
|
9783
10878
|
label: "PR Impact",
|