opencode-codebase-index 0.21.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/dist/cli.cjs +2154 -1391
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2237 -1474
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +1975 -1365
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2017 -1407
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +1699 -1087
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +1736 -1124
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +5 -3
package/dist/index.cjs
CHANGED
|
@@ -333,7 +333,7 @@ var require_ignore = __commonJS({
|
|
|
333
333
|
// path matching.
|
|
334
334
|
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
335
335
|
// @returns {TestResult} true if a file is ignored
|
|
336
|
-
test(
|
|
336
|
+
test(path28, checkUnignored, mode) {
|
|
337
337
|
let ignored = false;
|
|
338
338
|
let unignored = false;
|
|
339
339
|
let matchedRule;
|
|
@@ -342,7 +342,7 @@ var require_ignore = __commonJS({
|
|
|
342
342
|
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
343
343
|
return;
|
|
344
344
|
}
|
|
345
|
-
const matched = rule[mode].test(
|
|
345
|
+
const matched = rule[mode].test(path28);
|
|
346
346
|
if (!matched) {
|
|
347
347
|
return;
|
|
348
348
|
}
|
|
@@ -363,17 +363,17 @@ var require_ignore = __commonJS({
|
|
|
363
363
|
var throwError = (message, Ctor) => {
|
|
364
364
|
throw new Ctor(message);
|
|
365
365
|
};
|
|
366
|
-
var checkPath = (
|
|
367
|
-
if (!isString(
|
|
366
|
+
var checkPath = (path28, originalPath, doThrow) => {
|
|
367
|
+
if (!isString(path28)) {
|
|
368
368
|
return doThrow(
|
|
369
369
|
`path must be a string, but got \`${originalPath}\``,
|
|
370
370
|
TypeError
|
|
371
371
|
);
|
|
372
372
|
}
|
|
373
|
-
if (!
|
|
373
|
+
if (!path28) {
|
|
374
374
|
return doThrow(`path must not be empty`, TypeError);
|
|
375
375
|
}
|
|
376
|
-
if (checkPath.isNotRelative(
|
|
376
|
+
if (checkPath.isNotRelative(path28)) {
|
|
377
377
|
const r = "`path.relative()`d";
|
|
378
378
|
return doThrow(
|
|
379
379
|
`path should be a ${r} string, but got "${originalPath}"`,
|
|
@@ -382,7 +382,7 @@ var require_ignore = __commonJS({
|
|
|
382
382
|
}
|
|
383
383
|
return true;
|
|
384
384
|
};
|
|
385
|
-
var isNotRelative = (
|
|
385
|
+
var isNotRelative = (path28) => REGEX_TEST_INVALID_PATH.test(path28);
|
|
386
386
|
checkPath.isNotRelative = isNotRelative;
|
|
387
387
|
checkPath.convert = (p) => p;
|
|
388
388
|
var Ignore2 = class {
|
|
@@ -412,19 +412,19 @@ var require_ignore = __commonJS({
|
|
|
412
412
|
}
|
|
413
413
|
// @returns {TestResult}
|
|
414
414
|
_test(originalPath, cache, checkUnignored, slices) {
|
|
415
|
-
const
|
|
415
|
+
const path28 = originalPath && checkPath.convert(originalPath);
|
|
416
416
|
checkPath(
|
|
417
|
-
|
|
417
|
+
path28,
|
|
418
418
|
originalPath,
|
|
419
419
|
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
420
420
|
);
|
|
421
|
-
return this._t(
|
|
421
|
+
return this._t(path28, cache, checkUnignored, slices);
|
|
422
422
|
}
|
|
423
|
-
checkIgnore(
|
|
424
|
-
if (!REGEX_TEST_TRAILING_SLASH.test(
|
|
425
|
-
return this.test(
|
|
423
|
+
checkIgnore(path28) {
|
|
424
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path28)) {
|
|
425
|
+
return this.test(path28);
|
|
426
426
|
}
|
|
427
|
-
const slices =
|
|
427
|
+
const slices = path28.split(SLASH2).filter(Boolean);
|
|
428
428
|
slices.pop();
|
|
429
429
|
if (slices.length) {
|
|
430
430
|
const parent = this._t(
|
|
@@ -437,18 +437,18 @@ var require_ignore = __commonJS({
|
|
|
437
437
|
return parent;
|
|
438
438
|
}
|
|
439
439
|
}
|
|
440
|
-
return this._rules.test(
|
|
440
|
+
return this._rules.test(path28, false, MODE_CHECK_IGNORE);
|
|
441
441
|
}
|
|
442
|
-
_t(
|
|
443
|
-
if (
|
|
444
|
-
return cache[
|
|
442
|
+
_t(path28, cache, checkUnignored, slices) {
|
|
443
|
+
if (path28 in cache) {
|
|
444
|
+
return cache[path28];
|
|
445
445
|
}
|
|
446
446
|
if (!slices) {
|
|
447
|
-
slices =
|
|
447
|
+
slices = path28.split(SLASH2).filter(Boolean);
|
|
448
448
|
}
|
|
449
449
|
slices.pop();
|
|
450
450
|
if (!slices.length) {
|
|
451
|
-
return cache[
|
|
451
|
+
return cache[path28] = this._rules.test(path28, checkUnignored, MODE_IGNORE);
|
|
452
452
|
}
|
|
453
453
|
const parent = this._t(
|
|
454
454
|
slices.join(SLASH2) + SLASH2,
|
|
@@ -456,29 +456,29 @@ var require_ignore = __commonJS({
|
|
|
456
456
|
checkUnignored,
|
|
457
457
|
slices
|
|
458
458
|
);
|
|
459
|
-
return cache[
|
|
459
|
+
return cache[path28] = parent.ignored ? parent : this._rules.test(path28, checkUnignored, MODE_IGNORE);
|
|
460
460
|
}
|
|
461
|
-
ignores(
|
|
462
|
-
return this._test(
|
|
461
|
+
ignores(path28) {
|
|
462
|
+
return this._test(path28, this._ignoreCache, false).ignored;
|
|
463
463
|
}
|
|
464
464
|
createFilter() {
|
|
465
|
-
return (
|
|
465
|
+
return (path28) => !this.ignores(path28);
|
|
466
466
|
}
|
|
467
467
|
filter(paths) {
|
|
468
468
|
return makeArray(paths).filter(this.createFilter());
|
|
469
469
|
}
|
|
470
470
|
// @returns {TestResult}
|
|
471
|
-
test(
|
|
472
|
-
return this._test(
|
|
471
|
+
test(path28) {
|
|
472
|
+
return this._test(path28, this._testCache, true);
|
|
473
473
|
}
|
|
474
474
|
};
|
|
475
475
|
var factory = (options) => new Ignore2(options);
|
|
476
|
-
var isPathValid = (
|
|
476
|
+
var isPathValid = (path28) => checkPath(path28 && checkPath.convert(path28), path28, RETURN_FALSE);
|
|
477
477
|
var setupWindows = () => {
|
|
478
478
|
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
479
479
|
checkPath.convert = makePosix;
|
|
480
480
|
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
481
|
-
checkPath.isNotRelative = (
|
|
481
|
+
checkPath.isNotRelative = (path28) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path28) || isNotRelative(path28);
|
|
482
482
|
};
|
|
483
483
|
if (
|
|
484
484
|
// Detect `process` so that it can run in browsers.
|
|
@@ -663,7 +663,7 @@ __export(index_exports, {
|
|
|
663
663
|
module.exports = __toCommonJS(index_exports);
|
|
664
664
|
|
|
665
665
|
// src/adapters/opencode.ts
|
|
666
|
-
var
|
|
666
|
+
var path27 = __toESM(require("path"), 1);
|
|
667
667
|
var import_url = require("url");
|
|
668
668
|
|
|
669
669
|
// src/config/constants.ts
|
|
@@ -817,7 +817,8 @@ function getDefaultSearchConfig() {
|
|
|
817
817
|
contextLines: 0,
|
|
818
818
|
routingHints: true,
|
|
819
819
|
routingGraphHandoffHints: false,
|
|
820
|
-
routingHintRole: "system"
|
|
820
|
+
routingHintRole: "system",
|
|
821
|
+
communityBoost: 0
|
|
821
822
|
};
|
|
822
823
|
}
|
|
823
824
|
function getDefaultRerankerBaseUrl(provider) {
|
|
@@ -950,7 +951,8 @@ function parseConfig(raw) {
|
|
|
950
951
|
contextLines: typeof rawSearch.contextLines === "number" ? Math.min(50, Math.max(0, rawSearch.contextLines)) : defaultSearch.contextLines,
|
|
951
952
|
routingHints: typeof rawSearch.routingHints === "boolean" ? rawSearch.routingHints : defaultSearch.routingHints,
|
|
952
953
|
routingGraphHandoffHints: typeof rawSearch.routingGraphHandoffHints === "boolean" ? rawSearch.routingGraphHandoffHints : defaultSearch.routingGraphHandoffHints,
|
|
953
|
-
routingHintRole: rawSearch.routingHintRole === "developer" || rawSearch.routingHintRole === "system" ? rawSearch.routingHintRole : defaultSearch.routingHintRole
|
|
954
|
+
routingHintRole: rawSearch.routingHintRole === "developer" || rawSearch.routingHintRole === "system" ? rawSearch.routingHintRole : defaultSearch.routingHintRole,
|
|
955
|
+
communityBoost: typeof rawSearch.communityBoost === "number" && Number.isFinite(rawSearch.communityBoost) ? Math.min(1, Math.max(0, rawSearch.communityBoost)) : defaultSearch.communityBoost
|
|
954
956
|
};
|
|
955
957
|
const rawDebug = input.debug && typeof input.debug === "object" ? input.debug : {};
|
|
956
958
|
const debug = {
|
|
@@ -1742,7 +1744,7 @@ function loadMergedConfig(projectRoot, host) {
|
|
|
1742
1744
|
|
|
1743
1745
|
// src/tools/operations.ts
|
|
1744
1746
|
var import_fs13 = require("fs");
|
|
1745
|
-
var
|
|
1747
|
+
var path20 = __toESM(require("path"), 1);
|
|
1746
1748
|
|
|
1747
1749
|
// src/tools/knowledge-base-paths.ts
|
|
1748
1750
|
var path8 = __toESM(require("path"), 1);
|
|
@@ -1785,6 +1787,152 @@ function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot) {
|
|
|
1785
1787
|
);
|
|
1786
1788
|
}
|
|
1787
1789
|
|
|
1790
|
+
// src/tools/format-communities.ts
|
|
1791
|
+
function compareText(left, right) {
|
|
1792
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
1793
|
+
}
|
|
1794
|
+
function buildCodeCommunitiesResult(communities, centrality, couplings = [], options = {}) {
|
|
1795
|
+
const minSize = options.minSize ?? 1;
|
|
1796
|
+
const limit = options.limit ?? 20;
|
|
1797
|
+
const hubThreshold = options.hubThreshold ?? 5;
|
|
1798
|
+
const minCoupling = options.minCoupling ?? 1;
|
|
1799
|
+
const couplingLimit = options.couplingLimit ?? 20;
|
|
1800
|
+
const communityMap = /* @__PURE__ */ new Map();
|
|
1801
|
+
for (const c of communities) {
|
|
1802
|
+
let entry = communityMap.get(c.communityId);
|
|
1803
|
+
if (!entry) {
|
|
1804
|
+
entry = { label: c.communityLabel, members: [] };
|
|
1805
|
+
communityMap.set(c.communityId, entry);
|
|
1806
|
+
}
|
|
1807
|
+
entry.members.push(c);
|
|
1808
|
+
}
|
|
1809
|
+
const sortedCommunities = Array.from(communityMap.entries()).map(([id, entry]) => ({
|
|
1810
|
+
id,
|
|
1811
|
+
label: entry.label,
|
|
1812
|
+
symbolCount: entry.members.length,
|
|
1813
|
+
members: entry.members.map((m) => ({
|
|
1814
|
+
symbolId: m.symbolId,
|
|
1815
|
+
symbolName: m.symbolName,
|
|
1816
|
+
filePath: m.filePath
|
|
1817
|
+
})).sort((a, b) => compareText(a.symbolName, b.symbolName) || compareText(a.symbolId, b.symbolId))
|
|
1818
|
+
})).filter((c) => c.symbolCount >= minSize).sort((a, b) => b.symbolCount - a.symbolCount || compareText(a.label, b.label) || a.id - b.id).slice(0, limit);
|
|
1819
|
+
const communityBySymbol = new Map(communities.map((community) => [community.symbolId, community]));
|
|
1820
|
+
const communityLabelById = new Map(communities.map((community) => [community.communityId, community.communityLabel]));
|
|
1821
|
+
const hubNodes = centrality.map((c) => ({
|
|
1822
|
+
symbolId: c.symbolId,
|
|
1823
|
+
symbolName: c.symbolName,
|
|
1824
|
+
filePath: c.filePath,
|
|
1825
|
+
callerCount: c.callerCount,
|
|
1826
|
+
calleeCount: c.calleeCount,
|
|
1827
|
+
totalConnections: c.totalConnections,
|
|
1828
|
+
crossCommunityConnections: communityBySymbol.get(c.symbolId)?.crossCommunityConnections ?? 0
|
|
1829
|
+
})).filter((h) => h.crossCommunityConnections >= hubThreshold).sort(
|
|
1830
|
+
(a, b) => b.crossCommunityConnections - a.crossCommunityConnections || b.totalConnections - a.totalConnections || compareText(a.symbolId, b.symbolId)
|
|
1831
|
+
).slice(0, limit);
|
|
1832
|
+
const canonicalCoupling = (value) => Math.trunc(value);
|
|
1833
|
+
const couplingItems = couplings.map((entry) => {
|
|
1834
|
+
const relationships = entry.relationships ?? entry.representativeRelationships ?? [];
|
|
1835
|
+
const normalizedRelationships = relationships.map((relationship) => ({
|
|
1836
|
+
fromSymbolId: relationship.fromSymbolId,
|
|
1837
|
+
fromSymbolName: relationship.fromSymbolName,
|
|
1838
|
+
fromFilePath: relationship.fromFilePath,
|
|
1839
|
+
toSymbolId: relationship.toSymbolId,
|
|
1840
|
+
toSymbolName: relationship.toSymbolName,
|
|
1841
|
+
toFilePath: relationship.toFilePath
|
|
1842
|
+
})).sort(
|
|
1843
|
+
(left, right) => compareText(left.fromSymbolName, right.fromSymbolName) || compareText(left.fromSymbolId, right.fromSymbolId) || compareText(left.toSymbolName, right.toSymbolName) || compareText(left.toSymbolId, right.toSymbolId) || compareText(left.fromFilePath, right.fromFilePath) || compareText(left.toFilePath, right.toFilePath)
|
|
1844
|
+
).slice(0, 5);
|
|
1845
|
+
const communityA = canonicalCoupling(Math.min(entry.communityA, entry.communityB));
|
|
1846
|
+
const communityB = canonicalCoupling(Math.max(entry.communityA, entry.communityB));
|
|
1847
|
+
return {
|
|
1848
|
+
communityA,
|
|
1849
|
+
communityB,
|
|
1850
|
+
communityAName: communityLabelById.get(communityA) ?? `Community ${communityA}`,
|
|
1851
|
+
communityBName: communityLabelById.get(communityB) ?? `Community ${communityB}`,
|
|
1852
|
+
distinctConnections: canonicalCoupling(entry.count),
|
|
1853
|
+
representativeRelationships: normalizedRelationships
|
|
1854
|
+
};
|
|
1855
|
+
}).filter((entry) => entry.distinctConnections >= minCoupling).sort(
|
|
1856
|
+
(left, right) => right.distinctConnections - left.distinctConnections || compareText(left.communityAName, right.communityAName) || compareText(left.communityBName, right.communityBName) || compareText(left.communityAName + left.communityBName, right.communityAName + right.communityBName) || left.communityA - right.communityA || left.communityB - right.communityB
|
|
1857
|
+
).slice(0, Math.max(1, Math.floor(couplingLimit)));
|
|
1858
|
+
return {
|
|
1859
|
+
communities: sortedCommunities,
|
|
1860
|
+
hubNodes,
|
|
1861
|
+
totalSymbols: communities.length,
|
|
1862
|
+
totalCommunities: communityMap.size,
|
|
1863
|
+
couplings: couplingItems
|
|
1864
|
+
};
|
|
1865
|
+
}
|
|
1866
|
+
function formatCodeCommunities(result) {
|
|
1867
|
+
const lines = [];
|
|
1868
|
+
lines.push(`\u2192 Communities: ${result.totalCommunities} (${result.communities.length} shown, ${result.totalSymbols} symbols total)`);
|
|
1869
|
+
for (const community of result.communities) {
|
|
1870
|
+
lines.push(` Community ${community.id} (${community.label}): ${community.symbolCount} symbols`);
|
|
1871
|
+
const shownMembers = community.members.slice(0, 8);
|
|
1872
|
+
for (const m of shownMembers) {
|
|
1873
|
+
lines.push(` - ${m.symbolName} (${m.filePath})`);
|
|
1874
|
+
}
|
|
1875
|
+
if (community.members.length > 8) {
|
|
1876
|
+
lines.push(` ... and ${community.members.length - 8} more`);
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
if (result.hubNodes.length > 0) {
|
|
1880
|
+
lines.push(`\u2192 Hub nodes (${result.hubNodes.length} shown, cross-community connections):`);
|
|
1881
|
+
for (const hub of result.hubNodes) {
|
|
1882
|
+
lines.push(
|
|
1883
|
+
` - ${hub.symbolName} (${hub.crossCommunityConnections} cross-community, ${hub.callerCount} callers, ${hub.calleeCount} callees) at ${hub.filePath}`
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
} else {
|
|
1887
|
+
lines.push("\u2192 Hub nodes: none with significant cross-community connections");
|
|
1888
|
+
}
|
|
1889
|
+
if (result.couplings.length > 0) {
|
|
1890
|
+
lines.push(`\u2192 Community couplings: ${result.couplings.length} shown`);
|
|
1891
|
+
for (const coupling of result.couplings) {
|
|
1892
|
+
lines.push(` - ${coupling.communityAName} \u2194 ${coupling.communityBName}: ${coupling.distinctConnections} distinct connections`);
|
|
1893
|
+
for (const relationship of coupling.representativeRelationships) {
|
|
1894
|
+
lines.push(` - ${relationship.fromSymbolName} (${relationship.fromFilePath}) -> ${relationship.toSymbolName} (${relationship.toFilePath})`);
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
} else {
|
|
1898
|
+
lines.push("\u2192 Community couplings: none above minCoupling threshold");
|
|
1899
|
+
}
|
|
1900
|
+
return lines.join("\n");
|
|
1901
|
+
}
|
|
1902
|
+
|
|
1903
|
+
// src/tools/contracts.ts
|
|
1904
|
+
var CHUNK_TYPES = [
|
|
1905
|
+
"function",
|
|
1906
|
+
"class",
|
|
1907
|
+
"method",
|
|
1908
|
+
"interface",
|
|
1909
|
+
"type",
|
|
1910
|
+
"enum",
|
|
1911
|
+
"struct",
|
|
1912
|
+
"impl",
|
|
1913
|
+
"trait",
|
|
1914
|
+
"module",
|
|
1915
|
+
"other"
|
|
1916
|
+
];
|
|
1917
|
+
var CALL_GRAPH_DIRECTIONS = ["callers", "callees"];
|
|
1918
|
+
var RELATIONSHIP_TYPES = [
|
|
1919
|
+
"Call",
|
|
1920
|
+
"MethodCall",
|
|
1921
|
+
"Constructor",
|
|
1922
|
+
"Import",
|
|
1923
|
+
"Inherits",
|
|
1924
|
+
"Implements"
|
|
1925
|
+
];
|
|
1926
|
+
var INDEX_LOG_CATEGORIES = ["search", "embedding", "cache", "gc", "branch", "general"];
|
|
1927
|
+
var INDEX_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
1928
|
+
var CODE_COMMUNITIES_MIN_SIZE = 1;
|
|
1929
|
+
var CODE_COMMUNITIES_DEFAULT_LIMIT = 20;
|
|
1930
|
+
var CODE_COMMUNITIES_MAX_LIMIT = 100;
|
|
1931
|
+
var CODE_COMMUNITIES_DEFAULT_HUB_THRESHOLD = 5;
|
|
1932
|
+
var CODE_COMMUNITIES_MIN_COUPLING = 1;
|
|
1933
|
+
var CODE_COMMUNITIES_DEFAULT_COUPLING_LIMIT = 20;
|
|
1934
|
+
var CODE_COMMUNITIES_MAX_COUPLING_LIMIT = 100;
|
|
1935
|
+
|
|
1788
1936
|
// src/tools/context-pack.ts
|
|
1789
1937
|
var import_tiktoken = require("tiktoken");
|
|
1790
1938
|
|
|
@@ -2428,8 +2576,8 @@ function formatExactSearchHandoff(results) {
|
|
|
2428
2576
|
}
|
|
2429
2577
|
function formatContextEvidence(result, index) {
|
|
2430
2578
|
const symbol = result.name ? ` ${JSON.stringify(compactEvidenceValue(result.name, 80))}` : "";
|
|
2431
|
-
const
|
|
2432
|
-
return `[${index}] ${result.chunkType}${symbol} in ${
|
|
2579
|
+
const path28 = compactEvidenceValue(result.filePath, 120);
|
|
2580
|
+
return `[${index}] ${result.chunkType}${symbol} in ${path28}:${result.startLine}-${result.endLine} (score ${result.score.toFixed(2)})`;
|
|
2433
2581
|
}
|
|
2434
2582
|
function formatContextPack(heading, selected, candidateCount, duplicateCount, limitOmittedCount, budgetOmittedCount, includeExactSearchHandoff) {
|
|
2435
2583
|
const lines = selected.map((result, index) => formatContextEvidence(result, index + 1));
|
|
@@ -4442,7 +4590,7 @@ function saveConfig(projectRoot, config, host) {
|
|
|
4442
4590
|
|
|
4443
4591
|
// src/indexer/index.ts
|
|
4444
4592
|
var import_fs12 = require("fs");
|
|
4445
|
-
var
|
|
4593
|
+
var path19 = __toESM(require("path"), 1);
|
|
4446
4594
|
var import_perf_hooks = require("perf_hooks");
|
|
4447
4595
|
var import_child_process4 = require("child_process");
|
|
4448
4596
|
var import_util4 = require("util");
|
|
@@ -7034,6 +7182,9 @@ function createMockNativeBinding() {
|
|
|
7034
7182
|
detectCommunities() {
|
|
7035
7183
|
throw error;
|
|
7036
7184
|
}
|
|
7185
|
+
detectCommunityCouplings() {
|
|
7186
|
+
throw error;
|
|
7187
|
+
}
|
|
7037
7188
|
computeCentrality() {
|
|
7038
7189
|
throw error;
|
|
7039
7190
|
}
|
|
@@ -7271,6 +7422,18 @@ var Database = class _Database {
|
|
|
7271
7422
|
}
|
|
7272
7423
|
this.closed = true;
|
|
7273
7424
|
}
|
|
7425
|
+
beginWriteTransaction() {
|
|
7426
|
+
this.throwIfClosed();
|
|
7427
|
+
this.inner.beginWriteTransaction();
|
|
7428
|
+
}
|
|
7429
|
+
commitWriteTransaction() {
|
|
7430
|
+
this.throwIfClosed();
|
|
7431
|
+
this.inner.commitWriteTransaction();
|
|
7432
|
+
}
|
|
7433
|
+
rollbackWriteTransaction() {
|
|
7434
|
+
this.throwIfClosed();
|
|
7435
|
+
this.inner.rollbackWriteTransaction();
|
|
7436
|
+
}
|
|
7274
7437
|
embeddingExists(contentHash) {
|
|
7275
7438
|
this.throwIfClosed();
|
|
7276
7439
|
return this.inner.embeddingExists(contentHash);
|
|
@@ -7530,6 +7693,13 @@ var Database = class _Database {
|
|
|
7530
7693
|
this.throwIfClosed();
|
|
7531
7694
|
return this.inner.computeCentrality(branch);
|
|
7532
7695
|
}
|
|
7696
|
+
detectCommunityCouplings(branch) {
|
|
7697
|
+
this.throwIfClosed();
|
|
7698
|
+
return this.inner.detectCommunityCouplings(branch).map((entry) => ({
|
|
7699
|
+
...entry,
|
|
7700
|
+
relationships: entry.representativeRelationships ?? []
|
|
7701
|
+
}));
|
|
7702
|
+
}
|
|
7533
7703
|
};
|
|
7534
7704
|
|
|
7535
7705
|
// src/git/branch-materialization.ts
|
|
@@ -8299,6 +8469,21 @@ async function getChunkGitBlame(projectRoot, filePath, startLine, endLine) {
|
|
|
8299
8469
|
}
|
|
8300
8470
|
|
|
8301
8471
|
// src/indexer/search-ranking.ts
|
|
8472
|
+
function applyCommunityBoost(candidates, sameCommunityCandidateIds, boost) {
|
|
8473
|
+
if (boost <= 0 || sameCommunityCandidateIds.size === 0 || candidates.length <= 1) {
|
|
8474
|
+
return candidates;
|
|
8475
|
+
}
|
|
8476
|
+
const result = candidates.map((candidate) => sameCommunityCandidateIds.has(candidate.id) ? { ...candidate, score: candidate.score * (1 + boost) } : candidate);
|
|
8477
|
+
for (let index = 1; index < result.length; index += 1) {
|
|
8478
|
+
const candidate = result[index];
|
|
8479
|
+
const previous = result[index - 1];
|
|
8480
|
+
if (candidate && previous && sameCommunityCandidateIds.has(candidate.id) && !sameCommunityCandidateIds.has(previous.id) && candidate.score > previous.score) {
|
|
8481
|
+
result[index - 1] = candidate;
|
|
8482
|
+
result[index] = previous;
|
|
8483
|
+
}
|
|
8484
|
+
}
|
|
8485
|
+
return result;
|
|
8486
|
+
}
|
|
8302
8487
|
var RANK_HYBRID_CACHE_LIMIT = 256;
|
|
8303
8488
|
var rankHybridResultsCache = /* @__PURE__ */ new WeakMap();
|
|
8304
8489
|
function classifyQueryIntentRaw(query) {
|
|
@@ -8477,6 +8662,117 @@ function rankSemanticOnlyResults(query, semanticResults, options) {
|
|
|
8477
8662
|
});
|
|
8478
8663
|
}
|
|
8479
8664
|
|
|
8665
|
+
// src/tools/symbol-inference.ts
|
|
8666
|
+
var IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g;
|
|
8667
|
+
var QUOTED_BACKTICK_RE = /`([^`]+)`/g;
|
|
8668
|
+
var QUOTED_SINGLE_RE = /'([^'\\]+)'/g;
|
|
8669
|
+
var QUOTED_DOUBLE_RE = /"([^"]+)"/g;
|
|
8670
|
+
var SYMBOL_LIKE_RE = /^(?:[A-Za-z_$][A-Za-z0-9_$]*)$/;
|
|
8671
|
+
var CAMEL_CASE_RE = /^[a-z_][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*$/;
|
|
8672
|
+
var PASCAL_CASE_RE = /^[A-Z][A-Za-z0-9_$]*$/;
|
|
8673
|
+
var SNAKE_CASE_RE = /^[a-z][a-z0-9_]*_[a-z0-9_]+$/;
|
|
8674
|
+
var DEFINITION_INTENT_RE = /\b(where|defined|definition|define|declaration|symbol|function|method|class|interface|type)\b/i;
|
|
8675
|
+
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
8676
|
+
"a",
|
|
8677
|
+
"an",
|
|
8678
|
+
"and",
|
|
8679
|
+
"are",
|
|
8680
|
+
"at",
|
|
8681
|
+
"for",
|
|
8682
|
+
"find",
|
|
8683
|
+
"how",
|
|
8684
|
+
"i",
|
|
8685
|
+
"in",
|
|
8686
|
+
"is",
|
|
8687
|
+
"it",
|
|
8688
|
+
"of",
|
|
8689
|
+
"on",
|
|
8690
|
+
"that",
|
|
8691
|
+
"the",
|
|
8692
|
+
"definition",
|
|
8693
|
+
"show",
|
|
8694
|
+
"to",
|
|
8695
|
+
"where",
|
|
8696
|
+
"which",
|
|
8697
|
+
"what",
|
|
8698
|
+
"you",
|
|
8699
|
+
"your",
|
|
8700
|
+
"with"
|
|
8701
|
+
]);
|
|
8702
|
+
function stripCallSuffix(token) {
|
|
8703
|
+
return token.replace(/\(\s*\)$/, "");
|
|
8704
|
+
}
|
|
8705
|
+
function isLikelySymbolName(token) {
|
|
8706
|
+
if (!SYMBOL_LIKE_RE.test(token)) {
|
|
8707
|
+
return false;
|
|
8708
|
+
}
|
|
8709
|
+
if (STOP_WORDS.has(token.toLowerCase())) {
|
|
8710
|
+
return false;
|
|
8711
|
+
}
|
|
8712
|
+
return CAMEL_CASE_RE.test(token) || PASCAL_CASE_RE.test(token) || SNAKE_CASE_RE.test(token);
|
|
8713
|
+
}
|
|
8714
|
+
function extractQuotedIdentifiers(query) {
|
|
8715
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
8716
|
+
for (const match of query.matchAll(QUOTED_BACKTICK_RE)) {
|
|
8717
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
8718
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
8719
|
+
identifiers.add(candidate);
|
|
8720
|
+
}
|
|
8721
|
+
}
|
|
8722
|
+
for (const match of query.matchAll(QUOTED_SINGLE_RE)) {
|
|
8723
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
8724
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
8725
|
+
identifiers.add(candidate);
|
|
8726
|
+
}
|
|
8727
|
+
}
|
|
8728
|
+
for (const match of query.matchAll(QUOTED_DOUBLE_RE)) {
|
|
8729
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
8730
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
8731
|
+
identifiers.add(candidate);
|
|
8732
|
+
}
|
|
8733
|
+
}
|
|
8734
|
+
return [...identifiers];
|
|
8735
|
+
}
|
|
8736
|
+
function extractBareIdentifiers(query) {
|
|
8737
|
+
const unquoted = query.replace(QUOTED_BACKTICK_RE, " ").replace(QUOTED_SINGLE_RE, " ").replace(QUOTED_DOUBLE_RE, " ");
|
|
8738
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
8739
|
+
for (const match of unquoted.matchAll(IDENTIFIER_RE)) {
|
|
8740
|
+
const candidate = stripCallSuffix(match[0]);
|
|
8741
|
+
if (isLikelySymbolName(candidate)) {
|
|
8742
|
+
identifiers.add(candidate);
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
return [...identifiers];
|
|
8746
|
+
}
|
|
8747
|
+
function isSingleMeaningfulToken(query, symbol) {
|
|
8748
|
+
const tokens = query.replace(/[`'"()]/g, " ").split(/[^A-Za-z0-9_$]+/).map((token) => token.trim().toLowerCase()).filter((token) => token.length > 0).filter((token) => !STOP_WORDS.has(token));
|
|
8749
|
+
return tokens.length === 1 && tokens[0] === symbol.toLowerCase();
|
|
8750
|
+
}
|
|
8751
|
+
function inferExactSymbolFromQuery(query) {
|
|
8752
|
+
if (analyzeQueryIntent(query).explicitArtifactIntent) {
|
|
8753
|
+
return void 0;
|
|
8754
|
+
}
|
|
8755
|
+
const quoted = extractQuotedIdentifiers(query);
|
|
8756
|
+
if (quoted.length === 1) {
|
|
8757
|
+
return quoted[0];
|
|
8758
|
+
}
|
|
8759
|
+
if (quoted.length > 1) {
|
|
8760
|
+
return void 0;
|
|
8761
|
+
}
|
|
8762
|
+
const candidates = extractBareIdentifiers(query);
|
|
8763
|
+
if (candidates.length !== 1) {
|
|
8764
|
+
return void 0;
|
|
8765
|
+
}
|
|
8766
|
+
const candidate = candidates[0];
|
|
8767
|
+
if (DEFINITION_INTENT_RE.test(query)) {
|
|
8768
|
+
return candidate;
|
|
8769
|
+
}
|
|
8770
|
+
if (isSingleMeaningfulToken(query, candidate)) {
|
|
8771
|
+
return candidate;
|
|
8772
|
+
}
|
|
8773
|
+
return void 0;
|
|
8774
|
+
}
|
|
8775
|
+
|
|
8480
8776
|
// src/indexer/call-graph-constants.ts
|
|
8481
8777
|
var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
8482
8778
|
"function_declaration",
|
|
@@ -8979,22 +9275,6 @@ function getUniquePendingChunksFromRequests(requests) {
|
|
|
8979
9275
|
}
|
|
8980
9276
|
return Array.from(uniqueChunks.values());
|
|
8981
9277
|
}
|
|
8982
|
-
function coalesceFailedBatches(batches) {
|
|
8983
|
-
const grouped = /* @__PURE__ */ new Map();
|
|
8984
|
-
for (const batch of batches) {
|
|
8985
|
-
const key = `${batch.attemptCount}:${batch.lastAttempt}:${batch.error}`;
|
|
8986
|
-
const existing = grouped.get(key);
|
|
8987
|
-
if (!existing) {
|
|
8988
|
-
grouped.set(key, {
|
|
8989
|
-
...batch,
|
|
8990
|
-
chunks: [...batch.chunks]
|
|
8991
|
-
});
|
|
8992
|
-
continue;
|
|
8993
|
-
}
|
|
8994
|
-
existing.chunks.push(...batch.chunks);
|
|
8995
|
-
}
|
|
8996
|
-
return Array.from(grouped.values());
|
|
8997
|
-
}
|
|
8998
9278
|
function poolEmbeddingVectors(vectors, weights) {
|
|
8999
9279
|
const firstVector = vectors[0];
|
|
9000
9280
|
if (!firstVector) {
|
|
@@ -9027,66 +9307,325 @@ function hasAllEmbeddingParts(parts, expectedPartCount) {
|
|
|
9027
9307
|
return true;
|
|
9028
9308
|
}
|
|
9029
9309
|
|
|
9030
|
-
// src/indexer/
|
|
9031
|
-
var
|
|
9032
|
-
var
|
|
9033
|
-
var
|
|
9034
|
-
var
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
"class_declaration",
|
|
9041
|
-
"class_definition"
|
|
9042
|
-
]);
|
|
9043
|
-
var C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set(["class_specifier", "struct_specifier"]);
|
|
9044
|
-
function isCompatibleCFamilyCallTarget(language, callType, symbolKind) {
|
|
9045
|
-
if (language !== "c" && language !== "cpp") return true;
|
|
9046
|
-
if (symbolKind === "namespace_definition") return callType === "Import";
|
|
9047
|
-
const isTypeSymbol = C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES.has(symbolKind);
|
|
9048
|
-
if (callType === "Constructor" || callType === "Inherits" || callType === "Implements") {
|
|
9049
|
-
return isTypeSymbol;
|
|
9310
|
+
// src/indexer/failed-state-persistence.ts
|
|
9311
|
+
var fs2 = __toESM(require("fs"), 1);
|
|
9312
|
+
var import_node_crypto = require("crypto");
|
|
9313
|
+
var path18 = __toESM(require("path"), 1);
|
|
9314
|
+
var import_node_string_decoder = require("string_decoder");
|
|
9315
|
+
var CURRENT_FAILED_BATCH_VERSION = 1;
|
|
9316
|
+
var DEFAULT_MALFORMED_LINE_ACTION = "skip";
|
|
9317
|
+
function* readFailedBatchRecords(filePath, options = {}) {
|
|
9318
|
+
if (!fs2.existsSync(filePath)) {
|
|
9319
|
+
return;
|
|
9050
9320
|
}
|
|
9051
|
-
|
|
9052
|
-
|
|
9053
|
-
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
"
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
|
|
9066
|
-
|
|
9067
|
-
|
|
9068
|
-
|
|
9069
|
-
|
|
9070
|
-
|
|
9071
|
-
|
|
9072
|
-
|
|
9073
|
-
|
|
9074
|
-
|
|
9321
|
+
const fileFormat = detectFailedBatchFileFormat(filePath);
|
|
9322
|
+
if (fileFormat === "legacy") {
|
|
9323
|
+
yield* readLegacyFailedBatchRecords(filePath, options);
|
|
9324
|
+
return;
|
|
9325
|
+
}
|
|
9326
|
+
yield* readJsonlFailedBatchRecords(filePath, options);
|
|
9327
|
+
}
|
|
9328
|
+
function createFailedBatchWriter(targetPath) {
|
|
9329
|
+
const temporaryPath = createTemporaryPath(targetPath);
|
|
9330
|
+
let finalized = false;
|
|
9331
|
+
fs2.mkdirSync(path18.dirname(targetPath), { recursive: true });
|
|
9332
|
+
fs2.closeSync(fs2.openSync(temporaryPath, "w"));
|
|
9333
|
+
const write = (record) => {
|
|
9334
|
+
if (finalized) {
|
|
9335
|
+
throw new Error("Failed batch writer has been finalized");
|
|
9336
|
+
}
|
|
9337
|
+
const lines = record.chunks.map((chunk) => {
|
|
9338
|
+
const lineRecord = {
|
|
9339
|
+
version: CURRENT_FAILED_BATCH_VERSION,
|
|
9340
|
+
chunks: [chunk],
|
|
9341
|
+
error: record.error,
|
|
9342
|
+
attemptCount: record.attemptCount,
|
|
9343
|
+
lastAttempt: record.lastAttempt
|
|
9344
|
+
};
|
|
9345
|
+
return JSON.stringify(lineRecord);
|
|
9346
|
+
});
|
|
9347
|
+
if (lines.length === 0) {
|
|
9348
|
+
return;
|
|
9075
9349
|
}
|
|
9076
|
-
|
|
9077
|
-
|
|
9078
|
-
|
|
9350
|
+
fs2.mkdirSync(path18.dirname(targetPath), { recursive: true });
|
|
9351
|
+
fs2.appendFileSync(temporaryPath, `${lines.join("\n")}
|
|
9352
|
+
`, "utf-8");
|
|
9353
|
+
};
|
|
9354
|
+
const commit = () => {
|
|
9355
|
+
if (finalized) {
|
|
9356
|
+
return;
|
|
9079
9357
|
}
|
|
9080
|
-
|
|
9081
|
-
|
|
9082
|
-
|
|
9083
|
-
|
|
9084
|
-
|
|
9085
|
-
|
|
9358
|
+
fs2.mkdirSync(path18.dirname(targetPath), { recursive: true });
|
|
9359
|
+
fs2.renameSync(temporaryPath, targetPath);
|
|
9360
|
+
finalized = true;
|
|
9361
|
+
};
|
|
9362
|
+
const cleanup = () => {
|
|
9363
|
+
if (finalized) {
|
|
9364
|
+
return;
|
|
9086
9365
|
}
|
|
9087
|
-
|
|
9088
|
-
|
|
9089
|
-
|
|
9366
|
+
fs2.rmSync(temporaryPath, { force: true });
|
|
9367
|
+
};
|
|
9368
|
+
return {
|
|
9369
|
+
write,
|
|
9370
|
+
commit,
|
|
9371
|
+
cleanup,
|
|
9372
|
+
temporaryPath
|
|
9373
|
+
};
|
|
9374
|
+
}
|
|
9375
|
+
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
9376
|
+
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
9377
|
+
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
9378
|
+
if (trimmed.length === 0) {
|
|
9379
|
+
return;
|
|
9380
|
+
}
|
|
9381
|
+
let parsed;
|
|
9382
|
+
try {
|
|
9383
|
+
parsed = JSON.parse(trimmed);
|
|
9384
|
+
} catch (error) {
|
|
9385
|
+
handleMalformedLine(filePath, 1, trimmed, error, options);
|
|
9386
|
+
return;
|
|
9387
|
+
}
|
|
9388
|
+
if (!Array.isArray(parsed)) {
|
|
9389
|
+
handleMalformedLine(filePath, 1, trimmed, new Error("Expected legacy failed-batch file to contain a JSON array"), options);
|
|
9390
|
+
return;
|
|
9391
|
+
}
|
|
9392
|
+
for (const entry of parsed) {
|
|
9393
|
+
const normalized = normalizeFailedBatchRecord(entry);
|
|
9394
|
+
if (normalized) {
|
|
9395
|
+
yield normalized;
|
|
9396
|
+
}
|
|
9397
|
+
}
|
|
9398
|
+
}
|
|
9399
|
+
function* readJsonlFailedBatchRecords(filePath, options) {
|
|
9400
|
+
const handle = fs2.openSync(filePath, "r");
|
|
9401
|
+
const decoder = new import_node_string_decoder.StringDecoder("utf8");
|
|
9402
|
+
const readBuffer = Buffer.allocUnsafe(64 * 1024);
|
|
9403
|
+
let buffer = "";
|
|
9404
|
+
let lineNumber = 0;
|
|
9405
|
+
try {
|
|
9406
|
+
let bytesRead = 0;
|
|
9407
|
+
do {
|
|
9408
|
+
bytesRead = fs2.readSync(handle, readBuffer, 0, readBuffer.length, null);
|
|
9409
|
+
buffer += decoder.write(readBuffer.subarray(0, bytesRead));
|
|
9410
|
+
let newlineIndex = buffer.indexOf("\n");
|
|
9411
|
+
while (newlineIndex >= 0) {
|
|
9412
|
+
const rawLine = buffer.slice(0, newlineIndex);
|
|
9413
|
+
buffer = buffer.slice(newlineIndex + 1);
|
|
9414
|
+
lineNumber += 1;
|
|
9415
|
+
const normalized = parseFailedBatchLine(rawLine, filePath, lineNumber, options);
|
|
9416
|
+
if (normalized) {
|
|
9417
|
+
yield normalized;
|
|
9418
|
+
}
|
|
9419
|
+
newlineIndex = buffer.indexOf("\n");
|
|
9420
|
+
}
|
|
9421
|
+
} while (bytesRead > 0);
|
|
9422
|
+
buffer += decoder.end();
|
|
9423
|
+
const finalLine = buffer.trimEnd();
|
|
9424
|
+
if (finalLine.length > 0) {
|
|
9425
|
+
lineNumber += 1;
|
|
9426
|
+
const normalized = parseFailedBatchLine(finalLine, filePath, lineNumber, options);
|
|
9427
|
+
if (normalized) {
|
|
9428
|
+
yield normalized;
|
|
9429
|
+
}
|
|
9430
|
+
}
|
|
9431
|
+
} finally {
|
|
9432
|
+
fs2.closeSync(handle);
|
|
9433
|
+
}
|
|
9434
|
+
}
|
|
9435
|
+
function parseFailedBatchLine(rawLine, filePath, lineNumber, options) {
|
|
9436
|
+
const line = rawLine.trimEnd();
|
|
9437
|
+
if (line.length === 0) {
|
|
9438
|
+
return null;
|
|
9439
|
+
}
|
|
9440
|
+
try {
|
|
9441
|
+
const parsed = JSON.parse(line);
|
|
9442
|
+
const normalized = normalizeFailedBatchRecord(parsed);
|
|
9443
|
+
if (!normalized) {
|
|
9444
|
+
handleMalformedLine(filePath, lineNumber, line, new Error("Malformed failed-batch record"), options);
|
|
9445
|
+
return null;
|
|
9446
|
+
}
|
|
9447
|
+
return normalized;
|
|
9448
|
+
} catch (error) {
|
|
9449
|
+
handleMalformedLine(filePath, lineNumber, line, error, options);
|
|
9450
|
+
return null;
|
|
9451
|
+
}
|
|
9452
|
+
}
|
|
9453
|
+
function normalizeFailedBatchRecord(rawRecord) {
|
|
9454
|
+
if (!rawRecord || typeof rawRecord !== "object" || Array.isArray(rawRecord)) {
|
|
9455
|
+
return null;
|
|
9456
|
+
}
|
|
9457
|
+
const typed = rawRecord;
|
|
9458
|
+
const chunks = Array.isArray(typed.chunks) ? typed.chunks : null;
|
|
9459
|
+
if (!chunks || chunks.length === 0) {
|
|
9460
|
+
return null;
|
|
9461
|
+
}
|
|
9462
|
+
return {
|
|
9463
|
+
version: typeof typed.version === "number" && Number.isFinite(typed.version) ? typed.version : CURRENT_FAILED_BATCH_VERSION,
|
|
9464
|
+
chunks,
|
|
9465
|
+
error: typeof typed.error === "string" ? typed.error : "Unknown embedding error",
|
|
9466
|
+
attemptCount: typeof typed.attemptCount === "number" && Number.isFinite(typed.attemptCount) ? typed.attemptCount : 1,
|
|
9467
|
+
lastAttempt: typeof typed.lastAttempt === "string" ? typed.lastAttempt : (/* @__PURE__ */ new Date()).toISOString()
|
|
9468
|
+
};
|
|
9469
|
+
}
|
|
9470
|
+
function detectFailedBatchFileFormat(filePath) {
|
|
9471
|
+
const handle = fs2.openSync(filePath, "r");
|
|
9472
|
+
try {
|
|
9473
|
+
const buffer = Buffer.alloc(4096);
|
|
9474
|
+
const bytesRead = fs2.readSync(handle, buffer, 0, buffer.length, 0);
|
|
9475
|
+
if (bytesRead <= 0) {
|
|
9476
|
+
return "jsonl";
|
|
9477
|
+
}
|
|
9478
|
+
const prefix = stripLeadingBomAndWhitespace(buffer.subarray(0, bytesRead).toString("utf-8"));
|
|
9479
|
+
return prefix.startsWith("[") ? "legacy" : "jsonl";
|
|
9480
|
+
} finally {
|
|
9481
|
+
fs2.closeSync(handle);
|
|
9482
|
+
}
|
|
9483
|
+
}
|
|
9484
|
+
function stripLeadingBomAndWhitespace(value) {
|
|
9485
|
+
let result = value.trimStart();
|
|
9486
|
+
if (result.charCodeAt(0) === 65279) {
|
|
9487
|
+
result = result.slice(1);
|
|
9488
|
+
}
|
|
9489
|
+
return result;
|
|
9490
|
+
}
|
|
9491
|
+
function createTemporaryPath(targetPath) {
|
|
9492
|
+
const randomId = (0, import_node_crypto.createHash)("sha1").update(`${Date.now()}:${(0, import_node_crypto.randomBytes)(8).toString("hex")}`).digest("hex");
|
|
9493
|
+
const targetDir = path18.dirname(targetPath);
|
|
9494
|
+
const baseName = path18.basename(targetPath);
|
|
9495
|
+
return path18.join(targetDir, `.${baseName}.${randomId}.tmp`);
|
|
9496
|
+
}
|
|
9497
|
+
function handleMalformedLine(filePath, lineNumber, line, error, options) {
|
|
9498
|
+
const action = options.malformedLineAction ?? DEFAULT_MALFORMED_LINE_ACTION;
|
|
9499
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
9500
|
+
if (options.onMalformedLine) {
|
|
9501
|
+
options.onMalformedLine(normalizedError, line, lineNumber, filePath);
|
|
9502
|
+
}
|
|
9503
|
+
if (action === "fail") {
|
|
9504
|
+
throw normalizedError;
|
|
9505
|
+
}
|
|
9506
|
+
}
|
|
9507
|
+
|
|
9508
|
+
// src/indexer/file-batches.ts
|
|
9509
|
+
var INDEX_FILE_BATCH_LIMITS = Object.freeze({
|
|
9510
|
+
maxFiles: 64,
|
|
9511
|
+
maxBytes: 8 * 1024 * 1024
|
|
9512
|
+
});
|
|
9513
|
+
function* iterateOrderedFileBatches(items, getBytes, limits = INDEX_FILE_BATCH_LIMITS) {
|
|
9514
|
+
const maxFiles = Math.max(1, Math.floor(limits.maxFiles));
|
|
9515
|
+
const maxBytes = Math.max(1, Math.floor(limits.maxBytes));
|
|
9516
|
+
let batch = [];
|
|
9517
|
+
let batchBytes = 0;
|
|
9518
|
+
for (const item of items) {
|
|
9519
|
+
const itemBytes = Math.max(0, Math.floor(getBytes(item)));
|
|
9520
|
+
if (batch.length > 0 && (batch.length >= maxFiles || batchBytes + itemBytes > maxBytes)) {
|
|
9521
|
+
yield batch;
|
|
9522
|
+
batch = [];
|
|
9523
|
+
batchBytes = 0;
|
|
9524
|
+
}
|
|
9525
|
+
batch.push(item);
|
|
9526
|
+
batchBytes += itemBytes;
|
|
9527
|
+
}
|
|
9528
|
+
if (batch.length > 0) {
|
|
9529
|
+
yield batch;
|
|
9530
|
+
}
|
|
9531
|
+
}
|
|
9532
|
+
|
|
9533
|
+
// src/indexer/index.ts
|
|
9534
|
+
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "swift", "php", "apex", "zig", "gdscript", "matlab", "bash", "c", "cpp", "metal"]);
|
|
9535
|
+
var CASE_INSENSITIVE_LANGUAGES = /* @__PURE__ */ new Set(["apex", "php"]);
|
|
9536
|
+
function candidateOverlapsSymbol(candidate, symbol) {
|
|
9537
|
+
return candidate.metadata.filePath === symbol.filePath && candidate.metadata.startLine <= symbol.endLine && candidate.metadata.endLine >= symbol.startLine;
|
|
9538
|
+
}
|
|
9539
|
+
function resolveSameCommunityCandidateIds(query, candidates, database, branchCatalogKeys) {
|
|
9540
|
+
const anchorName = inferExactSymbolFromQuery(query);
|
|
9541
|
+
if (!anchorName || candidates.length === 0) {
|
|
9542
|
+
return /* @__PURE__ */ new Set();
|
|
9543
|
+
}
|
|
9544
|
+
const catalogs = branchCatalogKeys.map((branchKey) => ({
|
|
9545
|
+
branchKey,
|
|
9546
|
+
symbols: database.getSymbolsForBranch(branchKey)
|
|
9547
|
+
}));
|
|
9548
|
+
const exactAnchors = catalogs.flatMap(({ branchKey, symbols }) => symbols.filter((symbol) => symbol.name === anchorName).map((symbol) => ({ branchKey, symbol })));
|
|
9549
|
+
const anchors = exactAnchors.length > 0 ? exactAnchors : catalogs.flatMap(({ branchKey, symbols }) => symbols.filter((symbol) => symbol.name.toLowerCase() === anchorName.toLowerCase()).map((symbol) => ({ branchKey, symbol })));
|
|
9550
|
+
const uniqueAnchors = new Map(anchors.map((anchor2) => [anchor2.symbol.id, anchor2]));
|
|
9551
|
+
if (uniqueAnchors.size !== 1) {
|
|
9552
|
+
return /* @__PURE__ */ new Set();
|
|
9553
|
+
}
|
|
9554
|
+
const anchor = uniqueAnchors.values().next().value;
|
|
9555
|
+
const branchSymbols = catalogs.find((catalog) => catalog.branchKey === anchor.branchKey)?.symbols ?? [];
|
|
9556
|
+
const candidateSymbols = branchSymbols.filter(
|
|
9557
|
+
(symbol) => candidates.some((candidate) => candidateOverlapsSymbol(candidate, symbol))
|
|
9558
|
+
);
|
|
9559
|
+
const assignments = database.detectCommunities(
|
|
9560
|
+
anchor.branchKey,
|
|
9561
|
+
[anchor.symbol.id, ...candidateSymbols.map((symbol) => symbol.id)]
|
|
9562
|
+
);
|
|
9563
|
+
const anchorCommunity = assignments.find((assignment) => assignment.symbolId === anchor.symbol.id)?.communityId;
|
|
9564
|
+
if (anchorCommunity === void 0) {
|
|
9565
|
+
return /* @__PURE__ */ new Set();
|
|
9566
|
+
}
|
|
9567
|
+
const sameCommunitySymbolIds = new Set(assignments.filter((assignment) => assignment.communityId === anchorCommunity).map((assignment) => assignment.symbolId));
|
|
9568
|
+
return new Set(candidates.filter((candidate) => candidateSymbols.some(
|
|
9569
|
+
(symbol) => sameCommunitySymbolIds.has(symbol.id) && candidateOverlapsSymbol(candidate, symbol)
|
|
9570
|
+
)).map((candidate) => candidate.id));
|
|
9571
|
+
}
|
|
9572
|
+
var CALL_GRAPH_RESOLUTION_VERSION = "4";
|
|
9573
|
+
var PHP_FUNCTION_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
9574
|
+
"function_declaration",
|
|
9575
|
+
"function",
|
|
9576
|
+
"function_definition"
|
|
9577
|
+
]);
|
|
9578
|
+
var PHP_CLASS_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
9579
|
+
"class_declaration",
|
|
9580
|
+
"class_definition"
|
|
9581
|
+
]);
|
|
9582
|
+
var C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set(["class_specifier", "struct_specifier"]);
|
|
9583
|
+
function isCompatibleCFamilyCallTarget(language, callType, symbolKind) {
|
|
9584
|
+
if (language !== "c" && language !== "cpp") return true;
|
|
9585
|
+
if (symbolKind === "namespace_definition") return callType === "Import";
|
|
9586
|
+
const isTypeSymbol = C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES.has(symbolKind);
|
|
9587
|
+
if (callType === "Constructor" || callType === "Inherits" || callType === "Implements") {
|
|
9588
|
+
return isTypeSymbol;
|
|
9589
|
+
}
|
|
9590
|
+
return !isTypeSymbol;
|
|
9591
|
+
}
|
|
9592
|
+
var EXECUTABLE_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
9593
|
+
"function_declaration",
|
|
9594
|
+
"function",
|
|
9595
|
+
"arrow_function",
|
|
9596
|
+
"method_definition",
|
|
9597
|
+
"function_definition",
|
|
9598
|
+
"method_declaration",
|
|
9599
|
+
"function_item",
|
|
9600
|
+
"protocol_function_declaration",
|
|
9601
|
+
"init_declaration",
|
|
9602
|
+
"deinit_declaration",
|
|
9603
|
+
"subscript_declaration",
|
|
9604
|
+
"constructor_definition",
|
|
9605
|
+
"trigger_declaration",
|
|
9606
|
+
"test_declaration"
|
|
9607
|
+
]);
|
|
9608
|
+
function findEnclosingSymbol(symbols, line, column) {
|
|
9609
|
+
let best;
|
|
9610
|
+
for (const symbol of symbols) {
|
|
9611
|
+
if (line < symbol.startLine || line > symbol.endLine) continue;
|
|
9612
|
+
if (column !== void 0 && (line === symbol.startLine && column < symbol.startCol || line === symbol.endLine && column >= symbol.endCol)) {
|
|
9613
|
+
continue;
|
|
9614
|
+
}
|
|
9615
|
+
if (!best) {
|
|
9616
|
+
best = symbol;
|
|
9617
|
+
continue;
|
|
9618
|
+
}
|
|
9619
|
+
const span = symbol.endLine - symbol.startLine;
|
|
9620
|
+
const bestSpan = best.endLine - best.startLine;
|
|
9621
|
+
const isNarrowerPositionRange = column !== void 0 && span === bestSpan && symbol.startLine === best.startLine && symbol.endLine === best.endLine && symbol.startCol >= best.startCol && symbol.endCol <= best.endCol && (symbol.startCol > best.startCol || symbol.endCol < best.endCol);
|
|
9622
|
+
const isMoreSpecificTie = span === bestSpan && symbol.startLine === best.startLine && EXECUTABLE_SYMBOL_CHUNK_TYPES.has(symbol.kind) && !EXECUTABLE_SYMBOL_CHUNK_TYPES.has(best.kind);
|
|
9623
|
+
if (span < bestSpan || span === bestSpan && symbol.startLine > best.startLine || isNarrowerPositionRange || isMoreSpecificTie) {
|
|
9624
|
+
best = symbol;
|
|
9625
|
+
}
|
|
9626
|
+
}
|
|
9627
|
+
return best;
|
|
9628
|
+
}
|
|
9090
9629
|
function float32ArrayToBuffer(arr) {
|
|
9091
9630
|
const float32 = new Float32Array(arr);
|
|
9092
9631
|
return Buffer.from(float32.buffer);
|
|
@@ -9130,6 +9669,16 @@ function isSqliteCorruptionError(error) {
|
|
|
9130
9669
|
}
|
|
9131
9670
|
var STARTUP_WARNING_METADATA_KEY = "index.startupWarning";
|
|
9132
9671
|
var READER_ARTIFACT_RETRY_INTERVAL_MS = 1e3;
|
|
9672
|
+
function getFailedBatchGroupKey(record) {
|
|
9673
|
+
return `${record.attemptCount}:${record.lastAttempt}:${record.error}`;
|
|
9674
|
+
}
|
|
9675
|
+
function getPendingChunkId(rawChunk) {
|
|
9676
|
+
if (!rawChunk || typeof rawChunk !== "object") {
|
|
9677
|
+
return null;
|
|
9678
|
+
}
|
|
9679
|
+
const id = rawChunk.id;
|
|
9680
|
+
return typeof id === "string" ? id : null;
|
|
9681
|
+
}
|
|
9133
9682
|
function metadataFromBlame(blame) {
|
|
9134
9683
|
if (!blame) {
|
|
9135
9684
|
return {};
|
|
@@ -9177,9 +9726,9 @@ var SWIFT_PARSER_VERSION = "1";
|
|
|
9177
9726
|
var METAL_PARSER_VERSION = "1";
|
|
9178
9727
|
var SYMBOL_EXTRACTOR_VERSION = "1";
|
|
9179
9728
|
function isPathWithinRoot2(filePath, rootPath) {
|
|
9180
|
-
const normalizedFilePath =
|
|
9181
|
-
const normalizedRoot =
|
|
9182
|
-
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${
|
|
9729
|
+
const normalizedFilePath = path19.resolve(filePath);
|
|
9730
|
+
const normalizedRoot = path19.resolve(rootPath);
|
|
9731
|
+
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${path19.sep}`);
|
|
9183
9732
|
}
|
|
9184
9733
|
function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCandidates, database, branchChunkIds, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
9185
9734
|
if (combined.length === 0) {
|
|
@@ -9510,10 +10059,10 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
9510
10059
|
}
|
|
9511
10060
|
if (options?.directory) {
|
|
9512
10061
|
const candidatePath = canonicalizePathForComparison(
|
|
9513
|
-
|
|
10062
|
+
path19.resolve(projectRoot, candidate.metadata.filePath.replace(/\\/g, path19.sep))
|
|
9514
10063
|
);
|
|
9515
10064
|
const directoryPath = canonicalizePathForComparison(
|
|
9516
|
-
|
|
10065
|
+
path19.resolve(projectRoot, options.directory.trim().replace(/\\/g, path19.sep))
|
|
9517
10066
|
);
|
|
9518
10067
|
if (!isPathWithinRoot2(candidatePath, directoryPath)) return false;
|
|
9519
10068
|
}
|
|
@@ -9589,6 +10138,7 @@ var Indexer = class _Indexer {
|
|
|
9589
10138
|
readerArtifactFingerprint = null;
|
|
9590
10139
|
writerArtifactFingerprint = null;
|
|
9591
10140
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
10141
|
+
fileBatchLimits;
|
|
9592
10142
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
9593
10143
|
this.projectRoot = projectRoot;
|
|
9594
10144
|
this.projectIdentityHash = hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
@@ -9600,6 +10150,7 @@ var Indexer = class _Indexer {
|
|
|
9600
10150
|
}
|
|
9601
10151
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
9602
10152
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
10153
|
+
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
9603
10154
|
this.config = config;
|
|
9604
10155
|
this.host = host;
|
|
9605
10156
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -9617,26 +10168,26 @@ var Indexer = class _Indexer {
|
|
|
9617
10168
|
return this.indexPathOverride ?? resolveProjectIndexPath(this.projectRoot, this.config.scope, this.host);
|
|
9618
10169
|
}
|
|
9619
10170
|
toCanonicalFilePath(filePath) {
|
|
9620
|
-
if (!
|
|
10171
|
+
if (!path19.isAbsolute(filePath)) {
|
|
9621
10172
|
return this.resolveStoredFilePath(filePath, this.projectRoot);
|
|
9622
10173
|
}
|
|
9623
|
-
if (
|
|
10174
|
+
if (path19.resolve(this.materializedProjectRoot) === path19.resolve(this.projectRoot) || !isPathWithinRoot2(filePath, this.materializedProjectRoot)) {
|
|
9624
10175
|
return filePath;
|
|
9625
10176
|
}
|
|
9626
|
-
return
|
|
10177
|
+
return path19.resolve(this.projectRoot, path19.relative(this.materializedProjectRoot, filePath));
|
|
9627
10178
|
}
|
|
9628
10179
|
toStoredFilePath(filePath) {
|
|
9629
10180
|
const canonicalFilePath = this.toCanonicalFilePath(filePath);
|
|
9630
10181
|
if (this.config.scope !== "project" || !isPathWithinRoot2(canonicalFilePath, this.projectRoot)) {
|
|
9631
10182
|
return canonicalFilePath;
|
|
9632
10183
|
}
|
|
9633
|
-
return
|
|
10184
|
+
return path19.relative(this.projectRoot, canonicalFilePath).split(path19.sep).join("/");
|
|
9634
10185
|
}
|
|
9635
10186
|
resolveStoredFilePath(filePath, rootPath = this.projectRoot) {
|
|
9636
|
-
if (
|
|
10187
|
+
if (path19.isAbsolute(filePath)) {
|
|
9637
10188
|
return filePath;
|
|
9638
10189
|
}
|
|
9639
|
-
const resolvedPath =
|
|
10190
|
+
const resolvedPath = path19.resolve(rootPath, ...filePath.split("/"));
|
|
9640
10191
|
if (!isPathWithinRoot2(resolvedPath, rootPath)) {
|
|
9641
10192
|
throw new Error(`Stored project path escapes project root: ${JSON.stringify(filePath)}`);
|
|
9642
10193
|
}
|
|
@@ -9660,7 +10211,7 @@ var Indexer = class _Indexer {
|
|
|
9660
10211
|
}
|
|
9661
10212
|
toMaterializedFilePath(filePath) {
|
|
9662
10213
|
const storedFilePath = this.toStoredFilePath(filePath);
|
|
9663
|
-
if (
|
|
10214
|
+
if (path19.isAbsolute(storedFilePath)) {
|
|
9664
10215
|
return storedFilePath;
|
|
9665
10216
|
}
|
|
9666
10217
|
return this.resolveStoredFilePath(storedFilePath, this.materializedProjectRoot);
|
|
@@ -9677,10 +10228,10 @@ var Indexer = class _Indexer {
|
|
|
9677
10228
|
}
|
|
9678
10229
|
getRuntimeArtifactPath(fileName) {
|
|
9679
10230
|
const namespace = this.getRuntimeArtifactNamespace();
|
|
9680
|
-
if (!namespace) return
|
|
9681
|
-
const extension =
|
|
10231
|
+
if (!namespace) return path19.join(this.indexPath, fileName);
|
|
10232
|
+
const extension = path19.extname(fileName);
|
|
9682
10233
|
const baseName = fileName.slice(0, fileName.length - extension.length);
|
|
9683
|
-
return
|
|
10234
|
+
return path19.join(this.indexPath, `${baseName}.${namespace}${extension}`);
|
|
9684
10235
|
}
|
|
9685
10236
|
refreshRuntimeArtifactPaths() {
|
|
9686
10237
|
this.fileHashCachePath = this.getRuntimeArtifactPath("file-hashes.json");
|
|
@@ -9693,14 +10244,14 @@ var Indexer = class _Indexer {
|
|
|
9693
10244
|
getMaterializedKnowledgeBases() {
|
|
9694
10245
|
const canonicalProjectRoot = this.getCanonicalPath(this.projectRoot);
|
|
9695
10246
|
return this.config.knowledgeBases.map((knowledgeBase) => {
|
|
9696
|
-
const configuredPath =
|
|
10247
|
+
const configuredPath = path19.isAbsolute(knowledgeBase) ? knowledgeBase : path19.resolve(this.projectRoot, knowledgeBase);
|
|
9697
10248
|
const canonicalPath = this.getCanonicalPath(configuredPath);
|
|
9698
10249
|
if (!isPathWithinRoot2(canonicalPath, canonicalProjectRoot)) {
|
|
9699
10250
|
return canonicalPath;
|
|
9700
10251
|
}
|
|
9701
|
-
return
|
|
10252
|
+
return path19.resolve(
|
|
9702
10253
|
this.materializedProjectRoot,
|
|
9703
|
-
|
|
10254
|
+
path19.relative(canonicalProjectRoot, canonicalPath)
|
|
9704
10255
|
);
|
|
9705
10256
|
});
|
|
9706
10257
|
}
|
|
@@ -9708,7 +10259,7 @@ var Indexer = class _Indexer {
|
|
|
9708
10259
|
try {
|
|
9709
10260
|
return canonicalizePathForComparison(targetPath);
|
|
9710
10261
|
} catch {
|
|
9711
|
-
return
|
|
10262
|
+
return path19.resolve(targetPath);
|
|
9712
10263
|
}
|
|
9713
10264
|
}
|
|
9714
10265
|
isProjectOwnedIndexPath() {
|
|
@@ -9828,7 +10379,7 @@ var Indexer = class _Indexer {
|
|
|
9828
10379
|
atomicWriteSync(targetPath, data) {
|
|
9829
10380
|
const lease = this.requireActiveLease();
|
|
9830
10381
|
const tempPath = createLeaseTemporaryPath(targetPath, lease.owner, "tmp");
|
|
9831
|
-
(0, import_fs12.mkdirSync)(
|
|
10382
|
+
(0, import_fs12.mkdirSync)(path19.dirname(targetPath), { recursive: true });
|
|
9832
10383
|
try {
|
|
9833
10384
|
(0, import_fs12.writeFileSync)(tempPath, data);
|
|
9834
10385
|
(0, import_fs12.renameSync)(tempPath, targetPath);
|
|
@@ -9838,14 +10389,14 @@ var Indexer = class _Indexer {
|
|
|
9838
10389
|
}
|
|
9839
10390
|
saveInvertedIndex(invertedIndex) {
|
|
9840
10391
|
this.atomicWriteSync(
|
|
9841
|
-
|
|
10392
|
+
path19.join(this.indexPath, "inverted-index.json"),
|
|
9842
10393
|
invertedIndex.serialize()
|
|
9843
10394
|
);
|
|
9844
10395
|
}
|
|
9845
10396
|
getScopedRoots() {
|
|
9846
10397
|
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(this.projectRoot)]);
|
|
9847
10398
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
9848
|
-
roots.add(this.getCanonicalPath(
|
|
10399
|
+
roots.add(this.getCanonicalPath(path19.resolve(this.projectRoot, kbRoot)));
|
|
9849
10400
|
}
|
|
9850
10401
|
return Array.from(roots);
|
|
9851
10402
|
}
|
|
@@ -9861,6 +10412,9 @@ var Indexer = class _Indexer {
|
|
|
9861
10412
|
}
|
|
9862
10413
|
return `${this.projectIdentityHash}:${branchName}`;
|
|
9863
10414
|
}
|
|
10415
|
+
resolveBranchCatalogKey(branchName) {
|
|
10416
|
+
return branchName === void 0 ? this.getBranchCatalogKey() : this.getBranchCatalogKeyFor(branchName);
|
|
10417
|
+
}
|
|
9864
10418
|
getBranchCommitMetadataKey(catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
9865
10419
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
9866
10420
|
return `index.branchCommit.${hashContent(branchKey).slice(0, 24)}`;
|
|
@@ -9955,13 +10509,13 @@ var Indexer = class _Indexer {
|
|
|
9955
10509
|
if (Array.from(this.fileHashCache.keys()).some((filePath) => this.isFileInCurrentScope(filePath, roots))) {
|
|
9956
10510
|
return true;
|
|
9957
10511
|
}
|
|
9958
|
-
|
|
9959
|
-
(batch
|
|
10512
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10513
|
+
if (batch.chunks.some((chunk) => {
|
|
9960
10514
|
const filePath = getPendingChunkFilePath(chunk);
|
|
9961
10515
|
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
9962
|
-
})
|
|
9963
|
-
|
|
9964
|
-
|
|
10516
|
+
})) {
|
|
10517
|
+
return true;
|
|
10518
|
+
}
|
|
9965
10519
|
}
|
|
9966
10520
|
if (!this.database) {
|
|
9967
10521
|
return false;
|
|
@@ -10102,40 +10656,25 @@ var Indexer = class _Indexer {
|
|
|
10102
10656
|
}
|
|
10103
10657
|
this.saveFileHashCache();
|
|
10104
10658
|
}
|
|
10105
|
-
partitionFailedBatches(roots, maxChunkTokens) {
|
|
10106
|
-
const scoped = [];
|
|
10107
|
-
const retained = [];
|
|
10108
|
-
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10109
|
-
const scopedChunks = batch.chunks.filter((chunk) => {
|
|
10110
|
-
const filePath = getPendingChunkFilePath(chunk);
|
|
10111
|
-
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
10112
|
-
});
|
|
10113
|
-
const retainedChunks = batch.chunks.filter((chunk) => {
|
|
10114
|
-
const filePath = getPendingChunkFilePath(chunk);
|
|
10115
|
-
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
10116
|
-
});
|
|
10117
|
-
if (scopedChunks.length > 0) {
|
|
10118
|
-
const normalizedBatch = normalizeFailedBatch({ ...batch, chunks: scopedChunks }, maxChunkTokens);
|
|
10119
|
-
if (normalizedBatch) {
|
|
10120
|
-
scoped.push(normalizedBatch);
|
|
10121
|
-
}
|
|
10122
|
-
}
|
|
10123
|
-
if (retainedChunks.length > 0) {
|
|
10124
|
-
retained.push({ ...batch, chunks: retainedChunks });
|
|
10125
|
-
}
|
|
10126
|
-
}
|
|
10127
|
-
return { scoped, retained };
|
|
10128
|
-
}
|
|
10129
10659
|
clearScopedFailedBatches(roots) {
|
|
10130
|
-
|
|
10131
|
-
|
|
10660
|
+
this.rewriteFailedBatchState((chunk) => {
|
|
10661
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
10662
|
+
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
10663
|
+
});
|
|
10132
10664
|
}
|
|
10133
10665
|
hasForeignScopedFileHashData(roots) {
|
|
10134
10666
|
return Array.from(this.fileHashCache.keys()).some((filePath) => !this.isFileInCurrentScope(filePath, roots));
|
|
10135
10667
|
}
|
|
10136
10668
|
hasForeignScopedFailedBatches(roots) {
|
|
10137
|
-
const
|
|
10138
|
-
|
|
10669
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10670
|
+
if (batch.chunks.some((chunk) => {
|
|
10671
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
10672
|
+
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
10673
|
+
})) {
|
|
10674
|
+
return true;
|
|
10675
|
+
}
|
|
10676
|
+
}
|
|
10677
|
+
return false;
|
|
10139
10678
|
}
|
|
10140
10679
|
hasForeignScopedBranchData() {
|
|
10141
10680
|
if (!this.database || this.config.scope !== "global") {
|
|
@@ -10160,10 +10699,6 @@ var Indexer = class _Indexer {
|
|
|
10160
10699
|
}
|
|
10161
10700
|
);
|
|
10162
10701
|
}
|
|
10163
|
-
saveScopedFailedBatches(batches, roots) {
|
|
10164
|
-
const { retained } = this.partitionFailedBatches(roots);
|
|
10165
|
-
this.saveFailedBatches([...retained, ...batches]);
|
|
10166
|
-
}
|
|
10167
10702
|
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
10168
10703
|
const allMetadata = store.getAllMetadata();
|
|
10169
10704
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
@@ -10260,70 +10795,137 @@ var Indexer = class _Indexer {
|
|
|
10260
10795
|
}
|
|
10261
10796
|
this.logger.info("Recovery complete, next index will re-process all files");
|
|
10262
10797
|
}
|
|
10263
|
-
|
|
10264
|
-
|
|
10265
|
-
|
|
10266
|
-
|
|
10267
|
-
|
|
10798
|
+
*loadSerializedFailedBatches() {
|
|
10799
|
+
let warned = false;
|
|
10800
|
+
const warn = (error) => {
|
|
10801
|
+
if (warned) return;
|
|
10802
|
+
warned = true;
|
|
10268
10803
|
this.logger.warn("Failed to load failed batch state, skipping persisted retries", {
|
|
10269
10804
|
failedBatchesPath: this.failedBatchesPath,
|
|
10270
|
-
error:
|
|
10805
|
+
error: getErrorMessage4(error)
|
|
10271
10806
|
});
|
|
10272
|
-
|
|
10807
|
+
};
|
|
10808
|
+
try {
|
|
10809
|
+
for (const record of readFailedBatchRecords(this.failedBatchesPath, {
|
|
10810
|
+
malformedLineAction: "skip",
|
|
10811
|
+
onMalformedLine: (error) => warn(error)
|
|
10812
|
+
})) {
|
|
10813
|
+
yield {
|
|
10814
|
+
chunks: record.chunks,
|
|
10815
|
+
error: record.error,
|
|
10816
|
+
attemptCount: record.attemptCount,
|
|
10817
|
+
lastAttempt: record.lastAttempt
|
|
10818
|
+
};
|
|
10819
|
+
}
|
|
10820
|
+
} catch (error) {
|
|
10821
|
+
warn(error);
|
|
10273
10822
|
}
|
|
10274
10823
|
}
|
|
10275
|
-
|
|
10276
|
-
|
|
10277
|
-
|
|
10824
|
+
createFailedBatchWriteState() {
|
|
10825
|
+
return {
|
|
10826
|
+
writer: createFailedBatchWriter(this.failedBatchesPath),
|
|
10827
|
+
recordsWritten: 0
|
|
10828
|
+
};
|
|
10829
|
+
}
|
|
10830
|
+
writeFailedBatchRecord(state, record) {
|
|
10831
|
+
state.writer.write(record);
|
|
10832
|
+
state.recordsWritten += record.chunks.length;
|
|
10833
|
+
}
|
|
10834
|
+
finalizeFailedBatchWriteState(state) {
|
|
10835
|
+
if (state.recordsWritten > 0) {
|
|
10836
|
+
state.writer.commit();
|
|
10837
|
+
return;
|
|
10278
10838
|
}
|
|
10279
|
-
|
|
10280
|
-
|
|
10281
|
-
|
|
10282
|
-
|
|
10283
|
-
|
|
10284
|
-
|
|
10839
|
+
state.writer.cleanup();
|
|
10840
|
+
this.clearFailedBatchState();
|
|
10841
|
+
}
|
|
10842
|
+
clearFailedBatchState() {
|
|
10843
|
+
if ((0, import_fs12.existsSync)(this.failedBatchesPath)) {
|
|
10844
|
+
try {
|
|
10845
|
+
(0, import_fs12.unlinkSync)(this.failedBatchesPath);
|
|
10846
|
+
} catch {
|
|
10285
10847
|
}
|
|
10286
|
-
|
|
10287
|
-
chunks,
|
|
10288
|
-
error: typeof batch.error === "string" ? batch.error : "Unknown embedding error",
|
|
10289
|
-
attemptCount: typeof batch.attemptCount === "number" ? batch.attemptCount : 1,
|
|
10290
|
-
lastAttempt: typeof batch.lastAttempt === "string" ? batch.lastAttempt : (/* @__PURE__ */ new Date()).toISOString()
|
|
10291
|
-
};
|
|
10292
|
-
}).filter((batch) => batch !== null);
|
|
10848
|
+
}
|
|
10293
10849
|
}
|
|
10294
|
-
|
|
10295
|
-
|
|
10296
|
-
|
|
10297
|
-
|
|
10298
|
-
|
|
10299
|
-
|
|
10850
|
+
rewriteFailedBatchState(shouldRetain) {
|
|
10851
|
+
const state = this.createFailedBatchWriteState();
|
|
10852
|
+
try {
|
|
10853
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10854
|
+
const retainedChunks = batch.chunks.filter(shouldRetain);
|
|
10855
|
+
if (retainedChunks.length > 0) {
|
|
10856
|
+
this.writeFailedBatchRecord(state, { ...batch, chunks: retainedChunks });
|
|
10300
10857
|
}
|
|
10301
10858
|
}
|
|
10302
|
-
|
|
10859
|
+
this.finalizeFailedBatchWriteState(state);
|
|
10860
|
+
} catch (error) {
|
|
10861
|
+
state.writer.cleanup();
|
|
10862
|
+
throw error;
|
|
10863
|
+
}
|
|
10864
|
+
}
|
|
10865
|
+
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
10866
|
+
const state = this.createFailedBatchWriteState();
|
|
10867
|
+
const latestById = /* @__PURE__ */ new Map();
|
|
10868
|
+
try {
|
|
10869
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10870
|
+
for (const rawChunk of batch.chunks) {
|
|
10871
|
+
const filePath = getPendingChunkFilePath(rawChunk);
|
|
10872
|
+
const inScope = roots === null || filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
10873
|
+
if (!inScope) {
|
|
10874
|
+
this.writeFailedBatchRecord(state, { ...batch, chunks: [rawChunk] });
|
|
10875
|
+
continue;
|
|
10876
|
+
}
|
|
10877
|
+
if (!shouldProcess(filePath)) {
|
|
10878
|
+
continue;
|
|
10879
|
+
}
|
|
10880
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
10881
|
+
if (!chunkId) {
|
|
10882
|
+
continue;
|
|
10883
|
+
}
|
|
10884
|
+
const existing = latestById.get(chunkId);
|
|
10885
|
+
if (!existing || batch.attemptCount >= existing.attemptCount) {
|
|
10886
|
+
latestById.set(chunkId, {
|
|
10887
|
+
attemptCount: batch.attemptCount,
|
|
10888
|
+
error: batch.error,
|
|
10889
|
+
lastAttempt: batch.lastAttempt
|
|
10890
|
+
});
|
|
10891
|
+
}
|
|
10892
|
+
}
|
|
10893
|
+
}
|
|
10894
|
+
return { state, latestById };
|
|
10895
|
+
} catch (error) {
|
|
10896
|
+
state.writer.cleanup();
|
|
10897
|
+
throw error;
|
|
10303
10898
|
}
|
|
10304
|
-
this.atomicWriteSync(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
10305
10899
|
}
|
|
10306
|
-
|
|
10307
|
-
const
|
|
10308
|
-
for (const batch of this.
|
|
10309
|
-
for (const
|
|
10310
|
-
const
|
|
10311
|
-
if (!
|
|
10900
|
+
*iterateLatestFailedChunks(latestById, roots, shouldProcess, maxChunkTokens) {
|
|
10901
|
+
const yielded = /* @__PURE__ */ new Set();
|
|
10902
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10903
|
+
for (const rawChunk of batch.chunks) {
|
|
10904
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
10905
|
+
if (!chunkId || yielded.has(chunkId)) {
|
|
10312
10906
|
continue;
|
|
10313
10907
|
}
|
|
10314
|
-
|
|
10908
|
+
const latest = latestById.get(chunkId);
|
|
10909
|
+
if (!latest || latest.attemptCount !== batch.attemptCount || latest.error !== batch.error || latest.lastAttempt !== batch.lastAttempt) {
|
|
10315
10910
|
continue;
|
|
10316
10911
|
}
|
|
10317
|
-
const
|
|
10318
|
-
|
|
10319
|
-
|
|
10320
|
-
|
|
10321
|
-
|
|
10322
|
-
|
|
10912
|
+
const filePath = getPendingChunkFilePath(rawChunk);
|
|
10913
|
+
const inScope = roots === null || filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
10914
|
+
if (!inScope || !shouldProcess(filePath)) {
|
|
10915
|
+
continue;
|
|
10916
|
+
}
|
|
10917
|
+
const normalized = normalizeFailedBatch({ ...batch, chunks: [rawChunk] }, maxChunkTokens);
|
|
10918
|
+
const chunk = normalized?.chunks[0];
|
|
10919
|
+
if (!chunk) {
|
|
10920
|
+
continue;
|
|
10323
10921
|
}
|
|
10922
|
+
yielded.add(chunkId);
|
|
10923
|
+
yield {
|
|
10924
|
+
chunk,
|
|
10925
|
+
attemptCount: batch.attemptCount
|
|
10926
|
+
};
|
|
10324
10927
|
}
|
|
10325
10928
|
}
|
|
10326
|
-
return Array.from(retryableById.values());
|
|
10327
10929
|
}
|
|
10328
10930
|
getProviderRateLimits(provider) {
|
|
10329
10931
|
switch (provider) {
|
|
@@ -10348,31 +10950,267 @@ var Indexer = class _Indexer {
|
|
|
10348
10950
|
return { concurrency: 3, intervalMs: 1e3, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
10349
10951
|
}
|
|
10350
10952
|
}
|
|
10351
|
-
async
|
|
10352
|
-
const
|
|
10353
|
-
|
|
10354
|
-
|
|
10953
|
+
async processPendingChunkBatch(chunks, options) {
|
|
10954
|
+
const result = {
|
|
10955
|
+
indexedChunks: 0,
|
|
10956
|
+
failedChunks: 0,
|
|
10957
|
+
tokensUsed: 0,
|
|
10958
|
+
failedChunkIds: /* @__PURE__ */ new Set()
|
|
10959
|
+
};
|
|
10960
|
+
if (chunks.length === 0) {
|
|
10961
|
+
return result;
|
|
10355
10962
|
}
|
|
10356
|
-
const
|
|
10357
|
-
|
|
10358
|
-
|
|
10359
|
-
|
|
10360
|
-
|
|
10963
|
+
const chunksNeedingEmbedding = [];
|
|
10964
|
+
let cachedChunkCount = 0;
|
|
10965
|
+
if (options.reuseCachedEmbeddings && !options.forceReembed) {
|
|
10966
|
+
const missingHashes = new Set(options.database.getMissingEmbeddings(chunks.map((chunk) => chunk.contentHash)));
|
|
10967
|
+
for (const chunk of chunks) {
|
|
10968
|
+
if (missingHashes.has(chunk.contentHash)) {
|
|
10969
|
+
chunksNeedingEmbedding.push(chunk);
|
|
10970
|
+
continue;
|
|
10971
|
+
}
|
|
10972
|
+
const embeddingBuffer = options.database.getEmbedding(chunk.contentHash);
|
|
10973
|
+
if (!embeddingBuffer) {
|
|
10974
|
+
chunksNeedingEmbedding.push(chunk);
|
|
10975
|
+
continue;
|
|
10976
|
+
}
|
|
10977
|
+
options.store.add(chunk.id, Array.from(bufferToFloat32Array(embeddingBuffer)), chunk.metadata);
|
|
10978
|
+
options.invertedIndex.removeChunk(chunk.id);
|
|
10979
|
+
options.invertedIndex.addChunk(chunk.id, chunk.content);
|
|
10980
|
+
options.onSucceeded?.([chunk]);
|
|
10981
|
+
result.indexedChunks += 1;
|
|
10982
|
+
cachedChunkCount += 1;
|
|
10983
|
+
}
|
|
10984
|
+
} else {
|
|
10985
|
+
chunksNeedingEmbedding.push(...chunks);
|
|
10361
10986
|
}
|
|
10362
|
-
|
|
10363
|
-
|
|
10987
|
+
this.logger.cache("info", "Embedding cache lookup", {
|
|
10988
|
+
needsEmbedding: chunksNeedingEmbedding.length,
|
|
10989
|
+
fromCache: cachedChunkCount
|
|
10990
|
+
});
|
|
10991
|
+
if (cachedChunkCount > 0) {
|
|
10992
|
+
this.logger.recordChunksFromCache(cachedChunkCount);
|
|
10993
|
+
options.onProgress?.(result);
|
|
10364
10994
|
}
|
|
10365
|
-
|
|
10366
|
-
|
|
10367
|
-
|
|
10368
|
-
const
|
|
10369
|
-
|
|
10370
|
-
|
|
10371
|
-
|
|
10372
|
-
|
|
10373
|
-
|
|
10374
|
-
|
|
10375
|
-
|
|
10995
|
+
if (chunksNeedingEmbedding.length === 0) {
|
|
10996
|
+
return result;
|
|
10997
|
+
}
|
|
10998
|
+
const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk]));
|
|
10999
|
+
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
11000
|
+
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
11001
|
+
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
11002
|
+
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
11003
|
+
chunksNeedingEmbedding,
|
|
11004
|
+
getDynamicBatchOptions(options.configuredProviderInfo)
|
|
11005
|
+
);
|
|
11006
|
+
let fatalError;
|
|
11007
|
+
for (const requestBatch of requestBatches) {
|
|
11008
|
+
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
11009
|
+
const task = options.queue.add(async () => {
|
|
11010
|
+
if (options.rateLimitState.backoffMs > 0) {
|
|
11011
|
+
await new Promise((resolve15) => setTimeout(resolve15, options.rateLimitState.backoffMs));
|
|
11012
|
+
}
|
|
11013
|
+
try {
|
|
11014
|
+
const embeddingResult = await pRetry(
|
|
11015
|
+
async () => {
|
|
11016
|
+
const texts = requestBatch.map((request) => request.text);
|
|
11017
|
+
return options.provider.embedBatch(texts);
|
|
11018
|
+
},
|
|
11019
|
+
{
|
|
11020
|
+
retries: this.config.indexing.retries,
|
|
11021
|
+
minTimeout: Math.max(this.config.indexing.retryDelayMs, options.providerRateLimits.minRetryMs),
|
|
11022
|
+
maxTimeout: options.providerRateLimits.maxRetryMs,
|
|
11023
|
+
factor: 2,
|
|
11024
|
+
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError),
|
|
11025
|
+
onFailedAttempt: (error) => {
|
|
11026
|
+
const message = getErrorMessage4(error);
|
|
11027
|
+
if (isRateLimitError(error)) {
|
|
11028
|
+
options.rateLimitState.backoffMs = Math.min(
|
|
11029
|
+
options.providerRateLimits.maxRetryMs,
|
|
11030
|
+
(options.rateLimitState.backoffMs || options.providerRateLimits.minRetryMs) * 2
|
|
11031
|
+
);
|
|
11032
|
+
this.logger.embedding("warn", "Rate limited, backing off", {
|
|
11033
|
+
attempt: error.attemptNumber,
|
|
11034
|
+
retriesLeft: error.retriesLeft,
|
|
11035
|
+
backoffMs: options.rateLimitState.backoffMs
|
|
11036
|
+
});
|
|
11037
|
+
} else {
|
|
11038
|
+
this.logger.embedding("error", "Embedding batch failed", {
|
|
11039
|
+
attempt: error.attemptNumber,
|
|
11040
|
+
error: message
|
|
11041
|
+
});
|
|
11042
|
+
}
|
|
11043
|
+
}
|
|
11044
|
+
}
|
|
11045
|
+
);
|
|
11046
|
+
if (options.rateLimitState.backoffMs > 0) {
|
|
11047
|
+
options.rateLimitState.backoffMs = Math.max(0, options.rateLimitState.backoffMs - 2e3);
|
|
11048
|
+
}
|
|
11049
|
+
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
11050
|
+
requestBatch.forEach((request, index) => {
|
|
11051
|
+
if (result.failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
11052
|
+
return;
|
|
11053
|
+
}
|
|
11054
|
+
const vector = embeddingResult.embeddings[index];
|
|
11055
|
+
if (!vector) {
|
|
11056
|
+
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
11057
|
+
}
|
|
11058
|
+
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
11059
|
+
parts[request.partIndex] = {
|
|
11060
|
+
vector,
|
|
11061
|
+
tokenCount: request.tokenCount
|
|
11062
|
+
};
|
|
11063
|
+
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
11064
|
+
touchedChunkIds.add(request.chunk.id);
|
|
11065
|
+
});
|
|
11066
|
+
const pooledResults = [];
|
|
11067
|
+
for (const chunkId of touchedChunkIds) {
|
|
11068
|
+
if (result.failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
11069
|
+
continue;
|
|
11070
|
+
}
|
|
11071
|
+
const chunk = pendingChunksById.get(chunkId);
|
|
11072
|
+
if (!chunk) {
|
|
11073
|
+
continue;
|
|
11074
|
+
}
|
|
11075
|
+
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
11076
|
+
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
11077
|
+
continue;
|
|
11078
|
+
}
|
|
11079
|
+
const orderedParts = parts;
|
|
11080
|
+
pooledResults.push({
|
|
11081
|
+
chunk,
|
|
11082
|
+
vector: poolEmbeddingVectors(
|
|
11083
|
+
orderedParts.map((part) => part.vector),
|
|
11084
|
+
orderedParts.map((part) => part.tokenCount)
|
|
11085
|
+
)
|
|
11086
|
+
});
|
|
11087
|
+
}
|
|
11088
|
+
if (pooledResults.length > 0) {
|
|
11089
|
+
options.database.upsertEmbeddingsBatch(pooledResults.map(({ chunk, vector }) => ({
|
|
11090
|
+
contentHash: chunk.contentHash,
|
|
11091
|
+
embedding: float32ArrayToBuffer(vector),
|
|
11092
|
+
chunkText: chunk.storageText,
|
|
11093
|
+
model: options.configuredProviderInfo.modelInfo.model
|
|
11094
|
+
})));
|
|
11095
|
+
const succeededChunks = pooledResults.map(({ chunk }) => chunk);
|
|
11096
|
+
for (const { chunk, vector } of pooledResults) {
|
|
11097
|
+
completedVectorsByChunkId.set(chunk.id, vector);
|
|
11098
|
+
}
|
|
11099
|
+
for (const chunk of succeededChunks) {
|
|
11100
|
+
completedChunkIds.add(chunk.id);
|
|
11101
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
11102
|
+
}
|
|
11103
|
+
}
|
|
11104
|
+
result.tokensUsed += embeddingResult.totalTokensUsed;
|
|
11105
|
+
this.logger.recordEmbeddingApiCall(embeddingResult.totalTokensUsed);
|
|
11106
|
+
this.logger.embedding("debug", "Embedded batch", {
|
|
11107
|
+
batchSize: pooledResults.length,
|
|
11108
|
+
requestCount: requestBatch.length,
|
|
11109
|
+
tokens: embeddingResult.totalTokensUsed
|
|
11110
|
+
});
|
|
11111
|
+
} catch (error) {
|
|
11112
|
+
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id)).filter((chunk) => options.incrementRepeatedFailures || !result.failedChunkIds.has(chunk.id));
|
|
11113
|
+
const failureMessage = getErrorMessage4(error);
|
|
11114
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
11115
|
+
for (const chunk of failedChunks) {
|
|
11116
|
+
if (!result.failedChunkIds.has(chunk.id)) {
|
|
11117
|
+
result.failedChunkIds.add(chunk.id);
|
|
11118
|
+
result.failedChunks += 1;
|
|
11119
|
+
}
|
|
11120
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
11121
|
+
const attemptCount = (options.attemptCounts.get(chunk.id) ?? 0) + 1;
|
|
11122
|
+
options.attemptCounts.set(chunk.id, attemptCount);
|
|
11123
|
+
this.writeFailedBatchRecord(options.failedState, {
|
|
11124
|
+
chunks: [chunk],
|
|
11125
|
+
error: failureMessage,
|
|
11126
|
+
attemptCount,
|
|
11127
|
+
lastAttempt: failureTimestamp
|
|
11128
|
+
});
|
|
11129
|
+
}
|
|
11130
|
+
this.logger.recordEmbeddingError();
|
|
11131
|
+
this.logger.embedding("error", "Failed to embed batch after retries", {
|
|
11132
|
+
batchSize: failedChunks.length,
|
|
11133
|
+
requestCount: requestBatch.length,
|
|
11134
|
+
error: failureMessage
|
|
11135
|
+
});
|
|
11136
|
+
}
|
|
11137
|
+
options.onProgress?.(result);
|
|
11138
|
+
});
|
|
11139
|
+
void task.catch((error) => {
|
|
11140
|
+
fatalError ??= error;
|
|
11141
|
+
});
|
|
11142
|
+
}
|
|
11143
|
+
await options.queue.onIdle();
|
|
11144
|
+
if (fatalError !== void 0) {
|
|
11145
|
+
throw fatalError;
|
|
11146
|
+
}
|
|
11147
|
+
const orderedSucceededChunks = chunksNeedingEmbedding.filter((chunk) => completedVectorsByChunkId.has(chunk.id));
|
|
11148
|
+
if (orderedSucceededChunks.length > 0) {
|
|
11149
|
+
try {
|
|
11150
|
+
options.store.addBatch(orderedSucceededChunks.map((chunk) => ({
|
|
11151
|
+
id: chunk.id,
|
|
11152
|
+
vector: completedVectorsByChunkId.get(chunk.id),
|
|
11153
|
+
metadata: chunk.metadata
|
|
11154
|
+
})));
|
|
11155
|
+
for (const chunk of orderedSucceededChunks) {
|
|
11156
|
+
options.invertedIndex.removeChunk(chunk.id);
|
|
11157
|
+
options.invertedIndex.addChunk(chunk.id, chunk.content);
|
|
11158
|
+
}
|
|
11159
|
+
options.onSucceeded?.(orderedSucceededChunks);
|
|
11160
|
+
result.indexedChunks += orderedSucceededChunks.length;
|
|
11161
|
+
this.logger.recordChunksEmbedded(orderedSucceededChunks.length);
|
|
11162
|
+
} catch (error) {
|
|
11163
|
+
const failureMessage = getErrorMessage4(error);
|
|
11164
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
11165
|
+
for (const chunk of orderedSucceededChunks) {
|
|
11166
|
+
options.store.remove(chunk.id);
|
|
11167
|
+
options.invertedIndex.removeChunk(chunk.id);
|
|
11168
|
+
result.failedChunkIds.add(chunk.id);
|
|
11169
|
+
result.failedChunks += 1;
|
|
11170
|
+
const attemptCount = (options.attemptCounts.get(chunk.id) ?? 0) + 1;
|
|
11171
|
+
options.attemptCounts.set(chunk.id, attemptCount);
|
|
11172
|
+
this.writeFailedBatchRecord(options.failedState, {
|
|
11173
|
+
chunks: [chunk],
|
|
11174
|
+
error: failureMessage,
|
|
11175
|
+
attemptCount,
|
|
11176
|
+
lastAttempt: failureTimestamp
|
|
11177
|
+
});
|
|
11178
|
+
}
|
|
11179
|
+
this.logger.recordEmbeddingError();
|
|
11180
|
+
this.logger.embedding("error", "Failed to publish embedded chunks", {
|
|
11181
|
+
batchSize: orderedSucceededChunks.length,
|
|
11182
|
+
error: failureMessage
|
|
11183
|
+
});
|
|
11184
|
+
}
|
|
11185
|
+
options.onProgress?.(result);
|
|
11186
|
+
}
|
|
11187
|
+
return result;
|
|
11188
|
+
}
|
|
11189
|
+
async rerankCandidatesWithApi(query, candidates, options) {
|
|
11190
|
+
const reranker = this.config.reranker;
|
|
11191
|
+
if (!reranker || !reranker.enabled || candidates.length <= 1) {
|
|
11192
|
+
return candidates;
|
|
11193
|
+
}
|
|
11194
|
+
const queryIntent = analyzeQueryIntent(query);
|
|
11195
|
+
const preferSourcePaths = queryIntent.preferSourcePaths;
|
|
11196
|
+
const docIntent = queryIntent.primary === "docs";
|
|
11197
|
+
if (options?.definitionIntent === true) {
|
|
11198
|
+
return candidates;
|
|
11199
|
+
}
|
|
11200
|
+
if (options?.hasIdentifierHints === true && preferSourcePaths && !docIntent) {
|
|
11201
|
+
return candidates;
|
|
11202
|
+
}
|
|
11203
|
+
const topN = Math.min(reranker.topN, candidates.length);
|
|
11204
|
+
const head = candidates.slice(0, topN);
|
|
11205
|
+
const tail = candidates.slice(topN);
|
|
11206
|
+
const grouped = /* @__PURE__ */ new Map([
|
|
11207
|
+
["implementation", []],
|
|
11208
|
+
["documentation", []],
|
|
11209
|
+
["test", []],
|
|
11210
|
+
["config", []],
|
|
11211
|
+
["other", []]
|
|
11212
|
+
]);
|
|
11213
|
+
for (const candidate of head) {
|
|
10376
11214
|
const band = classifyExternalRerankBand(candidate, queryIntent);
|
|
10377
11215
|
grouped.get(band)?.push(candidate);
|
|
10378
11216
|
}
|
|
@@ -10589,12 +11427,12 @@ var Indexer = class _Indexer {
|
|
|
10589
11427
|
}
|
|
10590
11428
|
}
|
|
10591
11429
|
captureReaderArtifactFingerprint() {
|
|
10592
|
-
const storePath =
|
|
11430
|
+
const storePath = path19.join(this.indexPath, "vectors");
|
|
10593
11431
|
return {
|
|
10594
11432
|
vectors: `${this.getReaderFileFingerprint(storePath)}|${this.getReaderFileFingerprint(`${storePath}.meta.json`)}`,
|
|
10595
|
-
keyword: this.getReaderFileFingerprint(
|
|
10596
|
-
database: this.getReaderFileFingerprint(
|
|
10597
|
-
databaseIdentity: this.getReaderFileFingerprint(
|
|
11433
|
+
keyword: this.getReaderFileFingerprint(path19.join(this.indexPath, "inverted-index.json")),
|
|
11434
|
+
database: this.getReaderFileFingerprint(path19.join(this.indexPath, "codebase.db")),
|
|
11435
|
+
databaseIdentity: this.getReaderFileFingerprint(path19.join(this.indexPath, "codebase.db"), true)
|
|
10598
11436
|
};
|
|
10599
11437
|
}
|
|
10600
11438
|
refreshReaderArtifacts() {
|
|
@@ -10619,10 +11457,10 @@ var Indexer = class _Indexer {
|
|
|
10619
11457
|
issues.set(component, this.createReadIssue(component, message));
|
|
10620
11458
|
this.readerArtifactRetryAfter.set(component, Date.now() + READER_ARTIFACT_RETRY_INTERVAL_MS);
|
|
10621
11459
|
};
|
|
10622
|
-
const storePath =
|
|
11460
|
+
const storePath = path19.join(this.indexPath, "vectors");
|
|
10623
11461
|
const vectorMetadataPath = `${storePath}.meta.json`;
|
|
10624
|
-
const invertedIndexPath =
|
|
10625
|
-
const dbPath =
|
|
11462
|
+
const invertedIndexPath = path19.join(this.indexPath, "inverted-index.json");
|
|
11463
|
+
const dbPath = path19.join(this.indexPath, "codebase.db");
|
|
10626
11464
|
if (vectorsChanged || retryDue("vectors")) {
|
|
10627
11465
|
const vectorStoreExists = (0, import_fs12.existsSync)(storePath);
|
|
10628
11466
|
const vectorMetadataExists = (0, import_fs12.existsSync)(vectorMetadataPath);
|
|
@@ -10747,10 +11585,10 @@ var Indexer = class _Indexer {
|
|
|
10747
11585
|
}
|
|
10748
11586
|
}
|
|
10749
11587
|
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
10750
|
-
const storePath =
|
|
11588
|
+
const storePath = path19.join(this.indexPath, "vectors");
|
|
10751
11589
|
const vectorMetadataPath = `${storePath}.meta.json`;
|
|
10752
|
-
const invertedIndexPath =
|
|
10753
|
-
const dbPath =
|
|
11590
|
+
const invertedIndexPath = path19.join(this.indexPath, "inverted-index.json");
|
|
11591
|
+
const dbPath = path19.join(this.indexPath, "codebase.db");
|
|
10754
11592
|
let dbIsNew = !(0, import_fs12.existsSync)(dbPath);
|
|
10755
11593
|
const readerArtifactFingerprint = mode === "reader" ? this.captureReaderArtifactFingerprint() : null;
|
|
10756
11594
|
if (mode === "writer") {
|
|
@@ -10915,7 +11753,7 @@ var Indexer = class _Indexer {
|
|
|
10915
11753
|
if (await this.tryResetCorruptedIndex("running automatic orphan garbage collection", error)) {
|
|
10916
11754
|
return {
|
|
10917
11755
|
resetCorruptedIndex: true,
|
|
10918
|
-
warning: this.getCorruptedIndexWarning(
|
|
11756
|
+
warning: this.getCorruptedIndexWarning(path19.join(this.indexPath, "codebase.db"))
|
|
10919
11757
|
};
|
|
10920
11758
|
}
|
|
10921
11759
|
throw error;
|
|
@@ -10930,7 +11768,7 @@ var Indexer = class _Indexer {
|
|
|
10930
11768
|
return;
|
|
10931
11769
|
}
|
|
10932
11770
|
const retainedEntries = store.getAllMetadata().filter(({ key }) => !excludedSet.has(key));
|
|
10933
|
-
const storeBasePath =
|
|
11771
|
+
const storeBasePath = path19.join(this.indexPath, "vectors");
|
|
10934
11772
|
const storeIndexPath = storeBasePath;
|
|
10935
11773
|
const storeMetadataPath = `${storeBasePath}.meta.json`;
|
|
10936
11774
|
const lease = this.requireActiveLease();
|
|
@@ -11017,7 +11855,7 @@ var Indexer = class _Indexer {
|
|
|
11017
11855
|
const names = await import_fs12.promises.readdir(this.indexPath);
|
|
11018
11856
|
const runtimeStatePattern = /^(?:file-hashes|failed-batches)(?:\.[a-f0-9]{16})?\.json$/;
|
|
11019
11857
|
await Promise.all(
|
|
11020
|
-
names.filter((name) => runtimeStatePattern.test(name)).map((name) => import_fs12.promises.rm(
|
|
11858
|
+
names.filter((name) => runtimeStatePattern.test(name)).map((name) => import_fs12.promises.rm(path19.join(this.indexPath, name), { force: true }))
|
|
11021
11859
|
);
|
|
11022
11860
|
}
|
|
11023
11861
|
async resetLocalIndexArtifacts() {
|
|
@@ -11033,13 +11871,13 @@ var Indexer = class _Indexer {
|
|
|
11033
11871
|
this.readerArtifactRetryAfter.clear();
|
|
11034
11872
|
this.fileHashCache.clear();
|
|
11035
11873
|
const resetPaths = [
|
|
11036
|
-
|
|
11037
|
-
|
|
11038
|
-
|
|
11039
|
-
|
|
11040
|
-
|
|
11041
|
-
|
|
11042
|
-
|
|
11874
|
+
path19.join(this.indexPath, "codebase.db"),
|
|
11875
|
+
path19.join(this.indexPath, "codebase.db-shm"),
|
|
11876
|
+
path19.join(this.indexPath, "codebase.db-wal"),
|
|
11877
|
+
path19.join(this.indexPath, "vectors"),
|
|
11878
|
+
path19.join(this.indexPath, "vectors.usearch"),
|
|
11879
|
+
path19.join(this.indexPath, "vectors.meta.json"),
|
|
11880
|
+
path19.join(this.indexPath, "inverted-index.json")
|
|
11043
11881
|
];
|
|
11044
11882
|
await Promise.all(resetPaths.map((targetPath) => import_fs12.promises.rm(targetPath, { recursive: true, force: true })));
|
|
11045
11883
|
await this.removeProjectRuntimeStateArtifacts();
|
|
@@ -11049,7 +11887,7 @@ var Indexer = class _Indexer {
|
|
|
11049
11887
|
if (!isSqliteCorruptionError(error)) {
|
|
11050
11888
|
return false;
|
|
11051
11889
|
}
|
|
11052
|
-
const dbPath =
|
|
11890
|
+
const dbPath = path19.join(this.indexPath, "codebase.db");
|
|
11053
11891
|
const warning = this.getCorruptedIndexWarning(dbPath);
|
|
11054
11892
|
const errorMessage = getErrorMessage4(error);
|
|
11055
11893
|
if (this.config.scope === "global") {
|
|
@@ -11358,7 +12196,6 @@ var Indexer = class _Indexer {
|
|
|
11358
12196
|
skippedFiles: [],
|
|
11359
12197
|
parseFailures: []
|
|
11360
12198
|
};
|
|
11361
|
-
const failedBatchesForCurrentRun = [];
|
|
11362
12199
|
onProgress?.({
|
|
11363
12200
|
phase: "scanning",
|
|
11364
12201
|
filesProcessed: 0,
|
|
@@ -11373,14 +12210,10 @@ var Indexer = class _Indexer {
|
|
|
11373
12210
|
const reparseCachedMetalFiles = database.getMetadata(metalParserMetadataKey) !== METAL_PARSER_VERSION;
|
|
11374
12211
|
const symbolExtractorMetadataKey = this.getSymbolExtractorVersionMetadataKey();
|
|
11375
12212
|
const refreshCachedSymbols = database.getMetadata(symbolExtractorMetadataKey) !== SYMBOL_EXTRACTOR_VERSION;
|
|
11376
|
-
if (reparseCachedSwiftFiles && Array.from(this.fileHashCache.keys()).some(
|
|
11377
|
-
(filePath) => path18.extname(filePath).toLowerCase() === ".swift"
|
|
11378
|
-
)) {
|
|
12213
|
+
if (reparseCachedSwiftFiles && Array.from(this.fileHashCache.keys()).some((filePath) => path19.extname(filePath).toLowerCase() === ".swift")) {
|
|
11379
12214
|
this.logger.info("Reindexing cached Swift files for parser support");
|
|
11380
12215
|
}
|
|
11381
|
-
if (reparseCachedMetalFiles && Array.from(this.fileHashCache.keys()).some(
|
|
11382
|
-
(filePath) => path18.extname(filePath).toLowerCase() === ".metal"
|
|
11383
|
-
)) {
|
|
12216
|
+
if (reparseCachedMetalFiles && Array.from(this.fileHashCache.keys()).some((filePath) => path19.extname(filePath).toLowerCase() === ".metal")) {
|
|
11384
12217
|
this.logger.info("Reindexing cached Metal files for parser support");
|
|
11385
12218
|
}
|
|
11386
12219
|
const includePatterns = [...this.config.include, ...this.config.additionalInclude];
|
|
@@ -11402,46 +12235,44 @@ var Indexer = class _Indexer {
|
|
|
11402
12235
|
totalFiles: files.length,
|
|
11403
12236
|
skippedFiles: skipped.length
|
|
11404
12237
|
});
|
|
11405
|
-
const
|
|
12238
|
+
const changedFileDescriptors = [];
|
|
11406
12239
|
const unchangedFilePaths = /* @__PURE__ */ new Set();
|
|
11407
12240
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
11408
12241
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
11409
|
-
for (const
|
|
11410
|
-
const storedPath = this.toStoredFilePath(
|
|
11411
|
-
const currentHash = hashFile(
|
|
12242
|
+
for (const file of files) {
|
|
12243
|
+
const storedPath = this.toStoredFilePath(file.path);
|
|
12244
|
+
const currentHash = hashFile(file.path);
|
|
11412
12245
|
currentFileHashes.set(storedPath, currentHash);
|
|
11413
12246
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
11414
12247
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
11415
12248
|
(chunk) => chunk.language === "php" || chunk.language === "c" || chunk.language === "cpp"
|
|
11416
12249
|
);
|
|
11417
|
-
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles &&
|
|
11418
|
-
const requiresMetalParserUpgrade = reparseCachedMetalFiles &&
|
|
12250
|
+
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path19.extname(storedPath).toLowerCase() === ".swift";
|
|
12251
|
+
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path19.extname(storedPath).toLowerCase() === ".metal";
|
|
11419
12252
|
if (cachedHashMatches && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
11420
12253
|
unchangedFilePaths.add(storedPath);
|
|
11421
12254
|
this.logger.recordCacheHit();
|
|
11422
12255
|
} else {
|
|
11423
|
-
|
|
11424
|
-
|
|
12256
|
+
changedFileDescriptors.push({
|
|
12257
|
+
storedPath,
|
|
12258
|
+
materializedPath: file.path,
|
|
12259
|
+
hash: currentHash,
|
|
12260
|
+
sourceBytes: file.size
|
|
12261
|
+
});
|
|
11425
12262
|
this.logger.recordCacheMiss();
|
|
11426
12263
|
}
|
|
11427
12264
|
}
|
|
11428
12265
|
this.logger.cache("info", "File hash cache results", {
|
|
11429
12266
|
unchanged: unchangedFilePaths.size,
|
|
11430
|
-
changed:
|
|
12267
|
+
changed: changedFileDescriptors.length
|
|
11431
12268
|
});
|
|
11432
12269
|
onProgress?.({
|
|
11433
12270
|
phase: "parsing",
|
|
11434
|
-
filesProcessed:
|
|
12271
|
+
filesProcessed: unchangedFilePaths.size,
|
|
11435
12272
|
totalFiles: files.length,
|
|
11436
12273
|
chunksProcessed: 0,
|
|
11437
12274
|
totalChunks: 0
|
|
11438
12275
|
});
|
|
11439
|
-
const parseStartTime = import_perf_hooks.performance.now();
|
|
11440
|
-
const parsedFiles = parseFiles(changedFiles);
|
|
11441
|
-
const parseMs = import_perf_hooks.performance.now() - parseStartTime;
|
|
11442
|
-
this.logger.recordFilesParsed(parsedFiles.length);
|
|
11443
|
-
this.logger.recordParseDuration(parseMs);
|
|
11444
|
-
this.logger.debug("Parsed changed files", { parsedCount: parsedFiles.length, parseMs: parseMs.toFixed(2) });
|
|
11445
12276
|
const existingChunks = /* @__PURE__ */ new Map();
|
|
11446
12277
|
const existingChunksByFile = /* @__PURE__ */ new Map();
|
|
11447
12278
|
const existingMetadataById = /* @__PURE__ */ new Map();
|
|
@@ -11457,17 +12288,17 @@ var Indexer = class _Indexer {
|
|
|
11457
12288
|
}
|
|
11458
12289
|
existingChunks.set(key, metadata.hash);
|
|
11459
12290
|
existingMetadataById.set(key, metadata);
|
|
11460
|
-
const fileChunks = existingChunksByFile.get(metadata.filePath)
|
|
12291
|
+
const fileChunks = existingChunksByFile.get(metadata.filePath) ?? /* @__PURE__ */ new Set();
|
|
11461
12292
|
fileChunks.add(key);
|
|
11462
12293
|
existingChunksByFile.set(metadata.filePath, fileChunks);
|
|
11463
12294
|
}
|
|
11464
12295
|
const currentChunkIds = /* @__PURE__ */ new Set();
|
|
11465
|
-
const
|
|
11466
|
-
const
|
|
12296
|
+
const allSymbolIds = /* @__PURE__ */ new Set();
|
|
12297
|
+
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
12298
|
+
const retryableChunksWithExistingData = /* @__PURE__ */ new Set();
|
|
11467
12299
|
const gitBlameEnabled = this.config.indexing.gitBlame.enabled && isGitRepo(this.materializedProjectRoot);
|
|
11468
12300
|
let backfilledBlameMetadata = false;
|
|
11469
12301
|
for (const filePath of unchangedFilePaths) {
|
|
11470
|
-
currentFilePaths.add(filePath);
|
|
11471
12302
|
const fileChunks = existingChunksByFile.get(filePath);
|
|
11472
12303
|
if (fileChunks) {
|
|
11473
12304
|
for (const chunkId of fileChunks) {
|
|
@@ -11475,315 +12306,475 @@ var Indexer = class _Indexer {
|
|
|
11475
12306
|
}
|
|
11476
12307
|
}
|
|
11477
12308
|
}
|
|
11478
|
-
const
|
|
11479
|
-
|
|
11480
|
-
|
|
11481
|
-
|
|
11482
|
-
|
|
11483
|
-
|
|
11484
|
-
|
|
11485
|
-
|
|
11486
|
-
|
|
11487
|
-
|
|
11488
|
-
|
|
12309
|
+
const shouldRetryFailedPath = (filePath) => filePath !== null && currentFileHashes.has(filePath) && unchangedFilePaths.has(filePath);
|
|
12310
|
+
const failedProcessing = this.prepareFailedBatchProcessing(scopedRoots, shouldRetryFailedPath);
|
|
12311
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
12312
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
12313
|
+
const queue = new PQueue({
|
|
12314
|
+
concurrency: providerRateLimits.concurrency,
|
|
12315
|
+
interval: providerRateLimits.intervalMs,
|
|
12316
|
+
intervalCap: providerRateLimits.concurrency
|
|
12317
|
+
});
|
|
12318
|
+
const rateLimitState = { backoffMs: 0 };
|
|
12319
|
+
let writeTransactionActive = false;
|
|
12320
|
+
try {
|
|
12321
|
+
database.beginWriteTransaction();
|
|
12322
|
+
writeTransactionActive = true;
|
|
12323
|
+
const blameChunkDataBatch = [];
|
|
12324
|
+
if (gitBlameEnabled) {
|
|
12325
|
+
const backfillItems = [];
|
|
12326
|
+
for (const chunkId of currentChunkIds) {
|
|
12327
|
+
const metadata = existingMetadataById.get(chunkId);
|
|
12328
|
+
if (!metadata || hasBlameMetadata(metadata)) {
|
|
12329
|
+
continue;
|
|
12330
|
+
}
|
|
12331
|
+
const chunk = database.getChunk(chunkId);
|
|
12332
|
+
if (!chunk) {
|
|
12333
|
+
continue;
|
|
12334
|
+
}
|
|
12335
|
+
const blame = await getChunkGitBlame(
|
|
12336
|
+
this.materializedProjectRoot,
|
|
12337
|
+
this.toMaterializedFilePath(chunk.filePath),
|
|
12338
|
+
chunk.startLine,
|
|
12339
|
+
chunk.endLine
|
|
12340
|
+
);
|
|
12341
|
+
const blameMetadata = metadataFromBlame(blame);
|
|
12342
|
+
if (!blameMetadata.blameSha) {
|
|
12343
|
+
continue;
|
|
12344
|
+
}
|
|
12345
|
+
blameChunkDataBatch.push({
|
|
12346
|
+
...chunk,
|
|
12347
|
+
blameSha: blameMetadata.blameSha,
|
|
12348
|
+
blameAuthor: blameMetadata.blameAuthor,
|
|
12349
|
+
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
12350
|
+
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
12351
|
+
blameSummary: blameMetadata.blameSummary
|
|
12352
|
+
});
|
|
12353
|
+
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
12354
|
+
if (embeddingBuffer) {
|
|
12355
|
+
backfillItems.push({
|
|
12356
|
+
id: chunkId,
|
|
12357
|
+
vector: Array.from(bufferToFloat32Array(embeddingBuffer)),
|
|
12358
|
+
metadata: { ...metadata, ...blameMetadata }
|
|
12359
|
+
});
|
|
12360
|
+
}
|
|
11489
12361
|
}
|
|
11490
|
-
|
|
11491
|
-
|
|
11492
|
-
this.toMaterializedFilePath(chunk.filePath),
|
|
11493
|
-
chunk.startLine,
|
|
11494
|
-
chunk.endLine
|
|
11495
|
-
);
|
|
11496
|
-
const blameMetadata = metadataFromBlame(blame);
|
|
11497
|
-
if (!blameMetadata.blameSha) {
|
|
11498
|
-
continue;
|
|
12362
|
+
if (blameChunkDataBatch.length > 0) {
|
|
12363
|
+
database.upsertChunksBatch(blameChunkDataBatch);
|
|
11499
12364
|
}
|
|
11500
|
-
|
|
11501
|
-
|
|
11502
|
-
|
|
11503
|
-
blameAuthor: blameMetadata.blameAuthor,
|
|
11504
|
-
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
11505
|
-
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
11506
|
-
blameSummary: blameMetadata.blameSummary
|
|
11507
|
-
});
|
|
11508
|
-
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
11509
|
-
if (!embeddingBuffer) {
|
|
11510
|
-
continue;
|
|
12365
|
+
if (backfillItems.length > 0) {
|
|
12366
|
+
store.addBatch(backfillItems);
|
|
12367
|
+
backfilledBlameMetadata = true;
|
|
11511
12368
|
}
|
|
11512
|
-
backfillItems.push({
|
|
11513
|
-
id: chunkId,
|
|
11514
|
-
vector: Array.from(bufferToFloat32Array(embeddingBuffer)),
|
|
11515
|
-
metadata: {
|
|
11516
|
-
...metadata,
|
|
11517
|
-
...blameMetadata
|
|
11518
|
-
}
|
|
11519
|
-
});
|
|
11520
12369
|
}
|
|
11521
|
-
|
|
11522
|
-
|
|
11523
|
-
|
|
11524
|
-
|
|
11525
|
-
|
|
11526
|
-
for (const parsed of parsedFiles) {
|
|
11527
|
-
currentFilePaths.add(parsed.path);
|
|
11528
|
-
if (parsed.chunks.length === 0) {
|
|
11529
|
-
stats.parseFailures.push(path18.isAbsolute(parsed.path) ? path18.relative(this.projectRoot, parsed.path) : parsed.path);
|
|
11530
|
-
}
|
|
11531
|
-
let chunksToProcess = parsed.chunks;
|
|
11532
|
-
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
11533
|
-
const changedFile = changedFiles.find((f) => f.path === parsed.path);
|
|
11534
|
-
if (changedFile) {
|
|
11535
|
-
const textChunks = parseFileAsText(parsed.path, changedFile.content);
|
|
11536
|
-
chunksToProcess = textChunks;
|
|
12370
|
+
for (const filePath of unchangedFilePaths) {
|
|
12371
|
+
for (const symbol of database.getSymbolsByFile(filePath)) {
|
|
12372
|
+
if (!restrictExistingChunksToBranch || previousBranchSymbolIdSet.has(symbol.id)) {
|
|
12373
|
+
allSymbolIds.add(symbol.id);
|
|
12374
|
+
}
|
|
11537
12375
|
}
|
|
11538
12376
|
}
|
|
11539
|
-
|
|
11540
|
-
|
|
11541
|
-
|
|
11542
|
-
|
|
11543
|
-
|
|
11544
|
-
|
|
11545
|
-
const
|
|
11546
|
-
|
|
11547
|
-
|
|
11548
|
-
|
|
11549
|
-
|
|
11550
|
-
|
|
11551
|
-
|
|
11552
|
-
|
|
11553
|
-
|
|
11554
|
-
|
|
11555
|
-
|
|
11556
|
-
|
|
11557
|
-
|
|
11558
|
-
|
|
11559
|
-
|
|
11560
|
-
filePath: parsed.path,
|
|
11561
|
-
startLine: chunk.startLine,
|
|
11562
|
-
endLine: chunk.endLine,
|
|
11563
|
-
nodeType: chunk.chunkType,
|
|
11564
|
-
name: chunk.name,
|
|
11565
|
-
language: chunk.language,
|
|
11566
|
-
blameSha: blameMetadata.blameSha,
|
|
11567
|
-
blameAuthor: blameMetadata.blameAuthor,
|
|
11568
|
-
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
11569
|
-
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
11570
|
-
blameSummary: blameMetadata.blameSummary
|
|
12377
|
+
let processedChangedFiles = 0;
|
|
12378
|
+
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
12379
|
+
changedFileDescriptors,
|
|
12380
|
+
(descriptor) => descriptor.sourceBytes,
|
|
12381
|
+
this.fileBatchLimits
|
|
12382
|
+
)) {
|
|
12383
|
+
const loadedFiles = await Promise.all(descriptorBatch.map(async (descriptor) => ({
|
|
12384
|
+
path: descriptor.storedPath,
|
|
12385
|
+
content: await import_fs12.promises.readFile(descriptor.materializedPath, "utf-8"),
|
|
12386
|
+
hash: descriptor.hash
|
|
12387
|
+
})));
|
|
12388
|
+
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
12389
|
+
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
12390
|
+
const parseStartTime = import_perf_hooks.performance.now();
|
|
12391
|
+
const parsedFiles = parseFiles(loadedFiles);
|
|
12392
|
+
const parseMs = import_perf_hooks.performance.now() - parseStartTime;
|
|
12393
|
+
this.logger.recordFilesParsed(parsedFiles.length);
|
|
12394
|
+
this.logger.recordParseDuration(parseMs);
|
|
12395
|
+
this.logger.debug("Parsed changed file batch", {
|
|
12396
|
+
parsedCount: parsedFiles.length,
|
|
12397
|
+
parseMs: parseMs.toFixed(2)
|
|
11571
12398
|
});
|
|
11572
|
-
|
|
11573
|
-
|
|
12399
|
+
const chunkDataBatch = [];
|
|
12400
|
+
const pendingChunks = [];
|
|
12401
|
+
const symbolBatch = [];
|
|
12402
|
+
const edgeBatch = [];
|
|
12403
|
+
for (const parsed of parsedFiles) {
|
|
12404
|
+
const loadedFile = loadedByPath.get(parsed.path);
|
|
12405
|
+
const descriptor = descriptorByPath.get(parsed.path);
|
|
12406
|
+
if (!loadedFile || !descriptor) {
|
|
12407
|
+
throw new Error(`Parsed file was not present in its source batch: ${parsed.path}`);
|
|
12408
|
+
}
|
|
12409
|
+
if (parsed.chunks.length === 0) {
|
|
12410
|
+
stats.parseFailures.push(path19.isAbsolute(parsed.path) ? path19.relative(this.projectRoot, parsed.path) : parsed.path);
|
|
12411
|
+
}
|
|
12412
|
+
let chunksToProcess = parsed.chunks;
|
|
12413
|
+
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
12414
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
12415
|
+
}
|
|
12416
|
+
chunksToProcess = selectIndexableChunks(
|
|
12417
|
+
chunksToProcess,
|
|
12418
|
+
this.config.indexing.maxChunksPerFile,
|
|
12419
|
+
this.config.indexing.semanticOnly
|
|
12420
|
+
);
|
|
12421
|
+
for (const chunk of chunksToProcess) {
|
|
12422
|
+
const id = this.getPreparedChunkId(generateChunkId(parsed.path, chunk));
|
|
12423
|
+
const contentHash = generateChunkHash(chunk);
|
|
12424
|
+
const existingContentHash = existingChunks.get(id);
|
|
12425
|
+
const existingChunk = gitBlameEnabled ? database.getChunk(id) : null;
|
|
12426
|
+
const blame = gitBlameEnabled && existingContentHash !== contentHash ? await getChunkGitBlame(
|
|
12427
|
+
this.materializedProjectRoot,
|
|
12428
|
+
descriptor.materializedPath,
|
|
12429
|
+
chunk.startLine,
|
|
12430
|
+
chunk.endLine
|
|
12431
|
+
) : blameFromChunkData(existingChunk);
|
|
12432
|
+
const blameMetadata = metadataFromBlame(blame);
|
|
12433
|
+
currentChunkIds.add(id);
|
|
12434
|
+
chunkDataBatch.push({
|
|
12435
|
+
chunkId: id,
|
|
12436
|
+
contentHash,
|
|
12437
|
+
filePath: parsed.path,
|
|
12438
|
+
startLine: chunk.startLine,
|
|
12439
|
+
endLine: chunk.endLine,
|
|
12440
|
+
nodeType: chunk.chunkType,
|
|
12441
|
+
name: chunk.name,
|
|
12442
|
+
language: chunk.language,
|
|
12443
|
+
blameSha: blameMetadata.blameSha,
|
|
12444
|
+
blameAuthor: blameMetadata.blameAuthor,
|
|
12445
|
+
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
12446
|
+
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
12447
|
+
blameSummary: blameMetadata.blameSummary
|
|
12448
|
+
});
|
|
12449
|
+
if (existingContentHash === contentHash) {
|
|
12450
|
+
continue;
|
|
12451
|
+
}
|
|
12452
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens).map((text) => ({
|
|
12453
|
+
text,
|
|
12454
|
+
tokenCount: estimateTokens(text)
|
|
12455
|
+
}));
|
|
12456
|
+
pendingChunks.push({
|
|
12457
|
+
id,
|
|
12458
|
+
texts,
|
|
12459
|
+
storageText: createPendingChunkStorageText(texts),
|
|
12460
|
+
content: chunk.content,
|
|
12461
|
+
contentHash,
|
|
12462
|
+
metadata: {
|
|
12463
|
+
filePath: parsed.path,
|
|
12464
|
+
startLine: chunk.startLine,
|
|
12465
|
+
endLine: chunk.endLine,
|
|
12466
|
+
chunkType: chunk.chunkType,
|
|
12467
|
+
name: chunk.name,
|
|
12468
|
+
language: chunk.language,
|
|
12469
|
+
hash: contentHash,
|
|
12470
|
+
...blameMetadata
|
|
12471
|
+
}
|
|
12472
|
+
});
|
|
12473
|
+
}
|
|
12474
|
+
const fileSymbols = [];
|
|
12475
|
+
for (const parsedSymbol of parsed.symbols) {
|
|
12476
|
+
if (!CALL_GRAPH_SYMBOL_CHUNK_TYPES.has(parsedSymbol.kind)) {
|
|
12477
|
+
continue;
|
|
12478
|
+
}
|
|
12479
|
+
const preparedNamespace = this.getPreparedBranchNamespace();
|
|
12480
|
+
const symbolId = `sym_${hashContent(
|
|
12481
|
+
(preparedNamespace ? `${preparedNamespace}:` : "") + parsed.path + ":" + parsedSymbol.name + ":" + parsedSymbol.kind + ":" + parsedSymbol.startLine + ":" + parsedSymbol.startCol + ":" + descriptor.hash
|
|
12482
|
+
).slice(0, 16)}`;
|
|
12483
|
+
const symbol = {
|
|
12484
|
+
id: symbolId,
|
|
12485
|
+
filePath: parsed.path,
|
|
12486
|
+
name: parsedSymbol.name,
|
|
12487
|
+
kind: parsedSymbol.kind,
|
|
12488
|
+
startLine: parsedSymbol.startLine,
|
|
12489
|
+
startCol: parsedSymbol.startCol,
|
|
12490
|
+
endLine: parsedSymbol.endLine,
|
|
12491
|
+
endCol: parsedSymbol.endCol,
|
|
12492
|
+
language: parsedSymbol.language
|
|
12493
|
+
};
|
|
12494
|
+
fileSymbols.push(symbol);
|
|
12495
|
+
symbolBatch.push(symbol);
|
|
12496
|
+
allSymbolIds.add(symbolId);
|
|
12497
|
+
}
|
|
12498
|
+
const fileLanguage = parsed.symbols[0]?.language ?? parsed.chunks[0]?.language;
|
|
12499
|
+
if (!fileLanguage || !CALL_GRAPH_LANGUAGES.has(fileLanguage)) {
|
|
12500
|
+
continue;
|
|
12501
|
+
}
|
|
12502
|
+
const isCaseInsensitiveLanguage = CASE_INSENSITIVE_LANGUAGES.has(fileLanguage);
|
|
12503
|
+
const normalizeSymbolKey = (name) => isCaseInsensitiveLanguage ? name.toLowerCase() : name;
|
|
12504
|
+
const symbolsByName = /* @__PURE__ */ new Map();
|
|
12505
|
+
for (const symbol of fileSymbols) {
|
|
12506
|
+
const key = normalizeSymbolKey(symbol.name);
|
|
12507
|
+
const symbols = symbolsByName.get(key) ?? [];
|
|
12508
|
+
symbols.push(symbol);
|
|
12509
|
+
symbolsByName.set(key, symbols);
|
|
12510
|
+
}
|
|
12511
|
+
for (const site of extractCalls(loadedFile.content, fileLanguage)) {
|
|
12512
|
+
const enclosingSymbol = findEnclosingSymbol(fileSymbols, site.line, site.column);
|
|
12513
|
+
if (!enclosingSymbol) {
|
|
12514
|
+
continue;
|
|
12515
|
+
}
|
|
12516
|
+
let candidates = symbolsByName.get(normalizeSymbolKey(site.calleeName));
|
|
12517
|
+
if (fileLanguage === "php" && candidates) {
|
|
12518
|
+
if (site.callType === "Constructor") {
|
|
12519
|
+
candidates = candidates.filter((candidate) => PHP_CLASS_SYMBOL_CHUNK_TYPES.has(candidate.kind));
|
|
12520
|
+
} else if (site.callType === "Call") {
|
|
12521
|
+
candidates = candidates.filter((candidate) => PHP_FUNCTION_SYMBOL_CHUNK_TYPES.has(candidate.kind));
|
|
12522
|
+
}
|
|
12523
|
+
}
|
|
12524
|
+
candidates = candidates?.filter(
|
|
12525
|
+
(symbol) => isCompatibleCFamilyCallTarget(fileLanguage, site.callType, symbol.kind)
|
|
12526
|
+
);
|
|
12527
|
+
const resolvedTarget = candidates?.length === 1 ? candidates[0] : void 0;
|
|
12528
|
+
edgeBatch.push({
|
|
12529
|
+
id: `edge_${hashContent(
|
|
12530
|
+
enclosingSymbol.id + ":" + site.calleeName + ":" + site.line + ":" + site.column
|
|
12531
|
+
).slice(0, 16)}`,
|
|
12532
|
+
fromSymbolId: enclosingSymbol.id,
|
|
12533
|
+
targetName: site.calleeName,
|
|
12534
|
+
toSymbolId: resolvedTarget?.id,
|
|
12535
|
+
callType: site.callType,
|
|
12536
|
+
confidence: site.confidence,
|
|
12537
|
+
line: site.line,
|
|
12538
|
+
col: site.column,
|
|
12539
|
+
isResolved: resolvedTarget !== void 0
|
|
12540
|
+
});
|
|
12541
|
+
}
|
|
11574
12542
|
}
|
|
11575
|
-
|
|
11576
|
-
|
|
11577
|
-
tokenCount: estimateTokens(text)
|
|
11578
|
-
}));
|
|
11579
|
-
const metadata = {
|
|
11580
|
-
filePath: parsed.path,
|
|
11581
|
-
startLine: chunk.startLine,
|
|
11582
|
-
endLine: chunk.endLine,
|
|
11583
|
-
chunkType: chunk.chunkType,
|
|
11584
|
-
name: chunk.name,
|
|
11585
|
-
language: chunk.language,
|
|
11586
|
-
hash: contentHash,
|
|
11587
|
-
...blameMetadata
|
|
11588
|
-
};
|
|
11589
|
-
pendingChunks.push({
|
|
11590
|
-
id,
|
|
11591
|
-
texts,
|
|
11592
|
-
storageText: createPendingChunkStorageText(texts),
|
|
11593
|
-
content: chunk.content,
|
|
11594
|
-
contentHash,
|
|
11595
|
-
metadata
|
|
11596
|
-
});
|
|
11597
|
-
}
|
|
11598
|
-
}
|
|
11599
|
-
const retryableFailedChunks = this.collectRetryableFailedChunks(
|
|
11600
|
-
currentFileHashes,
|
|
11601
|
-
unchangedFilePaths,
|
|
11602
|
-
getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)
|
|
11603
|
-
);
|
|
11604
|
-
const retryableFailedAttemptCounts = /* @__PURE__ */ new Map();
|
|
11605
|
-
const retryableChunksWithExistingData = /* @__PURE__ */ new Set();
|
|
11606
|
-
if (retryableFailedChunks.length > 0) {
|
|
11607
|
-
const pendingChunkIds = new Set(pendingChunks.map((chunk) => chunk.id));
|
|
11608
|
-
for (const { chunk, attemptCount } of retryableFailedChunks) {
|
|
11609
|
-
retryableFailedAttemptCounts.set(chunk.id, attemptCount);
|
|
11610
|
-
if (existingChunks.has(chunk.id)) {
|
|
11611
|
-
retryableChunksWithExistingData.add(chunk.id);
|
|
12543
|
+
if (chunkDataBatch.length > 0) {
|
|
12544
|
+
database.upsertChunksBatch(chunkDataBatch);
|
|
11612
12545
|
}
|
|
11613
|
-
if (
|
|
11614
|
-
|
|
11615
|
-
pendingChunkIds.add(chunk.id);
|
|
11616
|
-
currentChunkIds.add(chunk.id);
|
|
12546
|
+
if (symbolBatch.length > 0) {
|
|
12547
|
+
database.upsertSymbolsBatch(symbolBatch);
|
|
11617
12548
|
}
|
|
11618
|
-
|
|
11619
|
-
|
|
11620
|
-
|
|
11621
|
-
|
|
11622
|
-
|
|
11623
|
-
|
|
11624
|
-
|
|
11625
|
-
|
|
11626
|
-
|
|
11627
|
-
|
|
11628
|
-
|
|
11629
|
-
for (const parsedSymbol of parsed.symbols) {
|
|
11630
|
-
if (!CALL_GRAPH_SYMBOL_CHUNK_TYPES.has(parsedSymbol.kind)) continue;
|
|
11631
|
-
const preparedNamespace = this.getPreparedBranchNamespace();
|
|
11632
|
-
const symbolId = `sym_${hashContent(
|
|
11633
|
-
(preparedNamespace ? `${preparedNamespace}:` : "") + parsed.path + ":" + parsedSymbol.name + ":" + parsedSymbol.kind + ":" + parsedSymbol.startLine + ":" + parsedSymbol.startCol + ":" + changedFile.hash
|
|
11634
|
-
).slice(0, 16)}`;
|
|
11635
|
-
const symbol = {
|
|
11636
|
-
id: symbolId,
|
|
11637
|
-
filePath: parsed.path,
|
|
11638
|
-
name: parsedSymbol.name,
|
|
11639
|
-
kind: parsedSymbol.kind,
|
|
11640
|
-
startLine: parsedSymbol.startLine,
|
|
11641
|
-
startCol: parsedSymbol.startCol,
|
|
11642
|
-
endLine: parsedSymbol.endLine,
|
|
11643
|
-
endCol: parsedSymbol.endCol,
|
|
11644
|
-
language: parsedSymbol.language
|
|
11645
|
-
};
|
|
11646
|
-
fileSymbols.push(symbol);
|
|
11647
|
-
allSymbolIds.add(symbolId);
|
|
11648
|
-
}
|
|
11649
|
-
const fileLanguage = parsed.symbols[0]?.language ?? parsed.chunks[0]?.language;
|
|
11650
|
-
const isCaseInsensitiveLanguage = !!fileLanguage && CASE_INSENSITIVE_LANGUAGES.has(fileLanguage);
|
|
11651
|
-
const normalizeSymbolKey = (name) => isCaseInsensitiveLanguage ? name.toLowerCase() : name;
|
|
11652
|
-
const symbolsByName = /* @__PURE__ */ new Map();
|
|
11653
|
-
for (const symbol of fileSymbols) {
|
|
11654
|
-
const key = normalizeSymbolKey(symbol.name);
|
|
11655
|
-
const existing = symbolsByName.get(key) ?? [];
|
|
11656
|
-
existing.push(symbol);
|
|
11657
|
-
symbolsByName.set(key, existing);
|
|
11658
|
-
}
|
|
11659
|
-
if (fileSymbols.length > 0) {
|
|
11660
|
-
database.upsertSymbolsBatch(fileSymbols);
|
|
11661
|
-
symbolsByFile.set(parsed.path, fileSymbols);
|
|
11662
|
-
}
|
|
11663
|
-
if (!fileLanguage || !CALL_GRAPH_LANGUAGES.has(fileLanguage)) continue;
|
|
11664
|
-
const callSites = extractCalls(changedFile.content, fileLanguage);
|
|
11665
|
-
if (callSites.length === 0) continue;
|
|
11666
|
-
const edges = [];
|
|
11667
|
-
for (const site of callSites) {
|
|
11668
|
-
const enclosingSymbol = findEnclosingSymbol(
|
|
11669
|
-
fileSymbols,
|
|
11670
|
-
site.line,
|
|
11671
|
-
site.column
|
|
11672
|
-
);
|
|
11673
|
-
if (!enclosingSymbol) continue;
|
|
11674
|
-
const edgeId = `edge_${hashContent(enclosingSymbol.id + ":" + site.calleeName + ":" + site.line + ":" + site.column).slice(0, 16)}`;
|
|
11675
|
-
edges.push({
|
|
11676
|
-
id: edgeId,
|
|
11677
|
-
fromSymbolId: enclosingSymbol.id,
|
|
11678
|
-
targetName: site.calleeName,
|
|
11679
|
-
toSymbolId: void 0,
|
|
11680
|
-
callType: site.callType,
|
|
11681
|
-
confidence: site.confidence,
|
|
11682
|
-
line: site.line,
|
|
11683
|
-
col: site.column,
|
|
11684
|
-
isResolved: false
|
|
12549
|
+
if (edgeBatch.length > 0) {
|
|
12550
|
+
database.upsertCallEdgesBatch(edgeBatch);
|
|
12551
|
+
}
|
|
12552
|
+
processedChangedFiles += descriptorBatch.length;
|
|
12553
|
+
stats.totalChunks += pendingChunks.length;
|
|
12554
|
+
onProgress?.({
|
|
12555
|
+
phase: "parsing",
|
|
12556
|
+
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
12557
|
+
totalFiles: files.length,
|
|
12558
|
+
chunksProcessed: stats.indexedChunks,
|
|
12559
|
+
totalChunks: stats.totalChunks
|
|
11685
12560
|
});
|
|
11686
|
-
|
|
11687
|
-
|
|
11688
|
-
|
|
11689
|
-
|
|
11690
|
-
|
|
11691
|
-
|
|
11692
|
-
|
|
11693
|
-
|
|
11694
|
-
|
|
11695
|
-
|
|
11696
|
-
|
|
11697
|
-
|
|
11698
|
-
|
|
11699
|
-
|
|
12561
|
+
if (pendingChunks.length > 0) {
|
|
12562
|
+
onProgress?.({
|
|
12563
|
+
phase: "embedding",
|
|
12564
|
+
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
12565
|
+
totalFiles: files.length,
|
|
12566
|
+
chunksProcessed: stats.indexedChunks,
|
|
12567
|
+
totalChunks: stats.totalChunks
|
|
12568
|
+
});
|
|
12569
|
+
const batchResult = await this.processPendingChunkBatch(pendingChunks, {
|
|
12570
|
+
store,
|
|
12571
|
+
provider,
|
|
12572
|
+
invertedIndex,
|
|
12573
|
+
database,
|
|
12574
|
+
configuredProviderInfo,
|
|
12575
|
+
queue,
|
|
12576
|
+
providerRateLimits,
|
|
12577
|
+
rateLimitState,
|
|
12578
|
+
failedState: failedProcessing.state,
|
|
12579
|
+
attemptCounts: /* @__PURE__ */ new Map(),
|
|
12580
|
+
forceReembed: forceScopedReembed,
|
|
12581
|
+
reuseCachedEmbeddings: true,
|
|
12582
|
+
incrementRepeatedFailures: true,
|
|
12583
|
+
onProgress: (batchProgress) => onProgress?.({
|
|
12584
|
+
phase: "embedding",
|
|
12585
|
+
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
12586
|
+
totalFiles: files.length,
|
|
12587
|
+
chunksProcessed: stats.indexedChunks + batchProgress.indexedChunks,
|
|
12588
|
+
totalChunks: stats.totalChunks
|
|
12589
|
+
})
|
|
12590
|
+
});
|
|
12591
|
+
stats.indexedChunks += batchResult.indexedChunks;
|
|
12592
|
+
stats.failedChunks += batchResult.failedChunks;
|
|
12593
|
+
stats.tokensUsed += batchResult.tokensUsed;
|
|
12594
|
+
for (const chunkId of batchResult.failedChunkIds) {
|
|
12595
|
+
failedChunkIds.add(chunkId);
|
|
12596
|
+
if (forceScopedReembed) {
|
|
12597
|
+
failedForcedChunkIds.add(chunkId);
|
|
11700
12598
|
}
|
|
11701
12599
|
}
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
12600
|
+
}
|
|
12601
|
+
}
|
|
12602
|
+
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
12603
|
+
failedProcessing.latestById,
|
|
12604
|
+
scopedRoots,
|
|
12605
|
+
shouldRetryFailedPath,
|
|
12606
|
+
maxChunkTokens
|
|
12607
|
+
);
|
|
12608
|
+
for (const retryBatch of iterateOrderedFileBatches(
|
|
12609
|
+
retryableFailedChunks,
|
|
12610
|
+
({ chunk }) => Buffer.byteLength(chunk.content, "utf-8"),
|
|
12611
|
+
this.fileBatchLimits
|
|
12612
|
+
)) {
|
|
12613
|
+
const pendingChunks = retryBatch.map(({ chunk }) => chunk);
|
|
12614
|
+
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
12615
|
+
for (const chunk of pendingChunks) {
|
|
12616
|
+
currentChunkIds.add(chunk.id);
|
|
12617
|
+
if (existingChunks.has(chunk.id)) {
|
|
12618
|
+
retryableChunksWithExistingData.add(chunk.id);
|
|
12619
|
+
}
|
|
12620
|
+
}
|
|
12621
|
+
stats.totalChunks += pendingChunks.length;
|
|
12622
|
+
onProgress?.({
|
|
12623
|
+
phase: "embedding",
|
|
12624
|
+
filesProcessed: files.length,
|
|
12625
|
+
totalFiles: files.length,
|
|
12626
|
+
chunksProcessed: stats.indexedChunks,
|
|
12627
|
+
totalChunks: stats.totalChunks
|
|
12628
|
+
});
|
|
12629
|
+
const batchResult = await this.processPendingChunkBatch(pendingChunks, {
|
|
12630
|
+
store,
|
|
12631
|
+
provider,
|
|
12632
|
+
invertedIndex,
|
|
12633
|
+
database,
|
|
12634
|
+
configuredProviderInfo,
|
|
12635
|
+
queue,
|
|
12636
|
+
providerRateLimits,
|
|
12637
|
+
rateLimitState,
|
|
12638
|
+
failedState: failedProcessing.state,
|
|
12639
|
+
attemptCounts,
|
|
12640
|
+
forceReembed: forceScopedReembed,
|
|
12641
|
+
reuseCachedEmbeddings: true,
|
|
12642
|
+
incrementRepeatedFailures: true,
|
|
12643
|
+
onProgress: (batchProgress) => onProgress?.({
|
|
12644
|
+
phase: "embedding",
|
|
12645
|
+
filesProcessed: files.length,
|
|
12646
|
+
totalFiles: files.length,
|
|
12647
|
+
chunksProcessed: stats.indexedChunks + batchProgress.indexedChunks,
|
|
12648
|
+
totalChunks: stats.totalChunks
|
|
12649
|
+
})
|
|
12650
|
+
});
|
|
12651
|
+
stats.indexedChunks += batchResult.indexedChunks;
|
|
12652
|
+
stats.failedChunks += batchResult.failedChunks;
|
|
12653
|
+
stats.tokensUsed += batchResult.tokensUsed;
|
|
12654
|
+
for (const chunkId of batchResult.failedChunkIds) {
|
|
12655
|
+
failedChunkIds.add(chunkId);
|
|
12656
|
+
if (forceScopedReembed) {
|
|
12657
|
+
failedForcedChunkIds.add(chunkId);
|
|
11707
12658
|
}
|
|
11708
12659
|
}
|
|
11709
12660
|
}
|
|
11710
|
-
|
|
11711
|
-
|
|
11712
|
-
|
|
11713
|
-
|
|
11714
|
-
if (!restrictExistingChunksToBranch || previousBranchSymbolIdSet.has(sym.id)) {
|
|
11715
|
-
allSymbolIds.add(sym.id);
|
|
12661
|
+
const removedChunkIds = [];
|
|
12662
|
+
for (const [chunkId] of existingChunks) {
|
|
12663
|
+
if (!currentChunkIds.has(chunkId)) {
|
|
12664
|
+
removedChunkIds.push(chunkId);
|
|
11716
12665
|
}
|
|
11717
12666
|
}
|
|
11718
|
-
|
|
11719
|
-
|
|
11720
|
-
|
|
11721
|
-
|
|
11722
|
-
|
|
12667
|
+
const removedCount = removedChunkIds.length;
|
|
12668
|
+
stats.existingChunks = currentChunkIds.size - stats.totalChunks;
|
|
12669
|
+
stats.removedChunks = removedCount;
|
|
12670
|
+
this.logger.recordChunksProcessed(currentChunkIds.size);
|
|
12671
|
+
this.logger.recordChunksRemoved(removedCount);
|
|
12672
|
+
this.logger.info("Chunk analysis complete", {
|
|
12673
|
+
pending: stats.totalChunks,
|
|
12674
|
+
existing: stats.existingChunks,
|
|
12675
|
+
removed: removedCount
|
|
12676
|
+
});
|
|
12677
|
+
if (stats.totalChunks === 0 && removedCount === 0) {
|
|
12678
|
+
const removedStoredChunks = this.replaceBranchCatalog(
|
|
12679
|
+
store,
|
|
12680
|
+
invertedIndex,
|
|
12681
|
+
database,
|
|
12682
|
+
branchCatalogKey,
|
|
12683
|
+
previousBranchChunkIds,
|
|
12684
|
+
Array.from(currentChunkIds),
|
|
12685
|
+
previousBranchSymbolIds,
|
|
12686
|
+
Array.from(allSymbolIds)
|
|
12687
|
+
);
|
|
12688
|
+
const vectorPath = path19.join(this.indexPath, "vectors");
|
|
12689
|
+
const shouldFingerprintLegacyPair = !store.hasFingerprint() && (0, import_fs12.existsSync)(vectorPath) && (0, import_fs12.existsSync)(`${vectorPath}.meta.json`);
|
|
12690
|
+
if (backfilledBlameMetadata || shouldFingerprintLegacyPair || removedStoredChunks) {
|
|
12691
|
+
store.save();
|
|
12692
|
+
}
|
|
12693
|
+
if (removedStoredChunks) {
|
|
12694
|
+
this.saveInvertedIndex(invertedIndex);
|
|
12695
|
+
}
|
|
12696
|
+
if (scopedRoots) {
|
|
12697
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12698
|
+
} else {
|
|
12699
|
+
this.fileHashCache = currentFileHashes;
|
|
12700
|
+
this.saveFileHashCache();
|
|
12701
|
+
}
|
|
12702
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12703
|
+
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12704
|
+
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12705
|
+
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
12706
|
+
this.saveBranchCommit(database, indexedCommit);
|
|
12707
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12708
|
+
this.indexCompatibility = { compatible: true };
|
|
12709
|
+
database.commitWriteTransaction();
|
|
12710
|
+
writeTransactionActive = false;
|
|
12711
|
+
stats.durationMs = Date.now() - startTime;
|
|
12712
|
+
onProgress?.({
|
|
12713
|
+
phase: "complete",
|
|
12714
|
+
filesProcessed: files.length,
|
|
12715
|
+
totalFiles: files.length,
|
|
12716
|
+
chunksProcessed: 0,
|
|
12717
|
+
totalChunks: 0
|
|
12718
|
+
});
|
|
12719
|
+
return stats;
|
|
11723
12720
|
}
|
|
11724
|
-
|
|
11725
|
-
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11732
|
-
|
|
11733
|
-
|
|
11734
|
-
|
|
11735
|
-
});
|
|
11736
|
-
if (pendingChunks.length === 0 && removedCount === 0) {
|
|
11737
|
-
const removedStoredChunks = this.replaceBranchCatalog(
|
|
11738
|
-
store,
|
|
11739
|
-
invertedIndex,
|
|
11740
|
-
database,
|
|
11741
|
-
branchCatalogKey,
|
|
11742
|
-
previousBranchChunkIds,
|
|
11743
|
-
Array.from(currentChunkIds),
|
|
11744
|
-
previousBranchSymbolIds,
|
|
11745
|
-
Array.from(allSymbolIds)
|
|
11746
|
-
);
|
|
11747
|
-
const vectorPath = path18.join(this.indexPath, "vectors");
|
|
11748
|
-
const shouldFingerprintLegacyPair = !store.hasFingerprint() && (0, import_fs12.existsSync)(vectorPath) && (0, import_fs12.existsSync)(`${vectorPath}.meta.json`);
|
|
11749
|
-
if (backfilledBlameMetadata || shouldFingerprintLegacyPair || removedStoredChunks) {
|
|
12721
|
+
if (stats.totalChunks === 0) {
|
|
12722
|
+
this.replaceBranchCatalog(
|
|
12723
|
+
store,
|
|
12724
|
+
invertedIndex,
|
|
12725
|
+
database,
|
|
12726
|
+
branchCatalogKey,
|
|
12727
|
+
previousBranchChunkIds,
|
|
12728
|
+
Array.from(currentChunkIds),
|
|
12729
|
+
previousBranchSymbolIds,
|
|
12730
|
+
Array.from(allSymbolIds)
|
|
12731
|
+
);
|
|
11750
12732
|
store.save();
|
|
11751
|
-
}
|
|
11752
|
-
if (removedStoredChunks) {
|
|
11753
12733
|
this.saveInvertedIndex(invertedIndex);
|
|
12734
|
+
if (scopedRoots) {
|
|
12735
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12736
|
+
} else {
|
|
12737
|
+
this.fileHashCache = currentFileHashes;
|
|
12738
|
+
this.saveFileHashCache();
|
|
12739
|
+
}
|
|
12740
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12741
|
+
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12742
|
+
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12743
|
+
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
12744
|
+
this.saveBranchCommit(database, indexedCommit);
|
|
12745
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12746
|
+
this.indexCompatibility = { compatible: true };
|
|
12747
|
+
database.commitWriteTransaction();
|
|
12748
|
+
writeTransactionActive = false;
|
|
12749
|
+
stats.durationMs = Date.now() - startTime;
|
|
12750
|
+
onProgress?.({
|
|
12751
|
+
phase: "complete",
|
|
12752
|
+
filesProcessed: files.length,
|
|
12753
|
+
totalFiles: files.length,
|
|
12754
|
+
chunksProcessed: 0,
|
|
12755
|
+
totalChunks: 0
|
|
12756
|
+
});
|
|
12757
|
+
return stats;
|
|
11754
12758
|
}
|
|
11755
|
-
if (scopedRoots) {
|
|
11756
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11757
|
-
this.clearScopedFailedBatches(scopedRoots);
|
|
11758
|
-
} else {
|
|
11759
|
-
this.fileHashCache = currentFileHashes;
|
|
11760
|
-
this.saveFileHashCache();
|
|
11761
|
-
this.saveFailedBatches([]);
|
|
11762
|
-
}
|
|
11763
|
-
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11764
|
-
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11765
|
-
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
11766
|
-
this.saveBranchCommit(database, indexedCommit);
|
|
11767
|
-
this.saveIndexMetadata(configuredProviderInfo);
|
|
11768
|
-
this.indexCompatibility = { compatible: true };
|
|
11769
|
-
stats.durationMs = Date.now() - startTime;
|
|
11770
12759
|
onProgress?.({
|
|
11771
|
-
phase: "
|
|
12760
|
+
phase: "storing",
|
|
11772
12761
|
filesProcessed: files.length,
|
|
11773
12762
|
totalFiles: files.length,
|
|
11774
|
-
chunksProcessed:
|
|
11775
|
-
totalChunks:
|
|
12763
|
+
chunksProcessed: stats.indexedChunks,
|
|
12764
|
+
totalChunks: stats.totalChunks
|
|
12765
|
+
});
|
|
12766
|
+
const branchChunkIds = Array.from(currentChunkIds).filter((chunkId) => {
|
|
12767
|
+
const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId);
|
|
12768
|
+
const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId);
|
|
12769
|
+
return !isNewlyFailed && !isForcedFailed;
|
|
11776
12770
|
});
|
|
11777
|
-
return stats;
|
|
11778
|
-
}
|
|
11779
|
-
if (pendingChunks.length === 0) {
|
|
11780
12771
|
this.replaceBranchCatalog(
|
|
11781
12772
|
store,
|
|
11782
12773
|
invertedIndex,
|
|
11783
12774
|
database,
|
|
11784
12775
|
branchCatalogKey,
|
|
11785
12776
|
previousBranchChunkIds,
|
|
11786
|
-
|
|
12777
|
+
branchChunkIds,
|
|
11787
12778
|
previousBranchSymbolIds,
|
|
11788
12779
|
Array.from(allSymbolIds)
|
|
11789
12780
|
);
|
|
@@ -11791,11 +12782,35 @@ var Indexer = class _Indexer {
|
|
|
11791
12782
|
this.saveInvertedIndex(invertedIndex);
|
|
11792
12783
|
if (scopedRoots) {
|
|
11793
12784
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11794
|
-
this.clearScopedFailedBatches(scopedRoots);
|
|
11795
12785
|
} else {
|
|
11796
12786
|
this.fileHashCache = currentFileHashes;
|
|
11797
12787
|
this.saveFileHashCache();
|
|
11798
|
-
|
|
12788
|
+
}
|
|
12789
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12790
|
+
database.commitWriteTransaction();
|
|
12791
|
+
writeTransactionActive = false;
|
|
12792
|
+
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
12793
|
+
const gcReset = await this.maybeRunOrphanGc();
|
|
12794
|
+
if (gcReset) {
|
|
12795
|
+
stats.durationMs = Date.now() - startTime;
|
|
12796
|
+
stats.warning = gcReset.warning;
|
|
12797
|
+
stats.resetCorruptedIndex = true;
|
|
12798
|
+
this.logger.recordIndexingEnd();
|
|
12799
|
+
this.logger.warn("Indexing ended after resetting corrupted local index during automatic GC", {
|
|
12800
|
+
files: stats.totalFiles,
|
|
12801
|
+
indexed: stats.indexedChunks,
|
|
12802
|
+
existing: stats.existingChunks,
|
|
12803
|
+
removed: stats.removedChunks,
|
|
12804
|
+
failed: stats.failedChunks,
|
|
12805
|
+
tokens: stats.tokensUsed,
|
|
12806
|
+
durationMs: stats.durationMs
|
|
12807
|
+
});
|
|
12808
|
+
return stats;
|
|
12809
|
+
}
|
|
12810
|
+
}
|
|
12811
|
+
stats.durationMs = Date.now() - startTime;
|
|
12812
|
+
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
12813
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11799
12814
|
}
|
|
11800
12815
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11801
12816
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
@@ -11803,307 +12818,40 @@ var Indexer = class _Indexer {
|
|
|
11803
12818
|
this.saveBranchCommit(database, indexedCommit);
|
|
11804
12819
|
this.saveIndexMetadata(configuredProviderInfo);
|
|
11805
12820
|
this.indexCompatibility = { compatible: true };
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
11810
|
-
|
|
11811
|
-
|
|
11812
|
-
|
|
11813
|
-
|
|
11814
|
-
|
|
11815
|
-
|
|
11816
|
-
|
|
11817
|
-
|
|
11818
|
-
filesProcessed: files.length,
|
|
11819
|
-
totalFiles: files.length,
|
|
11820
|
-
chunksProcessed: 0,
|
|
11821
|
-
totalChunks: pendingChunks.length
|
|
11822
|
-
});
|
|
11823
|
-
const allContentHashes = pendingChunks.map((c) => c.contentHash);
|
|
11824
|
-
const missingHashes = new Set(database.getMissingEmbeddings(allContentHashes));
|
|
11825
|
-
const forcedReembedChunkIds = forceScopedReembed ? new Set(pendingChunks.map((chunk) => chunk.id)) : /* @__PURE__ */ new Set();
|
|
11826
|
-
const chunksNeedingEmbedding = pendingChunks.filter((c) => forcedReembedChunkIds.has(c.id) || missingHashes.has(c.contentHash));
|
|
11827
|
-
const chunksWithExistingEmbedding = pendingChunks.filter((c) => !forcedReembedChunkIds.has(c.id) && !missingHashes.has(c.contentHash));
|
|
11828
|
-
this.logger.cache("info", "Embedding cache lookup", {
|
|
11829
|
-
needsEmbedding: chunksNeedingEmbedding.length,
|
|
11830
|
-
fromCache: chunksWithExistingEmbedding.length
|
|
11831
|
-
});
|
|
11832
|
-
this.logger.recordChunksFromCache(chunksWithExistingEmbedding.length);
|
|
11833
|
-
for (const chunk of chunksWithExistingEmbedding) {
|
|
11834
|
-
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
11835
|
-
if (embeddingBuffer) {
|
|
11836
|
-
const vector = bufferToFloat32Array(embeddingBuffer);
|
|
11837
|
-
store.add(chunk.id, Array.from(vector), chunk.metadata);
|
|
11838
|
-
invertedIndex.removeChunk(chunk.id);
|
|
11839
|
-
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
11840
|
-
stats.indexedChunks++;
|
|
11841
|
-
}
|
|
11842
|
-
}
|
|
11843
|
-
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
11844
|
-
const queue = new PQueue({
|
|
11845
|
-
concurrency: providerRateLimits.concurrency,
|
|
11846
|
-
interval: providerRateLimits.intervalMs,
|
|
11847
|
-
intervalCap: providerRateLimits.concurrency
|
|
11848
|
-
});
|
|
11849
|
-
const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk]));
|
|
11850
|
-
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
11851
|
-
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
11852
|
-
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
11853
|
-
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
11854
|
-
chunksNeedingEmbedding,
|
|
11855
|
-
getDynamicBatchOptions(configuredProviderInfo)
|
|
11856
|
-
);
|
|
11857
|
-
let rateLimitBackoffMs = 0;
|
|
11858
|
-
for (const requestBatch of requestBatches) {
|
|
11859
|
-
queue.add(async () => {
|
|
11860
|
-
if (rateLimitBackoffMs > 0) {
|
|
11861
|
-
await new Promise((resolve15) => setTimeout(resolve15, rateLimitBackoffMs));
|
|
11862
|
-
}
|
|
11863
|
-
try {
|
|
11864
|
-
const result = await pRetry(
|
|
11865
|
-
async () => {
|
|
11866
|
-
const texts = requestBatch.map((request) => request.text);
|
|
11867
|
-
return provider.embedBatch(texts);
|
|
11868
|
-
},
|
|
11869
|
-
{
|
|
11870
|
-
retries: this.config.indexing.retries,
|
|
11871
|
-
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
11872
|
-
maxTimeout: providerRateLimits.maxRetryMs,
|
|
11873
|
-
factor: 2,
|
|
11874
|
-
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError),
|
|
11875
|
-
onFailedAttempt: (error) => {
|
|
11876
|
-
const message = getErrorMessage4(error);
|
|
11877
|
-
if (isRateLimitError(error)) {
|
|
11878
|
-
rateLimitBackoffMs = Math.min(providerRateLimits.maxRetryMs, (rateLimitBackoffMs || providerRateLimits.minRetryMs) * 2);
|
|
11879
|
-
this.logger.embedding("warn", `Rate limited, backing off`, {
|
|
11880
|
-
attempt: error.attemptNumber,
|
|
11881
|
-
retriesLeft: error.retriesLeft,
|
|
11882
|
-
backoffMs: rateLimitBackoffMs
|
|
11883
|
-
});
|
|
11884
|
-
} else {
|
|
11885
|
-
this.logger.embedding("error", `Embedding batch failed`, {
|
|
11886
|
-
attempt: error.attemptNumber,
|
|
11887
|
-
error: message
|
|
11888
|
-
});
|
|
11889
|
-
}
|
|
11890
|
-
}
|
|
11891
|
-
}
|
|
11892
|
-
);
|
|
11893
|
-
if (rateLimitBackoffMs > 0) {
|
|
11894
|
-
rateLimitBackoffMs = Math.max(0, rateLimitBackoffMs - 2e3);
|
|
11895
|
-
}
|
|
11896
|
-
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
11897
|
-
requestBatch.forEach((request, idx) => {
|
|
11898
|
-
if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
11899
|
-
return;
|
|
11900
|
-
}
|
|
11901
|
-
const vector = result.embeddings[idx];
|
|
11902
|
-
if (!vector) {
|
|
11903
|
-
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
11904
|
-
}
|
|
11905
|
-
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
11906
|
-
parts[request.partIndex] = {
|
|
11907
|
-
vector,
|
|
11908
|
-
tokenCount: request.tokenCount
|
|
11909
|
-
};
|
|
11910
|
-
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
11911
|
-
touchedChunkIds.add(request.chunk.id);
|
|
11912
|
-
});
|
|
11913
|
-
const pooledResults = [];
|
|
11914
|
-
for (const chunkId of touchedChunkIds) {
|
|
11915
|
-
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
11916
|
-
continue;
|
|
11917
|
-
}
|
|
11918
|
-
const chunk = pendingChunksById.get(chunkId);
|
|
11919
|
-
if (!chunk) {
|
|
11920
|
-
continue;
|
|
11921
|
-
}
|
|
11922
|
-
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
11923
|
-
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
11924
|
-
continue;
|
|
11925
|
-
}
|
|
11926
|
-
const orderedParts = parts;
|
|
11927
|
-
pooledResults.push({
|
|
11928
|
-
chunk,
|
|
11929
|
-
vector: poolEmbeddingVectors(
|
|
11930
|
-
orderedParts.map((part) => part.vector),
|
|
11931
|
-
orderedParts.map((part) => part.tokenCount)
|
|
11932
|
-
)
|
|
11933
|
-
});
|
|
11934
|
-
}
|
|
11935
|
-
if (pooledResults.length > 0) {
|
|
11936
|
-
const items = pooledResults.map(({ chunk, vector }) => ({
|
|
11937
|
-
id: chunk.id,
|
|
11938
|
-
vector,
|
|
11939
|
-
metadata: chunk.metadata
|
|
11940
|
-
}));
|
|
11941
|
-
store.addBatch(items);
|
|
11942
|
-
const embeddingBatchItems = pooledResults.map(({ chunk, vector }) => ({
|
|
11943
|
-
contentHash: chunk.contentHash,
|
|
11944
|
-
embedding: float32ArrayToBuffer(vector),
|
|
11945
|
-
chunkText: chunk.storageText,
|
|
11946
|
-
model: configuredProviderInfo.modelInfo.model
|
|
11947
|
-
}));
|
|
11948
|
-
try {
|
|
11949
|
-
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
11950
|
-
} catch (dbError) {
|
|
11951
|
-
this.rebuildVectorStoreExcludingChunkIds(
|
|
11952
|
-
store,
|
|
11953
|
-
database,
|
|
11954
|
-
pooledResults.map(({ chunk }) => chunk.id)
|
|
11955
|
-
);
|
|
11956
|
-
throw dbError;
|
|
11957
|
-
}
|
|
11958
|
-
for (const { chunk } of pooledResults) {
|
|
11959
|
-
invertedIndex.removeChunk(chunk.id);
|
|
11960
|
-
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
11961
|
-
completedChunkIds.add(chunk.id);
|
|
11962
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
11963
|
-
}
|
|
11964
|
-
stats.indexedChunks += pooledResults.length;
|
|
11965
|
-
this.logger.recordChunksEmbedded(pooledResults.length);
|
|
11966
|
-
}
|
|
11967
|
-
stats.tokensUsed += result.totalTokensUsed;
|
|
11968
|
-
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
11969
|
-
this.logger.embedding("debug", `Embedded batch`, {
|
|
11970
|
-
batchSize: pooledResults.length,
|
|
11971
|
-
requestCount: requestBatch.length,
|
|
11972
|
-
tokens: result.totalTokensUsed
|
|
11973
|
-
});
|
|
11974
|
-
onProgress?.({
|
|
11975
|
-
phase: "embedding",
|
|
11976
|
-
filesProcessed: files.length,
|
|
11977
|
-
totalFiles: files.length,
|
|
11978
|
-
chunksProcessed: stats.indexedChunks,
|
|
11979
|
-
totalChunks: pendingChunks.length
|
|
11980
|
-
});
|
|
11981
|
-
} catch (error) {
|
|
11982
|
-
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id));
|
|
11983
|
-
const failureMessage = getErrorMessage4(error);
|
|
11984
|
-
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
11985
|
-
for (const chunk of failedChunks) {
|
|
11986
|
-
if (!failedChunkIds.has(chunk.id)) {
|
|
11987
|
-
failedChunkIds.add(chunk.id);
|
|
11988
|
-
stats.failedChunks += 1;
|
|
11989
|
-
}
|
|
11990
|
-
if (forceScopedReembed) {
|
|
11991
|
-
failedForcedChunkIds.add(chunk.id);
|
|
11992
|
-
}
|
|
11993
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
11994
|
-
const existingFailedBatchIndex = failedBatchesForCurrentRun.findIndex(
|
|
11995
|
-
(failedBatch2) => failedBatch2.chunks[0]?.id === chunk.id
|
|
11996
|
-
);
|
|
11997
|
-
const existingFailedBatch = existingFailedBatchIndex === -1 ? void 0 : failedBatchesForCurrentRun[existingFailedBatchIndex];
|
|
11998
|
-
const failedBatch = {
|
|
11999
|
-
chunks: [chunk],
|
|
12000
|
-
error: failureMessage,
|
|
12001
|
-
attemptCount: (existingFailedBatch?.attemptCount ?? retryableFailedAttemptCounts.get(chunk.id) ?? 0) + 1,
|
|
12002
|
-
lastAttempt: failureTimestamp
|
|
12003
|
-
};
|
|
12004
|
-
if (existingFailedBatchIndex === -1) {
|
|
12005
|
-
failedBatchesForCurrentRun.push(failedBatch);
|
|
12006
|
-
} else {
|
|
12007
|
-
failedBatchesForCurrentRun[existingFailedBatchIndex] = failedBatch;
|
|
12008
|
-
}
|
|
12009
|
-
}
|
|
12010
|
-
this.logger.recordEmbeddingError();
|
|
12011
|
-
this.logger.embedding("error", `Failed to embed batch after retries`, {
|
|
12012
|
-
batchSize: failedChunks.length,
|
|
12013
|
-
requestCount: requestBatch.length,
|
|
12014
|
-
error: failureMessage
|
|
12015
|
-
});
|
|
12016
|
-
}
|
|
12017
|
-
});
|
|
12018
|
-
}
|
|
12019
|
-
await queue.onIdle();
|
|
12020
|
-
if (scopedRoots) {
|
|
12021
|
-
this.saveScopedFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun), scopedRoots);
|
|
12022
|
-
} else {
|
|
12023
|
-
this.saveFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun));
|
|
12024
|
-
}
|
|
12025
|
-
onProgress?.({
|
|
12026
|
-
phase: "storing",
|
|
12027
|
-
filesProcessed: files.length,
|
|
12028
|
-
totalFiles: files.length,
|
|
12029
|
-
chunksProcessed: stats.indexedChunks,
|
|
12030
|
-
totalChunks: pendingChunks.length
|
|
12031
|
-
});
|
|
12032
|
-
const branchChunkIds = Array.from(currentChunkIds).filter(
|
|
12033
|
-
(chunkId) => {
|
|
12034
|
-
const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId);
|
|
12035
|
-
const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId);
|
|
12036
|
-
return !isNewlyFailed && !isForcedFailed;
|
|
12037
|
-
}
|
|
12038
|
-
);
|
|
12039
|
-
this.replaceBranchCatalog(
|
|
12040
|
-
store,
|
|
12041
|
-
invertedIndex,
|
|
12042
|
-
database,
|
|
12043
|
-
branchCatalogKey,
|
|
12044
|
-
previousBranchChunkIds,
|
|
12045
|
-
branchChunkIds,
|
|
12046
|
-
previousBranchSymbolIds,
|
|
12047
|
-
Array.from(allSymbolIds)
|
|
12048
|
-
);
|
|
12049
|
-
store.save();
|
|
12050
|
-
this.saveInvertedIndex(invertedIndex);
|
|
12051
|
-
if (scopedRoots) {
|
|
12052
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12053
|
-
} else {
|
|
12054
|
-
this.fileHashCache = currentFileHashes;
|
|
12055
|
-
this.saveFileHashCache();
|
|
12056
|
-
}
|
|
12057
|
-
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
12058
|
-
const gcReset = await this.maybeRunOrphanGc();
|
|
12059
|
-
if (gcReset) {
|
|
12060
|
-
stats.durationMs = Date.now() - startTime;
|
|
12061
|
-
stats.warning = gcReset.warning;
|
|
12062
|
-
stats.resetCorruptedIndex = true;
|
|
12063
|
-
this.logger.recordIndexingEnd();
|
|
12064
|
-
this.logger.warn("Indexing ended after resetting corrupted local index during automatic GC", {
|
|
12065
|
-
files: stats.totalFiles,
|
|
12066
|
-
indexed: stats.indexedChunks,
|
|
12067
|
-
existing: stats.existingChunks,
|
|
12068
|
-
removed: stats.removedChunks,
|
|
12069
|
-
failed: stats.failedChunks,
|
|
12070
|
-
tokens: stats.tokensUsed,
|
|
12071
|
-
durationMs: stats.durationMs
|
|
12072
|
-
});
|
|
12073
|
-
return stats;
|
|
12821
|
+
this.logger.recordIndexingEnd();
|
|
12822
|
+
this.logger.info("Indexing complete", {
|
|
12823
|
+
files: stats.totalFiles,
|
|
12824
|
+
indexed: stats.indexedChunks,
|
|
12825
|
+
existing: stats.existingChunks,
|
|
12826
|
+
removed: stats.removedChunks,
|
|
12827
|
+
failed: stats.failedChunks,
|
|
12828
|
+
tokens: stats.tokensUsed,
|
|
12829
|
+
durationMs: stats.durationMs
|
|
12830
|
+
});
|
|
12831
|
+
if (stats.failedChunks > 0) {
|
|
12832
|
+
stats.failedBatchesPath = this.failedBatchesPath;
|
|
12074
12833
|
}
|
|
12834
|
+
onProgress?.({
|
|
12835
|
+
phase: "complete",
|
|
12836
|
+
filesProcessed: files.length,
|
|
12837
|
+
totalFiles: files.length,
|
|
12838
|
+
chunksProcessed: stats.indexedChunks,
|
|
12839
|
+
totalChunks: stats.totalChunks
|
|
12840
|
+
});
|
|
12841
|
+
return stats;
|
|
12842
|
+
} catch (error) {
|
|
12843
|
+
failedProcessing.state.writer.cleanup();
|
|
12844
|
+
if (writeTransactionActive) {
|
|
12845
|
+
try {
|
|
12846
|
+
database.rollbackWriteTransaction();
|
|
12847
|
+
} catch (rollbackError) {
|
|
12848
|
+
this.logger.error("Failed to roll back indexing database transaction", {
|
|
12849
|
+
error: getErrorMessage4(rollbackError)
|
|
12850
|
+
});
|
|
12851
|
+
}
|
|
12852
|
+
}
|
|
12853
|
+
throw error;
|
|
12075
12854
|
}
|
|
12076
|
-
stats.durationMs = Date.now() - startTime;
|
|
12077
|
-
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
12078
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12079
|
-
}
|
|
12080
|
-
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12081
|
-
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12082
|
-
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
12083
|
-
this.saveBranchCommit(database, indexedCommit);
|
|
12084
|
-
this.saveIndexMetadata(configuredProviderInfo);
|
|
12085
|
-
this.indexCompatibility = { compatible: true };
|
|
12086
|
-
this.logger.recordIndexingEnd();
|
|
12087
|
-
this.logger.info("Indexing complete", {
|
|
12088
|
-
files: stats.totalFiles,
|
|
12089
|
-
indexed: stats.indexedChunks,
|
|
12090
|
-
existing: stats.existingChunks,
|
|
12091
|
-
removed: stats.removedChunks,
|
|
12092
|
-
failed: stats.failedChunks,
|
|
12093
|
-
tokens: stats.tokensUsed,
|
|
12094
|
-
durationMs: stats.durationMs
|
|
12095
|
-
});
|
|
12096
|
-
if (stats.failedChunks > 0) {
|
|
12097
|
-
stats.failedBatchesPath = this.failedBatchesPath;
|
|
12098
|
-
}
|
|
12099
|
-
onProgress?.({
|
|
12100
|
-
phase: "complete",
|
|
12101
|
-
filesProcessed: files.length,
|
|
12102
|
-
totalFiles: files.length,
|
|
12103
|
-
chunksProcessed: stats.indexedChunks,
|
|
12104
|
-
totalChunks: pendingChunks.length
|
|
12105
|
-
});
|
|
12106
|
-
return stats;
|
|
12107
12855
|
}
|
|
12108
12856
|
async getQueryEmbedding(query, provider) {
|
|
12109
12857
|
const now2 = Date.now();
|
|
@@ -12341,10 +13089,31 @@ var Indexer = class _Indexer {
|
|
|
12341
13089
|
const baseFiltered = tiered.filter(
|
|
12342
13090
|
(r) => matchesSearchFilters(r, options, this.config.search.minScore, this.projectRoot)
|
|
12343
13091
|
);
|
|
12344
|
-
|
|
13092
|
+
let communityRanked = baseFiltered;
|
|
13093
|
+
if (this.config.search.communityBoost > 0) {
|
|
13094
|
+
try {
|
|
13095
|
+
const sameCommunityCandidateIds = resolveSameCommunityCandidateIds(
|
|
13096
|
+
query,
|
|
13097
|
+
baseFiltered,
|
|
13098
|
+
database,
|
|
13099
|
+
this.getBranchCatalogKeys()
|
|
13100
|
+
);
|
|
13101
|
+
communityRanked = applyCommunityBoost(
|
|
13102
|
+
baseFiltered,
|
|
13103
|
+
sameCommunityCandidateIds,
|
|
13104
|
+
this.config.search.communityBoost
|
|
13105
|
+
);
|
|
13106
|
+
} catch (error) {
|
|
13107
|
+
this.logger.search("debug", "Community-aware ranking unavailable; using existing ranking", {
|
|
13108
|
+
query,
|
|
13109
|
+
error: getErrorMessage4(error)
|
|
13110
|
+
});
|
|
13111
|
+
}
|
|
13112
|
+
}
|
|
13113
|
+
const implementationOnly = communityRanked.filter(
|
|
12345
13114
|
(r) => isLikelyImplementationPath2(r.metadata.filePath) && isImplementationChunkType(r.metadata.chunkType)
|
|
12346
13115
|
);
|
|
12347
|
-
const filtered = (sourceIntent && hasCodeHints && implementationOnly.length > 0 ? implementationOnly :
|
|
13116
|
+
const filtered = (sourceIntent && hasCodeHints && implementationOnly.length > 0 ? implementationOnly : communityRanked).slice(0, maxResults);
|
|
12348
13117
|
const identifierFallback = !options?.definitionIntent && filtered.length === 0 && identifierHints.length > 0 ? buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, maxResults, union, true).filter((r) => matchesSearchFilters(r, options, this.config.search.minScore, this.projectRoot)).slice(0, maxResults) : [];
|
|
12349
13118
|
const finalResults = filtered.length > 0 ? filtered : identifierFallback;
|
|
12350
13119
|
const totalSearchMs = import_perf_hooks.performance.now() - searchStartTime;
|
|
@@ -12561,7 +13330,7 @@ var Indexer = class _Indexer {
|
|
|
12561
13330
|
this.saveFileHashCache();
|
|
12562
13331
|
database.clearAllIndexedData();
|
|
12563
13332
|
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
12564
|
-
this.
|
|
13333
|
+
this.clearFailedBatchState();
|
|
12565
13334
|
database.deleteMetadata("index.version");
|
|
12566
13335
|
database.deleteMetadata("index.pathStorageVersion");
|
|
12567
13336
|
database.deleteMetadata("index.embeddingProvider");
|
|
@@ -12692,7 +13461,7 @@ var Indexer = class _Indexer {
|
|
|
12692
13461
|
gcOrphanSymbols: 0,
|
|
12693
13462
|
gcOrphanCallEdges: 0,
|
|
12694
13463
|
resetCorruptedIndex: true,
|
|
12695
|
-
warning: this.getCorruptedIndexWarning(
|
|
13464
|
+
warning: this.getCorruptedIndexWarning(path19.join(this.indexPath, "codebase.db"))
|
|
12696
13465
|
};
|
|
12697
13466
|
}
|
|
12698
13467
|
this.logger.recordGc(removedCount, gcOrphanChunks, gcOrphanEmbeddings);
|
|
@@ -12722,180 +13491,94 @@ var Indexer = class _Indexer {
|
|
|
12722
13491
|
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
12723
13492
|
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
12724
13493
|
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
12725
|
-
const
|
|
12726
|
-
|
|
12727
|
-
|
|
13494
|
+
const failedProcessing = this.prepareFailedBatchProcessing(roots, () => true);
|
|
13495
|
+
if (failedProcessing.latestById.size === 0) {
|
|
13496
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12728
13497
|
return { succeeded: 0, failed: 0, remaining: 0 };
|
|
12729
13498
|
}
|
|
13499
|
+
const queue = new PQueue({ concurrency: 1 });
|
|
13500
|
+
const rateLimitState = { backoffMs: 0 };
|
|
12730
13501
|
let succeeded = 0;
|
|
12731
13502
|
let failed = 0;
|
|
12732
|
-
|
|
12733
|
-
|
|
12734
|
-
|
|
12735
|
-
|
|
12736
|
-
|
|
12737
|
-
|
|
12738
|
-
|
|
12739
|
-
const
|
|
12740
|
-
|
|
12741
|
-
|
|
12742
|
-
|
|
12743
|
-
|
|
12744
|
-
);
|
|
12745
|
-
|
|
12746
|
-
|
|
12747
|
-
|
|
12748
|
-
|
|
12749
|
-
|
|
12750
|
-
|
|
12751
|
-
|
|
12752
|
-
|
|
12753
|
-
|
|
12754
|
-
|
|
12755
|
-
|
|
12756
|
-
|
|
12757
|
-
|
|
12758
|
-
|
|
12759
|
-
|
|
12760
|
-
|
|
12761
|
-
|
|
12762
|
-
|
|
12763
|
-
|
|
12764
|
-
}
|
|
12765
|
-
const vector = result.embeddings[idx];
|
|
12766
|
-
if (!vector) {
|
|
12767
|
-
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
12768
|
-
}
|
|
12769
|
-
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
12770
|
-
parts[request.partIndex] = {
|
|
12771
|
-
vector,
|
|
12772
|
-
tokenCount: request.tokenCount
|
|
12773
|
-
};
|
|
12774
|
-
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
12775
|
-
touchedChunkIds.add(request.chunk.id);
|
|
12776
|
-
});
|
|
12777
|
-
for (const chunkId of touchedChunkIds) {
|
|
12778
|
-
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
12779
|
-
continue;
|
|
12780
|
-
}
|
|
12781
|
-
const chunk = batchChunksById.get(chunkId);
|
|
12782
|
-
if (!chunk) {
|
|
12783
|
-
continue;
|
|
12784
|
-
}
|
|
12785
|
-
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
12786
|
-
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
12787
|
-
continue;
|
|
12788
|
-
}
|
|
12789
|
-
const orderedParts = parts;
|
|
12790
|
-
pooledResults.push({
|
|
12791
|
-
chunk,
|
|
12792
|
-
vector: poolEmbeddingVectors(
|
|
12793
|
-
orderedParts.map((part) => part.vector),
|
|
12794
|
-
orderedParts.map((part) => part.tokenCount)
|
|
12795
|
-
)
|
|
12796
|
-
});
|
|
12797
|
-
}
|
|
12798
|
-
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
12799
|
-
} catch (error) {
|
|
12800
|
-
const failureMessage = String(error);
|
|
12801
|
-
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
12802
|
-
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id) && !failedChunkIds.has(chunk.id));
|
|
12803
|
-
for (const chunk of failedChunks) {
|
|
12804
|
-
failedChunkIds.add(chunk.id);
|
|
12805
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
12806
|
-
failedChunksForBatch.set(chunk.id, {
|
|
12807
|
-
chunks: [chunk],
|
|
12808
|
-
attemptCount: batch.attemptCount + 1,
|
|
12809
|
-
lastAttempt: failureTimestamp,
|
|
12810
|
-
error: failureMessage
|
|
12811
|
-
});
|
|
12812
|
-
}
|
|
12813
|
-
failed += failedChunks.length;
|
|
12814
|
-
this.logger.recordEmbeddingError();
|
|
12815
|
-
}
|
|
12816
|
-
}
|
|
12817
|
-
const successfulResults = pooledResults.filter(({ chunk }) => !failedChunkIds.has(chunk.id));
|
|
12818
|
-
const items = successfulResults.map(({ chunk, vector }) => ({
|
|
12819
|
-
id: chunk.id,
|
|
12820
|
-
vector,
|
|
12821
|
-
metadata: chunk.metadata
|
|
12822
|
-
}));
|
|
12823
|
-
if (items.length > 0) {
|
|
12824
|
-
store.addBatch(items);
|
|
12825
|
-
}
|
|
12826
|
-
if (successfulResults.length > 0) {
|
|
12827
|
-
try {
|
|
12828
|
-
database.upsertEmbeddingsBatch(
|
|
12829
|
-
successfulResults.map(({ chunk, vector }) => ({
|
|
12830
|
-
contentHash: chunk.contentHash,
|
|
12831
|
-
embedding: float32ArrayToBuffer(vector),
|
|
12832
|
-
chunkText: chunk.storageText,
|
|
12833
|
-
model: configuredProviderInfo.modelInfo.model
|
|
12834
|
-
}))
|
|
12835
|
-
);
|
|
12836
|
-
} catch (dbError) {
|
|
12837
|
-
this.rebuildVectorStoreExcludingChunkIds(
|
|
12838
|
-
store,
|
|
12839
|
-
database,
|
|
12840
|
-
successfulResults.map(({ chunk }) => chunk.id)
|
|
13503
|
+
try {
|
|
13504
|
+
const retryableChunks = this.iterateLatestFailedChunks(
|
|
13505
|
+
failedProcessing.latestById,
|
|
13506
|
+
roots,
|
|
13507
|
+
() => true,
|
|
13508
|
+
maxChunkTokens
|
|
13509
|
+
);
|
|
13510
|
+
for (const retryBatch of iterateOrderedFileBatches(
|
|
13511
|
+
retryableChunks,
|
|
13512
|
+
({ chunk }) => Buffer.byteLength(chunk.content, "utf-8"),
|
|
13513
|
+
this.fileBatchLimits
|
|
13514
|
+
)) {
|
|
13515
|
+
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
13516
|
+
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
13517
|
+
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
13518
|
+
store,
|
|
13519
|
+
provider,
|
|
13520
|
+
invertedIndex,
|
|
13521
|
+
database,
|
|
13522
|
+
configuredProviderInfo,
|
|
13523
|
+
queue,
|
|
13524
|
+
providerRateLimits,
|
|
13525
|
+
rateLimitState,
|
|
13526
|
+
failedState: failedProcessing.state,
|
|
13527
|
+
attemptCounts,
|
|
13528
|
+
forceReembed: false,
|
|
13529
|
+
reuseCachedEmbeddings: false,
|
|
13530
|
+
incrementRepeatedFailures: false,
|
|
13531
|
+
onSucceeded: (succeededChunks) => {
|
|
13532
|
+
database.addChunksToBranchBatch(
|
|
13533
|
+
this.getBranchCatalogKey(),
|
|
13534
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
12841
13535
|
);
|
|
12842
|
-
throw dbError;
|
|
12843
13536
|
}
|
|
12844
|
-
}
|
|
12845
|
-
|
|
12846
|
-
|
|
12847
|
-
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
12848
|
-
completedChunkIds.add(chunk.id);
|
|
12849
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
12850
|
-
}
|
|
12851
|
-
database.addChunksToBranchBatch(
|
|
12852
|
-
this.getBranchCatalogKey(),
|
|
12853
|
-
successfulResults.map(({ chunk }) => chunk.id)
|
|
12854
|
-
);
|
|
12855
|
-
this.logger.recordChunksEmbedded(successfulResults.length);
|
|
12856
|
-
succeeded += successfulResults.length;
|
|
12857
|
-
stillFailing.push(...failedChunksForBatch.values());
|
|
12858
|
-
} catch (error) {
|
|
12859
|
-
const failureMessage = getErrorMessage4(error);
|
|
12860
|
-
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
12861
|
-
const unaccountedChunks = batch.chunks.filter(
|
|
12862
|
-
(chunk) => !failedChunksForBatch.has(chunk.id) && !completedChunkIds.has(chunk.id)
|
|
12863
|
-
);
|
|
12864
|
-
for (const chunk of unaccountedChunks) {
|
|
12865
|
-
failedChunksForBatch.set(chunk.id, {
|
|
12866
|
-
chunks: [chunk],
|
|
12867
|
-
attemptCount: batch.attemptCount + 1,
|
|
12868
|
-
lastAttempt: failureTimestamp,
|
|
12869
|
-
error: failureMessage
|
|
12870
|
-
});
|
|
12871
|
-
}
|
|
12872
|
-
failed += unaccountedChunks.length;
|
|
12873
|
-
this.logger.recordEmbeddingError();
|
|
12874
|
-
stillFailing.push(...coalesceFailedBatches(Array.from(failedChunksForBatch.values())));
|
|
13537
|
+
});
|
|
13538
|
+
succeeded += batchResult.indexedChunks;
|
|
13539
|
+
failed += batchResult.failedChunks;
|
|
12875
13540
|
}
|
|
13541
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
13542
|
+
} catch (error) {
|
|
13543
|
+
failedProcessing.state.writer.cleanup();
|
|
13544
|
+
throw error;
|
|
12876
13545
|
}
|
|
12877
|
-
const
|
|
12878
|
-
if (roots) {
|
|
12879
|
-
this.saveFailedBatches([...retainedFailedBatches, ...persistedStillFailing]);
|
|
12880
|
-
} else {
|
|
12881
|
-
this.saveFailedBatches(persistedStillFailing);
|
|
12882
|
-
}
|
|
13546
|
+
const remaining = this.getFailedBatchesCount();
|
|
12883
13547
|
if (succeeded > 0) {
|
|
12884
13548
|
store.save();
|
|
12885
13549
|
this.saveInvertedIndex(invertedIndex);
|
|
12886
13550
|
}
|
|
12887
|
-
if (roots && succeeded > 0 &&
|
|
13551
|
+
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
12888
13552
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12889
13553
|
this.saveIndexMetadata(configuredProviderInfo);
|
|
12890
13554
|
this.indexCompatibility = { compatible: true };
|
|
12891
13555
|
}
|
|
12892
|
-
return { succeeded, failed, remaining
|
|
13556
|
+
return { succeeded, failed, remaining };
|
|
12893
13557
|
}
|
|
12894
13558
|
getFailedBatchesCount() {
|
|
12895
|
-
|
|
12896
|
-
|
|
13559
|
+
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
13560
|
+
const latestById = /* @__PURE__ */ new Map();
|
|
13561
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
13562
|
+
for (const rawChunk of batch.chunks) {
|
|
13563
|
+
const filePath = getPendingChunkFilePath(rawChunk);
|
|
13564
|
+
if (roots && (filePath === null || !this.isFileInCurrentScope(filePath, roots))) {
|
|
13565
|
+
continue;
|
|
13566
|
+
}
|
|
13567
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
13568
|
+
if (!chunkId) {
|
|
13569
|
+
continue;
|
|
13570
|
+
}
|
|
13571
|
+
const existing = latestById.get(chunkId);
|
|
13572
|
+
if (!existing || batch.attemptCount >= existing.attemptCount) {
|
|
13573
|
+
latestById.set(chunkId, {
|
|
13574
|
+
attemptCount: batch.attemptCount,
|
|
13575
|
+
error: batch.error,
|
|
13576
|
+
lastAttempt: batch.lastAttempt
|
|
13577
|
+
});
|
|
13578
|
+
}
|
|
13579
|
+
}
|
|
12897
13580
|
}
|
|
12898
|
-
return
|
|
13581
|
+
return new Set(Array.from(latestById.values(), getFailedBatchGroupKey)).size;
|
|
12899
13582
|
}
|
|
12900
13583
|
getCurrentBranch() {
|
|
12901
13584
|
return this.currentBranch;
|
|
@@ -13083,9 +13766,9 @@ var Indexer = class _Indexer {
|
|
|
13083
13766
|
this.requireReadableComponents(readIssues, "database");
|
|
13084
13767
|
let shortest = [];
|
|
13085
13768
|
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
13086
|
-
const
|
|
13087
|
-
if (
|
|
13088
|
-
shortest =
|
|
13769
|
+
const path28 = database.findShortestPath(fromName, toName, branchKey, maxDepth);
|
|
13770
|
+
if (path28.length > 0 && (shortest.length === 0 || path28.length < shortest.length)) {
|
|
13771
|
+
shortest = path28;
|
|
13089
13772
|
}
|
|
13090
13773
|
}
|
|
13091
13774
|
return shortest.map((hop) => this.resolveFilePathRecord(hop));
|
|
@@ -13133,13 +13816,13 @@ var Indexer = class _Indexer {
|
|
|
13133
13816
|
}
|
|
13134
13817
|
}
|
|
13135
13818
|
if (!found) continue;
|
|
13136
|
-
const
|
|
13819
|
+
const path28 = [];
|
|
13137
13820
|
let currentSymbolId = toSymbolId;
|
|
13138
13821
|
while (true) {
|
|
13139
13822
|
const symbol = symbolsById.get(currentSymbolId);
|
|
13140
13823
|
if (!symbol) break;
|
|
13141
13824
|
const parent = parentBySymbolId.get(currentSymbolId);
|
|
13142
|
-
|
|
13825
|
+
path28.push({
|
|
13143
13826
|
symbolId: symbol.id,
|
|
13144
13827
|
symbolName: symbol.name,
|
|
13145
13828
|
filePath: symbol.filePath,
|
|
@@ -13149,9 +13832,9 @@ var Indexer = class _Indexer {
|
|
|
13149
13832
|
if (!parent) break;
|
|
13150
13833
|
currentSymbolId = parent.parentId;
|
|
13151
13834
|
}
|
|
13152
|
-
|
|
13153
|
-
if (
|
|
13154
|
-
shortest =
|
|
13835
|
+
path28.reverse();
|
|
13836
|
+
if (path28.length > 0 && (shortest.length === 0 || path28.length < shortest.length)) {
|
|
13837
|
+
shortest = path28;
|
|
13155
13838
|
}
|
|
13156
13839
|
}
|
|
13157
13840
|
return shortest.map((hop) => this.resolveFilePathRecord(hop));
|
|
@@ -13170,13 +13853,13 @@ var Indexer = class _Indexer {
|
|
|
13170
13853
|
async getSymbolsForBranch(branch) {
|
|
13171
13854
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13172
13855
|
this.requireReadableComponents(readIssues, "database");
|
|
13173
|
-
const resolvedBranch =
|
|
13856
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13174
13857
|
return database.getSymbolsForBranch(resolvedBranch).map((symbol) => this.resolveFilePathRecord(symbol));
|
|
13175
13858
|
}
|
|
13176
13859
|
async getSymbolsForFiles(filePaths, branch) {
|
|
13177
13860
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13178
13861
|
this.requireReadableComponents(readIssues, "database");
|
|
13179
|
-
const resolvedBranch =
|
|
13862
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13180
13863
|
const storedFilePaths = filePaths.map((filePath) => this.toStoredFilePath(filePath));
|
|
13181
13864
|
return database.getSymbolsForFiles(storedFilePaths, resolvedBranch).map((symbol) => this.resolveFilePathRecord(symbol));
|
|
13182
13865
|
}
|
|
@@ -13189,13 +13872,26 @@ var Indexer = class _Indexer {
|
|
|
13189
13872
|
async detectCommunities(branch, symbolIds) {
|
|
13190
13873
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13191
13874
|
this.requireReadableComponents(readIssues, "database");
|
|
13192
|
-
const resolvedBranch =
|
|
13875
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13193
13876
|
return database.detectCommunities(resolvedBranch, symbolIds).map((entry) => this.resolveFilePathRecord(entry));
|
|
13194
13877
|
}
|
|
13878
|
+
async detectCommunityCouplings(branch) {
|
|
13879
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
13880
|
+
this.requireReadableComponents(readIssues, "database");
|
|
13881
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13882
|
+
return database.detectCommunityCouplings(resolvedBranch).map((entry) => ({
|
|
13883
|
+
...entry,
|
|
13884
|
+
relationships: (entry.relationships ?? entry.representativeRelationships ?? []).map((relationship) => ({
|
|
13885
|
+
...relationship,
|
|
13886
|
+
fromFilePath: this.resolveStoredFilePath(relationship.fromFilePath),
|
|
13887
|
+
toFilePath: this.resolveStoredFilePath(relationship.toFilePath)
|
|
13888
|
+
}))
|
|
13889
|
+
}));
|
|
13890
|
+
}
|
|
13195
13891
|
async computeCentrality(branch) {
|
|
13196
13892
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13197
13893
|
this.requireReadableComponents(readIssues, "database");
|
|
13198
|
-
const resolvedBranch =
|
|
13894
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13199
13895
|
return database.computeCentrality(resolvedBranch).map((entry) => this.resolveFilePathRecord(entry));
|
|
13200
13896
|
}
|
|
13201
13897
|
async getPrImpact(opts, onPreparationProgress) {
|
|
@@ -13289,7 +13985,7 @@ var Indexer = class _Indexer {
|
|
|
13289
13985
|
);
|
|
13290
13986
|
}
|
|
13291
13987
|
}
|
|
13292
|
-
const toStoredChangedFiles = (filePaths) => filePaths.map((filePath) => this.toStoredFilePath(
|
|
13988
|
+
const toStoredChangedFiles = (filePaths) => filePaths.map((filePath) => this.toStoredFilePath(path19.resolve(this.projectRoot, filePath)));
|
|
13293
13989
|
const storedChangedFiles = toStoredChangedFiles(changedFiles);
|
|
13294
13990
|
const directSymbols = database.getSymbolsForFiles(storedChangedFiles, branchKey);
|
|
13295
13991
|
const directIds = directSymbols.map((s) => s.id);
|
|
@@ -13438,12 +14134,12 @@ var Indexer = class _Indexer {
|
|
|
13438
14134
|
if (meta.filePath) filePaths.add(meta.filePath);
|
|
13439
14135
|
}
|
|
13440
14136
|
const directory = options?.directory?.replace(/\/$/, "");
|
|
13441
|
-
const absoluteDirectoryFilter = directory ?
|
|
14137
|
+
const absoluteDirectoryFilter = directory ? path19.resolve(this.projectRoot, directory) : void 0;
|
|
13442
14138
|
for (const filePath of filePaths) {
|
|
13443
14139
|
if (directory) {
|
|
13444
14140
|
const absoluteFilePath = this.resolveStoredFilePath(filePath);
|
|
13445
14141
|
const matchesRelative = filePath === directory || filePath.startsWith(directory + "/");
|
|
13446
|
-
const matchesProjectRelative = absoluteDirectoryFilter !== void 0 && (absoluteFilePath === absoluteDirectoryFilter || absoluteFilePath.startsWith(absoluteDirectoryFilter +
|
|
14142
|
+
const matchesProjectRelative = absoluteDirectoryFilter !== void 0 && (absoluteFilePath === absoluteDirectoryFilter || absoluteFilePath.startsWith(absoluteDirectoryFilter + path19.sep));
|
|
13447
14143
|
if (!matchesRelative && !matchesProjectRelative) {
|
|
13448
14144
|
continue;
|
|
13449
14145
|
}
|
|
@@ -13610,7 +14306,7 @@ function trimOrUndefined(value) {
|
|
|
13610
14306
|
return normalized || void 0;
|
|
13611
14307
|
}
|
|
13612
14308
|
function normalizeCallGraphPath(value) {
|
|
13613
|
-
let normalized =
|
|
14309
|
+
let normalized = path20.posix.normalize(value.trim().replaceAll("\\", "/"));
|
|
13614
14310
|
if (normalized.startsWith("./")) {
|
|
13615
14311
|
normalized = normalized.slice(2);
|
|
13616
14312
|
}
|
|
@@ -13794,12 +14490,12 @@ async function getCallGraphPath(projectRoot, host, from, to, maxDepth, fromFileP
|
|
|
13794
14490
|
if (fromResolution.status !== "resolved" || toResolution.status !== "resolved") {
|
|
13795
14491
|
return { from: fromResolution, to: toResolution, path: [] };
|
|
13796
14492
|
}
|
|
13797
|
-
const
|
|
14493
|
+
const path28 = await indexer.findCallPathBySymbolIds(
|
|
13798
14494
|
fromResolution.symbolId,
|
|
13799
14495
|
toResolution.symbolId,
|
|
13800
14496
|
maxDepth
|
|
13801
14497
|
);
|
|
13802
|
-
return { from: fromResolution, to: toResolution, path:
|
|
14498
|
+
return { from: fromResolution, to: toResolution, path: path28 };
|
|
13803
14499
|
}
|
|
13804
14500
|
async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
13805
14501
|
const root = getProjectRoot(projectRoot, host);
|
|
@@ -13863,6 +14559,34 @@ async function runIndexHealthCheck(projectRoot, host) {
|
|
|
13863
14559
|
return busyResult;
|
|
13864
14560
|
}
|
|
13865
14561
|
}
|
|
14562
|
+
async function getCodeCommunities(projectRoot, host, params) {
|
|
14563
|
+
await ensureAutoIndexReadyForRetrieval(projectRoot, host);
|
|
14564
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
14565
|
+
const [communities, centrality, couplings] = await Promise.all([
|
|
14566
|
+
indexer.detectCommunities(params.branch),
|
|
14567
|
+
indexer.computeCentrality(params.branch),
|
|
14568
|
+
indexer.detectCommunityCouplings(params.branch)
|
|
14569
|
+
]);
|
|
14570
|
+
return buildCodeCommunitiesResult(communities, centrality, couplings, {
|
|
14571
|
+
minSize: Math.max(CODE_COMMUNITIES_MIN_SIZE, Math.floor(params.minSize ?? CODE_COMMUNITIES_MIN_SIZE)),
|
|
14572
|
+
limit: Math.min(
|
|
14573
|
+
CODE_COMMUNITIES_MAX_LIMIT,
|
|
14574
|
+
Math.max(1, Math.floor(params.limit ?? CODE_COMMUNITIES_DEFAULT_LIMIT))
|
|
14575
|
+
),
|
|
14576
|
+
hubThreshold: Math.max(
|
|
14577
|
+
0,
|
|
14578
|
+
Math.floor(params.hubThreshold ?? CODE_COMMUNITIES_DEFAULT_HUB_THRESHOLD)
|
|
14579
|
+
),
|
|
14580
|
+
minCoupling: Math.max(
|
|
14581
|
+
CODE_COMMUNITIES_MIN_COUPLING,
|
|
14582
|
+
Math.floor(params.minCoupling ?? CODE_COMMUNITIES_MIN_COUPLING)
|
|
14583
|
+
),
|
|
14584
|
+
couplingLimit: Math.min(
|
|
14585
|
+
CODE_COMMUNITIES_MAX_COUPLING_LIMIT,
|
|
14586
|
+
Math.max(1, Math.floor(params.couplingLimit ?? CODE_COMMUNITIES_DEFAULT_COUPLING_LIMIT))
|
|
14587
|
+
)
|
|
14588
|
+
});
|
|
14589
|
+
}
|
|
13866
14590
|
async function getIndexMetrics(projectRoot, host, args = {}) {
|
|
13867
14591
|
const root = getProjectRoot(projectRoot, host);
|
|
13868
14592
|
const key = getIndexerCacheKey(root, host);
|
|
@@ -13954,8 +14678,8 @@ async function getIndexLogs(projectRoot, host, args) {
|
|
|
13954
14678
|
function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
13955
14679
|
const root = getProjectRoot(projectRoot, host);
|
|
13956
14680
|
const inputPath = knowledgeBasePath.trim();
|
|
13957
|
-
const normalizedPath =
|
|
13958
|
-
|
|
14681
|
+
const normalizedPath = path20.resolve(
|
|
14682
|
+
path20.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
|
|
13959
14683
|
);
|
|
13960
14684
|
if (!(0, import_fs13.existsSync)(normalizedPath)) {
|
|
13961
14685
|
return `Error: Directory does not exist: ${normalizedPath}`;
|
|
@@ -13991,7 +14715,7 @@ function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
|
13991
14715
|
}
|
|
13992
14716
|
}
|
|
13993
14717
|
for (const dotDir of sensitiveDotDirs) {
|
|
13994
|
-
const sensitiveDir =
|
|
14718
|
+
const sensitiveDir = path20.join(homeDir, dotDir);
|
|
13995
14719
|
if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
|
|
13996
14720
|
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath}`;
|
|
13997
14721
|
}
|
|
@@ -14054,7 +14778,7 @@ function listKnowledgeBases(projectRoot, host) {
|
|
|
14054
14778
|
}
|
|
14055
14779
|
result += "\n";
|
|
14056
14780
|
}
|
|
14057
|
-
const hasHostConfig = (0, import_fs13.existsSync)(
|
|
14781
|
+
const hasHostConfig = (0, import_fs13.existsSync)(path20.join(root, getHostProjectConfigRelativePath(host)));
|
|
14058
14782
|
if (hasHostConfig) {
|
|
14059
14783
|
result += `
|
|
14060
14784
|
Config sources: 1 file(s).`;
|
|
@@ -14180,7 +14904,7 @@ var ReaddirpStream = class extends import_node_stream.Readable {
|
|
|
14180
14904
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
14181
14905
|
const statMethod = opts.lstat ? import_promises.lstat : import_promises.stat;
|
|
14182
14906
|
if (wantBigintFsStats) {
|
|
14183
|
-
this._stat = (
|
|
14907
|
+
this._stat = (path28) => statMethod(path28, { bigint: true });
|
|
14184
14908
|
} else {
|
|
14185
14909
|
this._stat = statMethod;
|
|
14186
14910
|
}
|
|
@@ -14205,8 +14929,8 @@ var ReaddirpStream = class extends import_node_stream.Readable {
|
|
|
14205
14929
|
const par = this.parent;
|
|
14206
14930
|
const fil = par && par.files;
|
|
14207
14931
|
if (fil && fil.length > 0) {
|
|
14208
|
-
const { path:
|
|
14209
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
14932
|
+
const { path: path28, depth } = par;
|
|
14933
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path28));
|
|
14210
14934
|
const awaited = await Promise.all(slice);
|
|
14211
14935
|
for (const entry of awaited) {
|
|
14212
14936
|
if (!entry)
|
|
@@ -14246,21 +14970,21 @@ var ReaddirpStream = class extends import_node_stream.Readable {
|
|
|
14246
14970
|
this.reading = false;
|
|
14247
14971
|
}
|
|
14248
14972
|
}
|
|
14249
|
-
async _exploreDir(
|
|
14973
|
+
async _exploreDir(path28, depth) {
|
|
14250
14974
|
let files;
|
|
14251
14975
|
try {
|
|
14252
|
-
files = await (0, import_promises.readdir)(
|
|
14976
|
+
files = await (0, import_promises.readdir)(path28, this._rdOptions);
|
|
14253
14977
|
} catch (error) {
|
|
14254
14978
|
this._onError(error);
|
|
14255
14979
|
}
|
|
14256
|
-
return { files, depth, path:
|
|
14980
|
+
return { files, depth, path: path28 };
|
|
14257
14981
|
}
|
|
14258
|
-
async _formatEntry(dirent,
|
|
14982
|
+
async _formatEntry(dirent, path28) {
|
|
14259
14983
|
let entry;
|
|
14260
|
-
const
|
|
14984
|
+
const basename9 = this._isDirent ? dirent.name : dirent;
|
|
14261
14985
|
try {
|
|
14262
|
-
const fullPath = (0, import_node_path.resolve)((0, import_node_path.join)(
|
|
14263
|
-
entry = { path: (0, import_node_path.relative)(this._root, fullPath), fullPath, basename:
|
|
14986
|
+
const fullPath = (0, import_node_path.resolve)((0, import_node_path.join)(path28, basename9));
|
|
14987
|
+
entry = { path: (0, import_node_path.relative)(this._root, fullPath), fullPath, basename: basename9 };
|
|
14264
14988
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
14265
14989
|
} catch (err) {
|
|
14266
14990
|
this._onError(err);
|
|
@@ -14659,16 +15383,16 @@ var delFromSet = (main, prop, item) => {
|
|
|
14659
15383
|
};
|
|
14660
15384
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
14661
15385
|
var FsWatchInstances = /* @__PURE__ */ new Map();
|
|
14662
|
-
function createFsWatchInstance(
|
|
15386
|
+
function createFsWatchInstance(path28, options, listener, errHandler, emitRaw) {
|
|
14663
15387
|
const handleEvent = (rawEvent, evPath) => {
|
|
14664
|
-
listener(
|
|
14665
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
14666
|
-
if (evPath &&
|
|
14667
|
-
fsWatchBroadcast(sp.resolve(
|
|
15388
|
+
listener(path28);
|
|
15389
|
+
emitRaw(rawEvent, evPath, { watchedPath: path28 });
|
|
15390
|
+
if (evPath && path28 !== evPath) {
|
|
15391
|
+
fsWatchBroadcast(sp.resolve(path28, evPath), KEY_LISTENERS, sp.join(path28, evPath));
|
|
14668
15392
|
}
|
|
14669
15393
|
};
|
|
14670
15394
|
try {
|
|
14671
|
-
return (0, import_node_fs.watch)(
|
|
15395
|
+
return (0, import_node_fs.watch)(path28, {
|
|
14672
15396
|
persistent: options.persistent
|
|
14673
15397
|
}, handleEvent);
|
|
14674
15398
|
} catch (error) {
|
|
@@ -14684,12 +15408,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
14684
15408
|
listener(val1, val2, val3);
|
|
14685
15409
|
});
|
|
14686
15410
|
};
|
|
14687
|
-
var setFsWatchListener = (
|
|
15411
|
+
var setFsWatchListener = (path28, fullPath, options, handlers) => {
|
|
14688
15412
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
14689
15413
|
let cont = FsWatchInstances.get(fullPath);
|
|
14690
15414
|
let watcher;
|
|
14691
15415
|
if (!options.persistent) {
|
|
14692
|
-
watcher = createFsWatchInstance(
|
|
15416
|
+
watcher = createFsWatchInstance(path28, options, listener, errHandler, rawEmitter);
|
|
14693
15417
|
if (!watcher)
|
|
14694
15418
|
return;
|
|
14695
15419
|
return watcher.close.bind(watcher);
|
|
@@ -14700,7 +15424,7 @@ var setFsWatchListener = (path27, fullPath, options, handlers) => {
|
|
|
14700
15424
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
14701
15425
|
} else {
|
|
14702
15426
|
watcher = createFsWatchInstance(
|
|
14703
|
-
|
|
15427
|
+
path28,
|
|
14704
15428
|
options,
|
|
14705
15429
|
fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS),
|
|
14706
15430
|
errHandler,
|
|
@@ -14715,7 +15439,7 @@ var setFsWatchListener = (path27, fullPath, options, handlers) => {
|
|
|
14715
15439
|
cont.watcherUnusable = true;
|
|
14716
15440
|
if (isWindows && error.code === "EPERM") {
|
|
14717
15441
|
try {
|
|
14718
|
-
const fd = await (0, import_promises2.open)(
|
|
15442
|
+
const fd = await (0, import_promises2.open)(path28, "r");
|
|
14719
15443
|
await fd.close();
|
|
14720
15444
|
broadcastErr(error);
|
|
14721
15445
|
} catch (err) {
|
|
@@ -14746,7 +15470,7 @@ var setFsWatchListener = (path27, fullPath, options, handlers) => {
|
|
|
14746
15470
|
};
|
|
14747
15471
|
};
|
|
14748
15472
|
var FsWatchFileInstances = /* @__PURE__ */ new Map();
|
|
14749
|
-
var setFsWatchFileListener = (
|
|
15473
|
+
var setFsWatchFileListener = (path28, fullPath, options, handlers) => {
|
|
14750
15474
|
const { listener, rawEmitter } = handlers;
|
|
14751
15475
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
14752
15476
|
const copts = cont && cont.options;
|
|
@@ -14768,7 +15492,7 @@ var setFsWatchFileListener = (path27, fullPath, options, handlers) => {
|
|
|
14768
15492
|
});
|
|
14769
15493
|
const currmtime = curr.mtimeMs;
|
|
14770
15494
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
14771
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
15495
|
+
foreach(cont.listeners, (listener2) => listener2(path28, curr));
|
|
14772
15496
|
}
|
|
14773
15497
|
})
|
|
14774
15498
|
};
|
|
@@ -14798,13 +15522,13 @@ var NodeFsHandler = class {
|
|
|
14798
15522
|
* @param listener on fs change
|
|
14799
15523
|
* @returns closer for the watcher instance
|
|
14800
15524
|
*/
|
|
14801
|
-
_watchWithNodeFs(
|
|
15525
|
+
_watchWithNodeFs(path28, listener) {
|
|
14802
15526
|
const opts = this.fsw.options;
|
|
14803
|
-
const directory = sp.dirname(
|
|
14804
|
-
const
|
|
15527
|
+
const directory = sp.dirname(path28);
|
|
15528
|
+
const basename9 = sp.basename(path28);
|
|
14805
15529
|
const parent = this.fsw._getWatchedDir(directory);
|
|
14806
|
-
parent.add(
|
|
14807
|
-
const absolutePath = sp.resolve(
|
|
15530
|
+
parent.add(basename9);
|
|
15531
|
+
const absolutePath = sp.resolve(path28);
|
|
14808
15532
|
const options = {
|
|
14809
15533
|
persistent: opts.persistent
|
|
14810
15534
|
};
|
|
@@ -14813,13 +15537,13 @@ var NodeFsHandler = class {
|
|
|
14813
15537
|
let closer;
|
|
14814
15538
|
if (opts.usePolling) {
|
|
14815
15539
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
14816
|
-
options.interval = enableBin && isBinaryPath(
|
|
14817
|
-
closer = setFsWatchFileListener(
|
|
15540
|
+
options.interval = enableBin && isBinaryPath(basename9) ? opts.binaryInterval : opts.interval;
|
|
15541
|
+
closer = setFsWatchFileListener(path28, absolutePath, options, {
|
|
14818
15542
|
listener,
|
|
14819
15543
|
rawEmitter: this.fsw._emitRaw
|
|
14820
15544
|
});
|
|
14821
15545
|
} else {
|
|
14822
|
-
closer = setFsWatchListener(
|
|
15546
|
+
closer = setFsWatchListener(path28, absolutePath, options, {
|
|
14823
15547
|
listener,
|
|
14824
15548
|
errHandler: this._boundHandleError,
|
|
14825
15549
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -14835,13 +15559,13 @@ var NodeFsHandler = class {
|
|
|
14835
15559
|
if (this.fsw.closed) {
|
|
14836
15560
|
return;
|
|
14837
15561
|
}
|
|
14838
|
-
const
|
|
14839
|
-
const
|
|
14840
|
-
const parent = this.fsw._getWatchedDir(
|
|
15562
|
+
const dirname15 = sp.dirname(file);
|
|
15563
|
+
const basename9 = sp.basename(file);
|
|
15564
|
+
const parent = this.fsw._getWatchedDir(dirname15);
|
|
14841
15565
|
let prevStats = stats;
|
|
14842
|
-
if (parent.has(
|
|
15566
|
+
if (parent.has(basename9))
|
|
14843
15567
|
return;
|
|
14844
|
-
const listener = async (
|
|
15568
|
+
const listener = async (path28, newStats) => {
|
|
14845
15569
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
14846
15570
|
return;
|
|
14847
15571
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -14855,18 +15579,18 @@ var NodeFsHandler = class {
|
|
|
14855
15579
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
14856
15580
|
}
|
|
14857
15581
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
14858
|
-
this.fsw._closeFile(
|
|
15582
|
+
this.fsw._closeFile(path28);
|
|
14859
15583
|
prevStats = newStats2;
|
|
14860
15584
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
14861
15585
|
if (closer2)
|
|
14862
|
-
this.fsw._addPathCloser(
|
|
15586
|
+
this.fsw._addPathCloser(path28, closer2);
|
|
14863
15587
|
} else {
|
|
14864
15588
|
prevStats = newStats2;
|
|
14865
15589
|
}
|
|
14866
15590
|
} catch (error) {
|
|
14867
|
-
this.fsw._remove(
|
|
15591
|
+
this.fsw._remove(dirname15, basename9);
|
|
14868
15592
|
}
|
|
14869
|
-
} else if (parent.has(
|
|
15593
|
+
} else if (parent.has(basename9)) {
|
|
14870
15594
|
const at = newStats.atimeMs;
|
|
14871
15595
|
const mt = newStats.mtimeMs;
|
|
14872
15596
|
if (!at || at <= mt || mt !== prevStats.mtimeMs) {
|
|
@@ -14891,7 +15615,7 @@ var NodeFsHandler = class {
|
|
|
14891
15615
|
* @param item basename of this item
|
|
14892
15616
|
* @returns true if no more processing is needed for this entry.
|
|
14893
15617
|
*/
|
|
14894
|
-
async _handleSymlink(entry, directory,
|
|
15618
|
+
async _handleSymlink(entry, directory, path28, item) {
|
|
14895
15619
|
if (this.fsw.closed) {
|
|
14896
15620
|
return;
|
|
14897
15621
|
}
|
|
@@ -14901,7 +15625,7 @@ var NodeFsHandler = class {
|
|
|
14901
15625
|
this.fsw._incrReadyCount();
|
|
14902
15626
|
let linkPath;
|
|
14903
15627
|
try {
|
|
14904
|
-
linkPath = await (0, import_promises2.realpath)(
|
|
15628
|
+
linkPath = await (0, import_promises2.realpath)(path28);
|
|
14905
15629
|
} catch (e) {
|
|
14906
15630
|
this.fsw._emitReady();
|
|
14907
15631
|
return true;
|
|
@@ -14911,12 +15635,12 @@ var NodeFsHandler = class {
|
|
|
14911
15635
|
if (dir.has(item)) {
|
|
14912
15636
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
14913
15637
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
14914
|
-
this.fsw._emit(EV.CHANGE,
|
|
15638
|
+
this.fsw._emit(EV.CHANGE, path28, entry.stats);
|
|
14915
15639
|
}
|
|
14916
15640
|
} else {
|
|
14917
15641
|
dir.add(item);
|
|
14918
15642
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
14919
|
-
this.fsw._emit(EV.ADD,
|
|
15643
|
+
this.fsw._emit(EV.ADD, path28, entry.stats);
|
|
14920
15644
|
}
|
|
14921
15645
|
this.fsw._emitReady();
|
|
14922
15646
|
return true;
|
|
@@ -14946,9 +15670,9 @@ var NodeFsHandler = class {
|
|
|
14946
15670
|
return;
|
|
14947
15671
|
}
|
|
14948
15672
|
const item = entry.path;
|
|
14949
|
-
let
|
|
15673
|
+
let path28 = sp.join(directory, item);
|
|
14950
15674
|
current.add(item);
|
|
14951
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
15675
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path28, item)) {
|
|
14952
15676
|
return;
|
|
14953
15677
|
}
|
|
14954
15678
|
if (this.fsw.closed) {
|
|
@@ -14957,8 +15681,8 @@ var NodeFsHandler = class {
|
|
|
14957
15681
|
}
|
|
14958
15682
|
if (item === target || !target && !previous.has(item)) {
|
|
14959
15683
|
this.fsw._incrReadyCount();
|
|
14960
|
-
|
|
14961
|
-
this._addToNodeFs(
|
|
15684
|
+
path28 = sp.join(dir, sp.relative(dir, path28));
|
|
15685
|
+
this._addToNodeFs(path28, initialAdd, wh, depth + 1);
|
|
14962
15686
|
}
|
|
14963
15687
|
}).on(EV.ERROR, this._boundHandleError);
|
|
14964
15688
|
return new Promise((resolve15, reject) => {
|
|
@@ -15027,13 +15751,13 @@ var NodeFsHandler = class {
|
|
|
15027
15751
|
* @param depth Child path actually targeted for watch
|
|
15028
15752
|
* @param target Child path actually targeted for watch
|
|
15029
15753
|
*/
|
|
15030
|
-
async _addToNodeFs(
|
|
15754
|
+
async _addToNodeFs(path28, initialAdd, priorWh, depth, target) {
|
|
15031
15755
|
const ready = this.fsw._emitReady;
|
|
15032
|
-
if (this.fsw._isIgnored(
|
|
15756
|
+
if (this.fsw._isIgnored(path28) || this.fsw.closed) {
|
|
15033
15757
|
ready();
|
|
15034
15758
|
return false;
|
|
15035
15759
|
}
|
|
15036
|
-
const wh = this.fsw._getWatchHelpers(
|
|
15760
|
+
const wh = this.fsw._getWatchHelpers(path28);
|
|
15037
15761
|
if (priorWh) {
|
|
15038
15762
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
15039
15763
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -15049,8 +15773,8 @@ var NodeFsHandler = class {
|
|
|
15049
15773
|
const follow = this.fsw.options.followSymlinks;
|
|
15050
15774
|
let closer;
|
|
15051
15775
|
if (stats.isDirectory()) {
|
|
15052
|
-
const absPath = sp.resolve(
|
|
15053
|
-
const targetPath = follow ? await (0, import_promises2.realpath)(
|
|
15776
|
+
const absPath = sp.resolve(path28);
|
|
15777
|
+
const targetPath = follow ? await (0, import_promises2.realpath)(path28) : path28;
|
|
15054
15778
|
if (this.fsw.closed)
|
|
15055
15779
|
return;
|
|
15056
15780
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -15060,29 +15784,29 @@ var NodeFsHandler = class {
|
|
|
15060
15784
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
15061
15785
|
}
|
|
15062
15786
|
} else if (stats.isSymbolicLink()) {
|
|
15063
|
-
const targetPath = follow ? await (0, import_promises2.realpath)(
|
|
15787
|
+
const targetPath = follow ? await (0, import_promises2.realpath)(path28) : path28;
|
|
15064
15788
|
if (this.fsw.closed)
|
|
15065
15789
|
return;
|
|
15066
15790
|
const parent = sp.dirname(wh.watchPath);
|
|
15067
15791
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
15068
15792
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
15069
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
15793
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path28, wh, targetPath);
|
|
15070
15794
|
if (this.fsw.closed)
|
|
15071
15795
|
return;
|
|
15072
15796
|
if (targetPath !== void 0) {
|
|
15073
|
-
this.fsw._symlinkPaths.set(sp.resolve(
|
|
15797
|
+
this.fsw._symlinkPaths.set(sp.resolve(path28), targetPath);
|
|
15074
15798
|
}
|
|
15075
15799
|
} else {
|
|
15076
15800
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
15077
15801
|
}
|
|
15078
15802
|
ready();
|
|
15079
15803
|
if (closer)
|
|
15080
|
-
this.fsw._addPathCloser(
|
|
15804
|
+
this.fsw._addPathCloser(path28, closer);
|
|
15081
15805
|
return false;
|
|
15082
15806
|
} catch (error) {
|
|
15083
15807
|
if (this.fsw._handleError(error)) {
|
|
15084
15808
|
ready();
|
|
15085
|
-
return
|
|
15809
|
+
return path28;
|
|
15086
15810
|
}
|
|
15087
15811
|
}
|
|
15088
15812
|
}
|
|
@@ -15125,24 +15849,24 @@ function createPattern(matcher) {
|
|
|
15125
15849
|
}
|
|
15126
15850
|
return () => false;
|
|
15127
15851
|
}
|
|
15128
|
-
function normalizePath2(
|
|
15129
|
-
if (typeof
|
|
15852
|
+
function normalizePath2(path28) {
|
|
15853
|
+
if (typeof path28 !== "string")
|
|
15130
15854
|
throw new Error("string expected");
|
|
15131
|
-
|
|
15132
|
-
|
|
15855
|
+
path28 = sp2.normalize(path28);
|
|
15856
|
+
path28 = path28.replace(/\\/g, "/");
|
|
15133
15857
|
let prepend = false;
|
|
15134
|
-
if (
|
|
15858
|
+
if (path28.startsWith("//"))
|
|
15135
15859
|
prepend = true;
|
|
15136
|
-
|
|
15860
|
+
path28 = path28.replace(DOUBLE_SLASH_RE, "/");
|
|
15137
15861
|
if (prepend)
|
|
15138
|
-
|
|
15139
|
-
return
|
|
15862
|
+
path28 = "/" + path28;
|
|
15863
|
+
return path28;
|
|
15140
15864
|
}
|
|
15141
15865
|
function matchPatterns(patterns, testString, stats) {
|
|
15142
|
-
const
|
|
15866
|
+
const path28 = normalizePath2(testString);
|
|
15143
15867
|
for (let index = 0; index < patterns.length; index++) {
|
|
15144
15868
|
const pattern = patterns[index];
|
|
15145
|
-
if (pattern(
|
|
15869
|
+
if (pattern(path28, stats)) {
|
|
15146
15870
|
return true;
|
|
15147
15871
|
}
|
|
15148
15872
|
}
|
|
@@ -15180,19 +15904,19 @@ var toUnix = (string) => {
|
|
|
15180
15904
|
}
|
|
15181
15905
|
return str;
|
|
15182
15906
|
};
|
|
15183
|
-
var normalizePathToUnix = (
|
|
15184
|
-
var normalizeIgnored = (cwd = "") => (
|
|
15185
|
-
if (typeof
|
|
15186
|
-
return normalizePathToUnix(sp2.isAbsolute(
|
|
15907
|
+
var normalizePathToUnix = (path28) => toUnix(sp2.normalize(toUnix(path28)));
|
|
15908
|
+
var normalizeIgnored = (cwd = "") => (path28) => {
|
|
15909
|
+
if (typeof path28 === "string") {
|
|
15910
|
+
return normalizePathToUnix(sp2.isAbsolute(path28) ? path28 : sp2.join(cwd, path28));
|
|
15187
15911
|
} else {
|
|
15188
|
-
return
|
|
15912
|
+
return path28;
|
|
15189
15913
|
}
|
|
15190
15914
|
};
|
|
15191
|
-
var getAbsolutePath = (
|
|
15192
|
-
if (sp2.isAbsolute(
|
|
15193
|
-
return
|
|
15915
|
+
var getAbsolutePath = (path28, cwd) => {
|
|
15916
|
+
if (sp2.isAbsolute(path28)) {
|
|
15917
|
+
return path28;
|
|
15194
15918
|
}
|
|
15195
|
-
return sp2.join(cwd,
|
|
15919
|
+
return sp2.join(cwd, path28);
|
|
15196
15920
|
};
|
|
15197
15921
|
var EMPTY_SET = Object.freeze(/* @__PURE__ */ new Set());
|
|
15198
15922
|
var DirEntry = class {
|
|
@@ -15257,10 +15981,10 @@ var WatchHelper = class {
|
|
|
15257
15981
|
dirParts;
|
|
15258
15982
|
followSymlinks;
|
|
15259
15983
|
statMethod;
|
|
15260
|
-
constructor(
|
|
15984
|
+
constructor(path28, follow, fsw) {
|
|
15261
15985
|
this.fsw = fsw;
|
|
15262
|
-
const watchPath =
|
|
15263
|
-
this.path =
|
|
15986
|
+
const watchPath = path28;
|
|
15987
|
+
this.path = path28 = path28.replace(REPLACER_RE, "");
|
|
15264
15988
|
this.watchPath = watchPath;
|
|
15265
15989
|
this.fullWatchPath = sp2.resolve(watchPath);
|
|
15266
15990
|
this.dirParts = [];
|
|
@@ -15400,20 +16124,20 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15400
16124
|
this._closePromise = void 0;
|
|
15401
16125
|
let paths = unifyPaths(paths_);
|
|
15402
16126
|
if (cwd) {
|
|
15403
|
-
paths = paths.map((
|
|
15404
|
-
const absPath = getAbsolutePath(
|
|
16127
|
+
paths = paths.map((path28) => {
|
|
16128
|
+
const absPath = getAbsolutePath(path28, cwd);
|
|
15405
16129
|
return absPath;
|
|
15406
16130
|
});
|
|
15407
16131
|
}
|
|
15408
|
-
paths.forEach((
|
|
15409
|
-
this._removeIgnoredPath(
|
|
16132
|
+
paths.forEach((path28) => {
|
|
16133
|
+
this._removeIgnoredPath(path28);
|
|
15410
16134
|
});
|
|
15411
16135
|
this._userIgnored = void 0;
|
|
15412
16136
|
if (!this._readyCount)
|
|
15413
16137
|
this._readyCount = 0;
|
|
15414
16138
|
this._readyCount += paths.length;
|
|
15415
|
-
Promise.all(paths.map(async (
|
|
15416
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
16139
|
+
Promise.all(paths.map(async (path28) => {
|
|
16140
|
+
const res = await this._nodeFsHandler._addToNodeFs(path28, !_internal, void 0, 0, _origAdd);
|
|
15417
16141
|
if (res)
|
|
15418
16142
|
this._emitReady();
|
|
15419
16143
|
return res;
|
|
@@ -15435,17 +16159,17 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15435
16159
|
return this;
|
|
15436
16160
|
const paths = unifyPaths(paths_);
|
|
15437
16161
|
const { cwd } = this.options;
|
|
15438
|
-
paths.forEach((
|
|
15439
|
-
if (!sp2.isAbsolute(
|
|
16162
|
+
paths.forEach((path28) => {
|
|
16163
|
+
if (!sp2.isAbsolute(path28) && !this._closers.has(path28)) {
|
|
15440
16164
|
if (cwd)
|
|
15441
|
-
|
|
15442
|
-
|
|
16165
|
+
path28 = sp2.join(cwd, path28);
|
|
16166
|
+
path28 = sp2.resolve(path28);
|
|
15443
16167
|
}
|
|
15444
|
-
this._closePath(
|
|
15445
|
-
this._addIgnoredPath(
|
|
15446
|
-
if (this._watched.has(
|
|
16168
|
+
this._closePath(path28);
|
|
16169
|
+
this._addIgnoredPath(path28);
|
|
16170
|
+
if (this._watched.has(path28)) {
|
|
15447
16171
|
this._addIgnoredPath({
|
|
15448
|
-
path:
|
|
16172
|
+
path: path28,
|
|
15449
16173
|
recursive: true
|
|
15450
16174
|
});
|
|
15451
16175
|
}
|
|
@@ -15509,38 +16233,38 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15509
16233
|
* @param stats arguments to be passed with event
|
|
15510
16234
|
* @returns the error if defined, otherwise the value of the FSWatcher instance's `closed` flag
|
|
15511
16235
|
*/
|
|
15512
|
-
async _emit(event,
|
|
16236
|
+
async _emit(event, path28, stats) {
|
|
15513
16237
|
if (this.closed)
|
|
15514
16238
|
return;
|
|
15515
16239
|
const opts = this.options;
|
|
15516
16240
|
if (isWindows)
|
|
15517
|
-
|
|
16241
|
+
path28 = sp2.normalize(path28);
|
|
15518
16242
|
if (opts.cwd)
|
|
15519
|
-
|
|
15520
|
-
const args = [
|
|
16243
|
+
path28 = sp2.relative(opts.cwd, path28);
|
|
16244
|
+
const args = [path28];
|
|
15521
16245
|
if (stats != null)
|
|
15522
16246
|
args.push(stats);
|
|
15523
16247
|
const awf = opts.awaitWriteFinish;
|
|
15524
16248
|
let pw;
|
|
15525
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
16249
|
+
if (awf && (pw = this._pendingWrites.get(path28))) {
|
|
15526
16250
|
pw.lastChange = /* @__PURE__ */ new Date();
|
|
15527
16251
|
return this;
|
|
15528
16252
|
}
|
|
15529
16253
|
if (opts.atomic) {
|
|
15530
16254
|
if (event === EVENTS.UNLINK) {
|
|
15531
|
-
this._pendingUnlinks.set(
|
|
16255
|
+
this._pendingUnlinks.set(path28, [event, ...args]);
|
|
15532
16256
|
setTimeout(() => {
|
|
15533
|
-
this._pendingUnlinks.forEach((entry,
|
|
16257
|
+
this._pendingUnlinks.forEach((entry, path29) => {
|
|
15534
16258
|
this.emit(...entry);
|
|
15535
16259
|
this.emit(EVENTS.ALL, ...entry);
|
|
15536
|
-
this._pendingUnlinks.delete(
|
|
16260
|
+
this._pendingUnlinks.delete(path29);
|
|
15537
16261
|
});
|
|
15538
16262
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
15539
16263
|
return this;
|
|
15540
16264
|
}
|
|
15541
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
16265
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path28)) {
|
|
15542
16266
|
event = EVENTS.CHANGE;
|
|
15543
|
-
this._pendingUnlinks.delete(
|
|
16267
|
+
this._pendingUnlinks.delete(path28);
|
|
15544
16268
|
}
|
|
15545
16269
|
}
|
|
15546
16270
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -15558,16 +16282,16 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15558
16282
|
this.emitWithAll(event, args);
|
|
15559
16283
|
}
|
|
15560
16284
|
};
|
|
15561
|
-
this._awaitWriteFinish(
|
|
16285
|
+
this._awaitWriteFinish(path28, awf.stabilityThreshold, event, awfEmit);
|
|
15562
16286
|
return this;
|
|
15563
16287
|
}
|
|
15564
16288
|
if (event === EVENTS.CHANGE) {
|
|
15565
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
16289
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path28, 50);
|
|
15566
16290
|
if (isThrottled)
|
|
15567
16291
|
return this;
|
|
15568
16292
|
}
|
|
15569
16293
|
if (opts.alwaysStat && stats === void 0 && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
15570
|
-
const fullPath = opts.cwd ? sp2.join(opts.cwd,
|
|
16294
|
+
const fullPath = opts.cwd ? sp2.join(opts.cwd, path28) : path28;
|
|
15571
16295
|
let stats2;
|
|
15572
16296
|
try {
|
|
15573
16297
|
stats2 = await (0, import_promises3.stat)(fullPath);
|
|
@@ -15598,23 +16322,23 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15598
16322
|
* @param timeout duration of time to suppress duplicate actions
|
|
15599
16323
|
* @returns tracking object or false if action should be suppressed
|
|
15600
16324
|
*/
|
|
15601
|
-
_throttle(actionType,
|
|
16325
|
+
_throttle(actionType, path28, timeout) {
|
|
15602
16326
|
if (!this._throttled.has(actionType)) {
|
|
15603
16327
|
this._throttled.set(actionType, /* @__PURE__ */ new Map());
|
|
15604
16328
|
}
|
|
15605
16329
|
const action = this._throttled.get(actionType);
|
|
15606
16330
|
if (!action)
|
|
15607
16331
|
throw new Error("invalid throttle");
|
|
15608
|
-
const actionPath = action.get(
|
|
16332
|
+
const actionPath = action.get(path28);
|
|
15609
16333
|
if (actionPath) {
|
|
15610
16334
|
actionPath.count++;
|
|
15611
16335
|
return false;
|
|
15612
16336
|
}
|
|
15613
16337
|
let timeoutObject;
|
|
15614
16338
|
const clear = () => {
|
|
15615
|
-
const item = action.get(
|
|
16339
|
+
const item = action.get(path28);
|
|
15616
16340
|
const count = item ? item.count : 0;
|
|
15617
|
-
action.delete(
|
|
16341
|
+
action.delete(path28);
|
|
15618
16342
|
clearTimeout(timeoutObject);
|
|
15619
16343
|
if (item)
|
|
15620
16344
|
clearTimeout(item.timeoutObject);
|
|
@@ -15622,7 +16346,7 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15622
16346
|
};
|
|
15623
16347
|
timeoutObject = setTimeout(clear, timeout);
|
|
15624
16348
|
const thr = { timeoutObject, clear, count: 0 };
|
|
15625
|
-
action.set(
|
|
16349
|
+
action.set(path28, thr);
|
|
15626
16350
|
return thr;
|
|
15627
16351
|
}
|
|
15628
16352
|
_incrReadyCount() {
|
|
@@ -15636,44 +16360,44 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15636
16360
|
* @param event
|
|
15637
16361
|
* @param awfEmit Callback to be called when ready for event to be emitted.
|
|
15638
16362
|
*/
|
|
15639
|
-
_awaitWriteFinish(
|
|
16363
|
+
_awaitWriteFinish(path28, threshold, event, awfEmit) {
|
|
15640
16364
|
const awf = this.options.awaitWriteFinish;
|
|
15641
16365
|
if (typeof awf !== "object")
|
|
15642
16366
|
return;
|
|
15643
16367
|
const pollInterval = awf.pollInterval;
|
|
15644
16368
|
let timeoutHandler;
|
|
15645
|
-
let fullPath =
|
|
15646
|
-
if (this.options.cwd && !sp2.isAbsolute(
|
|
15647
|
-
fullPath = sp2.join(this.options.cwd,
|
|
16369
|
+
let fullPath = path28;
|
|
16370
|
+
if (this.options.cwd && !sp2.isAbsolute(path28)) {
|
|
16371
|
+
fullPath = sp2.join(this.options.cwd, path28);
|
|
15648
16372
|
}
|
|
15649
16373
|
const now2 = /* @__PURE__ */ new Date();
|
|
15650
16374
|
const writes = this._pendingWrites;
|
|
15651
16375
|
function awaitWriteFinishFn(prevStat) {
|
|
15652
16376
|
(0, import_node_fs2.stat)(fullPath, (err, curStat) => {
|
|
15653
|
-
if (err || !writes.has(
|
|
16377
|
+
if (err || !writes.has(path28)) {
|
|
15654
16378
|
if (err && err.code !== "ENOENT")
|
|
15655
16379
|
awfEmit(err);
|
|
15656
16380
|
return;
|
|
15657
16381
|
}
|
|
15658
16382
|
const now3 = Number(/* @__PURE__ */ new Date());
|
|
15659
16383
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
15660
|
-
writes.get(
|
|
16384
|
+
writes.get(path28).lastChange = now3;
|
|
15661
16385
|
}
|
|
15662
|
-
const pw = writes.get(
|
|
16386
|
+
const pw = writes.get(path28);
|
|
15663
16387
|
const df = now3 - pw.lastChange;
|
|
15664
16388
|
if (df >= threshold) {
|
|
15665
|
-
writes.delete(
|
|
16389
|
+
writes.delete(path28);
|
|
15666
16390
|
awfEmit(void 0, curStat);
|
|
15667
16391
|
} else {
|
|
15668
16392
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
15669
16393
|
}
|
|
15670
16394
|
});
|
|
15671
16395
|
}
|
|
15672
|
-
if (!writes.has(
|
|
15673
|
-
writes.set(
|
|
16396
|
+
if (!writes.has(path28)) {
|
|
16397
|
+
writes.set(path28, {
|
|
15674
16398
|
lastChange: now2,
|
|
15675
16399
|
cancelWait: () => {
|
|
15676
|
-
writes.delete(
|
|
16400
|
+
writes.delete(path28);
|
|
15677
16401
|
clearTimeout(timeoutHandler);
|
|
15678
16402
|
return event;
|
|
15679
16403
|
}
|
|
@@ -15684,8 +16408,8 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15684
16408
|
/**
|
|
15685
16409
|
* Determines whether user has asked to ignore this path.
|
|
15686
16410
|
*/
|
|
15687
|
-
_isIgnored(
|
|
15688
|
-
if (this.options.atomic && DOT_RE.test(
|
|
16411
|
+
_isIgnored(path28, stats) {
|
|
16412
|
+
if (this.options.atomic && DOT_RE.test(path28))
|
|
15689
16413
|
return true;
|
|
15690
16414
|
if (!this._userIgnored) {
|
|
15691
16415
|
const { cwd } = this.options;
|
|
@@ -15695,17 +16419,17 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15695
16419
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
15696
16420
|
this._userIgnored = anymatch(list, void 0);
|
|
15697
16421
|
}
|
|
15698
|
-
return this._userIgnored(
|
|
16422
|
+
return this._userIgnored(path28, stats);
|
|
15699
16423
|
}
|
|
15700
|
-
_isntIgnored(
|
|
15701
|
-
return !this._isIgnored(
|
|
16424
|
+
_isntIgnored(path28, stat4) {
|
|
16425
|
+
return !this._isIgnored(path28, stat4);
|
|
15702
16426
|
}
|
|
15703
16427
|
/**
|
|
15704
16428
|
* Provides a set of common helpers and properties relating to symlink handling.
|
|
15705
16429
|
* @param path file or directory pattern being watched
|
|
15706
16430
|
*/
|
|
15707
|
-
_getWatchHelpers(
|
|
15708
|
-
return new WatchHelper(
|
|
16431
|
+
_getWatchHelpers(path28) {
|
|
16432
|
+
return new WatchHelper(path28, this.options.followSymlinks, this);
|
|
15709
16433
|
}
|
|
15710
16434
|
// Directory helpers
|
|
15711
16435
|
// -----------------
|
|
@@ -15737,63 +16461,63 @@ var FSWatcher = class extends import_node_events.EventEmitter {
|
|
|
15737
16461
|
* @param item base path of item/directory
|
|
15738
16462
|
*/
|
|
15739
16463
|
_remove(directory, item, isDirectory) {
|
|
15740
|
-
const
|
|
15741
|
-
const fullPath = sp2.resolve(
|
|
15742
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
15743
|
-
if (!this._throttle("remove",
|
|
16464
|
+
const path28 = sp2.join(directory, item);
|
|
16465
|
+
const fullPath = sp2.resolve(path28);
|
|
16466
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path28) || this._watched.has(fullPath);
|
|
16467
|
+
if (!this._throttle("remove", path28, 100))
|
|
15744
16468
|
return;
|
|
15745
16469
|
if (!isDirectory && this._watched.size === 1) {
|
|
15746
16470
|
this.add(directory, item, true);
|
|
15747
16471
|
}
|
|
15748
|
-
const wp = this._getWatchedDir(
|
|
16472
|
+
const wp = this._getWatchedDir(path28);
|
|
15749
16473
|
const nestedDirectoryChildren = wp.getChildren();
|
|
15750
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
16474
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path28, nested));
|
|
15751
16475
|
const parent = this._getWatchedDir(directory);
|
|
15752
16476
|
const wasTracked = parent.has(item);
|
|
15753
16477
|
parent.remove(item);
|
|
15754
16478
|
if (this._symlinkPaths.has(fullPath)) {
|
|
15755
16479
|
this._symlinkPaths.delete(fullPath);
|
|
15756
16480
|
}
|
|
15757
|
-
let relPath =
|
|
16481
|
+
let relPath = path28;
|
|
15758
16482
|
if (this.options.cwd)
|
|
15759
|
-
relPath = sp2.relative(this.options.cwd,
|
|
16483
|
+
relPath = sp2.relative(this.options.cwd, path28);
|
|
15760
16484
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
15761
16485
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
15762
16486
|
if (event === EVENTS.ADD)
|
|
15763
16487
|
return;
|
|
15764
16488
|
}
|
|
15765
|
-
this._watched.delete(
|
|
16489
|
+
this._watched.delete(path28);
|
|
15766
16490
|
this._watched.delete(fullPath);
|
|
15767
16491
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
15768
|
-
if (wasTracked && !this._isIgnored(
|
|
15769
|
-
this._emit(eventName,
|
|
15770
|
-
this._closePath(
|
|
16492
|
+
if (wasTracked && !this._isIgnored(path28))
|
|
16493
|
+
this._emit(eventName, path28);
|
|
16494
|
+
this._closePath(path28);
|
|
15771
16495
|
}
|
|
15772
16496
|
/**
|
|
15773
16497
|
* Closes all watchers for a path
|
|
15774
16498
|
*/
|
|
15775
|
-
_closePath(
|
|
15776
|
-
this._closeFile(
|
|
15777
|
-
const dir = sp2.dirname(
|
|
15778
|
-
this._getWatchedDir(dir).remove(sp2.basename(
|
|
16499
|
+
_closePath(path28) {
|
|
16500
|
+
this._closeFile(path28);
|
|
16501
|
+
const dir = sp2.dirname(path28);
|
|
16502
|
+
this._getWatchedDir(dir).remove(sp2.basename(path28));
|
|
15779
16503
|
}
|
|
15780
16504
|
/**
|
|
15781
16505
|
* Closes only file-specific watchers
|
|
15782
16506
|
*/
|
|
15783
|
-
_closeFile(
|
|
15784
|
-
const closers = this._closers.get(
|
|
16507
|
+
_closeFile(path28) {
|
|
16508
|
+
const closers = this._closers.get(path28);
|
|
15785
16509
|
if (!closers)
|
|
15786
16510
|
return;
|
|
15787
16511
|
closers.forEach((closer) => closer());
|
|
15788
|
-
this._closers.delete(
|
|
16512
|
+
this._closers.delete(path28);
|
|
15789
16513
|
}
|
|
15790
|
-
_addPathCloser(
|
|
16514
|
+
_addPathCloser(path28, closer) {
|
|
15791
16515
|
if (!closer)
|
|
15792
16516
|
return;
|
|
15793
|
-
let list = this._closers.get(
|
|
16517
|
+
let list = this._closers.get(path28);
|
|
15794
16518
|
if (!list) {
|
|
15795
16519
|
list = [];
|
|
15796
|
-
this._closers.set(
|
|
16520
|
+
this._closers.set(path28, list);
|
|
15797
16521
|
}
|
|
15798
16522
|
list.push(closer);
|
|
15799
16523
|
}
|
|
@@ -15823,7 +16547,7 @@ function watch(paths, options = {}) {
|
|
|
15823
16547
|
var chokidar_default = { watch, FSWatcher };
|
|
15824
16548
|
|
|
15825
16549
|
// src/watcher/file-watcher.ts
|
|
15826
|
-
var
|
|
16550
|
+
var path21 = __toESM(require("path"), 1);
|
|
15827
16551
|
var FileWatcher = class {
|
|
15828
16552
|
watcher = null;
|
|
15829
16553
|
projectRoot;
|
|
@@ -15865,9 +16589,9 @@ var FileWatcher = class {
|
|
|
15865
16589
|
watchTargets = [this.projectRoot, this.configPath];
|
|
15866
16590
|
} else {
|
|
15867
16591
|
const externalConfigTargets = this.projectConfigPaths.filter((projectConfigPath) => {
|
|
15868
|
-
const relativeConfigPath =
|
|
16592
|
+
const relativeConfigPath = path21.relative(this.projectRoot, projectConfigPath);
|
|
15869
16593
|
return this.isOutsideProjectPath(relativeConfigPath);
|
|
15870
|
-
}).map((projectConfigPath) => (0, import_fs14.existsSync)(projectConfigPath) ? projectConfigPath : this.getNearestExistingDirectory(
|
|
16594
|
+
}).map((projectConfigPath) => (0, import_fs14.existsSync)(projectConfigPath) ? projectConfigPath : this.getNearestExistingDirectory(path21.dirname(projectConfigPath)));
|
|
15871
16595
|
const uniqueExternalConfigTargets = [...new Set(externalConfigTargets)];
|
|
15872
16596
|
if (uniqueExternalConfigTargets.length > 0) {
|
|
15873
16597
|
watchTargets = [this.projectRoot, ...uniqueExternalConfigTargets];
|
|
@@ -15875,7 +16599,7 @@ var FileWatcher = class {
|
|
|
15875
16599
|
}
|
|
15876
16600
|
const watcherOptions = {
|
|
15877
16601
|
ignored: (filePath) => {
|
|
15878
|
-
const relativePath =
|
|
16602
|
+
const relativePath = path21.relative(this.projectRoot, filePath);
|
|
15879
16603
|
if (!relativePath) return false;
|
|
15880
16604
|
if (this.isProjectConfigPathOrAncestor(relativePath)) {
|
|
15881
16605
|
return false;
|
|
@@ -15883,10 +16607,10 @@ var FileWatcher = class {
|
|
|
15883
16607
|
if (this.isOutsideProjectPath(relativePath)) {
|
|
15884
16608
|
return true;
|
|
15885
16609
|
}
|
|
15886
|
-
if (hasFilteredPathSegment(relativePath,
|
|
16610
|
+
if (hasFilteredPathSegment(relativePath, path21.sep)) {
|
|
15887
16611
|
return true;
|
|
15888
16612
|
}
|
|
15889
|
-
if (isRestrictedDirectory(relativePath,
|
|
16613
|
+
if (isRestrictedDirectory(relativePath, path21.sep)) {
|
|
15890
16614
|
return true;
|
|
15891
16615
|
}
|
|
15892
16616
|
if (ignoreFilter.ignores(relativePath)) {
|
|
@@ -15975,23 +16699,23 @@ var FileWatcher = class {
|
|
|
15975
16699
|
this.scheduleFlush();
|
|
15976
16700
|
}
|
|
15977
16701
|
isProjectConfigPath(filePath) {
|
|
15978
|
-
const relativePath =
|
|
15979
|
-
const normalizedRelativePath =
|
|
16702
|
+
const relativePath = path21.relative(this.projectRoot, filePath);
|
|
16703
|
+
const normalizedRelativePath = path21.normalize(relativePath);
|
|
15980
16704
|
return this.getProjectConfigRelativePaths().some((configPath) => configPath === normalizedRelativePath);
|
|
15981
16705
|
}
|
|
15982
16706
|
isProjectConfigPathOrAncestor(relativePath) {
|
|
15983
|
-
const normalizedRelativePath =
|
|
16707
|
+
const normalizedRelativePath = path21.normalize(relativePath);
|
|
15984
16708
|
return this.getProjectConfigRelativePaths().some(
|
|
15985
|
-
(configPath) => configPath === normalizedRelativePath || configPath.startsWith(`${normalizedRelativePath}${
|
|
16709
|
+
(configPath) => configPath === normalizedRelativePath || configPath.startsWith(`${normalizedRelativePath}${path21.sep}`)
|
|
15986
16710
|
);
|
|
15987
16711
|
}
|
|
15988
16712
|
isOutsideProjectPath(relativePath) {
|
|
15989
|
-
return relativePath === ".." || relativePath.startsWith(`..${
|
|
16713
|
+
return relativePath === ".." || relativePath.startsWith(`..${path21.sep}`) || path21.isAbsolute(relativePath);
|
|
15990
16714
|
}
|
|
15991
16715
|
getNearestExistingDirectory(directoryPath) {
|
|
15992
16716
|
let candidate = directoryPath;
|
|
15993
16717
|
while (!(0, import_fs14.existsSync)(candidate)) {
|
|
15994
|
-
const parent =
|
|
16718
|
+
const parent = path21.dirname(candidate);
|
|
15995
16719
|
if (parent === candidate) break;
|
|
15996
16720
|
candidate = parent;
|
|
15997
16721
|
}
|
|
@@ -15999,7 +16723,7 @@ var FileWatcher = class {
|
|
|
15999
16723
|
}
|
|
16000
16724
|
getProjectConfigRelativePaths() {
|
|
16001
16725
|
return this.projectConfigPaths.map(
|
|
16002
|
-
(configPath) =>
|
|
16726
|
+
(configPath) => path21.normalize(path21.relative(this.projectRoot, configPath))
|
|
16003
16727
|
);
|
|
16004
16728
|
}
|
|
16005
16729
|
scheduleFlush() {
|
|
@@ -16015,7 +16739,7 @@ var FileWatcher = class {
|
|
|
16015
16739
|
return;
|
|
16016
16740
|
}
|
|
16017
16741
|
const changes = Array.from(this.pendingChanges.entries()).map(
|
|
16018
|
-
([
|
|
16742
|
+
([path28, type]) => ({ path: path28, type })
|
|
16019
16743
|
);
|
|
16020
16744
|
this.pendingChanges.clear();
|
|
16021
16745
|
try {
|
|
@@ -16050,7 +16774,7 @@ var FileWatcher = class {
|
|
|
16050
16774
|
};
|
|
16051
16775
|
|
|
16052
16776
|
// src/watcher/git-head-watcher.ts
|
|
16053
|
-
var
|
|
16777
|
+
var path22 = __toESM(require("path"), 1);
|
|
16054
16778
|
var GitHeadWatcher = class {
|
|
16055
16779
|
watcher = null;
|
|
16056
16780
|
projectRoot;
|
|
@@ -16072,7 +16796,7 @@ var GitHeadWatcher = class {
|
|
|
16072
16796
|
this.onBranchChange = handler;
|
|
16073
16797
|
this.currentBranch = getCurrentBranch(this.projectRoot);
|
|
16074
16798
|
const headPath = getHeadPath(this.projectRoot);
|
|
16075
|
-
const refsPath =
|
|
16799
|
+
const refsPath = path22.join(this.projectRoot, ".git", "refs", "heads");
|
|
16076
16800
|
this.watcher = chokidar_default.watch([headPath, refsPath], {
|
|
16077
16801
|
persistent: true,
|
|
16078
16802
|
ignoreInitial: true,
|
|
@@ -16200,32 +16924,6 @@ function tool(input) {
|
|
|
16200
16924
|
}
|
|
16201
16925
|
tool.schema = import_zod.z;
|
|
16202
16926
|
|
|
16203
|
-
// src/tools/contracts.ts
|
|
16204
|
-
var CHUNK_TYPES = [
|
|
16205
|
-
"function",
|
|
16206
|
-
"class",
|
|
16207
|
-
"method",
|
|
16208
|
-
"interface",
|
|
16209
|
-
"type",
|
|
16210
|
-
"enum",
|
|
16211
|
-
"struct",
|
|
16212
|
-
"impl",
|
|
16213
|
-
"trait",
|
|
16214
|
-
"module",
|
|
16215
|
-
"other"
|
|
16216
|
-
];
|
|
16217
|
-
var CALL_GRAPH_DIRECTIONS = ["callers", "callees"];
|
|
16218
|
-
var RELATIONSHIP_TYPES = [
|
|
16219
|
-
"Call",
|
|
16220
|
-
"MethodCall",
|
|
16221
|
-
"Constructor",
|
|
16222
|
-
"Import",
|
|
16223
|
-
"Inherits",
|
|
16224
|
-
"Implements"
|
|
16225
|
-
];
|
|
16226
|
-
var INDEX_LOG_CATEGORIES = ["search", "embedding", "cache", "gc", "branch", "general"];
|
|
16227
|
-
var INDEX_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
16228
|
-
|
|
16229
16927
|
// src/tools/format-pr-impact.ts
|
|
16230
16928
|
function formatPrImpact(result) {
|
|
16231
16929
|
const lines = [];
|
|
@@ -16334,117 +17032,6 @@ var pr_impact = tool({
|
|
|
16334
17032
|
}
|
|
16335
17033
|
});
|
|
16336
17034
|
|
|
16337
|
-
// src/tools/symbol-inference.ts
|
|
16338
|
-
var IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g;
|
|
16339
|
-
var QUOTED_BACKTICK_RE = /`([^`]+)`/g;
|
|
16340
|
-
var QUOTED_SINGLE_RE = /'([^'\\]+)'/g;
|
|
16341
|
-
var QUOTED_DOUBLE_RE = /"([^"]+)"/g;
|
|
16342
|
-
var SYMBOL_LIKE_RE = /^(?:[A-Za-z_$][A-Za-z0-9_$]*)$/;
|
|
16343
|
-
var CAMEL_CASE_RE = /^[a-z_][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*$/;
|
|
16344
|
-
var PASCAL_CASE_RE = /^[A-Z][A-Za-z0-9_$]*$/;
|
|
16345
|
-
var SNAKE_CASE_RE = /^[a-z][a-z0-9_]*_[a-z0-9_]+$/;
|
|
16346
|
-
var DEFINITION_INTENT_RE = /\b(where|defined|definition|define|declaration|symbol|function|method|class|interface|type)\b/i;
|
|
16347
|
-
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
16348
|
-
"a",
|
|
16349
|
-
"an",
|
|
16350
|
-
"and",
|
|
16351
|
-
"are",
|
|
16352
|
-
"at",
|
|
16353
|
-
"for",
|
|
16354
|
-
"find",
|
|
16355
|
-
"how",
|
|
16356
|
-
"i",
|
|
16357
|
-
"in",
|
|
16358
|
-
"is",
|
|
16359
|
-
"it",
|
|
16360
|
-
"of",
|
|
16361
|
-
"on",
|
|
16362
|
-
"that",
|
|
16363
|
-
"the",
|
|
16364
|
-
"definition",
|
|
16365
|
-
"show",
|
|
16366
|
-
"to",
|
|
16367
|
-
"where",
|
|
16368
|
-
"which",
|
|
16369
|
-
"what",
|
|
16370
|
-
"you",
|
|
16371
|
-
"your",
|
|
16372
|
-
"with"
|
|
16373
|
-
]);
|
|
16374
|
-
function stripCallSuffix(token) {
|
|
16375
|
-
return token.replace(/\(\s*\)$/, "");
|
|
16376
|
-
}
|
|
16377
|
-
function isLikelySymbolName(token) {
|
|
16378
|
-
if (!SYMBOL_LIKE_RE.test(token)) {
|
|
16379
|
-
return false;
|
|
16380
|
-
}
|
|
16381
|
-
if (STOP_WORDS.has(token.toLowerCase())) {
|
|
16382
|
-
return false;
|
|
16383
|
-
}
|
|
16384
|
-
return CAMEL_CASE_RE.test(token) || PASCAL_CASE_RE.test(token) || SNAKE_CASE_RE.test(token);
|
|
16385
|
-
}
|
|
16386
|
-
function extractQuotedIdentifiers(query) {
|
|
16387
|
-
const identifiers = /* @__PURE__ */ new Set();
|
|
16388
|
-
for (const match of query.matchAll(QUOTED_BACKTICK_RE)) {
|
|
16389
|
-
const candidate = stripCallSuffix(match[1].trim());
|
|
16390
|
-
if (candidate && isLikelySymbolName(candidate)) {
|
|
16391
|
-
identifiers.add(candidate);
|
|
16392
|
-
}
|
|
16393
|
-
}
|
|
16394
|
-
for (const match of query.matchAll(QUOTED_SINGLE_RE)) {
|
|
16395
|
-
const candidate = stripCallSuffix(match[1].trim());
|
|
16396
|
-
if (candidate && isLikelySymbolName(candidate)) {
|
|
16397
|
-
identifiers.add(candidate);
|
|
16398
|
-
}
|
|
16399
|
-
}
|
|
16400
|
-
for (const match of query.matchAll(QUOTED_DOUBLE_RE)) {
|
|
16401
|
-
const candidate = stripCallSuffix(match[1].trim());
|
|
16402
|
-
if (candidate && isLikelySymbolName(candidate)) {
|
|
16403
|
-
identifiers.add(candidate);
|
|
16404
|
-
}
|
|
16405
|
-
}
|
|
16406
|
-
return [...identifiers];
|
|
16407
|
-
}
|
|
16408
|
-
function extractBareIdentifiers(query) {
|
|
16409
|
-
const unquoted = query.replace(QUOTED_BACKTICK_RE, " ").replace(QUOTED_SINGLE_RE, " ").replace(QUOTED_DOUBLE_RE, " ");
|
|
16410
|
-
const identifiers = /* @__PURE__ */ new Set();
|
|
16411
|
-
for (const match of unquoted.matchAll(IDENTIFIER_RE)) {
|
|
16412
|
-
const candidate = stripCallSuffix(match[0]);
|
|
16413
|
-
if (isLikelySymbolName(candidate)) {
|
|
16414
|
-
identifiers.add(candidate);
|
|
16415
|
-
}
|
|
16416
|
-
}
|
|
16417
|
-
return [...identifiers];
|
|
16418
|
-
}
|
|
16419
|
-
function isSingleMeaningfulToken(query, symbol) {
|
|
16420
|
-
const tokens = query.replace(/[`'"()]/g, " ").split(/[^A-Za-z0-9_$]+/).map((token) => token.trim().toLowerCase()).filter((token) => token.length > 0).filter((token) => !STOP_WORDS.has(token));
|
|
16421
|
-
return tokens.length === 1 && tokens[0] === symbol.toLowerCase();
|
|
16422
|
-
}
|
|
16423
|
-
function inferExactSymbolFromQuery(query) {
|
|
16424
|
-
if (analyzeQueryIntent(query).explicitArtifactIntent) {
|
|
16425
|
-
return void 0;
|
|
16426
|
-
}
|
|
16427
|
-
const quoted = extractQuotedIdentifiers(query);
|
|
16428
|
-
if (quoted.length === 1) {
|
|
16429
|
-
return quoted[0];
|
|
16430
|
-
}
|
|
16431
|
-
if (quoted.length > 1) {
|
|
16432
|
-
return void 0;
|
|
16433
|
-
}
|
|
16434
|
-
const candidates = extractBareIdentifiers(query);
|
|
16435
|
-
if (candidates.length !== 1) {
|
|
16436
|
-
return void 0;
|
|
16437
|
-
}
|
|
16438
|
-
const candidate = candidates[0];
|
|
16439
|
-
if (DEFINITION_INTENT_RE.test(query)) {
|
|
16440
|
-
return candidate;
|
|
16441
|
-
}
|
|
16442
|
-
if (isSingleMeaningfulToken(query, candidate)) {
|
|
16443
|
-
return candidate;
|
|
16444
|
-
}
|
|
16445
|
-
return void 0;
|
|
16446
|
-
}
|
|
16447
|
-
|
|
16448
17035
|
// src/tools/context-search.ts
|
|
16449
17036
|
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
16450
17037
|
var MAX_CONTEXT_RESULT_LIMIT = 100;
|
|
@@ -16795,7 +17382,7 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
16795
17382
|
const directory = input.directory ?? void 0;
|
|
16796
17383
|
const tokenBudget = input.tokenBudget ?? void 0;
|
|
16797
17384
|
if (from && to) {
|
|
16798
|
-
const
|
|
17385
|
+
const path28 = await getCallGraphPath(
|
|
16799
17386
|
projectRoot,
|
|
16800
17387
|
host,
|
|
16801
17388
|
from,
|
|
@@ -16804,25 +17391,25 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
16804
17391
|
fromFilePath,
|
|
16805
17392
|
toFilePath
|
|
16806
17393
|
);
|
|
16807
|
-
const pathText = formatCallGraphPathResult(
|
|
16808
|
-
if (
|
|
17394
|
+
const pathText = formatCallGraphPathResult(path28);
|
|
17395
|
+
if (path28.path.length > 0) {
|
|
16809
17396
|
const fitted2 = fitTextToContextBudget(
|
|
16810
17397
|
pathText,
|
|
16811
17398
|
tokenBudget
|
|
16812
17399
|
);
|
|
16813
17400
|
return {
|
|
16814
17401
|
text: fitted2.text,
|
|
16815
|
-
details: fittedDetails("path", fitted2,
|
|
17402
|
+
details: fittedDetails("path", fitted2, path28.path.length)
|
|
16816
17403
|
};
|
|
16817
17404
|
}
|
|
16818
|
-
if (
|
|
17405
|
+
if (path28.from.status !== "resolved" || path28.to.status !== "resolved") {
|
|
16819
17406
|
const fitted2 = fitTextToContextBudget(pathText, tokenBudget);
|
|
16820
17407
|
return {
|
|
16821
17408
|
text: fitted2.text,
|
|
16822
17409
|
details: fittedDetails("path", fitted2, 0)
|
|
16823
17410
|
};
|
|
16824
17411
|
}
|
|
16825
|
-
const resolvedFrom =
|
|
17412
|
+
const resolvedFrom = path28.from;
|
|
16826
17413
|
const { callers } = await getCallGraphData(projectRoot, host, {
|
|
16827
17414
|
name: to,
|
|
16828
17415
|
direction: "callers",
|
|
@@ -16967,7 +17554,7 @@ async function executeCallGraph(projectRoot, host, args) {
|
|
|
16967
17554
|
return { text: formatCallGraphResult(await getCallGraphData(projectRoot, host, args)) };
|
|
16968
17555
|
}
|
|
16969
17556
|
async function executeCallGraphPath(projectRoot, host, args) {
|
|
16970
|
-
const
|
|
17557
|
+
const path28 = await getCallGraphPath(
|
|
16971
17558
|
projectRoot,
|
|
16972
17559
|
host,
|
|
16973
17560
|
args.from,
|
|
@@ -16976,17 +17563,21 @@ async function executeCallGraphPath(projectRoot, host, args) {
|
|
|
16976
17563
|
args.fromFilePath,
|
|
16977
17564
|
args.toFilePath
|
|
16978
17565
|
);
|
|
16979
|
-
return { text: formatCallGraphPathResult(
|
|
17566
|
+
return { text: formatCallGraphPathResult(path28) };
|
|
17567
|
+
}
|
|
17568
|
+
async function executeCodeCommunities(projectRoot, host, args) {
|
|
17569
|
+
const result = await getCodeCommunities(projectRoot, host, args);
|
|
17570
|
+
return { text: formatCodeCommunities(result) };
|
|
16980
17571
|
}
|
|
16981
17572
|
|
|
16982
17573
|
// src/adapters/opencode/tools.ts
|
|
16983
17574
|
var import_fs15 = require("fs");
|
|
16984
17575
|
var os7 = __toESM(require("os"), 1);
|
|
16985
|
-
var
|
|
17576
|
+
var path25 = __toESM(require("path"), 1);
|
|
16986
17577
|
|
|
16987
17578
|
// src/tools/visualize/activity.ts
|
|
16988
17579
|
var import_child_process5 = require("child_process");
|
|
16989
|
-
var
|
|
17580
|
+
var path23 = __toESM(require("path"), 1);
|
|
16990
17581
|
function attachRecentActivity(data, projectRoot) {
|
|
16991
17582
|
const activity = readGitActivity(projectRoot);
|
|
16992
17583
|
const changes = activity.size > 0 ? buildGitChanges(data, activity, projectRoot) : buildGraphChanges(data);
|
|
@@ -17148,7 +17739,7 @@ function normalizePath3(filePath) {
|
|
|
17148
17739
|
return filePath.replace(/\\/g, "/");
|
|
17149
17740
|
}
|
|
17150
17741
|
function toGitRelativePath(projectRoot, filePath) {
|
|
17151
|
-
const relativePath =
|
|
17742
|
+
const relativePath = path23.isAbsolute(filePath) ? path23.relative(projectRoot, filePath) : filePath;
|
|
17152
17743
|
return normalizePath3(relativePath);
|
|
17153
17744
|
}
|
|
17154
17745
|
|
|
@@ -17406,7 +17997,7 @@ render();
|
|
|
17406
17997
|
}
|
|
17407
17998
|
|
|
17408
17999
|
// src/tools/visualize/transform.ts
|
|
17409
|
-
var
|
|
18000
|
+
var path24 = __toESM(require("path"), 1);
|
|
17410
18001
|
|
|
17411
18002
|
// src/tools/visualize/modules.ts
|
|
17412
18003
|
var MAX_MODULES = 18;
|
|
@@ -17666,7 +18257,7 @@ function transformForVisualization(symbols, edges, options = {}) {
|
|
|
17666
18257
|
filePath: s.filePath,
|
|
17667
18258
|
kind: s.kind,
|
|
17668
18259
|
line: s.startLine,
|
|
17669
|
-
directory:
|
|
18260
|
+
directory: path24.dirname(s.filePath),
|
|
17670
18261
|
moduleId: "",
|
|
17671
18262
|
moduleLabel: ""
|
|
17672
18263
|
}));
|
|
@@ -17908,6 +18499,20 @@ var remove_knowledge_base = tool({
|
|
|
17908
18499
|
return removeKnowledgeBase(context?.worktree, DEFAULT_HOST, args.path.trim());
|
|
17909
18500
|
}
|
|
17910
18501
|
});
|
|
18502
|
+
var code_communities = tool({
|
|
18503
|
+
description: "Discover natural module boundaries and hub symbols in the codebase using graph community detection. Clusters symbols by call-graph connectivity, reports community memberships, and identifies hub nodes with cross-community connections, and summarizes couplings between communities.",
|
|
18504
|
+
args: {
|
|
18505
|
+
branch: z3.string().optional().describe("Branch name to analyze (defaults to current branch)"),
|
|
18506
|
+
minSize: z3.number().int().min(CODE_COMMUNITIES_MIN_SIZE).optional().default(CODE_COMMUNITIES_MIN_SIZE).describe("Minimum community size to include (default: 1)"),
|
|
18507
|
+
limit: z3.number().int().min(1).max(CODE_COMMUNITIES_MAX_LIMIT).optional().default(CODE_COMMUNITIES_DEFAULT_LIMIT).describe("Maximum number of communities and hub nodes to return (default: 20)"),
|
|
18508
|
+
hubThreshold: z3.number().int().min(0).optional().default(CODE_COMMUNITIES_DEFAULT_HUB_THRESHOLD).describe("Minimum distinct cross-community neighbors to flag a hub node (default: 5)"),
|
|
18509
|
+
minCoupling: z3.number().int().min(CODE_COMMUNITIES_MIN_COUPLING).optional().default(CODE_COMMUNITIES_MIN_COUPLING).describe("Minimum distinct cross-community connection count to report a coupling (default: 1)"),
|
|
18510
|
+
couplingLimit: z3.number().int().min(1).max(CODE_COMMUNITIES_MAX_COUPLING_LIMIT).optional().default(CODE_COMMUNITIES_DEFAULT_COUPLING_LIMIT).describe("Maximum number of couplings to return (default: 20)")
|
|
18511
|
+
},
|
|
18512
|
+
async execute(args, context) {
|
|
18513
|
+
return (await executeCodeCommunities(context?.worktree, DEFAULT_HOST, args)).text;
|
|
18514
|
+
}
|
|
18515
|
+
});
|
|
17911
18516
|
var index_visualize = tool({
|
|
17912
18517
|
description: "Generate an interactive HTML visualization of recent code movement and the call graph. Starts with temporal onboarding context from Git history, then supports module, symbol, hotspot, and cycle drill-down.",
|
|
17913
18518
|
args: {
|
|
@@ -17933,7 +18538,7 @@ var index_visualize = tool({
|
|
|
17933
18538
|
return "No connected symbols found for visualization. Try including orphans with includeOrphans=true, or check that the call graph has resolved edges.";
|
|
17934
18539
|
}
|
|
17935
18540
|
const html = generateVisualizationHtml(vizData);
|
|
17936
|
-
const outputPath =
|
|
18541
|
+
const outputPath = path25.join(os7.tmpdir(), `call-graph-${Date.now()}.html`);
|
|
17937
18542
|
(0, import_fs15.writeFileSync)(outputPath, html, "utf-8");
|
|
17938
18543
|
let result = `Temporal call graph visualization generated: ${outputPath}
|
|
17939
18544
|
|
|
@@ -17969,6 +18574,7 @@ var TOOL_NAME = {
|
|
|
17969
18574
|
CALL_GRAPH: "call_graph",
|
|
17970
18575
|
CALL_GRAPH_PATH: "call_graph_path",
|
|
17971
18576
|
PR_IMPACT: "pr_impact",
|
|
18577
|
+
CODE_COMMUNITIES: "code_communities",
|
|
17972
18578
|
ADD_KNOWLEDGE_BASE: "add_knowledge_base",
|
|
17973
18579
|
LIST_KNOWLEDGE_BASES: "list_knowledge_bases",
|
|
17974
18580
|
REMOVE_KNOWLEDGE_BASE: "remove_knowledge_base",
|
|
@@ -17990,7 +18596,8 @@ var PORTABLE_TOOL_NAMES = [
|
|
|
17990
18596
|
TOOL_NAME.IMPLEMENTATION_LOOKUP,
|
|
17991
18597
|
TOOL_NAME.CALL_GRAPH,
|
|
17992
18598
|
TOOL_NAME.CALL_GRAPH_PATH,
|
|
17993
|
-
TOOL_NAME.PR_IMPACT
|
|
18599
|
+
TOOL_NAME.PR_IMPACT,
|
|
18600
|
+
TOOL_NAME.CODE_COMMUNITIES
|
|
17994
18601
|
];
|
|
17995
18602
|
var OPENCODE_TOOL_NAMES = [
|
|
17996
18603
|
TOOL_NAME.CODEBASE_CONTEXT,
|
|
@@ -18009,6 +18616,7 @@ var OPENCODE_TOOL_NAMES = [
|
|
|
18009
18616
|
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18010
18617
|
TOOL_NAME.REMOVE_KNOWLEDGE_BASE,
|
|
18011
18618
|
TOOL_NAME.PR_IMPACT,
|
|
18619
|
+
TOOL_NAME.CODE_COMMUNITIES,
|
|
18012
18620
|
TOOL_NAME.INDEX_VISUALIZE
|
|
18013
18621
|
];
|
|
18014
18622
|
var PI_TOOL_NAMES = [
|
|
@@ -18025,6 +18633,7 @@ var PI_TOOL_NAMES = [
|
|
|
18025
18633
|
TOOL_NAME.CALL_GRAPH,
|
|
18026
18634
|
TOOL_NAME.CALL_GRAPH_PATH,
|
|
18027
18635
|
TOOL_NAME.PR_IMPACT,
|
|
18636
|
+
TOOL_NAME.CODE_COMMUNITIES,
|
|
18028
18637
|
TOOL_NAME.PI_KNOWLEDGE_BASE_LIST,
|
|
18029
18638
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
18030
18639
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
@@ -18032,7 +18641,7 @@ var PI_TOOL_NAMES = [
|
|
|
18032
18641
|
|
|
18033
18642
|
// src/commands/loader.ts
|
|
18034
18643
|
var import_fs16 = require("fs");
|
|
18035
|
-
var
|
|
18644
|
+
var path26 = __toESM(require("path"), 1);
|
|
18036
18645
|
function parseFrontmatter(content) {
|
|
18037
18646
|
const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
|
|
18038
18647
|
const match = content.match(frontmatterRegex);
|
|
@@ -18058,7 +18667,7 @@ function loadCommandsFromDirectory(commandsDir) {
|
|
|
18058
18667
|
}
|
|
18059
18668
|
const files = (0, import_fs16.readdirSync)(commandsDir).filter((f) => f.endsWith(".md"));
|
|
18060
18669
|
for (const file of files) {
|
|
18061
|
-
const filePath =
|
|
18670
|
+
const filePath = path26.join(commandsDir, file);
|
|
18062
18671
|
let content;
|
|
18063
18672
|
try {
|
|
18064
18673
|
content = (0, import_fs16.readFileSync)(filePath, "utf-8");
|
|
@@ -18067,7 +18676,7 @@ function loadCommandsFromDirectory(commandsDir) {
|
|
|
18067
18676
|
throw new Error(`Failed to load command file ${filePath}: ${message}`);
|
|
18068
18677
|
}
|
|
18069
18678
|
const { frontmatter, body } = parseFrontmatter(content);
|
|
18070
|
-
const name =
|
|
18679
|
+
const name = path26.basename(file, ".md");
|
|
18071
18680
|
const description = frontmatter.description || `Run the ${name} command`;
|
|
18072
18681
|
commands.set(name, {
|
|
18073
18682
|
description,
|
|
@@ -18415,10 +19024,10 @@ function replaceActiveWatcher(projectRoot, nextWatcher) {
|
|
|
18415
19024
|
function getCommandsDir() {
|
|
18416
19025
|
let currentDir = process.cwd();
|
|
18417
19026
|
if (typeof import_meta2 !== "undefined" && import_meta2.url) {
|
|
18418
|
-
currentDir =
|
|
19027
|
+
currentDir = path27.dirname((0, import_url.fileURLToPath)(import_meta2.url));
|
|
18419
19028
|
}
|
|
18420
|
-
const packageRoot =
|
|
18421
|
-
return
|
|
19029
|
+
const packageRoot = path27.basename(currentDir) === "adapters" ? path27.join(currentDir, "..", "..") : path27.join(currentDir, "..");
|
|
19030
|
+
return path27.join(packageRoot, "commands");
|
|
18422
19031
|
}
|
|
18423
19032
|
function appendRoutingHints(output, hints, preferredRole) {
|
|
18424
19033
|
const preferredBucket = preferredRole === "developer" ? output.developer : output.system;
|
|
@@ -18481,6 +19090,7 @@ var plugin = async ({ directory, worktree }) => {
|
|
|
18481
19090
|
[TOOL_NAME.LIST_KNOWLEDGE_BASES]: list_knowledge_bases,
|
|
18482
19091
|
[TOOL_NAME.REMOVE_KNOWLEDGE_BASE]: remove_knowledge_base,
|
|
18483
19092
|
[TOOL_NAME.PR_IMPACT]: pr_impact,
|
|
19093
|
+
[TOOL_NAME.CODE_COMMUNITIES]: code_communities,
|
|
18484
19094
|
[TOOL_NAME.INDEX_VISUALIZE]: index_visualize
|
|
18485
19095
|
},
|
|
18486
19096
|
async "chat.message"(input, output) {
|