opencode-codebase-index 0.21.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/dist/cli.cjs +2154 -1391
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2237 -1474
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +1975 -1365
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2017 -1407
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +1699 -1087
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +1736 -1124
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +5 -3
package/dist/index.js
CHANGED
|
@@ -328,7 +328,7 @@ var require_ignore = __commonJS({
|
|
|
328
328
|
// path matching.
|
|
329
329
|
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
330
330
|
// @returns {TestResult} true if a file is ignored
|
|
331
|
-
test(
|
|
331
|
+
test(path28, checkUnignored, mode) {
|
|
332
332
|
let ignored = false;
|
|
333
333
|
let unignored = false;
|
|
334
334
|
let matchedRule;
|
|
@@ -337,7 +337,7 @@ var require_ignore = __commonJS({
|
|
|
337
337
|
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
338
338
|
return;
|
|
339
339
|
}
|
|
340
|
-
const matched = rule[mode].test(
|
|
340
|
+
const matched = rule[mode].test(path28);
|
|
341
341
|
if (!matched) {
|
|
342
342
|
return;
|
|
343
343
|
}
|
|
@@ -358,17 +358,17 @@ var require_ignore = __commonJS({
|
|
|
358
358
|
var throwError = (message, Ctor) => {
|
|
359
359
|
throw new Ctor(message);
|
|
360
360
|
};
|
|
361
|
-
var checkPath = (
|
|
362
|
-
if (!isString(
|
|
361
|
+
var checkPath = (path28, originalPath, doThrow) => {
|
|
362
|
+
if (!isString(path28)) {
|
|
363
363
|
return doThrow(
|
|
364
364
|
`path must be a string, but got \`${originalPath}\``,
|
|
365
365
|
TypeError
|
|
366
366
|
);
|
|
367
367
|
}
|
|
368
|
-
if (!
|
|
368
|
+
if (!path28) {
|
|
369
369
|
return doThrow(`path must not be empty`, TypeError);
|
|
370
370
|
}
|
|
371
|
-
if (checkPath.isNotRelative(
|
|
371
|
+
if (checkPath.isNotRelative(path28)) {
|
|
372
372
|
const r = "`path.relative()`d";
|
|
373
373
|
return doThrow(
|
|
374
374
|
`path should be a ${r} string, but got "${originalPath}"`,
|
|
@@ -377,7 +377,7 @@ var require_ignore = __commonJS({
|
|
|
377
377
|
}
|
|
378
378
|
return true;
|
|
379
379
|
};
|
|
380
|
-
var isNotRelative = (
|
|
380
|
+
var isNotRelative = (path28) => REGEX_TEST_INVALID_PATH.test(path28);
|
|
381
381
|
checkPath.isNotRelative = isNotRelative;
|
|
382
382
|
checkPath.convert = (p) => p;
|
|
383
383
|
var Ignore2 = class {
|
|
@@ -407,19 +407,19 @@ var require_ignore = __commonJS({
|
|
|
407
407
|
}
|
|
408
408
|
// @returns {TestResult}
|
|
409
409
|
_test(originalPath, cache, checkUnignored, slices) {
|
|
410
|
-
const
|
|
410
|
+
const path28 = originalPath && checkPath.convert(originalPath);
|
|
411
411
|
checkPath(
|
|
412
|
-
|
|
412
|
+
path28,
|
|
413
413
|
originalPath,
|
|
414
414
|
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
415
415
|
);
|
|
416
|
-
return this._t(
|
|
416
|
+
return this._t(path28, cache, checkUnignored, slices);
|
|
417
417
|
}
|
|
418
|
-
checkIgnore(
|
|
419
|
-
if (!REGEX_TEST_TRAILING_SLASH.test(
|
|
420
|
-
return this.test(
|
|
418
|
+
checkIgnore(path28) {
|
|
419
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path28)) {
|
|
420
|
+
return this.test(path28);
|
|
421
421
|
}
|
|
422
|
-
const slices =
|
|
422
|
+
const slices = path28.split(SLASH2).filter(Boolean);
|
|
423
423
|
slices.pop();
|
|
424
424
|
if (slices.length) {
|
|
425
425
|
const parent = this._t(
|
|
@@ -432,18 +432,18 @@ var require_ignore = __commonJS({
|
|
|
432
432
|
return parent;
|
|
433
433
|
}
|
|
434
434
|
}
|
|
435
|
-
return this._rules.test(
|
|
435
|
+
return this._rules.test(path28, false, MODE_CHECK_IGNORE);
|
|
436
436
|
}
|
|
437
|
-
_t(
|
|
438
|
-
if (
|
|
439
|
-
return cache[
|
|
437
|
+
_t(path28, cache, checkUnignored, slices) {
|
|
438
|
+
if (path28 in cache) {
|
|
439
|
+
return cache[path28];
|
|
440
440
|
}
|
|
441
441
|
if (!slices) {
|
|
442
|
-
slices =
|
|
442
|
+
slices = path28.split(SLASH2).filter(Boolean);
|
|
443
443
|
}
|
|
444
444
|
slices.pop();
|
|
445
445
|
if (!slices.length) {
|
|
446
|
-
return cache[
|
|
446
|
+
return cache[path28] = this._rules.test(path28, checkUnignored, MODE_IGNORE);
|
|
447
447
|
}
|
|
448
448
|
const parent = this._t(
|
|
449
449
|
slices.join(SLASH2) + SLASH2,
|
|
@@ -451,29 +451,29 @@ var require_ignore = __commonJS({
|
|
|
451
451
|
checkUnignored,
|
|
452
452
|
slices
|
|
453
453
|
);
|
|
454
|
-
return cache[
|
|
454
|
+
return cache[path28] = parent.ignored ? parent : this._rules.test(path28, checkUnignored, MODE_IGNORE);
|
|
455
455
|
}
|
|
456
|
-
ignores(
|
|
457
|
-
return this._test(
|
|
456
|
+
ignores(path28) {
|
|
457
|
+
return this._test(path28, this._ignoreCache, false).ignored;
|
|
458
458
|
}
|
|
459
459
|
createFilter() {
|
|
460
|
-
return (
|
|
460
|
+
return (path28) => !this.ignores(path28);
|
|
461
461
|
}
|
|
462
462
|
filter(paths) {
|
|
463
463
|
return makeArray(paths).filter(this.createFilter());
|
|
464
464
|
}
|
|
465
465
|
// @returns {TestResult}
|
|
466
|
-
test(
|
|
467
|
-
return this._test(
|
|
466
|
+
test(path28) {
|
|
467
|
+
return this._test(path28, this._testCache, true);
|
|
468
468
|
}
|
|
469
469
|
};
|
|
470
470
|
var factory = (options) => new Ignore2(options);
|
|
471
|
-
var isPathValid = (
|
|
471
|
+
var isPathValid = (path28) => checkPath(path28 && checkPath.convert(path28), path28, RETURN_FALSE);
|
|
472
472
|
var setupWindows = () => {
|
|
473
473
|
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
474
474
|
checkPath.convert = makePosix;
|
|
475
475
|
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
476
|
-
checkPath.isNotRelative = (
|
|
476
|
+
checkPath.isNotRelative = (path28) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path28) || isNotRelative(path28);
|
|
477
477
|
};
|
|
478
478
|
if (
|
|
479
479
|
// Detect `process` so that it can run in browsers.
|
|
@@ -651,7 +651,7 @@ var require_eventemitter3 = __commonJS({
|
|
|
651
651
|
});
|
|
652
652
|
|
|
653
653
|
// src/adapters/opencode.ts
|
|
654
|
-
import * as
|
|
654
|
+
import * as path27 from "path";
|
|
655
655
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
656
656
|
|
|
657
657
|
// src/config/constants.ts
|
|
@@ -805,7 +805,8 @@ function getDefaultSearchConfig() {
|
|
|
805
805
|
contextLines: 0,
|
|
806
806
|
routingHints: true,
|
|
807
807
|
routingGraphHandoffHints: false,
|
|
808
|
-
routingHintRole: "system"
|
|
808
|
+
routingHintRole: "system",
|
|
809
|
+
communityBoost: 0
|
|
809
810
|
};
|
|
810
811
|
}
|
|
811
812
|
function getDefaultRerankerBaseUrl(provider) {
|
|
@@ -938,7 +939,8 @@ function parseConfig(raw) {
|
|
|
938
939
|
contextLines: typeof rawSearch.contextLines === "number" ? Math.min(50, Math.max(0, rawSearch.contextLines)) : defaultSearch.contextLines,
|
|
939
940
|
routingHints: typeof rawSearch.routingHints === "boolean" ? rawSearch.routingHints : defaultSearch.routingHints,
|
|
940
941
|
routingGraphHandoffHints: typeof rawSearch.routingGraphHandoffHints === "boolean" ? rawSearch.routingGraphHandoffHints : defaultSearch.routingGraphHandoffHints,
|
|
941
|
-
routingHintRole: rawSearch.routingHintRole === "developer" || rawSearch.routingHintRole === "system" ? rawSearch.routingHintRole : defaultSearch.routingHintRole
|
|
942
|
+
routingHintRole: rawSearch.routingHintRole === "developer" || rawSearch.routingHintRole === "system" ? rawSearch.routingHintRole : defaultSearch.routingHintRole,
|
|
943
|
+
communityBoost: typeof rawSearch.communityBoost === "number" && Number.isFinite(rawSearch.communityBoost) ? Math.min(1, Math.max(0, rawSearch.communityBoost)) : defaultSearch.communityBoost
|
|
942
944
|
};
|
|
943
945
|
const rawDebug = input.debug && typeof input.debug === "object" ? input.debug : {};
|
|
944
946
|
const debug = {
|
|
@@ -1729,8 +1731,8 @@ function loadMergedConfig(projectRoot, host) {
|
|
|
1729
1731
|
}
|
|
1730
1732
|
|
|
1731
1733
|
// src/tools/operations.ts
|
|
1732
|
-
import { existsSync as
|
|
1733
|
-
import * as
|
|
1734
|
+
import { existsSync as existsSync12, realpathSync as realpathSync5, statSync as statSync5 } from "fs";
|
|
1735
|
+
import * as path20 from "path";
|
|
1734
1736
|
|
|
1735
1737
|
// src/tools/knowledge-base-paths.ts
|
|
1736
1738
|
import * as path8 from "path";
|
|
@@ -1773,6 +1775,152 @@ function findKnowledgeBasePathIndex(knowledgeBases, inputPath, projectRoot) {
|
|
|
1773
1775
|
);
|
|
1774
1776
|
}
|
|
1775
1777
|
|
|
1778
|
+
// src/tools/format-communities.ts
|
|
1779
|
+
function compareText(left, right) {
|
|
1780
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
1781
|
+
}
|
|
1782
|
+
function buildCodeCommunitiesResult(communities, centrality, couplings = [], options = {}) {
|
|
1783
|
+
const minSize = options.minSize ?? 1;
|
|
1784
|
+
const limit = options.limit ?? 20;
|
|
1785
|
+
const hubThreshold = options.hubThreshold ?? 5;
|
|
1786
|
+
const minCoupling = options.minCoupling ?? 1;
|
|
1787
|
+
const couplingLimit = options.couplingLimit ?? 20;
|
|
1788
|
+
const communityMap = /* @__PURE__ */ new Map();
|
|
1789
|
+
for (const c of communities) {
|
|
1790
|
+
let entry = communityMap.get(c.communityId);
|
|
1791
|
+
if (!entry) {
|
|
1792
|
+
entry = { label: c.communityLabel, members: [] };
|
|
1793
|
+
communityMap.set(c.communityId, entry);
|
|
1794
|
+
}
|
|
1795
|
+
entry.members.push(c);
|
|
1796
|
+
}
|
|
1797
|
+
const sortedCommunities = Array.from(communityMap.entries()).map(([id, entry]) => ({
|
|
1798
|
+
id,
|
|
1799
|
+
label: entry.label,
|
|
1800
|
+
symbolCount: entry.members.length,
|
|
1801
|
+
members: entry.members.map((m) => ({
|
|
1802
|
+
symbolId: m.symbolId,
|
|
1803
|
+
symbolName: m.symbolName,
|
|
1804
|
+
filePath: m.filePath
|
|
1805
|
+
})).sort((a, b) => compareText(a.symbolName, b.symbolName) || compareText(a.symbolId, b.symbolId))
|
|
1806
|
+
})).filter((c) => c.symbolCount >= minSize).sort((a, b) => b.symbolCount - a.symbolCount || compareText(a.label, b.label) || a.id - b.id).slice(0, limit);
|
|
1807
|
+
const communityBySymbol = new Map(communities.map((community) => [community.symbolId, community]));
|
|
1808
|
+
const communityLabelById = new Map(communities.map((community) => [community.communityId, community.communityLabel]));
|
|
1809
|
+
const hubNodes = centrality.map((c) => ({
|
|
1810
|
+
symbolId: c.symbolId,
|
|
1811
|
+
symbolName: c.symbolName,
|
|
1812
|
+
filePath: c.filePath,
|
|
1813
|
+
callerCount: c.callerCount,
|
|
1814
|
+
calleeCount: c.calleeCount,
|
|
1815
|
+
totalConnections: c.totalConnections,
|
|
1816
|
+
crossCommunityConnections: communityBySymbol.get(c.symbolId)?.crossCommunityConnections ?? 0
|
|
1817
|
+
})).filter((h) => h.crossCommunityConnections >= hubThreshold).sort(
|
|
1818
|
+
(a, b) => b.crossCommunityConnections - a.crossCommunityConnections || b.totalConnections - a.totalConnections || compareText(a.symbolId, b.symbolId)
|
|
1819
|
+
).slice(0, limit);
|
|
1820
|
+
const canonicalCoupling = (value) => Math.trunc(value);
|
|
1821
|
+
const couplingItems = couplings.map((entry) => {
|
|
1822
|
+
const relationships = entry.relationships ?? entry.representativeRelationships ?? [];
|
|
1823
|
+
const normalizedRelationships = relationships.map((relationship) => ({
|
|
1824
|
+
fromSymbolId: relationship.fromSymbolId,
|
|
1825
|
+
fromSymbolName: relationship.fromSymbolName,
|
|
1826
|
+
fromFilePath: relationship.fromFilePath,
|
|
1827
|
+
toSymbolId: relationship.toSymbolId,
|
|
1828
|
+
toSymbolName: relationship.toSymbolName,
|
|
1829
|
+
toFilePath: relationship.toFilePath
|
|
1830
|
+
})).sort(
|
|
1831
|
+
(left, right) => compareText(left.fromSymbolName, right.fromSymbolName) || compareText(left.fromSymbolId, right.fromSymbolId) || compareText(left.toSymbolName, right.toSymbolName) || compareText(left.toSymbolId, right.toSymbolId) || compareText(left.fromFilePath, right.fromFilePath) || compareText(left.toFilePath, right.toFilePath)
|
|
1832
|
+
).slice(0, 5);
|
|
1833
|
+
const communityA = canonicalCoupling(Math.min(entry.communityA, entry.communityB));
|
|
1834
|
+
const communityB = canonicalCoupling(Math.max(entry.communityA, entry.communityB));
|
|
1835
|
+
return {
|
|
1836
|
+
communityA,
|
|
1837
|
+
communityB,
|
|
1838
|
+
communityAName: communityLabelById.get(communityA) ?? `Community ${communityA}`,
|
|
1839
|
+
communityBName: communityLabelById.get(communityB) ?? `Community ${communityB}`,
|
|
1840
|
+
distinctConnections: canonicalCoupling(entry.count),
|
|
1841
|
+
representativeRelationships: normalizedRelationships
|
|
1842
|
+
};
|
|
1843
|
+
}).filter((entry) => entry.distinctConnections >= minCoupling).sort(
|
|
1844
|
+
(left, right) => right.distinctConnections - left.distinctConnections || compareText(left.communityAName, right.communityAName) || compareText(left.communityBName, right.communityBName) || compareText(left.communityAName + left.communityBName, right.communityAName + right.communityBName) || left.communityA - right.communityA || left.communityB - right.communityB
|
|
1845
|
+
).slice(0, Math.max(1, Math.floor(couplingLimit)));
|
|
1846
|
+
return {
|
|
1847
|
+
communities: sortedCommunities,
|
|
1848
|
+
hubNodes,
|
|
1849
|
+
totalSymbols: communities.length,
|
|
1850
|
+
totalCommunities: communityMap.size,
|
|
1851
|
+
couplings: couplingItems
|
|
1852
|
+
};
|
|
1853
|
+
}
|
|
1854
|
+
function formatCodeCommunities(result) {
|
|
1855
|
+
const lines = [];
|
|
1856
|
+
lines.push(`\u2192 Communities: ${result.totalCommunities} (${result.communities.length} shown, ${result.totalSymbols} symbols total)`);
|
|
1857
|
+
for (const community of result.communities) {
|
|
1858
|
+
lines.push(` Community ${community.id} (${community.label}): ${community.symbolCount} symbols`);
|
|
1859
|
+
const shownMembers = community.members.slice(0, 8);
|
|
1860
|
+
for (const m of shownMembers) {
|
|
1861
|
+
lines.push(` - ${m.symbolName} (${m.filePath})`);
|
|
1862
|
+
}
|
|
1863
|
+
if (community.members.length > 8) {
|
|
1864
|
+
lines.push(` ... and ${community.members.length - 8} more`);
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
if (result.hubNodes.length > 0) {
|
|
1868
|
+
lines.push(`\u2192 Hub nodes (${result.hubNodes.length} shown, cross-community connections):`);
|
|
1869
|
+
for (const hub of result.hubNodes) {
|
|
1870
|
+
lines.push(
|
|
1871
|
+
` - ${hub.symbolName} (${hub.crossCommunityConnections} cross-community, ${hub.callerCount} callers, ${hub.calleeCount} callees) at ${hub.filePath}`
|
|
1872
|
+
);
|
|
1873
|
+
}
|
|
1874
|
+
} else {
|
|
1875
|
+
lines.push("\u2192 Hub nodes: none with significant cross-community connections");
|
|
1876
|
+
}
|
|
1877
|
+
if (result.couplings.length > 0) {
|
|
1878
|
+
lines.push(`\u2192 Community couplings: ${result.couplings.length} shown`);
|
|
1879
|
+
for (const coupling of result.couplings) {
|
|
1880
|
+
lines.push(` - ${coupling.communityAName} \u2194 ${coupling.communityBName}: ${coupling.distinctConnections} distinct connections`);
|
|
1881
|
+
for (const relationship of coupling.representativeRelationships) {
|
|
1882
|
+
lines.push(` - ${relationship.fromSymbolName} (${relationship.fromFilePath}) -> ${relationship.toSymbolName} (${relationship.toFilePath})`);
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
} else {
|
|
1886
|
+
lines.push("\u2192 Community couplings: none above minCoupling threshold");
|
|
1887
|
+
}
|
|
1888
|
+
return lines.join("\n");
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// src/tools/contracts.ts
|
|
1892
|
+
var CHUNK_TYPES = [
|
|
1893
|
+
"function",
|
|
1894
|
+
"class",
|
|
1895
|
+
"method",
|
|
1896
|
+
"interface",
|
|
1897
|
+
"type",
|
|
1898
|
+
"enum",
|
|
1899
|
+
"struct",
|
|
1900
|
+
"impl",
|
|
1901
|
+
"trait",
|
|
1902
|
+
"module",
|
|
1903
|
+
"other"
|
|
1904
|
+
];
|
|
1905
|
+
var CALL_GRAPH_DIRECTIONS = ["callers", "callees"];
|
|
1906
|
+
var RELATIONSHIP_TYPES = [
|
|
1907
|
+
"Call",
|
|
1908
|
+
"MethodCall",
|
|
1909
|
+
"Constructor",
|
|
1910
|
+
"Import",
|
|
1911
|
+
"Inherits",
|
|
1912
|
+
"Implements"
|
|
1913
|
+
];
|
|
1914
|
+
var INDEX_LOG_CATEGORIES = ["search", "embedding", "cache", "gc", "branch", "general"];
|
|
1915
|
+
var INDEX_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
1916
|
+
var CODE_COMMUNITIES_MIN_SIZE = 1;
|
|
1917
|
+
var CODE_COMMUNITIES_DEFAULT_LIMIT = 20;
|
|
1918
|
+
var CODE_COMMUNITIES_MAX_LIMIT = 100;
|
|
1919
|
+
var CODE_COMMUNITIES_DEFAULT_HUB_THRESHOLD = 5;
|
|
1920
|
+
var CODE_COMMUNITIES_MIN_COUPLING = 1;
|
|
1921
|
+
var CODE_COMMUNITIES_DEFAULT_COUPLING_LIMIT = 20;
|
|
1922
|
+
var CODE_COMMUNITIES_MAX_COUPLING_LIMIT = 100;
|
|
1923
|
+
|
|
1776
1924
|
// src/tools/context-pack.ts
|
|
1777
1925
|
import { get_encoding } from "tiktoken";
|
|
1778
1926
|
|
|
@@ -2416,8 +2564,8 @@ function formatExactSearchHandoff(results) {
|
|
|
2416
2564
|
}
|
|
2417
2565
|
function formatContextEvidence(result, index) {
|
|
2418
2566
|
const symbol = result.name ? ` ${JSON.stringify(compactEvidenceValue(result.name, 80))}` : "";
|
|
2419
|
-
const
|
|
2420
|
-
return `[${index}] ${result.chunkType}${symbol} in ${
|
|
2567
|
+
const path28 = compactEvidenceValue(result.filePath, 120);
|
|
2568
|
+
return `[${index}] ${result.chunkType}${symbol} in ${path28}:${result.startLine}-${result.endLine} (score ${result.score.toFixed(2)})`;
|
|
2421
2569
|
}
|
|
2422
2570
|
function formatContextPack(heading, selected, candidateCount, duplicateCount, limitOmittedCount, budgetOmittedCount, includeExactSearchHandoff) {
|
|
2423
2571
|
const lines = selected.map((result, index) => formatContextEvidence(result, index + 1));
|
|
@@ -4439,8 +4587,8 @@ function saveConfig(projectRoot, config, host) {
|
|
|
4439
4587
|
}
|
|
4440
4588
|
|
|
4441
4589
|
// src/indexer/index.ts
|
|
4442
|
-
import { existsSync as
|
|
4443
|
-
import * as
|
|
4590
|
+
import { existsSync as existsSync11, readFileSync as readFileSync8, statSync as statSync4, writeFileSync as writeFileSync3, renameSync as renameSync3, unlinkSync as unlinkSync2, mkdirSync as mkdirSync4, promises as fsPromises3 } from "fs";
|
|
4591
|
+
import * as path19 from "path";
|
|
4444
4592
|
import { performance as performance2 } from "perf_hooks";
|
|
4445
4593
|
import { execFile as execFile5 } from "child_process";
|
|
4446
4594
|
import { promisify as promisify4 } from "util";
|
|
@@ -7031,6 +7179,9 @@ function createMockNativeBinding() {
|
|
|
7031
7179
|
detectCommunities() {
|
|
7032
7180
|
throw error;
|
|
7033
7181
|
}
|
|
7182
|
+
detectCommunityCouplings() {
|
|
7183
|
+
throw error;
|
|
7184
|
+
}
|
|
7034
7185
|
computeCentrality() {
|
|
7035
7186
|
throw error;
|
|
7036
7187
|
}
|
|
@@ -7268,6 +7419,18 @@ var Database = class _Database {
|
|
|
7268
7419
|
}
|
|
7269
7420
|
this.closed = true;
|
|
7270
7421
|
}
|
|
7422
|
+
beginWriteTransaction() {
|
|
7423
|
+
this.throwIfClosed();
|
|
7424
|
+
this.inner.beginWriteTransaction();
|
|
7425
|
+
}
|
|
7426
|
+
commitWriteTransaction() {
|
|
7427
|
+
this.throwIfClosed();
|
|
7428
|
+
this.inner.commitWriteTransaction();
|
|
7429
|
+
}
|
|
7430
|
+
rollbackWriteTransaction() {
|
|
7431
|
+
this.throwIfClosed();
|
|
7432
|
+
this.inner.rollbackWriteTransaction();
|
|
7433
|
+
}
|
|
7271
7434
|
embeddingExists(contentHash) {
|
|
7272
7435
|
this.throwIfClosed();
|
|
7273
7436
|
return this.inner.embeddingExists(contentHash);
|
|
@@ -7527,6 +7690,13 @@ var Database = class _Database {
|
|
|
7527
7690
|
this.throwIfClosed();
|
|
7528
7691
|
return this.inner.computeCentrality(branch);
|
|
7529
7692
|
}
|
|
7693
|
+
detectCommunityCouplings(branch) {
|
|
7694
|
+
this.throwIfClosed();
|
|
7695
|
+
return this.inner.detectCommunityCouplings(branch).map((entry) => ({
|
|
7696
|
+
...entry,
|
|
7697
|
+
relationships: entry.representativeRelationships ?? []
|
|
7698
|
+
}));
|
|
7699
|
+
}
|
|
7530
7700
|
};
|
|
7531
7701
|
|
|
7532
7702
|
// src/git/branch-materialization.ts
|
|
@@ -8296,6 +8466,21 @@ async function getChunkGitBlame(projectRoot, filePath, startLine, endLine) {
|
|
|
8296
8466
|
}
|
|
8297
8467
|
|
|
8298
8468
|
// src/indexer/search-ranking.ts
|
|
8469
|
+
function applyCommunityBoost(candidates, sameCommunityCandidateIds, boost) {
|
|
8470
|
+
if (boost <= 0 || sameCommunityCandidateIds.size === 0 || candidates.length <= 1) {
|
|
8471
|
+
return candidates;
|
|
8472
|
+
}
|
|
8473
|
+
const result = candidates.map((candidate) => sameCommunityCandidateIds.has(candidate.id) ? { ...candidate, score: candidate.score * (1 + boost) } : candidate);
|
|
8474
|
+
for (let index = 1; index < result.length; index += 1) {
|
|
8475
|
+
const candidate = result[index];
|
|
8476
|
+
const previous = result[index - 1];
|
|
8477
|
+
if (candidate && previous && sameCommunityCandidateIds.has(candidate.id) && !sameCommunityCandidateIds.has(previous.id) && candidate.score > previous.score) {
|
|
8478
|
+
result[index - 1] = candidate;
|
|
8479
|
+
result[index] = previous;
|
|
8480
|
+
}
|
|
8481
|
+
}
|
|
8482
|
+
return result;
|
|
8483
|
+
}
|
|
8299
8484
|
var RANK_HYBRID_CACHE_LIMIT = 256;
|
|
8300
8485
|
var rankHybridResultsCache = /* @__PURE__ */ new WeakMap();
|
|
8301
8486
|
function classifyQueryIntentRaw(query) {
|
|
@@ -8474,6 +8659,117 @@ function rankSemanticOnlyResults(query, semanticResults, options) {
|
|
|
8474
8659
|
});
|
|
8475
8660
|
}
|
|
8476
8661
|
|
|
8662
|
+
// src/tools/symbol-inference.ts
|
|
8663
|
+
var IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g;
|
|
8664
|
+
var QUOTED_BACKTICK_RE = /`([^`]+)`/g;
|
|
8665
|
+
var QUOTED_SINGLE_RE = /'([^'\\]+)'/g;
|
|
8666
|
+
var QUOTED_DOUBLE_RE = /"([^"]+)"/g;
|
|
8667
|
+
var SYMBOL_LIKE_RE = /^(?:[A-Za-z_$][A-Za-z0-9_$]*)$/;
|
|
8668
|
+
var CAMEL_CASE_RE = /^[a-z_][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*$/;
|
|
8669
|
+
var PASCAL_CASE_RE = /^[A-Z][A-Za-z0-9_$]*$/;
|
|
8670
|
+
var SNAKE_CASE_RE = /^[a-z][a-z0-9_]*_[a-z0-9_]+$/;
|
|
8671
|
+
var DEFINITION_INTENT_RE = /\b(where|defined|definition|define|declaration|symbol|function|method|class|interface|type)\b/i;
|
|
8672
|
+
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
8673
|
+
"a",
|
|
8674
|
+
"an",
|
|
8675
|
+
"and",
|
|
8676
|
+
"are",
|
|
8677
|
+
"at",
|
|
8678
|
+
"for",
|
|
8679
|
+
"find",
|
|
8680
|
+
"how",
|
|
8681
|
+
"i",
|
|
8682
|
+
"in",
|
|
8683
|
+
"is",
|
|
8684
|
+
"it",
|
|
8685
|
+
"of",
|
|
8686
|
+
"on",
|
|
8687
|
+
"that",
|
|
8688
|
+
"the",
|
|
8689
|
+
"definition",
|
|
8690
|
+
"show",
|
|
8691
|
+
"to",
|
|
8692
|
+
"where",
|
|
8693
|
+
"which",
|
|
8694
|
+
"what",
|
|
8695
|
+
"you",
|
|
8696
|
+
"your",
|
|
8697
|
+
"with"
|
|
8698
|
+
]);
|
|
8699
|
+
function stripCallSuffix(token) {
|
|
8700
|
+
return token.replace(/\(\s*\)$/, "");
|
|
8701
|
+
}
|
|
8702
|
+
function isLikelySymbolName(token) {
|
|
8703
|
+
if (!SYMBOL_LIKE_RE.test(token)) {
|
|
8704
|
+
return false;
|
|
8705
|
+
}
|
|
8706
|
+
if (STOP_WORDS.has(token.toLowerCase())) {
|
|
8707
|
+
return false;
|
|
8708
|
+
}
|
|
8709
|
+
return CAMEL_CASE_RE.test(token) || PASCAL_CASE_RE.test(token) || SNAKE_CASE_RE.test(token);
|
|
8710
|
+
}
|
|
8711
|
+
function extractQuotedIdentifiers(query) {
|
|
8712
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
8713
|
+
for (const match of query.matchAll(QUOTED_BACKTICK_RE)) {
|
|
8714
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
8715
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
8716
|
+
identifiers.add(candidate);
|
|
8717
|
+
}
|
|
8718
|
+
}
|
|
8719
|
+
for (const match of query.matchAll(QUOTED_SINGLE_RE)) {
|
|
8720
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
8721
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
8722
|
+
identifiers.add(candidate);
|
|
8723
|
+
}
|
|
8724
|
+
}
|
|
8725
|
+
for (const match of query.matchAll(QUOTED_DOUBLE_RE)) {
|
|
8726
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
8727
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
8728
|
+
identifiers.add(candidate);
|
|
8729
|
+
}
|
|
8730
|
+
}
|
|
8731
|
+
return [...identifiers];
|
|
8732
|
+
}
|
|
8733
|
+
function extractBareIdentifiers(query) {
|
|
8734
|
+
const unquoted = query.replace(QUOTED_BACKTICK_RE, " ").replace(QUOTED_SINGLE_RE, " ").replace(QUOTED_DOUBLE_RE, " ");
|
|
8735
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
8736
|
+
for (const match of unquoted.matchAll(IDENTIFIER_RE)) {
|
|
8737
|
+
const candidate = stripCallSuffix(match[0]);
|
|
8738
|
+
if (isLikelySymbolName(candidate)) {
|
|
8739
|
+
identifiers.add(candidate);
|
|
8740
|
+
}
|
|
8741
|
+
}
|
|
8742
|
+
return [...identifiers];
|
|
8743
|
+
}
|
|
8744
|
+
function isSingleMeaningfulToken(query, symbol) {
|
|
8745
|
+
const tokens = query.replace(/[`'"()]/g, " ").split(/[^A-Za-z0-9_$]+/).map((token) => token.trim().toLowerCase()).filter((token) => token.length > 0).filter((token) => !STOP_WORDS.has(token));
|
|
8746
|
+
return tokens.length === 1 && tokens[0] === symbol.toLowerCase();
|
|
8747
|
+
}
|
|
8748
|
+
function inferExactSymbolFromQuery(query) {
|
|
8749
|
+
if (analyzeQueryIntent(query).explicitArtifactIntent) {
|
|
8750
|
+
return void 0;
|
|
8751
|
+
}
|
|
8752
|
+
const quoted = extractQuotedIdentifiers(query);
|
|
8753
|
+
if (quoted.length === 1) {
|
|
8754
|
+
return quoted[0];
|
|
8755
|
+
}
|
|
8756
|
+
if (quoted.length > 1) {
|
|
8757
|
+
return void 0;
|
|
8758
|
+
}
|
|
8759
|
+
const candidates = extractBareIdentifiers(query);
|
|
8760
|
+
if (candidates.length !== 1) {
|
|
8761
|
+
return void 0;
|
|
8762
|
+
}
|
|
8763
|
+
const candidate = candidates[0];
|
|
8764
|
+
if (DEFINITION_INTENT_RE.test(query)) {
|
|
8765
|
+
return candidate;
|
|
8766
|
+
}
|
|
8767
|
+
if (isSingleMeaningfulToken(query, candidate)) {
|
|
8768
|
+
return candidate;
|
|
8769
|
+
}
|
|
8770
|
+
return void 0;
|
|
8771
|
+
}
|
|
8772
|
+
|
|
8477
8773
|
// src/indexer/call-graph-constants.ts
|
|
8478
8774
|
var CALL_GRAPH_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
8479
8775
|
"function_declaration",
|
|
@@ -8976,22 +9272,6 @@ function getUniquePendingChunksFromRequests(requests) {
|
|
|
8976
9272
|
}
|
|
8977
9273
|
return Array.from(uniqueChunks.values());
|
|
8978
9274
|
}
|
|
8979
|
-
function coalesceFailedBatches(batches) {
|
|
8980
|
-
const grouped = /* @__PURE__ */ new Map();
|
|
8981
|
-
for (const batch of batches) {
|
|
8982
|
-
const key = `${batch.attemptCount}:${batch.lastAttempt}:${batch.error}`;
|
|
8983
|
-
const existing = grouped.get(key);
|
|
8984
|
-
if (!existing) {
|
|
8985
|
-
grouped.set(key, {
|
|
8986
|
-
...batch,
|
|
8987
|
-
chunks: [...batch.chunks]
|
|
8988
|
-
});
|
|
8989
|
-
continue;
|
|
8990
|
-
}
|
|
8991
|
-
existing.chunks.push(...batch.chunks);
|
|
8992
|
-
}
|
|
8993
|
-
return Array.from(grouped.values());
|
|
8994
|
-
}
|
|
8995
9275
|
function poolEmbeddingVectors(vectors, weights) {
|
|
8996
9276
|
const firstVector = vectors[0];
|
|
8997
9277
|
if (!firstVector) {
|
|
@@ -9024,66 +9304,325 @@ function hasAllEmbeddingParts(parts, expectedPartCount) {
|
|
|
9024
9304
|
return true;
|
|
9025
9305
|
}
|
|
9026
9306
|
|
|
9027
|
-
// src/indexer/
|
|
9028
|
-
|
|
9029
|
-
|
|
9030
|
-
|
|
9031
|
-
|
|
9032
|
-
|
|
9033
|
-
|
|
9034
|
-
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
"class_declaration",
|
|
9038
|
-
"class_definition"
|
|
9039
|
-
]);
|
|
9040
|
-
var C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set(["class_specifier", "struct_specifier"]);
|
|
9041
|
-
function isCompatibleCFamilyCallTarget(language, callType, symbolKind) {
|
|
9042
|
-
if (language !== "c" && language !== "cpp") return true;
|
|
9043
|
-
if (symbolKind === "namespace_definition") return callType === "Import";
|
|
9044
|
-
const isTypeSymbol = C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES.has(symbolKind);
|
|
9045
|
-
if (callType === "Constructor" || callType === "Inherits" || callType === "Implements") {
|
|
9046
|
-
return isTypeSymbol;
|
|
9307
|
+
// src/indexer/failed-state-persistence.ts
|
|
9308
|
+
import * as fs2 from "fs";
|
|
9309
|
+
import { createHash, randomBytes as randomBytes3 } from "crypto";
|
|
9310
|
+
import * as path18 from "path";
|
|
9311
|
+
import { StringDecoder } from "string_decoder";
|
|
9312
|
+
var CURRENT_FAILED_BATCH_VERSION = 1;
|
|
9313
|
+
var DEFAULT_MALFORMED_LINE_ACTION = "skip";
|
|
9314
|
+
function* readFailedBatchRecords(filePath, options = {}) {
|
|
9315
|
+
if (!fs2.existsSync(filePath)) {
|
|
9316
|
+
return;
|
|
9047
9317
|
}
|
|
9048
|
-
|
|
9049
|
-
|
|
9050
|
-
|
|
9051
|
-
|
|
9052
|
-
|
|
9053
|
-
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
|
|
9059
|
-
"
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
|
|
9066
|
-
|
|
9067
|
-
|
|
9068
|
-
|
|
9069
|
-
|
|
9070
|
-
|
|
9071
|
-
|
|
9318
|
+
const fileFormat = detectFailedBatchFileFormat(filePath);
|
|
9319
|
+
if (fileFormat === "legacy") {
|
|
9320
|
+
yield* readLegacyFailedBatchRecords(filePath, options);
|
|
9321
|
+
return;
|
|
9322
|
+
}
|
|
9323
|
+
yield* readJsonlFailedBatchRecords(filePath, options);
|
|
9324
|
+
}
|
|
9325
|
+
function createFailedBatchWriter(targetPath) {
|
|
9326
|
+
const temporaryPath = createTemporaryPath(targetPath);
|
|
9327
|
+
let finalized = false;
|
|
9328
|
+
fs2.mkdirSync(path18.dirname(targetPath), { recursive: true });
|
|
9329
|
+
fs2.closeSync(fs2.openSync(temporaryPath, "w"));
|
|
9330
|
+
const write = (record) => {
|
|
9331
|
+
if (finalized) {
|
|
9332
|
+
throw new Error("Failed batch writer has been finalized");
|
|
9333
|
+
}
|
|
9334
|
+
const lines = record.chunks.map((chunk) => {
|
|
9335
|
+
const lineRecord = {
|
|
9336
|
+
version: CURRENT_FAILED_BATCH_VERSION,
|
|
9337
|
+
chunks: [chunk],
|
|
9338
|
+
error: record.error,
|
|
9339
|
+
attemptCount: record.attemptCount,
|
|
9340
|
+
lastAttempt: record.lastAttempt
|
|
9341
|
+
};
|
|
9342
|
+
return JSON.stringify(lineRecord);
|
|
9343
|
+
});
|
|
9344
|
+
if (lines.length === 0) {
|
|
9345
|
+
return;
|
|
9072
9346
|
}
|
|
9073
|
-
|
|
9074
|
-
|
|
9075
|
-
|
|
9347
|
+
fs2.mkdirSync(path18.dirname(targetPath), { recursive: true });
|
|
9348
|
+
fs2.appendFileSync(temporaryPath, `${lines.join("\n")}
|
|
9349
|
+
`, "utf-8");
|
|
9350
|
+
};
|
|
9351
|
+
const commit = () => {
|
|
9352
|
+
if (finalized) {
|
|
9353
|
+
return;
|
|
9076
9354
|
}
|
|
9077
|
-
|
|
9078
|
-
|
|
9079
|
-
|
|
9080
|
-
|
|
9081
|
-
|
|
9082
|
-
|
|
9355
|
+
fs2.mkdirSync(path18.dirname(targetPath), { recursive: true });
|
|
9356
|
+
fs2.renameSync(temporaryPath, targetPath);
|
|
9357
|
+
finalized = true;
|
|
9358
|
+
};
|
|
9359
|
+
const cleanup = () => {
|
|
9360
|
+
if (finalized) {
|
|
9361
|
+
return;
|
|
9083
9362
|
}
|
|
9084
|
-
|
|
9085
|
-
|
|
9086
|
-
|
|
9363
|
+
fs2.rmSync(temporaryPath, { force: true });
|
|
9364
|
+
};
|
|
9365
|
+
return {
|
|
9366
|
+
write,
|
|
9367
|
+
commit,
|
|
9368
|
+
cleanup,
|
|
9369
|
+
temporaryPath
|
|
9370
|
+
};
|
|
9371
|
+
}
|
|
9372
|
+
function* readLegacyFailedBatchRecords(filePath, options) {
|
|
9373
|
+
const rawData = fs2.readFileSync(filePath, "utf-8");
|
|
9374
|
+
const trimmed = stripLeadingBomAndWhitespace(rawData).trim();
|
|
9375
|
+
if (trimmed.length === 0) {
|
|
9376
|
+
return;
|
|
9377
|
+
}
|
|
9378
|
+
let parsed;
|
|
9379
|
+
try {
|
|
9380
|
+
parsed = JSON.parse(trimmed);
|
|
9381
|
+
} catch (error) {
|
|
9382
|
+
handleMalformedLine(filePath, 1, trimmed, error, options);
|
|
9383
|
+
return;
|
|
9384
|
+
}
|
|
9385
|
+
if (!Array.isArray(parsed)) {
|
|
9386
|
+
handleMalformedLine(filePath, 1, trimmed, new Error("Expected legacy failed-batch file to contain a JSON array"), options);
|
|
9387
|
+
return;
|
|
9388
|
+
}
|
|
9389
|
+
for (const entry of parsed) {
|
|
9390
|
+
const normalized = normalizeFailedBatchRecord(entry);
|
|
9391
|
+
if (normalized) {
|
|
9392
|
+
yield normalized;
|
|
9393
|
+
}
|
|
9394
|
+
}
|
|
9395
|
+
}
|
|
9396
|
+
function* readJsonlFailedBatchRecords(filePath, options) {
|
|
9397
|
+
const handle = fs2.openSync(filePath, "r");
|
|
9398
|
+
const decoder = new StringDecoder("utf8");
|
|
9399
|
+
const readBuffer = Buffer.allocUnsafe(64 * 1024);
|
|
9400
|
+
let buffer = "";
|
|
9401
|
+
let lineNumber = 0;
|
|
9402
|
+
try {
|
|
9403
|
+
let bytesRead = 0;
|
|
9404
|
+
do {
|
|
9405
|
+
bytesRead = fs2.readSync(handle, readBuffer, 0, readBuffer.length, null);
|
|
9406
|
+
buffer += decoder.write(readBuffer.subarray(0, bytesRead));
|
|
9407
|
+
let newlineIndex = buffer.indexOf("\n");
|
|
9408
|
+
while (newlineIndex >= 0) {
|
|
9409
|
+
const rawLine = buffer.slice(0, newlineIndex);
|
|
9410
|
+
buffer = buffer.slice(newlineIndex + 1);
|
|
9411
|
+
lineNumber += 1;
|
|
9412
|
+
const normalized = parseFailedBatchLine(rawLine, filePath, lineNumber, options);
|
|
9413
|
+
if (normalized) {
|
|
9414
|
+
yield normalized;
|
|
9415
|
+
}
|
|
9416
|
+
newlineIndex = buffer.indexOf("\n");
|
|
9417
|
+
}
|
|
9418
|
+
} while (bytesRead > 0);
|
|
9419
|
+
buffer += decoder.end();
|
|
9420
|
+
const finalLine = buffer.trimEnd();
|
|
9421
|
+
if (finalLine.length > 0) {
|
|
9422
|
+
lineNumber += 1;
|
|
9423
|
+
const normalized = parseFailedBatchLine(finalLine, filePath, lineNumber, options);
|
|
9424
|
+
if (normalized) {
|
|
9425
|
+
yield normalized;
|
|
9426
|
+
}
|
|
9427
|
+
}
|
|
9428
|
+
} finally {
|
|
9429
|
+
fs2.closeSync(handle);
|
|
9430
|
+
}
|
|
9431
|
+
}
|
|
9432
|
+
function parseFailedBatchLine(rawLine, filePath, lineNumber, options) {
|
|
9433
|
+
const line = rawLine.trimEnd();
|
|
9434
|
+
if (line.length === 0) {
|
|
9435
|
+
return null;
|
|
9436
|
+
}
|
|
9437
|
+
try {
|
|
9438
|
+
const parsed = JSON.parse(line);
|
|
9439
|
+
const normalized = normalizeFailedBatchRecord(parsed);
|
|
9440
|
+
if (!normalized) {
|
|
9441
|
+
handleMalformedLine(filePath, lineNumber, line, new Error("Malformed failed-batch record"), options);
|
|
9442
|
+
return null;
|
|
9443
|
+
}
|
|
9444
|
+
return normalized;
|
|
9445
|
+
} catch (error) {
|
|
9446
|
+
handleMalformedLine(filePath, lineNumber, line, error, options);
|
|
9447
|
+
return null;
|
|
9448
|
+
}
|
|
9449
|
+
}
|
|
9450
|
+
function normalizeFailedBatchRecord(rawRecord) {
|
|
9451
|
+
if (!rawRecord || typeof rawRecord !== "object" || Array.isArray(rawRecord)) {
|
|
9452
|
+
return null;
|
|
9453
|
+
}
|
|
9454
|
+
const typed = rawRecord;
|
|
9455
|
+
const chunks = Array.isArray(typed.chunks) ? typed.chunks : null;
|
|
9456
|
+
if (!chunks || chunks.length === 0) {
|
|
9457
|
+
return null;
|
|
9458
|
+
}
|
|
9459
|
+
return {
|
|
9460
|
+
version: typeof typed.version === "number" && Number.isFinite(typed.version) ? typed.version : CURRENT_FAILED_BATCH_VERSION,
|
|
9461
|
+
chunks,
|
|
9462
|
+
error: typeof typed.error === "string" ? typed.error : "Unknown embedding error",
|
|
9463
|
+
attemptCount: typeof typed.attemptCount === "number" && Number.isFinite(typed.attemptCount) ? typed.attemptCount : 1,
|
|
9464
|
+
lastAttempt: typeof typed.lastAttempt === "string" ? typed.lastAttempt : (/* @__PURE__ */ new Date()).toISOString()
|
|
9465
|
+
};
|
|
9466
|
+
}
|
|
9467
|
+
function detectFailedBatchFileFormat(filePath) {
|
|
9468
|
+
const handle = fs2.openSync(filePath, "r");
|
|
9469
|
+
try {
|
|
9470
|
+
const buffer = Buffer.alloc(4096);
|
|
9471
|
+
const bytesRead = fs2.readSync(handle, buffer, 0, buffer.length, 0);
|
|
9472
|
+
if (bytesRead <= 0) {
|
|
9473
|
+
return "jsonl";
|
|
9474
|
+
}
|
|
9475
|
+
const prefix = stripLeadingBomAndWhitespace(buffer.subarray(0, bytesRead).toString("utf-8"));
|
|
9476
|
+
return prefix.startsWith("[") ? "legacy" : "jsonl";
|
|
9477
|
+
} finally {
|
|
9478
|
+
fs2.closeSync(handle);
|
|
9479
|
+
}
|
|
9480
|
+
}
|
|
9481
|
+
function stripLeadingBomAndWhitespace(value) {
|
|
9482
|
+
let result = value.trimStart();
|
|
9483
|
+
if (result.charCodeAt(0) === 65279) {
|
|
9484
|
+
result = result.slice(1);
|
|
9485
|
+
}
|
|
9486
|
+
return result;
|
|
9487
|
+
}
|
|
9488
|
+
function createTemporaryPath(targetPath) {
|
|
9489
|
+
const randomId = createHash("sha1").update(`${Date.now()}:${randomBytes3(8).toString("hex")}`).digest("hex");
|
|
9490
|
+
const targetDir = path18.dirname(targetPath);
|
|
9491
|
+
const baseName = path18.basename(targetPath);
|
|
9492
|
+
return path18.join(targetDir, `.${baseName}.${randomId}.tmp`);
|
|
9493
|
+
}
|
|
9494
|
+
function handleMalformedLine(filePath, lineNumber, line, error, options) {
|
|
9495
|
+
const action = options.malformedLineAction ?? DEFAULT_MALFORMED_LINE_ACTION;
|
|
9496
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error));
|
|
9497
|
+
if (options.onMalformedLine) {
|
|
9498
|
+
options.onMalformedLine(normalizedError, line, lineNumber, filePath);
|
|
9499
|
+
}
|
|
9500
|
+
if (action === "fail") {
|
|
9501
|
+
throw normalizedError;
|
|
9502
|
+
}
|
|
9503
|
+
}
|
|
9504
|
+
|
|
9505
|
+
// src/indexer/file-batches.ts
|
|
9506
|
+
var INDEX_FILE_BATCH_LIMITS = Object.freeze({
|
|
9507
|
+
maxFiles: 64,
|
|
9508
|
+
maxBytes: 8 * 1024 * 1024
|
|
9509
|
+
});
|
|
9510
|
+
function* iterateOrderedFileBatches(items, getBytes, limits = INDEX_FILE_BATCH_LIMITS) {
|
|
9511
|
+
const maxFiles = Math.max(1, Math.floor(limits.maxFiles));
|
|
9512
|
+
const maxBytes = Math.max(1, Math.floor(limits.maxBytes));
|
|
9513
|
+
let batch = [];
|
|
9514
|
+
let batchBytes = 0;
|
|
9515
|
+
for (const item of items) {
|
|
9516
|
+
const itemBytes = Math.max(0, Math.floor(getBytes(item)));
|
|
9517
|
+
if (batch.length > 0 && (batch.length >= maxFiles || batchBytes + itemBytes > maxBytes)) {
|
|
9518
|
+
yield batch;
|
|
9519
|
+
batch = [];
|
|
9520
|
+
batchBytes = 0;
|
|
9521
|
+
}
|
|
9522
|
+
batch.push(item);
|
|
9523
|
+
batchBytes += itemBytes;
|
|
9524
|
+
}
|
|
9525
|
+
if (batch.length > 0) {
|
|
9526
|
+
yield batch;
|
|
9527
|
+
}
|
|
9528
|
+
}
|
|
9529
|
+
|
|
9530
|
+
// src/indexer/index.ts
|
|
9531
|
+
var CALL_GRAPH_LANGUAGES = /* @__PURE__ */ new Set(["typescript", "tsx", "javascript", "jsx", "python", "go", "rust", "swift", "php", "apex", "zig", "gdscript", "matlab", "bash", "c", "cpp", "metal"]);
|
|
9532
|
+
var CASE_INSENSITIVE_LANGUAGES = /* @__PURE__ */ new Set(["apex", "php"]);
|
|
9533
|
+
function candidateOverlapsSymbol(candidate, symbol) {
|
|
9534
|
+
return candidate.metadata.filePath === symbol.filePath && candidate.metadata.startLine <= symbol.endLine && candidate.metadata.endLine >= symbol.startLine;
|
|
9535
|
+
}
|
|
9536
|
+
function resolveSameCommunityCandidateIds(query, candidates, database, branchCatalogKeys) {
|
|
9537
|
+
const anchorName = inferExactSymbolFromQuery(query);
|
|
9538
|
+
if (!anchorName || candidates.length === 0) {
|
|
9539
|
+
return /* @__PURE__ */ new Set();
|
|
9540
|
+
}
|
|
9541
|
+
const catalogs = branchCatalogKeys.map((branchKey) => ({
|
|
9542
|
+
branchKey,
|
|
9543
|
+
symbols: database.getSymbolsForBranch(branchKey)
|
|
9544
|
+
}));
|
|
9545
|
+
const exactAnchors = catalogs.flatMap(({ branchKey, symbols }) => symbols.filter((symbol) => symbol.name === anchorName).map((symbol) => ({ branchKey, symbol })));
|
|
9546
|
+
const anchors = exactAnchors.length > 0 ? exactAnchors : catalogs.flatMap(({ branchKey, symbols }) => symbols.filter((symbol) => symbol.name.toLowerCase() === anchorName.toLowerCase()).map((symbol) => ({ branchKey, symbol })));
|
|
9547
|
+
const uniqueAnchors = new Map(anchors.map((anchor2) => [anchor2.symbol.id, anchor2]));
|
|
9548
|
+
if (uniqueAnchors.size !== 1) {
|
|
9549
|
+
return /* @__PURE__ */ new Set();
|
|
9550
|
+
}
|
|
9551
|
+
const anchor = uniqueAnchors.values().next().value;
|
|
9552
|
+
const branchSymbols = catalogs.find((catalog) => catalog.branchKey === anchor.branchKey)?.symbols ?? [];
|
|
9553
|
+
const candidateSymbols = branchSymbols.filter(
|
|
9554
|
+
(symbol) => candidates.some((candidate) => candidateOverlapsSymbol(candidate, symbol))
|
|
9555
|
+
);
|
|
9556
|
+
const assignments = database.detectCommunities(
|
|
9557
|
+
anchor.branchKey,
|
|
9558
|
+
[anchor.symbol.id, ...candidateSymbols.map((symbol) => symbol.id)]
|
|
9559
|
+
);
|
|
9560
|
+
const anchorCommunity = assignments.find((assignment) => assignment.symbolId === anchor.symbol.id)?.communityId;
|
|
9561
|
+
if (anchorCommunity === void 0) {
|
|
9562
|
+
return /* @__PURE__ */ new Set();
|
|
9563
|
+
}
|
|
9564
|
+
const sameCommunitySymbolIds = new Set(assignments.filter((assignment) => assignment.communityId === anchorCommunity).map((assignment) => assignment.symbolId));
|
|
9565
|
+
return new Set(candidates.filter((candidate) => candidateSymbols.some(
|
|
9566
|
+
(symbol) => sameCommunitySymbolIds.has(symbol.id) && candidateOverlapsSymbol(candidate, symbol)
|
|
9567
|
+
)).map((candidate) => candidate.id));
|
|
9568
|
+
}
|
|
9569
|
+
var CALL_GRAPH_RESOLUTION_VERSION = "4";
|
|
9570
|
+
var PHP_FUNCTION_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
9571
|
+
"function_declaration",
|
|
9572
|
+
"function",
|
|
9573
|
+
"function_definition"
|
|
9574
|
+
]);
|
|
9575
|
+
var PHP_CLASS_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
9576
|
+
"class_declaration",
|
|
9577
|
+
"class_definition"
|
|
9578
|
+
]);
|
|
9579
|
+
var C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set(["class_specifier", "struct_specifier"]);
|
|
9580
|
+
function isCompatibleCFamilyCallTarget(language, callType, symbolKind) {
|
|
9581
|
+
if (language !== "c" && language !== "cpp") return true;
|
|
9582
|
+
if (symbolKind === "namespace_definition") return callType === "Import";
|
|
9583
|
+
const isTypeSymbol = C_FAMILY_TYPE_SYMBOL_CHUNK_TYPES.has(symbolKind);
|
|
9584
|
+
if (callType === "Constructor" || callType === "Inherits" || callType === "Implements") {
|
|
9585
|
+
return isTypeSymbol;
|
|
9586
|
+
}
|
|
9587
|
+
return !isTypeSymbol;
|
|
9588
|
+
}
|
|
9589
|
+
var EXECUTABLE_SYMBOL_CHUNK_TYPES = /* @__PURE__ */ new Set([
|
|
9590
|
+
"function_declaration",
|
|
9591
|
+
"function",
|
|
9592
|
+
"arrow_function",
|
|
9593
|
+
"method_definition",
|
|
9594
|
+
"function_definition",
|
|
9595
|
+
"method_declaration",
|
|
9596
|
+
"function_item",
|
|
9597
|
+
"protocol_function_declaration",
|
|
9598
|
+
"init_declaration",
|
|
9599
|
+
"deinit_declaration",
|
|
9600
|
+
"subscript_declaration",
|
|
9601
|
+
"constructor_definition",
|
|
9602
|
+
"trigger_declaration",
|
|
9603
|
+
"test_declaration"
|
|
9604
|
+
]);
|
|
9605
|
+
function findEnclosingSymbol(symbols, line, column) {
|
|
9606
|
+
let best;
|
|
9607
|
+
for (const symbol of symbols) {
|
|
9608
|
+
if (line < symbol.startLine || line > symbol.endLine) continue;
|
|
9609
|
+
if (column !== void 0 && (line === symbol.startLine && column < symbol.startCol || line === symbol.endLine && column >= symbol.endCol)) {
|
|
9610
|
+
continue;
|
|
9611
|
+
}
|
|
9612
|
+
if (!best) {
|
|
9613
|
+
best = symbol;
|
|
9614
|
+
continue;
|
|
9615
|
+
}
|
|
9616
|
+
const span = symbol.endLine - symbol.startLine;
|
|
9617
|
+
const bestSpan = best.endLine - best.startLine;
|
|
9618
|
+
const isNarrowerPositionRange = column !== void 0 && span === bestSpan && symbol.startLine === best.startLine && symbol.endLine === best.endLine && symbol.startCol >= best.startCol && symbol.endCol <= best.endCol && (symbol.startCol > best.startCol || symbol.endCol < best.endCol);
|
|
9619
|
+
const isMoreSpecificTie = span === bestSpan && symbol.startLine === best.startLine && EXECUTABLE_SYMBOL_CHUNK_TYPES.has(symbol.kind) && !EXECUTABLE_SYMBOL_CHUNK_TYPES.has(best.kind);
|
|
9620
|
+
if (span < bestSpan || span === bestSpan && symbol.startLine > best.startLine || isNarrowerPositionRange || isMoreSpecificTie) {
|
|
9621
|
+
best = symbol;
|
|
9622
|
+
}
|
|
9623
|
+
}
|
|
9624
|
+
return best;
|
|
9625
|
+
}
|
|
9087
9626
|
function float32ArrayToBuffer(arr) {
|
|
9088
9627
|
const float32 = new Float32Array(arr);
|
|
9089
9628
|
return Buffer.from(float32.buffer);
|
|
@@ -9127,6 +9666,16 @@ function isSqliteCorruptionError(error) {
|
|
|
9127
9666
|
}
|
|
9128
9667
|
var STARTUP_WARNING_METADATA_KEY = "index.startupWarning";
|
|
9129
9668
|
var READER_ARTIFACT_RETRY_INTERVAL_MS = 1e3;
|
|
9669
|
+
function getFailedBatchGroupKey(record) {
|
|
9670
|
+
return `${record.attemptCount}:${record.lastAttempt}:${record.error}`;
|
|
9671
|
+
}
|
|
9672
|
+
function getPendingChunkId(rawChunk) {
|
|
9673
|
+
if (!rawChunk || typeof rawChunk !== "object") {
|
|
9674
|
+
return null;
|
|
9675
|
+
}
|
|
9676
|
+
const id = rawChunk.id;
|
|
9677
|
+
return typeof id === "string" ? id : null;
|
|
9678
|
+
}
|
|
9130
9679
|
function metadataFromBlame(blame) {
|
|
9131
9680
|
if (!blame) {
|
|
9132
9681
|
return {};
|
|
@@ -9174,9 +9723,9 @@ var SWIFT_PARSER_VERSION = "1";
|
|
|
9174
9723
|
var METAL_PARSER_VERSION = "1";
|
|
9175
9724
|
var SYMBOL_EXTRACTOR_VERSION = "1";
|
|
9176
9725
|
function isPathWithinRoot2(filePath, rootPath) {
|
|
9177
|
-
const normalizedFilePath =
|
|
9178
|
-
const normalizedRoot =
|
|
9179
|
-
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${
|
|
9726
|
+
const normalizedFilePath = path19.resolve(filePath);
|
|
9727
|
+
const normalizedRoot = path19.resolve(rootPath);
|
|
9728
|
+
return normalizedFilePath === normalizedRoot || normalizedFilePath.startsWith(`${normalizedRoot}${path19.sep}`);
|
|
9180
9729
|
}
|
|
9181
9730
|
function promoteIdentifierMatches(query, combined, semanticCandidates, keywordCandidates, database, branchChunkIds, prioritizeSourcePaths = classifyQueryIntentRaw(query) === "source") {
|
|
9182
9731
|
if (combined.length === 0) {
|
|
@@ -9507,10 +10056,10 @@ function matchesHardSearchFilters(candidate, options, projectRoot) {
|
|
|
9507
10056
|
}
|
|
9508
10057
|
if (options?.directory) {
|
|
9509
10058
|
const candidatePath = canonicalizePathForComparison(
|
|
9510
|
-
|
|
10059
|
+
path19.resolve(projectRoot, candidate.metadata.filePath.replace(/\\/g, path19.sep))
|
|
9511
10060
|
);
|
|
9512
10061
|
const directoryPath = canonicalizePathForComparison(
|
|
9513
|
-
|
|
10062
|
+
path19.resolve(projectRoot, options.directory.trim().replace(/\\/g, path19.sep))
|
|
9514
10063
|
);
|
|
9515
10064
|
if (!isPathWithinRoot2(candidatePath, directoryPath)) return false;
|
|
9516
10065
|
}
|
|
@@ -9586,6 +10135,7 @@ var Indexer = class _Indexer {
|
|
|
9586
10135
|
readerArtifactFingerprint = null;
|
|
9587
10136
|
writerArtifactFingerprint = null;
|
|
9588
10137
|
readerArtifactRetryAfter = /* @__PURE__ */ new Map();
|
|
10138
|
+
fileBatchLimits;
|
|
9589
10139
|
constructor(projectRoot, config, host, runtimeOptions = {}) {
|
|
9590
10140
|
this.projectRoot = projectRoot;
|
|
9591
10141
|
this.projectIdentityHash = hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16);
|
|
@@ -9597,6 +10147,7 @@ var Indexer = class _Indexer {
|
|
|
9597
10147
|
}
|
|
9598
10148
|
this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase();
|
|
9599
10149
|
this.indexPathOverride = runtimeOptions.indexPath;
|
|
10150
|
+
this.fileBatchLimits = runtimeOptions.fileBatchLimits;
|
|
9600
10151
|
this.config = config;
|
|
9601
10152
|
this.host = host;
|
|
9602
10153
|
if (isGitRepo(this.materializedProjectRoot)) {
|
|
@@ -9614,26 +10165,26 @@ var Indexer = class _Indexer {
|
|
|
9614
10165
|
return this.indexPathOverride ?? resolveProjectIndexPath(this.projectRoot, this.config.scope, this.host);
|
|
9615
10166
|
}
|
|
9616
10167
|
toCanonicalFilePath(filePath) {
|
|
9617
|
-
if (!
|
|
10168
|
+
if (!path19.isAbsolute(filePath)) {
|
|
9618
10169
|
return this.resolveStoredFilePath(filePath, this.projectRoot);
|
|
9619
10170
|
}
|
|
9620
|
-
if (
|
|
10171
|
+
if (path19.resolve(this.materializedProjectRoot) === path19.resolve(this.projectRoot) || !isPathWithinRoot2(filePath, this.materializedProjectRoot)) {
|
|
9621
10172
|
return filePath;
|
|
9622
10173
|
}
|
|
9623
|
-
return
|
|
10174
|
+
return path19.resolve(this.projectRoot, path19.relative(this.materializedProjectRoot, filePath));
|
|
9624
10175
|
}
|
|
9625
10176
|
toStoredFilePath(filePath) {
|
|
9626
10177
|
const canonicalFilePath = this.toCanonicalFilePath(filePath);
|
|
9627
10178
|
if (this.config.scope !== "project" || !isPathWithinRoot2(canonicalFilePath, this.projectRoot)) {
|
|
9628
10179
|
return canonicalFilePath;
|
|
9629
10180
|
}
|
|
9630
|
-
return
|
|
10181
|
+
return path19.relative(this.projectRoot, canonicalFilePath).split(path19.sep).join("/");
|
|
9631
10182
|
}
|
|
9632
10183
|
resolveStoredFilePath(filePath, rootPath = this.projectRoot) {
|
|
9633
|
-
if (
|
|
10184
|
+
if (path19.isAbsolute(filePath)) {
|
|
9634
10185
|
return filePath;
|
|
9635
10186
|
}
|
|
9636
|
-
const resolvedPath =
|
|
10187
|
+
const resolvedPath = path19.resolve(rootPath, ...filePath.split("/"));
|
|
9637
10188
|
if (!isPathWithinRoot2(resolvedPath, rootPath)) {
|
|
9638
10189
|
throw new Error(`Stored project path escapes project root: ${JSON.stringify(filePath)}`);
|
|
9639
10190
|
}
|
|
@@ -9657,7 +10208,7 @@ var Indexer = class _Indexer {
|
|
|
9657
10208
|
}
|
|
9658
10209
|
toMaterializedFilePath(filePath) {
|
|
9659
10210
|
const storedFilePath = this.toStoredFilePath(filePath);
|
|
9660
|
-
if (
|
|
10211
|
+
if (path19.isAbsolute(storedFilePath)) {
|
|
9661
10212
|
return storedFilePath;
|
|
9662
10213
|
}
|
|
9663
10214
|
return this.resolveStoredFilePath(storedFilePath, this.materializedProjectRoot);
|
|
@@ -9674,10 +10225,10 @@ var Indexer = class _Indexer {
|
|
|
9674
10225
|
}
|
|
9675
10226
|
getRuntimeArtifactPath(fileName) {
|
|
9676
10227
|
const namespace = this.getRuntimeArtifactNamespace();
|
|
9677
|
-
if (!namespace) return
|
|
9678
|
-
const extension =
|
|
10228
|
+
if (!namespace) return path19.join(this.indexPath, fileName);
|
|
10229
|
+
const extension = path19.extname(fileName);
|
|
9679
10230
|
const baseName = fileName.slice(0, fileName.length - extension.length);
|
|
9680
|
-
return
|
|
10231
|
+
return path19.join(this.indexPath, `${baseName}.${namespace}${extension}`);
|
|
9681
10232
|
}
|
|
9682
10233
|
refreshRuntimeArtifactPaths() {
|
|
9683
10234
|
this.fileHashCachePath = this.getRuntimeArtifactPath("file-hashes.json");
|
|
@@ -9690,14 +10241,14 @@ var Indexer = class _Indexer {
|
|
|
9690
10241
|
getMaterializedKnowledgeBases() {
|
|
9691
10242
|
const canonicalProjectRoot = this.getCanonicalPath(this.projectRoot);
|
|
9692
10243
|
return this.config.knowledgeBases.map((knowledgeBase) => {
|
|
9693
|
-
const configuredPath =
|
|
10244
|
+
const configuredPath = path19.isAbsolute(knowledgeBase) ? knowledgeBase : path19.resolve(this.projectRoot, knowledgeBase);
|
|
9694
10245
|
const canonicalPath = this.getCanonicalPath(configuredPath);
|
|
9695
10246
|
if (!isPathWithinRoot2(canonicalPath, canonicalProjectRoot)) {
|
|
9696
10247
|
return canonicalPath;
|
|
9697
10248
|
}
|
|
9698
|
-
return
|
|
10249
|
+
return path19.resolve(
|
|
9699
10250
|
this.materializedProjectRoot,
|
|
9700
|
-
|
|
10251
|
+
path19.relative(canonicalProjectRoot, canonicalPath)
|
|
9701
10252
|
);
|
|
9702
10253
|
});
|
|
9703
10254
|
}
|
|
@@ -9705,7 +10256,7 @@ var Indexer = class _Indexer {
|
|
|
9705
10256
|
try {
|
|
9706
10257
|
return canonicalizePathForComparison(targetPath);
|
|
9707
10258
|
} catch {
|
|
9708
|
-
return
|
|
10259
|
+
return path19.resolve(targetPath);
|
|
9709
10260
|
}
|
|
9710
10261
|
}
|
|
9711
10262
|
isProjectOwnedIndexPath() {
|
|
@@ -9781,7 +10332,7 @@ var Indexer = class _Indexer {
|
|
|
9781
10332
|
} catch (error) {
|
|
9782
10333
|
releaseError = error;
|
|
9783
10334
|
this.writerArtifactFingerprint = null;
|
|
9784
|
-
if (!
|
|
10335
|
+
if (!existsSync11(lease.lockPath) && this.activeIndexLease?.owner.token === lease.owner.token) {
|
|
9785
10336
|
this.activeIndexLease = null;
|
|
9786
10337
|
}
|
|
9787
10338
|
}
|
|
@@ -9799,11 +10350,11 @@ var Indexer = class _Indexer {
|
|
|
9799
10350
|
return this.activeIndexLease;
|
|
9800
10351
|
}
|
|
9801
10352
|
loadFileHashCache() {
|
|
9802
|
-
if (!
|
|
10353
|
+
if (!existsSync11(this.fileHashCachePath)) {
|
|
9803
10354
|
return;
|
|
9804
10355
|
}
|
|
9805
10356
|
try {
|
|
9806
|
-
const data =
|
|
10357
|
+
const data = readFileSync8(this.fileHashCachePath, "utf-8");
|
|
9807
10358
|
const parsed = JSON.parse(data);
|
|
9808
10359
|
this.fileHashCache = new Map(Object.entries(parsed));
|
|
9809
10360
|
} catch (error) {
|
|
@@ -9825,24 +10376,24 @@ var Indexer = class _Indexer {
|
|
|
9825
10376
|
atomicWriteSync(targetPath, data) {
|
|
9826
10377
|
const lease = this.requireActiveLease();
|
|
9827
10378
|
const tempPath = createLeaseTemporaryPath(targetPath, lease.owner, "tmp");
|
|
9828
|
-
|
|
10379
|
+
mkdirSync4(path19.dirname(targetPath), { recursive: true });
|
|
9829
10380
|
try {
|
|
9830
10381
|
writeFileSync3(tempPath, data);
|
|
9831
|
-
|
|
10382
|
+
renameSync3(tempPath, targetPath);
|
|
9832
10383
|
} finally {
|
|
9833
10384
|
removeLeaseTemporaryPath(tempPath);
|
|
9834
10385
|
}
|
|
9835
10386
|
}
|
|
9836
10387
|
saveInvertedIndex(invertedIndex) {
|
|
9837
10388
|
this.atomicWriteSync(
|
|
9838
|
-
|
|
10389
|
+
path19.join(this.indexPath, "inverted-index.json"),
|
|
9839
10390
|
invertedIndex.serialize()
|
|
9840
10391
|
);
|
|
9841
10392
|
}
|
|
9842
10393
|
getScopedRoots() {
|
|
9843
10394
|
const roots = /* @__PURE__ */ new Set([this.getCanonicalPath(this.projectRoot)]);
|
|
9844
10395
|
for (const kbRoot of this.config.knowledgeBases) {
|
|
9845
|
-
roots.add(this.getCanonicalPath(
|
|
10396
|
+
roots.add(this.getCanonicalPath(path19.resolve(this.projectRoot, kbRoot)));
|
|
9846
10397
|
}
|
|
9847
10398
|
return Array.from(roots);
|
|
9848
10399
|
}
|
|
@@ -9858,6 +10409,9 @@ var Indexer = class _Indexer {
|
|
|
9858
10409
|
}
|
|
9859
10410
|
return `${this.projectIdentityHash}:${branchName}`;
|
|
9860
10411
|
}
|
|
10412
|
+
resolveBranchCatalogKey(branchName) {
|
|
10413
|
+
return branchName === void 0 ? this.getBranchCatalogKey() : this.getBranchCatalogKeyFor(branchName);
|
|
10414
|
+
}
|
|
9861
10415
|
getBranchCommitMetadataKey(catalogIdentity = this.getBranchCatalogIdentity()) {
|
|
9862
10416
|
const branchKey = this.getBranchCatalogKeyFor(catalogIdentity);
|
|
9863
10417
|
return `index.branchCommit.${hashContent(branchKey).slice(0, 24)}`;
|
|
@@ -9952,13 +10506,13 @@ var Indexer = class _Indexer {
|
|
|
9952
10506
|
if (Array.from(this.fileHashCache.keys()).some((filePath) => this.isFileInCurrentScope(filePath, roots))) {
|
|
9953
10507
|
return true;
|
|
9954
10508
|
}
|
|
9955
|
-
|
|
9956
|
-
(batch
|
|
10509
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10510
|
+
if (batch.chunks.some((chunk) => {
|
|
9957
10511
|
const filePath = getPendingChunkFilePath(chunk);
|
|
9958
10512
|
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
9959
|
-
})
|
|
9960
|
-
|
|
9961
|
-
|
|
10513
|
+
})) {
|
|
10514
|
+
return true;
|
|
10515
|
+
}
|
|
9962
10516
|
}
|
|
9963
10517
|
if (!this.database) {
|
|
9964
10518
|
return false;
|
|
@@ -10099,40 +10653,25 @@ var Indexer = class _Indexer {
|
|
|
10099
10653
|
}
|
|
10100
10654
|
this.saveFileHashCache();
|
|
10101
10655
|
}
|
|
10102
|
-
partitionFailedBatches(roots, maxChunkTokens) {
|
|
10103
|
-
const scoped = [];
|
|
10104
|
-
const retained = [];
|
|
10105
|
-
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10106
|
-
const scopedChunks = batch.chunks.filter((chunk) => {
|
|
10107
|
-
const filePath = getPendingChunkFilePath(chunk);
|
|
10108
|
-
return filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
10109
|
-
});
|
|
10110
|
-
const retainedChunks = batch.chunks.filter((chunk) => {
|
|
10111
|
-
const filePath = getPendingChunkFilePath(chunk);
|
|
10112
|
-
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
10113
|
-
});
|
|
10114
|
-
if (scopedChunks.length > 0) {
|
|
10115
|
-
const normalizedBatch = normalizeFailedBatch({ ...batch, chunks: scopedChunks }, maxChunkTokens);
|
|
10116
|
-
if (normalizedBatch) {
|
|
10117
|
-
scoped.push(normalizedBatch);
|
|
10118
|
-
}
|
|
10119
|
-
}
|
|
10120
|
-
if (retainedChunks.length > 0) {
|
|
10121
|
-
retained.push({ ...batch, chunks: retainedChunks });
|
|
10122
|
-
}
|
|
10123
|
-
}
|
|
10124
|
-
return { scoped, retained };
|
|
10125
|
-
}
|
|
10126
10656
|
clearScopedFailedBatches(roots) {
|
|
10127
|
-
|
|
10128
|
-
|
|
10657
|
+
this.rewriteFailedBatchState((chunk) => {
|
|
10658
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
10659
|
+
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
10660
|
+
});
|
|
10129
10661
|
}
|
|
10130
10662
|
hasForeignScopedFileHashData(roots) {
|
|
10131
10663
|
return Array.from(this.fileHashCache.keys()).some((filePath) => !this.isFileInCurrentScope(filePath, roots));
|
|
10132
10664
|
}
|
|
10133
10665
|
hasForeignScopedFailedBatches(roots) {
|
|
10134
|
-
const
|
|
10135
|
-
|
|
10666
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10667
|
+
if (batch.chunks.some((chunk) => {
|
|
10668
|
+
const filePath = getPendingChunkFilePath(chunk);
|
|
10669
|
+
return filePath === null || !this.isFileInCurrentScope(filePath, roots);
|
|
10670
|
+
})) {
|
|
10671
|
+
return true;
|
|
10672
|
+
}
|
|
10673
|
+
}
|
|
10674
|
+
return false;
|
|
10136
10675
|
}
|
|
10137
10676
|
hasForeignScopedBranchData() {
|
|
10138
10677
|
if (!this.database || this.config.scope !== "global") {
|
|
@@ -10157,10 +10696,6 @@ var Indexer = class _Indexer {
|
|
|
10157
10696
|
}
|
|
10158
10697
|
);
|
|
10159
10698
|
}
|
|
10160
|
-
saveScopedFailedBatches(batches, roots) {
|
|
10161
|
-
const { retained } = this.partitionFailedBatches(roots);
|
|
10162
|
-
this.saveFailedBatches([...retained, ...batches]);
|
|
10163
|
-
}
|
|
10164
10699
|
clearSharedIndexProjectData(store, invertedIndex, database, roots) {
|
|
10165
10700
|
const allMetadata = store.getAllMetadata();
|
|
10166
10701
|
const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots));
|
|
@@ -10250,77 +10785,144 @@ var Indexer = class _Indexer {
|
|
|
10250
10785
|
});
|
|
10251
10786
|
}
|
|
10252
10787
|
if (this.config.scope === "global") {
|
|
10253
|
-
if (
|
|
10788
|
+
if (existsSync11(this.fileHashCachePath)) {
|
|
10254
10789
|
unlinkSync2(this.fileHashCachePath);
|
|
10255
10790
|
}
|
|
10256
10791
|
await this.healthCheckUnlocked();
|
|
10257
10792
|
}
|
|
10258
10793
|
this.logger.info("Recovery complete, next index will re-process all files");
|
|
10259
10794
|
}
|
|
10260
|
-
|
|
10261
|
-
|
|
10262
|
-
|
|
10263
|
-
|
|
10264
|
-
|
|
10795
|
+
*loadSerializedFailedBatches() {
|
|
10796
|
+
let warned = false;
|
|
10797
|
+
const warn = (error) => {
|
|
10798
|
+
if (warned) return;
|
|
10799
|
+
warned = true;
|
|
10265
10800
|
this.logger.warn("Failed to load failed batch state, skipping persisted retries", {
|
|
10266
10801
|
failedBatchesPath: this.failedBatchesPath,
|
|
10267
|
-
error:
|
|
10802
|
+
error: getErrorMessage4(error)
|
|
10268
10803
|
});
|
|
10269
|
-
|
|
10804
|
+
};
|
|
10805
|
+
try {
|
|
10806
|
+
for (const record of readFailedBatchRecords(this.failedBatchesPath, {
|
|
10807
|
+
malformedLineAction: "skip",
|
|
10808
|
+
onMalformedLine: (error) => warn(error)
|
|
10809
|
+
})) {
|
|
10810
|
+
yield {
|
|
10811
|
+
chunks: record.chunks,
|
|
10812
|
+
error: record.error,
|
|
10813
|
+
attemptCount: record.attemptCount,
|
|
10814
|
+
lastAttempt: record.lastAttempt
|
|
10815
|
+
};
|
|
10816
|
+
}
|
|
10817
|
+
} catch (error) {
|
|
10818
|
+
warn(error);
|
|
10270
10819
|
}
|
|
10271
10820
|
}
|
|
10272
|
-
|
|
10273
|
-
|
|
10274
|
-
|
|
10821
|
+
createFailedBatchWriteState() {
|
|
10822
|
+
return {
|
|
10823
|
+
writer: createFailedBatchWriter(this.failedBatchesPath),
|
|
10824
|
+
recordsWritten: 0
|
|
10825
|
+
};
|
|
10826
|
+
}
|
|
10827
|
+
writeFailedBatchRecord(state, record) {
|
|
10828
|
+
state.writer.write(record);
|
|
10829
|
+
state.recordsWritten += record.chunks.length;
|
|
10830
|
+
}
|
|
10831
|
+
finalizeFailedBatchWriteState(state) {
|
|
10832
|
+
if (state.recordsWritten > 0) {
|
|
10833
|
+
state.writer.commit();
|
|
10834
|
+
return;
|
|
10275
10835
|
}
|
|
10276
|
-
|
|
10277
|
-
|
|
10278
|
-
|
|
10279
|
-
|
|
10280
|
-
|
|
10281
|
-
|
|
10836
|
+
state.writer.cleanup();
|
|
10837
|
+
this.clearFailedBatchState();
|
|
10838
|
+
}
|
|
10839
|
+
clearFailedBatchState() {
|
|
10840
|
+
if (existsSync11(this.failedBatchesPath)) {
|
|
10841
|
+
try {
|
|
10842
|
+
unlinkSync2(this.failedBatchesPath);
|
|
10843
|
+
} catch {
|
|
10282
10844
|
}
|
|
10283
|
-
|
|
10284
|
-
chunks,
|
|
10285
|
-
error: typeof batch.error === "string" ? batch.error : "Unknown embedding error",
|
|
10286
|
-
attemptCount: typeof batch.attemptCount === "number" ? batch.attemptCount : 1,
|
|
10287
|
-
lastAttempt: typeof batch.lastAttempt === "string" ? batch.lastAttempt : (/* @__PURE__ */ new Date()).toISOString()
|
|
10288
|
-
};
|
|
10289
|
-
}).filter((batch) => batch !== null);
|
|
10845
|
+
}
|
|
10290
10846
|
}
|
|
10291
|
-
|
|
10292
|
-
|
|
10293
|
-
|
|
10294
|
-
|
|
10295
|
-
|
|
10296
|
-
|
|
10847
|
+
rewriteFailedBatchState(shouldRetain) {
|
|
10848
|
+
const state = this.createFailedBatchWriteState();
|
|
10849
|
+
try {
|
|
10850
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10851
|
+
const retainedChunks = batch.chunks.filter(shouldRetain);
|
|
10852
|
+
if (retainedChunks.length > 0) {
|
|
10853
|
+
this.writeFailedBatchRecord(state, { ...batch, chunks: retainedChunks });
|
|
10297
10854
|
}
|
|
10298
10855
|
}
|
|
10299
|
-
|
|
10856
|
+
this.finalizeFailedBatchWriteState(state);
|
|
10857
|
+
} catch (error) {
|
|
10858
|
+
state.writer.cleanup();
|
|
10859
|
+
throw error;
|
|
10860
|
+
}
|
|
10861
|
+
}
|
|
10862
|
+
prepareFailedBatchProcessing(roots, shouldProcess) {
|
|
10863
|
+
const state = this.createFailedBatchWriteState();
|
|
10864
|
+
const latestById = /* @__PURE__ */ new Map();
|
|
10865
|
+
try {
|
|
10866
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10867
|
+
for (const rawChunk of batch.chunks) {
|
|
10868
|
+
const filePath = getPendingChunkFilePath(rawChunk);
|
|
10869
|
+
const inScope = roots === null || filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
10870
|
+
if (!inScope) {
|
|
10871
|
+
this.writeFailedBatchRecord(state, { ...batch, chunks: [rawChunk] });
|
|
10872
|
+
continue;
|
|
10873
|
+
}
|
|
10874
|
+
if (!shouldProcess(filePath)) {
|
|
10875
|
+
continue;
|
|
10876
|
+
}
|
|
10877
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
10878
|
+
if (!chunkId) {
|
|
10879
|
+
continue;
|
|
10880
|
+
}
|
|
10881
|
+
const existing = latestById.get(chunkId);
|
|
10882
|
+
if (!existing || batch.attemptCount >= existing.attemptCount) {
|
|
10883
|
+
latestById.set(chunkId, {
|
|
10884
|
+
attemptCount: batch.attemptCount,
|
|
10885
|
+
error: batch.error,
|
|
10886
|
+
lastAttempt: batch.lastAttempt
|
|
10887
|
+
});
|
|
10888
|
+
}
|
|
10889
|
+
}
|
|
10890
|
+
}
|
|
10891
|
+
return { state, latestById };
|
|
10892
|
+
} catch (error) {
|
|
10893
|
+
state.writer.cleanup();
|
|
10894
|
+
throw error;
|
|
10300
10895
|
}
|
|
10301
|
-
this.atomicWriteSync(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
10302
10896
|
}
|
|
10303
|
-
|
|
10304
|
-
const
|
|
10305
|
-
for (const batch of this.
|
|
10306
|
-
for (const
|
|
10307
|
-
const
|
|
10308
|
-
if (!
|
|
10897
|
+
*iterateLatestFailedChunks(latestById, roots, shouldProcess, maxChunkTokens) {
|
|
10898
|
+
const yielded = /* @__PURE__ */ new Set();
|
|
10899
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
10900
|
+
for (const rawChunk of batch.chunks) {
|
|
10901
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
10902
|
+
if (!chunkId || yielded.has(chunkId)) {
|
|
10309
10903
|
continue;
|
|
10310
10904
|
}
|
|
10311
|
-
|
|
10905
|
+
const latest = latestById.get(chunkId);
|
|
10906
|
+
if (!latest || latest.attemptCount !== batch.attemptCount || latest.error !== batch.error || latest.lastAttempt !== batch.lastAttempt) {
|
|
10312
10907
|
continue;
|
|
10313
10908
|
}
|
|
10314
|
-
const
|
|
10315
|
-
|
|
10316
|
-
|
|
10317
|
-
|
|
10318
|
-
|
|
10319
|
-
|
|
10909
|
+
const filePath = getPendingChunkFilePath(rawChunk);
|
|
10910
|
+
const inScope = roots === null || filePath !== null && this.isFileInCurrentScope(filePath, roots);
|
|
10911
|
+
if (!inScope || !shouldProcess(filePath)) {
|
|
10912
|
+
continue;
|
|
10913
|
+
}
|
|
10914
|
+
const normalized = normalizeFailedBatch({ ...batch, chunks: [rawChunk] }, maxChunkTokens);
|
|
10915
|
+
const chunk = normalized?.chunks[0];
|
|
10916
|
+
if (!chunk) {
|
|
10917
|
+
continue;
|
|
10320
10918
|
}
|
|
10919
|
+
yielded.add(chunkId);
|
|
10920
|
+
yield {
|
|
10921
|
+
chunk,
|
|
10922
|
+
attemptCount: batch.attemptCount
|
|
10923
|
+
};
|
|
10321
10924
|
}
|
|
10322
10925
|
}
|
|
10323
|
-
return Array.from(retryableById.values());
|
|
10324
10926
|
}
|
|
10325
10927
|
getProviderRateLimits(provider) {
|
|
10326
10928
|
switch (provider) {
|
|
@@ -10345,31 +10947,267 @@ var Indexer = class _Indexer {
|
|
|
10345
10947
|
return { concurrency: 3, intervalMs: 1e3, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
10346
10948
|
}
|
|
10347
10949
|
}
|
|
10348
|
-
async
|
|
10349
|
-
const
|
|
10350
|
-
|
|
10351
|
-
|
|
10950
|
+
async processPendingChunkBatch(chunks, options) {
|
|
10951
|
+
const result = {
|
|
10952
|
+
indexedChunks: 0,
|
|
10953
|
+
failedChunks: 0,
|
|
10954
|
+
tokensUsed: 0,
|
|
10955
|
+
failedChunkIds: /* @__PURE__ */ new Set()
|
|
10956
|
+
};
|
|
10957
|
+
if (chunks.length === 0) {
|
|
10958
|
+
return result;
|
|
10352
10959
|
}
|
|
10353
|
-
const
|
|
10354
|
-
|
|
10355
|
-
|
|
10356
|
-
|
|
10357
|
-
|
|
10960
|
+
const chunksNeedingEmbedding = [];
|
|
10961
|
+
let cachedChunkCount = 0;
|
|
10962
|
+
if (options.reuseCachedEmbeddings && !options.forceReembed) {
|
|
10963
|
+
const missingHashes = new Set(options.database.getMissingEmbeddings(chunks.map((chunk) => chunk.contentHash)));
|
|
10964
|
+
for (const chunk of chunks) {
|
|
10965
|
+
if (missingHashes.has(chunk.contentHash)) {
|
|
10966
|
+
chunksNeedingEmbedding.push(chunk);
|
|
10967
|
+
continue;
|
|
10968
|
+
}
|
|
10969
|
+
const embeddingBuffer = options.database.getEmbedding(chunk.contentHash);
|
|
10970
|
+
if (!embeddingBuffer) {
|
|
10971
|
+
chunksNeedingEmbedding.push(chunk);
|
|
10972
|
+
continue;
|
|
10973
|
+
}
|
|
10974
|
+
options.store.add(chunk.id, Array.from(bufferToFloat32Array(embeddingBuffer)), chunk.metadata);
|
|
10975
|
+
options.invertedIndex.removeChunk(chunk.id);
|
|
10976
|
+
options.invertedIndex.addChunk(chunk.id, chunk.content);
|
|
10977
|
+
options.onSucceeded?.([chunk]);
|
|
10978
|
+
result.indexedChunks += 1;
|
|
10979
|
+
cachedChunkCount += 1;
|
|
10980
|
+
}
|
|
10981
|
+
} else {
|
|
10982
|
+
chunksNeedingEmbedding.push(...chunks);
|
|
10358
10983
|
}
|
|
10359
|
-
|
|
10360
|
-
|
|
10984
|
+
this.logger.cache("info", "Embedding cache lookup", {
|
|
10985
|
+
needsEmbedding: chunksNeedingEmbedding.length,
|
|
10986
|
+
fromCache: cachedChunkCount
|
|
10987
|
+
});
|
|
10988
|
+
if (cachedChunkCount > 0) {
|
|
10989
|
+
this.logger.recordChunksFromCache(cachedChunkCount);
|
|
10990
|
+
options.onProgress?.(result);
|
|
10361
10991
|
}
|
|
10362
|
-
|
|
10363
|
-
|
|
10364
|
-
|
|
10365
|
-
const
|
|
10366
|
-
|
|
10367
|
-
|
|
10368
|
-
|
|
10369
|
-
|
|
10370
|
-
|
|
10371
|
-
|
|
10372
|
-
|
|
10992
|
+
if (chunksNeedingEmbedding.length === 0) {
|
|
10993
|
+
return result;
|
|
10994
|
+
}
|
|
10995
|
+
const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk]));
|
|
10996
|
+
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
10997
|
+
const completedVectorsByChunkId = /* @__PURE__ */ new Map();
|
|
10998
|
+
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
10999
|
+
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
11000
|
+
chunksNeedingEmbedding,
|
|
11001
|
+
getDynamicBatchOptions(options.configuredProviderInfo)
|
|
11002
|
+
);
|
|
11003
|
+
let fatalError;
|
|
11004
|
+
for (const requestBatch of requestBatches) {
|
|
11005
|
+
await options.queue.onSizeLessThan(Math.max(1, options.providerRateLimits.concurrency));
|
|
11006
|
+
const task = options.queue.add(async () => {
|
|
11007
|
+
if (options.rateLimitState.backoffMs > 0) {
|
|
11008
|
+
await new Promise((resolve15) => setTimeout(resolve15, options.rateLimitState.backoffMs));
|
|
11009
|
+
}
|
|
11010
|
+
try {
|
|
11011
|
+
const embeddingResult = await pRetry(
|
|
11012
|
+
async () => {
|
|
11013
|
+
const texts = requestBatch.map((request) => request.text);
|
|
11014
|
+
return options.provider.embedBatch(texts);
|
|
11015
|
+
},
|
|
11016
|
+
{
|
|
11017
|
+
retries: this.config.indexing.retries,
|
|
11018
|
+
minTimeout: Math.max(this.config.indexing.retryDelayMs, options.providerRateLimits.minRetryMs),
|
|
11019
|
+
maxTimeout: options.providerRateLimits.maxRetryMs,
|
|
11020
|
+
factor: 2,
|
|
11021
|
+
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError),
|
|
11022
|
+
onFailedAttempt: (error) => {
|
|
11023
|
+
const message = getErrorMessage4(error);
|
|
11024
|
+
if (isRateLimitError(error)) {
|
|
11025
|
+
options.rateLimitState.backoffMs = Math.min(
|
|
11026
|
+
options.providerRateLimits.maxRetryMs,
|
|
11027
|
+
(options.rateLimitState.backoffMs || options.providerRateLimits.minRetryMs) * 2
|
|
11028
|
+
);
|
|
11029
|
+
this.logger.embedding("warn", "Rate limited, backing off", {
|
|
11030
|
+
attempt: error.attemptNumber,
|
|
11031
|
+
retriesLeft: error.retriesLeft,
|
|
11032
|
+
backoffMs: options.rateLimitState.backoffMs
|
|
11033
|
+
});
|
|
11034
|
+
} else {
|
|
11035
|
+
this.logger.embedding("error", "Embedding batch failed", {
|
|
11036
|
+
attempt: error.attemptNumber,
|
|
11037
|
+
error: message
|
|
11038
|
+
});
|
|
11039
|
+
}
|
|
11040
|
+
}
|
|
11041
|
+
}
|
|
11042
|
+
);
|
|
11043
|
+
if (options.rateLimitState.backoffMs > 0) {
|
|
11044
|
+
options.rateLimitState.backoffMs = Math.max(0, options.rateLimitState.backoffMs - 2e3);
|
|
11045
|
+
}
|
|
11046
|
+
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
11047
|
+
requestBatch.forEach((request, index) => {
|
|
11048
|
+
if (result.failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
11049
|
+
return;
|
|
11050
|
+
}
|
|
11051
|
+
const vector = embeddingResult.embeddings[index];
|
|
11052
|
+
if (!vector) {
|
|
11053
|
+
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
11054
|
+
}
|
|
11055
|
+
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
11056
|
+
parts[request.partIndex] = {
|
|
11057
|
+
vector,
|
|
11058
|
+
tokenCount: request.tokenCount
|
|
11059
|
+
};
|
|
11060
|
+
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
11061
|
+
touchedChunkIds.add(request.chunk.id);
|
|
11062
|
+
});
|
|
11063
|
+
const pooledResults = [];
|
|
11064
|
+
for (const chunkId of touchedChunkIds) {
|
|
11065
|
+
if (result.failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
11066
|
+
continue;
|
|
11067
|
+
}
|
|
11068
|
+
const chunk = pendingChunksById.get(chunkId);
|
|
11069
|
+
if (!chunk) {
|
|
11070
|
+
continue;
|
|
11071
|
+
}
|
|
11072
|
+
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
11073
|
+
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
11074
|
+
continue;
|
|
11075
|
+
}
|
|
11076
|
+
const orderedParts = parts;
|
|
11077
|
+
pooledResults.push({
|
|
11078
|
+
chunk,
|
|
11079
|
+
vector: poolEmbeddingVectors(
|
|
11080
|
+
orderedParts.map((part) => part.vector),
|
|
11081
|
+
orderedParts.map((part) => part.tokenCount)
|
|
11082
|
+
)
|
|
11083
|
+
});
|
|
11084
|
+
}
|
|
11085
|
+
if (pooledResults.length > 0) {
|
|
11086
|
+
options.database.upsertEmbeddingsBatch(pooledResults.map(({ chunk, vector }) => ({
|
|
11087
|
+
contentHash: chunk.contentHash,
|
|
11088
|
+
embedding: float32ArrayToBuffer(vector),
|
|
11089
|
+
chunkText: chunk.storageText,
|
|
11090
|
+
model: options.configuredProviderInfo.modelInfo.model
|
|
11091
|
+
})));
|
|
11092
|
+
const succeededChunks = pooledResults.map(({ chunk }) => chunk);
|
|
11093
|
+
for (const { chunk, vector } of pooledResults) {
|
|
11094
|
+
completedVectorsByChunkId.set(chunk.id, vector);
|
|
11095
|
+
}
|
|
11096
|
+
for (const chunk of succeededChunks) {
|
|
11097
|
+
completedChunkIds.add(chunk.id);
|
|
11098
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
11099
|
+
}
|
|
11100
|
+
}
|
|
11101
|
+
result.tokensUsed += embeddingResult.totalTokensUsed;
|
|
11102
|
+
this.logger.recordEmbeddingApiCall(embeddingResult.totalTokensUsed);
|
|
11103
|
+
this.logger.embedding("debug", "Embedded batch", {
|
|
11104
|
+
batchSize: pooledResults.length,
|
|
11105
|
+
requestCount: requestBatch.length,
|
|
11106
|
+
tokens: embeddingResult.totalTokensUsed
|
|
11107
|
+
});
|
|
11108
|
+
} catch (error) {
|
|
11109
|
+
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id)).filter((chunk) => options.incrementRepeatedFailures || !result.failedChunkIds.has(chunk.id));
|
|
11110
|
+
const failureMessage = getErrorMessage4(error);
|
|
11111
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
11112
|
+
for (const chunk of failedChunks) {
|
|
11113
|
+
if (!result.failedChunkIds.has(chunk.id)) {
|
|
11114
|
+
result.failedChunkIds.add(chunk.id);
|
|
11115
|
+
result.failedChunks += 1;
|
|
11116
|
+
}
|
|
11117
|
+
embeddingPartsByChunk.delete(chunk.id);
|
|
11118
|
+
const attemptCount = (options.attemptCounts.get(chunk.id) ?? 0) + 1;
|
|
11119
|
+
options.attemptCounts.set(chunk.id, attemptCount);
|
|
11120
|
+
this.writeFailedBatchRecord(options.failedState, {
|
|
11121
|
+
chunks: [chunk],
|
|
11122
|
+
error: failureMessage,
|
|
11123
|
+
attemptCount,
|
|
11124
|
+
lastAttempt: failureTimestamp
|
|
11125
|
+
});
|
|
11126
|
+
}
|
|
11127
|
+
this.logger.recordEmbeddingError();
|
|
11128
|
+
this.logger.embedding("error", "Failed to embed batch after retries", {
|
|
11129
|
+
batchSize: failedChunks.length,
|
|
11130
|
+
requestCount: requestBatch.length,
|
|
11131
|
+
error: failureMessage
|
|
11132
|
+
});
|
|
11133
|
+
}
|
|
11134
|
+
options.onProgress?.(result);
|
|
11135
|
+
});
|
|
11136
|
+
void task.catch((error) => {
|
|
11137
|
+
fatalError ??= error;
|
|
11138
|
+
});
|
|
11139
|
+
}
|
|
11140
|
+
await options.queue.onIdle();
|
|
11141
|
+
if (fatalError !== void 0) {
|
|
11142
|
+
throw fatalError;
|
|
11143
|
+
}
|
|
11144
|
+
const orderedSucceededChunks = chunksNeedingEmbedding.filter((chunk) => completedVectorsByChunkId.has(chunk.id));
|
|
11145
|
+
if (orderedSucceededChunks.length > 0) {
|
|
11146
|
+
try {
|
|
11147
|
+
options.store.addBatch(orderedSucceededChunks.map((chunk) => ({
|
|
11148
|
+
id: chunk.id,
|
|
11149
|
+
vector: completedVectorsByChunkId.get(chunk.id),
|
|
11150
|
+
metadata: chunk.metadata
|
|
11151
|
+
})));
|
|
11152
|
+
for (const chunk of orderedSucceededChunks) {
|
|
11153
|
+
options.invertedIndex.removeChunk(chunk.id);
|
|
11154
|
+
options.invertedIndex.addChunk(chunk.id, chunk.content);
|
|
11155
|
+
}
|
|
11156
|
+
options.onSucceeded?.(orderedSucceededChunks);
|
|
11157
|
+
result.indexedChunks += orderedSucceededChunks.length;
|
|
11158
|
+
this.logger.recordChunksEmbedded(orderedSucceededChunks.length);
|
|
11159
|
+
} catch (error) {
|
|
11160
|
+
const failureMessage = getErrorMessage4(error);
|
|
11161
|
+
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
11162
|
+
for (const chunk of orderedSucceededChunks) {
|
|
11163
|
+
options.store.remove(chunk.id);
|
|
11164
|
+
options.invertedIndex.removeChunk(chunk.id);
|
|
11165
|
+
result.failedChunkIds.add(chunk.id);
|
|
11166
|
+
result.failedChunks += 1;
|
|
11167
|
+
const attemptCount = (options.attemptCounts.get(chunk.id) ?? 0) + 1;
|
|
11168
|
+
options.attemptCounts.set(chunk.id, attemptCount);
|
|
11169
|
+
this.writeFailedBatchRecord(options.failedState, {
|
|
11170
|
+
chunks: [chunk],
|
|
11171
|
+
error: failureMessage,
|
|
11172
|
+
attemptCount,
|
|
11173
|
+
lastAttempt: failureTimestamp
|
|
11174
|
+
});
|
|
11175
|
+
}
|
|
11176
|
+
this.logger.recordEmbeddingError();
|
|
11177
|
+
this.logger.embedding("error", "Failed to publish embedded chunks", {
|
|
11178
|
+
batchSize: orderedSucceededChunks.length,
|
|
11179
|
+
error: failureMessage
|
|
11180
|
+
});
|
|
11181
|
+
}
|
|
11182
|
+
options.onProgress?.(result);
|
|
11183
|
+
}
|
|
11184
|
+
return result;
|
|
11185
|
+
}
|
|
11186
|
+
async rerankCandidatesWithApi(query, candidates, options) {
|
|
11187
|
+
const reranker = this.config.reranker;
|
|
11188
|
+
if (!reranker || !reranker.enabled || candidates.length <= 1) {
|
|
11189
|
+
return candidates;
|
|
11190
|
+
}
|
|
11191
|
+
const queryIntent = analyzeQueryIntent(query);
|
|
11192
|
+
const preferSourcePaths = queryIntent.preferSourcePaths;
|
|
11193
|
+
const docIntent = queryIntent.primary === "docs";
|
|
11194
|
+
if (options?.definitionIntent === true) {
|
|
11195
|
+
return candidates;
|
|
11196
|
+
}
|
|
11197
|
+
if (options?.hasIdentifierHints === true && preferSourcePaths && !docIntent) {
|
|
11198
|
+
return candidates;
|
|
11199
|
+
}
|
|
11200
|
+
const topN = Math.min(reranker.topN, candidates.length);
|
|
11201
|
+
const head = candidates.slice(0, topN);
|
|
11202
|
+
const tail = candidates.slice(topN);
|
|
11203
|
+
const grouped = /* @__PURE__ */ new Map([
|
|
11204
|
+
["implementation", []],
|
|
11205
|
+
["documentation", []],
|
|
11206
|
+
["test", []],
|
|
11207
|
+
["config", []],
|
|
11208
|
+
["other", []]
|
|
11209
|
+
]);
|
|
11210
|
+
for (const candidate of head) {
|
|
10373
11211
|
const band = classifyExternalRerankBand(candidate, queryIntent);
|
|
10374
11212
|
grouped.get(band)?.push(candidate);
|
|
10375
11213
|
}
|
|
@@ -10586,12 +11424,12 @@ var Indexer = class _Indexer {
|
|
|
10586
11424
|
}
|
|
10587
11425
|
}
|
|
10588
11426
|
captureReaderArtifactFingerprint() {
|
|
10589
|
-
const storePath =
|
|
11427
|
+
const storePath = path19.join(this.indexPath, "vectors");
|
|
10590
11428
|
return {
|
|
10591
11429
|
vectors: `${this.getReaderFileFingerprint(storePath)}|${this.getReaderFileFingerprint(`${storePath}.meta.json`)}`,
|
|
10592
|
-
keyword: this.getReaderFileFingerprint(
|
|
10593
|
-
database: this.getReaderFileFingerprint(
|
|
10594
|
-
databaseIdentity: this.getReaderFileFingerprint(
|
|
11430
|
+
keyword: this.getReaderFileFingerprint(path19.join(this.indexPath, "inverted-index.json")),
|
|
11431
|
+
database: this.getReaderFileFingerprint(path19.join(this.indexPath, "codebase.db")),
|
|
11432
|
+
databaseIdentity: this.getReaderFileFingerprint(path19.join(this.indexPath, "codebase.db"), true)
|
|
10595
11433
|
};
|
|
10596
11434
|
}
|
|
10597
11435
|
refreshReaderArtifacts() {
|
|
@@ -10616,13 +11454,13 @@ var Indexer = class _Indexer {
|
|
|
10616
11454
|
issues.set(component, this.createReadIssue(component, message));
|
|
10617
11455
|
this.readerArtifactRetryAfter.set(component, Date.now() + READER_ARTIFACT_RETRY_INTERVAL_MS);
|
|
10618
11456
|
};
|
|
10619
|
-
const storePath =
|
|
11457
|
+
const storePath = path19.join(this.indexPath, "vectors");
|
|
10620
11458
|
const vectorMetadataPath = `${storePath}.meta.json`;
|
|
10621
|
-
const invertedIndexPath =
|
|
10622
|
-
const dbPath =
|
|
11459
|
+
const invertedIndexPath = path19.join(this.indexPath, "inverted-index.json");
|
|
11460
|
+
const dbPath = path19.join(this.indexPath, "codebase.db");
|
|
10623
11461
|
if (vectorsChanged || retryDue("vectors")) {
|
|
10624
|
-
const vectorStoreExists =
|
|
10625
|
-
const vectorMetadataExists =
|
|
11462
|
+
const vectorStoreExists = existsSync11(storePath);
|
|
11463
|
+
const vectorMetadataExists = existsSync11(vectorMetadataPath);
|
|
10626
11464
|
if (vectorStoreExists && vectorMetadataExists) {
|
|
10627
11465
|
try {
|
|
10628
11466
|
const store = new VectorStore(storePath, this.configuredProviderInfo.modelInfo.dimensions);
|
|
@@ -10637,8 +11475,8 @@ var Indexer = class _Indexer {
|
|
|
10637
11475
|
setIssue("vectors", this.getVectorReadIssueMessage());
|
|
10638
11476
|
}
|
|
10639
11477
|
}
|
|
10640
|
-
if (keywordChanged || retryDue("keyword") || !
|
|
10641
|
-
if (
|
|
11478
|
+
if (keywordChanged || retryDue("keyword") || !existsSync11(invertedIndexPath) && (this.store?.count() ?? 0) > 0) {
|
|
11479
|
+
if (existsSync11(invertedIndexPath)) {
|
|
10642
11480
|
try {
|
|
10643
11481
|
const invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
10644
11482
|
invertedIndex.load();
|
|
@@ -10653,7 +11491,7 @@ var Indexer = class _Indexer {
|
|
|
10653
11491
|
}
|
|
10654
11492
|
}
|
|
10655
11493
|
if (databaseReplaced || databaseChanged && issues.has("database") || retryDue("database")) {
|
|
10656
|
-
if (
|
|
11494
|
+
if (existsSync11(dbPath)) {
|
|
10657
11495
|
try {
|
|
10658
11496
|
const database = Database.openReadOnly(dbPath);
|
|
10659
11497
|
if (this.database) {
|
|
@@ -10744,11 +11582,11 @@ var Indexer = class _Indexer {
|
|
|
10744
11582
|
}
|
|
10745
11583
|
}
|
|
10746
11584
|
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
10747
|
-
const storePath =
|
|
11585
|
+
const storePath = path19.join(this.indexPath, "vectors");
|
|
10748
11586
|
const vectorMetadataPath = `${storePath}.meta.json`;
|
|
10749
|
-
const invertedIndexPath =
|
|
10750
|
-
const dbPath =
|
|
10751
|
-
let dbIsNew = !
|
|
11587
|
+
const invertedIndexPath = path19.join(this.indexPath, "inverted-index.json");
|
|
11588
|
+
const dbPath = path19.join(this.indexPath, "codebase.db");
|
|
11589
|
+
let dbIsNew = !existsSync11(dbPath);
|
|
10752
11590
|
const readerArtifactFingerprint = mode === "reader" ? this.captureReaderArtifactFingerprint() : null;
|
|
10753
11591
|
if (mode === "writer") {
|
|
10754
11592
|
await fsPromises3.mkdir(this.indexPath, { recursive: true });
|
|
@@ -10767,14 +11605,14 @@ var Indexer = class _Indexer {
|
|
|
10767
11605
|
await this.resetLocalIndexArtifacts();
|
|
10768
11606
|
}
|
|
10769
11607
|
this.store = new VectorStore(storePath, dimensions);
|
|
10770
|
-
if (
|
|
11608
|
+
if (existsSync11(storePath) || existsSync11(vectorMetadataPath)) {
|
|
10771
11609
|
this.store.load();
|
|
10772
11610
|
}
|
|
10773
11611
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
10774
11612
|
try {
|
|
10775
11613
|
this.invertedIndex.load();
|
|
10776
11614
|
} catch {
|
|
10777
|
-
if (
|
|
11615
|
+
if (existsSync11(invertedIndexPath)) {
|
|
10778
11616
|
await fsPromises3.unlink(invertedIndexPath);
|
|
10779
11617
|
}
|
|
10780
11618
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
@@ -10792,8 +11630,8 @@ var Indexer = class _Indexer {
|
|
|
10792
11630
|
}
|
|
10793
11631
|
} else {
|
|
10794
11632
|
this.store = new VectorStore(storePath, dimensions);
|
|
10795
|
-
const vectorStoreExists =
|
|
10796
|
-
const vectorMetadataExists =
|
|
11633
|
+
const vectorStoreExists = existsSync11(storePath);
|
|
11634
|
+
const vectorMetadataExists = existsSync11(vectorMetadataPath);
|
|
10797
11635
|
const vectorReadFailureMessage = this.getVectorReadIssueMessage();
|
|
10798
11636
|
if (vectorStoreExists !== vectorMetadataExists) {
|
|
10799
11637
|
this.recordReadIssue("vectors", vectorReadFailureMessage);
|
|
@@ -10806,7 +11644,7 @@ var Indexer = class _Indexer {
|
|
|
10806
11644
|
}
|
|
10807
11645
|
}
|
|
10808
11646
|
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
10809
|
-
if (
|
|
11647
|
+
if (existsSync11(invertedIndexPath)) {
|
|
10810
11648
|
try {
|
|
10811
11649
|
this.invertedIndex.load();
|
|
10812
11650
|
} catch (error) {
|
|
@@ -10820,7 +11658,7 @@ var Indexer = class _Indexer {
|
|
|
10820
11658
|
} else if (this.store.count() > 0) {
|
|
10821
11659
|
this.recordReadIssue("keyword", this.getKeywordReadIssueMessage());
|
|
10822
11660
|
}
|
|
10823
|
-
if (
|
|
11661
|
+
if (existsSync11(dbPath)) {
|
|
10824
11662
|
try {
|
|
10825
11663
|
this.database = Database.openReadOnly(dbPath);
|
|
10826
11664
|
} catch (error) {
|
|
@@ -10912,7 +11750,7 @@ var Indexer = class _Indexer {
|
|
|
10912
11750
|
if (await this.tryResetCorruptedIndex("running automatic orphan garbage collection", error)) {
|
|
10913
11751
|
return {
|
|
10914
11752
|
resetCorruptedIndex: true,
|
|
10915
|
-
warning: this.getCorruptedIndexWarning(
|
|
11753
|
+
warning: this.getCorruptedIndexWarning(path19.join(this.indexPath, "codebase.db"))
|
|
10916
11754
|
};
|
|
10917
11755
|
}
|
|
10918
11756
|
throw error;
|
|
@@ -10927,7 +11765,7 @@ var Indexer = class _Indexer {
|
|
|
10927
11765
|
return;
|
|
10928
11766
|
}
|
|
10929
11767
|
const retainedEntries = store.getAllMetadata().filter(({ key }) => !excludedSet.has(key));
|
|
10930
|
-
const storeBasePath =
|
|
11768
|
+
const storeBasePath = path19.join(this.indexPath, "vectors");
|
|
10931
11769
|
const storeIndexPath = storeBasePath;
|
|
10932
11770
|
const storeMetadataPath = `${storeBasePath}.meta.json`;
|
|
10933
11771
|
const lease = this.requireActiveLease();
|
|
@@ -10937,19 +11775,19 @@ var Indexer = class _Indexer {
|
|
|
10937
11775
|
let backedUpMetadata = false;
|
|
10938
11776
|
let rebuiltCount = 0;
|
|
10939
11777
|
let skippedCount = 0;
|
|
10940
|
-
if (
|
|
11778
|
+
if (existsSync11(backupIndexPath)) {
|
|
10941
11779
|
unlinkSync2(backupIndexPath);
|
|
10942
11780
|
}
|
|
10943
|
-
if (
|
|
11781
|
+
if (existsSync11(backupMetadataPath)) {
|
|
10944
11782
|
unlinkSync2(backupMetadataPath);
|
|
10945
11783
|
}
|
|
10946
11784
|
try {
|
|
10947
|
-
if (
|
|
10948
|
-
|
|
11785
|
+
if (existsSync11(storeIndexPath)) {
|
|
11786
|
+
renameSync3(storeIndexPath, backupIndexPath);
|
|
10949
11787
|
backedUpIndex = true;
|
|
10950
11788
|
}
|
|
10951
|
-
if (
|
|
10952
|
-
|
|
11789
|
+
if (existsSync11(storeMetadataPath)) {
|
|
11790
|
+
renameSync3(storeMetadataPath, backupMetadataPath);
|
|
10953
11791
|
backedUpMetadata = true;
|
|
10954
11792
|
}
|
|
10955
11793
|
store.clear();
|
|
@@ -10969,10 +11807,10 @@ var Indexer = class _Indexer {
|
|
|
10969
11807
|
rebuiltCount += 1;
|
|
10970
11808
|
}
|
|
10971
11809
|
store.save();
|
|
10972
|
-
if (backedUpIndex &&
|
|
11810
|
+
if (backedUpIndex && existsSync11(backupIndexPath)) {
|
|
10973
11811
|
unlinkSync2(backupIndexPath);
|
|
10974
11812
|
}
|
|
10975
|
-
if (backedUpMetadata &&
|
|
11813
|
+
if (backedUpMetadata && existsSync11(backupMetadataPath)) {
|
|
10976
11814
|
unlinkSync2(backupMetadataPath);
|
|
10977
11815
|
}
|
|
10978
11816
|
this.logger.gc("info", "Rebuilt vector store to avoid native remove", {
|
|
@@ -10985,17 +11823,17 @@ var Indexer = class _Indexer {
|
|
|
10985
11823
|
store.clear();
|
|
10986
11824
|
} catch {
|
|
10987
11825
|
}
|
|
10988
|
-
if (
|
|
11826
|
+
if (existsSync11(storeIndexPath)) {
|
|
10989
11827
|
unlinkSync2(storeIndexPath);
|
|
10990
11828
|
}
|
|
10991
|
-
if (
|
|
11829
|
+
if (existsSync11(storeMetadataPath)) {
|
|
10992
11830
|
unlinkSync2(storeMetadataPath);
|
|
10993
11831
|
}
|
|
10994
|
-
if (backedUpIndex &&
|
|
10995
|
-
|
|
11832
|
+
if (backedUpIndex && existsSync11(backupIndexPath)) {
|
|
11833
|
+
renameSync3(backupIndexPath, storeIndexPath);
|
|
10996
11834
|
}
|
|
10997
|
-
if (backedUpMetadata &&
|
|
10998
|
-
|
|
11835
|
+
if (backedUpMetadata && existsSync11(backupMetadataPath)) {
|
|
11836
|
+
renameSync3(backupMetadataPath, storeMetadataPath);
|
|
10999
11837
|
}
|
|
11000
11838
|
if (backedUpIndex || backedUpMetadata) {
|
|
11001
11839
|
store.load();
|
|
@@ -11010,11 +11848,11 @@ var Indexer = class _Indexer {
|
|
|
11010
11848
|
return `Detected a corrupted local SQLite index at ${dbPath} and reset the local index. Run index_codebase to rebuild search data.`;
|
|
11011
11849
|
}
|
|
11012
11850
|
async removeProjectRuntimeStateArtifacts() {
|
|
11013
|
-
if (!
|
|
11851
|
+
if (!existsSync11(this.indexPath)) return;
|
|
11014
11852
|
const names = await fsPromises3.readdir(this.indexPath);
|
|
11015
11853
|
const runtimeStatePattern = /^(?:file-hashes|failed-batches)(?:\.[a-f0-9]{16})?\.json$/;
|
|
11016
11854
|
await Promise.all(
|
|
11017
|
-
names.filter((name) => runtimeStatePattern.test(name)).map((name) => fsPromises3.rm(
|
|
11855
|
+
names.filter((name) => runtimeStatePattern.test(name)).map((name) => fsPromises3.rm(path19.join(this.indexPath, name), { force: true }))
|
|
11018
11856
|
);
|
|
11019
11857
|
}
|
|
11020
11858
|
async resetLocalIndexArtifacts() {
|
|
@@ -11030,13 +11868,13 @@ var Indexer = class _Indexer {
|
|
|
11030
11868
|
this.readerArtifactRetryAfter.clear();
|
|
11031
11869
|
this.fileHashCache.clear();
|
|
11032
11870
|
const resetPaths = [
|
|
11033
|
-
|
|
11034
|
-
|
|
11035
|
-
|
|
11036
|
-
|
|
11037
|
-
|
|
11038
|
-
|
|
11039
|
-
|
|
11871
|
+
path19.join(this.indexPath, "codebase.db"),
|
|
11872
|
+
path19.join(this.indexPath, "codebase.db-shm"),
|
|
11873
|
+
path19.join(this.indexPath, "codebase.db-wal"),
|
|
11874
|
+
path19.join(this.indexPath, "vectors"),
|
|
11875
|
+
path19.join(this.indexPath, "vectors.usearch"),
|
|
11876
|
+
path19.join(this.indexPath, "vectors.meta.json"),
|
|
11877
|
+
path19.join(this.indexPath, "inverted-index.json")
|
|
11040
11878
|
];
|
|
11041
11879
|
await Promise.all(resetPaths.map((targetPath) => fsPromises3.rm(targetPath, { recursive: true, force: true })));
|
|
11042
11880
|
await this.removeProjectRuntimeStateArtifacts();
|
|
@@ -11046,7 +11884,7 @@ var Indexer = class _Indexer {
|
|
|
11046
11884
|
if (!isSqliteCorruptionError(error)) {
|
|
11047
11885
|
return false;
|
|
11048
11886
|
}
|
|
11049
|
-
const dbPath =
|
|
11887
|
+
const dbPath = path19.join(this.indexPath, "codebase.db");
|
|
11050
11888
|
const warning = this.getCorruptedIndexWarning(dbPath);
|
|
11051
11889
|
const errorMessage = getErrorMessage4(error);
|
|
11052
11890
|
if (this.config.scope === "global") {
|
|
@@ -11355,7 +12193,6 @@ var Indexer = class _Indexer {
|
|
|
11355
12193
|
skippedFiles: [],
|
|
11356
12194
|
parseFailures: []
|
|
11357
12195
|
};
|
|
11358
|
-
const failedBatchesForCurrentRun = [];
|
|
11359
12196
|
onProgress?.({
|
|
11360
12197
|
phase: "scanning",
|
|
11361
12198
|
filesProcessed: 0,
|
|
@@ -11370,14 +12207,10 @@ var Indexer = class _Indexer {
|
|
|
11370
12207
|
const reparseCachedMetalFiles = database.getMetadata(metalParserMetadataKey) !== METAL_PARSER_VERSION;
|
|
11371
12208
|
const symbolExtractorMetadataKey = this.getSymbolExtractorVersionMetadataKey();
|
|
11372
12209
|
const refreshCachedSymbols = database.getMetadata(symbolExtractorMetadataKey) !== SYMBOL_EXTRACTOR_VERSION;
|
|
11373
|
-
if (reparseCachedSwiftFiles && Array.from(this.fileHashCache.keys()).some(
|
|
11374
|
-
(filePath) => path18.extname(filePath).toLowerCase() === ".swift"
|
|
11375
|
-
)) {
|
|
12210
|
+
if (reparseCachedSwiftFiles && Array.from(this.fileHashCache.keys()).some((filePath) => path19.extname(filePath).toLowerCase() === ".swift")) {
|
|
11376
12211
|
this.logger.info("Reindexing cached Swift files for parser support");
|
|
11377
12212
|
}
|
|
11378
|
-
if (reparseCachedMetalFiles && Array.from(this.fileHashCache.keys()).some(
|
|
11379
|
-
(filePath) => path18.extname(filePath).toLowerCase() === ".metal"
|
|
11380
|
-
)) {
|
|
12213
|
+
if (reparseCachedMetalFiles && Array.from(this.fileHashCache.keys()).some((filePath) => path19.extname(filePath).toLowerCase() === ".metal")) {
|
|
11381
12214
|
this.logger.info("Reindexing cached Metal files for parser support");
|
|
11382
12215
|
}
|
|
11383
12216
|
const includePatterns = [...this.config.include, ...this.config.additionalInclude];
|
|
@@ -11399,46 +12232,44 @@ var Indexer = class _Indexer {
|
|
|
11399
12232
|
totalFiles: files.length,
|
|
11400
12233
|
skippedFiles: skipped.length
|
|
11401
12234
|
});
|
|
11402
|
-
const
|
|
12235
|
+
const changedFileDescriptors = [];
|
|
11403
12236
|
const unchangedFilePaths = /* @__PURE__ */ new Set();
|
|
11404
12237
|
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
11405
12238
|
const needsCallGraphResolutionMigration = database.getMetadata(this.getCallGraphResolutionMetadataKey()) !== CALL_GRAPH_RESOLUTION_VERSION;
|
|
11406
|
-
for (const
|
|
11407
|
-
const storedPath = this.toStoredFilePath(
|
|
11408
|
-
const currentHash = hashFile(
|
|
12239
|
+
for (const file of files) {
|
|
12240
|
+
const storedPath = this.toStoredFilePath(file.path);
|
|
12241
|
+
const currentHash = hashFile(file.path);
|
|
11409
12242
|
currentFileHashes.set(storedPath, currentHash);
|
|
11410
12243
|
const cachedHashMatches = this.fileHashCache.get(storedPath) === currentHash;
|
|
11411
12244
|
const needsCallGraphRefresh = cachedHashMatches && needsCallGraphResolutionMigration && database.getChunksByFile(storedPath).some(
|
|
11412
12245
|
(chunk) => chunk.language === "php" || chunk.language === "c" || chunk.language === "cpp"
|
|
11413
12246
|
);
|
|
11414
|
-
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles &&
|
|
11415
|
-
const requiresMetalParserUpgrade = reparseCachedMetalFiles &&
|
|
12247
|
+
const requiresSwiftParserUpgrade = reparseCachedSwiftFiles && path19.extname(storedPath).toLowerCase() === ".swift";
|
|
12248
|
+
const requiresMetalParserUpgrade = reparseCachedMetalFiles && path19.extname(storedPath).toLowerCase() === ".metal";
|
|
11416
12249
|
if (cachedHashMatches && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && !refreshCachedSymbols) {
|
|
11417
12250
|
unchangedFilePaths.add(storedPath);
|
|
11418
12251
|
this.logger.recordCacheHit();
|
|
11419
12252
|
} else {
|
|
11420
|
-
|
|
11421
|
-
|
|
12253
|
+
changedFileDescriptors.push({
|
|
12254
|
+
storedPath,
|
|
12255
|
+
materializedPath: file.path,
|
|
12256
|
+
hash: currentHash,
|
|
12257
|
+
sourceBytes: file.size
|
|
12258
|
+
});
|
|
11422
12259
|
this.logger.recordCacheMiss();
|
|
11423
12260
|
}
|
|
11424
12261
|
}
|
|
11425
12262
|
this.logger.cache("info", "File hash cache results", {
|
|
11426
12263
|
unchanged: unchangedFilePaths.size,
|
|
11427
|
-
changed:
|
|
12264
|
+
changed: changedFileDescriptors.length
|
|
11428
12265
|
});
|
|
11429
12266
|
onProgress?.({
|
|
11430
12267
|
phase: "parsing",
|
|
11431
|
-
filesProcessed:
|
|
12268
|
+
filesProcessed: unchangedFilePaths.size,
|
|
11432
12269
|
totalFiles: files.length,
|
|
11433
12270
|
chunksProcessed: 0,
|
|
11434
12271
|
totalChunks: 0
|
|
11435
12272
|
});
|
|
11436
|
-
const parseStartTime = performance2.now();
|
|
11437
|
-
const parsedFiles = parseFiles(changedFiles);
|
|
11438
|
-
const parseMs = performance2.now() - parseStartTime;
|
|
11439
|
-
this.logger.recordFilesParsed(parsedFiles.length);
|
|
11440
|
-
this.logger.recordParseDuration(parseMs);
|
|
11441
|
-
this.logger.debug("Parsed changed files", { parsedCount: parsedFiles.length, parseMs: parseMs.toFixed(2) });
|
|
11442
12273
|
const existingChunks = /* @__PURE__ */ new Map();
|
|
11443
12274
|
const existingChunksByFile = /* @__PURE__ */ new Map();
|
|
11444
12275
|
const existingMetadataById = /* @__PURE__ */ new Map();
|
|
@@ -11454,17 +12285,17 @@ var Indexer = class _Indexer {
|
|
|
11454
12285
|
}
|
|
11455
12286
|
existingChunks.set(key, metadata.hash);
|
|
11456
12287
|
existingMetadataById.set(key, metadata);
|
|
11457
|
-
const fileChunks = existingChunksByFile.get(metadata.filePath)
|
|
12288
|
+
const fileChunks = existingChunksByFile.get(metadata.filePath) ?? /* @__PURE__ */ new Set();
|
|
11458
12289
|
fileChunks.add(key);
|
|
11459
12290
|
existingChunksByFile.set(metadata.filePath, fileChunks);
|
|
11460
12291
|
}
|
|
11461
12292
|
const currentChunkIds = /* @__PURE__ */ new Set();
|
|
11462
|
-
const
|
|
11463
|
-
const
|
|
12293
|
+
const allSymbolIds = /* @__PURE__ */ new Set();
|
|
12294
|
+
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
12295
|
+
const retryableChunksWithExistingData = /* @__PURE__ */ new Set();
|
|
11464
12296
|
const gitBlameEnabled = this.config.indexing.gitBlame.enabled && isGitRepo(this.materializedProjectRoot);
|
|
11465
12297
|
let backfilledBlameMetadata = false;
|
|
11466
12298
|
for (const filePath of unchangedFilePaths) {
|
|
11467
|
-
currentFilePaths.add(filePath);
|
|
11468
12299
|
const fileChunks = existingChunksByFile.get(filePath);
|
|
11469
12300
|
if (fileChunks) {
|
|
11470
12301
|
for (const chunkId of fileChunks) {
|
|
@@ -11472,315 +12303,475 @@ var Indexer = class _Indexer {
|
|
|
11472
12303
|
}
|
|
11473
12304
|
}
|
|
11474
12305
|
}
|
|
11475
|
-
const
|
|
11476
|
-
|
|
11477
|
-
|
|
11478
|
-
|
|
11479
|
-
|
|
11480
|
-
|
|
11481
|
-
|
|
11482
|
-
|
|
11483
|
-
|
|
11484
|
-
|
|
11485
|
-
|
|
12306
|
+
const shouldRetryFailedPath = (filePath) => filePath !== null && currentFileHashes.has(filePath) && unchangedFilePaths.has(filePath);
|
|
12307
|
+
const failedProcessing = this.prepareFailedBatchProcessing(scopedRoots, shouldRetryFailedPath);
|
|
12308
|
+
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
12309
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
12310
|
+
const queue = new PQueue({
|
|
12311
|
+
concurrency: providerRateLimits.concurrency,
|
|
12312
|
+
interval: providerRateLimits.intervalMs,
|
|
12313
|
+
intervalCap: providerRateLimits.concurrency
|
|
12314
|
+
});
|
|
12315
|
+
const rateLimitState = { backoffMs: 0 };
|
|
12316
|
+
let writeTransactionActive = false;
|
|
12317
|
+
try {
|
|
12318
|
+
database.beginWriteTransaction();
|
|
12319
|
+
writeTransactionActive = true;
|
|
12320
|
+
const blameChunkDataBatch = [];
|
|
12321
|
+
if (gitBlameEnabled) {
|
|
12322
|
+
const backfillItems = [];
|
|
12323
|
+
for (const chunkId of currentChunkIds) {
|
|
12324
|
+
const metadata = existingMetadataById.get(chunkId);
|
|
12325
|
+
if (!metadata || hasBlameMetadata(metadata)) {
|
|
12326
|
+
continue;
|
|
12327
|
+
}
|
|
12328
|
+
const chunk = database.getChunk(chunkId);
|
|
12329
|
+
if (!chunk) {
|
|
12330
|
+
continue;
|
|
12331
|
+
}
|
|
12332
|
+
const blame = await getChunkGitBlame(
|
|
12333
|
+
this.materializedProjectRoot,
|
|
12334
|
+
this.toMaterializedFilePath(chunk.filePath),
|
|
12335
|
+
chunk.startLine,
|
|
12336
|
+
chunk.endLine
|
|
12337
|
+
);
|
|
12338
|
+
const blameMetadata = metadataFromBlame(blame);
|
|
12339
|
+
if (!blameMetadata.blameSha) {
|
|
12340
|
+
continue;
|
|
12341
|
+
}
|
|
12342
|
+
blameChunkDataBatch.push({
|
|
12343
|
+
...chunk,
|
|
12344
|
+
blameSha: blameMetadata.blameSha,
|
|
12345
|
+
blameAuthor: blameMetadata.blameAuthor,
|
|
12346
|
+
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
12347
|
+
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
12348
|
+
blameSummary: blameMetadata.blameSummary
|
|
12349
|
+
});
|
|
12350
|
+
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
12351
|
+
if (embeddingBuffer) {
|
|
12352
|
+
backfillItems.push({
|
|
12353
|
+
id: chunkId,
|
|
12354
|
+
vector: Array.from(bufferToFloat32Array(embeddingBuffer)),
|
|
12355
|
+
metadata: { ...metadata, ...blameMetadata }
|
|
12356
|
+
});
|
|
12357
|
+
}
|
|
11486
12358
|
}
|
|
11487
|
-
|
|
11488
|
-
|
|
11489
|
-
this.toMaterializedFilePath(chunk.filePath),
|
|
11490
|
-
chunk.startLine,
|
|
11491
|
-
chunk.endLine
|
|
11492
|
-
);
|
|
11493
|
-
const blameMetadata = metadataFromBlame(blame);
|
|
11494
|
-
if (!blameMetadata.blameSha) {
|
|
11495
|
-
continue;
|
|
12359
|
+
if (blameChunkDataBatch.length > 0) {
|
|
12360
|
+
database.upsertChunksBatch(blameChunkDataBatch);
|
|
11496
12361
|
}
|
|
11497
|
-
|
|
11498
|
-
|
|
11499
|
-
|
|
11500
|
-
blameAuthor: blameMetadata.blameAuthor,
|
|
11501
|
-
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
11502
|
-
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
11503
|
-
blameSummary: blameMetadata.blameSummary
|
|
11504
|
-
});
|
|
11505
|
-
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
11506
|
-
if (!embeddingBuffer) {
|
|
11507
|
-
continue;
|
|
12362
|
+
if (backfillItems.length > 0) {
|
|
12363
|
+
store.addBatch(backfillItems);
|
|
12364
|
+
backfilledBlameMetadata = true;
|
|
11508
12365
|
}
|
|
11509
|
-
backfillItems.push({
|
|
11510
|
-
id: chunkId,
|
|
11511
|
-
vector: Array.from(bufferToFloat32Array(embeddingBuffer)),
|
|
11512
|
-
metadata: {
|
|
11513
|
-
...metadata,
|
|
11514
|
-
...blameMetadata
|
|
11515
|
-
}
|
|
11516
|
-
});
|
|
11517
12366
|
}
|
|
11518
|
-
|
|
11519
|
-
|
|
11520
|
-
|
|
11521
|
-
|
|
11522
|
-
|
|
11523
|
-
for (const parsed of parsedFiles) {
|
|
11524
|
-
currentFilePaths.add(parsed.path);
|
|
11525
|
-
if (parsed.chunks.length === 0) {
|
|
11526
|
-
stats.parseFailures.push(path18.isAbsolute(parsed.path) ? path18.relative(this.projectRoot, parsed.path) : parsed.path);
|
|
11527
|
-
}
|
|
11528
|
-
let chunksToProcess = parsed.chunks;
|
|
11529
|
-
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
11530
|
-
const changedFile = changedFiles.find((f) => f.path === parsed.path);
|
|
11531
|
-
if (changedFile) {
|
|
11532
|
-
const textChunks = parseFileAsText(parsed.path, changedFile.content);
|
|
11533
|
-
chunksToProcess = textChunks;
|
|
12367
|
+
for (const filePath of unchangedFilePaths) {
|
|
12368
|
+
for (const symbol of database.getSymbolsByFile(filePath)) {
|
|
12369
|
+
if (!restrictExistingChunksToBranch || previousBranchSymbolIdSet.has(symbol.id)) {
|
|
12370
|
+
allSymbolIds.add(symbol.id);
|
|
12371
|
+
}
|
|
11534
12372
|
}
|
|
11535
12373
|
}
|
|
11536
|
-
|
|
11537
|
-
|
|
11538
|
-
|
|
11539
|
-
|
|
11540
|
-
|
|
11541
|
-
|
|
11542
|
-
const
|
|
11543
|
-
|
|
11544
|
-
|
|
11545
|
-
|
|
11546
|
-
|
|
11547
|
-
|
|
11548
|
-
|
|
11549
|
-
|
|
11550
|
-
|
|
11551
|
-
|
|
11552
|
-
|
|
11553
|
-
|
|
11554
|
-
|
|
11555
|
-
|
|
11556
|
-
|
|
11557
|
-
filePath: parsed.path,
|
|
11558
|
-
startLine: chunk.startLine,
|
|
11559
|
-
endLine: chunk.endLine,
|
|
11560
|
-
nodeType: chunk.chunkType,
|
|
11561
|
-
name: chunk.name,
|
|
11562
|
-
language: chunk.language,
|
|
11563
|
-
blameSha: blameMetadata.blameSha,
|
|
11564
|
-
blameAuthor: blameMetadata.blameAuthor,
|
|
11565
|
-
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
11566
|
-
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
11567
|
-
blameSummary: blameMetadata.blameSummary
|
|
12374
|
+
let processedChangedFiles = 0;
|
|
12375
|
+
for (const descriptorBatch of iterateOrderedFileBatches(
|
|
12376
|
+
changedFileDescriptors,
|
|
12377
|
+
(descriptor) => descriptor.sourceBytes,
|
|
12378
|
+
this.fileBatchLimits
|
|
12379
|
+
)) {
|
|
12380
|
+
const loadedFiles = await Promise.all(descriptorBatch.map(async (descriptor) => ({
|
|
12381
|
+
path: descriptor.storedPath,
|
|
12382
|
+
content: await fsPromises3.readFile(descriptor.materializedPath, "utf-8"),
|
|
12383
|
+
hash: descriptor.hash
|
|
12384
|
+
})));
|
|
12385
|
+
const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file]));
|
|
12386
|
+
const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor]));
|
|
12387
|
+
const parseStartTime = performance2.now();
|
|
12388
|
+
const parsedFiles = parseFiles(loadedFiles);
|
|
12389
|
+
const parseMs = performance2.now() - parseStartTime;
|
|
12390
|
+
this.logger.recordFilesParsed(parsedFiles.length);
|
|
12391
|
+
this.logger.recordParseDuration(parseMs);
|
|
12392
|
+
this.logger.debug("Parsed changed file batch", {
|
|
12393
|
+
parsedCount: parsedFiles.length,
|
|
12394
|
+
parseMs: parseMs.toFixed(2)
|
|
11568
12395
|
});
|
|
11569
|
-
|
|
11570
|
-
|
|
12396
|
+
const chunkDataBatch = [];
|
|
12397
|
+
const pendingChunks = [];
|
|
12398
|
+
const symbolBatch = [];
|
|
12399
|
+
const edgeBatch = [];
|
|
12400
|
+
for (const parsed of parsedFiles) {
|
|
12401
|
+
const loadedFile = loadedByPath.get(parsed.path);
|
|
12402
|
+
const descriptor = descriptorByPath.get(parsed.path);
|
|
12403
|
+
if (!loadedFile || !descriptor) {
|
|
12404
|
+
throw new Error(`Parsed file was not present in its source batch: ${parsed.path}`);
|
|
12405
|
+
}
|
|
12406
|
+
if (parsed.chunks.length === 0) {
|
|
12407
|
+
stats.parseFailures.push(path19.isAbsolute(parsed.path) ? path19.relative(this.projectRoot, parsed.path) : parsed.path);
|
|
12408
|
+
}
|
|
12409
|
+
let chunksToProcess = parsed.chunks;
|
|
12410
|
+
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
12411
|
+
chunksToProcess = parseFileAsText(parsed.path, loadedFile.content);
|
|
12412
|
+
}
|
|
12413
|
+
chunksToProcess = selectIndexableChunks(
|
|
12414
|
+
chunksToProcess,
|
|
12415
|
+
this.config.indexing.maxChunksPerFile,
|
|
12416
|
+
this.config.indexing.semanticOnly
|
|
12417
|
+
);
|
|
12418
|
+
for (const chunk of chunksToProcess) {
|
|
12419
|
+
const id = this.getPreparedChunkId(generateChunkId(parsed.path, chunk));
|
|
12420
|
+
const contentHash = generateChunkHash(chunk);
|
|
12421
|
+
const existingContentHash = existingChunks.get(id);
|
|
12422
|
+
const existingChunk = gitBlameEnabled ? database.getChunk(id) : null;
|
|
12423
|
+
const blame = gitBlameEnabled && existingContentHash !== contentHash ? await getChunkGitBlame(
|
|
12424
|
+
this.materializedProjectRoot,
|
|
12425
|
+
descriptor.materializedPath,
|
|
12426
|
+
chunk.startLine,
|
|
12427
|
+
chunk.endLine
|
|
12428
|
+
) : blameFromChunkData(existingChunk);
|
|
12429
|
+
const blameMetadata = metadataFromBlame(blame);
|
|
12430
|
+
currentChunkIds.add(id);
|
|
12431
|
+
chunkDataBatch.push({
|
|
12432
|
+
chunkId: id,
|
|
12433
|
+
contentHash,
|
|
12434
|
+
filePath: parsed.path,
|
|
12435
|
+
startLine: chunk.startLine,
|
|
12436
|
+
endLine: chunk.endLine,
|
|
12437
|
+
nodeType: chunk.chunkType,
|
|
12438
|
+
name: chunk.name,
|
|
12439
|
+
language: chunk.language,
|
|
12440
|
+
blameSha: blameMetadata.blameSha,
|
|
12441
|
+
blameAuthor: blameMetadata.blameAuthor,
|
|
12442
|
+
blameAuthorEmail: blameMetadata.blameAuthorEmail,
|
|
12443
|
+
blameCommittedAt: blameMetadata.blameCommittedAt,
|
|
12444
|
+
blameSummary: blameMetadata.blameSummary
|
|
12445
|
+
});
|
|
12446
|
+
if (existingContentHash === contentHash) {
|
|
12447
|
+
continue;
|
|
12448
|
+
}
|
|
12449
|
+
const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens).map((text) => ({
|
|
12450
|
+
text,
|
|
12451
|
+
tokenCount: estimateTokens(text)
|
|
12452
|
+
}));
|
|
12453
|
+
pendingChunks.push({
|
|
12454
|
+
id,
|
|
12455
|
+
texts,
|
|
12456
|
+
storageText: createPendingChunkStorageText(texts),
|
|
12457
|
+
content: chunk.content,
|
|
12458
|
+
contentHash,
|
|
12459
|
+
metadata: {
|
|
12460
|
+
filePath: parsed.path,
|
|
12461
|
+
startLine: chunk.startLine,
|
|
12462
|
+
endLine: chunk.endLine,
|
|
12463
|
+
chunkType: chunk.chunkType,
|
|
12464
|
+
name: chunk.name,
|
|
12465
|
+
language: chunk.language,
|
|
12466
|
+
hash: contentHash,
|
|
12467
|
+
...blameMetadata
|
|
12468
|
+
}
|
|
12469
|
+
});
|
|
12470
|
+
}
|
|
12471
|
+
const fileSymbols = [];
|
|
12472
|
+
for (const parsedSymbol of parsed.symbols) {
|
|
12473
|
+
if (!CALL_GRAPH_SYMBOL_CHUNK_TYPES.has(parsedSymbol.kind)) {
|
|
12474
|
+
continue;
|
|
12475
|
+
}
|
|
12476
|
+
const preparedNamespace = this.getPreparedBranchNamespace();
|
|
12477
|
+
const symbolId = `sym_${hashContent(
|
|
12478
|
+
(preparedNamespace ? `${preparedNamespace}:` : "") + parsed.path + ":" + parsedSymbol.name + ":" + parsedSymbol.kind + ":" + parsedSymbol.startLine + ":" + parsedSymbol.startCol + ":" + descriptor.hash
|
|
12479
|
+
).slice(0, 16)}`;
|
|
12480
|
+
const symbol = {
|
|
12481
|
+
id: symbolId,
|
|
12482
|
+
filePath: parsed.path,
|
|
12483
|
+
name: parsedSymbol.name,
|
|
12484
|
+
kind: parsedSymbol.kind,
|
|
12485
|
+
startLine: parsedSymbol.startLine,
|
|
12486
|
+
startCol: parsedSymbol.startCol,
|
|
12487
|
+
endLine: parsedSymbol.endLine,
|
|
12488
|
+
endCol: parsedSymbol.endCol,
|
|
12489
|
+
language: parsedSymbol.language
|
|
12490
|
+
};
|
|
12491
|
+
fileSymbols.push(symbol);
|
|
12492
|
+
symbolBatch.push(symbol);
|
|
12493
|
+
allSymbolIds.add(symbolId);
|
|
12494
|
+
}
|
|
12495
|
+
const fileLanguage = parsed.symbols[0]?.language ?? parsed.chunks[0]?.language;
|
|
12496
|
+
if (!fileLanguage || !CALL_GRAPH_LANGUAGES.has(fileLanguage)) {
|
|
12497
|
+
continue;
|
|
12498
|
+
}
|
|
12499
|
+
const isCaseInsensitiveLanguage = CASE_INSENSITIVE_LANGUAGES.has(fileLanguage);
|
|
12500
|
+
const normalizeSymbolKey = (name) => isCaseInsensitiveLanguage ? name.toLowerCase() : name;
|
|
12501
|
+
const symbolsByName = /* @__PURE__ */ new Map();
|
|
12502
|
+
for (const symbol of fileSymbols) {
|
|
12503
|
+
const key = normalizeSymbolKey(symbol.name);
|
|
12504
|
+
const symbols = symbolsByName.get(key) ?? [];
|
|
12505
|
+
symbols.push(symbol);
|
|
12506
|
+
symbolsByName.set(key, symbols);
|
|
12507
|
+
}
|
|
12508
|
+
for (const site of extractCalls(loadedFile.content, fileLanguage)) {
|
|
12509
|
+
const enclosingSymbol = findEnclosingSymbol(fileSymbols, site.line, site.column);
|
|
12510
|
+
if (!enclosingSymbol) {
|
|
12511
|
+
continue;
|
|
12512
|
+
}
|
|
12513
|
+
let candidates = symbolsByName.get(normalizeSymbolKey(site.calleeName));
|
|
12514
|
+
if (fileLanguage === "php" && candidates) {
|
|
12515
|
+
if (site.callType === "Constructor") {
|
|
12516
|
+
candidates = candidates.filter((candidate) => PHP_CLASS_SYMBOL_CHUNK_TYPES.has(candidate.kind));
|
|
12517
|
+
} else if (site.callType === "Call") {
|
|
12518
|
+
candidates = candidates.filter((candidate) => PHP_FUNCTION_SYMBOL_CHUNK_TYPES.has(candidate.kind));
|
|
12519
|
+
}
|
|
12520
|
+
}
|
|
12521
|
+
candidates = candidates?.filter(
|
|
12522
|
+
(symbol) => isCompatibleCFamilyCallTarget(fileLanguage, site.callType, symbol.kind)
|
|
12523
|
+
);
|
|
12524
|
+
const resolvedTarget = candidates?.length === 1 ? candidates[0] : void 0;
|
|
12525
|
+
edgeBatch.push({
|
|
12526
|
+
id: `edge_${hashContent(
|
|
12527
|
+
enclosingSymbol.id + ":" + site.calleeName + ":" + site.line + ":" + site.column
|
|
12528
|
+
).slice(0, 16)}`,
|
|
12529
|
+
fromSymbolId: enclosingSymbol.id,
|
|
12530
|
+
targetName: site.calleeName,
|
|
12531
|
+
toSymbolId: resolvedTarget?.id,
|
|
12532
|
+
callType: site.callType,
|
|
12533
|
+
confidence: site.confidence,
|
|
12534
|
+
line: site.line,
|
|
12535
|
+
col: site.column,
|
|
12536
|
+
isResolved: resolvedTarget !== void 0
|
|
12537
|
+
});
|
|
12538
|
+
}
|
|
11571
12539
|
}
|
|
11572
|
-
|
|
11573
|
-
|
|
11574
|
-
tokenCount: estimateTokens(text)
|
|
11575
|
-
}));
|
|
11576
|
-
const metadata = {
|
|
11577
|
-
filePath: parsed.path,
|
|
11578
|
-
startLine: chunk.startLine,
|
|
11579
|
-
endLine: chunk.endLine,
|
|
11580
|
-
chunkType: chunk.chunkType,
|
|
11581
|
-
name: chunk.name,
|
|
11582
|
-
language: chunk.language,
|
|
11583
|
-
hash: contentHash,
|
|
11584
|
-
...blameMetadata
|
|
11585
|
-
};
|
|
11586
|
-
pendingChunks.push({
|
|
11587
|
-
id,
|
|
11588
|
-
texts,
|
|
11589
|
-
storageText: createPendingChunkStorageText(texts),
|
|
11590
|
-
content: chunk.content,
|
|
11591
|
-
contentHash,
|
|
11592
|
-
metadata
|
|
11593
|
-
});
|
|
11594
|
-
}
|
|
11595
|
-
}
|
|
11596
|
-
const retryableFailedChunks = this.collectRetryableFailedChunks(
|
|
11597
|
-
currentFileHashes,
|
|
11598
|
-
unchangedFilePaths,
|
|
11599
|
-
getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)
|
|
11600
|
-
);
|
|
11601
|
-
const retryableFailedAttemptCounts = /* @__PURE__ */ new Map();
|
|
11602
|
-
const retryableChunksWithExistingData = /* @__PURE__ */ new Set();
|
|
11603
|
-
if (retryableFailedChunks.length > 0) {
|
|
11604
|
-
const pendingChunkIds = new Set(pendingChunks.map((chunk) => chunk.id));
|
|
11605
|
-
for (const { chunk, attemptCount } of retryableFailedChunks) {
|
|
11606
|
-
retryableFailedAttemptCounts.set(chunk.id, attemptCount);
|
|
11607
|
-
if (existingChunks.has(chunk.id)) {
|
|
11608
|
-
retryableChunksWithExistingData.add(chunk.id);
|
|
12540
|
+
if (chunkDataBatch.length > 0) {
|
|
12541
|
+
database.upsertChunksBatch(chunkDataBatch);
|
|
11609
12542
|
}
|
|
11610
|
-
if (
|
|
11611
|
-
|
|
11612
|
-
pendingChunkIds.add(chunk.id);
|
|
11613
|
-
currentChunkIds.add(chunk.id);
|
|
12543
|
+
if (symbolBatch.length > 0) {
|
|
12544
|
+
database.upsertSymbolsBatch(symbolBatch);
|
|
11614
12545
|
}
|
|
11615
|
-
|
|
11616
|
-
|
|
11617
|
-
|
|
11618
|
-
|
|
11619
|
-
|
|
11620
|
-
|
|
11621
|
-
|
|
11622
|
-
|
|
11623
|
-
|
|
11624
|
-
|
|
11625
|
-
|
|
11626
|
-
for (const parsedSymbol of parsed.symbols) {
|
|
11627
|
-
if (!CALL_GRAPH_SYMBOL_CHUNK_TYPES.has(parsedSymbol.kind)) continue;
|
|
11628
|
-
const preparedNamespace = this.getPreparedBranchNamespace();
|
|
11629
|
-
const symbolId = `sym_${hashContent(
|
|
11630
|
-
(preparedNamespace ? `${preparedNamespace}:` : "") + parsed.path + ":" + parsedSymbol.name + ":" + parsedSymbol.kind + ":" + parsedSymbol.startLine + ":" + parsedSymbol.startCol + ":" + changedFile.hash
|
|
11631
|
-
).slice(0, 16)}`;
|
|
11632
|
-
const symbol = {
|
|
11633
|
-
id: symbolId,
|
|
11634
|
-
filePath: parsed.path,
|
|
11635
|
-
name: parsedSymbol.name,
|
|
11636
|
-
kind: parsedSymbol.kind,
|
|
11637
|
-
startLine: parsedSymbol.startLine,
|
|
11638
|
-
startCol: parsedSymbol.startCol,
|
|
11639
|
-
endLine: parsedSymbol.endLine,
|
|
11640
|
-
endCol: parsedSymbol.endCol,
|
|
11641
|
-
language: parsedSymbol.language
|
|
11642
|
-
};
|
|
11643
|
-
fileSymbols.push(symbol);
|
|
11644
|
-
allSymbolIds.add(symbolId);
|
|
11645
|
-
}
|
|
11646
|
-
const fileLanguage = parsed.symbols[0]?.language ?? parsed.chunks[0]?.language;
|
|
11647
|
-
const isCaseInsensitiveLanguage = !!fileLanguage && CASE_INSENSITIVE_LANGUAGES.has(fileLanguage);
|
|
11648
|
-
const normalizeSymbolKey = (name) => isCaseInsensitiveLanguage ? name.toLowerCase() : name;
|
|
11649
|
-
const symbolsByName = /* @__PURE__ */ new Map();
|
|
11650
|
-
for (const symbol of fileSymbols) {
|
|
11651
|
-
const key = normalizeSymbolKey(symbol.name);
|
|
11652
|
-
const existing = symbolsByName.get(key) ?? [];
|
|
11653
|
-
existing.push(symbol);
|
|
11654
|
-
symbolsByName.set(key, existing);
|
|
11655
|
-
}
|
|
11656
|
-
if (fileSymbols.length > 0) {
|
|
11657
|
-
database.upsertSymbolsBatch(fileSymbols);
|
|
11658
|
-
symbolsByFile.set(parsed.path, fileSymbols);
|
|
11659
|
-
}
|
|
11660
|
-
if (!fileLanguage || !CALL_GRAPH_LANGUAGES.has(fileLanguage)) continue;
|
|
11661
|
-
const callSites = extractCalls(changedFile.content, fileLanguage);
|
|
11662
|
-
if (callSites.length === 0) continue;
|
|
11663
|
-
const edges = [];
|
|
11664
|
-
for (const site of callSites) {
|
|
11665
|
-
const enclosingSymbol = findEnclosingSymbol(
|
|
11666
|
-
fileSymbols,
|
|
11667
|
-
site.line,
|
|
11668
|
-
site.column
|
|
11669
|
-
);
|
|
11670
|
-
if (!enclosingSymbol) continue;
|
|
11671
|
-
const edgeId = `edge_${hashContent(enclosingSymbol.id + ":" + site.calleeName + ":" + site.line + ":" + site.column).slice(0, 16)}`;
|
|
11672
|
-
edges.push({
|
|
11673
|
-
id: edgeId,
|
|
11674
|
-
fromSymbolId: enclosingSymbol.id,
|
|
11675
|
-
targetName: site.calleeName,
|
|
11676
|
-
toSymbolId: void 0,
|
|
11677
|
-
callType: site.callType,
|
|
11678
|
-
confidence: site.confidence,
|
|
11679
|
-
line: site.line,
|
|
11680
|
-
col: site.column,
|
|
11681
|
-
isResolved: false
|
|
12546
|
+
if (edgeBatch.length > 0) {
|
|
12547
|
+
database.upsertCallEdgesBatch(edgeBatch);
|
|
12548
|
+
}
|
|
12549
|
+
processedChangedFiles += descriptorBatch.length;
|
|
12550
|
+
stats.totalChunks += pendingChunks.length;
|
|
12551
|
+
onProgress?.({
|
|
12552
|
+
phase: "parsing",
|
|
12553
|
+
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
12554
|
+
totalFiles: files.length,
|
|
12555
|
+
chunksProcessed: stats.indexedChunks,
|
|
12556
|
+
totalChunks: stats.totalChunks
|
|
11682
12557
|
});
|
|
11683
|
-
|
|
11684
|
-
|
|
11685
|
-
|
|
11686
|
-
|
|
11687
|
-
|
|
11688
|
-
|
|
11689
|
-
|
|
11690
|
-
|
|
11691
|
-
|
|
11692
|
-
|
|
11693
|
-
|
|
11694
|
-
|
|
11695
|
-
|
|
11696
|
-
|
|
12558
|
+
if (pendingChunks.length > 0) {
|
|
12559
|
+
onProgress?.({
|
|
12560
|
+
phase: "embedding",
|
|
12561
|
+
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
12562
|
+
totalFiles: files.length,
|
|
12563
|
+
chunksProcessed: stats.indexedChunks,
|
|
12564
|
+
totalChunks: stats.totalChunks
|
|
12565
|
+
});
|
|
12566
|
+
const batchResult = await this.processPendingChunkBatch(pendingChunks, {
|
|
12567
|
+
store,
|
|
12568
|
+
provider,
|
|
12569
|
+
invertedIndex,
|
|
12570
|
+
database,
|
|
12571
|
+
configuredProviderInfo,
|
|
12572
|
+
queue,
|
|
12573
|
+
providerRateLimits,
|
|
12574
|
+
rateLimitState,
|
|
12575
|
+
failedState: failedProcessing.state,
|
|
12576
|
+
attemptCounts: /* @__PURE__ */ new Map(),
|
|
12577
|
+
forceReembed: forceScopedReembed,
|
|
12578
|
+
reuseCachedEmbeddings: true,
|
|
12579
|
+
incrementRepeatedFailures: true,
|
|
12580
|
+
onProgress: (batchProgress) => onProgress?.({
|
|
12581
|
+
phase: "embedding",
|
|
12582
|
+
filesProcessed: unchangedFilePaths.size + processedChangedFiles,
|
|
12583
|
+
totalFiles: files.length,
|
|
12584
|
+
chunksProcessed: stats.indexedChunks + batchProgress.indexedChunks,
|
|
12585
|
+
totalChunks: stats.totalChunks
|
|
12586
|
+
})
|
|
12587
|
+
});
|
|
12588
|
+
stats.indexedChunks += batchResult.indexedChunks;
|
|
12589
|
+
stats.failedChunks += batchResult.failedChunks;
|
|
12590
|
+
stats.tokensUsed += batchResult.tokensUsed;
|
|
12591
|
+
for (const chunkId of batchResult.failedChunkIds) {
|
|
12592
|
+
failedChunkIds.add(chunkId);
|
|
12593
|
+
if (forceScopedReembed) {
|
|
12594
|
+
failedForcedChunkIds.add(chunkId);
|
|
11697
12595
|
}
|
|
11698
12596
|
}
|
|
11699
|
-
|
|
11700
|
-
|
|
11701
|
-
|
|
11702
|
-
|
|
11703
|
-
|
|
12597
|
+
}
|
|
12598
|
+
}
|
|
12599
|
+
const retryableFailedChunks = this.iterateLatestFailedChunks(
|
|
12600
|
+
failedProcessing.latestById,
|
|
12601
|
+
scopedRoots,
|
|
12602
|
+
shouldRetryFailedPath,
|
|
12603
|
+
maxChunkTokens
|
|
12604
|
+
);
|
|
12605
|
+
for (const retryBatch of iterateOrderedFileBatches(
|
|
12606
|
+
retryableFailedChunks,
|
|
12607
|
+
({ chunk }) => Buffer.byteLength(chunk.content, "utf-8"),
|
|
12608
|
+
this.fileBatchLimits
|
|
12609
|
+
)) {
|
|
12610
|
+
const pendingChunks = retryBatch.map(({ chunk }) => chunk);
|
|
12611
|
+
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
12612
|
+
for (const chunk of pendingChunks) {
|
|
12613
|
+
currentChunkIds.add(chunk.id);
|
|
12614
|
+
if (existingChunks.has(chunk.id)) {
|
|
12615
|
+
retryableChunksWithExistingData.add(chunk.id);
|
|
12616
|
+
}
|
|
12617
|
+
}
|
|
12618
|
+
stats.totalChunks += pendingChunks.length;
|
|
12619
|
+
onProgress?.({
|
|
12620
|
+
phase: "embedding",
|
|
12621
|
+
filesProcessed: files.length,
|
|
12622
|
+
totalFiles: files.length,
|
|
12623
|
+
chunksProcessed: stats.indexedChunks,
|
|
12624
|
+
totalChunks: stats.totalChunks
|
|
12625
|
+
});
|
|
12626
|
+
const batchResult = await this.processPendingChunkBatch(pendingChunks, {
|
|
12627
|
+
store,
|
|
12628
|
+
provider,
|
|
12629
|
+
invertedIndex,
|
|
12630
|
+
database,
|
|
12631
|
+
configuredProviderInfo,
|
|
12632
|
+
queue,
|
|
12633
|
+
providerRateLimits,
|
|
12634
|
+
rateLimitState,
|
|
12635
|
+
failedState: failedProcessing.state,
|
|
12636
|
+
attemptCounts,
|
|
12637
|
+
forceReembed: forceScopedReembed,
|
|
12638
|
+
reuseCachedEmbeddings: true,
|
|
12639
|
+
incrementRepeatedFailures: true,
|
|
12640
|
+
onProgress: (batchProgress) => onProgress?.({
|
|
12641
|
+
phase: "embedding",
|
|
12642
|
+
filesProcessed: files.length,
|
|
12643
|
+
totalFiles: files.length,
|
|
12644
|
+
chunksProcessed: stats.indexedChunks + batchProgress.indexedChunks,
|
|
12645
|
+
totalChunks: stats.totalChunks
|
|
12646
|
+
})
|
|
12647
|
+
});
|
|
12648
|
+
stats.indexedChunks += batchResult.indexedChunks;
|
|
12649
|
+
stats.failedChunks += batchResult.failedChunks;
|
|
12650
|
+
stats.tokensUsed += batchResult.tokensUsed;
|
|
12651
|
+
for (const chunkId of batchResult.failedChunkIds) {
|
|
12652
|
+
failedChunkIds.add(chunkId);
|
|
12653
|
+
if (forceScopedReembed) {
|
|
12654
|
+
failedForcedChunkIds.add(chunkId);
|
|
11704
12655
|
}
|
|
11705
12656
|
}
|
|
11706
12657
|
}
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
if (!restrictExistingChunksToBranch || previousBranchSymbolIdSet.has(sym.id)) {
|
|
11712
|
-
allSymbolIds.add(sym.id);
|
|
12658
|
+
const removedChunkIds = [];
|
|
12659
|
+
for (const [chunkId] of existingChunks) {
|
|
12660
|
+
if (!currentChunkIds.has(chunkId)) {
|
|
12661
|
+
removedChunkIds.push(chunkId);
|
|
11713
12662
|
}
|
|
11714
12663
|
}
|
|
11715
|
-
|
|
11716
|
-
|
|
11717
|
-
|
|
11718
|
-
|
|
11719
|
-
|
|
12664
|
+
const removedCount = removedChunkIds.length;
|
|
12665
|
+
stats.existingChunks = currentChunkIds.size - stats.totalChunks;
|
|
12666
|
+
stats.removedChunks = removedCount;
|
|
12667
|
+
this.logger.recordChunksProcessed(currentChunkIds.size);
|
|
12668
|
+
this.logger.recordChunksRemoved(removedCount);
|
|
12669
|
+
this.logger.info("Chunk analysis complete", {
|
|
12670
|
+
pending: stats.totalChunks,
|
|
12671
|
+
existing: stats.existingChunks,
|
|
12672
|
+
removed: removedCount
|
|
12673
|
+
});
|
|
12674
|
+
if (stats.totalChunks === 0 && removedCount === 0) {
|
|
12675
|
+
const removedStoredChunks = this.replaceBranchCatalog(
|
|
12676
|
+
store,
|
|
12677
|
+
invertedIndex,
|
|
12678
|
+
database,
|
|
12679
|
+
branchCatalogKey,
|
|
12680
|
+
previousBranchChunkIds,
|
|
12681
|
+
Array.from(currentChunkIds),
|
|
12682
|
+
previousBranchSymbolIds,
|
|
12683
|
+
Array.from(allSymbolIds)
|
|
12684
|
+
);
|
|
12685
|
+
const vectorPath = path19.join(this.indexPath, "vectors");
|
|
12686
|
+
const shouldFingerprintLegacyPair = !store.hasFingerprint() && existsSync11(vectorPath) && existsSync11(`${vectorPath}.meta.json`);
|
|
12687
|
+
if (backfilledBlameMetadata || shouldFingerprintLegacyPair || removedStoredChunks) {
|
|
12688
|
+
store.save();
|
|
12689
|
+
}
|
|
12690
|
+
if (removedStoredChunks) {
|
|
12691
|
+
this.saveInvertedIndex(invertedIndex);
|
|
12692
|
+
}
|
|
12693
|
+
if (scopedRoots) {
|
|
12694
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12695
|
+
} else {
|
|
12696
|
+
this.fileHashCache = currentFileHashes;
|
|
12697
|
+
this.saveFileHashCache();
|
|
12698
|
+
}
|
|
12699
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12700
|
+
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12701
|
+
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12702
|
+
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
12703
|
+
this.saveBranchCommit(database, indexedCommit);
|
|
12704
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12705
|
+
this.indexCompatibility = { compatible: true };
|
|
12706
|
+
database.commitWriteTransaction();
|
|
12707
|
+
writeTransactionActive = false;
|
|
12708
|
+
stats.durationMs = Date.now() - startTime;
|
|
12709
|
+
onProgress?.({
|
|
12710
|
+
phase: "complete",
|
|
12711
|
+
filesProcessed: files.length,
|
|
12712
|
+
totalFiles: files.length,
|
|
12713
|
+
chunksProcessed: 0,
|
|
12714
|
+
totalChunks: 0
|
|
12715
|
+
});
|
|
12716
|
+
return stats;
|
|
11720
12717
|
}
|
|
11721
|
-
|
|
11722
|
-
|
|
11723
|
-
|
|
11724
|
-
|
|
11725
|
-
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11732
|
-
});
|
|
11733
|
-
if (pendingChunks.length === 0 && removedCount === 0) {
|
|
11734
|
-
const removedStoredChunks = this.replaceBranchCatalog(
|
|
11735
|
-
store,
|
|
11736
|
-
invertedIndex,
|
|
11737
|
-
database,
|
|
11738
|
-
branchCatalogKey,
|
|
11739
|
-
previousBranchChunkIds,
|
|
11740
|
-
Array.from(currentChunkIds),
|
|
11741
|
-
previousBranchSymbolIds,
|
|
11742
|
-
Array.from(allSymbolIds)
|
|
11743
|
-
);
|
|
11744
|
-
const vectorPath = path18.join(this.indexPath, "vectors");
|
|
11745
|
-
const shouldFingerprintLegacyPair = !store.hasFingerprint() && existsSync10(vectorPath) && existsSync10(`${vectorPath}.meta.json`);
|
|
11746
|
-
if (backfilledBlameMetadata || shouldFingerprintLegacyPair || removedStoredChunks) {
|
|
12718
|
+
if (stats.totalChunks === 0) {
|
|
12719
|
+
this.replaceBranchCatalog(
|
|
12720
|
+
store,
|
|
12721
|
+
invertedIndex,
|
|
12722
|
+
database,
|
|
12723
|
+
branchCatalogKey,
|
|
12724
|
+
previousBranchChunkIds,
|
|
12725
|
+
Array.from(currentChunkIds),
|
|
12726
|
+
previousBranchSymbolIds,
|
|
12727
|
+
Array.from(allSymbolIds)
|
|
12728
|
+
);
|
|
11747
12729
|
store.save();
|
|
11748
|
-
}
|
|
11749
|
-
if (removedStoredChunks) {
|
|
11750
12730
|
this.saveInvertedIndex(invertedIndex);
|
|
12731
|
+
if (scopedRoots) {
|
|
12732
|
+
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12733
|
+
} else {
|
|
12734
|
+
this.fileHashCache = currentFileHashes;
|
|
12735
|
+
this.saveFileHashCache();
|
|
12736
|
+
}
|
|
12737
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12738
|
+
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12739
|
+
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12740
|
+
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
12741
|
+
this.saveBranchCommit(database, indexedCommit);
|
|
12742
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
12743
|
+
this.indexCompatibility = { compatible: true };
|
|
12744
|
+
database.commitWriteTransaction();
|
|
12745
|
+
writeTransactionActive = false;
|
|
12746
|
+
stats.durationMs = Date.now() - startTime;
|
|
12747
|
+
onProgress?.({
|
|
12748
|
+
phase: "complete",
|
|
12749
|
+
filesProcessed: files.length,
|
|
12750
|
+
totalFiles: files.length,
|
|
12751
|
+
chunksProcessed: 0,
|
|
12752
|
+
totalChunks: 0
|
|
12753
|
+
});
|
|
12754
|
+
return stats;
|
|
11751
12755
|
}
|
|
11752
|
-
if (scopedRoots) {
|
|
11753
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11754
|
-
this.clearScopedFailedBatches(scopedRoots);
|
|
11755
|
-
} else {
|
|
11756
|
-
this.fileHashCache = currentFileHashes;
|
|
11757
|
-
this.saveFileHashCache();
|
|
11758
|
-
this.saveFailedBatches([]);
|
|
11759
|
-
}
|
|
11760
|
-
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11761
|
-
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
11762
|
-
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
11763
|
-
this.saveBranchCommit(database, indexedCommit);
|
|
11764
|
-
this.saveIndexMetadata(configuredProviderInfo);
|
|
11765
|
-
this.indexCompatibility = { compatible: true };
|
|
11766
|
-
stats.durationMs = Date.now() - startTime;
|
|
11767
12756
|
onProgress?.({
|
|
11768
|
-
phase: "
|
|
12757
|
+
phase: "storing",
|
|
11769
12758
|
filesProcessed: files.length,
|
|
11770
12759
|
totalFiles: files.length,
|
|
11771
|
-
chunksProcessed:
|
|
11772
|
-
totalChunks:
|
|
12760
|
+
chunksProcessed: stats.indexedChunks,
|
|
12761
|
+
totalChunks: stats.totalChunks
|
|
12762
|
+
});
|
|
12763
|
+
const branchChunkIds = Array.from(currentChunkIds).filter((chunkId) => {
|
|
12764
|
+
const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId);
|
|
12765
|
+
const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId);
|
|
12766
|
+
return !isNewlyFailed && !isForcedFailed;
|
|
11773
12767
|
});
|
|
11774
|
-
return stats;
|
|
11775
|
-
}
|
|
11776
|
-
if (pendingChunks.length === 0) {
|
|
11777
12768
|
this.replaceBranchCatalog(
|
|
11778
12769
|
store,
|
|
11779
12770
|
invertedIndex,
|
|
11780
12771
|
database,
|
|
11781
12772
|
branchCatalogKey,
|
|
11782
12773
|
previousBranchChunkIds,
|
|
11783
|
-
|
|
12774
|
+
branchChunkIds,
|
|
11784
12775
|
previousBranchSymbolIds,
|
|
11785
12776
|
Array.from(allSymbolIds)
|
|
11786
12777
|
);
|
|
@@ -11788,11 +12779,35 @@ var Indexer = class _Indexer {
|
|
|
11788
12779
|
this.saveInvertedIndex(invertedIndex);
|
|
11789
12780
|
if (scopedRoots) {
|
|
11790
12781
|
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
11791
|
-
this.clearScopedFailedBatches(scopedRoots);
|
|
11792
12782
|
} else {
|
|
11793
12783
|
this.fileHashCache = currentFileHashes;
|
|
11794
12784
|
this.saveFileHashCache();
|
|
11795
|
-
|
|
12785
|
+
}
|
|
12786
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12787
|
+
database.commitWriteTransaction();
|
|
12788
|
+
writeTransactionActive = false;
|
|
12789
|
+
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
12790
|
+
const gcReset = await this.maybeRunOrphanGc();
|
|
12791
|
+
if (gcReset) {
|
|
12792
|
+
stats.durationMs = Date.now() - startTime;
|
|
12793
|
+
stats.warning = gcReset.warning;
|
|
12794
|
+
stats.resetCorruptedIndex = true;
|
|
12795
|
+
this.logger.recordIndexingEnd();
|
|
12796
|
+
this.logger.warn("Indexing ended after resetting corrupted local index during automatic GC", {
|
|
12797
|
+
files: stats.totalFiles,
|
|
12798
|
+
indexed: stats.indexedChunks,
|
|
12799
|
+
existing: stats.existingChunks,
|
|
12800
|
+
removed: stats.removedChunks,
|
|
12801
|
+
failed: stats.failedChunks,
|
|
12802
|
+
tokens: stats.tokensUsed,
|
|
12803
|
+
durationMs: stats.durationMs
|
|
12804
|
+
});
|
|
12805
|
+
return stats;
|
|
12806
|
+
}
|
|
12807
|
+
}
|
|
12808
|
+
stats.durationMs = Date.now() - startTime;
|
|
12809
|
+
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
12810
|
+
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
11796
12811
|
}
|
|
11797
12812
|
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
11798
12813
|
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
@@ -11800,307 +12815,40 @@ var Indexer = class _Indexer {
|
|
|
11800
12815
|
this.saveBranchCommit(database, indexedCommit);
|
|
11801
12816
|
this.saveIndexMetadata(configuredProviderInfo);
|
|
11802
12817
|
this.indexCompatibility = { compatible: true };
|
|
11803
|
-
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
11810
|
-
|
|
11811
|
-
|
|
11812
|
-
|
|
11813
|
-
|
|
11814
|
-
|
|
11815
|
-
filesProcessed: files.length,
|
|
11816
|
-
totalFiles: files.length,
|
|
11817
|
-
chunksProcessed: 0,
|
|
11818
|
-
totalChunks: pendingChunks.length
|
|
11819
|
-
});
|
|
11820
|
-
const allContentHashes = pendingChunks.map((c) => c.contentHash);
|
|
11821
|
-
const missingHashes = new Set(database.getMissingEmbeddings(allContentHashes));
|
|
11822
|
-
const forcedReembedChunkIds = forceScopedReembed ? new Set(pendingChunks.map((chunk) => chunk.id)) : /* @__PURE__ */ new Set();
|
|
11823
|
-
const chunksNeedingEmbedding = pendingChunks.filter((c) => forcedReembedChunkIds.has(c.id) || missingHashes.has(c.contentHash));
|
|
11824
|
-
const chunksWithExistingEmbedding = pendingChunks.filter((c) => !forcedReembedChunkIds.has(c.id) && !missingHashes.has(c.contentHash));
|
|
11825
|
-
this.logger.cache("info", "Embedding cache lookup", {
|
|
11826
|
-
needsEmbedding: chunksNeedingEmbedding.length,
|
|
11827
|
-
fromCache: chunksWithExistingEmbedding.length
|
|
11828
|
-
});
|
|
11829
|
-
this.logger.recordChunksFromCache(chunksWithExistingEmbedding.length);
|
|
11830
|
-
for (const chunk of chunksWithExistingEmbedding) {
|
|
11831
|
-
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
11832
|
-
if (embeddingBuffer) {
|
|
11833
|
-
const vector = bufferToFloat32Array(embeddingBuffer);
|
|
11834
|
-
store.add(chunk.id, Array.from(vector), chunk.metadata);
|
|
11835
|
-
invertedIndex.removeChunk(chunk.id);
|
|
11836
|
-
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
11837
|
-
stats.indexedChunks++;
|
|
11838
|
-
}
|
|
11839
|
-
}
|
|
11840
|
-
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
11841
|
-
const queue = new PQueue({
|
|
11842
|
-
concurrency: providerRateLimits.concurrency,
|
|
11843
|
-
interval: providerRateLimits.intervalMs,
|
|
11844
|
-
intervalCap: providerRateLimits.concurrency
|
|
11845
|
-
});
|
|
11846
|
-
const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk]));
|
|
11847
|
-
const embeddingPartsByChunk = /* @__PURE__ */ new Map();
|
|
11848
|
-
const completedChunkIds = /* @__PURE__ */ new Set();
|
|
11849
|
-
const failedChunkIds = /* @__PURE__ */ new Set();
|
|
11850
|
-
const requestBatches = createPendingEmbeddingRequestBatches(
|
|
11851
|
-
chunksNeedingEmbedding,
|
|
11852
|
-
getDynamicBatchOptions(configuredProviderInfo)
|
|
11853
|
-
);
|
|
11854
|
-
let rateLimitBackoffMs = 0;
|
|
11855
|
-
for (const requestBatch of requestBatches) {
|
|
11856
|
-
queue.add(async () => {
|
|
11857
|
-
if (rateLimitBackoffMs > 0) {
|
|
11858
|
-
await new Promise((resolve15) => setTimeout(resolve15, rateLimitBackoffMs));
|
|
11859
|
-
}
|
|
11860
|
-
try {
|
|
11861
|
-
const result = await pRetry(
|
|
11862
|
-
async () => {
|
|
11863
|
-
const texts = requestBatch.map((request) => request.text);
|
|
11864
|
-
return provider.embedBatch(texts);
|
|
11865
|
-
},
|
|
11866
|
-
{
|
|
11867
|
-
retries: this.config.indexing.retries,
|
|
11868
|
-
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
11869
|
-
maxTimeout: providerRateLimits.maxRetryMs,
|
|
11870
|
-
factor: 2,
|
|
11871
|
-
shouldRetry: (error) => !(error.error instanceof CustomProviderNonRetryableError),
|
|
11872
|
-
onFailedAttempt: (error) => {
|
|
11873
|
-
const message = getErrorMessage4(error);
|
|
11874
|
-
if (isRateLimitError(error)) {
|
|
11875
|
-
rateLimitBackoffMs = Math.min(providerRateLimits.maxRetryMs, (rateLimitBackoffMs || providerRateLimits.minRetryMs) * 2);
|
|
11876
|
-
this.logger.embedding("warn", `Rate limited, backing off`, {
|
|
11877
|
-
attempt: error.attemptNumber,
|
|
11878
|
-
retriesLeft: error.retriesLeft,
|
|
11879
|
-
backoffMs: rateLimitBackoffMs
|
|
11880
|
-
});
|
|
11881
|
-
} else {
|
|
11882
|
-
this.logger.embedding("error", `Embedding batch failed`, {
|
|
11883
|
-
attempt: error.attemptNumber,
|
|
11884
|
-
error: message
|
|
11885
|
-
});
|
|
11886
|
-
}
|
|
11887
|
-
}
|
|
11888
|
-
}
|
|
11889
|
-
);
|
|
11890
|
-
if (rateLimitBackoffMs > 0) {
|
|
11891
|
-
rateLimitBackoffMs = Math.max(0, rateLimitBackoffMs - 2e3);
|
|
11892
|
-
}
|
|
11893
|
-
const touchedChunkIds = /* @__PURE__ */ new Set();
|
|
11894
|
-
requestBatch.forEach((request, idx) => {
|
|
11895
|
-
if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) {
|
|
11896
|
-
return;
|
|
11897
|
-
}
|
|
11898
|
-
const vector = result.embeddings[idx];
|
|
11899
|
-
if (!vector) {
|
|
11900
|
-
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
11901
|
-
}
|
|
11902
|
-
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
11903
|
-
parts[request.partIndex] = {
|
|
11904
|
-
vector,
|
|
11905
|
-
tokenCount: request.tokenCount
|
|
11906
|
-
};
|
|
11907
|
-
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
11908
|
-
touchedChunkIds.add(request.chunk.id);
|
|
11909
|
-
});
|
|
11910
|
-
const pooledResults = [];
|
|
11911
|
-
for (const chunkId of touchedChunkIds) {
|
|
11912
|
-
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
11913
|
-
continue;
|
|
11914
|
-
}
|
|
11915
|
-
const chunk = pendingChunksById.get(chunkId);
|
|
11916
|
-
if (!chunk) {
|
|
11917
|
-
continue;
|
|
11918
|
-
}
|
|
11919
|
-
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
11920
|
-
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
11921
|
-
continue;
|
|
11922
|
-
}
|
|
11923
|
-
const orderedParts = parts;
|
|
11924
|
-
pooledResults.push({
|
|
11925
|
-
chunk,
|
|
11926
|
-
vector: poolEmbeddingVectors(
|
|
11927
|
-
orderedParts.map((part) => part.vector),
|
|
11928
|
-
orderedParts.map((part) => part.tokenCount)
|
|
11929
|
-
)
|
|
11930
|
-
});
|
|
11931
|
-
}
|
|
11932
|
-
if (pooledResults.length > 0) {
|
|
11933
|
-
const items = pooledResults.map(({ chunk, vector }) => ({
|
|
11934
|
-
id: chunk.id,
|
|
11935
|
-
vector,
|
|
11936
|
-
metadata: chunk.metadata
|
|
11937
|
-
}));
|
|
11938
|
-
store.addBatch(items);
|
|
11939
|
-
const embeddingBatchItems = pooledResults.map(({ chunk, vector }) => ({
|
|
11940
|
-
contentHash: chunk.contentHash,
|
|
11941
|
-
embedding: float32ArrayToBuffer(vector),
|
|
11942
|
-
chunkText: chunk.storageText,
|
|
11943
|
-
model: configuredProviderInfo.modelInfo.model
|
|
11944
|
-
}));
|
|
11945
|
-
try {
|
|
11946
|
-
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
11947
|
-
} catch (dbError) {
|
|
11948
|
-
this.rebuildVectorStoreExcludingChunkIds(
|
|
11949
|
-
store,
|
|
11950
|
-
database,
|
|
11951
|
-
pooledResults.map(({ chunk }) => chunk.id)
|
|
11952
|
-
);
|
|
11953
|
-
throw dbError;
|
|
11954
|
-
}
|
|
11955
|
-
for (const { chunk } of pooledResults) {
|
|
11956
|
-
invertedIndex.removeChunk(chunk.id);
|
|
11957
|
-
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
11958
|
-
completedChunkIds.add(chunk.id);
|
|
11959
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
11960
|
-
}
|
|
11961
|
-
stats.indexedChunks += pooledResults.length;
|
|
11962
|
-
this.logger.recordChunksEmbedded(pooledResults.length);
|
|
11963
|
-
}
|
|
11964
|
-
stats.tokensUsed += result.totalTokensUsed;
|
|
11965
|
-
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
11966
|
-
this.logger.embedding("debug", `Embedded batch`, {
|
|
11967
|
-
batchSize: pooledResults.length,
|
|
11968
|
-
requestCount: requestBatch.length,
|
|
11969
|
-
tokens: result.totalTokensUsed
|
|
11970
|
-
});
|
|
11971
|
-
onProgress?.({
|
|
11972
|
-
phase: "embedding",
|
|
11973
|
-
filesProcessed: files.length,
|
|
11974
|
-
totalFiles: files.length,
|
|
11975
|
-
chunksProcessed: stats.indexedChunks,
|
|
11976
|
-
totalChunks: pendingChunks.length
|
|
11977
|
-
});
|
|
11978
|
-
} catch (error) {
|
|
11979
|
-
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id));
|
|
11980
|
-
const failureMessage = getErrorMessage4(error);
|
|
11981
|
-
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
11982
|
-
for (const chunk of failedChunks) {
|
|
11983
|
-
if (!failedChunkIds.has(chunk.id)) {
|
|
11984
|
-
failedChunkIds.add(chunk.id);
|
|
11985
|
-
stats.failedChunks += 1;
|
|
11986
|
-
}
|
|
11987
|
-
if (forceScopedReembed) {
|
|
11988
|
-
failedForcedChunkIds.add(chunk.id);
|
|
11989
|
-
}
|
|
11990
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
11991
|
-
const existingFailedBatchIndex = failedBatchesForCurrentRun.findIndex(
|
|
11992
|
-
(failedBatch2) => failedBatch2.chunks[0]?.id === chunk.id
|
|
11993
|
-
);
|
|
11994
|
-
const existingFailedBatch = existingFailedBatchIndex === -1 ? void 0 : failedBatchesForCurrentRun[existingFailedBatchIndex];
|
|
11995
|
-
const failedBatch = {
|
|
11996
|
-
chunks: [chunk],
|
|
11997
|
-
error: failureMessage,
|
|
11998
|
-
attemptCount: (existingFailedBatch?.attemptCount ?? retryableFailedAttemptCounts.get(chunk.id) ?? 0) + 1,
|
|
11999
|
-
lastAttempt: failureTimestamp
|
|
12000
|
-
};
|
|
12001
|
-
if (existingFailedBatchIndex === -1) {
|
|
12002
|
-
failedBatchesForCurrentRun.push(failedBatch);
|
|
12003
|
-
} else {
|
|
12004
|
-
failedBatchesForCurrentRun[existingFailedBatchIndex] = failedBatch;
|
|
12005
|
-
}
|
|
12006
|
-
}
|
|
12007
|
-
this.logger.recordEmbeddingError();
|
|
12008
|
-
this.logger.embedding("error", `Failed to embed batch after retries`, {
|
|
12009
|
-
batchSize: failedChunks.length,
|
|
12010
|
-
requestCount: requestBatch.length,
|
|
12011
|
-
error: failureMessage
|
|
12012
|
-
});
|
|
12013
|
-
}
|
|
12014
|
-
});
|
|
12015
|
-
}
|
|
12016
|
-
await queue.onIdle();
|
|
12017
|
-
if (scopedRoots) {
|
|
12018
|
-
this.saveScopedFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun), scopedRoots);
|
|
12019
|
-
} else {
|
|
12020
|
-
this.saveFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun));
|
|
12021
|
-
}
|
|
12022
|
-
onProgress?.({
|
|
12023
|
-
phase: "storing",
|
|
12024
|
-
filesProcessed: files.length,
|
|
12025
|
-
totalFiles: files.length,
|
|
12026
|
-
chunksProcessed: stats.indexedChunks,
|
|
12027
|
-
totalChunks: pendingChunks.length
|
|
12028
|
-
});
|
|
12029
|
-
const branchChunkIds = Array.from(currentChunkIds).filter(
|
|
12030
|
-
(chunkId) => {
|
|
12031
|
-
const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId);
|
|
12032
|
-
const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId);
|
|
12033
|
-
return !isNewlyFailed && !isForcedFailed;
|
|
12034
|
-
}
|
|
12035
|
-
);
|
|
12036
|
-
this.replaceBranchCatalog(
|
|
12037
|
-
store,
|
|
12038
|
-
invertedIndex,
|
|
12039
|
-
database,
|
|
12040
|
-
branchCatalogKey,
|
|
12041
|
-
previousBranchChunkIds,
|
|
12042
|
-
branchChunkIds,
|
|
12043
|
-
previousBranchSymbolIds,
|
|
12044
|
-
Array.from(allSymbolIds)
|
|
12045
|
-
);
|
|
12046
|
-
store.save();
|
|
12047
|
-
this.saveInvertedIndex(invertedIndex);
|
|
12048
|
-
if (scopedRoots) {
|
|
12049
|
-
this.replaceScopedFileHashCache(currentFileHashes, scopedRoots);
|
|
12050
|
-
} else {
|
|
12051
|
-
this.fileHashCache = currentFileHashes;
|
|
12052
|
-
this.saveFileHashCache();
|
|
12053
|
-
}
|
|
12054
|
-
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
12055
|
-
const gcReset = await this.maybeRunOrphanGc();
|
|
12056
|
-
if (gcReset) {
|
|
12057
|
-
stats.durationMs = Date.now() - startTime;
|
|
12058
|
-
stats.warning = gcReset.warning;
|
|
12059
|
-
stats.resetCorruptedIndex = true;
|
|
12060
|
-
this.logger.recordIndexingEnd();
|
|
12061
|
-
this.logger.warn("Indexing ended after resetting corrupted local index during automatic GC", {
|
|
12062
|
-
files: stats.totalFiles,
|
|
12063
|
-
indexed: stats.indexedChunks,
|
|
12064
|
-
existing: stats.existingChunks,
|
|
12065
|
-
removed: stats.removedChunks,
|
|
12066
|
-
failed: stats.failedChunks,
|
|
12067
|
-
tokens: stats.tokensUsed,
|
|
12068
|
-
durationMs: stats.durationMs
|
|
12069
|
-
});
|
|
12070
|
-
return stats;
|
|
12818
|
+
this.logger.recordIndexingEnd();
|
|
12819
|
+
this.logger.info("Indexing complete", {
|
|
12820
|
+
files: stats.totalFiles,
|
|
12821
|
+
indexed: stats.indexedChunks,
|
|
12822
|
+
existing: stats.existingChunks,
|
|
12823
|
+
removed: stats.removedChunks,
|
|
12824
|
+
failed: stats.failedChunks,
|
|
12825
|
+
tokens: stats.tokensUsed,
|
|
12826
|
+
durationMs: stats.durationMs
|
|
12827
|
+
});
|
|
12828
|
+
if (stats.failedChunks > 0) {
|
|
12829
|
+
stats.failedBatchesPath = this.failedBatchesPath;
|
|
12071
12830
|
}
|
|
12831
|
+
onProgress?.({
|
|
12832
|
+
phase: "complete",
|
|
12833
|
+
filesProcessed: files.length,
|
|
12834
|
+
totalFiles: files.length,
|
|
12835
|
+
chunksProcessed: stats.indexedChunks,
|
|
12836
|
+
totalChunks: stats.totalChunks
|
|
12837
|
+
});
|
|
12838
|
+
return stats;
|
|
12839
|
+
} catch (error) {
|
|
12840
|
+
failedProcessing.state.writer.cleanup();
|
|
12841
|
+
if (writeTransactionActive) {
|
|
12842
|
+
try {
|
|
12843
|
+
database.rollbackWriteTransaction();
|
|
12844
|
+
} catch (rollbackError) {
|
|
12845
|
+
this.logger.error("Failed to roll back indexing database transaction", {
|
|
12846
|
+
error: getErrorMessage4(rollbackError)
|
|
12847
|
+
});
|
|
12848
|
+
}
|
|
12849
|
+
}
|
|
12850
|
+
throw error;
|
|
12072
12851
|
}
|
|
12073
|
-
stats.durationMs = Date.now() - startTime;
|
|
12074
|
-
if (forceScopedReembed && failedForcedChunkIds.size === 0) {
|
|
12075
|
-
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12076
|
-
}
|
|
12077
|
-
database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION);
|
|
12078
|
-
database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION);
|
|
12079
|
-
database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION);
|
|
12080
|
-
this.saveBranchCommit(database, indexedCommit);
|
|
12081
|
-
this.saveIndexMetadata(configuredProviderInfo);
|
|
12082
|
-
this.indexCompatibility = { compatible: true };
|
|
12083
|
-
this.logger.recordIndexingEnd();
|
|
12084
|
-
this.logger.info("Indexing complete", {
|
|
12085
|
-
files: stats.totalFiles,
|
|
12086
|
-
indexed: stats.indexedChunks,
|
|
12087
|
-
existing: stats.existingChunks,
|
|
12088
|
-
removed: stats.removedChunks,
|
|
12089
|
-
failed: stats.failedChunks,
|
|
12090
|
-
tokens: stats.tokensUsed,
|
|
12091
|
-
durationMs: stats.durationMs
|
|
12092
|
-
});
|
|
12093
|
-
if (stats.failedChunks > 0) {
|
|
12094
|
-
stats.failedBatchesPath = this.failedBatchesPath;
|
|
12095
|
-
}
|
|
12096
|
-
onProgress?.({
|
|
12097
|
-
phase: "complete",
|
|
12098
|
-
filesProcessed: files.length,
|
|
12099
|
-
totalFiles: files.length,
|
|
12100
|
-
chunksProcessed: stats.indexedChunks,
|
|
12101
|
-
totalChunks: pendingChunks.length
|
|
12102
|
-
});
|
|
12103
|
-
return stats;
|
|
12104
12852
|
}
|
|
12105
12853
|
async getQueryEmbedding(query, provider) {
|
|
12106
12854
|
const now2 = Date.now();
|
|
@@ -12338,10 +13086,31 @@ var Indexer = class _Indexer {
|
|
|
12338
13086
|
const baseFiltered = tiered.filter(
|
|
12339
13087
|
(r) => matchesSearchFilters(r, options, this.config.search.minScore, this.projectRoot)
|
|
12340
13088
|
);
|
|
12341
|
-
|
|
13089
|
+
let communityRanked = baseFiltered;
|
|
13090
|
+
if (this.config.search.communityBoost > 0) {
|
|
13091
|
+
try {
|
|
13092
|
+
const sameCommunityCandidateIds = resolveSameCommunityCandidateIds(
|
|
13093
|
+
query,
|
|
13094
|
+
baseFiltered,
|
|
13095
|
+
database,
|
|
13096
|
+
this.getBranchCatalogKeys()
|
|
13097
|
+
);
|
|
13098
|
+
communityRanked = applyCommunityBoost(
|
|
13099
|
+
baseFiltered,
|
|
13100
|
+
sameCommunityCandidateIds,
|
|
13101
|
+
this.config.search.communityBoost
|
|
13102
|
+
);
|
|
13103
|
+
} catch (error) {
|
|
13104
|
+
this.logger.search("debug", "Community-aware ranking unavailable; using existing ranking", {
|
|
13105
|
+
query,
|
|
13106
|
+
error: getErrorMessage4(error)
|
|
13107
|
+
});
|
|
13108
|
+
}
|
|
13109
|
+
}
|
|
13110
|
+
const implementationOnly = communityRanked.filter(
|
|
12342
13111
|
(r) => isLikelyImplementationPath2(r.metadata.filePath) && isImplementationChunkType(r.metadata.chunkType)
|
|
12343
13112
|
);
|
|
12344
|
-
const filtered = (sourceIntent && hasCodeHints && implementationOnly.length > 0 ? implementationOnly :
|
|
13113
|
+
const filtered = (sourceIntent && hasCodeHints && implementationOnly.length > 0 ? implementationOnly : communityRanked).slice(0, maxResults);
|
|
12345
13114
|
const identifierFallback = !options?.definitionIntent && filtered.length === 0 && identifierHints.length > 0 ? buildSymbolDefinitionLane(query, database, branchChunkIds, branchSymbolIds, maxResults, union, true).filter((r) => matchesSearchFilters(r, options, this.config.search.minScore, this.projectRoot)).slice(0, maxResults) : [];
|
|
12346
13115
|
const finalResults = filtered.length > 0 ? filtered : identifierFallback;
|
|
12347
13116
|
const totalSearchMs = performance2.now() - searchStartTime;
|
|
@@ -12558,7 +13327,7 @@ var Indexer = class _Indexer {
|
|
|
12558
13327
|
this.saveFileHashCache();
|
|
12559
13328
|
database.clearAllIndexedData();
|
|
12560
13329
|
this.deleteBranchCommitMetadata(database, clearedBranchKeys2);
|
|
12561
|
-
this.
|
|
13330
|
+
this.clearFailedBatchState();
|
|
12562
13331
|
database.deleteMetadata("index.version");
|
|
12563
13332
|
database.deleteMetadata("index.pathStorageVersion");
|
|
12564
13333
|
database.deleteMetadata("index.embeddingProvider");
|
|
@@ -12626,7 +13395,7 @@ var Indexer = class _Indexer {
|
|
|
12626
13395
|
const missingChunkKeys = [];
|
|
12627
13396
|
const chunkKeysByRemovedFile = /* @__PURE__ */ new Map();
|
|
12628
13397
|
for (const [filePath, chunkKeys] of filePathsToChunkKeys) {
|
|
12629
|
-
if (!
|
|
13398
|
+
if (!existsSync11(this.toMaterializedFilePath(filePath))) {
|
|
12630
13399
|
chunkKeysByRemovedFile.set(filePath, chunkKeys);
|
|
12631
13400
|
for (const key of chunkKeys) {
|
|
12632
13401
|
missingChunkKeys.push(key);
|
|
@@ -12689,7 +13458,7 @@ var Indexer = class _Indexer {
|
|
|
12689
13458
|
gcOrphanSymbols: 0,
|
|
12690
13459
|
gcOrphanCallEdges: 0,
|
|
12691
13460
|
resetCorruptedIndex: true,
|
|
12692
|
-
warning: this.getCorruptedIndexWarning(
|
|
13461
|
+
warning: this.getCorruptedIndexWarning(path19.join(this.indexPath, "codebase.db"))
|
|
12693
13462
|
};
|
|
12694
13463
|
}
|
|
12695
13464
|
this.logger.recordGc(removedCount, gcOrphanChunks, gcOrphanEmbeddings);
|
|
@@ -12719,180 +13488,94 @@ var Indexer = class _Indexer {
|
|
|
12719
13488
|
const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
|
|
12720
13489
|
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
12721
13490
|
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
12722
|
-
const
|
|
12723
|
-
|
|
12724
|
-
|
|
13491
|
+
const failedProcessing = this.prepareFailedBatchProcessing(roots, () => true);
|
|
13492
|
+
if (failedProcessing.latestById.size === 0) {
|
|
13493
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
12725
13494
|
return { succeeded: 0, failed: 0, remaining: 0 };
|
|
12726
13495
|
}
|
|
13496
|
+
const queue = new PQueue({ concurrency: 1 });
|
|
13497
|
+
const rateLimitState = { backoffMs: 0 };
|
|
12727
13498
|
let succeeded = 0;
|
|
12728
13499
|
let failed = 0;
|
|
12729
|
-
|
|
12730
|
-
|
|
12731
|
-
|
|
12732
|
-
|
|
12733
|
-
|
|
12734
|
-
|
|
12735
|
-
|
|
12736
|
-
const
|
|
12737
|
-
|
|
12738
|
-
|
|
12739
|
-
|
|
12740
|
-
|
|
12741
|
-
);
|
|
12742
|
-
|
|
12743
|
-
|
|
12744
|
-
|
|
12745
|
-
|
|
12746
|
-
|
|
12747
|
-
|
|
12748
|
-
|
|
12749
|
-
|
|
12750
|
-
|
|
12751
|
-
|
|
12752
|
-
|
|
12753
|
-
|
|
12754
|
-
|
|
12755
|
-
|
|
12756
|
-
|
|
12757
|
-
|
|
12758
|
-
|
|
12759
|
-
|
|
12760
|
-
|
|
12761
|
-
}
|
|
12762
|
-
const vector = result.embeddings[idx];
|
|
12763
|
-
if (!vector) {
|
|
12764
|
-
throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`);
|
|
12765
|
-
}
|
|
12766
|
-
const parts = embeddingPartsByChunk.get(request.chunk.id) ?? [];
|
|
12767
|
-
parts[request.partIndex] = {
|
|
12768
|
-
vector,
|
|
12769
|
-
tokenCount: request.tokenCount
|
|
12770
|
-
};
|
|
12771
|
-
embeddingPartsByChunk.set(request.chunk.id, parts);
|
|
12772
|
-
touchedChunkIds.add(request.chunk.id);
|
|
12773
|
-
});
|
|
12774
|
-
for (const chunkId of touchedChunkIds) {
|
|
12775
|
-
if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) {
|
|
12776
|
-
continue;
|
|
12777
|
-
}
|
|
12778
|
-
const chunk = batchChunksById.get(chunkId);
|
|
12779
|
-
if (!chunk) {
|
|
12780
|
-
continue;
|
|
12781
|
-
}
|
|
12782
|
-
const parts = embeddingPartsByChunk.get(chunk.id) ?? [];
|
|
12783
|
-
if (!hasAllEmbeddingParts(parts, chunk.texts.length)) {
|
|
12784
|
-
continue;
|
|
12785
|
-
}
|
|
12786
|
-
const orderedParts = parts;
|
|
12787
|
-
pooledResults.push({
|
|
12788
|
-
chunk,
|
|
12789
|
-
vector: poolEmbeddingVectors(
|
|
12790
|
-
orderedParts.map((part) => part.vector),
|
|
12791
|
-
orderedParts.map((part) => part.tokenCount)
|
|
12792
|
-
)
|
|
12793
|
-
});
|
|
12794
|
-
}
|
|
12795
|
-
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
12796
|
-
} catch (error) {
|
|
12797
|
-
const failureMessage = String(error);
|
|
12798
|
-
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
12799
|
-
const failedChunks = getUniquePendingChunksFromRequests(requestBatch).filter((chunk) => !completedChunkIds.has(chunk.id) && !failedChunkIds.has(chunk.id));
|
|
12800
|
-
for (const chunk of failedChunks) {
|
|
12801
|
-
failedChunkIds.add(chunk.id);
|
|
12802
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
12803
|
-
failedChunksForBatch.set(chunk.id, {
|
|
12804
|
-
chunks: [chunk],
|
|
12805
|
-
attemptCount: batch.attemptCount + 1,
|
|
12806
|
-
lastAttempt: failureTimestamp,
|
|
12807
|
-
error: failureMessage
|
|
12808
|
-
});
|
|
12809
|
-
}
|
|
12810
|
-
failed += failedChunks.length;
|
|
12811
|
-
this.logger.recordEmbeddingError();
|
|
12812
|
-
}
|
|
12813
|
-
}
|
|
12814
|
-
const successfulResults = pooledResults.filter(({ chunk }) => !failedChunkIds.has(chunk.id));
|
|
12815
|
-
const items = successfulResults.map(({ chunk, vector }) => ({
|
|
12816
|
-
id: chunk.id,
|
|
12817
|
-
vector,
|
|
12818
|
-
metadata: chunk.metadata
|
|
12819
|
-
}));
|
|
12820
|
-
if (items.length > 0) {
|
|
12821
|
-
store.addBatch(items);
|
|
12822
|
-
}
|
|
12823
|
-
if (successfulResults.length > 0) {
|
|
12824
|
-
try {
|
|
12825
|
-
database.upsertEmbeddingsBatch(
|
|
12826
|
-
successfulResults.map(({ chunk, vector }) => ({
|
|
12827
|
-
contentHash: chunk.contentHash,
|
|
12828
|
-
embedding: float32ArrayToBuffer(vector),
|
|
12829
|
-
chunkText: chunk.storageText,
|
|
12830
|
-
model: configuredProviderInfo.modelInfo.model
|
|
12831
|
-
}))
|
|
12832
|
-
);
|
|
12833
|
-
} catch (dbError) {
|
|
12834
|
-
this.rebuildVectorStoreExcludingChunkIds(
|
|
12835
|
-
store,
|
|
12836
|
-
database,
|
|
12837
|
-
successfulResults.map(({ chunk }) => chunk.id)
|
|
13500
|
+
try {
|
|
13501
|
+
const retryableChunks = this.iterateLatestFailedChunks(
|
|
13502
|
+
failedProcessing.latestById,
|
|
13503
|
+
roots,
|
|
13504
|
+
() => true,
|
|
13505
|
+
maxChunkTokens
|
|
13506
|
+
);
|
|
13507
|
+
for (const retryBatch of iterateOrderedFileBatches(
|
|
13508
|
+
retryableChunks,
|
|
13509
|
+
({ chunk }) => Buffer.byteLength(chunk.content, "utf-8"),
|
|
13510
|
+
this.fileBatchLimits
|
|
13511
|
+
)) {
|
|
13512
|
+
const chunks = retryBatch.map(({ chunk }) => chunk);
|
|
13513
|
+
const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount]));
|
|
13514
|
+
const batchResult = await this.processPendingChunkBatch(chunks, {
|
|
13515
|
+
store,
|
|
13516
|
+
provider,
|
|
13517
|
+
invertedIndex,
|
|
13518
|
+
database,
|
|
13519
|
+
configuredProviderInfo,
|
|
13520
|
+
queue,
|
|
13521
|
+
providerRateLimits,
|
|
13522
|
+
rateLimitState,
|
|
13523
|
+
failedState: failedProcessing.state,
|
|
13524
|
+
attemptCounts,
|
|
13525
|
+
forceReembed: false,
|
|
13526
|
+
reuseCachedEmbeddings: false,
|
|
13527
|
+
incrementRepeatedFailures: false,
|
|
13528
|
+
onSucceeded: (succeededChunks) => {
|
|
13529
|
+
database.addChunksToBranchBatch(
|
|
13530
|
+
this.getBranchCatalogKey(),
|
|
13531
|
+
succeededChunks.map((chunk) => chunk.id)
|
|
12838
13532
|
);
|
|
12839
|
-
throw dbError;
|
|
12840
13533
|
}
|
|
12841
|
-
}
|
|
12842
|
-
|
|
12843
|
-
|
|
12844
|
-
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
12845
|
-
completedChunkIds.add(chunk.id);
|
|
12846
|
-
embeddingPartsByChunk.delete(chunk.id);
|
|
12847
|
-
}
|
|
12848
|
-
database.addChunksToBranchBatch(
|
|
12849
|
-
this.getBranchCatalogKey(),
|
|
12850
|
-
successfulResults.map(({ chunk }) => chunk.id)
|
|
12851
|
-
);
|
|
12852
|
-
this.logger.recordChunksEmbedded(successfulResults.length);
|
|
12853
|
-
succeeded += successfulResults.length;
|
|
12854
|
-
stillFailing.push(...failedChunksForBatch.values());
|
|
12855
|
-
} catch (error) {
|
|
12856
|
-
const failureMessage = getErrorMessage4(error);
|
|
12857
|
-
const failureTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
12858
|
-
const unaccountedChunks = batch.chunks.filter(
|
|
12859
|
-
(chunk) => !failedChunksForBatch.has(chunk.id) && !completedChunkIds.has(chunk.id)
|
|
12860
|
-
);
|
|
12861
|
-
for (const chunk of unaccountedChunks) {
|
|
12862
|
-
failedChunksForBatch.set(chunk.id, {
|
|
12863
|
-
chunks: [chunk],
|
|
12864
|
-
attemptCount: batch.attemptCount + 1,
|
|
12865
|
-
lastAttempt: failureTimestamp,
|
|
12866
|
-
error: failureMessage
|
|
12867
|
-
});
|
|
12868
|
-
}
|
|
12869
|
-
failed += unaccountedChunks.length;
|
|
12870
|
-
this.logger.recordEmbeddingError();
|
|
12871
|
-
stillFailing.push(...coalesceFailedBatches(Array.from(failedChunksForBatch.values())));
|
|
13534
|
+
});
|
|
13535
|
+
succeeded += batchResult.indexedChunks;
|
|
13536
|
+
failed += batchResult.failedChunks;
|
|
12872
13537
|
}
|
|
13538
|
+
this.finalizeFailedBatchWriteState(failedProcessing.state);
|
|
13539
|
+
} catch (error) {
|
|
13540
|
+
failedProcessing.state.writer.cleanup();
|
|
13541
|
+
throw error;
|
|
12873
13542
|
}
|
|
12874
|
-
const
|
|
12875
|
-
if (roots) {
|
|
12876
|
-
this.saveFailedBatches([...retainedFailedBatches, ...persistedStillFailing]);
|
|
12877
|
-
} else {
|
|
12878
|
-
this.saveFailedBatches(persistedStillFailing);
|
|
12879
|
-
}
|
|
13543
|
+
const remaining = this.getFailedBatchesCount();
|
|
12880
13544
|
if (succeeded > 0) {
|
|
12881
13545
|
store.save();
|
|
12882
13546
|
this.saveInvertedIndex(invertedIndex);
|
|
12883
13547
|
}
|
|
12884
|
-
if (roots && succeeded > 0 &&
|
|
13548
|
+
if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) {
|
|
12885
13549
|
database.deleteMetadata(this.getProjectForceReembedMetadataKey());
|
|
12886
13550
|
this.saveIndexMetadata(configuredProviderInfo);
|
|
12887
13551
|
this.indexCompatibility = { compatible: true };
|
|
12888
13552
|
}
|
|
12889
|
-
return { succeeded, failed, remaining
|
|
13553
|
+
return { succeeded, failed, remaining };
|
|
12890
13554
|
}
|
|
12891
13555
|
getFailedBatchesCount() {
|
|
12892
|
-
|
|
12893
|
-
|
|
13556
|
+
const roots = this.config.scope === "global" ? this.getScopedRoots() : null;
|
|
13557
|
+
const latestById = /* @__PURE__ */ new Map();
|
|
13558
|
+
for (const batch of this.loadSerializedFailedBatches()) {
|
|
13559
|
+
for (const rawChunk of batch.chunks) {
|
|
13560
|
+
const filePath = getPendingChunkFilePath(rawChunk);
|
|
13561
|
+
if (roots && (filePath === null || !this.isFileInCurrentScope(filePath, roots))) {
|
|
13562
|
+
continue;
|
|
13563
|
+
}
|
|
13564
|
+
const chunkId = getPendingChunkId(rawChunk);
|
|
13565
|
+
if (!chunkId) {
|
|
13566
|
+
continue;
|
|
13567
|
+
}
|
|
13568
|
+
const existing = latestById.get(chunkId);
|
|
13569
|
+
if (!existing || batch.attemptCount >= existing.attemptCount) {
|
|
13570
|
+
latestById.set(chunkId, {
|
|
13571
|
+
attemptCount: batch.attemptCount,
|
|
13572
|
+
error: batch.error,
|
|
13573
|
+
lastAttempt: batch.lastAttempt
|
|
13574
|
+
});
|
|
13575
|
+
}
|
|
13576
|
+
}
|
|
12894
13577
|
}
|
|
12895
|
-
return
|
|
13578
|
+
return new Set(Array.from(latestById.values(), getFailedBatchGroupKey)).size;
|
|
12896
13579
|
}
|
|
12897
13580
|
getCurrentBranch() {
|
|
12898
13581
|
return this.currentBranch;
|
|
@@ -13080,9 +13763,9 @@ var Indexer = class _Indexer {
|
|
|
13080
13763
|
this.requireReadableComponents(readIssues, "database");
|
|
13081
13764
|
let shortest = [];
|
|
13082
13765
|
for (const branchKey of this.getBranchCatalogKeys()) {
|
|
13083
|
-
const
|
|
13084
|
-
if (
|
|
13085
|
-
shortest =
|
|
13766
|
+
const path28 = database.findShortestPath(fromName, toName, branchKey, maxDepth);
|
|
13767
|
+
if (path28.length > 0 && (shortest.length === 0 || path28.length < shortest.length)) {
|
|
13768
|
+
shortest = path28;
|
|
13086
13769
|
}
|
|
13087
13770
|
}
|
|
13088
13771
|
return shortest.map((hop) => this.resolveFilePathRecord(hop));
|
|
@@ -13130,13 +13813,13 @@ var Indexer = class _Indexer {
|
|
|
13130
13813
|
}
|
|
13131
13814
|
}
|
|
13132
13815
|
if (!found) continue;
|
|
13133
|
-
const
|
|
13816
|
+
const path28 = [];
|
|
13134
13817
|
let currentSymbolId = toSymbolId;
|
|
13135
13818
|
while (true) {
|
|
13136
13819
|
const symbol = symbolsById.get(currentSymbolId);
|
|
13137
13820
|
if (!symbol) break;
|
|
13138
13821
|
const parent = parentBySymbolId.get(currentSymbolId);
|
|
13139
|
-
|
|
13822
|
+
path28.push({
|
|
13140
13823
|
symbolId: symbol.id,
|
|
13141
13824
|
symbolName: symbol.name,
|
|
13142
13825
|
filePath: symbol.filePath,
|
|
@@ -13146,9 +13829,9 @@ var Indexer = class _Indexer {
|
|
|
13146
13829
|
if (!parent) break;
|
|
13147
13830
|
currentSymbolId = parent.parentId;
|
|
13148
13831
|
}
|
|
13149
|
-
|
|
13150
|
-
if (
|
|
13151
|
-
shortest =
|
|
13832
|
+
path28.reverse();
|
|
13833
|
+
if (path28.length > 0 && (shortest.length === 0 || path28.length < shortest.length)) {
|
|
13834
|
+
shortest = path28;
|
|
13152
13835
|
}
|
|
13153
13836
|
}
|
|
13154
13837
|
return shortest.map((hop) => this.resolveFilePathRecord(hop));
|
|
@@ -13167,13 +13850,13 @@ var Indexer = class _Indexer {
|
|
|
13167
13850
|
async getSymbolsForBranch(branch) {
|
|
13168
13851
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13169
13852
|
this.requireReadableComponents(readIssues, "database");
|
|
13170
|
-
const resolvedBranch =
|
|
13853
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13171
13854
|
return database.getSymbolsForBranch(resolvedBranch).map((symbol) => this.resolveFilePathRecord(symbol));
|
|
13172
13855
|
}
|
|
13173
13856
|
async getSymbolsForFiles(filePaths, branch) {
|
|
13174
13857
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13175
13858
|
this.requireReadableComponents(readIssues, "database");
|
|
13176
|
-
const resolvedBranch =
|
|
13859
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13177
13860
|
const storedFilePaths = filePaths.map((filePath) => this.toStoredFilePath(filePath));
|
|
13178
13861
|
return database.getSymbolsForFiles(storedFilePaths, resolvedBranch).map((symbol) => this.resolveFilePathRecord(symbol));
|
|
13179
13862
|
}
|
|
@@ -13186,13 +13869,26 @@ var Indexer = class _Indexer {
|
|
|
13186
13869
|
async detectCommunities(branch, symbolIds) {
|
|
13187
13870
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13188
13871
|
this.requireReadableComponents(readIssues, "database");
|
|
13189
|
-
const resolvedBranch =
|
|
13872
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13190
13873
|
return database.detectCommunities(resolvedBranch, symbolIds).map((entry) => this.resolveFilePathRecord(entry));
|
|
13191
13874
|
}
|
|
13875
|
+
async detectCommunityCouplings(branch) {
|
|
13876
|
+
const { database, readIssues } = await this.ensureInitialized();
|
|
13877
|
+
this.requireReadableComponents(readIssues, "database");
|
|
13878
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13879
|
+
return database.detectCommunityCouplings(resolvedBranch).map((entry) => ({
|
|
13880
|
+
...entry,
|
|
13881
|
+
relationships: (entry.relationships ?? entry.representativeRelationships ?? []).map((relationship) => ({
|
|
13882
|
+
...relationship,
|
|
13883
|
+
fromFilePath: this.resolveStoredFilePath(relationship.fromFilePath),
|
|
13884
|
+
toFilePath: this.resolveStoredFilePath(relationship.toFilePath)
|
|
13885
|
+
}))
|
|
13886
|
+
}));
|
|
13887
|
+
}
|
|
13192
13888
|
async computeCentrality(branch) {
|
|
13193
13889
|
const { database, readIssues } = await this.ensureInitialized();
|
|
13194
13890
|
this.requireReadableComponents(readIssues, "database");
|
|
13195
|
-
const resolvedBranch =
|
|
13891
|
+
const resolvedBranch = this.resolveBranchCatalogKey(branch);
|
|
13196
13892
|
return database.computeCentrality(resolvedBranch).map((entry) => this.resolveFilePathRecord(entry));
|
|
13197
13893
|
}
|
|
13198
13894
|
async getPrImpact(opts, onPreparationProgress) {
|
|
@@ -13286,7 +13982,7 @@ var Indexer = class _Indexer {
|
|
|
13286
13982
|
);
|
|
13287
13983
|
}
|
|
13288
13984
|
}
|
|
13289
|
-
const toStoredChangedFiles = (filePaths) => filePaths.map((filePath) => this.toStoredFilePath(
|
|
13985
|
+
const toStoredChangedFiles = (filePaths) => filePaths.map((filePath) => this.toStoredFilePath(path19.resolve(this.projectRoot, filePath)));
|
|
13290
13986
|
const storedChangedFiles = toStoredChangedFiles(changedFiles);
|
|
13291
13987
|
const directSymbols = database.getSymbolsForFiles(storedChangedFiles, branchKey);
|
|
13292
13988
|
const directIds = directSymbols.map((s) => s.id);
|
|
@@ -13435,12 +14131,12 @@ var Indexer = class _Indexer {
|
|
|
13435
14131
|
if (meta.filePath) filePaths.add(meta.filePath);
|
|
13436
14132
|
}
|
|
13437
14133
|
const directory = options?.directory?.replace(/\/$/, "");
|
|
13438
|
-
const absoluteDirectoryFilter = directory ?
|
|
14134
|
+
const absoluteDirectoryFilter = directory ? path19.resolve(this.projectRoot, directory) : void 0;
|
|
13439
14135
|
for (const filePath of filePaths) {
|
|
13440
14136
|
if (directory) {
|
|
13441
14137
|
const absoluteFilePath = this.resolveStoredFilePath(filePath);
|
|
13442
14138
|
const matchesRelative = filePath === directory || filePath.startsWith(directory + "/");
|
|
13443
|
-
const matchesProjectRelative = absoluteDirectoryFilter !== void 0 && (absoluteFilePath === absoluteDirectoryFilter || absoluteFilePath.startsWith(absoluteDirectoryFilter +
|
|
14139
|
+
const matchesProjectRelative = absoluteDirectoryFilter !== void 0 && (absoluteFilePath === absoluteDirectoryFilter || absoluteFilePath.startsWith(absoluteDirectoryFilter + path19.sep));
|
|
13444
14140
|
if (!matchesRelative && !matchesProjectRelative) {
|
|
13445
14141
|
continue;
|
|
13446
14142
|
}
|
|
@@ -13607,7 +14303,7 @@ function trimOrUndefined(value) {
|
|
|
13607
14303
|
return normalized || void 0;
|
|
13608
14304
|
}
|
|
13609
14305
|
function normalizeCallGraphPath(value) {
|
|
13610
|
-
let normalized =
|
|
14306
|
+
let normalized = path20.posix.normalize(value.trim().replaceAll("\\", "/"));
|
|
13611
14307
|
if (normalized.startsWith("./")) {
|
|
13612
14308
|
normalized = normalized.slice(2);
|
|
13613
14309
|
}
|
|
@@ -13791,12 +14487,12 @@ async function getCallGraphPath(projectRoot, host, from, to, maxDepth, fromFileP
|
|
|
13791
14487
|
if (fromResolution.status !== "resolved" || toResolution.status !== "resolved") {
|
|
13792
14488
|
return { from: fromResolution, to: toResolution, path: [] };
|
|
13793
14489
|
}
|
|
13794
|
-
const
|
|
14490
|
+
const path28 = await indexer.findCallPathBySymbolIds(
|
|
13795
14491
|
fromResolution.symbolId,
|
|
13796
14492
|
toResolution.symbolId,
|
|
13797
14493
|
maxDepth
|
|
13798
14494
|
);
|
|
13799
|
-
return { from: fromResolution, to: toResolution, path:
|
|
14495
|
+
return { from: fromResolution, to: toResolution, path: path28 };
|
|
13800
14496
|
}
|
|
13801
14497
|
async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
13802
14498
|
const root = getProjectRoot(projectRoot, host);
|
|
@@ -13860,6 +14556,34 @@ async function runIndexHealthCheck(projectRoot, host) {
|
|
|
13860
14556
|
return busyResult;
|
|
13861
14557
|
}
|
|
13862
14558
|
}
|
|
14559
|
+
async function getCodeCommunities(projectRoot, host, params) {
|
|
14560
|
+
await ensureAutoIndexReadyForRetrieval(projectRoot, host);
|
|
14561
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
14562
|
+
const [communities, centrality, couplings] = await Promise.all([
|
|
14563
|
+
indexer.detectCommunities(params.branch),
|
|
14564
|
+
indexer.computeCentrality(params.branch),
|
|
14565
|
+
indexer.detectCommunityCouplings(params.branch)
|
|
14566
|
+
]);
|
|
14567
|
+
return buildCodeCommunitiesResult(communities, centrality, couplings, {
|
|
14568
|
+
minSize: Math.max(CODE_COMMUNITIES_MIN_SIZE, Math.floor(params.minSize ?? CODE_COMMUNITIES_MIN_SIZE)),
|
|
14569
|
+
limit: Math.min(
|
|
14570
|
+
CODE_COMMUNITIES_MAX_LIMIT,
|
|
14571
|
+
Math.max(1, Math.floor(params.limit ?? CODE_COMMUNITIES_DEFAULT_LIMIT))
|
|
14572
|
+
),
|
|
14573
|
+
hubThreshold: Math.max(
|
|
14574
|
+
0,
|
|
14575
|
+
Math.floor(params.hubThreshold ?? CODE_COMMUNITIES_DEFAULT_HUB_THRESHOLD)
|
|
14576
|
+
),
|
|
14577
|
+
minCoupling: Math.max(
|
|
14578
|
+
CODE_COMMUNITIES_MIN_COUPLING,
|
|
14579
|
+
Math.floor(params.minCoupling ?? CODE_COMMUNITIES_MIN_COUPLING)
|
|
14580
|
+
),
|
|
14581
|
+
couplingLimit: Math.min(
|
|
14582
|
+
CODE_COMMUNITIES_MAX_COUPLING_LIMIT,
|
|
14583
|
+
Math.max(1, Math.floor(params.couplingLimit ?? CODE_COMMUNITIES_DEFAULT_COUPLING_LIMIT))
|
|
14584
|
+
)
|
|
14585
|
+
});
|
|
14586
|
+
}
|
|
13863
14587
|
async function getIndexMetrics(projectRoot, host, args = {}) {
|
|
13864
14588
|
const root = getProjectRoot(projectRoot, host);
|
|
13865
14589
|
const key = getIndexerCacheKey(root, host);
|
|
@@ -13951,10 +14675,10 @@ async function getIndexLogs(projectRoot, host, args) {
|
|
|
13951
14675
|
function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
13952
14676
|
const root = getProjectRoot(projectRoot, host);
|
|
13953
14677
|
const inputPath = knowledgeBasePath.trim();
|
|
13954
|
-
const normalizedPath =
|
|
13955
|
-
|
|
14678
|
+
const normalizedPath = path20.resolve(
|
|
14679
|
+
path20.isAbsolute(inputPath) ? inputPath : resolveKnowledgeBasePath(inputPath, root)
|
|
13956
14680
|
);
|
|
13957
|
-
if (!
|
|
14681
|
+
if (!existsSync12(normalizedPath)) {
|
|
13958
14682
|
return `Error: Directory does not exist: ${normalizedPath}`;
|
|
13959
14683
|
}
|
|
13960
14684
|
let realPath;
|
|
@@ -13988,7 +14712,7 @@ function addKnowledgeBase(projectRoot, host, knowledgeBasePath) {
|
|
|
13988
14712
|
}
|
|
13989
14713
|
}
|
|
13990
14714
|
for (const dotDir of sensitiveDotDirs) {
|
|
13991
|
-
const sensitiveDir =
|
|
14715
|
+
const sensitiveDir = path20.join(homeDir, dotDir);
|
|
13992
14716
|
if (sensitiveDir && (realPath === sensitiveDir || realPath.startsWith(`${sensitiveDir}/`))) {
|
|
13993
14717
|
return `Error: Adding sensitive directory as knowledge base is not allowed: ${normalizedPath}`;
|
|
13994
14718
|
}
|
|
@@ -14034,7 +14758,7 @@ function listKnowledgeBases(projectRoot, host) {
|
|
|
14034
14758
|
for (let i = 0; i < knowledgeBases.length; i++) {
|
|
14035
14759
|
const kb = knowledgeBases[i];
|
|
14036
14760
|
const resolvedPath = resolveKnowledgeBasePath(kb, root);
|
|
14037
|
-
const exists =
|
|
14761
|
+
const exists = existsSync12(resolvedPath);
|
|
14038
14762
|
result += `[${i + 1}] ${kb}
|
|
14039
14763
|
`;
|
|
14040
14764
|
result += ` Resolved: ${resolvedPath}
|
|
@@ -14051,7 +14775,7 @@ function listKnowledgeBases(projectRoot, host) {
|
|
|
14051
14775
|
}
|
|
14052
14776
|
result += "\n";
|
|
14053
14777
|
}
|
|
14054
|
-
const hasHostConfig =
|
|
14778
|
+
const hasHostConfig = existsSync12(path20.join(root, getHostProjectConfigRelativePath(host)));
|
|
14055
14779
|
if (hasHostConfig) {
|
|
14056
14780
|
result += `
|
|
14057
14781
|
Config sources: 1 file(s).`;
|
|
@@ -14085,7 +14809,7 @@ Run /index to rebuild the index without the removed knowledge base.`;
|
|
|
14085
14809
|
}
|
|
14086
14810
|
|
|
14087
14811
|
// src/watcher/file-watcher.ts
|
|
14088
|
-
import { existsSync as
|
|
14812
|
+
import { existsSync as existsSync13 } from "fs";
|
|
14089
14813
|
|
|
14090
14814
|
// node_modules/chokidar/index.js
|
|
14091
14815
|
import { EventEmitter as EventEmitter2 } from "events";
|
|
@@ -14177,7 +14901,7 @@ var ReaddirpStream = class extends Readable {
|
|
|
14177
14901
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
14178
14902
|
const statMethod = opts.lstat ? lstat : stat;
|
|
14179
14903
|
if (wantBigintFsStats) {
|
|
14180
|
-
this._stat = (
|
|
14904
|
+
this._stat = (path28) => statMethod(path28, { bigint: true });
|
|
14181
14905
|
} else {
|
|
14182
14906
|
this._stat = statMethod;
|
|
14183
14907
|
}
|
|
@@ -14202,8 +14926,8 @@ var ReaddirpStream = class extends Readable {
|
|
|
14202
14926
|
const par = this.parent;
|
|
14203
14927
|
const fil = par && par.files;
|
|
14204
14928
|
if (fil && fil.length > 0) {
|
|
14205
|
-
const { path:
|
|
14206
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
14929
|
+
const { path: path28, depth } = par;
|
|
14930
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path28));
|
|
14207
14931
|
const awaited = await Promise.all(slice);
|
|
14208
14932
|
for (const entry of awaited) {
|
|
14209
14933
|
if (!entry)
|
|
@@ -14243,21 +14967,21 @@ var ReaddirpStream = class extends Readable {
|
|
|
14243
14967
|
this.reading = false;
|
|
14244
14968
|
}
|
|
14245
14969
|
}
|
|
14246
|
-
async _exploreDir(
|
|
14970
|
+
async _exploreDir(path28, depth) {
|
|
14247
14971
|
let files;
|
|
14248
14972
|
try {
|
|
14249
|
-
files = await readdir(
|
|
14973
|
+
files = await readdir(path28, this._rdOptions);
|
|
14250
14974
|
} catch (error) {
|
|
14251
14975
|
this._onError(error);
|
|
14252
14976
|
}
|
|
14253
|
-
return { files, depth, path:
|
|
14977
|
+
return { files, depth, path: path28 };
|
|
14254
14978
|
}
|
|
14255
|
-
async _formatEntry(dirent,
|
|
14979
|
+
async _formatEntry(dirent, path28) {
|
|
14256
14980
|
let entry;
|
|
14257
|
-
const
|
|
14981
|
+
const basename9 = this._isDirent ? dirent.name : dirent;
|
|
14258
14982
|
try {
|
|
14259
|
-
const fullPath = presolve(pjoin(
|
|
14260
|
-
entry = { path: prelative(this._root, fullPath), fullPath, basename:
|
|
14983
|
+
const fullPath = presolve(pjoin(path28, basename9));
|
|
14984
|
+
entry = { path: prelative(this._root, fullPath), fullPath, basename: basename9 };
|
|
14261
14985
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
14262
14986
|
} catch (err) {
|
|
14263
14987
|
this._onError(err);
|
|
@@ -14656,16 +15380,16 @@ var delFromSet = (main, prop, item) => {
|
|
|
14656
15380
|
};
|
|
14657
15381
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
14658
15382
|
var FsWatchInstances = /* @__PURE__ */ new Map();
|
|
14659
|
-
function createFsWatchInstance(
|
|
15383
|
+
function createFsWatchInstance(path28, options, listener, errHandler, emitRaw) {
|
|
14660
15384
|
const handleEvent = (rawEvent, evPath) => {
|
|
14661
|
-
listener(
|
|
14662
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
14663
|
-
if (evPath &&
|
|
14664
|
-
fsWatchBroadcast(sp.resolve(
|
|
15385
|
+
listener(path28);
|
|
15386
|
+
emitRaw(rawEvent, evPath, { watchedPath: path28 });
|
|
15387
|
+
if (evPath && path28 !== evPath) {
|
|
15388
|
+
fsWatchBroadcast(sp.resolve(path28, evPath), KEY_LISTENERS, sp.join(path28, evPath));
|
|
14665
15389
|
}
|
|
14666
15390
|
};
|
|
14667
15391
|
try {
|
|
14668
|
-
return fs_watch(
|
|
15392
|
+
return fs_watch(path28, {
|
|
14669
15393
|
persistent: options.persistent
|
|
14670
15394
|
}, handleEvent);
|
|
14671
15395
|
} catch (error) {
|
|
@@ -14681,12 +15405,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
14681
15405
|
listener(val1, val2, val3);
|
|
14682
15406
|
});
|
|
14683
15407
|
};
|
|
14684
|
-
var setFsWatchListener = (
|
|
15408
|
+
var setFsWatchListener = (path28, fullPath, options, handlers) => {
|
|
14685
15409
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
14686
15410
|
let cont = FsWatchInstances.get(fullPath);
|
|
14687
15411
|
let watcher;
|
|
14688
15412
|
if (!options.persistent) {
|
|
14689
|
-
watcher = createFsWatchInstance(
|
|
15413
|
+
watcher = createFsWatchInstance(path28, options, listener, errHandler, rawEmitter);
|
|
14690
15414
|
if (!watcher)
|
|
14691
15415
|
return;
|
|
14692
15416
|
return watcher.close.bind(watcher);
|
|
@@ -14697,7 +15421,7 @@ var setFsWatchListener = (path27, fullPath, options, handlers) => {
|
|
|
14697
15421
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
14698
15422
|
} else {
|
|
14699
15423
|
watcher = createFsWatchInstance(
|
|
14700
|
-
|
|
15424
|
+
path28,
|
|
14701
15425
|
options,
|
|
14702
15426
|
fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS),
|
|
14703
15427
|
errHandler,
|
|
@@ -14712,7 +15436,7 @@ var setFsWatchListener = (path27, fullPath, options, handlers) => {
|
|
|
14712
15436
|
cont.watcherUnusable = true;
|
|
14713
15437
|
if (isWindows && error.code === "EPERM") {
|
|
14714
15438
|
try {
|
|
14715
|
-
const fd = await open(
|
|
15439
|
+
const fd = await open(path28, "r");
|
|
14716
15440
|
await fd.close();
|
|
14717
15441
|
broadcastErr(error);
|
|
14718
15442
|
} catch (err) {
|
|
@@ -14743,7 +15467,7 @@ var setFsWatchListener = (path27, fullPath, options, handlers) => {
|
|
|
14743
15467
|
};
|
|
14744
15468
|
};
|
|
14745
15469
|
var FsWatchFileInstances = /* @__PURE__ */ new Map();
|
|
14746
|
-
var setFsWatchFileListener = (
|
|
15470
|
+
var setFsWatchFileListener = (path28, fullPath, options, handlers) => {
|
|
14747
15471
|
const { listener, rawEmitter } = handlers;
|
|
14748
15472
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
14749
15473
|
const copts = cont && cont.options;
|
|
@@ -14765,7 +15489,7 @@ var setFsWatchFileListener = (path27, fullPath, options, handlers) => {
|
|
|
14765
15489
|
});
|
|
14766
15490
|
const currmtime = curr.mtimeMs;
|
|
14767
15491
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
14768
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
15492
|
+
foreach(cont.listeners, (listener2) => listener2(path28, curr));
|
|
14769
15493
|
}
|
|
14770
15494
|
})
|
|
14771
15495
|
};
|
|
@@ -14795,13 +15519,13 @@ var NodeFsHandler = class {
|
|
|
14795
15519
|
* @param listener on fs change
|
|
14796
15520
|
* @returns closer for the watcher instance
|
|
14797
15521
|
*/
|
|
14798
|
-
_watchWithNodeFs(
|
|
15522
|
+
_watchWithNodeFs(path28, listener) {
|
|
14799
15523
|
const opts = this.fsw.options;
|
|
14800
|
-
const directory = sp.dirname(
|
|
14801
|
-
const
|
|
15524
|
+
const directory = sp.dirname(path28);
|
|
15525
|
+
const basename9 = sp.basename(path28);
|
|
14802
15526
|
const parent = this.fsw._getWatchedDir(directory);
|
|
14803
|
-
parent.add(
|
|
14804
|
-
const absolutePath = sp.resolve(
|
|
15527
|
+
parent.add(basename9);
|
|
15528
|
+
const absolutePath = sp.resolve(path28);
|
|
14805
15529
|
const options = {
|
|
14806
15530
|
persistent: opts.persistent
|
|
14807
15531
|
};
|
|
@@ -14810,13 +15534,13 @@ var NodeFsHandler = class {
|
|
|
14810
15534
|
let closer;
|
|
14811
15535
|
if (opts.usePolling) {
|
|
14812
15536
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
14813
|
-
options.interval = enableBin && isBinaryPath(
|
|
14814
|
-
closer = setFsWatchFileListener(
|
|
15537
|
+
options.interval = enableBin && isBinaryPath(basename9) ? opts.binaryInterval : opts.interval;
|
|
15538
|
+
closer = setFsWatchFileListener(path28, absolutePath, options, {
|
|
14815
15539
|
listener,
|
|
14816
15540
|
rawEmitter: this.fsw._emitRaw
|
|
14817
15541
|
});
|
|
14818
15542
|
} else {
|
|
14819
|
-
closer = setFsWatchListener(
|
|
15543
|
+
closer = setFsWatchListener(path28, absolutePath, options, {
|
|
14820
15544
|
listener,
|
|
14821
15545
|
errHandler: this._boundHandleError,
|
|
14822
15546
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -14832,13 +15556,13 @@ var NodeFsHandler = class {
|
|
|
14832
15556
|
if (this.fsw.closed) {
|
|
14833
15557
|
return;
|
|
14834
15558
|
}
|
|
14835
|
-
const
|
|
14836
|
-
const
|
|
14837
|
-
const parent = this.fsw._getWatchedDir(
|
|
15559
|
+
const dirname15 = sp.dirname(file);
|
|
15560
|
+
const basename9 = sp.basename(file);
|
|
15561
|
+
const parent = this.fsw._getWatchedDir(dirname15);
|
|
14838
15562
|
let prevStats = stats;
|
|
14839
|
-
if (parent.has(
|
|
15563
|
+
if (parent.has(basename9))
|
|
14840
15564
|
return;
|
|
14841
|
-
const listener = async (
|
|
15565
|
+
const listener = async (path28, newStats) => {
|
|
14842
15566
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
14843
15567
|
return;
|
|
14844
15568
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -14852,18 +15576,18 @@ var NodeFsHandler = class {
|
|
|
14852
15576
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
14853
15577
|
}
|
|
14854
15578
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
14855
|
-
this.fsw._closeFile(
|
|
15579
|
+
this.fsw._closeFile(path28);
|
|
14856
15580
|
prevStats = newStats2;
|
|
14857
15581
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
14858
15582
|
if (closer2)
|
|
14859
|
-
this.fsw._addPathCloser(
|
|
15583
|
+
this.fsw._addPathCloser(path28, closer2);
|
|
14860
15584
|
} else {
|
|
14861
15585
|
prevStats = newStats2;
|
|
14862
15586
|
}
|
|
14863
15587
|
} catch (error) {
|
|
14864
|
-
this.fsw._remove(
|
|
15588
|
+
this.fsw._remove(dirname15, basename9);
|
|
14865
15589
|
}
|
|
14866
|
-
} else if (parent.has(
|
|
15590
|
+
} else if (parent.has(basename9)) {
|
|
14867
15591
|
const at = newStats.atimeMs;
|
|
14868
15592
|
const mt = newStats.mtimeMs;
|
|
14869
15593
|
if (!at || at <= mt || mt !== prevStats.mtimeMs) {
|
|
@@ -14888,7 +15612,7 @@ var NodeFsHandler = class {
|
|
|
14888
15612
|
* @param item basename of this item
|
|
14889
15613
|
* @returns true if no more processing is needed for this entry.
|
|
14890
15614
|
*/
|
|
14891
|
-
async _handleSymlink(entry, directory,
|
|
15615
|
+
async _handleSymlink(entry, directory, path28, item) {
|
|
14892
15616
|
if (this.fsw.closed) {
|
|
14893
15617
|
return;
|
|
14894
15618
|
}
|
|
@@ -14898,7 +15622,7 @@ var NodeFsHandler = class {
|
|
|
14898
15622
|
this.fsw._incrReadyCount();
|
|
14899
15623
|
let linkPath;
|
|
14900
15624
|
try {
|
|
14901
|
-
linkPath = await fsrealpath(
|
|
15625
|
+
linkPath = await fsrealpath(path28);
|
|
14902
15626
|
} catch (e) {
|
|
14903
15627
|
this.fsw._emitReady();
|
|
14904
15628
|
return true;
|
|
@@ -14908,12 +15632,12 @@ var NodeFsHandler = class {
|
|
|
14908
15632
|
if (dir.has(item)) {
|
|
14909
15633
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
14910
15634
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
14911
|
-
this.fsw._emit(EV.CHANGE,
|
|
15635
|
+
this.fsw._emit(EV.CHANGE, path28, entry.stats);
|
|
14912
15636
|
}
|
|
14913
15637
|
} else {
|
|
14914
15638
|
dir.add(item);
|
|
14915
15639
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
14916
|
-
this.fsw._emit(EV.ADD,
|
|
15640
|
+
this.fsw._emit(EV.ADD, path28, entry.stats);
|
|
14917
15641
|
}
|
|
14918
15642
|
this.fsw._emitReady();
|
|
14919
15643
|
return true;
|
|
@@ -14943,9 +15667,9 @@ var NodeFsHandler = class {
|
|
|
14943
15667
|
return;
|
|
14944
15668
|
}
|
|
14945
15669
|
const item = entry.path;
|
|
14946
|
-
let
|
|
15670
|
+
let path28 = sp.join(directory, item);
|
|
14947
15671
|
current.add(item);
|
|
14948
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
15672
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path28, item)) {
|
|
14949
15673
|
return;
|
|
14950
15674
|
}
|
|
14951
15675
|
if (this.fsw.closed) {
|
|
@@ -14954,8 +15678,8 @@ var NodeFsHandler = class {
|
|
|
14954
15678
|
}
|
|
14955
15679
|
if (item === target || !target && !previous.has(item)) {
|
|
14956
15680
|
this.fsw._incrReadyCount();
|
|
14957
|
-
|
|
14958
|
-
this._addToNodeFs(
|
|
15681
|
+
path28 = sp.join(dir, sp.relative(dir, path28));
|
|
15682
|
+
this._addToNodeFs(path28, initialAdd, wh, depth + 1);
|
|
14959
15683
|
}
|
|
14960
15684
|
}).on(EV.ERROR, this._boundHandleError);
|
|
14961
15685
|
return new Promise((resolve15, reject) => {
|
|
@@ -15024,13 +15748,13 @@ var NodeFsHandler = class {
|
|
|
15024
15748
|
* @param depth Child path actually targeted for watch
|
|
15025
15749
|
* @param target Child path actually targeted for watch
|
|
15026
15750
|
*/
|
|
15027
|
-
async _addToNodeFs(
|
|
15751
|
+
async _addToNodeFs(path28, initialAdd, priorWh, depth, target) {
|
|
15028
15752
|
const ready = this.fsw._emitReady;
|
|
15029
|
-
if (this.fsw._isIgnored(
|
|
15753
|
+
if (this.fsw._isIgnored(path28) || this.fsw.closed) {
|
|
15030
15754
|
ready();
|
|
15031
15755
|
return false;
|
|
15032
15756
|
}
|
|
15033
|
-
const wh = this.fsw._getWatchHelpers(
|
|
15757
|
+
const wh = this.fsw._getWatchHelpers(path28);
|
|
15034
15758
|
if (priorWh) {
|
|
15035
15759
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
15036
15760
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -15046,8 +15770,8 @@ var NodeFsHandler = class {
|
|
|
15046
15770
|
const follow = this.fsw.options.followSymlinks;
|
|
15047
15771
|
let closer;
|
|
15048
15772
|
if (stats.isDirectory()) {
|
|
15049
|
-
const absPath = sp.resolve(
|
|
15050
|
-
const targetPath = follow ? await fsrealpath(
|
|
15773
|
+
const absPath = sp.resolve(path28);
|
|
15774
|
+
const targetPath = follow ? await fsrealpath(path28) : path28;
|
|
15051
15775
|
if (this.fsw.closed)
|
|
15052
15776
|
return;
|
|
15053
15777
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -15057,29 +15781,29 @@ var NodeFsHandler = class {
|
|
|
15057
15781
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
15058
15782
|
}
|
|
15059
15783
|
} else if (stats.isSymbolicLink()) {
|
|
15060
|
-
const targetPath = follow ? await fsrealpath(
|
|
15784
|
+
const targetPath = follow ? await fsrealpath(path28) : path28;
|
|
15061
15785
|
if (this.fsw.closed)
|
|
15062
15786
|
return;
|
|
15063
15787
|
const parent = sp.dirname(wh.watchPath);
|
|
15064
15788
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
15065
15789
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
15066
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
15790
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path28, wh, targetPath);
|
|
15067
15791
|
if (this.fsw.closed)
|
|
15068
15792
|
return;
|
|
15069
15793
|
if (targetPath !== void 0) {
|
|
15070
|
-
this.fsw._symlinkPaths.set(sp.resolve(
|
|
15794
|
+
this.fsw._symlinkPaths.set(sp.resolve(path28), targetPath);
|
|
15071
15795
|
}
|
|
15072
15796
|
} else {
|
|
15073
15797
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
15074
15798
|
}
|
|
15075
15799
|
ready();
|
|
15076
15800
|
if (closer)
|
|
15077
|
-
this.fsw._addPathCloser(
|
|
15801
|
+
this.fsw._addPathCloser(path28, closer);
|
|
15078
15802
|
return false;
|
|
15079
15803
|
} catch (error) {
|
|
15080
15804
|
if (this.fsw._handleError(error)) {
|
|
15081
15805
|
ready();
|
|
15082
|
-
return
|
|
15806
|
+
return path28;
|
|
15083
15807
|
}
|
|
15084
15808
|
}
|
|
15085
15809
|
}
|
|
@@ -15122,24 +15846,24 @@ function createPattern(matcher) {
|
|
|
15122
15846
|
}
|
|
15123
15847
|
return () => false;
|
|
15124
15848
|
}
|
|
15125
|
-
function normalizePath2(
|
|
15126
|
-
if (typeof
|
|
15849
|
+
function normalizePath2(path28) {
|
|
15850
|
+
if (typeof path28 !== "string")
|
|
15127
15851
|
throw new Error("string expected");
|
|
15128
|
-
|
|
15129
|
-
|
|
15852
|
+
path28 = sp2.normalize(path28);
|
|
15853
|
+
path28 = path28.replace(/\\/g, "/");
|
|
15130
15854
|
let prepend = false;
|
|
15131
|
-
if (
|
|
15855
|
+
if (path28.startsWith("//"))
|
|
15132
15856
|
prepend = true;
|
|
15133
|
-
|
|
15857
|
+
path28 = path28.replace(DOUBLE_SLASH_RE, "/");
|
|
15134
15858
|
if (prepend)
|
|
15135
|
-
|
|
15136
|
-
return
|
|
15859
|
+
path28 = "/" + path28;
|
|
15860
|
+
return path28;
|
|
15137
15861
|
}
|
|
15138
15862
|
function matchPatterns(patterns, testString, stats) {
|
|
15139
|
-
const
|
|
15863
|
+
const path28 = normalizePath2(testString);
|
|
15140
15864
|
for (let index = 0; index < patterns.length; index++) {
|
|
15141
15865
|
const pattern = patterns[index];
|
|
15142
|
-
if (pattern(
|
|
15866
|
+
if (pattern(path28, stats)) {
|
|
15143
15867
|
return true;
|
|
15144
15868
|
}
|
|
15145
15869
|
}
|
|
@@ -15177,19 +15901,19 @@ var toUnix = (string) => {
|
|
|
15177
15901
|
}
|
|
15178
15902
|
return str;
|
|
15179
15903
|
};
|
|
15180
|
-
var normalizePathToUnix = (
|
|
15181
|
-
var normalizeIgnored = (cwd = "") => (
|
|
15182
|
-
if (typeof
|
|
15183
|
-
return normalizePathToUnix(sp2.isAbsolute(
|
|
15904
|
+
var normalizePathToUnix = (path28) => toUnix(sp2.normalize(toUnix(path28)));
|
|
15905
|
+
var normalizeIgnored = (cwd = "") => (path28) => {
|
|
15906
|
+
if (typeof path28 === "string") {
|
|
15907
|
+
return normalizePathToUnix(sp2.isAbsolute(path28) ? path28 : sp2.join(cwd, path28));
|
|
15184
15908
|
} else {
|
|
15185
|
-
return
|
|
15909
|
+
return path28;
|
|
15186
15910
|
}
|
|
15187
15911
|
};
|
|
15188
|
-
var getAbsolutePath = (
|
|
15189
|
-
if (sp2.isAbsolute(
|
|
15190
|
-
return
|
|
15912
|
+
var getAbsolutePath = (path28, cwd) => {
|
|
15913
|
+
if (sp2.isAbsolute(path28)) {
|
|
15914
|
+
return path28;
|
|
15191
15915
|
}
|
|
15192
|
-
return sp2.join(cwd,
|
|
15916
|
+
return sp2.join(cwd, path28);
|
|
15193
15917
|
};
|
|
15194
15918
|
var EMPTY_SET = Object.freeze(/* @__PURE__ */ new Set());
|
|
15195
15919
|
var DirEntry = class {
|
|
@@ -15254,10 +15978,10 @@ var WatchHelper = class {
|
|
|
15254
15978
|
dirParts;
|
|
15255
15979
|
followSymlinks;
|
|
15256
15980
|
statMethod;
|
|
15257
|
-
constructor(
|
|
15981
|
+
constructor(path28, follow, fsw) {
|
|
15258
15982
|
this.fsw = fsw;
|
|
15259
|
-
const watchPath =
|
|
15260
|
-
this.path =
|
|
15983
|
+
const watchPath = path28;
|
|
15984
|
+
this.path = path28 = path28.replace(REPLACER_RE, "");
|
|
15261
15985
|
this.watchPath = watchPath;
|
|
15262
15986
|
this.fullWatchPath = sp2.resolve(watchPath);
|
|
15263
15987
|
this.dirParts = [];
|
|
@@ -15397,20 +16121,20 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15397
16121
|
this._closePromise = void 0;
|
|
15398
16122
|
let paths = unifyPaths(paths_);
|
|
15399
16123
|
if (cwd) {
|
|
15400
|
-
paths = paths.map((
|
|
15401
|
-
const absPath = getAbsolutePath(
|
|
16124
|
+
paths = paths.map((path28) => {
|
|
16125
|
+
const absPath = getAbsolutePath(path28, cwd);
|
|
15402
16126
|
return absPath;
|
|
15403
16127
|
});
|
|
15404
16128
|
}
|
|
15405
|
-
paths.forEach((
|
|
15406
|
-
this._removeIgnoredPath(
|
|
16129
|
+
paths.forEach((path28) => {
|
|
16130
|
+
this._removeIgnoredPath(path28);
|
|
15407
16131
|
});
|
|
15408
16132
|
this._userIgnored = void 0;
|
|
15409
16133
|
if (!this._readyCount)
|
|
15410
16134
|
this._readyCount = 0;
|
|
15411
16135
|
this._readyCount += paths.length;
|
|
15412
|
-
Promise.all(paths.map(async (
|
|
15413
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
16136
|
+
Promise.all(paths.map(async (path28) => {
|
|
16137
|
+
const res = await this._nodeFsHandler._addToNodeFs(path28, !_internal, void 0, 0, _origAdd);
|
|
15414
16138
|
if (res)
|
|
15415
16139
|
this._emitReady();
|
|
15416
16140
|
return res;
|
|
@@ -15432,17 +16156,17 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15432
16156
|
return this;
|
|
15433
16157
|
const paths = unifyPaths(paths_);
|
|
15434
16158
|
const { cwd } = this.options;
|
|
15435
|
-
paths.forEach((
|
|
15436
|
-
if (!sp2.isAbsolute(
|
|
16159
|
+
paths.forEach((path28) => {
|
|
16160
|
+
if (!sp2.isAbsolute(path28) && !this._closers.has(path28)) {
|
|
15437
16161
|
if (cwd)
|
|
15438
|
-
|
|
15439
|
-
|
|
16162
|
+
path28 = sp2.join(cwd, path28);
|
|
16163
|
+
path28 = sp2.resolve(path28);
|
|
15440
16164
|
}
|
|
15441
|
-
this._closePath(
|
|
15442
|
-
this._addIgnoredPath(
|
|
15443
|
-
if (this._watched.has(
|
|
16165
|
+
this._closePath(path28);
|
|
16166
|
+
this._addIgnoredPath(path28);
|
|
16167
|
+
if (this._watched.has(path28)) {
|
|
15444
16168
|
this._addIgnoredPath({
|
|
15445
|
-
path:
|
|
16169
|
+
path: path28,
|
|
15446
16170
|
recursive: true
|
|
15447
16171
|
});
|
|
15448
16172
|
}
|
|
@@ -15506,38 +16230,38 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15506
16230
|
* @param stats arguments to be passed with event
|
|
15507
16231
|
* @returns the error if defined, otherwise the value of the FSWatcher instance's `closed` flag
|
|
15508
16232
|
*/
|
|
15509
|
-
async _emit(event,
|
|
16233
|
+
async _emit(event, path28, stats) {
|
|
15510
16234
|
if (this.closed)
|
|
15511
16235
|
return;
|
|
15512
16236
|
const opts = this.options;
|
|
15513
16237
|
if (isWindows)
|
|
15514
|
-
|
|
16238
|
+
path28 = sp2.normalize(path28);
|
|
15515
16239
|
if (opts.cwd)
|
|
15516
|
-
|
|
15517
|
-
const args = [
|
|
16240
|
+
path28 = sp2.relative(opts.cwd, path28);
|
|
16241
|
+
const args = [path28];
|
|
15518
16242
|
if (stats != null)
|
|
15519
16243
|
args.push(stats);
|
|
15520
16244
|
const awf = opts.awaitWriteFinish;
|
|
15521
16245
|
let pw;
|
|
15522
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
16246
|
+
if (awf && (pw = this._pendingWrites.get(path28))) {
|
|
15523
16247
|
pw.lastChange = /* @__PURE__ */ new Date();
|
|
15524
16248
|
return this;
|
|
15525
16249
|
}
|
|
15526
16250
|
if (opts.atomic) {
|
|
15527
16251
|
if (event === EVENTS.UNLINK) {
|
|
15528
|
-
this._pendingUnlinks.set(
|
|
16252
|
+
this._pendingUnlinks.set(path28, [event, ...args]);
|
|
15529
16253
|
setTimeout(() => {
|
|
15530
|
-
this._pendingUnlinks.forEach((entry,
|
|
16254
|
+
this._pendingUnlinks.forEach((entry, path29) => {
|
|
15531
16255
|
this.emit(...entry);
|
|
15532
16256
|
this.emit(EVENTS.ALL, ...entry);
|
|
15533
|
-
this._pendingUnlinks.delete(
|
|
16257
|
+
this._pendingUnlinks.delete(path29);
|
|
15534
16258
|
});
|
|
15535
16259
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
15536
16260
|
return this;
|
|
15537
16261
|
}
|
|
15538
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
16262
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path28)) {
|
|
15539
16263
|
event = EVENTS.CHANGE;
|
|
15540
|
-
this._pendingUnlinks.delete(
|
|
16264
|
+
this._pendingUnlinks.delete(path28);
|
|
15541
16265
|
}
|
|
15542
16266
|
}
|
|
15543
16267
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -15555,16 +16279,16 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15555
16279
|
this.emitWithAll(event, args);
|
|
15556
16280
|
}
|
|
15557
16281
|
};
|
|
15558
|
-
this._awaitWriteFinish(
|
|
16282
|
+
this._awaitWriteFinish(path28, awf.stabilityThreshold, event, awfEmit);
|
|
15559
16283
|
return this;
|
|
15560
16284
|
}
|
|
15561
16285
|
if (event === EVENTS.CHANGE) {
|
|
15562
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
16286
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path28, 50);
|
|
15563
16287
|
if (isThrottled)
|
|
15564
16288
|
return this;
|
|
15565
16289
|
}
|
|
15566
16290
|
if (opts.alwaysStat && stats === void 0 && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
15567
|
-
const fullPath = opts.cwd ? sp2.join(opts.cwd,
|
|
16291
|
+
const fullPath = opts.cwd ? sp2.join(opts.cwd, path28) : path28;
|
|
15568
16292
|
let stats2;
|
|
15569
16293
|
try {
|
|
15570
16294
|
stats2 = await stat3(fullPath);
|
|
@@ -15595,23 +16319,23 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15595
16319
|
* @param timeout duration of time to suppress duplicate actions
|
|
15596
16320
|
* @returns tracking object or false if action should be suppressed
|
|
15597
16321
|
*/
|
|
15598
|
-
_throttle(actionType,
|
|
16322
|
+
_throttle(actionType, path28, timeout) {
|
|
15599
16323
|
if (!this._throttled.has(actionType)) {
|
|
15600
16324
|
this._throttled.set(actionType, /* @__PURE__ */ new Map());
|
|
15601
16325
|
}
|
|
15602
16326
|
const action = this._throttled.get(actionType);
|
|
15603
16327
|
if (!action)
|
|
15604
16328
|
throw new Error("invalid throttle");
|
|
15605
|
-
const actionPath = action.get(
|
|
16329
|
+
const actionPath = action.get(path28);
|
|
15606
16330
|
if (actionPath) {
|
|
15607
16331
|
actionPath.count++;
|
|
15608
16332
|
return false;
|
|
15609
16333
|
}
|
|
15610
16334
|
let timeoutObject;
|
|
15611
16335
|
const clear = () => {
|
|
15612
|
-
const item = action.get(
|
|
16336
|
+
const item = action.get(path28);
|
|
15613
16337
|
const count = item ? item.count : 0;
|
|
15614
|
-
action.delete(
|
|
16338
|
+
action.delete(path28);
|
|
15615
16339
|
clearTimeout(timeoutObject);
|
|
15616
16340
|
if (item)
|
|
15617
16341
|
clearTimeout(item.timeoutObject);
|
|
@@ -15619,7 +16343,7 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15619
16343
|
};
|
|
15620
16344
|
timeoutObject = setTimeout(clear, timeout);
|
|
15621
16345
|
const thr = { timeoutObject, clear, count: 0 };
|
|
15622
|
-
action.set(
|
|
16346
|
+
action.set(path28, thr);
|
|
15623
16347
|
return thr;
|
|
15624
16348
|
}
|
|
15625
16349
|
_incrReadyCount() {
|
|
@@ -15633,44 +16357,44 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15633
16357
|
* @param event
|
|
15634
16358
|
* @param awfEmit Callback to be called when ready for event to be emitted.
|
|
15635
16359
|
*/
|
|
15636
|
-
_awaitWriteFinish(
|
|
16360
|
+
_awaitWriteFinish(path28, threshold, event, awfEmit) {
|
|
15637
16361
|
const awf = this.options.awaitWriteFinish;
|
|
15638
16362
|
if (typeof awf !== "object")
|
|
15639
16363
|
return;
|
|
15640
16364
|
const pollInterval = awf.pollInterval;
|
|
15641
16365
|
let timeoutHandler;
|
|
15642
|
-
let fullPath =
|
|
15643
|
-
if (this.options.cwd && !sp2.isAbsolute(
|
|
15644
|
-
fullPath = sp2.join(this.options.cwd,
|
|
16366
|
+
let fullPath = path28;
|
|
16367
|
+
if (this.options.cwd && !sp2.isAbsolute(path28)) {
|
|
16368
|
+
fullPath = sp2.join(this.options.cwd, path28);
|
|
15645
16369
|
}
|
|
15646
16370
|
const now2 = /* @__PURE__ */ new Date();
|
|
15647
16371
|
const writes = this._pendingWrites;
|
|
15648
16372
|
function awaitWriteFinishFn(prevStat) {
|
|
15649
16373
|
statcb(fullPath, (err, curStat) => {
|
|
15650
|
-
if (err || !writes.has(
|
|
16374
|
+
if (err || !writes.has(path28)) {
|
|
15651
16375
|
if (err && err.code !== "ENOENT")
|
|
15652
16376
|
awfEmit(err);
|
|
15653
16377
|
return;
|
|
15654
16378
|
}
|
|
15655
16379
|
const now3 = Number(/* @__PURE__ */ new Date());
|
|
15656
16380
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
15657
|
-
writes.get(
|
|
16381
|
+
writes.get(path28).lastChange = now3;
|
|
15658
16382
|
}
|
|
15659
|
-
const pw = writes.get(
|
|
16383
|
+
const pw = writes.get(path28);
|
|
15660
16384
|
const df = now3 - pw.lastChange;
|
|
15661
16385
|
if (df >= threshold) {
|
|
15662
|
-
writes.delete(
|
|
16386
|
+
writes.delete(path28);
|
|
15663
16387
|
awfEmit(void 0, curStat);
|
|
15664
16388
|
} else {
|
|
15665
16389
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
15666
16390
|
}
|
|
15667
16391
|
});
|
|
15668
16392
|
}
|
|
15669
|
-
if (!writes.has(
|
|
15670
|
-
writes.set(
|
|
16393
|
+
if (!writes.has(path28)) {
|
|
16394
|
+
writes.set(path28, {
|
|
15671
16395
|
lastChange: now2,
|
|
15672
16396
|
cancelWait: () => {
|
|
15673
|
-
writes.delete(
|
|
16397
|
+
writes.delete(path28);
|
|
15674
16398
|
clearTimeout(timeoutHandler);
|
|
15675
16399
|
return event;
|
|
15676
16400
|
}
|
|
@@ -15681,8 +16405,8 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15681
16405
|
/**
|
|
15682
16406
|
* Determines whether user has asked to ignore this path.
|
|
15683
16407
|
*/
|
|
15684
|
-
_isIgnored(
|
|
15685
|
-
if (this.options.atomic && DOT_RE.test(
|
|
16408
|
+
_isIgnored(path28, stats) {
|
|
16409
|
+
if (this.options.atomic && DOT_RE.test(path28))
|
|
15686
16410
|
return true;
|
|
15687
16411
|
if (!this._userIgnored) {
|
|
15688
16412
|
const { cwd } = this.options;
|
|
@@ -15692,17 +16416,17 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15692
16416
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
15693
16417
|
this._userIgnored = anymatch(list, void 0);
|
|
15694
16418
|
}
|
|
15695
|
-
return this._userIgnored(
|
|
16419
|
+
return this._userIgnored(path28, stats);
|
|
15696
16420
|
}
|
|
15697
|
-
_isntIgnored(
|
|
15698
|
-
return !this._isIgnored(
|
|
16421
|
+
_isntIgnored(path28, stat4) {
|
|
16422
|
+
return !this._isIgnored(path28, stat4);
|
|
15699
16423
|
}
|
|
15700
16424
|
/**
|
|
15701
16425
|
* Provides a set of common helpers and properties relating to symlink handling.
|
|
15702
16426
|
* @param path file or directory pattern being watched
|
|
15703
16427
|
*/
|
|
15704
|
-
_getWatchHelpers(
|
|
15705
|
-
return new WatchHelper(
|
|
16428
|
+
_getWatchHelpers(path28) {
|
|
16429
|
+
return new WatchHelper(path28, this.options.followSymlinks, this);
|
|
15706
16430
|
}
|
|
15707
16431
|
// Directory helpers
|
|
15708
16432
|
// -----------------
|
|
@@ -15734,63 +16458,63 @@ var FSWatcher = class extends EventEmitter2 {
|
|
|
15734
16458
|
* @param item base path of item/directory
|
|
15735
16459
|
*/
|
|
15736
16460
|
_remove(directory, item, isDirectory) {
|
|
15737
|
-
const
|
|
15738
|
-
const fullPath = sp2.resolve(
|
|
15739
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
15740
|
-
if (!this._throttle("remove",
|
|
16461
|
+
const path28 = sp2.join(directory, item);
|
|
16462
|
+
const fullPath = sp2.resolve(path28);
|
|
16463
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path28) || this._watched.has(fullPath);
|
|
16464
|
+
if (!this._throttle("remove", path28, 100))
|
|
15741
16465
|
return;
|
|
15742
16466
|
if (!isDirectory && this._watched.size === 1) {
|
|
15743
16467
|
this.add(directory, item, true);
|
|
15744
16468
|
}
|
|
15745
|
-
const wp = this._getWatchedDir(
|
|
16469
|
+
const wp = this._getWatchedDir(path28);
|
|
15746
16470
|
const nestedDirectoryChildren = wp.getChildren();
|
|
15747
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
16471
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path28, nested));
|
|
15748
16472
|
const parent = this._getWatchedDir(directory);
|
|
15749
16473
|
const wasTracked = parent.has(item);
|
|
15750
16474
|
parent.remove(item);
|
|
15751
16475
|
if (this._symlinkPaths.has(fullPath)) {
|
|
15752
16476
|
this._symlinkPaths.delete(fullPath);
|
|
15753
16477
|
}
|
|
15754
|
-
let relPath =
|
|
16478
|
+
let relPath = path28;
|
|
15755
16479
|
if (this.options.cwd)
|
|
15756
|
-
relPath = sp2.relative(this.options.cwd,
|
|
16480
|
+
relPath = sp2.relative(this.options.cwd, path28);
|
|
15757
16481
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
15758
16482
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
15759
16483
|
if (event === EVENTS.ADD)
|
|
15760
16484
|
return;
|
|
15761
16485
|
}
|
|
15762
|
-
this._watched.delete(
|
|
16486
|
+
this._watched.delete(path28);
|
|
15763
16487
|
this._watched.delete(fullPath);
|
|
15764
16488
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
15765
|
-
if (wasTracked && !this._isIgnored(
|
|
15766
|
-
this._emit(eventName,
|
|
15767
|
-
this._closePath(
|
|
16489
|
+
if (wasTracked && !this._isIgnored(path28))
|
|
16490
|
+
this._emit(eventName, path28);
|
|
16491
|
+
this._closePath(path28);
|
|
15768
16492
|
}
|
|
15769
16493
|
/**
|
|
15770
16494
|
* Closes all watchers for a path
|
|
15771
16495
|
*/
|
|
15772
|
-
_closePath(
|
|
15773
|
-
this._closeFile(
|
|
15774
|
-
const dir = sp2.dirname(
|
|
15775
|
-
this._getWatchedDir(dir).remove(sp2.basename(
|
|
16496
|
+
_closePath(path28) {
|
|
16497
|
+
this._closeFile(path28);
|
|
16498
|
+
const dir = sp2.dirname(path28);
|
|
16499
|
+
this._getWatchedDir(dir).remove(sp2.basename(path28));
|
|
15776
16500
|
}
|
|
15777
16501
|
/**
|
|
15778
16502
|
* Closes only file-specific watchers
|
|
15779
16503
|
*/
|
|
15780
|
-
_closeFile(
|
|
15781
|
-
const closers = this._closers.get(
|
|
16504
|
+
_closeFile(path28) {
|
|
16505
|
+
const closers = this._closers.get(path28);
|
|
15782
16506
|
if (!closers)
|
|
15783
16507
|
return;
|
|
15784
16508
|
closers.forEach((closer) => closer());
|
|
15785
|
-
this._closers.delete(
|
|
16509
|
+
this._closers.delete(path28);
|
|
15786
16510
|
}
|
|
15787
|
-
_addPathCloser(
|
|
16511
|
+
_addPathCloser(path28, closer) {
|
|
15788
16512
|
if (!closer)
|
|
15789
16513
|
return;
|
|
15790
|
-
let list = this._closers.get(
|
|
16514
|
+
let list = this._closers.get(path28);
|
|
15791
16515
|
if (!list) {
|
|
15792
16516
|
list = [];
|
|
15793
|
-
this._closers.set(
|
|
16517
|
+
this._closers.set(path28, list);
|
|
15794
16518
|
}
|
|
15795
16519
|
list.push(closer);
|
|
15796
16520
|
}
|
|
@@ -15820,7 +16544,7 @@ function watch(paths, options = {}) {
|
|
|
15820
16544
|
var chokidar_default = { watch, FSWatcher };
|
|
15821
16545
|
|
|
15822
16546
|
// src/watcher/file-watcher.ts
|
|
15823
|
-
import * as
|
|
16547
|
+
import * as path21 from "path";
|
|
15824
16548
|
var FileWatcher = class {
|
|
15825
16549
|
watcher = null;
|
|
15826
16550
|
projectRoot;
|
|
@@ -15862,9 +16586,9 @@ var FileWatcher = class {
|
|
|
15862
16586
|
watchTargets = [this.projectRoot, this.configPath];
|
|
15863
16587
|
} else {
|
|
15864
16588
|
const externalConfigTargets = this.projectConfigPaths.filter((projectConfigPath) => {
|
|
15865
|
-
const relativeConfigPath =
|
|
16589
|
+
const relativeConfigPath = path21.relative(this.projectRoot, projectConfigPath);
|
|
15866
16590
|
return this.isOutsideProjectPath(relativeConfigPath);
|
|
15867
|
-
}).map((projectConfigPath) =>
|
|
16591
|
+
}).map((projectConfigPath) => existsSync13(projectConfigPath) ? projectConfigPath : this.getNearestExistingDirectory(path21.dirname(projectConfigPath)));
|
|
15868
16592
|
const uniqueExternalConfigTargets = [...new Set(externalConfigTargets)];
|
|
15869
16593
|
if (uniqueExternalConfigTargets.length > 0) {
|
|
15870
16594
|
watchTargets = [this.projectRoot, ...uniqueExternalConfigTargets];
|
|
@@ -15872,7 +16596,7 @@ var FileWatcher = class {
|
|
|
15872
16596
|
}
|
|
15873
16597
|
const watcherOptions = {
|
|
15874
16598
|
ignored: (filePath) => {
|
|
15875
|
-
const relativePath =
|
|
16599
|
+
const relativePath = path21.relative(this.projectRoot, filePath);
|
|
15876
16600
|
if (!relativePath) return false;
|
|
15877
16601
|
if (this.isProjectConfigPathOrAncestor(relativePath)) {
|
|
15878
16602
|
return false;
|
|
@@ -15880,10 +16604,10 @@ var FileWatcher = class {
|
|
|
15880
16604
|
if (this.isOutsideProjectPath(relativePath)) {
|
|
15881
16605
|
return true;
|
|
15882
16606
|
}
|
|
15883
|
-
if (hasFilteredPathSegment(relativePath,
|
|
16607
|
+
if (hasFilteredPathSegment(relativePath, path21.sep)) {
|
|
15884
16608
|
return true;
|
|
15885
16609
|
}
|
|
15886
|
-
if (isRestrictedDirectory(relativePath,
|
|
16610
|
+
if (isRestrictedDirectory(relativePath, path21.sep)) {
|
|
15887
16611
|
return true;
|
|
15888
16612
|
}
|
|
15889
16613
|
if (ignoreFilter.ignores(relativePath)) {
|
|
@@ -15972,23 +16696,23 @@ var FileWatcher = class {
|
|
|
15972
16696
|
this.scheduleFlush();
|
|
15973
16697
|
}
|
|
15974
16698
|
isProjectConfigPath(filePath) {
|
|
15975
|
-
const relativePath =
|
|
15976
|
-
const normalizedRelativePath =
|
|
16699
|
+
const relativePath = path21.relative(this.projectRoot, filePath);
|
|
16700
|
+
const normalizedRelativePath = path21.normalize(relativePath);
|
|
15977
16701
|
return this.getProjectConfigRelativePaths().some((configPath) => configPath === normalizedRelativePath);
|
|
15978
16702
|
}
|
|
15979
16703
|
isProjectConfigPathOrAncestor(relativePath) {
|
|
15980
|
-
const normalizedRelativePath =
|
|
16704
|
+
const normalizedRelativePath = path21.normalize(relativePath);
|
|
15981
16705
|
return this.getProjectConfigRelativePaths().some(
|
|
15982
|
-
(configPath) => configPath === normalizedRelativePath || configPath.startsWith(`${normalizedRelativePath}${
|
|
16706
|
+
(configPath) => configPath === normalizedRelativePath || configPath.startsWith(`${normalizedRelativePath}${path21.sep}`)
|
|
15983
16707
|
);
|
|
15984
16708
|
}
|
|
15985
16709
|
isOutsideProjectPath(relativePath) {
|
|
15986
|
-
return relativePath === ".." || relativePath.startsWith(`..${
|
|
16710
|
+
return relativePath === ".." || relativePath.startsWith(`..${path21.sep}`) || path21.isAbsolute(relativePath);
|
|
15987
16711
|
}
|
|
15988
16712
|
getNearestExistingDirectory(directoryPath) {
|
|
15989
16713
|
let candidate = directoryPath;
|
|
15990
|
-
while (!
|
|
15991
|
-
const parent =
|
|
16714
|
+
while (!existsSync13(candidate)) {
|
|
16715
|
+
const parent = path21.dirname(candidate);
|
|
15992
16716
|
if (parent === candidate) break;
|
|
15993
16717
|
candidate = parent;
|
|
15994
16718
|
}
|
|
@@ -15996,7 +16720,7 @@ var FileWatcher = class {
|
|
|
15996
16720
|
}
|
|
15997
16721
|
getProjectConfigRelativePaths() {
|
|
15998
16722
|
return this.projectConfigPaths.map(
|
|
15999
|
-
(configPath) =>
|
|
16723
|
+
(configPath) => path21.normalize(path21.relative(this.projectRoot, configPath))
|
|
16000
16724
|
);
|
|
16001
16725
|
}
|
|
16002
16726
|
scheduleFlush() {
|
|
@@ -16012,7 +16736,7 @@ var FileWatcher = class {
|
|
|
16012
16736
|
return;
|
|
16013
16737
|
}
|
|
16014
16738
|
const changes = Array.from(this.pendingChanges.entries()).map(
|
|
16015
|
-
([
|
|
16739
|
+
([path28, type]) => ({ path: path28, type })
|
|
16016
16740
|
);
|
|
16017
16741
|
this.pendingChanges.clear();
|
|
16018
16742
|
try {
|
|
@@ -16047,7 +16771,7 @@ var FileWatcher = class {
|
|
|
16047
16771
|
};
|
|
16048
16772
|
|
|
16049
16773
|
// src/watcher/git-head-watcher.ts
|
|
16050
|
-
import * as
|
|
16774
|
+
import * as path22 from "path";
|
|
16051
16775
|
var GitHeadWatcher = class {
|
|
16052
16776
|
watcher = null;
|
|
16053
16777
|
projectRoot;
|
|
@@ -16069,7 +16793,7 @@ var GitHeadWatcher = class {
|
|
|
16069
16793
|
this.onBranchChange = handler;
|
|
16070
16794
|
this.currentBranch = getCurrentBranch(this.projectRoot);
|
|
16071
16795
|
const headPath = getHeadPath(this.projectRoot);
|
|
16072
|
-
const refsPath =
|
|
16796
|
+
const refsPath = path22.join(this.projectRoot, ".git", "refs", "heads");
|
|
16073
16797
|
this.watcher = chokidar_default.watch([headPath, refsPath], {
|
|
16074
16798
|
persistent: true,
|
|
16075
16799
|
ignoreInitial: true,
|
|
@@ -16197,32 +16921,6 @@ function tool(input) {
|
|
|
16197
16921
|
}
|
|
16198
16922
|
tool.schema = z;
|
|
16199
16923
|
|
|
16200
|
-
// src/tools/contracts.ts
|
|
16201
|
-
var CHUNK_TYPES = [
|
|
16202
|
-
"function",
|
|
16203
|
-
"class",
|
|
16204
|
-
"method",
|
|
16205
|
-
"interface",
|
|
16206
|
-
"type",
|
|
16207
|
-
"enum",
|
|
16208
|
-
"struct",
|
|
16209
|
-
"impl",
|
|
16210
|
-
"trait",
|
|
16211
|
-
"module",
|
|
16212
|
-
"other"
|
|
16213
|
-
];
|
|
16214
|
-
var CALL_GRAPH_DIRECTIONS = ["callers", "callees"];
|
|
16215
|
-
var RELATIONSHIP_TYPES = [
|
|
16216
|
-
"Call",
|
|
16217
|
-
"MethodCall",
|
|
16218
|
-
"Constructor",
|
|
16219
|
-
"Import",
|
|
16220
|
-
"Inherits",
|
|
16221
|
-
"Implements"
|
|
16222
|
-
];
|
|
16223
|
-
var INDEX_LOG_CATEGORIES = ["search", "embedding", "cache", "gc", "branch", "general"];
|
|
16224
|
-
var INDEX_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
16225
|
-
|
|
16226
16924
|
// src/tools/format-pr-impact.ts
|
|
16227
16925
|
function formatPrImpact(result) {
|
|
16228
16926
|
const lines = [];
|
|
@@ -16331,117 +17029,6 @@ var pr_impact = tool({
|
|
|
16331
17029
|
}
|
|
16332
17030
|
});
|
|
16333
17031
|
|
|
16334
|
-
// src/tools/symbol-inference.ts
|
|
16335
|
-
var IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g;
|
|
16336
|
-
var QUOTED_BACKTICK_RE = /`([^`]+)`/g;
|
|
16337
|
-
var QUOTED_SINGLE_RE = /'([^'\\]+)'/g;
|
|
16338
|
-
var QUOTED_DOUBLE_RE = /"([^"]+)"/g;
|
|
16339
|
-
var SYMBOL_LIKE_RE = /^(?:[A-Za-z_$][A-Za-z0-9_$]*)$/;
|
|
16340
|
-
var CAMEL_CASE_RE = /^[a-z_][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*$/;
|
|
16341
|
-
var PASCAL_CASE_RE = /^[A-Z][A-Za-z0-9_$]*$/;
|
|
16342
|
-
var SNAKE_CASE_RE = /^[a-z][a-z0-9_]*_[a-z0-9_]+$/;
|
|
16343
|
-
var DEFINITION_INTENT_RE = /\b(where|defined|definition|define|declaration|symbol|function|method|class|interface|type)\b/i;
|
|
16344
|
-
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
16345
|
-
"a",
|
|
16346
|
-
"an",
|
|
16347
|
-
"and",
|
|
16348
|
-
"are",
|
|
16349
|
-
"at",
|
|
16350
|
-
"for",
|
|
16351
|
-
"find",
|
|
16352
|
-
"how",
|
|
16353
|
-
"i",
|
|
16354
|
-
"in",
|
|
16355
|
-
"is",
|
|
16356
|
-
"it",
|
|
16357
|
-
"of",
|
|
16358
|
-
"on",
|
|
16359
|
-
"that",
|
|
16360
|
-
"the",
|
|
16361
|
-
"definition",
|
|
16362
|
-
"show",
|
|
16363
|
-
"to",
|
|
16364
|
-
"where",
|
|
16365
|
-
"which",
|
|
16366
|
-
"what",
|
|
16367
|
-
"you",
|
|
16368
|
-
"your",
|
|
16369
|
-
"with"
|
|
16370
|
-
]);
|
|
16371
|
-
function stripCallSuffix(token) {
|
|
16372
|
-
return token.replace(/\(\s*\)$/, "");
|
|
16373
|
-
}
|
|
16374
|
-
function isLikelySymbolName(token) {
|
|
16375
|
-
if (!SYMBOL_LIKE_RE.test(token)) {
|
|
16376
|
-
return false;
|
|
16377
|
-
}
|
|
16378
|
-
if (STOP_WORDS.has(token.toLowerCase())) {
|
|
16379
|
-
return false;
|
|
16380
|
-
}
|
|
16381
|
-
return CAMEL_CASE_RE.test(token) || PASCAL_CASE_RE.test(token) || SNAKE_CASE_RE.test(token);
|
|
16382
|
-
}
|
|
16383
|
-
function extractQuotedIdentifiers(query) {
|
|
16384
|
-
const identifiers = /* @__PURE__ */ new Set();
|
|
16385
|
-
for (const match of query.matchAll(QUOTED_BACKTICK_RE)) {
|
|
16386
|
-
const candidate = stripCallSuffix(match[1].trim());
|
|
16387
|
-
if (candidate && isLikelySymbolName(candidate)) {
|
|
16388
|
-
identifiers.add(candidate);
|
|
16389
|
-
}
|
|
16390
|
-
}
|
|
16391
|
-
for (const match of query.matchAll(QUOTED_SINGLE_RE)) {
|
|
16392
|
-
const candidate = stripCallSuffix(match[1].trim());
|
|
16393
|
-
if (candidate && isLikelySymbolName(candidate)) {
|
|
16394
|
-
identifiers.add(candidate);
|
|
16395
|
-
}
|
|
16396
|
-
}
|
|
16397
|
-
for (const match of query.matchAll(QUOTED_DOUBLE_RE)) {
|
|
16398
|
-
const candidate = stripCallSuffix(match[1].trim());
|
|
16399
|
-
if (candidate && isLikelySymbolName(candidate)) {
|
|
16400
|
-
identifiers.add(candidate);
|
|
16401
|
-
}
|
|
16402
|
-
}
|
|
16403
|
-
return [...identifiers];
|
|
16404
|
-
}
|
|
16405
|
-
function extractBareIdentifiers(query) {
|
|
16406
|
-
const unquoted = query.replace(QUOTED_BACKTICK_RE, " ").replace(QUOTED_SINGLE_RE, " ").replace(QUOTED_DOUBLE_RE, " ");
|
|
16407
|
-
const identifiers = /* @__PURE__ */ new Set();
|
|
16408
|
-
for (const match of unquoted.matchAll(IDENTIFIER_RE)) {
|
|
16409
|
-
const candidate = stripCallSuffix(match[0]);
|
|
16410
|
-
if (isLikelySymbolName(candidate)) {
|
|
16411
|
-
identifiers.add(candidate);
|
|
16412
|
-
}
|
|
16413
|
-
}
|
|
16414
|
-
return [...identifiers];
|
|
16415
|
-
}
|
|
16416
|
-
function isSingleMeaningfulToken(query, symbol) {
|
|
16417
|
-
const tokens = query.replace(/[`'"()]/g, " ").split(/[^A-Za-z0-9_$]+/).map((token) => token.trim().toLowerCase()).filter((token) => token.length > 0).filter((token) => !STOP_WORDS.has(token));
|
|
16418
|
-
return tokens.length === 1 && tokens[0] === symbol.toLowerCase();
|
|
16419
|
-
}
|
|
16420
|
-
function inferExactSymbolFromQuery(query) {
|
|
16421
|
-
if (analyzeQueryIntent(query).explicitArtifactIntent) {
|
|
16422
|
-
return void 0;
|
|
16423
|
-
}
|
|
16424
|
-
const quoted = extractQuotedIdentifiers(query);
|
|
16425
|
-
if (quoted.length === 1) {
|
|
16426
|
-
return quoted[0];
|
|
16427
|
-
}
|
|
16428
|
-
if (quoted.length > 1) {
|
|
16429
|
-
return void 0;
|
|
16430
|
-
}
|
|
16431
|
-
const candidates = extractBareIdentifiers(query);
|
|
16432
|
-
if (candidates.length !== 1) {
|
|
16433
|
-
return void 0;
|
|
16434
|
-
}
|
|
16435
|
-
const candidate = candidates[0];
|
|
16436
|
-
if (DEFINITION_INTENT_RE.test(query)) {
|
|
16437
|
-
return candidate;
|
|
16438
|
-
}
|
|
16439
|
-
if (isSingleMeaningfulToken(query, candidate)) {
|
|
16440
|
-
return candidate;
|
|
16441
|
-
}
|
|
16442
|
-
return void 0;
|
|
16443
|
-
}
|
|
16444
|
-
|
|
16445
17032
|
// src/tools/context-search.ts
|
|
16446
17033
|
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
16447
17034
|
var MAX_CONTEXT_RESULT_LIMIT = 100;
|
|
@@ -16792,7 +17379,7 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
16792
17379
|
const directory = input.directory ?? void 0;
|
|
16793
17380
|
const tokenBudget = input.tokenBudget ?? void 0;
|
|
16794
17381
|
if (from && to) {
|
|
16795
|
-
const
|
|
17382
|
+
const path28 = await getCallGraphPath(
|
|
16796
17383
|
projectRoot,
|
|
16797
17384
|
host,
|
|
16798
17385
|
from,
|
|
@@ -16801,25 +17388,25 @@ async function resolveCodebaseContextUnmeasured(projectRoot, host, input) {
|
|
|
16801
17388
|
fromFilePath,
|
|
16802
17389
|
toFilePath
|
|
16803
17390
|
);
|
|
16804
|
-
const pathText = formatCallGraphPathResult(
|
|
16805
|
-
if (
|
|
17391
|
+
const pathText = formatCallGraphPathResult(path28);
|
|
17392
|
+
if (path28.path.length > 0) {
|
|
16806
17393
|
const fitted2 = fitTextToContextBudget(
|
|
16807
17394
|
pathText,
|
|
16808
17395
|
tokenBudget
|
|
16809
17396
|
);
|
|
16810
17397
|
return {
|
|
16811
17398
|
text: fitted2.text,
|
|
16812
|
-
details: fittedDetails("path", fitted2,
|
|
17399
|
+
details: fittedDetails("path", fitted2, path28.path.length)
|
|
16813
17400
|
};
|
|
16814
17401
|
}
|
|
16815
|
-
if (
|
|
17402
|
+
if (path28.from.status !== "resolved" || path28.to.status !== "resolved") {
|
|
16816
17403
|
const fitted2 = fitTextToContextBudget(pathText, tokenBudget);
|
|
16817
17404
|
return {
|
|
16818
17405
|
text: fitted2.text,
|
|
16819
17406
|
details: fittedDetails("path", fitted2, 0)
|
|
16820
17407
|
};
|
|
16821
17408
|
}
|
|
16822
|
-
const resolvedFrom =
|
|
17409
|
+
const resolvedFrom = path28.from;
|
|
16823
17410
|
const { callers } = await getCallGraphData(projectRoot, host, {
|
|
16824
17411
|
name: to,
|
|
16825
17412
|
direction: "callers",
|
|
@@ -16964,7 +17551,7 @@ async function executeCallGraph(projectRoot, host, args) {
|
|
|
16964
17551
|
return { text: formatCallGraphResult(await getCallGraphData(projectRoot, host, args)) };
|
|
16965
17552
|
}
|
|
16966
17553
|
async function executeCallGraphPath(projectRoot, host, args) {
|
|
16967
|
-
const
|
|
17554
|
+
const path28 = await getCallGraphPath(
|
|
16968
17555
|
projectRoot,
|
|
16969
17556
|
host,
|
|
16970
17557
|
args.from,
|
|
@@ -16973,17 +17560,21 @@ async function executeCallGraphPath(projectRoot, host, args) {
|
|
|
16973
17560
|
args.fromFilePath,
|
|
16974
17561
|
args.toFilePath
|
|
16975
17562
|
);
|
|
16976
|
-
return { text: formatCallGraphPathResult(
|
|
17563
|
+
return { text: formatCallGraphPathResult(path28) };
|
|
17564
|
+
}
|
|
17565
|
+
async function executeCodeCommunities(projectRoot, host, args) {
|
|
17566
|
+
const result = await getCodeCommunities(projectRoot, host, args);
|
|
17567
|
+
return { text: formatCodeCommunities(result) };
|
|
16977
17568
|
}
|
|
16978
17569
|
|
|
16979
17570
|
// src/adapters/opencode/tools.ts
|
|
16980
17571
|
import { writeFileSync as writeFileSync4 } from "fs";
|
|
16981
17572
|
import * as os7 from "os";
|
|
16982
|
-
import * as
|
|
17573
|
+
import * as path25 from "path";
|
|
16983
17574
|
|
|
16984
17575
|
// src/tools/visualize/activity.ts
|
|
16985
17576
|
import { execFileSync } from "child_process";
|
|
16986
|
-
import * as
|
|
17577
|
+
import * as path23 from "path";
|
|
16987
17578
|
function attachRecentActivity(data, projectRoot) {
|
|
16988
17579
|
const activity = readGitActivity(projectRoot);
|
|
16989
17580
|
const changes = activity.size > 0 ? buildGitChanges(data, activity, projectRoot) : buildGraphChanges(data);
|
|
@@ -17145,7 +17736,7 @@ function normalizePath3(filePath) {
|
|
|
17145
17736
|
return filePath.replace(/\\/g, "/");
|
|
17146
17737
|
}
|
|
17147
17738
|
function toGitRelativePath(projectRoot, filePath) {
|
|
17148
|
-
const relativePath =
|
|
17739
|
+
const relativePath = path23.isAbsolute(filePath) ? path23.relative(projectRoot, filePath) : filePath;
|
|
17149
17740
|
return normalizePath3(relativePath);
|
|
17150
17741
|
}
|
|
17151
17742
|
|
|
@@ -17403,7 +17994,7 @@ render();
|
|
|
17403
17994
|
}
|
|
17404
17995
|
|
|
17405
17996
|
// src/tools/visualize/transform.ts
|
|
17406
|
-
import * as
|
|
17997
|
+
import * as path24 from "path";
|
|
17407
17998
|
|
|
17408
17999
|
// src/tools/visualize/modules.ts
|
|
17409
18000
|
var MAX_MODULES = 18;
|
|
@@ -17663,7 +18254,7 @@ function transformForVisualization(symbols, edges, options = {}) {
|
|
|
17663
18254
|
filePath: s.filePath,
|
|
17664
18255
|
kind: s.kind,
|
|
17665
18256
|
line: s.startLine,
|
|
17666
|
-
directory:
|
|
18257
|
+
directory: path24.dirname(s.filePath),
|
|
17667
18258
|
moduleId: "",
|
|
17668
18259
|
moduleLabel: ""
|
|
17669
18260
|
}));
|
|
@@ -17905,6 +18496,20 @@ var remove_knowledge_base = tool({
|
|
|
17905
18496
|
return removeKnowledgeBase(context?.worktree, DEFAULT_HOST, args.path.trim());
|
|
17906
18497
|
}
|
|
17907
18498
|
});
|
|
18499
|
+
var code_communities = tool({
|
|
18500
|
+
description: "Discover natural module boundaries and hub symbols in the codebase using graph community detection. Clusters symbols by call-graph connectivity, reports community memberships, and identifies hub nodes with cross-community connections, and summarizes couplings between communities.",
|
|
18501
|
+
args: {
|
|
18502
|
+
branch: z3.string().optional().describe("Branch name to analyze (defaults to current branch)"),
|
|
18503
|
+
minSize: z3.number().int().min(CODE_COMMUNITIES_MIN_SIZE).optional().default(CODE_COMMUNITIES_MIN_SIZE).describe("Minimum community size to include (default: 1)"),
|
|
18504
|
+
limit: z3.number().int().min(1).max(CODE_COMMUNITIES_MAX_LIMIT).optional().default(CODE_COMMUNITIES_DEFAULT_LIMIT).describe("Maximum number of communities and hub nodes to return (default: 20)"),
|
|
18505
|
+
hubThreshold: z3.number().int().min(0).optional().default(CODE_COMMUNITIES_DEFAULT_HUB_THRESHOLD).describe("Minimum distinct cross-community neighbors to flag a hub node (default: 5)"),
|
|
18506
|
+
minCoupling: z3.number().int().min(CODE_COMMUNITIES_MIN_COUPLING).optional().default(CODE_COMMUNITIES_MIN_COUPLING).describe("Minimum distinct cross-community connection count to report a coupling (default: 1)"),
|
|
18507
|
+
couplingLimit: z3.number().int().min(1).max(CODE_COMMUNITIES_MAX_COUPLING_LIMIT).optional().default(CODE_COMMUNITIES_DEFAULT_COUPLING_LIMIT).describe("Maximum number of couplings to return (default: 20)")
|
|
18508
|
+
},
|
|
18509
|
+
async execute(args, context) {
|
|
18510
|
+
return (await executeCodeCommunities(context?.worktree, DEFAULT_HOST, args)).text;
|
|
18511
|
+
}
|
|
18512
|
+
});
|
|
17908
18513
|
var index_visualize = tool({
|
|
17909
18514
|
description: "Generate an interactive HTML visualization of recent code movement and the call graph. Starts with temporal onboarding context from Git history, then supports module, symbol, hotspot, and cycle drill-down.",
|
|
17910
18515
|
args: {
|
|
@@ -17930,7 +18535,7 @@ var index_visualize = tool({
|
|
|
17930
18535
|
return "No connected symbols found for visualization. Try including orphans with includeOrphans=true, or check that the call graph has resolved edges.";
|
|
17931
18536
|
}
|
|
17932
18537
|
const html = generateVisualizationHtml(vizData);
|
|
17933
|
-
const outputPath =
|
|
18538
|
+
const outputPath = path25.join(os7.tmpdir(), `call-graph-${Date.now()}.html`);
|
|
17934
18539
|
writeFileSync4(outputPath, html, "utf-8");
|
|
17935
18540
|
let result = `Temporal call graph visualization generated: ${outputPath}
|
|
17936
18541
|
|
|
@@ -17966,6 +18571,7 @@ var TOOL_NAME = {
|
|
|
17966
18571
|
CALL_GRAPH: "call_graph",
|
|
17967
18572
|
CALL_GRAPH_PATH: "call_graph_path",
|
|
17968
18573
|
PR_IMPACT: "pr_impact",
|
|
18574
|
+
CODE_COMMUNITIES: "code_communities",
|
|
17969
18575
|
ADD_KNOWLEDGE_BASE: "add_knowledge_base",
|
|
17970
18576
|
LIST_KNOWLEDGE_BASES: "list_knowledge_bases",
|
|
17971
18577
|
REMOVE_KNOWLEDGE_BASE: "remove_knowledge_base",
|
|
@@ -17987,7 +18593,8 @@ var PORTABLE_TOOL_NAMES = [
|
|
|
17987
18593
|
TOOL_NAME.IMPLEMENTATION_LOOKUP,
|
|
17988
18594
|
TOOL_NAME.CALL_GRAPH,
|
|
17989
18595
|
TOOL_NAME.CALL_GRAPH_PATH,
|
|
17990
|
-
TOOL_NAME.PR_IMPACT
|
|
18596
|
+
TOOL_NAME.PR_IMPACT,
|
|
18597
|
+
TOOL_NAME.CODE_COMMUNITIES
|
|
17991
18598
|
];
|
|
17992
18599
|
var OPENCODE_TOOL_NAMES = [
|
|
17993
18600
|
TOOL_NAME.CODEBASE_CONTEXT,
|
|
@@ -18006,6 +18613,7 @@ var OPENCODE_TOOL_NAMES = [
|
|
|
18006
18613
|
TOOL_NAME.LIST_KNOWLEDGE_BASES,
|
|
18007
18614
|
TOOL_NAME.REMOVE_KNOWLEDGE_BASE,
|
|
18008
18615
|
TOOL_NAME.PR_IMPACT,
|
|
18616
|
+
TOOL_NAME.CODE_COMMUNITIES,
|
|
18009
18617
|
TOOL_NAME.INDEX_VISUALIZE
|
|
18010
18618
|
];
|
|
18011
18619
|
var PI_TOOL_NAMES = [
|
|
@@ -18022,14 +18630,15 @@ var PI_TOOL_NAMES = [
|
|
|
18022
18630
|
TOOL_NAME.CALL_GRAPH,
|
|
18023
18631
|
TOOL_NAME.CALL_GRAPH_PATH,
|
|
18024
18632
|
TOOL_NAME.PR_IMPACT,
|
|
18633
|
+
TOOL_NAME.CODE_COMMUNITIES,
|
|
18025
18634
|
TOOL_NAME.PI_KNOWLEDGE_BASE_LIST,
|
|
18026
18635
|
TOOL_NAME.PI_KNOWLEDGE_BASE_ADD,
|
|
18027
18636
|
TOOL_NAME.PI_KNOWLEDGE_BASE_REMOVE
|
|
18028
18637
|
];
|
|
18029
18638
|
|
|
18030
18639
|
// src/commands/loader.ts
|
|
18031
|
-
import { existsSync as
|
|
18032
|
-
import * as
|
|
18640
|
+
import { existsSync as existsSync14, readdirSync as readdirSync3, readFileSync as readFileSync9 } from "fs";
|
|
18641
|
+
import * as path26 from "path";
|
|
18033
18642
|
function parseFrontmatter(content) {
|
|
18034
18643
|
const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
|
|
18035
18644
|
const match = content.match(frontmatterRegex);
|
|
@@ -18050,21 +18659,21 @@ function parseFrontmatter(content) {
|
|
|
18050
18659
|
}
|
|
18051
18660
|
function loadCommandsFromDirectory(commandsDir) {
|
|
18052
18661
|
const commands = /* @__PURE__ */ new Map();
|
|
18053
|
-
if (!
|
|
18662
|
+
if (!existsSync14(commandsDir)) {
|
|
18054
18663
|
return commands;
|
|
18055
18664
|
}
|
|
18056
18665
|
const files = readdirSync3(commandsDir).filter((f) => f.endsWith(".md"));
|
|
18057
18666
|
for (const file of files) {
|
|
18058
|
-
const filePath =
|
|
18667
|
+
const filePath = path26.join(commandsDir, file);
|
|
18059
18668
|
let content;
|
|
18060
18669
|
try {
|
|
18061
|
-
content =
|
|
18670
|
+
content = readFileSync9(filePath, "utf-8");
|
|
18062
18671
|
} catch (error) {
|
|
18063
18672
|
const message = error instanceof Error ? error.message : String(error);
|
|
18064
18673
|
throw new Error(`Failed to load command file ${filePath}: ${message}`);
|
|
18065
18674
|
}
|
|
18066
18675
|
const { frontmatter, body } = parseFrontmatter(content);
|
|
18067
|
-
const name =
|
|
18676
|
+
const name = path26.basename(file, ".md");
|
|
18068
18677
|
const description = frontmatter.description || `Run the ${name} command`;
|
|
18069
18678
|
commands.set(name, {
|
|
18070
18679
|
description,
|
|
@@ -18411,10 +19020,10 @@ function replaceActiveWatcher(projectRoot, nextWatcher) {
|
|
|
18411
19020
|
function getCommandsDir() {
|
|
18412
19021
|
let currentDir = process.cwd();
|
|
18413
19022
|
if (typeof import.meta !== "undefined" && import.meta.url) {
|
|
18414
|
-
currentDir =
|
|
19023
|
+
currentDir = path27.dirname(fileURLToPath2(import.meta.url));
|
|
18415
19024
|
}
|
|
18416
|
-
const packageRoot =
|
|
18417
|
-
return
|
|
19025
|
+
const packageRoot = path27.basename(currentDir) === "adapters" ? path27.join(currentDir, "..", "..") : path27.join(currentDir, "..");
|
|
19026
|
+
return path27.join(packageRoot, "commands");
|
|
18418
19027
|
}
|
|
18419
19028
|
function appendRoutingHints(output, hints, preferredRole) {
|
|
18420
19029
|
const preferredBucket = preferredRole === "developer" ? output.developer : output.system;
|
|
@@ -18477,6 +19086,7 @@ var plugin = async ({ directory, worktree }) => {
|
|
|
18477
19086
|
[TOOL_NAME.LIST_KNOWLEDGE_BASES]: list_knowledge_bases,
|
|
18478
19087
|
[TOOL_NAME.REMOVE_KNOWLEDGE_BASE]: remove_knowledge_base,
|
|
18479
19088
|
[TOOL_NAME.PR_IMPACT]: pr_impact,
|
|
19089
|
+
[TOOL_NAME.CODE_COMMUNITIES]: code_communities,
|
|
18480
19090
|
[TOOL_NAME.INDEX_VISUALIZE]: index_visualize
|
|
18481
19091
|
},
|
|
18482
19092
|
async "chat.message"(input, output) {
|