@neuralsea/workspace-indexer 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-IXKSW7IX.js → chunk-ENM3P2PS.js} +56 -3
- package/dist/cli.cjs +56 -3
- package/dist/cli.js +1 -1
- package/dist/index.cjs +56 -3
- package/dist/index.d.cts +40 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.js +1 -1
- package/package.json +1 -1
|
@@ -1621,6 +1621,16 @@ var RepoFileIndexer = class {
|
|
|
1621
1621
|
return;
|
|
1622
1622
|
}
|
|
1623
1623
|
this.emit({ type: "repo/index/file/start", repoRoot: this.repoRoot, path: posixRelPath });
|
|
1624
|
+
const stage = (stageName, stageStartedAt) => {
|
|
1625
|
+
this.emit({
|
|
1626
|
+
type: "repo/index/file/stage",
|
|
1627
|
+
repoRoot: this.repoRoot,
|
|
1628
|
+
path: posixRelPath,
|
|
1629
|
+
stage: stageName,
|
|
1630
|
+
ms: Date.now() - stageStartedAt
|
|
1631
|
+
});
|
|
1632
|
+
};
|
|
1633
|
+
const readStartedAt = Date.now();
|
|
1624
1634
|
const abs = path9.join(this.repoRoot, fromPosixPath(posixRelPath));
|
|
1625
1635
|
let stat;
|
|
1626
1636
|
try {
|
|
@@ -1644,16 +1654,20 @@ var RepoFileIndexer = class {
|
|
|
1644
1654
|
}
|
|
1645
1655
|
const raw = buf.toString("utf8");
|
|
1646
1656
|
const redacted = this.applyRedactions(raw);
|
|
1657
|
+
stage("read", readStartedAt);
|
|
1647
1658
|
const fileHash = sha256Hex(redacted);
|
|
1648
1659
|
const prev = this.store.getFileHash(posixRelPath);
|
|
1649
1660
|
if (prev === fileHash) {
|
|
1650
1661
|
this.emit({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "unchanged" });
|
|
1651
1662
|
return;
|
|
1652
1663
|
}
|
|
1664
|
+
const chunkStartedAt = Date.now();
|
|
1653
1665
|
const { language, chunks } = chunkSource(posixRelPath, redacted, this.config.chunk);
|
|
1666
|
+
stage("chunk", chunkStartedAt);
|
|
1654
1667
|
let imports = [];
|
|
1655
1668
|
let exports = [];
|
|
1656
1669
|
if (language === "typescript" || language === "javascript") {
|
|
1670
|
+
const relationsStartedAt = Date.now();
|
|
1657
1671
|
const rel = extractTsRelations(posixRelPath, redacted);
|
|
1658
1672
|
imports = rel.imports;
|
|
1659
1673
|
exports = rel.exports;
|
|
@@ -1661,12 +1675,14 @@ var RepoFileIndexer = class {
|
|
|
1661
1675
|
this.store.setEdges(posixRelPath, "export", rel.exports);
|
|
1662
1676
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "import", rel.imports);
|
|
1663
1677
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "export", rel.exports);
|
|
1678
|
+
stage("relations", relationsStartedAt);
|
|
1664
1679
|
} else {
|
|
1665
1680
|
this.store.setEdges(posixRelPath, "import", []);
|
|
1666
1681
|
this.store.setEdges(posixRelPath, "export", []);
|
|
1667
1682
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "import", []);
|
|
1668
1683
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "export", []);
|
|
1669
1684
|
}
|
|
1685
|
+
const synopsisStartedAt = Date.now();
|
|
1670
1686
|
const synopsisText = buildSynopsis(posixRelPath, language, redacted);
|
|
1671
1687
|
const synopsis = synopsisText.trim() ? [
|
|
1672
1688
|
{
|
|
@@ -1691,14 +1707,18 @@ var RepoFileIndexer = class {
|
|
|
1691
1707
|
}
|
|
1692
1708
|
] : [];
|
|
1693
1709
|
const combined = [...synopsis, ...headerChunk, ...chunks.map((c) => ({ ...c, kind: "chunk" }))];
|
|
1710
|
+
stage("synopsis", synopsisStartedAt);
|
|
1711
|
+
const embedStartedAt = Date.now();
|
|
1694
1712
|
const embedTexts = [];
|
|
1695
1713
|
const embedPlan = [];
|
|
1696
1714
|
const embeddings = combined.map(() => null);
|
|
1715
|
+
let cached2 = 0;
|
|
1697
1716
|
for (let i = 0; i < combined.length; i++) {
|
|
1698
1717
|
const ch = combined[i];
|
|
1699
|
-
const
|
|
1700
|
-
if (
|
|
1701
|
-
embeddings[i] =
|
|
1718
|
+
const cachedEmb = this.embeddingCache.get(this.embedder.id, ch.contentHash);
|
|
1719
|
+
if (cachedEmb) {
|
|
1720
|
+
embeddings[i] = cachedEmb;
|
|
1721
|
+
cached2++;
|
|
1702
1722
|
continue;
|
|
1703
1723
|
}
|
|
1704
1724
|
embedTexts.push(
|
|
@@ -1730,11 +1750,22 @@ ${ch.text}`
|
|
|
1730
1750
|
this.embeddingCache.put(this.embedder.id, plan.contentHash, vecs[j]);
|
|
1731
1751
|
}
|
|
1732
1752
|
}
|
|
1753
|
+
this.emit({
|
|
1754
|
+
type: "repo/index/file/embed/done",
|
|
1755
|
+
repoRoot: this.repoRoot,
|
|
1756
|
+
path: posixRelPath,
|
|
1757
|
+
chunks: combined.length,
|
|
1758
|
+
cached: cached2,
|
|
1759
|
+
computed: embedTexts.length,
|
|
1760
|
+
ms: Date.now() - embedStartedAt
|
|
1761
|
+
});
|
|
1762
|
+
stage("embed", embedStartedAt);
|
|
1733
1763
|
const fileMtime = stat.mtimeMs;
|
|
1734
1764
|
const ftsMode = this.config.storage.ftsMode;
|
|
1735
1765
|
const storeText = this.config.storage.storeText;
|
|
1736
1766
|
const oldChunkIds = this.store.listChunksForFile(posixRelPath).map((r) => r.id);
|
|
1737
1767
|
const points = [];
|
|
1768
|
+
const storeStartedAt = Date.now();
|
|
1738
1769
|
const rows = combined.map((ch, i) => {
|
|
1739
1770
|
const id = sha256Hex(
|
|
1740
1771
|
`${this.repoId}:${posixRelPath}:${ch.kind}:${ch.startLine}:${ch.endLine}:${ch.contentHash}`
|
|
@@ -1762,7 +1793,19 @@ ${ch.text}`
|
|
|
1762
1793
|
this.store.replaceChunksForFile(posixRelPath, rows);
|
|
1763
1794
|
this.workspaceStore?.upsertFile(this.repoId, posixRelPath, fileHash, fileMtime, language, stat.size);
|
|
1764
1795
|
this.workspaceStore?.replaceChunksForFile(this.repoId, this.repoRoot, posixRelPath, rows);
|
|
1796
|
+
stage("store", storeStartedAt);
|
|
1797
|
+
const symbolsStartedAt = Date.now();
|
|
1765
1798
|
const symbolOut = await this.indexSymbolsIfEnabled(posixRelPath, language, redacted, fileHash);
|
|
1799
|
+
this.emit({
|
|
1800
|
+
type: "repo/index/file/symbols",
|
|
1801
|
+
repoRoot: this.repoRoot,
|
|
1802
|
+
path: posixRelPath,
|
|
1803
|
+
symbols: symbolOut?.symbols?.length ?? 0,
|
|
1804
|
+
edges: symbolOut?.edges?.length ?? 0,
|
|
1805
|
+
ms: Date.now() - symbolsStartedAt
|
|
1806
|
+
});
|
|
1807
|
+
stage("symbols", symbolsStartedAt);
|
|
1808
|
+
const graphStartedAt = Date.now();
|
|
1766
1809
|
const head = this.getHead();
|
|
1767
1810
|
if (this.graphStore && head) {
|
|
1768
1811
|
await this.graphStore.replaceFileGraph({
|
|
@@ -1778,7 +1821,10 @@ ${ch.text}`
|
|
|
1778
1821
|
symbolEdges: (symbolOut?.edges ?? []).map((e) => ({ fromId: e.fromId, toId: e.toId, kind: e.kind }))
|
|
1779
1822
|
}).catch(() => void 0);
|
|
1780
1823
|
}
|
|
1824
|
+
stage("graph", graphStartedAt);
|
|
1825
|
+
const vectorStartedAt = Date.now();
|
|
1781
1826
|
await this.ensureVectorUpToDate(points, oldChunkIds);
|
|
1827
|
+
stage("vector", vectorStartedAt);
|
|
1782
1828
|
this.emit({
|
|
1783
1829
|
type: "repo/index/file/done",
|
|
1784
1830
|
repoRoot: this.repoRoot,
|
|
@@ -2389,11 +2435,14 @@ var RepoIndexer = class {
|
|
|
2389
2435
|
const uniq3 = Array.from(new Set(posixPaths)).slice(0, maxFiles);
|
|
2390
2436
|
for (const p of uniq3) {
|
|
2391
2437
|
if (opts?.signal?.aborted) return;
|
|
2438
|
+
const startedAt = Date.now();
|
|
2439
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/start", repoRoot: this.repoRoot, path: p });
|
|
2392
2440
|
const abs = path12.join(this.repoRoot, p.split("/").join(path12.sep));
|
|
2393
2441
|
let text = "";
|
|
2394
2442
|
try {
|
|
2395
2443
|
text = fs9.readFileSync(abs, "utf8");
|
|
2396
2444
|
} catch {
|
|
2445
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/done", repoRoot: this.repoRoot, path: p, edges: 0, ms: Date.now() - startedAt });
|
|
2397
2446
|
continue;
|
|
2398
2447
|
}
|
|
2399
2448
|
const contentHash = this.store.getFileHash(p) ?? void 0;
|
|
@@ -2411,6 +2460,7 @@ var RepoIndexer = class {
|
|
|
2411
2460
|
fromPath: p,
|
|
2412
2461
|
edges: normalized.map((e) => ({ fromId: e.fromId, toId: e.toId, kind: e.kind, toPath: e.toPath }))
|
|
2413
2462
|
}).catch(() => void 0);
|
|
2463
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/done", repoRoot: this.repoRoot, path: p, edges: normalized.length, ms: Date.now() - startedAt });
|
|
2414
2464
|
}
|
|
2415
2465
|
}
|
|
2416
2466
|
async watch() {
|
|
@@ -4236,11 +4286,14 @@ var WorkspaceIndexer = class {
|
|
|
4236
4286
|
if (seeds.length >= 4) break;
|
|
4237
4287
|
}
|
|
4238
4288
|
if (seeds.length > 0) {
|
|
4289
|
+
const gs = Date.now();
|
|
4290
|
+
this.emitProgress({ type: "workspace/retrieve/graph/start", workspaceRoot: this.workspaceRoot, seeds: seeds.length });
|
|
4239
4291
|
graphNeighborFiles = await this.graphStore.neighborFiles({
|
|
4240
4292
|
seeds,
|
|
4241
4293
|
limit: profile.name === "architecture" ? 16 : 10,
|
|
4242
4294
|
kinds: ["definition", "reference", "implementation", "typeDefinition"]
|
|
4243
4295
|
});
|
|
4296
|
+
this.emitProgress({ type: "workspace/retrieve/graph/done", workspaceRoot: this.workspaceRoot, neighbors: graphNeighborFiles.length, ms: Date.now() - gs });
|
|
4244
4297
|
}
|
|
4245
4298
|
}
|
|
4246
4299
|
} catch {
|
package/dist/cli.cjs
CHANGED
|
@@ -1604,6 +1604,16 @@ var RepoFileIndexer = class {
|
|
|
1604
1604
|
return;
|
|
1605
1605
|
}
|
|
1606
1606
|
this.emit({ type: "repo/index/file/start", repoRoot: this.repoRoot, path: posixRelPath });
|
|
1607
|
+
const stage = (stageName, stageStartedAt) => {
|
|
1608
|
+
this.emit({
|
|
1609
|
+
type: "repo/index/file/stage",
|
|
1610
|
+
repoRoot: this.repoRoot,
|
|
1611
|
+
path: posixRelPath,
|
|
1612
|
+
stage: stageName,
|
|
1613
|
+
ms: Date.now() - stageStartedAt
|
|
1614
|
+
});
|
|
1615
|
+
};
|
|
1616
|
+
const readStartedAt = Date.now();
|
|
1607
1617
|
const abs = import_node_path9.default.join(this.repoRoot, fromPosixPath(posixRelPath));
|
|
1608
1618
|
let stat;
|
|
1609
1619
|
try {
|
|
@@ -1627,16 +1637,20 @@ var RepoFileIndexer = class {
|
|
|
1627
1637
|
}
|
|
1628
1638
|
const raw = buf.toString("utf8");
|
|
1629
1639
|
const redacted = this.applyRedactions(raw);
|
|
1640
|
+
stage("read", readStartedAt);
|
|
1630
1641
|
const fileHash = sha256Hex(redacted);
|
|
1631
1642
|
const prev = this.store.getFileHash(posixRelPath);
|
|
1632
1643
|
if (prev === fileHash) {
|
|
1633
1644
|
this.emit({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "unchanged" });
|
|
1634
1645
|
return;
|
|
1635
1646
|
}
|
|
1647
|
+
const chunkStartedAt = Date.now();
|
|
1636
1648
|
const { language, chunks } = chunkSource(posixRelPath, redacted, this.config.chunk);
|
|
1649
|
+
stage("chunk", chunkStartedAt);
|
|
1637
1650
|
let imports = [];
|
|
1638
1651
|
let exports2 = [];
|
|
1639
1652
|
if (language === "typescript" || language === "javascript") {
|
|
1653
|
+
const relationsStartedAt = Date.now();
|
|
1640
1654
|
const rel = extractTsRelations(posixRelPath, redacted);
|
|
1641
1655
|
imports = rel.imports;
|
|
1642
1656
|
exports2 = rel.exports;
|
|
@@ -1644,12 +1658,14 @@ var RepoFileIndexer = class {
|
|
|
1644
1658
|
this.store.setEdges(posixRelPath, "export", rel.exports);
|
|
1645
1659
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "import", rel.imports);
|
|
1646
1660
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "export", rel.exports);
|
|
1661
|
+
stage("relations", relationsStartedAt);
|
|
1647
1662
|
} else {
|
|
1648
1663
|
this.store.setEdges(posixRelPath, "import", []);
|
|
1649
1664
|
this.store.setEdges(posixRelPath, "export", []);
|
|
1650
1665
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "import", []);
|
|
1651
1666
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "export", []);
|
|
1652
1667
|
}
|
|
1668
|
+
const synopsisStartedAt = Date.now();
|
|
1653
1669
|
const synopsisText = buildSynopsis(posixRelPath, language, redacted);
|
|
1654
1670
|
const synopsis = synopsisText.trim() ? [
|
|
1655
1671
|
{
|
|
@@ -1674,14 +1690,18 @@ var RepoFileIndexer = class {
|
|
|
1674
1690
|
}
|
|
1675
1691
|
] : [];
|
|
1676
1692
|
const combined = [...synopsis, ...headerChunk, ...chunks.map((c) => ({ ...c, kind: "chunk" }))];
|
|
1693
|
+
stage("synopsis", synopsisStartedAt);
|
|
1694
|
+
const embedStartedAt = Date.now();
|
|
1677
1695
|
const embedTexts = [];
|
|
1678
1696
|
const embedPlan = [];
|
|
1679
1697
|
const embeddings = combined.map(() => null);
|
|
1698
|
+
let cached2 = 0;
|
|
1680
1699
|
for (let i = 0; i < combined.length; i++) {
|
|
1681
1700
|
const ch = combined[i];
|
|
1682
|
-
const
|
|
1683
|
-
if (
|
|
1684
|
-
embeddings[i] =
|
|
1701
|
+
const cachedEmb = this.embeddingCache.get(this.embedder.id, ch.contentHash);
|
|
1702
|
+
if (cachedEmb) {
|
|
1703
|
+
embeddings[i] = cachedEmb;
|
|
1704
|
+
cached2++;
|
|
1685
1705
|
continue;
|
|
1686
1706
|
}
|
|
1687
1707
|
embedTexts.push(
|
|
@@ -1713,11 +1733,22 @@ ${ch.text}`
|
|
|
1713
1733
|
this.embeddingCache.put(this.embedder.id, plan.contentHash, vecs[j]);
|
|
1714
1734
|
}
|
|
1715
1735
|
}
|
|
1736
|
+
this.emit({
|
|
1737
|
+
type: "repo/index/file/embed/done",
|
|
1738
|
+
repoRoot: this.repoRoot,
|
|
1739
|
+
path: posixRelPath,
|
|
1740
|
+
chunks: combined.length,
|
|
1741
|
+
cached: cached2,
|
|
1742
|
+
computed: embedTexts.length,
|
|
1743
|
+
ms: Date.now() - embedStartedAt
|
|
1744
|
+
});
|
|
1745
|
+
stage("embed", embedStartedAt);
|
|
1716
1746
|
const fileMtime = stat.mtimeMs;
|
|
1717
1747
|
const ftsMode = this.config.storage.ftsMode;
|
|
1718
1748
|
const storeText = this.config.storage.storeText;
|
|
1719
1749
|
const oldChunkIds = this.store.listChunksForFile(posixRelPath).map((r) => r.id);
|
|
1720
1750
|
const points = [];
|
|
1751
|
+
const storeStartedAt = Date.now();
|
|
1721
1752
|
const rows = combined.map((ch, i) => {
|
|
1722
1753
|
const id = sha256Hex(
|
|
1723
1754
|
`${this.repoId}:${posixRelPath}:${ch.kind}:${ch.startLine}:${ch.endLine}:${ch.contentHash}`
|
|
@@ -1745,7 +1776,19 @@ ${ch.text}`
|
|
|
1745
1776
|
this.store.replaceChunksForFile(posixRelPath, rows);
|
|
1746
1777
|
this.workspaceStore?.upsertFile(this.repoId, posixRelPath, fileHash, fileMtime, language, stat.size);
|
|
1747
1778
|
this.workspaceStore?.replaceChunksForFile(this.repoId, this.repoRoot, posixRelPath, rows);
|
|
1779
|
+
stage("store", storeStartedAt);
|
|
1780
|
+
const symbolsStartedAt = Date.now();
|
|
1748
1781
|
const symbolOut = await this.indexSymbolsIfEnabled(posixRelPath, language, redacted, fileHash);
|
|
1782
|
+
this.emit({
|
|
1783
|
+
type: "repo/index/file/symbols",
|
|
1784
|
+
repoRoot: this.repoRoot,
|
|
1785
|
+
path: posixRelPath,
|
|
1786
|
+
symbols: symbolOut?.symbols?.length ?? 0,
|
|
1787
|
+
edges: symbolOut?.edges?.length ?? 0,
|
|
1788
|
+
ms: Date.now() - symbolsStartedAt
|
|
1789
|
+
});
|
|
1790
|
+
stage("symbols", symbolsStartedAt);
|
|
1791
|
+
const graphStartedAt = Date.now();
|
|
1749
1792
|
const head = this.getHead();
|
|
1750
1793
|
if (this.graphStore && head) {
|
|
1751
1794
|
await this.graphStore.replaceFileGraph({
|
|
@@ -1761,7 +1804,10 @@ ${ch.text}`
|
|
|
1761
1804
|
symbolEdges: (symbolOut?.edges ?? []).map((e) => ({ fromId: e.fromId, toId: e.toId, kind: e.kind }))
|
|
1762
1805
|
}).catch(() => void 0);
|
|
1763
1806
|
}
|
|
1807
|
+
stage("graph", graphStartedAt);
|
|
1808
|
+
const vectorStartedAt = Date.now();
|
|
1764
1809
|
await this.ensureVectorUpToDate(points, oldChunkIds);
|
|
1810
|
+
stage("vector", vectorStartedAt);
|
|
1765
1811
|
this.emit({
|
|
1766
1812
|
type: "repo/index/file/done",
|
|
1767
1813
|
repoRoot: this.repoRoot,
|
|
@@ -2401,11 +2447,14 @@ var RepoIndexer = class {
|
|
|
2401
2447
|
const uniq3 = Array.from(new Set(posixPaths)).slice(0, maxFiles);
|
|
2402
2448
|
for (const p of uniq3) {
|
|
2403
2449
|
if (opts?.signal?.aborted) return;
|
|
2450
|
+
const startedAt = Date.now();
|
|
2451
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/start", repoRoot: this.repoRoot, path: p });
|
|
2404
2452
|
const abs = import_node_path12.default.join(this.repoRoot, p.split("/").join(import_node_path12.default.sep));
|
|
2405
2453
|
let text = "";
|
|
2406
2454
|
try {
|
|
2407
2455
|
text = import_node_fs9.default.readFileSync(abs, "utf8");
|
|
2408
2456
|
} catch {
|
|
2457
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/done", repoRoot: this.repoRoot, path: p, edges: 0, ms: Date.now() - startedAt });
|
|
2409
2458
|
continue;
|
|
2410
2459
|
}
|
|
2411
2460
|
const contentHash = this.store.getFileHash(p) ?? void 0;
|
|
@@ -2423,6 +2472,7 @@ var RepoIndexer = class {
|
|
|
2423
2472
|
fromPath: p,
|
|
2424
2473
|
edges: normalized.map((e) => ({ fromId: e.fromId, toId: e.toId, kind: e.kind, toPath: e.toPath }))
|
|
2425
2474
|
}).catch(() => void 0);
|
|
2475
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/done", repoRoot: this.repoRoot, path: p, edges: normalized.length, ms: Date.now() - startedAt });
|
|
2426
2476
|
}
|
|
2427
2477
|
}
|
|
2428
2478
|
async watch() {
|
|
@@ -4248,11 +4298,14 @@ var WorkspaceIndexer = class {
|
|
|
4248
4298
|
if (seeds.length >= 4) break;
|
|
4249
4299
|
}
|
|
4250
4300
|
if (seeds.length > 0) {
|
|
4301
|
+
const gs = Date.now();
|
|
4302
|
+
this.emitProgress({ type: "workspace/retrieve/graph/start", workspaceRoot: this.workspaceRoot, seeds: seeds.length });
|
|
4251
4303
|
graphNeighborFiles = await this.graphStore.neighborFiles({
|
|
4252
4304
|
seeds,
|
|
4253
4305
|
limit: profile.name === "architecture" ? 16 : 10,
|
|
4254
4306
|
kinds: ["definition", "reference", "implementation", "typeDefinition"]
|
|
4255
4307
|
});
|
|
4308
|
+
this.emitProgress({ type: "workspace/retrieve/graph/done", workspaceRoot: this.workspaceRoot, neighbors: graphNeighborFiles.length, ms: Date.now() - gs });
|
|
4256
4309
|
}
|
|
4257
4310
|
}
|
|
4258
4311
|
} catch {
|
package/dist/cli.js
CHANGED
package/dist/index.cjs
CHANGED
|
@@ -3408,6 +3408,16 @@ var RepoFileIndexer = class {
|
|
|
3408
3408
|
return;
|
|
3409
3409
|
}
|
|
3410
3410
|
this.emit({ type: "repo/index/file/start", repoRoot: this.repoRoot, path: posixRelPath });
|
|
3411
|
+
const stage = (stageName, stageStartedAt) => {
|
|
3412
|
+
this.emit({
|
|
3413
|
+
type: "repo/index/file/stage",
|
|
3414
|
+
repoRoot: this.repoRoot,
|
|
3415
|
+
path: posixRelPath,
|
|
3416
|
+
stage: stageName,
|
|
3417
|
+
ms: Date.now() - stageStartedAt
|
|
3418
|
+
});
|
|
3419
|
+
};
|
|
3420
|
+
const readStartedAt = Date.now();
|
|
3411
3421
|
const abs = import_node_path14.default.join(this.repoRoot, fromPosixPath(posixRelPath));
|
|
3412
3422
|
let stat;
|
|
3413
3423
|
try {
|
|
@@ -3431,16 +3441,20 @@ var RepoFileIndexer = class {
|
|
|
3431
3441
|
}
|
|
3432
3442
|
const raw = buf.toString("utf8");
|
|
3433
3443
|
const redacted = this.applyRedactions(raw);
|
|
3444
|
+
stage("read", readStartedAt);
|
|
3434
3445
|
const fileHash = sha256Hex(redacted);
|
|
3435
3446
|
const prev = this.store.getFileHash(posixRelPath);
|
|
3436
3447
|
if (prev === fileHash) {
|
|
3437
3448
|
this.emit({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "unchanged" });
|
|
3438
3449
|
return;
|
|
3439
3450
|
}
|
|
3451
|
+
const chunkStartedAt = Date.now();
|
|
3440
3452
|
const { language, chunks } = chunkSource(posixRelPath, redacted, this.config.chunk);
|
|
3453
|
+
stage("chunk", chunkStartedAt);
|
|
3441
3454
|
let imports = [];
|
|
3442
3455
|
let exports2 = [];
|
|
3443
3456
|
if (language === "typescript" || language === "javascript") {
|
|
3457
|
+
const relationsStartedAt = Date.now();
|
|
3444
3458
|
const rel = extractTsRelations(posixRelPath, redacted);
|
|
3445
3459
|
imports = rel.imports;
|
|
3446
3460
|
exports2 = rel.exports;
|
|
@@ -3448,12 +3462,14 @@ var RepoFileIndexer = class {
|
|
|
3448
3462
|
this.store.setEdges(posixRelPath, "export", rel.exports);
|
|
3449
3463
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "import", rel.imports);
|
|
3450
3464
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "export", rel.exports);
|
|
3465
|
+
stage("relations", relationsStartedAt);
|
|
3451
3466
|
} else {
|
|
3452
3467
|
this.store.setEdges(posixRelPath, "import", []);
|
|
3453
3468
|
this.store.setEdges(posixRelPath, "export", []);
|
|
3454
3469
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "import", []);
|
|
3455
3470
|
this.workspaceStore?.setEdges(this.repoId, posixRelPath, "export", []);
|
|
3456
3471
|
}
|
|
3472
|
+
const synopsisStartedAt = Date.now();
|
|
3457
3473
|
const synopsisText = buildSynopsis(posixRelPath, language, redacted);
|
|
3458
3474
|
const synopsis = synopsisText.trim() ? [
|
|
3459
3475
|
{
|
|
@@ -3478,14 +3494,18 @@ var RepoFileIndexer = class {
|
|
|
3478
3494
|
}
|
|
3479
3495
|
] : [];
|
|
3480
3496
|
const combined = [...synopsis, ...headerChunk, ...chunks.map((c) => ({ ...c, kind: "chunk" }))];
|
|
3497
|
+
stage("synopsis", synopsisStartedAt);
|
|
3498
|
+
const embedStartedAt = Date.now();
|
|
3481
3499
|
const embedTexts = [];
|
|
3482
3500
|
const embedPlan = [];
|
|
3483
3501
|
const embeddings = combined.map(() => null);
|
|
3502
|
+
let cached2 = 0;
|
|
3484
3503
|
for (let i = 0; i < combined.length; i++) {
|
|
3485
3504
|
const ch = combined[i];
|
|
3486
|
-
const
|
|
3487
|
-
if (
|
|
3488
|
-
embeddings[i] =
|
|
3505
|
+
const cachedEmb = this.embeddingCache.get(this.embedder.id, ch.contentHash);
|
|
3506
|
+
if (cachedEmb) {
|
|
3507
|
+
embeddings[i] = cachedEmb;
|
|
3508
|
+
cached2++;
|
|
3489
3509
|
continue;
|
|
3490
3510
|
}
|
|
3491
3511
|
embedTexts.push(
|
|
@@ -3517,11 +3537,22 @@ ${ch.text}`
|
|
|
3517
3537
|
this.embeddingCache.put(this.embedder.id, plan.contentHash, vecs[j]);
|
|
3518
3538
|
}
|
|
3519
3539
|
}
|
|
3540
|
+
this.emit({
|
|
3541
|
+
type: "repo/index/file/embed/done",
|
|
3542
|
+
repoRoot: this.repoRoot,
|
|
3543
|
+
path: posixRelPath,
|
|
3544
|
+
chunks: combined.length,
|
|
3545
|
+
cached: cached2,
|
|
3546
|
+
computed: embedTexts.length,
|
|
3547
|
+
ms: Date.now() - embedStartedAt
|
|
3548
|
+
});
|
|
3549
|
+
stage("embed", embedStartedAt);
|
|
3520
3550
|
const fileMtime = stat.mtimeMs;
|
|
3521
3551
|
const ftsMode = this.config.storage.ftsMode;
|
|
3522
3552
|
const storeText = this.config.storage.storeText;
|
|
3523
3553
|
const oldChunkIds = this.store.listChunksForFile(posixRelPath).map((r) => r.id);
|
|
3524
3554
|
const points = [];
|
|
3555
|
+
const storeStartedAt = Date.now();
|
|
3525
3556
|
const rows = combined.map((ch, i) => {
|
|
3526
3557
|
const id = sha256Hex(
|
|
3527
3558
|
`${this.repoId}:${posixRelPath}:${ch.kind}:${ch.startLine}:${ch.endLine}:${ch.contentHash}`
|
|
@@ -3549,7 +3580,19 @@ ${ch.text}`
|
|
|
3549
3580
|
this.store.replaceChunksForFile(posixRelPath, rows);
|
|
3550
3581
|
this.workspaceStore?.upsertFile(this.repoId, posixRelPath, fileHash, fileMtime, language, stat.size);
|
|
3551
3582
|
this.workspaceStore?.replaceChunksForFile(this.repoId, this.repoRoot, posixRelPath, rows);
|
|
3583
|
+
stage("store", storeStartedAt);
|
|
3584
|
+
const symbolsStartedAt = Date.now();
|
|
3552
3585
|
const symbolOut = await this.indexSymbolsIfEnabled(posixRelPath, language, redacted, fileHash);
|
|
3586
|
+
this.emit({
|
|
3587
|
+
type: "repo/index/file/symbols",
|
|
3588
|
+
repoRoot: this.repoRoot,
|
|
3589
|
+
path: posixRelPath,
|
|
3590
|
+
symbols: symbolOut?.symbols?.length ?? 0,
|
|
3591
|
+
edges: symbolOut?.edges?.length ?? 0,
|
|
3592
|
+
ms: Date.now() - symbolsStartedAt
|
|
3593
|
+
});
|
|
3594
|
+
stage("symbols", symbolsStartedAt);
|
|
3595
|
+
const graphStartedAt = Date.now();
|
|
3553
3596
|
const head = this.getHead();
|
|
3554
3597
|
if (this.graphStore && head) {
|
|
3555
3598
|
await this.graphStore.replaceFileGraph({
|
|
@@ -3565,7 +3608,10 @@ ${ch.text}`
|
|
|
3565
3608
|
symbolEdges: (symbolOut?.edges ?? []).map((e) => ({ fromId: e.fromId, toId: e.toId, kind: e.kind }))
|
|
3566
3609
|
}).catch(() => void 0);
|
|
3567
3610
|
}
|
|
3611
|
+
stage("graph", graphStartedAt);
|
|
3612
|
+
const vectorStartedAt = Date.now();
|
|
3568
3613
|
await this.ensureVectorUpToDate(points, oldChunkIds);
|
|
3614
|
+
stage("vector", vectorStartedAt);
|
|
3569
3615
|
this.emit({
|
|
3570
3616
|
type: "repo/index/file/done",
|
|
3571
3617
|
repoRoot: this.repoRoot,
|
|
@@ -4176,11 +4222,14 @@ var RepoIndexer = class {
|
|
|
4176
4222
|
const uniq3 = Array.from(new Set(posixPaths)).slice(0, maxFiles);
|
|
4177
4223
|
for (const p of uniq3) {
|
|
4178
4224
|
if (opts?.signal?.aborted) return;
|
|
4225
|
+
const startedAt = Date.now();
|
|
4226
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/start", repoRoot: this.repoRoot, path: p });
|
|
4179
4227
|
const abs = import_node_path17.default.join(this.repoRoot, p.split("/").join(import_node_path17.default.sep));
|
|
4180
4228
|
let text = "";
|
|
4181
4229
|
try {
|
|
4182
4230
|
text = import_node_fs12.default.readFileSync(abs, "utf8");
|
|
4183
4231
|
} catch {
|
|
4232
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/done", repoRoot: this.repoRoot, path: p, edges: 0, ms: Date.now() - startedAt });
|
|
4184
4233
|
continue;
|
|
4185
4234
|
}
|
|
4186
4235
|
const contentHash = this.store.getFileHash(p) ?? void 0;
|
|
@@ -4198,6 +4247,7 @@ var RepoIndexer = class {
|
|
|
4198
4247
|
fromPath: p,
|
|
4199
4248
|
edges: normalized.map((e) => ({ fromId: e.fromId, toId: e.toId, kind: e.kind, toPath: e.toPath }))
|
|
4200
4249
|
}).catch(() => void 0);
|
|
4250
|
+
this.emitProgress({ type: "repo/symbolGraph/expand/done", repoRoot: this.repoRoot, path: p, edges: normalized.length, ms: Date.now() - startedAt });
|
|
4201
4251
|
}
|
|
4202
4252
|
}
|
|
4203
4253
|
async watch() {
|
|
@@ -4784,11 +4834,14 @@ var WorkspaceIndexer = class {
|
|
|
4784
4834
|
if (seeds.length >= 4) break;
|
|
4785
4835
|
}
|
|
4786
4836
|
if (seeds.length > 0) {
|
|
4837
|
+
const gs = Date.now();
|
|
4838
|
+
this.emitProgress({ type: "workspace/retrieve/graph/start", workspaceRoot: this.workspaceRoot, seeds: seeds.length });
|
|
4787
4839
|
graphNeighborFiles = await this.graphStore.neighborFiles({
|
|
4788
4840
|
seeds,
|
|
4789
4841
|
limit: profile.name === "architecture" ? 16 : 10,
|
|
4790
4842
|
kinds: ["definition", "reference", "implementation", "typeDefinition"]
|
|
4791
4843
|
});
|
|
4844
|
+
this.emitProgress({ type: "workspace/retrieve/graph/done", workspaceRoot: this.workspaceRoot, neighbors: graphNeighborFiles.length, ms: Date.now() - gs });
|
|
4792
4845
|
}
|
|
4793
4846
|
}
|
|
4794
4847
|
} catch {
|
package/dist/index.d.cts
CHANGED
|
@@ -253,6 +253,15 @@ type IndexerProgressEvent = {
|
|
|
253
253
|
lexical: number;
|
|
254
254
|
merged: number;
|
|
255
255
|
};
|
|
256
|
+
} | {
|
|
257
|
+
type: "workspace/retrieve/graph/start";
|
|
258
|
+
workspaceRoot: string;
|
|
259
|
+
seeds: number;
|
|
260
|
+
} | {
|
|
261
|
+
type: "workspace/retrieve/graph/done";
|
|
262
|
+
workspaceRoot: string;
|
|
263
|
+
ms: number;
|
|
264
|
+
neighbors: number;
|
|
256
265
|
} | {
|
|
257
266
|
type: "repo/open";
|
|
258
267
|
repoRoot: string;
|
|
@@ -267,11 +276,32 @@ type IndexerProgressEvent = {
|
|
|
267
276
|
type: "repo/index/file/start";
|
|
268
277
|
repoRoot: string;
|
|
269
278
|
path: string;
|
|
279
|
+
} | {
|
|
280
|
+
type: "repo/index/file/stage";
|
|
281
|
+
repoRoot: string;
|
|
282
|
+
path: string;
|
|
283
|
+
stage: "read" | "chunk" | "relations" | "synopsis" | "embed" | "store" | "symbols" | "graph" | "vector";
|
|
284
|
+
ms?: number;
|
|
270
285
|
} | {
|
|
271
286
|
type: "repo/index/file/skip";
|
|
272
287
|
repoRoot: string;
|
|
273
288
|
path: string;
|
|
274
289
|
reason: string;
|
|
290
|
+
} | {
|
|
291
|
+
type: "repo/index/file/embed/done";
|
|
292
|
+
repoRoot: string;
|
|
293
|
+
path: string;
|
|
294
|
+
chunks: number;
|
|
295
|
+
cached: number;
|
|
296
|
+
computed: number;
|
|
297
|
+
ms: number;
|
|
298
|
+
} | {
|
|
299
|
+
type: "repo/index/file/symbols";
|
|
300
|
+
repoRoot: string;
|
|
301
|
+
path: string;
|
|
302
|
+
symbols: number;
|
|
303
|
+
edges: number;
|
|
304
|
+
ms: number;
|
|
275
305
|
} | {
|
|
276
306
|
type: "repo/index/file/done";
|
|
277
307
|
repoRoot: string;
|
|
@@ -306,6 +336,16 @@ type IndexerProgressEvent = {
|
|
|
306
336
|
type: "repo/vector/flush";
|
|
307
337
|
repoRoot: string;
|
|
308
338
|
kind: string;
|
|
339
|
+
} | {
|
|
340
|
+
type: "repo/symbolGraph/expand/start";
|
|
341
|
+
repoRoot: string;
|
|
342
|
+
path: string;
|
|
343
|
+
} | {
|
|
344
|
+
type: "repo/symbolGraph/expand/done";
|
|
345
|
+
repoRoot: string;
|
|
346
|
+
path: string;
|
|
347
|
+
edges: number;
|
|
348
|
+
ms: number;
|
|
309
349
|
} | {
|
|
310
350
|
type: "repo/watch/start";
|
|
311
351
|
repoRoot: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -253,6 +253,15 @@ type IndexerProgressEvent = {
|
|
|
253
253
|
lexical: number;
|
|
254
254
|
merged: number;
|
|
255
255
|
};
|
|
256
|
+
} | {
|
|
257
|
+
type: "workspace/retrieve/graph/start";
|
|
258
|
+
workspaceRoot: string;
|
|
259
|
+
seeds: number;
|
|
260
|
+
} | {
|
|
261
|
+
type: "workspace/retrieve/graph/done";
|
|
262
|
+
workspaceRoot: string;
|
|
263
|
+
ms: number;
|
|
264
|
+
neighbors: number;
|
|
256
265
|
} | {
|
|
257
266
|
type: "repo/open";
|
|
258
267
|
repoRoot: string;
|
|
@@ -267,11 +276,32 @@ type IndexerProgressEvent = {
|
|
|
267
276
|
type: "repo/index/file/start";
|
|
268
277
|
repoRoot: string;
|
|
269
278
|
path: string;
|
|
279
|
+
} | {
|
|
280
|
+
type: "repo/index/file/stage";
|
|
281
|
+
repoRoot: string;
|
|
282
|
+
path: string;
|
|
283
|
+
stage: "read" | "chunk" | "relations" | "synopsis" | "embed" | "store" | "symbols" | "graph" | "vector";
|
|
284
|
+
ms?: number;
|
|
270
285
|
} | {
|
|
271
286
|
type: "repo/index/file/skip";
|
|
272
287
|
repoRoot: string;
|
|
273
288
|
path: string;
|
|
274
289
|
reason: string;
|
|
290
|
+
} | {
|
|
291
|
+
type: "repo/index/file/embed/done";
|
|
292
|
+
repoRoot: string;
|
|
293
|
+
path: string;
|
|
294
|
+
chunks: number;
|
|
295
|
+
cached: number;
|
|
296
|
+
computed: number;
|
|
297
|
+
ms: number;
|
|
298
|
+
} | {
|
|
299
|
+
type: "repo/index/file/symbols";
|
|
300
|
+
repoRoot: string;
|
|
301
|
+
path: string;
|
|
302
|
+
symbols: number;
|
|
303
|
+
edges: number;
|
|
304
|
+
ms: number;
|
|
275
305
|
} | {
|
|
276
306
|
type: "repo/index/file/done";
|
|
277
307
|
repoRoot: string;
|
|
@@ -306,6 +336,16 @@ type IndexerProgressEvent = {
|
|
|
306
336
|
type: "repo/vector/flush";
|
|
307
337
|
repoRoot: string;
|
|
308
338
|
kind: string;
|
|
339
|
+
} | {
|
|
340
|
+
type: "repo/symbolGraph/expand/start";
|
|
341
|
+
repoRoot: string;
|
|
342
|
+
path: string;
|
|
343
|
+
} | {
|
|
344
|
+
type: "repo/symbolGraph/expand/done";
|
|
345
|
+
repoRoot: string;
|
|
346
|
+
path: string;
|
|
347
|
+
edges: number;
|
|
348
|
+
ms: number;
|
|
309
349
|
} | {
|
|
310
350
|
type: "repo/watch/start";
|
|
311
351
|
repoRoot: string;
|
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@neuralsea/workspace-indexer",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.4",
|
|
4
4
|
"description": "Local-first multi-repo workspace indexer (semantic embeddings + git-aware incremental updates + hybrid retrieval profiles) for AI agents.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|