@neat.is/core 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-66Z5IEFY.js → chunk-ICFH326H.js} +2 -2
- package/dist/{chunk-FIXKIYNF.js → chunk-VABXPLDT.js} +457 -206
- package/dist/chunk-VABXPLDT.js.map +1 -0
- package/dist/cli.cjs +601 -306
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +100 -53
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +451 -221
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +2 -2
- package/dist/neatd.cjs +470 -228
- package/dist/neatd.cjs.map +1 -1
- package/dist/neatd.js +16 -4
- package/dist/neatd.js.map +1 -1
- package/dist/server.cjs +420 -190
- package/dist/server.cjs.map +1 -1
- package/dist/server.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-FIXKIYNF.js.map +0 -1
- /package/dist/{chunk-66Z5IEFY.js.map → chunk-ICFH326H.js.map} +0 -0
|
@@ -427,19 +427,19 @@ function confidenceFromMix(edges, now = Date.now()) {
|
|
|
427
427
|
function longestIncomingWalk(graph, start, maxDepth) {
|
|
428
428
|
let best = { path: [start], edges: [] };
|
|
429
429
|
const visited = /* @__PURE__ */ new Set([start]);
|
|
430
|
-
function step(node,
|
|
431
|
-
if (
|
|
432
|
-
best = { path: [...
|
|
430
|
+
function step(node, path33, edges) {
|
|
431
|
+
if (path33.length > best.path.length) {
|
|
432
|
+
best = { path: [...path33], edges: [...edges] };
|
|
433
433
|
}
|
|
434
|
-
if (
|
|
434
|
+
if (path33.length - 1 >= maxDepth) return;
|
|
435
435
|
const incoming = bestEdgeBySource(graph, graph.inboundEdges(node));
|
|
436
436
|
for (const [srcId, edge] of incoming) {
|
|
437
437
|
if (visited.has(srcId)) continue;
|
|
438
438
|
visited.add(srcId);
|
|
439
|
-
|
|
439
|
+
path33.push(srcId);
|
|
440
440
|
edges.push(edge);
|
|
441
|
-
step(srcId,
|
|
442
|
-
|
|
441
|
+
step(srcId, path33, edges);
|
|
442
|
+
path33.pop();
|
|
443
443
|
edges.pop();
|
|
444
444
|
visited.delete(srcId);
|
|
445
445
|
}
|
|
@@ -1560,16 +1560,50 @@ async function readErrorEvents(errorsPath) {
|
|
|
1560
1560
|
}
|
|
1561
1561
|
}
|
|
1562
1562
|
|
|
1563
|
+
// src/extract/errors.ts
|
|
1564
|
+
import { promises as fs4 } from "fs";
|
|
1565
|
+
import path4 from "path";
|
|
1566
|
+
var sink = [];
|
|
1567
|
+
function recordExtractionError(producer, file, err) {
|
|
1568
|
+
const e = err instanceof Error ? err : new Error(String(err));
|
|
1569
|
+
sink.push({
|
|
1570
|
+
producer,
|
|
1571
|
+
file,
|
|
1572
|
+
error: e.message,
|
|
1573
|
+
stack: e.stack,
|
|
1574
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1575
|
+
source: "extract"
|
|
1576
|
+
});
|
|
1577
|
+
console.warn(`[neat] ${producer} skipped ${file}: ${e.message}`);
|
|
1578
|
+
}
|
|
1579
|
+
function drainExtractionErrors() {
|
|
1580
|
+
return sink.splice(0, sink.length);
|
|
1581
|
+
}
|
|
1582
|
+
async function writeExtractionErrors(errors, errorsPath) {
|
|
1583
|
+
if (errors.length === 0) return;
|
|
1584
|
+
await fs4.mkdir(path4.dirname(errorsPath), { recursive: true });
|
|
1585
|
+
const lines = errors.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
1586
|
+
await fs4.appendFile(errorsPath, lines, "utf8");
|
|
1587
|
+
}
|
|
1588
|
+
function isStrictExtractionEnabled() {
|
|
1589
|
+
const raw = process.env.NEAT_STRICT_EXTRACTION;
|
|
1590
|
+
return raw === "1" || raw === "true";
|
|
1591
|
+
}
|
|
1592
|
+
function formatExtractionBanner(count) {
|
|
1593
|
+
if (count === 1) return `[neat] 1 file skipped due to parse errors`;
|
|
1594
|
+
return `[neat] ${count} files skipped due to parse errors`;
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1563
1597
|
// src/extract/services.ts
|
|
1564
|
-
import { promises as
|
|
1565
|
-
import
|
|
1598
|
+
import { promises as fs8 } from "fs";
|
|
1599
|
+
import path8 from "path";
|
|
1566
1600
|
import ignore from "ignore";
|
|
1567
1601
|
import { minimatch as minimatch2 } from "minimatch";
|
|
1568
1602
|
import { NodeType as NodeType4, serviceId as serviceId2 } from "@neat.is/types";
|
|
1569
1603
|
|
|
1570
1604
|
// src/extract/shared.ts
|
|
1571
|
-
import { promises as
|
|
1572
|
-
import
|
|
1605
|
+
import { promises as fs5 } from "fs";
|
|
1606
|
+
import path5 from "path";
|
|
1573
1607
|
import { parse as parseYaml } from "yaml";
|
|
1574
1608
|
import { extractedEdgeId as extractedEdgeId2 } from "@neat.is/types";
|
|
1575
1609
|
var SERVICE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([".js", ".mjs", ".cjs", ".ts", ".tsx", ".py"]);
|
|
@@ -1583,26 +1617,124 @@ var IGNORED_DIRS = /* @__PURE__ */ new Set([
|
|
|
1583
1617
|
".next"
|
|
1584
1618
|
]);
|
|
1585
1619
|
function isConfigFile(name) {
|
|
1586
|
-
const ext =
|
|
1620
|
+
const ext = path5.extname(name);
|
|
1587
1621
|
if (CONFIG_FILE_EXTENSIONS.has(ext)) return { match: true, fileType: ext.slice(1) };
|
|
1588
|
-
if (name === ".env" || name.startsWith(".env."))
|
|
1622
|
+
if (name === ".env" || name.startsWith(".env.")) {
|
|
1623
|
+
if (isEnvTemplateFile(name)) return { match: false, fileType: "" };
|
|
1624
|
+
return { match: true, fileType: "env" };
|
|
1625
|
+
}
|
|
1589
1626
|
return { match: false, fileType: "" };
|
|
1590
1627
|
}
|
|
1628
|
+
function isTestPath(filePath) {
|
|
1629
|
+
const normalised = filePath.replace(/\\/g, "/");
|
|
1630
|
+
const segments = normalised.split("/");
|
|
1631
|
+
for (const seg of segments) {
|
|
1632
|
+
if (seg === "__tests__" || seg === "__fixtures__" || seg === "integration-tests") {
|
|
1633
|
+
return true;
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
const base = segments[segments.length - 1] ?? "";
|
|
1637
|
+
return /\.(spec|test)\.(?:tsx?|jsx?|mjs|cjs|py)$/i.test(base);
|
|
1638
|
+
}
|
|
1639
|
+
function isEnvTemplateFile(name) {
|
|
1640
|
+
if (name === ".env.template" || name === ".env.example" || name === ".env.sample") {
|
|
1641
|
+
return true;
|
|
1642
|
+
}
|
|
1643
|
+
return /^\.env\.[^.]+\.(?:template|example|sample)$/i.test(name);
|
|
1644
|
+
}
|
|
1645
|
+
function maskCommentsInSource(src) {
|
|
1646
|
+
const len = src.length;
|
|
1647
|
+
const out = new Array(len);
|
|
1648
|
+
let i = 0;
|
|
1649
|
+
let inString = 0;
|
|
1650
|
+
let escaped = false;
|
|
1651
|
+
while (i < len) {
|
|
1652
|
+
const c = src[i];
|
|
1653
|
+
if (inString !== 0) {
|
|
1654
|
+
out[i] = c;
|
|
1655
|
+
if (escaped) {
|
|
1656
|
+
escaped = false;
|
|
1657
|
+
} else if (c === "\\") {
|
|
1658
|
+
escaped = true;
|
|
1659
|
+
} else if (c === inString) {
|
|
1660
|
+
inString = 0;
|
|
1661
|
+
}
|
|
1662
|
+
i++;
|
|
1663
|
+
continue;
|
|
1664
|
+
}
|
|
1665
|
+
if (c === "/" && i + 1 < len) {
|
|
1666
|
+
const next = src[i + 1];
|
|
1667
|
+
if (next === "/") {
|
|
1668
|
+
out[i] = " ";
|
|
1669
|
+
out[i + 1] = " ";
|
|
1670
|
+
let j = i + 2;
|
|
1671
|
+
while (j < len && src[j] !== "\n") {
|
|
1672
|
+
out[j] = " ";
|
|
1673
|
+
j++;
|
|
1674
|
+
}
|
|
1675
|
+
i = j;
|
|
1676
|
+
continue;
|
|
1677
|
+
}
|
|
1678
|
+
if (next === "*") {
|
|
1679
|
+
out[i] = " ";
|
|
1680
|
+
out[i + 1] = " ";
|
|
1681
|
+
let j = i + 2;
|
|
1682
|
+
while (j < len) {
|
|
1683
|
+
if (src[j] === "\n") {
|
|
1684
|
+
out[j] = "\n";
|
|
1685
|
+
j++;
|
|
1686
|
+
continue;
|
|
1687
|
+
}
|
|
1688
|
+
if (src[j] === "*" && j + 1 < len && src[j + 1] === "/") {
|
|
1689
|
+
out[j] = " ";
|
|
1690
|
+
out[j + 1] = " ";
|
|
1691
|
+
j += 2;
|
|
1692
|
+
break;
|
|
1693
|
+
}
|
|
1694
|
+
out[j] = " ";
|
|
1695
|
+
j++;
|
|
1696
|
+
}
|
|
1697
|
+
i = j;
|
|
1698
|
+
continue;
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
out[i] = c;
|
|
1702
|
+
if (c === "'" || c === '"' || c === "`") inString = c;
|
|
1703
|
+
i++;
|
|
1704
|
+
}
|
|
1705
|
+
return out.join("");
|
|
1706
|
+
}
|
|
1707
|
+
var URL_LIKE = /^(?:[a-z][a-z0-9+.-]*:)?\/\//i;
|
|
1708
|
+
function urlMatchesHost(urlString, host) {
|
|
1709
|
+
if (typeof urlString !== "string" || urlString.length === 0) return false;
|
|
1710
|
+
if (!URL_LIKE.test(urlString)) return false;
|
|
1711
|
+
const [wantedHost, wantedPort] = host.split(":");
|
|
1712
|
+
let parsed;
|
|
1713
|
+
try {
|
|
1714
|
+
const candidate = urlString.startsWith("//") ? `http:${urlString}` : urlString;
|
|
1715
|
+
parsed = new URL(candidate);
|
|
1716
|
+
} catch {
|
|
1717
|
+
return false;
|
|
1718
|
+
}
|
|
1719
|
+
if (parsed.hostname.toLowerCase() !== (wantedHost ?? "").toLowerCase()) return false;
|
|
1720
|
+
if (wantedPort && parsed.port !== wantedPort) return false;
|
|
1721
|
+
return true;
|
|
1722
|
+
}
|
|
1591
1723
|
function cleanVersion(raw) {
|
|
1592
1724
|
if (!raw) return void 0;
|
|
1593
1725
|
return raw.replace(/^[\^~><=v\s]+/, "").trim() || void 0;
|
|
1594
1726
|
}
|
|
1595
1727
|
async function readJson(filePath) {
|
|
1596
|
-
const raw = await
|
|
1728
|
+
const raw = await fs5.readFile(filePath, "utf8");
|
|
1597
1729
|
return JSON.parse(raw);
|
|
1598
1730
|
}
|
|
1599
1731
|
async function readYaml(filePath) {
|
|
1600
|
-
const raw = await
|
|
1732
|
+
const raw = await fs5.readFile(filePath, "utf8");
|
|
1601
1733
|
return parseYaml(raw);
|
|
1602
1734
|
}
|
|
1603
1735
|
async function exists(p) {
|
|
1604
1736
|
try {
|
|
1605
|
-
await
|
|
1737
|
+
await fs5.access(p);
|
|
1606
1738
|
return true;
|
|
1607
1739
|
} catch {
|
|
1608
1740
|
return false;
|
|
@@ -1610,8 +1742,8 @@ async function exists(p) {
|
|
|
1610
1742
|
}
|
|
1611
1743
|
|
|
1612
1744
|
// src/extract/python.ts
|
|
1613
|
-
import { promises as
|
|
1614
|
-
import
|
|
1745
|
+
import { promises as fs6 } from "fs";
|
|
1746
|
+
import path6 from "path";
|
|
1615
1747
|
import { parse as parseToml } from "smol-toml";
|
|
1616
1748
|
var REQUIREMENT_LINE = /^\s*([A-Za-z0-9_.-]+)(?:\[[^\]]*\])?\s*(?:(==)\s*([A-Za-z0-9_.+-]+))?/;
|
|
1617
1749
|
function parseRequirementsTxt(content) {
|
|
@@ -1644,25 +1776,25 @@ function depsFromPyProject(pyproject) {
|
|
|
1644
1776
|
return out;
|
|
1645
1777
|
}
|
|
1646
1778
|
async function discoverPythonService(serviceDir) {
|
|
1647
|
-
const pyprojectPath =
|
|
1648
|
-
const requirementsPath =
|
|
1649
|
-
const setupPath =
|
|
1779
|
+
const pyprojectPath = path6.join(serviceDir, "pyproject.toml");
|
|
1780
|
+
const requirementsPath = path6.join(serviceDir, "requirements.txt");
|
|
1781
|
+
const setupPath = path6.join(serviceDir, "setup.py");
|
|
1650
1782
|
const hasPyproject = await exists(pyprojectPath);
|
|
1651
1783
|
const hasRequirements = await exists(requirementsPath);
|
|
1652
1784
|
const hasSetup = await exists(setupPath);
|
|
1653
1785
|
if (!hasPyproject && !hasRequirements && !hasSetup) return null;
|
|
1654
|
-
let name =
|
|
1786
|
+
let name = path6.basename(serviceDir);
|
|
1655
1787
|
let version;
|
|
1656
1788
|
const dependencies = {};
|
|
1657
1789
|
if (hasPyproject) {
|
|
1658
|
-
const raw = await
|
|
1790
|
+
const raw = await fs6.readFile(pyprojectPath, "utf8");
|
|
1659
1791
|
const pyproject = parseToml(raw);
|
|
1660
1792
|
name = pyproject.project?.name ?? pyproject.tool?.poetry?.name ?? name;
|
|
1661
1793
|
version = pyproject.project?.version ?? pyproject.tool?.poetry?.version ?? void 0;
|
|
1662
1794
|
Object.assign(dependencies, depsFromPyProject(pyproject));
|
|
1663
1795
|
}
|
|
1664
1796
|
if (hasRequirements) {
|
|
1665
|
-
const raw = await
|
|
1797
|
+
const raw = await fs6.readFile(requirementsPath, "utf8");
|
|
1666
1798
|
Object.assign(dependencies, parseRequirementsTxt(raw));
|
|
1667
1799
|
}
|
|
1668
1800
|
return { name, version, dependencies };
|
|
@@ -1676,17 +1808,17 @@ function pythonToPackage(service) {
|
|
|
1676
1808
|
}
|
|
1677
1809
|
|
|
1678
1810
|
// src/extract/owners.ts
|
|
1679
|
-
import { promises as
|
|
1680
|
-
import
|
|
1811
|
+
import { promises as fs7 } from "fs";
|
|
1812
|
+
import path7 from "path";
|
|
1681
1813
|
import { minimatch } from "minimatch";
|
|
1682
1814
|
async function loadCodeowners(scanPath) {
|
|
1683
1815
|
const candidates = [
|
|
1684
|
-
|
|
1685
|
-
|
|
1816
|
+
path7.join(scanPath, "CODEOWNERS"),
|
|
1817
|
+
path7.join(scanPath, ".github", "CODEOWNERS")
|
|
1686
1818
|
];
|
|
1687
1819
|
for (const file of candidates) {
|
|
1688
1820
|
if (await exists(file)) {
|
|
1689
|
-
const raw = await
|
|
1821
|
+
const raw = await fs7.readFile(file, "utf8");
|
|
1690
1822
|
return parseCodeowners(raw);
|
|
1691
1823
|
}
|
|
1692
1824
|
}
|
|
@@ -1704,7 +1836,7 @@ function parseCodeowners(raw) {
|
|
|
1704
1836
|
return { rules };
|
|
1705
1837
|
}
|
|
1706
1838
|
function matchOwner(file, repoPath) {
|
|
1707
|
-
const normalized = repoPath.split(
|
|
1839
|
+
const normalized = repoPath.split(path7.sep).join("/");
|
|
1708
1840
|
for (const rule of file.rules) {
|
|
1709
1841
|
if (matchesPattern(rule.pattern, normalized)) return rule.owners;
|
|
1710
1842
|
}
|
|
@@ -1720,7 +1852,7 @@ function matchesPattern(rawPattern, repoPath) {
|
|
|
1720
1852
|
return false;
|
|
1721
1853
|
}
|
|
1722
1854
|
async function readPackageJsonAuthor(serviceDir) {
|
|
1723
|
-
const pkgPath =
|
|
1855
|
+
const pkgPath = path7.join(serviceDir, "package.json");
|
|
1724
1856
|
if (!await exists(pkgPath)) return null;
|
|
1725
1857
|
try {
|
|
1726
1858
|
const pkg = await readJson(pkgPath);
|
|
@@ -1757,21 +1889,21 @@ function workspaceGlobs(pkg) {
|
|
|
1757
1889
|
return null;
|
|
1758
1890
|
}
|
|
1759
1891
|
async function loadGitignore(scanPath) {
|
|
1760
|
-
const gitignorePath =
|
|
1892
|
+
const gitignorePath = path8.join(scanPath, ".gitignore");
|
|
1761
1893
|
if (!await exists(gitignorePath)) return null;
|
|
1762
|
-
const raw = await
|
|
1894
|
+
const raw = await fs8.readFile(gitignorePath, "utf8");
|
|
1763
1895
|
return ignore().add(raw);
|
|
1764
1896
|
}
|
|
1765
1897
|
async function walkDirs(start, scanPath, options, visit) {
|
|
1766
1898
|
async function recurse(current, depth) {
|
|
1767
1899
|
if (depth > options.maxDepth) return;
|
|
1768
|
-
const entries = await
|
|
1900
|
+
const entries = await fs8.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
1769
1901
|
for (const entry of entries) {
|
|
1770
1902
|
if (!entry.isDirectory()) continue;
|
|
1771
1903
|
if (IGNORED_DIRS.has(entry.name)) continue;
|
|
1772
|
-
const child =
|
|
1904
|
+
const child = path8.join(current, entry.name);
|
|
1773
1905
|
if (options.ig) {
|
|
1774
|
-
const rel =
|
|
1906
|
+
const rel = path8.relative(scanPath, child).split(path8.sep).join("/");
|
|
1775
1907
|
if (rel && options.ig.ignores(rel + "/")) continue;
|
|
1776
1908
|
}
|
|
1777
1909
|
await visit(child);
|
|
@@ -1786,8 +1918,8 @@ async function expandWorkspaceGlobs(scanPath, globs) {
|
|
|
1786
1918
|
for (const raw of globs) {
|
|
1787
1919
|
const pattern = raw.replace(/^\.\//, "");
|
|
1788
1920
|
if (!pattern.includes("*")) {
|
|
1789
|
-
const candidate =
|
|
1790
|
-
if (await exists(
|
|
1921
|
+
const candidate = path8.join(scanPath, pattern);
|
|
1922
|
+
if (await exists(path8.join(candidate, "package.json"))) found.add(candidate);
|
|
1791
1923
|
continue;
|
|
1792
1924
|
}
|
|
1793
1925
|
const segments = pattern.split("/");
|
|
@@ -1796,13 +1928,13 @@ async function expandWorkspaceGlobs(scanPath, globs) {
|
|
|
1796
1928
|
if (seg.includes("*")) break;
|
|
1797
1929
|
staticSegments.push(seg);
|
|
1798
1930
|
}
|
|
1799
|
-
const start =
|
|
1931
|
+
const start = path8.join(scanPath, ...staticSegments);
|
|
1800
1932
|
if (!await exists(start)) continue;
|
|
1801
1933
|
const hasDoubleStar = pattern.includes("**");
|
|
1802
1934
|
const walkDepth = hasDoubleStar ? scanDepth : Math.max(0, segments.length - staticSegments.length - 1);
|
|
1803
1935
|
await walkDirs(start, scanPath, { maxDepth: walkDepth, ig: null }, async (dir) => {
|
|
1804
|
-
const rel =
|
|
1805
|
-
if (minimatch2(rel, pattern) && await exists(
|
|
1936
|
+
const rel = path8.relative(scanPath, dir).split(path8.sep).join("/");
|
|
1937
|
+
if (minimatch2(rel, pattern) && await exists(path8.join(dir, "package.json"))) {
|
|
1806
1938
|
found.add(dir);
|
|
1807
1939
|
}
|
|
1808
1940
|
});
|
|
@@ -1810,15 +1942,13 @@ async function expandWorkspaceGlobs(scanPath, globs) {
|
|
|
1810
1942
|
return [...found];
|
|
1811
1943
|
}
|
|
1812
1944
|
async function discoverNodeService(scanPath, dir) {
|
|
1813
|
-
const pkgPath =
|
|
1945
|
+
const pkgPath = path8.join(dir, "package.json");
|
|
1814
1946
|
if (!await exists(pkgPath)) return null;
|
|
1815
1947
|
let pkg;
|
|
1816
1948
|
try {
|
|
1817
1949
|
pkg = await readJson(pkgPath);
|
|
1818
1950
|
} catch (err) {
|
|
1819
|
-
|
|
1820
|
-
`[neat] services skipped ${path7.relative(scanPath, pkgPath)}: ${err.message}`
|
|
1821
|
-
);
|
|
1951
|
+
recordExtractionError("services", path8.relative(scanPath, pkgPath), err);
|
|
1822
1952
|
return null;
|
|
1823
1953
|
}
|
|
1824
1954
|
if (!pkg.name) return null;
|
|
@@ -1829,7 +1959,7 @@ async function discoverNodeService(scanPath, dir) {
|
|
|
1829
1959
|
language: "javascript",
|
|
1830
1960
|
version: pkg.version,
|
|
1831
1961
|
dependencies: pkg.dependencies ?? {},
|
|
1832
|
-
repoPath:
|
|
1962
|
+
repoPath: path8.relative(scanPath, dir),
|
|
1833
1963
|
...pkg.engines?.node ? { nodeEngine: pkg.engines.node } : {}
|
|
1834
1964
|
};
|
|
1835
1965
|
return { pkg, dir, node };
|
|
@@ -1845,19 +1975,21 @@ async function discoverPyService(scanPath, dir) {
|
|
|
1845
1975
|
language: "python",
|
|
1846
1976
|
version: py.version,
|
|
1847
1977
|
dependencies: py.dependencies,
|
|
1848
|
-
repoPath:
|
|
1978
|
+
repoPath: path8.relative(scanPath, dir)
|
|
1849
1979
|
};
|
|
1850
1980
|
return { pkg, dir, node };
|
|
1851
1981
|
}
|
|
1852
1982
|
async function discoverServices(scanPath) {
|
|
1853
|
-
const rootPkgPath =
|
|
1983
|
+
const rootPkgPath = path8.join(scanPath, "package.json");
|
|
1854
1984
|
let rootPkg = null;
|
|
1855
1985
|
if (await exists(rootPkgPath)) {
|
|
1856
1986
|
try {
|
|
1857
1987
|
rootPkg = await readJson(rootPkgPath);
|
|
1858
1988
|
} catch (err) {
|
|
1859
|
-
|
|
1860
|
-
|
|
1989
|
+
recordExtractionError(
|
|
1990
|
+
"services workspaces",
|
|
1991
|
+
path8.relative(scanPath, rootPkgPath),
|
|
1992
|
+
err
|
|
1861
1993
|
);
|
|
1862
1994
|
}
|
|
1863
1995
|
}
|
|
@@ -1873,9 +2005,9 @@ async function discoverServices(scanPath) {
|
|
|
1873
2005
|
scanPath,
|
|
1874
2006
|
{ maxDepth: parseScanDepth(), ig },
|
|
1875
2007
|
async (dir) => {
|
|
1876
|
-
if (await exists(
|
|
2008
|
+
if (await exists(path8.join(dir, "package.json"))) {
|
|
1877
2009
|
candidateDirs.push(dir);
|
|
1878
|
-
} else if (await exists(
|
|
2010
|
+
} else if (await exists(path8.join(dir, "pyproject.toml")) || await exists(path8.join(dir, "requirements.txt")) || await exists(path8.join(dir, "setup.py"))) {
|
|
1879
2011
|
candidateDirs.push(dir);
|
|
1880
2012
|
}
|
|
1881
2013
|
}
|
|
@@ -1889,8 +2021,8 @@ async function discoverServices(scanPath) {
|
|
|
1889
2021
|
if (!service) continue;
|
|
1890
2022
|
const existingDir = seen.get(service.node.name);
|
|
1891
2023
|
if (existingDir !== void 0) {
|
|
1892
|
-
const a =
|
|
1893
|
-
const b =
|
|
2024
|
+
const a = path8.relative(scanPath, existingDir) || ".";
|
|
2025
|
+
const b = path8.relative(scanPath, dir) || ".";
|
|
1894
2026
|
console.warn(
|
|
1895
2027
|
`[neat] duplicate package name "${service.node.name}" \u2014 keeping ${a}, ignoring ${b}`
|
|
1896
2028
|
);
|
|
@@ -1926,8 +2058,8 @@ function addServiceNodes(graph, services) {
|
|
|
1926
2058
|
}
|
|
1927
2059
|
|
|
1928
2060
|
// src/extract/aliases.ts
|
|
1929
|
-
import
|
|
1930
|
-
import { promises as
|
|
2061
|
+
import path9 from "path";
|
|
2062
|
+
import { promises as fs9 } from "fs";
|
|
1931
2063
|
import { parseAllDocuments } from "yaml";
|
|
1932
2064
|
import { NodeType as NodeType5 } from "@neat.is/types";
|
|
1933
2065
|
var K8S_KINDS_WITH_HOSTNAMES = /* @__PURE__ */ new Set([
|
|
@@ -1954,14 +2086,14 @@ function indexServicesByName(services) {
|
|
|
1954
2086
|
const map = /* @__PURE__ */ new Map();
|
|
1955
2087
|
for (const s of services) {
|
|
1956
2088
|
map.set(s.node.name, s.node.id);
|
|
1957
|
-
map.set(
|
|
2089
|
+
map.set(path9.basename(s.dir), s.node.id);
|
|
1958
2090
|
}
|
|
1959
2091
|
return map;
|
|
1960
2092
|
}
|
|
1961
2093
|
async function collectComposeAliases(graph, scanPath, serviceIndex) {
|
|
1962
2094
|
let composePath = null;
|
|
1963
2095
|
for (const name of ["docker-compose.yml", "docker-compose.yaml"]) {
|
|
1964
|
-
const abs =
|
|
2096
|
+
const abs = path9.join(scanPath, name);
|
|
1965
2097
|
if (await exists(abs)) {
|
|
1966
2098
|
composePath = abs;
|
|
1967
2099
|
break;
|
|
@@ -1972,8 +2104,10 @@ async function collectComposeAliases(graph, scanPath, serviceIndex) {
|
|
|
1972
2104
|
try {
|
|
1973
2105
|
compose = await readYaml(composePath);
|
|
1974
2106
|
} catch (err) {
|
|
1975
|
-
|
|
1976
|
-
|
|
2107
|
+
recordExtractionError(
|
|
2108
|
+
"aliases compose",
|
|
2109
|
+
path9.relative(scanPath, composePath),
|
|
2110
|
+
err
|
|
1977
2111
|
);
|
|
1978
2112
|
return;
|
|
1979
2113
|
}
|
|
@@ -2015,15 +2149,13 @@ function parseDockerfileLabels(content) {
|
|
|
2015
2149
|
}
|
|
2016
2150
|
async function collectDockerfileAliases(graph, services) {
|
|
2017
2151
|
for (const service of services) {
|
|
2018
|
-
const dockerfilePath =
|
|
2152
|
+
const dockerfilePath = path9.join(service.dir, "Dockerfile");
|
|
2019
2153
|
if (!await exists(dockerfilePath)) continue;
|
|
2020
2154
|
let content;
|
|
2021
2155
|
try {
|
|
2022
|
-
content = await
|
|
2156
|
+
content = await fs9.readFile(dockerfilePath, "utf8");
|
|
2023
2157
|
} catch (err) {
|
|
2024
|
-
|
|
2025
|
-
`[neat] aliases dockerfile skipped ${dockerfilePath}: ${err.message}`
|
|
2026
|
-
);
|
|
2158
|
+
recordExtractionError("aliases dockerfile", dockerfilePath, err);
|
|
2027
2159
|
continue;
|
|
2028
2160
|
}
|
|
2029
2161
|
const aliases = parseDockerfileLabels(content);
|
|
@@ -2033,13 +2165,13 @@ async function collectDockerfileAliases(graph, services) {
|
|
|
2033
2165
|
async function walkYamlFiles(start, depth = 0, max = 5) {
|
|
2034
2166
|
if (depth > max) return [];
|
|
2035
2167
|
const out = [];
|
|
2036
|
-
const entries = await
|
|
2168
|
+
const entries = await fs9.readdir(start, { withFileTypes: true }).catch(() => []);
|
|
2037
2169
|
for (const entry of entries) {
|
|
2038
2170
|
if (entry.isDirectory()) {
|
|
2039
2171
|
if (IGNORED_DIRS.has(entry.name)) continue;
|
|
2040
|
-
out.push(...await walkYamlFiles(
|
|
2041
|
-
} else if (entry.isFile() && CONFIG_FILE_EXTENSIONS.has(
|
|
2042
|
-
out.push(
|
|
2172
|
+
out.push(...await walkYamlFiles(path9.join(start, entry.name), depth + 1, max));
|
|
2173
|
+
} else if (entry.isFile() && CONFIG_FILE_EXTENSIONS.has(path9.extname(entry.name))) {
|
|
2174
|
+
out.push(path9.join(start, entry.name));
|
|
2043
2175
|
}
|
|
2044
2176
|
}
|
|
2045
2177
|
return out;
|
|
@@ -2066,7 +2198,7 @@ function k8sServiceTarget(doc, byName) {
|
|
|
2066
2198
|
async function collectK8sAliases(graph, scanPath, serviceIndex) {
|
|
2067
2199
|
const files = await walkYamlFiles(scanPath);
|
|
2068
2200
|
for (const file of files) {
|
|
2069
|
-
const content = await
|
|
2201
|
+
const content = await fs9.readFile(file, "utf8");
|
|
2070
2202
|
let docs;
|
|
2071
2203
|
try {
|
|
2072
2204
|
docs = parseAllDocuments(content).map((d) => d.toJSON());
|
|
@@ -2090,13 +2222,13 @@ async function addServiceAliases(graph, scanPath, services) {
|
|
|
2090
2222
|
}
|
|
2091
2223
|
|
|
2092
2224
|
// src/extract/databases/index.ts
|
|
2093
|
-
import
|
|
2225
|
+
import path17 from "path";
|
|
2094
2226
|
import { EdgeType as EdgeType3, NodeType as NodeType6, Provenance as Provenance4, databaseId as databaseId2 } from "@neat.is/types";
|
|
2095
2227
|
|
|
2096
2228
|
// src/extract/databases/db-config-yaml.ts
|
|
2097
|
-
import
|
|
2229
|
+
import path10 from "path";
|
|
2098
2230
|
async function parse(serviceDir) {
|
|
2099
|
-
const yamlPath =
|
|
2231
|
+
const yamlPath = path10.join(serviceDir, "db-config.yaml");
|
|
2100
2232
|
if (!await exists(yamlPath)) return [];
|
|
2101
2233
|
const raw = await readYaml(yamlPath);
|
|
2102
2234
|
return [
|
|
@@ -2113,12 +2245,12 @@ async function parse(serviceDir) {
|
|
|
2113
2245
|
var dbConfigYamlParser = { name: "db-config.yaml", parse };
|
|
2114
2246
|
|
|
2115
2247
|
// src/extract/databases/dotenv.ts
|
|
2116
|
-
import { promises as
|
|
2117
|
-
import
|
|
2248
|
+
import { promises as fs11 } from "fs";
|
|
2249
|
+
import path12 from "path";
|
|
2118
2250
|
|
|
2119
2251
|
// src/extract/databases/shared.ts
|
|
2120
|
-
import { promises as
|
|
2121
|
-
import
|
|
2252
|
+
import { promises as fs10 } from "fs";
|
|
2253
|
+
import path11 from "path";
|
|
2122
2254
|
function schemeToEngine(scheme) {
|
|
2123
2255
|
const s = scheme.toLowerCase().split("+")[0];
|
|
2124
2256
|
switch (s) {
|
|
@@ -2157,14 +2289,14 @@ function parseConnectionString(url) {
|
|
|
2157
2289
|
}
|
|
2158
2290
|
async function readIfExists(filePath) {
|
|
2159
2291
|
try {
|
|
2160
|
-
return await
|
|
2292
|
+
return await fs10.readFile(filePath, "utf8");
|
|
2161
2293
|
} catch {
|
|
2162
2294
|
return null;
|
|
2163
2295
|
}
|
|
2164
2296
|
}
|
|
2165
2297
|
async function findFirst(serviceDir, candidates) {
|
|
2166
2298
|
for (const rel of candidates) {
|
|
2167
|
-
const abs =
|
|
2299
|
+
const abs = path11.join(serviceDir, rel);
|
|
2168
2300
|
const content = await readIfExists(abs);
|
|
2169
2301
|
if (content !== null) return abs;
|
|
2170
2302
|
}
|
|
@@ -2215,15 +2347,15 @@ function parseDotenvLine(line) {
|
|
|
2215
2347
|
return { key, value };
|
|
2216
2348
|
}
|
|
2217
2349
|
async function parse2(serviceDir) {
|
|
2218
|
-
const entries = await
|
|
2350
|
+
const entries = await fs11.readdir(serviceDir, { withFileTypes: true }).catch(() => []);
|
|
2219
2351
|
const configs = [];
|
|
2220
2352
|
const seen = /* @__PURE__ */ new Set();
|
|
2221
2353
|
for (const entry of entries) {
|
|
2222
2354
|
if (!entry.isFile()) continue;
|
|
2223
2355
|
const match = isConfigFile(entry.name);
|
|
2224
2356
|
if (!match.match || match.fileType !== "env") continue;
|
|
2225
|
-
const filePath =
|
|
2226
|
-
const content = await
|
|
2357
|
+
const filePath = path12.join(serviceDir, entry.name);
|
|
2358
|
+
const content = await fs11.readFile(filePath, "utf8");
|
|
2227
2359
|
for (const line of content.split("\n")) {
|
|
2228
2360
|
const parsed = parseDotenvLine(line);
|
|
2229
2361
|
if (!parsed) continue;
|
|
@@ -2241,9 +2373,9 @@ async function parse2(serviceDir) {
|
|
|
2241
2373
|
var dotenvParser = { name: ".env", parse: parse2 };
|
|
2242
2374
|
|
|
2243
2375
|
// src/extract/databases/prisma.ts
|
|
2244
|
-
import
|
|
2376
|
+
import path13 from "path";
|
|
2245
2377
|
async function parse3(serviceDir) {
|
|
2246
|
-
const schemaPath =
|
|
2378
|
+
const schemaPath = path13.join(serviceDir, "prisma", "schema.prisma");
|
|
2247
2379
|
const content = await readIfExists(schemaPath);
|
|
2248
2380
|
if (!content) return [];
|
|
2249
2381
|
const block = content.match(/datasource\s+\w+\s*\{([^}]*)\}/s);
|
|
@@ -2372,10 +2504,10 @@ async function parse5(serviceDir) {
|
|
|
2372
2504
|
var knexParser = { name: "knex", parse: parse5 };
|
|
2373
2505
|
|
|
2374
2506
|
// src/extract/databases/ormconfig.ts
|
|
2375
|
-
import
|
|
2507
|
+
import path14 from "path";
|
|
2376
2508
|
async function parse6(serviceDir) {
|
|
2377
2509
|
for (const candidate of ["ormconfig.json", "ormconfig.yaml", "ormconfig.yml"]) {
|
|
2378
|
-
const abs =
|
|
2510
|
+
const abs = path14.join(serviceDir, candidate);
|
|
2379
2511
|
if (!await exists(abs)) continue;
|
|
2380
2512
|
const raw = candidate.endsWith(".json") ? await readJson(abs) : await readYaml(abs);
|
|
2381
2513
|
const entries = Array.isArray(raw) ? raw : [raw];
|
|
@@ -2433,9 +2565,9 @@ async function parse7(serviceDir) {
|
|
|
2433
2565
|
var typeormParser = { name: "typeorm", parse: parse7 };
|
|
2434
2566
|
|
|
2435
2567
|
// src/extract/databases/sequelize.ts
|
|
2436
|
-
import
|
|
2568
|
+
import path15 from "path";
|
|
2437
2569
|
async function parse8(serviceDir) {
|
|
2438
|
-
const configPath =
|
|
2570
|
+
const configPath = path15.join(serviceDir, "config", "config.json");
|
|
2439
2571
|
if (!await exists(configPath)) return [];
|
|
2440
2572
|
const raw = await readJson(configPath);
|
|
2441
2573
|
const out = [];
|
|
@@ -2461,7 +2593,7 @@ async function parse8(serviceDir) {
|
|
|
2461
2593
|
var sequelizeParser = { name: "sequelize", parse: parse8 };
|
|
2462
2594
|
|
|
2463
2595
|
// src/extract/databases/docker-compose.ts
|
|
2464
|
-
import
|
|
2596
|
+
import path16 from "path";
|
|
2465
2597
|
function portFromService(svc) {
|
|
2466
2598
|
for (const raw of svc.ports ?? []) {
|
|
2467
2599
|
const str = String(raw);
|
|
@@ -2488,7 +2620,7 @@ function databaseFromEnv(svc) {
|
|
|
2488
2620
|
}
|
|
2489
2621
|
async function parse9(serviceDir) {
|
|
2490
2622
|
for (const name of ["docker-compose.yml", "docker-compose.yaml"]) {
|
|
2491
|
-
const abs =
|
|
2623
|
+
const abs = path16.join(serviceDir, name);
|
|
2492
2624
|
if (!await exists(abs)) continue;
|
|
2493
2625
|
const raw = await readYaml(abs);
|
|
2494
2626
|
if (!raw?.services) return [];
|
|
@@ -2666,11 +2798,12 @@ async function addDatabasesAndCompat(graph, services, scanPath) {
|
|
|
2666
2798
|
target: dbNode.id,
|
|
2667
2799
|
type: EdgeType3.CONNECTS_TO,
|
|
2668
2800
|
provenance: Provenance4.EXTRACTED,
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2801
|
+
// ADR-032 / #140 — every EXTRACTED edge carries evidence.file.
|
|
2802
|
+
// Ghost-edge cleanup keys retirement on this; the conditional
|
|
2803
|
+
// sourceFile spread that used to live here was a v0.1.x leftover.
|
|
2804
|
+
evidence: {
|
|
2805
|
+
file: path17.relative(scanPath, config.sourceFile).split(path17.sep).join("/")
|
|
2806
|
+
}
|
|
2674
2807
|
};
|
|
2675
2808
|
if (!graph.hasEdge(edge.id)) {
|
|
2676
2809
|
graph.addEdgeWithKey(edge.id, edge.source, edge.target, edge);
|
|
@@ -2695,15 +2828,15 @@ async function addDatabasesAndCompat(graph, services, scanPath) {
|
|
|
2695
2828
|
}
|
|
2696
2829
|
|
|
2697
2830
|
// src/extract/configs.ts
|
|
2698
|
-
import { promises as
|
|
2699
|
-
import
|
|
2831
|
+
import { promises as fs12 } from "fs";
|
|
2832
|
+
import path18 from "path";
|
|
2700
2833
|
import { EdgeType as EdgeType4, NodeType as NodeType7, Provenance as Provenance5, configId } from "@neat.is/types";
|
|
2701
2834
|
async function walkConfigFiles(dir) {
|
|
2702
2835
|
const out = [];
|
|
2703
2836
|
async function walk(current) {
|
|
2704
|
-
const entries = await
|
|
2837
|
+
const entries = await fs12.readdir(current, { withFileTypes: true });
|
|
2705
2838
|
for (const entry of entries) {
|
|
2706
|
-
const full =
|
|
2839
|
+
const full = path18.join(current, entry.name);
|
|
2707
2840
|
if (entry.isDirectory()) {
|
|
2708
2841
|
if (!IGNORED_DIRS.has(entry.name)) await walk(full);
|
|
2709
2842
|
} else if (entry.isFile() && isConfigFile(entry.name).match) {
|
|
@@ -2720,13 +2853,13 @@ async function addConfigNodes(graph, services, scanPath) {
|
|
|
2720
2853
|
for (const service of services) {
|
|
2721
2854
|
const configFiles = await walkConfigFiles(service.dir);
|
|
2722
2855
|
for (const file of configFiles) {
|
|
2723
|
-
const relPath =
|
|
2856
|
+
const relPath = path18.relative(scanPath, file);
|
|
2724
2857
|
const node = {
|
|
2725
2858
|
id: configId(relPath),
|
|
2726
2859
|
type: NodeType7.ConfigNode,
|
|
2727
|
-
name:
|
|
2860
|
+
name: path18.basename(file),
|
|
2728
2861
|
path: relPath,
|
|
2729
|
-
fileType: isConfigFile(
|
|
2862
|
+
fileType: isConfigFile(path18.basename(file)).fileType
|
|
2730
2863
|
};
|
|
2731
2864
|
if (!graph.hasNode(node.id)) {
|
|
2732
2865
|
graph.addNode(node.id, node);
|
|
@@ -2738,7 +2871,7 @@ async function addConfigNodes(graph, services, scanPath) {
|
|
|
2738
2871
|
target: node.id,
|
|
2739
2872
|
type: EdgeType4.CONFIGURED_BY,
|
|
2740
2873
|
provenance: Provenance5.EXTRACTED,
|
|
2741
|
-
evidence: { file: relPath.split(
|
|
2874
|
+
evidence: { file: relPath.split(path18.sep).join("/") }
|
|
2742
2875
|
};
|
|
2743
2876
|
if (!graph.hasEdge(edge.id)) {
|
|
2744
2877
|
graph.addEdgeWithKey(edge.id, edge.source, edge.target, edge);
|
|
@@ -2753,24 +2886,24 @@ async function addConfigNodes(graph, services, scanPath) {
|
|
|
2753
2886
|
import { EdgeType as EdgeType6, NodeType as NodeType8, Provenance as Provenance7 } from "@neat.is/types";
|
|
2754
2887
|
|
|
2755
2888
|
// src/extract/calls/http.ts
|
|
2756
|
-
import
|
|
2889
|
+
import path20 from "path";
|
|
2757
2890
|
import Parser from "tree-sitter";
|
|
2758
2891
|
import JavaScript from "tree-sitter-javascript";
|
|
2759
2892
|
import Python from "tree-sitter-python";
|
|
2760
2893
|
import { EdgeType as EdgeType5, Provenance as Provenance6 } from "@neat.is/types";
|
|
2761
2894
|
|
|
2762
2895
|
// src/extract/calls/shared.ts
|
|
2763
|
-
import { promises as
|
|
2764
|
-
import
|
|
2896
|
+
import { promises as fs13 } from "fs";
|
|
2897
|
+
import path19 from "path";
|
|
2765
2898
|
async function walkSourceFiles(dir) {
|
|
2766
2899
|
const out = [];
|
|
2767
2900
|
async function walk(current) {
|
|
2768
|
-
const entries = await
|
|
2901
|
+
const entries = await fs13.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
2769
2902
|
for (const entry of entries) {
|
|
2770
|
-
const full =
|
|
2903
|
+
const full = path19.join(current, entry.name);
|
|
2771
2904
|
if (entry.isDirectory()) {
|
|
2772
2905
|
if (!IGNORED_DIRS.has(entry.name)) await walk(full);
|
|
2773
|
-
} else if (entry.isFile() && SERVICE_FILE_EXTENSIONS.has(
|
|
2906
|
+
} else if (entry.isFile() && SERVICE_FILE_EXTENSIONS.has(path19.extname(entry.name))) {
|
|
2774
2907
|
out.push(full);
|
|
2775
2908
|
}
|
|
2776
2909
|
}
|
|
@@ -2783,7 +2916,7 @@ async function loadSourceFiles(dir) {
|
|
|
2783
2916
|
const out = [];
|
|
2784
2917
|
for (const p of paths) {
|
|
2785
2918
|
try {
|
|
2786
|
-
const content = await
|
|
2919
|
+
const content = await fs13.readFile(p, "utf8");
|
|
2787
2920
|
out.push({ path: p, content });
|
|
2788
2921
|
} catch {
|
|
2789
2922
|
}
|
|
@@ -2802,8 +2935,27 @@ function snippet(text, line) {
|
|
|
2802
2935
|
|
|
2803
2936
|
// src/extract/calls/http.ts
|
|
2804
2937
|
var STRING_LITERAL_NODE_TYPES = /* @__PURE__ */ new Set(["string_fragment", "string_content"]);
|
|
2938
|
+
var JSX_EXTERNAL_LINK_TAGS = /* @__PURE__ */ new Set(["a", "Link", "NavLink", "ExternalLink", "Anchor"]);
|
|
2939
|
+
function isInsideJsxExternalLink(node) {
|
|
2940
|
+
let cursor = node.parent;
|
|
2941
|
+
while (cursor) {
|
|
2942
|
+
if (cursor.type === "jsx_attribute") {
|
|
2943
|
+
let owner = cursor.parent;
|
|
2944
|
+
while (owner && owner.type !== "jsx_opening_element" && owner.type !== "jsx_self_closing_element") {
|
|
2945
|
+
owner = owner.parent;
|
|
2946
|
+
}
|
|
2947
|
+
if (!owner) return false;
|
|
2948
|
+
const tagNode = owner.namedChild(0);
|
|
2949
|
+
const tagName = tagNode?.text ?? "";
|
|
2950
|
+
const right = tagName.includes(".") ? tagName.split(".").pop() : tagName;
|
|
2951
|
+
return JSX_EXTERNAL_LINK_TAGS.has(right);
|
|
2952
|
+
}
|
|
2953
|
+
cursor = cursor.parent;
|
|
2954
|
+
}
|
|
2955
|
+
return false;
|
|
2956
|
+
}
|
|
2805
2957
|
function collectStringLiterals(node, out) {
|
|
2806
|
-
if (STRING_LITERAL_NODE_TYPES.has(node.type)) out.push(node.text);
|
|
2958
|
+
if (STRING_LITERAL_NODE_TYPES.has(node.type)) out.push({ text: node.text, node });
|
|
2807
2959
|
for (let i = 0; i < node.namedChildCount; i++) {
|
|
2808
2960
|
const child = node.namedChild(i);
|
|
2809
2961
|
if (child) collectStringLiterals(child, out);
|
|
@@ -2815,8 +2967,9 @@ function callsFromSource(source, parser, knownHosts) {
|
|
|
2815
2967
|
collectStringLiterals(tree.rootNode, literals);
|
|
2816
2968
|
const targets = /* @__PURE__ */ new Set();
|
|
2817
2969
|
for (const lit of literals) {
|
|
2970
|
+
if (isInsideJsxExternalLink(lit.node)) continue;
|
|
2818
2971
|
for (const host of knownHosts) {
|
|
2819
|
-
if (lit.
|
|
2972
|
+
if (urlMatchesHost(lit.text, host)) {
|
|
2820
2973
|
targets.add(host);
|
|
2821
2974
|
}
|
|
2822
2975
|
}
|
|
@@ -2839,9 +2992,9 @@ async function addHttpCallEdges(graph, services) {
|
|
|
2839
2992
|
const knownHosts = /* @__PURE__ */ new Set();
|
|
2840
2993
|
const hostToNodeId = /* @__PURE__ */ new Map();
|
|
2841
2994
|
for (const service of services) {
|
|
2842
|
-
knownHosts.add(
|
|
2995
|
+
knownHosts.add(path20.basename(service.dir));
|
|
2843
2996
|
knownHosts.add(service.pkg.name);
|
|
2844
|
-
hostToNodeId.set(
|
|
2997
|
+
hostToNodeId.set(path20.basename(service.dir), service.node.id);
|
|
2845
2998
|
hostToNodeId.set(service.pkg.name, service.node.id);
|
|
2846
2999
|
}
|
|
2847
3000
|
let edgesAdded = 0;
|
|
@@ -2849,14 +3002,13 @@ async function addHttpCallEdges(graph, services) {
|
|
|
2849
3002
|
const files = await loadSourceFiles(service.dir);
|
|
2850
3003
|
const seenTargets = /* @__PURE__ */ new Map();
|
|
2851
3004
|
for (const file of files) {
|
|
2852
|
-
|
|
3005
|
+
if (isTestPath(file.path)) continue;
|
|
3006
|
+
const parser = path20.extname(file.path) === ".py" ? pyParser : jsParser;
|
|
2853
3007
|
let targets;
|
|
2854
3008
|
try {
|
|
2855
3009
|
targets = callsFromSource(file.content, parser, knownHosts);
|
|
2856
3010
|
} catch (err) {
|
|
2857
|
-
|
|
2858
|
-
`[neat] http call extraction skipped ${file.path}: ${err.message}`
|
|
2859
|
-
);
|
|
3011
|
+
recordExtractionError("http call extraction", file.path, err);
|
|
2860
3012
|
continue;
|
|
2861
3013
|
}
|
|
2862
3014
|
for (const t of targets) {
|
|
@@ -2877,7 +3029,7 @@ async function addHttpCallEdges(graph, services) {
|
|
|
2877
3029
|
type: EdgeType5.CALLS,
|
|
2878
3030
|
provenance: Provenance6.EXTRACTED,
|
|
2879
3031
|
evidence: {
|
|
2880
|
-
file:
|
|
3032
|
+
file: path20.relative(service.dir, evidenceFile.file),
|
|
2881
3033
|
line,
|
|
2882
3034
|
snippet: snippet(fileContent, line)
|
|
2883
3035
|
}
|
|
@@ -2892,7 +3044,7 @@ async function addHttpCallEdges(graph, services) {
|
|
|
2892
3044
|
}
|
|
2893
3045
|
|
|
2894
3046
|
// src/extract/calls/kafka.ts
|
|
2895
|
-
import
|
|
3047
|
+
import path21 from "path";
|
|
2896
3048
|
import { infraId } from "@neat.is/types";
|
|
2897
3049
|
var PRODUCER_TOPIC_RE = /(?:producer|kafkaProducer)[\s\S]{0,40}?\.send\s*\(\s*\{[\s\S]{0,200}?topic\s*:\s*['"`]([^'"`]+)['"`]/g;
|
|
2898
3050
|
var CONSUMER_TOPIC_RE = /(?:consumer|kafkaConsumer)[\s\S]{0,40}?\.(?:subscribe|run)\s*\(\s*\{[\s\S]{0,200}?topic[s]?\s*:\s*(?:\[\s*)?['"`]([^'"`]+)['"`]/g;
|
|
@@ -2919,7 +3071,7 @@ function kafkaEndpointsFromFile(file, serviceDir) {
|
|
|
2919
3071
|
kind: "kafka-topic",
|
|
2920
3072
|
edgeType,
|
|
2921
3073
|
evidence: {
|
|
2922
|
-
file:
|
|
3074
|
+
file: path21.relative(serviceDir, file.path),
|
|
2923
3075
|
line,
|
|
2924
3076
|
snippet: snippet(file.content, line)
|
|
2925
3077
|
}
|
|
@@ -2931,7 +3083,7 @@ function kafkaEndpointsFromFile(file, serviceDir) {
|
|
|
2931
3083
|
}
|
|
2932
3084
|
|
|
2933
3085
|
// src/extract/calls/redis.ts
|
|
2934
|
-
import
|
|
3086
|
+
import path22 from "path";
|
|
2935
3087
|
import { infraId as infraId2 } from "@neat.is/types";
|
|
2936
3088
|
var REDIS_URL_RE = /redis(?:s)?:\/\/(?:[^@'"`\s]+@)?([^:/'"`\s]+)(?::(\d+))?/g;
|
|
2937
3089
|
function redisEndpointsFromFile(file, serviceDir) {
|
|
@@ -2950,7 +3102,7 @@ function redisEndpointsFromFile(file, serviceDir) {
|
|
|
2950
3102
|
kind: "redis",
|
|
2951
3103
|
edgeType: "CALLS",
|
|
2952
3104
|
evidence: {
|
|
2953
|
-
file:
|
|
3105
|
+
file: path22.relative(serviceDir, file.path),
|
|
2954
3106
|
line,
|
|
2955
3107
|
snippet: snippet(file.content, line)
|
|
2956
3108
|
}
|
|
@@ -2960,7 +3112,7 @@ function redisEndpointsFromFile(file, serviceDir) {
|
|
|
2960
3112
|
}
|
|
2961
3113
|
|
|
2962
3114
|
// src/extract/calls/aws.ts
|
|
2963
|
-
import
|
|
3115
|
+
import path23 from "path";
|
|
2964
3116
|
import { infraId as infraId3 } from "@neat.is/types";
|
|
2965
3117
|
var S3_BUCKET_RE = /Bucket\s*:\s*['"`]([^'"`]+)['"`]/g;
|
|
2966
3118
|
var DYNAMO_TABLE_RE = /TableName\s*:\s*['"`]([^'"`]+)['"`]/g;
|
|
@@ -2990,7 +3142,7 @@ function awsEndpointsFromFile(file, serviceDir) {
|
|
|
2990
3142
|
kind,
|
|
2991
3143
|
edgeType: "CALLS",
|
|
2992
3144
|
evidence: {
|
|
2993
|
-
file:
|
|
3145
|
+
file: path23.relative(serviceDir, file.path),
|
|
2994
3146
|
line,
|
|
2995
3147
|
snippet: snippet(file.content, line)
|
|
2996
3148
|
}
|
|
@@ -3014,16 +3166,42 @@ function awsEndpointsFromFile(file, serviceDir) {
|
|
|
3014
3166
|
}
|
|
3015
3167
|
|
|
3016
3168
|
// src/extract/calls/grpc.ts
|
|
3017
|
-
import
|
|
3169
|
+
import path24 from "path";
|
|
3018
3170
|
import { infraId as infraId4 } from "@neat.is/types";
|
|
3019
3171
|
var GRPC_CLIENT_RE = /new\s+([A-Z][A-Za-z0-9_]*)Client\s*\(\s*['"`]?([^,'"`)]+)?/g;
|
|
3172
|
+
var AWS_SDK_IMPORT_RE = /(?:from\s+['"`]|require\(\s*['"`])@aws-sdk\/client-([a-z0-9-]+)['"`]/g;
|
|
3173
|
+
var GRPC_IMPORT_RE = /(?:from\s+['"`]|require\(\s*['"`])@grpc\/grpc-js['"`]|_grpc_pb['"`]/;
|
|
3020
3174
|
function isLikelyAddress(value) {
|
|
3021
3175
|
if (!value) return false;
|
|
3022
3176
|
return /:\d{2,5}$/.test(value) || value.includes(".");
|
|
3023
3177
|
}
|
|
3178
|
+
function normaliseForMatch(s) {
|
|
3179
|
+
return s.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
3180
|
+
}
|
|
3181
|
+
function readImports(content) {
|
|
3182
|
+
const awsSdkSuffixes = /* @__PURE__ */ new Map();
|
|
3183
|
+
AWS_SDK_IMPORT_RE.lastIndex = 0;
|
|
3184
|
+
let m;
|
|
3185
|
+
while ((m = AWS_SDK_IMPORT_RE.exec(content)) !== null) {
|
|
3186
|
+
const raw = m[1];
|
|
3187
|
+
awsSdkSuffixes.set(normaliseForMatch(raw), raw);
|
|
3188
|
+
}
|
|
3189
|
+
return {
|
|
3190
|
+
awsSdkSuffixes,
|
|
3191
|
+
hasGrpcImport: GRPC_IMPORT_RE.test(content)
|
|
3192
|
+
};
|
|
3193
|
+
}
|
|
3194
|
+
function classifyClient(symbol, ctx) {
|
|
3195
|
+
const key = normaliseForMatch(symbol);
|
|
3196
|
+
const awsRaw = ctx.awsSdkSuffixes.get(key);
|
|
3197
|
+
if (awsRaw) return { kind: `aws-${awsRaw}` };
|
|
3198
|
+
if (ctx.hasGrpcImport) return { kind: "grpc-service" };
|
|
3199
|
+
return null;
|
|
3200
|
+
}
|
|
3024
3201
|
function grpcEndpointsFromFile(file, serviceDir) {
|
|
3025
3202
|
const out = [];
|
|
3026
3203
|
const seen = /* @__PURE__ */ new Set();
|
|
3204
|
+
const ctx = readImports(file.content);
|
|
3027
3205
|
GRPC_CLIENT_RE.lastIndex = 0;
|
|
3028
3206
|
let m;
|
|
3029
3207
|
while ((m = GRPC_CLIENT_RE.exec(file.content)) !== null) {
|
|
@@ -3031,15 +3209,18 @@ function grpcEndpointsFromFile(file, serviceDir) {
|
|
|
3031
3209
|
const addr = m[2]?.trim();
|
|
3032
3210
|
const name = isLikelyAddress(addr) ? addr : symbol;
|
|
3033
3211
|
if (seen.has(name)) continue;
|
|
3212
|
+
const classified = classifyClient(symbol, ctx);
|
|
3213
|
+
if (!classified) continue;
|
|
3034
3214
|
seen.add(name);
|
|
3215
|
+
const { kind } = classified;
|
|
3035
3216
|
const line = lineOf(file.content, m[0]);
|
|
3036
3217
|
out.push({
|
|
3037
|
-
infraId: infraId4(
|
|
3218
|
+
infraId: infraId4(kind, name),
|
|
3038
3219
|
name,
|
|
3039
|
-
kind
|
|
3220
|
+
kind,
|
|
3040
3221
|
edgeType: "CALLS",
|
|
3041
3222
|
evidence: {
|
|
3042
|
-
file:
|
|
3223
|
+
file: path24.relative(serviceDir, file.path),
|
|
3043
3224
|
line,
|
|
3044
3225
|
snippet: snippet(file.content, line)
|
|
3045
3226
|
}
|
|
@@ -3059,6 +3240,9 @@ function edgeTypeFromEndpoint(ep) {
|
|
|
3059
3240
|
return EdgeType6.CALLS;
|
|
3060
3241
|
}
|
|
3061
3242
|
}
|
|
3243
|
+
function isAwsKind(kind) {
|
|
3244
|
+
return kind.startsWith("aws-") || kind.startsWith("s3") || kind.startsWith("dynamodb");
|
|
3245
|
+
}
|
|
3062
3246
|
async function addExternalEndpointEdges(graph, services) {
|
|
3063
3247
|
let nodesAdded = 0;
|
|
3064
3248
|
let edgesAdded = 0;
|
|
@@ -3066,10 +3250,13 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
3066
3250
|
const files = await loadSourceFiles(service.dir);
|
|
3067
3251
|
const endpoints = [];
|
|
3068
3252
|
for (const file of files) {
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
endpoints.push(...
|
|
3253
|
+
if (isTestPath(file.path)) continue;
|
|
3254
|
+
const masked = maskCommentsInSource(file.content);
|
|
3255
|
+
const maskedFile = { path: file.path, content: masked };
|
|
3256
|
+
endpoints.push(...kafkaEndpointsFromFile(maskedFile, service.dir));
|
|
3257
|
+
endpoints.push(...redisEndpointsFromFile(maskedFile, service.dir));
|
|
3258
|
+
endpoints.push(...awsEndpointsFromFile(maskedFile, service.dir));
|
|
3259
|
+
endpoints.push(...grpcEndpointsFromFile(maskedFile, service.dir));
|
|
3073
3260
|
}
|
|
3074
3261
|
if (endpoints.length === 0) continue;
|
|
3075
3262
|
const seenEdges = /* @__PURE__ */ new Set();
|
|
@@ -3079,7 +3266,10 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
3079
3266
|
id: ep.infraId,
|
|
3080
3267
|
type: NodeType8.InfraNode,
|
|
3081
3268
|
name: ep.name,
|
|
3082
|
-
|
|
3269
|
+
// #238 — `aws-*` covers AWS-SDK client kinds (aws-s3, aws-dynamodb,
|
|
3270
|
+
// aws-cognito-identity-provider, …); `s3-` / `dynamodb-` cover the
|
|
3271
|
+
// bucket / table kinds from aws.ts.
|
|
3272
|
+
provider: isAwsKind(ep.kind) ? "aws" : "self",
|
|
3083
3273
|
kind: ep.kind
|
|
3084
3274
|
};
|
|
3085
3275
|
graph.addNode(node.id, node);
|
|
@@ -3115,7 +3305,7 @@ async function addCallEdges(graph, services) {
|
|
|
3115
3305
|
}
|
|
3116
3306
|
|
|
3117
3307
|
// src/extract/infra/docker-compose.ts
|
|
3118
|
-
import
|
|
3308
|
+
import path25 from "path";
|
|
3119
3309
|
import { EdgeType as EdgeType7, Provenance as Provenance8 } from "@neat.is/types";
|
|
3120
3310
|
|
|
3121
3311
|
// src/extract/infra/shared.ts
|
|
@@ -3152,7 +3342,7 @@ function dependsOnList(value) {
|
|
|
3152
3342
|
}
|
|
3153
3343
|
function serviceNameToServiceNode(name, services) {
|
|
3154
3344
|
for (const s of services) {
|
|
3155
|
-
if (s.node.name === name ||
|
|
3345
|
+
if (s.node.name === name || path25.basename(s.dir) === name) return s.node.id;
|
|
3156
3346
|
}
|
|
3157
3347
|
return null;
|
|
3158
3348
|
}
|
|
@@ -3161,7 +3351,7 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
3161
3351
|
let edgesAdded = 0;
|
|
3162
3352
|
let composePath = null;
|
|
3163
3353
|
for (const name of ["docker-compose.yml", "docker-compose.yaml"]) {
|
|
3164
|
-
const abs =
|
|
3354
|
+
const abs = path25.join(scanPath, name);
|
|
3165
3355
|
if (await exists(abs)) {
|
|
3166
3356
|
composePath = abs;
|
|
3167
3357
|
break;
|
|
@@ -3172,13 +3362,15 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
3172
3362
|
try {
|
|
3173
3363
|
compose = await readYaml(composePath);
|
|
3174
3364
|
} catch (err) {
|
|
3175
|
-
|
|
3176
|
-
|
|
3365
|
+
recordExtractionError(
|
|
3366
|
+
"infra docker-compose",
|
|
3367
|
+
path25.relative(scanPath, composePath),
|
|
3368
|
+
err
|
|
3177
3369
|
);
|
|
3178
3370
|
return { nodesAdded, edgesAdded };
|
|
3179
3371
|
}
|
|
3180
3372
|
if (!compose?.services) return { nodesAdded, edgesAdded };
|
|
3181
|
-
const evidenceFile =
|
|
3373
|
+
const evidenceFile = path25.relative(scanPath, composePath).split(path25.sep).join("/");
|
|
3182
3374
|
const composeNameToNodeId = /* @__PURE__ */ new Map();
|
|
3183
3375
|
for (const [composeName, svc] of Object.entries(compose.services)) {
|
|
3184
3376
|
const matchedServiceId = serviceNameToServiceNode(composeName, services);
|
|
@@ -3218,8 +3410,8 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
3218
3410
|
}
|
|
3219
3411
|
|
|
3220
3412
|
// src/extract/infra/dockerfile.ts
|
|
3221
|
-
import
|
|
3222
|
-
import { promises as
|
|
3413
|
+
import path26 from "path";
|
|
3414
|
+
import { promises as fs14 } from "fs";
|
|
3223
3415
|
import { EdgeType as EdgeType8, Provenance as Provenance9 } from "@neat.is/types";
|
|
3224
3416
|
function runtimeImage(content) {
|
|
3225
3417
|
const lines = content.split("\n");
|
|
@@ -3239,14 +3431,16 @@ async function addDockerfileRuntimes(graph, services, scanPath) {
|
|
|
3239
3431
|
let nodesAdded = 0;
|
|
3240
3432
|
let edgesAdded = 0;
|
|
3241
3433
|
for (const service of services) {
|
|
3242
|
-
const dockerfilePath =
|
|
3434
|
+
const dockerfilePath = path26.join(service.dir, "Dockerfile");
|
|
3243
3435
|
if (!await exists(dockerfilePath)) continue;
|
|
3244
3436
|
let content;
|
|
3245
3437
|
try {
|
|
3246
|
-
content = await
|
|
3438
|
+
content = await fs14.readFile(dockerfilePath, "utf8");
|
|
3247
3439
|
} catch (err) {
|
|
3248
|
-
|
|
3249
|
-
|
|
3440
|
+
recordExtractionError(
|
|
3441
|
+
"infra dockerfile",
|
|
3442
|
+
path26.relative(scanPath, dockerfilePath),
|
|
3443
|
+
err
|
|
3250
3444
|
);
|
|
3251
3445
|
continue;
|
|
3252
3446
|
}
|
|
@@ -3266,7 +3460,7 @@ async function addDockerfileRuntimes(graph, services, scanPath) {
|
|
|
3266
3460
|
type: EdgeType8.RUNS_ON,
|
|
3267
3461
|
provenance: Provenance9.EXTRACTED,
|
|
3268
3462
|
evidence: {
|
|
3269
|
-
file:
|
|
3463
|
+
file: path26.relative(scanPath, dockerfilePath).split(path26.sep).join("/")
|
|
3270
3464
|
}
|
|
3271
3465
|
};
|
|
3272
3466
|
graph.addEdgeWithKey(edgeId, edge.source, edge.target, edge);
|
|
@@ -3277,19 +3471,19 @@ async function addDockerfileRuntimes(graph, services, scanPath) {
|
|
|
3277
3471
|
}
|
|
3278
3472
|
|
|
3279
3473
|
// src/extract/infra/terraform.ts
|
|
3280
|
-
import { promises as
|
|
3281
|
-
import
|
|
3474
|
+
import { promises as fs15 } from "fs";
|
|
3475
|
+
import path27 from "path";
|
|
3282
3476
|
var RESOURCE_RE = /resource\s+"(aws_[A-Za-z0-9_]+)"\s+"([A-Za-z0-9_-]+)"/g;
|
|
3283
3477
|
async function walkTfFiles(start, depth = 0, max = 5) {
|
|
3284
3478
|
if (depth > max) return [];
|
|
3285
3479
|
const out = [];
|
|
3286
|
-
const entries = await
|
|
3480
|
+
const entries = await fs15.readdir(start, { withFileTypes: true }).catch(() => []);
|
|
3287
3481
|
for (const entry of entries) {
|
|
3288
3482
|
if (entry.isDirectory()) {
|
|
3289
3483
|
if (IGNORED_DIRS.has(entry.name) || entry.name === ".terraform") continue;
|
|
3290
|
-
out.push(...await walkTfFiles(
|
|
3484
|
+
out.push(...await walkTfFiles(path27.join(start, entry.name), depth + 1, max));
|
|
3291
3485
|
} else if (entry.isFile() && entry.name.endsWith(".tf")) {
|
|
3292
|
-
out.push(
|
|
3486
|
+
out.push(path27.join(start, entry.name));
|
|
3293
3487
|
}
|
|
3294
3488
|
}
|
|
3295
3489
|
return out;
|
|
@@ -3298,7 +3492,7 @@ async function addTerraformResources(graph, scanPath) {
|
|
|
3298
3492
|
let nodesAdded = 0;
|
|
3299
3493
|
const files = await walkTfFiles(scanPath);
|
|
3300
3494
|
for (const file of files) {
|
|
3301
|
-
const content = await
|
|
3495
|
+
const content = await fs15.readFile(file, "utf8");
|
|
3302
3496
|
RESOURCE_RE.lastIndex = 0;
|
|
3303
3497
|
let m;
|
|
3304
3498
|
while ((m = RESOURCE_RE.exec(content)) !== null) {
|
|
@@ -3315,8 +3509,8 @@ async function addTerraformResources(graph, scanPath) {
|
|
|
3315
3509
|
}
|
|
3316
3510
|
|
|
3317
3511
|
// src/extract/infra/k8s.ts
|
|
3318
|
-
import { promises as
|
|
3319
|
-
import
|
|
3512
|
+
import { promises as fs16 } from "fs";
|
|
3513
|
+
import path28 from "path";
|
|
3320
3514
|
import { parseAllDocuments as parseAllDocuments2 } from "yaml";
|
|
3321
3515
|
var K8S_KIND_TO_INFRA_KIND = {
|
|
3322
3516
|
Service: "k8s-service",
|
|
@@ -3330,13 +3524,13 @@ var K8S_KIND_TO_INFRA_KIND = {
|
|
|
3330
3524
|
async function walkYamlFiles2(start, depth = 0, max = 5) {
|
|
3331
3525
|
if (depth > max) return [];
|
|
3332
3526
|
const out = [];
|
|
3333
|
-
const entries = await
|
|
3527
|
+
const entries = await fs16.readdir(start, { withFileTypes: true }).catch(() => []);
|
|
3334
3528
|
for (const entry of entries) {
|
|
3335
3529
|
if (entry.isDirectory()) {
|
|
3336
3530
|
if (IGNORED_DIRS.has(entry.name)) continue;
|
|
3337
|
-
out.push(...await walkYamlFiles2(
|
|
3338
|
-
} else if (entry.isFile() && CONFIG_FILE_EXTENSIONS.has(
|
|
3339
|
-
out.push(
|
|
3531
|
+
out.push(...await walkYamlFiles2(path28.join(start, entry.name), depth + 1, max));
|
|
3532
|
+
} else if (entry.isFile() && CONFIG_FILE_EXTENSIONS.has(path28.extname(entry.name))) {
|
|
3533
|
+
out.push(path28.join(start, entry.name));
|
|
3340
3534
|
}
|
|
3341
3535
|
}
|
|
3342
3536
|
return out;
|
|
@@ -3345,7 +3539,7 @@ async function addK8sResources(graph, scanPath) {
|
|
|
3345
3539
|
let nodesAdded = 0;
|
|
3346
3540
|
const files = await walkYamlFiles2(scanPath);
|
|
3347
3541
|
for (const file of files) {
|
|
3348
|
-
const content = await
|
|
3542
|
+
const content = await fs16.readFile(file, "utf8");
|
|
3349
3543
|
let docs;
|
|
3350
3544
|
try {
|
|
3351
3545
|
docs = parseAllDocuments2(content).map((d) => d.toJSON());
|
|
@@ -3379,9 +3573,45 @@ async function addInfra(graph, scanPath, services) {
|
|
|
3379
3573
|
};
|
|
3380
3574
|
}
|
|
3381
3575
|
|
|
3576
|
+
// src/extract/retire.ts
|
|
3577
|
+
import { existsSync } from "fs";
|
|
3578
|
+
import path29 from "path";
|
|
3579
|
+
import { Provenance as Provenance10 } from "@neat.is/types";
|
|
3580
|
+
function retireEdgesByFile(graph, file) {
|
|
3581
|
+
const normalized = file.split("\\").join("/");
|
|
3582
|
+
const toDrop = [];
|
|
3583
|
+
graph.forEachEdge((id, attrs) => {
|
|
3584
|
+
const edge = attrs;
|
|
3585
|
+
if (edge.provenance !== Provenance10.EXTRACTED) return;
|
|
3586
|
+
if (!edge.evidence?.file) return;
|
|
3587
|
+
if (edge.evidence.file === normalized) toDrop.push(id);
|
|
3588
|
+
});
|
|
3589
|
+
for (const id of toDrop) graph.dropEdge(id);
|
|
3590
|
+
return toDrop.length;
|
|
3591
|
+
}
|
|
3592
|
+
function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
|
|
3593
|
+
const toDrop = [];
|
|
3594
|
+
const bases = [scanPath, ...serviceDirs];
|
|
3595
|
+
graph.forEachEdge((id, attrs) => {
|
|
3596
|
+
const edge = attrs;
|
|
3597
|
+
if (edge.provenance !== Provenance10.EXTRACTED) return;
|
|
3598
|
+
const evidenceFile = edge.evidence?.file;
|
|
3599
|
+
if (!evidenceFile) return;
|
|
3600
|
+
if (path29.isAbsolute(evidenceFile)) {
|
|
3601
|
+
if (!existsSync(evidenceFile)) toDrop.push(id);
|
|
3602
|
+
return;
|
|
3603
|
+
}
|
|
3604
|
+
const found = bases.some((base) => existsSync(path29.join(base, evidenceFile)));
|
|
3605
|
+
if (!found) toDrop.push(id);
|
|
3606
|
+
});
|
|
3607
|
+
for (const id of toDrop) graph.dropEdge(id);
|
|
3608
|
+
return toDrop.length;
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3382
3611
|
// src/extract/index.ts
|
|
3383
3612
|
async function extractFromDirectory(graph, scanPath, opts = {}) {
|
|
3384
3613
|
await ensureCompatLoaded();
|
|
3614
|
+
drainExtractionErrors();
|
|
3385
3615
|
const services = await discoverServices(scanPath);
|
|
3386
3616
|
const phase1Nodes = addServiceNodes(graph, services);
|
|
3387
3617
|
await addServiceAliases(graph, scanPath, services);
|
|
@@ -3389,12 +3619,30 @@ async function extractFromDirectory(graph, scanPath, opts = {}) {
|
|
|
3389
3619
|
const phase3 = await addConfigNodes(graph, services, scanPath);
|
|
3390
3620
|
const phase4 = await addCallEdges(graph, services);
|
|
3391
3621
|
const phase5 = await addInfra(graph, scanPath, services);
|
|
3622
|
+
const ghostsRetired = retireExtractedEdgesByMissingFile(
|
|
3623
|
+
graph,
|
|
3624
|
+
scanPath,
|
|
3625
|
+
services.map((s) => s.dir)
|
|
3626
|
+
);
|
|
3392
3627
|
const frontiersPromoted = promoteFrontierNodes(graph);
|
|
3393
3628
|
if (opts.onPolicyTrigger) await opts.onPolicyTrigger(graph);
|
|
3629
|
+
const errorEntries = drainExtractionErrors();
|
|
3630
|
+
if (opts.errorsPath && errorEntries.length > 0) {
|
|
3631
|
+
try {
|
|
3632
|
+
await writeExtractionErrors(errorEntries, opts.errorsPath);
|
|
3633
|
+
} catch (err) {
|
|
3634
|
+
console.warn(
|
|
3635
|
+
`[neat] failed to write extraction errors to ${opts.errorsPath}: ${err.message}`
|
|
3636
|
+
);
|
|
3637
|
+
}
|
|
3638
|
+
}
|
|
3394
3639
|
const result = {
|
|
3395
3640
|
nodesAdded: phase1Nodes + phase2.nodesAdded + phase3.nodesAdded + phase4.nodesAdded + phase5.nodesAdded,
|
|
3396
3641
|
edgesAdded: phase2.edgesAdded + phase3.edgesAdded + phase4.edgesAdded + phase5.edgesAdded,
|
|
3397
|
-
frontiersPromoted
|
|
3642
|
+
frontiersPromoted,
|
|
3643
|
+
extractionErrors: errorEntries.length,
|
|
3644
|
+
errorEntries,
|
|
3645
|
+
ghostsRetired
|
|
3398
3646
|
};
|
|
3399
3647
|
emitNeatEvent({
|
|
3400
3648
|
type: "extraction-complete",
|
|
@@ -3410,8 +3658,8 @@ async function extractFromDirectory(graph, scanPath, opts = {}) {
|
|
|
3410
3658
|
}
|
|
3411
3659
|
|
|
3412
3660
|
// src/persist.ts
|
|
3413
|
-
import { promises as
|
|
3414
|
-
import
|
|
3661
|
+
import { promises as fs17 } from "fs";
|
|
3662
|
+
import path30 from "path";
|
|
3415
3663
|
var SCHEMA_VERSION = 2;
|
|
3416
3664
|
function migrateV1ToV2(payload) {
|
|
3417
3665
|
const nodes = payload.graph.nodes;
|
|
@@ -3425,7 +3673,7 @@ function migrateV1ToV2(payload) {
|
|
|
3425
3673
|
return { ...payload, schemaVersion: 2 };
|
|
3426
3674
|
}
|
|
3427
3675
|
async function ensureDir(filePath) {
|
|
3428
|
-
await
|
|
3676
|
+
await fs17.mkdir(path30.dirname(filePath), { recursive: true });
|
|
3429
3677
|
}
|
|
3430
3678
|
async function saveGraphToDisk(graph, outPath) {
|
|
3431
3679
|
await ensureDir(outPath);
|
|
@@ -3435,13 +3683,13 @@ async function saveGraphToDisk(graph, outPath) {
|
|
|
3435
3683
|
graph: graph.export()
|
|
3436
3684
|
};
|
|
3437
3685
|
const tmp = `${outPath}.tmp`;
|
|
3438
|
-
await
|
|
3439
|
-
await
|
|
3686
|
+
await fs17.writeFile(tmp, JSON.stringify(payload), "utf8");
|
|
3687
|
+
await fs17.rename(tmp, outPath);
|
|
3440
3688
|
}
|
|
3441
3689
|
async function loadGraphFromDisk(graph, outPath) {
|
|
3442
3690
|
let raw;
|
|
3443
3691
|
try {
|
|
3444
|
-
raw = await
|
|
3692
|
+
raw = await fs17.readFile(outPath, "utf8");
|
|
3445
3693
|
} catch (err) {
|
|
3446
3694
|
if (err.code === "ENOENT") return;
|
|
3447
3695
|
throw err;
|
|
@@ -3493,7 +3741,7 @@ function startPersistLoop(graph, outPath, intervalMs = 6e4) {
|
|
|
3493
3741
|
}
|
|
3494
3742
|
|
|
3495
3743
|
// src/diff.ts
|
|
3496
|
-
import { promises as
|
|
3744
|
+
import { promises as fs18 } from "fs";
|
|
3497
3745
|
async function loadSnapshotForDiff(target) {
|
|
3498
3746
|
if (/^https?:\/\//i.test(target)) {
|
|
3499
3747
|
const res = await fetch(target);
|
|
@@ -3502,7 +3750,7 @@ async function loadSnapshotForDiff(target) {
|
|
|
3502
3750
|
}
|
|
3503
3751
|
return await res.json();
|
|
3504
3752
|
}
|
|
3505
|
-
const raw = await
|
|
3753
|
+
const raw = await fs18.readFile(target, "utf8");
|
|
3506
3754
|
return JSON.parse(raw);
|
|
3507
3755
|
}
|
|
3508
3756
|
function indexEntries(entries) {
|
|
@@ -3569,23 +3817,23 @@ function canonicalJson(value) {
|
|
|
3569
3817
|
}
|
|
3570
3818
|
|
|
3571
3819
|
// src/projects.ts
|
|
3572
|
-
import
|
|
3820
|
+
import path31 from "path";
|
|
3573
3821
|
function pathsForProject(project, baseDir) {
|
|
3574
3822
|
if (project === DEFAULT_PROJECT) {
|
|
3575
3823
|
return {
|
|
3576
|
-
snapshotPath:
|
|
3577
|
-
errorsPath:
|
|
3578
|
-
staleEventsPath:
|
|
3579
|
-
embeddingsCachePath:
|
|
3580
|
-
policyViolationsPath:
|
|
3824
|
+
snapshotPath: path31.join(baseDir, "graph.json"),
|
|
3825
|
+
errorsPath: path31.join(baseDir, "errors.ndjson"),
|
|
3826
|
+
staleEventsPath: path31.join(baseDir, "stale-events.ndjson"),
|
|
3827
|
+
embeddingsCachePath: path31.join(baseDir, "embeddings.json"),
|
|
3828
|
+
policyViolationsPath: path31.join(baseDir, "policy-violations.ndjson")
|
|
3581
3829
|
};
|
|
3582
3830
|
}
|
|
3583
3831
|
return {
|
|
3584
|
-
snapshotPath:
|
|
3585
|
-
errorsPath:
|
|
3586
|
-
staleEventsPath:
|
|
3587
|
-
embeddingsCachePath:
|
|
3588
|
-
policyViolationsPath:
|
|
3832
|
+
snapshotPath: path31.join(baseDir, `${project}.json`),
|
|
3833
|
+
errorsPath: path31.join(baseDir, `errors.${project}.ndjson`),
|
|
3834
|
+
staleEventsPath: path31.join(baseDir, `stale-events.${project}.ndjson`),
|
|
3835
|
+
embeddingsCachePath: path31.join(baseDir, `embeddings.${project}.json`),
|
|
3836
|
+
policyViolationsPath: path31.join(baseDir, `policy-violations.${project}.ndjson`)
|
|
3589
3837
|
};
|
|
3590
3838
|
}
|
|
3591
3839
|
var Projects = class {
|
|
@@ -3624,9 +3872,9 @@ function parseExtraProjects(raw) {
|
|
|
3624
3872
|
}
|
|
3625
3873
|
|
|
3626
3874
|
// src/registry.ts
|
|
3627
|
-
import { promises as
|
|
3875
|
+
import { promises as fs19 } from "fs";
|
|
3628
3876
|
import os2 from "os";
|
|
3629
|
-
import
|
|
3877
|
+
import path32 from "path";
|
|
3630
3878
|
import {
|
|
3631
3879
|
RegistryFileSchema
|
|
3632
3880
|
} from "@neat.is/types";
|
|
@@ -3634,41 +3882,41 @@ var LOCK_TIMEOUT_MS = 5e3;
|
|
|
3634
3882
|
var LOCK_RETRY_MS = 50;
|
|
3635
3883
|
function neatHome() {
|
|
3636
3884
|
const override = process.env.NEAT_HOME;
|
|
3637
|
-
if (override && override.length > 0) return
|
|
3638
|
-
return
|
|
3885
|
+
if (override && override.length > 0) return path32.resolve(override);
|
|
3886
|
+
return path32.join(os2.homedir(), ".neat");
|
|
3639
3887
|
}
|
|
3640
3888
|
function registryPath() {
|
|
3641
|
-
return
|
|
3889
|
+
return path32.join(neatHome(), "projects.json");
|
|
3642
3890
|
}
|
|
3643
3891
|
function registryLockPath() {
|
|
3644
|
-
return
|
|
3892
|
+
return path32.join(neatHome(), "projects.json.lock");
|
|
3645
3893
|
}
|
|
3646
3894
|
async function normalizeProjectPath(input) {
|
|
3647
|
-
const resolved =
|
|
3895
|
+
const resolved = path32.resolve(input);
|
|
3648
3896
|
try {
|
|
3649
|
-
return await
|
|
3897
|
+
return await fs19.realpath(resolved);
|
|
3650
3898
|
} catch {
|
|
3651
3899
|
return resolved;
|
|
3652
3900
|
}
|
|
3653
3901
|
}
|
|
3654
3902
|
async function writeAtomically(target, contents) {
|
|
3655
|
-
await
|
|
3903
|
+
await fs19.mkdir(path32.dirname(target), { recursive: true });
|
|
3656
3904
|
const tmp = `${target}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
3657
|
-
const fd = await
|
|
3905
|
+
const fd = await fs19.open(tmp, "w");
|
|
3658
3906
|
try {
|
|
3659
3907
|
await fd.writeFile(contents, "utf8");
|
|
3660
3908
|
await fd.sync();
|
|
3661
3909
|
} finally {
|
|
3662
3910
|
await fd.close();
|
|
3663
3911
|
}
|
|
3664
|
-
await
|
|
3912
|
+
await fs19.rename(tmp, target);
|
|
3665
3913
|
}
|
|
3666
3914
|
async function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS) {
|
|
3667
3915
|
const deadline = Date.now() + timeoutMs;
|
|
3668
|
-
await
|
|
3916
|
+
await fs19.mkdir(path32.dirname(lockPath), { recursive: true });
|
|
3669
3917
|
while (true) {
|
|
3670
3918
|
try {
|
|
3671
|
-
const fd = await
|
|
3919
|
+
const fd = await fs19.open(lockPath, "wx");
|
|
3672
3920
|
await fd.close();
|
|
3673
3921
|
return;
|
|
3674
3922
|
} catch (err) {
|
|
@@ -3684,7 +3932,7 @@ async function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS) {
|
|
|
3684
3932
|
}
|
|
3685
3933
|
}
|
|
3686
3934
|
async function releaseLock(lockPath) {
|
|
3687
|
-
await
|
|
3935
|
+
await fs19.unlink(lockPath).catch(() => {
|
|
3688
3936
|
});
|
|
3689
3937
|
}
|
|
3690
3938
|
async function withLock(fn) {
|
|
@@ -3700,7 +3948,7 @@ async function readRegistry() {
|
|
|
3700
3948
|
const file = registryPath();
|
|
3701
3949
|
let raw;
|
|
3702
3950
|
try {
|
|
3703
|
-
raw = await
|
|
3951
|
+
raw = await fs19.readFile(file, "utf8");
|
|
3704
3952
|
} catch (err) {
|
|
3705
3953
|
if (err.code === "ENOENT") {
|
|
3706
3954
|
return { version: 1, projects: [] };
|
|
@@ -3803,7 +4051,7 @@ import {
|
|
|
3803
4051
|
EdgeType as EdgeType9,
|
|
3804
4052
|
NodeType as NodeType10,
|
|
3805
4053
|
parseEdgeId,
|
|
3806
|
-
Provenance as
|
|
4054
|
+
Provenance as Provenance11
|
|
3807
4055
|
} from "@neat.is/types";
|
|
3808
4056
|
function bucketKey(source, target, type) {
|
|
3809
4057
|
return `${type}|${source}|${target}`;
|
|
@@ -3817,20 +4065,20 @@ function bucketEdges(graph) {
|
|
|
3817
4065
|
const key = bucketKey(e.source, e.target, e.type);
|
|
3818
4066
|
const cur = buckets.get(key) ?? { source: e.source, target: e.target, type: e.type };
|
|
3819
4067
|
switch (provenance) {
|
|
3820
|
-
case
|
|
4068
|
+
case Provenance11.EXTRACTED:
|
|
3821
4069
|
cur.extracted = e;
|
|
3822
4070
|
break;
|
|
3823
|
-
case
|
|
4071
|
+
case Provenance11.OBSERVED:
|
|
3824
4072
|
cur.observed = e;
|
|
3825
4073
|
break;
|
|
3826
|
-
case
|
|
4074
|
+
case Provenance11.INFERRED:
|
|
3827
4075
|
cur.inferred = e;
|
|
3828
4076
|
break;
|
|
3829
|
-
case
|
|
4077
|
+
case Provenance11.FRONTIER:
|
|
3830
4078
|
cur.frontier = e;
|
|
3831
4079
|
break;
|
|
3832
4080
|
default:
|
|
3833
|
-
if (e.provenance ===
|
|
4081
|
+
if (e.provenance === Provenance11.STALE) cur.stale = e;
|
|
3834
4082
|
}
|
|
3835
4083
|
buckets.set(key, cur);
|
|
3836
4084
|
});
|
|
@@ -3845,7 +4093,7 @@ function hasAnyObservedFromSource(graph, sourceId) {
|
|
|
3845
4093
|
if (!graph.hasNode(sourceId)) return false;
|
|
3846
4094
|
for (const edgeId of graph.outboundEdges(sourceId)) {
|
|
3847
4095
|
const e = graph.getEdgeAttributes(edgeId);
|
|
3848
|
-
if (e.provenance ===
|
|
4096
|
+
if (e.provenance === Provenance11.OBSERVED) return true;
|
|
3849
4097
|
}
|
|
3850
4098
|
return false;
|
|
3851
4099
|
}
|
|
@@ -3906,7 +4154,7 @@ function declaredHostFor(svc) {
|
|
|
3906
4154
|
function hasExtractedConfiguredBy(graph, svcId) {
|
|
3907
4155
|
for (const edgeId of graph.outboundEdges(svcId)) {
|
|
3908
4156
|
const e = graph.getEdgeAttributes(edgeId);
|
|
3909
|
-
if (e.type === EdgeType9.CONFIGURED_BY && e.provenance ===
|
|
4157
|
+
if (e.type === EdgeType9.CONFIGURED_BY && e.provenance === Provenance11.EXTRACTED) {
|
|
3910
4158
|
return true;
|
|
3911
4159
|
}
|
|
3912
4160
|
}
|
|
@@ -3920,7 +4168,7 @@ function detectHostMismatch(graph, svcId, svc) {
|
|
|
3920
4168
|
for (const edgeId of graph.outboundEdges(svcId)) {
|
|
3921
4169
|
const edge = graph.getEdgeAttributes(edgeId);
|
|
3922
4170
|
if (edge.type !== EdgeType9.CONNECTS_TO) continue;
|
|
3923
|
-
if (edge.provenance !==
|
|
4171
|
+
if (edge.provenance !== Provenance11.OBSERVED) continue;
|
|
3924
4172
|
const target = graph.getNodeAttributes(edge.target);
|
|
3925
4173
|
if (target.type !== NodeType10.DatabaseNode) continue;
|
|
3926
4174
|
const observedHost = target.host?.trim();
|
|
@@ -3945,7 +4193,7 @@ function detectCompatDivergences(graph, svcId, svc) {
|
|
|
3945
4193
|
for (const edgeId of graph.outboundEdges(svcId)) {
|
|
3946
4194
|
const edge = graph.getEdgeAttributes(edgeId);
|
|
3947
4195
|
if (edge.type !== EdgeType9.CONNECTS_TO) continue;
|
|
3948
|
-
if (edge.provenance !==
|
|
4196
|
+
if (edge.provenance !== Provenance11.OBSERVED) continue;
|
|
3949
4197
|
const target = graph.getNodeAttributes(edge.target);
|
|
3950
4198
|
if (target.type !== NodeType10.DatabaseNode) continue;
|
|
3951
4199
|
for (const pair of compatPairs()) {
|
|
@@ -4545,6 +4793,8 @@ export {
|
|
|
4545
4793
|
readStaleEvents,
|
|
4546
4794
|
startStalenessLoop,
|
|
4547
4795
|
readErrorEvents,
|
|
4796
|
+
isStrictExtractionEnabled,
|
|
4797
|
+
formatExtractionBanner,
|
|
4548
4798
|
discoverServices,
|
|
4549
4799
|
addServiceNodes,
|
|
4550
4800
|
addServiceAliases,
|
|
@@ -4552,6 +4802,7 @@ export {
|
|
|
4552
4802
|
addConfigNodes,
|
|
4553
4803
|
addCallEdges,
|
|
4554
4804
|
addInfra,
|
|
4805
|
+
retireEdgesByFile,
|
|
4555
4806
|
extractFromDirectory,
|
|
4556
4807
|
saveGraphToDisk,
|
|
4557
4808
|
loadGraphFromDisk,
|
|
@@ -4575,4 +4826,4 @@ export {
|
|
|
4575
4826
|
removeProject,
|
|
4576
4827
|
buildApi
|
|
4577
4828
|
};
|
|
4578
|
-
//# sourceMappingURL=chunk-
|
|
4829
|
+
//# sourceMappingURL=chunk-VABXPLDT.js.map
|