@neat.is/core 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RBWL4HRB.js → chunk-7FTK47JQ.js} +277 -103
- package/dist/chunk-7FTK47JQ.js.map +1 -0
- package/dist/{chunk-6GXBAR3M.js → chunk-LS6NS72S.js} +37 -3
- package/dist/chunk-LS6NS72S.js.map +1 -0
- package/dist/cli.cjs +600 -219
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +294 -80
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +314 -115
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2 -2
- package/dist/neatd.cjs +314 -115
- package/dist/neatd.cjs.map +1 -1
- package/dist/neatd.js +2 -2
- package/dist/server.cjs +263 -98
- package/dist/server.cjs.map +1 -1
- package/dist/server.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-6GXBAR3M.js.map +0 -1
- package/dist/chunk-RBWL4HRB.js.map +0 -1
package/dist/server.cjs
CHANGED
|
@@ -610,7 +610,7 @@ function getGraph(project = DEFAULT_PROJECT) {
|
|
|
610
610
|
init_cjs_shims();
|
|
611
611
|
var import_fastify = __toESM(require("fastify"), 1);
|
|
612
612
|
var import_cors = __toESM(require("@fastify/cors"), 1);
|
|
613
|
-
var
|
|
613
|
+
var import_types23 = require("@neat.is/types");
|
|
614
614
|
|
|
615
615
|
// src/divergences.ts
|
|
616
616
|
init_cjs_shims();
|
|
@@ -1267,6 +1267,7 @@ function gradedConfidence(edge) {
|
|
|
1267
1267
|
}
|
|
1268
1268
|
function detectMissingDivergences(graph, bucket) {
|
|
1269
1269
|
const out = [];
|
|
1270
|
+
if (bucket.type === import_types2.EdgeType.CONTAINS) return out;
|
|
1270
1271
|
if (bucket.extracted && !bucket.observed) {
|
|
1271
1272
|
if (!nodeIsFrontier(graph, bucket.target)) {
|
|
1272
1273
|
out.push({
|
|
@@ -1857,6 +1858,86 @@ function hostFromUrl(u) {
|
|
|
1857
1858
|
function pickAddress(span) {
|
|
1858
1859
|
return pickAttr(span, "server.address", "net.peer.name", "net.host.name") ?? hostFromUrl(pickAttr(span, "url.full", "http.url"));
|
|
1859
1860
|
}
|
|
1861
|
+
var CODE_FILEPATH_ATTR = "code.filepath";
|
|
1862
|
+
var CODE_LINENO_ATTR = "code.lineno";
|
|
1863
|
+
var CODE_FUNCTION_ATTR = "code.function";
|
|
1864
|
+
function toPosix(p) {
|
|
1865
|
+
return p.split("\\").join("/");
|
|
1866
|
+
}
|
|
1867
|
+
function languageForExt(relPath) {
|
|
1868
|
+
const dot = relPath.lastIndexOf(".");
|
|
1869
|
+
if (dot === -1) return void 0;
|
|
1870
|
+
switch (relPath.slice(dot).toLowerCase()) {
|
|
1871
|
+
case ".py":
|
|
1872
|
+
return "python";
|
|
1873
|
+
case ".ts":
|
|
1874
|
+
case ".tsx":
|
|
1875
|
+
return "typescript";
|
|
1876
|
+
case ".js":
|
|
1877
|
+
case ".jsx":
|
|
1878
|
+
case ".mjs":
|
|
1879
|
+
case ".cjs":
|
|
1880
|
+
return "javascript";
|
|
1881
|
+
default:
|
|
1882
|
+
return void 0;
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
function relPathForRuntimeFile(filepath, serviceNode) {
|
|
1886
|
+
let p = toPosix(filepath).replace(/^file:\/\//, "");
|
|
1887
|
+
const root = serviceNode?.repoPath;
|
|
1888
|
+
if (root && root !== "." && root.length > 0) {
|
|
1889
|
+
const rootPosix = toPosix(root);
|
|
1890
|
+
const anchor = `/${rootPosix}/`;
|
|
1891
|
+
const idx = p.lastIndexOf(anchor);
|
|
1892
|
+
if (idx !== -1) return p.slice(idx + anchor.length);
|
|
1893
|
+
const base = rootPosix.split("/").filter(Boolean).pop();
|
|
1894
|
+
if (base) {
|
|
1895
|
+
const baseAnchor = `/${base}/`;
|
|
1896
|
+
const bidx = p.lastIndexOf(baseAnchor);
|
|
1897
|
+
if (bidx !== -1) return p.slice(bidx + baseAnchor.length);
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
p = p.replace(/^[A-Za-z]:/, "").replace(/^\/+/, "");
|
|
1901
|
+
return p.length > 0 ? p : null;
|
|
1902
|
+
}
|
|
1903
|
+
function callSiteFromSpan(span, serviceNode) {
|
|
1904
|
+
const filepath = span.attributes[CODE_FILEPATH_ATTR];
|
|
1905
|
+
if (typeof filepath !== "string" || filepath.length === 0) return null;
|
|
1906
|
+
const relPath = relPathForRuntimeFile(filepath, serviceNode);
|
|
1907
|
+
if (!relPath) return null;
|
|
1908
|
+
const linenoRaw = span.attributes[CODE_LINENO_ATTR];
|
|
1909
|
+
const line = typeof linenoRaw === "number" && Number.isFinite(linenoRaw) ? linenoRaw : void 0;
|
|
1910
|
+
const fnRaw = span.attributes[CODE_FUNCTION_ATTR];
|
|
1911
|
+
const fn = typeof fnRaw === "string" && fnRaw.length > 0 ? fnRaw : void 0;
|
|
1912
|
+
return { relPath, ...line !== void 0 ? { line } : {}, ...fn ? { fn } : {} };
|
|
1913
|
+
}
|
|
1914
|
+
function ensureObservedFileNode(graph, serviceName, serviceNodeId, callSite) {
|
|
1915
|
+
const fileNodeId = (0, import_types4.fileId)(serviceName, callSite.relPath);
|
|
1916
|
+
if (!graph.hasNode(fileNodeId)) {
|
|
1917
|
+
const language = languageForExt(callSite.relPath);
|
|
1918
|
+
const node = {
|
|
1919
|
+
id: fileNodeId,
|
|
1920
|
+
type: import_types4.NodeType.FileNode,
|
|
1921
|
+
service: serviceName,
|
|
1922
|
+
path: callSite.relPath,
|
|
1923
|
+
...language ? { language } : {},
|
|
1924
|
+
discoveredVia: "otel"
|
|
1925
|
+
};
|
|
1926
|
+
graph.addNode(fileNodeId, node);
|
|
1927
|
+
}
|
|
1928
|
+
const containsId = makeObservedEdgeId(import_types4.EdgeType.CONTAINS, serviceNodeId, fileNodeId);
|
|
1929
|
+
if (!graph.hasEdge(containsId)) {
|
|
1930
|
+
const edge = {
|
|
1931
|
+
id: containsId,
|
|
1932
|
+
source: serviceNodeId,
|
|
1933
|
+
target: fileNodeId,
|
|
1934
|
+
type: import_types4.EdgeType.CONTAINS,
|
|
1935
|
+
provenance: import_types4.Provenance.OBSERVED
|
|
1936
|
+
};
|
|
1937
|
+
graph.addEdgeWithKey(containsId, serviceNodeId, fileNodeId, edge);
|
|
1938
|
+
}
|
|
1939
|
+
return fileNodeId;
|
|
1940
|
+
}
|
|
1860
1941
|
function makeObservedEdgeId(type, source, target) {
|
|
1861
1942
|
return (0, import_types4.observedEdgeId)(source, target, type);
|
|
1862
1943
|
}
|
|
@@ -2074,6 +2155,9 @@ async function handleSpan(ctx, span) {
|
|
|
2074
2155
|
const sourceId = ensureServiceNode(ctx.graph, span.service, env);
|
|
2075
2156
|
const isError = span.statusCode === 2;
|
|
2076
2157
|
cacheSpanService(span, nowMs);
|
|
2158
|
+
const sourceServiceNode = ctx.graph.getNodeAttributes(sourceId);
|
|
2159
|
+
const callSite = callSiteFromSpan(span, sourceServiceNode);
|
|
2160
|
+
const observedSource = () => callSite ? ensureObservedFileNode(ctx.graph, span.service, sourceId, callSite) : sourceId;
|
|
2077
2161
|
let affectedNode = sourceId;
|
|
2078
2162
|
if (span.dbSystem) {
|
|
2079
2163
|
const host = pickAddress(span);
|
|
@@ -2083,7 +2167,7 @@ async function handleSpan(ctx, span) {
|
|
|
2083
2167
|
const result = upsertObservedEdge(
|
|
2084
2168
|
ctx.graph,
|
|
2085
2169
|
import_types4.EdgeType.CONNECTS_TO,
|
|
2086
|
-
|
|
2170
|
+
observedSource(),
|
|
2087
2171
|
targetId,
|
|
2088
2172
|
ts,
|
|
2089
2173
|
isError
|
|
@@ -2099,7 +2183,7 @@ async function handleSpan(ctx, span) {
|
|
|
2099
2183
|
upsertObservedEdge(
|
|
2100
2184
|
ctx.graph,
|
|
2101
2185
|
import_types4.EdgeType.CALLS,
|
|
2102
|
-
|
|
2186
|
+
observedSource(),
|
|
2103
2187
|
targetId,
|
|
2104
2188
|
ts,
|
|
2105
2189
|
isError
|
|
@@ -2111,7 +2195,7 @@ async function handleSpan(ctx, span) {
|
|
|
2111
2195
|
upsertObservedEdge(
|
|
2112
2196
|
ctx.graph,
|
|
2113
2197
|
import_types4.EdgeType.CALLS,
|
|
2114
|
-
|
|
2198
|
+
observedSource(),
|
|
2115
2199
|
frontierNodeId,
|
|
2116
2200
|
ts,
|
|
2117
2201
|
isError
|
|
@@ -3742,7 +3826,7 @@ async function addConfigNodes(graph, services, scanPath) {
|
|
|
3742
3826
|
|
|
3743
3827
|
// src/extract/calls/index.ts
|
|
3744
3828
|
init_cjs_shims();
|
|
3745
|
-
var
|
|
3829
|
+
var import_types16 = require("@neat.is/types");
|
|
3746
3830
|
|
|
3747
3831
|
// src/extract/calls/http.ts
|
|
3748
3832
|
init_cjs_shims();
|
|
@@ -3750,12 +3834,13 @@ var import_node_path20 = __toESM(require("path"), 1);
|
|
|
3750
3834
|
var import_tree_sitter = __toESM(require("tree-sitter"), 1);
|
|
3751
3835
|
var import_tree_sitter_javascript = __toESM(require("tree-sitter-javascript"), 1);
|
|
3752
3836
|
var import_tree_sitter_python = __toESM(require("tree-sitter-python"), 1);
|
|
3753
|
-
var
|
|
3837
|
+
var import_types11 = require("@neat.is/types");
|
|
3754
3838
|
|
|
3755
3839
|
// src/extract/calls/shared.ts
|
|
3756
3840
|
init_cjs_shims();
|
|
3757
3841
|
var import_node_fs13 = require("fs");
|
|
3758
3842
|
var import_node_path19 = __toESM(require("path"), 1);
|
|
3843
|
+
var import_types10 = require("@neat.is/types");
|
|
3759
3844
|
async function walkSourceFiles(dir) {
|
|
3760
3845
|
const out = [];
|
|
3761
3846
|
async function walk(current) {
|
|
@@ -3795,6 +3880,58 @@ function snippet(text, line) {
|
|
|
3795
3880
|
const lines = text.split("\n");
|
|
3796
3881
|
return (lines[line - 1] ?? "").trim();
|
|
3797
3882
|
}
|
|
3883
|
+
function toPosix2(p) {
|
|
3884
|
+
return p.split("\\").join("/");
|
|
3885
|
+
}
|
|
3886
|
+
function languageForPath(relPath) {
|
|
3887
|
+
switch (import_node_path19.default.extname(relPath).toLowerCase()) {
|
|
3888
|
+
case ".py":
|
|
3889
|
+
return "python";
|
|
3890
|
+
case ".ts":
|
|
3891
|
+
case ".tsx":
|
|
3892
|
+
return "typescript";
|
|
3893
|
+
case ".js":
|
|
3894
|
+
case ".jsx":
|
|
3895
|
+
case ".mjs":
|
|
3896
|
+
case ".cjs":
|
|
3897
|
+
return "javascript";
|
|
3898
|
+
default:
|
|
3899
|
+
return void 0;
|
|
3900
|
+
}
|
|
3901
|
+
}
|
|
3902
|
+
function ensureFileNode(graph, serviceName, serviceNodeId, relPath) {
|
|
3903
|
+
let nodesAdded = 0;
|
|
3904
|
+
let edgesAdded = 0;
|
|
3905
|
+
const fileNodeId = (0, import_types10.fileId)(serviceName, relPath);
|
|
3906
|
+
if (!graph.hasNode(fileNodeId)) {
|
|
3907
|
+
const language = languageForPath(relPath);
|
|
3908
|
+
const node = {
|
|
3909
|
+
id: fileNodeId,
|
|
3910
|
+
type: import_types10.NodeType.FileNode,
|
|
3911
|
+
service: serviceName,
|
|
3912
|
+
path: relPath,
|
|
3913
|
+
...language ? { language } : {},
|
|
3914
|
+
discoveredVia: "static"
|
|
3915
|
+
};
|
|
3916
|
+
graph.addNode(fileNodeId, node);
|
|
3917
|
+
nodesAdded++;
|
|
3918
|
+
}
|
|
3919
|
+
const containsId = (0, import_types10.extractedEdgeId)(serviceNodeId, fileNodeId, import_types10.EdgeType.CONTAINS);
|
|
3920
|
+
if (!graph.hasEdge(containsId)) {
|
|
3921
|
+
const edge = {
|
|
3922
|
+
id: containsId,
|
|
3923
|
+
source: serviceNodeId,
|
|
3924
|
+
target: fileNodeId,
|
|
3925
|
+
type: import_types10.EdgeType.CONTAINS,
|
|
3926
|
+
provenance: import_types10.Provenance.EXTRACTED,
|
|
3927
|
+
confidence: (0, import_types10.confidenceForExtracted)("structural"),
|
|
3928
|
+
evidence: { file: relPath }
|
|
3929
|
+
};
|
|
3930
|
+
graph.addEdgeWithKey(containsId, serviceNodeId, fileNodeId, edge);
|
|
3931
|
+
edgesAdded++;
|
|
3932
|
+
}
|
|
3933
|
+
return { fileNodeId, nodesAdded, edgesAdded };
|
|
3934
|
+
}
|
|
3798
3935
|
|
|
3799
3936
|
// src/extract/calls/http.ts
|
|
3800
3937
|
var STRING_LITERAL_NODE_TYPES = /* @__PURE__ */ new Set(["string_fragment", "string_content"]);
|
|
@@ -3828,16 +3965,16 @@ function callsFromSource(source, parser, knownHosts) {
|
|
|
3828
3965
|
const tree = parser.parse(source);
|
|
3829
3966
|
const literals = [];
|
|
3830
3967
|
collectStringLiterals(tree.rootNode, literals);
|
|
3831
|
-
const
|
|
3968
|
+
const out = [];
|
|
3832
3969
|
for (const lit of literals) {
|
|
3833
3970
|
if (isInsideJsxExternalLink(lit.node)) continue;
|
|
3834
3971
|
for (const host of knownHosts) {
|
|
3835
3972
|
if (urlMatchesHost(lit.text, host)) {
|
|
3836
|
-
|
|
3973
|
+
out.push({ host, line: lit.node.startPosition.row + 1 });
|
|
3837
3974
|
}
|
|
3838
3975
|
}
|
|
3839
3976
|
}
|
|
3840
|
-
return
|
|
3977
|
+
return out;
|
|
3841
3978
|
}
|
|
3842
3979
|
function makeJsParser() {
|
|
3843
3980
|
const p = new import_tree_sitter.default();
|
|
@@ -3860,71 +3997,78 @@ async function addHttpCallEdges(graph, services) {
|
|
|
3860
3997
|
hostToNodeId.set(import_node_path20.default.basename(service.dir), service.node.id);
|
|
3861
3998
|
hostToNodeId.set(service.pkg.name, service.node.id);
|
|
3862
3999
|
}
|
|
4000
|
+
let nodesAdded = 0;
|
|
3863
4001
|
let edgesAdded = 0;
|
|
3864
4002
|
for (const service of services) {
|
|
3865
4003
|
const files = await loadSourceFiles(service.dir);
|
|
3866
|
-
const
|
|
4004
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3867
4005
|
for (const file of files) {
|
|
3868
4006
|
if (isTestPath(file.path)) continue;
|
|
3869
4007
|
const parser = import_node_path20.default.extname(file.path) === ".py" ? pyParser : jsParser;
|
|
3870
|
-
let
|
|
4008
|
+
let sites;
|
|
3871
4009
|
try {
|
|
3872
|
-
|
|
4010
|
+
sites = callsFromSource(file.content, parser, knownHosts);
|
|
3873
4011
|
} catch (err) {
|
|
3874
4012
|
recordExtractionError("http call extraction", file.path, err);
|
|
3875
4013
|
continue;
|
|
3876
4014
|
}
|
|
3877
|
-
|
|
3878
|
-
|
|
4015
|
+
if (sites.length === 0) continue;
|
|
4016
|
+
const relFile = toPosix2(import_node_path20.default.relative(service.dir, file.path));
|
|
4017
|
+
for (const site of sites) {
|
|
4018
|
+
const targetId = hostToNodeId.get(site.host);
|
|
3879
4019
|
if (!targetId || targetId === service.node.id) continue;
|
|
3880
|
-
|
|
3881
|
-
|
|
4020
|
+
const dedupKey = `${relFile}|${targetId}`;
|
|
4021
|
+
if (seen.has(dedupKey)) continue;
|
|
4022
|
+
seen.add(dedupKey);
|
|
4023
|
+
const confidence = (0, import_types11.confidenceForExtracted)("hostname-shape-match");
|
|
4024
|
+
const ev = {
|
|
4025
|
+
file: relFile,
|
|
4026
|
+
line: site.line,
|
|
4027
|
+
snippet: snippet(file.content, site.line)
|
|
4028
|
+
};
|
|
4029
|
+
if (!(0, import_types11.passesExtractedFloor)(confidence)) {
|
|
4030
|
+
noteExtractedDropped({
|
|
4031
|
+
source: service.node.id,
|
|
4032
|
+
target: targetId,
|
|
4033
|
+
type: import_types11.EdgeType.CALLS,
|
|
4034
|
+
confidence,
|
|
4035
|
+
confidenceKind: "hostname-shape-match",
|
|
4036
|
+
evidence: ev
|
|
4037
|
+
});
|
|
4038
|
+
continue;
|
|
4039
|
+
}
|
|
4040
|
+
const { fileNodeId, nodesAdded: n, edgesAdded: e } = ensureFileNode(
|
|
4041
|
+
graph,
|
|
4042
|
+
service.pkg.name,
|
|
4043
|
+
service.node.id,
|
|
4044
|
+
relFile
|
|
4045
|
+
);
|
|
4046
|
+
nodesAdded += n;
|
|
4047
|
+
edgesAdded += e;
|
|
4048
|
+
const edgeId = (0, import_types5.extractedEdgeId)(fileNodeId, targetId, import_types11.EdgeType.CALLS);
|
|
4049
|
+
if (!graph.hasEdge(edgeId)) {
|
|
4050
|
+
const edge = {
|
|
4051
|
+
id: edgeId,
|
|
4052
|
+
source: fileNodeId,
|
|
4053
|
+
target: targetId,
|
|
4054
|
+
type: import_types11.EdgeType.CALLS,
|
|
4055
|
+
provenance: import_types11.Provenance.EXTRACTED,
|
|
4056
|
+
confidence,
|
|
4057
|
+
evidence: ev
|
|
4058
|
+
};
|
|
4059
|
+
graph.addEdgeWithKey(edgeId, fileNodeId, targetId, edge);
|
|
4060
|
+
edgesAdded++;
|
|
3882
4061
|
}
|
|
3883
|
-
}
|
|
3884
|
-
}
|
|
3885
|
-
for (const [targetId, evidenceFile] of seenTargets) {
|
|
3886
|
-
const fileContent = files.find((f) => f.path === evidenceFile.file)?.content ?? "";
|
|
3887
|
-
const line = lineOf(fileContent, `//${evidenceFile.host}`);
|
|
3888
|
-
const confidence = (0, import_types10.confidenceForExtracted)("hostname-shape-match");
|
|
3889
|
-
const ev = {
|
|
3890
|
-
file: import_node_path20.default.relative(service.dir, evidenceFile.file),
|
|
3891
|
-
line,
|
|
3892
|
-
snippet: snippet(fileContent, line)
|
|
3893
|
-
};
|
|
3894
|
-
const edgeId = (0, import_types5.extractedEdgeId)(service.node.id, targetId, import_types10.EdgeType.CALLS);
|
|
3895
|
-
if (!(0, import_types10.passesExtractedFloor)(confidence)) {
|
|
3896
|
-
noteExtractedDropped({
|
|
3897
|
-
source: service.node.id,
|
|
3898
|
-
target: targetId,
|
|
3899
|
-
type: import_types10.EdgeType.CALLS,
|
|
3900
|
-
confidence,
|
|
3901
|
-
confidenceKind: "hostname-shape-match",
|
|
3902
|
-
evidence: ev
|
|
3903
|
-
});
|
|
3904
|
-
continue;
|
|
3905
|
-
}
|
|
3906
|
-
const edge = {
|
|
3907
|
-
id: edgeId,
|
|
3908
|
-
source: service.node.id,
|
|
3909
|
-
target: targetId,
|
|
3910
|
-
type: import_types10.EdgeType.CALLS,
|
|
3911
|
-
provenance: import_types10.Provenance.EXTRACTED,
|
|
3912
|
-
confidence,
|
|
3913
|
-
evidence: ev
|
|
3914
|
-
};
|
|
3915
|
-
if (!graph.hasEdge(edge.id)) {
|
|
3916
|
-
graph.addEdgeWithKey(edge.id, edge.source, edge.target, edge);
|
|
3917
|
-
edgesAdded++;
|
|
3918
4062
|
}
|
|
3919
4063
|
}
|
|
3920
4064
|
}
|
|
3921
|
-
return edgesAdded;
|
|
4065
|
+
return { nodesAdded, edgesAdded };
|
|
3922
4066
|
}
|
|
3923
4067
|
|
|
3924
4068
|
// src/extract/calls/kafka.ts
|
|
3925
4069
|
init_cjs_shims();
|
|
3926
4070
|
var import_node_path21 = __toESM(require("path"), 1);
|
|
3927
|
-
var
|
|
4071
|
+
var import_types12 = require("@neat.is/types");
|
|
3928
4072
|
var PRODUCER_TOPIC_RE = /(?:producer|kafkaProducer)[\s\S]{0,40}?\.send\s*\(\s*\{[\s\S]{0,200}?topic\s*:\s*['"`]([^'"`]+)['"`]/g;
|
|
3929
4073
|
var CONSUMER_TOPIC_RE = /(?:consumer|kafkaConsumer)[\s\S]{0,40}?\.(?:subscribe|run)\s*\(\s*\{[\s\S]{0,200}?topic[s]?\s*:\s*(?:\[\s*)?['"`]([^'"`]+)['"`]/g;
|
|
3930
4074
|
function findAll(re, text) {
|
|
@@ -3945,7 +4089,7 @@ function kafkaEndpointsFromFile(file, serviceDir) {
|
|
|
3945
4089
|
seen.add(key);
|
|
3946
4090
|
const line = lineOf(file.content, topic);
|
|
3947
4091
|
out.push({
|
|
3948
|
-
infraId: (0,
|
|
4092
|
+
infraId: (0, import_types12.infraId)("kafka-topic", topic),
|
|
3949
4093
|
name: topic,
|
|
3950
4094
|
kind: "kafka-topic",
|
|
3951
4095
|
edgeType,
|
|
@@ -3968,7 +4112,7 @@ function kafkaEndpointsFromFile(file, serviceDir) {
|
|
|
3968
4112
|
// src/extract/calls/redis.ts
|
|
3969
4113
|
init_cjs_shims();
|
|
3970
4114
|
var import_node_path22 = __toESM(require("path"), 1);
|
|
3971
|
-
var
|
|
4115
|
+
var import_types13 = require("@neat.is/types");
|
|
3972
4116
|
var REDIS_URL_RE = /redis(?:s)?:\/\/(?:[^@'"`\s]+@)?([^:/'"`\s]+)(?::(\d+))?/g;
|
|
3973
4117
|
function redisEndpointsFromFile(file, serviceDir) {
|
|
3974
4118
|
const out = [];
|
|
@@ -3981,7 +4125,7 @@ function redisEndpointsFromFile(file, serviceDir) {
|
|
|
3981
4125
|
seen.add(host);
|
|
3982
4126
|
const line = lineOf(file.content, host);
|
|
3983
4127
|
out.push({
|
|
3984
|
-
infraId: (0,
|
|
4128
|
+
infraId: (0, import_types13.infraId)("redis", host),
|
|
3985
4129
|
name: host,
|
|
3986
4130
|
kind: "redis",
|
|
3987
4131
|
edgeType: "CALLS",
|
|
@@ -4002,7 +4146,7 @@ function redisEndpointsFromFile(file, serviceDir) {
|
|
|
4002
4146
|
// src/extract/calls/aws.ts
|
|
4003
4147
|
init_cjs_shims();
|
|
4004
4148
|
var import_node_path23 = __toESM(require("path"), 1);
|
|
4005
|
-
var
|
|
4149
|
+
var import_types14 = require("@neat.is/types");
|
|
4006
4150
|
var S3_BUCKET_RE = /Bucket\s*:\s*['"`]([^'"`]+)['"`]/g;
|
|
4007
4151
|
var DYNAMO_TABLE_RE = /TableName\s*:\s*['"`]([^'"`]+)['"`]/g;
|
|
4008
4152
|
function hasMarker(text, markers) {
|
|
@@ -4026,7 +4170,7 @@ function awsEndpointsFromFile(file, serviceDir) {
|
|
|
4026
4170
|
seen.add(key);
|
|
4027
4171
|
const line = lineOf(file.content, name);
|
|
4028
4172
|
out.push({
|
|
4029
|
-
infraId: (0,
|
|
4173
|
+
infraId: (0, import_types14.infraId)(kind, name),
|
|
4030
4174
|
name,
|
|
4031
4175
|
kind,
|
|
4032
4176
|
edgeType: "CALLS",
|
|
@@ -4061,7 +4205,7 @@ function awsEndpointsFromFile(file, serviceDir) {
|
|
|
4061
4205
|
// src/extract/calls/grpc.ts
|
|
4062
4206
|
init_cjs_shims();
|
|
4063
4207
|
var import_node_path24 = __toESM(require("path"), 1);
|
|
4064
|
-
var
|
|
4208
|
+
var import_types15 = require("@neat.is/types");
|
|
4065
4209
|
var GRPC_CLIENT_RE = /new\s+([A-Z][A-Za-z0-9_]*)Client\s*\(\s*['"`]?([^,'"`)]+)?/g;
|
|
4066
4210
|
var AWS_SDK_IMPORT_RE = /(?:from\s+['"`]|require\(\s*['"`])@aws-sdk\/client-([a-z0-9-]+)['"`]/g;
|
|
4067
4211
|
var GRPC_IMPORT_RE = /(?:from\s+['"`]|require\(\s*['"`])@grpc\/grpc-js['"`]|_grpc_pb['"`]/;
|
|
@@ -4109,7 +4253,7 @@ function grpcEndpointsFromFile(file, serviceDir) {
|
|
|
4109
4253
|
const { kind } = classified;
|
|
4110
4254
|
const line = lineOf(file.content, m[0]);
|
|
4111
4255
|
out.push({
|
|
4112
|
-
infraId: (0,
|
|
4256
|
+
infraId: (0, import_types15.infraId)(kind, name),
|
|
4113
4257
|
name,
|
|
4114
4258
|
kind,
|
|
4115
4259
|
edgeType: "CALLS",
|
|
@@ -4131,11 +4275,11 @@ function grpcEndpointsFromFile(file, serviceDir) {
|
|
|
4131
4275
|
function edgeTypeFromEndpoint(ep) {
|
|
4132
4276
|
switch (ep.edgeType) {
|
|
4133
4277
|
case "PUBLISHES_TO":
|
|
4134
|
-
return
|
|
4278
|
+
return import_types16.EdgeType.PUBLISHES_TO;
|
|
4135
4279
|
case "CONSUMES_FROM":
|
|
4136
|
-
return
|
|
4280
|
+
return import_types16.EdgeType.CONSUMES_FROM;
|
|
4137
4281
|
default:
|
|
4138
|
-
return
|
|
4282
|
+
return import_types16.EdgeType.CALLS;
|
|
4139
4283
|
}
|
|
4140
4284
|
}
|
|
4141
4285
|
function isAwsKind(kind) {
|
|
@@ -4162,7 +4306,7 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
4162
4306
|
if (!graph.hasNode(ep.infraId)) {
|
|
4163
4307
|
const node = {
|
|
4164
4308
|
id: ep.infraId,
|
|
4165
|
-
type:
|
|
4309
|
+
type: import_types16.NodeType.InfraNode,
|
|
4166
4310
|
name: ep.name,
|
|
4167
4311
|
// #238 — `aws-*` covers AWS-SDK client kinds (aws-s3, aws-dynamodb,
|
|
4168
4312
|
// aws-cognito-identity-provider, …); `s3-` / `dynamodb-` cover the
|
|
@@ -4174,11 +4318,8 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
4174
4318
|
nodesAdded++;
|
|
4175
4319
|
}
|
|
4176
4320
|
const edgeType = edgeTypeFromEndpoint(ep);
|
|
4177
|
-
const
|
|
4178
|
-
if (
|
|
4179
|
-
seenEdges.add(edgeId);
|
|
4180
|
-
const confidence = (0, import_types15.confidenceForExtracted)(ep.confidenceKind);
|
|
4181
|
-
if (!(0, import_types15.passesExtractedFloor)(confidence)) {
|
|
4321
|
+
const confidence = (0, import_types16.confidenceForExtracted)(ep.confidenceKind);
|
|
4322
|
+
if (!(0, import_types16.passesExtractedFloor)(confidence)) {
|
|
4182
4323
|
noteExtractedDropped({
|
|
4183
4324
|
source: service.node.id,
|
|
4184
4325
|
target: ep.infraId,
|
|
@@ -4189,13 +4330,25 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
4189
4330
|
});
|
|
4190
4331
|
continue;
|
|
4191
4332
|
}
|
|
4333
|
+
const relFile = toPosix2(ep.evidence.file);
|
|
4334
|
+
const { fileNodeId, nodesAdded: n, edgesAdded: e } = ensureFileNode(
|
|
4335
|
+
graph,
|
|
4336
|
+
service.pkg.name,
|
|
4337
|
+
service.node.id,
|
|
4338
|
+
relFile
|
|
4339
|
+
);
|
|
4340
|
+
nodesAdded += n;
|
|
4341
|
+
edgesAdded += e;
|
|
4342
|
+
const edgeId = (0, import_types5.extractedEdgeId)(fileNodeId, ep.infraId, edgeType);
|
|
4343
|
+
if (seenEdges.has(edgeId)) continue;
|
|
4344
|
+
seenEdges.add(edgeId);
|
|
4192
4345
|
if (!graph.hasEdge(edgeId)) {
|
|
4193
4346
|
const edge = {
|
|
4194
4347
|
id: edgeId,
|
|
4195
|
-
source:
|
|
4348
|
+
source: fileNodeId,
|
|
4196
4349
|
target: ep.infraId,
|
|
4197
4350
|
type: edgeType,
|
|
4198
|
-
provenance:
|
|
4351
|
+
provenance: import_types16.Provenance.EXTRACTED,
|
|
4199
4352
|
confidence,
|
|
4200
4353
|
evidence: ep.evidence
|
|
4201
4354
|
};
|
|
@@ -4207,11 +4360,11 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
4207
4360
|
return { nodesAdded, edgesAdded };
|
|
4208
4361
|
}
|
|
4209
4362
|
async function addCallEdges(graph, services) {
|
|
4210
|
-
const
|
|
4363
|
+
const http = await addHttpCallEdges(graph, services);
|
|
4211
4364
|
const ext = await addExternalEndpointEdges(graph, services);
|
|
4212
4365
|
return {
|
|
4213
|
-
nodesAdded: ext.nodesAdded,
|
|
4214
|
-
edgesAdded:
|
|
4366
|
+
nodesAdded: http.nodesAdded + ext.nodesAdded,
|
|
4367
|
+
edgesAdded: http.edgesAdded + ext.edgesAdded
|
|
4215
4368
|
};
|
|
4216
4369
|
}
|
|
4217
4370
|
|
|
@@ -4221,15 +4374,15 @@ init_cjs_shims();
|
|
|
4221
4374
|
// src/extract/infra/docker-compose.ts
|
|
4222
4375
|
init_cjs_shims();
|
|
4223
4376
|
var import_node_path25 = __toESM(require("path"), 1);
|
|
4224
|
-
var
|
|
4377
|
+
var import_types18 = require("@neat.is/types");
|
|
4225
4378
|
|
|
4226
4379
|
// src/extract/infra/shared.ts
|
|
4227
4380
|
init_cjs_shims();
|
|
4228
|
-
var
|
|
4381
|
+
var import_types17 = require("@neat.is/types");
|
|
4229
4382
|
function makeInfraNode(kind, name, provider = "self", extras) {
|
|
4230
4383
|
return {
|
|
4231
|
-
id: (0,
|
|
4232
|
-
type:
|
|
4384
|
+
id: (0, import_types17.infraId)(kind, name),
|
|
4385
|
+
type: import_types17.NodeType.InfraNode,
|
|
4233
4386
|
name,
|
|
4234
4387
|
provider,
|
|
4235
4388
|
kind,
|
|
@@ -4308,15 +4461,15 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
4308
4461
|
for (const dep of dependsOnList(svc.depends_on)) {
|
|
4309
4462
|
const targetId = composeNameToNodeId.get(dep);
|
|
4310
4463
|
if (!targetId) continue;
|
|
4311
|
-
const edgeId = (0, import_types5.extractedEdgeId)(sourceId, targetId,
|
|
4464
|
+
const edgeId = (0, import_types5.extractedEdgeId)(sourceId, targetId, import_types18.EdgeType.DEPENDS_ON);
|
|
4312
4465
|
if (graph.hasEdge(edgeId)) continue;
|
|
4313
4466
|
const edge = {
|
|
4314
4467
|
id: edgeId,
|
|
4315
4468
|
source: sourceId,
|
|
4316
4469
|
target: targetId,
|
|
4317
|
-
type:
|
|
4318
|
-
provenance:
|
|
4319
|
-
confidence: (0,
|
|
4470
|
+
type: import_types18.EdgeType.DEPENDS_ON,
|
|
4471
|
+
provenance: import_types18.Provenance.EXTRACTED,
|
|
4472
|
+
confidence: (0, import_types18.confidenceForExtracted)("structural"),
|
|
4320
4473
|
evidence: { file: evidenceFile }
|
|
4321
4474
|
};
|
|
4322
4475
|
graph.addEdgeWithKey(edgeId, edge.source, edge.target, edge);
|
|
@@ -4330,7 +4483,7 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
4330
4483
|
init_cjs_shims();
|
|
4331
4484
|
var import_node_path26 = __toESM(require("path"), 1);
|
|
4332
4485
|
var import_node_fs14 = require("fs");
|
|
4333
|
-
var
|
|
4486
|
+
var import_types19 = require("@neat.is/types");
|
|
4334
4487
|
function runtimeImage(content) {
|
|
4335
4488
|
const lines = content.split("\n");
|
|
4336
4489
|
let last = null;
|
|
@@ -4369,15 +4522,15 @@ async function addDockerfileRuntimes(graph, services, scanPath) {
|
|
|
4369
4522
|
graph.addNode(node.id, node);
|
|
4370
4523
|
nodesAdded++;
|
|
4371
4524
|
}
|
|
4372
|
-
const edgeId = (0, import_types5.extractedEdgeId)(service.node.id, node.id,
|
|
4525
|
+
const edgeId = (0, import_types5.extractedEdgeId)(service.node.id, node.id, import_types19.EdgeType.RUNS_ON);
|
|
4373
4526
|
if (!graph.hasEdge(edgeId)) {
|
|
4374
4527
|
const edge = {
|
|
4375
4528
|
id: edgeId,
|
|
4376
4529
|
source: service.node.id,
|
|
4377
4530
|
target: node.id,
|
|
4378
|
-
type:
|
|
4379
|
-
provenance:
|
|
4380
|
-
confidence: (0,
|
|
4531
|
+
type: import_types19.EdgeType.RUNS_ON,
|
|
4532
|
+
provenance: import_types19.Provenance.EXTRACTED,
|
|
4533
|
+
confidence: (0, import_types19.confidenceForExtracted)("structural"),
|
|
4381
4534
|
evidence: {
|
|
4382
4535
|
file: import_node_path26.default.relative(scanPath, dockerfilePath).split(import_node_path26.default.sep).join("/")
|
|
4383
4536
|
}
|
|
@@ -4505,13 +4658,24 @@ var import_node_path30 = __toESM(require("path"), 1);
|
|
|
4505
4658
|
init_cjs_shims();
|
|
4506
4659
|
var import_node_fs17 = require("fs");
|
|
4507
4660
|
var import_node_path29 = __toESM(require("path"), 1);
|
|
4508
|
-
var
|
|
4661
|
+
var import_types20 = require("@neat.is/types");
|
|
4662
|
+
function dropOrphanedFileNodes(graph) {
|
|
4663
|
+
const orphans = [];
|
|
4664
|
+
graph.forEachNode((id, attrs) => {
|
|
4665
|
+
if (attrs.type !== import_types20.NodeType.FileNode) return;
|
|
4666
|
+
if (graph.inboundEdges(id).length === 0 && graph.outboundEdges(id).length === 0) {
|
|
4667
|
+
orphans.push(id);
|
|
4668
|
+
}
|
|
4669
|
+
});
|
|
4670
|
+
for (const id of orphans) graph.dropNode(id);
|
|
4671
|
+
return orphans.length;
|
|
4672
|
+
}
|
|
4509
4673
|
function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
|
|
4510
4674
|
const toDrop = [];
|
|
4511
4675
|
const bases = [scanPath, ...serviceDirs];
|
|
4512
4676
|
graph.forEachEdge((id, attrs) => {
|
|
4513
4677
|
const edge = attrs;
|
|
4514
|
-
if (edge.provenance !==
|
|
4678
|
+
if (edge.provenance !== import_types20.Provenance.EXTRACTED) return;
|
|
4515
4679
|
const evidenceFile = edge.evidence?.file;
|
|
4516
4680
|
if (!evidenceFile) return;
|
|
4517
4681
|
if (import_node_path29.default.isAbsolute(evidenceFile)) {
|
|
@@ -4522,6 +4686,7 @@ function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
|
|
|
4522
4686
|
if (!found) toDrop.push(id);
|
|
4523
4687
|
});
|
|
4524
4688
|
for (const id of toDrop) graph.dropEdge(id);
|
|
4689
|
+
dropOrphanedFileNodes(graph);
|
|
4525
4690
|
return toDrop.length;
|
|
4526
4691
|
}
|
|
4527
4692
|
|
|
@@ -4668,7 +4833,7 @@ function canonicalJson(value) {
|
|
|
4668
4833
|
init_cjs_shims();
|
|
4669
4834
|
var import_node_fs19 = require("fs");
|
|
4670
4835
|
var import_node_path31 = __toESM(require("path"), 1);
|
|
4671
|
-
var
|
|
4836
|
+
var import_types21 = require("@neat.is/types");
|
|
4672
4837
|
var SCHEMA_VERSION = 4;
|
|
4673
4838
|
function migrateV1ToV2(payload) {
|
|
4674
4839
|
const nodes = payload.graph.nodes;
|
|
@@ -4690,12 +4855,12 @@ function migrateV2ToV3(payload) {
|
|
|
4690
4855
|
for (const edge of edges) {
|
|
4691
4856
|
const attrs = edge.attributes;
|
|
4692
4857
|
if (!attrs || attrs.provenance !== "FRONTIER") continue;
|
|
4693
|
-
attrs.provenance =
|
|
4858
|
+
attrs.provenance = import_types21.Provenance.OBSERVED;
|
|
4694
4859
|
const type = typeof attrs.type === "string" ? attrs.type : void 0;
|
|
4695
4860
|
const source = typeof attrs.source === "string" ? attrs.source : void 0;
|
|
4696
4861
|
const target = typeof attrs.target === "string" ? attrs.target : void 0;
|
|
4697
4862
|
if (type && source && target) {
|
|
4698
|
-
const newId = (0,
|
|
4863
|
+
const newId = (0, import_types21.observedEdgeId)(source, target, type);
|
|
4699
4864
|
attrs.id = newId;
|
|
4700
4865
|
if (edge.key) edge.key = newId;
|
|
4701
4866
|
}
|
|
@@ -4838,7 +5003,7 @@ init_cjs_shims();
|
|
|
4838
5003
|
var import_node_fs20 = require("fs");
|
|
4839
5004
|
var import_node_os2 = __toESM(require("os"), 1);
|
|
4840
5005
|
var import_node_path33 = __toESM(require("path"), 1);
|
|
4841
|
-
var
|
|
5006
|
+
var import_types22 = require("@neat.is/types");
|
|
4842
5007
|
function neatHome() {
|
|
4843
5008
|
const override = process.env.NEAT_HOME;
|
|
4844
5009
|
if (override && override.length > 0) return import_node_path33.default.resolve(override);
|
|
@@ -4859,7 +5024,7 @@ async function readRegistry() {
|
|
|
4859
5024
|
throw err;
|
|
4860
5025
|
}
|
|
4861
5026
|
const parsed = JSON.parse(raw);
|
|
4862
|
-
return
|
|
5027
|
+
return import_types22.RegistryFileSchema.parse(parsed);
|
|
4863
5028
|
}
|
|
4864
5029
|
async function getProject(name) {
|
|
4865
5030
|
const reg = await readRegistry();
|
|
@@ -5061,11 +5226,11 @@ function registerRoutes(scope, ctx) {
|
|
|
5061
5226
|
const candidates = req.query.type.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
5062
5227
|
const parsed = [];
|
|
5063
5228
|
for (const c of candidates) {
|
|
5064
|
-
const r =
|
|
5229
|
+
const r = import_types23.DivergenceTypeSchema.safeParse(c);
|
|
5065
5230
|
if (!r.success) {
|
|
5066
5231
|
return reply.code(400).send({
|
|
5067
5232
|
error: `unknown divergence type "${c}"`,
|
|
5068
|
-
allowed:
|
|
5233
|
+
allowed: import_types23.DivergenceTypeSchema.options
|
|
5069
5234
|
});
|
|
5070
5235
|
}
|
|
5071
5236
|
parsed.push(r.data);
|
|
@@ -5278,7 +5443,7 @@ function registerRoutes(scope, ctx) {
|
|
|
5278
5443
|
const log = new PolicyViolationsLog(proj.paths.policyViolationsPath);
|
|
5279
5444
|
let violations = await log.readAll();
|
|
5280
5445
|
if (req.query.severity) {
|
|
5281
|
-
const sev =
|
|
5446
|
+
const sev = import_types23.PolicySeveritySchema.safeParse(req.query.severity);
|
|
5282
5447
|
if (!sev.success) {
|
|
5283
5448
|
return reply.code(400).send({
|
|
5284
5449
|
error: "invalid severity",
|
|
@@ -5295,7 +5460,7 @@ function registerRoutes(scope, ctx) {
|
|
|
5295
5460
|
scope.post("/policies/check", async (req, reply) => {
|
|
5296
5461
|
const proj = resolveProject(registry, req, reply, ctx.bootstrap);
|
|
5297
5462
|
if (!proj) return;
|
|
5298
|
-
const parsed =
|
|
5463
|
+
const parsed = import_types23.PoliciesCheckBodySchema.safeParse(req.body ?? {});
|
|
5299
5464
|
if (!parsed.success) {
|
|
5300
5465
|
return reply.code(400).send({
|
|
5301
5466
|
error: "invalid /policies/check body",
|