@neat.is/core 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RBWL4HRB.js → chunk-7FTK47JQ.js} +277 -103
- package/dist/chunk-7FTK47JQ.js.map +1 -0
- package/dist/{chunk-NON5AXOR.js → chunk-LS6NS72S.js} +2 -2
- package/dist/cli.cjs +402 -145
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +107 -16
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +280 -115
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2 -2
- package/dist/neatd.cjs +280 -115
- package/dist/neatd.cjs.map +1 -1
- package/dist/neatd.js +2 -2
- package/dist/server.cjs +263 -98
- package/dist/server.cjs.map +1 -1
- package/dist/server.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-RBWL4HRB.js.map +0 -1
- /package/dist/{chunk-NON5AXOR.js.map → chunk-LS6NS72S.js.map} +0 -0
|
@@ -1027,6 +1027,7 @@ import {
|
|
|
1027
1027
|
confidenceForObservedSignal,
|
|
1028
1028
|
databaseId,
|
|
1029
1029
|
extractedEdgeId,
|
|
1030
|
+
fileId,
|
|
1030
1031
|
frontierId,
|
|
1031
1032
|
inferredEdgeId,
|
|
1032
1033
|
observedEdgeId,
|
|
@@ -1094,6 +1095,86 @@ function hostFromUrl(u) {
|
|
|
1094
1095
|
function pickAddress(span) {
|
|
1095
1096
|
return pickAttr(span, "server.address", "net.peer.name", "net.host.name") ?? hostFromUrl(pickAttr(span, "url.full", "http.url"));
|
|
1096
1097
|
}
|
|
1098
|
+
var CODE_FILEPATH_ATTR = "code.filepath";
|
|
1099
|
+
var CODE_LINENO_ATTR = "code.lineno";
|
|
1100
|
+
var CODE_FUNCTION_ATTR = "code.function";
|
|
1101
|
+
function toPosix(p) {
|
|
1102
|
+
return p.split("\\").join("/");
|
|
1103
|
+
}
|
|
1104
|
+
function languageForExt(relPath) {
|
|
1105
|
+
const dot = relPath.lastIndexOf(".");
|
|
1106
|
+
if (dot === -1) return void 0;
|
|
1107
|
+
switch (relPath.slice(dot).toLowerCase()) {
|
|
1108
|
+
case ".py":
|
|
1109
|
+
return "python";
|
|
1110
|
+
case ".ts":
|
|
1111
|
+
case ".tsx":
|
|
1112
|
+
return "typescript";
|
|
1113
|
+
case ".js":
|
|
1114
|
+
case ".jsx":
|
|
1115
|
+
case ".mjs":
|
|
1116
|
+
case ".cjs":
|
|
1117
|
+
return "javascript";
|
|
1118
|
+
default:
|
|
1119
|
+
return void 0;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
function relPathForRuntimeFile(filepath, serviceNode) {
|
|
1123
|
+
let p = toPosix(filepath).replace(/^file:\/\//, "");
|
|
1124
|
+
const root = serviceNode?.repoPath;
|
|
1125
|
+
if (root && root !== "." && root.length > 0) {
|
|
1126
|
+
const rootPosix = toPosix(root);
|
|
1127
|
+
const anchor = `/${rootPosix}/`;
|
|
1128
|
+
const idx = p.lastIndexOf(anchor);
|
|
1129
|
+
if (idx !== -1) return p.slice(idx + anchor.length);
|
|
1130
|
+
const base = rootPosix.split("/").filter(Boolean).pop();
|
|
1131
|
+
if (base) {
|
|
1132
|
+
const baseAnchor = `/${base}/`;
|
|
1133
|
+
const bidx = p.lastIndexOf(baseAnchor);
|
|
1134
|
+
if (bidx !== -1) return p.slice(bidx + baseAnchor.length);
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
p = p.replace(/^[A-Za-z]:/, "").replace(/^\/+/, "");
|
|
1138
|
+
return p.length > 0 ? p : null;
|
|
1139
|
+
}
|
|
1140
|
+
function callSiteFromSpan(span, serviceNode) {
|
|
1141
|
+
const filepath = span.attributes[CODE_FILEPATH_ATTR];
|
|
1142
|
+
if (typeof filepath !== "string" || filepath.length === 0) return null;
|
|
1143
|
+
const relPath = relPathForRuntimeFile(filepath, serviceNode);
|
|
1144
|
+
if (!relPath) return null;
|
|
1145
|
+
const linenoRaw = span.attributes[CODE_LINENO_ATTR];
|
|
1146
|
+
const line = typeof linenoRaw === "number" && Number.isFinite(linenoRaw) ? linenoRaw : void 0;
|
|
1147
|
+
const fnRaw = span.attributes[CODE_FUNCTION_ATTR];
|
|
1148
|
+
const fn = typeof fnRaw === "string" && fnRaw.length > 0 ? fnRaw : void 0;
|
|
1149
|
+
return { relPath, ...line !== void 0 ? { line } : {}, ...fn ? { fn } : {} };
|
|
1150
|
+
}
|
|
1151
|
+
function ensureObservedFileNode(graph, serviceName, serviceNodeId, callSite) {
|
|
1152
|
+
const fileNodeId = fileId(serviceName, callSite.relPath);
|
|
1153
|
+
if (!graph.hasNode(fileNodeId)) {
|
|
1154
|
+
const language = languageForExt(callSite.relPath);
|
|
1155
|
+
const node = {
|
|
1156
|
+
id: fileNodeId,
|
|
1157
|
+
type: NodeType3.FileNode,
|
|
1158
|
+
service: serviceName,
|
|
1159
|
+
path: callSite.relPath,
|
|
1160
|
+
...language ? { language } : {},
|
|
1161
|
+
discoveredVia: "otel"
|
|
1162
|
+
};
|
|
1163
|
+
graph.addNode(fileNodeId, node);
|
|
1164
|
+
}
|
|
1165
|
+
const containsId = makeObservedEdgeId(EdgeType2.CONTAINS, serviceNodeId, fileNodeId);
|
|
1166
|
+
if (!graph.hasEdge(containsId)) {
|
|
1167
|
+
const edge = {
|
|
1168
|
+
id: containsId,
|
|
1169
|
+
source: serviceNodeId,
|
|
1170
|
+
target: fileNodeId,
|
|
1171
|
+
type: EdgeType2.CONTAINS,
|
|
1172
|
+
provenance: Provenance.OBSERVED
|
|
1173
|
+
};
|
|
1174
|
+
graph.addEdgeWithKey(containsId, serviceNodeId, fileNodeId, edge);
|
|
1175
|
+
}
|
|
1176
|
+
return fileNodeId;
|
|
1177
|
+
}
|
|
1097
1178
|
function makeObservedEdgeId(type, source, target) {
|
|
1098
1179
|
return observedEdgeId(source, target, type);
|
|
1099
1180
|
}
|
|
@@ -1336,6 +1417,9 @@ async function handleSpan(ctx, span) {
|
|
|
1336
1417
|
const sourceId = ensureServiceNode(ctx.graph, span.service, env);
|
|
1337
1418
|
const isError = span.statusCode === 2;
|
|
1338
1419
|
cacheSpanService(span, nowMs);
|
|
1420
|
+
const sourceServiceNode = ctx.graph.getNodeAttributes(sourceId);
|
|
1421
|
+
const callSite = callSiteFromSpan(span, sourceServiceNode);
|
|
1422
|
+
const observedSource = () => callSite ? ensureObservedFileNode(ctx.graph, span.service, sourceId, callSite) : sourceId;
|
|
1339
1423
|
let affectedNode = sourceId;
|
|
1340
1424
|
if (span.dbSystem) {
|
|
1341
1425
|
const host = pickAddress(span);
|
|
@@ -1345,7 +1429,7 @@ async function handleSpan(ctx, span) {
|
|
|
1345
1429
|
const result = upsertObservedEdge(
|
|
1346
1430
|
ctx.graph,
|
|
1347
1431
|
EdgeType2.CONNECTS_TO,
|
|
1348
|
-
|
|
1432
|
+
observedSource(),
|
|
1349
1433
|
targetId,
|
|
1350
1434
|
ts,
|
|
1351
1435
|
isError
|
|
@@ -1361,7 +1445,7 @@ async function handleSpan(ctx, span) {
|
|
|
1361
1445
|
upsertObservedEdge(
|
|
1362
1446
|
ctx.graph,
|
|
1363
1447
|
EdgeType2.CALLS,
|
|
1364
|
-
|
|
1448
|
+
observedSource(),
|
|
1365
1449
|
targetId,
|
|
1366
1450
|
ts,
|
|
1367
1451
|
isError
|
|
@@ -1373,7 +1457,7 @@ async function handleSpan(ctx, span) {
|
|
|
1373
1457
|
upsertObservedEdge(
|
|
1374
1458
|
ctx.graph,
|
|
1375
1459
|
EdgeType2.CALLS,
|
|
1376
|
-
|
|
1460
|
+
observedSource(),
|
|
1377
1461
|
frontierNodeId,
|
|
1378
1462
|
ts,
|
|
1379
1463
|
isError
|
|
@@ -3010,10 +3094,10 @@ async function addConfigNodes(graph, services, scanPath) {
|
|
|
3010
3094
|
|
|
3011
3095
|
// src/extract/calls/index.ts
|
|
3012
3096
|
import {
|
|
3013
|
-
EdgeType as
|
|
3014
|
-
NodeType as
|
|
3015
|
-
Provenance as
|
|
3016
|
-
confidenceForExtracted as
|
|
3097
|
+
EdgeType as EdgeType7,
|
|
3098
|
+
NodeType as NodeType9,
|
|
3099
|
+
Provenance as Provenance6,
|
|
3100
|
+
confidenceForExtracted as confidenceForExtracted5,
|
|
3017
3101
|
passesExtractedFloor as passesExtractedFloor2
|
|
3018
3102
|
} from "@neat.is/types";
|
|
3019
3103
|
|
|
@@ -3023,15 +3107,23 @@ import Parser from "tree-sitter";
|
|
|
3023
3107
|
import JavaScript from "tree-sitter-javascript";
|
|
3024
3108
|
import Python from "tree-sitter-python";
|
|
3025
3109
|
import {
|
|
3026
|
-
EdgeType as
|
|
3027
|
-
Provenance as
|
|
3028
|
-
confidenceForExtracted as
|
|
3110
|
+
EdgeType as EdgeType6,
|
|
3111
|
+
Provenance as Provenance5,
|
|
3112
|
+
confidenceForExtracted as confidenceForExtracted4,
|
|
3029
3113
|
passesExtractedFloor
|
|
3030
3114
|
} from "@neat.is/types";
|
|
3031
3115
|
|
|
3032
3116
|
// src/extract/calls/shared.ts
|
|
3033
3117
|
import { promises as fs13 } from "fs";
|
|
3034
3118
|
import path19 from "path";
|
|
3119
|
+
import {
|
|
3120
|
+
EdgeType as EdgeType5,
|
|
3121
|
+
NodeType as NodeType8,
|
|
3122
|
+
Provenance as Provenance4,
|
|
3123
|
+
confidenceForExtracted as confidenceForExtracted3,
|
|
3124
|
+
extractedEdgeId as extractedEdgeId3,
|
|
3125
|
+
fileId as fileId2
|
|
3126
|
+
} from "@neat.is/types";
|
|
3035
3127
|
async function walkSourceFiles(dir) {
|
|
3036
3128
|
const out = [];
|
|
3037
3129
|
async function walk(current) {
|
|
@@ -3071,6 +3163,58 @@ function snippet(text, line) {
|
|
|
3071
3163
|
const lines = text.split("\n");
|
|
3072
3164
|
return (lines[line - 1] ?? "").trim();
|
|
3073
3165
|
}
|
|
3166
|
+
function toPosix2(p) {
|
|
3167
|
+
return p.split("\\").join("/");
|
|
3168
|
+
}
|
|
3169
|
+
function languageForPath(relPath) {
|
|
3170
|
+
switch (path19.extname(relPath).toLowerCase()) {
|
|
3171
|
+
case ".py":
|
|
3172
|
+
return "python";
|
|
3173
|
+
case ".ts":
|
|
3174
|
+
case ".tsx":
|
|
3175
|
+
return "typescript";
|
|
3176
|
+
case ".js":
|
|
3177
|
+
case ".jsx":
|
|
3178
|
+
case ".mjs":
|
|
3179
|
+
case ".cjs":
|
|
3180
|
+
return "javascript";
|
|
3181
|
+
default:
|
|
3182
|
+
return void 0;
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
function ensureFileNode(graph, serviceName, serviceNodeId, relPath) {
|
|
3186
|
+
let nodesAdded = 0;
|
|
3187
|
+
let edgesAdded = 0;
|
|
3188
|
+
const fileNodeId = fileId2(serviceName, relPath);
|
|
3189
|
+
if (!graph.hasNode(fileNodeId)) {
|
|
3190
|
+
const language = languageForPath(relPath);
|
|
3191
|
+
const node = {
|
|
3192
|
+
id: fileNodeId,
|
|
3193
|
+
type: NodeType8.FileNode,
|
|
3194
|
+
service: serviceName,
|
|
3195
|
+
path: relPath,
|
|
3196
|
+
...language ? { language } : {},
|
|
3197
|
+
discoveredVia: "static"
|
|
3198
|
+
};
|
|
3199
|
+
graph.addNode(fileNodeId, node);
|
|
3200
|
+
nodesAdded++;
|
|
3201
|
+
}
|
|
3202
|
+
const containsId = extractedEdgeId3(serviceNodeId, fileNodeId, EdgeType5.CONTAINS);
|
|
3203
|
+
if (!graph.hasEdge(containsId)) {
|
|
3204
|
+
const edge = {
|
|
3205
|
+
id: containsId,
|
|
3206
|
+
source: serviceNodeId,
|
|
3207
|
+
target: fileNodeId,
|
|
3208
|
+
type: EdgeType5.CONTAINS,
|
|
3209
|
+
provenance: Provenance4.EXTRACTED,
|
|
3210
|
+
confidence: confidenceForExtracted3("structural"),
|
|
3211
|
+
evidence: { file: relPath }
|
|
3212
|
+
};
|
|
3213
|
+
graph.addEdgeWithKey(containsId, serviceNodeId, fileNodeId, edge);
|
|
3214
|
+
edgesAdded++;
|
|
3215
|
+
}
|
|
3216
|
+
return { fileNodeId, nodesAdded, edgesAdded };
|
|
3217
|
+
}
|
|
3074
3218
|
|
|
3075
3219
|
// src/extract/calls/http.ts
|
|
3076
3220
|
var STRING_LITERAL_NODE_TYPES = /* @__PURE__ */ new Set(["string_fragment", "string_content"]);
|
|
@@ -3104,16 +3248,16 @@ function callsFromSource(source, parser, knownHosts) {
|
|
|
3104
3248
|
const tree = parser.parse(source);
|
|
3105
3249
|
const literals = [];
|
|
3106
3250
|
collectStringLiterals(tree.rootNode, literals);
|
|
3107
|
-
const
|
|
3251
|
+
const out = [];
|
|
3108
3252
|
for (const lit of literals) {
|
|
3109
3253
|
if (isInsideJsxExternalLink(lit.node)) continue;
|
|
3110
3254
|
for (const host of knownHosts) {
|
|
3111
3255
|
if (urlMatchesHost(lit.text, host)) {
|
|
3112
|
-
|
|
3256
|
+
out.push({ host, line: lit.node.startPosition.row + 1 });
|
|
3113
3257
|
}
|
|
3114
3258
|
}
|
|
3115
3259
|
}
|
|
3116
|
-
return
|
|
3260
|
+
return out;
|
|
3117
3261
|
}
|
|
3118
3262
|
function makeJsParser() {
|
|
3119
3263
|
const p = new Parser();
|
|
@@ -3136,65 +3280,72 @@ async function addHttpCallEdges(graph, services) {
|
|
|
3136
3280
|
hostToNodeId.set(path20.basename(service.dir), service.node.id);
|
|
3137
3281
|
hostToNodeId.set(service.pkg.name, service.node.id);
|
|
3138
3282
|
}
|
|
3283
|
+
let nodesAdded = 0;
|
|
3139
3284
|
let edgesAdded = 0;
|
|
3140
3285
|
for (const service of services) {
|
|
3141
3286
|
const files = await loadSourceFiles(service.dir);
|
|
3142
|
-
const
|
|
3287
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3143
3288
|
for (const file of files) {
|
|
3144
3289
|
if (isTestPath(file.path)) continue;
|
|
3145
3290
|
const parser = path20.extname(file.path) === ".py" ? pyParser : jsParser;
|
|
3146
|
-
let
|
|
3291
|
+
let sites;
|
|
3147
3292
|
try {
|
|
3148
|
-
|
|
3293
|
+
sites = callsFromSource(file.content, parser, knownHosts);
|
|
3149
3294
|
} catch (err) {
|
|
3150
3295
|
recordExtractionError("http call extraction", file.path, err);
|
|
3151
3296
|
continue;
|
|
3152
3297
|
}
|
|
3153
|
-
|
|
3154
|
-
|
|
3298
|
+
if (sites.length === 0) continue;
|
|
3299
|
+
const relFile = toPosix2(path20.relative(service.dir, file.path));
|
|
3300
|
+
for (const site of sites) {
|
|
3301
|
+
const targetId = hostToNodeId.get(site.host);
|
|
3155
3302
|
if (!targetId || targetId === service.node.id) continue;
|
|
3156
|
-
|
|
3157
|
-
|
|
3303
|
+
const dedupKey = `${relFile}|${targetId}`;
|
|
3304
|
+
if (seen.has(dedupKey)) continue;
|
|
3305
|
+
seen.add(dedupKey);
|
|
3306
|
+
const confidence = confidenceForExtracted4("hostname-shape-match");
|
|
3307
|
+
const ev = {
|
|
3308
|
+
file: relFile,
|
|
3309
|
+
line: site.line,
|
|
3310
|
+
snippet: snippet(file.content, site.line)
|
|
3311
|
+
};
|
|
3312
|
+
if (!passesExtractedFloor(confidence)) {
|
|
3313
|
+
noteExtractedDropped({
|
|
3314
|
+
source: service.node.id,
|
|
3315
|
+
target: targetId,
|
|
3316
|
+
type: EdgeType6.CALLS,
|
|
3317
|
+
confidence,
|
|
3318
|
+
confidenceKind: "hostname-shape-match",
|
|
3319
|
+
evidence: ev
|
|
3320
|
+
});
|
|
3321
|
+
continue;
|
|
3322
|
+
}
|
|
3323
|
+
const { fileNodeId, nodesAdded: n, edgesAdded: e } = ensureFileNode(
|
|
3324
|
+
graph,
|
|
3325
|
+
service.pkg.name,
|
|
3326
|
+
service.node.id,
|
|
3327
|
+
relFile
|
|
3328
|
+
);
|
|
3329
|
+
nodesAdded += n;
|
|
3330
|
+
edgesAdded += e;
|
|
3331
|
+
const edgeId = extractedEdgeId2(fileNodeId, targetId, EdgeType6.CALLS);
|
|
3332
|
+
if (!graph.hasEdge(edgeId)) {
|
|
3333
|
+
const edge = {
|
|
3334
|
+
id: edgeId,
|
|
3335
|
+
source: fileNodeId,
|
|
3336
|
+
target: targetId,
|
|
3337
|
+
type: EdgeType6.CALLS,
|
|
3338
|
+
provenance: Provenance5.EXTRACTED,
|
|
3339
|
+
confidence,
|
|
3340
|
+
evidence: ev
|
|
3341
|
+
};
|
|
3342
|
+
graph.addEdgeWithKey(edgeId, fileNodeId, targetId, edge);
|
|
3343
|
+
edgesAdded++;
|
|
3158
3344
|
}
|
|
3159
|
-
}
|
|
3160
|
-
}
|
|
3161
|
-
for (const [targetId, evidenceFile] of seenTargets) {
|
|
3162
|
-
const fileContent = files.find((f) => f.path === evidenceFile.file)?.content ?? "";
|
|
3163
|
-
const line = lineOf(fileContent, `//${evidenceFile.host}`);
|
|
3164
|
-
const confidence = confidenceForExtracted3("hostname-shape-match");
|
|
3165
|
-
const ev = {
|
|
3166
|
-
file: path20.relative(service.dir, evidenceFile.file),
|
|
3167
|
-
line,
|
|
3168
|
-
snippet: snippet(fileContent, line)
|
|
3169
|
-
};
|
|
3170
|
-
const edgeId = extractedEdgeId2(service.node.id, targetId, EdgeType5.CALLS);
|
|
3171
|
-
if (!passesExtractedFloor(confidence)) {
|
|
3172
|
-
noteExtractedDropped({
|
|
3173
|
-
source: service.node.id,
|
|
3174
|
-
target: targetId,
|
|
3175
|
-
type: EdgeType5.CALLS,
|
|
3176
|
-
confidence,
|
|
3177
|
-
confidenceKind: "hostname-shape-match",
|
|
3178
|
-
evidence: ev
|
|
3179
|
-
});
|
|
3180
|
-
continue;
|
|
3181
|
-
}
|
|
3182
|
-
const edge = {
|
|
3183
|
-
id: edgeId,
|
|
3184
|
-
source: service.node.id,
|
|
3185
|
-
target: targetId,
|
|
3186
|
-
type: EdgeType5.CALLS,
|
|
3187
|
-
provenance: Provenance4.EXTRACTED,
|
|
3188
|
-
confidence,
|
|
3189
|
-
evidence: ev
|
|
3190
|
-
};
|
|
3191
|
-
if (!graph.hasEdge(edge.id)) {
|
|
3192
|
-
graph.addEdgeWithKey(edge.id, edge.source, edge.target, edge);
|
|
3193
|
-
edgesAdded++;
|
|
3194
3345
|
}
|
|
3195
3346
|
}
|
|
3196
3347
|
}
|
|
3197
|
-
return edgesAdded;
|
|
3348
|
+
return { nodesAdded, edgesAdded };
|
|
3198
3349
|
}
|
|
3199
3350
|
|
|
3200
3351
|
// src/extract/calls/kafka.ts
|
|
@@ -3403,11 +3554,11 @@ function grpcEndpointsFromFile(file, serviceDir) {
|
|
|
3403
3554
|
function edgeTypeFromEndpoint(ep) {
|
|
3404
3555
|
switch (ep.edgeType) {
|
|
3405
3556
|
case "PUBLISHES_TO":
|
|
3406
|
-
return
|
|
3557
|
+
return EdgeType7.PUBLISHES_TO;
|
|
3407
3558
|
case "CONSUMES_FROM":
|
|
3408
|
-
return
|
|
3559
|
+
return EdgeType7.CONSUMES_FROM;
|
|
3409
3560
|
default:
|
|
3410
|
-
return
|
|
3561
|
+
return EdgeType7.CALLS;
|
|
3411
3562
|
}
|
|
3412
3563
|
}
|
|
3413
3564
|
function isAwsKind(kind) {
|
|
@@ -3434,7 +3585,7 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
3434
3585
|
if (!graph.hasNode(ep.infraId)) {
|
|
3435
3586
|
const node = {
|
|
3436
3587
|
id: ep.infraId,
|
|
3437
|
-
type:
|
|
3588
|
+
type: NodeType9.InfraNode,
|
|
3438
3589
|
name: ep.name,
|
|
3439
3590
|
// #238 — `aws-*` covers AWS-SDK client kinds (aws-s3, aws-dynamodb,
|
|
3440
3591
|
// aws-cognito-identity-provider, …); `s3-` / `dynamodb-` cover the
|
|
@@ -3446,10 +3597,7 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
3446
3597
|
nodesAdded++;
|
|
3447
3598
|
}
|
|
3448
3599
|
const edgeType = edgeTypeFromEndpoint(ep);
|
|
3449
|
-
const
|
|
3450
|
-
if (seenEdges.has(edgeId)) continue;
|
|
3451
|
-
seenEdges.add(edgeId);
|
|
3452
|
-
const confidence = confidenceForExtracted4(ep.confidenceKind);
|
|
3600
|
+
const confidence = confidenceForExtracted5(ep.confidenceKind);
|
|
3453
3601
|
if (!passesExtractedFloor2(confidence)) {
|
|
3454
3602
|
noteExtractedDropped({
|
|
3455
3603
|
source: service.node.id,
|
|
@@ -3461,13 +3609,25 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
3461
3609
|
});
|
|
3462
3610
|
continue;
|
|
3463
3611
|
}
|
|
3612
|
+
const relFile = toPosix2(ep.evidence.file);
|
|
3613
|
+
const { fileNodeId, nodesAdded: n, edgesAdded: e } = ensureFileNode(
|
|
3614
|
+
graph,
|
|
3615
|
+
service.pkg.name,
|
|
3616
|
+
service.node.id,
|
|
3617
|
+
relFile
|
|
3618
|
+
);
|
|
3619
|
+
nodesAdded += n;
|
|
3620
|
+
edgesAdded += e;
|
|
3621
|
+
const edgeId = extractedEdgeId2(fileNodeId, ep.infraId, edgeType);
|
|
3622
|
+
if (seenEdges.has(edgeId)) continue;
|
|
3623
|
+
seenEdges.add(edgeId);
|
|
3464
3624
|
if (!graph.hasEdge(edgeId)) {
|
|
3465
3625
|
const edge = {
|
|
3466
3626
|
id: edgeId,
|
|
3467
|
-
source:
|
|
3627
|
+
source: fileNodeId,
|
|
3468
3628
|
target: ep.infraId,
|
|
3469
3629
|
type: edgeType,
|
|
3470
|
-
provenance:
|
|
3630
|
+
provenance: Provenance6.EXTRACTED,
|
|
3471
3631
|
confidence,
|
|
3472
3632
|
evidence: ep.evidence
|
|
3473
3633
|
};
|
|
@@ -3479,24 +3639,24 @@ async function addExternalEndpointEdges(graph, services) {
|
|
|
3479
3639
|
return { nodesAdded, edgesAdded };
|
|
3480
3640
|
}
|
|
3481
3641
|
async function addCallEdges(graph, services) {
|
|
3482
|
-
const
|
|
3642
|
+
const http = await addHttpCallEdges(graph, services);
|
|
3483
3643
|
const ext = await addExternalEndpointEdges(graph, services);
|
|
3484
3644
|
return {
|
|
3485
|
-
nodesAdded: ext.nodesAdded,
|
|
3486
|
-
edgesAdded:
|
|
3645
|
+
nodesAdded: http.nodesAdded + ext.nodesAdded,
|
|
3646
|
+
edgesAdded: http.edgesAdded + ext.edgesAdded
|
|
3487
3647
|
};
|
|
3488
3648
|
}
|
|
3489
3649
|
|
|
3490
3650
|
// src/extract/infra/docker-compose.ts
|
|
3491
3651
|
import path25 from "path";
|
|
3492
|
-
import { EdgeType as
|
|
3652
|
+
import { EdgeType as EdgeType8, Provenance as Provenance7, confidenceForExtracted as confidenceForExtracted6 } from "@neat.is/types";
|
|
3493
3653
|
|
|
3494
3654
|
// src/extract/infra/shared.ts
|
|
3495
|
-
import { NodeType as
|
|
3655
|
+
import { NodeType as NodeType10, infraId as infraId5 } from "@neat.is/types";
|
|
3496
3656
|
function makeInfraNode(kind, name, provider = "self", extras) {
|
|
3497
3657
|
return {
|
|
3498
3658
|
id: infraId5(kind, name),
|
|
3499
|
-
type:
|
|
3659
|
+
type: NodeType10.InfraNode,
|
|
3500
3660
|
name,
|
|
3501
3661
|
provider,
|
|
3502
3662
|
kind,
|
|
@@ -3575,15 +3735,15 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
3575
3735
|
for (const dep of dependsOnList(svc.depends_on)) {
|
|
3576
3736
|
const targetId = composeNameToNodeId.get(dep);
|
|
3577
3737
|
if (!targetId) continue;
|
|
3578
|
-
const edgeId = extractedEdgeId2(sourceId, targetId,
|
|
3738
|
+
const edgeId = extractedEdgeId2(sourceId, targetId, EdgeType8.DEPENDS_ON);
|
|
3579
3739
|
if (graph.hasEdge(edgeId)) continue;
|
|
3580
3740
|
const edge = {
|
|
3581
3741
|
id: edgeId,
|
|
3582
3742
|
source: sourceId,
|
|
3583
3743
|
target: targetId,
|
|
3584
|
-
type:
|
|
3585
|
-
provenance:
|
|
3586
|
-
confidence:
|
|
3744
|
+
type: EdgeType8.DEPENDS_ON,
|
|
3745
|
+
provenance: Provenance7.EXTRACTED,
|
|
3746
|
+
confidence: confidenceForExtracted6("structural"),
|
|
3587
3747
|
evidence: { file: evidenceFile }
|
|
3588
3748
|
};
|
|
3589
3749
|
graph.addEdgeWithKey(edgeId, edge.source, edge.target, edge);
|
|
@@ -3596,7 +3756,7 @@ async function addComposeInfra(graph, scanPath, services) {
|
|
|
3596
3756
|
// src/extract/infra/dockerfile.ts
|
|
3597
3757
|
import path26 from "path";
|
|
3598
3758
|
import { promises as fs14 } from "fs";
|
|
3599
|
-
import { EdgeType as
|
|
3759
|
+
import { EdgeType as EdgeType9, Provenance as Provenance8, confidenceForExtracted as confidenceForExtracted7 } from "@neat.is/types";
|
|
3600
3760
|
function runtimeImage(content) {
|
|
3601
3761
|
const lines = content.split("\n");
|
|
3602
3762
|
let last = null;
|
|
@@ -3635,15 +3795,15 @@ async function addDockerfileRuntimes(graph, services, scanPath) {
|
|
|
3635
3795
|
graph.addNode(node.id, node);
|
|
3636
3796
|
nodesAdded++;
|
|
3637
3797
|
}
|
|
3638
|
-
const edgeId = extractedEdgeId2(service.node.id, node.id,
|
|
3798
|
+
const edgeId = extractedEdgeId2(service.node.id, node.id, EdgeType9.RUNS_ON);
|
|
3639
3799
|
if (!graph.hasEdge(edgeId)) {
|
|
3640
3800
|
const edge = {
|
|
3641
3801
|
id: edgeId,
|
|
3642
3802
|
source: service.node.id,
|
|
3643
3803
|
target: node.id,
|
|
3644
|
-
type:
|
|
3645
|
-
provenance:
|
|
3646
|
-
confidence:
|
|
3804
|
+
type: EdgeType9.RUNS_ON,
|
|
3805
|
+
provenance: Provenance8.EXTRACTED,
|
|
3806
|
+
confidence: confidenceForExtracted7("structural"),
|
|
3647
3807
|
evidence: {
|
|
3648
3808
|
file: path26.relative(scanPath, dockerfilePath).split(path26.sep).join("/")
|
|
3649
3809
|
}
|
|
@@ -3768,17 +3928,29 @@ import path30 from "path";
|
|
|
3768
3928
|
// src/extract/retire.ts
|
|
3769
3929
|
import { existsSync } from "fs";
|
|
3770
3930
|
import path29 from "path";
|
|
3771
|
-
import { Provenance as
|
|
3931
|
+
import { NodeType as NodeType11, Provenance as Provenance9 } from "@neat.is/types";
|
|
3932
|
+
function dropOrphanedFileNodes(graph) {
|
|
3933
|
+
const orphans = [];
|
|
3934
|
+
graph.forEachNode((id, attrs) => {
|
|
3935
|
+
if (attrs.type !== NodeType11.FileNode) return;
|
|
3936
|
+
if (graph.inboundEdges(id).length === 0 && graph.outboundEdges(id).length === 0) {
|
|
3937
|
+
orphans.push(id);
|
|
3938
|
+
}
|
|
3939
|
+
});
|
|
3940
|
+
for (const id of orphans) graph.dropNode(id);
|
|
3941
|
+
return orphans.length;
|
|
3942
|
+
}
|
|
3772
3943
|
function retireEdgesByFile(graph, file) {
|
|
3773
3944
|
const normalized = file.split("\\").join("/");
|
|
3774
3945
|
const toDrop = [];
|
|
3775
3946
|
graph.forEachEdge((id, attrs) => {
|
|
3776
3947
|
const edge = attrs;
|
|
3777
|
-
if (edge.provenance !==
|
|
3948
|
+
if (edge.provenance !== Provenance9.EXTRACTED) return;
|
|
3778
3949
|
if (!edge.evidence?.file) return;
|
|
3779
3950
|
if (edge.evidence.file === normalized) toDrop.push(id);
|
|
3780
3951
|
});
|
|
3781
3952
|
for (const id of toDrop) graph.dropEdge(id);
|
|
3953
|
+
dropOrphanedFileNodes(graph);
|
|
3782
3954
|
return toDrop.length;
|
|
3783
3955
|
}
|
|
3784
3956
|
function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
|
|
@@ -3786,7 +3958,7 @@ function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
|
|
|
3786
3958
|
const bases = [scanPath, ...serviceDirs];
|
|
3787
3959
|
graph.forEachEdge((id, attrs) => {
|
|
3788
3960
|
const edge = attrs;
|
|
3789
|
-
if (edge.provenance !==
|
|
3961
|
+
if (edge.provenance !== Provenance9.EXTRACTED) return;
|
|
3790
3962
|
const evidenceFile = edge.evidence?.file;
|
|
3791
3963
|
if (!evidenceFile) return;
|
|
3792
3964
|
if (path29.isAbsolute(evidenceFile)) {
|
|
@@ -3797,6 +3969,7 @@ function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
|
|
|
3797
3969
|
if (!found) toDrop.push(id);
|
|
3798
3970
|
});
|
|
3799
3971
|
for (const id of toDrop) graph.dropEdge(id);
|
|
3972
|
+
dropOrphanedFileNodes(graph);
|
|
3800
3973
|
return toDrop.length;
|
|
3801
3974
|
}
|
|
3802
3975
|
|
|
@@ -3865,10 +4038,10 @@ async function extractFromDirectory(graph, scanPath, opts = {}) {
|
|
|
3865
4038
|
// src/divergences.ts
|
|
3866
4039
|
import {
|
|
3867
4040
|
DivergenceResultSchema,
|
|
3868
|
-
EdgeType as
|
|
3869
|
-
NodeType as
|
|
4041
|
+
EdgeType as EdgeType10,
|
|
4042
|
+
NodeType as NodeType12,
|
|
3870
4043
|
parseEdgeId,
|
|
3871
|
-
Provenance as
|
|
4044
|
+
Provenance as Provenance10
|
|
3872
4045
|
} from "@neat.is/types";
|
|
3873
4046
|
function bucketKey(source, target, type) {
|
|
3874
4047
|
return `${type}|${source}|${target}`;
|
|
@@ -3882,17 +4055,17 @@ function bucketEdges(graph) {
|
|
|
3882
4055
|
const key = bucketKey(e.source, e.target, e.type);
|
|
3883
4056
|
const cur = buckets.get(key) ?? { source: e.source, target: e.target, type: e.type };
|
|
3884
4057
|
switch (provenance) {
|
|
3885
|
-
case
|
|
4058
|
+
case Provenance10.EXTRACTED:
|
|
3886
4059
|
cur.extracted = e;
|
|
3887
4060
|
break;
|
|
3888
|
-
case
|
|
4061
|
+
case Provenance10.OBSERVED:
|
|
3889
4062
|
cur.observed = e;
|
|
3890
4063
|
break;
|
|
3891
|
-
case
|
|
4064
|
+
case Provenance10.INFERRED:
|
|
3892
4065
|
cur.inferred = e;
|
|
3893
4066
|
break;
|
|
3894
4067
|
default:
|
|
3895
|
-
if (e.provenance ===
|
|
4068
|
+
if (e.provenance === Provenance10.STALE) cur.stale = e;
|
|
3896
4069
|
}
|
|
3897
4070
|
buckets.set(key, cur);
|
|
3898
4071
|
});
|
|
@@ -3901,7 +4074,7 @@ function bucketEdges(graph) {
|
|
|
3901
4074
|
function nodeIsFrontier(graph, nodeId) {
|
|
3902
4075
|
if (!graph.hasNode(nodeId)) return false;
|
|
3903
4076
|
const attrs = graph.getNodeAttributes(nodeId);
|
|
3904
|
-
return attrs.type ===
|
|
4077
|
+
return attrs.type === NodeType12.FrontierNode;
|
|
3905
4078
|
}
|
|
3906
4079
|
function clampConfidence(n) {
|
|
3907
4080
|
if (!Number.isFinite(n)) return 0;
|
|
@@ -3922,6 +4095,7 @@ function gradedConfidence(edge) {
|
|
|
3922
4095
|
}
|
|
3923
4096
|
function detectMissingDivergences(graph, bucket) {
|
|
3924
4097
|
const out = [];
|
|
4098
|
+
if (bucket.type === EdgeType10.CONTAINS) return out;
|
|
3925
4099
|
if (bucket.extracted && !bucket.observed) {
|
|
3926
4100
|
if (!nodeIsFrontier(graph, bucket.target)) {
|
|
3927
4101
|
out.push({
|
|
@@ -3962,7 +4136,7 @@ function declaredHostFor(svc) {
|
|
|
3962
4136
|
function hasExtractedConfiguredBy(graph, svcId) {
|
|
3963
4137
|
for (const edgeId of graph.outboundEdges(svcId)) {
|
|
3964
4138
|
const e = graph.getEdgeAttributes(edgeId);
|
|
3965
|
-
if (e.type ===
|
|
4139
|
+
if (e.type === EdgeType10.CONFIGURED_BY && e.provenance === Provenance10.EXTRACTED) {
|
|
3966
4140
|
return true;
|
|
3967
4141
|
}
|
|
3968
4142
|
}
|
|
@@ -3975,10 +4149,10 @@ function detectHostMismatch(graph, svcId, svc) {
|
|
|
3975
4149
|
const out = [];
|
|
3976
4150
|
for (const edgeId of graph.outboundEdges(svcId)) {
|
|
3977
4151
|
const edge = graph.getEdgeAttributes(edgeId);
|
|
3978
|
-
if (edge.type !==
|
|
3979
|
-
if (edge.provenance !==
|
|
4152
|
+
if (edge.type !== EdgeType10.CONNECTS_TO) continue;
|
|
4153
|
+
if (edge.provenance !== Provenance10.OBSERVED) continue;
|
|
3980
4154
|
const target = graph.getNodeAttributes(edge.target);
|
|
3981
|
-
if (target.type !==
|
|
4155
|
+
if (target.type !== NodeType12.DatabaseNode) continue;
|
|
3982
4156
|
const observedHost = target.host?.trim();
|
|
3983
4157
|
if (!observedHost) continue;
|
|
3984
4158
|
if (observedHost === declaredHost) continue;
|
|
@@ -4000,10 +4174,10 @@ function detectCompatDivergences(graph, svcId, svc) {
|
|
|
4000
4174
|
const deps = svc.dependencies ?? {};
|
|
4001
4175
|
for (const edgeId of graph.outboundEdges(svcId)) {
|
|
4002
4176
|
const edge = graph.getEdgeAttributes(edgeId);
|
|
4003
|
-
if (edge.type !==
|
|
4004
|
-
if (edge.provenance !==
|
|
4177
|
+
if (edge.type !== EdgeType10.CONNECTS_TO) continue;
|
|
4178
|
+
if (edge.provenance !== Provenance10.OBSERVED) continue;
|
|
4005
4179
|
const target = graph.getNodeAttributes(edge.target);
|
|
4006
|
-
if (target.type !==
|
|
4180
|
+
if (target.type !== NodeType12.DatabaseNode) continue;
|
|
4007
4181
|
for (const pair of compatPairs()) {
|
|
4008
4182
|
if (pair.engine !== target.engine) continue;
|
|
4009
4183
|
const declared = deps[pair.driver];
|
|
@@ -4064,7 +4238,7 @@ function computeDivergences(graph, opts = {}) {
|
|
|
4064
4238
|
}
|
|
4065
4239
|
graph.forEachNode((nodeId, attrs) => {
|
|
4066
4240
|
const n = attrs;
|
|
4067
|
-
if (n.type !==
|
|
4241
|
+
if (n.type !== NodeType12.ServiceNode) return;
|
|
4068
4242
|
const svc = n;
|
|
4069
4243
|
for (const d of detectHostMismatch(graph, nodeId, svc)) all.push(d);
|
|
4070
4244
|
for (const d of detectCompatDivergences(graph, nodeId, svc)) all.push(d);
|
|
@@ -4107,7 +4281,7 @@ function computeDivergences(graph, opts = {}) {
|
|
|
4107
4281
|
// src/persist.ts
|
|
4108
4282
|
import { promises as fs17 } from "fs";
|
|
4109
4283
|
import path31 from "path";
|
|
4110
|
-
import { Provenance as
|
|
4284
|
+
import { Provenance as Provenance11, observedEdgeId as observedEdgeId2 } from "@neat.is/types";
|
|
4111
4285
|
var SCHEMA_VERSION = 4;
|
|
4112
4286
|
function migrateV1ToV2(payload) {
|
|
4113
4287
|
const nodes = payload.graph.nodes;
|
|
@@ -4129,7 +4303,7 @@ function migrateV2ToV3(payload) {
|
|
|
4129
4303
|
for (const edge of edges) {
|
|
4130
4304
|
const attrs = edge.attributes;
|
|
4131
4305
|
if (!attrs || attrs.provenance !== "FRONTIER") continue;
|
|
4132
|
-
attrs.provenance =
|
|
4306
|
+
attrs.provenance = Provenance11.OBSERVED;
|
|
4133
4307
|
const type = typeof attrs.type === "string" ? attrs.type : void 0;
|
|
4134
4308
|
const source = typeof attrs.source === "string" ? attrs.source : void 0;
|
|
4135
4309
|
const target = typeof attrs.target === "string" ? attrs.target : void 0;
|
|
@@ -5137,4 +5311,4 @@ export {
|
|
|
5137
5311
|
removeProject,
|
|
5138
5312
|
buildApi
|
|
5139
5313
|
};
|
|
5140
|
-
//# sourceMappingURL=chunk-
|
|
5314
|
+
//# sourceMappingURL=chunk-7FTK47JQ.js.map
|