@neat.is/core 0.9.13-dev.20260901 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-WE3AFQYL.js → chunk-NV4WWJSU.js} +78 -7
- package/dist/chunk-NV4WWJSU.js.map +1 -0
- package/dist/{chunk-RBZNXA5L.js → chunk-SSLPBQWY.js} +162 -38
- package/dist/chunk-SSLPBQWY.js.map +1 -0
- package/dist/cli.cjs +691 -84
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +23 -3
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +225 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +2 -2
- package/dist/neatd.cjs +225 -44
- package/dist/neatd.cjs.map +1 -1
- package/dist/neatd.js +2 -2
- package/dist/server.cjs +128 -37
- package/dist/server.cjs.map +1 -1
- package/dist/server.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-RBZNXA5L.js.map +0 -1
- package/dist/chunk-WE3AFQYL.js.map +0 -1
package/dist/index.d.cts
CHANGED
|
@@ -183,6 +183,10 @@ interface ObservedSignal {
|
|
|
183
183
|
callSite?: ConnectorCallSite;
|
|
184
184
|
columns?: string[];
|
|
185
185
|
incident?: ConnectorIncident;
|
|
186
|
+
deployState?: {
|
|
187
|
+
image?: string;
|
|
188
|
+
readyReplicas?: number;
|
|
189
|
+
};
|
|
186
190
|
}
|
|
187
191
|
|
|
188
192
|
/**
|
|
@@ -436,6 +440,7 @@ interface StalenessLoopOptions {
|
|
|
436
440
|
staleEventsPath?: string;
|
|
437
441
|
project?: string;
|
|
438
442
|
onPolicyTrigger?: (graph: NeatGraph) => Promise<void> | void;
|
|
443
|
+
onReconcile?: (graph: NeatGraph) => Promise<void> | void;
|
|
439
444
|
}
|
|
440
445
|
declare function startStalenessLoop(graph: NeatGraph, options?: StalenessLoopOptions): () => void;
|
|
441
446
|
declare function readErrorEvents(errorsPath: string, opts?: {
|
package/dist/index.d.ts
CHANGED
|
@@ -183,6 +183,10 @@ interface ObservedSignal {
|
|
|
183
183
|
callSite?: ConnectorCallSite;
|
|
184
184
|
columns?: string[];
|
|
185
185
|
incident?: ConnectorIncident;
|
|
186
|
+
deployState?: {
|
|
187
|
+
image?: string;
|
|
188
|
+
readyReplicas?: number;
|
|
189
|
+
};
|
|
186
190
|
}
|
|
187
191
|
|
|
188
192
|
/**
|
|
@@ -436,6 +440,7 @@ interface StalenessLoopOptions {
|
|
|
436
440
|
staleEventsPath?: string;
|
|
437
441
|
project?: string;
|
|
438
442
|
onPolicyTrigger?: (graph: NeatGraph) => Promise<void> | void;
|
|
443
|
+
onReconcile?: (graph: NeatGraph) => Promise<void> | void;
|
|
439
444
|
}
|
|
440
445
|
declare function startStalenessLoop(graph: NeatGraph, options?: StalenessLoopOptions): () => void;
|
|
441
446
|
declare function readErrorEvents(errorsPath: string, opts?: {
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
routeSpanToProject,
|
|
3
3
|
startDaemon
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-NV4WWJSU.js";
|
|
5
5
|
import {
|
|
6
6
|
ProjectNameCollisionError,
|
|
7
7
|
addProject,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
thresholdForEdgeType,
|
|
38
38
|
touchLastSeen,
|
|
39
39
|
writeAtomically
|
|
40
|
-
} from "./chunk-
|
|
40
|
+
} from "./chunk-SSLPBQWY.js";
|
|
41
41
|
import {
|
|
42
42
|
startOtelGrpcReceiver
|
|
43
43
|
} from "./chunk-ERE47MCR.js";
|
package/dist/neatd.cjs
CHANGED
|
@@ -1247,13 +1247,19 @@ function resolveOwningService(graph, nodeId) {
|
|
|
1247
1247
|
}
|
|
1248
1248
|
return null;
|
|
1249
1249
|
}
|
|
1250
|
+
function isFrontierEdge(e) {
|
|
1251
|
+
return e.provenance === import_types.Provenance.FRONTIER;
|
|
1252
|
+
}
|
|
1253
|
+
function rankOf(p) {
|
|
1254
|
+
return p === import_types.Provenance.FRONTIER ? -1 : import_types.PROV_RANK[p];
|
|
1255
|
+
}
|
|
1250
1256
|
function bestEdgeBySource(graph, edgeIds) {
|
|
1251
1257
|
const best = /* @__PURE__ */ new Map();
|
|
1252
1258
|
for (const id of edgeIds) {
|
|
1253
1259
|
const e = graph.getEdgeAttributes(id);
|
|
1254
|
-
if (isFrontierNode(graph, e.source)) continue;
|
|
1260
|
+
if (isFrontierNode(graph, e.source) || isFrontierEdge(e)) continue;
|
|
1255
1261
|
const cur = best.get(e.source);
|
|
1256
|
-
if (!cur ||
|
|
1262
|
+
if (!cur || rankOf(e.provenance) > rankOf(cur.provenance)) {
|
|
1257
1263
|
best.set(e.source, e);
|
|
1258
1264
|
}
|
|
1259
1265
|
}
|
|
@@ -1263,9 +1269,9 @@ function bestEdgeByTarget(graph, edgeIds) {
|
|
|
1263
1269
|
const best = /* @__PURE__ */ new Map();
|
|
1264
1270
|
for (const id of edgeIds) {
|
|
1265
1271
|
const e = graph.getEdgeAttributes(id);
|
|
1266
|
-
if (isFrontierNode(graph, e.target)) continue;
|
|
1272
|
+
if (isFrontierNode(graph, e.target) || isFrontierEdge(e)) continue;
|
|
1267
1273
|
const cur = best.get(e.target);
|
|
1268
|
-
if (!cur ||
|
|
1274
|
+
if (!cur || rankOf(e.provenance) > rankOf(cur.provenance)) {
|
|
1269
1275
|
best.set(e.target, e);
|
|
1270
1276
|
}
|
|
1271
1277
|
}
|
|
@@ -1275,7 +1281,10 @@ var PROVENANCE_CEILING = {
|
|
|
1275
1281
|
OBSERVED: 1,
|
|
1276
1282
|
INFERRED: 0.7,
|
|
1277
1283
|
EXTRACTED: 0.5,
|
|
1278
|
-
STALE: 0.3
|
|
1284
|
+
STALE: 0.3,
|
|
1285
|
+
// A FRONTIER surface (ADR-226) is a proposal about a cause NEAT could not observe
|
|
1286
|
+
// — the least-trusted claim it makes, below STALE (which was at least once seen).
|
|
1287
|
+
FRONTIER: 0.2
|
|
1279
1288
|
};
|
|
1280
1289
|
function volumeWeight(spanCount) {
|
|
1281
1290
|
if (!spanCount || spanCount <= 0) return 0.5;
|
|
@@ -1510,7 +1519,7 @@ function rootCauseFromIncidents(nodeId, incidents, errorEvent) {
|
|
|
1510
1519
|
});
|
|
1511
1520
|
}
|
|
1512
1521
|
function isFailingCallEdge(e) {
|
|
1513
|
-
return e.type === import_types.EdgeType.CALLS && (e.signal?.errorCount ?? 0) > 0;
|
|
1522
|
+
return e.type === import_types.EdgeType.CALLS && !isFrontierEdge(e) && (e.signal?.errorCount ?? 0) > 0;
|
|
1514
1523
|
}
|
|
1515
1524
|
function callSourcesForService(graph, serviceId17) {
|
|
1516
1525
|
const ids = [serviceId17];
|
|
@@ -1533,8 +1542,8 @@ function failingCallDominates(e, id, curEdge, curId) {
|
|
|
1533
1542
|
const ec = e.signal?.errorCount ?? 0;
|
|
1534
1543
|
const cc = curEdge.signal?.errorCount ?? 0;
|
|
1535
1544
|
if (ec !== cc) return ec > cc;
|
|
1536
|
-
if (
|
|
1537
|
-
return
|
|
1545
|
+
if (rankOf(e.provenance) !== rankOf(curEdge.provenance)) {
|
|
1546
|
+
return rankOf(e.provenance) > rankOf(curEdge.provenance);
|
|
1538
1547
|
}
|
|
1539
1548
|
return id < curId;
|
|
1540
1549
|
}
|
|
@@ -1585,11 +1594,11 @@ function dominantStaleCall(graph, serviceId17, visited) {
|
|
|
1585
1594
|
for (const edgeId of graph.outboundEdges(src)) {
|
|
1586
1595
|
const e = graph.getEdgeAttributes(edgeId);
|
|
1587
1596
|
if (e.type !== import_types.EdgeType.CALLS) continue;
|
|
1588
|
-
if (isFrontierNode(graph, e.target)) continue;
|
|
1597
|
+
if (isFrontierNode(graph, e.target) || isFrontierEdge(e)) continue;
|
|
1589
1598
|
const owner = resolveOwningService(graph, e.target);
|
|
1590
1599
|
if (!owner || visited.has(owner.id)) continue;
|
|
1591
1600
|
const cur = bestByCallee.get(owner.id);
|
|
1592
|
-
if (!cur ||
|
|
1601
|
+
if (!cur || rankOf(e.provenance) > rankOf(cur.provenance)) {
|
|
1593
1602
|
bestByCallee.set(owner.id, e);
|
|
1594
1603
|
}
|
|
1595
1604
|
}
|
|
@@ -1862,6 +1871,7 @@ function nodeContext(graph, nodeId, incidents, now = Date.now()) {
|
|
|
1862
1871
|
for (const edgeId of graph.inboundEdges(n)) {
|
|
1863
1872
|
const e = graph.getEdgeAttributes(edgeId);
|
|
1864
1873
|
if (e.type === import_types.EdgeType.CONTAINS) continue;
|
|
1874
|
+
if (isFrontierEdge(e)) continue;
|
|
1865
1875
|
errorsFromCallers += e.signal?.errorCount ?? 0;
|
|
1866
1876
|
inboundVolume += e.callCount ?? e.signal?.spanCount ?? 0;
|
|
1867
1877
|
if (e.provenance === import_types.Provenance.STALE) stale = true;
|
|
@@ -1875,6 +1885,7 @@ function nodeContext(graph, nodeId, incidents, now = Date.now()) {
|
|
|
1875
1885
|
for (const edgeId of graph.outboundEdges(n)) {
|
|
1876
1886
|
const e = graph.getEdgeAttributes(edgeId);
|
|
1877
1887
|
if (e.type === import_types.EdgeType.CONTAINS) continue;
|
|
1888
|
+
if (isFrontierEdge(e)) continue;
|
|
1878
1889
|
outboundVolume += e.callCount ?? e.signal?.spanCount ?? 0;
|
|
1879
1890
|
if (e.type === import_types.EdgeType.CALLS) outboundErrors += e.signal?.errorCount ?? 0;
|
|
1880
1891
|
if (e.provenance === import_types.Provenance.STALE) stale = true;
|
|
@@ -1908,6 +1919,11 @@ function isSaturated(ctx) {
|
|
|
1908
1919
|
function isBoundaryTimeoutSymptom(ctx) {
|
|
1909
1920
|
return ctx.errorsEmittedHere > 0 && ctx.boundaryTimeout === true && ctx.errorsFromCallers === 0 && ctx.hasOutboundDeps === true && ctx.observedErroringDownstream !== true;
|
|
1910
1921
|
}
|
|
1922
|
+
var UNREACHABLE_INBOUND_ERROR_RATE = 0.5;
|
|
1923
|
+
var UNREACHABLE_MIN_INBOUND = 3;
|
|
1924
|
+
function isUnreachableSeed(ctx) {
|
|
1925
|
+
return ctx.callCount >= UNREACHABLE_MIN_INBOUND && ctx.errorsFromCallers > 0 && ctx.errorsFromCallers >= UNREACHABLE_INBOUND_ERROR_RATE * ctx.callCount && ctx.errorsEmittedHere === 0 && ctx.outboundVolume === 0;
|
|
1926
|
+
}
|
|
1911
1927
|
function classifyNode(ctx) {
|
|
1912
1928
|
if (ctx.errorsEmittedHere > 0) {
|
|
1913
1929
|
if ((ctx.stale || isSaturated(ctx)) && ctx.errorsFromCallers >= ctx.errorsEmittedHere) {
|
|
@@ -1916,6 +1932,7 @@ function classifyNode(ctx) {
|
|
|
1916
1932
|
if (isBoundaryTimeoutSymptom(ctx)) return "symptom-only";
|
|
1917
1933
|
return "primary-failure";
|
|
1918
1934
|
}
|
|
1935
|
+
if (isUnreachableSeed(ctx)) return "unreachable";
|
|
1919
1936
|
if (ctx.errorsFromCallers > 0) return "symptom-only";
|
|
1920
1937
|
return "unrelated";
|
|
1921
1938
|
}
|
|
@@ -2152,22 +2169,44 @@ function routeMatches(declared, incident) {
|
|
|
2152
2169
|
const b = normalizeRoute(incident);
|
|
2153
2170
|
return a === b || a.startsWith(b) || b.startsWith(a);
|
|
2154
2171
|
}
|
|
2155
|
-
function
|
|
2156
|
-
const
|
|
2157
|
-
const callees = declaredOutboundCallees(graph, boundaryNode);
|
|
2172
|
+
function firstDeclaredCallee(graph, node, route) {
|
|
2173
|
+
const callees = declaredOutboundCallees(graph, node);
|
|
2158
2174
|
if (callees.length === 0) return null;
|
|
2159
2175
|
const narrowed = route !== void 0 ? callees.filter((c) => c.route !== void 0 && routeMatches(c.route, route)) : callees;
|
|
2160
2176
|
const chosen = narrowed.length === 1 ? narrowed[0] : callees.length === 1 ? callees[0] : null;
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
if (
|
|
2167
|
-
|
|
2168
|
-
|
|
2177
|
+
return chosen ? chosen.target : null;
|
|
2178
|
+
}
|
|
2179
|
+
function observedOutboundNextServices(graph, nodeId) {
|
|
2180
|
+
const targets = /* @__PURE__ */ new Set();
|
|
2181
|
+
for (const n of nodeScope(graph, nodeId)) {
|
|
2182
|
+
if (!graph.hasNode(n)) continue;
|
|
2183
|
+
for (const edgeId of graph.outboundEdges(n)) {
|
|
2184
|
+
const e = graph.getEdgeAttributes(edgeId);
|
|
2185
|
+
if (!DEP_EDGE_TYPES.has(e.type)) continue;
|
|
2186
|
+
if (e.provenance !== import_types.Provenance.OBSERVED) continue;
|
|
2187
|
+
if (e.target !== nodeId) targets.add(e.target);
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
return [...targets];
|
|
2191
|
+
}
|
|
2192
|
+
function resolveHangHop(graph, boundaryNode, incidents) {
|
|
2193
|
+
const route = boundaryIncidentRoute(boundaryNode, incidents);
|
|
2194
|
+
const own = firstDeclaredCallee(graph, boundaryNode, route);
|
|
2195
|
+
if (own) {
|
|
2196
|
+
return route !== void 0 ? { source: boundaryNode, target: own, route } : { source: boundaryNode, target: own };
|
|
2197
|
+
}
|
|
2198
|
+
const resolved = observedOutboundNextServices(graph, boundaryNode).map((n) => ({ source: n, target: firstDeclaredCallee(graph, n, route) })).filter((r) => r.target !== null);
|
|
2199
|
+
if (resolved.length === 1) {
|
|
2200
|
+
const hop = resolved[0];
|
|
2201
|
+
return route !== void 0 ? { ...hop, route } : hop;
|
|
2169
2202
|
}
|
|
2170
|
-
return
|
|
2203
|
+
return null;
|
|
2204
|
+
}
|
|
2205
|
+
function stagedHangProposal(graph, boundaryNode, incidents) {
|
|
2206
|
+
const hop = resolveHangHop(graph, boundaryNode, incidents);
|
|
2207
|
+
if (!hop) return null;
|
|
2208
|
+
if (!graph.hasEdge((0, import_types.frontierEdgeId)(hop.source, hop.target, import_types.EdgeType.CALLS))) return null;
|
|
2209
|
+
return hop.route !== void 0 ? { node: hop.target, route: hop.route } : { node: hop.target };
|
|
2171
2210
|
}
|
|
2172
2211
|
function enrichWithNavigation(graph, errorNodeId, tagged, incidents, now) {
|
|
2173
2212
|
const legacy = tagged.result;
|
|
@@ -2202,29 +2241,39 @@ function enrichWithNavigation(graph, errorNodeId, tagged, incidents, now) {
|
|
|
2202
2241
|
confidence: Math.min(legacy.confidence, 0.4),
|
|
2203
2242
|
...lastProv ? { provenance: lastProv } : {}
|
|
2204
2243
|
});
|
|
2244
|
+
} else if (seedCtx && isUnreachableSeed(seedCtx)) {
|
|
2245
|
+
const name = displayNameOf(seedNode);
|
|
2246
|
+
candidates.push({
|
|
2247
|
+
node: seedNode,
|
|
2248
|
+
classification: "unreachable",
|
|
2249
|
+
reason: `${name} is unreachable: its callers' requests fail (${seedCtx.errorsFromCallers} erroring inbound calls) and it produced no telemetry of its own \u2014 no server spans, no outbound calls \u2014 so it never served. The failure is observed, but its cause is not in the trace: a startup failure, a crash before the first span, or an unschedulable / unhealthy pod. Inspect ${name}'s deploy state and logs \u2014 there is no code fault to find in the graph here.`,
|
|
2250
|
+
context: seedCtx,
|
|
2251
|
+
confidence: legacy.confidence,
|
|
2252
|
+
provenance: import_types.Provenance.OBSERVED
|
|
2253
|
+
});
|
|
2205
2254
|
} else if (seedCtx && isBoundaryTimeoutSymptom(seedCtx)) {
|
|
2206
|
-
const pointer = structuralUpstreamPointer(graph, seedNode, incidents);
|
|
2207
2255
|
const seedName = displayNameOf(seedNode);
|
|
2208
|
-
const
|
|
2209
|
-
|
|
2210
|
-
const causeName = displayNameOf(pointer.node);
|
|
2211
|
-
candidates.push({
|
|
2212
|
-
node: pointer.node,
|
|
2213
|
-
classification: "primary-failure",
|
|
2214
|
-
reason: `${causeName} is the likely root cause (structural, INFERRED \u2014 not observed): the boundary ${seedName} only timed out waiting, and the actual culprit hung without exporting a span, so this is walked from the declared call graph${routeNote}, not from runtime signal. Restore instrumentation / inspect ${causeName} to confirm.`,
|
|
2215
|
-
context: nodeContext(graph, pointer.node, incidents, now),
|
|
2216
|
-
confidence: Math.min(legacy.confidence, PROVENANCE_CEILING.EXTRACTED),
|
|
2217
|
-
provenance: import_types.Provenance.INFERRED
|
|
2218
|
-
});
|
|
2219
|
-
}
|
|
2256
|
+
const proposal = stagedHangProposal(graph, seedNode, incidents);
|
|
2257
|
+
const routeNote = proposal?.route ? ` serving ${proposal.route}` : "";
|
|
2220
2258
|
candidates.push({
|
|
2221
2259
|
node: seedNode,
|
|
2222
2260
|
classification: "symptom-only",
|
|
2223
|
-
reason:
|
|
2261
|
+
reason: proposal ? `${seedName} timed out waiting on a downstream that hung and exported no span \u2014 a boundary reporting an upstream fault, not the fault itself. The cause is unobservable; NEAT proposes the staged FRONTIER surface below, a hypothesis to confirm, not an observed cause.` : `${seedName} timed out but nothing it can see downstream is erroring \u2014 the cause is downstream and unobserved (the culprit hung and exported no span). No FRONTIER surface${routeNote} is staged, so no cause is proposed; inspect ${seedName}'s declared downstream and restore instrumentation.`,
|
|
2224
2262
|
context: seedCtx,
|
|
2225
2263
|
confidence: Math.min(legacy.confidence, 0.3),
|
|
2226
2264
|
...lastProv ? { provenance: lastProv } : {}
|
|
2227
2265
|
});
|
|
2266
|
+
if (proposal) {
|
|
2267
|
+
const causeName = displayNameOf(proposal.node);
|
|
2268
|
+
candidates.push({
|
|
2269
|
+
node: proposal.node,
|
|
2270
|
+
classification: "primary-failure",
|
|
2271
|
+
reason: `${causeName} is the proposed hang cause (FRONTIER \u2014 a hypothesis, not observed): the boundary ${seedName} only timed out waiting and the culprit hung without exporting a span, so NEAT staged a FRONTIER surface on the unobservable hop${routeNote}. It graduates to OBSERVED when ${causeName} comes back; until then, inspect it / restore instrumentation to confirm, or it is culled.`,
|
|
2272
|
+
context: nodeContext(graph, proposal.node, incidents, now),
|
|
2273
|
+
confidence: Math.min(legacy.confidence, PROVENANCE_CEILING.FRONTIER),
|
|
2274
|
+
provenance: import_types.Provenance.FRONTIER
|
|
2275
|
+
});
|
|
2276
|
+
}
|
|
2228
2277
|
} else if (staleChain) {
|
|
2229
2278
|
const culprit = staleChain.culprit;
|
|
2230
2279
|
const culpritName = displayNameOf(culprit);
|
|
@@ -2288,6 +2337,9 @@ function fixRecommendationForTop(top, seedNode, legacy) {
|
|
|
2288
2337
|
return legacy.fixRecommendation;
|
|
2289
2338
|
}
|
|
2290
2339
|
const name = top.node.replace(/^service:/, "");
|
|
2340
|
+
if (top.classification === "symptom-only" && top.context.boundaryTimeout === true) {
|
|
2341
|
+
return `${name} timed out waiting on a downstream that hung and exported no span \u2014 the cause is unobservable from traces. Inspect the FRONTIER surface NEAT staged on the unobservable hop (see candidates); it graduates to OBSERVED when that service recovers. Restore instrumentation on that path to confirm, rather than treating ${name} as the fault.`;
|
|
2342
|
+
}
|
|
2291
2343
|
if (top.provenance === import_types.Provenance.STALE) {
|
|
2292
2344
|
return `Live telemetry for this path has gone quiet; the last-observed topology traces the failure downstream to ${name}. Restore instrumentation (or re-run with live traces) to confirm, then inspect ${name}.`;
|
|
2293
2345
|
}
|
|
@@ -6249,6 +6301,14 @@ function mergeColumnsAt(graph, tableNodeId, columns, provenance, confidence) {
|
|
|
6249
6301
|
function mergeObservedColumns(graph, tableNodeId, columns) {
|
|
6250
6302
|
mergeColumnsAt(graph, tableNodeId, columns, import_types8.Provenance.OBSERVED, OBSERVED_COLUMN_CONFIDENCE);
|
|
6251
6303
|
}
|
|
6304
|
+
function mergeObservedDeployState(graph, serviceNodeId, state) {
|
|
6305
|
+
if (!graph.hasNode(serviceNodeId)) return;
|
|
6306
|
+
const attrs = {};
|
|
6307
|
+
if (typeof state.image === "string" && state.image.length > 0) attrs["observedImage"] = state.image;
|
|
6308
|
+
if (typeof state.readyReplicas === "number") attrs["observedReadyReplicas"] = state.readyReplicas;
|
|
6309
|
+
if (Object.keys(attrs).length === 0) return;
|
|
6310
|
+
graph.mergeNodeAttributes(serviceNodeId, attrs);
|
|
6311
|
+
}
|
|
6252
6312
|
function ensureDatabaseNode(graph, host, engine) {
|
|
6253
6313
|
const id = (0, import_types8.databaseId)(host);
|
|
6254
6314
|
if (graph.hasNode(id)) return id;
|
|
@@ -6945,6 +7005,32 @@ function promoteFrontierNodes(graph, opts = {}) {
|
|
|
6945
7005
|
}
|
|
6946
7006
|
return promoted;
|
|
6947
7007
|
}
|
|
7008
|
+
function stageFrontierEdge(graph, source, target, type) {
|
|
7009
|
+
if (!graph.hasNode(source) || !graph.hasNode(target)) return false;
|
|
7010
|
+
if (graph.hasEdge((0, import_types8.observedEdgeId)(source, target, type))) return false;
|
|
7011
|
+
const key = (0, import_types8.frontierEdgeId)(source, target, type);
|
|
7012
|
+
if (graph.hasEdge(key)) return false;
|
|
7013
|
+
graph.addEdgeWithKey(key, source, target, {
|
|
7014
|
+
id: key,
|
|
7015
|
+
source,
|
|
7016
|
+
target,
|
|
7017
|
+
type,
|
|
7018
|
+
provenance: import_types8.Provenance.FRONTIER
|
|
7019
|
+
});
|
|
7020
|
+
return true;
|
|
7021
|
+
}
|
|
7022
|
+
function promoteFrontierEdges(graph) {
|
|
7023
|
+
const graduated = [];
|
|
7024
|
+
graph.forEachEdge((edgeId, attrs) => {
|
|
7025
|
+
const e = attrs;
|
|
7026
|
+
if (e.provenance !== import_types8.Provenance.FRONTIER) return;
|
|
7027
|
+
if (graph.hasEdge((0, import_types8.observedEdgeId)(e.source, e.target, e.type))) {
|
|
7028
|
+
graduated.push(edgeId);
|
|
7029
|
+
}
|
|
7030
|
+
});
|
|
7031
|
+
for (const edgeId of graduated) graph.dropEdge(edgeId);
|
|
7032
|
+
return graduated.length;
|
|
7033
|
+
}
|
|
6948
7034
|
function rewireFrontierEdges(graph, frontierId2, serviceId17) {
|
|
6949
7035
|
const inbound = [...graph.inboundEdges(frontierId2)];
|
|
6950
7036
|
const outbound = [...graph.outboundEdges(frontierId2)];
|
|
@@ -7050,6 +7136,7 @@ function startStalenessLoop(graph, options = {}) {
|
|
|
7050
7136
|
project: options.project
|
|
7051
7137
|
});
|
|
7052
7138
|
if (options.onPolicyTrigger) await options.onPolicyTrigger(graph);
|
|
7139
|
+
if (options.onReconcile) await options.onReconcile(graph);
|
|
7053
7140
|
} catch (err) {
|
|
7054
7141
|
console.error("staleness tick failed", err);
|
|
7055
7142
|
}
|
|
@@ -16968,6 +17055,25 @@ function detectColumnDrift(node) {
|
|
|
16968
17055
|
}
|
|
16969
17056
|
return out;
|
|
16970
17057
|
}
|
|
17058
|
+
var RECOMMENDATION_DEPLOY_IMAGE_MISMATCH = "The running image differs from the manifest-declared image \u2014 the rollout has not taken (the old ReplicaSet is still serving). Re-apply the deployment or investigate why the new image failed to roll out.";
|
|
17059
|
+
var DEPLOY_DIVERGENCE_CONFIDENCE = 0.9;
|
|
17060
|
+
function detectDeployDivergence(svc) {
|
|
17061
|
+
const out = [];
|
|
17062
|
+
if (typeof svc.declaredImage === "string" && typeof svc.observedImage === "string" && svc.declaredImage !== svc.observedImage) {
|
|
17063
|
+
out.push({
|
|
17064
|
+
type: "deploy-mismatch",
|
|
17065
|
+
kind: "image",
|
|
17066
|
+
source: svc.id,
|
|
17067
|
+
target: svc.id,
|
|
17068
|
+
declaredImage: svc.declaredImage,
|
|
17069
|
+
observedImage: svc.observedImage,
|
|
17070
|
+
confidence: DEPLOY_DIVERGENCE_CONFIDENCE,
|
|
17071
|
+
reason: `Service ${svc.name} declares image ${svc.declaredImage} but its running pods report ${svc.observedImage} \u2014 the deploy has not taken (the old version is still serving), so no incident fires.`,
|
|
17072
|
+
recommendation: RECOMMENDATION_DEPLOY_IMAGE_MISMATCH
|
|
17073
|
+
});
|
|
17074
|
+
}
|
|
17075
|
+
return out;
|
|
17076
|
+
}
|
|
16971
17077
|
var SYMBOL_MISMATCH_PATTERNS = [
|
|
16972
17078
|
{
|
|
16973
17079
|
// "'ListProductsResponse' object has no attribute 'products_list'" and kin.
|
|
@@ -17295,6 +17401,7 @@ function computeDivergences(graph, opts = {}) {
|
|
|
17295
17401
|
const svc = n;
|
|
17296
17402
|
for (const d of detectHostMismatch(graph, nodeId, svc)) all.push(d);
|
|
17297
17403
|
for (const d of detectCompatDivergences(graph, nodeId, svc)) all.push(d);
|
|
17404
|
+
for (const d of detectDeployDivergence(svc)) all.push(d);
|
|
17298
17405
|
return;
|
|
17299
17406
|
}
|
|
17300
17407
|
if (n.type === import_types60.NodeType.InfraNode && n.kind === "sql-table") {
|
|
@@ -17335,7 +17442,12 @@ function computeDivergences(graph, opts = {}) {
|
|
|
17335
17442
|
// orders last so at equal confidence the structural and symbol divergences
|
|
17336
17443
|
// lead, per the contract ("rank below the definitive structural and symbol
|
|
17337
17444
|
// divergences").
|
|
17338
|
-
"observed-failing": 6
|
|
17445
|
+
"observed-failing": 6,
|
|
17446
|
+
// Deploy mismatch (ADR-225) is a definitive structural declared-vs-observed
|
|
17447
|
+
// divergence carrying high confidence (0.9), so the confidence sort already
|
|
17448
|
+
// places it among the structural leaders; this slot only breaks an exact
|
|
17449
|
+
// confidence tie and sits last as a stable tiebreaker.
|
|
17450
|
+
"deploy-mismatch": 7
|
|
17339
17451
|
};
|
|
17340
17452
|
filtered.sort((a, b) => {
|
|
17341
17453
|
if (b.confidence !== a.confidence) return b.confidence - a.confidence;
|
|
@@ -18875,6 +18987,11 @@ async function runConnectorPoll(connector, ctx, graph, resolveTarget) {
|
|
|
18875
18987
|
unresolved++;
|
|
18876
18988
|
continue;
|
|
18877
18989
|
}
|
|
18990
|
+
if (signal.deployState) {
|
|
18991
|
+
ensureServiceNode(graph, resolved.serviceName, NO_ENV);
|
|
18992
|
+
mergeObservedDeployState(graph, resolved.targetNodeId, signal.deployState);
|
|
18993
|
+
continue;
|
|
18994
|
+
}
|
|
18878
18995
|
if (signal.incident) {
|
|
18879
18996
|
ensureServiceNode(graph, resolved.serviceName, NO_ENV);
|
|
18880
18997
|
if (!ctx.errorsPath || !graph.hasNode(resolved.targetNodeId)) {
|
|
@@ -23149,6 +23266,36 @@ async function buildApi(opts) {
|
|
|
23149
23266
|
// src/daemon.ts
|
|
23150
23267
|
init_otel();
|
|
23151
23268
|
|
|
23269
|
+
// src/hang-sensor.ts
|
|
23270
|
+
init_cjs_shims();
|
|
23271
|
+
var import_types101 = require("@neat.is/types");
|
|
23272
|
+
var RECONCILE_INCIDENT_LIMIT = 500;
|
|
23273
|
+
function stageHangSurfaces(graph, incidents) {
|
|
23274
|
+
const boundaries = /* @__PURE__ */ new Set();
|
|
23275
|
+
for (const ev of incidents) {
|
|
23276
|
+
if (ev.affectedNode && graph.hasNode(ev.affectedNode)) boundaries.add(ev.affectedNode);
|
|
23277
|
+
}
|
|
23278
|
+
let staged = 0;
|
|
23279
|
+
for (const boundary of boundaries) {
|
|
23280
|
+
const ctx = nodeContext(graph, boundary, incidents);
|
|
23281
|
+
if (!isBoundaryTimeoutSymptom(ctx)) continue;
|
|
23282
|
+
const hop = resolveHangHop(graph, boundary, incidents);
|
|
23283
|
+
if (!hop) continue;
|
|
23284
|
+
if (stageFrontierEdge(graph, hop.source, hop.target, import_types101.EdgeType.CALLS)) staged++;
|
|
23285
|
+
}
|
|
23286
|
+
return staged;
|
|
23287
|
+
}
|
|
23288
|
+
async function reconcileFrontierSurfaces(graph, errorsPath) {
|
|
23289
|
+
promoteFrontierEdges(graph);
|
|
23290
|
+
let incidents = [];
|
|
23291
|
+
try {
|
|
23292
|
+
incidents = await readErrorEvents(errorsPath, { limit: RECONCILE_INCIDENT_LIMIT });
|
|
23293
|
+
} catch {
|
|
23294
|
+
incidents = [];
|
|
23295
|
+
}
|
|
23296
|
+
stageHangSurfaces(graph, incidents);
|
|
23297
|
+
}
|
|
23298
|
+
|
|
23152
23299
|
// src/connectors/kubernetes/index.ts
|
|
23153
23300
|
init_cjs_shims();
|
|
23154
23301
|
|
|
@@ -23326,6 +23473,7 @@ var IMAGE_PULL_REASONS = /* @__PURE__ */ new Set(["ImagePullBackOff", "ErrImageP
|
|
|
23326
23473
|
var CRASH_LOOP_REASON = "CrashLoopBackOff";
|
|
23327
23474
|
var FIELD_SEP5 = "\0";
|
|
23328
23475
|
var K8S_TARGET_KIND = "k8s-workload";
|
|
23476
|
+
var K8S_DEPLOY_STATE = "deploy-state";
|
|
23329
23477
|
function packK8sTargetName(identity) {
|
|
23330
23478
|
return [identity.serviceName, identity.fault].join(FIELD_SEP5);
|
|
23331
23479
|
}
|
|
@@ -23457,18 +23605,47 @@ function mapDeploymentToSignal(deployment, pods, config) {
|
|
|
23457
23605
|
}
|
|
23458
23606
|
};
|
|
23459
23607
|
}
|
|
23608
|
+
function observedDeployState(deployment, pods) {
|
|
23609
|
+
const readyReplicas = typeof deployment.status?.readyReplicas === "number" ? deployment.status.readyReplicas : 0;
|
|
23610
|
+
let image;
|
|
23611
|
+
for (const pod of podsForDeployment(deployment, pods)) {
|
|
23612
|
+
for (const cs of pod.status?.containerStatuses ?? []) {
|
|
23613
|
+
if (cs.state?.running && typeof cs.image === "string" && cs.image.length > 0) {
|
|
23614
|
+
image = cs.image;
|
|
23615
|
+
break;
|
|
23616
|
+
}
|
|
23617
|
+
}
|
|
23618
|
+
if (image) break;
|
|
23619
|
+
}
|
|
23620
|
+
return image !== void 0 ? { image, readyReplicas } : { readyReplicas };
|
|
23621
|
+
}
|
|
23622
|
+
function deployStateSignal(deployment, pods, config) {
|
|
23623
|
+
const name = deployment.metadata?.name;
|
|
23624
|
+
if (typeof name !== "string" || name.length === 0) return null;
|
|
23625
|
+
const serviceName = serviceNameFor(deployment, config);
|
|
23626
|
+
return {
|
|
23627
|
+
targetKind: K8S_TARGET_KIND,
|
|
23628
|
+
targetName: packK8sTargetName({ serviceName, fault: K8S_DEPLOY_STATE }),
|
|
23629
|
+
callCount: 0,
|
|
23630
|
+
errorCount: 0,
|
|
23631
|
+
lastObservedIso: nowIso2(),
|
|
23632
|
+
deployState: observedDeployState(deployment, pods)
|
|
23633
|
+
};
|
|
23634
|
+
}
|
|
23460
23635
|
function mapWorkloadsToSignals(deployments, pods, config) {
|
|
23461
23636
|
const out = [];
|
|
23462
23637
|
for (const deployment of deployments) {
|
|
23463
|
-
const
|
|
23464
|
-
if (
|
|
23638
|
+
const deployState = deployStateSignal(deployment, pods, config);
|
|
23639
|
+
if (deployState) out.push(deployState);
|
|
23640
|
+
const incident = mapDeploymentToSignal(deployment, pods, config);
|
|
23641
|
+
if (incident) out.push(incident);
|
|
23465
23642
|
}
|
|
23466
23643
|
return out;
|
|
23467
23644
|
}
|
|
23468
23645
|
|
|
23469
23646
|
// src/connectors/kubernetes/resolve.ts
|
|
23470
23647
|
init_cjs_shims();
|
|
23471
|
-
var
|
|
23648
|
+
var import_types103 = require("@neat.is/types");
|
|
23472
23649
|
var NO_ENV3 = "unknown";
|
|
23473
23650
|
function createK8sResolveTarget(graph) {
|
|
23474
23651
|
return (signal) => {
|
|
@@ -23479,7 +23656,7 @@ function createK8sResolveTarget(graph) {
|
|
|
23479
23656
|
return {
|
|
23480
23657
|
targetNodeId: resolveFusedServiceId(graph, serviceName, NO_ENV3),
|
|
23481
23658
|
serviceName,
|
|
23482
|
-
edgeType:
|
|
23659
|
+
edgeType: import_types103.EdgeType.CALLS
|
|
23483
23660
|
};
|
|
23484
23661
|
};
|
|
23485
23662
|
}
|
|
@@ -23614,7 +23791,7 @@ function unroutedErrorsPath(neatHome4) {
|
|
|
23614
23791
|
}
|
|
23615
23792
|
|
|
23616
23793
|
// src/daemon.ts
|
|
23617
|
-
var
|
|
23794
|
+
var import_types106 = require("@neat.is/types");
|
|
23618
23795
|
function daemonJsonPath(scanPath) {
|
|
23619
23796
|
return import_node_path77.default.join(scanPath, "neat-out", "daemon.json");
|
|
23620
23797
|
}
|
|
@@ -23757,7 +23934,7 @@ function spanBelongsToSingleProject(graph, project, serviceName) {
|
|
|
23757
23934
|
if (!serviceName) return true;
|
|
23758
23935
|
if (serviceNameMatchesProject(serviceName, project)) return true;
|
|
23759
23936
|
return graph.someNode(
|
|
23760
|
-
(_id, attrs) => attrs.type ===
|
|
23937
|
+
(_id, attrs) => attrs.type === import_types106.NodeType.ServiceNode && attrs.name === serviceName
|
|
23761
23938
|
);
|
|
23762
23939
|
}
|
|
23763
23940
|
async function bootstrapProject(entry2, connectors = [], neatHome4) {
|
|
@@ -23801,7 +23978,11 @@ async function bootstrapProject(entry2, connectors = [], neatHome4) {
|
|
|
23801
23978
|
const stopPersist = startPersistLoop(graph, outPath, { exitOnSignal: false });
|
|
23802
23979
|
const stopStaleness = startStalenessLoop(graph, {
|
|
23803
23980
|
staleEventsPath: paths.staleEventsPath,
|
|
23804
|
-
project: entry2.name
|
|
23981
|
+
project: entry2.name,
|
|
23982
|
+
// Graduate FRONTIER surfaces whose OBSERVED twin has arrived + stage surfaces
|
|
23983
|
+
// for current hangs (ADR-226), each post-ingest tick — the periodic point
|
|
23984
|
+
// where OBSERVED state has settled and the errors ledger is current.
|
|
23985
|
+
onReconcile: (g) => reconcileFrontierSurfaces(g, paths.errorsPath)
|
|
23805
23986
|
});
|
|
23806
23987
|
const stopConnectors = await startConnectorPolling({
|
|
23807
23988
|
project: entry2.name,
|