@neat.is/core 0.7.10 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-6T7ZHODF.js → chunk-3PXRQH53.js} +420 -20
- package/dist/chunk-3PXRQH53.js.map +1 -0
- package/dist/{chunk-D4OX6MUX.js → chunk-OJ6BW63H.js} +2 -2
- package/dist/cli.cjs +417 -18
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2 -2
- package/dist/index.cjs +417 -18
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +4 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +2 -2
- package/dist/neatd.cjs +417 -18
- package/dist/neatd.cjs.map +1 -1
- package/dist/neatd.js +2 -2
- package/dist/server.cjs +417 -18
- package/dist/server.cjs.map +1 -1
- package/dist/server.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-6T7ZHODF.js.map +0 -1
- /package/dist/{chunk-D4OX6MUX.js.map → chunk-OJ6BW63H.js.map} +0 -0
package/dist/index.d.cts
CHANGED
|
@@ -444,8 +444,11 @@ declare function startStalenessLoop(graph: NeatGraph, options?: StalenessLoopOpt
|
|
|
444
444
|
declare function readErrorEvents(errorsPath: string): Promise<ErrorEvent[]>;
|
|
445
445
|
|
|
446
446
|
declare function confidenceForEdge(edge: GraphEdge, now?: number): number;
|
|
447
|
-
declare function getRootCause(graph: NeatGraph, errorNodeId: string, errorEvent?: ErrorEvent, incidents?: ErrorEvent[]): RootCauseResult | null;
|
|
448
447
|
declare function getBlastRadius(graph: NeatGraph, nodeId: string, maxDepth?: number): BlastRadiusResult;
|
|
448
|
+
declare function getRootCause(graph: NeatGraph, errorNodeId: string, errorEvent?: ErrorEvent, incidents?: ErrorEvent[], opts?: {
|
|
449
|
+
navigation?: boolean;
|
|
450
|
+
now?: number;
|
|
451
|
+
}): RootCauseResult | null;
|
|
449
452
|
|
|
450
453
|
interface PersistedNodeEntry {
|
|
451
454
|
key?: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -444,8 +444,11 @@ declare function startStalenessLoop(graph: NeatGraph, options?: StalenessLoopOpt
|
|
|
444
444
|
declare function readErrorEvents(errorsPath: string): Promise<ErrorEvent[]>;
|
|
445
445
|
|
|
446
446
|
declare function confidenceForEdge(edge: GraphEdge, now?: number): number;
|
|
447
|
-
declare function getRootCause(graph: NeatGraph, errorNodeId: string, errorEvent?: ErrorEvent, incidents?: ErrorEvent[]): RootCauseResult | null;
|
|
448
447
|
declare function getBlastRadius(graph: NeatGraph, nodeId: string, maxDepth?: number): BlastRadiusResult;
|
|
448
|
+
declare function getRootCause(graph: NeatGraph, errorNodeId: string, errorEvent?: ErrorEvent, incidents?: ErrorEvent[], opts?: {
|
|
449
|
+
navigation?: boolean;
|
|
450
|
+
now?: number;
|
|
451
|
+
}): RootCauseResult | null;
|
|
449
452
|
|
|
450
453
|
interface PersistedNodeEntry {
|
|
451
454
|
key?: string;
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
routeSpanToProject,
|
|
3
3
|
startDaemon
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-OJ6BW63H.js";
|
|
5
5
|
import {
|
|
6
6
|
ProjectNameCollisionError,
|
|
7
7
|
addProject,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
thresholdForEdgeType,
|
|
38
38
|
touchLastSeen,
|
|
39
39
|
writeAtomically
|
|
40
|
-
} from "./chunk-
|
|
40
|
+
} from "./chunk-3PXRQH53.js";
|
|
41
41
|
import {
|
|
42
42
|
startOtelGrpcReceiver
|
|
43
43
|
} from "./chunk-FCO5Z3RW.js";
|
package/dist/neatd.cjs
CHANGED
|
@@ -1400,7 +1400,7 @@ var rootCauseShapes = {
|
|
|
1400
1400
|
[import_types.NodeType.FileNode]: fileRootCauseShape,
|
|
1401
1401
|
[import_types.NodeType.SymbolNode]: symbolRootCauseShape
|
|
1402
1402
|
};
|
|
1403
|
-
function
|
|
1403
|
+
function legacyRootCause(graph, errorNodeId, errorEvent, incidents) {
|
|
1404
1404
|
if (!graph.hasNode(errorNodeId)) return null;
|
|
1405
1405
|
const origin = graph.getNodeAttributes(errorNodeId);
|
|
1406
1406
|
const shape = rootCauseShapes[origin.type];
|
|
@@ -1657,7 +1657,9 @@ function getObservedDependencies(graph, nodeId) {
|
|
|
1657
1657
|
dependencies: [],
|
|
1658
1658
|
observed: false,
|
|
1659
1659
|
inboundObservedCount: 0,
|
|
1660
|
-
hasExtractedOutbound: false
|
|
1660
|
+
hasExtractedOutbound: false,
|
|
1661
|
+
inboundVolume: 0,
|
|
1662
|
+
window: "lifetime"
|
|
1661
1663
|
});
|
|
1662
1664
|
}
|
|
1663
1665
|
const attrs = graph.getNodeAttributes(nodeId);
|
|
@@ -1688,11 +1690,19 @@ function getObservedDependencies(graph, nodeId) {
|
|
|
1688
1690
|
}
|
|
1689
1691
|
}
|
|
1690
1692
|
let inboundObservedCount = 0;
|
|
1693
|
+
let inboundVolume = 0;
|
|
1694
|
+
let inboundLastObserved;
|
|
1691
1695
|
for (const tgt of scope) {
|
|
1692
1696
|
for (const edgeId of graph.inboundEdges(tgt)) {
|
|
1693
1697
|
const e = graph.getEdgeAttributes(edgeId);
|
|
1694
1698
|
if (e.type === import_types.EdgeType.CONTAINS) continue;
|
|
1695
|
-
if (e.provenance === import_types.Provenance.OBSERVED)
|
|
1699
|
+
if (e.provenance === import_types.Provenance.OBSERVED) {
|
|
1700
|
+
inboundObservedCount += 1;
|
|
1701
|
+
inboundVolume += e.callCount ?? e.signal?.spanCount ?? 1;
|
|
1702
|
+
if (e.lastObserved && (!inboundLastObserved || e.lastObserved > inboundLastObserved)) {
|
|
1703
|
+
inboundLastObserved = e.lastObserved;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1696
1706
|
}
|
|
1697
1707
|
}
|
|
1698
1708
|
dependencies.sort(
|
|
@@ -1703,7 +1713,291 @@ function getObservedDependencies(graph, nodeId) {
|
|
|
1703
1713
|
dependencies,
|
|
1704
1714
|
observed: dependencies.length > 0 || inboundObservedCount > 0,
|
|
1705
1715
|
inboundObservedCount,
|
|
1706
|
-
hasExtractedOutbound
|
|
1716
|
+
hasExtractedOutbound,
|
|
1717
|
+
// The signal is cumulative, so the honest window label is "lifetime" (ADR-190).
|
|
1718
|
+
inboundVolume,
|
|
1719
|
+
window: "lifetime",
|
|
1720
|
+
...inboundLastObserved ? { inboundLastObserved } : {}
|
|
1721
|
+
});
|
|
1722
|
+
}
|
|
1723
|
+
var SATURATION_P95_MS = 1e3;
|
|
1724
|
+
function nodeScope(graph, nodeId) {
|
|
1725
|
+
const scope = [nodeId];
|
|
1726
|
+
if (!graph.hasNode(nodeId)) return scope;
|
|
1727
|
+
const attrs = graph.getNodeAttributes(nodeId);
|
|
1728
|
+
if (attrs.type === import_types.NodeType.ServiceNode) {
|
|
1729
|
+
for (const edgeId of graph.outboundEdges(nodeId)) {
|
|
1730
|
+
const e = graph.getEdgeAttributes(edgeId);
|
|
1731
|
+
if (e.type !== import_types.EdgeType.CONTAINS) continue;
|
|
1732
|
+
const owned = graph.getNodeAttributes(e.target);
|
|
1733
|
+
if (owned.type === import_types.NodeType.FileNode) scope.push(e.target);
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
return scope;
|
|
1737
|
+
}
|
|
1738
|
+
function incidentCountForNode(nodeId, incidents) {
|
|
1739
|
+
if (!incidents || incidents.length === 0) return 0;
|
|
1740
|
+
return incidents.filter((ev) => incidentMatchesNode(ev, nodeId)).length;
|
|
1741
|
+
}
|
|
1742
|
+
function nodeContext(graph, nodeId, incidents, now = Date.now()) {
|
|
1743
|
+
const scope = nodeScope(graph, nodeId);
|
|
1744
|
+
let errorsFromCallers = 0;
|
|
1745
|
+
let inboundVolume = 0;
|
|
1746
|
+
let outboundVolume = 0;
|
|
1747
|
+
let outboundErrors = 0;
|
|
1748
|
+
let latestInboundMs;
|
|
1749
|
+
let latencyP95Ms;
|
|
1750
|
+
let stale = false;
|
|
1751
|
+
for (const n of scope) {
|
|
1752
|
+
if (!graph.hasNode(n)) continue;
|
|
1753
|
+
for (const edgeId of graph.inboundEdges(n)) {
|
|
1754
|
+
const e = graph.getEdgeAttributes(edgeId);
|
|
1755
|
+
if (e.type === import_types.EdgeType.CONTAINS) continue;
|
|
1756
|
+
errorsFromCallers += e.signal?.errorCount ?? 0;
|
|
1757
|
+
inboundVolume += e.callCount ?? e.signal?.spanCount ?? 0;
|
|
1758
|
+
if (e.provenance === import_types.Provenance.STALE) stale = true;
|
|
1759
|
+
const p95 = e.signal?.latencyMs?.p95;
|
|
1760
|
+
if (p95 !== void 0) latencyP95Ms = Math.max(latencyP95Ms ?? 0, p95);
|
|
1761
|
+
if (e.lastObserved) {
|
|
1762
|
+
const t = Date.parse(e.lastObserved);
|
|
1763
|
+
if (Number.isFinite(t)) latestInboundMs = Math.max(latestInboundMs ?? 0, t);
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
for (const edgeId of graph.outboundEdges(n)) {
|
|
1767
|
+
const e = graph.getEdgeAttributes(edgeId);
|
|
1768
|
+
if (e.type === import_types.EdgeType.CONTAINS) continue;
|
|
1769
|
+
outboundVolume += e.callCount ?? e.signal?.spanCount ?? 0;
|
|
1770
|
+
if (e.type === import_types.EdgeType.CALLS) outboundErrors += e.signal?.errorCount ?? 0;
|
|
1771
|
+
if (e.provenance === import_types.Provenance.STALE) stale = true;
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
const errorsEmittedHere = incidentCountForNode(nodeId, incidents) + outboundErrors;
|
|
1775
|
+
const lastObservedAgeMs = latestInboundMs !== void 0 ? Math.max(0, now - latestInboundMs) : void 0;
|
|
1776
|
+
return {
|
|
1777
|
+
errorsEmittedHere,
|
|
1778
|
+
errorsFromCallers,
|
|
1779
|
+
callCount: inboundVolume,
|
|
1780
|
+
outboundVolume,
|
|
1781
|
+
...lastObservedAgeMs !== void 0 ? { lastObservedAgeMs } : {},
|
|
1782
|
+
...latencyP95Ms !== void 0 ? { latencyP95Ms } : {},
|
|
1783
|
+
stale
|
|
1784
|
+
};
|
|
1785
|
+
}
|
|
1786
|
+
function isSaturated(ctx) {
|
|
1787
|
+
return ctx.latencyP95Ms !== void 0 && ctx.latencyP95Ms >= SATURATION_P95_MS;
|
|
1788
|
+
}
|
|
1789
|
+
function classifyNode(ctx) {
|
|
1790
|
+
if (ctx.errorsEmittedHere > 0) {
|
|
1791
|
+
if ((ctx.stale || isSaturated(ctx)) && ctx.errorsFromCallers >= ctx.errorsEmittedHere) {
|
|
1792
|
+
return "symptom-only";
|
|
1793
|
+
}
|
|
1794
|
+
return "primary-failure";
|
|
1795
|
+
}
|
|
1796
|
+
if (ctx.errorsFromCallers > 0) return "symptom-only";
|
|
1797
|
+
return "unrelated";
|
|
1798
|
+
}
|
|
1799
|
+
function isVictimSeed(ctx) {
|
|
1800
|
+
return ctx.errorsFromCallers > 0 && (ctx.stale || isSaturated(ctx)) && ctx.errorsEmittedHere <= ctx.errorsFromCallers;
|
|
1801
|
+
}
|
|
1802
|
+
function grainOf(graph, nodeId) {
|
|
1803
|
+
if (!graph.hasNode(nodeId)) return "unknown";
|
|
1804
|
+
const t = graph.getNodeAttributes(nodeId).type;
|
|
1805
|
+
if (t === import_types.NodeType.ServiceNode) return "service";
|
|
1806
|
+
if (t === import_types.NodeType.FileNode) return "file";
|
|
1807
|
+
if (t === import_types.NodeType.SymbolNode) return "symbol";
|
|
1808
|
+
return t;
|
|
1809
|
+
}
|
|
1810
|
+
function findPath(graph, from, to, direction, maxDepth) {
|
|
1811
|
+
if (!graph.hasNode(from) || !graph.hasNode(to)) return null;
|
|
1812
|
+
if (from === to) return { nodes: [from], edges: [] };
|
|
1813
|
+
const queue = [{ nodeId: from, depth: 0, nodes: [from], edges: [] }];
|
|
1814
|
+
const enqueued = /* @__PURE__ */ new Set([from]);
|
|
1815
|
+
while (queue.length > 0) {
|
|
1816
|
+
const frame = queue.shift();
|
|
1817
|
+
if (frame.depth >= maxDepth) continue;
|
|
1818
|
+
const best = direction === "up" ? bestEdgeBySource(graph, graph.inboundEdges(frame.nodeId)) : bestEdgeByTarget(graph, graph.outboundEdges(frame.nodeId));
|
|
1819
|
+
const neighbours = [...best.entries()].sort((a, b) => a[0].localeCompare(b[0]));
|
|
1820
|
+
for (const [nid, edge] of neighbours) {
|
|
1821
|
+
if (nid === to) return { nodes: [...frame.nodes, nid], edges: [...frame.edges, edge] };
|
|
1822
|
+
if (enqueued.has(nid)) continue;
|
|
1823
|
+
enqueued.add(nid);
|
|
1824
|
+
queue.push({
|
|
1825
|
+
nodeId: nid,
|
|
1826
|
+
depth: frame.depth + 1,
|
|
1827
|
+
nodes: [...frame.nodes, nid],
|
|
1828
|
+
edges: [...frame.edges, edge]
|
|
1829
|
+
});
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
return null;
|
|
1833
|
+
}
|
|
1834
|
+
var EMPTY_CONTEXT = {
|
|
1835
|
+
errorsEmittedHere: 0,
|
|
1836
|
+
errorsFromCallers: 0,
|
|
1837
|
+
callCount: 0,
|
|
1838
|
+
outboundVolume: 0,
|
|
1839
|
+
stale: false
|
|
1840
|
+
};
|
|
1841
|
+
function expandNode(graph, nodeId, direction, incidents, now = Date.now()) {
|
|
1842
|
+
if (!graph.hasNode(nodeId)) {
|
|
1843
|
+
return import_types.ExpandResultSchema.parse({
|
|
1844
|
+
origin: nodeId,
|
|
1845
|
+
direction,
|
|
1846
|
+
node: { id: nodeId, classification: "unrelated", context: EMPTY_CONTEXT },
|
|
1847
|
+
neighbours: []
|
|
1848
|
+
});
|
|
1849
|
+
}
|
|
1850
|
+
const ctx = nodeContext(graph, nodeId, incidents, now);
|
|
1851
|
+
const best = direction === "up" ? bestEdgeBySource(graph, graph.inboundEdges(nodeId)) : bestEdgeByTarget(graph, graph.outboundEdges(nodeId));
|
|
1852
|
+
const neighbours = [];
|
|
1853
|
+
for (const [nid, edge] of best) {
|
|
1854
|
+
if (edge.type === import_types.EdgeType.CONTAINS) continue;
|
|
1855
|
+
const nctx = nodeContext(graph, nid, incidents, now);
|
|
1856
|
+
neighbours.push({
|
|
1857
|
+
node: nid,
|
|
1858
|
+
edgeType: edge.type,
|
|
1859
|
+
provenance: edge.provenance,
|
|
1860
|
+
classification: classifyNode(nctx),
|
|
1861
|
+
context: nctx
|
|
1862
|
+
});
|
|
1863
|
+
}
|
|
1864
|
+
neighbours.sort((a, b) => a.node.localeCompare(b.node));
|
|
1865
|
+
return import_types.ExpandResultSchema.parse({
|
|
1866
|
+
origin: nodeId,
|
|
1867
|
+
direction,
|
|
1868
|
+
node: { id: nodeId, classification: classifyNode(ctx), context: ctx },
|
|
1869
|
+
neighbours
|
|
1870
|
+
});
|
|
1871
|
+
}
|
|
1872
|
+
function relate(graph, a, b, maxDepth = ROOT_CAUSE_MAX_DEPTH) {
|
|
1873
|
+
const buildPath = (fp) => ({
|
|
1874
|
+
nodes: fp.nodes,
|
|
1875
|
+
edgeTypes: fp.edges.map((e) => e.type),
|
|
1876
|
+
provenance: fp.edges.map((e) => e.provenance),
|
|
1877
|
+
grain: fp.nodes.map((n) => grainOf(graph, n)),
|
|
1878
|
+
// The failure runs end to end when every hop carries error / latency / alert
|
|
1879
|
+
// signal — that is what turns reachability into cause-confirmation.
|
|
1880
|
+
carriesSignal: fp.edges.length > 0 && fp.edges.every(
|
|
1881
|
+
(e) => (e.signal?.errorCount ?? 0) > 0 || e.signal?.latencyMs !== void 0 || e.signal?.anomalous !== void 0
|
|
1882
|
+
)
|
|
1883
|
+
});
|
|
1884
|
+
if (!graph.hasNode(a) || !graph.hasNode(b)) {
|
|
1885
|
+
return import_types.RelateResultSchema.parse({
|
|
1886
|
+
a,
|
|
1887
|
+
b,
|
|
1888
|
+
related: false,
|
|
1889
|
+
direction: null,
|
|
1890
|
+
paths: [],
|
|
1891
|
+
note: !graph.hasNode(a) ? `node not found: ${a}` : `node not found: ${b}`
|
|
1892
|
+
});
|
|
1893
|
+
}
|
|
1894
|
+
const down = findPath(graph, a, b, "down", maxDepth);
|
|
1895
|
+
const up = findPath(graph, a, b, "up", maxDepth);
|
|
1896
|
+
if (!down && !up) {
|
|
1897
|
+
return import_types.RelateResultSchema.parse({
|
|
1898
|
+
a,
|
|
1899
|
+
b,
|
|
1900
|
+
related: false,
|
|
1901
|
+
direction: null,
|
|
1902
|
+
paths: [],
|
|
1903
|
+
note: `no path within ${maxDepth} hops`
|
|
1904
|
+
});
|
|
1905
|
+
}
|
|
1906
|
+
const paths = [];
|
|
1907
|
+
const direction = down ? "a->b" : "b->a";
|
|
1908
|
+
if (down) paths.push(buildPath(down));
|
|
1909
|
+
if (up) paths.push(buildPath(up));
|
|
1910
|
+
const endpointsFine = grainOf(graph, a) !== "service" && grainOf(graph, b) !== "service";
|
|
1911
|
+
const grainGap = endpointsFine && paths[0].grain.some((g) => g === "service");
|
|
1912
|
+
return import_types.RelateResultSchema.parse({
|
|
1913
|
+
a,
|
|
1914
|
+
b,
|
|
1915
|
+
related: true,
|
|
1916
|
+
direction,
|
|
1917
|
+
paths,
|
|
1918
|
+
...grainGap ? { grainGap: true } : {}
|
|
1919
|
+
});
|
|
1920
|
+
}
|
|
1921
|
+
function findLoadOrigin(graph, alertNodeId, incidents, now) {
|
|
1922
|
+
const upstream = getBlastRadius(graph, alertNodeId).affectedNodes.map((n) => n.nodeId);
|
|
1923
|
+
let best = null;
|
|
1924
|
+
for (const nid of upstream) {
|
|
1925
|
+
if (nid === alertNodeId) continue;
|
|
1926
|
+
const ctx = nodeContext(graph, nid, incidents, now);
|
|
1927
|
+
if (ctx.outboundVolume === 0) continue;
|
|
1928
|
+
const isSource = ctx.callCount === 0;
|
|
1929
|
+
const better = !best || (isSource !== best.isSource ? isSource : ctx.outboundVolume !== best.ctx.outboundVolume ? ctx.outboundVolume > best.ctx.outboundVolume : nid < best.node);
|
|
1930
|
+
if (better) best = { node: nid, ctx, isSource };
|
|
1931
|
+
}
|
|
1932
|
+
return best ? { node: best.node, ctx: best.ctx } : null;
|
|
1933
|
+
}
|
|
1934
|
+
function getRootCause(graph, errorNodeId, errorEvent, incidents, opts) {
|
|
1935
|
+
const legacy = legacyRootCause(graph, errorNodeId, errorEvent, incidents);
|
|
1936
|
+
if (!legacy) return null;
|
|
1937
|
+
const navigation = opts?.navigation ?? process.env.NEAT_RCA_NAVIGATION !== "0";
|
|
1938
|
+
if (!navigation) return legacy;
|
|
1939
|
+
return enrichWithNavigation(graph, errorNodeId, legacy, incidents, opts?.now ?? Date.now());
|
|
1940
|
+
}
|
|
1941
|
+
function enrichWithNavigation(graph, errorNodeId, legacy, incidents, now) {
|
|
1942
|
+
const seedNode = legacy.rootCauseNode;
|
|
1943
|
+
const seedCtx = graph.hasNode(seedNode) ? nodeContext(graph, seedNode, incidents, now) : null;
|
|
1944
|
+
const lastProv = legacy.edgeProvenances[legacy.edgeProvenances.length - 1];
|
|
1945
|
+
const candidates = [];
|
|
1946
|
+
if (seedCtx && isVictimSeed(seedCtx)) {
|
|
1947
|
+
const origin = findLoadOrigin(graph, errorNodeId, incidents, now);
|
|
1948
|
+
if (origin) {
|
|
1949
|
+
const path70 = findPath(graph, errorNodeId, origin.node, "up", ROOT_CAUSE_MAX_DEPTH);
|
|
1950
|
+
const originConfidence = confidenceFromMix(path70?.edges ?? [], now);
|
|
1951
|
+
candidates.push({
|
|
1952
|
+
node: origin.node,
|
|
1953
|
+
classification: "primary-failure",
|
|
1954
|
+
reason: `Highest-volume upstream source (${origin.ctx.outboundVolume} observed outbound calls) driving a saturated/stale subgraph; the alerting path decays downstream into a starved victim rather than a fault at the callee.`,
|
|
1955
|
+
context: origin.ctx,
|
|
1956
|
+
confidence: Math.max(0.3, Math.min(0.8, originConfidence || 0.5)),
|
|
1957
|
+
provenance: import_types.Provenance.OBSERVED
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
const staleNote = seedCtx.stale ? "; the node has gone STALE" : "";
|
|
1961
|
+
const satNote = isSaturated(seedCtx) ? `; inbound p95 ${Math.round(seedCtx.latencyP95Ms)}ms is saturated` : "";
|
|
1962
|
+
candidates.push({
|
|
1963
|
+
node: seedNode,
|
|
1964
|
+
classification: "symptom-only",
|
|
1965
|
+
reason: `Errors arrive from callers (${seedCtx.errorsFromCallers}) but none originate here${staleNote}${satNote} \u2014 a downstream victim of load, not the fault.`,
|
|
1966
|
+
context: seedCtx,
|
|
1967
|
+
confidence: Math.min(legacy.confidence, 0.4),
|
|
1968
|
+
...lastProv ? { provenance: lastProv } : {}
|
|
1969
|
+
});
|
|
1970
|
+
} else {
|
|
1971
|
+
candidates.push({
|
|
1972
|
+
node: seedNode,
|
|
1973
|
+
classification: "primary-failure",
|
|
1974
|
+
reason: legacy.rootCauseReason,
|
|
1975
|
+
context: seedCtx ?? EMPTY_CONTEXT,
|
|
1976
|
+
confidence: legacy.confidence,
|
|
1977
|
+
...lastProv ? { provenance: lastProv } : {}
|
|
1978
|
+
});
|
|
1979
|
+
}
|
|
1980
|
+
const top = candidates[0];
|
|
1981
|
+
let traversalPath = legacy.traversalPath;
|
|
1982
|
+
let edgeProvenances = legacy.edgeProvenances;
|
|
1983
|
+
if (top.node !== seedNode) {
|
|
1984
|
+
const path70 = findPath(graph, errorNodeId, top.node, "up", ROOT_CAUSE_MAX_DEPTH);
|
|
1985
|
+
if (path70) {
|
|
1986
|
+
traversalPath = path70.nodes;
|
|
1987
|
+
edgeProvenances = path70.edges.map((e) => e.provenance);
|
|
1988
|
+
} else {
|
|
1989
|
+
traversalPath = [errorNodeId, top.node];
|
|
1990
|
+
edgeProvenances = [top.provenance ?? import_types.Provenance.OBSERVED];
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
return import_types.RootCauseResultSchema.parse({
|
|
1994
|
+
rootCauseNode: top.node,
|
|
1995
|
+
rootCauseReason: top.reason,
|
|
1996
|
+
traversalPath,
|
|
1997
|
+
edgeProvenances,
|
|
1998
|
+
confidence: top.confidence,
|
|
1999
|
+
...legacy.fixRecommendation ? { fixRecommendation: legacy.fixRecommendation } : {},
|
|
2000
|
+
candidates
|
|
1707
2001
|
});
|
|
1708
2002
|
}
|
|
1709
2003
|
|
|
@@ -4586,6 +4880,63 @@ function columnIsObserved(col) {
|
|
|
4586
4880
|
return col.provenances.includes(import_types7.Provenance.OBSERVED);
|
|
4587
4881
|
}
|
|
4588
4882
|
|
|
4883
|
+
// src/latency-digest.ts
|
|
4884
|
+
init_cjs_shims();
|
|
4885
|
+
var SUB = 16;
|
|
4886
|
+
var MIN_EXP = -10;
|
|
4887
|
+
var MAX_EXP = 22;
|
|
4888
|
+
var OCTAVES = MAX_EXP - MIN_EXP + 1;
|
|
4889
|
+
var OVERFLOW_INDEX = OCTAVES * SUB + 1;
|
|
4890
|
+
function latencyBucketIndex(ms) {
|
|
4891
|
+
if (!Number.isFinite(ms) || ms <= 0) return 0;
|
|
4892
|
+
const e = Math.floor(Math.log2(ms));
|
|
4893
|
+
if (e < MIN_EXP) return 0;
|
|
4894
|
+
if (e > MAX_EXP) return OVERFLOW_INDEX;
|
|
4895
|
+
const base = 2 ** e;
|
|
4896
|
+
const raw = Math.floor((ms / base - 1) * SUB);
|
|
4897
|
+
const s = raw < 0 ? 0 : raw >= SUB ? SUB - 1 : raw;
|
|
4898
|
+
return (e - MIN_EXP) * SUB + s + 1;
|
|
4899
|
+
}
|
|
4900
|
+
function bucketRepresentativeMs(index) {
|
|
4901
|
+
if (index <= 0) return 0;
|
|
4902
|
+
if (index >= OVERFLOW_INDEX) return 2 ** (MAX_EXP + 1);
|
|
4903
|
+
const zeroBased = index - 1;
|
|
4904
|
+
const e = MIN_EXP + Math.floor(zeroBased / SUB);
|
|
4905
|
+
const s = zeroBased % SUB;
|
|
4906
|
+
const base = 2 ** e;
|
|
4907
|
+
const lower = base * (1 + s / SUB);
|
|
4908
|
+
const upper = base * (1 + (s + 1) / SUB);
|
|
4909
|
+
return (lower + upper) / 2;
|
|
4910
|
+
}
|
|
4911
|
+
function recordLatency(hist, ms) {
|
|
4912
|
+
const key = String(latencyBucketIndex(ms));
|
|
4913
|
+
hist[key] = (hist[key] ?? 0) + 1;
|
|
4914
|
+
return hist;
|
|
4915
|
+
}
|
|
4916
|
+
function quantile(hist, q) {
|
|
4917
|
+
const entries = Object.entries(hist).map(([k, c]) => [Number(k), c]).filter(([idx, c]) => Number.isFinite(idx) && c > 0).sort((a, b) => a[0] - b[0]);
|
|
4918
|
+
let total = 0;
|
|
4919
|
+
for (const [, c] of entries) total += c;
|
|
4920
|
+
if (total === 0) return 0;
|
|
4921
|
+
const rank = Math.max(1, Math.ceil(q * total));
|
|
4922
|
+
let cumulative = 0;
|
|
4923
|
+
for (const [idx, c] of entries) {
|
|
4924
|
+
cumulative += c;
|
|
4925
|
+
if (cumulative >= rank) return bucketRepresentativeMs(idx);
|
|
4926
|
+
}
|
|
4927
|
+
return bucketRepresentativeMs(entries[entries.length - 1][0]);
|
|
4928
|
+
}
|
|
4929
|
+
function round(ms) {
|
|
4930
|
+
return Math.round(ms * 100) / 100;
|
|
4931
|
+
}
|
|
4932
|
+
function latencyPercentiles(hist) {
|
|
4933
|
+
if (!hist) return void 0;
|
|
4934
|
+
let total = 0;
|
|
4935
|
+
for (const c of Object.values(hist)) total += c;
|
|
4936
|
+
if (total === 0) return void 0;
|
|
4937
|
+
return { p50: round(quantile(hist, 0.5)), p95: round(quantile(hist, 0.95)) };
|
|
4938
|
+
}
|
|
4939
|
+
|
|
4589
4940
|
// src/ingest.ts
|
|
4590
4941
|
var HOUR_MS = 60 * 60 * 1e3;
|
|
4591
4942
|
var DAY_MS = 24 * HOUR_MS;
|
|
@@ -5281,7 +5632,7 @@ function ensureFrontierNode(graph, host, ts) {
|
|
|
5281
5632
|
graph.addNode(id, node);
|
|
5282
5633
|
return id;
|
|
5283
5634
|
}
|
|
5284
|
-
function upsertObservedEdge(graph, type, source, target, ts, isError = false, evidence) {
|
|
5635
|
+
function upsertObservedEdge(graph, type, source, target, ts, isError = false, evidence, durationMs) {
|
|
5285
5636
|
if (!graph.hasNode(source) || !graph.hasNode(target)) return null;
|
|
5286
5637
|
const grain = source.startsWith("file:") || source.startsWith("symbol:") ? "file" : "service";
|
|
5287
5638
|
const id = makeObservedEdgeId(type, source, target);
|
|
@@ -5289,10 +5640,15 @@ function upsertObservedEdge(graph, type, source, target, ts, isError = false, ev
|
|
|
5289
5640
|
const existing = graph.getEdgeAttributes(id);
|
|
5290
5641
|
const newSpanCount = (existing.signal?.spanCount ?? existing.callCount ?? 0) + 1;
|
|
5291
5642
|
const newErrorCount = (existing.signal?.errorCount ?? 0) + (isError ? 1 : 0);
|
|
5643
|
+
const latencyHist2 = durationMs !== void 0 ? recordLatency({ ...existing.signal?.latencyHist ?? {} }, durationMs) : existing.signal?.latencyHist;
|
|
5644
|
+
const latencyMs2 = latencyPercentiles(latencyHist2) ?? existing.signal?.latencyMs;
|
|
5292
5645
|
const newSignal = {
|
|
5293
5646
|
spanCount: newSpanCount,
|
|
5294
5647
|
errorCount: newErrorCount,
|
|
5295
|
-
lastObservedAgeMs: 0
|
|
5648
|
+
lastObservedAgeMs: 0,
|
|
5649
|
+
...latencyHist2 ? { latencyHist: latencyHist2 } : {},
|
|
5650
|
+
...latencyMs2 ? { latencyMs: latencyMs2 } : {},
|
|
5651
|
+
...existing.signal?.anomalous !== void 0 ? { anomalous: existing.signal.anomalous } : {}
|
|
5296
5652
|
};
|
|
5297
5653
|
const updated = {
|
|
5298
5654
|
...existing,
|
|
@@ -5307,10 +5663,14 @@ function upsertObservedEdge(graph, type, source, target, ts, isError = false, ev
|
|
|
5307
5663
|
graph.replaceEdgeAttributes(id, updated);
|
|
5308
5664
|
return { edge: updated, created: false };
|
|
5309
5665
|
}
|
|
5666
|
+
const latencyHist = durationMs !== void 0 ? recordLatency({}, durationMs) : void 0;
|
|
5667
|
+
const latencyMs = latencyHist ? latencyPercentiles(latencyHist) : void 0;
|
|
5310
5668
|
const signal = {
|
|
5311
5669
|
spanCount: 1,
|
|
5312
5670
|
errorCount: isError ? 1 : 0,
|
|
5313
|
-
lastObservedAgeMs: 0
|
|
5671
|
+
lastObservedAgeMs: 0,
|
|
5672
|
+
...latencyHist ? { latencyHist } : {},
|
|
5673
|
+
...latencyMs ? { latencyMs } : {}
|
|
5314
5674
|
};
|
|
5315
5675
|
const edge = {
|
|
5316
5676
|
id,
|
|
@@ -5535,6 +5895,7 @@ async function handleSpan(ctx, span) {
|
|
|
5535
5895
|
}
|
|
5536
5896
|
const sourceId = ensureServiceNode(ctx.graph, span.service, env);
|
|
5537
5897
|
const isError = span.statusCode === 2;
|
|
5898
|
+
const durationMs = span.durationNanos > 0n ? Number(span.durationNanos) / 1e6 : void 0;
|
|
5538
5899
|
const sourceServiceNode = ctx.graph.getNodeAttributes(sourceId);
|
|
5539
5900
|
const callSite = callSiteFromSpan(span, sourceServiceNode, ctx.scanPath);
|
|
5540
5901
|
cacheSpanService(span, nowMs, callSite);
|
|
@@ -5573,7 +5934,8 @@ async function handleSpan(ctx, span) {
|
|
|
5573
5934
|
targetId,
|
|
5574
5935
|
ts,
|
|
5575
5936
|
isError,
|
|
5576
|
-
callSiteEvidence
|
|
5937
|
+
callSiteEvidence,
|
|
5938
|
+
durationMs
|
|
5577
5939
|
);
|
|
5578
5940
|
if (result) affectedNode = targetId;
|
|
5579
5941
|
if (span.dbSystem === "mongodb" && span.dbCollection) {
|
|
@@ -5585,7 +5947,8 @@ async function handleSpan(ctx, span) {
|
|
|
5585
5947
|
collectionId,
|
|
5586
5948
|
ts,
|
|
5587
5949
|
isError,
|
|
5588
|
-
callSiteEvidence
|
|
5950
|
+
callSiteEvidence,
|
|
5951
|
+
durationMs
|
|
5589
5952
|
);
|
|
5590
5953
|
}
|
|
5591
5954
|
if (span.dbTable) {
|
|
@@ -5597,7 +5960,8 @@ async function handleSpan(ctx, span) {
|
|
|
5597
5960
|
tableId,
|
|
5598
5961
|
ts,
|
|
5599
5962
|
isError,
|
|
5600
|
-
callSiteEvidence
|
|
5963
|
+
callSiteEvidence,
|
|
5964
|
+
durationMs
|
|
5601
5965
|
);
|
|
5602
5966
|
mergeObservedColumns(ctx.graph, tableId, span.dbColumns);
|
|
5603
5967
|
}
|
|
@@ -5616,7 +5980,8 @@ async function handleSpan(ctx, span) {
|
|
|
5616
5980
|
targetId,
|
|
5617
5981
|
ts,
|
|
5618
5982
|
isError,
|
|
5619
|
-
callSiteEvidence
|
|
5983
|
+
callSiteEvidence,
|
|
5984
|
+
durationMs
|
|
5620
5985
|
);
|
|
5621
5986
|
if (result) affectedNode = targetId;
|
|
5622
5987
|
} else if (span.graphqlOperationName && span.graphqlOperationType && spanServesGraphqlOperation(span.kind)) {
|
|
@@ -5633,7 +5998,8 @@ async function handleSpan(ctx, span) {
|
|
|
5633
5998
|
targetId,
|
|
5634
5999
|
ts,
|
|
5635
6000
|
isError,
|
|
5636
|
-
callSiteEvidence
|
|
6001
|
+
callSiteEvidence,
|
|
6002
|
+
durationMs
|
|
5637
6003
|
);
|
|
5638
6004
|
if (result) affectedNode = targetId;
|
|
5639
6005
|
} else if (span.rpcSystem === "grpc" && span.rpcService && span.rpcMethod && spanServesGrpcMethod(span.kind)) {
|
|
@@ -5645,7 +6011,8 @@ async function handleSpan(ctx, span) {
|
|
|
5645
6011
|
targetId,
|
|
5646
6012
|
ts,
|
|
5647
6013
|
isError,
|
|
5648
|
-
callSiteEvidence
|
|
6014
|
+
callSiteEvidence,
|
|
6015
|
+
durationMs
|
|
5649
6016
|
);
|
|
5650
6017
|
if (result) affectedNode = targetId;
|
|
5651
6018
|
} else if (span.websocketChannel && spanServesWebsocketChannel(span.kind)) {
|
|
@@ -5661,7 +6028,8 @@ async function handleSpan(ctx, span) {
|
|
|
5661
6028
|
targetId,
|
|
5662
6029
|
ts,
|
|
5663
6030
|
isError,
|
|
5664
|
-
callSiteEvidence
|
|
6031
|
+
callSiteEvidence,
|
|
6032
|
+
durationMs
|
|
5665
6033
|
);
|
|
5666
6034
|
if (result) affectedNode = targetId;
|
|
5667
6035
|
} else {
|
|
@@ -5677,7 +6045,8 @@ async function handleSpan(ctx, span) {
|
|
|
5677
6045
|
targetId,
|
|
5678
6046
|
ts,
|
|
5679
6047
|
isError,
|
|
5680
|
-
callSiteEvidence
|
|
6048
|
+
callSiteEvidence,
|
|
6049
|
+
durationMs
|
|
5681
6050
|
);
|
|
5682
6051
|
affectedNode = targetId;
|
|
5683
6052
|
resolvedViaAddress = true;
|
|
@@ -5690,7 +6059,8 @@ async function handleSpan(ctx, span) {
|
|
|
5690
6059
|
frontierNodeId,
|
|
5691
6060
|
ts,
|
|
5692
6061
|
isError,
|
|
5693
|
-
callSiteEvidence
|
|
6062
|
+
callSiteEvidence,
|
|
6063
|
+
durationMs
|
|
5694
6064
|
);
|
|
5695
6065
|
affectedNode = frontierNodeId;
|
|
5696
6066
|
resolvedViaAddress = true;
|
|
@@ -5716,7 +6086,8 @@ async function handleSpan(ctx, span) {
|
|
|
5716
6086
|
sourceId,
|
|
5717
6087
|
ts,
|
|
5718
6088
|
isError,
|
|
5719
|
-
fallbackEvidence
|
|
6089
|
+
fallbackEvidence,
|
|
6090
|
+
durationMs
|
|
5720
6091
|
);
|
|
5721
6092
|
}
|
|
5722
6093
|
}
|
|
@@ -5730,7 +6101,7 @@ async function handleSpan(ctx, span) {
|
|
|
5730
6101
|
);
|
|
5731
6102
|
if (routeNodeId) {
|
|
5732
6103
|
const routeSvc = ctx.graph.getNodeAttributes(routeNodeId).service;
|
|
5733
|
-
upsertObservedEdge(ctx.graph, import_types8.EdgeType.CONTAINS, (0, import_types8.serviceId)(routeSvc), routeNodeId, ts, isError);
|
|
6104
|
+
upsertObservedEdge(ctx.graph, import_types8.EdgeType.CONTAINS, (0, import_types8.serviceId)(routeSvc), routeNodeId, ts, isError, void 0, durationMs);
|
|
5734
6105
|
}
|
|
5735
6106
|
}
|
|
5736
6107
|
if (span.statusCode === 2) {
|
|
@@ -18415,6 +18786,34 @@ function registerRoutes(scope, ctx) {
|
|
|
18415
18786
|
}
|
|
18416
18787
|
return getBlastRadius(proj.graph, nodeId, depth);
|
|
18417
18788
|
});
|
|
18789
|
+
scope.get("/graph/expand/:nodeId", async (req2, reply) => {
|
|
18790
|
+
const proj = resolveProject(registry, req2, reply, ctx.bootstrap, ctx.singleProject);
|
|
18791
|
+
if (!proj) return;
|
|
18792
|
+
const { nodeId } = req2.params;
|
|
18793
|
+
if (!proj.graph.hasNode(nodeId)) {
|
|
18794
|
+
return reply.code(404).send({ error: "node not found", id: nodeId });
|
|
18795
|
+
}
|
|
18796
|
+
const direction = req2.query.direction;
|
|
18797
|
+
if (direction !== "up" && direction !== "down") {
|
|
18798
|
+
return reply.code(400).send({ error: 'direction must be "up" or "down"' });
|
|
18799
|
+
}
|
|
18800
|
+
const epath = errorsPathFor(proj);
|
|
18801
|
+
const incidents = epath ? await readErrorEvents(epath) : [];
|
|
18802
|
+
return expandNode(proj.graph, nodeId, direction, incidents);
|
|
18803
|
+
});
|
|
18804
|
+
scope.get("/graph/relate", async (req2, reply) => {
|
|
18805
|
+
const proj = resolveProject(registry, req2, reply, ctx.bootstrap, ctx.singleProject);
|
|
18806
|
+
if (!proj) return;
|
|
18807
|
+
const { a, b } = req2.query;
|
|
18808
|
+
if (!a || !b) {
|
|
18809
|
+
return reply.code(400).send({ error: "both a and b query params are required" });
|
|
18810
|
+
}
|
|
18811
|
+
const maxDepth = req2.query.maxDepth ? Number(req2.query.maxDepth) : void 0;
|
|
18812
|
+
if (maxDepth !== void 0 && (!Number.isFinite(maxDepth) || maxDepth < 1)) {
|
|
18813
|
+
return reply.code(400).send({ error: "maxDepth must be a positive integer" });
|
|
18814
|
+
}
|
|
18815
|
+
return relate(proj.graph, a, b, maxDepth);
|
|
18816
|
+
});
|
|
18418
18817
|
scope.get("/search", async (req2, reply) => {
|
|
18419
18818
|
const proj = resolveProject(registry, req2, reply, ctx.bootstrap, ctx.singleProject);
|
|
18420
18819
|
if (!proj) return;
|