@skyhook-io/radar-app 1.13.4 → 1.13.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/package.json +1 -1
  2. package/src/RadarApp.tsx +5 -0
  3. package/src/api/client.ts +4 -0
  4. package/src/api/diagnose.ts +61 -15
  5. package/src/components/ConnectionErrorView.test.tsx +30 -0
  6. package/src/components/ConnectionErrorView.tsx +31 -19
  7. package/src/components/diagnose/AgentCase.tsx +131 -0
  8. package/src/components/diagnose/DiagnoseSurface.test.tsx +0 -20
  9. package/src/components/diagnose/DiagnoseSurface.tsx +32 -195
  10. package/src/components/diagnose/InvestigationEvidencePane.test.tsx +783 -7
  11. package/src/components/diagnose/InvestigationEvidencePane.tsx +774 -133
  12. package/src/components/diagnose/InvestigationView.tsx +222 -5
  13. package/src/components/diagnose/diagnoseEvidenceTypes.ts +22 -1
  14. package/src/components/diagnose/investigationCase.test.tsx +1568 -0
  15. package/src/components/diagnose/investigationCase.ts +439 -0
  16. package/src/components/diagnose/investigationEvidence.test.ts +1566 -151
  17. package/src/components/diagnose/investigationEvidence.ts +831 -43
  18. package/src/components/diagnose/investigationEvidenceKinds.ts +218 -0
  19. package/src/components/diagnose/investigationEvidencePresentation.test.ts +31 -0
  20. package/src/components/diagnose/investigationEvidencePresentation.ts +1 -0
  21. package/src/components/diagnose/investigationMetrics.test.ts +712 -0
  22. package/src/components/diagnose/investigationMetrics.ts +393 -0
  23. package/src/components/diagnose/investigationSourceFocus.ts +2 -0
  24. package/src/components/diagnose/investigationState.test.ts +322 -0
  25. package/src/components/diagnose/investigationState.ts +147 -25
  26. package/src/components/diagnose/parts.test.tsx +482 -0
  27. package/src/components/diagnose/parts.tsx +418 -41
  28. package/src/components/resource/PrometheusChartsGrid.tsx +181 -79
  29. package/src/components/resources/PodFilePreview.test.tsx +131 -0
  30. package/src/components/resources/PodFilePreview.tsx +394 -0
  31. package/src/components/resources/PodFilesystemModal.tsx +157 -67
  32. package/src/context/DiagnoseCustomization.test.tsx +26 -0
  33. package/src/context/DiagnoseCustomization.tsx +14 -2
  34. package/src/index.ts +1 -0
  35. package/src/utils/shell-safe.test.ts +25 -1
  36. package/src/utils/shell-safe.ts +16 -0
@@ -6,6 +6,7 @@ import {
6
6
  investigationEvidenceSourceDomId,
7
7
  investigationEvidenceSourceId,
8
8
  investigationEvidenceSubjectRef,
9
+ metricsScope,
9
10
  projectInvestigationEvidence,
10
11
  resolveInvestigationRootCauseEvidence,
11
12
  type InvestigationEvidenceGroup,
@@ -560,7 +561,7 @@ describe("root-cause evidence resolution", () => {
560
561
  const projection = project([
561
562
  tool(
562
563
  "metrics-1",
563
- "query_prometheus",
564
+ "discover_metrics",
564
565
  { result: [1] },
565
566
  { evidenceRef: ref },
566
567
  ),
@@ -858,10 +859,10 @@ describe("semantic diagnose evidence projection", () => {
858
859
  expect(limitationText).toContain("selected 2 of 5");
859
860
  expect(limitationText).toContain("includes 2 of 8 selected log lines");
860
861
  expect(limitationText).toContain("not the container's full log history");
861
- expect(limitationText).toContain("Additional crash-cause");
862
+ expect(limitationText).toContain("part of the crash-cause candidates");
862
863
  expect(limitationText).toContain("received 1 of 3 event groups");
863
864
  expect(limitationText).toContain("rbac denied");
864
- expect(limitationText).toContain("Referenced-by relationships");
865
+ expect(limitationText).toContain("referenced-by relationships");
865
866
  expect(limitationText).toContain("recent-change result limit");
866
867
  expect(result.coverage).toEqual({
867
868
  attempted: 1,
@@ -2595,7 +2596,7 @@ describe("strict evidence adapters", () => {
2595
2596
 
2596
2597
  it("keeps unknown tools in Activity and limits invalid known contracts", () => {
2597
2598
  const result = project([
2598
- tool("other", "query_prometheus", { data: "ignored" }),
2599
+ tool("other", "discover_metrics", { data: "ignored" }),
2599
2600
  tool("bad-events", "get_events", { events: [{ nope: true }] }),
2600
2601
  ]);
2601
2602
  expect(result.sources.map((source) => source.stepId)).toEqual([
@@ -2848,10 +2849,10 @@ describe("honest zero and partial-result states", () => {
2848
2849
  group.latest.data.checked === "events",
2849
2850
  )?.latest,
2850
2851
  ).toMatchObject({
2851
- title: "No matching warning events",
2852
- data: {
2853
- message: "The warning-event query completed and returned no groups.",
2854
- },
2852
+ // The title and the scope beside it are the whole answer here; a body
2853
+ // would only restate them.
2854
+ title: "No warning events",
2855
+ data: { message: undefined },
2855
2856
  });
2856
2857
  });
2857
2858
 
@@ -3427,7 +3428,9 @@ describe("workload logs adapter", () => {
3427
3428
  expect(malformed.limitations[0].message).toContain(
3428
3429
  "couldn't summarize this investigation step",
3429
3430
  );
3430
- expect(malformed.limitations[2].message).toContain("3 resolved pods");
3431
+ expect(malformed.limitations[2].message).toContain(
3432
+ "Radar found 3 pods but got no logs from them",
3433
+ );
3431
3434
  });
3432
3435
  });
3433
3436
 
@@ -3742,7 +3745,7 @@ describe("prometheus rules adapter", () => {
3742
3745
  expect(firing[0].latest).toMatchObject({
3743
3746
  tier: "checked",
3744
3747
  relevance: "target",
3745
- title: "No firing alert rules name this Deployment",
3748
+ title: "No firing alerts matched this Deployment",
3746
3749
  summary: "Deployment shop/api",
3747
3750
  data: {
3748
3751
  type: "receipt",
@@ -4009,9 +4012,7 @@ describe("prometheus rules adapter", () => {
4009
4012
  ]),
4010
4013
  ).toHaveLength(0);
4011
4014
  expect(run([])).toHaveLength(1);
4012
- expect(run([])[0].latest.title).toBe(
4013
- "No firing alert rules name this Node",
4014
- );
4015
+ expect(run([])[0].latest.title).toBe("No firing alerts matched this Node");
4015
4016
  });
4016
4017
 
4017
4018
  it("keeps recording rules off the card list and flags truncation and evaluation health", () => {
@@ -4407,7 +4408,11 @@ describe("subject permissions adapter", () => {
4407
4408
  "perm",
4408
4409
  "get_subject_permissions",
4409
4410
  {
4410
- subject: { kind: "ServiceAccount", namespace: "shop", name: "api-sa" },
4411
+ subject: {
4412
+ kind: "ServiceAccount",
4413
+ namespace: "shop",
4414
+ name: "api-sa",
4415
+ },
4411
4416
  accessCheck: {
4412
4417
  verb: "get",
4413
4418
  resource: "secrets",
@@ -4424,7 +4429,9 @@ describe("subject permissions adapter", () => {
4424
4429
  const [group] = groupsOf(projection.groups, "permissions");
4425
4430
  expect(group.latest.title).toContain("Could not check whether");
4426
4431
  expect(group.latest.title).not.toContain("cannot get");
4427
- expect(group.latest.summary).toContain("webhook authorizer returned partial data");
4432
+ expect(group.latest.summary).toContain(
4433
+ "webhook authorizer returned partial data",
4434
+ );
4428
4435
  expect(group.latest.summary).not.toContain("No RBAC rule allows it");
4429
4436
  // The gap still reaches the coverage strip as well.
4430
4437
  expect(projection.limitations).toContainEqual(
@@ -4444,7 +4451,11 @@ describe("subject permissions adapter", () => {
4444
4451
  "perm",
4445
4452
  "get_subject_permissions",
4446
4453
  {
4447
- subject: { kind: "ServiceAccount", namespace: "shop", name: "api-sa" },
4454
+ subject: {
4455
+ kind: "ServiceAccount",
4456
+ namespace: "shop",
4457
+ name: "api-sa",
4458
+ },
4448
4459
  accessCheck: {
4449
4460
  verb: "get",
4450
4461
  resource: "secrets",
@@ -4987,169 +4998,1573 @@ describe("card leads and deep-link subjects", () => {
4987
4998
  });
4988
4999
  });
4989
5000
 
4990
- describe("live-run follow-ups", () => {
4991
- it("reads the producer's nil-slice null as an empty events or changes result", () => {
5001
+ describe("query_prometheus evidence", () => {
5002
+ const rangeSeries = [
5003
+ {
5004
+ labels: { pod: "api-7f6-abc" },
5005
+ dataPoints: [
5006
+ { timestamp: 1_788_679_043, value: 169_377_792 },
5007
+ { timestamp: 1_788_679_068, value: 171_401_216 },
5008
+ ],
5009
+ },
5010
+ ];
5011
+ // The pods a diagnose bundle established as the target's own, so an agent
5012
+ // query naming exactly them is target evidence. A prefix over the same
5013
+ // namespace is not: "api-.*" also selects api-worker's pods.
5014
+ const establishedPods = ["api-7f6-abc", "api-7f6-def"];
5015
+ const podSetPattern = `^(${establishedPods.join("|")})$`;
5016
+ const targetSelectors = [
5017
+ {
5018
+ metric: "container_memory_working_set_bytes",
5019
+ matchers: [
5020
+ { label: "namespace", op: "=", value: "shop" },
5021
+ { label: "pod", op: "=~", value: podSetPattern },
5022
+ ],
5023
+ },
5024
+ ];
5025
+ const prefixSelectors = [
5026
+ {
5027
+ metric: "container_memory_working_set_bytes",
5028
+ matchers: [
5029
+ { label: "namespace", op: "=", value: "shop" },
5030
+ { label: "pod", op: "=~", value: "api-.*" },
5031
+ ],
5032
+ },
5033
+ ];
5034
+ // A diagnose of the target that lists its pods, so a later query can be
5035
+ // proved to be about them.
5036
+ function podsBundle() {
5037
+ return tool(
5038
+ "diag-pods",
5039
+ "diagnose",
5040
+ { resource: deployment, pods: 2, podNames: establishedPods },
5041
+ {
5042
+ summary: JSON.stringify({
5043
+ kind: "Deployment",
5044
+ namespace: "shop",
5045
+ name: "api",
5046
+ }),
5047
+ },
5048
+ );
5049
+ }
5050
+ function promResult(patch: Record<string, unknown> = {}) {
5051
+ return {
5052
+ query: `sum(container_memory_working_set_bytes{namespace="shop",pod=~"${podSetPattern}"})`,
5053
+ type: "range",
5054
+ start: "2026-09-06T07:17:23Z",
5055
+ end: "2026-09-06T09:17:23Z",
5056
+ step: "25s",
5057
+ resultType: "matrix",
5058
+ seriesCount: 1,
5059
+ series: rangeSeries,
5060
+ selectors: targetSelectors,
5061
+ ...patch,
5062
+ };
5063
+ }
5064
+
5065
+ it("keeps an unrecognised metric's own name as the title", () => {
4992
5066
  const projection = project([
4993
5067
  tool(
4994
- "events-none",
4995
- "get_events",
4996
- { events: null },
4997
- {
4998
- summary: JSON.stringify({
4999
- kind: "Pod",
5000
- namespace: "shop",
5001
- name: "api",
5002
- }),
5003
- },
5068
+ "prom",
5069
+ "query_prometheus",
5070
+ promResult({
5071
+ query: 'sum(acme_widget_queue_depth{namespace="shop"})',
5072
+ selectors: [
5073
+ {
5074
+ metric: "acme_widget_queue_depth",
5075
+ matchers: [{ label: "namespace", op: "=", value: "shop" }],
5076
+ },
5077
+ ],
5078
+ }),
5004
5079
  ),
5080
+ ]);
5081
+ const [group] = groupsOf(projection.groups, "metrics");
5082
+ expect(group.latest.title).toBe("acme_widget_queue_depth");
5083
+ // No family label means the summary keeps its generic lead-in rather than
5084
+ // repeating the name already in the title.
5085
+ expect(group.latest.summary).toContain("Prometheus · 1 series");
5086
+ });
5087
+
5088
+ it("keeps the generic title when the selectors name no metric", () => {
5089
+ const projection = project([
5005
5090
  tool(
5006
- "changes-none",
5007
- "get_changes",
5008
- { changes: null },
5009
- {
5010
- summary: JSON.stringify({
5011
- kind: "Deployment",
5012
- namespace: "shop",
5013
- name: "api",
5014
- since: "24h",
5015
- }),
5016
- },
5091
+ "prom",
5092
+ "query_prometheus",
5093
+ promResult({
5094
+ query: 'sum({namespace="shop"})',
5095
+ selectors: [
5096
+ {
5097
+ metric: "",
5098
+ matchers: [{ label: "namespace", op: "=", value: "shop" }],
5099
+ },
5100
+ ],
5101
+ }),
5017
5102
  ),
5018
5103
  ]);
5019
- // A complete, successful events query that returned nothing is a checked
5020
- // receipt for its scope; the producer marks a denied namespace separately.
5021
- expect(
5022
- projection.groups.map((group) => [
5023
- group.latest.data.type,
5024
- group.latest.title,
5025
- ]),
5026
- ).toEqual([["receipt", "No events matched"]]);
5027
- expect(
5028
- projection.limitations.map((limitation) => [
5029
- limitation.source,
5030
- limitation.kind,
5031
- ]),
5032
- ).toEqual([["Recent changes", "unknown"]]);
5104
+ const [group] = groupsOf(projection.groups, "metrics");
5105
+ expect(group.latest.title).toBe("Prometheus metrics");
5106
+ expect(group.latest.summary).toBe("1 series · 2h window · 25s step");
5033
5107
  });
5034
5108
 
5035
- it("merges identical startup blockers across pods into one card with the pod list", () => {
5036
- const blocker = (
5037
- name: string,
5038
- message = "1 node(s) no free host ports",
5039
- ) => ({
5040
- kind: "Pod",
5041
- name,
5042
- reason: "Unschedulable",
5043
- severity: "critical",
5044
- message,
5045
- });
5109
+ it("charts a range query scoped to the target as target evidence", () => {
5046
5110
  const projection = project([
5047
- tool("diagnose", "diagnose", {
5048
- resource: {
5049
- apiVersion: "apps/v1",
5050
- kind: "DaemonSet",
5051
- metadata: { namespace: "opencost", name: "node-exporter" },
5052
- },
5053
- resourceContext: { tier: "basic" },
5054
- pods: 3,
5055
- startupBlockers: [
5056
- blocker("node-exporter-a"),
5057
- blocker("node-exporter-b"),
5058
- blocker("node-exporter-c", "0/10 nodes: insufficient memory"),
5059
- ],
5111
+ podsBundle(),
5112
+ tool("prom", "query_prometheus", promResult(), {
5113
+ summary: JSON.stringify({ query: "sum(...)", type: "range" }),
5060
5114
  }),
5061
5115
  ]);
5062
- const startup = projection.groups.filter(
5063
- (group) => group.latest.data.type === "startup",
5116
+ const [group] = groupsOf(projection.groups, "metrics");
5117
+ expect(group).toBeDefined();
5118
+ expect(group.latest.relevance).toBe("target");
5119
+ expect(group.latest.tier).toBe("supporting");
5120
+ expect(group.latest.tone).toBe("neutral");
5121
+ expect(group.latest.title).toBe("Memory working set · Deployment shop/api");
5122
+ expect(group.latest.summary).toBe(
5123
+ "container_memory_working_set_bytes · 1 series · 2h window · 25s step",
5064
5124
  );
5065
- expect(
5066
- startup.map((group) => [
5067
- group.latest.summary,
5068
- group.latest.data.type === "startup" ? group.latest.data.pods : null,
5069
- ]),
5070
- ).toEqual([
5071
- [
5072
- "2 pods · 1 node(s) no free host ports",
5073
- ["node-exporter-a", "node-exporter-b"],
5074
- ],
5075
- ["0/10 nodes: insufficient memory", undefined],
5125
+ const data = group.latest.data;
5126
+ if (data.type !== "metrics") throw new Error("expected metrics");
5127
+ expect(data.mode).toBe("range");
5128
+ expect(data.origin).toBe("query");
5129
+ expect(data.subject).toEqual({
5130
+ kind: "Deployment",
5131
+ group: "apps",
5132
+ namespace: "shop",
5133
+ name: "api",
5134
+ });
5135
+ expect(data.unit).toBe("bytes");
5136
+ expect(data.series).toEqual(rangeSeries);
5137
+ expect(investigationEvidenceSubjectRef(data)).toEqual(data.subject);
5138
+ // The chart and the diagnose bundle that established its pods.
5139
+ expect(projection.coverage.projected).toBe(2);
5140
+ });
5141
+
5142
+ it("claims a unit only for a single bare metric that states one", () => {
5143
+ const projection = project([
5144
+ tool(
5145
+ "bare",
5146
+ "query_prometheus",
5147
+ promResult({
5148
+ query:
5149
+ 'container_memory_working_set_bytes{namespace="shop",pod=~"api-.*"}',
5150
+ }),
5151
+ ),
5152
+ tool(
5153
+ "rate",
5154
+ "query_prometheus",
5155
+ promResult({
5156
+ query:
5157
+ 'rate(container_cpu_usage_seconds_total{namespace="shop",pod=~"api-.*"}[5m])',
5158
+ selectors: [
5159
+ {
5160
+ metric: "container_cpu_usage_seconds_total",
5161
+ matchers: targetSelectors[0].matchers,
5162
+ },
5163
+ ],
5164
+ }),
5165
+ ),
5076
5166
  ]);
5077
- // A single pod keeps the per-pod identity so saved runs still match.
5078
- expect(startup[1].id).not.toBe(startup[0].id);
5167
+ const units = groupsOf(projection.groups, "metrics").map((group) =>
5168
+ group.latest.data.type === "metrics" ? group.latest.data.unit : "?",
5169
+ );
5170
+ expect(units).toEqual(["bytes", ""]);
5079
5171
  });
5080
5172
 
5081
- it("folds pods' startup blocker into the classified issue that repeats it", () => {
5082
- const blocker = (name: string) => ({
5083
- kind: "Pod",
5084
- name,
5085
- reason: "Unschedulable",
5086
- severity: "critical",
5087
- message: "1 node(s) no free host ports",
5088
- });
5173
+ it("renders an instant query as values, not a chart", () => {
5089
5174
  const projection = project([
5090
- tool("diagnose", "diagnose", {
5091
- resource: {
5092
- apiVersion: "apps/v1",
5093
- kind: "DaemonSet",
5094
- metadata: { namespace: "opencost", name: "node-exporter" },
5095
- },
5096
- resourceContext: { tier: "basic" },
5097
- pods: 2,
5098
- relatedIssues: [
5099
- {
5100
- id: "issue-unsched",
5101
- severity: "critical",
5102
- source: "scheduling",
5103
- category: "unschedulable",
5104
- category_group: "startup",
5105
- grouping_scope: "workload",
5106
- kind: "DaemonSet",
5107
- namespace: "opencost",
5108
- name: "node-exporter",
5109
- reason: "Unschedulable",
5110
- message: "1 node(s) no free host ports",
5111
- },
5112
- ],
5113
- startupBlockers: [
5114
- blocker("node-exporter-a"),
5115
- blocker("node-exporter-b"),
5116
- ],
5117
- }),
5175
+ tool(
5176
+ "instant",
5177
+ "query_prometheus",
5178
+ promResult({
5179
+ type: "instant",
5180
+ start: undefined,
5181
+ end: undefined,
5182
+ step: undefined,
5183
+ resultType: "vector",
5184
+ series: [
5185
+ {
5186
+ labels: { pod: "api-7f6-abc" },
5187
+ dataPoints: [{ timestamp: 1_788_679_068, value: 3 }],
5188
+ },
5189
+ ],
5190
+ }),
5191
+ ),
5118
5192
  ]);
5119
- expect(
5120
- projection.groups.filter((group) => group.latest.data.type === "startup"),
5121
- ).toHaveLength(0);
5122
- const [issue] = projection.groups.filter(
5123
- (group) => group.latest.data.type === "issue",
5193
+ const [group] = groupsOf(projection.groups, "metrics");
5194
+ expect(group.latest.title).toBe("Memory working set");
5195
+ expect(group.latest.summary).toBe(
5196
+ "container_memory_working_set_bytes · 1 series",
5197
+ );
5198
+ expect(group.latest.data.type === "metrics" && group.latest.data.mode).toBe(
5199
+ "instant",
5124
5200
  );
5125
- expect(
5126
- issue.latest.data.type === "issue" && issue.latest.data.pods,
5127
- ).toEqual(["node-exporter-a", "node-exporter-b"]);
5128
5201
  });
5129
5202
 
5130
- it("files a denied events namespace as an access limitation, not a receipt", () => {
5203
+ it("reports a truncated result as a limitation with its cardinality, never as a chart", () => {
5131
5204
  const projection = project([
5132
5205
  tool(
5133
- "events-denied",
5134
- "get_events",
5135
- { events: [], accessDenied: true },
5136
- {
5137
- summary: JSON.stringify({ namespace: "locked", kind: "Pod" }),
5138
- },
5206
+ "big",
5207
+ "query_prometheus",
5208
+ promResult({
5209
+ series: [],
5210
+ truncated: true,
5211
+ summary: {
5212
+ seriesCount: 812,
5213
+ totalDataPoints: 194_880,
5214
+ labelCardinality: { pod: 812, container: 3 },
5215
+ suggestion: "topk(5, ...)",
5216
+ },
5217
+ note: "Result exceeded the 96 KiB cap.",
5218
+ }),
5139
5219
  ),
5140
5220
  ]);
5141
- expect(projection.groups).toHaveLength(0);
5142
- expect(
5143
- projection.limitations.map((limitation) => [
5144
- limitation.source,
5145
- limitation.kind,
5146
- ]),
5147
- ).toEqual([["Events", "error"]]);
5221
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
5222
+ expect(projection.limitations).toHaveLength(1);
5223
+ expect(projection.limitations[0].kind).toBe("truncated");
5224
+ expect(projection.limitations[0].message).toContain("812 series");
5148
5225
  expect(projection.limitations[0].message).toContain(
5149
- "not readable with your permissions",
5226
+ "812 distinct pod values",
5227
+ );
5228
+ expect(projection.limitations[0].message).toContain("Result exceeded");
5229
+ expect(projection.coverage.limited).toBe(1);
5230
+ });
5231
+
5232
+ it("keeps a query that matched nothing as a fact about the window", () => {
5233
+ const projection = project([
5234
+ tool("none", "query_prometheus", promResult({ series: [] })),
5235
+ ]);
5236
+ const [group] = groupsOf(projection.groups, "metrics");
5237
+ expect(group.latest.summary).toBe(
5238
+ "container_memory_working_set_bytes · No series matched · 2h window · 25s step",
5150
5239
  );
5151
5240
  });
5152
5241
 
5242
+ it("classifies scope from the producer's selectors, never from the expression text", () => {
5243
+ const withNamespace = (
5244
+ metric: string,
5245
+ extra: Array<{ label: string; op: string; value: string }> = [],
5246
+ ) => ({
5247
+ metric,
5248
+ matchers: [{ label: "namespace", op: "=", value: "shop" }, ...extra],
5249
+ });
5250
+ const cases: Array<[string, Record<string, unknown>, string]> = [
5251
+ ["namespace only", { selectors: [withNamespace("up")] }, "broader"],
5252
+ [
5253
+ "workload label",
5254
+ {
5255
+ selectors: [
5256
+ withNamespace("kube_deployment_status_replicas_available", [
5257
+ { label: "deployment", op: "=", value: "api" },
5258
+ ]),
5259
+ ],
5260
+ },
5261
+ "target",
5262
+ ],
5263
+ [
5264
+ "pod named by the workload's prefix, never established",
5265
+ {
5266
+ selectors: [
5267
+ withNamespace("kube_pod_container_status_restarts_total", [
5268
+ { label: "pod", op: "=", value: "api-7f6-abc" },
5269
+ ]),
5270
+ ],
5271
+ },
5272
+ "producer-related",
5273
+ ],
5274
+ [
5275
+ "owner join over the namespace",
5276
+ {
5277
+ selectors: [
5278
+ withNamespace("kube_pod_owner", [
5279
+ { label: "owner_kind", op: "=", value: "ReplicaSet" },
5280
+ { label: "owner_name", op: "=~", value: "^(api-7f6)$" },
5281
+ ]),
5282
+ ],
5283
+ },
5284
+ "producer-related",
5285
+ ],
5286
+ [
5287
+ "container alone",
5288
+ {
5289
+ selectors: [
5290
+ withNamespace("container_memory_working_set_bytes", [
5291
+ { label: "container", op: "=", value: "api" },
5292
+ ]),
5293
+ ],
5294
+ },
5295
+ "broader",
5296
+ ],
5297
+ [
5298
+ "sibling workload",
5299
+ {
5300
+ selectors: [
5301
+ withNamespace("kube_deployment_status_replicas_available", [
5302
+ { label: "deployment", op: "=", value: "worker" },
5303
+ ]),
5304
+ ],
5305
+ },
5306
+ "producer-related",
5307
+ ],
5308
+ [
5309
+ "pod regex with an optional dash",
5310
+ {
5311
+ selectors: [
5312
+ withNamespace("kube_pod_container_status_restarts_total", [
5313
+ { label: "pod", op: "=~", value: "api-?.*" },
5314
+ ]),
5315
+ ],
5316
+ },
5317
+ "producer-related",
5318
+ ],
5319
+ [
5320
+ "pod regex alternation behind the target prefix",
5321
+ {
5322
+ selectors: [
5323
+ withNamespace("kube_pod_container_status_restarts_total", [
5324
+ { label: "pod", op: "=~", value: "api-.*|worker-.*" },
5325
+ ]),
5326
+ ],
5327
+ },
5328
+ "producer-related",
5329
+ ],
5330
+ [
5331
+ "workload alternation",
5332
+ {
5333
+ selectors: [
5334
+ withNamespace("kube_deployment_status_replicas_available", [
5335
+ { label: "deployment", op: "=~", value: "api|worker" },
5336
+ ]),
5337
+ ],
5338
+ },
5339
+ "producer-related",
5340
+ ],
5341
+ [
5342
+ "statefulset label on a deployment target",
5343
+ {
5344
+ selectors: [
5345
+ withNamespace("kube_statefulset_status_replicas_ready", [
5346
+ { label: "statefulset", op: "=", value: "api" },
5347
+ ]),
5348
+ ],
5349
+ },
5350
+ "producer-related",
5351
+ ],
5352
+ [
5353
+ "braced target numerator over a bare cluster denominator",
5354
+ {
5355
+ query:
5356
+ 'sum(container_memory_working_set_bytes{namespace="shop",pod=~"api-.*"}) / scalar(sum(machine_memory_bytes))',
5357
+ selectors: [
5358
+ ...targetSelectors,
5359
+ { metric: "machine_memory_bytes", matchers: [] },
5360
+ ],
5361
+ },
5362
+ "broader",
5363
+ ],
5364
+ [
5365
+ "other namespace",
5366
+ {
5367
+ selectors: [
5368
+ {
5369
+ metric: "up",
5370
+ matchers: [
5371
+ { label: "namespace", op: "=", value: "other" },
5372
+ { label: "pod", op: "=~", value: "api-.*" },
5373
+ ],
5374
+ },
5375
+ ],
5376
+ },
5377
+ "broader",
5378
+ ],
5379
+ ["empty selectors", { selectors: [] }, "broader"],
5380
+ [
5381
+ "unknown selectors",
5382
+ { selectors: targetSelectors, selectorsUnknown: true },
5383
+ "broader",
5384
+ ],
5385
+ ];
5386
+ for (const [name, patch, expected] of cases) {
5387
+ const projection = project([
5388
+ tool(`case-${name}`, "query_prometheus", promResult(patch)),
5389
+ ]);
5390
+ const [group] = groupsOf(projection.groups, "metrics");
5391
+ expect(group?.latest.relevance, name).toBe(expected);
5392
+ expect(
5393
+ group?.latest.data.type === "metrics"
5394
+ ? group.latest.data.subject
5395
+ : "missing",
5396
+ name,
5397
+ ).toEqual(
5398
+ expected === "target"
5399
+ ? {
5400
+ kind: "Deployment",
5401
+ group: "apps",
5402
+ namespace: "shop",
5403
+ name: "api",
5404
+ }
5405
+ : undefined,
5406
+ );
5407
+ expect(group?.latest.tier, name).toBe(
5408
+ expected === "broader" ? "context" : "supporting",
5409
+ );
5410
+ }
5411
+ });
5412
+
5413
+ // The diagnose prompt tells the agent to scope a query as `pod=~'^(a|b)$'`.
5414
+ // A Pod target has to recognise its own name written that way, or following
5415
+ // our own instruction demotes the target's own series.
5416
+ it("accepts the anchored set form the prompt asks for on a Pod target", () => {
5417
+ const podTarget = {
5418
+ kind: "Pod",
5419
+ group: "",
5420
+ namespace: "shop",
5421
+ name: "api-7f6-abc",
5422
+ };
5423
+ const podSelector = (value: string) => [
5424
+ {
5425
+ metric: "container_cpu_usage_seconds_total",
5426
+ matchers: [
5427
+ { label: "namespace", op: "=", value: "shop" },
5428
+ { label: "pod", op: "=~", value },
5429
+ ],
5430
+ },
5431
+ ];
5432
+ expect(metricsScope(podTarget, podSelector("api-7f6-abc"), false)).toBe(
5433
+ "target",
5434
+ );
5435
+ expect(metricsScope(podTarget, podSelector("^(api-7f6-abc)$"), false)).toBe(
5436
+ "target",
5437
+ );
5438
+ // A set that also names a neighbour is not evidence about this pod alone,
5439
+ // and a prefix still selects pods this one does not stand for.
5440
+ expect(
5441
+ metricsScope(
5442
+ podTarget,
5443
+ podSelector("^(api-7f6-abc|api-7f6-def)$"),
5444
+ false,
5445
+ ),
5446
+ ).toBe("producer-related");
5447
+ expect(metricsScope(podTarget, podSelector("api-7f6-.*"), false)).toBe(
5448
+ "producer-related",
5449
+ );
5450
+ });
5451
+
5452
+ it("treats an unescaped dot in a workload regex as naming more than the target", () => {
5453
+ const dotted = { ...target, name: "api.v2" };
5454
+ const selector = (value: string) => [
5455
+ {
5456
+ metric: "kube_deployment_status_replicas_available",
5457
+ matchers: [
5458
+ { label: "namespace", op: "=", value: "shop" },
5459
+ { label: "deployment", op: "=~", value },
5460
+ ],
5461
+ },
5462
+ ];
5463
+ expect(metricsScope(dotted, selector("api.v2"), false)).toBe(
5464
+ "producer-related",
5465
+ );
5466
+ expect(metricsScope(dotted, selector("api\\.v2"), false)).toBe("target");
5467
+ expect(
5468
+ metricsScope(
5469
+ dotted,
5470
+ [
5471
+ {
5472
+ metric: "up",
5473
+ matchers: [
5474
+ { label: "namespace", op: "=", value: "shop" },
5475
+ { label: "pod", op: "=~", value: "api\\.v2-.*" },
5476
+ ],
5477
+ },
5478
+ ],
5479
+ false,
5480
+ ),
5481
+ ).toBe("producer-related");
5482
+ expect(
5483
+ metricsScope(
5484
+ dotted,
5485
+ [
5486
+ {
5487
+ metric: "up",
5488
+ matchers: [
5489
+ { label: "namespace", op: "=", value: "shop" },
5490
+ { label: "pod", op: "=~", value: "api.v2-.*" },
5491
+ ],
5492
+ },
5493
+ ],
5494
+ false,
5495
+ ),
5496
+ ).toBe("producer-related");
5497
+ });
5498
+
5499
+ it("names a Pod target only by its exact name and accepts an anchored namespace regex", () => {
5500
+ const pod = { ...target, kind: "Pod", group: "", name: "api-7f6-abc" };
5501
+ const selector = (
5502
+ namespace: { op: string; value: string },
5503
+ podMatcher: { op: string; value: string },
5504
+ ) => [
5505
+ {
5506
+ metric: "container_memory_working_set_bytes",
5507
+ matchers: [
5508
+ { label: "namespace", ...namespace },
5509
+ { label: "pod", ...podMatcher },
5510
+ ],
5511
+ },
5512
+ ];
5513
+ const exactNs = { op: "=", value: "shop" };
5514
+ expect(
5515
+ metricsScope(
5516
+ pod,
5517
+ selector(exactNs, { op: "=~", value: "api-7f6-abc-.*" }),
5518
+ false,
5519
+ ),
5520
+ ).toBe("producer-related");
5521
+ expect(
5522
+ metricsScope(
5523
+ pod,
5524
+ selector(exactNs, { op: "=", value: "api-7f6-abc" }),
5525
+ false,
5526
+ ),
5527
+ ).toBe("target");
5528
+ expect(
5529
+ metricsScope(
5530
+ pod,
5531
+ selector(exactNs, { op: "=~", value: "api-7f6-abc" }),
5532
+ false,
5533
+ ),
5534
+ ).toBe("target");
5535
+ expect(
5536
+ metricsScope(
5537
+ pod,
5538
+ selector(exactNs, { op: "=", value: "api-7f6-abc-extra" }),
5539
+ false,
5540
+ ),
5541
+ ).toBe("producer-related");
5542
+ expect(
5543
+ metricsScope(
5544
+ target,
5545
+ selector({ op: "=~", value: "shop" }, { op: "=~", value: "api-.*" }),
5546
+ false,
5547
+ ),
5548
+ ).toBe("producer-related");
5549
+ expect(
5550
+ metricsScope(
5551
+ target,
5552
+ selector(
5553
+ { op: "=~", value: "shop" },
5554
+ {
5555
+ op: "=~",
5556
+ value: "^(api-7f6-abc)$",
5557
+ },
5558
+ ),
5559
+ false,
5560
+ new Set(["api-7f6-abc"]),
5561
+ ),
5562
+ ).toBe("target");
5563
+ expect(
5564
+ metricsScope(
5565
+ target,
5566
+ selector(
5567
+ { op: "=~", value: "shop|other" },
5568
+ { op: "=~", value: "api-.*" },
5569
+ ),
5570
+ false,
5571
+ ),
5572
+ ).toBe("broader");
5573
+ expect(
5574
+ metricsScope(
5575
+ target,
5576
+ selector(
5577
+ { op: "!=", value: "kube-system" },
5578
+ { op: "=~", value: "api-.*" },
5579
+ ),
5580
+ false,
5581
+ ),
5582
+ ).toBe("broader");
5583
+ });
5584
+
5585
+ // Membership can also be established by a workload-logs read, which the
5586
+ // main pass adapts into groups as it goes. Classifying a metrics query
5587
+ // against those groups while they are still being built made the verdict
5588
+ // depend on which tool the agent happened to call first: the same facts,
5589
+ // read in the other order, demoted the same chart.
5590
+ it("classifies a metrics query the same whichever order the agent worked in", () => {
5591
+ const loggedPod = "api-68c7b766dc-fmphn";
5592
+ const exactLoggedPod = {
5593
+ metric: "container_memory_working_set_bytes",
5594
+ matchers: [
5595
+ { label: "namespace", op: "=", value: "shop" },
5596
+ { label: "pod", op: "=~", value: `^(${loggedPod})$` },
5597
+ ],
5598
+ };
5599
+ const logs = () =>
5600
+ tool("wl-logs", "get_workload_logs", workloadLogsPayload, {
5601
+ summary: workloadLogsArgs,
5602
+ });
5603
+ const metrics = () =>
5604
+ tool(
5605
+ "prom",
5606
+ "query_prometheus",
5607
+ promResult({
5608
+ selectors: [exactLoggedPod],
5609
+ query: `sum(container_memory_working_set_bytes{namespace="shop",pod=~"^(${loggedPod})$"})`,
5610
+ }),
5611
+ );
5612
+
5613
+ const logsFirst = groupsOf(project([logs(), metrics()]).groups, "metrics");
5614
+ const metricsFirst = groupsOf(
5615
+ project([metrics(), logs()]).groups,
5616
+ "metrics",
5617
+ );
5618
+ expect(logsFirst[0].latest.relevance).toBe("target");
5619
+ expect(metricsFirst[0].latest.relevance).toBe(
5620
+ logsFirst[0].latest.relevance,
5621
+ );
5622
+
5623
+ // Across turns too: the read that names the pod may arrive in a later one.
5624
+ const acrossTurns = groupsOf(
5625
+ project([metrics()], [logs()]).groups,
5626
+ "metrics",
5627
+ );
5628
+ expect(acrossTurns[0].latest.relevance).toBe("target");
5629
+ });
5630
+
5631
+ it("proves pod membership from the bundle instead of the workload's name", () => {
5632
+ const withBundle = project([
5633
+ podsBundle(),
5634
+ tool("prom", "query_prometheus", promResult()),
5635
+ ]);
5636
+ expect(groupsOf(withBundle.groups, "metrics")[0].latest.relevance).toBe(
5637
+ "target",
5638
+ );
5639
+
5640
+ // The same exact set with no bundle to establish it, and the prefix that
5641
+ // would also select a sibling workload's pods: kept, never promoted.
5642
+ const unproven = project([tool("prom", "query_prometheus", promResult())]);
5643
+ expect(groupsOf(unproven.groups, "metrics")[0].latest.relevance).toBe(
5644
+ "producer-related",
5645
+ );
5646
+ const prefixed = project([
5647
+ podsBundle(),
5648
+ tool(
5649
+ "prom",
5650
+ "query_prometheus",
5651
+ promResult({ selectors: prefixSelectors }),
5652
+ ),
5653
+ ]);
5654
+ const prefixGroup = groupsOf(prefixed.groups, "metrics")[0];
5655
+ expect(prefixGroup.latest.relevance).toBe("producer-related");
5656
+ expect(
5657
+ prefixGroup.latest.data.type === "metrics" &&
5658
+ prefixGroup.latest.data.subject,
5659
+ ).toBeUndefined();
5660
+
5661
+ // A subset of the established pods is still about the target.
5662
+ const subset = project([
5663
+ podsBundle(),
5664
+ tool(
5665
+ "prom",
5666
+ "query_prometheus",
5667
+ promResult({
5668
+ selectors: [
5669
+ {
5670
+ metric: "container_memory_working_set_bytes",
5671
+ matchers: [
5672
+ { label: "namespace", op: "=", value: "shop" },
5673
+ { label: "pod", op: "=", value: "api-7f6-abc" },
5674
+ ],
5675
+ },
5676
+ ],
5677
+ }),
5678
+ ),
5679
+ ]);
5680
+ expect(groupsOf(subset.groups, "metrics")[0].latest.relevance).toBe(
5681
+ "target",
5682
+ );
5683
+ });
5684
+
5685
+ it("proves membership whatever order the agent ran its tools in", () => {
5686
+ // The agent may know the pod names from an earlier turn and chart them
5687
+ // before it calls diagnose. The same facts must read the same way.
5688
+ const queryFirst = project([
5689
+ tool("prom", "query_prometheus", promResult()),
5690
+ podsBundle(),
5691
+ ]);
5692
+ expect(groupsOf(queryFirst.groups, "metrics")[0].latest.relevance).toBe(
5693
+ "target",
5694
+ );
5695
+ });
5696
+
5697
+ it("will not establish target pods from a diagnose that never confirmed success", () => {
5698
+ // This set is what proves a Prometheus selector is about the target, so it
5699
+ // takes confirmed success — the same test the sources use — rather than
5700
+ // merely the absence of an error.
5701
+ const unconfirmed = {
5702
+ ...podsBundle(),
5703
+ isError: undefined as unknown as boolean,
5704
+ };
5705
+ const projection = project([
5706
+ unconfirmed,
5707
+ tool(
5708
+ "prom",
5709
+ "query_prometheus",
5710
+ promResult({
5711
+ query: `sum(container_memory_working_set_bytes{namespace="shop",pod=~"${podSetPattern}"})`,
5712
+ selectors: [
5713
+ {
5714
+ metric: "container_memory_working_set_bytes",
5715
+ matchers: [
5716
+ { label: "namespace", op: "=", value: "shop" },
5717
+ { label: "pod", op: "=~", value: podSetPattern },
5718
+ ],
5719
+ },
5720
+ ],
5721
+ }),
5722
+ ),
5723
+ ]);
5724
+ expect(groupsOf(projection.groups, "metrics")[0].latest.relevance).not.toBe(
5725
+ "target",
5726
+ );
5727
+ });
5728
+
5729
+ it("does not let a generic workload label claim another kind's series", () => {
5730
+ const workloadSelectors = (type?: string) => [
5731
+ {
5732
+ metric: "container_cpu_usage_seconds_total",
5733
+ matchers: [
5734
+ { label: "namespace", op: "=", value: "shop" },
5735
+ { label: "workload", op: "=", value: "api" },
5736
+ ...(type ? [{ label: "workload_type", op: "=", value: type }] : []),
5737
+ ],
5738
+ },
5739
+ ];
5740
+ const asDeployment = project([
5741
+ tool(
5742
+ "prom",
5743
+ "query_prometheus",
5744
+ promResult({ selectors: workloadSelectors("deployment") }),
5745
+ ),
5746
+ ]);
5747
+ expect(groupsOf(asDeployment.groups, "metrics")[0].latest.relevance).toBe(
5748
+ "target",
5749
+ );
5750
+ // A StatefulSet also called api in the same namespace is a different
5751
+ // workload, and the series says so.
5752
+ const asStatefulSet = project([
5753
+ tool(
5754
+ "prom",
5755
+ "query_prometheus",
5756
+ promResult({ selectors: workloadSelectors("statefulset") }),
5757
+ ),
5758
+ ]);
5759
+ expect(groupsOf(asStatefulSet.groups, "metrics")[0].latest.relevance).toBe(
5760
+ "producer-related",
5761
+ );
5762
+
5763
+ // The same exclusion written as a regex. Reading only `=` let a sibling's
5764
+ // chart take the target's identity and its change markers.
5765
+ const regexSelectors = (value: string) => [
5766
+ {
5767
+ metric: "container_cpu_usage_seconds_total",
5768
+ matchers: [
5769
+ { label: "namespace", op: "=", value: "shop" },
5770
+ { label: "workload", op: "=", value: "api" },
5771
+ { label: "workload_type", op: "=~", value },
5772
+ ],
5773
+ },
5774
+ ];
5775
+ for (const [value, relevance] of [
5776
+ ["statefulset", "producer-related"],
5777
+ ["^(statefulset)$", "producer-related"],
5778
+ // A set that includes the target kind could be the target, so it is no
5779
+ // conflict; an inexact regex proves nothing either way.
5780
+ ["^(deployment|statefulset)$", "target"],
5781
+ [".*", "target"],
5782
+ ] as const) {
5783
+ const projection = project([
5784
+ tool(
5785
+ "prom",
5786
+ "query_prometheus",
5787
+ promResult({ selectors: regexSelectors(value) }),
5788
+ ),
5789
+ ]);
5790
+ expect(groupsOf(projection.groups, "metrics")[0].latest.relevance).toBe(
5791
+ relevance,
5792
+ );
5793
+ }
5794
+ });
5795
+
5796
+ it("refuses an unescaped dot as proof that a pod set names the target", () => {
5797
+ const dotted = ["api.v2-0"];
5798
+ const bundle = tool(
5799
+ "diag-dotted",
5800
+ "diagnose",
5801
+ { resource: deployment, pods: 1, podNames: dotted },
5802
+ {
5803
+ summary: JSON.stringify({
5804
+ kind: "Deployment",
5805
+ namespace: "shop",
5806
+ name: "api",
5807
+ }),
5808
+ },
5809
+ );
5810
+ const podSelector = (value: string) => [
5811
+ {
5812
+ metric: "container_memory_working_set_bytes",
5813
+ matchers: [
5814
+ { label: "namespace", op: "=", value: "shop" },
5815
+ { label: "pod", op: "=~", value },
5816
+ ],
5817
+ },
5818
+ ];
5819
+ // `^(api.v2-0)$` also selects api-v2-0, so it names more than the pod it
5820
+ // appears to and cannot prove membership.
5821
+ const wildcard = project([
5822
+ bundle,
5823
+ tool(
5824
+ "prom",
5825
+ "query_prometheus",
5826
+ promResult({ selectors: podSelector("^(api.v2-0)$") }),
5827
+ ),
5828
+ ]);
5829
+ expect(groupsOf(wildcard.groups, "metrics")[0].latest.relevance).toBe(
5830
+ "producer-related",
5831
+ );
5832
+ const escaped = project([
5833
+ bundle,
5834
+ tool(
5835
+ "prom",
5836
+ "query_prometheus",
5837
+ promResult({ selectors: podSelector("^(api\\.v2-0)$") }),
5838
+ ),
5839
+ ]);
5840
+ expect(groupsOf(escaped.groups, "metrics")[0].latest.relevance).toBe(
5841
+ "target",
5842
+ );
5843
+ });
5844
+
5845
+ it("rejects a result without the producer's selector inventory", () => {
5846
+ const projection = project([
5847
+ tool("old", "query_prometheus", promResult({ selectors: undefined })),
5848
+ tool(
5849
+ "bad-series",
5850
+ "query_prometheus",
5851
+ promResult({
5852
+ series: [{ labels: {}, dataPoints: [{ timestamp: "x", value: 1 }] }],
5853
+ }),
5854
+ ),
5855
+ ]);
5856
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
5857
+ expect(projection.limitations.map((item) => item.kind)).toEqual([
5858
+ "unknown",
5859
+ ]);
5860
+ expect(projection.limitations[0].source).toBe("Prometheus query");
5861
+ expect(projection.limitations[0].sources).toHaveLength(2);
5862
+ });
5863
+
5864
+ it("merges the same query into one card with revisions", () => {
5865
+ const projection = project(
5866
+ [tool("first", "query_prometheus", promResult())],
5867
+ [
5868
+ tool(
5869
+ "second",
5870
+ "query_prometheus",
5871
+ promResult({
5872
+ series: [
5873
+ {
5874
+ labels: { pod: "api-7f6-abc" },
5875
+ dataPoints: [{ timestamp: 1_788_679_043, value: 1 }],
5876
+ },
5877
+ ],
5878
+ }),
5879
+ ),
5880
+ ],
5881
+ );
5882
+ const groups = groupsOf(projection.groups, "metrics");
5883
+ expect(groups).toHaveLength(1);
5884
+ expect(groups[0].observations).toHaveLength(2);
5885
+ expect(groups[0].observations[1].changedFromPrevious).toBe(true);
5886
+ });
5887
+ });
5888
+
5889
+ describe("resource cards scaled by an HPA", () => {
5890
+ const scaled = (state: string) =>
5891
+ tool("workload", "diagnose", {
5892
+ resource: {
5893
+ apiVersion: "apps/v1",
5894
+ kind: "Deployment",
5895
+ metadata: { namespace: "shop", name: "api" },
5896
+ },
5897
+ resourceContext: {
5898
+ tier: "basic",
5899
+ workloadSummary: { replicas: { desired: 5, ready: 5, available: 5 } },
5900
+ scaledBy: [
5901
+ {
5902
+ kind: "HorizontalPodAutoscaler",
5903
+ namespace: "shop",
5904
+ name: "api-hpa",
5905
+ hpaSummary: {
5906
+ state,
5907
+ summary: "summary from Radar",
5908
+ bounds: { min: 1, max: 5, current: 5, desired: 5 },
5909
+ },
5910
+ },
5911
+ ],
5912
+ },
5913
+ });
5914
+
5915
+ it("lifts a healthy workload whose autoscaler cannot act", () => {
5916
+ const result = project([scaled("metrics_unavailable")]);
5917
+ const card = groupsOf(result.groups, "resource")[0].latest;
5918
+ expect(card.tier).toBe("supporting");
5919
+ expect(card.tone).toBe("warning");
5920
+ expect(card.summary).toBe("5/5 replicas ready · HPA: Metrics unavailable");
5921
+ });
5922
+
5923
+ it("treats a maxed-out autoscaler as adverse", () => {
5924
+ const card = groupsOf(
5925
+ project([scaled("limited_max")]).groups,
5926
+ "resource",
5927
+ )[0].latest;
5928
+ expect(card.tone).toBe("warning");
5929
+ expect(card.summary).toBe("5/5 replicas ready · HPA: Maxed");
5930
+ });
5931
+
5932
+ it("leaves an autoscaler that is simply scaling as context", () => {
5933
+ const card = groupsOf(project([scaled("scaling_up")]).groups, "resource")[0]
5934
+ .latest;
5935
+ expect(card.tier).toBe("context");
5936
+ expect(card.tone).toBe("neutral");
5937
+ expect(card.summary).toBe("5/5 replicas ready");
5938
+ });
5939
+ });
5940
+
5941
+ describe("diagnose metrics evidence", () => {
5942
+ const window = {
5943
+ start: "2026-09-06T07:00:00Z",
5944
+ end: "2026-09-06T08:00:00Z",
5945
+ step: "1m2s",
5946
+ };
5947
+ const samples = [
5948
+ { timestamp: Date.parse(window.start) / 1000, value: 0.003457 },
5949
+ { timestamp: Date.parse(window.end) / 1000, value: 0.004 },
5950
+ ];
5951
+ function vitals(patch: Record<string, unknown> = {}) {
5952
+ return {
5953
+ window,
5954
+ pods: 2,
5955
+ series: [
5956
+ {
5957
+ category: "cpu",
5958
+ unit: "cores",
5959
+ query:
5960
+ "sum(rate(container_cpu_usage_seconds_total{container!='',namespace='shop',pod=~'^(api-a|api-b)$'}[5m]))",
5961
+ series: [{ labels: {}, dataPoints: samples }],
5962
+ },
5963
+ {
5964
+ category: "memory",
5965
+ unit: "bytes",
5966
+ query:
5967
+ "sum(max by (pod,namespace,container) (container_memory_working_set_bytes{container!='',namespace='shop',pod=~'^(api-a|api-b)$'}))",
5968
+ series: [{ labels: {}, dataPoints: samples }],
5969
+ },
5970
+ {
5971
+ category: "restarts",
5972
+ unit: "count",
5973
+ query:
5974
+ "sum(round(increase(kube_pod_container_status_restarts_total{namespace='shop',pod=~'^(api-a|api-b)$'}[1h])))",
5975
+ series: [],
5976
+ },
5977
+ ],
5978
+ ...patch,
5979
+ };
5980
+ }
5981
+ const args = { kind: "Deployment", namespace: "shop", name: "api" };
5982
+
5983
+ it("charts each captured category as target evidence with the diagnosed resource as subject", () => {
5984
+ const projection = project([
5985
+ tool(
5986
+ "diag",
5987
+ "diagnose",
5988
+ { resource: deployment, pods: 2, metrics: vitals() },
5989
+ { summary: JSON.stringify(args) },
5990
+ ),
5991
+ ]);
5992
+ const groups = groupsOf(projection.groups, "metrics");
5993
+ expect(groups.map((group) => group.latest.title)).toEqual([
5994
+ "CPU usage · Deployment shop/api",
5995
+ "Memory working set · Deployment shop/api",
5996
+ "Restarts · Deployment shop/api",
5997
+ ]);
5998
+ for (const group of groups) {
5999
+ expect(group.latest.relevance).toBe("target");
6000
+ expect(group.latest.tier).toBe("supporting");
6001
+ expect(group.latest.tone).toBe("neutral");
6002
+ const data = group.latest.data;
6003
+ if (data.type !== "metrics") throw new Error("expected metrics");
6004
+ expect(data.origin).toBe("diagnose");
6005
+ expect(data.mode).toBe("range");
6006
+ expect(data.start).toBe(window.start);
6007
+ expect(data.end).toBe(window.end);
6008
+ expect(data.step).toBe(window.step);
6009
+ expect(data.pods).toBe(2);
6010
+ expect(data.partial).toBe(false);
6011
+ expect(data.truncated).toBe(false);
6012
+ expect(data.subject).toEqual({
6013
+ kind: "Deployment",
6014
+ group: "apps",
6015
+ namespace: "shop",
6016
+ name: "api",
6017
+ });
6018
+ expect(investigationEvidenceSubjectRef(data)).toEqual(data.subject);
6019
+ }
6020
+ const [cpu, , restarts] = groups;
6021
+ expect(cpu.latest.summary).toBe("2 current pods · 60m window · 1m2s step");
6022
+ const cpuData = cpu.latest.data;
6023
+ if (cpuData.type !== "metrics") throw new Error("expected metrics");
6024
+ expect(cpuData.unit).toBe("cores");
6025
+ expect(cpuData.label).toBe("CPU usage");
6026
+ expect(cpuData.query).toContain("pod=~'^(api-a|api-b)$'");
6027
+ expect(cpuData.series).toEqual([{ labels: {}, dataPoints: samples }]);
6028
+ expect(restarts.latest.summary).toBe(
6029
+ "No samples in the window · 60m window · 1m2s step",
6030
+ );
6031
+ expect(
6032
+ projection.limitations.filter(
6033
+ (item) => item.source === "Workload metrics",
6034
+ ),
6035
+ ).toHaveLength(0);
6036
+ });
6037
+
6038
+ it("merges a re-diagnose of the same workload into the same chart per category", () => {
6039
+ const later = vitals({
6040
+ window: { ...window, end: "2026-09-06T08:30:00Z" },
6041
+ });
6042
+ const projection = project(
6043
+ [
6044
+ tool(
6045
+ "diag-1",
6046
+ "diagnose",
6047
+ { resource: deployment, pods: 2, metrics: vitals() },
6048
+ { summary: JSON.stringify(args) },
6049
+ ),
6050
+ ],
6051
+ [
6052
+ tool(
6053
+ "diag-2",
6054
+ "diagnose",
6055
+ { resource: deployment, pods: 2, metrics: later },
6056
+ { summary: JSON.stringify(args) },
6057
+ ),
6058
+ ],
6059
+ );
6060
+ const groups = groupsOf(projection.groups, "metrics");
6061
+ expect(groups).toHaveLength(3);
6062
+ expect(groups.every((group) => group.observations.length === 2)).toBe(true);
6063
+ const data = groups[0].latest.data;
6064
+ if (data.type !== "metrics") throw new Error("expected metrics");
6065
+ expect(data.end).toBe("2026-09-06T08:30:00Z");
6066
+ });
6067
+
6068
+ it("reports a producer error as a limitation and still charts what answered", () => {
6069
+ const projection = project([
6070
+ tool(
6071
+ "diag",
6072
+ "diagnose",
6073
+ {
6074
+ resource: deployment,
6075
+ pods: 2,
6076
+ metrics: vitals({
6077
+ series: [vitals().series[1]],
6078
+ error:
6079
+ "cpu: prom: query error from prometheus: cpu exploded (execution)",
6080
+ }),
6081
+ },
6082
+ { summary: JSON.stringify(args) },
6083
+ ),
6084
+ ]);
6085
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(1);
6086
+ const limitation = projection.limitations.find(
6087
+ (item) => item.source === "Workload metrics",
6088
+ );
6089
+ expect(limitation?.kind).toBe("error");
6090
+ expect(limitation?.message).toContain("cpu exploded");
6091
+ });
6092
+
6093
+ it("reports an unreachable Prometheus without inventing a chart", () => {
6094
+ const projection = project([
6095
+ tool(
6096
+ "diag",
6097
+ "diagnose",
6098
+ {
6099
+ resource: deployment,
6100
+ pods: 2,
6101
+ metrics: vitals({
6102
+ series: [],
6103
+ error:
6104
+ "prometheus unreachable: dial tcp 10.0.0.9:9090: connection refused",
6105
+ }),
6106
+ },
6107
+ { summary: JSON.stringify(args) },
6108
+ ),
6109
+ ]);
6110
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
6111
+ const limitation = projection.limitations.find(
6112
+ (item) => item.source === "Workload metrics",
6113
+ );
6114
+ expect(limitation?.kind).toBe("error");
6115
+ expect(limitation?.message).toBe(
6116
+ "prometheus unreachable: dial tcp 10.0.0.9:9090: connection refused",
6117
+ );
6118
+ });
6119
+
6120
+ it("says nothing when diagnose carried no metrics field", () => {
6121
+ const projection = project([
6122
+ tool(
6123
+ "diag",
6124
+ "diagnose",
6125
+ { resource: deployment, pods: 2 },
6126
+ { summary: JSON.stringify(args) },
6127
+ ),
6128
+ ]);
6129
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
6130
+ expect(
6131
+ projection.limitations.some((item) => item.source === "Workload metrics"),
6132
+ ).toBe(false);
6133
+ });
6134
+
6135
+ it("notes a partial pod set in the summary and keeps the cap on the card", () => {
6136
+ const projection = project([
6137
+ tool(
6138
+ "diag",
6139
+ "diagnose",
6140
+ {
6141
+ resource: deployment,
6142
+ pods: 55,
6143
+ metrics: vitals({ pods: 50, partial: true, omittedPods: 5 }),
6144
+ },
6145
+ { summary: JSON.stringify(args) },
6146
+ ),
6147
+ ]);
6148
+ const [cpu] = groupsOf(projection.groups, "metrics");
6149
+ expect(cpu.latest.summary).toBe(
6150
+ "first 50 of 55 current pods · 60m window · 1m2s step · partial pod set",
6151
+ );
6152
+ const data = cpu.latest.data;
6153
+ if (data.type !== "metrics") throw new Error("expected metrics");
6154
+ expect(data.pods).toBe(50);
6155
+ expect(data.partial).toBe(true);
6156
+ });
6157
+
6158
+ it("keeps a neighbour's vitals broader with the neighbour as subject", () => {
6159
+ const worker = {
6160
+ ...deployment,
6161
+ metadata: { namespace: "shop", name: "worker" },
6162
+ };
6163
+ const projection = project([
6164
+ tool(
6165
+ "diag-worker",
6166
+ "diagnose",
6167
+ { resource: worker, pods: 2, metrics: vitals() },
6168
+ {
6169
+ summary: JSON.stringify({
6170
+ kind: "Deployment",
6171
+ namespace: "shop",
6172
+ name: "worker",
6173
+ }),
6174
+ },
6175
+ ),
6176
+ ]);
6177
+ const groups = groupsOf(projection.groups, "metrics");
6178
+ expect(groups).toHaveLength(3);
6179
+ for (const group of groups) {
6180
+ expect(group.latest.relevance).toBe("broader");
6181
+ expect(group.latest.tier).toBe("context");
6182
+ const data = group.latest.data;
6183
+ if (data.type !== "metrics") throw new Error("expected metrics");
6184
+ expect(data.subject).toEqual({
6185
+ kind: "Deployment",
6186
+ group: "apps",
6187
+ namespace: "shop",
6188
+ name: "worker",
6189
+ });
6190
+ }
6191
+ });
6192
+
6193
+ it("rejects a malformed metrics field instead of charting it", () => {
6194
+ const malformed: Record<string, unknown>[] = [
6195
+ { window: { start: window.start }, pods: "two", series: [] },
6196
+ { window: { ...window, end: "not a date" }, pods: 2, series: [] },
6197
+ { window: { ...window, end: window.start }, pods: 2, series: [] },
6198
+ { window, pods: -1, series: [] },
6199
+ { window, pods: 1.5, series: [] },
6200
+ { window, pods: 2, omittedPods: -3, series: [] },
6201
+ ];
6202
+ for (const metrics of malformed) {
6203
+ const projection = project([
6204
+ tool(
6205
+ "diag",
6206
+ "diagnose",
6207
+ { resource: deployment, pods: 2, metrics },
6208
+ { summary: JSON.stringify(args) },
6209
+ ),
6210
+ ]);
6211
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
6212
+ expect(
6213
+ projection.limitations.some(
6214
+ (item) =>
6215
+ item.source === "Workload metrics" && item.kind === "unknown",
6216
+ ),
6217
+ ).toBe(true);
6218
+ }
6219
+ });
6220
+
6221
+ it("keeps same-named workloads in different API groups on separate charts", () => {
6222
+ const rollout = {
6223
+ ...deployment,
6224
+ apiVersion: "argoproj.io/v1alpha1",
6225
+ kind: "Rollout",
6226
+ };
6227
+ const projection = project([
6228
+ tool(
6229
+ "diag-deployment",
6230
+ "diagnose",
6231
+ { resource: deployment, pods: 2, metrics: vitals() },
6232
+ { summary: JSON.stringify(args) },
6233
+ ),
6234
+ tool(
6235
+ "diag-rollout",
6236
+ "diagnose",
6237
+ { resource: rollout, pods: 2, metrics: vitals() },
6238
+ { summary: JSON.stringify(args) },
6239
+ ),
6240
+ ]);
6241
+ const groups = groupsOf(projection.groups, "metrics");
6242
+ expect(groups).toHaveLength(6);
6243
+ expect(groups.every((group) => group.observations.length === 1)).toBe(true);
6244
+ expect(new Set(groups.map((group) => group.identity)).size).toBe(6);
6245
+ });
6246
+
6247
+ it("rejects inherited object keys posing as categories", () => {
6248
+ for (const category of ["__proto__", "constructor", "toString"]) {
6249
+ const projection = project([
6250
+ tool(
6251
+ "diag",
6252
+ "diagnose",
6253
+ {
6254
+ resource: deployment,
6255
+ pods: 2,
6256
+ metrics: vitals({
6257
+ series: [{ ...vitals().series[0], category }],
6258
+ }),
6259
+ },
6260
+ { summary: JSON.stringify(args) },
6261
+ ),
6262
+ ]);
6263
+ expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
6264
+ expect(
6265
+ projection.limitations.some(
6266
+ (item) =>
6267
+ item.source === "Workload metrics" && item.kind === "unknown",
6268
+ ),
6269
+ ).toBe(true);
6270
+ }
6271
+ });
6272
+
6273
+ it("rejects a category outside the contract or a series without a unit", () => {
6274
+ const projection = project([
6275
+ tool(
6276
+ "diag",
6277
+ "diagnose",
6278
+ {
6279
+ resource: deployment,
6280
+ pods: 2,
6281
+ metrics: vitals({
6282
+ series: [
6283
+ { ...vitals().series[0], category: "latency" },
6284
+ { ...vitals().series[1], unit: undefined },
6285
+ vitals().series[2],
6286
+ ],
6287
+ }),
6288
+ },
6289
+ { summary: JSON.stringify(args) },
6290
+ ),
6291
+ ]);
6292
+ const groups = groupsOf(projection.groups, "metrics");
6293
+ expect(groups.map((group) => group.latest.title)).toEqual([
6294
+ "Restarts · Deployment shop/api",
6295
+ ]);
6296
+ expect(
6297
+ projection.limitations.some(
6298
+ (item) => item.source === "Workload metrics" && item.kind === "unknown",
6299
+ ),
6300
+ ).toBe(true);
6301
+ });
6302
+ });
6303
+
6304
+ describe("live-run follow-ups", () => {
6305
+ it("reads the producer's nil-slice null as an empty events or changes result", () => {
6306
+ const projection = project([
6307
+ tool(
6308
+ "events-none",
6309
+ "get_events",
6310
+ { events: null },
6311
+ {
6312
+ summary: JSON.stringify({
6313
+ kind: "Pod",
6314
+ namespace: "shop",
6315
+ name: "api",
6316
+ }),
6317
+ },
6318
+ ),
6319
+ tool(
6320
+ "changes-none",
6321
+ "get_changes",
6322
+ { changes: null },
6323
+ {
6324
+ summary: JSON.stringify({
6325
+ kind: "Deployment",
6326
+ namespace: "shop",
6327
+ name: "api",
6328
+ since: "24h",
6329
+ }),
6330
+ },
6331
+ ),
6332
+ ]);
6333
+ // A complete, successful events query that returned nothing is a checked
6334
+ // receipt for its scope; the producer marks a denied namespace separately.
6335
+ expect(
6336
+ projection.groups.map((group) => [
6337
+ group.latest.data.type,
6338
+ group.latest.title,
6339
+ ]),
6340
+ ).toEqual([["receipt", "No events in this window"]]);
6341
+ expect(
6342
+ projection.limitations.map((limitation) => [
6343
+ limitation.source,
6344
+ limitation.kind,
6345
+ ]),
6346
+ ).toEqual([["Recent changes", "unknown"]]);
6347
+ });
6348
+
6349
+ it("keeps one identity for a blocker whether one pod holds it or several", () => {
6350
+ const blocker = (name: string) => ({
6351
+ kind: "Pod",
6352
+ name,
6353
+ reason: "Unschedulable",
6354
+ severity: "critical",
6355
+ message: "1 node(s) no free host ports",
6356
+ });
6357
+ const bundle = (pods: string[]) => ({
6358
+ resource: {
6359
+ apiVersion: "apps/v1",
6360
+ kind: "DaemonSet",
6361
+ metadata: { namespace: "opencost", name: "node-exporter" },
6362
+ },
6363
+ resourceContext: { tier: "basic" },
6364
+ pods: pods.length,
6365
+ startupBlockers: pods.map(blocker),
6366
+ });
6367
+ // Two pods recover to one between diagnoses. The blocker is the same
6368
+ // finding, so it must update its card rather than open a second one
6369
+ // beside it saying the same thing.
6370
+ const projection = projectInvestigationEvidence(
6371
+ [
6372
+ {
6373
+ status: "done",
6374
+ timeline: [tool("first", "diagnose", bundle(["a", "b"]))],
6375
+ },
6376
+ {
6377
+ status: "done",
6378
+ verify: true,
6379
+ timeline: [tool("second", "diagnose", bundle(["a"]))],
6380
+ },
6381
+ ],
6382
+ {
6383
+ kind: "DaemonSet",
6384
+ group: "apps",
6385
+ namespace: "opencost",
6386
+ name: "node-exporter",
6387
+ },
6388
+ );
6389
+ const startup = projection.groups.filter(
6390
+ (group) => group.latest.data.type === "startup",
6391
+ );
6392
+ expect(startup).toHaveLength(1);
6393
+ expect(startup[0].observations).toHaveLength(2);
6394
+ expect(startup[0].latest.summary).toBe("1 node(s) no free host ports");
6395
+ });
6396
+
6397
+ it("merges identical startup blockers across pods into one card with the pod list", () => {
6398
+ const blocker = (
6399
+ name: string,
6400
+ message = "1 node(s) no free host ports",
6401
+ ) => ({
6402
+ kind: "Pod",
6403
+ name,
6404
+ reason: "Unschedulable",
6405
+ severity: "critical",
6406
+ message,
6407
+ });
6408
+ const projection = project([
6409
+ tool("diagnose", "diagnose", {
6410
+ resource: {
6411
+ apiVersion: "apps/v1",
6412
+ kind: "DaemonSet",
6413
+ metadata: { namespace: "opencost", name: "node-exporter" },
6414
+ },
6415
+ resourceContext: { tier: "basic" },
6416
+ pods: 3,
6417
+ startupBlockers: [
6418
+ blocker("node-exporter-a"),
6419
+ blocker("node-exporter-b"),
6420
+ blocker("node-exporter-c", "0/10 nodes: insufficient memory"),
6421
+ ],
6422
+ }),
6423
+ ]);
6424
+ const startup = projection.groups.filter(
6425
+ (group) => group.latest.data.type === "startup",
6426
+ );
6427
+ expect(
6428
+ startup.map((group) => [
6429
+ group.latest.summary,
6430
+ group.latest.data.type === "startup" ? group.latest.data.pods : null,
6431
+ ]),
6432
+ ).toEqual([
6433
+ [
6434
+ "2 pods · 1 node(s) no free host ports",
6435
+ ["node-exporter-a", "node-exporter-b"],
6436
+ ],
6437
+ // A blocker only one pod holds still carries that pod, so the pods a
6438
+ // later Prometheus query is proved against do not depend on how many
6439
+ // happened to share a reason.
6440
+ ["0/10 nodes: insufficient memory", ["node-exporter-c"]],
6441
+ ]);
6442
+ expect(startup[1].id).not.toBe(startup[0].id);
6443
+ });
6444
+
6445
+ it("folds pods' startup blocker into the classified issue that repeats it", () => {
6446
+ const blocker = (name: string) => ({
6447
+ kind: "Pod",
6448
+ name,
6449
+ reason: "Unschedulable",
6450
+ severity: "critical",
6451
+ message: "1 node(s) no free host ports",
6452
+ });
6453
+ const projection = project([
6454
+ tool("diagnose", "diagnose", {
6455
+ resource: {
6456
+ apiVersion: "apps/v1",
6457
+ kind: "DaemonSet",
6458
+ metadata: { namespace: "opencost", name: "node-exporter" },
6459
+ },
6460
+ resourceContext: { tier: "basic" },
6461
+ pods: 2,
6462
+ relatedIssues: [
6463
+ {
6464
+ id: "issue-unsched",
6465
+ severity: "critical",
6466
+ source: "scheduling",
6467
+ category: "unschedulable",
6468
+ category_group: "startup",
6469
+ grouping_scope: "workload",
6470
+ kind: "DaemonSet",
6471
+ namespace: "opencost",
6472
+ name: "node-exporter",
6473
+ reason: "Unschedulable",
6474
+ message: "1 node(s) no free host ports",
6475
+ },
6476
+ ],
6477
+ startupBlockers: [
6478
+ blocker("node-exporter-a"),
6479
+ blocker("node-exporter-b"),
6480
+ ],
6481
+ }),
6482
+ ]);
6483
+ expect(
6484
+ projection.groups.filter((group) => group.latest.data.type === "startup"),
6485
+ ).toHaveLength(0);
6486
+ const [issue] = projection.groups.filter(
6487
+ (group) => group.latest.data.type === "issue",
6488
+ );
6489
+ expect(
6490
+ issue.latest.data.type === "issue" && issue.latest.data.pods,
6491
+ ).toEqual(["node-exporter-a", "node-exporter-b"]);
6492
+ });
6493
+
6494
+ it("files a denied events namespace as an access limitation, not a receipt", () => {
6495
+ const projection = project([
6496
+ tool(
6497
+ "events-denied",
6498
+ "get_events",
6499
+ { events: [], accessDenied: true },
6500
+ {
6501
+ summary: JSON.stringify({ namespace: "locked", kind: "Pod" }),
6502
+ },
6503
+ ),
6504
+ ]);
6505
+ expect(projection.groups).toHaveLength(0);
6506
+ expect(
6507
+ projection.limitations.map((limitation) => [
6508
+ limitation.source,
6509
+ limitation.kind,
6510
+ ]),
6511
+ ).toEqual([["Events", "error"]]);
6512
+ expect(projection.limitations[0].message).toContain(
6513
+ "not readable with your permissions",
6514
+ );
6515
+ });
6516
+
6517
+ it("keeps a cluster-wide events read that was narrowed to readable namespaces from reading as a clean cluster", () => {
6518
+ const empty = project([
6519
+ tool(
6520
+ "events-partial-empty",
6521
+ "get_events",
6522
+ { events: [], partialScope: true, scopeNamespaces: ["alpha", "beta"] },
6523
+ { summary: JSON.stringify({}) },
6524
+ ),
6525
+ ]);
6526
+ const [receipt] = groupsOf(empty.groups, "receipt");
6527
+ expect(receipt.latest.title).toBe(
6528
+ "No events in the namespaces you can read",
6529
+ );
6530
+ const narrowedBody =
6531
+ receipt.latest.data.type === "receipt" ? receipt.latest.data.message : "";
6532
+ expect(narrowedBody).toContain("alpha, beta");
6533
+ // The load-bearing half: an empty answer from the namespaces a reader can
6534
+ // see must never read as an answer about the cluster.
6535
+ expect(narrowedBody).toContain("does not clear the cluster");
6536
+ expect(empty.limitations).toHaveLength(0);
6537
+
6538
+ const found = project([
6539
+ tool(
6540
+ "events-partial",
6541
+ "get_events",
6542
+ {
6543
+ events: [
6544
+ {
6545
+ type: "Warning",
6546
+ reason: "BackOff",
6547
+ message: "back-off restarting",
6548
+ count: 3,
6549
+ lastTimestamp: "2026-09-02T09:00:00Z",
6550
+ involvedObject: { kind: "Pod", namespace: "alpha", name: "api" },
6551
+ },
6552
+ ],
6553
+ partialScope: true,
6554
+ scopeNamespaces: ["alpha", "beta"],
6555
+ },
6556
+ { summary: JSON.stringify({}) },
6557
+ ),
6558
+ ]);
6559
+ expect(
6560
+ found.limitations.map((limitation) => [
6561
+ limitation.source,
6562
+ limitation.kind,
6563
+ ]),
6564
+ ).toEqual([["Events", "unknown"]]);
6565
+ expect(found.limitations[0].message).toContain("alpha, beta");
6566
+ });
6567
+
5153
6568
  it("keeps repeated change reads of one resource on one card and names the window", () => {
5154
6569
  const change = {
5155
6570
  kind: "Deployment",