@skyhook-io/radar-app 1.13.3 → 1.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/App.tsx +68 -23
- package/src/RadarApp.tsx +17 -1
- package/src/api/client.ts +28 -10
- package/src/api/diagnose.ts +61 -15
- package/src/components/ConnectionErrorView.test.tsx +30 -0
- package/src/components/ConnectionErrorView.tsx +31 -19
- package/src/components/diagnose/AgentCase.tsx +131 -0
- package/src/components/diagnose/DiagnoseContext.test.ts +95 -0
- package/src/components/diagnose/DiagnoseContext.tsx +402 -77
- package/src/components/diagnose/DiagnoseSurface.test.tsx +53 -54
- package/src/components/diagnose/DiagnoseSurface.tsx +198 -286
- package/src/components/diagnose/Home.test.tsx +23 -1
- package/src/components/diagnose/Home.tsx +49 -3
- package/src/components/diagnose/InvestigationEvidencePane.test.tsx +1446 -5
- package/src/components/diagnose/InvestigationEvidencePane.tsx +1373 -126
- package/src/components/diagnose/InvestigationView.tsx +227 -5
- package/src/components/diagnose/LocalDiagnoseAction.tsx +2 -3
- package/src/components/diagnose/diagnoseEvidenceTypes.ts +22 -1
- package/src/components/diagnose/investigationCase.test.tsx +1568 -0
- package/src/components/diagnose/investigationCase.ts +439 -0
- package/src/components/diagnose/investigationEvidence.test.ts +3525 -17
- package/src/components/diagnose/investigationEvidence.ts +2246 -166
- package/src/components/diagnose/investigationEvidenceKinds.ts +218 -0
- package/src/components/diagnose/investigationEvidencePresentation.test.ts +31 -0
- package/src/components/diagnose/investigationEvidencePresentation.ts +1 -0
- package/src/components/diagnose/investigationMetrics.test.ts +712 -0
- package/src/components/diagnose/investigationMetrics.ts +393 -0
- package/src/components/diagnose/investigationSourceFocus.ts +6 -0
- package/src/components/diagnose/investigationState.test.ts +322 -0
- package/src/components/diagnose/investigationState.ts +147 -25
- package/src/components/diagnose/parts.test.tsx +537 -1
- package/src/components/diagnose/parts.tsx +486 -42
- package/src/components/resource/HPACharts.render.test.tsx +77 -0
- package/src/components/resource/HPACharts.tsx +32 -25
- package/src/components/resource/PrometheusChartsGrid.tsx +181 -79
- package/src/components/resources/PodFilePreview.test.tsx +131 -0
- package/src/components/resources/PodFilePreview.tsx +394 -0
- package/src/components/resources/PodFilesystemModal.tsx +157 -67
- package/src/components/resources/ResourcesView.tsx +27 -2
- package/src/context/DiagnoseCustomization.test.tsx +26 -0
- package/src/context/DiagnoseCustomization.tsx +14 -2
- package/src/index.ts +8 -5
- package/src/utils/shell-safe.test.ts +25 -1
- package/src/utils/shell-safe.ts +16 -0
|
@@ -3,13 +3,17 @@ import {
|
|
|
3
3
|
defaultConditionTone,
|
|
4
4
|
displayKind,
|
|
5
5
|
englishPlural,
|
|
6
|
+
hpaStateLabel,
|
|
7
|
+
hpaStateLevel,
|
|
6
8
|
kindToPlural,
|
|
7
9
|
stripAnsi,
|
|
10
|
+
type HPADiagnosisState,
|
|
8
11
|
type Issue,
|
|
9
12
|
type IssueRecentChange,
|
|
10
13
|
type Topology,
|
|
11
14
|
} from "@skyhook-io/k8s-ui";
|
|
12
15
|
import { fnv1a32 } from "@skyhook-io/k8s-ui/utils/structure-hash";
|
|
16
|
+
import type { TimeSeries } from "@skyhook-io/k8s-ui/components/charts";
|
|
13
17
|
import { apiVersionToGroup } from "../../utils/navigation";
|
|
14
18
|
|
|
15
19
|
import {
|
|
@@ -28,6 +32,7 @@ import {
|
|
|
28
32
|
} from "./diagnoseEvidenceTypes";
|
|
29
33
|
import type { RootCauseEvidence } from "../../api/diagnose";
|
|
30
34
|
import { investigationResourceEvidenceSummary } from "./investigationResourceEvidenceModel";
|
|
35
|
+
import { metricsUnitForExpression } from "./investigationMetrics";
|
|
31
36
|
|
|
32
37
|
/**
|
|
33
38
|
* The projection deliberately consumes only the small, structural portion of
|
|
@@ -89,7 +94,11 @@ export type InvestigationEvidenceKind =
|
|
|
89
94
|
| "relationships"
|
|
90
95
|
| "topology"
|
|
91
96
|
| "inventory"
|
|
92
|
-
| "receipt"
|
|
97
|
+
| "receipt"
|
|
98
|
+
| "alerts"
|
|
99
|
+
| "helm"
|
|
100
|
+
| "permissions"
|
|
101
|
+
| "metrics";
|
|
93
102
|
|
|
94
103
|
type InvestigationSemanticDomain = "issue" | "startup" | "crash" | "dns";
|
|
95
104
|
|
|
@@ -240,6 +249,100 @@ export interface InvestigationNetworkEvidence {
|
|
|
240
249
|
routes: InvestigationNetworkRoute[];
|
|
241
250
|
}
|
|
242
251
|
|
|
252
|
+
/** One Prometheus rule as `get_prometheus_rules` flattens it (group stamped on). */
|
|
253
|
+
export interface InvestigationAlertRule {
|
|
254
|
+
group: string;
|
|
255
|
+
name: string;
|
|
256
|
+
type: string;
|
|
257
|
+
state?: string;
|
|
258
|
+
health?: string;
|
|
259
|
+
query?: string;
|
|
260
|
+
labels: Record<string, string>;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** One active instance of an alerting rule, with its own label set. */
|
|
264
|
+
export interface InvestigationAlertInstance {
|
|
265
|
+
state: string;
|
|
266
|
+
activeAt?: string;
|
|
267
|
+
value?: string;
|
|
268
|
+
labels: Record<string, string>;
|
|
269
|
+
/** The instance's labels name the investigated resource. */
|
|
270
|
+
namesTarget: boolean;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export interface InvestigationHelmOperation {
|
|
274
|
+
kind: string;
|
|
275
|
+
status: string;
|
|
276
|
+
message: string;
|
|
277
|
+
revision?: number;
|
|
278
|
+
failedRevision?: number;
|
|
279
|
+
rollbackRevision?: number;
|
|
280
|
+
updated?: string;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
export interface InvestigationHelmOwnedResource {
|
|
284
|
+
kind: string;
|
|
285
|
+
apiVersion?: string;
|
|
286
|
+
name: string;
|
|
287
|
+
namespace: string;
|
|
288
|
+
status?: string;
|
|
289
|
+
ready?: string;
|
|
290
|
+
message?: string;
|
|
291
|
+
summary?: string;
|
|
292
|
+
issue?: string;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
export interface InvestigationHelmRelease {
|
|
296
|
+
name: string;
|
|
297
|
+
namespace: string;
|
|
298
|
+
/** Set only when Helm stores the release metadata elsewhere. */
|
|
299
|
+
storageNamespace?: string;
|
|
300
|
+
chart: string;
|
|
301
|
+
chartVersion: string;
|
|
302
|
+
appVersion?: string;
|
|
303
|
+
status: string;
|
|
304
|
+
revision: number;
|
|
305
|
+
updated: string;
|
|
306
|
+
description?: string;
|
|
307
|
+
resourceHealth?: string;
|
|
308
|
+
healthIssue?: string;
|
|
309
|
+
healthSummary?: string;
|
|
310
|
+
managedByFluxHelmRelease?: string;
|
|
311
|
+
lastOperation?: InvestigationHelmOperation;
|
|
312
|
+
resources: InvestigationHelmOwnedResource[];
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export interface InvestigationPermissionSubject {
|
|
316
|
+
kind: string;
|
|
317
|
+
namespace?: string;
|
|
318
|
+
name: string;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
export interface InvestigationAccessCheck {
|
|
322
|
+
verb: string;
|
|
323
|
+
group?: string;
|
|
324
|
+
resource: string;
|
|
325
|
+
subresource?: string;
|
|
326
|
+
/** Empty means a cluster-scoped resource or cluster-wide request. */
|
|
327
|
+
namespace: string;
|
|
328
|
+
resourceName?: string;
|
|
329
|
+
allowed: boolean;
|
|
330
|
+
denied: boolean;
|
|
331
|
+
reason?: string;
|
|
332
|
+
evaluationError?: string;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export interface InvestigationPermissionBinding {
|
|
336
|
+
bindingKind: string;
|
|
337
|
+
bindingNamespace?: string;
|
|
338
|
+
bindingName: string;
|
|
339
|
+
roleKind: string;
|
|
340
|
+
roleNamespace?: string;
|
|
341
|
+
roleName: string;
|
|
342
|
+
rulesCount: number;
|
|
343
|
+
inheritedFromGroup?: string;
|
|
344
|
+
}
|
|
345
|
+
|
|
243
346
|
export type InvestigationEvidenceData =
|
|
244
347
|
| {
|
|
245
348
|
type: "issue";
|
|
@@ -250,12 +353,20 @@ export type InvestigationEvidenceData =
|
|
|
250
353
|
* that they explain the resource under investigation.
|
|
251
354
|
*/
|
|
252
355
|
relevance: "target" | "producer-related" | "broader";
|
|
356
|
+
/** Pods whose startup blocker repeats this issue word for word. */
|
|
357
|
+
pods?: string[];
|
|
253
358
|
}
|
|
254
359
|
| {
|
|
255
360
|
type: "startup";
|
|
256
361
|
blocker: DiagnosisStartupBlocker;
|
|
257
362
|
/** Exact blocker object when the diagnosis producer established it. */
|
|
258
363
|
subject?: DiagnosisResourceRef;
|
|
364
|
+
/**
|
|
365
|
+
* Every pod the producer reported with this exact blocker. A DaemonSet
|
|
366
|
+
* with seven pending pods is one finding, not seven; the card carries the
|
|
367
|
+
* pod list instead of repeating itself.
|
|
368
|
+
*/
|
|
369
|
+
pods?: string[];
|
|
259
370
|
}
|
|
260
371
|
| {
|
|
261
372
|
type: "crash";
|
|
@@ -291,6 +402,8 @@ export type InvestigationEvidenceData =
|
|
|
291
402
|
changes: IssueRecentChange[];
|
|
292
403
|
scope: string;
|
|
293
404
|
changeContext?: DiagnosisChangeContext;
|
|
405
|
+
/** The resource whose change history the producer read, when it named one. */
|
|
406
|
+
subject?: { kind?: string; namespace?: string; name: string };
|
|
294
407
|
}
|
|
295
408
|
| { type: "dns"; dns: DiagnosisDNSContext }
|
|
296
409
|
| { type: "network"; network: InvestigationNetworkEvidence }
|
|
@@ -315,10 +428,69 @@ export type InvestigationEvidenceData =
|
|
|
315
428
|
}
|
|
316
429
|
| {
|
|
317
430
|
type: "receipt";
|
|
318
|
-
checked:
|
|
431
|
+
checked:
|
|
432
|
+
"issues" | "events" | "changes" | "inventory" | "logs" | "alerts";
|
|
319
433
|
scope: string;
|
|
320
|
-
|
|
321
|
-
|
|
434
|
+
/**
|
|
435
|
+
* Only when there is something to add. The card already shows the title
|
|
436
|
+
* and the scope, so a line that restates either costs the reader a read
|
|
437
|
+
* and returns nothing; this carries the reason the answer is what it is,
|
|
438
|
+
* or the limit of what it proves.
|
|
439
|
+
*/
|
|
440
|
+
message?: string;
|
|
441
|
+
}
|
|
442
|
+
| {
|
|
443
|
+
type: "alerts";
|
|
444
|
+
rule: InvestigationAlertRule;
|
|
445
|
+
instances: InvestigationAlertInstance[];
|
|
446
|
+
annotations: Record<string, string>;
|
|
447
|
+
}
|
|
448
|
+
| { type: "helm"; release: InvestigationHelmRelease }
|
|
449
|
+
| {
|
|
450
|
+
type: "permissions";
|
|
451
|
+
subject: InvestigationPermissionSubject;
|
|
452
|
+
/** Present for the access-check response shape. */
|
|
453
|
+
accessCheck?: InvestigationAccessCheck;
|
|
454
|
+
/** Present for the subject-permissions response shape. */
|
|
455
|
+
bindings?: InvestigationPermissionBinding[];
|
|
456
|
+
flatRulesCount?: number;
|
|
457
|
+
truncated?: boolean;
|
|
458
|
+
usedByPods?: string[];
|
|
459
|
+
podsTotal?: number;
|
|
460
|
+
}
|
|
461
|
+
| InvestigationMetricsEvidence;
|
|
462
|
+
/** One PromQL vector selector as the producer tokenized it. */
|
|
463
|
+
export interface InvestigationMetricsSelector {
|
|
464
|
+
metric: string;
|
|
465
|
+
matchers: Array<{ label: string; op: string; value: string }>;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* A Prometheus result captured during the investigation. `origin` says which
|
|
470
|
+
* producer ran the query; the shape is shared so every metrics card renders
|
|
471
|
+
* the same way. `subject` is set only when the query's selectors are all
|
|
472
|
+
* scoped to the investigation target.
|
|
473
|
+
*/
|
|
474
|
+
export interface InvestigationMetricsEvidence {
|
|
475
|
+
type: "metrics";
|
|
476
|
+
origin: "query" | "diagnose";
|
|
477
|
+
query: string;
|
|
478
|
+
mode: "range" | "instant";
|
|
479
|
+
start?: string;
|
|
480
|
+
end?: string;
|
|
481
|
+
step?: string;
|
|
482
|
+
unit?: string;
|
|
483
|
+
label?: string;
|
|
484
|
+
series: TimeSeries[];
|
|
485
|
+
truncated: boolean;
|
|
486
|
+
summary?: unknown;
|
|
487
|
+
note?: string;
|
|
488
|
+
selectors?: InvestigationMetricsSelector[];
|
|
489
|
+
selectorsUnknown?: boolean;
|
|
490
|
+
subject?: DiagnosisResourceRef;
|
|
491
|
+
pods?: number;
|
|
492
|
+
partial?: boolean;
|
|
493
|
+
}
|
|
322
494
|
|
|
323
495
|
export interface InvestigationEvidenceObservation {
|
|
324
496
|
source: InvestigationEvidenceSource;
|
|
@@ -485,6 +657,31 @@ export function investigationEvidenceSubjectRef(
|
|
|
485
657
|
return data.network.subject;
|
|
486
658
|
case "relationships":
|
|
487
659
|
return data.root;
|
|
660
|
+
case "helm":
|
|
661
|
+
// A resource ref cannot carry the storage namespace, and opening the
|
|
662
|
+
// release by name alone would read the wrong storage.
|
|
663
|
+
if (
|
|
664
|
+
data.release.storageNamespace !== undefined &&
|
|
665
|
+
data.release.storageNamespace !== data.release.namespace
|
|
666
|
+
) {
|
|
667
|
+
return undefined;
|
|
668
|
+
}
|
|
669
|
+
return {
|
|
670
|
+
kind: "HelmRelease",
|
|
671
|
+
group: "helm.sh",
|
|
672
|
+
namespace: data.release.namespace,
|
|
673
|
+
name: data.release.name,
|
|
674
|
+
};
|
|
675
|
+
case "permissions":
|
|
676
|
+
// Users and Groups are principals, not Kubernetes objects.
|
|
677
|
+
if (data.subject.kind !== "ServiceAccount") return undefined;
|
|
678
|
+
return {
|
|
679
|
+
kind: "ServiceAccount",
|
|
680
|
+
namespace: data.subject.namespace,
|
|
681
|
+
name: data.subject.name,
|
|
682
|
+
};
|
|
683
|
+
case "metrics":
|
|
684
|
+
return data.subject;
|
|
488
685
|
default:
|
|
489
686
|
return undefined;
|
|
490
687
|
}
|
|
@@ -521,6 +718,16 @@ export function investigationEvidenceSourceDomId(sourceId: string): string {
|
|
|
521
718
|
|
|
522
719
|
const investigationEvidenceRefRe = /^ev_[a-z2-7]{26,128}_[a-z2-7]{26,128}$/;
|
|
523
720
|
|
|
721
|
+
export function isInvestigationEvidenceRef(value: string): boolean {
|
|
722
|
+
return investigationEvidenceRefRe.test(value);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
export function investigationSourceArgs(
|
|
726
|
+
source: InvestigationEvidenceSource,
|
|
727
|
+
): Record<string, unknown> | undefined {
|
|
728
|
+
return record(source.args ? parseJSON(source.args) : undefined);
|
|
729
|
+
}
|
|
730
|
+
|
|
524
731
|
export function resolveInvestigationRootCauseEvidence(
|
|
525
732
|
projection: InvestigationEvidenceProjection,
|
|
526
733
|
evidence: RootCauseEvidence | undefined,
|
|
@@ -1116,6 +1323,11 @@ const INVESTIGATION_RESULT_LABELS: Readonly<Record<string, string>> = {
|
|
|
1116
1323
|
get_changes: "Recent changes",
|
|
1117
1324
|
get_neighborhood: "Relationships",
|
|
1118
1325
|
get_topology: "Topology",
|
|
1326
|
+
get_workload_logs: "Workload logs",
|
|
1327
|
+
get_prometheus_rules: "Alert rules",
|
|
1328
|
+
get_helm_release: "Helm release",
|
|
1329
|
+
get_subject_permissions: "Permissions",
|
|
1330
|
+
query_prometheus: "Prometheus query",
|
|
1119
1331
|
};
|
|
1120
1332
|
|
|
1121
1333
|
function investigationResultLabel(source: InvestigationEvidenceSource): string {
|
|
@@ -1209,6 +1421,13 @@ class ProjectionBuilder {
|
|
|
1209
1421
|
Set<InvestigationSemanticDomain>
|
|
1210
1422
|
>();
|
|
1211
1423
|
|
|
1424
|
+
/**
|
|
1425
|
+
* Pods a target-scoped diagnose bundle listed as the workload's own, by
|
|
1426
|
+
* controller ownership. Membership for an agent's pod-level Prometheus
|
|
1427
|
+
* query is proved against this set, never inferred from a name.
|
|
1428
|
+
*/
|
|
1429
|
+
readonly establishedTargetPods = new Set<string>();
|
|
1430
|
+
|
|
1212
1431
|
private readonly groupByIdentity = new Map<
|
|
1213
1432
|
string,
|
|
1214
1433
|
InvestigationEvidenceGroup
|
|
@@ -1254,11 +1473,13 @@ class ProjectionBuilder {
|
|
|
1254
1473
|
kind === "startup" ||
|
|
1255
1474
|
kind === "crash" ||
|
|
1256
1475
|
(kind === "receipt" && identity.startsWith("previous-log-absence:"));
|
|
1257
|
-
const mapKey =
|
|
1476
|
+
const mapKey = groupKey(
|
|
1477
|
+
kind,
|
|
1478
|
+
identity,
|
|
1258
1479
|
partitionByRelevance
|
|
1259
|
-
?
|
|
1260
|
-
:
|
|
1261
|
-
|
|
1480
|
+
? `${observation.relevance}\u0000${scopeFromArgs(source)}`
|
|
1481
|
+
: undefined,
|
|
1482
|
+
);
|
|
1262
1483
|
let group = this.groupByIdentity.get(mapKey);
|
|
1263
1484
|
const previousObservation = group?.observations.at(-1);
|
|
1264
1485
|
const changedFromPrevious = previousObservation
|
|
@@ -1297,7 +1518,10 @@ class ProjectionBuilder {
|
|
|
1297
1518
|
// target was already established by a scoped producer. Keep the broad read
|
|
1298
1519
|
// in revision history, but never let weaker provenance replace the card's
|
|
1299
1520
|
// authoritative observation. Equal provenance still advances normally.
|
|
1521
|
+
// An alert rule's relevance is derived from its live instances, so a later
|
|
1522
|
+
// read of the same rule is a state transition, not weaker provenance.
|
|
1300
1523
|
if (
|
|
1524
|
+
kind === "alerts" ||
|
|
1301
1525
|
relevanceRank[next.relevance] <= relevanceRank[group.latest.relevance]
|
|
1302
1526
|
) {
|
|
1303
1527
|
group.latest = next;
|
|
@@ -1306,6 +1530,14 @@ class ProjectionBuilder {
|
|
|
1306
1530
|
if (next.tier === "checked") this.checkedSources.add(source.id);
|
|
1307
1531
|
}
|
|
1308
1532
|
|
|
1533
|
+
/** Relevance of the current card for an unpartitioned identity, if observed. */
|
|
1534
|
+
latestRelevance(
|
|
1535
|
+
kind: InvestigationEvidenceKind,
|
|
1536
|
+
identity: string,
|
|
1537
|
+
): InvestigationEvidenceRelevance | undefined {
|
|
1538
|
+
return this.groupByIdentity.get(groupKey(kind, identity))?.latest.relevance;
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1309
1541
|
limit(
|
|
1310
1542
|
source: InvestigationEvidenceSource,
|
|
1311
1543
|
label: string,
|
|
@@ -1337,6 +1569,18 @@ class ProjectionBuilder {
|
|
|
1337
1569
|
}
|
|
1338
1570
|
}
|
|
1339
1571
|
|
|
1572
|
+
/**
|
|
1573
|
+
* The one place a group's map key is built. NUL separates the parts so no
|
|
1574
|
+
* kind or identity text can collide with another kind's key.
|
|
1575
|
+
*/
|
|
1576
|
+
function groupKey(
|
|
1577
|
+
kind: InvestigationEvidenceKind,
|
|
1578
|
+
identity: string,
|
|
1579
|
+
partition?: string,
|
|
1580
|
+
): string {
|
|
1581
|
+
return `${kind}\u0000${identity}${partition === undefined ? "" : `\u0000${partition}`}`;
|
|
1582
|
+
}
|
|
1583
|
+
|
|
1340
1584
|
export function evidenceSemanticSnapshot(
|
|
1341
1585
|
observation: Pick<
|
|
1342
1586
|
InvestigationEvidenceObservation,
|
|
@@ -1352,23 +1596,35 @@ export function evidenceSemanticSnapshot(
|
|
|
1352
1596
|
});
|
|
1353
1597
|
}
|
|
1354
1598
|
const data =
|
|
1355
|
-
observation.data.type === "
|
|
1599
|
+
observation.data.type === "alerts"
|
|
1356
1600
|
? {
|
|
1357
1601
|
type: observation.data.type,
|
|
1358
|
-
//
|
|
1359
|
-
//
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
severity: observation.data.issue.severity,
|
|
1367
|
-
cause: observation.data.issue.cause,
|
|
1368
|
-
message: observation.data.issue.message,
|
|
1369
|
-
},
|
|
1602
|
+
// An instance's sampled value and activation time move on every
|
|
1603
|
+
// evaluation; the finding is which instances exist and their state.
|
|
1604
|
+
rule: observation.data.rule,
|
|
1605
|
+
instances: observation.data.instances.map((instance) => ({
|
|
1606
|
+
state: instance.state,
|
|
1607
|
+
labels: instance.labels,
|
|
1608
|
+
})),
|
|
1609
|
+
annotations: observation.data.annotations,
|
|
1370
1610
|
}
|
|
1371
|
-
: observation.data
|
|
1611
|
+
: observation.data.type === "issue"
|
|
1612
|
+
? {
|
|
1613
|
+
type: observation.data.type,
|
|
1614
|
+
// These are the finding and details shown in the evidence card.
|
|
1615
|
+
// Collection timestamps and detector bookkeeping do not change it.
|
|
1616
|
+
issue: {
|
|
1617
|
+
kind: observation.data.issue.kind,
|
|
1618
|
+
group: observation.data.issue.group,
|
|
1619
|
+
namespace: observation.data.issue.namespace,
|
|
1620
|
+
name: observation.data.issue.name,
|
|
1621
|
+
reason: observation.data.issue.reason,
|
|
1622
|
+
severity: observation.data.issue.severity,
|
|
1623
|
+
cause: observation.data.issue.cause,
|
|
1624
|
+
message: observation.data.issue.message,
|
|
1625
|
+
},
|
|
1626
|
+
}
|
|
1627
|
+
: observation.data;
|
|
1372
1628
|
return JSON.stringify({
|
|
1373
1629
|
tone: observation.tone,
|
|
1374
1630
|
title: observation.title,
|
|
@@ -1394,7 +1650,7 @@ function addNarrowHint(
|
|
|
1394
1650
|
source,
|
|
1395
1651
|
label,
|
|
1396
1652
|
label +
|
|
1397
|
-
"
|
|
1653
|
+
" returned part of the matching results to keep this investigation fast. More may exist.",
|
|
1398
1654
|
"truncated",
|
|
1399
1655
|
);
|
|
1400
1656
|
}
|
|
@@ -1417,7 +1673,7 @@ function addResourceContextLimitations(
|
|
|
1417
1673
|
builder.limit(
|
|
1418
1674
|
source,
|
|
1419
1675
|
omitted.field,
|
|
1420
|
-
`
|
|
1676
|
+
`Radar left out part of the resource context (${omitted.reason.replaceAll("_", " ")}).`,
|
|
1421
1677
|
omitted.reason === "budget_exceeded" ? "truncated" : "unknown",
|
|
1422
1678
|
);
|
|
1423
1679
|
}
|
|
@@ -1426,7 +1682,7 @@ function addResourceContextLimitations(
|
|
|
1426
1682
|
builder.limit(
|
|
1427
1683
|
source,
|
|
1428
1684
|
"Relationships",
|
|
1429
|
-
`
|
|
1685
|
+
`Radar returned ${shown} of ${context.referencedBy.total} referenced-by relationships.`,
|
|
1430
1686
|
"truncated",
|
|
1431
1687
|
);
|
|
1432
1688
|
}
|
|
@@ -1434,7 +1690,7 @@ function addResourceContextLimitations(
|
|
|
1434
1690
|
builder.limit(
|
|
1435
1691
|
source,
|
|
1436
1692
|
"Application references",
|
|
1437
|
-
"
|
|
1693
|
+
"Radar returned part of the stale Secret references.",
|
|
1438
1694
|
"truncated",
|
|
1439
1695
|
);
|
|
1440
1696
|
}
|
|
@@ -1449,7 +1705,7 @@ function addIssueLimitations(
|
|
|
1449
1705
|
builder.limit(
|
|
1450
1706
|
source,
|
|
1451
1707
|
`Radar Issue ${value.id}`,
|
|
1452
|
-
"
|
|
1708
|
+
"Radar returned part of the affected-resource list.",
|
|
1453
1709
|
"truncated",
|
|
1454
1710
|
);
|
|
1455
1711
|
}
|
|
@@ -1470,11 +1726,16 @@ function resourceObservationSummary(
|
|
|
1470
1726
|
if (gitOps?.sync) return `Sync ${gitOps.sync}`;
|
|
1471
1727
|
if (gitOps?.suspended) return "Reconciliation suspended";
|
|
1472
1728
|
const replicas = context?.workloadSummary?.replicas;
|
|
1729
|
+
const adverseScaler = adverseScalerStates(context)[0];
|
|
1473
1730
|
if (replicas?.desired !== undefined) {
|
|
1474
1731
|
const desired = replicas.desired;
|
|
1475
1732
|
const ready = replicas.ready ?? 0;
|
|
1476
|
-
|
|
1733
|
+
const readiness = `${ready}/${desired} replicas ready`;
|
|
1734
|
+
return adverseScaler
|
|
1735
|
+
? `${readiness} · HPA: ${hpaStateLabel(adverseScaler)}`
|
|
1736
|
+
: readiness;
|
|
1477
1737
|
}
|
|
1738
|
+
if (adverseScaler) return `HPA: ${hpaStateLabel(adverseScaler)}`;
|
|
1478
1739
|
if (context?.statusSummary?.phase) return context.statusSummary.phase;
|
|
1479
1740
|
if (context?.issueSummary?.topReason) return context.issueSummary.topReason;
|
|
1480
1741
|
return (
|
|
@@ -1484,6 +1745,22 @@ function resourceObservationSummary(
|
|
|
1484
1745
|
);
|
|
1485
1746
|
}
|
|
1486
1747
|
|
|
1748
|
+
// A scaler that cannot act (no metrics, cannot read the target) or is pinned
|
|
1749
|
+
// at its ceiling is a captured Radar fact about the workload, so it lifts the
|
|
1750
|
+
// card the way an adverse condition does. Ordinary scale-ups and scale-downs
|
|
1751
|
+
// are the autoscaler working and stay as detail.
|
|
1752
|
+
function adverseScalerStates(
|
|
1753
|
+
context: InvestigationResourceContext | undefined,
|
|
1754
|
+
): HPADiagnosisState[] {
|
|
1755
|
+
return (context?.scaledBy ?? []).flatMap((scaler) => {
|
|
1756
|
+
const state = scaler.hpaSummary?.state;
|
|
1757
|
+
if (!state) return [];
|
|
1758
|
+
return hpaStateLevel(state) === "unhealthy" || state === "limited_max"
|
|
1759
|
+
? [state]
|
|
1760
|
+
: [];
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1487
1764
|
function addResourceObservation(
|
|
1488
1765
|
builder: ProjectionBuilder,
|
|
1489
1766
|
source: InvestigationEvidenceSource,
|
|
@@ -1521,7 +1798,11 @@ function addResourceObservation(
|
|
|
1521
1798
|
gitOpsDiagnosis.operationPhase?.toLowerCase() ?? "",
|
|
1522
1799
|
)),
|
|
1523
1800
|
);
|
|
1524
|
-
const hasAdverseState =
|
|
1801
|
+
const hasAdverseState =
|
|
1802
|
+
replicaShortfall ||
|
|
1803
|
+
adverseCondition ||
|
|
1804
|
+
gitOpsAdverse ||
|
|
1805
|
+
adverseScalerStates(context).length > 0;
|
|
1525
1806
|
const intendedTier: InvestigationEvidenceTier =
|
|
1526
1807
|
critical && !hasDetailedCriticalIssue
|
|
1527
1808
|
? "key"
|
|
@@ -1568,6 +1849,7 @@ function addIssueObservation(
|
|
|
1568
1849
|
source: InvestigationEvidenceSource,
|
|
1569
1850
|
value: Issue,
|
|
1570
1851
|
producerRelevance: InvestigationEvidenceRelevance = "broader",
|
|
1852
|
+
pods?: string[],
|
|
1571
1853
|
): void {
|
|
1572
1854
|
const matchesTarget = resourceMatchesTarget(builder.target, {
|
|
1573
1855
|
kind: value.kind,
|
|
@@ -1587,7 +1869,12 @@ function addIssueObservation(
|
|
|
1587
1869
|
relevance,
|
|
1588
1870
|
title: value.reason,
|
|
1589
1871
|
summary: value.cause || value.message,
|
|
1590
|
-
data: {
|
|
1872
|
+
data: {
|
|
1873
|
+
type: "issue",
|
|
1874
|
+
issue: value,
|
|
1875
|
+
relevance,
|
|
1876
|
+
...(pods && pods.length > 0 ? { pods } : {}),
|
|
1877
|
+
},
|
|
1591
1878
|
});
|
|
1592
1879
|
addIssueLimitations(builder, source, value);
|
|
1593
1880
|
}
|
|
@@ -1636,6 +1923,9 @@ function addEvents(
|
|
|
1636
1923
|
complete = true,
|
|
1637
1924
|
emptyIsAuthoritative = false,
|
|
1638
1925
|
relevance: InvestigationEvidenceRelevance = "broader",
|
|
1926
|
+
emptyReceipt: { title: string; message?: string } = {
|
|
1927
|
+
title: "No warning events",
|
|
1928
|
+
},
|
|
1639
1929
|
): void {
|
|
1640
1930
|
const scope = scopeFromArgs(source);
|
|
1641
1931
|
if (values.length === 0) {
|
|
@@ -1653,17 +1943,18 @@ function addEvents(
|
|
|
1653
1943
|
tier: evidenceTierForRelevance("checked", relevance),
|
|
1654
1944
|
relevance,
|
|
1655
1945
|
tone: "neutral",
|
|
1656
|
-
title:
|
|
1946
|
+
title: emptyReceipt.title,
|
|
1657
1947
|
summary: scope,
|
|
1658
1948
|
data: {
|
|
1659
1949
|
type: "receipt",
|
|
1660
1950
|
checked: "events",
|
|
1661
1951
|
scope,
|
|
1662
|
-
message:
|
|
1952
|
+
message: emptyReceipt.message,
|
|
1663
1953
|
},
|
|
1664
1954
|
});
|
|
1665
1955
|
return;
|
|
1666
1956
|
}
|
|
1957
|
+
const lead = leadEvent(values);
|
|
1667
1958
|
builder.observe(identity, "events", source, {
|
|
1668
1959
|
tier: evidenceTierForRelevance(
|
|
1669
1960
|
values.some((item) => item.type.toLowerCase() === "warning")
|
|
@@ -1676,11 +1967,32 @@ function addEvents(
|
|
|
1676
1967
|
? "warning"
|
|
1677
1968
|
: "info",
|
|
1678
1969
|
title: "Kubernetes events",
|
|
1679
|
-
summary: `${
|
|
1970
|
+
summary: `${lead.reason}: ${lead.message}${
|
|
1971
|
+
values.length > 1 ? ` · ${values.length} event groups` : ""
|
|
1972
|
+
} · ${scope}`,
|
|
1680
1973
|
data: { type: "events", events: values, scope },
|
|
1681
1974
|
});
|
|
1682
1975
|
}
|
|
1683
1976
|
|
|
1977
|
+
/**
|
|
1978
|
+
* The newest warning, else the newest event. The card lead must be what
|
|
1979
|
+
* happened, not how many groups the producer returned.
|
|
1980
|
+
*/
|
|
1981
|
+
function leadEvent(
|
|
1982
|
+
values: InvestigationEventEvidence[],
|
|
1983
|
+
): InvestigationEventEvidence {
|
|
1984
|
+
const newest = (items: InvestigationEventEvidence[]) =>
|
|
1985
|
+
items.reduce((best, item) =>
|
|
1986
|
+
Date.parse(item.lastTimestamp) > Date.parse(best.lastTimestamp)
|
|
1987
|
+
? item
|
|
1988
|
+
: best,
|
|
1989
|
+
);
|
|
1990
|
+
const warnings = values.filter(
|
|
1991
|
+
(item) => item.type.toLowerCase() === "warning",
|
|
1992
|
+
);
|
|
1993
|
+
return newest(warnings.length > 0 ? warnings : values);
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1684
1996
|
function addChanges(
|
|
1685
1997
|
builder: ProjectionBuilder,
|
|
1686
1998
|
source: InvestigationEvidenceSource,
|
|
@@ -1690,6 +2002,8 @@ function addChanges(
|
|
|
1690
2002
|
complete = true,
|
|
1691
2003
|
emptyIsAuthoritative = false,
|
|
1692
2004
|
relevance: InvestigationEvidenceRelevance = "broader",
|
|
2005
|
+
subject?: { kind?: string; namespace?: string; name: string },
|
|
2006
|
+
window?: string,
|
|
1693
2007
|
): void {
|
|
1694
2008
|
const scope = scopeFromArgs(source);
|
|
1695
2009
|
if (values.length === 0 && !changeContext?.changed) {
|
|
@@ -1708,13 +2022,12 @@ function addChanges(
|
|
|
1708
2022
|
tier: evidenceTierForRelevance("checked", relevance),
|
|
1709
2023
|
relevance,
|
|
1710
2024
|
tone: "neutral",
|
|
1711
|
-
title: "No
|
|
2025
|
+
title: "No recorded changes in this window",
|
|
1712
2026
|
summary: scope,
|
|
1713
2027
|
data: {
|
|
1714
2028
|
type: "receipt",
|
|
1715
2029
|
checked: "changes",
|
|
1716
2030
|
scope,
|
|
1717
|
-
message: "The requested change window returned no tracked changes.",
|
|
1718
2031
|
},
|
|
1719
2032
|
});
|
|
1720
2033
|
return;
|
|
@@ -1728,7 +2041,7 @@ function addChanges(
|
|
|
1728
2041
|
),
|
|
1729
2042
|
relevance,
|
|
1730
2043
|
tone: "info",
|
|
1731
|
-
title:
|
|
2044
|
+
title: `Recent changes${window ? ` · last ${window}` : ""}`,
|
|
1732
2045
|
summary: changeContext?.changed
|
|
1733
2046
|
? changeContext.what === "The workload's Pod template changed" &&
|
|
1734
2047
|
changeContext.when
|
|
@@ -1740,10 +2053,23 @@ function addChanges(
|
|
|
1740
2053
|
changes: values,
|
|
1741
2054
|
scope,
|
|
1742
2055
|
changeContext,
|
|
2056
|
+
subject,
|
|
1743
2057
|
},
|
|
1744
2058
|
});
|
|
1745
2059
|
}
|
|
1746
2060
|
|
|
2061
|
+
function changesSubjectFromArgs(
|
|
2062
|
+
source: InvestigationEvidenceSource,
|
|
2063
|
+
): { kind?: string; namespace?: string; name: string } | undefined {
|
|
2064
|
+
const args = record(source.args ? parseJSON(source.args) : undefined);
|
|
2065
|
+
if (!nonEmptyString(args?.name)) return undefined;
|
|
2066
|
+
return {
|
|
2067
|
+
...(nonEmptyString(args.kind) ? { kind: args.kind } : {}),
|
|
2068
|
+
...(nonEmptyString(args.namespace) ? { namespace: args.namespace } : {}),
|
|
2069
|
+
name: args.name,
|
|
2070
|
+
};
|
|
2071
|
+
}
|
|
2072
|
+
|
|
1747
2073
|
function addLogs(
|
|
1748
2074
|
builder: ProjectionBuilder,
|
|
1749
2075
|
source: InvestigationEvidenceSource,
|
|
@@ -2034,6 +2360,7 @@ function adaptDiagnose(
|
|
|
2034
2360
|
});
|
|
2035
2361
|
const relatedRelevance: InvestigationEvidenceRelevance =
|
|
2036
2362
|
bundleRelevance === "target" ? "producer-related" : "broader";
|
|
2363
|
+
|
|
2037
2364
|
const bundledRowRelevance = (
|
|
2038
2365
|
kind: string,
|
|
2039
2366
|
name: string,
|
|
@@ -2068,11 +2395,57 @@ function adaptDiagnose(
|
|
|
2068
2395
|
if (!relatedValid) {
|
|
2069
2396
|
invalidPayload(builder, source, "Classified issues");
|
|
2070
2397
|
}
|
|
2398
|
+
const blockersRaw = value.startupBlockers;
|
|
2399
|
+
let blockersValid = blockersRaw === undefined || Array.isArray(blockersRaw);
|
|
2400
|
+
// Pods of one workload usually share a blocker word for word. Merge those
|
|
2401
|
+
// into one finding with the pod list; anything else keeps its own card and
|
|
2402
|
+
// its own identity so saved runs match as before.
|
|
2403
|
+
const blockerGroups: Array<{
|
|
2404
|
+
blocker: DiagnosisStartupBlocker;
|
|
2405
|
+
pods: string[];
|
|
2406
|
+
foldedInto?: Issue;
|
|
2407
|
+
}> = [];
|
|
2408
|
+
if (Array.isArray(blockersRaw)) {
|
|
2409
|
+
const merged = new Map<string, (typeof blockerGroups)[number]>();
|
|
2410
|
+
for (const raw of blockersRaw) {
|
|
2411
|
+
const blocker = startupBlocker(raw);
|
|
2412
|
+
if (!blocker) {
|
|
2413
|
+
blockersValid = false;
|
|
2414
|
+
invalidPayload(builder, source, "Startup evidence");
|
|
2415
|
+
continue;
|
|
2416
|
+
}
|
|
2417
|
+
const key =
|
|
2418
|
+
blocker.kind === "Pod"
|
|
2419
|
+
? `${blocker.reason} ${blocker.severity} ${blocker.message}`
|
|
2420
|
+
: undefined;
|
|
2421
|
+
const existing = key ? merged.get(key) : undefined;
|
|
2422
|
+
if (existing) {
|
|
2423
|
+
if (!existing.pods.includes(blocker.name)) {
|
|
2424
|
+
existing.pods.push(blocker.name);
|
|
2425
|
+
}
|
|
2426
|
+
continue;
|
|
2427
|
+
}
|
|
2428
|
+
const entry = { blocker, pods: [blocker.name] };
|
|
2429
|
+
if (key) merged.set(key, entry);
|
|
2430
|
+
blockerGroups.push(entry);
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2071
2433
|
for (const item of related) {
|
|
2072
2434
|
// `diagnose` is itself scoped to one resource and declares these rows as
|
|
2073
2435
|
// related evidence. That producer contract is stronger than a broad
|
|
2074
2436
|
// `issues` query, even when the related row is a child Pod.
|
|
2075
|
-
|
|
2437
|
+
// A classified issue and the pods' startup blocker are the same fact
|
|
2438
|
+
// seen from the workload and from its pods; one card carries both.
|
|
2439
|
+
const folded = blockerGroups.find(
|
|
2440
|
+
(group) =>
|
|
2441
|
+
!group.foldedInto &&
|
|
2442
|
+
group.blocker.kind === "Pod" &&
|
|
2443
|
+
group.blocker.reason === item.reason &&
|
|
2444
|
+
(group.blocker.message === item.message ||
|
|
2445
|
+
group.blocker.message === item.cause),
|
|
2446
|
+
);
|
|
2447
|
+
if (folded) folded.foldedInto = item;
|
|
2448
|
+
addIssueObservation(builder, source, item, relatedRelevance, folded?.pods);
|
|
2076
2449
|
}
|
|
2077
2450
|
|
|
2078
2451
|
const context = contextFrom(value.resourceContext);
|
|
@@ -2097,14 +2470,12 @@ function adaptDiagnose(
|
|
|
2097
2470
|
tier: evidenceTierForRelevance("checked", bundleRelevance),
|
|
2098
2471
|
relevance: bundleRelevance,
|
|
2099
2472
|
tone: "neutral",
|
|
2100
|
-
title: "
|
|
2473
|
+
title: "Radar's diagnosis found no live issues",
|
|
2101
2474
|
summary: scope,
|
|
2102
2475
|
data: {
|
|
2103
2476
|
type: "receipt",
|
|
2104
2477
|
checked: "issues",
|
|
2105
2478
|
scope,
|
|
2106
|
-
message:
|
|
2107
|
-
"Radar's workload diagnosis completed without a classified live issue for this resource.",
|
|
2108
2479
|
},
|
|
2109
2480
|
});
|
|
2110
2481
|
}
|
|
@@ -2120,18 +2491,19 @@ function adaptDiagnose(
|
|
|
2120
2491
|
bundleRelevance,
|
|
2121
2492
|
);
|
|
2122
2493
|
|
|
2123
|
-
const blockersRaw = value.startupBlockers;
|
|
2124
|
-
let blockersValid = blockersRaw === undefined || Array.isArray(blockersRaw);
|
|
2125
2494
|
if (Array.isArray(blockersRaw)) {
|
|
2126
|
-
for (const
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2495
|
+
for (const { blocker, pods, foldedInto } of blockerGroups) {
|
|
2496
|
+
if (foldedInto) continue;
|
|
2497
|
+
// One blocker keeps one identity however many pods share it. Keying a
|
|
2498
|
+
// merged card differently from a single-pod one made a re-diagnose that
|
|
2499
|
+
// crossed one pod look like a new fact, so the same reason appeared
|
|
2500
|
+
// twice; a Pod blocker is the same finding whether it holds one pod or
|
|
2501
|
+
// nine.
|
|
2502
|
+
const grouped = blocker.kind === "Pod" && pods.length > 1;
|
|
2133
2503
|
builder.observe(
|
|
2134
|
-
|
|
2504
|
+
blocker.kind === "Pod"
|
|
2505
|
+
? `startup:Pod:${blocker.reason}:${fnv1a32(blocker.message).toString(36)}`
|
|
2506
|
+
: `startup:${blocker.kind}:${blocker.name}:${blocker.reason}`,
|
|
2135
2507
|
"startup",
|
|
2136
2508
|
source,
|
|
2137
2509
|
{
|
|
@@ -2142,30 +2514,39 @@ function adaptDiagnose(
|
|
|
2142
2514
|
relevance: bundledRowRelevance(blocker.kind, blocker.name),
|
|
2143
2515
|
tone: diagnosisSeverityTone(blocker.severity),
|
|
2144
2516
|
title: blocker.reason,
|
|
2145
|
-
summary:
|
|
2517
|
+
summary: grouped
|
|
2518
|
+
? `${pods.length} pods · ${blocker.message}`
|
|
2519
|
+
: blocker.message,
|
|
2146
2520
|
data: {
|
|
2147
2521
|
type: "startup",
|
|
2148
2522
|
blocker,
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2523
|
+
// Always carry the pods, whether one or nine: they are what puts
|
|
2524
|
+
// this workload's pods into the set a later Prometheus query is
|
|
2525
|
+
// proved against, and a merged card that dropped them quietly
|
|
2526
|
+
// weakened attribution elsewhere.
|
|
2527
|
+
pods,
|
|
2528
|
+
subject: grouped
|
|
2529
|
+
? undefined
|
|
2530
|
+
: (() => {
|
|
2531
|
+
const namespace = resource.metadata.namespace;
|
|
2532
|
+
if (!namespace) return undefined;
|
|
2533
|
+
const rootGroup = apiVersionToGroup(resource.apiVersion);
|
|
2534
|
+
const group =
|
|
2535
|
+
blocker.kind === resource.kind
|
|
2536
|
+
? rootGroup
|
|
2537
|
+
: blocker.kind === "Pod"
|
|
2538
|
+
? ""
|
|
2539
|
+
: blocker.kind === "ReplicaSet"
|
|
2540
|
+
? "apps"
|
|
2541
|
+
: undefined;
|
|
2542
|
+
if (group === undefined) return undefined;
|
|
2543
|
+
return {
|
|
2544
|
+
kind: blocker.kind,
|
|
2545
|
+
...(group ? { group } : {}),
|
|
2546
|
+
namespace,
|
|
2547
|
+
name: blocker.name,
|
|
2548
|
+
};
|
|
2549
|
+
})(),
|
|
2169
2550
|
},
|
|
2170
2551
|
},
|
|
2171
2552
|
);
|
|
@@ -2217,7 +2598,7 @@ function adaptDiagnose(
|
|
|
2217
2598
|
builder.limit(
|
|
2218
2599
|
source,
|
|
2219
2600
|
"Crash evidence",
|
|
2220
|
-
"
|
|
2601
|
+
"Radar returned part of the crash-cause candidates.",
|
|
2221
2602
|
"truncated",
|
|
2222
2603
|
);
|
|
2223
2604
|
}
|
|
@@ -2268,7 +2649,7 @@ function adaptDiagnose(
|
|
|
2268
2649
|
tier: evidenceTierForRelevance("checked", logRelevance),
|
|
2269
2650
|
relevance: logRelevance,
|
|
2270
2651
|
tone: "neutral",
|
|
2271
|
-
title: "No previous
|
|
2652
|
+
title: "No previous logs expected",
|
|
2272
2653
|
summary: `${item.pod} / ${item.container}`,
|
|
2273
2654
|
data: {
|
|
2274
2655
|
type: "receipt",
|
|
@@ -2450,6 +2831,11 @@ function adaptDiagnose(
|
|
|
2450
2831
|
if (value.changeContext !== undefined && !changeContext) {
|
|
2451
2832
|
invalidPayload(builder, source, "Change correlation");
|
|
2452
2833
|
}
|
|
2834
|
+
const diagnoseChangesSubject = {
|
|
2835
|
+
kind: resource.kind,
|
|
2836
|
+
namespace: resource.metadata.namespace,
|
|
2837
|
+
name: resource.metadata.name,
|
|
2838
|
+
};
|
|
2453
2839
|
if (Array.isArray(changesRaw)) {
|
|
2454
2840
|
const changes = changesRaw
|
|
2455
2841
|
.map(recentChange)
|
|
@@ -2466,6 +2852,7 @@ function adaptDiagnose(
|
|
|
2466
2852
|
changesCoverageLimitedValid,
|
|
2467
2853
|
true,
|
|
2468
2854
|
relatedRelevance,
|
|
2855
|
+
diagnoseChangesSubject,
|
|
2469
2856
|
);
|
|
2470
2857
|
} else {
|
|
2471
2858
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2486,6 +2873,7 @@ function adaptDiagnose(
|
|
|
2486
2873
|
changesCoverageLimitedValid,
|
|
2487
2874
|
true,
|
|
2488
2875
|
relatedRelevance,
|
|
2876
|
+
diagnoseChangesSubject,
|
|
2489
2877
|
);
|
|
2490
2878
|
} else if (changesRaw !== undefined) {
|
|
2491
2879
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2493,6 +2881,21 @@ function adaptDiagnose(
|
|
|
2493
2881
|
if (nonEmptyString(value.recentChangesError)) {
|
|
2494
2882
|
builder.limit(source, "Recent changes", value.recentChangesError, "error");
|
|
2495
2883
|
}
|
|
2884
|
+
|
|
2885
|
+
addDiagnoseMetrics(
|
|
2886
|
+
builder,
|
|
2887
|
+
source,
|
|
2888
|
+
value.metrics,
|
|
2889
|
+
{
|
|
2890
|
+
kind: resource.kind,
|
|
2891
|
+
...(apiVersionToGroup(resource.apiVersion)
|
|
2892
|
+
? { group: apiVersionToGroup(resource.apiVersion) }
|
|
2893
|
+
: {}),
|
|
2894
|
+
namespace: resource.metadata.namespace,
|
|
2895
|
+
name: resource.metadata.name,
|
|
2896
|
+
},
|
|
2897
|
+
bundleRelevance,
|
|
2898
|
+
);
|
|
2496
2899
|
if (value.recentChangesSaturated === true) {
|
|
2497
2900
|
builder.limit(
|
|
2498
2901
|
source,
|
|
@@ -2681,14 +3084,12 @@ function adaptIssues(
|
|
|
2681
3084
|
tier: evidenceTierForRelevance("checked", relevance),
|
|
2682
3085
|
relevance,
|
|
2683
3086
|
tone: "neutral",
|
|
2684
|
-
title: "No
|
|
3087
|
+
title: "No live issues matched this search",
|
|
2685
3088
|
summary: scope,
|
|
2686
3089
|
data: {
|
|
2687
3090
|
type: "receipt",
|
|
2688
3091
|
checked: "issues",
|
|
2689
3092
|
scope,
|
|
2690
|
-
message:
|
|
2691
|
-
"Radar's live-issue query completed and returned no matching issues.",
|
|
2692
3093
|
},
|
|
2693
3094
|
});
|
|
2694
3095
|
} else {
|
|
@@ -2784,6 +3185,11 @@ function adaptGetResource(
|
|
|
2784
3185
|
invalidPayload(builder, source, "Events");
|
|
2785
3186
|
}
|
|
2786
3187
|
}
|
|
3188
|
+
const resourceChangesSubject = {
|
|
3189
|
+
kind: resource.kind,
|
|
3190
|
+
namespace: resource.metadata.namespace,
|
|
3191
|
+
name: resource.metadata.name,
|
|
3192
|
+
};
|
|
2787
3193
|
const recentChangesRaw = value.recentChanges;
|
|
2788
3194
|
const recentChangesSaturatedRaw = value.recentChangesSaturated;
|
|
2789
3195
|
const recentChangesCoverageLimitedRaw = value.recentChangesCoverageLimited;
|
|
@@ -2814,6 +3220,7 @@ function adaptGetResource(
|
|
|
2814
3220
|
!nonEmptyString(value.recentChangesError),
|
|
2815
3221
|
true,
|
|
2816
3222
|
relevance,
|
|
3223
|
+
resourceChangesSubject,
|
|
2817
3224
|
);
|
|
2818
3225
|
} else {
|
|
2819
3226
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2833,6 +3240,7 @@ function adaptGetResource(
|
|
|
2833
3240
|
recentChangesCoverageLimitedRaw === false,
|
|
2834
3241
|
true,
|
|
2835
3242
|
relevance,
|
|
3243
|
+
resourceChangesSubject,
|
|
2836
3244
|
);
|
|
2837
3245
|
} else if (recentChangesRaw !== undefined) {
|
|
2838
3246
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2891,7 +3299,7 @@ function adaptListResources(
|
|
|
2891
3299
|
builder.limit(
|
|
2892
3300
|
source,
|
|
2893
3301
|
"Resource inventory",
|
|
2894
|
-
`Radar found no matching resources for ${scope}
|
|
3302
|
+
`Radar found no matching resources for ${scope}. Anything you do not have permission to read was not searched.`,
|
|
2895
3303
|
"unknown",
|
|
2896
3304
|
);
|
|
2897
3305
|
return;
|
|
@@ -2920,27 +3328,82 @@ function adaptEvents(
|
|
|
2920
3328
|
payload: unknown,
|
|
2921
3329
|
): void {
|
|
2922
3330
|
const value = record(payload);
|
|
2923
|
-
|
|
3331
|
+
// The producer serializes an empty result as a nil slice, which is null.
|
|
3332
|
+
const eventsRaw = value?.events === null ? [] : value?.events;
|
|
3333
|
+
if (!value || !Array.isArray(eventsRaw)) {
|
|
2924
3334
|
invalidPayload(builder, source);
|
|
2925
3335
|
return;
|
|
2926
3336
|
}
|
|
2927
|
-
const events =
|
|
3337
|
+
const events = eventsRaw
|
|
2928
3338
|
.map(event)
|
|
2929
3339
|
.filter((item): item is InvestigationEventEvidence => Boolean(item));
|
|
2930
|
-
if (events.length !==
|
|
3340
|
+
if (events.length !== eventsRaw.length) {
|
|
2931
3341
|
invalidPayload(builder, source);
|
|
2932
3342
|
return;
|
|
2933
3343
|
}
|
|
2934
3344
|
addNarrowHint(builder, source, value);
|
|
3345
|
+
// The producer answers a namespace the caller cannot read with an empty
|
|
3346
|
+
// list and marks it. Only that case is a coverage gap. Any other complete,
|
|
3347
|
+
// successful empty read answers the question it asked and is filed as a
|
|
3348
|
+
// checked receipt; a gap there would contradict an events card from another
|
|
3349
|
+
// call in the same turn. The receipt names the one remaining ambiguity for
|
|
3350
|
+
// producers that predate the marker.
|
|
3351
|
+
if (value.accessDenied === true) {
|
|
3352
|
+
builder.limit(
|
|
3353
|
+
source,
|
|
3354
|
+
"Events",
|
|
3355
|
+
`Events in ${scopeFromArgs(source)} are not readable with your permissions.`,
|
|
3356
|
+
"error",
|
|
3357
|
+
);
|
|
3358
|
+
return;
|
|
3359
|
+
}
|
|
3360
|
+
// A cluster-wide read the producer narrowed to the caller's namespaces
|
|
3361
|
+
// answers for those alone, so neither its empty receipt nor its card may
|
|
3362
|
+
// stand for the cluster.
|
|
3363
|
+
const narrowedTo = narrowedEventScope(value);
|
|
3364
|
+
if (narrowedTo && events.length > 0) {
|
|
3365
|
+
builder.limit(
|
|
3366
|
+
source,
|
|
3367
|
+
"Events",
|
|
3368
|
+
`This cluster-wide events read covered only the namespaces you can read (${narrowedTo}).`,
|
|
3369
|
+
"unknown",
|
|
3370
|
+
);
|
|
3371
|
+
}
|
|
2935
3372
|
addEvents(
|
|
2936
3373
|
builder,
|
|
2937
3374
|
source,
|
|
2938
3375
|
events,
|
|
2939
3376
|
`events:${source.args ?? scopeFromArgs(source)}`,
|
|
2940
3377
|
!nonEmptyString(value.narrowHint),
|
|
2941
|
-
|
|
3378
|
+
true,
|
|
2942
3379
|
sourceArgsRelevance(builder, source),
|
|
3380
|
+
narrowedTo
|
|
3381
|
+
? {
|
|
3382
|
+
title: "No events in the namespaces you can read",
|
|
3383
|
+
message: `Read ${narrowedTo}. Namespaces outside your permissions were not read, so this does not clear the cluster.`,
|
|
3384
|
+
}
|
|
3385
|
+
: {
|
|
3386
|
+
title: "No events in this window",
|
|
3387
|
+
message:
|
|
3388
|
+
"Events outside this window or its filters are not covered, and a namespace you cannot read also returns nothing.",
|
|
3389
|
+
},
|
|
3390
|
+
);
|
|
3391
|
+
}
|
|
3392
|
+
|
|
3393
|
+
/**
|
|
3394
|
+
* Names the namespaces a cluster-wide events read was actually narrowed to,
|
|
3395
|
+
* or undefined when the read covered everything the query asked for.
|
|
3396
|
+
*/
|
|
3397
|
+
function narrowedEventScope(
|
|
3398
|
+
value: Record<string, unknown>,
|
|
3399
|
+
): string | undefined {
|
|
3400
|
+
if (value.partialScope !== true) return undefined;
|
|
3401
|
+
const namespaces = (stringArray(value.scopeNamespaces) ?? []).filter(
|
|
3402
|
+
nonEmptyString,
|
|
2943
3403
|
);
|
|
3404
|
+
return namespaces.length > 0
|
|
3405
|
+
? namespaces.join(", ")
|
|
3406
|
+
: "the namespaces you can read";
|
|
2944
3407
|
}
|
|
2945
3408
|
|
|
2946
3409
|
function adaptPodLogs(
|
|
@@ -2978,14 +3441,16 @@ function adaptChanges(
|
|
|
2978
3441
|
payload: unknown,
|
|
2979
3442
|
): void {
|
|
2980
3443
|
const value = record(payload);
|
|
2981
|
-
|
|
3444
|
+
// The producer serializes an empty result as a nil slice, which is null.
|
|
3445
|
+
const changesRaw = value?.changes === null ? [] : value?.changes;
|
|
3446
|
+
if (!value || !Array.isArray(changesRaw)) {
|
|
2982
3447
|
invalidPayload(builder, source, "Recent changes");
|
|
2983
3448
|
return;
|
|
2984
3449
|
}
|
|
2985
|
-
const changes =
|
|
3450
|
+
const changes = changesRaw
|
|
2986
3451
|
.map(recentChange)
|
|
2987
3452
|
.filter((item): item is IssueRecentChange => Boolean(item));
|
|
2988
|
-
if (changes.length !==
|
|
3453
|
+
if (changes.length !== changesRaw.length) {
|
|
2989
3454
|
invalidPayload(builder, source, "Recent changes");
|
|
2990
3455
|
return;
|
|
2991
3456
|
}
|
|
@@ -3001,15 +3466,20 @@ function adaptChanges(
|
|
|
3001
3466
|
} else if (value.sourcesErrored !== undefined) {
|
|
3002
3467
|
invalidPayload(builder, source, "Recent changes source coverage");
|
|
3003
3468
|
}
|
|
3469
|
+
// Reads of one resource's history are revisions of one card however the
|
|
3470
|
+
// window or cap differs; the window is stated in the title instead.
|
|
3471
|
+
const args = record(source.args ? parseJSON(source.args) : undefined);
|
|
3004
3472
|
addChanges(
|
|
3005
3473
|
builder,
|
|
3006
3474
|
source,
|
|
3007
3475
|
changes,
|
|
3008
|
-
`changes:${
|
|
3476
|
+
`changes:${scopeFromArgs(source)}`,
|
|
3009
3477
|
undefined,
|
|
3010
3478
|
!nonEmptyString(value.narrowHint) && sourceErrors === 0,
|
|
3011
3479
|
false,
|
|
3012
3480
|
sourceArgsRelevance(builder, source),
|
|
3481
|
+
changesSubjectFromArgs(source),
|
|
3482
|
+
nonEmptyString(args?.since) ? args.since : undefined,
|
|
3013
3483
|
);
|
|
3014
3484
|
}
|
|
3015
3485
|
|
|
@@ -3199,87 +3669,1680 @@ function adaptTopology(
|
|
|
3199
3669
|
);
|
|
3200
3670
|
}
|
|
3201
3671
|
|
|
3202
|
-
const
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
> = {
|
|
3210
|
-
diagnose: adaptDiagnose,
|
|
3211
|
-
issues: adaptIssues,
|
|
3212
|
-
get_resource: adaptGetResource,
|
|
3213
|
-
list_resources: adaptListResources,
|
|
3214
|
-
get_events: adaptEvents,
|
|
3215
|
-
get_pod_logs: adaptPodLogs,
|
|
3216
|
-
get_changes: adaptChanges,
|
|
3217
|
-
get_neighborhood: adaptNeighborhood,
|
|
3218
|
-
get_topology: adaptTopology,
|
|
3672
|
+
const WORKLOAD_LOG_KINDS: Readonly<Record<string, string>> = {
|
|
3673
|
+
deployments: "Deployment",
|
|
3674
|
+
statefulsets: "StatefulSet",
|
|
3675
|
+
daemonsets: "DaemonSet",
|
|
3676
|
+
rollouts: "Rollout",
|
|
3677
|
+
jobs: "Job",
|
|
3678
|
+
workflows: "Workflow",
|
|
3219
3679
|
};
|
|
3220
3680
|
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
):
|
|
3225
|
-
const
|
|
3226
|
-
|
|
3227
|
-
const
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3681
|
+
/** `get_workload_logs` states what it read as `<plural>/<namespace>/<name>`. */
|
|
3682
|
+
function workloadLogsSubject(
|
|
3683
|
+
value: string,
|
|
3684
|
+
): { kind: string; namespace: string; name: string } | undefined {
|
|
3685
|
+
const parts = value.split("/");
|
|
3686
|
+
if (parts.length !== 3) return undefined;
|
|
3687
|
+
const kind = WORKLOAD_LOG_KINDS[parts[0]];
|
|
3688
|
+
if (!kind || !parts[1] || !parts[2]) return undefined;
|
|
3689
|
+
return { kind, namespace: parts[1], name: parts[2] };
|
|
3690
|
+
}
|
|
3691
|
+
|
|
3692
|
+
function adaptWorkloadLogs(
|
|
3693
|
+
builder: ProjectionBuilder,
|
|
3694
|
+
source: InvestigationEvidenceSource,
|
|
3695
|
+
payload: unknown,
|
|
3696
|
+
): void {
|
|
3697
|
+
const value = record(payload);
|
|
3698
|
+
const workload = nonEmptyString(value?.workload)
|
|
3699
|
+
? workloadLogsSubject(value.workload)
|
|
3700
|
+
: undefined;
|
|
3701
|
+
if (!value || !workload || !nonNegativeInteger(value.pods)) {
|
|
3702
|
+
invalidPayload(builder, source);
|
|
3703
|
+
return;
|
|
3704
|
+
}
|
|
3705
|
+
// The producer names the workload without an API group; as with other
|
|
3706
|
+
// group-less producers, the investigation target supplies that dimension.
|
|
3707
|
+
const workloadRelevance = relevanceForResource(builder, {
|
|
3708
|
+
...workload,
|
|
3709
|
+
group: builder.target.group,
|
|
3710
|
+
});
|
|
3711
|
+
const scope = `${displayKind(workload.kind)} ${workload.namespace}/${workload.name}`;
|
|
3712
|
+
const previous = previousFromArgs(source);
|
|
3713
|
+
addNarrowHint(builder, source, value);
|
|
3714
|
+
if (nonEmptyString(value.logsError)) {
|
|
3715
|
+
builder.limit(source, "Workload logs", value.logsError, "error");
|
|
3716
|
+
return;
|
|
3717
|
+
}
|
|
3718
|
+
if (value.pods === 0) {
|
|
3719
|
+
// The producer replaces the stream list with a message when it resolved
|
|
3720
|
+
// no pods; any other shape is not this response.
|
|
3721
|
+
if (
|
|
3722
|
+
typeof value.logs !== "string" ||
|
|
3723
|
+
(value.emptyMessage !== undefined &&
|
|
3724
|
+
typeof value.emptyMessage !== "string") ||
|
|
3725
|
+
value.narrowHint !== undefined
|
|
3726
|
+
) {
|
|
3727
|
+
invalidPayload(builder, source);
|
|
3728
|
+
return;
|
|
3729
|
+
}
|
|
3730
|
+
if (!source.confirmedSuccess) return;
|
|
3731
|
+
// The producer's own sentence when it gave one; otherwise nothing, because
|
|
3732
|
+
// "no pods to read logs from" is already the whole answer.
|
|
3733
|
+
const message = nonEmptyString(value.emptyMessage)
|
|
3734
|
+
? value.emptyMessage
|
|
3735
|
+
: nonEmptyString(value.logs)
|
|
3736
|
+
? value.logs
|
|
3737
|
+
: undefined;
|
|
3738
|
+
builder.observe(
|
|
3739
|
+
`workload-logs:${previous ? "previous" : "current"}:${scope}`,
|
|
3740
|
+
"receipt",
|
|
3741
|
+
source,
|
|
3742
|
+
{
|
|
3743
|
+
tier: evidenceTierForRelevance("checked", workloadRelevance),
|
|
3744
|
+
relevance: workloadRelevance,
|
|
3745
|
+
tone: "neutral",
|
|
3746
|
+
title: "No pods to read logs from",
|
|
3747
|
+
summary: scope,
|
|
3748
|
+
data: { type: "receipt", checked: "logs", scope, message },
|
|
3749
|
+
},
|
|
3750
|
+
);
|
|
3751
|
+
return;
|
|
3752
|
+
}
|
|
3753
|
+
const logsRaw = value.logs;
|
|
3754
|
+
const noStreams = `Radar found ${value.pods} pod${value.pods === 1 ? "" : "s"} but got no logs from ${value.pods === 1 ? "it" : "them"}, so the logs were not checked.`;
|
|
3755
|
+
if (!Array.isArray(logsRaw)) {
|
|
3756
|
+
if (logsRaw === undefined || logsRaw === null) {
|
|
3757
|
+
builder.limit(source, "Workload logs", noStreams, "unknown");
|
|
3758
|
+
} else {
|
|
3759
|
+
invalidPayload(builder, source);
|
|
3760
|
+
}
|
|
3761
|
+
return;
|
|
3762
|
+
}
|
|
3763
|
+
const warnings = stringArray(value.warnings) ?? [];
|
|
3764
|
+
for (const raw of logsRaw) {
|
|
3765
|
+
const entry = parseLogEntry(raw);
|
|
3766
|
+
if (!entry) {
|
|
3767
|
+
invalidPayload(
|
|
3768
|
+
builder,
|
|
3769
|
+
source,
|
|
3770
|
+
previous ? "Previous logs" : "Current logs",
|
|
3771
|
+
);
|
|
3772
|
+
continue;
|
|
3773
|
+
}
|
|
3774
|
+
// A Pod target is named exactly by its own row; a workload target relates
|
|
3775
|
+
// to its rows the way diagnose relates to the pods it resolved.
|
|
3776
|
+
const rowRelevance: InvestigationEvidenceRelevance = resourceMatchesTarget(
|
|
3777
|
+
builder.target,
|
|
3778
|
+
{
|
|
3779
|
+
kind: "Pod",
|
|
3780
|
+
group: "",
|
|
3781
|
+
namespace: workload.namespace,
|
|
3782
|
+
name: entry.pod,
|
|
3783
|
+
},
|
|
3784
|
+
)
|
|
3785
|
+
? "target"
|
|
3786
|
+
: workloadRelevance === "target"
|
|
3787
|
+
? "producer-related"
|
|
3788
|
+
: "broader";
|
|
3789
|
+
addLogs(
|
|
3790
|
+
builder,
|
|
3791
|
+
source,
|
|
3792
|
+
entry,
|
|
3793
|
+
previous,
|
|
3794
|
+
warnings,
|
|
3795
|
+
rowRelevance,
|
|
3796
|
+
workload.namespace,
|
|
3797
|
+
);
|
|
3798
|
+
}
|
|
3799
|
+
if (logsRaw.length === 0) {
|
|
3800
|
+
builder.limit(source, "Workload logs", noStreams, "unknown");
|
|
3801
|
+
}
|
|
3802
|
+
}
|
|
3803
|
+
|
|
3804
|
+
function stringRecord(value: unknown): Record<string, string> | undefined {
|
|
3805
|
+
const candidate = record(value);
|
|
3806
|
+
if (!candidate) return undefined;
|
|
3807
|
+
return Object.values(candidate).every((item) => typeof item === "string")
|
|
3808
|
+
? (candidate as Record<string, string>)
|
|
3809
|
+
: undefined;
|
|
3810
|
+
}
|
|
3811
|
+
|
|
3812
|
+
/**
|
|
3813
|
+
* Pods a producer has already tied to the investigated workload in this
|
|
3814
|
+
* projection: the streams diagnose or a workload-logs read resolved for it,
|
|
3815
|
+
* its crash candidates, and its Pod startup blockers. A pod name matched
|
|
3816
|
+
* against this set is producer-established membership, never inference from
|
|
3817
|
+
* a name prefix, which a sibling such as `api-worker` would also satisfy.
|
|
3818
|
+
*/
|
|
3819
|
+
function producerEstablishedTargetPods(
|
|
3820
|
+
builder: ProjectionBuilder,
|
|
3821
|
+
): Set<string> {
|
|
3822
|
+
const pods = new Set<string>(builder.establishedTargetPods);
|
|
3823
|
+
const namespace = builder.target.namespace;
|
|
3824
|
+
if (!namespace) return pods;
|
|
3825
|
+
for (const group of builder.groups) {
|
|
3826
|
+
for (const observation of group.observations) {
|
|
3827
|
+
if (observation.relevance === "broader") continue;
|
|
3828
|
+
const data = observation.data;
|
|
3829
|
+
if (data.type === "logs" && data.namespace === namespace) {
|
|
3830
|
+
pods.add(data.pod);
|
|
3831
|
+
} else if (data.type === "crash" && data.namespace === namespace) {
|
|
3832
|
+
for (const pod of data.crash.pods) pods.add(pod);
|
|
3833
|
+
} else if (data.type === "startup" && data.blocker.kind === "Pod") {
|
|
3834
|
+
// The blocker names the pods it holds whether the card merged them or
|
|
3835
|
+
// not; the subject is only set when there is exactly one, so reading
|
|
3836
|
+
// it alone lost the whole set the moment a second pod appeared.
|
|
3837
|
+
for (const pod of data.pods ?? []) pods.add(pod);
|
|
3838
|
+
}
|
|
3839
|
+
}
|
|
3840
|
+
}
|
|
3841
|
+
return pods;
|
|
3842
|
+
}
|
|
3843
|
+
|
|
3844
|
+
// kube-state-metrics label names for the owning workload.
|
|
3845
|
+
const WORKLOAD_LABEL_BY_KIND: Readonly<Record<string, string>> = {
|
|
3846
|
+
deployment: "deployment",
|
|
3847
|
+
statefulset: "statefulset",
|
|
3848
|
+
daemonset: "daemonset",
|
|
3849
|
+
rollout: "rollout",
|
|
3850
|
+
job: "job_name",
|
|
3851
|
+
cronjob: "cronjob",
|
|
3852
|
+
};
|
|
3853
|
+
|
|
3854
|
+
/**
|
|
3855
|
+
* Whether a Prometheus label set names the investigated resource: the target
|
|
3856
|
+
* namespace plus a label identifying it directly, or a `pod` label naming a
|
|
3857
|
+
* pod a producer already tied to it.
|
|
3858
|
+
*/
|
|
3859
|
+
function labelsNameTarget(
|
|
3860
|
+
target: InvestigationEvidenceTarget,
|
|
3861
|
+
labels: Record<string, string>,
|
|
3862
|
+
targetPods: ReadonlySet<string>,
|
|
3863
|
+
): boolean {
|
|
3864
|
+
if (!target.namespace || labels.namespace !== target.namespace) return false;
|
|
3865
|
+
const kind = target.kind.toLowerCase();
|
|
3866
|
+
if (kind === "pod") return labels.pod === target.name;
|
|
3867
|
+
const workloadLabel = WORKLOAD_LABEL_BY_KIND[kind];
|
|
3868
|
+
if (workloadLabel && labels[workloadLabel] === target.name) return true;
|
|
3869
|
+
if (
|
|
3870
|
+
labels.workload === target.name &&
|
|
3871
|
+
(labels.workload_type === undefined ||
|
|
3872
|
+
labels.workload_type.toLowerCase() === kind)
|
|
3873
|
+
) {
|
|
3874
|
+
return true;
|
|
3875
|
+
}
|
|
3876
|
+
return labels.pod !== undefined && targetPods.has(labels.pod);
|
|
3877
|
+
}
|
|
3878
|
+
|
|
3879
|
+
/**
|
|
3880
|
+
* The label set places the instance in another namespace outright. Only a
|
|
3881
|
+
* namespaced target can be excluded this way; for a cluster-scoped target
|
|
3882
|
+
* every namespace label is "different" and proves nothing.
|
|
3883
|
+
*/
|
|
3884
|
+
function labelsPlaceElsewhere(
|
|
3885
|
+
target: InvestigationEvidenceTarget,
|
|
3886
|
+
labels: Record<string, string>,
|
|
3887
|
+
): boolean {
|
|
3888
|
+
return (
|
|
3889
|
+
Boolean(target.namespace) &&
|
|
3890
|
+
nonEmptyString(labels.namespace) &&
|
|
3891
|
+
labels.namespace !== target.namespace
|
|
3892
|
+
);
|
|
3893
|
+
}
|
|
3894
|
+
|
|
3895
|
+
function alertRule(value: unknown):
|
|
3896
|
+
| {
|
|
3897
|
+
rule: InvestigationAlertRule;
|
|
3898
|
+
instances: Omit<InvestigationAlertInstance, "namesTarget">[];
|
|
3899
|
+
annotations: Record<string, string>;
|
|
3900
|
+
}
|
|
3901
|
+
| undefined {
|
|
3902
|
+
const candidate = record(value);
|
|
3903
|
+
if (
|
|
3904
|
+
!candidate ||
|
|
3905
|
+
!nonEmptyString(candidate.group) ||
|
|
3906
|
+
!nonEmptyString(candidate.name) ||
|
|
3907
|
+
!nonEmptyString(candidate.type) ||
|
|
3908
|
+
typeof candidate.query !== "string"
|
|
3909
|
+
) {
|
|
3910
|
+
return undefined;
|
|
3911
|
+
}
|
|
3912
|
+
for (const field of ["state", "health"] as const) {
|
|
3913
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
3914
|
+
return undefined;
|
|
3915
|
+
}
|
|
3916
|
+
const labels =
|
|
3917
|
+
candidate.labels === undefined ? {} : stringRecord(candidate.labels);
|
|
3918
|
+
const annotations =
|
|
3919
|
+
candidate.annotations === undefined
|
|
3920
|
+
? {}
|
|
3921
|
+
: stringRecord(candidate.annotations);
|
|
3922
|
+
if (!labels || !annotations) return undefined;
|
|
3923
|
+
const alertsRaw = candidate.alerts === undefined ? [] : candidate.alerts;
|
|
3924
|
+
if (!Array.isArray(alertsRaw)) return undefined;
|
|
3925
|
+
const instances: Omit<InvestigationAlertInstance, "namesTarget">[] = [];
|
|
3926
|
+
for (const raw of alertsRaw) {
|
|
3927
|
+
const instance = record(raw);
|
|
3928
|
+
const instanceLabels =
|
|
3929
|
+
instance?.labels === undefined ? {} : stringRecord(instance.labels);
|
|
3930
|
+
if (
|
|
3931
|
+
!instance ||
|
|
3932
|
+
!nonEmptyString(instance.state) ||
|
|
3933
|
+
!instanceLabels ||
|
|
3934
|
+
(instance.activeAt !== undefined &&
|
|
3935
|
+
typeof instance.activeAt !== "string") ||
|
|
3936
|
+
(instance.value !== undefined && typeof instance.value !== "string")
|
|
3937
|
+
) {
|
|
3938
|
+
return undefined;
|
|
3939
|
+
}
|
|
3940
|
+
instances.push({
|
|
3941
|
+
state: instance.state,
|
|
3942
|
+
activeAt: instance.activeAt as string | undefined,
|
|
3943
|
+
value: instance.value as string | undefined,
|
|
3944
|
+
labels: instanceLabels,
|
|
3945
|
+
});
|
|
3946
|
+
}
|
|
3947
|
+
return {
|
|
3948
|
+
rule: {
|
|
3949
|
+
group: candidate.group,
|
|
3950
|
+
name: candidate.name,
|
|
3951
|
+
type: candidate.type,
|
|
3952
|
+
state: candidate.state as string | undefined,
|
|
3953
|
+
health: candidate.health as string | undefined,
|
|
3954
|
+
query: candidate.query,
|
|
3955
|
+
labels,
|
|
3956
|
+
},
|
|
3957
|
+
instances,
|
|
3958
|
+
annotations,
|
|
3959
|
+
};
|
|
3960
|
+
}
|
|
3961
|
+
|
|
3962
|
+
function targetIdentity(target: InvestigationEvidenceTarget): string {
|
|
3963
|
+
return `${displayKind(target.kind)} ${target.namespace ? `${target.namespace}/` : ""}${target.name}`;
|
|
3964
|
+
}
|
|
3965
|
+
|
|
3966
|
+
function adaptPrometheusRules(
|
|
3967
|
+
builder: ProjectionBuilder,
|
|
3968
|
+
source: InvestigationEvidenceSource,
|
|
3969
|
+
payload: unknown,
|
|
3970
|
+
): void {
|
|
3971
|
+
const value = record(payload);
|
|
3972
|
+
if (
|
|
3973
|
+
!value ||
|
|
3974
|
+
!Array.isArray(value.rules) ||
|
|
3975
|
+
typeof value.count !== "number" ||
|
|
3976
|
+
value.count !== value.rules.length ||
|
|
3977
|
+
(value.truncated !== undefined && typeof value.truncated !== "boolean") ||
|
|
3978
|
+
(value.note !== undefined && typeof value.note !== "string")
|
|
3979
|
+
) {
|
|
3980
|
+
invalidPayload(builder, source);
|
|
3981
|
+
return;
|
|
3982
|
+
}
|
|
3983
|
+
const rules = value.rules
|
|
3984
|
+
.map(alertRule)
|
|
3985
|
+
.filter((item): item is NonNullable<typeof item> => Boolean(item));
|
|
3986
|
+
if (rules.length !== value.rules.length) {
|
|
3987
|
+
invalidPayload(builder, source);
|
|
3988
|
+
return;
|
|
3989
|
+
}
|
|
3990
|
+
const args = record(source.args ? parseJSON(source.args) : undefined);
|
|
3991
|
+
const stateFilter = nonEmptyString(args?.state)
|
|
3992
|
+
? args.state.toLowerCase()
|
|
3993
|
+
: undefined;
|
|
3994
|
+
const typeFilter = nonEmptyString(args?.type)
|
|
3995
|
+
? args.type.toLowerCase()
|
|
3996
|
+
: undefined;
|
|
3997
|
+
const truncated = value.truncated === true;
|
|
3998
|
+
if (truncated) {
|
|
3999
|
+
builder.limit(
|
|
4000
|
+
source,
|
|
4001
|
+
"Alert rules",
|
|
4002
|
+
`The rule list was capped, so additional matching rules may exist${nonEmptyString(value.note) ? `; ${value.note}` : ""}.`,
|
|
4003
|
+
"truncated",
|
|
4004
|
+
);
|
|
4005
|
+
}
|
|
4006
|
+
const identity = targetIdentity(builder.target);
|
|
4007
|
+
const targetPods = producerEstablishedTargetPods(builder);
|
|
4008
|
+
// Rule names are not unique within a group, and group names are unique
|
|
4009
|
+
// only per rule file, which the producer does not return. The expression
|
|
4010
|
+
// and static labels distinguish rules; identical rows within one response
|
|
4011
|
+
// cannot be told apart at all, so they stay scoped to this read rather
|
|
4012
|
+
// than inherit each other's history in whatever order they arrive.
|
|
4013
|
+
const ruleDefinition = (rule: InvestigationAlertRule) =>
|
|
4014
|
+
`alerts:${rule.group}:${rule.name}:${stableHash(
|
|
4015
|
+
JSON.stringify([rule.query, Object.entries(rule.labels).sort()]),
|
|
4016
|
+
)}`;
|
|
4017
|
+
const definitionCounts = new Map<string, number>();
|
|
4018
|
+
for (const { rule } of rules) {
|
|
4019
|
+
const definition = ruleDefinition(rule);
|
|
4020
|
+
definitionCounts.set(
|
|
4021
|
+
definition,
|
|
4022
|
+
(definitionCounts.get(definition) ?? 0) + 1,
|
|
4023
|
+
);
|
|
4024
|
+
}
|
|
4025
|
+
const seenRuleIdentities = new Map<string, number>();
|
|
4026
|
+
let namesTarget = false;
|
|
4027
|
+
let healthGap = false;
|
|
4028
|
+
let alertingCount = 0;
|
|
4029
|
+
let everyInstanceElsewhere = true;
|
|
4030
|
+
for (const { rule, instances: rawInstances, annotations } of rules) {
|
|
4031
|
+
// Recording rules carry no state and never describe a resource.
|
|
4032
|
+
if (rule.type.toLowerCase() !== "alerting") continue;
|
|
4033
|
+
alertingCount += 1;
|
|
4034
|
+
const instances = rawInstances.map((instance) => ({
|
|
4035
|
+
...instance,
|
|
4036
|
+
namesTarget: labelsNameTarget(
|
|
4037
|
+
builder.target,
|
|
4038
|
+
instance.labels,
|
|
4039
|
+
targetPods,
|
|
4040
|
+
),
|
|
4041
|
+
}));
|
|
4042
|
+
if (
|
|
4043
|
+
instances.length === 0 ||
|
|
4044
|
+
instances.some(
|
|
4045
|
+
(instance) => !labelsPlaceElsewhere(builder.target, instance.labels),
|
|
4046
|
+
)
|
|
4047
|
+
) {
|
|
4048
|
+
everyInstanceElsewhere = false;
|
|
4049
|
+
}
|
|
4050
|
+
const definition = ruleDefinition(rule);
|
|
4051
|
+
const ambiguous = (definitionCounts.get(definition) ?? 0) > 1;
|
|
4052
|
+
const ordinal = seenRuleIdentities.get(definition) ?? 0;
|
|
4053
|
+
seenRuleIdentities.set(definition, ordinal + 1);
|
|
4054
|
+
const ruleIdentity = ambiguous
|
|
4055
|
+
? `${definition}:${source.id}:${ordinal}`
|
|
4056
|
+
: definition;
|
|
4057
|
+
const previousRelevance = ambiguous
|
|
4058
|
+
? undefined
|
|
4059
|
+
: builder.latestRelevance("alerts", ruleIdentity);
|
|
4060
|
+
const relevance: InvestigationEvidenceRelevance = instances.some(
|
|
4061
|
+
(instance) => instance.namesTarget,
|
|
4062
|
+
)
|
|
4063
|
+
? "target"
|
|
4064
|
+
: labelsNameTarget(builder.target, rule.labels, targetPods)
|
|
4065
|
+
? "producer-related"
|
|
4066
|
+
: // The same rule read again with no instances left has resolved for
|
|
4067
|
+
// the target it named before; keep that provenance so the
|
|
4068
|
+
// resolution stays visible instead of the stale firing card.
|
|
4069
|
+
instances.length === 0 &&
|
|
4070
|
+
previousRelevance !== undefined &&
|
|
4071
|
+
previousRelevance !== "broader"
|
|
4072
|
+
? previousRelevance
|
|
4073
|
+
: "broader";
|
|
4074
|
+
if (relevance !== "broader") namesTarget = true;
|
|
4075
|
+
const state = (rule.state ?? "").toLowerCase();
|
|
4076
|
+
const active = state === "firing" || state === "pending";
|
|
4077
|
+
const severity = (rule.labels.severity ?? "").toLowerCase();
|
|
4078
|
+
const targetInstances = instances.filter(
|
|
4079
|
+
(instance) => instance.namesTarget,
|
|
4080
|
+
).length;
|
|
4081
|
+
builder.observe(ruleIdentity, "alerts", source, {
|
|
4082
|
+
tier: evidenceTierForRelevance(
|
|
4083
|
+
active ? "supporting" : "context",
|
|
4084
|
+
relevance,
|
|
4085
|
+
),
|
|
4086
|
+
relevance,
|
|
4087
|
+
tone:
|
|
4088
|
+
state === "firing"
|
|
4089
|
+
? severity === "critical"
|
|
4090
|
+
? "error"
|
|
4091
|
+
: "alert"
|
|
4092
|
+
: state === "pending"
|
|
4093
|
+
? "warning"
|
|
4094
|
+
: "neutral",
|
|
4095
|
+
title: `${rule.name}${state ? ` ${state}` : ""}`,
|
|
4096
|
+
summary:
|
|
4097
|
+
instances.length === 0
|
|
4098
|
+
? `No active instances · ${rule.group}`
|
|
4099
|
+
: `${instances.length} active instance${instances.length === 1 ? "" : "s"}${
|
|
4100
|
+
targetInstances > 0
|
|
4101
|
+
? `, ${targetInstances} naming ${identity}`
|
|
4102
|
+
: ""
|
|
4103
|
+
} · ${rule.group}`,
|
|
4104
|
+
data: { type: "alerts", rule, instances, annotations },
|
|
4105
|
+
});
|
|
4106
|
+
const health = (rule.health ?? "").toLowerCase();
|
|
4107
|
+
if (health === "err") {
|
|
4108
|
+
healthGap = true;
|
|
4109
|
+
builder.limit(
|
|
4110
|
+
source,
|
|
4111
|
+
`Alert rule ${rule.name}`,
|
|
4112
|
+
"Prometheus reported an evaluation error for this rule, so its state may be stale or missing.",
|
|
4113
|
+
"error",
|
|
4114
|
+
);
|
|
4115
|
+
} else if (health !== "ok") {
|
|
4116
|
+
// Anything but a reported "ok" evaluation, including an unrecognized
|
|
4117
|
+
// value, is an unevaluated rule for the purpose of a negative.
|
|
4118
|
+
healthGap = true;
|
|
4119
|
+
builder.limit(
|
|
4120
|
+
source,
|
|
4121
|
+
`Alert rule ${rule.name}`,
|
|
4122
|
+
"Prometheus has not reported a successful evaluation for this rule, so its state is unknown.",
|
|
4123
|
+
"unknown",
|
|
4124
|
+
);
|
|
4125
|
+
}
|
|
4126
|
+
}
|
|
4127
|
+
// A negative is only as strong as the query: an active-state filter over
|
|
4128
|
+
// every alerting rule (a name or group filter never looked at the rest), a
|
|
4129
|
+
// complete list, every rule evaluated, and every returned instance placed
|
|
4130
|
+
// in another namespace outright. An instance this projection merely fails
|
|
4131
|
+
// to recognize is not evidence of absence.
|
|
4132
|
+
if (
|
|
4133
|
+
source.confirmedSuccess &&
|
|
4134
|
+
!truncated &&
|
|
4135
|
+
!namesTarget &&
|
|
4136
|
+
!healthGap &&
|
|
4137
|
+
typeFilter !== "record" &&
|
|
4138
|
+
!nonEmptyString(args?.name) &&
|
|
4139
|
+
!nonEmptyString(args?.group) &&
|
|
4140
|
+
(stateFilter === "firing" || stateFilter === "pending") &&
|
|
4141
|
+
(alertingCount === 0 || everyInstanceElsewhere)
|
|
4142
|
+
) {
|
|
4143
|
+
const filter = `state=${stateFilter}`;
|
|
4144
|
+
builder.observe(`alerts:receipt:${filter}`, "receipt", source, {
|
|
4145
|
+
tier: "checked",
|
|
4146
|
+
relevance: "target",
|
|
4147
|
+
tone: "neutral",
|
|
4148
|
+
title: `No ${stateFilter} alerts matched this ${displayKind(builder.target.kind)}`,
|
|
4149
|
+
summary: identity,
|
|
4150
|
+
data: {
|
|
4151
|
+
type: "receipt",
|
|
4152
|
+
checked: "alerts",
|
|
4153
|
+
scope: identity,
|
|
4154
|
+
message:
|
|
4155
|
+
alertingCount === 0
|
|
4156
|
+
? `0 alerting rules returned (filter: ${filter}).`
|
|
4157
|
+
: `${alertingCount} alerting rule${alertingCount === 1 ? "" : "s"} returned (filter: ${filter}), every instance in another namespace; none names ${identity}.`,
|
|
4158
|
+
},
|
|
4159
|
+
});
|
|
4160
|
+
}
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4163
|
+
function helmOwnedResource(
|
|
4164
|
+
value: unknown,
|
|
4165
|
+
): InvestigationHelmOwnedResource | undefined {
|
|
4166
|
+
const candidate = record(value);
|
|
4167
|
+
if (
|
|
4168
|
+
!candidate ||
|
|
4169
|
+
!nonEmptyString(candidate.kind) ||
|
|
4170
|
+
!nonEmptyString(candidate.name) ||
|
|
4171
|
+
typeof candidate.namespace !== "string"
|
|
4172
|
+
) {
|
|
4173
|
+
return undefined;
|
|
4174
|
+
}
|
|
4175
|
+
for (const field of [
|
|
4176
|
+
"apiVersion",
|
|
4177
|
+
"status",
|
|
4178
|
+
"ready",
|
|
4179
|
+
"message",
|
|
4180
|
+
"summary",
|
|
4181
|
+
"issue",
|
|
4182
|
+
] as const) {
|
|
4183
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
4184
|
+
return undefined;
|
|
4185
|
+
}
|
|
4186
|
+
return candidate as unknown as InvestigationHelmOwnedResource;
|
|
4187
|
+
}
|
|
4188
|
+
|
|
4189
|
+
function helmOperation(value: unknown): InvestigationHelmOperation | undefined {
|
|
4190
|
+
const candidate = record(value);
|
|
4191
|
+
if (
|
|
4192
|
+
!candidate ||
|
|
4193
|
+
!nonEmptyString(candidate.kind) ||
|
|
4194
|
+
!nonEmptyString(candidate.status) ||
|
|
4195
|
+
typeof candidate.message !== "string"
|
|
4196
|
+
) {
|
|
4197
|
+
return undefined;
|
|
4198
|
+
}
|
|
4199
|
+
for (const field of [
|
|
4200
|
+
"revision",
|
|
4201
|
+
"failedRevision",
|
|
4202
|
+
"rollbackRevision",
|
|
4203
|
+
] as const) {
|
|
4204
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "number")
|
|
4205
|
+
return undefined;
|
|
4206
|
+
}
|
|
4207
|
+
if (candidate.updated !== undefined && typeof candidate.updated !== "string")
|
|
4208
|
+
return undefined;
|
|
4209
|
+
return candidate as unknown as InvestigationHelmOperation;
|
|
4210
|
+
}
|
|
4211
|
+
|
|
4212
|
+
function adaptHelmRelease(
|
|
4213
|
+
builder: ProjectionBuilder,
|
|
4214
|
+
source: InvestigationEvidenceSource,
|
|
4215
|
+
payload: unknown,
|
|
4216
|
+
): void {
|
|
4217
|
+
const value = record(payload);
|
|
4218
|
+
if (
|
|
4219
|
+
!value ||
|
|
4220
|
+
!nonEmptyString(value.name) ||
|
|
4221
|
+
!nonEmptyString(value.namespace) ||
|
|
4222
|
+
typeof value.chart !== "string" ||
|
|
4223
|
+
typeof value.chartVersion !== "string" ||
|
|
4224
|
+
!nonEmptyString(value.status) ||
|
|
4225
|
+
typeof value.revision !== "number" ||
|
|
4226
|
+
typeof value.updated !== "string"
|
|
4227
|
+
) {
|
|
4228
|
+
invalidPayload(builder, source);
|
|
4229
|
+
return;
|
|
4230
|
+
}
|
|
4231
|
+
for (const field of [
|
|
4232
|
+
"appVersion",
|
|
4233
|
+
"description",
|
|
4234
|
+
"storageNamespace",
|
|
4235
|
+
"resourceHealth",
|
|
4236
|
+
"healthIssue",
|
|
4237
|
+
"healthSummary",
|
|
4238
|
+
"managedByFluxHelmRelease",
|
|
4239
|
+
] as const) {
|
|
4240
|
+
if (value[field] !== undefined && typeof value[field] !== "string") {
|
|
4241
|
+
invalidPayload(builder, source);
|
|
4242
|
+
return;
|
|
4243
|
+
}
|
|
4244
|
+
}
|
|
4245
|
+
const resourcesRaw =
|
|
4246
|
+
value.resources === undefined || value.resources === null
|
|
4247
|
+
? []
|
|
4248
|
+
: value.resources;
|
|
4249
|
+
if (!Array.isArray(resourcesRaw)) {
|
|
4250
|
+
invalidPayload(builder, source, "Helm resources");
|
|
4251
|
+
return;
|
|
4252
|
+
}
|
|
4253
|
+
const resources = resourcesRaw
|
|
4254
|
+
.map(helmOwnedResource)
|
|
4255
|
+
.filter((item): item is InvestigationHelmOwnedResource => Boolean(item));
|
|
4256
|
+
if (resources.length !== resourcesRaw.length) {
|
|
4257
|
+
invalidPayload(builder, source, "Helm resources");
|
|
4258
|
+
return;
|
|
4259
|
+
}
|
|
4260
|
+
const lastOperation =
|
|
4261
|
+
value.lastOperation === undefined
|
|
4262
|
+
? undefined
|
|
4263
|
+
: helmOperation(value.lastOperation);
|
|
4264
|
+
if (value.lastOperation !== undefined && !lastOperation) {
|
|
4265
|
+
invalidPayload(builder, source, "Helm operation");
|
|
4266
|
+
return;
|
|
4267
|
+
}
|
|
4268
|
+
const release: InvestigationHelmRelease = {
|
|
4269
|
+
name: value.name,
|
|
4270
|
+
namespace: value.namespace,
|
|
4271
|
+
chart: value.chart,
|
|
4272
|
+
chartVersion: value.chartVersion,
|
|
4273
|
+
appVersion: value.appVersion as string | undefined,
|
|
4274
|
+
status: value.status,
|
|
4275
|
+
revision: value.revision,
|
|
4276
|
+
updated: value.updated,
|
|
4277
|
+
description: value.description as string | undefined,
|
|
4278
|
+
...(nonEmptyString(value.storageNamespace)
|
|
4279
|
+
? { storageNamespace: value.storageNamespace }
|
|
4280
|
+
: {}),
|
|
4281
|
+
resourceHealth: value.resourceHealth as string | undefined,
|
|
4282
|
+
healthIssue: value.healthIssue as string | undefined,
|
|
4283
|
+
healthSummary: value.healthSummary as string | undefined,
|
|
4284
|
+
managedByFluxHelmRelease: value.managedByFluxHelmRelease as
|
|
4285
|
+
string | undefined,
|
|
4286
|
+
lastOperation,
|
|
4287
|
+
resources,
|
|
4288
|
+
};
|
|
4289
|
+
// The release manages the target when the target is among the resources
|
|
4290
|
+
// Helm rendered for it; a release is never the investigated object itself.
|
|
4291
|
+
// Without an API version the owned row cannot tell colliding kinds apart
|
|
4292
|
+
// (CNPG vs CAPI Cluster), so it establishes nothing.
|
|
4293
|
+
const relevance: InvestigationEvidenceRelevance = resources.some(
|
|
4294
|
+
(owned) =>
|
|
4295
|
+
owned.apiVersion !== undefined &&
|
|
4296
|
+
resourceMatchesTarget(builder.target, {
|
|
4297
|
+
kind: owned.kind,
|
|
4298
|
+
group: apiVersionToGroup(owned.apiVersion),
|
|
4299
|
+
namespace: owned.namespace || undefined,
|
|
4300
|
+
name: owned.name,
|
|
4301
|
+
}),
|
|
4302
|
+
)
|
|
4303
|
+
? "producer-related"
|
|
4304
|
+
: "broader";
|
|
4305
|
+
const status = release.status.toLowerCase();
|
|
4306
|
+
const operationFailed =
|
|
4307
|
+
lastOperation?.status === "failed" ||
|
|
4308
|
+
lastOperation?.status === "stuck_pending";
|
|
4309
|
+
const adverse =
|
|
4310
|
+
status !== "deployed" || Boolean(release.healthIssue) || operationFailed;
|
|
4311
|
+
const chartLabel = `${release.chart}${release.chartVersion ? ` ${release.chartVersion}` : ""}`;
|
|
4312
|
+
// Helm keys a release by where its metadata is stored; two releases with
|
|
4313
|
+
// one name and namespace can live in different storage namespaces.
|
|
4314
|
+
const storage = release.storageNamespace ?? release.namespace;
|
|
4315
|
+
builder.observe(
|
|
4316
|
+
`helm:${storage}:${release.namespace}:${release.name}`,
|
|
4317
|
+
"helm",
|
|
4318
|
+
source,
|
|
4319
|
+
{
|
|
4320
|
+
tier: evidenceTierForRelevance(
|
|
4321
|
+
adverse ? "supporting" : "context",
|
|
4322
|
+
relevance,
|
|
4323
|
+
),
|
|
4324
|
+
relevance,
|
|
4325
|
+
tone:
|
|
4326
|
+
status.includes("failed") || lastOperation?.status === "failed"
|
|
4327
|
+
? "error"
|
|
4328
|
+
: adverse
|
|
4329
|
+
? "warning"
|
|
4330
|
+
: "info",
|
|
4331
|
+
title: `Helm release ${release.namespace}/${release.name}`,
|
|
4332
|
+
summary: release.healthIssue
|
|
4333
|
+
? `${release.status} · ${release.healthIssue}`
|
|
4334
|
+
: `${chartLabel} · ${release.status} · revision ${release.revision}`,
|
|
4335
|
+
data: { type: "helm", release },
|
|
4336
|
+
},
|
|
4337
|
+
);
|
|
4338
|
+
// Every comparison the producer attempted and could not finish, not just the
|
|
4339
|
+
// values read. Four of these are Cloud-role denials, so dropping any of them
|
|
4340
|
+
// lets a reader see a release card whose comparison was withheld and take it
|
|
4341
|
+
// for complete. The labels follow the producer's own fields: `diff` is the
|
|
4342
|
+
// manifest, `valuesDiff` the values — naming one for the other sends a reader
|
|
4343
|
+
// to the wrong thing.
|
|
4344
|
+
for (const [field, label] of [
|
|
4345
|
+
["valuesError", "Helm values"],
|
|
4346
|
+
["valuesDiffError", "Helm values diff"],
|
|
4347
|
+
["diffError", "Helm manifest diff"],
|
|
4348
|
+
["notesDiffError", "Helm notes diff"],
|
|
4349
|
+
["resourceDiffError", "Helm resource diff"],
|
|
4350
|
+
] as const) {
|
|
4351
|
+
const message = value[field];
|
|
4352
|
+
if (nonEmptyString(message)) {
|
|
4353
|
+
builder.limit(source, label, message, "error");
|
|
4354
|
+
}
|
|
4355
|
+
}
|
|
4356
|
+
}
|
|
4357
|
+
|
|
4358
|
+
function permissionSubject(
|
|
4359
|
+
value: unknown,
|
|
4360
|
+
): InvestigationPermissionSubject | undefined {
|
|
4361
|
+
const candidate = record(value);
|
|
4362
|
+
if (
|
|
4363
|
+
!candidate ||
|
|
4364
|
+
!nonEmptyString(candidate.kind) ||
|
|
4365
|
+
!nonEmptyString(candidate.name) ||
|
|
4366
|
+
(candidate.namespace !== undefined &&
|
|
4367
|
+
typeof candidate.namespace !== "string")
|
|
4368
|
+
) {
|
|
4369
|
+
return undefined;
|
|
4370
|
+
}
|
|
4371
|
+
return {
|
|
4372
|
+
kind: candidate.kind,
|
|
4373
|
+
name: candidate.name,
|
|
4374
|
+
...(nonEmptyString(candidate.namespace)
|
|
4375
|
+
? { namespace: candidate.namespace }
|
|
4376
|
+
: {}),
|
|
4377
|
+
};
|
|
4378
|
+
}
|
|
4379
|
+
|
|
4380
|
+
function accessCheck(value: unknown): InvestigationAccessCheck | undefined {
|
|
4381
|
+
const candidate = record(value);
|
|
4382
|
+
if (
|
|
4383
|
+
!candidate ||
|
|
4384
|
+
!nonEmptyString(candidate.verb) ||
|
|
4385
|
+
!nonEmptyString(candidate.resource) ||
|
|
4386
|
+
typeof candidate.namespace !== "string" ||
|
|
4387
|
+
typeof candidate.allowed !== "boolean" ||
|
|
4388
|
+
(candidate.denied !== undefined && typeof candidate.denied !== "boolean")
|
|
4389
|
+
) {
|
|
4390
|
+
return undefined;
|
|
4391
|
+
}
|
|
4392
|
+
for (const field of [
|
|
4393
|
+
"group",
|
|
4394
|
+
"subresource",
|
|
4395
|
+
"resourceName",
|
|
4396
|
+
"reason",
|
|
4397
|
+
"evaluationError",
|
|
4398
|
+
] as const) {
|
|
4399
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
4400
|
+
return undefined;
|
|
4401
|
+
}
|
|
4402
|
+
return {
|
|
4403
|
+
...(candidate as unknown as InvestigationAccessCheck),
|
|
4404
|
+
denied: candidate.denied === true,
|
|
4405
|
+
};
|
|
4406
|
+
}
|
|
4407
|
+
|
|
4408
|
+
function permissionBinding(
|
|
4409
|
+
value: unknown,
|
|
4410
|
+
): InvestigationPermissionBinding | undefined {
|
|
4411
|
+
const candidate = record(value);
|
|
4412
|
+
if (
|
|
4413
|
+
!candidate ||
|
|
4414
|
+
!nonEmptyString(candidate.bindingKind) ||
|
|
4415
|
+
!nonEmptyString(candidate.bindingName) ||
|
|
4416
|
+
!nonEmptyString(candidate.roleKind) ||
|
|
4417
|
+
!nonEmptyString(candidate.roleName) ||
|
|
4418
|
+
!nonNegativeInteger(candidate.rulesCount)
|
|
4419
|
+
) {
|
|
4420
|
+
return undefined;
|
|
4421
|
+
}
|
|
4422
|
+
for (const field of [
|
|
4423
|
+
"bindingNamespace",
|
|
4424
|
+
"roleNamespace",
|
|
4425
|
+
"inheritedFromGroup",
|
|
4426
|
+
] as const) {
|
|
4427
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
4428
|
+
return undefined;
|
|
4429
|
+
}
|
|
4430
|
+
return candidate as unknown as InvestigationPermissionBinding;
|
|
4431
|
+
}
|
|
4432
|
+
|
|
4433
|
+
// Kinds whose spec carries a PodSpec, by API group. Anything else with a
|
|
4434
|
+
// `spec.template.spec` (an ApplicationSet, for one) templates something that
|
|
4435
|
+
// is not a Pod and runs as no account at all.
|
|
4436
|
+
const POD_TEMPLATE_KINDS: Readonly<Record<string, readonly string[]>> = {
|
|
4437
|
+
"": ["Pod"],
|
|
4438
|
+
apps: ["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet"],
|
|
4439
|
+
batch: ["Job", "CronJob"],
|
|
4440
|
+
"argoproj.io": ["Rollout"],
|
|
4441
|
+
};
|
|
4442
|
+
|
|
4443
|
+
function podSpecServiceAccount(
|
|
4444
|
+
resource: InvestigationKubernetesResource,
|
|
4445
|
+
): string | undefined {
|
|
4446
|
+
const group = apiVersionToGroup(resource.apiVersion);
|
|
4447
|
+
if (!POD_TEMPLATE_KINDS[group]?.includes(resource.kind)) return undefined;
|
|
4448
|
+
const spec = record(resource.spec);
|
|
4449
|
+
const podSpec =
|
|
4450
|
+
resource.kind === "Pod"
|
|
4451
|
+
? spec
|
|
4452
|
+
: resource.kind === "CronJob"
|
|
4453
|
+
? record(
|
|
4454
|
+
record(record(record(spec?.jobTemplate)?.spec)?.template)?.spec,
|
|
4455
|
+
)
|
|
4456
|
+
: record(record(spec?.template)?.spec);
|
|
4457
|
+
if (!podSpec) return undefined;
|
|
4458
|
+
// An unset serviceAccountName runs as the namespace's `default` account.
|
|
4459
|
+
return nonEmptyString(podSpec.serviceAccountName)
|
|
4460
|
+
? podSpec.serviceAccountName
|
|
4461
|
+
: "default";
|
|
4462
|
+
}
|
|
4463
|
+
|
|
4464
|
+
/**
|
|
4465
|
+
* The ServiceAccount the target runs as, read from a resource observation of
|
|
4466
|
+
* the target in the same turn. Only the captured resource can state this; the
|
|
4467
|
+
* permission subject is never assumed related by name alone.
|
|
4468
|
+
*/
|
|
4469
|
+
function targetServiceAccountName(
|
|
4470
|
+
builder: ProjectionBuilder,
|
|
4471
|
+
source: InvestigationEvidenceSource,
|
|
4472
|
+
): string | undefined {
|
|
4473
|
+
let newest: InvestigationEvidenceObservation | undefined;
|
|
4474
|
+
for (const group of builder.groups) {
|
|
4475
|
+
if (group.kind !== "resource") continue;
|
|
4476
|
+
for (const observation of group.observations) {
|
|
4477
|
+
if (
|
|
4478
|
+
observation.source.turnIndex !== source.turnIndex ||
|
|
4479
|
+
observation.relevance !== "target" ||
|
|
4480
|
+
observation.data.type !== "resource" ||
|
|
4481
|
+
(newest && observation.source.order < newest.source.order)
|
|
4482
|
+
) {
|
|
4483
|
+
continue;
|
|
4484
|
+
}
|
|
4485
|
+
newest = observation;
|
|
4486
|
+
}
|
|
4487
|
+
}
|
|
4488
|
+
if (!newest || newest.data.type !== "resource") return undefined;
|
|
4489
|
+
return (
|
|
4490
|
+
newest.data.resourceContext?.uses?.serviceAccount?.name ??
|
|
4491
|
+
podSpecServiceAccount(newest.data.resource)
|
|
4492
|
+
);
|
|
4493
|
+
}
|
|
4494
|
+
|
|
4495
|
+
function adaptSubjectPermissions(
|
|
4496
|
+
builder: ProjectionBuilder,
|
|
4497
|
+
source: InvestigationEvidenceSource,
|
|
4498
|
+
payload: unknown,
|
|
4499
|
+
): void {
|
|
4500
|
+
const value = record(payload);
|
|
4501
|
+
const subject = value ? permissionSubject(value.subject) : undefined;
|
|
4502
|
+
if (!value || !subject) {
|
|
4503
|
+
invalidPayload(builder, source);
|
|
4504
|
+
return;
|
|
4505
|
+
}
|
|
4506
|
+
// Only a captured resource can tie a principal to the target. Any other
|
|
4507
|
+
// subject the agent chose to check is context, however adverse the answer.
|
|
4508
|
+
const relevance: InvestigationEvidenceRelevance =
|
|
4509
|
+
subject.kind === "ServiceAccount" &&
|
|
4510
|
+
subject.namespace !== undefined &&
|
|
4511
|
+
subject.namespace === builder.target.namespace &&
|
|
4512
|
+
targetServiceAccountName(builder, source) === subject.name
|
|
4513
|
+
? "target"
|
|
4514
|
+
: "broader";
|
|
4515
|
+
const subjectLabel = `${displayKind(subject.kind)} ${subject.namespace ? `${subject.namespace}/` : ""}${subject.name}`;
|
|
4516
|
+
const subjectKey = `${subject.kind}:${subject.namespace ?? ""}:${subject.name}`;
|
|
4517
|
+
|
|
4518
|
+
if (value.accessCheck !== undefined) {
|
|
4519
|
+
const check = accessCheck(value.accessCheck);
|
|
4520
|
+
if (!check) {
|
|
4521
|
+
invalidPayload(builder, source, "Access check");
|
|
4522
|
+
return;
|
|
4523
|
+
}
|
|
4524
|
+
const resourceLabel = `${check.resource}${check.subresource ? `/${check.subresource}` : ""}${check.group ? `.${check.group}` : ""}`;
|
|
4525
|
+
// An authorizer that could not decide has established neither answer. The
|
|
4526
|
+
// evaluation error already reaches the coverage strip below, but a card
|
|
4527
|
+
// reading "cannot verb resource" is the part an operator acts on, and a
|
|
4528
|
+
// webhook returning partial data is not a denial.
|
|
4529
|
+
// Only when the authorizer decided nothing. Kubernetes returns this error
|
|
4530
|
+
// alongside a real verdict too — one webhook failing while another allows —
|
|
4531
|
+
// and rewriting a decided allow or deny as "could not check" loses the
|
|
4532
|
+
// answer and puts an allow in the warning list.
|
|
4533
|
+
const unresolved =
|
|
4534
|
+
nonEmptyString(check.evaluationError) && !check.allowed && !check.denied;
|
|
4535
|
+
const denied = !check.allowed && !unresolved;
|
|
4536
|
+
const verdict = unresolved
|
|
4537
|
+
? `Could not be evaluated: ${check.evaluationError}`
|
|
4538
|
+
: check.reason ||
|
|
4539
|
+
(check.allowed
|
|
4540
|
+
? "Allowed by RBAC"
|
|
4541
|
+
: check.denied
|
|
4542
|
+
? "Explicitly denied"
|
|
4543
|
+
: "No RBAC rule allows it");
|
|
4544
|
+
builder.observe(
|
|
4545
|
+
`permissions:check:${subjectKey}:${check.verb}:${check.group ?? ""}:${check.resource}:${check.subresource ?? ""}:${check.namespace}:${check.resourceName ?? ""}`,
|
|
4546
|
+
"permissions",
|
|
4547
|
+
source,
|
|
4548
|
+
{
|
|
4549
|
+
tier: evidenceTierForRelevance(
|
|
4550
|
+
denied || unresolved ? "supporting" : "context",
|
|
4551
|
+
relevance,
|
|
4552
|
+
),
|
|
4553
|
+
relevance,
|
|
4554
|
+
tone: denied || unresolved ? "warning" : "info",
|
|
4555
|
+
title: unresolved
|
|
4556
|
+
? `Could not check whether ${subjectLabel} can ${check.verb} ${resourceLabel}`
|
|
4557
|
+
: `${subjectLabel} ${denied ? "cannot" : "can"} ${check.verb} ${resourceLabel}`,
|
|
4558
|
+
summary: `${verdict} · ${check.namespace ? `namespace ${check.namespace}` : "cluster-wide"}${check.resourceName ? ` · ${check.resourceName}` : ""}`,
|
|
4559
|
+
data: { type: "permissions", subject, accessCheck: check },
|
|
4560
|
+
},
|
|
4561
|
+
);
|
|
4562
|
+
if (nonEmptyString(check.evaluationError)) {
|
|
4563
|
+
builder.limit(source, "Permissions", check.evaluationError, "error");
|
|
4564
|
+
}
|
|
4565
|
+
return;
|
|
4566
|
+
}
|
|
4567
|
+
|
|
4568
|
+
const bindingsRaw = value.bindings;
|
|
4569
|
+
if (!Array.isArray(bindingsRaw)) {
|
|
4570
|
+
invalidPayload(builder, source);
|
|
4571
|
+
return;
|
|
4572
|
+
}
|
|
4573
|
+
const bindings = bindingsRaw
|
|
4574
|
+
.map(permissionBinding)
|
|
4575
|
+
.filter((item): item is InvestigationPermissionBinding => Boolean(item));
|
|
4576
|
+
if (
|
|
4577
|
+
bindings.length !== bindingsRaw.length ||
|
|
4578
|
+
!Array.isArray(value.flatRules) ||
|
|
4579
|
+
!value.flatRules.every((rule) => stringArray(record(rule)?.verbs)) ||
|
|
4580
|
+
(value.truncated !== undefined && typeof value.truncated !== "boolean") ||
|
|
4581
|
+
(value.podsTotal !== undefined && !nonNegativeInteger(value.podsTotal)) ||
|
|
4582
|
+
(value.usedByPods !== undefined && !stringArray(value.usedByPods))
|
|
4583
|
+
) {
|
|
4584
|
+
invalidPayload(builder, source);
|
|
4585
|
+
return;
|
|
4586
|
+
}
|
|
4587
|
+
const truncated = value.truncated === true;
|
|
4588
|
+
const flatRulesCount = value.flatRules.length;
|
|
4589
|
+
const usedByPods = stringArray(value.usedByPods) ?? [];
|
|
4590
|
+
builder.observe(`permissions:subject:${subjectKey}`, "permissions", source, {
|
|
4591
|
+
tier: evidenceTierForRelevance("context", relevance),
|
|
4592
|
+
relevance,
|
|
4593
|
+
tone: "info",
|
|
4594
|
+
title: `Permissions of ${subjectLabel}`,
|
|
4595
|
+
summary: `${bindings.length} binding${bindings.length === 1 ? "" : "s"} · ${flatRulesCount}${truncated ? "+" : ""} effective rule${flatRulesCount === 1 && !truncated ? "" : "s"}`,
|
|
4596
|
+
data: {
|
|
4597
|
+
type: "permissions",
|
|
4598
|
+
subject,
|
|
4599
|
+
bindings,
|
|
4600
|
+
flatRulesCount,
|
|
4601
|
+
truncated,
|
|
4602
|
+
usedByPods,
|
|
4603
|
+
podsTotal: value.podsTotal as number | undefined,
|
|
4604
|
+
},
|
|
4605
|
+
});
|
|
4606
|
+
if (truncated) {
|
|
4607
|
+
builder.limit(
|
|
4608
|
+
source,
|
|
4609
|
+
"Permissions",
|
|
4610
|
+
nonEmptyString(value.narrowHint)
|
|
4611
|
+
? value.narrowHint
|
|
4612
|
+
: "The effective rule list was truncated; it is not the subject's complete permission set.",
|
|
4613
|
+
"truncated",
|
|
4614
|
+
);
|
|
4615
|
+
}
|
|
4616
|
+
if (nonEmptyString(value.subjectWarning)) {
|
|
4617
|
+
builder.limit(source, "Permissions", value.subjectWarning, "unknown");
|
|
4618
|
+
}
|
|
4619
|
+
if (
|
|
4620
|
+
typeof value.podsTotal === "number" &&
|
|
4621
|
+
value.podsTotal > usedByPods.length
|
|
4622
|
+
) {
|
|
4623
|
+
builder.limit(
|
|
4624
|
+
source,
|
|
4625
|
+
"Permissions",
|
|
4626
|
+
`Only ${usedByPods.length} of ${value.podsTotal} pods running as this subject were listed.`,
|
|
4627
|
+
"truncated",
|
|
4628
|
+
);
|
|
4629
|
+
}
|
|
4630
|
+
}
|
|
4631
|
+
|
|
4632
|
+
function timeSeries(value: unknown): TimeSeries | undefined {
|
|
4633
|
+
const item = record(value);
|
|
4634
|
+
if (!item || !Array.isArray(item.dataPoints)) return undefined;
|
|
4635
|
+
const labels = record(item.labels) ?? {};
|
|
4636
|
+
if (Object.values(labels).some((label) => typeof label !== "string")) {
|
|
4637
|
+
return undefined;
|
|
4638
|
+
}
|
|
4639
|
+
const dataPoints: TimeSeries["dataPoints"] = [];
|
|
4640
|
+
for (const raw of item.dataPoints) {
|
|
4641
|
+
const point = record(raw);
|
|
4642
|
+
if (!point || typeof point.timestamp !== "number") return undefined;
|
|
4643
|
+
if (
|
|
4644
|
+
point.value !== undefined &&
|
|
4645
|
+
point.value !== null &&
|
|
4646
|
+
typeof point.value !== "number"
|
|
4647
|
+
) {
|
|
4648
|
+
return undefined;
|
|
4649
|
+
}
|
|
4650
|
+
dataPoints.push({
|
|
4651
|
+
timestamp: point.timestamp,
|
|
4652
|
+
value: typeof point.value === "number" ? point.value : null,
|
|
4653
|
+
});
|
|
4654
|
+
}
|
|
4655
|
+
return { labels: labels as Record<string, string>, dataPoints };
|
|
4656
|
+
}
|
|
4657
|
+
|
|
4658
|
+
function metricsSelector(
|
|
4659
|
+
value: unknown,
|
|
4660
|
+
): InvestigationMetricsSelector | undefined {
|
|
4661
|
+
const item = record(value);
|
|
4662
|
+
if (!item || typeof item.metric !== "string" || !Array.isArray(item.matchers))
|
|
4663
|
+
return undefined;
|
|
4664
|
+
const matchers: InvestigationMetricsSelector["matchers"] = [];
|
|
4665
|
+
for (const raw of item.matchers) {
|
|
4666
|
+
const matcher = record(raw);
|
|
4667
|
+
if (
|
|
4668
|
+
!matcher ||
|
|
4669
|
+
typeof matcher.label !== "string" ||
|
|
4670
|
+
typeof matcher.op !== "string" ||
|
|
4671
|
+
typeof matcher.value !== "string"
|
|
4672
|
+
) {
|
|
4673
|
+
return undefined;
|
|
4674
|
+
}
|
|
4675
|
+
matchers.push({
|
|
4676
|
+
label: matcher.label,
|
|
4677
|
+
op: matcher.op,
|
|
4678
|
+
value: matcher.value,
|
|
4679
|
+
});
|
|
4680
|
+
}
|
|
4681
|
+
return { metric: item.metric, matchers };
|
|
4682
|
+
}
|
|
4683
|
+
|
|
4684
|
+
/**
|
|
4685
|
+
* How one selector relates to the investigated resource. "target": it names
|
|
4686
|
+
* the target itself (an identity label, or pods a producer established as
|
|
4687
|
+
* the target's own). "related": it mentions the target's namespace and
|
|
4688
|
+
* something in it the target cannot be proved to own (a pod named by prefix,
|
|
4689
|
+
* a sibling workload, an owner series). "namespace": only the namespace.
|
|
4690
|
+
* "foreign": another namespace or none.
|
|
4691
|
+
*/
|
|
4692
|
+
type SelectorVerdict = "target" | "related" | "namespace" | "foreign";
|
|
4693
|
+
|
|
4694
|
+
const VERDICT_RANK: Record<SelectorVerdict, number> = {
|
|
4695
|
+
foreign: 3,
|
|
4696
|
+
target: 2,
|
|
4697
|
+
related: 1,
|
|
4698
|
+
namespace: 0,
|
|
4699
|
+
};
|
|
4700
|
+
|
|
4701
|
+
/**
|
|
4702
|
+
* A regex matcher names the target only in the exact forms the plan admits,
|
|
4703
|
+
* with the name's regex metacharacters (a dot in a Kubernetes name) escaped
|
|
4704
|
+
* or absent. "api-?.*" also selects "apiworker-…" and an unescaped "api.v2"
|
|
4705
|
+
* also selects "api-v2", so neither may claim the target.
|
|
4706
|
+
*/
|
|
4707
|
+
function regexNamesExactly(
|
|
4708
|
+
value: string,
|
|
4709
|
+
name: string,
|
|
4710
|
+
suffix: string,
|
|
4711
|
+
): boolean {
|
|
4712
|
+
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4713
|
+
return value === `${escaped}${suffix}`;
|
|
4714
|
+
}
|
|
4715
|
+
|
|
4716
|
+
/**
|
|
4717
|
+
* The pod names an exact-set regex `^(a|b)$` (or the unanchored `a|b`, which
|
|
4718
|
+
* Prometheus anchors) lists. A dot must be escaped to count: `^(api.v2-0)$`
|
|
4719
|
+
* also selects `api-v2-0`, so it names more than the pod it appears to and
|
|
4720
|
+
* cannot prove membership. Undefined for any other shape.
|
|
4721
|
+
*/
|
|
4722
|
+
function exactPodSet(value: string): string[] | undefined {
|
|
4723
|
+
const body =
|
|
4724
|
+
value.startsWith("^(") && value.endsWith(")$") ? value.slice(2, -2) : value;
|
|
4725
|
+
if (body === "" || /[^A-Za-z0-9\-|\\.]/.test(body)) return undefined;
|
|
4726
|
+
// Every dot has to arrive escaped; an unescaped one is a wildcard.
|
|
4727
|
+
if (/(^|[^\\])\./.test(body.replace(/\\\\/g, ""))) return undefined;
|
|
4728
|
+
const names = body.split("|").map((name) => name.replace(/\\\./g, "."));
|
|
4729
|
+
return names.every((name) => /^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(name))
|
|
4730
|
+
? names
|
|
4731
|
+
: undefined;
|
|
4732
|
+
}
|
|
4733
|
+
|
|
4734
|
+
/**
|
|
4735
|
+
* Whether a workload identity label names the target: the label
|
|
4736
|
+
* kube-state-metrics uses for the kind (`deployment`, `job_name`, …) or a
|
|
4737
|
+
* generic `workload` label, with the exact name.
|
|
4738
|
+
*/
|
|
4739
|
+
function identityLabelNamesTarget(
|
|
4740
|
+
target: InvestigationEvidenceTarget,
|
|
4741
|
+
label: string,
|
|
4742
|
+
value: string,
|
|
4743
|
+
exact: boolean,
|
|
4744
|
+
): boolean {
|
|
4745
|
+
const kind = target.kind.toLowerCase();
|
|
4746
|
+
const workloadLabel = WORKLOAD_LABEL_BY_KIND[kind];
|
|
4747
|
+
// `deployment=api` states the kind in the label itself; `workload=api` does
|
|
4748
|
+
// not, so a `workload_type` naming another kind vetoes it in
|
|
4749
|
+
// classifySelector, which is the only place that sees the whole selector.
|
|
4750
|
+
if (label !== workloadLabel && label !== "workload") return false;
|
|
4751
|
+
return exact
|
|
4752
|
+
? value === target.name
|
|
4753
|
+
: regexNamesExactly(value, target.name, "");
|
|
4754
|
+
}
|
|
4755
|
+
|
|
4756
|
+
function classifyMatcher(
|
|
4757
|
+
target: InvestigationEvidenceTarget,
|
|
4758
|
+
matcher: InvestigationMetricsSelector["matchers"][number],
|
|
4759
|
+
establishedPods: ReadonlySet<string>,
|
|
4760
|
+
): SelectorVerdict {
|
|
4761
|
+
const { label, op, value } = matcher;
|
|
4762
|
+
const kind = target.kind.toLowerCase();
|
|
4763
|
+
if (label === "pod") {
|
|
4764
|
+
if (kind === "pod") {
|
|
4765
|
+
if (op === "=") return value === target.name ? "target" : "related";
|
|
4766
|
+
if (op === "=~") {
|
|
4767
|
+
if (regexNamesExactly(value, target.name, "")) return "target";
|
|
4768
|
+
// `^(name)$` is the form the diagnose prompt asks the agent to write,
|
|
4769
|
+
// so a Pod target has to recognise its own name in it. The set must
|
|
4770
|
+
// name nothing else: a query covering the target and a neighbour is
|
|
4771
|
+
// not evidence about the target alone.
|
|
4772
|
+
const names = exactPodSet(value);
|
|
4773
|
+
return names && names.every((name) => name === target.name)
|
|
4774
|
+
? "target"
|
|
4775
|
+
: "related";
|
|
4776
|
+
}
|
|
4777
|
+
return "related";
|
|
4778
|
+
}
|
|
4779
|
+
if (op === "=") return establishedPods.has(value) ? "target" : "related";
|
|
4780
|
+
if (op === "=~") {
|
|
4781
|
+
const names = exactPodSet(value);
|
|
4782
|
+
if (names && names.every((name) => establishedPods.has(name))) {
|
|
4783
|
+
return "target";
|
|
4784
|
+
}
|
|
4785
|
+
return "related";
|
|
4786
|
+
}
|
|
4787
|
+
return "related";
|
|
4788
|
+
}
|
|
4789
|
+
if (label === "namespace" || label === "container") return "namespace";
|
|
4790
|
+
if (op === "=" && identityLabelNamesTarget(target, label, value, true)) {
|
|
4791
|
+
return "target";
|
|
4792
|
+
}
|
|
4793
|
+
if (op === "=~" && identityLabelNamesTarget(target, label, value, false)) {
|
|
4794
|
+
return "target";
|
|
4795
|
+
}
|
|
4796
|
+
// Owner series (kube_pod_owner, kube_replicaset_owner) and every other
|
|
4797
|
+
// label in the namespace: about the namespace's things, not proved to be
|
|
4798
|
+
// the target's.
|
|
4799
|
+
return "related";
|
|
4800
|
+
}
|
|
4801
|
+
|
|
4802
|
+
function classifySelector(
|
|
4803
|
+
target: InvestigationEvidenceTarget,
|
|
4804
|
+
selector: InvestigationMetricsSelector,
|
|
4805
|
+
establishedPods: ReadonlySet<string>,
|
|
4806
|
+
): SelectorVerdict {
|
|
4807
|
+
// `workload` is generic, so a `workload_type` naming another kind means
|
|
4808
|
+
// the series is about a different workload that happens to share a name.
|
|
4809
|
+
// A regex counts when it names an exact set, the same way the namespace
|
|
4810
|
+
// matcher below does: `workload_type=~"statefulset"` excludes a Deployment
|
|
4811
|
+
// as plainly as `=` does, and reading only `=` let a sibling's chart take
|
|
4812
|
+
// the target's identity. A set that includes the target kind is no conflict,
|
|
4813
|
+
// and an inexact regex says nothing either way.
|
|
4814
|
+
const workloadType = selector.matchers.find(
|
|
4815
|
+
(matcher) =>
|
|
4816
|
+
matcher.label === "workload_type" &&
|
|
4817
|
+
(matcher.op === "=" || matcher.op === "=~"),
|
|
4818
|
+
);
|
|
4819
|
+
const workloadTypeNames =
|
|
4820
|
+
workloadType === undefined
|
|
4821
|
+
? undefined
|
|
4822
|
+
: workloadType.op === "="
|
|
4823
|
+
? [workloadType.value]
|
|
4824
|
+
: exactPodSet(workloadType.value);
|
|
4825
|
+
const workloadTypeConflicts =
|
|
4826
|
+
workloadTypeNames !== undefined &&
|
|
4827
|
+
workloadTypeNames.length > 0 &&
|
|
4828
|
+
!workloadTypeNames.some(
|
|
4829
|
+
(name) => name.toLowerCase() === target.kind.toLowerCase(),
|
|
4830
|
+
);
|
|
4831
|
+
// Prometheus anchors regex matchers, so namespace=~"shop" is exact too.
|
|
4832
|
+
const inNamespace = selector.matchers.some(
|
|
4833
|
+
(matcher) =>
|
|
4834
|
+
matcher.label === "namespace" &&
|
|
4835
|
+
((matcher.op === "=" && matcher.value === target.namespace) ||
|
|
4836
|
+
(matcher.op === "=~" &&
|
|
4837
|
+
regexNamesExactly(matcher.value, target.namespace ?? "", ""))),
|
|
4838
|
+
);
|
|
4839
|
+
if (!inNamespace) return "foreign";
|
|
4840
|
+
let verdict: SelectorVerdict = "namespace";
|
|
4841
|
+
for (const matcher of selector.matchers) {
|
|
4842
|
+
if (workloadTypeConflicts && matcher.label === "workload") continue;
|
|
4843
|
+
const candidate = classifyMatcher(target, matcher, establishedPods);
|
|
4844
|
+
if (candidate === "foreign") return "foreign";
|
|
4845
|
+
if (VERDICT_RANK[candidate] > VERDICT_RANK[verdict]) verdict = candidate;
|
|
4846
|
+
}
|
|
4847
|
+
return verdict;
|
|
4848
|
+
}
|
|
4849
|
+
|
|
4850
|
+
/**
|
|
4851
|
+
* A metrics result is about the target only when every selector names it:
|
|
4852
|
+
* the target namespace plus pods a producer established as the target's own
|
|
4853
|
+
* or an identity label with its exact name. A selector that only mentions
|
|
4854
|
+
* something in the namespace the target cannot be proved to own (a pod
|
|
4855
|
+
* prefix, a sibling, an owner join) makes the result producer-related: kept,
|
|
4856
|
+
* not promoted, no change markers. A namespace-only query, a bare or foreign
|
|
4857
|
+
* selector, or an inventory the producer could not extract is broader.
|
|
4858
|
+
*/
|
|
4859
|
+
export function metricsScope(
|
|
4860
|
+
target: InvestigationEvidenceTarget,
|
|
4861
|
+
selectors: readonly InvestigationMetricsSelector[],
|
|
4862
|
+
selectorsUnknown: boolean,
|
|
4863
|
+
establishedPods: ReadonlySet<string> = new Set(),
|
|
4864
|
+
): InvestigationEvidenceRelevance {
|
|
4865
|
+
if (selectorsUnknown || selectors.length === 0 || !target.namespace) {
|
|
4866
|
+
return "broader";
|
|
4867
|
+
}
|
|
4868
|
+
let allTarget = true;
|
|
4869
|
+
let anyRelated = false;
|
|
4870
|
+
for (const selector of selectors) {
|
|
4871
|
+
const verdict = classifySelector(target, selector, establishedPods);
|
|
4872
|
+
if (verdict === "foreign") return "broader";
|
|
4873
|
+
if (verdict !== "target") allTarget = false;
|
|
4874
|
+
if (verdict === "related") anyRelated = true;
|
|
4875
|
+
}
|
|
4876
|
+
if (allTarget) return "target";
|
|
4877
|
+
return anyRelated ? "producer-related" : "broader";
|
|
4878
|
+
}
|
|
4879
|
+
|
|
4880
|
+
function metricsWindowLabel(data: {
|
|
4881
|
+
mode: "range" | "instant";
|
|
4882
|
+
start?: string;
|
|
4883
|
+
end?: string;
|
|
4884
|
+
step?: string;
|
|
4885
|
+
}): string | undefined {
|
|
4886
|
+
if (data.mode !== "range" || !data.start || !data.end) return undefined;
|
|
4887
|
+
const startMs = Date.parse(data.start);
|
|
4888
|
+
const endMs = Date.parse(data.end);
|
|
4889
|
+
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs <= startMs)
|
|
4890
|
+
return undefined;
|
|
4891
|
+
const minutes = Math.round((endMs - startMs) / 60_000);
|
|
4892
|
+
const window =
|
|
4893
|
+
minutes >= 60 * 24 * 2
|
|
4894
|
+
? `${Math.round(minutes / (60 * 24))}d`
|
|
4895
|
+
: minutes >= 120
|
|
4896
|
+
? `${Math.round(minutes / 60)}h`
|
|
4897
|
+
: `${minutes}m`;
|
|
4898
|
+
return data.step
|
|
4899
|
+
? `${window} window · ${data.step} step`
|
|
4900
|
+
: `${window} window`;
|
|
4901
|
+
}
|
|
4902
|
+
|
|
4903
|
+
/**
|
|
4904
|
+
* One half of a Go↔TS contract: `diagnoseMetricsCategories` in
|
|
4905
|
+
* internal/mcp/tools_diagnose_metrics.go decides which categories the producer
|
|
4906
|
+
* captures, and a series whose category is missing here is discarded. Change
|
|
4907
|
+
* both together.
|
|
4908
|
+
*/
|
|
4909
|
+
const DIAGNOSE_METRICS_LABELS: Record<string, string> = {
|
|
4910
|
+
cpu: "CPU usage",
|
|
4911
|
+
memory: "Memory working set",
|
|
4912
|
+
restarts: "Restarts",
|
|
4913
|
+
};
|
|
4914
|
+
|
|
4915
|
+
/**
|
|
4916
|
+
* The vitals `diagnose` captured inside the same call: one chart per category
|
|
4917
|
+
* over the exact pods the bundle covers. They take the bundle's relevance, so a
|
|
4918
|
+
* neighbour's diagnose never charts as evidence for the target, and they carry
|
|
4919
|
+
* the diagnosed resource as their subject so recorded changes can be marked
|
|
4920
|
+
* on them. An absent field means Prometheus was not available or the read was
|
|
4921
|
+
* not permitted, which is not a collection failure the producer reported.
|
|
4922
|
+
*/
|
|
4923
|
+
function addDiagnoseMetrics(
|
|
4924
|
+
builder: ProjectionBuilder,
|
|
4925
|
+
source: InvestigationEvidenceSource,
|
|
4926
|
+
raw: unknown,
|
|
4927
|
+
subject: DiagnosisResourceRef,
|
|
4928
|
+
relevance: InvestigationEvidenceRelevance,
|
|
4929
|
+
): void {
|
|
4930
|
+
if (raw === undefined) return;
|
|
4931
|
+
const value = record(raw);
|
|
4932
|
+
const window = record(value?.window);
|
|
4933
|
+
if (
|
|
4934
|
+
!value ||
|
|
4935
|
+
!window ||
|
|
4936
|
+
!nonEmptyString(window.start) ||
|
|
4937
|
+
!nonEmptyString(window.end) ||
|
|
4938
|
+
!nonEmptyString(window.step) ||
|
|
4939
|
+
!(Date.parse(window.end) > Date.parse(window.start)) ||
|
|
4940
|
+
!nonNegativeInteger(value.pods) ||
|
|
4941
|
+
(value.partial !== undefined && typeof value.partial !== "boolean") ||
|
|
4942
|
+
(value.omittedPods !== undefined &&
|
|
4943
|
+
!nonNegativeInteger(value.omittedPods)) ||
|
|
4944
|
+
(value.error !== undefined && typeof value.error !== "string") ||
|
|
4945
|
+
!Array.isArray(value.series)
|
|
4946
|
+
) {
|
|
4947
|
+
invalidPayload(builder, source, "Workload metrics");
|
|
4948
|
+
return;
|
|
4949
|
+
}
|
|
4950
|
+
const scope = scopeFromArgs(source);
|
|
4951
|
+
const partial = value.partial === true;
|
|
4952
|
+
// The pods the chart covers: the ones the workload controlled when the
|
|
4953
|
+
// bundle was captured, which a rollout during the window can miss.
|
|
4954
|
+
const podsLabel = partial
|
|
4955
|
+
? `first ${value.pods} of ${typeof value.omittedPods === "number" ? value.pods + value.omittedPods : "the"} current pods`
|
|
4956
|
+
: `${value.pods} current pod${value.pods === 1 ? "" : "s"}`;
|
|
4957
|
+
const windowLabel = metricsWindowLabel({
|
|
4958
|
+
mode: "range",
|
|
4959
|
+
start: window.start,
|
|
4960
|
+
end: window.end,
|
|
4961
|
+
step: window.step,
|
|
4962
|
+
});
|
|
4963
|
+
for (const rawEntry of value.series) {
|
|
4964
|
+
const entry = record(rawEntry);
|
|
4965
|
+
const rawSeries = entry?.series;
|
|
4966
|
+
const series = Array.isArray(rawSeries)
|
|
4967
|
+
? rawSeries
|
|
4968
|
+
.map(timeSeries)
|
|
4969
|
+
.filter((item): item is TimeSeries => Boolean(item))
|
|
4970
|
+
: undefined;
|
|
4971
|
+
if (
|
|
4972
|
+
!entry ||
|
|
4973
|
+
!nonEmptyString(entry.category) ||
|
|
4974
|
+
!Object.hasOwn(DIAGNOSE_METRICS_LABELS, entry.category) ||
|
|
4975
|
+
!nonEmptyString(entry.query) ||
|
|
4976
|
+
!nonEmptyString(entry.unit) ||
|
|
4977
|
+
!Array.isArray(rawSeries) ||
|
|
4978
|
+
!series ||
|
|
4979
|
+
series.length !== rawSeries.length
|
|
4980
|
+
) {
|
|
4981
|
+
invalidPayload(builder, source, "Workload metrics");
|
|
4982
|
+
continue;
|
|
4983
|
+
}
|
|
4984
|
+
const label = DIAGNOSE_METRICS_LABELS[entry.category];
|
|
4985
|
+
const data: InvestigationMetricsEvidence = {
|
|
4986
|
+
type: "metrics",
|
|
4987
|
+
origin: "diagnose",
|
|
4988
|
+
query: entry.query,
|
|
4989
|
+
mode: "range",
|
|
4990
|
+
start: window.start,
|
|
4991
|
+
end: window.end,
|
|
4992
|
+
step: window.step,
|
|
4993
|
+
unit: entry.unit,
|
|
4994
|
+
label,
|
|
4995
|
+
series,
|
|
4996
|
+
truncated: false,
|
|
4997
|
+
subject,
|
|
4998
|
+
pods: value.pods,
|
|
4999
|
+
partial,
|
|
5000
|
+
};
|
|
5001
|
+
builder.observe(
|
|
5002
|
+
`metrics:diagnose:${subject.group ?? ""}:${subject.kind}:${subject.namespace ?? ""}:${subject.name}:${entry.category}`,
|
|
5003
|
+
"metrics",
|
|
5004
|
+
source,
|
|
5005
|
+
{
|
|
5006
|
+
tier: evidenceTierForRelevance("supporting", relevance),
|
|
5007
|
+
relevance,
|
|
5008
|
+
tone: "neutral",
|
|
5009
|
+
title: `${label} · ${scope}`,
|
|
5010
|
+
summary: [
|
|
5011
|
+
series.length === 0 ? "No samples in the window" : podsLabel,
|
|
5012
|
+
windowLabel,
|
|
5013
|
+
partial ? "partial pod set" : undefined,
|
|
5014
|
+
]
|
|
5015
|
+
.filter((part): part is string => Boolean(part))
|
|
5016
|
+
.join(" · "),
|
|
5017
|
+
data,
|
|
5018
|
+
},
|
|
5019
|
+
);
|
|
5020
|
+
}
|
|
5021
|
+
if (nonEmptyString(value.error)) {
|
|
5022
|
+
builder.limit(source, "Workload metrics", value.error, "error");
|
|
5023
|
+
}
|
|
5024
|
+
}
|
|
5025
|
+
|
|
5026
|
+
// Radar's reading of a metric name, so an agent-written query gets a title a
|
|
5027
|
+
// reader can scan instead of the raw series name. Deliberately small and exact:
|
|
5028
|
+
// an unrecognised metric keeps its own name rather than being handed a meaning
|
|
5029
|
+
// Radar cannot derive from it.
|
|
5030
|
+
const METRIC_FAMILY_LABELS: Record<string, string> = {
|
|
5031
|
+
container_cpu_usage_seconds_total: "CPU usage",
|
|
5032
|
+
container_cpu_cfs_throttled_seconds_total: "CPU throttling",
|
|
5033
|
+
container_memory_working_set_bytes: "Memory working set",
|
|
5034
|
+
container_memory_usage_bytes: "Memory usage",
|
|
5035
|
+
container_memory_rss: "Memory RSS",
|
|
5036
|
+
kube_pod_container_status_restarts_total: "Restarts",
|
|
5037
|
+
container_network_receive_bytes_total: "Network received",
|
|
5038
|
+
container_network_transmit_bytes_total: "Network transmitted",
|
|
5039
|
+
container_fs_usage_bytes: "Filesystem usage",
|
|
5040
|
+
up: "Scrape target up",
|
|
5041
|
+
};
|
|
5042
|
+
|
|
5043
|
+
function adaptQueryPrometheus(
|
|
5044
|
+
builder: ProjectionBuilder,
|
|
5045
|
+
source: InvestigationEvidenceSource,
|
|
5046
|
+
payload: unknown,
|
|
5047
|
+
): void {
|
|
5048
|
+
const value = record(payload);
|
|
5049
|
+
const mode = value?.type;
|
|
5050
|
+
if (
|
|
5051
|
+
!value ||
|
|
5052
|
+
!nonEmptyString(value.query) ||
|
|
5053
|
+
(mode !== "range" && mode !== "instant") ||
|
|
5054
|
+
!Array.isArray(value.series) ||
|
|
5055
|
+
!Array.isArray(value.selectors) ||
|
|
5056
|
+
(value.selectorsUnknown !== undefined &&
|
|
5057
|
+
typeof value.selectorsUnknown !== "boolean") ||
|
|
5058
|
+
(value.truncated !== undefined && typeof value.truncated !== "boolean")
|
|
5059
|
+
) {
|
|
5060
|
+
invalidPayload(builder, source, "Prometheus query");
|
|
5061
|
+
return;
|
|
5062
|
+
}
|
|
5063
|
+
const series = value.series
|
|
5064
|
+
.map(timeSeries)
|
|
5065
|
+
.filter((item): item is TimeSeries => Boolean(item));
|
|
5066
|
+
const selectors = value.selectors
|
|
5067
|
+
.map(metricsSelector)
|
|
5068
|
+
.filter((item): item is InvestigationMetricsSelector => Boolean(item));
|
|
5069
|
+
if (
|
|
5070
|
+
series.length !== value.series.length ||
|
|
5071
|
+
selectors.length !== value.selectors.length
|
|
5072
|
+
) {
|
|
5073
|
+
invalidPayload(builder, source, "Prometheus query");
|
|
5074
|
+
return;
|
|
5075
|
+
}
|
|
5076
|
+
const selectorsUnknown = value.selectorsUnknown === true;
|
|
5077
|
+
const relevance = metricsScope(
|
|
5078
|
+
builder.target,
|
|
5079
|
+
selectors,
|
|
5080
|
+
selectorsUnknown,
|
|
5081
|
+
producerEstablishedTargetPods(builder),
|
|
5082
|
+
);
|
|
5083
|
+
const start = nonEmptyString(value.start) ? value.start : undefined;
|
|
5084
|
+
const end = nonEmptyString(value.end) ? value.end : undefined;
|
|
5085
|
+
const step = nonEmptyString(value.step) ? value.step : undefined;
|
|
5086
|
+
const note = nonEmptyString(value.note) ? value.note : undefined;
|
|
5087
|
+
if (value.truncated === true) {
|
|
5088
|
+
const summary = record(value.summary);
|
|
5089
|
+
const cardinality = record(summary?.labelCardinality);
|
|
5090
|
+
const widest = cardinality
|
|
5091
|
+
? Object.entries(cardinality).find(
|
|
5092
|
+
([, count]) => typeof count === "number",
|
|
5093
|
+
)
|
|
5094
|
+
: undefined;
|
|
5095
|
+
const parts = [
|
|
5096
|
+
typeof summary?.seriesCount === "number"
|
|
5097
|
+
? `${summary.seriesCount} series`
|
|
5098
|
+
: undefined,
|
|
5099
|
+
typeof summary?.totalDataPoints === "number"
|
|
5100
|
+
? `${summary.totalDataPoints} samples`
|
|
5101
|
+
: undefined,
|
|
5102
|
+
widest ? `${widest[1]} distinct ${widest[0]} values` : undefined,
|
|
5103
|
+
].filter((part): part is string => Boolean(part));
|
|
5104
|
+
builder.limit(
|
|
5105
|
+
source,
|
|
5106
|
+
"Prometheus query",
|
|
5107
|
+
`The result of \`${value.query}\` was too large to keep${parts.length ? ` (${parts.join(", ")})` : ""}, so Radar cannot chart it.${note ? ` ${note}` : ""}`,
|
|
5108
|
+
"truncated",
|
|
5109
|
+
);
|
|
5110
|
+
return;
|
|
5111
|
+
}
|
|
5112
|
+
const windowLabel = metricsWindowLabel({ mode, start, end, step });
|
|
5113
|
+
// One metric names the card; the title is Radar's reading of the query,
|
|
5114
|
+
// never the agent's, so a wrong claim cannot become the card's identity.
|
|
5115
|
+
// A bare matcher block such as `{namespace="shop"}` names no metric.
|
|
5116
|
+
const metricNames = [
|
|
5117
|
+
...new Set(
|
|
5118
|
+
selectors.flatMap((selector) =>
|
|
5119
|
+
selector.metric.trim() ? [selector.metric.trim()] : [],
|
|
5120
|
+
),
|
|
5121
|
+
),
|
|
5122
|
+
];
|
|
5123
|
+
// A recognised family names the card; the scope is added only when Radar
|
|
5124
|
+
// already decided the query is about the target, so the title never widens
|
|
5125
|
+
// or narrows what the relevance check concluded.
|
|
5126
|
+
const family =
|
|
5127
|
+
metricNames.length === 1 ? METRIC_FAMILY_LABELS[metricNames[0]] : undefined;
|
|
5128
|
+
const scope =
|
|
5129
|
+
relevance === "target"
|
|
5130
|
+
? `${displayKind(builder.target.kind)} ${builder.target.namespace ? `${builder.target.namespace}/` : ""}${builder.target.name}`
|
|
5131
|
+
: undefined;
|
|
5132
|
+
const title = family
|
|
5133
|
+
? scope
|
|
5134
|
+
? `${family} · ${scope}`
|
|
5135
|
+
: family
|
|
5136
|
+
: metricNames.length === 1
|
|
5137
|
+
? metricNames[0]
|
|
5138
|
+
: mode === "range"
|
|
5139
|
+
? "Prometheus metrics"
|
|
5140
|
+
: "Prometheus values";
|
|
5141
|
+
const data: InvestigationMetricsEvidence = {
|
|
5142
|
+
type: "metrics",
|
|
5143
|
+
origin: "query",
|
|
5144
|
+
query: value.query,
|
|
5145
|
+
mode,
|
|
5146
|
+
start,
|
|
5147
|
+
end,
|
|
5148
|
+
step,
|
|
5149
|
+
unit: metricsUnitForExpression(value.query, selectors),
|
|
5150
|
+
series,
|
|
5151
|
+
truncated: false,
|
|
5152
|
+
note,
|
|
5153
|
+
selectors,
|
|
5154
|
+
selectorsUnknown,
|
|
5155
|
+
subject:
|
|
5156
|
+
relevance === "target"
|
|
5157
|
+
? {
|
|
5158
|
+
kind: builder.target.kind,
|
|
5159
|
+
...(builder.target.group ? { group: builder.target.group } : {}),
|
|
5160
|
+
namespace: builder.target.namespace,
|
|
5161
|
+
name: builder.target.name,
|
|
5162
|
+
}
|
|
5163
|
+
: undefined,
|
|
5164
|
+
};
|
|
5165
|
+
builder.observe(
|
|
5166
|
+
`metrics:${mode}:${value.query}:${start ?? ""}:${end ?? ""}:${step ?? ""}`,
|
|
5167
|
+
"metrics",
|
|
5168
|
+
source,
|
|
5169
|
+
{
|
|
5170
|
+
tier: evidenceTierForRelevance("supporting", relevance),
|
|
5171
|
+
relevance,
|
|
5172
|
+
tone: "neutral",
|
|
5173
|
+
title,
|
|
5174
|
+
summary: [
|
|
5175
|
+
// When a family named the card, the metric name moves here so it stays
|
|
5176
|
+
// visible; otherwise it is already the title.
|
|
5177
|
+
family
|
|
5178
|
+
? metricNames[0]
|
|
5179
|
+
: metricNames.length === 1
|
|
5180
|
+
? "Prometheus"
|
|
5181
|
+
: undefined,
|
|
5182
|
+
series.length === 0 ? "No series matched" : `${series.length} series`,
|
|
5183
|
+
windowLabel,
|
|
5184
|
+
]
|
|
5185
|
+
.filter((part): part is string => Boolean(part))
|
|
5186
|
+
.join(" · "),
|
|
5187
|
+
data,
|
|
5188
|
+
},
|
|
5189
|
+
);
|
|
5190
|
+
}
|
|
5191
|
+
|
|
5192
|
+
/**
|
|
5193
|
+
* Adapters whose verdict depends on which pods a producer established as the
|
|
5194
|
+
* target's. They run after every other tool in the transcript, so the answer
|
|
5195
|
+
* does not depend on whether the agent happened to query Prometheus before or
|
|
5196
|
+
* after the read that named the pods. Their source order is captured before
|
|
5197
|
+
* they are queued, so deferring the classification does not reorder evidence.
|
|
5198
|
+
*/
|
|
5199
|
+
const MEMBERSHIP_DEPENDENT_ADAPTERS = new Set([
|
|
5200
|
+
"query_prometheus",
|
|
5201
|
+
"get_prometheus_rules",
|
|
5202
|
+
]);
|
|
5203
|
+
|
|
5204
|
+
const ADAPTERS: Record<
|
|
5205
|
+
string,
|
|
5206
|
+
(
|
|
5207
|
+
builder: ProjectionBuilder,
|
|
5208
|
+
source: InvestigationEvidenceSource,
|
|
5209
|
+
payload: unknown,
|
|
5210
|
+
) => void
|
|
5211
|
+
> = {
|
|
5212
|
+
diagnose: adaptDiagnose,
|
|
5213
|
+
issues: adaptIssues,
|
|
5214
|
+
get_resource: adaptGetResource,
|
|
5215
|
+
list_resources: adaptListResources,
|
|
5216
|
+
get_events: adaptEvents,
|
|
5217
|
+
get_pod_logs: adaptPodLogs,
|
|
5218
|
+
get_changes: adaptChanges,
|
|
5219
|
+
get_neighborhood: adaptNeighborhood,
|
|
5220
|
+
get_topology: adaptTopology,
|
|
5221
|
+
get_workload_logs: adaptWorkloadLogs,
|
|
5222
|
+
get_prometheus_rules: adaptPrometheusRules,
|
|
5223
|
+
get_helm_release: adaptHelmRelease,
|
|
5224
|
+
get_subject_permissions: adaptSubjectPermissions,
|
|
5225
|
+
query_prometheus: adaptQueryPrometheus,
|
|
5226
|
+
};
|
|
5227
|
+
|
|
5228
|
+
/**
|
|
5229
|
+
* The pods every diagnose of the target listed as its own, gathered before
|
|
5230
|
+
* anything is classified. A metrics query is target evidence when it names
|
|
5231
|
+
* pods a producer established, and that must not depend on whether the agent
|
|
5232
|
+
* happened to run diagnose before or after the query: the same investigation
|
|
5233
|
+
* would otherwise read differently for the same facts.
|
|
5234
|
+
*/
|
|
5235
|
+
function collectEstablishedTargetPods(
|
|
5236
|
+
builder: ProjectionBuilder,
|
|
5237
|
+
turns: readonly InvestigationEvidenceTurn[],
|
|
5238
|
+
): void {
|
|
5239
|
+
for (const turn of turns) {
|
|
5240
|
+
for (const item of turn.timeline) {
|
|
5241
|
+
if (
|
|
5242
|
+
item.kind !== "tool" ||
|
|
5243
|
+
item.tool !== "diagnose" ||
|
|
5244
|
+
item.radarEvidence !== true ||
|
|
5245
|
+
item.status !== "done" ||
|
|
5246
|
+
// Confirmed success, the same test the sources use. This set is what
|
|
5247
|
+
// proves a Prometheus selector is about the target, so a result that
|
|
5248
|
+
// never said it succeeded must not put pods into it.
|
|
5249
|
+
item.isError !== false ||
|
|
5250
|
+
!nonEmptyString(item.result)
|
|
5251
|
+
) {
|
|
5252
|
+
continue;
|
|
5253
|
+
}
|
|
5254
|
+
const value = record(parseJSON(item.result));
|
|
5255
|
+
const resource = kubernetesResource(value?.resource);
|
|
5256
|
+
if (!value || !resource || !Array.isArray(value.podNames)) continue;
|
|
5257
|
+
if (
|
|
5258
|
+
relevanceForResource(builder, {
|
|
5259
|
+
kind: resource.kind,
|
|
5260
|
+
group: apiVersionToGroup(resource.apiVersion),
|
|
5261
|
+
namespace: resource.metadata.namespace,
|
|
5262
|
+
name: resource.metadata.name,
|
|
5263
|
+
}) !== "target"
|
|
5264
|
+
) {
|
|
5265
|
+
continue;
|
|
5266
|
+
}
|
|
5267
|
+
for (const pod of value.podNames) {
|
|
5268
|
+
if (nonEmptyString(pod)) builder.establishedTargetPods.add(pod);
|
|
5269
|
+
}
|
|
5270
|
+
}
|
|
5271
|
+
}
|
|
5272
|
+
}
|
|
5273
|
+
|
|
5274
|
+
export function projectInvestigationEvidence(
|
|
5275
|
+
turns: readonly InvestigationEvidenceTurn[],
|
|
5276
|
+
target: InvestigationEvidenceTarget,
|
|
5277
|
+
): InvestigationEvidenceProjection {
|
|
5278
|
+
const builder = new ProjectionBuilder(target);
|
|
5279
|
+
collectEstablishedTargetPods(builder, turns);
|
|
5280
|
+
const evidenceRefSources: InvestigationEvidenceSource[] = [];
|
|
5281
|
+
const citableSources: InvestigationEvidenceSource[] = [];
|
|
5282
|
+
const deferred: {
|
|
5283
|
+
adapt: (
|
|
5284
|
+
builder: ProjectionBuilder,
|
|
5285
|
+
source: InvestigationEvidenceSource,
|
|
5286
|
+
payload: unknown,
|
|
5287
|
+
) => void;
|
|
5288
|
+
source: InvestigationEvidenceSource;
|
|
5289
|
+
payload: unknown;
|
|
5290
|
+
}[] = [];
|
|
5291
|
+
let order = 0;
|
|
5292
|
+
for (const [turnIndex, turn] of turns.entries()) {
|
|
5293
|
+
for (const [timelineIndex, item] of turn.timeline.entries()) {
|
|
5294
|
+
const itemOrder = order;
|
|
5295
|
+
order += 1;
|
|
5296
|
+
if (item.kind !== "tool") continue;
|
|
5297
|
+
// Full-local agents may load user MCP servers whose bare tool names collide
|
|
5298
|
+
// with Radar's. Only results matched by the server to the active private
|
|
5299
|
+
// transport ledger may enter the surface labelled "Radar evidence".
|
|
5300
|
+
if (item.radarEvidence !== true) continue;
|
|
5301
|
+
const source: InvestigationEvidenceSource = {
|
|
5302
|
+
id: investigationEvidenceSourceId(turnIndex, item.id),
|
|
5303
|
+
turnIndex,
|
|
5304
|
+
timelineIndex,
|
|
5305
|
+
stepId: item.id,
|
|
5306
|
+
tool: item.tool,
|
|
5307
|
+
args: item.summary,
|
|
5308
|
+
order: itemOrder,
|
|
5309
|
+
phase: turn.verify
|
|
5310
|
+
? "verification"
|
|
5311
|
+
: turn.apply
|
|
5312
|
+
? "apply"
|
|
5313
|
+
: turn.question
|
|
5314
|
+
? "followup"
|
|
5315
|
+
: "initial",
|
|
5316
|
+
confirmedSuccess: item.status === "done" && item.isError === false,
|
|
5317
|
+
evidenceRef: item.evidenceRef,
|
|
5318
|
+
};
|
|
5319
|
+
if (item.evidenceRef) evidenceRefSources.push(source);
|
|
5320
|
+
if (item.status !== "done") continue;
|
|
5321
|
+
if (
|
|
5322
|
+
item.evidenceRef &&
|
|
5323
|
+
investigationEvidenceRefRe.test(item.evidenceRef) &&
|
|
5324
|
+
item.isError === false &&
|
|
5325
|
+
!item.truncated &&
|
|
5326
|
+
nonEmptyString(item.result)
|
|
5327
|
+
) {
|
|
5328
|
+
citableSources.push(source);
|
|
5329
|
+
}
|
|
5330
|
+
const adapt = ADAPTERS[item.tool];
|
|
5331
|
+
if (!adapt) continue;
|
|
5332
|
+
builder.addSource(source);
|
|
5333
|
+
if (item.isError === true) {
|
|
5334
|
+
builder.limit(
|
|
5335
|
+
source,
|
|
5336
|
+
investigationResultLabel(source),
|
|
5337
|
+
item.result || "This investigation step failed.",
|
|
5338
|
+
"error",
|
|
5339
|
+
);
|
|
5340
|
+
continue;
|
|
5341
|
+
}
|
|
5342
|
+
if (item.truncated) {
|
|
5343
|
+
builder.limit(
|
|
5344
|
+
source,
|
|
5345
|
+
investigationResultLabel(source),
|
|
3283
5346
|
"Only part of this investigation result was saved, so Radar could not summarize it here.",
|
|
3284
5347
|
"truncated",
|
|
3285
5348
|
);
|
|
@@ -3299,7 +5362,11 @@ export function projectInvestigationEvidence(
|
|
|
3299
5362
|
invalidPayload(builder, source);
|
|
3300
5363
|
continue;
|
|
3301
5364
|
}
|
|
3302
|
-
|
|
5365
|
+
if (MEMBERSHIP_DEPENDENT_ADAPTERS.has(item.tool)) {
|
|
5366
|
+
deferred.push({ adapt, source, payload });
|
|
5367
|
+
} else {
|
|
5368
|
+
adapt(builder, source, payload);
|
|
5369
|
+
}
|
|
3303
5370
|
if (item.isError !== false) {
|
|
3304
5371
|
builder.limit(
|
|
3305
5372
|
source,
|
|
@@ -3311,6 +5378,13 @@ export function projectInvestigationEvidence(
|
|
|
3311
5378
|
}
|
|
3312
5379
|
}
|
|
3313
5380
|
|
|
5381
|
+
// Membership is settled now: every read that could name one of the target's
|
|
5382
|
+
// pods has been adapted, so these see the same set whatever order the agent
|
|
5383
|
+
// worked in.
|
|
5384
|
+
for (const { adapt, source, payload } of deferred) {
|
|
5385
|
+
adapt(builder, source, payload);
|
|
5386
|
+
}
|
|
5387
|
+
|
|
3314
5388
|
const tierRank: Record<InvestigationEvidenceTier, number> = {
|
|
3315
5389
|
key: 0,
|
|
3316
5390
|
supporting: 1,
|
|
@@ -3430,6 +5504,7 @@ export function projectInvestigationEvidence(
|
|
|
3430
5504
|
}
|
|
3431
5505
|
break;
|
|
3432
5506
|
case "inventory":
|
|
5507
|
+
case "alerts":
|
|
3433
5508
|
break;
|
|
3434
5509
|
}
|
|
3435
5510
|
continue;
|
|
@@ -3478,7 +5553,12 @@ export function projectInvestigationEvidence(
|
|
|
3478
5553
|
|
|
3479
5554
|
return {
|
|
3480
5555
|
groups: builder.groups,
|
|
3481
|
-
|
|
5556
|
+
// Qualifications read in the order the investigation produced them, which
|
|
5557
|
+
// is the transcript's order and not the order the adapters happened to
|
|
5558
|
+
// run in: the membership-dependent ones are adapted last.
|
|
5559
|
+
limitations: [...builder.limitations].sort(
|
|
5560
|
+
(a, b) => a.firstOrder - b.firstOrder,
|
|
5561
|
+
),
|
|
3482
5562
|
sources: builder.sources,
|
|
3483
5563
|
evidenceRefSources,
|
|
3484
5564
|
citableSources,
|