@skyhook-io/radar-app 1.13.3 → 1.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/App.tsx +68 -23
- package/src/RadarApp.tsx +12 -1
- package/src/api/client.ts +24 -10
- package/src/components/diagnose/DiagnoseContext.test.ts +95 -0
- package/src/components/diagnose/DiagnoseContext.tsx +402 -77
- package/src/components/diagnose/DiagnoseSurface.test.tsx +53 -34
- package/src/components/diagnose/DiagnoseSurface.tsx +167 -92
- package/src/components/diagnose/Home.test.tsx +23 -1
- package/src/components/diagnose/Home.tsx +49 -3
- package/src/components/diagnose/InvestigationEvidencePane.test.tsx +665 -0
- package/src/components/diagnose/InvestigationEvidencePane.tsx +631 -25
- package/src/components/diagnose/InvestigationView.tsx +5 -0
- package/src/components/diagnose/LocalDiagnoseAction.tsx +2 -3
- package/src/components/diagnose/investigationEvidence.test.ts +2102 -9
- package/src/components/diagnose/investigationEvidence.ts +1357 -65
- package/src/components/diagnose/investigationSourceFocus.ts +4 -0
- package/src/components/diagnose/parts.test.tsx +55 -1
- package/src/components/diagnose/parts.tsx +68 -1
- package/src/components/resource/HPACharts.render.test.tsx +77 -0
- package/src/components/resource/HPACharts.tsx +32 -25
- package/src/components/resources/ResourcesView.tsx +27 -2
- package/src/index.ts +7 -5
|
@@ -89,7 +89,10 @@ export type InvestigationEvidenceKind =
|
|
|
89
89
|
| "relationships"
|
|
90
90
|
| "topology"
|
|
91
91
|
| "inventory"
|
|
92
|
-
| "receipt"
|
|
92
|
+
| "receipt"
|
|
93
|
+
| "alerts"
|
|
94
|
+
| "helm"
|
|
95
|
+
| "permissions";
|
|
93
96
|
|
|
94
97
|
type InvestigationSemanticDomain = "issue" | "startup" | "crash" | "dns";
|
|
95
98
|
|
|
@@ -240,6 +243,100 @@ export interface InvestigationNetworkEvidence {
|
|
|
240
243
|
routes: InvestigationNetworkRoute[];
|
|
241
244
|
}
|
|
242
245
|
|
|
246
|
+
/** One Prometheus rule as `get_prometheus_rules` flattens it (group stamped on). */
|
|
247
|
+
export interface InvestigationAlertRule {
|
|
248
|
+
group: string;
|
|
249
|
+
name: string;
|
|
250
|
+
type: string;
|
|
251
|
+
state?: string;
|
|
252
|
+
health?: string;
|
|
253
|
+
query?: string;
|
|
254
|
+
labels: Record<string, string>;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** One active instance of an alerting rule, with its own label set. */
|
|
258
|
+
export interface InvestigationAlertInstance {
|
|
259
|
+
state: string;
|
|
260
|
+
activeAt?: string;
|
|
261
|
+
value?: string;
|
|
262
|
+
labels: Record<string, string>;
|
|
263
|
+
/** The instance's labels name the investigated resource. */
|
|
264
|
+
namesTarget: boolean;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export interface InvestigationHelmOperation {
|
|
268
|
+
kind: string;
|
|
269
|
+
status: string;
|
|
270
|
+
message: string;
|
|
271
|
+
revision?: number;
|
|
272
|
+
failedRevision?: number;
|
|
273
|
+
rollbackRevision?: number;
|
|
274
|
+
updated?: string;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export interface InvestigationHelmOwnedResource {
|
|
278
|
+
kind: string;
|
|
279
|
+
apiVersion?: string;
|
|
280
|
+
name: string;
|
|
281
|
+
namespace: string;
|
|
282
|
+
status?: string;
|
|
283
|
+
ready?: string;
|
|
284
|
+
message?: string;
|
|
285
|
+
summary?: string;
|
|
286
|
+
issue?: string;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
export interface InvestigationHelmRelease {
|
|
290
|
+
name: string;
|
|
291
|
+
namespace: string;
|
|
292
|
+
/** Set only when Helm stores the release metadata elsewhere. */
|
|
293
|
+
storageNamespace?: string;
|
|
294
|
+
chart: string;
|
|
295
|
+
chartVersion: string;
|
|
296
|
+
appVersion?: string;
|
|
297
|
+
status: string;
|
|
298
|
+
revision: number;
|
|
299
|
+
updated: string;
|
|
300
|
+
description?: string;
|
|
301
|
+
resourceHealth?: string;
|
|
302
|
+
healthIssue?: string;
|
|
303
|
+
healthSummary?: string;
|
|
304
|
+
managedByFluxHelmRelease?: string;
|
|
305
|
+
lastOperation?: InvestigationHelmOperation;
|
|
306
|
+
resources: InvestigationHelmOwnedResource[];
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export interface InvestigationPermissionSubject {
|
|
310
|
+
kind: string;
|
|
311
|
+
namespace?: string;
|
|
312
|
+
name: string;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export interface InvestigationAccessCheck {
|
|
316
|
+
verb: string;
|
|
317
|
+
group?: string;
|
|
318
|
+
resource: string;
|
|
319
|
+
subresource?: string;
|
|
320
|
+
/** Empty means a cluster-scoped resource or cluster-wide request. */
|
|
321
|
+
namespace: string;
|
|
322
|
+
resourceName?: string;
|
|
323
|
+
allowed: boolean;
|
|
324
|
+
denied: boolean;
|
|
325
|
+
reason?: string;
|
|
326
|
+
evaluationError?: string;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
export interface InvestigationPermissionBinding {
|
|
330
|
+
bindingKind: string;
|
|
331
|
+
bindingNamespace?: string;
|
|
332
|
+
bindingName: string;
|
|
333
|
+
roleKind: string;
|
|
334
|
+
roleNamespace?: string;
|
|
335
|
+
roleName: string;
|
|
336
|
+
rulesCount: number;
|
|
337
|
+
inheritedFromGroup?: string;
|
|
338
|
+
}
|
|
339
|
+
|
|
243
340
|
export type InvestigationEvidenceData =
|
|
244
341
|
| {
|
|
245
342
|
type: "issue";
|
|
@@ -250,12 +347,20 @@ export type InvestigationEvidenceData =
|
|
|
250
347
|
* that they explain the resource under investigation.
|
|
251
348
|
*/
|
|
252
349
|
relevance: "target" | "producer-related" | "broader";
|
|
350
|
+
/** Pods whose startup blocker repeats this issue word for word. */
|
|
351
|
+
pods?: string[];
|
|
253
352
|
}
|
|
254
353
|
| {
|
|
255
354
|
type: "startup";
|
|
256
355
|
blocker: DiagnosisStartupBlocker;
|
|
257
356
|
/** Exact blocker object when the diagnosis producer established it. */
|
|
258
357
|
subject?: DiagnosisResourceRef;
|
|
358
|
+
/**
|
|
359
|
+
* Every pod the producer reported with this exact blocker. A DaemonSet
|
|
360
|
+
* with seven pending pods is one finding, not seven; the card carries the
|
|
361
|
+
* pod list instead of repeating itself.
|
|
362
|
+
*/
|
|
363
|
+
pods?: string[];
|
|
259
364
|
}
|
|
260
365
|
| {
|
|
261
366
|
type: "crash";
|
|
@@ -291,6 +396,8 @@ export type InvestigationEvidenceData =
|
|
|
291
396
|
changes: IssueRecentChange[];
|
|
292
397
|
scope: string;
|
|
293
398
|
changeContext?: DiagnosisChangeContext;
|
|
399
|
+
/** The resource whose change history the producer read, when it named one. */
|
|
400
|
+
subject?: { kind?: string; namespace?: string; name: string };
|
|
294
401
|
}
|
|
295
402
|
| { type: "dns"; dns: DiagnosisDNSContext }
|
|
296
403
|
| { type: "network"; network: InvestigationNetworkEvidence }
|
|
@@ -315,9 +422,29 @@ export type InvestigationEvidenceData =
|
|
|
315
422
|
}
|
|
316
423
|
| {
|
|
317
424
|
type: "receipt";
|
|
318
|
-
checked:
|
|
425
|
+
checked:
|
|
426
|
+
"issues" | "events" | "changes" | "inventory" | "logs" | "alerts";
|
|
319
427
|
scope: string;
|
|
320
428
|
message: string;
|
|
429
|
+
}
|
|
430
|
+
| {
|
|
431
|
+
type: "alerts";
|
|
432
|
+
rule: InvestigationAlertRule;
|
|
433
|
+
instances: InvestigationAlertInstance[];
|
|
434
|
+
annotations: Record<string, string>;
|
|
435
|
+
}
|
|
436
|
+
| { type: "helm"; release: InvestigationHelmRelease }
|
|
437
|
+
| {
|
|
438
|
+
type: "permissions";
|
|
439
|
+
subject: InvestigationPermissionSubject;
|
|
440
|
+
/** Present for the access-check response shape. */
|
|
441
|
+
accessCheck?: InvestigationAccessCheck;
|
|
442
|
+
/** Present for the subject-permissions response shape. */
|
|
443
|
+
bindings?: InvestigationPermissionBinding[];
|
|
444
|
+
flatRulesCount?: number;
|
|
445
|
+
truncated?: boolean;
|
|
446
|
+
usedByPods?: string[];
|
|
447
|
+
podsTotal?: number;
|
|
321
448
|
};
|
|
322
449
|
|
|
323
450
|
export interface InvestigationEvidenceObservation {
|
|
@@ -485,6 +612,29 @@ export function investigationEvidenceSubjectRef(
|
|
|
485
612
|
return data.network.subject;
|
|
486
613
|
case "relationships":
|
|
487
614
|
return data.root;
|
|
615
|
+
case "helm":
|
|
616
|
+
// A resource ref cannot carry the storage namespace, and opening the
|
|
617
|
+
// release by name alone would read the wrong storage.
|
|
618
|
+
if (
|
|
619
|
+
data.release.storageNamespace !== undefined &&
|
|
620
|
+
data.release.storageNamespace !== data.release.namespace
|
|
621
|
+
) {
|
|
622
|
+
return undefined;
|
|
623
|
+
}
|
|
624
|
+
return {
|
|
625
|
+
kind: "HelmRelease",
|
|
626
|
+
group: "helm.sh",
|
|
627
|
+
namespace: data.release.namespace,
|
|
628
|
+
name: data.release.name,
|
|
629
|
+
};
|
|
630
|
+
case "permissions":
|
|
631
|
+
// Users and Groups are principals, not Kubernetes objects.
|
|
632
|
+
if (data.subject.kind !== "ServiceAccount") return undefined;
|
|
633
|
+
return {
|
|
634
|
+
kind: "ServiceAccount",
|
|
635
|
+
namespace: data.subject.namespace,
|
|
636
|
+
name: data.subject.name,
|
|
637
|
+
};
|
|
488
638
|
default:
|
|
489
639
|
return undefined;
|
|
490
640
|
}
|
|
@@ -1116,6 +1266,10 @@ const INVESTIGATION_RESULT_LABELS: Readonly<Record<string, string>> = {
|
|
|
1116
1266
|
get_changes: "Recent changes",
|
|
1117
1267
|
get_neighborhood: "Relationships",
|
|
1118
1268
|
get_topology: "Topology",
|
|
1269
|
+
get_workload_logs: "Workload logs",
|
|
1270
|
+
get_prometheus_rules: "Alert rules",
|
|
1271
|
+
get_helm_release: "Helm release",
|
|
1272
|
+
get_subject_permissions: "Permissions",
|
|
1119
1273
|
};
|
|
1120
1274
|
|
|
1121
1275
|
function investigationResultLabel(source: InvestigationEvidenceSource): string {
|
|
@@ -1254,11 +1408,13 @@ class ProjectionBuilder {
|
|
|
1254
1408
|
kind === "startup" ||
|
|
1255
1409
|
kind === "crash" ||
|
|
1256
1410
|
(kind === "receipt" && identity.startsWith("previous-log-absence:"));
|
|
1257
|
-
const mapKey =
|
|
1411
|
+
const mapKey = groupKey(
|
|
1412
|
+
kind,
|
|
1413
|
+
identity,
|
|
1258
1414
|
partitionByRelevance
|
|
1259
|
-
?
|
|
1260
|
-
:
|
|
1261
|
-
|
|
1415
|
+
? `${observation.relevance}\u0000${scopeFromArgs(source)}`
|
|
1416
|
+
: undefined,
|
|
1417
|
+
);
|
|
1262
1418
|
let group = this.groupByIdentity.get(mapKey);
|
|
1263
1419
|
const previousObservation = group?.observations.at(-1);
|
|
1264
1420
|
const changedFromPrevious = previousObservation
|
|
@@ -1297,7 +1453,10 @@ class ProjectionBuilder {
|
|
|
1297
1453
|
// target was already established by a scoped producer. Keep the broad read
|
|
1298
1454
|
// in revision history, but never let weaker provenance replace the card's
|
|
1299
1455
|
// authoritative observation. Equal provenance still advances normally.
|
|
1456
|
+
// An alert rule's relevance is derived from its live instances, so a later
|
|
1457
|
+
// read of the same rule is a state transition, not weaker provenance.
|
|
1300
1458
|
if (
|
|
1459
|
+
kind === "alerts" ||
|
|
1301
1460
|
relevanceRank[next.relevance] <= relevanceRank[group.latest.relevance]
|
|
1302
1461
|
) {
|
|
1303
1462
|
group.latest = next;
|
|
@@ -1306,6 +1465,14 @@ class ProjectionBuilder {
|
|
|
1306
1465
|
if (next.tier === "checked") this.checkedSources.add(source.id);
|
|
1307
1466
|
}
|
|
1308
1467
|
|
|
1468
|
+
/** Relevance of the current card for an unpartitioned identity, if observed. */
|
|
1469
|
+
latestRelevance(
|
|
1470
|
+
kind: InvestigationEvidenceKind,
|
|
1471
|
+
identity: string,
|
|
1472
|
+
): InvestigationEvidenceRelevance | undefined {
|
|
1473
|
+
return this.groupByIdentity.get(groupKey(kind, identity))?.latest.relevance;
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1309
1476
|
limit(
|
|
1310
1477
|
source: InvestigationEvidenceSource,
|
|
1311
1478
|
label: string,
|
|
@@ -1337,6 +1504,18 @@ class ProjectionBuilder {
|
|
|
1337
1504
|
}
|
|
1338
1505
|
}
|
|
1339
1506
|
|
|
1507
|
+
/**
|
|
1508
|
+
* The one place a group's map key is built. NUL separates the parts so no
|
|
1509
|
+
* kind or identity text can collide with another kind's key.
|
|
1510
|
+
*/
|
|
1511
|
+
function groupKey(
|
|
1512
|
+
kind: InvestigationEvidenceKind,
|
|
1513
|
+
identity: string,
|
|
1514
|
+
partition?: string,
|
|
1515
|
+
): string {
|
|
1516
|
+
return `${kind}\u0000${identity}${partition === undefined ? "" : `\u0000${partition}`}`;
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1340
1519
|
export function evidenceSemanticSnapshot(
|
|
1341
1520
|
observation: Pick<
|
|
1342
1521
|
InvestigationEvidenceObservation,
|
|
@@ -1352,23 +1531,35 @@ export function evidenceSemanticSnapshot(
|
|
|
1352
1531
|
});
|
|
1353
1532
|
}
|
|
1354
1533
|
const data =
|
|
1355
|
-
observation.data.type === "
|
|
1534
|
+
observation.data.type === "alerts"
|
|
1356
1535
|
? {
|
|
1357
1536
|
type: observation.data.type,
|
|
1358
|
-
//
|
|
1359
|
-
//
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
severity: observation.data.issue.severity,
|
|
1367
|
-
cause: observation.data.issue.cause,
|
|
1368
|
-
message: observation.data.issue.message,
|
|
1369
|
-
},
|
|
1537
|
+
// An instance's sampled value and activation time move on every
|
|
1538
|
+
// evaluation; the finding is which instances exist and their state.
|
|
1539
|
+
rule: observation.data.rule,
|
|
1540
|
+
instances: observation.data.instances.map((instance) => ({
|
|
1541
|
+
state: instance.state,
|
|
1542
|
+
labels: instance.labels,
|
|
1543
|
+
})),
|
|
1544
|
+
annotations: observation.data.annotations,
|
|
1370
1545
|
}
|
|
1371
|
-
: observation.data
|
|
1546
|
+
: observation.data.type === "issue"
|
|
1547
|
+
? {
|
|
1548
|
+
type: observation.data.type,
|
|
1549
|
+
// These are the finding and details shown in the evidence card.
|
|
1550
|
+
// Collection timestamps and detector bookkeeping do not change it.
|
|
1551
|
+
issue: {
|
|
1552
|
+
kind: observation.data.issue.kind,
|
|
1553
|
+
group: observation.data.issue.group,
|
|
1554
|
+
namespace: observation.data.issue.namespace,
|
|
1555
|
+
name: observation.data.issue.name,
|
|
1556
|
+
reason: observation.data.issue.reason,
|
|
1557
|
+
severity: observation.data.issue.severity,
|
|
1558
|
+
cause: observation.data.issue.cause,
|
|
1559
|
+
message: observation.data.issue.message,
|
|
1560
|
+
},
|
|
1561
|
+
}
|
|
1562
|
+
: observation.data;
|
|
1372
1563
|
return JSON.stringify({
|
|
1373
1564
|
tone: observation.tone,
|
|
1374
1565
|
title: observation.title,
|
|
@@ -1568,6 +1759,7 @@ function addIssueObservation(
|
|
|
1568
1759
|
source: InvestigationEvidenceSource,
|
|
1569
1760
|
value: Issue,
|
|
1570
1761
|
producerRelevance: InvestigationEvidenceRelevance = "broader",
|
|
1762
|
+
pods?: string[],
|
|
1571
1763
|
): void {
|
|
1572
1764
|
const matchesTarget = resourceMatchesTarget(builder.target, {
|
|
1573
1765
|
kind: value.kind,
|
|
@@ -1587,7 +1779,12 @@ function addIssueObservation(
|
|
|
1587
1779
|
relevance,
|
|
1588
1780
|
title: value.reason,
|
|
1589
1781
|
summary: value.cause || value.message,
|
|
1590
|
-
data: {
|
|
1782
|
+
data: {
|
|
1783
|
+
type: "issue",
|
|
1784
|
+
issue: value,
|
|
1785
|
+
relevance,
|
|
1786
|
+
...(pods && pods.length > 0 ? { pods } : {}),
|
|
1787
|
+
},
|
|
1591
1788
|
});
|
|
1592
1789
|
addIssueLimitations(builder, source, value);
|
|
1593
1790
|
}
|
|
@@ -1636,6 +1833,10 @@ function addEvents(
|
|
|
1636
1833
|
complete = true,
|
|
1637
1834
|
emptyIsAuthoritative = false,
|
|
1638
1835
|
relevance: InvestigationEvidenceRelevance = "broader",
|
|
1836
|
+
emptyReceipt: { title: string; message: string } = {
|
|
1837
|
+
title: "No matching warning events",
|
|
1838
|
+
message: "The warning-event query completed and returned no groups.",
|
|
1839
|
+
},
|
|
1639
1840
|
): void {
|
|
1640
1841
|
const scope = scopeFromArgs(source);
|
|
1641
1842
|
if (values.length === 0) {
|
|
@@ -1653,17 +1854,18 @@ function addEvents(
|
|
|
1653
1854
|
tier: evidenceTierForRelevance("checked", relevance),
|
|
1654
1855
|
relevance,
|
|
1655
1856
|
tone: "neutral",
|
|
1656
|
-
title:
|
|
1857
|
+
title: emptyReceipt.title,
|
|
1657
1858
|
summary: scope,
|
|
1658
1859
|
data: {
|
|
1659
1860
|
type: "receipt",
|
|
1660
1861
|
checked: "events",
|
|
1661
1862
|
scope,
|
|
1662
|
-
message:
|
|
1863
|
+
message: emptyReceipt.message,
|
|
1663
1864
|
},
|
|
1664
1865
|
});
|
|
1665
1866
|
return;
|
|
1666
1867
|
}
|
|
1868
|
+
const lead = leadEvent(values);
|
|
1667
1869
|
builder.observe(identity, "events", source, {
|
|
1668
1870
|
tier: evidenceTierForRelevance(
|
|
1669
1871
|
values.some((item) => item.type.toLowerCase() === "warning")
|
|
@@ -1676,11 +1878,32 @@ function addEvents(
|
|
|
1676
1878
|
? "warning"
|
|
1677
1879
|
: "info",
|
|
1678
1880
|
title: "Kubernetes events",
|
|
1679
|
-
summary: `${
|
|
1881
|
+
summary: `${lead.reason}: ${lead.message}${
|
|
1882
|
+
values.length > 1 ? ` · ${values.length} event groups` : ""
|
|
1883
|
+
} · ${scope}`,
|
|
1680
1884
|
data: { type: "events", events: values, scope },
|
|
1681
1885
|
});
|
|
1682
1886
|
}
|
|
1683
1887
|
|
|
1888
|
+
/**
|
|
1889
|
+
* The newest warning, else the newest event. The card lead must be what
|
|
1890
|
+
* happened, not how many groups the producer returned.
|
|
1891
|
+
*/
|
|
1892
|
+
function leadEvent(
|
|
1893
|
+
values: InvestigationEventEvidence[],
|
|
1894
|
+
): InvestigationEventEvidence {
|
|
1895
|
+
const newest = (items: InvestigationEventEvidence[]) =>
|
|
1896
|
+
items.reduce((best, item) =>
|
|
1897
|
+
Date.parse(item.lastTimestamp) > Date.parse(best.lastTimestamp)
|
|
1898
|
+
? item
|
|
1899
|
+
: best,
|
|
1900
|
+
);
|
|
1901
|
+
const warnings = values.filter(
|
|
1902
|
+
(item) => item.type.toLowerCase() === "warning",
|
|
1903
|
+
);
|
|
1904
|
+
return newest(warnings.length > 0 ? warnings : values);
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1684
1907
|
function addChanges(
|
|
1685
1908
|
builder: ProjectionBuilder,
|
|
1686
1909
|
source: InvestigationEvidenceSource,
|
|
@@ -1690,6 +1913,8 @@ function addChanges(
|
|
|
1690
1913
|
complete = true,
|
|
1691
1914
|
emptyIsAuthoritative = false,
|
|
1692
1915
|
relevance: InvestigationEvidenceRelevance = "broader",
|
|
1916
|
+
subject?: { kind?: string; namespace?: string; name: string },
|
|
1917
|
+
window?: string,
|
|
1693
1918
|
): void {
|
|
1694
1919
|
const scope = scopeFromArgs(source);
|
|
1695
1920
|
if (values.length === 0 && !changeContext?.changed) {
|
|
@@ -1728,7 +1953,7 @@ function addChanges(
|
|
|
1728
1953
|
),
|
|
1729
1954
|
relevance,
|
|
1730
1955
|
tone: "info",
|
|
1731
|
-
title:
|
|
1956
|
+
title: `Recent changes${window ? ` · last ${window}` : ""}`,
|
|
1732
1957
|
summary: changeContext?.changed
|
|
1733
1958
|
? changeContext.what === "The workload's Pod template changed" &&
|
|
1734
1959
|
changeContext.when
|
|
@@ -1740,10 +1965,23 @@ function addChanges(
|
|
|
1740
1965
|
changes: values,
|
|
1741
1966
|
scope,
|
|
1742
1967
|
changeContext,
|
|
1968
|
+
subject,
|
|
1743
1969
|
},
|
|
1744
1970
|
});
|
|
1745
1971
|
}
|
|
1746
1972
|
|
|
1973
|
+
function changesSubjectFromArgs(
|
|
1974
|
+
source: InvestigationEvidenceSource,
|
|
1975
|
+
): { kind?: string; namespace?: string; name: string } | undefined {
|
|
1976
|
+
const args = record(source.args ? parseJSON(source.args) : undefined);
|
|
1977
|
+
if (!nonEmptyString(args?.name)) return undefined;
|
|
1978
|
+
return {
|
|
1979
|
+
...(nonEmptyString(args.kind) ? { kind: args.kind } : {}),
|
|
1980
|
+
...(nonEmptyString(args.namespace) ? { namespace: args.namespace } : {}),
|
|
1981
|
+
name: args.name,
|
|
1982
|
+
};
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1747
1985
|
function addLogs(
|
|
1748
1986
|
builder: ProjectionBuilder,
|
|
1749
1987
|
source: InvestigationEvidenceSource,
|
|
@@ -2068,11 +2306,57 @@ function adaptDiagnose(
|
|
|
2068
2306
|
if (!relatedValid) {
|
|
2069
2307
|
invalidPayload(builder, source, "Classified issues");
|
|
2070
2308
|
}
|
|
2309
|
+
const blockersRaw = value.startupBlockers;
|
|
2310
|
+
let blockersValid = blockersRaw === undefined || Array.isArray(blockersRaw);
|
|
2311
|
+
// Pods of one workload usually share a blocker word for word. Merge those
|
|
2312
|
+
// into one finding with the pod list; anything else keeps its own card and
|
|
2313
|
+
// its own identity so saved runs match as before.
|
|
2314
|
+
const blockerGroups: Array<{
|
|
2315
|
+
blocker: DiagnosisStartupBlocker;
|
|
2316
|
+
pods: string[];
|
|
2317
|
+
foldedInto?: Issue;
|
|
2318
|
+
}> = [];
|
|
2319
|
+
if (Array.isArray(blockersRaw)) {
|
|
2320
|
+
const merged = new Map<string, (typeof blockerGroups)[number]>();
|
|
2321
|
+
for (const raw of blockersRaw) {
|
|
2322
|
+
const blocker = startupBlocker(raw);
|
|
2323
|
+
if (!blocker) {
|
|
2324
|
+
blockersValid = false;
|
|
2325
|
+
invalidPayload(builder, source, "Startup evidence");
|
|
2326
|
+
continue;
|
|
2327
|
+
}
|
|
2328
|
+
const key =
|
|
2329
|
+
blocker.kind === "Pod"
|
|
2330
|
+
? `${blocker.reason} ${blocker.severity} ${blocker.message}`
|
|
2331
|
+
: undefined;
|
|
2332
|
+
const existing = key ? merged.get(key) : undefined;
|
|
2333
|
+
if (existing) {
|
|
2334
|
+
if (!existing.pods.includes(blocker.name)) {
|
|
2335
|
+
existing.pods.push(blocker.name);
|
|
2336
|
+
}
|
|
2337
|
+
continue;
|
|
2338
|
+
}
|
|
2339
|
+
const entry = { blocker, pods: [blocker.name] };
|
|
2340
|
+
if (key) merged.set(key, entry);
|
|
2341
|
+
blockerGroups.push(entry);
|
|
2342
|
+
}
|
|
2343
|
+
}
|
|
2071
2344
|
for (const item of related) {
|
|
2072
2345
|
// `diagnose` is itself scoped to one resource and declares these rows as
|
|
2073
2346
|
// related evidence. That producer contract is stronger than a broad
|
|
2074
2347
|
// `issues` query, even when the related row is a child Pod.
|
|
2075
|
-
|
|
2348
|
+
// A classified issue and the pods' startup blocker are the same fact
|
|
2349
|
+
// seen from the workload and from its pods; one card carries both.
|
|
2350
|
+
const folded = blockerGroups.find(
|
|
2351
|
+
(group) =>
|
|
2352
|
+
!group.foldedInto &&
|
|
2353
|
+
group.blocker.kind === "Pod" &&
|
|
2354
|
+
group.blocker.reason === item.reason &&
|
|
2355
|
+
(group.blocker.message === item.message ||
|
|
2356
|
+
group.blocker.message === item.cause),
|
|
2357
|
+
);
|
|
2358
|
+
if (folded) folded.foldedInto = item;
|
|
2359
|
+
addIssueObservation(builder, source, item, relatedRelevance, folded?.pods);
|
|
2076
2360
|
}
|
|
2077
2361
|
|
|
2078
2362
|
const context = contextFrom(value.resourceContext);
|
|
@@ -2120,18 +2404,14 @@ function adaptDiagnose(
|
|
|
2120
2404
|
bundleRelevance,
|
|
2121
2405
|
);
|
|
2122
2406
|
|
|
2123
|
-
const blockersRaw = value.startupBlockers;
|
|
2124
|
-
let blockersValid = blockersRaw === undefined || Array.isArray(blockersRaw);
|
|
2125
2407
|
if (Array.isArray(blockersRaw)) {
|
|
2126
|
-
for (const
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
blockersValid = false;
|
|
2130
|
-
invalidPayload(builder, source, "Startup evidence");
|
|
2131
|
-
continue;
|
|
2132
|
-
}
|
|
2408
|
+
for (const { blocker, pods, foldedInto } of blockerGroups) {
|
|
2409
|
+
if (foldedInto) continue;
|
|
2410
|
+
const grouped = blocker.kind === "Pod" && pods.length > 1;
|
|
2133
2411
|
builder.observe(
|
|
2134
|
-
|
|
2412
|
+
grouped
|
|
2413
|
+
? `startup:Pod:${blocker.reason}:${fnv1a32(blocker.message).toString(36)}`
|
|
2414
|
+
: `startup:${blocker.kind}:${blocker.name}:${blocker.reason}`,
|
|
2135
2415
|
"startup",
|
|
2136
2416
|
source,
|
|
2137
2417
|
{
|
|
@@ -2142,30 +2422,35 @@ function adaptDiagnose(
|
|
|
2142
2422
|
relevance: bundledRowRelevance(blocker.kind, blocker.name),
|
|
2143
2423
|
tone: diagnosisSeverityTone(blocker.severity),
|
|
2144
2424
|
title: blocker.reason,
|
|
2145
|
-
summary:
|
|
2425
|
+
summary: grouped
|
|
2426
|
+
? `${pods.length} pods · ${blocker.message}`
|
|
2427
|
+
: blocker.message,
|
|
2146
2428
|
data: {
|
|
2147
2429
|
type: "startup",
|
|
2148
2430
|
blocker,
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2431
|
+
...(grouped ? { pods } : {}),
|
|
2432
|
+
subject: grouped
|
|
2433
|
+
? undefined
|
|
2434
|
+
: (() => {
|
|
2435
|
+
const namespace = resource.metadata.namespace;
|
|
2436
|
+
if (!namespace) return undefined;
|
|
2437
|
+
const rootGroup = apiVersionToGroup(resource.apiVersion);
|
|
2438
|
+
const group =
|
|
2439
|
+
blocker.kind === resource.kind
|
|
2440
|
+
? rootGroup
|
|
2441
|
+
: blocker.kind === "Pod"
|
|
2442
|
+
? ""
|
|
2443
|
+
: blocker.kind === "ReplicaSet"
|
|
2444
|
+
? "apps"
|
|
2445
|
+
: undefined;
|
|
2446
|
+
if (group === undefined) return undefined;
|
|
2447
|
+
return {
|
|
2448
|
+
kind: blocker.kind,
|
|
2449
|
+
...(group ? { group } : {}),
|
|
2450
|
+
namespace,
|
|
2451
|
+
name: blocker.name,
|
|
2452
|
+
};
|
|
2453
|
+
})(),
|
|
2169
2454
|
},
|
|
2170
2455
|
},
|
|
2171
2456
|
);
|
|
@@ -2450,6 +2735,11 @@ function adaptDiagnose(
|
|
|
2450
2735
|
if (value.changeContext !== undefined && !changeContext) {
|
|
2451
2736
|
invalidPayload(builder, source, "Change correlation");
|
|
2452
2737
|
}
|
|
2738
|
+
const diagnoseChangesSubject = {
|
|
2739
|
+
kind: resource.kind,
|
|
2740
|
+
namespace: resource.metadata.namespace,
|
|
2741
|
+
name: resource.metadata.name,
|
|
2742
|
+
};
|
|
2453
2743
|
if (Array.isArray(changesRaw)) {
|
|
2454
2744
|
const changes = changesRaw
|
|
2455
2745
|
.map(recentChange)
|
|
@@ -2466,6 +2756,7 @@ function adaptDiagnose(
|
|
|
2466
2756
|
changesCoverageLimitedValid,
|
|
2467
2757
|
true,
|
|
2468
2758
|
relatedRelevance,
|
|
2759
|
+
diagnoseChangesSubject,
|
|
2469
2760
|
);
|
|
2470
2761
|
} else {
|
|
2471
2762
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2486,6 +2777,7 @@ function adaptDiagnose(
|
|
|
2486
2777
|
changesCoverageLimitedValid,
|
|
2487
2778
|
true,
|
|
2488
2779
|
relatedRelevance,
|
|
2780
|
+
diagnoseChangesSubject,
|
|
2489
2781
|
);
|
|
2490
2782
|
} else if (changesRaw !== undefined) {
|
|
2491
2783
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2784,6 +3076,11 @@ function adaptGetResource(
|
|
|
2784
3076
|
invalidPayload(builder, source, "Events");
|
|
2785
3077
|
}
|
|
2786
3078
|
}
|
|
3079
|
+
const resourceChangesSubject = {
|
|
3080
|
+
kind: resource.kind,
|
|
3081
|
+
namespace: resource.metadata.namespace,
|
|
3082
|
+
name: resource.metadata.name,
|
|
3083
|
+
};
|
|
2787
3084
|
const recentChangesRaw = value.recentChanges;
|
|
2788
3085
|
const recentChangesSaturatedRaw = value.recentChangesSaturated;
|
|
2789
3086
|
const recentChangesCoverageLimitedRaw = value.recentChangesCoverageLimited;
|
|
@@ -2814,6 +3111,7 @@ function adaptGetResource(
|
|
|
2814
3111
|
!nonEmptyString(value.recentChangesError),
|
|
2815
3112
|
true,
|
|
2816
3113
|
relevance,
|
|
3114
|
+
resourceChangesSubject,
|
|
2817
3115
|
);
|
|
2818
3116
|
} else {
|
|
2819
3117
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2833,6 +3131,7 @@ function adaptGetResource(
|
|
|
2833
3131
|
recentChangesCoverageLimitedRaw === false,
|
|
2834
3132
|
true,
|
|
2835
3133
|
relevance,
|
|
3134
|
+
resourceChangesSubject,
|
|
2836
3135
|
);
|
|
2837
3136
|
} else if (recentChangesRaw !== undefined) {
|
|
2838
3137
|
invalidPayload(builder, source, "Recent changes");
|
|
@@ -2920,26 +3219,48 @@ function adaptEvents(
|
|
|
2920
3219
|
payload: unknown,
|
|
2921
3220
|
): void {
|
|
2922
3221
|
const value = record(payload);
|
|
2923
|
-
|
|
3222
|
+
// The producer serializes an empty result as a nil slice, which is null.
|
|
3223
|
+
const eventsRaw = value?.events === null ? [] : value?.events;
|
|
3224
|
+
if (!value || !Array.isArray(eventsRaw)) {
|
|
2924
3225
|
invalidPayload(builder, source);
|
|
2925
3226
|
return;
|
|
2926
3227
|
}
|
|
2927
|
-
const events =
|
|
3228
|
+
const events = eventsRaw
|
|
2928
3229
|
.map(event)
|
|
2929
3230
|
.filter((item): item is InvestigationEventEvidence => Boolean(item));
|
|
2930
|
-
if (events.length !==
|
|
3231
|
+
if (events.length !== eventsRaw.length) {
|
|
2931
3232
|
invalidPayload(builder, source);
|
|
2932
3233
|
return;
|
|
2933
3234
|
}
|
|
2934
3235
|
addNarrowHint(builder, source, value);
|
|
3236
|
+
// The producer answers a namespace the caller cannot read with an empty
|
|
3237
|
+
// list and marks it. Only that case is a coverage gap. Any other complete,
|
|
3238
|
+
// successful empty read answers the question it asked and is filed as a
|
|
3239
|
+
// checked receipt; a gap there would contradict an events card from another
|
|
3240
|
+
// call in the same turn. The receipt names the one remaining ambiguity for
|
|
3241
|
+
// producers that predate the marker.
|
|
3242
|
+
if (value.accessDenied === true) {
|
|
3243
|
+
builder.limit(
|
|
3244
|
+
source,
|
|
3245
|
+
"Events",
|
|
3246
|
+
`Events in ${scopeFromArgs(source)} are not readable with your permissions.`,
|
|
3247
|
+
"error",
|
|
3248
|
+
);
|
|
3249
|
+
return;
|
|
3250
|
+
}
|
|
2935
3251
|
addEvents(
|
|
2936
3252
|
builder,
|
|
2937
3253
|
source,
|
|
2938
3254
|
events,
|
|
2939
3255
|
`events:${source.args ?? scopeFromArgs(source)}`,
|
|
2940
3256
|
!nonEmptyString(value.narrowHint),
|
|
2941
|
-
|
|
3257
|
+
true,
|
|
2942
3258
|
sourceArgsRelevance(builder, source),
|
|
3259
|
+
{
|
|
3260
|
+
title: "No events matched",
|
|
3261
|
+
message:
|
|
3262
|
+
"The events query completed and returned nothing for this scope. Events outside its window or filters are not covered; a namespace you cannot read also returns nothing.",
|
|
3263
|
+
},
|
|
2943
3264
|
);
|
|
2944
3265
|
}
|
|
2945
3266
|
|
|
@@ -2978,14 +3299,16 @@ function adaptChanges(
|
|
|
2978
3299
|
payload: unknown,
|
|
2979
3300
|
): void {
|
|
2980
3301
|
const value = record(payload);
|
|
2981
|
-
|
|
3302
|
+
// The producer serializes an empty result as a nil slice, which is null.
|
|
3303
|
+
const changesRaw = value?.changes === null ? [] : value?.changes;
|
|
3304
|
+
if (!value || !Array.isArray(changesRaw)) {
|
|
2982
3305
|
invalidPayload(builder, source, "Recent changes");
|
|
2983
3306
|
return;
|
|
2984
3307
|
}
|
|
2985
|
-
const changes =
|
|
3308
|
+
const changes = changesRaw
|
|
2986
3309
|
.map(recentChange)
|
|
2987
3310
|
.filter((item): item is IssueRecentChange => Boolean(item));
|
|
2988
|
-
if (changes.length !==
|
|
3311
|
+
if (changes.length !== changesRaw.length) {
|
|
2989
3312
|
invalidPayload(builder, source, "Recent changes");
|
|
2990
3313
|
return;
|
|
2991
3314
|
}
|
|
@@ -3001,15 +3324,20 @@ function adaptChanges(
|
|
|
3001
3324
|
} else if (value.sourcesErrored !== undefined) {
|
|
3002
3325
|
invalidPayload(builder, source, "Recent changes source coverage");
|
|
3003
3326
|
}
|
|
3327
|
+
// Reads of one resource's history are revisions of one card however the
|
|
3328
|
+
// window or cap differs; the window is stated in the title instead.
|
|
3329
|
+
const args = record(source.args ? parseJSON(source.args) : undefined);
|
|
3004
3330
|
addChanges(
|
|
3005
3331
|
builder,
|
|
3006
3332
|
source,
|
|
3007
3333
|
changes,
|
|
3008
|
-
`changes:${
|
|
3334
|
+
`changes:${scopeFromArgs(source)}`,
|
|
3009
3335
|
undefined,
|
|
3010
3336
|
!nonEmptyString(value.narrowHint) && sourceErrors === 0,
|
|
3011
3337
|
false,
|
|
3012
3338
|
sourceArgsRelevance(builder, source),
|
|
3339
|
+
changesSubjectFromArgs(source),
|
|
3340
|
+
nonEmptyString(args?.since) ? args.since : undefined,
|
|
3013
3341
|
);
|
|
3014
3342
|
}
|
|
3015
3343
|
|
|
@@ -3199,6 +3527,965 @@ function adaptTopology(
|
|
|
3199
3527
|
);
|
|
3200
3528
|
}
|
|
3201
3529
|
|
|
3530
|
+
const WORKLOAD_LOG_KINDS: Readonly<Record<string, string>> = {
|
|
3531
|
+
deployments: "Deployment",
|
|
3532
|
+
statefulsets: "StatefulSet",
|
|
3533
|
+
daemonsets: "DaemonSet",
|
|
3534
|
+
rollouts: "Rollout",
|
|
3535
|
+
jobs: "Job",
|
|
3536
|
+
workflows: "Workflow",
|
|
3537
|
+
};
|
|
3538
|
+
|
|
3539
|
+
/** `get_workload_logs` states what it read as `<plural>/<namespace>/<name>`. */
|
|
3540
|
+
function workloadLogsSubject(
|
|
3541
|
+
value: string,
|
|
3542
|
+
): { kind: string; namespace: string; name: string } | undefined {
|
|
3543
|
+
const parts = value.split("/");
|
|
3544
|
+
if (parts.length !== 3) return undefined;
|
|
3545
|
+
const kind = WORKLOAD_LOG_KINDS[parts[0]];
|
|
3546
|
+
if (!kind || !parts[1] || !parts[2]) return undefined;
|
|
3547
|
+
return { kind, namespace: parts[1], name: parts[2] };
|
|
3548
|
+
}
|
|
3549
|
+
|
|
3550
|
+
function adaptWorkloadLogs(
|
|
3551
|
+
builder: ProjectionBuilder,
|
|
3552
|
+
source: InvestigationEvidenceSource,
|
|
3553
|
+
payload: unknown,
|
|
3554
|
+
): void {
|
|
3555
|
+
const value = record(payload);
|
|
3556
|
+
const workload = nonEmptyString(value?.workload)
|
|
3557
|
+
? workloadLogsSubject(value.workload)
|
|
3558
|
+
: undefined;
|
|
3559
|
+
if (!value || !workload || !nonNegativeInteger(value.pods)) {
|
|
3560
|
+
invalidPayload(builder, source);
|
|
3561
|
+
return;
|
|
3562
|
+
}
|
|
3563
|
+
// The producer names the workload without an API group; as with other
|
|
3564
|
+
// group-less producers, the investigation target supplies that dimension.
|
|
3565
|
+
const workloadRelevance = relevanceForResource(builder, {
|
|
3566
|
+
...workload,
|
|
3567
|
+
group: builder.target.group,
|
|
3568
|
+
});
|
|
3569
|
+
const scope = `${displayKind(workload.kind)} ${workload.namespace}/${workload.name}`;
|
|
3570
|
+
const previous = previousFromArgs(source);
|
|
3571
|
+
addNarrowHint(builder, source, value);
|
|
3572
|
+
if (nonEmptyString(value.logsError)) {
|
|
3573
|
+
builder.limit(source, "Workload logs", value.logsError, "error");
|
|
3574
|
+
return;
|
|
3575
|
+
}
|
|
3576
|
+
if (value.pods === 0) {
|
|
3577
|
+
// The producer replaces the stream list with a message when it resolved
|
|
3578
|
+
// no pods; any other shape is not this response.
|
|
3579
|
+
if (
|
|
3580
|
+
typeof value.logs !== "string" ||
|
|
3581
|
+
(value.emptyMessage !== undefined &&
|
|
3582
|
+
typeof value.emptyMessage !== "string") ||
|
|
3583
|
+
value.narrowHint !== undefined
|
|
3584
|
+
) {
|
|
3585
|
+
invalidPayload(builder, source);
|
|
3586
|
+
return;
|
|
3587
|
+
}
|
|
3588
|
+
if (!source.confirmedSuccess) return;
|
|
3589
|
+
const message = nonEmptyString(value.emptyMessage)
|
|
3590
|
+
? value.emptyMessage
|
|
3591
|
+
: nonEmptyString(value.logs)
|
|
3592
|
+
? value.logs
|
|
3593
|
+
: "The workload resolved no pods, so there were no log streams to read.";
|
|
3594
|
+
builder.observe(
|
|
3595
|
+
`workload-logs:${previous ? "previous" : "current"}:${scope}`,
|
|
3596
|
+
"receipt",
|
|
3597
|
+
source,
|
|
3598
|
+
{
|
|
3599
|
+
tier: evidenceTierForRelevance("checked", workloadRelevance),
|
|
3600
|
+
relevance: workloadRelevance,
|
|
3601
|
+
tone: "neutral",
|
|
3602
|
+
title: "No pods to read logs from",
|
|
3603
|
+
summary: scope,
|
|
3604
|
+
data: { type: "receipt", checked: "logs", scope, message },
|
|
3605
|
+
},
|
|
3606
|
+
);
|
|
3607
|
+
return;
|
|
3608
|
+
}
|
|
3609
|
+
const logsRaw = value.logs;
|
|
3610
|
+
const noStreams = `No log streams were returned for the ${value.pods} resolved pod${value.pods === 1 ? "" : "s"}, so Radar could not evaluate them.`;
|
|
3611
|
+
if (!Array.isArray(logsRaw)) {
|
|
3612
|
+
if (logsRaw === undefined || logsRaw === null) {
|
|
3613
|
+
builder.limit(source, "Workload logs", noStreams, "unknown");
|
|
3614
|
+
} else {
|
|
3615
|
+
invalidPayload(builder, source);
|
|
3616
|
+
}
|
|
3617
|
+
return;
|
|
3618
|
+
}
|
|
3619
|
+
const warnings = stringArray(value.warnings) ?? [];
|
|
3620
|
+
for (const raw of logsRaw) {
|
|
3621
|
+
const entry = parseLogEntry(raw);
|
|
3622
|
+
if (!entry) {
|
|
3623
|
+
invalidPayload(
|
|
3624
|
+
builder,
|
|
3625
|
+
source,
|
|
3626
|
+
previous ? "Previous logs" : "Current logs",
|
|
3627
|
+
);
|
|
3628
|
+
continue;
|
|
3629
|
+
}
|
|
3630
|
+
// A Pod target is named exactly by its own row; a workload target relates
|
|
3631
|
+
// to its rows the way diagnose relates to the pods it resolved.
|
|
3632
|
+
const rowRelevance: InvestigationEvidenceRelevance = resourceMatchesTarget(
|
|
3633
|
+
builder.target,
|
|
3634
|
+
{
|
|
3635
|
+
kind: "Pod",
|
|
3636
|
+
group: "",
|
|
3637
|
+
namespace: workload.namespace,
|
|
3638
|
+
name: entry.pod,
|
|
3639
|
+
},
|
|
3640
|
+
)
|
|
3641
|
+
? "target"
|
|
3642
|
+
: workloadRelevance === "target"
|
|
3643
|
+
? "producer-related"
|
|
3644
|
+
: "broader";
|
|
3645
|
+
addLogs(
|
|
3646
|
+
builder,
|
|
3647
|
+
source,
|
|
3648
|
+
entry,
|
|
3649
|
+
previous,
|
|
3650
|
+
warnings,
|
|
3651
|
+
rowRelevance,
|
|
3652
|
+
workload.namespace,
|
|
3653
|
+
);
|
|
3654
|
+
}
|
|
3655
|
+
if (logsRaw.length === 0) {
|
|
3656
|
+
builder.limit(source, "Workload logs", noStreams, "unknown");
|
|
3657
|
+
}
|
|
3658
|
+
}
|
|
3659
|
+
|
|
3660
|
+
function stringRecord(value: unknown): Record<string, string> | undefined {
|
|
3661
|
+
const candidate = record(value);
|
|
3662
|
+
if (!candidate) return undefined;
|
|
3663
|
+
return Object.values(candidate).every((item) => typeof item === "string")
|
|
3664
|
+
? (candidate as Record<string, string>)
|
|
3665
|
+
: undefined;
|
|
3666
|
+
}
|
|
3667
|
+
|
|
3668
|
+
/**
|
|
3669
|
+
* Pods a producer has already tied to the investigated workload in this
|
|
3670
|
+
* projection: the streams diagnose or a workload-logs read resolved for it,
|
|
3671
|
+
* its crash candidates, and its Pod startup blockers. A pod name matched
|
|
3672
|
+
* against this set is producer-established membership, never inference from
|
|
3673
|
+
* a name prefix, which a sibling such as `api-worker` would also satisfy.
|
|
3674
|
+
*/
|
|
3675
|
+
function producerEstablishedTargetPods(
|
|
3676
|
+
builder: ProjectionBuilder,
|
|
3677
|
+
): Set<string> {
|
|
3678
|
+
const pods = new Set<string>();
|
|
3679
|
+
const namespace = builder.target.namespace;
|
|
3680
|
+
if (!namespace) return pods;
|
|
3681
|
+
for (const group of builder.groups) {
|
|
3682
|
+
for (const observation of group.observations) {
|
|
3683
|
+
if (observation.relevance === "broader") continue;
|
|
3684
|
+
const data = observation.data;
|
|
3685
|
+
if (data.type === "logs" && data.namespace === namespace) {
|
|
3686
|
+
pods.add(data.pod);
|
|
3687
|
+
} else if (data.type === "crash" && data.namespace === namespace) {
|
|
3688
|
+
for (const pod of data.crash.pods) pods.add(pod);
|
|
3689
|
+
} else if (
|
|
3690
|
+
data.type === "startup" &&
|
|
3691
|
+
data.subject?.kind === "Pod" &&
|
|
3692
|
+
data.subject.namespace === namespace
|
|
3693
|
+
) {
|
|
3694
|
+
pods.add(data.subject.name);
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
return pods;
|
|
3699
|
+
}
|
|
3700
|
+
|
|
3701
|
+
// kube-state-metrics label names for the owning workload.
|
|
3702
|
+
const WORKLOAD_LABEL_BY_KIND: Readonly<Record<string, string>> = {
|
|
3703
|
+
deployment: "deployment",
|
|
3704
|
+
statefulset: "statefulset",
|
|
3705
|
+
daemonset: "daemonset",
|
|
3706
|
+
rollout: "rollout",
|
|
3707
|
+
job: "job_name",
|
|
3708
|
+
cronjob: "cronjob",
|
|
3709
|
+
};
|
|
3710
|
+
|
|
3711
|
+
/**
|
|
3712
|
+
* Whether a Prometheus label set names the investigated resource: the target
|
|
3713
|
+
* namespace plus a label identifying it directly, or a `pod` label naming a
|
|
3714
|
+
* pod a producer already tied to it.
|
|
3715
|
+
*/
|
|
3716
|
+
function labelsNameTarget(
|
|
3717
|
+
target: InvestigationEvidenceTarget,
|
|
3718
|
+
labels: Record<string, string>,
|
|
3719
|
+
targetPods: ReadonlySet<string>,
|
|
3720
|
+
): boolean {
|
|
3721
|
+
if (!target.namespace || labels.namespace !== target.namespace) return false;
|
|
3722
|
+
const kind = target.kind.toLowerCase();
|
|
3723
|
+
if (kind === "pod") return labels.pod === target.name;
|
|
3724
|
+
const workloadLabel = WORKLOAD_LABEL_BY_KIND[kind];
|
|
3725
|
+
if (workloadLabel && labels[workloadLabel] === target.name) return true;
|
|
3726
|
+
if (
|
|
3727
|
+
labels.workload === target.name &&
|
|
3728
|
+
(labels.workload_type === undefined ||
|
|
3729
|
+
labels.workload_type.toLowerCase() === kind)
|
|
3730
|
+
) {
|
|
3731
|
+
return true;
|
|
3732
|
+
}
|
|
3733
|
+
return labels.pod !== undefined && targetPods.has(labels.pod);
|
|
3734
|
+
}
|
|
3735
|
+
|
|
3736
|
+
/**
|
|
3737
|
+
* The label set places the instance in another namespace outright. Only a
|
|
3738
|
+
* namespaced target can be excluded this way; for a cluster-scoped target
|
|
3739
|
+
* every namespace label is "different" and proves nothing.
|
|
3740
|
+
*/
|
|
3741
|
+
function labelsPlaceElsewhere(
|
|
3742
|
+
target: InvestigationEvidenceTarget,
|
|
3743
|
+
labels: Record<string, string>,
|
|
3744
|
+
): boolean {
|
|
3745
|
+
return (
|
|
3746
|
+
Boolean(target.namespace) &&
|
|
3747
|
+
nonEmptyString(labels.namespace) &&
|
|
3748
|
+
labels.namespace !== target.namespace
|
|
3749
|
+
);
|
|
3750
|
+
}
|
|
3751
|
+
|
|
3752
|
+
function alertRule(value: unknown):
|
|
3753
|
+
| {
|
|
3754
|
+
rule: InvestigationAlertRule;
|
|
3755
|
+
instances: Omit<InvestigationAlertInstance, "namesTarget">[];
|
|
3756
|
+
annotations: Record<string, string>;
|
|
3757
|
+
}
|
|
3758
|
+
| undefined {
|
|
3759
|
+
const candidate = record(value);
|
|
3760
|
+
if (
|
|
3761
|
+
!candidate ||
|
|
3762
|
+
!nonEmptyString(candidate.group) ||
|
|
3763
|
+
!nonEmptyString(candidate.name) ||
|
|
3764
|
+
!nonEmptyString(candidate.type) ||
|
|
3765
|
+
typeof candidate.query !== "string"
|
|
3766
|
+
) {
|
|
3767
|
+
return undefined;
|
|
3768
|
+
}
|
|
3769
|
+
for (const field of ["state", "health"] as const) {
|
|
3770
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
3771
|
+
return undefined;
|
|
3772
|
+
}
|
|
3773
|
+
const labels =
|
|
3774
|
+
candidate.labels === undefined ? {} : stringRecord(candidate.labels);
|
|
3775
|
+
const annotations =
|
|
3776
|
+
candidate.annotations === undefined
|
|
3777
|
+
? {}
|
|
3778
|
+
: stringRecord(candidate.annotations);
|
|
3779
|
+
if (!labels || !annotations) return undefined;
|
|
3780
|
+
const alertsRaw = candidate.alerts === undefined ? [] : candidate.alerts;
|
|
3781
|
+
if (!Array.isArray(alertsRaw)) return undefined;
|
|
3782
|
+
const instances: Omit<InvestigationAlertInstance, "namesTarget">[] = [];
|
|
3783
|
+
for (const raw of alertsRaw) {
|
|
3784
|
+
const instance = record(raw);
|
|
3785
|
+
const instanceLabels =
|
|
3786
|
+
instance?.labels === undefined ? {} : stringRecord(instance.labels);
|
|
3787
|
+
if (
|
|
3788
|
+
!instance ||
|
|
3789
|
+
!nonEmptyString(instance.state) ||
|
|
3790
|
+
!instanceLabels ||
|
|
3791
|
+
(instance.activeAt !== undefined &&
|
|
3792
|
+
typeof instance.activeAt !== "string") ||
|
|
3793
|
+
(instance.value !== undefined && typeof instance.value !== "string")
|
|
3794
|
+
) {
|
|
3795
|
+
return undefined;
|
|
3796
|
+
}
|
|
3797
|
+
instances.push({
|
|
3798
|
+
state: instance.state,
|
|
3799
|
+
activeAt: instance.activeAt as string | undefined,
|
|
3800
|
+
value: instance.value as string | undefined,
|
|
3801
|
+
labels: instanceLabels,
|
|
3802
|
+
});
|
|
3803
|
+
}
|
|
3804
|
+
return {
|
|
3805
|
+
rule: {
|
|
3806
|
+
group: candidate.group,
|
|
3807
|
+
name: candidate.name,
|
|
3808
|
+
type: candidate.type,
|
|
3809
|
+
state: candidate.state as string | undefined,
|
|
3810
|
+
health: candidate.health as string | undefined,
|
|
3811
|
+
query: candidate.query,
|
|
3812
|
+
labels,
|
|
3813
|
+
},
|
|
3814
|
+
instances,
|
|
3815
|
+
annotations,
|
|
3816
|
+
};
|
|
3817
|
+
}
|
|
3818
|
+
|
|
3819
|
+
function targetIdentity(target: InvestigationEvidenceTarget): string {
|
|
3820
|
+
return `${displayKind(target.kind)} ${target.namespace ? `${target.namespace}/` : ""}${target.name}`;
|
|
3821
|
+
}
|
|
3822
|
+
|
|
3823
|
+
function adaptPrometheusRules(
|
|
3824
|
+
builder: ProjectionBuilder,
|
|
3825
|
+
source: InvestigationEvidenceSource,
|
|
3826
|
+
payload: unknown,
|
|
3827
|
+
): void {
|
|
3828
|
+
const value = record(payload);
|
|
3829
|
+
if (
|
|
3830
|
+
!value ||
|
|
3831
|
+
!Array.isArray(value.rules) ||
|
|
3832
|
+
typeof value.count !== "number" ||
|
|
3833
|
+
value.count !== value.rules.length ||
|
|
3834
|
+
(value.truncated !== undefined && typeof value.truncated !== "boolean") ||
|
|
3835
|
+
(value.note !== undefined && typeof value.note !== "string")
|
|
3836
|
+
) {
|
|
3837
|
+
invalidPayload(builder, source);
|
|
3838
|
+
return;
|
|
3839
|
+
}
|
|
3840
|
+
const rules = value.rules
|
|
3841
|
+
.map(alertRule)
|
|
3842
|
+
.filter((item): item is NonNullable<typeof item> => Boolean(item));
|
|
3843
|
+
if (rules.length !== value.rules.length) {
|
|
3844
|
+
invalidPayload(builder, source);
|
|
3845
|
+
return;
|
|
3846
|
+
}
|
|
3847
|
+
const args = record(source.args ? parseJSON(source.args) : undefined);
|
|
3848
|
+
const stateFilter = nonEmptyString(args?.state)
|
|
3849
|
+
? args.state.toLowerCase()
|
|
3850
|
+
: undefined;
|
|
3851
|
+
const typeFilter = nonEmptyString(args?.type)
|
|
3852
|
+
? args.type.toLowerCase()
|
|
3853
|
+
: undefined;
|
|
3854
|
+
const truncated = value.truncated === true;
|
|
3855
|
+
if (truncated) {
|
|
3856
|
+
builder.limit(
|
|
3857
|
+
source,
|
|
3858
|
+
"Alert rules",
|
|
3859
|
+
`The rule list was capped, so additional matching rules may exist${nonEmptyString(value.note) ? `; ${value.note}` : ""}.`,
|
|
3860
|
+
"truncated",
|
|
3861
|
+
);
|
|
3862
|
+
}
|
|
3863
|
+
const identity = targetIdentity(builder.target);
|
|
3864
|
+
const targetPods = producerEstablishedTargetPods(builder);
|
|
3865
|
+
// Rule names are not unique within a group, and group names are unique
|
|
3866
|
+
// only per rule file, which the producer does not return. The expression
|
|
3867
|
+
// and static labels distinguish rules; identical rows within one response
|
|
3868
|
+
// cannot be told apart at all, so they stay scoped to this read rather
|
|
3869
|
+
// than inherit each other's history in whatever order they arrive.
|
|
3870
|
+
const ruleDefinition = (rule: InvestigationAlertRule) =>
|
|
3871
|
+
`alerts:${rule.group}:${rule.name}:${stableHash(
|
|
3872
|
+
JSON.stringify([rule.query, Object.entries(rule.labels).sort()]),
|
|
3873
|
+
)}`;
|
|
3874
|
+
const definitionCounts = new Map<string, number>();
|
|
3875
|
+
for (const { rule } of rules) {
|
|
3876
|
+
const definition = ruleDefinition(rule);
|
|
3877
|
+
definitionCounts.set(
|
|
3878
|
+
definition,
|
|
3879
|
+
(definitionCounts.get(definition) ?? 0) + 1,
|
|
3880
|
+
);
|
|
3881
|
+
}
|
|
3882
|
+
const seenRuleIdentities = new Map<string, number>();
|
|
3883
|
+
let namesTarget = false;
|
|
3884
|
+
let healthGap = false;
|
|
3885
|
+
let alertingCount = 0;
|
|
3886
|
+
let everyInstanceElsewhere = true;
|
|
3887
|
+
for (const { rule, instances: rawInstances, annotations } of rules) {
|
|
3888
|
+
// Recording rules carry no state and never describe a resource.
|
|
3889
|
+
if (rule.type.toLowerCase() !== "alerting") continue;
|
|
3890
|
+
alertingCount += 1;
|
|
3891
|
+
const instances = rawInstances.map((instance) => ({
|
|
3892
|
+
...instance,
|
|
3893
|
+
namesTarget: labelsNameTarget(
|
|
3894
|
+
builder.target,
|
|
3895
|
+
instance.labels,
|
|
3896
|
+
targetPods,
|
|
3897
|
+
),
|
|
3898
|
+
}));
|
|
3899
|
+
if (
|
|
3900
|
+
instances.length === 0 ||
|
|
3901
|
+
instances.some(
|
|
3902
|
+
(instance) => !labelsPlaceElsewhere(builder.target, instance.labels),
|
|
3903
|
+
)
|
|
3904
|
+
) {
|
|
3905
|
+
everyInstanceElsewhere = false;
|
|
3906
|
+
}
|
|
3907
|
+
const definition = ruleDefinition(rule);
|
|
3908
|
+
const ambiguous = (definitionCounts.get(definition) ?? 0) > 1;
|
|
3909
|
+
const ordinal = seenRuleIdentities.get(definition) ?? 0;
|
|
3910
|
+
seenRuleIdentities.set(definition, ordinal + 1);
|
|
3911
|
+
const ruleIdentity = ambiguous
|
|
3912
|
+
? `${definition}:${source.id}:${ordinal}`
|
|
3913
|
+
: definition;
|
|
3914
|
+
const previousRelevance = ambiguous
|
|
3915
|
+
? undefined
|
|
3916
|
+
: builder.latestRelevance("alerts", ruleIdentity);
|
|
3917
|
+
const relevance: InvestigationEvidenceRelevance = instances.some(
|
|
3918
|
+
(instance) => instance.namesTarget,
|
|
3919
|
+
)
|
|
3920
|
+
? "target"
|
|
3921
|
+
: labelsNameTarget(builder.target, rule.labels, targetPods)
|
|
3922
|
+
? "producer-related"
|
|
3923
|
+
: // The same rule read again with no instances left has resolved for
|
|
3924
|
+
// the target it named before; keep that provenance so the
|
|
3925
|
+
// resolution stays visible instead of the stale firing card.
|
|
3926
|
+
instances.length === 0 &&
|
|
3927
|
+
previousRelevance !== undefined &&
|
|
3928
|
+
previousRelevance !== "broader"
|
|
3929
|
+
? previousRelevance
|
|
3930
|
+
: "broader";
|
|
3931
|
+
if (relevance !== "broader") namesTarget = true;
|
|
3932
|
+
const state = (rule.state ?? "").toLowerCase();
|
|
3933
|
+
const active = state === "firing" || state === "pending";
|
|
3934
|
+
const severity = (rule.labels.severity ?? "").toLowerCase();
|
|
3935
|
+
const targetInstances = instances.filter(
|
|
3936
|
+
(instance) => instance.namesTarget,
|
|
3937
|
+
).length;
|
|
3938
|
+
builder.observe(ruleIdentity, "alerts", source, {
|
|
3939
|
+
tier: evidenceTierForRelevance(
|
|
3940
|
+
active ? "supporting" : "context",
|
|
3941
|
+
relevance,
|
|
3942
|
+
),
|
|
3943
|
+
relevance,
|
|
3944
|
+
tone:
|
|
3945
|
+
state === "firing"
|
|
3946
|
+
? severity === "critical"
|
|
3947
|
+
? "error"
|
|
3948
|
+
: "alert"
|
|
3949
|
+
: state === "pending"
|
|
3950
|
+
? "warning"
|
|
3951
|
+
: "neutral",
|
|
3952
|
+
title: `${rule.name}${state ? ` ${state}` : ""}`,
|
|
3953
|
+
summary:
|
|
3954
|
+
instances.length === 0
|
|
3955
|
+
? `No active instances · ${rule.group}`
|
|
3956
|
+
: `${instances.length} active instance${instances.length === 1 ? "" : "s"}${
|
|
3957
|
+
targetInstances > 0
|
|
3958
|
+
? `, ${targetInstances} naming ${identity}`
|
|
3959
|
+
: ""
|
|
3960
|
+
} · ${rule.group}`,
|
|
3961
|
+
data: { type: "alerts", rule, instances, annotations },
|
|
3962
|
+
});
|
|
3963
|
+
const health = (rule.health ?? "").toLowerCase();
|
|
3964
|
+
if (health === "err") {
|
|
3965
|
+
healthGap = true;
|
|
3966
|
+
builder.limit(
|
|
3967
|
+
source,
|
|
3968
|
+
`Alert rule ${rule.name}`,
|
|
3969
|
+
"Prometheus reported an evaluation error for this rule, so its state may be stale or missing.",
|
|
3970
|
+
"error",
|
|
3971
|
+
);
|
|
3972
|
+
} else if (health !== "ok") {
|
|
3973
|
+
// Anything but a reported "ok" evaluation, including an unrecognized
|
|
3974
|
+
// value, is an unevaluated rule for the purpose of a negative.
|
|
3975
|
+
healthGap = true;
|
|
3976
|
+
builder.limit(
|
|
3977
|
+
source,
|
|
3978
|
+
`Alert rule ${rule.name}`,
|
|
3979
|
+
"Prometheus has not reported a successful evaluation for this rule, so its state is unknown.",
|
|
3980
|
+
"unknown",
|
|
3981
|
+
);
|
|
3982
|
+
}
|
|
3983
|
+
}
|
|
3984
|
+
// A negative is only as strong as the query: an active-state filter over
|
|
3985
|
+
// every alerting rule (a name or group filter never looked at the rest), a
|
|
3986
|
+
// complete list, every rule evaluated, and every returned instance placed
|
|
3987
|
+
// in another namespace outright. An instance this projection merely fails
|
|
3988
|
+
// to recognize is not evidence of absence.
|
|
3989
|
+
if (
|
|
3990
|
+
source.confirmedSuccess &&
|
|
3991
|
+
!truncated &&
|
|
3992
|
+
!namesTarget &&
|
|
3993
|
+
!healthGap &&
|
|
3994
|
+
typeFilter !== "record" &&
|
|
3995
|
+
!nonEmptyString(args?.name) &&
|
|
3996
|
+
!nonEmptyString(args?.group) &&
|
|
3997
|
+
(stateFilter === "firing" || stateFilter === "pending") &&
|
|
3998
|
+
(alertingCount === 0 || everyInstanceElsewhere)
|
|
3999
|
+
) {
|
|
4000
|
+
const filter = `state=${stateFilter}`;
|
|
4001
|
+
builder.observe(`alerts:receipt:${filter}`, "receipt", source, {
|
|
4002
|
+
tier: "checked",
|
|
4003
|
+
relevance: "target",
|
|
4004
|
+
tone: "neutral",
|
|
4005
|
+
title: `No ${stateFilter} alert rules name this ${displayKind(builder.target.kind)}`,
|
|
4006
|
+
summary: identity,
|
|
4007
|
+
data: {
|
|
4008
|
+
type: "receipt",
|
|
4009
|
+
checked: "alerts",
|
|
4010
|
+
scope: identity,
|
|
4011
|
+
message:
|
|
4012
|
+
alertingCount === 0
|
|
4013
|
+
? `0 alerting rules returned (filter: ${filter}).`
|
|
4014
|
+
: `${alertingCount} alerting rule${alertingCount === 1 ? "" : "s"} returned (filter: ${filter}), every instance in another namespace; none names ${identity}.`,
|
|
4015
|
+
},
|
|
4016
|
+
});
|
|
4017
|
+
}
|
|
4018
|
+
}
|
|
4019
|
+
|
|
4020
|
+
function helmOwnedResource(
|
|
4021
|
+
value: unknown,
|
|
4022
|
+
): InvestigationHelmOwnedResource | undefined {
|
|
4023
|
+
const candidate = record(value);
|
|
4024
|
+
if (
|
|
4025
|
+
!candidate ||
|
|
4026
|
+
!nonEmptyString(candidate.kind) ||
|
|
4027
|
+
!nonEmptyString(candidate.name) ||
|
|
4028
|
+
typeof candidate.namespace !== "string"
|
|
4029
|
+
) {
|
|
4030
|
+
return undefined;
|
|
4031
|
+
}
|
|
4032
|
+
for (const field of [
|
|
4033
|
+
"apiVersion",
|
|
4034
|
+
"status",
|
|
4035
|
+
"ready",
|
|
4036
|
+
"message",
|
|
4037
|
+
"summary",
|
|
4038
|
+
"issue",
|
|
4039
|
+
] as const) {
|
|
4040
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
4041
|
+
return undefined;
|
|
4042
|
+
}
|
|
4043
|
+
return candidate as unknown as InvestigationHelmOwnedResource;
|
|
4044
|
+
}
|
|
4045
|
+
|
|
4046
|
+
function helmOperation(value: unknown): InvestigationHelmOperation | undefined {
|
|
4047
|
+
const candidate = record(value);
|
|
4048
|
+
if (
|
|
4049
|
+
!candidate ||
|
|
4050
|
+
!nonEmptyString(candidate.kind) ||
|
|
4051
|
+
!nonEmptyString(candidate.status) ||
|
|
4052
|
+
typeof candidate.message !== "string"
|
|
4053
|
+
) {
|
|
4054
|
+
return undefined;
|
|
4055
|
+
}
|
|
4056
|
+
for (const field of [
|
|
4057
|
+
"revision",
|
|
4058
|
+
"failedRevision",
|
|
4059
|
+
"rollbackRevision",
|
|
4060
|
+
] as const) {
|
|
4061
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "number")
|
|
4062
|
+
return undefined;
|
|
4063
|
+
}
|
|
4064
|
+
if (candidate.updated !== undefined && typeof candidate.updated !== "string")
|
|
4065
|
+
return undefined;
|
|
4066
|
+
return candidate as unknown as InvestigationHelmOperation;
|
|
4067
|
+
}
|
|
4068
|
+
|
|
4069
|
+
function adaptHelmRelease(
|
|
4070
|
+
builder: ProjectionBuilder,
|
|
4071
|
+
source: InvestigationEvidenceSource,
|
|
4072
|
+
payload: unknown,
|
|
4073
|
+
): void {
|
|
4074
|
+
const value = record(payload);
|
|
4075
|
+
if (
|
|
4076
|
+
!value ||
|
|
4077
|
+
!nonEmptyString(value.name) ||
|
|
4078
|
+
!nonEmptyString(value.namespace) ||
|
|
4079
|
+
typeof value.chart !== "string" ||
|
|
4080
|
+
typeof value.chartVersion !== "string" ||
|
|
4081
|
+
!nonEmptyString(value.status) ||
|
|
4082
|
+
typeof value.revision !== "number" ||
|
|
4083
|
+
typeof value.updated !== "string"
|
|
4084
|
+
) {
|
|
4085
|
+
invalidPayload(builder, source);
|
|
4086
|
+
return;
|
|
4087
|
+
}
|
|
4088
|
+
for (const field of [
|
|
4089
|
+
"appVersion",
|
|
4090
|
+
"description",
|
|
4091
|
+
"storageNamespace",
|
|
4092
|
+
"resourceHealth",
|
|
4093
|
+
"healthIssue",
|
|
4094
|
+
"healthSummary",
|
|
4095
|
+
"managedByFluxHelmRelease",
|
|
4096
|
+
] as const) {
|
|
4097
|
+
if (value[field] !== undefined && typeof value[field] !== "string") {
|
|
4098
|
+
invalidPayload(builder, source);
|
|
4099
|
+
return;
|
|
4100
|
+
}
|
|
4101
|
+
}
|
|
4102
|
+
const resourcesRaw =
|
|
4103
|
+
value.resources === undefined || value.resources === null
|
|
4104
|
+
? []
|
|
4105
|
+
: value.resources;
|
|
4106
|
+
if (!Array.isArray(resourcesRaw)) {
|
|
4107
|
+
invalidPayload(builder, source, "Helm resources");
|
|
4108
|
+
return;
|
|
4109
|
+
}
|
|
4110
|
+
const resources = resourcesRaw
|
|
4111
|
+
.map(helmOwnedResource)
|
|
4112
|
+
.filter((item): item is InvestigationHelmOwnedResource => Boolean(item));
|
|
4113
|
+
if (resources.length !== resourcesRaw.length) {
|
|
4114
|
+
invalidPayload(builder, source, "Helm resources");
|
|
4115
|
+
return;
|
|
4116
|
+
}
|
|
4117
|
+
const lastOperation =
|
|
4118
|
+
value.lastOperation === undefined
|
|
4119
|
+
? undefined
|
|
4120
|
+
: helmOperation(value.lastOperation);
|
|
4121
|
+
if (value.lastOperation !== undefined && !lastOperation) {
|
|
4122
|
+
invalidPayload(builder, source, "Helm operation");
|
|
4123
|
+
return;
|
|
4124
|
+
}
|
|
4125
|
+
const release: InvestigationHelmRelease = {
|
|
4126
|
+
name: value.name,
|
|
4127
|
+
namespace: value.namespace,
|
|
4128
|
+
chart: value.chart,
|
|
4129
|
+
chartVersion: value.chartVersion,
|
|
4130
|
+
appVersion: value.appVersion as string | undefined,
|
|
4131
|
+
status: value.status,
|
|
4132
|
+
revision: value.revision,
|
|
4133
|
+
updated: value.updated,
|
|
4134
|
+
description: value.description as string | undefined,
|
|
4135
|
+
...(nonEmptyString(value.storageNamespace)
|
|
4136
|
+
? { storageNamespace: value.storageNamespace }
|
|
4137
|
+
: {}),
|
|
4138
|
+
resourceHealth: value.resourceHealth as string | undefined,
|
|
4139
|
+
healthIssue: value.healthIssue as string | undefined,
|
|
4140
|
+
healthSummary: value.healthSummary as string | undefined,
|
|
4141
|
+
managedByFluxHelmRelease: value.managedByFluxHelmRelease as
|
|
4142
|
+
string | undefined,
|
|
4143
|
+
lastOperation,
|
|
4144
|
+
resources,
|
|
4145
|
+
};
|
|
4146
|
+
// The release manages the target when the target is among the resources
|
|
4147
|
+
// Helm rendered for it; a release is never the investigated object itself.
|
|
4148
|
+
// Without an API version the owned row cannot tell colliding kinds apart
|
|
4149
|
+
// (CNPG vs CAPI Cluster), so it establishes nothing.
|
|
4150
|
+
const relevance: InvestigationEvidenceRelevance = resources.some(
|
|
4151
|
+
(owned) =>
|
|
4152
|
+
owned.apiVersion !== undefined &&
|
|
4153
|
+
resourceMatchesTarget(builder.target, {
|
|
4154
|
+
kind: owned.kind,
|
|
4155
|
+
group: apiVersionToGroup(owned.apiVersion),
|
|
4156
|
+
namespace: owned.namespace || undefined,
|
|
4157
|
+
name: owned.name,
|
|
4158
|
+
}),
|
|
4159
|
+
)
|
|
4160
|
+
? "producer-related"
|
|
4161
|
+
: "broader";
|
|
4162
|
+
const status = release.status.toLowerCase();
|
|
4163
|
+
const operationFailed =
|
|
4164
|
+
lastOperation?.status === "failed" ||
|
|
4165
|
+
lastOperation?.status === "stuck_pending";
|
|
4166
|
+
const adverse =
|
|
4167
|
+
status !== "deployed" || Boolean(release.healthIssue) || operationFailed;
|
|
4168
|
+
const chartLabel = `${release.chart}${release.chartVersion ? ` ${release.chartVersion}` : ""}`;
|
|
4169
|
+
// Helm keys a release by where its metadata is stored; two releases with
|
|
4170
|
+
// one name and namespace can live in different storage namespaces.
|
|
4171
|
+
const storage = release.storageNamespace ?? release.namespace;
|
|
4172
|
+
builder.observe(
|
|
4173
|
+
`helm:${storage}:${release.namespace}:${release.name}`,
|
|
4174
|
+
"helm",
|
|
4175
|
+
source,
|
|
4176
|
+
{
|
|
4177
|
+
tier: evidenceTierForRelevance(
|
|
4178
|
+
adverse ? "supporting" : "context",
|
|
4179
|
+
relevance,
|
|
4180
|
+
),
|
|
4181
|
+
relevance,
|
|
4182
|
+
tone:
|
|
4183
|
+
status.includes("failed") || lastOperation?.status === "failed"
|
|
4184
|
+
? "error"
|
|
4185
|
+
: adverse
|
|
4186
|
+
? "warning"
|
|
4187
|
+
: "info",
|
|
4188
|
+
title: `Helm release ${release.namespace}/${release.name}`,
|
|
4189
|
+
summary: release.healthIssue
|
|
4190
|
+
? `${release.status} · ${release.healthIssue}`
|
|
4191
|
+
: `${chartLabel} · ${release.status} · revision ${release.revision}`,
|
|
4192
|
+
data: { type: "helm", release },
|
|
4193
|
+
},
|
|
4194
|
+
);
|
|
4195
|
+
// Every comparison the producer attempted and could not finish, not just the
|
|
4196
|
+
// values read. Four of these are Cloud-role denials, so dropping any of them
|
|
4197
|
+
// lets a reader see a release card whose comparison was withheld and take it
|
|
4198
|
+
// for complete. The labels follow the producer's own fields: `diff` is the
|
|
4199
|
+
// manifest, `valuesDiff` the values — naming one for the other sends a reader
|
|
4200
|
+
// to the wrong thing.
|
|
4201
|
+
for (const [field, label] of [
|
|
4202
|
+
["valuesError", "Helm values"],
|
|
4203
|
+
["valuesDiffError", "Helm values diff"],
|
|
4204
|
+
["diffError", "Helm manifest diff"],
|
|
4205
|
+
["notesDiffError", "Helm notes diff"],
|
|
4206
|
+
["resourceDiffError", "Helm resource diff"],
|
|
4207
|
+
] as const) {
|
|
4208
|
+
const message = value[field];
|
|
4209
|
+
if (nonEmptyString(message)) {
|
|
4210
|
+
builder.limit(source, label, message, "error");
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
}
|
|
4214
|
+
|
|
4215
|
+
function permissionSubject(
|
|
4216
|
+
value: unknown,
|
|
4217
|
+
): InvestigationPermissionSubject | undefined {
|
|
4218
|
+
const candidate = record(value);
|
|
4219
|
+
if (
|
|
4220
|
+
!candidate ||
|
|
4221
|
+
!nonEmptyString(candidate.kind) ||
|
|
4222
|
+
!nonEmptyString(candidate.name) ||
|
|
4223
|
+
(candidate.namespace !== undefined &&
|
|
4224
|
+
typeof candidate.namespace !== "string")
|
|
4225
|
+
) {
|
|
4226
|
+
return undefined;
|
|
4227
|
+
}
|
|
4228
|
+
return {
|
|
4229
|
+
kind: candidate.kind,
|
|
4230
|
+
name: candidate.name,
|
|
4231
|
+
...(nonEmptyString(candidate.namespace)
|
|
4232
|
+
? { namespace: candidate.namespace }
|
|
4233
|
+
: {}),
|
|
4234
|
+
};
|
|
4235
|
+
}
|
|
4236
|
+
|
|
4237
|
+
function accessCheck(value: unknown): InvestigationAccessCheck | undefined {
|
|
4238
|
+
const candidate = record(value);
|
|
4239
|
+
if (
|
|
4240
|
+
!candidate ||
|
|
4241
|
+
!nonEmptyString(candidate.verb) ||
|
|
4242
|
+
!nonEmptyString(candidate.resource) ||
|
|
4243
|
+
typeof candidate.namespace !== "string" ||
|
|
4244
|
+
typeof candidate.allowed !== "boolean" ||
|
|
4245
|
+
(candidate.denied !== undefined && typeof candidate.denied !== "boolean")
|
|
4246
|
+
) {
|
|
4247
|
+
return undefined;
|
|
4248
|
+
}
|
|
4249
|
+
for (const field of [
|
|
4250
|
+
"group",
|
|
4251
|
+
"subresource",
|
|
4252
|
+
"resourceName",
|
|
4253
|
+
"reason",
|
|
4254
|
+
"evaluationError",
|
|
4255
|
+
] as const) {
|
|
4256
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
4257
|
+
return undefined;
|
|
4258
|
+
}
|
|
4259
|
+
return {
|
|
4260
|
+
...(candidate as unknown as InvestigationAccessCheck),
|
|
4261
|
+
denied: candidate.denied === true,
|
|
4262
|
+
};
|
|
4263
|
+
}
|
|
4264
|
+
|
|
4265
|
+
function permissionBinding(
|
|
4266
|
+
value: unknown,
|
|
4267
|
+
): InvestigationPermissionBinding | undefined {
|
|
4268
|
+
const candidate = record(value);
|
|
4269
|
+
if (
|
|
4270
|
+
!candidate ||
|
|
4271
|
+
!nonEmptyString(candidate.bindingKind) ||
|
|
4272
|
+
!nonEmptyString(candidate.bindingName) ||
|
|
4273
|
+
!nonEmptyString(candidate.roleKind) ||
|
|
4274
|
+
!nonEmptyString(candidate.roleName) ||
|
|
4275
|
+
!nonNegativeInteger(candidate.rulesCount)
|
|
4276
|
+
) {
|
|
4277
|
+
return undefined;
|
|
4278
|
+
}
|
|
4279
|
+
for (const field of [
|
|
4280
|
+
"bindingNamespace",
|
|
4281
|
+
"roleNamespace",
|
|
4282
|
+
"inheritedFromGroup",
|
|
4283
|
+
] as const) {
|
|
4284
|
+
if (candidate[field] !== undefined && typeof candidate[field] !== "string")
|
|
4285
|
+
return undefined;
|
|
4286
|
+
}
|
|
4287
|
+
return candidate as unknown as InvestigationPermissionBinding;
|
|
4288
|
+
}
|
|
4289
|
+
|
|
4290
|
+
// Kinds whose spec carries a PodSpec, by API group. Anything else with a
|
|
4291
|
+
// `spec.template.spec` (an ApplicationSet, for one) templates something that
|
|
4292
|
+
// is not a Pod and runs as no account at all.
|
|
4293
|
+
const POD_TEMPLATE_KINDS: Readonly<Record<string, readonly string[]>> = {
|
|
4294
|
+
"": ["Pod"],
|
|
4295
|
+
apps: ["Deployment", "StatefulSet", "DaemonSet", "ReplicaSet"],
|
|
4296
|
+
batch: ["Job", "CronJob"],
|
|
4297
|
+
"argoproj.io": ["Rollout"],
|
|
4298
|
+
};
|
|
4299
|
+
|
|
4300
|
+
function podSpecServiceAccount(
|
|
4301
|
+
resource: InvestigationKubernetesResource,
|
|
4302
|
+
): string | undefined {
|
|
4303
|
+
const group = apiVersionToGroup(resource.apiVersion);
|
|
4304
|
+
if (!POD_TEMPLATE_KINDS[group]?.includes(resource.kind)) return undefined;
|
|
4305
|
+
const spec = record(resource.spec);
|
|
4306
|
+
const podSpec =
|
|
4307
|
+
resource.kind === "Pod"
|
|
4308
|
+
? spec
|
|
4309
|
+
: resource.kind === "CronJob"
|
|
4310
|
+
? record(
|
|
4311
|
+
record(record(record(spec?.jobTemplate)?.spec)?.template)?.spec,
|
|
4312
|
+
)
|
|
4313
|
+
: record(record(spec?.template)?.spec);
|
|
4314
|
+
if (!podSpec) return undefined;
|
|
4315
|
+
// An unset serviceAccountName runs as the namespace's `default` account.
|
|
4316
|
+
return nonEmptyString(podSpec.serviceAccountName)
|
|
4317
|
+
? podSpec.serviceAccountName
|
|
4318
|
+
: "default";
|
|
4319
|
+
}
|
|
4320
|
+
|
|
4321
|
+
/**
|
|
4322
|
+
* The ServiceAccount the target runs as, read from a resource observation of
|
|
4323
|
+
* the target in the same turn. Only the captured resource can state this; the
|
|
4324
|
+
* permission subject is never assumed related by name alone.
|
|
4325
|
+
*/
|
|
4326
|
+
function targetServiceAccountName(
|
|
4327
|
+
builder: ProjectionBuilder,
|
|
4328
|
+
source: InvestigationEvidenceSource,
|
|
4329
|
+
): string | undefined {
|
|
4330
|
+
let newest: InvestigationEvidenceObservation | undefined;
|
|
4331
|
+
for (const group of builder.groups) {
|
|
4332
|
+
if (group.kind !== "resource") continue;
|
|
4333
|
+
for (const observation of group.observations) {
|
|
4334
|
+
if (
|
|
4335
|
+
observation.source.turnIndex !== source.turnIndex ||
|
|
4336
|
+
observation.relevance !== "target" ||
|
|
4337
|
+
observation.data.type !== "resource" ||
|
|
4338
|
+
(newest && observation.source.order < newest.source.order)
|
|
4339
|
+
) {
|
|
4340
|
+
continue;
|
|
4341
|
+
}
|
|
4342
|
+
newest = observation;
|
|
4343
|
+
}
|
|
4344
|
+
}
|
|
4345
|
+
if (!newest || newest.data.type !== "resource") return undefined;
|
|
4346
|
+
return (
|
|
4347
|
+
newest.data.resourceContext?.uses?.serviceAccount?.name ??
|
|
4348
|
+
podSpecServiceAccount(newest.data.resource)
|
|
4349
|
+
);
|
|
4350
|
+
}
|
|
4351
|
+
|
|
4352
|
+
function adaptSubjectPermissions(
|
|
4353
|
+
builder: ProjectionBuilder,
|
|
4354
|
+
source: InvestigationEvidenceSource,
|
|
4355
|
+
payload: unknown,
|
|
4356
|
+
): void {
|
|
4357
|
+
const value = record(payload);
|
|
4358
|
+
const subject = value ? permissionSubject(value.subject) : undefined;
|
|
4359
|
+
if (!value || !subject) {
|
|
4360
|
+
invalidPayload(builder, source);
|
|
4361
|
+
return;
|
|
4362
|
+
}
|
|
4363
|
+
// Only a captured resource can tie a principal to the target. Any other
|
|
4364
|
+
// subject the agent chose to check is context, however adverse the answer.
|
|
4365
|
+
const relevance: InvestigationEvidenceRelevance =
|
|
4366
|
+
subject.kind === "ServiceAccount" &&
|
|
4367
|
+
subject.namespace !== undefined &&
|
|
4368
|
+
subject.namespace === builder.target.namespace &&
|
|
4369
|
+
targetServiceAccountName(builder, source) === subject.name
|
|
4370
|
+
? "target"
|
|
4371
|
+
: "broader";
|
|
4372
|
+
const subjectLabel = `${displayKind(subject.kind)} ${subject.namespace ? `${subject.namespace}/` : ""}${subject.name}`;
|
|
4373
|
+
const subjectKey = `${subject.kind}:${subject.namespace ?? ""}:${subject.name}`;
|
|
4374
|
+
|
|
4375
|
+
if (value.accessCheck !== undefined) {
|
|
4376
|
+
const check = accessCheck(value.accessCheck);
|
|
4377
|
+
if (!check) {
|
|
4378
|
+
invalidPayload(builder, source, "Access check");
|
|
4379
|
+
return;
|
|
4380
|
+
}
|
|
4381
|
+
const resourceLabel = `${check.resource}${check.subresource ? `/${check.subresource}` : ""}${check.group ? `.${check.group}` : ""}`;
|
|
4382
|
+
// An authorizer that could not decide has established neither answer. The
|
|
4383
|
+
// evaluation error already reaches the coverage strip below, but a card
|
|
4384
|
+
// reading "cannot verb resource" is the part an operator acts on, and a
|
|
4385
|
+
// webhook returning partial data is not a denial.
|
|
4386
|
+
// Only when the authorizer decided nothing. Kubernetes returns this error
|
|
4387
|
+
// alongside a real verdict too — one webhook failing while another allows —
|
|
4388
|
+
// and rewriting a decided allow or deny as "could not check" loses the
|
|
4389
|
+
// answer and puts an allow in the warning list.
|
|
4390
|
+
const unresolved =
|
|
4391
|
+
nonEmptyString(check.evaluationError) && !check.allowed && !check.denied;
|
|
4392
|
+
const denied = !check.allowed && !unresolved;
|
|
4393
|
+
const verdict = unresolved
|
|
4394
|
+
? `Could not be evaluated: ${check.evaluationError}`
|
|
4395
|
+
: check.reason ||
|
|
4396
|
+
(check.allowed
|
|
4397
|
+
? "Allowed by RBAC"
|
|
4398
|
+
: check.denied
|
|
4399
|
+
? "Explicitly denied"
|
|
4400
|
+
: "No RBAC rule allows it");
|
|
4401
|
+
builder.observe(
|
|
4402
|
+
`permissions:check:${subjectKey}:${check.verb}:${check.group ?? ""}:${check.resource}:${check.subresource ?? ""}:${check.namespace}:${check.resourceName ?? ""}`,
|
|
4403
|
+
"permissions",
|
|
4404
|
+
source,
|
|
4405
|
+
{
|
|
4406
|
+
tier: evidenceTierForRelevance(
|
|
4407
|
+
denied || unresolved ? "supporting" : "context",
|
|
4408
|
+
relevance,
|
|
4409
|
+
),
|
|
4410
|
+
relevance,
|
|
4411
|
+
tone: denied || unresolved ? "warning" : "info",
|
|
4412
|
+
title: unresolved
|
|
4413
|
+
? `Could not check whether ${subjectLabel} can ${check.verb} ${resourceLabel}`
|
|
4414
|
+
: `${subjectLabel} ${denied ? "cannot" : "can"} ${check.verb} ${resourceLabel}`,
|
|
4415
|
+
summary: `${verdict} · ${check.namespace ? `namespace ${check.namespace}` : "cluster-wide"}${check.resourceName ? ` · ${check.resourceName}` : ""}`,
|
|
4416
|
+
data: { type: "permissions", subject, accessCheck: check },
|
|
4417
|
+
},
|
|
4418
|
+
);
|
|
4419
|
+
if (nonEmptyString(check.evaluationError)) {
|
|
4420
|
+
builder.limit(source, "Permissions", check.evaluationError, "error");
|
|
4421
|
+
}
|
|
4422
|
+
return;
|
|
4423
|
+
}
|
|
4424
|
+
|
|
4425
|
+
const bindingsRaw = value.bindings;
|
|
4426
|
+
if (!Array.isArray(bindingsRaw)) {
|
|
4427
|
+
invalidPayload(builder, source);
|
|
4428
|
+
return;
|
|
4429
|
+
}
|
|
4430
|
+
const bindings = bindingsRaw
|
|
4431
|
+
.map(permissionBinding)
|
|
4432
|
+
.filter((item): item is InvestigationPermissionBinding => Boolean(item));
|
|
4433
|
+
if (
|
|
4434
|
+
bindings.length !== bindingsRaw.length ||
|
|
4435
|
+
!Array.isArray(value.flatRules) ||
|
|
4436
|
+
!value.flatRules.every((rule) => stringArray(record(rule)?.verbs)) ||
|
|
4437
|
+
(value.truncated !== undefined && typeof value.truncated !== "boolean") ||
|
|
4438
|
+
(value.podsTotal !== undefined && !nonNegativeInteger(value.podsTotal)) ||
|
|
4439
|
+
(value.usedByPods !== undefined && !stringArray(value.usedByPods))
|
|
4440
|
+
) {
|
|
4441
|
+
invalidPayload(builder, source);
|
|
4442
|
+
return;
|
|
4443
|
+
}
|
|
4444
|
+
const truncated = value.truncated === true;
|
|
4445
|
+
const flatRulesCount = value.flatRules.length;
|
|
4446
|
+
const usedByPods = stringArray(value.usedByPods) ?? [];
|
|
4447
|
+
builder.observe(`permissions:subject:${subjectKey}`, "permissions", source, {
|
|
4448
|
+
tier: evidenceTierForRelevance("context", relevance),
|
|
4449
|
+
relevance,
|
|
4450
|
+
tone: "info",
|
|
4451
|
+
title: `Permissions of ${subjectLabel}`,
|
|
4452
|
+
summary: `${bindings.length} binding${bindings.length === 1 ? "" : "s"} · ${flatRulesCount}${truncated ? "+" : ""} effective rule${flatRulesCount === 1 && !truncated ? "" : "s"}`,
|
|
4453
|
+
data: {
|
|
4454
|
+
type: "permissions",
|
|
4455
|
+
subject,
|
|
4456
|
+
bindings,
|
|
4457
|
+
flatRulesCount,
|
|
4458
|
+
truncated,
|
|
4459
|
+
usedByPods,
|
|
4460
|
+
podsTotal: value.podsTotal as number | undefined,
|
|
4461
|
+
},
|
|
4462
|
+
});
|
|
4463
|
+
if (truncated) {
|
|
4464
|
+
builder.limit(
|
|
4465
|
+
source,
|
|
4466
|
+
"Permissions",
|
|
4467
|
+
nonEmptyString(value.narrowHint)
|
|
4468
|
+
? value.narrowHint
|
|
4469
|
+
: "The effective rule list was truncated; it is not the subject's complete permission set.",
|
|
4470
|
+
"truncated",
|
|
4471
|
+
);
|
|
4472
|
+
}
|
|
4473
|
+
if (nonEmptyString(value.subjectWarning)) {
|
|
4474
|
+
builder.limit(source, "Permissions", value.subjectWarning, "unknown");
|
|
4475
|
+
}
|
|
4476
|
+
if (
|
|
4477
|
+
typeof value.podsTotal === "number" &&
|
|
4478
|
+
value.podsTotal > usedByPods.length
|
|
4479
|
+
) {
|
|
4480
|
+
builder.limit(
|
|
4481
|
+
source,
|
|
4482
|
+
"Permissions",
|
|
4483
|
+
`Only ${usedByPods.length} of ${value.podsTotal} pods running as this subject were listed.`,
|
|
4484
|
+
"truncated",
|
|
4485
|
+
);
|
|
4486
|
+
}
|
|
4487
|
+
}
|
|
4488
|
+
|
|
3202
4489
|
const ADAPTERS: Record<
|
|
3203
4490
|
string,
|
|
3204
4491
|
(
|
|
@@ -3216,6 +4503,10 @@ const ADAPTERS: Record<
|
|
|
3216
4503
|
get_changes: adaptChanges,
|
|
3217
4504
|
get_neighborhood: adaptNeighborhood,
|
|
3218
4505
|
get_topology: adaptTopology,
|
|
4506
|
+
get_workload_logs: adaptWorkloadLogs,
|
|
4507
|
+
get_prometheus_rules: adaptPrometheusRules,
|
|
4508
|
+
get_helm_release: adaptHelmRelease,
|
|
4509
|
+
get_subject_permissions: adaptSubjectPermissions,
|
|
3219
4510
|
};
|
|
3220
4511
|
|
|
3221
4512
|
export function projectInvestigationEvidence(
|
|
@@ -3430,6 +4721,7 @@ export function projectInvestigationEvidence(
|
|
|
3430
4721
|
}
|
|
3431
4722
|
break;
|
|
3432
4723
|
case "inventory":
|
|
4724
|
+
case "alerts":
|
|
3433
4725
|
break;
|
|
3434
4726
|
}
|
|
3435
4727
|
continue;
|