@skyhook-io/radar-app 1.13.3 → 1.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/App.tsx +68 -23
- package/src/RadarApp.tsx +17 -1
- package/src/api/client.ts +28 -10
- package/src/api/diagnose.ts +61 -15
- package/src/components/ConnectionErrorView.test.tsx +30 -0
- package/src/components/ConnectionErrorView.tsx +31 -19
- package/src/components/diagnose/AgentCase.tsx +131 -0
- package/src/components/diagnose/DiagnoseContext.test.ts +95 -0
- package/src/components/diagnose/DiagnoseContext.tsx +402 -77
- package/src/components/diagnose/DiagnoseSurface.test.tsx +53 -54
- package/src/components/diagnose/DiagnoseSurface.tsx +198 -286
- package/src/components/diagnose/Home.test.tsx +23 -1
- package/src/components/diagnose/Home.tsx +49 -3
- package/src/components/diagnose/InvestigationEvidencePane.test.tsx +1446 -5
- package/src/components/diagnose/InvestigationEvidencePane.tsx +1373 -126
- package/src/components/diagnose/InvestigationView.tsx +227 -5
- package/src/components/diagnose/LocalDiagnoseAction.tsx +2 -3
- package/src/components/diagnose/diagnoseEvidenceTypes.ts +22 -1
- package/src/components/diagnose/investigationCase.test.tsx +1568 -0
- package/src/components/diagnose/investigationCase.ts +439 -0
- package/src/components/diagnose/investigationEvidence.test.ts +3525 -17
- package/src/components/diagnose/investigationEvidence.ts +2246 -166
- package/src/components/diagnose/investigationEvidenceKinds.ts +218 -0
- package/src/components/diagnose/investigationEvidencePresentation.test.ts +31 -0
- package/src/components/diagnose/investigationEvidencePresentation.ts +1 -0
- package/src/components/diagnose/investigationMetrics.test.ts +712 -0
- package/src/components/diagnose/investigationMetrics.ts +393 -0
- package/src/components/diagnose/investigationSourceFocus.ts +6 -0
- package/src/components/diagnose/investigationState.test.ts +322 -0
- package/src/components/diagnose/investigationState.ts +147 -25
- package/src/components/diagnose/parts.test.tsx +537 -1
- package/src/components/diagnose/parts.tsx +486 -42
- package/src/components/resource/HPACharts.render.test.tsx +77 -0
- package/src/components/resource/HPACharts.tsx +32 -25
- package/src/components/resource/PrometheusChartsGrid.tsx +181 -79
- package/src/components/resources/PodFilePreview.test.tsx +131 -0
- package/src/components/resources/PodFilePreview.tsx +394 -0
- package/src/components/resources/PodFilesystemModal.tsx +157 -67
- package/src/components/resources/ResourcesView.tsx +27 -2
- package/src/context/DiagnoseCustomization.test.tsx +26 -0
- package/src/context/DiagnoseCustomization.tsx +14 -2
- package/src/index.ts +8 -5
- package/src/utils/shell-safe.test.ts +25 -1
- package/src/utils/shell-safe.ts +16 -0
|
@@ -6,9 +6,11 @@ import {
|
|
|
6
6
|
investigationEvidenceSourceDomId,
|
|
7
7
|
investigationEvidenceSourceId,
|
|
8
8
|
investigationEvidenceSubjectRef,
|
|
9
|
+
metricsScope,
|
|
9
10
|
projectInvestigationEvidence,
|
|
10
11
|
resolveInvestigationRootCauseEvidence,
|
|
11
12
|
type InvestigationEvidenceGroup,
|
|
13
|
+
type InvestigationEvidenceTarget,
|
|
12
14
|
type InvestigationEvidenceTimelineItem,
|
|
13
15
|
type InvestigationEvidenceTurn,
|
|
14
16
|
} from "./investigationEvidence";
|
|
@@ -559,7 +561,7 @@ describe("root-cause evidence resolution", () => {
|
|
|
559
561
|
const projection = project([
|
|
560
562
|
tool(
|
|
561
563
|
"metrics-1",
|
|
562
|
-
"
|
|
564
|
+
"discover_metrics",
|
|
563
565
|
{ result: [1] },
|
|
564
566
|
{ evidenceRef: ref },
|
|
565
567
|
),
|
|
@@ -857,10 +859,10 @@ describe("semantic diagnose evidence projection", () => {
|
|
|
857
859
|
expect(limitationText).toContain("selected 2 of 5");
|
|
858
860
|
expect(limitationText).toContain("includes 2 of 8 selected log lines");
|
|
859
861
|
expect(limitationText).toContain("not the container's full log history");
|
|
860
|
-
expect(limitationText).toContain("
|
|
862
|
+
expect(limitationText).toContain("part of the crash-cause candidates");
|
|
861
863
|
expect(limitationText).toContain("received 1 of 3 event groups");
|
|
862
864
|
expect(limitationText).toContain("rbac denied");
|
|
863
|
-
expect(limitationText).toContain("
|
|
865
|
+
expect(limitationText).toContain("referenced-by relationships");
|
|
864
866
|
expect(limitationText).toContain("recent-change result limit");
|
|
865
867
|
expect(result.coverage).toEqual({
|
|
866
868
|
attempted: 1,
|
|
@@ -943,7 +945,9 @@ describe("strict evidence adapters", () => {
|
|
|
943
945
|
]);
|
|
944
946
|
const events = groupsOf(result.groups, "events")[0].latest;
|
|
945
947
|
|
|
946
|
-
expect(events.summary).toBe(
|
|
948
|
+
expect(events.summary).toBe(
|
|
949
|
+
"BackOff: Back-off restarting failed container · Deployment shop/api",
|
|
950
|
+
);
|
|
947
951
|
expect(events.data).toMatchObject({
|
|
948
952
|
type: "events",
|
|
949
953
|
scope: "Deployment shop/api",
|
|
@@ -2592,7 +2596,7 @@ describe("strict evidence adapters", () => {
|
|
|
2592
2596
|
|
|
2593
2597
|
it("keeps unknown tools in Activity and limits invalid known contracts", () => {
|
|
2594
2598
|
const result = project([
|
|
2595
|
-
tool("other", "
|
|
2599
|
+
tool("other", "discover_metrics", { data: "ignored" }),
|
|
2596
2600
|
tool("bad-events", "get_events", { events: [{ nope: true }] }),
|
|
2597
2601
|
]);
|
|
2598
2602
|
expect(result.sources.map((source) => source.stepId)).toEqual([
|
|
@@ -2815,7 +2819,8 @@ describe("honest zero and partial-result states", () => {
|
|
|
2815
2819
|
);
|
|
2816
2820
|
// list_resources uses [] for some RBAC-filtered reads, so its emptiness is
|
|
2817
2821
|
// never a positive receipt even when the transport itself succeeded. The
|
|
2818
|
-
//
|
|
2822
|
+
// events producer marks a denied namespace explicitly, so its empty result
|
|
2823
|
+
// is a receipt; the changes producer still has the ambiguity. The targeted
|
|
2819
2824
|
// semantic diagnose bundle is authoritative because access is checked before
|
|
2820
2825
|
// that workload path runs.
|
|
2821
2826
|
expect(
|
|
@@ -2826,16 +2831,15 @@ describe("honest zero and partial-result states", () => {
|
|
|
2826
2831
|
),
|
|
2827
2832
|
).toBe(false);
|
|
2828
2833
|
expect(result.limitations.map((item) => item.source)).toEqual(
|
|
2829
|
-
expect.arrayContaining([
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
]),
|
|
2834
|
+
expect.arrayContaining(["Recent changes", "Resource inventory"]),
|
|
2835
|
+
);
|
|
2836
|
+
expect(result.limitations.map((item) => item.source)).not.toContain(
|
|
2837
|
+
"Events",
|
|
2834
2838
|
);
|
|
2835
2839
|
expect(result.coverage).toEqual({
|
|
2836
2840
|
attempted: 5,
|
|
2837
|
-
projected:
|
|
2838
|
-
limited:
|
|
2841
|
+
projected: 3,
|
|
2842
|
+
limited: 2,
|
|
2839
2843
|
checked: 1,
|
|
2840
2844
|
});
|
|
2841
2845
|
expect(
|
|
@@ -2845,10 +2849,10 @@ describe("honest zero and partial-result states", () => {
|
|
|
2845
2849
|
group.latest.data.checked === "events",
|
|
2846
2850
|
)?.latest,
|
|
2847
2851
|
).toMatchObject({
|
|
2848
|
-
title
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
},
|
|
2852
|
+
// The title and the scope beside it are the whole answer here; a body
|
|
2853
|
+
// would only restate them.
|
|
2854
|
+
title: "No warning events",
|
|
2855
|
+
data: { message: undefined },
|
|
2852
2856
|
});
|
|
2853
2857
|
});
|
|
2854
2858
|
|
|
@@ -3107,3 +3111,3507 @@ describe("honest zero and partial-result states", () => {
|
|
|
3107
3111
|
]);
|
|
3108
3112
|
});
|
|
3109
3113
|
});
|
|
3114
|
+
|
|
3115
|
+
// The get_workload_logs response shape, with the stream rows fetchPodLogs
|
|
3116
|
+
// emits per pod and container.
|
|
3117
|
+
const workloadLogsPayload = {
|
|
3118
|
+
workload: "deployments/shop/api",
|
|
3119
|
+
pods: 2,
|
|
3120
|
+
logs: [
|
|
3121
|
+
{
|
|
3122
|
+
pod: "api-68c7b766dc-fmphn",
|
|
3123
|
+
container: "api",
|
|
3124
|
+
logs: {
|
|
3125
|
+
lines: [
|
|
3126
|
+
"2026-09-07T08:00:20.234687659Z [31m[Nest] 1 - [39m09/07/2026, 8:00:20 AM [31m ERROR[39m [38;5;3m[MongooseModule] [39m[31mUnable to connect to the database. Retrying (9)...[39m",
|
|
3127
|
+
"2026-09-07T08:00:20.234756499Z MongoServerError: Authentication failed.",
|
|
3128
|
+
],
|
|
3129
|
+
totalLines: 50,
|
|
3130
|
+
matchedLines: 4,
|
|
3131
|
+
fallback: false,
|
|
3132
|
+
},
|
|
3133
|
+
},
|
|
3134
|
+
{
|
|
3135
|
+
pod: "api-68c7b766dc-fmphn",
|
|
3136
|
+
container: "istio-proxy",
|
|
3137
|
+
logs: {
|
|
3138
|
+
lines: ["2026-09-07T08:00:19Z info Envoy proxy is ready"],
|
|
3139
|
+
totalLines: 12,
|
|
3140
|
+
matchedLines: 0,
|
|
3141
|
+
fallback: true,
|
|
3142
|
+
},
|
|
3143
|
+
},
|
|
3144
|
+
{
|
|
3145
|
+
pod: "api-68c7b766dc-q2zxr",
|
|
3146
|
+
container: "api",
|
|
3147
|
+
logs: {
|
|
3148
|
+
lines: [
|
|
3149
|
+
"2026-09-07T08:00:21Z MongoServerError: Authentication failed.",
|
|
3150
|
+
],
|
|
3151
|
+
totalLines: 50,
|
|
3152
|
+
matchedLines: 1,
|
|
3153
|
+
fallback: false,
|
|
3154
|
+
},
|
|
3155
|
+
},
|
|
3156
|
+
],
|
|
3157
|
+
narrowHint:
|
|
3158
|
+
"at least one pod's log stream tailed to 50 lines (cap reached) — narrow with since= (e.g. 10m), grep= regex, container=, or raise tail_lines",
|
|
3159
|
+
warnings: [
|
|
3160
|
+
"1 of 2 pod(s) have container restarts on record; the error(s) that killed prior containers are in the previous instance's logs — call again with `previous: true` to see them.",
|
|
3161
|
+
],
|
|
3162
|
+
};
|
|
3163
|
+
|
|
3164
|
+
const workloadLogsArgs = JSON.stringify({
|
|
3165
|
+
namespace: "shop",
|
|
3166
|
+
name: "api",
|
|
3167
|
+
kind: "deployment",
|
|
3168
|
+
tail_lines: 50,
|
|
3169
|
+
});
|
|
3170
|
+
|
|
3171
|
+
describe("workload logs adapter", () => {
|
|
3172
|
+
it("keeps one multi-container result as one source with several observations", () => {
|
|
3173
|
+
const ref = evidenceRef("a", "b");
|
|
3174
|
+
const projection = project([
|
|
3175
|
+
tool("wl-logs", "get_workload_logs", workloadLogsPayload, {
|
|
3176
|
+
summary: workloadLogsArgs,
|
|
3177
|
+
evidenceRef: ref,
|
|
3178
|
+
}),
|
|
3179
|
+
]);
|
|
3180
|
+
const logs = groupsOf(projection.groups, "logs");
|
|
3181
|
+
expect(logs.map((group) => group.identity)).toEqual([
|
|
3182
|
+
"logs:current:api-68c7b766dc-fmphn:api",
|
|
3183
|
+
"logs:current:api-68c7b766dc-fmphn:istio-proxy",
|
|
3184
|
+
"logs:current:api-68c7b766dc-q2zxr:api",
|
|
3185
|
+
]);
|
|
3186
|
+
expect(projection.sources).toHaveLength(1);
|
|
3187
|
+
expect(projection.coverage).toMatchObject({
|
|
3188
|
+
attempted: 1,
|
|
3189
|
+
projected: 1,
|
|
3190
|
+
limited: 1,
|
|
3191
|
+
});
|
|
3192
|
+
expect(new Set(logs.map((group) => group.latest.source.id)).size).toBe(1);
|
|
3193
|
+
// Diagnose relates a target workload to its pods the same way.
|
|
3194
|
+
expect(logs.map((group) => group.latest.relevance)).toEqual([
|
|
3195
|
+
"producer-related",
|
|
3196
|
+
"producer-related",
|
|
3197
|
+
"producer-related",
|
|
3198
|
+
]);
|
|
3199
|
+
expect(logs[0].latest.tier).toBe("supporting");
|
|
3200
|
+
expect(logs[0].latest.tone).toBe("warning");
|
|
3201
|
+
expect(logs[0].latest.data).toMatchObject({
|
|
3202
|
+
type: "logs",
|
|
3203
|
+
namespace: "shop",
|
|
3204
|
+
previous: false,
|
|
3205
|
+
warnings: workloadLogsPayload.warnings,
|
|
3206
|
+
});
|
|
3207
|
+
expect(
|
|
3208
|
+
(logs[0].latest.data as { logs?: { lines: string[] } }).logs?.lines[0],
|
|
3209
|
+
).not.toContain("[");
|
|
3210
|
+
expect(logs[1].latest.tier).toBe("context");
|
|
3211
|
+
expect(investigationEvidenceSubjectRef(logs[2].latest.data)).toEqual({
|
|
3212
|
+
kind: "Pod",
|
|
3213
|
+
namespace: "shop",
|
|
3214
|
+
name: "api-68c7b766dc-q2zxr",
|
|
3215
|
+
});
|
|
3216
|
+
|
|
3217
|
+
const resolution = resolveInvestigationRootCauseEvidence(
|
|
3218
|
+
projection,
|
|
3219
|
+
{ status: "linked", refs: [ref] },
|
|
3220
|
+
0,
|
|
3221
|
+
);
|
|
3222
|
+
expect(resolution.status).toBe("linked");
|
|
3223
|
+
expect(resolution.links[0].source.stepId).toBe("wl-logs");
|
|
3224
|
+
expect(resolution.links[0].originalGroupId).toBe(logs[0].id);
|
|
3225
|
+
expect(projection.sources[0].primaryGroupId).toBe(logs[0].id);
|
|
3226
|
+
expect(
|
|
3227
|
+
projection.limitations.map((limitation) => limitation.source),
|
|
3228
|
+
).toEqual(["Workload logs"]);
|
|
3229
|
+
expect(projection.limitations[0].kind).toBe("truncated");
|
|
3230
|
+
});
|
|
3231
|
+
|
|
3232
|
+
it("accepts every workload kind the producer resolves, including Rollouts", () => {
|
|
3233
|
+
const rollout = projectInvestigationEvidence(
|
|
3234
|
+
[
|
|
3235
|
+
{
|
|
3236
|
+
timeline: [
|
|
3237
|
+
tool(
|
|
3238
|
+
"wl-rollout",
|
|
3239
|
+
"get_workload_logs",
|
|
3240
|
+
{ ...workloadLogsPayload, workload: "rollouts/shop/api" },
|
|
3241
|
+
{
|
|
3242
|
+
summary: JSON.stringify({
|
|
3243
|
+
namespace: "shop",
|
|
3244
|
+
name: "api",
|
|
3245
|
+
kind: "rollout",
|
|
3246
|
+
}),
|
|
3247
|
+
},
|
|
3248
|
+
),
|
|
3249
|
+
],
|
|
3250
|
+
},
|
|
3251
|
+
],
|
|
3252
|
+
{ kind: "Rollout", group: "argoproj.io", namespace: "shop", name: "api" },
|
|
3253
|
+
);
|
|
3254
|
+
expect(
|
|
3255
|
+
groupsOf(rollout.groups, "logs").map((group) => group.latest.relevance),
|
|
3256
|
+
).toEqual(["producer-related", "producer-related", "producer-related"]);
|
|
3257
|
+
expect(rollout.limitations.map((limitation) => limitation.kind)).toEqual([
|
|
3258
|
+
"truncated",
|
|
3259
|
+
]);
|
|
3260
|
+
});
|
|
3261
|
+
|
|
3262
|
+
it("scopes a sibling workload's logs as broader and a Pod target's own row as target", () => {
|
|
3263
|
+
const sibling = project([
|
|
3264
|
+
tool(
|
|
3265
|
+
"wl-sibling",
|
|
3266
|
+
"get_workload_logs",
|
|
3267
|
+
{ ...workloadLogsPayload, workload: "deployments/shop/api-worker" },
|
|
3268
|
+
{
|
|
3269
|
+
summary: JSON.stringify({
|
|
3270
|
+
namespace: "shop",
|
|
3271
|
+
name: "api-worker",
|
|
3272
|
+
kind: "deployment",
|
|
3273
|
+
}),
|
|
3274
|
+
},
|
|
3275
|
+
),
|
|
3276
|
+
]);
|
|
3277
|
+
expect(
|
|
3278
|
+
groupsOf(sibling.groups, "logs").map((group) => group.latest.relevance),
|
|
3279
|
+
).toEqual(["broader", "broader", "broader"]);
|
|
3280
|
+
expect(
|
|
3281
|
+
groupsOf(sibling.groups, "logs").map((group) => group.latest.tier),
|
|
3282
|
+
).toEqual(["context", "context", "context"]);
|
|
3283
|
+
|
|
3284
|
+
const podTarget = projectInvestigationEvidence(
|
|
3285
|
+
[
|
|
3286
|
+
{
|
|
3287
|
+
timeline: [
|
|
3288
|
+
tool("wl-pod", "get_workload_logs", workloadLogsPayload, {
|
|
3289
|
+
summary: workloadLogsArgs,
|
|
3290
|
+
}),
|
|
3291
|
+
],
|
|
3292
|
+
},
|
|
3293
|
+
],
|
|
3294
|
+
{
|
|
3295
|
+
kind: "Pod",
|
|
3296
|
+
group: "",
|
|
3297
|
+
namespace: "shop",
|
|
3298
|
+
name: "api-68c7b766dc-q2zxr",
|
|
3299
|
+
},
|
|
3300
|
+
);
|
|
3301
|
+
expect(
|
|
3302
|
+
groupsOf(podTarget.groups, "logs").map((group) => group.latest.relevance),
|
|
3303
|
+
).toEqual(["broader", "broader", "target"]);
|
|
3304
|
+
});
|
|
3305
|
+
|
|
3306
|
+
it("reports a failed read as a limitation and no pods as a scoped receipt", () => {
|
|
3307
|
+
const failed = project([
|
|
3308
|
+
tool(
|
|
3309
|
+
"wl-error",
|
|
3310
|
+
"get_workload_logs",
|
|
3311
|
+
{
|
|
3312
|
+
workload: "deployments/shop/api",
|
|
3313
|
+
pods: 2,
|
|
3314
|
+
logsError: "no kube client in request context",
|
|
3315
|
+
},
|
|
3316
|
+
{ summary: workloadLogsArgs },
|
|
3317
|
+
),
|
|
3318
|
+
]);
|
|
3319
|
+
expect(failed.groups).toHaveLength(0);
|
|
3320
|
+
expect(failed.limitations).toEqual([
|
|
3321
|
+
expect.objectContaining({
|
|
3322
|
+
source: "Workload logs",
|
|
3323
|
+
message: "no kube client in request context",
|
|
3324
|
+
kind: "error",
|
|
3325
|
+
}),
|
|
3326
|
+
]);
|
|
3327
|
+
|
|
3328
|
+
const empty = project([
|
|
3329
|
+
tool(
|
|
3330
|
+
"wl-empty",
|
|
3331
|
+
"get_workload_logs",
|
|
3332
|
+
{
|
|
3333
|
+
workload: "jobs/shop/api",
|
|
3334
|
+
pods: 0,
|
|
3335
|
+
logs: "No pods found for this Job yet. Check scheduling, admission, or controller events.",
|
|
3336
|
+
emptyReason: "no-pods",
|
|
3337
|
+
emptyMessage:
|
|
3338
|
+
"No pods found for this Job yet. Check scheduling, admission, or controller events.",
|
|
3339
|
+
command: "kubectl logs job/api -n shop",
|
|
3340
|
+
},
|
|
3341
|
+
{
|
|
3342
|
+
summary: JSON.stringify({
|
|
3343
|
+
namespace: "shop",
|
|
3344
|
+
name: "api",
|
|
3345
|
+
kind: "job",
|
|
3346
|
+
}),
|
|
3347
|
+
},
|
|
3348
|
+
),
|
|
3349
|
+
]);
|
|
3350
|
+
const receipt = groupsOf(empty.groups, "receipt")[0].latest;
|
|
3351
|
+
expect(receipt.tier).toBe("context");
|
|
3352
|
+
expect(receipt.relevance).toBe("broader");
|
|
3353
|
+
expect(receipt.data).toMatchObject({
|
|
3354
|
+
type: "receipt",
|
|
3355
|
+
checked: "logs",
|
|
3356
|
+
scope: "Job shop/api",
|
|
3357
|
+
message:
|
|
3358
|
+
"No pods found for this Job yet. Check scheduling, admission, or controller events.",
|
|
3359
|
+
});
|
|
3360
|
+
expect(empty.limitations).toHaveLength(0);
|
|
3361
|
+
|
|
3362
|
+
const unconfirmed = project([
|
|
3363
|
+
tool(
|
|
3364
|
+
"wl-unconfirmed",
|
|
3365
|
+
"get_workload_logs",
|
|
3366
|
+
{ workload: "deployments/shop/api", pods: 0, logs: "no pods" },
|
|
3367
|
+
{ summary: workloadLogsArgs, isError: undefined },
|
|
3368
|
+
),
|
|
3369
|
+
]);
|
|
3370
|
+
expect(groupsOf(unconfirmed.groups, "receipt")).toHaveLength(0);
|
|
3371
|
+
});
|
|
3372
|
+
|
|
3373
|
+
it("rejects malformed payloads and per-stream errors honestly", () => {
|
|
3374
|
+
const malformed = project([
|
|
3375
|
+
tool(
|
|
3376
|
+
"wl-bad",
|
|
3377
|
+
"get_workload_logs",
|
|
3378
|
+
{ workload: "pods/shop/api", pods: 1, logs: [] },
|
|
3379
|
+
{ summary: workloadLogsArgs },
|
|
3380
|
+
),
|
|
3381
|
+
tool(
|
|
3382
|
+
"wl-stream-error",
|
|
3383
|
+
"get_workload_logs",
|
|
3384
|
+
{
|
|
3385
|
+
workload: "deployments/shop/api",
|
|
3386
|
+
pods: 1,
|
|
3387
|
+
logs: [
|
|
3388
|
+
{
|
|
3389
|
+
pod: "api-68c7b766dc-fmphn",
|
|
3390
|
+
container: "api",
|
|
3391
|
+
error: "failed to get logs: container is waiting to start",
|
|
3392
|
+
},
|
|
3393
|
+
],
|
|
3394
|
+
},
|
|
3395
|
+
{ summary: workloadLogsArgs },
|
|
3396
|
+
),
|
|
3397
|
+
tool(
|
|
3398
|
+
"wl-null",
|
|
3399
|
+
"get_workload_logs",
|
|
3400
|
+
{ workload: "deployments/shop/api", pods: 3, logs: null },
|
|
3401
|
+
{ summary: workloadLogsArgs },
|
|
3402
|
+
),
|
|
3403
|
+
tool(
|
|
3404
|
+
"wl-zero-with-streams",
|
|
3405
|
+
"get_workload_logs",
|
|
3406
|
+
{
|
|
3407
|
+
workload: "deployments/shop/api",
|
|
3408
|
+
pods: 0,
|
|
3409
|
+
logs: workloadLogsPayload.logs,
|
|
3410
|
+
},
|
|
3411
|
+
{ summary: workloadLogsArgs },
|
|
3412
|
+
),
|
|
3413
|
+
]);
|
|
3414
|
+
expect(malformed.groups).toHaveLength(0);
|
|
3415
|
+
expect(
|
|
3416
|
+
malformed.limitations.map((limitation) => [
|
|
3417
|
+
limitation.source,
|
|
3418
|
+
limitation.kind,
|
|
3419
|
+
]),
|
|
3420
|
+
).toEqual([
|
|
3421
|
+
["Workload logs", "unknown"],
|
|
3422
|
+
["api-68c7b766dc-fmphn / api", "error"],
|
|
3423
|
+
["Workload logs", "unknown"],
|
|
3424
|
+
]);
|
|
3425
|
+
expect(malformed.limitations[0].sources.map((item) => item.stepId)).toEqual(
|
|
3426
|
+
["wl-bad", "wl-zero-with-streams"],
|
|
3427
|
+
);
|
|
3428
|
+
expect(malformed.limitations[0].message).toContain(
|
|
3429
|
+
"couldn't summarize this investigation step",
|
|
3430
|
+
);
|
|
3431
|
+
expect(malformed.limitations[2].message).toContain(
|
|
3432
|
+
"Radar found 3 pods but got no logs from them",
|
|
3433
|
+
);
|
|
3434
|
+
});
|
|
3435
|
+
});
|
|
3436
|
+
|
|
3437
|
+
const firingInstance = {
|
|
3438
|
+
state: "firing",
|
|
3439
|
+
activeAt: "2026-09-07T07:58:00Z",
|
|
3440
|
+
value: "1e+00",
|
|
3441
|
+
labels: {
|
|
3442
|
+
alertname: "KubePodCrashLooping",
|
|
3443
|
+
namespace: "shop",
|
|
3444
|
+
pod: "api-68c7b766dc-fmphn",
|
|
3445
|
+
container: "api",
|
|
3446
|
+
severity: "warning",
|
|
3447
|
+
},
|
|
3448
|
+
};
|
|
3449
|
+
|
|
3450
|
+
function alertingRule(
|
|
3451
|
+
patch: Record<string, unknown> = {},
|
|
3452
|
+
): Record<string, unknown> {
|
|
3453
|
+
return {
|
|
3454
|
+
group: "kubernetes-apps",
|
|
3455
|
+
type: "alerting",
|
|
3456
|
+
name: "KubePodCrashLooping",
|
|
3457
|
+
query:
|
|
3458
|
+
'max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}[5m]) >= 1',
|
|
3459
|
+
state: "firing",
|
|
3460
|
+
health: "ok",
|
|
3461
|
+
labels: { severity: "warning" },
|
|
3462
|
+
annotations: {
|
|
3463
|
+
summary: "Pod is crash looping.",
|
|
3464
|
+
description:
|
|
3465
|
+
"Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: CrashLoopBackOff).",
|
|
3466
|
+
},
|
|
3467
|
+
alerts: [firingInstance],
|
|
3468
|
+
...patch,
|
|
3469
|
+
};
|
|
3470
|
+
}
|
|
3471
|
+
|
|
3472
|
+
const rulesArgs = JSON.stringify({ state: "firing" });
|
|
3473
|
+
|
|
3474
|
+
/** A diagnose read that ties the given pods to the target as its log streams. */
|
|
3475
|
+
function diagnoseWithPods(...pods: string[]) {
|
|
3476
|
+
return tool("diagnose-pods", "diagnose", {
|
|
3477
|
+
resource: deployment,
|
|
3478
|
+
resourceContext: { tier: "basic" },
|
|
3479
|
+
logsCurrent: pods.map((pod) => ({
|
|
3480
|
+
pod,
|
|
3481
|
+
container: "api",
|
|
3482
|
+
logs: {
|
|
3483
|
+
lines: ["ready"],
|
|
3484
|
+
totalLines: 1,
|
|
3485
|
+
matchedLines: 0,
|
|
3486
|
+
fallback: true,
|
|
3487
|
+
},
|
|
3488
|
+
})),
|
|
3489
|
+
});
|
|
3490
|
+
}
|
|
3491
|
+
|
|
3492
|
+
describe("prometheus rules adapter", () => {
|
|
3493
|
+
it("relates a rule to the target through its instances, never the rule alone", () => {
|
|
3494
|
+
const projection = project([
|
|
3495
|
+
diagnoseWithPods("api-68c7b766dc-fmphn"),
|
|
3496
|
+
tool(
|
|
3497
|
+
"rules",
|
|
3498
|
+
"get_prometheus_rules",
|
|
3499
|
+
{
|
|
3500
|
+
count: 4,
|
|
3501
|
+
rules: [
|
|
3502
|
+
alertingRule(),
|
|
3503
|
+
alertingRule({
|
|
3504
|
+
name: "KubeDeploymentReplicasMismatch",
|
|
3505
|
+
labels: { severity: "warning", namespace: "shop" },
|
|
3506
|
+
alerts: [
|
|
3507
|
+
{
|
|
3508
|
+
state: "firing",
|
|
3509
|
+
labels: {
|
|
3510
|
+
namespace: "shop",
|
|
3511
|
+
deployment: "api-worker",
|
|
3512
|
+
},
|
|
3513
|
+
},
|
|
3514
|
+
],
|
|
3515
|
+
}),
|
|
3516
|
+
alertingRule({
|
|
3517
|
+
name: "TargetDown",
|
|
3518
|
+
group: "general",
|
|
3519
|
+
labels: { severity: "critical" },
|
|
3520
|
+
alerts: [
|
|
3521
|
+
{
|
|
3522
|
+
state: "firing",
|
|
3523
|
+
labels: { namespace: "monitoring", job: "node-exporter" },
|
|
3524
|
+
},
|
|
3525
|
+
],
|
|
3526
|
+
}),
|
|
3527
|
+
alertingRule({
|
|
3528
|
+
name: "SiblingCrash",
|
|
3529
|
+
alerts: [
|
|
3530
|
+
{
|
|
3531
|
+
state: "firing",
|
|
3532
|
+
labels: {
|
|
3533
|
+
namespace: "shop",
|
|
3534
|
+
pod: "api-worker-5d8f7c9b6-x2k9p",
|
|
3535
|
+
},
|
|
3536
|
+
},
|
|
3537
|
+
],
|
|
3538
|
+
}),
|
|
3539
|
+
],
|
|
3540
|
+
},
|
|
3541
|
+
{ summary: rulesArgs },
|
|
3542
|
+
),
|
|
3543
|
+
]);
|
|
3544
|
+
const alerts = groupsOf(projection.groups, "alerts");
|
|
3545
|
+
expect(alerts.map((group) => group.latest.relevance)).toEqual([
|
|
3546
|
+
"target",
|
|
3547
|
+
"broader",
|
|
3548
|
+
"broader",
|
|
3549
|
+
"broader",
|
|
3550
|
+
]);
|
|
3551
|
+
expect(alerts.map((group) => group.latest.tier)).toEqual([
|
|
3552
|
+
"supporting",
|
|
3553
|
+
"context",
|
|
3554
|
+
"context",
|
|
3555
|
+
"context",
|
|
3556
|
+
]);
|
|
3557
|
+
expect(alerts[0].latest.tone).toBe("alert");
|
|
3558
|
+
expect(alerts[2].latest.tone).toBe("error");
|
|
3559
|
+
expect(alerts[0].latest.title).toBe("KubePodCrashLooping firing");
|
|
3560
|
+
expect(alerts[0].latest.summary).toBe(
|
|
3561
|
+
"1 active instance, 1 naming Deployment shop/api · kubernetes-apps",
|
|
3562
|
+
);
|
|
3563
|
+
expect(alerts[0].latest.data).toMatchObject({
|
|
3564
|
+
type: "alerts",
|
|
3565
|
+
rule: { group: "kubernetes-apps", state: "firing", health: "ok" },
|
|
3566
|
+
instances: [{ namesTarget: true, value: "1e+00" }],
|
|
3567
|
+
annotations: { summary: "Pod is crash looping." },
|
|
3568
|
+
});
|
|
3569
|
+
expect(alerts[3].latest.data).toMatchObject({
|
|
3570
|
+
instances: [{ namesTarget: false }],
|
|
3571
|
+
});
|
|
3572
|
+
expect(
|
|
3573
|
+
groupsOf(projection.groups, "receipt").filter((group) =>
|
|
3574
|
+
group.identity.startsWith("alerts:"),
|
|
3575
|
+
),
|
|
3576
|
+
).toHaveLength(0);
|
|
3577
|
+
expect(
|
|
3578
|
+
projection.limitations.filter((limitation) =>
|
|
3579
|
+
limitation.source.startsWith("Alert"),
|
|
3580
|
+
),
|
|
3581
|
+
).toHaveLength(0);
|
|
3582
|
+
});
|
|
3583
|
+
|
|
3584
|
+
it("uses rule-level labels only as producer-related context", () => {
|
|
3585
|
+
const projection = project([
|
|
3586
|
+
tool(
|
|
3587
|
+
"rules",
|
|
3588
|
+
"get_prometheus_rules",
|
|
3589
|
+
{
|
|
3590
|
+
count: 1,
|
|
3591
|
+
rules: [
|
|
3592
|
+
alertingRule({
|
|
3593
|
+
name: "ApiDown",
|
|
3594
|
+
labels: {
|
|
3595
|
+
severity: "warning",
|
|
3596
|
+
namespace: "shop",
|
|
3597
|
+
deployment: "api",
|
|
3598
|
+
},
|
|
3599
|
+
alerts: [
|
|
3600
|
+
{
|
|
3601
|
+
state: "firing",
|
|
3602
|
+
labels: { namespace: "shop", instance: "10.0.0.4:8080" },
|
|
3603
|
+
},
|
|
3604
|
+
],
|
|
3605
|
+
}),
|
|
3606
|
+
],
|
|
3607
|
+
},
|
|
3608
|
+
{ summary: rulesArgs },
|
|
3609
|
+
),
|
|
3610
|
+
]);
|
|
3611
|
+
const alert = groupsOf(projection.groups, "alerts")[0].latest;
|
|
3612
|
+
expect(alert.relevance).toBe("producer-related");
|
|
3613
|
+
expect(alert.tier).toBe("supporting");
|
|
3614
|
+
expect(alert.summary).toBe("1 active instance · kubernetes-apps");
|
|
3615
|
+
expect(groupsOf(projection.groups, "receipt")).toHaveLength(0);
|
|
3616
|
+
});
|
|
3617
|
+
|
|
3618
|
+
it("matches workload labels per kind and pod labels only through producer-established pods", () => {
|
|
3619
|
+
const run = (
|
|
3620
|
+
target: InvestigationEvidenceTarget,
|
|
3621
|
+
labels: object,
|
|
3622
|
+
capturedPods: string[] = [],
|
|
3623
|
+
) =>
|
|
3624
|
+
groupsOf(
|
|
3625
|
+
projectInvestigationEvidence(
|
|
3626
|
+
[
|
|
3627
|
+
{
|
|
3628
|
+
timeline: [
|
|
3629
|
+
...(capturedPods.length > 0
|
|
3630
|
+
? [
|
|
3631
|
+
tool("diagnose-pods", "diagnose", {
|
|
3632
|
+
resource: {
|
|
3633
|
+
...deployment,
|
|
3634
|
+
kind: target.kind,
|
|
3635
|
+
metadata: {
|
|
3636
|
+
namespace: target.namespace,
|
|
3637
|
+
name: target.name,
|
|
3638
|
+
},
|
|
3639
|
+
},
|
|
3640
|
+
resourceContext: { tier: "basic" },
|
|
3641
|
+
logsCurrent: capturedPods.map((pod) => ({
|
|
3642
|
+
pod,
|
|
3643
|
+
container: "app",
|
|
3644
|
+
logs: {
|
|
3645
|
+
lines: ["ready"],
|
|
3646
|
+
totalLines: 1,
|
|
3647
|
+
matchedLines: 0,
|
|
3648
|
+
fallback: true,
|
|
3649
|
+
},
|
|
3650
|
+
})),
|
|
3651
|
+
}),
|
|
3652
|
+
]
|
|
3653
|
+
: []),
|
|
3654
|
+
tool(
|
|
3655
|
+
"rules",
|
|
3656
|
+
"get_prometheus_rules",
|
|
3657
|
+
{
|
|
3658
|
+
count: 1,
|
|
3659
|
+
rules: [
|
|
3660
|
+
alertingRule({ alerts: [{ state: "firing", labels }] }),
|
|
3661
|
+
],
|
|
3662
|
+
},
|
|
3663
|
+
{ summary: rulesArgs },
|
|
3664
|
+
),
|
|
3665
|
+
],
|
|
3666
|
+
},
|
|
3667
|
+
],
|
|
3668
|
+
target,
|
|
3669
|
+
).groups,
|
|
3670
|
+
"alerts",
|
|
3671
|
+
)[0].latest.relevance;
|
|
3672
|
+
const statefulSet = { ...target, kind: "StatefulSet", name: "db" };
|
|
3673
|
+
expect(run(target, { namespace: "shop", deployment: "api" })).toBe(
|
|
3674
|
+
"target",
|
|
3675
|
+
);
|
|
3676
|
+
expect(run(target, { namespace: "other", deployment: "api" })).toBe(
|
|
3677
|
+
"broader",
|
|
3678
|
+
);
|
|
3679
|
+
expect(run(target, { namespace: "shop", statefulset: "api" })).toBe(
|
|
3680
|
+
"broader",
|
|
3681
|
+
);
|
|
3682
|
+
expect(
|
|
3683
|
+
run(target, {
|
|
3684
|
+
namespace: "shop",
|
|
3685
|
+
workload: "api",
|
|
3686
|
+
workload_type: "deployment",
|
|
3687
|
+
}),
|
|
3688
|
+
).toBe("target");
|
|
3689
|
+
expect(
|
|
3690
|
+
run(target, {
|
|
3691
|
+
namespace: "shop",
|
|
3692
|
+
workload: "api",
|
|
3693
|
+
workload_type: "statefulset",
|
|
3694
|
+
}),
|
|
3695
|
+
).toBe("broader");
|
|
3696
|
+
// A controller-shaped pod name proves nothing on its own: a sibling
|
|
3697
|
+
// ReplicaSet `api-worker` produces `api-worker-abcde` too.
|
|
3698
|
+
expect(
|
|
3699
|
+
run(target, { namespace: "shop", pod: "api-68c7b766dc-fmphn" }),
|
|
3700
|
+
).toBe("broader");
|
|
3701
|
+
expect(
|
|
3702
|
+
run(target, { namespace: "shop", pod: "api-68c7b766dc-fmphn" }, [
|
|
3703
|
+
"api-68c7b766dc-fmphn",
|
|
3704
|
+
]),
|
|
3705
|
+
).toBe("target");
|
|
3706
|
+
expect(
|
|
3707
|
+
run(target, { namespace: "shop", pod: "api-worker-abcde" }, [
|
|
3708
|
+
"api-68c7b766dc-fmphn",
|
|
3709
|
+
]),
|
|
3710
|
+
).toBe("broader");
|
|
3711
|
+
expect(run(statefulSet, { namespace: "shop", pod: "db-0" }, ["db-0"])).toBe(
|
|
3712
|
+
"target",
|
|
3713
|
+
);
|
|
3714
|
+
expect(
|
|
3715
|
+
run(statefulSet, { namespace: "other", pod: "db-0" }, ["db-0"]),
|
|
3716
|
+
).toBe("broader");
|
|
3717
|
+
expect(
|
|
3718
|
+
run(target, { pod: "api-68c7b766dc-fmphn" }, ["api-68c7b766dc-fmphn"]),
|
|
3719
|
+
).toBe("broader");
|
|
3720
|
+
});
|
|
3721
|
+
|
|
3722
|
+
it("issues a scoped receipt only for a complete, unfiltered, evaluated active-state query", () => {
|
|
3723
|
+
const unrelated = alertingRule({
|
|
3724
|
+
name: "TargetDown",
|
|
3725
|
+
alerts: [{ state: "firing", labels: { namespace: "monitoring" } }],
|
|
3726
|
+
});
|
|
3727
|
+
const receipt = (
|
|
3728
|
+
payload: Record<string, unknown>,
|
|
3729
|
+
args: Record<string, unknown>,
|
|
3730
|
+
) =>
|
|
3731
|
+
groupsOf(
|
|
3732
|
+
project([
|
|
3733
|
+
tool("rules", "get_prometheus_rules", payload, {
|
|
3734
|
+
summary: JSON.stringify(args),
|
|
3735
|
+
}),
|
|
3736
|
+
]).groups,
|
|
3737
|
+
"receipt",
|
|
3738
|
+
);
|
|
3739
|
+
|
|
3740
|
+
const firing = receipt(
|
|
3741
|
+
{ count: 1, rules: [unrelated] },
|
|
3742
|
+
{ state: "firing" },
|
|
3743
|
+
);
|
|
3744
|
+
expect(firing).toHaveLength(1);
|
|
3745
|
+
expect(firing[0].latest).toMatchObject({
|
|
3746
|
+
tier: "checked",
|
|
3747
|
+
relevance: "target",
|
|
3748
|
+
title: "No firing alerts matched this Deployment",
|
|
3749
|
+
summary: "Deployment shop/api",
|
|
3750
|
+
data: {
|
|
3751
|
+
type: "receipt",
|
|
3752
|
+
checked: "alerts",
|
|
3753
|
+
scope: "Deployment shop/api",
|
|
3754
|
+
message:
|
|
3755
|
+
"1 alerting rule returned (filter: state=firing), every instance in another namespace; none names Deployment shop/api.",
|
|
3756
|
+
},
|
|
3757
|
+
});
|
|
3758
|
+
|
|
3759
|
+
expect(
|
|
3760
|
+
receipt(
|
|
3761
|
+
{
|
|
3762
|
+
count: 0,
|
|
3763
|
+
rules: [],
|
|
3764
|
+
note: "no rules matched — drop filters to see what exists, or the backend may have no rules configured",
|
|
3765
|
+
},
|
|
3766
|
+
{ state: "pending" },
|
|
3767
|
+
)[0].latest.data,
|
|
3768
|
+
).toMatchObject({
|
|
3769
|
+
message: "0 alerting rules returned (filter: state=pending).",
|
|
3770
|
+
});
|
|
3771
|
+
|
|
3772
|
+
// A name or group filter never looked at the other rules.
|
|
3773
|
+
expect(
|
|
3774
|
+
receipt(
|
|
3775
|
+
{ count: 1, rules: [unrelated] },
|
|
3776
|
+
{ state: "firing", name: "Target" },
|
|
3777
|
+
),
|
|
3778
|
+
).toHaveLength(0);
|
|
3779
|
+
expect(
|
|
3780
|
+
receipt({ count: 0, rules: [] }, { state: "firing", group: "general" }),
|
|
3781
|
+
).toHaveLength(0);
|
|
3782
|
+
// An instance this projection cannot place is not evidence of absence.
|
|
3783
|
+
expect(
|
|
3784
|
+
receipt(
|
|
3785
|
+
{
|
|
3786
|
+
count: 1,
|
|
3787
|
+
rules: [
|
|
3788
|
+
alertingRule({
|
|
3789
|
+
name: "Vague",
|
|
3790
|
+
alerts: [
|
|
3791
|
+
{ state: "firing", labels: { instance: "10.0.0.4:9100" } },
|
|
3792
|
+
],
|
|
3793
|
+
}),
|
|
3794
|
+
],
|
|
3795
|
+
},
|
|
3796
|
+
{ state: "firing" },
|
|
3797
|
+
),
|
|
3798
|
+
).toHaveLength(0);
|
|
3799
|
+
expect(
|
|
3800
|
+
receipt(
|
|
3801
|
+
{
|
|
3802
|
+
count: 1,
|
|
3803
|
+
rules: [
|
|
3804
|
+
alertingRule({
|
|
3805
|
+
name: "SameNamespace",
|
|
3806
|
+
alerts: [
|
|
3807
|
+
{
|
|
3808
|
+
state: "firing",
|
|
3809
|
+
labels: { namespace: "shop", pod: "other-abc" },
|
|
3810
|
+
},
|
|
3811
|
+
],
|
|
3812
|
+
}),
|
|
3813
|
+
],
|
|
3814
|
+
},
|
|
3815
|
+
{ state: "firing" },
|
|
3816
|
+
),
|
|
3817
|
+
).toHaveLength(0);
|
|
3818
|
+
expect(receipt({ count: 1, rules: [unrelated] }, {})).toHaveLength(0);
|
|
3819
|
+
expect(
|
|
3820
|
+
receipt({ count: 1, rules: [unrelated] }, { state: "inactive" }),
|
|
3821
|
+
).toHaveLength(0);
|
|
3822
|
+
expect(
|
|
3823
|
+
receipt(
|
|
3824
|
+
{ count: 1, rules: [alertingRule({ type: "recording", name: "x" })] },
|
|
3825
|
+
{ type: "record", state: "firing" },
|
|
3826
|
+
),
|
|
3827
|
+
).toHaveLength(0);
|
|
3828
|
+
expect(
|
|
3829
|
+
receipt(
|
|
3830
|
+
{
|
|
3831
|
+
count: 1,
|
|
3832
|
+
rules: [unrelated],
|
|
3833
|
+
truncated: true,
|
|
3834
|
+
note: "narrow with name, group, state, or type filters",
|
|
3835
|
+
},
|
|
3836
|
+
{ state: "firing" },
|
|
3837
|
+
),
|
|
3838
|
+
).toHaveLength(0);
|
|
3839
|
+
expect(
|
|
3840
|
+
receipt(
|
|
3841
|
+
{ count: 1, rules: [{ ...unrelated, health: "err" }] },
|
|
3842
|
+
{ state: "firing" },
|
|
3843
|
+
),
|
|
3844
|
+
).toHaveLength(0);
|
|
3845
|
+
expect(
|
|
3846
|
+
receipt(
|
|
3847
|
+
{ count: 1, rules: [{ ...unrelated, health: "degraded" }] },
|
|
3848
|
+
{ state: "firing" },
|
|
3849
|
+
),
|
|
3850
|
+
).toHaveLength(0);
|
|
3851
|
+
expect(
|
|
3852
|
+
receipt(
|
|
3853
|
+
{ count: 1, rules: [{ ...unrelated, health: undefined }] },
|
|
3854
|
+
{ state: "firing" },
|
|
3855
|
+
),
|
|
3856
|
+
).toHaveLength(0);
|
|
3857
|
+
});
|
|
3858
|
+
|
|
3859
|
+
it("advances the card through alert state transitions and keeps distinct rules apart", () => {
|
|
3860
|
+
const firingForTarget = alertingRule();
|
|
3861
|
+
const resolved = alertingRule({ state: "inactive", alerts: [] });
|
|
3862
|
+
const firingElsewhere = alertingRule({
|
|
3863
|
+
alerts: [
|
|
3864
|
+
{
|
|
3865
|
+
state: "firing",
|
|
3866
|
+
labels: { namespace: "shop", pod: "api-worker-5d8f7c9b6-x2k9p" },
|
|
3867
|
+
},
|
|
3868
|
+
],
|
|
3869
|
+
});
|
|
3870
|
+
const run = (...later: Record<string, unknown>[]) =>
|
|
3871
|
+
groupsOf(
|
|
3872
|
+
project(
|
|
3873
|
+
[
|
|
3874
|
+
diagnoseWithPods("api-68c7b766dc-fmphn"),
|
|
3875
|
+
tool(
|
|
3876
|
+
"rules-1",
|
|
3877
|
+
"get_prometheus_rules",
|
|
3878
|
+
{ count: 1, rules: [firingForTarget] },
|
|
3879
|
+
{ summary: rulesArgs },
|
|
3880
|
+
),
|
|
3881
|
+
],
|
|
3882
|
+
[
|
|
3883
|
+
tool(
|
|
3884
|
+
"rules-2",
|
|
3885
|
+
"get_prometheus_rules",
|
|
3886
|
+
{ count: later.length, rules: later },
|
|
3887
|
+
{ summary: JSON.stringify({}) },
|
|
3888
|
+
),
|
|
3889
|
+
],
|
|
3890
|
+
).groups,
|
|
3891
|
+
"alerts",
|
|
3892
|
+
);
|
|
3893
|
+
|
|
3894
|
+
const [resolvedGroup] = run(resolved);
|
|
3895
|
+
expect(resolvedGroup.observations).toHaveLength(2);
|
|
3896
|
+
expect(resolvedGroup.latest).toMatchObject({
|
|
3897
|
+
title: "KubePodCrashLooping inactive",
|
|
3898
|
+
relevance: "target",
|
|
3899
|
+
tier: "context",
|
|
3900
|
+
tone: "neutral",
|
|
3901
|
+
summary: "No active instances · kubernetes-apps",
|
|
3902
|
+
});
|
|
3903
|
+
expect(resolvedGroup.latest).toBe(resolvedGroup.chronologicalLatest);
|
|
3904
|
+
expect(resolvedGroup.historical).toBe(false);
|
|
3905
|
+
|
|
3906
|
+
const [movedGroup] = run(firingElsewhere);
|
|
3907
|
+
expect(movedGroup.observations).toHaveLength(2);
|
|
3908
|
+
expect(movedGroup.latest.relevance).toBe("broader");
|
|
3909
|
+
expect(movedGroup.latest.summary).toBe(
|
|
3910
|
+
"1 active instance · kubernetes-apps",
|
|
3911
|
+
);
|
|
3912
|
+
|
|
3913
|
+
const distinct = groupsOf(
|
|
3914
|
+
project([
|
|
3915
|
+
tool(
|
|
3916
|
+
"rules",
|
|
3917
|
+
"get_prometheus_rules",
|
|
3918
|
+
{
|
|
3919
|
+
count: 2,
|
|
3920
|
+
rules: [
|
|
3921
|
+
alertingRule({ labels: { severity: "warning", team: "a" } }),
|
|
3922
|
+
alertingRule({
|
|
3923
|
+
query: "vector(1)",
|
|
3924
|
+
labels: { severity: "critical", team: "b" },
|
|
3925
|
+
}),
|
|
3926
|
+
],
|
|
3927
|
+
},
|
|
3928
|
+
{ summary: rulesArgs },
|
|
3929
|
+
),
|
|
3930
|
+
]).groups,
|
|
3931
|
+
"alerts",
|
|
3932
|
+
);
|
|
3933
|
+
expect(distinct).toHaveLength(2);
|
|
3934
|
+
expect(distinct.every((group) => group.observations.length === 1)).toBe(
|
|
3935
|
+
true,
|
|
3936
|
+
);
|
|
3937
|
+
|
|
3938
|
+
// Identical definitions from different rule files arrive as identical
|
|
3939
|
+
// rows: two rules this read cannot tell apart, so neither inherits the
|
|
3940
|
+
// other's history on a later read.
|
|
3941
|
+
const twin = () => alertingRule({ alerts: [], state: "inactive" });
|
|
3942
|
+
const twins = groupsOf(
|
|
3943
|
+
project(
|
|
3944
|
+
[
|
|
3945
|
+
diagnoseWithPods("api-68c7b766dc-fmphn"),
|
|
3946
|
+
tool(
|
|
3947
|
+
"rules-1",
|
|
3948
|
+
"get_prometheus_rules",
|
|
3949
|
+
{ count: 2, rules: [alertingRule(), twin()] },
|
|
3950
|
+
{ summary: rulesArgs },
|
|
3951
|
+
),
|
|
3952
|
+
],
|
|
3953
|
+
[
|
|
3954
|
+
tool(
|
|
3955
|
+
"rules-2",
|
|
3956
|
+
"get_prometheus_rules",
|
|
3957
|
+
{ count: 2, rules: [alertingRule(), twin()] },
|
|
3958
|
+
{ summary: rulesArgs },
|
|
3959
|
+
),
|
|
3960
|
+
],
|
|
3961
|
+
).groups,
|
|
3962
|
+
"alerts",
|
|
3963
|
+
);
|
|
3964
|
+
expect(twins).toHaveLength(4);
|
|
3965
|
+
expect(twins.every((group) => group.observations.length === 1)).toBe(true);
|
|
3966
|
+
expect(twins.map((group) => group.latest.relevance)).toEqual([
|
|
3967
|
+
"target",
|
|
3968
|
+
"broader",
|
|
3969
|
+
"target",
|
|
3970
|
+
"broader",
|
|
3971
|
+
]);
|
|
3972
|
+
expect(twins.map((group) => group.latest.title)).toEqual([
|
|
3973
|
+
"KubePodCrashLooping firing",
|
|
3974
|
+
"KubePodCrashLooping inactive",
|
|
3975
|
+
"KubePodCrashLooping firing",
|
|
3976
|
+
"KubePodCrashLooping inactive",
|
|
3977
|
+
]);
|
|
3978
|
+
});
|
|
3979
|
+
|
|
3980
|
+
it("never proves absence by namespace for a cluster-scoped target", () => {
|
|
3981
|
+
const node = { kind: "Node", group: "", name: "worker-1" };
|
|
3982
|
+
const run = (rules: Record<string, unknown>[]) =>
|
|
3983
|
+
groupsOf(
|
|
3984
|
+
projectInvestigationEvidence(
|
|
3985
|
+
[
|
|
3986
|
+
{
|
|
3987
|
+
timeline: [
|
|
3988
|
+
tool(
|
|
3989
|
+
"rules",
|
|
3990
|
+
"get_prometheus_rules",
|
|
3991
|
+
{ count: rules.length, rules },
|
|
3992
|
+
{ summary: rulesArgs },
|
|
3993
|
+
),
|
|
3994
|
+
],
|
|
3995
|
+
},
|
|
3996
|
+
],
|
|
3997
|
+
node,
|
|
3998
|
+
).groups,
|
|
3999
|
+
"receipt",
|
|
4000
|
+
);
|
|
4001
|
+
expect(
|
|
4002
|
+
run([
|
|
4003
|
+
alertingRule({
|
|
4004
|
+
name: "KubeNodeNotReady",
|
|
4005
|
+
alerts: [
|
|
4006
|
+
{
|
|
4007
|
+
state: "firing",
|
|
4008
|
+
labels: { node: "worker-1", namespace: "kube-system" },
|
|
4009
|
+
},
|
|
4010
|
+
],
|
|
4011
|
+
}),
|
|
4012
|
+
]),
|
|
4013
|
+
).toHaveLength(0);
|
|
4014
|
+
expect(run([])).toHaveLength(1);
|
|
4015
|
+
expect(run([])[0].latest.title).toBe("No firing alerts matched this Node");
|
|
4016
|
+
});
|
|
4017
|
+
|
|
4018
|
+
it("keeps recording rules off the card list and flags truncation and evaluation health", () => {
|
|
4019
|
+
const projection = project([
|
|
4020
|
+
tool(
|
|
4021
|
+
"rules",
|
|
4022
|
+
"get_prometheus_rules",
|
|
4023
|
+
{
|
|
4024
|
+
count: 3,
|
|
4025
|
+
rules: [
|
|
4026
|
+
alertingRule({
|
|
4027
|
+
type: "recording",
|
|
4028
|
+
name: "namespace:container_cpu_usage_seconds_total:sum_rate",
|
|
4029
|
+
alerts: undefined,
|
|
4030
|
+
state: undefined,
|
|
4031
|
+
}),
|
|
4032
|
+
alertingRule({ health: "err", alerts: [] }),
|
|
4033
|
+
alertingRule({
|
|
4034
|
+
name: "KubeContainerWaiting",
|
|
4035
|
+
state: "inactive",
|
|
4036
|
+
health: "unknown",
|
|
4037
|
+
alerts: [],
|
|
4038
|
+
}),
|
|
4039
|
+
],
|
|
4040
|
+
truncated: true,
|
|
4041
|
+
note: "narrow with name, group, state, or type filters",
|
|
4042
|
+
},
|
|
4043
|
+
{ summary: rulesArgs },
|
|
4044
|
+
),
|
|
4045
|
+
]);
|
|
4046
|
+
const alerts = groupsOf(projection.groups, "alerts");
|
|
4047
|
+
expect(alerts.map((group) => group.latest.title)).toEqual([
|
|
4048
|
+
"KubePodCrashLooping firing",
|
|
4049
|
+
"KubeContainerWaiting inactive",
|
|
4050
|
+
]);
|
|
4051
|
+
expect(alerts[1].latest).toMatchObject({
|
|
4052
|
+
tier: "context",
|
|
4053
|
+
tone: "neutral",
|
|
4054
|
+
relevance: "broader",
|
|
4055
|
+
summary: "No active instances · kubernetes-apps",
|
|
4056
|
+
});
|
|
4057
|
+
expect(
|
|
4058
|
+
projection.limitations.map((limitation) => [
|
|
4059
|
+
limitation.source,
|
|
4060
|
+
limitation.kind,
|
|
4061
|
+
]),
|
|
4062
|
+
).toEqual([
|
|
4063
|
+
["Alert rules", "truncated"],
|
|
4064
|
+
["Alert rule KubePodCrashLooping", "error"],
|
|
4065
|
+
["Alert rule KubeContainerWaiting", "unknown"],
|
|
4066
|
+
]);
|
|
4067
|
+
expect(projection.limitations[0].message).toContain(
|
|
4068
|
+
"narrow with name, group, state, or type filters",
|
|
4069
|
+
);
|
|
4070
|
+
expect(groupsOf(projection.groups, "receipt")).toHaveLength(0);
|
|
4071
|
+
});
|
|
4072
|
+
|
|
4073
|
+
it("does not let sampled values churn revision history, and rejects malformed rules", () => {
|
|
4074
|
+
const later = {
|
|
4075
|
+
...firingInstance,
|
|
4076
|
+
value: "2e+00",
|
|
4077
|
+
activeAt: "2026-09-07T08:10:00Z",
|
|
4078
|
+
};
|
|
4079
|
+
const projection = project(
|
|
4080
|
+
[
|
|
4081
|
+
tool(
|
|
4082
|
+
"rules-1",
|
|
4083
|
+
"get_prometheus_rules",
|
|
4084
|
+
{ count: 1, rules: [alertingRule()] },
|
|
4085
|
+
{ summary: rulesArgs },
|
|
4086
|
+
),
|
|
4087
|
+
],
|
|
4088
|
+
[
|
|
4089
|
+
tool(
|
|
4090
|
+
"rules-2",
|
|
4091
|
+
"get_prometheus_rules",
|
|
4092
|
+
{ count: 1, rules: [alertingRule({ alerts: [later] })] },
|
|
4093
|
+
{ summary: rulesArgs },
|
|
4094
|
+
),
|
|
4095
|
+
],
|
|
4096
|
+
);
|
|
4097
|
+
const group = groupsOf(projection.groups, "alerts")[0];
|
|
4098
|
+
expect(group.observations).toHaveLength(2);
|
|
4099
|
+
expect(group.observations[1].changedFromPrevious).toBe(false);
|
|
4100
|
+
|
|
4101
|
+
const malformed = project([
|
|
4102
|
+
tool(
|
|
4103
|
+
"rules-bad",
|
|
4104
|
+
"get_prometheus_rules",
|
|
4105
|
+
{ count: 1, rules: [{ name: "NoGroup", type: "alerting" }] },
|
|
4106
|
+
{ summary: rulesArgs },
|
|
4107
|
+
),
|
|
4108
|
+
tool(
|
|
4109
|
+
"rules-bad-instance",
|
|
4110
|
+
"get_prometheus_rules",
|
|
4111
|
+
{ count: 1, rules: [alertingRule({ alerts: [{ labels: {} }] })] },
|
|
4112
|
+
{ summary: rulesArgs },
|
|
4113
|
+
),
|
|
4114
|
+
]);
|
|
4115
|
+
expect(malformed.groups).toHaveLength(0);
|
|
4116
|
+
expect(malformed.limitations).toHaveLength(1);
|
|
4117
|
+
expect(malformed.limitations[0].sources).toHaveLength(2);
|
|
4118
|
+
expect(malformed.limitations[0].source).toBe("Alert rules");
|
|
4119
|
+
|
|
4120
|
+
const envelope = project([
|
|
4121
|
+
tool(
|
|
4122
|
+
"rules-count",
|
|
4123
|
+
"get_prometheus_rules",
|
|
4124
|
+
{ count: 2, rules: [alertingRule()] },
|
|
4125
|
+
{ summary: rulesArgs },
|
|
4126
|
+
),
|
|
4127
|
+
tool(
|
|
4128
|
+
"rules-truncated",
|
|
4129
|
+
"get_prometheus_rules",
|
|
4130
|
+
{ count: 0, rules: [], truncated: "true" },
|
|
4131
|
+
{ summary: rulesArgs },
|
|
4132
|
+
),
|
|
4133
|
+
tool(
|
|
4134
|
+
"rules-no-query",
|
|
4135
|
+
"get_prometheus_rules",
|
|
4136
|
+
{ count: 1, rules: [alertingRule({ query: undefined })] },
|
|
4137
|
+
{ summary: rulesArgs },
|
|
4138
|
+
),
|
|
4139
|
+
]);
|
|
4140
|
+
expect(envelope.groups).toHaveLength(0);
|
|
4141
|
+
expect(envelope.limitations[0].sources).toHaveLength(3);
|
|
4142
|
+
});
|
|
4143
|
+
});
|
|
4144
|
+
|
|
4145
|
+
const helmRelease = {
|
|
4146
|
+
name: "shop",
|
|
4147
|
+
namespace: "shop",
|
|
4148
|
+
chart: "shop",
|
|
4149
|
+
chartVersion: "1.4.2",
|
|
4150
|
+
appVersion: "2.0.0",
|
|
4151
|
+
status: "deployed",
|
|
4152
|
+
revision: 7,
|
|
4153
|
+
updated: "2026-09-07T07:00:00Z",
|
|
4154
|
+
description: "Upgrade complete",
|
|
4155
|
+
resources: [
|
|
4156
|
+
{
|
|
4157
|
+
kind: "Deployment",
|
|
4158
|
+
apiVersion: "apps/v1",
|
|
4159
|
+
name: "api",
|
|
4160
|
+
namespace: "shop",
|
|
4161
|
+
status: "Running",
|
|
4162
|
+
ready: "0/1",
|
|
4163
|
+
},
|
|
4164
|
+
{
|
|
4165
|
+
kind: "Service",
|
|
4166
|
+
apiVersion: "v1",
|
|
4167
|
+
name: "api",
|
|
4168
|
+
namespace: "shop",
|
|
4169
|
+
status: "Active",
|
|
4170
|
+
},
|
|
4171
|
+
],
|
|
4172
|
+
};
|
|
4173
|
+
|
|
4174
|
+
describe("helm release adapter", () => {
|
|
4175
|
+
it("relates a release through the resources it manages", () => {
|
|
4176
|
+
const projection = project([
|
|
4177
|
+
tool("helm", "get_helm_release", helmRelease, {
|
|
4178
|
+
summary: JSON.stringify({ namespace: "shop", name: "shop" }),
|
|
4179
|
+
}),
|
|
4180
|
+
tool(
|
|
4181
|
+
"helm-other",
|
|
4182
|
+
"get_helm_release",
|
|
4183
|
+
{
|
|
4184
|
+
...helmRelease,
|
|
4185
|
+
name: "ingress",
|
|
4186
|
+
resources: [
|
|
4187
|
+
{
|
|
4188
|
+
kind: "Deployment",
|
|
4189
|
+
apiVersion: "apps/v1",
|
|
4190
|
+
name: "api",
|
|
4191
|
+
namespace: "ingress",
|
|
4192
|
+
},
|
|
4193
|
+
],
|
|
4194
|
+
},
|
|
4195
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "ingress" }) },
|
|
4196
|
+
),
|
|
4197
|
+
]);
|
|
4198
|
+
const releases = groupsOf(projection.groups, "helm");
|
|
4199
|
+
expect(releases.map((group) => group.latest.relevance)).toEqual([
|
|
4200
|
+
"producer-related",
|
|
4201
|
+
"broader",
|
|
4202
|
+
]);
|
|
4203
|
+
expect(releases[0].latest).toMatchObject({
|
|
4204
|
+
tier: "context",
|
|
4205
|
+
tone: "info",
|
|
4206
|
+
title: "Helm release shop/shop",
|
|
4207
|
+
summary: "shop 1.4.2 · deployed · revision 7",
|
|
4208
|
+
});
|
|
4209
|
+
expect(investigationEvidenceSubjectRef(releases[0].latest.data)).toEqual({
|
|
4210
|
+
kind: "HelmRelease",
|
|
4211
|
+
group: "helm.sh",
|
|
4212
|
+
namespace: "shop",
|
|
4213
|
+
name: "shop",
|
|
4214
|
+
});
|
|
4215
|
+
});
|
|
4216
|
+
|
|
4217
|
+
it("never fills a missing owned-resource API version from the target", () => {
|
|
4218
|
+
const projection = project([
|
|
4219
|
+
tool(
|
|
4220
|
+
"helm-no-version",
|
|
4221
|
+
"get_helm_release",
|
|
4222
|
+
{
|
|
4223
|
+
...helmRelease,
|
|
4224
|
+
resources: [{ kind: "Deployment", name: "api", namespace: "shop" }],
|
|
4225
|
+
},
|
|
4226
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4227
|
+
),
|
|
4228
|
+
tool(
|
|
4229
|
+
"helm-other-group",
|
|
4230
|
+
"get_helm_release",
|
|
4231
|
+
{
|
|
4232
|
+
...helmRelease,
|
|
4233
|
+
name: "shop-x",
|
|
4234
|
+
resources: [
|
|
4235
|
+
{
|
|
4236
|
+
kind: "Deployment",
|
|
4237
|
+
apiVersion: "example.io/v1",
|
|
4238
|
+
name: "api",
|
|
4239
|
+
namespace: "shop",
|
|
4240
|
+
},
|
|
4241
|
+
],
|
|
4242
|
+
},
|
|
4243
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop-x" }) },
|
|
4244
|
+
),
|
|
4245
|
+
]);
|
|
4246
|
+
expect(
|
|
4247
|
+
groupsOf(projection.groups, "helm").map(
|
|
4248
|
+
(group) => group.latest.relevance,
|
|
4249
|
+
),
|
|
4250
|
+
).toEqual(["broader", "broader"]);
|
|
4251
|
+
});
|
|
4252
|
+
|
|
4253
|
+
it("keys a release by its storage namespace and withholds a link it cannot express", () => {
|
|
4254
|
+
const projection = project([
|
|
4255
|
+
tool(
|
|
4256
|
+
"helm-a",
|
|
4257
|
+
"get_helm_release",
|
|
4258
|
+
{ ...helmRelease, storageNamespace: "flux-a" },
|
|
4259
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4260
|
+
),
|
|
4261
|
+
tool(
|
|
4262
|
+
"helm-b",
|
|
4263
|
+
"get_helm_release",
|
|
4264
|
+
{ ...helmRelease, storageNamespace: "flux-b", revision: 9 },
|
|
4265
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4266
|
+
),
|
|
4267
|
+
]);
|
|
4268
|
+
const releases = groupsOf(projection.groups, "helm");
|
|
4269
|
+
expect(releases.map((group) => group.identity)).toEqual([
|
|
4270
|
+
"helm:flux-a:shop:shop",
|
|
4271
|
+
"helm:flux-b:shop:shop",
|
|
4272
|
+
]);
|
|
4273
|
+
expect(releases[0].latest.data).toMatchObject({
|
|
4274
|
+
release: { storageNamespace: "flux-a" },
|
|
4275
|
+
});
|
|
4276
|
+
expect(investigationEvidenceSubjectRef(releases[0].latest.data)).toBe(
|
|
4277
|
+
undefined,
|
|
4278
|
+
);
|
|
4279
|
+
});
|
|
4280
|
+
|
|
4281
|
+
it("treats a non-deployed status, a health issue, or a failed operation as adverse", () => {
|
|
4282
|
+
const projection = project([
|
|
4283
|
+
tool(
|
|
4284
|
+
"helm-failed",
|
|
4285
|
+
"get_helm_release",
|
|
4286
|
+
{ ...helmRelease, status: "failed", description: "Upgrade failed" },
|
|
4287
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4288
|
+
),
|
|
4289
|
+
tool(
|
|
4290
|
+
"helm-issue",
|
|
4291
|
+
"get_helm_release",
|
|
4292
|
+
{
|
|
4293
|
+
...helmRelease,
|
|
4294
|
+
name: "shop-cache",
|
|
4295
|
+
resourceHealth: "unhealthy",
|
|
4296
|
+
healthIssue: "Deployment shop/cache has 0/1 ready replicas",
|
|
4297
|
+
healthSummary: "1 of 2 resources unhealthy",
|
|
4298
|
+
},
|
|
4299
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop-cache" }) },
|
|
4300
|
+
),
|
|
4301
|
+
tool(
|
|
4302
|
+
"helm-stuck",
|
|
4303
|
+
"get_helm_release",
|
|
4304
|
+
{
|
|
4305
|
+
...helmRelease,
|
|
4306
|
+
name: "shop-jobs",
|
|
4307
|
+
status: "pending-upgrade",
|
|
4308
|
+
lastOperation: {
|
|
4309
|
+
kind: "pending",
|
|
4310
|
+
status: "stuck_pending",
|
|
4311
|
+
source: "helm_status",
|
|
4312
|
+
confidence: "high",
|
|
4313
|
+
message: "Revision 8 has been pending-upgrade for 42m",
|
|
4314
|
+
revision: 8,
|
|
4315
|
+
},
|
|
4316
|
+
valuesError:
|
|
4317
|
+
'Radar Cloud role "viewer" cannot view Helm release values (requires member or higher)',
|
|
4318
|
+
diffError:
|
|
4319
|
+
'Radar Cloud role "viewer" cannot view Helm release diffs (requires member or higher)',
|
|
4320
|
+
valuesDiffError: "values diff failed",
|
|
4321
|
+
notesDiffError: "notes diff failed",
|
|
4322
|
+
resourceDiffError: "resource diff failed",
|
|
4323
|
+
},
|
|
4324
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop-jobs" }) },
|
|
4325
|
+
),
|
|
4326
|
+
]);
|
|
4327
|
+
const releases = groupsOf(projection.groups, "helm");
|
|
4328
|
+
expect(
|
|
4329
|
+
releases.map((group) => [group.latest.tier, group.latest.tone]),
|
|
4330
|
+
).toEqual([
|
|
4331
|
+
["supporting", "error"],
|
|
4332
|
+
["supporting", "warning"],
|
|
4333
|
+
["supporting", "warning"],
|
|
4334
|
+
]);
|
|
4335
|
+
expect(releases[1].latest.summary).toBe(
|
|
4336
|
+
"deployed · Deployment shop/cache has 0/1 ready replicas",
|
|
4337
|
+
);
|
|
4338
|
+
expect(releases[2].latest.data).toMatchObject({
|
|
4339
|
+
type: "helm",
|
|
4340
|
+
release: {
|
|
4341
|
+
lastOperation: { kind: "pending", status: "stuck_pending" },
|
|
4342
|
+
},
|
|
4343
|
+
});
|
|
4344
|
+
// Every comparison the producer could not finish has to reach the reader:
|
|
4345
|
+
// three of these are role denials, and a release card that omits a withheld
|
|
4346
|
+
// diff without saying so reads as a complete one.
|
|
4347
|
+
expect(projection.limitations).toEqual([
|
|
4348
|
+
expect.objectContaining({ source: "Helm values", kind: "error" }),
|
|
4349
|
+
expect.objectContaining({ source: "Helm values diff", kind: "error" }),
|
|
4350
|
+
expect.objectContaining({ source: "Helm manifest diff", kind: "error" }),
|
|
4351
|
+
expect.objectContaining({ source: "Helm notes diff", kind: "error" }),
|
|
4352
|
+
expect.objectContaining({ source: "Helm resource diff", kind: "error" }),
|
|
4353
|
+
]);
|
|
4354
|
+
});
|
|
4355
|
+
|
|
4356
|
+
it("rejects malformed release payloads", () => {
|
|
4357
|
+
const projection = project([
|
|
4358
|
+
tool(
|
|
4359
|
+
"helm-bad",
|
|
4360
|
+
"get_helm_release",
|
|
4361
|
+
{ ...helmRelease, revision: "7" },
|
|
4362
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4363
|
+
),
|
|
4364
|
+
tool(
|
|
4365
|
+
"helm-bad-resource",
|
|
4366
|
+
"get_helm_release",
|
|
4367
|
+
{ ...helmRelease, resources: [{ kind: "Deployment" }] },
|
|
4368
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4369
|
+
),
|
|
4370
|
+
tool(
|
|
4371
|
+
"helm-bad-operation",
|
|
4372
|
+
"get_helm_release",
|
|
4373
|
+
{ ...helmRelease, lastOperation: { kind: "rollback" } },
|
|
4374
|
+
{ summary: JSON.stringify({ namespace: "shop", name: "shop" }) },
|
|
4375
|
+
),
|
|
4376
|
+
]);
|
|
4377
|
+
expect(projection.groups).toHaveLength(0);
|
|
4378
|
+
expect(
|
|
4379
|
+
projection.limitations.map((limitation) => limitation.source),
|
|
4380
|
+
).toEqual(["Helm release", "Helm resources", "Helm operation"]);
|
|
4381
|
+
});
|
|
4382
|
+
});
|
|
4383
|
+
|
|
4384
|
+
const deploymentWithServiceAccount = {
|
|
4385
|
+
...deployment,
|
|
4386
|
+
spec: { template: { spec: { serviceAccountName: "api-sa" } } },
|
|
4387
|
+
};
|
|
4388
|
+
|
|
4389
|
+
const permissionsArgs = JSON.stringify({
|
|
4390
|
+
kind: "ServiceAccount",
|
|
4391
|
+
namespace: "shop",
|
|
4392
|
+
name: "api-sa",
|
|
4393
|
+
verb: "get",
|
|
4394
|
+
resource: "secrets",
|
|
4395
|
+
});
|
|
4396
|
+
|
|
4397
|
+
describe("subject permissions adapter", () => {
|
|
4398
|
+
it("does not turn an unevaluated access check into a denial", () => {
|
|
4399
|
+
// A webhook authorizer returning partial data has established neither
|
|
4400
|
+
// answer. Reading it as "cannot" sends an operator after an RBAC grant
|
|
4401
|
+
// that may already exist.
|
|
4402
|
+
const projection = project([
|
|
4403
|
+
tool("diagnose", "diagnose", {
|
|
4404
|
+
resource: deploymentWithServiceAccount,
|
|
4405
|
+
resourceContext: { tier: "basic" },
|
|
4406
|
+
}),
|
|
4407
|
+
tool(
|
|
4408
|
+
"perm",
|
|
4409
|
+
"get_subject_permissions",
|
|
4410
|
+
{
|
|
4411
|
+
subject: {
|
|
4412
|
+
kind: "ServiceAccount",
|
|
4413
|
+
namespace: "shop",
|
|
4414
|
+
name: "api-sa",
|
|
4415
|
+
},
|
|
4416
|
+
accessCheck: {
|
|
4417
|
+
verb: "get",
|
|
4418
|
+
resource: "secrets",
|
|
4419
|
+
namespace: "shop",
|
|
4420
|
+
allowed: false,
|
|
4421
|
+
denied: false,
|
|
4422
|
+
reason: "",
|
|
4423
|
+
evaluationError: "webhook authorizer returned partial data",
|
|
4424
|
+
},
|
|
4425
|
+
},
|
|
4426
|
+
{ summary: permissionsArgs },
|
|
4427
|
+
),
|
|
4428
|
+
]);
|
|
4429
|
+
const [group] = groupsOf(projection.groups, "permissions");
|
|
4430
|
+
expect(group.latest.title).toContain("Could not check whether");
|
|
4431
|
+
expect(group.latest.title).not.toContain("cannot get");
|
|
4432
|
+
expect(group.latest.summary).toContain(
|
|
4433
|
+
"webhook authorizer returned partial data",
|
|
4434
|
+
);
|
|
4435
|
+
expect(group.latest.summary).not.toContain("No RBAC rule allows it");
|
|
4436
|
+
// The gap still reaches the coverage strip as well.
|
|
4437
|
+
expect(projection.limitations).toContainEqual(
|
|
4438
|
+
expect.objectContaining({ source: "Permissions", kind: "error" }),
|
|
4439
|
+
);
|
|
4440
|
+
});
|
|
4441
|
+
|
|
4442
|
+
it("keeps a decided verdict that arrived with an evaluation error", () => {
|
|
4443
|
+
// Kubernetes returns the error alongside a real answer too — one webhook
|
|
4444
|
+
// failing while another allows. That answer is still the answer.
|
|
4445
|
+
const projection = project([
|
|
4446
|
+
tool("diagnose", "diagnose", {
|
|
4447
|
+
resource: deploymentWithServiceAccount,
|
|
4448
|
+
resourceContext: { tier: "basic" },
|
|
4449
|
+
}),
|
|
4450
|
+
tool(
|
|
4451
|
+
"perm",
|
|
4452
|
+
"get_subject_permissions",
|
|
4453
|
+
{
|
|
4454
|
+
subject: {
|
|
4455
|
+
kind: "ServiceAccount",
|
|
4456
|
+
namespace: "shop",
|
|
4457
|
+
name: "api-sa",
|
|
4458
|
+
},
|
|
4459
|
+
accessCheck: {
|
|
4460
|
+
verb: "get",
|
|
4461
|
+
resource: "secrets",
|
|
4462
|
+
namespace: "shop",
|
|
4463
|
+
allowed: true,
|
|
4464
|
+
denied: false,
|
|
4465
|
+
reason: "allowed by one authorizer",
|
|
4466
|
+
evaluationError: "a second webhook did not answer",
|
|
4467
|
+
},
|
|
4468
|
+
},
|
|
4469
|
+
{ summary: permissionsArgs },
|
|
4470
|
+
),
|
|
4471
|
+
]);
|
|
4472
|
+
const [group] = groupsOf(projection.groups, "permissions");
|
|
4473
|
+
expect(group.latest.title).toContain("can get");
|
|
4474
|
+
expect(group.latest.title).not.toContain("Could not check");
|
|
4475
|
+
expect(group.latest.tone).toBe("info");
|
|
4476
|
+
// And the error is still reported, just not as the verdict.
|
|
4477
|
+
expect(projection.limitations).toContainEqual(
|
|
4478
|
+
expect.objectContaining({ source: "Permissions", kind: "error" }),
|
|
4479
|
+
);
|
|
4480
|
+
});
|
|
4481
|
+
|
|
4482
|
+
it("binds an access check to the target's ServiceAccount seen in the same turn", () => {
|
|
4483
|
+
const check = {
|
|
4484
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name: "api-sa" },
|
|
4485
|
+
accessCheck: {
|
|
4486
|
+
verb: "get",
|
|
4487
|
+
resource: "secrets",
|
|
4488
|
+
namespace: "shop",
|
|
4489
|
+
allowed: false,
|
|
4490
|
+
reason: "",
|
|
4491
|
+
},
|
|
4492
|
+
};
|
|
4493
|
+
const projection = project([
|
|
4494
|
+
tool("diagnose", "diagnose", {
|
|
4495
|
+
resource: deploymentWithServiceAccount,
|
|
4496
|
+
resourceContext: { tier: "basic" },
|
|
4497
|
+
}),
|
|
4498
|
+
tool("perm", "get_subject_permissions", check, {
|
|
4499
|
+
summary: permissionsArgs,
|
|
4500
|
+
}),
|
|
4501
|
+
tool(
|
|
4502
|
+
"perm-other",
|
|
4503
|
+
"get_subject_permissions",
|
|
4504
|
+
{
|
|
4505
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name: "other" },
|
|
4506
|
+
accessCheck: {
|
|
4507
|
+
verb: "list",
|
|
4508
|
+
group: "apps",
|
|
4509
|
+
resource: "deployments",
|
|
4510
|
+
subresource: "scale",
|
|
4511
|
+
namespace: "",
|
|
4512
|
+
allowed: true,
|
|
4513
|
+
reason: 'RBAC: allowed by ClusterRoleBinding "admin"',
|
|
4514
|
+
},
|
|
4515
|
+
},
|
|
4516
|
+
{
|
|
4517
|
+
summary: JSON.stringify({
|
|
4518
|
+
kind: "ServiceAccount",
|
|
4519
|
+
namespace: "shop",
|
|
4520
|
+
name: "other",
|
|
4521
|
+
}),
|
|
4522
|
+
},
|
|
4523
|
+
),
|
|
4524
|
+
]);
|
|
4525
|
+
const permissions = groupsOf(projection.groups, "permissions");
|
|
4526
|
+
expect(permissions.map((group) => group.latest.relevance)).toEqual([
|
|
4527
|
+
"target",
|
|
4528
|
+
"broader",
|
|
4529
|
+
]);
|
|
4530
|
+
expect(permissions[0].latest).toMatchObject({
|
|
4531
|
+
tier: "supporting",
|
|
4532
|
+
tone: "warning",
|
|
4533
|
+
title: "Service Account shop/api-sa cannot get secrets",
|
|
4534
|
+
summary: "No RBAC rule allows it · namespace shop",
|
|
4535
|
+
});
|
|
4536
|
+
expect(permissions[0].latest.data).toMatchObject({
|
|
4537
|
+
type: "permissions",
|
|
4538
|
+
accessCheck: { allowed: false, denied: false },
|
|
4539
|
+
});
|
|
4540
|
+
expect(permissions[1].latest).toMatchObject({
|
|
4541
|
+
tier: "context",
|
|
4542
|
+
tone: "info",
|
|
4543
|
+
title: "Service Account shop/other can list deployments/scale.apps",
|
|
4544
|
+
summary: 'RBAC: allowed by ClusterRoleBinding "admin" · cluster-wide',
|
|
4545
|
+
});
|
|
4546
|
+
expect(investigationEvidenceSubjectRef(permissions[0].latest.data)).toEqual(
|
|
4547
|
+
{ kind: "ServiceAccount", namespace: "shop", name: "api-sa" },
|
|
4548
|
+
);
|
|
4549
|
+
});
|
|
4550
|
+
|
|
4551
|
+
it("uses the producer's serviceAccount reference and the default account when the template omits one", () => {
|
|
4552
|
+
const withContext = project([
|
|
4553
|
+
tool("diagnose", "diagnose", {
|
|
4554
|
+
resource: deployment,
|
|
4555
|
+
resourceContext: {
|
|
4556
|
+
tier: "diagnostic",
|
|
4557
|
+
uses: {
|
|
4558
|
+
serviceAccount: {
|
|
4559
|
+
kind: "ServiceAccount",
|
|
4560
|
+
namespace: "shop",
|
|
4561
|
+
name: "ctx-sa",
|
|
4562
|
+
},
|
|
4563
|
+
},
|
|
4564
|
+
},
|
|
4565
|
+
}),
|
|
4566
|
+
tool(
|
|
4567
|
+
"perm",
|
|
4568
|
+
"get_subject_permissions",
|
|
4569
|
+
{
|
|
4570
|
+
subject: {
|
|
4571
|
+
kind: "ServiceAccount",
|
|
4572
|
+
namespace: "shop",
|
|
4573
|
+
name: "ctx-sa",
|
|
4574
|
+
},
|
|
4575
|
+
bindings: [],
|
|
4576
|
+
flatRules: [],
|
|
4577
|
+
},
|
|
4578
|
+
{ summary: permissionsArgs },
|
|
4579
|
+
),
|
|
4580
|
+
]);
|
|
4581
|
+
expect(
|
|
4582
|
+
groupsOf(withContext.groups, "permissions")[0].latest.relevance,
|
|
4583
|
+
).toBe("target");
|
|
4584
|
+
|
|
4585
|
+
const withDefault = project([
|
|
4586
|
+
tool("resource", "get_resource", {
|
|
4587
|
+
...deployment,
|
|
4588
|
+
spec: { template: { spec: {} } },
|
|
4589
|
+
}),
|
|
4590
|
+
tool(
|
|
4591
|
+
"perm",
|
|
4592
|
+
"get_subject_permissions",
|
|
4593
|
+
{
|
|
4594
|
+
subject: {
|
|
4595
|
+
kind: "ServiceAccount",
|
|
4596
|
+
namespace: "shop",
|
|
4597
|
+
name: "default",
|
|
4598
|
+
},
|
|
4599
|
+
bindings: [],
|
|
4600
|
+
flatRules: [],
|
|
4601
|
+
},
|
|
4602
|
+
{ summary: permissionsArgs },
|
|
4603
|
+
),
|
|
4604
|
+
]);
|
|
4605
|
+
expect(
|
|
4606
|
+
groupsOf(withDefault.groups, "permissions")[0].latest.relevance,
|
|
4607
|
+
).toBe("target");
|
|
4608
|
+
|
|
4609
|
+
const priorTurn = project(
|
|
4610
|
+
[
|
|
4611
|
+
tool("diagnose", "diagnose", {
|
|
4612
|
+
resource: deploymentWithServiceAccount,
|
|
4613
|
+
resourceContext: { tier: "basic" },
|
|
4614
|
+
}),
|
|
4615
|
+
],
|
|
4616
|
+
[
|
|
4617
|
+
tool(
|
|
4618
|
+
"perm",
|
|
4619
|
+
"get_subject_permissions",
|
|
4620
|
+
{
|
|
4621
|
+
subject: {
|
|
4622
|
+
kind: "ServiceAccount",
|
|
4623
|
+
namespace: "shop",
|
|
4624
|
+
name: "api-sa",
|
|
4625
|
+
},
|
|
4626
|
+
bindings: [],
|
|
4627
|
+
flatRules: [],
|
|
4628
|
+
},
|
|
4629
|
+
{ summary: permissionsArgs },
|
|
4630
|
+
),
|
|
4631
|
+
],
|
|
4632
|
+
);
|
|
4633
|
+
expect(groupsOf(priorTurn.groups, "permissions")[0].latest.relevance).toBe(
|
|
4634
|
+
"broader",
|
|
4635
|
+
);
|
|
4636
|
+
|
|
4637
|
+
// The newest captured read of the target decides which account it runs as.
|
|
4638
|
+
const rotated = project([
|
|
4639
|
+
tool("resource-old", "get_resource", deploymentWithServiceAccount),
|
|
4640
|
+
tool("resource-new", "get_resource", {
|
|
4641
|
+
...deployment,
|
|
4642
|
+
spec: { template: { spec: { serviceAccountName: "api-sa-v2" } } },
|
|
4643
|
+
}),
|
|
4644
|
+
tool(
|
|
4645
|
+
"perm-old",
|
|
4646
|
+
"get_subject_permissions",
|
|
4647
|
+
{
|
|
4648
|
+
subject: {
|
|
4649
|
+
kind: "ServiceAccount",
|
|
4650
|
+
namespace: "shop",
|
|
4651
|
+
name: "api-sa",
|
|
4652
|
+
},
|
|
4653
|
+
bindings: [],
|
|
4654
|
+
flatRules: [],
|
|
4655
|
+
},
|
|
4656
|
+
{ summary: permissionsArgs },
|
|
4657
|
+
),
|
|
4658
|
+
tool(
|
|
4659
|
+
"perm-new",
|
|
4660
|
+
"get_subject_permissions",
|
|
4661
|
+
{
|
|
4662
|
+
subject: {
|
|
4663
|
+
kind: "ServiceAccount",
|
|
4664
|
+
namespace: "shop",
|
|
4665
|
+
name: "api-sa-v2",
|
|
4666
|
+
},
|
|
4667
|
+
bindings: [],
|
|
4668
|
+
flatRules: [],
|
|
4669
|
+
},
|
|
4670
|
+
{ summary: permissionsArgs },
|
|
4671
|
+
),
|
|
4672
|
+
]);
|
|
4673
|
+
expect(
|
|
4674
|
+
groupsOf(rotated.groups, "permissions").map(
|
|
4675
|
+
(group) => group.latest.relevance,
|
|
4676
|
+
),
|
|
4677
|
+
).toEqual(["broader", "target"]);
|
|
4678
|
+
});
|
|
4679
|
+
|
|
4680
|
+
it("reads a PodSpec only from kinds that carry one", () => {
|
|
4681
|
+
const check = (name: string) => ({
|
|
4682
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name },
|
|
4683
|
+
accessCheck: {
|
|
4684
|
+
verb: "get",
|
|
4685
|
+
resource: "secrets",
|
|
4686
|
+
namespace: "shop",
|
|
4687
|
+
allowed: false,
|
|
4688
|
+
},
|
|
4689
|
+
});
|
|
4690
|
+
const applicationSet = project([
|
|
4691
|
+
tool("resource", "get_resource", {
|
|
4692
|
+
apiVersion: "argoproj.io/v1alpha1",
|
|
4693
|
+
kind: "ApplicationSet",
|
|
4694
|
+
metadata: { namespace: "shop", name: "api" },
|
|
4695
|
+
spec: { template: { spec: { project: "default" } } },
|
|
4696
|
+
}),
|
|
4697
|
+
tool("perm", "get_subject_permissions", check("default"), {
|
|
4698
|
+
summary: permissionsArgs,
|
|
4699
|
+
}),
|
|
4700
|
+
]);
|
|
4701
|
+
expect(
|
|
4702
|
+
groupsOf(applicationSet.groups, "permissions")[0].latest.relevance,
|
|
4703
|
+
).toBe("broader");
|
|
4704
|
+
|
|
4705
|
+
const cronJob = projectInvestigationEvidence(
|
|
4706
|
+
[
|
|
4707
|
+
{
|
|
4708
|
+
timeline: [
|
|
4709
|
+
tool("resource", "get_resource", {
|
|
4710
|
+
apiVersion: "batch/v1",
|
|
4711
|
+
kind: "CronJob",
|
|
4712
|
+
metadata: { namespace: "shop", name: "api" },
|
|
4713
|
+
spec: {
|
|
4714
|
+
jobTemplate: {
|
|
4715
|
+
spec: {
|
|
4716
|
+
template: { spec: { serviceAccountName: "cron-sa" } },
|
|
4717
|
+
},
|
|
4718
|
+
},
|
|
4719
|
+
},
|
|
4720
|
+
}),
|
|
4721
|
+
tool("perm", "get_subject_permissions", check("cron-sa"), {
|
|
4722
|
+
summary: permissionsArgs,
|
|
4723
|
+
}),
|
|
4724
|
+
],
|
|
4725
|
+
},
|
|
4726
|
+
],
|
|
4727
|
+
{ kind: "CronJob", group: "batch", namespace: "shop", name: "api" },
|
|
4728
|
+
);
|
|
4729
|
+
expect(groupsOf(cronJob.groups, "permissions")[0].latest.relevance).toBe(
|
|
4730
|
+
"target",
|
|
4731
|
+
);
|
|
4732
|
+
});
|
|
4733
|
+
|
|
4734
|
+
it("keeps a denial for an unrelated principal out of the lead evidence", () => {
|
|
4735
|
+
const projection = project([
|
|
4736
|
+
tool("diagnose", "diagnose", {
|
|
4737
|
+
resource: deploymentWithServiceAccount,
|
|
4738
|
+
resourceContext: { tier: "basic" },
|
|
4739
|
+
}),
|
|
4740
|
+
tool(
|
|
4741
|
+
"perm-cross",
|
|
4742
|
+
"get_subject_permissions",
|
|
4743
|
+
{
|
|
4744
|
+
subject: {
|
|
4745
|
+
kind: "ServiceAccount",
|
|
4746
|
+
namespace: "unrelated",
|
|
4747
|
+
name: "other",
|
|
4748
|
+
},
|
|
4749
|
+
accessCheck: {
|
|
4750
|
+
verb: "get",
|
|
4751
|
+
resource: "secrets",
|
|
4752
|
+
namespace: "unrelated",
|
|
4753
|
+
allowed: false,
|
|
4754
|
+
},
|
|
4755
|
+
},
|
|
4756
|
+
{
|
|
4757
|
+
summary: JSON.stringify({
|
|
4758
|
+
kind: "ServiceAccount",
|
|
4759
|
+
namespace: "unrelated",
|
|
4760
|
+
name: "other",
|
|
4761
|
+
}),
|
|
4762
|
+
},
|
|
4763
|
+
),
|
|
4764
|
+
]);
|
|
4765
|
+
const denial = groupsOf(projection.groups, "permissions")[0].latest;
|
|
4766
|
+
expect(denial.relevance).toBe("broader");
|
|
4767
|
+
expect(denial.tier).toBe("context");
|
|
4768
|
+
expect(denial.tone).toBe("warning");
|
|
4769
|
+
});
|
|
4770
|
+
|
|
4771
|
+
it("renders the subject response as counts and keeps every coverage caveat", () => {
|
|
4772
|
+
const projection = project([
|
|
4773
|
+
tool(
|
|
4774
|
+
"perm",
|
|
4775
|
+
"get_subject_permissions",
|
|
4776
|
+
{
|
|
4777
|
+
subject: {
|
|
4778
|
+
kind: "ServiceAccount",
|
|
4779
|
+
namespace: "shop",
|
|
4780
|
+
name: "api-sa",
|
|
4781
|
+
},
|
|
4782
|
+
bindings: [
|
|
4783
|
+
{
|
|
4784
|
+
bindingKind: "RoleBinding",
|
|
4785
|
+
bindingNamespace: "shop",
|
|
4786
|
+
bindingName: "api-reader",
|
|
4787
|
+
roleKind: "Role",
|
|
4788
|
+
roleNamespace: "shop",
|
|
4789
|
+
roleName: "reader",
|
|
4790
|
+
rulesCount: 3,
|
|
4791
|
+
},
|
|
4792
|
+
{
|
|
4793
|
+
bindingKind: "ClusterRoleBinding",
|
|
4794
|
+
bindingName: "system:basic-user",
|
|
4795
|
+
roleKind: "ClusterRole",
|
|
4796
|
+
roleName: "system:basic-user",
|
|
4797
|
+
rulesCount: 1,
|
|
4798
|
+
inheritedFromGroup: "system:authenticated",
|
|
4799
|
+
},
|
|
4800
|
+
],
|
|
4801
|
+
flatRules: [
|
|
4802
|
+
{ verbs: ["get"], resources: ["pods"] },
|
|
4803
|
+
{ verbs: ["list"], resources: ["pods"] },
|
|
4804
|
+
],
|
|
4805
|
+
truncated: true,
|
|
4806
|
+
usedByPods: ["shop/api-68c7b766dc-fmphn"],
|
|
4807
|
+
podsTotal: 3,
|
|
4808
|
+
narrowHint:
|
|
4809
|
+
"rule list truncated — the subject has more rules than shown; do not treat this list as the subject's complete permissions",
|
|
4810
|
+
},
|
|
4811
|
+
{ summary: permissionsArgs },
|
|
4812
|
+
),
|
|
4813
|
+
tool(
|
|
4814
|
+
"perm-missing",
|
|
4815
|
+
"get_subject_permissions",
|
|
4816
|
+
{
|
|
4817
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name: "ghost" },
|
|
4818
|
+
bindings: [],
|
|
4819
|
+
flatRules: [],
|
|
4820
|
+
subjectWarning:
|
|
4821
|
+
'no ServiceAccount "ghost" exists in namespace "shop" — this empty result reflects a subject that was never found, not an account without permissions. Check the name and namespace.',
|
|
4822
|
+
},
|
|
4823
|
+
{ summary: permissionsArgs },
|
|
4824
|
+
),
|
|
4825
|
+
]);
|
|
4826
|
+
const permissions = groupsOf(projection.groups, "permissions");
|
|
4827
|
+
expect(permissions[0].latest).toMatchObject({
|
|
4828
|
+
tier: "context",
|
|
4829
|
+
tone: "info",
|
|
4830
|
+
relevance: "broader",
|
|
4831
|
+
title: "Permissions of Service Account shop/api-sa",
|
|
4832
|
+
summary: "2 bindings · 2+ effective rules",
|
|
4833
|
+
});
|
|
4834
|
+
expect(permissions[0].latest.data).toMatchObject({
|
|
4835
|
+
type: "permissions",
|
|
4836
|
+
flatRulesCount: 2,
|
|
4837
|
+
truncated: true,
|
|
4838
|
+
usedByPods: ["shop/api-68c7b766dc-fmphn"],
|
|
4839
|
+
podsTotal: 3,
|
|
4840
|
+
});
|
|
4841
|
+
expect(permissions[1].latest.summary).toBe(
|
|
4842
|
+
"0 bindings · 0 effective rules",
|
|
4843
|
+
);
|
|
4844
|
+
expect(
|
|
4845
|
+
projection.limitations.map((limitation) => [
|
|
4846
|
+
limitation.source,
|
|
4847
|
+
limitation.kind,
|
|
4848
|
+
]),
|
|
4849
|
+
).toEqual([
|
|
4850
|
+
["Permissions", "truncated"],
|
|
4851
|
+
["Permissions", "truncated"],
|
|
4852
|
+
["Permissions", "unknown"],
|
|
4853
|
+
]);
|
|
4854
|
+
expect(projection.limitations[0].message).toContain("rule list truncated");
|
|
4855
|
+
expect(projection.limitations[1].message).toBe(
|
|
4856
|
+
"Only 1 of 3 pods running as this subject were listed.",
|
|
4857
|
+
);
|
|
4858
|
+
});
|
|
4859
|
+
|
|
4860
|
+
it("rejects malformed permission payloads", () => {
|
|
4861
|
+
const projection = project([
|
|
4862
|
+
tool(
|
|
4863
|
+
"perm-bad",
|
|
4864
|
+
"get_subject_permissions",
|
|
4865
|
+
{ subject: { kind: "ServiceAccount" }, bindings: [], flatRules: [] },
|
|
4866
|
+
{ summary: permissionsArgs },
|
|
4867
|
+
),
|
|
4868
|
+
tool(
|
|
4869
|
+
"perm-bad-check",
|
|
4870
|
+
"get_subject_permissions",
|
|
4871
|
+
{
|
|
4872
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name: "x" },
|
|
4873
|
+
accessCheck: { verb: "get", resource: "secrets", namespace: "shop" },
|
|
4874
|
+
},
|
|
4875
|
+
{ summary: permissionsArgs },
|
|
4876
|
+
),
|
|
4877
|
+
tool(
|
|
4878
|
+
"perm-bad-binding",
|
|
4879
|
+
"get_subject_permissions",
|
|
4880
|
+
{
|
|
4881
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name: "x" },
|
|
4882
|
+
bindings: [{ bindingKind: "RoleBinding" }],
|
|
4883
|
+
flatRules: [],
|
|
4884
|
+
},
|
|
4885
|
+
{ summary: permissionsArgs },
|
|
4886
|
+
),
|
|
4887
|
+
tool(
|
|
4888
|
+
"perm-bad-rule",
|
|
4889
|
+
"get_subject_permissions",
|
|
4890
|
+
{
|
|
4891
|
+
subject: { kind: "ServiceAccount", namespace: "shop", name: "x" },
|
|
4892
|
+
bindings: [],
|
|
4893
|
+
flatRules: [{ resources: ["pods"] }],
|
|
4894
|
+
},
|
|
4895
|
+
{ summary: permissionsArgs },
|
|
4896
|
+
),
|
|
4897
|
+
]);
|
|
4898
|
+
expect(projection.groups).toHaveLength(0);
|
|
4899
|
+
expect(
|
|
4900
|
+
projection.limitations.map((limitation) => limitation.source),
|
|
4901
|
+
).toEqual(["Permissions", "Access check"]);
|
|
4902
|
+
expect(projection.limitations[0].sources).toHaveLength(3);
|
|
4903
|
+
});
|
|
4904
|
+
});
|
|
4905
|
+
|
|
4906
|
+
describe("card leads and deep-link subjects", () => {
|
|
4907
|
+
it("leads an events card with the newest warning's reason and message", () => {
|
|
4908
|
+
const projection = project([
|
|
4909
|
+
tool(
|
|
4910
|
+
"events",
|
|
4911
|
+
"get_events",
|
|
4912
|
+
{
|
|
4913
|
+
events: [
|
|
4914
|
+
{
|
|
4915
|
+
reason: "Scheduled",
|
|
4916
|
+
message: "Successfully assigned shop/api-abc to node-1",
|
|
4917
|
+
type: "Normal",
|
|
4918
|
+
count: 1,
|
|
4919
|
+
lastTimestamp: "2026-09-02T11:00:00Z",
|
|
4920
|
+
},
|
|
4921
|
+
warningEvent,
|
|
4922
|
+
{
|
|
4923
|
+
reason: "Unhealthy",
|
|
4924
|
+
message: "Readiness probe failed: connection refused",
|
|
4925
|
+
type: "Warning",
|
|
4926
|
+
count: 12,
|
|
4927
|
+
lastTimestamp: "2026-09-02T10:30:00Z",
|
|
4928
|
+
},
|
|
4929
|
+
],
|
|
4930
|
+
},
|
|
4931
|
+
{
|
|
4932
|
+
summary: JSON.stringify({
|
|
4933
|
+
kind: "Deployment",
|
|
4934
|
+
namespace: "shop",
|
|
4935
|
+
name: "api",
|
|
4936
|
+
}),
|
|
4937
|
+
},
|
|
4938
|
+
),
|
|
4939
|
+
]);
|
|
4940
|
+
expect(groupsOf(projection.groups, "events")[0].latest.summary).toBe(
|
|
4941
|
+
"Unhealthy: Readiness probe failed: connection refused · 3 event groups · Deployment shop/api",
|
|
4942
|
+
);
|
|
4943
|
+
});
|
|
4944
|
+
|
|
4945
|
+
it("carries the producer's change subject so a card can open its Timeline", () => {
|
|
4946
|
+
const change = {
|
|
4947
|
+
kind: "Deployment",
|
|
4948
|
+
apiVersion: "apps/v1",
|
|
4949
|
+
namespace: "shop",
|
|
4950
|
+
name: "api",
|
|
4951
|
+
changeType: "update",
|
|
4952
|
+
timestamp: "2026-09-02T09:00:00Z",
|
|
4953
|
+
};
|
|
4954
|
+
const projection = project([
|
|
4955
|
+
tool(
|
|
4956
|
+
"changes",
|
|
4957
|
+
"get_changes",
|
|
4958
|
+
{ changes: [change] },
|
|
4959
|
+
{
|
|
4960
|
+
summary: JSON.stringify({
|
|
4961
|
+
kind: "Deployment",
|
|
4962
|
+
namespace: "shop",
|
|
4963
|
+
name: "api",
|
|
4964
|
+
}),
|
|
4965
|
+
},
|
|
4966
|
+
),
|
|
4967
|
+
tool(
|
|
4968
|
+
"changes-ns",
|
|
4969
|
+
"get_changes",
|
|
4970
|
+
{ changes: [change] },
|
|
4971
|
+
{ summary: JSON.stringify({ namespace: "shop" }) },
|
|
4972
|
+
),
|
|
4973
|
+
tool("diagnose", "diagnose", {
|
|
4974
|
+
resource: deployment,
|
|
4975
|
+
resourceContext: { tier: "basic" },
|
|
4976
|
+
recentChanges: [change],
|
|
4977
|
+
}),
|
|
4978
|
+
tool("resource", "get_resource", {
|
|
4979
|
+
resource: {
|
|
4980
|
+
...deployment,
|
|
4981
|
+
metadata: { namespace: "shop", name: "web" },
|
|
4982
|
+
},
|
|
4983
|
+
resourceContext: { tier: "basic" },
|
|
4984
|
+
recentChanges: [change],
|
|
4985
|
+
recentChangesSaturated: false,
|
|
4986
|
+
recentChangesCoverageLimited: false,
|
|
4987
|
+
}),
|
|
4988
|
+
]);
|
|
4989
|
+
const changes = groupsOf(projection.groups, "changes").map(
|
|
4990
|
+
(group) => (group.latest.data as { subject?: unknown }).subject,
|
|
4991
|
+
);
|
|
4992
|
+
expect(changes).toEqual([
|
|
4993
|
+
{ kind: "Deployment", namespace: "shop", name: "api" },
|
|
4994
|
+
undefined,
|
|
4995
|
+
{ kind: "Deployment", namespace: "shop", name: "api" },
|
|
4996
|
+
{ kind: "Deployment", namespace: "shop", name: "web" },
|
|
4997
|
+
]);
|
|
4998
|
+
});
|
|
4999
|
+
});
|
|
5000
|
+
|
|
5001
|
+
describe("query_prometheus evidence", () => {
|
|
5002
|
+
const rangeSeries = [
|
|
5003
|
+
{
|
|
5004
|
+
labels: { pod: "api-7f6-abc" },
|
|
5005
|
+
dataPoints: [
|
|
5006
|
+
{ timestamp: 1_788_679_043, value: 169_377_792 },
|
|
5007
|
+
{ timestamp: 1_788_679_068, value: 171_401_216 },
|
|
5008
|
+
],
|
|
5009
|
+
},
|
|
5010
|
+
];
|
|
5011
|
+
// The pods a diagnose bundle established as the target's own, so an agent
|
|
5012
|
+
// query naming exactly them is target evidence. A prefix over the same
|
|
5013
|
+
// namespace is not: "api-.*" also selects api-worker's pods.
|
|
5014
|
+
const establishedPods = ["api-7f6-abc", "api-7f6-def"];
|
|
5015
|
+
const podSetPattern = `^(${establishedPods.join("|")})$`;
|
|
5016
|
+
const targetSelectors = [
|
|
5017
|
+
{
|
|
5018
|
+
metric: "container_memory_working_set_bytes",
|
|
5019
|
+
matchers: [
|
|
5020
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5021
|
+
{ label: "pod", op: "=~", value: podSetPattern },
|
|
5022
|
+
],
|
|
5023
|
+
},
|
|
5024
|
+
];
|
|
5025
|
+
const prefixSelectors = [
|
|
5026
|
+
{
|
|
5027
|
+
metric: "container_memory_working_set_bytes",
|
|
5028
|
+
matchers: [
|
|
5029
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5030
|
+
{ label: "pod", op: "=~", value: "api-.*" },
|
|
5031
|
+
],
|
|
5032
|
+
},
|
|
5033
|
+
];
|
|
5034
|
+
// A diagnose of the target that lists its pods, so a later query can be
|
|
5035
|
+
// proved to be about them.
|
|
5036
|
+
function podsBundle() {
|
|
5037
|
+
return tool(
|
|
5038
|
+
"diag-pods",
|
|
5039
|
+
"diagnose",
|
|
5040
|
+
{ resource: deployment, pods: 2, podNames: establishedPods },
|
|
5041
|
+
{
|
|
5042
|
+
summary: JSON.stringify({
|
|
5043
|
+
kind: "Deployment",
|
|
5044
|
+
namespace: "shop",
|
|
5045
|
+
name: "api",
|
|
5046
|
+
}),
|
|
5047
|
+
},
|
|
5048
|
+
);
|
|
5049
|
+
}
|
|
5050
|
+
function promResult(patch: Record<string, unknown> = {}) {
|
|
5051
|
+
return {
|
|
5052
|
+
query: `sum(container_memory_working_set_bytes{namespace="shop",pod=~"${podSetPattern}"})`,
|
|
5053
|
+
type: "range",
|
|
5054
|
+
start: "2026-09-06T07:17:23Z",
|
|
5055
|
+
end: "2026-09-06T09:17:23Z",
|
|
5056
|
+
step: "25s",
|
|
5057
|
+
resultType: "matrix",
|
|
5058
|
+
seriesCount: 1,
|
|
5059
|
+
series: rangeSeries,
|
|
5060
|
+
selectors: targetSelectors,
|
|
5061
|
+
...patch,
|
|
5062
|
+
};
|
|
5063
|
+
}
|
|
5064
|
+
|
|
5065
|
+
it("keeps an unrecognised metric's own name as the title", () => {
|
|
5066
|
+
const projection = project([
|
|
5067
|
+
tool(
|
|
5068
|
+
"prom",
|
|
5069
|
+
"query_prometheus",
|
|
5070
|
+
promResult({
|
|
5071
|
+
query: 'sum(acme_widget_queue_depth{namespace="shop"})',
|
|
5072
|
+
selectors: [
|
|
5073
|
+
{
|
|
5074
|
+
metric: "acme_widget_queue_depth",
|
|
5075
|
+
matchers: [{ label: "namespace", op: "=", value: "shop" }],
|
|
5076
|
+
},
|
|
5077
|
+
],
|
|
5078
|
+
}),
|
|
5079
|
+
),
|
|
5080
|
+
]);
|
|
5081
|
+
const [group] = groupsOf(projection.groups, "metrics");
|
|
5082
|
+
expect(group.latest.title).toBe("acme_widget_queue_depth");
|
|
5083
|
+
// No family label means the summary keeps its generic lead-in rather than
|
|
5084
|
+
// repeating the name already in the title.
|
|
5085
|
+
expect(group.latest.summary).toContain("Prometheus · 1 series");
|
|
5086
|
+
});
|
|
5087
|
+
|
|
5088
|
+
it("keeps the generic title when the selectors name no metric", () => {
|
|
5089
|
+
const projection = project([
|
|
5090
|
+
tool(
|
|
5091
|
+
"prom",
|
|
5092
|
+
"query_prometheus",
|
|
5093
|
+
promResult({
|
|
5094
|
+
query: 'sum({namespace="shop"})',
|
|
5095
|
+
selectors: [
|
|
5096
|
+
{
|
|
5097
|
+
metric: "",
|
|
5098
|
+
matchers: [{ label: "namespace", op: "=", value: "shop" }],
|
|
5099
|
+
},
|
|
5100
|
+
],
|
|
5101
|
+
}),
|
|
5102
|
+
),
|
|
5103
|
+
]);
|
|
5104
|
+
const [group] = groupsOf(projection.groups, "metrics");
|
|
5105
|
+
expect(group.latest.title).toBe("Prometheus metrics");
|
|
5106
|
+
expect(group.latest.summary).toBe("1 series · 2h window · 25s step");
|
|
5107
|
+
});
|
|
5108
|
+
|
|
5109
|
+
it("charts a range query scoped to the target as target evidence", () => {
|
|
5110
|
+
const projection = project([
|
|
5111
|
+
podsBundle(),
|
|
5112
|
+
tool("prom", "query_prometheus", promResult(), {
|
|
5113
|
+
summary: JSON.stringify({ query: "sum(...)", type: "range" }),
|
|
5114
|
+
}),
|
|
5115
|
+
]);
|
|
5116
|
+
const [group] = groupsOf(projection.groups, "metrics");
|
|
5117
|
+
expect(group).toBeDefined();
|
|
5118
|
+
expect(group.latest.relevance).toBe("target");
|
|
5119
|
+
expect(group.latest.tier).toBe("supporting");
|
|
5120
|
+
expect(group.latest.tone).toBe("neutral");
|
|
5121
|
+
expect(group.latest.title).toBe("Memory working set · Deployment shop/api");
|
|
5122
|
+
expect(group.latest.summary).toBe(
|
|
5123
|
+
"container_memory_working_set_bytes · 1 series · 2h window · 25s step",
|
|
5124
|
+
);
|
|
5125
|
+
const data = group.latest.data;
|
|
5126
|
+
if (data.type !== "metrics") throw new Error("expected metrics");
|
|
5127
|
+
expect(data.mode).toBe("range");
|
|
5128
|
+
expect(data.origin).toBe("query");
|
|
5129
|
+
expect(data.subject).toEqual({
|
|
5130
|
+
kind: "Deployment",
|
|
5131
|
+
group: "apps",
|
|
5132
|
+
namespace: "shop",
|
|
5133
|
+
name: "api",
|
|
5134
|
+
});
|
|
5135
|
+
expect(data.unit).toBe("bytes");
|
|
5136
|
+
expect(data.series).toEqual(rangeSeries);
|
|
5137
|
+
expect(investigationEvidenceSubjectRef(data)).toEqual(data.subject);
|
|
5138
|
+
// The chart and the diagnose bundle that established its pods.
|
|
5139
|
+
expect(projection.coverage.projected).toBe(2);
|
|
5140
|
+
});
|
|
5141
|
+
|
|
5142
|
+
it("claims a unit only for a single bare metric that states one", () => {
|
|
5143
|
+
const projection = project([
|
|
5144
|
+
tool(
|
|
5145
|
+
"bare",
|
|
5146
|
+
"query_prometheus",
|
|
5147
|
+
promResult({
|
|
5148
|
+
query:
|
|
5149
|
+
'container_memory_working_set_bytes{namespace="shop",pod=~"api-.*"}',
|
|
5150
|
+
}),
|
|
5151
|
+
),
|
|
5152
|
+
tool(
|
|
5153
|
+
"rate",
|
|
5154
|
+
"query_prometheus",
|
|
5155
|
+
promResult({
|
|
5156
|
+
query:
|
|
5157
|
+
'rate(container_cpu_usage_seconds_total{namespace="shop",pod=~"api-.*"}[5m])',
|
|
5158
|
+
selectors: [
|
|
5159
|
+
{
|
|
5160
|
+
metric: "container_cpu_usage_seconds_total",
|
|
5161
|
+
matchers: targetSelectors[0].matchers,
|
|
5162
|
+
},
|
|
5163
|
+
],
|
|
5164
|
+
}),
|
|
5165
|
+
),
|
|
5166
|
+
]);
|
|
5167
|
+
const units = groupsOf(projection.groups, "metrics").map((group) =>
|
|
5168
|
+
group.latest.data.type === "metrics" ? group.latest.data.unit : "?",
|
|
5169
|
+
);
|
|
5170
|
+
expect(units).toEqual(["bytes", ""]);
|
|
5171
|
+
});
|
|
5172
|
+
|
|
5173
|
+
it("renders an instant query as values, not a chart", () => {
|
|
5174
|
+
const projection = project([
|
|
5175
|
+
tool(
|
|
5176
|
+
"instant",
|
|
5177
|
+
"query_prometheus",
|
|
5178
|
+
promResult({
|
|
5179
|
+
type: "instant",
|
|
5180
|
+
start: undefined,
|
|
5181
|
+
end: undefined,
|
|
5182
|
+
step: undefined,
|
|
5183
|
+
resultType: "vector",
|
|
5184
|
+
series: [
|
|
5185
|
+
{
|
|
5186
|
+
labels: { pod: "api-7f6-abc" },
|
|
5187
|
+
dataPoints: [{ timestamp: 1_788_679_068, value: 3 }],
|
|
5188
|
+
},
|
|
5189
|
+
],
|
|
5190
|
+
}),
|
|
5191
|
+
),
|
|
5192
|
+
]);
|
|
5193
|
+
const [group] = groupsOf(projection.groups, "metrics");
|
|
5194
|
+
expect(group.latest.title).toBe("Memory working set");
|
|
5195
|
+
expect(group.latest.summary).toBe(
|
|
5196
|
+
"container_memory_working_set_bytes · 1 series",
|
|
5197
|
+
);
|
|
5198
|
+
expect(group.latest.data.type === "metrics" && group.latest.data.mode).toBe(
|
|
5199
|
+
"instant",
|
|
5200
|
+
);
|
|
5201
|
+
});
|
|
5202
|
+
|
|
5203
|
+
it("reports a truncated result as a limitation with its cardinality, never as a chart", () => {
|
|
5204
|
+
const projection = project([
|
|
5205
|
+
tool(
|
|
5206
|
+
"big",
|
|
5207
|
+
"query_prometheus",
|
|
5208
|
+
promResult({
|
|
5209
|
+
series: [],
|
|
5210
|
+
truncated: true,
|
|
5211
|
+
summary: {
|
|
5212
|
+
seriesCount: 812,
|
|
5213
|
+
totalDataPoints: 194_880,
|
|
5214
|
+
labelCardinality: { pod: 812, container: 3 },
|
|
5215
|
+
suggestion: "topk(5, ...)",
|
|
5216
|
+
},
|
|
5217
|
+
note: "Result exceeded the 96 KiB cap.",
|
|
5218
|
+
}),
|
|
5219
|
+
),
|
|
5220
|
+
]);
|
|
5221
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
|
|
5222
|
+
expect(projection.limitations).toHaveLength(1);
|
|
5223
|
+
expect(projection.limitations[0].kind).toBe("truncated");
|
|
5224
|
+
expect(projection.limitations[0].message).toContain("812 series");
|
|
5225
|
+
expect(projection.limitations[0].message).toContain(
|
|
5226
|
+
"812 distinct pod values",
|
|
5227
|
+
);
|
|
5228
|
+
expect(projection.limitations[0].message).toContain("Result exceeded");
|
|
5229
|
+
expect(projection.coverage.limited).toBe(1);
|
|
5230
|
+
});
|
|
5231
|
+
|
|
5232
|
+
it("keeps a query that matched nothing as a fact about the window", () => {
|
|
5233
|
+
const projection = project([
|
|
5234
|
+
tool("none", "query_prometheus", promResult({ series: [] })),
|
|
5235
|
+
]);
|
|
5236
|
+
const [group] = groupsOf(projection.groups, "metrics");
|
|
5237
|
+
expect(group.latest.summary).toBe(
|
|
5238
|
+
"container_memory_working_set_bytes · No series matched · 2h window · 25s step",
|
|
5239
|
+
);
|
|
5240
|
+
});
|
|
5241
|
+
|
|
5242
|
+
it("classifies scope from the producer's selectors, never from the expression text", () => {
|
|
5243
|
+
const withNamespace = (
|
|
5244
|
+
metric: string,
|
|
5245
|
+
extra: Array<{ label: string; op: string; value: string }> = [],
|
|
5246
|
+
) => ({
|
|
5247
|
+
metric,
|
|
5248
|
+
matchers: [{ label: "namespace", op: "=", value: "shop" }, ...extra],
|
|
5249
|
+
});
|
|
5250
|
+
const cases: Array<[string, Record<string, unknown>, string]> = [
|
|
5251
|
+
["namespace only", { selectors: [withNamespace("up")] }, "broader"],
|
|
5252
|
+
[
|
|
5253
|
+
"workload label",
|
|
5254
|
+
{
|
|
5255
|
+
selectors: [
|
|
5256
|
+
withNamespace("kube_deployment_status_replicas_available", [
|
|
5257
|
+
{ label: "deployment", op: "=", value: "api" },
|
|
5258
|
+
]),
|
|
5259
|
+
],
|
|
5260
|
+
},
|
|
5261
|
+
"target",
|
|
5262
|
+
],
|
|
5263
|
+
[
|
|
5264
|
+
"pod named by the workload's prefix, never established",
|
|
5265
|
+
{
|
|
5266
|
+
selectors: [
|
|
5267
|
+
withNamespace("kube_pod_container_status_restarts_total", [
|
|
5268
|
+
{ label: "pod", op: "=", value: "api-7f6-abc" },
|
|
5269
|
+
]),
|
|
5270
|
+
],
|
|
5271
|
+
},
|
|
5272
|
+
"producer-related",
|
|
5273
|
+
],
|
|
5274
|
+
[
|
|
5275
|
+
"owner join over the namespace",
|
|
5276
|
+
{
|
|
5277
|
+
selectors: [
|
|
5278
|
+
withNamespace("kube_pod_owner", [
|
|
5279
|
+
{ label: "owner_kind", op: "=", value: "ReplicaSet" },
|
|
5280
|
+
{ label: "owner_name", op: "=~", value: "^(api-7f6)$" },
|
|
5281
|
+
]),
|
|
5282
|
+
],
|
|
5283
|
+
},
|
|
5284
|
+
"producer-related",
|
|
5285
|
+
],
|
|
5286
|
+
[
|
|
5287
|
+
"container alone",
|
|
5288
|
+
{
|
|
5289
|
+
selectors: [
|
|
5290
|
+
withNamespace("container_memory_working_set_bytes", [
|
|
5291
|
+
{ label: "container", op: "=", value: "api" },
|
|
5292
|
+
]),
|
|
5293
|
+
],
|
|
5294
|
+
},
|
|
5295
|
+
"broader",
|
|
5296
|
+
],
|
|
5297
|
+
[
|
|
5298
|
+
"sibling workload",
|
|
5299
|
+
{
|
|
5300
|
+
selectors: [
|
|
5301
|
+
withNamespace("kube_deployment_status_replicas_available", [
|
|
5302
|
+
{ label: "deployment", op: "=", value: "worker" },
|
|
5303
|
+
]),
|
|
5304
|
+
],
|
|
5305
|
+
},
|
|
5306
|
+
"producer-related",
|
|
5307
|
+
],
|
|
5308
|
+
[
|
|
5309
|
+
"pod regex with an optional dash",
|
|
5310
|
+
{
|
|
5311
|
+
selectors: [
|
|
5312
|
+
withNamespace("kube_pod_container_status_restarts_total", [
|
|
5313
|
+
{ label: "pod", op: "=~", value: "api-?.*" },
|
|
5314
|
+
]),
|
|
5315
|
+
],
|
|
5316
|
+
},
|
|
5317
|
+
"producer-related",
|
|
5318
|
+
],
|
|
5319
|
+
[
|
|
5320
|
+
"pod regex alternation behind the target prefix",
|
|
5321
|
+
{
|
|
5322
|
+
selectors: [
|
|
5323
|
+
withNamespace("kube_pod_container_status_restarts_total", [
|
|
5324
|
+
{ label: "pod", op: "=~", value: "api-.*|worker-.*" },
|
|
5325
|
+
]),
|
|
5326
|
+
],
|
|
5327
|
+
},
|
|
5328
|
+
"producer-related",
|
|
5329
|
+
],
|
|
5330
|
+
[
|
|
5331
|
+
"workload alternation",
|
|
5332
|
+
{
|
|
5333
|
+
selectors: [
|
|
5334
|
+
withNamespace("kube_deployment_status_replicas_available", [
|
|
5335
|
+
{ label: "deployment", op: "=~", value: "api|worker" },
|
|
5336
|
+
]),
|
|
5337
|
+
],
|
|
5338
|
+
},
|
|
5339
|
+
"producer-related",
|
|
5340
|
+
],
|
|
5341
|
+
[
|
|
5342
|
+
"statefulset label on a deployment target",
|
|
5343
|
+
{
|
|
5344
|
+
selectors: [
|
|
5345
|
+
withNamespace("kube_statefulset_status_replicas_ready", [
|
|
5346
|
+
{ label: "statefulset", op: "=", value: "api" },
|
|
5347
|
+
]),
|
|
5348
|
+
],
|
|
5349
|
+
},
|
|
5350
|
+
"producer-related",
|
|
5351
|
+
],
|
|
5352
|
+
[
|
|
5353
|
+
"braced target numerator over a bare cluster denominator",
|
|
5354
|
+
{
|
|
5355
|
+
query:
|
|
5356
|
+
'sum(container_memory_working_set_bytes{namespace="shop",pod=~"api-.*"}) / scalar(sum(machine_memory_bytes))',
|
|
5357
|
+
selectors: [
|
|
5358
|
+
...targetSelectors,
|
|
5359
|
+
{ metric: "machine_memory_bytes", matchers: [] },
|
|
5360
|
+
],
|
|
5361
|
+
},
|
|
5362
|
+
"broader",
|
|
5363
|
+
],
|
|
5364
|
+
[
|
|
5365
|
+
"other namespace",
|
|
5366
|
+
{
|
|
5367
|
+
selectors: [
|
|
5368
|
+
{
|
|
5369
|
+
metric: "up",
|
|
5370
|
+
matchers: [
|
|
5371
|
+
{ label: "namespace", op: "=", value: "other" },
|
|
5372
|
+
{ label: "pod", op: "=~", value: "api-.*" },
|
|
5373
|
+
],
|
|
5374
|
+
},
|
|
5375
|
+
],
|
|
5376
|
+
},
|
|
5377
|
+
"broader",
|
|
5378
|
+
],
|
|
5379
|
+
["empty selectors", { selectors: [] }, "broader"],
|
|
5380
|
+
[
|
|
5381
|
+
"unknown selectors",
|
|
5382
|
+
{ selectors: targetSelectors, selectorsUnknown: true },
|
|
5383
|
+
"broader",
|
|
5384
|
+
],
|
|
5385
|
+
];
|
|
5386
|
+
for (const [name, patch, expected] of cases) {
|
|
5387
|
+
const projection = project([
|
|
5388
|
+
tool(`case-${name}`, "query_prometheus", promResult(patch)),
|
|
5389
|
+
]);
|
|
5390
|
+
const [group] = groupsOf(projection.groups, "metrics");
|
|
5391
|
+
expect(group?.latest.relevance, name).toBe(expected);
|
|
5392
|
+
expect(
|
|
5393
|
+
group?.latest.data.type === "metrics"
|
|
5394
|
+
? group.latest.data.subject
|
|
5395
|
+
: "missing",
|
|
5396
|
+
name,
|
|
5397
|
+
).toEqual(
|
|
5398
|
+
expected === "target"
|
|
5399
|
+
? {
|
|
5400
|
+
kind: "Deployment",
|
|
5401
|
+
group: "apps",
|
|
5402
|
+
namespace: "shop",
|
|
5403
|
+
name: "api",
|
|
5404
|
+
}
|
|
5405
|
+
: undefined,
|
|
5406
|
+
);
|
|
5407
|
+
expect(group?.latest.tier, name).toBe(
|
|
5408
|
+
expected === "broader" ? "context" : "supporting",
|
|
5409
|
+
);
|
|
5410
|
+
}
|
|
5411
|
+
});
|
|
5412
|
+
|
|
5413
|
+
// The diagnose prompt tells the agent to scope a query as `pod=~'^(a|b)$'`.
|
|
5414
|
+
// A Pod target has to recognise its own name written that way, or following
|
|
5415
|
+
// our own instruction demotes the target's own series.
|
|
5416
|
+
it("accepts the anchored set form the prompt asks for on a Pod target", () => {
|
|
5417
|
+
const podTarget = {
|
|
5418
|
+
kind: "Pod",
|
|
5419
|
+
group: "",
|
|
5420
|
+
namespace: "shop",
|
|
5421
|
+
name: "api-7f6-abc",
|
|
5422
|
+
};
|
|
5423
|
+
const podSelector = (value: string) => [
|
|
5424
|
+
{
|
|
5425
|
+
metric: "container_cpu_usage_seconds_total",
|
|
5426
|
+
matchers: [
|
|
5427
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5428
|
+
{ label: "pod", op: "=~", value },
|
|
5429
|
+
],
|
|
5430
|
+
},
|
|
5431
|
+
];
|
|
5432
|
+
expect(metricsScope(podTarget, podSelector("api-7f6-abc"), false)).toBe(
|
|
5433
|
+
"target",
|
|
5434
|
+
);
|
|
5435
|
+
expect(metricsScope(podTarget, podSelector("^(api-7f6-abc)$"), false)).toBe(
|
|
5436
|
+
"target",
|
|
5437
|
+
);
|
|
5438
|
+
// A set that also names a neighbour is not evidence about this pod alone,
|
|
5439
|
+
// and a prefix still selects pods this one does not stand for.
|
|
5440
|
+
expect(
|
|
5441
|
+
metricsScope(
|
|
5442
|
+
podTarget,
|
|
5443
|
+
podSelector("^(api-7f6-abc|api-7f6-def)$"),
|
|
5444
|
+
false,
|
|
5445
|
+
),
|
|
5446
|
+
).toBe("producer-related");
|
|
5447
|
+
expect(metricsScope(podTarget, podSelector("api-7f6-.*"), false)).toBe(
|
|
5448
|
+
"producer-related",
|
|
5449
|
+
);
|
|
5450
|
+
});
|
|
5451
|
+
|
|
5452
|
+
it("treats an unescaped dot in a workload regex as naming more than the target", () => {
|
|
5453
|
+
const dotted = { ...target, name: "api.v2" };
|
|
5454
|
+
const selector = (value: string) => [
|
|
5455
|
+
{
|
|
5456
|
+
metric: "kube_deployment_status_replicas_available",
|
|
5457
|
+
matchers: [
|
|
5458
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5459
|
+
{ label: "deployment", op: "=~", value },
|
|
5460
|
+
],
|
|
5461
|
+
},
|
|
5462
|
+
];
|
|
5463
|
+
expect(metricsScope(dotted, selector("api.v2"), false)).toBe(
|
|
5464
|
+
"producer-related",
|
|
5465
|
+
);
|
|
5466
|
+
expect(metricsScope(dotted, selector("api\\.v2"), false)).toBe("target");
|
|
5467
|
+
expect(
|
|
5468
|
+
metricsScope(
|
|
5469
|
+
dotted,
|
|
5470
|
+
[
|
|
5471
|
+
{
|
|
5472
|
+
metric: "up",
|
|
5473
|
+
matchers: [
|
|
5474
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5475
|
+
{ label: "pod", op: "=~", value: "api\\.v2-.*" },
|
|
5476
|
+
],
|
|
5477
|
+
},
|
|
5478
|
+
],
|
|
5479
|
+
false,
|
|
5480
|
+
),
|
|
5481
|
+
).toBe("producer-related");
|
|
5482
|
+
expect(
|
|
5483
|
+
metricsScope(
|
|
5484
|
+
dotted,
|
|
5485
|
+
[
|
|
5486
|
+
{
|
|
5487
|
+
metric: "up",
|
|
5488
|
+
matchers: [
|
|
5489
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5490
|
+
{ label: "pod", op: "=~", value: "api.v2-.*" },
|
|
5491
|
+
],
|
|
5492
|
+
},
|
|
5493
|
+
],
|
|
5494
|
+
false,
|
|
5495
|
+
),
|
|
5496
|
+
).toBe("producer-related");
|
|
5497
|
+
});
|
|
5498
|
+
|
|
5499
|
+
it("names a Pod target only by its exact name and accepts an anchored namespace regex", () => {
|
|
5500
|
+
const pod = { ...target, kind: "Pod", group: "", name: "api-7f6-abc" };
|
|
5501
|
+
const selector = (
|
|
5502
|
+
namespace: { op: string; value: string },
|
|
5503
|
+
podMatcher: { op: string; value: string },
|
|
5504
|
+
) => [
|
|
5505
|
+
{
|
|
5506
|
+
metric: "container_memory_working_set_bytes",
|
|
5507
|
+
matchers: [
|
|
5508
|
+
{ label: "namespace", ...namespace },
|
|
5509
|
+
{ label: "pod", ...podMatcher },
|
|
5510
|
+
],
|
|
5511
|
+
},
|
|
5512
|
+
];
|
|
5513
|
+
const exactNs = { op: "=", value: "shop" };
|
|
5514
|
+
expect(
|
|
5515
|
+
metricsScope(
|
|
5516
|
+
pod,
|
|
5517
|
+
selector(exactNs, { op: "=~", value: "api-7f6-abc-.*" }),
|
|
5518
|
+
false,
|
|
5519
|
+
),
|
|
5520
|
+
).toBe("producer-related");
|
|
5521
|
+
expect(
|
|
5522
|
+
metricsScope(
|
|
5523
|
+
pod,
|
|
5524
|
+
selector(exactNs, { op: "=", value: "api-7f6-abc" }),
|
|
5525
|
+
false,
|
|
5526
|
+
),
|
|
5527
|
+
).toBe("target");
|
|
5528
|
+
expect(
|
|
5529
|
+
metricsScope(
|
|
5530
|
+
pod,
|
|
5531
|
+
selector(exactNs, { op: "=~", value: "api-7f6-abc" }),
|
|
5532
|
+
false,
|
|
5533
|
+
),
|
|
5534
|
+
).toBe("target");
|
|
5535
|
+
expect(
|
|
5536
|
+
metricsScope(
|
|
5537
|
+
pod,
|
|
5538
|
+
selector(exactNs, { op: "=", value: "api-7f6-abc-extra" }),
|
|
5539
|
+
false,
|
|
5540
|
+
),
|
|
5541
|
+
).toBe("producer-related");
|
|
5542
|
+
expect(
|
|
5543
|
+
metricsScope(
|
|
5544
|
+
target,
|
|
5545
|
+
selector({ op: "=~", value: "shop" }, { op: "=~", value: "api-.*" }),
|
|
5546
|
+
false,
|
|
5547
|
+
),
|
|
5548
|
+
).toBe("producer-related");
|
|
5549
|
+
expect(
|
|
5550
|
+
metricsScope(
|
|
5551
|
+
target,
|
|
5552
|
+
selector(
|
|
5553
|
+
{ op: "=~", value: "shop" },
|
|
5554
|
+
{
|
|
5555
|
+
op: "=~",
|
|
5556
|
+
value: "^(api-7f6-abc)$",
|
|
5557
|
+
},
|
|
5558
|
+
),
|
|
5559
|
+
false,
|
|
5560
|
+
new Set(["api-7f6-abc"]),
|
|
5561
|
+
),
|
|
5562
|
+
).toBe("target");
|
|
5563
|
+
expect(
|
|
5564
|
+
metricsScope(
|
|
5565
|
+
target,
|
|
5566
|
+
selector(
|
|
5567
|
+
{ op: "=~", value: "shop|other" },
|
|
5568
|
+
{ op: "=~", value: "api-.*" },
|
|
5569
|
+
),
|
|
5570
|
+
false,
|
|
5571
|
+
),
|
|
5572
|
+
).toBe("broader");
|
|
5573
|
+
expect(
|
|
5574
|
+
metricsScope(
|
|
5575
|
+
target,
|
|
5576
|
+
selector(
|
|
5577
|
+
{ op: "!=", value: "kube-system" },
|
|
5578
|
+
{ op: "=~", value: "api-.*" },
|
|
5579
|
+
),
|
|
5580
|
+
false,
|
|
5581
|
+
),
|
|
5582
|
+
).toBe("broader");
|
|
5583
|
+
});
|
|
5584
|
+
|
|
5585
|
+
// Membership can also be established by a workload-logs read, which the
|
|
5586
|
+
// main pass adapts into groups as it goes. Classifying a metrics query
|
|
5587
|
+
// against those groups while they are still being built made the verdict
|
|
5588
|
+
// depend on which tool the agent happened to call first: the same facts,
|
|
5589
|
+
// read in the other order, demoted the same chart.
|
|
5590
|
+
it("classifies a metrics query the same whichever order the agent worked in", () => {
|
|
5591
|
+
const loggedPod = "api-68c7b766dc-fmphn";
|
|
5592
|
+
const exactLoggedPod = {
|
|
5593
|
+
metric: "container_memory_working_set_bytes",
|
|
5594
|
+
matchers: [
|
|
5595
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5596
|
+
{ label: "pod", op: "=~", value: `^(${loggedPod})$` },
|
|
5597
|
+
],
|
|
5598
|
+
};
|
|
5599
|
+
const logs = () =>
|
|
5600
|
+
tool("wl-logs", "get_workload_logs", workloadLogsPayload, {
|
|
5601
|
+
summary: workloadLogsArgs,
|
|
5602
|
+
});
|
|
5603
|
+
const metrics = () =>
|
|
5604
|
+
tool(
|
|
5605
|
+
"prom",
|
|
5606
|
+
"query_prometheus",
|
|
5607
|
+
promResult({
|
|
5608
|
+
selectors: [exactLoggedPod],
|
|
5609
|
+
query: `sum(container_memory_working_set_bytes{namespace="shop",pod=~"^(${loggedPod})$"})`,
|
|
5610
|
+
}),
|
|
5611
|
+
);
|
|
5612
|
+
|
|
5613
|
+
const logsFirst = groupsOf(project([logs(), metrics()]).groups, "metrics");
|
|
5614
|
+
const metricsFirst = groupsOf(
|
|
5615
|
+
project([metrics(), logs()]).groups,
|
|
5616
|
+
"metrics",
|
|
5617
|
+
);
|
|
5618
|
+
expect(logsFirst[0].latest.relevance).toBe("target");
|
|
5619
|
+
expect(metricsFirst[0].latest.relevance).toBe(
|
|
5620
|
+
logsFirst[0].latest.relevance,
|
|
5621
|
+
);
|
|
5622
|
+
|
|
5623
|
+
// Across turns too: the read that names the pod may arrive in a later one.
|
|
5624
|
+
const acrossTurns = groupsOf(
|
|
5625
|
+
project([metrics()], [logs()]).groups,
|
|
5626
|
+
"metrics",
|
|
5627
|
+
);
|
|
5628
|
+
expect(acrossTurns[0].latest.relevance).toBe("target");
|
|
5629
|
+
});
|
|
5630
|
+
|
|
5631
|
+
it("proves pod membership from the bundle instead of the workload's name", () => {
|
|
5632
|
+
const withBundle = project([
|
|
5633
|
+
podsBundle(),
|
|
5634
|
+
tool("prom", "query_prometheus", promResult()),
|
|
5635
|
+
]);
|
|
5636
|
+
expect(groupsOf(withBundle.groups, "metrics")[0].latest.relevance).toBe(
|
|
5637
|
+
"target",
|
|
5638
|
+
);
|
|
5639
|
+
|
|
5640
|
+
// The same exact set with no bundle to establish it, and the prefix that
|
|
5641
|
+
// would also select a sibling workload's pods: kept, never promoted.
|
|
5642
|
+
const unproven = project([tool("prom", "query_prometheus", promResult())]);
|
|
5643
|
+
expect(groupsOf(unproven.groups, "metrics")[0].latest.relevance).toBe(
|
|
5644
|
+
"producer-related",
|
|
5645
|
+
);
|
|
5646
|
+
const prefixed = project([
|
|
5647
|
+
podsBundle(),
|
|
5648
|
+
tool(
|
|
5649
|
+
"prom",
|
|
5650
|
+
"query_prometheus",
|
|
5651
|
+
promResult({ selectors: prefixSelectors }),
|
|
5652
|
+
),
|
|
5653
|
+
]);
|
|
5654
|
+
const prefixGroup = groupsOf(prefixed.groups, "metrics")[0];
|
|
5655
|
+
expect(prefixGroup.latest.relevance).toBe("producer-related");
|
|
5656
|
+
expect(
|
|
5657
|
+
prefixGroup.latest.data.type === "metrics" &&
|
|
5658
|
+
prefixGroup.latest.data.subject,
|
|
5659
|
+
).toBeUndefined();
|
|
5660
|
+
|
|
5661
|
+
// A subset of the established pods is still about the target.
|
|
5662
|
+
const subset = project([
|
|
5663
|
+
podsBundle(),
|
|
5664
|
+
tool(
|
|
5665
|
+
"prom",
|
|
5666
|
+
"query_prometheus",
|
|
5667
|
+
promResult({
|
|
5668
|
+
selectors: [
|
|
5669
|
+
{
|
|
5670
|
+
metric: "container_memory_working_set_bytes",
|
|
5671
|
+
matchers: [
|
|
5672
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5673
|
+
{ label: "pod", op: "=", value: "api-7f6-abc" },
|
|
5674
|
+
],
|
|
5675
|
+
},
|
|
5676
|
+
],
|
|
5677
|
+
}),
|
|
5678
|
+
),
|
|
5679
|
+
]);
|
|
5680
|
+
expect(groupsOf(subset.groups, "metrics")[0].latest.relevance).toBe(
|
|
5681
|
+
"target",
|
|
5682
|
+
);
|
|
5683
|
+
});
|
|
5684
|
+
|
|
5685
|
+
it("proves membership whatever order the agent ran its tools in", () => {
|
|
5686
|
+
// The agent may know the pod names from an earlier turn and chart them
|
|
5687
|
+
// before it calls diagnose. The same facts must read the same way.
|
|
5688
|
+
const queryFirst = project([
|
|
5689
|
+
tool("prom", "query_prometheus", promResult()),
|
|
5690
|
+
podsBundle(),
|
|
5691
|
+
]);
|
|
5692
|
+
expect(groupsOf(queryFirst.groups, "metrics")[0].latest.relevance).toBe(
|
|
5693
|
+
"target",
|
|
5694
|
+
);
|
|
5695
|
+
});
|
|
5696
|
+
|
|
5697
|
+
it("will not establish target pods from a diagnose that never confirmed success", () => {
|
|
5698
|
+
// This set is what proves a Prometheus selector is about the target, so it
|
|
5699
|
+
// takes confirmed success — the same test the sources use — rather than
|
|
5700
|
+
// merely the absence of an error.
|
|
5701
|
+
const unconfirmed = {
|
|
5702
|
+
...podsBundle(),
|
|
5703
|
+
isError: undefined as unknown as boolean,
|
|
5704
|
+
};
|
|
5705
|
+
const projection = project([
|
|
5706
|
+
unconfirmed,
|
|
5707
|
+
tool(
|
|
5708
|
+
"prom",
|
|
5709
|
+
"query_prometheus",
|
|
5710
|
+
promResult({
|
|
5711
|
+
query: `sum(container_memory_working_set_bytes{namespace="shop",pod=~"${podSetPattern}"})`,
|
|
5712
|
+
selectors: [
|
|
5713
|
+
{
|
|
5714
|
+
metric: "container_memory_working_set_bytes",
|
|
5715
|
+
matchers: [
|
|
5716
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5717
|
+
{ label: "pod", op: "=~", value: podSetPattern },
|
|
5718
|
+
],
|
|
5719
|
+
},
|
|
5720
|
+
],
|
|
5721
|
+
}),
|
|
5722
|
+
),
|
|
5723
|
+
]);
|
|
5724
|
+
expect(groupsOf(projection.groups, "metrics")[0].latest.relevance).not.toBe(
|
|
5725
|
+
"target",
|
|
5726
|
+
);
|
|
5727
|
+
});
|
|
5728
|
+
|
|
5729
|
+
it("does not let a generic workload label claim another kind's series", () => {
|
|
5730
|
+
const workloadSelectors = (type?: string) => [
|
|
5731
|
+
{
|
|
5732
|
+
metric: "container_cpu_usage_seconds_total",
|
|
5733
|
+
matchers: [
|
|
5734
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5735
|
+
{ label: "workload", op: "=", value: "api" },
|
|
5736
|
+
...(type ? [{ label: "workload_type", op: "=", value: type }] : []),
|
|
5737
|
+
],
|
|
5738
|
+
},
|
|
5739
|
+
];
|
|
5740
|
+
const asDeployment = project([
|
|
5741
|
+
tool(
|
|
5742
|
+
"prom",
|
|
5743
|
+
"query_prometheus",
|
|
5744
|
+
promResult({ selectors: workloadSelectors("deployment") }),
|
|
5745
|
+
),
|
|
5746
|
+
]);
|
|
5747
|
+
expect(groupsOf(asDeployment.groups, "metrics")[0].latest.relevance).toBe(
|
|
5748
|
+
"target",
|
|
5749
|
+
);
|
|
5750
|
+
// A StatefulSet also called api in the same namespace is a different
|
|
5751
|
+
// workload, and the series says so.
|
|
5752
|
+
const asStatefulSet = project([
|
|
5753
|
+
tool(
|
|
5754
|
+
"prom",
|
|
5755
|
+
"query_prometheus",
|
|
5756
|
+
promResult({ selectors: workloadSelectors("statefulset") }),
|
|
5757
|
+
),
|
|
5758
|
+
]);
|
|
5759
|
+
expect(groupsOf(asStatefulSet.groups, "metrics")[0].latest.relevance).toBe(
|
|
5760
|
+
"producer-related",
|
|
5761
|
+
);
|
|
5762
|
+
|
|
5763
|
+
// The same exclusion written as a regex. Reading only `=` let a sibling's
|
|
5764
|
+
// chart take the target's identity and its change markers.
|
|
5765
|
+
const regexSelectors = (value: string) => [
|
|
5766
|
+
{
|
|
5767
|
+
metric: "container_cpu_usage_seconds_total",
|
|
5768
|
+
matchers: [
|
|
5769
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5770
|
+
{ label: "workload", op: "=", value: "api" },
|
|
5771
|
+
{ label: "workload_type", op: "=~", value },
|
|
5772
|
+
],
|
|
5773
|
+
},
|
|
5774
|
+
];
|
|
5775
|
+
for (const [value, relevance] of [
|
|
5776
|
+
["statefulset", "producer-related"],
|
|
5777
|
+
["^(statefulset)$", "producer-related"],
|
|
5778
|
+
// A set that includes the target kind could be the target, so it is no
|
|
5779
|
+
// conflict; an inexact regex proves nothing either way.
|
|
5780
|
+
["^(deployment|statefulset)$", "target"],
|
|
5781
|
+
[".*", "target"],
|
|
5782
|
+
] as const) {
|
|
5783
|
+
const projection = project([
|
|
5784
|
+
tool(
|
|
5785
|
+
"prom",
|
|
5786
|
+
"query_prometheus",
|
|
5787
|
+
promResult({ selectors: regexSelectors(value) }),
|
|
5788
|
+
),
|
|
5789
|
+
]);
|
|
5790
|
+
expect(groupsOf(projection.groups, "metrics")[0].latest.relevance).toBe(
|
|
5791
|
+
relevance,
|
|
5792
|
+
);
|
|
5793
|
+
}
|
|
5794
|
+
});
|
|
5795
|
+
|
|
5796
|
+
it("refuses an unescaped dot as proof that a pod set names the target", () => {
|
|
5797
|
+
const dotted = ["api.v2-0"];
|
|
5798
|
+
const bundle = tool(
|
|
5799
|
+
"diag-dotted",
|
|
5800
|
+
"diagnose",
|
|
5801
|
+
{ resource: deployment, pods: 1, podNames: dotted },
|
|
5802
|
+
{
|
|
5803
|
+
summary: JSON.stringify({
|
|
5804
|
+
kind: "Deployment",
|
|
5805
|
+
namespace: "shop",
|
|
5806
|
+
name: "api",
|
|
5807
|
+
}),
|
|
5808
|
+
},
|
|
5809
|
+
);
|
|
5810
|
+
const podSelector = (value: string) => [
|
|
5811
|
+
{
|
|
5812
|
+
metric: "container_memory_working_set_bytes",
|
|
5813
|
+
matchers: [
|
|
5814
|
+
{ label: "namespace", op: "=", value: "shop" },
|
|
5815
|
+
{ label: "pod", op: "=~", value },
|
|
5816
|
+
],
|
|
5817
|
+
},
|
|
5818
|
+
];
|
|
5819
|
+
// `^(api.v2-0)$` also selects api-v2-0, so it names more than the pod it
|
|
5820
|
+
// appears to and cannot prove membership.
|
|
5821
|
+
const wildcard = project([
|
|
5822
|
+
bundle,
|
|
5823
|
+
tool(
|
|
5824
|
+
"prom",
|
|
5825
|
+
"query_prometheus",
|
|
5826
|
+
promResult({ selectors: podSelector("^(api.v2-0)$") }),
|
|
5827
|
+
),
|
|
5828
|
+
]);
|
|
5829
|
+
expect(groupsOf(wildcard.groups, "metrics")[0].latest.relevance).toBe(
|
|
5830
|
+
"producer-related",
|
|
5831
|
+
);
|
|
5832
|
+
const escaped = project([
|
|
5833
|
+
bundle,
|
|
5834
|
+
tool(
|
|
5835
|
+
"prom",
|
|
5836
|
+
"query_prometheus",
|
|
5837
|
+
promResult({ selectors: podSelector("^(api\\.v2-0)$") }),
|
|
5838
|
+
),
|
|
5839
|
+
]);
|
|
5840
|
+
expect(groupsOf(escaped.groups, "metrics")[0].latest.relevance).toBe(
|
|
5841
|
+
"target",
|
|
5842
|
+
);
|
|
5843
|
+
});
|
|
5844
|
+
|
|
5845
|
+
it("rejects a result without the producer's selector inventory", () => {
|
|
5846
|
+
const projection = project([
|
|
5847
|
+
tool("old", "query_prometheus", promResult({ selectors: undefined })),
|
|
5848
|
+
tool(
|
|
5849
|
+
"bad-series",
|
|
5850
|
+
"query_prometheus",
|
|
5851
|
+
promResult({
|
|
5852
|
+
series: [{ labels: {}, dataPoints: [{ timestamp: "x", value: 1 }] }],
|
|
5853
|
+
}),
|
|
5854
|
+
),
|
|
5855
|
+
]);
|
|
5856
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
|
|
5857
|
+
expect(projection.limitations.map((item) => item.kind)).toEqual([
|
|
5858
|
+
"unknown",
|
|
5859
|
+
]);
|
|
5860
|
+
expect(projection.limitations[0].source).toBe("Prometheus query");
|
|
5861
|
+
expect(projection.limitations[0].sources).toHaveLength(2);
|
|
5862
|
+
});
|
|
5863
|
+
|
|
5864
|
+
it("merges the same query into one card with revisions", () => {
|
|
5865
|
+
const projection = project(
|
|
5866
|
+
[tool("first", "query_prometheus", promResult())],
|
|
5867
|
+
[
|
|
5868
|
+
tool(
|
|
5869
|
+
"second",
|
|
5870
|
+
"query_prometheus",
|
|
5871
|
+
promResult({
|
|
5872
|
+
series: [
|
|
5873
|
+
{
|
|
5874
|
+
labels: { pod: "api-7f6-abc" },
|
|
5875
|
+
dataPoints: [{ timestamp: 1_788_679_043, value: 1 }],
|
|
5876
|
+
},
|
|
5877
|
+
],
|
|
5878
|
+
}),
|
|
5879
|
+
),
|
|
5880
|
+
],
|
|
5881
|
+
);
|
|
5882
|
+
const groups = groupsOf(projection.groups, "metrics");
|
|
5883
|
+
expect(groups).toHaveLength(1);
|
|
5884
|
+
expect(groups[0].observations).toHaveLength(2);
|
|
5885
|
+
expect(groups[0].observations[1].changedFromPrevious).toBe(true);
|
|
5886
|
+
});
|
|
5887
|
+
});
|
|
5888
|
+
|
|
5889
|
+
describe("resource cards scaled by an HPA", () => {
|
|
5890
|
+
const scaled = (state: string) =>
|
|
5891
|
+
tool("workload", "diagnose", {
|
|
5892
|
+
resource: {
|
|
5893
|
+
apiVersion: "apps/v1",
|
|
5894
|
+
kind: "Deployment",
|
|
5895
|
+
metadata: { namespace: "shop", name: "api" },
|
|
5896
|
+
},
|
|
5897
|
+
resourceContext: {
|
|
5898
|
+
tier: "basic",
|
|
5899
|
+
workloadSummary: { replicas: { desired: 5, ready: 5, available: 5 } },
|
|
5900
|
+
scaledBy: [
|
|
5901
|
+
{
|
|
5902
|
+
kind: "HorizontalPodAutoscaler",
|
|
5903
|
+
namespace: "shop",
|
|
5904
|
+
name: "api-hpa",
|
|
5905
|
+
hpaSummary: {
|
|
5906
|
+
state,
|
|
5907
|
+
summary: "summary from Radar",
|
|
5908
|
+
bounds: { min: 1, max: 5, current: 5, desired: 5 },
|
|
5909
|
+
},
|
|
5910
|
+
},
|
|
5911
|
+
],
|
|
5912
|
+
},
|
|
5913
|
+
});
|
|
5914
|
+
|
|
5915
|
+
it("lifts a healthy workload whose autoscaler cannot act", () => {
|
|
5916
|
+
const result = project([scaled("metrics_unavailable")]);
|
|
5917
|
+
const card = groupsOf(result.groups, "resource")[0].latest;
|
|
5918
|
+
expect(card.tier).toBe("supporting");
|
|
5919
|
+
expect(card.tone).toBe("warning");
|
|
5920
|
+
expect(card.summary).toBe("5/5 replicas ready · HPA: Metrics unavailable");
|
|
5921
|
+
});
|
|
5922
|
+
|
|
5923
|
+
it("treats a maxed-out autoscaler as adverse", () => {
|
|
5924
|
+
const card = groupsOf(
|
|
5925
|
+
project([scaled("limited_max")]).groups,
|
|
5926
|
+
"resource",
|
|
5927
|
+
)[0].latest;
|
|
5928
|
+
expect(card.tone).toBe("warning");
|
|
5929
|
+
expect(card.summary).toBe("5/5 replicas ready · HPA: Maxed");
|
|
5930
|
+
});
|
|
5931
|
+
|
|
5932
|
+
it("leaves an autoscaler that is simply scaling as context", () => {
|
|
5933
|
+
const card = groupsOf(project([scaled("scaling_up")]).groups, "resource")[0]
|
|
5934
|
+
.latest;
|
|
5935
|
+
expect(card.tier).toBe("context");
|
|
5936
|
+
expect(card.tone).toBe("neutral");
|
|
5937
|
+
expect(card.summary).toBe("5/5 replicas ready");
|
|
5938
|
+
});
|
|
5939
|
+
});
|
|
5940
|
+
|
|
5941
|
+
describe("diagnose metrics evidence", () => {
|
|
5942
|
+
const window = {
|
|
5943
|
+
start: "2026-09-06T07:00:00Z",
|
|
5944
|
+
end: "2026-09-06T08:00:00Z",
|
|
5945
|
+
step: "1m2s",
|
|
5946
|
+
};
|
|
5947
|
+
const samples = [
|
|
5948
|
+
{ timestamp: Date.parse(window.start) / 1000, value: 0.003457 },
|
|
5949
|
+
{ timestamp: Date.parse(window.end) / 1000, value: 0.004 },
|
|
5950
|
+
];
|
|
5951
|
+
function vitals(patch: Record<string, unknown> = {}) {
|
|
5952
|
+
return {
|
|
5953
|
+
window,
|
|
5954
|
+
pods: 2,
|
|
5955
|
+
series: [
|
|
5956
|
+
{
|
|
5957
|
+
category: "cpu",
|
|
5958
|
+
unit: "cores",
|
|
5959
|
+
query:
|
|
5960
|
+
"sum(rate(container_cpu_usage_seconds_total{container!='',namespace='shop',pod=~'^(api-a|api-b)$'}[5m]))",
|
|
5961
|
+
series: [{ labels: {}, dataPoints: samples }],
|
|
5962
|
+
},
|
|
5963
|
+
{
|
|
5964
|
+
category: "memory",
|
|
5965
|
+
unit: "bytes",
|
|
5966
|
+
query:
|
|
5967
|
+
"sum(max by (pod,namespace,container) (container_memory_working_set_bytes{container!='',namespace='shop',pod=~'^(api-a|api-b)$'}))",
|
|
5968
|
+
series: [{ labels: {}, dataPoints: samples }],
|
|
5969
|
+
},
|
|
5970
|
+
{
|
|
5971
|
+
category: "restarts",
|
|
5972
|
+
unit: "count",
|
|
5973
|
+
query:
|
|
5974
|
+
"sum(round(increase(kube_pod_container_status_restarts_total{namespace='shop',pod=~'^(api-a|api-b)$'}[1h])))",
|
|
5975
|
+
series: [],
|
|
5976
|
+
},
|
|
5977
|
+
],
|
|
5978
|
+
...patch,
|
|
5979
|
+
};
|
|
5980
|
+
}
|
|
5981
|
+
const args = { kind: "Deployment", namespace: "shop", name: "api" };
|
|
5982
|
+
|
|
5983
|
+
it("charts each captured category as target evidence with the diagnosed resource as subject", () => {
|
|
5984
|
+
const projection = project([
|
|
5985
|
+
tool(
|
|
5986
|
+
"diag",
|
|
5987
|
+
"diagnose",
|
|
5988
|
+
{ resource: deployment, pods: 2, metrics: vitals() },
|
|
5989
|
+
{ summary: JSON.stringify(args) },
|
|
5990
|
+
),
|
|
5991
|
+
]);
|
|
5992
|
+
const groups = groupsOf(projection.groups, "metrics");
|
|
5993
|
+
expect(groups.map((group) => group.latest.title)).toEqual([
|
|
5994
|
+
"CPU usage · Deployment shop/api",
|
|
5995
|
+
"Memory working set · Deployment shop/api",
|
|
5996
|
+
"Restarts · Deployment shop/api",
|
|
5997
|
+
]);
|
|
5998
|
+
for (const group of groups) {
|
|
5999
|
+
expect(group.latest.relevance).toBe("target");
|
|
6000
|
+
expect(group.latest.tier).toBe("supporting");
|
|
6001
|
+
expect(group.latest.tone).toBe("neutral");
|
|
6002
|
+
const data = group.latest.data;
|
|
6003
|
+
if (data.type !== "metrics") throw new Error("expected metrics");
|
|
6004
|
+
expect(data.origin).toBe("diagnose");
|
|
6005
|
+
expect(data.mode).toBe("range");
|
|
6006
|
+
expect(data.start).toBe(window.start);
|
|
6007
|
+
expect(data.end).toBe(window.end);
|
|
6008
|
+
expect(data.step).toBe(window.step);
|
|
6009
|
+
expect(data.pods).toBe(2);
|
|
6010
|
+
expect(data.partial).toBe(false);
|
|
6011
|
+
expect(data.truncated).toBe(false);
|
|
6012
|
+
expect(data.subject).toEqual({
|
|
6013
|
+
kind: "Deployment",
|
|
6014
|
+
group: "apps",
|
|
6015
|
+
namespace: "shop",
|
|
6016
|
+
name: "api",
|
|
6017
|
+
});
|
|
6018
|
+
expect(investigationEvidenceSubjectRef(data)).toEqual(data.subject);
|
|
6019
|
+
}
|
|
6020
|
+
const [cpu, , restarts] = groups;
|
|
6021
|
+
expect(cpu.latest.summary).toBe("2 current pods · 60m window · 1m2s step");
|
|
6022
|
+
const cpuData = cpu.latest.data;
|
|
6023
|
+
if (cpuData.type !== "metrics") throw new Error("expected metrics");
|
|
6024
|
+
expect(cpuData.unit).toBe("cores");
|
|
6025
|
+
expect(cpuData.label).toBe("CPU usage");
|
|
6026
|
+
expect(cpuData.query).toContain("pod=~'^(api-a|api-b)$'");
|
|
6027
|
+
expect(cpuData.series).toEqual([{ labels: {}, dataPoints: samples }]);
|
|
6028
|
+
expect(restarts.latest.summary).toBe(
|
|
6029
|
+
"No samples in the window · 60m window · 1m2s step",
|
|
6030
|
+
);
|
|
6031
|
+
expect(
|
|
6032
|
+
projection.limitations.filter(
|
|
6033
|
+
(item) => item.source === "Workload metrics",
|
|
6034
|
+
),
|
|
6035
|
+
).toHaveLength(0);
|
|
6036
|
+
});
|
|
6037
|
+
|
|
6038
|
+
it("merges a re-diagnose of the same workload into the same chart per category", () => {
|
|
6039
|
+
const later = vitals({
|
|
6040
|
+
window: { ...window, end: "2026-09-06T08:30:00Z" },
|
|
6041
|
+
});
|
|
6042
|
+
const projection = project(
|
|
6043
|
+
[
|
|
6044
|
+
tool(
|
|
6045
|
+
"diag-1",
|
|
6046
|
+
"diagnose",
|
|
6047
|
+
{ resource: deployment, pods: 2, metrics: vitals() },
|
|
6048
|
+
{ summary: JSON.stringify(args) },
|
|
6049
|
+
),
|
|
6050
|
+
],
|
|
6051
|
+
[
|
|
6052
|
+
tool(
|
|
6053
|
+
"diag-2",
|
|
6054
|
+
"diagnose",
|
|
6055
|
+
{ resource: deployment, pods: 2, metrics: later },
|
|
6056
|
+
{ summary: JSON.stringify(args) },
|
|
6057
|
+
),
|
|
6058
|
+
],
|
|
6059
|
+
);
|
|
6060
|
+
const groups = groupsOf(projection.groups, "metrics");
|
|
6061
|
+
expect(groups).toHaveLength(3);
|
|
6062
|
+
expect(groups.every((group) => group.observations.length === 2)).toBe(true);
|
|
6063
|
+
const data = groups[0].latest.data;
|
|
6064
|
+
if (data.type !== "metrics") throw new Error("expected metrics");
|
|
6065
|
+
expect(data.end).toBe("2026-09-06T08:30:00Z");
|
|
6066
|
+
});
|
|
6067
|
+
|
|
6068
|
+
it("reports a producer error as a limitation and still charts what answered", () => {
|
|
6069
|
+
const projection = project([
|
|
6070
|
+
tool(
|
|
6071
|
+
"diag",
|
|
6072
|
+
"diagnose",
|
|
6073
|
+
{
|
|
6074
|
+
resource: deployment,
|
|
6075
|
+
pods: 2,
|
|
6076
|
+
metrics: vitals({
|
|
6077
|
+
series: [vitals().series[1]],
|
|
6078
|
+
error:
|
|
6079
|
+
"cpu: prom: query error from prometheus: cpu exploded (execution)",
|
|
6080
|
+
}),
|
|
6081
|
+
},
|
|
6082
|
+
{ summary: JSON.stringify(args) },
|
|
6083
|
+
),
|
|
6084
|
+
]);
|
|
6085
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(1);
|
|
6086
|
+
const limitation = projection.limitations.find(
|
|
6087
|
+
(item) => item.source === "Workload metrics",
|
|
6088
|
+
);
|
|
6089
|
+
expect(limitation?.kind).toBe("error");
|
|
6090
|
+
expect(limitation?.message).toContain("cpu exploded");
|
|
6091
|
+
});
|
|
6092
|
+
|
|
6093
|
+
it("reports an unreachable Prometheus without inventing a chart", () => {
|
|
6094
|
+
const projection = project([
|
|
6095
|
+
tool(
|
|
6096
|
+
"diag",
|
|
6097
|
+
"diagnose",
|
|
6098
|
+
{
|
|
6099
|
+
resource: deployment,
|
|
6100
|
+
pods: 2,
|
|
6101
|
+
metrics: vitals({
|
|
6102
|
+
series: [],
|
|
6103
|
+
error:
|
|
6104
|
+
"prometheus unreachable: dial tcp 10.0.0.9:9090: connection refused",
|
|
6105
|
+
}),
|
|
6106
|
+
},
|
|
6107
|
+
{ summary: JSON.stringify(args) },
|
|
6108
|
+
),
|
|
6109
|
+
]);
|
|
6110
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
|
|
6111
|
+
const limitation = projection.limitations.find(
|
|
6112
|
+
(item) => item.source === "Workload metrics",
|
|
6113
|
+
);
|
|
6114
|
+
expect(limitation?.kind).toBe("error");
|
|
6115
|
+
expect(limitation?.message).toBe(
|
|
6116
|
+
"prometheus unreachable: dial tcp 10.0.0.9:9090: connection refused",
|
|
6117
|
+
);
|
|
6118
|
+
});
|
|
6119
|
+
|
|
6120
|
+
it("says nothing when diagnose carried no metrics field", () => {
|
|
6121
|
+
const projection = project([
|
|
6122
|
+
tool(
|
|
6123
|
+
"diag",
|
|
6124
|
+
"diagnose",
|
|
6125
|
+
{ resource: deployment, pods: 2 },
|
|
6126
|
+
{ summary: JSON.stringify(args) },
|
|
6127
|
+
),
|
|
6128
|
+
]);
|
|
6129
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
|
|
6130
|
+
expect(
|
|
6131
|
+
projection.limitations.some((item) => item.source === "Workload metrics"),
|
|
6132
|
+
).toBe(false);
|
|
6133
|
+
});
|
|
6134
|
+
|
|
6135
|
+
it("notes a partial pod set in the summary and keeps the cap on the card", () => {
|
|
6136
|
+
const projection = project([
|
|
6137
|
+
tool(
|
|
6138
|
+
"diag",
|
|
6139
|
+
"diagnose",
|
|
6140
|
+
{
|
|
6141
|
+
resource: deployment,
|
|
6142
|
+
pods: 55,
|
|
6143
|
+
metrics: vitals({ pods: 50, partial: true, omittedPods: 5 }),
|
|
6144
|
+
},
|
|
6145
|
+
{ summary: JSON.stringify(args) },
|
|
6146
|
+
),
|
|
6147
|
+
]);
|
|
6148
|
+
const [cpu] = groupsOf(projection.groups, "metrics");
|
|
6149
|
+
expect(cpu.latest.summary).toBe(
|
|
6150
|
+
"first 50 of 55 current pods · 60m window · 1m2s step · partial pod set",
|
|
6151
|
+
);
|
|
6152
|
+
const data = cpu.latest.data;
|
|
6153
|
+
if (data.type !== "metrics") throw new Error("expected metrics");
|
|
6154
|
+
expect(data.pods).toBe(50);
|
|
6155
|
+
expect(data.partial).toBe(true);
|
|
6156
|
+
});
|
|
6157
|
+
|
|
6158
|
+
it("keeps a neighbour's vitals broader with the neighbour as subject", () => {
|
|
6159
|
+
const worker = {
|
|
6160
|
+
...deployment,
|
|
6161
|
+
metadata: { namespace: "shop", name: "worker" },
|
|
6162
|
+
};
|
|
6163
|
+
const projection = project([
|
|
6164
|
+
tool(
|
|
6165
|
+
"diag-worker",
|
|
6166
|
+
"diagnose",
|
|
6167
|
+
{ resource: worker, pods: 2, metrics: vitals() },
|
|
6168
|
+
{
|
|
6169
|
+
summary: JSON.stringify({
|
|
6170
|
+
kind: "Deployment",
|
|
6171
|
+
namespace: "shop",
|
|
6172
|
+
name: "worker",
|
|
6173
|
+
}),
|
|
6174
|
+
},
|
|
6175
|
+
),
|
|
6176
|
+
]);
|
|
6177
|
+
const groups = groupsOf(projection.groups, "metrics");
|
|
6178
|
+
expect(groups).toHaveLength(3);
|
|
6179
|
+
for (const group of groups) {
|
|
6180
|
+
expect(group.latest.relevance).toBe("broader");
|
|
6181
|
+
expect(group.latest.tier).toBe("context");
|
|
6182
|
+
const data = group.latest.data;
|
|
6183
|
+
if (data.type !== "metrics") throw new Error("expected metrics");
|
|
6184
|
+
expect(data.subject).toEqual({
|
|
6185
|
+
kind: "Deployment",
|
|
6186
|
+
group: "apps",
|
|
6187
|
+
namespace: "shop",
|
|
6188
|
+
name: "worker",
|
|
6189
|
+
});
|
|
6190
|
+
}
|
|
6191
|
+
});
|
|
6192
|
+
|
|
6193
|
+
it("rejects a malformed metrics field instead of charting it", () => {
|
|
6194
|
+
const malformed: Record<string, unknown>[] = [
|
|
6195
|
+
{ window: { start: window.start }, pods: "two", series: [] },
|
|
6196
|
+
{ window: { ...window, end: "not a date" }, pods: 2, series: [] },
|
|
6197
|
+
{ window: { ...window, end: window.start }, pods: 2, series: [] },
|
|
6198
|
+
{ window, pods: -1, series: [] },
|
|
6199
|
+
{ window, pods: 1.5, series: [] },
|
|
6200
|
+
{ window, pods: 2, omittedPods: -3, series: [] },
|
|
6201
|
+
];
|
|
6202
|
+
for (const metrics of malformed) {
|
|
6203
|
+
const projection = project([
|
|
6204
|
+
tool(
|
|
6205
|
+
"diag",
|
|
6206
|
+
"diagnose",
|
|
6207
|
+
{ resource: deployment, pods: 2, metrics },
|
|
6208
|
+
{ summary: JSON.stringify(args) },
|
|
6209
|
+
),
|
|
6210
|
+
]);
|
|
6211
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
|
|
6212
|
+
expect(
|
|
6213
|
+
projection.limitations.some(
|
|
6214
|
+
(item) =>
|
|
6215
|
+
item.source === "Workload metrics" && item.kind === "unknown",
|
|
6216
|
+
),
|
|
6217
|
+
).toBe(true);
|
|
6218
|
+
}
|
|
6219
|
+
});
|
|
6220
|
+
|
|
6221
|
+
it("keeps same-named workloads in different API groups on separate charts", () => {
|
|
6222
|
+
const rollout = {
|
|
6223
|
+
...deployment,
|
|
6224
|
+
apiVersion: "argoproj.io/v1alpha1",
|
|
6225
|
+
kind: "Rollout",
|
|
6226
|
+
};
|
|
6227
|
+
const projection = project([
|
|
6228
|
+
tool(
|
|
6229
|
+
"diag-deployment",
|
|
6230
|
+
"diagnose",
|
|
6231
|
+
{ resource: deployment, pods: 2, metrics: vitals() },
|
|
6232
|
+
{ summary: JSON.stringify(args) },
|
|
6233
|
+
),
|
|
6234
|
+
tool(
|
|
6235
|
+
"diag-rollout",
|
|
6236
|
+
"diagnose",
|
|
6237
|
+
{ resource: rollout, pods: 2, metrics: vitals() },
|
|
6238
|
+
{ summary: JSON.stringify(args) },
|
|
6239
|
+
),
|
|
6240
|
+
]);
|
|
6241
|
+
const groups = groupsOf(projection.groups, "metrics");
|
|
6242
|
+
expect(groups).toHaveLength(6);
|
|
6243
|
+
expect(groups.every((group) => group.observations.length === 1)).toBe(true);
|
|
6244
|
+
expect(new Set(groups.map((group) => group.identity)).size).toBe(6);
|
|
6245
|
+
});
|
|
6246
|
+
|
|
6247
|
+
it("rejects inherited object keys posing as categories", () => {
|
|
6248
|
+
for (const category of ["__proto__", "constructor", "toString"]) {
|
|
6249
|
+
const projection = project([
|
|
6250
|
+
tool(
|
|
6251
|
+
"diag",
|
|
6252
|
+
"diagnose",
|
|
6253
|
+
{
|
|
6254
|
+
resource: deployment,
|
|
6255
|
+
pods: 2,
|
|
6256
|
+
metrics: vitals({
|
|
6257
|
+
series: [{ ...vitals().series[0], category }],
|
|
6258
|
+
}),
|
|
6259
|
+
},
|
|
6260
|
+
{ summary: JSON.stringify(args) },
|
|
6261
|
+
),
|
|
6262
|
+
]);
|
|
6263
|
+
expect(groupsOf(projection.groups, "metrics")).toHaveLength(0);
|
|
6264
|
+
expect(
|
|
6265
|
+
projection.limitations.some(
|
|
6266
|
+
(item) =>
|
|
6267
|
+
item.source === "Workload metrics" && item.kind === "unknown",
|
|
6268
|
+
),
|
|
6269
|
+
).toBe(true);
|
|
6270
|
+
}
|
|
6271
|
+
});
|
|
6272
|
+
|
|
6273
|
+
it("rejects a category outside the contract or a series without a unit", () => {
|
|
6274
|
+
const projection = project([
|
|
6275
|
+
tool(
|
|
6276
|
+
"diag",
|
|
6277
|
+
"diagnose",
|
|
6278
|
+
{
|
|
6279
|
+
resource: deployment,
|
|
6280
|
+
pods: 2,
|
|
6281
|
+
metrics: vitals({
|
|
6282
|
+
series: [
|
|
6283
|
+
{ ...vitals().series[0], category: "latency" },
|
|
6284
|
+
{ ...vitals().series[1], unit: undefined },
|
|
6285
|
+
vitals().series[2],
|
|
6286
|
+
],
|
|
6287
|
+
}),
|
|
6288
|
+
},
|
|
6289
|
+
{ summary: JSON.stringify(args) },
|
|
6290
|
+
),
|
|
6291
|
+
]);
|
|
6292
|
+
const groups = groupsOf(projection.groups, "metrics");
|
|
6293
|
+
expect(groups.map((group) => group.latest.title)).toEqual([
|
|
6294
|
+
"Restarts · Deployment shop/api",
|
|
6295
|
+
]);
|
|
6296
|
+
expect(
|
|
6297
|
+
projection.limitations.some(
|
|
6298
|
+
(item) => item.source === "Workload metrics" && item.kind === "unknown",
|
|
6299
|
+
),
|
|
6300
|
+
).toBe(true);
|
|
6301
|
+
});
|
|
6302
|
+
});
|
|
6303
|
+
|
|
6304
|
+
describe("live-run follow-ups", () => {
|
|
6305
|
+
it("reads the producer's nil-slice null as an empty events or changes result", () => {
|
|
6306
|
+
const projection = project([
|
|
6307
|
+
tool(
|
|
6308
|
+
"events-none",
|
|
6309
|
+
"get_events",
|
|
6310
|
+
{ events: null },
|
|
6311
|
+
{
|
|
6312
|
+
summary: JSON.stringify({
|
|
6313
|
+
kind: "Pod",
|
|
6314
|
+
namespace: "shop",
|
|
6315
|
+
name: "api",
|
|
6316
|
+
}),
|
|
6317
|
+
},
|
|
6318
|
+
),
|
|
6319
|
+
tool(
|
|
6320
|
+
"changes-none",
|
|
6321
|
+
"get_changes",
|
|
6322
|
+
{ changes: null },
|
|
6323
|
+
{
|
|
6324
|
+
summary: JSON.stringify({
|
|
6325
|
+
kind: "Deployment",
|
|
6326
|
+
namespace: "shop",
|
|
6327
|
+
name: "api",
|
|
6328
|
+
since: "24h",
|
|
6329
|
+
}),
|
|
6330
|
+
},
|
|
6331
|
+
),
|
|
6332
|
+
]);
|
|
6333
|
+
// A complete, successful events query that returned nothing is a checked
|
|
6334
|
+
// receipt for its scope; the producer marks a denied namespace separately.
|
|
6335
|
+
expect(
|
|
6336
|
+
projection.groups.map((group) => [
|
|
6337
|
+
group.latest.data.type,
|
|
6338
|
+
group.latest.title,
|
|
6339
|
+
]),
|
|
6340
|
+
).toEqual([["receipt", "No events in this window"]]);
|
|
6341
|
+
expect(
|
|
6342
|
+
projection.limitations.map((limitation) => [
|
|
6343
|
+
limitation.source,
|
|
6344
|
+
limitation.kind,
|
|
6345
|
+
]),
|
|
6346
|
+
).toEqual([["Recent changes", "unknown"]]);
|
|
6347
|
+
});
|
|
6348
|
+
|
|
6349
|
+
it("keeps one identity for a blocker whether one pod holds it or several", () => {
|
|
6350
|
+
const blocker = (name: string) => ({
|
|
6351
|
+
kind: "Pod",
|
|
6352
|
+
name,
|
|
6353
|
+
reason: "Unschedulable",
|
|
6354
|
+
severity: "critical",
|
|
6355
|
+
message: "1 node(s) no free host ports",
|
|
6356
|
+
});
|
|
6357
|
+
const bundle = (pods: string[]) => ({
|
|
6358
|
+
resource: {
|
|
6359
|
+
apiVersion: "apps/v1",
|
|
6360
|
+
kind: "DaemonSet",
|
|
6361
|
+
metadata: { namespace: "opencost", name: "node-exporter" },
|
|
6362
|
+
},
|
|
6363
|
+
resourceContext: { tier: "basic" },
|
|
6364
|
+
pods: pods.length,
|
|
6365
|
+
startupBlockers: pods.map(blocker),
|
|
6366
|
+
});
|
|
6367
|
+
// Two pods recover to one between diagnoses. The blocker is the same
|
|
6368
|
+
// finding, so it must update its card rather than open a second one
|
|
6369
|
+
// beside it saying the same thing.
|
|
6370
|
+
const projection = projectInvestigationEvidence(
|
|
6371
|
+
[
|
|
6372
|
+
{
|
|
6373
|
+
status: "done",
|
|
6374
|
+
timeline: [tool("first", "diagnose", bundle(["a", "b"]))],
|
|
6375
|
+
},
|
|
6376
|
+
{
|
|
6377
|
+
status: "done",
|
|
6378
|
+
verify: true,
|
|
6379
|
+
timeline: [tool("second", "diagnose", bundle(["a"]))],
|
|
6380
|
+
},
|
|
6381
|
+
],
|
|
6382
|
+
{
|
|
6383
|
+
kind: "DaemonSet",
|
|
6384
|
+
group: "apps",
|
|
6385
|
+
namespace: "opencost",
|
|
6386
|
+
name: "node-exporter",
|
|
6387
|
+
},
|
|
6388
|
+
);
|
|
6389
|
+
const startup = projection.groups.filter(
|
|
6390
|
+
(group) => group.latest.data.type === "startup",
|
|
6391
|
+
);
|
|
6392
|
+
expect(startup).toHaveLength(1);
|
|
6393
|
+
expect(startup[0].observations).toHaveLength(2);
|
|
6394
|
+
expect(startup[0].latest.summary).toBe("1 node(s) no free host ports");
|
|
6395
|
+
});
|
|
6396
|
+
|
|
6397
|
+
it("merges identical startup blockers across pods into one card with the pod list", () => {
|
|
6398
|
+
const blocker = (
|
|
6399
|
+
name: string,
|
|
6400
|
+
message = "1 node(s) no free host ports",
|
|
6401
|
+
) => ({
|
|
6402
|
+
kind: "Pod",
|
|
6403
|
+
name,
|
|
6404
|
+
reason: "Unschedulable",
|
|
6405
|
+
severity: "critical",
|
|
6406
|
+
message,
|
|
6407
|
+
});
|
|
6408
|
+
const projection = project([
|
|
6409
|
+
tool("diagnose", "diagnose", {
|
|
6410
|
+
resource: {
|
|
6411
|
+
apiVersion: "apps/v1",
|
|
6412
|
+
kind: "DaemonSet",
|
|
6413
|
+
metadata: { namespace: "opencost", name: "node-exporter" },
|
|
6414
|
+
},
|
|
6415
|
+
resourceContext: { tier: "basic" },
|
|
6416
|
+
pods: 3,
|
|
6417
|
+
startupBlockers: [
|
|
6418
|
+
blocker("node-exporter-a"),
|
|
6419
|
+
blocker("node-exporter-b"),
|
|
6420
|
+
blocker("node-exporter-c", "0/10 nodes: insufficient memory"),
|
|
6421
|
+
],
|
|
6422
|
+
}),
|
|
6423
|
+
]);
|
|
6424
|
+
const startup = projection.groups.filter(
|
|
6425
|
+
(group) => group.latest.data.type === "startup",
|
|
6426
|
+
);
|
|
6427
|
+
expect(
|
|
6428
|
+
startup.map((group) => [
|
|
6429
|
+
group.latest.summary,
|
|
6430
|
+
group.latest.data.type === "startup" ? group.latest.data.pods : null,
|
|
6431
|
+
]),
|
|
6432
|
+
).toEqual([
|
|
6433
|
+
[
|
|
6434
|
+
"2 pods · 1 node(s) no free host ports",
|
|
6435
|
+
["node-exporter-a", "node-exporter-b"],
|
|
6436
|
+
],
|
|
6437
|
+
// A blocker only one pod holds still carries that pod, so the pods a
|
|
6438
|
+
// later Prometheus query is proved against do not depend on how many
|
|
6439
|
+
// happened to share a reason.
|
|
6440
|
+
["0/10 nodes: insufficient memory", ["node-exporter-c"]],
|
|
6441
|
+
]);
|
|
6442
|
+
expect(startup[1].id).not.toBe(startup[0].id);
|
|
6443
|
+
});
|
|
6444
|
+
|
|
6445
|
+
it("folds pods' startup blocker into the classified issue that repeats it", () => {
|
|
6446
|
+
const blocker = (name: string) => ({
|
|
6447
|
+
kind: "Pod",
|
|
6448
|
+
name,
|
|
6449
|
+
reason: "Unschedulable",
|
|
6450
|
+
severity: "critical",
|
|
6451
|
+
message: "1 node(s) no free host ports",
|
|
6452
|
+
});
|
|
6453
|
+
const projection = project([
|
|
6454
|
+
tool("diagnose", "diagnose", {
|
|
6455
|
+
resource: {
|
|
6456
|
+
apiVersion: "apps/v1",
|
|
6457
|
+
kind: "DaemonSet",
|
|
6458
|
+
metadata: { namespace: "opencost", name: "node-exporter" },
|
|
6459
|
+
},
|
|
6460
|
+
resourceContext: { tier: "basic" },
|
|
6461
|
+
pods: 2,
|
|
6462
|
+
relatedIssues: [
|
|
6463
|
+
{
|
|
6464
|
+
id: "issue-unsched",
|
|
6465
|
+
severity: "critical",
|
|
6466
|
+
source: "scheduling",
|
|
6467
|
+
category: "unschedulable",
|
|
6468
|
+
category_group: "startup",
|
|
6469
|
+
grouping_scope: "workload",
|
|
6470
|
+
kind: "DaemonSet",
|
|
6471
|
+
namespace: "opencost",
|
|
6472
|
+
name: "node-exporter",
|
|
6473
|
+
reason: "Unschedulable",
|
|
6474
|
+
message: "1 node(s) no free host ports",
|
|
6475
|
+
},
|
|
6476
|
+
],
|
|
6477
|
+
startupBlockers: [
|
|
6478
|
+
blocker("node-exporter-a"),
|
|
6479
|
+
blocker("node-exporter-b"),
|
|
6480
|
+
],
|
|
6481
|
+
}),
|
|
6482
|
+
]);
|
|
6483
|
+
expect(
|
|
6484
|
+
projection.groups.filter((group) => group.latest.data.type === "startup"),
|
|
6485
|
+
).toHaveLength(0);
|
|
6486
|
+
const [issue] = projection.groups.filter(
|
|
6487
|
+
(group) => group.latest.data.type === "issue",
|
|
6488
|
+
);
|
|
6489
|
+
expect(
|
|
6490
|
+
issue.latest.data.type === "issue" && issue.latest.data.pods,
|
|
6491
|
+
).toEqual(["node-exporter-a", "node-exporter-b"]);
|
|
6492
|
+
});
|
|
6493
|
+
|
|
6494
|
+
it("files a denied events namespace as an access limitation, not a receipt", () => {
|
|
6495
|
+
const projection = project([
|
|
6496
|
+
tool(
|
|
6497
|
+
"events-denied",
|
|
6498
|
+
"get_events",
|
|
6499
|
+
{ events: [], accessDenied: true },
|
|
6500
|
+
{
|
|
6501
|
+
summary: JSON.stringify({ namespace: "locked", kind: "Pod" }),
|
|
6502
|
+
},
|
|
6503
|
+
),
|
|
6504
|
+
]);
|
|
6505
|
+
expect(projection.groups).toHaveLength(0);
|
|
6506
|
+
expect(
|
|
6507
|
+
projection.limitations.map((limitation) => [
|
|
6508
|
+
limitation.source,
|
|
6509
|
+
limitation.kind,
|
|
6510
|
+
]),
|
|
6511
|
+
).toEqual([["Events", "error"]]);
|
|
6512
|
+
expect(projection.limitations[0].message).toContain(
|
|
6513
|
+
"not readable with your permissions",
|
|
6514
|
+
);
|
|
6515
|
+
});
|
|
6516
|
+
|
|
6517
|
+
it("keeps a cluster-wide events read that was narrowed to readable namespaces from reading as a clean cluster", () => {
|
|
6518
|
+
const empty = project([
|
|
6519
|
+
tool(
|
|
6520
|
+
"events-partial-empty",
|
|
6521
|
+
"get_events",
|
|
6522
|
+
{ events: [], partialScope: true, scopeNamespaces: ["alpha", "beta"] },
|
|
6523
|
+
{ summary: JSON.stringify({}) },
|
|
6524
|
+
),
|
|
6525
|
+
]);
|
|
6526
|
+
const [receipt] = groupsOf(empty.groups, "receipt");
|
|
6527
|
+
expect(receipt.latest.title).toBe(
|
|
6528
|
+
"No events in the namespaces you can read",
|
|
6529
|
+
);
|
|
6530
|
+
const narrowedBody =
|
|
6531
|
+
receipt.latest.data.type === "receipt" ? receipt.latest.data.message : "";
|
|
6532
|
+
expect(narrowedBody).toContain("alpha, beta");
|
|
6533
|
+
// The load-bearing half: an empty answer from the namespaces a reader can
|
|
6534
|
+
// see must never read as an answer about the cluster.
|
|
6535
|
+
expect(narrowedBody).toContain("does not clear the cluster");
|
|
6536
|
+
expect(empty.limitations).toHaveLength(0);
|
|
6537
|
+
|
|
6538
|
+
const found = project([
|
|
6539
|
+
tool(
|
|
6540
|
+
"events-partial",
|
|
6541
|
+
"get_events",
|
|
6542
|
+
{
|
|
6543
|
+
events: [
|
|
6544
|
+
{
|
|
6545
|
+
type: "Warning",
|
|
6546
|
+
reason: "BackOff",
|
|
6547
|
+
message: "back-off restarting",
|
|
6548
|
+
count: 3,
|
|
6549
|
+
lastTimestamp: "2026-09-02T09:00:00Z",
|
|
6550
|
+
involvedObject: { kind: "Pod", namespace: "alpha", name: "api" },
|
|
6551
|
+
},
|
|
6552
|
+
],
|
|
6553
|
+
partialScope: true,
|
|
6554
|
+
scopeNamespaces: ["alpha", "beta"],
|
|
6555
|
+
},
|
|
6556
|
+
{ summary: JSON.stringify({}) },
|
|
6557
|
+
),
|
|
6558
|
+
]);
|
|
6559
|
+
expect(
|
|
6560
|
+
found.limitations.map((limitation) => [
|
|
6561
|
+
limitation.source,
|
|
6562
|
+
limitation.kind,
|
|
6563
|
+
]),
|
|
6564
|
+
).toEqual([["Events", "unknown"]]);
|
|
6565
|
+
expect(found.limitations[0].message).toContain("alpha, beta");
|
|
6566
|
+
});
|
|
6567
|
+
|
|
6568
|
+
it("keeps repeated change reads of one resource on one card and names the window", () => {
|
|
6569
|
+
const change = {
|
|
6570
|
+
kind: "Deployment",
|
|
6571
|
+
apiVersion: "apps/v1",
|
|
6572
|
+
namespace: "shop",
|
|
6573
|
+
name: "api",
|
|
6574
|
+
changeType: "update",
|
|
6575
|
+
timestamp: "2026-09-02T09:00:00Z",
|
|
6576
|
+
};
|
|
6577
|
+
const older = { ...change, timestamp: "2026-09-01T09:00:00Z" };
|
|
6578
|
+
const read = (
|
|
6579
|
+
id: string,
|
|
6580
|
+
args: Record<string, unknown>,
|
|
6581
|
+
changes: unknown[],
|
|
6582
|
+
) =>
|
|
6583
|
+
tool(id, "get_changes", { changes }, { summary: JSON.stringify(args) });
|
|
6584
|
+
const projection = project([
|
|
6585
|
+
read(
|
|
6586
|
+
"changes-24h",
|
|
6587
|
+
{ namespace: "shop", name: "api", since: "24h", kind: "Deployment" },
|
|
6588
|
+
[change],
|
|
6589
|
+
),
|
|
6590
|
+
read(
|
|
6591
|
+
"changes-48h",
|
|
6592
|
+
{
|
|
6593
|
+
kind: "Deployment",
|
|
6594
|
+
namespace: "shop",
|
|
6595
|
+
name: "api",
|
|
6596
|
+
since: "48h",
|
|
6597
|
+
limit: 30,
|
|
6598
|
+
},
|
|
6599
|
+
[change, older],
|
|
6600
|
+
),
|
|
6601
|
+
read("changes-ns", { namespace: "shop", since: "24h" }, [change]),
|
|
6602
|
+
]);
|
|
6603
|
+
const changes = groupsOf(projection.groups, "changes");
|
|
6604
|
+
expect(changes.map((group) => group.identity)).toEqual([
|
|
6605
|
+
"changes:Deployment shop/api",
|
|
6606
|
+
"changes:namespace shop",
|
|
6607
|
+
]);
|
|
6608
|
+
expect(changes[0].observations).toHaveLength(2);
|
|
6609
|
+
expect(changes[0].observations.map((o) => o.title)).toEqual([
|
|
6610
|
+
"Recent changes · last 24h",
|
|
6611
|
+
"Recent changes · last 48h",
|
|
6612
|
+
]);
|
|
6613
|
+
expect(changes[0].latest.title).toBe("Recent changes · last 48h");
|
|
6614
|
+
expect(changes[0].latest.summary).toBe("2 changes · Deployment shop/api");
|
|
6615
|
+
expect(changes[1].latest.title).toBe("Recent changes · last 24h");
|
|
6616
|
+
});
|
|
6617
|
+
});
|