@skyhook-io/k8s-ui 1.16.3 → 1.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. package/README.md +41 -0
  2. package/package.json +3 -3
  3. package/src/components/applications/ApplicationDetail.test.tsx +15 -0
  4. package/src/components/applications/ApplicationDetail.tsx +17 -15
  5. package/src/components/charts/AreaChart.interaction.test.tsx +118 -0
  6. package/src/components/charts/AreaChart.test.tsx +25 -0
  7. package/src/components/charts/AreaChart.tsx +141 -12
  8. package/src/components/charts/index.ts +1 -0
  9. package/src/components/checks/ChecksView.tsx +18 -1
  10. package/src/components/checks/types.ts +2 -13
  11. package/src/components/cnpg/CNPGBackupSummary.tsx +57 -9
  12. package/src/components/cnpg/CNPGClusterHASection.test.tsx +90 -0
  13. package/src/components/cnpg/CNPGClusterHASection.tsx +331 -0
  14. package/src/components/cnpg/CNPGClusterSummary.test.tsx +275 -0
  15. package/src/components/cnpg/CNPGClusterSummary.tsx +310 -96
  16. package/src/components/cnpg/CNPGConnectSection.test.tsx +64 -0
  17. package/src/components/cnpg/CNPGConnectSection.tsx +178 -0
  18. package/src/components/cnpg/CNPGDeclarativeSummary.tsx +157 -26
  19. package/src/components/cnpg/CNPGImageCatalogSummary.tsx +7 -4
  20. package/src/components/cnpg/CNPGLogicalPath.test.tsx +22 -0
  21. package/src/components/cnpg/CNPGLogicalPath.tsx +150 -0
  22. package/src/components/cnpg/CNPGObjectStoreSummary.tsx +9 -6
  23. package/src/components/cnpg/CNPGObjectSummary.test.tsx +193 -2
  24. package/src/components/cnpg/CNPGPoolerScheduling.test.tsx +36 -0
  25. package/src/components/cnpg/CNPGPoolerSummary.tsx +241 -14
  26. package/src/components/cnpg/CNPGSharedSummary.tsx +15 -8
  27. package/src/components/cnpg/CNPGWALArchivingFact.test.tsx +39 -0
  28. package/src/components/cnpg/backupRuns.test.ts +17 -0
  29. package/src/components/cnpg/backupRuns.ts +19 -0
  30. package/src/components/cnpg/connect.test.ts +124 -0
  31. package/src/components/cnpg/connect.ts +155 -0
  32. package/src/components/cnpg/databaseRole.test.ts +38 -0
  33. package/src/components/cnpg/databaseRole.ts +80 -0
  34. package/src/components/cnpg/ha.test.ts +446 -0
  35. package/src/components/cnpg/ha.ts +621 -0
  36. package/src/components/cnpg/index.ts +18 -0
  37. package/src/components/cnpg/logicalReplication.test.ts +121 -0
  38. package/src/components/cnpg/logicalReplication.ts +252 -0
  39. package/src/components/cnpg/pooler.test.ts +84 -0
  40. package/src/components/cnpg/pooler.ts +225 -0
  41. package/src/components/cnpg/primitives.tsx +21 -117
  42. package/src/components/cnpg/problems.ts +12 -0
  43. package/src/components/cnpg/relations.test.ts +63 -13
  44. package/src/components/cnpg/relations.ts +61 -52
  45. package/src/components/cnpg/schedule.ts +35 -0
  46. package/src/components/cnpg/workspace-disk.test.ts +97 -0
  47. package/src/components/cnpg/workspace-fleet-metrics.test.ts +250 -0
  48. package/src/components/cnpg/workspace-problems.test.ts +175 -0
  49. package/src/components/cnpg/workspace.test.ts +380 -9
  50. package/src/components/cnpg/workspace.ts +1137 -136
  51. package/src/components/dock/DockContext.test.tsx +32 -0
  52. package/src/components/dock/DockContext.tsx +18 -4
  53. package/src/components/dock/LocalTerminalTab.test.tsx +40 -0
  54. package/src/components/dock/LocalTerminalTab.tsx +4 -3
  55. package/src/components/dock/TerminalTab.tsx +20 -8
  56. package/src/components/facts/certainty.tsx +36 -0
  57. package/src/components/facts/facts.test.tsx +24 -0
  58. package/src/components/facts/facts.tsx +62 -0
  59. package/src/components/facts/index.ts +6 -0
  60. package/src/components/facts/managed-by.test.tsx +17 -0
  61. package/src/components/facts/managed-by.tsx +36 -0
  62. package/src/components/facts/public-exports.test.tsx +35 -0
  63. package/src/components/gitops/GitOpsGraphFilterRail.tsx +6 -2
  64. package/src/components/gitops/GitOpsHealthSourceNotice.test.tsx +38 -0
  65. package/src/components/gitops/GitOpsHealthSourceNotice.tsx +13 -3
  66. package/src/components/gitops/GitOpsTableView.tsx +9 -24
  67. package/src/components/gitops/detail-helpers.ts +1 -1
  68. package/src/components/gitops/insights/ArgoResourceDiff.test.tsx +115 -0
  69. package/src/components/gitops/insights/ArgoResourceDiff.tsx +22 -5
  70. package/src/components/gitops/insights/DriftPanel.tsx +62 -0
  71. package/src/components/gitops/insights/GitOpsInsightViews.tsx +29 -83
  72. package/src/components/gitops/tree/GitOpsTreeGraph.test.ts +26 -1
  73. package/src/components/gitops/tree/GitOpsTreeGraph.tsx +40 -60
  74. package/src/components/gitops/tree/layout.test.ts +111 -0
  75. package/src/components/gitops/tree/layout.ts +128 -0
  76. package/src/components/gitops/tree/tree-helpers.test.ts +17 -0
  77. package/src/components/gitops/tree/tree-helpers.ts +7 -1
  78. package/src/components/issues/ConditionalCleanup.test.tsx +82 -0
  79. package/src/components/issues/IssuesView.tsx +40 -5
  80. package/src/components/issues/ResourceIssuesSection.test.tsx +28 -0
  81. package/src/components/issues/ResourceIssuesSection.tsx +3 -0
  82. package/src/components/issues/index.ts +3 -1
  83. package/src/components/issues/issue-timing.ts +14 -3
  84. package/src/components/issues/issues.test.ts +50 -2
  85. package/src/components/issues/severity.ts +21 -0
  86. package/src/components/issues/types.ts +23 -0
  87. package/src/components/logs/LogCore.theme.test.tsx +81 -3
  88. package/src/components/logs/LogCore.tsx +25 -17
  89. package/src/components/logs/LogToolbarSelects.tsx +3 -1
  90. package/src/components/logs/LogsViewer.test.tsx +291 -0
  91. package/src/components/logs/LogsViewer.tsx +72 -11
  92. package/src/components/logs/StructuredLogLine.test.tsx +76 -0
  93. package/src/components/logs/StructuredLogLine.tsx +22 -30
  94. package/src/components/logs/WorkloadLogsViewer.notice.test.tsx +59 -0
  95. package/src/components/logs/WorkloadLogsViewer.test.tsx +65 -0
  96. package/src/components/logs/WorkloadLogsViewer.tsx +44 -10
  97. package/src/components/logs/useLogBuffer.test.ts +8 -0
  98. package/src/components/logs/useLogStream.ts +28 -1
  99. package/src/components/problems/index.ts +1 -0
  100. package/src/components/problems/problems.test.tsx +63 -0
  101. package/src/components/problems/problems.tsx +165 -0
  102. package/src/components/resources/ActiveFilterBar.test.tsx +78 -0
  103. package/src/components/resources/ActiveFilterBar.tsx +86 -0
  104. package/src/components/resources/ResourcesSidebar.keyboard.test.tsx +42 -0
  105. package/src/components/resources/ResourcesSidebar.test.tsx +36 -2
  106. package/src/components/resources/ResourcesSidebar.tsx +92 -41
  107. package/src/components/resources/ResourcesView.kind-filter-memory.test.ts +52 -0
  108. package/src/components/resources/ResourcesView.kind-view.test.tsx +149 -0
  109. package/src/components/resources/ResourcesView.per-container.test.ts +56 -1
  110. package/src/components/resources/ResourcesView.tsx +309 -117
  111. package/src/components/resources/curated-column-ownership.test.ts +3 -3
  112. package/src/components/resources/renderers/AnalysisRunRenderer.tsx +3 -3
  113. package/src/components/resources/renderers/ArgoApplicationRenderer.tsx +6 -5
  114. package/src/components/resources/renderers/AutoscalingReferences.test.tsx +286 -0
  115. package/src/components/resources/renderers/CAPIClusterRenderer.tsx +5 -4
  116. package/src/components/resources/renderers/CAPIExactReferenceNavigation.test.tsx +27 -0
  117. package/src/components/resources/renderers/CAPIKubeadmControlPlaneRenderer.tsx +2 -2
  118. package/src/components/resources/renderers/CAPIMachineDeploymentRenderer.tsx +2 -2
  119. package/src/components/resources/renderers/CAPIMachinePoolRenderer.tsx +2 -2
  120. package/src/components/resources/renderers/CAPIMachineRenderer.tsx +2 -2
  121. package/src/components/resources/renderers/CAPIMachineSetRenderer.tsx +2 -2
  122. package/src/components/resources/renderers/CNPGBackupRenderer.test.tsx +23 -0
  123. package/src/components/resources/renderers/CNPGBackupRenderer.tsx +8 -19
  124. package/src/components/resources/renderers/CNPGClusterRenderer.tsx +7 -8
  125. package/src/components/resources/renderers/CNPGDeclarativeRenderer.tsx +25 -0
  126. package/src/components/resources/renderers/CNPGScheduledBackupRenderer.test.tsx +32 -0
  127. package/src/components/resources/renderers/CNPGScheduledBackupRenderer.tsx +9 -17
  128. package/src/components/resources/renderers/CNPGShared.tsx +2 -2
  129. package/src/components/resources/renderers/ClusterComplianceReportRenderer.tsx +2 -2
  130. package/src/components/resources/renderers/CompositeRenderer.navigation.test.tsx +35 -0
  131. package/src/components/resources/renderers/CompositeRenderer.tsx +3 -3
  132. package/src/components/resources/renderers/CompositionRenderer.tsx +3 -3
  133. package/src/components/resources/renderers/ConfigAuditReportRenderer.tsx +2 -2
  134. package/src/components/resources/renderers/ContourExactReferenceNavigation.test.tsx +34 -0
  135. package/src/components/resources/renderers/ContourHTTPProxyRenderer.tsx +11 -10
  136. package/src/components/resources/renderers/CronJobRenderer.tsx +6 -4
  137. package/src/components/resources/renderers/CronWorkflowRenderer.tsx +3 -2
  138. package/src/components/resources/renderers/EndpointSliceRenderer.test.tsx +62 -0
  139. package/src/components/resources/renderers/EndpointSliceRenderer.tsx +15 -14
  140. package/src/components/resources/renderers/EventRenderer.tsx +3 -3
  141. package/src/components/resources/renderers/ExperimentRenderer.tsx +2 -2
  142. package/src/components/resources/renderers/ExposedSecretReportRenderer.tsx +2 -2
  143. package/src/components/resources/renderers/ExternalSecretRenderer.tsx +2 -2
  144. package/src/components/resources/renderers/FluxHelmReleaseRenderer.tsx +4 -4
  145. package/src/components/resources/renderers/GRPCRouteRenderer.tsx +11 -24
  146. package/src/components/resources/renderers/GatewayRouteReferences.test.tsx +63 -0
  147. package/src/components/resources/renderers/GitRepositoryRenderer.tsx +3 -3
  148. package/src/components/resources/renderers/HPARenderer.tsx +57 -17
  149. package/src/components/resources/renderers/HTTPRouteRenderer.tsx +12 -25
  150. package/src/components/resources/renderers/HelmRepositoryRenderer.tsx +3 -3
  151. package/src/components/resources/renderers/IngressClassReferences.test.tsx +52 -0
  152. package/src/components/resources/renderers/IngressClassRenderer.tsx +10 -3
  153. package/src/components/resources/renderers/IngressRenderer.defaultBackend.test.tsx +33 -0
  154. package/src/components/resources/renderers/IngressRenderer.tsx +36 -5
  155. package/src/components/resources/renderers/IstioDestinationRuleRenderer.tsx +9 -9
  156. package/src/components/resources/renderers/IstioExactReferenceNavigation.test.tsx +34 -0
  157. package/src/components/resources/renderers/IstioVirtualServiceRenderer.tsx +5 -8
  158. package/src/components/resources/renderers/KarpenterNodeClaimRenderer.tsx +2 -14
  159. package/src/components/resources/renderers/KedaCredentials.test.tsx +85 -0
  160. package/src/components/resources/renderers/KedaScaledObjectRenderer.tsx +15 -13
  161. package/src/components/resources/renderers/KedaTriggerAuthRenderer.tsx +6 -3
  162. package/src/components/resources/renderers/KnativeEventingRenderer.tsx +24 -31
  163. package/src/components/resources/renderers/KnativeExactReferenceNavigation.test.tsx +47 -0
  164. package/src/components/resources/renderers/KnativeFlowRenderer.tsx +6 -7
  165. package/src/components/resources/renderers/KnativeNetworkingRenderer.tsx +7 -9
  166. package/src/components/resources/renderers/KnativeReferenceLink.tsx +14 -0
  167. package/src/components/resources/renderers/KnativeReferenceLinks.test.tsx +60 -0
  168. package/src/components/resources/renderers/KnativeSourceRenderer.tsx +9 -11
  169. package/src/components/resources/renderers/KustomizationRenderer.tsx +2 -2
  170. package/src/components/resources/renderers/KyvernoGlobalContextRenderer.tsx +3 -2
  171. package/src/components/resources/renderers/KyvernoQueueRenderers.tsx +4 -3
  172. package/src/components/resources/renderers/LeaseRenderer.tsx +3 -15
  173. package/src/components/resources/renderers/ManagedResourceRenderer.tsx +2 -2
  174. package/src/components/resources/renderers/MetricsUnavailableNotice.tsx +3 -2
  175. package/src/components/resources/renderers/NamespaceRenderer.test.ts +69 -2
  176. package/src/components/resources/renderers/NamespaceRenderer.tsx +6 -1
  177. package/src/components/resources/renderers/NodeRenderer.test.tsx +81 -1
  178. package/src/components/resources/renderers/NodeRenderer.tsx +56 -15
  179. package/src/components/resources/renderers/OCIRepositoryRenderer.tsx +3 -3
  180. package/src/components/resources/renderers/PodClassReferences.test.tsx +42 -0
  181. package/src/components/resources/renderers/PodNomination.test.tsx +57 -0
  182. package/src/components/resources/renderers/PodRenderer.test.tsx +32 -0
  183. package/src/components/resources/renderers/PodRenderer.tsx +62 -36
  184. package/src/components/resources/renderers/RolloutRenderer.test.tsx +1 -1
  185. package/src/components/resources/renderers/RolloutRenderer.tsx +3 -2
  186. package/src/components/resources/renderers/SbomReportRenderer.tsx +2 -2
  187. package/src/components/resources/renderers/SecretRenderer.test.tsx +88 -0
  188. package/src/components/resources/renderers/SecretRenderer.tsx +34 -5
  189. package/src/components/resources/renderers/ServiceRenderer.test.tsx +43 -0
  190. package/src/components/resources/renderers/ServiceRenderer.tsx +11 -3
  191. package/src/components/resources/renderers/SimpleRouteRenderer.tsx +9 -22
  192. package/src/components/resources/renderers/VPARenderer.tsx +17 -11
  193. package/src/components/resources/renderers/VeleroBSLRenderer.tsx +5 -7
  194. package/src/components/resources/renderers/VeleroBackupRenderer.tsx +3 -3
  195. package/src/components/resources/renderers/VeleroBackupRepositoryRenderer.tsx +2 -2
  196. package/src/components/resources/renderers/VeleroRestoreRenderer.tsx +3 -3
  197. package/src/components/resources/renderers/VeleroScheduleRenderer.tsx +2 -2
  198. package/src/components/resources/renderers/VulnerabilityReportRenderer.tsx +2 -2
  199. package/src/components/resources/renderers/WorkflowRenderer.tsx +4 -3
  200. package/src/components/resources/renderers/kyverno-modern-cells.tsx +1 -1
  201. package/src/components/resources/renderers/rollout/ReplicaSetProgression.tsx +2 -2
  202. package/src/components/resources/resource-utils-capi.ts +1 -1
  203. package/src/components/resources/resource-utils-cnpg.test.ts +35 -10
  204. package/src/components/resources/resource-utils-cnpg.ts +51 -24
  205. package/src/components/resources/resource-utils-contour.test.ts +44 -0
  206. package/src/components/resources/resource-utils-contour.ts +16 -0
  207. package/src/components/resources/resource-utils-istio.ts +12 -8
  208. package/src/components/resources/resource-utils-keda.ts +1 -7
  209. package/src/components/resources/resource-utils-knative.ts +18 -0
  210. package/src/components/resources/resource-utils-kyverno.ts +1 -7
  211. package/src/components/resources/resource-utils-velero.ts +0 -12
  212. package/src/components/resources/resource-utils.ts +99 -39
  213. package/src/components/resources/rules-column-width.test.ts +1 -1
  214. package/src/components/resources/summarize-scheduler-message.test.ts +30 -0
  215. package/src/components/shared/ActionConfirmDialog.test.tsx +154 -0
  216. package/src/components/shared/ActionConfirmDialog.tsx +246 -0
  217. package/src/components/shared/CreateResourceDialog.test.tsx +330 -0
  218. package/src/components/shared/CreateResourceDialog.tsx +126 -80
  219. package/src/components/shared/DetailShell.test.tsx +80 -0
  220. package/src/components/shared/DetailShell.tsx +85 -10
  221. package/src/components/shared/DrainPlanDialog.tsx +1 -0
  222. package/src/components/shared/EditableYamlView.test.tsx +7 -0
  223. package/src/components/shared/EditableYamlView.tsx +1 -1
  224. package/src/components/shared/GitOpsWriteWarning.test.tsx +86 -0
  225. package/src/components/shared/GitOpsWriteWarning.tsx +135 -0
  226. package/src/components/shared/ResourceActionsBar.tsx +20 -24
  227. package/src/components/shared/ResourceRendererDispatch.tsx +13 -10
  228. package/src/components/shared/SetImageDialog.test.ts +75 -6
  229. package/src/components/shared/SetImageDialog.tsx +92 -84
  230. package/src/components/shared/index.ts +4 -1
  231. package/src/components/timeline/TimelineList.tsx +2 -15
  232. package/src/components/timeline/TimelineSwimlanes.tsx +3 -11
  233. package/src/components/timeline/timeline-filters.ts +1 -0
  234. package/src/components/topology/K8sResourceNode.test.ts +18 -1
  235. package/src/components/topology/K8sResourceNode.tsx +44 -2
  236. package/src/components/topology/TopologyFilterSidebar.tsx +1 -0
  237. package/src/components/topology/TopologyGraph.tsx +8 -74
  238. package/src/components/topology/TopologySearch.tsx +1 -0
  239. package/src/components/topology/internet-groups.test.ts +35 -0
  240. package/src/components/topology/internet-groups.ts +55 -0
  241. package/src/components/topology/layout.ts +1 -1
  242. package/src/components/ui/Badge.tsx +1 -0
  243. package/src/components/ui/FoldSection.tsx +68 -0
  244. package/src/components/ui/ForceDeleteConfirmDialog.test.tsx +62 -0
  245. package/src/components/ui/ForceDeleteConfirmDialog.tsx +24 -12
  246. package/src/components/ui/RefLink.tsx +20 -0
  247. package/src/components/ui/RowActionMenu.tsx +8 -3
  248. package/src/components/ui/ScheduleValue.test.tsx +5 -0
  249. package/src/components/ui/ScheduleValue.tsx +3 -2
  250. package/src/components/ui/crd-definition-navigation.test.tsx +160 -0
  251. package/src/components/ui/drawer-components.test.tsx +175 -1
  252. package/src/components/ui/drawer-components.tsx +103 -36
  253. package/src/components/ui/index.ts +3 -1
  254. package/src/components/ui/monacoEditorFeatures.ts +77 -0
  255. package/src/components/ui/monacoRuntime.ts +14 -19
  256. package/src/components/ui/status-tone.test.ts +10 -1
  257. package/src/components/ui/status-tone.tsx +29 -0
  258. package/src/components/workload/WorkloadView.deleted.test.tsx +101 -0
  259. package/src/components/workload/WorkloadView.deletedDialog.test.tsx +45 -0
  260. package/src/components/workload/WorkloadView.relationships.test.tsx +37 -0
  261. package/src/components/workload/WorkloadView.secretSave.test.tsx +62 -0
  262. package/src/components/workload/WorkloadView.spec.test.tsx +149 -0
  263. package/src/components/workload/WorkloadView.tsx +297 -104
  264. package/src/index.ts +14 -1
  265. package/src/monaco-deep.d.ts +19 -12
  266. package/src/theme/components.css +13 -0
  267. package/src/types/capacity.ts +21 -2
  268. package/src/types/core.ts +60 -2
  269. package/src/types/gitops-flux-status.test.ts +32 -1
  270. package/src/types/gitops-insights.ts +19 -2
  271. package/src/types/gitops-tree.ts +10 -0
  272. package/src/types/gitops.ts +26 -6
  273. package/src/utils/api-resources.ts +7 -0
  274. package/src/utils/application-history.ts +1 -0
  275. package/src/utils/cnpg-backup.test.ts +26 -0
  276. package/src/utils/cnpg-backup.ts +70 -0
  277. package/src/utils/format.test.ts +49 -2
  278. package/src/utils/format.ts +9 -1
  279. package/src/utils/gateway-references.test.ts +17 -0
  280. package/src/utils/gateway-references.ts +29 -0
  281. package/src/utils/gitops-owner.ts +3 -2
  282. package/src/utils/gitops-write-guard.test.ts +401 -0
  283. package/src/utils/gitops-write-guard.ts +509 -0
  284. package/src/utils/grant.test.ts +19 -0
  285. package/src/utils/grant.ts +35 -0
  286. package/src/utils/index.ts +6 -0
  287. package/src/utils/ingress-class-references.ts +22 -0
  288. package/src/utils/k8s-errors.ts +7 -7
  289. package/src/utils/log-format.test.ts +41 -0
  290. package/src/utils/log-format.ts +71 -15
  291. package/src/utils/log-level.ts +12 -8
  292. package/src/utils/navigation.test.ts +70 -0
  293. package/src/utils/navigation.ts +91 -8
  294. package/src/utils/node-lifecycle.test.ts +36 -0
  295. package/src/utils/node-lifecycle.ts +77 -0
  296. package/src/utils/object-reference-navigation.test.ts +43 -0
  297. package/src/utils/resource-hierarchy.test.ts +69 -0
  298. package/src/utils/resource-hierarchy.ts +41 -7
  299. package/src/utils/resource-icons.ts +2 -0
  300. package/src/utils/resource-refs.test.ts +23 -0
  301. package/src/utils/resource-refs.ts +30 -0
  302. package/src/utils/skeleton-yaml.test.ts +21 -0
  303. package/src/utils/skeleton-yaml.ts +16 -0
  304. package/src/utils/url-path.test.ts +35 -0
  305. package/src/utils/url-path.ts +21 -0
  306. package/src/utils/validators.test.ts +4 -13
  307. package/src/utils/validators.ts +5 -19
  308. package/src/utils/view-transition.test.ts +35 -1
  309. package/src/utils/view-transition.ts +16 -1
@@ -1,18 +1,31 @@
1
+ import { cnpgBackupDeclaration, cnpgBarmanPlugin } from '../../utils/cnpg-backup'
1
2
  // CloudNativePG workspace model: pure derivations over the /api/cnpg/workspace
2
3
  // payload. Every fact here is something the cluster actually reports; when it
3
4
  // does not report something the value is "unknown", never zero or healthy.
4
5
 
5
- import type { HealthLevel } from '../resources/resource-utils'
6
+ import { backupsForScheduledBackup, cnpgScheduleDestinationBlocker, cnpgBackupMatchesCluster, cnpgArchiveMatchesCluster, targetCluster } from './relations'
7
+ import { cnpgRoleState } from './databaseRole'
8
+ import { formatAge, summarizeSchedulerMessage, type HealthLevel } from '../resources/resource-utils'
9
+ import { worseTone } from '../ui/status-tone'
10
+ import { type Fact } from '../facts'
11
+ import { type ProblemOrigin, type ProblemDetails, type ProblemProvenance } from '../problems'
12
+ import type { ResourceRef } from '../../types/core'
13
+ import { formatGrant, type Grant } from '../../utils/grant'
14
+ import { issueReasonTitle, issueTitle } from '../issues/severity'
6
15
  import {
7
16
  CNPG_BARMAN_PLUGIN_NAME,
8
17
  getCNPGClusterBackupConfig,
9
18
  getCNPGClusterBarmanPlugin,
10
19
  getCNPGClusterImageTag,
20
+ getCNPGClusterIsReplica,
11
21
  getCNPGClusterStatus,
12
22
  getCNPGObjectStoreRecoveryWindows,
13
23
  isApiGroup,
14
24
  } from '../resources/resource-utils-cnpg'
15
25
 
26
+ export type CNPGFactTone = HealthLevel
27
+ export type CNPGFact = Fact
28
+
16
29
  export const CNPG_WORKSPACE_KEYS = [
17
30
  'clusters',
18
31
  'backups',
@@ -21,6 +34,7 @@ export const CNPG_WORKSPACE_KEYS = [
21
34
  'databases',
22
35
  'publications',
23
36
  'subscriptions',
37
+ 'databaseRoles',
24
38
  'imageCatalogs',
25
39
  'clusterImageCatalogs',
26
40
  'objectStores',
@@ -29,12 +43,14 @@ export const CNPG_WORKSPACE_KEYS = [
29
43
 
30
44
  export type CNPGWorkspaceKey = (typeof CNPG_WORKSPACE_KEYS)[number]
31
45
 
32
- export type CNPGCoverageState = 'full' | 'partial' | 'denied' | 'notInstalled' | 'syncing' | 'error'
46
+ export type CNPGCoverageState = 'full' | 'partial' | 'denied' | 'notInstalled' | 'syncing' | 'uncached' | 'error'
33
47
 
34
48
  export interface CNPGKindCoverage {
35
49
  state: CNPGCoverageState
36
50
  /** Denied namespaces, named only when the caller supplied the candidate list. */
37
51
  deniedNamespaces?: string[]
52
+ /** Namespaces the caller may read but Radar's cache does not hold, named under the same rule. */
53
+ uncachedNamespaces?: string[]
38
54
  /** For partial coverage: the namespaces that were read. */
39
55
  allowedNamespaces?: string[]
40
56
  }
@@ -73,6 +89,14 @@ export interface CNPGWorkspaceResponse {
73
89
  issues: CNPGWorkspaceIssue[]
74
90
  audit: CNPGAuditFinding[]
75
91
  backupsOmitted: number
92
+ /** Each ScheduledBackup's schedule as the operator reads it, keyed "namespace/name"; absent when it cannot be parsed. */
93
+ scheduleReadings?: Record<string, string>
94
+ /** The manager of each object that has one, keyed "Kind/namespace/name". */
95
+ managedBy?: Record<string, ResourceRef>
96
+ /** Pods of the Jobs a Cluster controls (initdb, join, restore…), never counted as instances. */
97
+ jobPods?: any[]
98
+ /** Where the caller's Jobs were read; a Job Pod is returned only there. Absent from a Radar that predates it. */
99
+ jobCoverage?: CNPGKindCoverage
76
100
  }
77
101
 
78
102
  export const CNPG_KIND_BY_KEY: Record<CNPGWorkspaceKey, { kind: string; group: string; plural: string }> = {
@@ -83,6 +107,7 @@ export const CNPG_KIND_BY_KEY: Record<CNPGWorkspaceKey, { kind: string; group: s
83
107
  databases: { kind: 'Database', group: 'postgresql.cnpg.io', plural: 'databases' },
84
108
  publications: { kind: 'Publication', group: 'postgresql.cnpg.io', plural: 'publications' },
85
109
  subscriptions: { kind: 'Subscription', group: 'postgresql.cnpg.io', plural: 'subscriptions' },
110
+ databaseRoles: { kind: 'DatabaseRole', group: 'postgresql.cnpg.io', plural: 'databaseroles' },
86
111
  imageCatalogs: { kind: 'ImageCatalog', group: 'postgresql.cnpg.io', plural: 'imagecatalogs' },
87
112
  clusterImageCatalogs: { kind: 'ClusterImageCatalog', group: 'postgresql.cnpg.io', plural: 'clusterimagecatalogs' },
88
113
  objectStores: { kind: 'ObjectStore', group: 'barmancloud.cnpg.io', plural: 'objectstores' },
@@ -93,37 +118,31 @@ export function isCNPGWorkspaceKind(kind: string, group: string | undefined): bo
93
118
  return Object.values(CNPG_KIND_BY_KEY).some((k) => k.group !== '' && k.group === (group ?? '') && k.kind === kind)
94
119
  }
95
120
 
96
- /** The value is observed, derived, or not available from the cluster. */
97
- export type CNPGFactTone = HealthLevel
98
-
99
- export interface CNPGFact {
100
- text: string
101
- tone: CNPGFactTone
102
- /** Where the value comes from, shown next to it so claims carry their source. */
103
- source?: string
104
- /** A timestamp the text refers to; the UI renders it as an age. */
105
- at?: string
106
- }
107
-
108
121
  export type CNPGProblemCategory = 'availability' | 'protection' | 'declarations' | 'pooling'
109
122
 
110
123
  export const CNPG_PROBLEM_CATEGORIES: { id: CNPGProblemCategory; label: string }[] = [
111
124
  { id: 'availability', label: 'Availability' },
112
- { id: 'protection', label: 'Protection' },
125
+ { id: 'protection', label: 'Backups' },
113
126
  { id: 'declarations', label: 'Declarations' },
114
127
  { id: 'pooling', label: 'Pooling' },
115
128
  ]
116
129
 
117
- export interface CNPGProblem {
118
- /** Stable identity for keys. */
119
- id: string
130
+ /**
131
+ * A problem in the CloudNativePG workspace, categorised by the workspace's
132
+ * screens. `instance` names the instance a Cluster-level problem is about,
133
+ * e.g. the standby an HA slot is kept for.
134
+ */
135
+ export type CNPGProblem = ProblemDetails<CNPGProblemCategory> & ProblemProvenance & {
120
136
  severity: 'critical' | 'warning' | 'posture'
121
- category: CNPGProblemCategory
122
- title: string
123
- detail?: string
124
- /** The object the evidence is about (may be the Cluster or a child object). */
125
- subject: { kind: string; group: string; namespace: string; name: string }
126
- source: 'issue' | 'audit'
137
+ /** The measurement matched by name rather than by UID. */
138
+ unverifiedMatch?: boolean
139
+ reason?: string
140
+ slot?: string
141
+ instance?: string
142
+ /** The cnpg.io/jobRole of the Cluster's own Job whose Pod this is about: a cause, not an instance's symptom. */
143
+ job?: string
144
+ /** The Cluster's own Ready condition: a roll-up of the other problems, shown after them. */
145
+ rollup?: boolean
127
146
  }
128
147
 
129
148
  export interface CNPGInstance {
@@ -135,15 +154,15 @@ export interface CNPGInstance {
135
154
  }
136
155
 
137
156
  export interface CNPGProtectionFacts {
138
- schedule: CNPGFact & { names: string[] }
139
- destination: CNPGFact & {
157
+ schedule: Fact & { names: string[] }
158
+ destination: Fact & {
140
159
  method: 'plugin' | 'barmanObjectStore' | 'volumeSnapshot' | 'none'
141
160
  objectStore?: string
142
161
  }
143
- lastSuccessfulBackup: CNPGFact
144
- walArchiving: CNPGFact
145
- recoveryWindow: CNPGFact & { from?: string }
146
- restoreValidation: CNPGFact & { restoredInto?: { namespace: string; name: string } }
162
+ lastSuccessfulBackup: Fact
163
+ walArchiving: Fact & { state?: 'no_destination' | 'failing' | 'archiving' | 'unknown'; operatorCondition?: { type: string; status: string; message?: string; lastTransitionTime?: string } }
164
+ recoveryWindow: Fact & { from?: string }
165
+ restoreValidation: Fact & { restoredInto?: { namespace: string; name: string } }
147
166
  }
148
167
 
149
168
  export interface CNPGFleetRow {
@@ -153,15 +172,26 @@ export interface CNPGFleetRow {
153
172
  cluster: any
154
173
  controllerStatus: { text: string; level: HealthLevel }
155
174
  instances: { ready: number | null; desired: number | null }
175
+ /**
176
+ * Ready instances counted from the instance Pods' Ready condition; absent
177
+ * when Pods are not readable here or any Pod's readiness is unknown.
178
+ */
179
+ podReadiness?: { ready: number; total: number }
180
+ /** status.readyInstances claims more ready instances than the Pods show: CNPG status is stale or lagging. */
181
+ readinessContradicted?: boolean
182
+ /** status.currentPrimary is not the Pod labelled primary; status may be stale, or a failover is under way. */
183
+ primaryConflict?: { status: string; labelled: string }
156
184
  pods: CNPGInstance[]
157
185
  replicaCluster: { source?: string } | null
158
186
  hibernated: boolean
159
187
  pgVersion: string | null
160
188
  catalog: { kind: string; name: string } | null
161
- replication: CNPGFact
162
- protection: CNPGProtectionFacts & { summary: CNPGFact }
163
- declarations: { summary: CNPGFact; total: number; failed: number; pending: number }
189
+ replication: Fact
190
+ protection: CNPGProtectionFacts & { summary: Fact }
191
+ declarations: { summary: Fact; total: number; failed: number; pending: number }
164
192
  poolers: string[]
193
+ /** The Pooler objects behind `poolers`, for their type and Service port. */
194
+ poolerObjects?: any[]
165
195
  /** False when Poolers are not readable in this cluster's namespace, so an empty list means unknown. */
166
196
  poolersKnown: boolean
167
197
  problems: CNPGProblem[]
@@ -169,7 +199,12 @@ export interface CNPGFleetRow {
169
199
  attention: boolean
170
200
  categories: Set<CNPGProblemCategory>
171
201
  /** GitOps owner recorded on the Cluster, when it carries the standard labels. */
172
- gitops: CNPGGitOpsSource | null
202
+ /** The GitOps or Helm object that manages the Cluster, as the server detected it. */
203
+ managedBy?: ResourceRef
204
+ /** Fullest measured volume, set by applyCNPGDisk; absent when no disk reading was requested. */
205
+ disk?: Fact
206
+ /** Growth of the fastest-growing volume, set by applyCNPGFleetMetrics when measured. */
207
+ diskGrowth?: Fact
173
208
  }
174
209
 
175
210
  export interface CNPGFleet {
@@ -185,8 +220,195 @@ const PROTECTION_ISSUE_REASONS = new Set([
185
220
  'CNPGLastBackupFailed',
186
221
  'CNPGBackupFailed',
187
222
  'CNPGScheduledBackupMissed',
223
+ 'CNPGScheduledRunNoBackup',
224
+ 'CNPGScheduleDestinationMissing',
188
225
  ])
189
226
 
227
+ // What an instance Pod's bare reason means, said about the Pod.
228
+ const CNPG_POD_REASON_SENTENCES: Record<string, string> = {
229
+ ReadinessProbeFailed: 'not ready (readiness probe failing)',
230
+ // The issue does not say whether the Pod is serving now (it may have come
231
+ // back within the settle window), so restarts are worded as past.
232
+ LivenessProbeFailed: 'restarted recently (liveness probe failing)',
233
+ CrashLoopBackOff: 'restarted recently (CrashLoopBackOff)',
234
+ HighRestartCount: 'restarted repeatedly',
235
+ OOMKilled: 'killed for running out of memory (OOMKilled)',
236
+ ImagePullBackOff: 'cannot pull its image (ImagePullBackOff)',
237
+ ErrImagePull: 'cannot pull its image (ErrImagePull)',
238
+ }
239
+
240
+ // Plain headlines for CNPG issues whose message carries the operator's own
241
+ // condition text; that message becomes the detail beneath.
242
+ // Reasons the Issues page already titles come from issueReasonTitle, so a
243
+ // problem reads the same here and there; these are the rest.
244
+ const CNPG_REASON_TITLES: Record<string, string> = {
245
+ CNPGBackupFailed: 'Backup failed',
246
+ CNPGScheduledBackupMissed: 'A scheduled backup did not run',
247
+ CNPGCertificateExpiring: 'A certificate expires soon',
248
+ CNPGCertificateExpired: 'A certificate has expired',
249
+ }
250
+
251
+ /**
252
+ * A problem's headline and detail from an issue. Known CNPG reasons get a
253
+ * short plain title with the operator's message beneath; otherwise the
254
+ * message is the title, unless it is empty or only the reason token (e.g.
255
+ * "ReadinessProbeFailed"), which is turned into a sentence about the subject.
256
+ */
257
+ export function cnpgIssueText(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'name' | 'reason' | 'message' | 'cause' | 'first_seen'>): { title: string; detail?: string } {
258
+ let message = issue.message?.trim() ?? ''
259
+ const cause = issue.cause?.trim() || undefined
260
+ // The run's time is the issue's first_seen, not part of the message.
261
+ if (issue.reason === 'CNPGScheduledRunNoBackup' && issue.first_seen && message) message = `${message} ${formatAge(issue.first_seen)} ago`
262
+ if (['CNPGScheduleDestinationMissing', 'CNPGInstanceReadinessMismatch', 'CNPGPrimaryLabelMismatch'].includes(issue.reason)) return { title: message, detail: cause }
263
+ const known = issueReasonTitle(issue.reason) ?? CNPG_REASON_TITLES[issue.reason]
264
+ if (known) return { title: known, detail: [stripTitlePrefix(message, known), cause].filter(Boolean).join(' ') || undefined }
265
+ if (message && message !== issue.reason && /\s/.test(message)) return { title: message, detail: cause }
266
+ const token = message || issue.reason
267
+ const sentence = CNPG_POD_REASON_SENTENCES[token]
268
+ if (sentence) return { title: `${issue.name} ${sentence}`, detail: cause }
269
+ const words = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase()
270
+ return { title: `${issue.kind} ${issue.name}: ${words}`, detail: cause }
271
+ }
272
+
273
+ // "Backup failed: cannot proceed…" under the title "Backup failed" repeats it.
274
+ function stripTitlePrefix(message: string, title: string): string {
275
+ if (!message.toLowerCase().startsWith(title.toLowerCase())) return message
276
+ const rest = message.slice(title.length).replace(/^[\s:;,.\-–—]+/, '')
277
+ return rest ? rest[0].toUpperCase() + rest.slice(1) : ''
278
+ }
279
+
280
+ /**
281
+ * Failed Backups of one cluster that failed for the same reason become one
282
+ * problem: "3 backups failed: <reason>", about the latest of them, the others
283
+ * named in alsoAbout. Each Backup is otherwise its own issue, and a schedule
284
+ * failing every night would list the same sentence over and over.
285
+ */
286
+ type IssueProblem = CNPGProblem
287
+
288
+ export function cnpgCollapseBackupFailures(problems: IssueProblem[], backupTimes: Map<string, number> = new Map()): IssueProblem[] {
289
+ const groups = new Map<string, IssueProblem[]>()
290
+ const out: IssueProblem[] = []
291
+ for (const p of problems) {
292
+ if (p.reason !== 'CNPGBackupFailed' || p.subject.kind !== 'Backup') {
293
+ out.push(p)
294
+ continue
295
+ }
296
+ const key = `${p.severity}\x00${p.detail ?? ''}`
297
+ groups.set(key, [...(groups.get(key) ?? []), p])
298
+ }
299
+ for (const list of groups.values()) {
300
+ if (list.length === 1) {
301
+ out.push(list[0])
302
+ continue
303
+ }
304
+ const at = (p: IssueProblem) => backupTimes.get(p.subject.name) ?? 0
305
+ const sorted = [...list].sort((a, b) => at(b) - at(a) || b.subject.name.localeCompare(a.subject.name))
306
+ const latest = sorted[0]
307
+ out.push({
308
+ ...latest,
309
+ id: `backups-failed:${latest.subject.namespace}:${latest.detail ?? ''}`,
310
+ title: latest.detail ? `${list.length} backups failed: ${latest.detail[0].toLowerCase()}${latest.detail.slice(1)}` : `${list.length} backups failed`,
311
+ detail: undefined,
312
+ alsoAbout: sorted.slice(1).map((p) => ({ kind: p.subject.kind, name: p.subject.name })),
313
+ })
314
+ }
315
+ return out
316
+ }
317
+
318
+ /**
319
+ * "Latest backup failed" restates a failed-Backup problem when that problem
320
+ * is about the cluster's newest Backup, so the duplicate is dropped and the
321
+ * count stays honest. With the newest Backup unknown, both stay.
322
+ */
323
+ export function cnpgFoldLastBackupFailed(problems: IssueProblem[], newestBackup: string | undefined): CNPGProblem[] {
324
+ const covered =
325
+ !!newestBackup &&
326
+ problems.some(
327
+ (p) =>
328
+ p.reason === 'CNPGBackupFailed' &&
329
+ p.subject.kind === 'Backup' &&
330
+ (p.subject.name === newestBackup || p.alsoAbout?.some((o) => o.kind === 'Backup' && o.name === newestBackup)),
331
+ )
332
+ return problems
333
+ .filter((p) => !(covered && p.reason === 'CNPGLastBackupFailed'))
334
+ }
335
+
336
+ // When each of the cluster's Backups started (status.startedAt, else its
337
+ // creation), by name: the issues about Backups carry no time of their own.
338
+ function backupTimesOf(cluster: any, backups: any[]): Map<string, number> {
339
+ const out = new Map<string, number>()
340
+ for (const b of backups) {
341
+ if (!cnpgBackupMatchesCluster(b, cluster)) continue
342
+ out.set(b.metadata.name, Date.parse(b?.status?.startedAt ?? b?.metadata?.creationTimestamp ?? '') || 0)
343
+ }
344
+ return out
345
+ }
346
+
347
+
348
+ // Each entry names what the Go detector (internal/issues/source_cnpg*.go and
349
+ // the Pod detector) actually reads. "Reported by CNPG" only where the operator
350
+ // itself wrote the failure; a threshold or comparison Radar applies is a
351
+ // "Radar check".
352
+ const CNPG_CONDITION_ORIGINS: Record<string, string> = {
353
+ CNPGLastBackupFailed: 'Cluster LastBackupSucceeded condition',
354
+ CNPGClusterTerminal: 'Cluster status.phase',
355
+ CNPGClusterUnrecoverable: 'Cluster status.phase',
356
+ CNPGClusterPluginFailure: 'Cluster status.phase',
357
+ CNPGClusterFailingOver: 'Cluster status.phase',
358
+ CNPGClusterWaitingForUser: 'Cluster status.phase',
359
+ CNPGDeclarativeNotApplied: 'status.applied and status.message',
360
+ }
361
+
362
+ const POD_ORIGINS: Record<string, ProblemOrigin> = {
363
+ ReadinessProbeFailed: { label: 'Pod readiness probe', detail: 'Kubelet probe-failure events and the Pod\'s Ready condition' },
364
+ LivenessProbeFailed: { label: 'Pod liveness probe', detail: 'Kubelet probe-failure events and container restarts' },
365
+ ReadinessProbeInvalid: { label: 'Radar check of the probe', detail: 'The readiness probe names a port the container does not declare' },
366
+ LivenessProbeInvalid: { label: 'Radar check of the probe', detail: 'The liveness probe names a port the container does not declare' },
367
+ HighRestartCount: { label: 'Radar check of restarts', detail: 'More than 3 restarts on a container that is still unhealthy' },
368
+ InitContainerStalled: { label: 'Radar check of init containers', detail: 'An init container has not finished' },
369
+ Unschedulable: { label: 'Kubernetes scheduler', detail: "The Pod's PodScheduled condition (reason Unschedulable) and the scheduler's message" },
370
+ }
371
+
372
+ /**
373
+ * Where an issue's evidence comes from, in user terms: what CloudNativePG
374
+ * reported, a Backup's or Pod's own status, or Radar's own check. A reason this
375
+ * does not know reads "Detected by Radar" rather than a guessed source.
376
+ */
377
+ export function cnpgIssueOrigin(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'reason'>): ProblemOrigin {
378
+ const condition = CNPG_CONDITION_ORIGINS[issue.reason]
379
+ if (condition) return { label: 'Reported by CNPG', detail: condition }
380
+ switch (issue.reason) {
381
+ case 'CNPGWALArchivingFailing':
382
+ return issue.kind === 'Backup'
383
+ ? { label: 'Backup status', detail: 'Backup status.phase walArchivingFailing and status.error' }
384
+ : { label: 'Reported by CNPG', detail: 'Cluster ContinuousArchiving condition' }
385
+ case 'CNPGBackupFailed':
386
+ return { label: 'Backup status', detail: 'Backup status.phase and status.error' }
387
+ case 'CNPGClusterDegraded':
388
+ return { label: 'Radar check of ready instances', detail: 'spec.instances against status.readyInstances, unless the phase, hibernation or fencing explains it' }
389
+ case 'CNPGScheduleDestinationMissing':
390
+ return { label: 'Radar check of the backup destination', detail: 'ScheduledBackup method against its target Cluster spec' }
391
+ case 'CNPGInstanceReadinessMismatch':
392
+ return { label: 'Radar check of instance readiness', detail: 'Instance Pod Ready conditions against Cluster status.readyInstances' }
393
+ case 'CNPGPrimaryLabelMismatch':
394
+ return { label: 'Radar check of the primary', detail: 'Instance Pod role labels against Cluster status.currentPrimary' }
395
+ case 'CNPGScheduledRunNoBackup':
396
+ return { label: 'Radar check of the backup schedule', detail: 'The schedule, read as the operator does, against the cluster\'s newest successful backup' }
397
+ case 'CNPGScheduledBackupMissed':
398
+ return { label: 'Radar check of the backup schedule', detail: 'ScheduledBackup status.nextScheduleTime passed more than 10 minutes ago' }
399
+ case 'CNPGCertificateExpiring':
400
+ case 'CNPGCertificateExpired':
401
+ return { label: 'Certificate expiry (from Cluster status)', detail: 'Cluster status.certificates.expirations, compared with now' }
402
+ }
403
+ if (issue.kind === 'Pod') return POD_ORIGINS[issue.reason] ?? { label: 'Pod status' }
404
+ return { label: 'Detected by Radar' }
405
+ }
406
+
407
+ /** The headline alone; see cnpgIssueText. */
408
+ export function cnpgIssueTitle(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'name' | 'reason' | 'message'>): string {
409
+ return cnpgIssueText(issue).title
410
+ }
411
+
190
412
  export function cnpgIssueCategory(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'reason'>): CNPGProblemCategory {
191
413
  if (PROTECTION_ISSUE_REASONS.has(issue.reason)) return 'protection'
192
414
  switch (issue.kind) {
@@ -197,6 +419,7 @@ export function cnpgIssueCategory(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'reas
197
419
  case 'Database':
198
420
  case 'Publication':
199
421
  case 'Subscription':
422
+ case 'DatabaseRole':
200
423
  return 'declarations'
201
424
  case 'Pooler':
202
425
  return 'pooling'
@@ -215,7 +438,7 @@ function specClusterName(obj: any): string | undefined {
215
438
  }
216
439
 
217
440
  function coverageOf(resp: CNPGWorkspaceResponse, k: CNPGWorkspaceKey): CNPGKindCoverage {
218
- return resp.coverage?.[k] ?? { state: 'notInstalled' }
441
+ return resp.coverage[k] ?? { state: 'uncached' }
219
442
  }
220
443
 
221
444
  /** Coverage is usable in a namespace when that namespace's objects were read. */
@@ -224,23 +447,35 @@ export function coverageReadable(cov: CNPGKindCoverage, namespace?: string): boo
224
447
  if (cov.state === 'partial') {
225
448
  if (!namespace) return false
226
449
  if (cov.allowedNamespaces) return cov.allowedNamespaces.includes(namespace)
227
- if (cov.deniedNamespaces) return !cov.deniedNamespaces.includes(namespace)
450
+ if (cov.deniedNamespaces || cov.uncachedNamespaces) return !cov.deniedNamespaces?.includes(namespace) && !cov.uncachedNamespaces?.includes(namespace)
228
451
  return false
229
452
  }
230
453
  return false
231
454
  }
232
455
 
233
- function coverageUnavailableText(cov: CNPGKindCoverage, what: string): string {
456
+ /**
457
+ * Why a kind's objects were not read (in `namespace`, when given), worded for
458
+ * a fact. A namespace Radar's cache does not hold is never called "no access";
459
+ * a partial read that names neither cause says only that it was not read.
460
+ */
461
+ export function cnpgCoverageGap(cov: CNPGKindCoverage, what: string, namespace?: string, notInstalled = 'Not installed'): string {
462
+ // A namespace the server names wins over the kind's overall state: an
463
+ // uncached scope can still name namespaces the caller was denied.
464
+ if (namespace && cov.deniedNamespaces?.includes(namespace)) return `No access to ${what}`
465
+ if (namespace && cov.uncachedNamespaces?.includes(namespace)) return `Radar does not cache ${what} in ${namespace}`
234
466
  switch (cov.state) {
235
467
  case 'denied':
236
- case 'partial':
237
468
  return `No access to ${what}`
469
+ case 'partial':
470
+ return namespace ? `${what} not read in ${namespace}` : `${what} not read`
471
+ case 'uncached':
472
+ return `Radar does not cache ${what}`
238
473
  case 'syncing':
239
474
  return 'Loading…'
240
475
  case 'error':
241
476
  return `Could not read ${what}`
242
477
  default:
243
- return 'Not installed'
478
+ return notInstalled
244
479
  }
245
480
  }
246
481
 
@@ -267,37 +502,24 @@ function podReady(pod: any): boolean | null {
267
502
  return ready.status === 'True'
268
503
  }
269
504
 
270
- export type CNPGGitOpsSource = { tool: 'argocd' | 'flux'; name: string; namespace?: string }
271
-
272
- /** The GitOps owner recorded on an object's standard Argo CD / Flux labels. */
273
- export function cnpgGitOpsSource(obj: any): CNPGGitOpsSource | null {
274
- const labels = obj?.metadata?.labels ?? {}
275
- const annotations = obj?.metadata?.annotations ?? {}
276
- const argo = labels['argocd.argoproj.io/instance']
277
- if (argo) return { tool: 'argocd', name: argo }
278
- const tracking = annotations['argocd.argoproj.io/tracking-id']
279
- if (typeof tracking === 'string' && tracking.includes(':')) {
280
- return { tool: 'argocd', name: tracking.split(':')[0] }
281
- }
282
- const fluxName = labels['kustomize.toolkit.fluxcd.io/name'] || labels['helm.toolkit.fluxcd.io/name']
283
- if (fluxName) {
284
- return {
285
- tool: 'flux',
286
- name: fluxName,
287
- namespace: labels['kustomize.toolkit.fluxcd.io/namespace'] || labels['helm.toolkit.fluxcd.io/namespace'],
288
- }
289
- }
290
- return null
505
+ /** The manager the server detected for a workspace object (Argo CD, Flux or Helm), if any. */
506
+ export function cnpgManagedBy(ws: Pick<CNPGWorkspaceResponse, 'managedBy'> | null | undefined, obj: any): ResourceRef | undefined {
507
+ const kind = obj?.kind
508
+ const name = obj?.metadata?.name
509
+ if (!kind || !name) return undefined
510
+ return ws?.managedBy?.[`${kind}/${obj?.metadata?.namespace ?? ''}/${name}`]
291
511
  }
292
512
 
293
513
  function scheduleFact(
294
514
  cluster: any,
295
515
  schedules: any[],
296
516
  cov: CNPGKindCoverage,
517
+ readings: Record<string, string> = {},
518
+ backups: any[] = [],
297
519
  ): CNPGProtectionFacts['schedule'] {
298
520
  const ns = cluster.metadata?.namespace
299
521
  if (!coverageReadable(cov, ns)) {
300
- return { text: coverageUnavailableText(cov, 'ScheduledBackups'), tone: 'unknown', names: [] }
522
+ return { text: cnpgCoverageGap(cov, 'ScheduledBackups', ns), tone: 'unknown', names: [] }
301
523
  }
302
524
  const mine = schedules.filter((s) => s.metadata?.namespace === ns && specClusterName(s) === cluster.metadata?.name)
303
525
  if (mine.length === 0) return { text: 'No declarative schedule', tone: 'neutral', names: [] }
@@ -307,15 +529,21 @@ function scheduleFact(
307
529
  return { text: mine.length === 1 ? 'Schedule suspended' : 'All schedules suspended', tone: 'degraded', names }
308
530
  }
309
531
  const cron = active[0]?.spec?.schedule
532
+ const reading = readings[`${ns}/${active[0]?.metadata?.name}`]
533
+ const blockers = active.map((s) => cnpgScheduleDestinationBlocker(s, [cluster])).filter((b): b is string => !!b)
534
+ const cadence = active.length === 1 ? reading || cron : undefined
535
+ const run = active.some((s) => s.status?.lastScheduleTime || backupsForScheduledBackup(s, backups).length > 0)
310
536
  return {
311
- text: active.length === 1 ? (cron ? `Scheduled · ${cron}` : 'Scheduled') : `${active.length} schedules`,
312
- tone: 'healthy',
537
+ text: [active.length === 1 ? 'Enabled' : `${active.length} enabled schedules`, blockers.length || run ? cadence : undefined, blockers.length ? `blocked: ${[...new Set(blockers)].map((blocker) => blocker[0].toLowerCase() + blocker.slice(1)).join(', ')}` : !run ? 'not run yet' : undefined].filter(Boolean).join(' · '),
538
+ tone: blockers.length ? 'degraded' : 'neutral',
313
539
  names,
540
+ ...(active.length === 1 && cron ? { source: `ScheduledBackup ${active[0]?.metadata?.name} · cron ${cron}` } : {}),
314
541
  }
315
542
  }
316
543
 
317
544
  function destinationFact(cluster: any): CNPGProtectionFacts['destination'] {
318
- const plugin = getCNPGClusterBarmanPlugin(cluster)
545
+ const declaration = cnpgBackupDeclaration(cluster)
546
+ const plugin = cnpgBarmanPlugin(declaration)
319
547
  if (plugin?.barmanObjectName) {
320
548
  return {
321
549
  text: `ObjectStore ${plugin.barmanObjectName}`,
@@ -324,11 +552,10 @@ function destinationFact(cluster: any): CNPGProtectionFacts['destination'] {
324
552
  objectStore: plugin.barmanObjectName,
325
553
  }
326
554
  }
327
- const cfg = getCNPGClusterBackupConfig(cluster)
328
- if (cfg.destinationPath) {
329
- return { text: cfg.destinationPath, tone: 'neutral', method: 'barmanObjectStore' }
555
+ if (declaration.inTreeDestination) {
556
+ return { text: declaration.inTreeDestination, tone: 'neutral', method: 'barmanObjectStore' }
330
557
  }
331
- if (cluster?.spec?.backup?.volumeSnapshot) {
558
+ if (declaration.snapshotsConfigured) {
332
559
  return { text: 'Volume snapshots', tone: 'neutral', method: 'volumeSnapshot' }
333
560
  }
334
561
  return { text: 'No destination configured', tone: 'neutral', method: 'none' }
@@ -359,14 +586,12 @@ function lastBackupFact(
359
586
  storesUnreadable: CNPGKindCoverage | null,
360
587
  ): CNPGProtectionFacts['lastSuccessfulBackup'] {
361
588
  const ns = cluster.metadata?.namespace
362
- const name = cluster.metadata?.name
363
589
  const candidates: { at: string; source: string }[] = []
364
590
  if (coverageReadable(backupsCov, ns)) {
365
591
  const completed = backups
366
592
  .filter(
367
593
  (b) =>
368
- b.metadata?.namespace === ns &&
369
- specClusterName(b) === name &&
594
+ cnpgBackupMatchesCluster(b, cluster) &&
370
595
  isApiGroup(b.apiVersion, 'postgresql.cnpg.io') &&
371
596
  b.status?.phase === 'completed',
372
597
  )
@@ -377,26 +602,109 @@ function lastBackupFact(
377
602
  if (window?.lastSuccess) candidates.push({ at: window.lastSuccess, source: `ObjectStore ${window.store} status` })
378
603
  const cfg = getCNPGClusterBackupConfig(cluster)
379
604
  if (!cfg.plugin && cfg.lastSuccessfulBackup) candidates.push({ at: cfg.lastSuccessfulBackup, source: 'Cluster status' })
380
- if (candidates.length === 0) {
605
+ const createdAt = Date.parse(cluster.metadata?.creationTimestamp ?? '')
606
+ const current = candidates.filter((candidate) => !Number.isFinite(createdAt) || Date.parse(candidate.at) >= createdAt)
607
+ if (current.length === 0) {
381
608
  if (!coverageReadable(backupsCov, ns)) {
382
- return { text: coverageUnavailableText(backupsCov, 'Backups'), tone: 'unknown' }
609
+ return { text: cnpgCoverageGap(backupsCov, 'Backups', ns), tone: 'unknown', source: 'Backups not read' }
383
610
  }
384
- if (storesUnreadable) return { text: coverageUnavailableText(storesUnreadable, 'ObjectStores'), tone: 'unknown' }
385
- return { text: 'None observed', tone: 'unknown' }
611
+ if (storesUnreadable) return { text: cnpgCoverageGap(storesUnreadable, 'ObjectStores', ns), tone: 'unknown' }
612
+ return { text: 'No successful backup yet', tone: 'degraded', source: 'Backups read in this namespace; none completed' }
386
613
  }
387
- const best = candidates.reduce((a, b) => (Date.parse(a.at) >= Date.parse(b.at) ? a : b))
614
+ const best = current.reduce((a, b) => (Date.parse(a.at) >= Date.parse(b.at) ? a : b))
388
615
  return { text: 'Completed', tone: 'healthy', at: best.at, source: best.source }
389
616
  }
390
617
 
391
- function walFact(cluster: any): CNPGFact {
618
+ export const CNPG_NO_WAL_ARCHIVE_DESTINATION = 'Not archived: no destination configured'
619
+
620
+ function walFact(cluster: any): CNPGProtectionFacts['walArchiving'] {
392
621
  const conds = cluster?.status?.conditions
393
622
  const c = Array.isArray(conds) ? conds.find((x: any) => x?.type === 'ContinuousArchiving') : null
394
- if (!c) return { text: 'Not reported', tone: 'unknown', source: 'Cluster status' }
395
- if (c.status === 'True') return { text: 'Archiving', tone: 'healthy', source: 'ContinuousArchiving condition' }
396
- if (c.status === 'False') {
397
- return { text: c.message ? `Failing · ${c.message}` : 'Failing', tone: 'unhealthy', source: 'ContinuousArchiving condition' }
623
+ const declaration = cnpgBackupDeclaration(cluster)
624
+ const plugin = cnpgBarmanPlugin(declaration)
625
+ const archivers = declaration.plugins.filter((p) => p.enabled && p.isWALArchiver)
626
+ const archiveConfigured = archivers.length > 0 || !!declaration.inTreeDestination
627
+ if (c?.status === 'False' && archiveConfigured) {
628
+ return { state: 'failing', text: 'Failing', tone: 'unhealthy', source: 'ContinuousArchiving condition', ...(c.lastTransitionTime ? { at: c.lastTransitionTime, atMeaning: 'since' as const } : {}), ...(c.message ? { detail: c.message } : {}) }
398
629
  }
399
- return { text: 'Unknown', tone: 'unknown', source: 'ContinuousArchiving condition' }
630
+ const destinationKnown = !!(plugin?.isWALArchiver && plugin.barmanObjectName) || !!declaration.inTreeDestination
631
+ const customArchiver = archivers.some((p: any) => p.name !== plugin?.name)
632
+ if (!destinationKnown && !customArchiver) {
633
+ return {
634
+ state: 'no_destination', text: CNPG_NO_WAL_ARCHIVE_DESTINATION,
635
+ tone: 'neutral',
636
+ source: 'Cluster spec',
637
+ detail: "WAL is not archived to recovery storage, so point-in-time recovery is unavailable." + (c?.status === 'True' ? " CloudNativePG still reports archiving as working because, with no destination, it accepts each WAL file without keeping it." : ''),
638
+ ...(c ? { operatorCondition: { type: c.type, status: c.status, message: c.message, lastTransitionTime: c.lastTransitionTime } } : {}),
639
+ }
640
+ }
641
+ if (!c) return { state: 'unknown', text: 'Not reported', tone: 'unknown', source: 'Cluster status' }
642
+ if (c.status === 'True') {
643
+ return { state: 'archiving', text: 'CNPG reports archiving', tone: 'healthy', source: 'Cluster status · ContinuousArchiving=True', ...(customArchiver && !destinationKnown ? { detail: 'Archive plugin declared; its destination is not assessed here' } : {}), ...(c.lastTransitionTime ? { at: c.lastTransitionTime, atMeaning: 'since' as const } : {}) }
644
+ }
645
+ return { state: 'unknown', text: 'Unknown', tone: 'unknown', source: 'ContinuousArchiving condition' }
646
+ }
647
+
648
+ export const CNPG_RESTORE_VALIDATION_ANNOTATION = 'radar.skyhook.io/restore-validation'
649
+ /** Radars of other versions read the note on the Cluster; a format this one does not know is skipped. */
650
+ export const CNPG_RESTORE_VALIDATION_VERSION = 1
651
+
652
+ export interface CNPGRestoreValidationNote {
653
+ version: number
654
+ recordedAt: string
655
+ recordedBy?: string
656
+ checked: string
657
+ targetTime?: string
658
+ source?: { namespace: string; name: string; uid?: string; verified: boolean }
659
+ target?: { namespace: string; name: string; uid?: string; verified: boolean }
660
+ }
661
+
662
+ /** The validation note recorded on a restored Cluster, or null when absent, malformed or in another format. */
663
+ export function getCNPGRestoreValidation(cluster: any): CNPGRestoreValidationNote | null {
664
+ const raw = cluster?.metadata?.annotations?.[CNPG_RESTORE_VALIDATION_ANNOTATION]
665
+ if (typeof raw !== 'string' || !raw.trim()) return null
666
+ try {
667
+ const v = JSON.parse(raw)
668
+ if (v?.version !== CNPG_RESTORE_VALIDATION_VERSION) return null
669
+ if (typeof v?.recordedAt !== 'string' || typeof v?.checked !== 'string' || !v.checked) return null
670
+ if (v.target?.uid && v.target.uid !== cluster.metadata?.uid) return null
671
+ return v as CNPGRestoreValidationNote
672
+ } catch {
673
+ return null
674
+ }
675
+ }
676
+
677
+ function noteIsAbout(note: CNPGRestoreValidationNote, cluster: any): boolean {
678
+ return !!note.source?.uid && note.source.uid === cluster.metadata?.uid
679
+ }
680
+
681
+ /** A recovery declaration compatible with this live source, excluding known predecessor evidence. */
682
+ export function cnpgRecoveryMatchesCluster(restored: any, cluster: any, backups: any[]): boolean {
683
+ if (restored === cluster || restored.metadata?.namespace !== cluster.metadata?.namespace) return false
684
+ const recovery = restored.spec?.bootstrap?.recovery
685
+ if (!recovery) return false
686
+ const created = Date.parse(cluster.metadata?.creationTimestamp ?? '')
687
+ const restoredAt = Date.parse(restored.metadata?.creationTimestamp ?? '')
688
+ const targetAt = Date.parse(recovery.recoveryTarget?.targetTime ?? '')
689
+ if (Number.isFinite(created) && (restoredAt < created || targetAt < created)) return false
690
+ const note = getCNPGRestoreValidation(restored)
691
+ if (note?.source?.uid && !noteIsAbout(note, cluster)) return false
692
+ const ns = cluster.metadata?.namespace
693
+ if (recovery.backup?.name) {
694
+ const backup = backups.find((b) => b.metadata?.namespace === ns && b.metadata?.name === recovery.backup.name)
695
+ return cnpgBackupMatchesCluster(backup, cluster)
696
+ }
697
+ const source = (restored.spec?.externalClusters ?? []).find((e: any) => e?.name === recovery.source)
698
+ if (!source) return false
699
+ const params = source.plugin?.name === CNPG_BARMAN_PLUGIN_NAME ? source.plugin.parameters : undefined
700
+ const external = source.barmanObjectStore
701
+ const archiveMatch = params?.barmanObjectName
702
+ ? cnpgArchiveMatchesCluster(cluster, { kind: 'objectStore', objectStore: params.barmanObjectName, serverName: params.serverName || recovery.source })
703
+ : external && cnpgArchiveMatchesCluster(cluster, { kind: 'inTree', barmanObjectStore: external, serverName: external.serverName || recovery.source })
704
+ if (!archiveMatch) return false
705
+ const id = recovery.recoveryTarget?.backupID
706
+ const pinned = id ? backups.filter((b) => b.metadata?.namespace === ns && b.spec?.cluster?.name === cluster.metadata?.name && b.status?.backupId === id) : []
707
+ return pinned.length === 0 || pinned.some((b) => cnpgBackupMatchesCluster(b, cluster))
400
708
  }
401
709
 
402
710
  function restoreValidationFact(
@@ -405,26 +713,24 @@ function restoreValidationFact(
405
713
  backups: any[],
406
714
  backupsReadable: boolean,
407
715
  ): CNPGProtectionFacts['restoreValidation'] {
408
- const plugin = getCNPGClusterBarmanPlugin(cluster)
409
- const server = plugin?.serverName || cluster.metadata?.name
410
- const store = plugin?.barmanObjectName
411
716
  const ns = cluster.metadata?.namespace
412
- const name = cluster.metadata?.name
413
- const restored = allClusters.find((c) => {
414
- if (c === cluster || c.metadata?.namespace !== ns) return false
415
- const recovery = c.spec?.bootstrap?.recovery
416
- if (!recovery) return false
417
- const sourceName = recovery.source
418
- if (sourceName) {
419
- const ext = (c.spec?.externalClusters ?? []).find((e: any) => e?.name === sourceName)
420
- const params = ext?.plugin?.name === CNPG_BARMAN_PLUGIN_NAME ? ext.plugin.parameters : undefined
421
- if (store && params?.barmanObjectName === store && (params?.serverName || sourceName) === server) return true
717
+ const restoredFromThis = allClusters.filter((c) => cnpgRecoveryMatchesCluster(c, cluster, backups))
718
+ const noted = restoredFromThis
719
+ .map((c) => ({ c, note: getCNPGRestoreValidation(c) }))
720
+ .filter((x): x is { c: any; note: CNPGRestoreValidationNote } => !!x.note && noteIsAbout(x.note, cluster))
721
+ .sort((a, b) => Date.parse(b.note.recordedAt) - Date.parse(a.note.recordedAt))[0]
722
+ if (noted) {
723
+ const rname = noted.c.metadata?.name
724
+ const by = noted.note.recordedBy ? `by ${noted.note.recordedBy}` : 'by a user Radar could not identify'
725
+ return {
726
+ text: 'Validation recorded',
727
+ tone: 'neutral',
728
+ at: noted.note.recordedAt,
729
+ source: `Recorded ${by} on ${rname}${noted.note.targetTime ? ` (target ${noted.note.targetTime})` : ''}: ${noted.note.checked.length > 140 ? `${noted.note.checked.slice(0, 140)}…` : noted.note.checked}. A person's note, not a check Radar ran.`,
730
+ restoredInto: { namespace: noted.c.metadata?.namespace, name: rname },
422
731
  }
423
- const backupName = recovery.backup?.name
424
- if (!backupName) return false
425
- const backup = backups.find((b) => b.metadata?.namespace === ns && b.metadata?.name === backupName)
426
- return specClusterName(backup) === name
427
- })
732
+ }
733
+ const restored = restoredFromThis[0]
428
734
  if (!restored) {
429
735
  // A recovery by Backup name is only attributable when that Backup could be read.
430
736
  const unresolved = !backupsReadable && allClusters.some((c) => c !== cluster && c.metadata?.namespace === ns && c.spec?.bootstrap?.recovery?.backup?.name)
@@ -432,24 +738,26 @@ function restoreValidationFact(
432
738
  return { text: 'None recorded', tone: 'unknown', source: 'Kubernetes does not record restore tests' }
433
739
  }
434
740
  const rname = restored.metadata?.name
741
+ const fromArchive = !!restored.spec?.bootstrap?.recovery?.source
742
+ const sourceDescription = fromArchive ? 'the same archive currently configured for this Cluster' : "this cluster's backups"
435
743
  const ready = typeof restored.status?.readyInstances === 'number' && restored.status.readyInstances > 0
436
744
  if (!ready) {
437
745
  return {
438
- text: `Recovery declared in ${rname}`,
746
+ text: `${fromArchive ? 'Archive recovery' : 'Recovery'} declared in ${rname}`,
439
747
  tone: 'unknown',
440
- source: `Cluster ${rname} bootstraps from this cluster's backups but has no ready instance yet`,
748
+ source: `Cluster ${rname} bootstraps from ${sourceDescription} but has no ready instance yet`,
441
749
  restoredInto: { namespace: restored.metadata?.namespace, name: rname },
442
750
  }
443
751
  }
444
752
  return {
445
- text: `Restored into ${rname}`,
753
+ text: `${fromArchive ? 'Archive restored' : 'Restored'} into ${rname}`,
446
754
  tone: 'neutral',
447
- source: `Cluster ${rname} bootstrapped from this cluster's backups and has ready instances · created ${restored.metadata?.creationTimestamp ?? 'unknown'}. This proves one recovery, not that today's backups restore.`,
755
+ source: `Cluster ${rname} bootstrapped from ${sourceDescription} and has ready instances · created ${restored.metadata?.creationTimestamp ?? 'unknown'}. No validation note is recorded on it; this proves one recovery, not that today's backups restore.`,
448
756
  restoredInto: { namespace: restored.metadata?.namespace, name: rname },
449
757
  }
450
758
  }
451
759
 
452
- function protectionSummary(p: CNPGProtectionFacts): CNPGFact {
760
+ function protectionSummary(p: CNPGProtectionFacts): Fact {
453
761
  if (p.walArchiving.tone === 'unhealthy') return { text: 'WAL archiving failing', tone: 'unhealthy' }
454
762
  if (p.destination.method === 'none' && p.schedule.names.length === 0 && p.schedule.tone !== 'unknown') {
455
763
  return { text: 'No backup destination or schedule', tone: 'neutral' }
@@ -475,47 +783,99 @@ function pgVersion(cluster: any): string | null {
475
783
  return typeof major === 'number' ? String(major) : null
476
784
  }
477
785
 
478
- function replicationFact(cluster: any, pods: CNPGInstance[], hibernated: boolean, podsCov: CNPGKindCoverage): CNPGFact {
786
+ function replicationFact(cluster: any, pods: CNPGInstance[], hibernated: boolean, podsCov: CNPGKindCoverage): Fact {
479
787
  if (hibernated) return { text: 'Hibernated', tone: 'neutral' }
480
788
  const desired = cluster?.spec?.instances
481
- if (desired === 1) return { text: 'Single instance', tone: 'neutral' }
789
+ if (desired === 1) {
790
+ return getCNPGClusterIsReplica(cluster)
791
+ ? { text: 'Single instance · upstream feed not assessed', tone: 'unknown', source: 'The designated primary replays WAL from an external cluster; Radar does not assess that feed' }
792
+ : { text: 'Single instance', tone: 'neutral' }
793
+ }
482
794
  if (!coverageReadable(podsCov, cluster?.metadata?.namespace)) {
483
- return { text: coverageUnavailableText(podsCov, 'Pods'), tone: 'unknown' }
795
+ return { text: cnpgCoverageGap(podsCov, 'Pods', cluster?.metadata?.namespace), tone: 'unknown' }
484
796
  }
485
797
  const replicas = pods.filter((p) => p.role === 'replica')
486
798
  const readyReplicas = replicas.filter((p) => p.ready === true).length
487
799
  if (replicas.length === 0) return { text: 'No replica pods observed', tone: 'unknown' }
488
800
  return {
489
- text: `${readyReplicas}/${replicas.length} replicas ready · lag unknown`,
801
+ text: `${readyReplicas}/${replicas.length} Pods ready · lag unknown`,
490
802
  tone: 'unknown',
491
- source: 'Pod readiness does not show whether a replica is streaming',
803
+ source: CNPG_LAG_UNMEASURED_SOURCE,
492
804
  }
493
805
  }
494
806
 
807
+ const CNPG_JOB_PURPOSE: Record<string, string> = {
808
+ initdb: 'First instance',
809
+ join: 'New standby',
810
+ 'full-recovery': 'Restore into',
811
+ 'snapshot-recovery': 'Restore into',
812
+ pgbasebackup: 'Clone into',
813
+ import: 'Import into',
814
+ 'major-upgrade': 'Major upgrade of',
815
+ }
816
+
817
+ /** What a Cluster's own Job is for, naming the instance it builds ("New standby pg-2"). */
818
+ export function cnpgJobPurpose(role: string | undefined, instance: string | undefined, pod: string): string {
819
+ const purpose = role ? CNPG_JOB_PURPOSE[role] : undefined
820
+ if (purpose && instance) return `${purpose} ${instance}`
821
+ return `${role ? `${role} ` : ''}Job Pod ${pod}`
822
+ }
823
+
824
+ interface CNPGJobPod {
825
+ role?: string
826
+ instance?: string
827
+ }
828
+
829
+ function jobPodIndex(resp: CNPGWorkspaceResponse): Map<string, CNPGJobPod> {
830
+ const idx = new Map<string, CNPGJobPod>()
831
+ for (const p of resp.jobPods ?? []) {
832
+ const labels = p?.metadata?.labels ?? {}
833
+ idx.set(`${p.metadata?.namespace}/${p.metadata?.name}`, { role: labels['cnpg.io/jobRole'], instance: labels['cnpg.io/instanceName'] })
834
+ }
835
+ return idx
836
+ }
837
+
495
838
  function problemsFor(
496
839
  cluster: any,
497
840
  issues: CNPGWorkspaceIssue[],
498
841
  audit: CNPGAuditFinding[],
499
842
  children: Map<string, string>,
843
+ backupTimes: Map<string, number> = new Map(),
844
+ jobPods: Map<string, CNPGJobPod> = new Map(),
500
845
  ): CNPGProblem[] {
501
846
  const ns = cluster.metadata?.namespace
502
847
  const name = cluster.metadata?.name
503
848
  const out: CNPGProblem[] = []
849
+ const fromIssues: IssueProblem[] = []
504
850
  for (const issue of issues) {
505
851
  if ((issue.namespace ?? '') !== ns) continue
506
852
  const isSelf = issue.kind === 'Cluster' && issue.name === name
507
853
  const owner = children.get(`${issue.kind}/${ns}/${issue.name}`)
508
854
  if (!isSelf && owner !== name) continue
509
- out.push({
855
+ const job = issue.kind === 'Pod' ? jobPods.get(`${ns}/${issue.name}`) : undefined
856
+ const text = cnpgIssueText(issue)
857
+ fromIssues.push({
510
858
  id: `${issue.id}:${issue.kind}/${issue.name}`,
511
- severity: issue.severity,
859
+ // A standby that cannot join costs redundancy, not service: the primary keeps serving.
860
+ severity: job?.role === 'join' ? 'warning' : issue.severity,
512
861
  category: cnpgIssueCategory(issue),
513
- title: issue.message || issue.reason,
514
- detail: issue.cause || undefined,
862
+ ...(job
863
+ ? {
864
+ title: `${cnpgJobPurpose(job.role, job.instance, issue.name)}: ${issue.category ? issueTitle({ category: issue.category, reason: issue.reason }) : text.title}`,
865
+ detail: [issue.message?.trim(), issue.cause?.trim()].filter(Boolean).join(' ') || undefined,
866
+ job: job.role ?? 'job',
867
+ instance: job.instance,
868
+ }
869
+ : text),
515
870
  subject: { kind: issue.kind, group: issue.group ?? '', namespace: ns, name: issue.name },
516
871
  source: 'issue',
872
+ origin: cnpgIssueOrigin(issue),
873
+ reason: issue.reason,
874
+ ...(isSelf && issue.reason.startsWith('Ready:') ? { rollup: true } : {}),
517
875
  })
518
876
  }
877
+ const newestBackup = [...backupTimes.entries()].sort((a, b) => b[1] - a[1])[0]?.[0]
878
+ out.push(...cnpgFoldLastBackupFailed(cnpgCollapseBackupFailures(fromIssues, backupTimes), newestBackup))
519
879
  for (const f of audit) {
520
880
  if (f.kind !== 'Cluster' || f.name !== name || (f.namespace ?? '') !== ns) continue
521
881
  out.push({
@@ -528,8 +888,62 @@ function problemsFor(
528
888
  source: 'audit',
529
889
  })
530
890
  }
531
- const rank = { critical: 0, warning: 1, posture: 2 } as const
532
- return out.sort((a, b) => rank[a.severity] - rank[b.severity] || a.title.localeCompare(b.title))
891
+ return out.sort(cnpgCompareProblems)
892
+ }
893
+
894
+ /**
895
+ * The ready count to show for a cluster: CNPG's status count, or the Pods'
896
+ * own count when the status claims more than the Pods show.
897
+ */
898
+ export function cnpgReadyInstances(row: Pick<CNPGFleetRow, 'instances' | 'podReadiness' | 'readinessContradicted'>): { text: string; podText?: string; tone?: HealthLevel; note?: string } {
899
+ if (row.instances.ready === null) {
900
+ return {
901
+ text: 'Not reported by the operator',
902
+ podText: row.podReadiness
903
+ ? row.instances.desired !== null
904
+ ? `${row.podReadiness.ready} of ${row.instances.desired} instance Pods ready`
905
+ : `${row.podReadiness.ready} instance Pod${row.podReadiness.ready === 1 ? '' : 's'} ready`
906
+ : undefined,
907
+ }
908
+ }
909
+ const desired = row.instances.desired ?? '–'
910
+ if (row.readinessContradicted && row.podReadiness) {
911
+ return {
912
+ text: `${row.podReadiness.ready}/${desired}`,
913
+ tone: row.podReadiness.ready === 0 ? 'unhealthy' : 'degraded',
914
+ note: `Counted from the instance Pods' Ready condition. CNPG status reports ${row.instances.ready} ready, so the status may be stale.`,
915
+ }
916
+ }
917
+ return { text: `${row.instances.ready ?? '–'}/${desired}` }
918
+ }
919
+
920
+ const CNPG_SEVERITY_RANK = { critical: 0, warning: 1, posture: 2 } as const
921
+
922
+ /**
923
+ * The order problems are shown in, everywhere: most severe first, then a
924
+ * cause before its symptoms. One instance Pod's state (a failing probe, a
925
+ * crash loop) is usually the symptom of a cluster-level problem (archiving,
926
+ * backups, replication, reconciliation, declarations) or of a Job the Cluster
927
+ * runs (an initdb or join Pod that cannot start), so at equal severity those
928
+ * come first; the Cluster's own Ready condition sums them all up and comes last.
929
+ */
930
+ export function cnpgCompareProblems(a: CNPGProblem, b: CNPGProblem): number {
931
+ const symptom = (p: CNPGProblem) => (p.rollup ? 2 : p.subject.kind === 'Pod' && p.subject.group === '' && !p.job ? 1 : 0)
932
+ return CNPG_SEVERITY_RANK[a.severity] - CNPG_SEVERITY_RANK[b.severity] || symptom(a) - symptom(b) || a.title.localeCompare(b.title)
933
+ }
934
+
935
+ function sortProblems(list: CNPGProblem[]): CNPGProblem[] {
936
+ return list.sort(cnpgCompareProblems)
937
+ }
938
+
939
+ function primaryConflictOf(cluster: any, pods: any[]): CNPGFleetRow['primaryConflict'] {
940
+ const status = cluster?.status?.currentPrimary
941
+ if (!status) return undefined
942
+ const labelled = pods
943
+ .filter((p) => (p?.metadata?.labels?.['cnpg.io/instanceRole'] ?? p?.metadata?.labels?.role) === 'primary')
944
+ .map((p) => p.metadata?.name as string)
945
+ if (labelled.length === 0 || labelled.includes(status)) return undefined
946
+ return { status, labelled: labelled.sort()[0] }
533
947
  }
534
948
 
535
949
  /** Index "Kind/ns/name" → owning cluster name, from each child's spec.cluster.name. */
@@ -537,6 +951,7 @@ function childIndex(resp: CNPGWorkspaceResponse): Map<string, string> {
537
951
  const idx = new Map<string, string>()
538
952
  const add = (kind: string, list: any[] | undefined) => {
539
953
  for (const o of list ?? []) {
954
+ if (kind === 'Backup' && !targetCluster(o, resp.objects.clusters ?? [])) continue
540
955
  const c = specClusterName(o)
541
956
  if (c) idx.set(`${kind}/${o.metadata?.namespace}/${o.metadata?.name}`, c)
542
957
  }
@@ -547,7 +962,8 @@ function childIndex(resp: CNPGWorkspaceResponse): Map<string, string> {
547
962
  add('Database', resp.objects.databases)
548
963
  add('Publication', resp.objects.publications)
549
964
  add('Subscription', resp.objects.subscriptions)
550
- for (const p of resp.objects.pods ?? []) {
965
+ add('DatabaseRole', resp.objects.databaseRoles)
966
+ for (const p of [...(resp.objects.pods ?? []), ...(resp.jobPods ?? [])]) {
551
967
  const c = p?.metadata?.labels?.['cnpg.io/cluster']
552
968
  if (c) idx.set(`Pod/${p.metadata?.namespace}/${p.metadata?.name}`, c)
553
969
  }
@@ -561,21 +977,24 @@ function declarationsFor(cluster: any, resp: CNPGWorkspaceResponse): CNPGFleetRo
561
977
  ['databases', resp.objects.databases ?? []],
562
978
  ['publications', resp.objects.publications ?? []],
563
979
  ['subscriptions', resp.objects.subscriptions ?? []],
980
+ ['databaseRoles', resp.objects.databaseRoles ?? []],
564
981
  ]
565
982
  let total = 0
566
983
  let failed = 0
567
984
  let pending = 0
568
- let unreadable = false
985
+ const unread: string[] = []
986
+ const labels = { databases: 'Databases', publications: 'Publications', subscriptions: 'Subscriptions', databaseRoles: 'DatabaseRoles' }
569
987
  for (const [k, list] of lists) {
570
988
  if (!coverageReadable(coverageOf(resp, k), ns)) {
571
- if (coverageOf(resp, k).state !== 'notInstalled') unreadable = true
989
+ if (coverageOf(resp, k).state !== 'notInstalled') unread.push(`${labels[k as keyof typeof labels]} ${resp.coverage[k] ? 'not read' : 'not read by this Radar'}`)
572
990
  continue
573
991
  }
574
992
  for (const o of list) {
575
993
  if (o.metadata?.namespace !== ns || specClusterName(o) !== name) continue
576
994
  total++
577
- if (o.status?.applied === false) failed++
578
- else if (o.status?.applied !== true) pending++
995
+ const state = cnpgRoleState(o)
996
+ if (state === 'failed') failed++
997
+ else if (state === 'pending') pending++
579
998
  }
580
999
  }
581
1000
  const roleStatus = cluster?.status?.managedRolesStatus
@@ -588,9 +1007,10 @@ function declarationsFor(cluster: any, resp: CNPGWorkspaceResponse): CNPGFleetRo
588
1007
  if (failedRoles.has(r.name)) failed++
589
1008
  else if (!reconciledRoles.has(r.name)) pending++
590
1009
  }
591
- let summary: CNPGFact
1010
+ const unreadable = unread.length > 0
1011
+ let summary: Fact
592
1012
  if (total === 0) {
593
- summary = unreadable ? { text: 'No access to some declarations', tone: 'unknown' } : { text: 'None declared', tone: 'neutral' }
1013
+ summary = { text: 'None declared', tone: 'neutral' }
594
1014
  } else if (failed > 0) {
595
1015
  summary = { text: `${failed} of ${total} not reconciled`, tone: 'degraded' }
596
1016
  } else if (pending > 0) {
@@ -598,7 +1018,10 @@ function declarationsFor(cluster: any, resp: CNPGWorkspaceResponse): CNPGFleetRo
598
1018
  } else {
599
1019
  summary = { text: `${total} reconciled`, tone: 'healthy' }
600
1020
  }
601
- if (unreadable && total > 0) summary = { ...summary, source: 'Some declaration kinds are not readable' }
1021
+ if (unreadable) {
1022
+ const count = total === 0 ? 'Reconciliation unknown' : failed > 0 ? `≥${failed} not reconciled` : pending > 0 ? `≥${pending} pending` : `≥${total} reconciled`
1023
+ summary = { text: `${count}; ${unread.join('; ')}`, tone: failed > 0 ? 'degraded' : 'unknown' }
1024
+ }
602
1025
  return { summary, total, failed, pending }
603
1026
  }
604
1027
 
@@ -607,6 +1030,7 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
607
1030
  const pods = resp.objects.pods ?? []
608
1031
  const stores = resp.objects.objectStores ?? []
609
1032
  const children = childIndex(resp)
1033
+ const jobs = jobPodIndex(resp)
610
1034
  const poolers = resp.objects.poolers ?? []
611
1035
 
612
1036
  const rows: CNPGFleetRow[] = clusters.map((cluster) => {
@@ -631,7 +1055,7 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
631
1055
  const storesCov = coverageOf(resp, 'objectStores')
632
1056
  const storesUnreadable = !!getCNPGClusterBarmanPlugin(cluster)?.barmanObjectName && !coverageReadable(storesCov, ns)
633
1057
  const protection: CNPGProtectionFacts = {
634
- schedule: scheduleFact(cluster, resp.objects.scheduledBackups ?? [], coverageOf(resp, 'scheduledBackups')),
1058
+ schedule: scheduleFact(cluster, resp.objects.scheduledBackups ?? [], coverageOf(resp, 'scheduledBackups'), resp.scheduleReadings, resp.objects.backups ?? []),
635
1059
  destination: destinationFact(cluster),
636
1060
  lastSuccessfulBackup: lastBackupFact(cluster, resp.objects.backups ?? [], coverageOf(resp, 'backups'), window, storesUnreadable ? storesCov : null),
637
1061
  walArchiving: wal,
@@ -645,15 +1069,26 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
645
1069
  source: `ObjectStore ${window.store} status (earliest point)`,
646
1070
  }
647
1071
  : storesUnreadable
648
- ? { text: coverageUnavailableText(storesCov, 'ObjectStores'), tone: 'unknown' }
649
- : { text: 'Not reported', tone: 'unknown' },
1072
+ ? { text: cnpgCoverageGap(storesCov, 'ObjectStores', ns), tone: 'unknown' }
1073
+ : destinationFact(cluster).method === 'none' ? { text: 'None: no backup destination', tone: 'neutral' } : { text: 'Not reported', tone: 'unknown' },
650
1074
  restoreValidation: restoreValidationFact(cluster, clusters, resp.objects.backups ?? [], coverageReadable(coverageOf(resp, 'backups'), ns)),
651
1075
  }
652
- const problems = problemsFor(cluster, resp.issues ?? [], resp.audit ?? [], children)
1076
+ const podsReadable = coverageReadable(coverageOf(resp, 'pods'), ns)
1077
+ const podReadiness =
1078
+ podsReadable && resp.objects.pods !== undefined && instancePods.every((p) => p.ready !== null)
1079
+ ? { ready: instancePods.filter((p) => p.ready).length, total: instancePods.length }
1080
+ : undefined
1081
+ const readinessContradicted = !hibernated && !!podReadiness && readyInstances !== null && podReadiness.ready < readyInstances
1082
+ const primaryConflict = podsReadable ? primaryConflictOf(cluster, pods.filter((p) => p.metadata?.namespace === ns && p.metadata?.labels?.['cnpg.io/cluster'] === name)) : undefined
1083
+ let problems = sortProblems([
1084
+ ...problemsFor(cluster, resp.issues ?? [], resp.audit ?? [], children, backupTimesOf(cluster, resp.objects.backups ?? []), jobs),
1085
+
1086
+ ])
1087
+ problems = problems.map((p) => p.origin?.label === 'Kubernetes scheduler' ? { ...p, detail: summarizeSchedulerMessage(p.detail, { plain: true }), rawDetail: p.detail, rawDetailLabel: 'Scheduler message' } : p)
653
1088
  const categories = new Set<CNPGProblemCategory>(
654
1089
  problems.filter((p) => p.severity !== 'posture').map((p) => p.category),
655
1090
  )
656
- const replica = cluster?.spec?.replica?.enabled ? { source: cluster.spec.replica.source } : null
1091
+ const replica = cluster && getCNPGClusterIsReplica(cluster) ? { source: cluster.spec.replica.source } : null
657
1092
 
658
1093
  return {
659
1094
  key: key(ns, name),
@@ -662,6 +1097,9 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
662
1097
  cluster,
663
1098
  controllerStatus: { text: status.text, level: status.level },
664
1099
  instances: { ready: readyInstances, desired },
1100
+ ...(podReadiness ? { podReadiness } : {}),
1101
+ ...(readinessContradicted ? { readinessContradicted } : {}),
1102
+ ...(primaryConflict ? { primaryConflict } : {}),
665
1103
  pods: instancePods,
666
1104
  replicaCluster: replica,
667
1105
  hibernated,
@@ -670,6 +1108,7 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
670
1108
  replication: replicationFact(cluster, instancePods, hibernated, coverageOf(resp, 'pods')),
671
1109
  protection: { ...protection, summary: protectionSummary(protection) },
672
1110
  declarations: declarationsFor(cluster, resp),
1111
+ poolerObjects: poolers.filter((p) => p.metadata?.namespace === ns && specClusterName(p) === name),
673
1112
  poolers: poolers
674
1113
  .filter((p) => p.metadata?.namespace === ns && specClusterName(p) === name)
675
1114
  .map((p) => p.metadata?.name),
@@ -677,20 +1116,49 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
677
1116
  problems,
678
1117
  attention: problems.some((p) => p.severity !== 'posture'),
679
1118
  categories,
680
- gitops: cnpgGitOpsSource(cluster),
1119
+ managedBy: cnpgManagedBy(resp, cluster),
681
1120
  }
682
1121
  })
683
1122
 
684
- rows.sort((a, b) => Number(b.attention) - Number(a.attention) || a.namespace.localeCompare(b.namespace) || a.name.localeCompare(b.name))
685
-
686
- const categoryCounts = { availability: 0, protection: 0, declarations: 0, pooling: 0 } as Record<CNPGProblemCategory, number>
687
- for (const r of rows) for (const c of r.categories) categoryCounts[c]++
688
-
689
1123
  const incompleteKinds = CNPG_WORKSPACE_KEYS.filter((k) => {
690
1124
  const s = coverageOf(resp, k).state
691
- return s === 'partial' || s === 'denied' || s === 'syncing' || s === 'error'
1125
+ return s === 'partial' || s === 'denied' || s === 'syncing' || s === 'error' || s === 'uncached'
692
1126
  })
693
1127
 
1128
+ return finishFleet(rows, incompleteKinds)
1129
+ }
1130
+
1131
+ function urgencyOf(row: CNPGFleetRow): { worst: number; urgent: number; total: number } {
1132
+ let worst = 3
1133
+ let urgent = 0
1134
+ for (const p of row.problems) {
1135
+ worst = Math.min(worst, CNPG_SEVERITY_RANK[p.severity])
1136
+ if (p.severity !== 'posture') urgent++
1137
+ }
1138
+ return { worst, urgent, total: row.problems.length }
1139
+ }
1140
+
1141
+ /**
1142
+ * Worst problem first (critical, warning, posture, none), then the most
1143
+ * attention-level problems, then all problems, then namespace/name — the same
1144
+ * order in every filter, so a row never jumps when the filter changes.
1145
+ */
1146
+ export function compareCNPGFleetUrgency(a: CNPGFleetRow, b: CNPGFleetRow): number {
1147
+ const ua = urgencyOf(a)
1148
+ const ub = urgencyOf(b)
1149
+ return (
1150
+ ua.worst - ub.worst ||
1151
+ ub.urgent - ua.urgent ||
1152
+ ub.total - ua.total ||
1153
+ a.namespace.localeCompare(b.namespace) ||
1154
+ a.name.localeCompare(b.name)
1155
+ )
1156
+ }
1157
+
1158
+ function finishFleet(rows: CNPGFleetRow[], incompleteKinds: CNPGWorkspaceKey[]): CNPGFleet {
1159
+ rows.sort(compareCNPGFleetUrgency)
1160
+ const categoryCounts = { availability: 0, protection: 0, declarations: 0, pooling: 0 } as Record<CNPGProblemCategory, number>
1161
+ for (const r of rows) for (const c of r.categories) categoryCounts[c]++
694
1162
  return {
695
1163
  rows,
696
1164
  attentionCount: rows.filter((r) => r.attention).length,
@@ -698,3 +1166,536 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
698
1166
  incompleteKinds,
699
1167
  }
700
1168
  }
1169
+
1170
+ /** One cluster's answer from /api/cnpg/disk. */
1171
+ export interface CNPGDiskReading {
1172
+ namespace: string
1173
+ name: string
1174
+ /** ok | partial | noSeries | noPrometheus | denied | unavailable | error | notRead | ambiguous | scopeMismatch */
1175
+ state: string
1176
+ grant?: Grant
1177
+ reason?: string
1178
+ claims: number
1179
+ measured: number
1180
+ max?: {
1181
+ claim: string
1182
+ instance: string
1183
+ role: string
1184
+ tablespace?: string
1185
+ usedBytes: number
1186
+ capacityBytes: number
1187
+ ratio: number
1188
+ }
1189
+ isolation?: CNPGMetricIsolation
1190
+ }
1191
+
1192
+ export const CNPG_DISK_WARNING_RATIO = 0.8
1193
+ export const CNPG_DISK_CRITICAL_RATIO = 0.9
1194
+
1195
+ export const CNPG_DISK_SOURCE = 'kubelet volume stats via Prometheus'
1196
+
1197
+ export function cnpgVolumeRoleLabel(role: string, tablespace?: string): string {
1198
+ switch (role) {
1199
+ case 'PG_DATA':
1200
+ return 'data volume'
1201
+ case 'PG_WAL':
1202
+ return 'WAL volume'
1203
+ case 'PG_TABLESPACE':
1204
+ return tablespace ? `tablespace ${tablespace} volume` : 'tablespace volume'
1205
+ default:
1206
+ return 'volume'
1207
+ }
1208
+ }
1209
+
1210
+ export function cnpgDiskTone(ratio: number): HealthLevel {
1211
+ if (ratio >= CNPG_DISK_CRITICAL_RATIO) return 'unhealthy'
1212
+ if (ratio >= CNPG_DISK_WARNING_RATIO) return 'degraded'
1213
+ return 'healthy'
1214
+ }
1215
+
1216
+ /** The fleet and summary "Storage" fact: the fullest measured volume, or why there is none. */
1217
+ export function cnpgDiskFact(r: CNPGDiskReading | undefined): Fact {
1218
+ if (!r) return { text: 'Not read', tone: 'unknown' }
1219
+ if (r.max && (r.state === 'ok' || r.state === 'partial')) {
1220
+ const partial = r.state === 'partial' ? ` · ${r.measured} of ${r.claims} volumes measured` : ''
1221
+ return {
1222
+ text: `${Math.round(r.max.ratio * 100)}% used`,
1223
+ tone: cnpgDiskTone(r.max.ratio),
1224
+ source: `Fullest: ${cnpgVolumeRoleLabel(r.max.role, r.max.tablespace)} of ${r.max.instance}, ${formatBytes(r.max.usedBytes)} of ${formatBytes(r.max.capacityBytes)} · ${CNPG_DISK_SOURCE}${partial}.${isolationCaveat(r.isolation)}`,
1225
+ }
1226
+ }
1227
+ switch (r.state) {
1228
+ case 'denied':
1229
+ return { text: 'No access', tone: 'unknown', source: r.grant ? `Needs ${formatGrant(r.grant)}` : r.reason }
1230
+ case 'noPrometheus':
1231
+ return { text: 'No usage metrics', tone: 'unknown', source: CNPG_PROMETHEUS_NOT_CONNECTED, detail: r.reason }
1232
+ case 'noSeries':
1233
+ case 'ok':
1234
+ case 'partial':
1235
+ return { text: 'No usage metrics', tone: 'unknown', source: r.reason ?? 'Used space needs Prometheus with kubelet volume stats' }
1236
+ case 'notRead':
1237
+ return { text: 'Not measured', tone: 'unknown', source: r.reason }
1238
+ default:
1239
+ return { text: 'Unavailable', tone: 'unknown', source: r.reason }
1240
+ }
1241
+ }
1242
+
1243
+ /**
1244
+ * Joins /api/cnpg/disk into the fleet: every row gets its disk fact, and a
1245
+ * volume at or past the warning threshold becomes a problem, so the cluster
1246
+ * needs attention. Only a measurement raises one, and the endpoint returns
1247
+ * measurements only to callers holding the claim and metrics grants.
1248
+ */
1249
+ export function applyCNPGDisk(fleet: CNPGFleet, readings: CNPGDiskReading[] | undefined): CNPGFleet {
1250
+ if (!readings) return fleet
1251
+ const byKey = new Map(readings.map((r) => [key(r.namespace, r.name), r]))
1252
+ const rows = fleet.rows.map((row) => {
1253
+ const reading = byKey.get(row.key)
1254
+ const next: CNPGFleetRow = { ...row, disk: cnpgDiskFact(reading) }
1255
+ const max = reading?.max
1256
+ if (!max || max.ratio < CNPG_DISK_WARNING_RATIO || (reading.state !== 'ok' && reading.state !== 'partial')) return next
1257
+ const problem: CNPGProblem = {
1258
+ id: `disk:${row.key}`,
1259
+ severity: max.ratio >= CNPG_DISK_CRITICAL_RATIO ? 'critical' : 'warning',
1260
+ category: 'availability',
1261
+ title: `The ${cnpgVolumeRoleLabel(max.role, max.tablespace)} of ${max.instance} is ${Math.round(max.ratio * 100)}% full`,
1262
+ detail: `${formatBytes(max.usedBytes)} of ${formatBytes(max.capacityBytes)} used, from ${CNPG_DISK_SOURCE}.${isolationCaveat(reading.isolation)}`,
1263
+ subject: { kind: 'Cluster', group: 'postgresql.cnpg.io', namespace: row.namespace, name: row.name },
1264
+ source: 'measurement',
1265
+ ...(reading.isolation?.mode === 'unverified' ? { measuredBy: 'kubelet, matched by claim name', unverifiedMatch: true } : {}),
1266
+ }
1267
+ const problems = [...row.problems, problem].sort(cnpgCompareProblems)
1268
+ return { ...next, problems, attention: true, categories: new Set([...row.categories, problem.category]) }
1269
+ })
1270
+ return finishFleet(rows, fleet.incompleteKinds)
1271
+ }
1272
+
1273
+ const CNPG_PLUGIN_PHASES: Record<string, 'unknownPlugin' | 'pluginError'> = {
1274
+ 'Cluster cannot proceed to reconciliation due to an unknown plugin being required': 'unknownPlugin',
1275
+ 'Cluster cannot proceed to reconciliation due to an error while interacting with plugins': 'pluginError',
1276
+ }
1277
+
1278
+ /** Whether the Cluster's phase says the operator is stuck on a CNPG-I plugin, and which way. */
1279
+ export function cnpgPluginPhase(cluster: any): 'unknownPlugin' | 'pluginError' | null {
1280
+ return CNPG_PLUGIN_PHASES[cluster?.status?.phase] ?? null
1281
+ }
1282
+
1283
+ /** The CNPG-I plugins a Cluster names in spec.plugins. */
1284
+ export function cnpgClusterPlugins(cluster: any): string[] {
1285
+ const plugins = cluster?.spec?.plugins
1286
+ return Array.isArray(plugins) ? plugins.map((p: any) => p?.name).filter((n: unknown): n is string => typeof n === 'string' && !!n) : []
1287
+ }
1288
+
1289
+ /** Bytes in binary units with IEC labels (GiB), as every CloudNativePG view prints them. */
1290
+ export function cnpgFormatBytes(n: number): string {
1291
+ const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
1292
+ let v = n
1293
+ let i = 0
1294
+ while (v >= 1024 && i < u.length - 1) {
1295
+ v /= 1024
1296
+ i++
1297
+ }
1298
+ return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`
1299
+ }
1300
+
1301
+ const formatBytes = cnpgFormatBytes
1302
+
1303
+ // ---------------------------------------------------------------------------
1304
+ // Standbys and replication slots: one finding, whichever source saw it
1305
+ // ---------------------------------------------------------------------------
1306
+
1307
+ /**
1308
+ * Retained WAL at which an inactive physical slot becomes a concern. An HA
1309
+ * slot only goes inactive when its standby stops consuming, and the WAL it
1310
+ * pins keeps growing until that standby catches up or the slot is dropped.
1311
+ */
1312
+ export const CNPG_SLOT_RETENTION_WARNING_BYTES = 1024 ** 3
1313
+
1314
+ /** The id of a standby's "not receiving WAL" problem: the fleet (Prometheus) and the cluster page (instance manager) raise the same one. */
1315
+ export function cnpgStandbyProblemId(rowKey: string, pod: string): string {
1316
+ return `standby:${rowKey}:${pod}`
1317
+ }
1318
+
1319
+ /** The id of an inactive slot's retention problem, shared like cnpgStandbyProblemId. */
1320
+ export function cnpgSlotProblemId(rowKey: string, slot: string): string {
1321
+ return `slot:${rowKey}:${slot}`
1322
+ }
1323
+
1324
+ function haSlotPrefix(cluster: any): string {
1325
+ const p = cluster?.spec?.replicationSlots?.highAvailability?.slotPrefix
1326
+ return typeof p === 'string' && p ? p : '_cnpg_'
1327
+ }
1328
+
1329
+ /** The HA slot CloudNativePG keeps for an instance: the prefix (default `_cnpg_`) then the instance name with `-` as `_`. */
1330
+ export function cnpgHASlotName(cluster: any, instance: string): string {
1331
+ return `${haSlotPrefix(cluster)}${instance.replace(/-/g, '_')}`
1332
+ }
1333
+
1334
+ /** The instance a managed HA slot serves, or undefined when the slot is not one of this cluster's HA slots. */
1335
+ export function cnpgHASlotInstance(cluster: any, slot: string, instances: string[]): string | undefined {
1336
+ return instances.find((i) => cnpgHASlotName(cluster, i) === slot)
1337
+ }
1338
+
1339
+ const CLUSTER_GROUP = 'postgresql.cnpg.io'
1340
+
1341
+ function clusterSubject(row: Pick<CNPGFleetRow, 'namespace' | 'name'>) {
1342
+ return { kind: 'Cluster', group: CLUSTER_GROUP, namespace: row.namespace, name: row.name }
1343
+ }
1344
+
1345
+ export interface CNPGStandbyGap {
1346
+ pod: string
1347
+ /** What was observed, in words, e.g. "its WAL receiver is down", "replay is paused". */
1348
+ evidence: string[]
1349
+ measuredBy: string
1350
+ sourceDetail: string
1351
+ /** Every expected standby is out: there is no failover target with current data. */
1352
+ noneReceiving?: boolean
1353
+ unverifiedMatch?: boolean
1354
+ }
1355
+
1356
+ /** A standby that receives nothing from the primary. Its replay lag reads 0 because nothing new arrives to replay. */
1357
+ export function cnpgStandbyNotReceivingProblem(row: Pick<CNPGFleetRow, 'key' | 'namespace' | 'name'>, gap: CNPGStandbyGap): CNPGProblem {
1358
+ return {
1359
+ id: cnpgStandbyProblemId(row.key, gap.pod),
1360
+ reason: 'CNPGStandbyNotReceiving',
1361
+ severity: gap.noneReceiving ? 'critical' : 'warning',
1362
+ category: 'availability',
1363
+ title: `${gap.pod} is not receiving WAL from the primary`,
1364
+ shortTitle: `${gap.pod} not receiving WAL`,
1365
+ detail: `${capitalize(gap.evidence.join('; '))}. While it does not stream, the primary keeps WAL for its slot, and it falls behind unless it replays WAL from the archive; a replay lag of 0 does not mean it is caught up, only that nothing new reached it.`,
1366
+ subject: { kind: 'Pod', group: '', namespace: row.namespace, name: gap.pod },
1367
+ source: 'measurement',
1368
+ measuredBy: gap.measuredBy,
1369
+ sourceDetail: gap.sourceDetail,
1370
+ ...(gap.unverifiedMatch ? { unverifiedMatch: true } : {}),
1371
+ }
1372
+ }
1373
+
1374
+ export interface CNPGSlotRetention {
1375
+ slot: string
1376
+ /** The instance holding the slot (the primary for an HA slot). */
1377
+ pod: string
1378
+ bytes: number
1379
+ /** The standby an HA slot serves, when the name matches one of the cluster's instances. */
1380
+ standby?: string
1381
+ measuredBy: string
1382
+ sourceDetail: string
1383
+ unverifiedMatch?: boolean
1384
+ }
1385
+
1386
+ export function cnpgSlotRetentionProblem(row: Pick<CNPGFleetRow, 'key' | 'namespace' | 'name'>, r: CNPGSlotRetention): CNPGProblem {
1387
+ const forWhom = r.standby ? ` for ${r.standby}` : ''
1388
+ return {
1389
+ id: cnpgSlotProblemId(row.key, r.slot),
1390
+ reason: 'CNPGInactiveSlot',
1391
+ slot: r.slot,
1392
+ severity: 'warning',
1393
+ category: 'availability',
1394
+ title: `Inactive slot ${r.slot} holds ${formatBytes(r.bytes)} of WAL on ${r.pod}${forWhom}`,
1395
+ shortTitle: `${formatBytes(r.bytes)} of WAL held${forWhom || ` by ${r.slot}`}`,
1396
+ detail: `PostgreSQL keeps every WAL file the slot still needs until ${r.standby ? `${r.standby} catches up` : 'its consumer catches up'} or the slot is dropped, so this grows while the slot stays inactive. Storage shows it beside the volume's other WAL.`,
1397
+ subject: clusterSubject(row),
1398
+ source: 'measurement',
1399
+ measuredBy: r.measuredBy,
1400
+ sourceDetail: r.sourceDetail,
1401
+ ...(r.standby ? { instance: r.standby } : {}),
1402
+ ...(r.unverifiedMatch ? { unverifiedMatch: true } : {}),
1403
+ }
1404
+ }
1405
+
1406
+ /**
1407
+ * Adds problems to a row, replacing any with the same id, and drops those a
1408
+ * newer, complete read has disproven (`drop`); attention and categories
1409
+ * follow from what remains.
1410
+ */
1411
+ export function cnpgWithProblems(row: CNPGFleetRow, added: CNPGProblem[], drop?: (p: CNPGProblem) => boolean): CNPGFleetRow {
1412
+ if (added.length === 0 && !drop) return row
1413
+ const ids = new Set(added.map((p) => p.id))
1414
+ const problems = [...row.problems.filter((p) => !ids.has(p.id) && !drop?.(p)), ...added].sort(cnpgCompareProblems)
1415
+ if (problems.length === row.problems.length && problems.every((p, i) => p === row.problems[i])) return row
1416
+ return {
1417
+ ...row,
1418
+ problems,
1419
+ attention: problems.some((p) => p.severity !== 'posture'),
1420
+ categories: new Set(problems.filter((p) => p.severity !== 'posture').map((p) => p.category)),
1421
+ }
1422
+ }
1423
+
1424
+ function capitalize(s: string): string {
1425
+ return s ? s[0].toUpperCase() + s.slice(1) : s
1426
+ }
1427
+
1428
+ /** The short per-fact source when Radar has no Prometheus; the reason goes in `detail`. */
1429
+ export const CNPG_PROMETHEUS_NOT_CONNECTED = 'Prometheus not connected'
1430
+
1431
+ const CNPG_LAG_UNMEASURED_SOURCE = 'Pod readiness does not show whether a replica is streaming'
1432
+
1433
+ /** One cluster's answer from /api/cnpg/fleet-metrics. */
1434
+ export interface CNPGFleetMetricsReading {
1435
+ namespace: string
1436
+ name: string
1437
+ /** ok | noStandby | noSeries | denied | ambiguous | scopeMismatch | error | notRead */
1438
+ lag: {
1439
+ state: string
1440
+ grant?: Grant
1441
+ reason?: string
1442
+ seconds?: number
1443
+ pod?: string
1444
+ /** Standbys whose replay lag was read; `seconds` covers only these. */
1445
+ lagStandbys?: number
1446
+ /** The worst standby's lowest recorded lag over `sustainedWindow`, across every scrape of it; it was already reporting by the window's start. */
1447
+ sustainedSeconds?: number
1448
+ sustainedPod?: string
1449
+ sustainedWindow?: string
1450
+ isolation?: CNPGMetricIsolation
1451
+ /** Instances reporting that they are in recovery (standbys). */
1452
+ standbys?: number
1453
+ /** Standbys whose WAL receiver is up (cnpg_pg_replication_is_wal_receiver_up = 1). */
1454
+ receiving?: number
1455
+ /** Standbys whose WAL receiver is down: their lag reads 0 because nothing arrives. */
1456
+ receiverDown?: string[]
1457
+ /** The exporter reported no receiver state (or the read failed), so streaming is not established from lag alone. */
1458
+ receiverUnknown?: boolean
1459
+ receiverReason?: string
1460
+ /** Standbys whose receiver was down in every sample over `receiverDownWindow`: only these raise a problem, since a restarting standby is briefly down. */
1461
+ receiverDownSustained?: string[]
1462
+ receiverDownWindow?: string
1463
+ }
1464
+ /** Inactive physical replication slots and the WAL each keeps: ok | noSeries | denied | ambiguous | scopeMismatch | error | notRead */
1465
+ slots?: {
1466
+ state: string
1467
+ grant?: Grant
1468
+ reason?: string
1469
+ /** Null unless `state` is ok; [] when ok and none is inactive. */
1470
+ inactive?: { slot: string; pod: string; role?: 'primary' | 'standby'; bytes: number | null }[] | null
1471
+ /** Inactive slots beyond the per-cluster cap. */
1472
+ omitted?: number
1473
+ isolation?: CNPGMetricIsolation
1474
+ }
1475
+ /** ok | noSeries | denied | unavailable | error | notRead */
1476
+ growth: { state: string; grant?: Grant; reason?: string; bytesPerHour?: number; claim?: string; instance?: string; isolation?: CNPGMetricIsolation }
1477
+ }
1478
+
1479
+ /** How Prometheus series were tied to one cluster; `unverified` matched only by namespace and Pod or claim names. */
1480
+ export interface CNPGMetricIsolation {
1481
+ mode: 'configured' | 'verified' | 'unverified'
1482
+ note: string
1483
+ }
1484
+
1485
+ // A finding stated as this cluster's must say when its series were matched by name alone.
1486
+ function isolationCaveat(iso: CNPGMetricIsolation | undefined): string {
1487
+ return iso?.mode === 'unverified' ? ` ${iso.note}.` : ''
1488
+ }
1489
+
1490
+ export interface CNPGFleetMetricsSources {
1491
+ /** prometheus, or none when Radar has no Prometheus (`reason` says why). */
1492
+ source: 'prometheus' | 'none'
1493
+ reason?: string
1494
+ lagSource?: string
1495
+ growthSource?: string
1496
+ }
1497
+
1498
+ export function cnpgLagTone(seconds: number): HealthLevel {
1499
+ if (seconds >= 30) return 'unhealthy'
1500
+ if (seconds >= 5) return 'degraded'
1501
+ return 'healthy'
1502
+ }
1503
+
1504
+ /**
1505
+ * Replication's tone from the primary's pg_stat_replication: a missing
1506
+ * standby is degraded, and the lag of the ones that do stream can make it
1507
+ * worse. Missing standbys never hide a severe lag.
1508
+ */
1509
+ export function cnpgReplicationTone(streaming: number, expected: number, maxLagSeconds: number | undefined): HealthLevel {
1510
+ const missing: HealthLevel = streaming < expected ? 'degraded' : 'healthy'
1511
+ return maxLagSeconds === undefined ? missing : worseTone(missing, cnpgLagTone(maxLagSeconds))
1512
+ }
1513
+
1514
+ /**
1515
+ * The one way a replay lag reads: milliseconds below a second, one decimal
1516
+ * below 10 s, whole seconds below 100 s, then whole minutes, then hours and
1517
+ * minutes. Rounded down, so a lower bound stays one.
1518
+ */
1519
+ export function cnpgFormatLag(s: number): string {
1520
+ if (s <= 0) return '0 s'
1521
+ if (s < 1) return `${Math.floor(s * 1000)} ms`
1522
+ if (s < 10) return `${(Math.floor(s * 10) / 10).toFixed(1)} s`
1523
+ if (s < 100) return `${Math.floor(s)} s`
1524
+ const minutes = Math.floor(s / 60)
1525
+ if (minutes < 60) return `${minutes} min`
1526
+ const m = minutes % 60
1527
+ return m === 0 ? `${Math.floor(minutes / 60)} h` : `${Math.floor(minutes / 60)} h ${m} min`
1528
+ }
1529
+
1530
+ function measuredReplication(base: Fact, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources, designated?: string, expectedStandbys?: number | null): Fact {
1531
+ const prefix = base.text.replace(/ · lag unknown$/, '')
1532
+ if (src.source === 'none') {
1533
+ return { text: `${prefix} · lag unknown`, tone: 'unknown', source: CNPG_PROMETHEUS_NOT_CONNECTED, detail: src.reason }
1534
+ }
1535
+ const lag = reading?.lag
1536
+ switch (lag?.state) {
1537
+ case 'ok': {
1538
+ const down = (lag.receiverDown ?? []).filter((pod) => pod !== designated)
1539
+ if (down.length > 0) {
1540
+ return {
1541
+ text: `${prefix} · ${down.length === 1 ? `${down[0]} not receiving WAL` : `${down.length} standbys not receiving WAL`}`,
1542
+ tone: lag.receiving === 0 && down.length === lag.standbys ? 'unhealthy' : 'degraded',
1543
+ source: `WAL receiver down on ${down.join(', ')} (cnpg_pg_replication_is_wal_receiver_up = 0) · ${src.lagSource ?? 'Prometheus'}.${isolationCaveat(lag.isolation)}`,
1544
+ }
1545
+ }
1546
+ if (lag.seconds === undefined) break
1547
+ // The lag covers the standbys whose lag was read; one that is not read is unknown, not caught up.
1548
+ const partial = expectedStandbys !== undefined && expectedStandbys !== null && lag.lagStandbys !== undefined && lag.lagStandbys < expectedStandbys
1549
+ const coverage = partial ? ` (${lag.lagStandbys} of ${expectedStandbys} ${designated !== undefined ? 'instances' : 'standbys'} reporting)` : ''
1550
+ const lagTone = cnpgLagTone(lag.seconds)
1551
+ // A replica cluster's lag is replay behind what arrived; a dead upstream
1552
+ // feed delivers nothing and still reads 0.
1553
+ const upstream = designated !== undefined
1554
+ return {
1555
+ text: `${prefix} · lag ${cnpgFormatLag(lag.seconds)}${coverage}${lag.receiverUnknown ? ' · streaming unverified' : ''}${upstream ? ' · upstream feed not assessed' : ''}`,
1556
+ tone: lag.receiverUnknown || ((partial || upstream) && lagTone === 'healthy') ? 'unknown' : lagTone,
1557
+ source: `Largest standby replay lag, ${lag.pod ?? 'a standby'} · ${src.lagSource ?? 'Prometheus'}.${lag.receiverUnknown ? ' The exporter reported no WAL receiver state, and a standby that receives nothing also reads 0.' : ''}${upstream ? ' The designated primary replays WAL from an external cluster; Radar does not assess that feed.' : ''}${isolationCaveat(lag.isolation)}`,
1558
+ }
1559
+ }
1560
+ case 'noStandby':
1561
+ return { text: `${prefix} · lag unknown`, tone: 'unknown', source: `No standby reports lag: ${lag.reason ?? 'no instance reports being a standby'} · ${src.lagSource ?? 'Prometheus'}` }
1562
+ case 'denied':
1563
+ return { text: `${prefix} · lag unknown`, tone: 'unknown', source: lag.grant ? `Needs ${formatGrant(lag.grant)}` : lag.reason }
1564
+ }
1565
+ return { text: `${prefix} · lag unknown`, tone: 'unknown', source: lag?.reason ?? 'Replication lag needs Prometheus scraping the CNPG exporter' }
1566
+ }
1567
+
1568
+ /** Volume growth of the fastest-growing claim, as a fact. */
1569
+ export function cnpgDiskGrowthFact(reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): Fact | undefined {
1570
+ const g = reading?.growth
1571
+ if (src.source === 'none' || !g || g.state !== 'ok' || g.bytesPerHour === undefined) return undefined
1572
+ const perDay = g.bytesPerHour * 24
1573
+ const text = Math.abs(perDay) < 1024 ? 'flat over 6 h' : `${perDay > 0 ? '+' : '−'}${formatBytes(Math.abs(perDay))}/day`
1574
+ return { text, tone: 'neutral', source: `Fastest-growing: ${g.claim ?? 'a volume'}${g.instance ? ` of ${g.instance}` : ''} · ${src.growthSource ?? 'Prometheus'}.${isolationCaveat(g.isolation)}` }
1575
+ }
1576
+
1577
+ /**
1578
+ * Joins /api/cnpg/fleet-metrics into the fleet: a cluster whose replication
1579
+ * fact is only Pod readiness gets its measured standby lag, or says why it has
1580
+ * none; the disk growth, when measured, lands on `diskGrowth`. Only lag that
1581
+ * stayed high for the whole sustained window raises a problem; a spike is
1582
+ * shown, not judged, and growth is never judged.
1583
+ */
1584
+ export function applyCNPGFleetMetrics(fleet: CNPGFleet, readings: CNPGFleetMetricsReading[] | undefined, src: CNPGFleetMetricsSources | undefined): CNPGFleet {
1585
+ if (!src) return fleet
1586
+ const byKey = new Map((readings ?? []).map((r) => [key(r.namespace, r.name), r]))
1587
+ const rows = fleet.rows.map((row) => {
1588
+ const reading = byKey.get(row.key)
1589
+ const next: CNPGFleetRow = { ...row, diskGrowth: cnpgDiskGrowthFact(reading, src) }
1590
+ if (row.replication.source === CNPG_LAG_UNMEASURED_SOURCE) next.replication = measuredReplication(
1591
+ row.replication,
1592
+ reading,
1593
+ src,
1594
+ row.replicaCluster ? (row.cluster?.status?.currentPrimary ?? '') : undefined,
1595
+ // Every instance of a replica cluster is in recovery, its designated primary included.
1596
+ row.instances.desired !== null ? (row.replicaCluster ? row.instances.desired : Math.max(0, row.instances.desired - 1)) : null,
1597
+ )
1598
+ const sustained = sustainedLagProblem(row, reading, src)
1599
+ return cnpgWithProblems(next, [...(sustained ? [sustained] : []), ...fleetStandbyProblems(row, reading, src), ...fleetSlotProblems(row, reading, src)])
1600
+ })
1601
+ return finishFleet(rows, fleet.incompleteKinds)
1602
+ }
1603
+
1604
+ function fleetStandbyProblems(row: CNPGFleetRow, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): CNPGProblem[] {
1605
+ const lag = reading?.lag
1606
+ if (src.source !== 'prometheus' || lag?.state !== 'ok' || row.hibernated) return []
1607
+ // A replica cluster's designated primary is in recovery too, and may be fed
1608
+ // from the WAL archive with no receiver at all.
1609
+ const designated = row.replicaCluster ? row.cluster?.status?.currentPrimary : undefined
1610
+ const down = (lag.receiverDownSustained ?? []).filter((pod) => pod !== designated)
1611
+ const window = lag.receiverDownWindow ? formatWindowWords(lag.receiverDownWindow) : 'several minutes'
1612
+ return down.map((pod) =>
1613
+ cnpgStandbyNotReceivingProblem(row, {
1614
+ pod,
1615
+ evidence: [`its WAL receiver was down in every sample Prometheus recorded over the last ${window}`],
1616
+ measuredBy: lag.isolation?.mode === 'unverified' ? 'Prometheus, matched by Pod name' : 'Prometheus',
1617
+ sourceDetail: `cnpg_pg_replication_is_wal_receiver_up = 0 while cnpg_pg_replication_in_recovery = 1 · ${src.lagSource ?? 'Prometheus'}`,
1618
+ noneReceiving: noStandbyReceives(row, lag),
1619
+ unverifiedMatch: lag.isolation?.mode === 'unverified',
1620
+ }),
1621
+ )
1622
+ }
1623
+
1624
+ // "None receives" only when every expected standby reported its receiver and
1625
+ // every one was down; one that did not report leaves it open.
1626
+ function noStandbyReceives(row: CNPGFleetRow, lag: CNPGFleetMetricsReading['lag']): boolean {
1627
+ const expected = row.instances.desired !== null ? Math.max(0, row.instances.desired - 1) : null
1628
+ const down = lag.receiverDownSustained?.length ?? 0
1629
+ return lag.receiving === 0 && expected !== null && expected > 0 && lag.standbys === expected && down === expected
1630
+ }
1631
+
1632
+ function fleetSlotProblems(row: CNPGFleetRow, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): CNPGProblem[] {
1633
+ const slots = reading?.slots
1634
+ if (src.source !== 'prometheus' || slots?.state !== 'ok' || !slots.inactive) return []
1635
+ // CloudNativePG copies HA slots to the standbys, where nothing streams from
1636
+ // them, so a standby's copy is always inactive. Only the primary's copy
1637
+ // means a consumer stopped.
1638
+ const instances = row.pods.map((p) => p.name)
1639
+ return slots.inactive
1640
+ .filter((s): s is typeof s & { bytes: number } => s.role === 'primary' && s.bytes !== null && s.bytes >= CNPG_SLOT_RETENTION_WARNING_BYTES)
1641
+ .map((s) =>
1642
+ cnpgSlotRetentionProblem(row, {
1643
+ slot: s.slot,
1644
+ pod: s.pod,
1645
+ bytes: s.bytes,
1646
+ standby: cnpgHASlotInstance(row.cluster, s.slot, instances),
1647
+ measuredBy: slots.isolation?.mode === 'unverified' ? 'Prometheus, matched by Pod name' : 'Prometheus',
1648
+ sourceDetail: 'cnpg_pg_replication_slots_pg_wal_lsn_diff where cnpg_pg_replication_slots_active = 0 (physical slots)',
1649
+ unverifiedMatch: slots.isolation?.mode === 'unverified',
1650
+ }),
1651
+ )
1652
+ }
1653
+
1654
+ export const CNPG_SUSTAINED_LAG_WARNING_SECONDS = 30
1655
+ export const CNPG_SUSTAINED_LAG_CRITICAL_SECONDS = 300
1656
+
1657
+ /** The id of a cluster's sustained-lag problem, so other views can find it among the row's problems. */
1658
+ export function cnpgSustainedLagProblemId(rowKey: string): string {
1659
+ return `lag:${rowKey}`
1660
+ }
1661
+
1662
+ function sustainedLagProblem(row: CNPGFleetRow, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): CNPGProblem | undefined {
1663
+ const lag = reading?.lag
1664
+ const floor = lag?.sustainedSeconds
1665
+ if (src.source !== 'prometheus' || lag?.state !== 'ok' || floor === undefined || floor < CNPG_SUSTAINED_LAG_WARNING_SECONDS) return undefined
1666
+ const window = lag.sustainedWindow ? formatWindowWords(lag.sustainedWindow) : 'several minutes'
1667
+ const pod = lag.sustainedPod ?? 'A standby'
1668
+ // The query proves every recorded sample was at least the floor and that
1669
+ // the series existed at the window's start, not that samples were continuous.
1670
+ return {
1671
+ id: cnpgSustainedLagProblemId(row.key),
1672
+ reason: 'CNPGSustainedLag',
1673
+ severity: floor >= CNPG_SUSTAINED_LAG_CRITICAL_SECONDS ? 'critical' : 'warning',
1674
+ category: 'availability',
1675
+ title: `${pod} ≥ ${cnpgFormatLag(floor)} behind in every sample for ${formatWindowShort(lag.sustainedWindow)}`,
1676
+ shortTitle: `${pod}: all samples ≥ ${cnpgFormatLag(floor)} behind (${formatWindowShort(lag.sustainedWindow)})`,
1677
+ detail: `Lowest replay lag in the samples Prometheus recorded over the last ${window}. If Prometheus missed some scrapes, those moments aren't included. If it was still that far behind, a failover to it would lose or wait on that much WAL.${isolationCaveat(lag.isolation)}`,
1678
+ subject: { kind: 'Cluster', group: 'postgresql.cnpg.io', namespace: row.namespace, name: row.name },
1679
+ source: 'measurement',
1680
+ measuredBy: lag.isolation?.mode === 'unverified' ? 'Prometheus, matched by Pod name' : 'Prometheus',
1681
+ unverifiedMatch: lag.isolation?.mode === 'unverified',
1682
+ sourceDetail: src.lagSource ?? 'Prometheus',
1683
+ }
1684
+ }
1685
+
1686
+ // "10m0s" as "10 min"; "1h0m0s" as "1 h", for a title that must stay short.
1687
+ function formatWindowShort(d: string | undefined): string {
1688
+ const m = d ? /^(?:(\d+)h)?(?:(\d+)m)?(?:0s)?$/.exec(d) : null
1689
+ if (!m) return d ?? 'minutes'
1690
+ const minutes = Number(m[1] ?? 0) * 60 + Number(m[2] ?? 0)
1691
+ return minutes >= 60 && minutes % 60 === 0 ? `${minutes / 60} h` : `${minutes} min`
1692
+ }
1693
+
1694
+ // "10m0s" as "10 minutes"; "1h0m0s" as "1 hour".
1695
+ function formatWindowWords(d: string): string {
1696
+ const m = /^(?:(\d+)h)?(?:(\d+)m)?(?:0s)?$/.exec(d)
1697
+ if (!m) return d
1698
+ const minutes = Number(m[1] ?? 0) * 60 + Number(m[2] ?? 0)
1699
+ if (minutes >= 60 && minutes % 60 === 0) return minutes === 60 ? '1 hour' : `${minutes / 60} hours`
1700
+ return minutes === 1 ? '1 minute' : `${minutes} minutes`
1701
+ }