@skyhook-io/k8s-ui 1.16.3 → 1.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -0
- package/package.json +3 -3
- package/src/components/applications/ApplicationDetail.test.tsx +15 -0
- package/src/components/applications/ApplicationDetail.tsx +17 -15
- package/src/components/charts/AreaChart.interaction.test.tsx +118 -0
- package/src/components/charts/AreaChart.test.tsx +25 -0
- package/src/components/charts/AreaChart.tsx +141 -12
- package/src/components/charts/index.ts +1 -0
- package/src/components/checks/ChecksView.tsx +18 -1
- package/src/components/checks/types.ts +2 -13
- package/src/components/cnpg/CNPGBackupSummary.tsx +57 -9
- package/src/components/cnpg/CNPGClusterHASection.test.tsx +90 -0
- package/src/components/cnpg/CNPGClusterHASection.tsx +331 -0
- package/src/components/cnpg/CNPGClusterSummary.test.tsx +275 -0
- package/src/components/cnpg/CNPGClusterSummary.tsx +310 -96
- package/src/components/cnpg/CNPGConnectSection.test.tsx +64 -0
- package/src/components/cnpg/CNPGConnectSection.tsx +178 -0
- package/src/components/cnpg/CNPGDeclarativeSummary.tsx +157 -26
- package/src/components/cnpg/CNPGImageCatalogSummary.tsx +7 -4
- package/src/components/cnpg/CNPGLogicalPath.test.tsx +22 -0
- package/src/components/cnpg/CNPGLogicalPath.tsx +150 -0
- package/src/components/cnpg/CNPGObjectStoreSummary.tsx +9 -6
- package/src/components/cnpg/CNPGObjectSummary.test.tsx +193 -2
- package/src/components/cnpg/CNPGPoolerScheduling.test.tsx +36 -0
- package/src/components/cnpg/CNPGPoolerSummary.tsx +241 -14
- package/src/components/cnpg/CNPGSharedSummary.tsx +15 -8
- package/src/components/cnpg/CNPGWALArchivingFact.test.tsx +39 -0
- package/src/components/cnpg/backupRuns.test.ts +17 -0
- package/src/components/cnpg/backupRuns.ts +19 -0
- package/src/components/cnpg/connect.test.ts +124 -0
- package/src/components/cnpg/connect.ts +155 -0
- package/src/components/cnpg/databaseRole.test.ts +38 -0
- package/src/components/cnpg/databaseRole.ts +80 -0
- package/src/components/cnpg/ha.test.ts +446 -0
- package/src/components/cnpg/ha.ts +621 -0
- package/src/components/cnpg/index.ts +18 -0
- package/src/components/cnpg/logicalReplication.test.ts +121 -0
- package/src/components/cnpg/logicalReplication.ts +252 -0
- package/src/components/cnpg/pooler.test.ts +84 -0
- package/src/components/cnpg/pooler.ts +225 -0
- package/src/components/cnpg/primitives.tsx +21 -117
- package/src/components/cnpg/problems.ts +12 -0
- package/src/components/cnpg/relations.test.ts +63 -13
- package/src/components/cnpg/relations.ts +61 -52
- package/src/components/cnpg/schedule.ts +35 -0
- package/src/components/cnpg/workspace-disk.test.ts +97 -0
- package/src/components/cnpg/workspace-fleet-metrics.test.ts +250 -0
- package/src/components/cnpg/workspace-problems.test.ts +175 -0
- package/src/components/cnpg/workspace.test.ts +380 -9
- package/src/components/cnpg/workspace.ts +1137 -136
- package/src/components/dock/DockContext.test.tsx +32 -0
- package/src/components/dock/DockContext.tsx +18 -4
- package/src/components/dock/LocalTerminalTab.test.tsx +40 -0
- package/src/components/dock/LocalTerminalTab.tsx +4 -3
- package/src/components/dock/TerminalTab.tsx +20 -8
- package/src/components/facts/certainty.tsx +36 -0
- package/src/components/facts/facts.test.tsx +24 -0
- package/src/components/facts/facts.tsx +62 -0
- package/src/components/facts/index.ts +6 -0
- package/src/components/facts/managed-by.test.tsx +17 -0
- package/src/components/facts/managed-by.tsx +36 -0
- package/src/components/facts/public-exports.test.tsx +35 -0
- package/src/components/gitops/GitOpsGraphFilterRail.tsx +6 -2
- package/src/components/gitops/GitOpsHealthSourceNotice.test.tsx +38 -0
- package/src/components/gitops/GitOpsHealthSourceNotice.tsx +13 -3
- package/src/components/gitops/GitOpsTableView.tsx +9 -24
- package/src/components/gitops/detail-helpers.ts +1 -1
- package/src/components/gitops/insights/ArgoResourceDiff.test.tsx +115 -0
- package/src/components/gitops/insights/ArgoResourceDiff.tsx +22 -5
- package/src/components/gitops/insights/DriftPanel.tsx +62 -0
- package/src/components/gitops/insights/GitOpsInsightViews.tsx +29 -83
- package/src/components/gitops/tree/GitOpsTreeGraph.test.ts +26 -1
- package/src/components/gitops/tree/GitOpsTreeGraph.tsx +40 -60
- package/src/components/gitops/tree/layout.test.ts +111 -0
- package/src/components/gitops/tree/layout.ts +128 -0
- package/src/components/gitops/tree/tree-helpers.test.ts +17 -0
- package/src/components/gitops/tree/tree-helpers.ts +7 -1
- package/src/components/issues/ConditionalCleanup.test.tsx +82 -0
- package/src/components/issues/IssuesView.tsx +40 -5
- package/src/components/issues/ResourceIssuesSection.test.tsx +28 -0
- package/src/components/issues/ResourceIssuesSection.tsx +3 -0
- package/src/components/issues/index.ts +3 -1
- package/src/components/issues/issue-timing.ts +14 -3
- package/src/components/issues/issues.test.ts +50 -2
- package/src/components/issues/severity.ts +21 -0
- package/src/components/issues/types.ts +23 -0
- package/src/components/logs/LogCore.theme.test.tsx +81 -3
- package/src/components/logs/LogCore.tsx +25 -17
- package/src/components/logs/LogToolbarSelects.tsx +3 -1
- package/src/components/logs/LogsViewer.test.tsx +291 -0
- package/src/components/logs/LogsViewer.tsx +72 -11
- package/src/components/logs/StructuredLogLine.test.tsx +76 -0
- package/src/components/logs/StructuredLogLine.tsx +22 -30
- package/src/components/logs/WorkloadLogsViewer.notice.test.tsx +59 -0
- package/src/components/logs/WorkloadLogsViewer.test.tsx +65 -0
- package/src/components/logs/WorkloadLogsViewer.tsx +44 -10
- package/src/components/logs/useLogBuffer.test.ts +8 -0
- package/src/components/logs/useLogStream.ts +28 -1
- package/src/components/problems/index.ts +1 -0
- package/src/components/problems/problems.test.tsx +63 -0
- package/src/components/problems/problems.tsx +165 -0
- package/src/components/resources/ActiveFilterBar.test.tsx +78 -0
- package/src/components/resources/ActiveFilterBar.tsx +86 -0
- package/src/components/resources/ResourcesSidebar.keyboard.test.tsx +42 -0
- package/src/components/resources/ResourcesSidebar.test.tsx +36 -2
- package/src/components/resources/ResourcesSidebar.tsx +92 -41
- package/src/components/resources/ResourcesView.kind-filter-memory.test.ts +52 -0
- package/src/components/resources/ResourcesView.kind-view.test.tsx +149 -0
- package/src/components/resources/ResourcesView.per-container.test.ts +56 -1
- package/src/components/resources/ResourcesView.tsx +309 -117
- package/src/components/resources/curated-column-ownership.test.ts +3 -3
- package/src/components/resources/renderers/AnalysisRunRenderer.tsx +3 -3
- package/src/components/resources/renderers/ArgoApplicationRenderer.tsx +6 -5
- package/src/components/resources/renderers/AutoscalingReferences.test.tsx +286 -0
- package/src/components/resources/renderers/CAPIClusterRenderer.tsx +5 -4
- package/src/components/resources/renderers/CAPIExactReferenceNavigation.test.tsx +27 -0
- package/src/components/resources/renderers/CAPIKubeadmControlPlaneRenderer.tsx +2 -2
- package/src/components/resources/renderers/CAPIMachineDeploymentRenderer.tsx +2 -2
- package/src/components/resources/renderers/CAPIMachinePoolRenderer.tsx +2 -2
- package/src/components/resources/renderers/CAPIMachineRenderer.tsx +2 -2
- package/src/components/resources/renderers/CAPIMachineSetRenderer.tsx +2 -2
- package/src/components/resources/renderers/CNPGBackupRenderer.test.tsx +23 -0
- package/src/components/resources/renderers/CNPGBackupRenderer.tsx +8 -19
- package/src/components/resources/renderers/CNPGClusterRenderer.tsx +7 -8
- package/src/components/resources/renderers/CNPGDeclarativeRenderer.tsx +25 -0
- package/src/components/resources/renderers/CNPGScheduledBackupRenderer.test.tsx +32 -0
- package/src/components/resources/renderers/CNPGScheduledBackupRenderer.tsx +9 -17
- package/src/components/resources/renderers/CNPGShared.tsx +2 -2
- package/src/components/resources/renderers/ClusterComplianceReportRenderer.tsx +2 -2
- package/src/components/resources/renderers/CompositeRenderer.navigation.test.tsx +35 -0
- package/src/components/resources/renderers/CompositeRenderer.tsx +3 -3
- package/src/components/resources/renderers/CompositionRenderer.tsx +3 -3
- package/src/components/resources/renderers/ConfigAuditReportRenderer.tsx +2 -2
- package/src/components/resources/renderers/ContourExactReferenceNavigation.test.tsx +34 -0
- package/src/components/resources/renderers/ContourHTTPProxyRenderer.tsx +11 -10
- package/src/components/resources/renderers/CronJobRenderer.tsx +6 -4
- package/src/components/resources/renderers/CronWorkflowRenderer.tsx +3 -2
- package/src/components/resources/renderers/EndpointSliceRenderer.test.tsx +62 -0
- package/src/components/resources/renderers/EndpointSliceRenderer.tsx +15 -14
- package/src/components/resources/renderers/EventRenderer.tsx +3 -3
- package/src/components/resources/renderers/ExperimentRenderer.tsx +2 -2
- package/src/components/resources/renderers/ExposedSecretReportRenderer.tsx +2 -2
- package/src/components/resources/renderers/ExternalSecretRenderer.tsx +2 -2
- package/src/components/resources/renderers/FluxHelmReleaseRenderer.tsx +4 -4
- package/src/components/resources/renderers/GRPCRouteRenderer.tsx +11 -24
- package/src/components/resources/renderers/GatewayRouteReferences.test.tsx +63 -0
- package/src/components/resources/renderers/GitRepositoryRenderer.tsx +3 -3
- package/src/components/resources/renderers/HPARenderer.tsx +57 -17
- package/src/components/resources/renderers/HTTPRouteRenderer.tsx +12 -25
- package/src/components/resources/renderers/HelmRepositoryRenderer.tsx +3 -3
- package/src/components/resources/renderers/IngressClassReferences.test.tsx +52 -0
- package/src/components/resources/renderers/IngressClassRenderer.tsx +10 -3
- package/src/components/resources/renderers/IngressRenderer.defaultBackend.test.tsx +33 -0
- package/src/components/resources/renderers/IngressRenderer.tsx +36 -5
- package/src/components/resources/renderers/IstioDestinationRuleRenderer.tsx +9 -9
- package/src/components/resources/renderers/IstioExactReferenceNavigation.test.tsx +34 -0
- package/src/components/resources/renderers/IstioVirtualServiceRenderer.tsx +5 -8
- package/src/components/resources/renderers/KarpenterNodeClaimRenderer.tsx +2 -14
- package/src/components/resources/renderers/KedaCredentials.test.tsx +85 -0
- package/src/components/resources/renderers/KedaScaledObjectRenderer.tsx +15 -13
- package/src/components/resources/renderers/KedaTriggerAuthRenderer.tsx +6 -3
- package/src/components/resources/renderers/KnativeEventingRenderer.tsx +24 -31
- package/src/components/resources/renderers/KnativeExactReferenceNavigation.test.tsx +47 -0
- package/src/components/resources/renderers/KnativeFlowRenderer.tsx +6 -7
- package/src/components/resources/renderers/KnativeNetworkingRenderer.tsx +7 -9
- package/src/components/resources/renderers/KnativeReferenceLink.tsx +14 -0
- package/src/components/resources/renderers/KnativeReferenceLinks.test.tsx +60 -0
- package/src/components/resources/renderers/KnativeSourceRenderer.tsx +9 -11
- package/src/components/resources/renderers/KustomizationRenderer.tsx +2 -2
- package/src/components/resources/renderers/KyvernoGlobalContextRenderer.tsx +3 -2
- package/src/components/resources/renderers/KyvernoQueueRenderers.tsx +4 -3
- package/src/components/resources/renderers/LeaseRenderer.tsx +3 -15
- package/src/components/resources/renderers/ManagedResourceRenderer.tsx +2 -2
- package/src/components/resources/renderers/MetricsUnavailableNotice.tsx +3 -2
- package/src/components/resources/renderers/NamespaceRenderer.test.ts +69 -2
- package/src/components/resources/renderers/NamespaceRenderer.tsx +6 -1
- package/src/components/resources/renderers/NodeRenderer.test.tsx +81 -1
- package/src/components/resources/renderers/NodeRenderer.tsx +56 -15
- package/src/components/resources/renderers/OCIRepositoryRenderer.tsx +3 -3
- package/src/components/resources/renderers/PodClassReferences.test.tsx +42 -0
- package/src/components/resources/renderers/PodNomination.test.tsx +57 -0
- package/src/components/resources/renderers/PodRenderer.test.tsx +32 -0
- package/src/components/resources/renderers/PodRenderer.tsx +62 -36
- package/src/components/resources/renderers/RolloutRenderer.test.tsx +1 -1
- package/src/components/resources/renderers/RolloutRenderer.tsx +3 -2
- package/src/components/resources/renderers/SbomReportRenderer.tsx +2 -2
- package/src/components/resources/renderers/SecretRenderer.test.tsx +88 -0
- package/src/components/resources/renderers/SecretRenderer.tsx +34 -5
- package/src/components/resources/renderers/ServiceRenderer.test.tsx +43 -0
- package/src/components/resources/renderers/ServiceRenderer.tsx +11 -3
- package/src/components/resources/renderers/SimpleRouteRenderer.tsx +9 -22
- package/src/components/resources/renderers/VPARenderer.tsx +17 -11
- package/src/components/resources/renderers/VeleroBSLRenderer.tsx +5 -7
- package/src/components/resources/renderers/VeleroBackupRenderer.tsx +3 -3
- package/src/components/resources/renderers/VeleroBackupRepositoryRenderer.tsx +2 -2
- package/src/components/resources/renderers/VeleroRestoreRenderer.tsx +3 -3
- package/src/components/resources/renderers/VeleroScheduleRenderer.tsx +2 -2
- package/src/components/resources/renderers/VulnerabilityReportRenderer.tsx +2 -2
- package/src/components/resources/renderers/WorkflowRenderer.tsx +4 -3
- package/src/components/resources/renderers/kyverno-modern-cells.tsx +1 -1
- package/src/components/resources/renderers/rollout/ReplicaSetProgression.tsx +2 -2
- package/src/components/resources/resource-utils-capi.ts +1 -1
- package/src/components/resources/resource-utils-cnpg.test.ts +35 -10
- package/src/components/resources/resource-utils-cnpg.ts +51 -24
- package/src/components/resources/resource-utils-contour.test.ts +44 -0
- package/src/components/resources/resource-utils-contour.ts +16 -0
- package/src/components/resources/resource-utils-istio.ts +12 -8
- package/src/components/resources/resource-utils-keda.ts +1 -7
- package/src/components/resources/resource-utils-knative.ts +18 -0
- package/src/components/resources/resource-utils-kyverno.ts +1 -7
- package/src/components/resources/resource-utils-velero.ts +0 -12
- package/src/components/resources/resource-utils.ts +99 -39
- package/src/components/resources/rules-column-width.test.ts +1 -1
- package/src/components/resources/summarize-scheduler-message.test.ts +30 -0
- package/src/components/shared/ActionConfirmDialog.test.tsx +154 -0
- package/src/components/shared/ActionConfirmDialog.tsx +246 -0
- package/src/components/shared/CreateResourceDialog.test.tsx +330 -0
- package/src/components/shared/CreateResourceDialog.tsx +126 -80
- package/src/components/shared/DetailShell.test.tsx +80 -0
- package/src/components/shared/DetailShell.tsx +85 -10
- package/src/components/shared/DrainPlanDialog.tsx +1 -0
- package/src/components/shared/EditableYamlView.test.tsx +7 -0
- package/src/components/shared/EditableYamlView.tsx +1 -1
- package/src/components/shared/GitOpsWriteWarning.test.tsx +86 -0
- package/src/components/shared/GitOpsWriteWarning.tsx +135 -0
- package/src/components/shared/ResourceActionsBar.tsx +20 -24
- package/src/components/shared/ResourceRendererDispatch.tsx +13 -10
- package/src/components/shared/SetImageDialog.test.ts +75 -6
- package/src/components/shared/SetImageDialog.tsx +92 -84
- package/src/components/shared/index.ts +4 -1
- package/src/components/timeline/TimelineList.tsx +2 -15
- package/src/components/timeline/TimelineSwimlanes.tsx +3 -11
- package/src/components/timeline/timeline-filters.ts +1 -0
- package/src/components/topology/K8sResourceNode.test.ts +18 -1
- package/src/components/topology/K8sResourceNode.tsx +44 -2
- package/src/components/topology/TopologyFilterSidebar.tsx +1 -0
- package/src/components/topology/TopologyGraph.tsx +8 -74
- package/src/components/topology/TopologySearch.tsx +1 -0
- package/src/components/topology/internet-groups.test.ts +35 -0
- package/src/components/topology/internet-groups.ts +55 -0
- package/src/components/topology/layout.ts +1 -1
- package/src/components/ui/Badge.tsx +1 -0
- package/src/components/ui/FoldSection.tsx +68 -0
- package/src/components/ui/ForceDeleteConfirmDialog.test.tsx +62 -0
- package/src/components/ui/ForceDeleteConfirmDialog.tsx +24 -12
- package/src/components/ui/RefLink.tsx +20 -0
- package/src/components/ui/RowActionMenu.tsx +8 -3
- package/src/components/ui/ScheduleValue.test.tsx +5 -0
- package/src/components/ui/ScheduleValue.tsx +3 -2
- package/src/components/ui/crd-definition-navigation.test.tsx +160 -0
- package/src/components/ui/drawer-components.test.tsx +175 -1
- package/src/components/ui/drawer-components.tsx +103 -36
- package/src/components/ui/index.ts +3 -1
- package/src/components/ui/monacoEditorFeatures.ts +77 -0
- package/src/components/ui/monacoRuntime.ts +14 -19
- package/src/components/ui/status-tone.test.ts +10 -1
- package/src/components/ui/status-tone.tsx +29 -0
- package/src/components/workload/WorkloadView.deleted.test.tsx +101 -0
- package/src/components/workload/WorkloadView.deletedDialog.test.tsx +45 -0
- package/src/components/workload/WorkloadView.relationships.test.tsx +37 -0
- package/src/components/workload/WorkloadView.secretSave.test.tsx +62 -0
- package/src/components/workload/WorkloadView.spec.test.tsx +149 -0
- package/src/components/workload/WorkloadView.tsx +297 -104
- package/src/index.ts +14 -1
- package/src/monaco-deep.d.ts +19 -12
- package/src/theme/components.css +13 -0
- package/src/types/capacity.ts +21 -2
- package/src/types/core.ts +60 -2
- package/src/types/gitops-flux-status.test.ts +32 -1
- package/src/types/gitops-insights.ts +19 -2
- package/src/types/gitops-tree.ts +10 -0
- package/src/types/gitops.ts +26 -6
- package/src/utils/api-resources.ts +7 -0
- package/src/utils/application-history.ts +1 -0
- package/src/utils/cnpg-backup.test.ts +26 -0
- package/src/utils/cnpg-backup.ts +70 -0
- package/src/utils/format.test.ts +49 -2
- package/src/utils/format.ts +9 -1
- package/src/utils/gateway-references.test.ts +17 -0
- package/src/utils/gateway-references.ts +29 -0
- package/src/utils/gitops-owner.ts +3 -2
- package/src/utils/gitops-write-guard.test.ts +401 -0
- package/src/utils/gitops-write-guard.ts +509 -0
- package/src/utils/grant.test.ts +19 -0
- package/src/utils/grant.ts +35 -0
- package/src/utils/index.ts +6 -0
- package/src/utils/ingress-class-references.ts +22 -0
- package/src/utils/k8s-errors.ts +7 -7
- package/src/utils/log-format.test.ts +41 -0
- package/src/utils/log-format.ts +71 -15
- package/src/utils/log-level.ts +12 -8
- package/src/utils/navigation.test.ts +70 -0
- package/src/utils/navigation.ts +91 -8
- package/src/utils/node-lifecycle.test.ts +36 -0
- package/src/utils/node-lifecycle.ts +77 -0
- package/src/utils/object-reference-navigation.test.ts +43 -0
- package/src/utils/resource-hierarchy.test.ts +69 -0
- package/src/utils/resource-hierarchy.ts +41 -7
- package/src/utils/resource-icons.ts +2 -0
- package/src/utils/resource-refs.test.ts +23 -0
- package/src/utils/resource-refs.ts +30 -0
- package/src/utils/skeleton-yaml.test.ts +21 -0
- package/src/utils/skeleton-yaml.ts +16 -0
- package/src/utils/url-path.test.ts +35 -0
- package/src/utils/url-path.ts +21 -0
- package/src/utils/validators.test.ts +4 -13
- package/src/utils/validators.ts +5 -19
- package/src/utils/view-transition.test.ts +35 -1
- package/src/utils/view-transition.ts +16 -1
|
@@ -1,18 +1,31 @@
|
|
|
1
|
+
import { cnpgBackupDeclaration, cnpgBarmanPlugin } from '../../utils/cnpg-backup'
|
|
1
2
|
// CloudNativePG workspace model: pure derivations over the /api/cnpg/workspace
|
|
2
3
|
// payload. Every fact here is something the cluster actually reports; when it
|
|
3
4
|
// does not report something the value is "unknown", never zero or healthy.
|
|
4
5
|
|
|
5
|
-
import
|
|
6
|
+
import { backupsForScheduledBackup, cnpgScheduleDestinationBlocker, cnpgBackupMatchesCluster, cnpgArchiveMatchesCluster, targetCluster } from './relations'
|
|
7
|
+
import { cnpgRoleState } from './databaseRole'
|
|
8
|
+
import { formatAge, summarizeSchedulerMessage, type HealthLevel } from '../resources/resource-utils'
|
|
9
|
+
import { worseTone } from '../ui/status-tone'
|
|
10
|
+
import { type Fact } from '../facts'
|
|
11
|
+
import { type ProblemOrigin, type ProblemDetails, type ProblemProvenance } from '../problems'
|
|
12
|
+
import type { ResourceRef } from '../../types/core'
|
|
13
|
+
import { formatGrant, type Grant } from '../../utils/grant'
|
|
14
|
+
import { issueReasonTitle, issueTitle } from '../issues/severity'
|
|
6
15
|
import {
|
|
7
16
|
CNPG_BARMAN_PLUGIN_NAME,
|
|
8
17
|
getCNPGClusterBackupConfig,
|
|
9
18
|
getCNPGClusterBarmanPlugin,
|
|
10
19
|
getCNPGClusterImageTag,
|
|
20
|
+
getCNPGClusterIsReplica,
|
|
11
21
|
getCNPGClusterStatus,
|
|
12
22
|
getCNPGObjectStoreRecoveryWindows,
|
|
13
23
|
isApiGroup,
|
|
14
24
|
} from '../resources/resource-utils-cnpg'
|
|
15
25
|
|
|
26
|
+
export type CNPGFactTone = HealthLevel
|
|
27
|
+
export type CNPGFact = Fact
|
|
28
|
+
|
|
16
29
|
export const CNPG_WORKSPACE_KEYS = [
|
|
17
30
|
'clusters',
|
|
18
31
|
'backups',
|
|
@@ -21,6 +34,7 @@ export const CNPG_WORKSPACE_KEYS = [
|
|
|
21
34
|
'databases',
|
|
22
35
|
'publications',
|
|
23
36
|
'subscriptions',
|
|
37
|
+
'databaseRoles',
|
|
24
38
|
'imageCatalogs',
|
|
25
39
|
'clusterImageCatalogs',
|
|
26
40
|
'objectStores',
|
|
@@ -29,12 +43,14 @@ export const CNPG_WORKSPACE_KEYS = [
|
|
|
29
43
|
|
|
30
44
|
export type CNPGWorkspaceKey = (typeof CNPG_WORKSPACE_KEYS)[number]
|
|
31
45
|
|
|
32
|
-
export type CNPGCoverageState = 'full' | 'partial' | 'denied' | 'notInstalled' | 'syncing' | 'error'
|
|
46
|
+
export type CNPGCoverageState = 'full' | 'partial' | 'denied' | 'notInstalled' | 'syncing' | 'uncached' | 'error'
|
|
33
47
|
|
|
34
48
|
export interface CNPGKindCoverage {
|
|
35
49
|
state: CNPGCoverageState
|
|
36
50
|
/** Denied namespaces, named only when the caller supplied the candidate list. */
|
|
37
51
|
deniedNamespaces?: string[]
|
|
52
|
+
/** Namespaces the caller may read but Radar's cache does not hold, named under the same rule. */
|
|
53
|
+
uncachedNamespaces?: string[]
|
|
38
54
|
/** For partial coverage: the namespaces that were read. */
|
|
39
55
|
allowedNamespaces?: string[]
|
|
40
56
|
}
|
|
@@ -73,6 +89,14 @@ export interface CNPGWorkspaceResponse {
|
|
|
73
89
|
issues: CNPGWorkspaceIssue[]
|
|
74
90
|
audit: CNPGAuditFinding[]
|
|
75
91
|
backupsOmitted: number
|
|
92
|
+
/** Each ScheduledBackup's schedule as the operator reads it, keyed "namespace/name"; absent when it cannot be parsed. */
|
|
93
|
+
scheduleReadings?: Record<string, string>
|
|
94
|
+
/** The manager of each object that has one, keyed "Kind/namespace/name". */
|
|
95
|
+
managedBy?: Record<string, ResourceRef>
|
|
96
|
+
/** Pods of the Jobs a Cluster controls (initdb, join, restore…), never counted as instances. */
|
|
97
|
+
jobPods?: any[]
|
|
98
|
+
/** Where the caller's Jobs were read; a Job Pod is returned only there. Absent from a Radar that predates it. */
|
|
99
|
+
jobCoverage?: CNPGKindCoverage
|
|
76
100
|
}
|
|
77
101
|
|
|
78
102
|
export const CNPG_KIND_BY_KEY: Record<CNPGWorkspaceKey, { kind: string; group: string; plural: string }> = {
|
|
@@ -83,6 +107,7 @@ export const CNPG_KIND_BY_KEY: Record<CNPGWorkspaceKey, { kind: string; group: s
|
|
|
83
107
|
databases: { kind: 'Database', group: 'postgresql.cnpg.io', plural: 'databases' },
|
|
84
108
|
publications: { kind: 'Publication', group: 'postgresql.cnpg.io', plural: 'publications' },
|
|
85
109
|
subscriptions: { kind: 'Subscription', group: 'postgresql.cnpg.io', plural: 'subscriptions' },
|
|
110
|
+
databaseRoles: { kind: 'DatabaseRole', group: 'postgresql.cnpg.io', plural: 'databaseroles' },
|
|
86
111
|
imageCatalogs: { kind: 'ImageCatalog', group: 'postgresql.cnpg.io', plural: 'imagecatalogs' },
|
|
87
112
|
clusterImageCatalogs: { kind: 'ClusterImageCatalog', group: 'postgresql.cnpg.io', plural: 'clusterimagecatalogs' },
|
|
88
113
|
objectStores: { kind: 'ObjectStore', group: 'barmancloud.cnpg.io', plural: 'objectstores' },
|
|
@@ -93,37 +118,31 @@ export function isCNPGWorkspaceKind(kind: string, group: string | undefined): bo
|
|
|
93
118
|
return Object.values(CNPG_KIND_BY_KEY).some((k) => k.group !== '' && k.group === (group ?? '') && k.kind === kind)
|
|
94
119
|
}
|
|
95
120
|
|
|
96
|
-
/** The value is observed, derived, or not available from the cluster. */
|
|
97
|
-
export type CNPGFactTone = HealthLevel
|
|
98
|
-
|
|
99
|
-
export interface CNPGFact {
|
|
100
|
-
text: string
|
|
101
|
-
tone: CNPGFactTone
|
|
102
|
-
/** Where the value comes from, shown next to it so claims carry their source. */
|
|
103
|
-
source?: string
|
|
104
|
-
/** A timestamp the text refers to; the UI renders it as an age. */
|
|
105
|
-
at?: string
|
|
106
|
-
}
|
|
107
|
-
|
|
108
121
|
export type CNPGProblemCategory = 'availability' | 'protection' | 'declarations' | 'pooling'
|
|
109
122
|
|
|
110
123
|
export const CNPG_PROBLEM_CATEGORIES: { id: CNPGProblemCategory; label: string }[] = [
|
|
111
124
|
{ id: 'availability', label: 'Availability' },
|
|
112
|
-
{ id: 'protection', label: '
|
|
125
|
+
{ id: 'protection', label: 'Backups' },
|
|
113
126
|
{ id: 'declarations', label: 'Declarations' },
|
|
114
127
|
{ id: 'pooling', label: 'Pooling' },
|
|
115
128
|
]
|
|
116
129
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
130
|
+
/**
|
|
131
|
+
* A problem in the CloudNativePG workspace, categorised by the workspace's
|
|
132
|
+
* screens. `instance` names the instance a Cluster-level problem is about,
|
|
133
|
+
* e.g. the standby an HA slot is kept for.
|
|
134
|
+
*/
|
|
135
|
+
export type CNPGProblem = ProblemDetails<CNPGProblemCategory> & ProblemProvenance & {
|
|
120
136
|
severity: 'critical' | 'warning' | 'posture'
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
137
|
+
/** The measurement matched by name rather than by UID. */
|
|
138
|
+
unverifiedMatch?: boolean
|
|
139
|
+
reason?: string
|
|
140
|
+
slot?: string
|
|
141
|
+
instance?: string
|
|
142
|
+
/** The cnpg.io/jobRole of the Cluster's own Job whose Pod this is about: a cause, not an instance's symptom. */
|
|
143
|
+
job?: string
|
|
144
|
+
/** The Cluster's own Ready condition: a roll-up of the other problems, shown after them. */
|
|
145
|
+
rollup?: boolean
|
|
127
146
|
}
|
|
128
147
|
|
|
129
148
|
export interface CNPGInstance {
|
|
@@ -135,15 +154,15 @@ export interface CNPGInstance {
|
|
|
135
154
|
}
|
|
136
155
|
|
|
137
156
|
export interface CNPGProtectionFacts {
|
|
138
|
-
schedule:
|
|
139
|
-
destination:
|
|
157
|
+
schedule: Fact & { names: string[] }
|
|
158
|
+
destination: Fact & {
|
|
140
159
|
method: 'plugin' | 'barmanObjectStore' | 'volumeSnapshot' | 'none'
|
|
141
160
|
objectStore?: string
|
|
142
161
|
}
|
|
143
|
-
lastSuccessfulBackup:
|
|
144
|
-
walArchiving:
|
|
145
|
-
recoveryWindow:
|
|
146
|
-
restoreValidation:
|
|
162
|
+
lastSuccessfulBackup: Fact
|
|
163
|
+
walArchiving: Fact & { state?: 'no_destination' | 'failing' | 'archiving' | 'unknown'; operatorCondition?: { type: string; status: string; message?: string; lastTransitionTime?: string } }
|
|
164
|
+
recoveryWindow: Fact & { from?: string }
|
|
165
|
+
restoreValidation: Fact & { restoredInto?: { namespace: string; name: string } }
|
|
147
166
|
}
|
|
148
167
|
|
|
149
168
|
export interface CNPGFleetRow {
|
|
@@ -153,15 +172,26 @@ export interface CNPGFleetRow {
|
|
|
153
172
|
cluster: any
|
|
154
173
|
controllerStatus: { text: string; level: HealthLevel }
|
|
155
174
|
instances: { ready: number | null; desired: number | null }
|
|
175
|
+
/**
|
|
176
|
+
* Ready instances counted from the instance Pods' Ready condition; absent
|
|
177
|
+
* when Pods are not readable here or any Pod's readiness is unknown.
|
|
178
|
+
*/
|
|
179
|
+
podReadiness?: { ready: number; total: number }
|
|
180
|
+
/** status.readyInstances claims more ready instances than the Pods show: CNPG status is stale or lagging. */
|
|
181
|
+
readinessContradicted?: boolean
|
|
182
|
+
/** status.currentPrimary is not the Pod labelled primary; status may be stale, or a failover is under way. */
|
|
183
|
+
primaryConflict?: { status: string; labelled: string }
|
|
156
184
|
pods: CNPGInstance[]
|
|
157
185
|
replicaCluster: { source?: string } | null
|
|
158
186
|
hibernated: boolean
|
|
159
187
|
pgVersion: string | null
|
|
160
188
|
catalog: { kind: string; name: string } | null
|
|
161
|
-
replication:
|
|
162
|
-
protection: CNPGProtectionFacts & { summary:
|
|
163
|
-
declarations: { summary:
|
|
189
|
+
replication: Fact
|
|
190
|
+
protection: CNPGProtectionFacts & { summary: Fact }
|
|
191
|
+
declarations: { summary: Fact; total: number; failed: number; pending: number }
|
|
164
192
|
poolers: string[]
|
|
193
|
+
/** The Pooler objects behind `poolers`, for their type and Service port. */
|
|
194
|
+
poolerObjects?: any[]
|
|
165
195
|
/** False when Poolers are not readable in this cluster's namespace, so an empty list means unknown. */
|
|
166
196
|
poolersKnown: boolean
|
|
167
197
|
problems: CNPGProblem[]
|
|
@@ -169,7 +199,12 @@ export interface CNPGFleetRow {
|
|
|
169
199
|
attention: boolean
|
|
170
200
|
categories: Set<CNPGProblemCategory>
|
|
171
201
|
/** GitOps owner recorded on the Cluster, when it carries the standard labels. */
|
|
172
|
-
|
|
202
|
+
/** The GitOps or Helm object that manages the Cluster, as the server detected it. */
|
|
203
|
+
managedBy?: ResourceRef
|
|
204
|
+
/** Fullest measured volume, set by applyCNPGDisk; absent when no disk reading was requested. */
|
|
205
|
+
disk?: Fact
|
|
206
|
+
/** Growth of the fastest-growing volume, set by applyCNPGFleetMetrics when measured. */
|
|
207
|
+
diskGrowth?: Fact
|
|
173
208
|
}
|
|
174
209
|
|
|
175
210
|
export interface CNPGFleet {
|
|
@@ -185,8 +220,195 @@ const PROTECTION_ISSUE_REASONS = new Set([
|
|
|
185
220
|
'CNPGLastBackupFailed',
|
|
186
221
|
'CNPGBackupFailed',
|
|
187
222
|
'CNPGScheduledBackupMissed',
|
|
223
|
+
'CNPGScheduledRunNoBackup',
|
|
224
|
+
'CNPGScheduleDestinationMissing',
|
|
188
225
|
])
|
|
189
226
|
|
|
227
|
+
// What an instance Pod's bare reason means, said about the Pod.
|
|
228
|
+
const CNPG_POD_REASON_SENTENCES: Record<string, string> = {
|
|
229
|
+
ReadinessProbeFailed: 'not ready (readiness probe failing)',
|
|
230
|
+
// The issue does not say whether the Pod is serving now (it may have come
|
|
231
|
+
// back within the settle window), so restarts are worded as past.
|
|
232
|
+
LivenessProbeFailed: 'restarted recently (liveness probe failing)',
|
|
233
|
+
CrashLoopBackOff: 'restarted recently (CrashLoopBackOff)',
|
|
234
|
+
HighRestartCount: 'restarted repeatedly',
|
|
235
|
+
OOMKilled: 'killed for running out of memory (OOMKilled)',
|
|
236
|
+
ImagePullBackOff: 'cannot pull its image (ImagePullBackOff)',
|
|
237
|
+
ErrImagePull: 'cannot pull its image (ErrImagePull)',
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Plain headlines for CNPG issues whose message carries the operator's own
|
|
241
|
+
// condition text; that message becomes the detail beneath.
|
|
242
|
+
// Reasons the Issues page already titles come from issueReasonTitle, so a
|
|
243
|
+
// problem reads the same here and there; these are the rest.
|
|
244
|
+
const CNPG_REASON_TITLES: Record<string, string> = {
|
|
245
|
+
CNPGBackupFailed: 'Backup failed',
|
|
246
|
+
CNPGScheduledBackupMissed: 'A scheduled backup did not run',
|
|
247
|
+
CNPGCertificateExpiring: 'A certificate expires soon',
|
|
248
|
+
CNPGCertificateExpired: 'A certificate has expired',
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* A problem's headline and detail from an issue. Known CNPG reasons get a
|
|
253
|
+
* short plain title with the operator's message beneath; otherwise the
|
|
254
|
+
* message is the title, unless it is empty or only the reason token (e.g.
|
|
255
|
+
* "ReadinessProbeFailed"), which is turned into a sentence about the subject.
|
|
256
|
+
*/
|
|
257
|
+
export function cnpgIssueText(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'name' | 'reason' | 'message' | 'cause' | 'first_seen'>): { title: string; detail?: string } {
|
|
258
|
+
let message = issue.message?.trim() ?? ''
|
|
259
|
+
const cause = issue.cause?.trim() || undefined
|
|
260
|
+
// The run's time is the issue's first_seen, not part of the message.
|
|
261
|
+
if (issue.reason === 'CNPGScheduledRunNoBackup' && issue.first_seen && message) message = `${message} ${formatAge(issue.first_seen)} ago`
|
|
262
|
+
if (['CNPGScheduleDestinationMissing', 'CNPGInstanceReadinessMismatch', 'CNPGPrimaryLabelMismatch'].includes(issue.reason)) return { title: message, detail: cause }
|
|
263
|
+
const known = issueReasonTitle(issue.reason) ?? CNPG_REASON_TITLES[issue.reason]
|
|
264
|
+
if (known) return { title: known, detail: [stripTitlePrefix(message, known), cause].filter(Boolean).join(' ') || undefined }
|
|
265
|
+
if (message && message !== issue.reason && /\s/.test(message)) return { title: message, detail: cause }
|
|
266
|
+
const token = message || issue.reason
|
|
267
|
+
const sentence = CNPG_POD_REASON_SENTENCES[token]
|
|
268
|
+
if (sentence) return { title: `${issue.name} ${sentence}`, detail: cause }
|
|
269
|
+
const words = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase()
|
|
270
|
+
return { title: `${issue.kind} ${issue.name}: ${words}`, detail: cause }
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// "Backup failed: cannot proceed…" under the title "Backup failed" repeats it.
|
|
274
|
+
function stripTitlePrefix(message: string, title: string): string {
|
|
275
|
+
if (!message.toLowerCase().startsWith(title.toLowerCase())) return message
|
|
276
|
+
const rest = message.slice(title.length).replace(/^[\s:;,.\-–—]+/, '')
|
|
277
|
+
return rest ? rest[0].toUpperCase() + rest.slice(1) : ''
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Failed Backups of one cluster that failed for the same reason become one
|
|
282
|
+
* problem: "3 backups failed: <reason>", about the latest of them, the others
|
|
283
|
+
* named in alsoAbout. Each Backup is otherwise its own issue, and a schedule
|
|
284
|
+
* failing every night would list the same sentence over and over.
|
|
285
|
+
*/
|
|
286
|
+
type IssueProblem = CNPGProblem
|
|
287
|
+
|
|
288
|
+
export function cnpgCollapseBackupFailures(problems: IssueProblem[], backupTimes: Map<string, number> = new Map()): IssueProblem[] {
|
|
289
|
+
const groups = new Map<string, IssueProblem[]>()
|
|
290
|
+
const out: IssueProblem[] = []
|
|
291
|
+
for (const p of problems) {
|
|
292
|
+
if (p.reason !== 'CNPGBackupFailed' || p.subject.kind !== 'Backup') {
|
|
293
|
+
out.push(p)
|
|
294
|
+
continue
|
|
295
|
+
}
|
|
296
|
+
const key = `${p.severity}\x00${p.detail ?? ''}`
|
|
297
|
+
groups.set(key, [...(groups.get(key) ?? []), p])
|
|
298
|
+
}
|
|
299
|
+
for (const list of groups.values()) {
|
|
300
|
+
if (list.length === 1) {
|
|
301
|
+
out.push(list[0])
|
|
302
|
+
continue
|
|
303
|
+
}
|
|
304
|
+
const at = (p: IssueProblem) => backupTimes.get(p.subject.name) ?? 0
|
|
305
|
+
const sorted = [...list].sort((a, b) => at(b) - at(a) || b.subject.name.localeCompare(a.subject.name))
|
|
306
|
+
const latest = sorted[0]
|
|
307
|
+
out.push({
|
|
308
|
+
...latest,
|
|
309
|
+
id: `backups-failed:${latest.subject.namespace}:${latest.detail ?? ''}`,
|
|
310
|
+
title: latest.detail ? `${list.length} backups failed: ${latest.detail[0].toLowerCase()}${latest.detail.slice(1)}` : `${list.length} backups failed`,
|
|
311
|
+
detail: undefined,
|
|
312
|
+
alsoAbout: sorted.slice(1).map((p) => ({ kind: p.subject.kind, name: p.subject.name })),
|
|
313
|
+
})
|
|
314
|
+
}
|
|
315
|
+
return out
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* "Latest backup failed" restates a failed-Backup problem when that problem
|
|
320
|
+
* is about the cluster's newest Backup, so the duplicate is dropped and the
|
|
321
|
+
* count stays honest. With the newest Backup unknown, both stay.
|
|
322
|
+
*/
|
|
323
|
+
export function cnpgFoldLastBackupFailed(problems: IssueProblem[], newestBackup: string | undefined): CNPGProblem[] {
|
|
324
|
+
const covered =
|
|
325
|
+
!!newestBackup &&
|
|
326
|
+
problems.some(
|
|
327
|
+
(p) =>
|
|
328
|
+
p.reason === 'CNPGBackupFailed' &&
|
|
329
|
+
p.subject.kind === 'Backup' &&
|
|
330
|
+
(p.subject.name === newestBackup || p.alsoAbout?.some((o) => o.kind === 'Backup' && o.name === newestBackup)),
|
|
331
|
+
)
|
|
332
|
+
return problems
|
|
333
|
+
.filter((p) => !(covered && p.reason === 'CNPGLastBackupFailed'))
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// When each of the cluster's Backups started (status.startedAt, else its
|
|
337
|
+
// creation), by name: the issues about Backups carry no time of their own.
|
|
338
|
+
function backupTimesOf(cluster: any, backups: any[]): Map<string, number> {
|
|
339
|
+
const out = new Map<string, number>()
|
|
340
|
+
for (const b of backups) {
|
|
341
|
+
if (!cnpgBackupMatchesCluster(b, cluster)) continue
|
|
342
|
+
out.set(b.metadata.name, Date.parse(b?.status?.startedAt ?? b?.metadata?.creationTimestamp ?? '') || 0)
|
|
343
|
+
}
|
|
344
|
+
return out
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
// Each entry names what the Go detector (internal/issues/source_cnpg*.go and
|
|
349
|
+
// the Pod detector) actually reads. "Reported by CNPG" only where the operator
|
|
350
|
+
// itself wrote the failure; a threshold or comparison Radar applies is a
|
|
351
|
+
// "Radar check".
|
|
352
|
+
const CNPG_CONDITION_ORIGINS: Record<string, string> = {
|
|
353
|
+
CNPGLastBackupFailed: 'Cluster LastBackupSucceeded condition',
|
|
354
|
+
CNPGClusterTerminal: 'Cluster status.phase',
|
|
355
|
+
CNPGClusterUnrecoverable: 'Cluster status.phase',
|
|
356
|
+
CNPGClusterPluginFailure: 'Cluster status.phase',
|
|
357
|
+
CNPGClusterFailingOver: 'Cluster status.phase',
|
|
358
|
+
CNPGClusterWaitingForUser: 'Cluster status.phase',
|
|
359
|
+
CNPGDeclarativeNotApplied: 'status.applied and status.message',
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const POD_ORIGINS: Record<string, ProblemOrigin> = {
|
|
363
|
+
ReadinessProbeFailed: { label: 'Pod readiness probe', detail: 'Kubelet probe-failure events and the Pod\'s Ready condition' },
|
|
364
|
+
LivenessProbeFailed: { label: 'Pod liveness probe', detail: 'Kubelet probe-failure events and container restarts' },
|
|
365
|
+
ReadinessProbeInvalid: { label: 'Radar check of the probe', detail: 'The readiness probe names a port the container does not declare' },
|
|
366
|
+
LivenessProbeInvalid: { label: 'Radar check of the probe', detail: 'The liveness probe names a port the container does not declare' },
|
|
367
|
+
HighRestartCount: { label: 'Radar check of restarts', detail: 'More than 3 restarts on a container that is still unhealthy' },
|
|
368
|
+
InitContainerStalled: { label: 'Radar check of init containers', detail: 'An init container has not finished' },
|
|
369
|
+
Unschedulable: { label: 'Kubernetes scheduler', detail: "The Pod's PodScheduled condition (reason Unschedulable) and the scheduler's message" },
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* Where an issue's evidence comes from, in user terms: what CloudNativePG
|
|
374
|
+
* reported, a Backup's or Pod's own status, or Radar's own check. A reason this
|
|
375
|
+
* does not know reads "Detected by Radar" rather than a guessed source.
|
|
376
|
+
*/
|
|
377
|
+
export function cnpgIssueOrigin(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'reason'>): ProblemOrigin {
|
|
378
|
+
const condition = CNPG_CONDITION_ORIGINS[issue.reason]
|
|
379
|
+
if (condition) return { label: 'Reported by CNPG', detail: condition }
|
|
380
|
+
switch (issue.reason) {
|
|
381
|
+
case 'CNPGWALArchivingFailing':
|
|
382
|
+
return issue.kind === 'Backup'
|
|
383
|
+
? { label: 'Backup status', detail: 'Backup status.phase walArchivingFailing and status.error' }
|
|
384
|
+
: { label: 'Reported by CNPG', detail: 'Cluster ContinuousArchiving condition' }
|
|
385
|
+
case 'CNPGBackupFailed':
|
|
386
|
+
return { label: 'Backup status', detail: 'Backup status.phase and status.error' }
|
|
387
|
+
case 'CNPGClusterDegraded':
|
|
388
|
+
return { label: 'Radar check of ready instances', detail: 'spec.instances against status.readyInstances, unless the phase, hibernation or fencing explains it' }
|
|
389
|
+
case 'CNPGScheduleDestinationMissing':
|
|
390
|
+
return { label: 'Radar check of the backup destination', detail: 'ScheduledBackup method against its target Cluster spec' }
|
|
391
|
+
case 'CNPGInstanceReadinessMismatch':
|
|
392
|
+
return { label: 'Radar check of instance readiness', detail: 'Instance Pod Ready conditions against Cluster status.readyInstances' }
|
|
393
|
+
case 'CNPGPrimaryLabelMismatch':
|
|
394
|
+
return { label: 'Radar check of the primary', detail: 'Instance Pod role labels against Cluster status.currentPrimary' }
|
|
395
|
+
case 'CNPGScheduledRunNoBackup':
|
|
396
|
+
return { label: 'Radar check of the backup schedule', detail: 'The schedule, read as the operator does, against the cluster\'s newest successful backup' }
|
|
397
|
+
case 'CNPGScheduledBackupMissed':
|
|
398
|
+
return { label: 'Radar check of the backup schedule', detail: 'ScheduledBackup status.nextScheduleTime passed more than 10 minutes ago' }
|
|
399
|
+
case 'CNPGCertificateExpiring':
|
|
400
|
+
case 'CNPGCertificateExpired':
|
|
401
|
+
return { label: 'Certificate expiry (from Cluster status)', detail: 'Cluster status.certificates.expirations, compared with now' }
|
|
402
|
+
}
|
|
403
|
+
if (issue.kind === 'Pod') return POD_ORIGINS[issue.reason] ?? { label: 'Pod status' }
|
|
404
|
+
return { label: 'Detected by Radar' }
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** The headline alone; see cnpgIssueText. */
|
|
408
|
+
export function cnpgIssueTitle(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'name' | 'reason' | 'message'>): string {
|
|
409
|
+
return cnpgIssueText(issue).title
|
|
410
|
+
}
|
|
411
|
+
|
|
190
412
|
export function cnpgIssueCategory(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'reason'>): CNPGProblemCategory {
|
|
191
413
|
if (PROTECTION_ISSUE_REASONS.has(issue.reason)) return 'protection'
|
|
192
414
|
switch (issue.kind) {
|
|
@@ -197,6 +419,7 @@ export function cnpgIssueCategory(issue: Pick<CNPGWorkspaceIssue, 'kind' | 'reas
|
|
|
197
419
|
case 'Database':
|
|
198
420
|
case 'Publication':
|
|
199
421
|
case 'Subscription':
|
|
422
|
+
case 'DatabaseRole':
|
|
200
423
|
return 'declarations'
|
|
201
424
|
case 'Pooler':
|
|
202
425
|
return 'pooling'
|
|
@@ -215,7 +438,7 @@ function specClusterName(obj: any): string | undefined {
|
|
|
215
438
|
}
|
|
216
439
|
|
|
217
440
|
function coverageOf(resp: CNPGWorkspaceResponse, k: CNPGWorkspaceKey): CNPGKindCoverage {
|
|
218
|
-
return resp.coverage
|
|
441
|
+
return resp.coverage[k] ?? { state: 'uncached' }
|
|
219
442
|
}
|
|
220
443
|
|
|
221
444
|
/** Coverage is usable in a namespace when that namespace's objects were read. */
|
|
@@ -224,23 +447,35 @@ export function coverageReadable(cov: CNPGKindCoverage, namespace?: string): boo
|
|
|
224
447
|
if (cov.state === 'partial') {
|
|
225
448
|
if (!namespace) return false
|
|
226
449
|
if (cov.allowedNamespaces) return cov.allowedNamespaces.includes(namespace)
|
|
227
|
-
if (cov.deniedNamespaces) return !cov.deniedNamespaces.includes(namespace)
|
|
450
|
+
if (cov.deniedNamespaces || cov.uncachedNamespaces) return !cov.deniedNamespaces?.includes(namespace) && !cov.uncachedNamespaces?.includes(namespace)
|
|
228
451
|
return false
|
|
229
452
|
}
|
|
230
453
|
return false
|
|
231
454
|
}
|
|
232
455
|
|
|
233
|
-
|
|
456
|
+
/**
|
|
457
|
+
* Why a kind's objects were not read (in `namespace`, when given), worded for
|
|
458
|
+
* a fact. A namespace Radar's cache does not hold is never called "no access";
|
|
459
|
+
* a partial read that names neither cause says only that it was not read.
|
|
460
|
+
*/
|
|
461
|
+
export function cnpgCoverageGap(cov: CNPGKindCoverage, what: string, namespace?: string, notInstalled = 'Not installed'): string {
|
|
462
|
+
// A namespace the server names wins over the kind's overall state: an
|
|
463
|
+
// uncached scope can still name namespaces the caller was denied.
|
|
464
|
+
if (namespace && cov.deniedNamespaces?.includes(namespace)) return `No access to ${what}`
|
|
465
|
+
if (namespace && cov.uncachedNamespaces?.includes(namespace)) return `Radar does not cache ${what} in ${namespace}`
|
|
234
466
|
switch (cov.state) {
|
|
235
467
|
case 'denied':
|
|
236
|
-
case 'partial':
|
|
237
468
|
return `No access to ${what}`
|
|
469
|
+
case 'partial':
|
|
470
|
+
return namespace ? `${what} not read in ${namespace}` : `${what} not read`
|
|
471
|
+
case 'uncached':
|
|
472
|
+
return `Radar does not cache ${what}`
|
|
238
473
|
case 'syncing':
|
|
239
474
|
return 'Loading…'
|
|
240
475
|
case 'error':
|
|
241
476
|
return `Could not read ${what}`
|
|
242
477
|
default:
|
|
243
|
-
return
|
|
478
|
+
return notInstalled
|
|
244
479
|
}
|
|
245
480
|
}
|
|
246
481
|
|
|
@@ -267,37 +502,24 @@ function podReady(pod: any): boolean | null {
|
|
|
267
502
|
return ready.status === 'True'
|
|
268
503
|
}
|
|
269
504
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
const argo = labels['argocd.argoproj.io/instance']
|
|
277
|
-
if (argo) return { tool: 'argocd', name: argo }
|
|
278
|
-
const tracking = annotations['argocd.argoproj.io/tracking-id']
|
|
279
|
-
if (typeof tracking === 'string' && tracking.includes(':')) {
|
|
280
|
-
return { tool: 'argocd', name: tracking.split(':')[0] }
|
|
281
|
-
}
|
|
282
|
-
const fluxName = labels['kustomize.toolkit.fluxcd.io/name'] || labels['helm.toolkit.fluxcd.io/name']
|
|
283
|
-
if (fluxName) {
|
|
284
|
-
return {
|
|
285
|
-
tool: 'flux',
|
|
286
|
-
name: fluxName,
|
|
287
|
-
namespace: labels['kustomize.toolkit.fluxcd.io/namespace'] || labels['helm.toolkit.fluxcd.io/namespace'],
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
return null
|
|
505
|
+
/** The manager the server detected for a workspace object (Argo CD, Flux or Helm), if any. */
|
|
506
|
+
export function cnpgManagedBy(ws: Pick<CNPGWorkspaceResponse, 'managedBy'> | null | undefined, obj: any): ResourceRef | undefined {
|
|
507
|
+
const kind = obj?.kind
|
|
508
|
+
const name = obj?.metadata?.name
|
|
509
|
+
if (!kind || !name) return undefined
|
|
510
|
+
return ws?.managedBy?.[`${kind}/${obj?.metadata?.namespace ?? ''}/${name}`]
|
|
291
511
|
}
|
|
292
512
|
|
|
293
513
|
function scheduleFact(
|
|
294
514
|
cluster: any,
|
|
295
515
|
schedules: any[],
|
|
296
516
|
cov: CNPGKindCoverage,
|
|
517
|
+
readings: Record<string, string> = {},
|
|
518
|
+
backups: any[] = [],
|
|
297
519
|
): CNPGProtectionFacts['schedule'] {
|
|
298
520
|
const ns = cluster.metadata?.namespace
|
|
299
521
|
if (!coverageReadable(cov, ns)) {
|
|
300
|
-
return { text:
|
|
522
|
+
return { text: cnpgCoverageGap(cov, 'ScheduledBackups', ns), tone: 'unknown', names: [] }
|
|
301
523
|
}
|
|
302
524
|
const mine = schedules.filter((s) => s.metadata?.namespace === ns && specClusterName(s) === cluster.metadata?.name)
|
|
303
525
|
if (mine.length === 0) return { text: 'No declarative schedule', tone: 'neutral', names: [] }
|
|
@@ -307,15 +529,21 @@ function scheduleFact(
|
|
|
307
529
|
return { text: mine.length === 1 ? 'Schedule suspended' : 'All schedules suspended', tone: 'degraded', names }
|
|
308
530
|
}
|
|
309
531
|
const cron = active[0]?.spec?.schedule
|
|
532
|
+
const reading = readings[`${ns}/${active[0]?.metadata?.name}`]
|
|
533
|
+
const blockers = active.map((s) => cnpgScheduleDestinationBlocker(s, [cluster])).filter((b): b is string => !!b)
|
|
534
|
+
const cadence = active.length === 1 ? reading || cron : undefined
|
|
535
|
+
const run = active.some((s) => s.status?.lastScheduleTime || backupsForScheduledBackup(s, backups).length > 0)
|
|
310
536
|
return {
|
|
311
|
-
text: active.length === 1 ?
|
|
312
|
-
tone: '
|
|
537
|
+
text: [active.length === 1 ? 'Enabled' : `${active.length} enabled schedules`, blockers.length || run ? cadence : undefined, blockers.length ? `blocked: ${[...new Set(blockers)].map((blocker) => blocker[0].toLowerCase() + blocker.slice(1)).join(', ')}` : !run ? 'not run yet' : undefined].filter(Boolean).join(' · '),
|
|
538
|
+
tone: blockers.length ? 'degraded' : 'neutral',
|
|
313
539
|
names,
|
|
540
|
+
...(active.length === 1 && cron ? { source: `ScheduledBackup ${active[0]?.metadata?.name} · cron ${cron}` } : {}),
|
|
314
541
|
}
|
|
315
542
|
}
|
|
316
543
|
|
|
317
544
|
function destinationFact(cluster: any): CNPGProtectionFacts['destination'] {
|
|
318
|
-
const
|
|
545
|
+
const declaration = cnpgBackupDeclaration(cluster)
|
|
546
|
+
const plugin = cnpgBarmanPlugin(declaration)
|
|
319
547
|
if (plugin?.barmanObjectName) {
|
|
320
548
|
return {
|
|
321
549
|
text: `ObjectStore ${plugin.barmanObjectName}`,
|
|
@@ -324,11 +552,10 @@ function destinationFact(cluster: any): CNPGProtectionFacts['destination'] {
|
|
|
324
552
|
objectStore: plugin.barmanObjectName,
|
|
325
553
|
}
|
|
326
554
|
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
return { text: cfg.destinationPath, tone: 'neutral', method: 'barmanObjectStore' }
|
|
555
|
+
if (declaration.inTreeDestination) {
|
|
556
|
+
return { text: declaration.inTreeDestination, tone: 'neutral', method: 'barmanObjectStore' }
|
|
330
557
|
}
|
|
331
|
-
if (
|
|
558
|
+
if (declaration.snapshotsConfigured) {
|
|
332
559
|
return { text: 'Volume snapshots', tone: 'neutral', method: 'volumeSnapshot' }
|
|
333
560
|
}
|
|
334
561
|
return { text: 'No destination configured', tone: 'neutral', method: 'none' }
|
|
@@ -359,14 +586,12 @@ function lastBackupFact(
|
|
|
359
586
|
storesUnreadable: CNPGKindCoverage | null,
|
|
360
587
|
): CNPGProtectionFacts['lastSuccessfulBackup'] {
|
|
361
588
|
const ns = cluster.metadata?.namespace
|
|
362
|
-
const name = cluster.metadata?.name
|
|
363
589
|
const candidates: { at: string; source: string }[] = []
|
|
364
590
|
if (coverageReadable(backupsCov, ns)) {
|
|
365
591
|
const completed = backups
|
|
366
592
|
.filter(
|
|
367
593
|
(b) =>
|
|
368
|
-
b
|
|
369
|
-
specClusterName(b) === name &&
|
|
594
|
+
cnpgBackupMatchesCluster(b, cluster) &&
|
|
370
595
|
isApiGroup(b.apiVersion, 'postgresql.cnpg.io') &&
|
|
371
596
|
b.status?.phase === 'completed',
|
|
372
597
|
)
|
|
@@ -377,26 +602,109 @@ function lastBackupFact(
|
|
|
377
602
|
if (window?.lastSuccess) candidates.push({ at: window.lastSuccess, source: `ObjectStore ${window.store} status` })
|
|
378
603
|
const cfg = getCNPGClusterBackupConfig(cluster)
|
|
379
604
|
if (!cfg.plugin && cfg.lastSuccessfulBackup) candidates.push({ at: cfg.lastSuccessfulBackup, source: 'Cluster status' })
|
|
380
|
-
|
|
605
|
+
const createdAt = Date.parse(cluster.metadata?.creationTimestamp ?? '')
|
|
606
|
+
const current = candidates.filter((candidate) => !Number.isFinite(createdAt) || Date.parse(candidate.at) >= createdAt)
|
|
607
|
+
if (current.length === 0) {
|
|
381
608
|
if (!coverageReadable(backupsCov, ns)) {
|
|
382
|
-
return { text:
|
|
609
|
+
return { text: cnpgCoverageGap(backupsCov, 'Backups', ns), tone: 'unknown', source: 'Backups not read' }
|
|
383
610
|
}
|
|
384
|
-
if (storesUnreadable) return { text:
|
|
385
|
-
return { text: '
|
|
611
|
+
if (storesUnreadable) return { text: cnpgCoverageGap(storesUnreadable, 'ObjectStores', ns), tone: 'unknown' }
|
|
612
|
+
return { text: 'No successful backup yet', tone: 'degraded', source: 'Backups read in this namespace; none completed' }
|
|
386
613
|
}
|
|
387
|
-
const best =
|
|
614
|
+
const best = current.reduce((a, b) => (Date.parse(a.at) >= Date.parse(b.at) ? a : b))
|
|
388
615
|
return { text: 'Completed', tone: 'healthy', at: best.at, source: best.source }
|
|
389
616
|
}
|
|
390
617
|
|
|
391
|
-
|
|
618
|
+
export const CNPG_NO_WAL_ARCHIVE_DESTINATION = 'Not archived: no destination configured'
|
|
619
|
+
|
|
620
|
+
function walFact(cluster: any): CNPGProtectionFacts['walArchiving'] {
|
|
392
621
|
const conds = cluster?.status?.conditions
|
|
393
622
|
const c = Array.isArray(conds) ? conds.find((x: any) => x?.type === 'ContinuousArchiving') : null
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
623
|
+
const declaration = cnpgBackupDeclaration(cluster)
|
|
624
|
+
const plugin = cnpgBarmanPlugin(declaration)
|
|
625
|
+
const archivers = declaration.plugins.filter((p) => p.enabled && p.isWALArchiver)
|
|
626
|
+
const archiveConfigured = archivers.length > 0 || !!declaration.inTreeDestination
|
|
627
|
+
if (c?.status === 'False' && archiveConfigured) {
|
|
628
|
+
return { state: 'failing', text: 'Failing', tone: 'unhealthy', source: 'ContinuousArchiving condition', ...(c.lastTransitionTime ? { at: c.lastTransitionTime, atMeaning: 'since' as const } : {}), ...(c.message ? { detail: c.message } : {}) }
|
|
398
629
|
}
|
|
399
|
-
|
|
630
|
+
const destinationKnown = !!(plugin?.isWALArchiver && plugin.barmanObjectName) || !!declaration.inTreeDestination
|
|
631
|
+
const customArchiver = archivers.some((p: any) => p.name !== plugin?.name)
|
|
632
|
+
if (!destinationKnown && !customArchiver) {
|
|
633
|
+
return {
|
|
634
|
+
state: 'no_destination', text: CNPG_NO_WAL_ARCHIVE_DESTINATION,
|
|
635
|
+
tone: 'neutral',
|
|
636
|
+
source: 'Cluster spec',
|
|
637
|
+
detail: "WAL is not archived to recovery storage, so point-in-time recovery is unavailable." + (c?.status === 'True' ? " CloudNativePG still reports archiving as working because, with no destination, it accepts each WAL file without keeping it." : ''),
|
|
638
|
+
...(c ? { operatorCondition: { type: c.type, status: c.status, message: c.message, lastTransitionTime: c.lastTransitionTime } } : {}),
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
if (!c) return { state: 'unknown', text: 'Not reported', tone: 'unknown', source: 'Cluster status' }
|
|
642
|
+
if (c.status === 'True') {
|
|
643
|
+
return { state: 'archiving', text: 'CNPG reports archiving', tone: 'healthy', source: 'Cluster status · ContinuousArchiving=True', ...(customArchiver && !destinationKnown ? { detail: 'Archive plugin declared; its destination is not assessed here' } : {}), ...(c.lastTransitionTime ? { at: c.lastTransitionTime, atMeaning: 'since' as const } : {}) }
|
|
644
|
+
}
|
|
645
|
+
return { state: 'unknown', text: 'Unknown', tone: 'unknown', source: 'ContinuousArchiving condition' }
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
export const CNPG_RESTORE_VALIDATION_ANNOTATION = 'radar.skyhook.io/restore-validation'
|
|
649
|
+
/** Radars of other versions read the note on the Cluster; a format this one does not know is skipped. */
|
|
650
|
+
export const CNPG_RESTORE_VALIDATION_VERSION = 1
|
|
651
|
+
|
|
652
|
+
export interface CNPGRestoreValidationNote {
|
|
653
|
+
version: number
|
|
654
|
+
recordedAt: string
|
|
655
|
+
recordedBy?: string
|
|
656
|
+
checked: string
|
|
657
|
+
targetTime?: string
|
|
658
|
+
source?: { namespace: string; name: string; uid?: string; verified: boolean }
|
|
659
|
+
target?: { namespace: string; name: string; uid?: string; verified: boolean }
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/** The validation note recorded on a restored Cluster, or null when absent, malformed or in another format. */
|
|
663
|
+
export function getCNPGRestoreValidation(cluster: any): CNPGRestoreValidationNote | null {
|
|
664
|
+
const raw = cluster?.metadata?.annotations?.[CNPG_RESTORE_VALIDATION_ANNOTATION]
|
|
665
|
+
if (typeof raw !== 'string' || !raw.trim()) return null
|
|
666
|
+
try {
|
|
667
|
+
const v = JSON.parse(raw)
|
|
668
|
+
if (v?.version !== CNPG_RESTORE_VALIDATION_VERSION) return null
|
|
669
|
+
if (typeof v?.recordedAt !== 'string' || typeof v?.checked !== 'string' || !v.checked) return null
|
|
670
|
+
if (v.target?.uid && v.target.uid !== cluster.metadata?.uid) return null
|
|
671
|
+
return v as CNPGRestoreValidationNote
|
|
672
|
+
} catch {
|
|
673
|
+
return null
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
function noteIsAbout(note: CNPGRestoreValidationNote, cluster: any): boolean {
|
|
678
|
+
return !!note.source?.uid && note.source.uid === cluster.metadata?.uid
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/** A recovery declaration compatible with this live source, excluding known predecessor evidence. */
|
|
682
|
+
export function cnpgRecoveryMatchesCluster(restored: any, cluster: any, backups: any[]): boolean {
|
|
683
|
+
if (restored === cluster || restored.metadata?.namespace !== cluster.metadata?.namespace) return false
|
|
684
|
+
const recovery = restored.spec?.bootstrap?.recovery
|
|
685
|
+
if (!recovery) return false
|
|
686
|
+
const created = Date.parse(cluster.metadata?.creationTimestamp ?? '')
|
|
687
|
+
const restoredAt = Date.parse(restored.metadata?.creationTimestamp ?? '')
|
|
688
|
+
const targetAt = Date.parse(recovery.recoveryTarget?.targetTime ?? '')
|
|
689
|
+
if (Number.isFinite(created) && (restoredAt < created || targetAt < created)) return false
|
|
690
|
+
const note = getCNPGRestoreValidation(restored)
|
|
691
|
+
if (note?.source?.uid && !noteIsAbout(note, cluster)) return false
|
|
692
|
+
const ns = cluster.metadata?.namespace
|
|
693
|
+
if (recovery.backup?.name) {
|
|
694
|
+
const backup = backups.find((b) => b.metadata?.namespace === ns && b.metadata?.name === recovery.backup.name)
|
|
695
|
+
return cnpgBackupMatchesCluster(backup, cluster)
|
|
696
|
+
}
|
|
697
|
+
const source = (restored.spec?.externalClusters ?? []).find((e: any) => e?.name === recovery.source)
|
|
698
|
+
if (!source) return false
|
|
699
|
+
const params = source.plugin?.name === CNPG_BARMAN_PLUGIN_NAME ? source.plugin.parameters : undefined
|
|
700
|
+
const external = source.barmanObjectStore
|
|
701
|
+
const archiveMatch = params?.barmanObjectName
|
|
702
|
+
? cnpgArchiveMatchesCluster(cluster, { kind: 'objectStore', objectStore: params.barmanObjectName, serverName: params.serverName || recovery.source })
|
|
703
|
+
: external && cnpgArchiveMatchesCluster(cluster, { kind: 'inTree', barmanObjectStore: external, serverName: external.serverName || recovery.source })
|
|
704
|
+
if (!archiveMatch) return false
|
|
705
|
+
const id = recovery.recoveryTarget?.backupID
|
|
706
|
+
const pinned = id ? backups.filter((b) => b.metadata?.namespace === ns && b.spec?.cluster?.name === cluster.metadata?.name && b.status?.backupId === id) : []
|
|
707
|
+
return pinned.length === 0 || pinned.some((b) => cnpgBackupMatchesCluster(b, cluster))
|
|
400
708
|
}
|
|
401
709
|
|
|
402
710
|
function restoreValidationFact(
|
|
@@ -405,26 +713,24 @@ function restoreValidationFact(
|
|
|
405
713
|
backups: any[],
|
|
406
714
|
backupsReadable: boolean,
|
|
407
715
|
): CNPGProtectionFacts['restoreValidation'] {
|
|
408
|
-
const plugin = getCNPGClusterBarmanPlugin(cluster)
|
|
409
|
-
const server = plugin?.serverName || cluster.metadata?.name
|
|
410
|
-
const store = plugin?.barmanObjectName
|
|
411
716
|
const ns = cluster.metadata?.namespace
|
|
412
|
-
const
|
|
413
|
-
const
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
717
|
+
const restoredFromThis = allClusters.filter((c) => cnpgRecoveryMatchesCluster(c, cluster, backups))
|
|
718
|
+
const noted = restoredFromThis
|
|
719
|
+
.map((c) => ({ c, note: getCNPGRestoreValidation(c) }))
|
|
720
|
+
.filter((x): x is { c: any; note: CNPGRestoreValidationNote } => !!x.note && noteIsAbout(x.note, cluster))
|
|
721
|
+
.sort((a, b) => Date.parse(b.note.recordedAt) - Date.parse(a.note.recordedAt))[0]
|
|
722
|
+
if (noted) {
|
|
723
|
+
const rname = noted.c.metadata?.name
|
|
724
|
+
const by = noted.note.recordedBy ? `by ${noted.note.recordedBy}` : 'by a user Radar could not identify'
|
|
725
|
+
return {
|
|
726
|
+
text: 'Validation recorded',
|
|
727
|
+
tone: 'neutral',
|
|
728
|
+
at: noted.note.recordedAt,
|
|
729
|
+
source: `Recorded ${by} on ${rname}${noted.note.targetTime ? ` (target ${noted.note.targetTime})` : ''}: ${noted.note.checked.length > 140 ? `${noted.note.checked.slice(0, 140)}…` : noted.note.checked}. A person's note, not a check Radar ran.`,
|
|
730
|
+
restoredInto: { namespace: noted.c.metadata?.namespace, name: rname },
|
|
422
731
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
const backup = backups.find((b) => b.metadata?.namespace === ns && b.metadata?.name === backupName)
|
|
426
|
-
return specClusterName(backup) === name
|
|
427
|
-
})
|
|
732
|
+
}
|
|
733
|
+
const restored = restoredFromThis[0]
|
|
428
734
|
if (!restored) {
|
|
429
735
|
// A recovery by Backup name is only attributable when that Backup could be read.
|
|
430
736
|
const unresolved = !backupsReadable && allClusters.some((c) => c !== cluster && c.metadata?.namespace === ns && c.spec?.bootstrap?.recovery?.backup?.name)
|
|
@@ -432,24 +738,26 @@ function restoreValidationFact(
|
|
|
432
738
|
return { text: 'None recorded', tone: 'unknown', source: 'Kubernetes does not record restore tests' }
|
|
433
739
|
}
|
|
434
740
|
const rname = restored.metadata?.name
|
|
741
|
+
const fromArchive = !!restored.spec?.bootstrap?.recovery?.source
|
|
742
|
+
const sourceDescription = fromArchive ? 'the same archive currently configured for this Cluster' : "this cluster's backups"
|
|
435
743
|
const ready = typeof restored.status?.readyInstances === 'number' && restored.status.readyInstances > 0
|
|
436
744
|
if (!ready) {
|
|
437
745
|
return {
|
|
438
|
-
text:
|
|
746
|
+
text: `${fromArchive ? 'Archive recovery' : 'Recovery'} declared in ${rname}`,
|
|
439
747
|
tone: 'unknown',
|
|
440
|
-
source: `Cluster ${rname} bootstraps from
|
|
748
|
+
source: `Cluster ${rname} bootstraps from ${sourceDescription} but has no ready instance yet`,
|
|
441
749
|
restoredInto: { namespace: restored.metadata?.namespace, name: rname },
|
|
442
750
|
}
|
|
443
751
|
}
|
|
444
752
|
return {
|
|
445
|
-
text:
|
|
753
|
+
text: `${fromArchive ? 'Archive restored' : 'Restored'} into ${rname}`,
|
|
446
754
|
tone: 'neutral',
|
|
447
|
-
source: `Cluster ${rname} bootstrapped from
|
|
755
|
+
source: `Cluster ${rname} bootstrapped from ${sourceDescription} and has ready instances · created ${restored.metadata?.creationTimestamp ?? 'unknown'}. No validation note is recorded on it; this proves one recovery, not that today's backups restore.`,
|
|
448
756
|
restoredInto: { namespace: restored.metadata?.namespace, name: rname },
|
|
449
757
|
}
|
|
450
758
|
}
|
|
451
759
|
|
|
452
|
-
function protectionSummary(p: CNPGProtectionFacts):
|
|
760
|
+
function protectionSummary(p: CNPGProtectionFacts): Fact {
|
|
453
761
|
if (p.walArchiving.tone === 'unhealthy') return { text: 'WAL archiving failing', tone: 'unhealthy' }
|
|
454
762
|
if (p.destination.method === 'none' && p.schedule.names.length === 0 && p.schedule.tone !== 'unknown') {
|
|
455
763
|
return { text: 'No backup destination or schedule', tone: 'neutral' }
|
|
@@ -475,47 +783,99 @@ function pgVersion(cluster: any): string | null {
|
|
|
475
783
|
return typeof major === 'number' ? String(major) : null
|
|
476
784
|
}
|
|
477
785
|
|
|
478
|
-
function replicationFact(cluster: any, pods: CNPGInstance[], hibernated: boolean, podsCov: CNPGKindCoverage):
|
|
786
|
+
function replicationFact(cluster: any, pods: CNPGInstance[], hibernated: boolean, podsCov: CNPGKindCoverage): Fact {
|
|
479
787
|
if (hibernated) return { text: 'Hibernated', tone: 'neutral' }
|
|
480
788
|
const desired = cluster?.spec?.instances
|
|
481
|
-
if (desired === 1)
|
|
789
|
+
if (desired === 1) {
|
|
790
|
+
return getCNPGClusterIsReplica(cluster)
|
|
791
|
+
? { text: 'Single instance · upstream feed not assessed', tone: 'unknown', source: 'The designated primary replays WAL from an external cluster; Radar does not assess that feed' }
|
|
792
|
+
: { text: 'Single instance', tone: 'neutral' }
|
|
793
|
+
}
|
|
482
794
|
if (!coverageReadable(podsCov, cluster?.metadata?.namespace)) {
|
|
483
|
-
return { text:
|
|
795
|
+
return { text: cnpgCoverageGap(podsCov, 'Pods', cluster?.metadata?.namespace), tone: 'unknown' }
|
|
484
796
|
}
|
|
485
797
|
const replicas = pods.filter((p) => p.role === 'replica')
|
|
486
798
|
const readyReplicas = replicas.filter((p) => p.ready === true).length
|
|
487
799
|
if (replicas.length === 0) return { text: 'No replica pods observed', tone: 'unknown' }
|
|
488
800
|
return {
|
|
489
|
-
text: `${readyReplicas}/${replicas.length}
|
|
801
|
+
text: `${readyReplicas}/${replicas.length} Pods ready · lag unknown`,
|
|
490
802
|
tone: 'unknown',
|
|
491
|
-
source:
|
|
803
|
+
source: CNPG_LAG_UNMEASURED_SOURCE,
|
|
492
804
|
}
|
|
493
805
|
}
|
|
494
806
|
|
|
807
|
+
const CNPG_JOB_PURPOSE: Record<string, string> = {
|
|
808
|
+
initdb: 'First instance',
|
|
809
|
+
join: 'New standby',
|
|
810
|
+
'full-recovery': 'Restore into',
|
|
811
|
+
'snapshot-recovery': 'Restore into',
|
|
812
|
+
pgbasebackup: 'Clone into',
|
|
813
|
+
import: 'Import into',
|
|
814
|
+
'major-upgrade': 'Major upgrade of',
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/** What a Cluster's own Job is for, naming the instance it builds ("New standby pg-2"). */
|
|
818
|
+
export function cnpgJobPurpose(role: string | undefined, instance: string | undefined, pod: string): string {
|
|
819
|
+
const purpose = role ? CNPG_JOB_PURPOSE[role] : undefined
|
|
820
|
+
if (purpose && instance) return `${purpose} ${instance}`
|
|
821
|
+
return `${role ? `${role} ` : ''}Job Pod ${pod}`
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
interface CNPGJobPod {
|
|
825
|
+
role?: string
|
|
826
|
+
instance?: string
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
function jobPodIndex(resp: CNPGWorkspaceResponse): Map<string, CNPGJobPod> {
|
|
830
|
+
const idx = new Map<string, CNPGJobPod>()
|
|
831
|
+
for (const p of resp.jobPods ?? []) {
|
|
832
|
+
const labels = p?.metadata?.labels ?? {}
|
|
833
|
+
idx.set(`${p.metadata?.namespace}/${p.metadata?.name}`, { role: labels['cnpg.io/jobRole'], instance: labels['cnpg.io/instanceName'] })
|
|
834
|
+
}
|
|
835
|
+
return idx
|
|
836
|
+
}
|
|
837
|
+
|
|
495
838
|
function problemsFor(
|
|
496
839
|
cluster: any,
|
|
497
840
|
issues: CNPGWorkspaceIssue[],
|
|
498
841
|
audit: CNPGAuditFinding[],
|
|
499
842
|
children: Map<string, string>,
|
|
843
|
+
backupTimes: Map<string, number> = new Map(),
|
|
844
|
+
jobPods: Map<string, CNPGJobPod> = new Map(),
|
|
500
845
|
): CNPGProblem[] {
|
|
501
846
|
const ns = cluster.metadata?.namespace
|
|
502
847
|
const name = cluster.metadata?.name
|
|
503
848
|
const out: CNPGProblem[] = []
|
|
849
|
+
const fromIssues: IssueProblem[] = []
|
|
504
850
|
for (const issue of issues) {
|
|
505
851
|
if ((issue.namespace ?? '') !== ns) continue
|
|
506
852
|
const isSelf = issue.kind === 'Cluster' && issue.name === name
|
|
507
853
|
const owner = children.get(`${issue.kind}/${ns}/${issue.name}`)
|
|
508
854
|
if (!isSelf && owner !== name) continue
|
|
509
|
-
|
|
855
|
+
const job = issue.kind === 'Pod' ? jobPods.get(`${ns}/${issue.name}`) : undefined
|
|
856
|
+
const text = cnpgIssueText(issue)
|
|
857
|
+
fromIssues.push({
|
|
510
858
|
id: `${issue.id}:${issue.kind}/${issue.name}`,
|
|
511
|
-
|
|
859
|
+
// A standby that cannot join costs redundancy, not service: the primary keeps serving.
|
|
860
|
+
severity: job?.role === 'join' ? 'warning' : issue.severity,
|
|
512
861
|
category: cnpgIssueCategory(issue),
|
|
513
|
-
|
|
514
|
-
|
|
862
|
+
...(job
|
|
863
|
+
? {
|
|
864
|
+
title: `${cnpgJobPurpose(job.role, job.instance, issue.name)}: ${issue.category ? issueTitle({ category: issue.category, reason: issue.reason }) : text.title}`,
|
|
865
|
+
detail: [issue.message?.trim(), issue.cause?.trim()].filter(Boolean).join(' ') || undefined,
|
|
866
|
+
job: job.role ?? 'job',
|
|
867
|
+
instance: job.instance,
|
|
868
|
+
}
|
|
869
|
+
: text),
|
|
515
870
|
subject: { kind: issue.kind, group: issue.group ?? '', namespace: ns, name: issue.name },
|
|
516
871
|
source: 'issue',
|
|
872
|
+
origin: cnpgIssueOrigin(issue),
|
|
873
|
+
reason: issue.reason,
|
|
874
|
+
...(isSelf && issue.reason.startsWith('Ready:') ? { rollup: true } : {}),
|
|
517
875
|
})
|
|
518
876
|
}
|
|
877
|
+
const newestBackup = [...backupTimes.entries()].sort((a, b) => b[1] - a[1])[0]?.[0]
|
|
878
|
+
out.push(...cnpgFoldLastBackupFailed(cnpgCollapseBackupFailures(fromIssues, backupTimes), newestBackup))
|
|
519
879
|
for (const f of audit) {
|
|
520
880
|
if (f.kind !== 'Cluster' || f.name !== name || (f.namespace ?? '') !== ns) continue
|
|
521
881
|
out.push({
|
|
@@ -528,8 +888,62 @@ function problemsFor(
|
|
|
528
888
|
source: 'audit',
|
|
529
889
|
})
|
|
530
890
|
}
|
|
531
|
-
|
|
532
|
-
|
|
891
|
+
return out.sort(cnpgCompareProblems)
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
/**
|
|
895
|
+
* The ready count to show for a cluster: CNPG's status count, or the Pods'
|
|
896
|
+
* own count when the status claims more than the Pods show.
|
|
897
|
+
*/
|
|
898
|
+
export function cnpgReadyInstances(row: Pick<CNPGFleetRow, 'instances' | 'podReadiness' | 'readinessContradicted'>): { text: string; podText?: string; tone?: HealthLevel; note?: string } {
|
|
899
|
+
if (row.instances.ready === null) {
|
|
900
|
+
return {
|
|
901
|
+
text: 'Not reported by the operator',
|
|
902
|
+
podText: row.podReadiness
|
|
903
|
+
? row.instances.desired !== null
|
|
904
|
+
? `${row.podReadiness.ready} of ${row.instances.desired} instance Pods ready`
|
|
905
|
+
: `${row.podReadiness.ready} instance Pod${row.podReadiness.ready === 1 ? '' : 's'} ready`
|
|
906
|
+
: undefined,
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
const desired = row.instances.desired ?? '–'
|
|
910
|
+
if (row.readinessContradicted && row.podReadiness) {
|
|
911
|
+
return {
|
|
912
|
+
text: `${row.podReadiness.ready}/${desired}`,
|
|
913
|
+
tone: row.podReadiness.ready === 0 ? 'unhealthy' : 'degraded',
|
|
914
|
+
note: `Counted from the instance Pods' Ready condition. CNPG status reports ${row.instances.ready} ready, so the status may be stale.`,
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
return { text: `${row.instances.ready ?? '–'}/${desired}` }
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
const CNPG_SEVERITY_RANK = { critical: 0, warning: 1, posture: 2 } as const
|
|
921
|
+
|
|
922
|
+
/**
|
|
923
|
+
* The order problems are shown in, everywhere: most severe first, then a
|
|
924
|
+
* cause before its symptoms. One instance Pod's state (a failing probe, a
|
|
925
|
+
* crash loop) is usually the symptom of a cluster-level problem (archiving,
|
|
926
|
+
* backups, replication, reconciliation, declarations) or of a Job the Cluster
|
|
927
|
+
* runs (an initdb or join Pod that cannot start), so at equal severity those
|
|
928
|
+
* come first; the Cluster's own Ready condition sums them all up and comes last.
|
|
929
|
+
*/
|
|
930
|
+
export function cnpgCompareProblems(a: CNPGProblem, b: CNPGProblem): number {
|
|
931
|
+
const symptom = (p: CNPGProblem) => (p.rollup ? 2 : p.subject.kind === 'Pod' && p.subject.group === '' && !p.job ? 1 : 0)
|
|
932
|
+
return CNPG_SEVERITY_RANK[a.severity] - CNPG_SEVERITY_RANK[b.severity] || symptom(a) - symptom(b) || a.title.localeCompare(b.title)
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
function sortProblems(list: CNPGProblem[]): CNPGProblem[] {
|
|
936
|
+
return list.sort(cnpgCompareProblems)
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
function primaryConflictOf(cluster: any, pods: any[]): CNPGFleetRow['primaryConflict'] {
|
|
940
|
+
const status = cluster?.status?.currentPrimary
|
|
941
|
+
if (!status) return undefined
|
|
942
|
+
const labelled = pods
|
|
943
|
+
.filter((p) => (p?.metadata?.labels?.['cnpg.io/instanceRole'] ?? p?.metadata?.labels?.role) === 'primary')
|
|
944
|
+
.map((p) => p.metadata?.name as string)
|
|
945
|
+
if (labelled.length === 0 || labelled.includes(status)) return undefined
|
|
946
|
+
return { status, labelled: labelled.sort()[0] }
|
|
533
947
|
}
|
|
534
948
|
|
|
535
949
|
/** Index "Kind/ns/name" → owning cluster name, from each child's spec.cluster.name. */
|
|
@@ -537,6 +951,7 @@ function childIndex(resp: CNPGWorkspaceResponse): Map<string, string> {
|
|
|
537
951
|
const idx = new Map<string, string>()
|
|
538
952
|
const add = (kind: string, list: any[] | undefined) => {
|
|
539
953
|
for (const o of list ?? []) {
|
|
954
|
+
if (kind === 'Backup' && !targetCluster(o, resp.objects.clusters ?? [])) continue
|
|
540
955
|
const c = specClusterName(o)
|
|
541
956
|
if (c) idx.set(`${kind}/${o.metadata?.namespace}/${o.metadata?.name}`, c)
|
|
542
957
|
}
|
|
@@ -547,7 +962,8 @@ function childIndex(resp: CNPGWorkspaceResponse): Map<string, string> {
|
|
|
547
962
|
add('Database', resp.objects.databases)
|
|
548
963
|
add('Publication', resp.objects.publications)
|
|
549
964
|
add('Subscription', resp.objects.subscriptions)
|
|
550
|
-
|
|
965
|
+
add('DatabaseRole', resp.objects.databaseRoles)
|
|
966
|
+
for (const p of [...(resp.objects.pods ?? []), ...(resp.jobPods ?? [])]) {
|
|
551
967
|
const c = p?.metadata?.labels?.['cnpg.io/cluster']
|
|
552
968
|
if (c) idx.set(`Pod/${p.metadata?.namespace}/${p.metadata?.name}`, c)
|
|
553
969
|
}
|
|
@@ -561,21 +977,24 @@ function declarationsFor(cluster: any, resp: CNPGWorkspaceResponse): CNPGFleetRo
|
|
|
561
977
|
['databases', resp.objects.databases ?? []],
|
|
562
978
|
['publications', resp.objects.publications ?? []],
|
|
563
979
|
['subscriptions', resp.objects.subscriptions ?? []],
|
|
980
|
+
['databaseRoles', resp.objects.databaseRoles ?? []],
|
|
564
981
|
]
|
|
565
982
|
let total = 0
|
|
566
983
|
let failed = 0
|
|
567
984
|
let pending = 0
|
|
568
|
-
|
|
985
|
+
const unread: string[] = []
|
|
986
|
+
const labels = { databases: 'Databases', publications: 'Publications', subscriptions: 'Subscriptions', databaseRoles: 'DatabaseRoles' }
|
|
569
987
|
for (const [k, list] of lists) {
|
|
570
988
|
if (!coverageReadable(coverageOf(resp, k), ns)) {
|
|
571
|
-
if (coverageOf(resp, k).state !== 'notInstalled')
|
|
989
|
+
if (coverageOf(resp, k).state !== 'notInstalled') unread.push(`${labels[k as keyof typeof labels]} ${resp.coverage[k] ? 'not read' : 'not read by this Radar'}`)
|
|
572
990
|
continue
|
|
573
991
|
}
|
|
574
992
|
for (const o of list) {
|
|
575
993
|
if (o.metadata?.namespace !== ns || specClusterName(o) !== name) continue
|
|
576
994
|
total++
|
|
577
|
-
|
|
578
|
-
|
|
995
|
+
const state = cnpgRoleState(o)
|
|
996
|
+
if (state === 'failed') failed++
|
|
997
|
+
else if (state === 'pending') pending++
|
|
579
998
|
}
|
|
580
999
|
}
|
|
581
1000
|
const roleStatus = cluster?.status?.managedRolesStatus
|
|
@@ -588,9 +1007,10 @@ function declarationsFor(cluster: any, resp: CNPGWorkspaceResponse): CNPGFleetRo
|
|
|
588
1007
|
if (failedRoles.has(r.name)) failed++
|
|
589
1008
|
else if (!reconciledRoles.has(r.name)) pending++
|
|
590
1009
|
}
|
|
591
|
-
|
|
1010
|
+
const unreadable = unread.length > 0
|
|
1011
|
+
let summary: Fact
|
|
592
1012
|
if (total === 0) {
|
|
593
|
-
summary =
|
|
1013
|
+
summary = { text: 'None declared', tone: 'neutral' }
|
|
594
1014
|
} else if (failed > 0) {
|
|
595
1015
|
summary = { text: `${failed} of ${total} not reconciled`, tone: 'degraded' }
|
|
596
1016
|
} else if (pending > 0) {
|
|
@@ -598,7 +1018,10 @@ function declarationsFor(cluster: any, resp: CNPGWorkspaceResponse): CNPGFleetRo
|
|
|
598
1018
|
} else {
|
|
599
1019
|
summary = { text: `${total} reconciled`, tone: 'healthy' }
|
|
600
1020
|
}
|
|
601
|
-
if (unreadable
|
|
1021
|
+
if (unreadable) {
|
|
1022
|
+
const count = total === 0 ? 'Reconciliation unknown' : failed > 0 ? `≥${failed} not reconciled` : pending > 0 ? `≥${pending} pending` : `≥${total} reconciled`
|
|
1023
|
+
summary = { text: `${count}; ${unread.join('; ')}`, tone: failed > 0 ? 'degraded' : 'unknown' }
|
|
1024
|
+
}
|
|
602
1025
|
return { summary, total, failed, pending }
|
|
603
1026
|
}
|
|
604
1027
|
|
|
@@ -607,6 +1030,7 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
607
1030
|
const pods = resp.objects.pods ?? []
|
|
608
1031
|
const stores = resp.objects.objectStores ?? []
|
|
609
1032
|
const children = childIndex(resp)
|
|
1033
|
+
const jobs = jobPodIndex(resp)
|
|
610
1034
|
const poolers = resp.objects.poolers ?? []
|
|
611
1035
|
|
|
612
1036
|
const rows: CNPGFleetRow[] = clusters.map((cluster) => {
|
|
@@ -631,7 +1055,7 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
631
1055
|
const storesCov = coverageOf(resp, 'objectStores')
|
|
632
1056
|
const storesUnreadable = !!getCNPGClusterBarmanPlugin(cluster)?.barmanObjectName && !coverageReadable(storesCov, ns)
|
|
633
1057
|
const protection: CNPGProtectionFacts = {
|
|
634
|
-
schedule: scheduleFact(cluster, resp.objects.scheduledBackups ?? [], coverageOf(resp, 'scheduledBackups')),
|
|
1058
|
+
schedule: scheduleFact(cluster, resp.objects.scheduledBackups ?? [], coverageOf(resp, 'scheduledBackups'), resp.scheduleReadings, resp.objects.backups ?? []),
|
|
635
1059
|
destination: destinationFact(cluster),
|
|
636
1060
|
lastSuccessfulBackup: lastBackupFact(cluster, resp.objects.backups ?? [], coverageOf(resp, 'backups'), window, storesUnreadable ? storesCov : null),
|
|
637
1061
|
walArchiving: wal,
|
|
@@ -645,15 +1069,26 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
645
1069
|
source: `ObjectStore ${window.store} status (earliest point)`,
|
|
646
1070
|
}
|
|
647
1071
|
: storesUnreadable
|
|
648
|
-
? { text:
|
|
649
|
-
: { text: 'Not reported', tone: 'unknown' },
|
|
1072
|
+
? { text: cnpgCoverageGap(storesCov, 'ObjectStores', ns), tone: 'unknown' }
|
|
1073
|
+
: destinationFact(cluster).method === 'none' ? { text: 'None: no backup destination', tone: 'neutral' } : { text: 'Not reported', tone: 'unknown' },
|
|
650
1074
|
restoreValidation: restoreValidationFact(cluster, clusters, resp.objects.backups ?? [], coverageReadable(coverageOf(resp, 'backups'), ns)),
|
|
651
1075
|
}
|
|
652
|
-
const
|
|
1076
|
+
const podsReadable = coverageReadable(coverageOf(resp, 'pods'), ns)
|
|
1077
|
+
const podReadiness =
|
|
1078
|
+
podsReadable && resp.objects.pods !== undefined && instancePods.every((p) => p.ready !== null)
|
|
1079
|
+
? { ready: instancePods.filter((p) => p.ready).length, total: instancePods.length }
|
|
1080
|
+
: undefined
|
|
1081
|
+
const readinessContradicted = !hibernated && !!podReadiness && readyInstances !== null && podReadiness.ready < readyInstances
|
|
1082
|
+
const primaryConflict = podsReadable ? primaryConflictOf(cluster, pods.filter((p) => p.metadata?.namespace === ns && p.metadata?.labels?.['cnpg.io/cluster'] === name)) : undefined
|
|
1083
|
+
let problems = sortProblems([
|
|
1084
|
+
...problemsFor(cluster, resp.issues ?? [], resp.audit ?? [], children, backupTimesOf(cluster, resp.objects.backups ?? []), jobs),
|
|
1085
|
+
|
|
1086
|
+
])
|
|
1087
|
+
problems = problems.map((p) => p.origin?.label === 'Kubernetes scheduler' ? { ...p, detail: summarizeSchedulerMessage(p.detail, { plain: true }), rawDetail: p.detail, rawDetailLabel: 'Scheduler message' } : p)
|
|
653
1088
|
const categories = new Set<CNPGProblemCategory>(
|
|
654
1089
|
problems.filter((p) => p.severity !== 'posture').map((p) => p.category),
|
|
655
1090
|
)
|
|
656
|
-
const replica = cluster
|
|
1091
|
+
const replica = cluster && getCNPGClusterIsReplica(cluster) ? { source: cluster.spec.replica.source } : null
|
|
657
1092
|
|
|
658
1093
|
return {
|
|
659
1094
|
key: key(ns, name),
|
|
@@ -662,6 +1097,9 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
662
1097
|
cluster,
|
|
663
1098
|
controllerStatus: { text: status.text, level: status.level },
|
|
664
1099
|
instances: { ready: readyInstances, desired },
|
|
1100
|
+
...(podReadiness ? { podReadiness } : {}),
|
|
1101
|
+
...(readinessContradicted ? { readinessContradicted } : {}),
|
|
1102
|
+
...(primaryConflict ? { primaryConflict } : {}),
|
|
665
1103
|
pods: instancePods,
|
|
666
1104
|
replicaCluster: replica,
|
|
667
1105
|
hibernated,
|
|
@@ -670,6 +1108,7 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
670
1108
|
replication: replicationFact(cluster, instancePods, hibernated, coverageOf(resp, 'pods')),
|
|
671
1109
|
protection: { ...protection, summary: protectionSummary(protection) },
|
|
672
1110
|
declarations: declarationsFor(cluster, resp),
|
|
1111
|
+
poolerObjects: poolers.filter((p) => p.metadata?.namespace === ns && specClusterName(p) === name),
|
|
673
1112
|
poolers: poolers
|
|
674
1113
|
.filter((p) => p.metadata?.namespace === ns && specClusterName(p) === name)
|
|
675
1114
|
.map((p) => p.metadata?.name),
|
|
@@ -677,20 +1116,49 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
677
1116
|
problems,
|
|
678
1117
|
attention: problems.some((p) => p.severity !== 'posture'),
|
|
679
1118
|
categories,
|
|
680
|
-
|
|
1119
|
+
managedBy: cnpgManagedBy(resp, cluster),
|
|
681
1120
|
}
|
|
682
1121
|
})
|
|
683
1122
|
|
|
684
|
-
rows.sort((a, b) => Number(b.attention) - Number(a.attention) || a.namespace.localeCompare(b.namespace) || a.name.localeCompare(b.name))
|
|
685
|
-
|
|
686
|
-
const categoryCounts = { availability: 0, protection: 0, declarations: 0, pooling: 0 } as Record<CNPGProblemCategory, number>
|
|
687
|
-
for (const r of rows) for (const c of r.categories) categoryCounts[c]++
|
|
688
|
-
|
|
689
1123
|
const incompleteKinds = CNPG_WORKSPACE_KEYS.filter((k) => {
|
|
690
1124
|
const s = coverageOf(resp, k).state
|
|
691
|
-
return s === 'partial' || s === 'denied' || s === 'syncing' || s === 'error'
|
|
1125
|
+
return s === 'partial' || s === 'denied' || s === 'syncing' || s === 'error' || s === 'uncached'
|
|
692
1126
|
})
|
|
693
1127
|
|
|
1128
|
+
return finishFleet(rows, incompleteKinds)
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
function urgencyOf(row: CNPGFleetRow): { worst: number; urgent: number; total: number } {
|
|
1132
|
+
let worst = 3
|
|
1133
|
+
let urgent = 0
|
|
1134
|
+
for (const p of row.problems) {
|
|
1135
|
+
worst = Math.min(worst, CNPG_SEVERITY_RANK[p.severity])
|
|
1136
|
+
if (p.severity !== 'posture') urgent++
|
|
1137
|
+
}
|
|
1138
|
+
return { worst, urgent, total: row.problems.length }
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
/**
|
|
1142
|
+
* Worst problem first (critical, warning, posture, none), then the most
|
|
1143
|
+
* attention-level problems, then all problems, then namespace/name — the same
|
|
1144
|
+
* order in every filter, so a row never jumps when the filter changes.
|
|
1145
|
+
*/
|
|
1146
|
+
export function compareCNPGFleetUrgency(a: CNPGFleetRow, b: CNPGFleetRow): number {
|
|
1147
|
+
const ua = urgencyOf(a)
|
|
1148
|
+
const ub = urgencyOf(b)
|
|
1149
|
+
return (
|
|
1150
|
+
ua.worst - ub.worst ||
|
|
1151
|
+
ub.urgent - ua.urgent ||
|
|
1152
|
+
ub.total - ua.total ||
|
|
1153
|
+
a.namespace.localeCompare(b.namespace) ||
|
|
1154
|
+
a.name.localeCompare(b.name)
|
|
1155
|
+
)
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
function finishFleet(rows: CNPGFleetRow[], incompleteKinds: CNPGWorkspaceKey[]): CNPGFleet {
|
|
1159
|
+
rows.sort(compareCNPGFleetUrgency)
|
|
1160
|
+
const categoryCounts = { availability: 0, protection: 0, declarations: 0, pooling: 0 } as Record<CNPGProblemCategory, number>
|
|
1161
|
+
for (const r of rows) for (const c of r.categories) categoryCounts[c]++
|
|
694
1162
|
return {
|
|
695
1163
|
rows,
|
|
696
1164
|
attentionCount: rows.filter((r) => r.attention).length,
|
|
@@ -698,3 +1166,536 @@ export function buildCNPGFleet(resp: CNPGWorkspaceResponse): CNPGFleet {
|
|
|
698
1166
|
incompleteKinds,
|
|
699
1167
|
}
|
|
700
1168
|
}
|
|
1169
|
+
|
|
1170
|
+
/** One cluster's answer from /api/cnpg/disk. */
|
|
1171
|
+
export interface CNPGDiskReading {
|
|
1172
|
+
namespace: string
|
|
1173
|
+
name: string
|
|
1174
|
+
/** ok | partial | noSeries | noPrometheus | denied | unavailable | error | notRead | ambiguous | scopeMismatch */
|
|
1175
|
+
state: string
|
|
1176
|
+
grant?: Grant
|
|
1177
|
+
reason?: string
|
|
1178
|
+
claims: number
|
|
1179
|
+
measured: number
|
|
1180
|
+
max?: {
|
|
1181
|
+
claim: string
|
|
1182
|
+
instance: string
|
|
1183
|
+
role: string
|
|
1184
|
+
tablespace?: string
|
|
1185
|
+
usedBytes: number
|
|
1186
|
+
capacityBytes: number
|
|
1187
|
+
ratio: number
|
|
1188
|
+
}
|
|
1189
|
+
isolation?: CNPGMetricIsolation
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
export const CNPG_DISK_WARNING_RATIO = 0.8
|
|
1193
|
+
export const CNPG_DISK_CRITICAL_RATIO = 0.9
|
|
1194
|
+
|
|
1195
|
+
export const CNPG_DISK_SOURCE = 'kubelet volume stats via Prometheus'
|
|
1196
|
+
|
|
1197
|
+
export function cnpgVolumeRoleLabel(role: string, tablespace?: string): string {
|
|
1198
|
+
switch (role) {
|
|
1199
|
+
case 'PG_DATA':
|
|
1200
|
+
return 'data volume'
|
|
1201
|
+
case 'PG_WAL':
|
|
1202
|
+
return 'WAL volume'
|
|
1203
|
+
case 'PG_TABLESPACE':
|
|
1204
|
+
return tablespace ? `tablespace ${tablespace} volume` : 'tablespace volume'
|
|
1205
|
+
default:
|
|
1206
|
+
return 'volume'
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
export function cnpgDiskTone(ratio: number): HealthLevel {
|
|
1211
|
+
if (ratio >= CNPG_DISK_CRITICAL_RATIO) return 'unhealthy'
|
|
1212
|
+
if (ratio >= CNPG_DISK_WARNING_RATIO) return 'degraded'
|
|
1213
|
+
return 'healthy'
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
/** The fleet and summary "Storage" fact: the fullest measured volume, or why there is none. */
|
|
1217
|
+
export function cnpgDiskFact(r: CNPGDiskReading | undefined): Fact {
|
|
1218
|
+
if (!r) return { text: 'Not read', tone: 'unknown' }
|
|
1219
|
+
if (r.max && (r.state === 'ok' || r.state === 'partial')) {
|
|
1220
|
+
const partial = r.state === 'partial' ? ` · ${r.measured} of ${r.claims} volumes measured` : ''
|
|
1221
|
+
return {
|
|
1222
|
+
text: `${Math.round(r.max.ratio * 100)}% used`,
|
|
1223
|
+
tone: cnpgDiskTone(r.max.ratio),
|
|
1224
|
+
source: `Fullest: ${cnpgVolumeRoleLabel(r.max.role, r.max.tablespace)} of ${r.max.instance}, ${formatBytes(r.max.usedBytes)} of ${formatBytes(r.max.capacityBytes)} · ${CNPG_DISK_SOURCE}${partial}.${isolationCaveat(r.isolation)}`,
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
switch (r.state) {
|
|
1228
|
+
case 'denied':
|
|
1229
|
+
return { text: 'No access', tone: 'unknown', source: r.grant ? `Needs ${formatGrant(r.grant)}` : r.reason }
|
|
1230
|
+
case 'noPrometheus':
|
|
1231
|
+
return { text: 'No usage metrics', tone: 'unknown', source: CNPG_PROMETHEUS_NOT_CONNECTED, detail: r.reason }
|
|
1232
|
+
case 'noSeries':
|
|
1233
|
+
case 'ok':
|
|
1234
|
+
case 'partial':
|
|
1235
|
+
return { text: 'No usage metrics', tone: 'unknown', source: r.reason ?? 'Used space needs Prometheus with kubelet volume stats' }
|
|
1236
|
+
case 'notRead':
|
|
1237
|
+
return { text: 'Not measured', tone: 'unknown', source: r.reason }
|
|
1238
|
+
default:
|
|
1239
|
+
return { text: 'Unavailable', tone: 'unknown', source: r.reason }
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
/**
|
|
1244
|
+
* Joins /api/cnpg/disk into the fleet: every row gets its disk fact, and a
|
|
1245
|
+
* volume at or past the warning threshold becomes a problem, so the cluster
|
|
1246
|
+
* needs attention. Only a measurement raises one, and the endpoint returns
|
|
1247
|
+
* measurements only to callers holding the claim and metrics grants.
|
|
1248
|
+
*/
|
|
1249
|
+
export function applyCNPGDisk(fleet: CNPGFleet, readings: CNPGDiskReading[] | undefined): CNPGFleet {
|
|
1250
|
+
if (!readings) return fleet
|
|
1251
|
+
const byKey = new Map(readings.map((r) => [key(r.namespace, r.name), r]))
|
|
1252
|
+
const rows = fleet.rows.map((row) => {
|
|
1253
|
+
const reading = byKey.get(row.key)
|
|
1254
|
+
const next: CNPGFleetRow = { ...row, disk: cnpgDiskFact(reading) }
|
|
1255
|
+
const max = reading?.max
|
|
1256
|
+
if (!max || max.ratio < CNPG_DISK_WARNING_RATIO || (reading.state !== 'ok' && reading.state !== 'partial')) return next
|
|
1257
|
+
const problem: CNPGProblem = {
|
|
1258
|
+
id: `disk:${row.key}`,
|
|
1259
|
+
severity: max.ratio >= CNPG_DISK_CRITICAL_RATIO ? 'critical' : 'warning',
|
|
1260
|
+
category: 'availability',
|
|
1261
|
+
title: `The ${cnpgVolumeRoleLabel(max.role, max.tablespace)} of ${max.instance} is ${Math.round(max.ratio * 100)}% full`,
|
|
1262
|
+
detail: `${formatBytes(max.usedBytes)} of ${formatBytes(max.capacityBytes)} used, from ${CNPG_DISK_SOURCE}.${isolationCaveat(reading.isolation)}`,
|
|
1263
|
+
subject: { kind: 'Cluster', group: 'postgresql.cnpg.io', namespace: row.namespace, name: row.name },
|
|
1264
|
+
source: 'measurement',
|
|
1265
|
+
...(reading.isolation?.mode === 'unverified' ? { measuredBy: 'kubelet, matched by claim name', unverifiedMatch: true } : {}),
|
|
1266
|
+
}
|
|
1267
|
+
const problems = [...row.problems, problem].sort(cnpgCompareProblems)
|
|
1268
|
+
return { ...next, problems, attention: true, categories: new Set([...row.categories, problem.category]) }
|
|
1269
|
+
})
|
|
1270
|
+
return finishFleet(rows, fleet.incompleteKinds)
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
const CNPG_PLUGIN_PHASES: Record<string, 'unknownPlugin' | 'pluginError'> = {
|
|
1274
|
+
'Cluster cannot proceed to reconciliation due to an unknown plugin being required': 'unknownPlugin',
|
|
1275
|
+
'Cluster cannot proceed to reconciliation due to an error while interacting with plugins': 'pluginError',
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
/** Whether the Cluster's phase says the operator is stuck on a CNPG-I plugin, and which way. */
|
|
1279
|
+
export function cnpgPluginPhase(cluster: any): 'unknownPlugin' | 'pluginError' | null {
|
|
1280
|
+
return CNPG_PLUGIN_PHASES[cluster?.status?.phase] ?? null
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
/** The CNPG-I plugins a Cluster names in spec.plugins. */
|
|
1284
|
+
export function cnpgClusterPlugins(cluster: any): string[] {
|
|
1285
|
+
const plugins = cluster?.spec?.plugins
|
|
1286
|
+
return Array.isArray(plugins) ? plugins.map((p: any) => p?.name).filter((n: unknown): n is string => typeof n === 'string' && !!n) : []
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
/** Bytes in binary units with IEC labels (GiB), as every CloudNativePG view prints them. */
|
|
1290
|
+
export function cnpgFormatBytes(n: number): string {
|
|
1291
|
+
const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
|
|
1292
|
+
let v = n
|
|
1293
|
+
let i = 0
|
|
1294
|
+
while (v >= 1024 && i < u.length - 1) {
|
|
1295
|
+
v /= 1024
|
|
1296
|
+
i++
|
|
1297
|
+
}
|
|
1298
|
+
return `${v.toFixed(v >= 10 || i === 0 ? 0 : 1)} ${u[i]}`
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
const formatBytes = cnpgFormatBytes
|
|
1302
|
+
|
|
1303
|
+
// ---------------------------------------------------------------------------
|
|
1304
|
+
// Standbys and replication slots: one finding, whichever source saw it
|
|
1305
|
+
// ---------------------------------------------------------------------------
|
|
1306
|
+
|
|
1307
|
+
/**
|
|
1308
|
+
* Retained WAL at which an inactive physical slot becomes a concern. An HA
|
|
1309
|
+
* slot only goes inactive when its standby stops consuming, and the WAL it
|
|
1310
|
+
* pins keeps growing until that standby catches up or the slot is dropped.
|
|
1311
|
+
*/
|
|
1312
|
+
export const CNPG_SLOT_RETENTION_WARNING_BYTES = 1024 ** 3
|
|
1313
|
+
|
|
1314
|
+
/** The id of a standby's "not receiving WAL" problem: the fleet (Prometheus) and the cluster page (instance manager) raise the same one. */
|
|
1315
|
+
export function cnpgStandbyProblemId(rowKey: string, pod: string): string {
|
|
1316
|
+
return `standby:${rowKey}:${pod}`
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
/** The id of an inactive slot's retention problem, shared like cnpgStandbyProblemId. */
|
|
1320
|
+
export function cnpgSlotProblemId(rowKey: string, slot: string): string {
|
|
1321
|
+
return `slot:${rowKey}:${slot}`
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
function haSlotPrefix(cluster: any): string {
|
|
1325
|
+
const p = cluster?.spec?.replicationSlots?.highAvailability?.slotPrefix
|
|
1326
|
+
return typeof p === 'string' && p ? p : '_cnpg_'
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
/** The HA slot CloudNativePG keeps for an instance: the prefix (default `_cnpg_`) then the instance name with `-` as `_`. */
|
|
1330
|
+
export function cnpgHASlotName(cluster: any, instance: string): string {
|
|
1331
|
+
return `${haSlotPrefix(cluster)}${instance.replace(/-/g, '_')}`
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
/** The instance a managed HA slot serves, or undefined when the slot is not one of this cluster's HA slots. */
|
|
1335
|
+
export function cnpgHASlotInstance(cluster: any, slot: string, instances: string[]): string | undefined {
|
|
1336
|
+
return instances.find((i) => cnpgHASlotName(cluster, i) === slot)
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
const CLUSTER_GROUP = 'postgresql.cnpg.io'
|
|
1340
|
+
|
|
1341
|
+
function clusterSubject(row: Pick<CNPGFleetRow, 'namespace' | 'name'>) {
|
|
1342
|
+
return { kind: 'Cluster', group: CLUSTER_GROUP, namespace: row.namespace, name: row.name }
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
export interface CNPGStandbyGap {
|
|
1346
|
+
pod: string
|
|
1347
|
+
/** What was observed, in words, e.g. "its WAL receiver is down", "replay is paused". */
|
|
1348
|
+
evidence: string[]
|
|
1349
|
+
measuredBy: string
|
|
1350
|
+
sourceDetail: string
|
|
1351
|
+
/** Every expected standby is out: there is no failover target with current data. */
|
|
1352
|
+
noneReceiving?: boolean
|
|
1353
|
+
unverifiedMatch?: boolean
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
/** A standby that receives nothing from the primary. Its replay lag reads 0 because nothing new arrives to replay. */
|
|
1357
|
+
export function cnpgStandbyNotReceivingProblem(row: Pick<CNPGFleetRow, 'key' | 'namespace' | 'name'>, gap: CNPGStandbyGap): CNPGProblem {
|
|
1358
|
+
return {
|
|
1359
|
+
id: cnpgStandbyProblemId(row.key, gap.pod),
|
|
1360
|
+
reason: 'CNPGStandbyNotReceiving',
|
|
1361
|
+
severity: gap.noneReceiving ? 'critical' : 'warning',
|
|
1362
|
+
category: 'availability',
|
|
1363
|
+
title: `${gap.pod} is not receiving WAL from the primary`,
|
|
1364
|
+
shortTitle: `${gap.pod} not receiving WAL`,
|
|
1365
|
+
detail: `${capitalize(gap.evidence.join('; '))}. While it does not stream, the primary keeps WAL for its slot, and it falls behind unless it replays WAL from the archive; a replay lag of 0 does not mean it is caught up, only that nothing new reached it.`,
|
|
1366
|
+
subject: { kind: 'Pod', group: '', namespace: row.namespace, name: gap.pod },
|
|
1367
|
+
source: 'measurement',
|
|
1368
|
+
measuredBy: gap.measuredBy,
|
|
1369
|
+
sourceDetail: gap.sourceDetail,
|
|
1370
|
+
...(gap.unverifiedMatch ? { unverifiedMatch: true } : {}),
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
export interface CNPGSlotRetention {
|
|
1375
|
+
slot: string
|
|
1376
|
+
/** The instance holding the slot (the primary for an HA slot). */
|
|
1377
|
+
pod: string
|
|
1378
|
+
bytes: number
|
|
1379
|
+
/** The standby an HA slot serves, when the name matches one of the cluster's instances. */
|
|
1380
|
+
standby?: string
|
|
1381
|
+
measuredBy: string
|
|
1382
|
+
sourceDetail: string
|
|
1383
|
+
unverifiedMatch?: boolean
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
export function cnpgSlotRetentionProblem(row: Pick<CNPGFleetRow, 'key' | 'namespace' | 'name'>, r: CNPGSlotRetention): CNPGProblem {
|
|
1387
|
+
const forWhom = r.standby ? ` for ${r.standby}` : ''
|
|
1388
|
+
return {
|
|
1389
|
+
id: cnpgSlotProblemId(row.key, r.slot),
|
|
1390
|
+
reason: 'CNPGInactiveSlot',
|
|
1391
|
+
slot: r.slot,
|
|
1392
|
+
severity: 'warning',
|
|
1393
|
+
category: 'availability',
|
|
1394
|
+
title: `Inactive slot ${r.slot} holds ${formatBytes(r.bytes)} of WAL on ${r.pod}${forWhom}`,
|
|
1395
|
+
shortTitle: `${formatBytes(r.bytes)} of WAL held${forWhom || ` by ${r.slot}`}`,
|
|
1396
|
+
detail: `PostgreSQL keeps every WAL file the slot still needs until ${r.standby ? `${r.standby} catches up` : 'its consumer catches up'} or the slot is dropped, so this grows while the slot stays inactive. Storage shows it beside the volume's other WAL.`,
|
|
1397
|
+
subject: clusterSubject(row),
|
|
1398
|
+
source: 'measurement',
|
|
1399
|
+
measuredBy: r.measuredBy,
|
|
1400
|
+
sourceDetail: r.sourceDetail,
|
|
1401
|
+
...(r.standby ? { instance: r.standby } : {}),
|
|
1402
|
+
...(r.unverifiedMatch ? { unverifiedMatch: true } : {}),
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
/**
|
|
1407
|
+
* Adds problems to a row, replacing any with the same id, and drops those a
|
|
1408
|
+
* newer, complete read has disproven (`drop`); attention and categories
|
|
1409
|
+
* follow from what remains.
|
|
1410
|
+
*/
|
|
1411
|
+
export function cnpgWithProblems(row: CNPGFleetRow, added: CNPGProblem[], drop?: (p: CNPGProblem) => boolean): CNPGFleetRow {
|
|
1412
|
+
if (added.length === 0 && !drop) return row
|
|
1413
|
+
const ids = new Set(added.map((p) => p.id))
|
|
1414
|
+
const problems = [...row.problems.filter((p) => !ids.has(p.id) && !drop?.(p)), ...added].sort(cnpgCompareProblems)
|
|
1415
|
+
if (problems.length === row.problems.length && problems.every((p, i) => p === row.problems[i])) return row
|
|
1416
|
+
return {
|
|
1417
|
+
...row,
|
|
1418
|
+
problems,
|
|
1419
|
+
attention: problems.some((p) => p.severity !== 'posture'),
|
|
1420
|
+
categories: new Set(problems.filter((p) => p.severity !== 'posture').map((p) => p.category)),
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
function capitalize(s: string): string {
|
|
1425
|
+
return s ? s[0].toUpperCase() + s.slice(1) : s
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
/** The short per-fact source when Radar has no Prometheus; the reason goes in `detail`. */
|
|
1429
|
+
export const CNPG_PROMETHEUS_NOT_CONNECTED = 'Prometheus not connected'
|
|
1430
|
+
|
|
1431
|
+
const CNPG_LAG_UNMEASURED_SOURCE = 'Pod readiness does not show whether a replica is streaming'
|
|
1432
|
+
|
|
1433
|
+
/** One cluster's answer from /api/cnpg/fleet-metrics. */
|
|
1434
|
+
export interface CNPGFleetMetricsReading {
|
|
1435
|
+
namespace: string
|
|
1436
|
+
name: string
|
|
1437
|
+
/** ok | noStandby | noSeries | denied | ambiguous | scopeMismatch | error | notRead */
|
|
1438
|
+
lag: {
|
|
1439
|
+
state: string
|
|
1440
|
+
grant?: Grant
|
|
1441
|
+
reason?: string
|
|
1442
|
+
seconds?: number
|
|
1443
|
+
pod?: string
|
|
1444
|
+
/** Standbys whose replay lag was read; `seconds` covers only these. */
|
|
1445
|
+
lagStandbys?: number
|
|
1446
|
+
/** The worst standby's lowest recorded lag over `sustainedWindow`, across every scrape of it; it was already reporting by the window's start. */
|
|
1447
|
+
sustainedSeconds?: number
|
|
1448
|
+
sustainedPod?: string
|
|
1449
|
+
sustainedWindow?: string
|
|
1450
|
+
isolation?: CNPGMetricIsolation
|
|
1451
|
+
/** Instances reporting that they are in recovery (standbys). */
|
|
1452
|
+
standbys?: number
|
|
1453
|
+
/** Standbys whose WAL receiver is up (cnpg_pg_replication_is_wal_receiver_up = 1). */
|
|
1454
|
+
receiving?: number
|
|
1455
|
+
/** Standbys whose WAL receiver is down: their lag reads 0 because nothing arrives. */
|
|
1456
|
+
receiverDown?: string[]
|
|
1457
|
+
/** The exporter reported no receiver state (or the read failed), so streaming is not established from lag alone. */
|
|
1458
|
+
receiverUnknown?: boolean
|
|
1459
|
+
receiverReason?: string
|
|
1460
|
+
/** Standbys whose receiver was down in every sample over `receiverDownWindow`: only these raise a problem, since a restarting standby is briefly down. */
|
|
1461
|
+
receiverDownSustained?: string[]
|
|
1462
|
+
receiverDownWindow?: string
|
|
1463
|
+
}
|
|
1464
|
+
/** Inactive physical replication slots and the WAL each keeps: ok | noSeries | denied | ambiguous | scopeMismatch | error | notRead */
|
|
1465
|
+
slots?: {
|
|
1466
|
+
state: string
|
|
1467
|
+
grant?: Grant
|
|
1468
|
+
reason?: string
|
|
1469
|
+
/** Null unless `state` is ok; [] when ok and none is inactive. */
|
|
1470
|
+
inactive?: { slot: string; pod: string; role?: 'primary' | 'standby'; bytes: number | null }[] | null
|
|
1471
|
+
/** Inactive slots beyond the per-cluster cap. */
|
|
1472
|
+
omitted?: number
|
|
1473
|
+
isolation?: CNPGMetricIsolation
|
|
1474
|
+
}
|
|
1475
|
+
/** ok | noSeries | denied | unavailable | error | notRead */
|
|
1476
|
+
growth: { state: string; grant?: Grant; reason?: string; bytesPerHour?: number; claim?: string; instance?: string; isolation?: CNPGMetricIsolation }
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/** How Prometheus series were tied to one cluster; `unverified` matched only by namespace and Pod or claim names. */
|
|
1480
|
+
export interface CNPGMetricIsolation {
|
|
1481
|
+
mode: 'configured' | 'verified' | 'unverified'
|
|
1482
|
+
note: string
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
// A finding stated as this cluster's must say when its series were matched by name alone.
|
|
1486
|
+
function isolationCaveat(iso: CNPGMetricIsolation | undefined): string {
|
|
1487
|
+
return iso?.mode === 'unverified' ? ` ${iso.note}.` : ''
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
export interface CNPGFleetMetricsSources {
|
|
1491
|
+
/** prometheus, or none when Radar has no Prometheus (`reason` says why). */
|
|
1492
|
+
source: 'prometheus' | 'none'
|
|
1493
|
+
reason?: string
|
|
1494
|
+
lagSource?: string
|
|
1495
|
+
growthSource?: string
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
export function cnpgLagTone(seconds: number): HealthLevel {
|
|
1499
|
+
if (seconds >= 30) return 'unhealthy'
|
|
1500
|
+
if (seconds >= 5) return 'degraded'
|
|
1501
|
+
return 'healthy'
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
/**
|
|
1505
|
+
* Replication's tone from the primary's pg_stat_replication: a missing
|
|
1506
|
+
* standby is degraded, and the lag of the ones that do stream can make it
|
|
1507
|
+
* worse. Missing standbys never hide a severe lag.
|
|
1508
|
+
*/
|
|
1509
|
+
export function cnpgReplicationTone(streaming: number, expected: number, maxLagSeconds: number | undefined): HealthLevel {
|
|
1510
|
+
const missing: HealthLevel = streaming < expected ? 'degraded' : 'healthy'
|
|
1511
|
+
return maxLagSeconds === undefined ? missing : worseTone(missing, cnpgLagTone(maxLagSeconds))
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
/**
|
|
1515
|
+
* The one way a replay lag reads: milliseconds below a second, one decimal
|
|
1516
|
+
* below 10 s, whole seconds below 100 s, then whole minutes, then hours and
|
|
1517
|
+
* minutes. Rounded down, so a lower bound stays one.
|
|
1518
|
+
*/
|
|
1519
|
+
export function cnpgFormatLag(s: number): string {
|
|
1520
|
+
if (s <= 0) return '0 s'
|
|
1521
|
+
if (s < 1) return `${Math.floor(s * 1000)} ms`
|
|
1522
|
+
if (s < 10) return `${(Math.floor(s * 10) / 10).toFixed(1)} s`
|
|
1523
|
+
if (s < 100) return `${Math.floor(s)} s`
|
|
1524
|
+
const minutes = Math.floor(s / 60)
|
|
1525
|
+
if (minutes < 60) return `${minutes} min`
|
|
1526
|
+
const m = minutes % 60
|
|
1527
|
+
return m === 0 ? `${Math.floor(minutes / 60)} h` : `${Math.floor(minutes / 60)} h ${m} min`
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
function measuredReplication(base: Fact, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources, designated?: string, expectedStandbys?: number | null): Fact {
|
|
1531
|
+
const prefix = base.text.replace(/ · lag unknown$/, '')
|
|
1532
|
+
if (src.source === 'none') {
|
|
1533
|
+
return { text: `${prefix} · lag unknown`, tone: 'unknown', source: CNPG_PROMETHEUS_NOT_CONNECTED, detail: src.reason }
|
|
1534
|
+
}
|
|
1535
|
+
const lag = reading?.lag
|
|
1536
|
+
switch (lag?.state) {
|
|
1537
|
+
case 'ok': {
|
|
1538
|
+
const down = (lag.receiverDown ?? []).filter((pod) => pod !== designated)
|
|
1539
|
+
if (down.length > 0) {
|
|
1540
|
+
return {
|
|
1541
|
+
text: `${prefix} · ${down.length === 1 ? `${down[0]} not receiving WAL` : `${down.length} standbys not receiving WAL`}`,
|
|
1542
|
+
tone: lag.receiving === 0 && down.length === lag.standbys ? 'unhealthy' : 'degraded',
|
|
1543
|
+
source: `WAL receiver down on ${down.join(', ')} (cnpg_pg_replication_is_wal_receiver_up = 0) · ${src.lagSource ?? 'Prometheus'}.${isolationCaveat(lag.isolation)}`,
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
if (lag.seconds === undefined) break
|
|
1547
|
+
// The lag covers the standbys whose lag was read; one that is not read is unknown, not caught up.
|
|
1548
|
+
const partial = expectedStandbys !== undefined && expectedStandbys !== null && lag.lagStandbys !== undefined && lag.lagStandbys < expectedStandbys
|
|
1549
|
+
const coverage = partial ? ` (${lag.lagStandbys} of ${expectedStandbys} ${designated !== undefined ? 'instances' : 'standbys'} reporting)` : ''
|
|
1550
|
+
const lagTone = cnpgLagTone(lag.seconds)
|
|
1551
|
+
// A replica cluster's lag is replay behind what arrived; a dead upstream
|
|
1552
|
+
// feed delivers nothing and still reads 0.
|
|
1553
|
+
const upstream = designated !== undefined
|
|
1554
|
+
return {
|
|
1555
|
+
text: `${prefix} · lag ${cnpgFormatLag(lag.seconds)}${coverage}${lag.receiverUnknown ? ' · streaming unverified' : ''}${upstream ? ' · upstream feed not assessed' : ''}`,
|
|
1556
|
+
tone: lag.receiverUnknown || ((partial || upstream) && lagTone === 'healthy') ? 'unknown' : lagTone,
|
|
1557
|
+
source: `Largest standby replay lag, ${lag.pod ?? 'a standby'} · ${src.lagSource ?? 'Prometheus'}.${lag.receiverUnknown ? ' The exporter reported no WAL receiver state, and a standby that receives nothing also reads 0.' : ''}${upstream ? ' The designated primary replays WAL from an external cluster; Radar does not assess that feed.' : ''}${isolationCaveat(lag.isolation)}`,
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
case 'noStandby':
|
|
1561
|
+
return { text: `${prefix} · lag unknown`, tone: 'unknown', source: `No standby reports lag: ${lag.reason ?? 'no instance reports being a standby'} · ${src.lagSource ?? 'Prometheus'}` }
|
|
1562
|
+
case 'denied':
|
|
1563
|
+
return { text: `${prefix} · lag unknown`, tone: 'unknown', source: lag.grant ? `Needs ${formatGrant(lag.grant)}` : lag.reason }
|
|
1564
|
+
}
|
|
1565
|
+
return { text: `${prefix} · lag unknown`, tone: 'unknown', source: lag?.reason ?? 'Replication lag needs Prometheus scraping the CNPG exporter' }
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
/** Volume growth of the fastest-growing claim, as a fact. */
|
|
1569
|
+
export function cnpgDiskGrowthFact(reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): Fact | undefined {
|
|
1570
|
+
const g = reading?.growth
|
|
1571
|
+
if (src.source === 'none' || !g || g.state !== 'ok' || g.bytesPerHour === undefined) return undefined
|
|
1572
|
+
const perDay = g.bytesPerHour * 24
|
|
1573
|
+
const text = Math.abs(perDay) < 1024 ? 'flat over 6 h' : `${perDay > 0 ? '+' : '−'}${formatBytes(Math.abs(perDay))}/day`
|
|
1574
|
+
return { text, tone: 'neutral', source: `Fastest-growing: ${g.claim ?? 'a volume'}${g.instance ? ` of ${g.instance}` : ''} · ${src.growthSource ?? 'Prometheus'}.${isolationCaveat(g.isolation)}` }
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
/**
|
|
1578
|
+
* Joins /api/cnpg/fleet-metrics into the fleet: a cluster whose replication
|
|
1579
|
+
* fact is only Pod readiness gets its measured standby lag, or says why it has
|
|
1580
|
+
* none; the disk growth, when measured, lands on `diskGrowth`. Only lag that
|
|
1581
|
+
* stayed high for the whole sustained window raises a problem; a spike is
|
|
1582
|
+
* shown, not judged, and growth is never judged.
|
|
1583
|
+
*/
|
|
1584
|
+
export function applyCNPGFleetMetrics(fleet: CNPGFleet, readings: CNPGFleetMetricsReading[] | undefined, src: CNPGFleetMetricsSources | undefined): CNPGFleet {
|
|
1585
|
+
if (!src) return fleet
|
|
1586
|
+
const byKey = new Map((readings ?? []).map((r) => [key(r.namespace, r.name), r]))
|
|
1587
|
+
const rows = fleet.rows.map((row) => {
|
|
1588
|
+
const reading = byKey.get(row.key)
|
|
1589
|
+
const next: CNPGFleetRow = { ...row, diskGrowth: cnpgDiskGrowthFact(reading, src) }
|
|
1590
|
+
if (row.replication.source === CNPG_LAG_UNMEASURED_SOURCE) next.replication = measuredReplication(
|
|
1591
|
+
row.replication,
|
|
1592
|
+
reading,
|
|
1593
|
+
src,
|
|
1594
|
+
row.replicaCluster ? (row.cluster?.status?.currentPrimary ?? '') : undefined,
|
|
1595
|
+
// Every instance of a replica cluster is in recovery, its designated primary included.
|
|
1596
|
+
row.instances.desired !== null ? (row.replicaCluster ? row.instances.desired : Math.max(0, row.instances.desired - 1)) : null,
|
|
1597
|
+
)
|
|
1598
|
+
const sustained = sustainedLagProblem(row, reading, src)
|
|
1599
|
+
return cnpgWithProblems(next, [...(sustained ? [sustained] : []), ...fleetStandbyProblems(row, reading, src), ...fleetSlotProblems(row, reading, src)])
|
|
1600
|
+
})
|
|
1601
|
+
return finishFleet(rows, fleet.incompleteKinds)
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
function fleetStandbyProblems(row: CNPGFleetRow, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): CNPGProblem[] {
|
|
1605
|
+
const lag = reading?.lag
|
|
1606
|
+
if (src.source !== 'prometheus' || lag?.state !== 'ok' || row.hibernated) return []
|
|
1607
|
+
// A replica cluster's designated primary is in recovery too, and may be fed
|
|
1608
|
+
// from the WAL archive with no receiver at all.
|
|
1609
|
+
const designated = row.replicaCluster ? row.cluster?.status?.currentPrimary : undefined
|
|
1610
|
+
const down = (lag.receiverDownSustained ?? []).filter((pod) => pod !== designated)
|
|
1611
|
+
const window = lag.receiverDownWindow ? formatWindowWords(lag.receiverDownWindow) : 'several minutes'
|
|
1612
|
+
return down.map((pod) =>
|
|
1613
|
+
cnpgStandbyNotReceivingProblem(row, {
|
|
1614
|
+
pod,
|
|
1615
|
+
evidence: [`its WAL receiver was down in every sample Prometheus recorded over the last ${window}`],
|
|
1616
|
+
measuredBy: lag.isolation?.mode === 'unverified' ? 'Prometheus, matched by Pod name' : 'Prometheus',
|
|
1617
|
+
sourceDetail: `cnpg_pg_replication_is_wal_receiver_up = 0 while cnpg_pg_replication_in_recovery = 1 · ${src.lagSource ?? 'Prometheus'}`,
|
|
1618
|
+
noneReceiving: noStandbyReceives(row, lag),
|
|
1619
|
+
unverifiedMatch: lag.isolation?.mode === 'unverified',
|
|
1620
|
+
}),
|
|
1621
|
+
)
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
// "None receives" only when every expected standby reported its receiver and
|
|
1625
|
+
// every one was down; one that did not report leaves it open.
|
|
1626
|
+
function noStandbyReceives(row: CNPGFleetRow, lag: CNPGFleetMetricsReading['lag']): boolean {
|
|
1627
|
+
const expected = row.instances.desired !== null ? Math.max(0, row.instances.desired - 1) : null
|
|
1628
|
+
const down = lag.receiverDownSustained?.length ?? 0
|
|
1629
|
+
return lag.receiving === 0 && expected !== null && expected > 0 && lag.standbys === expected && down === expected
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
function fleetSlotProblems(row: CNPGFleetRow, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): CNPGProblem[] {
|
|
1633
|
+
const slots = reading?.slots
|
|
1634
|
+
if (src.source !== 'prometheus' || slots?.state !== 'ok' || !slots.inactive) return []
|
|
1635
|
+
// CloudNativePG copies HA slots to the standbys, where nothing streams from
|
|
1636
|
+
// them, so a standby's copy is always inactive. Only the primary's copy
|
|
1637
|
+
// means a consumer stopped.
|
|
1638
|
+
const instances = row.pods.map((p) => p.name)
|
|
1639
|
+
return slots.inactive
|
|
1640
|
+
.filter((s): s is typeof s & { bytes: number } => s.role === 'primary' && s.bytes !== null && s.bytes >= CNPG_SLOT_RETENTION_WARNING_BYTES)
|
|
1641
|
+
.map((s) =>
|
|
1642
|
+
cnpgSlotRetentionProblem(row, {
|
|
1643
|
+
slot: s.slot,
|
|
1644
|
+
pod: s.pod,
|
|
1645
|
+
bytes: s.bytes,
|
|
1646
|
+
standby: cnpgHASlotInstance(row.cluster, s.slot, instances),
|
|
1647
|
+
measuredBy: slots.isolation?.mode === 'unverified' ? 'Prometheus, matched by Pod name' : 'Prometheus',
|
|
1648
|
+
sourceDetail: 'cnpg_pg_replication_slots_pg_wal_lsn_diff where cnpg_pg_replication_slots_active = 0 (physical slots)',
|
|
1649
|
+
unverifiedMatch: slots.isolation?.mode === 'unverified',
|
|
1650
|
+
}),
|
|
1651
|
+
)
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
export const CNPG_SUSTAINED_LAG_WARNING_SECONDS = 30
|
|
1655
|
+
export const CNPG_SUSTAINED_LAG_CRITICAL_SECONDS = 300
|
|
1656
|
+
|
|
1657
|
+
/** The id of a cluster's sustained-lag problem, so other views can find it among the row's problems. */
|
|
1658
|
+
export function cnpgSustainedLagProblemId(rowKey: string): string {
|
|
1659
|
+
return `lag:${rowKey}`
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
function sustainedLagProblem(row: CNPGFleetRow, reading: CNPGFleetMetricsReading | undefined, src: CNPGFleetMetricsSources): CNPGProblem | undefined {
|
|
1663
|
+
const lag = reading?.lag
|
|
1664
|
+
const floor = lag?.sustainedSeconds
|
|
1665
|
+
if (src.source !== 'prometheus' || lag?.state !== 'ok' || floor === undefined || floor < CNPG_SUSTAINED_LAG_WARNING_SECONDS) return undefined
|
|
1666
|
+
const window = lag.sustainedWindow ? formatWindowWords(lag.sustainedWindow) : 'several minutes'
|
|
1667
|
+
const pod = lag.sustainedPod ?? 'A standby'
|
|
1668
|
+
// The query proves every recorded sample was at least the floor and that
|
|
1669
|
+
// the series existed at the window's start, not that samples were continuous.
|
|
1670
|
+
return {
|
|
1671
|
+
id: cnpgSustainedLagProblemId(row.key),
|
|
1672
|
+
reason: 'CNPGSustainedLag',
|
|
1673
|
+
severity: floor >= CNPG_SUSTAINED_LAG_CRITICAL_SECONDS ? 'critical' : 'warning',
|
|
1674
|
+
category: 'availability',
|
|
1675
|
+
title: `${pod} ≥ ${cnpgFormatLag(floor)} behind in every sample for ${formatWindowShort(lag.sustainedWindow)}`,
|
|
1676
|
+
shortTitle: `${pod}: all samples ≥ ${cnpgFormatLag(floor)} behind (${formatWindowShort(lag.sustainedWindow)})`,
|
|
1677
|
+
detail: `Lowest replay lag in the samples Prometheus recorded over the last ${window}. If Prometheus missed some scrapes, those moments aren't included. If it was still that far behind, a failover to it would lose or wait on that much WAL.${isolationCaveat(lag.isolation)}`,
|
|
1678
|
+
subject: { kind: 'Cluster', group: 'postgresql.cnpg.io', namespace: row.namespace, name: row.name },
|
|
1679
|
+
source: 'measurement',
|
|
1680
|
+
measuredBy: lag.isolation?.mode === 'unverified' ? 'Prometheus, matched by Pod name' : 'Prometheus',
|
|
1681
|
+
unverifiedMatch: lag.isolation?.mode === 'unverified',
|
|
1682
|
+
sourceDetail: src.lagSource ?? 'Prometheus',
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
// "10m0s" as "10 min"; "1h0m0s" as "1 h", for a title that must stay short.
|
|
1687
|
+
function formatWindowShort(d: string | undefined): string {
|
|
1688
|
+
const m = d ? /^(?:(\d+)h)?(?:(\d+)m)?(?:0s)?$/.exec(d) : null
|
|
1689
|
+
if (!m) return d ?? 'minutes'
|
|
1690
|
+
const minutes = Number(m[1] ?? 0) * 60 + Number(m[2] ?? 0)
|
|
1691
|
+
return minutes >= 60 && minutes % 60 === 0 ? `${minutes / 60} h` : `${minutes} min`
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
// "10m0s" as "10 minutes"; "1h0m0s" as "1 hour".
|
|
1695
|
+
function formatWindowWords(d: string): string {
|
|
1696
|
+
const m = /^(?:(\d+)h)?(?:(\d+)m)?(?:0s)?$/.exec(d)
|
|
1697
|
+
if (!m) return d
|
|
1698
|
+
const minutes = Number(m[1] ?? 0) * 60 + Number(m[2] ?? 0)
|
|
1699
|
+
if (minutes >= 60 && minutes % 60 === 0) return minutes === 60 ? '1 hour' : `${minutes / 60} hours`
|
|
1700
|
+
return minutes === 1 ? '1 minute' : `${minutes} minutes`
|
|
1701
|
+
}
|