@oneuptime/common 14.0.2 → 14.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/Alert.ts +55 -0
- package/Models/DatabaseModels/AlertFeed.ts +2 -0
- package/Models/DatabaseModels/DatabaseServer.ts +1650 -0
- package/Models/DatabaseModels/DatabaseServerEndpoint.ts +540 -0
- package/Models/DatabaseModels/DatabaseServerFeed.ts +625 -0
- package/Models/DatabaseModels/DatabaseServerLabelRule.ts +514 -0
- package/Models/DatabaseModels/DatabaseServerOwnerRule.ts +596 -0
- package/Models/DatabaseModels/DatabaseServerOwnerTeam.ts +491 -0
- package/Models/DatabaseModels/DatabaseServerOwnerUser.ts +490 -0
- package/Models/DatabaseModels/Incident.ts +55 -0
- package/Models/DatabaseModels/IncidentAlert.ts +540 -0
- package/Models/DatabaseModels/IncidentFeed.ts +2 -0
- package/Models/DatabaseModels/Index.ts +18 -0
- package/Models/DatabaseModels/NetworkDeviceDiscoveryScan.ts +26 -0
- package/Models/DatabaseModels/Project.ts +60 -0
- package/Models/DatabaseModels/RumApplication.ts +52 -1
- package/Models/DatabaseModels/ScheduledMaintenance.ts +55 -0
- package/Models/DatabaseModels/TelemetryException.ts +2 -0
- package/Models/DatabaseModels/UserProjectSsoConsent.ts +146 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794800000000-AddSessionReplaySameOriginTracePropagation.ts +28 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794900000000-AddIncidentAlert.ts +89 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1795000000000-AddDatabaseServerTables.ts +673 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1795100000000-AddUserProjectSsoConsent.ts +51 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +8 -0
- package/Server/Middleware/MultipartFormData.ts +1 -1
- package/Server/Middleware/TelemetryIngest.ts +15 -8
- package/Server/Services/AlertReminderRuleService.ts +100 -24
- package/Server/Services/DatabaseServerEndpointService.ts +895 -0
- package/Server/Services/DatabaseServerFeedService.ts +128 -0
- package/Server/Services/DatabaseServerLabelRuleEngineService.ts +340 -0
- package/Server/Services/DatabaseServerLabelRuleService.ts +14 -0
- package/Server/Services/DatabaseServerOwnerRuleEngineService.ts +425 -0
- package/Server/Services/DatabaseServerOwnerRuleService.ts +14 -0
- package/Server/Services/DatabaseServerOwnerTeamService.ts +180 -0
- package/Server/Services/DatabaseServerOwnerUserService.ts +173 -0
- package/Server/Services/DatabaseServerService.ts +4546 -0
- package/Server/Services/DatabaseService.ts +14 -8
- package/Server/Services/DockerResourceService.ts +43 -2
- package/Server/Services/IncidentAlertService.ts +1811 -0
- package/Server/Services/IncidentReminderRuleService.ts +103 -23
- package/Server/Services/IncidentService.ts +210 -10
- package/Server/Services/IncidentStateTimelineService.ts +24 -0
- package/Server/Services/Index.ts +18 -0
- package/Server/Services/InventoryItemService.ts +113 -5
- package/Server/Services/IoTDeviceService.ts +145 -37
- package/Server/Services/OpenTelemetryIngestService.ts +19 -0
- package/Server/Services/PodmanResourceService.ts +43 -2
- package/Server/Services/ScheduledMaintenanceReminderRuleService.ts +107 -23
- package/Server/Services/ServiceService.ts +1 -1
- package/Server/Services/TelemetryUsageBillingService.ts +29 -2
- package/Server/Services/UserProjectSsoConsentService.ts +74 -0
- package/Server/Services/UserService.ts +14 -1
- package/Server/Types/Database/Permissions/OwnerTableRegistry.ts +18 -0
- package/Server/Types/Database/QueryHelper.ts +30 -1
- package/Server/Utils/AI/Toolbox/ResourceTools.ts +69 -1
- package/Server/Utils/Database/AffectedResourceRelations.ts +6 -0
- package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts +188 -6
- package/Server/Utils/Monitor/MonitorAlert.ts +28 -1
- package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts +2262 -755
- package/Server/Utils/Monitor/MonitorIncident.ts +28 -1
- package/Server/Utils/Monitor/MonitorMaintenanceSuppression.ts +268 -6
- package/Server/Utils/Monitor/MonitorResource.ts +47 -11
- package/Server/Utils/Monitor/MonitorResourceContext.ts +1 -0
- package/Server/Utils/Monitor/MonitorStepResourceIdentity.ts +239 -21
- package/Server/Utils/Monitor/PerSeriesResolutionRootCause.ts +43 -0
- package/Server/Utils/Monitor/SeriesResourceLabels.ts +291 -0
- package/Server/Utils/Monitor/SeriesResourceLinker.ts +111 -12
- package/Server/Utils/Rules/RuleRun/RuleRunRegistry.ts +25 -0
- package/Server/Utils/SessionReplay/SessionReplayGateCache.ts +24 -2
- package/Server/Utils/StartServer.ts +26 -6
- package/Server/Utils/TeamMember/ProjectLeaveResourceCleanup.ts +2 -0
- package/Server/Utils/Telemetry/DatabaseEndpointDiscovery.ts +688 -0
- package/Server/Utils/Telemetry/EntityRegistry.ts +5 -1
- package/Server/Utils/Telemetry/LogExceptionExtractor.ts +18 -1
- package/Server/Utils/Telemetry/ResourceEntityFilter.ts +175 -12
- package/Server/Utils/Telemetry/ResourceFacetResolver.ts +12 -0
- package/Server/Utils/Telemetry/ServiceDependencyDiscovery.ts +606 -36
- package/Server/Utils/Telemetry/Telemetry.ts +16 -0
- package/Server/Utils/Telemetry/TelemetryEntity.ts +22 -22
- package/Server/Utils/Telemetry/TelemetryIngestionKeyGuard.ts +16 -10
- package/Tests/App/Accounts/SignedInRedirect.test.tsx +241 -0
- package/Tests/App/AdminDashboard/AdminHeaderSmallScreens.test.tsx +221 -4
- package/Tests/App/Dashboard/AIInsightsResourcePageContext.test.ts +42 -0
- package/Tests/App/Dashboard/AffectedResourcesDisplay.test.tsx +100 -0
- package/Tests/App/Dashboard/AffectedResourcesPicker.test.tsx +86 -0
- package/Tests/App/Dashboard/AffectedResourcesPickerHiddenTypes.test.tsx +2 -0
- package/Tests/App/Dashboard/AffectedResourcesSummaryStep.test.tsx +19 -0
- package/Tests/App/Dashboard/BreadcrumbCoverage.test.tsx +14 -0
- package/Tests/App/Dashboard/BulkIncidentLinkActions.test.tsx +869 -0
- package/Tests/App/Dashboard/CephMetricTooltips.test.tsx +1777 -0
- package/Tests/App/Dashboard/CloudFleetSummary.test.ts +8 -0
- package/Tests/App/Dashboard/CloudMetricTooltips.test.tsx +887 -0
- package/Tests/App/Dashboard/ContainerHostMetricTooltips.test.tsx +1309 -0
- package/Tests/App/Dashboard/CorrelateFilterBuilder.test.tsx +26 -1
- package/Tests/App/Dashboard/DashboardAppSsoError.test.tsx +501 -0
- package/Tests/App/Dashboard/DashboardHeaderRequests.test.tsx +64 -1
- package/Tests/App/Dashboard/DashboardHeaderSmallScreens.test.tsx +178 -18
- package/Tests/App/Dashboard/DashboardMasterPageSsoRedirect.test.tsx +444 -0
- package/Tests/App/Dashboard/DashboardNavBarForeignHiddenRule.test.tsx +573 -0
- package/Tests/App/Dashboard/DatabaseChartAxes.test.tsx +431 -0
- package/Tests/App/Dashboard/DatabaseContainerDetailBadge.test.tsx +249 -0
- package/Tests/App/Dashboard/DatabaseDashboardE2EFixes.test.ts +681 -0
- package/Tests/App/Dashboard/DatabaseDocumentationMarkdown.test.ts +1801 -0
- package/Tests/App/Dashboard/DatabaseFleetSummaryRefresh.test.tsx +280 -0
- package/Tests/App/Dashboard/DatabaseFormHelpers.test.tsx +321 -0
- package/Tests/App/Dashboard/DatabaseMetricMonitorLink.test.tsx +795 -0
- package/Tests/App/Dashboard/DatabaseOverviewHeader.test.tsx +580 -0
- package/Tests/App/Dashboard/DatabaseRecommendationsEngineMetrics.test.ts +270 -0
- package/Tests/App/Dashboard/DatabaseServerComponents.test.tsx +1486 -0
- package/Tests/App/Dashboard/DatabaseServerEngineMetricsFilter.test.ts +223 -0
- package/Tests/App/Dashboard/DatabaseServerPresentation.test.ts +435 -0
- package/Tests/App/Dashboard/DatabaseServerSummary.test.ts +186 -0
- package/Tests/App/Dashboard/DatabaseServerTelemetryQueries.test.ts +1553 -0
- package/Tests/App/Dashboard/DatabaseTableTruncation.test.tsx +326 -0
- package/Tests/App/Dashboard/DatabaseTelemetryScope.test.ts +512 -0
- package/Tests/App/Dashboard/DatabaseTelemetryTabs.test.tsx +774 -0
- package/Tests/App/Dashboard/DatabaseViewLayout.test.tsx +233 -0
- package/Tests/App/Dashboard/DatabaseWorkloadBadge.test.tsx +972 -0
- package/Tests/App/Dashboard/DiscoveryReviewInventoryRefresh.test.tsx +5 -0
- package/Tests/App/Dashboard/DiscoveryReviewUnnamedHostHint.test.tsx +1061 -0
- package/Tests/App/Dashboard/DiscoveryScanLivePage.test.tsx +39 -0
- package/Tests/App/Dashboard/DockerSwarmClustersListTooltips.test.tsx +414 -0
- package/Tests/App/Dashboard/DockerSwarmMetricTooltips.test.ts +796 -0
- package/Tests/App/Dashboard/DockerSwarmOverviewTooltips.test.tsx +747 -0
- package/Tests/App/Dashboard/EventOverlayScope.test.ts +147 -0
- package/Tests/App/Dashboard/EventOverlayScopeHook.test.tsx +1 -0
- package/Tests/App/Dashboard/ExceptionMonitorEnvironmentFilter.test.tsx +38 -18
- package/Tests/App/Dashboard/HostDetailViewTooltips.test.tsx +725 -0
- package/Tests/App/Dashboard/HostListColumnTooltips.test.tsx +472 -0
- package/Tests/App/Dashboard/HostMetricTooltips.test.ts +709 -0
- package/Tests/App/Dashboard/HostOverviewTooltips.test.tsx +698 -0
- package/Tests/App/Dashboard/HostTooltipHarness.ts +124 -0
- package/Tests/App/Dashboard/HostsListTooltips.test.tsx +395 -0
- package/Tests/App/Dashboard/IncidentAlertLinkDialog.test.tsx +691 -0
- package/Tests/App/Dashboard/IncidentAlertLinkHelpers.test.ts +458 -0
- package/Tests/App/Dashboard/IncidentAlertLinkPages.test.tsx +828 -0
- package/Tests/App/Dashboard/IncidentAlertLinkSideMenus.test.tsx +231 -0
- package/Tests/App/Dashboard/IncidentCreateFromAlerts.test.tsx +689 -0
- package/Tests/App/Dashboard/IoTMetricTooltips.test.tsx +1123 -0
- package/Tests/App/Dashboard/KubernetesClusterMetricTooltips.test.ts +837 -0
- package/Tests/App/Dashboard/KubernetesClusterOverviewTooltips.test.tsx +699 -0
- package/Tests/App/Dashboard/KubernetesContainersTabTooltips.test.tsx +320 -0
- package/Tests/App/Dashboard/KubernetesCostTooltips.test.tsx +502 -0
- package/Tests/App/Dashboard/KubernetesInsightsTooltips.test.tsx +222 -0
- package/Tests/App/Dashboard/KubernetesResourceMetricTooltips.test.ts +561 -0
- package/Tests/App/Dashboard/KubernetesResourcePagesTooltips.test.tsx +1168 -0
- package/Tests/App/Dashboard/LinkIncidentAlertModal.test.tsx +154 -0
- package/Tests/App/Dashboard/LogsResourceFacetChips.test.tsx +1 -0
- package/Tests/App/Dashboard/MetricDescriptionRules.ts +170 -0
- package/Tests/App/Dashboard/MetricDescriptionsCatalog.test.ts +206 -0
- package/Tests/App/Dashboard/MetricDescriptionsConsistency.test.ts +275 -0
- package/Tests/App/Dashboard/MetricsViewerDrillDown.test.tsx +310 -0
- package/Tests/App/Dashboard/MetricsViewerEntityScopedRows.test.tsx +495 -0
- package/Tests/App/Dashboard/MonitorCreateFromMonitorBackedDevice.test.tsx +132 -1
- package/Tests/App/Dashboard/NavigationSearchKeywords.test.tsx +5 -0
- package/Tests/App/Dashboard/NetworkDeviceTooltips.test.tsx +852 -0
- package/Tests/App/Dashboard/NetworkDiagnosticsTooltips.test.tsx +556 -0
- package/Tests/App/Dashboard/NetworkMetricTooltips.test.ts +1348 -0
- package/Tests/App/Dashboard/NetworkSiteTooltips.test.tsx +679 -0
- package/Tests/App/Dashboard/ProductOrientation.test.tsx +4 -3
- package/Tests/App/Dashboard/ProxmoxMetricTooltips.test.ts +592 -0
- package/Tests/App/Dashboard/ProxmoxOverviewTooltips.test.tsx +1002 -0
- package/Tests/App/Dashboard/ProxmoxResourcePageTooltips.test.tsx +923 -0
- package/Tests/App/Dashboard/RecommendationsLoadedContext.test.tsx +349 -0
- package/Tests/App/Dashboard/ResourceArchiveSettings.test.tsx +62 -0
- package/Tests/App/Dashboard/ResourceFeedPageWiring.test.ts +8 -1
- package/Tests/App/Dashboard/ResourceFeedPagesFilterSort.test.tsx +14 -0
- package/Tests/App/Dashboard/ResourceOverviewTileTooltips.test.tsx +1066 -0
- package/Tests/App/Dashboard/RumOverviewPage.test.tsx +1195 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceOverviewPage.test.tsx +2 -0
- package/Tests/App/Dashboard/ServerlessMetricTooltips.test.tsx +474 -0
- package/Tests/App/Dashboard/ServiceMapDatabaseLink.test.tsx +1085 -0
- package/Tests/App/Dashboard/ServiceMetricTooltips.test.tsx +772 -0
- package/Tests/App/Dashboard/SessionReplayAuditTable.test.tsx +56 -1
- package/Tests/App/Dashboard/TelemetryMetricsSignals.test.ts +1207 -0
- package/Tests/App/Dashboard/TelemetryResourceRetentionSettings.test.tsx +10 -0
- package/Tests/App/Dashboard/TelemetryResourceTabTooltips.test.tsx +437 -0
- package/Tests/App/Dashboard/TracesResourceFacets.test.tsx +2 -1
- package/Tests/App/Dashboard/VMwareMetricTooltips.test.ts +801 -0
- package/Tests/App/Dashboard/VMwareOverviewTooltips.test.tsx +601 -0
- package/Tests/App/Dashboard/VMwareResourcePagesTooltips.test.tsx +646 -0
- package/Tests/App/Dashboard/VMwareTooltipHarness.ts +145 -0
- package/Tests/App/StatusPage/StatusPageHeaderForeignHiddenRule.test.tsx +284 -0
- package/Tests/ClassStrings.ts +178 -0
- package/Tests/Fixtures/MobilePasskeyUser.json +1 -2
- package/Tests/ForeignHiddenRuleGuard.ts +2152 -0
- package/Tests/Models/DatabaseModels/DatabaseServerModels.test.ts +2250 -0
- package/Tests/Models/DatabaseModels/IncidentAlertModel.test.ts +554 -0
- package/Tests/Models/DatabaseModels/ResourceFeedModels.test.ts +24 -5
- package/Tests/Models/DatabaseModels/SessionReplayModels.test.ts +117 -0
- package/Tests/Models/DatabaseModels/StaticRuleCriteriaInheritance.test.ts +4 -2
- package/Tests/RenderedMarkup.ts +518 -0
- package/Tests/ResponsiveVisibility.test.ts +490 -4
- package/Tests/ResponsiveVisibility.ts +505 -52
- package/Tests/Server/Infrastructure/Postgres/AddConfigurableRuleCriteriaMigration.test.ts +3 -0
- package/Tests/Server/Infrastructure/Postgres/AddDatabaseServerTablesMigration.test.ts +541 -0
- package/Tests/Server/Infrastructure/Postgres/AddIncidentAlertMigration.test.ts +341 -0
- package/Tests/Server/Infrastructure/Postgres/AddSessionReplaySameOriginTracePropagationMigration.test.ts +247 -0
- package/Tests/Server/Infrastructure/Postgres/AddUserProjectSsoConsentMigration.test.ts +234 -0
- package/Tests/Server/Services/ContainerSnapshotUpsertPostgres.test.ts +319 -0
- package/Tests/Server/Services/CrossProjectRelationWriteGuard.test.ts +13 -0
- package/Tests/Server/Services/DatabaseServerEndpointService.test.ts +2033 -0
- package/Tests/Server/Services/DatabaseServerFeedAndOwnerServices.test.ts +529 -0
- package/Tests/Server/Services/DatabaseServerLifecycle.test.ts +3302 -0
- package/Tests/Server/Services/DatabaseServerRegistries.test.ts +603 -0
- package/Tests/Server/Services/DatabaseServerRuleEngines.test.ts +754 -0
- package/Tests/Server/Services/DatabaseServerService.test.ts +3558 -0
- package/Tests/Server/Services/DatabaseServerSqlPostgres.test.ts +2045 -0
- package/Tests/Server/Services/DatabaseServiceUpdateColumnsIfUnlockedPostgres.test.ts +362 -0
- package/Tests/Server/Services/DatabaseServiceUpdateColumnsIfUnlockedReply.test.ts +210 -0
- package/Tests/Server/Services/HookFreeWriteLengthClamp.test.ts +18 -0
- package/Tests/Server/Services/IncidentAlertPostgres.test.ts +903 -0
- package/Tests/Server/Services/IncidentAlertRegistration.test.ts +96 -0
- package/Tests/Server/Services/IncidentAlertService.test.ts +2225 -0
- package/Tests/Server/Services/IncidentAlertSloLinkSurvivesEdit.test.ts +2 -0
- package/Tests/Server/Services/IncidentAlertStateCascade.test.ts +970 -0
- package/Tests/Server/Services/IncidentCreateFromAlerts.test.ts +1284 -0
- package/Tests/Server/Services/IncomingCallPolicyEscalationRuleTarget.test.ts +225 -0
- package/Tests/Server/Services/InventoryItemDisplayName.test.ts +280 -2
- package/Tests/Server/Services/InventoryUpsertSchemaParity.test.ts +8 -2
- package/Tests/Server/Services/IoTDeviceLatestMetricsPostgres.test.ts +689 -0
- package/Tests/Server/Services/IoTDeviceService.test.ts +379 -0
- package/Tests/Server/Services/LatestMetricsValuesShape.test.ts +360 -0
- package/Tests/Server/Services/LlmModelPriceService.test.ts +402 -0
- package/Tests/Server/Services/OpenTelemetryResourceRetentionMemo.test.ts +7 -1
- package/Tests/Server/Services/OwnerRowUniqueness.test.ts +61 -9
- package/Tests/Server/Services/ReminderRuleLookupPostgres.test.ts +2250 -0
- package/Tests/Server/Services/ReminderRuleRefreshOpenSubjectsPaging.test.ts +653 -0
- package/Tests/Server/Services/ResourceUpdateLastSeenLivenessFallback.test.ts +68 -20
- package/Tests/Server/Services/StaticRuleCriteriaRuntimeCoverage.test.ts +3 -3
- package/Tests/Server/Services/TelemetryResourceFacetCounting.test.ts +9 -0
- package/Tests/Server/Services/UserProjectSsoConsentPostgres.test.ts +153 -0
- package/Tests/Server/Services/UserProjectSsoConsentService.test.ts +192 -0
- package/Tests/Server/Services/UserServiceFirstMasterAdminElection.test.ts +71 -0
- package/Tests/Server/Services/UserSignupIdTakeoverPostgres.test.ts +238 -0
- package/Tests/Server/Services/WorkspaceNotificationSummaryHelpers.test.ts +592 -0
- package/Tests/Server/Types/Billing/PayAsYouGoMeteredAuthorization.test.ts +3 -0
- package/Tests/Server/Types/Database/EventOverlayScopePostgres.test.ts +1 -0
- package/Tests/Server/Types/Database/FeedEventTypeFilterQuery.test.ts +11 -0
- package/Tests/Server/Types/Database/Permissions/InfrastructureOwnershipScope.test.ts +8 -0
- package/Tests/Server/Types/Database/QueryHelperBooleanForCriteriaBackedRule.test.ts +1094 -0
- package/Tests/Server/Utils/AI/ResourceTools.test.ts +134 -0
- package/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.test.ts +41 -0
- package/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteriaAnomalyUnits.test.ts +279 -9
- package/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteriaDisplayContext.test.ts +933 -0
- package/Tests/Server/Utils/Monitor/DatabaseMaintenanceSuppression.test.ts +582 -0
- package/Tests/Server/Utils/Monitor/DatabaseServerMonitorLinking.test.ts +753 -0
- package/Tests/Server/Utils/Monitor/MonitorActionTimestamps.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/MonitorAlertCriteriaReferences.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/MonitorAlertResourceLinking.test.ts +119 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluator.test.ts +60 -25
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorAffectedResourceEdgeCases.test.ts +1719 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorKubernetesAffectedResources.test.ts +1447 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorKubernetesRatioEmail.test.ts +352 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorPlatformMetricUnits.test.ts +717 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorPlatformRawScan.test.ts +2125 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorPlatformSeriesAffectedResources.test.ts +1288 -0
- package/Tests/Server/Utils/Monitor/MonitorDependencySuppressionCreatorSkip.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/MonitorIncidentCriteriaReferences.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/MonitorIncidentMemberRoleMembership.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/MonitorIncidentResourceLinking.test.ts +88 -0
- package/Tests/Server/Utils/Monitor/MonitorMaintenanceSuppression.test.ts +205 -0
- package/Tests/Server/Utils/Monitor/MonitorResourceContext.test.ts +62 -0
- package/Tests/Server/Utils/Monitor/MonitorResourceDatabaseMaintenance.test.ts +803 -0
- package/Tests/Server/Utils/Monitor/MonitorStepResourceIdentity.test.ts +133 -3
- package/Tests/Server/Utils/Monitor/MonitorSummaryPersistence.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/PerSeriesRecoveryResolution.test.ts +472 -1
- package/Tests/Server/Utils/Monitor/PerSeriesSecondEntityAlerting.test.ts +1 -0
- package/Tests/Server/Utils/Monitor/SeriesResourceLinker.test.ts +236 -2
- package/Tests/Server/Utils/SeriesResourceLabels.test.ts +70 -0
- package/Tests/Server/Utils/SessionReplay/SessionReplayGateCachePolicy.test.ts +103 -0
- package/Tests/Server/Utils/StartServerProtobufContentType.test.ts +54 -0
- package/Tests/Server/Utils/Telemetry/DatabaseEndpointDiscovery.test.ts +1895 -0
- package/Tests/Server/Utils/Telemetry/EntityRegistry.test.ts +7 -0
- package/Tests/Server/Utils/Telemetry/LogExceptionExtractorSeverityGate.test.ts +90 -0
- package/Tests/Server/Utils/Telemetry/ResourceEntityFilter.test.ts +570 -0
- package/Tests/Server/Utils/Telemetry/ResourceFacetResolver.test.ts +97 -2
- package/Tests/Server/Utils/Telemetry/ServiceDependencyDiscovery.test.ts +1337 -0
- package/Tests/Server/Utils/Telemetry/TelemetryEntity.test.ts +295 -0
- package/Tests/Types/AI/AIResourceContext.test.ts +30 -0
- package/Tests/Types/DatabaseServer/DatabaseContainerClassifier.test.ts +3644 -0
- package/Tests/Types/DatabaseServer/DatabaseContainerCommand.test.ts +340 -0
- package/Tests/Types/DatabaseServer/DatabaseContainerCommandCorpus.ts +826 -0
- package/Tests/Types/DatabaseServer/DatabaseContainerCommandPostgres.test.ts +150 -0
- package/Tests/Types/DatabaseServer/DatabaseEndpoint.test.ts +2657 -0
- package/Tests/Types/DatabaseServer/DatabaseServerDiscoverySource.test.ts +64 -0
- package/Tests/Types/DatabaseServer/DatabaseServerEntityKeys.test.ts +952 -0
- package/Tests/Types/DatabaseServer/DatabaseServerMetricCatalog.test.ts +1161 -0
- package/Tests/Types/DatabaseServer/DatabaseSystem.test.ts +1514 -0
- package/Tests/Types/DatabaseServer/DatabaseTelemetryResolver.test.ts +1558 -0
- package/Tests/Types/Monitor/CephAlertTemplates.test.ts +255 -6
- package/Tests/Types/Monitor/DatabaseAlertTemplates.test.ts +2254 -0
- package/Tests/Types/Monitor/KubernetesAlertTemplates.test.ts +298 -0
- package/Tests/Types/Monitor/KubernetesMetricCatalog.test.ts +271 -0
- package/Tests/Types/Monitor/Recommendation/MonitorRecommendationCatalog.test.ts +253 -9
- package/Tests/Types/Monitor/Recommendation/MonitorRecommendationCoverage.test.ts +13 -5
- package/Tests/Types/Monitor/Recommendation/MonitorRecommendationNotificationMode.test.ts +4 -2
- package/Tests/Types/Monitor/Recommendation/MonitorRecommendationUtil.test.ts +111 -2
- package/Tests/Types/Monitor/Recommendation/RecommendationCriteriaBuilder.test.ts +61 -0
- package/Tests/Types/Monitor/SeriesContext/SeriesLabelDisplay.test.ts +114 -0
- package/Tests/Types/Monitor/Utils/KubernetesAgentEmittedMetrics.ts +181 -0
- package/Tests/Types/NetworkDevice/DiscoveredHostNamingStatus.test.ts +401 -0
- package/Tests/Types/Rum/WebVitals.test.ts +173 -0
- package/Tests/Types/Telemetry/ResourceEntityFacet.test.ts +4 -1
- package/Tests/Types/Telemetry/ResourceFacetCatalog.test.ts +24 -2
- package/Tests/UI/Components/Card.test.tsx +93 -2
- package/Tests/UI/Components/FeedOptions.test.ts +6 -0
- package/Tests/UI/Components/FeedOptionsButton.test.tsx +8 -1
- package/Tests/UI/Components/FieldLabelInfoCardTooltip.test.tsx +413 -0
- package/Tests/UI/Components/ForeignHiddenRuleResilience.test.tsx +1147 -0
- package/Tests/UI/Components/HeaderRightRail.test.tsx +41 -1
- package/Tests/UI/Components/InfoTooltip.test.tsx +242 -0
- package/Tests/UI/Components/LogsEntityNames.test.ts +16 -1
- package/Tests/UI/Components/LogsFacetSidebarHiddenFacets.test.tsx +7 -3
- package/Tests/UI/Components/ModelTable/BaseModelTableBulkDelete.test.tsx +207 -3
- package/Tests/UI/Components/ModelTable/BaseModelTableBulkDeleteVerb.test.tsx +367 -0
- package/Tests/UI/Components/ModelTable/BaseModelTableCreateButtonTitle.test.tsx +275 -0
- package/Tests/UI/Components/ResourceFacetConfigs.test.ts +2 -1
- package/Tests/UI/Components/RuleCriteria/RuleCriteriaModelTable.test.tsx +3 -3
- package/Tests/UI/Components/SrOnlyForeignRule.test.tsx +994 -0
- package/Tests/UI/Components/StatusPage/ResourceListSelection.test.tsx +34 -1
- package/Tests/UI/Components/TableHeaderTooltip.test.tsx +662 -0
- package/Tests/UI/Components/TelemetryFacetSidebarHiddenFacets.test.tsx +4 -1
- package/Tests/UI/ForeignHiddenRuleGuard.test.ts +1951 -0
- package/Tests/UI/Rum/PrivacySummaryCard.test.tsx +166 -9
- package/Tests/UI/Rum/RecordingHealthTooltips.test.tsx +447 -0
- package/Tests/UI/Rum/ReplayCorrelationPanel.test.tsx +164 -0
- package/Tests/UI/Rum/ReplayFrameCapture.test.ts +6750 -0
- package/Tests/UI/Rum/ReplayPlayerChrome.test.tsx +19 -1
- package/Tests/UI/Rum/ReplayRail.test.tsx +723 -12
- package/Tests/UI/Rum/ReplayRailDetail.test.tsx +86 -0
- package/Tests/UI/Rum/ReplayScreenshot.test.ts +2124 -0
- package/Tests/UI/Rum/ReplayScreenshotActions.test.tsx +3288 -0
- package/Tests/UI/Rum/ReplayStage.test.tsx +62 -0
- package/Tests/UI/Rum/ReplayStageOverlays.test.tsx +983 -2
- package/Tests/UI/Rum/ReplayTabSwitcher.test.tsx +19 -1
- package/Tests/UI/Rum/SessionReplaySettingsPolicyLoading.test.tsx +149 -0
- package/Tests/UI/Rum/SessionReplaySetupGuide.test.tsx +107 -6
- package/Tests/UI/Rum/SessionReplayUsersTableTooltips.test.tsx +330 -0
- package/Tests/UI/Rum/UserFlowTooltips.test.tsx +886 -0
- package/Tests/UI/TailwindPlayCdnForeignHiddenRule.test.ts +1005 -0
- package/Tests/UI/ThemeGroupHoverReveal.test.ts +847 -0
- package/Tests/UI/Utils/Breadcrumb/fixtures/RealBreadcrumbTrails.ts +80 -0
- package/Tests/UI/Utils/Breadcrumb/fixtures/RealRoutePatterns.ts +23 -0
- package/Tests/UI/Utils/ModelAPISsoRedirect.test.ts +127 -0
- package/Tests/UI/Utils/Navigation.test.ts +102 -0
- package/Tests/UI/Utils/PermissionGate.test.ts +76 -0
- package/Tests/UI/Utils/Telemetry/TelemetryEntityNames.test.ts +23 -0
- package/Tests/Utils/Incident/IncidentFromAlerts.test.ts +696 -0
- package/Tests/Utils/LabelRuleImportExport.test.ts +5 -0
- package/Tests/Utils/MetricUnitUtil.test.ts +227 -2
- package/Tests/Utils/Monitor/MetricValueFormatter.test.ts +288 -2
- package/Tests/Utils/Monitor/PlatformResourceIdentityAndUnits.test.ts +1559 -0
- package/Tests/Utils/NetworkDiscovery/DiscoveredHostNamingDiagnosis.test.ts +2546 -0
- package/Tests/Utils/NetworkDiscovery/DiscoveredHostUtil.test.ts +315 -0
- package/Tests/Utils/NetworkDiscovery/DiscoveryReverseDnsChain.test.ts +336 -1
- package/Tests/Utils/NetworkDiscovery/NetbiosNameUtil.test.ts +244 -0
- package/Tests/Utils/Rum/SessionTraceState.test.ts +239 -0
- package/Tests/Utils/ValueFormatter.test.ts +50 -0
- package/Tests/tsconfig.json +16 -0
- package/Types/AI/AIResourceContext.ts +7 -0
- package/Types/Dashboard/DashboardTemplates.ts +2 -1
- package/Types/DatabaseServer/DatabaseConnectionSpan.ts +77 -0
- package/Types/DatabaseServer/DatabaseContainerClassifier.ts +2068 -0
- package/Types/DatabaseServer/DatabaseContainerCommand.ts +801 -0
- package/Types/DatabaseServer/DatabaseEndpoint.ts +1943 -0
- package/Types/DatabaseServer/DatabaseServerDiscoverySource.ts +68 -0
- package/Types/DatabaseServer/DatabaseServerMetricCatalog.ts +966 -0
- package/Types/DatabaseServer/DatabaseSystem.ts +2124 -0
- package/Types/DatabaseServer/DatabaseTelemetryResolver.ts +555 -0
- package/Types/Email/EmailTemplateType.ts +1 -0
- package/Types/Incident/IncidentAlertLink.ts +38 -0
- package/Types/Monitor/CephAlertTemplates.ts +18 -0
- package/Types/Monitor/DatabaseAlertTemplates.ts +2020 -0
- package/Types/Monitor/KubernetesAlertTemplates.ts +64 -24
- package/Types/Monitor/KubernetesMetricCatalog.ts +118 -19
- package/Types/Monitor/MetricMonitor/MetricCriteriaContext.ts +14 -0
- package/Types/Monitor/MetricMonitor/MetricMonitorResponse.ts +93 -10
- package/Types/Monitor/Recommendation/MonitorRecommendationCatalog.ts +109 -8
- package/Types/Monitor/Recommendation/MonitorRecommendationTypes.ts +36 -7
- package/Types/Monitor/Recommendation/MonitorRecommendationUtil.ts +43 -2
- package/Types/Monitor/Recommendation/RecommendationCriteriaBuilder.ts +41 -6
- package/Types/Monitor/SeriesContext/SeriesLabelDisplay.ts +26 -0
- package/Types/NetworkDevice/DiscoveredHostNamingStatus.ts +147 -0
- package/Types/Permission.ts +315 -0
- package/Types/Probe/ProbeApiIngestResponse.ts +8 -0
- package/Types/Rules/RuleCriteriaFieldRegistry.ts +10 -0
- package/Types/Rules/RuleRun.ts +12 -0
- package/Types/Rum/SessionReplay.ts +25 -6
- package/Types/Rum/WebVitals.ts +52 -0
- package/Types/Telemetry/EntityType.ts +23 -0
- package/Types/Telemetry/ResourceFacetCatalog.ts +14 -0
- package/Types/Telemetry/ServiceType.ts +8 -0
- package/UI/Components/Banner/Banner.tsx +1 -1
- package/UI/Components/Card/Card.tsx +1 -1
- package/UI/Components/Charts/ChartGroup/ChartGroup.tsx +2 -2
- package/UI/Components/Charts/Line/LineChart.tsx +7 -0
- package/UI/Components/CommandPalette/CommandPalette.tsx +1 -1
- package/UI/Components/Date/StartAndEndDate.tsx +5 -5
- package/UI/Components/Detail/FieldLabel.tsx +8 -0
- package/UI/Components/Diagram/HorizontalStepChain.tsx +1 -1
- package/UI/Components/Feed/FeedOptionsButton.tsx +1 -1
- package/UI/Components/Forms/BasicForm.tsx +1 -1
- package/UI/Components/Header/Header.tsx +1 -1
- package/UI/Components/Header/HeaderIconDropdownButton.tsx +1 -1
- package/UI/Components/HeaderAlert/HeaderAlert.tsx +1 -1
- package/UI/Components/InfoCard/InfoCard.tsx +47 -1
- package/UI/Components/LogsViewer/components/FacetValueRow.tsx +1 -1
- package/UI/Components/LogsViewer/components/LogsFacetSidebar.tsx +1 -1
- package/UI/Components/Modal/ModalFooter.tsx +1 -1
- package/UI/Components/ModelDetail/CardModelDetail.tsx +2 -2
- package/UI/Components/ModelTable/BaseModelTable.tsx +99 -21
- package/UI/Components/ModelTable/Column.ts +5 -0
- package/UI/Components/Navbar/NavBar.tsx +9 -2
- package/UI/Components/Navbar/NavBarMenuModal.tsx +2 -2
- package/UI/Components/Page/Page.tsx +1 -1
- package/UI/Components/Pagination/Pagination.tsx +3 -3
- package/UI/Components/ProgressButtons/ProgressButtonItem.tsx +1 -1
- package/UI/Components/SideMenu/SideMenu.tsx +4 -1
- package/UI/Components/StatusPage/ResourceGroupNavigator.tsx +4 -2
- package/UI/Components/StatusPage/ResourceList.tsx +6 -6
- package/UI/Components/Table/TableHeader.tsx +47 -2
- package/UI/Components/Table/Types/Column.ts +7 -0
- package/UI/Components/TelemetryViewer/components/TelemetryFacetSidebar.tsx +1 -1
- package/UI/Components/TelemetryViewer/components/TelemetryFacetValueRow.tsx +1 -1
- package/UI/Components/Tooltip/InfoTooltip.tsx +78 -0
- package/UI/Components/Workflow/ColumnEditor/ModelQueryBuilder.tsx +1 -1
- package/UI/Components/Workflow/ComponentsModal.tsx +2 -2
- package/UI/Components/Workflow/StepTraceViewer.tsx +1 -1
- package/UI/Styles/Theme.css +8 -1
- package/UI/Utils/Navigation.ts +89 -42
- package/UI/Utils/PermissionGate.ts +11 -2
- package/UI/Utils/Telemetry/TelemetryEntityNames.ts +7 -0
- package/UI/Utils/TelemetryService.ts +4 -3
- package/Utils/Incident/IncidentFromAlerts.ts +516 -0
- package/Utils/LabelRuleImportExport.ts +2 -0
- package/Utils/MetricUnitUtil.ts +103 -11
- package/Utils/Monitor/MetricValueFormatter.ts +66 -13
- package/Utils/Monitor/PlatformMetricUnitUtil.ts +215 -0
- package/Utils/Monitor/PlatformResourceIdentity.ts +514 -0
- package/Utils/NetworkDiscovery/DiscoveredHostNamingDiagnosis.ts +425 -0
- package/Utils/NetworkDiscovery/DiscoveredHostUtil.ts +32 -0
- package/Utils/NetworkDiscovery/NetbiosNameUtil.ts +73 -0
- package/Utils/Rum/SessionTraceState.ts +212 -0
- package/Utils/Telemetry/DatabaseServerEntityKeys.ts +275 -0
- package/Utils/Telemetry/EntityKey.ts +176 -0
- package/Utils/ValueFormatter.ts +16 -0
- package/build/dist/Models/DatabaseModels/Alert.js +54 -0
- package/build/dist/Models/DatabaseModels/Alert.js.map +1 -1
- package/build/dist/Models/DatabaseModels/AlertFeed.js +2 -0
- package/build/dist/Models/DatabaseModels/AlertFeed.js.map +1 -1
- package/build/dist/Models/DatabaseModels/DatabaseServer.js +1683 -0
- package/build/dist/Models/DatabaseModels/DatabaseServer.js.map +1 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerEndpoint.js +559 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerEndpoint.js.map +1 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerFeed.js +648 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerFeed.js.map +1 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerLabelRule.js +522 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerLabelRule.js.map +1 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerOwnerRule.js +603 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerOwnerRule.js.map +1 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerOwnerTeam.js +506 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerOwnerTeam.js.map +1 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerOwnerUser.js +505 -0
- package/build/dist/Models/DatabaseModels/DatabaseServerOwnerUser.js.map +1 -0
- package/build/dist/Models/DatabaseModels/Incident.js +54 -0
- package/build/dist/Models/DatabaseModels/Incident.js.map +1 -1
- package/build/dist/Models/DatabaseModels/IncidentAlert.js +553 -0
- package/build/dist/Models/DatabaseModels/IncidentAlert.js.map +1 -0
- package/build/dist/Models/DatabaseModels/IncidentFeed.js +2 -0
- package/build/dist/Models/DatabaseModels/IncidentFeed.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +18 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/NetworkDeviceDiscoveryScan.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Project.js +62 -0
- package/build/dist/Models/DatabaseModels/Project.js.map +1 -1
- package/build/dist/Models/DatabaseModels/RumApplication.js +53 -1
- package/build/dist/Models/DatabaseModels/RumApplication.js.map +1 -1
- package/build/dist/Models/DatabaseModels/ScheduledMaintenance.js +54 -0
- package/build/dist/Models/DatabaseModels/ScheduledMaintenance.js.map +1 -1
- package/build/dist/Models/DatabaseModels/TelemetryException.js +2 -0
- package/build/dist/Models/DatabaseModels/TelemetryException.js.map +1 -1
- package/build/dist/Models/DatabaseModels/UserProjectSsoConsent.js +162 -0
- package/build/dist/Models/DatabaseModels/UserProjectSsoConsent.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794800000000-AddSessionReplaySameOriginTracePropagation.js +20 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794800000000-AddSessionReplaySameOriginTracePropagation.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794900000000-AddIncidentAlert.js +42 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794900000000-AddIncidentAlert.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1795000000000-AddDatabaseServerTables.js +240 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1795000000000-AddDatabaseServerTables.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1795100000000-AddUserProjectSsoConsent.js +26 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1795100000000-AddUserProjectSsoConsent.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +8 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Middleware/MultipartFormData.js +1 -1
- package/build/dist/Server/Middleware/TelemetryIngest.js +15 -8
- package/build/dist/Server/Middleware/TelemetryIngest.js.map +1 -1
- package/build/dist/Server/Services/AlertReminderRuleService.js +72 -23
- package/build/dist/Server/Services/AlertReminderRuleService.js.map +1 -1
- package/build/dist/Server/Services/DatabaseServerEndpointService.js +694 -0
- package/build/dist/Server/Services/DatabaseServerEndpointService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerFeedService.js +110 -0
- package/build/dist/Server/Services/DatabaseServerFeedService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerLabelRuleEngineService.js +272 -0
- package/build/dist/Server/Services/DatabaseServerLabelRuleEngineService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerLabelRuleService.js +13 -0
- package/build/dist/Server/Services/DatabaseServerLabelRuleService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerOwnerRuleEngineService.js +340 -0
- package/build/dist/Server/Services/DatabaseServerOwnerRuleEngineService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerOwnerRuleService.js +13 -0
- package/build/dist/Server/Services/DatabaseServerOwnerRuleService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerOwnerTeamService.js +160 -0
- package/build/dist/Server/Services/DatabaseServerOwnerTeamService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerOwnerUserService.js +153 -0
- package/build/dist/Server/Services/DatabaseServerOwnerUserService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseServerService.js +3272 -0
- package/build/dist/Server/Services/DatabaseServerService.js.map +1 -0
- package/build/dist/Server/Services/DatabaseService.js +8 -4
- package/build/dist/Server/Services/DatabaseService.js.map +1 -1
- package/build/dist/Server/Services/DockerResourceService.js +30 -3
- package/build/dist/Server/Services/DockerResourceService.js.map +1 -1
- package/build/dist/Server/Services/IncidentAlertService.js +1370 -0
- package/build/dist/Server/Services/IncidentAlertService.js.map +1 -0
- package/build/dist/Server/Services/IncidentReminderRuleService.js +73 -23
- package/build/dist/Server/Services/IncidentReminderRuleService.js.map +1 -1
- package/build/dist/Server/Services/IncidentService.js +164 -12
- package/build/dist/Server/Services/IncidentService.js.map +1 -1
- package/build/dist/Server/Services/IncidentStateTimelineService.js +21 -0
- package/build/dist/Server/Services/IncidentStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +18 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/InventoryItemService.js +88 -5
- package/build/dist/Server/Services/InventoryItemService.js.map +1 -1
- package/build/dist/Server/Services/IoTDeviceService.js +111 -24
- package/build/dist/Server/Services/IoTDeviceService.js.map +1 -1
- package/build/dist/Server/Services/OpenTelemetryIngestService.js +24 -9
- package/build/dist/Server/Services/OpenTelemetryIngestService.js.map +1 -1
- package/build/dist/Server/Services/PodmanResourceService.js +30 -3
- package/build/dist/Server/Services/PodmanResourceService.js.map +1 -1
- package/build/dist/Server/Services/ScheduledMaintenanceReminderRuleService.js +75 -23
- package/build/dist/Server/Services/ScheduledMaintenanceReminderRuleService.js.map +1 -1
- package/build/dist/Server/Services/ServiceService.js +1 -1
- package/build/dist/Server/Services/ServiceService.js.map +1 -1
- package/build/dist/Server/Services/TelemetryUsageBillingService.js +23 -2
- package/build/dist/Server/Services/TelemetryUsageBillingService.js.map +1 -1
- package/build/dist/Server/Services/UserProjectSsoConsentService.js +79 -0
- package/build/dist/Server/Services/UserProjectSsoConsentService.js.map +1 -0
- package/build/dist/Server/Services/UserService.js +13 -1
- package/build/dist/Server/Services/UserService.js.map +1 -1
- package/build/dist/Server/Types/Database/Permissions/OwnerTableRegistry.js +18 -0
- package/build/dist/Server/Types/Database/Permissions/OwnerTableRegistry.js.map +1 -1
- package/build/dist/Server/Types/Database/QueryHelper.js +26 -1
- package/build/dist/Server/Types/Database/QueryHelper.js.map +1 -1
- package/build/dist/Server/Utils/AI/Toolbox/ResourceTools.js +46 -0
- package/build/dist/Server/Utils/AI/Toolbox/ResourceTools.js.map +1 -1
- package/build/dist/Server/Utils/Database/AffectedResourceRelations.js +6 -0
- package/build/dist/Server/Utils/Database/AffectedResourceRelations.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.js +134 -13
- package/build/dist/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js +22 -6
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js +1489 -524
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js +22 -6
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorMaintenanceSuppression.js +196 -6
- package/build/dist/Server/Utils/Monitor/MonitorMaintenanceSuppression.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorResource.js +36 -9
- package/build/dist/Server/Utils/Monitor/MonitorResource.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorResourceContext.js +1 -0
- package/build/dist/Server/Utils/Monitor/MonitorResourceContext.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorStepResourceIdentity.js +175 -17
- package/build/dist/Server/Utils/Monitor/MonitorStepResourceIdentity.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/PerSeriesResolutionRootCause.js +29 -0
- package/build/dist/Server/Utils/Monitor/PerSeriesResolutionRootCause.js.map +1 -0
- package/build/dist/Server/Utils/Monitor/SeriesResourceLabels.js +211 -0
- package/build/dist/Server/Utils/Monitor/SeriesResourceLabels.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/SeriesResourceLinker.js +82 -10
- package/build/dist/Server/Utils/Monitor/SeriesResourceLinker.js.map +1 -1
- package/build/dist/Server/Utils/Rules/RuleRun/RuleRunRegistry.js +25 -0
- package/build/dist/Server/Utils/Rules/RuleRun/RuleRunRegistry.js.map +1 -1
- package/build/dist/Server/Utils/SessionReplay/SessionReplayGateCache.js +8 -0
- package/build/dist/Server/Utils/SessionReplay/SessionReplayGateCache.js.map +1 -1
- package/build/dist/Server/Utils/StartServer.js +20 -4
- package/build/dist/Server/Utils/StartServer.js.map +1 -1
- package/build/dist/Server/Utils/TeamMember/ProjectLeaveResourceCleanup.js +2 -0
- package/build/dist/Server/Utils/TeamMember/ProjectLeaveResourceCleanup.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/DatabaseEndpointDiscovery.js +442 -0
- package/build/dist/Server/Utils/Telemetry/DatabaseEndpointDiscovery.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/EntityRegistry.js +5 -1
- package/build/dist/Server/Utils/Telemetry/EntityRegistry.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/LogExceptionExtractor.js +15 -1
- package/build/dist/Server/Utils/Telemetry/LogExceptionExtractor.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/ResourceEntityFilter.js +121 -5
- package/build/dist/Server/Utils/Telemetry/ResourceEntityFilter.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/ResourceFacetResolver.js +12 -0
- package/build/dist/Server/Utils/Telemetry/ResourceFacetResolver.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/ServiceDependencyDiscovery.js +438 -36
- package/build/dist/Server/Utils/Telemetry/ServiceDependencyDiscovery.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/Telemetry.js +12 -0
- package/build/dist/Server/Utils/Telemetry/Telemetry.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/TelemetryEntity.js +17 -18
- package/build/dist/Server/Utils/Telemetry/TelemetryEntity.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/TelemetryIngestionKeyGuard.js +16 -10
- package/build/dist/Server/Utils/Telemetry/TelemetryIngestionKeyGuard.js.map +1 -1
- package/build/dist/Types/AI/AIResourceContext.js +7 -0
- package/build/dist/Types/AI/AIResourceContext.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardTemplates.js +2 -1
- package/build/dist/Types/Dashboard/DashboardTemplates.js.map +1 -1
- package/build/dist/Types/DatabaseServer/DatabaseConnectionSpan.js +71 -0
- package/build/dist/Types/DatabaseServer/DatabaseConnectionSpan.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseContainerClassifier.js +1462 -0
- package/build/dist/Types/DatabaseServer/DatabaseContainerClassifier.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseContainerCommand.js +691 -0
- package/build/dist/Types/DatabaseServer/DatabaseContainerCommand.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseEndpoint.js +1407 -0
- package/build/dist/Types/DatabaseServer/DatabaseEndpoint.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseServerDiscoverySource.js +60 -0
- package/build/dist/Types/DatabaseServer/DatabaseServerDiscoverySource.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseServerMetricCatalog.js +804 -0
- package/build/dist/Types/DatabaseServer/DatabaseServerMetricCatalog.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseSystem.js +1860 -0
- package/build/dist/Types/DatabaseServer/DatabaseSystem.js.map +1 -0
- package/build/dist/Types/DatabaseServer/DatabaseTelemetryResolver.js +406 -0
- package/build/dist/Types/DatabaseServer/DatabaseTelemetryResolver.js.map +1 -0
- package/build/dist/Types/Email/EmailTemplateType.js +1 -0
- package/build/dist/Types/Email/EmailTemplateType.js.map +1 -1
- package/build/dist/Types/Incident/IncidentAlertLink.js +34 -0
- package/build/dist/Types/Incident/IncidentAlertLink.js.map +1 -0
- package/build/dist/Types/Monitor/CephAlertTemplates.js +18 -0
- package/build/dist/Types/Monitor/CephAlertTemplates.js.map +1 -1
- package/build/dist/Types/Monitor/DatabaseAlertTemplates.js +1518 -0
- package/build/dist/Types/Monitor/DatabaseAlertTemplates.js.map +1 -0
- package/build/dist/Types/Monitor/KubernetesAlertTemplates.js +57 -24
- package/build/dist/Types/Monitor/KubernetesAlertTemplates.js.map +1 -1
- package/build/dist/Types/Monitor/KubernetesMetricCatalog.js +104 -18
- package/build/dist/Types/Monitor/KubernetesMetricCatalog.js.map +1 -1
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationCatalog.js +80 -3
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationCatalog.js.map +1 -1
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationTypes.js +10 -3
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationTypes.js.map +1 -1
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationUtil.js +29 -0
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationUtil.js.map +1 -1
- package/build/dist/Types/Monitor/Recommendation/RecommendationCriteriaBuilder.js +21 -6
- package/build/dist/Types/Monitor/Recommendation/RecommendationCriteriaBuilder.js.map +1 -1
- package/build/dist/Types/Monitor/SeriesContext/SeriesLabelDisplay.js +25 -0
- package/build/dist/Types/Monitor/SeriesContext/SeriesLabelDisplay.js.map +1 -1
- package/build/dist/Types/NetworkDevice/DiscoveredHostNamingStatus.js +136 -0
- package/build/dist/Types/NetworkDevice/DiscoveredHostNamingStatus.js.map +1 -0
- package/build/dist/Types/Permission.js +276 -0
- package/build/dist/Types/Permission.js.map +1 -1
- package/build/dist/Types/Rules/RuleCriteriaFieldRegistry.js +10 -0
- package/build/dist/Types/Rules/RuleCriteriaFieldRegistry.js.map +1 -1
- package/build/dist/Types/Rules/RuleRun.js +4 -0
- package/build/dist/Types/Rules/RuleRun.js.map +1 -1
- package/build/dist/Types/Rum/WebVitals.js +25 -0
- package/build/dist/Types/Rum/WebVitals.js.map +1 -1
- package/build/dist/Types/Telemetry/EntityType.js +23 -0
- package/build/dist/Types/Telemetry/EntityType.js.map +1 -1
- package/build/dist/Types/Telemetry/ResourceFacetCatalog.js +14 -0
- package/build/dist/Types/Telemetry/ResourceFacetCatalog.js.map +1 -1
- package/build/dist/Types/Telemetry/ServiceType.js +8 -0
- package/build/dist/Types/Telemetry/ServiceType.js.map +1 -1
- package/build/dist/UI/Components/Banner/Banner.js +1 -1
- package/build/dist/UI/Components/Banner/Banner.js.map +1 -1
- package/build/dist/UI/Components/Card/Card.js +1 -1
- package/build/dist/UI/Components/Card/Card.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js +2 -2
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js.map +1 -1
- package/build/dist/UI/Components/Charts/Line/LineChart.js +1 -1
- package/build/dist/UI/Components/Charts/Line/LineChart.js.map +1 -1
- package/build/dist/UI/Components/CommandPalette/CommandPalette.js +1 -1
- package/build/dist/UI/Components/CommandPalette/CommandPalette.js.map +1 -1
- package/build/dist/UI/Components/Date/StartAndEndDate.js +5 -5
- package/build/dist/UI/Components/Date/StartAndEndDate.js.map +1 -1
- package/build/dist/UI/Components/Detail/FieldLabel.js +2 -0
- package/build/dist/UI/Components/Detail/FieldLabel.js.map +1 -1
- package/build/dist/UI/Components/Diagram/HorizontalStepChain.js +1 -1
- package/build/dist/UI/Components/Diagram/HorizontalStepChain.js.map +1 -1
- package/build/dist/UI/Components/Feed/FeedOptionsButton.js +1 -1
- package/build/dist/UI/Components/Feed/FeedOptionsButton.js.map +1 -1
- package/build/dist/UI/Components/Forms/BasicForm.js +1 -1
- package/build/dist/UI/Components/Forms/BasicForm.js.map +1 -1
- package/build/dist/UI/Components/Header/HeaderIconDropdownButton.js +1 -1
- package/build/dist/UI/Components/Header/HeaderIconDropdownButton.js.map +1 -1
- package/build/dist/UI/Components/HeaderAlert/HeaderAlert.js +1 -1
- package/build/dist/UI/Components/HeaderAlert/HeaderAlert.js.map +1 -1
- package/build/dist/UI/Components/InfoCard/InfoCard.js +22 -1
- package/build/dist/UI/Components/InfoCard/InfoCard.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/FacetValueRow.js +1 -1
- package/build/dist/UI/Components/LogsViewer/components/FacetValueRow.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsFacetSidebar.js +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsFacetSidebar.js.map +1 -1
- package/build/dist/UI/Components/Modal/ModalFooter.js +1 -1
- package/build/dist/UI/Components/Modal/ModalFooter.js.map +1 -1
- package/build/dist/UI/Components/ModelDetail/CardModelDetail.js +2 -2
- package/build/dist/UI/Components/ModelDetail/CardModelDetail.js.map +1 -1
- package/build/dist/UI/Components/ModelTable/BaseModelTable.js +70 -18
- package/build/dist/UI/Components/ModelTable/BaseModelTable.js.map +1 -1
- package/build/dist/UI/Components/Navbar/NavBar.js +9 -2
- package/build/dist/UI/Components/Navbar/NavBar.js.map +1 -1
- package/build/dist/UI/Components/Navbar/NavBarMenuModal.js +2 -2
- package/build/dist/UI/Components/Navbar/NavBarMenuModal.js.map +1 -1
- package/build/dist/UI/Components/Page/Page.js +1 -1
- package/build/dist/UI/Components/Page/Page.js.map +1 -1
- package/build/dist/UI/Components/Pagination/Pagination.js +3 -3
- package/build/dist/UI/Components/Pagination/Pagination.js.map +1 -1
- package/build/dist/UI/Components/ProgressButtons/ProgressButtonItem.js +1 -1
- package/build/dist/UI/Components/ProgressButtons/ProgressButtonItem.js.map +1 -1
- package/build/dist/UI/Components/SideMenu/SideMenu.js +4 -1
- package/build/dist/UI/Components/SideMenu/SideMenu.js.map +1 -1
- package/build/dist/UI/Components/StatusPage/ResourceGroupNavigator.js +3 -1
- package/build/dist/UI/Components/StatusPage/ResourceGroupNavigator.js.map +1 -1
- package/build/dist/UI/Components/StatusPage/ResourceList.js +6 -6
- package/build/dist/UI/Components/StatusPage/ResourceList.js.map +1 -1
- package/build/dist/UI/Components/Table/TableHeader.js +18 -2
- package/build/dist/UI/Components/Table/TableHeader.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryFacetSidebar.js +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryFacetSidebar.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryFacetValueRow.js +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryFacetValueRow.js.map +1 -1
- package/build/dist/UI/Components/Tooltip/InfoTooltip.js +28 -0
- package/build/dist/UI/Components/Tooltip/InfoTooltip.js.map +1 -0
- package/build/dist/UI/Components/Workflow/ColumnEditor/ModelQueryBuilder.js +1 -1
- package/build/dist/UI/Components/Workflow/ColumnEditor/ModelQueryBuilder.js.map +1 -1
- package/build/dist/UI/Components/Workflow/ComponentsModal.js +2 -2
- package/build/dist/UI/Components/Workflow/ComponentsModal.js.map +1 -1
- package/build/dist/UI/Components/Workflow/StepTraceViewer.js +1 -1
- package/build/dist/UI/Components/Workflow/StepTraceViewer.js.map +1 -1
- package/build/dist/UI/Utils/Navigation.js +66 -33
- package/build/dist/UI/Utils/Navigation.js.map +1 -1
- package/build/dist/UI/Utils/PermissionGate.js +4 -2
- package/build/dist/UI/Utils/PermissionGate.js.map +1 -1
- package/build/dist/UI/Utils/Telemetry/TelemetryEntityNames.js +7 -0
- package/build/dist/UI/Utils/Telemetry/TelemetryEntityNames.js.map +1 -1
- package/build/dist/UI/Utils/TelemetryService.js +3 -2
- package/build/dist/UI/Utils/TelemetryService.js.map +1 -1
- package/build/dist/Utils/Incident/IncidentFromAlerts.js +306 -0
- package/build/dist/Utils/Incident/IncidentFromAlerts.js.map +1 -0
- package/build/dist/Utils/LabelRuleImportExport.js +2 -0
- package/build/dist/Utils/LabelRuleImportExport.js.map +1 -1
- package/build/dist/Utils/MetricUnitUtil.js +77 -13
- package/build/dist/Utils/MetricUnitUtil.js.map +1 -1
- package/build/dist/Utils/Monitor/MetricValueFormatter.js +58 -13
- package/build/dist/Utils/Monitor/MetricValueFormatter.js.map +1 -1
- package/build/dist/Utils/Monitor/PlatformMetricUnitUtil.js +161 -0
- package/build/dist/Utils/Monitor/PlatformMetricUnitUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/PlatformResourceIdentity.js +323 -0
- package/build/dist/Utils/Monitor/PlatformResourceIdentity.js.map +1 -0
- package/build/dist/Utils/NetworkDiscovery/DiscoveredHostNamingDiagnosis.js +270 -0
- package/build/dist/Utils/NetworkDiscovery/DiscoveredHostNamingDiagnosis.js.map +1 -0
- package/build/dist/Utils/NetworkDiscovery/DiscoveredHostUtil.js +23 -0
- package/build/dist/Utils/NetworkDiscovery/DiscoveredHostUtil.js.map +1 -1
- package/build/dist/Utils/NetworkDiscovery/NetbiosNameUtil.js +61 -0
- package/build/dist/Utils/NetworkDiscovery/NetbiosNameUtil.js.map +1 -1
- package/build/dist/Utils/Rum/SessionTraceState.js +154 -0
- package/build/dist/Utils/Rum/SessionTraceState.js.map +1 -0
- package/build/dist/Utils/Telemetry/DatabaseServerEntityKeys.js +178 -0
- package/build/dist/Utils/Telemetry/DatabaseServerEntityKeys.js.map +1 -0
- package/build/dist/Utils/Telemetry/EntityKey.js +133 -0
- package/build/dist/Utils/Telemetry/EntityKey.js.map +1 -1
- package/build/dist/Utils/ValueFormatter.js +15 -0
- package/build/dist/Utils/ValueFormatter.js.map +1 -1
- package/package.json +1 -1
- package/tsconfig.tests.json +7 -5
|
@@ -6,10 +6,12 @@ import CustomCodeMonitoringCriteria from "./Criteria/CustomCodeMonitorCriteria";
|
|
|
6
6
|
import IncomingEmailCriteria from "./Criteria/IncomingEmailCriteria";
|
|
7
7
|
import IncomingRequestCriteria from "./Criteria/IncomingRequestCriteria";
|
|
8
8
|
import IncomingRequestIncidentGrouping from "./IncomingRequestIncidentGrouping";
|
|
9
|
+
import { IOT_DEVICE_ID_ATTRIBUTE_KEY } from "./IoTDeviceAbsenceSeries";
|
|
9
10
|
import PerEntityCriteriaFanOut, {
|
|
10
11
|
FanOutEntity,
|
|
11
12
|
} from "./PerEntityCriteriaFanOut";
|
|
12
13
|
import SSLMonitorCriteria from "./Criteria/SSLMonitorCriteria";
|
|
14
|
+
import CompareCriteria from "./Criteria/CompareCriteria";
|
|
13
15
|
import ServerMonitorCriteria from "./Criteria/ServerMonitorCriteria";
|
|
14
16
|
import SyntheticMonitoringCriteria from "./Criteria/SyntheticMonitor";
|
|
15
17
|
import LogMonitorCriteria from "./Criteria/LogMonitorCriteria";
|
|
@@ -71,6 +73,8 @@ import { JSONObject } from "../../../Types/JSON";
|
|
|
71
73
|
import Dictionary from "../../../Types/Dictionary";
|
|
72
74
|
import InBetween from "../../../Types/BaseDatabase/InBetween";
|
|
73
75
|
import MetricQueryConfigData from "../../../Types/Metrics/MetricQueryConfigData";
|
|
76
|
+
import MetricFormulaConfigData from "../../../Types/Metrics/MetricFormulaConfigData";
|
|
77
|
+
import MetricsViewConfig from "../../../Types/Metrics/MetricsViewConfig";
|
|
74
78
|
import MetricExplorerUrl from "../../../Utils/Metrics/MetricExplorerUrl";
|
|
75
79
|
import {
|
|
76
80
|
CrossSignalQueryParams,
|
|
@@ -87,6 +91,7 @@ import Hostname from "../../../Types/API/Hostname";
|
|
|
87
91
|
import Port from "../../../Types/Port";
|
|
88
92
|
import { DashboardClientUrl } from "../../EnvironmentConfig";
|
|
89
93
|
import MetricMonitorResponse, {
|
|
94
|
+
PlatformResourceBreakdownSource,
|
|
90
95
|
KubernetesAffectedResource,
|
|
91
96
|
KubernetesResourceBreakdown,
|
|
92
97
|
ProxmoxAffectedResource,
|
|
@@ -107,6 +112,7 @@ import MetricCriteriaContext, {
|
|
|
107
112
|
import MonitorStepDockerMonitor from "../../../Types/Monitor/MonitorStepDockerMonitor";
|
|
108
113
|
import MonitorStepHostMonitor from "../../../Types/Monitor/MonitorStepHostMonitor";
|
|
109
114
|
import MonitorStepPodmanMonitor from "../../../Types/Monitor/MonitorStepPodmanMonitor";
|
|
115
|
+
import MonitorStepIoTMonitor from "../../../Types/Monitor/MonitorStepIoTMonitor";
|
|
110
116
|
import MonitorStepProxmoxMonitor from "../../../Types/Monitor/MonitorStepProxmoxMonitor";
|
|
111
117
|
import MonitorStepVMwareMonitor from "../../../Types/Monitor/MonitorStepVMwareMonitor";
|
|
112
118
|
import MonitorStepCephMonitor from "../../../Types/Monitor/MonitorStepCephMonitor";
|
|
@@ -117,6 +123,17 @@ import { getProxmoxMetricByMetricName } from "../../../Types/Monitor/ProxmoxMetr
|
|
|
117
123
|
import { getVMwareMetricByMetricName } from "../../../Types/Monitor/VMwareMetricCatalog";
|
|
118
124
|
import { getCephMetricByMetricName } from "../../../Types/Monitor/CephMetricCatalog";
|
|
119
125
|
import { getDockerSwarmMetricByMetricName } from "../../../Types/Monitor/DockerSwarmMetricCatalog";
|
|
126
|
+
import PlatformMetricUnitUtil from "../../../Utils/Monitor/PlatformMetricUnitUtil";
|
|
127
|
+
import MetricUnitUtil from "../../../Utils/MetricUnitUtil";
|
|
128
|
+
import PlatformResourceIdentity, {
|
|
129
|
+
CephResourceIdentity,
|
|
130
|
+
DockerSwarmResourceIdentity,
|
|
131
|
+
KubernetesResourceIdentity,
|
|
132
|
+
ProxmoxResourceIdentity,
|
|
133
|
+
VMwareResourceIdentity,
|
|
134
|
+
} from "../../../Utils/Monitor/PlatformResourceIdentity";
|
|
135
|
+
import MetricAliasData from "../../../Types/Metrics/MetricAliasData";
|
|
136
|
+
import MetricFormulaEvaluator from "../../../Utils/Metrics/MetricFormulaEvaluator";
|
|
120
137
|
|
|
121
138
|
/**
|
|
122
139
|
* A cross-signal deep link into a telemetry explorer, plus the scope
|
|
@@ -135,6 +152,83 @@ export interface TelemetryExplorerDeepLink {
|
|
|
135
152
|
interface AffectedResourceBreachPredicate {
|
|
136
153
|
matches: (value: number) => boolean;
|
|
137
154
|
worstIsLowest: boolean;
|
|
155
|
+
/*
|
|
156
|
+
* Which end of a raw-scan resource's window to judge it by, when neither
|
|
157
|
+
* "the highest" nor "the lowest" is right on its own. Absent means:
|
|
158
|
+
* the lowest when worstIsLowest, else the highest. Receives the two
|
|
159
|
+
* ends already in the criteria's comparison unit.
|
|
160
|
+
*/
|
|
161
|
+
pickEnd?:
|
|
162
|
+
| ((ends: { highest: number; lowest: number }) => "highest" | "lowest")
|
|
163
|
+
| undefined;
|
|
164
|
+
/*
|
|
165
|
+
* What a raw-scan list falls back to when no single row satisfies
|
|
166
|
+
* `matches` — a criteria that compared an AGGREGATE across resources (a
|
|
167
|
+
* Sum of in-flight requests over three API servers) can breach while no
|
|
168
|
+
* one resource crosses the threshold on its own.
|
|
169
|
+
*/
|
|
170
|
+
fallbackMatches?: ((value: number) => boolean) | undefined;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/*
|
|
174
|
+
* The platforms whose monitors carry a per-resource breakdown.
|
|
175
|
+
*/
|
|
176
|
+
type PlatformName =
|
|
177
|
+
| "kubernetes"
|
|
178
|
+
| "proxmox"
|
|
179
|
+
| "vmware"
|
|
180
|
+
| "dockerSwarm"
|
|
181
|
+
| "ceph";
|
|
182
|
+
|
|
183
|
+
interface CriteriaMetricTargetComponent {
|
|
184
|
+
alias: string;
|
|
185
|
+
metricName: string | undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The query or formula a matched criteria compared against its threshold.
|
|
190
|
+
*/
|
|
191
|
+
interface CriteriaMetricTarget {
|
|
192
|
+
alias: string;
|
|
193
|
+
isFormula: boolean;
|
|
194
|
+
/*
|
|
195
|
+
* The unit the criteria compared its threshold in: the filter's own
|
|
196
|
+
* threshold unit, else the query's legendUnit (what the worker converted
|
|
197
|
+
* the samples into). Undefined means the metric's native unit.
|
|
198
|
+
*/
|
|
199
|
+
comparisonUnit: string | undefined;
|
|
200
|
+
/** The query's metric; undefined for a formula. */
|
|
201
|
+
metricName: string | undefined;
|
|
202
|
+
/** The legend the user gave the query or formula, when it is not just the alias. */
|
|
203
|
+
displayName: string | undefined;
|
|
204
|
+
formulaExpression: string | undefined;
|
|
205
|
+
/** For a formula: the variable behind each alias it references. */
|
|
206
|
+
components: Array<CriteriaMetricTargetComponent>;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* One row of a platform "Affected Resources" list: the resource, its
|
|
211
|
+
* worst value (null for a series that matched with no data) and that
|
|
212
|
+
* value already rendered with its unit (without emphasis — the list
|
|
213
|
+
* bolds it, a sentence may not).
|
|
214
|
+
*/
|
|
215
|
+
interface PlatformAffectedRow<I> {
|
|
216
|
+
identity: I;
|
|
217
|
+
value: number | null;
|
|
218
|
+
formattedValue: string;
|
|
219
|
+
/*
|
|
220
|
+
* Set when the row's value belongs to a DIFFERENT filter than the one
|
|
221
|
+
* the list is about — a series that matched an "Any" criteria on its
|
|
222
|
+
* other filter. It names that filter's metric next to the value, and
|
|
223
|
+
* the row is never ranked against rows in the list's own unit.
|
|
224
|
+
*/
|
|
225
|
+
valueNote?: string | undefined;
|
|
226
|
+
/*
|
|
227
|
+
* The series' labels, when none of them names a platform object (a
|
|
228
|
+
* monitor grouped by a PVC name, say). The row is then titled by them
|
|
229
|
+
* instead of as an anonymous "Cluster".
|
|
230
|
+
*/
|
|
231
|
+
seriesLabels?: JSONObject | undefined;
|
|
138
232
|
}
|
|
139
233
|
|
|
140
234
|
export default class MonitorCriteriaEvaluator {
|
|
@@ -331,6 +425,7 @@ export default class MonitorCriteriaEvaluator {
|
|
|
331
425
|
monitorStep: input.monitorStep,
|
|
332
426
|
monitor: input.monitor,
|
|
333
427
|
criteriaInstance: criteriaInstance,
|
|
428
|
+
perSeriesMatches: perSeriesMatches,
|
|
334
429
|
});
|
|
335
430
|
|
|
336
431
|
/*
|
|
@@ -720,6 +815,9 @@ ${contextBlock}
|
|
|
720
815
|
labels: entry.labels,
|
|
721
816
|
rootCause: rootCauseLines.join("\n"),
|
|
722
817
|
metricContext: matched[0]?.context,
|
|
818
|
+
metricContexts: matched.map((r: MetricSeriesEvaluationResult) => {
|
|
819
|
+
return r.context;
|
|
820
|
+
}),
|
|
723
821
|
});
|
|
724
822
|
}
|
|
725
823
|
|
|
@@ -1278,6 +1376,11 @@ ${contextBlock}
|
|
|
1278
1376
|
monitorStep: MonitorStep;
|
|
1279
1377
|
monitor: Monitor;
|
|
1280
1378
|
criteriaInstance?: MonitorCriteriaInstance;
|
|
1379
|
+
/*
|
|
1380
|
+
* The series this criteria matched, for a grouped monitor. The
|
|
1381
|
+
* platform contexts list exactly these as the affected resources.
|
|
1382
|
+
*/
|
|
1383
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
1281
1384
|
}): Promise<string | null> {
|
|
1282
1385
|
// Handle Kubernetes monitors with rich resource context
|
|
1283
1386
|
if (input.monitor.monitorType === MonitorType.Kubernetes) {
|
|
@@ -1301,6 +1404,11 @@ ${contextBlock}
|
|
|
1301
1404
|
return MonitorCriteriaEvaluator.buildPodmanRootCauseContext(input);
|
|
1302
1405
|
}
|
|
1303
1406
|
|
|
1407
|
+
// Handle IoT device monitors with fleet / device context
|
|
1408
|
+
if (input.monitor.monitorType === MonitorType.IoTDevice) {
|
|
1409
|
+
return MonitorCriteriaEvaluator.buildIoTRootCauseContext(input);
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1304
1412
|
// Handle Proxmox monitors with resource context
|
|
1305
1413
|
if (input.monitor.monitorType === MonitorType.Proxmox) {
|
|
1306
1414
|
return MonitorCriteriaEvaluator.buildProxmoxRootCauseContext(input);
|
|
@@ -1450,19 +1558,15 @@ ${contextBlock}
|
|
|
1450
1558
|
return sections.join("\n");
|
|
1451
1559
|
}
|
|
1452
1560
|
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
*/
|
|
1463
|
-
const ctx: MetricCriteriaContext | undefined = (
|
|
1464
|
-
input.criteriaInstance.data?.filters || []
|
|
1465
|
-
)
|
|
1561
|
+
/*
|
|
1562
|
+
* The first populated metric context across the instance's filters.
|
|
1563
|
+
* Only metric-value filters populate this at evaluation time, so this
|
|
1564
|
+
* effectively returns the context for the filter that ran.
|
|
1565
|
+
*/
|
|
1566
|
+
private static getMetricCriteriaContext(
|
|
1567
|
+
criteriaInstance: MonitorCriteriaInstance | undefined,
|
|
1568
|
+
): MetricCriteriaContext | undefined {
|
|
1569
|
+
return (criteriaInstance?.data?.filters || [])
|
|
1466
1570
|
.map((f: CriteriaFilter) => {
|
|
1467
1571
|
return f.metricCriteriaContext;
|
|
1468
1572
|
})
|
|
@@ -1471,6 +1575,15 @@ ${contextBlock}
|
|
|
1471
1575
|
return Boolean(c);
|
|
1472
1576
|
},
|
|
1473
1577
|
);
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
private static buildMetricRootCauseContext(input: {
|
|
1581
|
+
criteriaInstance: MonitorCriteriaInstance;
|
|
1582
|
+
monitor: Monitor;
|
|
1583
|
+
monitorStep?: MonitorStep | undefined;
|
|
1584
|
+
}): string | null {
|
|
1585
|
+
const ctx: MetricCriteriaContext | undefined =
|
|
1586
|
+
MonitorCriteriaEvaluator.getMetricCriteriaContext(input.criteriaInstance);
|
|
1474
1587
|
|
|
1475
1588
|
if (!ctx) {
|
|
1476
1589
|
return null;
|
|
@@ -1571,6 +1684,10 @@ ${contextBlock}
|
|
|
1571
1684
|
unit: ctx.unit,
|
|
1572
1685
|
unitHeuristicMetricName: unitHeuristicMetricName,
|
|
1573
1686
|
components: ctx.components || [],
|
|
1687
|
+
seriesLabelKeys: [
|
|
1688
|
+
...ctx.groupBy,
|
|
1689
|
+
...Object.keys(ctx.seriesLabels || {}),
|
|
1690
|
+
],
|
|
1574
1691
|
})}`,
|
|
1575
1692
|
);
|
|
1576
1693
|
}
|
|
@@ -1579,6 +1696,7 @@ ${contextBlock}
|
|
|
1579
1696
|
MonitorCriteriaEvaluator.buildMetricExplorerDeepLink({
|
|
1580
1697
|
monitor: input.monitor,
|
|
1581
1698
|
ctx,
|
|
1699
|
+
monitorStep: input.monitorStep,
|
|
1582
1700
|
});
|
|
1583
1701
|
|
|
1584
1702
|
if (deepLink) {
|
|
@@ -1649,6 +1767,52 @@ ${contextBlock}
|
|
|
1649
1767
|
* the client-side markdown viewer can localize them to the viewer's
|
|
1650
1768
|
* timezone (without losing the canonical instant).
|
|
1651
1769
|
*/
|
|
1770
|
+
/**
|
|
1771
|
+
* The attributes worth printing under a breaching sample.
|
|
1772
|
+
*
|
|
1773
|
+
* A grouped monitor's sample does not carry its group-by values at the
|
|
1774
|
+
* top level: the worker nests the datapoint's whole attribute map under
|
|
1775
|
+
* a single `attributes` key. Printed as-is that became
|
|
1776
|
+
* "`attributes`: `[object Object]`" under every sample — the one line
|
|
1777
|
+
* that should have said which host breached. The series' own label keys
|
|
1778
|
+
* are lifted out of it instead; the rest of the datapoint's attributes
|
|
1779
|
+
* (every resource attribute the exporter stamped) are not what the
|
|
1780
|
+
* reader is looking for here.
|
|
1781
|
+
*/
|
|
1782
|
+
private static flattenSampleAttributes(input: {
|
|
1783
|
+
attributes: JSONObject;
|
|
1784
|
+
seriesLabelKeys: Set<string>;
|
|
1785
|
+
}): Record<string, unknown> {
|
|
1786
|
+
const flattened: Record<string, unknown> = {};
|
|
1787
|
+
|
|
1788
|
+
for (const key of Object.keys(input.attributes)) {
|
|
1789
|
+
const value: unknown = input.attributes[key];
|
|
1790
|
+
|
|
1791
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
1792
|
+
for (const nestedKey of Object.keys(value as JSONObject)) {
|
|
1793
|
+
const nestedValue: unknown = (value as JSONObject)[nestedKey];
|
|
1794
|
+
|
|
1795
|
+
if (
|
|
1796
|
+
input.seriesLabelKeys.has(nestedKey) &&
|
|
1797
|
+
flattened[nestedKey] === undefined &&
|
|
1798
|
+
nestedValue !== undefined &&
|
|
1799
|
+
nestedValue !== null &&
|
|
1800
|
+
typeof nestedValue !== "object"
|
|
1801
|
+
) {
|
|
1802
|
+
flattened[nestedKey] = nestedValue;
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
continue;
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
if (flattened[key] === undefined) {
|
|
1809
|
+
flattened[key] = value;
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
return flattened;
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1652
1816
|
private static formatBreachingSamplesSection(input: {
|
|
1653
1817
|
samples: Array<MetricBreachingSample>;
|
|
1654
1818
|
totalSamples?: number | undefined;
|
|
@@ -1659,6 +1823,12 @@ ${contextBlock}
|
|
|
1659
1823
|
*/
|
|
1660
1824
|
unitHeuristicMetricName: string | undefined;
|
|
1661
1825
|
components: Array<MetricComponent>;
|
|
1826
|
+
/*
|
|
1827
|
+
* The keys that identify the series (its group-by keys). A grouped
|
|
1828
|
+
* sample carries its datapoint's whole attribute map nested under
|
|
1829
|
+
* `attributes`; only these keys are lifted out of it.
|
|
1830
|
+
*/
|
|
1831
|
+
seriesLabelKeys?: Array<string> | undefined;
|
|
1662
1832
|
}): string {
|
|
1663
1833
|
const MAX_SAMPLES_SHOWN: number = 20;
|
|
1664
1834
|
|
|
@@ -1680,9 +1850,22 @@ ${contextBlock}
|
|
|
1680
1850
|
* Collect attribute keys that appear on any displayed sample, so every
|
|
1681
1851
|
* item lists the attributes it has in the same order.
|
|
1682
1852
|
*/
|
|
1853
|
+
const seriesLabelKeys: Set<string> = new Set<string>(
|
|
1854
|
+
input.seriesLabelKeys || [],
|
|
1855
|
+
);
|
|
1856
|
+
const sampleAttributes: Map<
|
|
1857
|
+
MetricBreachingSample,
|
|
1858
|
+
Record<string, unknown>
|
|
1859
|
+
> = new Map();
|
|
1683
1860
|
const attrKeySet: Set<string> = new Set<string>();
|
|
1684
1861
|
for (const s of displayedSamples) {
|
|
1685
|
-
|
|
1862
|
+
const flattened: Record<string, unknown> =
|
|
1863
|
+
MonitorCriteriaEvaluator.flattenSampleAttributes({
|
|
1864
|
+
attributes: s.attributes || {},
|
|
1865
|
+
seriesLabelKeys: seriesLabelKeys,
|
|
1866
|
+
});
|
|
1867
|
+
sampleAttributes.set(s, flattened);
|
|
1868
|
+
for (const k of Object.keys(flattened)) {
|
|
1686
1869
|
attrKeySet.add(k);
|
|
1687
1870
|
}
|
|
1688
1871
|
}
|
|
@@ -1735,7 +1918,7 @@ ${contextBlock}
|
|
|
1735
1918
|
* RootCauseList.code keeps whatever they contain inside the span.
|
|
1736
1919
|
*/
|
|
1737
1920
|
for (const k of attrKeys) {
|
|
1738
|
-
const v: unknown = (s
|
|
1921
|
+
const v: unknown = sampleAttributes.get(s)?.[k];
|
|
1739
1922
|
|
|
1740
1923
|
if (v === undefined || v === null) {
|
|
1741
1924
|
continue;
|
|
@@ -1800,6 +1983,7 @@ ${contextBlock}
|
|
|
1800
1983
|
private static buildMetricExplorerDeepLink(input: {
|
|
1801
1984
|
monitor: Monitor;
|
|
1802
1985
|
ctx: MetricCriteriaContext;
|
|
1986
|
+
monitorStep?: MonitorStep | undefined;
|
|
1803
1987
|
}): string | null {
|
|
1804
1988
|
const projectId: string | undefined = input.monitor.projectId?.toString();
|
|
1805
1989
|
|
|
@@ -1813,27 +1997,56 @@ ${contextBlock}
|
|
|
1813
1997
|
* for its own URL round-trip — so this deep link can never drift
|
|
1814
1998
|
* from what the explorer parses.
|
|
1815
1999
|
*/
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
2000
|
+
let queryConfigs: Array<MetricQueryConfigData> = [
|
|
2001
|
+
{
|
|
2002
|
+
metricQueryData: {
|
|
2003
|
+
filterData: {
|
|
2004
|
+
metricName: input.ctx.metricName,
|
|
2005
|
+
attributes: MetricExplorerUrl.sanitizeAttributes(
|
|
2006
|
+
input.ctx.filterAttributes,
|
|
2007
|
+
),
|
|
2008
|
+
...(input.ctx.aggregationType
|
|
2009
|
+
? { aggegationType: input.ctx.aggregationType }
|
|
2010
|
+
: {}),
|
|
2011
|
+
},
|
|
1826
2012
|
},
|
|
1827
2013
|
},
|
|
1828
|
-
|
|
2014
|
+
];
|
|
2015
|
+
let formulaConfigs: Array<MetricFormulaConfigData> = [];
|
|
2016
|
+
|
|
2017
|
+
/*
|
|
2018
|
+
* A formula's metricName is its EXPRESSION, not a metric the explorer
|
|
2019
|
+
* can chart — linking it as one opened an empty chart. Open the
|
|
2020
|
+
* step's own queries and formulas instead, which is what the formula
|
|
2021
|
+
* was computed from. With no step config to rebuild it from, no link
|
|
2022
|
+
* beats a dead one.
|
|
2023
|
+
*/
|
|
2024
|
+
if (input.ctx.isFormula) {
|
|
2025
|
+
const metricViewConfig: MetricsViewConfig | undefined =
|
|
2026
|
+
MonitorStep.getMetricsViewConfig(input.monitorStep);
|
|
2027
|
+
|
|
2028
|
+
if (
|
|
2029
|
+
!metricViewConfig ||
|
|
2030
|
+
!(metricViewConfig.formulaConfigs || []).some(
|
|
2031
|
+
(formula: MetricFormulaConfigData) => {
|
|
2032
|
+
return formula.metricAliasData?.metricVariable === input.ctx.alias;
|
|
2033
|
+
},
|
|
2034
|
+
)
|
|
2035
|
+
) {
|
|
2036
|
+
return null;
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
queryConfigs = metricViewConfig.queryConfigs || [];
|
|
2040
|
+
formulaConfigs = metricViewConfig.formulaConfigs || [];
|
|
2041
|
+
}
|
|
1829
2042
|
|
|
1830
2043
|
const breachWindow: { startTime: Date; endTime: Date } =
|
|
1831
2044
|
MonitorCriteriaEvaluator.getMetricBreachExplorerWindow(input.ctx);
|
|
1832
2045
|
|
|
1833
2046
|
const urlParams: Dictionary<string> =
|
|
1834
2047
|
MetricExplorerUrl.buildQueryParamsFromMetricViewData({
|
|
1835
|
-
queryConfigs:
|
|
1836
|
-
formulaConfigs:
|
|
2048
|
+
queryConfigs: queryConfigs,
|
|
2049
|
+
formulaConfigs: formulaConfigs,
|
|
1837
2050
|
startAndEndDate: new InBetween(
|
|
1838
2051
|
breachWindow.startTime,
|
|
1839
2052
|
breachWindow.endTime,
|
|
@@ -2018,7 +2231,8 @@ ${contextBlock}
|
|
|
2018
2231
|
}
|
|
2019
2232
|
|
|
2020
2233
|
/**
|
|
2021
|
-
* The unit for a value in a per-platform "Affected Resources" list
|
|
2234
|
+
* The unit for a value in a per-platform "Affected Resources" list built
|
|
2235
|
+
* from the worker's raw datapoint scan.
|
|
2022
2236
|
*
|
|
2023
2237
|
* These rows are NOT the same numbers as the criteria's breaching
|
|
2024
2238
|
* samples: the worker collects them with a raw `MetricService.findBy`
|
|
@@ -2028,349 +2242,1413 @@ ${contextBlock}
|
|
|
2028
2242
|
* query's legendUnit would therefore attach a confidently wrong unit to
|
|
2029
2243
|
* a correct number.
|
|
2030
2244
|
*
|
|
2031
|
-
* The catalog is the
|
|
2245
|
+
* The catalog is the first source precisely because it also declares
|
|
2032
2246
|
* the native unit. It is the same lookup, on the same metric name, that
|
|
2033
2247
|
* the worker already ran to resolve `metricFriendlyName`, so the unit
|
|
2034
2248
|
* shown here can never disagree with the metric name shown beside it.
|
|
2249
|
+
* The unit the worker recorded on the breakdown — the one the exporter
|
|
2250
|
+
* declared — covers every metric the catalog does not know, which is
|
|
2251
|
+
* how k8s.node.allocatable_memory reads "258 GB" instead of the bare
|
|
2252
|
+
* "257760964608" it used to.
|
|
2035
2253
|
*/
|
|
2036
2254
|
private static getPlatformMetricUnit(input: {
|
|
2037
|
-
platform:
|
|
2255
|
+
platform: PlatformName;
|
|
2256
|
+
metricName: string;
|
|
2257
|
+
metricUnit?: string | undefined;
|
|
2258
|
+
}): string | undefined {
|
|
2259
|
+
/*
|
|
2260
|
+
* The vcenter receiver's utilization metrics are already 0–100
|
|
2261
|
+
* percentages (unit "%"), never [0, 1] ratios — the catalog says "%"
|
|
2262
|
+
* and the formatter renders it as-is, so 92.5 reads "92.50%" and
|
|
2263
|
+
* never "9250.00%".
|
|
2264
|
+
*/
|
|
2265
|
+
return (
|
|
2266
|
+
PlatformMetricUnitUtil.getCatalogUnit({
|
|
2267
|
+
platform: input.platform,
|
|
2268
|
+
metricName: input.metricName,
|
|
2269
|
+
}) ||
|
|
2270
|
+
input.metricUnit ||
|
|
2271
|
+
undefined
|
|
2272
|
+
);
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
/**
|
|
2276
|
+
* Render the value of one resource from the worker's raw scan, in the
|
|
2277
|
+
* unit getPlatformMetricUnit resolves. Unemphasised: the list bolds it
|
|
2278
|
+
* (it is the one number on the line a reader scans for), a sentence
|
|
2279
|
+
* may not.
|
|
2280
|
+
*/
|
|
2281
|
+
private static formatPlatformMetricValue(input: {
|
|
2282
|
+
platform: PlatformName;
|
|
2283
|
+
metricName: string;
|
|
2284
|
+
value: number;
|
|
2285
|
+
metricUnit?: string | undefined;
|
|
2286
|
+
}): string {
|
|
2287
|
+
return MetricValueFormatter.format({
|
|
2288
|
+
value: input.value,
|
|
2289
|
+
unit: MonitorCriteriaEvaluator.getPlatformMetricUnit({
|
|
2290
|
+
platform: input.platform,
|
|
2291
|
+
metricName: input.metricName,
|
|
2292
|
+
metricUnit: input.metricUnit,
|
|
2293
|
+
}),
|
|
2294
|
+
metricName: input.metricName,
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
/**
|
|
2299
|
+
* The catalog's human name for a platform metric, or undefined when the
|
|
2300
|
+
* catalog does not know it.
|
|
2301
|
+
*/
|
|
2302
|
+
private static getPlatformMetricFriendlyName(input: {
|
|
2303
|
+
platform: PlatformName;
|
|
2038
2304
|
metricName: string;
|
|
2039
2305
|
}): string | undefined {
|
|
2040
2306
|
switch (input.platform) {
|
|
2041
2307
|
case "kubernetes":
|
|
2042
|
-
return getKubernetesMetricByMetricName(input.metricName)?.
|
|
2308
|
+
return getKubernetesMetricByMetricName(input.metricName)?.friendlyName;
|
|
2043
2309
|
case "proxmox":
|
|
2044
|
-
return getProxmoxMetricByMetricName(input.metricName)?.
|
|
2310
|
+
return getProxmoxMetricByMetricName(input.metricName)?.friendlyName;
|
|
2045
2311
|
case "vmware":
|
|
2046
|
-
|
|
2047
|
-
* The vcenter receiver's utilization metrics are already 0–100
|
|
2048
|
-
* percentages (unit "%"), never [0, 1] ratios — the catalog
|
|
2049
|
-
* says "%" and the formatter renders it as-is, so 92.5 reads
|
|
2050
|
-
* "92.50%" and never "9250.00%".
|
|
2051
|
-
*/
|
|
2052
|
-
return getVMwareMetricByMetricName(input.metricName)?.unit;
|
|
2312
|
+
return getVMwareMetricByMetricName(input.metricName)?.friendlyName;
|
|
2053
2313
|
case "dockerSwarm":
|
|
2054
|
-
return getDockerSwarmMetricByMetricName(input.metricName)?.
|
|
2314
|
+
return getDockerSwarmMetricByMetricName(input.metricName)?.friendlyName;
|
|
2055
2315
|
case "ceph":
|
|
2056
|
-
return getCephMetricByMetricName(input.metricName)?.
|
|
2316
|
+
return getCephMetricByMetricName(input.metricName)?.friendlyName;
|
|
2057
2317
|
default:
|
|
2058
2318
|
return undefined;
|
|
2059
2319
|
}
|
|
2060
2320
|
}
|
|
2061
2321
|
|
|
2062
2322
|
/**
|
|
2063
|
-
*
|
|
2064
|
-
*
|
|
2065
|
-
*
|
|
2066
|
-
*
|
|
2067
|
-
* it is the one number on the line a reader scans for.
|
|
2323
|
+
* "Node Memory Usage (`k8s.node.memory.usage`)", or just the code-quoted
|
|
2324
|
+
* name when there is no friendlier one. The old line always printed
|
|
2325
|
+
* both, which is how the email came to read
|
|
2326
|
+
* "k8s.node.allocatable_memory (`k8s.node.allocatable_memory`)".
|
|
2068
2327
|
*/
|
|
2069
|
-
private static
|
|
2070
|
-
platform: "kubernetes" | "proxmox" | "vmware" | "dockerSwarm" | "ceph";
|
|
2328
|
+
private static describePlatformMetricName(input: {
|
|
2071
2329
|
metricName: string;
|
|
2072
|
-
|
|
2330
|
+
friendlyName?: string | undefined;
|
|
2073
2331
|
}): string {
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
})}**`;
|
|
2332
|
+
const friendlyName: string = (input.friendlyName || "").trim();
|
|
2333
|
+
|
|
2334
|
+
if (!friendlyName || friendlyName === input.metricName) {
|
|
2335
|
+
return `\`${input.metricName}\``;
|
|
2336
|
+
}
|
|
2337
|
+
|
|
2338
|
+
return `${friendlyName} (\`${input.metricName}\`)`;
|
|
2082
2339
|
}
|
|
2083
2340
|
|
|
2084
|
-
|
|
2085
|
-
*
|
|
2086
|
-
*
|
|
2087
|
-
*
|
|
2088
|
-
*
|
|
2089
|
-
* that is the thing that is actually breaching; everything above it in
|
|
2090
|
-
* the hierarchy is listed underneath as context. A series with no
|
|
2091
|
-
* Kubernetes identity at all is a cluster-level series, so it is titled
|
|
2092
|
-
* by the cluster.
|
|
2341
|
+
/**
|
|
2342
|
+
* What the matched criteria actually measured: the query or formula its
|
|
2343
|
+
* metric-value filter points at, resolved the same way
|
|
2344
|
+
* MetricMonitorCriteria resolves it (alias match, falling back to the
|
|
2345
|
+
* first query when the alias names nothing).
|
|
2093
2346
|
*
|
|
2094
|
-
*
|
|
2095
|
-
*
|
|
2096
|
-
*
|
|
2097
|
-
* the pod that tells them apart buried in the bullets.
|
|
2347
|
+
* Returns null when the criteria has no metric-value filter — the
|
|
2348
|
+
* platform context then falls back to describing the breakdown, as it
|
|
2349
|
+
* always has.
|
|
2098
2350
|
*/
|
|
2099
|
-
private static
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
}):
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2351
|
+
private static resolveCriteriaMetricTarget(input: {
|
|
2352
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
2353
|
+
monitorStep: MonitorStep;
|
|
2354
|
+
metricResponse: MetricMonitorResponse;
|
|
2355
|
+
}): CriteriaMetricTarget | null {
|
|
2356
|
+
/*
|
|
2357
|
+
* The step's own config, exactly as MetricMonitorCriteria reads it,
|
|
2358
|
+
* so the target named here is the one the criteria was evaluated on.
|
|
2359
|
+
*/
|
|
2360
|
+
const stepConfig: MetricsViewConfig | undefined =
|
|
2361
|
+
MonitorStep.getMetricsViewConfig(input.monitorStep);
|
|
2362
|
+
const metricViewConfig: MetricsViewConfig | undefined =
|
|
2363
|
+
stepConfig && stepConfig.queryConfigs?.length > 0
|
|
2364
|
+
? stepConfig
|
|
2365
|
+
: input.metricResponse.metricViewConfig || stepConfig;
|
|
2108
2366
|
|
|
2109
|
-
|
|
2110
|
-
|
|
2367
|
+
const metricFilters: Array<CriteriaFilter> = (
|
|
2368
|
+
input.criteriaInstance?.data?.filters || []
|
|
2369
|
+
).filter((f: CriteriaFilter) => {
|
|
2370
|
+
return f.checkOn === CheckOn.MetricValue;
|
|
2371
|
+
});
|
|
2111
2372
|
|
|
2112
|
-
if (
|
|
2113
|
-
|
|
2114
|
-
name = resource.podName
|
|
2115
|
-
? `${AffectedResourceList.code(resource.containerName)} in pod ${AffectedResourceList.code(resource.podName)}`
|
|
2116
|
-
: AffectedResourceList.code(resource.containerName);
|
|
2117
|
-
} else if (resource.podName) {
|
|
2118
|
-
kind = "Pod";
|
|
2119
|
-
name = AffectedResourceList.code(resource.podName);
|
|
2120
|
-
} else if (resource.workloadName) {
|
|
2121
|
-
kind = workloadLabel;
|
|
2122
|
-
name = AffectedResourceList.code(resource.workloadName);
|
|
2123
|
-
} else if (resource.nodeName) {
|
|
2124
|
-
kind = "Node";
|
|
2125
|
-
name = AffectedResourceList.code(resource.nodeName);
|
|
2126
|
-
} else if (resource.namespace) {
|
|
2127
|
-
kind = "Namespace";
|
|
2128
|
-
name = AffectedResourceList.code(resource.namespace);
|
|
2373
|
+
if (metricFilters.length === 0) {
|
|
2374
|
+
return null;
|
|
2129
2375
|
}
|
|
2130
2376
|
|
|
2131
|
-
|
|
2377
|
+
/*
|
|
2378
|
+
* The filter that actually fired, when the criteria has several. An
|
|
2379
|
+
* "Any" criteria (ceph-pg-damaged: PG_DAMAGED > 0 OR OSD_SCRUB_ERRORS
|
|
2380
|
+
* > 0) is described — and its list taken from — whichever of them
|
|
2381
|
+
* breached. Following the first filter instead found no scan for an
|
|
2382
|
+
* inactive PG_DAMAGED check and dropped the list the scrub errors had.
|
|
2383
|
+
* The evaluation leaves each filter's context on it; one with
|
|
2384
|
+
* breaching samples is one that fired.
|
|
2385
|
+
*/
|
|
2386
|
+
const firedFilter: CriteriaFilter | undefined = metricFilters.find(
|
|
2387
|
+
(f: CriteriaFilter) => {
|
|
2388
|
+
const context: MetricCriteriaContext | undefined =
|
|
2389
|
+
f.metricCriteriaContext;
|
|
2132
2390
|
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
}
|
|
2138
|
-
|
|
2391
|
+
return Boolean(
|
|
2392
|
+
(context?.breachingSamples && context.breachingSamples.length > 0) ||
|
|
2393
|
+
context?.breachingSample,
|
|
2394
|
+
);
|
|
2395
|
+
},
|
|
2396
|
+
);
|
|
2139
2397
|
|
|
2140
|
-
const
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2398
|
+
const filter: CriteriaFilter =
|
|
2399
|
+
firedFilter ||
|
|
2400
|
+
metricFilters.find((f: CriteriaFilter) => {
|
|
2401
|
+
return Boolean(f.metricMonitorOptions?.metricAlias);
|
|
2402
|
+
}) ||
|
|
2403
|
+
metricFilters[0]!;
|
|
2404
|
+
|
|
2405
|
+
const thresholdUnit: string | undefined =
|
|
2406
|
+
filter.metricMonitorOptions?.thresholdUnit || undefined;
|
|
2407
|
+
|
|
2408
|
+
const alias: string = filter.metricMonitorOptions?.metricAlias || "";
|
|
2409
|
+
const queryConfigs: Array<MetricQueryConfigData> =
|
|
2410
|
+
metricViewConfig?.queryConfigs || [];
|
|
2411
|
+
const formulaConfigs: Array<MetricFormulaConfigData> =
|
|
2412
|
+
metricViewConfig?.formulaConfigs || [];
|
|
2413
|
+
|
|
2414
|
+
const aliasOf: (config: {
|
|
2415
|
+
metricAliasData?: MetricAliasData | undefined;
|
|
2416
|
+
}) => string = (config: {
|
|
2417
|
+
metricAliasData?: MetricAliasData | undefined;
|
|
2418
|
+
}): string => {
|
|
2419
|
+
return config.metricAliasData?.metricVariable || "";
|
|
2420
|
+
};
|
|
2144
2421
|
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2422
|
+
const query: MetricQueryConfigData | undefined = alias
|
|
2423
|
+
? queryConfigs.find((q: MetricQueryConfigData) => {
|
|
2424
|
+
return aliasOf(q) === alias;
|
|
2425
|
+
})
|
|
2426
|
+
: undefined;
|
|
2427
|
+
|
|
2428
|
+
const formula: MetricFormulaConfigData | undefined =
|
|
2429
|
+
!query && alias
|
|
2430
|
+
? formulaConfigs.find((f: MetricFormulaConfigData) => {
|
|
2431
|
+
return aliasOf(f) === alias;
|
|
2432
|
+
})
|
|
2433
|
+
: undefined;
|
|
2434
|
+
|
|
2435
|
+
if (formula) {
|
|
2436
|
+
const expression: string = formula.metricFormulaData?.metricFormula || "";
|
|
2437
|
+
|
|
2438
|
+
const components: Array<CriteriaMetricTargetComponent> =
|
|
2439
|
+
MetricFormulaEvaluator.getReferencedVariables(expression).map(
|
|
2440
|
+
(variable: string): CriteriaMetricTargetComponent => {
|
|
2441
|
+
const componentQuery: MetricQueryConfigData | undefined =
|
|
2442
|
+
queryConfigs.find((q: MetricQueryConfigData) => {
|
|
2443
|
+
return aliasOf(q).toLowerCase() === variable.toLowerCase();
|
|
2444
|
+
});
|
|
2445
|
+
|
|
2446
|
+
return {
|
|
2447
|
+
alias: componentQuery ? aliasOf(componentQuery) : variable,
|
|
2448
|
+
metricName:
|
|
2449
|
+
(componentQuery?.metricQueryData?.filterData?.metricName as
|
|
2450
|
+
| string
|
|
2451
|
+
| undefined) || undefined,
|
|
2452
|
+
};
|
|
2453
|
+
},
|
|
2454
|
+
);
|
|
2455
|
+
|
|
2456
|
+
return {
|
|
2457
|
+
alias: alias,
|
|
2458
|
+
isFormula: true,
|
|
2459
|
+
comparisonUnit:
|
|
2460
|
+
thresholdUnit || formula.metricAliasData?.legendUnit || undefined,
|
|
2461
|
+
metricName: undefined,
|
|
2462
|
+
displayName: MetricMonitorCriteria.getAliasDisplayName({
|
|
2463
|
+
aliasData: formula.metricAliasData,
|
|
2464
|
+
metricAlias: alias,
|
|
2465
|
+
}),
|
|
2466
|
+
formulaExpression: expression || undefined,
|
|
2467
|
+
components: components,
|
|
2468
|
+
};
|
|
2150
2469
|
}
|
|
2151
2470
|
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2471
|
+
const matchedQuery: MetricQueryConfigData | undefined =
|
|
2472
|
+
query || queryConfigs[0];
|
|
2473
|
+
|
|
2474
|
+
if (!matchedQuery) {
|
|
2475
|
+
return null;
|
|
2157
2476
|
}
|
|
2158
2477
|
|
|
2159
2478
|
return {
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2479
|
+
alias: aliasOf(matchedQuery) || alias,
|
|
2480
|
+
isFormula: false,
|
|
2481
|
+
comparisonUnit:
|
|
2482
|
+
thresholdUnit || matchedQuery.metricAliasData?.legendUnit || undefined,
|
|
2483
|
+
metricName:
|
|
2484
|
+
(matchedQuery.metricQueryData?.filterData?.metricName as
|
|
2485
|
+
| string
|
|
2486
|
+
| undefined) || undefined,
|
|
2487
|
+
displayName: MetricMonitorCriteria.getAliasDisplayName({
|
|
2488
|
+
aliasData: matchedQuery.metricAliasData,
|
|
2489
|
+
metricAlias: aliasOf(matchedQuery) || alias,
|
|
2490
|
+
}),
|
|
2491
|
+
formulaExpression: undefined,
|
|
2492
|
+
components: [],
|
|
2164
2493
|
};
|
|
2165
2494
|
}
|
|
2166
2495
|
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2496
|
+
/**
|
|
2497
|
+
* The metric the platform's root-cause analysis should reason about. For
|
|
2498
|
+
* a plain query that is the query's metric. A formula has no metric name
|
|
2499
|
+
* of its own, so it is analysed as its first query component — the
|
|
2500
|
+
* numerator of every ratio template (k8s.node.memory.usage for
|
|
2501
|
+
* "used ÷ allocatable"), which is what the analysis branches key on.
|
|
2502
|
+
*/
|
|
2503
|
+
private static getAnalysisMetricName(
|
|
2504
|
+
target: CriteriaMetricTarget | null,
|
|
2505
|
+
): string | undefined {
|
|
2506
|
+
if (!target) {
|
|
2507
|
+
return undefined;
|
|
2508
|
+
}
|
|
2178
2509
|
|
|
2179
|
-
if (!
|
|
2180
|
-
return
|
|
2510
|
+
if (!target.isFormula) {
|
|
2511
|
+
return target.metricName;
|
|
2181
2512
|
}
|
|
2182
2513
|
|
|
2183
|
-
|
|
2514
|
+
return target.components.find(
|
|
2515
|
+
(component: CriteriaMetricTargetComponent) => {
|
|
2516
|
+
return Boolean(component.metricName);
|
|
2517
|
+
},
|
|
2518
|
+
)?.metricName;
|
|
2519
|
+
}
|
|
2184
2520
|
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2521
|
+
/**
|
|
2522
|
+
* The "- Metric:" lines of a platform's details block.
|
|
2523
|
+
*
|
|
2524
|
+
* They name what the criteria compared. A formula is named by its
|
|
2525
|
+
* legend and spelled out with the metric behind each variable, because
|
|
2526
|
+
* "Node Memory Utilization (%)" is what the threshold was about and the
|
|
2527
|
+
* two metrics are what an engineer greps for. With no criteria to go on,
|
|
2528
|
+
* the breakdown's own metric is named, as before.
|
|
2529
|
+
*/
|
|
2530
|
+
private static describeCriteriaMetric(input: {
|
|
2531
|
+
platform: PlatformName;
|
|
2532
|
+
target: CriteriaMetricTarget | null;
|
|
2533
|
+
breakdown?: { metricName: string; metricFriendlyName: string } | undefined;
|
|
2534
|
+
fallbackMetricName?: string | undefined;
|
|
2535
|
+
}): Array<string> {
|
|
2536
|
+
const target: CriteriaMetricTarget | null = input.target;
|
|
2537
|
+
|
|
2538
|
+
const describeMetric: (metricName: string) => string = (
|
|
2539
|
+
metricName: string,
|
|
2540
|
+
): string => {
|
|
2541
|
+
const breakdownFriendlyName: string | undefined =
|
|
2542
|
+
input.breakdown?.metricName === metricName
|
|
2543
|
+
? input.breakdown.metricFriendlyName
|
|
2544
|
+
: undefined;
|
|
2545
|
+
|
|
2546
|
+
return MonitorCriteriaEvaluator.describePlatformMetricName({
|
|
2547
|
+
metricName: metricName,
|
|
2548
|
+
friendlyName:
|
|
2549
|
+
breakdownFriendlyName ||
|
|
2550
|
+
MonitorCriteriaEvaluator.getPlatformMetricFriendlyName({
|
|
2551
|
+
platform: input.platform,
|
|
2552
|
+
metricName: metricName,
|
|
2553
|
+
}),
|
|
2554
|
+
});
|
|
2555
|
+
};
|
|
2191
2556
|
|
|
2192
|
-
if (
|
|
2193
|
-
|
|
2194
|
-
`-
|
|
2195
|
-
|
|
2196
|
-
}
|
|
2557
|
+
if (target?.isFormula) {
|
|
2558
|
+
const lines: Array<string> = [
|
|
2559
|
+
`- Metric: ${target.displayName || `\`${target.alias}\``}`,
|
|
2560
|
+
];
|
|
2197
2561
|
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2562
|
+
if (target.formulaExpression) {
|
|
2563
|
+
lines.push(`- Formula: \`${target.formulaExpression}\``);
|
|
2564
|
+
}
|
|
2201
2565
|
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
* healthy node in the analysis below.
|
|
2210
|
-
*/
|
|
2211
|
-
const breach: AffectedResourceBreachPredicate =
|
|
2212
|
-
MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
|
|
2213
|
-
criteriaInstance: input.criteriaInstance,
|
|
2214
|
-
floorEqualityFiresOnFall: true,
|
|
2215
|
-
});
|
|
2566
|
+
for (const component of target.components) {
|
|
2567
|
+
if (component.metricName) {
|
|
2568
|
+
lines.push(
|
|
2569
|
+
` - \`${component.alias}\` = ${describeMetric(component.metricName)}`,
|
|
2570
|
+
);
|
|
2571
|
+
}
|
|
2572
|
+
}
|
|
2216
2573
|
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2574
|
+
return lines;
|
|
2575
|
+
}
|
|
2576
|
+
|
|
2577
|
+
if (target?.metricName) {
|
|
2578
|
+
return [`- Metric: ${describeMetric(target.metricName)}`];
|
|
2579
|
+
}
|
|
2580
|
+
|
|
2581
|
+
if (input.breakdown) {
|
|
2582
|
+
return [
|
|
2583
|
+
`- Metric: ${MonitorCriteriaEvaluator.describePlatformMetricName({
|
|
2584
|
+
metricName: input.breakdown.metricName,
|
|
2585
|
+
friendlyName: input.breakdown.metricFriendlyName,
|
|
2586
|
+
})}`,
|
|
2587
|
+
];
|
|
2588
|
+
}
|
|
2589
|
+
|
|
2590
|
+
if (input.fallbackMetricName) {
|
|
2591
|
+
return [`- Metric: \`${input.fallbackMetricName}\``];
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
return [];
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2597
|
+
/**
|
|
2598
|
+
* Pick the raw-scan breakdown that measured what the criteria compared.
|
|
2599
|
+
*
|
|
2600
|
+
* The worker returns one breakdown per query, tagged with the query's
|
|
2601
|
+
* alias. Only the one scanned for the criteria's own query may be shown:
|
|
2602
|
+
* a sibling query's scan is a different quantity (allocatable bytes
|
|
2603
|
+
* under a utilization threshold), and a formula has no raw scan at all
|
|
2604
|
+
* — its per-resource values only exist per series.
|
|
2605
|
+
*
|
|
2606
|
+
* A breakdown built before the tag existed is trusted the way it always
|
|
2607
|
+
* was, unless the criteria targets a formula.
|
|
2608
|
+
*/
|
|
2609
|
+
private static selectPlatformBreakdown<
|
|
2610
|
+
B extends PlatformResourceBreakdownSource & { metricName: string },
|
|
2611
|
+
>(input: {
|
|
2612
|
+
breakdowns?: Array<B> | undefined;
|
|
2613
|
+
legacyBreakdown?: B | undefined;
|
|
2614
|
+
target: CriteriaMetricTarget | null;
|
|
2615
|
+
}): B | undefined {
|
|
2616
|
+
const candidates: Array<B> =
|
|
2617
|
+
input.breakdowns && input.breakdowns.length > 0
|
|
2618
|
+
? input.breakdowns
|
|
2619
|
+
: input.legacyBreakdown
|
|
2620
|
+
? [input.legacyBreakdown]
|
|
2621
|
+
: [];
|
|
2622
|
+
|
|
2623
|
+
if (candidates.length === 0) {
|
|
2624
|
+
return undefined;
|
|
2625
|
+
}
|
|
2626
|
+
|
|
2627
|
+
const target: CriteriaMetricTarget | null = input.target;
|
|
2628
|
+
|
|
2629
|
+
if (!target) {
|
|
2630
|
+
return candidates.length === 1 ? candidates[0] : undefined;
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
if (target.isFormula) {
|
|
2634
|
+
return undefined;
|
|
2635
|
+
}
|
|
2636
|
+
|
|
2637
|
+
const byAlias: B | undefined = candidates.find((b: B) => {
|
|
2638
|
+
return (
|
|
2639
|
+
Boolean(b.metricAlias) &&
|
|
2640
|
+
b.metricAlias!.toLowerCase() === target.alias.toLowerCase()
|
|
2641
|
+
);
|
|
2642
|
+
});
|
|
2643
|
+
|
|
2644
|
+
if (byAlias) {
|
|
2645
|
+
return byAlias;
|
|
2646
|
+
}
|
|
2647
|
+
|
|
2648
|
+
return candidates.find((b: B) => {
|
|
2649
|
+
return !b.metricAlias;
|
|
2650
|
+
});
|
|
2651
|
+
}
|
|
2652
|
+
|
|
2653
|
+
/**
|
|
2654
|
+
* The "Affected Resources" rows of a GROUPED platform monitor: one per
|
|
2655
|
+
* series that satisfied the matched criteria.
|
|
2656
|
+
*
|
|
2657
|
+
* This is the list the criteria actually produced. Each row is valued
|
|
2658
|
+
* with its series' worst breaching sample, in the same display unit —
|
|
2659
|
+
* and formatted by the same formatter — as the "Filter Conditions Met"
|
|
2660
|
+
* sentence above it, so the node the sentence says "ranged from 86.26%
|
|
2661
|
+
* to 88.82%" is listed at 88.82%, not at its 257760964608 bytes of
|
|
2662
|
+
* allocatable memory. Series that did not breach are not listed at all.
|
|
2663
|
+
*
|
|
2664
|
+
* The resource is named from the series' group-by labels, the same
|
|
2665
|
+
* attribute keys the worker's raw scan reads, then given whatever
|
|
2666
|
+
* context those labels imply but do not carry (a pod's node and
|
|
2667
|
+
* workload) from one of the series' own datapoints.
|
|
2668
|
+
*/
|
|
2669
|
+
private static buildSeriesAffectedRows<I>(input: {
|
|
2670
|
+
perSeriesMatches: Array<PerSeriesCriteriaMatch>;
|
|
2671
|
+
worstIsLowest: boolean;
|
|
2672
|
+
toIdentity: (attributes: JSONObject) => I;
|
|
2673
|
+
withContext: (series: I, context: I) => I;
|
|
2674
|
+
seriesContextAttributes: (fingerprint: string) => Array<JSONObject>;
|
|
2675
|
+
targetAlias?: string | undefined;
|
|
2676
|
+
namesObject: (identity: I) => boolean;
|
|
2677
|
+
}): Array<PlatformAffectedRow<I>> {
|
|
2678
|
+
const targetAlias: string = (input.targetAlias || "").toLowerCase();
|
|
2679
|
+
|
|
2680
|
+
const rows: Array<PlatformAffectedRow<I>> = input.perSeriesMatches.map(
|
|
2681
|
+
(match: PerSeriesCriteriaMatch): PlatformAffectedRow<I> => {
|
|
2682
|
+
/*
|
|
2683
|
+
* Value the row with the filter the list is about. Under an "Any"
|
|
2684
|
+
* criteria a series may have matched on another filter only; its
|
|
2685
|
+
* value is then a different metric in a different unit, so it is
|
|
2686
|
+
* shown with that metric named and never ranked against the rest.
|
|
2687
|
+
*/
|
|
2688
|
+
const matchedContexts: Array<MetricCriteriaContext> =
|
|
2689
|
+
match.metricContexts && match.metricContexts.length > 0
|
|
2690
|
+
? match.metricContexts
|
|
2691
|
+
: match.metricContext
|
|
2692
|
+
? [match.metricContext]
|
|
2693
|
+
: [];
|
|
2694
|
+
|
|
2695
|
+
const targetContext: MetricCriteriaContext | undefined = targetAlias
|
|
2696
|
+
? matchedContexts.find((c: MetricCriteriaContext) => {
|
|
2697
|
+
return (c.alias || "").toLowerCase() === targetAlias;
|
|
2698
|
+
})
|
|
2699
|
+
: matchedContexts[0];
|
|
2700
|
+
|
|
2701
|
+
const context: MetricCriteriaContext | undefined =
|
|
2702
|
+
targetContext || matchedContexts[0];
|
|
2703
|
+
|
|
2704
|
+
const valueNote: string | undefined =
|
|
2705
|
+
!targetContext && context
|
|
2706
|
+
? context.displayName || `\`${context.alias}\``
|
|
2707
|
+
: undefined;
|
|
2708
|
+
|
|
2709
|
+
const seriesIdentity: I = input.toIdentity(
|
|
2710
|
+
PlatformResourceIdentity.withBothAttributeSpellings(
|
|
2711
|
+
match.labels || {},
|
|
2712
|
+
),
|
|
2713
|
+
);
|
|
2714
|
+
|
|
2715
|
+
const sample: MetricBreachingSample | undefined =
|
|
2716
|
+
context?.breachingSamples?.[0] || context?.breachingSample;
|
|
2717
|
+
const sampleAttributes: JSONObject | undefined =
|
|
2718
|
+
MonitorCriteriaEvaluator.getSampleRawAttributes(sample);
|
|
2719
|
+
|
|
2720
|
+
/*
|
|
2721
|
+
* The breaching sample's own datapoint first; then a datapoint of
|
|
2722
|
+
* each of the series' queries, because a FORMULA's sample carries
|
|
2723
|
+
* only the group-by labels — a Proxmox ratio over `id` would
|
|
2724
|
+
* otherwise never learn whether the id is a node or a guest.
|
|
2725
|
+
* with<Platform>Context only ever borrows what the series' labels
|
|
2726
|
+
* guarantee is shared, so a row from any of them is safe.
|
|
2727
|
+
*/
|
|
2728
|
+
const contexts: Array<JSONObject> = [
|
|
2729
|
+
...(sampleAttributes ? [sampleAttributes] : []),
|
|
2730
|
+
...input.seriesContextAttributes(match.fingerprint),
|
|
2731
|
+
];
|
|
2732
|
+
|
|
2733
|
+
const identity: I = contexts.reduce(
|
|
2734
|
+
(current: I, attributes: JSONObject): I => {
|
|
2735
|
+
return input.withContext(
|
|
2736
|
+
current,
|
|
2737
|
+
input.toIdentity(
|
|
2738
|
+
PlatformResourceIdentity.withBothAttributeSpellings(attributes),
|
|
2739
|
+
),
|
|
2740
|
+
);
|
|
2741
|
+
},
|
|
2742
|
+
seriesIdentity,
|
|
2743
|
+
);
|
|
2744
|
+
|
|
2745
|
+
const value: number | null = MonitorCriteriaEvaluator.getWorstValue({
|
|
2746
|
+
context: context,
|
|
2747
|
+
worstIsLowest: input.worstIsLowest,
|
|
2748
|
+
});
|
|
2749
|
+
|
|
2750
|
+
const seriesLabels: JSONObject | undefined = !input.namesObject(
|
|
2751
|
+
identity,
|
|
2752
|
+
)
|
|
2753
|
+
? MonitorCriteriaEvaluator.getNonEmptyLabels(match.labels)
|
|
2754
|
+
: undefined;
|
|
2755
|
+
|
|
2756
|
+
return {
|
|
2757
|
+
identity: identity,
|
|
2758
|
+
value: value,
|
|
2759
|
+
...(valueNote ? { valueNote: valueNote } : {}),
|
|
2760
|
+
...(seriesLabels ? { seriesLabels: seriesLabels } : {}),
|
|
2761
|
+
formattedValue:
|
|
2762
|
+
value === null
|
|
2763
|
+
? "no data"
|
|
2764
|
+
: MetricValueFormatter.format({
|
|
2765
|
+
value: value,
|
|
2766
|
+
unit: context?.unit,
|
|
2767
|
+
metricName:
|
|
2768
|
+
MonitorCriteriaEvaluator.metricNameForUnitHeuristics({
|
|
2769
|
+
metricName: context?.metricName,
|
|
2770
|
+
isFormula: context?.isFormula,
|
|
2771
|
+
}),
|
|
2772
|
+
}),
|
|
2773
|
+
};
|
|
2774
|
+
},
|
|
2775
|
+
);
|
|
2776
|
+
|
|
2777
|
+
return MonitorCriteriaEvaluator.sortAffectedRows({
|
|
2778
|
+
rows: rows,
|
|
2779
|
+
worstIsLowest: input.worstIsLowest,
|
|
2780
|
+
});
|
|
2781
|
+
}
|
|
2782
|
+
|
|
2783
|
+
/**
|
|
2784
|
+
* The "Affected Resources" rows of an UNGROUPED platform monitor, from
|
|
2785
|
+
* the worker's raw datapoint scan of the criteria's own query: every
|
|
2786
|
+
* resource the matched criteria's direction says breached, worst first.
|
|
2787
|
+
*/
|
|
2788
|
+
private static buildScanAffectedRows<
|
|
2789
|
+
R extends { metricValue: number; lowestMetricValue?: number | undefined },
|
|
2790
|
+
>(input: {
|
|
2791
|
+
platform: PlatformName;
|
|
2792
|
+
breakdown: {
|
|
2793
|
+
metricName: string;
|
|
2794
|
+
metricUnit?: string | undefined;
|
|
2795
|
+
affectedResources: Array<R>;
|
|
2796
|
+
};
|
|
2797
|
+
breach: AffectedResourceBreachPredicate;
|
|
2798
|
+
comparisonUnit?: string | undefined;
|
|
2799
|
+
}): Array<PlatformAffectedRow<R>> {
|
|
2800
|
+
/*
|
|
2801
|
+
* The scan's values are in the metric's own unit, but the criteria
|
|
2802
|
+
* compared its threshold in the filter's threshold unit (or the
|
|
2803
|
+
* query's legendUnit). Judge each row in that unit — a "< 1 min"
|
|
2804
|
+
* uptime criteria must not test 30 seconds against 1 — and show it in
|
|
2805
|
+
* its own.
|
|
2806
|
+
*/
|
|
2807
|
+
const valueUnit: string | undefined = PlatformMetricUnitUtil.getMetricUnit({
|
|
2808
|
+
platform: input.platform,
|
|
2809
|
+
metricName: input.breakdown.metricName,
|
|
2810
|
+
declaredUnit: input.breakdown.metricUnit,
|
|
2811
|
+
});
|
|
2812
|
+
|
|
2813
|
+
const toComparisonUnit: (value: number) => number = (
|
|
2814
|
+
value: number,
|
|
2815
|
+
): number => {
|
|
2816
|
+
if (
|
|
2817
|
+
!valueUnit ||
|
|
2818
|
+
!input.comparisonUnit ||
|
|
2819
|
+
valueUnit === input.comparisonUnit
|
|
2820
|
+
) {
|
|
2821
|
+
return value;
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
return MetricUnitUtil.convertToMetricUnit({
|
|
2825
|
+
value: value,
|
|
2826
|
+
fromUnit: valueUnit,
|
|
2827
|
+
metricUnit: input.comparisonUnit,
|
|
2828
|
+
});
|
|
2829
|
+
};
|
|
2830
|
+
|
|
2831
|
+
const collect: (
|
|
2832
|
+
matches: (value: number) => boolean,
|
|
2833
|
+
) => Array<PlatformAffectedRow<R>> = (
|
|
2834
|
+
matches: (value: number) => boolean,
|
|
2835
|
+
): Array<PlatformAffectedRow<R>> => {
|
|
2836
|
+
return MonitorCriteriaEvaluator.collectScanRows<R>({
|
|
2837
|
+
platform: input.platform,
|
|
2838
|
+
breakdown: input.breakdown,
|
|
2839
|
+
breach: input.breach,
|
|
2840
|
+
matches: matches,
|
|
2841
|
+
toComparisonUnit: toComparisonUnit,
|
|
2842
|
+
});
|
|
2843
|
+
};
|
|
2844
|
+
|
|
2845
|
+
let rows: Array<PlatformAffectedRow<R>> = collect(input.breach.matches);
|
|
2846
|
+
|
|
2847
|
+
if (rows.length === 0 && input.breach.fallbackMatches) {
|
|
2848
|
+
rows = collect(input.breach.fallbackMatches);
|
|
2849
|
+
}
|
|
2850
|
+
|
|
2851
|
+
return MonitorCriteriaEvaluator.sortAffectedRows({
|
|
2852
|
+
rows: rows,
|
|
2853
|
+
worstIsLowest: input.breach.worstIsLowest,
|
|
2854
|
+
});
|
|
2855
|
+
}
|
|
2856
|
+
|
|
2857
|
+
/*
|
|
2858
|
+
* One pass of buildScanAffectedRows: the resources `matches` accepts,
|
|
2859
|
+
* each judged in the comparison unit and shown in its own.
|
|
2860
|
+
*/
|
|
2861
|
+
private static collectScanRows<
|
|
2862
|
+
R extends { metricValue: number; lowestMetricValue?: number | undefined },
|
|
2863
|
+
>(input: {
|
|
2864
|
+
platform: PlatformName;
|
|
2865
|
+
breakdown: {
|
|
2866
|
+
metricName: string;
|
|
2867
|
+
metricUnit?: string | undefined;
|
|
2868
|
+
affectedResources: Array<R>;
|
|
2869
|
+
};
|
|
2870
|
+
breach: AffectedResourceBreachPredicate;
|
|
2871
|
+
matches: (value: number) => boolean;
|
|
2872
|
+
toComparisonUnit: (value: number) => number;
|
|
2873
|
+
}): Array<PlatformAffectedRow<R>> {
|
|
2874
|
+
const toComparisonUnit: (value: number) => number = input.toComparisonUnit;
|
|
2875
|
+
const rows: Array<PlatformAffectedRow<R>> = [];
|
|
2876
|
+
|
|
2877
|
+
for (const resource of input.breakdown.affectedResources) {
|
|
2878
|
+
const highest: number = resource.metricValue;
|
|
2879
|
+
const lowest: number = resource.lowestMetricValue ?? resource.metricValue;
|
|
2880
|
+
|
|
2881
|
+
/*
|
|
2882
|
+
* A fall criteria breached on the resource's lowest sample in the
|
|
2883
|
+
* window, not its highest.
|
|
2884
|
+
*/
|
|
2885
|
+
const end: "highest" | "lowest" = input.breach.pickEnd
|
|
2886
|
+
? input.breach.pickEnd({
|
|
2887
|
+
highest: toComparisonUnit(highest),
|
|
2888
|
+
lowest: toComparisonUnit(lowest),
|
|
2230
2889
|
})
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2890
|
+
: input.breach.worstIsLowest
|
|
2891
|
+
? "lowest"
|
|
2892
|
+
: "highest";
|
|
2893
|
+
|
|
2894
|
+
const value: number = end === "lowest" ? lowest : highest;
|
|
2895
|
+
|
|
2896
|
+
if (!input.matches(toComparisonUnit(value))) {
|
|
2897
|
+
continue;
|
|
2898
|
+
}
|
|
2899
|
+
|
|
2900
|
+
rows.push({
|
|
2901
|
+
identity: resource,
|
|
2902
|
+
value: value,
|
|
2903
|
+
formattedValue: MonitorCriteriaEvaluator.formatPlatformMetricValue({
|
|
2904
|
+
platform: input.platform,
|
|
2905
|
+
metricName: input.breakdown.metricName,
|
|
2906
|
+
metricUnit: input.breakdown.metricUnit,
|
|
2907
|
+
value: value,
|
|
2908
|
+
}),
|
|
2909
|
+
});
|
|
2910
|
+
}
|
|
2911
|
+
|
|
2912
|
+
return rows;
|
|
2913
|
+
}
|
|
2914
|
+
|
|
2915
|
+
/*
|
|
2916
|
+
* Worst first. Rows valued by another filter (valueNote) follow, in the
|
|
2917
|
+
* order they came — their numbers are in another unit and cannot be
|
|
2918
|
+
* ranked against these — and a series with no value (a no-data trigger)
|
|
2919
|
+
* comes last.
|
|
2920
|
+
*/
|
|
2921
|
+
private static sortAffectedRows<I>(input: {
|
|
2922
|
+
rows: Array<PlatformAffectedRow<I>>;
|
|
2923
|
+
worstIsLowest: boolean;
|
|
2924
|
+
}): Array<PlatformAffectedRow<I>> {
|
|
2925
|
+
const group: (row: PlatformAffectedRow<I>) => number = (
|
|
2926
|
+
row: PlatformAffectedRow<I>,
|
|
2927
|
+
): number => {
|
|
2928
|
+
if (row.value === null) {
|
|
2929
|
+
return 2;
|
|
2930
|
+
}
|
|
2931
|
+
|
|
2932
|
+
return row.valueNote ? 1 : 0;
|
|
2933
|
+
};
|
|
2934
|
+
|
|
2935
|
+
return [...input.rows].sort(
|
|
2936
|
+
(a: PlatformAffectedRow<I>, b: PlatformAffectedRow<I>): number => {
|
|
2937
|
+
if (group(a) !== group(b)) {
|
|
2938
|
+
return group(a) - group(b);
|
|
2939
|
+
}
|
|
2940
|
+
|
|
2941
|
+
if (group(a) !== 0 || a.value === null || b.value === null) {
|
|
2942
|
+
return 0;
|
|
2943
|
+
}
|
|
2944
|
+
|
|
2945
|
+
return input.worstIsLowest ? a.value - b.value : b.value - a.value;
|
|
2946
|
+
},
|
|
2947
|
+
);
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
/*
|
|
2951
|
+
* The value cell of a platform list entry: bold, and — for a row valued
|
|
2952
|
+
* by another filter than the list's — followed by that filter's metric.
|
|
2953
|
+
*/
|
|
2954
|
+
private static renderAffectedRowValue<I>(
|
|
2955
|
+
row: PlatformAffectedRow<I>,
|
|
2956
|
+
): string {
|
|
2957
|
+
return row.valueNote
|
|
2958
|
+
? `**${row.formattedValue}** (${row.valueNote})`
|
|
2959
|
+
: `**${row.formattedValue}**`;
|
|
2960
|
+
}
|
|
2961
|
+
|
|
2962
|
+
private static getNonEmptyLabels(
|
|
2963
|
+
labels: JSONObject | undefined,
|
|
2964
|
+
): JSONObject | undefined {
|
|
2965
|
+
const result: JSONObject = {};
|
|
2966
|
+
|
|
2967
|
+
for (const key of Object.keys(labels || {})) {
|
|
2968
|
+
const value: unknown = (labels || {})[key];
|
|
2969
|
+
|
|
2970
|
+
if (value !== undefined && value !== null && value !== "") {
|
|
2971
|
+
result[key] = value as JSONObject[string];
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
|
|
2975
|
+
return Object.keys(result).length > 0 ? result : undefined;
|
|
2976
|
+
}
|
|
2977
|
+
|
|
2978
|
+
/*
|
|
2979
|
+
* A list entry for a series whose labels name no platform object: titled
|
|
2980
|
+
* "Series" and named by its labels (their stored `resource.` prefix
|
|
2981
|
+
* dropped), so rows stay distinguishable.
|
|
2982
|
+
*/
|
|
2983
|
+
private static getSeriesLabelEntry(input: {
|
|
2984
|
+
labels: JSONObject;
|
|
2985
|
+
value: string;
|
|
2986
|
+
}): AffectedResourceListEntry {
|
|
2987
|
+
const name: string = Object.keys(input.labels)
|
|
2988
|
+
.map((key: string) => {
|
|
2989
|
+
const displayKey: string = key.startsWith("resource.")
|
|
2990
|
+
? key.substring("resource.".length)
|
|
2991
|
+
: key;
|
|
2992
|
+
|
|
2993
|
+
return AffectedResourceList.code(
|
|
2994
|
+
`${displayKey}=${String(input.labels[key])}`,
|
|
2995
|
+
);
|
|
2996
|
+
})
|
|
2997
|
+
.join(", ");
|
|
2998
|
+
|
|
2999
|
+
return {
|
|
3000
|
+
kind: "Series",
|
|
3001
|
+
name: name,
|
|
3002
|
+
value: input.value,
|
|
3003
|
+
details: [],
|
|
3004
|
+
};
|
|
3005
|
+
}
|
|
3006
|
+
|
|
3007
|
+
/**
|
|
3008
|
+
* A series' worst value in the display unit: its worst breaching sample,
|
|
3009
|
+
* or — when the criteria compared an aggregate no single sample crossed
|
|
3010
|
+
* (a Sum) — its worst sample in the window. Null for a series that
|
|
3011
|
+
* matched with no samples at all (the no-data policy).
|
|
3012
|
+
*/
|
|
3013
|
+
private static getWorstValue(input: {
|
|
3014
|
+
context: MetricCriteriaContext | undefined;
|
|
3015
|
+
worstIsLowest: boolean;
|
|
3016
|
+
}): number | null {
|
|
3017
|
+
const breachingValues: Array<number> = (
|
|
3018
|
+
input.context?.breachingSamples ||
|
|
3019
|
+
(input.context?.breachingSample ? [input.context.breachingSample] : [])
|
|
3020
|
+
)
|
|
3021
|
+
.map((sample: MetricBreachingSample) => {
|
|
3022
|
+
return sample.value;
|
|
3023
|
+
})
|
|
3024
|
+
.filter((value: number) => {
|
|
3025
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
3026
|
+
});
|
|
3027
|
+
|
|
3028
|
+
if (breachingValues.length > 0) {
|
|
3029
|
+
return input.worstIsLowest
|
|
3030
|
+
? Math.min(...breachingValues)
|
|
3031
|
+
: Math.max(...breachingValues);
|
|
3032
|
+
}
|
|
3033
|
+
|
|
3034
|
+
const range: { min: number; max: number } | undefined =
|
|
3035
|
+
input.context?.sampleValueRange;
|
|
3036
|
+
|
|
3037
|
+
if (range) {
|
|
3038
|
+
return input.worstIsLowest ? range.min : range.max;
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
return null;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
/**
|
|
3045
|
+
* One datapoint's attributes from each QUERY slot of a series (the
|
|
3046
|
+
* formula slots after them carry only group-by labels). Per-series query
|
|
3047
|
+
* rows keep their datapoint's full attribute map under `attributes`.
|
|
3048
|
+
*/
|
|
3049
|
+
private static getSeriesQueryRowAttributes(input: {
|
|
3050
|
+
seriesBreakdown: Array<MetricSeriesResult> | undefined;
|
|
3051
|
+
fingerprint: string;
|
|
3052
|
+
queryCount: number;
|
|
3053
|
+
}): Array<JSONObject> {
|
|
3054
|
+
const series: MetricSeriesResult | undefined = (
|
|
3055
|
+
input.seriesBreakdown || []
|
|
3056
|
+
).find((s: MetricSeriesResult) => {
|
|
3057
|
+
return s.fingerprint === input.fingerprint;
|
|
3058
|
+
});
|
|
3059
|
+
|
|
3060
|
+
if (!series) {
|
|
3061
|
+
return [];
|
|
3062
|
+
}
|
|
3063
|
+
|
|
3064
|
+
const result: Array<JSONObject> = [];
|
|
3065
|
+
|
|
3066
|
+
for (let i: number = 0; i < input.queryCount; i++) {
|
|
3067
|
+
const row: JSONObject | undefined = series.aggregatedResults[i]
|
|
3068
|
+
?.data?.[0] as unknown as JSONObject | undefined;
|
|
3069
|
+
const attributes: unknown = row?.["attributes"];
|
|
3070
|
+
|
|
3071
|
+
if (attributes && typeof attributes === "object") {
|
|
3072
|
+
result.push(attributes as JSONObject);
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
|
|
3076
|
+
return result;
|
|
3077
|
+
}
|
|
3078
|
+
|
|
3079
|
+
/**
|
|
3080
|
+
* The raw datapoint attributes a breaching sample carries. A sample from
|
|
3081
|
+
* a per-series query row keeps the datapoint's full attribute map under
|
|
3082
|
+
* `attributes`; a formula row carries only its group attributes there.
|
|
3083
|
+
*/
|
|
3084
|
+
private static getSampleRawAttributes(
|
|
3085
|
+
sample: MetricBreachingSample | undefined,
|
|
3086
|
+
): JSONObject | undefined {
|
|
3087
|
+
if (!sample || !sample.attributes) {
|
|
3088
|
+
return undefined;
|
|
3089
|
+
}
|
|
3090
|
+
|
|
3091
|
+
const nested: unknown = sample.attributes["attributes"];
|
|
3092
|
+
|
|
3093
|
+
if (nested && typeof nested === "object" && !Array.isArray(nested)) {
|
|
3094
|
+
return nested as JSONObject;
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
return sample.attributes;
|
|
3098
|
+
}
|
|
3099
|
+
|
|
3100
|
+
/**
|
|
3101
|
+
* The rows of a platform's "Affected Resources" list for the matched
|
|
3102
|
+
* criteria, from whichever source measured what the criteria compared:
|
|
3103
|
+
* the per-series matches of a grouped monitor, else the raw scan of the
|
|
3104
|
+
* criteria's own query. Null when neither applies (an ungrouped formula
|
|
3105
|
+
* has no per-resource values to list).
|
|
3106
|
+
*/
|
|
3107
|
+
private static getPlatformAffectedRows<
|
|
3108
|
+
I,
|
|
3109
|
+
R extends I & {
|
|
3110
|
+
metricValue: number;
|
|
3111
|
+
lowestMetricValue?: number | undefined;
|
|
3112
|
+
},
|
|
3113
|
+
>(input: {
|
|
3114
|
+
platform: PlatformName;
|
|
3115
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
3116
|
+
breakdown:
|
|
3117
|
+
| {
|
|
3118
|
+
metricName: string;
|
|
3119
|
+
metricUnit?: string | undefined;
|
|
3120
|
+
affectedResources: Array<R>;
|
|
3121
|
+
}
|
|
3122
|
+
| undefined;
|
|
3123
|
+
breach: AffectedResourceBreachPredicate;
|
|
3124
|
+
toIdentity: (attributes: JSONObject) => I;
|
|
3125
|
+
withContext: (series: I, context: I) => I;
|
|
3126
|
+
/*
|
|
3127
|
+
* Whether an identity names a specific object the platform's list can
|
|
3128
|
+
* title a row with. A series whose labels name none — grouped by a PVC
|
|
3129
|
+
* name, or only by a kind (`pve.type`) — is titled by its labels.
|
|
3130
|
+
*/
|
|
3131
|
+
namesObject: (identity: I) => boolean;
|
|
3132
|
+
metricResponse: MetricMonitorResponse;
|
|
3133
|
+
monitorStep: MonitorStep;
|
|
3134
|
+
target: CriteriaMetricTarget | null;
|
|
3135
|
+
}): Array<PlatformAffectedRow<I>> | null {
|
|
3136
|
+
if (input.perSeriesMatches && input.perSeriesMatches.length > 0) {
|
|
3137
|
+
const queryCount: number =
|
|
3138
|
+
MonitorStep.getMetricsViewConfig(input.monitorStep)?.queryConfigs
|
|
3139
|
+
?.length || 0;
|
|
3140
|
+
|
|
3141
|
+
return MonitorCriteriaEvaluator.buildSeriesAffectedRows<I>({
|
|
3142
|
+
perSeriesMatches: input.perSeriesMatches,
|
|
3143
|
+
worstIsLowest: input.breach.worstIsLowest,
|
|
3144
|
+
toIdentity: input.toIdentity,
|
|
3145
|
+
withContext: input.withContext,
|
|
3146
|
+
namesObject: input.namesObject,
|
|
3147
|
+
targetAlias: input.target?.alias,
|
|
3148
|
+
seriesContextAttributes: (fingerprint: string): Array<JSONObject> => {
|
|
3149
|
+
return MonitorCriteriaEvaluator.getSeriesQueryRowAttributes({
|
|
3150
|
+
seriesBreakdown: input.metricResponse.seriesBreakdown,
|
|
3151
|
+
fingerprint: fingerprint,
|
|
3152
|
+
queryCount: queryCount,
|
|
3153
|
+
});
|
|
3154
|
+
},
|
|
3155
|
+
});
|
|
3156
|
+
}
|
|
3157
|
+
|
|
3158
|
+
if (!input.breakdown || input.breakdown.affectedResources.length === 0) {
|
|
3159
|
+
return null;
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
return MonitorCriteriaEvaluator.buildScanAffectedRows<R>({
|
|
3163
|
+
platform: input.platform,
|
|
3164
|
+
breakdown: input.breakdown,
|
|
3165
|
+
breach: input.breach,
|
|
3166
|
+
comparisonUnit: input.target?.comparisonUnit,
|
|
3167
|
+
});
|
|
3168
|
+
}
|
|
3169
|
+
|
|
3170
|
+
/*
|
|
3171
|
+
* One Kubernetes resource as an "Affected Resources" list item.
|
|
3172
|
+
*
|
|
3173
|
+
* The item is titled by the most specific object the series names —
|
|
3174
|
+
* container, then pod, then workload, then node, then namespace — because
|
|
3175
|
+
* that is the thing that is actually breaching; everything above it in
|
|
3176
|
+
* the hierarchy is listed underneath as context. A series with no
|
|
3177
|
+
* Kubernetes identity at all is a cluster-level series, so it is titled
|
|
3178
|
+
* by the cluster.
|
|
3179
|
+
*
|
|
3180
|
+
* A container is titled together with its pod. Every replica of a
|
|
3181
|
+
* workload runs a container of the same name, so "Container `checkout`"
|
|
3182
|
+
* alone would title three crash-looping replicas identically and leave
|
|
3183
|
+
* the pod that tells them apart buried in the bullets.
|
|
3184
|
+
*/
|
|
3185
|
+
private static getKubernetesAffectedResourceEntry(input: {
|
|
3186
|
+
resource: KubernetesResourceIdentity;
|
|
3187
|
+
clusterName: string;
|
|
3188
|
+
value: string;
|
|
3189
|
+
}): AffectedResourceListEntry {
|
|
3190
|
+
const resource: KubernetesResourceIdentity = input.resource;
|
|
3191
|
+
|
|
3192
|
+
// "Deployment", "StatefulSet", ... — the worker only ever sets one of those.
|
|
3193
|
+
const workloadLabel: string = resource.workloadType || "Workload";
|
|
3194
|
+
|
|
3195
|
+
let kind: string = "Cluster";
|
|
3196
|
+
let name: string = AffectedResourceList.code(input.clusterName);
|
|
3197
|
+
|
|
3198
|
+
if (resource.containerName) {
|
|
3199
|
+
kind = "Container";
|
|
3200
|
+
name = resource.podName
|
|
3201
|
+
? `${AffectedResourceList.code(resource.containerName)} in pod ${AffectedResourceList.code(resource.podName)}`
|
|
3202
|
+
: AffectedResourceList.code(resource.containerName);
|
|
3203
|
+
} else if (resource.podName) {
|
|
3204
|
+
kind = "Pod";
|
|
3205
|
+
name = AffectedResourceList.code(resource.podName);
|
|
3206
|
+
} else if (resource.workloadName) {
|
|
3207
|
+
kind = workloadLabel;
|
|
3208
|
+
name = AffectedResourceList.code(resource.workloadName);
|
|
3209
|
+
} else if (resource.nodeName) {
|
|
3210
|
+
kind = "Node";
|
|
3211
|
+
name = AffectedResourceList.code(resource.nodeName);
|
|
3212
|
+
} else if (resource.namespace) {
|
|
3213
|
+
kind = "Namespace";
|
|
3214
|
+
name = AffectedResourceList.code(resource.namespace);
|
|
3215
|
+
}
|
|
3216
|
+
|
|
3217
|
+
const details: Array<AffectedResourceListDetail> = [];
|
|
3218
|
+
|
|
3219
|
+
if (resource.namespace && kind !== "Namespace") {
|
|
3220
|
+
details.push({
|
|
3221
|
+
label: "Namespace",
|
|
3222
|
+
value: AffectedResourceList.code(resource.namespace),
|
|
3223
|
+
});
|
|
3224
|
+
}
|
|
3225
|
+
|
|
3226
|
+
const titledByWorkload: boolean =
|
|
3227
|
+
!resource.containerName &&
|
|
3228
|
+
!resource.podName &&
|
|
3229
|
+
Boolean(resource.workloadName);
|
|
3230
|
+
|
|
3231
|
+
if (resource.workloadName && !titledByWorkload) {
|
|
3232
|
+
details.push({
|
|
3233
|
+
label: workloadLabel,
|
|
3234
|
+
value: AffectedResourceList.code(resource.workloadName),
|
|
3235
|
+
});
|
|
3236
|
+
}
|
|
3237
|
+
|
|
3238
|
+
if (resource.nodeName && kind !== "Node") {
|
|
3239
|
+
details.push({
|
|
3240
|
+
label: "Node",
|
|
3241
|
+
value: AffectedResourceList.code(resource.nodeName),
|
|
3242
|
+
});
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
return {
|
|
3246
|
+
kind: kind,
|
|
3247
|
+
name: name,
|
|
3248
|
+
value: input.value,
|
|
3249
|
+
details: details,
|
|
3250
|
+
};
|
|
3251
|
+
}
|
|
3252
|
+
|
|
3253
|
+
private static async buildKubernetesRootCauseContext(input: {
|
|
3254
|
+
dataToProcess: DataToProcess;
|
|
3255
|
+
monitorStep: MonitorStep;
|
|
3256
|
+
monitor: Monitor;
|
|
3257
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
3258
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
3259
|
+
}): Promise<string | null> {
|
|
3260
|
+
const metricResponse: MetricMonitorResponse =
|
|
3261
|
+
input.dataToProcess as MetricMonitorResponse;
|
|
3262
|
+
|
|
3263
|
+
const target: CriteriaMetricTarget | null =
|
|
3264
|
+
MonitorCriteriaEvaluator.resolveCriteriaMetricTarget({
|
|
3265
|
+
criteriaInstance: input.criteriaInstance,
|
|
3266
|
+
monitorStep: input.monitorStep,
|
|
3267
|
+
metricResponse: metricResponse,
|
|
3268
|
+
});
|
|
3269
|
+
|
|
3270
|
+
const breakdown: KubernetesResourceBreakdown | undefined =
|
|
3271
|
+
MonitorCriteriaEvaluator.selectPlatformBreakdown({
|
|
3272
|
+
breakdowns: metricResponse.kubernetesResourceBreakdowns,
|
|
3273
|
+
legacyBreakdown: metricResponse.kubernetesResourceBreakdown,
|
|
3274
|
+
target: target,
|
|
3275
|
+
});
|
|
3276
|
+
|
|
3277
|
+
const hasSeries: boolean = Boolean(
|
|
3278
|
+
input.perSeriesMatches && input.perSeriesMatches.length > 0,
|
|
3279
|
+
);
|
|
3280
|
+
|
|
3281
|
+
/*
|
|
3282
|
+
* Nothing to say only when there is neither a list nor a monitor to
|
|
3283
|
+
* describe. An ungrouped formula criteria has no raw scan of its own
|
|
3284
|
+
* and no series, but its cluster and formula are still worth naming.
|
|
3285
|
+
*/
|
|
3286
|
+
if (
|
|
3287
|
+
!breakdown &&
|
|
3288
|
+
!hasSeries &&
|
|
3289
|
+
!input.monitorStep.data?.kubernetesMonitor
|
|
3290
|
+
) {
|
|
3291
|
+
return null;
|
|
3292
|
+
}
|
|
3293
|
+
|
|
3294
|
+
const clusterName: string =
|
|
3295
|
+
breakdown?.clusterName ||
|
|
3296
|
+
input.monitorStep.data?.kubernetesMonitor?.clusterIdentifier ||
|
|
3297
|
+
"Unknown";
|
|
3298
|
+
|
|
3299
|
+
/*
|
|
3300
|
+
* The metric the analysis below reasons about: the criteria's own
|
|
3301
|
+
* query, or a formula's numerator. Without a criteria to go on, the
|
|
3302
|
+
* breakdown's metric, as before.
|
|
3303
|
+
*/
|
|
3304
|
+
const analysisMetricName: string =
|
|
3305
|
+
MonitorCriteriaEvaluator.getAnalysisMetricName(target) ||
|
|
3306
|
+
breakdown?.metricName ||
|
|
3307
|
+
"";
|
|
3308
|
+
|
|
3309
|
+
const sections: Array<string> = [];
|
|
3310
|
+
|
|
3311
|
+
// Cluster context
|
|
3312
|
+
const clusterDetails: Array<string> = [];
|
|
3313
|
+
clusterDetails.push(`- Cluster: ${clusterName}`);
|
|
3314
|
+
clusterDetails.push(
|
|
3315
|
+
...MonitorCriteriaEvaluator.describeCriteriaMetric({
|
|
3316
|
+
platform: "kubernetes",
|
|
3317
|
+
target: target,
|
|
3318
|
+
breakdown: breakdown,
|
|
3319
|
+
}),
|
|
3320
|
+
);
|
|
3321
|
+
|
|
3322
|
+
/*
|
|
3323
|
+
* The worker records the query's attribute filters under the stored,
|
|
3324
|
+
* `resource.`-prefixed key; the bare key never occurs, which is why
|
|
3325
|
+
* this line never rendered.
|
|
3326
|
+
*/
|
|
3327
|
+
const namespaceFilter: string | undefined =
|
|
3328
|
+
breakdown?.attributes["resource.k8s.namespace.name"] ||
|
|
3329
|
+
breakdown?.attributes["k8s.namespace.name"] ||
|
|
3330
|
+
input.monitorStep.data?.kubernetesMonitor?.resourceFilters?.namespace;
|
|
3331
|
+
|
|
3332
|
+
if (namespaceFilter) {
|
|
3333
|
+
clusterDetails.push(`- Namespace: ${namespaceFilter}`);
|
|
3334
|
+
}
|
|
3335
|
+
|
|
3336
|
+
sections.push(
|
|
3337
|
+
`**Kubernetes Cluster Details**\n${clusterDetails.join("\n")}`,
|
|
3338
|
+
);
|
|
3339
|
+
|
|
3340
|
+
/*
|
|
3341
|
+
* Keep the rows that satisfy the criteria that just matched, worst
|
|
3342
|
+
* first — NOT simply the non-zero rows. k8s-node-not-ready fires on
|
|
3343
|
+
* `k8s.node.condition_ready = 0`, so the zero rows are the NotReady
|
|
3344
|
+
* nodes; the old hardcoded `> 0` dropped exactly those and named a
|
|
3345
|
+
* healthy node in the analysis below.
|
|
3346
|
+
*/
|
|
3347
|
+
const breach: AffectedResourceBreachPredicate =
|
|
3348
|
+
MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
|
|
3349
|
+
criteriaInstance: input.criteriaInstance,
|
|
3350
|
+
floorEqualityFiresOnFall: true,
|
|
3351
|
+
});
|
|
3352
|
+
|
|
3353
|
+
const rows: Array<PlatformAffectedRow<KubernetesResourceIdentity>> | null =
|
|
3354
|
+
MonitorCriteriaEvaluator.getPlatformAffectedRows<
|
|
3355
|
+
KubernetesResourceIdentity,
|
|
3356
|
+
KubernetesAffectedResource
|
|
3357
|
+
>({
|
|
3358
|
+
platform: "kubernetes",
|
|
3359
|
+
perSeriesMatches: input.perSeriesMatches,
|
|
3360
|
+
namesObject: (identity: KubernetesResourceIdentity): boolean => {
|
|
3361
|
+
return Boolean(
|
|
3362
|
+
identity.containerName ||
|
|
3363
|
+
identity.podName ||
|
|
3364
|
+
identity.workloadName ||
|
|
3365
|
+
identity.nodeName ||
|
|
3366
|
+
identity.namespace,
|
|
2237
3367
|
);
|
|
3368
|
+
},
|
|
3369
|
+
metricResponse: metricResponse,
|
|
3370
|
+
monitorStep: input.monitorStep,
|
|
3371
|
+
target: target,
|
|
3372
|
+
breakdown: breakdown,
|
|
3373
|
+
breach: breach,
|
|
3374
|
+
toIdentity: PlatformResourceIdentity.kubernetes,
|
|
3375
|
+
withContext: PlatformResourceIdentity.withKubernetesContext,
|
|
3376
|
+
});
|
|
3377
|
+
|
|
3378
|
+
if (!rows || rows.length === 0) {
|
|
3379
|
+
return sections.join("\n");
|
|
3380
|
+
}
|
|
3381
|
+
|
|
3382
|
+
// Show top 10 affected resources
|
|
3383
|
+
const rowsToShow: Array<PlatformAffectedRow<KubernetesResourceIdentity>> =
|
|
3384
|
+
rows.slice(0, AffectedResourceList.MAX_ENTRIES);
|
|
3385
|
+
|
|
3386
|
+
sections.push(
|
|
3387
|
+
AffectedResourceList.render({
|
|
3388
|
+
heading: "Affected Resources",
|
|
3389
|
+
overflowNoun: "affected resources",
|
|
3390
|
+
totalCount: rows.length,
|
|
3391
|
+
entries: rowsToShow.map(
|
|
3392
|
+
(
|
|
3393
|
+
row: PlatformAffectedRow<KubernetesResourceIdentity>,
|
|
3394
|
+
): AffectedResourceListEntry => {
|
|
3395
|
+
return row.seriesLabels
|
|
3396
|
+
? MonitorCriteriaEvaluator.getSeriesLabelEntry({
|
|
3397
|
+
labels: row.seriesLabels,
|
|
3398
|
+
value: MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
3399
|
+
})
|
|
3400
|
+
: MonitorCriteriaEvaluator.getKubernetesAffectedResourceEntry({
|
|
3401
|
+
resource: row.identity,
|
|
3402
|
+
clusterName: clusterName,
|
|
3403
|
+
value: MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
3404
|
+
});
|
|
3405
|
+
},
|
|
3406
|
+
),
|
|
3407
|
+
}),
|
|
3408
|
+
);
|
|
3409
|
+
|
|
3410
|
+
const topRow: PlatformAffectedRow<KubernetesResourceIdentity> =
|
|
3411
|
+
rowsToShow[0]!;
|
|
3412
|
+
const topResource: KubernetesAffectedResource = {
|
|
3413
|
+
...topRow.identity,
|
|
3414
|
+
metricValue: topRow.value ?? 0,
|
|
3415
|
+
};
|
|
3416
|
+
|
|
3417
|
+
/*
|
|
3418
|
+
* Add root cause analysis based on metric type — unless the worst row
|
|
3419
|
+
* is a series that matched with no data at all, or one valued by a
|
|
3420
|
+
* different filter than the one analysed, which no sentence below
|
|
3421
|
+
* ("memory usage is at ...") can describe.
|
|
3422
|
+
*/
|
|
3423
|
+
const analysis: string | null =
|
|
3424
|
+
topRow.value === null || topRow.valueNote
|
|
3425
|
+
? null
|
|
3426
|
+
: MonitorCriteriaEvaluator.buildKubernetesRootCauseAnalysis({
|
|
3427
|
+
breakdown: {
|
|
3428
|
+
clusterName: clusterName,
|
|
3429
|
+
metricName: analysisMetricName,
|
|
3430
|
+
metricFriendlyName:
|
|
3431
|
+
(target?.isFormula ? target.displayName : undefined) ||
|
|
3432
|
+
(breakdown?.metricName === analysisMetricName
|
|
3433
|
+
? breakdown.metricFriendlyName
|
|
3434
|
+
: undefined) ||
|
|
3435
|
+
MonitorCriteriaEvaluator.getPlatformMetricFriendlyName({
|
|
3436
|
+
platform: "kubernetes",
|
|
3437
|
+
metricName: analysisMetricName,
|
|
3438
|
+
}) ||
|
|
3439
|
+
analysisMetricName,
|
|
3440
|
+
affectedResources: [],
|
|
3441
|
+
attributes: breakdown?.attributes || {},
|
|
3442
|
+
metricUnit: breakdown?.metricUnit,
|
|
3443
|
+
},
|
|
3444
|
+
topResource: topResource,
|
|
3445
|
+
topResourceValue: topRow.formattedValue,
|
|
3446
|
+
target: target,
|
|
3447
|
+
});
|
|
3448
|
+
|
|
3449
|
+
if (analysis) {
|
|
3450
|
+
sections.push(`\n\n**Root Cause Analysis**\n${analysis}`);
|
|
3451
|
+
}
|
|
3452
|
+
|
|
3453
|
+
// Fetch recent container logs for the top affected resource during CrashLoopBackOff
|
|
3454
|
+
if (
|
|
3455
|
+
(analysisMetricName === "k8s.container.restarts" ||
|
|
3456
|
+
analysisMetricName.includes("restart")) &&
|
|
3457
|
+
input.monitor.projectId
|
|
3458
|
+
) {
|
|
3459
|
+
try {
|
|
3460
|
+
const logAttributes: Record<string, string> = {};
|
|
3461
|
+
|
|
3462
|
+
if (clusterName) {
|
|
3463
|
+
logAttributes["resource.k8s.cluster.name"] = clusterName;
|
|
3464
|
+
}
|
|
3465
|
+
|
|
3466
|
+
if (topResource.podName) {
|
|
3467
|
+
logAttributes["resource.k8s.pod.name"] = topResource.podName;
|
|
3468
|
+
}
|
|
3469
|
+
|
|
3470
|
+
if (topResource.containerName) {
|
|
3471
|
+
logAttributes["resource.k8s.container.name"] =
|
|
3472
|
+
topResource.containerName;
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3475
|
+
if (topResource.namespace) {
|
|
3476
|
+
logAttributes["resource.k8s.namespace.name"] = topResource.namespace;
|
|
3477
|
+
}
|
|
3478
|
+
|
|
3479
|
+
const now: Date = OneUptimeDate.getCurrentDate();
|
|
3480
|
+
const fifteenMinutesAgo: Date = OneUptimeDate.addRemoveMinutes(
|
|
3481
|
+
now,
|
|
3482
|
+
-15,
|
|
3483
|
+
);
|
|
3484
|
+
|
|
3485
|
+
const logs: Array<JSONObject> =
|
|
3486
|
+
await LogAggregationService.getExportLogs({
|
|
3487
|
+
projectId: input.monitor.projectId,
|
|
3488
|
+
startTime: fifteenMinutesAgo,
|
|
3489
|
+
endTime: now,
|
|
3490
|
+
limit: 50,
|
|
3491
|
+
attributes: logAttributes,
|
|
3492
|
+
});
|
|
3493
|
+
|
|
3494
|
+
if (logs.length > 0) {
|
|
3495
|
+
const logLines: Array<string> = logs.map((log: JSONObject) => {
|
|
3496
|
+
const timestamp: string = log["time"] ? String(log["time"]) : "";
|
|
3497
|
+
const severity: string = log["severityText"]
|
|
3498
|
+
? String(log["severityText"])
|
|
3499
|
+
: "INFO";
|
|
3500
|
+
const body: string = log["body"] ? String(log["body"]) : "";
|
|
3501
|
+
return `\`${timestamp}\` **${severity}** ${body}`;
|
|
3502
|
+
});
|
|
2238
3503
|
|
|
2239
|
-
|
|
2240
|
-
|
|
3504
|
+
sections.push(
|
|
3505
|
+
`\n\n**Recent Container Logs** (${topResource.podName || "unknown pod"} / ${topResource.containerName || "unknown container"}, last 15 minutes)\n\n${logLines.join("\n\n")}`,
|
|
3506
|
+
);
|
|
3507
|
+
}
|
|
3508
|
+
} catch (err) {
|
|
3509
|
+
const k8sLogAttributes: LogAttributes = {
|
|
3510
|
+
projectId: input.monitor.projectId?.toString(),
|
|
3511
|
+
};
|
|
3512
|
+
logger.error(
|
|
3513
|
+
"Failed to fetch container logs for root cause context",
|
|
3514
|
+
k8sLogAttributes,
|
|
3515
|
+
);
|
|
3516
|
+
logger.error(err, k8sLogAttributes);
|
|
2241
3517
|
}
|
|
3518
|
+
}
|
|
2242
3519
|
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
sortedResources.slice(0, AffectedResourceList.MAX_ENTRIES);
|
|
3520
|
+
return sections.join("\n");
|
|
3521
|
+
}
|
|
2246
3522
|
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
);
|
|
3523
|
+
/**
|
|
3524
|
+
* The "- Metric:" line of a telemetry resource's details block: what
|
|
3525
|
+
* the matched criteria actually compared.
|
|
3526
|
+
*
|
|
3527
|
+
* It used to name `queryConfigs[0]` unconditionally, so a criteria on a
|
|
3528
|
+
* formula — Host CPU busy `(host_cpu_user + host_cpu_system) * 100`,
|
|
3529
|
+
* Docker restarts `max - min` — was reported as its first operand. The
|
|
3530
|
+
* metric context MetricMonitorCriteria records at evaluation time
|
|
3531
|
+
* already says whether the criteria resolved to a query or a formula
|
|
3532
|
+
* (including its fall back to the first query when the alias names
|
|
3533
|
+
* nothing), so the line is read from there.
|
|
3534
|
+
*
|
|
3535
|
+
* A formula is named by its legend, which is what the threshold was
|
|
3536
|
+
* written against ("CPU Busy (%)"); its expression and the metric behind
|
|
3537
|
+
* each variable are spelled out in the Metric Details below. With no
|
|
3538
|
+
* metric context — a criteria with no metric-value filter — the step's
|
|
3539
|
+
* first query is named, as before.
|
|
3540
|
+
*/
|
|
3541
|
+
private static describeCriteriaMetricLine(input: {
|
|
3542
|
+
ctx: MetricCriteriaContext | undefined;
|
|
3543
|
+
metricViewConfig: MetricsViewConfig | undefined;
|
|
3544
|
+
}): string | null {
|
|
3545
|
+
const ctx: MetricCriteriaContext | undefined = input.ctx;
|
|
2271
3546
|
|
|
2272
|
-
|
|
2273
|
-
const
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
3547
|
+
if (ctx?.isFormula) {
|
|
3548
|
+
const formula: MetricFormulaConfigData | undefined = (
|
|
3549
|
+
input.metricViewConfig?.formulaConfigs || []
|
|
3550
|
+
).find((f: MetricFormulaConfigData) => {
|
|
3551
|
+
return f.metricAliasData?.metricVariable === ctx.alias;
|
|
3552
|
+
});
|
|
2278
3553
|
|
|
2279
|
-
|
|
2280
|
-
|
|
3554
|
+
const legend: string = (
|
|
3555
|
+
formula?.metricAliasData?.legend ||
|
|
3556
|
+
formula?.metricAliasData?.title ||
|
|
3557
|
+
""
|
|
3558
|
+
).trim();
|
|
3559
|
+
|
|
3560
|
+
if (legend && legend !== ctx.alias) {
|
|
3561
|
+
return `- Metric: ${legend}`;
|
|
2281
3562
|
}
|
|
2282
3563
|
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
(breakdown.metricName === "k8s.container.restarts" ||
|
|
2286
|
-
breakdown.metricName.includes("restart")) &&
|
|
2287
|
-
input.monitor.projectId
|
|
2288
|
-
) {
|
|
2289
|
-
const topResource: KubernetesAffectedResource = resourcesToShow[0]!;
|
|
3564
|
+
return `- Metric: \`${ctx.alias}\` (formula)`;
|
|
3565
|
+
}
|
|
2290
3566
|
|
|
2291
|
-
|
|
2292
|
-
|
|
3567
|
+
if (ctx?.metricName) {
|
|
3568
|
+
return `- Metric: \`${ctx.metricName}\``;
|
|
3569
|
+
}
|
|
2293
3570
|
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
3571
|
+
const firstQueryMetricName: string | undefined = input.metricViewConfig
|
|
3572
|
+
?.queryConfigs?.[0]?.metricQueryData?.filterData?.metricName as
|
|
3573
|
+
| string
|
|
3574
|
+
| undefined;
|
|
2297
3575
|
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
3576
|
+
return firstQueryMetricName
|
|
3577
|
+
? `- Metric: \`${firstQueryMetricName}\``
|
|
3578
|
+
: null;
|
|
3579
|
+
}
|
|
2301
3580
|
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
3581
|
+
/**
|
|
3582
|
+
* Root-cause context for a telemetry monitor that watches one host,
|
|
3583
|
+
* container set or device fleet: the resource it is scoped to, then the
|
|
3584
|
+
* same Metric Details and Breaching Samples a Metrics monitor gets.
|
|
3585
|
+
*
|
|
3586
|
+
* These monitors used to follow the identity lines with a "Metric
|
|
3587
|
+
* Summary" that only counted the data points in each result — one line
|
|
3588
|
+
* per query AND per formula, so the Host CPU template printed three
|
|
3589
|
+
* identical "- 5 metric data point(s) returned" lines and never the
|
|
3590
|
+
* value that breached. MetricMonitorCriteria fills the filter's metric
|
|
3591
|
+
* context for every metric-backed monitor type, so the unit, formula,
|
|
3592
|
+
* components and unit-formatted samples were there all along.
|
|
3593
|
+
*
|
|
3594
|
+
* `identityLines` is null when the step carries no config for the
|
|
3595
|
+
* monitor type; the metric details are still rendered from the context.
|
|
3596
|
+
*/
|
|
3597
|
+
private static buildTelemetryResourceRootCauseContext(input: {
|
|
3598
|
+
heading: string;
|
|
3599
|
+
identityLines: Array<string> | null;
|
|
3600
|
+
monitor: Monitor;
|
|
3601
|
+
monitorStep: MonitorStep;
|
|
3602
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
3603
|
+
}): string | null {
|
|
3604
|
+
const sections: Array<string> = [];
|
|
2306
3605
|
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
3606
|
+
if (input.identityLines) {
|
|
3607
|
+
const metricLine: string | null =
|
|
3608
|
+
MonitorCriteriaEvaluator.describeCriteriaMetricLine({
|
|
3609
|
+
ctx: MonitorCriteriaEvaluator.getMetricCriteriaContext(
|
|
3610
|
+
input.criteriaInstance,
|
|
3611
|
+
),
|
|
3612
|
+
metricViewConfig: MonitorStep.getMetricsViewConfig(input.monitorStep),
|
|
3613
|
+
});
|
|
2311
3614
|
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
-15,
|
|
2316
|
-
);
|
|
3615
|
+
const lines: Array<string> = metricLine
|
|
3616
|
+
? [...input.identityLines, metricLine]
|
|
3617
|
+
: input.identityLines;
|
|
2317
3618
|
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
projectId: input.monitor.projectId,
|
|
2321
|
-
startTime: fifteenMinutesAgo,
|
|
2322
|
-
endTime: now,
|
|
2323
|
-
limit: 50,
|
|
2324
|
-
attributes: logAttributes,
|
|
2325
|
-
});
|
|
3619
|
+
sections.push(`**${input.heading}**\n${lines.join("\n")}`);
|
|
3620
|
+
}
|
|
2326
3621
|
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
return `\`${timestamp}\` **${severity}** ${body}`;
|
|
2335
|
-
});
|
|
3622
|
+
const metricDetails: string | null = input.criteriaInstance
|
|
3623
|
+
? MonitorCriteriaEvaluator.buildMetricRootCauseContext({
|
|
3624
|
+
criteriaInstance: input.criteriaInstance,
|
|
3625
|
+
monitor: input.monitor,
|
|
3626
|
+
monitorStep: input.monitorStep,
|
|
3627
|
+
})
|
|
3628
|
+
: null;
|
|
2336
3629
|
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
} catch (err) {
|
|
2342
|
-
const k8sLogAttributes: LogAttributes = {
|
|
2343
|
-
projectId: input.monitor.projectId?.toString(),
|
|
2344
|
-
};
|
|
2345
|
-
logger.error(
|
|
2346
|
-
"Failed to fetch container logs for root cause context",
|
|
2347
|
-
k8sLogAttributes,
|
|
2348
|
-
);
|
|
2349
|
-
logger.error(err, k8sLogAttributes);
|
|
2350
|
-
}
|
|
2351
|
-
}
|
|
3630
|
+
if (metricDetails) {
|
|
3631
|
+
sections.push(
|
|
3632
|
+
sections.length > 0 ? `\n\n${metricDetails}` : metricDetails,
|
|
3633
|
+
);
|
|
2352
3634
|
}
|
|
2353
3635
|
|
|
2354
|
-
return sections.join("\n");
|
|
3636
|
+
return sections.length > 0 ? sections.join("\n") : null;
|
|
2355
3637
|
}
|
|
2356
3638
|
|
|
2357
3639
|
private static buildDockerRootCauseContext(input: {
|
|
2358
|
-
dataToProcess: DataToProcess;
|
|
2359
3640
|
monitorStep: MonitorStep;
|
|
2360
3641
|
monitor: Monitor;
|
|
3642
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
2361
3643
|
}): string | null {
|
|
2362
|
-
const metricResponse: MetricMonitorResponse =
|
|
2363
|
-
input.dataToProcess as MetricMonitorResponse;
|
|
2364
|
-
|
|
2365
|
-
const sections: Array<string> = [];
|
|
2366
|
-
|
|
2367
3644
|
// Docker host context
|
|
2368
3645
|
const dockerMonitor: MonitorStepDockerMonitor | undefined =
|
|
2369
3646
|
input.monitorStep.data?.dockerMonitor;
|
|
2370
3647
|
|
|
3648
|
+
let hostDetails: Array<string> | null = null;
|
|
3649
|
+
|
|
2371
3650
|
if (dockerMonitor) {
|
|
2372
|
-
|
|
2373
|
-
hostDetails.push(`- Host: ${dockerMonitor.hostIdentifier || "Unknown"}`);
|
|
3651
|
+
hostDetails = [`- Host: ${dockerMonitor.hostIdentifier || "Unknown"}`];
|
|
2374
3652
|
|
|
2375
3653
|
if (dockerMonitor.containerFilters?.containerName) {
|
|
2376
3654
|
hostDetails.push(
|
|
@@ -2383,112 +3661,50 @@ ${contextBlock}
|
|
|
2383
3661
|
`- Container Image Filter: ${dockerMonitor.containerFilters.containerImage}`,
|
|
2384
3662
|
);
|
|
2385
3663
|
}
|
|
2386
|
-
|
|
2387
|
-
// Add metric name from the query config
|
|
2388
|
-
if (
|
|
2389
|
-
dockerMonitor.metricViewConfig?.queryConfigs?.length > 0 &&
|
|
2390
|
-
dockerMonitor.metricViewConfig.queryConfigs[0]
|
|
2391
|
-
) {
|
|
2392
|
-
const metricName: string = dockerMonitor.metricViewConfig
|
|
2393
|
-
.queryConfigs[0].metricQueryData?.filterData?.metricName as string;
|
|
2394
|
-
if (metricName) {
|
|
2395
|
-
hostDetails.push(`- Metric: \`${metricName}\``);
|
|
2396
|
-
}
|
|
2397
|
-
}
|
|
2398
|
-
|
|
2399
|
-
sections.push(`**Docker Host Details**\n${hostDetails.join("\n")}`);
|
|
2400
|
-
}
|
|
2401
|
-
|
|
2402
|
-
// Metric results summary
|
|
2403
|
-
if (metricResponse.metricResult && metricResponse.metricResult.length > 0) {
|
|
2404
|
-
const resultDetails: Array<string> = [];
|
|
2405
|
-
|
|
2406
|
-
for (const result of metricResponse.metricResult) {
|
|
2407
|
-
if (result.data && result.data.length > 0) {
|
|
2408
|
-
resultDetails.push(
|
|
2409
|
-
`- ${result.data.length} metric data point(s) returned`,
|
|
2410
|
-
);
|
|
2411
|
-
}
|
|
2412
|
-
}
|
|
2413
|
-
|
|
2414
|
-
if (resultDetails.length > 0) {
|
|
2415
|
-
sections.push(`\n\n**Metric Summary**\n${resultDetails.join("\n")}`);
|
|
2416
|
-
}
|
|
2417
3664
|
}
|
|
2418
3665
|
|
|
2419
|
-
return
|
|
3666
|
+
return MonitorCriteriaEvaluator.buildTelemetryResourceRootCauseContext({
|
|
3667
|
+
heading: "Docker Host Details",
|
|
3668
|
+
identityLines: hostDetails,
|
|
3669
|
+
monitor: input.monitor,
|
|
3670
|
+
monitorStep: input.monitorStep,
|
|
3671
|
+
criteriaInstance: input.criteriaInstance,
|
|
3672
|
+
});
|
|
2420
3673
|
}
|
|
2421
3674
|
|
|
2422
3675
|
private static buildHostRootCauseContext(input: {
|
|
2423
|
-
dataToProcess: DataToProcess;
|
|
2424
3676
|
monitorStep: MonitorStep;
|
|
2425
3677
|
monitor: Monitor;
|
|
3678
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
2426
3679
|
}): string | null {
|
|
2427
|
-
const metricResponse: MetricMonitorResponse =
|
|
2428
|
-
input.dataToProcess as MetricMonitorResponse;
|
|
2429
|
-
|
|
2430
|
-
const sections: Array<string> = [];
|
|
2431
|
-
|
|
2432
3680
|
// Host context
|
|
2433
3681
|
const hostMonitor: MonitorStepHostMonitor | undefined =
|
|
2434
3682
|
input.monitorStep.data?.hostMonitor;
|
|
2435
3683
|
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
const metricName: string = hostMonitor.metricViewConfig.queryConfigs[0]
|
|
2446
|
-
.metricQueryData?.filterData?.metricName as string;
|
|
2447
|
-
if (metricName) {
|
|
2448
|
-
hostDetails.push(`- Metric: \`${metricName}\``);
|
|
2449
|
-
}
|
|
2450
|
-
}
|
|
2451
|
-
|
|
2452
|
-
sections.push(`**Host Details**\n${hostDetails.join("\n")}`);
|
|
2453
|
-
}
|
|
2454
|
-
|
|
2455
|
-
// Metric results summary
|
|
2456
|
-
if (metricResponse.metricResult && metricResponse.metricResult.length > 0) {
|
|
2457
|
-
const resultDetails: Array<string> = [];
|
|
2458
|
-
|
|
2459
|
-
for (const result of metricResponse.metricResult) {
|
|
2460
|
-
if (result.data && result.data.length > 0) {
|
|
2461
|
-
resultDetails.push(
|
|
2462
|
-
`- ${result.data.length} metric data point(s) returned`,
|
|
2463
|
-
);
|
|
2464
|
-
}
|
|
2465
|
-
}
|
|
2466
|
-
|
|
2467
|
-
if (resultDetails.length > 0) {
|
|
2468
|
-
sections.push(`\n\n**Metric Summary**\n${resultDetails.join("\n")}`);
|
|
2469
|
-
}
|
|
2470
|
-
}
|
|
2471
|
-
|
|
2472
|
-
return sections.length > 0 ? sections.join("\n") : null;
|
|
3684
|
+
return MonitorCriteriaEvaluator.buildTelemetryResourceRootCauseContext({
|
|
3685
|
+
heading: "Host Details",
|
|
3686
|
+
identityLines: hostMonitor
|
|
3687
|
+
? [`- Host: ${hostMonitor.hostIdentifier || "Unknown"}`]
|
|
3688
|
+
: null,
|
|
3689
|
+
monitor: input.monitor,
|
|
3690
|
+
monitorStep: input.monitorStep,
|
|
3691
|
+
criteriaInstance: input.criteriaInstance,
|
|
3692
|
+
});
|
|
2473
3693
|
}
|
|
2474
3694
|
|
|
2475
3695
|
private static buildPodmanRootCauseContext(input: {
|
|
2476
|
-
dataToProcess: DataToProcess;
|
|
2477
3696
|
monitorStep: MonitorStep;
|
|
2478
3697
|
monitor: Monitor;
|
|
3698
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
2479
3699
|
}): string | null {
|
|
2480
|
-
const metricResponse: MetricMonitorResponse =
|
|
2481
|
-
input.dataToProcess as MetricMonitorResponse;
|
|
2482
|
-
|
|
2483
|
-
const sections: Array<string> = [];
|
|
2484
|
-
|
|
2485
3700
|
// Podman host context
|
|
2486
3701
|
const podmanMonitor: MonitorStepPodmanMonitor | undefined =
|
|
2487
3702
|
input.monitorStep.data?.podmanMonitor;
|
|
2488
3703
|
|
|
3704
|
+
let hostDetails: Array<string> | null = null;
|
|
3705
|
+
|
|
2489
3706
|
if (podmanMonitor) {
|
|
2490
|
-
|
|
2491
|
-
hostDetails.push(`- Host: ${podmanMonitor.hostIdentifier || "Unknown"}`);
|
|
3707
|
+
hostDetails = [`- Host: ${podmanMonitor.hostIdentifier || "Unknown"}`];
|
|
2492
3708
|
|
|
2493
3709
|
if (podmanMonitor.containerFilters?.containerName) {
|
|
2494
3710
|
hostDetails.push(
|
|
@@ -2501,40 +3717,78 @@ ${contextBlock}
|
|
|
2501
3717
|
`- Container Image Filter: ${podmanMonitor.containerFilters.containerImage}`,
|
|
2502
3718
|
);
|
|
2503
3719
|
}
|
|
3720
|
+
}
|
|
3721
|
+
|
|
3722
|
+
return MonitorCriteriaEvaluator.buildTelemetryResourceRootCauseContext({
|
|
3723
|
+
heading: "Podman Host Details",
|
|
3724
|
+
identityLines: hostDetails,
|
|
3725
|
+
monitor: input.monitor,
|
|
3726
|
+
monitorStep: input.monitorStep,
|
|
3727
|
+
criteriaInstance: input.criteriaInstance,
|
|
3728
|
+
});
|
|
3729
|
+
}
|
|
3730
|
+
|
|
3731
|
+
private static buildIoTRootCauseContext(input: {
|
|
3732
|
+
monitorStep: MonitorStep;
|
|
3733
|
+
monitor: Monitor;
|
|
3734
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
3735
|
+
}): string | null {
|
|
3736
|
+
// IoT fleet / device context
|
|
3737
|
+
const iotMonitor: MonitorStepIoTMonitor | undefined =
|
|
3738
|
+
input.monitorStep.data?.iotMonitor;
|
|
3739
|
+
|
|
3740
|
+
let deviceDetails: Array<string> | null = null;
|
|
3741
|
+
|
|
3742
|
+
if (iotMonitor) {
|
|
3743
|
+
deviceDetails = [`- Fleet: ${iotMonitor.fleetIdentifier || "Unknown"}`];
|
|
3744
|
+
|
|
3745
|
+
/*
|
|
3746
|
+
* The device that breached. Every IoT template groups by the
|
|
3747
|
+
* `device.id` label, so the matched series names it; an ungrouped
|
|
3748
|
+
* monitor has no series labels and falls back to the step's own
|
|
3749
|
+
* device filter below.
|
|
3750
|
+
*/
|
|
3751
|
+
const breachingDeviceId: unknown =
|
|
3752
|
+
MonitorCriteriaEvaluator.getMetricCriteriaContext(
|
|
3753
|
+
input.criteriaInstance,
|
|
3754
|
+
)?.seriesLabels?.[IOT_DEVICE_ID_ATTRIBUTE_KEY];
|
|
2504
3755
|
|
|
2505
|
-
// Add metric name from the query config
|
|
2506
3756
|
if (
|
|
2507
|
-
|
|
2508
|
-
|
|
3757
|
+
breachingDeviceId !== undefined &&
|
|
3758
|
+
breachingDeviceId !== null &&
|
|
3759
|
+
String(breachingDeviceId) !== ""
|
|
2509
3760
|
) {
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
hostDetails.push(`- Metric: \`${metricName}\``);
|
|
2514
|
-
}
|
|
3761
|
+
deviceDetails.push(
|
|
3762
|
+
`- Device: ${AffectedResourceList.code(String(breachingDeviceId))}`,
|
|
3763
|
+
);
|
|
2515
3764
|
}
|
|
2516
3765
|
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
const resultDetails: Array<string> = [];
|
|
3766
|
+
if (iotMonitor.resourceFilters?.deviceId) {
|
|
3767
|
+
deviceDetails.push(
|
|
3768
|
+
`- Device ID Filter: ${iotMonitor.resourceFilters.deviceId}`,
|
|
3769
|
+
);
|
|
3770
|
+
}
|
|
2523
3771
|
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
);
|
|
2529
|
-
}
|
|
3772
|
+
if (iotMonitor.resourceFilters?.deviceType) {
|
|
3773
|
+
deviceDetails.push(
|
|
3774
|
+
`- Device Type Filter: ${iotMonitor.resourceFilters.deviceType}`,
|
|
3775
|
+
);
|
|
2530
3776
|
}
|
|
2531
3777
|
|
|
2532
|
-
if (
|
|
2533
|
-
|
|
3778
|
+
if (iotMonitor.resourceFilters?.scope) {
|
|
3779
|
+
deviceDetails.push(
|
|
3780
|
+
`- Scope Filter: ${iotMonitor.resourceFilters.scope}`,
|
|
3781
|
+
);
|
|
2534
3782
|
}
|
|
2535
3783
|
}
|
|
2536
3784
|
|
|
2537
|
-
return
|
|
3785
|
+
return MonitorCriteriaEvaluator.buildTelemetryResourceRootCauseContext({
|
|
3786
|
+
heading: "IoT Device Details",
|
|
3787
|
+
identityLines: deviceDetails,
|
|
3788
|
+
monitor: input.monitor,
|
|
3789
|
+
monitorStep: input.monitorStep,
|
|
3790
|
+
criteriaInstance: input.criteriaInstance,
|
|
3791
|
+
});
|
|
2538
3792
|
}
|
|
2539
3793
|
|
|
2540
3794
|
/*
|
|
@@ -2544,7 +3798,7 @@ ${contextBlock}
|
|
|
2544
3798
|
* does not produce is shown as it arrived rather than guessed at.
|
|
2545
3799
|
*/
|
|
2546
3800
|
private static getProxmoxAffectedResourceKind(
|
|
2547
|
-
resource:
|
|
3801
|
+
resource: ProxmoxResourceIdentity,
|
|
2548
3802
|
): string {
|
|
2549
3803
|
switch (resource.resourceType) {
|
|
2550
3804
|
case "qemu":
|
|
@@ -2580,11 +3834,11 @@ ${contextBlock}
|
|
|
2580
3834
|
* its name next to its `qemu/100`-style id, and the node it runs on.
|
|
2581
3835
|
*/
|
|
2582
3836
|
private static getProxmoxAffectedResourceEntry(input: {
|
|
2583
|
-
resource:
|
|
3837
|
+
resource: ProxmoxResourceIdentity;
|
|
2584
3838
|
clusterName: string;
|
|
2585
3839
|
value: string;
|
|
2586
3840
|
}): AffectedResourceListEntry {
|
|
2587
|
-
const resource:
|
|
3841
|
+
const resource: ProxmoxResourceIdentity = input.resource;
|
|
2588
3842
|
|
|
2589
3843
|
let kind: string =
|
|
2590
3844
|
MonitorCriteriaEvaluator.getProxmoxAffectedResourceKind(resource);
|
|
@@ -2627,41 +3881,50 @@ ${contextBlock}
|
|
|
2627
3881
|
dataToProcess: DataToProcess;
|
|
2628
3882
|
monitorStep: MonitorStep;
|
|
2629
3883
|
monitor: Monitor;
|
|
3884
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
3885
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
2630
3886
|
}): string | null {
|
|
2631
3887
|
const metricResponse: MetricMonitorResponse =
|
|
2632
3888
|
input.dataToProcess as MetricMonitorResponse;
|
|
2633
3889
|
|
|
2634
|
-
const breakdown: ProxmoxResourceBreakdown | undefined =
|
|
2635
|
-
metricResponse.proxmoxResourceBreakdown;
|
|
2636
|
-
|
|
2637
|
-
const sections: Array<string> = [];
|
|
2638
|
-
|
|
2639
3890
|
// Proxmox cluster context
|
|
2640
3891
|
const proxmoxMonitor: MonitorStepProxmoxMonitor | undefined =
|
|
2641
3892
|
input.monitorStep.data?.proxmoxMonitor;
|
|
2642
3893
|
|
|
3894
|
+
const target: CriteriaMetricTarget | null =
|
|
3895
|
+
MonitorCriteriaEvaluator.resolveCriteriaMetricTarget({
|
|
3896
|
+
criteriaInstance: input.criteriaInstance,
|
|
3897
|
+
monitorStep: input.monitorStep,
|
|
3898
|
+
metricResponse: metricResponse,
|
|
3899
|
+
});
|
|
3900
|
+
|
|
3901
|
+
const breakdown: ProxmoxResourceBreakdown | undefined =
|
|
3902
|
+
MonitorCriteriaEvaluator.selectPlatformBreakdown({
|
|
3903
|
+
breakdowns: metricResponse.proxmoxResourceBreakdowns,
|
|
3904
|
+
legacyBreakdown: metricResponse.proxmoxResourceBreakdown,
|
|
3905
|
+
target: target,
|
|
3906
|
+
});
|
|
3907
|
+
|
|
3908
|
+
const clusterName: string =
|
|
3909
|
+
breakdown?.clusterName || proxmoxMonitor?.clusterIdentifier || "Unknown";
|
|
3910
|
+
|
|
3911
|
+
const sections: Array<string> = [];
|
|
3912
|
+
|
|
2643
3913
|
if (proxmoxMonitor || breakdown) {
|
|
2644
3914
|
const clusterDetails: Array<string> = [];
|
|
3915
|
+
clusterDetails.push(`- Cluster: ${clusterName}`);
|
|
2645
3916
|
clusterDetails.push(
|
|
2646
|
-
|
|
3917
|
+
...MonitorCriteriaEvaluator.describeCriteriaMetric({
|
|
3918
|
+
platform: "proxmox",
|
|
3919
|
+
target: target,
|
|
3920
|
+
breakdown: breakdown,
|
|
3921
|
+
fallbackMetricName: proxmoxMonitor?.metricViewConfig
|
|
3922
|
+
?.queryConfigs?.[0]?.metricQueryData?.filterData?.metricName as
|
|
3923
|
+
| string
|
|
3924
|
+
| undefined,
|
|
3925
|
+
}),
|
|
2647
3926
|
);
|
|
2648
3927
|
|
|
2649
|
-
if (breakdown) {
|
|
2650
|
-
clusterDetails.push(
|
|
2651
|
-
`- Metric: ${breakdown.metricFriendlyName} (\`${breakdown.metricName}\`)`,
|
|
2652
|
-
);
|
|
2653
|
-
} else if (
|
|
2654
|
-
proxmoxMonitor &&
|
|
2655
|
-
proxmoxMonitor.metricViewConfig?.queryConfigs?.length > 0 &&
|
|
2656
|
-
proxmoxMonitor.metricViewConfig.queryConfigs[0]
|
|
2657
|
-
) {
|
|
2658
|
-
const metricName: string = proxmoxMonitor.metricViewConfig
|
|
2659
|
-
.queryConfigs[0].metricQueryData?.filterData?.metricName as string;
|
|
2660
|
-
if (metricName) {
|
|
2661
|
-
clusterDetails.push(`- Metric: \`${metricName}\``);
|
|
2662
|
-
}
|
|
2663
|
-
}
|
|
2664
|
-
|
|
2665
3928
|
if (proxmoxMonitor?.resourceFilters?.scope) {
|
|
2666
3929
|
clusterDetails.push(
|
|
2667
3930
|
`- Scope Filter: ${proxmoxMonitor.resourceFilters.scope}`,
|
|
@@ -2694,66 +3957,79 @@ ${contextBlock}
|
|
|
2694
3957
|
// Affected resources: a ranked list of kind, name and node
|
|
2695
3958
|
let renderedBreakdownList: boolean = false;
|
|
2696
3959
|
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
/*
|
|
2716
|
-
* Skip the list when no row carries any identity label
|
|
2717
|
-
* (cluster-wide series) — it would add nothing.
|
|
2718
|
-
*/
|
|
2719
|
-
const hasIdentity: boolean = sortedResources.some(
|
|
2720
|
-
(r: ProxmoxAffectedResource) => {
|
|
2721
|
-
return r.resourceId || r.resourceName || r.nodeName;
|
|
3960
|
+
/*
|
|
3961
|
+
* The resources the matched criteria breached, worst first, top 10 —
|
|
3962
|
+
* NOT simply the non-zero rows. pve-node-offline fires on `pve_up < 1`,
|
|
3963
|
+
* so the zero rows ARE the down nodes; the old `> 0` listed every
|
|
3964
|
+
* healthy node under it instead.
|
|
3965
|
+
*/
|
|
3966
|
+
const rows: Array<PlatformAffectedRow<ProxmoxResourceIdentity>> | null =
|
|
3967
|
+
MonitorCriteriaEvaluator.getPlatformAffectedRows<
|
|
3968
|
+
ProxmoxResourceIdentity,
|
|
3969
|
+
ProxmoxAffectedResource
|
|
3970
|
+
>({
|
|
3971
|
+
platform: "proxmox",
|
|
3972
|
+
perSeriesMatches: input.perSeriesMatches,
|
|
3973
|
+
namesObject: (identity: ProxmoxResourceIdentity): boolean => {
|
|
3974
|
+
return Boolean(
|
|
3975
|
+
identity.resourceId || identity.resourceName || identity.nodeName,
|
|
3976
|
+
);
|
|
2722
3977
|
},
|
|
2723
|
-
|
|
3978
|
+
metricResponse: metricResponse,
|
|
3979
|
+
monitorStep: input.monitorStep,
|
|
3980
|
+
target: target,
|
|
3981
|
+
breakdown: breakdown,
|
|
3982
|
+
breach: MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
|
|
3983
|
+
criteriaInstance: input.criteriaInstance,
|
|
3984
|
+
}),
|
|
3985
|
+
toIdentity: PlatformResourceIdentity.proxmox,
|
|
3986
|
+
withContext: PlatformResourceIdentity.withProxmoxContext,
|
|
3987
|
+
});
|
|
2724
3988
|
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
3989
|
+
/*
|
|
3990
|
+
* Skip the list when no row carries any identity label
|
|
3991
|
+
* (cluster-wide series) — it would add nothing.
|
|
3992
|
+
*/
|
|
3993
|
+
const hasIdentity: boolean = (rows || []).some(
|
|
3994
|
+
(row: PlatformAffectedRow<ProxmoxResourceIdentity>) => {
|
|
3995
|
+
return Boolean(
|
|
3996
|
+
row.seriesLabels ||
|
|
3997
|
+
row.identity.resourceId ||
|
|
3998
|
+
row.identity.resourceName ||
|
|
3999
|
+
row.identity.nodeName,
|
|
4000
|
+
);
|
|
4001
|
+
},
|
|
4002
|
+
);
|
|
2728
4003
|
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
4004
|
+
if (rows && rows.length > 0 && hasIdentity) {
|
|
4005
|
+
sections.push(
|
|
4006
|
+
AffectedResourceList.render({
|
|
4007
|
+
heading: "Affected Resources",
|
|
4008
|
+
overflowNoun: "affected resources",
|
|
4009
|
+
totalCount: rows.length,
|
|
4010
|
+
entries: rows
|
|
4011
|
+
.slice(0, AffectedResourceList.MAX_ENTRIES)
|
|
4012
|
+
.map(
|
|
2735
4013
|
(
|
|
2736
|
-
|
|
4014
|
+
row: PlatformAffectedRow<ProxmoxResourceIdentity>,
|
|
2737
4015
|
): AffectedResourceListEntry => {
|
|
2738
|
-
return
|
|
2739
|
-
{
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
);
|
|
4016
|
+
return row.seriesLabels
|
|
4017
|
+
? MonitorCriteriaEvaluator.getSeriesLabelEntry({
|
|
4018
|
+
labels: row.seriesLabels,
|
|
4019
|
+
value:
|
|
4020
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
4021
|
+
})
|
|
4022
|
+
: MonitorCriteriaEvaluator.getProxmoxAffectedResourceEntry({
|
|
4023
|
+
resource: row.identity,
|
|
4024
|
+
clusterName: clusterName,
|
|
4025
|
+
value:
|
|
4026
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
4027
|
+
});
|
|
2751
4028
|
},
|
|
2752
4029
|
),
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
}
|
|
4030
|
+
}),
|
|
4031
|
+
);
|
|
4032
|
+
renderedBreakdownList = true;
|
|
2757
4033
|
}
|
|
2758
4034
|
|
|
2759
4035
|
// Metric results summary (fallback context when no list rendered)
|
|
@@ -2790,7 +4066,7 @@ ${contextBlock}
|
|
|
2790
4066
|
* host and pool checks.
|
|
2791
4067
|
*/
|
|
2792
4068
|
private static getVMwareAffectedResourceKind(
|
|
2793
|
-
resource:
|
|
4069
|
+
resource: VMwareResourceIdentity,
|
|
2794
4070
|
): string | undefined {
|
|
2795
4071
|
if (resource.vmName || resource.vmId) {
|
|
2796
4072
|
return "Virtual Machine";
|
|
@@ -2825,11 +4101,11 @@ ${contextBlock}
|
|
|
2825
4101
|
* listed beneath it.
|
|
2826
4102
|
*/
|
|
2827
4103
|
private static getVMwareAffectedResourceEntry(input: {
|
|
2828
|
-
resource:
|
|
4104
|
+
resource: VMwareResourceIdentity;
|
|
2829
4105
|
vcenterName: string;
|
|
2830
4106
|
value: string;
|
|
2831
4107
|
}): AffectedResourceListEntry {
|
|
2832
|
-
const resource:
|
|
4108
|
+
const resource: VMwareResourceIdentity = input.resource;
|
|
2833
4109
|
|
|
2834
4110
|
const kind: string | undefined =
|
|
2835
4111
|
MonitorCriteriaEvaluator.getVMwareAffectedResourceKind(resource);
|
|
@@ -2915,41 +4191,48 @@ ${contextBlock}
|
|
|
2915
4191
|
dataToProcess: DataToProcess;
|
|
2916
4192
|
monitorStep: MonitorStep;
|
|
2917
4193
|
monitor: Monitor;
|
|
4194
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
4195
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
2918
4196
|
}): string | null {
|
|
2919
4197
|
const metricResponse: MetricMonitorResponse =
|
|
2920
4198
|
input.dataToProcess as MetricMonitorResponse;
|
|
2921
4199
|
|
|
2922
|
-
const breakdown: VMwareResourceBreakdown | undefined =
|
|
2923
|
-
metricResponse.vmwareResourceBreakdown;
|
|
2924
|
-
|
|
2925
|
-
const sections: Array<string> = [];
|
|
2926
|
-
|
|
2927
4200
|
// vCenter context
|
|
2928
4201
|
const vmwareMonitor: MonitorStepVMwareMonitor | undefined =
|
|
2929
4202
|
input.monitorStep.data?.vmwareMonitor;
|
|
2930
4203
|
|
|
4204
|
+
const target: CriteriaMetricTarget | null =
|
|
4205
|
+
MonitorCriteriaEvaluator.resolveCriteriaMetricTarget({
|
|
4206
|
+
criteriaInstance: input.criteriaInstance,
|
|
4207
|
+
monitorStep: input.monitorStep,
|
|
4208
|
+
metricResponse: metricResponse,
|
|
4209
|
+
});
|
|
4210
|
+
|
|
4211
|
+
const breakdown: VMwareResourceBreakdown | undefined =
|
|
4212
|
+
MonitorCriteriaEvaluator.selectPlatformBreakdown({
|
|
4213
|
+
breakdowns: metricResponse.vmwareResourceBreakdowns,
|
|
4214
|
+
legacyBreakdown: metricResponse.vmwareResourceBreakdown,
|
|
4215
|
+
target: target,
|
|
4216
|
+
});
|
|
4217
|
+
|
|
4218
|
+
const vcenterName: string =
|
|
4219
|
+
breakdown?.vcenterName || vmwareMonitor?.vcenterIdentifier || "Unknown";
|
|
4220
|
+
|
|
4221
|
+
const sections: Array<string> = [];
|
|
4222
|
+
|
|
2931
4223
|
if (vmwareMonitor || breakdown) {
|
|
2932
4224
|
const vcenterDetails: Array<string> = [];
|
|
4225
|
+
vcenterDetails.push(`- vCenter: ${vcenterName}`);
|
|
2933
4226
|
vcenterDetails.push(
|
|
2934
|
-
|
|
4227
|
+
...MonitorCriteriaEvaluator.describeCriteriaMetric({
|
|
4228
|
+
platform: "vmware",
|
|
4229
|
+
target: target,
|
|
4230
|
+
breakdown: breakdown,
|
|
4231
|
+
fallbackMetricName: vmwareMonitor?.metricViewConfig?.queryConfigs?.[0]
|
|
4232
|
+
?.metricQueryData?.filterData?.metricName as string | undefined,
|
|
4233
|
+
}),
|
|
2935
4234
|
);
|
|
2936
4235
|
|
|
2937
|
-
if (breakdown) {
|
|
2938
|
-
vcenterDetails.push(
|
|
2939
|
-
`- Metric: ${breakdown.metricFriendlyName} (\`${breakdown.metricName}\`)`,
|
|
2940
|
-
);
|
|
2941
|
-
} else if (
|
|
2942
|
-
vmwareMonitor &&
|
|
2943
|
-
vmwareMonitor.metricViewConfig?.queryConfigs?.length > 0 &&
|
|
2944
|
-
vmwareMonitor.metricViewConfig.queryConfigs[0]
|
|
2945
|
-
) {
|
|
2946
|
-
const metricName: string = vmwareMonitor.metricViewConfig
|
|
2947
|
-
.queryConfigs[0].metricQueryData?.filterData?.metricName as string;
|
|
2948
|
-
if (metricName) {
|
|
2949
|
-
vcenterDetails.push(`- Metric: \`${metricName}\``);
|
|
2950
|
-
}
|
|
2951
|
-
}
|
|
2952
|
-
|
|
2953
4236
|
/*
|
|
2954
4237
|
* The worker maps each resource filter to an equality on the
|
|
2955
4238
|
* matching `resource.vcenter.*` attribute; surface the same
|
|
@@ -2997,66 +4280,81 @@ ${contextBlock}
|
|
|
2997
4280
|
// Affected resources: a ranked list of object, host and cluster
|
|
2998
4281
|
let renderedBreakdownList: boolean = false;
|
|
2999
4282
|
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
return r.metricValue > 0;
|
|
3015
|
-
})
|
|
3016
|
-
.sort((a: VMwareAffectedResource, b: VMwareAffectedResource) => {
|
|
3017
|
-
return b.metricValue - a.metricValue;
|
|
3018
|
-
});
|
|
3019
|
-
|
|
3020
|
-
/*
|
|
3021
|
-
* Skip the list when no row carries any identity attribute — it
|
|
3022
|
-
* would add nothing. Every vcenter-receiver series carries at
|
|
3023
|
-
* least `vcenter.datacenter.name`, so this only trips on a
|
|
3024
|
-
* breakdown the worker could not attribute at all.
|
|
3025
|
-
*/
|
|
3026
|
-
const hasIdentity: boolean = sortedResources.some(
|
|
3027
|
-
(r: VMwareAffectedResource) => {
|
|
4283
|
+
/*
|
|
4284
|
+
* The objects the matched criteria breached, worst first, top 10. For
|
|
4285
|
+
* count metrics filtered to an unhealthy state
|
|
4286
|
+
* (`vcenter.datacenter.host.count{status=red}`) the non-zero rows ARE
|
|
4287
|
+
* the objects with a problem.
|
|
4288
|
+
*/
|
|
4289
|
+
const rows: Array<PlatformAffectedRow<VMwareResourceIdentity>> | null =
|
|
4290
|
+
MonitorCriteriaEvaluator.getPlatformAffectedRows<
|
|
4291
|
+
VMwareResourceIdentity,
|
|
4292
|
+
VMwareAffectedResource
|
|
4293
|
+
>({
|
|
4294
|
+
platform: "vmware",
|
|
4295
|
+
perSeriesMatches: input.perSeriesMatches,
|
|
4296
|
+
namesObject: (identity: VMwareResourceIdentity): boolean => {
|
|
3028
4297
|
return Boolean(
|
|
3029
|
-
MonitorCriteriaEvaluator.getVMwareAffectedResourceKind(
|
|
4298
|
+
MonitorCriteriaEvaluator.getVMwareAffectedResourceKind(identity),
|
|
3030
4299
|
);
|
|
3031
4300
|
},
|
|
3032
|
-
|
|
4301
|
+
metricResponse: metricResponse,
|
|
4302
|
+
monitorStep: input.monitorStep,
|
|
4303
|
+
target: target,
|
|
4304
|
+
breakdown: breakdown,
|
|
4305
|
+
breach: MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
|
|
4306
|
+
criteriaInstance: input.criteriaInstance,
|
|
4307
|
+
}),
|
|
4308
|
+
toIdentity: PlatformResourceIdentity.vmware,
|
|
4309
|
+
withContext: PlatformResourceIdentity.withVMwareContext,
|
|
4310
|
+
});
|
|
3033
4311
|
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
resource: resource,
|
|
3047
|
-
vcenterName: breakdown.vcenterName,
|
|
3048
|
-
value: MonitorCriteriaEvaluator.formatPlatformResourceValue({
|
|
3049
|
-
platform: "vmware",
|
|
3050
|
-
metricName: breakdown.metricName,
|
|
3051
|
-
value: resource.metricValue,
|
|
3052
|
-
}),
|
|
3053
|
-
});
|
|
3054
|
-
},
|
|
4312
|
+
/*
|
|
4313
|
+
* Skip the list when no row carries any identity attribute — it
|
|
4314
|
+
* would add nothing. Every vcenter-receiver series carries at
|
|
4315
|
+
* least `vcenter.datacenter.name`, so this only trips on a
|
|
4316
|
+
* breakdown the worker could not attribute at all.
|
|
4317
|
+
*/
|
|
4318
|
+
const hasIdentity: boolean = (rows || []).some(
|
|
4319
|
+
(row: PlatformAffectedRow<VMwareResourceIdentity>) => {
|
|
4320
|
+
return Boolean(
|
|
4321
|
+
row.seriesLabels ||
|
|
4322
|
+
MonitorCriteriaEvaluator.getVMwareAffectedResourceKind(
|
|
4323
|
+
row.identity,
|
|
3055
4324
|
),
|
|
3056
|
-
}),
|
|
3057
4325
|
);
|
|
3058
|
-
|
|
3059
|
-
|
|
4326
|
+
},
|
|
4327
|
+
);
|
|
4328
|
+
|
|
4329
|
+
if (rows && rows.length > 0 && hasIdentity) {
|
|
4330
|
+
sections.push(
|
|
4331
|
+
AffectedResourceList.render({
|
|
4332
|
+
heading: "Affected Resources",
|
|
4333
|
+
overflowNoun: "affected resources",
|
|
4334
|
+
totalCount: rows.length,
|
|
4335
|
+
entries: rows
|
|
4336
|
+
.slice(0, AffectedResourceList.MAX_ENTRIES)
|
|
4337
|
+
.map(
|
|
4338
|
+
(
|
|
4339
|
+
row: PlatformAffectedRow<VMwareResourceIdentity>,
|
|
4340
|
+
): AffectedResourceListEntry => {
|
|
4341
|
+
return row.seriesLabels
|
|
4342
|
+
? MonitorCriteriaEvaluator.getSeriesLabelEntry({
|
|
4343
|
+
labels: row.seriesLabels,
|
|
4344
|
+
value:
|
|
4345
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
4346
|
+
})
|
|
4347
|
+
: MonitorCriteriaEvaluator.getVMwareAffectedResourceEntry({
|
|
4348
|
+
resource: row.identity,
|
|
4349
|
+
vcenterName: vcenterName,
|
|
4350
|
+
value:
|
|
4351
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
4352
|
+
});
|
|
4353
|
+
},
|
|
4354
|
+
),
|
|
4355
|
+
}),
|
|
4356
|
+
);
|
|
4357
|
+
renderedBreakdownList = true;
|
|
3060
4358
|
}
|
|
3061
4359
|
|
|
3062
4360
|
// Metric results summary (fallback context when no list rendered)
|
|
@@ -3088,11 +4386,11 @@ ${contextBlock}
|
|
|
3088
4386
|
* service it belongs to and the node it is scheduled on beneath it.
|
|
3089
4387
|
*/
|
|
3090
4388
|
private static getDockerSwarmAffectedResourceEntry(input: {
|
|
3091
|
-
resource:
|
|
4389
|
+
resource: DockerSwarmResourceIdentity;
|
|
3092
4390
|
clusterName: string;
|
|
3093
4391
|
value: string;
|
|
3094
4392
|
}): AffectedResourceListEntry {
|
|
3095
|
-
const resource:
|
|
4393
|
+
const resource: DockerSwarmResourceIdentity = input.resource;
|
|
3096
4394
|
|
|
3097
4395
|
let kind: string = "Cluster";
|
|
3098
4396
|
let name: string = AffectedResourceList.code(input.clusterName);
|
|
@@ -3136,41 +4434,52 @@ ${contextBlock}
|
|
|
3136
4434
|
dataToProcess: DataToProcess;
|
|
3137
4435
|
monitorStep: MonitorStep;
|
|
3138
4436
|
monitor: Monitor;
|
|
4437
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
4438
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
3139
4439
|
}): string | null {
|
|
3140
4440
|
const metricResponse: MetricMonitorResponse =
|
|
3141
4441
|
input.dataToProcess as MetricMonitorResponse;
|
|
3142
4442
|
|
|
3143
|
-
const breakdown: DockerSwarmResourceBreakdown | undefined =
|
|
3144
|
-
metricResponse.dockerSwarmResourceBreakdown;
|
|
3145
|
-
|
|
3146
|
-
const sections: Array<string> = [];
|
|
3147
|
-
|
|
3148
4443
|
// Docker Swarm cluster context
|
|
3149
4444
|
const dockerSwarmMonitor: MonitorStepDockerSwarmMonitor | undefined =
|
|
3150
4445
|
input.monitorStep.data?.dockerSwarmMonitor;
|
|
3151
4446
|
|
|
4447
|
+
const target: CriteriaMetricTarget | null =
|
|
4448
|
+
MonitorCriteriaEvaluator.resolveCriteriaMetricTarget({
|
|
4449
|
+
criteriaInstance: input.criteriaInstance,
|
|
4450
|
+
monitorStep: input.monitorStep,
|
|
4451
|
+
metricResponse: metricResponse,
|
|
4452
|
+
});
|
|
4453
|
+
|
|
4454
|
+
const breakdown: DockerSwarmResourceBreakdown | undefined =
|
|
4455
|
+
MonitorCriteriaEvaluator.selectPlatformBreakdown({
|
|
4456
|
+
breakdowns: metricResponse.dockerSwarmResourceBreakdowns,
|
|
4457
|
+
legacyBreakdown: metricResponse.dockerSwarmResourceBreakdown,
|
|
4458
|
+
target: target,
|
|
4459
|
+
});
|
|
4460
|
+
|
|
4461
|
+
const clusterName: string =
|
|
4462
|
+
breakdown?.clusterName ||
|
|
4463
|
+
dockerSwarmMonitor?.clusterIdentifier ||
|
|
4464
|
+
"Unknown";
|
|
4465
|
+
|
|
4466
|
+
const sections: Array<string> = [];
|
|
4467
|
+
|
|
3152
4468
|
if (dockerSwarmMonitor || breakdown) {
|
|
3153
4469
|
const clusterDetails: Array<string> = [];
|
|
4470
|
+
clusterDetails.push(`- Cluster: ${clusterName}`);
|
|
3154
4471
|
clusterDetails.push(
|
|
3155
|
-
|
|
4472
|
+
...MonitorCriteriaEvaluator.describeCriteriaMetric({
|
|
4473
|
+
platform: "dockerSwarm",
|
|
4474
|
+
target: target,
|
|
4475
|
+
breakdown: breakdown,
|
|
4476
|
+
fallbackMetricName: dockerSwarmMonitor?.metricViewConfig
|
|
4477
|
+
?.queryConfigs?.[0]?.metricQueryData?.filterData?.metricName as
|
|
4478
|
+
| string
|
|
4479
|
+
| undefined,
|
|
4480
|
+
}),
|
|
3156
4481
|
);
|
|
3157
4482
|
|
|
3158
|
-
if (breakdown) {
|
|
3159
|
-
clusterDetails.push(
|
|
3160
|
-
`- Metric: ${breakdown.metricFriendlyName} (\`${breakdown.metricName}\`)`,
|
|
3161
|
-
);
|
|
3162
|
-
} else if (
|
|
3163
|
-
dockerSwarmMonitor &&
|
|
3164
|
-
dockerSwarmMonitor.metricViewConfig?.queryConfigs?.length > 0 &&
|
|
3165
|
-
dockerSwarmMonitor.metricViewConfig.queryConfigs[0]
|
|
3166
|
-
) {
|
|
3167
|
-
const metricName: string = dockerSwarmMonitor.metricViewConfig
|
|
3168
|
-
.queryConfigs[0].metricQueryData?.filterData?.metricName as string;
|
|
3169
|
-
if (metricName) {
|
|
3170
|
-
clusterDetails.push(`- Metric: \`${metricName}\``);
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3173
|
-
|
|
3174
4483
|
if (dockerSwarmMonitor?.resourceFilters?.serviceName) {
|
|
3175
4484
|
clusterDetails.push(
|
|
3176
4485
|
`- Service Filter: ${dockerSwarmMonitor.resourceFilters.serviceName}`,
|
|
@@ -3203,68 +4512,82 @@ ${contextBlock}
|
|
|
3203
4512
|
// Affected tasks: a ranked list of task, service and node
|
|
3204
4513
|
let renderedBreakdownList: boolean = false;
|
|
3205
4514
|
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
(
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
);
|
|
3225
|
-
|
|
3226
|
-
/*
|
|
3227
|
-
* Skip the list when no row carries any identity label
|
|
3228
|
-
* (cluster-wide series) — it would add nothing.
|
|
3229
|
-
*/
|
|
3230
|
-
const hasIdentity: boolean = sortedResources.some(
|
|
3231
|
-
(r: DockerSwarmAffectedResource) => {
|
|
3232
|
-
return r.containerName || r.serviceName || r.nodeName;
|
|
4515
|
+
/*
|
|
4516
|
+
* The tasks the matched criteria breached, worst first, top 10. The
|
|
4517
|
+
* task-down template fires on `container.uptime < 60`, so the worst
|
|
4518
|
+
* task is the one with the LOWEST uptime; the old `> 0`,
|
|
4519
|
+
* highest-first list put the healthiest, longest-running tasks at the
|
|
4520
|
+
* top and pushed the restarting one off the end.
|
|
4521
|
+
*/
|
|
4522
|
+
const rows: Array<PlatformAffectedRow<DockerSwarmResourceIdentity>> | null =
|
|
4523
|
+
MonitorCriteriaEvaluator.getPlatformAffectedRows<
|
|
4524
|
+
DockerSwarmResourceIdentity,
|
|
4525
|
+
DockerSwarmAffectedResource
|
|
4526
|
+
>({
|
|
4527
|
+
platform: "dockerSwarm",
|
|
4528
|
+
perSeriesMatches: input.perSeriesMatches,
|
|
4529
|
+
namesObject: (identity: DockerSwarmResourceIdentity): boolean => {
|
|
4530
|
+
return Boolean(
|
|
4531
|
+
identity.containerName || identity.serviceName || identity.nodeName,
|
|
4532
|
+
);
|
|
3233
4533
|
},
|
|
3234
|
-
|
|
4534
|
+
metricResponse: metricResponse,
|
|
4535
|
+
monitorStep: input.monitorStep,
|
|
4536
|
+
target: target,
|
|
4537
|
+
breakdown: breakdown,
|
|
4538
|
+
breach: MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
|
|
4539
|
+
criteriaInstance: input.criteriaInstance,
|
|
4540
|
+
}),
|
|
4541
|
+
toIdentity: PlatformResourceIdentity.dockerSwarm,
|
|
4542
|
+
withContext: PlatformResourceIdentity.withDockerSwarmContext,
|
|
4543
|
+
});
|
|
3235
4544
|
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
4545
|
+
/*
|
|
4546
|
+
* Skip the list when no row carries any identity label
|
|
4547
|
+
* (cluster-wide series) — it would add nothing.
|
|
4548
|
+
*/
|
|
4549
|
+
const hasIdentity: boolean = (rows || []).some(
|
|
4550
|
+
(row: PlatformAffectedRow<DockerSwarmResourceIdentity>) => {
|
|
4551
|
+
return Boolean(
|
|
4552
|
+
row.seriesLabels ||
|
|
4553
|
+
row.identity.containerName ||
|
|
4554
|
+
row.identity.serviceName ||
|
|
4555
|
+
row.identity.nodeName,
|
|
4556
|
+
);
|
|
4557
|
+
},
|
|
4558
|
+
);
|
|
3239
4559
|
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
4560
|
+
if (rows && rows.length > 0 && hasIdentity) {
|
|
4561
|
+
sections.push(
|
|
4562
|
+
AffectedResourceList.render({
|
|
4563
|
+
heading: "Affected Tasks",
|
|
4564
|
+
overflowNoun: "affected tasks",
|
|
4565
|
+
totalCount: rows.length,
|
|
4566
|
+
entries: rows
|
|
4567
|
+
.slice(0, AffectedResourceList.MAX_ENTRIES)
|
|
4568
|
+
.map(
|
|
3246
4569
|
(
|
|
3247
|
-
|
|
4570
|
+
row: PlatformAffectedRow<DockerSwarmResourceIdentity>,
|
|
3248
4571
|
): AffectedResourceListEntry => {
|
|
3249
|
-
return
|
|
3250
|
-
{
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
4572
|
+
return row.seriesLabels
|
|
4573
|
+
? MonitorCriteriaEvaluator.getSeriesLabelEntry({
|
|
4574
|
+
labels: row.seriesLabels,
|
|
4575
|
+
value:
|
|
4576
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
4577
|
+
})
|
|
4578
|
+
: MonitorCriteriaEvaluator.getDockerSwarmAffectedResourceEntry(
|
|
3254
4579
|
{
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
value:
|
|
4580
|
+
resource: row.identity,
|
|
4581
|
+
clusterName: clusterName,
|
|
4582
|
+
value:
|
|
4583
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
3258
4584
|
},
|
|
3259
|
-
)
|
|
3260
|
-
},
|
|
3261
|
-
);
|
|
4585
|
+
);
|
|
3262
4586
|
},
|
|
3263
4587
|
),
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
}
|
|
4588
|
+
}),
|
|
4589
|
+
);
|
|
4590
|
+
renderedBreakdownList = true;
|
|
3268
4591
|
}
|
|
3269
4592
|
|
|
3270
4593
|
// Metric results summary (fallback context when no list rendered)
|
|
@@ -3322,11 +4645,26 @@ ${contextBlock}
|
|
|
3322
4645
|
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
3323
4646
|
floorEqualityFiresOnFall?: boolean | undefined;
|
|
3324
4647
|
}): AffectedResourceBreachPredicate {
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
4648
|
+
/*
|
|
4649
|
+
* Thresholds read exactly as the criteria evaluation reads them. The
|
|
4650
|
+
* dashboard's criteria form saves the threshold as a string ("60"), so
|
|
4651
|
+
* a `typeof === "number"` test dropped every filter a user had edited
|
|
4652
|
+
* and quietly put their criteria back on the `> 0`, highest-first rule.
|
|
4653
|
+
*/
|
|
4654
|
+
const metricFilters: Array<{ filter: CriteriaFilter; threshold: number }> =
|
|
4655
|
+
(input.criteriaInstance?.data?.filters || [])
|
|
4656
|
+
.filter((f: CriteriaFilter) => {
|
|
4657
|
+
return f.checkOn === CheckOn.MetricValue;
|
|
4658
|
+
})
|
|
4659
|
+
.map((f: CriteriaFilter) => {
|
|
4660
|
+
return {
|
|
4661
|
+
filter: f,
|
|
4662
|
+
threshold: CompareCriteria.convertToNumber(f.value) as number,
|
|
4663
|
+
};
|
|
4664
|
+
})
|
|
4665
|
+
.filter((f: { filter: CriteriaFilter; threshold: number | null }) => {
|
|
4666
|
+
return f.threshold !== null;
|
|
4667
|
+
});
|
|
3330
4668
|
|
|
3331
4669
|
const opensIncidentOrAlert: boolean =
|
|
3332
4670
|
input.criteriaInstance?.data?.createIncidents === true ||
|
|
@@ -3334,44 +4672,126 @@ ${contextBlock}
|
|
|
3334
4672
|
|
|
3335
4673
|
const firesWhenMetricFalls: boolean =
|
|
3336
4674
|
metricFilters.length > 0 &&
|
|
3337
|
-
metricFilters.every(
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
4675
|
+
metricFilters.every(
|
|
4676
|
+
(f: { filter: CriteriaFilter; threshold: number }) => {
|
|
4677
|
+
if (
|
|
4678
|
+
f.filter.filterType === FilterType.LessThan ||
|
|
4679
|
+
f.filter.filterType === FilterType.LessThanOrEqualTo
|
|
4680
|
+
) {
|
|
4681
|
+
return true;
|
|
4682
|
+
}
|
|
3344
4683
|
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
4684
|
+
return (
|
|
4685
|
+
input.floorEqualityFiresOnFall === true &&
|
|
4686
|
+
opensIncidentOrAlert &&
|
|
4687
|
+
f.filter.filterType === FilterType.EqualTo &&
|
|
4688
|
+
f.threshold <= 0
|
|
4689
|
+
);
|
|
4690
|
+
},
|
|
4691
|
+
);
|
|
3352
4692
|
|
|
3353
|
-
|
|
4693
|
+
/*
|
|
4694
|
+
* A firing criteria that asks for one exact, non-zero value —
|
|
4695
|
+
* k8s-pod-pending fires on phase `= 1` (Pending) — lists exactly the
|
|
4696
|
+
* rows that were at that value. The `> 0` rule below listed every pod
|
|
4697
|
+
* with a phase, which is all of them, led by the one with the highest
|
|
4698
|
+
* code (Unknown), and sent the reader to `kubectl describe` the wrong
|
|
4699
|
+
* pod.
|
|
4700
|
+
*
|
|
4701
|
+
* A row was at the value if either end of its window was: pod-pending
|
|
4702
|
+
* watches the phase at its Min (a pod that has since started Running
|
|
4703
|
+
* still breached), a memory-pressure `= 1` watches the Max. `= 0` stays
|
|
4704
|
+
* with the floor rule above, which each platform opts into.
|
|
4705
|
+
*/
|
|
4706
|
+
const exactValues: Array<number> = metricFilters.map(
|
|
4707
|
+
(f: { filter: CriteriaFilter; threshold: number }) => {
|
|
4708
|
+
return f.threshold;
|
|
4709
|
+
},
|
|
4710
|
+
);
|
|
4711
|
+
|
|
4712
|
+
const firesOnExactValue: boolean =
|
|
4713
|
+
!firesWhenMetricFalls &&
|
|
4714
|
+
opensIncidentOrAlert &&
|
|
4715
|
+
metricFilters.length > 0 &&
|
|
4716
|
+
metricFilters.every(
|
|
4717
|
+
(f: { filter: CriteriaFilter; threshold: number }) => {
|
|
4718
|
+
return f.filter.filterType === FilterType.EqualTo && f.threshold > 0;
|
|
4719
|
+
},
|
|
4720
|
+
);
|
|
4721
|
+
|
|
4722
|
+
if (firesOnExactValue) {
|
|
4723
|
+
return {
|
|
4724
|
+
matches: (value: number): boolean => {
|
|
4725
|
+
return exactValues.includes(value);
|
|
4726
|
+
},
|
|
4727
|
+
worstIsLowest: false,
|
|
4728
|
+
pickEnd: (ends: {
|
|
4729
|
+
highest: number;
|
|
4730
|
+
lowest: number;
|
|
4731
|
+
}): "highest" | "lowest" => {
|
|
4732
|
+
return exactValues.includes(ends.highest) ? "highest" : "lowest";
|
|
4733
|
+
},
|
|
4734
|
+
};
|
|
4735
|
+
}
|
|
4736
|
+
|
|
4737
|
+
/*
|
|
4738
|
+
* A criteria that fires when the metric RISES lists the rows past its
|
|
4739
|
+
* threshold. The blanket `> 0` listed every resource with any value at
|
|
4740
|
+
* all — the healthy nodes at 50% under a "> 90%" alert. When no single
|
|
4741
|
+
* row is past it (the criteria compared a sum or an average across
|
|
4742
|
+
* resources) the list still falls back to the non-zero rows.
|
|
4743
|
+
*/
|
|
4744
|
+
const firesWhenMetricRises: boolean =
|
|
4745
|
+
metricFilters.length > 0 &&
|
|
4746
|
+
metricFilters.every(
|
|
4747
|
+
(f: { filter: CriteriaFilter; threshold: number }) => {
|
|
4748
|
+
return (
|
|
4749
|
+
f.filter.filterType === FilterType.GreaterThan ||
|
|
4750
|
+
f.filter.filterType === FilterType.GreaterThanOrEqualTo
|
|
4751
|
+
);
|
|
4752
|
+
},
|
|
4753
|
+
);
|
|
4754
|
+
|
|
4755
|
+
const isNonZero: (value: number) => boolean = (value: number): boolean => {
|
|
4756
|
+
return value > 0;
|
|
4757
|
+
};
|
|
4758
|
+
|
|
4759
|
+
if (firesWhenMetricRises && !firesWhenMetricFalls) {
|
|
3354
4760
|
return {
|
|
3355
4761
|
matches: (value: number): boolean => {
|
|
3356
|
-
return
|
|
4762
|
+
return metricFilters.some(
|
|
4763
|
+
(f: { filter: CriteriaFilter; threshold: number }) => {
|
|
4764
|
+
return f.filter.filterType === FilterType.GreaterThan
|
|
4765
|
+
? value > f.threshold
|
|
4766
|
+
: value >= f.threshold;
|
|
4767
|
+
},
|
|
4768
|
+
);
|
|
3357
4769
|
},
|
|
3358
4770
|
worstIsLowest: false,
|
|
4771
|
+
fallbackMatches: isNonZero,
|
|
4772
|
+
};
|
|
4773
|
+
}
|
|
4774
|
+
|
|
4775
|
+
if (!firesWhenMetricFalls) {
|
|
4776
|
+
return {
|
|
4777
|
+
matches: isNonZero,
|
|
4778
|
+
worstIsLowest: false,
|
|
3359
4779
|
};
|
|
3360
4780
|
}
|
|
3361
4781
|
|
|
3362
4782
|
return {
|
|
3363
4783
|
matches: (value: number): boolean => {
|
|
3364
|
-
return metricFilters.some(
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
}
|
|
4784
|
+
return metricFilters.some(
|
|
4785
|
+
(f: { filter: CriteriaFilter; threshold: number }) => {
|
|
4786
|
+
if (f.filter.filterType === FilterType.EqualTo) {
|
|
4787
|
+
return value === f.threshold;
|
|
4788
|
+
}
|
|
3370
4789
|
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
4790
|
+
return f.filter.filterType === FilterType.LessThan
|
|
4791
|
+
? value < f.threshold
|
|
4792
|
+
: value <= f.threshold;
|
|
4793
|
+
},
|
|
4794
|
+
);
|
|
3375
4795
|
},
|
|
3376
4796
|
/*
|
|
3377
4797
|
* A criteria that fires when the metric falls ranks the smallest
|
|
@@ -3386,12 +4806,12 @@ ${contextBlock}
|
|
|
3386
4806
|
* the pool and host it belongs to beneath it.
|
|
3387
4807
|
*/
|
|
3388
4808
|
private static getCephAffectedResourceEntry(input: {
|
|
3389
|
-
resource:
|
|
4809
|
+
resource: CephResourceIdentity;
|
|
3390
4810
|
clusterName: string;
|
|
3391
4811
|
metricName: string;
|
|
3392
4812
|
value: string;
|
|
3393
4813
|
}): AffectedResourceListEntry {
|
|
3394
|
-
const resource:
|
|
4814
|
+
const resource: CephResourceIdentity = input.resource;
|
|
3395
4815
|
|
|
3396
4816
|
/*
|
|
3397
4817
|
* The worker stores every series' `name` label as poolName, but only
|
|
@@ -3465,41 +4885,47 @@ ${contextBlock}
|
|
|
3465
4885
|
monitorStep: MonitorStep;
|
|
3466
4886
|
monitor: Monitor;
|
|
3467
4887
|
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
4888
|
+
perSeriesMatches?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
3468
4889
|
}): string | null {
|
|
3469
4890
|
const metricResponse: MetricMonitorResponse =
|
|
3470
4891
|
input.dataToProcess as MetricMonitorResponse;
|
|
3471
4892
|
|
|
3472
|
-
const breakdown: CephResourceBreakdown | undefined =
|
|
3473
|
-
metricResponse.cephResourceBreakdown;
|
|
3474
|
-
|
|
3475
|
-
const sections: Array<string> = [];
|
|
3476
|
-
|
|
3477
4893
|
// Ceph cluster context
|
|
3478
4894
|
const cephMonitor: MonitorStepCephMonitor | undefined =
|
|
3479
4895
|
input.monitorStep.data?.cephMonitor;
|
|
3480
4896
|
|
|
4897
|
+
const target: CriteriaMetricTarget | null =
|
|
4898
|
+
MonitorCriteriaEvaluator.resolveCriteriaMetricTarget({
|
|
4899
|
+
criteriaInstance: input.criteriaInstance,
|
|
4900
|
+
monitorStep: input.monitorStep,
|
|
4901
|
+
metricResponse: metricResponse,
|
|
4902
|
+
});
|
|
4903
|
+
|
|
4904
|
+
const breakdown: CephResourceBreakdown | undefined =
|
|
4905
|
+
MonitorCriteriaEvaluator.selectPlatformBreakdown({
|
|
4906
|
+
breakdowns: metricResponse.cephResourceBreakdowns,
|
|
4907
|
+
legacyBreakdown: metricResponse.cephResourceBreakdown,
|
|
4908
|
+
target: target,
|
|
4909
|
+
});
|
|
4910
|
+
|
|
4911
|
+
const clusterName: string =
|
|
4912
|
+
breakdown?.clusterName || cephMonitor?.clusterIdentifier || "Unknown";
|
|
4913
|
+
|
|
4914
|
+
const sections: Array<string> = [];
|
|
4915
|
+
|
|
3481
4916
|
if (cephMonitor || breakdown) {
|
|
3482
4917
|
const clusterDetails: Array<string> = [];
|
|
4918
|
+
clusterDetails.push(`- Cluster: ${clusterName}`);
|
|
3483
4919
|
clusterDetails.push(
|
|
3484
|
-
|
|
4920
|
+
...MonitorCriteriaEvaluator.describeCriteriaMetric({
|
|
4921
|
+
platform: "ceph",
|
|
4922
|
+
target: target,
|
|
4923
|
+
breakdown: breakdown,
|
|
4924
|
+
fallbackMetricName: cephMonitor?.metricViewConfig?.queryConfigs?.[0]
|
|
4925
|
+
?.metricQueryData?.filterData?.metricName as string | undefined,
|
|
4926
|
+
}),
|
|
3485
4927
|
);
|
|
3486
4928
|
|
|
3487
|
-
if (breakdown) {
|
|
3488
|
-
clusterDetails.push(
|
|
3489
|
-
`- Metric: ${breakdown.metricFriendlyName} (\`${breakdown.metricName}\`)`,
|
|
3490
|
-
);
|
|
3491
|
-
} else if (
|
|
3492
|
-
cephMonitor &&
|
|
3493
|
-
cephMonitor.metricViewConfig?.queryConfigs?.length > 0 &&
|
|
3494
|
-
cephMonitor.metricViewConfig.queryConfigs[0]
|
|
3495
|
-
) {
|
|
3496
|
-
const metricName: string = cephMonitor.metricViewConfig.queryConfigs[0]
|
|
3497
|
-
.metricQueryData?.filterData?.metricName as string;
|
|
3498
|
-
if (metricName) {
|
|
3499
|
-
clusterDetails.push(`- Metric: \`${metricName}\``);
|
|
3500
|
-
}
|
|
3501
|
-
}
|
|
3502
|
-
|
|
3503
4929
|
if (cephMonitor?.resourceFilters?.osdId) {
|
|
3504
4930
|
clusterDetails.push(
|
|
3505
4931
|
`- OSD Filter: ${cephMonitor.resourceFilters.osdId}`,
|
|
@@ -3518,68 +4944,94 @@ ${contextBlock}
|
|
|
3518
4944
|
// Affected resources: a ranked list of daemon, pool and host
|
|
3519
4945
|
let renderedBreakdownList: boolean = false;
|
|
3520
4946
|
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
4947
|
+
/*
|
|
4948
|
+
* Keep the rows that satisfy the criteria that just matched, worst
|
|
4949
|
+
* first, top 10 — NOT simply the non-zero rows. For ceph_osd_up /
|
|
4950
|
+
* ceph_osd_in / ceph_mon_quorum_status the criteria is `< 1`, so
|
|
4951
|
+
* the zero rows are the whole point of the incident.
|
|
4952
|
+
*/
|
|
4953
|
+
const rows: Array<PlatformAffectedRow<CephResourceIdentity>> | null =
|
|
4954
|
+
MonitorCriteriaEvaluator.getPlatformAffectedRows<
|
|
4955
|
+
CephResourceIdentity,
|
|
4956
|
+
CephAffectedResource
|
|
4957
|
+
>({
|
|
4958
|
+
platform: "ceph",
|
|
4959
|
+
perSeriesMatches: input.perSeriesMatches,
|
|
4960
|
+
namesObject: (identity: CephResourceIdentity): boolean => {
|
|
4961
|
+
return Boolean(
|
|
4962
|
+
identity.daemon ||
|
|
4963
|
+
identity.poolId ||
|
|
4964
|
+
identity.poolName ||
|
|
4965
|
+
identity.hostname,
|
|
4966
|
+
);
|
|
4967
|
+
},
|
|
4968
|
+
metricResponse: metricResponse,
|
|
4969
|
+
monitorStep: input.monitorStep,
|
|
4970
|
+
target: target,
|
|
4971
|
+
breakdown: breakdown,
|
|
4972
|
+
breach: MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
|
|
3530
4973
|
criteriaInstance: input.criteriaInstance,
|
|
3531
|
-
})
|
|
4974
|
+
}),
|
|
4975
|
+
toIdentity: PlatformResourceIdentity.ceph,
|
|
4976
|
+
withContext: PlatformResourceIdentity.withCephContext,
|
|
4977
|
+
});
|
|
3532
4978
|
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
4979
|
+
/*
|
|
4980
|
+
* Skip the list when no row carries any identity label
|
|
4981
|
+
* (cluster-wide series like ceph_health_status) — it would add
|
|
4982
|
+
* nothing.
|
|
4983
|
+
*/
|
|
4984
|
+
const hasIdentity: boolean = (rows || []).some(
|
|
4985
|
+
(row: PlatformAffectedRow<CephResourceIdentity>) => {
|
|
4986
|
+
return Boolean(
|
|
4987
|
+
row.seriesLabels ||
|
|
4988
|
+
row.identity.daemon ||
|
|
4989
|
+
row.identity.poolId ||
|
|
4990
|
+
row.identity.poolName ||
|
|
4991
|
+
row.identity.hostname,
|
|
4992
|
+
);
|
|
4993
|
+
},
|
|
4994
|
+
);
|
|
3544
4995
|
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
},
|
|
3554
|
-
);
|
|
4996
|
+
/*
|
|
4997
|
+
* Which metric the rows are about, for the entry's "Health Check"
|
|
4998
|
+
* titling: the criteria's own query, else the breakdown's.
|
|
4999
|
+
*/
|
|
5000
|
+
const rowsMetricName: string =
|
|
5001
|
+
MonitorCriteriaEvaluator.getAnalysisMetricName(target) ||
|
|
5002
|
+
breakdown?.metricName ||
|
|
5003
|
+
"";
|
|
3555
5004
|
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
5005
|
+
if (rows && rows.length > 0 && hasIdentity) {
|
|
5006
|
+
sections.push(
|
|
5007
|
+
AffectedResourceList.render({
|
|
5008
|
+
heading: "Affected Resources",
|
|
5009
|
+
overflowNoun: "affected resources",
|
|
5010
|
+
totalCount: rows.length,
|
|
5011
|
+
entries: rows
|
|
5012
|
+
.slice(0, AffectedResourceList.MAX_ENTRIES)
|
|
5013
|
+
.map(
|
|
5014
|
+
(
|
|
5015
|
+
row: PlatformAffectedRow<CephResourceIdentity>,
|
|
5016
|
+
): AffectedResourceListEntry => {
|
|
5017
|
+
return row.seriesLabels
|
|
5018
|
+
? MonitorCriteriaEvaluator.getSeriesLabelEntry({
|
|
5019
|
+
labels: row.seriesLabels,
|
|
5020
|
+
value:
|
|
5021
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
5022
|
+
})
|
|
5023
|
+
: MonitorCriteriaEvaluator.getCephAffectedResourceEntry({
|
|
5024
|
+
resource: row.identity,
|
|
5025
|
+
clusterName: clusterName,
|
|
5026
|
+
metricName: rowsMetricName,
|
|
5027
|
+
value:
|
|
5028
|
+
MonitorCriteriaEvaluator.renderAffectedRowValue(row),
|
|
5029
|
+
});
|
|
3577
5030
|
},
|
|
3578
5031
|
),
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
}
|
|
5032
|
+
}),
|
|
5033
|
+
);
|
|
5034
|
+
renderedBreakdownList = true;
|
|
3583
5035
|
}
|
|
3584
5036
|
|
|
3585
5037
|
// Metric results summary (fallback context when no list rendered)
|
|
@@ -3609,6 +5061,15 @@ ${contextBlock}
|
|
|
3609
5061
|
private static buildKubernetesRootCauseAnalysis(input: {
|
|
3610
5062
|
breakdown: KubernetesResourceBreakdown;
|
|
3611
5063
|
topResource: KubernetesAffectedResource;
|
|
5064
|
+
/*
|
|
5065
|
+
* The top resource's value, already rendered in the unit the criteria
|
|
5066
|
+
* compared it in. Required when the value did not come from the raw
|
|
5067
|
+
* scan of `breakdown.metricName` — a formula's value is a percentage
|
|
5068
|
+
* although its numerator (the metric the branches below key on) is
|
|
5069
|
+
* bytes.
|
|
5070
|
+
*/
|
|
5071
|
+
topResourceValue?: string | undefined;
|
|
5072
|
+
target?: CriteriaMetricTarget | null | undefined;
|
|
3612
5073
|
}): string | null {
|
|
3613
5074
|
const { breakdown, topResource } = input;
|
|
3614
5075
|
const metricName: string = breakdown.metricName;
|
|
@@ -3626,14 +5087,14 @@ ${contextBlock}
|
|
|
3626
5087
|
* kubeletstats cores gauge that four other places in this repo warn
|
|
3627
5088
|
* is misnamed, so 1.4 cores in use read as "1.4% CPU utilization".
|
|
3628
5089
|
*/
|
|
3629
|
-
const topResourceValue: string =
|
|
3630
|
-
|
|
3631
|
-
|
|
5090
|
+
const topResourceValue: string =
|
|
5091
|
+
input.topResourceValue ||
|
|
5092
|
+
MonitorCriteriaEvaluator.formatPlatformMetricValue({
|
|
3632
5093
|
platform: "kubernetes",
|
|
3633
5094
|
metricName: metricName,
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
5095
|
+
metricUnit: breakdown.metricUnit,
|
|
5096
|
+
value: topResource.metricValue,
|
|
5097
|
+
});
|
|
3637
5098
|
|
|
3638
5099
|
if (
|
|
3639
5100
|
metricName === "k8s.container.restarts" ||
|
|
@@ -3691,6 +5152,12 @@ ${contextBlock}
|
|
|
3691
5152
|
);
|
|
3692
5153
|
} else if (
|
|
3693
5154
|
metricName === "k8s.node.cpu.utilization" ||
|
|
5155
|
+
/*
|
|
5156
|
+
* The numerator of the "High Node CPU Utilization" ratio template
|
|
5157
|
+
* (k8s.node.cpu.usage ÷ k8s.node.allocatable_cpu × 100), which is
|
|
5158
|
+
* what a formula criteria is analysed as.
|
|
5159
|
+
*/
|
|
5160
|
+
metricName === "k8s.node.cpu.usage" ||
|
|
3694
5161
|
/*
|
|
3695
5162
|
* Node-scoped ONLY. The bare substring test used to swallow every
|
|
3696
5163
|
* pod- and container-scoped CPU metric — `k8s.pod.cpu.utilization`,
|
|
@@ -3738,7 +5205,14 @@ ${contextBlock}
|
|
|
3738
5205
|
`Recommended actions: Check memory consumers with \`kubectl top pods --all-namespaces --sort-by=memory\` and review pod memory limits. Consider scaling the cluster or adding nodes with more memory.`,
|
|
3739
5206
|
);
|
|
3740
5207
|
} else if (
|
|
3741
|
-
|
|
5208
|
+
/*
|
|
5209
|
+
* The k8s_cluster receiver has no unavailable-replicas series. The
|
|
5210
|
+
* replica-mismatch template is a formula (`desired - available`),
|
|
5211
|
+
* analysed as its first operand, k8s.deployment.desired, so it lands
|
|
5212
|
+
* in the generic branch below, which names the formula. What still
|
|
5213
|
+
* reaches this one is a custom monitor on an "unavailable" series
|
|
5214
|
+
* such as kube-state-metrics' `kube_deployment_status_replicas_unavailable`.
|
|
5215
|
+
*/
|
|
3742
5216
|
metricName.includes("unavailable")
|
|
3743
5217
|
) {
|
|
3744
5218
|
lines.push(
|
|
@@ -3807,15 +5281,48 @@ ${contextBlock}
|
|
|
3807
5281
|
`Recommended actions: Check DaemonSet status with \`kubectl describe daemonset ${topResource.workloadName || "<daemonset>"}\` and verify node labels and taints.`,
|
|
3808
5282
|
);
|
|
3809
5283
|
} else {
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
5284
|
+
/*
|
|
5285
|
+
* Generic Kubernetes context. It names what the criteria compared —
|
|
5286
|
+
* a formula by its legend and expression, since the metric the
|
|
5287
|
+
* branches above key on is only its numerator — and the worst
|
|
5288
|
+
* resource's value, so the reader does not have to find it in the
|
|
5289
|
+
* list above.
|
|
5290
|
+
*/
|
|
5291
|
+
const target: CriteriaMetricTarget | null | undefined = input.target;
|
|
5292
|
+
|
|
5293
|
+
if (target?.isFormula) {
|
|
5294
|
+
const formulaName: string = target.displayName || `\`${target.alias}\``;
|
|
5295
|
+
const expression: string = target.formulaExpression
|
|
5296
|
+
? ` (\`${target.formulaExpression}\`)`
|
|
5297
|
+
: "";
|
|
5298
|
+
lines.push(
|
|
5299
|
+
`${formulaName}${expression} has breached the configured threshold.`,
|
|
5300
|
+
);
|
|
5301
|
+
} else {
|
|
5302
|
+
const friendlyName: string =
|
|
5303
|
+
breakdown.metricFriendlyName &&
|
|
5304
|
+
breakdown.metricFriendlyName !== metricName
|
|
5305
|
+
? ` (${breakdown.metricFriendlyName})`
|
|
5306
|
+
: "";
|
|
5307
|
+
lines.push(
|
|
5308
|
+
`Kubernetes metric \`${metricName}\`${friendlyName} has breached the configured threshold.`,
|
|
5309
|
+
);
|
|
5310
|
+
}
|
|
3814
5311
|
if (topResource.podName) {
|
|
3815
|
-
lines.push(
|
|
5312
|
+
lines.push(
|
|
5313
|
+
`Most affected pod: \`${topResource.podName}\` (**${topResourceValue}**)`,
|
|
5314
|
+
);
|
|
5315
|
+
} else if (topResource.workloadName) {
|
|
5316
|
+
lines.push(
|
|
5317
|
+
`Most affected ${topResource.workloadType || "workload"}: \`${topResource.workloadName}\` (**${topResourceValue}**)`,
|
|
5318
|
+
);
|
|
3816
5319
|
}
|
|
3817
5320
|
if (topResource.nodeName) {
|
|
3818
|
-
lines.push(
|
|
5321
|
+
lines.push(
|
|
5322
|
+
topResource.podName
|
|
5323
|
+
? `Most affected node: \`${topResource.nodeName}\``
|
|
5324
|
+
: `Most affected node: \`${topResource.nodeName}\` (**${topResourceValue}**)`,
|
|
5325
|
+
);
|
|
3819
5326
|
}
|
|
3820
5327
|
lines.push(
|
|
3821
5328
|
`Recommended actions: Investigate the affected resources using \`kubectl describe\` and \`kubectl logs\` commands.`,
|