@oneuptime/common 13.0.8 → 14.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/AuditLog.ts +23 -15
- package/Models/DatabaseModels/GlobalConfig.ts +24 -0
- package/Models/DatabaseModels/Index.ts +4 -0
- package/Models/DatabaseModels/Monitor.ts +3 -2
- package/Models/DatabaseModels/MonitorTemplate.ts +2 -1
- package/Models/DatabaseModels/ServiceLevelObjectiveFeed.ts +2 -0
- package/Models/DatabaseModels/ServiceLevelObjectiveLabelRule.ts +522 -0
- package/Models/DatabaseModels/ServiceLevelObjectiveMonitorRule.ts +38 -4
- package/Models/DatabaseModels/ServiceLevelObjectiveOwnerRule.ts +603 -0
- package/Scripts/generate-service-worker.js +65 -4
- package/Server/API/BaseAPI.ts +26 -5
- package/Server/API/GlobalConfigAPI.ts +228 -462
- package/Server/API/ProjectOIDC.ts +25 -16
- package/Server/API/ProjectSSO.ts +24 -16
- package/Server/API/StatusPageAPI.ts +100 -42
- package/Server/API/TeamMemberAPI.ts +10 -1
- package/Server/Enterprise/EnterpriseEdition.ts +649 -0
- package/Server/Enterprise/EnterpriseFeature.ts +48 -0
- package/Server/Enterprise/EnterpriseLicenseSnapshot.ts +192 -0
- package/Server/Enterprise/EnterpriseServerModule.ts +233 -0
- package/Server/EnvironmentConfig.ts +147 -2
- package/Server/Infrastructure/Postgres/SchemaMigrations/1793900000000-AddEnterpriseEditionFirstSeenAt.ts +25 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794100000000-AddSloLabelAndOwnerRules.ts +248 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794200000000-AddSloMonitorRuleMonitorType.ts +19 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +6 -0
- package/Server/Middleware/UserAuthorization.ts +38 -16
- package/Server/Services/AuditLogService.ts +84 -1079
- package/Server/Services/DatabaseService.ts +10 -0
- package/Server/Services/EnterpriseLicenseService.ts +38 -0
- package/Server/Services/GlobalConfigService.ts +80 -1
- package/Server/Services/Index.ts +4 -0
- package/Server/Services/MonitorProbeService.ts +116 -20
- package/Server/Services/MonitorService.ts +59 -6
- package/Server/Services/MonitorStatusTimelineService.ts +284 -0
- package/Server/Services/MonitorTemplateService.ts +26 -0
- package/Server/Services/ProjectService.ts +284 -0
- package/Server/Services/RumSessionPinService.ts +25 -6
- package/Server/Services/ServiceLevelObjectiveLabelRuleEngineService.ts +385 -0
- package/Server/Services/ServiceLevelObjectiveLabelRuleService.ts +51 -0
- package/Server/Services/ServiceLevelObjectiveMonitorRuleEngineService.ts +92 -16
- package/Server/Services/ServiceLevelObjectiveMonitorRuleService.ts +36 -1
- package/Server/Services/ServiceLevelObjectiveOwnerRuleEngineService.ts +521 -0
- package/Server/Services/ServiceLevelObjectiveOwnerRuleService.ts +51 -0
- package/Server/Services/ServiceLevelObjectiveService.ts +28 -0
- package/Server/Services/StatusPagePrivateUserService.ts +7 -3
- package/Server/Services/StatusPageService.ts +78 -1
- package/Server/Services/TeamMemberService.ts +11 -0
- package/Server/Services/TeamService.ts +10 -0
- package/Server/Services/UserService.ts +10 -17
- package/Server/Types/Database/Permissions/CreatePermission.ts +15 -0
- package/Server/Types/Database/Permissions/EditionPermission.ts +208 -18
- package/Server/Types/Database/Permissions/TablePermission.ts +10 -3
- package/Server/Types/Database/Permissions/UpdatePermission.ts +29 -0
- package/Server/Types/Markdown.ts +41 -0
- package/Server/Types/Workflow/Components/Telegram/SendMessageToChat.ts +27 -1
- package/Server/Utils/AI/Toolbox/SloTools.ts +1 -0
- package/Server/Utils/CommunityEditionSsoReport.ts +361 -0
- package/Server/Utils/EditionEnforcement.ts +233 -0
- package/Server/Utils/FrontendEnvironment.ts +41 -2
- package/Server/Utils/InstanceHealth/RedisHealth.ts +12 -277
- package/Server/Utils/Monitor/AffectedResourceList.ts +148 -0
- package/Server/Utils/Monitor/Criteria/CompareCriteria.ts +1 -1
- package/Server/Utils/Monitor/Criteria/EvaluateOverTime.ts +10 -1
- package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts +1 -1
- package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts +720 -465
- package/Server/Utils/Monitor/MonitoringIntervalValidator.ts +116 -0
- package/Server/Utils/Monitor/RootCauseList.ts +139 -0
- package/Server/Utils/Rules/RuleRun/RuleRunRegistry.ts +28 -0
- package/Server/Utils/Slo/SloRulePatternValidator.ts +42 -0
- package/Server/Utils/Workspace/Slack/Slack.ts +423 -10
- package/Tests/App/Accounts/LoginPasskey.test.tsx +18 -0
- package/Tests/App/Accounts/LoginSsoEdition.test.tsx +616 -0
- package/Tests/App/AdminDashboard/AdminAuthenticationSsoEdition.test.tsx +217 -0
- package/Tests/App/AdminDashboard/AdminEnterprisePluginPage.test.tsx +357 -0
- package/Tests/App/AdminDashboard/EnterpriseLicensesShell.test.tsx +214 -0
- package/Tests/App/AdminDashboard/GlobalSSOEnterpriseShells.test.tsx +271 -0
- package/Tests/App/AdminDashboard/HealthShells.test.tsx +659 -0
- package/Tests/App/Dashboard/AuditLogsAndComplianceShells.test.tsx +572 -0
- package/Tests/App/Dashboard/EnterpriseFeatureUpgrade.test.tsx +547 -0
- package/Tests/App/Dashboard/EnterprisePluginPage.test.tsx +667 -0
- package/Tests/App/Dashboard/FeedFilterSortIntegration.test.tsx +1632 -0
- package/Tests/App/Dashboard/FeedOptionsLocales.test.ts +250 -0
- package/Tests/App/Dashboard/FeedOrdering.test.ts +279 -9
- package/Tests/App/Dashboard/IdentityEnterpriseShells.test.tsx +319 -0
- package/Tests/App/Dashboard/InvestigationFeedRefresh.test.tsx +60 -1
- package/Tests/App/Dashboard/InvestigationPanel.test.tsx +594 -3
- package/Tests/App/Dashboard/OnboardingSsoEmptyState.test.tsx +185 -0
- package/Tests/App/Dashboard/ProductOrientation.test.tsx +646 -0
- package/Tests/App/Dashboard/ResourceFeed.test.tsx +612 -2
- package/Tests/App/Dashboard/ResourceFeedPageWiring.test.ts +30 -0
- package/Tests/App/Dashboard/ResourceFeedPagesFilterSort.test.tsx +1202 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceFeedRefresh.test.tsx +52 -1
- package/Tests/App/Dashboard/SecurityEventConnectionDiagnostics.test.tsx +45 -9
- package/Tests/App/Dashboard/SecurityEventDetailPanel.test.tsx +399 -0
- package/Tests/App/Dashboard/SecurityEventListRow.test.tsx +214 -0
- package/Tests/App/Dashboard/SecurityEventsEmptyState.test.tsx +1 -83
- package/Tests/App/Dashboard/SecurityEventsNavTabs.test.tsx +248 -0
- package/Tests/App/Dashboard/SecurityEventsViewer.test.tsx +1203 -0
- package/Tests/App/Dashboard/SloFeed.test.tsx +106 -1
- package/Tests/App/Dashboard/SloMonitorRuleCriteria.test.tsx +566 -0
- package/Tests/App/Dashboard/SloNavigationWiring.test.tsx +6 -4
- package/Tests/App/Dashboard/SloNoticeBanner.test.tsx +186 -0
- package/Tests/App/Dashboard/SloOverviewKpiStrip.test.tsx +388 -44
- package/Tests/App/Dashboard/SloOverviewLayout.test.tsx +613 -0
- package/Tests/App/Dashboard/UserEmailCallSites.test.tsx +11 -91
- package/Tests/App/StatusPage/StatusPageSsoEmptyState.test.tsx +319 -0
- package/Tests/Helpers/RefreshAwareApiScan.ts +757 -0
- package/Tests/Helpers/TenantScopedRequestScan.ts +515 -0
- package/Tests/Models/DatabaseModels/ResourceFeedModels.test.ts +0 -8
- package/Tests/Models/DatabaseModels/SloLabelAndOwnerRuleModels.test.ts +404 -0
- package/Tests/Models/DatabaseModels/SloOverhaulModels.test.ts +17 -1
- package/Tests/Models/DatabaseModels/StaticRuleCriteriaInheritance.test.ts +4 -2
- package/Tests/Scripts/GenerateServiceWorker.test.ts +931 -0
- package/Tests/Server/API/BaseAPICreateItemDoesNotMutateModel.test.ts +130 -0
- package/Tests/Server/API/GlobalConfigLicense.test.ts +742 -497
- package/Tests/Server/API/OnCallReadinessAPI.test.ts +35 -909
- package/Tests/Server/API/SsoProviderListingsEdition.test.ts +432 -0
- package/Tests/Server/API/TeamMemberLeaveScimLockEdition.test.ts +252 -0
- package/Tests/Server/Enterprise/EnterpriseEdition.test.ts +824 -0
- package/Tests/Server/Enterprise/EnterpriseFeatureActive.test.ts +643 -0
- package/Tests/Server/Enterprise/EnterpriseLicenseSnapshot.test.ts +294 -0
- package/Tests/Server/Enterprise/EnterpriseModelFeatureMap.test.ts +93 -0
- package/Tests/Server/Enterprise/EnterpriseOnlyModulesLiveInEe.test.ts +582 -0
- package/Tests/Server/Enterprise/EnterpriseServerModule.test.ts +247 -0
- package/Tests/Server/Enterprise/FakeEnterpriseModule.ts +502 -0
- package/Tests/Server/Enterprise/TestBillingFlag.ts +70 -0
- package/Tests/Server/EnvironmentConfigEnterpriseEdition.test.ts +314 -0
- package/Tests/Server/Infrastructure/ContainerBootConfiguration.test.ts +289 -0
- package/Tests/Server/Infrastructure/E2EClickHouseFixtureAccess.test.ts +5 -1
- package/Tests/Server/Infrastructure/Postgres/AddConfigurableRuleCriteriaMigration.test.ts +3 -0
- package/Tests/Server/Infrastructure/Postgres/AddSloLabelAndOwnerRulesMigration.test.ts +407 -0
- package/Tests/Server/Infrastructure/Postgres/AddSloMonitorRuleMonitorTypeMigration.test.ts +171 -0
- package/Tests/Server/Infrastructure/Postgres/SloProductOverhaulMigration.test.ts +88 -4
- package/Tests/Server/Middleware/UserAuthorization.test.ts +37 -2
- package/Tests/Server/Middleware/UserAuthorizationEditionEnforcement.test.ts +516 -0
- package/Tests/Server/Services/AuditLogService.test.ts +460 -990
- package/Tests/Server/Services/CreateAndReadResponsesKeepIds.test.ts +263 -0
- package/Tests/Server/Services/DatabaseServiceRuleCriteriaValidation.test.ts +104 -0
- package/Tests/Server/Services/EnterpriseLicenseServiceKeyGeneration.test.ts +151 -0
- package/Tests/Server/Services/GlobalConfigServiceLicenseColumns.test.ts +357 -0
- package/Tests/Server/Services/MonitorDestinationInfoIPv6.test.ts +101 -0
- package/Tests/Server/Services/MonitorProbeServiceIntervalScheduling.test.ts +333 -0
- package/Tests/Server/Services/MonitorStatusTimelineDailyAggregates.test.ts +303 -0
- package/Tests/Server/Services/ProjectServiceAuditLogLicense.test.ts +701 -0
- package/Tests/Server/Services/ProjectServiceAuditLogSettingsCache.test.ts +31 -35
- package/Tests/Server/Services/RuleRun/LabelRuleEnginesGroupBRun.test.ts +48 -1
- package/Tests/Server/Services/RuleRun/OwnerRuleEnginesGroupBRun.test.ts +30 -0
- package/Tests/Server/Services/ScimPushGroupsTeamLocksEdition.test.ts +431 -0
- package/Tests/Server/Services/ServiceLevelObjectiveLabelAndOwnerRuleEngines.test.ts +1170 -0
- package/Tests/Server/Services/ServiceLevelObjectiveLabelAndOwnerRuleServices.test.ts +172 -0
- package/Tests/Server/Services/ServiceLevelObjectiveMonitorRuleEngineService.test.ts +323 -0
- package/Tests/Server/Services/ServiceLevelObjectiveMonitorRuleService.test.ts +115 -0
- package/Tests/Server/Services/ServiceLevelObjectiveMonitorRuleTriggers.test.ts +21 -2
- package/Tests/Server/Services/ServiceLevelObjectiveMonitorTypePostgres.test.ts +480 -0
- package/Tests/Server/Services/ServiceLevelObjectiveService.test.ts +162 -0
- package/Tests/Server/Services/SsoRequirementReadMasking.test.ts +419 -0
- package/Tests/Server/Services/SsoRequirementWriteGuard.test.ts +769 -0
- package/Tests/Server/Services/StaticRuleCriteriaRuntimeCoverage.test.ts +45 -6
- package/Tests/Server/Services/StatusPagePrivateUserInvitationPolicy.test.ts +94 -0
- package/Tests/Server/Services/UserAuthenticationService.test.ts +35 -0
- package/Tests/Server/Services/UserServiceSeatLimit.test.ts +94 -56
- package/Tests/Server/Types/Database/FeedEventTypeFilterQuery.test.ts +745 -0
- package/Tests/Server/Types/Database/Permissions/EditionPermission.test.ts +1823 -0
- package/Tests/Server/Types/Markdown.test.ts +183 -1
- package/Tests/Server/Types/Workflow/Components/ApiComponentSsrf.test.ts +49 -0
- package/Tests/Server/Types/Workflow/Components/TelegramSendMessageToChat.test.ts +331 -0
- package/Tests/Server/Utils/AI/SloTools.test.ts +10 -0
- package/Tests/Server/Utils/CommunityEditionSsoReport.test.ts +565 -0
- package/Tests/Server/Utils/EditionEnforcement.test.ts +303 -0
- package/Tests/Server/Utils/FrontendEnvironment.test.ts +367 -0
- package/Tests/Server/Utils/InstanceHealth/RedisHealth.test.ts +38 -418
- package/Tests/Server/Utils/Monitor/AffectedResourceList.test.ts +518 -0
- package/Tests/Server/Utils/Monitor/Criteria/EvaluateOverTime.test.ts +24 -4
- package/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteriaAnomalyUnits.test.ts +1 -1
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluator.test.ts +1339 -96
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorBreachingSamples.test.ts +1101 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorMetricUnits.test.ts +106 -80
- package/Tests/Server/Utils/Monitor/MonitoringIntervalValidator.test.ts +188 -0
- package/Tests/Server/Utils/Monitor/RootCauseList.test.ts +252 -0
- package/Tests/Server/Utils/Rules/RuleRun/RuleRunPermission.test.ts +154 -0
- package/Tests/Server/Utils/SSRFProtection.test.ts +111 -0
- package/Tests/Server/Utils/Slo/SloRulePatternValidator.test.ts +67 -0
- package/Tests/Server/Utils/VM/VMRunnerSsrf.test.ts +213 -0
- package/Tests/Server/Utils/Workspace/Slack/SlackMarkdownSectionSplit.test.ts +1209 -0
- package/Tests/Types/API/HostnameFromAuthority.test.ts +20 -10
- package/Tests/Types/API/HostnameIPv6.test.ts +149 -0
- package/Tests/Types/API/Route.test.ts +56 -0
- package/Tests/Types/API/URLSchemeLikePath.test.ts +304 -0
- package/Tests/Types/Database/CanonicalModelInstance.test.ts +190 -0
- package/Tests/UI/Components/EditionLabelLapseCopy.ts +104 -0
- package/Tests/UI/Components/EditionLabelLicenseManagerWiring.test.tsx +254 -0
- package/Tests/UI/Components/EditionLabelLicenseRefresh.test.tsx +84 -178
- package/Tests/UI/Components/EditionLabelLicenseStatus.test.tsx +1586 -0
- package/Tests/UI/Components/EditionLabelLicenseWritesGuard.test.tsx +157 -0
- package/Tests/UI/Components/FeedOptions.test.ts +1404 -0
- package/Tests/UI/Components/FeedOptionsButton.test.tsx +2603 -0
- package/Tests/UI/Components/Graphs/DayUptimeGraph.test.tsx +49 -2
- package/Tests/UI/Components/Icon.test.tsx +26 -0
- package/Tests/UI/Components/MarkdownConverters.test.ts +684 -0
- package/Tests/UI/Components/ModelListEmptyState.test.tsx +289 -0
- package/Tests/UI/Components/RuleCriteria/RuleCriteriaModelTable.test.tsx +3 -3
- package/Tests/UI/Components/TelemetryViewer/TelemetryViewerEmptyContent.test.tsx +131 -0
- package/Tests/UI/Components/useFeedItemsViewKey.test.tsx +1017 -0
- package/Tests/UI/Components/useFeedOptions.test.tsx +1107 -0
- package/Tests/UI/EsbuildEnterpriseAlias.test.ts +1127 -0
- package/Tests/Utils/HostAddressUtil.test.ts +138 -0
- package/Tests/Utils/LabelRuleImportExport.test.ts +102 -0
- package/Tests/Utils/Monitor/MonitorDestinationUtil.test.ts +313 -0
- package/Tests/Utils/Monitor/MonitoringIntervalUtil.test.ts +304 -0
- package/Tests/Utils/Slo/SloDuration.test.ts +115 -24
- package/Tests/Utils/Slo/SloHealth.test.ts +93 -10
- package/Tests/Utils/Slo/SloMonitorRuleCriteria.test.ts +109 -0
- package/Tests/Utils/Slo/SloOverviewText.test.ts +64 -0
- package/Tests/Utils/StatusPage/UptimeDailyAggregateUtil.test.ts +248 -0
- package/Types/API/Hostname.ts +63 -16
- package/Types/API/Route.ts +19 -2
- package/Types/API/URL.ts +26 -2
- package/Types/Database/AccessControl/ColumnAccessControl.ts +19 -7
- package/Types/Database/AccessControl/ColumnBillingAccessControl.ts +19 -7
- package/Types/Database/AccessControl/OwnerOnlyColumn.ts +15 -6
- package/Types/Database/CanonicalModelInstance.ts +102 -0
- package/Types/Database/TableColumn.ts +20 -6
- package/Types/EnterpriseLicense/EnterpriseLicensePeriods.ts +29 -0
- package/Types/IP/IPv6.ts +6 -1
- package/Types/Monitor/MetricMonitor/MetricCriteriaContext.ts +6 -5
- package/Types/Monitor/MetricMonitor/MetricMonitorResponse.ts +9 -0
- package/Types/Monitor/MonitorStep.ts +6 -1
- package/Types/Monitor/ProxmoxAlertTemplates.ts +1 -1
- package/Types/Permission.ts +90 -0
- package/Types/Rules/RuleCriteriaFieldRegistry.ts +11 -0
- package/Types/Rules/RuleRun.ts +4 -0
- package/Types/StatusPage/UptimeDailyAggregate.ts +110 -0
- package/UI/Components/EditionLabel/EditionLabel.tsx +796 -1195
- package/UI/Components/EditionLabel/LicenseManager.ts +119 -0
- package/UI/Components/Feed/FeedOptions.ts +427 -0
- package/UI/Components/Feed/FeedOptionsButton.tsx +692 -0
- package/UI/Components/Feed/useFeedItems.ts +28 -9
- package/UI/Components/Feed/useFeedOptions.ts +225 -0
- package/UI/Components/Graphs/DayUptimeGraph.tsx +123 -15
- package/UI/Components/Icon/Icon.tsx +1 -1
- package/UI/Components/Markdown.tsx/MarkdownConverters.ts +0 -0
- package/UI/Components/ModelList/ModelList.tsx +11 -1
- package/UI/Components/MonitorGraphs/Uptime.tsx +69 -0
- package/UI/Components/TelemetryViewer/TelemetryViewer.tsx +28 -12
- package/UI/esbuild-enterprise.js +263 -0
- package/Utils/EnterpriseLicense/EnterpriseLicenseUsage.ts +7 -2
- package/Utils/HostAddressUtil.ts +94 -0
- package/Utils/LabelRuleImportExport.ts +27 -9
- package/Utils/Monitor/MetricValueFormatter.ts +2 -2
- package/Utils/Monitor/MonitorDestinationUtil.ts +157 -0
- package/Utils/Monitor/MonitoringIntervalUtil.ts +436 -0
- package/Utils/Rules/MonitorTypeRuleCriteria.ts +31 -0
- package/Utils/Slo/SloDuration.ts +22 -6
- package/Utils/Slo/SloHealth.ts +8 -7
- package/Utils/Slo/SloMonitorRuleCriteria.ts +33 -1
- package/Utils/Slo/SloOverviewText.ts +33 -0
- package/Utils/Slo/SloProjection.ts +4 -3
- package/Utils/StatusPage/UptimeDailyAggregateUtil.ts +192 -0
- package/build/dist/Models/AnalyticsModels/AuditLog.js +23 -15
- package/build/dist/Models/AnalyticsModels/AuditLog.js.map +1 -1
- package/build/dist/Models/DatabaseModels/GlobalConfig.js +25 -0
- package/build/dist/Models/DatabaseModels/GlobalConfig.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +4 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Monitor.js +2 -2
- package/build/dist/Models/DatabaseModels/Monitor.js.map +1 -1
- package/build/dist/Models/DatabaseModels/MonitorTemplate.js +2 -1
- package/build/dist/Models/DatabaseModels/MonitorTemplate.js.map +1 -1
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveFeed.js +2 -0
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveFeed.js.map +1 -1
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveLabelRule.js +530 -0
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveLabelRule.js.map +1 -0
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveMonitorRule.js +39 -4
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveMonitorRule.js.map +1 -1
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveOwnerRule.js +612 -0
- package/build/dist/Models/DatabaseModels/ServiceLevelObjectiveOwnerRule.js.map +1 -0
- package/build/dist/Server/API/BaseAPI.js +26 -5
- package/build/dist/Server/API/BaseAPI.js.map +1 -1
- package/build/dist/Server/API/GlobalConfigAPI.js +148 -344
- package/build/dist/Server/API/GlobalConfigAPI.js.map +1 -1
- package/build/dist/Server/API/ProjectOIDC.js +24 -16
- package/build/dist/Server/API/ProjectOIDC.js.map +1 -1
- package/build/dist/Server/API/ProjectSSO.js +24 -16
- package/build/dist/Server/API/ProjectSSO.js.map +1 -1
- package/build/dist/Server/API/StatusPageAPI.js +93 -43
- package/build/dist/Server/API/StatusPageAPI.js.map +1 -1
- package/build/dist/Server/API/TeamMemberAPI.js +8 -1
- package/build/dist/Server/API/TeamMemberAPI.js.map +1 -1
- package/build/dist/Server/Enterprise/EnterpriseEdition.js +466 -0
- package/build/dist/Server/Enterprise/EnterpriseEdition.js.map +1 -0
- package/build/dist/Server/Enterprise/EnterpriseFeature.js +39 -0
- package/build/dist/Server/Enterprise/EnterpriseFeature.js.map +1 -0
- package/build/dist/Server/Enterprise/EnterpriseLicenseSnapshot.js +70 -0
- package/build/dist/Server/Enterprise/EnterpriseLicenseSnapshot.js.map +1 -0
- package/build/dist/Server/Enterprise/EnterpriseServerModule.js +84 -0
- package/build/dist/Server/Enterprise/EnterpriseServerModule.js.map +1 -0
- package/build/dist/Server/EnvironmentConfig.js +84 -2
- package/build/dist/Server/EnvironmentConfig.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1793900000000-AddEnterpriseEditionFirstSeenAt.js +18 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1793900000000-AddEnterpriseEditionFirstSeenAt.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794100000000-AddSloLabelAndOwnerRules.js +93 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794100000000-AddSloLabelAndOwnerRules.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794200000000-AddSloMonitorRuleMonitorType.js +12 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794200000000-AddSloMonitorRuleMonitorType.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +6 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Middleware/UserAuthorization.js +30 -11
- package/build/dist/Server/Middleware/UserAuthorization.js.map +1 -1
- package/build/dist/Server/Services/AuditLogService.js +71 -740
- package/build/dist/Server/Services/AuditLogService.js.map +1 -1
- package/build/dist/Server/Services/DatabaseService.js +7 -0
- package/build/dist/Server/Services/DatabaseService.js.map +1 -1
- package/build/dist/Server/Services/EnterpriseLicenseService.js +34 -0
- package/build/dist/Server/Services/EnterpriseLicenseService.js.map +1 -1
- package/build/dist/Server/Services/GlobalConfigService.js +63 -0
- package/build/dist/Server/Services/GlobalConfigService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +4 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/MonitorProbeService.js +90 -17
- package/build/dist/Server/Services/MonitorProbeService.js.map +1 -1
- package/build/dist/Server/Services/MonitorService.js +48 -6
- package/build/dist/Server/Services/MonitorService.js.map +1 -1
- package/build/dist/Server/Services/MonitorStatusTimelineService.js +242 -0
- package/build/dist/Server/Services/MonitorStatusTimelineService.js.map +1 -1
- package/build/dist/Server/Services/MonitorTemplateService.js +16 -0
- package/build/dist/Server/Services/MonitorTemplateService.js.map +1 -1
- package/build/dist/Server/Services/ProjectService.js +214 -0
- package/build/dist/Server/Services/ProjectService.js.map +1 -1
- package/build/dist/Server/Services/RumSessionPinService.js +24 -6
- package/build/dist/Server/Services/RumSessionPinService.js.map +1 -1
- package/build/dist/Server/Services/ServiceLevelObjectiveLabelRuleEngineService.js +295 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveLabelRuleEngineService.js.map +1 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveLabelRuleService.js +56 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveLabelRuleService.js.map +1 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveMonitorRuleEngineService.js +59 -14
- package/build/dist/Server/Services/ServiceLevelObjectiveMonitorRuleEngineService.js.map +1 -1
- package/build/dist/Server/Services/ServiceLevelObjectiveMonitorRuleService.js +22 -1
- package/build/dist/Server/Services/ServiceLevelObjectiveMonitorRuleService.js.map +1 -1
- package/build/dist/Server/Services/ServiceLevelObjectiveOwnerRuleEngineService.js +400 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveOwnerRuleEngineService.js.map +1 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveOwnerRuleService.js +56 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveOwnerRuleService.js.map +1 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveService.js +21 -0
- package/build/dist/Server/Services/ServiceLevelObjectiveService.js.map +1 -1
- package/build/dist/Server/Services/StatusPagePrivateUserService.js +7 -3
- package/build/dist/Server/Services/StatusPagePrivateUserService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageService.js +71 -0
- package/build/dist/Server/Services/StatusPageService.js.map +1 -1
- package/build/dist/Server/Services/TeamMemberService.js +10 -0
- package/build/dist/Server/Services/TeamMemberService.js.map +1 -1
- package/build/dist/Server/Services/TeamService.js +9 -0
- package/build/dist/Server/Services/TeamService.js.map +1 -1
- package/build/dist/Server/Services/UserService.js +10 -16
- package/build/dist/Server/Services/UserService.js.map +1 -1
- package/build/dist/Server/Types/Database/Permissions/CreatePermission.js +10 -0
- package/build/dist/Server/Types/Database/Permissions/CreatePermission.js.map +1 -1
- package/build/dist/Server/Types/Database/Permissions/EditionPermission.js +154 -14
- package/build/dist/Server/Types/Database/Permissions/EditionPermission.js.map +1 -1
- package/build/dist/Server/Types/Database/Permissions/TablePermission.js +10 -3
- package/build/dist/Server/Types/Database/Permissions/TablePermission.js.map +1 -1
- package/build/dist/Server/Types/Database/Permissions/UpdatePermission.js +17 -0
- package/build/dist/Server/Types/Database/Permissions/UpdatePermission.js.map +1 -1
- package/build/dist/Server/Types/Markdown.js +31 -0
- package/build/dist/Server/Types/Markdown.js.map +1 -1
- package/build/dist/Server/Types/Workflow/Components/Telegram/SendMessageToChat.js +20 -1
- package/build/dist/Server/Types/Workflow/Components/Telegram/SendMessageToChat.js.map +1 -1
- package/build/dist/Server/Utils/AI/Toolbox/SloTools.js +1 -0
- package/build/dist/Server/Utils/AI/Toolbox/SloTools.js.map +1 -1
- package/build/dist/Server/Utils/CommunityEditionSsoReport.js +223 -0
- package/build/dist/Server/Utils/CommunityEditionSsoReport.js.map +1 -0
- package/build/dist/Server/Utils/EditionEnforcement.js +187 -0
- package/build/dist/Server/Utils/EditionEnforcement.js.map +1 -0
- package/build/dist/Server/Utils/FrontendEnvironment.js +31 -2
- package/build/dist/Server/Utils/FrontendEnvironment.js.map +1 -1
- package/build/dist/Server/Utils/InstanceHealth/RedisHealth.js +6 -193
- package/build/dist/Server/Utils/InstanceHealth/RedisHealth.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/AffectedResourceList.js +67 -0
- package/build/dist/Server/Utils/Monitor/AffectedResourceList.js.map +1 -0
- package/build/dist/Server/Utils/Monitor/Criteria/CompareCriteria.js +1 -1
- package/build/dist/Server/Utils/Monitor/Criteria/EvaluateOverTime.js +8 -1
- package/build/dist/Server/Utils/Monitor/Criteria/EvaluateOverTime.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.js +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js +585 -367
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitoringIntervalValidator.js +88 -0
- package/build/dist/Server/Utils/Monitor/MonitoringIntervalValidator.js.map +1 -0
- package/build/dist/Server/Utils/Monitor/RootCauseList.js +101 -0
- package/build/dist/Server/Utils/Monitor/RootCauseList.js.map +1 -0
- package/build/dist/Server/Utils/Rules/RuleRun/RuleRunRegistry.js +28 -0
- package/build/dist/Server/Utils/Rules/RuleRun/RuleRunRegistry.js.map +1 -1
- package/build/dist/Server/Utils/Slo/SloRulePatternValidator.js +28 -0
- package/build/dist/Server/Utils/Slo/SloRulePatternValidator.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/Slack/Slack.js +331 -11
- package/build/dist/Server/Utils/Workspace/Slack/Slack.js.map +1 -1
- package/build/dist/Types/API/Hostname.js +59 -11
- package/build/dist/Types/API/Hostname.js.map +1 -1
- package/build/dist/Types/API/Route.js +18 -2
- package/build/dist/Types/API/Route.js.map +1 -1
- package/build/dist/Types/API/URL.js +23 -2
- package/build/dist/Types/API/URL.js.map +1 -1
- package/build/dist/Types/Database/AccessControl/ColumnAccessControl.js +19 -7
- package/build/dist/Types/Database/AccessControl/ColumnAccessControl.js.map +1 -1
- package/build/dist/Types/Database/AccessControl/ColumnBillingAccessControl.js +19 -7
- package/build/dist/Types/Database/AccessControl/ColumnBillingAccessControl.js.map +1 -1
- package/build/dist/Types/Database/AccessControl/OwnerOnlyColumn.js +13 -6
- package/build/dist/Types/Database/AccessControl/OwnerOnlyColumn.js.map +1 -1
- package/build/dist/Types/Database/CanonicalModelInstance.js +88 -0
- package/build/dist/Types/Database/CanonicalModelInstance.js.map +1 -0
- package/build/dist/Types/Database/TableColumn.js +19 -6
- package/build/dist/Types/Database/TableColumn.js.map +1 -1
- package/build/dist/Types/EnterpriseLicense/EnterpriseLicensePeriods.js +28 -0
- package/build/dist/Types/EnterpriseLicense/EnterpriseLicensePeriods.js.map +1 -0
- package/build/dist/Types/IP/IPv6.js +6 -1
- package/build/dist/Types/IP/IPv6.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorStep.js +5 -0
- package/build/dist/Types/Monitor/MonitorStep.js.map +1 -1
- package/build/dist/Types/Monitor/ProxmoxAlertTemplates.js +1 -1
- package/build/dist/Types/Permission.js +84 -0
- package/build/dist/Types/Permission.js.map +1 -1
- package/build/dist/Types/Rules/RuleCriteriaFieldRegistry.js +11 -0
- package/build/dist/Types/Rules/RuleCriteriaFieldRegistry.js.map +1 -1
- package/build/dist/Types/Rules/RuleRun.js +4 -0
- package/build/dist/Types/Rules/RuleRun.js.map +1 -1
- package/build/dist/Types/StatusPage/UptimeDailyAggregate.js +32 -0
- package/build/dist/Types/StatusPage/UptimeDailyAggregate.js.map +1 -0
- package/build/dist/UI/Components/EditionLabel/EditionLabel.js +414 -585
- package/build/dist/UI/Components/EditionLabel/EditionLabel.js.map +1 -1
- package/build/dist/UI/Components/EditionLabel/LicenseManager.js +2 -0
- package/build/dist/UI/Components/EditionLabel/LicenseManager.js.map +1 -0
- package/build/dist/UI/Components/Feed/FeedOptions.js +235 -0
- package/build/dist/UI/Components/Feed/FeedOptions.js.map +1 -0
- package/build/dist/UI/Components/Feed/FeedOptionsButton.js +346 -0
- package/build/dist/UI/Components/Feed/FeedOptionsButton.js.map +1 -0
- package/build/dist/UI/Components/Feed/useFeedItems.js +17 -7
- package/build/dist/UI/Components/Feed/useFeedItems.js.map +1 -1
- package/build/dist/UI/Components/Feed/useFeedOptions.js +130 -0
- package/build/dist/UI/Components/Feed/useFeedOptions.js.map +1 -0
- package/build/dist/UI/Components/Graphs/DayUptimeGraph.js +90 -17
- package/build/dist/UI/Components/Graphs/DayUptimeGraph.js.map +1 -1
- package/build/dist/UI/Components/Icon/Icon.js +1 -1
- package/build/dist/UI/Components/Icon/Icon.js.map +1 -1
- package/build/dist/UI/Components/Markdown.tsx/MarkdownConverters.js +0 -0
- package/build/dist/UI/Components/Markdown.tsx/MarkdownConverters.js.map +1 -1
- package/build/dist/UI/Components/ModelList/ModelList.js +4 -1
- package/build/dist/UI/Components/ModelList/ModelList.js.map +1 -1
- package/build/dist/UI/Components/MonitorGraphs/Uptime.js +42 -1
- package/build/dist/UI/Components/MonitorGraphs/Uptime.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/TelemetryViewer.js +2 -2
- package/build/dist/UI/Components/TelemetryViewer/TelemetryViewer.js.map +1 -1
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseUsage.js +6 -2
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseUsage.js.map +1 -1
- package/build/dist/Utils/HostAddressUtil.js +80 -0
- package/build/dist/Utils/HostAddressUtil.js.map +1 -0
- package/build/dist/Utils/LabelRuleImportExport.js +22 -6
- package/build/dist/Utils/LabelRuleImportExport.js.map +1 -1
- package/build/dist/Utils/Monitor/MetricValueFormatter.js +2 -2
- package/build/dist/Utils/Monitor/MonitorDestinationUtil.js +127 -0
- package/build/dist/Utils/Monitor/MonitorDestinationUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitoringIntervalUtil.js +374 -0
- package/build/dist/Utils/Monitor/MonitoringIntervalUtil.js.map +1 -0
- package/build/dist/Utils/Rules/MonitorTypeRuleCriteria.js +18 -0
- package/build/dist/Utils/Rules/MonitorTypeRuleCriteria.js.map +1 -0
- package/build/dist/Utils/Slo/SloDuration.js +13 -5
- package/build/dist/Utils/Slo/SloDuration.js.map +1 -1
- package/build/dist/Utils/Slo/SloHealth.js +8 -7
- package/build/dist/Utils/Slo/SloHealth.js.map +1 -1
- package/build/dist/Utils/Slo/SloMonitorRuleCriteria.js +16 -1
- package/build/dist/Utils/Slo/SloMonitorRuleCriteria.js.map +1 -1
- package/build/dist/Utils/Slo/SloOverviewText.js +16 -0
- package/build/dist/Utils/Slo/SloOverviewText.js.map +1 -1
- package/build/dist/Utils/Slo/SloProjection.js.map +1 -1
- package/build/dist/Utils/StatusPage/UptimeDailyAggregateUtil.js +148 -0
- package/build/dist/Utils/StatusPage/UptimeDailyAggregateUtil.js.map +1 -0
- package/jest.config.json +3 -1
- package/package.json +1 -1
- package/test-setup.sh +103 -34
- package/tsconfig.json +13 -0
- package/Server/API/EnterpriseLicenseAPI.ts +0 -925
- package/Server/API/TeamComplianceAPI.ts +0 -158
- package/Server/Middleware/SCIMAuthorization.ts +0 -141
- package/Server/Services/TeamComplianceService.ts +0 -733
- package/Server/Utils/EnterpriseLicense/EnterpriseLicenseSeatUtil.ts +0 -186
- package/Server/Utils/InstanceHealth/PostgresHealth.ts +0 -286
- package/Tests/App/AdminDashboard/EnterpriseLicenseInstanceActivity.test.tsx +0 -451
- package/Tests/App/Dashboard/AuditLogsTable.test.tsx +0 -598
- package/Tests/App/Dashboard/SecurityEventsExplorer.test.tsx +0 -761
- package/Tests/App/Dashboard/SecurityEventsSideMenu.test.tsx +0 -247
- package/Tests/App/Dashboard/SecurityEventsTableColumns.test.tsx +0 -382
- package/Tests/App/Dashboard/SecurityEventsVolumeChart.test.tsx +0 -321
- package/Tests/Server/API/EnterpriseLicenseReportUserCount.test.ts +0 -1851
- package/Tests/Server/Middleware/SCIMAuthorization.test.ts +0 -525
- package/Tests/Server/Services/TeamComplianceServiceBehaviour.test.ts +0 -1845
- package/Tests/Server/Utils/EnterpriseLicense/EnterpriseLicenseSeatUtil.test.ts +0 -495
- package/Tests/Server/Utils/InstanceHealth/PostgresHealth.test.ts +0 -337
- package/Tests/Utils/EnterpriseLicenseSeats.test.ts +0 -576
- package/Tests/Utils/EnterpriseLicenseSync.test.ts +0 -568
- package/Utils/EnterpriseLicense/EnterpriseLicenseSeats.ts +0 -287
- package/Utils/EnterpriseLicense/EnterpriseLicenseSync.ts +0 -244
- package/build/dist/Server/API/EnterpriseLicenseAPI.js +0 -646
- package/build/dist/Server/API/EnterpriseLicenseAPI.js.map +0 -1
- package/build/dist/Server/API/TeamComplianceAPI.js +0 -111
- package/build/dist/Server/API/TeamComplianceAPI.js.map +0 -1
- package/build/dist/Server/Middleware/SCIMAuthorization.js +0 -124
- package/build/dist/Server/Middleware/SCIMAuthorization.js.map +0 -1
- package/build/dist/Server/Services/TeamComplianceService.js +0 -537
- package/build/dist/Server/Services/TeamComplianceService.js.map +0 -1
- package/build/dist/Server/Utils/EnterpriseLicense/EnterpriseLicenseSeatUtil.js +0 -145
- package/build/dist/Server/Utils/EnterpriseLicense/EnterpriseLicenseSeatUtil.js.map +0 -1
- package/build/dist/Server/Utils/InstanceHealth/PostgresHealth.js +0 -194
- package/build/dist/Server/Utils/InstanceHealth/PostgresHealth.js.map +0 -1
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseSeats.js +0 -168
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseSeats.js.map +0 -1
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseSync.js +0 -174
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseSync.js.map +0 -1
|
@@ -6,7 +6,11 @@ import MetricMonitorResponse, {
|
|
|
6
6
|
VMwareAffectedResource,
|
|
7
7
|
CephAffectedResource,
|
|
8
8
|
KubernetesAffectedResource,
|
|
9
|
+
DockerSwarmAffectedResource,
|
|
9
10
|
} from "../../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
|
|
11
|
+
import Markdown, {
|
|
12
|
+
MarkdownContentType,
|
|
13
|
+
} from "../../../../Server/Types/Markdown";
|
|
10
14
|
import {
|
|
11
15
|
getProxmoxAlertTemplateById,
|
|
12
16
|
ProxmoxAlertTemplate,
|
|
@@ -19,6 +23,10 @@ import {
|
|
|
19
23
|
getVMwareAlertTemplateById,
|
|
20
24
|
VMwareAlertTemplate,
|
|
21
25
|
} from "../../../../Types/Monitor/VMwareAlertTemplates";
|
|
26
|
+
import {
|
|
27
|
+
getKubernetesAlertTemplateById,
|
|
28
|
+
KubernetesAlertTemplate,
|
|
29
|
+
} from "../../../../Types/Monitor/KubernetesAlertTemplates";
|
|
22
30
|
import ObjectID from "../../../../Types/ObjectID";
|
|
23
31
|
import MonitorCriteriaInstance from "../../../../Types/Monitor/MonitorCriteriaInstance";
|
|
24
32
|
import {
|
|
@@ -36,12 +44,14 @@ import FilterCondition from "../../../../Types/Filter/FilterCondition";
|
|
|
36
44
|
* the incident root-cause context. These tests drive the evaluator's
|
|
37
45
|
* Proxmox/Ceph branches directly and lock in the render contract:
|
|
38
46
|
*
|
|
39
|
-
* -
|
|
40
|
-
*
|
|
47
|
+
* - the breakdown is a ranked list, not a table: one numbered item per
|
|
48
|
+
* resource titled "**Kind** `name` — **value**", with the rest of
|
|
49
|
+
* its identity (a Proxmox guest's node, a Ceph daemon's pool and
|
|
50
|
+
* host, a vSphere object's host and cluster) as nested bullets,
|
|
41
51
|
* - zero-value rows are dropped (supplementary context — the
|
|
42
52
|
* per-series criteria still alert on them), worst value first,
|
|
43
53
|
* top 10 with an "... and N more" suffix,
|
|
44
|
-
* - identity-less (cluster-wide) breakdowns render NO
|
|
54
|
+
* - identity-less (cluster-wide) breakdowns render NO list and fall
|
|
45
55
|
* back to the metric summary,
|
|
46
56
|
* - the cluster context lines surface the monitor step's
|
|
47
57
|
* clusterIdentifier and resource filters (the pve.scope / pve.id /
|
|
@@ -68,6 +78,17 @@ type EvaluatorPrivate = {
|
|
|
68
78
|
monitor: Monitor;
|
|
69
79
|
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
70
80
|
}) => string | null;
|
|
81
|
+
buildKubernetesRootCauseContext: (input: {
|
|
82
|
+
dataToProcess: unknown;
|
|
83
|
+
monitorStep: MonitorStep;
|
|
84
|
+
monitor: Monitor;
|
|
85
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
86
|
+
}) => Promise<string | null>;
|
|
87
|
+
buildDockerSwarmRootCauseContext: (input: {
|
|
88
|
+
dataToProcess: unknown;
|
|
89
|
+
monitorStep: MonitorStep;
|
|
90
|
+
monitor: Monitor;
|
|
91
|
+
}) => string | null;
|
|
71
92
|
buildKubernetesRootCauseAnalysis: (input: {
|
|
72
93
|
breakdown: {
|
|
73
94
|
clusterName: string;
|
|
@@ -142,12 +163,12 @@ function metricResponse(
|
|
|
142
163
|
}
|
|
143
164
|
|
|
144
165
|
describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
145
|
-
test("renders
|
|
166
|
+
test("renders a ranked list of kind, name and node: zero rows dropped, worst first", () => {
|
|
146
167
|
const affectedResources: Array<ProxmoxAffectedResource> = [
|
|
147
168
|
/*
|
|
148
169
|
* pve_cpu_usage_ratio is a [0, 1] ratio — the catalog says so and
|
|
149
170
|
* pve-exporter reports it that way — so these are 42% and 97% of a
|
|
150
|
-
* node's CPU, and the
|
|
171
|
+
* node's CPU, and the list is expected to say exactly that.
|
|
151
172
|
*/
|
|
152
173
|
{
|
|
153
174
|
resourceId: "qemu/100",
|
|
@@ -165,7 +186,7 @@ describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
|
165
186
|
nodeName: "pve2",
|
|
166
187
|
metricValue: 0.97,
|
|
167
188
|
},
|
|
168
|
-
// Zero-value row — must be dropped from the
|
|
189
|
+
// Zero-value row — must be dropped from the list.
|
|
169
190
|
{
|
|
170
191
|
resourceId: "qemu/102",
|
|
171
192
|
resourceName: "idle-vm",
|
|
@@ -195,7 +216,9 @@ describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
|
195
216
|
expect(context).toContain("- Cluster: prod-cluster");
|
|
196
217
|
expect(context).toContain("- Metric: CPU Usage (`pve_cpu_usage_ratio`)");
|
|
197
218
|
|
|
198
|
-
|
|
219
|
+
// A list, not a table: no GFM table row survives anywhere.
|
|
220
|
+
expect(context).not.toContain("| --- |");
|
|
221
|
+
expect(context).not.toMatch(/^\|/m);
|
|
199
222
|
// Worst (0.97) sorts above 0.42; the zero row is gone entirely.
|
|
200
223
|
const dbIndex: number = context!.indexOf("`db-vm` (`qemu/101`)");
|
|
201
224
|
const webIndex: number = context!.indexOf("`web-vm` (`qemu/100`)");
|
|
@@ -205,20 +228,26 @@ describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
|
205
228
|
expect(context).toContain("**Affected Resources** (2 total)");
|
|
206
229
|
|
|
207
230
|
/*
|
|
208
|
-
* The
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
* otherwise have to recognise as a percentage themselves.
|
|
231
|
+
* The value is rendered in the unit the catalog declares for the
|
|
232
|
+
* metric — a "ratio" on a `_ratio`-suffixed metric is a fraction, so
|
|
233
|
+
* the reader gets "97.00%" rather than the bare "0.97" they would
|
|
234
|
+
* otherwise have to recognise as a percentage themselves. A qemu
|
|
235
|
+
* guest is titled "Virtual Machine", and the node it runs on sits
|
|
236
|
+
* under it.
|
|
212
237
|
*/
|
|
213
238
|
expect(context).toContain(
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
239
|
+
[
|
|
240
|
+
"**Affected Resources** (2 total)",
|
|
241
|
+
"",
|
|
242
|
+
"1. **Virtual Machine** `db-vm` (`qemu/101`) — **97.00%**",
|
|
243
|
+
" - Node: `pve2`",
|
|
244
|
+
"2. **Virtual Machine** `web-vm` (`qemu/100`) — **42.00%**",
|
|
245
|
+
" - Node: `pve1`",
|
|
246
|
+
].join("\n"),
|
|
218
247
|
);
|
|
219
248
|
});
|
|
220
249
|
|
|
221
|
-
test("renders
|
|
250
|
+
test("renders byte values at human scale", () => {
|
|
222
251
|
const context: string | null = Evaluator.buildProxmoxRootCauseContext({
|
|
223
252
|
dataToProcess: metricResponse({
|
|
224
253
|
proxmoxResourceBreakdown: {
|
|
@@ -247,12 +276,12 @@ describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
|
247
276
|
* 3am should not have to divide by 2^30 in their head.
|
|
248
277
|
*/
|
|
249
278
|
expect(context).toContain(
|
|
250
|
-
"
|
|
279
|
+
"1. **Virtual Machine** `db-vm` (`qemu/101`) — **8.59 GB**",
|
|
251
280
|
);
|
|
252
281
|
expect(context).not.toContain("**8589934592**");
|
|
253
282
|
});
|
|
254
283
|
|
|
255
|
-
test("caps the
|
|
284
|
+
test("caps the list at 10 items, worst first, with an overflow suffix", () => {
|
|
256
285
|
const affectedResources: Array<ProxmoxAffectedResource> = [];
|
|
257
286
|
for (let i: number = 1; i <= 12; i++) {
|
|
258
287
|
affectedResources.push({
|
|
@@ -286,13 +315,18 @@ describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
|
286
315
|
expect(context).toContain("**Affected Resources** (12 total)");
|
|
287
316
|
expect(context).toContain("*... and 2 more affected resources*");
|
|
288
317
|
// The worst rows survive the cap; the mildest two are cut.
|
|
289
|
-
expect(context).toContain("**12**");
|
|
290
|
-
expect(context).toContain("**3**");
|
|
291
|
-
expect(context).not.toContain("
|
|
292
|
-
expect(context).not.toContain("
|
|
318
|
+
expect(context).toContain("1. **Virtual Machine** `qemu/112` — **12**");
|
|
319
|
+
expect(context).toContain("10. **Virtual Machine** `qemu/103` — **3**");
|
|
320
|
+
expect(context).not.toContain("`qemu/102`");
|
|
321
|
+
expect(context).not.toContain("`qemu/101`");
|
|
322
|
+
expect(context).not.toContain("11. ");
|
|
323
|
+
// The summary is its own paragraph, not a continuation of item 10.
|
|
324
|
+
expect(context).toContain(
|
|
325
|
+
"10. **Virtual Machine** `qemu/103` — **3**\n\n*... and 2 more affected resources*",
|
|
326
|
+
);
|
|
293
327
|
});
|
|
294
328
|
|
|
295
|
-
test("identity-less (cluster-wide) breakdowns render no
|
|
329
|
+
test("identity-less (cluster-wide) breakdowns render no list and fall back to the metric summary", () => {
|
|
296
330
|
const context: string | null = Evaluator.buildProxmoxRootCauseContext({
|
|
297
331
|
dataToProcess: metricResponse({
|
|
298
332
|
proxmoxResourceBreakdown: {
|
|
@@ -311,7 +345,8 @@ describe("MonitorCriteriaEvaluator - Proxmox root cause breakdown", () => {
|
|
|
311
345
|
monitor: new Monitor(),
|
|
312
346
|
});
|
|
313
347
|
|
|
314
|
-
expect(context).not.toContain("
|
|
348
|
+
expect(context).not.toContain("**Affected Resources**");
|
|
349
|
+
expect(context).not.toContain("1. ");
|
|
315
350
|
expect(context).toContain("**Metric Summary**");
|
|
316
351
|
expect(context).toContain("- 2 metric data point(s) returned");
|
|
317
352
|
});
|
|
@@ -346,12 +381,13 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
346
381
|
/*
|
|
347
382
|
* VMware twin of the Proxmox block. The differences are deliberate:
|
|
348
383
|
* identity comes from the vcenter receiver's RESOURCE attributes
|
|
349
|
-
* (stored `resource.vcenter.*`),
|
|
350
|
-
*
|
|
351
|
-
* are already 0–100 percentages (catalog unit
|
|
352
|
-
* ratios — so the
|
|
384
|
+
* (stored `resource.vcenter.*`), each item is titled by the vSphere
|
|
385
|
+
* object (VM, host, pool, ...) with its host and cluster beneath it,
|
|
386
|
+
* and utilization metrics are already 0–100 percentages (catalog unit
|
|
387
|
+
* "%") — never [0, 1] ratios — so the value must print as-is and never
|
|
388
|
+
* ×100.
|
|
353
389
|
*/
|
|
354
|
-
test("renders
|
|
390
|
+
test("renders a ranked list of object, host and cluster: zero rows dropped, worst first, % as-is", () => {
|
|
355
391
|
const affectedResources: Array<VMwareAffectedResource> = [
|
|
356
392
|
{
|
|
357
393
|
datacenterName: "DC1",
|
|
@@ -365,7 +401,7 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
365
401
|
hostName: "esx-02",
|
|
366
402
|
metricValue: 97.25,
|
|
367
403
|
},
|
|
368
|
-
// Zero-value row — must be dropped from the
|
|
404
|
+
// Zero-value row — must be dropped from the list.
|
|
369
405
|
{
|
|
370
406
|
datacenterName: "DC1",
|
|
371
407
|
clusterName: "prod-cluster",
|
|
@@ -395,7 +431,7 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
395
431
|
"- Metric: Host CPU Utilization (`vcenter.host.cpu.utilization`)",
|
|
396
432
|
);
|
|
397
433
|
|
|
398
|
-
expect(context).toContain("|
|
|
434
|
+
expect(context).not.toContain("| --- |");
|
|
399
435
|
// Worst (97.25) sorts above 42.5; the zero row is gone entirely.
|
|
400
436
|
const worstIndex: number = context!.indexOf("`esx-02`");
|
|
401
437
|
const mildIndex: number = context!.indexOf("`esx-01`");
|
|
@@ -407,15 +443,18 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
407
443
|
/*
|
|
408
444
|
* The catalog declares "%" for the receiver's utilization metrics,
|
|
409
445
|
* whose values are already percentages: 97.25 must read "97.25%",
|
|
410
|
-
* never "9725.00%". A host IS the resource, so
|
|
411
|
-
*
|
|
446
|
+
* never "9725.00%". A host IS the resource, so there is no "Host:"
|
|
447
|
+
* detail repeating it — only the cluster it belongs to.
|
|
412
448
|
*/
|
|
413
449
|
expect(context).toContain(
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
450
|
+
[
|
|
451
|
+
"1. **Host** `esx-02` — **97.25%**",
|
|
452
|
+
" - Cluster: `prod-cluster`",
|
|
453
|
+
"2. **Host** `esx-01` — **42.50%**",
|
|
454
|
+
" - Cluster: `prod-cluster`",
|
|
455
|
+
].join("\n"),
|
|
418
456
|
);
|
|
457
|
+
expect(context).not.toContain("- Host: `esx-0");
|
|
419
458
|
expect(context).not.toContain("9725");
|
|
420
459
|
});
|
|
421
460
|
|
|
@@ -441,11 +480,11 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
441
480
|
monitor: new Monitor(),
|
|
442
481
|
});
|
|
443
482
|
|
|
444
|
-
expect(context).toContain("
|
|
483
|
+
expect(context).toContain("1. **Host** `esx-01` — **0.42%**");
|
|
445
484
|
expect(context).not.toContain("42.00%");
|
|
446
485
|
});
|
|
447
486
|
|
|
448
|
-
test("renders
|
|
487
|
+
test("renders VMs with their parent host and cluster, and bytes at human scale", () => {
|
|
449
488
|
const context: string | null = Evaluator.buildVMwareRootCauseContext({
|
|
450
489
|
dataToProcess: metricResponse({
|
|
451
490
|
vmwareResourceBreakdown: {
|
|
@@ -480,22 +519,25 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
480
519
|
|
|
481
520
|
/*
|
|
482
521
|
* A VM row carries a resource pool AND a host, but it is still a
|
|
483
|
-
* VM: the
|
|
484
|
-
*
|
|
485
|
-
*
|
|
522
|
+
* VM: the title must say so (VM check runs first), the Host detail
|
|
523
|
+
* is the ESXi host running it, and the on-call engineer should not
|
|
524
|
+
* have to divide by 2^30 in their head. A VM on a standalone ESXi
|
|
525
|
+
* host has no cluster, so it simply has no Cluster detail.
|
|
486
526
|
*/
|
|
487
527
|
expect(context).toContain(
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
528
|
+
[
|
|
529
|
+
"1. **Virtual Machine** `db-01` — **8.59 GB**",
|
|
530
|
+
" - Host: `esx-02`",
|
|
531
|
+
" - Cluster: `prod-cluster`",
|
|
532
|
+
"2. **Virtual Machine** `lab-01` — **1.07 GB**",
|
|
533
|
+
" - Host: `esx-standalone`",
|
|
534
|
+
].join("\n"),
|
|
493
535
|
);
|
|
494
536
|
expect(context).not.toContain("**8589934592**");
|
|
495
537
|
expect(context).not.toContain("Resource Pool");
|
|
496
538
|
});
|
|
497
539
|
|
|
498
|
-
test("renders
|
|
540
|
+
test("renders datastores, clusters, datacenters and resource pools by their own identity", () => {
|
|
499
541
|
const context: string | null = Evaluator.buildVMwareRootCauseContext({
|
|
500
542
|
dataToProcess: metricResponse({
|
|
501
543
|
vmwareResourceBreakdown: {
|
|
@@ -533,15 +575,18 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
533
575
|
* `{hosts}` is an annotation-only unit, so counts render as bare
|
|
534
576
|
* integers. Each kind resolves by the most specific identity
|
|
535
577
|
* attribute it carries: a datacenter-only row is a Datacenter, a
|
|
536
|
-
* cluster
|
|
537
|
-
*
|
|
538
|
-
*
|
|
578
|
+
* cluster shows no duplicate Cluster detail, a pool shows its name
|
|
579
|
+
* with the inventory path (pool names are only unique within a
|
|
580
|
+
* parent).
|
|
539
581
|
*/
|
|
540
|
-
expect(context).toContain("| `DC1` | Datacenter | - | - | **4** |");
|
|
541
|
-
expect(context).toContain("| `prod-cluster` | Cluster | - | - | **3** |");
|
|
542
|
-
expect(context).toContain("| `vsan-ds-01` | Datastore | - | - | **2** |");
|
|
543
582
|
expect(context).toContain(
|
|
544
|
-
|
|
583
|
+
[
|
|
584
|
+
"1. **Datacenter** `DC1` — **4**",
|
|
585
|
+
"2. **Cluster** `prod-cluster` — **3**",
|
|
586
|
+
"3. **Datastore** `vsan-ds-01` — **2**",
|
|
587
|
+
"4. **Resource Pool** `batch` (`/DC1/host/prod-cluster/Resources/batch`) — **1**",
|
|
588
|
+
" - Cluster: `prod-cluster`",
|
|
589
|
+
].join("\n"),
|
|
545
590
|
);
|
|
546
591
|
});
|
|
547
592
|
|
|
@@ -563,12 +608,10 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
563
608
|
});
|
|
564
609
|
|
|
565
610
|
// Unit from the catalog ("ms"), not a bare, unit-less 85.
|
|
566
|
-
expect(context).toMatch(
|
|
567
|
-
/\| `esx-01` \| Host \| - \| - \| \*\*85(\.00)? ms\*\* \|/,
|
|
568
|
-
);
|
|
611
|
+
expect(context).toMatch(/1\. \*\*Host\*\* `esx-01` — \*\*85(\.00)? ms\*\*/);
|
|
569
612
|
});
|
|
570
613
|
|
|
571
|
-
test("caps the
|
|
614
|
+
test("caps the list at 10 items, worst first, with an overflow suffix", () => {
|
|
572
615
|
const affectedResources: Array<VMwareAffectedResource> = [];
|
|
573
616
|
for (let i: number = 1; i <= 12; i++) {
|
|
574
617
|
affectedResources.push({
|
|
@@ -595,13 +638,13 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
595
638
|
expect(context).toContain("**Affected Resources** (12 total)");
|
|
596
639
|
expect(context).toContain("*... and 2 more affected resources*");
|
|
597
640
|
// The worst rows survive the cap; the mildest two are cut.
|
|
598
|
-
expect(context).toContain("`esx-12`");
|
|
599
|
-
expect(context).toContain("`esx-03`");
|
|
641
|
+
expect(context).toContain("1. **Host** `esx-12` — **12**");
|
|
642
|
+
expect(context).toContain("10. **Host** `esx-03` — **3**");
|
|
600
643
|
expect(context).not.toContain("`esx-02`");
|
|
601
644
|
expect(context).not.toContain("`esx-01`");
|
|
602
645
|
});
|
|
603
646
|
|
|
604
|
-
test("identity-less breakdowns render no
|
|
647
|
+
test("identity-less breakdowns render no list and fall back to the metric summary", () => {
|
|
605
648
|
const context: string | null = Evaluator.buildVMwareRootCauseContext({
|
|
606
649
|
dataToProcess: metricResponse({
|
|
607
650
|
vmwareResourceBreakdown: {
|
|
@@ -620,9 +663,7 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
620
663
|
monitor: new Monitor(),
|
|
621
664
|
});
|
|
622
665
|
|
|
623
|
-
expect(context).not.toContain(
|
|
624
|
-
"| Resource | Kind | Host | Cluster | Value |",
|
|
625
|
-
);
|
|
666
|
+
expect(context).not.toContain("**Affected Resources**");
|
|
626
667
|
expect(context).toContain("**Metric Summary**");
|
|
627
668
|
expect(context).toContain("- 2 metric data point(s) returned");
|
|
628
669
|
});
|
|
@@ -678,7 +719,7 @@ describe("MonitorCriteriaEvaluator - VMware root cause breakdown", () => {
|
|
|
678
719
|
});
|
|
679
720
|
|
|
680
721
|
describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
681
|
-
test("renders
|
|
722
|
+
test("renders a ranked list of daemons and pools with pool name+id", () => {
|
|
682
723
|
const affectedResources: Array<CephAffectedResource> = [
|
|
683
724
|
{ daemon: "osd.3", hostname: "ceph-node-1", metricValue: 250 },
|
|
684
725
|
{ poolId: "2", poolName: "rbd", metricValue: 91 },
|
|
@@ -706,15 +747,20 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
706
747
|
"- Metric: OSD Apply Latency (`ceph_osd_apply_latency_ms`)",
|
|
707
748
|
);
|
|
708
749
|
|
|
709
|
-
expect(context).toContain("| Daemon | Pool | Host | Value |");
|
|
710
750
|
/*
|
|
711
751
|
* ceph_osd_apply_latency_ms carries the catalog unit "ms", so the
|
|
712
|
-
*
|
|
752
|
+
* value names it. 250 ms stays "250 ms" rather than scaling to
|
|
713
753
|
* seconds — the ladder only moves a value when the magnitude asks
|
|
714
|
-
* for it.
|
|
754
|
+
* for it. A row with no daemon is titled by its pool.
|
|
715
755
|
*/
|
|
716
|
-
expect(context).toContain(
|
|
717
|
-
|
|
756
|
+
expect(context).toContain(
|
|
757
|
+
[
|
|
758
|
+
"1. **Daemon** `osd.3` — **250 ms**",
|
|
759
|
+
" - Host: `ceph-node-1`",
|
|
760
|
+
"2. **Pool** `rbd` (`2`) — **91 ms**",
|
|
761
|
+
].join("\n"),
|
|
762
|
+
);
|
|
763
|
+
expect(context).not.toContain("| --- |");
|
|
718
764
|
expect(context).not.toContain("osd.5");
|
|
719
765
|
expect(context).toContain("**Affected Resources** (2 total)");
|
|
720
766
|
|
|
@@ -726,14 +772,14 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
726
772
|
});
|
|
727
773
|
|
|
728
774
|
/*
|
|
729
|
-
* BACKWARD COMPATIBILITY for the platform
|
|
775
|
+
* BACKWARD COMPATIBILITY for the platform breakdowns.
|
|
730
776
|
*
|
|
731
777
|
* A metric whose catalog unit is "count" or absent has no dimension to
|
|
732
|
-
* report, and its
|
|
778
|
+
* report, and its value must keep every digit — never a chart's
|
|
733
779
|
* abbreviated "1.2K", and never the noise "250 count". ceph_osd_up is
|
|
734
780
|
* `unit: "count"`; ceph_health_status has no unit at all.
|
|
735
781
|
*/
|
|
736
|
-
test("a count metric keeps exact bare digits in the
|
|
782
|
+
test("a count metric keeps exact bare digits in the value", () => {
|
|
737
783
|
const context: string | null = Evaluator.buildCephRootCauseContext({
|
|
738
784
|
dataToProcess: metricResponse({
|
|
739
785
|
cephResourceBreakdown: {
|
|
@@ -750,16 +796,18 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
750
796
|
monitor: new Monitor(),
|
|
751
797
|
});
|
|
752
798
|
|
|
753
|
-
expect(context).toContain(
|
|
799
|
+
expect(context).toContain(
|
|
800
|
+
"1. **Daemon** `osd.3` — **5000**\n - Host: `ceph-node-1`",
|
|
801
|
+
);
|
|
754
802
|
expect(context).not.toContain("5K");
|
|
755
803
|
expect(context).not.toContain("count");
|
|
756
804
|
});
|
|
757
805
|
|
|
758
806
|
/*
|
|
759
|
-
* ORDERING INVARIANT. The
|
|
807
|
+
* ORDERING INVARIANT. The list sorts and filters on the RAW numeric
|
|
760
808
|
* metricValue and only formats at render time. If a refactor ever sorted
|
|
761
809
|
* the formatted strings instead, "1.07 GB" would sort below "900 KB" and
|
|
762
|
-
* the worst-first
|
|
810
|
+
* the worst-first list would silently invert — putting the healthiest
|
|
763
811
|
* daemon at the top of an incident.
|
|
764
812
|
*/
|
|
765
813
|
test("rows still sort worst-first on the raw value, not the formatted string", () => {
|
|
@@ -780,8 +828,8 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
780
828
|
monitor: new Monitor(),
|
|
781
829
|
});
|
|
782
830
|
|
|
783
|
-
expect(context).toContain("
|
|
784
|
-
expect(context).toContain("
|
|
831
|
+
expect(context).toContain("1. **Daemon** `osd.bigger` — **1.07 GB**");
|
|
832
|
+
expect(context).toContain("2. **Daemon** `osd.smaller` — **922 KB**");
|
|
785
833
|
|
|
786
834
|
// "1.07 GB" < "922 KB" as strings — the raw numbers must drive this.
|
|
787
835
|
const biggerIndex: number = context!.indexOf("`osd.bigger`");
|
|
@@ -790,7 +838,7 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
790
838
|
expect(smallerIndex).toBeGreaterThan(biggerIndex);
|
|
791
839
|
});
|
|
792
840
|
|
|
793
|
-
test("cluster-wide series (ceph_health_status) render no
|
|
841
|
+
test("cluster-wide series (ceph_health_status) render no list", () => {
|
|
794
842
|
const context: string | null = Evaluator.buildCephRootCauseContext({
|
|
795
843
|
dataToProcess: metricResponse({
|
|
796
844
|
cephResourceBreakdown: {
|
|
@@ -806,7 +854,7 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
806
854
|
});
|
|
807
855
|
|
|
808
856
|
expect(context).toContain("- Cluster: prod-cluster");
|
|
809
|
-
expect(context).not.toContain("
|
|
857
|
+
expect(context).not.toContain("**Affected Resources**");
|
|
810
858
|
});
|
|
811
859
|
|
|
812
860
|
test("surfaces the worker's resource-filter mapping (ceph_daemon / pool_id) in the cluster context", () => {
|
|
@@ -835,16 +883,16 @@ describe("MonitorCriteriaEvaluator - Ceph root cause breakdown", () => {
|
|
|
835
883
|
* ceph_osd_up / ceph_osd_in / ceph_mon_quorum_status are 0/1 gauges and
|
|
836
884
|
* the OSD Down / OSD Out / Quorum Degraded criteria fire on `< 1`, so the
|
|
837
885
|
* ZERO rows are the down daemons — the filter dropped exactly the daemons
|
|
838
|
-
* the incident was about and rendered a
|
|
886
|
+
* the incident was about and rendered a list of the HEALTHY ones, under
|
|
839
887
|
* an incident description that ends "Check the root cause for the affected
|
|
840
888
|
* ceph_daemon label."
|
|
841
889
|
*
|
|
842
890
|
* The predicate now mirrors the criteria that matched, but ONLY inverts
|
|
843
891
|
* for a criteria that fires when the metric FALLS. Every upward comparison
|
|
844
892
|
* (`> 0` health checks, PG counts, latency, capacity ratios) and every
|
|
845
|
-
* `= 0` RECOVERY criteria keeps the old `> 0`, worst-highest-first
|
|
893
|
+
* `= 0` RECOVERY criteria keeps the old `> 0`, worst-highest-first list
|
|
846
894
|
* byte for byte — the recovery case matters because deriving the predicate
|
|
847
|
-
* from an `= 0` criteria would newly render an "Affected Resources"
|
|
895
|
+
* from an `= 0` criteria would newly render an "Affected Resources" list
|
|
848
896
|
* of every resource sitting at 0 on the all-clear.
|
|
849
897
|
*/
|
|
850
898
|
describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate", () => {
|
|
@@ -933,8 +981,9 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
933
981
|
],
|
|
934
982
|
});
|
|
935
983
|
|
|
936
|
-
expect(context).toContain(
|
|
937
|
-
|
|
984
|
+
expect(context).toContain(
|
|
985
|
+
"1. **Daemon** `osd.3` — **0**\n - Host: `ceph-node-1`",
|
|
986
|
+
);
|
|
938
987
|
expect(context).not.toContain("osd.4");
|
|
939
988
|
expect(context).not.toContain("osd.5");
|
|
940
989
|
expect(context).toContain("**Affected Resources** (1 total)");
|
|
@@ -960,8 +1009,8 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
960
1009
|
});
|
|
961
1010
|
|
|
962
1011
|
expect(context).toContain("**Affected Resources** (2 total)");
|
|
963
|
-
expect(context).toContain("`osd.1`");
|
|
964
|
-
expect(context).toContain("`osd.2`");
|
|
1012
|
+
expect(context).toContain("1. **Daemon** `osd.1` — **0**");
|
|
1013
|
+
expect(context).toContain("2. **Daemon** `osd.2` — **0**");
|
|
965
1014
|
expect(context).not.toContain("`osd.10`");
|
|
966
1015
|
expect(context).not.toContain("*... and");
|
|
967
1016
|
});
|
|
@@ -1009,8 +1058,13 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1009
1058
|
],
|
|
1010
1059
|
});
|
|
1011
1060
|
|
|
1012
|
-
expect(context).toContain(
|
|
1013
|
-
|
|
1061
|
+
expect(context).toContain(
|
|
1062
|
+
[
|
|
1063
|
+
"1. **Daemon** `osd.3` — **250 ms**",
|
|
1064
|
+
" - Host: `ceph-node-1`",
|
|
1065
|
+
"2. **Pool** `rbd` (`2`) — **91 ms**",
|
|
1066
|
+
].join("\n"),
|
|
1067
|
+
);
|
|
1014
1068
|
expect(context).not.toContain("osd.5");
|
|
1015
1069
|
expect(context).toContain("**Affected Resources** (2 total)");
|
|
1016
1070
|
|
|
@@ -1020,7 +1074,7 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1020
1074
|
expect(poolIndex).toBeGreaterThan(osdIndex);
|
|
1021
1075
|
});
|
|
1022
1076
|
|
|
1023
|
-
test("an `= 0` recovery criteria does NOT turn the all-clear into a
|
|
1077
|
+
test("an `= 0` recovery criteria does NOT turn the all-clear into a list of zeroes", () => {
|
|
1024
1078
|
const context: string | null = cephContext({
|
|
1025
1079
|
metricName: "ceph_pg_degraded",
|
|
1026
1080
|
criteriaInstance: syntheticCriteria([
|
|
@@ -1036,7 +1090,7 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1036
1090
|
],
|
|
1037
1091
|
});
|
|
1038
1092
|
|
|
1039
|
-
expect(context).not.toContain("
|
|
1093
|
+
expect(context).not.toContain("**Pool**");
|
|
1040
1094
|
expect(context).not.toContain("**Affected Resources**");
|
|
1041
1095
|
});
|
|
1042
1096
|
|
|
@@ -1049,7 +1103,7 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1049
1103
|
],
|
|
1050
1104
|
});
|
|
1051
1105
|
|
|
1052
|
-
expect(context).toContain("
|
|
1106
|
+
expect(context).toContain("1. **Daemon** `osd.4` — **1**");
|
|
1053
1107
|
expect(context).not.toContain("osd.3");
|
|
1054
1108
|
});
|
|
1055
1109
|
|
|
@@ -1061,7 +1115,7 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1061
1115
|
],
|
|
1062
1116
|
});
|
|
1063
1117
|
|
|
1064
|
-
expect(context).toContain("
|
|
1118
|
+
expect(context).toContain("1. **Daemon** `osd.4` — **1**");
|
|
1065
1119
|
expect(context).not.toContain("osd.3");
|
|
1066
1120
|
});
|
|
1067
1121
|
|
|
@@ -1080,7 +1134,7 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1080
1134
|
],
|
|
1081
1135
|
});
|
|
1082
1136
|
|
|
1083
|
-
expect(context).toContain("
|
|
1137
|
+
expect(context).toContain("1. **Daemon** `osd.4` — **1**");
|
|
1084
1138
|
expect(context).not.toContain("osd.3");
|
|
1085
1139
|
});
|
|
1086
1140
|
|
|
@@ -1104,11 +1158,411 @@ describe("MonitorCriteriaEvaluator - Ceph affected-resources breach predicate",
|
|
|
1104
1158
|
],
|
|
1105
1159
|
});
|
|
1106
1160
|
|
|
1107
|
-
expect(context).toContain("
|
|
1161
|
+
expect(context).toContain("1. **Daemon** `osd.4` — **1**");
|
|
1162
|
+
expect(context).not.toContain("osd.3");
|
|
1163
|
+
});
|
|
1164
|
+
|
|
1165
|
+
test("an `= 0` criteria that opens an incident still keeps the `> 0` fallback (only Kubernetes opts in)", () => {
|
|
1166
|
+
/*
|
|
1167
|
+
* Kubernetes treats a FIRING `= 0` as a fall (node Ready = 0). Ceph
|
|
1168
|
+
* must not: every Ceph `= 0` criteria is a recovery, and the shared
|
|
1169
|
+
* predicate only inverts for it when the caller asks.
|
|
1170
|
+
*/
|
|
1171
|
+
const firingEqualsZero: MonitorCriteriaInstance = syntheticCriteria([
|
|
1172
|
+
{
|
|
1173
|
+
checkOn: CheckOn.MetricValue,
|
|
1174
|
+
filterType: FilterType.EqualTo,
|
|
1175
|
+
value: 0,
|
|
1176
|
+
},
|
|
1177
|
+
]);
|
|
1178
|
+
firingEqualsZero.data!.createIncidents = true;
|
|
1179
|
+
firingEqualsZero.data!.createAlerts = true;
|
|
1180
|
+
|
|
1181
|
+
const context: string | null = cephContext({
|
|
1182
|
+
criteriaInstance: firingEqualsZero,
|
|
1183
|
+
affectedResources: [
|
|
1184
|
+
{ daemon: "osd.3", metricValue: 0 },
|
|
1185
|
+
{ daemon: "osd.4", metricValue: 1 },
|
|
1186
|
+
],
|
|
1187
|
+
});
|
|
1188
|
+
|
|
1189
|
+
expect(context).toContain("1. **Daemon** `osd.4` — **1**");
|
|
1108
1190
|
expect(context).not.toContain("osd.3");
|
|
1109
1191
|
});
|
|
1110
1192
|
});
|
|
1111
1193
|
|
|
1194
|
+
/*
|
|
1195
|
+
* REGRESSION: the Kubernetes "Affected Resources" list named the HEALTHY
|
|
1196
|
+
* resources for criteria that fire when the metric FALLS.
|
|
1197
|
+
*
|
|
1198
|
+
* buildKubernetesRootCauseContext filtered the breakdown with a hardcoded
|
|
1199
|
+
* `metricValue > 0` and sorted highest first. k8s-node-not-ready fires on
|
|
1200
|
+
* `k8s.node.condition_ready = 0` and k8s-etcd-no-leader on
|
|
1201
|
+
* `etcd_server_has_leader = 0`, so the zero rows ARE the breach: the
|
|
1202
|
+
* filter dropped the NotReady node, listed the Ready ones, and the Root
|
|
1203
|
+
* Cause Analysis told the reader "Node `node-ok-1` is reporting NotReady
|
|
1204
|
+
* (value: 1)" and to `kubectl describe node node-ok-1`.
|
|
1205
|
+
*
|
|
1206
|
+
* The worker also kept only the HIGHEST sample per resource, so a node
|
|
1207
|
+
* that was NotReady for one scrape and Ready for the rest reached the
|
|
1208
|
+
* evaluator as 1. It now also records the lowest sample, which is what a
|
|
1209
|
+
* fall criteria breached on.
|
|
1210
|
+
*/
|
|
1211
|
+
describe("MonitorCriteriaEvaluator - Kubernetes affected-resources breach predicate", () => {
|
|
1212
|
+
function kubernetesStep(templateId: string): MonitorStep {
|
|
1213
|
+
const template: KubernetesAlertTemplate | undefined =
|
|
1214
|
+
getKubernetesAlertTemplateById(templateId);
|
|
1215
|
+
if (!template) {
|
|
1216
|
+
throw new Error(`${templateId} template missing`);
|
|
1217
|
+
}
|
|
1218
|
+
return template.getMonitorStep(templateArgs());
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
function criteriaInstances(
|
|
1222
|
+
monitorStep: MonitorStep,
|
|
1223
|
+
): Array<MonitorCriteriaInstance> {
|
|
1224
|
+
return monitorStep.data!.monitorCriteria!.data!
|
|
1225
|
+
.monitorCriteriaInstanceArray;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
/** The template's FIRING criteria (the one that opens the incident). */
|
|
1229
|
+
function firingCriteria(monitorStep: MonitorStep): MonitorCriteriaInstance {
|
|
1230
|
+
const instance: MonitorCriteriaInstance | undefined =
|
|
1231
|
+
criteriaInstances(monitorStep)[0];
|
|
1232
|
+
if (!instance) {
|
|
1233
|
+
throw new Error("firing criteria instance missing");
|
|
1234
|
+
}
|
|
1235
|
+
return instance;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
/** The template's RECOVERY criteria. */
|
|
1239
|
+
function recoveryCriteria(monitorStep: MonitorStep): MonitorCriteriaInstance {
|
|
1240
|
+
const instances: Array<MonitorCriteriaInstance> =
|
|
1241
|
+
criteriaInstances(monitorStep);
|
|
1242
|
+
const instance: MonitorCriteriaInstance | undefined =
|
|
1243
|
+
instances[instances.length - 1];
|
|
1244
|
+
if (!instance) {
|
|
1245
|
+
throw new Error("recovery criteria instance missing");
|
|
1246
|
+
}
|
|
1247
|
+
return instance;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
function syntheticCriteria(input: {
|
|
1251
|
+
filters: Array<CriteriaFilter>;
|
|
1252
|
+
opensIncident: boolean;
|
|
1253
|
+
}): MonitorCriteriaInstance {
|
|
1254
|
+
const instance: MonitorCriteriaInstance = new MonitorCriteriaInstance();
|
|
1255
|
+
instance.data = {
|
|
1256
|
+
monitorStatusId: undefined,
|
|
1257
|
+
filterCondition: FilterCondition.Any,
|
|
1258
|
+
filters: input.filters,
|
|
1259
|
+
incidents: [],
|
|
1260
|
+
alerts: [],
|
|
1261
|
+
name: "synthetic",
|
|
1262
|
+
description: "synthetic",
|
|
1263
|
+
createIncidents: input.opensIncident,
|
|
1264
|
+
createAlerts: input.opensIncident,
|
|
1265
|
+
id: ObjectID.generate().toString(),
|
|
1266
|
+
};
|
|
1267
|
+
return instance;
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
async function kubernetesContext(input: {
|
|
1271
|
+
metricName: string;
|
|
1272
|
+
affectedResources: Array<KubernetesAffectedResource>;
|
|
1273
|
+
monitorStep?: MonitorStep | undefined;
|
|
1274
|
+
criteriaInstance?: MonitorCriteriaInstance | undefined;
|
|
1275
|
+
}): Promise<string | null> {
|
|
1276
|
+
return Evaluator.buildKubernetesRootCauseContext({
|
|
1277
|
+
dataToProcess: metricResponse({
|
|
1278
|
+
kubernetesResourceBreakdown: {
|
|
1279
|
+
clusterName: "prod-cluster",
|
|
1280
|
+
metricName: input.metricName,
|
|
1281
|
+
metricFriendlyName: "Friendly Name",
|
|
1282
|
+
affectedResources: input.affectedResources,
|
|
1283
|
+
attributes: {},
|
|
1284
|
+
},
|
|
1285
|
+
}),
|
|
1286
|
+
monitorStep: input.monitorStep || kubernetesStep("k8s-node-not-ready"),
|
|
1287
|
+
monitor: new Monitor(),
|
|
1288
|
+
criteriaInstance: input.criteriaInstance,
|
|
1289
|
+
});
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
describe("k8s-node-not-ready", () => {
|
|
1293
|
+
const nodeNotReadyStep: MonitorStep = kubernetesStep("k8s-node-not-ready");
|
|
1294
|
+
|
|
1295
|
+
test("the real firing criteria is an incident-opening `= 0` on the metric value", () => {
|
|
1296
|
+
/*
|
|
1297
|
+
* Guards the fixture: if the template stops firing on `= 0`, the
|
|
1298
|
+
* rest of this suite is testing nothing.
|
|
1299
|
+
*/
|
|
1300
|
+
const criteria: MonitorCriteriaInstance =
|
|
1301
|
+
firingCriteria(nodeNotReadyStep);
|
|
1302
|
+
const filters: Array<CriteriaFilter> = criteria.data?.filters || [];
|
|
1303
|
+
|
|
1304
|
+
expect(filters.length).toBeGreaterThan(0);
|
|
1305
|
+
expect(filters[0]!.checkOn).toBe(CheckOn.MetricValue);
|
|
1306
|
+
expect(filters[0]!.filterType).toBe(FilterType.EqualTo);
|
|
1307
|
+
expect(filters[0]!.value).toBe(0);
|
|
1308
|
+
expect(criteria.data?.createIncidents).toBe(true);
|
|
1309
|
+
});
|
|
1310
|
+
|
|
1311
|
+
test("lists only the NotReady node and the analysis names it", async () => {
|
|
1312
|
+
const context: string | null = await kubernetesContext({
|
|
1313
|
+
metricName: "k8s.node.condition_ready",
|
|
1314
|
+
monitorStep: nodeNotReadyStep,
|
|
1315
|
+
criteriaInstance: firingCriteria(nodeNotReadyStep),
|
|
1316
|
+
affectedResources: [
|
|
1317
|
+
// The NotReady node. The old `> 0` filter removed exactly this row.
|
|
1318
|
+
{ nodeName: "node-notready", metricValue: 0 },
|
|
1319
|
+
{ nodeName: "node-ok-1", metricValue: 1 },
|
|
1320
|
+
],
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
expect(context).toContain(
|
|
1324
|
+
[
|
|
1325
|
+
"**Affected Resources** (1 total)",
|
|
1326
|
+
"",
|
|
1327
|
+
"1. **Node** `node-notready` — **0**",
|
|
1328
|
+
].join("\n"),
|
|
1329
|
+
);
|
|
1330
|
+
expect(context).toContain(
|
|
1331
|
+
"Node `node-notready` is reporting NotReady (value: 0).",
|
|
1332
|
+
);
|
|
1333
|
+
expect(context).toContain("kubectl describe node node-notready");
|
|
1334
|
+
expect(context).not.toContain("node-ok-1");
|
|
1335
|
+
});
|
|
1336
|
+
|
|
1337
|
+
test("a node that was NotReady for only part of the window is still listed", async () => {
|
|
1338
|
+
/*
|
|
1339
|
+
* The worker's highest sample for a flapping node is 1; its lowest
|
|
1340
|
+
* is the 0 the Min-aggregated criteria actually fired on.
|
|
1341
|
+
*/
|
|
1342
|
+
const context: string | null = await kubernetesContext({
|
|
1343
|
+
metricName: "k8s.node.condition_ready",
|
|
1344
|
+
monitorStep: nodeNotReadyStep,
|
|
1345
|
+
criteriaInstance: firingCriteria(nodeNotReadyStep),
|
|
1346
|
+
affectedResources: [
|
|
1347
|
+
{ nodeName: "node-ok-1", metricValue: 1, lowestMetricValue: 1 },
|
|
1348
|
+
{ nodeName: "node-flapping", metricValue: 1, lowestMetricValue: 0 },
|
|
1349
|
+
],
|
|
1350
|
+
});
|
|
1351
|
+
|
|
1352
|
+
expect(context).toContain("**Affected Resources** (1 total)");
|
|
1353
|
+
expect(context).toContain("1. **Node** `node-flapping` — **0**");
|
|
1354
|
+
expect(context).toContain(
|
|
1355
|
+
"Node `node-flapping` is reporting NotReady (value: 0).",
|
|
1356
|
+
);
|
|
1357
|
+
expect(context).not.toContain("node-ok-1");
|
|
1358
|
+
});
|
|
1359
|
+
|
|
1360
|
+
test("every NotReady node survives the top-10 cap", async () => {
|
|
1361
|
+
const affectedResources: Array<KubernetesAffectedResource> = [
|
|
1362
|
+
{ nodeName: "node-down-a", metricValue: 0 },
|
|
1363
|
+
{ nodeName: "node-down-b", metricValue: 0 },
|
|
1364
|
+
];
|
|
1365
|
+
|
|
1366
|
+
/*
|
|
1367
|
+
* Twelve Ready nodes — enough to overflow the ten-row slice if the
|
|
1368
|
+
* predicate ever stops filtering them out.
|
|
1369
|
+
*/
|
|
1370
|
+
for (let i: number = 10; i < 22; i++) {
|
|
1371
|
+
affectedResources.push({ nodeName: `node-ok-${i}`, metricValue: 1 });
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
const context: string | null = await kubernetesContext({
|
|
1375
|
+
metricName: "k8s.node.condition_ready",
|
|
1376
|
+
monitorStep: nodeNotReadyStep,
|
|
1377
|
+
criteriaInstance: firingCriteria(nodeNotReadyStep),
|
|
1378
|
+
affectedResources,
|
|
1379
|
+
});
|
|
1380
|
+
|
|
1381
|
+
expect(context).toContain("**Affected Resources** (2 total)");
|
|
1382
|
+
expect(context).toContain("`node-down-a`");
|
|
1383
|
+
expect(context).toContain("`node-down-b`");
|
|
1384
|
+
expect(context).not.toContain("node-ok-");
|
|
1385
|
+
expect(context).not.toContain("*... and");
|
|
1386
|
+
});
|
|
1387
|
+
|
|
1388
|
+
test("the recovery criteria (an upward comparison) keeps the `> 0` fallback", async () => {
|
|
1389
|
+
const context: string | null = await kubernetesContext({
|
|
1390
|
+
metricName: "k8s.node.condition_ready",
|
|
1391
|
+
monitorStep: nodeNotReadyStep,
|
|
1392
|
+
criteriaInstance: recoveryCriteria(nodeNotReadyStep),
|
|
1393
|
+
affectedResources: [
|
|
1394
|
+
{ nodeName: "node-notready", metricValue: 0 },
|
|
1395
|
+
{ nodeName: "node-ok-1", metricValue: 1 },
|
|
1396
|
+
],
|
|
1397
|
+
});
|
|
1398
|
+
|
|
1399
|
+
expect(context).toContain("1. **Node** `node-ok-1` — **1**");
|
|
1400
|
+
expect(context).not.toContain("node-notready");
|
|
1401
|
+
});
|
|
1402
|
+
});
|
|
1403
|
+
|
|
1404
|
+
test("k8s-etcd-no-leader lists the leaderless member, not the healthy one", async () => {
|
|
1405
|
+
const etcdStep: MonitorStep = kubernetesStep("k8s-etcd-no-leader");
|
|
1406
|
+
const criteria: MonitorCriteriaInstance = firingCriteria(etcdStep);
|
|
1407
|
+
|
|
1408
|
+
// Fixture guard: the real criteria is an incident-opening `= 0`.
|
|
1409
|
+
expect(criteria.data?.filters[0]!.filterType).toBe(FilterType.EqualTo);
|
|
1410
|
+
expect(criteria.data?.filters[0]!.value).toBe(0);
|
|
1411
|
+
expect(criteria.data?.createIncidents).toBe(true);
|
|
1412
|
+
|
|
1413
|
+
const context: string | null = await kubernetesContext({
|
|
1414
|
+
metricName: "etcd_server_has_leader",
|
|
1415
|
+
monitorStep: etcdStep,
|
|
1416
|
+
criteriaInstance: criteria,
|
|
1417
|
+
affectedResources: [
|
|
1418
|
+
{ podName: "etcd-cp-1", metricValue: 1 },
|
|
1419
|
+
{ podName: "etcd-cp-2", metricValue: 0 },
|
|
1420
|
+
],
|
|
1421
|
+
});
|
|
1422
|
+
|
|
1423
|
+
expect(context).toContain("**Affected Resources** (1 total)");
|
|
1424
|
+
expect(context).toContain("1. **Pod** `etcd-cp-2` — **0**");
|
|
1425
|
+
expect(context).toContain("Most affected pod: `etcd-cp-2`");
|
|
1426
|
+
expect(context).not.toContain("etcd-cp-1");
|
|
1427
|
+
});
|
|
1428
|
+
|
|
1429
|
+
test("an `= 0` RECOVERY criteria does NOT turn the all-clear into a list of zeroes", async () => {
|
|
1430
|
+
const replicaStep: MonitorStep = kubernetesStep(
|
|
1431
|
+
"k8s-deployment-replica-mismatch",
|
|
1432
|
+
);
|
|
1433
|
+
const criteria: MonitorCriteriaInstance = recoveryCriteria(replicaStep);
|
|
1434
|
+
|
|
1435
|
+
// Fixture guard: the real recovery is a non-incident `= 0`.
|
|
1436
|
+
expect(criteria.data?.filters[0]!.filterType).toBe(FilterType.EqualTo);
|
|
1437
|
+
expect(criteria.data?.filters[0]!.value).toBe(0);
|
|
1438
|
+
expect(criteria.data?.createIncidents).toBe(false);
|
|
1439
|
+
expect(criteria.data?.createAlerts).toBe(false);
|
|
1440
|
+
|
|
1441
|
+
const context: string | null = await kubernetesContext({
|
|
1442
|
+
metricName: "k8s.deployment.unavailable_replicas",
|
|
1443
|
+
monitorStep: replicaStep,
|
|
1444
|
+
criteriaInstance: criteria,
|
|
1445
|
+
affectedResources: [
|
|
1446
|
+
{ workloadType: "Deployment", workloadName: "web", metricValue: 0 },
|
|
1447
|
+
{ workloadType: "Deployment", workloadName: "api", metricValue: 0 },
|
|
1448
|
+
],
|
|
1449
|
+
});
|
|
1450
|
+
|
|
1451
|
+
expect(context).toContain("**Kubernetes Cluster Details**");
|
|
1452
|
+
expect(context).not.toContain("**Affected Resources**");
|
|
1453
|
+
expect(context).not.toContain("**Root Cause Analysis**");
|
|
1454
|
+
});
|
|
1455
|
+
|
|
1456
|
+
test("a `> 0` criteria renders exactly as the hardcoded filter did, on the highest sample", async () => {
|
|
1457
|
+
const replicaStep: MonitorStep = kubernetesStep(
|
|
1458
|
+
"k8s-deployment-replica-mismatch",
|
|
1459
|
+
);
|
|
1460
|
+
|
|
1461
|
+
const context: string | null = await kubernetesContext({
|
|
1462
|
+
metricName: "k8s.deployment.unavailable_replicas",
|
|
1463
|
+
monitorStep: replicaStep,
|
|
1464
|
+
criteriaInstance: firingCriteria(replicaStep),
|
|
1465
|
+
affectedResources: [
|
|
1466
|
+
{
|
|
1467
|
+
workloadType: "Deployment",
|
|
1468
|
+
workloadName: "web",
|
|
1469
|
+
metricValue: 3,
|
|
1470
|
+
lowestMetricValue: 0,
|
|
1471
|
+
},
|
|
1472
|
+
{ workloadType: "Deployment", workloadName: "api", metricValue: 5 },
|
|
1473
|
+
{ workloadType: "Deployment", workloadName: "ok", metricValue: 0 },
|
|
1474
|
+
],
|
|
1475
|
+
});
|
|
1476
|
+
|
|
1477
|
+
expect(context).toContain("**Affected Resources** (2 total)");
|
|
1478
|
+
expect(context).toContain(
|
|
1479
|
+
[
|
|
1480
|
+
"1. **Deployment** `api` — **5**",
|
|
1481
|
+
"2. **Deployment** `web` — **3**",
|
|
1482
|
+
].join("\n"),
|
|
1483
|
+
);
|
|
1484
|
+
expect(context).not.toContain("`ok`");
|
|
1485
|
+
|
|
1486
|
+
// Still worst-HIGHEST-first for an upward comparison.
|
|
1487
|
+
const apiIndex: number = context!.indexOf("`api`");
|
|
1488
|
+
const webIndex: number = context!.indexOf("`web`");
|
|
1489
|
+
expect(apiIndex).toBeGreaterThan(-1);
|
|
1490
|
+
expect(webIndex).toBeGreaterThan(apiIndex);
|
|
1491
|
+
expect(context).toContain(
|
|
1492
|
+
"Deployment `api` has **5** unavailable replica(s).",
|
|
1493
|
+
);
|
|
1494
|
+
});
|
|
1495
|
+
|
|
1496
|
+
test("a `<` criteria keeps the breaching rows and sorts worst (lowest) first", async () => {
|
|
1497
|
+
const context: string | null = await kubernetesContext({
|
|
1498
|
+
metricName: "k8s.deployment.available",
|
|
1499
|
+
criteriaInstance: syntheticCriteria({
|
|
1500
|
+
opensIncident: true,
|
|
1501
|
+
filters: [
|
|
1502
|
+
{
|
|
1503
|
+
checkOn: CheckOn.MetricValue,
|
|
1504
|
+
filterType: FilterType.LessThan,
|
|
1505
|
+
value: 2,
|
|
1506
|
+
},
|
|
1507
|
+
],
|
|
1508
|
+
}),
|
|
1509
|
+
affectedResources: [
|
|
1510
|
+
{ workloadType: "Deployment", workloadName: "mild", metricValue: 1 },
|
|
1511
|
+
{ workloadType: "Deployment", workloadName: "fine", metricValue: 3 },
|
|
1512
|
+
{ workloadType: "Deployment", workloadName: "worst", metricValue: 0 },
|
|
1513
|
+
],
|
|
1514
|
+
});
|
|
1515
|
+
|
|
1516
|
+
expect(context).toContain("**Affected Resources** (2 total)");
|
|
1517
|
+
expect(context).not.toContain("`fine`");
|
|
1518
|
+
|
|
1519
|
+
const worstIndex: number = context!.indexOf("`worst`");
|
|
1520
|
+
const mildIndex: number = context!.indexOf("`mild`");
|
|
1521
|
+
expect(worstIndex).toBeGreaterThan(-1);
|
|
1522
|
+
expect(mildIndex).toBeGreaterThan(worstIndex);
|
|
1523
|
+
});
|
|
1524
|
+
|
|
1525
|
+
test("a firing equality ABOVE the floor keeps the `> 0` fallback", async () => {
|
|
1526
|
+
/*
|
|
1527
|
+
* `= 1` on a 0/1 pressure condition fires when the metric RISES, so
|
|
1528
|
+
* it must not be read as a fall.
|
|
1529
|
+
*/
|
|
1530
|
+
const context: string | null = await kubernetesContext({
|
|
1531
|
+
metricName: "k8s.node.condition_memory_pressure",
|
|
1532
|
+
criteriaInstance: syntheticCriteria({
|
|
1533
|
+
opensIncident: true,
|
|
1534
|
+
filters: [
|
|
1535
|
+
{
|
|
1536
|
+
checkOn: CheckOn.MetricValue,
|
|
1537
|
+
filterType: FilterType.EqualTo,
|
|
1538
|
+
value: 1,
|
|
1539
|
+
},
|
|
1540
|
+
],
|
|
1541
|
+
}),
|
|
1542
|
+
affectedResources: [
|
|
1543
|
+
{ nodeName: "node-pressured", metricValue: 1, lowestMetricValue: 0 },
|
|
1544
|
+
{ nodeName: "node-ok-1", metricValue: 0 },
|
|
1545
|
+
],
|
|
1546
|
+
});
|
|
1547
|
+
|
|
1548
|
+
expect(context).toContain("1. **Node** `node-pressured` — **1**");
|
|
1549
|
+
expect(context).not.toContain("node-ok-1");
|
|
1550
|
+
});
|
|
1551
|
+
|
|
1552
|
+
test("no criteriaInstance falls back to dropping zero rows", async () => {
|
|
1553
|
+
const context: string | null = await kubernetesContext({
|
|
1554
|
+
metricName: "k8s.node.condition_ready",
|
|
1555
|
+
affectedResources: [
|
|
1556
|
+
{ nodeName: "node-notready", metricValue: 0 },
|
|
1557
|
+
{ nodeName: "node-ok-1", metricValue: 1 },
|
|
1558
|
+
],
|
|
1559
|
+
});
|
|
1560
|
+
|
|
1561
|
+
expect(context).toContain("1. **Node** `node-ok-1` — **1**");
|
|
1562
|
+
expect(context).not.toContain("node-notready");
|
|
1563
|
+
});
|
|
1564
|
+
});
|
|
1565
|
+
|
|
1112
1566
|
/*
|
|
1113
1567
|
* REGRESSION: buildKubernetesRootCauseAnalysis routed POD-scoped metrics
|
|
1114
1568
|
* into the NODE branches.
|
|
@@ -1333,3 +1787,792 @@ describe("MonitorCriteriaEvaluator - Kubernetes root cause analysis scoping", ()
|
|
|
1333
1787
|
});
|
|
1334
1788
|
});
|
|
1335
1789
|
});
|
|
1790
|
+
|
|
1791
|
+
/*
|
|
1792
|
+
* The Kubernetes "Affected Resources" block — the one the table-to-list
|
|
1793
|
+
* change was made for. A pod-restart incident on a GKE cluster used to
|
|
1794
|
+
* carry a six-column table (Namespace / Workload Type / Workload / Pod /
|
|
1795
|
+
* Node / Value) that wrapped every identifier in the email card. Each
|
|
1796
|
+
* resource is now one numbered item titled by the most specific object
|
|
1797
|
+
* the series names, with the rest of its identity as nested bullets.
|
|
1798
|
+
*/
|
|
1799
|
+
describe("MonitorCriteriaEvaluator - Kubernetes affected resources list", () => {
|
|
1800
|
+
// The first line of a top-level list item: "1. ", "10. ", ...
|
|
1801
|
+
const NUMBERED_ITEM: RegExp = /^\d+\. /;
|
|
1802
|
+
|
|
1803
|
+
async function kubernetesContext(input: {
|
|
1804
|
+
affectedResources: Array<KubernetesAffectedResource>;
|
|
1805
|
+
metricName?: string;
|
|
1806
|
+
clusterName?: string;
|
|
1807
|
+
}): Promise<string | null> {
|
|
1808
|
+
return Evaluator.buildKubernetesRootCauseContext({
|
|
1809
|
+
dataToProcess: metricResponse({
|
|
1810
|
+
kubernetesResourceBreakdown: {
|
|
1811
|
+
clusterName:
|
|
1812
|
+
input.clusterName === undefined ? "prod-gke" : input.clusterName,
|
|
1813
|
+
metricName: input.metricName || "k8s.container.restarts",
|
|
1814
|
+
metricFriendlyName: "Container Restarts",
|
|
1815
|
+
affectedResources: input.affectedResources,
|
|
1816
|
+
attributes: {},
|
|
1817
|
+
},
|
|
1818
|
+
}),
|
|
1819
|
+
monitorStep: new MonitorStep(),
|
|
1820
|
+
// No projectId, so the restart branch never reaches for container logs.
|
|
1821
|
+
monitor: new Monitor(),
|
|
1822
|
+
});
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
function podRestart(
|
|
1826
|
+
overrides: Partial<KubernetesAffectedResource>,
|
|
1827
|
+
): KubernetesAffectedResource {
|
|
1828
|
+
return {
|
|
1829
|
+
namespace: "default",
|
|
1830
|
+
workloadType: "Deployment",
|
|
1831
|
+
workloadName: "oneuptime-worker",
|
|
1832
|
+
podName: "oneuptime-worker-6b67db69c9-gtcp6",
|
|
1833
|
+
nodeName: "gke-prod-default-pool-662f6819-e9gq",
|
|
1834
|
+
metricValue: 2,
|
|
1835
|
+
...overrides,
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
test("renders the screenshot's pod restarts as a ranked list, not a table", async () => {
|
|
1840
|
+
const affectedResources: Array<KubernetesAffectedResource> = [
|
|
1841
|
+
podRestart({ metricValue: 2 }),
|
|
1842
|
+
podRestart({
|
|
1843
|
+
workloadType: "Job",
|
|
1844
|
+
workloadName: "oneuptime-migrate-267",
|
|
1845
|
+
podName: "oneuptime-migrate-267-k64qm",
|
|
1846
|
+
nodeName: "gke-prod-default-pool-662f6819-c6w3",
|
|
1847
|
+
metricValue: 3,
|
|
1848
|
+
}),
|
|
1849
|
+
// kube-proxy is a static pod: no owning workload at all.
|
|
1850
|
+
podRestart({
|
|
1851
|
+
namespace: "kube-system",
|
|
1852
|
+
workloadType: undefined,
|
|
1853
|
+
workloadName: undefined,
|
|
1854
|
+
podName: "kube-proxy-gke-prod-default-pool-662f6819-c6w3",
|
|
1855
|
+
nodeName: "gke-prod-default-pool-662f6819-c6w3",
|
|
1856
|
+
metricValue: 1,
|
|
1857
|
+
}),
|
|
1858
|
+
];
|
|
1859
|
+
|
|
1860
|
+
const context: string | null = await kubernetesContext({
|
|
1861
|
+
affectedResources,
|
|
1862
|
+
});
|
|
1863
|
+
|
|
1864
|
+
expect(context).toContain("**Kubernetes Cluster Details**");
|
|
1865
|
+
expect(context).toContain(
|
|
1866
|
+
[
|
|
1867
|
+
"**Affected Resources** (3 total)",
|
|
1868
|
+
"",
|
|
1869
|
+
"1. **Pod** `oneuptime-migrate-267-k64qm` — **3**",
|
|
1870
|
+
" - Namespace: `default`",
|
|
1871
|
+
" - Job: `oneuptime-migrate-267`",
|
|
1872
|
+
" - Node: `gke-prod-default-pool-662f6819-c6w3`",
|
|
1873
|
+
"2. **Pod** `oneuptime-worker-6b67db69c9-gtcp6` — **2**",
|
|
1874
|
+
" - Namespace: `default`",
|
|
1875
|
+
" - Deployment: `oneuptime-worker`",
|
|
1876
|
+
" - Node: `gke-prod-default-pool-662f6819-e9gq`",
|
|
1877
|
+
"3. **Pod** `kube-proxy-gke-prod-default-pool-662f6819-c6w3` — **1**",
|
|
1878
|
+
" - Namespace: `kube-system`",
|
|
1879
|
+
" - Node: `gke-prod-default-pool-662f6819-c6w3`",
|
|
1880
|
+
].join("\n"),
|
|
1881
|
+
);
|
|
1882
|
+
|
|
1883
|
+
// None of the old table survives.
|
|
1884
|
+
expect(context).not.toContain("| Namespace |");
|
|
1885
|
+
expect(context).not.toContain("Workload Type");
|
|
1886
|
+
expect(context).not.toContain("| --- |");
|
|
1887
|
+
expect(context).not.toMatch(/^\|/m);
|
|
1888
|
+
// A missing workload is left out, not printed as "-".
|
|
1889
|
+
expect(context).not.toContain(": -");
|
|
1890
|
+
expect(context).not.toContain("| - |");
|
|
1891
|
+
});
|
|
1892
|
+
|
|
1893
|
+
test("caps the list at 10 with an '... and N more' summary, like the 83-pod incident", async () => {
|
|
1894
|
+
const affectedResources: Array<KubernetesAffectedResource> = [];
|
|
1895
|
+
for (let i: number = 1; i <= 83; i++) {
|
|
1896
|
+
affectedResources.push(
|
|
1897
|
+
podRestart({ podName: `worker-${i}`, metricValue: i }),
|
|
1898
|
+
);
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
const context: string | null = await kubernetesContext({
|
|
1902
|
+
affectedResources,
|
|
1903
|
+
});
|
|
1904
|
+
|
|
1905
|
+
expect(context).toContain("**Affected Resources** (83 total)");
|
|
1906
|
+
expect(context).toContain("1. **Pod** `worker-83` — **83**");
|
|
1907
|
+
expect(context).toContain(
|
|
1908
|
+
[
|
|
1909
|
+
"10. **Pod** `worker-74` — **74**",
|
|
1910
|
+
" - Namespace: `default`",
|
|
1911
|
+
" - Deployment: `oneuptime-worker`",
|
|
1912
|
+
" - Node: `gke-prod-default-pool-662f6819-e9gq`",
|
|
1913
|
+
"",
|
|
1914
|
+
"*... and 73 more affected resources*",
|
|
1915
|
+
].join("\n"),
|
|
1916
|
+
);
|
|
1917
|
+
expect(context).not.toContain("11. ");
|
|
1918
|
+
expect(context).not.toContain("`worker-73`");
|
|
1919
|
+
});
|
|
1920
|
+
|
|
1921
|
+
test("a container-level series is titled by the container and its pod", async () => {
|
|
1922
|
+
const context: string | null = await kubernetesContext({
|
|
1923
|
+
affectedResources: [
|
|
1924
|
+
podRestart({ containerName: "app", podName: "checkout-7d9f" }),
|
|
1925
|
+
],
|
|
1926
|
+
});
|
|
1927
|
+
|
|
1928
|
+
expect(context).toContain(
|
|
1929
|
+
[
|
|
1930
|
+
"1. **Container** `app` in pod `checkout-7d9f` — **2**",
|
|
1931
|
+
" - Namespace: `default`",
|
|
1932
|
+
" - Deployment: `oneuptime-worker`",
|
|
1933
|
+
" - Node: `gke-prod-default-pool-662f6819-e9gq`",
|
|
1934
|
+
].join("\n"),
|
|
1935
|
+
);
|
|
1936
|
+
// The pod is in the title, so it is not repeated as a bullet.
|
|
1937
|
+
expect(context).not.toContain("- Pod:");
|
|
1938
|
+
});
|
|
1939
|
+
|
|
1940
|
+
/*
|
|
1941
|
+
* Every replica of a workload runs a container of the same name. Titled
|
|
1942
|
+
* by the container alone, three crash-looping replicas would read
|
|
1943
|
+
* "Container `checkout`" three times over.
|
|
1944
|
+
*/
|
|
1945
|
+
test("replicas that share a container name still get distinct titles", async () => {
|
|
1946
|
+
const context: string | null = await kubernetesContext({
|
|
1947
|
+
affectedResources: [
|
|
1948
|
+
podRestart({
|
|
1949
|
+
containerName: "checkout",
|
|
1950
|
+
podName: "checkout-7d9f6c5b8-abcde",
|
|
1951
|
+
metricValue: 5,
|
|
1952
|
+
}),
|
|
1953
|
+
podRestart({
|
|
1954
|
+
containerName: "checkout",
|
|
1955
|
+
podName: "checkout-7d9f6c5b8-fghij",
|
|
1956
|
+
metricValue: 4,
|
|
1957
|
+
}),
|
|
1958
|
+
podRestart({
|
|
1959
|
+
containerName: "checkout",
|
|
1960
|
+
podName: "checkout-7d9f6c5b8-klmno",
|
|
1961
|
+
metricValue: 3,
|
|
1962
|
+
}),
|
|
1963
|
+
],
|
|
1964
|
+
});
|
|
1965
|
+
|
|
1966
|
+
const titles: Array<string> = (context || "")
|
|
1967
|
+
.split("\n")
|
|
1968
|
+
.filter((line: string) => {
|
|
1969
|
+
return NUMBERED_ITEM.test(line);
|
|
1970
|
+
});
|
|
1971
|
+
|
|
1972
|
+
expect(titles).toEqual([
|
|
1973
|
+
"1. **Container** `checkout` in pod `checkout-7d9f6c5b8-abcde` — **5**",
|
|
1974
|
+
"2. **Container** `checkout` in pod `checkout-7d9f6c5b8-fghij` — **4**",
|
|
1975
|
+
"3. **Container** `checkout` in pod `checkout-7d9f6c5b8-klmno` — **3**",
|
|
1976
|
+
]);
|
|
1977
|
+
expect(new Set(titles).size).toBe(3);
|
|
1978
|
+
});
|
|
1979
|
+
|
|
1980
|
+
test("a container without a pod is titled by the container alone", async () => {
|
|
1981
|
+
const context: string | null = await kubernetesContext({
|
|
1982
|
+
affectedResources: [
|
|
1983
|
+
{ containerName: "app", namespace: "default", metricValue: 2 },
|
|
1984
|
+
],
|
|
1985
|
+
});
|
|
1986
|
+
|
|
1987
|
+
expect(context).toContain(
|
|
1988
|
+
"1. **Container** `app` — **2**\n - Namespace: `default`",
|
|
1989
|
+
);
|
|
1990
|
+
});
|
|
1991
|
+
|
|
1992
|
+
test("a node-level series is titled by the node and has nothing to list under it", async () => {
|
|
1993
|
+
const context: string | null = await kubernetesContext({
|
|
1994
|
+
metricName: "k8s.node.memory.usage",
|
|
1995
|
+
affectedResources: [
|
|
1996
|
+
{ nodeName: "gke-prod-pool-a", metricValue: 8589934592 },
|
|
1997
|
+
{ nodeName: "gke-prod-pool-b", metricValue: 1073741824 },
|
|
1998
|
+
],
|
|
1999
|
+
});
|
|
2000
|
+
|
|
2001
|
+
expect(context).toContain(
|
|
2002
|
+
[
|
|
2003
|
+
"1. **Node** `gke-prod-pool-a` — **8.59 GB**",
|
|
2004
|
+
"2. **Node** `gke-prod-pool-b` — **1.07 GB**",
|
|
2005
|
+
].join("\n"),
|
|
2006
|
+
);
|
|
2007
|
+
expect(context).not.toContain("- Node:");
|
|
2008
|
+
});
|
|
2009
|
+
|
|
2010
|
+
test("a workload-level series is titled by its workload type and does not repeat it", async () => {
|
|
2011
|
+
const context: string | null = await kubernetesContext({
|
|
2012
|
+
metricName: "k8s.deployment.unavailable_replicas",
|
|
2013
|
+
affectedResources: [
|
|
2014
|
+
{
|
|
2015
|
+
namespace: "payments",
|
|
2016
|
+
workloadType: "StatefulSet",
|
|
2017
|
+
workloadName: "ledger",
|
|
2018
|
+
metricValue: 2,
|
|
2019
|
+
},
|
|
2020
|
+
],
|
|
2021
|
+
});
|
|
2022
|
+
|
|
2023
|
+
expect(context).toContain(
|
|
2024
|
+
"1. **StatefulSet** `ledger` — **2**\n - Namespace: `payments`",
|
|
2025
|
+
);
|
|
2026
|
+
expect(context).not.toContain("- StatefulSet:");
|
|
2027
|
+
});
|
|
2028
|
+
|
|
2029
|
+
test("a workload name without a type falls back to the generic 'Workload' label", async () => {
|
|
2030
|
+
const context: string | null = await kubernetesContext({
|
|
2031
|
+
affectedResources: [
|
|
2032
|
+
podRestart({ workloadType: undefined, workloadName: "mystery" }),
|
|
2033
|
+
],
|
|
2034
|
+
});
|
|
2035
|
+
|
|
2036
|
+
expect(context).toContain(" - Workload: `mystery`");
|
|
2037
|
+
});
|
|
2038
|
+
|
|
2039
|
+
test("a workload-level series without a type is titled 'Workload'", async () => {
|
|
2040
|
+
const context: string | null = await kubernetesContext({
|
|
2041
|
+
affectedResources: [
|
|
2042
|
+
{ namespace: "payments", workloadName: "ledger", metricValue: 1 },
|
|
2043
|
+
],
|
|
2044
|
+
});
|
|
2045
|
+
|
|
2046
|
+
expect(context).toContain(
|
|
2047
|
+
"1. **Workload** `ledger` — **1**\n - Namespace: `payments`",
|
|
2048
|
+
);
|
|
2049
|
+
});
|
|
2050
|
+
|
|
2051
|
+
test("a namespace-level series is titled by the namespace", async () => {
|
|
2052
|
+
const context: string | null = await kubernetesContext({
|
|
2053
|
+
affectedResources: [{ namespace: "payments", metricValue: 4 }],
|
|
2054
|
+
});
|
|
2055
|
+
|
|
2056
|
+
expect(context).toContain("1. **Namespace** `payments` — **4**");
|
|
2057
|
+
expect(context).not.toContain("- Namespace:");
|
|
2058
|
+
});
|
|
2059
|
+
|
|
2060
|
+
test("a series with no Kubernetes identity is a cluster-level series and is titled by the cluster", async () => {
|
|
2061
|
+
const context: string | null = await kubernetesContext({
|
|
2062
|
+
affectedResources: [{ metricValue: 5 }],
|
|
2063
|
+
});
|
|
2064
|
+
|
|
2065
|
+
expect(context).toContain("1. **Cluster** `prod-gke` — **5**");
|
|
2066
|
+
});
|
|
2067
|
+
|
|
2068
|
+
test("items of different shapes in one breakdown are each titled by their own kind", async () => {
|
|
2069
|
+
const context: string | null = await kubernetesContext({
|
|
2070
|
+
affectedResources: [
|
|
2071
|
+
podRestart({ containerName: "app", metricValue: 4 }),
|
|
2072
|
+
podRestart({ metricValue: 3 }),
|
|
2073
|
+
{ nodeName: "gke-prod-pool-a", metricValue: 2 },
|
|
2074
|
+
{ metricValue: 1 },
|
|
2075
|
+
],
|
|
2076
|
+
});
|
|
2077
|
+
|
|
2078
|
+
const titles: Array<string> = (context || "")
|
|
2079
|
+
.split("\n")
|
|
2080
|
+
.filter((line: string) => {
|
|
2081
|
+
return NUMBERED_ITEM.test(line);
|
|
2082
|
+
});
|
|
2083
|
+
|
|
2084
|
+
expect(titles).toEqual([
|
|
2085
|
+
"1. **Container** `app` in pod `oneuptime-worker-6b67db69c9-gtcp6` — **4**",
|
|
2086
|
+
"2. **Pod** `oneuptime-worker-6b67db69c9-gtcp6` — **3**",
|
|
2087
|
+
"3. **Node** `gke-prod-pool-a` — **2**",
|
|
2088
|
+
"4. **Cluster** `prod-gke` — **1**",
|
|
2089
|
+
]);
|
|
2090
|
+
});
|
|
2091
|
+
|
|
2092
|
+
test("zero-value resources are dropped, and an all-zero breakdown renders no list", async () => {
|
|
2093
|
+
const some: string | null = await kubernetesContext({
|
|
2094
|
+
affectedResources: [
|
|
2095
|
+
podRestart({ podName: "healthy", metricValue: 0 }),
|
|
2096
|
+
podRestart({ podName: "crashing", metricValue: 7 }),
|
|
2097
|
+
],
|
|
2098
|
+
});
|
|
2099
|
+
|
|
2100
|
+
expect(some).toContain("**Affected Resources** (1 total)");
|
|
2101
|
+
expect(some).toContain("1. **Pod** `crashing` — **7**");
|
|
2102
|
+
expect(some).not.toContain("healthy");
|
|
2103
|
+
|
|
2104
|
+
const none: string | null = await kubernetesContext({
|
|
2105
|
+
affectedResources: [podRestart({ metricValue: 0 })],
|
|
2106
|
+
});
|
|
2107
|
+
|
|
2108
|
+
expect(none).toContain("**Kubernetes Cluster Details**");
|
|
2109
|
+
expect(none).not.toContain("**Affected Resources**");
|
|
2110
|
+
expect(none).not.toContain("1. ");
|
|
2111
|
+
});
|
|
2112
|
+
|
|
2113
|
+
test("the Root Cause Analysis still names the worst resource and follows the list as its own block", async () => {
|
|
2114
|
+
const context: string | null = await kubernetesContext({
|
|
2115
|
+
affectedResources: [
|
|
2116
|
+
podRestart({ containerName: "app", podName: "checkout-7d9f" }),
|
|
2117
|
+
],
|
|
2118
|
+
});
|
|
2119
|
+
|
|
2120
|
+
expect(context).toContain(
|
|
2121
|
+
" - Node: `gke-prod-default-pool-662f6819-e9gq`\n\n\n**Root Cause Analysis**",
|
|
2122
|
+
);
|
|
2123
|
+
expect(context).toContain("kubectl logs checkout-7d9f -c app --previous");
|
|
2124
|
+
});
|
|
2125
|
+
|
|
2126
|
+
test("a name containing a backtick cannot break out of its code span", async () => {
|
|
2127
|
+
const context: string | null = await kubernetesContext({
|
|
2128
|
+
affectedResources: [podRestart({ podName: "odd`pod" })],
|
|
2129
|
+
});
|
|
2130
|
+
|
|
2131
|
+
expect(context).toContain("1. **Pod** `` odd`pod `` — **2**");
|
|
2132
|
+
});
|
|
2133
|
+
|
|
2134
|
+
test("the whole root cause renders to an email as nested lists, not a table", async () => {
|
|
2135
|
+
const context: string | null = await kubernetesContext({
|
|
2136
|
+
affectedResources: [
|
|
2137
|
+
podRestart({ metricValue: 3 }),
|
|
2138
|
+
podRestart({ podName: "second", metricValue: 2 }),
|
|
2139
|
+
],
|
|
2140
|
+
});
|
|
2141
|
+
|
|
2142
|
+
const html: string = await Markdown.convertToHTML(
|
|
2143
|
+
context!,
|
|
2144
|
+
MarkdownContentType.Email,
|
|
2145
|
+
);
|
|
2146
|
+
|
|
2147
|
+
expect(html).not.toContain("<table");
|
|
2148
|
+
expect(html).not.toContain("<th");
|
|
2149
|
+
expect((html.match(/<ol /g) || []).length).toBe(1);
|
|
2150
|
+
// Cluster details list + one nested details list per resource.
|
|
2151
|
+
expect((html.match(/<ul /g) || []).length).toBe(3);
|
|
2152
|
+
expect(html).toMatch(
|
|
2153
|
+
/<li style="[^"]+"><strong>Pod<\/strong> <code[^>]*>oneuptime-worker-6b67db69c9-gtcp6<\/code> — <strong>3<\/strong><ul style="[^"]+"><li style="[^"]+">Namespace: <code[^>]*>default<\/code><\/li>/,
|
|
2154
|
+
);
|
|
2155
|
+
});
|
|
2156
|
+
});
|
|
2157
|
+
|
|
2158
|
+
describe("MonitorCriteriaEvaluator - Docker Swarm affected tasks list", () => {
|
|
2159
|
+
function swarmContext(input: {
|
|
2160
|
+
affectedResources: Array<DockerSwarmAffectedResource>;
|
|
2161
|
+
metricResult?: Array<any>;
|
|
2162
|
+
}): string | null {
|
|
2163
|
+
return Evaluator.buildDockerSwarmRootCauseContext({
|
|
2164
|
+
dataToProcess: metricResponse({
|
|
2165
|
+
dockerSwarmResourceBreakdown: {
|
|
2166
|
+
clusterName: "swarm-prod",
|
|
2167
|
+
metricName: "container.pids.count",
|
|
2168
|
+
metricFriendlyName: "Process Count",
|
|
2169
|
+
affectedResources: input.affectedResources,
|
|
2170
|
+
attributes: {},
|
|
2171
|
+
},
|
|
2172
|
+
metricResult: input.metricResult || [],
|
|
2173
|
+
}),
|
|
2174
|
+
monitorStep: new MonitorStep(),
|
|
2175
|
+
monitor: new Monitor(),
|
|
2176
|
+
});
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
test("renders each task with its service and node beneath it, worst first", () => {
|
|
2180
|
+
const context: string | null = swarmContext({
|
|
2181
|
+
affectedResources: [
|
|
2182
|
+
{
|
|
2183
|
+
containerName: "web.1.abc123",
|
|
2184
|
+
serviceName: "web",
|
|
2185
|
+
nodeName: "worker-1",
|
|
2186
|
+
metricValue: 40,
|
|
2187
|
+
},
|
|
2188
|
+
{
|
|
2189
|
+
containerName: "api.2.def456",
|
|
2190
|
+
serviceName: "api",
|
|
2191
|
+
nodeName: "worker-2",
|
|
2192
|
+
metricValue: 90,
|
|
2193
|
+
},
|
|
2194
|
+
// Zero row dropped.
|
|
2195
|
+
{
|
|
2196
|
+
containerName: "idle.1.xyz",
|
|
2197
|
+
serviceName: "idle",
|
|
2198
|
+
nodeName: "worker-3",
|
|
2199
|
+
metricValue: 0,
|
|
2200
|
+
},
|
|
2201
|
+
],
|
|
2202
|
+
});
|
|
2203
|
+
|
|
2204
|
+
expect(context).toContain("**Docker Swarm Cluster Details**");
|
|
2205
|
+
expect(context).toContain(
|
|
2206
|
+
[
|
|
2207
|
+
"**Affected Tasks** (2 total)",
|
|
2208
|
+
"",
|
|
2209
|
+
"1. **Task** `api.2.def456` — **90**",
|
|
2210
|
+
" - Service: `api`",
|
|
2211
|
+
" - Node: `worker-2`",
|
|
2212
|
+
"2. **Task** `web.1.abc123` — **40**",
|
|
2213
|
+
" - Service: `web`",
|
|
2214
|
+
" - Node: `worker-1`",
|
|
2215
|
+
].join("\n"),
|
|
2216
|
+
);
|
|
2217
|
+
expect(context).not.toContain("idle");
|
|
2218
|
+
expect(context).not.toContain("Task / Container");
|
|
2219
|
+
expect(context).not.toContain("| --- |");
|
|
2220
|
+
});
|
|
2221
|
+
|
|
2222
|
+
test("a series without a task is titled by its service, then its node", () => {
|
|
2223
|
+
const context: string | null = swarmContext({
|
|
2224
|
+
affectedResources: [
|
|
2225
|
+
{ serviceName: "web", nodeName: "worker-1", metricValue: 2 },
|
|
2226
|
+
{ nodeName: "worker-2", metricValue: 1 },
|
|
2227
|
+
],
|
|
2228
|
+
});
|
|
2229
|
+
|
|
2230
|
+
expect(context).toContain(
|
|
2231
|
+
[
|
|
2232
|
+
"1. **Service** `web` — **2**",
|
|
2233
|
+
" - Node: `worker-1`",
|
|
2234
|
+
"2. **Node** `worker-2` — **1**",
|
|
2235
|
+
].join("\n"),
|
|
2236
|
+
);
|
|
2237
|
+
});
|
|
2238
|
+
|
|
2239
|
+
test("overflow is counted in tasks", () => {
|
|
2240
|
+
const affectedResources: Array<DockerSwarmAffectedResource> = [];
|
|
2241
|
+
for (let i: number = 1; i <= 13; i++) {
|
|
2242
|
+
affectedResources.push({
|
|
2243
|
+
containerName: `web.${i}.task`,
|
|
2244
|
+
serviceName: "web",
|
|
2245
|
+
metricValue: i,
|
|
2246
|
+
});
|
|
2247
|
+
}
|
|
2248
|
+
|
|
2249
|
+
const context: string | null = swarmContext({ affectedResources });
|
|
2250
|
+
|
|
2251
|
+
expect(context).toContain("**Affected Tasks** (13 total)");
|
|
2252
|
+
expect(context).toContain("*... and 3 more affected tasks*");
|
|
2253
|
+
expect(context).toContain("10. **Task** `web.4.task` — **4**");
|
|
2254
|
+
});
|
|
2255
|
+
|
|
2256
|
+
test("a row with no identity inside an attributed breakdown is titled by the cluster", () => {
|
|
2257
|
+
const context: string | null = swarmContext({
|
|
2258
|
+
affectedResources: [
|
|
2259
|
+
{ containerName: "web.1", serviceName: "web", metricValue: 2 },
|
|
2260
|
+
{ metricValue: 1 },
|
|
2261
|
+
],
|
|
2262
|
+
});
|
|
2263
|
+
|
|
2264
|
+
expect(context).toContain(
|
|
2265
|
+
[
|
|
2266
|
+
"1. **Task** `web.1` — **2**",
|
|
2267
|
+
" - Service: `web`",
|
|
2268
|
+
"2. **Cluster** `swarm-prod` — **1**",
|
|
2269
|
+
].join("\n"),
|
|
2270
|
+
);
|
|
2271
|
+
});
|
|
2272
|
+
|
|
2273
|
+
test("an identity-less breakdown renders no list and falls back to the metric summary", () => {
|
|
2274
|
+
const context: string | null = swarmContext({
|
|
2275
|
+
affectedResources: [{ metricValue: 3 }],
|
|
2276
|
+
metricResult: [{ data: [{}, {}] }],
|
|
2277
|
+
});
|
|
2278
|
+
|
|
2279
|
+
expect(context).not.toContain("**Affected Tasks**");
|
|
2280
|
+
expect(context).toContain("**Metric Summary**");
|
|
2281
|
+
});
|
|
2282
|
+
});
|
|
2283
|
+
|
|
2284
|
+
describe("MonitorCriteriaEvaluator - per-platform resource kinds in the list", () => {
|
|
2285
|
+
function proxmoxContext(
|
|
2286
|
+
affectedResources: Array<ProxmoxAffectedResource>,
|
|
2287
|
+
): string | null {
|
|
2288
|
+
return Evaluator.buildProxmoxRootCauseContext({
|
|
2289
|
+
dataToProcess: metricResponse({
|
|
2290
|
+
proxmoxResourceBreakdown: {
|
|
2291
|
+
clusterName: "prod-cluster",
|
|
2292
|
+
metricName: "pve_replication_failed_syncs",
|
|
2293
|
+
metricFriendlyName: "Failed Replication Syncs",
|
|
2294
|
+
affectedResources,
|
|
2295
|
+
attributes: {},
|
|
2296
|
+
},
|
|
2297
|
+
}),
|
|
2298
|
+
monitorStep: proxmoxStep(),
|
|
2299
|
+
monitor: new Monitor(),
|
|
2300
|
+
});
|
|
2301
|
+
}
|
|
2302
|
+
|
|
2303
|
+
test("Proxmox: each pve.type gets a readable kind, and a node does not list itself as its node", () => {
|
|
2304
|
+
const context: string | null = proxmoxContext([
|
|
2305
|
+
{
|
|
2306
|
+
resourceId: "lxc/200",
|
|
2307
|
+
resourceName: "cache",
|
|
2308
|
+
resourceType: "lxc",
|
|
2309
|
+
scope: "guest",
|
|
2310
|
+
nodeName: "pve1",
|
|
2311
|
+
metricValue: 5,
|
|
2312
|
+
},
|
|
2313
|
+
{
|
|
2314
|
+
resourceId: "node/pve1",
|
|
2315
|
+
resourceName: "pve1",
|
|
2316
|
+
resourceType: "node",
|
|
2317
|
+
scope: "node",
|
|
2318
|
+
nodeName: "pve1",
|
|
2319
|
+
metricValue: 4,
|
|
2320
|
+
},
|
|
2321
|
+
{
|
|
2322
|
+
resourceId: "storage/pve1/local-lvm",
|
|
2323
|
+
resourceName: "local-lvm",
|
|
2324
|
+
resourceType: "storage",
|
|
2325
|
+
scope: "storage",
|
|
2326
|
+
nodeName: "pve1",
|
|
2327
|
+
metricValue: 3,
|
|
2328
|
+
},
|
|
2329
|
+
]);
|
|
2330
|
+
|
|
2331
|
+
expect(context).toContain(
|
|
2332
|
+
[
|
|
2333
|
+
"1. **Container** `cache` (`lxc/200`) — **5**",
|
|
2334
|
+
" - Node: `pve1`",
|
|
2335
|
+
"2. **Node** `pve1` (`node/pve1`) — **4**",
|
|
2336
|
+
"3. **Storage** `local-lvm` (`storage/pve1/local-lvm`) — **3**",
|
|
2337
|
+
" - Node: `pve1`",
|
|
2338
|
+
].join("\n"),
|
|
2339
|
+
);
|
|
2340
|
+
});
|
|
2341
|
+
|
|
2342
|
+
test("Proxmox: an unknown pve.type is shown as it arrived, and scope fills in when type is missing", () => {
|
|
2343
|
+
const context: string | null = proxmoxContext([
|
|
2344
|
+
{ resourceId: "sdn/zone1", resourceType: "sdn", metricValue: 2 },
|
|
2345
|
+
{ resourceId: "qemu/300", scope: "guest", metricValue: 1 },
|
|
2346
|
+
]);
|
|
2347
|
+
|
|
2348
|
+
expect(context).toContain("1. **sdn** `sdn/zone1` — **2**");
|
|
2349
|
+
expect(context).toContain("2. **Guest** `qemu/300` — **1**");
|
|
2350
|
+
});
|
|
2351
|
+
|
|
2352
|
+
/*
|
|
2353
|
+
* The agent's transform/pve-identity processor sets pve.scope=cluster
|
|
2354
|
+
* and leaves pve.type unset on `cluster/*` series, so this is the one
|
|
2355
|
+
* scope-only input that really arrives.
|
|
2356
|
+
*/
|
|
2357
|
+
test("Proxmox: cluster/* series (scope only) are titled Cluster; no type or scope falls back to Resource", () => {
|
|
2358
|
+
const context: string | null = proxmoxContext([
|
|
2359
|
+
{ resourceId: "cluster/prod", scope: "cluster", metricValue: 4 },
|
|
2360
|
+
{ resourceId: "node/pve3", scope: "node", metricValue: 3 },
|
|
2361
|
+
{ resourceId: "pool/backups", scope: "pool", metricValue: 2 },
|
|
2362
|
+
{ resourceId: "qemu/100", metricValue: 1 },
|
|
2363
|
+
]);
|
|
2364
|
+
|
|
2365
|
+
expect(context).toContain(
|
|
2366
|
+
[
|
|
2367
|
+
"1. **Cluster** `cluster/prod` — **4**",
|
|
2368
|
+
"2. **Node** `node/pve3` — **3**",
|
|
2369
|
+
"3. **pool** `pool/backups` — **2**",
|
|
2370
|
+
"4. **Resource** `qemu/100` — **1**",
|
|
2371
|
+
].join("\n"),
|
|
2372
|
+
);
|
|
2373
|
+
});
|
|
2374
|
+
|
|
2375
|
+
test("Proxmox: a node-only row is titled by the node; a row with no identity by the cluster", () => {
|
|
2376
|
+
const context: string | null = proxmoxContext([
|
|
2377
|
+
{ nodeName: "pve2", metricValue: 2 },
|
|
2378
|
+
{ metricValue: 1 },
|
|
2379
|
+
]);
|
|
2380
|
+
|
|
2381
|
+
expect(context).toContain(
|
|
2382
|
+
"1. **Node** `pve2` — **2**\n2. **Cluster** `prod-cluster` — **1**",
|
|
2383
|
+
);
|
|
2384
|
+
});
|
|
2385
|
+
|
|
2386
|
+
test("VMware: a VM with no display name is named by its instance UUID", () => {
|
|
2387
|
+
const context: string | null = Evaluator.buildVMwareRootCauseContext({
|
|
2388
|
+
dataToProcess: metricResponse({
|
|
2389
|
+
vmwareResourceBreakdown: {
|
|
2390
|
+
vcenterName: "vcsa-prod",
|
|
2391
|
+
metricName: "vcenter.vm.disk.usage",
|
|
2392
|
+
metricFriendlyName: "VM Disk Usage",
|
|
2393
|
+
affectedResources: [
|
|
2394
|
+
{
|
|
2395
|
+
datacenterName: "DC1",
|
|
2396
|
+
hostName: "esx-01",
|
|
2397
|
+
vmId: "5029abcd-1111-2222-3333-444455556666",
|
|
2398
|
+
metricValue: 1073741824,
|
|
2399
|
+
},
|
|
2400
|
+
],
|
|
2401
|
+
attributes: {},
|
|
2402
|
+
},
|
|
2403
|
+
}),
|
|
2404
|
+
monitorStep: vmwareStep(),
|
|
2405
|
+
monitor: new Monitor(),
|
|
2406
|
+
});
|
|
2407
|
+
|
|
2408
|
+
expect(context).toContain(
|
|
2409
|
+
"1. **Virtual Machine** `5029abcd-1111-2222-3333-444455556666` — **1.07 GB**\n - Host: `esx-01`",
|
|
2410
|
+
);
|
|
2411
|
+
});
|
|
2412
|
+
|
|
2413
|
+
test("VMware: a pool on a standalone host lists its owner host; a row with no identity is the vCenter", () => {
|
|
2414
|
+
const context: string | null = Evaluator.buildVMwareRootCauseContext({
|
|
2415
|
+
dataToProcess: metricResponse({
|
|
2416
|
+
vmwareResourceBreakdown: {
|
|
2417
|
+
vcenterName: "vcsa-prod",
|
|
2418
|
+
metricName: "vcenter.datacenter.vm.count",
|
|
2419
|
+
metricFriendlyName: "Datacenter VM Count",
|
|
2420
|
+
affectedResources: [
|
|
2421
|
+
{
|
|
2422
|
+
datacenterName: "DC1",
|
|
2423
|
+
hostName: "esx-standalone",
|
|
2424
|
+
resourcePoolName: "Resources",
|
|
2425
|
+
resourcePoolPath: "/DC1/host/esx-standalone/Resources",
|
|
2426
|
+
metricValue: 2,
|
|
2427
|
+
},
|
|
2428
|
+
{ metricValue: 1 },
|
|
2429
|
+
],
|
|
2430
|
+
attributes: {},
|
|
2431
|
+
},
|
|
2432
|
+
}),
|
|
2433
|
+
monitorStep: vmwareStep(),
|
|
2434
|
+
monitor: new Monitor(),
|
|
2435
|
+
});
|
|
2436
|
+
|
|
2437
|
+
expect(context).toContain(
|
|
2438
|
+
[
|
|
2439
|
+
"1. **Resource Pool** `Resources` (`/DC1/host/esx-standalone/Resources`) — **2**",
|
|
2440
|
+
" - Host: `esx-standalone`",
|
|
2441
|
+
"2. **vCenter** `vcsa-prod` — **1**",
|
|
2442
|
+
].join("\n"),
|
|
2443
|
+
);
|
|
2444
|
+
});
|
|
2445
|
+
|
|
2446
|
+
/*
|
|
2447
|
+
* The worker stores every Ceph series' `name` label as poolName. On
|
|
2448
|
+
* ceph_health_detail that label is the health check — RECENT_CRASH,
|
|
2449
|
+
* OSD_NEARFULL — not a pool, and there is no pool_id beside it.
|
|
2450
|
+
*/
|
|
2451
|
+
test("Ceph: a ceph_health_detail series is titled as a health check, not a pool", () => {
|
|
2452
|
+
const template: CephAlertTemplate | undefined =
|
|
2453
|
+
getCephAlertTemplateById("ceph-daemon-crash");
|
|
2454
|
+
if (!template) {
|
|
2455
|
+
throw new Error("ceph-daemon-crash template missing");
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
const step: MonitorStep = template.getMonitorStep(templateArgs());
|
|
2459
|
+
|
|
2460
|
+
const context: string | null = Evaluator.buildCephRootCauseContext({
|
|
2461
|
+
dataToProcess: metricResponse({
|
|
2462
|
+
cephResourceBreakdown: {
|
|
2463
|
+
clusterName: "prod-cluster",
|
|
2464
|
+
metricName: "ceph_health_detail",
|
|
2465
|
+
metricFriendlyName: "Health Detail",
|
|
2466
|
+
affectedResources: [{ poolName: "RECENT_CRASH", metricValue: 1 }],
|
|
2467
|
+
attributes: {},
|
|
2468
|
+
},
|
|
2469
|
+
}),
|
|
2470
|
+
monitorStep: step,
|
|
2471
|
+
monitor: new Monitor(),
|
|
2472
|
+
criteriaInstance:
|
|
2473
|
+
step.data!.monitorCriteria!.data!.monitorCriteriaInstanceArray[0],
|
|
2474
|
+
});
|
|
2475
|
+
|
|
2476
|
+
expect(context).toContain(
|
|
2477
|
+
"**Affected Resources** (1 total)\n\n1. **Health Check** `RECENT_CRASH` — **1**",
|
|
2478
|
+
);
|
|
2479
|
+
expect(context).not.toContain("**Pool**");
|
|
2480
|
+
expect(context).not.toContain("- Pool:");
|
|
2481
|
+
});
|
|
2482
|
+
|
|
2483
|
+
test("Ceph: a name without a pool id on another metric is a generic resource, and a daemon keeps it as a detail", () => {
|
|
2484
|
+
const context: string | null = Evaluator.buildCephRootCauseContext({
|
|
2485
|
+
dataToProcess: metricResponse({
|
|
2486
|
+
cephResourceBreakdown: {
|
|
2487
|
+
clusterName: "prod-cluster",
|
|
2488
|
+
metricName: "ceph_fs_metadata",
|
|
2489
|
+
metricFriendlyName: "Filesystem Metadata",
|
|
2490
|
+
affectedResources: [
|
|
2491
|
+
{ poolName: "cephfs", metricValue: 2 },
|
|
2492
|
+
{
|
|
2493
|
+
daemon: "mds.a",
|
|
2494
|
+
poolName: "cephfs",
|
|
2495
|
+
hostname: "ceph-node-1",
|
|
2496
|
+
metricValue: 1,
|
|
2497
|
+
},
|
|
2498
|
+
],
|
|
2499
|
+
attributes: {},
|
|
2500
|
+
},
|
|
2501
|
+
}),
|
|
2502
|
+
monitorStep: cephStep(),
|
|
2503
|
+
monitor: new Monitor(),
|
|
2504
|
+
});
|
|
2505
|
+
|
|
2506
|
+
expect(context).toContain(
|
|
2507
|
+
[
|
|
2508
|
+
"1. **Resource** `cephfs` — **2**",
|
|
2509
|
+
"2. **Daemon** `mds.a` — **1**",
|
|
2510
|
+
" - Name: `cephfs`",
|
|
2511
|
+
" - Host: `ceph-node-1`",
|
|
2512
|
+
].join("\n"),
|
|
2513
|
+
);
|
|
2514
|
+
expect(context).not.toContain("**Pool**");
|
|
2515
|
+
});
|
|
2516
|
+
|
|
2517
|
+
test("Ceph: a pool id alone still makes a pool", () => {
|
|
2518
|
+
const context: string | null = Evaluator.buildCephRootCauseContext({
|
|
2519
|
+
dataToProcess: metricResponse({
|
|
2520
|
+
cephResourceBreakdown: {
|
|
2521
|
+
clusterName: "prod-cluster",
|
|
2522
|
+
metricName: "ceph_pool_stored",
|
|
2523
|
+
metricFriendlyName: "Pool Stored",
|
|
2524
|
+
affectedResources: [{ poolId: "7", metricValue: 3 }],
|
|
2525
|
+
attributes: {},
|
|
2526
|
+
},
|
|
2527
|
+
}),
|
|
2528
|
+
monitorStep: cephStep(),
|
|
2529
|
+
monitor: new Monitor(),
|
|
2530
|
+
});
|
|
2531
|
+
|
|
2532
|
+
expect(context).toContain("1. **Pool** `7` — **3 B**");
|
|
2533
|
+
});
|
|
2534
|
+
|
|
2535
|
+
test("Ceph: a daemon lists its pool and host; a pool lists its host; a host and a cluster stand alone", () => {
|
|
2536
|
+
const context: string | null = Evaluator.buildCephRootCauseContext({
|
|
2537
|
+
dataToProcess: metricResponse({
|
|
2538
|
+
cephResourceBreakdown: {
|
|
2539
|
+
clusterName: "prod-cluster",
|
|
2540
|
+
metricName: "ceph_osd_apply_latency_ms",
|
|
2541
|
+
metricFriendlyName: "OSD Apply Latency",
|
|
2542
|
+
affectedResources: [
|
|
2543
|
+
{
|
|
2544
|
+
daemon: "osd.7",
|
|
2545
|
+
poolId: "2",
|
|
2546
|
+
poolName: "rbd",
|
|
2547
|
+
hostname: "ceph-node-1",
|
|
2548
|
+
metricValue: 400,
|
|
2549
|
+
},
|
|
2550
|
+
{
|
|
2551
|
+
poolId: "3",
|
|
2552
|
+
poolName: "cephfs_data",
|
|
2553
|
+
hostname: "ceph-node-2",
|
|
2554
|
+
metricValue: 300,
|
|
2555
|
+
},
|
|
2556
|
+
{ hostname: "ceph-node-3", metricValue: 200 },
|
|
2557
|
+
{ metricValue: 100 },
|
|
2558
|
+
],
|
|
2559
|
+
attributes: {},
|
|
2560
|
+
},
|
|
2561
|
+
}),
|
|
2562
|
+
monitorStep: cephStep(),
|
|
2563
|
+
monitor: new Monitor(),
|
|
2564
|
+
});
|
|
2565
|
+
|
|
2566
|
+
expect(context).toContain(
|
|
2567
|
+
[
|
|
2568
|
+
"1. **Daemon** `osd.7` — **400 ms**",
|
|
2569
|
+
" - Pool: `rbd` (`2`)",
|
|
2570
|
+
" - Host: `ceph-node-1`",
|
|
2571
|
+
"2. **Pool** `cephfs_data` (`3`) — **300 ms**",
|
|
2572
|
+
" - Host: `ceph-node-2`",
|
|
2573
|
+
"3. **Host** `ceph-node-3` — **200 ms**",
|
|
2574
|
+
"4. **Cluster** `prod-cluster` — **100 ms**",
|
|
2575
|
+
].join("\n"),
|
|
2576
|
+
);
|
|
2577
|
+
});
|
|
2578
|
+
});
|