@oneuptime/common 14.0.1 → 14.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/AlertEpisodeMember.ts +8 -1
- package/Models/DatabaseModels/AlertFeed.ts +2 -0
- package/Models/DatabaseModels/AutoRemediationSuggestion.ts +63 -0
- package/Models/DatabaseModels/IncidentAlert.ts +540 -0
- package/Models/DatabaseModels/IncidentEpisodeMember.ts +8 -1
- package/Models/DatabaseModels/IncidentEpisodePublicNote.ts +9 -2
- package/Models/DatabaseModels/IncidentFeed.ts +2 -0
- package/Models/DatabaseModels/IncidentPublicNote.ts +9 -2
- package/Models/DatabaseModels/Index.ts +2 -0
- package/Models/DatabaseModels/KubernetesCluster.ts +429 -0
- package/Models/DatabaseModels/Monitor.ts +48 -0
- package/Models/DatabaseModels/Project.ts +60 -0
- package/Models/DatabaseModels/RumApplication.ts +52 -1
- package/Models/DatabaseModels/Runner.ts +5 -4
- package/Models/DatabaseModels/RunnerJob.ts +66 -0
- package/Models/DatabaseModels/ScheduledMaintenancePublicNote.ts +10 -2
- package/Models/DatabaseModels/WorkflowVariable.ts +479 -2
- package/Models/DatabaseModels/WorkspaceProjectAuthToken.ts +79 -25
- package/Server/API/AIInvestigationAPI.ts +341 -0
- package/Server/API/AutoRemediationAPI.ts +184 -2
- package/Server/API/CommonAPI.ts +70 -0
- package/Server/API/KubernetesClusterAiAccessAPI.ts +793 -0
- package/Server/API/MicrosoftTeamsAPI.ts +770 -315
- package/Server/API/MonitorAPI.ts +140 -0
- package/Server/API/OnCallScheduleTimelineAPI.ts +5 -21
- package/Server/API/SlackAPI.ts +447 -99
- package/Server/API/StatusPageAPI.ts +146 -26
- package/Server/API/TelemetryAPI.ts +216 -5
- package/Server/API/UserTwoFactorBackupCodeAPI.ts +1 -0
- package/Server/API/WorkflowVariableAPI.ts +156 -0
- package/Server/Infrastructure/GlobalCache.ts +35 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794300000000-AddOAuth2WorkflowVariables.ts +97 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794400000000-AddKubernetesClusterAiAccess.ts +129 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794500000000-AddIncomingEmailCustomAddressToMonitor.ts +25 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794600000000-AddAuthTokenExpiresAtToWorkspaceProjectAuthToken.ts +20 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794700000000-RemoveAppAccessTokenFromWorkspaceProjectAuthTokenMiscData.ts +38 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794800000000-AddSessionReplaySameOriginTracePropagation.ts +28 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1794900000000-AddIncidentAlert.ts +89 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +14 -0
- package/Server/Middleware/MasterAdminAuthorization.ts +28 -0
- package/Server/Middleware/TelemetryIngest.ts +47 -2
- package/Server/Middleware/UserAuthorization.ts +56 -2
- package/Server/Services/AIAgentService.ts +1 -0
- package/Server/Services/AccessTokenService.ts +121 -48
- package/Server/Services/AlertEpisodeMemberService.ts +9 -0
- package/Server/Services/AlertEpisodeService.ts +14 -1
- package/Server/Services/AlertService.ts +134 -22
- package/Server/Services/AutoRemediationRuleEngineService.ts +968 -39
- package/Server/Services/AutoRemediationRuleService.ts +118 -0
- package/Server/Services/BillingService.ts +1 -0
- package/Server/Services/InboundEmail/Providers/SendGridInboundProvider.ts +16 -22
- package/Server/Services/IncidentAlertService.ts +1811 -0
- package/Server/Services/IncidentEpisodeMemberService.ts +9 -0
- package/Server/Services/IncidentEpisodePublicNoteService.ts +66 -0
- package/Server/Services/IncidentEpisodeRoleMemberService.ts +27 -0
- package/Server/Services/IncidentEpisodeService.ts +14 -1
- package/Server/Services/IncidentGroupingEngineService.ts +11 -0
- package/Server/Services/IncidentMemberService.ts +21 -0
- package/Server/Services/IncidentPublicNoteService.ts +63 -0
- package/Server/Services/IncidentService.ts +330 -30
- package/Server/Services/IncidentStateTimelineService.ts +39 -0
- package/Server/Services/IncidentTemplateService.ts +83 -3
- package/Server/Services/Index.ts +2 -0
- package/Server/Services/KubernetesClusterAiAccessService.ts +1943 -0
- package/Server/Services/KubernetesClusterService.ts +1570 -1
- package/Server/Services/LogAggregationService.ts +14 -6
- package/Server/Services/MonitorService.ts +651 -52
- package/Server/Services/MonitorStatusTimelineService.ts +639 -2
- package/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.ts +2 -0
- package/Server/Services/OnCallDutyPolicyEscalationRuleTeamService.ts +2 -0
- package/Server/Services/OnCallDutyPolicyEscalationRuleUserService.ts +2 -0
- package/Server/Services/OnCallDutyPolicyScheduleService.ts +3 -0
- package/Server/Services/OnCallNotificationAlertingService.ts +36 -12
- package/Server/Services/OnCallSetupReminderService.ts +1 -0
- package/Server/Services/ProbeService.ts +7 -1
- package/Server/Services/ProjectService.ts +11 -1
- package/Server/Services/RunbookCredentialService.ts +26 -1
- package/Server/Services/RunbookSecretService.ts +36 -0
- package/Server/Services/RunnerJobService.ts +941 -23
- package/Server/Services/RunnerService.ts +439 -6
- package/Server/Services/ScheduledMaintenancePublicNoteService.ts +66 -0
- package/Server/Services/ScheduledMaintenanceService.ts +124 -19
- package/Server/Services/ScheduledMaintenanceTemplateService.ts +82 -3
- package/Server/Services/ServiceLevelObjectiveService.ts +14 -1
- package/Server/Services/StatusPagePrivateUserService.ts +1 -0
- package/Server/Services/StatusPageService.ts +301 -74
- package/Server/Services/StatusPageSubscriberService.ts +4 -0
- package/Server/Services/TeamMemberService.ts +309 -7
- package/Server/Services/TraceAggregationService.ts +10 -6
- package/Server/Services/UserEmailService.ts +1 -0
- package/Server/Services/UserNotificationMethodAdminService.ts +1 -0
- package/Server/Services/UserNotificationRuleAdminService.ts +1 -0
- package/Server/Services/UserNotificationRuleService.ts +4 -0
- package/Server/Services/UserService.ts +122 -1
- package/Server/Services/WorkflowVariableService.ts +593 -8
- package/Server/Services/WorkspaceNotificationRuleService.ts +588 -8
- package/Server/Services/WorkspaceProjectAuthTokenService.ts +170 -1
- package/Server/Types/Database/QueryHelper.ts +22 -0
- package/Server/Types/Markdown.ts +502 -27
- package/Server/Utils/AI/ClusterAccess/ClusterAccessContext.ts +182 -0
- package/Server/Utils/AI/ClusterAccess/KubectlInvestigationToolkit.ts +525 -0
- package/Server/Utils/AI/ClusterAccess/KubectlJobRunner.ts +854 -0
- package/Server/Utils/AI/CodeFix/CodeFixAgentAuth.ts +22 -2
- package/Server/Utils/AI/Remediation/RemediationCommandTools.ts +1916 -307
- package/Server/Utils/AI/Remediation/RemediationExecutionRunner.ts +1601 -74
- package/Server/Utils/AI/SRE/AIInvestigationEngine.ts +139 -7
- package/Server/Utils/AI/SRE/AlertInvestigationRunner.ts +58 -1
- package/Server/Utils/AI/SRE/IncidentInvestigationRunner.ts +54 -1
- package/Server/Utils/AI/Toolbox/AIActionTools.ts +14 -6
- package/Server/Utils/AI/Toolbox/ResourceTools.ts +4 -2
- package/Server/Utils/AutoRemediation/CommandPlanExecutor.ts +1341 -150
- package/Server/Utils/AutoRemediation/RemediationVerifier.ts +375 -4
- package/Server/Utils/Database/AffectedResourceRelations.ts +127 -0
- package/Server/Utils/Database/ProjectScopedReferenceValidator.ts +240 -1
- package/Server/Utils/EmailRollup/EmailRollupFlushRunner.ts +1 -0
- package/Server/Utils/Monitor/MonitorAlert.ts +83 -9
- package/Server/Utils/Monitor/MonitorCriteriaObservationBuilder.ts +5 -1
- package/Server/Utils/Monitor/MonitorIncident.ts +104 -9
- package/Server/Utils/Monitor/MonitorPayloadRedaction.ts +124 -7
- package/Server/Utils/Monitor/ProbeMonitorsNotification.ts +407 -0
- package/Server/Utils/OnCall/OnCallShiftReminderRunner.ts +1 -0
- package/Server/Utils/PushNotificationUtil.ts +45 -0
- package/Server/Utils/Rules/OwnerRuleAssignment.ts +40 -3
- package/Server/Utils/Runbook/RunbookExecutePermission.ts +56 -6
- package/Server/Utils/SessionReplay/SessionReplayGateCache.ts +24 -2
- package/Server/Utils/SessionReplay/SessionReplayReadService.ts +223 -33
- package/Server/Utils/SessionReplay/SessionReplayUserFlowReadService.ts +559 -0
- package/Server/Utils/StartServer.ts +4 -3
- package/Server/Utils/TeamMember/ProjectLeaveResourceCleanup.ts +528 -0
- package/Server/Utils/TwoFactorBackupCodeNotification.ts +1 -0
- package/Server/Utils/UserPermission/UserPermission.ts +16 -0
- package/Server/Utils/Workflow/OAuth2TokenClient.ts +704 -0
- package/Server/Utils/Workflow/WorkflowVariableOAuthToken.ts +318 -0
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.ts +47 -7
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/AlertEpisode.ts +57 -16
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.ts +193 -214
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/IncidentEpisode.ts +57 -13
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/Monitor.ts +54 -17
- package/Server/Utils/Workspace/MicrosoftTeams/Actions/ScheduledMaintenance.ts +179 -161
- package/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.ts +281 -167
- package/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeamsServiceUrl.ts +114 -0
- package/Server/Utils/Workspace/MicrosoftTeams/ReactionNoteSync.ts +895 -0
- package/Server/Utils/Workspace/Slack/Actions/ActionTypes.ts +7 -9
- package/Server/Utils/Workspace/Slack/Actions/Alert.ts +96 -227
- package/Server/Utils/Workspace/Slack/Actions/AlertEpisode.ts +93 -214
- package/Server/Utils/Workspace/Slack/Actions/Auth.ts +30 -3
- package/Server/Utils/Workspace/Slack/Actions/Authorization.ts +99 -0
- package/Server/Utils/Workspace/Slack/Actions/Incident.ts +117 -283
- package/Server/Utils/Workspace/Slack/Actions/IncidentEpisode.ts +101 -196
- package/Server/Utils/Workspace/Slack/Actions/ReactionNote.ts +329 -0
- package/Server/Utils/Workspace/Slack/Actions/ScheduledMaintenance.ts +111 -287
- package/Server/Utils/Workspace/Slack/Slack.ts +86 -17
- package/Server/Utils/Workspace/Slack/app-manifest.json +2 -0
- package/Server/Utils/Workspace/Workspace.ts +171 -54
- package/Server/Utils/Workspace/WorkspaceActionAuthorization.ts +252 -0
- package/Server/Utils/Workspace/WorkspaceBase.ts +39 -0
- package/Server/Utils/Workspace/WorkspaceOAuthState.ts +280 -0
- package/Server/Utils/Workspace/WorkspaceProjectReferenceValidator.ts +96 -0
- package/Server/Utils/Workspace/WorkspaceReactionNote.ts +620 -0
- package/Tests/App/Dashboard/AttributesJSONDashboardSurfaces.test.tsx +496 -0
- package/Tests/App/Dashboard/AutoRemediationRulesTableCommandRunners.test.tsx +152 -0
- package/Tests/App/Dashboard/BreadcrumbCoverage.test.tsx +32 -0
- package/Tests/App/Dashboard/BulkIncidentLinkActions.test.tsx +869 -0
- package/Tests/App/Dashboard/CephMetricTooltips.test.tsx +1777 -0
- package/Tests/App/Dashboard/CloudFleetSummary.test.ts +8 -0
- package/Tests/App/Dashboard/CloudMetricTooltips.test.tsx +887 -0
- package/Tests/App/Dashboard/ClusterAccessNotice.test.tsx +794 -0
- package/Tests/App/Dashboard/ContainerHostMetricTooltips.test.tsx +1309 -0
- package/Tests/App/Dashboard/CreateOAuthWorkflowVariableModal.test.tsx +840 -0
- package/Tests/App/Dashboard/CreateWorkflowModal.test.tsx +653 -0
- package/Tests/App/Dashboard/DatabaseMonitorStepForm.test.tsx +197 -0
- package/Tests/App/Dashboard/DatabaseMonitorView.test.tsx +376 -0
- package/Tests/App/Dashboard/DisabledWarningRobustness.test.tsx +262 -0
- package/Tests/App/Dashboard/DiscoveryScanLivePage.test.tsx +39 -0
- package/Tests/App/Dashboard/DockerSwarmClustersListTooltips.test.tsx +414 -0
- package/Tests/App/Dashboard/DockerSwarmMetricTooltips.test.ts +796 -0
- package/Tests/App/Dashboard/DockerSwarmOverviewTooltips.test.tsx +747 -0
- package/Tests/App/Dashboard/EventNotes.test.tsx +2080 -0
- package/Tests/App/Dashboard/EventNotesPages.test.tsx +486 -0
- package/Tests/App/Dashboard/EventNotesUtil.test.ts +933 -0
- package/Tests/App/Dashboard/ExceptionMonitorEnvironmentFilter.test.tsx +238 -0
- package/Tests/App/Dashboard/HostDetailViewTooltips.test.tsx +725 -0
- package/Tests/App/Dashboard/HostListColumnTooltips.test.tsx +472 -0
- package/Tests/App/Dashboard/HostMetricTooltips.test.ts +709 -0
- package/Tests/App/Dashboard/HostOverviewTooltips.test.tsx +698 -0
- package/Tests/App/Dashboard/HostTooltipHarness.ts +124 -0
- package/Tests/App/Dashboard/HostsListTooltips.test.tsx +395 -0
- package/Tests/App/Dashboard/IncidentAlertLinkDialog.test.tsx +691 -0
- package/Tests/App/Dashboard/IncidentAlertLinkHelpers.test.ts +458 -0
- package/Tests/App/Dashboard/IncidentAlertLinkPages.test.tsx +828 -0
- package/Tests/App/Dashboard/IncidentAlertLinkSideMenus.test.tsx +231 -0
- package/Tests/App/Dashboard/IncidentChangeStateNotifyDefault.test.tsx +665 -0
- package/Tests/App/Dashboard/IncidentCreateFromAlerts.test.tsx +689 -0
- package/Tests/App/Dashboard/IncidentEpisodeFeedPublicNoteNotifyDefault.test.tsx +494 -0
- package/Tests/App/Dashboard/IncidentEpisodeOverviewNotifyDefault.test.tsx +706 -0
- package/Tests/App/Dashboard/IncidentFeedPublicNoteNotifyDefault.test.tsx +472 -0
- package/Tests/App/Dashboard/IncidentOverviewNotifyDefault.test.tsx +858 -0
- package/Tests/App/Dashboard/IncomingEmailAddressSettings.test.tsx +567 -0
- package/Tests/App/Dashboard/InvestigationClusterToolFormat.test.ts +241 -0
- package/Tests/App/Dashboard/InvestigationEvidenceKubectl.test.tsx +322 -0
- package/Tests/App/Dashboard/InvestigationPanel.test.tsx +12 -0
- package/Tests/App/Dashboard/InvestigationPanelClusterAccess.test.tsx +476 -0
- package/Tests/App/Dashboard/InvestigationRunDetailsKubectl.test.tsx +383 -0
- package/Tests/App/Dashboard/IoTMetricTooltips.test.tsx +1123 -0
- package/Tests/App/Dashboard/KubectlActivitySummary.test.tsx +366 -0
- package/Tests/App/Dashboard/KubernetesClusterAiAccessSettings.test.ts +2886 -0
- package/Tests/App/Dashboard/KubernetesClusterAiPage.test.tsx +2688 -0
- package/Tests/App/Dashboard/KubernetesClusterAiPageServerParity.test.ts +915 -0
- package/Tests/App/Dashboard/KubernetesClusterMetricTooltips.test.ts +837 -0
- package/Tests/App/Dashboard/KubernetesClusterOverviewTooltips.test.tsx +699 -0
- package/Tests/App/Dashboard/KubernetesContainersTabTooltips.test.tsx +320 -0
- package/Tests/App/Dashboard/KubernetesCostTooltips.test.tsx +502 -0
- package/Tests/App/Dashboard/KubernetesInsightsTooltips.test.tsx +222 -0
- package/Tests/App/Dashboard/KubernetesResourceMetricTooltips.test.ts +561 -0
- package/Tests/App/Dashboard/KubernetesResourcePagesTooltips.test.tsx +1168 -0
- package/Tests/App/Dashboard/LayerDateTimeFieldTimezoneName.test.tsx +66 -0
- package/Tests/App/Dashboard/LinkIncidentAlertModal.test.tsx +154 -0
- package/Tests/App/Dashboard/LogMonitorPreview.test.tsx +174 -0
- package/Tests/App/Dashboard/MetricDescriptionRules.ts +170 -0
- package/Tests/App/Dashboard/MetricDescriptionsCatalog.test.ts +206 -0
- package/Tests/App/Dashboard/MetricDescriptionsConsistency.test.ts +275 -0
- package/Tests/App/Dashboard/MonitorFeedRefresh.test.tsx +279 -0
- package/Tests/App/Dashboard/MonitorMetricQueryConfig.test.ts +244 -0
- package/Tests/App/Dashboard/MonitorOpenWorkCard.test.tsx +381 -0
- package/Tests/App/Dashboard/MonitorOpenWorkHook.test.tsx +784 -0
- package/Tests/App/Dashboard/MonitorOverviewData.test.tsx +2199 -0
- package/Tests/App/Dashboard/MonitorOverviewHero.test.tsx +1169 -0
- package/Tests/App/Dashboard/MonitorOverviewInput.test.ts +1127 -0
- package/Tests/App/Dashboard/MonitorOverviewPage.test.tsx +2136 -0
- package/Tests/App/Dashboard/MonitorOverviewSelectReadable.test.ts +368 -0
- package/Tests/App/Dashboard/MonitorOverviewStatBar.test.tsx +526 -0
- package/Tests/App/Dashboard/MonitorOverviewSupportingCards.test.tsx +542 -0
- package/Tests/App/Dashboard/MonitorOwnersCombine.test.ts +299 -0
- package/Tests/App/Dashboard/MonitorProbesCard.test.tsx +602 -0
- package/Tests/App/Dashboard/MonitorResponseTimeCard.test.tsx +386 -0
- package/Tests/App/Dashboard/MonitorSetupAndConnectionCards.test.tsx +612 -0
- package/Tests/App/Dashboard/MonitorStatusChangesCard.test.tsx +257 -0
- package/Tests/App/Dashboard/MonitorSummaryTestMonitorButton.test.tsx +360 -1
- package/Tests/App/Dashboard/MonitorUptimeHistoryCard.test.tsx +835 -0
- package/Tests/App/Dashboard/MonitorUptimeSummaryHook.test.tsx +971 -0
- package/Tests/App/Dashboard/MonitorViewLayout.test.tsx +580 -0
- package/Tests/App/Dashboard/NetworkDeviceTooltips.test.tsx +852 -0
- package/Tests/App/Dashboard/NetworkDiagnosticsTooltips.test.tsx +556 -0
- package/Tests/App/Dashboard/NetworkMetricTooltips.test.ts +1348 -0
- package/Tests/App/Dashboard/NetworkSiteTooltips.test.tsx +679 -0
- package/Tests/App/Dashboard/OnCallLayerRestrictionTimesPreserved.test.tsx +441 -0
- package/Tests/App/Dashboard/OnCallNavbar.test.tsx +214 -0
- package/Tests/App/Dashboard/OnCallSchedulePreviewCalendarDisplay.test.tsx +1389 -0
- package/Tests/App/Dashboard/OnCallSchedulePreviewOverrides.test.tsx +205 -1
- package/Tests/App/Dashboard/OnCallScheduleTimeInputsSettingsZone.test.tsx +198 -0
- package/Tests/App/Dashboard/ProxmoxMetricTooltips.test.ts +592 -0
- package/Tests/App/Dashboard/ProxmoxOverviewTooltips.test.tsx +1002 -0
- package/Tests/App/Dashboard/ProxmoxResourcePageTooltips.test.tsx +923 -0
- package/Tests/App/Dashboard/PublicNotePagesNotifyDefault.test.tsx +524 -0
- package/Tests/App/Dashboard/RemediationSuggestionCardApproval.test.tsx +702 -0
- package/Tests/App/Dashboard/RemoveUserFromProject.test.tsx +427 -0
- package/Tests/App/Dashboard/ResourceArchiveSettings.test.tsx +4 -0
- package/Tests/App/Dashboard/ResourceOverviewTileTooltips.test.tsx +1066 -0
- package/Tests/App/Dashboard/RumOverviewPage.test.tsx +1195 -0
- package/Tests/App/Dashboard/RumSessionReplaySideMenu.test.tsx +3 -1
- package/Tests/App/Dashboard/RunbookStepTypeMaps.test.ts +166 -0
- package/Tests/App/Dashboard/RunnerKubernetesAgentForm.test.tsx +622 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceChangeStateNotifyDefault.test.tsx +1491 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceFeedPublicNoteNotifyDefault.test.tsx +489 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceOverviewNotifyDefault.test.tsx +909 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceOverviewPage.test.tsx +14 -0
- package/Tests/App/Dashboard/ScheduledMaintenanceWindowStatsTranslation.test.tsx +307 -0
- package/Tests/App/Dashboard/SecurityEventAttributeColumnPicker.test.tsx +542 -0
- package/Tests/App/Dashboard/SecurityEventConnectionFormModal.test.tsx +174 -0
- package/Tests/App/Dashboard/SecurityEventConnectionsEmptyState.test.tsx +973 -212
- package/Tests/App/Dashboard/SecurityEventConnectionsTable.test.tsx +491 -52
- package/Tests/App/Dashboard/SecurityEventDetailPanel.test.tsx +301 -0
- package/Tests/App/Dashboard/SecurityEventListRow.test.tsx +158 -1
- package/Tests/App/Dashboard/SecurityEventProviderTile.test.tsx +869 -0
- package/Tests/App/Dashboard/SecurityEventsEmptyState.test.tsx +959 -94
- package/Tests/App/Dashboard/SecurityEventsNoResults.test.tsx +2 -0
- package/Tests/App/Dashboard/SecurityEventsViewer.test.tsx +274 -1
- package/Tests/App/Dashboard/ServerlessMetricTooltips.test.tsx +474 -0
- package/Tests/App/Dashboard/ServiceMetricTooltips.test.tsx +772 -0
- package/Tests/App/Dashboard/SessionReplayAuditTable.test.tsx +56 -1
- package/Tests/App/Dashboard/TeamMembershipRemoval.test.ts +900 -0
- package/Tests/App/Dashboard/TeamMembershipRemovalPages.test.tsx +460 -0
- package/Tests/App/Dashboard/TelemetryMetricsSignals.test.ts +1207 -0
- package/Tests/App/Dashboard/TelemetryResourceTabTooltips.test.tsx +437 -0
- package/Tests/App/Dashboard/TimezoneElements.test.tsx +321 -0
- package/Tests/App/Dashboard/TimezoneSelectButton.test.tsx +634 -0
- package/Tests/App/Dashboard/TraceExplorer.test.tsx +260 -0
- package/Tests/App/Dashboard/TraceMonitorPreview.test.tsx +182 -0
- package/Tests/App/Dashboard/UserTimezoneInit.test.tsx +544 -0
- package/Tests/App/Dashboard/VMwareMetricTooltips.test.ts +801 -0
- package/Tests/App/Dashboard/VMwareOverviewTooltips.test.tsx +601 -0
- package/Tests/App/Dashboard/VMwareResourcePagesTooltips.test.tsx +646 -0
- package/Tests/App/Dashboard/VMwareTooltipHarness.ts +145 -0
- package/Tests/App/Dashboard/WorkflowVariableModals.test.tsx +1630 -0
- package/Tests/App/Dashboard/WorkflowVariableRoutes.test.tsx +603 -0
- package/Tests/App/Dashboard/WorkflowVariableTokenStatus.test.tsx +589 -0
- package/Tests/App/Dashboard/WorkflowVariableUtil.test.ts +1543 -0
- package/Tests/App/Dashboard/WorkflowVariableView.test.tsx +3227 -0
- package/Tests/App/Dashboard/WorkflowVariablesTable.test.tsx +1277 -395
- package/Tests/App/StatusPage/StatusPageOverviewMonitorGroupUptime.test.tsx +714 -0
- package/Tests/App/StatusPage/StatusPageOverviewUptimeBars.NewYork.test.tsx +1009 -0
- package/Tests/App/StatusPage/StatusPageOverviewUptimeBars.test.tsx +1092 -0
- package/Tests/Models/DatabaseModels/DomainRoleTierCoverage.test.ts +5 -0
- package/Tests/Models/DatabaseModels/IncidentAlertModel.test.ts +554 -0
- package/Tests/Models/DatabaseModels/KubernetesClusterAiAccessColumns.test.ts +520 -0
- package/Tests/Models/DatabaseModels/MonitorSecretKeyColumnAccessControl.test.ts +66 -2
- package/Tests/Models/DatabaseModels/SessionReplayModels.test.ts +117 -0
- package/Tests/Models/DatabaseModels/WorkspaceProjectAuthTokenAccessControl.test.ts +236 -0
- package/Tests/Models/IncidentEpisodePublicNoteNotifyColumnContract.test.ts +103 -0
- package/Tests/Models/IncidentPublicNoteNotifyColumnContract.test.ts +96 -0
- package/Tests/Models/ScheduledMaintenancePublicNoteNotifyColumnContract.test.ts +111 -0
- package/Tests/Server/API/AIInvestigationAPI.test.ts +976 -0
- package/Tests/Server/API/AutoRemediationApproveKubectlRecheck.test.ts +836 -0
- package/Tests/Server/API/CommonAPIAssertCanCreateTable.test.ts +565 -0
- package/Tests/Server/API/CommonAPIAssertCanReadTable.test.ts +282 -0
- package/Tests/Server/API/KubernetesClusterAiAccessAPI.test.ts +1744 -0
- package/Tests/Server/API/LogHistogramAPI.test.ts +371 -0
- package/Tests/Server/API/MicrosoftTeamsLegacyWebhookRemoval.test.ts +125 -30
- package/Tests/Server/API/MicrosoftTeamsOAuthState.test.ts +1121 -0
- package/Tests/Server/API/MonitorUptimeSummaryAPI.test.ts +685 -0
- package/Tests/Server/API/SessionReplayAPI.test.ts +602 -1
- package/Tests/Server/API/SlackChannelCacheAPI.test.ts +358 -0
- package/Tests/Server/API/SlackOAuthState.test.ts +688 -0
- package/Tests/Server/API/StatusPageOverviewMonitorGroupUptime.test.ts +527 -0
- package/Tests/Server/API/StatusPageTestEmailReportAuth.test.ts +2 -0
- package/Tests/Server/API/StatusPageUptimeAPI.test.ts +427 -0
- package/Tests/Server/API/UserProjectsAPI.test.ts +4 -0
- package/Tests/Server/API/WorkflowVariableAPI.test.ts +404 -0
- package/Tests/Server/API/WorkspaceOAuthTestHelpers.ts +289 -0
- package/Tests/Server/API/WorkspaceSendTestNotificationRoutes.test.ts +2064 -0
- package/Tests/Server/Infrastructure/GlobalCache.test.ts +86 -0
- package/Tests/Server/Infrastructure/Postgres/AddIncidentAlertMigration.test.ts +341 -0
- package/Tests/Server/Infrastructure/Postgres/AddKubernetesClusterAiAccessMigration.test.ts +589 -0
- package/Tests/Server/Infrastructure/Postgres/AddOAuth2WorkflowVariablesMigration.test.ts +254 -0
- package/Tests/Server/Infrastructure/Postgres/AddSessionReplaySameOriginTracePropagationMigration.test.ts +247 -0
- package/Tests/Server/Middleware/BlockedUserMiddleware.test.ts +403 -0
- package/Tests/Server/Middleware/MasterAdminAuthorization.test.ts +13 -0
- package/Tests/Server/Middleware/PublicRouteUserMiddleware.test.ts +2 -0
- package/Tests/Server/Middleware/RemovedProjectMemberAuthorization.test.ts +469 -0
- package/Tests/Server/Middleware/TelemetryIngestBrowserKey.test.ts +77 -1
- package/Tests/Server/Middleware/TelemetryIngestKubernetesAgentRunnerPinnedKey.test.ts +211 -0
- package/Tests/Server/Middleware/TelemetryIngestKubernetesAgentRunnerRateLimit.test.ts +358 -0
- package/Tests/Server/Middleware/UserAuthorization.test.ts +97 -0
- package/Tests/Server/Middleware/UserAuthorizationEditionEnforcement.test.ts +2 -0
- package/Tests/Server/Services/AccessTokenProjectUserGrant.test.ts +14 -2
- package/Tests/Server/Services/AccessTokenRemovedMemberAccess.test.ts +1044 -0
- package/Tests/Server/Services/AddOwnersIdempotency.test.ts +5 -0
- package/Tests/Server/Services/AlertEpisodeMemberCreatePermission.test.ts +139 -0
- package/Tests/Server/Services/AutoRemediationClusterRuleEngine.test.ts +1361 -0
- package/Tests/Server/Services/AutoRemediationRuleCommandRunners.test.ts +375 -0
- package/Tests/Server/Services/CrossProjectRelationWriteGuard.test.ts +2384 -0
- package/Tests/Server/Services/FindOwnersProjectMembership.test.ts +267 -0
- package/Tests/Server/Services/InboundEmail/SendGridInboundProviderAddress.test.ts +90 -0
- package/Tests/Server/Services/IncidentAlertPostgres.test.ts +903 -0
- package/Tests/Server/Services/IncidentAlertRegistration.test.ts +96 -0
- package/Tests/Server/Services/IncidentAlertService.test.ts +2225 -0
- package/Tests/Server/Services/IncidentAlertStateCascade.test.ts +970 -0
- package/Tests/Server/Services/IncidentCreateFromAlerts.test.ts +1284 -0
- package/Tests/Server/Services/IncidentEpisodeMemberCreatePermission.test.ts +143 -0
- package/Tests/Server/Services/IncidentEpisodePublicNoteNotifyDefault.test.ts +805 -0
- package/Tests/Server/Services/IncidentPublicNoteNotifyDefault.test.ts +724 -0
- package/Tests/Server/Services/IncidentRoleProjectMembership.test.ts +233 -0
- package/Tests/Server/Services/KubernetesClusterAiAccessAgentRunnerAssignments.test.ts +372 -0
- package/Tests/Server/Services/KubernetesClusterAiAccessBindingGuard.test.ts +1876 -0
- package/Tests/Server/Services/KubernetesClusterAiAccessDeletedRunnerRemedy.test.ts +620 -0
- package/Tests/Server/Services/KubernetesClusterAiAccessGapNextSteps.test.ts +421 -0
- package/Tests/Server/Services/KubernetesClusterAiAccessService.test.ts +3662 -0
- package/Tests/Server/Services/KubernetesClusterAiAccessSubjectClusters.test.ts +405 -0
- package/Tests/Server/Services/KubernetesClusterAiSettingsPermission.test.ts +975 -0
- package/Tests/Server/Services/KubernetesClusterServiceAiAccessFeed.test.ts +497 -0
- package/Tests/Server/Services/KubernetesClusterServiceDeleteSettlesRemediation.test.ts +328 -0
- package/Tests/Server/Services/LogAggregationService.test.ts +99 -0
- package/Tests/Server/Services/MonitorServiceIncomingEmailCustomAddress.test.ts +447 -0
- package/Tests/Server/Services/MonitorServiceProbeFlagSync.test.ts +1334 -0
- package/Tests/Server/Services/MonitorServiceProbeStatusGrouping.test.ts +1640 -0
- package/Tests/Server/Services/MonitorServiceRefreshCurrentStatus.test.ts +225 -0
- package/Tests/Server/Services/MonitorStatusBridgeProbePath.test.ts +8 -8
- package/Tests/Server/Services/MonitorStatusTimelineDailyAggregates.test.ts +264 -1
- package/Tests/Server/Services/MonitorStatusTimelineMergedDowntime.test.ts +284 -0
- package/Tests/Server/Services/MonitorUptimeSummaryService.test.ts +1043 -0
- package/Tests/Server/Services/NetworkDeviceRuleEngines.test.ts +5 -0
- package/Tests/Server/Services/OnCallUndeliverablePageOwnerMembership.test.ts +359 -0
- package/Tests/Server/Services/ProbeServiceDashboardLink.test.ts +191 -0
- package/Tests/Server/Services/ProjectOwnersProjectMembership.test.ts +221 -0
- package/Tests/Server/Services/RuleRun/IncidentAlertMaintenanceRuleEnginesRun.test.ts +5 -0
- package/Tests/Server/Services/RuleRun/OwnerRuleEnginesGroupARun.test.ts +28 -0
- package/Tests/Server/Services/RuleRun/OwnerRuleEnginesGroupBRun.test.ts +9 -1
- package/Tests/Server/Services/RunnerJobEnqueueAiCommandAgentRunner.test.ts +230 -0
- package/Tests/Server/Services/RunnerJobEnqueueKubectl.test.ts +978 -0
- package/Tests/Server/Services/RunnerJobKubectlWriteScope.test.ts +866 -0
- package/Tests/Server/Services/RunnerJobSubmitResultRedaction.test.ts +322 -0
- package/Tests/Server/Services/RunnerJobSubmitResultUnstorableText.test.ts +357 -0
- package/Tests/Server/Services/RunnerJobTimeoutReason.test.ts +227 -0
- package/Tests/Server/Services/RunnerServiceAgentName.test.ts +581 -0
- package/Tests/Server/Services/ScheduledMaintenancePublicNoteNotifyDefault.test.ts +802 -0
- package/Tests/Server/Services/ScheduledMaintenanceSubscriberTemplateFormats.test.ts +53 -0
- package/Tests/Server/Services/ServiceLevelObjectiveLabelAndOwnerRuleEngines.test.ts +5 -0
- package/Tests/Server/Services/ServiceLevelObjectiveService.test.ts +12 -0
- package/Tests/Server/Services/StatusPageSubscriberReport.test.ts +1176 -1
- package/Tests/Server/Services/TeamMemberBillingSeatSync.test.ts +9 -0
- package/Tests/Server/Services/TeamMemberOnCallCleanup.test.ts +18 -0
- package/Tests/Server/Services/TeamMemberProjectMembership.test.ts +255 -0
- package/Tests/Server/Services/TeamMemberRemovalRevokesProjectAccess.test.ts +600 -0
- package/Tests/Server/Services/TeamMemberResourceCleanup.test.ts +731 -0
- package/Tests/Server/Services/TeamMemberWorkspaceAccountCleanup.test.ts +164 -0
- package/Tests/Server/Services/TraceAggregationService.test.ts +64 -0
- package/Tests/Server/Services/UserServiceBlockedUser.test.ts +293 -0
- package/Tests/Server/Services/WorkflowVariableOAuthHooks.test.ts +1016 -0
- package/Tests/Server/Services/WorkflowVariableOAuthPostgres.test.ts +543 -0
- package/Tests/Server/Services/WorkflowVariableRenameUniqueness.test.ts +16 -5
- package/Tests/Server/Services/WorkspaceNotificationRuleNoteDelivery.test.ts +473 -0
- package/Tests/Server/Services/WorkspaceNotificationRuleSendTestNotification.test.ts +3690 -0
- package/Tests/Server/Services/WorkspaceProjectAuthTokenMiscDataSecrets.test.ts +374 -0
- package/Tests/Server/Services/WorkspaceProjectAuthTokenSlackChannelCache.test.ts +143 -0
- package/Tests/Server/Types/Database/Permissions/WorkflowVariableOAuthColumnPermission.test.ts +250 -0
- package/Tests/Server/Types/Database/QueryHelperJsonContains.test.ts +96 -0
- package/Tests/Server/Types/Markdown.test.ts +1033 -0
- package/Tests/Server/Utils/AI/ClusterAccessContext.test.ts +285 -0
- package/Tests/Server/Utils/AI/CodeFix/CodeFixAgentAuth.test.ts +159 -1
- package/Tests/Server/Utils/AI/InvestigationClusterAccessWiring.test.ts +752 -0
- package/Tests/Server/Utils/AI/KubectlInvestigationToolkit.test.ts +1840 -0
- package/Tests/Server/Utils/AI/KubectlJobRunner.test.ts +1067 -0
- package/Tests/Server/Utils/AI/KubectlJobRunnerFinishedJob.test.ts +355 -0
- package/Tests/Server/Utils/AI/PostIncidentStatusUpdateTool.test.ts +496 -0
- package/Tests/Server/Utils/AI/RemediationCommandTools.test.ts +305 -0
- package/Tests/Server/Utils/AI/RemediationCommandToolsKubectl.test.ts +1365 -0
- package/Tests/Server/Utils/AI/RemediationCommandToolsKubectlOutcome.test.ts +639 -0
- package/Tests/Server/Utils/AI/RemediationCommandToolsLiveCluster.test.ts +1585 -0
- package/Tests/Server/Utils/AI/RemediationCommandToolsRunnerScope.test.ts +1065 -0
- package/Tests/Server/Utils/AI/RemediationExecutionRunner.test.ts +2138 -1
- package/Tests/Server/Utils/AI/RemediationExecutionRunnerClusterMode.test.ts +1035 -0
- package/Tests/Server/Utils/AI/RemediationKubectlCopyParity.test.ts +517 -0
- package/Tests/Server/Utils/AI/ResourceTools.test.ts +27 -1
- package/Tests/Server/Utils/AutoRemediation/CommandPlanExecutor.test.ts +1371 -24
- package/Tests/Server/Utils/AutoRemediation/CommandPlanExecutorRollbackRace.test.ts +962 -0
- package/Tests/Server/Utils/AutoRemediation/CommandPlanExecutorUnknownRun.test.ts +518 -0
- package/Tests/Server/Utils/AutoRemediation/CommandPlanVerification.test.ts +939 -2
- package/Tests/Server/Utils/Database/ProjectScopedRelationReferences.test.ts +371 -0
- package/Tests/Server/Utils/Monitor/AffectedResourceList.test.ts +64 -0
- package/Tests/Server/Utils/Monitor/DatabaseMetricObservationRemediation.test.ts +102 -0
- package/Tests/Server/Utils/Monitor/MonitorAlertCriteriaReferences.test.ts +362 -0
- package/Tests/Server/Utils/Monitor/MonitorCriteriaEvaluatorBreachingSamples.test.ts +55 -0
- package/Tests/Server/Utils/Monitor/MonitorEmailAddressRedaction.test.ts +255 -0
- package/Tests/Server/Utils/Monitor/MonitorIncidentCriteriaReferences.test.ts +347 -0
- package/Tests/Server/Utils/Monitor/MonitorIncidentMemberRoleMembership.test.ts +203 -0
- package/Tests/Server/Utils/Monitor/ProbeMonitorsNotification.test.ts +1147 -0
- package/Tests/Server/Utils/PushNotificationUtilCreators.test.ts +79 -0
- package/Tests/Server/Utils/ResourceFeed/ResourceFeedUtil.test.ts +21 -1
- package/Tests/Server/Utils/Rules/OwnerRuleAssignment.test.ts +155 -1
- package/Tests/Server/Utils/SessionReplay/SessionReplayGateCachePolicy.test.ts +103 -0
- package/Tests/Server/Utils/SessionReplay/SessionReplayReadServiceQueries.test.ts +288 -0
- package/Tests/Server/Utils/SessionReplay/SessionReplayUserFlowReadService.test.ts +304 -0
- package/Tests/Server/Utils/Telemetry/TelemetryIngestionKeyGuard.test.ts +40 -3
- package/Tests/Server/Utils/Workflow/OAuth2TokenClient.test.ts +702 -0
- package/Tests/Server/Utils/Workflow/OAuth2TokenClientHttpIntegration.test.ts +507 -0
- package/Tests/Server/Utils/Workflow/WorkflowVariableOAuthToken.test.ts +831 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsActionAuthorization.test.ts +63 -1
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsAppAccessToken.test.ts +256 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsChannelSend.test.ts +60 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsChatActionAuthorization.test.ts +442 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsChats.test.ts +44 -2
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsDirectMessage.test.ts +55 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsReactionNoteSync.test.ts +1522 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsServiceUrl.test.ts +130 -0
- package/Tests/Server/Utils/Workspace/MicrosoftTeamsTeamInstalls.test.ts +4 -4
- package/Tests/Server/Utils/Workspace/Slack/SlackActionAuthorization.test.ts +691 -0
- package/Tests/Server/Utils/Workspace/Slack/SlackMessageByTimestamp.test.ts +253 -0
- package/Tests/Server/Utils/Workspace/Slack/SlackReactionNote.test.ts +574 -0
- package/Tests/Server/Utils/Workspace/WorkspaceActionAuthorization.test.ts +488 -0
- package/Tests/Server/Utils/Workspace/WorkspaceCreateProjectReferences.test.ts +861 -0
- package/Tests/Server/Utils/Workspace/WorkspaceMessageAuthor.test.ts +635 -0
- package/Tests/Server/Utils/Workspace/WorkspaceOAuthState.test.ts +548 -0
- package/Tests/Server/Utils/Workspace/WorkspaceReactionNote.test.ts +839 -0
- package/Tests/Server/Utils/Workspace/WorkspaceSendErrorMessage.test.ts +725 -0
- package/Tests/Types/AutoRemediation/AiRemediationCommandPlan.test.ts +283 -9
- package/Tests/Types/DateBrowserLocalWallClock.test.ts +280 -0
- package/Tests/Types/DateLocalShortMonthNameFallback.test.ts +389 -0
- package/Tests/Types/DateUserTimezone.test.ts +177 -4
- package/Tests/Types/Kubernetes/KubernetesAgentChartAiAccess.test.ts +306 -0
- package/Tests/Types/Kubernetes/KubernetesClusterAiAccess.test.ts +283 -0
- package/Tests/Types/Kubernetes/KubernetesRunnerPostureWriteScope.test.ts +68 -0
- package/Tests/Types/Monitor/DatabaseMonitor/SqlServerPlatform.test.ts +144 -0
- package/Tests/Types/Monitor/MonitorStepExceptionMonitor.test.ts +160 -0
- package/Tests/Types/Monitor/Recommendation/MonitorRecommendationCoverage.test.ts +53 -0
- package/Tests/Types/Rum/WebVitals.test.ts +173 -0
- package/Tests/Types/Runbook/RunbookStepType.test.ts +91 -16
- package/Tests/Types/Runner/RunnerLiveStatus.test.ts +14 -3
- package/Tests/Types/SecurityEvent/Connectors/SecurityEventConnectorCatalog.test.ts +15 -0
- package/Tests/Types/StatusPage/PublicNoteSubscriberNotificationDefault.test.ts +1868 -0
- package/Tests/Types/Timezone.test.ts +5 -0
- package/Tests/Types/TimezoneAlias.test.ts +1034 -0
- package/Tests/Types/Workflow/JiraTemplateFixtures.ts +833 -0
- package/Tests/Types/Workflow/JiraTemplateScripts.test.ts +5414 -0
- package/Tests/Types/Workflow/JiraTemplateSimulation.test.ts +4803 -0
- package/Tests/Types/Workflow/JiraTemplates.test.ts +5429 -0
- package/Tests/Types/Workflow/Templates.test.ts +79 -4
- package/Tests/Types/Workflow/WorkflowVariableOAuth.test.ts +520 -0
- package/Tests/Types/Workspace/WorkspaceNoteReaction.test.ts +177 -0
- package/Tests/UI/Components/AttributesJSON/AttributesJSONView.test.tsx +390 -0
- package/Tests/UI/Components/AttributesJSON/CopyAttributesAsJSONButton.test.tsx +566 -0
- package/Tests/UI/Components/AttributesJSON/JSONHighlight.test.ts +131 -0
- package/Tests/UI/Components/BasicFormDropdownNormalization.test.tsx +276 -1
- package/Tests/UI/Components/Calendar/Calendar.test.tsx +1611 -0
- package/Tests/UI/Components/Calendar/CalendarDayLayout.test.ts +1486 -0
- package/Tests/UI/Components/Calendar/CalendarDayLayoutDst.test.ts +632 -0
- package/Tests/UI/Components/Calendar/CalendarDst.test.tsx +388 -0
- package/Tests/UI/Components/Charts/ChartGroupPlotFloor.test.tsx +170 -0
- package/Tests/UI/Components/Charts/HistogramSelection.test.ts +282 -0
- package/Tests/UI/Components/CustomTimeRangeModal.test.tsx +6 -0
- package/Tests/UI/Components/CustomTimeRangeModalTimezoneTranslation.test.tsx +285 -0
- package/Tests/UI/Components/Dropdown.test.tsx +999 -2
- package/Tests/UI/Components/FieldLabelInfoCardTooltip.test.tsx +413 -0
- package/Tests/UI/Components/Forms/FormFieldCheckboxTranslation.test.tsx +216 -0
- package/Tests/UI/Components/Forms/FormFieldTimezoneDescriptionTranslation.test.tsx +461 -0
- package/Tests/UI/Components/Forms/IncidentEpisodePublicNoteNotifyPayload.test.tsx +641 -0
- package/Tests/UI/Components/Forms/IncidentPublicNoteNotifyPayload.test.tsx +346 -0
- package/Tests/UI/Components/Forms/ScheduledMaintenancePublicNoteNotifyPayload.test.tsx +493 -0
- package/Tests/UI/Components/Graphs/DayUptimeGraph.test.tsx +911 -2
- package/Tests/UI/Components/Graphs/DayUptimeGraphTimezone.NewYork.test.tsx +730 -0
- package/Tests/UI/Components/Graphs/DayUptimeGraphTimezone.Tokyo.test.tsx +506 -0
- package/Tests/UI/Components/HistogramClickToZoom.test.tsx +619 -0
- package/Tests/UI/Components/InfoTooltip.test.tsx +242 -0
- package/Tests/UI/Components/LogDetailsPanelAttributesJSON.test.tsx +248 -0
- package/Tests/UI/Components/LogsViewerHistogramZoom.test.tsx +47 -1
- package/Tests/UI/Components/LogsViewerSmallScreens.test.tsx +620 -0
- package/Tests/UI/Components/ModelPageModelChange.test.tsx +242 -3
- package/Tests/UI/Components/ModelTable/BaseModelTableBulkDelete.test.tsx +207 -3
- package/Tests/UI/Components/ModelTable/BaseModelTableBulkDeleteVerb.test.tsx +367 -0
- package/Tests/UI/Components/ModelTable/BaseModelTableCreateButtonTitle.test.tsx +275 -0
- package/Tests/UI/Components/ModelTable/BaseModelTableDeleteConfirmation.test.tsx +521 -0
- package/Tests/UI/Components/MonitorGraphs/Uptime.test.tsx +406 -2
- package/Tests/UI/Components/MonitorGraphs/UptimeBarDayModal.test.tsx +169 -2
- package/Tests/UI/Components/TableHeaderTooltip.test.tsx +662 -0
- package/Tests/UI/Components/TelemetryViewer/TelemetryViewerHistogramClickZoom.test.tsx +266 -0
- package/Tests/UI/Components/TelemetryViewer/TelemetryViewerSmallScreens.test.tsx +439 -0
- package/Tests/UI/Components/TimezoneDropdownLegacyValues.test.tsx +704 -0
- package/Tests/UI/Components/UseHistogramZoom.test.tsx +89 -1
- package/Tests/UI/Components/UseLogsHistogram.test.tsx +175 -0
- package/Tests/UI/Monitor/MonitorSummarySnapshotRender.test.tsx +5 -5
- package/Tests/UI/Rum/PrivacySummaryCard.test.tsx +166 -9
- package/Tests/UI/Rum/RecordingHealthTooltips.test.tsx +447 -0
- package/Tests/UI/Rum/ReplayCorrelationPanel.test.tsx +164 -0
- package/Tests/UI/Rum/ReplayFrameCapture.test.ts +6750 -0
- package/Tests/UI/Rum/ReplayRail.test.tsx +723 -12
- package/Tests/UI/Rum/ReplayRailDetail.test.tsx +86 -0
- package/Tests/UI/Rum/ReplayScreenshot.test.ts +2124 -0
- package/Tests/UI/Rum/ReplayScreenshotActions.test.tsx +3288 -0
- package/Tests/UI/Rum/ReplayStage.test.tsx +62 -0
- package/Tests/UI/Rum/ReplayStageOverlays.test.tsx +983 -2
- package/Tests/UI/Rum/SessionReplaySettingsPolicyLoading.test.tsx +149 -0
- package/Tests/UI/Rum/SessionReplaySetupGuide.test.tsx +107 -6
- package/Tests/UI/Rum/SessionReplayUsersTableTooltips.test.tsx +330 -0
- package/Tests/UI/Rum/UserFlowTooltips.test.tsx +886 -0
- package/Tests/UI/Telemetry/TelemetryDetailPanel.test.tsx +28 -0
- package/Tests/UI/Utils/Breadcrumb/fixtures/RealBreadcrumbTrails.ts +10 -0
- package/Tests/UI/Utils/Breadcrumb/fixtures/RealRoutePatterns.ts +2 -0
- package/Tests/UI/Utils/PermissionGate.test.ts +76 -0
- package/Tests/UI/Utils/ProjectUsersModelAPI.test.ts +51 -0
- package/Tests/UI/Utils/Timezone.test.ts +747 -33
- package/Tests/UI/Workspace/SendTestNotificationButton.test.tsx +1854 -0
- package/Tests/UI/Workspace/WorkspaceCardsSendTestNotification.test.tsx +1864 -0
- package/Tests/Utils/AI/InvestigationReport.test.ts +435 -7
- package/Tests/Utils/AiRemediation/KubectlAllowlistValidatorParity.test.ts +1084 -0
- package/Tests/Utils/AiRemediation/KubectlOutputRedactor.test.ts +2531 -0
- package/Tests/Utils/AiRemediation/KubectlPolicy.test.ts +1895 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyAllowlist.test.ts +428 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyChartRbacParity.test.ts +828 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyCustomResourceScope.test.ts +573 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyDenials.test.ts +1525 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyExposeOverrides.test.ts +275 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyNodePatch.test.ts +428 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyObjectWords.test.ts +195 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyPatchParents.test.ts +637 -0
- package/Tests/Utils/AiRemediation/KubectlPolicyPodSecurityBaseline.test.ts +322 -0
- package/Tests/Utils/AiRemediation/KubectlPolicySafeWriteScope.test.ts +1026 -0
- package/Tests/Utils/AiRemediation/KubectlWaitBudget.test.ts +267 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScope.test.ts +390 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeCallerParity.test.ts +777 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeClusterScoped.test.ts +613 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeEntryPoint.test.ts +576 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeKubectlSpellings.test.ts +752 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeObjectWords.test.ts +358 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeOverrides.test.ts +293 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopeParityCases.ts +598 -0
- package/Tests/Utils/AiRemediation/KubectlWriteScopePolicyParity.test.ts +403 -0
- package/Tests/Utils/Dashboard/DashboardClockComponent.test.ts +154 -5
- package/Tests/Utils/Incident/IncidentFromAlerts.test.ts +696 -0
- package/Tests/Utils/Monitor/IncomingEmailMonitorAddress.test.ts +478 -0
- package/Tests/Utils/Monitor/MonitorCheckScheduleUtil.test.ts +646 -0
- package/Tests/Utils/Monitor/MonitorOverviewCriteriaUtil.test.ts +292 -0
- package/Tests/Utils/Monitor/MonitorOverviewFamily.test.ts +274 -0
- package/Tests/Utils/Monitor/MonitorOverviewPresentationUtil.test.ts +3100 -0
- package/Tests/Utils/Monitor/MonitorOverviewProbeUtil.test.ts +1412 -0
- package/Tests/Utils/Monitor/MonitorOverviewTargetUtil.test.ts +374 -0
- package/Tests/Utils/Monitor/MonitorStatusHistoryUtil.test.ts +332 -0
- package/Tests/Utils/Monitor/MonitorUptimeSummaryUtil.test.ts +946 -0
- package/Tests/Utils/Rum/SessionIdentity.test.ts +94 -0
- package/Tests/Utils/Rum/SessionReplayWafCompatibility.test.ts +840 -0
- package/Tests/Utils/Rum/SessionReplayWireEncoding.test.ts +208 -0
- package/Tests/Utils/Rum/SessionTraceState.test.ts +239 -0
- package/Tests/Utils/Rum/UserFlow.test.ts +1010 -0
- package/Tests/Utils/StatusPage/MonitorGroupMergedDowntimeUtil.test.ts +217 -0
- package/Tests/Utils/StatusPage/ResourceUptime.test.ts +1272 -1
- package/Tests/Utils/StatusPage/UptimeDailyAggregateTotals.test.ts +298 -0
- package/Tests/Utils/StatusPage/UptimeDailyAggregateUtil.test.ts +649 -1
- package/Tests/Utils/StatusPage/UptimeDailyAggregateWindow.test.ts +314 -0
- package/Tests/Utils/Telemetry/AttributesJSON.test.ts +454 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtil.test.ts +418 -2
- package/Tests/Utils/Uptime/DayUptimeGraphUtilMidnightDst.test.ts +160 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtilTimezone.Auckland.test.ts +45 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtilTimezone.Kolkata.test.ts +40 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtilTimezone.London.test.ts +147 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtilTimezone.NewYork.test.ts +48 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtilTimezone.Tokyo.test.ts +39 -0
- package/Tests/Utils/Uptime/DayUptimeGraphUtilTimezoneCases.ts +1015 -0
- package/Tests/Utils/Uptime/UptimeUtil.test.ts +75 -0
- package/Tests/Utils/Uptime/UptimeUtilChronologicalOrder.test.ts +1279 -0
- package/Types/AutoRemediation/AiRemediationCommandPlan.ts +203 -5
- package/Types/Date.ts +121 -7
- package/Types/Email/EmailMessage.ts +8 -0
- package/Types/Email/EmailTemplateType.ts +1 -0
- package/Types/Exception/ExceptionMessages.ts +1 -0
- package/Types/Incident/IncidentAlertLink.ts +38 -0
- package/Types/Kubernetes/KubernetesClusterAiAccess.ts +488 -0
- package/Types/Kubernetes/KubernetesClusterAiAccessPermissions.ts +54 -0
- package/Types/Kubernetes/KubernetesClusterAiAccessToolNames.ts +17 -0
- package/Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse.ts +9 -0
- package/Types/Monitor/DatabaseMonitor/SqlServerPlatform.ts +117 -0
- package/Types/Monitor/KubernetesAlertTemplates.ts +6 -5
- package/Types/Monitor/MonitorStepExceptionMonitor.ts +56 -0
- package/Types/Monitor/MonitorUptimeSummary.ts +113 -0
- package/Types/Monitor/Recommendation/MonitorRecommendationUtil.ts +11 -0
- package/Types/Permission.ts +48 -0
- package/Types/Rum/SessionReplay.ts +80 -8
- package/Types/Rum/SessionReplayApi.ts +44 -1
- package/Types/Rum/UserFlow.ts +187 -0
- package/Types/Rum/WebVitals.ts +52 -0
- package/Types/Runbook/RunbookStepType.ts +10 -0
- package/Types/Runbook/RunnerJobOrigin.ts +15 -0
- package/Types/SecurityEvent/Connectors/SecurityEventConnectorCatalog.ts +6 -2
- package/Types/StatusPage/MergedDowntimeTotals.ts +17 -0
- package/Types/StatusPage/PublicNoteSubscriberNotificationDefault.ts +148 -0
- package/Types/StatusPage/UptimeDailyAggregate.ts +15 -0
- package/Types/Telemetry/TelemetryIngestSurface.ts +9 -0
- package/Types/Timezone.ts +5 -0
- package/Types/TimezoneAlias.ts +385 -0
- package/Types/Workflow/Templates.ts +3041 -2
- package/Types/Workflow/WorkflowVariableOAuth.ts +476 -0
- package/Types/Workspace/WorkspaceNoteReaction.ts +132 -0
- package/Typings/Index.d.ts +24 -0
- package/UI/Components/AttributesJSON/AttributesJSONPreferences.ts +175 -0
- package/UI/Components/AttributesJSON/AttributesJSONView.tsx +138 -0
- package/UI/Components/AttributesJSON/AttributesViewToggle.tsx +79 -0
- package/UI/Components/AttributesJSON/BracesGlyph.tsx +30 -0
- package/UI/Components/AttributesJSON/CopyAttributesAsJSONButton.tsx +491 -0
- package/UI/Components/AttributesJSON/JSONHighlight.ts +77 -0
- package/UI/Components/Calendar/Calendar.css +83 -0
- package/UI/Components/Calendar/Calendar.tsx +136 -7
- package/UI/Components/Calendar/CalendarDayLayout.ts +236 -0
- package/UI/Components/Charts/ChartGroup/ChartGroup.tsx +23 -1
- package/UI/Components/Charts/Utils/HistogramSelection.ts +117 -0
- package/UI/Components/Charts/Utils/useHistogramRangeSelection.ts +269 -0
- package/UI/Components/Charts/Utils/useHistogramZoom.ts +26 -2
- package/UI/Components/Date/CustomTimeRangeModal.tsx +26 -3
- package/UI/Components/Detail/FieldLabel.tsx +8 -0
- package/UI/Components/Dropdown/Dropdown.tsx +58 -9
- package/UI/Components/Forms/Fields/FormField.tsx +49 -14
- package/UI/Components/Graphs/DayUptimeGraph.tsx +166 -25
- package/UI/Components/Graphs/UptimeBarTooltip.tsx +10 -2
- package/UI/Components/Graphs/UptimeDaySummary.tsx +8 -0
- package/UI/Components/InfoCard/InfoCard.tsx +47 -1
- package/UI/Components/LogsViewer/LogsViewer.tsx +52 -5
- package/UI/Components/LogsViewer/components/ColumnSelector.tsx +13 -2
- package/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.tsx +13 -1
- package/UI/Components/LogsViewer/components/LogDetailsPanel.tsx +91 -76
- package/UI/Components/LogsViewer/components/LogsFacetSidebar.tsx +15 -1
- package/UI/Components/LogsViewer/components/LogsHistogram.tsx +28 -106
- package/UI/Components/LogsViewer/components/LogsViewerToolbar.tsx +54 -6
- package/UI/Components/LogsViewer/components/SavedViewsDropdown.tsx +8 -2
- package/UI/Components/LogsViewer/useLogsHistogram.ts +39 -6
- package/UI/Components/ModelTable/BaseModelTable.tsx +161 -31
- package/UI/Components/ModelTable/Column.ts +5 -0
- package/UI/Components/MonitorGraphs/Uptime.tsx +12 -0
- package/UI/Components/MonitorGraphs/UptimeBarDayModal.tsx +10 -2
- package/UI/Components/Page/ModelPage.tsx +59 -3
- package/UI/Components/Table/TableHeader.tsx +47 -2
- package/UI/Components/Table/Types/Column.ts +7 -0
- package/UI/Components/TelemetryViewer/TelemetryViewer.tsx +69 -5
- package/UI/Components/TelemetryViewer/components/TelemetryDetailPanel.tsx +12 -1
- package/UI/Components/TelemetryViewer/components/TelemetryFacetSidebar.tsx +15 -1
- package/UI/Components/TelemetryViewer/components/TelemetryHistogram.tsx +36 -104
- package/UI/Components/TelemetryViewer/components/TelemetrySearchBar.tsx +9 -1
- package/UI/Components/Tooltip/InfoTooltip.tsx +78 -0
- package/UI/Utils/ModelAPI/ProjectUsersModelAPI.ts +19 -1
- package/UI/Utils/PermissionGate.ts +11 -2
- package/UI/Utils/Timezone.ts +69 -28
- package/Utils/AI/InvestigationReport.ts +195 -25
- package/Utils/AiRemediation/KubectlOutputRedactor.ts +1724 -0
- package/Utils/AiRemediation/KubectlPolicy.ts +4136 -0
- package/Utils/AiRemediation/KubectlWaitBudget.ts +156 -0
- package/Utils/AiRemediation/KubectlWriteScope.ts +1964 -0
- package/Utils/Dashboard/Components/DashboardClockComponent.ts +7 -6
- package/Utils/Incident/IncidentFromAlerts.ts +516 -0
- package/Utils/Monitor/IncomingEmailMonitorAddress.ts +292 -0
- package/Utils/Monitor/MonitorCheckScheduleUtil.ts +654 -0
- package/Utils/Monitor/MonitorOverviewCriteriaUtil.ts +188 -0
- package/Utils/Monitor/MonitorOverviewFamily.ts +263 -0
- package/Utils/Monitor/MonitorOverviewPresentationUtil.ts +1958 -0
- package/Utils/Monitor/MonitorOverviewProbeUtil.ts +979 -0
- package/Utils/Monitor/MonitorOverviewTargetUtil.ts +299 -0
- package/Utils/Monitor/MonitorStatusHistoryUtil.ts +201 -0
- package/Utils/Monitor/MonitorUptimeSummaryUtil.ts +638 -0
- package/Utils/Rum/SessionIdentity.ts +44 -0
- package/Utils/Rum/SessionReplayWireEncoding.ts +137 -0
- package/Utils/Rum/SessionTraceState.ts +212 -0
- package/Utils/Rum/UserFlow.ts +1527 -0
- package/Utils/StatusPage/MonitorGroupMergedDowntimeUtil.ts +107 -0
- package/Utils/StatusPage/ResourceUptime.ts +99 -0
- package/Utils/StatusPage/UptimeDailyAggregateUtil.ts +197 -0
- package/Utils/Telemetry/AttributesJSON.ts +367 -0
- package/Utils/Uptime/DayUptimeGraphUtil.ts +165 -0
- package/Utils/Uptime/UptimeUtil.ts +122 -38
- package/build/dist/Models/DatabaseModels/AlertEpisodeMember.js +8 -1
- package/build/dist/Models/DatabaseModels/AlertEpisodeMember.js.map +1 -1
- package/build/dist/Models/DatabaseModels/AlertFeed.js +2 -0
- package/build/dist/Models/DatabaseModels/AlertFeed.js.map +1 -1
- package/build/dist/Models/DatabaseModels/AutoRemediationSuggestion.js +62 -0
- package/build/dist/Models/DatabaseModels/AutoRemediationSuggestion.js.map +1 -1
- package/build/dist/Models/DatabaseModels/IncidentAlert.js +553 -0
- package/build/dist/Models/DatabaseModels/IncidentAlert.js.map +1 -0
- package/build/dist/Models/DatabaseModels/IncidentEpisodeMember.js +8 -1
- package/build/dist/Models/DatabaseModels/IncidentEpisodeMember.js.map +1 -1
- package/build/dist/Models/DatabaseModels/IncidentEpisodePublicNote.js +10 -3
- package/build/dist/Models/DatabaseModels/IncidentEpisodePublicNote.js.map +1 -1
- package/build/dist/Models/DatabaseModels/IncidentFeed.js +2 -0
- package/build/dist/Models/DatabaseModels/IncidentFeed.js.map +1 -1
- package/build/dist/Models/DatabaseModels/IncidentPublicNote.js +10 -3
- package/build/dist/Models/DatabaseModels/IncidentPublicNote.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +2 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/KubernetesCluster.js +434 -0
- package/build/dist/Models/DatabaseModels/KubernetesCluster.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Monitor.js +49 -0
- package/build/dist/Models/DatabaseModels/Monitor.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Project.js +62 -0
- package/build/dist/Models/DatabaseModels/Project.js.map +1 -1
- package/build/dist/Models/DatabaseModels/RumApplication.js +53 -1
- package/build/dist/Models/DatabaseModels/RumApplication.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Runner.js +4 -4
- package/build/dist/Models/DatabaseModels/Runner.js.map +1 -1
- package/build/dist/Models/DatabaseModels/RunnerJob.js +66 -0
- package/build/dist/Models/DatabaseModels/RunnerJob.js.map +1 -1
- package/build/dist/Models/DatabaseModels/ScheduledMaintenancePublicNote.js +11 -3
- package/build/dist/Models/DatabaseModels/ScheduledMaintenancePublicNote.js.map +1 -1
- package/build/dist/Models/DatabaseModels/WorkflowVariable.js +487 -3
- package/build/dist/Models/DatabaseModels/WorkflowVariable.js.map +1 -1
- package/build/dist/Models/DatabaseModels/WorkspaceProjectAuthToken.js +64 -16
- package/build/dist/Models/DatabaseModels/WorkspaceProjectAuthToken.js.map +1 -1
- package/build/dist/Server/API/AIInvestigationAPI.js +205 -0
- package/build/dist/Server/API/AIInvestigationAPI.js.map +1 -1
- package/build/dist/Server/API/AutoRemediationAPI.js +115 -2
- package/build/dist/Server/API/AutoRemediationAPI.js.map +1 -1
- package/build/dist/Server/API/CommonAPI.js +49 -0
- package/build/dist/Server/API/CommonAPI.js.map +1 -1
- package/build/dist/Server/API/KubernetesClusterAiAccessAPI.js +564 -0
- package/build/dist/Server/API/KubernetesClusterAiAccessAPI.js.map +1 -0
- package/build/dist/Server/API/MicrosoftTeamsAPI.js +465 -176
- package/build/dist/Server/API/MicrosoftTeamsAPI.js.map +1 -1
- package/build/dist/Server/API/MonitorAPI.js +107 -1
- package/build/dist/Server/API/MonitorAPI.js.map +1 -1
- package/build/dist/Server/API/OnCallScheduleTimelineAPI.js +5 -16
- package/build/dist/Server/API/OnCallScheduleTimelineAPI.js.map +1 -1
- package/build/dist/Server/API/SlackAPI.js +322 -78
- package/build/dist/Server/API/SlackAPI.js.map +1 -1
- package/build/dist/Server/API/StatusPageAPI.js +123 -17
- package/build/dist/Server/API/StatusPageAPI.js.map +1 -1
- package/build/dist/Server/API/TelemetryAPI.js +144 -5
- package/build/dist/Server/API/TelemetryAPI.js.map +1 -1
- package/build/dist/Server/API/UserTwoFactorBackupCodeAPI.js +1 -0
- package/build/dist/Server/API/UserTwoFactorBackupCodeAPI.js.map +1 -1
- package/build/dist/Server/API/WorkflowVariableAPI.js +109 -0
- package/build/dist/Server/API/WorkflowVariableAPI.js.map +1 -0
- package/build/dist/Server/Infrastructure/GlobalCache.js +29 -0
- package/build/dist/Server/Infrastructure/GlobalCache.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794300000000-AddOAuth2WorkflowVariables.js +38 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794300000000-AddOAuth2WorkflowVariables.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794400000000-AddKubernetesClusterAiAccess.js +54 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794400000000-AddKubernetesClusterAiAccess.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794500000000-AddIncomingEmailCustomAddressToMonitor.js +14 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794500000000-AddIncomingEmailCustomAddressToMonitor.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794600000000-AddAuthTokenExpiresAtToWorkspaceProjectAuthToken.js +12 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794600000000-AddAuthTokenExpiresAtToWorkspaceProjectAuthToken.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794700000000-RemoveAppAccessTokenFromWorkspaceProjectAuthTokenMiscData.js +32 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794700000000-RemoveAppAccessTokenFromWorkspaceProjectAuthTokenMiscData.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794800000000-AddSessionReplaySameOriginTracePropagation.js +20 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794800000000-AddSessionReplaySameOriginTracePropagation.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794900000000-AddIncidentAlert.js +42 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1794900000000-AddIncidentAlert.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +14 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Middleware/MasterAdminAuthorization.js +21 -0
- package/build/dist/Server/Middleware/MasterAdminAuthorization.js.map +1 -1
- package/build/dist/Server/Middleware/TelemetryIngest.js +32 -2
- package/build/dist/Server/Middleware/TelemetryIngest.js.map +1 -1
- package/build/dist/Server/Middleware/UserAuthorization.js +39 -6
- package/build/dist/Server/Middleware/UserAuthorization.js.map +1 -1
- package/build/dist/Server/Services/AIAgentService.js +1 -0
- package/build/dist/Server/Services/AIAgentService.js.map +1 -1
- package/build/dist/Server/Services/AccessTokenService.js +87 -42
- package/build/dist/Server/Services/AccessTokenService.js.map +1 -1
- package/build/dist/Server/Services/AlertEpisodeMemberService.js +4 -0
- package/build/dist/Server/Services/AlertEpisodeMemberService.js.map +1 -1
- package/build/dist/Server/Services/AlertEpisodeService.js +12 -1
- package/build/dist/Server/Services/AlertEpisodeService.js.map +1 -1
- package/build/dist/Server/Services/AlertService.js +110 -23
- package/build/dist/Server/Services/AlertService.js.map +1 -1
- package/build/dist/Server/Services/AutoRemediationRuleEngineService.js +716 -30
- package/build/dist/Server/Services/AutoRemediationRuleEngineService.js.map +1 -1
- package/build/dist/Server/Services/AutoRemediationRuleService.js +100 -0
- package/build/dist/Server/Services/AutoRemediationRuleService.js.map +1 -1
- package/build/dist/Server/Services/BillingService.js +1 -0
- package/build/dist/Server/Services/BillingService.js.map +1 -1
- package/build/dist/Server/Services/InboundEmail/Providers/SendGridInboundProvider.js +11 -16
- package/build/dist/Server/Services/InboundEmail/Providers/SendGridInboundProvider.js.map +1 -1
- package/build/dist/Server/Services/IncidentAlertService.js +1370 -0
- package/build/dist/Server/Services/IncidentAlertService.js.map +1 -0
- package/build/dist/Server/Services/IncidentEpisodeMemberService.js +4 -0
- package/build/dist/Server/Services/IncidentEpisodeMemberService.js.map +1 -1
- package/build/dist/Server/Services/IncidentEpisodePublicNoteService.js +48 -0
- package/build/dist/Server/Services/IncidentEpisodePublicNoteService.js.map +1 -1
- package/build/dist/Server/Services/IncidentEpisodeRoleMemberService.js +21 -0
- package/build/dist/Server/Services/IncidentEpisodeRoleMemberService.js.map +1 -1
- package/build/dist/Server/Services/IncidentEpisodeService.js +12 -1
- package/build/dist/Server/Services/IncidentEpisodeService.js.map +1 -1
- package/build/dist/Server/Services/IncidentGroupingEngineService.js +8 -0
- package/build/dist/Server/Services/IncidentGroupingEngineService.js.map +1 -1
- package/build/dist/Server/Services/IncidentMemberService.js +14 -0
- package/build/dist/Server/Services/IncidentMemberService.js.map +1 -1
- package/build/dist/Server/Services/IncidentPublicNoteService.js +48 -0
- package/build/dist/Server/Services/IncidentPublicNoteService.js.map +1 -1
- package/build/dist/Server/Services/IncidentService.js +263 -33
- package/build/dist/Server/Services/IncidentService.js.map +1 -1
- package/build/dist/Server/Services/IncidentStateTimelineService.js +33 -0
- package/build/dist/Server/Services/IncidentStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/IncidentTemplateService.js +68 -4
- package/build/dist/Server/Services/IncidentTemplateService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +2 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/KubernetesClusterAiAccessService.js +1470 -0
- package/build/dist/Server/Services/KubernetesClusterAiAccessService.js.map +1 -0
- package/build/dist/Server/Services/KubernetesClusterService.js +1097 -0
- package/build/dist/Server/Services/KubernetesClusterService.js.map +1 -1
- package/build/dist/Server/Services/LogAggregationService.js +14 -6
- package/build/dist/Server/Services/LogAggregationService.js.map +1 -1
- package/build/dist/Server/Services/MonitorService.js +487 -48
- package/build/dist/Server/Services/MonitorService.js.map +1 -1
- package/build/dist/Server/Services/MonitorStatusTimelineService.js +540 -3
- package/build/dist/Server/Services/MonitorStatusTimelineService.js.map +1 -1
- package/build/dist/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.js +2 -0
- package/build/dist/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.js.map +1 -1
- package/build/dist/Server/Services/OnCallDutyPolicyEscalationRuleTeamService.js +2 -0
- package/build/dist/Server/Services/OnCallDutyPolicyEscalationRuleTeamService.js.map +1 -1
- package/build/dist/Server/Services/OnCallDutyPolicyEscalationRuleUserService.js +2 -0
- package/build/dist/Server/Services/OnCallDutyPolicyEscalationRuleUserService.js.map +1 -1
- package/build/dist/Server/Services/OnCallDutyPolicyScheduleService.js +3 -0
- package/build/dist/Server/Services/OnCallDutyPolicyScheduleService.js.map +1 -1
- package/build/dist/Server/Services/OnCallNotificationAlertingService.js +29 -7
- package/build/dist/Server/Services/OnCallNotificationAlertingService.js.map +1 -1
- package/build/dist/Server/Services/OnCallSetupReminderService.js +1 -0
- package/build/dist/Server/Services/OnCallSetupReminderService.js.map +1 -1
- package/build/dist/Server/Services/ProbeService.js +7 -1
- package/build/dist/Server/Services/ProbeService.js.map +1 -1
- package/build/dist/Server/Services/ProjectService.js +11 -1
- package/build/dist/Server/Services/ProjectService.js.map +1 -1
- package/build/dist/Server/Services/RunbookCredentialService.js +24 -0
- package/build/dist/Server/Services/RunbookCredentialService.js.map +1 -1
- package/build/dist/Server/Services/RunbookSecretService.js +44 -0
- package/build/dist/Server/Services/RunbookSecretService.js.map +1 -1
- package/build/dist/Server/Services/RunnerJobService.js +641 -21
- package/build/dist/Server/Services/RunnerJobService.js.map +1 -1
- package/build/dist/Server/Services/RunnerService.js +344 -5
- package/build/dist/Server/Services/RunnerService.js.map +1 -1
- package/build/dist/Server/Services/ScheduledMaintenancePublicNoteService.js +48 -0
- package/build/dist/Server/Services/ScheduledMaintenancePublicNoteService.js.map +1 -1
- package/build/dist/Server/Services/ScheduledMaintenanceService.js +102 -20
- package/build/dist/Server/Services/ScheduledMaintenanceService.js.map +1 -1
- package/build/dist/Server/Services/ScheduledMaintenanceTemplateService.js +67 -4
- package/build/dist/Server/Services/ScheduledMaintenanceTemplateService.js.map +1 -1
- package/build/dist/Server/Services/ServiceLevelObjectiveService.js +12 -1
- package/build/dist/Server/Services/ServiceLevelObjectiveService.js.map +1 -1
- package/build/dist/Server/Services/StatusPagePrivateUserService.js +1 -0
- package/build/dist/Server/Services/StatusPagePrivateUserService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageService.js +230 -32
- package/build/dist/Server/Services/StatusPageService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageSubscriberService.js +4 -0
- package/build/dist/Server/Services/StatusPageSubscriberService.js.map +1 -1
- package/build/dist/Server/Services/TeamMemberService.js +270 -11
- package/build/dist/Server/Services/TeamMemberService.js.map +1 -1
- package/build/dist/Server/Services/TraceAggregationService.js +10 -6
- package/build/dist/Server/Services/TraceAggregationService.js.map +1 -1
- package/build/dist/Server/Services/UserEmailService.js +1 -0
- package/build/dist/Server/Services/UserEmailService.js.map +1 -1
- package/build/dist/Server/Services/UserNotificationMethodAdminService.js +1 -0
- package/build/dist/Server/Services/UserNotificationMethodAdminService.js.map +1 -1
- package/build/dist/Server/Services/UserNotificationRuleAdminService.js +1 -0
- package/build/dist/Server/Services/UserNotificationRuleAdminService.js.map +1 -1
- package/build/dist/Server/Services/UserNotificationRuleService.js +4 -0
- package/build/dist/Server/Services/UserNotificationRuleService.js.map +1 -1
- package/build/dist/Server/Services/UserService.js +107 -1
- package/build/dist/Server/Services/UserService.js.map +1 -1
- package/build/dist/Server/Services/WorkflowVariableService.js +396 -7
- package/build/dist/Server/Services/WorkflowVariableService.js.map +1 -1
- package/build/dist/Server/Services/WorkspaceNotificationRuleService.js +463 -7
- package/build/dist/Server/Services/WorkspaceNotificationRuleService.js.map +1 -1
- package/build/dist/Server/Services/WorkspaceProjectAuthTokenService.js +144 -2
- package/build/dist/Server/Services/WorkspaceProjectAuthTokenService.js.map +1 -1
- package/build/dist/Server/Types/Database/QueryHelper.js +20 -0
- package/build/dist/Server/Types/Database/QueryHelper.js.map +1 -1
- package/build/dist/Server/Types/Markdown.js +394 -27
- package/build/dist/Server/Types/Markdown.js.map +1 -1
- package/build/dist/Server/Utils/AI/ClusterAccess/ClusterAccessContext.js +123 -0
- package/build/dist/Server/Utils/AI/ClusterAccess/ClusterAccessContext.js.map +1 -0
- package/build/dist/Server/Utils/AI/ClusterAccess/KubectlInvestigationToolkit.js +344 -0
- package/build/dist/Server/Utils/AI/ClusterAccess/KubectlInvestigationToolkit.js.map +1 -0
- package/build/dist/Server/Utils/AI/ClusterAccess/KubectlJobRunner.js +587 -0
- package/build/dist/Server/Utils/AI/ClusterAccess/KubectlJobRunner.js.map +1 -0
- package/build/dist/Server/Utils/AI/CodeFix/CodeFixAgentAuth.js +18 -2
- package/build/dist/Server/Utils/AI/CodeFix/CodeFixAgentAuth.js.map +1 -1
- package/build/dist/Server/Utils/AI/Remediation/RemediationCommandTools.js +1380 -239
- package/build/dist/Server/Utils/AI/Remediation/RemediationCommandTools.js.map +1 -1
- package/build/dist/Server/Utils/AI/Remediation/RemediationExecutionRunner.js +1099 -68
- package/build/dist/Server/Utils/AI/Remediation/RemediationExecutionRunner.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/AIInvestigationEngine.js +88 -6
- package/build/dist/Server/Utils/AI/SRE/AIInvestigationEngine.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/AlertInvestigationRunner.js +45 -11
- package/build/dist/Server/Utils/AI/SRE/AlertInvestigationRunner.js.map +1 -1
- package/build/dist/Server/Utils/AI/SRE/IncidentInvestigationRunner.js +41 -11
- package/build/dist/Server/Utils/AI/SRE/IncidentInvestigationRunner.js.map +1 -1
- package/build/dist/Server/Utils/AI/Toolbox/AIActionTools.js +11 -4
- package/build/dist/Server/Utils/AI/Toolbox/AIActionTools.js.map +1 -1
- package/build/dist/Server/Utils/AI/Toolbox/ResourceTools.js +4 -2
- package/build/dist/Server/Utils/AI/Toolbox/ResourceTools.js.map +1 -1
- package/build/dist/Server/Utils/AutoRemediation/CommandPlanExecutor.js +997 -141
- package/build/dist/Server/Utils/AutoRemediation/CommandPlanExecutor.js.map +1 -1
- package/build/dist/Server/Utils/AutoRemediation/RemediationVerifier.js +299 -4
- package/build/dist/Server/Utils/AutoRemediation/RemediationVerifier.js.map +1 -1
- package/build/dist/Server/Utils/Database/AffectedResourceRelations.js +120 -0
- package/build/dist/Server/Utils/Database/AffectedResourceRelations.js.map +1 -0
- package/build/dist/Server/Utils/Database/ProjectScopedReferenceValidator.js +160 -1
- package/build/dist/Server/Utils/Database/ProjectScopedReferenceValidator.js.map +1 -1
- package/build/dist/Server/Utils/EmailRollup/EmailRollupFlushRunner.js +1 -0
- package/build/dist/Server/Utils/EmailRollup/EmailRollupFlushRunner.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js +59 -12
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaObservationBuilder.js +5 -1
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaObservationBuilder.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js +84 -22
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorPayloadRedaction.js +70 -5
- package/build/dist/Server/Utils/Monitor/MonitorPayloadRedaction.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/ProbeMonitorsNotification.js +249 -0
- package/build/dist/Server/Utils/Monitor/ProbeMonitorsNotification.js.map +1 -0
- package/build/dist/Server/Utils/OnCall/OnCallShiftReminderRunner.js +1 -0
- package/build/dist/Server/Utils/OnCall/OnCallShiftReminderRunner.js.map +1 -1
- package/build/dist/Server/Utils/PushNotificationUtil.js +27 -0
- package/build/dist/Server/Utils/PushNotificationUtil.js.map +1 -1
- package/build/dist/Server/Utils/Rules/OwnerRuleAssignment.js +26 -3
- package/build/dist/Server/Utils/Rules/OwnerRuleAssignment.js.map +1 -1
- package/build/dist/Server/Utils/Runbook/RunbookExecutePermission.js +38 -3
- package/build/dist/Server/Utils/Runbook/RunbookExecutePermission.js.map +1 -1
- package/build/dist/Server/Utils/SessionReplay/SessionReplayGateCache.js +8 -0
- package/build/dist/Server/Utils/SessionReplay/SessionReplayGateCache.js.map +1 -1
- package/build/dist/Server/Utils/SessionReplay/SessionReplayReadService.js +143 -23
- package/build/dist/Server/Utils/SessionReplay/SessionReplayReadService.js.map +1 -1
- package/build/dist/Server/Utils/SessionReplay/SessionReplayUserFlowReadService.js +406 -0
- package/build/dist/Server/Utils/SessionReplay/SessionReplayUserFlowReadService.js.map +1 -0
- package/build/dist/Server/Utils/StartServer.js +4 -3
- package/build/dist/Server/Utils/StartServer.js.map +1 -1
- package/build/dist/Server/Utils/TeamMember/ProjectLeaveResourceCleanup.js +382 -0
- package/build/dist/Server/Utils/TeamMember/ProjectLeaveResourceCleanup.js.map +1 -0
- package/build/dist/Server/Utils/TwoFactorBackupCodeNotification.js +1 -0
- package/build/dist/Server/Utils/TwoFactorBackupCodeNotification.js.map +1 -1
- package/build/dist/Server/Utils/UserPermission/UserPermission.js +10 -0
- package/build/dist/Server/Utils/UserPermission/UserPermission.js.map +1 -1
- package/build/dist/Server/Utils/Workflow/OAuth2TokenClient.js +447 -0
- package/build/dist/Server/Utils/Workflow/OAuth2TokenClient.js.map +1 -0
- package/build/dist/Server/Utils/Workflow/WorkflowVariableOAuthToken.js +202 -0
- package/build/dist/Server/Utils/Workflow/WorkflowVariableOAuthToken.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.js +39 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Alert.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/AlertEpisode.js +44 -4
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/AlertEpisode.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.js +150 -191
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Incident.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/IncidentEpisode.js +44 -4
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/IncidentEpisode.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Monitor.js +39 -17
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/Monitor.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/ScheduledMaintenance.js +151 -145
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/Actions/ScheduledMaintenance.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.js +221 -127
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeams.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeamsServiceUrl.js +91 -0
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/MicrosoftTeamsServiceUrl.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/ReactionNoteSync.js +673 -0
- package/build/dist/Server/Utils/Workspace/MicrosoftTeams/ReactionNoteSync.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/Slack/Actions/ActionTypes.js +3 -9
- package/build/dist/Server/Utils/Workspace/Slack/Actions/ActionTypes.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Alert.js +65 -166
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Alert.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Actions/AlertEpisode.js +63 -157
- package/build/dist/Server/Utils/Workspace/Slack/Actions/AlertEpisode.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Auth.js +23 -3
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Auth.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Authorization.js +92 -0
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Authorization.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Incident.js +81 -210
- package/build/dist/Server/Utils/Workspace/Slack/Actions/Incident.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Actions/IncidentEpisode.js +68 -138
- package/build/dist/Server/Utils/Workspace/Slack/Actions/IncidentEpisode.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Actions/ReactionNote.js +240 -0
- package/build/dist/Server/Utils/Workspace/Slack/Actions/ReactionNote.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/Slack/Actions/ScheduledMaintenance.js +85 -212
- package/build/dist/Server/Utils/Workspace/Slack/Actions/ScheduledMaintenance.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/Slack.js +60 -17
- package/build/dist/Server/Utils/Workspace/Slack/Slack.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/Slack/app-manifest.json +2 -0
- package/build/dist/Server/Utils/Workspace/Workspace.js +134 -41
- package/build/dist/Server/Utils/Workspace/Workspace.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/WorkspaceActionAuthorization.js +197 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceActionAuthorization.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceBase.js +32 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceBase.js.map +1 -1
- package/build/dist/Server/Utils/Workspace/WorkspaceOAuthState.js +199 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceOAuthState.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceProjectReferenceValidator.js +91 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceProjectReferenceValidator.js.map +1 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceReactionNote.js +450 -0
- package/build/dist/Server/Utils/Workspace/WorkspaceReactionNote.js.map +1 -0
- package/build/dist/Types/AutoRemediation/AiRemediationCommandPlan.js +103 -5
- package/build/dist/Types/AutoRemediation/AiRemediationCommandPlan.js.map +1 -1
- package/build/dist/Types/Date.js +97 -7
- package/build/dist/Types/Date.js.map +1 -1
- package/build/dist/Types/Email/EmailTemplateType.js +1 -0
- package/build/dist/Types/Email/EmailTemplateType.js.map +1 -1
- package/build/dist/Types/Exception/ExceptionMessages.js +1 -0
- package/build/dist/Types/Exception/ExceptionMessages.js.map +1 -1
- package/build/dist/Types/Incident/IncidentAlertLink.js +34 -0
- package/build/dist/Types/Incident/IncidentAlertLink.js.map +1 -0
- package/build/dist/Types/Kubernetes/KubernetesClusterAiAccess.js +245 -0
- package/build/dist/Types/Kubernetes/KubernetesClusterAiAccess.js.map +1 -0
- package/build/dist/Types/Kubernetes/KubernetesClusterAiAccessPermissions.js +49 -0
- package/build/dist/Types/Kubernetes/KubernetesClusterAiAccessPermissions.js.map +1 -0
- package/build/dist/Types/Kubernetes/KubernetesClusterAiAccessToolNames.js +16 -0
- package/build/dist/Types/Kubernetes/KubernetesClusterAiAccessToolNames.js.map +1 -0
- package/build/dist/Types/Monitor/DatabaseMonitor/SqlServerPlatform.js +104 -0
- package/build/dist/Types/Monitor/DatabaseMonitor/SqlServerPlatform.js.map +1 -0
- package/build/dist/Types/Monitor/KubernetesAlertTemplates.js +6 -5
- package/build/dist/Types/Monitor/KubernetesAlertTemplates.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorStepExceptionMonitor.js +32 -0
- package/build/dist/Types/Monitor/MonitorStepExceptionMonitor.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorUptimeSummary.js +52 -0
- package/build/dist/Types/Monitor/MonitorUptimeSummary.js.map +1 -0
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationUtil.js +9 -1
- package/build/dist/Types/Monitor/Recommendation/MonitorRecommendationUtil.js.map +1 -1
- package/build/dist/Types/Permission.js +42 -0
- package/build/dist/Types/Permission.js.map +1 -1
- package/build/dist/Types/Rum/SessionReplay.js +52 -2
- package/build/dist/Types/Rum/SessionReplay.js.map +1 -1
- package/build/dist/Types/Rum/SessionReplayApi.js +11 -0
- package/build/dist/Types/Rum/SessionReplayApi.js.map +1 -1
- package/build/dist/Types/Rum/UserFlow.js +115 -0
- package/build/dist/Types/Rum/UserFlow.js.map +1 -0
- package/build/dist/Types/Rum/WebVitals.js +25 -0
- package/build/dist/Types/Rum/WebVitals.js.map +1 -1
- package/build/dist/Types/Runbook/RunbookStepType.js +10 -0
- package/build/dist/Types/Runbook/RunbookStepType.js.map +1 -1
- package/build/dist/Types/Runbook/RunnerJobOrigin.js +14 -0
- package/build/dist/Types/Runbook/RunnerJobOrigin.js.map +1 -1
- package/build/dist/Types/SecurityEvent/Connectors/SecurityEventConnectorCatalog.js +2 -2
- package/build/dist/Types/SecurityEvent/Connectors/SecurityEventConnectorCatalog.js.map +1 -1
- package/build/dist/Types/StatusPage/MergedDowntimeTotals.js +2 -0
- package/build/dist/Types/StatusPage/MergedDowntimeTotals.js.map +1 -0
- package/build/dist/Types/StatusPage/PublicNoteSubscriberNotificationDefault.js +65 -0
- package/build/dist/Types/StatusPage/PublicNoteSubscriberNotificationDefault.js.map +1 -0
- package/build/dist/Types/Telemetry/TelemetryIngestSurface.js +8 -0
- package/build/dist/Types/Telemetry/TelemetryIngestSurface.js.map +1 -1
- package/build/dist/Types/Timezone.js +5 -0
- package/build/dist/Types/Timezone.js.map +1 -1
- package/build/dist/Types/TimezoneAlias.js +340 -0
- package/build/dist/Types/TimezoneAlias.js.map +1 -0
- package/build/dist/Types/Workflow/Templates.js +2694 -2
- package/build/dist/Types/Workflow/Templates.js.map +1 -1
- package/build/dist/Types/Workflow/WorkflowVariableOAuth.js +314 -0
- package/build/dist/Types/Workflow/WorkflowVariableOAuth.js.map +1 -0
- package/build/dist/Types/Workspace/WorkspaceNoteReaction.js +98 -0
- package/build/dist/Types/Workspace/WorkspaceNoteReaction.js.map +1 -0
- package/build/dist/UI/Components/AttributesJSON/AttributesJSONPreferences.js +95 -0
- package/build/dist/UI/Components/AttributesJSON/AttributesJSONPreferences.js.map +1 -0
- package/build/dist/UI/Components/AttributesJSON/AttributesJSONView.js +44 -0
- package/build/dist/UI/Components/AttributesJSON/AttributesJSONView.js.map +1 -0
- package/build/dist/UI/Components/AttributesJSON/AttributesViewToggle.js +27 -0
- package/build/dist/UI/Components/AttributesJSON/AttributesViewToggle.js.map +1 -0
- package/build/dist/UI/Components/AttributesJSON/BracesGlyph.js +9 -0
- package/build/dist/UI/Components/AttributesJSON/BracesGlyph.js.map +1 -0
- package/build/dist/UI/Components/AttributesJSON/CopyAttributesAsJSONButton.js +260 -0
- package/build/dist/UI/Components/AttributesJSON/CopyAttributesAsJSONButton.js.map +1 -0
- package/build/dist/UI/Components/AttributesJSON/JSONHighlight.js +56 -0
- package/build/dist/UI/Components/AttributesJSON/JSONHighlight.js.map +1 -0
- package/build/dist/UI/Components/Calendar/Calendar.js +94 -7
- package/build/dist/UI/Components/Calendar/Calendar.js.map +1 -1
- package/build/dist/UI/Components/Calendar/CalendarDayLayout.js +136 -0
- package/build/dist/UI/Components/Calendar/CalendarDayLayout.js.map +1 -0
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js +1 -1
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js.map +1 -1
- package/build/dist/UI/Components/Charts/Utils/HistogramSelection.js +72 -0
- package/build/dist/UI/Components/Charts/Utils/HistogramSelection.js.map +1 -0
- package/build/dist/UI/Components/Charts/Utils/useHistogramRangeSelection.js +178 -0
- package/build/dist/UI/Components/Charts/Utils/useHistogramRangeSelection.js.map +1 -0
- package/build/dist/UI/Components/Charts/Utils/useHistogramZoom.js +13 -1
- package/build/dist/UI/Components/Charts/Utils/useHistogramZoom.js.map +1 -1
- package/build/dist/UI/Components/Date/CustomTimeRangeModal.js +20 -4
- package/build/dist/UI/Components/Date/CustomTimeRangeModal.js.map +1 -1
- package/build/dist/UI/Components/Detail/FieldLabel.js +2 -0
- package/build/dist/UI/Components/Detail/FieldLabel.js.map +1 -1
- package/build/dist/UI/Components/Dropdown/Dropdown.js +33 -9
- package/build/dist/UI/Components/Dropdown/Dropdown.js.map +1 -1
- package/build/dist/UI/Components/Forms/Fields/FormField.js +36 -13
- package/build/dist/UI/Components/Forms/Fields/FormField.js.map +1 -1
- package/build/dist/UI/Components/Graphs/DayUptimeGraph.js +117 -11
- package/build/dist/UI/Components/Graphs/DayUptimeGraph.js.map +1 -1
- package/build/dist/UI/Components/Graphs/UptimeBarTooltip.js +5 -1
- package/build/dist/UI/Components/Graphs/UptimeBarTooltip.js.map +1 -1
- package/build/dist/UI/Components/Graphs/UptimeDaySummary.js.map +1 -1
- package/build/dist/UI/Components/InfoCard/InfoCard.js +22 -1
- package/build/dist/UI/Components/InfoCard/InfoCard.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js +36 -8
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/ColumnSelector.js +10 -2
- package/build/dist/UI/Components/LogsViewer/components/ColumnSelector.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.js +9 -1
- package/build/dist/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogDetailsPanel.js +16 -15
- package/build/dist/UI/Components/LogsViewer/components/LogDetailsPanel.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsFacetSidebar.js +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsFacetSidebar.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsHistogram.js +14 -74
- package/build/dist/UI/Components/LogsViewer/components/LogsHistogram.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js +22 -5
- package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/SavedViewsDropdown.js +9 -2
- package/build/dist/UI/Components/LogsViewer/components/SavedViewsDropdown.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/useLogsHistogram.js +16 -5
- package/build/dist/UI/Components/LogsViewer/useLogsHistogram.js.map +1 -1
- package/build/dist/UI/Components/ModelTable/BaseModelTable.js +89 -22
- package/build/dist/UI/Components/ModelTable/BaseModelTable.js.map +1 -1
- package/build/dist/UI/Components/MonitorGraphs/Uptime.js +7 -1
- package/build/dist/UI/Components/MonitorGraphs/Uptime.js.map +1 -1
- package/build/dist/UI/Components/MonitorGraphs/UptimeBarDayModal.js +5 -1
- package/build/dist/UI/Components/MonitorGraphs/UptimeBarDayModal.js.map +1 -1
- package/build/dist/UI/Components/Page/ModelPage.js +37 -3
- package/build/dist/UI/Components/Page/ModelPage.js.map +1 -1
- package/build/dist/UI/Components/Table/TableHeader.js +18 -2
- package/build/dist/UI/Components/Table/TableHeader.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/TelemetryViewer.js +27 -6
- package/build/dist/UI/Components/TelemetryViewer/TelemetryViewer.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryDetailPanel.js +7 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryDetailPanel.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryFacetSidebar.js +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryFacetSidebar.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogram.js +17 -77
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetryHistogram.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetrySearchBar.js +10 -1
- package/build/dist/UI/Components/TelemetryViewer/components/TelemetrySearchBar.js.map +1 -1
- package/build/dist/UI/Components/Tooltip/InfoTooltip.js +28 -0
- package/build/dist/UI/Components/Tooltip/InfoTooltip.js.map +1 -0
- package/build/dist/UI/Utils/ModelAPI/ProjectUsersModelAPI.js +15 -1
- package/build/dist/UI/Utils/ModelAPI/ProjectUsersModelAPI.js.map +1 -1
- package/build/dist/UI/Utils/PermissionGate.js +4 -2
- package/build/dist/UI/Utils/PermissionGate.js.map +1 -1
- package/build/dist/UI/Utils/Timezone.js +52 -17
- package/build/dist/UI/Utils/Timezone.js.map +1 -1
- package/build/dist/Utils/AI/InvestigationReport.js +116 -21
- package/build/dist/Utils/AI/InvestigationReport.js.map +1 -1
- package/build/dist/Utils/AiRemediation/KubectlOutputRedactor.js +1209 -0
- package/build/dist/Utils/AiRemediation/KubectlOutputRedactor.js.map +1 -0
- package/build/dist/Utils/AiRemediation/KubectlPolicy.js +2973 -0
- package/build/dist/Utils/AiRemediation/KubectlPolicy.js.map +1 -0
- package/build/dist/Utils/AiRemediation/KubectlWaitBudget.js +99 -0
- package/build/dist/Utils/AiRemediation/KubectlWaitBudget.js.map +1 -0
- package/build/dist/Utils/AiRemediation/KubectlWriteScope.js +1429 -0
- package/build/dist/Utils/AiRemediation/KubectlWriteScope.js.map +1 -0
- package/build/dist/Utils/Dashboard/Components/DashboardClockComponent.js +7 -6
- package/build/dist/Utils/Dashboard/Components/DashboardClockComponent.js.map +1 -1
- package/build/dist/Utils/Incident/IncidentFromAlerts.js +306 -0
- package/build/dist/Utils/Incident/IncidentFromAlerts.js.map +1 -0
- package/build/dist/Utils/Monitor/IncomingEmailMonitorAddress.js +214 -0
- package/build/dist/Utils/Monitor/IncomingEmailMonitorAddress.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorCheckScheduleUtil.js +471 -0
- package/build/dist/Utils/Monitor/MonitorCheckScheduleUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorOverviewCriteriaUtil.js +107 -0
- package/build/dist/Utils/Monitor/MonitorOverviewCriteriaUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorOverviewFamily.js +191 -0
- package/build/dist/Utils/Monitor/MonitorOverviewFamily.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorOverviewPresentationUtil.js +1259 -0
- package/build/dist/Utils/Monitor/MonitorOverviewPresentationUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorOverviewProbeUtil.js +593 -0
- package/build/dist/Utils/Monitor/MonitorOverviewProbeUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorOverviewTargetUtil.js +195 -0
- package/build/dist/Utils/Monitor/MonitorOverviewTargetUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorStatusHistoryUtil.js +127 -0
- package/build/dist/Utils/Monitor/MonitorStatusHistoryUtil.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorUptimeSummaryUtil.js +464 -0
- package/build/dist/Utils/Monitor/MonitorUptimeSummaryUtil.js.map +1 -0
- package/build/dist/Utils/Rum/SessionIdentity.js +31 -1
- package/build/dist/Utils/Rum/SessionIdentity.js.map +1 -1
- package/build/dist/Utils/Rum/SessionReplayWireEncoding.js +128 -0
- package/build/dist/Utils/Rum/SessionReplayWireEncoding.js.map +1 -0
- package/build/dist/Utils/Rum/SessionTraceState.js +154 -0
- package/build/dist/Utils/Rum/SessionTraceState.js.map +1 -0
- package/build/dist/Utils/Rum/UserFlow.js +941 -0
- package/build/dist/Utils/Rum/UserFlow.js.map +1 -0
- package/build/dist/Utils/StatusPage/MonitorGroupMergedDowntimeUtil.js +82 -0
- package/build/dist/Utils/StatusPage/MonitorGroupMergedDowntimeUtil.js.map +1 -0
- package/build/dist/Utils/StatusPage/ResourceUptime.js +46 -0
- package/build/dist/Utils/StatusPage/ResourceUptime.js.map +1 -1
- package/build/dist/Utils/StatusPage/UptimeDailyAggregateUtil.js +148 -0
- package/build/dist/Utils/StatusPage/UptimeDailyAggregateUtil.js.map +1 -1
- package/build/dist/Utils/Telemetry/AttributesJSON.js +245 -0
- package/build/dist/Utils/Telemetry/AttributesJSON.js.map +1 -0
- package/build/dist/Utils/Uptime/DayUptimeGraphUtil.js +103 -0
- package/build/dist/Utils/Uptime/DayUptimeGraphUtil.js.map +1 -1
- package/build/dist/Utils/Uptime/UptimeUtil.js +91 -27
- package/build/dist/Utils/Uptime/UptimeUtil.js.map +1 -1
- package/jest.config.json +2 -1
- package/package.json +2 -1
- package/tsconfig.tests.json +36 -0
- package/Tests/UI/Components/EmptyState/EmptyStateGuideLinks.test.tsx +0 -337
- package/UI/Components/EmptyState/EmptyStateGuideLinks.tsx +0 -124
- package/build/dist/UI/Components/EmptyState/EmptyStateGuideLinks.js +0 -47
- package/build/dist/UI/Components/EmptyState/EmptyStateGuideLinks.js.map +0 -1
|
@@ -0,0 +1,3662 @@
|
|
|
1
|
+
import KubernetesClusterAiAccessService, {
|
|
2
|
+
KubernetesAgentRegistrationRefusedException,
|
|
3
|
+
KubernetesClusterAiAccessProjectGates,
|
|
4
|
+
MAX_KUBERNETES_AGENT_RUNNERS_PER_PROJECT,
|
|
5
|
+
MAX_KUBERNETES_AGENT_RUNNER_NAME_LENGTH,
|
|
6
|
+
MAX_NEW_KUBERNETES_AGENT_RUNNERS_PER_PROJECT_PER_HOUR,
|
|
7
|
+
RegisterKubernetesAgentRunnerResult,
|
|
8
|
+
REMEDIATION_WRITE_ACCESS_NEXT_STEP,
|
|
9
|
+
getKubernetesAgentRunnerNameForCluster,
|
|
10
|
+
getPreviousInstanceRetryAfterSeconds,
|
|
11
|
+
} from "../../../Server/Services/KubernetesClusterAiAccessService";
|
|
12
|
+
import KubernetesClusterFeedService from "../../../Server/Services/KubernetesClusterFeedService";
|
|
13
|
+
import KubernetesClusterService from "../../../Server/Services/KubernetesClusterService";
|
|
14
|
+
import LlmProviderService from "../../../Server/Services/LlmProviderService";
|
|
15
|
+
import ProjectService from "../../../Server/Services/ProjectService";
|
|
16
|
+
import RunbookCredentialService from "../../../Server/Services/RunbookCredentialService";
|
|
17
|
+
import RunbookSecretService from "../../../Server/Services/RunbookSecretService";
|
|
18
|
+
import RunnerService from "../../../Server/Services/RunnerService";
|
|
19
|
+
import logger from "../../../Server/Utils/Logger";
|
|
20
|
+
import KubernetesCluster from "../../../Models/DatabaseModels/KubernetesCluster";
|
|
21
|
+
import LlmProvider from "../../../Models/DatabaseModels/LlmProvider";
|
|
22
|
+
import Project from "../../../Models/DatabaseModels/Project";
|
|
23
|
+
import RunbookCredential from "../../../Models/DatabaseModels/RunbookCredential";
|
|
24
|
+
import Runner, {
|
|
25
|
+
RunnerConnectionStatus,
|
|
26
|
+
} from "../../../Models/DatabaseModels/Runner";
|
|
27
|
+
import OneUptimeDate from "../../../Types/Date";
|
|
28
|
+
import BadDataException from "../../../Types/Exception/BadDataException";
|
|
29
|
+
import ForbiddenException from "../../../Types/Exception/ForbiddenException";
|
|
30
|
+
import TooManyRequestsException from "../../../Types/Exception/TooManyRequestsException";
|
|
31
|
+
import ObjectID from "../../../Types/ObjectID";
|
|
32
|
+
import PositiveNumber from "../../../Types/PositiveNumber";
|
|
33
|
+
import {
|
|
34
|
+
KubernetesAiAccessGap,
|
|
35
|
+
KubernetesAiAccessGapCode,
|
|
36
|
+
KubernetesAiRemediationMode,
|
|
37
|
+
KubernetesClusterAiAccessStatus,
|
|
38
|
+
KubernetesRunnerPosture,
|
|
39
|
+
getKubernetesAgentRunnerName,
|
|
40
|
+
isTransientKubernetesAgentRegistrationRefusal,
|
|
41
|
+
} from "../../../Types/Kubernetes/KubernetesClusterAiAccess";
|
|
42
|
+
import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
|
|
43
|
+
|
|
44
|
+
/*
|
|
45
|
+
* Contract under test — the readiness computation behind a cluster's AI
|
|
46
|
+
* page and the in-cluster Runner's self-registration:
|
|
47
|
+
*
|
|
48
|
+
* - every reason OneUptime AI cannot reach or change a cluster becomes a
|
|
49
|
+
* gap with a next step, and readiness is derived ONLY from gaps: no
|
|
50
|
+
* Runner, an offline Runner, a Runner without the AI-commands capability,
|
|
51
|
+
* an external Runner without a Kubernetes credential, a read-only
|
|
52
|
+
* in-cluster Runner asked to remediate, and every project gate;
|
|
53
|
+
* - "in-cluster" access is only ever the in-cluster Runner OF THIS CLUSTER:
|
|
54
|
+
* a bound Runner that runs inside a different cluster (or never said
|
|
55
|
+
* which) is a gap, never credential-less access to the wrong cluster;
|
|
56
|
+
* - investigation and remediation readiness are independent: a gap that
|
|
57
|
+
* only blocks remediation never reads as "AI cannot investigate";
|
|
58
|
+
* - registration with the ingestion key creates the agent Runner row with
|
|
59
|
+
* exactly the kubectl capability (never runbooks or code fixes), binds a
|
|
60
|
+
* never-configured cluster with investigation on and remediation
|
|
61
|
+
* "ask for approval" when the chart granted writes (keeping a mode an
|
|
62
|
+
* operator chose beforehand), rotates the key on a restart WITHOUT
|
|
63
|
+
* touching the operator's switches, never steals a
|
|
64
|
+
* cluster an operator bound to a different Runner, never re-binds a
|
|
65
|
+
* cluster an operator cleared, only reuses a Runner that is THIS
|
|
66
|
+
* cluster's agent, refuses to re-key a Runner that is online unless the
|
|
67
|
+
* caller proves it holds the current key, bounds how many agent Runner
|
|
68
|
+
* rows an ingestion key can mint, and records every bind and rotation on
|
|
69
|
+
* the cluster's feed.
|
|
70
|
+
*/
|
|
71
|
+
|
|
72
|
+
const PROJECT_ID: ObjectID = new ObjectID(
|
|
73
|
+
"22222222-2222-4222-8222-222222222222",
|
|
74
|
+
);
|
|
75
|
+
const CLUSTER_ID: ObjectID = new ObjectID(
|
|
76
|
+
"33333333-3333-4333-8333-333333333333",
|
|
77
|
+
);
|
|
78
|
+
const RUNNER_ID: ObjectID = new ObjectID(
|
|
79
|
+
"44444444-4444-4444-8444-444444444444",
|
|
80
|
+
);
|
|
81
|
+
const CREDENTIAL_ID: ObjectID = new ObjectID(
|
|
82
|
+
"55555555-5555-4555-8555-555555555555",
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
const READY_GATES: KubernetesClusterAiAccessProjectGates = {
|
|
86
|
+
isAiEnabled: true,
|
|
87
|
+
isAutoRemediationEnabled: true,
|
|
88
|
+
isAiCommandExecutionEnabled: true,
|
|
89
|
+
hasLlmProvider: true,
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
function fakeCluster(
|
|
93
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
94
|
+
): KubernetesCluster {
|
|
95
|
+
return {
|
|
96
|
+
id: CLUSTER_ID,
|
|
97
|
+
_id: CLUSTER_ID.toString(),
|
|
98
|
+
projectId: PROJECT_ID,
|
|
99
|
+
name: "prod-us",
|
|
100
|
+
clusterIdentifier: "prod-us",
|
|
101
|
+
aiAccessRunnerId: RUNNER_ID,
|
|
102
|
+
aiAccessCredentialId: undefined,
|
|
103
|
+
isAiInvestigationEnabled: true,
|
|
104
|
+
aiRemediationMode: KubernetesAiRemediationMode.RequireApproval,
|
|
105
|
+
aiKubectlCommandAllowlist: undefined,
|
|
106
|
+
...overrides,
|
|
107
|
+
} as unknown as KubernetesCluster;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function fakeRunner(overrides: Partial<Record<string, unknown>> = {}): Runner {
|
|
111
|
+
return {
|
|
112
|
+
id: RUNNER_ID,
|
|
113
|
+
_id: RUNNER_ID.toString(),
|
|
114
|
+
name: "kubernetes-agent/prod-us",
|
|
115
|
+
lastAlive: OneUptimeDate.getCurrentDate(),
|
|
116
|
+
canRunAiCommands: true,
|
|
117
|
+
/*
|
|
118
|
+
* What registration gives an agent row. Readiness now reads these for
|
|
119
|
+
* an offline agent row (they would block its re-registration), so the
|
|
120
|
+
* fixture carries the agent defaults rather than leaving them unset
|
|
121
|
+
* (which reads as "Runs Runbooks" on).
|
|
122
|
+
*/
|
|
123
|
+
canRunRunbooks: false,
|
|
124
|
+
canRunCodeFixTasks: false,
|
|
125
|
+
hostInfo: {
|
|
126
|
+
kubernetes: {
|
|
127
|
+
inCluster: true,
|
|
128
|
+
allowWrites: true,
|
|
129
|
+
clusterIdentifier: "prod-us",
|
|
130
|
+
},
|
|
131
|
+
},
|
|
132
|
+
...overrides,
|
|
133
|
+
} as unknown as Runner;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function gapCodes(
|
|
137
|
+
status: KubernetesClusterAiAccessStatus,
|
|
138
|
+
): Array<KubernetesAiAccessGapCode> {
|
|
139
|
+
return status.gaps.map((gap: KubernetesAiAccessGap) => {
|
|
140
|
+
return gap.code;
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
describe("KubernetesClusterAiAccessService.getStatusForClusterModel", () => {
|
|
145
|
+
/*
|
|
146
|
+
* What an offline agent Runner of this cluster holds beyond the agent
|
|
147
|
+
* defaults (readiness asks, because holdings stop it re-registering).
|
|
148
|
+
* Nothing, unless a test says otherwise.
|
|
149
|
+
*/
|
|
150
|
+
let credentialCountSpy: jest.SpyInstance;
|
|
151
|
+
let secretCountSpy: jest.SpyInstance;
|
|
152
|
+
let boundClusterCountSpy: jest.SpyInstance;
|
|
153
|
+
|
|
154
|
+
beforeEach(() => {
|
|
155
|
+
jest.spyOn(logger, "error").mockImplementation((): void => {
|
|
156
|
+
return undefined;
|
|
157
|
+
});
|
|
158
|
+
credentialCountSpy = jest
|
|
159
|
+
.spyOn(RunbookCredentialService, "countBy")
|
|
160
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
161
|
+
secretCountSpy = jest
|
|
162
|
+
.spyOn(RunbookSecretService, "countBy")
|
|
163
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
164
|
+
boundClusterCountSpy = jest
|
|
165
|
+
.spyOn(KubernetesClusterService, "countBy")
|
|
166
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
afterEach(() => {
|
|
170
|
+
jest.restoreAllMocks();
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it("is ready for both when an online in-cluster Runner with writes is bound and every gate passes", async () => {
|
|
174
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
175
|
+
|
|
176
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
177
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
178
|
+
cluster: fakeCluster(),
|
|
179
|
+
gates: READY_GATES,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
expect(status.gaps).toEqual([]);
|
|
183
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
184
|
+
expect(status.isRemediationReady).toBe(true);
|
|
185
|
+
expect(status.accessMethod).toBe("in_cluster");
|
|
186
|
+
expect(status.runner?.isOnline).toBe(true);
|
|
187
|
+
expect(status.runner?.posture?.allowWrites).toBe(true);
|
|
188
|
+
expect(status.kubectlAllowlist).toEqual([]);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
/*
|
|
192
|
+
* The Runner's write scope reaches the status (and so the AI page and
|
|
193
|
+
* the planner) as the Runner reported it: the namespaces it lets AI
|
|
194
|
+
* change, its own namespace, and whether node operations are allowed.
|
|
195
|
+
*/
|
|
196
|
+
it("reports the bound Runner's write scope in its posture", async () => {
|
|
197
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
198
|
+
fakeRunner({
|
|
199
|
+
hostInfo: {
|
|
200
|
+
kubernetes: {
|
|
201
|
+
inCluster: true,
|
|
202
|
+
allowWrites: true,
|
|
203
|
+
clusterIdentifier: "prod-us",
|
|
204
|
+
writeNamespaces: ["web", "api"],
|
|
205
|
+
podNamespace: "oneuptime-agent",
|
|
206
|
+
allowNodeOperations: false,
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
}),
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
213
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
214
|
+
cluster: fakeCluster(),
|
|
215
|
+
gates: READY_GATES,
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
expect(status.runner?.posture?.writeNamespaces).toEqual(["web", "api"]);
|
|
219
|
+
expect(status.runner?.posture?.podNamespace).toBe("oneuptime-agent");
|
|
220
|
+
expect(status.runner?.posture?.allowNodeOperations).toBe(false);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("reports no_runner_bound with the install step and blocks both when the cluster has no Runner and none is registered for it", async () => {
|
|
224
|
+
/*
|
|
225
|
+
* The status now looks for this cluster's agent Runner row before
|
|
226
|
+
* choosing the next step (see the "installed but not selected" block
|
|
227
|
+
* below); with none registered, the one-command install is the step.
|
|
228
|
+
*/
|
|
229
|
+
const findOneBy: jest.SpyInstance = jest
|
|
230
|
+
.spyOn(RunnerService, "findOneBy")
|
|
231
|
+
.mockResolvedValue(null);
|
|
232
|
+
|
|
233
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
234
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
235
|
+
cluster: fakeCluster({ aiAccessRunnerId: undefined }),
|
|
236
|
+
gates: READY_GATES,
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
expect(findOneBy).toHaveBeenCalledTimes(1);
|
|
240
|
+
expect(gapCodes(status)).toEqual(["no_runner_bound"]);
|
|
241
|
+
expect(status.gaps[0]?.blocks).toBe("both");
|
|
242
|
+
expect(status.gaps[0]?.title).toBe("No Runner can reach this cluster");
|
|
243
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
244
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
245
|
+
expect(status.isRemediationReady).toBe(false);
|
|
246
|
+
expect(status.runner).toBeNull();
|
|
247
|
+
expect(status.accessMethod).toBe("none");
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
/*
|
|
251
|
+
* The FK on aiAccessRunnerId is ON DELETE SET NULL, so deleting the bound
|
|
252
|
+
* Runner clears the binding in the same statement: the only way to see a
|
|
253
|
+
* bound id with no Runner row is the race between reading the cluster and
|
|
254
|
+
* reading its Runner while a delete lands.
|
|
255
|
+
*/
|
|
256
|
+
it("reports runner_missing only for the race with a Runner delete, and says so", async () => {
|
|
257
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(null);
|
|
258
|
+
|
|
259
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
260
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
261
|
+
cluster: fakeCluster(),
|
|
262
|
+
gates: READY_GATES,
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
expect(gapCodes(status)).toEqual(["runner_missing"]);
|
|
266
|
+
expect(status.gaps[0]?.title).toBe("The bound Runner was just deleted");
|
|
267
|
+
/*
|
|
268
|
+
* Round three: gap next steps are shown verbatim on incident pages too,
|
|
269
|
+
* where "this page" would be the incident. They name the cluster's AI
|
|
270
|
+
* page instead (was "Reload this page").
|
|
271
|
+
*/
|
|
272
|
+
expect(status.gaps[0]?.nextStep).toContain("Reload the cluster's AI page");
|
|
273
|
+
expect(status.gaps[0]?.nextStep).not.toContain("this page");
|
|
274
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
/*
|
|
278
|
+
* A cluster left unbound while its agent Runner already exists (the
|
|
279
|
+
* binding was cleared, its Runner was deleted and re-registered, or the
|
|
280
|
+
* cluster row was recreated by telemetry). Re-running the helm upgrade
|
|
281
|
+
* would change nothing there — a registering Runner never re-binds a
|
|
282
|
+
* cluster that was configured before — so the step that works is
|
|
283
|
+
* selecting the Runner on the AI page.
|
|
284
|
+
*/
|
|
285
|
+
describe("no Runner bound, but this cluster's agent Runner is registered", () => {
|
|
286
|
+
it("tells the operator to select that Runner on the cluster's AI page, not to run helm", async () => {
|
|
287
|
+
const findOneBy: jest.SpyInstance = jest
|
|
288
|
+
.spyOn(RunnerService, "findOneBy")
|
|
289
|
+
.mockResolvedValue(fakeRunner());
|
|
290
|
+
|
|
291
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
292
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
293
|
+
cluster: fakeCluster({ aiAccessRunnerId: undefined }),
|
|
294
|
+
gates: READY_GATES,
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
expect(gapCodes(status)).toEqual(["no_runner_bound"]);
|
|
298
|
+
expect(status.gaps[0]?.title).toBe(
|
|
299
|
+
"The in-cluster Runner is installed but not selected",
|
|
300
|
+
);
|
|
301
|
+
expect(status.gaps[0]?.description).toContain("and online");
|
|
302
|
+
expect(status.gaps[0]?.nextStep).toContain(
|
|
303
|
+
'Select the kubernetes-agent Runner "kubernetes-agent/prod-us"',
|
|
304
|
+
);
|
|
305
|
+
// Round three: named, not "on this page" (shown on incident pages too).
|
|
306
|
+
expect(status.gaps[0]?.nextStep).toContain("on the cluster's AI page");
|
|
307
|
+
expect(status.gaps[0]?.nextStep).not.toContain("this page");
|
|
308
|
+
expect(status.gaps[0]?.nextStep).not.toContain("aiAccess.enabled=true");
|
|
309
|
+
expect(status.gaps[0]?.blocks).toBe("both");
|
|
310
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
311
|
+
expect(status.runner).toBeNull();
|
|
312
|
+
|
|
313
|
+
// Looked up in this project, by the agent name, case-insensitively.
|
|
314
|
+
const query: Record<string, unknown> = (
|
|
315
|
+
findOneBy.mock.calls[0]![0] as { query: Record<string, unknown> }
|
|
316
|
+
).query;
|
|
317
|
+
expect(query["projectId"]).toBe(PROJECT_ID);
|
|
318
|
+
expect(typeof query["name"]).not.toBe("string");
|
|
319
|
+
expect(query["name"]).toBeDefined();
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
it("says the Runner is not online when it is not, and still points at selecting it", async () => {
|
|
323
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
324
|
+
fakeRunner({
|
|
325
|
+
lastAlive: OneUptimeDate.getSomeMinutesAgo(30),
|
|
326
|
+
}),
|
|
327
|
+
);
|
|
328
|
+
|
|
329
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
330
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
331
|
+
cluster: fakeCluster({ aiAccessRunnerId: undefined }),
|
|
332
|
+
gates: READY_GATES,
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
expect(status.gaps[0]?.description).toContain("not online right now");
|
|
336
|
+
expect(status.gaps[0]?.nextStep).toContain("Select the kubernetes-agent");
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
it("keeps the install step when the row by that name reports a different cluster", async () => {
|
|
340
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
341
|
+
fakeRunner({
|
|
342
|
+
hostInfo: {
|
|
343
|
+
kubernetes: { inCluster: true, clusterIdentifier: "prod-eu" },
|
|
344
|
+
},
|
|
345
|
+
}),
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
349
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
350
|
+
cluster: fakeCluster({ aiAccessRunnerId: undefined }),
|
|
351
|
+
gates: READY_GATES,
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
expect(status.gaps[0]?.title).toBe("No Runner can reach this cluster");
|
|
355
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it("never throws: a failed lookup falls back to the install step", async () => {
|
|
359
|
+
jest
|
|
360
|
+
.spyOn(RunnerService, "findOneBy")
|
|
361
|
+
.mockRejectedValue(new Error("db down"));
|
|
362
|
+
|
|
363
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
364
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
365
|
+
cluster: fakeCluster({ aiAccessRunnerId: undefined }),
|
|
366
|
+
gates: READY_GATES,
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
expect(gapCodes(status)).toEqual(["no_runner_bound"]);
|
|
370
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
it("does not look anything up for a cluster row without an identifier", async () => {
|
|
374
|
+
const findOneBy: jest.SpyInstance = jest.spyOn(
|
|
375
|
+
RunnerService,
|
|
376
|
+
"findOneBy",
|
|
377
|
+
);
|
|
378
|
+
|
|
379
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
380
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
381
|
+
cluster: fakeCluster({
|
|
382
|
+
aiAccessRunnerId: undefined,
|
|
383
|
+
clusterIdentifier: " ",
|
|
384
|
+
}),
|
|
385
|
+
gates: READY_GATES,
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
expect(findOneBy).not.toHaveBeenCalled();
|
|
389
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
390
|
+
});
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
/*
|
|
394
|
+
* A Runner that signs off (/runner-ingest/disconnect — what a helm
|
|
395
|
+
* uninstall or turning aiAccess off does to the agent's pod) is offline
|
|
396
|
+
* at once, not "Connected" until its last heartbeat ages out.
|
|
397
|
+
*/
|
|
398
|
+
describe("a Runner that signed off", () => {
|
|
399
|
+
it("is offline and blocks both, with the uninstall-aware copy and not the 'check the pod' step", async () => {
|
|
400
|
+
const findOneBy: jest.SpyInstance = jest
|
|
401
|
+
.spyOn(RunnerService, "findOneBy")
|
|
402
|
+
.mockResolvedValue(
|
|
403
|
+
fakeRunner({
|
|
404
|
+
lastAlive: OneUptimeDate.getCurrentDate(),
|
|
405
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
406
|
+
}),
|
|
407
|
+
);
|
|
408
|
+
|
|
409
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
410
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
411
|
+
cluster: fakeCluster(),
|
|
412
|
+
gates: READY_GATES,
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
expect(status.runner?.isOnline).toBe(false);
|
|
416
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
417
|
+
expect(status.isRemediationReady).toBe(false);
|
|
418
|
+
expect(gapCodes(status)).toEqual(["runner_offline"]);
|
|
419
|
+
expect(status.gaps[0]?.title).toBe("The in-cluster Runner signed off");
|
|
420
|
+
expect(status.gaps[0]?.description).toContain("uninstalled");
|
|
421
|
+
// Round three: was "clear the Runner on this page".
|
|
422
|
+
expect(status.gaps[0]?.nextStep).toContain(
|
|
423
|
+
"clear the Runner on the cluster's AI page",
|
|
424
|
+
);
|
|
425
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
426
|
+
expect(status.gaps[0]?.nextStep).not.toContain("component=ai-runner");
|
|
427
|
+
|
|
428
|
+
// The sign-off is read, not guessed.
|
|
429
|
+
const select: Record<string, unknown> = (
|
|
430
|
+
findOneBy.mock.calls[0]![0] as { select: Record<string, unknown> }
|
|
431
|
+
).select;
|
|
432
|
+
expect(select["connectionStatus"]).toBe(true);
|
|
433
|
+
});
|
|
434
|
+
|
|
435
|
+
it("uses plain copy for a dashboard Runner that signed off", async () => {
|
|
436
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
437
|
+
fakeRunner({
|
|
438
|
+
name: "office-runner",
|
|
439
|
+
hostInfo: { hostname: "x" },
|
|
440
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
441
|
+
}),
|
|
442
|
+
);
|
|
443
|
+
|
|
444
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
445
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
446
|
+
cluster: fakeCluster(),
|
|
447
|
+
gates: READY_GATES,
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
expect(status.gaps[0]?.code).toBe("runner_offline");
|
|
451
|
+
expect(status.gaps[0]?.title).toBe("The Runner signed off");
|
|
452
|
+
expect(status.gaps[0]?.nextStep).toContain("Start the Runner container");
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it("negative control: the same Runner Connected is online and ready", async () => {
|
|
456
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
457
|
+
fakeRunner({
|
|
458
|
+
lastAlive: OneUptimeDate.getCurrentDate(),
|
|
459
|
+
connectionStatus: RunnerConnectionStatus.Connected,
|
|
460
|
+
}),
|
|
461
|
+
);
|
|
462
|
+
|
|
463
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
464
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
465
|
+
cluster: fakeCluster(),
|
|
466
|
+
gates: READY_GATES,
|
|
467
|
+
});
|
|
468
|
+
|
|
469
|
+
expect(status.runner?.isOnline).toBe(true);
|
|
470
|
+
expect(status.gaps).toEqual([]);
|
|
471
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
472
|
+
expect(status.isRemediationReady).toBe(true);
|
|
473
|
+
});
|
|
474
|
+
|
|
475
|
+
it("negative control: a row that never heartbeated is 'never connected', not 'signed off'", async () => {
|
|
476
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
477
|
+
fakeRunner({
|
|
478
|
+
lastAlive: undefined,
|
|
479
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
480
|
+
}),
|
|
481
|
+
);
|
|
482
|
+
|
|
483
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
484
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
485
|
+
cluster: fakeCluster(),
|
|
486
|
+
gates: READY_GATES,
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
expect(status.gaps[0]?.title).toBe("The Runner has never connected");
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
/*
|
|
493
|
+
* A restarted agent pod cannot present the key its predecessor held, so
|
|
494
|
+
* a row holding more than the agent defaults is refused on every
|
|
495
|
+
* registration — "it reconnects within a minute" would be false.
|
|
496
|
+
*/
|
|
497
|
+
describe("an offline agent Runner that holds more than the defaults", () => {
|
|
498
|
+
it('says it cannot re-register and names "Runs Runbooks" instead of promising a reconnect', async () => {
|
|
499
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
500
|
+
fakeRunner({
|
|
501
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
502
|
+
canRunRunbooks: true,
|
|
503
|
+
}),
|
|
504
|
+
);
|
|
505
|
+
|
|
506
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
507
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
508
|
+
cluster: fakeCluster(),
|
|
509
|
+
gates: READY_GATES,
|
|
510
|
+
});
|
|
511
|
+
|
|
512
|
+
expect(gapCodes(status)).toEqual(["runner_offline"]);
|
|
513
|
+
const gap: KubernetesAiAccessGap = status.gaps[0]!;
|
|
514
|
+
expect(gap.title).toContain("cannot re-register");
|
|
515
|
+
expect(gap.description).toContain('"Runs Runbooks" is on');
|
|
516
|
+
expect(gap.nextStep).toContain(
|
|
517
|
+
'on Runner "kubernetes-agent/prod-us": turn off "Runs Runbooks"',
|
|
518
|
+
);
|
|
519
|
+
expect(gap.nextStep).toContain("delete the Runner");
|
|
520
|
+
/*
|
|
521
|
+
* Round four: deleting the Runner leaves this cluster unbound (the
|
|
522
|
+
* FK nulls the binding and registration never re-binds a cluster
|
|
523
|
+
* that had a Runner), so the delete path names its second step.
|
|
524
|
+
* Was "... — or delete the Runner, and the in-cluster Runner
|
|
525
|
+
* registers a fresh one within a minute."
|
|
526
|
+
*/
|
|
527
|
+
expect(gap.nextStep).toContain(
|
|
528
|
+
"select it on the cluster's AI page as its Runner",
|
|
529
|
+
);
|
|
530
|
+
expect(gap.nextStep).toContain("still bound to this cluster");
|
|
531
|
+
expect(gap.nextStep).not.toContain("reconnects within a minute");
|
|
532
|
+
expect(gap.nextStep).not.toContain("aiAccess.enabled=true");
|
|
533
|
+
expect(gap.blocks).toBe("both");
|
|
534
|
+
});
|
|
535
|
+
|
|
536
|
+
it("names assigned credentials, secrets and other clusters bound to it, counted for THIS cluster", async () => {
|
|
537
|
+
credentialCountSpy.mockResolvedValue(new PositiveNumber(1));
|
|
538
|
+
secretCountSpy.mockResolvedValue(new PositiveNumber(2));
|
|
539
|
+
boundClusterCountSpy.mockResolvedValue(new PositiveNumber(1));
|
|
540
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
541
|
+
fakeRunner({
|
|
542
|
+
lastAlive: OneUptimeDate.getSomeMinutesAgo(30),
|
|
543
|
+
}),
|
|
544
|
+
);
|
|
545
|
+
|
|
546
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
547
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
548
|
+
cluster: fakeCluster(),
|
|
549
|
+
gates: READY_GATES,
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
const gap: KubernetesAiAccessGap = status.gaps[0]!;
|
|
553
|
+
expect(gap.description).toContain("1 Runner credential(s) assigned");
|
|
554
|
+
expect(gap.description).toContain("2 runbook secret(s) assigned");
|
|
555
|
+
expect(gap.description).toContain(
|
|
556
|
+
"the AI access Runner of 1 other cluster(s)",
|
|
557
|
+
);
|
|
558
|
+
// "Runs Runbooks" is off on this row, so it is not named.
|
|
559
|
+
expect(gap.description).not.toContain("Runs Runbooks");
|
|
560
|
+
|
|
561
|
+
const clusterQuery: Record<string, unknown> = (
|
|
562
|
+
boundClusterCountSpy.mock.calls[0]![0] as {
|
|
563
|
+
query: Record<string, unknown>;
|
|
564
|
+
}
|
|
565
|
+
).query;
|
|
566
|
+
expect(clusterQuery["_id"]).toBeDefined();
|
|
567
|
+
expect(typeof clusterQuery["_id"]).not.toBe("string");
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
it("negative control: a signed-off agent row with nothing extra keeps the reconnect wording", async () => {
|
|
571
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
572
|
+
fakeRunner({
|
|
573
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
574
|
+
}),
|
|
575
|
+
);
|
|
576
|
+
|
|
577
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
578
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
579
|
+
cluster: fakeCluster(),
|
|
580
|
+
gates: READY_GATES,
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
expect(status.gaps[0]?.title).toBe("The in-cluster Runner signed off");
|
|
584
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
it("negative control: an online agent row is never counted", async () => {
|
|
588
|
+
jest
|
|
589
|
+
.spyOn(RunnerService, "findOneBy")
|
|
590
|
+
.mockResolvedValue(fakeRunner({ canRunRunbooks: true }));
|
|
591
|
+
|
|
592
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
593
|
+
cluster: fakeCluster(),
|
|
594
|
+
gates: READY_GATES,
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
expect(credentialCountSpy).not.toHaveBeenCalled();
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
it("negative control: another cluster's agent row is not judged by this cluster's holdings", async () => {
|
|
601
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
602
|
+
fakeRunner({
|
|
603
|
+
name: "kubernetes-agent/prod-eu",
|
|
604
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
605
|
+
canRunRunbooks: true,
|
|
606
|
+
hostInfo: {
|
|
607
|
+
kubernetes: { inCluster: true, clusterIdentifier: "prod-eu" },
|
|
608
|
+
},
|
|
609
|
+
}),
|
|
610
|
+
);
|
|
611
|
+
|
|
612
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
613
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
614
|
+
cluster: fakeCluster(),
|
|
615
|
+
gates: READY_GATES,
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
expect(status.gaps[0]?.title).toBe("The in-cluster Runner signed off");
|
|
619
|
+
expect(credentialCountSpy).not.toHaveBeenCalled();
|
|
620
|
+
});
|
|
621
|
+
|
|
622
|
+
it("never throws: a failed holdings lookup falls back to the plain offline gap", async () => {
|
|
623
|
+
credentialCountSpy.mockRejectedValue(new Error("db down"));
|
|
624
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
625
|
+
fakeRunner({
|
|
626
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
627
|
+
}),
|
|
628
|
+
);
|
|
629
|
+
|
|
630
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
631
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
632
|
+
cluster: fakeCluster(),
|
|
633
|
+
gates: READY_GATES,
|
|
634
|
+
});
|
|
635
|
+
|
|
636
|
+
expect(status.gaps[0]?.title).toBe("The in-cluster Runner signed off");
|
|
637
|
+
});
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
it("still reports the sign-off once the last heartbeat has also aged out", async () => {
|
|
641
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
642
|
+
fakeRunner({
|
|
643
|
+
lastAlive: OneUptimeDate.getSomeMinutesAgo(30),
|
|
644
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
645
|
+
}),
|
|
646
|
+
);
|
|
647
|
+
|
|
648
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
649
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
650
|
+
cluster: fakeCluster(),
|
|
651
|
+
gates: READY_GATES,
|
|
652
|
+
});
|
|
653
|
+
|
|
654
|
+
// Signed off AND stale: the sign-off is the more precise explanation.
|
|
655
|
+
expect(status.gaps[0]?.title).toBe("The in-cluster Runner signed off");
|
|
656
|
+
expect(status.runner?.isOnline).toBe(false);
|
|
657
|
+
});
|
|
658
|
+
});
|
|
659
|
+
|
|
660
|
+
it("reports runner_offline (never connected vs stale) and keeps the Runner summary", async () => {
|
|
661
|
+
jest
|
|
662
|
+
.spyOn(RunnerService, "findOneBy")
|
|
663
|
+
.mockResolvedValue(fakeRunner({ lastAlive: undefined }));
|
|
664
|
+
|
|
665
|
+
const never: KubernetesClusterAiAccessStatus =
|
|
666
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
667
|
+
cluster: fakeCluster(),
|
|
668
|
+
gates: READY_GATES,
|
|
669
|
+
});
|
|
670
|
+
|
|
671
|
+
expect(gapCodes(never)).toEqual(["runner_offline"]);
|
|
672
|
+
expect(never.gaps[0]?.title).toContain("never connected");
|
|
673
|
+
expect(never.runner?.isOnline).toBe(false);
|
|
674
|
+
|
|
675
|
+
jest.restoreAllMocks();
|
|
676
|
+
jest
|
|
677
|
+
.spyOn(RunnerService, "findOneBy")
|
|
678
|
+
.mockResolvedValue(
|
|
679
|
+
fakeRunner({ lastAlive: OneUptimeDate.getSomeMinutesAgo(30) }),
|
|
680
|
+
);
|
|
681
|
+
|
|
682
|
+
const stale: KubernetesClusterAiAccessStatus =
|
|
683
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
684
|
+
cluster: fakeCluster(),
|
|
685
|
+
gates: READY_GATES,
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
expect(gapCodes(stale)).toEqual(["runner_offline"]);
|
|
689
|
+
expect(stale.gaps[0]?.title).toBe("The Runner is offline");
|
|
690
|
+
expect(stale.gaps[0]?.nextStep).toContain("component=ai-runner");
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
it("reports runner_ai_commands_disabled when the capability is off", async () => {
|
|
694
|
+
jest
|
|
695
|
+
.spyOn(RunnerService, "findOneBy")
|
|
696
|
+
.mockResolvedValue(fakeRunner({ canRunAiCommands: false }));
|
|
697
|
+
|
|
698
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
699
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
700
|
+
cluster: fakeCluster(),
|
|
701
|
+
gates: READY_GATES,
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
expect(gapCodes(status)).toEqual(["runner_ai_commands_disabled"]);
|
|
705
|
+
expect(status.runner?.canRunAiCommands).toBe(false);
|
|
706
|
+
});
|
|
707
|
+
|
|
708
|
+
it("requires a Kubernetes credential assigned to an external Runner", async () => {
|
|
709
|
+
jest
|
|
710
|
+
.spyOn(RunnerService, "findOneBy")
|
|
711
|
+
.mockResolvedValue(
|
|
712
|
+
fakeRunner({ name: "office-runner", hostInfo: { hostname: "x" } }),
|
|
713
|
+
);
|
|
714
|
+
|
|
715
|
+
const noCredential: KubernetesClusterAiAccessStatus =
|
|
716
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
717
|
+
cluster: fakeCluster(),
|
|
718
|
+
gates: READY_GATES,
|
|
719
|
+
});
|
|
720
|
+
|
|
721
|
+
expect(gapCodes(noCredential)).toEqual(["credential_missing"]);
|
|
722
|
+
expect(noCredential.accessMethod).toBe("none");
|
|
723
|
+
|
|
724
|
+
// Credential exists but is not assigned to this Runner.
|
|
725
|
+
jest.spyOn(RunbookCredentialService, "findOneBy").mockResolvedValue({
|
|
726
|
+
id: CREDENTIAL_ID,
|
|
727
|
+
_id: CREDENTIAL_ID.toString(),
|
|
728
|
+
name: "prod kubeconfig",
|
|
729
|
+
credentialType: "Kubernetes",
|
|
730
|
+
runners: [],
|
|
731
|
+
} as unknown as RunbookCredential);
|
|
732
|
+
|
|
733
|
+
const unassigned: KubernetesClusterAiAccessStatus =
|
|
734
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
735
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
736
|
+
gates: READY_GATES,
|
|
737
|
+
});
|
|
738
|
+
|
|
739
|
+
expect(gapCodes(unassigned)).toEqual(["credential_missing"]);
|
|
740
|
+
expect(unassigned.gaps[0]?.description).toContain("not assigned");
|
|
741
|
+
|
|
742
|
+
// Assigned Kubernetes credential: ready, and the id travels on the status.
|
|
743
|
+
jest.restoreAllMocks();
|
|
744
|
+
jest
|
|
745
|
+
.spyOn(RunnerService, "findOneBy")
|
|
746
|
+
.mockResolvedValue(
|
|
747
|
+
fakeRunner({ name: "office-runner", hostInfo: { hostname: "x" } }),
|
|
748
|
+
);
|
|
749
|
+
jest.spyOn(RunbookCredentialService, "findOneBy").mockResolvedValue({
|
|
750
|
+
id: CREDENTIAL_ID,
|
|
751
|
+
_id: CREDENTIAL_ID.toString(),
|
|
752
|
+
name: "prod kubeconfig",
|
|
753
|
+
credentialType: "Kubernetes",
|
|
754
|
+
runners: [{ id: RUNNER_ID, _id: RUNNER_ID.toString() }],
|
|
755
|
+
} as unknown as RunbookCredential);
|
|
756
|
+
|
|
757
|
+
const assigned: KubernetesClusterAiAccessStatus =
|
|
758
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
759
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
760
|
+
gates: READY_GATES,
|
|
761
|
+
});
|
|
762
|
+
|
|
763
|
+
expect(assigned.gaps).toEqual([]);
|
|
764
|
+
expect(assigned.accessMethod).toBe("credential");
|
|
765
|
+
expect(assigned.credentialId).toBe(CREDENTIAL_ID.toString());
|
|
766
|
+
expect(assigned.credentialName).toBe("prod kubeconfig");
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
/*
|
|
770
|
+
* The cross-cluster hole: the dashboard lets an operator bind ANY Runner,
|
|
771
|
+
* and a Runner that is in-cluster in cluster B would run every command
|
|
772
|
+
* for cluster A against B's ServiceAccount. In-cluster access therefore
|
|
773
|
+
* exists only for the Runner whose posture names THIS cluster.
|
|
774
|
+
*/
|
|
775
|
+
describe("in-cluster access is only the in-cluster Runner of this cluster", () => {
|
|
776
|
+
it("refuses the in-cluster Runner of a different cluster with a gap that blocks both", async () => {
|
|
777
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
778
|
+
fakeRunner({
|
|
779
|
+
name: "kubernetes-agent/prod-eu",
|
|
780
|
+
hostInfo: {
|
|
781
|
+
kubernetes: {
|
|
782
|
+
inCluster: true,
|
|
783
|
+
allowWrites: true,
|
|
784
|
+
clusterIdentifier: "prod-eu",
|
|
785
|
+
},
|
|
786
|
+
},
|
|
787
|
+
}),
|
|
788
|
+
);
|
|
789
|
+
|
|
790
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
791
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
792
|
+
cluster: fakeCluster(),
|
|
793
|
+
gates: READY_GATES,
|
|
794
|
+
});
|
|
795
|
+
|
|
796
|
+
expect(gapCodes(status)).toEqual(["runner_cluster_mismatch"]);
|
|
797
|
+
expect(status.gaps[0]?.blocks).toBe("both");
|
|
798
|
+
expect(status.gaps[0]?.title).toContain("different cluster");
|
|
799
|
+
expect(status.gaps[0]?.description).toContain('"prod-eu"');
|
|
800
|
+
expect(status.gaps[0]?.description).toContain('"prod-us"');
|
|
801
|
+
expect(status.gaps[0]?.nextStep).toContain("aiAccess.enabled=true");
|
|
802
|
+
expect(status.accessMethod).toBe("none");
|
|
803
|
+
expect(status.credentialId).toBeUndefined();
|
|
804
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
805
|
+
expect(status.isRemediationReady).toBe(false);
|
|
806
|
+
// The Runner is still described, so the page can say WHICH Runner.
|
|
807
|
+
expect(status.runner?.name).toBe("kubernetes-agent/prod-eu");
|
|
808
|
+
expect(status.runner?.posture?.clusterIdentifier).toBe("prod-eu");
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
it("refuses a pod Runner that never said which cluster it runs in", async () => {
|
|
812
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
813
|
+
fakeRunner({
|
|
814
|
+
name: "pod-runner",
|
|
815
|
+
hostInfo: { kubernetes: { inCluster: true, allowWrites: true } },
|
|
816
|
+
}),
|
|
817
|
+
);
|
|
818
|
+
|
|
819
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
820
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
821
|
+
cluster: fakeCluster(),
|
|
822
|
+
gates: READY_GATES,
|
|
823
|
+
});
|
|
824
|
+
|
|
825
|
+
expect(gapCodes(status)).toEqual(["runner_cluster_mismatch"]);
|
|
826
|
+
expect(status.gaps[0]?.title).toContain("did not report which cluster");
|
|
827
|
+
expect(status.accessMethod).toBe("none");
|
|
828
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
829
|
+
});
|
|
830
|
+
|
|
831
|
+
it("never treats a blank cluster identifier on the cluster row as a match", async () => {
|
|
832
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
833
|
+
fakeRunner({
|
|
834
|
+
hostInfo: {
|
|
835
|
+
kubernetes: {
|
|
836
|
+
inCluster: true,
|
|
837
|
+
allowWrites: true,
|
|
838
|
+
clusterIdentifier: "",
|
|
839
|
+
},
|
|
840
|
+
},
|
|
841
|
+
}),
|
|
842
|
+
);
|
|
843
|
+
|
|
844
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
845
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
846
|
+
cluster: fakeCluster({ clusterIdentifier: "" }),
|
|
847
|
+
gates: READY_GATES,
|
|
848
|
+
});
|
|
849
|
+
|
|
850
|
+
expect(gapCodes(status)).toEqual(["runner_cluster_mismatch"]);
|
|
851
|
+
expect(status.accessMethod).toBe("none");
|
|
852
|
+
});
|
|
853
|
+
|
|
854
|
+
it("matches the cluster identifier case-insensitively and with surrounding whitespace", async () => {
|
|
855
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
856
|
+
fakeRunner({
|
|
857
|
+
hostInfo: {
|
|
858
|
+
kubernetes: {
|
|
859
|
+
inCluster: true,
|
|
860
|
+
allowWrites: true,
|
|
861
|
+
clusterIdentifier: " PROD-us ",
|
|
862
|
+
},
|
|
863
|
+
},
|
|
864
|
+
}),
|
|
865
|
+
);
|
|
866
|
+
|
|
867
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
868
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
869
|
+
cluster: fakeCluster({ clusterIdentifier: "prod-us" }),
|
|
870
|
+
gates: READY_GATES,
|
|
871
|
+
});
|
|
872
|
+
|
|
873
|
+
expect(status.gaps).toEqual([]);
|
|
874
|
+
expect(status.accessMethod).toBe("in_cluster");
|
|
875
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
876
|
+
});
|
|
877
|
+
|
|
878
|
+
/*
|
|
879
|
+
* Changed with the credential-exfiltration fix: this used to be the
|
|
880
|
+
* supported "another cluster's agent + an assigned credential" setup
|
|
881
|
+
* and read as ready. A kubernetes-agent Runner is minted and re-keyed
|
|
882
|
+
* with the telemetry ingestion key, so it is never handed credential
|
|
883
|
+
* material — the claim path refuses it — and readiness must say so
|
|
884
|
+
* instead of promising access that would fail on every command.
|
|
885
|
+
*/
|
|
886
|
+
it("refuses a credential on another cluster's agent Runner with its own gap, never 'credential' access", async () => {
|
|
887
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
888
|
+
fakeRunner({
|
|
889
|
+
name: "kubernetes-agent/prod-eu",
|
|
890
|
+
hostInfo: {
|
|
891
|
+
kubernetes: {
|
|
892
|
+
inCluster: true,
|
|
893
|
+
allowWrites: true,
|
|
894
|
+
clusterIdentifier: "prod-eu",
|
|
895
|
+
},
|
|
896
|
+
},
|
|
897
|
+
}),
|
|
898
|
+
);
|
|
899
|
+
const credentialLookup: jest.SpyInstance = jest
|
|
900
|
+
.spyOn(RunbookCredentialService, "findOneBy")
|
|
901
|
+
.mockResolvedValue({
|
|
902
|
+
id: CREDENTIAL_ID,
|
|
903
|
+
_id: CREDENTIAL_ID.toString(),
|
|
904
|
+
name: "prod-us kubeconfig",
|
|
905
|
+
credentialType: "Kubernetes",
|
|
906
|
+
runners: [{ id: RUNNER_ID, _id: RUNNER_ID.toString() }],
|
|
907
|
+
} as unknown as RunbookCredential);
|
|
908
|
+
|
|
909
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
910
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
911
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
912
|
+
gates: READY_GATES,
|
|
913
|
+
});
|
|
914
|
+
|
|
915
|
+
expect(gapCodes(status)).toEqual(["credential_on_agent_runner"]);
|
|
916
|
+
expect(status.gaps[0]?.blocks).toBe("both");
|
|
917
|
+
expect(status.gaps[0]?.description).toContain(
|
|
918
|
+
'the in-cluster Runner of cluster "prod-eu"',
|
|
919
|
+
);
|
|
920
|
+
expect(status.gaps[0]?.nextStep).toContain(
|
|
921
|
+
"Create a Runner under Project Settings → Runners",
|
|
922
|
+
);
|
|
923
|
+
expect(status.accessMethod).toBe("none");
|
|
924
|
+
expect(status.credentialId).toBeUndefined();
|
|
925
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
926
|
+
expect(status.isRemediationReady).toBe(false);
|
|
927
|
+
// The credential's details are moot: an agent row never carries one.
|
|
928
|
+
expect(credentialLookup).not.toHaveBeenCalled();
|
|
929
|
+
});
|
|
930
|
+
|
|
931
|
+
it("keyed on the server-owned NAME: an agent row whose heartbeat dropped its posture still cannot carry a credential", async () => {
|
|
932
|
+
jest
|
|
933
|
+
.spyOn(RunnerService, "findOneBy")
|
|
934
|
+
.mockResolvedValue(
|
|
935
|
+
fakeRunner({ name: "kubernetes-agent/prod-eu", hostInfo: {} }),
|
|
936
|
+
);
|
|
937
|
+
jest.spyOn(RunbookCredentialService, "findOneBy").mockResolvedValue({
|
|
938
|
+
id: CREDENTIAL_ID,
|
|
939
|
+
_id: CREDENTIAL_ID.toString(),
|
|
940
|
+
name: "prod-us kubeconfig",
|
|
941
|
+
credentialType: "Kubernetes",
|
|
942
|
+
runners: [{ id: RUNNER_ID, _id: RUNNER_ID.toString() }],
|
|
943
|
+
} as unknown as RunbookCredential);
|
|
944
|
+
|
|
945
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
946
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
947
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
948
|
+
gates: READY_GATES,
|
|
949
|
+
});
|
|
950
|
+
|
|
951
|
+
expect(gapCodes(status)).toEqual(["credential_on_agent_runner"]);
|
|
952
|
+
expect(status.accessMethod).toBe("none");
|
|
953
|
+
});
|
|
954
|
+
|
|
955
|
+
/*
|
|
956
|
+
* An agent row renamed (only root can rename one now) keeps its agent
|
|
957
|
+
* posture, and the posture alone marks it: the one "is an agent row"
|
|
958
|
+
* rule fails closed on the name OR the posture.
|
|
959
|
+
*/
|
|
960
|
+
it("keyed on the name OR the posture: a row with another cluster's agent posture but no marker cannot carry a credential", async () => {
|
|
961
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
962
|
+
fakeRunner({
|
|
963
|
+
name: "prod-eu in-cluster runner",
|
|
964
|
+
hostInfo: {
|
|
965
|
+
kubernetes: {
|
|
966
|
+
inCluster: true,
|
|
967
|
+
allowWrites: true,
|
|
968
|
+
clusterIdentifier: "prod-eu",
|
|
969
|
+
},
|
|
970
|
+
},
|
|
971
|
+
}),
|
|
972
|
+
);
|
|
973
|
+
jest.spyOn(RunbookCredentialService, "findOneBy").mockResolvedValue({
|
|
974
|
+
id: CREDENTIAL_ID,
|
|
975
|
+
_id: CREDENTIAL_ID.toString(),
|
|
976
|
+
name: "prod-us kubeconfig",
|
|
977
|
+
credentialType: "Kubernetes",
|
|
978
|
+
runners: [{ id: RUNNER_ID, _id: RUNNER_ID.toString() }],
|
|
979
|
+
} as unknown as RunbookCredential);
|
|
980
|
+
|
|
981
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
982
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
983
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
984
|
+
gates: READY_GATES,
|
|
985
|
+
});
|
|
986
|
+
|
|
987
|
+
expect(gapCodes(status)).toEqual(["credential_on_agent_runner"]);
|
|
988
|
+
expect(status.accessMethod).toBe("none");
|
|
989
|
+
expect(status.credentialId).toBeUndefined();
|
|
990
|
+
});
|
|
991
|
+
|
|
992
|
+
it("keyed on a case variant of the marker too", async () => {
|
|
993
|
+
jest
|
|
994
|
+
.spyOn(RunnerService, "findOneBy")
|
|
995
|
+
.mockResolvedValue(
|
|
996
|
+
fakeRunner({ name: "Kubernetes-Agent/prod-eu", hostInfo: {} }),
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1000
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1001
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
1002
|
+
gates: READY_GATES,
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
expect(gapCodes(status)).toEqual(["credential_on_agent_runner"]);
|
|
1006
|
+
});
|
|
1007
|
+
|
|
1008
|
+
/*
|
|
1009
|
+
* Before round two this fixture reported a cluster identity too (an
|
|
1010
|
+
* agent posture). An ordinary Runner in a pod never does — only the
|
|
1011
|
+
* kubernetes-agent binary reports one — and such a posture now marks
|
|
1012
|
+
* the row as an agent (the case above).
|
|
1013
|
+
*/
|
|
1014
|
+
it("negative control: a dashboard Runner that happens to live in a pod still uses an assigned credential", async () => {
|
|
1015
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
1016
|
+
fakeRunner({
|
|
1017
|
+
name: "pod-runner",
|
|
1018
|
+
hostInfo: {
|
|
1019
|
+
kubernetes: {
|
|
1020
|
+
inCluster: true,
|
|
1021
|
+
allowWrites: true,
|
|
1022
|
+
},
|
|
1023
|
+
},
|
|
1024
|
+
}),
|
|
1025
|
+
);
|
|
1026
|
+
jest.spyOn(RunbookCredentialService, "findOneBy").mockResolvedValue({
|
|
1027
|
+
id: CREDENTIAL_ID,
|
|
1028
|
+
_id: CREDENTIAL_ID.toString(),
|
|
1029
|
+
name: "prod-us kubeconfig",
|
|
1030
|
+
credentialType: "Kubernetes",
|
|
1031
|
+
runners: [{ id: RUNNER_ID, _id: RUNNER_ID.toString() }],
|
|
1032
|
+
} as unknown as RunbookCredential);
|
|
1033
|
+
|
|
1034
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1035
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1036
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
1037
|
+
gates: READY_GATES,
|
|
1038
|
+
});
|
|
1039
|
+
|
|
1040
|
+
// The credential names the API server, so the pod's cluster is moot.
|
|
1041
|
+
expect(status.gaps).toEqual([]);
|
|
1042
|
+
expect(status.accessMethod).toBe("credential");
|
|
1043
|
+
expect(status.credentialId).toBe(CREDENTIAL_ID.toString());
|
|
1044
|
+
});
|
|
1045
|
+
|
|
1046
|
+
it("negative control: THIS cluster's agent with a credential also selected stays in-cluster (the credential is ignored)", async () => {
|
|
1047
|
+
const credentialLookup: jest.SpyInstance = jest.spyOn(
|
|
1048
|
+
RunbookCredentialService,
|
|
1049
|
+
"findOneBy",
|
|
1050
|
+
);
|
|
1051
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
1052
|
+
|
|
1053
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1054
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1055
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
1056
|
+
gates: READY_GATES,
|
|
1057
|
+
});
|
|
1058
|
+
|
|
1059
|
+
expect(status.gaps).toEqual([]);
|
|
1060
|
+
expect(status.accessMethod).toBe("in_cluster");
|
|
1061
|
+
expect(status.credentialId).toBeUndefined();
|
|
1062
|
+
expect(credentialLookup).not.toHaveBeenCalled();
|
|
1063
|
+
});
|
|
1064
|
+
|
|
1065
|
+
it("still reports the credential gap, not a mismatch, when the bound credential is unusable", async () => {
|
|
1066
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
1067
|
+
fakeRunner({
|
|
1068
|
+
/*
|
|
1069
|
+
* A dashboard Runner living in a pod: an agent row here would be
|
|
1070
|
+
* the credential_on_agent_runner case above. Before round two
|
|
1071
|
+
* this fixture also named a cluster — an agent posture, which
|
|
1072
|
+
* only the kubernetes-agent binary reports and which now marks
|
|
1073
|
+
* the row as an agent on its own.
|
|
1074
|
+
*/
|
|
1075
|
+
name: "pod-runner",
|
|
1076
|
+
hostInfo: {
|
|
1077
|
+
kubernetes: {
|
|
1078
|
+
inCluster: true,
|
|
1079
|
+
allowWrites: true,
|
|
1080
|
+
},
|
|
1081
|
+
},
|
|
1082
|
+
}),
|
|
1083
|
+
);
|
|
1084
|
+
jest.spyOn(RunbookCredentialService, "findOneBy").mockResolvedValue(null);
|
|
1085
|
+
|
|
1086
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1087
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1088
|
+
cluster: fakeCluster({ aiAccessCredentialId: CREDENTIAL_ID }),
|
|
1089
|
+
gates: READY_GATES,
|
|
1090
|
+
});
|
|
1091
|
+
|
|
1092
|
+
expect(gapCodes(status)).toEqual(["credential_missing"]);
|
|
1093
|
+
expect(status.accessMethod).toBe("none");
|
|
1094
|
+
});
|
|
1095
|
+
|
|
1096
|
+
it("reports both the mismatch and the read-only gap for a read-only agent of another cluster", async () => {
|
|
1097
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
1098
|
+
fakeRunner({
|
|
1099
|
+
hostInfo: {
|
|
1100
|
+
kubernetes: {
|
|
1101
|
+
inCluster: true,
|
|
1102
|
+
allowWrites: false,
|
|
1103
|
+
clusterIdentifier: "prod-eu",
|
|
1104
|
+
},
|
|
1105
|
+
},
|
|
1106
|
+
}),
|
|
1107
|
+
);
|
|
1108
|
+
|
|
1109
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1110
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1111
|
+
cluster: fakeCluster(),
|
|
1112
|
+
gates: READY_GATES,
|
|
1113
|
+
});
|
|
1114
|
+
|
|
1115
|
+
expect(gapCodes(status)).toEqual([
|
|
1116
|
+
"runner_cluster_mismatch",
|
|
1117
|
+
"remediation_write_access_missing",
|
|
1118
|
+
]);
|
|
1119
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
1120
|
+
expect(status.isRemediationReady).toBe(false);
|
|
1121
|
+
});
|
|
1122
|
+
});
|
|
1123
|
+
|
|
1124
|
+
it("blocks only remediation when the in-cluster Runner is read-only", async () => {
|
|
1125
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
1126
|
+
fakeRunner({
|
|
1127
|
+
hostInfo: {
|
|
1128
|
+
kubernetes: {
|
|
1129
|
+
inCluster: true,
|
|
1130
|
+
allowWrites: false,
|
|
1131
|
+
clusterIdentifier: "prod-us",
|
|
1132
|
+
},
|
|
1133
|
+
},
|
|
1134
|
+
}),
|
|
1135
|
+
);
|
|
1136
|
+
|
|
1137
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1138
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1139
|
+
cluster: fakeCluster(),
|
|
1140
|
+
gates: READY_GATES,
|
|
1141
|
+
});
|
|
1142
|
+
|
|
1143
|
+
expect(gapCodes(status)).toEqual(["remediation_write_access_missing"]);
|
|
1144
|
+
expect(status.gaps[0]?.blocks).toBe("remediation");
|
|
1145
|
+
expect(status.gaps[0]?.nextStep).toContain(
|
|
1146
|
+
"aiAccess.remediation.enabled=true",
|
|
1147
|
+
);
|
|
1148
|
+
expect(status.gaps[0]?.nextStep).toBe(REMEDIATION_WRITE_ACCESS_NEXT_STEP);
|
|
1149
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
1150
|
+
expect(status.isRemediationReady).toBe(false);
|
|
1151
|
+
});
|
|
1152
|
+
|
|
1153
|
+
/*
|
|
1154
|
+
* Round three: the read-only gap used to give only a bare
|
|
1155
|
+
* "--set aiAccess.remediation.enabled=true", which grants write RBAC
|
|
1156
|
+
* cluster-wide and to every node. It now points at the AI page's
|
|
1157
|
+
* write-access section (the complete commands) and names the two values
|
|
1158
|
+
* that bound where the write role reaches.
|
|
1159
|
+
*/
|
|
1160
|
+
it("points the read-only gap at the AI page's write-access section and the values that scope it", () => {
|
|
1161
|
+
expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
|
|
1162
|
+
"--set aiAccess.remediation.enabled=true",
|
|
1163
|
+
);
|
|
1164
|
+
expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
|
|
1165
|
+
'"Let AI apply fixes (write access)" on the cluster\'s AI page',
|
|
1166
|
+
);
|
|
1167
|
+
expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
|
|
1168
|
+
"aiAccess.remediation.namespaces",
|
|
1169
|
+
);
|
|
1170
|
+
expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
|
|
1171
|
+
"aiAccess.remediation.nodeOperations=false",
|
|
1172
|
+
);
|
|
1173
|
+
expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain("cluster-wide");
|
|
1174
|
+
expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).not.toContain("this page");
|
|
1175
|
+
});
|
|
1176
|
+
|
|
1177
|
+
it("does not raise the read-only gap when remediation is off", async () => {
|
|
1178
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
1179
|
+
fakeRunner({
|
|
1180
|
+
hostInfo: {
|
|
1181
|
+
kubernetes: {
|
|
1182
|
+
inCluster: true,
|
|
1183
|
+
allowWrites: false,
|
|
1184
|
+
clusterIdentifier: "prod-us",
|
|
1185
|
+
},
|
|
1186
|
+
},
|
|
1187
|
+
}),
|
|
1188
|
+
);
|
|
1189
|
+
|
|
1190
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1191
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1192
|
+
cluster: fakeCluster({
|
|
1193
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
1194
|
+
}),
|
|
1195
|
+
gates: READY_GATES,
|
|
1196
|
+
});
|
|
1197
|
+
|
|
1198
|
+
expect(gapCodes(status)).toEqual(["remediation_disabled"]);
|
|
1199
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
1200
|
+
expect(status.isRemediationReady).toBe(false);
|
|
1201
|
+
});
|
|
1202
|
+
|
|
1203
|
+
it("keeps remediation ready when only investigation is switched off", async () => {
|
|
1204
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
1205
|
+
|
|
1206
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1207
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1208
|
+
cluster: fakeCluster({ isAiInvestigationEnabled: false }),
|
|
1209
|
+
gates: READY_GATES,
|
|
1210
|
+
});
|
|
1211
|
+
|
|
1212
|
+
expect(gapCodes(status)).toEqual(["investigation_disabled"]);
|
|
1213
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
1214
|
+
expect(status.isRemediationReady).toBe(true);
|
|
1215
|
+
});
|
|
1216
|
+
|
|
1217
|
+
it("turns every failed project gate into its own gap", async () => {
|
|
1218
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
1219
|
+
|
|
1220
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1221
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1222
|
+
cluster: fakeCluster(),
|
|
1223
|
+
gates: {
|
|
1224
|
+
isAiEnabled: false,
|
|
1225
|
+
isAutoRemediationEnabled: false,
|
|
1226
|
+
isAiCommandExecutionEnabled: false,
|
|
1227
|
+
hasLlmProvider: false,
|
|
1228
|
+
},
|
|
1229
|
+
});
|
|
1230
|
+
|
|
1231
|
+
expect(gapCodes(status)).toEqual([
|
|
1232
|
+
"project_ai_disabled",
|
|
1233
|
+
"llm_provider_missing",
|
|
1234
|
+
"project_auto_remediation_disabled",
|
|
1235
|
+
"project_ai_command_execution_disabled",
|
|
1236
|
+
]);
|
|
1237
|
+
expect(status.isInvestigationReady).toBe(false);
|
|
1238
|
+
expect(status.isRemediationReady).toBe(false);
|
|
1239
|
+
});
|
|
1240
|
+
|
|
1241
|
+
it("is remediation-ready in BypassApproval mode with a write-capable Runner and reports the mode verbatim", async () => {
|
|
1242
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
1243
|
+
|
|
1244
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1245
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1246
|
+
cluster: fakeCluster({
|
|
1247
|
+
aiRemediationMode: KubernetesAiRemediationMode.BypassApproval,
|
|
1248
|
+
}),
|
|
1249
|
+
gates: READY_GATES,
|
|
1250
|
+
});
|
|
1251
|
+
|
|
1252
|
+
expect(status.gaps).toEqual([]);
|
|
1253
|
+
expect(status.remediationMode).toBe(
|
|
1254
|
+
KubernetesAiRemediationMode.BypassApproval,
|
|
1255
|
+
);
|
|
1256
|
+
expect(status.isRemediationReady).toBe(true);
|
|
1257
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
1258
|
+
});
|
|
1259
|
+
|
|
1260
|
+
it("raises the read-only gap for a BypassApproval cluster whose Runner cannot write", async () => {
|
|
1261
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(
|
|
1262
|
+
fakeRunner({
|
|
1263
|
+
hostInfo: {
|
|
1264
|
+
kubernetes: {
|
|
1265
|
+
inCluster: true,
|
|
1266
|
+
allowWrites: false,
|
|
1267
|
+
clusterIdentifier: "prod-us",
|
|
1268
|
+
},
|
|
1269
|
+
},
|
|
1270
|
+
}),
|
|
1271
|
+
);
|
|
1272
|
+
|
|
1273
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1274
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1275
|
+
cluster: fakeCluster({
|
|
1276
|
+
aiRemediationMode: KubernetesAiRemediationMode.BypassApproval,
|
|
1277
|
+
}),
|
|
1278
|
+
gates: READY_GATES,
|
|
1279
|
+
});
|
|
1280
|
+
|
|
1281
|
+
expect(gapCodes(status)).toEqual(["remediation_write_access_missing"]);
|
|
1282
|
+
expect(status.remediationMode).toBe(
|
|
1283
|
+
KubernetesAiRemediationMode.BypassApproval,
|
|
1284
|
+
);
|
|
1285
|
+
expect(status.isRemediationReady).toBe(false);
|
|
1286
|
+
expect(status.isInvestigationReady).toBe(true);
|
|
1287
|
+
});
|
|
1288
|
+
|
|
1289
|
+
it("names every enabling mode in the remediation_disabled next step", async () => {
|
|
1290
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
1291
|
+
|
|
1292
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1293
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1294
|
+
cluster: fakeCluster({
|
|
1295
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
1296
|
+
}),
|
|
1297
|
+
gates: READY_GATES,
|
|
1298
|
+
});
|
|
1299
|
+
|
|
1300
|
+
expect(status.gaps[0]?.code).toBe("remediation_disabled");
|
|
1301
|
+
expect(status.gaps[0]?.nextStep).toContain("Ask for approval");
|
|
1302
|
+
expect(status.gaps[0]?.nextStep).toContain("Automatic");
|
|
1303
|
+
expect(status.gaps[0]?.nextStep).toContain("Bypass approval");
|
|
1304
|
+
});
|
|
1305
|
+
|
|
1306
|
+
it("normalizes a JSON-string allowlist and an unknown mode fails closed to Disabled", async () => {
|
|
1307
|
+
jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(fakeRunner());
|
|
1308
|
+
|
|
1309
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
1310
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
1311
|
+
cluster: fakeCluster({
|
|
1312
|
+
aiRemediationMode: "Yolo",
|
|
1313
|
+
aiKubectlCommandAllowlist: '["kubectl set image *", " "]',
|
|
1314
|
+
}),
|
|
1315
|
+
gates: READY_GATES,
|
|
1316
|
+
});
|
|
1317
|
+
|
|
1318
|
+
expect(status.remediationMode).toBe(KubernetesAiRemediationMode.Disabled);
|
|
1319
|
+
expect(status.kubectlAllowlist).toEqual(["kubectl set image *"]);
|
|
1320
|
+
});
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
describe("KubernetesClusterAiAccessService.normalizeRemediationMode", () => {
|
|
1324
|
+
it("accepts every declared mode verbatim", () => {
|
|
1325
|
+
for (const mode of Object.values(KubernetesAiRemediationMode)) {
|
|
1326
|
+
expect(
|
|
1327
|
+
KubernetesClusterAiAccessService.normalizeRemediationMode(mode),
|
|
1328
|
+
).toBe(mode);
|
|
1329
|
+
}
|
|
1330
|
+
});
|
|
1331
|
+
|
|
1332
|
+
it("fails closed to Disabled for anything else, including case variants", () => {
|
|
1333
|
+
for (const value of [
|
|
1334
|
+
undefined,
|
|
1335
|
+
null,
|
|
1336
|
+
"",
|
|
1337
|
+
"automatic",
|
|
1338
|
+
"bypassapproval",
|
|
1339
|
+
"Bypass",
|
|
1340
|
+
"BYPASSAPPROVAL",
|
|
1341
|
+
" BypassApproval",
|
|
1342
|
+
42,
|
|
1343
|
+
true,
|
|
1344
|
+
{},
|
|
1345
|
+
["BypassApproval"],
|
|
1346
|
+
]) {
|
|
1347
|
+
expect(
|
|
1348
|
+
KubernetesClusterAiAccessService.normalizeRemediationMode(value),
|
|
1349
|
+
).toBe(KubernetesAiRemediationMode.Disabled);
|
|
1350
|
+
}
|
|
1351
|
+
});
|
|
1352
|
+
});
|
|
1353
|
+
|
|
1354
|
+
describe("KubernetesClusterAiAccessService.getFirstBindRemediationMode", () => {
|
|
1355
|
+
it("fills in the chart's intent only when the mode is still at its default", () => {
|
|
1356
|
+
expect(
|
|
1357
|
+
KubernetesClusterAiAccessService.getFirstBindRemediationMode({
|
|
1358
|
+
currentMode: KubernetesAiRemediationMode.Disabled,
|
|
1359
|
+
allowWrites: true,
|
|
1360
|
+
}),
|
|
1361
|
+
).toBe(KubernetesAiRemediationMode.RequireApproval);
|
|
1362
|
+
expect(
|
|
1363
|
+
KubernetesClusterAiAccessService.getFirstBindRemediationMode({
|
|
1364
|
+
currentMode: KubernetesAiRemediationMode.Disabled,
|
|
1365
|
+
allowWrites: false,
|
|
1366
|
+
}),
|
|
1367
|
+
).toBe(KubernetesAiRemediationMode.Disabled);
|
|
1368
|
+
// A missing or unrecognised value is the default too.
|
|
1369
|
+
expect(
|
|
1370
|
+
KubernetesClusterAiAccessService.getFirstBindRemediationMode({
|
|
1371
|
+
currentMode: undefined,
|
|
1372
|
+
allowWrites: true,
|
|
1373
|
+
}),
|
|
1374
|
+
).toBe(KubernetesAiRemediationMode.RequireApproval);
|
|
1375
|
+
expect(
|
|
1376
|
+
KubernetesClusterAiAccessService.getFirstBindRemediationMode({
|
|
1377
|
+
currentMode: "bypassapproval",
|
|
1378
|
+
allowWrites: true,
|
|
1379
|
+
}),
|
|
1380
|
+
).toBe(KubernetesAiRemediationMode.RequireApproval);
|
|
1381
|
+
});
|
|
1382
|
+
|
|
1383
|
+
it("keeps every mode an operator chose, whatever the chart granted", () => {
|
|
1384
|
+
for (const chosen of [
|
|
1385
|
+
KubernetesAiRemediationMode.RequireApproval,
|
|
1386
|
+
KubernetesAiRemediationMode.Automatic,
|
|
1387
|
+
KubernetesAiRemediationMode.BypassApproval,
|
|
1388
|
+
]) {
|
|
1389
|
+
for (const allowWrites of [true, false]) {
|
|
1390
|
+
expect(
|
|
1391
|
+
KubernetesClusterAiAccessService.getFirstBindRemediationMode({
|
|
1392
|
+
currentMode: chosen,
|
|
1393
|
+
allowWrites,
|
|
1394
|
+
}),
|
|
1395
|
+
).toBe(chosen);
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
});
|
|
1399
|
+
});
|
|
1400
|
+
|
|
1401
|
+
describe("KubernetesClusterAiAccessService.getProjectGates", () => {
|
|
1402
|
+
afterEach(() => {
|
|
1403
|
+
jest.restoreAllMocks();
|
|
1404
|
+
});
|
|
1405
|
+
|
|
1406
|
+
it("reads kill switches as enabled-by-default and command execution as opt-in", async () => {
|
|
1407
|
+
jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
|
|
1408
|
+
enableAi: undefined,
|
|
1409
|
+
enableAutoRemediation: undefined,
|
|
1410
|
+
enableAiCommandExecution: undefined,
|
|
1411
|
+
} as unknown as Project);
|
|
1412
|
+
jest
|
|
1413
|
+
.spyOn(LlmProviderService, "getLLMProviderForProject")
|
|
1414
|
+
.mockResolvedValue({ id: ObjectID.generate() } as unknown as LlmProvider);
|
|
1415
|
+
|
|
1416
|
+
const gates: KubernetesClusterAiAccessProjectGates =
|
|
1417
|
+
await KubernetesClusterAiAccessService.getProjectGates(PROJECT_ID);
|
|
1418
|
+
|
|
1419
|
+
expect(gates).toEqual({
|
|
1420
|
+
isAiEnabled: true,
|
|
1421
|
+
isAutoRemediationEnabled: true,
|
|
1422
|
+
isAiCommandExecutionEnabled: false,
|
|
1423
|
+
hasLlmProvider: true,
|
|
1424
|
+
});
|
|
1425
|
+
});
|
|
1426
|
+
|
|
1427
|
+
it("treats a provider lookup failure as no provider", async () => {
|
|
1428
|
+
jest.spyOn(logger, "error").mockImplementation((): void => {
|
|
1429
|
+
return undefined;
|
|
1430
|
+
});
|
|
1431
|
+
jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
|
|
1432
|
+
enableAiCommandExecution: true,
|
|
1433
|
+
} as unknown as Project);
|
|
1434
|
+
jest
|
|
1435
|
+
.spyOn(LlmProviderService, "getLLMProviderForProject")
|
|
1436
|
+
.mockRejectedValue(new Error("boom"));
|
|
1437
|
+
|
|
1438
|
+
const gates: KubernetesClusterAiAccessProjectGates =
|
|
1439
|
+
await KubernetesClusterAiAccessService.getProjectGates(PROJECT_ID);
|
|
1440
|
+
|
|
1441
|
+
expect(gates.hasLlmProvider).toBe(false);
|
|
1442
|
+
expect(gates.isAiCommandExecutionEnabled).toBe(true);
|
|
1443
|
+
});
|
|
1444
|
+
});
|
|
1445
|
+
|
|
1446
|
+
describe("KubernetesClusterAiAccessService.registerKubernetesAgentRunner", () => {
|
|
1447
|
+
const OTHER_RUNNER_ID: ObjectID = new ObjectID(
|
|
1448
|
+
"66666666-6666-4666-8666-666666666666",
|
|
1449
|
+
);
|
|
1450
|
+
// Hex letters on purpose: the case-variant near-miss below must differ.
|
|
1451
|
+
const CURRENT_KEY: string = "9a9b9c9d-9e9f-4a9b-8c9d-9e9f9a9b9c9d";
|
|
1452
|
+
|
|
1453
|
+
let clusterUpdates: Array<Record<string, unknown>>;
|
|
1454
|
+
let runnerUpdates: Array<{ id: string; data: Record<string, unknown> }>;
|
|
1455
|
+
let createdRunner: Runner | null;
|
|
1456
|
+
let feedItems: Array<Record<string, unknown>>;
|
|
1457
|
+
let countBySpy: jest.SpyInstance;
|
|
1458
|
+
let warnSpy: jest.SpyInstance;
|
|
1459
|
+
// What an unproven re-key checks the row holds beyond the agent defaults.
|
|
1460
|
+
let credentialCountSpy: jest.SpyInstance;
|
|
1461
|
+
let secretCountSpy: jest.SpyInstance;
|
|
1462
|
+
let boundClusterCountSpy: jest.SpyInstance;
|
|
1463
|
+
|
|
1464
|
+
/*
|
|
1465
|
+
* An agent Runner row as the last heartbeat left it, OFFLINE: the pod it
|
|
1466
|
+
* belonged to stopped half an hour ago, which is the restart case that
|
|
1467
|
+
* must always be admitted. It holds exactly what registration gave it:
|
|
1468
|
+
* kubectl only, no runbooks, no code fixes.
|
|
1469
|
+
*/
|
|
1470
|
+
function offlineAgentRunner(
|
|
1471
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
1472
|
+
): Runner {
|
|
1473
|
+
return fakeRunner({
|
|
1474
|
+
key: CURRENT_KEY,
|
|
1475
|
+
lastAlive: OneUptimeDate.getSomeMinutesAgo(30),
|
|
1476
|
+
connectionStatus: RunnerConnectionStatus.Connected,
|
|
1477
|
+
canRunRunbooks: false,
|
|
1478
|
+
canRunCodeFixTasks: false,
|
|
1479
|
+
...overrides,
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
// The same row while its pod is still heartbeating.
|
|
1484
|
+
function onlineAgentRunner(
|
|
1485
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
1486
|
+
): Runner {
|
|
1487
|
+
return fakeRunner({
|
|
1488
|
+
key: CURRENT_KEY,
|
|
1489
|
+
lastAlive: OneUptimeDate.getCurrentDate(),
|
|
1490
|
+
connectionStatus: RunnerConnectionStatus.Connected,
|
|
1491
|
+
canRunRunbooks: false,
|
|
1492
|
+
canRunCodeFixTasks: false,
|
|
1493
|
+
...overrides,
|
|
1494
|
+
});
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
// The row after its pod signed off on a clean shutdown (a helm upgrade).
|
|
1498
|
+
function signedOffAgentRunner(
|
|
1499
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
1500
|
+
): Runner {
|
|
1501
|
+
return onlineAgentRunner({
|
|
1502
|
+
connectionStatus: RunnerConnectionStatus.Disconnected,
|
|
1503
|
+
...overrides,
|
|
1504
|
+
});
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
/*
|
|
1508
|
+
* Answers the two Runner lookups registration makes — by agent name, and
|
|
1509
|
+
* by bound id — separately, so a test can put a different row behind
|
|
1510
|
+
* each. Registration finds the row it re-keys by NAME only, so without an
|
|
1511
|
+
* explicit `byName` the name lookup answers with the bound row exactly
|
|
1512
|
+
* when that row carries the name being looked up (compared the way
|
|
1513
|
+
* findWithSameText compares), as Postgres would.
|
|
1514
|
+
*/
|
|
1515
|
+
function mockRunnerLookups(data: {
|
|
1516
|
+
bound?: Runner | null | undefined;
|
|
1517
|
+
byName?: Runner | null | undefined;
|
|
1518
|
+
}): jest.SpyInstance {
|
|
1519
|
+
return jest
|
|
1520
|
+
.spyOn(RunnerService, "findOneBy")
|
|
1521
|
+
.mockImplementation(async (args: unknown): Promise<Runner | null> => {
|
|
1522
|
+
const query: Record<string, unknown> = (
|
|
1523
|
+
args as { query: Record<string, unknown> }
|
|
1524
|
+
).query;
|
|
1525
|
+
|
|
1526
|
+
if (query["_id"]) {
|
|
1527
|
+
return data.bound ?? null;
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
if (data.byName !== undefined) {
|
|
1531
|
+
return data.byName;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
const nameFilter: {
|
|
1535
|
+
objectLiteralParameters?: Record<string, unknown>;
|
|
1536
|
+
} = query["name"] as {
|
|
1537
|
+
objectLiteralParameters?: Record<string, unknown>;
|
|
1538
|
+
};
|
|
1539
|
+
const lookedUp: unknown = Object.values(
|
|
1540
|
+
nameFilter?.objectLiteralParameters || {},
|
|
1541
|
+
)[0];
|
|
1542
|
+
|
|
1543
|
+
return data.bound &&
|
|
1544
|
+
typeof lookedUp === "string" &&
|
|
1545
|
+
(data.bound.name || "").trim().toLowerCase() === lookedUp
|
|
1546
|
+
? data.bound
|
|
1547
|
+
: null;
|
|
1548
|
+
});
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
beforeEach(() => {
|
|
1552
|
+
clusterUpdates = [];
|
|
1553
|
+
runnerUpdates = [];
|
|
1554
|
+
createdRunner = null;
|
|
1555
|
+
feedItems = [];
|
|
1556
|
+
|
|
1557
|
+
warnSpy = jest.spyOn(logger, "warn").mockImplementation((): void => {
|
|
1558
|
+
return undefined;
|
|
1559
|
+
});
|
|
1560
|
+
jest.spyOn(logger, "info").mockImplementation((): void => {
|
|
1561
|
+
return undefined;
|
|
1562
|
+
});
|
|
1563
|
+
jest.spyOn(logger, "error").mockImplementation((): void => {
|
|
1564
|
+
return undefined;
|
|
1565
|
+
});
|
|
1566
|
+
|
|
1567
|
+
jest
|
|
1568
|
+
.spyOn(KubernetesClusterService, "findOrCreateByClusterIdentifier")
|
|
1569
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
1570
|
+
jest
|
|
1571
|
+
.spyOn(KubernetesClusterService, "updateOneById")
|
|
1572
|
+
.mockImplementation(async (args: unknown): Promise<never> => {
|
|
1573
|
+
clusterUpdates.push((args as { data: Record<string, unknown> }).data);
|
|
1574
|
+
return undefined as never;
|
|
1575
|
+
});
|
|
1576
|
+
jest
|
|
1577
|
+
.spyOn(RunnerService, "updateOneById")
|
|
1578
|
+
.mockImplementation(async (args: unknown): Promise<never> => {
|
|
1579
|
+
const call: { id: ObjectID; data: Record<string, unknown> } = args as {
|
|
1580
|
+
id: ObjectID;
|
|
1581
|
+
data: Record<string, unknown>;
|
|
1582
|
+
};
|
|
1583
|
+
runnerUpdates.push({ id: call.id.toString(), data: call.data });
|
|
1584
|
+
return undefined as never;
|
|
1585
|
+
});
|
|
1586
|
+
jest
|
|
1587
|
+
.spyOn(RunnerService, "create")
|
|
1588
|
+
.mockImplementation(async (args: unknown): Promise<Runner> => {
|
|
1589
|
+
const data: Runner = (args as { data: Runner }).data;
|
|
1590
|
+
data.id = RUNNER_ID;
|
|
1591
|
+
createdRunner = data;
|
|
1592
|
+
return data;
|
|
1593
|
+
});
|
|
1594
|
+
countBySpy = jest
|
|
1595
|
+
.spyOn(RunnerService, "countBy")
|
|
1596
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
1597
|
+
credentialCountSpy = jest
|
|
1598
|
+
.spyOn(RunbookCredentialService, "countBy")
|
|
1599
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
1600
|
+
secretCountSpy = jest
|
|
1601
|
+
.spyOn(RunbookSecretService, "countBy")
|
|
1602
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
1603
|
+
boundClusterCountSpy = jest
|
|
1604
|
+
.spyOn(KubernetesClusterService, "countBy")
|
|
1605
|
+
.mockResolvedValue(new PositiveNumber(0));
|
|
1606
|
+
jest
|
|
1607
|
+
.spyOn(KubernetesClusterFeedService, "createKubernetesClusterFeedItem")
|
|
1608
|
+
.mockImplementation(async (args: unknown): Promise<void> => {
|
|
1609
|
+
feedItems.push(args as Record<string, unknown>);
|
|
1610
|
+
});
|
|
1611
|
+
});
|
|
1612
|
+
|
|
1613
|
+
afterEach(() => {
|
|
1614
|
+
jest.restoreAllMocks();
|
|
1615
|
+
});
|
|
1616
|
+
|
|
1617
|
+
it("rejects an empty cluster name", async () => {
|
|
1618
|
+
await expect(
|
|
1619
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1620
|
+
projectId: PROJECT_ID,
|
|
1621
|
+
clusterIdentifier: " ",
|
|
1622
|
+
posture: { allowWrites: true },
|
|
1623
|
+
}),
|
|
1624
|
+
).rejects.toThrow("clusterName is required.");
|
|
1625
|
+
});
|
|
1626
|
+
|
|
1627
|
+
describe("first bind of a never-configured cluster", () => {
|
|
1628
|
+
// A cluster row as ingest creates it: every AI switch at its default.
|
|
1629
|
+
function neverConfiguredCluster(
|
|
1630
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
1631
|
+
): KubernetesCluster {
|
|
1632
|
+
return fakeCluster({
|
|
1633
|
+
aiAccessRunnerId: undefined,
|
|
1634
|
+
isAiInvestigationEnabled: false,
|
|
1635
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
1636
|
+
...overrides,
|
|
1637
|
+
});
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
/*
|
|
1641
|
+
* The write scope the Runner reports at registration is stored at once,
|
|
1642
|
+
* so the enqueue chokepoint (and the AI page) know it before the first
|
|
1643
|
+
* heartbeat — and the service, not the body, says which cluster and that
|
|
1644
|
+
* it is in-cluster.
|
|
1645
|
+
*/
|
|
1646
|
+
it("stores the Runner's reported write scope with the posture, the cluster identity its own", async () => {
|
|
1647
|
+
jest
|
|
1648
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1649
|
+
.mockResolvedValue(neverConfiguredCluster());
|
|
1650
|
+
mockRunnerLookups({});
|
|
1651
|
+
|
|
1652
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1653
|
+
projectId: PROJECT_ID,
|
|
1654
|
+
clusterIdentifier: "prod-us",
|
|
1655
|
+
posture: {
|
|
1656
|
+
allowWrites: true,
|
|
1657
|
+
writeNamespaces: ["web"],
|
|
1658
|
+
podNamespace: "oneuptime-agent",
|
|
1659
|
+
allowNodeOperations: false,
|
|
1660
|
+
clusterIdentifier: "someone-else",
|
|
1661
|
+
inCluster: false,
|
|
1662
|
+
},
|
|
1663
|
+
});
|
|
1664
|
+
|
|
1665
|
+
expect(
|
|
1666
|
+
(createdRunner!.hostInfo as { kubernetes: Record<string, unknown> })
|
|
1667
|
+
.kubernetes,
|
|
1668
|
+
).toEqual({
|
|
1669
|
+
allowWrites: true,
|
|
1670
|
+
writeNamespaces: ["web"],
|
|
1671
|
+
podNamespace: "oneuptime-agent",
|
|
1672
|
+
allowNodeOperations: false,
|
|
1673
|
+
clusterIdentifier: "prod-us",
|
|
1674
|
+
inCluster: true,
|
|
1675
|
+
});
|
|
1676
|
+
});
|
|
1677
|
+
|
|
1678
|
+
it("creates a kubectl-only Runner, binds the cluster, turns investigation on and remediation to ask-for-approval", async () => {
|
|
1679
|
+
jest
|
|
1680
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1681
|
+
.mockResolvedValue(neverConfiguredCluster());
|
|
1682
|
+
mockRunnerLookups({});
|
|
1683
|
+
|
|
1684
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
1685
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1686
|
+
projectId: PROJECT_ID,
|
|
1687
|
+
clusterIdentifier: "prod-us",
|
|
1688
|
+
agentVersion: "9.1.0",
|
|
1689
|
+
posture: { allowWrites: true, kubectlVersion: "v1.31.4" },
|
|
1690
|
+
});
|
|
1691
|
+
|
|
1692
|
+
expect(result.isFirstBind).toBe(true);
|
|
1693
|
+
expect(result.isBoundToCluster).toBe(true);
|
|
1694
|
+
expect(result.bindingState).toBe("first_bind");
|
|
1695
|
+
expect(result.runnerId.toString()).toBe(RUNNER_ID.toString());
|
|
1696
|
+
expect(result.runnerKey).toHaveLength(36);
|
|
1697
|
+
|
|
1698
|
+
expect(createdRunner).not.toBeNull();
|
|
1699
|
+
expect(createdRunner!.name).toBe(getKubernetesAgentRunnerName("prod-us"));
|
|
1700
|
+
expect(createdRunner!.canRunAiCommands).toBe(true);
|
|
1701
|
+
expect(createdRunner!.canRunRunbooks).toBe(false);
|
|
1702
|
+
expect(createdRunner!.canRunCodeFixTasks).toBe(false);
|
|
1703
|
+
expect(createdRunner!.key).toBe(result.runnerKey);
|
|
1704
|
+
expect(
|
|
1705
|
+
(createdRunner!.hostInfo as { kubernetes: Record<string, unknown> })
|
|
1706
|
+
.kubernetes,
|
|
1707
|
+
).toMatchObject({
|
|
1708
|
+
inCluster: true,
|
|
1709
|
+
allowWrites: true,
|
|
1710
|
+
clusterIdentifier: "prod-us",
|
|
1711
|
+
kubectlVersion: "v1.31.4",
|
|
1712
|
+
});
|
|
1713
|
+
|
|
1714
|
+
expect(clusterUpdates).toHaveLength(1);
|
|
1715
|
+
expect(clusterUpdates[0]).toMatchObject({
|
|
1716
|
+
aiAccessRunnerId: RUNNER_ID,
|
|
1717
|
+
aiAccessCredentialId: null,
|
|
1718
|
+
isAiInvestigationEnabled: true,
|
|
1719
|
+
aiRemediationMode: KubernetesAiRemediationMode.RequireApproval,
|
|
1720
|
+
});
|
|
1721
|
+
/*
|
|
1722
|
+
* The first bind stamps the never-cleared "configured" marker, so a
|
|
1723
|
+
* later Runner delete (the FK nulls the binding) can never make this
|
|
1724
|
+
* cluster look never-configured and first-bind it again.
|
|
1725
|
+
*/
|
|
1726
|
+
expect(clusterUpdates[0]!["aiAccessConfiguredAt"]).toBeInstanceOf(Date);
|
|
1727
|
+
|
|
1728
|
+
// Recorded on the cluster feed so an operator can see it happened.
|
|
1729
|
+
expect(feedItems).toHaveLength(1);
|
|
1730
|
+
expect(feedItems[0]!["kubernetesClusterId"]).toBe(CLUSTER_ID);
|
|
1731
|
+
expect(feedItems[0]!["projectId"]).toBe(PROJECT_ID);
|
|
1732
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
1733
|
+
"was bound as this cluster's AI access Runner",
|
|
1734
|
+
);
|
|
1735
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
1736
|
+
'AI remediation is "Ask for approval"',
|
|
1737
|
+
);
|
|
1738
|
+
});
|
|
1739
|
+
|
|
1740
|
+
it("without write RBAC leaves remediation Disabled", async () => {
|
|
1741
|
+
jest
|
|
1742
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1743
|
+
.mockResolvedValue(neverConfiguredCluster());
|
|
1744
|
+
mockRunnerLookups({});
|
|
1745
|
+
|
|
1746
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1747
|
+
projectId: PROJECT_ID,
|
|
1748
|
+
clusterIdentifier: "prod-us",
|
|
1749
|
+
posture: { allowWrites: false },
|
|
1750
|
+
});
|
|
1751
|
+
|
|
1752
|
+
expect(clusterUpdates[0]).toMatchObject({
|
|
1753
|
+
isAiInvestigationEnabled: true,
|
|
1754
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
1755
|
+
});
|
|
1756
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
1757
|
+
"AI remediation is Disabled",
|
|
1758
|
+
);
|
|
1759
|
+
});
|
|
1760
|
+
|
|
1761
|
+
/*
|
|
1762
|
+
* The dashboard is the control plane even before a Runner exists: an
|
|
1763
|
+
* operator may set the mode on the AI page and install the chart
|
|
1764
|
+
* afterwards. The chart's intent only fills in a switch still at its
|
|
1765
|
+
* default; it never downgrades a chosen mode to "ask for approval",
|
|
1766
|
+
* and never lifts one either.
|
|
1767
|
+
*/
|
|
1768
|
+
it("keeps a remediation mode the operator chose before the Runner registered", async () => {
|
|
1769
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
1770
|
+
neverConfiguredCluster({
|
|
1771
|
+
aiRemediationMode: KubernetesAiRemediationMode.Automatic,
|
|
1772
|
+
}),
|
|
1773
|
+
);
|
|
1774
|
+
mockRunnerLookups({});
|
|
1775
|
+
|
|
1776
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
1777
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1778
|
+
projectId: PROJECT_ID,
|
|
1779
|
+
clusterIdentifier: "prod-us",
|
|
1780
|
+
posture: { allowWrites: true },
|
|
1781
|
+
});
|
|
1782
|
+
|
|
1783
|
+
expect(result.bindingState).toBe("first_bind");
|
|
1784
|
+
expect(clusterUpdates).toHaveLength(1);
|
|
1785
|
+
expect(clusterUpdates[0]).toMatchObject({
|
|
1786
|
+
aiAccessRunnerId: RUNNER_ID,
|
|
1787
|
+
isAiInvestigationEnabled: true,
|
|
1788
|
+
aiRemediationMode: KubernetesAiRemediationMode.Automatic,
|
|
1789
|
+
});
|
|
1790
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
1791
|
+
'AI remediation is "Automatic"',
|
|
1792
|
+
);
|
|
1793
|
+
});
|
|
1794
|
+
|
|
1795
|
+
it("keeps a chosen write mode on a read-only chart too (the readiness gap handles it, not a silent downgrade)", async () => {
|
|
1796
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
1797
|
+
neverConfiguredCluster({
|
|
1798
|
+
aiRemediationMode: KubernetesAiRemediationMode.BypassApproval,
|
|
1799
|
+
}),
|
|
1800
|
+
);
|
|
1801
|
+
mockRunnerLookups({});
|
|
1802
|
+
|
|
1803
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1804
|
+
projectId: PROJECT_ID,
|
|
1805
|
+
clusterIdentifier: "prod-us",
|
|
1806
|
+
posture: { allowWrites: false },
|
|
1807
|
+
});
|
|
1808
|
+
|
|
1809
|
+
expect(clusterUpdates[0]).toMatchObject({
|
|
1810
|
+
aiRemediationMode: KubernetesAiRemediationMode.BypassApproval,
|
|
1811
|
+
});
|
|
1812
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
1813
|
+
'AI remediation is "Bypass approval"',
|
|
1814
|
+
);
|
|
1815
|
+
});
|
|
1816
|
+
|
|
1817
|
+
it("treats an unknown stored mode as the default and applies the chart's intent", async () => {
|
|
1818
|
+
jest
|
|
1819
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1820
|
+
.mockResolvedValue(
|
|
1821
|
+
neverConfiguredCluster({ aiRemediationMode: "Yolo" }),
|
|
1822
|
+
);
|
|
1823
|
+
mockRunnerLookups({});
|
|
1824
|
+
|
|
1825
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1826
|
+
projectId: PROJECT_ID,
|
|
1827
|
+
clusterIdentifier: "prod-us",
|
|
1828
|
+
posture: { allowWrites: true },
|
|
1829
|
+
});
|
|
1830
|
+
|
|
1831
|
+
expect(clusterUpdates[0]).toMatchObject({
|
|
1832
|
+
aiRemediationMode: KubernetesAiRemediationMode.RequireApproval,
|
|
1833
|
+
});
|
|
1834
|
+
});
|
|
1835
|
+
|
|
1836
|
+
it("looks the agent Runner row up by name case-insensitively", async () => {
|
|
1837
|
+
jest
|
|
1838
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1839
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
1840
|
+
const findOneBy: jest.SpyInstance = mockRunnerLookups({});
|
|
1841
|
+
|
|
1842
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1843
|
+
projectId: PROJECT_ID,
|
|
1844
|
+
clusterIdentifier: "Prod-US",
|
|
1845
|
+
posture: { allowWrites: true },
|
|
1846
|
+
});
|
|
1847
|
+
|
|
1848
|
+
const nameQuery: Record<string, unknown> = (
|
|
1849
|
+
findOneBy.mock.calls[0]![0] as { query: Record<string, unknown> }
|
|
1850
|
+
).query;
|
|
1851
|
+
|
|
1852
|
+
expect(nameQuery["projectId"]).toBe(PROJECT_ID);
|
|
1853
|
+
/*
|
|
1854
|
+
* A findWithSameText operator, not the raw string: a chart that
|
|
1855
|
+
* registers "Prod-US" must find the row a "prod-us" registration made
|
|
1856
|
+
* instead of tripping the unique-name guard on a second create.
|
|
1857
|
+
*/
|
|
1858
|
+
expect(typeof nameQuery["name"]).not.toBe("string");
|
|
1859
|
+
expect(nameQuery["name"]).toBeDefined();
|
|
1860
|
+
});
|
|
1861
|
+
});
|
|
1862
|
+
|
|
1863
|
+
describe("re-registration of this cluster's agent (a pod restart)", () => {
|
|
1864
|
+
it("rotates the key and posture of the OFFLINE bound agent Runner but never touches the operator's switches", async () => {
|
|
1865
|
+
jest
|
|
1866
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1867
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
1868
|
+
mockRunnerLookups({ bound: offlineAgentRunner() });
|
|
1869
|
+
|
|
1870
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
1871
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1872
|
+
projectId: PROJECT_ID,
|
|
1873
|
+
clusterIdentifier: "prod-us",
|
|
1874
|
+
posture: { allowWrites: false },
|
|
1875
|
+
});
|
|
1876
|
+
|
|
1877
|
+
expect(result.isFirstBind).toBe(false);
|
|
1878
|
+
expect(result.isBoundToCluster).toBe(true);
|
|
1879
|
+
expect(result.bindingState).toBe("already_bound");
|
|
1880
|
+
expect(createdRunner).toBeNull();
|
|
1881
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
1882
|
+
expect(runnerUpdates[0]!.id).toBe(RUNNER_ID.toString());
|
|
1883
|
+
expect(runnerUpdates[0]!.data["key"]).toBe(result.runnerKey);
|
|
1884
|
+
expect(result.runnerKey).not.toBe(CURRENT_KEY);
|
|
1885
|
+
expect(
|
|
1886
|
+
(
|
|
1887
|
+
runnerUpdates[0]!.data["hostInfo"] as {
|
|
1888
|
+
kubernetes: { allowWrites: boolean; clusterIdentifier: string };
|
|
1889
|
+
}
|
|
1890
|
+
).kubernetes,
|
|
1891
|
+
).toMatchObject({ allowWrites: false, clusterIdentifier: "prod-us" });
|
|
1892
|
+
// No cluster write at all: the dashboard is the control plane.
|
|
1893
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
1894
|
+
// Creation brakes are for new rows only.
|
|
1895
|
+
expect(countBySpy).not.toHaveBeenCalled();
|
|
1896
|
+
const feed: string = String(feedItems[0]!["feedInfoInMarkdown"]);
|
|
1897
|
+
expect(feed).toContain("key was rotated");
|
|
1898
|
+
// Said plainly: no key was presented, and why it was admitted anyway.
|
|
1899
|
+
expect(feed).toContain("WITHOUT proof of continuity");
|
|
1900
|
+
expect(feed).toContain("had stopped heartbeating");
|
|
1901
|
+
});
|
|
1902
|
+
|
|
1903
|
+
/*
|
|
1904
|
+
* Round three (follow-up e): the write scope a restarted pod reports is
|
|
1905
|
+
* stored on the re-keyed row at once, replacing what the previous pod
|
|
1906
|
+
* reported, so a scope changed by a helm upgrade is known before the
|
|
1907
|
+
* first heartbeat. The service, not the body, still says which cluster.
|
|
1908
|
+
*/
|
|
1909
|
+
it("stores the restarted Runner's reported write scope on the re-keyed row", async () => {
|
|
1910
|
+
jest
|
|
1911
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1912
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
1913
|
+
mockRunnerLookups({
|
|
1914
|
+
bound: offlineAgentRunner({
|
|
1915
|
+
hostInfo: {
|
|
1916
|
+
kubernetes: {
|
|
1917
|
+
inCluster: true,
|
|
1918
|
+
allowWrites: true,
|
|
1919
|
+
clusterIdentifier: "prod-us",
|
|
1920
|
+
writeNamespaces: ["legacy"],
|
|
1921
|
+
allowNodeOperations: true,
|
|
1922
|
+
},
|
|
1923
|
+
},
|
|
1924
|
+
}),
|
|
1925
|
+
});
|
|
1926
|
+
|
|
1927
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1928
|
+
projectId: PROJECT_ID,
|
|
1929
|
+
clusterIdentifier: "prod-us",
|
|
1930
|
+
posture: {
|
|
1931
|
+
allowWrites: true,
|
|
1932
|
+
writeNamespaces: ["web", "api"],
|
|
1933
|
+
podNamespace: "oneuptime-agent",
|
|
1934
|
+
allowNodeOperations: false,
|
|
1935
|
+
clusterIdentifier: "someone-else",
|
|
1936
|
+
inCluster: false,
|
|
1937
|
+
},
|
|
1938
|
+
});
|
|
1939
|
+
|
|
1940
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
1941
|
+
expect(
|
|
1942
|
+
(
|
|
1943
|
+
runnerUpdates[0]!.data["hostInfo"] as {
|
|
1944
|
+
kubernetes: Record<string, unknown>;
|
|
1945
|
+
}
|
|
1946
|
+
).kubernetes,
|
|
1947
|
+
).toEqual({
|
|
1948
|
+
allowWrites: true,
|
|
1949
|
+
writeNamespaces: ["web", "api"],
|
|
1950
|
+
podNamespace: "oneuptime-agent",
|
|
1951
|
+
allowNodeOperations: false,
|
|
1952
|
+
clusterIdentifier: "prod-us",
|
|
1953
|
+
inCluster: true,
|
|
1954
|
+
});
|
|
1955
|
+
});
|
|
1956
|
+
|
|
1957
|
+
/*
|
|
1958
|
+
* Round four, defence in depth: an agent image older than the write
|
|
1959
|
+
* scope's registration fields registers with neither writeNamespaces
|
|
1960
|
+
* nor podNamespace (only its heartbeat sends them). Storing that body
|
|
1961
|
+
* as it is emptied the scope the row held, so until the first
|
|
1962
|
+
* heartbeat every server-side check read the Runner as cluster-wide
|
|
1963
|
+
* with no namespace of its own. A field the body leaves out keeps what
|
|
1964
|
+
* the row last reported; a field it sends wins.
|
|
1965
|
+
*/
|
|
1966
|
+
describe("a registration body that leaves out the write scope (an older agent image)", () => {
|
|
1967
|
+
const STORED_SCOPE: Record<string, unknown> = {
|
|
1968
|
+
inCluster: true,
|
|
1969
|
+
allowWrites: true,
|
|
1970
|
+
clusterIdentifier: "prod-us",
|
|
1971
|
+
writeNamespaces: ["web", "api"],
|
|
1972
|
+
podNamespace: "oneuptime-agent",
|
|
1973
|
+
allowNodeOperations: true,
|
|
1974
|
+
};
|
|
1975
|
+
|
|
1976
|
+
async function storedAfterRegistering(
|
|
1977
|
+
posture: KubernetesRunnerPosture,
|
|
1978
|
+
): Promise<Record<string, unknown>> {
|
|
1979
|
+
jest
|
|
1980
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
1981
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
1982
|
+
mockRunnerLookups({
|
|
1983
|
+
bound: offlineAgentRunner({
|
|
1984
|
+
hostInfo: { kubernetes: { ...STORED_SCOPE } },
|
|
1985
|
+
}),
|
|
1986
|
+
});
|
|
1987
|
+
|
|
1988
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
1989
|
+
projectId: PROJECT_ID,
|
|
1990
|
+
clusterIdentifier: "prod-us",
|
|
1991
|
+
posture,
|
|
1992
|
+
});
|
|
1993
|
+
|
|
1994
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
1995
|
+
|
|
1996
|
+
return (
|
|
1997
|
+
runnerUpdates[0]!.data["hostInfo"] as {
|
|
1998
|
+
kubernetes: Record<string, unknown>;
|
|
1999
|
+
}
|
|
2000
|
+
).kubernetes;
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
it("keeps the write namespaces and the pod namespace the row holds", async () => {
|
|
2004
|
+
const stored: Record<string, unknown> = await storedAfterRegistering({
|
|
2005
|
+
allowWrites: true,
|
|
2006
|
+
allowNodeOperations: false,
|
|
2007
|
+
writeNamespaces: undefined,
|
|
2008
|
+
podNamespace: undefined,
|
|
2009
|
+
});
|
|
2010
|
+
|
|
2011
|
+
expect(stored["writeNamespaces"]).toEqual(["web", "api"]);
|
|
2012
|
+
expect(stored["podNamespace"]).toBe("oneuptime-agent");
|
|
2013
|
+
// What the body did send still replaces what the row held.
|
|
2014
|
+
expect(stored["allowNodeOperations"]).toBe(false);
|
|
2015
|
+
expect(stored["clusterIdentifier"]).toBe("prod-us");
|
|
2016
|
+
expect(stored["inCluster"]).toBe(true);
|
|
2017
|
+
});
|
|
2018
|
+
|
|
2019
|
+
it("keeps each field it leaves out on its own", async () => {
|
|
2020
|
+
const stored: Record<string, unknown> = await storedAfterRegistering({
|
|
2021
|
+
allowWrites: true,
|
|
2022
|
+
writeNamespaces: ["payments"],
|
|
2023
|
+
});
|
|
2024
|
+
|
|
2025
|
+
expect(stored["writeNamespaces"]).toEqual(["payments"]);
|
|
2026
|
+
expect(stored["podNamespace"]).toBe("oneuptime-agent");
|
|
2027
|
+
});
|
|
2028
|
+
|
|
2029
|
+
// Negative control: an explicit empty list is the chart's cluster-wide scope.
|
|
2030
|
+
it("negative control: a body that sends an empty list stores cluster-wide", async () => {
|
|
2031
|
+
const stored: Record<string, unknown> = await storedAfterRegistering({
|
|
2032
|
+
allowWrites: true,
|
|
2033
|
+
writeNamespaces: [],
|
|
2034
|
+
podNamespace: "oneuptime-agent",
|
|
2035
|
+
});
|
|
2036
|
+
|
|
2037
|
+
expect(stored["writeNamespaces"]).toEqual([]);
|
|
2038
|
+
expect(stored["podNamespace"]).toBe("oneuptime-agent");
|
|
2039
|
+
});
|
|
2040
|
+
|
|
2041
|
+
it("negative control: a first registration has nothing to keep", async () => {
|
|
2042
|
+
jest
|
|
2043
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2044
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
2045
|
+
mockRunnerLookups({});
|
|
2046
|
+
|
|
2047
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2048
|
+
projectId: PROJECT_ID,
|
|
2049
|
+
clusterIdentifier: "prod-us",
|
|
2050
|
+
posture: { allowWrites: true },
|
|
2051
|
+
});
|
|
2052
|
+
|
|
2053
|
+
const stored: Record<string, unknown> = (
|
|
2054
|
+
createdRunner!.hostInfo as { kubernetes: Record<string, unknown> }
|
|
2055
|
+
).kubernetes;
|
|
2056
|
+
|
|
2057
|
+
expect(stored["writeNamespaces"]).toBeUndefined();
|
|
2058
|
+
expect(stored["podNamespace"]).toBeUndefined();
|
|
2059
|
+
});
|
|
2060
|
+
|
|
2061
|
+
it("keepStoredWriteScope leaves a posture alone when the row reported none", () => {
|
|
2062
|
+
const reported: KubernetesRunnerPosture = {
|
|
2063
|
+
allowWrites: true,
|
|
2064
|
+
inCluster: true,
|
|
2065
|
+
};
|
|
2066
|
+
|
|
2067
|
+
expect(
|
|
2068
|
+
KubernetesClusterAiAccessService.keepStoredWriteScope({
|
|
2069
|
+
reported,
|
|
2070
|
+
stored: undefined,
|
|
2071
|
+
}),
|
|
2072
|
+
).toEqual(reported);
|
|
2073
|
+
expect(
|
|
2074
|
+
KubernetesClusterAiAccessService.keepStoredWriteScope({
|
|
2075
|
+
reported,
|
|
2076
|
+
stored: { inCluster: true, podNamespace: undefined },
|
|
2077
|
+
}),
|
|
2078
|
+
).toEqual(reported);
|
|
2079
|
+
});
|
|
2080
|
+
});
|
|
2081
|
+
|
|
2082
|
+
it("also admits a Runner that signed off (Disconnected) even though its last heartbeat is recent", async () => {
|
|
2083
|
+
jest
|
|
2084
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2085
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2086
|
+
mockRunnerLookups({
|
|
2087
|
+
bound: signedOffAgentRunner(),
|
|
2088
|
+
});
|
|
2089
|
+
|
|
2090
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2091
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2092
|
+
projectId: PROJECT_ID,
|
|
2093
|
+
clusterIdentifier: "prod-us",
|
|
2094
|
+
posture: { allowWrites: true },
|
|
2095
|
+
});
|
|
2096
|
+
|
|
2097
|
+
expect(result.bindingState).toBe("already_bound");
|
|
2098
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2099
|
+
expect(runnerUpdates[0]!.data["connectionStatus"]).toBe(
|
|
2100
|
+
RunnerConnectionStatus.Connected,
|
|
2101
|
+
);
|
|
2102
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
2103
|
+
"the previous instance had signed off",
|
|
2104
|
+
);
|
|
2105
|
+
});
|
|
2106
|
+
|
|
2107
|
+
it("says a rotation WITH the current key was proven, and without the takeover warning", async () => {
|
|
2108
|
+
jest
|
|
2109
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2110
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2111
|
+
mockRunnerLookups({ bound: offlineAgentRunner() });
|
|
2112
|
+
|
|
2113
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2114
|
+
projectId: PROJECT_ID,
|
|
2115
|
+
clusterIdentifier: "prod-us",
|
|
2116
|
+
posture: { allowWrites: true },
|
|
2117
|
+
previousRunnerKey: CURRENT_KEY,
|
|
2118
|
+
});
|
|
2119
|
+
|
|
2120
|
+
const feed: string = String(feedItems[0]!["feedInfoInMarkdown"]);
|
|
2121
|
+
expect(feed).toContain("It presented its current key");
|
|
2122
|
+
expect(feed).not.toContain("WITHOUT proof");
|
|
2123
|
+
expect(feed).not.toContain("telemetry ingestion key");
|
|
2124
|
+
});
|
|
2125
|
+
|
|
2126
|
+
it("warns about a possible takeover on a rotation WITHOUT proof, in a different color", async () => {
|
|
2127
|
+
jest
|
|
2128
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2129
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2130
|
+
mockRunnerLookups({ bound: offlineAgentRunner() });
|
|
2131
|
+
|
|
2132
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2133
|
+
projectId: PROJECT_ID,
|
|
2134
|
+
clusterIdentifier: "prod-us",
|
|
2135
|
+
posture: { allowWrites: true },
|
|
2136
|
+
});
|
|
2137
|
+
|
|
2138
|
+
const unproven: Record<string, unknown> = feedItems[0]!;
|
|
2139
|
+
expect(String(unproven["feedInfoInMarkdown"])).toContain(
|
|
2140
|
+
"telemetry ingestion key",
|
|
2141
|
+
);
|
|
2142
|
+
|
|
2143
|
+
feedItems.length = 0;
|
|
2144
|
+
mockRunnerLookups({ bound: offlineAgentRunner() });
|
|
2145
|
+
|
|
2146
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2147
|
+
projectId: PROJECT_ID,
|
|
2148
|
+
clusterIdentifier: "prod-us",
|
|
2149
|
+
posture: { allowWrites: true },
|
|
2150
|
+
previousRunnerKey: CURRENT_KEY,
|
|
2151
|
+
});
|
|
2152
|
+
|
|
2153
|
+
expect(feedItems[0]!["displayColor"]).not.toEqual(
|
|
2154
|
+
unproven["displayColor"],
|
|
2155
|
+
);
|
|
2156
|
+
});
|
|
2157
|
+
|
|
2158
|
+
it("admits a Runner that never heartbeated at all", async () => {
|
|
2159
|
+
jest
|
|
2160
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2161
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2162
|
+
mockRunnerLookups({
|
|
2163
|
+
bound: offlineAgentRunner({ lastAlive: undefined }),
|
|
2164
|
+
});
|
|
2165
|
+
|
|
2166
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2167
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2168
|
+
projectId: PROJECT_ID,
|
|
2169
|
+
clusterIdentifier: "prod-us",
|
|
2170
|
+
posture: { allowWrites: true },
|
|
2171
|
+
});
|
|
2172
|
+
|
|
2173
|
+
expect(result.bindingState).toBe("already_bound");
|
|
2174
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2175
|
+
});
|
|
2176
|
+
});
|
|
2177
|
+
|
|
2178
|
+
/*
|
|
2179
|
+
* The ingestion key is held by every collector and CI job in the project.
|
|
2180
|
+
* With it alone, nobody may evict the live pod and take its identity —
|
|
2181
|
+
* the identity every kubectl job for that cluster is targeted at.
|
|
2182
|
+
*/
|
|
2183
|
+
describe("re-keying a Runner that is ONLINE needs proof of continuity", () => {
|
|
2184
|
+
it("refuses with 403, rotates nothing, writes nothing and never discloses the current key", async () => {
|
|
2185
|
+
jest
|
|
2186
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2187
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2188
|
+
mockRunnerLookups({ bound: onlineAgentRunner() });
|
|
2189
|
+
|
|
2190
|
+
let thrown: unknown = null;
|
|
2191
|
+
|
|
2192
|
+
try {
|
|
2193
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2194
|
+
projectId: PROJECT_ID,
|
|
2195
|
+
clusterIdentifier: "prod-us",
|
|
2196
|
+
posture: { allowWrites: true },
|
|
2197
|
+
});
|
|
2198
|
+
} catch (error) {
|
|
2199
|
+
thrown = error;
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
expect(thrown).toBeInstanceOf(ForbiddenException);
|
|
2203
|
+
const message: string = (thrown as Error).message;
|
|
2204
|
+
expect(message).toContain("is online");
|
|
2205
|
+
expect(message).toContain("previousRunnerKey");
|
|
2206
|
+
expect(message).not.toContain(CURRENT_KEY);
|
|
2207
|
+
|
|
2208
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
2209
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2210
|
+
expect(createdRunner).toBeNull();
|
|
2211
|
+
expect(feedItems).toHaveLength(0);
|
|
2212
|
+
|
|
2213
|
+
// Logged for the operator, without the key.
|
|
2214
|
+
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
2215
|
+
expect(String(warnSpy.mock.calls[0]![0])).toContain("refused to re-key");
|
|
2216
|
+
expect(String(warnSpy.mock.calls[0]![0])).not.toContain(CURRENT_KEY);
|
|
2217
|
+
});
|
|
2218
|
+
|
|
2219
|
+
it("admits a registration that presents the Runner's current key", async () => {
|
|
2220
|
+
jest
|
|
2221
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2222
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2223
|
+
mockRunnerLookups({ bound: onlineAgentRunner() });
|
|
2224
|
+
|
|
2225
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2226
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2227
|
+
projectId: PROJECT_ID,
|
|
2228
|
+
clusterIdentifier: "prod-us",
|
|
2229
|
+
posture: { allowWrites: true },
|
|
2230
|
+
previousRunnerKey: CURRENT_KEY,
|
|
2231
|
+
});
|
|
2232
|
+
|
|
2233
|
+
expect(result.bindingState).toBe("already_bound");
|
|
2234
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2235
|
+
expect(runnerUpdates[0]!.data["key"]).toBe(result.runnerKey);
|
|
2236
|
+
expect(result.runnerKey).not.toBe(CURRENT_KEY);
|
|
2237
|
+
expect(warnSpy).not.toHaveBeenCalled();
|
|
2238
|
+
});
|
|
2239
|
+
|
|
2240
|
+
it("refuses a wrong, a near-miss and an empty previousRunnerKey alike", async () => {
|
|
2241
|
+
jest
|
|
2242
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2243
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2244
|
+
mockRunnerLookups({ bound: onlineAgentRunner() });
|
|
2245
|
+
|
|
2246
|
+
for (const presented of [
|
|
2247
|
+
"00000000-0000-4000-8000-000000000000",
|
|
2248
|
+
CURRENT_KEY.slice(0, -1),
|
|
2249
|
+
`${CURRENT_KEY}x`,
|
|
2250
|
+
CURRENT_KEY.toUpperCase(),
|
|
2251
|
+
"",
|
|
2252
|
+
]) {
|
|
2253
|
+
await expect(
|
|
2254
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2255
|
+
projectId: PROJECT_ID,
|
|
2256
|
+
clusterIdentifier: "prod-us",
|
|
2257
|
+
posture: { allowWrites: true },
|
|
2258
|
+
previousRunnerKey: presented,
|
|
2259
|
+
}),
|
|
2260
|
+
).rejects.toThrow(ForbiddenException);
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
2264
|
+
});
|
|
2265
|
+
|
|
2266
|
+
it("applies the same rule to the by-name agent row of an unbound cluster", async () => {
|
|
2267
|
+
jest
|
|
2268
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2269
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
2270
|
+
mockRunnerLookups({ byName: onlineAgentRunner() });
|
|
2271
|
+
|
|
2272
|
+
await expect(
|
|
2273
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2274
|
+
projectId: PROJECT_ID,
|
|
2275
|
+
clusterIdentifier: "prod-us",
|
|
2276
|
+
posture: { allowWrites: true },
|
|
2277
|
+
}),
|
|
2278
|
+
).rejects.toThrow(ForbiddenException);
|
|
2279
|
+
|
|
2280
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
2281
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2282
|
+
});
|
|
2283
|
+
});
|
|
2284
|
+
|
|
2285
|
+
describe("bounds on what an ingestion key can mint", () => {
|
|
2286
|
+
it("refuses a new agent Runner once the project holds the maximum, with 429", async () => {
|
|
2287
|
+
jest
|
|
2288
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2289
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
2290
|
+
mockRunnerLookups({});
|
|
2291
|
+
countBySpy.mockResolvedValue(
|
|
2292
|
+
new PositiveNumber(MAX_KUBERNETES_AGENT_RUNNERS_PER_PROJECT),
|
|
2293
|
+
);
|
|
2294
|
+
|
|
2295
|
+
await expect(
|
|
2296
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2297
|
+
projectId: PROJECT_ID,
|
|
2298
|
+
clusterIdentifier: "prod-us",
|
|
2299
|
+
posture: { allowWrites: true },
|
|
2300
|
+
}),
|
|
2301
|
+
).rejects.toThrow(TooManyRequestsException);
|
|
2302
|
+
|
|
2303
|
+
expect(createdRunner).toBeNull();
|
|
2304
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2305
|
+
expect(feedItems).toHaveLength(0);
|
|
2306
|
+
|
|
2307
|
+
// Counted on this project's agent rows only.
|
|
2308
|
+
const query: Record<string, unknown> = (
|
|
2309
|
+
countBySpy.mock.calls[0]![0] as { query: Record<string, unknown> }
|
|
2310
|
+
).query;
|
|
2311
|
+
expect(query["projectId"]).toBe(PROJECT_ID);
|
|
2312
|
+
expect(query["name"]).toBeDefined();
|
|
2313
|
+
});
|
|
2314
|
+
|
|
2315
|
+
it("refuses a new agent Runner once the hourly creation brake is reached", async () => {
|
|
2316
|
+
jest
|
|
2317
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2318
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
2319
|
+
mockRunnerLookups({});
|
|
2320
|
+
countBySpy
|
|
2321
|
+
.mockResolvedValueOnce(new PositiveNumber(3))
|
|
2322
|
+
.mockResolvedValueOnce(
|
|
2323
|
+
new PositiveNumber(
|
|
2324
|
+
MAX_NEW_KUBERNETES_AGENT_RUNNERS_PER_PROJECT_PER_HOUR,
|
|
2325
|
+
),
|
|
2326
|
+
);
|
|
2327
|
+
|
|
2328
|
+
await expect(
|
|
2329
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2330
|
+
projectId: PROJECT_ID,
|
|
2331
|
+
clusterIdentifier: "prod-us",
|
|
2332
|
+
posture: { allowWrites: true },
|
|
2333
|
+
}),
|
|
2334
|
+
).rejects.toThrow(/in the last hour/);
|
|
2335
|
+
|
|
2336
|
+
expect(createdRunner).toBeNull();
|
|
2337
|
+
|
|
2338
|
+
const hourlyQuery: Record<string, unknown> = (
|
|
2339
|
+
countBySpy.mock.calls[1]![0] as { query: Record<string, unknown> }
|
|
2340
|
+
).query;
|
|
2341
|
+
expect(hourlyQuery["createdAt"]).toBeDefined();
|
|
2342
|
+
});
|
|
2343
|
+
|
|
2344
|
+
it("admits one below each brake", async () => {
|
|
2345
|
+
jest
|
|
2346
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2347
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
2348
|
+
mockRunnerLookups({});
|
|
2349
|
+
countBySpy
|
|
2350
|
+
.mockResolvedValueOnce(
|
|
2351
|
+
new PositiveNumber(MAX_KUBERNETES_AGENT_RUNNERS_PER_PROJECT - 1),
|
|
2352
|
+
)
|
|
2353
|
+
.mockResolvedValueOnce(
|
|
2354
|
+
new PositiveNumber(
|
|
2355
|
+
MAX_NEW_KUBERNETES_AGENT_RUNNERS_PER_PROJECT_PER_HOUR - 1,
|
|
2356
|
+
),
|
|
2357
|
+
);
|
|
2358
|
+
|
|
2359
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2360
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2361
|
+
projectId: PROJECT_ID,
|
|
2362
|
+
clusterIdentifier: "prod-us",
|
|
2363
|
+
posture: { allowWrites: true },
|
|
2364
|
+
});
|
|
2365
|
+
|
|
2366
|
+
expect(result.bindingState).toBe("first_bind");
|
|
2367
|
+
expect(createdRunner).not.toBeNull();
|
|
2368
|
+
});
|
|
2369
|
+
});
|
|
2370
|
+
|
|
2371
|
+
/*
|
|
2372
|
+
* The ping-pong finding: cluster A is bound (via the dashboard) to the
|
|
2373
|
+
* in-cluster Runner of cluster B. A's pod registering must not re-key B's
|
|
2374
|
+
* pod and overwrite its posture; it gets its own row and A stays as the
|
|
2375
|
+
* operator left it.
|
|
2376
|
+
*/
|
|
2377
|
+
describe("a bound Runner that is another cluster's agent is never reused", () => {
|
|
2378
|
+
it("creates this cluster's own agent row and leaves the other cluster's Runner and the binding alone", async () => {
|
|
2379
|
+
jest
|
|
2380
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2381
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: OTHER_RUNNER_ID }));
|
|
2382
|
+
mockRunnerLookups({
|
|
2383
|
+
bound: onlineAgentRunner({
|
|
2384
|
+
id: OTHER_RUNNER_ID,
|
|
2385
|
+
_id: OTHER_RUNNER_ID.toString(),
|
|
2386
|
+
name: "kubernetes-agent/prod-eu",
|
|
2387
|
+
hostInfo: {
|
|
2388
|
+
kubernetes: {
|
|
2389
|
+
inCluster: true,
|
|
2390
|
+
allowWrites: true,
|
|
2391
|
+
clusterIdentifier: "prod-eu",
|
|
2392
|
+
},
|
|
2393
|
+
},
|
|
2394
|
+
}),
|
|
2395
|
+
});
|
|
2396
|
+
|
|
2397
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2398
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2399
|
+
projectId: PROJECT_ID,
|
|
2400
|
+
clusterIdentifier: "prod-us",
|
|
2401
|
+
posture: { allowWrites: true },
|
|
2402
|
+
});
|
|
2403
|
+
|
|
2404
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
2405
|
+
expect(result.isFirstBind).toBe(false);
|
|
2406
|
+
expect(result.bindingState).toBe("bound_to_other_runner");
|
|
2407
|
+
expect(result.runnerId.toString()).toBe(RUNNER_ID.toString());
|
|
2408
|
+
|
|
2409
|
+
// prod-eu's live pod keeps its key and its posture.
|
|
2410
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
2411
|
+
expect(createdRunner).not.toBeNull();
|
|
2412
|
+
expect(createdRunner!.name).toBe(getKubernetesAgentRunnerName("prod-us"));
|
|
2413
|
+
expect(
|
|
2414
|
+
(createdRunner!.hostInfo as { kubernetes: Record<string, unknown> })
|
|
2415
|
+
.kubernetes["clusterIdentifier"],
|
|
2416
|
+
).toBe("prod-us");
|
|
2417
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2418
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
2419
|
+
"bound to a different Runner",
|
|
2420
|
+
);
|
|
2421
|
+
});
|
|
2422
|
+
|
|
2423
|
+
it("re-keys this cluster's existing by-name row instead of the other cluster's bound Runner", async () => {
|
|
2424
|
+
jest
|
|
2425
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2426
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: OTHER_RUNNER_ID }));
|
|
2427
|
+
mockRunnerLookups({
|
|
2428
|
+
bound: onlineAgentRunner({
|
|
2429
|
+
id: OTHER_RUNNER_ID,
|
|
2430
|
+
_id: OTHER_RUNNER_ID.toString(),
|
|
2431
|
+
name: "kubernetes-agent/prod-eu",
|
|
2432
|
+
hostInfo: {
|
|
2433
|
+
kubernetes: { inCluster: true, clusterIdentifier: "prod-eu" },
|
|
2434
|
+
},
|
|
2435
|
+
}),
|
|
2436
|
+
byName: offlineAgentRunner(),
|
|
2437
|
+
});
|
|
2438
|
+
|
|
2439
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2440
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2441
|
+
projectId: PROJECT_ID,
|
|
2442
|
+
clusterIdentifier: "prod-us",
|
|
2443
|
+
posture: { allowWrites: true },
|
|
2444
|
+
});
|
|
2445
|
+
|
|
2446
|
+
expect(result.bindingState).toBe("bound_to_other_runner");
|
|
2447
|
+
expect(result.runnerId.toString()).toBe(RUNNER_ID.toString());
|
|
2448
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2449
|
+
expect(runnerUpdates[0]!.id).toBe(RUNNER_ID.toString());
|
|
2450
|
+
expect(runnerUpdates[0]!.id).not.toBe(OTHER_RUNNER_ID.toString());
|
|
2451
|
+
expect(createdRunner).toBeNull();
|
|
2452
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2453
|
+
});
|
|
2454
|
+
|
|
2455
|
+
it("treats a bound pod Runner with no cluster identity as not this cluster's agent", async () => {
|
|
2456
|
+
jest
|
|
2457
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2458
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: OTHER_RUNNER_ID }));
|
|
2459
|
+
mockRunnerLookups({
|
|
2460
|
+
bound: onlineAgentRunner({
|
|
2461
|
+
id: OTHER_RUNNER_ID,
|
|
2462
|
+
_id: OTHER_RUNNER_ID.toString(),
|
|
2463
|
+
name: "pod-runner",
|
|
2464
|
+
hostInfo: { kubernetes: { inCluster: true } },
|
|
2465
|
+
}),
|
|
2466
|
+
});
|
|
2467
|
+
|
|
2468
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2469
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2470
|
+
projectId: PROJECT_ID,
|
|
2471
|
+
clusterIdentifier: "prod-us",
|
|
2472
|
+
posture: { allowWrites: true },
|
|
2473
|
+
});
|
|
2474
|
+
|
|
2475
|
+
expect(result.bindingState).toBe("bound_to_other_runner");
|
|
2476
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
2477
|
+
expect(createdRunner).not.toBeNull();
|
|
2478
|
+
});
|
|
2479
|
+
|
|
2480
|
+
/*
|
|
2481
|
+
* Round one re-keyed the BOUND row whenever its posture named this
|
|
2482
|
+
* cluster. A posture is whatever the Runner last heartbeated, so a row
|
|
2483
|
+
* renamed out of the agent marker (its name guards gone) was still
|
|
2484
|
+
* handed the ingestion-key identity. The row is found by NAME only.
|
|
2485
|
+
*/
|
|
2486
|
+
it("never re-keys a bound row that only CLAIMS to be this cluster's agent by its posture", async () => {
|
|
2487
|
+
jest
|
|
2488
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2489
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: OTHER_RUNNER_ID }));
|
|
2490
|
+
const lookups: jest.SpyInstance = mockRunnerLookups({
|
|
2491
|
+
bound: offlineAgentRunner({
|
|
2492
|
+
id: OTHER_RUNNER_ID,
|
|
2493
|
+
_id: OTHER_RUNNER_ID.toString(),
|
|
2494
|
+
name: "prod-us in-cluster runner",
|
|
2495
|
+
}),
|
|
2496
|
+
});
|
|
2497
|
+
|
|
2498
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2499
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2500
|
+
projectId: PROJECT_ID,
|
|
2501
|
+
clusterIdentifier: "prod-us",
|
|
2502
|
+
posture: { allowWrites: true },
|
|
2503
|
+
});
|
|
2504
|
+
|
|
2505
|
+
expect(result.bindingState).toBe("bound_to_other_runner");
|
|
2506
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
2507
|
+
// The bound row keeps its key; this cluster's agent row is minted.
|
|
2508
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
2509
|
+
expect(createdRunner).not.toBeNull();
|
|
2510
|
+
expect(createdRunner!.name).toBe("kubernetes-agent/prod-us");
|
|
2511
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2512
|
+
|
|
2513
|
+
// The row to re-key was looked up by name, first.
|
|
2514
|
+
const firstQuery: Record<string, unknown> = (
|
|
2515
|
+
lookups.mock.calls[0]![0] as { query: Record<string, unknown> }
|
|
2516
|
+
).query;
|
|
2517
|
+
expect(firstQuery["_id"]).toBeUndefined();
|
|
2518
|
+
expect(firstQuery["name"]).toBeDefined();
|
|
2519
|
+
});
|
|
2520
|
+
|
|
2521
|
+
it("does reuse the bound agent when its posture names this cluster in a different case", async () => {
|
|
2522
|
+
jest
|
|
2523
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2524
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2525
|
+
mockRunnerLookups({
|
|
2526
|
+
bound: offlineAgentRunner({
|
|
2527
|
+
hostInfo: {
|
|
2528
|
+
kubernetes: { inCluster: true, clusterIdentifier: "PROD-US" },
|
|
2529
|
+
},
|
|
2530
|
+
}),
|
|
2531
|
+
});
|
|
2532
|
+
|
|
2533
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2534
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2535
|
+
projectId: PROJECT_ID,
|
|
2536
|
+
clusterIdentifier: "prod-us",
|
|
2537
|
+
posture: { allowWrites: true },
|
|
2538
|
+
});
|
|
2539
|
+
|
|
2540
|
+
expect(result.bindingState).toBe("already_bound");
|
|
2541
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2542
|
+
expect(runnerUpdates[0]!.id).toBe(RUNNER_ID.toString());
|
|
2543
|
+
expect(createdRunner).toBeNull();
|
|
2544
|
+
});
|
|
2545
|
+
});
|
|
2546
|
+
|
|
2547
|
+
it("does not steal a cluster an operator bound to an external Runner", async () => {
|
|
2548
|
+
jest
|
|
2549
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2550
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
2551
|
+
mockRunnerLookups({
|
|
2552
|
+
// The bound Runner: an operator's external Runner, no posture.
|
|
2553
|
+
bound: onlineAgentRunner({ name: "office-runner", hostInfo: {} }),
|
|
2554
|
+
// The agent Runner row for this cluster, previously registered.
|
|
2555
|
+
byName: offlineAgentRunner({
|
|
2556
|
+
id: OTHER_RUNNER_ID,
|
|
2557
|
+
_id: OTHER_RUNNER_ID.toString(),
|
|
2558
|
+
}),
|
|
2559
|
+
});
|
|
2560
|
+
|
|
2561
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2562
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2563
|
+
projectId: PROJECT_ID,
|
|
2564
|
+
clusterIdentifier: "prod-us",
|
|
2565
|
+
posture: { allowWrites: true },
|
|
2566
|
+
});
|
|
2567
|
+
|
|
2568
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
2569
|
+
expect(result.bindingState).toBe("bound_to_other_runner");
|
|
2570
|
+
expect(result.runnerId.toString()).toBe(OTHER_RUNNER_ID.toString());
|
|
2571
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2572
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2573
|
+
expect(runnerUpdates[0]!.id).toBe(OTHER_RUNNER_ID.toString());
|
|
2574
|
+
});
|
|
2575
|
+
|
|
2576
|
+
/*
|
|
2577
|
+
* Rewritten: this used to mock a bound id pointing at a missing Runner
|
|
2578
|
+
* and expect a take-over. The binding's FK is ON DELETE SET NULL, so that
|
|
2579
|
+
* state only exists in the race between reading the cluster and reading
|
|
2580
|
+
* its Runner while a delete lands — and a Runner delete is how an
|
|
2581
|
+
* operator revokes or resets it, so even then nothing is re-bound.
|
|
2582
|
+
*/
|
|
2583
|
+
it("race only: a bound id whose Runner row vanished mid-registration is never re-bound and no switch moves", async () => {
|
|
2584
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
2585
|
+
fakeCluster({
|
|
2586
|
+
aiAccessRunnerId: RUNNER_ID,
|
|
2587
|
+
isAiInvestigationEnabled: false,
|
|
2588
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
2589
|
+
}),
|
|
2590
|
+
);
|
|
2591
|
+
mockRunnerLookups({});
|
|
2592
|
+
|
|
2593
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2594
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2595
|
+
projectId: PROJECT_ID,
|
|
2596
|
+
clusterIdentifier: "prod-us",
|
|
2597
|
+
posture: { allowWrites: true },
|
|
2598
|
+
});
|
|
2599
|
+
|
|
2600
|
+
expect(result.isFirstBind).toBe(false);
|
|
2601
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
2602
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
2603
|
+
expect(createdRunner).not.toBeNull();
|
|
2604
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2605
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).not.toContain(
|
|
2606
|
+
"took over",
|
|
2607
|
+
);
|
|
2608
|
+
});
|
|
2609
|
+
|
|
2610
|
+
/*
|
|
2611
|
+
* What Postgres actually leaves after an operator deletes the bound agent
|
|
2612
|
+
* Runner: the FK (ON DELETE SET NULL) has already cleared
|
|
2613
|
+
* aiAccessRunnerId, and the row is gone, so there is no agent row by name
|
|
2614
|
+
* either. The never-cleared aiAccessRunnerBoundAt marker remembers that a
|
|
2615
|
+
* Runner WAS bound (round one keyed this on aiAccessConfiguredAt, which
|
|
2616
|
+
* also left an operator who configured the cluster before installing the
|
|
2617
|
+
* chart unbound — known follow-up 12).
|
|
2618
|
+
*/
|
|
2619
|
+
describe("re-registration after the bound agent Runner was deleted (the FK ON DELETE SET NULL state)", () => {
|
|
2620
|
+
function clusterAfterRunnerDelete(
|
|
2621
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
2622
|
+
): KubernetesCluster {
|
|
2623
|
+
return fakeCluster({
|
|
2624
|
+
aiAccessRunnerId: undefined,
|
|
2625
|
+
aiAccessCredentialId: undefined,
|
|
2626
|
+
isAiInvestigationEnabled: false,
|
|
2627
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
2628
|
+
aiAccessLastVerifiedAt: undefined,
|
|
2629
|
+
aiAccessLastError: undefined,
|
|
2630
|
+
aiAccessConfiguredAt: OneUptimeDate.getSomeMinutesAgo(60 * 24),
|
|
2631
|
+
aiAccessRunnerBoundAt: OneUptimeDate.getSomeMinutesAgo(60 * 24),
|
|
2632
|
+
...overrides,
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2636
|
+
it("does not re-enable switches an operator turned off before any command ran", async () => {
|
|
2637
|
+
jest
|
|
2638
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2639
|
+
.mockResolvedValue(clusterAfterRunnerDelete());
|
|
2640
|
+
mockRunnerLookups({});
|
|
2641
|
+
|
|
2642
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2643
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2644
|
+
projectId: PROJECT_ID,
|
|
2645
|
+
clusterIdentifier: "prod-us",
|
|
2646
|
+
posture: { allowWrites: true },
|
|
2647
|
+
});
|
|
2648
|
+
|
|
2649
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
2650
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
2651
|
+
expect(result.isFirstBind).toBe(false);
|
|
2652
|
+
// A fresh Runner row for the pod, but the cluster is not touched.
|
|
2653
|
+
expect(createdRunner).not.toBeNull();
|
|
2654
|
+
expect(createdRunner!.name).toBe("kubernetes-agent/prod-us");
|
|
2655
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2656
|
+
});
|
|
2657
|
+
|
|
2658
|
+
it("says honestly what may have happened: cleared by an operator, OR its Runner was deleted", async () => {
|
|
2659
|
+
jest
|
|
2660
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2661
|
+
.mockResolvedValue(clusterAfterRunnerDelete());
|
|
2662
|
+
mockRunnerLookups({});
|
|
2663
|
+
|
|
2664
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2665
|
+
projectId: PROJECT_ID,
|
|
2666
|
+
clusterIdentifier: "prod-us",
|
|
2667
|
+
posture: { allowWrites: true },
|
|
2668
|
+
});
|
|
2669
|
+
|
|
2670
|
+
const feed: string = String(feedItems[0]!["feedInfoInMarkdown"]);
|
|
2671
|
+
expect(feed).toContain(
|
|
2672
|
+
"the binding was cleared by an operator, or the Runner it was bound to was deleted",
|
|
2673
|
+
);
|
|
2674
|
+
expect(feed).toContain("no AI switch was changed");
|
|
2675
|
+
expect(feed).toContain("Select **kubernetes-agent/prod-us**");
|
|
2676
|
+
});
|
|
2677
|
+
|
|
2678
|
+
it("keeps an operator's enabled switches exactly as they are, too", async () => {
|
|
2679
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
2680
|
+
clusterAfterRunnerDelete({
|
|
2681
|
+
isAiInvestigationEnabled: true,
|
|
2682
|
+
aiRemediationMode: KubernetesAiRemediationMode.Automatic,
|
|
2683
|
+
aiAccessLastVerifiedAt: OneUptimeDate.getSomeMinutesAgo(90),
|
|
2684
|
+
}),
|
|
2685
|
+
);
|
|
2686
|
+
mockRunnerLookups({});
|
|
2687
|
+
|
|
2688
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2689
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2690
|
+
projectId: PROJECT_ID,
|
|
2691
|
+
clusterIdentifier: "prod-us",
|
|
2692
|
+
posture: { allowWrites: false },
|
|
2693
|
+
});
|
|
2694
|
+
|
|
2695
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
2696
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2697
|
+
});
|
|
2698
|
+
|
|
2699
|
+
it("afterwards, the cluster's status points at selecting the new Runner, not at helm", async () => {
|
|
2700
|
+
jest
|
|
2701
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2702
|
+
.mockResolvedValue(clusterAfterRunnerDelete());
|
|
2703
|
+
mockRunnerLookups({});
|
|
2704
|
+
|
|
2705
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2706
|
+
projectId: PROJECT_ID,
|
|
2707
|
+
clusterIdentifier: "prod-us",
|
|
2708
|
+
posture: { allowWrites: true },
|
|
2709
|
+
});
|
|
2710
|
+
|
|
2711
|
+
// The row registration just created is what the status now finds.
|
|
2712
|
+
mockRunnerLookups({ byName: createdRunner });
|
|
2713
|
+
|
|
2714
|
+
const status: KubernetesClusterAiAccessStatus =
|
|
2715
|
+
await KubernetesClusterAiAccessService.getStatusForClusterModel({
|
|
2716
|
+
cluster: clusterAfterRunnerDelete(),
|
|
2717
|
+
gates: READY_GATES,
|
|
2718
|
+
});
|
|
2719
|
+
|
|
2720
|
+
expect(gapCodes(status)).toContain("no_runner_bound");
|
|
2721
|
+
const gap: KubernetesAiAccessGap = status.gaps.find(
|
|
2722
|
+
(candidate: KubernetesAiAccessGap) => {
|
|
2723
|
+
return candidate.code === "no_runner_bound";
|
|
2724
|
+
},
|
|
2725
|
+
)!;
|
|
2726
|
+
expect(gap.nextStep).toContain(
|
|
2727
|
+
'Select the kubernetes-agent Runner "kubernetes-agent/prod-us"',
|
|
2728
|
+
);
|
|
2729
|
+
expect(gap.nextStep).not.toContain("aiAccess.enabled=true");
|
|
2730
|
+
});
|
|
2731
|
+
|
|
2732
|
+
it("negative control: without either marker (and no other history) the same cluster still first-binds", async () => {
|
|
2733
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
2734
|
+
clusterAfterRunnerDelete({
|
|
2735
|
+
aiAccessConfiguredAt: undefined,
|
|
2736
|
+
aiAccessRunnerBoundAt: undefined,
|
|
2737
|
+
}),
|
|
2738
|
+
);
|
|
2739
|
+
mockRunnerLookups({});
|
|
2740
|
+
|
|
2741
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2742
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2743
|
+
projectId: PROJECT_ID,
|
|
2744
|
+
clusterIdentifier: "prod-us",
|
|
2745
|
+
posture: { allowWrites: true },
|
|
2746
|
+
});
|
|
2747
|
+
|
|
2748
|
+
expect(result.bindingState).toBe("first_bind");
|
|
2749
|
+
expect(clusterUpdates[0]).toMatchObject({
|
|
2750
|
+
isAiInvestigationEnabled: true,
|
|
2751
|
+
aiRemediationMode: KubernetesAiRemediationMode.RequireApproval,
|
|
2752
|
+
});
|
|
2753
|
+
});
|
|
2754
|
+
|
|
2755
|
+
/*
|
|
2756
|
+
* The marker that decides it is "a Runner was bound", not "configured":
|
|
2757
|
+
* with only aiAccessConfiguredAt left, the cluster is the pre-configured
|
|
2758
|
+
* case below and is bound, switches untouched.
|
|
2759
|
+
*/
|
|
2760
|
+
it("negative control: with only the configured marker it binds, keeping the operator's switches", async () => {
|
|
2761
|
+
jest
|
|
2762
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2763
|
+
.mockResolvedValue(
|
|
2764
|
+
clusterAfterRunnerDelete({ aiAccessRunnerBoundAt: undefined }),
|
|
2765
|
+
);
|
|
2766
|
+
mockRunnerLookups({});
|
|
2767
|
+
|
|
2768
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2769
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2770
|
+
projectId: PROJECT_ID,
|
|
2771
|
+
clusterIdentifier: "prod-us",
|
|
2772
|
+
posture: { allowWrites: true },
|
|
2773
|
+
});
|
|
2774
|
+
|
|
2775
|
+
expect(result.bindingState).toBe("bound_keeping_operator_settings");
|
|
2776
|
+
});
|
|
2777
|
+
});
|
|
2778
|
+
|
|
2779
|
+
/*
|
|
2780
|
+
* Known follow-up 12: an operator who opens the AI page and picks a
|
|
2781
|
+
* remediation mode (or flips a switch) BEFORE installing the chart stamps
|
|
2782
|
+
* aiAccessConfiguredAt. That must not strand the cluster unbound: no
|
|
2783
|
+
* Runner was ever bound, so the agent Runner is bound — and not one of the
|
|
2784
|
+
* operator's settings moves.
|
|
2785
|
+
*/
|
|
2786
|
+
describe("a cluster configured on the AI page before the chart was installed", () => {
|
|
2787
|
+
function preConfiguredCluster(
|
|
2788
|
+
overrides: Partial<Record<string, unknown>> = {},
|
|
2789
|
+
): KubernetesCluster {
|
|
2790
|
+
return fakeCluster({
|
|
2791
|
+
aiAccessRunnerId: undefined,
|
|
2792
|
+
aiAccessCredentialId: undefined,
|
|
2793
|
+
isAiInvestigationEnabled: false,
|
|
2794
|
+
aiRemediationMode: KubernetesAiRemediationMode.Automatic,
|
|
2795
|
+
aiAccessLastVerifiedAt: undefined,
|
|
2796
|
+
aiAccessLastError: undefined,
|
|
2797
|
+
aiAccessConfiguredAt: OneUptimeDate.getSomeMinutesAgo(10),
|
|
2798
|
+
aiAccessRunnerBoundAt: undefined,
|
|
2799
|
+
...overrides,
|
|
2800
|
+
});
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
it("binds the agent Runner and changes no switch", async () => {
|
|
2804
|
+
jest
|
|
2805
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2806
|
+
.mockResolvedValue(preConfiguredCluster());
|
|
2807
|
+
mockRunnerLookups({});
|
|
2808
|
+
|
|
2809
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2810
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2811
|
+
projectId: PROJECT_ID,
|
|
2812
|
+
clusterIdentifier: "prod-us",
|
|
2813
|
+
posture: { allowWrites: true },
|
|
2814
|
+
});
|
|
2815
|
+
|
|
2816
|
+
expect(result.bindingState).toBe("bound_keeping_operator_settings");
|
|
2817
|
+
expect(result.isBoundToCluster).toBe(true);
|
|
2818
|
+
expect(result.isFirstBind).toBe(false);
|
|
2819
|
+
expect(createdRunner).not.toBeNull();
|
|
2820
|
+
|
|
2821
|
+
expect(clusterUpdates).toHaveLength(1);
|
|
2822
|
+
// Only the binding and its marker: no switch, no credential, no mode.
|
|
2823
|
+
expect(Object.keys(clusterUpdates[0]!).sort()).toEqual(
|
|
2824
|
+
["aiAccessRunnerBoundAt", "aiAccessRunnerId"].sort(),
|
|
2825
|
+
);
|
|
2826
|
+
expect(String(clusterUpdates[0]!["aiAccessRunnerId"])).toBe(
|
|
2827
|
+
RUNNER_ID.toString(),
|
|
2828
|
+
);
|
|
2829
|
+
expect(clusterUpdates[0]!["aiAccessRunnerBoundAt"]).toBeInstanceOf(Date);
|
|
2830
|
+
});
|
|
2831
|
+
|
|
2832
|
+
it("says so honestly on the feed: bound, and the operator's settings kept", async () => {
|
|
2833
|
+
jest
|
|
2834
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2835
|
+
.mockResolvedValue(preConfiguredCluster());
|
|
2836
|
+
mockRunnerLookups({});
|
|
2837
|
+
|
|
2838
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2839
|
+
projectId: PROJECT_ID,
|
|
2840
|
+
clusterIdentifier: "prod-us",
|
|
2841
|
+
posture: { allowWrites: true },
|
|
2842
|
+
});
|
|
2843
|
+
|
|
2844
|
+
const feed: string = String(feedItems[0]!["feedInfoInMarkdown"]);
|
|
2845
|
+
expect(feed).toContain("was bound as this cluster's AI access Runner");
|
|
2846
|
+
expect(feed).toContain("no AI setting was changed");
|
|
2847
|
+
expect(feed).toContain("AI investigation with kubectl is off");
|
|
2848
|
+
expect(feed).toContain('AI remediation is "Automatic"');
|
|
2849
|
+
expect(feed).not.toContain("left unbound");
|
|
2850
|
+
});
|
|
2851
|
+
|
|
2852
|
+
it("keeps an investigation switch the operator turned on, too", async () => {
|
|
2853
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
2854
|
+
preConfiguredCluster({
|
|
2855
|
+
isAiInvestigationEnabled: true,
|
|
2856
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
2857
|
+
}),
|
|
2858
|
+
);
|
|
2859
|
+
mockRunnerLookups({});
|
|
2860
|
+
|
|
2861
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2862
|
+
projectId: PROJECT_ID,
|
|
2863
|
+
clusterIdentifier: "prod-us",
|
|
2864
|
+
posture: { allowWrites: true },
|
|
2865
|
+
});
|
|
2866
|
+
|
|
2867
|
+
// Disabled stays Disabled: the chart's intent never overrides a choice.
|
|
2868
|
+
expect(clusterUpdates[0]).not.toHaveProperty("aiRemediationMode");
|
|
2869
|
+
expect(clusterUpdates[0]).not.toHaveProperty("isAiInvestigationEnabled");
|
|
2870
|
+
});
|
|
2871
|
+
|
|
2872
|
+
it("also re-keys and binds this cluster's agent row that already existed", async () => {
|
|
2873
|
+
jest
|
|
2874
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2875
|
+
.mockResolvedValue(preConfiguredCluster());
|
|
2876
|
+
mockRunnerLookups({ byName: offlineAgentRunner() });
|
|
2877
|
+
|
|
2878
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2879
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2880
|
+
projectId: PROJECT_ID,
|
|
2881
|
+
clusterIdentifier: "prod-us",
|
|
2882
|
+
posture: { allowWrites: true },
|
|
2883
|
+
});
|
|
2884
|
+
|
|
2885
|
+
expect(result.bindingState).toBe("bound_keeping_operator_settings");
|
|
2886
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
2887
|
+
expect(clusterUpdates[0]!["aiAccessRunnerId"]).toBe(RUNNER_ID);
|
|
2888
|
+
});
|
|
2889
|
+
|
|
2890
|
+
it("negative control: once a Runner was bound, the same cluster is left unbound", async () => {
|
|
2891
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
2892
|
+
preConfiguredCluster({
|
|
2893
|
+
aiAccessRunnerBoundAt: OneUptimeDate.getSomeMinutesAgo(5),
|
|
2894
|
+
}),
|
|
2895
|
+
);
|
|
2896
|
+
mockRunnerLookups({});
|
|
2897
|
+
|
|
2898
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2899
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2900
|
+
projectId: PROJECT_ID,
|
|
2901
|
+
clusterIdentifier: "prod-us",
|
|
2902
|
+
posture: { allowWrites: true },
|
|
2903
|
+
});
|
|
2904
|
+
|
|
2905
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
2906
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2907
|
+
});
|
|
2908
|
+
|
|
2909
|
+
it("negative control: a recorded command outcome also leaves it unbound", async () => {
|
|
2910
|
+
for (const history of [
|
|
2911
|
+
{ aiAccessLastVerifiedAt: OneUptimeDate.getSomeMinutesAgo(5) },
|
|
2912
|
+
{ aiAccessLastError: "Forbidden" },
|
|
2913
|
+
]) {
|
|
2914
|
+
clusterUpdates.length = 0;
|
|
2915
|
+
jest
|
|
2916
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
2917
|
+
.mockResolvedValue(preConfiguredCluster(history));
|
|
2918
|
+
mockRunnerLookups({});
|
|
2919
|
+
|
|
2920
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2921
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2922
|
+
projectId: PROJECT_ID,
|
|
2923
|
+
clusterIdentifier: "prod-us",
|
|
2924
|
+
posture: { allowWrites: true },
|
|
2925
|
+
});
|
|
2926
|
+
|
|
2927
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
2928
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
2929
|
+
}
|
|
2930
|
+
});
|
|
2931
|
+
|
|
2932
|
+
it("a first bind stamps the bound marker too", async () => {
|
|
2933
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
2934
|
+
preConfiguredCluster({
|
|
2935
|
+
aiAccessConfiguredAt: undefined,
|
|
2936
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
2937
|
+
}),
|
|
2938
|
+
);
|
|
2939
|
+
mockRunnerLookups({});
|
|
2940
|
+
|
|
2941
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
2942
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
2943
|
+
projectId: PROJECT_ID,
|
|
2944
|
+
clusterIdentifier: "prod-us",
|
|
2945
|
+
posture: { allowWrites: true },
|
|
2946
|
+
});
|
|
2947
|
+
|
|
2948
|
+
expect(result.bindingState).toBe("first_bind");
|
|
2949
|
+
expect(clusterUpdates[0]!["aiAccessRunnerBoundAt"]).toBeInstanceOf(Date);
|
|
2950
|
+
expect(clusterUpdates[0]!["aiAccessConfiguredAt"]).toBeInstanceOf(Date);
|
|
2951
|
+
});
|
|
2952
|
+
});
|
|
2953
|
+
|
|
2954
|
+
/*
|
|
2955
|
+
* The credential-exfiltration finding: an offline (or signed-off) agent
|
|
2956
|
+
* row may be re-keyed by anyone holding the ingestion key, and the offline
|
|
2957
|
+
* window is predictable (every helm upgrade opens one). So a row an
|
|
2958
|
+
* operator entrusted with more than the agent defaults is never handed to
|
|
2959
|
+
* a registration that cannot present its current key.
|
|
2960
|
+
*/
|
|
2961
|
+
describe("an unproven re-key of a row that holds more than the agent defaults is refused", () => {
|
|
2962
|
+
type Holding = {
|
|
2963
|
+
name: string;
|
|
2964
|
+
arrange: () => Partial<Record<string, unknown>>;
|
|
2965
|
+
expectInMessage: string;
|
|
2966
|
+
};
|
|
2967
|
+
|
|
2968
|
+
const holdings: Array<Holding> = [
|
|
2969
|
+
{
|
|
2970
|
+
name: "a Runner credential assigned to it",
|
|
2971
|
+
arrange: (): Partial<Record<string, unknown>> => {
|
|
2972
|
+
credentialCountSpy.mockResolvedValue(new PositiveNumber(1));
|
|
2973
|
+
return {};
|
|
2974
|
+
},
|
|
2975
|
+
expectInMessage: "1 Runner credential(s) assigned",
|
|
2976
|
+
},
|
|
2977
|
+
{
|
|
2978
|
+
name: "a runbook secret assigned to it",
|
|
2979
|
+
arrange: (): Partial<Record<string, unknown>> => {
|
|
2980
|
+
secretCountSpy.mockResolvedValue(new PositiveNumber(2));
|
|
2981
|
+
return {};
|
|
2982
|
+
},
|
|
2983
|
+
expectInMessage: "2 runbook secret(s) assigned",
|
|
2984
|
+
},
|
|
2985
|
+
{
|
|
2986
|
+
name: "another cluster bound to it",
|
|
2987
|
+
arrange: (): Partial<Record<string, unknown>> => {
|
|
2988
|
+
boundClusterCountSpy.mockResolvedValue(new PositiveNumber(1));
|
|
2989
|
+
return {};
|
|
2990
|
+
},
|
|
2991
|
+
expectInMessage: "the AI access Runner of 1 other cluster(s)",
|
|
2992
|
+
},
|
|
2993
|
+
{
|
|
2994
|
+
name: '"Runs Runbooks" turned on',
|
|
2995
|
+
arrange: (): Partial<Record<string, unknown>> => {
|
|
2996
|
+
return { canRunRunbooks: true };
|
|
2997
|
+
},
|
|
2998
|
+
expectInMessage: '"Runs Runbooks" is on',
|
|
2999
|
+
},
|
|
3000
|
+
{
|
|
3001
|
+
name: '"Runs AI Code Fixes" turned on',
|
|
3002
|
+
arrange: (): Partial<Record<string, unknown>> => {
|
|
3003
|
+
return { canRunCodeFixTasks: true };
|
|
3004
|
+
},
|
|
3005
|
+
expectInMessage: '"Runs AI Code Fixes" is on',
|
|
3006
|
+
},
|
|
3007
|
+
];
|
|
3008
|
+
|
|
3009
|
+
function itRefusesTheUnprovenReKey(holding: Holding, state: string): void {
|
|
3010
|
+
it(`refuses a ${state} row with ${holding.name}: 403, nothing written, no key handed out`, async () => {
|
|
3011
|
+
const overrides: Partial<Record<string, unknown>> = holding.arrange();
|
|
3012
|
+
jest
|
|
3013
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3014
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3015
|
+
mockRunnerLookups({
|
|
3016
|
+
bound:
|
|
3017
|
+
state === "signed off"
|
|
3018
|
+
? signedOffAgentRunner(overrides)
|
|
3019
|
+
: offlineAgentRunner(overrides),
|
|
3020
|
+
});
|
|
3021
|
+
|
|
3022
|
+
let thrown: unknown = null;
|
|
3023
|
+
|
|
3024
|
+
try {
|
|
3025
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3026
|
+
projectId: PROJECT_ID,
|
|
3027
|
+
clusterIdentifier: "prod-us",
|
|
3028
|
+
posture: { allowWrites: true },
|
|
3029
|
+
});
|
|
3030
|
+
} catch (error) {
|
|
3031
|
+
thrown = error;
|
|
3032
|
+
}
|
|
3033
|
+
|
|
3034
|
+
expect(thrown).toBeInstanceOf(ForbiddenException);
|
|
3035
|
+
const message: string = (thrown as Error).message;
|
|
3036
|
+
expect(message).toContain(holding.expectInMessage);
|
|
3037
|
+
expect(message).toContain("delete the Runner");
|
|
3038
|
+
// Round four: the delete path names its second step.
|
|
3039
|
+
expect(message).toContain(
|
|
3040
|
+
"select it on the cluster's AI page as its Runner",
|
|
3041
|
+
);
|
|
3042
|
+
expect(message).not.toContain(CURRENT_KEY);
|
|
3043
|
+
|
|
3044
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
3045
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
3046
|
+
expect(createdRunner).toBeNull();
|
|
3047
|
+
expect(feedItems).toHaveLength(0);
|
|
3048
|
+
expect(String(warnSpy.mock.calls[0]![0])).toContain(
|
|
3049
|
+
"without proof of continuity",
|
|
3050
|
+
);
|
|
3051
|
+
});
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
for (const holding of holdings) {
|
|
3055
|
+
for (const state of ["signed off", "offline"]) {
|
|
3056
|
+
itRefusesTheUnprovenReKey(holding, state);
|
|
3057
|
+
}
|
|
3058
|
+
}
|
|
3059
|
+
|
|
3060
|
+
it("counts credentials and secrets assigned to THIS row in this project, and other clusters bound to it", async () => {
|
|
3061
|
+
jest
|
|
3062
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3063
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3064
|
+
mockRunnerLookups({ bound: signedOffAgentRunner() });
|
|
3065
|
+
|
|
3066
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3067
|
+
projectId: PROJECT_ID,
|
|
3068
|
+
clusterIdentifier: "prod-us",
|
|
3069
|
+
posture: { allowWrites: true },
|
|
3070
|
+
});
|
|
3071
|
+
|
|
3072
|
+
for (const spy of [credentialCountSpy, secretCountSpy]) {
|
|
3073
|
+
const query: Record<string, unknown> = (
|
|
3074
|
+
spy.mock.calls[0]![0] as { query: Record<string, unknown> }
|
|
3075
|
+
).query;
|
|
3076
|
+
expect(query["projectId"]).toBe(PROJECT_ID);
|
|
3077
|
+
expect(
|
|
3078
|
+
(query["runners"] as Array<ObjectID>).map((id: ObjectID) => {
|
|
3079
|
+
return id.toString();
|
|
3080
|
+
}),
|
|
3081
|
+
).toEqual([RUNNER_ID.toString()]);
|
|
3082
|
+
}
|
|
3083
|
+
|
|
3084
|
+
const clusterQuery: Record<string, unknown> = (
|
|
3085
|
+
boundClusterCountSpy.mock.calls[0]![0] as {
|
|
3086
|
+
query: Record<string, unknown>;
|
|
3087
|
+
}
|
|
3088
|
+
).query;
|
|
3089
|
+
expect(clusterQuery["projectId"]).toBe(PROJECT_ID);
|
|
3090
|
+
expect(String(clusterQuery["aiAccessRunnerId"])).toBe(
|
|
3091
|
+
RUNNER_ID.toString(),
|
|
3092
|
+
);
|
|
3093
|
+
// "Other than this cluster": this cluster's own binding is expected.
|
|
3094
|
+
expect(clusterQuery["_id"]).toBeDefined();
|
|
3095
|
+
expect(typeof clusterQuery["_id"]).not.toBe("string");
|
|
3096
|
+
});
|
|
3097
|
+
|
|
3098
|
+
it("the same refusal applies to the by-name row of an unbound cluster", async () => {
|
|
3099
|
+
credentialCountSpy.mockResolvedValue(new PositiveNumber(1));
|
|
3100
|
+
jest
|
|
3101
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3102
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
3103
|
+
mockRunnerLookups({ byName: signedOffAgentRunner() });
|
|
3104
|
+
|
|
3105
|
+
await expect(
|
|
3106
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3107
|
+
projectId: PROJECT_ID,
|
|
3108
|
+
clusterIdentifier: "prod-us",
|
|
3109
|
+
posture: { allowWrites: true },
|
|
3110
|
+
}),
|
|
3111
|
+
).rejects.toThrow(ForbiddenException);
|
|
3112
|
+
|
|
3113
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
3114
|
+
expect(createdRunner).toBeNull();
|
|
3115
|
+
});
|
|
3116
|
+
|
|
3117
|
+
it("admits the row when the registration proves continuity with its current key, whatever it holds", async () => {
|
|
3118
|
+
credentialCountSpy.mockResolvedValue(new PositiveNumber(1));
|
|
3119
|
+
jest
|
|
3120
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3121
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3122
|
+
mockRunnerLookups({
|
|
3123
|
+
bound: signedOffAgentRunner({ canRunRunbooks: true }),
|
|
3124
|
+
});
|
|
3125
|
+
|
|
3126
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3127
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3128
|
+
projectId: PROJECT_ID,
|
|
3129
|
+
clusterIdentifier: "prod-us",
|
|
3130
|
+
posture: { allowWrites: true },
|
|
3131
|
+
previousRunnerKey: CURRENT_KEY,
|
|
3132
|
+
});
|
|
3133
|
+
|
|
3134
|
+
expect(result.bindingState).toBe("already_bound");
|
|
3135
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
3136
|
+
// The proof short-circuits: nothing is counted.
|
|
3137
|
+
expect(credentialCountSpy).not.toHaveBeenCalled();
|
|
3138
|
+
});
|
|
3139
|
+
|
|
3140
|
+
it("negative control: a signed-off row with nothing beyond the defaults is still re-keyed without a key", async () => {
|
|
3141
|
+
jest
|
|
3142
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3143
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3144
|
+
mockRunnerLookups({ bound: signedOffAgentRunner() });
|
|
3145
|
+
|
|
3146
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3147
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3148
|
+
projectId: PROJECT_ID,
|
|
3149
|
+
clusterIdentifier: "prod-us",
|
|
3150
|
+
posture: { allowWrites: true },
|
|
3151
|
+
});
|
|
3152
|
+
|
|
3153
|
+
expect(result.bindingState).toBe("already_bound");
|
|
3154
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
3155
|
+
expect(credentialCountSpy).toHaveBeenCalledTimes(1);
|
|
3156
|
+
expect(secretCountSpy).toHaveBeenCalledTimes(1);
|
|
3157
|
+
expect(boundClusterCountSpy).toHaveBeenCalledTimes(1);
|
|
3158
|
+
});
|
|
3159
|
+
|
|
3160
|
+
it("an online row is refused for being online before anything is counted", async () => {
|
|
3161
|
+
jest
|
|
3162
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3163
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3164
|
+
mockRunnerLookups({ bound: onlineAgentRunner() });
|
|
3165
|
+
|
|
3166
|
+
await expect(
|
|
3167
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3168
|
+
projectId: PROJECT_ID,
|
|
3169
|
+
clusterIdentifier: "prod-us",
|
|
3170
|
+
posture: { allowWrites: true },
|
|
3171
|
+
}),
|
|
3172
|
+
).rejects.toThrow(/is online/);
|
|
3173
|
+
|
|
3174
|
+
expect(credentialCountSpy).not.toHaveBeenCalled();
|
|
3175
|
+
});
|
|
3176
|
+
});
|
|
3177
|
+
|
|
3178
|
+
/*
|
|
3179
|
+
* The agent Runner is named "kubernetes-agent/<cluster>", and Runner.name
|
|
3180
|
+
* is a 100-character column. Every cluster name the cluster row itself
|
|
3181
|
+
* accepts must still get a Runner row.
|
|
3182
|
+
*/
|
|
3183
|
+
describe("long cluster names", () => {
|
|
3184
|
+
const LONG_84: string = `arn:aws:eks:us-east-1:123456789012:cluster/${"p".repeat(84 - 43)}`;
|
|
3185
|
+
const LONG_100: string = "c".repeat(100);
|
|
3186
|
+
|
|
3187
|
+
it("keeps the exact name for an 83-character identifier (existing rows keep matching)", () => {
|
|
3188
|
+
const id: string = "a".repeat(83);
|
|
3189
|
+
|
|
3190
|
+
expect(getKubernetesAgentRunnerNameForCluster(id)).toBe(
|
|
3191
|
+
`kubernetes-agent/${id}`,
|
|
3192
|
+
);
|
|
3193
|
+
expect(getKubernetesAgentRunnerNameForCluster("prod-us")).toBe(
|
|
3194
|
+
getKubernetesAgentRunnerName("prod-us"),
|
|
3195
|
+
);
|
|
3196
|
+
});
|
|
3197
|
+
|
|
3198
|
+
it("bounds 84- and 100-character identifiers to the Runner name column, deterministically", () => {
|
|
3199
|
+
for (const id of [LONG_84, LONG_100]) {
|
|
3200
|
+
const name: string = getKubernetesAgentRunnerNameForCluster(id);
|
|
3201
|
+
|
|
3202
|
+
expect(LONG_84).toHaveLength(84);
|
|
3203
|
+
expect(name.length).toBeLessThanOrEqual(
|
|
3204
|
+
MAX_KUBERNETES_AGENT_RUNNER_NAME_LENGTH,
|
|
3205
|
+
);
|
|
3206
|
+
expect(name.startsWith("kubernetes-agent/")).toBe(true);
|
|
3207
|
+
expect(name).toBe(getKubernetesAgentRunnerNameForCluster(id));
|
|
3208
|
+
}
|
|
3209
|
+
});
|
|
3210
|
+
|
|
3211
|
+
it("maps case variants of one long identifier to the same row, and different long identifiers to different rows", () => {
|
|
3212
|
+
const lower: string = `${"x".repeat(90)}-prod`;
|
|
3213
|
+
const upper: string = lower.toUpperCase();
|
|
3214
|
+
const sibling: string = `${"x".repeat(90)}-test`;
|
|
3215
|
+
|
|
3216
|
+
expect(getKubernetesAgentRunnerNameForCluster(upper).toLowerCase()).toBe(
|
|
3217
|
+
getKubernetesAgentRunnerNameForCluster(lower).toLowerCase(),
|
|
3218
|
+
);
|
|
3219
|
+
expect(getKubernetesAgentRunnerNameForCluster(sibling)).not.toBe(
|
|
3220
|
+
getKubernetesAgentRunnerNameForCluster(lower),
|
|
3221
|
+
);
|
|
3222
|
+
});
|
|
3223
|
+
|
|
3224
|
+
it("registers a 100-character cluster name with a Runner name that fits, then finds that row again", async () => {
|
|
3225
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3226
|
+
fakeCluster({
|
|
3227
|
+
aiAccessRunnerId: undefined,
|
|
3228
|
+
clusterIdentifier: LONG_100,
|
|
3229
|
+
}),
|
|
3230
|
+
);
|
|
3231
|
+
const lookups: jest.SpyInstance = mockRunnerLookups({});
|
|
3232
|
+
|
|
3233
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3234
|
+
projectId: PROJECT_ID,
|
|
3235
|
+
clusterIdentifier: LONG_100,
|
|
3236
|
+
posture: { allowWrites: true },
|
|
3237
|
+
});
|
|
3238
|
+
|
|
3239
|
+
expect(createdRunner).not.toBeNull();
|
|
3240
|
+
expect(createdRunner!.name!.length).toBeLessThanOrEqual(
|
|
3241
|
+
MAX_KUBERNETES_AGENT_RUNNER_NAME_LENGTH,
|
|
3242
|
+
);
|
|
3243
|
+
expect(createdRunner!.name).toBe(
|
|
3244
|
+
getKubernetesAgentRunnerNameForCluster(LONG_100),
|
|
3245
|
+
);
|
|
3246
|
+
|
|
3247
|
+
// The lookup asked for exactly that name (case-insensitively).
|
|
3248
|
+
const nameFilter: { objectLiteralParameters?: Record<string, unknown> } =
|
|
3249
|
+
(lookups.mock.calls[0]![0] as { query: Record<string, unknown> }).query[
|
|
3250
|
+
"name"
|
|
3251
|
+
] as {
|
|
3252
|
+
objectLiteralParameters?: Record<string, unknown>;
|
|
3253
|
+
};
|
|
3254
|
+
expect(Object.values(nameFilter.objectLiteralParameters || {})).toEqual([
|
|
3255
|
+
createdRunner!.name!.toLowerCase(),
|
|
3256
|
+
]);
|
|
3257
|
+
|
|
3258
|
+
// A later restart re-finds that row instead of creating another.
|
|
3259
|
+
const created: Runner = createdRunner!;
|
|
3260
|
+
createdRunner = null;
|
|
3261
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3262
|
+
fakeCluster({
|
|
3263
|
+
aiAccessRunnerId: RUNNER_ID,
|
|
3264
|
+
clusterIdentifier: LONG_100,
|
|
3265
|
+
}),
|
|
3266
|
+
);
|
|
3267
|
+
mockRunnerLookups({
|
|
3268
|
+
bound: offlineAgentRunner({
|
|
3269
|
+
name: created.name,
|
|
3270
|
+
hostInfo: created.hostInfo,
|
|
3271
|
+
}),
|
|
3272
|
+
});
|
|
3273
|
+
|
|
3274
|
+
const again: RegisterKubernetesAgentRunnerResult =
|
|
3275
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3276
|
+
projectId: PROJECT_ID,
|
|
3277
|
+
clusterIdentifier: LONG_100,
|
|
3278
|
+
posture: { allowWrites: true },
|
|
3279
|
+
});
|
|
3280
|
+
|
|
3281
|
+
expect(again.bindingState).toBe("already_bound");
|
|
3282
|
+
expect(createdRunner).toBeNull();
|
|
3283
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
3284
|
+
});
|
|
3285
|
+
|
|
3286
|
+
it("refuses a name longer than the cluster column with a clear 400, before any cluster row is created", async () => {
|
|
3287
|
+
const findOrCreate: jest.SpyInstance =
|
|
3288
|
+
KubernetesClusterService.findOrCreateByClusterIdentifier as unknown as jest.SpyInstance;
|
|
3289
|
+
|
|
3290
|
+
let thrown: unknown = null;
|
|
3291
|
+
|
|
3292
|
+
try {
|
|
3293
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3294
|
+
projectId: PROJECT_ID,
|
|
3295
|
+
clusterIdentifier: "d".repeat(101),
|
|
3296
|
+
posture: { allowWrites: true },
|
|
3297
|
+
});
|
|
3298
|
+
} catch (error) {
|
|
3299
|
+
thrown = error;
|
|
3300
|
+
}
|
|
3301
|
+
|
|
3302
|
+
expect(thrown).toBeInstanceOf(BadDataException);
|
|
3303
|
+
expect((thrown as Error).message).toContain("101 characters");
|
|
3304
|
+
expect((thrown as Error).message).toContain("up to 100 characters");
|
|
3305
|
+
expect(findOrCreate).not.toHaveBeenCalled();
|
|
3306
|
+
expect(createdRunner).toBeNull();
|
|
3307
|
+
});
|
|
3308
|
+
|
|
3309
|
+
it("refuses to reuse a row by this name that reports it belongs to a different cluster", async () => {
|
|
3310
|
+
jest
|
|
3311
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3312
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
3313
|
+
mockRunnerLookups({
|
|
3314
|
+
byName: offlineAgentRunner({
|
|
3315
|
+
hostInfo: {
|
|
3316
|
+
kubernetes: { inCluster: true, clusterIdentifier: "prod-eu" },
|
|
3317
|
+
},
|
|
3318
|
+
}),
|
|
3319
|
+
});
|
|
3320
|
+
|
|
3321
|
+
await expect(
|
|
3322
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3323
|
+
projectId: PROJECT_ID,
|
|
3324
|
+
clusterIdentifier: "prod-us",
|
|
3325
|
+
posture: { allowWrites: true },
|
|
3326
|
+
}),
|
|
3327
|
+
).rejects.toThrow(
|
|
3328
|
+
/reports that it is the in-cluster Runner of a different cluster/,
|
|
3329
|
+
);
|
|
3330
|
+
|
|
3331
|
+
expect(runnerUpdates).toHaveLength(0);
|
|
3332
|
+
expect(createdRunner).toBeNull();
|
|
3333
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
3334
|
+
});
|
|
3335
|
+
});
|
|
3336
|
+
|
|
3337
|
+
/*
|
|
3338
|
+
* The silent re-enable finding: an operator who cleared the Runner (and
|
|
3339
|
+
* turned the switches off) must not have AI turned back on by the next
|
|
3340
|
+
* pod restart. An unbound cluster WITH an AI-access history was cleared
|
|
3341
|
+
* on purpose; only a cluster with no history at all is "never
|
|
3342
|
+
* configured".
|
|
3343
|
+
*/
|
|
3344
|
+
describe("a cluster an operator unbound stays unbound", () => {
|
|
3345
|
+
/*
|
|
3346
|
+
* Round one read "an agent row already existed" as the history that
|
|
3347
|
+
* meant "cleared on purpose"; the record of that is now the
|
|
3348
|
+
* never-cleared aiAccessRunnerBoundAt marker every bind stamps.
|
|
3349
|
+
*/
|
|
3350
|
+
it("re-keys the existing agent row but neither re-binds nor flips a switch", async () => {
|
|
3351
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3352
|
+
fakeCluster({
|
|
3353
|
+
aiAccessRunnerId: undefined,
|
|
3354
|
+
isAiInvestigationEnabled: false,
|
|
3355
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
3356
|
+
aiAccessRunnerBoundAt: OneUptimeDate.getSomeMinutesAgo(60),
|
|
3357
|
+
}),
|
|
3358
|
+
);
|
|
3359
|
+
mockRunnerLookups({ byName: offlineAgentRunner() });
|
|
3360
|
+
|
|
3361
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3362
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3363
|
+
projectId: PROJECT_ID,
|
|
3364
|
+
clusterIdentifier: "prod-us",
|
|
3365
|
+
posture: { allowWrites: true },
|
|
3366
|
+
});
|
|
3367
|
+
|
|
3368
|
+
expect(result.isFirstBind).toBe(false);
|
|
3369
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
3370
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
3371
|
+
expect(result.runnerId.toString()).toBe(RUNNER_ID.toString());
|
|
3372
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
3373
|
+
expect(runnerUpdates).toHaveLength(1);
|
|
3374
|
+
expect(runnerUpdates[0]!.data["key"]).toBe(result.runnerKey);
|
|
3375
|
+
expect(createdRunner).toBeNull();
|
|
3376
|
+
expect(String(feedItems[0]!["feedInfoInMarkdown"])).toContain(
|
|
3377
|
+
"cleared by an operator",
|
|
3378
|
+
);
|
|
3379
|
+
});
|
|
3380
|
+
|
|
3381
|
+
it("also holds when the Runner row is gone but the cluster has a recorded access check", async () => {
|
|
3382
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3383
|
+
fakeCluster({
|
|
3384
|
+
aiAccessRunnerId: undefined,
|
|
3385
|
+
isAiInvestigationEnabled: false,
|
|
3386
|
+
aiAccessLastVerifiedAt: OneUptimeDate.getSomeMinutesAgo(60 * 24 * 7),
|
|
3387
|
+
}),
|
|
3388
|
+
);
|
|
3389
|
+
mockRunnerLookups({});
|
|
3390
|
+
|
|
3391
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3392
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3393
|
+
projectId: PROJECT_ID,
|
|
3394
|
+
clusterIdentifier: "prod-us",
|
|
3395
|
+
posture: { allowWrites: true },
|
|
3396
|
+
});
|
|
3397
|
+
|
|
3398
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
3399
|
+
expect(result.isBoundToCluster).toBe(false);
|
|
3400
|
+
expect(createdRunner).not.toBeNull();
|
|
3401
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
3402
|
+
});
|
|
3403
|
+
|
|
3404
|
+
it("also holds when the only trace is a recorded access error", async () => {
|
|
3405
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3406
|
+
fakeCluster({
|
|
3407
|
+
aiAccessRunnerId: undefined,
|
|
3408
|
+
aiAccessLastError: "kubectl: connection refused",
|
|
3409
|
+
}),
|
|
3410
|
+
);
|
|
3411
|
+
mockRunnerLookups({});
|
|
3412
|
+
|
|
3413
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3414
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3415
|
+
projectId: PROJECT_ID,
|
|
3416
|
+
clusterIdentifier: "prod-us",
|
|
3417
|
+
posture: { allowWrites: true },
|
|
3418
|
+
});
|
|
3419
|
+
|
|
3420
|
+
expect(result.bindingState).toBe("left_unbound_by_operator");
|
|
3421
|
+
expect(clusterUpdates).toHaveLength(0);
|
|
3422
|
+
});
|
|
3423
|
+
|
|
3424
|
+
it("still first-binds a cluster with switches at their defaults and no history", async () => {
|
|
3425
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3426
|
+
fakeCluster({
|
|
3427
|
+
aiAccessRunnerId: undefined,
|
|
3428
|
+
isAiInvestigationEnabled: false,
|
|
3429
|
+
aiRemediationMode: KubernetesAiRemediationMode.Disabled,
|
|
3430
|
+
aiAccessLastVerifiedAt: undefined,
|
|
3431
|
+
aiAccessLastError: undefined,
|
|
3432
|
+
}),
|
|
3433
|
+
);
|
|
3434
|
+
mockRunnerLookups({});
|
|
3435
|
+
|
|
3436
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3437
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3438
|
+
projectId: PROJECT_ID,
|
|
3439
|
+
clusterIdentifier: "prod-us",
|
|
3440
|
+
posture: { allowWrites: true },
|
|
3441
|
+
});
|
|
3442
|
+
|
|
3443
|
+
expect(result.bindingState).toBe("first_bind");
|
|
3444
|
+
expect(clusterUpdates).toHaveLength(1);
|
|
3445
|
+
});
|
|
3446
|
+
});
|
|
3447
|
+
|
|
3448
|
+
/*
|
|
3449
|
+
* Every 403 says WHICH refusal it is, so the Runner can tell a wait that
|
|
3450
|
+
* clears on its own from one that needs an operator (and retry the first
|
|
3451
|
+
* when it says). Round one read every 403 as "the previous instance is
|
|
3452
|
+
* still online".
|
|
3453
|
+
*/
|
|
3454
|
+
describe("registration refusals carry a machine-readable reason", () => {
|
|
3455
|
+
async function refusalOf(
|
|
3456
|
+
promise: Promise<unknown>,
|
|
3457
|
+
): Promise<KubernetesAgentRegistrationRefusedException> {
|
|
3458
|
+
try {
|
|
3459
|
+
await promise;
|
|
3460
|
+
} catch (error) {
|
|
3461
|
+
expect(error).toBeInstanceOf(
|
|
3462
|
+
KubernetesAgentRegistrationRefusedException,
|
|
3463
|
+
);
|
|
3464
|
+
// Still a 403 for anything that reads it as a ForbiddenException.
|
|
3465
|
+
expect(error).toBeInstanceOf(ForbiddenException);
|
|
3466
|
+
return error as KubernetesAgentRegistrationRefusedException;
|
|
3467
|
+
}
|
|
3468
|
+
|
|
3469
|
+
throw new Error("expected the registration to be refused");
|
|
3470
|
+
}
|
|
3471
|
+
|
|
3472
|
+
it("previous_instance_online, with when to try again", async () => {
|
|
3473
|
+
jest
|
|
3474
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3475
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3476
|
+
mockRunnerLookups({
|
|
3477
|
+
bound: onlineAgentRunner({
|
|
3478
|
+
lastAlive: OneUptimeDate.getSomeMinutesAgo(2),
|
|
3479
|
+
}),
|
|
3480
|
+
});
|
|
3481
|
+
|
|
3482
|
+
const refusal: KubernetesAgentRegistrationRefusedException =
|
|
3483
|
+
await refusalOf(
|
|
3484
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3485
|
+
projectId: PROJECT_ID,
|
|
3486
|
+
clusterIdentifier: "prod-us",
|
|
3487
|
+
posture: { allowWrites: true },
|
|
3488
|
+
}),
|
|
3489
|
+
);
|
|
3490
|
+
|
|
3491
|
+
expect(refusal.reason).toBe("previous_instance_online");
|
|
3492
|
+
expect(
|
|
3493
|
+
isTransientKubernetesAgentRegistrationRefusal(refusal.reason),
|
|
3494
|
+
).toBe(true);
|
|
3495
|
+
// Admitted once the last heartbeat is 5 minutes old: ~3 minutes away.
|
|
3496
|
+
expect(refusal.retryAfterSeconds).toBeGreaterThanOrEqual(170);
|
|
3497
|
+
expect(refusal.retryAfterSeconds).toBeLessThanOrEqual(181);
|
|
3498
|
+
});
|
|
3499
|
+
|
|
3500
|
+
it("runner_holds_more_than_defaults, with the instruction first and no retry hint", async () => {
|
|
3501
|
+
jest
|
|
3502
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3503
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: RUNNER_ID }));
|
|
3504
|
+
mockRunnerLookups({
|
|
3505
|
+
bound: signedOffAgentRunner({ canRunRunbooks: true }),
|
|
3506
|
+
});
|
|
3507
|
+
|
|
3508
|
+
const refusal: KubernetesAgentRegistrationRefusedException =
|
|
3509
|
+
await refusalOf(
|
|
3510
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3511
|
+
projectId: PROJECT_ID,
|
|
3512
|
+
clusterIdentifier: "prod-us",
|
|
3513
|
+
posture: { allowWrites: true },
|
|
3514
|
+
}),
|
|
3515
|
+
);
|
|
3516
|
+
|
|
3517
|
+
expect(refusal.reason).toBe("runner_holds_more_than_defaults");
|
|
3518
|
+
expect(
|
|
3519
|
+
isTransientKubernetesAgentRegistrationRefusal(refusal.reason),
|
|
3520
|
+
).toBe(false);
|
|
3521
|
+
expect(refusal.retryAfterSeconds).toBeUndefined();
|
|
3522
|
+
expect(
|
|
3523
|
+
refusal.message.startsWith(
|
|
3524
|
+
'Under Project Settings → Runners, on Runner "kubernetes-agent/prod-us": turn off "Runs Runbooks"',
|
|
3525
|
+
),
|
|
3526
|
+
).toBe(true);
|
|
3527
|
+
// Only what it actually holds is asked for.
|
|
3528
|
+
expect(refusal.message).not.toContain('turn off "Runs AI Code Fixes"');
|
|
3529
|
+
expect(refusal.message).not.toContain("unassign");
|
|
3530
|
+
});
|
|
3531
|
+
|
|
3532
|
+
/*
|
|
3533
|
+
* The Runner logs only the first 500 characters of the server's reason.
|
|
3534
|
+
* The whole instruction must fit in them even for a long cluster name
|
|
3535
|
+
* and every kind of holding at once.
|
|
3536
|
+
*/
|
|
3537
|
+
it("keeps the full instruction within the first 500 characters for a 100-character cluster name", async () => {
|
|
3538
|
+
const longName: string = "x".repeat(100);
|
|
3539
|
+
credentialCountSpy.mockResolvedValue(new PositiveNumber(3));
|
|
3540
|
+
secretCountSpy.mockResolvedValue(new PositiveNumber(2));
|
|
3541
|
+
boundClusterCountSpy.mockResolvedValue(new PositiveNumber(4));
|
|
3542
|
+
jest.spyOn(KubernetesClusterService, "findOneBy").mockResolvedValue(
|
|
3543
|
+
fakeCluster({
|
|
3544
|
+
aiAccessRunnerId: RUNNER_ID,
|
|
3545
|
+
clusterIdentifier: longName,
|
|
3546
|
+
}),
|
|
3547
|
+
);
|
|
3548
|
+
mockRunnerLookups({
|
|
3549
|
+
bound: signedOffAgentRunner({
|
|
3550
|
+
name: getKubernetesAgentRunnerNameForCluster(longName),
|
|
3551
|
+
canRunRunbooks: true,
|
|
3552
|
+
canRunCodeFixTasks: true,
|
|
3553
|
+
hostInfo: {
|
|
3554
|
+
kubernetes: { inCluster: true, clusterIdentifier: longName },
|
|
3555
|
+
},
|
|
3556
|
+
}),
|
|
3557
|
+
});
|
|
3558
|
+
|
|
3559
|
+
const refusal: KubernetesAgentRegistrationRefusedException =
|
|
3560
|
+
await refusalOf(
|
|
3561
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3562
|
+
projectId: PROJECT_ID,
|
|
3563
|
+
clusterIdentifier: longName,
|
|
3564
|
+
posture: { allowWrites: true },
|
|
3565
|
+
}),
|
|
3566
|
+
);
|
|
3567
|
+
|
|
3568
|
+
const head: string = refusal.message.slice(0, 500);
|
|
3569
|
+
expect(head).toContain("delete the Runner");
|
|
3570
|
+
// Round four: the delete path's second step is part of the instruction.
|
|
3571
|
+
expect(head).toContain(
|
|
3572
|
+
"select it on the cluster's AI page as its Runner",
|
|
3573
|
+
);
|
|
3574
|
+
expect(head).toContain('"Runs Runbooks"');
|
|
3575
|
+
expect(head).toContain('"Runs AI Code Fixes"');
|
|
3576
|
+
expect(head).toContain("3 Runner credential(s)");
|
|
3577
|
+
expect(head).toContain("2 runbook secret(s)");
|
|
3578
|
+
expect(head).toContain("4 other cluster(s)");
|
|
3579
|
+
});
|
|
3580
|
+
|
|
3581
|
+
it("runner_belongs_to_another_cluster, with no retry hint", async () => {
|
|
3582
|
+
jest
|
|
3583
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3584
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
3585
|
+
mockRunnerLookups({
|
|
3586
|
+
byName: offlineAgentRunner({
|
|
3587
|
+
hostInfo: {
|
|
3588
|
+
kubernetes: { inCluster: true, clusterIdentifier: "prod-eu" },
|
|
3589
|
+
},
|
|
3590
|
+
}),
|
|
3591
|
+
});
|
|
3592
|
+
|
|
3593
|
+
const refusal: KubernetesAgentRegistrationRefusedException =
|
|
3594
|
+
await refusalOf(
|
|
3595
|
+
KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3596
|
+
projectId: PROJECT_ID,
|
|
3597
|
+
clusterIdentifier: "prod-us",
|
|
3598
|
+
posture: { allowWrites: true },
|
|
3599
|
+
}),
|
|
3600
|
+
);
|
|
3601
|
+
|
|
3602
|
+
expect(refusal.reason).toBe("runner_belongs_to_another_cluster");
|
|
3603
|
+
expect(refusal.retryAfterSeconds).toBeUndefined();
|
|
3604
|
+
/*
|
|
3605
|
+
* Round four: was "Rename or delete Runner ...". A non-root rename of
|
|
3606
|
+
* an agent-named Runner is refused (RunnerService), so the one step
|
|
3607
|
+
* an operator can take is the delete — and then selecting the fresh
|
|
3608
|
+
* Runner, since a deleted Runner's cluster is left unbound.
|
|
3609
|
+
*/
|
|
3610
|
+
expect(refusal.message.startsWith("Delete Runner")).toBe(true);
|
|
3611
|
+
expect(refusal.message).toContain(
|
|
3612
|
+
'select it on the AI page of cluster "prod-us" as its Runner',
|
|
3613
|
+
);
|
|
3614
|
+
});
|
|
3615
|
+
|
|
3616
|
+
it("getPreviousInstanceRetryAfterSeconds counts down to the end of the alive window, never below 1s", () => {
|
|
3617
|
+
const now: Date = new Date("2026-09-22T12:00:00.000Z");
|
|
3618
|
+
|
|
3619
|
+
expect(
|
|
3620
|
+
getPreviousInstanceRetryAfterSeconds({
|
|
3621
|
+
lastAlive: new Date("2026-09-22T11:59:00.000Z"),
|
|
3622
|
+
now,
|
|
3623
|
+
}),
|
|
3624
|
+
).toBe(240);
|
|
3625
|
+
expect(
|
|
3626
|
+
getPreviousInstanceRetryAfterSeconds({
|
|
3627
|
+
lastAlive: new Date("2026-09-22T11:55:00.500Z"),
|
|
3628
|
+
now,
|
|
3629
|
+
}),
|
|
3630
|
+
).toBe(1);
|
|
3631
|
+
expect(
|
|
3632
|
+
getPreviousInstanceRetryAfterSeconds({
|
|
3633
|
+
lastAlive: new Date("2026-09-22T11:00:00.000Z"),
|
|
3634
|
+
now,
|
|
3635
|
+
}),
|
|
3636
|
+
).toBe(1);
|
|
3637
|
+
expect(
|
|
3638
|
+
getPreviousInstanceRetryAfterSeconds({ lastAlive: undefined, now }),
|
|
3639
|
+
).toBe(1);
|
|
3640
|
+
});
|
|
3641
|
+
});
|
|
3642
|
+
|
|
3643
|
+
it("survives a feed write failure: the registration still succeeds", async () => {
|
|
3644
|
+
jest
|
|
3645
|
+
.spyOn(KubernetesClusterService, "findOneBy")
|
|
3646
|
+
.mockResolvedValue(fakeCluster({ aiAccessRunnerId: undefined }));
|
|
3647
|
+
mockRunnerLookups({});
|
|
3648
|
+
(
|
|
3649
|
+
KubernetesClusterFeedService.createKubernetesClusterFeedItem as unknown as jest.SpyInstance
|
|
3650
|
+
).mockRejectedValue(new Error("feed is down"));
|
|
3651
|
+
|
|
3652
|
+
const result: RegisterKubernetesAgentRunnerResult =
|
|
3653
|
+
await KubernetesClusterAiAccessService.registerKubernetesAgentRunner({
|
|
3654
|
+
projectId: PROJECT_ID,
|
|
3655
|
+
clusterIdentifier: "prod-us",
|
|
3656
|
+
posture: { allowWrites: true },
|
|
3657
|
+
});
|
|
3658
|
+
|
|
3659
|
+
expect(result.bindingState).toBe("first_bind");
|
|
3660
|
+
expect(clusterUpdates).toHaveLength(1);
|
|
3661
|
+
});
|
|
3662
|
+
});
|