@oneuptime/common 11.5.9 → 11.5.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/Index.ts +13 -0
- package/Models/AnalyticsModels/MetricItemAggMV1mByContainer.ts +178 -0
- package/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.ts +179 -0
- package/Models/AnalyticsModels/MetricItemAggMV1mByService.ts +179 -0
- package/Models/AnalyticsModels/NetworkFlow.ts +397 -0
- package/Models/DatabaseModels/EnterpriseLicenseInstance.ts +18 -0
- package/Models/DatabaseModels/GlobalConfig.ts +36 -0
- package/Models/DatabaseModels/Index.ts +2 -0
- package/Models/DatabaseModels/MarketingConversion.ts +198 -0
- package/Models/DatabaseModels/MetricSavedView.ts +34 -0
- package/Models/DatabaseModels/MetricType.ts +74 -0
- package/Models/DatabaseModels/NetworkDevice.ts +332 -0
- package/Models/DatabaseModels/NetworkDeviceDiscoveryScan.ts +113 -0
- package/Models/DatabaseModels/NetworkInterface.ts +58 -0
- package/Models/DatabaseModels/Project.ts +37 -0
- package/Models/DatabaseModels/User.ts +37 -0
- package/Server/API/BaseAnalyticsAPI.ts +23 -0
- package/Server/API/EnterpriseLicenseAPI.ts +14 -0
- package/Server/EnvironmentConfig.ts +84 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784191414522-AddCounterSemanticsToMetricType.ts +38 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784191414600-AddViewTypeToMetricSavedView.ts +28 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784211212164-AddNetworkDeviceInventoryAndDiscoverySchedule.ts +108 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784218257664-AddEnterpriseLicenseNotificationColumns.ts +31 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784293516000-AddAttributionColumnsToUserAndProject.ts +29 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784298000000-AddMarketingConversionTable.ts +35 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1784401962564-MigrationName.ts +101 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +14 -0
- package/Server/Services/AnalyticsDatabaseService.ts +70 -1
- package/Server/Services/IncidentService.ts +13 -0
- package/Server/Services/Index.ts +10 -0
- package/Server/Services/MarketingConversionService.ts +10 -0
- package/Server/Services/MetricItemAggMV1mByContainerService.ts +31 -0
- package/Server/Services/MetricItemAggMV1mByK8sClusterService.ts +31 -0
- package/Server/Services/MetricItemAggMV1mByServiceService.ts +31 -0
- package/Server/Services/MetricService.ts +970 -122
- package/Server/Services/MonitorService.ts +14 -0
- package/Server/Services/NetworkDeviceOwnerTeamService.ts +42 -0
- package/Server/Services/NetworkDeviceOwnerUserService.ts +106 -0
- package/Server/Services/NetworkFlowService.ts +11 -0
- package/Server/Services/OnCallDutyPolicyService.ts +11 -1
- package/Server/Services/ProjectService.ts +136 -0
- package/Server/Services/SpanService.ts +13 -9
- package/Server/Services/StatusPageService.ts +10 -0
- package/Server/Services/TeamMemberService.ts +11 -0
- package/Server/Services/UserService.ts +59 -0
- package/Server/Types/AnalyticsDatabase/AggregateBy.ts +37 -7
- package/Server/Utils/Attribution.ts +99 -0
- package/Server/Utils/Marketing/ConversionUploadProvider.ts +103 -0
- package/Server/Utils/Marketing/ConversionUploadProviders.ts +22 -0
- package/Server/Utils/Marketing/Providers/GoogleAds.ts +270 -0
- package/Server/Utils/Marketing/Providers/LinkedIn.ts +147 -0
- package/Server/Utils/Marketing/Providers/Meta.ts +148 -0
- package/Server/Utils/Marketing/Providers/MicrosoftAds.ts +196 -0
- package/Server/Utils/Marketing/Providers/Reddit.ts +178 -0
- package/Server/Utils/Monitor/MonitorAlert.ts +71 -7
- package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts +31 -19
- package/Server/Utils/Monitor/MonitorIncident.ts +71 -7
- package/Server/Utils/Monitor/MonitorMetricUtil.ts +57 -0
- package/Server/Utils/Monitor/MonitorTemplateUtil.ts +94 -0
- package/Server/Utils/Monitor/NetworkDeviceHydrationUtil.ts +15 -0
- package/Server/Utils/Monitor/NetworkInventoryUtil.ts +125 -14
- package/Server/Utils/ProductAnalytics.ts +133 -0
- package/Server/Utils/Telemetry/Telemetry.ts +39 -0
- package/Server/Utils/Telemetry/TelemetryEntity.ts +21 -0
- package/Tests/Server/API/BaseAnalyticsAPI.test.ts +72 -0
- package/Tests/Server/Infrastructure/SemaphorePermit.test.ts +9 -1
- package/Tests/Server/Services/AnalyticsDatabaseService.test.ts +21 -2
- package/Tests/Server/Services/MetricEntityMVKeyParity.test.ts +173 -0
- package/Tests/Server/Services/MetricServiceAggregate.test.ts +1391 -0
- package/Tests/Server/Services/NetworkDeviceOwnerServices.test.ts +260 -0
- package/Tests/Server/Utils/Monitor/MonitorTemplateUtilNetworkDevice.test.ts +389 -0
- package/Tests/Server/Utils/Monitor/NetworkInventoryUtil.test.ts +764 -0
- package/Tests/Types/BaseDatabase/AggregationIntervalUtil.test.ts +111 -2
- package/Tests/Types/Database/EndsWith.test.ts +64 -0
- package/Tests/Types/Database/GreaterThan.test.ts +62 -0
- package/Tests/Types/Database/GreaterThanOrEqual.test.ts +65 -0
- package/Tests/Types/Database/GreaterThanOrNull.test.ts +63 -0
- package/Tests/Types/Database/Includes.test.ts +65 -0
- package/Tests/Types/Database/IncludesAll.test.ts +65 -0
- package/Tests/Types/Database/IncludesNone.test.ts +65 -0
- package/Tests/Types/Database/IsNull.test.ts +44 -0
- package/Tests/Types/Database/LessThan.test.ts +62 -0
- package/Tests/Types/Database/LessThanOrEqual.test.ts +62 -0
- package/Tests/Types/Database/LessThanOrNull.test.ts +62 -0
- package/Tests/Types/Database/MultiSearch.test.ts +81 -0
- package/Tests/Types/Database/NotContains.test.ts +64 -0
- package/Tests/Types/Database/NotNull.test.ts +44 -0
- package/Tests/Types/Database/StartsWith.test.ts +64 -0
- package/Tests/Types/Monitor/MonitorStepSqlMonitor.test.ts +83 -1
- package/Tests/Types/Monitor/SnmpV3EnumParsing.test.ts +247 -0
- package/Tests/Types/Monitor/SnmpVendorTemplate.test.ts +263 -0
- package/Tests/UI/Components/XAxis.test.ts +5 -3
- package/Tests/Utils/EnterpriseLicenseUsage.test.ts +76 -0
- package/Tests/Utils/Metrics/MetricExplorerUrl.test.ts +622 -0
- package/Tests/Utils/Metrics/MetricFormulaEvaluator.test.ts +474 -0
- package/Tests/Utils/Telemetry/HeartbeatAvailability.test.ts +52 -4
- package/Types/AnalyticsDatabase/AnalyticsTableName.ts +11 -0
- package/Types/BaseDatabase/AggregateBy.ts +31 -2
- package/Types/BaseDatabase/AggregatedResult.ts +22 -0
- package/Types/BaseDatabase/AggregationInterval.ts +11 -0
- package/Types/BaseDatabase/AggregationIntervalUtil.ts +30 -5
- package/Types/Dashboard/DashboardComponents/DashboardTraceChartComponent.ts +16 -0
- package/Types/Email/EmailTemplateType.ts +3 -0
- package/Types/Marketing/MarketingConversion.ts +11 -0
- package/Types/Metrics/MetricQueryConfigData.ts +21 -0
- package/Types/Metrics/MetricQueryData.ts +8 -0
- package/Types/Metrics/MetricViewData.ts +8 -0
- package/Types/Monitor/MonitorMetricType.ts +8 -0
- package/Types/Monitor/MonitorStep.ts +9 -0
- package/Types/Monitor/MonitorStepSqlMonitor.ts +12 -0
- package/Types/Monitor/MonitorType.ts +2 -1
- package/Types/Monitor/SSLMonitor/SslMonitorResponse.ts +17 -12
- package/Types/Monitor/SnmpMonitor/CdpNeighbor.ts +13 -0
- package/Types/Monitor/SnmpMonitor/NetworkTopology.ts +37 -3
- package/Types/Monitor/SnmpMonitor/SnmpAuthProtocol.ts +46 -0
- package/Types/Monitor/SnmpMonitor/SnmpEntityInfo.ts +15 -0
- package/Types/Monitor/SnmpMonitor/SnmpInterface.ts +8 -0
- package/Types/Monitor/SnmpMonitor/SnmpMonitorResponse.ts +17 -8
- package/Types/Monitor/SnmpMonitor/SnmpPrivProtocol.ts +53 -0
- package/Types/Monitor/SnmpMonitor/SnmpSecurityLevel.ts +43 -0
- package/Types/Monitor/SnmpMonitor/SnmpSystemInfo.ts +16 -0
- package/Types/Monitor/SnmpMonitor/SnmpVendorTemplate.ts +119 -3
- package/Types/NetFlow/NetworkFlowRecord.ts +25 -0
- package/Types/Syslog/SyslogMessage.ts +18 -0
- package/Types/Telemetry/TelemetrySavedViewState.ts +9 -0
- package/Types/Telemetry/TelemetrySavedViewType.ts +12 -0
- package/UI/Components/Charts/Area/AreaChart.tsx +51 -0
- package/UI/Components/Charts/Bar/BarChart.tsx +51 -1
- package/UI/Components/Charts/ChartGroup/ChartGroup.tsx +97 -26
- package/UI/Components/Charts/ChartLibrary/AreaChart/AreaChart.tsx +328 -1
- package/UI/Components/Charts/ChartLibrary/BarChart/BarChart.tsx +115 -0
- package/UI/Components/Charts/ChartLibrary/LineChart/LineChart.tsx +328 -1
- package/UI/Components/Charts/ChartLibrary/Types/ChartDataPoint.ts +7 -0
- package/UI/Components/Charts/ChartLibrary/Types/FormattedReferenceRegion.ts +15 -0
- package/UI/Components/Charts/ChartLibrary/Types/FormattedTimeReferenceLine.ts +13 -0
- package/UI/Components/Charts/Line/LineChart.tsx +51 -0
- package/UI/Components/Charts/Types/ReferenceRegionProps.ts +12 -0
- package/UI/Components/Charts/Types/TimeReferenceLineProps.ts +12 -0
- package/UI/Components/Charts/Types/XAxis/XAxisPrecision.ts +1 -0
- package/UI/Components/Charts/Utils/DataPoint.ts +0 -0
- package/UI/Components/Charts/Utils/TimeAnnotation.ts +169 -0
- package/UI/Components/Charts/Utils/XAxis.ts +28 -4
- package/UI/Components/Icon/Icon.tsx +33 -13
- package/UI/Components/Markdown.tsx/MarkdownViewer.tsx +9 -0
- package/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.ts +67 -0
- package/UI/Components/Page/ModelPage.tsx +6 -1
- package/UI/Components/TelemetryViewer/components/SavedViewsDropdown.tsx +33 -4
- package/UI/Styles/Theme.css +10 -0
- package/UI/Utils/User.ts +41 -2
- package/Utils/Analytics.ts +8 -6
- package/Utils/EnterpriseLicense/EnterpriseLicenseUsage.ts +74 -0
- package/Utils/Metrics/MetricExplorerUrl.ts +771 -0
- package/Utils/Metrics/MetricFormulaEvaluator.ts +410 -46
- package/Utils/Monitor/NetworkTopologyUtil.ts +261 -49
- package/Utils/ValueFormatter.ts +22 -1
- package/build/dist/Models/AnalyticsModels/Index.js +13 -0
- package/build/dist/Models/AnalyticsModels/Index.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByContainer.js +159 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByContainer.js.map +1 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.js +158 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.js.map +1 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByService.js +160 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByService.js.map +1 -0
- package/build/dist/Models/AnalyticsModels/NetworkFlow.js +351 -0
- package/build/dist/Models/AnalyticsModels/NetworkFlow.js.map +1 -0
- package/build/dist/Models/DatabaseModels/EnterpriseLicenseInstance.js +19 -0
- package/build/dist/Models/DatabaseModels/EnterpriseLicenseInstance.js.map +1 -1
- package/build/dist/Models/DatabaseModels/GlobalConfig.js +38 -0
- package/build/dist/Models/DatabaseModels/GlobalConfig.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +2 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/MarketingConversion.js +219 -0
- package/build/dist/Models/DatabaseModels/MarketingConversion.js.map +1 -0
- package/build/dist/Models/DatabaseModels/MetricSavedView.js +35 -0
- package/build/dist/Models/DatabaseModels/MetricSavedView.js.map +1 -1
- package/build/dist/Models/DatabaseModels/MetricType.js +76 -0
- package/build/dist/Models/DatabaseModels/MetricType.js.map +1 -1
- package/build/dist/Models/DatabaseModels/NetworkDevice.js +341 -0
- package/build/dist/Models/DatabaseModels/NetworkDevice.js.map +1 -1
- package/build/dist/Models/DatabaseModels/NetworkDeviceDiscoveryScan.js +116 -0
- package/build/dist/Models/DatabaseModels/NetworkDeviceDiscoveryScan.js.map +1 -1
- package/build/dist/Models/DatabaseModels/NetworkInterface.js +60 -0
- package/build/dist/Models/DatabaseModels/NetworkInterface.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Project.js +38 -0
- package/build/dist/Models/DatabaseModels/Project.js.map +1 -1
- package/build/dist/Models/DatabaseModels/User.js +38 -0
- package/build/dist/Models/DatabaseModels/User.js.map +1 -1
- package/build/dist/Server/API/BaseAnalyticsAPI.js +19 -0
- package/build/dist/Server/API/BaseAnalyticsAPI.js.map +1 -1
- package/build/dist/Server/API/EnterpriseLicenseAPI.js +8 -0
- package/build/dist/Server/API/EnterpriseLicenseAPI.js.map +1 -1
- package/build/dist/Server/EnvironmentConfig.js +51 -0
- package/build/dist/Server/EnvironmentConfig.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784191414522-AddCounterSemanticsToMetricType.js +27 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784191414522-AddCounterSemanticsToMetricType.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784191414600-AddViewTypeToMetricSavedView.js +21 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784191414600-AddViewTypeToMetricSavedView.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784211212164-AddNetworkDeviceInventoryAndDiscoverySchedule.js +47 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784211212164-AddNetworkDeviceInventoryAndDiscoverySchedule.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784218257664-AddEnterpriseLicenseNotificationColumns.js +16 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784218257664-AddEnterpriseLicenseNotificationColumns.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784293516000-AddAttributionColumnsToUserAndProject.js +18 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784293516000-AddAttributionColumnsToUserAndProject.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784298000000-AddMarketingConversionTable.js +20 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784298000000-AddMarketingConversionTable.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784401962564-MigrationName.js +40 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1784401962564-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +14 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +58 -4
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Services/IncidentService.js +12 -0
- package/build/dist/Server/Services/IncidentService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +10 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/MarketingConversionService.js +9 -0
- package/build/dist/Server/Services/MarketingConversionService.js.map +1 -0
- package/build/dist/Server/Services/MetricItemAggMV1mByContainerService.js +29 -0
- package/build/dist/Server/Services/MetricItemAggMV1mByContainerService.js.map +1 -0
- package/build/dist/Server/Services/MetricItemAggMV1mByK8sClusterService.js +29 -0
- package/build/dist/Server/Services/MetricItemAggMV1mByK8sClusterService.js.map +1 -0
- package/build/dist/Server/Services/MetricItemAggMV1mByServiceService.js +29 -0
- package/build/dist/Server/Services/MetricItemAggMV1mByServiceService.js.map +1 -0
- package/build/dist/Server/Services/MetricService.js +761 -125
- package/build/dist/Server/Services/MetricService.js.map +1 -1
- package/build/dist/Server/Services/MonitorService.js +18 -5
- package/build/dist/Server/Services/MonitorService.js.map +1 -1
- package/build/dist/Server/Services/NetworkDeviceOwnerTeamService.js +44 -0
- package/build/dist/Server/Services/NetworkDeviceOwnerTeamService.js.map +1 -1
- package/build/dist/Server/Services/NetworkDeviceOwnerUserService.js +88 -0
- package/build/dist/Server/Services/NetworkDeviceOwnerUserService.js.map +1 -1
- package/build/dist/Server/Services/NetworkFlowService.js +9 -0
- package/build/dist/Server/Services/NetworkFlowService.js.map +1 -0
- package/build/dist/Server/Services/OnCallDutyPolicyService.js +15 -6
- package/build/dist/Server/Services/OnCallDutyPolicyService.js.map +1 -1
- package/build/dist/Server/Services/ProjectService.js +107 -6
- package/build/dist/Server/Services/ProjectService.js.map +1 -1
- package/build/dist/Server/Services/SpanService.js +8 -6
- package/build/dist/Server/Services/SpanService.js.map +1 -1
- package/build/dist/Server/Services/StatusPageService.js +10 -0
- package/build/dist/Server/Services/StatusPageService.js.map +1 -1
- package/build/dist/Server/Services/TeamMemberService.js +11 -0
- package/build/dist/Server/Services/TeamMemberService.js.map +1 -1
- package/build/dist/Server/Services/UserService.js +52 -0
- package/build/dist/Server/Services/UserService.js.map +1 -1
- package/build/dist/Server/Types/AnalyticsDatabase/AggregateBy.js +33 -7
- package/build/dist/Server/Types/AnalyticsDatabase/AggregateBy.js.map +1 -1
- package/build/dist/Server/Utils/Attribution.js +74 -0
- package/build/dist/Server/Utils/Attribution.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/ConversionUploadProvider.js +40 -0
- package/build/dist/Server/Utils/Marketing/ConversionUploadProvider.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/ConversionUploadProviders.js +20 -0
- package/build/dist/Server/Utils/Marketing/ConversionUploadProviders.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/Providers/GoogleAds.js +177 -0
- package/build/dist/Server/Utils/Marketing/Providers/GoogleAds.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/Providers/LinkedIn.js +113 -0
- package/build/dist/Server/Utils/Marketing/Providers/LinkedIn.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/Providers/Meta.js +109 -0
- package/build/dist/Server/Utils/Marketing/Providers/Meta.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/Providers/MicrosoftAds.js +130 -0
- package/build/dist/Server/Utils/Marketing/Providers/MicrosoftAds.js.map +1 -0
- package/build/dist/Server/Utils/Marketing/Providers/Reddit.js +126 -0
- package/build/dist/Server/Utils/Marketing/Providers/Reddit.js.map +1 -0
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js +53 -9
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js +21 -10
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js +54 -11
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js +43 -3
- package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorTemplateUtil.js +68 -0
- package/build/dist/Server/Utils/Monitor/MonitorTemplateUtil.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/NetworkDeviceHydrationUtil.js +15 -0
- package/build/dist/Server/Utils/Monitor/NetworkDeviceHydrationUtil.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/NetworkInventoryUtil.js +100 -15
- package/build/dist/Server/Utils/Monitor/NetworkInventoryUtil.js.map +1 -1
- package/build/dist/Server/Utils/ProductAnalytics.js +111 -0
- package/build/dist/Server/Utils/ProductAnalytics.js.map +1 -0
- package/build/dist/Server/Utils/Telemetry/Telemetry.js +32 -5
- package/build/dist/Server/Utils/Telemetry/Telemetry.js.map +1 -1
- package/build/dist/Server/Utils/Telemetry/TelemetryEntity.js +17 -0
- package/build/dist/Server/Utils/Telemetry/TelemetryEntity.js.map +1 -1
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js +11 -0
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js.map +1 -1
- package/build/dist/Types/BaseDatabase/AggregationInterval.js +11 -0
- package/build/dist/Types/BaseDatabase/AggregationInterval.js.map +1 -1
- package/build/dist/Types/BaseDatabase/AggregationIntervalUtil.js +33 -5
- package/build/dist/Types/BaseDatabase/AggregationIntervalUtil.js.map +1 -1
- package/build/dist/Types/Email/EmailTemplateType.js +2 -0
- package/build/dist/Types/Email/EmailTemplateType.js.map +1 -1
- package/build/dist/Types/Marketing/MarketingConversion.js +13 -0
- package/build/dist/Types/Marketing/MarketingConversion.js.map +1 -0
- package/build/dist/Types/Monitor/MonitorMetricType.js +7 -0
- package/build/dist/Types/Monitor/MonitorMetricType.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorStep.js +6 -0
- package/build/dist/Types/Monitor/MonitorStep.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorStepSqlMonitor.js +3 -0
- package/build/dist/Types/Monitor/MonitorStepSqlMonitor.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorType.js +2 -1
- package/build/dist/Types/Monitor/MonitorType.js.map +1 -1
- package/build/dist/Types/Monitor/SnmpMonitor/CdpNeighbor.js +2 -0
- package/build/dist/Types/Monitor/SnmpMonitor/CdpNeighbor.js.map +1 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpAuthProtocol.js +41 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpAuthProtocol.js.map +1 -1
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpEntityInfo.js +2 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpEntityInfo.js.map +1 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpPrivProtocol.js +48 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpPrivProtocol.js.map +1 -1
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpSecurityLevel.js +38 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpSecurityLevel.js.map +1 -1
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpSystemInfo.js +2 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpSystemInfo.js.map +1 -0
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpVendorTemplate.js +96 -3
- package/build/dist/Types/Monitor/SnmpMonitor/SnmpVendorTemplate.js.map +1 -1
- package/build/dist/Types/NetFlow/NetworkFlowRecord.js +2 -0
- package/build/dist/Types/NetFlow/NetworkFlowRecord.js.map +1 -0
- package/build/dist/Types/Syslog/SyslogMessage.js +2 -0
- package/build/dist/Types/Syslog/SyslogMessage.js.map +1 -0
- package/build/dist/Types/Telemetry/TelemetrySavedViewState.js +0 -7
- package/build/dist/Types/Telemetry/TelemetrySavedViewState.js.map +1 -1
- package/build/dist/Types/Telemetry/TelemetrySavedViewType.js +13 -0
- package/build/dist/Types/Telemetry/TelemetrySavedViewType.js.map +1 -0
- package/build/dist/UI/Components/Charts/Area/AreaChart.js +25 -1
- package/build/dist/UI/Components/Charts/Area/AreaChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/Bar/BarChart.js +26 -2
- package/build/dist/UI/Components/Charts/Bar/BarChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js +36 -12
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/AreaChart/AreaChart.js +208 -6
- package/build/dist/UI/Components/Charts/ChartLibrary/AreaChart/AreaChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/BarChart/BarChart.js +52 -3
- package/build/dist/UI/Components/Charts/ChartLibrary/BarChart/BarChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/LineChart/LineChart.js +208 -6
- package/build/dist/UI/Components/Charts/ChartLibrary/LineChart/LineChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/Types/ChartDataPoint.js +6 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/Types/ChartDataPoint.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/Types/FormattedReferenceRegion.js +2 -0
- package/build/dist/UI/Components/Charts/ChartLibrary/Types/FormattedReferenceRegion.js.map +1 -0
- package/build/dist/UI/Components/Charts/ChartLibrary/Types/FormattedTimeReferenceLine.js +2 -0
- package/build/dist/UI/Components/Charts/ChartLibrary/Types/FormattedTimeReferenceLine.js.map +1 -0
- package/build/dist/UI/Components/Charts/Line/LineChart.js +25 -1
- package/build/dist/UI/Components/Charts/Line/LineChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/Types/ReferenceRegionProps.js +2 -0
- package/build/dist/UI/Components/Charts/Types/ReferenceRegionProps.js.map +1 -0
- package/build/dist/UI/Components/Charts/Types/TimeReferenceLineProps.js +2 -0
- package/build/dist/UI/Components/Charts/Types/TimeReferenceLineProps.js.map +1 -0
- package/build/dist/UI/Components/Charts/Types/XAxis/XAxisPrecision.js +1 -0
- package/build/dist/UI/Components/Charts/Types/XAxis/XAxisPrecision.js.map +1 -1
- package/build/dist/UI/Components/Charts/Utils/DataPoint.js +0 -0
- package/build/dist/UI/Components/Charts/Utils/DataPoint.js.map +1 -1
- package/build/dist/UI/Components/Charts/Utils/TimeAnnotation.js +121 -0
- package/build/dist/UI/Components/Charts/Utils/TimeAnnotation.js.map +1 -0
- package/build/dist/UI/Components/Charts/Utils/XAxis.js +27 -4
- package/build/dist/UI/Components/Charts/Utils/XAxis.js.map +1 -1
- package/build/dist/UI/Components/Icon/Icon.js +15 -8
- package/build/dist/UI/Components/Icon/Icon.js.map +1 -1
- package/build/dist/UI/Components/Markdown.tsx/MarkdownViewer.js.map +1 -1
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.js +58 -0
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.js.map +1 -1
- package/build/dist/UI/Components/Page/ModelPage.js +7 -1
- package/build/dist/UI/Components/Page/ModelPage.js.map +1 -1
- package/build/dist/UI/Components/TelemetryViewer/components/SavedViewsDropdown.js +10 -4
- package/build/dist/UI/Components/TelemetryViewer/components/SavedViewsDropdown.js.map +1 -1
- package/build/dist/UI/Utils/User.js +35 -2
- package/build/dist/UI/Utils/User.js.map +1 -1
- package/build/dist/Utils/Analytics.js +8 -5
- package/build/dist/Utils/Analytics.js.map +1 -1
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseUsage.js +57 -0
- package/build/dist/Utils/EnterpriseLicense/EnterpriseLicenseUsage.js.map +1 -1
- package/build/dist/Utils/Metrics/MetricExplorerUrl.js +418 -0
- package/build/dist/Utils/Metrics/MetricExplorerUrl.js.map +1 -0
- package/build/dist/Utils/Metrics/MetricFormulaEvaluator.js +296 -37
- package/build/dist/Utils/Metrics/MetricFormulaEvaluator.js.map +1 -1
- package/build/dist/Utils/Monitor/NetworkTopologyUtil.js +176 -40
- package/build/dist/Utils/Monitor/NetworkTopologyUtil.js.map +1 -1
- package/build/dist/Utils/ValueFormatter.js +20 -1
- package/build/dist/Utils/ValueFormatter.js.map +1 -1
- package/package.json +4 -3
|
@@ -7,8 +7,13 @@ import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper
|
|
|
7
7
|
import AggregationType, { getPercentileLevel, isPercentileAggregation, } from "../../Types/BaseDatabase/AggregationType";
|
|
8
8
|
import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
|
|
9
9
|
import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
|
|
10
|
+
import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
|
|
10
11
|
import BadDataException from "../../Types/Exception/BadDataException";
|
|
11
|
-
import { keyForHost } from "../../Utils/Telemetry/EntityKey";
|
|
12
|
+
import { canonicalizeEntityValue, keyForHost, keyForKubernetesCluster, keyForService, } from "../../Utils/Telemetry/EntityKey";
|
|
13
|
+
import { keyForContainer } from "../Utils/Telemetry/TelemetryEntity";
|
|
14
|
+
import { getEntityBudget } from "../Utils/Telemetry/EntityRegistry";
|
|
15
|
+
import TelemetryEntityService from "./TelemetryEntityService";
|
|
16
|
+
import EntityType from "../../Types/Telemetry/EntityType";
|
|
12
17
|
import ObjectID from "../../Types/ObjectID";
|
|
13
18
|
import logger from "../Utils/Logger";
|
|
14
19
|
/*
|
|
@@ -30,6 +35,42 @@ const MAX_GROUP_BY_ATTRIBUTE_KEYS = 10;
|
|
|
30
35
|
const MAX_GROUP_BY_ATTRIBUTE_KEY_LENGTH = 256;
|
|
31
36
|
const POINT_TYPE_CACHE_TTL_MS = 10 * 60 * 1000;
|
|
32
37
|
const POINT_TYPE_CACHE_MAX_ENTRIES = 5000;
|
|
38
|
+
/*
|
|
39
|
+
* Deliberately short: the registry gains a row for a NEW namespaced
|
|
40
|
+
* service variant asynchronously (throttled reconcile), and until this
|
|
41
|
+
* cache expires a routed service query would keep serving the stale
|
|
42
|
+
* (smaller) key set. 60s bounds that staleness to about one dashboard
|
|
43
|
+
* refresh while still collapsing the burst of per-metric aggregates a
|
|
44
|
+
* single refresh fires.
|
|
45
|
+
*/
|
|
46
|
+
const SERVICE_ENTITY_KEYS_CACHE_TTL_MS = 60 * 1000;
|
|
47
|
+
const SERVICE_ENTITY_KEYS_CACHE_MAX_ENTRIES = 5000;
|
|
48
|
+
const ENTITY_MV_ROUTES = [
|
|
49
|
+
{
|
|
50
|
+
attributeKey: "resource.host.name",
|
|
51
|
+
tableName: AnalyticsTableName.MetricItemAggMV1mByHostV2,
|
|
52
|
+
keyColumn: "hostEntityKey",
|
|
53
|
+
keyForValue: keyForHost,
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
attributeKey: "resource.k8s.cluster.name",
|
|
57
|
+
tableName: AnalyticsTableName.MetricItemAggMV1mByK8sCluster,
|
|
58
|
+
keyColumn: "k8sClusterEntityKey",
|
|
59
|
+
keyForValue: keyForKubernetesCluster,
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
attributeKey: "resource.container.id",
|
|
63
|
+
tableName: AnalyticsTableName.MetricItemAggMV1mByContainer,
|
|
64
|
+
keyColumn: "containerEntityKey",
|
|
65
|
+
keyForValue: keyForContainer,
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
attributeKey: "resource.service.name",
|
|
69
|
+
tableName: AnalyticsTableName.MetricItemAggMV1mByService,
|
|
70
|
+
keyColumn: "serviceEntityKey",
|
|
71
|
+
keyForValue: null,
|
|
72
|
+
},
|
|
73
|
+
];
|
|
33
74
|
export class MetricService extends AnalyticsDatabaseService {
|
|
34
75
|
constructor(clickhouseDatabase) {
|
|
35
76
|
super({ modelType: Metric, database: clickhouseDatabase });
|
|
@@ -48,6 +89,23 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
48
89
|
*/
|
|
49
90
|
this.pointTypeHintByAggregate = new WeakMap();
|
|
50
91
|
this.inFlightPointTypeLookups = new Map();
|
|
92
|
+
/*
|
|
93
|
+
* Registry-resolved serviceEntityKey set per aggregateBy call, for
|
|
94
|
+
* queries filtering by `resource.service.name`. Same hand-over pattern
|
|
95
|
+
* as the point-type hint: toAggregateStatement is synchronous, so the
|
|
96
|
+
* async Postgres lookup happens in aggregateBy and is keyed on the
|
|
97
|
+
* (shared) aggregateBy object. An empty array means "registry has no
|
|
98
|
+
* rows for this name" — the builder then refuses to route (raw path).
|
|
99
|
+
*/
|
|
100
|
+
this.serviceEntityKeysHintByAggregate = new WeakMap();
|
|
101
|
+
/*
|
|
102
|
+
* (projectId|canonical service name) -> serviceEntityKey set. Bounded +
|
|
103
|
+
* short-TTL'd (see SERVICE_ENTITY_KEYS_CACHE_TTL_MS): a dashboard
|
|
104
|
+
* refresh fires one aggregate per metric name, all for the same
|
|
105
|
+
* service, and each would otherwise hit Postgres.
|
|
106
|
+
*/
|
|
107
|
+
this.serviceEntityKeysCache = new Map();
|
|
108
|
+
this.inFlightServiceEntityKeyLookups = new Map();
|
|
51
109
|
}
|
|
52
110
|
async aggregateBy(aggregateBy) {
|
|
53
111
|
/*
|
|
@@ -64,9 +122,197 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
64
122
|
aggregateBy.groupByAttributeKeys.length === 0)) {
|
|
65
123
|
const pointType = await this.resolveMetricPointType(aggregateBy);
|
|
66
124
|
this.pointTypeHintByAggregate.set(aggregateBy, pointType);
|
|
125
|
+
/*
|
|
126
|
+
* Service-scoped queries need the registry-resolved serviceEntityKey
|
|
127
|
+
* set before the synchronous statement builder runs. Skipped when
|
|
128
|
+
* the point type already rules out MV routing (distribution
|
|
129
|
+
* metrics never touch the rollups).
|
|
130
|
+
*/
|
|
131
|
+
if (!pointType || !DISTRIBUTION_POINT_TYPES.includes(pointType)) {
|
|
132
|
+
const serviceKeys = await this.resolveServiceEntityKeysHint(aggregateBy);
|
|
133
|
+
if (serviceKeys) {
|
|
134
|
+
this.serviceEntityKeysHintByAggregate.set(aggregateBy, serviceKeys);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
67
137
|
}
|
|
68
138
|
return super.aggregateBy(aggregateBy);
|
|
69
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* Resolves the serviceEntityKey set for a query whose only entity
|
|
142
|
+
* filter is a `resource.service.name` equality — the shape (and the
|
|
143
|
+
* only shape) the entity-MV builder routes to the per-service rollup.
|
|
144
|
+
* Returns null for every other query shape so no lookup is wasted.
|
|
145
|
+
*/
|
|
146
|
+
async resolveServiceEntityKeysHint(aggregateBy) {
|
|
147
|
+
const queryRecord = aggregateBy.query || {};
|
|
148
|
+
// service + entityScope is never routed (see the entity-MV builder).
|
|
149
|
+
if (queryRecord["entityScope"] !== undefined &&
|
|
150
|
+
queryRecord["entityScope"] !== null) {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
const attrs = queryRecord["attributes"];
|
|
154
|
+
if (!attrs || typeof attrs !== "object") {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
const attrEntries = Object.entries(attrs);
|
|
158
|
+
if (attrEntries.length !== 1) {
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
const [attrKey, attrValue] = attrEntries[0];
|
|
162
|
+
if (attrKey !== "resource.service.name" ||
|
|
163
|
+
typeof attrValue !== "string" ||
|
|
164
|
+
!attrValue) {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
const projectIdValue = queryRecord["projectId"];
|
|
168
|
+
let projectId = "";
|
|
169
|
+
if (projectIdValue instanceof ObjectID) {
|
|
170
|
+
projectId = projectIdValue.toString();
|
|
171
|
+
}
|
|
172
|
+
else if (typeof projectIdValue === "string") {
|
|
173
|
+
projectId = projectIdValue;
|
|
174
|
+
}
|
|
175
|
+
if (!projectId) {
|
|
176
|
+
return null;
|
|
177
|
+
}
|
|
178
|
+
const cacheKey = `${projectId}|${canonicalizeEntityValue(attrValue)}`;
|
|
179
|
+
const cached = this.serviceEntityKeysCache.get(cacheKey);
|
|
180
|
+
if (cached && cached.expiresAt > Date.now()) {
|
|
181
|
+
return cached.keys;
|
|
182
|
+
}
|
|
183
|
+
const inFlight = this.inFlightServiceEntityKeyLookups.get(cacheKey);
|
|
184
|
+
if (inFlight) {
|
|
185
|
+
return inFlight;
|
|
186
|
+
}
|
|
187
|
+
const lookup = this.lookupServiceEntityKeys(projectId, attrValue)
|
|
188
|
+
.then((keys) => {
|
|
189
|
+
this.evictExpiredServiceEntityKeyCacheEntries();
|
|
190
|
+
this.serviceEntityKeysCache.set(cacheKey, {
|
|
191
|
+
keys,
|
|
192
|
+
expiresAt: Date.now() + SERVICE_ENTITY_KEYS_CACHE_TTL_MS,
|
|
193
|
+
});
|
|
194
|
+
return keys;
|
|
195
|
+
})
|
|
196
|
+
.finally(() => {
|
|
197
|
+
this.inFlightServiceEntityKeyLookups.delete(cacheKey);
|
|
198
|
+
});
|
|
199
|
+
this.inFlightServiceEntityKeyLookups.set(cacheKey, lookup);
|
|
200
|
+
return lookup;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* All serviceEntityKeys the ingest pipeline can have stamped for rows
|
|
204
|
+
* reporting this `service.name`, from the Postgres TelemetryEntity
|
|
205
|
+
* registry. Service identity folds `service.namespace` into the key
|
|
206
|
+
* when the resource carries one (see the service resolver in
|
|
207
|
+
* Common/Server/Utils/Telemetry/TelemetryEntity), so one name maps to
|
|
208
|
+
* one key per namespace variant — only the registry knows them all.
|
|
209
|
+
*
|
|
210
|
+
* `displayName` equals the canonicalized service.name for every
|
|
211
|
+
* Service row the registry mints (deriveDisplayName prefers the
|
|
212
|
+
* `service.name` identifying attribute), so the query is indexed and
|
|
213
|
+
* tiny; the identifyingAttributes re-check below guards against a
|
|
214
|
+
* displayName that came from some other identity shape. Returns []
|
|
215
|
+
* on a registry miss, on any lookup failure, or when the project's
|
|
216
|
+
* Service registry is at/over its entity budget (new variants stop
|
|
217
|
+
* minting rows there, so the key set can be permanently partial) —
|
|
218
|
+
* the caller then refuses to route (an MV query must never return
|
|
219
|
+
* less data than the raw predicate it replaces, and rows of a
|
|
220
|
+
* namespaced service the registry does not know about would be
|
|
221
|
+
* silently dropped).
|
|
222
|
+
*/
|
|
223
|
+
async lookupServiceEntityKeys(projectId, serviceName) {
|
|
224
|
+
try {
|
|
225
|
+
const canonicalName = canonicalizeEntityValue(serviceName);
|
|
226
|
+
const rows = await TelemetryEntityService.findBy({
|
|
227
|
+
query: {
|
|
228
|
+
projectId: new ObjectID(projectId),
|
|
229
|
+
entityType: EntityType.Service,
|
|
230
|
+
displayName: canonicalName,
|
|
231
|
+
},
|
|
232
|
+
select: {
|
|
233
|
+
entityKey: true,
|
|
234
|
+
identifyingAttributes: true,
|
|
235
|
+
},
|
|
236
|
+
limit: LIMIT_PER_PROJECT,
|
|
237
|
+
skip: 0,
|
|
238
|
+
props: { isRoot: true },
|
|
239
|
+
});
|
|
240
|
+
const keys = new Set();
|
|
241
|
+
for (const row of rows) {
|
|
242
|
+
const identifying = row.identifyingAttributes || {};
|
|
243
|
+
if (identifying["service.name"] !== canonicalName) {
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
if (typeof row.entityKey === "string" && row.entityKey) {
|
|
247
|
+
keys.add(row.entityKey);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
if (keys.size === 0) {
|
|
251
|
+
return [];
|
|
252
|
+
}
|
|
253
|
+
/*
|
|
254
|
+
* Budget-capped projects must not route. At/over the per-type
|
|
255
|
+
* entity budget, ingest stops minting NEW Service registry rows
|
|
256
|
+
* forever (TelemetryEntityService.beforeCreate) while namespaced
|
|
257
|
+
* serviceEntityKeys keep flowing on signal rows — so a non-empty
|
|
258
|
+
* registry key set may be PERMANENTLY missing identity variants,
|
|
259
|
+
* and a routed `serviceEntityKey IN (...)` would silently return
|
|
260
|
+
* less data than the raw attributes predicate it replaces (the
|
|
261
|
+
* invariant documented above). Same signal as the ingest-side
|
|
262
|
+
* gate; the [] is cached for 60s alongside key hits, so the extra
|
|
263
|
+
* countBy amortizes to one per (project, service name) per minute.
|
|
264
|
+
*/
|
|
265
|
+
const serviceEntityCount = await TelemetryEntityService.countBy({
|
|
266
|
+
query: {
|
|
267
|
+
projectId: new ObjectID(projectId),
|
|
268
|
+
entityType: EntityType.Service,
|
|
269
|
+
},
|
|
270
|
+
props: { isRoot: true },
|
|
271
|
+
});
|
|
272
|
+
if (serviceEntityCount.toNumber() >= getEntityBudget(EntityType.Service)) {
|
|
273
|
+
logger.debug("Service entity registry is budget-capped; refusing MV routing");
|
|
274
|
+
return [];
|
|
275
|
+
}
|
|
276
|
+
/*
|
|
277
|
+
* Union in the deterministic namespace-less key: a service that
|
|
278
|
+
* ALSO reports without a namespace may not have minted its bare
|
|
279
|
+
* registry row yet (reconcile is throttled and budget-gated), and
|
|
280
|
+
* adding a key can only widen the MV result, never narrow it.
|
|
281
|
+
*/
|
|
282
|
+
keys.add(keyForService(projectId, serviceName));
|
|
283
|
+
return Array.from(keys).sort();
|
|
284
|
+
}
|
|
285
|
+
catch (err) {
|
|
286
|
+
/*
|
|
287
|
+
* Lookup failures must not fail the aggregation — without the hint
|
|
288
|
+
* the builder falls back to the raw table, which is always correct.
|
|
289
|
+
*/
|
|
290
|
+
logger.debug("Service entity key lookup failed");
|
|
291
|
+
logger.debug(err);
|
|
292
|
+
return [];
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
evictExpiredServiceEntityKeyCacheEntries() {
|
|
296
|
+
if (this.serviceEntityKeysCache.size < SERVICE_ENTITY_KEYS_CACHE_MAX_ENTRIES) {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const now = Date.now();
|
|
300
|
+
for (const [key, entry] of this.serviceEntityKeysCache) {
|
|
301
|
+
if (entry.expiresAt <= now) {
|
|
302
|
+
this.serviceEntityKeysCache.delete(key);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
// Mirrors evictExpiredPointTypeCacheEntries: never clear() wholesale.
|
|
306
|
+
while (this.serviceEntityKeysCache.size >= SERVICE_ENTITY_KEYS_CACHE_MAX_ENTRIES) {
|
|
307
|
+
const oldestKey = this.serviceEntityKeysCache
|
|
308
|
+
.keys()
|
|
309
|
+
.next().value;
|
|
310
|
+
if (oldestKey === undefined) {
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
this.serviceEntityKeysCache.delete(oldestKey);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
70
316
|
async resolveMetricPointType(aggregateBy) {
|
|
71
317
|
const metricName = this.getExactMetricNameFromQuery(aggregateBy.query);
|
|
72
318
|
if (!metricName) {
|
|
@@ -244,6 +490,117 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
244
490
|
});
|
|
245
491
|
statement.append(`) AS attributes`);
|
|
246
492
|
}
|
|
493
|
+
/**
|
|
494
|
+
* Validated Top-K spec, or null when the caller did not request one.
|
|
495
|
+
* `count` is parameter-bound where it reaches SQL and `rankBy` only
|
|
496
|
+
* ever selects between two hardcoded aggregate names, so this is
|
|
497
|
+
* shape validation (like getSanitizedGroupByAttributeKeys), not
|
|
498
|
+
* injection defense.
|
|
499
|
+
*/
|
|
500
|
+
getSanitizedTopK(aggregateBy) {
|
|
501
|
+
const topK = aggregateBy.topK;
|
|
502
|
+
if (!topK) {
|
|
503
|
+
return null;
|
|
504
|
+
}
|
|
505
|
+
const count = Math.floor(Number(topK.count));
|
|
506
|
+
if (!Number.isFinite(count) || count < 1 || count > LIMIT_PER_PROJECT) {
|
|
507
|
+
throw new BadDataException(`topK.count must be a positive integer <= ${LIMIT_PER_PROJECT}.`);
|
|
508
|
+
}
|
|
509
|
+
if (topK.rankBy !== "max" && topK.rankBy !== "avg") {
|
|
510
|
+
throw new BadDataException(`topK.rankBy must be 'max' or 'avg'.`);
|
|
511
|
+
}
|
|
512
|
+
return { count, rankBy: topK.rankBy };
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Appends the raw-table expressions that identify a group — model
|
|
516
|
+
* group-by columns as identifiers, attribute keys as parameter-bound
|
|
517
|
+
* `attributes[...]` lookups — in a FIXED order, so the Top-K
|
|
518
|
+
* IN-restriction tuple, the ranking subquery's SELECT/GROUP BY, and
|
|
519
|
+
* the totalGroups counter always line up element-for-element.
|
|
520
|
+
*/
|
|
521
|
+
appendRawGroupExpressionList(statement, groupByKeys, attributeGroupKeys) {
|
|
522
|
+
let isFirst = true;
|
|
523
|
+
for (const key of groupByKeys) {
|
|
524
|
+
if (!isFirst) {
|
|
525
|
+
statement.append(`, `);
|
|
526
|
+
}
|
|
527
|
+
statement.append(key);
|
|
528
|
+
isFirst = false;
|
|
529
|
+
}
|
|
530
|
+
for (const key of attributeGroupKeys) {
|
|
531
|
+
if (!isFirst) {
|
|
532
|
+
statement.append(`, `);
|
|
533
|
+
}
|
|
534
|
+
statement.append(SQL `attributes[${{ value: key, type: TableColumnType.Text }}]`);
|
|
535
|
+
isFirst = false;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Appends the Top-K group restriction predicate:
|
|
540
|
+
*
|
|
541
|
+
* ` AND (<group exprs>) GLOBAL IN (SELECT <group exprs> FROM <table>
|
|
542
|
+
* WHERE ... GROUP BY <group exprs>
|
|
543
|
+
* ORDER BY max|avg(<column>) DESC LIMIT <k>)`
|
|
544
|
+
*
|
|
545
|
+
* The ranking subquery scores each group over the WHOLE query window
|
|
546
|
+
* (no time bucketing) so a series that spiked early ranks the same as
|
|
547
|
+
* one spiking late. Must be appended in raw-table WHERE scope — the
|
|
548
|
+
* innermost WHERE of the surrounding builder — where the group
|
|
549
|
+
* expressions are valid. Ranking uses plain max/avg of the raw
|
|
550
|
+
* column (not the distribution-aware expressions): for distribution
|
|
551
|
+
* metrics that scores by per-export-interval sums, which preserves
|
|
552
|
+
* relative group ordering well enough for series selection.
|
|
553
|
+
*
|
|
554
|
+
* GLOBAL IN is load-bearing: this predicate sits in a query on the
|
|
555
|
+
* Distributed Metric table whose subquery reads the SAME Distributed
|
|
556
|
+
* table, which multi-shard ClickHouse rejects outright as a
|
|
557
|
+
* double-distributed subquery (Code 288, distributed_product_mode =
|
|
558
|
+
* 'deny' by default) — every grouped Top-K chart would 500 on a
|
|
559
|
+
* 2+-shard cluster. A shard-local rewrite would be wrong anyway: the
|
|
560
|
+
* sharding key is cityHash64(projectId, name, primaryEntityId), so
|
|
561
|
+
* one project+metric's groups span shards and the ranking must be
|
|
562
|
+
* computed once globally, not per shard. On a single shard GLOBAL is
|
|
563
|
+
* a semantic no-op. (The mutable-metric twin below deliberately stays
|
|
564
|
+
* plain IN: its predicate lives in an initiator-evaluated outer query
|
|
565
|
+
* over a subquery-FROM, which is not subject to the denial.)
|
|
566
|
+
*/
|
|
567
|
+
appendTopKGroupRestriction(data) {
|
|
568
|
+
const statement = data.statement;
|
|
569
|
+
const aggregationColumn = data.aggregateBy.aggregateColumnName.toString();
|
|
570
|
+
statement.append(` AND (`);
|
|
571
|
+
this.appendRawGroupExpressionList(statement, data.groupByKeys, data.attributeGroupKeys);
|
|
572
|
+
statement.append(`) GLOBAL IN (SELECT `);
|
|
573
|
+
this.appendRawGroupExpressionList(statement, data.groupByKeys, data.attributeGroupKeys);
|
|
574
|
+
statement.append(` FROM ${data.databaseName}.${this.model.tableName} WHERE TRUE `);
|
|
575
|
+
statement.append(this.statementGenerator.toWhereStatement(data.aggregateBy.query));
|
|
576
|
+
statement.append(this.getRetentionReadFilter());
|
|
577
|
+
statement.append(` GROUP BY `);
|
|
578
|
+
this.appendRawGroupExpressionList(statement, data.groupByKeys, data.attributeGroupKeys);
|
|
579
|
+
statement.append(` ORDER BY ${data.topK.rankBy}(${aggregationColumn}) DESC`);
|
|
580
|
+
statement.append(SQL ` LIMIT ${{
|
|
581
|
+
value: data.topK.count,
|
|
582
|
+
type: TableColumnType.Number,
|
|
583
|
+
}}`);
|
|
584
|
+
statement.append(`)`);
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Appends `, (SELECT uniqExact(<group exprs>) FROM <table>
|
|
588
|
+
* WHERE ...) AS __total_groups` — a scalar subquery counting every
|
|
589
|
+
* distinct group in the window BEFORE Top-K trims them. The value is
|
|
590
|
+
* constant across result rows; AnalyticsDatabaseService._aggregateBy
|
|
591
|
+
* reads it off the first row into AggregatedResult.totalGroups and it
|
|
592
|
+
* never reaches the per-row payload. Must be appended in SELECT-list
|
|
593
|
+
* position of the outer query.
|
|
594
|
+
*/
|
|
595
|
+
appendTotalGroupsColumn(data) {
|
|
596
|
+
const statement = data.statement;
|
|
597
|
+
statement.append(`, (SELECT uniqExact(`);
|
|
598
|
+
this.appendRawGroupExpressionList(statement, data.groupByKeys, data.attributeGroupKeys);
|
|
599
|
+
statement.append(`) FROM ${data.databaseName}.${this.model.tableName} WHERE TRUE `);
|
|
600
|
+
statement.append(this.statementGenerator.toWhereStatement(data.aggregateBy.query));
|
|
601
|
+
statement.append(this.getRetentionReadFilter());
|
|
602
|
+
statement.append(`) AS __total_groups`);
|
|
603
|
+
}
|
|
247
604
|
/*
|
|
248
605
|
* Cascade deletes from `MetricItemV3` into the aggregating
|
|
249
606
|
* materialized-view target tables.
|
|
@@ -260,10 +617,11 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
260
617
|
* The cascade only runs when the caller scoped the delete by
|
|
261
618
|
* `primaryEntityId`. Global time-based purges (TTL cleanup) are handled
|
|
262
619
|
* by each MV table's own `retentionDate TTL DELETE`, so cascading those
|
|
263
|
-
* would pointlessly scan the whole MV. The per-
|
|
264
|
-
* (`MetricItemAggMV1mByHostV2`
|
|
265
|
-
* `
|
|
266
|
-
*
|
|
620
|
+
* would pointlessly scan the whole MV. The per-entity MVs
|
|
621
|
+
* (`MetricItemAggMV1mByHostV2` / `...ByService` / `...ByK8sCluster` /
|
|
622
|
+
* `...ByContainer`) are keyed by their scalar entity-key column rather
|
|
623
|
+
* than `primaryEntityId`, so an entity-scoped delete has nothing to
|
|
624
|
+
* remove there — skip them.
|
|
267
625
|
*/
|
|
268
626
|
async deleteBy(deleteBy) {
|
|
269
627
|
await super.deleteBy(deleteBy);
|
|
@@ -374,11 +732,12 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
374
732
|
const attributeGroupKeys = this.getSanitizedGroupByAttributeKeys(aggregateBy);
|
|
375
733
|
if (!isPercentileAggregation(aggregateBy.aggregationType)) {
|
|
376
734
|
/*
|
|
377
|
-
* Try the per-
|
|
378
|
-
*
|
|
379
|
-
* the only
|
|
380
|
-
* (no
|
|
381
|
-
* to the project/primaryEntityId MV, then to the base
|
|
735
|
+
* Try the per-entity MVs first — entity detail pages (host, k8s
|
|
736
|
+
* cluster, container, service) are the dominant filtered path and
|
|
737
|
+
* the entity-keyed rollups are the only ones that can serve them.
|
|
738
|
+
* If none applies (no entity filter, or extra attrs/groupBy), fall
|
|
739
|
+
* through to the project/primaryEntityId MV, then to the base
|
|
740
|
+
* table.
|
|
382
741
|
*
|
|
383
742
|
* Distribution metrics (histograms/summaries) must skip the
|
|
384
743
|
* MVs entirely: their states collapse `value`, which for
|
|
@@ -388,9 +747,9 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
388
747
|
*/
|
|
389
748
|
if (attributeGroupKeys.length === 0 &&
|
|
390
749
|
!this.isDistributionMetricAggregate(aggregateBy)) {
|
|
391
|
-
const
|
|
392
|
-
if (
|
|
393
|
-
return
|
|
750
|
+
const entityMvStatement = this.tryBuildEntityAggregateMVStatement(aggregateBy);
|
|
751
|
+
if (entityMvStatement) {
|
|
752
|
+
return entityMvStatement;
|
|
394
753
|
}
|
|
395
754
|
const mvStatement = this.tryBuildMinuteAggregateMVStatement(aggregateBy);
|
|
396
755
|
if (mvStatement) {
|
|
@@ -528,12 +887,41 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
528
887
|
* attribute combination.
|
|
529
888
|
*/
|
|
530
889
|
this.appendAttributeGroupMapColumn(statement, attributeGroupKeys);
|
|
890
|
+
/*
|
|
891
|
+
* Server-side Top-K: rank groups over the whole window, restrict
|
|
892
|
+
* the bucketed quantile to the winners, and surface the pre-trim
|
|
893
|
+
* group count. Only meaningful when the aggregation is grouped —
|
|
894
|
+
* p95-by-host must get the same series selection as the scalar
|
|
895
|
+
* paths.
|
|
896
|
+
*/
|
|
897
|
+
const percentileTopK = this.getSanitizedTopK(aggregateBy);
|
|
898
|
+
const percentileApplyTopK = Boolean(percentileTopK &&
|
|
899
|
+
(groupByKeys.length > 0 || attributeGroupKeys.length > 0));
|
|
900
|
+
if (percentileApplyTopK) {
|
|
901
|
+
this.appendTotalGroupsColumn({
|
|
902
|
+
statement,
|
|
903
|
+
databaseName,
|
|
904
|
+
aggregateBy,
|
|
905
|
+
groupByKeys,
|
|
906
|
+
attributeGroupKeys,
|
|
907
|
+
});
|
|
908
|
+
}
|
|
531
909
|
statement.append(SQL ` FROM (`);
|
|
532
910
|
statement.append(`SELECT ${innerSelectClause}`);
|
|
533
911
|
this.appendAttributeGroupExtractionColumns(statement, attributeGroupKeys);
|
|
534
912
|
statement.append(` FROM ${databaseName}.${this.model.tableName} WHERE TRUE `);
|
|
535
913
|
statement.append(whereStatement);
|
|
536
914
|
statement.append(this.getRetentionReadFilter());
|
|
915
|
+
if (percentileApplyTopK) {
|
|
916
|
+
this.appendTopKGroupRestriction({
|
|
917
|
+
statement,
|
|
918
|
+
databaseName,
|
|
919
|
+
aggregateBy,
|
|
920
|
+
topK: percentileTopK,
|
|
921
|
+
groupByKeys,
|
|
922
|
+
attributeGroupKeys,
|
|
923
|
+
});
|
|
924
|
+
}
|
|
537
925
|
statement.append(SQL `) `);
|
|
538
926
|
/*
|
|
539
927
|
* `Total` collapses the window into one row per group: the timestamp
|
|
@@ -574,16 +962,23 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
574
962
|
}} `);
|
|
575
963
|
/*
|
|
576
964
|
* Match the read-path settings the base aggregator now appends (see
|
|
577
|
-
* AnalyticsDatabaseService.toAggregateStatement)
|
|
578
|
-
*
|
|
579
|
-
*
|
|
965
|
+
* AnalyticsDatabaseService.toAggregateStatement), including the 45s
|
|
966
|
+
* execution cap and the caller-selected overflow mode. The
|
|
967
|
+
* percentile path bypasses the base method, so we mirror them here
|
|
968
|
+
* to keep cluster behavior consistent across aggregation kinds.
|
|
969
|
+
*
|
|
970
|
+
* transform_null_in=1 rides along only when the Top-K restriction
|
|
971
|
+
* is present: grouping by a Nullable model column (metricPointType,
|
|
972
|
+
* unit, ...) puts NULL group keys in play, and under the ClickHouse
|
|
973
|
+
* default (transform_null_in=0) NULL never matches an IN — a
|
|
974
|
+
* NULL-keyed group could win a ranking slot yet return zero rows.
|
|
975
|
+
* Query-wide it is safe here: user-filter IN lists are
|
|
976
|
+
* parameter-bound scalar arrays that never carry NULL.
|
|
580
977
|
*/
|
|
581
978
|
statement.append(getQuerySettings({
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
max_threads: 4,
|
|
586
|
-
},
|
|
979
|
+
maxExecutionTimeInSeconds: 45,
|
|
980
|
+
timeoutOverflowMode: this.getTimeoutOverflowMode(aggregateBy),
|
|
981
|
+
additionalSettings: Object.assign({ optimize_aggregation_in_order: 1, optimize_move_to_prewhere: 1, max_threads: 4 }, (percentileApplyTopK ? { transform_null_in: 1 } : {})),
|
|
587
982
|
}));
|
|
588
983
|
const columns = [
|
|
589
984
|
aggregationColumn,
|
|
@@ -691,11 +1086,27 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
691
1086
|
const sortStatement = this.statementGenerator.toSortStatement(aggregateBy.sort);
|
|
692
1087
|
const aggregationExpression = this.getDistributionAwareAggregationExpression(aggregateBy.aggregationType, aggregationColumn);
|
|
693
1088
|
const statement = SQL ``;
|
|
1089
|
+
/*
|
|
1090
|
+
* Server-side Top-K: rank groups over the whole window, restrict
|
|
1091
|
+
* the bucketed aggregation to the winners, and surface the pre-trim
|
|
1092
|
+
* group count. Only meaningful when the aggregation is grouped.
|
|
1093
|
+
*/
|
|
1094
|
+
const scalarTopK = this.getSanitizedTopK(aggregateBy);
|
|
1095
|
+
const scalarApplyTopK = Boolean(scalarTopK && (groupByKeys.length > 0 || attributeGroupKeys.length > 0));
|
|
694
1096
|
statement.append(`SELECT ${aggregationExpression} as ${aggregationColumn}, ${AggregateUtil.buildBucketTimestampSelect(resolvedInterval, aggregationTimestampColumn)}`);
|
|
695
1097
|
for (const key of groupByKeys) {
|
|
696
1098
|
statement.append(`, ${key}`);
|
|
697
1099
|
}
|
|
698
1100
|
this.appendAttributeGroupMapColumn(statement, attributeGroupKeys);
|
|
1101
|
+
if (scalarApplyTopK) {
|
|
1102
|
+
this.appendTotalGroupsColumn({
|
|
1103
|
+
statement,
|
|
1104
|
+
databaseName,
|
|
1105
|
+
aggregateBy,
|
|
1106
|
+
groupByKeys,
|
|
1107
|
+
attributeGroupKeys,
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
699
1110
|
if (attributeGroupKeys.length > 0) {
|
|
700
1111
|
/*
|
|
701
1112
|
* Subquery form: extract the selected keys into `__attr_grp_<i>`
|
|
@@ -722,12 +1133,32 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
722
1133
|
statement.append(` FROM ${databaseName}.${this.model.tableName} WHERE TRUE `);
|
|
723
1134
|
statement.append(whereStatement);
|
|
724
1135
|
statement.append(this.getRetentionReadFilter());
|
|
1136
|
+
if (scalarApplyTopK) {
|
|
1137
|
+
this.appendTopKGroupRestriction({
|
|
1138
|
+
statement,
|
|
1139
|
+
databaseName,
|
|
1140
|
+
aggregateBy,
|
|
1141
|
+
topK: scalarTopK,
|
|
1142
|
+
groupByKeys,
|
|
1143
|
+
attributeGroupKeys,
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
725
1146
|
statement.append(SQL `) `);
|
|
726
1147
|
}
|
|
727
1148
|
else {
|
|
728
1149
|
statement.append(` FROM ${databaseName}.${this.model.tableName} WHERE TRUE `);
|
|
729
1150
|
statement.append(whereStatement);
|
|
730
1151
|
statement.append(this.getRetentionReadFilter());
|
|
1152
|
+
if (scalarApplyTopK) {
|
|
1153
|
+
this.appendTopKGroupRestriction({
|
|
1154
|
+
statement,
|
|
1155
|
+
databaseName,
|
|
1156
|
+
aggregateBy,
|
|
1157
|
+
topK: scalarTopK,
|
|
1158
|
+
groupByKeys,
|
|
1159
|
+
attributeGroupKeys,
|
|
1160
|
+
});
|
|
1161
|
+
}
|
|
731
1162
|
}
|
|
732
1163
|
/*
|
|
733
1164
|
* `Total` drops the time bucket from GROUP BY (it becomes an
|
|
@@ -765,12 +1196,14 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
765
1196
|
value: Number(aggregateBy.skip),
|
|
766
1197
|
type: TableColumnType.Number,
|
|
767
1198
|
}} `);
|
|
1199
|
+
/*
|
|
1200
|
+
* transform_null_in=1 only when the Top-K IN restriction is present
|
|
1201
|
+
* — see the percentile path's settings comment for the rationale.
|
|
1202
|
+
*/
|
|
768
1203
|
statement.append(getQuerySettings({
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
max_threads: 4,
|
|
773
|
-
},
|
|
1204
|
+
maxExecutionTimeInSeconds: 45,
|
|
1205
|
+
timeoutOverflowMode: this.getTimeoutOverflowMode(aggregateBy),
|
|
1206
|
+
additionalSettings: Object.assign({ optimize_aggregation_in_order: 1, optimize_move_to_prewhere: 1, max_threads: 4 }, (scalarApplyTopK ? { transform_null_in: 1 } : {})),
|
|
774
1207
|
}));
|
|
775
1208
|
const columns = [
|
|
776
1209
|
aggregationColumn,
|
|
@@ -802,10 +1235,28 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
802
1235
|
const innerWhereStatement = this.statementGenerator.toWhereStatement(this.getMutableMetricInnerQuery(aggregateBy.query));
|
|
803
1236
|
const outerWhereStatement = this.statementGenerator.toWhereStatement(aggregateBy.query);
|
|
804
1237
|
const sortStatement = this.statementGenerator.toSortStatement(aggregateBy.sort);
|
|
1238
|
+
/*
|
|
1239
|
+
* Server-side Top-K for the (model-column) grouped mutable path.
|
|
1240
|
+
* The ranking subquery and totalGroups counter each re-run the
|
|
1241
|
+
* dedup subquery (buildMutableMetricDedupSource) — acceptable
|
|
1242
|
+
* because the mutable table is small by design (bounded-cardinality
|
|
1243
|
+
* business metrics), unlike the raw Metric table.
|
|
1244
|
+
*/
|
|
1245
|
+
const mutableTopK = this.getSanitizedTopK(aggregateBy);
|
|
1246
|
+
const mutableGroupByKeys = aggregateBy.groupBy
|
|
1247
|
+
? Object.keys(aggregateBy.groupBy)
|
|
1248
|
+
: [];
|
|
1249
|
+
const mutableApplyTopK = Boolean(mutableTopK && mutableGroupByKeys.length > 0);
|
|
805
1250
|
const statement = SQL ``;
|
|
1251
|
+
statement.append(SQL `SELECT `).append(select.statement);
|
|
1252
|
+
if (mutableApplyTopK) {
|
|
1253
|
+
statement.append(`, (SELECT uniqExact(`);
|
|
1254
|
+
this.appendRawGroupExpressionList(statement, mutableGroupByKeys, []);
|
|
1255
|
+
statement.append(`)`);
|
|
1256
|
+
statement.append(this.buildMutableMetricDedupSource(databaseName, aggregateBy));
|
|
1257
|
+
statement.append(`) AS __total_groups`);
|
|
1258
|
+
}
|
|
806
1259
|
statement
|
|
807
|
-
.append(SQL `SELECT `)
|
|
808
|
-
.append(select.statement)
|
|
809
1260
|
.append(SQL `
|
|
810
1261
|
FROM (
|
|
811
1262
|
SELECT
|
|
@@ -838,6 +1289,28 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
838
1289
|
`)
|
|
839
1290
|
.append(outerWhereStatement)
|
|
840
1291
|
.append(SQL ` AND retentionDate >= now()`);
|
|
1292
|
+
if (mutableApplyTopK) {
|
|
1293
|
+
/*
|
|
1294
|
+
* Plain (non-GLOBAL) IN is deliberate here, unlike
|
|
1295
|
+
* appendTopKGroupRestriction: this predicate sits in the OUTER
|
|
1296
|
+
* query whose FROM is the argMax-dedup subquery — the initiator
|
|
1297
|
+
* evaluates it, so it is not a double-distributed subquery and
|
|
1298
|
+
* multi-shard ClickHouse accepts it as-is.
|
|
1299
|
+
*/
|
|
1300
|
+
statement.append(` AND (`);
|
|
1301
|
+
this.appendRawGroupExpressionList(statement, mutableGroupByKeys, []);
|
|
1302
|
+
statement.append(`) IN (SELECT `);
|
|
1303
|
+
this.appendRawGroupExpressionList(statement, mutableGroupByKeys, []);
|
|
1304
|
+
statement.append(this.buildMutableMetricDedupSource(databaseName, aggregateBy));
|
|
1305
|
+
statement.append(` GROUP BY `);
|
|
1306
|
+
this.appendRawGroupExpressionList(statement, mutableGroupByKeys, []);
|
|
1307
|
+
statement.append(` ORDER BY ${mutableTopK.rankBy}(${aggregateBy.aggregateColumnName.toString()}) DESC`);
|
|
1308
|
+
statement.append(SQL ` LIMIT ${{
|
|
1309
|
+
value: mutableTopK.count,
|
|
1310
|
+
type: TableColumnType.Number,
|
|
1311
|
+
}}`);
|
|
1312
|
+
statement.append(`)`);
|
|
1313
|
+
}
|
|
841
1314
|
/*
|
|
842
1315
|
* Mirror the base builder's Total handling: the SELECT above (shared
|
|
843
1316
|
* toAggregateSelectStatement) already emits `min(time)` for a `Total`
|
|
@@ -884,11 +1357,16 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
884
1357
|
value: Number(aggregateBy.skip),
|
|
885
1358
|
type: TableColumnType.Number,
|
|
886
1359
|
}}`);
|
|
1360
|
+
/*
|
|
1361
|
+
* transform_null_in=1 only when the Top-K IN restriction is present
|
|
1362
|
+
* — see the percentile path's settings comment for the rationale
|
|
1363
|
+
* (mutable group-by keys are Nullable model columns too, e.g.
|
|
1364
|
+
* primaryEntityType).
|
|
1365
|
+
*/
|
|
887
1366
|
statement.append(getQuerySettings({
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
},
|
|
1367
|
+
maxExecutionTimeInSeconds: 45,
|
|
1368
|
+
timeoutOverflowMode: this.getTimeoutOverflowMode(aggregateBy),
|
|
1369
|
+
additionalSettings: Object.assign({ optimize_move_to_prewhere: 1, max_threads: 4 }, (mutableApplyTopK ? { transform_null_in: 1 } : {})),
|
|
892
1370
|
}));
|
|
893
1371
|
logger.debug("Mutable metric aggregate statement", {
|
|
894
1372
|
metricName: metricName || "",
|
|
@@ -901,6 +1379,51 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
901
1379
|
columns: select.columns,
|
|
902
1380
|
};
|
|
903
1381
|
}
|
|
1382
|
+
/**
|
|
1383
|
+
* A fresh `FROM (...) WHERE ...` fragment over the argMax-deduped
|
|
1384
|
+
* mutable-metric rows — the same source the main mutable aggregate
|
|
1385
|
+
* reads from — for the Top-K ranking subquery and totalGroups
|
|
1386
|
+
* counter, which need their own scan of the deduped rows (the
|
|
1387
|
+
* versioned raw rows must never be ranked directly, or a metric
|
|
1388
|
+
* updated N times would rank by its stalest value).
|
|
1389
|
+
*/
|
|
1390
|
+
buildMutableMetricDedupSource(databaseName, aggregateBy) {
|
|
1391
|
+
const source = SQL ``;
|
|
1392
|
+
source
|
|
1393
|
+
.append(SQL `
|
|
1394
|
+
FROM (
|
|
1395
|
+
SELECT
|
|
1396
|
+
projectId,
|
|
1397
|
+
name,
|
|
1398
|
+
primaryEntityId,
|
|
1399
|
+
primaryEntityType,
|
|
1400
|
+
metricPointId,
|
|
1401
|
+
argMax(metricPointType, version) AS metricPointType,
|
|
1402
|
+
argMax(time, version) AS time,
|
|
1403
|
+
argMax(timeUnixNano, version) AS timeUnixNano,
|
|
1404
|
+
argMax(attributes, version) AS attributes,
|
|
1405
|
+
argMax(attributeKeys, version) AS attributeKeys,
|
|
1406
|
+
argMax(value, version) AS value,
|
|
1407
|
+
argMax(retentionDate, version) AS retentionDate,
|
|
1408
|
+
argMax(isDeleted, version) AS isDeleted
|
|
1409
|
+
FROM ${databaseName}.${AnalyticsTableName.MutableMetric}
|
|
1410
|
+
WHERE TRUE
|
|
1411
|
+
`)
|
|
1412
|
+
.append(this.statementGenerator.toWhereStatement(this.getMutableMetricInnerQuery(aggregateBy.query)))
|
|
1413
|
+
.append(SQL `
|
|
1414
|
+
GROUP BY
|
|
1415
|
+
projectId,
|
|
1416
|
+
name,
|
|
1417
|
+
primaryEntityId,
|
|
1418
|
+
primaryEntityType,
|
|
1419
|
+
metricPointId
|
|
1420
|
+
)
|
|
1421
|
+
WHERE isDeleted = false
|
|
1422
|
+
`)
|
|
1423
|
+
.append(this.statementGenerator.toWhereStatement(aggregateBy.query))
|
|
1424
|
+
.append(SQL ` AND retentionDate >= now()`);
|
|
1425
|
+
return source;
|
|
1426
|
+
}
|
|
904
1427
|
getMutableMetricInnerQuery(query) {
|
|
905
1428
|
const queryRecord = query;
|
|
906
1429
|
const allowedInnerKeys = [
|
|
@@ -1012,10 +1535,11 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1012
1535
|
});
|
|
1013
1536
|
/*
|
|
1014
1537
|
* The MV is bucketed at 1 minute, so every time-bucketed interval
|
|
1015
|
-
* (Minute /
|
|
1016
|
-
* acceptable — the honored interval flows into the
|
|
1017
|
-
* `Total` (whole-window, no bucketing) is the one
|
|
1018
|
-
* to the raw-table builder, which knows how to
|
|
1538
|
+
* (Minute / FiveMinutes / ... / Year) is >= MV resolution and
|
|
1539
|
+
* acceptable — the honored interval flows into the shared bucket
|
|
1540
|
+
* expression below. `Total` (whole-window, no bucketing) is the one
|
|
1541
|
+
* exception: fall back to the raw-table builder, which knows how to
|
|
1542
|
+
* collapse the window.
|
|
1019
1543
|
*/
|
|
1020
1544
|
if (AggregateUtil.isTotalAggregation(interval)) {
|
|
1021
1545
|
return null;
|
|
@@ -1058,7 +1582,6 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1058
1582
|
this.useDefaultDatabase();
|
|
1059
1583
|
}
|
|
1060
1584
|
const databaseName = this.database.getDatasourceOptions().database;
|
|
1061
|
-
const intervalLower = interval.toLowerCase();
|
|
1062
1585
|
let mergedExpr;
|
|
1063
1586
|
if (aggType === AggregationType.Sum) {
|
|
1064
1587
|
mergedExpr = `sumMerge(valueSumState)`;
|
|
@@ -1085,7 +1608,7 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1085
1608
|
const nonTimeWhere = this.statementGenerator.toWhereStatement(this.stripTimeFromQuery(aggregateBy.query));
|
|
1086
1609
|
const sortStatement = this.statementGenerator.toSortStatement(aggregateBy.sort);
|
|
1087
1610
|
const statement = SQL ``;
|
|
1088
|
-
statement.append(`SELECT ${mergedExpr} as value,
|
|
1611
|
+
statement.append(`SELECT ${mergedExpr} as value, ${AggregateUtil.buildBucketTimestampExpression(interval, "bucketTime")} as time`);
|
|
1089
1612
|
statement.append(SQL ` FROM ${databaseName}.MetricItemAggMV1m`);
|
|
1090
1613
|
statement.append(` WHERE bucketTime >= toDateTime('${this.formatDateTime(aggregateBy.startTimestamp)}') AND bucketTime <= toDateTime('${this.formatDateTime(aggregateBy.endTimestamp)}')${this.getRetentionReadFilter()}`);
|
|
1091
1614
|
statement.append(SQL ` `).append(nonTimeWhere);
|
|
@@ -1100,6 +1623,8 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1100
1623
|
type: TableColumnType.Number,
|
|
1101
1624
|
}} `);
|
|
1102
1625
|
statement.append(getQuerySettings({
|
|
1626
|
+
maxExecutionTimeInSeconds: 45,
|
|
1627
|
+
timeoutOverflowMode: this.getTimeoutOverflowMode(aggregateBy),
|
|
1103
1628
|
additionalSettings: {
|
|
1104
1629
|
optimize_aggregation_in_order: 1,
|
|
1105
1630
|
optimize_move_to_prewhere: 1,
|
|
@@ -1121,30 +1646,55 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1121
1646
|
};
|
|
1122
1647
|
}
|
|
1123
1648
|
/*
|
|
1124
|
-
*
|
|
1649
|
+
* Entity-scoped materialized-view fast path (per-entity 1-minute
|
|
1650
|
+
* rollups, keyed by the ingest-stamped scalar entity-key columns).
|
|
1651
|
+
* Entity keys canonicalize their value (trim + lowercase), so spelling
|
|
1652
|
+
* drift in the reported identity still lands on one rollup stream.
|
|
1125
1653
|
*
|
|
1126
|
-
*
|
|
1127
|
-
*
|
|
1128
|
-
*
|
|
1129
|
-
*
|
|
1130
|
-
*
|
|
1131
|
-
*
|
|
1654
|
+
* Routing decision table — ALL of these outer gates must hold:
|
|
1655
|
+
* Sum/Avg/Min/Max/Count over `value` bucketed by `time`; no group-by
|
|
1656
|
+
* of any kind (an entity-key-keyed MV cannot label legend series);
|
|
1657
|
+
* not a distribution metric and not a percentile (enforced by the
|
|
1658
|
+
* caller); a bucketed (non-Total) interval; a projectId (the entity
|
|
1659
|
+
* key is tenant-scoped by construction); and no query columns beyond
|
|
1660
|
+
* projectId/name/time/attributes/entityScope.
|
|
1661
|
+
* Then exactly ONE of these entity-filter shapes routes:
|
|
1132
1662
|
*
|
|
1133
|
-
*
|
|
1134
|
-
*
|
|
1135
|
-
*
|
|
1136
|
-
*
|
|
1137
|
-
*
|
|
1138
|
-
*
|
|
1139
|
-
*
|
|
1140
|
-
*
|
|
1663
|
+
* attributes == {resource.host.name: v}
|
|
1664
|
+
* -> MetricItemAggMV1mByHostV2, hostEntityKey = keyForHost(projectId, v)
|
|
1665
|
+
* attributes == {resource.k8s.cluster.name: v}
|
|
1666
|
+
* -> MetricItemAggMV1mByK8sCluster, k8sClusterEntityKey = keyForKubernetesCluster(projectId, v)
|
|
1667
|
+
* attributes == {resource.container.id: v}
|
|
1668
|
+
* -> MetricItemAggMV1mByContainer, containerEntityKey = keyForContainer(projectId, v)
|
|
1669
|
+
* attributes == {resource.service.name: v}
|
|
1670
|
+
* -> MetricItemAggMV1mByService, serviceEntityKey IN (<registry key set>)
|
|
1671
|
+
* The key set is resolved asynchronously in aggregateBy() from
|
|
1672
|
+
* the Postgres TelemetryEntity registry — service identity folds
|
|
1673
|
+
* service.namespace into the key at ingest, so one name can map
|
|
1674
|
+
* to several keys. No registry rows -> NO routing (raw path).
|
|
1675
|
+
* entityScope only, attributeKey in {host/k8s-cluster/container} and
|
|
1676
|
+
* entityKeys verified byte-equal to [keyFor<type>(projectId, attributeValue)]
|
|
1677
|
+
* -> same MV as the matching attribute shape. The scope's attribute
|
|
1678
|
+
* OR-fallback only adds rows ingested before the entity-key
|
|
1679
|
+
* columns existed — the same retention-bounded gap the original
|
|
1680
|
+
* per-host path accepted.
|
|
1681
|
+
* attributes(single recognized key) + entityScope agreeing on
|
|
1682
|
+
* attributeKey/attributeValue (the host/k8s Metrics-tab sparkline
|
|
1683
|
+
* shape; the scope is then redundant — A AND (B OR A) === A)
|
|
1684
|
+
* -> same MV as the attribute shape (service excluded).
|
|
1685
|
+
*
|
|
1686
|
+
* Everything else falls through (minute MV, then raw): service +
|
|
1687
|
+
* entityScope (a bare-key translation would drop namespaced variants),
|
|
1688
|
+
* unverifiable or multi-key scopes, extra attribute filters, non-string
|
|
1689
|
+
* values, k8s pod/node identities (composite cluster(+namespace)+uid —
|
|
1690
|
+
* not derivable from a single attribute), and any other query column.
|
|
1691
|
+
* When in doubt this method returns null; the raw path is always
|
|
1692
|
+
* correct.
|
|
1141
1693
|
*
|
|
1142
|
-
*
|
|
1143
|
-
*
|
|
1144
|
-
* shape (columns: aggregateColumn, timestampColumn) matches
|
|
1145
|
-
* the base statement so downstream code needs no changes.
|
|
1694
|
+
* The result row shape (columns: aggregateColumn, timestampColumn)
|
|
1695
|
+
* matches the base statement so downstream code needs no changes.
|
|
1146
1696
|
*/
|
|
1147
|
-
|
|
1697
|
+
tryBuildEntityAggregateMVStatement(aggregateBy) {
|
|
1148
1698
|
const aggType = aggregateBy.aggregationType;
|
|
1149
1699
|
const supported = [
|
|
1150
1700
|
AggregationType.Sum,
|
|
@@ -1167,84 +1717,57 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1167
1717
|
aggregateBy.groupByAttributeKeys.length > 0) {
|
|
1168
1718
|
return null;
|
|
1169
1719
|
}
|
|
1720
|
+
const queryRecord = aggregateBy.query || {};
|
|
1170
1721
|
/*
|
|
1171
|
-
*
|
|
1172
|
-
*
|
|
1173
|
-
*
|
|
1174
|
-
* canonical Overview/Metrics-page pattern). Anything else —
|
|
1175
|
-
* extra attribute filters, NotEqual, Search, etc. — has to
|
|
1176
|
-
* fall back so the result stays correct.
|
|
1722
|
+
* The entity keys fold the tenant in (sha256(projectId|type|...)), so
|
|
1723
|
+
* MV rows can only be located when the query is project-scoped.
|
|
1724
|
+
* Dashboard reads always are; anything else falls back safely.
|
|
1177
1725
|
*/
|
|
1178
|
-
const
|
|
1179
|
-
|
|
1180
|
-
if (
|
|
1181
|
-
|
|
1726
|
+
const projectIdValue = queryRecord["projectId"];
|
|
1727
|
+
let projectId = "";
|
|
1728
|
+
if (projectIdValue instanceof ObjectID) {
|
|
1729
|
+
projectId = projectIdValue.toString();
|
|
1182
1730
|
}
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
return null;
|
|
1731
|
+
else if (typeof projectIdValue === "string") {
|
|
1732
|
+
projectId = projectIdValue;
|
|
1186
1733
|
}
|
|
1187
|
-
|
|
1188
|
-
if (attrKey !== "resource.host.name") {
|
|
1734
|
+
if (!projectId) {
|
|
1189
1735
|
return null;
|
|
1190
1736
|
}
|
|
1191
|
-
|
|
1737
|
+
const resolved = this.resolveEntityMVRouteAndKeys(aggregateBy, queryRecord, projectId);
|
|
1738
|
+
if (!resolved) {
|
|
1192
1739
|
return null;
|
|
1193
1740
|
}
|
|
1194
1741
|
/*
|
|
1195
|
-
*
|
|
1196
|
-
* other query key (primaryEntityId,
|
|
1197
|
-
*
|
|
1198
|
-
*
|
|
1199
|
-
*
|
|
1742
|
+
* Each MV only carries projectId/name/<entity key>/bucketTime. Any
|
|
1743
|
+
* other query key (primaryEntityId, entityKeys, ...) would compile to
|
|
1744
|
+
* a WHERE over a column the MV does not have, so fall back to the raw
|
|
1745
|
+
* table for those. Mirrors tryBuildMinuteAggregateMVStatement.
|
|
1746
|
+
* `attributes` and `entityScope` are validated and rewritten into the
|
|
1747
|
+
* entity-key predicate by resolveEntityMVRouteAndKeys above.
|
|
1200
1748
|
*/
|
|
1201
1749
|
const mvQueryableColumns = [
|
|
1202
1750
|
"projectId",
|
|
1203
1751
|
"name",
|
|
1204
1752
|
"time", // stripped below; bucketTime range is added explicitly
|
|
1205
|
-
"attributes",
|
|
1753
|
+
"attributes",
|
|
1754
|
+
"entityScope",
|
|
1206
1755
|
];
|
|
1207
1756
|
for (const queryKey of Object.keys(queryRecord)) {
|
|
1208
1757
|
if (!mvQueryableColumns.includes(queryKey)) {
|
|
1209
1758
|
return null;
|
|
1210
1759
|
}
|
|
1211
1760
|
}
|
|
1212
|
-
const hostIdentifier = typeof attrValue === "string" ? attrValue : "";
|
|
1213
|
-
if (!hostIdentifier) {
|
|
1214
|
-
return null;
|
|
1215
|
-
}
|
|
1216
|
-
/*
|
|
1217
|
-
* The entity key folds the tenant in (sha256(projectId|host|...)), so
|
|
1218
|
-
* the MV row can only be located when the query is project-scoped.
|
|
1219
|
-
* Dashboard reads always are; anything else falls back safely.
|
|
1220
|
-
*/
|
|
1221
|
-
const projectIdValue = queryRecord["projectId"];
|
|
1222
|
-
let projectId = "";
|
|
1223
|
-
if (projectIdValue instanceof ObjectID) {
|
|
1224
|
-
projectId = projectIdValue.toString();
|
|
1225
|
-
}
|
|
1226
|
-
else if (typeof projectIdValue === "string") {
|
|
1227
|
-
projectId = projectIdValue;
|
|
1228
|
-
}
|
|
1229
|
-
if (!projectId) {
|
|
1230
|
-
return null;
|
|
1231
|
-
}
|
|
1232
|
-
/*
|
|
1233
|
-
* Same canonicalized key the ingest pipeline stamps into
|
|
1234
|
-
* MetricItemV3.hostEntityKey (and the V2 MV groups by) — byte-equality
|
|
1235
|
-
* is what makes this lookup correct, see Common/Utils/Telemetry/EntityKey.
|
|
1236
|
-
*/
|
|
1237
|
-
const hostEntityKey = keyForHost(projectId, hostIdentifier);
|
|
1238
1761
|
const interval = AggregateUtil.getAggregationInterval({
|
|
1239
1762
|
startDate: aggregateBy.startTimestamp,
|
|
1240
1763
|
endDate: aggregateBy.endTimestamp,
|
|
1241
1764
|
aggregationInterval: aggregateBy.aggregationInterval,
|
|
1242
1765
|
});
|
|
1243
1766
|
/*
|
|
1244
|
-
* `Total` (whole-window, no bucketing) is not served by
|
|
1245
|
-
* time-bucketed per-
|
|
1246
|
-
* Every other interval is >= the
|
|
1247
|
-
* into the
|
|
1767
|
+
* `Total` (whole-window, no bucketing) is not served by these
|
|
1768
|
+
* time-bucketed per-entity MVs — fall back to the raw-table builder.
|
|
1769
|
+
* Every other interval is >= the MVs' 1-minute resolution and flows
|
|
1770
|
+
* into the shared bucket expression below.
|
|
1248
1771
|
*/
|
|
1249
1772
|
if (AggregateUtil.isTotalAggregation(interval)) {
|
|
1250
1773
|
return null;
|
|
@@ -1253,7 +1776,6 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1253
1776
|
this.useDefaultDatabase();
|
|
1254
1777
|
}
|
|
1255
1778
|
const databaseName = this.database.getDatasourceOptions().database;
|
|
1256
|
-
const intervalLower = interval.toLowerCase();
|
|
1257
1779
|
let mergedExpr;
|
|
1258
1780
|
if (aggType === AggregationType.Sum) {
|
|
1259
1781
|
mergedExpr = `sumMerge(valueSumState)`;
|
|
@@ -1271,22 +1793,32 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1271
1793
|
mergedExpr = `if(countMerge(valueCountState) = 0, 0, sumMerge(valueSumState) / countMerge(valueCountState))`;
|
|
1272
1794
|
}
|
|
1273
1795
|
/*
|
|
1274
|
-
* Strip
|
|
1275
|
-
*
|
|
1276
|
-
*
|
|
1277
|
-
*
|
|
1796
|
+
* Strip `time` (column doesn't exist on the MV; we inject an
|
|
1797
|
+
* explicit bucketTime range below) plus `attributes`/`entityScope`
|
|
1798
|
+
* (the entity filter is now an explicit predicate against the MV's
|
|
1799
|
+
* entity-key column).
|
|
1278
1800
|
*/
|
|
1279
|
-
const filteredQuery = this.
|
|
1801
|
+
const filteredQuery = this.stripEntityFilterAndTimeFromQuery(aggregateBy.query);
|
|
1280
1802
|
const nonTimeWhere = this.statementGenerator.toWhereStatement(filteredQuery);
|
|
1281
1803
|
const sortStatement = this.statementGenerator.toSortStatement(aggregateBy.sort);
|
|
1282
1804
|
const statement = SQL ``;
|
|
1283
|
-
statement.append(`SELECT ${mergedExpr} as value,
|
|
1284
|
-
statement.append(SQL ` FROM ${databaseName}
|
|
1805
|
+
statement.append(`SELECT ${mergedExpr} as value, ${AggregateUtil.buildBucketTimestampExpression(interval, "bucketTime")} as time`);
|
|
1806
|
+
statement.append(SQL ` FROM ${databaseName}.`);
|
|
1807
|
+
statement.append(resolved.route.tableName);
|
|
1285
1808
|
statement.append(` WHERE bucketTime >= toDateTime('${this.formatDateTime(aggregateBy.startTimestamp)}') AND bucketTime <= toDateTime('${this.formatDateTime(aggregateBy.endTimestamp)}')${this.getRetentionReadFilter()}`);
|
|
1286
|
-
statement.append(
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1809
|
+
statement.append(` AND ${resolved.route.keyColumn}`);
|
|
1810
|
+
if (resolved.keys.length === 1) {
|
|
1811
|
+
statement.append(SQL ` = ${{
|
|
1812
|
+
value: resolved.keys[0],
|
|
1813
|
+
type: TableColumnType.Text,
|
|
1814
|
+
}}`);
|
|
1815
|
+
}
|
|
1816
|
+
else {
|
|
1817
|
+
statement.append(SQL ` IN ${{
|
|
1818
|
+
value: resolved.keys,
|
|
1819
|
+
type: TableColumnType.ArrayText,
|
|
1820
|
+
}}`);
|
|
1821
|
+
}
|
|
1290
1822
|
statement.append(SQL ` `).append(nonTimeWhere);
|
|
1291
1823
|
statement.append(SQL ` GROUP BY `).append(`time`);
|
|
1292
1824
|
statement.append(SQL ` ORDER BY `).append(sortStatement);
|
|
@@ -1299,13 +1831,15 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1299
1831
|
type: TableColumnType.Number,
|
|
1300
1832
|
}} `);
|
|
1301
1833
|
statement.append(getQuerySettings({
|
|
1834
|
+
maxExecutionTimeInSeconds: 45,
|
|
1835
|
+
timeoutOverflowMode: this.getTimeoutOverflowMode(aggregateBy),
|
|
1302
1836
|
additionalSettings: {
|
|
1303
1837
|
optimize_aggregation_in_order: 1,
|
|
1304
1838
|
optimize_move_to_prewhere: 1,
|
|
1305
1839
|
max_threads: 4,
|
|
1306
1840
|
},
|
|
1307
1841
|
}));
|
|
1308
|
-
logger.debug(`${this.model.tableName}
|
|
1842
|
+
logger.debug(`${this.model.tableName} Entity MV Aggregate Statement`, {
|
|
1309
1843
|
tableName: this.model.tableName,
|
|
1310
1844
|
});
|
|
1311
1845
|
logger.debug(statement, {
|
|
@@ -1319,13 +1853,115 @@ export class MetricService extends AnalyticsDatabaseService {
|
|
|
1319
1853
|
],
|
|
1320
1854
|
};
|
|
1321
1855
|
}
|
|
1322
|
-
|
|
1856
|
+
/**
|
|
1857
|
+
* The single entity filter of a query, resolved to the MV route that
|
|
1858
|
+
* serves it and the exact entity-key set to filter by — or null when
|
|
1859
|
+
* no branch of the routing decision table (see
|
|
1860
|
+
* tryBuildEntityAggregateMVStatement) matches. Never guesses: every
|
|
1861
|
+
* shape it accepts is one whose MV predicate provably covers the raw
|
|
1862
|
+
* predicate it replaces (modulo the accepted pre-entity-key-column
|
|
1863
|
+
* rows).
|
|
1864
|
+
*/
|
|
1865
|
+
resolveEntityMVRouteAndKeys(aggregateBy, queryRecord, projectId) {
|
|
1866
|
+
var _a;
|
|
1867
|
+
const attrs = queryRecord["attributes"];
|
|
1868
|
+
if (attrs !== undefined && attrs !== null && typeof attrs !== "object") {
|
|
1869
|
+
return null;
|
|
1870
|
+
}
|
|
1871
|
+
const attrEntries = attrs && typeof attrs === "object"
|
|
1872
|
+
? Object.entries(attrs)
|
|
1873
|
+
: [];
|
|
1874
|
+
if (attrEntries.length > 1) {
|
|
1875
|
+
return null;
|
|
1876
|
+
}
|
|
1877
|
+
const scopeValue = queryRecord["entityScope"];
|
|
1878
|
+
const scope = scopeValue !== undefined &&
|
|
1879
|
+
scopeValue !== null &&
|
|
1880
|
+
typeof scopeValue === "object"
|
|
1881
|
+
? scopeValue
|
|
1882
|
+
: null;
|
|
1883
|
+
// A present-but-malformed entityScope is a filter we cannot honor.
|
|
1884
|
+
if (scopeValue !== undefined && scopeValue !== null && !scope) {
|
|
1885
|
+
return null;
|
|
1886
|
+
}
|
|
1887
|
+
if (attrEntries.length === 1) {
|
|
1888
|
+
const [attrKey, attrValue] = attrEntries[0];
|
|
1889
|
+
const route = ENTITY_MV_ROUTES.find((candidate) => {
|
|
1890
|
+
return candidate.attributeKey === attrKey;
|
|
1891
|
+
});
|
|
1892
|
+
if (!route || typeof attrValue !== "string" || !attrValue) {
|
|
1893
|
+
return null;
|
|
1894
|
+
}
|
|
1895
|
+
if (scope) {
|
|
1896
|
+
/*
|
|
1897
|
+
* The host/k8s Metrics tabs send the attribute filter AND a
|
|
1898
|
+
* matching entityScope in the same query. When the scope is
|
|
1899
|
+
* provably the SAME entity filter (same attributeKey/value), the
|
|
1900
|
+
* conjunction reduces to the attribute equality — the scope's
|
|
1901
|
+
* OR-fallback is subsumed — so route on the attribute alone. Any
|
|
1902
|
+
* disagreement means a genuine second filter: fall back. Service
|
|
1903
|
+
* is excluded outright: its scope keys cannot be verified against
|
|
1904
|
+
* a computed key (namespace variants).
|
|
1905
|
+
*/
|
|
1906
|
+
if (route.keyForValue === null) {
|
|
1907
|
+
return null;
|
|
1908
|
+
}
|
|
1909
|
+
if (scope.attributeKey !== attrKey ||
|
|
1910
|
+
String((_a = scope.attributeValue) !== null && _a !== void 0 ? _a : "") !== attrValue) {
|
|
1911
|
+
return null;
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
if (route.keyForValue === null) {
|
|
1915
|
+
/*
|
|
1916
|
+
* Service: the registry-resolved key set was handed over by
|
|
1917
|
+
* aggregateBy(). Absent or empty (sync caller, registry miss,
|
|
1918
|
+
* lookup failure) -> raw path.
|
|
1919
|
+
*/
|
|
1920
|
+
const hintKeys = this.serviceEntityKeysHintByAggregate.get(aggregateBy);
|
|
1921
|
+
if (!hintKeys || hintKeys.length === 0) {
|
|
1922
|
+
return null;
|
|
1923
|
+
}
|
|
1924
|
+
return { route, keys: hintKeys };
|
|
1925
|
+
}
|
|
1926
|
+
return { route, keys: [route.keyForValue(projectId, attrValue)] };
|
|
1927
|
+
}
|
|
1928
|
+
// No attribute filter: entityScope-only shape.
|
|
1929
|
+
if (!scope) {
|
|
1930
|
+
return null;
|
|
1931
|
+
}
|
|
1932
|
+
const route = ENTITY_MV_ROUTES.find((candidate) => {
|
|
1933
|
+
return candidate.attributeKey === scope.attributeKey;
|
|
1934
|
+
});
|
|
1935
|
+
if (!route || route.keyForValue === null) {
|
|
1936
|
+
return null;
|
|
1937
|
+
}
|
|
1938
|
+
const scopeAttributeValue = scope.attributeValue;
|
|
1939
|
+
if (typeof scopeAttributeValue !== "string" || !scopeAttributeValue) {
|
|
1940
|
+
return null;
|
|
1941
|
+
}
|
|
1942
|
+
/*
|
|
1943
|
+
* `hasAny(entityKeys, [...])` is only translatable to the scalar
|
|
1944
|
+
* entity-key column when every listed key IS the key this scope's
|
|
1945
|
+
* attribute value derives to — recompute it server-side and require
|
|
1946
|
+
* byte equality (a foreign or extra key would make the raw predicate
|
|
1947
|
+
* match rows the MV predicate cannot).
|
|
1948
|
+
*/
|
|
1949
|
+
const expectedKey = route.keyForValue(projectId, scopeAttributeValue);
|
|
1950
|
+
const scopeKeys = Array.isArray(scope.entityKeys)
|
|
1951
|
+
? scope.entityKeys
|
|
1952
|
+
: [];
|
|
1953
|
+
if (scopeKeys.length !== 1 || scopeKeys[0] !== expectedKey) {
|
|
1954
|
+
return null;
|
|
1955
|
+
}
|
|
1956
|
+
return { route, keys: [expectedKey] };
|
|
1957
|
+
}
|
|
1958
|
+
stripEntityFilterAndTimeFromQuery(query) {
|
|
1323
1959
|
if (!query || typeof query !== "object") {
|
|
1324
1960
|
return query;
|
|
1325
1961
|
}
|
|
1326
1962
|
const out = {};
|
|
1327
1963
|
for (const [k, v] of Object.entries(query)) {
|
|
1328
|
-
if (k === "time" || k === "attributes") {
|
|
1964
|
+
if (k === "time" || k === "attributes" || k === "entityScope") {
|
|
1329
1965
|
continue;
|
|
1330
1966
|
}
|
|
1331
1967
|
out[k] = v;
|