@oneuptime/common 11.5.0 → 11.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Server/Infrastructure/Postgres/SchemaMigrations/1783650000000-AddAdditionalParamsToLlmProvider.ts +2 -2
- package/Server/Infrastructure/Postgres/SchemaMigrations/1783937343400-MigrationName.ts +386 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
- package/Server/Services/AnalyticsDatabaseService.ts +59 -7
- package/Server/Services/MetricService.ts +124 -28
- package/Server/Services/SpanService.ts +11 -0
- package/Server/Types/AnalyticsDatabase/AggregateBy.ts +39 -0
- package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +8 -5
- package/Tests/Server/Services/AnalyticsDatabaseService.test.ts +84 -0
- package/Tests/Server/Services/MetricServiceAggregate.test.ts +167 -0
- package/Tests/Types/BaseDatabase/AggregationIntervalUtil.test.ts +40 -0
- package/Types/BaseDatabase/AggregateBy.ts +11 -1
- package/Types/BaseDatabase/AggregationInterval.ts +10 -0
- package/Types/BaseDatabase/AggregationIntervalUtil.ts +23 -0
- package/Types/Metrics/MetricFormulaConfigData.ts +5 -0
- package/Types/Metrics/MetricQueryConfigData.ts +19 -0
- package/UI/Components/Charts/Area/AreaChart.tsx +15 -2
- package/UI/Components/Charts/Bar/BarChart.tsx +16 -2
- package/UI/Components/Charts/ChartLibrary/AreaChart/AreaChart.tsx +31 -32
- package/UI/Components/Charts/ChartLibrary/BarChart/BarChart.tsx +41 -17
- package/UI/Components/Charts/ChartLibrary/LineChart/LineChart.tsx +38 -34
- package/UI/Components/Charts/ChartLibrary/SparkChart/SparkChart.tsx +4 -3
- package/UI/Components/Charts/ChartLibrary/Utils/ChartColors.ts +48 -10
- package/UI/Components/Charts/Line/LineChart.tsx +15 -2
- package/UI/Components/MoreMenu/MoreMenu.tsx +61 -2
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783650000000-AddAdditionalParamsToLlmProvider.js +2 -2
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783650000000-AddAdditionalParamsToLlmProvider.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783937343400-MigrationName.js +384 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1783937343400-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +43 -5
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Services/MetricService.js +110 -23
- package/build/dist/Server/Services/MetricService.js.map +1 -1
- package/build/dist/Server/Services/SpanService.js +10 -0
- package/build/dist/Server/Services/SpanService.js.map +1 -1
- package/build/dist/Server/Types/AnalyticsDatabase/AggregateBy.js +32 -0
- package/build/dist/Server/Types/AnalyticsDatabase/AggregateBy.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +2 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
- package/build/dist/Types/BaseDatabase/AggregationInterval.js +10 -0
- package/build/dist/Types/BaseDatabase/AggregationInterval.js.map +1 -1
- package/build/dist/Types/BaseDatabase/AggregationIntervalUtil.js +13 -0
- package/build/dist/Types/BaseDatabase/AggregationIntervalUtil.js.map +1 -1
- package/build/dist/UI/Components/Charts/Area/AreaChart.js +3 -1
- package/build/dist/UI/Components/Charts/Area/AreaChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/Bar/BarChart.js +3 -1
- package/build/dist/UI/Components/Charts/Bar/BarChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/AreaChart/AreaChart.js +13 -9
- package/build/dist/UI/Components/Charts/ChartLibrary/AreaChart/AreaChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/BarChart/BarChart.js +19 -7
- package/build/dist/UI/Components/Charts/ChartLibrary/BarChart/BarChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/LineChart/LineChart.js +21 -9
- package/build/dist/UI/Components/Charts/ChartLibrary/LineChart/LineChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/SparkChart/SparkChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartLibrary/Utils/ChartColors.js +26 -0
- package/build/dist/UI/Components/Charts/ChartLibrary/Utils/ChartColors.js.map +1 -1
- package/build/dist/UI/Components/Charts/Line/LineChart.js +3 -1
- package/build/dist/UI/Components/Charts/Line/LineChart.js.map +1 -1
- package/build/dist/UI/Components/MoreMenu/MoreMenu.js +44 -4
- package/build/dist/UI/Components/MoreMenu/MoreMenu.js.map +1 -1
- package/package.json +1 -1
|
@@ -560,10 +560,14 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
560
560
|
aggregateBy.aggregateColumnName.toString();
|
|
561
561
|
const aggregationTimestampColumn: string =
|
|
562
562
|
aggregateBy.aggregationTimestampColumnName.toString();
|
|
563
|
-
const
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
563
|
+
const resolvedInterval: AggregationInterval =
|
|
564
|
+
AggregateUtil.getAggregationInterval({
|
|
565
|
+
startDate: aggregateBy.startTimestamp!,
|
|
566
|
+
endDate: aggregateBy.endTimestamp!,
|
|
567
|
+
aggregationInterval: aggregateBy.aggregationInterval,
|
|
568
|
+
});
|
|
569
|
+
const bucketByTime: boolean =
|
|
570
|
+
!AggregateUtil.isTotalAggregation(resolvedInterval);
|
|
567
571
|
|
|
568
572
|
/*
|
|
569
573
|
* Group-by columns from the caller need to be carried through the
|
|
@@ -671,7 +675,7 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
671
675
|
* always whole numbers in practice).
|
|
672
676
|
*/
|
|
673
677
|
statement.append(
|
|
674
|
-
`SELECT quantileExactWeighted(${percentileLevel})(__pcl_pair.1, toUInt64(greatest(0, round(__pcl_pair.2)))) as ${aggregationColumn},
|
|
678
|
+
`SELECT quantileExactWeighted(${percentileLevel})(__pcl_pair.1, toUInt64(greatest(0, round(__pcl_pair.2)))) as ${aggregationColumn}, ${AggregateUtil.buildBucketTimestampSelect(resolvedInterval, aggregationTimestampColumn)}`,
|
|
675
679
|
);
|
|
676
680
|
|
|
677
681
|
for (const key of groupByKeys) {
|
|
@@ -699,13 +703,33 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
699
703
|
statement.append(this.getRetentionReadFilter());
|
|
700
704
|
statement.append(SQL`) `);
|
|
701
705
|
|
|
702
|
-
|
|
706
|
+
/*
|
|
707
|
+
* `Total` collapses the window into one row per group: the timestamp
|
|
708
|
+
* becomes an aggregate (`min(...)`) and drops out of GROUP BY, and
|
|
709
|
+
* when nothing else is grouped the clause is omitted entirely.
|
|
710
|
+
*/
|
|
711
|
+
const percentileGroupByTerms: Array<string> = [];
|
|
712
|
+
if (bucketByTime) {
|
|
713
|
+
percentileGroupByTerms.push(aggregationTimestampColumn);
|
|
714
|
+
}
|
|
703
715
|
for (const key of groupByKeys) {
|
|
704
|
-
|
|
716
|
+
percentileGroupByTerms.push(key);
|
|
705
717
|
}
|
|
706
718
|
attributeGroupKeys.forEach((_key: string, index: number) => {
|
|
707
|
-
|
|
719
|
+
percentileGroupByTerms.push(`__attr_grp_${index}`);
|
|
708
720
|
});
|
|
721
|
+
if (percentileGroupByTerms.length > 0) {
|
|
722
|
+
statement
|
|
723
|
+
.append(SQL` GROUP BY `)
|
|
724
|
+
.append(percentileGroupByTerms.join(", "));
|
|
725
|
+
} else {
|
|
726
|
+
/*
|
|
727
|
+
* Group-less Total: suppress the single phantom row an empty window
|
|
728
|
+
* would otherwise return (min(time) → 1970 epoch). Only reachable
|
|
729
|
+
* under Total — a bucketed query always groups by the time column.
|
|
730
|
+
*/
|
|
731
|
+
statement.append(SQL` HAVING count() > 0`);
|
|
732
|
+
}
|
|
709
733
|
|
|
710
734
|
statement.append(SQL` ORDER BY `).append(sortStatement);
|
|
711
735
|
|
|
@@ -832,10 +856,14 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
832
856
|
aggregateBy.aggregateColumnName.toString();
|
|
833
857
|
const aggregationTimestampColumn: string =
|
|
834
858
|
aggregateBy.aggregationTimestampColumnName.toString();
|
|
835
|
-
const
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
859
|
+
const resolvedInterval: AggregationInterval =
|
|
860
|
+
AggregateUtil.getAggregationInterval({
|
|
861
|
+
startDate: aggregateBy.startTimestamp!,
|
|
862
|
+
endDate: aggregateBy.endTimestamp!,
|
|
863
|
+
aggregationInterval: aggregateBy.aggregationInterval,
|
|
864
|
+
});
|
|
865
|
+
const bucketByTime: boolean =
|
|
866
|
+
!AggregateUtil.isTotalAggregation(resolvedInterval);
|
|
839
867
|
|
|
840
868
|
const groupByKeys: Array<string> = [];
|
|
841
869
|
if (aggregateBy.groupBy) {
|
|
@@ -871,7 +899,7 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
871
899
|
const statement: Statement = SQL``;
|
|
872
900
|
|
|
873
901
|
statement.append(
|
|
874
|
-
`SELECT ${aggregationExpression} as ${aggregationColumn},
|
|
902
|
+
`SELECT ${aggregationExpression} as ${aggregationColumn}, ${AggregateUtil.buildBucketTimestampSelect(resolvedInterval, aggregationTimestampColumn)}`,
|
|
875
903
|
);
|
|
876
904
|
for (const key of groupByKeys) {
|
|
877
905
|
statement.append(`, ${key}`);
|
|
@@ -916,13 +944,32 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
916
944
|
statement.append(this.getRetentionReadFilter());
|
|
917
945
|
}
|
|
918
946
|
|
|
919
|
-
|
|
947
|
+
/*
|
|
948
|
+
* `Total` drops the time bucket from GROUP BY (it becomes an
|
|
949
|
+
* aggregate `min(...)`), leaving one row per group — or a single
|
|
950
|
+
* global row when nothing else is grouped, in which case the GROUP
|
|
951
|
+
* BY clause is omitted entirely.
|
|
952
|
+
*/
|
|
953
|
+
const scalarGroupByTerms: Array<string> = [];
|
|
954
|
+
if (bucketByTime) {
|
|
955
|
+
scalarGroupByTerms.push(aggregationTimestampColumn);
|
|
956
|
+
}
|
|
920
957
|
for (const key of groupByKeys) {
|
|
921
|
-
|
|
958
|
+
scalarGroupByTerms.push(key);
|
|
922
959
|
}
|
|
923
960
|
attributeGroupKeys.forEach((_key: string, index: number) => {
|
|
924
|
-
|
|
961
|
+
scalarGroupByTerms.push(`__attr_grp_${index}`);
|
|
925
962
|
});
|
|
963
|
+
if (scalarGroupByTerms.length > 0) {
|
|
964
|
+
statement.append(SQL` GROUP BY `).append(scalarGroupByTerms.join(", "));
|
|
965
|
+
} else {
|
|
966
|
+
/*
|
|
967
|
+
* Group-less Total: suppress the single phantom row an empty window
|
|
968
|
+
* would otherwise return (min(time) → 1970 epoch). Only reachable
|
|
969
|
+
* under Total — a bucketed query always groups by the time column.
|
|
970
|
+
*/
|
|
971
|
+
statement.append(SQL` HAVING count() > 0`);
|
|
972
|
+
}
|
|
926
973
|
|
|
927
974
|
statement.append(SQL` ORDER BY `).append(sortStatement);
|
|
928
975
|
|
|
@@ -1044,18 +1091,54 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
1044
1091
|
.append(outerWhereStatement)
|
|
1045
1092
|
.append(SQL` AND retentionDate >= now()`);
|
|
1046
1093
|
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1094
|
+
/*
|
|
1095
|
+
* Mirror the base builder's Total handling: the SELECT above (shared
|
|
1096
|
+
* toAggregateSelectStatement) already emits `min(time)` for a `Total`
|
|
1097
|
+
* interval, so the time column must NOT appear in GROUP BY — grouping
|
|
1098
|
+
* by an aggregate alias is a ClickHouse error. Omit the clause
|
|
1099
|
+
* entirely when nothing else is grouped.
|
|
1100
|
+
*/
|
|
1101
|
+
const mutableResolvedInterval: AggregationInterval =
|
|
1102
|
+
AggregateUtil.getAggregationInterval({
|
|
1103
|
+
startDate: aggregateBy.startTimestamp!,
|
|
1104
|
+
endDate: aggregateBy.endTimestamp!,
|
|
1105
|
+
aggregationInterval: aggregateBy.aggregationInterval,
|
|
1106
|
+
});
|
|
1107
|
+
const mutableBucketByTime: boolean = !AggregateUtil.isTotalAggregation(
|
|
1108
|
+
mutableResolvedInterval,
|
|
1109
|
+
);
|
|
1110
|
+
const mutableHasGroupBy: boolean = Boolean(
|
|
1111
|
+
aggregateBy.groupBy && Object.keys(aggregateBy.groupBy).length > 0,
|
|
1112
|
+
);
|
|
1050
1113
|
|
|
1051
|
-
if (
|
|
1114
|
+
if (mutableBucketByTime) {
|
|
1115
|
+
statement
|
|
1116
|
+
.append(SQL` GROUP BY `)
|
|
1117
|
+
.append(`${aggregateBy.aggregationTimestampColumnName.toString()}`);
|
|
1118
|
+
if (mutableHasGroupBy) {
|
|
1119
|
+
statement
|
|
1120
|
+
.append(SQL` , `)
|
|
1121
|
+
.append(
|
|
1122
|
+
this.statementGenerator.toGroupByStatement(aggregateBy.groupBy!),
|
|
1123
|
+
);
|
|
1124
|
+
}
|
|
1125
|
+
} else if (mutableHasGroupBy) {
|
|
1052
1126
|
statement
|
|
1053
|
-
.append(SQL`
|
|
1127
|
+
.append(SQL` GROUP BY `)
|
|
1054
1128
|
.append(
|
|
1055
|
-
this.statementGenerator.toGroupByStatement(aggregateBy.groupBy),
|
|
1129
|
+
this.statementGenerator.toGroupByStatement(aggregateBy.groupBy!),
|
|
1056
1130
|
);
|
|
1057
1131
|
}
|
|
1058
1132
|
|
|
1133
|
+
/*
|
|
1134
|
+
* Group-less Total: suppress the single phantom row an empty window
|
|
1135
|
+
* would otherwise return (min(time) → 1970 epoch). No-op for bucketed
|
|
1136
|
+
* or grouped queries.
|
|
1137
|
+
*/
|
|
1138
|
+
if (!mutableBucketByTime && !mutableHasGroupBy) {
|
|
1139
|
+
statement.append(SQL` HAVING count() > 0`);
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1059
1142
|
statement.append(SQL` ORDER BY `).append(sortStatement);
|
|
1060
1143
|
|
|
1061
1144
|
statement.append(
|
|
@@ -1236,14 +1319,18 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
1236
1319
|
const interval: AggregationInterval = AggregateUtil.getAggregationInterval({
|
|
1237
1320
|
startDate: aggregateBy.startTimestamp!,
|
|
1238
1321
|
endDate: aggregateBy.endTimestamp!,
|
|
1322
|
+
aggregationInterval: aggregateBy.aggregationInterval,
|
|
1239
1323
|
});
|
|
1240
1324
|
/*
|
|
1241
|
-
* The MV is bucketed at 1 minute, so
|
|
1242
|
-
* (Minute / Hour / Day / Week / Month / Year)
|
|
1243
|
-
*
|
|
1244
|
-
*
|
|
1325
|
+
* The MV is bucketed at 1 minute, so every time-bucketed interval
|
|
1326
|
+
* (Minute / Hour / Day / Week / Month / Year) is >= MV resolution and
|
|
1327
|
+
* acceptable — the honored interval flows into the date_trunc below.
|
|
1328
|
+
* `Total` (whole-window, no bucketing) is the one exception: fall back
|
|
1329
|
+
* to the raw-table builder, which knows how to collapse the window.
|
|
1245
1330
|
*/
|
|
1246
|
-
|
|
1331
|
+
if (AggregateUtil.isTotalAggregation(interval)) {
|
|
1332
|
+
return null;
|
|
1333
|
+
}
|
|
1247
1334
|
|
|
1248
1335
|
const queryRecord: Record<string, unknown> =
|
|
1249
1336
|
(aggregateBy.query as unknown as Record<string, unknown>) || {};
|
|
@@ -1510,8 +1597,17 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
|
|
|
1510
1597
|
const interval: AggregationInterval = AggregateUtil.getAggregationInterval({
|
|
1511
1598
|
startDate: aggregateBy.startTimestamp!,
|
|
1512
1599
|
endDate: aggregateBy.endTimestamp!,
|
|
1600
|
+
aggregationInterval: aggregateBy.aggregationInterval,
|
|
1513
1601
|
});
|
|
1514
|
-
|
|
1602
|
+
/*
|
|
1603
|
+
* `Total` (whole-window, no bucketing) is not served by this
|
|
1604
|
+
* time-bucketed per-host MV — fall back to the raw-table builder.
|
|
1605
|
+
* Every other interval is >= the MV's 1-minute resolution and flows
|
|
1606
|
+
* into the date_trunc below.
|
|
1607
|
+
*/
|
|
1608
|
+
if (AggregateUtil.isTotalAggregation(interval)) {
|
|
1609
|
+
return null;
|
|
1610
|
+
}
|
|
1515
1611
|
|
|
1516
1612
|
if (!this.database) {
|
|
1517
1613
|
this.useDefaultDatabase();
|
|
@@ -245,6 +245,17 @@ export class SpanService extends AnalyticsDatabaseService<Span> {
|
|
|
245
245
|
return null;
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
+
/*
|
|
249
|
+
* This projection path derives its bucket interval from the window
|
|
250
|
+
* and buckets by minute; it does not implement the explicit
|
|
251
|
+
* `aggregationInterval` override (nor the `Total` whole-window mode).
|
|
252
|
+
* When a caller pins an interval, fall back to the generic base
|
|
253
|
+
* builder, which honors it.
|
|
254
|
+
*/
|
|
255
|
+
if (aggregateBy.aggregationInterval !== undefined) {
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
|
|
248
259
|
/*
|
|
249
260
|
* Only the aggregates whose states the projection stores. P95 etc.
|
|
250
261
|
* must fall back: the projection has no quantile(0.95) state and
|
|
@@ -16,12 +16,51 @@ export class AggregateUtil {
|
|
|
16
16
|
* code reconstructing the server's bucket grid (heartbeat
|
|
17
17
|
* availability charts) always agrees with the interval the
|
|
18
18
|
* statement generator compiles into `toStartOfInterval(...)`.
|
|
19
|
+
*
|
|
20
|
+
* `aggregationInterval`, when provided, pins the bucket size
|
|
21
|
+
* independent of the window (including `Total` for a single
|
|
22
|
+
* whole-window bucket per group).
|
|
19
23
|
*/
|
|
20
24
|
@CaptureSpan()
|
|
21
25
|
public static getAggregationInterval(data: {
|
|
22
26
|
startDate: Date;
|
|
23
27
|
endDate: Date;
|
|
28
|
+
aggregationInterval?: AggregationInterval | undefined;
|
|
24
29
|
}): AggregationInterval {
|
|
25
30
|
return AggregationIntervalUtil.getAggregationIntervalForWindow(data);
|
|
26
31
|
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* True when the resolved interval means "no time bucketing" — the
|
|
35
|
+
* entire window collapses into one aggregate per group.
|
|
36
|
+
*/
|
|
37
|
+
public static isTotalAggregation(interval: AggregationInterval): boolean {
|
|
38
|
+
return interval === AggregationInterval.Total;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The SELECT fragment that produces the bucket timestamp column,
|
|
43
|
+
* already aliased to `timestampColumn`. Shared by every aggregate SQL
|
|
44
|
+
* builder so the `Total` (whole-window) shape stays identical across
|
|
45
|
+
* paths:
|
|
46
|
+
*
|
|
47
|
+
* - normal interval → `date_trunc('<i>', toStartOfInterval(col, INTERVAL 1 <i>)) as col`
|
|
48
|
+
* - Total → `min(col) as col` (one row per group; the
|
|
49
|
+
* earliest sample timestamp in the window is the bucket label)
|
|
50
|
+
*
|
|
51
|
+
* `timestampColumn` is a model-validated identifier (see
|
|
52
|
+
* AnalyticsDatabaseService._aggregateBy), so raw interpolation here is
|
|
53
|
+
* consistent with the surrounding builders.
|
|
54
|
+
*/
|
|
55
|
+
public static buildBucketTimestampSelect(
|
|
56
|
+
resolvedInterval: AggregationInterval,
|
|
57
|
+
timestampColumn: string,
|
|
58
|
+
): string {
|
|
59
|
+
if (AggregateUtil.isTotalAggregation(resolvedInterval)) {
|
|
60
|
+
return `min(${timestampColumn}) as ${timestampColumn}`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const interval: string = resolvedInterval.toLowerCase();
|
|
64
|
+
return `date_trunc('${interval}', toStartOfInterval(${timestampColumn}, INTERVAL 1 ${interval})) as ${timestampColumn}`;
|
|
65
|
+
}
|
|
27
66
|
}
|
|
@@ -51,6 +51,7 @@ import JSONFunctions from "../../../Types/JSONFunctions";
|
|
|
51
51
|
import AggregateBy, {
|
|
52
52
|
AggregateUtil,
|
|
53
53
|
} from "../../Types/AnalyticsDatabase/AggregateBy";
|
|
54
|
+
import AggregationInterval from "../../../Types/BaseDatabase/AggregationInterval";
|
|
54
55
|
import CaptureSpan from "../Telemetry/CaptureSpan";
|
|
55
56
|
import { getPercentileLevel } from "../../../Types/BaseDatabase/AggregationType";
|
|
56
57
|
|
|
@@ -1194,10 +1195,12 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1194
1195
|
|
|
1195
1196
|
const selectStatement: Statement = new Statement();
|
|
1196
1197
|
|
|
1197
|
-
const aggregationInterval:
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1198
|
+
const aggregationInterval: AggregationInterval =
|
|
1199
|
+
AggregateUtil.getAggregationInterval({
|
|
1200
|
+
startDate: aggregateBy.startTimestamp!,
|
|
1201
|
+
endDate: aggregateBy.endTimestamp!,
|
|
1202
|
+
aggregationInterval: aggregateBy.aggregationInterval,
|
|
1203
|
+
});
|
|
1201
1204
|
const aggregationColumn: string =
|
|
1202
1205
|
aggregateBy.aggregateColumnName.toString();
|
|
1203
1206
|
const aggregationTimestampColumn: string =
|
|
@@ -1212,7 +1215,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1212
1215
|
: `${aggregateBy.aggregationType.toLocaleLowerCase()}(${aggregationColumn})`;
|
|
1213
1216
|
|
|
1214
1217
|
selectStatement.append(
|
|
1215
|
-
`${aggregationExpression} as ${aggregationColumn},
|
|
1218
|
+
`${aggregationExpression} as ${aggregationColumn}, ${AggregateUtil.buildBucketTimestampSelect(aggregationInterval, aggregationTimestampColumn)}`,
|
|
1216
1219
|
);
|
|
1217
1220
|
|
|
1218
1221
|
const columns: Array<string> = [
|
|
@@ -11,6 +11,7 @@ import AnalyticsTableEngine from "../../../Types/AnalyticsDatabase/AnalyticsTabl
|
|
|
11
11
|
import AnalyticsTableColumn from "../../../Types/AnalyticsDatabase/TableColumn";
|
|
12
12
|
import TableColumnType from "../../../Types/AnalyticsDatabase/TableColumnType";
|
|
13
13
|
import AggregationType from "../../../Types/BaseDatabase/AggregationType";
|
|
14
|
+
import SortOrder from "../../../Types/BaseDatabase/SortOrder";
|
|
14
15
|
import BadDataException from "../../../Types/Exception/BadDataException";
|
|
15
16
|
import GenericObject from "../../../Types/GenericObject";
|
|
16
17
|
import ObjectID from "../../../Types/ObjectID";
|
|
@@ -314,6 +315,89 @@ describe("AnalyticsDatabaseService", () => {
|
|
|
314
315
|
),
|
|
315
316
|
).rejects.toThrow("Invalid aggregationTimestampColumnName");
|
|
316
317
|
});
|
|
318
|
+
|
|
319
|
+
test("should reject an unrecognized aggregationInterval", async () => {
|
|
320
|
+
await expect(
|
|
321
|
+
service.aggregateBy(
|
|
322
|
+
makeAggregateBy({ aggregationInterval: "NotAnInterval" }),
|
|
323
|
+
),
|
|
324
|
+
).rejects.toThrow("Invalid aggregationInterval");
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
/*
|
|
329
|
+
* The base AnalyticsDatabaseService.toAggregateStatement is what every
|
|
330
|
+
* analytics model that does NOT override aggregation (Log, AuditLog, …)
|
|
331
|
+
* uses. Exercise its aggregationInterval / Total GROUP BY assembly here,
|
|
332
|
+
* since MetricService's own builders bypass it.
|
|
333
|
+
*/
|
|
334
|
+
describe("toAggregateStatement aggregationInterval (base path)", () => {
|
|
335
|
+
beforeEach(() => {
|
|
336
|
+
jest.spyOn(logger, "debug").mockImplementation(() => {
|
|
337
|
+
return undefined!;
|
|
338
|
+
});
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
afterEach(() => {
|
|
342
|
+
jest.restoreAllMocks();
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
const makeBaseAggregateBy: (overrides?: Record<string, unknown>) => any = (
|
|
346
|
+
overrides: Record<string, unknown> = {},
|
|
347
|
+
): any => {
|
|
348
|
+
return {
|
|
349
|
+
aggregationType: AggregationType.Sum,
|
|
350
|
+
aggregateColumnName: "column_2",
|
|
351
|
+
aggregationTimestampColumnName: "column_ObjectID",
|
|
352
|
+
startTimestamp: new Date("2024-01-01"),
|
|
353
|
+
endTimestamp: new Date("2024-01-02"),
|
|
354
|
+
query: {},
|
|
355
|
+
sort: { column_ObjectID: SortOrder.Ascending },
|
|
356
|
+
limit: 10,
|
|
357
|
+
skip: 0,
|
|
358
|
+
props: { isRoot: true },
|
|
359
|
+
...overrides,
|
|
360
|
+
};
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
test("derives the window bucket and groups by time when interval is omitted", () => {
|
|
364
|
+
const query: string = service.toAggregateStatement(makeBaseAggregateBy())
|
|
365
|
+
.statement.query;
|
|
366
|
+
|
|
367
|
+
// 1-day window derives an Hour bucket, grouped by the time column.
|
|
368
|
+
expect(query).toContain(
|
|
369
|
+
"date_trunc('hour', toStartOfInterval(column_ObjectID, INTERVAL 1 hour))",
|
|
370
|
+
);
|
|
371
|
+
expect(query).toContain("GROUP BY column_ObjectID");
|
|
372
|
+
expect(query).not.toContain("min(column_ObjectID)");
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
test("Total with no group-by drops the time bucket and guards empty windows", () => {
|
|
376
|
+
const query: string = service.toAggregateStatement(
|
|
377
|
+
makeBaseAggregateBy({ aggregationInterval: "Total" }),
|
|
378
|
+
).statement.query;
|
|
379
|
+
|
|
380
|
+
expect(query).toContain("min(column_ObjectID) as column_ObjectID");
|
|
381
|
+
expect(query).not.toContain("date_trunc");
|
|
382
|
+
expect(query).not.toContain("GROUP BY");
|
|
383
|
+
expect(query).toContain("HAVING count() > 0");
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
test("Total with a model-column group-by keeps the column, drops the time bucket", () => {
|
|
387
|
+
const query: string = service.toAggregateStatement(
|
|
388
|
+
makeBaseAggregateBy({
|
|
389
|
+
aggregationInterval: "Total",
|
|
390
|
+
groupBy: { column_1: true },
|
|
391
|
+
}),
|
|
392
|
+
).statement.query;
|
|
393
|
+
|
|
394
|
+
expect(query).toContain("min(column_ObjectID) as column_ObjectID");
|
|
395
|
+
expect(query).toContain("GROUP BY");
|
|
396
|
+
// The time column must not be a grouping key under Total.
|
|
397
|
+
expect(query).not.toContain("GROUP BY column_ObjectID");
|
|
398
|
+
// A grouped query never needs the empty-window guard.
|
|
399
|
+
expect(query).not.toContain("HAVING count() > 0");
|
|
400
|
+
});
|
|
317
401
|
});
|
|
318
402
|
|
|
319
403
|
describe("toDeleteStatement", () => {
|
|
@@ -3,6 +3,7 @@ import { MetricService } from "../../../Server/Services/MetricService";
|
|
|
3
3
|
import Metric from "../../../Models/AnalyticsModels/Metric";
|
|
4
4
|
import AggregateBy from "../../../Server/Types/AnalyticsDatabase/AggregateBy";
|
|
5
5
|
import { Statement } from "../../../Server/Utils/AnalyticsDatabase/Statement";
|
|
6
|
+
import AggregationInterval from "../../../Types/BaseDatabase/AggregationInterval";
|
|
6
7
|
import AggregationType from "../../../Types/BaseDatabase/AggregationType";
|
|
7
8
|
import InBetween from "../../../Types/BaseDatabase/InBetween";
|
|
8
9
|
import SortOrder from "../../../Types/BaseDatabase/SortOrder";
|
|
@@ -191,4 +192,170 @@ describe("MetricService aggregate statement generation", () => {
|
|
|
191
192
|
expect(query).toContain("GROUP BY time");
|
|
192
193
|
});
|
|
193
194
|
});
|
|
195
|
+
|
|
196
|
+
/*
|
|
197
|
+
* The `aggregationInterval` override lets callers pin the time bucket
|
|
198
|
+
* independent of the query window (the two scenarios from the customer
|
|
199
|
+
* thread: "daily uptime per host over a week" and "one weekly average
|
|
200
|
+
* per host"). The buildAggregateBy window is 5 minutes, so its
|
|
201
|
+
* window-derived interval is Minute — every assertion below proves the
|
|
202
|
+
* override wins over that default.
|
|
203
|
+
*/
|
|
204
|
+
describe("explicit aggregationInterval override", () => {
|
|
205
|
+
it("pins a Day bucket over the window-derived Minute (per-host daily)", () => {
|
|
206
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
207
|
+
aggregationInterval: AggregationInterval.Day,
|
|
208
|
+
groupByAttributeKeys: ["host.name"],
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
const query: string = getQuery(aggregateBy);
|
|
212
|
+
|
|
213
|
+
expect(query).toContain(
|
|
214
|
+
"date_trunc('day', toStartOfInterval(time, INTERVAL 1 day))",
|
|
215
|
+
);
|
|
216
|
+
expect(query).toContain("GROUP BY time, __attr_grp_0");
|
|
217
|
+
// The 5-minute window's default Minute bucket must NOT leak through.
|
|
218
|
+
expect(query).not.toContain("date_trunc('minute'");
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it("honors an explicit Day interval on the minute MV fast path", () => {
|
|
222
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
223
|
+
aggregationInterval: AggregationInterval.Day,
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
const query: string = getQuery(aggregateBy);
|
|
227
|
+
|
|
228
|
+
expect(query).toContain("MetricItemAggMV1m");
|
|
229
|
+
expect(query).toContain("date_trunc('day'");
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it("honors an explicit Day interval on the percentile path", () => {
|
|
233
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
234
|
+
aggregationType: AggregationType.P90,
|
|
235
|
+
aggregationInterval: AggregationInterval.Day,
|
|
236
|
+
groupByAttributeKeys: ["host.name"],
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
const query: string = getQuery(aggregateBy);
|
|
240
|
+
|
|
241
|
+
expect(query).toContain("quantileExactWeighted(0.9)");
|
|
242
|
+
expect(query).toContain("date_trunc('day'");
|
|
243
|
+
expect(query).toContain("GROUP BY time, __attr_grp_0");
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
it("reproduces the window-derived bucket SQL when omitted (regression)", () => {
|
|
247
|
+
/*
|
|
248
|
+
* Distribution hint forces the base scalar builder (skips the MVs).
|
|
249
|
+
* The 5-minute window must still derive a Minute bucket, unchanged.
|
|
250
|
+
*/
|
|
251
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy();
|
|
252
|
+
(
|
|
253
|
+
service as unknown as {
|
|
254
|
+
pointTypeHintByAggregate: WeakMap<AggregateBy<Metric>, string | null>;
|
|
255
|
+
}
|
|
256
|
+
).pointTypeHintByAggregate.set(aggregateBy, "Histogram");
|
|
257
|
+
|
|
258
|
+
const query: string = getQuery(aggregateBy);
|
|
259
|
+
|
|
260
|
+
expect(query).toContain(
|
|
261
|
+
"date_trunc('minute', toStartOfInterval(time, INTERVAL 1 minute))",
|
|
262
|
+
);
|
|
263
|
+
expect(query).toContain("GROUP BY time");
|
|
264
|
+
expect(query).not.toContain("min(time) as time");
|
|
265
|
+
expect(query).not.toContain("HAVING count() > 0");
|
|
266
|
+
});
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
/*
|
|
270
|
+
* `Total` = no time bucketing: the entire window collapses into a single
|
|
271
|
+
* aggregate per group. The bucket timestamp becomes `min(time)` (an
|
|
272
|
+
* aggregate, not a group key), so `time` must drop out of GROUP BY —
|
|
273
|
+
* and the clause disappears entirely when nothing else is grouped.
|
|
274
|
+
*/
|
|
275
|
+
describe("Total (whole-window) aggregation", () => {
|
|
276
|
+
it("collapses the window into one row per host (scalar Avg)", () => {
|
|
277
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
278
|
+
aggregationInterval: AggregationInterval.Total,
|
|
279
|
+
groupByAttributeKeys: ["host.name"],
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
const query: string = getQuery(aggregateBy);
|
|
283
|
+
|
|
284
|
+
expect(query).toContain("min(time) as time");
|
|
285
|
+
expect(query).not.toContain("date_trunc");
|
|
286
|
+
expect(query).toContain("GROUP BY __attr_grp_0");
|
|
287
|
+
expect(query).not.toContain("GROUP BY time");
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
it("emits a single global aggregate with no GROUP BY when nothing is grouped", () => {
|
|
291
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
292
|
+
aggregationInterval: AggregationInterval.Total,
|
|
293
|
+
/*
|
|
294
|
+
* An attribute filter (not a host filter) blocks every MV and
|
|
295
|
+
* routes to the base scalar builder without a group-by.
|
|
296
|
+
*/
|
|
297
|
+
query: {
|
|
298
|
+
projectId: projectId,
|
|
299
|
+
name: "http.client.request.duration",
|
|
300
|
+
time: new InBetween(startDate, endDate),
|
|
301
|
+
attributes: { "oneuptime.service.name": "starship-web" },
|
|
302
|
+
} as unknown as AggregateBy<Metric>["query"],
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
const query: string = getQuery(aggregateBy);
|
|
306
|
+
|
|
307
|
+
expect(query).not.toContain("MetricItemAggMV1m");
|
|
308
|
+
expect(query).toContain("min(time) as time");
|
|
309
|
+
expect(query).not.toContain("GROUP BY");
|
|
310
|
+
// The phantom row an empty window would return is suppressed.
|
|
311
|
+
expect(query).toContain("HAVING count() > 0");
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
it("falls back from the minute MV to the base table for Total", () => {
|
|
315
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
316
|
+
aggregationInterval: AggregationInterval.Total,
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
const query: string = getQuery(aggregateBy);
|
|
320
|
+
|
|
321
|
+
// The time-bucketed MV cannot serve a whole-window aggregation.
|
|
322
|
+
expect(query).not.toContain("MetricItemAggMV1m");
|
|
323
|
+
expect(query).toContain("min(time) as time");
|
|
324
|
+
expect(query).not.toContain("GROUP BY");
|
|
325
|
+
expect(query).toContain("HAVING count() > 0");
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
it("keeps a model-column groupBy while dropping the time bucket (Total)", () => {
|
|
329
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
330
|
+
aggregationInterval: AggregationInterval.Total,
|
|
331
|
+
groupBy: { metricPointType: true } as AggregateBy<Metric>["groupBy"],
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
const query: string = getQuery(aggregateBy);
|
|
335
|
+
|
|
336
|
+
expect(query).toContain("min(time) as time");
|
|
337
|
+
// The model column survives as the sole grouping key...
|
|
338
|
+
expect(query).toContain("GROUP BY metricPointType");
|
|
339
|
+
// ...and the time bucket is gone (it is now an aggregate).
|
|
340
|
+
expect(query).not.toContain("GROUP BY time");
|
|
341
|
+
expect(query).not.toContain("date_trunc");
|
|
342
|
+
// A grouped query never emits the empty-window guard.
|
|
343
|
+
expect(query).not.toContain("HAVING count() > 0");
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
it("collapses the window into one row per host on the percentile path", () => {
|
|
347
|
+
const aggregateBy: AggregateBy<Metric> = buildAggregateBy({
|
|
348
|
+
aggregationType: AggregationType.P90,
|
|
349
|
+
aggregationInterval: AggregationInterval.Total,
|
|
350
|
+
groupByAttributeKeys: ["host.name"],
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
const query: string = getQuery(aggregateBy);
|
|
354
|
+
|
|
355
|
+
expect(query).toContain("quantileExactWeighted(0.9)");
|
|
356
|
+
expect(query).toContain("min(time) as time");
|
|
357
|
+
expect(query).toContain("GROUP BY __attr_grp_0");
|
|
358
|
+
expect(query).not.toContain("date_trunc");
|
|
359
|
+
});
|
|
360
|
+
});
|
|
194
361
|
});
|
|
@@ -53,6 +53,34 @@ describe("AggregationIntervalUtil", () => {
|
|
|
53
53
|
},
|
|
54
54
|
);
|
|
55
55
|
|
|
56
|
+
test("an explicit override wins over the window-derived interval", () => {
|
|
57
|
+
// A 2-hour window derives Minute, but the override pins Day.
|
|
58
|
+
expect(
|
|
59
|
+
AggregationIntervalUtil.getAggregationIntervalForWindow({
|
|
60
|
+
...windowOf(2 * HOUR),
|
|
61
|
+
aggregationInterval: AggregationInterval.Day,
|
|
62
|
+
}),
|
|
63
|
+
).toBe(AggregationInterval.Day);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("the Total override is returned verbatim (whole-window bucketing)", () => {
|
|
67
|
+
expect(
|
|
68
|
+
AggregationIntervalUtil.getAggregationIntervalForWindow({
|
|
69
|
+
...windowOf(14 * DAY),
|
|
70
|
+
aggregationInterval: AggregationInterval.Total,
|
|
71
|
+
}),
|
|
72
|
+
).toBe(AggregationInterval.Total);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("an invalid override falls through to the window-derived interval", () => {
|
|
76
|
+
expect(
|
|
77
|
+
AggregationIntervalUtil.getAggregationIntervalForWindow({
|
|
78
|
+
...windowOf(2 * HOUR),
|
|
79
|
+
aggregationInterval: "NotAnInterval" as AggregationInterval,
|
|
80
|
+
}),
|
|
81
|
+
).toBe(AggregationInterval.Minute);
|
|
82
|
+
});
|
|
83
|
+
|
|
56
84
|
test("returns the widest interval as the window grows", () => {
|
|
57
85
|
const order: Array<AggregationInterval> = [
|
|
58
86
|
AggregationInterval.Minute,
|
|
@@ -105,6 +133,18 @@ describe("AggregationIntervalUtil", () => {
|
|
|
105
133
|
},
|
|
106
134
|
);
|
|
107
135
|
|
|
136
|
+
test("Total has no fixed width, so it maps to a very large sentinel > Year", () => {
|
|
137
|
+
expect(
|
|
138
|
+
AggregationIntervalUtil.getAggregationIntervalMs(
|
|
139
|
+
AggregationInterval.Total,
|
|
140
|
+
),
|
|
141
|
+
).toBeGreaterThan(
|
|
142
|
+
AggregationIntervalUtil.getAggregationIntervalMs(
|
|
143
|
+
AggregationInterval.Year,
|
|
144
|
+
),
|
|
145
|
+
);
|
|
146
|
+
});
|
|
147
|
+
|
|
108
148
|
test("intervals are strictly increasing in width", () => {
|
|
109
149
|
const ordered: Array<AggregationInterval> = [
|
|
110
150
|
AggregationInterval.Minute,
|