@oneuptime/common 11.6.0 → 11.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts +41 -24
- package/Models/AnalyticsModels/MetricBaselineHourly.ts +2 -0
- package/Models/AnalyticsModels/MetricItemAggMV1m.ts +5 -2
- package/Models/AnalyticsModels/MetricItemAggMV1mByContainer.ts +5 -2
- package/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.ts +5 -2
- package/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.ts +5 -2
- package/Models/AnalyticsModels/MetricItemAggMV1mByService.ts +5 -2
- package/Server/Services/AnalyticsDatabaseService.ts +13 -1
- package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +41 -4
- package/Tests/Models/AnalyticsModels/AnalyticsBaseModel.test.ts +122 -0
- package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts +164 -3
- package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts +99 -0
- package/Tests/Types/HashCode.test.ts +41 -0
- package/Tests/Types/Log/LogQueryParser.test.ts +186 -0
- package/Tests/Utils/Dashboard/VariableInterpolation.test.ts +197 -0
- package/Tests/Utils/ValueFormatter.test.ts +305 -0
- package/Types/AnalyticsDatabase/TableColumn.ts +23 -0
- package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js +25 -20
- package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricBaselineHourly.js +2 -0
- package/build/dist/Models/AnalyticsModels/MetricBaselineHourly.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1m.js +5 -2
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1m.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByContainer.js +5 -2
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByContainer.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.js +5 -2
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.js +5 -2
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByService.js +5 -2
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByService.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +9 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +29 -4
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
- package/build/dist/Types/AnalyticsDatabase/TableColumn.js +7 -0
- package/build/dist/Types/AnalyticsDatabase/TableColumn.js.map +1 -1
- package/package.json +1 -1
|
@@ -77,6 +77,15 @@ export default class AnalyticsBaseModel extends CommonModel {
|
|
|
77
77
|
materializedViews?: Array<MaterializedView> | undefined;
|
|
78
78
|
enableMCP?: boolean | undefined;
|
|
79
79
|
ttlExpression?: string | undefined; // e.g. "retentionDate DELETE"
|
|
80
|
+
/*
|
|
81
|
+
* Generic event/entity tables carry `_id` and `createdAt`. Derived
|
|
82
|
+
* AggregatingMergeTree targets do not represent independently-addressable
|
|
83
|
+
* rows: their identity is exactly their aggregation key, and an arbitrary
|
|
84
|
+
* creation timestamp has no stable meaning after merges. Those models set
|
|
85
|
+
* this false so ClickHouse can enforce that every non-key column is an
|
|
86
|
+
* aggregate measure. Defaults true for backward compatibility.
|
|
87
|
+
*/
|
|
88
|
+
includeBaseColumns?: boolean | undefined;
|
|
80
89
|
/*
|
|
81
90
|
* Column that `findBy` falls back to when the caller doesn't
|
|
82
91
|
* specify a `sort`. Defaults to `createdAt` (matching the legacy
|
|
@@ -100,31 +109,39 @@ export default class AnalyticsBaseModel extends CommonModel {
|
|
|
100
109
|
this.tableEngine = data.tableEngine;
|
|
101
110
|
}
|
|
102
111
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
112
|
+
if (data.includeBaseColumns !== false) {
|
|
113
|
+
columns.push(
|
|
114
|
+
new AnalyticsTableColumn({
|
|
115
|
+
key: "_id",
|
|
116
|
+
title: "ID",
|
|
117
|
+
description: "ID of this object",
|
|
118
|
+
required: true,
|
|
119
|
+
type: TableColumnType.ObjectID,
|
|
120
|
+
/*
|
|
121
|
+
* Ids are UUIDv7 (time-ordered — see ObjectID.generateTimeOrdered),
|
|
122
|
+
* so consecutive rows share their 48-bit timestamp prefix and ZSTD
|
|
123
|
+
* compresses them well; random v4 ids were incompressible.
|
|
124
|
+
*/
|
|
125
|
+
codec: { codec: "ZSTD", level: 1 },
|
|
126
|
+
}),
|
|
127
|
+
);
|
|
118
128
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
129
|
+
columns.push(
|
|
130
|
+
new AnalyticsTableColumn({
|
|
131
|
+
key: "createdAt",
|
|
132
|
+
title: "Created",
|
|
133
|
+
description: "Date and Time when the object was created.",
|
|
134
|
+
required: true,
|
|
135
|
+
type: TableColumnType.Date,
|
|
136
|
+
}),
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (data.includeBaseColumns === false && !data.defaultSortColumn) {
|
|
141
|
+
throw new BadDataException(
|
|
142
|
+
"defaultSortColumn is required when includeBaseColumns is false because createdAt is not part of the table schema",
|
|
143
|
+
);
|
|
144
|
+
}
|
|
128
145
|
|
|
129
146
|
if (!data.primaryKeys || data.primaryKeys.length === 0) {
|
|
130
147
|
throw new BadDataException("Primary keys are required");
|
|
@@ -234,6 +234,8 @@ GROUP BY projectId, name, primaryEntityId, day, hourOfWeek`,
|
|
|
234
234
|
tableSettings:
|
|
235
235
|
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
236
236
|
ttlExpression: "day + INTERVAL 90 DAY",
|
|
237
|
+
includeBaseColumns: false,
|
|
238
|
+
defaultSortColumn: "day",
|
|
237
239
|
});
|
|
238
240
|
}
|
|
239
241
|
}
|
|
@@ -110,9 +110,10 @@ export default class MetricItemAggMV1m extends AnalyticsBaseModel {
|
|
|
110
110
|
key: "retentionDate",
|
|
111
111
|
title: "Retention Date",
|
|
112
112
|
description:
|
|
113
|
-
"
|
|
113
|
+
"Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
|
|
114
114
|
required: true,
|
|
115
115
|
type: TableColumnType.Date,
|
|
116
|
+
simpleAggregateFunction: "max",
|
|
116
117
|
});
|
|
117
118
|
|
|
118
119
|
super({
|
|
@@ -156,7 +157,7 @@ SELECT
|
|
|
156
157
|
countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
|
|
157
158
|
minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
|
|
158
159
|
maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
|
|
159
|
-
|
|
160
|
+
maxSimpleState(retentionDate) AS retentionDate
|
|
160
161
|
FROM MetricItemV3
|
|
161
162
|
GROUP BY projectId, name, primaryEntityId, bucketTime`,
|
|
162
163
|
},
|
|
@@ -172,6 +173,8 @@ GROUP BY projectId, name, primaryEntityId, bucketTime`,
|
|
|
172
173
|
tableSettings:
|
|
173
174
|
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
174
175
|
ttlExpression: "retentionDate DELETE",
|
|
176
|
+
includeBaseColumns: false,
|
|
177
|
+
defaultSortColumn: "bucketTime",
|
|
175
178
|
});
|
|
176
179
|
}
|
|
177
180
|
}
|
|
@@ -111,9 +111,10 @@ export default class MetricItemAggMV1mByContainer extends AnalyticsBaseModel {
|
|
|
111
111
|
key: "retentionDate",
|
|
112
112
|
title: "Retention Date",
|
|
113
113
|
description:
|
|
114
|
-
"
|
|
114
|
+
"Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
|
|
115
115
|
required: true,
|
|
116
116
|
type: TableColumnType.Date,
|
|
117
|
+
simpleAggregateFunction: "max",
|
|
117
118
|
});
|
|
118
119
|
|
|
119
120
|
super({
|
|
@@ -159,7 +160,7 @@ SELECT
|
|
|
159
160
|
countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
|
|
160
161
|
minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
|
|
161
162
|
maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
|
|
162
|
-
|
|
163
|
+
maxSimpleState(retentionDate) AS retentionDate
|
|
163
164
|
FROM MetricItemV3
|
|
164
165
|
WHERE containerEntityKey != ''
|
|
165
166
|
GROUP BY projectId, name, containerEntityKey, bucketTime`,
|
|
@@ -173,6 +174,8 @@ GROUP BY projectId, name, containerEntityKey, bucketTime`,
|
|
|
173
174
|
tableSettings:
|
|
174
175
|
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
175
176
|
ttlExpression: "retentionDate DELETE",
|
|
177
|
+
includeBaseColumns: false,
|
|
178
|
+
defaultSortColumn: "bucketTime",
|
|
176
179
|
});
|
|
177
180
|
}
|
|
178
181
|
}
|
|
@@ -114,9 +114,10 @@ export default class MetricItemAggMV1mByHostV2 extends AnalyticsBaseModel {
|
|
|
114
114
|
key: "retentionDate",
|
|
115
115
|
title: "Retention Date",
|
|
116
116
|
description:
|
|
117
|
-
"
|
|
117
|
+
"Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
|
|
118
118
|
required: true,
|
|
119
119
|
type: TableColumnType.Date,
|
|
120
|
+
simpleAggregateFunction: "max",
|
|
120
121
|
});
|
|
121
122
|
|
|
122
123
|
super({
|
|
@@ -162,7 +163,7 @@ SELECT
|
|
|
162
163
|
countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
|
|
163
164
|
minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
|
|
164
165
|
maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
|
|
165
|
-
|
|
166
|
+
maxSimpleState(retentionDate) AS retentionDate
|
|
166
167
|
FROM MetricItemV3
|
|
167
168
|
WHERE hostEntityKey != ''
|
|
168
169
|
GROUP BY projectId, name, hostEntityKey, bucketTime`,
|
|
@@ -176,6 +177,8 @@ GROUP BY projectId, name, hostEntityKey, bucketTime`,
|
|
|
176
177
|
tableSettings:
|
|
177
178
|
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
178
179
|
ttlExpression: "retentionDate DELETE",
|
|
180
|
+
includeBaseColumns: false,
|
|
181
|
+
defaultSortColumn: "bucketTime",
|
|
179
182
|
});
|
|
180
183
|
}
|
|
181
184
|
}
|
|
@@ -110,9 +110,10 @@ export default class MetricItemAggMV1mByK8sCluster extends AnalyticsBaseModel {
|
|
|
110
110
|
key: "retentionDate",
|
|
111
111
|
title: "Retention Date",
|
|
112
112
|
description:
|
|
113
|
-
"
|
|
113
|
+
"Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
|
|
114
114
|
required: true,
|
|
115
115
|
type: TableColumnType.Date,
|
|
116
|
+
simpleAggregateFunction: "max",
|
|
116
117
|
});
|
|
117
118
|
|
|
118
119
|
super({
|
|
@@ -160,7 +161,7 @@ SELECT
|
|
|
160
161
|
countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
|
|
161
162
|
minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
|
|
162
163
|
maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
|
|
163
|
-
|
|
164
|
+
maxSimpleState(retentionDate) AS retentionDate
|
|
164
165
|
FROM MetricItemV3
|
|
165
166
|
WHERE k8sClusterEntityKey != ''
|
|
166
167
|
GROUP BY projectId, name, k8sClusterEntityKey, bucketTime`,
|
|
@@ -174,6 +175,8 @@ GROUP BY projectId, name, k8sClusterEntityKey, bucketTime`,
|
|
|
174
175
|
tableSettings:
|
|
175
176
|
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
176
177
|
ttlExpression: "retentionDate DELETE",
|
|
178
|
+
includeBaseColumns: false,
|
|
179
|
+
defaultSortColumn: "bucketTime",
|
|
177
180
|
});
|
|
178
181
|
}
|
|
179
182
|
}
|
|
@@ -112,9 +112,10 @@ export default class MetricItemAggMV1mByService extends AnalyticsBaseModel {
|
|
|
112
112
|
key: "retentionDate",
|
|
113
113
|
title: "Retention Date",
|
|
114
114
|
description:
|
|
115
|
-
"
|
|
115
|
+
"Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
|
|
116
116
|
required: true,
|
|
117
117
|
type: TableColumnType.Date,
|
|
118
|
+
simpleAggregateFunction: "max",
|
|
118
119
|
});
|
|
119
120
|
|
|
120
121
|
super({
|
|
@@ -160,7 +161,7 @@ SELECT
|
|
|
160
161
|
countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
|
|
161
162
|
minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
|
|
162
163
|
maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
|
|
163
|
-
|
|
164
|
+
maxSimpleState(retentionDate) AS retentionDate
|
|
164
165
|
FROM MetricItemV3
|
|
165
166
|
WHERE serviceEntityKey != ''
|
|
166
167
|
GROUP BY projectId, name, serviceEntityKey, bucketTime`,
|
|
@@ -174,6 +175,8 @@ GROUP BY projectId, name, serviceEntityKey, bucketTime`,
|
|
|
174
175
|
tableSettings:
|
|
175
176
|
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
176
177
|
ttlExpression: "retentionDate DELETE",
|
|
178
|
+
includeBaseColumns: false,
|
|
179
|
+
defaultSortColumn: "bucketTime",
|
|
177
180
|
});
|
|
178
181
|
}
|
|
179
182
|
}
|
|
@@ -983,7 +983,19 @@ export default class AnalyticsDatabaseService<
|
|
|
983
983
|
onBeforeFind.select = {} as any;
|
|
984
984
|
}
|
|
985
985
|
|
|
986
|
-
|
|
986
|
+
/*
|
|
987
|
+
* Derived aggregate targets deliberately omit AnalyticsBaseModel's
|
|
988
|
+
* synthetic `_id`: the aggregation key is the row identity. Only force
|
|
989
|
+
* `_id` into generic reads when the model actually declares it.
|
|
990
|
+
*/
|
|
991
|
+
if (
|
|
992
|
+
this.model.tableColumns.some(
|
|
993
|
+
(column: AnalyticsTableColumn): boolean => {
|
|
994
|
+
return column.key === "_id";
|
|
995
|
+
},
|
|
996
|
+
) &&
|
|
997
|
+
!(onBeforeFind.select as any)["_id"]
|
|
998
|
+
) {
|
|
987
999
|
(onBeforeFind.select as any)["_id"] = true;
|
|
988
1000
|
}
|
|
989
1001
|
|
|
@@ -69,6 +69,8 @@ export interface EntityScopeQueryValue {
|
|
|
69
69
|
attributeValue: string;
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
+
const SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
73
|
+
|
|
72
74
|
export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
73
75
|
public model!: TBaseModel;
|
|
74
76
|
public modelType!: { new (): TBaseModel };
|
|
@@ -1361,6 +1363,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1361
1363
|
* column. Scalar types ignore the rest of the column.
|
|
1362
1364
|
*/
|
|
1363
1365
|
public toColumnType(column: AnalyticsTableColumn): Statement {
|
|
1366
|
+
const simpleAggregateFunction: string | undefined =
|
|
1367
|
+
column.simpleAggregateFunction;
|
|
1368
|
+
|
|
1369
|
+
if (simpleAggregateFunction !== undefined) {
|
|
1370
|
+
if (column.type === TableColumnType.AggregateFunction) {
|
|
1371
|
+
throw new BadDataException(
|
|
1372
|
+
`Column ${column.key} cannot declare both AggregateFunction and SimpleAggregateFunction.`,
|
|
1373
|
+
);
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
/*
|
|
1377
|
+
* The function name is emitted as raw ClickHouse DDL, so accept only a
|
|
1378
|
+
* plain function identifier. Besides catching accidental whitespace /
|
|
1379
|
+
* empty definitions, this prevents a model field from injecting another
|
|
1380
|
+
* type or table clause into generated schema SQL.
|
|
1381
|
+
*/
|
|
1382
|
+
if (
|
|
1383
|
+
simpleAggregateFunction.trim() !== simpleAggregateFunction ||
|
|
1384
|
+
!SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX.test(simpleAggregateFunction)
|
|
1385
|
+
) {
|
|
1386
|
+
throw new BadDataException(
|
|
1387
|
+
`Column ${column.key} has invalid simpleAggregateFunction "${simpleAggregateFunction}".`,
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1364
1392
|
if (column.type === TableColumnType.AggregateFunction) {
|
|
1365
1393
|
const def: string | undefined = column.aggregateFunctionDefinition;
|
|
1366
1394
|
if (!def) {
|
|
@@ -1371,7 +1399,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1371
1399
|
return SQL`AggregateFunction(`.append(def).append(SQL`)`);
|
|
1372
1400
|
}
|
|
1373
1401
|
|
|
1374
|
-
const
|
|
1402
|
+
const scalarStatement: Statement | undefined = {
|
|
1375
1403
|
[TableColumnType.Text]: SQL`String`,
|
|
1376
1404
|
[TableColumnType.ObjectID]: SQL`String`,
|
|
1377
1405
|
[TableColumnType.Boolean]: SQL`Bool`,
|
|
@@ -1394,13 +1422,21 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1394
1422
|
[TableColumnType.UInt64]: SQL`UInt64`,
|
|
1395
1423
|
}[column.type];
|
|
1396
1424
|
|
|
1397
|
-
if (!
|
|
1425
|
+
if (!scalarStatement) {
|
|
1398
1426
|
throw new BadDataException(
|
|
1399
1427
|
`Unknown column type: ${column.type}. Please add support for this column type.`,
|
|
1400
1428
|
);
|
|
1401
1429
|
}
|
|
1402
1430
|
|
|
1403
|
-
|
|
1431
|
+
if (simpleAggregateFunction) {
|
|
1432
|
+
return SQL`SimpleAggregateFunction(`
|
|
1433
|
+
.append(simpleAggregateFunction)
|
|
1434
|
+
.append(SQL`, `)
|
|
1435
|
+
.append(scalarStatement)
|
|
1436
|
+
.append(SQL`)`);
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
return scalarStatement;
|
|
1404
1440
|
}
|
|
1405
1441
|
|
|
1406
1442
|
/**
|
|
@@ -1413,7 +1449,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1413
1449
|
*/
|
|
1414
1450
|
public toFullColumnType(column: AnalyticsTableColumn): Statement {
|
|
1415
1451
|
const isAggregateFunction: boolean =
|
|
1416
|
-
column.type === TableColumnType.AggregateFunction
|
|
1452
|
+
column.type === TableColumnType.AggregateFunction ||
|
|
1453
|
+
Boolean(column.simpleAggregateFunction);
|
|
1417
1454
|
|
|
1418
1455
|
let typeStatement: Statement = this.toColumnType(column);
|
|
1419
1456
|
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { describe, expect, test } from "@jest/globals";
|
|
2
|
+
import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
|
|
3
|
+
import AnalyticsTableEngine from "../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
4
|
+
import AnalyticsTableColumn from "../../../Types/AnalyticsDatabase/TableColumn";
|
|
5
|
+
import TableColumnType from "../../../Types/AnalyticsDatabase/TableColumnType";
|
|
6
|
+
|
|
7
|
+
type ModelOptions = ConstructorParameters<typeof AnalyticsBaseModel>[0];
|
|
8
|
+
|
|
9
|
+
function projectIdColumn(): AnalyticsTableColumn {
|
|
10
|
+
return new AnalyticsTableColumn({
|
|
11
|
+
key: "projectId",
|
|
12
|
+
title: "Project ID",
|
|
13
|
+
description: "Tenant",
|
|
14
|
+
required: true,
|
|
15
|
+
type: TableColumnType.Text,
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function modelOptions(overrides: Partial<ModelOptions> = {}): ModelOptions {
|
|
20
|
+
return {
|
|
21
|
+
tableName: "TestAnalyticsTable",
|
|
22
|
+
singularName: "Test row",
|
|
23
|
+
pluralName: "Test rows",
|
|
24
|
+
tableColumns: [projectIdColumn()],
|
|
25
|
+
primaryKeys: ["projectId"],
|
|
26
|
+
sortKeys: ["projectId"],
|
|
27
|
+
partitionKey: "projectId",
|
|
28
|
+
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
29
|
+
...overrides,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe("AnalyticsBaseModel base-column policy", () => {
|
|
34
|
+
test("adds _id and createdAt by default for regular analytics models", () => {
|
|
35
|
+
const suppliedColumns: Array<AnalyticsTableColumn> = [projectIdColumn()];
|
|
36
|
+
const model: AnalyticsBaseModel = new AnalyticsBaseModel(
|
|
37
|
+
modelOptions({ tableColumns: suppliedColumns }),
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
expect(
|
|
41
|
+
model.tableColumns.map((column: AnalyticsTableColumn) => {
|
|
42
|
+
return column.key;
|
|
43
|
+
}),
|
|
44
|
+
).toEqual(["projectId", "_id", "createdAt"]);
|
|
45
|
+
expect(model.getTableColumn("_id")?.type).toBe(TableColumnType.ObjectID);
|
|
46
|
+
expect(model.getTableColumn("_id")?.codec).toEqual({
|
|
47
|
+
codec: "ZSTD",
|
|
48
|
+
level: 1,
|
|
49
|
+
});
|
|
50
|
+
expect(model.getTableColumn("createdAt")?.type).toBe(TableColumnType.Date);
|
|
51
|
+
|
|
52
|
+
// Construction copies the caller's array instead of mutating it.
|
|
53
|
+
expect(
|
|
54
|
+
suppliedColumns.map((column: AnalyticsTableColumn) => {
|
|
55
|
+
return column.key;
|
|
56
|
+
}),
|
|
57
|
+
).toEqual(["projectId"]);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("omits synthetic columns when includeBaseColumns is false", () => {
|
|
61
|
+
const model: AnalyticsBaseModel = new AnalyticsBaseModel(
|
|
62
|
+
modelOptions({
|
|
63
|
+
includeBaseColumns: false,
|
|
64
|
+
defaultSortColumn: "projectId",
|
|
65
|
+
}),
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
expect(
|
|
69
|
+
model.tableColumns.map((column: AnalyticsTableColumn) => {
|
|
70
|
+
return column.key;
|
|
71
|
+
}),
|
|
72
|
+
).toEqual(["projectId"]);
|
|
73
|
+
expect(model.getTableColumn("_id")).toBeNull();
|
|
74
|
+
expect(model.getTableColumn("createdAt")).toBeNull();
|
|
75
|
+
expect(model.defaultSortColumn).toBe("projectId");
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("requires a real default sort column when base columns are omitted", () => {
|
|
79
|
+
expect(() => {
|
|
80
|
+
return new AnalyticsBaseModel(
|
|
81
|
+
modelOptions({
|
|
82
|
+
includeBaseColumns: false,
|
|
83
|
+
}),
|
|
84
|
+
);
|
|
85
|
+
}).toThrow(
|
|
86
|
+
"defaultSortColumn is required when includeBaseColumns is false",
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("rejects createdAt as a strict model's default sort because it is absent", () => {
|
|
91
|
+
expect(() => {
|
|
92
|
+
return new AnalyticsBaseModel(
|
|
93
|
+
modelOptions({
|
|
94
|
+
includeBaseColumns: false,
|
|
95
|
+
defaultSortColumn: "createdAt",
|
|
96
|
+
}),
|
|
97
|
+
);
|
|
98
|
+
}).toThrow("defaultSortColumn createdAt is not part of tableColumns");
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("still validates primary and sort keys against the strict column set", () => {
|
|
102
|
+
expect(() => {
|
|
103
|
+
return new AnalyticsBaseModel(
|
|
104
|
+
modelOptions({
|
|
105
|
+
includeBaseColumns: false,
|
|
106
|
+
defaultSortColumn: "projectId",
|
|
107
|
+
sortKeys: ["projectId", "createdAt"],
|
|
108
|
+
}),
|
|
109
|
+
);
|
|
110
|
+
}).toThrow("Sort key createdAt is not part of tableColumns");
|
|
111
|
+
|
|
112
|
+
expect(() => {
|
|
113
|
+
return new AnalyticsBaseModel(
|
|
114
|
+
modelOptions({
|
|
115
|
+
includeBaseColumns: false,
|
|
116
|
+
defaultSortColumn: "projectId",
|
|
117
|
+
primaryKeys: ["_id"],
|
|
118
|
+
}),
|
|
119
|
+
);
|
|
120
|
+
}).toThrow("Primary key _id is not part of tableColumns");
|
|
121
|
+
});
|
|
122
|
+
});
|
|
@@ -15,13 +15,21 @@ import {
|
|
|
15
15
|
} from "../../../../Server/Utils/AnalyticsDatabase/ClusterConfig";
|
|
16
16
|
import UpdateBy from "../../../../Server/Types/AnalyticsDatabase/UpdateBy";
|
|
17
17
|
import "../../TestingUtils/Init";
|
|
18
|
-
import AnalyticsBaseModel
|
|
18
|
+
import AnalyticsBaseModel, {
|
|
19
|
+
AnalyticsBaseModelType,
|
|
20
|
+
} from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
|
|
19
21
|
import Route from "../../../../Types/API/Route";
|
|
20
22
|
import AnalyticsTableEngine from "../../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
21
23
|
import AnalyticsTableColumn, {
|
|
22
24
|
SkipIndexType,
|
|
23
25
|
} from "../../../../Types/AnalyticsDatabase/TableColumn";
|
|
24
26
|
import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
|
|
27
|
+
import MetricItemAggMV1m from "../../../../Models/AnalyticsModels/MetricItemAggMV1m";
|
|
28
|
+
import MetricItemAggMV1mByHostV2 from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByHostV2";
|
|
29
|
+
import MetricItemAggMV1mByService from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByService";
|
|
30
|
+
import MetricItemAggMV1mByK8sCluster from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster";
|
|
31
|
+
import MetricItemAggMV1mByContainer from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByContainer";
|
|
32
|
+
import MetricBaselineHourly from "../../../../Models/AnalyticsModels/MetricBaselineHourly";
|
|
25
33
|
|
|
26
34
|
const CLUSTER_ENV_KEY: string = "CLICKHOUSE_CLUSTER_NAME";
|
|
27
35
|
const SHARDING_ENV_KEY: string = "CLICKHOUSE_SHARDING_KEY";
|
|
@@ -167,12 +175,38 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
|
|
|
167
175
|
required: true,
|
|
168
176
|
type: TableColumnType.ObjectID,
|
|
169
177
|
}),
|
|
178
|
+
new AnalyticsTableColumn({
|
|
179
|
+
key: "bucketTime",
|
|
180
|
+
title: "Bucket",
|
|
181
|
+
description: "Bucket",
|
|
182
|
+
required: true,
|
|
183
|
+
type: TableColumnType.Date,
|
|
184
|
+
}),
|
|
185
|
+
new AnalyticsTableColumn({
|
|
186
|
+
key: "retentionDate",
|
|
187
|
+
title: "Retention",
|
|
188
|
+
description: "Retention",
|
|
189
|
+
required: true,
|
|
190
|
+
type: TableColumnType.Date,
|
|
191
|
+
simpleAggregateFunction: "max",
|
|
192
|
+
}),
|
|
193
|
+
new AnalyticsTableColumn({
|
|
194
|
+
key: "valueState",
|
|
195
|
+
title: "Value",
|
|
196
|
+
description: "Value",
|
|
197
|
+
required: true,
|
|
198
|
+
type: TableColumnType.AggregateFunction,
|
|
199
|
+
aggregateFunctionDefinition: "sum, Float64",
|
|
200
|
+
}),
|
|
170
201
|
],
|
|
171
202
|
crudApiPath: new Route("agg"),
|
|
172
|
-
primaryKeys: ["projectId"],
|
|
173
|
-
sortKeys: ["projectId"],
|
|
203
|
+
primaryKeys: ["projectId", "bucketTime"],
|
|
204
|
+
sortKeys: ["projectId", "bucketTime"],
|
|
174
205
|
partitionKey: "toYYYYMM(bucketTime)",
|
|
175
206
|
tableEngine: AnalyticsTableEngine.AggregatingMergeTree,
|
|
207
|
+
ttlExpression: "retentionDate DELETE",
|
|
208
|
+
includeBaseColumns: false,
|
|
209
|
+
defaultSortColumn: "bucketTime",
|
|
176
210
|
});
|
|
177
211
|
}
|
|
178
212
|
}
|
|
@@ -215,6 +249,133 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
|
|
|
215
249
|
expect(fullText(stmt)).toContain("MetricItemAggMV1mLocal");
|
|
216
250
|
});
|
|
217
251
|
|
|
252
|
+
test("AggregatingMergeTree CREATE uses real measures without the dimension bypass", () => {
|
|
253
|
+
const stmt: Statement = aggregatingGen.toTableCreateStatement();
|
|
254
|
+
expect(stmt.query).toContain("SimpleAggregateFunction(max, DateTime)");
|
|
255
|
+
expect(stmt.query).toContain("AggregateFunction(sum, Float64)");
|
|
256
|
+
expect(fullText(stmt)).not.toContain('"_id"');
|
|
257
|
+
expect(fullText(stmt)).not.toContain('"createdAt"');
|
|
258
|
+
expect(stmt.query).not.toContain("allow_dimensions_outside_sorting_key");
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
test("all metric aggregate models satisfy the strict off-key measure invariant", () => {
|
|
262
|
+
const models: Array<AnalyticsBaseModel> = [
|
|
263
|
+
new MetricItemAggMV1m(),
|
|
264
|
+
new MetricItemAggMV1mByHostV2(),
|
|
265
|
+
new MetricItemAggMV1mByService(),
|
|
266
|
+
new MetricItemAggMV1mByK8sCluster(),
|
|
267
|
+
new MetricItemAggMV1mByContainer(),
|
|
268
|
+
new MetricBaselineHourly(),
|
|
269
|
+
];
|
|
270
|
+
|
|
271
|
+
for (const model of models) {
|
|
272
|
+
expect(model.tableEngine).toBe(
|
|
273
|
+
AnalyticsTableEngine.AggregatingMergeTree,
|
|
274
|
+
);
|
|
275
|
+
expect(model.getTableColumn("_id")).toBeNull();
|
|
276
|
+
expect(model.getTableColumn("createdAt")).toBeNull();
|
|
277
|
+
expect(model.getTableColumn("updatedAt")).toBeNull();
|
|
278
|
+
expect(model.defaultSortColumn).toBeTruthy();
|
|
279
|
+
expect(model.sortKeys).toContain(model.defaultSortColumn);
|
|
280
|
+
expect(model.tableSettings).not.toContain(
|
|
281
|
+
"allow_dimensions_outside_sorting_key",
|
|
282
|
+
);
|
|
283
|
+
|
|
284
|
+
for (const column of model.tableColumns) {
|
|
285
|
+
if (model.sortKeys.includes(column.key)) {
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
expect(
|
|
289
|
+
column.type === TableColumnType.AggregateFunction ||
|
|
290
|
+
Boolean(column.simpleAggregateFunction),
|
|
291
|
+
).toBe(true);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
const retentionDate: AnalyticsTableColumn | null =
|
|
295
|
+
model.getTableColumn("retentionDate");
|
|
296
|
+
if (retentionDate) {
|
|
297
|
+
expect(retentionDate.simpleAggregateFunction).toBe("max");
|
|
298
|
+
expect(model.materializedViews[0]?.query).toContain(
|
|
299
|
+
"maxSimpleState(retentionDate) AS retentionDate",
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
test("every concrete metric aggregate model generates strict ClickHouse DDL", () => {
|
|
306
|
+
const modelTypes: Array<AnalyticsBaseModelType> = [
|
|
307
|
+
MetricItemAggMV1m,
|
|
308
|
+
MetricItemAggMV1mByHostV2,
|
|
309
|
+
MetricItemAggMV1mByService,
|
|
310
|
+
MetricItemAggMV1mByK8sCluster,
|
|
311
|
+
MetricItemAggMV1mByContainer,
|
|
312
|
+
MetricBaselineHourly,
|
|
313
|
+
];
|
|
314
|
+
|
|
315
|
+
for (const modelType of modelTypes) {
|
|
316
|
+
const model: AnalyticsBaseModel = new modelType();
|
|
317
|
+
const generator: StatementGenerator<AnalyticsBaseModel> =
|
|
318
|
+
new StatementGenerator<AnalyticsBaseModel>({
|
|
319
|
+
modelType,
|
|
320
|
+
database: ClickhouseAppInstance,
|
|
321
|
+
});
|
|
322
|
+
const statement: Statement = generator.toTableCreateStatement();
|
|
323
|
+
const ddl: string = fullText(statement);
|
|
324
|
+
|
|
325
|
+
expect(statement.query).toContain(
|
|
326
|
+
"ENGINE = ReplicatedAggregatingMergeTree",
|
|
327
|
+
);
|
|
328
|
+
expect(ddl).toContain(`${model.tableName}Local`);
|
|
329
|
+
expect(ddl).not.toContain('"_id"');
|
|
330
|
+
expect(ddl).not.toContain('"createdAt"');
|
|
331
|
+
expect(ddl).not.toContain('"updatedAt"');
|
|
332
|
+
expect(statement.query).not.toContain(
|
|
333
|
+
"allow_dimensions_outside_sorting_key",
|
|
334
|
+
);
|
|
335
|
+
|
|
336
|
+
if (model.getTableColumn("retentionDate")) {
|
|
337
|
+
expect(statement.query).toContain(
|
|
338
|
+
"retentionDate SimpleAggregateFunction(max, DateTime)",
|
|
339
|
+
);
|
|
340
|
+
expect(statement.query).toContain("TTL retentionDate DELETE");
|
|
341
|
+
} else {
|
|
342
|
+
expect(statement.query).not.toContain("retentionDate");
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
test("retention materialized views emit the state type their targets declare", () => {
|
|
348
|
+
const models: Array<AnalyticsBaseModel> = [
|
|
349
|
+
new MetricItemAggMV1m(),
|
|
350
|
+
new MetricItemAggMV1mByHostV2(),
|
|
351
|
+
new MetricItemAggMV1mByService(),
|
|
352
|
+
new MetricItemAggMV1mByK8sCluster(),
|
|
353
|
+
new MetricItemAggMV1mByContainer(),
|
|
354
|
+
];
|
|
355
|
+
|
|
356
|
+
for (const model of models) {
|
|
357
|
+
const query: string = model.materializedViews[0]?.query || "";
|
|
358
|
+
expect(query).toContain(
|
|
359
|
+
"maxSimpleState(retentionDate) AS retentionDate",
|
|
360
|
+
);
|
|
361
|
+
expect(query).not.toContain("max(retentionDate) AS retentionDate");
|
|
362
|
+
expect(
|
|
363
|
+
model.getTableColumn("retentionDate")?.simpleAggregateFunction,
|
|
364
|
+
).toBe("max");
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const baseline: MetricBaselineHourly = new MetricBaselineHourly();
|
|
368
|
+
expect(baseline.getTableColumn("retentionDate")).toBeNull();
|
|
369
|
+
expect(baseline.materializedViews[0]?.query).not.toContain(
|
|
370
|
+
"retentionDate",
|
|
371
|
+
);
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
test("plain MergeTree CREATE does NOT add the aggregating-only setting", () => {
|
|
375
|
+
const stmt: Statement = spanGen.toTableCreateStatement();
|
|
376
|
+
expect(stmt.query).not.toContain("allow_dimensions_outside_sorting_key");
|
|
377
|
+
});
|
|
378
|
+
|
|
218
379
|
test("Distributed wrapper uses the model sharding key and local table", () => {
|
|
219
380
|
const q: string = spanGen.toDistributedTableCreateStatement().query;
|
|
220
381
|
expect(q).toContain("ON CLUSTER 'oneuptime'");
|