@oneuptime/common 11.6.0 → 11.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts +41 -24
  2. package/Models/AnalyticsModels/MetricBaselineHourly.ts +2 -0
  3. package/Models/AnalyticsModels/MetricItemAggMV1m.ts +5 -2
  4. package/Models/AnalyticsModels/MetricItemAggMV1mByContainer.ts +5 -2
  5. package/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.ts +5 -2
  6. package/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.ts +5 -2
  7. package/Models/AnalyticsModels/MetricItemAggMV1mByService.ts +5 -2
  8. package/Server/Services/AnalyticsDatabaseService.ts +13 -1
  9. package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +41 -4
  10. package/Tests/Models/AnalyticsModels/AnalyticsBaseModel.test.ts +122 -0
  11. package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts +164 -3
  12. package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts +99 -0
  13. package/Tests/Types/HashCode.test.ts +41 -0
  14. package/Tests/Types/Log/LogQueryParser.test.ts +186 -0
  15. package/Tests/Utils/Dashboard/VariableInterpolation.test.ts +197 -0
  16. package/Tests/Utils/ValueFormatter.test.ts +305 -0
  17. package/Types/AnalyticsDatabase/TableColumn.ts +23 -0
  18. package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js +25 -20
  19. package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js.map +1 -1
  20. package/build/dist/Models/AnalyticsModels/MetricBaselineHourly.js +2 -0
  21. package/build/dist/Models/AnalyticsModels/MetricBaselineHourly.js.map +1 -1
  22. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1m.js +5 -2
  23. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1m.js.map +1 -1
  24. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByContainer.js +5 -2
  25. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByContainer.js.map +1 -1
  26. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.js +5 -2
  27. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.js.map +1 -1
  28. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.js +5 -2
  29. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster.js.map +1 -1
  30. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByService.js +5 -2
  31. package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByService.js.map +1 -1
  32. package/build/dist/Server/Services/AnalyticsDatabaseService.js +9 -1
  33. package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
  34. package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +29 -4
  35. package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
  36. package/build/dist/Types/AnalyticsDatabase/TableColumn.js +7 -0
  37. package/build/dist/Types/AnalyticsDatabase/TableColumn.js.map +1 -1
  38. package/package.json +1 -1
@@ -77,6 +77,15 @@ export default class AnalyticsBaseModel extends CommonModel {
77
77
  materializedViews?: Array<MaterializedView> | undefined;
78
78
  enableMCP?: boolean | undefined;
79
79
  ttlExpression?: string | undefined; // e.g. "retentionDate DELETE"
80
+ /*
81
+ * Generic event/entity tables carry `_id` and `createdAt`. Derived
82
+ * AggregatingMergeTree targets do not represent independently-addressable
83
+ * rows: their identity is exactly their aggregation key, and an arbitrary
84
+ * creation timestamp has no stable meaning after merges. Those models set
85
+ * this false so ClickHouse can enforce that every non-key column is an
86
+ * aggregate measure. Defaults true for backward compatibility.
87
+ */
88
+ includeBaseColumns?: boolean | undefined;
80
89
  /*
81
90
  * Column that `findBy` falls back to when the caller doesn't
82
91
  * specify a `sort`. Defaults to `createdAt` (matching the legacy
@@ -100,31 +109,39 @@ export default class AnalyticsBaseModel extends CommonModel {
100
109
  this.tableEngine = data.tableEngine;
101
110
  }
102
111
 
103
- columns.push(
104
- new AnalyticsTableColumn({
105
- key: "_id",
106
- title: "ID",
107
- description: "ID of this object",
108
- required: true,
109
- type: TableColumnType.ObjectID,
110
- /*
111
- * Ids are UUIDv7 (time-ordered — see ObjectID.generateTimeOrdered),
112
- * so consecutive rows share their 48-bit timestamp prefix and ZSTD
113
- * compresses them well; random v4 ids were incompressible.
114
- */
115
- codec: { codec: "ZSTD", level: 1 },
116
- }),
117
- );
112
+ if (data.includeBaseColumns !== false) {
113
+ columns.push(
114
+ new AnalyticsTableColumn({
115
+ key: "_id",
116
+ title: "ID",
117
+ description: "ID of this object",
118
+ required: true,
119
+ type: TableColumnType.ObjectID,
120
+ /*
121
+ * Ids are UUIDv7 (time-ordered see ObjectID.generateTimeOrdered),
122
+ * so consecutive rows share their 48-bit timestamp prefix and ZSTD
123
+ * compresses them well; random v4 ids were incompressible.
124
+ */
125
+ codec: { codec: "ZSTD", level: 1 },
126
+ }),
127
+ );
118
128
 
119
- columns.push(
120
- new AnalyticsTableColumn({
121
- key: "createdAt",
122
- title: "Created",
123
- description: "Date and Time when the object was created.",
124
- required: true,
125
- type: TableColumnType.Date,
126
- }),
127
- );
129
+ columns.push(
130
+ new AnalyticsTableColumn({
131
+ key: "createdAt",
132
+ title: "Created",
133
+ description: "Date and Time when the object was created.",
134
+ required: true,
135
+ type: TableColumnType.Date,
136
+ }),
137
+ );
138
+ }
139
+
140
+ if (data.includeBaseColumns === false && !data.defaultSortColumn) {
141
+ throw new BadDataException(
142
+ "defaultSortColumn is required when includeBaseColumns is false because createdAt is not part of the table schema",
143
+ );
144
+ }
128
145
 
129
146
  if (!data.primaryKeys || data.primaryKeys.length === 0) {
130
147
  throw new BadDataException("Primary keys are required");
@@ -234,6 +234,8 @@ GROUP BY projectId, name, primaryEntityId, day, hourOfWeek`,
234
234
  tableSettings:
235
235
  "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
236
236
  ttlExpression: "day + INTERVAL 90 DAY",
237
+ includeBaseColumns: false,
238
+ defaultSortColumn: "day",
237
239
  });
238
240
  }
239
241
  }
@@ -110,9 +110,10 @@ export default class MetricItemAggMV1m extends AnalyticsBaseModel {
110
110
  key: "retentionDate",
111
111
  title: "Retention Date",
112
112
  description:
113
- "Date after which this row is eligible for TTL deletion. Computed by the MV as max(retentionDate) per bucket inherits from the source MetricItemV3 rows.",
113
+ "Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
114
114
  required: true,
115
115
  type: TableColumnType.Date,
116
+ simpleAggregateFunction: "max",
116
117
  });
117
118
 
118
119
  super({
@@ -156,7 +157,7 @@ SELECT
156
157
  countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
157
158
  minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
158
159
  maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
159
- max(retentionDate) AS retentionDate
160
+ maxSimpleState(retentionDate) AS retentionDate
160
161
  FROM MetricItemV3
161
162
  GROUP BY projectId, name, primaryEntityId, bucketTime`,
162
163
  },
@@ -172,6 +173,8 @@ GROUP BY projectId, name, primaryEntityId, bucketTime`,
172
173
  tableSettings:
173
174
  "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
174
175
  ttlExpression: "retentionDate DELETE",
176
+ includeBaseColumns: false,
177
+ defaultSortColumn: "bucketTime",
175
178
  });
176
179
  }
177
180
  }
@@ -111,9 +111,10 @@ export default class MetricItemAggMV1mByContainer extends AnalyticsBaseModel {
111
111
  key: "retentionDate",
112
112
  title: "Retention Date",
113
113
  description:
114
- "Date after which this row is eligible for TTL deletion. Computed by the MV as max(retentionDate) per bucket inherits from the source MetricItemV3 rows.",
114
+ "Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
115
115
  required: true,
116
116
  type: TableColumnType.Date,
117
+ simpleAggregateFunction: "max",
117
118
  });
118
119
 
119
120
  super({
@@ -159,7 +160,7 @@ SELECT
159
160
  countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
160
161
  minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
161
162
  maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
162
- max(retentionDate) AS retentionDate
163
+ maxSimpleState(retentionDate) AS retentionDate
163
164
  FROM MetricItemV3
164
165
  WHERE containerEntityKey != ''
165
166
  GROUP BY projectId, name, containerEntityKey, bucketTime`,
@@ -173,6 +174,8 @@ GROUP BY projectId, name, containerEntityKey, bucketTime`,
173
174
  tableSettings:
174
175
  "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
175
176
  ttlExpression: "retentionDate DELETE",
177
+ includeBaseColumns: false,
178
+ defaultSortColumn: "bucketTime",
176
179
  });
177
180
  }
178
181
  }
@@ -114,9 +114,10 @@ export default class MetricItemAggMV1mByHostV2 extends AnalyticsBaseModel {
114
114
  key: "retentionDate",
115
115
  title: "Retention Date",
116
116
  description:
117
- "Date after which this row is eligible for TTL deletion. Computed by the MV as max(retentionDate) per bucket inherits from the source MetricItemV3 rows.",
117
+ "Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
118
118
  required: true,
119
119
  type: TableColumnType.Date,
120
+ simpleAggregateFunction: "max",
120
121
  });
121
122
 
122
123
  super({
@@ -162,7 +163,7 @@ SELECT
162
163
  countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
163
164
  minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
164
165
  maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
165
- max(retentionDate) AS retentionDate
166
+ maxSimpleState(retentionDate) AS retentionDate
166
167
  FROM MetricItemV3
167
168
  WHERE hostEntityKey != ''
168
169
  GROUP BY projectId, name, hostEntityKey, bucketTime`,
@@ -176,6 +177,8 @@ GROUP BY projectId, name, hostEntityKey, bucketTime`,
176
177
  tableSettings:
177
178
  "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
178
179
  ttlExpression: "retentionDate DELETE",
180
+ includeBaseColumns: false,
181
+ defaultSortColumn: "bucketTime",
179
182
  });
180
183
  }
181
184
  }
@@ -110,9 +110,10 @@ export default class MetricItemAggMV1mByK8sCluster extends AnalyticsBaseModel {
110
110
  key: "retentionDate",
111
111
  title: "Retention Date",
112
112
  description:
113
- "Date after which this row is eligible for TTL deletion. Computed by the MV as max(retentionDate) per bucket inherits from the source MetricItemV3 rows.",
113
+ "Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
114
114
  required: true,
115
115
  type: TableColumnType.Date,
116
+ simpleAggregateFunction: "max",
116
117
  });
117
118
 
118
119
  super({
@@ -160,7 +161,7 @@ SELECT
160
161
  countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
161
162
  minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
162
163
  maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
163
- max(retentionDate) AS retentionDate
164
+ maxSimpleState(retentionDate) AS retentionDate
164
165
  FROM MetricItemV3
165
166
  WHERE k8sClusterEntityKey != ''
166
167
  GROUP BY projectId, name, k8sClusterEntityKey, bucketTime`,
@@ -174,6 +175,8 @@ GROUP BY projectId, name, k8sClusterEntityKey, bucketTime`,
174
175
  tableSettings:
175
176
  "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
176
177
  ttlExpression: "retentionDate DELETE",
178
+ includeBaseColumns: false,
179
+ defaultSortColumn: "bucketTime",
177
180
  });
178
181
  }
179
182
  }
@@ -112,9 +112,10 @@ export default class MetricItemAggMV1mByService extends AnalyticsBaseModel {
112
112
  key: "retentionDate",
113
113
  title: "Retention Date",
114
114
  description:
115
- "Date after which this row is eligible for TTL deletion. Computed by the MV as max(retentionDate) per bucket inherits from the source MetricItemV3 rows.",
115
+ "Latest date after which this aggregate bucket is eligible for TTL deletion. Stored as SimpleAggregateFunction(max, DateTime) so partial rows keep the maximum expiry when AggregatingMergeTree merges them.",
116
116
  required: true,
117
117
  type: TableColumnType.Date,
118
+ simpleAggregateFunction: "max",
118
119
  });
119
120
 
120
121
  super({
@@ -160,7 +161,7 @@ SELECT
160
161
  countState(toFloat64(coalesce(value, sum, 0))) AS valueCountState,
161
162
  minState(toFloat64(coalesce(value, sum, 0))) AS valueMinState,
162
163
  maxState(toFloat64(coalesce(value, sum, 0))) AS valueMaxState,
163
- max(retentionDate) AS retentionDate
164
+ maxSimpleState(retentionDate) AS retentionDate
164
165
  FROM MetricItemV3
165
166
  WHERE serviceEntityKey != ''
166
167
  GROUP BY projectId, name, serviceEntityKey, bucketTime`,
@@ -174,6 +175,8 @@ GROUP BY projectId, name, serviceEntityKey, bucketTime`,
174
175
  tableSettings:
175
176
  "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
176
177
  ttlExpression: "retentionDate DELETE",
178
+ includeBaseColumns: false,
179
+ defaultSortColumn: "bucketTime",
177
180
  });
178
181
  }
179
182
  }
@@ -983,7 +983,19 @@ export default class AnalyticsDatabaseService<
983
983
  onBeforeFind.select = {} as any;
984
984
  }
985
985
 
986
- if (!(onBeforeFind.select as any)["_id"]) {
986
+ /*
987
+ * Derived aggregate targets deliberately omit AnalyticsBaseModel's
988
+ * synthetic `_id`: the aggregation key is the row identity. Only force
989
+ * `_id` into generic reads when the model actually declares it.
990
+ */
991
+ if (
992
+ this.model.tableColumns.some(
993
+ (column: AnalyticsTableColumn): boolean => {
994
+ return column.key === "_id";
995
+ },
996
+ ) &&
997
+ !(onBeforeFind.select as any)["_id"]
998
+ ) {
987
999
  (onBeforeFind.select as any)["_id"] = true;
988
1000
  }
989
1001
 
@@ -69,6 +69,8 @@ export interface EntityScopeQueryValue {
69
69
  attributeValue: string;
70
70
  }
71
71
 
72
+ const SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;
73
+
72
74
  export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
73
75
  public model!: TBaseModel;
74
76
  public modelType!: { new (): TBaseModel };
@@ -1361,6 +1363,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
1361
1363
  * column. Scalar types ignore the rest of the column.
1362
1364
  */
1363
1365
  public toColumnType(column: AnalyticsTableColumn): Statement {
1366
+ const simpleAggregateFunction: string | undefined =
1367
+ column.simpleAggregateFunction;
1368
+
1369
+ if (simpleAggregateFunction !== undefined) {
1370
+ if (column.type === TableColumnType.AggregateFunction) {
1371
+ throw new BadDataException(
1372
+ `Column ${column.key} cannot declare both AggregateFunction and SimpleAggregateFunction.`,
1373
+ );
1374
+ }
1375
+
1376
+ /*
1377
+ * The function name is emitted as raw ClickHouse DDL, so accept only a
1378
+ * plain function identifier. Besides catching accidental whitespace /
1379
+ * empty definitions, this prevents a model field from injecting another
1380
+ * type or table clause into generated schema SQL.
1381
+ */
1382
+ if (
1383
+ simpleAggregateFunction.trim() !== simpleAggregateFunction ||
1384
+ !SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX.test(simpleAggregateFunction)
1385
+ ) {
1386
+ throw new BadDataException(
1387
+ `Column ${column.key} has invalid simpleAggregateFunction "${simpleAggregateFunction}".`,
1388
+ );
1389
+ }
1390
+ }
1391
+
1364
1392
  if (column.type === TableColumnType.AggregateFunction) {
1365
1393
  const def: string | undefined = column.aggregateFunctionDefinition;
1366
1394
  if (!def) {
@@ -1371,7 +1399,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
1371
1399
  return SQL`AggregateFunction(`.append(def).append(SQL`)`);
1372
1400
  }
1373
1401
 
1374
- const statement: Statement | undefined = {
1402
+ const scalarStatement: Statement | undefined = {
1375
1403
  [TableColumnType.Text]: SQL`String`,
1376
1404
  [TableColumnType.ObjectID]: SQL`String`,
1377
1405
  [TableColumnType.Boolean]: SQL`Bool`,
@@ -1394,13 +1422,21 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
1394
1422
  [TableColumnType.UInt64]: SQL`UInt64`,
1395
1423
  }[column.type];
1396
1424
 
1397
- if (!statement) {
1425
+ if (!scalarStatement) {
1398
1426
  throw new BadDataException(
1399
1427
  `Unknown column type: ${column.type}. Please add support for this column type.`,
1400
1428
  );
1401
1429
  }
1402
1430
 
1403
- return statement;
1431
+ if (simpleAggregateFunction) {
1432
+ return SQL`SimpleAggregateFunction(`
1433
+ .append(simpleAggregateFunction)
1434
+ .append(SQL`, `)
1435
+ .append(scalarStatement)
1436
+ .append(SQL`)`);
1437
+ }
1438
+
1439
+ return scalarStatement;
1404
1440
  }
1405
1441
 
1406
1442
  /**
@@ -1413,7 +1449,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
1413
1449
  */
1414
1450
  public toFullColumnType(column: AnalyticsTableColumn): Statement {
1415
1451
  const isAggregateFunction: boolean =
1416
- column.type === TableColumnType.AggregateFunction;
1452
+ column.type === TableColumnType.AggregateFunction ||
1453
+ Boolean(column.simpleAggregateFunction);
1417
1454
 
1418
1455
  let typeStatement: Statement = this.toColumnType(column);
1419
1456
 
@@ -0,0 +1,122 @@
1
+ import { describe, expect, test } from "@jest/globals";
2
+ import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
3
+ import AnalyticsTableEngine from "../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
4
+ import AnalyticsTableColumn from "../../../Types/AnalyticsDatabase/TableColumn";
5
+ import TableColumnType from "../../../Types/AnalyticsDatabase/TableColumnType";
6
+
7
+ type ModelOptions = ConstructorParameters<typeof AnalyticsBaseModel>[0];
8
+
9
+ function projectIdColumn(): AnalyticsTableColumn {
10
+ return new AnalyticsTableColumn({
11
+ key: "projectId",
12
+ title: "Project ID",
13
+ description: "Tenant",
14
+ required: true,
15
+ type: TableColumnType.Text,
16
+ });
17
+ }
18
+
19
+ function modelOptions(overrides: Partial<ModelOptions> = {}): ModelOptions {
20
+ return {
21
+ tableName: "TestAnalyticsTable",
22
+ singularName: "Test row",
23
+ pluralName: "Test rows",
24
+ tableColumns: [projectIdColumn()],
25
+ primaryKeys: ["projectId"],
26
+ sortKeys: ["projectId"],
27
+ partitionKey: "projectId",
28
+ tableEngine: AnalyticsTableEngine.MergeTree,
29
+ ...overrides,
30
+ };
31
+ }
32
+
33
+ describe("AnalyticsBaseModel base-column policy", () => {
34
+ test("adds _id and createdAt by default for regular analytics models", () => {
35
+ const suppliedColumns: Array<AnalyticsTableColumn> = [projectIdColumn()];
36
+ const model: AnalyticsBaseModel = new AnalyticsBaseModel(
37
+ modelOptions({ tableColumns: suppliedColumns }),
38
+ );
39
+
40
+ expect(
41
+ model.tableColumns.map((column: AnalyticsTableColumn) => {
42
+ return column.key;
43
+ }),
44
+ ).toEqual(["projectId", "_id", "createdAt"]);
45
+ expect(model.getTableColumn("_id")?.type).toBe(TableColumnType.ObjectID);
46
+ expect(model.getTableColumn("_id")?.codec).toEqual({
47
+ codec: "ZSTD",
48
+ level: 1,
49
+ });
50
+ expect(model.getTableColumn("createdAt")?.type).toBe(TableColumnType.Date);
51
+
52
+ // Construction copies the caller's array instead of mutating it.
53
+ expect(
54
+ suppliedColumns.map((column: AnalyticsTableColumn) => {
55
+ return column.key;
56
+ }),
57
+ ).toEqual(["projectId"]);
58
+ });
59
+
60
+ test("omits synthetic columns when includeBaseColumns is false", () => {
61
+ const model: AnalyticsBaseModel = new AnalyticsBaseModel(
62
+ modelOptions({
63
+ includeBaseColumns: false,
64
+ defaultSortColumn: "projectId",
65
+ }),
66
+ );
67
+
68
+ expect(
69
+ model.tableColumns.map((column: AnalyticsTableColumn) => {
70
+ return column.key;
71
+ }),
72
+ ).toEqual(["projectId"]);
73
+ expect(model.getTableColumn("_id")).toBeNull();
74
+ expect(model.getTableColumn("createdAt")).toBeNull();
75
+ expect(model.defaultSortColumn).toBe("projectId");
76
+ });
77
+
78
+ test("requires a real default sort column when base columns are omitted", () => {
79
+ expect(() => {
80
+ return new AnalyticsBaseModel(
81
+ modelOptions({
82
+ includeBaseColumns: false,
83
+ }),
84
+ );
85
+ }).toThrow(
86
+ "defaultSortColumn is required when includeBaseColumns is false",
87
+ );
88
+ });
89
+
90
+ test("rejects createdAt as a strict model's default sort because it is absent", () => {
91
+ expect(() => {
92
+ return new AnalyticsBaseModel(
93
+ modelOptions({
94
+ includeBaseColumns: false,
95
+ defaultSortColumn: "createdAt",
96
+ }),
97
+ );
98
+ }).toThrow("defaultSortColumn createdAt is not part of tableColumns");
99
+ });
100
+
101
+ test("still validates primary and sort keys against the strict column set", () => {
102
+ expect(() => {
103
+ return new AnalyticsBaseModel(
104
+ modelOptions({
105
+ includeBaseColumns: false,
106
+ defaultSortColumn: "projectId",
107
+ sortKeys: ["projectId", "createdAt"],
108
+ }),
109
+ );
110
+ }).toThrow("Sort key createdAt is not part of tableColumns");
111
+
112
+ expect(() => {
113
+ return new AnalyticsBaseModel(
114
+ modelOptions({
115
+ includeBaseColumns: false,
116
+ defaultSortColumn: "projectId",
117
+ primaryKeys: ["_id"],
118
+ }),
119
+ );
120
+ }).toThrow("Primary key _id is not part of tableColumns");
121
+ });
122
+ });
@@ -15,13 +15,21 @@ import {
15
15
  } from "../../../../Server/Utils/AnalyticsDatabase/ClusterConfig";
16
16
  import UpdateBy from "../../../../Server/Types/AnalyticsDatabase/UpdateBy";
17
17
  import "../../TestingUtils/Init";
18
- import AnalyticsBaseModel from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
18
+ import AnalyticsBaseModel, {
19
+ AnalyticsBaseModelType,
20
+ } from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
19
21
  import Route from "../../../../Types/API/Route";
20
22
  import AnalyticsTableEngine from "../../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
21
23
  import AnalyticsTableColumn, {
22
24
  SkipIndexType,
23
25
  } from "../../../../Types/AnalyticsDatabase/TableColumn";
24
26
  import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
27
+ import MetricItemAggMV1m from "../../../../Models/AnalyticsModels/MetricItemAggMV1m";
28
+ import MetricItemAggMV1mByHostV2 from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByHostV2";
29
+ import MetricItemAggMV1mByService from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByService";
30
+ import MetricItemAggMV1mByK8sCluster from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster";
31
+ import MetricItemAggMV1mByContainer from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByContainer";
32
+ import MetricBaselineHourly from "../../../../Models/AnalyticsModels/MetricBaselineHourly";
25
33
 
26
34
  const CLUSTER_ENV_KEY: string = "CLICKHOUSE_CLUSTER_NAME";
27
35
  const SHARDING_ENV_KEY: string = "CLICKHOUSE_SHARDING_KEY";
@@ -167,12 +175,38 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
167
175
  required: true,
168
176
  type: TableColumnType.ObjectID,
169
177
  }),
178
+ new AnalyticsTableColumn({
179
+ key: "bucketTime",
180
+ title: "Bucket",
181
+ description: "Bucket",
182
+ required: true,
183
+ type: TableColumnType.Date,
184
+ }),
185
+ new AnalyticsTableColumn({
186
+ key: "retentionDate",
187
+ title: "Retention",
188
+ description: "Retention",
189
+ required: true,
190
+ type: TableColumnType.Date,
191
+ simpleAggregateFunction: "max",
192
+ }),
193
+ new AnalyticsTableColumn({
194
+ key: "valueState",
195
+ title: "Value",
196
+ description: "Value",
197
+ required: true,
198
+ type: TableColumnType.AggregateFunction,
199
+ aggregateFunctionDefinition: "sum, Float64",
200
+ }),
170
201
  ],
171
202
  crudApiPath: new Route("agg"),
172
- primaryKeys: ["projectId"],
173
- sortKeys: ["projectId"],
203
+ primaryKeys: ["projectId", "bucketTime"],
204
+ sortKeys: ["projectId", "bucketTime"],
174
205
  partitionKey: "toYYYYMM(bucketTime)",
175
206
  tableEngine: AnalyticsTableEngine.AggregatingMergeTree,
207
+ ttlExpression: "retentionDate DELETE",
208
+ includeBaseColumns: false,
209
+ defaultSortColumn: "bucketTime",
176
210
  });
177
211
  }
178
212
  }
@@ -215,6 +249,133 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
215
249
  expect(fullText(stmt)).toContain("MetricItemAggMV1mLocal");
216
250
  });
217
251
 
252
+ test("AggregatingMergeTree CREATE uses real measures without the dimension bypass", () => {
253
+ const stmt: Statement = aggregatingGen.toTableCreateStatement();
254
+ expect(stmt.query).toContain("SimpleAggregateFunction(max, DateTime)");
255
+ expect(stmt.query).toContain("AggregateFunction(sum, Float64)");
256
+ expect(fullText(stmt)).not.toContain('"_id"');
257
+ expect(fullText(stmt)).not.toContain('"createdAt"');
258
+ expect(stmt.query).not.toContain("allow_dimensions_outside_sorting_key");
259
+ });
260
+
261
+ test("all metric aggregate models satisfy the strict off-key measure invariant", () => {
262
+ const models: Array<AnalyticsBaseModel> = [
263
+ new MetricItemAggMV1m(),
264
+ new MetricItemAggMV1mByHostV2(),
265
+ new MetricItemAggMV1mByService(),
266
+ new MetricItemAggMV1mByK8sCluster(),
267
+ new MetricItemAggMV1mByContainer(),
268
+ new MetricBaselineHourly(),
269
+ ];
270
+
271
+ for (const model of models) {
272
+ expect(model.tableEngine).toBe(
273
+ AnalyticsTableEngine.AggregatingMergeTree,
274
+ );
275
+ expect(model.getTableColumn("_id")).toBeNull();
276
+ expect(model.getTableColumn("createdAt")).toBeNull();
277
+ expect(model.getTableColumn("updatedAt")).toBeNull();
278
+ expect(model.defaultSortColumn).toBeTruthy();
279
+ expect(model.sortKeys).toContain(model.defaultSortColumn);
280
+ expect(model.tableSettings).not.toContain(
281
+ "allow_dimensions_outside_sorting_key",
282
+ );
283
+
284
+ for (const column of model.tableColumns) {
285
+ if (model.sortKeys.includes(column.key)) {
286
+ continue;
287
+ }
288
+ expect(
289
+ column.type === TableColumnType.AggregateFunction ||
290
+ Boolean(column.simpleAggregateFunction),
291
+ ).toBe(true);
292
+ }
293
+
294
+ const retentionDate: AnalyticsTableColumn | null =
295
+ model.getTableColumn("retentionDate");
296
+ if (retentionDate) {
297
+ expect(retentionDate.simpleAggregateFunction).toBe("max");
298
+ expect(model.materializedViews[0]?.query).toContain(
299
+ "maxSimpleState(retentionDate) AS retentionDate",
300
+ );
301
+ }
302
+ }
303
+ });
304
+
305
+ test("every concrete metric aggregate model generates strict ClickHouse DDL", () => {
306
+ const modelTypes: Array<AnalyticsBaseModelType> = [
307
+ MetricItemAggMV1m,
308
+ MetricItemAggMV1mByHostV2,
309
+ MetricItemAggMV1mByService,
310
+ MetricItemAggMV1mByK8sCluster,
311
+ MetricItemAggMV1mByContainer,
312
+ MetricBaselineHourly,
313
+ ];
314
+
315
+ for (const modelType of modelTypes) {
316
+ const model: AnalyticsBaseModel = new modelType();
317
+ const generator: StatementGenerator<AnalyticsBaseModel> =
318
+ new StatementGenerator<AnalyticsBaseModel>({
319
+ modelType,
320
+ database: ClickhouseAppInstance,
321
+ });
322
+ const statement: Statement = generator.toTableCreateStatement();
323
+ const ddl: string = fullText(statement);
324
+
325
+ expect(statement.query).toContain(
326
+ "ENGINE = ReplicatedAggregatingMergeTree",
327
+ );
328
+ expect(ddl).toContain(`${model.tableName}Local`);
329
+ expect(ddl).not.toContain('"_id"');
330
+ expect(ddl).not.toContain('"createdAt"');
331
+ expect(ddl).not.toContain('"updatedAt"');
332
+ expect(statement.query).not.toContain(
333
+ "allow_dimensions_outside_sorting_key",
334
+ );
335
+
336
+ if (model.getTableColumn("retentionDate")) {
337
+ expect(statement.query).toContain(
338
+ "retentionDate SimpleAggregateFunction(max, DateTime)",
339
+ );
340
+ expect(statement.query).toContain("TTL retentionDate DELETE");
341
+ } else {
342
+ expect(statement.query).not.toContain("retentionDate");
343
+ }
344
+ }
345
+ });
346
+
347
+ test("retention materialized views emit the state type their targets declare", () => {
348
+ const models: Array<AnalyticsBaseModel> = [
349
+ new MetricItemAggMV1m(),
350
+ new MetricItemAggMV1mByHostV2(),
351
+ new MetricItemAggMV1mByService(),
352
+ new MetricItemAggMV1mByK8sCluster(),
353
+ new MetricItemAggMV1mByContainer(),
354
+ ];
355
+
356
+ for (const model of models) {
357
+ const query: string = model.materializedViews[0]?.query || "";
358
+ expect(query).toContain(
359
+ "maxSimpleState(retentionDate) AS retentionDate",
360
+ );
361
+ expect(query).not.toContain("max(retentionDate) AS retentionDate");
362
+ expect(
363
+ model.getTableColumn("retentionDate")?.simpleAggregateFunction,
364
+ ).toBe("max");
365
+ }
366
+
367
+ const baseline: MetricBaselineHourly = new MetricBaselineHourly();
368
+ expect(baseline.getTableColumn("retentionDate")).toBeNull();
369
+ expect(baseline.materializedViews[0]?.query).not.toContain(
370
+ "retentionDate",
371
+ );
372
+ });
373
+
374
+ test("plain MergeTree CREATE does NOT add the aggregating-only setting", () => {
375
+ const stmt: Statement = spanGen.toTableCreateStatement();
376
+ expect(stmt.query).not.toContain("allow_dimensions_outside_sorting_key");
377
+ });
378
+
218
379
  test("Distributed wrapper uses the model sharding key and local table", () => {
219
380
  const q: string = spanGen.toDistributedTableCreateStatement().query;
220
381
  expect(q).toContain("ON CLUSTER 'oneuptime'");