@oneuptime/common 11.0.14 → 11.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts +23 -0
- package/Models/AnalyticsModels/AuditLog.ts +2 -0
- package/Models/AnalyticsModels/ExceptionInstance.ts +5 -0
- package/Models/AnalyticsModels/Log.ts +10 -0
- package/Models/AnalyticsModels/Metric.ts +7 -0
- package/Models/AnalyticsModels/MetricBaselineHourly.ts +5 -0
- package/Models/AnalyticsModels/MetricItemAggMV1m.ts +5 -0
- package/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.ts +2 -0
- package/Models/AnalyticsModels/MonitorLog.ts +2 -0
- package/Models/AnalyticsModels/Profile.ts +9 -0
- package/Models/AnalyticsModels/ProfileSample.ts +2 -0
- package/Models/AnalyticsModels/Span.ts +5 -0
- package/Server/EnvironmentConfig.ts +25 -0
- package/Server/Services/AnalyticsDatabaseService.ts +27 -14
- package/Server/Utils/AnalyticsDatabase/ClusterConfig.ts +177 -0
- package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +101 -21
- package/Server/Utils/CodeRepository/CodeRepository.ts +8 -0
- package/Server/Utils/Execute.ts +9 -0
- package/Server/Utils/Monitor/MonitorTemplateUtil.ts +4 -0
- package/Tests/Server/Services/AnalyticsDatabaseService.test.ts +8 -3
- package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts +296 -0
- package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts +5 -3
- package/Types/Monitor/ExternalStatusPageMonitor/ExternalStatusPageMonitorResponse.ts +5 -0
- package/Types/Monitor/ExternalStatusPageProviderType.ts +1 -0
- package/Types/Monitor/MonitorCriteriaInstance.ts +34 -14
- package/Types/Monitor/MonitorStepExternalStatusPageMonitor.ts +5 -1
- package/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.ts +14 -1
- package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js +8 -0
- package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/AuditLog.js +2 -0
- package/build/dist/Models/AnalyticsModels/AuditLog.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/ExceptionInstance.js +5 -0
- package/build/dist/Models/AnalyticsModels/ExceptionInstance.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Log.js +10 -0
- package/build/dist/Models/AnalyticsModels/Log.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Metric.js +7 -0
- package/build/dist/Models/AnalyticsModels/Metric.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricBaselineHourly.js +5 -0
- package/build/dist/Models/AnalyticsModels/MetricBaselineHourly.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1m.js +5 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1m.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.js +2 -0
- package/build/dist/Models/AnalyticsModels/MetricItemAggMV1mByHostV2.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MonitorLog.js +2 -0
- package/build/dist/Models/AnalyticsModels/MonitorLog.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Profile.js +9 -0
- package/build/dist/Models/AnalyticsModels/Profile.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/ProfileSample.js +2 -0
- package/build/dist/Models/AnalyticsModels/ProfileSample.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Span.js +5 -0
- package/build/dist/Models/AnalyticsModels/Span.js.map +1 -1
- package/build/dist/Server/EnvironmentConfig.js +21 -0
- package/build/dist/Server/EnvironmentConfig.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +22 -14
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/ClusterConfig.js +144 -0
- package/build/dist/Server/Utils/AnalyticsDatabase/ClusterConfig.js.map +1 -0
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +68 -15
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
- package/build/dist/Server/Utils/CodeRepository/CodeRepository.js +2 -6
- package/build/dist/Server/Utils/CodeRepository/CodeRepository.js.map +1 -1
- package/build/dist/Server/Utils/Execute.js +3 -1
- package/build/dist/Server/Utils/Execute.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorTemplateUtil.js +4 -0
- package/build/dist/Server/Utils/Monitor/MonitorTemplateUtil.js.map +1 -1
- package/build/dist/Types/Monitor/ExternalStatusPageProviderType.js +1 -0
- package/build/dist/Types/Monitor/ExternalStatusPageProviderType.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorCriteriaInstance.js +34 -14
- package/build/dist/Types/Monitor/MonitorCriteriaInstance.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorStepExternalStatusPageMonitor.js +3 -0
- package/build/dist/Types/Monitor/MonitorStepExternalStatusPageMonitor.js.map +1 -1
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.js +13 -1
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.js.map +1 -1
- package/package.json +1 -1
|
@@ -6,6 +6,13 @@ import Sort from "../../Types/AnalyticsDatabase/Sort";
|
|
|
6
6
|
import UpdateBy from "../../Types/AnalyticsDatabase/UpdateBy";
|
|
7
7
|
import logger from "../Logger";
|
|
8
8
|
import { SQL, Statement } from "./Statement";
|
|
9
|
+
import {
|
|
10
|
+
adaptTableSettingsForStorage,
|
|
11
|
+
getDistributedEngine,
|
|
12
|
+
getStorageEngine,
|
|
13
|
+
getStorageTableName,
|
|
14
|
+
onClusterClause,
|
|
15
|
+
} from "./ClusterConfig";
|
|
9
16
|
import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
|
|
10
17
|
import CommonModel, {
|
|
11
18
|
Record as AnalyticsRecord,
|
|
@@ -78,12 +85,24 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
78
85
|
const setStatement: Statement = this.toSetStatement(updateBy.data);
|
|
79
86
|
const whereStatement: Statement = this.toWhereStatement(updateBy.query);
|
|
80
87
|
|
|
88
|
+
/*
|
|
89
|
+
* `ALTER TABLE … UPDATE` is a mutation and cannot target a Distributed
|
|
90
|
+
* table, so in cluster mode it runs against the local storage table and is
|
|
91
|
+
* dispatched to every shard via ON CLUSTER (replicated within each shard by
|
|
92
|
+
* Keeper). onClusterClause() is appended as RAW SQL — it is not an
|
|
93
|
+
* identifier and must not become a {pN:Identifier} parameter — and is empty
|
|
94
|
+
* in single-node mode, leaving the original statement unchanged.
|
|
95
|
+
*/
|
|
81
96
|
/* eslint-disable prettier/prettier */
|
|
82
97
|
const statement: Statement = SQL`
|
|
83
|
-
ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
|
|
84
|
-
this.model.tableName
|
|
85
|
-
}
|
|
86
|
-
|
|
98
|
+
ALTER TABLE ${this.database.getDatasourceOptions().database!}.${getStorageTableName(
|
|
99
|
+
this.model.tableName,
|
|
100
|
+
)}`
|
|
101
|
+
.append(onClusterClause())
|
|
102
|
+
.append(
|
|
103
|
+
SQL`
|
|
104
|
+
UPDATE `,
|
|
105
|
+
)
|
|
87
106
|
.append(setStatement)
|
|
88
107
|
.append(
|
|
89
108
|
SQL`
|
|
@@ -1089,9 +1108,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1089
1108
|
): Promise<Statement> {
|
|
1090
1109
|
const statement: string = `ALTER TABLE ${
|
|
1091
1110
|
this.database.getDatasourceOptions().database
|
|
1092
|
-
}.${
|
|
1093
|
-
this.model.tableName
|
|
1094
|
-
} RENAME COLUMN IF EXISTS ${oldColumnName} TO ${newColumnName}`;
|
|
1111
|
+
}.${getStorageTableName(
|
|
1112
|
+
this.model.tableName,
|
|
1113
|
+
)} RENAME COLUMN IF EXISTS ${oldColumnName} TO ${newColumnName}`;
|
|
1095
1114
|
|
|
1096
1115
|
return SQL`${statement}`;
|
|
1097
1116
|
}
|
|
@@ -1309,9 +1328,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1309
1328
|
}
|
|
1310
1329
|
|
|
1311
1330
|
const statement: Statement = SQL`
|
|
1312
|
-
ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
|
|
1313
|
-
this.model.tableName
|
|
1314
|
-
} ADD COLUMN IF NOT EXISTS `.append(columnDef);
|
|
1331
|
+
ALTER TABLE ${this.database.getDatasourceOptions().database!}.${getStorageTableName(
|
|
1332
|
+
this.model.tableName,
|
|
1333
|
+
)} ADD COLUMN IF NOT EXISTS `.append(columnDef);
|
|
1315
1334
|
|
|
1316
1335
|
logger.debug(`${this.model.tableName} Add Column Statement`);
|
|
1317
1336
|
logger.debug(statement);
|
|
@@ -1341,7 +1360,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1341
1360
|
const databaseName: string = this.database.getDatasourceOptions().database!;
|
|
1342
1361
|
const statement: Statement = new Statement();
|
|
1343
1362
|
statement.append(
|
|
1344
|
-
`ALTER TABLE ${databaseName}.${this.model.tableName} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
|
|
1363
|
+
`ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
|
|
1345
1364
|
);
|
|
1346
1365
|
|
|
1347
1366
|
logger.debug(`${this.model.tableName} Add Skip Index Statement`);
|
|
@@ -1352,7 +1371,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1352
1371
|
|
|
1353
1372
|
public toDropSkipIndexStatement(indexName: string): string {
|
|
1354
1373
|
const databaseName: string = this.database.getDatasourceOptions().database!;
|
|
1355
|
-
const statement: string = `ALTER TABLE ${databaseName}.${this.model.tableName} DROP INDEX IF EXISTS ${indexName}`;
|
|
1374
|
+
const statement: string = `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} DROP INDEX IF EXISTS ${indexName}`;
|
|
1356
1375
|
|
|
1357
1376
|
logger.debug(`${this.model.tableName} Drop Skip Index Statement`);
|
|
1358
1377
|
logger.debug(statement);
|
|
@@ -1362,7 +1381,9 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1362
1381
|
|
|
1363
1382
|
public toDropColumnStatement(columnName: string): string {
|
|
1364
1383
|
const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
|
|
1365
|
-
.database!}.${
|
|
1384
|
+
.database!}.${getStorageTableName(
|
|
1385
|
+
this.model.tableName,
|
|
1386
|
+
)} DROP COLUMN IF EXISTS ${columnName}`;
|
|
1366
1387
|
|
|
1367
1388
|
logger.debug(`${this.model.tableName} Drop Column Statement`);
|
|
1368
1389
|
logger.debug(statement);
|
|
@@ -1377,16 +1398,40 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1377
1398
|
);
|
|
1378
1399
|
|
|
1379
1400
|
/*
|
|
1380
|
-
* special case - ClickHouse does not support using a query parameter
|
|
1381
|
-
*
|
|
1401
|
+
* special case - ClickHouse does not support using a query parameter to
|
|
1402
|
+
* specify the table name, engine, or ON CLUSTER clause, so these are
|
|
1403
|
+
* interpolated as raw SQL (the SQL tag only parameterizes ${{value,type}}
|
|
1404
|
+
* objects, not plain string interpolations).
|
|
1405
|
+
*
|
|
1406
|
+
* In cluster mode this builds the LOCAL storage table
|
|
1407
|
+
* (`<tableName>Local` with a Replicated* engine) `ON CLUSTER '<name>'`; the
|
|
1408
|
+
* app-facing Distributed table is created separately via
|
|
1409
|
+
* toDistributedTableCreateStatement(). In single-node mode it builds the
|
|
1410
|
+
* model's own table with its plain engine, exactly as before.
|
|
1382
1411
|
*/
|
|
1383
|
-
const tableEngineStatement: string =
|
|
1412
|
+
const tableEngineStatement: string = getStorageEngine(
|
|
1413
|
+
this.model.tableEngine,
|
|
1414
|
+
);
|
|
1415
|
+
|
|
1416
|
+
const storageTableName: string = getStorageTableName(this.model.tableName);
|
|
1417
|
+
|
|
1418
|
+
const onCluster: string = onClusterClause();
|
|
1384
1419
|
|
|
1385
1420
|
const partitionKey: string = this.model.partitionKey;
|
|
1386
1421
|
|
|
1387
1422
|
const statement: Statement = SQL`
|
|
1388
|
-
CREATE TABLE IF NOT EXISTS ${databaseName}.${
|
|
1389
|
-
|
|
1423
|
+
CREATE TABLE IF NOT EXISTS ${databaseName}.${storageTableName}`
|
|
1424
|
+
/*
|
|
1425
|
+
* ON CLUSTER is appended as RAW SQL — the SQL tag turns every ${..}
|
|
1426
|
+
* interpolation into a {pN:Identifier} parameter, which would wrongly
|
|
1427
|
+
* quote the whole " ON CLUSTER '<name>'" clause as a single identifier.
|
|
1428
|
+
* onCluster is "" in single-node mode, so this is a no-op there.
|
|
1429
|
+
*/
|
|
1430
|
+
.append(onCluster)
|
|
1431
|
+
.append(
|
|
1432
|
+
SQL`
|
|
1433
|
+
(\n`,
|
|
1434
|
+
)
|
|
1390
1435
|
.append(columnsStatement)
|
|
1391
1436
|
.append(
|
|
1392
1437
|
SQL`
|
|
@@ -1426,10 +1471,15 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1426
1471
|
|
|
1427
1472
|
/*
|
|
1428
1473
|
* Append table-level SETTINGS if specified (e.g. ttl_only_drop_parts = 1
|
|
1429
|
-
* so TTL drops whole time-partitions instead of rewriting parts).
|
|
1474
|
+
* so TTL drops whole time-partitions instead of rewriting parts). In
|
|
1475
|
+
* cluster mode the non-replicated dedup window is rewritten to its
|
|
1476
|
+
* replicated equivalent so insert idempotency survives.
|
|
1430
1477
|
*/
|
|
1431
|
-
|
|
1432
|
-
|
|
1478
|
+
const tableSettings: string | undefined = adaptTableSettingsForStorage(
|
|
1479
|
+
this.model.tableSettings,
|
|
1480
|
+
);
|
|
1481
|
+
if (tableSettings) {
|
|
1482
|
+
statement.append(`\nSETTINGS ${tableSettings}`);
|
|
1433
1483
|
}
|
|
1434
1484
|
|
|
1435
1485
|
/* eslint-enable prettier/prettier */
|
|
@@ -1439,4 +1489,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
1439
1489
|
|
|
1440
1490
|
return statement;
|
|
1441
1491
|
}
|
|
1492
|
+
|
|
1493
|
+
/*
|
|
1494
|
+
* The app-facing Distributed table that wraps the model's local storage table.
|
|
1495
|
+
* Built with `AS <db>.<local>` so its column layout is copied from — and stays
|
|
1496
|
+
* identical to — the local table, and with `CREATE OR REPLACE` so re-running it
|
|
1497
|
+
* every boot atomically re-syncs the wrapper after a column is reconciled onto
|
|
1498
|
+
* the local table (the Distributed table holds no data, so the replace is cheap
|
|
1499
|
+
* and lossless). The Distributed engine routes writes by the model's sharding
|
|
1500
|
+
* key and scatter-gathers reads across all shards.
|
|
1501
|
+
*/
|
|
1502
|
+
public toDistributedTableCreateStatement(): Statement {
|
|
1503
|
+
const databaseName: string = this.database.getDatasourceOptions().database!;
|
|
1504
|
+
const distributedTableName: string = this.model.tableName;
|
|
1505
|
+
const localTableName: string = getStorageTableName(this.model.tableName);
|
|
1506
|
+
const onCluster: string = onClusterClause();
|
|
1507
|
+
const distributedEngine: string = getDistributedEngine(
|
|
1508
|
+
localTableName,
|
|
1509
|
+
this.model.shardingKey,
|
|
1510
|
+
);
|
|
1511
|
+
|
|
1512
|
+
const statement: Statement = new Statement();
|
|
1513
|
+
statement.append(
|
|
1514
|
+
`CREATE OR REPLACE TABLE ${databaseName}.${distributedTableName}${onCluster} AS ${databaseName}.${localTableName} ENGINE = ${distributedEngine}`,
|
|
1515
|
+
);
|
|
1516
|
+
|
|
1517
|
+
logger.debug(`${this.model.tableName} Distributed Table Create Statement`);
|
|
1518
|
+
logger.debug(statement);
|
|
1519
|
+
|
|
1520
|
+
return statement;
|
|
1521
|
+
}
|
|
1442
1522
|
}
|
|
@@ -397,6 +397,13 @@ export default class CodeRepositoryUtil {
|
|
|
397
397
|
public static async getCommitAuthorsWithFiles(data: {
|
|
398
398
|
repoPath: string;
|
|
399
399
|
path?: string | undefined;
|
|
400
|
+
/*
|
|
401
|
+
* Hard upper bound (ms) on the `git log` invocation. Over a full-history repo
|
|
402
|
+
* this walk can take a very long time, so callers on (or near) a request path
|
|
403
|
+
* should bound it; on timeout the child is killed and this rejects, letting
|
|
404
|
+
* the caller fall back gracefully.
|
|
405
|
+
*/
|
|
406
|
+
timeoutInMS?: number | undefined;
|
|
400
407
|
}): Promise<
|
|
401
408
|
Array<{
|
|
402
409
|
authorName: string;
|
|
@@ -428,6 +435,7 @@ export default class CodeRepositoryUtil {
|
|
|
428
435
|
cwd: path.resolve(data.repoPath),
|
|
429
436
|
// git log over a full repo can easily exceed the 1 MB default.
|
|
430
437
|
maxBuffer: 256 * 1024 * 1024,
|
|
438
|
+
...(data.timeoutInMS ? { timeoutInMS: data.timeoutInMS } : {}),
|
|
431
439
|
});
|
|
432
440
|
|
|
433
441
|
const commits: Array<{
|
package/Server/Utils/Execute.ts
CHANGED
|
@@ -59,6 +59,12 @@ export default class Execute {
|
|
|
59
59
|
* can emit large output (e.g. `git log` over a whole repo) should raise this.
|
|
60
60
|
*/
|
|
61
61
|
maxBuffer?: number | undefined;
|
|
62
|
+
/*
|
|
63
|
+
* Hard upper bound (ms) on how long the child may run. When exceeded the
|
|
64
|
+
* child is force-killed (SIGKILL) and the promise rejects, so a slow command
|
|
65
|
+
* (e.g. `git log` over a huge full-history repo) can never hang the caller.
|
|
66
|
+
*/
|
|
67
|
+
timeoutInMS?: number | undefined;
|
|
62
68
|
}): Promise<string> {
|
|
63
69
|
return new Promise(
|
|
64
70
|
(
|
|
@@ -71,6 +77,9 @@ export default class Execute {
|
|
|
71
77
|
{
|
|
72
78
|
cwd: data.cwd,
|
|
73
79
|
...(data.maxBuffer ? { maxBuffer: data.maxBuffer } : {}),
|
|
80
|
+
...(data.timeoutInMS
|
|
81
|
+
? { timeout: data.timeoutInMS, killSignal: "SIGKILL" }
|
|
82
|
+
: {}),
|
|
74
83
|
},
|
|
75
84
|
(err: ExecException | null, stdout: string, stderr: string) => {
|
|
76
85
|
if (err) {
|
|
@@ -399,6 +399,9 @@ export default class MonitorTemplateUtil {
|
|
|
399
399
|
failureCause: externalStatusPageResponse?.failureCause,
|
|
400
400
|
overallStatus: externalStatusPageResponse?.overallStatus,
|
|
401
401
|
activeIncidentCount: externalStatusPageResponse?.activeIncidentCount,
|
|
402
|
+
provider: externalStatusPageResponse?.provider,
|
|
403
|
+
componentGroup: externalStatusPageResponse?.componentGroupName,
|
|
404
|
+
componentName: externalStatusPageResponse?.componentName,
|
|
402
405
|
} as JSONObject;
|
|
403
406
|
|
|
404
407
|
// Add component statuses
|
|
@@ -410,6 +413,7 @@ export default class MonitorTemplateUtil {
|
|
|
410
413
|
name: component.name,
|
|
411
414
|
status: component.status,
|
|
412
415
|
description: component.description,
|
|
416
|
+
groupName: component.groupName,
|
|
413
417
|
};
|
|
414
418
|
},
|
|
415
419
|
);
|
|
@@ -356,13 +356,18 @@ describe("AnalyticsDatabaseService", () => {
|
|
|
356
356
|
tableName: "<table-name>",
|
|
357
357
|
});
|
|
358
358
|
|
|
359
|
+
/*
|
|
360
|
+
* Cluster mode: deletes are an ALTER ... DELETE mutation on the local
|
|
361
|
+
* table, dispatched ON CLUSTER (lightweight DELETE can't hit a Distributed
|
|
362
|
+
* table).
|
|
363
|
+
*/
|
|
359
364
|
expect(statement.query).toBe(
|
|
360
|
-
"
|
|
361
|
-
"WHERE TRUE <where-statement>",
|
|
365
|
+
"ALTER TABLE {p0:Identifier}.{p1:Identifier} ON CLUSTER 'oneuptime'\n" +
|
|
366
|
+
"DELETE WHERE TRUE <where-statement>",
|
|
362
367
|
);
|
|
363
368
|
expect(statement.query_params).toStrictEqual({
|
|
364
369
|
p0: "oneuptime",
|
|
365
|
-
p1: "<table-name>",
|
|
370
|
+
p1: "<table-name>Local",
|
|
366
371
|
});
|
|
367
372
|
});
|
|
368
373
|
});
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { ClickhouseAppInstance } from "../../../../Server/Infrastructure/ClickhouseDatabase";
|
|
2
|
+
import StatementGenerator from "../../../../Server/Utils/AnalyticsDatabase/StatementGenerator";
|
|
3
|
+
import {
|
|
4
|
+
SQL,
|
|
5
|
+
Statement,
|
|
6
|
+
} from "../../../../Server/Utils/AnalyticsDatabase/Statement";
|
|
7
|
+
import {
|
|
8
|
+
adaptTableSettingsForStorage,
|
|
9
|
+
applyClusterToMaterializedViewQuery,
|
|
10
|
+
getClickhouseClusterName,
|
|
11
|
+
getDistributedEngine,
|
|
12
|
+
getStorageEngine,
|
|
13
|
+
getStorageTableName,
|
|
14
|
+
onClusterClause,
|
|
15
|
+
} from "../../../../Server/Utils/AnalyticsDatabase/ClusterConfig";
|
|
16
|
+
import UpdateBy from "../../../../Server/Types/AnalyticsDatabase/UpdateBy";
|
|
17
|
+
import "../../TestingUtils/Init";
|
|
18
|
+
import AnalyticsBaseModel from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
|
|
19
|
+
import Route from "../../../../Types/API/Route";
|
|
20
|
+
import AnalyticsTableEngine from "../../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
21
|
+
import AnalyticsTableColumn, {
|
|
22
|
+
SkipIndexType,
|
|
23
|
+
} from "../../../../Types/AnalyticsDatabase/TableColumn";
|
|
24
|
+
import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
|
|
25
|
+
|
|
26
|
+
const CLUSTER_ENV_KEY: string = "CLICKHOUSE_CLUSTER_NAME";
|
|
27
|
+
const SHARDING_ENV_KEY: string = "CLICKHOUSE_SHARDING_KEY";
|
|
28
|
+
|
|
29
|
+
/*
|
|
30
|
+
* Flatten a Statement into a single string (raw SQL + serialized identifier
|
|
31
|
+
* params) so assertions don't have to care whether a name landed in the query
|
|
32
|
+
* text or in query_params as a {pN:Identifier} parameter.
|
|
33
|
+
*/
|
|
34
|
+
function fullText(statement: Statement | string): string {
|
|
35
|
+
if (typeof statement === "string") {
|
|
36
|
+
return statement;
|
|
37
|
+
}
|
|
38
|
+
return statement.query + " :: " + JSON.stringify(statement.query_params);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe("ClickHouse cluster-aware schema (always-on)", () => {
|
|
42
|
+
const originalCluster: string | undefined = process.env[CLUSTER_ENV_KEY];
|
|
43
|
+
const originalSharding: string | undefined = process.env[SHARDING_ENV_KEY];
|
|
44
|
+
|
|
45
|
+
afterEach(() => {
|
|
46
|
+
if (originalCluster === undefined) {
|
|
47
|
+
delete process.env[CLUSTER_ENV_KEY];
|
|
48
|
+
} else {
|
|
49
|
+
process.env[CLUSTER_ENV_KEY] = originalCluster;
|
|
50
|
+
}
|
|
51
|
+
if (originalSharding === undefined) {
|
|
52
|
+
delete process.env[SHARDING_ENV_KEY];
|
|
53
|
+
} else {
|
|
54
|
+
process.env[SHARDING_ENV_KEY] = originalSharding;
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
describe("ClusterConfig helpers", () => {
|
|
59
|
+
test("cluster name defaults to 'oneuptime' and is overridable", () => {
|
|
60
|
+
delete process.env[CLUSTER_ENV_KEY];
|
|
61
|
+
expect(getClickhouseClusterName()).toBe("oneuptime");
|
|
62
|
+
expect(onClusterClause()).toBe(" ON CLUSTER 'oneuptime'");
|
|
63
|
+
|
|
64
|
+
process.env[CLUSTER_ENV_KEY] = "ext_cluster";
|
|
65
|
+
expect(getClickhouseClusterName()).toBe("ext_cluster");
|
|
66
|
+
expect(onClusterClause()).toBe(" ON CLUSTER 'ext_cluster'");
|
|
67
|
+
|
|
68
|
+
// whitespace-only falls back to the default
|
|
69
|
+
process.env[CLUSTER_ENV_KEY] = " ";
|
|
70
|
+
expect(getClickhouseClusterName()).toBe("oneuptime");
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("storage table name always gets the Local suffix", () => {
|
|
74
|
+
expect(getStorageTableName("SpanItemV3")).toBe("SpanItemV3Local");
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("engines always map to their Replicated variant", () => {
|
|
78
|
+
expect(getStorageEngine(AnalyticsTableEngine.MergeTree)).toBe(
|
|
79
|
+
"ReplicatedMergeTree",
|
|
80
|
+
);
|
|
81
|
+
expect(getStorageEngine(AnalyticsTableEngine.AggregatingMergeTree)).toBe(
|
|
82
|
+
"ReplicatedAggregatingMergeTree",
|
|
83
|
+
);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("table settings always swap to the replicated dedup window", () => {
|
|
87
|
+
expect(
|
|
88
|
+
adaptTableSettingsForStorage(
|
|
89
|
+
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
90
|
+
),
|
|
91
|
+
).toBe(
|
|
92
|
+
"ttl_only_drop_parts = 1, replicated_deduplication_window = 10000",
|
|
93
|
+
);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("distributed engine: model key, with global override winning", () => {
|
|
97
|
+
delete process.env[SHARDING_ENV_KEY];
|
|
98
|
+
// model sharding key is used
|
|
99
|
+
expect(
|
|
100
|
+
getDistributedEngine("SpanItemV3Local", "cityHash64(traceId)"),
|
|
101
|
+
).toContain("SpanItemV3Local, cityHash64(traceId))");
|
|
102
|
+
// no model key -> default cityHash64(projectId)
|
|
103
|
+
expect(getDistributedEngine("LogItemV3Local")).toContain(
|
|
104
|
+
"cityHash64(projectId))",
|
|
105
|
+
);
|
|
106
|
+
// global override wins over the model key
|
|
107
|
+
process.env[SHARDING_ENV_KEY] = "rand()";
|
|
108
|
+
expect(
|
|
109
|
+
getDistributedEngine("SpanItemV3Local", "cityHash64(traceId)"),
|
|
110
|
+
).toContain("SpanItemV3Local, rand())");
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe("StatementGenerator DDL", () => {
|
|
115
|
+
class SpanModel extends AnalyticsBaseModel {
|
|
116
|
+
public constructor() {
|
|
117
|
+
super({
|
|
118
|
+
tableName: "SpanItemV3",
|
|
119
|
+
singularName: "Span",
|
|
120
|
+
pluralName: "Spans",
|
|
121
|
+
tableColumns: [
|
|
122
|
+
new AnalyticsTableColumn({
|
|
123
|
+
key: "projectId",
|
|
124
|
+
title: "Project",
|
|
125
|
+
description: "Project",
|
|
126
|
+
required: true,
|
|
127
|
+
type: TableColumnType.ObjectID,
|
|
128
|
+
}),
|
|
129
|
+
new AnalyticsTableColumn({
|
|
130
|
+
key: "traceId",
|
|
131
|
+
title: "Trace",
|
|
132
|
+
description: "Trace",
|
|
133
|
+
required: true,
|
|
134
|
+
type: TableColumnType.Text,
|
|
135
|
+
skipIndex: {
|
|
136
|
+
name: "idx_trace_id",
|
|
137
|
+
type: SkipIndexType.BloomFilter,
|
|
138
|
+
params: [0.01],
|
|
139
|
+
granularity: 1,
|
|
140
|
+
},
|
|
141
|
+
}),
|
|
142
|
+
],
|
|
143
|
+
crudApiPath: new Route("span"),
|
|
144
|
+
primaryKeys: ["projectId"],
|
|
145
|
+
sortKeys: ["projectId"],
|
|
146
|
+
partitionKey: "toYYYYMMDD(startTime)",
|
|
147
|
+
shardingKey: "cityHash64(traceId)",
|
|
148
|
+
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
149
|
+
ttlExpression: "retentionDate DELETE",
|
|
150
|
+
tableSettings:
|
|
151
|
+
"ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
class AggregatingModel extends AnalyticsBaseModel {
|
|
157
|
+
public constructor() {
|
|
158
|
+
super({
|
|
159
|
+
tableName: "MetricItemAggMV1m",
|
|
160
|
+
singularName: "Agg",
|
|
161
|
+
pluralName: "Aggs",
|
|
162
|
+
tableColumns: [
|
|
163
|
+
new AnalyticsTableColumn({
|
|
164
|
+
key: "projectId",
|
|
165
|
+
title: "Project",
|
|
166
|
+
description: "Project",
|
|
167
|
+
required: true,
|
|
168
|
+
type: TableColumnType.ObjectID,
|
|
169
|
+
}),
|
|
170
|
+
],
|
|
171
|
+
crudApiPath: new Route("agg"),
|
|
172
|
+
primaryKeys: ["projectId"],
|
|
173
|
+
sortKeys: ["projectId"],
|
|
174
|
+
partitionKey: "toYYYYMM(bucketTime)",
|
|
175
|
+
tableEngine: AnalyticsTableEngine.AggregatingMergeTree,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
let spanGen: StatementGenerator<SpanModel>;
|
|
181
|
+
let aggregatingGen: StatementGenerator<AggregatingModel>;
|
|
182
|
+
|
|
183
|
+
beforeEach(() => {
|
|
184
|
+
delete process.env[CLUSTER_ENV_KEY];
|
|
185
|
+
delete process.env[SHARDING_ENV_KEY];
|
|
186
|
+
spanGen = new StatementGenerator<SpanModel>({
|
|
187
|
+
modelType: SpanModel,
|
|
188
|
+
database: ClickhouseAppInstance,
|
|
189
|
+
});
|
|
190
|
+
aggregatingGen = new StatementGenerator<AggregatingModel>({
|
|
191
|
+
modelType: AggregatingModel,
|
|
192
|
+
database: ClickhouseAppInstance,
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test("CREATE TABLE builds the local Replicated table ON CLUSTER with replicated dedup", () => {
|
|
197
|
+
const stmt: Statement = spanGen.toTableCreateStatement();
|
|
198
|
+
expect(stmt.query).toContain("ENGINE = ReplicatedMergeTree");
|
|
199
|
+
expect(stmt.query).toContain("ON CLUSTER 'oneuptime'");
|
|
200
|
+
expect(stmt.query).toContain("replicated_deduplication_window = 10000");
|
|
201
|
+
expect(stmt.query).not.toContain("non_replicated_deduplication_window");
|
|
202
|
+
expect(fullText(stmt)).toContain("SpanItemV3Local");
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("cluster name override flows into the DDL", () => {
|
|
206
|
+
process.env[CLUSTER_ENV_KEY] = "ext_cluster";
|
|
207
|
+
expect(spanGen.toTableCreateStatement().query).toContain(
|
|
208
|
+
"ON CLUSTER 'ext_cluster'",
|
|
209
|
+
);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
test("AggregatingMergeTree maps to ReplicatedAggregatingMergeTree", () => {
|
|
213
|
+
const stmt: Statement = aggregatingGen.toTableCreateStatement();
|
|
214
|
+
expect(stmt.query).toContain("ENGINE = ReplicatedAggregatingMergeTree");
|
|
215
|
+
expect(fullText(stmt)).toContain("MetricItemAggMV1mLocal");
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test("Distributed wrapper uses the model sharding key and local table", () => {
|
|
219
|
+
const q: string = spanGen.toDistributedTableCreateStatement().query;
|
|
220
|
+
expect(q).toContain("ON CLUSTER 'oneuptime'");
|
|
221
|
+
expect(q).toContain(
|
|
222
|
+
"Distributed('oneuptime', oneuptime, SpanItemV3Local, cityHash64(traceId))",
|
|
223
|
+
);
|
|
224
|
+
expect(q).toContain("AS ");
|
|
225
|
+
expect(q).toContain("SpanItemV3 "); // the app-facing distributed name
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
test("global sharding-key override beats the model key in the Distributed engine", () => {
|
|
229
|
+
process.env[SHARDING_ENV_KEY] = "rand()";
|
|
230
|
+
expect(spanGen.toDistributedTableCreateStatement().query).toContain(
|
|
231
|
+
"SpanItemV3Local, rand())",
|
|
232
|
+
);
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
test("schema ALTERs target the local table", () => {
|
|
236
|
+
const column: AnalyticsTableColumn = new SpanModel().tableColumns[1]!;
|
|
237
|
+
expect(fullText(spanGen.toAddColumnStatement(column))).toContain(
|
|
238
|
+
"SpanItemV3Local",
|
|
239
|
+
);
|
|
240
|
+
expect(fullText(spanGen.toAddSkipIndexStatement(column)!)).toContain(
|
|
241
|
+
"SpanItemV3Local",
|
|
242
|
+
);
|
|
243
|
+
expect(spanGen.toDropColumnStatement("traceId")).toContain(
|
|
244
|
+
"SpanItemV3Local",
|
|
245
|
+
);
|
|
246
|
+
expect(spanGen.toDropSkipIndexStatement("idx_trace_id")).toContain(
|
|
247
|
+
"SpanItemV3Local",
|
|
248
|
+
);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
test("ALTER UPDATE mutates the local table with ON CLUSTER", () => {
|
|
252
|
+
spanGen.toSetStatement = jest.fn(() => {
|
|
253
|
+
return SQL`name = 'x'`;
|
|
254
|
+
});
|
|
255
|
+
spanGen.toWhereStatement = jest.fn(() => {
|
|
256
|
+
return SQL` AND projectId = 'p'`;
|
|
257
|
+
});
|
|
258
|
+
const updateBy: UpdateBy<SpanModel> = {
|
|
259
|
+
data: new SpanModel(),
|
|
260
|
+
query: {},
|
|
261
|
+
props: {},
|
|
262
|
+
};
|
|
263
|
+
const stmt: Statement = spanGen.toUpdateStatement(updateBy);
|
|
264
|
+
expect(stmt.query).toContain("ON CLUSTER 'oneuptime'");
|
|
265
|
+
expect(stmt.query).toContain("UPDATE");
|
|
266
|
+
expect(fullText(stmt)).toContain("SpanItemV3Local");
|
|
267
|
+
});
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
describe("applyClusterToMaterializedViewQuery", () => {
|
|
271
|
+
const MV_QUERY: string = `CREATE MATERIALIZED VIEW IF NOT EXISTS MetricItemAggMV1m_mv
|
|
272
|
+
TO MetricItemAggMV1m
|
|
273
|
+
AS
|
|
274
|
+
SELECT
|
|
275
|
+
projectId,
|
|
276
|
+
toStartOfMinute(time) AS bucketTime,
|
|
277
|
+
sumState(toFloat64(coalesce(value, sum, 0))) AS valueSumState
|
|
278
|
+
FROM MetricItemV3
|
|
279
|
+
GROUP BY projectId, bucketTime`;
|
|
280
|
+
|
|
281
|
+
test("injects ON CLUSTER and retargets TO/FROM at local tables", () => {
|
|
282
|
+
delete process.env[CLUSTER_ENV_KEY];
|
|
283
|
+
const out: string = applyClusterToMaterializedViewQuery(MV_QUERY);
|
|
284
|
+
expect(out).toContain(
|
|
285
|
+
"CREATE MATERIALIZED VIEW IF NOT EXISTS MetricItemAggMV1m_mv ON CLUSTER 'oneuptime'",
|
|
286
|
+
);
|
|
287
|
+
expect(out).toContain("TO MetricItemAggMV1mLocal");
|
|
288
|
+
expect(out).toContain("FROM MetricItemV3Local");
|
|
289
|
+
// aggregate columns / time bucketing must NOT be rewritten
|
|
290
|
+
expect(out).toContain("toStartOfMinute(time) AS bucketTime");
|
|
291
|
+
expect(out).toContain("sumState(toFloat64(coalesce(value, sum, 0)))");
|
|
292
|
+
expect(out).not.toContain("MetricItemAggMV1mLocalLocal");
|
|
293
|
+
expect(out).not.toContain("MetricItemV3LocalLocal");
|
|
294
|
+
});
|
|
295
|
+
});
|
|
296
|
+
});
|
|
@@ -111,10 +111,11 @@ describe("StatementGenerator", () => {
|
|
|
111
111
|
expect(jest.mocked(logger.debug)).toHaveBeenNthCalledWith(2, statement);
|
|
112
112
|
|
|
113
113
|
/* eslint-disable prettier/prettier */
|
|
114
|
+
// Cluster mode: mutation targets the local table and dispatches ON CLUSTER.
|
|
114
115
|
expectStatement(
|
|
115
116
|
statement,
|
|
116
117
|
SQL`
|
|
117
|
-
ALTER TABLE ${"oneuptime"}.${"<table-name>"}
|
|
118
|
+
ALTER TABLE ${"oneuptime"}.${"<table-name>Local"} ON CLUSTER 'oneuptime'
|
|
118
119
|
UPDATE <set-statement>
|
|
119
120
|
WHERE TRUE <where-statement>
|
|
120
121
|
`,
|
|
@@ -868,12 +869,13 @@ describe("StatementGenerator", () => {
|
|
|
868
869
|
expect(jest.mocked(logger.debug)).toHaveBeenNthCalledWith(2, statement);
|
|
869
870
|
|
|
870
871
|
/* eslint-disable prettier/prettier */
|
|
872
|
+
// Cluster mode: the local <table>Local table, Replicated engine, ON CLUSTER.
|
|
871
873
|
const expectedStatement: Statement = SQL`
|
|
872
|
-
CREATE TABLE IF NOT EXISTS ${"oneuptime"}.${"<table-name>"}
|
|
874
|
+
CREATE TABLE IF NOT EXISTS ${"oneuptime"}.${"<table-name>Local"} ON CLUSTER 'oneuptime'
|
|
873
875
|
(
|
|
874
876
|
<columns-create-statement>
|
|
875
877
|
)
|
|
876
|
-
ENGINE =
|
|
878
|
+
ENGINE = ReplicatedMergeTree
|
|
877
879
|
PARTITION BY (column_ObjectID)
|
|
878
880
|
|
|
879
881
|
PRIMARY KEY (${"column_ObjectID"})
|
|
@@ -4,6 +4,7 @@ export interface ExternalStatusPageComponentStatus {
|
|
|
4
4
|
name: string;
|
|
5
5
|
status: string;
|
|
6
6
|
description?: string | undefined;
|
|
7
|
+
groupName?: string | undefined; // the component group this component belongs to, if any
|
|
7
8
|
}
|
|
8
9
|
|
|
9
10
|
export default interface ExternalStatusPageMonitorResponse {
|
|
@@ -13,6 +14,10 @@ export default interface ExternalStatusPageMonitorResponse {
|
|
|
13
14
|
activeIncidentCount: number;
|
|
14
15
|
responseTimeInMs: number;
|
|
15
16
|
failureCause: string;
|
|
17
|
+
// Echoes of the resolved query scope, so consumers (summary view, templates) can show what was queried.
|
|
18
|
+
provider?: string | undefined;
|
|
19
|
+
componentGroupName?: string | undefined;
|
|
20
|
+
componentName?: string | undefined;
|
|
16
21
|
rawBody?: string | undefined;
|
|
17
22
|
isTimeout?: boolean | undefined;
|
|
18
23
|
probeAttempts?: Array<ProbeAttempt> | undefined;
|