@oneuptime/common 10.0.25 → 10.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts +10 -0
- package/Models/AnalyticsModels/ExceptionInstance.ts +55 -1
- package/Models/AnalyticsModels/Log.ts +51 -1
- package/Models/AnalyticsModels/Metric.ts +35 -1
- package/Models/AnalyticsModels/MonitorLog.ts +20 -0
- package/Models/AnalyticsModels/Span.ts +47 -1
- package/Models/DatabaseModels/GlobalConfig.ts +18 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1772350000000-MigrationName.ts +23 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +2 -0
- package/Server/Services/LogAggregationService.ts +25 -24
- package/Server/Utils/AnalyticsDatabase/Statement.ts +3 -1
- package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +44 -1
- package/Server/Utils/Monitor/MonitorLogUtil.ts +92 -17
- package/Server/Utils/Monitor/MonitorMetricUtil.ts +77 -11
- package/Types/AnalyticsDatabase/TableColumn.ts +43 -0
- package/Types/Log/LogQueryToFilter.ts +4 -2
- package/UI/Components/LogsViewer/LogsViewer.tsx +32 -26
- package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js +8 -0
- package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/ExceptionInstance.js +49 -1
- package/build/dist/Models/AnalyticsModels/ExceptionInstance.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Log.js +45 -1
- package/build/dist/Models/AnalyticsModels/Log.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Metric.js +29 -1
- package/build/dist/Models/AnalyticsModels/Metric.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MonitorLog.js +16 -0
- package/build/dist/Models/AnalyticsModels/MonitorLog.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Span.js +41 -1
- package/build/dist/Models/AnalyticsModels/Span.js.map +1 -1
- package/build/dist/Models/DatabaseModels/GlobalConfig.js +19 -0
- package/build/dist/Models/DatabaseModels/GlobalConfig.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1772350000000-MigrationName.js +14 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1772350000000-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +2 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/LogAggregationService.js +19 -19
- package/build/dist/Server/Services/LogAggregationService.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js +3 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +31 -0
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorLogUtil.js +69 -13
- package/build/dist/Server/Utils/Monitor/MonitorLogUtil.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js +61 -11
- package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js.map +1 -1
- package/build/dist/Types/AnalyticsDatabase/TableColumn.js +22 -0
- package/build/dist/Types/AnalyticsDatabase/TableColumn.js.map +1 -1
- package/build/dist/Types/Log/LogQueryToFilter.js +4 -2
- package/build/dist/Types/Log/LogQueryToFilter.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js +22 -16
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
- package/package.json +1 -1
|
@@ -45,6 +45,7 @@ export default class AnalyticsBaseModel extends CommonModel {
|
|
|
45
45
|
projections?: Array<Projection> | undefined;
|
|
46
46
|
materializedViews?: Array<MaterializedView> | undefined;
|
|
47
47
|
enableMCP?: boolean | undefined;
|
|
48
|
+
ttlExpression?: string | undefined; // e.g. "retentionDate DELETE"
|
|
48
49
|
}) {
|
|
49
50
|
super({
|
|
50
51
|
tableColumns: data.tableColumns,
|
|
@@ -148,6 +149,7 @@ export default class AnalyticsBaseModel extends CommonModel {
|
|
|
148
149
|
this.projections = data.projections || [];
|
|
149
150
|
this.materializedViews = data.materializedViews || [];
|
|
150
151
|
this.enableMCP = data.enableMCP || false;
|
|
152
|
+
this.ttlExpression = data.ttlExpression || "";
|
|
151
153
|
}
|
|
152
154
|
|
|
153
155
|
private _enableWorkflowOn: EnableWorkflowOn | undefined;
|
|
@@ -282,6 +284,14 @@ export default class AnalyticsBaseModel extends CommonModel {
|
|
|
282
284
|
this._enableMCP = v;
|
|
283
285
|
}
|
|
284
286
|
|
|
287
|
+
private _ttlExpression: string = "";
|
|
288
|
+
public get ttlExpression(): string {
|
|
289
|
+
return this._ttlExpression;
|
|
290
|
+
}
|
|
291
|
+
public set ttlExpression(v: string) {
|
|
292
|
+
this._ttlExpression = v;
|
|
293
|
+
}
|
|
294
|
+
|
|
285
295
|
public getTenantColumn(): AnalyticsTableColumn | null {
|
|
286
296
|
const column: AnalyticsTableColumn | undefined = this.tableColumns.find(
|
|
287
297
|
(column: AnalyticsTableColumn) => {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
|
|
2
2
|
import Route from "../../Types/API/Route";
|
|
3
3
|
import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
4
|
-
import AnalyticsTableColumn
|
|
4
|
+
import AnalyticsTableColumn, {
|
|
5
|
+
SkipIndexType,
|
|
6
|
+
} from "../../Types/AnalyticsDatabase/TableColumn";
|
|
5
7
|
import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
|
|
6
8
|
import ObjectID from "../../Types/ObjectID";
|
|
7
9
|
import Permission from "../../Types/Permission";
|
|
@@ -108,6 +110,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
108
110
|
description: "Exception Type", // Examples: java.net.ConnectException; OSError; etc.
|
|
109
111
|
required: false,
|
|
110
112
|
type: TableColumnType.Text,
|
|
113
|
+
skipIndex: {
|
|
114
|
+
name: "idx_exception_type",
|
|
115
|
+
type: SkipIndexType.BloomFilter,
|
|
116
|
+
params: [0.01],
|
|
117
|
+
granularity: 1,
|
|
118
|
+
},
|
|
111
119
|
accessControl: {
|
|
112
120
|
read: [
|
|
113
121
|
Permission.ProjectOwner,
|
|
@@ -131,6 +139,10 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
131
139
|
description: "Exception Stack Trace", // Examples: Division by zero; Can't convert 'int' object to str implicitly
|
|
132
140
|
required: false,
|
|
133
141
|
type: TableColumnType.Text,
|
|
142
|
+
codec: {
|
|
143
|
+
codec: "ZSTD",
|
|
144
|
+
level: 3,
|
|
145
|
+
},
|
|
134
146
|
accessControl: {
|
|
135
147
|
read: [
|
|
136
148
|
Permission.ProjectOwner,
|
|
@@ -154,6 +166,10 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
154
166
|
description: "Exception Message", // Examples: Division by zero; Can't convert 'int' object to str implicitly
|
|
155
167
|
required: false,
|
|
156
168
|
type: TableColumnType.Text,
|
|
169
|
+
codec: {
|
|
170
|
+
codec: "ZSTD",
|
|
171
|
+
level: 3,
|
|
172
|
+
},
|
|
157
173
|
accessControl: {
|
|
158
174
|
read: [
|
|
159
175
|
Permission.ProjectOwner,
|
|
@@ -225,6 +241,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
225
241
|
description: "ID of the trace",
|
|
226
242
|
required: false,
|
|
227
243
|
type: TableColumnType.Text,
|
|
244
|
+
skipIndex: {
|
|
245
|
+
name: "idx_trace_id",
|
|
246
|
+
type: SkipIndexType.BloomFilter,
|
|
247
|
+
params: [0.01],
|
|
248
|
+
granularity: 1,
|
|
249
|
+
},
|
|
228
250
|
accessControl: {
|
|
229
251
|
read: [
|
|
230
252
|
Permission.ProjectOwner,
|
|
@@ -248,6 +270,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
248
270
|
description: "ID of the span",
|
|
249
271
|
required: false,
|
|
250
272
|
type: TableColumnType.Text,
|
|
273
|
+
skipIndex: {
|
|
274
|
+
name: "idx_span_id",
|
|
275
|
+
type: SkipIndexType.BloomFilter,
|
|
276
|
+
params: [0.01],
|
|
277
|
+
granularity: 1,
|
|
278
|
+
},
|
|
251
279
|
accessControl: {
|
|
252
280
|
read: [
|
|
253
281
|
Permission.ProjectOwner,
|
|
@@ -271,6 +299,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
271
299
|
description: "Fingerprint of the exception",
|
|
272
300
|
required: true,
|
|
273
301
|
type: TableColumnType.Text,
|
|
302
|
+
skipIndex: {
|
|
303
|
+
name: "idx_fingerprint",
|
|
304
|
+
type: SkipIndexType.BloomFilter,
|
|
305
|
+
params: [0.01],
|
|
306
|
+
granularity: 1,
|
|
307
|
+
},
|
|
274
308
|
accessControl: {
|
|
275
309
|
read: [
|
|
276
310
|
Permission.ProjectOwner,
|
|
@@ -406,6 +440,16 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
406
440
|
},
|
|
407
441
|
});
|
|
408
442
|
|
|
443
|
+
const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
|
|
444
|
+
key: "retentionDate",
|
|
445
|
+
title: "Retention Date",
|
|
446
|
+
description:
|
|
447
|
+
"Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
|
|
448
|
+
required: true,
|
|
449
|
+
type: TableColumnType.Date,
|
|
450
|
+
defaultValue: undefined,
|
|
451
|
+
});
|
|
452
|
+
|
|
409
453
|
super({
|
|
410
454
|
tableName: "ExceptionItem",
|
|
411
455
|
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
@@ -459,11 +503,13 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
459
503
|
environmentColumn,
|
|
460
504
|
parsedFramesColumn,
|
|
461
505
|
attributesColumn,
|
|
506
|
+
retentionDateColumn,
|
|
462
507
|
],
|
|
463
508
|
projections: [],
|
|
464
509
|
sortKeys: ["projectId", "time", "serviceId", "fingerprint"],
|
|
465
510
|
primaryKeys: ["projectId", "time", "serviceId", "fingerprint"],
|
|
466
511
|
partitionKey: "sipHash64(projectId) % 16",
|
|
512
|
+
ttlExpression: "retentionDate DELETE",
|
|
467
513
|
});
|
|
468
514
|
}
|
|
469
515
|
|
|
@@ -602,4 +648,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
|
|
|
602
648
|
public set parsedFrames(v: string | undefined) {
|
|
603
649
|
this.setColumnValue("parsedFrames", v);
|
|
604
650
|
}
|
|
651
|
+
|
|
652
|
+
public get retentionDate(): Date | undefined {
|
|
653
|
+
return this.getColumnValue("retentionDate") as Date | undefined;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
public set retentionDate(v: Date | undefined) {
|
|
657
|
+
this.setColumnValue("retentionDate", v);
|
|
658
|
+
}
|
|
605
659
|
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
|
|
2
2
|
import Route from "../../Types/API/Route";
|
|
3
3
|
import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
4
|
-
import AnalyticsTableColumn
|
|
4
|
+
import AnalyticsTableColumn, {
|
|
5
|
+
SkipIndexType,
|
|
6
|
+
} from "../../Types/AnalyticsDatabase/TableColumn";
|
|
5
7
|
import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
|
|
6
8
|
import { JSONObject } from "../../Types/JSON";
|
|
7
9
|
import ObjectID from "../../Types/ObjectID";
|
|
@@ -109,6 +111,12 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
109
111
|
description: "Log Severity Text",
|
|
110
112
|
required: true,
|
|
111
113
|
type: TableColumnType.Text,
|
|
114
|
+
skipIndex: {
|
|
115
|
+
name: "idx_severity",
|
|
116
|
+
type: SkipIndexType.Set,
|
|
117
|
+
params: [10],
|
|
118
|
+
granularity: 4,
|
|
119
|
+
},
|
|
112
120
|
accessControl: {
|
|
113
121
|
read: [
|
|
114
122
|
Permission.ProjectOwner,
|
|
@@ -205,6 +213,12 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
205
213
|
description: "ID of the trace",
|
|
206
214
|
required: false,
|
|
207
215
|
type: TableColumnType.Text,
|
|
216
|
+
skipIndex: {
|
|
217
|
+
name: "idx_trace_id",
|
|
218
|
+
type: SkipIndexType.BloomFilter,
|
|
219
|
+
params: [0.01],
|
|
220
|
+
granularity: 1,
|
|
221
|
+
},
|
|
208
222
|
accessControl: {
|
|
209
223
|
read: [
|
|
210
224
|
Permission.ProjectOwner,
|
|
@@ -228,6 +242,12 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
228
242
|
description: "ID of the span",
|
|
229
243
|
required: false,
|
|
230
244
|
type: TableColumnType.Text,
|
|
245
|
+
skipIndex: {
|
|
246
|
+
name: "idx_span_id",
|
|
247
|
+
type: SkipIndexType.BloomFilter,
|
|
248
|
+
params: [0.01],
|
|
249
|
+
granularity: 1,
|
|
250
|
+
},
|
|
231
251
|
accessControl: {
|
|
232
252
|
read: [
|
|
233
253
|
Permission.ProjectOwner,
|
|
@@ -251,6 +271,16 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
251
271
|
description: "Body of the Log",
|
|
252
272
|
required: false,
|
|
253
273
|
type: TableColumnType.Text,
|
|
274
|
+
skipIndex: {
|
|
275
|
+
name: "idx_body",
|
|
276
|
+
type: SkipIndexType.TokenBF,
|
|
277
|
+
params: [10240, 3, 0],
|
|
278
|
+
granularity: 4,
|
|
279
|
+
},
|
|
280
|
+
codec: {
|
|
281
|
+
codec: "ZSTD",
|
|
282
|
+
level: 3,
|
|
283
|
+
},
|
|
254
284
|
accessControl: {
|
|
255
285
|
read: [
|
|
256
286
|
Permission.ProjectOwner,
|
|
@@ -268,6 +298,16 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
268
298
|
},
|
|
269
299
|
});
|
|
270
300
|
|
|
301
|
+
const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
|
|
302
|
+
key: "retentionDate",
|
|
303
|
+
title: "Retention Date",
|
|
304
|
+
description:
|
|
305
|
+
"Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
|
|
306
|
+
required: true,
|
|
307
|
+
type: TableColumnType.Date,
|
|
308
|
+
defaultValue: undefined,
|
|
309
|
+
});
|
|
310
|
+
|
|
271
311
|
super({
|
|
272
312
|
tableName: "LogItem",
|
|
273
313
|
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
@@ -312,11 +352,13 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
312
352
|
traceIdColumn,
|
|
313
353
|
spanIdColumn,
|
|
314
354
|
bodyColumn,
|
|
355
|
+
retentionDateColumn,
|
|
315
356
|
],
|
|
316
357
|
projections: [],
|
|
317
358
|
sortKeys: ["projectId", "time", "serviceId"],
|
|
318
359
|
primaryKeys: ["projectId", "time", "serviceId"],
|
|
319
360
|
partitionKey: "sipHash64(projectId) % 16",
|
|
361
|
+
ttlExpression: "retentionDate DELETE",
|
|
320
362
|
});
|
|
321
363
|
}
|
|
322
364
|
|
|
@@ -407,4 +449,12 @@ export default class Log extends AnalyticsBaseModel {
|
|
|
407
449
|
public set spanId(v: string | undefined) {
|
|
408
450
|
this.setColumnValue("spanId", v);
|
|
409
451
|
}
|
|
452
|
+
|
|
453
|
+
public get retentionDate(): Date | undefined {
|
|
454
|
+
return this.getColumnValue("retentionDate") as Date | undefined;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
public set retentionDate(v: Date | undefined) {
|
|
458
|
+
this.setColumnValue("retentionDate", v);
|
|
459
|
+
}
|
|
410
460
|
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
|
|
2
2
|
import Route from "../../Types/API/Route";
|
|
3
3
|
import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
4
|
-
import AnalyticsTableColumn
|
|
4
|
+
import AnalyticsTableColumn, {
|
|
5
|
+
SkipIndexType,
|
|
6
|
+
} from "../../Types/AnalyticsDatabase/TableColumn";
|
|
5
7
|
import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
|
|
6
8
|
import { JSONObject } from "../../Types/JSON";
|
|
7
9
|
import ObjectID from "../../Types/ObjectID";
|
|
@@ -83,6 +85,12 @@ export default class Metric extends AnalyticsBaseModel {
|
|
|
83
85
|
description: "Type of the service that this telemetry belongs to",
|
|
84
86
|
required: false,
|
|
85
87
|
type: TableColumnType.Text,
|
|
88
|
+
skipIndex: {
|
|
89
|
+
name: "idx_service_type",
|
|
90
|
+
type: SkipIndexType.Set,
|
|
91
|
+
params: [5],
|
|
92
|
+
granularity: 4,
|
|
93
|
+
},
|
|
86
94
|
accessControl: {
|
|
87
95
|
read: [
|
|
88
96
|
Permission.ProjectOwner,
|
|
@@ -107,6 +115,12 @@ export default class Metric extends AnalyticsBaseModel {
|
|
|
107
115
|
description: "Name of the Metric",
|
|
108
116
|
required: true,
|
|
109
117
|
type: TableColumnType.Text,
|
|
118
|
+
skipIndex: {
|
|
119
|
+
name: "idx_name",
|
|
120
|
+
type: SkipIndexType.BloomFilter,
|
|
121
|
+
params: [0.01],
|
|
122
|
+
granularity: 1,
|
|
123
|
+
},
|
|
110
124
|
accessControl: {
|
|
111
125
|
read: [
|
|
112
126
|
Permission.ProjectOwner,
|
|
@@ -503,6 +517,16 @@ export default class Metric extends AnalyticsBaseModel {
|
|
|
503
517
|
},
|
|
504
518
|
);
|
|
505
519
|
|
|
520
|
+
const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
|
|
521
|
+
key: "retentionDate",
|
|
522
|
+
title: "Retention Date",
|
|
523
|
+
description:
|
|
524
|
+
"Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
|
|
525
|
+
required: true,
|
|
526
|
+
type: TableColumnType.Date,
|
|
527
|
+
defaultValue: undefined,
|
|
528
|
+
});
|
|
529
|
+
|
|
506
530
|
super({
|
|
507
531
|
tableName: "MetricItem",
|
|
508
532
|
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
@@ -556,11 +580,13 @@ export default class Metric extends AnalyticsBaseModel {
|
|
|
556
580
|
maxColumn,
|
|
557
581
|
bucketCountsColumn,
|
|
558
582
|
explicitBoundsColumn,
|
|
583
|
+
retentionDateColumn,
|
|
559
584
|
],
|
|
560
585
|
projections: [],
|
|
561
586
|
sortKeys: ["projectId", "time", "serviceId"],
|
|
562
587
|
primaryKeys: ["projectId", "time", "serviceId"],
|
|
563
588
|
partitionKey: "sipHash64(projectId) % 16",
|
|
589
|
+
ttlExpression: "retentionDate DELETE",
|
|
564
590
|
});
|
|
565
591
|
}
|
|
566
592
|
|
|
@@ -727,4 +753,12 @@ export default class Metric extends AnalyticsBaseModel {
|
|
|
727
753
|
public set explicitBounds(v: Array<number> | undefined) {
|
|
728
754
|
this.setColumnValue("explicitBounds", v);
|
|
729
755
|
}
|
|
756
|
+
|
|
757
|
+
public get retentionDate(): Date | undefined {
|
|
758
|
+
return this.getColumnValue("retentionDate") as Date | undefined;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
public set retentionDate(v: Date | undefined) {
|
|
762
|
+
this.setColumnValue("retentionDate", v);
|
|
763
|
+
}
|
|
730
764
|
}
|
|
@@ -103,6 +103,16 @@ export default class MonitorLog extends AnalyticsBaseModel {
|
|
|
103
103
|
},
|
|
104
104
|
});
|
|
105
105
|
|
|
106
|
+
const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
|
|
107
|
+
key: "retentionDate",
|
|
108
|
+
title: "Retention Date",
|
|
109
|
+
description:
|
|
110
|
+
"Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
|
|
111
|
+
required: true,
|
|
112
|
+
type: TableColumnType.Date,
|
|
113
|
+
defaultValue: undefined,
|
|
114
|
+
});
|
|
115
|
+
|
|
106
116
|
super({
|
|
107
117
|
tableName: "MonitorLog",
|
|
108
118
|
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
@@ -140,11 +150,13 @@ export default class MonitorLog extends AnalyticsBaseModel {
|
|
|
140
150
|
monitorIdColumn,
|
|
141
151
|
timeColumn,
|
|
142
152
|
logBodyColumn,
|
|
153
|
+
retentionDateColumn,
|
|
143
154
|
],
|
|
144
155
|
projections: [],
|
|
145
156
|
sortKeys: ["projectId", "time", "monitorId"],
|
|
146
157
|
primaryKeys: ["projectId", "time", "monitorId"],
|
|
147
158
|
partitionKey: "sipHash64(projectId) % 16",
|
|
159
|
+
ttlExpression: "retentionDate DELETE",
|
|
148
160
|
});
|
|
149
161
|
}
|
|
150
162
|
|
|
@@ -176,4 +188,12 @@ export default class MonitorLog extends AnalyticsBaseModel {
|
|
|
176
188
|
public set logBody(v: JSONObject | undefined) {
|
|
177
189
|
this.setColumnValue("logBody", v);
|
|
178
190
|
}
|
|
191
|
+
|
|
192
|
+
public get retentionDate(): Date | undefined {
|
|
193
|
+
return this.getColumnValue("retentionDate") as Date | undefined;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
public set retentionDate(v: Date | undefined) {
|
|
197
|
+
this.setColumnValue("retentionDate", v);
|
|
198
|
+
}
|
|
179
199
|
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
|
|
2
2
|
import Route from "../../Types/API/Route";
|
|
3
3
|
import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
|
|
4
|
-
import AnalyticsTableColumn
|
|
4
|
+
import AnalyticsTableColumn, {
|
|
5
|
+
SkipIndexType,
|
|
6
|
+
} from "../../Types/AnalyticsDatabase/TableColumn";
|
|
5
7
|
import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
|
|
6
8
|
import { JSONObject } from "../../Types/JSON";
|
|
7
9
|
import ObjectID from "../../Types/ObjectID";
|
|
@@ -212,6 +214,12 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
212
214
|
description: "ID of the trace",
|
|
213
215
|
required: true,
|
|
214
216
|
type: TableColumnType.Text,
|
|
217
|
+
skipIndex: {
|
|
218
|
+
name: "idx_trace_id",
|
|
219
|
+
type: SkipIndexType.BloomFilter,
|
|
220
|
+
params: [0.01],
|
|
221
|
+
granularity: 1,
|
|
222
|
+
},
|
|
215
223
|
accessControl: {
|
|
216
224
|
read: [
|
|
217
225
|
Permission.ProjectOwner,
|
|
@@ -235,6 +243,12 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
235
243
|
description: "ID of the span",
|
|
236
244
|
required: true,
|
|
237
245
|
type: TableColumnType.Text,
|
|
246
|
+
skipIndex: {
|
|
247
|
+
name: "idx_span_id",
|
|
248
|
+
type: SkipIndexType.BloomFilter,
|
|
249
|
+
params: [0.01],
|
|
250
|
+
granularity: 1,
|
|
251
|
+
},
|
|
238
252
|
accessControl: {
|
|
239
253
|
read: [
|
|
240
254
|
Permission.ProjectOwner,
|
|
@@ -400,6 +414,12 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
400
414
|
description: "Status Code",
|
|
401
415
|
required: false,
|
|
402
416
|
type: TableColumnType.Number,
|
|
417
|
+
skipIndex: {
|
|
418
|
+
name: "idx_status_code",
|
|
419
|
+
type: SkipIndexType.Set,
|
|
420
|
+
params: [5],
|
|
421
|
+
granularity: 4,
|
|
422
|
+
},
|
|
403
423
|
accessControl: {
|
|
404
424
|
read: [
|
|
405
425
|
Permission.ProjectOwner,
|
|
@@ -446,6 +466,12 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
446
466
|
description: "Name of the span",
|
|
447
467
|
required: false,
|
|
448
468
|
type: TableColumnType.Text,
|
|
469
|
+
skipIndex: {
|
|
470
|
+
name: "idx_name",
|
|
471
|
+
type: SkipIndexType.TokenBF,
|
|
472
|
+
params: [10240, 3, 0],
|
|
473
|
+
granularity: 4,
|
|
474
|
+
},
|
|
449
475
|
accessControl: {
|
|
450
476
|
read: [
|
|
451
477
|
Permission.ProjectOwner,
|
|
@@ -486,6 +512,16 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
486
512
|
},
|
|
487
513
|
});
|
|
488
514
|
|
|
515
|
+
const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
|
|
516
|
+
key: "retentionDate",
|
|
517
|
+
title: "Retention Date",
|
|
518
|
+
description:
|
|
519
|
+
"Date after which this row is eligible for TTL deletion, computed at ingest time as startTime + service.retainTelemetryDataForDays",
|
|
520
|
+
required: true,
|
|
521
|
+
type: TableColumnType.Date,
|
|
522
|
+
defaultValue: undefined,
|
|
523
|
+
});
|
|
524
|
+
|
|
489
525
|
super({
|
|
490
526
|
tableName: "SpanItem",
|
|
491
527
|
tableEngine: AnalyticsTableEngine.MergeTree,
|
|
@@ -538,11 +574,13 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
538
574
|
statusMessageColumn,
|
|
539
575
|
nameColumn,
|
|
540
576
|
kindColumn,
|
|
577
|
+
retentionDateColumn,
|
|
541
578
|
],
|
|
542
579
|
projections: [],
|
|
543
580
|
sortKeys: ["projectId", "startTime", "serviceId", "traceId"],
|
|
544
581
|
primaryKeys: ["projectId", "startTime", "serviceId", "traceId"],
|
|
545
582
|
partitionKey: "sipHash64(projectId) % 16",
|
|
583
|
+
ttlExpression: "retentionDate DELETE",
|
|
546
584
|
});
|
|
547
585
|
}
|
|
548
586
|
|
|
@@ -697,4 +735,12 @@ export default class Span extends AnalyticsBaseModel {
|
|
|
697
735
|
public set statusMessage(v: string | undefined) {
|
|
698
736
|
this.setColumnValue("statusMessage", v);
|
|
699
737
|
}
|
|
738
|
+
|
|
739
|
+
public get retentionDate(): Date | undefined {
|
|
740
|
+
return this.getColumnValue("retentionDate") as Date | undefined;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
public set retentionDate(v: Date | undefined) {
|
|
744
|
+
this.setColumnValue("retentionDate", v);
|
|
745
|
+
}
|
|
700
746
|
}
|
|
@@ -605,4 +605,22 @@ export default class GlobalConfig extends GlobalConfigModel {
|
|
|
605
605
|
unique: true,
|
|
606
606
|
})
|
|
607
607
|
public monitorLogRetentionInDays?: number = undefined;
|
|
608
|
+
|
|
609
|
+
@ColumnAccessControl({
|
|
610
|
+
create: [],
|
|
611
|
+
read: [],
|
|
612
|
+
update: [],
|
|
613
|
+
})
|
|
614
|
+
@TableColumn({
|
|
615
|
+
type: TableColumnType.Number,
|
|
616
|
+
title: "Monitor Metric Retention Days",
|
|
617
|
+
description:
|
|
618
|
+
"Number of days to retain monitor metrics. Monitor metrics older than this will be automatically deleted. Default is 1 day.",
|
|
619
|
+
})
|
|
620
|
+
@Column({
|
|
621
|
+
type: ColumnType.Number,
|
|
622
|
+
nullable: true,
|
|
623
|
+
unique: true,
|
|
624
|
+
})
|
|
625
|
+
public monitorMetricRetentionInDays?: number = undefined;
|
|
608
626
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { MigrationInterface, QueryRunner } from "typeorm";
|
|
2
|
+
|
|
3
|
+
export class MigrationName1772350000000 implements MigrationInterface {
|
|
4
|
+
public name = "MigrationName1772350000000";
|
|
5
|
+
|
|
6
|
+
public async up(queryRunner: QueryRunner): Promise<void> {
|
|
7
|
+
await queryRunner.query(
|
|
8
|
+
`ALTER TABLE "GlobalConfig" ADD "monitorMetricRetentionInDays" integer`,
|
|
9
|
+
);
|
|
10
|
+
await queryRunner.query(
|
|
11
|
+
`ALTER TABLE "GlobalConfig" ADD CONSTRAINT "UQ_monitor_metric_retention" UNIQUE ("monitorMetricRetentionInDays")`,
|
|
12
|
+
);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
public async down(queryRunner: QueryRunner): Promise<void> {
|
|
16
|
+
await queryRunner.query(
|
|
17
|
+
`ALTER TABLE "GlobalConfig" DROP CONSTRAINT "UQ_monitor_metric_retention"`,
|
|
18
|
+
);
|
|
19
|
+
await queryRunner.query(
|
|
20
|
+
`ALTER TABLE "GlobalConfig" DROP COLUMN "monitorMetricRetentionInDays"`,
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -261,6 +261,7 @@ import { MigrationName1770834237090 } from "./1770834237090-MigrationName";
|
|
|
261
261
|
import { MigrationName1770834237091 } from "./1770834237091-MigrationName";
|
|
262
262
|
import { MigrationName1772111896988 } from "./1772111896988-MigrationName";
|
|
263
263
|
import { MigrationName1772280000000 } from "./1772280000000-MigrationName";
|
|
264
|
+
import { MigrationName1772350000000 } from "./1772350000000-MigrationName";
|
|
264
265
|
|
|
265
266
|
export default [
|
|
266
267
|
InitialMigration,
|
|
@@ -526,4 +527,5 @@ export default [
|
|
|
526
527
|
MigrationName1770834237091,
|
|
527
528
|
MigrationName1772111896988,
|
|
528
529
|
MigrationName1772280000000,
|
|
530
|
+
MigrationName1772350000000,
|
|
529
531
|
];
|
|
@@ -4,6 +4,7 @@ import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
|
|
|
4
4
|
import { JSONObject } from "../../Types/JSON";
|
|
5
5
|
import ObjectID from "../../Types/ObjectID";
|
|
6
6
|
import BadDataException from "../../Types/Exception/BadDataException";
|
|
7
|
+
import Includes from "../../Types/BaseDatabase/Includes";
|
|
7
8
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
8
9
|
import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
|
|
9
10
|
|
|
@@ -204,39 +205,43 @@ export class LogAggregationService {
|
|
|
204
205
|
>,
|
|
205
206
|
): void {
|
|
206
207
|
if (request.serviceIds && request.serviceIds.length > 0) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
208
|
+
statement.append(
|
|
209
|
+
SQL` AND serviceId IN (${{
|
|
210
|
+
type: TableColumnType.ObjectID,
|
|
211
|
+
value: new Includes(
|
|
212
|
+
request.serviceIds.map((id: ObjectID) => {
|
|
213
|
+
return id.toString();
|
|
214
|
+
}),
|
|
215
|
+
),
|
|
216
|
+
}})`,
|
|
211
217
|
);
|
|
212
|
-
statement.append(` AND serviceId IN (${idStrings.join(",")})`);
|
|
213
218
|
}
|
|
214
219
|
|
|
215
220
|
if (request.severityTexts && request.severityTexts.length > 0) {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
221
|
+
statement.append(
|
|
222
|
+
SQL` AND severityText IN (${{
|
|
223
|
+
type: TableColumnType.Text,
|
|
224
|
+
value: new Includes(request.severityTexts),
|
|
225
|
+
}})`,
|
|
220
226
|
);
|
|
221
|
-
statement.append(` AND severityText IN (${sevStrings.join(",")})`);
|
|
222
227
|
}
|
|
223
228
|
|
|
224
229
|
if (request.traceIds && request.traceIds.length > 0) {
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
230
|
+
statement.append(
|
|
231
|
+
SQL` AND traceId IN (${{
|
|
232
|
+
type: TableColumnType.Text,
|
|
233
|
+
value: new Includes(request.traceIds),
|
|
234
|
+
}})`,
|
|
229
235
|
);
|
|
230
|
-
statement.append(` AND traceId IN (${traceStrings.join(",")})`);
|
|
231
236
|
}
|
|
232
237
|
|
|
233
238
|
if (request.spanIds && request.spanIds.length > 0) {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
239
|
+
statement.append(
|
|
240
|
+
SQL` AND spanId IN (${{
|
|
241
|
+
type: TableColumnType.Text,
|
|
242
|
+
value: new Includes(request.spanIds),
|
|
243
|
+
}})`,
|
|
238
244
|
);
|
|
239
|
-
statement.append(` AND spanId IN (${spanStrings.join(",")})`);
|
|
240
245
|
}
|
|
241
246
|
|
|
242
247
|
if (request.bodySearchText && request.bodySearchText.trim().length > 0) {
|
|
@@ -275,10 +280,6 @@ export class LogAggregationService {
|
|
|
275
280
|
throw new BadDataException("Invalid facetKey");
|
|
276
281
|
}
|
|
277
282
|
}
|
|
278
|
-
|
|
279
|
-
private static escapeSingleQuotes(value: string): string {
|
|
280
|
-
return value.replace(/'/g, "\\'");
|
|
281
|
-
}
|
|
282
283
|
}
|
|
283
284
|
|
|
284
285
|
export default LogAggregationService;
|
|
@@ -9,6 +9,7 @@ import LessThan from "../../../Types/BaseDatabase/LessThan";
|
|
|
9
9
|
import LessThanOrEqual from "../../../Types/BaseDatabase/LessThanOrEqual";
|
|
10
10
|
import LessThanOrNull from "../../../Types/BaseDatabase/LessThanOrNull";
|
|
11
11
|
import GreaterThanOrNull from "../../../Types/BaseDatabase/GreaterThanOrNull";
|
|
12
|
+
import NotEqual from "../../../Types/BaseDatabase/NotEqual";
|
|
12
13
|
import Search from "../../../Types/BaseDatabase/Search";
|
|
13
14
|
import OneUptimeDate from "../../../Types/Date";
|
|
14
15
|
import Dictionary from "../../../Types/Dictionary";
|
|
@@ -103,7 +104,8 @@ export class Statement implements BaseQueryParams {
|
|
|
103
104
|
v.value instanceof GreaterThan ||
|
|
104
105
|
v.value instanceof GreaterThanOrEqual ||
|
|
105
106
|
v.value instanceof LessThanOrNull ||
|
|
106
|
-
v.value instanceof GreaterThanOrNull
|
|
107
|
+
v.value instanceof GreaterThanOrNull ||
|
|
108
|
+
v.value instanceof NotEqual
|
|
107
109
|
) {
|
|
108
110
|
finalValue = v.value.value;
|
|
109
111
|
} else if (v.value instanceof Includes) {
|