@oneuptime/common 10.0.25 → 10.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts +10 -0
  2. package/Models/AnalyticsModels/ExceptionInstance.ts +55 -1
  3. package/Models/AnalyticsModels/Log.ts +51 -1
  4. package/Models/AnalyticsModels/Metric.ts +35 -1
  5. package/Models/AnalyticsModels/MonitorLog.ts +20 -0
  6. package/Models/AnalyticsModels/Span.ts +47 -1
  7. package/Models/DatabaseModels/GlobalConfig.ts +18 -0
  8. package/Server/Infrastructure/Postgres/SchemaMigrations/1772350000000-MigrationName.ts +23 -0
  9. package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +2 -0
  10. package/Server/Services/LogAggregationService.ts +25 -24
  11. package/Server/Utils/AnalyticsDatabase/Statement.ts +3 -1
  12. package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +44 -1
  13. package/Server/Utils/Monitor/MonitorLogUtil.ts +92 -17
  14. package/Server/Utils/Monitor/MonitorMetricUtil.ts +77 -11
  15. package/Types/AnalyticsDatabase/TableColumn.ts +43 -0
  16. package/Types/Log/LogQueryToFilter.ts +4 -2
  17. package/UI/Components/LogsViewer/LogsViewer.tsx +32 -26
  18. package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js +8 -0
  19. package/build/dist/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.js.map +1 -1
  20. package/build/dist/Models/AnalyticsModels/ExceptionInstance.js +49 -1
  21. package/build/dist/Models/AnalyticsModels/ExceptionInstance.js.map +1 -1
  22. package/build/dist/Models/AnalyticsModels/Log.js +45 -1
  23. package/build/dist/Models/AnalyticsModels/Log.js.map +1 -1
  24. package/build/dist/Models/AnalyticsModels/Metric.js +29 -1
  25. package/build/dist/Models/AnalyticsModels/Metric.js.map +1 -1
  26. package/build/dist/Models/AnalyticsModels/MonitorLog.js +16 -0
  27. package/build/dist/Models/AnalyticsModels/MonitorLog.js.map +1 -1
  28. package/build/dist/Models/AnalyticsModels/Span.js +41 -1
  29. package/build/dist/Models/AnalyticsModels/Span.js.map +1 -1
  30. package/build/dist/Models/DatabaseModels/GlobalConfig.js +19 -0
  31. package/build/dist/Models/DatabaseModels/GlobalConfig.js.map +1 -1
  32. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1772350000000-MigrationName.js +14 -0
  33. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1772350000000-MigrationName.js.map +1 -0
  34. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +2 -0
  35. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
  36. package/build/dist/Server/Services/LogAggregationService.js +19 -19
  37. package/build/dist/Server/Services/LogAggregationService.js.map +1 -1
  38. package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js +3 -1
  39. package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js.map +1 -1
  40. package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +31 -0
  41. package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
  42. package/build/dist/Server/Utils/Monitor/MonitorLogUtil.js +69 -13
  43. package/build/dist/Server/Utils/Monitor/MonitorLogUtil.js.map +1 -1
  44. package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js +61 -11
  45. package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js.map +1 -1
  46. package/build/dist/Types/AnalyticsDatabase/TableColumn.js +22 -0
  47. package/build/dist/Types/AnalyticsDatabase/TableColumn.js.map +1 -1
  48. package/build/dist/Types/Log/LogQueryToFilter.js +4 -2
  49. package/build/dist/Types/Log/LogQueryToFilter.js.map +1 -1
  50. package/build/dist/UI/Components/LogsViewer/LogsViewer.js +22 -16
  51. package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
  52. package/package.json +1 -1
@@ -45,6 +45,7 @@ export default class AnalyticsBaseModel extends CommonModel {
45
45
  projections?: Array<Projection> | undefined;
46
46
  materializedViews?: Array<MaterializedView> | undefined;
47
47
  enableMCP?: boolean | undefined;
48
+ ttlExpression?: string | undefined; // e.g. "retentionDate DELETE"
48
49
  }) {
49
50
  super({
50
51
  tableColumns: data.tableColumns,
@@ -148,6 +149,7 @@ export default class AnalyticsBaseModel extends CommonModel {
148
149
  this.projections = data.projections || [];
149
150
  this.materializedViews = data.materializedViews || [];
150
151
  this.enableMCP = data.enableMCP || false;
152
+ this.ttlExpression = data.ttlExpression || "";
151
153
  }
152
154
 
153
155
  private _enableWorkflowOn: EnableWorkflowOn | undefined;
@@ -282,6 +284,14 @@ export default class AnalyticsBaseModel extends CommonModel {
282
284
  this._enableMCP = v;
283
285
  }
284
286
 
287
+ private _ttlExpression: string = "";
288
+ public get ttlExpression(): string {
289
+ return this._ttlExpression;
290
+ }
291
+ public set ttlExpression(v: string) {
292
+ this._ttlExpression = v;
293
+ }
294
+
285
295
  public getTenantColumn(): AnalyticsTableColumn | null {
286
296
  const column: AnalyticsTableColumn | undefined = this.tableColumns.find(
287
297
  (column: AnalyticsTableColumn) => {
@@ -1,7 +1,9 @@
1
1
  import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
2
2
  import Route from "../../Types/API/Route";
3
3
  import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
4
- import AnalyticsTableColumn from "../../Types/AnalyticsDatabase/TableColumn";
4
+ import AnalyticsTableColumn, {
5
+ SkipIndexType,
6
+ } from "../../Types/AnalyticsDatabase/TableColumn";
5
7
  import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
6
8
  import ObjectID from "../../Types/ObjectID";
7
9
  import Permission from "../../Types/Permission";
@@ -108,6 +110,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
108
110
  description: "Exception Type", // Examples: java.net.ConnectException; OSError; etc.
109
111
  required: false,
110
112
  type: TableColumnType.Text,
113
+ skipIndex: {
114
+ name: "idx_exception_type",
115
+ type: SkipIndexType.BloomFilter,
116
+ params: [0.01],
117
+ granularity: 1,
118
+ },
111
119
  accessControl: {
112
120
  read: [
113
121
  Permission.ProjectOwner,
@@ -131,6 +139,10 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
131
139
  description: "Exception Stack Trace", // Examples: Division by zero; Can't convert 'int' object to str implicitly
132
140
  required: false,
133
141
  type: TableColumnType.Text,
142
+ codec: {
143
+ codec: "ZSTD",
144
+ level: 3,
145
+ },
134
146
  accessControl: {
135
147
  read: [
136
148
  Permission.ProjectOwner,
@@ -154,6 +166,10 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
154
166
  description: "Exception Message", // Examples: Division by zero; Can't convert 'int' object to str implicitly
155
167
  required: false,
156
168
  type: TableColumnType.Text,
169
+ codec: {
170
+ codec: "ZSTD",
171
+ level: 3,
172
+ },
157
173
  accessControl: {
158
174
  read: [
159
175
  Permission.ProjectOwner,
@@ -225,6 +241,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
225
241
  description: "ID of the trace",
226
242
  required: false,
227
243
  type: TableColumnType.Text,
244
+ skipIndex: {
245
+ name: "idx_trace_id",
246
+ type: SkipIndexType.BloomFilter,
247
+ params: [0.01],
248
+ granularity: 1,
249
+ },
228
250
  accessControl: {
229
251
  read: [
230
252
  Permission.ProjectOwner,
@@ -248,6 +270,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
248
270
  description: "ID of the span",
249
271
  required: false,
250
272
  type: TableColumnType.Text,
273
+ skipIndex: {
274
+ name: "idx_span_id",
275
+ type: SkipIndexType.BloomFilter,
276
+ params: [0.01],
277
+ granularity: 1,
278
+ },
251
279
  accessControl: {
252
280
  read: [
253
281
  Permission.ProjectOwner,
@@ -271,6 +299,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
271
299
  description: "Fingerprint of the exception",
272
300
  required: true,
273
301
  type: TableColumnType.Text,
302
+ skipIndex: {
303
+ name: "idx_fingerprint",
304
+ type: SkipIndexType.BloomFilter,
305
+ params: [0.01],
306
+ granularity: 1,
307
+ },
274
308
  accessControl: {
275
309
  read: [
276
310
  Permission.ProjectOwner,
@@ -406,6 +440,16 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
406
440
  },
407
441
  });
408
442
 
443
+ const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
444
+ key: "retentionDate",
445
+ title: "Retention Date",
446
+ description:
447
+ "Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
448
+ required: true,
449
+ type: TableColumnType.Date,
450
+ defaultValue: undefined,
451
+ });
452
+
409
453
  super({
410
454
  tableName: "ExceptionItem",
411
455
  tableEngine: AnalyticsTableEngine.MergeTree,
@@ -459,11 +503,13 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
459
503
  environmentColumn,
460
504
  parsedFramesColumn,
461
505
  attributesColumn,
506
+ retentionDateColumn,
462
507
  ],
463
508
  projections: [],
464
509
  sortKeys: ["projectId", "time", "serviceId", "fingerprint"],
465
510
  primaryKeys: ["projectId", "time", "serviceId", "fingerprint"],
466
511
  partitionKey: "sipHash64(projectId) % 16",
512
+ ttlExpression: "retentionDate DELETE",
467
513
  });
468
514
  }
469
515
 
@@ -602,4 +648,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
602
648
  public set parsedFrames(v: string | undefined) {
603
649
  this.setColumnValue("parsedFrames", v);
604
650
  }
651
+
652
+ public get retentionDate(): Date | undefined {
653
+ return this.getColumnValue("retentionDate") as Date | undefined;
654
+ }
655
+
656
+ public set retentionDate(v: Date | undefined) {
657
+ this.setColumnValue("retentionDate", v);
658
+ }
605
659
  }
@@ -1,7 +1,9 @@
1
1
  import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
2
2
  import Route from "../../Types/API/Route";
3
3
  import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
4
- import AnalyticsTableColumn from "../../Types/AnalyticsDatabase/TableColumn";
4
+ import AnalyticsTableColumn, {
5
+ SkipIndexType,
6
+ } from "../../Types/AnalyticsDatabase/TableColumn";
5
7
  import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
6
8
  import { JSONObject } from "../../Types/JSON";
7
9
  import ObjectID from "../../Types/ObjectID";
@@ -109,6 +111,12 @@ export default class Log extends AnalyticsBaseModel {
109
111
  description: "Log Severity Text",
110
112
  required: true,
111
113
  type: TableColumnType.Text,
114
+ skipIndex: {
115
+ name: "idx_severity",
116
+ type: SkipIndexType.Set,
117
+ params: [10],
118
+ granularity: 4,
119
+ },
112
120
  accessControl: {
113
121
  read: [
114
122
  Permission.ProjectOwner,
@@ -205,6 +213,12 @@ export default class Log extends AnalyticsBaseModel {
205
213
  description: "ID of the trace",
206
214
  required: false,
207
215
  type: TableColumnType.Text,
216
+ skipIndex: {
217
+ name: "idx_trace_id",
218
+ type: SkipIndexType.BloomFilter,
219
+ params: [0.01],
220
+ granularity: 1,
221
+ },
208
222
  accessControl: {
209
223
  read: [
210
224
  Permission.ProjectOwner,
@@ -228,6 +242,12 @@ export default class Log extends AnalyticsBaseModel {
228
242
  description: "ID of the span",
229
243
  required: false,
230
244
  type: TableColumnType.Text,
245
+ skipIndex: {
246
+ name: "idx_span_id",
247
+ type: SkipIndexType.BloomFilter,
248
+ params: [0.01],
249
+ granularity: 1,
250
+ },
231
251
  accessControl: {
232
252
  read: [
233
253
  Permission.ProjectOwner,
@@ -251,6 +271,16 @@ export default class Log extends AnalyticsBaseModel {
251
271
  description: "Body of the Log",
252
272
  required: false,
253
273
  type: TableColumnType.Text,
274
+ skipIndex: {
275
+ name: "idx_body",
276
+ type: SkipIndexType.TokenBF,
277
+ params: [10240, 3, 0],
278
+ granularity: 4,
279
+ },
280
+ codec: {
281
+ codec: "ZSTD",
282
+ level: 3,
283
+ },
254
284
  accessControl: {
255
285
  read: [
256
286
  Permission.ProjectOwner,
@@ -268,6 +298,16 @@ export default class Log extends AnalyticsBaseModel {
268
298
  },
269
299
  });
270
300
 
301
+ const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
302
+ key: "retentionDate",
303
+ title: "Retention Date",
304
+ description:
305
+ "Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
306
+ required: true,
307
+ type: TableColumnType.Date,
308
+ defaultValue: undefined,
309
+ });
310
+
271
311
  super({
272
312
  tableName: "LogItem",
273
313
  tableEngine: AnalyticsTableEngine.MergeTree,
@@ -312,11 +352,13 @@ export default class Log extends AnalyticsBaseModel {
312
352
  traceIdColumn,
313
353
  spanIdColumn,
314
354
  bodyColumn,
355
+ retentionDateColumn,
315
356
  ],
316
357
  projections: [],
317
358
  sortKeys: ["projectId", "time", "serviceId"],
318
359
  primaryKeys: ["projectId", "time", "serviceId"],
319
360
  partitionKey: "sipHash64(projectId) % 16",
361
+ ttlExpression: "retentionDate DELETE",
320
362
  });
321
363
  }
322
364
 
@@ -407,4 +449,12 @@ export default class Log extends AnalyticsBaseModel {
407
449
  public set spanId(v: string | undefined) {
408
450
  this.setColumnValue("spanId", v);
409
451
  }
452
+
453
+ public get retentionDate(): Date | undefined {
454
+ return this.getColumnValue("retentionDate") as Date | undefined;
455
+ }
456
+
457
+ public set retentionDate(v: Date | undefined) {
458
+ this.setColumnValue("retentionDate", v);
459
+ }
410
460
  }
@@ -1,7 +1,9 @@
1
1
  import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
2
2
  import Route from "../../Types/API/Route";
3
3
  import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
4
- import AnalyticsTableColumn from "../../Types/AnalyticsDatabase/TableColumn";
4
+ import AnalyticsTableColumn, {
5
+ SkipIndexType,
6
+ } from "../../Types/AnalyticsDatabase/TableColumn";
5
7
  import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
6
8
  import { JSONObject } from "../../Types/JSON";
7
9
  import ObjectID from "../../Types/ObjectID";
@@ -83,6 +85,12 @@ export default class Metric extends AnalyticsBaseModel {
83
85
  description: "Type of the service that this telemetry belongs to",
84
86
  required: false,
85
87
  type: TableColumnType.Text,
88
+ skipIndex: {
89
+ name: "idx_service_type",
90
+ type: SkipIndexType.Set,
91
+ params: [5],
92
+ granularity: 4,
93
+ },
86
94
  accessControl: {
87
95
  read: [
88
96
  Permission.ProjectOwner,
@@ -107,6 +115,12 @@ export default class Metric extends AnalyticsBaseModel {
107
115
  description: "Name of the Metric",
108
116
  required: true,
109
117
  type: TableColumnType.Text,
118
+ skipIndex: {
119
+ name: "idx_name",
120
+ type: SkipIndexType.BloomFilter,
121
+ params: [0.01],
122
+ granularity: 1,
123
+ },
110
124
  accessControl: {
111
125
  read: [
112
126
  Permission.ProjectOwner,
@@ -503,6 +517,16 @@ export default class Metric extends AnalyticsBaseModel {
503
517
  },
504
518
  );
505
519
 
520
+ const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
521
+ key: "retentionDate",
522
+ title: "Retention Date",
523
+ description:
524
+ "Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
525
+ required: true,
526
+ type: TableColumnType.Date,
527
+ defaultValue: undefined,
528
+ });
529
+
506
530
  super({
507
531
  tableName: "MetricItem",
508
532
  tableEngine: AnalyticsTableEngine.MergeTree,
@@ -556,11 +580,13 @@ export default class Metric extends AnalyticsBaseModel {
556
580
  maxColumn,
557
581
  bucketCountsColumn,
558
582
  explicitBoundsColumn,
583
+ retentionDateColumn,
559
584
  ],
560
585
  projections: [],
561
586
  sortKeys: ["projectId", "time", "serviceId"],
562
587
  primaryKeys: ["projectId", "time", "serviceId"],
563
588
  partitionKey: "sipHash64(projectId) % 16",
589
+ ttlExpression: "retentionDate DELETE",
564
590
  });
565
591
  }
566
592
 
@@ -727,4 +753,12 @@ export default class Metric extends AnalyticsBaseModel {
727
753
  public set explicitBounds(v: Array<number> | undefined) {
728
754
  this.setColumnValue("explicitBounds", v);
729
755
  }
756
+
757
+ public get retentionDate(): Date | undefined {
758
+ return this.getColumnValue("retentionDate") as Date | undefined;
759
+ }
760
+
761
+ public set retentionDate(v: Date | undefined) {
762
+ this.setColumnValue("retentionDate", v);
763
+ }
730
764
  }
@@ -103,6 +103,16 @@ export default class MonitorLog extends AnalyticsBaseModel {
103
103
  },
104
104
  });
105
105
 
106
+ const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
107
+ key: "retentionDate",
108
+ title: "Retention Date",
109
+ description:
110
+ "Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
111
+ required: true,
112
+ type: TableColumnType.Date,
113
+ defaultValue: undefined,
114
+ });
115
+
106
116
  super({
107
117
  tableName: "MonitorLog",
108
118
  tableEngine: AnalyticsTableEngine.MergeTree,
@@ -140,11 +150,13 @@ export default class MonitorLog extends AnalyticsBaseModel {
140
150
  monitorIdColumn,
141
151
  timeColumn,
142
152
  logBodyColumn,
153
+ retentionDateColumn,
143
154
  ],
144
155
  projections: [],
145
156
  sortKeys: ["projectId", "time", "monitorId"],
146
157
  primaryKeys: ["projectId", "time", "monitorId"],
147
158
  partitionKey: "sipHash64(projectId) % 16",
159
+ ttlExpression: "retentionDate DELETE",
148
160
  });
149
161
  }
150
162
 
@@ -176,4 +188,12 @@ export default class MonitorLog extends AnalyticsBaseModel {
176
188
  public set logBody(v: JSONObject | undefined) {
177
189
  this.setColumnValue("logBody", v);
178
190
  }
191
+
192
+ public get retentionDate(): Date | undefined {
193
+ return this.getColumnValue("retentionDate") as Date | undefined;
194
+ }
195
+
196
+ public set retentionDate(v: Date | undefined) {
197
+ this.setColumnValue("retentionDate", v);
198
+ }
179
199
  }
@@ -1,7 +1,9 @@
1
1
  import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
2
2
  import Route from "../../Types/API/Route";
3
3
  import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
4
- import AnalyticsTableColumn from "../../Types/AnalyticsDatabase/TableColumn";
4
+ import AnalyticsTableColumn, {
5
+ SkipIndexType,
6
+ } from "../../Types/AnalyticsDatabase/TableColumn";
5
7
  import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
6
8
  import { JSONObject } from "../../Types/JSON";
7
9
  import ObjectID from "../../Types/ObjectID";
@@ -212,6 +214,12 @@ export default class Span extends AnalyticsBaseModel {
212
214
  description: "ID of the trace",
213
215
  required: true,
214
216
  type: TableColumnType.Text,
217
+ skipIndex: {
218
+ name: "idx_trace_id",
219
+ type: SkipIndexType.BloomFilter,
220
+ params: [0.01],
221
+ granularity: 1,
222
+ },
215
223
  accessControl: {
216
224
  read: [
217
225
  Permission.ProjectOwner,
@@ -235,6 +243,12 @@ export default class Span extends AnalyticsBaseModel {
235
243
  description: "ID of the span",
236
244
  required: true,
237
245
  type: TableColumnType.Text,
246
+ skipIndex: {
247
+ name: "idx_span_id",
248
+ type: SkipIndexType.BloomFilter,
249
+ params: [0.01],
250
+ granularity: 1,
251
+ },
238
252
  accessControl: {
239
253
  read: [
240
254
  Permission.ProjectOwner,
@@ -400,6 +414,12 @@ export default class Span extends AnalyticsBaseModel {
400
414
  description: "Status Code",
401
415
  required: false,
402
416
  type: TableColumnType.Number,
417
+ skipIndex: {
418
+ name: "idx_status_code",
419
+ type: SkipIndexType.Set,
420
+ params: [5],
421
+ granularity: 4,
422
+ },
403
423
  accessControl: {
404
424
  read: [
405
425
  Permission.ProjectOwner,
@@ -446,6 +466,12 @@ export default class Span extends AnalyticsBaseModel {
446
466
  description: "Name of the span",
447
467
  required: false,
448
468
  type: TableColumnType.Text,
469
+ skipIndex: {
470
+ name: "idx_name",
471
+ type: SkipIndexType.TokenBF,
472
+ params: [10240, 3, 0],
473
+ granularity: 4,
474
+ },
449
475
  accessControl: {
450
476
  read: [
451
477
  Permission.ProjectOwner,
@@ -486,6 +512,16 @@ export default class Span extends AnalyticsBaseModel {
486
512
  },
487
513
  });
488
514
 
515
+ const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
516
+ key: "retentionDate",
517
+ title: "Retention Date",
518
+ description:
519
+ "Date after which this row is eligible for TTL deletion, computed at ingest time as startTime + service.retainTelemetryDataForDays",
520
+ required: true,
521
+ type: TableColumnType.Date,
522
+ defaultValue: undefined,
523
+ });
524
+
489
525
  super({
490
526
  tableName: "SpanItem",
491
527
  tableEngine: AnalyticsTableEngine.MergeTree,
@@ -538,11 +574,13 @@ export default class Span extends AnalyticsBaseModel {
538
574
  statusMessageColumn,
539
575
  nameColumn,
540
576
  kindColumn,
577
+ retentionDateColumn,
541
578
  ],
542
579
  projections: [],
543
580
  sortKeys: ["projectId", "startTime", "serviceId", "traceId"],
544
581
  primaryKeys: ["projectId", "startTime", "serviceId", "traceId"],
545
582
  partitionKey: "sipHash64(projectId) % 16",
583
+ ttlExpression: "retentionDate DELETE",
546
584
  });
547
585
  }
548
586
 
@@ -697,4 +735,12 @@ export default class Span extends AnalyticsBaseModel {
697
735
  public set statusMessage(v: string | undefined) {
698
736
  this.setColumnValue("statusMessage", v);
699
737
  }
738
+
739
+ public get retentionDate(): Date | undefined {
740
+ return this.getColumnValue("retentionDate") as Date | undefined;
741
+ }
742
+
743
+ public set retentionDate(v: Date | undefined) {
744
+ this.setColumnValue("retentionDate", v);
745
+ }
700
746
  }
@@ -605,4 +605,22 @@ export default class GlobalConfig extends GlobalConfigModel {
605
605
  unique: true,
606
606
  })
607
607
  public monitorLogRetentionInDays?: number = undefined;
608
+
609
+ @ColumnAccessControl({
610
+ create: [],
611
+ read: [],
612
+ update: [],
613
+ })
614
+ @TableColumn({
615
+ type: TableColumnType.Number,
616
+ title: "Monitor Metric Retention Days",
617
+ description:
618
+ "Number of days to retain monitor metrics. Monitor metrics older than this will be automatically deleted. Default is 1 day.",
619
+ })
620
+ @Column({
621
+ type: ColumnType.Number,
622
+ nullable: true,
623
+ unique: true,
624
+ })
625
+ public monitorMetricRetentionInDays?: number = undefined;
608
626
  }
@@ -0,0 +1,23 @@
1
+ import { MigrationInterface, QueryRunner } from "typeorm";
2
+
3
+ export class MigrationName1772350000000 implements MigrationInterface {
4
+ public name = "MigrationName1772350000000";
5
+
6
+ public async up(queryRunner: QueryRunner): Promise<void> {
7
+ await queryRunner.query(
8
+ `ALTER TABLE "GlobalConfig" ADD "monitorMetricRetentionInDays" integer`,
9
+ );
10
+ await queryRunner.query(
11
+ `ALTER TABLE "GlobalConfig" ADD CONSTRAINT "UQ_monitor_metric_retention" UNIQUE ("monitorMetricRetentionInDays")`,
12
+ );
13
+ }
14
+
15
+ public async down(queryRunner: QueryRunner): Promise<void> {
16
+ await queryRunner.query(
17
+ `ALTER TABLE "GlobalConfig" DROP CONSTRAINT "UQ_monitor_metric_retention"`,
18
+ );
19
+ await queryRunner.query(
20
+ `ALTER TABLE "GlobalConfig" DROP COLUMN "monitorMetricRetentionInDays"`,
21
+ );
22
+ }
23
+ }
@@ -261,6 +261,7 @@ import { MigrationName1770834237090 } from "./1770834237090-MigrationName";
261
261
  import { MigrationName1770834237091 } from "./1770834237091-MigrationName";
262
262
  import { MigrationName1772111896988 } from "./1772111896988-MigrationName";
263
263
  import { MigrationName1772280000000 } from "./1772280000000-MigrationName";
264
+ import { MigrationName1772350000000 } from "./1772350000000-MigrationName";
264
265
 
265
266
  export default [
266
267
  InitialMigration,
@@ -526,4 +527,5 @@ export default [
526
527
  MigrationName1770834237091,
527
528
  MigrationName1772111896988,
528
529
  MigrationName1772280000000,
530
+ MigrationName1772350000000,
529
531
  ];
@@ -4,6 +4,7 @@ import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
4
4
  import { JSONObject } from "../../Types/JSON";
5
5
  import ObjectID from "../../Types/ObjectID";
6
6
  import BadDataException from "../../Types/Exception/BadDataException";
7
+ import Includes from "../../Types/BaseDatabase/Includes";
7
8
  import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
8
9
  import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
9
10
 
@@ -204,39 +205,43 @@ export class LogAggregationService {
204
205
  >,
205
206
  ): void {
206
207
  if (request.serviceIds && request.serviceIds.length > 0) {
207
- const idStrings: Array<string> = request.serviceIds.map(
208
- (id: ObjectID): string => {
209
- return `'${id.toString()}'`;
210
- },
208
+ statement.append(
209
+ SQL` AND serviceId IN (${{
210
+ type: TableColumnType.ObjectID,
211
+ value: new Includes(
212
+ request.serviceIds.map((id: ObjectID) => {
213
+ return id.toString();
214
+ }),
215
+ ),
216
+ }})`,
211
217
  );
212
- statement.append(` AND serviceId IN (${idStrings.join(",")})`);
213
218
  }
214
219
 
215
220
  if (request.severityTexts && request.severityTexts.length > 0) {
216
- const sevStrings: Array<string> = request.severityTexts.map(
217
- (s: string): string => {
218
- return `'${LogAggregationService.escapeSingleQuotes(s)}'`;
219
- },
221
+ statement.append(
222
+ SQL` AND severityText IN (${{
223
+ type: TableColumnType.Text,
224
+ value: new Includes(request.severityTexts),
225
+ }})`,
220
226
  );
221
- statement.append(` AND severityText IN (${sevStrings.join(",")})`);
222
227
  }
223
228
 
224
229
  if (request.traceIds && request.traceIds.length > 0) {
225
- const traceStrings: Array<string> = request.traceIds.map(
226
- (s: string): string => {
227
- return `'${LogAggregationService.escapeSingleQuotes(s)}'`;
228
- },
230
+ statement.append(
231
+ SQL` AND traceId IN (${{
232
+ type: TableColumnType.Text,
233
+ value: new Includes(request.traceIds),
234
+ }})`,
229
235
  );
230
- statement.append(` AND traceId IN (${traceStrings.join(",")})`);
231
236
  }
232
237
 
233
238
  if (request.spanIds && request.spanIds.length > 0) {
234
- const spanStrings: Array<string> = request.spanIds.map(
235
- (s: string): string => {
236
- return `'${LogAggregationService.escapeSingleQuotes(s)}'`;
237
- },
239
+ statement.append(
240
+ SQL` AND spanId IN (${{
241
+ type: TableColumnType.Text,
242
+ value: new Includes(request.spanIds),
243
+ }})`,
238
244
  );
239
- statement.append(` AND spanId IN (${spanStrings.join(",")})`);
240
245
  }
241
246
 
242
247
  if (request.bodySearchText && request.bodySearchText.trim().length > 0) {
@@ -275,10 +280,6 @@ export class LogAggregationService {
275
280
  throw new BadDataException("Invalid facetKey");
276
281
  }
277
282
  }
278
-
279
- private static escapeSingleQuotes(value: string): string {
280
- return value.replace(/'/g, "\\'");
281
- }
282
283
  }
283
284
 
284
285
  export default LogAggregationService;
@@ -9,6 +9,7 @@ import LessThan from "../../../Types/BaseDatabase/LessThan";
9
9
  import LessThanOrEqual from "../../../Types/BaseDatabase/LessThanOrEqual";
10
10
  import LessThanOrNull from "../../../Types/BaseDatabase/LessThanOrNull";
11
11
  import GreaterThanOrNull from "../../../Types/BaseDatabase/GreaterThanOrNull";
12
+ import NotEqual from "../../../Types/BaseDatabase/NotEqual";
12
13
  import Search from "../../../Types/BaseDatabase/Search";
13
14
  import OneUptimeDate from "../../../Types/Date";
14
15
  import Dictionary from "../../../Types/Dictionary";
@@ -103,7 +104,8 @@ export class Statement implements BaseQueryParams {
103
104
  v.value instanceof GreaterThan ||
104
105
  v.value instanceof GreaterThanOrEqual ||
105
106
  v.value instanceof LessThanOrNull ||
106
- v.value instanceof GreaterThanOrNull
107
+ v.value instanceof GreaterThanOrNull ||
108
+ v.value instanceof NotEqual
107
109
  ) {
108
110
  finalValue = v.value.value;
109
111
  } else if (v.value instanceof Includes) {