@oneuptime/common 10.0.30 → 10.0.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/Models/AnalyticsModels/ExceptionInstance.ts +29 -4
  2. package/Models/AnalyticsModels/Log.ts +110 -4
  3. package/Models/AnalyticsModels/Metric.ts +16 -9
  4. package/Models/AnalyticsModels/MonitorLog.ts +4 -2
  5. package/Models/AnalyticsModels/Span.ts +79 -6
  6. package/Models/DatabaseModels/Index.ts +8 -0
  7. package/Models/DatabaseModels/LogDropFilter.ts +480 -0
  8. package/Models/DatabaseModels/LogPipeline.ts +412 -0
  9. package/Models/DatabaseModels/LogPipelineProcessor.ts +430 -0
  10. package/Models/DatabaseModels/LogScrubRule.ts +516 -0
  11. package/Server/API/TelemetryAPI.ts +261 -0
  12. package/Server/Infrastructure/Postgres/SchemaMigrations/1773402621107-MigrationName.ts +131 -0
  13. package/Server/Infrastructure/Postgres/SchemaMigrations/1773414578773-MigrationName.ts +79 -0
  14. package/Server/Infrastructure/Postgres/SchemaMigrations/1773500000000-MigrationName.ts +41 -0
  15. package/Server/Infrastructure/Postgres/SchemaMigrations/1773676206197-MigrationName.ts +57 -0
  16. package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
  17. package/Server/Services/AnalyticsDatabaseService.ts +61 -0
  18. package/Server/Services/LogAggregationService.ts +238 -1
  19. package/Server/Services/LogDropFilterService.ts +10 -0
  20. package/Server/Services/LogPipelineProcessorService.ts +10 -0
  21. package/Server/Services/LogPipelineService.ts +10 -0
  22. package/Server/Services/LogScrubRuleService.ts +10 -0
  23. package/Server/Services/TelemetryAttributeService.ts +4 -6
  24. package/Server/Utils/AnalyticsDatabase/Statement.ts +15 -1
  25. package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +126 -11
  26. package/Tests/Server/Services/LogAggregationService.test.ts +3 -2
  27. package/Types/AnalyticsDatabase/AnalyticsTableName.ts +9 -0
  28. package/Types/AnalyticsDatabase/TableColumnType.ts +4 -0
  29. package/Types/Date.ts +22 -0
  30. package/Types/Log/LogDropFilterAction.ts +6 -0
  31. package/Types/Log/LogPipelineProcessorType.ts +44 -0
  32. package/Types/Log/LogScrubAction.ts +7 -0
  33. package/Types/Log/LogScrubPatternType.ts +10 -0
  34. package/Types/Permission.ts +174 -0
  35. package/UI/Components/LogsViewer/LogsViewer.tsx +152 -4
  36. package/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.tsx +92 -0
  37. package/UI/Components/LogsViewer/components/LogDetailsPanel.tsx +332 -117
  38. package/UI/Components/LogsViewer/components/LogSearchBar.tsx +294 -274
  39. package/UI/Components/LogsViewer/components/LogsAnalyticsView.tsx +513 -234
  40. package/UI/Components/LogsViewer/components/LogsFilterCard.tsx +37 -29
  41. package/UI/Components/LogsViewer/components/LogsTable.tsx +6 -1
  42. package/UI/Components/LogsViewer/components/LogsViewerToolbar.tsx +106 -0
  43. package/UI/Utils/LogExport.ts +160 -0
  44. package/build/dist/Models/AnalyticsModels/ExceptionInstance.js +28 -4
  45. package/build/dist/Models/AnalyticsModels/ExceptionInstance.js.map +1 -1
  46. package/build/dist/Models/AnalyticsModels/Log.js +97 -4
  47. package/build/dist/Models/AnalyticsModels/Log.js.map +1 -1
  48. package/build/dist/Models/AnalyticsModels/Metric.js +16 -9
  49. package/build/dist/Models/AnalyticsModels/Metric.js.map +1 -1
  50. package/build/dist/Models/AnalyticsModels/MonitorLog.js +4 -2
  51. package/build/dist/Models/AnalyticsModels/MonitorLog.js.map +1 -1
  52. package/build/dist/Models/AnalyticsModels/Span.js +73 -6
  53. package/build/dist/Models/AnalyticsModels/Span.js.map +1 -1
  54. package/build/dist/Models/DatabaseModels/Index.js +8 -0
  55. package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
  56. package/build/dist/Models/DatabaseModels/LogDropFilter.js +508 -0
  57. package/build/dist/Models/DatabaseModels/LogDropFilter.js.map +1 -0
  58. package/build/dist/Models/DatabaseModels/LogPipeline.js +438 -0
  59. package/build/dist/Models/DatabaseModels/LogPipeline.js.map +1 -0
  60. package/build/dist/Models/DatabaseModels/LogPipelineProcessor.js +452 -0
  61. package/build/dist/Models/DatabaseModels/LogPipelineProcessor.js.map +1 -0
  62. package/build/dist/Models/DatabaseModels/LogScrubRule.js +545 -0
  63. package/build/dist/Models/DatabaseModels/LogScrubRule.js.map +1 -0
  64. package/build/dist/Server/API/TelemetryAPI.js +155 -0
  65. package/build/dist/Server/API/TelemetryAPI.js.map +1 -1
  66. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773402621107-MigrationName.js +52 -0
  67. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773402621107-MigrationName.js.map +1 -0
  68. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773414578773-MigrationName.js +34 -0
  69. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773414578773-MigrationName.js.map +1 -0
  70. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773500000000-MigrationName.js +22 -0
  71. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773500000000-MigrationName.js.map +1 -0
  72. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773676206197-MigrationName.js +26 -0
  73. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773676206197-MigrationName.js.map +1 -0
  74. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
  75. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
  76. package/build/dist/Server/Services/AnalyticsDatabaseService.js +30 -0
  77. package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
  78. package/build/dist/Server/Services/LogAggregationService.js +188 -1
  79. package/build/dist/Server/Services/LogAggregationService.js.map +1 -1
  80. package/build/dist/Server/Services/LogDropFilterService.js +9 -0
  81. package/build/dist/Server/Services/LogDropFilterService.js.map +1 -0
  82. package/build/dist/Server/Services/LogPipelineProcessorService.js +9 -0
  83. package/build/dist/Server/Services/LogPipelineProcessorService.js.map +1 -0
  84. package/build/dist/Server/Services/LogPipelineService.js +9 -0
  85. package/build/dist/Server/Services/LogPipelineService.js.map +1 -0
  86. package/build/dist/Server/Services/LogScrubRuleService.js +9 -0
  87. package/build/dist/Server/Services/LogScrubRuleService.js.map +1 -0
  88. package/build/dist/Server/Services/TelemetryAttributeService.js +4 -6
  89. package/build/dist/Server/Services/TelemetryAttributeService.js.map +1 -1
  90. package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js +13 -1
  91. package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js.map +1 -1
  92. package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +89 -2
  93. package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
  94. package/build/dist/Tests/Server/Services/LogAggregationService.test.js +3 -2
  95. package/build/dist/Tests/Server/Services/LogAggregationService.test.js.map +1 -1
  96. package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js +10 -0
  97. package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js.map +1 -0
  98. package/build/dist/Types/AnalyticsDatabase/TableColumnType.js +4 -0
  99. package/build/dist/Types/AnalyticsDatabase/TableColumnType.js.map +1 -1
  100. package/build/dist/Types/Date.js +16 -0
  101. package/build/dist/Types/Date.js.map +1 -1
  102. package/build/dist/Types/Log/LogDropFilterAction.js +7 -0
  103. package/build/dist/Types/Log/LogDropFilterAction.js.map +1 -0
  104. package/build/dist/Types/Log/LogPipelineProcessorType.js +9 -0
  105. package/build/dist/Types/Log/LogPipelineProcessorType.js.map +1 -0
  106. package/build/dist/Types/Log/LogScrubAction.js +8 -0
  107. package/build/dist/Types/Log/LogScrubAction.js.map +1 -0
  108. package/build/dist/Types/Log/LogScrubPatternType.js +11 -0
  109. package/build/dist/Types/Log/LogScrubPatternType.js.map +1 -0
  110. package/build/dist/Types/Permission.js +152 -0
  111. package/build/dist/Types/Permission.js.map +1 -1
  112. package/build/dist/UI/Components/LogsViewer/LogsViewer.js +124 -11
  113. package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
  114. package/build/dist/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.js +36 -0
  115. package/build/dist/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.js.map +1 -0
  116. package/build/dist/UI/Components/LogsViewer/components/LogDetailsPanel.js +114 -4
  117. package/build/dist/UI/Components/LogsViewer/components/LogDetailsPanel.js.map +1 -1
  118. package/build/dist/UI/Components/LogsViewer/components/LogSearchBar.js +17 -5
  119. package/build/dist/UI/Components/LogsViewer/components/LogSearchBar.js.map +1 -1
  120. package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js +229 -122
  121. package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js.map +1 -1
  122. package/build/dist/UI/Components/LogsViewer/components/LogsFilterCard.js +5 -4
  123. package/build/dist/UI/Components/LogsViewer/components/LogsFilterCard.js.map +1 -1
  124. package/build/dist/UI/Components/LogsViewer/components/LogsTable.js +4 -1
  125. package/build/dist/UI/Components/LogsViewer/components/LogsTable.js.map +1 -1
  126. package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js +28 -0
  127. package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js.map +1 -1
  128. package/build/dist/UI/Utils/LogExport.js +129 -0
  129. package/build/dist/UI/Utils/LogExport.js.map +1 -0
  130. package/package.json +1 -1
@@ -5,6 +5,7 @@ import { JSONObject } from "../../Types/JSON";
5
5
  import ObjectID from "../../Types/ObjectID";
6
6
  import BadDataException from "../../Types/Exception/BadDataException";
7
7
  import Includes from "../../Types/BaseDatabase/Includes";
8
+ import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
8
9
  import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
9
10
  import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
10
11
 
@@ -82,7 +83,7 @@ export interface AnalyticsTableRow {
82
83
 
83
84
  export class LogAggregationService {
84
85
  private static readonly DEFAULT_FACET_LIMIT: number = 500;
85
- private static readonly TABLE_NAME: string = "LogItem";
86
+ private static readonly TABLE_NAME: string = AnalyticsTableName.Log;
86
87
  private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
87
88
  "severityText",
88
89
  "serviceId",
@@ -641,6 +642,242 @@ export class LogAggregationService {
641
642
  }
642
643
  }
643
644
 
645
+ @CaptureSpan()
646
+ public static async getExportLogs(request: {
647
+ projectId: ObjectID;
648
+ startTime: Date;
649
+ endTime: Date;
650
+ limit: number;
651
+ serviceIds?: Array<ObjectID> | undefined;
652
+ severityTexts?: Array<string> | undefined;
653
+ bodySearchText?: string | undefined;
654
+ traceIds?: Array<string> | undefined;
655
+ spanIds?: Array<string> | undefined;
656
+ }): Promise<Array<JSONObject>> {
657
+ const maxLimit: number = Math.min(request.limit || 10000, 10000);
658
+
659
+ const statement: Statement = SQL`
660
+ SELECT
661
+ time,
662
+ serviceId,
663
+ severityText,
664
+ severityNumber,
665
+ body,
666
+ traceId,
667
+ spanId,
668
+ attributes
669
+ FROM ${LogAggregationService.TABLE_NAME}
670
+ WHERE projectId = ${{
671
+ type: TableColumnType.ObjectID,
672
+ value: request.projectId,
673
+ }}
674
+ AND time >= ${{
675
+ type: TableColumnType.Date,
676
+ value: request.startTime,
677
+ }}
678
+ AND time <= ${{
679
+ type: TableColumnType.Date,
680
+ value: request.endTime,
681
+ }}
682
+ `;
683
+
684
+ LogAggregationService.appendCommonFilters(statement, request);
685
+
686
+ statement.append(
687
+ SQL` ORDER BY time DESC LIMIT ${{
688
+ type: TableColumnType.Number,
689
+ value: maxLimit,
690
+ }}`,
691
+ );
692
+
693
+ const dbResult: Results = await LogDatabaseService.executeQuery(statement);
694
+ const response: DbJSONResponse = await dbResult.json<{
695
+ data?: Array<JSONObject>;
696
+ }>();
697
+
698
+ return response.data || [];
699
+ }
700
+
701
+ @CaptureSpan()
702
+ public static async getLogContext(request: {
703
+ projectId: ObjectID;
704
+ serviceId: ObjectID;
705
+ time: Date;
706
+ logId: string;
707
+ count: number;
708
+ }): Promise<{ before: Array<JSONObject>; after: Array<JSONObject> }> {
709
+ const count: number = Math.min(request.count || 5, 20);
710
+
711
+ const beforeStatement: Statement = SQL`
712
+ SELECT
713
+ _id,
714
+ time,
715
+ timeUnixNano,
716
+ serviceId,
717
+ severityText,
718
+ severityNumber,
719
+ body,
720
+ traceId,
721
+ spanId,
722
+ attributes
723
+ FROM ${LogAggregationService.TABLE_NAME}
724
+ WHERE projectId = ${{
725
+ type: TableColumnType.ObjectID,
726
+ value: request.projectId,
727
+ }}
728
+ AND serviceId = ${{
729
+ type: TableColumnType.ObjectID,
730
+ value: request.serviceId,
731
+ }}
732
+ AND time <= ${{
733
+ type: TableColumnType.Date,
734
+ value: request.time,
735
+ }}
736
+ AND _id != ${{
737
+ type: TableColumnType.Text,
738
+ value: request.logId,
739
+ }}
740
+ ORDER BY time DESC, timeUnixNano DESC
741
+ LIMIT ${{
742
+ type: TableColumnType.Number,
743
+ value: count,
744
+ }}
745
+ `;
746
+
747
+ const afterStatement: Statement = SQL`
748
+ SELECT
749
+ _id,
750
+ time,
751
+ timeUnixNano,
752
+ serviceId,
753
+ severityText,
754
+ severityNumber,
755
+ body,
756
+ traceId,
757
+ spanId,
758
+ attributes
759
+ FROM ${LogAggregationService.TABLE_NAME}
760
+ WHERE projectId = ${{
761
+ type: TableColumnType.ObjectID,
762
+ value: request.projectId,
763
+ }}
764
+ AND serviceId = ${{
765
+ type: TableColumnType.ObjectID,
766
+ value: request.serviceId,
767
+ }}
768
+ AND time >= ${{
769
+ type: TableColumnType.Date,
770
+ value: request.time,
771
+ }}
772
+ AND _id != ${{
773
+ type: TableColumnType.Text,
774
+ value: request.logId,
775
+ }}
776
+ ORDER BY time ASC, timeUnixNano ASC
777
+ LIMIT ${{
778
+ type: TableColumnType.Number,
779
+ value: count,
780
+ }}
781
+ `;
782
+
783
+ const [beforeResult, afterResult] = await Promise.all([
784
+ LogDatabaseService.executeQuery(beforeStatement),
785
+ LogDatabaseService.executeQuery(afterStatement),
786
+ ]);
787
+
788
+ const beforeResponse: DbJSONResponse = await beforeResult.json<{
789
+ data?: Array<JSONObject>;
790
+ }>();
791
+ const afterResponse: DbJSONResponse = await afterResult.json<{
792
+ data?: Array<JSONObject>;
793
+ }>();
794
+
795
+ const beforeRows: Array<JSONObject> = (beforeResponse.data || []).reverse();
796
+ const afterRows: Array<JSONObject> = afterResponse.data || [];
797
+
798
+ return { before: beforeRows, after: afterRows };
799
+ }
800
+
801
+ @CaptureSpan()
802
+ public static async getDropFilterEstimate(request: {
803
+ projectId: ObjectID;
804
+ startTime: Date;
805
+ endTime: Date;
806
+ filterQuery: string;
807
+ serviceIds?: Array<ObjectID> | undefined;
808
+ severityTexts?: Array<string> | undefined;
809
+ bodySearchText?: string | undefined;
810
+ }): Promise<{
811
+ totalLogs: number;
812
+ matchingLogs: number;
813
+ estimatedReductionPercent: number;
814
+ }> {
815
+ // Get total count
816
+ const totalStatement: Statement = SQL`
817
+ SELECT count() AS cnt
818
+ FROM ${LogAggregationService.TABLE_NAME}
819
+ WHERE projectId = ${{
820
+ type: TableColumnType.ObjectID,
821
+ value: request.projectId,
822
+ }}
823
+ AND time >= ${{
824
+ type: TableColumnType.Date,
825
+ value: request.startTime,
826
+ }}
827
+ AND time <= ${{
828
+ type: TableColumnType.Date,
829
+ value: request.endTime,
830
+ }}
831
+ `;
832
+
833
+ LogAggregationService.appendCommonFilters(totalStatement, request);
834
+
835
+ // Get matching count using the filter query as body search
836
+ const matchStatement: Statement = SQL`
837
+ SELECT count() AS cnt
838
+ FROM ${LogAggregationService.TABLE_NAME}
839
+ WHERE projectId = ${{
840
+ type: TableColumnType.ObjectID,
841
+ value: request.projectId,
842
+ }}
843
+ AND time >= ${{
844
+ type: TableColumnType.Date,
845
+ value: request.startTime,
846
+ }}
847
+ AND time <= ${{
848
+ type: TableColumnType.Date,
849
+ value: request.endTime,
850
+ }}
851
+ `;
852
+
853
+ LogAggregationService.appendCommonFilters(matchStatement, {
854
+ ...request,
855
+ bodySearchText: request.filterQuery,
856
+ });
857
+
858
+ const [totalResult, matchResult] = await Promise.all([
859
+ LogDatabaseService.executeQuery(totalStatement),
860
+ LogDatabaseService.executeQuery(matchStatement),
861
+ ]);
862
+
863
+ const totalResponse: DbJSONResponse = await totalResult.json<{
864
+ data?: Array<JSONObject>;
865
+ }>();
866
+ const matchResponse: DbJSONResponse = await matchResult.json<{
867
+ data?: Array<JSONObject>;
868
+ }>();
869
+
870
+ const totalData: Array<JSONObject> = totalResponse.data || [];
871
+ const matchData: Array<JSONObject> = matchResponse.data || [];
872
+
873
+ const totalLogs: number = Number(totalData[0]?.["cnt"] || 0);
874
+ const matchingLogs: number = Number(matchData[0]?.["cnt"] || 0);
875
+ const estimatedReductionPercent: number =
876
+ totalLogs > 0 ? Math.round((matchingLogs / totalLogs) * 100) : 0;
877
+
878
+ return { totalLogs, matchingLogs, estimatedReductionPercent };
879
+ }
880
+
644
881
  private static isTopLevelColumn(key: string): boolean {
645
882
  return LogAggregationService.TOP_LEVEL_COLUMNS.has(key);
646
883
  }
@@ -0,0 +1,10 @@
1
+ import DatabaseService from "./DatabaseService";
2
+ import Model from "../../Models/DatabaseModels/LogDropFilter";
3
+
4
+ export class Service extends DatabaseService<Model> {
5
+ public constructor() {
6
+ super(Model);
7
+ }
8
+ }
9
+
10
+ export default new Service();
@@ -0,0 +1,10 @@
1
+ import DatabaseService from "./DatabaseService";
2
+ import Model from "../../Models/DatabaseModels/LogPipelineProcessor";
3
+
4
+ export class Service extends DatabaseService<Model> {
5
+ public constructor() {
6
+ super(Model);
7
+ }
8
+ }
9
+
10
+ export default new Service();
@@ -0,0 +1,10 @@
1
+ import DatabaseService from "./DatabaseService";
2
+ import Model from "../../Models/DatabaseModels/LogPipeline";
3
+
4
+ export class Service extends DatabaseService<Model> {
5
+ public constructor() {
6
+ super(Model);
7
+ }
8
+ }
9
+
10
+ export default new Service();
@@ -0,0 +1,10 @@
1
+ import DatabaseService from "./DatabaseService";
2
+ import Model from "../../Models/DatabaseModels/LogScrubRule";
3
+
4
+ export class Service extends DatabaseService<Model> {
5
+ public constructor() {
6
+ super(Model);
7
+ }
8
+ }
9
+
10
+ export default new Service();
@@ -227,8 +227,8 @@ export class TelemetryAttributeService {
227
227
  WITH filtered AS (
228
228
  SELECT arrayJoin(
229
229
  if(
230
- ${data.attributeKeysColumn} IS NULL OR empty(${data.attributeKeysColumn}),
231
- JSONExtractKeys(${data.attributesColumn}),
230
+ empty(${data.attributeKeysColumn}),
231
+ mapKeys(${data.attributesColumn}),
232
232
  ${data.attributeKeysColumn}
233
233
  )
234
234
  ) AS attribute
@@ -238,10 +238,8 @@ export class TelemetryAttributeService {
238
238
  value: data.projectId,
239
239
  }}
240
240
  AND (
241
- ${data.attributeKeysColumn} IS NOT NULL OR (
242
- ${data.attributesColumn} IS NOT NULL AND
243
- ${data.attributesColumn} != ''
244
- )
241
+ NOT empty(${data.attributeKeysColumn}) OR
242
+ NOT empty(${data.attributesColumn})
245
243
  )
246
244
  AND ${data.timeColumn} >= ${{
247
245
  type: TableColumnType.Date,
@@ -120,7 +120,11 @@ export class Statement implements BaseQueryParams {
120
120
  finalValue = v.value.values;
121
121
  }
122
122
  } else if (v.value instanceof Date) {
123
- finalValue = OneUptimeDate.toClickhouseDateTime(v.value);
123
+ if (typeof v !== "string" && v.type === TableColumnType.DateTime64) {
124
+ finalValue = OneUptimeDate.toClickhouseDateTime64(v.value);
125
+ } else {
126
+ finalValue = OneUptimeDate.toClickhouseDateTime(v.value);
127
+ }
124
128
  } else {
125
129
  finalValue = v.value;
126
130
  }
@@ -136,6 +140,15 @@ export class Statement implements BaseQueryParams {
136
140
  finalValue = OneUptimeDate.toClickhouseDateTime(finalValue);
137
141
  }
138
142
 
143
+ if (
144
+ typeof v !== "string" &&
145
+ v.type === TableColumnType.DateTime64 &&
146
+ !(v.value instanceof Date)
147
+ ) {
148
+ finalValue = OneUptimeDate.fromString(finalValue as string);
149
+ finalValue = OneUptimeDate.toClickhouseDateTime64(finalValue);
150
+ }
151
+
139
152
  return finalValue;
140
153
  }
141
154
 
@@ -176,6 +189,7 @@ export class Statement implements BaseQueryParams {
176
189
  [TableColumnType.Number]: "Int32",
177
190
  [TableColumnType.Decimal]: "Double",
178
191
  [TableColumnType.Date]: "DateTime",
192
+ [TableColumnType.DateTime64]: "DateTime64(9)",
179
193
  [TableColumnType.JSON]: "JSON",
180
194
  [TableColumnType.ArrayNumber]: "Array(Int32)",
181
195
  [TableColumnType.ArrayText]: "Array(String)",
@@ -8,7 +8,7 @@ import logger from "../Logger";
8
8
  import { SQL, Statement } from "./Statement";
9
9
  import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
10
10
  import CommonModel, {
11
- Record,
11
+ Record as AnalyticsRecord,
12
12
  RecordValue,
13
13
  } from "../../../Models/AnalyticsModels/AnalyticsBaseModel/CommonModel";
14
14
  import AnalyticsTableColumn, {
@@ -85,7 +85,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
85
85
  return columnNames;
86
86
  }
87
87
 
88
- public getRecordValuesStatement(record: Record): string {
88
+ public getRecordValuesStatement(record: AnalyticsRecord): string {
89
89
  let valueStatement: string = "";
90
90
 
91
91
  for (const value of record) {
@@ -106,7 +106,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
106
106
  return valueStatement;
107
107
  }
108
108
 
109
- public getValuesStatement(records: Array<Record>): string {
109
+ public getValuesStatement(records: Array<AnalyticsRecord>): string {
110
110
  let statement: string = "";
111
111
  for (const record of records) {
112
112
  statement += `(${this.getRecordValuesStatement(record)}), `;
@@ -126,10 +126,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
126
126
  this.model.getTableColumns(),
127
127
  );
128
128
 
129
- const records: Array<Record> = [];
129
+ const records: Array<AnalyticsRecord> = [];
130
130
 
131
131
  for (const item of data.item) {
132
- const record: Record = this.getRecord(item);
132
+ const record: AnalyticsRecord = this.getRecord(item);
133
133
  records.push(record);
134
134
  }
135
135
 
@@ -154,8 +154,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
154
154
  return statement;
155
155
  }
156
156
 
157
- private getRecord(item: CommonModel): Record {
158
- const record: Record = [];
157
+ private getRecord(item: CommonModel): AnalyticsRecord {
158
+ const record: AnalyticsRecord = [];
159
159
 
160
160
  for (const column of item.getTableColumns()) {
161
161
  const value: RecordValue | undefined = this.sanitizeValue(
@@ -251,6 +251,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
251
251
  value = `CAST(${this.escapeStringLiteral(value.toString())} AS Int128)`;
252
252
  }
253
253
 
254
+ if (column.type === TableColumnType.BigNumber) {
255
+ if (typeof value === "string") {
256
+ value = parseInt(value);
257
+ }
258
+ }
259
+
260
+ if (column.type === TableColumnType.ArrayBigNumber) {
261
+ value = `[${(value as Array<number>)
262
+ .map((v: number) => {
263
+ if (v && typeof v !== "number") {
264
+ v = parseFloat(v);
265
+ return isNaN(v) ? "NULL" : v;
266
+ }
267
+ return v;
268
+ })
269
+ .join(", ")}]`;
270
+ }
271
+
272
+ if (column.type === TableColumnType.MapStringString) {
273
+ const mapObj: Record<string, string> = value as Record<string, string>;
274
+ const entries: Array<string> = Object.entries(mapObj).map(
275
+ ([k, v]: [string, string]) => {
276
+ return `${this.escapeStringLiteral(k)}, ${this.escapeStringLiteral(v)}`;
277
+ },
278
+ );
279
+ value = `map(${entries.join(", ")})`;
280
+ }
281
+
254
282
  return value;
255
283
  }
256
284
 
@@ -387,6 +415,28 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
387
415
  } else {
388
416
  whereStatement.append(SQL`AND ${key} IS NULL`);
389
417
  }
418
+ } else if (
419
+ tableColumn.type === TableColumnType.MapStringString &&
420
+ typeof value === "object"
421
+ ) {
422
+ const mapValue: Record<string, string> = value as Record<
423
+ string,
424
+ string
425
+ >;
426
+ for (const mapKey in mapValue) {
427
+ if (mapValue[mapKey] === undefined) {
428
+ continue;
429
+ }
430
+ whereStatement.append(
431
+ SQL`AND ${key}[${{
432
+ value: mapKey,
433
+ type: TableColumnType.Text,
434
+ }}] = ${{
435
+ value: mapValue[mapKey] as string,
436
+ type: TableColumnType.Text,
437
+ }}`,
438
+ );
439
+ }
390
440
  } else if (
391
441
  (tableColumn.type === TableColumnType.JSON ||
392
442
  tableColumn.type === TableColumnType.JSONArray) &&
@@ -640,6 +690,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
640
690
  );
641
691
  }
642
692
 
693
+ // Append projections after indexes
694
+ if (this.model.projections && this.model.projections.length > 0) {
695
+ for (const projection of this.model.projections) {
696
+ columns.append(`, PROJECTION ${projection.name} (${projection.query})`);
697
+ }
698
+ }
699
+
643
700
  return columns;
644
701
  }
645
702
 
@@ -649,13 +706,16 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
649
706
  return {
650
707
  String: TableColumnType.Text,
651
708
  Int32: TableColumnType.Number,
652
- Int64: TableColumnType.LongNumber,
709
+ Int64: TableColumnType.BigNumber,
653
710
  Int128: TableColumnType.LongNumber,
654
711
  Float32: TableColumnType.Decimal,
655
712
  Float64: TableColumnType.Decimal,
656
713
  DateTime: TableColumnType.Date,
714
+ "DateTime64(9)": TableColumnType.DateTime64,
657
715
  "Array(String)": TableColumnType.ArrayText,
658
716
  "Array(Int32)": TableColumnType.ArrayNumber,
717
+ "Array(Int64)": TableColumnType.ArrayBigNumber,
718
+ "Map(String, String)": TableColumnType.MapStringString,
659
719
  JSON: TableColumnType.JSON, //JSONArray is also JSON
660
720
  Bool: TableColumnType.Boolean,
661
721
  }[clickhouseType];
@@ -671,11 +731,15 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
671
731
  [TableColumnType.IP]: SQL`String`,
672
732
  [TableColumnType.Port]: SQL`String`,
673
733
  [TableColumnType.Date]: SQL`DateTime`,
734
+ [TableColumnType.DateTime64]: SQL`DateTime64(9)`,
674
735
  [TableColumnType.JSON]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
675
736
  [TableColumnType.JSONArray]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
676
737
  [TableColumnType.ArrayNumber]: SQL`Array(Int32)`,
738
+ [TableColumnType.ArrayBigNumber]: SQL`Array(Int64)`,
677
739
  [TableColumnType.ArrayText]: SQL`Array(String)`,
678
740
  [TableColumnType.LongNumber]: SQL`Int128`,
741
+ [TableColumnType.BigNumber]: SQL`Int64`,
742
+ [TableColumnType.MapStringString]: SQL`Map(String, String)`,
679
743
  }[type];
680
744
 
681
745
  if (!statement) {
@@ -700,12 +764,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
700
764
  }
701
765
 
702
766
  public toAddColumnStatement(column: AnalyticsTableColumn): Statement {
767
+ // Build column definition without skip index (indexes must be added separately via ADD INDEX)
768
+ const columnDef: Statement = new Statement();
769
+
770
+ columnDef
771
+ .append(column.key)
772
+ .append(SQL` `)
773
+ .append(
774
+ column.required
775
+ ? this.toColumnType(column.type)
776
+ : SQL`Nullable(`
777
+ .append(this.toColumnType(column.type))
778
+ .append(SQL`)`),
779
+ );
780
+
781
+ if (column.codec) {
782
+ const codecStr: string =
783
+ column.codec.level !== undefined
784
+ ? `${column.codec.codec}(${column.codec.level})`
785
+ : column.codec.codec;
786
+ columnDef.append(` CODEC(${codecStr})`);
787
+ }
788
+
703
789
  const statement: Statement = SQL`
704
790
  ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
705
791
  this.model.tableName
706
- } ADD COLUMN IF NOT EXISTS `.append(
707
- this.toColumnsCreateStatement([column]),
708
- );
792
+ } ADD COLUMN IF NOT EXISTS `.append(columnDef);
709
793
 
710
794
  logger.debug(`${this.model.tableName} Add Column Statement`);
711
795
  logger.debug(statement);
@@ -713,6 +797,37 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
713
797
  return statement;
714
798
  }
715
799
 
800
+ public toAddSkipIndexStatement(
801
+ column: AnalyticsTableColumn,
802
+ ): Statement | null {
803
+ if (!column.skipIndex) {
804
+ return null;
805
+ }
806
+
807
+ const idx: AnalyticsTableColumn["skipIndex"] = column.skipIndex;
808
+ const paramsStr: string =
809
+ idx.params && idx.params.length > 0 ? `(${idx.params.join(", ")})` : "";
810
+
811
+ const needsAssumeNotNull: boolean =
812
+ !column.required &&
813
+ (idx.type === SkipIndexType.TokenBF ||
814
+ idx.type === SkipIndexType.NgramBF);
815
+ const columnExpr: string = needsAssumeNotNull
816
+ ? `assumeNotNull(${column.key})`
817
+ : column.key;
818
+
819
+ const databaseName: string = this.database.getDatasourceOptions().database!;
820
+ const statement: Statement = new Statement();
821
+ statement.append(
822
+ `ALTER TABLE ${databaseName}.${this.model.tableName} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
823
+ );
824
+
825
+ logger.debug(`${this.model.tableName} Add Skip Index Statement`);
826
+ logger.debug(statement);
827
+
828
+ return statement;
829
+ }
830
+
716
831
  public toDropColumnStatement(columnName: string): string {
717
832
  const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
718
833
  .database!}.${this.model.tableName} DROP COLUMN IF EXISTS ${columnName}`;
@@ -2,6 +2,7 @@ import LogAggregationService, {
2
2
  FacetRequest,
3
3
  } from "../../../Server/Services/LogAggregationService";
4
4
  import { Statement } from "../../../Server/Utils/AnalyticsDatabase/Statement";
5
+ import AnalyticsTableName from "../../../Types/AnalyticsDatabase/AnalyticsTableName";
5
6
  import ObjectID from "../../../Types/ObjectID";
6
7
  import OneUptimeDate from "../../../Types/Date";
7
8
  import { describe, expect, test } from "@jest/globals";
@@ -35,7 +36,7 @@ describe("LogAggregationService", () => {
35
36
 
36
37
  expect(statement.query_params).toStrictEqual({
37
38
  p0: "severityText",
38
- p1: "LogItem",
39
+ p1: AnalyticsTableName.Log,
39
40
  p2: defaultRequest.projectId.toString(),
40
41
  p3: OneUptimeDate.toClickhouseDateTime(defaultRequest.startTime),
41
42
  p4: OneUptimeDate.toClickhouseDateTime(defaultRequest.endTime),
@@ -55,7 +56,7 @@ describe("LogAggregationService", () => {
55
56
 
56
57
  expect(statement.query_params).toStrictEqual({
57
58
  p0: facetKey,
58
- p1: "LogItem",
59
+ p1: AnalyticsTableName.Log,
59
60
  p2: defaultRequest.projectId.toString(),
60
61
  p3: OneUptimeDate.toClickhouseDateTime(defaultRequest.startTime),
61
62
  p4: OneUptimeDate.toClickhouseDateTime(defaultRequest.endTime),
@@ -0,0 +1,9 @@
1
+ enum AnalyticsTableName {
2
+ Log = "LogItemV2",
3
+ Metric = "MetricItemV2",
4
+ ExceptionInstance = "ExceptionItemV2",
5
+ Span = "SpanItemV2",
6
+ MonitorLog = "MonitorLogV2",
7
+ }
8
+
9
+ export default AnalyticsTableName;
@@ -10,8 +10,12 @@ enum ColumnType {
10
10
  ArrayNumber = "Array of Numbers",
11
11
  ArrayText = "Array of Text",
12
12
  LongNumber = "Long Number",
13
+ BigNumber = "Big Number",
14
+ DateTime64 = "DateTime64",
13
15
  IP = "IP",
14
16
  Port = "Port",
17
+ MapStringString = "Map(String, String)",
18
+ ArrayBigNumber = "Array of Big Numbers",
15
19
  }
16
20
 
17
21
  export default ColumnType;
package/Types/Date.ts CHANGED
@@ -1524,4 +1524,26 @@ export default class OneUptimeDate {
1524
1524
  const parsedDate: Date = this.fromString(date);
1525
1525
  return moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
1526
1526
  }
1527
+
1528
+ public static toClickhouseDateTime64(
1529
+ date: Date | string,
1530
+ nanoTimestamp?: number,
1531
+ ): string {
1532
+ const parsedDate: Date = this.fromString(date);
1533
+ const base: string = moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
1534
+
1535
+ let nanoFraction: string;
1536
+
1537
+ if (nanoTimestamp !== undefined && nanoTimestamp > 0) {
1538
+ // Extract sub-second nanoseconds from the unix nano timestamp
1539
+ const subSecondNanos: number = nanoTimestamp % 1_000_000_000;
1540
+ nanoFraction = subSecondNanos.toString().padStart(9, "0");
1541
+ } else {
1542
+ // Fall back to milliseconds from the Date object
1543
+ const ms: number = parsedDate.getMilliseconds();
1544
+ nanoFraction = (ms * 1_000_000).toString().padStart(9, "0");
1545
+ }
1546
+
1547
+ return `${base}.${nanoFraction}`;
1548
+ }
1527
1549
  }
@@ -0,0 +1,6 @@
1
+ enum LogDropFilterAction {
2
+ Drop = "drop",
3
+ Sample = "sample",
4
+ }
5
+
6
+ export default LogDropFilterAction;