@oneuptime/common 10.0.30 → 10.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/ExceptionInstance.ts +29 -4
- package/Models/AnalyticsModels/Log.ts +110 -4
- package/Models/AnalyticsModels/Metric.ts +16 -9
- package/Models/AnalyticsModels/MonitorLog.ts +4 -2
- package/Models/AnalyticsModels/Span.ts +79 -6
- package/Models/DatabaseModels/Index.ts +8 -0
- package/Models/DatabaseModels/LogDropFilter.ts +480 -0
- package/Models/DatabaseModels/LogPipeline.ts +412 -0
- package/Models/DatabaseModels/LogPipelineProcessor.ts +430 -0
- package/Models/DatabaseModels/LogScrubRule.ts +516 -0
- package/Server/API/TelemetryAPI.ts +261 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1773402621107-MigrationName.ts +131 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1773414578773-MigrationName.ts +79 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1773500000000-MigrationName.ts +41 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1773676206197-MigrationName.ts +57 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
- package/Server/Services/AnalyticsDatabaseService.ts +61 -0
- package/Server/Services/LogAggregationService.ts +238 -1
- package/Server/Services/LogDropFilterService.ts +10 -0
- package/Server/Services/LogPipelineProcessorService.ts +10 -0
- package/Server/Services/LogPipelineService.ts +10 -0
- package/Server/Services/LogScrubRuleService.ts +10 -0
- package/Server/Services/TelemetryAttributeService.ts +4 -6
- package/Server/Utils/AnalyticsDatabase/Statement.ts +15 -1
- package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +126 -11
- package/Tests/Server/Services/LogAggregationService.test.ts +3 -2
- package/Types/AnalyticsDatabase/AnalyticsTableName.ts +9 -0
- package/Types/AnalyticsDatabase/TableColumnType.ts +4 -0
- package/Types/Date.ts +22 -0
- package/Types/Log/LogDropFilterAction.ts +6 -0
- package/Types/Log/LogPipelineProcessorType.ts +44 -0
- package/Types/Log/LogScrubAction.ts +7 -0
- package/Types/Log/LogScrubPatternType.ts +10 -0
- package/Types/Permission.ts +174 -0
- package/UI/Components/LogsViewer/LogsViewer.tsx +152 -4
- package/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.tsx +92 -0
- package/UI/Components/LogsViewer/components/LogDetailsPanel.tsx +332 -117
- package/UI/Components/LogsViewer/components/LogSearchBar.tsx +294 -274
- package/UI/Components/LogsViewer/components/LogsAnalyticsView.tsx +513 -234
- package/UI/Components/LogsViewer/components/LogsFilterCard.tsx +37 -29
- package/UI/Components/LogsViewer/components/LogsTable.tsx +6 -1
- package/UI/Components/LogsViewer/components/LogsViewerToolbar.tsx +106 -0
- package/UI/Utils/LogExport.ts +160 -0
- package/build/dist/Models/AnalyticsModels/ExceptionInstance.js +28 -4
- package/build/dist/Models/AnalyticsModels/ExceptionInstance.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Log.js +97 -4
- package/build/dist/Models/AnalyticsModels/Log.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Metric.js +16 -9
- package/build/dist/Models/AnalyticsModels/Metric.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/MonitorLog.js +4 -2
- package/build/dist/Models/AnalyticsModels/MonitorLog.js.map +1 -1
- package/build/dist/Models/AnalyticsModels/Span.js +73 -6
- package/build/dist/Models/AnalyticsModels/Span.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Index.js +8 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/LogDropFilter.js +508 -0
- package/build/dist/Models/DatabaseModels/LogDropFilter.js.map +1 -0
- package/build/dist/Models/DatabaseModels/LogPipeline.js +438 -0
- package/build/dist/Models/DatabaseModels/LogPipeline.js.map +1 -0
- package/build/dist/Models/DatabaseModels/LogPipelineProcessor.js +452 -0
- package/build/dist/Models/DatabaseModels/LogPipelineProcessor.js.map +1 -0
- package/build/dist/Models/DatabaseModels/LogScrubRule.js +545 -0
- package/build/dist/Models/DatabaseModels/LogScrubRule.js.map +1 -0
- package/build/dist/Server/API/TelemetryAPI.js +155 -0
- package/build/dist/Server/API/TelemetryAPI.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773402621107-MigrationName.js +52 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773402621107-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773414578773-MigrationName.js +34 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773414578773-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773500000000-MigrationName.js +22 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773500000000-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773676206197-MigrationName.js +26 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773676206197-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +30 -0
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Services/LogAggregationService.js +188 -1
- package/build/dist/Server/Services/LogAggregationService.js.map +1 -1
- package/build/dist/Server/Services/LogDropFilterService.js +9 -0
- package/build/dist/Server/Services/LogDropFilterService.js.map +1 -0
- package/build/dist/Server/Services/LogPipelineProcessorService.js +9 -0
- package/build/dist/Server/Services/LogPipelineProcessorService.js.map +1 -0
- package/build/dist/Server/Services/LogPipelineService.js +9 -0
- package/build/dist/Server/Services/LogPipelineService.js.map +1 -0
- package/build/dist/Server/Services/LogScrubRuleService.js +9 -0
- package/build/dist/Server/Services/LogScrubRuleService.js.map +1 -0
- package/build/dist/Server/Services/TelemetryAttributeService.js +4 -6
- package/build/dist/Server/Services/TelemetryAttributeService.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js +13 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js +89 -2
- package/build/dist/Server/Utils/AnalyticsDatabase/StatementGenerator.js.map +1 -1
- package/build/dist/Tests/Server/Services/LogAggregationService.test.js +3 -2
- package/build/dist/Tests/Server/Services/LogAggregationService.test.js.map +1 -1
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js +10 -0
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js.map +1 -0
- package/build/dist/Types/AnalyticsDatabase/TableColumnType.js +4 -0
- package/build/dist/Types/AnalyticsDatabase/TableColumnType.js.map +1 -1
- package/build/dist/Types/Date.js +16 -0
- package/build/dist/Types/Date.js.map +1 -1
- package/build/dist/Types/Log/LogDropFilterAction.js +7 -0
- package/build/dist/Types/Log/LogDropFilterAction.js.map +1 -0
- package/build/dist/Types/Log/LogPipelineProcessorType.js +9 -0
- package/build/dist/Types/Log/LogPipelineProcessorType.js.map +1 -0
- package/build/dist/Types/Log/LogScrubAction.js +8 -0
- package/build/dist/Types/Log/LogScrubAction.js.map +1 -0
- package/build/dist/Types/Log/LogScrubPatternType.js +11 -0
- package/build/dist/Types/Log/LogScrubPatternType.js.map +1 -0
- package/build/dist/Types/Permission.js +152 -0
- package/build/dist/Types/Permission.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js +124 -11
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.js +36 -0
- package/build/dist/UI/Components/LogsViewer/components/KeyboardShortcutsHelp.js.map +1 -0
- package/build/dist/UI/Components/LogsViewer/components/LogDetailsPanel.js +114 -4
- package/build/dist/UI/Components/LogsViewer/components/LogDetailsPanel.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogSearchBar.js +17 -5
- package/build/dist/UI/Components/LogsViewer/components/LogSearchBar.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js +229 -122
- package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsFilterCard.js +5 -4
- package/build/dist/UI/Components/LogsViewer/components/LogsFilterCard.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsTable.js +4 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsTable.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js +28 -0
- package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js.map +1 -1
- package/build/dist/UI/Utils/LogExport.js +129 -0
- package/build/dist/UI/Utils/LogExport.js.map +1 -0
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import { JSONObject } from "../../Types/JSON";
|
|
|
5
5
|
import ObjectID from "../../Types/ObjectID";
|
|
6
6
|
import BadDataException from "../../Types/Exception/BadDataException";
|
|
7
7
|
import Includes from "../../Types/BaseDatabase/Includes";
|
|
8
|
+
import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
|
|
8
9
|
import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
|
|
9
10
|
import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
|
|
10
11
|
|
|
@@ -82,7 +83,7 @@ export interface AnalyticsTableRow {
|
|
|
82
83
|
|
|
83
84
|
export class LogAggregationService {
|
|
84
85
|
private static readonly DEFAULT_FACET_LIMIT: number = 500;
|
|
85
|
-
private static readonly TABLE_NAME: string =
|
|
86
|
+
private static readonly TABLE_NAME: string = AnalyticsTableName.Log;
|
|
86
87
|
private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
|
|
87
88
|
"severityText",
|
|
88
89
|
"serviceId",
|
|
@@ -641,6 +642,242 @@ export class LogAggregationService {
|
|
|
641
642
|
}
|
|
642
643
|
}
|
|
643
644
|
|
|
645
|
+
@CaptureSpan()
|
|
646
|
+
public static async getExportLogs(request: {
|
|
647
|
+
projectId: ObjectID;
|
|
648
|
+
startTime: Date;
|
|
649
|
+
endTime: Date;
|
|
650
|
+
limit: number;
|
|
651
|
+
serviceIds?: Array<ObjectID> | undefined;
|
|
652
|
+
severityTexts?: Array<string> | undefined;
|
|
653
|
+
bodySearchText?: string | undefined;
|
|
654
|
+
traceIds?: Array<string> | undefined;
|
|
655
|
+
spanIds?: Array<string> | undefined;
|
|
656
|
+
}): Promise<Array<JSONObject>> {
|
|
657
|
+
const maxLimit: number = Math.min(request.limit || 10000, 10000);
|
|
658
|
+
|
|
659
|
+
const statement: Statement = SQL`
|
|
660
|
+
SELECT
|
|
661
|
+
time,
|
|
662
|
+
serviceId,
|
|
663
|
+
severityText,
|
|
664
|
+
severityNumber,
|
|
665
|
+
body,
|
|
666
|
+
traceId,
|
|
667
|
+
spanId,
|
|
668
|
+
attributes
|
|
669
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
670
|
+
WHERE projectId = ${{
|
|
671
|
+
type: TableColumnType.ObjectID,
|
|
672
|
+
value: request.projectId,
|
|
673
|
+
}}
|
|
674
|
+
AND time >= ${{
|
|
675
|
+
type: TableColumnType.Date,
|
|
676
|
+
value: request.startTime,
|
|
677
|
+
}}
|
|
678
|
+
AND time <= ${{
|
|
679
|
+
type: TableColumnType.Date,
|
|
680
|
+
value: request.endTime,
|
|
681
|
+
}}
|
|
682
|
+
`;
|
|
683
|
+
|
|
684
|
+
LogAggregationService.appendCommonFilters(statement, request);
|
|
685
|
+
|
|
686
|
+
statement.append(
|
|
687
|
+
SQL` ORDER BY time DESC LIMIT ${{
|
|
688
|
+
type: TableColumnType.Number,
|
|
689
|
+
value: maxLimit,
|
|
690
|
+
}}`,
|
|
691
|
+
);
|
|
692
|
+
|
|
693
|
+
const dbResult: Results = await LogDatabaseService.executeQuery(statement);
|
|
694
|
+
const response: DbJSONResponse = await dbResult.json<{
|
|
695
|
+
data?: Array<JSONObject>;
|
|
696
|
+
}>();
|
|
697
|
+
|
|
698
|
+
return response.data || [];
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
@CaptureSpan()
|
|
702
|
+
public static async getLogContext(request: {
|
|
703
|
+
projectId: ObjectID;
|
|
704
|
+
serviceId: ObjectID;
|
|
705
|
+
time: Date;
|
|
706
|
+
logId: string;
|
|
707
|
+
count: number;
|
|
708
|
+
}): Promise<{ before: Array<JSONObject>; after: Array<JSONObject> }> {
|
|
709
|
+
const count: number = Math.min(request.count || 5, 20);
|
|
710
|
+
|
|
711
|
+
const beforeStatement: Statement = SQL`
|
|
712
|
+
SELECT
|
|
713
|
+
_id,
|
|
714
|
+
time,
|
|
715
|
+
timeUnixNano,
|
|
716
|
+
serviceId,
|
|
717
|
+
severityText,
|
|
718
|
+
severityNumber,
|
|
719
|
+
body,
|
|
720
|
+
traceId,
|
|
721
|
+
spanId,
|
|
722
|
+
attributes
|
|
723
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
724
|
+
WHERE projectId = ${{
|
|
725
|
+
type: TableColumnType.ObjectID,
|
|
726
|
+
value: request.projectId,
|
|
727
|
+
}}
|
|
728
|
+
AND serviceId = ${{
|
|
729
|
+
type: TableColumnType.ObjectID,
|
|
730
|
+
value: request.serviceId,
|
|
731
|
+
}}
|
|
732
|
+
AND time <= ${{
|
|
733
|
+
type: TableColumnType.Date,
|
|
734
|
+
value: request.time,
|
|
735
|
+
}}
|
|
736
|
+
AND _id != ${{
|
|
737
|
+
type: TableColumnType.Text,
|
|
738
|
+
value: request.logId,
|
|
739
|
+
}}
|
|
740
|
+
ORDER BY time DESC, timeUnixNano DESC
|
|
741
|
+
LIMIT ${{
|
|
742
|
+
type: TableColumnType.Number,
|
|
743
|
+
value: count,
|
|
744
|
+
}}
|
|
745
|
+
`;
|
|
746
|
+
|
|
747
|
+
const afterStatement: Statement = SQL`
|
|
748
|
+
SELECT
|
|
749
|
+
_id,
|
|
750
|
+
time,
|
|
751
|
+
timeUnixNano,
|
|
752
|
+
serviceId,
|
|
753
|
+
severityText,
|
|
754
|
+
severityNumber,
|
|
755
|
+
body,
|
|
756
|
+
traceId,
|
|
757
|
+
spanId,
|
|
758
|
+
attributes
|
|
759
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
760
|
+
WHERE projectId = ${{
|
|
761
|
+
type: TableColumnType.ObjectID,
|
|
762
|
+
value: request.projectId,
|
|
763
|
+
}}
|
|
764
|
+
AND serviceId = ${{
|
|
765
|
+
type: TableColumnType.ObjectID,
|
|
766
|
+
value: request.serviceId,
|
|
767
|
+
}}
|
|
768
|
+
AND time >= ${{
|
|
769
|
+
type: TableColumnType.Date,
|
|
770
|
+
value: request.time,
|
|
771
|
+
}}
|
|
772
|
+
AND _id != ${{
|
|
773
|
+
type: TableColumnType.Text,
|
|
774
|
+
value: request.logId,
|
|
775
|
+
}}
|
|
776
|
+
ORDER BY time ASC, timeUnixNano ASC
|
|
777
|
+
LIMIT ${{
|
|
778
|
+
type: TableColumnType.Number,
|
|
779
|
+
value: count,
|
|
780
|
+
}}
|
|
781
|
+
`;
|
|
782
|
+
|
|
783
|
+
const [beforeResult, afterResult] = await Promise.all([
|
|
784
|
+
LogDatabaseService.executeQuery(beforeStatement),
|
|
785
|
+
LogDatabaseService.executeQuery(afterStatement),
|
|
786
|
+
]);
|
|
787
|
+
|
|
788
|
+
const beforeResponse: DbJSONResponse = await beforeResult.json<{
|
|
789
|
+
data?: Array<JSONObject>;
|
|
790
|
+
}>();
|
|
791
|
+
const afterResponse: DbJSONResponse = await afterResult.json<{
|
|
792
|
+
data?: Array<JSONObject>;
|
|
793
|
+
}>();
|
|
794
|
+
|
|
795
|
+
const beforeRows: Array<JSONObject> = (beforeResponse.data || []).reverse();
|
|
796
|
+
const afterRows: Array<JSONObject> = afterResponse.data || [];
|
|
797
|
+
|
|
798
|
+
return { before: beforeRows, after: afterRows };
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
@CaptureSpan()
|
|
802
|
+
public static async getDropFilterEstimate(request: {
|
|
803
|
+
projectId: ObjectID;
|
|
804
|
+
startTime: Date;
|
|
805
|
+
endTime: Date;
|
|
806
|
+
filterQuery: string;
|
|
807
|
+
serviceIds?: Array<ObjectID> | undefined;
|
|
808
|
+
severityTexts?: Array<string> | undefined;
|
|
809
|
+
bodySearchText?: string | undefined;
|
|
810
|
+
}): Promise<{
|
|
811
|
+
totalLogs: number;
|
|
812
|
+
matchingLogs: number;
|
|
813
|
+
estimatedReductionPercent: number;
|
|
814
|
+
}> {
|
|
815
|
+
// Get total count
|
|
816
|
+
const totalStatement: Statement = SQL`
|
|
817
|
+
SELECT count() AS cnt
|
|
818
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
819
|
+
WHERE projectId = ${{
|
|
820
|
+
type: TableColumnType.ObjectID,
|
|
821
|
+
value: request.projectId,
|
|
822
|
+
}}
|
|
823
|
+
AND time >= ${{
|
|
824
|
+
type: TableColumnType.Date,
|
|
825
|
+
value: request.startTime,
|
|
826
|
+
}}
|
|
827
|
+
AND time <= ${{
|
|
828
|
+
type: TableColumnType.Date,
|
|
829
|
+
value: request.endTime,
|
|
830
|
+
}}
|
|
831
|
+
`;
|
|
832
|
+
|
|
833
|
+
LogAggregationService.appendCommonFilters(totalStatement, request);
|
|
834
|
+
|
|
835
|
+
// Get matching count using the filter query as body search
|
|
836
|
+
const matchStatement: Statement = SQL`
|
|
837
|
+
SELECT count() AS cnt
|
|
838
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
839
|
+
WHERE projectId = ${{
|
|
840
|
+
type: TableColumnType.ObjectID,
|
|
841
|
+
value: request.projectId,
|
|
842
|
+
}}
|
|
843
|
+
AND time >= ${{
|
|
844
|
+
type: TableColumnType.Date,
|
|
845
|
+
value: request.startTime,
|
|
846
|
+
}}
|
|
847
|
+
AND time <= ${{
|
|
848
|
+
type: TableColumnType.Date,
|
|
849
|
+
value: request.endTime,
|
|
850
|
+
}}
|
|
851
|
+
`;
|
|
852
|
+
|
|
853
|
+
LogAggregationService.appendCommonFilters(matchStatement, {
|
|
854
|
+
...request,
|
|
855
|
+
bodySearchText: request.filterQuery,
|
|
856
|
+
});
|
|
857
|
+
|
|
858
|
+
const [totalResult, matchResult] = await Promise.all([
|
|
859
|
+
LogDatabaseService.executeQuery(totalStatement),
|
|
860
|
+
LogDatabaseService.executeQuery(matchStatement),
|
|
861
|
+
]);
|
|
862
|
+
|
|
863
|
+
const totalResponse: DbJSONResponse = await totalResult.json<{
|
|
864
|
+
data?: Array<JSONObject>;
|
|
865
|
+
}>();
|
|
866
|
+
const matchResponse: DbJSONResponse = await matchResult.json<{
|
|
867
|
+
data?: Array<JSONObject>;
|
|
868
|
+
}>();
|
|
869
|
+
|
|
870
|
+
const totalData: Array<JSONObject> = totalResponse.data || [];
|
|
871
|
+
const matchData: Array<JSONObject> = matchResponse.data || [];
|
|
872
|
+
|
|
873
|
+
const totalLogs: number = Number(totalData[0]?.["cnt"] || 0);
|
|
874
|
+
const matchingLogs: number = Number(matchData[0]?.["cnt"] || 0);
|
|
875
|
+
const estimatedReductionPercent: number =
|
|
876
|
+
totalLogs > 0 ? Math.round((matchingLogs / totalLogs) * 100) : 0;
|
|
877
|
+
|
|
878
|
+
return { totalLogs, matchingLogs, estimatedReductionPercent };
|
|
879
|
+
}
|
|
880
|
+
|
|
644
881
|
private static isTopLevelColumn(key: string): boolean {
|
|
645
882
|
return LogAggregationService.TOP_LEVEL_COLUMNS.has(key);
|
|
646
883
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import DatabaseService from "./DatabaseService";
|
|
2
|
+
import Model from "../../Models/DatabaseModels/LogPipelineProcessor";
|
|
3
|
+
|
|
4
|
+
export class Service extends DatabaseService<Model> {
|
|
5
|
+
public constructor() {
|
|
6
|
+
super(Model);
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export default new Service();
|
|
@@ -227,8 +227,8 @@ export class TelemetryAttributeService {
|
|
|
227
227
|
WITH filtered AS (
|
|
228
228
|
SELECT arrayJoin(
|
|
229
229
|
if(
|
|
230
|
-
|
|
231
|
-
|
|
230
|
+
empty(${data.attributeKeysColumn}),
|
|
231
|
+
mapKeys(${data.attributesColumn}),
|
|
232
232
|
${data.attributeKeysColumn}
|
|
233
233
|
)
|
|
234
234
|
) AS attribute
|
|
@@ -238,10 +238,8 @@ export class TelemetryAttributeService {
|
|
|
238
238
|
value: data.projectId,
|
|
239
239
|
}}
|
|
240
240
|
AND (
|
|
241
|
-
${data.attributeKeysColumn}
|
|
242
|
-
|
|
243
|
-
${data.attributesColumn} != ''
|
|
244
|
-
)
|
|
241
|
+
NOT empty(${data.attributeKeysColumn}) OR
|
|
242
|
+
NOT empty(${data.attributesColumn})
|
|
245
243
|
)
|
|
246
244
|
AND ${data.timeColumn} >= ${{
|
|
247
245
|
type: TableColumnType.Date,
|
|
@@ -120,7 +120,11 @@ export class Statement implements BaseQueryParams {
|
|
|
120
120
|
finalValue = v.value.values;
|
|
121
121
|
}
|
|
122
122
|
} else if (v.value instanceof Date) {
|
|
123
|
-
|
|
123
|
+
if (typeof v !== "string" && v.type === TableColumnType.DateTime64) {
|
|
124
|
+
finalValue = OneUptimeDate.toClickhouseDateTime64(v.value);
|
|
125
|
+
} else {
|
|
126
|
+
finalValue = OneUptimeDate.toClickhouseDateTime(v.value);
|
|
127
|
+
}
|
|
124
128
|
} else {
|
|
125
129
|
finalValue = v.value;
|
|
126
130
|
}
|
|
@@ -136,6 +140,15 @@ export class Statement implements BaseQueryParams {
|
|
|
136
140
|
finalValue = OneUptimeDate.toClickhouseDateTime(finalValue);
|
|
137
141
|
}
|
|
138
142
|
|
|
143
|
+
if (
|
|
144
|
+
typeof v !== "string" &&
|
|
145
|
+
v.type === TableColumnType.DateTime64 &&
|
|
146
|
+
!(v.value instanceof Date)
|
|
147
|
+
) {
|
|
148
|
+
finalValue = OneUptimeDate.fromString(finalValue as string);
|
|
149
|
+
finalValue = OneUptimeDate.toClickhouseDateTime64(finalValue);
|
|
150
|
+
}
|
|
151
|
+
|
|
139
152
|
return finalValue;
|
|
140
153
|
}
|
|
141
154
|
|
|
@@ -176,6 +189,7 @@ export class Statement implements BaseQueryParams {
|
|
|
176
189
|
[TableColumnType.Number]: "Int32",
|
|
177
190
|
[TableColumnType.Decimal]: "Double",
|
|
178
191
|
[TableColumnType.Date]: "DateTime",
|
|
192
|
+
[TableColumnType.DateTime64]: "DateTime64(9)",
|
|
179
193
|
[TableColumnType.JSON]: "JSON",
|
|
180
194
|
[TableColumnType.ArrayNumber]: "Array(Int32)",
|
|
181
195
|
[TableColumnType.ArrayText]: "Array(String)",
|
|
@@ -8,7 +8,7 @@ import logger from "../Logger";
|
|
|
8
8
|
import { SQL, Statement } from "./Statement";
|
|
9
9
|
import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
|
|
10
10
|
import CommonModel, {
|
|
11
|
-
Record,
|
|
11
|
+
Record as AnalyticsRecord,
|
|
12
12
|
RecordValue,
|
|
13
13
|
} from "../../../Models/AnalyticsModels/AnalyticsBaseModel/CommonModel";
|
|
14
14
|
import AnalyticsTableColumn, {
|
|
@@ -85,7 +85,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
85
85
|
return columnNames;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
-
public getRecordValuesStatement(record:
|
|
88
|
+
public getRecordValuesStatement(record: AnalyticsRecord): string {
|
|
89
89
|
let valueStatement: string = "";
|
|
90
90
|
|
|
91
91
|
for (const value of record) {
|
|
@@ -106,7 +106,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
106
106
|
return valueStatement;
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
public getValuesStatement(records: Array<
|
|
109
|
+
public getValuesStatement(records: Array<AnalyticsRecord>): string {
|
|
110
110
|
let statement: string = "";
|
|
111
111
|
for (const record of records) {
|
|
112
112
|
statement += `(${this.getRecordValuesStatement(record)}), `;
|
|
@@ -126,10 +126,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
126
126
|
this.model.getTableColumns(),
|
|
127
127
|
);
|
|
128
128
|
|
|
129
|
-
const records: Array<
|
|
129
|
+
const records: Array<AnalyticsRecord> = [];
|
|
130
130
|
|
|
131
131
|
for (const item of data.item) {
|
|
132
|
-
const record:
|
|
132
|
+
const record: AnalyticsRecord = this.getRecord(item);
|
|
133
133
|
records.push(record);
|
|
134
134
|
}
|
|
135
135
|
|
|
@@ -154,8 +154,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
154
154
|
return statement;
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
-
private getRecord(item: CommonModel):
|
|
158
|
-
const record:
|
|
157
|
+
private getRecord(item: CommonModel): AnalyticsRecord {
|
|
158
|
+
const record: AnalyticsRecord = [];
|
|
159
159
|
|
|
160
160
|
for (const column of item.getTableColumns()) {
|
|
161
161
|
const value: RecordValue | undefined = this.sanitizeValue(
|
|
@@ -251,6 +251,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
251
251
|
value = `CAST(${this.escapeStringLiteral(value.toString())} AS Int128)`;
|
|
252
252
|
}
|
|
253
253
|
|
|
254
|
+
if (column.type === TableColumnType.BigNumber) {
|
|
255
|
+
if (typeof value === "string") {
|
|
256
|
+
value = parseInt(value);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
if (column.type === TableColumnType.ArrayBigNumber) {
|
|
261
|
+
value = `[${(value as Array<number>)
|
|
262
|
+
.map((v: number) => {
|
|
263
|
+
if (v && typeof v !== "number") {
|
|
264
|
+
v = parseFloat(v);
|
|
265
|
+
return isNaN(v) ? "NULL" : v;
|
|
266
|
+
}
|
|
267
|
+
return v;
|
|
268
|
+
})
|
|
269
|
+
.join(", ")}]`;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
if (column.type === TableColumnType.MapStringString) {
|
|
273
|
+
const mapObj: Record<string, string> = value as Record<string, string>;
|
|
274
|
+
const entries: Array<string> = Object.entries(mapObj).map(
|
|
275
|
+
([k, v]: [string, string]) => {
|
|
276
|
+
return `${this.escapeStringLiteral(k)}, ${this.escapeStringLiteral(v)}`;
|
|
277
|
+
},
|
|
278
|
+
);
|
|
279
|
+
value = `map(${entries.join(", ")})`;
|
|
280
|
+
}
|
|
281
|
+
|
|
254
282
|
return value;
|
|
255
283
|
}
|
|
256
284
|
|
|
@@ -387,6 +415,28 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
387
415
|
} else {
|
|
388
416
|
whereStatement.append(SQL`AND ${key} IS NULL`);
|
|
389
417
|
}
|
|
418
|
+
} else if (
|
|
419
|
+
tableColumn.type === TableColumnType.MapStringString &&
|
|
420
|
+
typeof value === "object"
|
|
421
|
+
) {
|
|
422
|
+
const mapValue: Record<string, string> = value as Record<
|
|
423
|
+
string,
|
|
424
|
+
string
|
|
425
|
+
>;
|
|
426
|
+
for (const mapKey in mapValue) {
|
|
427
|
+
if (mapValue[mapKey] === undefined) {
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
whereStatement.append(
|
|
431
|
+
SQL`AND ${key}[${{
|
|
432
|
+
value: mapKey,
|
|
433
|
+
type: TableColumnType.Text,
|
|
434
|
+
}}] = ${{
|
|
435
|
+
value: mapValue[mapKey] as string,
|
|
436
|
+
type: TableColumnType.Text,
|
|
437
|
+
}}`,
|
|
438
|
+
);
|
|
439
|
+
}
|
|
390
440
|
} else if (
|
|
391
441
|
(tableColumn.type === TableColumnType.JSON ||
|
|
392
442
|
tableColumn.type === TableColumnType.JSONArray) &&
|
|
@@ -640,6 +690,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
640
690
|
);
|
|
641
691
|
}
|
|
642
692
|
|
|
693
|
+
// Append projections after indexes
|
|
694
|
+
if (this.model.projections && this.model.projections.length > 0) {
|
|
695
|
+
for (const projection of this.model.projections) {
|
|
696
|
+
columns.append(`, PROJECTION ${projection.name} (${projection.query})`);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
643
700
|
return columns;
|
|
644
701
|
}
|
|
645
702
|
|
|
@@ -649,13 +706,16 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
649
706
|
return {
|
|
650
707
|
String: TableColumnType.Text,
|
|
651
708
|
Int32: TableColumnType.Number,
|
|
652
|
-
Int64: TableColumnType.
|
|
709
|
+
Int64: TableColumnType.BigNumber,
|
|
653
710
|
Int128: TableColumnType.LongNumber,
|
|
654
711
|
Float32: TableColumnType.Decimal,
|
|
655
712
|
Float64: TableColumnType.Decimal,
|
|
656
713
|
DateTime: TableColumnType.Date,
|
|
714
|
+
"DateTime64(9)": TableColumnType.DateTime64,
|
|
657
715
|
"Array(String)": TableColumnType.ArrayText,
|
|
658
716
|
"Array(Int32)": TableColumnType.ArrayNumber,
|
|
717
|
+
"Array(Int64)": TableColumnType.ArrayBigNumber,
|
|
718
|
+
"Map(String, String)": TableColumnType.MapStringString,
|
|
659
719
|
JSON: TableColumnType.JSON, //JSONArray is also JSON
|
|
660
720
|
Bool: TableColumnType.Boolean,
|
|
661
721
|
}[clickhouseType];
|
|
@@ -671,11 +731,15 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
671
731
|
[TableColumnType.IP]: SQL`String`,
|
|
672
732
|
[TableColumnType.Port]: SQL`String`,
|
|
673
733
|
[TableColumnType.Date]: SQL`DateTime`,
|
|
734
|
+
[TableColumnType.DateTime64]: SQL`DateTime64(9)`,
|
|
674
735
|
[TableColumnType.JSON]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
|
|
675
736
|
[TableColumnType.JSONArray]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
|
|
676
737
|
[TableColumnType.ArrayNumber]: SQL`Array(Int32)`,
|
|
738
|
+
[TableColumnType.ArrayBigNumber]: SQL`Array(Int64)`,
|
|
677
739
|
[TableColumnType.ArrayText]: SQL`Array(String)`,
|
|
678
740
|
[TableColumnType.LongNumber]: SQL`Int128`,
|
|
741
|
+
[TableColumnType.BigNumber]: SQL`Int64`,
|
|
742
|
+
[TableColumnType.MapStringString]: SQL`Map(String, String)`,
|
|
679
743
|
}[type];
|
|
680
744
|
|
|
681
745
|
if (!statement) {
|
|
@@ -700,12 +764,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
700
764
|
}
|
|
701
765
|
|
|
702
766
|
public toAddColumnStatement(column: AnalyticsTableColumn): Statement {
|
|
767
|
+
// Build column definition without skip index (indexes must be added separately via ADD INDEX)
|
|
768
|
+
const columnDef: Statement = new Statement();
|
|
769
|
+
|
|
770
|
+
columnDef
|
|
771
|
+
.append(column.key)
|
|
772
|
+
.append(SQL` `)
|
|
773
|
+
.append(
|
|
774
|
+
column.required
|
|
775
|
+
? this.toColumnType(column.type)
|
|
776
|
+
: SQL`Nullable(`
|
|
777
|
+
.append(this.toColumnType(column.type))
|
|
778
|
+
.append(SQL`)`),
|
|
779
|
+
);
|
|
780
|
+
|
|
781
|
+
if (column.codec) {
|
|
782
|
+
const codecStr: string =
|
|
783
|
+
column.codec.level !== undefined
|
|
784
|
+
? `${column.codec.codec}(${column.codec.level})`
|
|
785
|
+
: column.codec.codec;
|
|
786
|
+
columnDef.append(` CODEC(${codecStr})`);
|
|
787
|
+
}
|
|
788
|
+
|
|
703
789
|
const statement: Statement = SQL`
|
|
704
790
|
ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
|
|
705
791
|
this.model.tableName
|
|
706
|
-
} ADD COLUMN IF NOT EXISTS `.append(
|
|
707
|
-
this.toColumnsCreateStatement([column]),
|
|
708
|
-
);
|
|
792
|
+
} ADD COLUMN IF NOT EXISTS `.append(columnDef);
|
|
709
793
|
|
|
710
794
|
logger.debug(`${this.model.tableName} Add Column Statement`);
|
|
711
795
|
logger.debug(statement);
|
|
@@ -713,6 +797,37 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
|
|
|
713
797
|
return statement;
|
|
714
798
|
}
|
|
715
799
|
|
|
800
|
+
public toAddSkipIndexStatement(
|
|
801
|
+
column: AnalyticsTableColumn,
|
|
802
|
+
): Statement | null {
|
|
803
|
+
if (!column.skipIndex) {
|
|
804
|
+
return null;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
const idx: AnalyticsTableColumn["skipIndex"] = column.skipIndex;
|
|
808
|
+
const paramsStr: string =
|
|
809
|
+
idx.params && idx.params.length > 0 ? `(${idx.params.join(", ")})` : "";
|
|
810
|
+
|
|
811
|
+
const needsAssumeNotNull: boolean =
|
|
812
|
+
!column.required &&
|
|
813
|
+
(idx.type === SkipIndexType.TokenBF ||
|
|
814
|
+
idx.type === SkipIndexType.NgramBF);
|
|
815
|
+
const columnExpr: string = needsAssumeNotNull
|
|
816
|
+
? `assumeNotNull(${column.key})`
|
|
817
|
+
: column.key;
|
|
818
|
+
|
|
819
|
+
const databaseName: string = this.database.getDatasourceOptions().database!;
|
|
820
|
+
const statement: Statement = new Statement();
|
|
821
|
+
statement.append(
|
|
822
|
+
`ALTER TABLE ${databaseName}.${this.model.tableName} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
|
|
823
|
+
);
|
|
824
|
+
|
|
825
|
+
logger.debug(`${this.model.tableName} Add Skip Index Statement`);
|
|
826
|
+
logger.debug(statement);
|
|
827
|
+
|
|
828
|
+
return statement;
|
|
829
|
+
}
|
|
830
|
+
|
|
716
831
|
public toDropColumnStatement(columnName: string): string {
|
|
717
832
|
const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
|
|
718
833
|
.database!}.${this.model.tableName} DROP COLUMN IF EXISTS ${columnName}`;
|
|
@@ -2,6 +2,7 @@ import LogAggregationService, {
|
|
|
2
2
|
FacetRequest,
|
|
3
3
|
} from "../../../Server/Services/LogAggregationService";
|
|
4
4
|
import { Statement } from "../../../Server/Utils/AnalyticsDatabase/Statement";
|
|
5
|
+
import AnalyticsTableName from "../../../Types/AnalyticsDatabase/AnalyticsTableName";
|
|
5
6
|
import ObjectID from "../../../Types/ObjectID";
|
|
6
7
|
import OneUptimeDate from "../../../Types/Date";
|
|
7
8
|
import { describe, expect, test } from "@jest/globals";
|
|
@@ -35,7 +36,7 @@ describe("LogAggregationService", () => {
|
|
|
35
36
|
|
|
36
37
|
expect(statement.query_params).toStrictEqual({
|
|
37
38
|
p0: "severityText",
|
|
38
|
-
p1:
|
|
39
|
+
p1: AnalyticsTableName.Log,
|
|
39
40
|
p2: defaultRequest.projectId.toString(),
|
|
40
41
|
p3: OneUptimeDate.toClickhouseDateTime(defaultRequest.startTime),
|
|
41
42
|
p4: OneUptimeDate.toClickhouseDateTime(defaultRequest.endTime),
|
|
@@ -55,7 +56,7 @@ describe("LogAggregationService", () => {
|
|
|
55
56
|
|
|
56
57
|
expect(statement.query_params).toStrictEqual({
|
|
57
58
|
p0: facetKey,
|
|
58
|
-
p1:
|
|
59
|
+
p1: AnalyticsTableName.Log,
|
|
59
60
|
p2: defaultRequest.projectId.toString(),
|
|
60
61
|
p3: OneUptimeDate.toClickhouseDateTime(defaultRequest.startTime),
|
|
61
62
|
p4: OneUptimeDate.toClickhouseDateTime(defaultRequest.endTime),
|
|
@@ -10,8 +10,12 @@ enum ColumnType {
|
|
|
10
10
|
ArrayNumber = "Array of Numbers",
|
|
11
11
|
ArrayText = "Array of Text",
|
|
12
12
|
LongNumber = "Long Number",
|
|
13
|
+
BigNumber = "Big Number",
|
|
14
|
+
DateTime64 = "DateTime64",
|
|
13
15
|
IP = "IP",
|
|
14
16
|
Port = "Port",
|
|
17
|
+
MapStringString = "Map(String, String)",
|
|
18
|
+
ArrayBigNumber = "Array of Big Numbers",
|
|
15
19
|
}
|
|
16
20
|
|
|
17
21
|
export default ColumnType;
|
package/Types/Date.ts
CHANGED
|
@@ -1524,4 +1524,26 @@ export default class OneUptimeDate {
|
|
|
1524
1524
|
const parsedDate: Date = this.fromString(date);
|
|
1525
1525
|
return moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
|
|
1526
1526
|
}
|
|
1527
|
+
|
|
1528
|
+
public static toClickhouseDateTime64(
|
|
1529
|
+
date: Date | string,
|
|
1530
|
+
nanoTimestamp?: number,
|
|
1531
|
+
): string {
|
|
1532
|
+
const parsedDate: Date = this.fromString(date);
|
|
1533
|
+
const base: string = moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
|
|
1534
|
+
|
|
1535
|
+
let nanoFraction: string;
|
|
1536
|
+
|
|
1537
|
+
if (nanoTimestamp !== undefined && nanoTimestamp > 0) {
|
|
1538
|
+
// Extract sub-second nanoseconds from the unix nano timestamp
|
|
1539
|
+
const subSecondNanos: number = nanoTimestamp % 1_000_000_000;
|
|
1540
|
+
nanoFraction = subSecondNanos.toString().padStart(9, "0");
|
|
1541
|
+
} else {
|
|
1542
|
+
// Fall back to milliseconds from the Date object
|
|
1543
|
+
const ms: number = parsedDate.getMilliseconds();
|
|
1544
|
+
nanoFraction = (ms * 1_000_000).toString().padStart(9, "0");
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
return `${base}.${nanoFraction}`;
|
|
1548
|
+
}
|
|
1527
1549
|
}
|