@oneuptime/common 10.0.28 → 10.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/Index.ts +2 -0
- package/Models/DatabaseModels/LogSavedView.ts +466 -0
- package/Server/API/TelemetryAPI.ts +146 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1772355000000-AddLogSavedView.ts +48 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1773344537755-MigrationName.ts +91 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
- package/Server/Services/LogAggregationService.ts +387 -0
- package/Server/Services/LogSavedViewService.ts +109 -0
- package/Server/Types/Workflow/Components/BaseModel/OnTriggerBaseModel.ts +15 -0
- package/Server/Utils/Express.ts +1 -0
- package/Server/Utils/OpenAPI.ts +28 -0
- package/Server/Utils/StartServer.ts +20 -1
- package/UI/Components/LogsViewer/LogsViewer.tsx +204 -64
- package/UI/Components/LogsViewer/components/ColumnSelector.tsx +270 -0
- package/UI/Components/LogsViewer/components/LiveLogsToggle.tsx +3 -3
- package/UI/Components/LogsViewer/components/LogTimeRangePicker.tsx +2 -2
- package/UI/Components/LogsViewer/components/LogsAnalyticsView.tsx +699 -0
- package/UI/Components/LogsViewer/components/LogsFacetSidebar.tsx +46 -1
- package/UI/Components/LogsViewer/components/LogsFilterCard.tsx +3 -3
- package/UI/Components/LogsViewer/components/LogsTable.tsx +288 -103
- package/UI/Components/LogsViewer/components/LogsViewerToolbar.tsx +113 -11
- package/UI/Components/LogsViewer/components/SavedViewsDropdown.tsx +175 -0
- package/UI/Components/LogsViewer/types.ts +96 -0
- package/build/dist/Models/DatabaseModels/Index.js +2 -0
- package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
- package/build/dist/Models/DatabaseModels/LogSavedView.js +496 -0
- package/build/dist/Models/DatabaseModels/LogSavedView.js.map +1 -0
- package/build/dist/Server/API/TelemetryAPI.js +88 -0
- package/build/dist/Server/API/TelemetryAPI.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1772355000000-AddLogSavedView.js +44 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1772355000000-AddLogSavedView.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773344537755-MigrationName.js +38 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1773344537755-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/LogAggregationService.js +249 -0
- package/build/dist/Server/Services/LogAggregationService.js.map +1 -1
- package/build/dist/Server/Services/LogSavedViewService.js +82 -0
- package/build/dist/Server/Services/LogSavedViewService.js.map +1 -0
- package/build/dist/Server/Types/Workflow/Components/BaseModel/OnTriggerBaseModel.js +15 -0
- package/build/dist/Server/Types/Workflow/Components/BaseModel/OnTriggerBaseModel.js.map +1 -1
- package/build/dist/Server/Utils/Express.js +1 -0
- package/build/dist/Server/Utils/Express.js.map +1 -1
- package/build/dist/Server/Utils/OpenAPI.js +24 -0
- package/build/dist/Server/Utils/OpenAPI.js.map +1 -1
- package/build/dist/Server/Utils/StartServer.js +17 -2
- package/build/dist/Server/Utils/StartServer.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js +77 -8
- package/build/dist/UI/Components/LogsViewer/LogsViewer.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/ColumnSelector.js +115 -0
- package/build/dist/UI/Components/LogsViewer/components/ColumnSelector.js.map +1 -0
- package/build/dist/UI/Components/LogsViewer/components/LiveLogsToggle.js +3 -3
- package/build/dist/UI/Components/LogsViewer/components/LiveLogsToggle.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogTimeRangePicker.js +2 -2
- package/build/dist/UI/Components/LogsViewer/components/LogTimeRangePicker.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js +379 -0
- package/build/dist/UI/Components/LogsViewer/components/LogsAnalyticsView.js.map +1 -0
- package/build/dist/UI/Components/LogsViewer/components/LogsFacetSidebar.js +27 -13
- package/build/dist/UI/Components/LogsViewer/components/LogsFacetSidebar.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsFilterCard.js +3 -3
- package/build/dist/UI/Components/LogsViewer/components/LogsFilterCard.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsTable.js +118 -49
- package/build/dist/UI/Components/LogsViewer/components/LogsTable.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js +35 -11
- package/build/dist/UI/Components/LogsViewer/components/LogsViewerToolbar.js.map +1 -1
- package/build/dist/UI/Components/LogsViewer/components/SavedViewsDropdown.js +58 -0
- package/build/dist/UI/Components/LogsViewer/components/SavedViewsDropdown.js.map +1 -0
- package/build/dist/UI/Components/LogsViewer/types.js +60 -1
- package/build/dist/UI/Components/LogsViewer/types.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { MigrationInterface, QueryRunner } from "typeorm";
|
|
2
|
+
|
|
3
|
+
export class MigrationName1773344537755 implements MigrationInterface {
|
|
4
|
+
public name = "MigrationName1773344537755";
|
|
5
|
+
|
|
6
|
+
public async up(queryRunner: QueryRunner): Promise<void> {
|
|
7
|
+
await queryRunner.query(
|
|
8
|
+
`ALTER TABLE "LogSavedView" DROP CONSTRAINT "FK_56e1d744839c4e59c50de300a9d"`,
|
|
9
|
+
);
|
|
10
|
+
await queryRunner.query(
|
|
11
|
+
`ALTER TABLE "LogSavedView" DROP CONSTRAINT "FK_fa55f4b8cb6e6ce31554b7b021f"`,
|
|
12
|
+
);
|
|
13
|
+
await queryRunner.query(
|
|
14
|
+
`ALTER TABLE "LogSavedView" DROP CONSTRAINT "FK_8bd2b62c5f269dc8b2c74da0f27"`,
|
|
15
|
+
);
|
|
16
|
+
await queryRunner.query(
|
|
17
|
+
`DROP INDEX "public"."IDX_56e1d744839c4e59c50de300a9"`,
|
|
18
|
+
);
|
|
19
|
+
await queryRunner.query(
|
|
20
|
+
`DROP INDEX "public"."IDX_80241afbecf0a3749cc775f93f"`,
|
|
21
|
+
);
|
|
22
|
+
await queryRunner.query(`ALTER TABLE "LogSavedView" DROP COLUMN "name"`);
|
|
23
|
+
await queryRunner.query(
|
|
24
|
+
`ALTER TABLE "LogSavedView" ADD "name" character varying(50) NOT NULL`,
|
|
25
|
+
);
|
|
26
|
+
await queryRunner.query(
|
|
27
|
+
`ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
|
|
28
|
+
);
|
|
29
|
+
await queryRunner.query(
|
|
30
|
+
`ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
|
|
31
|
+
);
|
|
32
|
+
await queryRunner.query(
|
|
33
|
+
`CREATE INDEX "IDX_fd2dce79ac83d56416311f50e5" ON "LogSavedView" ("projectId") `,
|
|
34
|
+
);
|
|
35
|
+
await queryRunner.query(
|
|
36
|
+
`CREATE INDEX "IDX_0f23bc454fb9a7dbecaeee6b93" ON "LogSavedView" ("isDefault") `,
|
|
37
|
+
);
|
|
38
|
+
await queryRunner.query(
|
|
39
|
+
`ALTER TABLE "LogSavedView" ADD CONSTRAINT "FK_fd2dce79ac83d56416311f50e52" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
|
40
|
+
);
|
|
41
|
+
await queryRunner.query(
|
|
42
|
+
`ALTER TABLE "LogSavedView" ADD CONSTRAINT "FK_93663ad4128292e6a57f5950ab9" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
|
|
43
|
+
);
|
|
44
|
+
await queryRunner.query(
|
|
45
|
+
`ALTER TABLE "LogSavedView" ADD CONSTRAINT "FK_a7817e3946945d28ef65a81e173" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
public async down(queryRunner: QueryRunner): Promise<void> {
|
|
50
|
+
await queryRunner.query(
|
|
51
|
+
`ALTER TABLE "LogSavedView" DROP CONSTRAINT "FK_a7817e3946945d28ef65a81e173"`,
|
|
52
|
+
);
|
|
53
|
+
await queryRunner.query(
|
|
54
|
+
`ALTER TABLE "LogSavedView" DROP CONSTRAINT "FK_93663ad4128292e6a57f5950ab9"`,
|
|
55
|
+
);
|
|
56
|
+
await queryRunner.query(
|
|
57
|
+
`ALTER TABLE "LogSavedView" DROP CONSTRAINT "FK_fd2dce79ac83d56416311f50e52"`,
|
|
58
|
+
);
|
|
59
|
+
await queryRunner.query(
|
|
60
|
+
`DROP INDEX "public"."IDX_0f23bc454fb9a7dbecaeee6b93"`,
|
|
61
|
+
);
|
|
62
|
+
await queryRunner.query(
|
|
63
|
+
`DROP INDEX "public"."IDX_fd2dce79ac83d56416311f50e5"`,
|
|
64
|
+
);
|
|
65
|
+
await queryRunner.query(
|
|
66
|
+
`ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
|
|
67
|
+
);
|
|
68
|
+
await queryRunner.query(
|
|
69
|
+
`ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
|
|
70
|
+
);
|
|
71
|
+
await queryRunner.query(`ALTER TABLE "LogSavedView" DROP COLUMN "name"`);
|
|
72
|
+
await queryRunner.query(
|
|
73
|
+
`ALTER TABLE "LogSavedView" ADD "name" character varying(100) NOT NULL`,
|
|
74
|
+
);
|
|
75
|
+
await queryRunner.query(
|
|
76
|
+
`CREATE INDEX "IDX_80241afbecf0a3749cc775f93f" ON "LogSavedView" ("isDefault") `,
|
|
77
|
+
);
|
|
78
|
+
await queryRunner.query(
|
|
79
|
+
`CREATE INDEX "IDX_56e1d744839c4e59c50de300a9" ON "LogSavedView" ("projectId") `,
|
|
80
|
+
);
|
|
81
|
+
await queryRunner.query(
|
|
82
|
+
`ALTER TABLE "LogSavedView" ADD CONSTRAINT "FK_8bd2b62c5f269dc8b2c74da0f27" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
|
|
83
|
+
);
|
|
84
|
+
await queryRunner.query(
|
|
85
|
+
`ALTER TABLE "LogSavedView" ADD CONSTRAINT "FK_fa55f4b8cb6e6ce31554b7b021f" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
|
|
86
|
+
);
|
|
87
|
+
await queryRunner.query(
|
|
88
|
+
`ALTER TABLE "LogSavedView" ADD CONSTRAINT "FK_56e1d744839c4e59c50de300a9d" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -262,6 +262,8 @@ import { MigrationName1770834237091 } from "./1770834237091-MigrationName";
|
|
|
262
262
|
import { MigrationName1772111896988 } from "./1772111896988-MigrationName";
|
|
263
263
|
import { MigrationName1772280000000 } from "./1772280000000-MigrationName";
|
|
264
264
|
import { MigrationName1772350000000 } from "./1772350000000-MigrationName";
|
|
265
|
+
import { AddLogSavedView1772355000000 } from "./1772355000000-AddLogSavedView";
|
|
266
|
+
import { MigrationName1773344537755 } from "./1773344537755-MigrationName";
|
|
265
267
|
|
|
266
268
|
export default [
|
|
267
269
|
InitialMigration,
|
|
@@ -528,4 +530,6 @@ export default [
|
|
|
528
530
|
MigrationName1772111896988,
|
|
529
531
|
MigrationName1772280000000,
|
|
530
532
|
MigrationName1772350000000,
|
|
533
|
+
AddLogSavedView1772355000000,
|
|
534
|
+
MigrationName1773344537755,
|
|
531
535
|
];
|
|
@@ -44,6 +44,42 @@ export interface FacetRequest {
|
|
|
44
44
|
spanIds?: Array<string> | undefined;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
export type AnalyticsChartType = "timeseries" | "toplist" | "table";
|
|
48
|
+
export type AnalyticsAggregation = "count" | "unique";
|
|
49
|
+
|
|
50
|
+
export interface AnalyticsRequest {
|
|
51
|
+
projectId: ObjectID;
|
|
52
|
+
startTime: Date;
|
|
53
|
+
endTime: Date;
|
|
54
|
+
bucketSizeInMinutes: number;
|
|
55
|
+
chartType: AnalyticsChartType;
|
|
56
|
+
groupBy?: Array<string> | undefined;
|
|
57
|
+
aggregation: AnalyticsAggregation;
|
|
58
|
+
aggregationField?: string | undefined;
|
|
59
|
+
serviceIds?: Array<ObjectID> | undefined;
|
|
60
|
+
severityTexts?: Array<string> | undefined;
|
|
61
|
+
bodySearchText?: string | undefined;
|
|
62
|
+
traceIds?: Array<string> | undefined;
|
|
63
|
+
spanIds?: Array<string> | undefined;
|
|
64
|
+
limit?: number | undefined;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface AnalyticsTimeseriesRow {
|
|
68
|
+
time: string;
|
|
69
|
+
count: number;
|
|
70
|
+
groupValues: Record<string, string>;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface AnalyticsTopItem {
|
|
74
|
+
value: string;
|
|
75
|
+
count: number;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface AnalyticsTableRow {
|
|
79
|
+
groupValues: Record<string, string>;
|
|
80
|
+
count: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
47
83
|
export class LogAggregationService {
|
|
48
84
|
private static readonly DEFAULT_FACET_LIMIT: number = 500;
|
|
49
85
|
private static readonly TABLE_NAME: string = "LogItem";
|
|
@@ -197,6 +233,357 @@ export class LogAggregationService {
|
|
|
197
233
|
return statement;
|
|
198
234
|
}
|
|
199
235
|
|
|
236
|
+
private static readonly DEFAULT_ANALYTICS_LIMIT: number = 10;
|
|
237
|
+
private static readonly MAX_GROUP_BY_DIMENSIONS: number = 2;
|
|
238
|
+
|
|
239
|
+
@CaptureSpan()
|
|
240
|
+
public static async getAnalyticsTimeseries(
|
|
241
|
+
request: AnalyticsRequest,
|
|
242
|
+
): Promise<Array<AnalyticsTimeseriesRow>> {
|
|
243
|
+
const statement: Statement =
|
|
244
|
+
LogAggregationService.buildAnalyticsTimeseriesStatement(request);
|
|
245
|
+
|
|
246
|
+
const dbResult: Results = await LogDatabaseService.executeQuery(statement);
|
|
247
|
+
const response: DbJSONResponse = await dbResult.json<{
|
|
248
|
+
data?: Array<JSONObject>;
|
|
249
|
+
}>();
|
|
250
|
+
|
|
251
|
+
const rows: Array<JSONObject> = response.data || [];
|
|
252
|
+
const groupByKeys: Array<string> = request.groupBy || [];
|
|
253
|
+
|
|
254
|
+
return rows.map((row: JSONObject): AnalyticsTimeseriesRow => {
|
|
255
|
+
const groupValues: Record<string, string> = {};
|
|
256
|
+
|
|
257
|
+
for (const key of groupByKeys) {
|
|
258
|
+
const alias: string = LogAggregationService.groupByAlias(key);
|
|
259
|
+
groupValues[key] = String(row[alias] || "");
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
return {
|
|
263
|
+
time: String(row["bucket"] || ""),
|
|
264
|
+
count: Number(row["cnt"] || 0),
|
|
265
|
+
groupValues,
|
|
266
|
+
};
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
@CaptureSpan()
|
|
271
|
+
public static async getAnalyticsTopList(
|
|
272
|
+
request: AnalyticsRequest,
|
|
273
|
+
): Promise<Array<AnalyticsTopItem>> {
|
|
274
|
+
if (!request.groupBy || request.groupBy.length === 0) {
|
|
275
|
+
throw new BadDataException(
|
|
276
|
+
"groupBy with at least one dimension is required for top list",
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const statement: Statement =
|
|
281
|
+
LogAggregationService.buildAnalyticsTopListStatement(request);
|
|
282
|
+
|
|
283
|
+
const dbResult: Results = await LogDatabaseService.executeQuery(statement);
|
|
284
|
+
const response: DbJSONResponse = await dbResult.json<{
|
|
285
|
+
data?: Array<JSONObject>;
|
|
286
|
+
}>();
|
|
287
|
+
|
|
288
|
+
const rows: Array<JSONObject> = response.data || [];
|
|
289
|
+
|
|
290
|
+
return rows
|
|
291
|
+
.map((row: JSONObject): AnalyticsTopItem => {
|
|
292
|
+
return {
|
|
293
|
+
value: String(row["val"] || ""),
|
|
294
|
+
count: Number(row["cnt"] || 0),
|
|
295
|
+
};
|
|
296
|
+
})
|
|
297
|
+
.filter((item: AnalyticsTopItem): boolean => {
|
|
298
|
+
return item.value.length > 0;
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
@CaptureSpan()
|
|
303
|
+
public static async getAnalyticsTable(
|
|
304
|
+
request: AnalyticsRequest,
|
|
305
|
+
): Promise<Array<AnalyticsTableRow>> {
|
|
306
|
+
if (!request.groupBy || request.groupBy.length === 0) {
|
|
307
|
+
throw new BadDataException(
|
|
308
|
+
"groupBy with at least one dimension is required for table",
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const statement: Statement =
|
|
313
|
+
LogAggregationService.buildAnalyticsTableStatement(request);
|
|
314
|
+
|
|
315
|
+
const dbResult: Results = await LogDatabaseService.executeQuery(statement);
|
|
316
|
+
const response: DbJSONResponse = await dbResult.json<{
|
|
317
|
+
data?: Array<JSONObject>;
|
|
318
|
+
}>();
|
|
319
|
+
|
|
320
|
+
const rows: Array<JSONObject> = response.data || [];
|
|
321
|
+
const groupByKeys: Array<string> = request.groupBy;
|
|
322
|
+
|
|
323
|
+
return rows.map((row: JSONObject): AnalyticsTableRow => {
|
|
324
|
+
const groupValues: Record<string, string> = {};
|
|
325
|
+
|
|
326
|
+
for (const key of groupByKeys) {
|
|
327
|
+
const alias: string = LogAggregationService.groupByAlias(key);
|
|
328
|
+
groupValues[key] = String(row[alias] || "");
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
return {
|
|
332
|
+
groupValues,
|
|
333
|
+
count: Number(row["cnt"] || 0),
|
|
334
|
+
};
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
private static groupByAlias(key: string): string {
|
|
339
|
+
if (LogAggregationService.isTopLevelColumn(key)) {
|
|
340
|
+
return key;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// For attribute keys, use a sanitized alias
|
|
344
|
+
return `attr_${key.replace(/[^a-zA-Z0-9_]/g, "_")}`;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
private static appendGroupBySelect(
|
|
348
|
+
statement: Statement,
|
|
349
|
+
groupByKeys: Array<string>,
|
|
350
|
+
): void {
|
|
351
|
+
for (const key of groupByKeys) {
|
|
352
|
+
LogAggregationService.validateFacetKey(key);
|
|
353
|
+
|
|
354
|
+
if (LogAggregationService.isTopLevelColumn(key)) {
|
|
355
|
+
statement.append(`, toString(${key}) AS ${key}`);
|
|
356
|
+
} else {
|
|
357
|
+
const alias: string = LogAggregationService.groupByAlias(key);
|
|
358
|
+
statement.append(
|
|
359
|
+
SQL`, JSONExtractRaw(attributes, ${{
|
|
360
|
+
type: TableColumnType.Text,
|
|
361
|
+
value: key,
|
|
362
|
+
}}) AS ${alias}`,
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
private static appendGroupByClause(
|
|
369
|
+
statement: Statement,
|
|
370
|
+
groupByKeys: Array<string>,
|
|
371
|
+
): void {
|
|
372
|
+
for (const key of groupByKeys) {
|
|
373
|
+
if (LogAggregationService.isTopLevelColumn(key)) {
|
|
374
|
+
statement.append(`, ${key}`);
|
|
375
|
+
} else {
|
|
376
|
+
const alias: string = LogAggregationService.groupByAlias(key);
|
|
377
|
+
statement.append(`, ${alias}`);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
private static getAggregationExpression(request: AnalyticsRequest): string {
|
|
383
|
+
if (request.aggregation === "unique" && request.aggregationField) {
|
|
384
|
+
LogAggregationService.validateFacetKey(request.aggregationField);
|
|
385
|
+
|
|
386
|
+
if (LogAggregationService.isTopLevelColumn(request.aggregationField)) {
|
|
387
|
+
return `uniqExact(${request.aggregationField})`;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
return `uniqExact(JSONExtractRaw(attributes, '${request.aggregationField.replace(/'/g, "\\'")}'))`;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
return "count()";
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
private static validateGroupBy(groupBy: Array<string> | undefined): void {
|
|
397
|
+
if (!groupBy) {
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
if (groupBy.length > LogAggregationService.MAX_GROUP_BY_DIMENSIONS) {
|
|
402
|
+
throw new BadDataException(
|
|
403
|
+
`groupBy supports at most ${LogAggregationService.MAX_GROUP_BY_DIMENSIONS} dimensions`,
|
|
404
|
+
);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
for (const key of groupBy) {
|
|
408
|
+
LogAggregationService.validateFacetKey(key);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
private static buildAnalyticsTimeseriesStatement(
|
|
413
|
+
request: AnalyticsRequest,
|
|
414
|
+
): Statement {
|
|
415
|
+
LogAggregationService.validateGroupBy(request.groupBy);
|
|
416
|
+
|
|
417
|
+
const intervalSeconds: number = request.bucketSizeInMinutes * 60;
|
|
418
|
+
const aggExpr: string =
|
|
419
|
+
LogAggregationService.getAggregationExpression(request);
|
|
420
|
+
|
|
421
|
+
const statement: Statement = SQL`
|
|
422
|
+
SELECT
|
|
423
|
+
toStartOfInterval(time, INTERVAL ${{
|
|
424
|
+
type: TableColumnType.Number,
|
|
425
|
+
value: intervalSeconds,
|
|
426
|
+
}} SECOND) AS bucket`;
|
|
427
|
+
|
|
428
|
+
statement.append(`, ${aggExpr} AS cnt`);
|
|
429
|
+
|
|
430
|
+
if (request.groupBy && request.groupBy.length > 0) {
|
|
431
|
+
LogAggregationService.appendGroupBySelect(statement, request.groupBy);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
statement.append(
|
|
435
|
+
SQL`
|
|
436
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
437
|
+
WHERE projectId = ${{
|
|
438
|
+
type: TableColumnType.ObjectID,
|
|
439
|
+
value: request.projectId,
|
|
440
|
+
}}
|
|
441
|
+
AND time >= ${{
|
|
442
|
+
type: TableColumnType.Date,
|
|
443
|
+
value: request.startTime,
|
|
444
|
+
}}
|
|
445
|
+
AND time <= ${{
|
|
446
|
+
type: TableColumnType.Date,
|
|
447
|
+
value: request.endTime,
|
|
448
|
+
}}`,
|
|
449
|
+
);
|
|
450
|
+
|
|
451
|
+
LogAggregationService.appendCommonFilters(statement, request);
|
|
452
|
+
|
|
453
|
+
statement.append(" GROUP BY bucket");
|
|
454
|
+
|
|
455
|
+
if (request.groupBy && request.groupBy.length > 0) {
|
|
456
|
+
LogAggregationService.appendGroupByClause(statement, request.groupBy);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
statement.append(" ORDER BY bucket ASC");
|
|
460
|
+
|
|
461
|
+
return statement;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
private static buildAnalyticsTopListStatement(
|
|
465
|
+
request: AnalyticsRequest,
|
|
466
|
+
): Statement {
|
|
467
|
+
const groupByKey: string = request.groupBy![0]!;
|
|
468
|
+
LogAggregationService.validateFacetKey(groupByKey);
|
|
469
|
+
|
|
470
|
+
const limit: number =
|
|
471
|
+
request.limit ?? LogAggregationService.DEFAULT_ANALYTICS_LIMIT;
|
|
472
|
+
const aggExpr: string =
|
|
473
|
+
LogAggregationService.getAggregationExpression(request);
|
|
474
|
+
|
|
475
|
+
const isTopLevel: boolean =
|
|
476
|
+
LogAggregationService.isTopLevelColumn(groupByKey);
|
|
477
|
+
|
|
478
|
+
const statement: Statement = new Statement();
|
|
479
|
+
|
|
480
|
+
if (isTopLevel) {
|
|
481
|
+
statement.append(
|
|
482
|
+
`SELECT toString(${groupByKey}) AS val, ${aggExpr} AS cnt FROM ${LogAggregationService.TABLE_NAME}`,
|
|
483
|
+
);
|
|
484
|
+
} else {
|
|
485
|
+
statement.append(`SELECT JSONExtractRaw(attributes, `);
|
|
486
|
+
statement.append(
|
|
487
|
+
SQL`${{
|
|
488
|
+
type: TableColumnType.Text,
|
|
489
|
+
value: groupByKey,
|
|
490
|
+
}}`,
|
|
491
|
+
);
|
|
492
|
+
statement.append(
|
|
493
|
+
`) AS val, ${aggExpr} AS cnt FROM ${LogAggregationService.TABLE_NAME}`,
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
statement.append(
|
|
498
|
+
SQL` WHERE projectId = ${{
|
|
499
|
+
type: TableColumnType.ObjectID,
|
|
500
|
+
value: request.projectId,
|
|
501
|
+
}} AND time >= ${{
|
|
502
|
+
type: TableColumnType.Date,
|
|
503
|
+
value: request.startTime,
|
|
504
|
+
}} AND time <= ${{
|
|
505
|
+
type: TableColumnType.Date,
|
|
506
|
+
value: request.endTime,
|
|
507
|
+
}}`,
|
|
508
|
+
);
|
|
509
|
+
|
|
510
|
+
if (!isTopLevel) {
|
|
511
|
+
statement.append(
|
|
512
|
+
SQL` AND JSONHas(attributes, ${{
|
|
513
|
+
type: TableColumnType.Text,
|
|
514
|
+
value: groupByKey,
|
|
515
|
+
}}) = 1`,
|
|
516
|
+
);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
LogAggregationService.appendCommonFilters(statement, request);
|
|
520
|
+
|
|
521
|
+
statement.append(
|
|
522
|
+
SQL` GROUP BY val ORDER BY cnt DESC LIMIT ${{
|
|
523
|
+
type: TableColumnType.Number,
|
|
524
|
+
value: limit,
|
|
525
|
+
}}`,
|
|
526
|
+
);
|
|
527
|
+
|
|
528
|
+
return statement;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
private static buildAnalyticsTableStatement(
|
|
532
|
+
request: AnalyticsRequest,
|
|
533
|
+
): Statement {
|
|
534
|
+
LogAggregationService.validateGroupBy(request.groupBy);
|
|
535
|
+
|
|
536
|
+
const groupByKeys: Array<string> = request.groupBy!;
|
|
537
|
+
const limit: number =
|
|
538
|
+
request.limit ?? LogAggregationService.DEFAULT_ANALYTICS_LIMIT;
|
|
539
|
+
const aggExpr: string =
|
|
540
|
+
LogAggregationService.getAggregationExpression(request);
|
|
541
|
+
|
|
542
|
+
const statement: Statement = new Statement();
|
|
543
|
+
statement.append(`SELECT ${aggExpr} AS cnt`);
|
|
544
|
+
|
|
545
|
+
LogAggregationService.appendGroupBySelect(statement, groupByKeys);
|
|
546
|
+
|
|
547
|
+
statement.append(
|
|
548
|
+
SQL`
|
|
549
|
+
FROM ${LogAggregationService.TABLE_NAME}
|
|
550
|
+
WHERE projectId = ${{
|
|
551
|
+
type: TableColumnType.ObjectID,
|
|
552
|
+
value: request.projectId,
|
|
553
|
+
}}
|
|
554
|
+
AND time >= ${{
|
|
555
|
+
type: TableColumnType.Date,
|
|
556
|
+
value: request.startTime,
|
|
557
|
+
}}
|
|
558
|
+
AND time <= ${{
|
|
559
|
+
type: TableColumnType.Date,
|
|
560
|
+
value: request.endTime,
|
|
561
|
+
}}`,
|
|
562
|
+
);
|
|
563
|
+
|
|
564
|
+
LogAggregationService.appendCommonFilters(statement, request);
|
|
565
|
+
|
|
566
|
+
// Build GROUP BY from aliases
|
|
567
|
+
const aliases: Array<string> = groupByKeys.map((key: string) => {
|
|
568
|
+
if (LogAggregationService.isTopLevelColumn(key)) {
|
|
569
|
+
return key;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
return LogAggregationService.groupByAlias(key);
|
|
573
|
+
});
|
|
574
|
+
|
|
575
|
+
statement.append(` GROUP BY ${aliases.join(", ")}`);
|
|
576
|
+
|
|
577
|
+
statement.append(
|
|
578
|
+
SQL` ORDER BY cnt DESC LIMIT ${{
|
|
579
|
+
type: TableColumnType.Number,
|
|
580
|
+
value: limit,
|
|
581
|
+
}}`,
|
|
582
|
+
);
|
|
583
|
+
|
|
584
|
+
return statement;
|
|
585
|
+
}
|
|
586
|
+
|
|
200
587
|
private static appendCommonFilters(
|
|
201
588
|
statement: Statement,
|
|
202
589
|
request: Pick<
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import DatabaseService from "./DatabaseService";
|
|
2
|
+
import Model from "../../Models/DatabaseModels/LogSavedView";
|
|
3
|
+
import CreateBy from "../Types/Database/CreateBy";
|
|
4
|
+
import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
|
|
5
|
+
import UpdateBy from "../Types/Database/UpdateBy";
|
|
6
|
+
import ObjectID from "../../Types/ObjectID";
|
|
7
|
+
import QueryHelper from "../Types/Database/QueryHelper";
|
|
8
|
+
import LIMIT_MAX from "../../Types/Database/LimitMax";
|
|
9
|
+
|
|
10
|
+
export class Service extends DatabaseService<Model> {
|
|
11
|
+
public constructor() {
|
|
12
|
+
super(Model);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
protected override async onBeforeCreate(
|
|
16
|
+
createBy: CreateBy<Model>,
|
|
17
|
+
): Promise<OnCreate<Model>> {
|
|
18
|
+
if (createBy.data.isDefault === undefined && createBy.data.projectId) {
|
|
19
|
+
const existingDefaultView: Model | null = await this.findOneBy({
|
|
20
|
+
query: {
|
|
21
|
+
projectId: createBy.data.projectId,
|
|
22
|
+
isDefault: true,
|
|
23
|
+
},
|
|
24
|
+
select: {
|
|
25
|
+
_id: true,
|
|
26
|
+
},
|
|
27
|
+
props: {
|
|
28
|
+
isRoot: true,
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
createBy.data.isDefault = !existingDefaultView;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (createBy.data.projectId) {
|
|
36
|
+
await this.unsetOtherDefaultsIfNeeded({
|
|
37
|
+
projectId: createBy.data.projectId,
|
|
38
|
+
isDefault: createBy.data.isDefault || false,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return { createBy, carryForward: null };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
protected override async onBeforeUpdate(
|
|
46
|
+
updateBy: UpdateBy<Model>,
|
|
47
|
+
): Promise<OnUpdate<Model>> {
|
|
48
|
+
if (updateBy.data.isDefault !== true) {
|
|
49
|
+
return { updateBy, carryForward: null };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const itemsToUpdate: Array<Model> = await this.findBy({
|
|
53
|
+
query: updateBy.query,
|
|
54
|
+
select: {
|
|
55
|
+
_id: true,
|
|
56
|
+
projectId: true,
|
|
57
|
+
},
|
|
58
|
+
props: {
|
|
59
|
+
isRoot: true,
|
|
60
|
+
},
|
|
61
|
+
limit: LIMIT_MAX,
|
|
62
|
+
skip: 0,
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
for (const item of itemsToUpdate) {
|
|
66
|
+
if (item.projectId) {
|
|
67
|
+
await this.unsetOtherDefaultsIfNeeded({
|
|
68
|
+
projectId: item.projectId,
|
|
69
|
+
isDefault: true,
|
|
70
|
+
excludeIds: item._id ? [item._id] : [],
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return { updateBy, carryForward: null };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private async unsetOtherDefaultsIfNeeded(data: {
|
|
79
|
+
projectId?: ObjectID;
|
|
80
|
+
isDefault?: boolean;
|
|
81
|
+
excludeIds?: Array<string>;
|
|
82
|
+
}): Promise<void> {
|
|
83
|
+
if (!data.projectId || !data.isDefault) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
await this.updateBy({
|
|
88
|
+
query: {
|
|
89
|
+
projectId: data.projectId,
|
|
90
|
+
isDefault: true,
|
|
91
|
+
...(data.excludeIds && data.excludeIds.length > 0
|
|
92
|
+
? {
|
|
93
|
+
_id: QueryHelper.notInOrNull(data.excludeIds),
|
|
94
|
+
}
|
|
95
|
+
: {}),
|
|
96
|
+
},
|
|
97
|
+
data: {
|
|
98
|
+
isDefault: false,
|
|
99
|
+
},
|
|
100
|
+
props: {
|
|
101
|
+
isRoot: true,
|
|
102
|
+
},
|
|
103
|
+
limit: LIMIT_MAX,
|
|
104
|
+
skip: 0,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export default new Service();
|
|
@@ -138,6 +138,21 @@ export default class OnTriggerBaseModel<
|
|
|
138
138
|
select = JSONFunctions.parse(select) as Select<TBaseModel>;
|
|
139
139
|
}
|
|
140
140
|
|
|
141
|
+
/*
|
|
142
|
+
* Convert string "true"/"false" values to booleans in select.
|
|
143
|
+
* Workflow arguments may pass select values as strings (e.g. "true")
|
|
144
|
+
* which causes TypeORM to iterate string characters as property indices.
|
|
145
|
+
*/
|
|
146
|
+
if (select && typeof select === "object") {
|
|
147
|
+
for (const key in select) {
|
|
148
|
+
if ((select as any)[key] === "true") {
|
|
149
|
+
(select as any)[key] = true;
|
|
150
|
+
} else if ((select as any)[key] === "false") {
|
|
151
|
+
(select as any)[key] = false;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
141
156
|
const model: TBaseModel | null = await this.service!.findOneById({
|
|
142
157
|
id: new ObjectID(data["_id"].toString()),
|
|
143
158
|
props: {
|
package/Server/Utils/Express.ts
CHANGED
|
@@ -21,6 +21,7 @@ export type NextFunction = express.NextFunction;
|
|
|
21
21
|
export const ExpressStatic: GenericFunction = express.static;
|
|
22
22
|
export const ExpressJson: GenericFunction = express.json;
|
|
23
23
|
export const ExpressUrlEncoded: GenericFunction = express.urlencoded;
|
|
24
|
+
export const ExpressRaw: GenericFunction = express.raw;
|
|
24
25
|
|
|
25
26
|
export type ProbeRequest = {
|
|
26
27
|
id: ObjectID;
|
package/Server/Utils/OpenAPI.ts
CHANGED
|
@@ -257,6 +257,20 @@ export default class OpenAPIUtil {
|
|
|
257
257
|
query: { $ref: `#/components/schemas/${querySchemaName}` },
|
|
258
258
|
select: { $ref: `#/components/schemas/${selectSchemaName}` },
|
|
259
259
|
sort: { $ref: `#/components/schemas/${sortSchemaName}` },
|
|
260
|
+
limit: {
|
|
261
|
+
type: "number",
|
|
262
|
+
description:
|
|
263
|
+
"Maximum number of items to return. Defaults to 10.",
|
|
264
|
+
default: 10,
|
|
265
|
+
minimum: 1,
|
|
266
|
+
},
|
|
267
|
+
skip: {
|
|
268
|
+
type: "number",
|
|
269
|
+
description:
|
|
270
|
+
"Number of items to skip for pagination. Defaults to 0.",
|
|
271
|
+
default: 0,
|
|
272
|
+
minimum: 0,
|
|
273
|
+
},
|
|
260
274
|
},
|
|
261
275
|
},
|
|
262
276
|
},
|
|
@@ -891,6 +905,20 @@ export default class OpenAPIUtil {
|
|
|
891
905
|
select: { $ref: `#/components/schemas/${selectSchemaName}` },
|
|
892
906
|
sort: { $ref: `#/components/schemas/${sortSchemaName}` },
|
|
893
907
|
groupBy: { $ref: `#/components/schemas/${groupBySchemaName}` },
|
|
908
|
+
limit: {
|
|
909
|
+
type: "number",
|
|
910
|
+
description:
|
|
911
|
+
"Maximum number of items to return. Defaults to 10.",
|
|
912
|
+
default: 10,
|
|
913
|
+
minimum: 1,
|
|
914
|
+
},
|
|
915
|
+
skip: {
|
|
916
|
+
type: "number",
|
|
917
|
+
description:
|
|
918
|
+
"Number of items to skip for pagination. Defaults to 0.",
|
|
919
|
+
default: 0,
|
|
920
|
+
minimum: 0,
|
|
921
|
+
},
|
|
894
922
|
},
|
|
895
923
|
},
|
|
896
924
|
},
|