@oneuptime/common 10.0.31 → 10.0.34

package/Server/Middleware/WhatsAppAuthorization.ts

@@ -0,0 +1,87 @@
+import {
+  ExpressResponse,
+  NextFunction,
+  OneUptimeRequest,
+} from "../Utils/Express";
+import Response from "../Utils/Response";
+import BadDataException from "../../Types/Exception/BadDataException";
+import GlobalConfig from "../../Models/DatabaseModels/GlobalConfig";
+import GlobalConfigService from "../Services/GlobalConfigService";
+import ObjectID from "../../Types/ObjectID";
+import crypto from "crypto";
+import logger from "../Utils/Logger";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+export default class WhatsAppAuthorization {
+  @CaptureSpan()
+  public static async isAuthorizedWhatsAppRequest(
+    req: OneUptimeRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ): Promise<void> {
+    logger.debug("Starting WhatsApp webhook signature verification");
+
+    const signature: string | undefined = req.headers["x-hub-signature-256"] as
+      | string
+      | undefined;
+
+    if (!signature) {
+      logger.error(
+        "WhatsApp webhook request missing X-Hub-Signature-256 header.",
+      );
+      return Response.sendErrorResponse(
+        req,
+        res,
+        new BadDataException("Missing X-Hub-Signature-256 header."),
+      );
+    }
+
+    const globalConfig: GlobalConfig | null =
+      await GlobalConfigService.findOneBy({
+        query: {
+          _id: ObjectID.getZeroObjectID().toString(),
+        },
+        props: {
+          isRoot: true,
+        },
+        select: {
+          metaWhatsAppAppSecret: true,
+        },
+      });
+
+    const appSecret: string | undefined =
+      globalConfig?.metaWhatsAppAppSecret?.trim() || undefined;
+
+    if (!appSecret) {
+      logger.error(
+        "Meta WhatsApp App Secret is not configured. Cannot verify webhook signature.",
+      );
+      return Response.sendErrorResponse(
+        req,
+        res,
+        new BadDataException("Meta WhatsApp App Secret is not configured."),
+      );
+    }
+
+    const rawBody: string = req.rawBody || "";
+
+    const expectedSignature: string = `sha256=${crypto.createHmac("sha256", appSecret).update(rawBody).digest("hex")}`;
+
+    if (
+      !crypto.timingSafeEqual(
+        Buffer.from(expectedSignature) as Uint8Array,
+        Buffer.from(signature) as Uint8Array,
+      )
+    ) {
+      logger.error("WhatsApp webhook signature verification failed.");
+      return Response.sendErrorResponse(
+        req,
+        res,
+        new BadDataException("WhatsApp webhook signature verification failed."),
+      );
+    }
+
+    logger.debug("WhatsApp webhook signature verified successfully");
+    next();
+  }
+}

package/Server/Services/AnalyticsDatabaseService.ts

@@ -238,6 +238,13 @@ export default class AnalyticsDatabaseService<
     const statement: Statement =
       this.statementGenerator.toAddColumnStatement(column);
     await this.execute(statement);
+
+    // Add skip index separately (ClickHouse requires ADD INDEX as a separate ALTER statement)
+    const indexStatement: Statement | null =
+      this.statementGenerator.toAddSkipIndexStatement(column);
+    if (indexStatement) {
+      await this.execute(indexStatement);
+    }
   }
 
   @CaptureSpan()
@@ -247,6 +254,60 @@ export default class AnalyticsDatabaseService<
     );
   }
 
+  public async doesColumnExist(columnName: string): Promise<boolean> {
+    const tableName: string = this.model.tableName;
+    const result: { data: Array<JSONObject> } = await (
+      await this.executeQuery(
+        `SELECT count() as cnt FROM system.columns WHERE database = currentDatabase() AND table = '${tableName}' AND name = '${columnName}'`,
+      )
+    ).json();
+
+    const rows: Array<JSONObject> = result.data || [];
+
+    return rows.length > 0 && Number(rows[0]!["cnt"]) > 0;
+  }
+
+  public async getColumnCodec(columnName: string): Promise<string> {
+    const tableName: string = this.model.tableName;
+    const result: { data: Array<JSONObject> } = await (
+      await this.executeQuery(
+        `SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = '${tableName}' AND name = '${columnName}'`,
+      )
+    ).json();
+
+    const rows: Array<JSONObject> = result.data || [];
+
+    if (rows.length === 0) {
+      return "";
+    }
+
+    return (rows[0]!["compression_codec"] as string) || "";
+  }
+
+  public async setColumnCodecIfNotSet(data: {
+    columnName: string;
+    columnType: string;
+    codec: string;
+    expectedCodecValue: string;
+  }): Promise<void> {
+    const tableName: string = this.model.tableName;
+    const currentCodec: string = await this.getColumnCodec(data.columnName);
+
+    if (currentCodec === data.expectedCodecValue) {
+      logger.info(
+        `${tableName}.${data.columnName} already has ${data.expectedCodecValue}, skipping`,
+      );
+      return;
+    }
+
+    await this.execute(
+      `ALTER TABLE ${tableName} MODIFY COLUMN ${data.columnName} ${data.columnType} CODEC(${data.codec}) SETTINGS mutations_sync=0`,
+    );
+    logger.info(
+      `Applied ${data.codec} codec to ${tableName}.${data.columnName} (async)`,
+    );
+  }
+
   @CaptureSpan()
   public async findBy(findBy: FindBy<TBaseModel>): Promise<Array<TBaseModel>> {
     return await this._findBy(findBy);

package/Server/Services/LogAggregationService.ts

@@ -5,6 +5,7 @@ import { JSONObject } from "../../Types/JSON";
 import ObjectID from "../../Types/ObjectID";
 import BadDataException from "../../Types/Exception/BadDataException";
 import Includes from "../../Types/BaseDatabase/Includes";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
 
@@ -82,7 +83,7 @@ export interface AnalyticsTableRow {
 
 export class LogAggregationService {
   private static readonly DEFAULT_FACET_LIMIT: number = 500;
-  private static readonly TABLE_NAME: string = "LogItem";
+  private static readonly TABLE_NAME: string = AnalyticsTableName.Log;
   private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
     "severityText",
     "serviceId",
@@ -641,6 +642,242 @@ export class LogAggregationService {
     }
   }
 
+  @CaptureSpan()
+  public static async getExportLogs(request: {
+    projectId: ObjectID;
+    startTime: Date;
+    endTime: Date;
+    limit: number;
+    serviceIds?: Array<ObjectID> | undefined;
+    severityTexts?: Array<string> | undefined;
+    bodySearchText?: string | undefined;
+    traceIds?: Array<string> | undefined;
+    spanIds?: Array<string> | undefined;
+  }): Promise<Array<JSONObject>> {
+    const maxLimit: number = Math.min(request.limit || 10000, 10000);
+
+    const statement: Statement = SQL`
+      SELECT
+        time,
+        serviceId,
+        severityText,
+        severityNumber,
+        body,
+        traceId,
+        spanId,
+        attributes
+      FROM ${LogAggregationService.TABLE_NAME}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }}
+        AND time >= ${{
+          type: TableColumnType.Date,
+          value: request.startTime,
+        }}
+        AND time <= ${{
+          type: TableColumnType.Date,
+          value: request.endTime,
+        }}
+    `;
+
+    LogAggregationService.appendCommonFilters(statement, request);
+
+    statement.append(
+      SQL` ORDER BY time DESC LIMIT ${{
+        type: TableColumnType.Number,
+        value: maxLimit,
+      }}`,
+    );
+
+    const dbResult: Results = await LogDatabaseService.executeQuery(statement);
+    const response: DbJSONResponse = await dbResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    return response.data || [];
+  }
+
+  @CaptureSpan()
+  public static async getLogContext(request: {
+    projectId: ObjectID;
+    serviceId: ObjectID;
+    time: Date;
+    logId: string;
+    count: number;
+  }): Promise<{ before: Array<JSONObject>; after: Array<JSONObject> }> {
+    const count: number = Math.min(request.count || 5, 20);
+
+    const beforeStatement: Statement = SQL`
+      SELECT
+        _id,
+        time,
+        timeUnixNano,
+        serviceId,
+        severityText,
+        severityNumber,
+        body,
+        traceId,
+        spanId,
+        attributes
+      FROM ${LogAggregationService.TABLE_NAME}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }}
+        AND serviceId = ${{
+          type: TableColumnType.ObjectID,
+          value: request.serviceId,
+        }}
+        AND time <= ${{
+          type: TableColumnType.Date,
+          value: request.time,
+        }}
+        AND _id != ${{
+          type: TableColumnType.Text,
+          value: request.logId,
+        }}
+      ORDER BY time DESC, timeUnixNano DESC
+      LIMIT ${{
+        type: TableColumnType.Number,
+        value: count,
+      }}
+    `;
+
+    const afterStatement: Statement = SQL`
+      SELECT
+        _id,
+        time,
+        timeUnixNano,
+        serviceId,
+        severityText,
+        severityNumber,
+        body,
+        traceId,
+        spanId,
+        attributes
+      FROM ${LogAggregationService.TABLE_NAME}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }}
+        AND serviceId = ${{
+          type: TableColumnType.ObjectID,
+          value: request.serviceId,
+        }}
+        AND time >= ${{
+          type: TableColumnType.Date,
+          value: request.time,
+        }}
+        AND _id != ${{
+          type: TableColumnType.Text,
+          value: request.logId,
+        }}
+      ORDER BY time ASC, timeUnixNano ASC
+      LIMIT ${{
+        type: TableColumnType.Number,
+        value: count,
+      }}
+    `;
+
+    const [beforeResult, afterResult] = await Promise.all([
+      LogDatabaseService.executeQuery(beforeStatement),
+      LogDatabaseService.executeQuery(afterStatement),
+    ]);
+
+    const beforeResponse: DbJSONResponse = await beforeResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+    const afterResponse: DbJSONResponse = await afterResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    const beforeRows: Array<JSONObject> = (beforeResponse.data || []).reverse();
+    const afterRows: Array<JSONObject> = afterResponse.data || [];
+
+    return { before: beforeRows, after: afterRows };
+  }
+
+  @CaptureSpan()
+  public static async getDropFilterEstimate(request: {
+    projectId: ObjectID;
+    startTime: Date;
+    endTime: Date;
+    filterQuery: string;
+    serviceIds?: Array<ObjectID> | undefined;
+    severityTexts?: Array<string> | undefined;
+    bodySearchText?: string | undefined;
+  }): Promise<{
+    totalLogs: number;
+    matchingLogs: number;
+    estimatedReductionPercent: number;
+  }> {
+    // Get total count
+    const totalStatement: Statement = SQL`
+      SELECT count() AS cnt
+      FROM ${LogAggregationService.TABLE_NAME}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }}
+        AND time >= ${{
+          type: TableColumnType.Date,
+          value: request.startTime,
+        }}
+        AND time <= ${{
+          type: TableColumnType.Date,
+          value: request.endTime,
+        }}
+    `;
+
+    LogAggregationService.appendCommonFilters(totalStatement, request);
+
+    // Get matching count using the filter query as body search
+    const matchStatement: Statement = SQL`
+      SELECT count() AS cnt
+      FROM ${LogAggregationService.TABLE_NAME}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }}
+        AND time >= ${{
+          type: TableColumnType.Date,
+          value: request.startTime,
+        }}
+        AND time <= ${{
+          type: TableColumnType.Date,
+          value: request.endTime,
+        }}
+    `;
+
+    LogAggregationService.appendCommonFilters(matchStatement, {
+      ...request,
+      bodySearchText: request.filterQuery,
+    });
+
+    const [totalResult, matchResult] = await Promise.all([
+      LogDatabaseService.executeQuery(totalStatement),
+      LogDatabaseService.executeQuery(matchStatement),
+    ]);
+
+    const totalResponse: DbJSONResponse = await totalResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+    const matchResponse: DbJSONResponse = await matchResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    const totalData: Array<JSONObject> = totalResponse.data || [];
+    const matchData: Array<JSONObject> = matchResponse.data || [];
+
+    const totalLogs: number = Number(totalData[0]?.["cnt"] || 0);
+    const matchingLogs: number = Number(matchData[0]?.["cnt"] || 0);
+    const estimatedReductionPercent: number =
+      totalLogs > 0 ? Math.round((matchingLogs / totalLogs) * 100) : 0;
+
+    return { totalLogs, matchingLogs, estimatedReductionPercent };
+  }
+
   private static isTopLevelColumn(key: string): boolean {
     return LogAggregationService.TOP_LEVEL_COLUMNS.has(key);
   }

package/Server/Services/LogDropFilterService.ts

@@ -0,0 +1,10 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/LogDropFilter";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+
+export default new Service();

package/Server/Services/LogPipelineProcessorService.ts

@@ -0,0 +1,10 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/LogPipelineProcessor";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+
+export default new Service();

package/Server/Services/LogPipelineService.ts

@@ -0,0 +1,10 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/LogPipeline";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+
+export default new Service();

package/Server/Services/LogScrubRuleService.ts

@@ -0,0 +1,10 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/LogScrubRule";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+
+export default new Service();

package/Server/Services/TelemetryAttributeService.ts

@@ -227,8 +227,8 @@ export class TelemetryAttributeService {
       WITH filtered AS (
         SELECT arrayJoin(
             if(
-              ${data.attributeKeysColumn} IS NULL OR empty(${data.attributeKeysColumn}),
-              JSONExtractKeys(${data.attributesColumn}),
+              empty(${data.attributeKeysColumn}),
+              mapKeys(${data.attributesColumn}),
               ${data.attributeKeysColumn}
             )
           ) AS attribute
@@ -238,10 +238,8 @@ export class TelemetryAttributeService {
           value: data.projectId,
         }}
           AND (
-            ${data.attributeKeysColumn} IS NOT NULL OR (
-              ${data.attributesColumn} IS NOT NULL AND
-              ${data.attributesColumn} != ''
-            )
+            NOT empty(${data.attributeKeysColumn}) OR
+            NOT empty(${data.attributesColumn})
           )
           AND ${data.timeColumn} >= ${{
             type: TableColumnType.Date,

package/Server/Utils/AnalyticsDatabase/Statement.ts

@@ -120,7 +120,11 @@ export class Statement implements BaseQueryParams {
         finalValue = v.value.values;
       }
     } else if (v.value instanceof Date) {
-      finalValue = OneUptimeDate.toClickhouseDateTime(v.value);
+      if (typeof v !== "string" && v.type === TableColumnType.DateTime64) {
+        finalValue = OneUptimeDate.toClickhouseDateTime64(v.value);
+      } else {
+        finalValue = OneUptimeDate.toClickhouseDateTime(v.value);
+      }
     } else {
       finalValue = v.value;
     }
@@ -136,6 +140,15 @@ export class Statement implements BaseQueryParams {
       finalValue = OneUptimeDate.toClickhouseDateTime(finalValue);
     }
 
+    if (
+      typeof v !== "string" &&
+      v.type === TableColumnType.DateTime64 &&
+      !(v.value instanceof Date)
+    ) {
+      finalValue = OneUptimeDate.fromString(finalValue as string);
+      finalValue = OneUptimeDate.toClickhouseDateTime64(finalValue);
+    }
+
     return finalValue;
   }
 
@@ -176,6 +189,7 @@ export class Statement implements BaseQueryParams {
       [TableColumnType.Number]: "Int32",
       [TableColumnType.Decimal]: "Double",
       [TableColumnType.Date]: "DateTime",
+      [TableColumnType.DateTime64]: "DateTime64(9)",
       [TableColumnType.JSON]: "JSON",
       [TableColumnType.ArrayNumber]: "Array(Int32)",
       [TableColumnType.ArrayText]: "Array(String)",