@oneuptime/common 11.3.9 → 11.3.11

package/Server/Services/IncidentStateTimelineService.ts

@@ -1,7 +1,8 @@
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
-import { OnCreate, OnDelete } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
+import UpdateBy from "../Types/Database/UpdateBy";
 import DatabaseService from "./DatabaseService";
 import IncidentPublicNoteService from "./IncidentPublicNoteService";
 import IncidentService from "./IncidentService";
@@ -643,6 +644,113 @@ ${createdItem.rootCause}`,
     return createdItem;
   }
 
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<IncidentStateTimeline>,
+  ): Promise<OnUpdate<IncidentStateTimeline>> {
+    const incidentIds: Array<ObjectID> =
+      await this.getIncidentIdsForTimelineQuery(updateBy);
+
+    return {
+      updateBy,
+      carryForward: incidentIds,
+    };
+  }
+
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<IncidentStateTimeline>,
+    updatedItemIds: ObjectID[],
+  ): Promise<OnUpdate<IncidentStateTimeline>> {
+    const incidentIds: Array<ObjectID> = [
+      ...(Array.isArray(onUpdate.carryForward)
+        ? (onUpdate.carryForward as Array<ObjectID>)
+        : []),
+    ];
+
+    if (updatedItemIds.length > 0) {
+      const updatedTimelines: Array<IncidentStateTimeline> = await this.findBy({
+        query: {
+          _id: QueryHelper.any(updatedItemIds),
+        },
+        select: {
+          incidentId: true,
+        },
+        skip: 0,
+        limit: LIMIT_PER_PROJECT,
+        props: {
+          isRoot: true,
+          ignoreHooks: true,
+        },
+      });
+
+      incidentIds.push(...this.getIncidentIdsFromTimelines(updatedTimelines));
+    }
+
+    this.refreshIncidentMetricsForIncidentIds(incidentIds);
+
+    return onUpdate;
+  }
+
+  private async getIncidentIdsForTimelineQuery(
+    updateBy: UpdateBy<IncidentStateTimeline>,
+  ): Promise<Array<ObjectID>> {
+    const timelines: Array<IncidentStateTimeline> = await this.findBy({
+      query: updateBy.query,
+      select: {
+        incidentId: true,
+      },
+      skip: updateBy.skip,
+      limit: updateBy.limit,
+      props: {
+        isRoot: true,
+        ignoreHooks: true,
+      },
+    });
+
+    return this.getIncidentIdsFromTimelines(timelines);
+  }
+
+  private getIncidentIdsFromTimelines(
+    timelines: Array<IncidentStateTimeline>,
+  ): Array<ObjectID> {
+    const incidentIds: Map<string, ObjectID> = new Map<string, ObjectID>();
+
+    for (const timeline of timelines) {
+      if (timeline.incidentId) {
+        incidentIds.set(timeline.incidentId.toString(), timeline.incidentId);
+      }
+    }
+
+    return Array.from(incidentIds.values());
+  }
+
+  private refreshIncidentMetricsForIncidentIds(
+    incidentIds: Array<ObjectID>,
+  ): void {
+    const uniqueIncidentIds: Map<string, ObjectID> = new Map<
+      string,
+      ObjectID
+    >();
+
+    for (const incidentId of incidentIds) {
+      uniqueIncidentIds.set(incidentId.toString(), incidentId);
+    }
+
+    for (const incidentId of uniqueIncidentIds.values()) {
+      IncidentService.refreshIncidentMetrics({
+        incidentId: incidentId,
+      }).catch((error: Error) => {
+        logger.error(`Error while refreshing incident metrics:`, {
+          incidentId: incidentId.toString(),
+        } as LogAttributes);
+        logger.error(error, {
+          incidentId: incidentId.toString(),
+        } as LogAttributes);
+      });
+    }
+  }
+
   private async isLastIncidentState(data: {
     projectId: ObjectID;
     incidentStateId: ObjectID;
@@ -1027,6 +1135,17 @@ ${createdItem.rootCause}`,
           },
         });
       }
+
+      IncidentService.refreshIncidentMetrics({
+        incidentId: incidentId,
+      }).catch((error: Error) => {
+        logger.error(`Error while refreshing incident metrics:`, {
+          incidentId: incidentId?.toString(),
+        } as LogAttributes);
+        logger.error(error, {
+          incidentId: incidentId?.toString(),
+        } as LogAttributes);
+      });
     }
 
     return onDelete;

package/Server/Services/Index.ts

@@ -61,6 +61,7 @@ import MailService from "./MailService";
 import MetricService from "./MetricService";
 import MetricItemAggMV1mService from "./MetricItemAggMV1mService";
 import MetricItemAggMV1mByHostV2Service from "./MetricItemAggMV1mByHostV2Service";
+import MutableMetricService from "./MutableMetricService";
 import MetricBaselineService from "./MetricBaselineService";
 import MonitorCustomFieldService from "./MonitorCustomFieldService";
 import MonitorGroupOwnerTeamService from "./MonitorGroupOwnerTeamService";
@@ -499,6 +500,7 @@ export const AnalyticsServices: Array<
   LogService,
   SpanService,
   MetricService,
+  MutableMetricService,
   /*
    * Materialized-view target tables. The auto-create flow runs
    * `CREATE TABLE IF NOT EXISTS` for these, idempotent with the legacy

package/Server/Services/MetricAggregationService.ts

@@ -1,6 +1,7 @@
 import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
 import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
 import MetricService from "./MetricService";
+import { MutableMetricService as MutableMetricServiceClass } from "./MutableMetricService";
 import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
 import { JSONObject } from "../../Types/JSON";
 import ObjectID from "../../Types/ObjectID";
@@ -40,6 +41,8 @@ export interface FacetRequest extends MetricFilters {
 export class MetricAggregationService {
   private static readonly DEFAULT_FACET_LIMIT: number = 500;
   private static readonly TABLE_NAME: string = AnalyticsTableName.Metric;
+  private static readonly MUTABLE_TABLE_NAME: string =
+    AnalyticsTableName.MutableMetric;
   private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
     "primaryEntityId",
     "name",
@@ -102,26 +105,34 @@ export class MetricAggregationService {
     const isTopLevelColumn: boolean =
       isResourceFacet ||
       MetricAggregationService.isTopLevelColumn(request.facetKey);
+    const useMutableMetricTable: boolean =
+      MetricAggregationService.shouldUseMutableMetricTable(request);
 
     const statement: Statement = new Statement();
 
     if (isResourceFacet) {
       statement.append(
-        SQL`SELECT toString(primaryEntityId) AS val, count() AS cnt FROM ${MetricAggregationService.TABLE_NAME}`,
+        SQL`SELECT toString(primaryEntityId) AS val, count() AS cnt`,
       );
     } else if (isTopLevelColumn) {
       statement.append(
-        SQL`SELECT toString(${request.facetKey}) AS val, count() AS cnt FROM ${MetricAggregationService.TABLE_NAME}`,
+        SQL`SELECT toString(${request.facetKey}) AS val, count() AS cnt`,
       );
     } else {
       statement.append(
         SQL`SELECT attributes[${{
           type: TableColumnType.Text,
           value: request.facetKey,
-        }}] AS val, count() AS cnt FROM ${MetricAggregationService.TABLE_NAME}`,
+        }}] AS val, count() AS cnt`,
       );
     }
 
+    MetricAggregationService.appendFacetSourceTable(
+      statement,
+      request,
+      useMutableMetricTable,
+    );
+
     statement.append(
       SQL` WHERE projectId = ${{
         type: TableColumnType.ObjectID,
@@ -135,6 +146,10 @@ export class MetricAggregationService {
       }}`,
     );
 
+    if (useMutableMetricTable) {
+      statement.append(SQL` AND isDeleted = false`);
+    }
+
     if (isResourceFacet) {
       statement.append(
         SQL` AND primaryEntityType = ${{
@@ -187,6 +202,61 @@ export class MetricAggregationService {
     return statement;
   }
 
+  private static shouldUseMutableMetricTable(request: FacetRequest): boolean {
+    if (!request.metricNames || request.metricNames.length === 0) {
+      return false;
+    }
+
+    return request.metricNames.every((metricName: string): boolean => {
+      return MutableMetricServiceClass.isMutableMetricName(metricName);
+    });
+  }
+
+  private static appendFacetSourceTable(
+    statement: Statement,
+    request: FacetRequest,
+    useMutableMetricTable: boolean,
+  ): void {
+    if (!useMutableMetricTable) {
+      statement.append(SQL` FROM ${MetricAggregationService.TABLE_NAME}`);
+      return;
+    }
+
+    statement.append(
+      SQL` FROM (
+        SELECT
+          projectId,
+          name,
+          primaryEntityId,
+          primaryEntityType,
+          metricPointId,
+          argMax(time, version) AS time,
+          argMax(attributes, version) AS attributes,
+          argMax(retentionDate, version) AS retentionDate,
+          argMax(isDeleted, version) AS isDeleted
+        FROM ${MetricAggregationService.MUTABLE_TABLE_NAME}
+        WHERE projectId = ${{
+          type: TableColumnType.ObjectID,
+          value: request.projectId,
+        }}`,
+    );
+
+    if (request.metricNames && request.metricNames.length > 0) {
+      statement.append(
+        SQL` AND name IN (${{
+          type: TableColumnType.Text,
+          value: new Includes(request.metricNames),
+        }})`,
+      );
+    }
+
+    statement.append(
+      SQL`
+        GROUP BY projectId, name, primaryEntityId, primaryEntityType, metricPointId
+      )`,
+    );
+  }
+
   private static appendCommonFilters(
     statement: Statement,
     request: MetricFilters,

package/Server/Services/MetricService.ts

@@ -1,5 +1,6 @@
 import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
 import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import { MutableMetricService as MutableMetricServiceClass } from "./MutableMetricService";
 import Metric from "../../Models/AnalyticsModels/Metric";
 import AggregateBy, {
   AggregateUtil,
@@ -172,6 +173,15 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
     statement: Statement;
     columns: Array<string>;
   } {
+    const mutableMetricStatement: {
+      statement: Statement;
+      columns: Array<string>;
+    } | null = this.tryBuildMutableMetricAggregateStatement(aggregateBy);
+
+    if (mutableMetricStatement) {
+      return mutableMetricStatement;
+    }
+
     if (!isPercentileAggregation(aggregateBy.aggregationType)) {
       /*
        * Try the per-host MV first — host detail pages are the
@@ -387,6 +397,224 @@ export class MetricService extends AnalyticsDatabaseService<Metric> {
     return { statement, columns };
   }
 
+  private tryBuildMutableMetricAggregateStatement(
+    aggregateBy: AggregateBy<Metric>,
+  ): { statement: Statement; columns: Array<string> } | null {
+    const metricName: string | null = this.getExactMetricNameFromQuery(
+      aggregateBy.query,
+    );
+
+    if (
+      !MutableMetricServiceClass.isMutableMetricName(metricName || undefined)
+    ) {
+      return null;
+    }
+
+    if (!this.canRouteAggregateToMutableMetricTable(aggregateBy)) {
+      return null;
+    }
+
+    if (!this.database) {
+      this.useDefaultDatabase();
+    }
+
+    const databaseName: string = this.database.getDatasourceOptions().database!;
+
+    const select: { statement: Statement; columns: Array<string> } =
+      this.statementGenerator.toAggregateSelectStatement(aggregateBy);
+
+    const innerWhereStatement: Statement =
+      this.statementGenerator.toWhereStatement(
+        this.getMutableMetricInnerQuery(aggregateBy.query),
+      );
+    const outerWhereStatement: Statement =
+      this.statementGenerator.toWhereStatement(aggregateBy.query);
+    const sortStatement: Statement = this.statementGenerator.toSortStatement(
+      aggregateBy.sort!,
+    );
+
+    const statement: Statement = SQL``;
+
+    statement
+      .append(SQL`SELECT `)
+      .append(select.statement)
+      .append(
+        SQL`
+          FROM (
+            SELECT
+              projectId,
+              name,
+              primaryEntityId,
+              primaryEntityType,
+              metricPointId,
+              argMax(metricPointType, version) AS metricPointType,
+              argMax(time, version) AS time,
+              argMax(timeUnixNano, version) AS timeUnixNano,
+              argMax(attributes, version) AS attributes,
+              argMax(attributeKeys, version) AS attributeKeys,
+              argMax(value, version) AS value,
+              argMax(retentionDate, version) AS retentionDate,
+              argMax(isDeleted, version) AS isDeleted
+            FROM ${databaseName}.${AnalyticsTableName.MutableMetric}
+            WHERE TRUE
+        `,
+      )
+      .append(innerWhereStatement)
+      .append(
+        SQL`
+            GROUP BY
+              projectId,
+              name,
+              primaryEntityId,
+              primaryEntityType,
+              metricPointId
+          )
+          WHERE isDeleted = false
+        `,
+      )
+      .append(outerWhereStatement)
+      .append(SQL` AND retentionDate >= now()`);
+
+    statement
+      .append(SQL` GROUP BY `)
+      .append(`${aggregateBy.aggregationTimestampColumnName.toString()}`);
+
+    if (aggregateBy.groupBy && Object.keys(aggregateBy.groupBy).length > 0) {
+      statement
+        .append(SQL` , `)
+        .append(
+          this.statementGenerator.toGroupByStatement(aggregateBy.groupBy),
+        );
+    }
+
+    statement.append(SQL` ORDER BY `).append(sortStatement);
+
+    statement.append(
+      SQL` LIMIT ${{
+        value: Number(aggregateBy.limit),
+        type: TableColumnType.Number,
+      }}`,
+    );
+
+    statement.append(
+      SQL` OFFSET ${{
+        value: Number(aggregateBy.skip),
+        type: TableColumnType.Number,
+      }}`,
+    );
+
+    statement.append(
+      getQuerySettings({
+        additionalSettings: {
+          optimize_move_to_prewhere: 1,
+          max_threads: 4,
+        },
+      }),
+    );
+
+    logger.debug("Mutable metric aggregate statement", {
+      metricName: metricName || "",
+    } as LogAttributes);
+    logger.debug(statement, {
+      metricName: metricName || "",
+    } as LogAttributes);
+
+    return {
+      statement,
+      columns: select.columns,
+    };
+  }
+
+  private getMutableMetricInnerQuery(query: Query<Metric>): Query<Metric> {
+    const queryRecord: Record<string, unknown> = query as unknown as Record<
+      string,
+      unknown
+    >;
+    const allowedInnerKeys: Array<string> = [
+      "projectId",
+      "name",
+      "primaryEntityId",
+      "primaryEntityType",
+    ];
+    const innerQuery: Record<string, unknown> = {};
+
+    for (const key of allowedInnerKeys) {
+      if (queryRecord[key] !== undefined) {
+        innerQuery[key] = queryRecord[key];
+      }
+    }
+
+    return innerQuery as unknown as Query<Metric>;
+  }
+
+  private getExactMetricNameFromQuery(query: Query<Metric>): string | null {
+    const queryRecord: Record<string, unknown> = query as unknown as Record<
+      string,
+      unknown
+    >;
+    const metricName: unknown = queryRecord["name"];
+
+    if (typeof metricName !== "string") {
+      return null;
+    }
+
+    return metricName;
+  }
+
+  private canRouteAggregateToMutableMetricTable(
+    aggregateBy: AggregateBy<Metric>,
+  ): boolean {
+    const supportedColumns: Set<string> = new Set<string>([
+      "projectId",
+      "name",
+      "primaryEntityId",
+      "primaryEntityType",
+      "metricPointType",
+      "time",
+      "timeUnixNano",
+      "attributes",
+      "attributeKeys",
+      "value",
+      "retentionDate",
+    ]);
+
+    const queryRecord: Record<string, unknown> =
+      aggregateBy.query as unknown as Record<string, unknown>;
+
+    for (const key of Object.keys(queryRecord)) {
+      if (!supportedColumns.has(key)) {
+        return false;
+      }
+    }
+
+    if (
+      !supportedColumns.has(aggregateBy.aggregateColumnName.toString()) ||
+      !supportedColumns.has(
+        aggregateBy.aggregationTimestampColumnName.toString(),
+      )
+    ) {
+      return false;
+    }
+
+    if (aggregateBy.groupBy) {
+      for (const key of Object.keys(aggregateBy.groupBy)) {
+        if (!supportedColumns.has(key)) {
+          return false;
+        }
+      }
+    }
+
+    if (aggregateBy.sort) {
+      for (const key of Object.keys(aggregateBy.sort)) {
+        if (!supportedColumns.has(key)) {
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
   /*
    * Materialized-view fast path for scalar aggregations.
    *